Merge pull request #107514 from Profpatsch/tree-sitter-fetch-all-grammars

Tree sitter fetch all grammars

24 files changed, 354 insertions, 76 deletions

diff --git a/pkgs/development/tools/parsing/tree-sitter/default.nix b/pkgs/development/tools/parsing/tree-sitter/default.nix
index 57c12cce93e..72150b5ac0b 100644
--- a/pkgs/development/tools/parsing/tree-sitter/default.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/default.nix
@@ -1,6 +1,6 @@
 { lib, stdenv
 , fetchgit, fetchFromGitHub, fetchurl
-, writeShellScript, runCommand, which
+, writeShellScript, runCommand, which, formats
 , rustPlatform, jq, nix-prefetch-git, xe, curl, emscripten
 , Security
 , callPackage
@@ -29,7 +29,7 @@ let
   };
 
   update-all-grammars = import ./update.nix {
-    inherit writeShellScript nix-prefetch-git curl jq xe src;
+    inherit writeShellScript nix-prefetch-git curl jq xe src formats lib;
   };
 
   fetchGrammar = (v: fetchgit {inherit (v) url rev sha256 fetchSubmodules; });
@@ -43,14 +43,22 @@ let
 
   builtGrammars = let
     change = name: grammar:
-      callPackage ./library.nix {} {
+      callPackage ./grammar.nix {} {
         language = name;
         inherit version;
         source = fetchGrammar grammar;
       };
   in
-    # typescript doesn't have parser.c in the same place as others
-    lib.mapAttrs change (removeAttrs (import ./grammars) ["typescript"]);
+    lib.mapAttrs change (removeAttrs (import ./grammars) [
+      # TODO these don't have parser.c in the same place as others.
+      # They might require more elaborate builds?
+      #  /nix/…/src/parser.c: No such file or directory
+      "tree-sitter-typescript"
+      #  /nix/…/src/parser.c: No such file or directory
+      "tree-sitter-ocaml"
+      # /nix/…/src/parser.c:1:10: fatal error: tree_sitter/parser.h: No such file or directory
+      "tree-sitter-razor"
+    ]);
 
 in rustPlatform.buildRustPackage {
   pname = "tree-sitter";
@@ -93,6 +101,11 @@ in rustPlatform.buildRustPackage {
     };
     inherit grammars;
     inherit builtGrammars;
+
+    tests = {
+      # make sure all grammars build
+      builtGrammars = lib.recurseIntoAttrs builtGrammars;
+    };
   };
 
   meta = {
diff --git a/pkgs/development/tools/parsing/tree-sitter/library.nix b/pkgs/development/tools/parsing/tree-sitter/grammar.nix
index f78939b22ff..f92e0d79426 100644
--- a/pkgs/development/tools/parsing/tree-sitter/library.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/grammar.nix
@@ -15,7 +15,7 @@
 
 stdenv.mkDerivation {
 
-  pname = "tree-sitter-${language}-library";
+  pname = "${language}-grammar";
   inherit version;
 
   src = source;
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix b/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix
index e9cb0a1db6e..e1648b84be5 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix
@@ -1,18 +1,31 @@
 {
-  bash = (builtins.fromJSON (builtins.readFile ./tree-sitter-bash.json));
-  c = (builtins.fromJSON (builtins.readFile ./tree-sitter-c.json));
-  cpp = (builtins.fromJSON (builtins.readFile ./tree-sitter-cpp.json));
-  embedded-template = (builtins.fromJSON (builtins.readFile ./tree-sitter-embedded-template.json));
-  go = (builtins.fromJSON (builtins.readFile ./tree-sitter-go.json));
-  html = (builtins.fromJSON (builtins.readFile ./tree-sitter-html.json));
-  java = (builtins.fromJSON (builtins.readFile ./tree-sitter-java.json));
-  javascript = (builtins.fromJSON (builtins.readFile ./tree-sitter-javascript.json));
-  jsdoc = (builtins.fromJSON (builtins.readFile ./tree-sitter-jsdoc.json));
-  json = (builtins.fromJSON (builtins.readFile ./tree-sitter-json.json));
-  lua = (builtins.fromJSON (builtins.readFile ./tree-sitter-lua.json));
-  php = (builtins.fromJSON (builtins.readFile ./tree-sitter-php.json));
-  python = (builtins.fromJSON (builtins.readFile ./tree-sitter-python.json));
-  ruby = (builtins.fromJSON (builtins.readFile ./tree-sitter-ruby.json));
-  rust = (builtins.fromJSON (builtins.readFile ./tree-sitter-rust.json));
-  typescript = (builtins.fromJSON (builtins.readFile ./tree-sitter-typescript.json));
+  tree-sitter-agda = (builtins.fromJSON (builtins.readFile ./tree-sitter-agda.json));
+  tree-sitter-bash = (builtins.fromJSON (builtins.readFile ./tree-sitter-bash.json));
+  tree-sitter-c = (builtins.fromJSON (builtins.readFile ./tree-sitter-c.json));
+  tree-sitter-c-sharp = (builtins.fromJSON (builtins.readFile ./tree-sitter-c-sharp.json));
+  tree-sitter-cpp = (builtins.fromJSON (builtins.readFile ./tree-sitter-cpp.json));
+  tree-sitter-css = (builtins.fromJSON (builtins.readFile ./tree-sitter-css.json));
+  tree-sitter-embedded-template = (builtins.fromJSON (builtins.readFile ./tree-sitter-embedded-template.json));
+  tree-sitter-fluent = (builtins.fromJSON (builtins.readFile ./tree-sitter-fluent.json));
+  tree-sitter-go = (builtins.fromJSON (builtins.readFile ./tree-sitter-go.json));
+  tree-sitter-haskell = (builtins.fromJSON (builtins.readFile ./tree-sitter-haskell.json));
+  tree-sitter-html = (builtins.fromJSON (builtins.readFile ./tree-sitter-html.json));
+  tree-sitter-java = (builtins.fromJSON (builtins.readFile ./tree-sitter-java.json));
+  tree-sitter-javascript = (builtins.fromJSON (builtins.readFile ./tree-sitter-javascript.json));
+  tree-sitter-jsdoc = (builtins.fromJSON (builtins.readFile ./tree-sitter-jsdoc.json));
+  tree-sitter-json = (builtins.fromJSON (builtins.readFile ./tree-sitter-json.json));
+  tree-sitter-julia = (builtins.fromJSON (builtins.readFile ./tree-sitter-julia.json));
+  tree-sitter-lua = (builtins.fromJSON (builtins.readFile ./tree-sitter-lua.json));
+  tree-sitter-ocaml = (builtins.fromJSON (builtins.readFile ./tree-sitter-ocaml.json));
+  tree-sitter-php = (builtins.fromJSON (builtins.readFile ./tree-sitter-php.json));
+  tree-sitter-python = (builtins.fromJSON (builtins.readFile ./tree-sitter-python.json));
+  tree-sitter-ql = (builtins.fromJSON (builtins.readFile ./tree-sitter-ql.json));
+  tree-sitter-razor = (builtins.fromJSON (builtins.readFile ./tree-sitter-razor.json));
+  tree-sitter-regex = (builtins.fromJSON (builtins.readFile ./tree-sitter-regex.json));
+  tree-sitter-ruby = (builtins.fromJSON (builtins.readFile ./tree-sitter-ruby.json));
+  tree-sitter-rust = (builtins.fromJSON (builtins.readFile ./tree-sitter-rust.json));
+  tree-sitter-scala = (builtins.fromJSON (builtins.readFile ./tree-sitter-scala.json));
+  tree-sitter-swift = (builtins.fromJSON (builtins.readFile ./tree-sitter-swift.json));
+  tree-sitter-typescript = (builtins.fromJSON (builtins.readFile ./tree-sitter-typescript.json));
+  tree-sitter-verilog = (builtins.fromJSON (builtins.readFile ./tree-sitter-verilog.json));
 }
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-agda.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-agda.json
new file mode 100644
index 00000000000..f16877e9465
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-agda.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-agda",
+  "rev": "d710ff14d15ddee3764fd73a0837c6c4c8c913e9",
+  "date": "2019-09-20T18:06:06+08:00",
+  "path": "/nix/store/wqz9v9znaiwhhqi19hgig9bn0yvl4i9s-tree-sitter-agda",
+  "sha256": "1wpfj47l97pxk3i9rzdylqipy849r482fnj3lmx8byhalv7z1vm6",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-c-sharp.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-c-sharp.json
new file mode 100644
index 00000000000..6498e40497e
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-c-sharp.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-c-sharp",
+  "rev": "b8bff2a389ecc098dbb7e8abee492816a5eb42db",
+  "date": "2020-11-15T07:54:17+00:00",
+  "path": "/nix/store/h5p71g661hbyzcdcj6xff8y5pcsivpa4-tree-sitter-c-sharp",
+  "sha256": "0x78s2wgd8b6pwjzbmc9fgp0ivdmxv39wikig1m55slai6yq51wh",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json
index 16aef8ee33e..a4538b1a932 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json
@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-cpp",
-  "rev": "fb8250eef8b4cf1ce104806c50dc206e388b0e72",
-  "date": "2020-11-02T09:46:58-08:00",
-  "path": "/nix/store/qmza43f0fraa111bg50vaxsgnrs5kqs1-tree-sitter-cpp",
-  "sha256": "1sbk5a2p1fh6798naiswsap6fpj7n7bqhwd3xdyi35xf613qpwsi",
+  "rev": "a35a275df92e7583df38f2de2562361f2b69987e",
+  "date": "2020-12-13T11:27:21-08:00",
+  "path": "/nix/store/l0mv4q1xdxz94ym1nl73y52i1yr9zcgi-tree-sitter-cpp",
+  "sha256": "130vizybkm11j3lpzmf183myz0vjxq75mpy6qz48rrkidhnrlryk",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-css.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-css.json
new file mode 100644
index 00000000000..b8609c0bd17
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-css.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-css",
+  "rev": "23f2cb97d47860c517f67f03e1f4b621d5bd2085",
+  "date": "2020-05-14T14:44:30-07:00",
+  "path": "/nix/store/r5pkz9kly0mhgrmqzdzdsr6d1dpqavld-tree-sitter-css",
+  "sha256": "17svpf36p0p7spppzhm3fi833zpdl2l1scg34r6d4vcbv7dknrjy",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-fluent.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-fluent.json
new file mode 100644
index 00000000000..f40b8465b7d
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-fluent.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-fluent",
+  "rev": "858fdd6f1e81992e00d3541bfb31bac9365d7a47",
+  "date": "2018-06-18T13:00:38-07:00",
+  "path": "/nix/store/zbj8abdlrqi9swm8qn8rhpqmjwcz145f-tree-sitter-fluent",
+  "sha256": "0528v9w0cs73p9048xrddb1wpdhr92sn1sw8yyqfrq5sq0danr9k",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-haskell.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-haskell.json
new file mode 100644
index 00000000000..10d2dd64ac2
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-haskell.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-haskell",
+  "rev": "2a0aa1cb5f1b787a4056a29fa0791e87846e33fb",
+  "date": "2018-11-03T09:56:20-07:00",
+  "path": "/nix/store/9xszs4xi51qr72laxr67zxnh8y2br0gy-tree-sitter-haskell",
+  "sha256": "0z0nfip5m0yrjgm49j5nld7jkgyjdmps101xhbng39pwfnh1av83",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json
index 68008d393d1..2c563f3fbfa 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json
@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-javascript",
-  "rev": "852f11b394804ac2a8986f8bcaafe77753635667",
-  "date": "2020-10-27T13:43:40-04:00",
-  "path": "/nix/store/7a9nd3prxbv2izvilqdxf58his097ak7-tree-sitter-javascript",
-  "sha256": "1cdqx75fm7fvna0iymw7n03a7f6gayfg97qwqz3himi0775fz9ir",
+  "rev": "3f8b62f9befd3cb3b4cb0de22f6595a0aadf76ca",
+  "date": "2020-12-02T10:20:20-08:00",
+  "path": "/nix/store/c17bf7sjq95lank5ygbglv8j48i5z9w3-tree-sitter-javascript",
+  "sha256": "0fjq1jzrzd8c8rfxkh2s25gnqlyc19k3a8i3r1129kakisn1288k",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-julia.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-julia.json
new file mode 100644
index 00000000000..2cf5f06dc19
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-julia.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-julia",
+  "rev": "6a0863f1ce3fcf6f99dc0addb7886dcbd27c5a48",
+  "date": "2020-09-08T19:39:52-07:00",
+  "path": "/nix/store/xn5nii9mi2aw7xdabyxlglha2vk12h1w-tree-sitter-julia",
+  "sha256": "07ds4wzgvnkgkq07izdglkk8lgpqrylvrs96afnwxi56bnzs8sbv",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-lua.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-lua.json
index 267d25a52a0..804d613357e 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-lua.json
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-lua.json
@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/nvim-treesitter/tree-sitter-lua",
-  "rev": "278b0ea1f1f3d86afc86faaca0cbbf6c01182a54",
-  "date": "2020-09-08T18:29:23+02:00",
-  "path": "/nix/store/xkaqsw030bf6zd6pivrzrg3d50j1ka1k-tree-sitter-lua",
-  "sha256": "0i8jdp0bbl02h58hfhpild4v3rvy8yk5r5l90navvfxw4ad4f6f3",
+  "rev": "97b757ad3546d68dc1131e6ffb9b1d2a750eea29",
+  "date": "2020-11-25T21:23:57+01:00",
+  "path": "/nix/store/irz0dikk9dahkayi41chcznqq1i3wr84-tree-sitter-lua",
+  "sha256": "1nhg139vk3xm6ip4mhr29z0rprfg4q417z2vlvkz3m9wp8gppzls",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ocaml.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ocaml.json
new file mode 100644
index 00000000000..32488ebb73e
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ocaml.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-ocaml",
+  "rev": "4ec9ee414dadc2b6e7325a9f8124d02f6cd8c250",
+  "date": "2020-09-18T02:13:40+02:00",
+  "path": "/nix/store/dszjdnwnbziqxav2khs85026msm6fasz-tree-sitter-ocaml",
+  "sha256": "0wy85940fhmrnz7c1gk6xkipm8ixzalq5q4i7mcc6wnjiiwq60gx",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json
index 402c4c087b1..86551acbee0 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json
@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-python",
-  "rev": "d245768132eb6cb74d8a394ca4d29dc57169b096",
-  "date": "2020-10-27T13:20:04-04:00",
-  "path": "/nix/store/afv7ibxpf5c2shafa89cnza88hsfb7kj-tree-sitter-python",
-  "sha256": "075r5i2id2rn76xm3pcrn5cpvj63dlaxcpvphig39a4c9f9hrpdx",
+  "rev": "65b486b42fe0188e4be344092151042bf1bc6d9a",
+  "date": "2020-12-15T09:33:20-08:00",
+  "path": "/nix/store/k9jziqzyxq2bv55pwl03jcmmca83fjyp-tree-sitter-python",
+  "sha256": "1yxqdlmp0jybm7vvza1ni5a320vrviqkd14pnpcrg9ilzq23mlsh",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ql.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ql.json
new file mode 100644
index 00000000000..27042b6ef95
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ql.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-ql",
+  "rev": "a0d688d62dcb9cbc7c53f0d98343c458b3776b3d",
+  "date": "2020-09-16T12:56:09-07:00",
+  "path": "/nix/store/dfdaf6wg80dfw5fvdiir7n9nj6j30g3g-tree-sitter-ql",
+  "sha256": "0f6rfhrbvpg8czfa7mld45by3rp628bs6fyl47a8mn18w6x0n5g2",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-razor.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-razor.json
new file mode 100644
index 00000000000..6905f4145aa
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-razor.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-razor",
+  "rev": "60edbd8e798e416f5226a746396efa6a8614fa9b",
+  "date": "2016-07-08T15:17:50-07:00",
+  "path": "/nix/store/2vkxw48wc0xf07awi65r4rg2m1lpqq6c-tree-sitter-razor",
+  "sha256": "1ldsn9nxldxkxdy7irywx88d6a56q05f6907lypzass07piyqp2i",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-regex.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-regex.json
new file mode 100644
index 00000000000..d04c1bdab86
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-regex.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-regex",
+  "rev": "be2e415b5716615530234d179dc27c32b7a1d86b",
+  "date": "2020-05-14T15:05:54-07:00",
+  "path": "/nix/store/acbcq9lgh5pf6hq3329fa253yfvaj6ji-tree-sitter-regex",
+  "sha256": "0qxbafbwfj64bwhj3455jgkidpdzqwrlsqm3rckq2pi75flnkv42",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json
index d5a1767a23c..027478e99f8 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json
@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-ruby",
-  "rev": "f86d90cce578e28c824afdcdbdca7d031b88c0b0",
-  "date": "2020-11-02T20:49:49-08:00",
-  "path": "/nix/store/gs0hhb1dlgmv0y60zyyz7ki30yvpddfx-tree-sitter-ruby",
-  "sha256": "1n1irdjfmv27p5ia2vqzv8zfsq19pxi0as0wrkb7z7s1lqr1gwrb",
+  "rev": "bb572f60e9538bd11fbde95a54f97522073f1e06",
+  "date": "2020-12-20T12:25:50-08:00",
+  "path": "/nix/store/hkjgvxh8l4drk1z4zkhmd0wcj876x5y1-tree-sitter-ruby",
+  "sha256": "198sfmrb3mxvpfmlwmwfmb3qs075ih0bjmr0ffrdy4nqm4a00cb8",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json
index 4e0e2e2f7f4..38c9e44f2f6 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json
@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-rust",
-  "rev": "cf47ff80cbd8bdc6dd649a005c5f5d73f9c347e2",
-  "date": "2020-11-04T06:23:40-08:00",
-  "path": "/nix/store/8cvjxw8cz5jrkqwvgq88r7rfm269xwk1-tree-sitter-rust",
-  "sha256": "0yzcag0yy3ncxwpnb3fmsw8bxaidp7z8kbl8wwaxaj2vvxxf6sam",
+  "rev": "8746bd4b584b8063ee8e445bf31015e887417d33",
+  "date": "2020-11-23T17:16:42-08:00",
+  "path": "/nix/store/1zzxvza23wsdyazw47lhjvrs0za6wjpn-tree-sitter-rust",
+  "sha256": "0bhxfyq8ycnp90pqvr6cf3gpq1vax8a34kaq85dmbrc6ar8a7ap6",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-scala.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-scala.json
new file mode 100644
index 00000000000..8d9959cf4cf
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-scala.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-scala",
+  "rev": "211bb726bb5857f872247b600c7c1808e641a8d4",
+  "date": "2020-07-13T13:31:00-07:00",
+  "path": "/nix/store/6q66gzabxazr2581dgp4pp5wwyk2p6mw-tree-sitter-scala",
+  "sha256": "096ps5za8gxmq61gdd3xdk8cif07vb4v8asls2kdwm6jazm82777",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-swift.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-swift.json
new file mode 100644
index 00000000000..8f73380e379
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-swift.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-swift",
+  "rev": "a22fa5e19bae50098e2252ea96cba3aba43f4c58",
+  "date": "2019-10-24T19:04:02-06:00",
+  "path": "/nix/store/pk5xk8yp6vanbar75bhfrs104w0k1ph0-tree-sitter-swift",
+  "sha256": "14b40lmwrnyvdz2wiv684kfh4fvqfhbj1dgrx81ppmy7hsz7jcq7",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json
index d31f72754bf..a875232eab6 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json
@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-typescript",
-  "rev": "73afadbd117a8e8551758af9c3a522ef46452119",
-  "date": "2020-10-20T12:15:52-07:00",
-  "path": "/nix/store/wrgp1j5l50cigv5cmlxikw693a55i6g9-tree-sitter-typescript",
-  "sha256": "0dsbmcvjzys9s229drp1l8anram7d6nsx1nixl5m9znw54hr4w9p",
+  "rev": "a3a4becef889692724bb1c2191f8fc4bce6573f9",
+  "date": "2020-12-15T13:34:55-08:00",
+  "path": "/nix/store/dra5wbhgxkkhphm1mhgv87hsfw0g18nw-tree-sitter-typescript",
+  "sha256": "0ir7lmzwnbf1zwbl4bfsib1jilg8pvfc1nabaq2n2rixvghlvf3d",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-verilog.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-verilog.json
new file mode 100644
index 00000000000..7ab79c6f2d5
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-verilog.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-verilog",
+  "rev": "ad551aae2649da56582bc0557478a7dc979c0be3",
+  "date": "2020-10-13T17:40:47-07:00",
+  "path": "/nix/store/nfaxfqrqkxpwaq8rnk7kcp28nnj8y6m2-tree-sitter-verilog",
+  "sha256": "0cy29i200rnc34d237s19r6a1n5vv4d3wgwpbywxg6ahcankc34m",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/update.nix b/pkgs/development/tools/parsing/tree-sitter/update.nix
index 2a3575a44d2..7b0b16b403f 100644
--- a/pkgs/development/tools/parsing/tree-sitter/update.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/update.nix
@@ -1,66 +1,188 @@
-{ writeShellScript, nix-prefetch-git
+{ writeShellScript, nix-prefetch-git, formats, lib
 , curl, jq, xe
 , src }:
 
 let
-  # print all the grammar names mentioned in the fetch-fixtures script
-  getGrammarNames = writeShellScript "get-grammars.sh" ''
+  # Grammars we want to fetch from the tree-sitter github orga
+  knownTreeSitterOrgGrammarRepos = [
+    "tree-sitter-javascript"
+    "tree-sitter-c"
+    "tree-sitter-swift"
+    "tree-sitter-json"
+    "tree-sitter-cpp"
+    "tree-sitter-ruby"
+    "tree-sitter-razor"
+    "tree-sitter-go"
+    "tree-sitter-c-sharp"
+    "tree-sitter-python"
+    "tree-sitter-typescript"
+    "tree-sitter-rust"
+    "tree-sitter-bash"
+    "tree-sitter-php"
+    "tree-sitter-java"
+    "tree-sitter-scala"
+    "tree-sitter-ocaml"
+    "tree-sitter-julia"
+    "tree-sitter-agda"
+    "tree-sitter-fluent"
+    "tree-sitter-html"
+    "tree-sitter-haskell"
+    "tree-sitter-regex"
+    "tree-sitter-css"
+    "tree-sitter-verilog"
+    "tree-sitter-jsdoc"
+    "tree-sitter-ql"
+    "tree-sitter-embedded-template"
+  ];
+  knownTreeSitterOrgGrammarReposJson = jsonFile "known-tree-sitter-org-grammar-repos" knownTreeSitterOrgGrammarRepos;
+
+  # repos of the tree-sitter github orga we want to ignore (not grammars)
+  ignoredTreeSitterOrgRepos = [
+    "tree-sitter"
+    "tree-sitter-cli"
+    # this is the haskell language bindings, tree-sitter-haskell is the grammar
+    "haskell-tree-sitter"
+    # this is the ruby language bindings, tree-sitter-ruby is the grammar
+    "ruby-tree-sitter"
+    # this is the (unmaintained) rust language bindings, tree-sitter-rust is the grammar
+    "rust-tree-sitter"
+    # this is the nodejs language bindings, tree-sitter-javascript is the grammar
+    "node-tree-sitter"
+    # this is the python language bindings, tree-sitter-python is the grammar
+    "py-tree-sitter"
+    # afl fuzzing for tree sitter
+    "afl-tree-sitter"
+    # archived
+    "highlight-schema"
+    # website
+    "tree-sitter.github.io"
+  ];
+  ignoredTreeSitterOrgReposJson = jsonFile "ignored-tree-sitter-org-repos" ignoredTreeSitterOrgRepos;
+
+  # Additional grammars that are not in the official github orga.
+  # If you need a grammar that already exists in the official orga,
+  # make sure to give it a different name.
+  otherGrammars = {
+    "tree-sitter-lua" = {
+      orga = "nvim-treesitter";
+      repo = "tree-sitter-lua";
+    };
+  };
+
+  allGrammars =
+    let
+      treeSitterOrgaGrammars =
+        lib.listToAttrs (map (repo:
+          { name = repo;
+            value = {
+              orga = "tree-sitter";
+              inherit repo;
+            };
+          })
+        knownTreeSitterOrgGrammarRepos);
+
+    in
+      mergeAttrsUnique otherGrammars treeSitterOrgaGrammars;
+
+  # TODO: move to lib
+  mergeAttrsUnique = left: right:
+    let intersect = lib.intersectLists (lib.attrNames left) (lib.attrNames right); in
+    assert
+      lib.assertMsg (intersect == [])
+        (lib.concatStringsSep "\n" [
+          "mergeAttrsUnique: keys in attrset overlapping:"
+          "left: ${lib.generators.toPretty {} (lib.getAttrs intersect left)}"
+          "right: ${lib.generators.toPretty {} (lib.getAttrs intersect right)}"
+        ]);
+    left // right;
+
+
+
+  jsonFile = name: val: (formats.json {}).generate name val;
+
+  # check the tree-sitter orga repos
+  checkTreeSitterRepos = writeShellScript "get-grammars.sh" ''
     set -euo pipefail
-    sed -ne 's/^fetch_grammar \(\S*\).*$/\1/p' \
-      ${src}/script/fetch-fixtures
+    res=$(${jq}/bin/jq \
+      --slurpfile known "${knownTreeSitterOrgGrammarReposJson}" \
+      --slurpfile ignore "${ignoredTreeSitterOrgReposJson}" \
+      '. - ($known[0] + $ignore[0])' \
+      )
+    if [ ! "$res" == "[]" ]; then
+      echo "These repositories are neither known nor ignored:" 1>&2
+      echo "$res" 1>&2
+      exit 1
+    fi
   '';
 
   # TODO
   urlEscape = x: x;
-  # TODO
-  urlEscapeSh = writeShellScript "escape-url" ''printf '%s' "$1"'';
 
   # generic bash script to find the latest github release for a repo
-  latestGithubRelease = { owner }: writeShellScript "latest-github-release" ''
+  latestGithubRelease = { orga, repo }: writeShellScript "latest-github-release" ''
     set -euo pipefail
-    repo="$1"
     res=$(${curl}/bin/curl \
       --silent \
-      "https://api.github.com/repos/${urlEscape owner}/$(${urlEscapeSh} "$repo")/releases/latest")
-    if [[ "$(printf "%s" "$res" | ${jq}/bin/jq '.message')" =~ "rate limit" ]]; then
+      "https://api.github.com/repos/${urlEscape orga}/${urlEscape repo}/releases/latest")
+    if [[ "$(printf "%s" "$res" | ${jq}/bin/jq '.message?')" =~ "rate limit" ]]; then
       echo "rate limited" >&2
     fi
     release=$(printf "%s" "$res" | ${jq}/bin/jq '.tag_name')
     # github sometimes returns an empty list even tough there are releases
     if [ "$release" = "null" ]; then
-      echo "uh-oh, latest for $repo is not there, using HEAD" >&2
+      echo "uh-oh, latest for ${orga + "/" + repo} is not there, using HEAD" >&2
       release="HEAD"
     fi
     echo "$release"
   '';
 
+  # find the latest repos of a github organization
+  latestGithubRepos = { orga }: writeShellScript "latest-github-repos" ''
+    set -euo pipefail
+    res=$(${curl}/bin/curl \
+      --silent \
+      'https://api.github.com/orgs/${urlEscape orga}/repos?per_page=100')
+
+    if [[ "$(printf "%s" "$res" | ${jq}/bin/jq '.message?')" =~ "rate limit" ]]; then
+      echo "rate limited" >&2   #
+    fi
+
+    printf "%s" "$res" | ${jq}/bin/jq 'map(.name)' \
+      || echo "failed $res"
+  '';
+
   # update one tree-sitter grammar repo and print their nix-prefetch-git output
-  updateGrammar = { owner }: writeShellScript "update-grammar.sh" ''
+  updateGrammar = { orga, repo }: writeShellScript "update-grammar.sh" ''
     set -euo pipefail
-    repo="$1"
-    latest="$(${latestGithubRelease { inherit owner; }} "$repo")"
-    echo "Fetching latest release ($latest) of $repo …" >&2
+    latest="$(${latestGithubRelease { inherit orga repo; }})"
+    echo "Fetching latest release ($latest) of ${repo} …" >&2
     ${nix-prefetch-git}/bin/nix-prefetch-git \
       --quiet \
       --no-deepClone \
-      --url "https://github.com/${urlEscape owner}/$(${urlEscapeSh} "$repo")" \
+      --url "https://github.com/${urlEscape orga}/${urlEscape repo}" \
       --rev "$latest"
     '';
 
+  foreachSh = attrs: f:
+    lib.concatMapStringsSep "\n" f
+    (lib.mapAttrsToList (k: v: { name = k; } // v) attrs);
+
   update-all-grammars = writeShellScript "update-all-grammars.sh" ''
     set -euo pipefail
-    grammarNames=$(${getGrammarNames})
+    echo "fetching list of grammars" 1>&2
+    treeSitterRepos=$(${latestGithubRepos { orga = "tree-sitter"; }})
+    echo "checking the tree-sitter repo list against the grammars we know" 1>&2
+    printf '%s' "$treeSitterRepos" | ${checkTreeSitterRepos}
     outputDir="${toString ./.}/grammars"
+    echo "writing files to $outputDir" 1>&2
     mkdir -p "$outputDir"
-    updateCommand=$(printf \
-      '${updateGrammar { owner = "tree-sitter"; }} "$1" > "%s/$1.json"' \
-      "$outputDir")
-    printf '%s' "$grammarNames" \
-      | ${xe}/bin/xe printf "tree-sitter-%s\n" {} \
-      | ${xe}/bin/xe -j2 -s "$updateCommand"
+    ${foreachSh allGrammars
+      ({name, orga, repo}: ''${updateGrammar { inherit orga repo; }} > $outputDir/${name}.json'')}
     ( echo "{"
-      printf '%s' "$grammarNames" \
-        | ${xe}/bin/xe -s 'printf "  %s = (builtins.fromJSON (builtins.readFile ./tree-sitter-%s.json));\n" "$1" "$1"'
+      ${foreachSh allGrammars
+        ({name, ...}: ''
+           # indentation hack
+             printf "  %s = (builtins.fromJSON (builtins.readFile ./%s.json));\n" "${name}" "${name}"'')}
       echo "}" ) \
       > "$outputDir/default.nix"
   '';