diff options
Diffstat (limited to 'pkgs/development/tools')
24 files changed, 354 insertions, 76 deletions
diff --git a/pkgs/development/tools/parsing/tree-sitter/default.nix b/pkgs/development/tools/parsing/tree-sitter/default.nix index 57c12cce93e..72150b5ac0b 100644 --- a/pkgs/development/tools/parsing/tree-sitter/default.nix +++ b/pkgs/development/tools/parsing/tree-sitter/default.nix @@ -1,6 +1,6 @@ { lib, stdenv , fetchgit, fetchFromGitHub, fetchurl -, writeShellScript, runCommand, which +, writeShellScript, runCommand, which, formats , rustPlatform, jq, nix-prefetch-git, xe, curl, emscripten , Security , callPackage @@ -29,7 +29,7 @@ let }; update-all-grammars = import ./update.nix { - inherit writeShellScript nix-prefetch-git curl jq xe src; + inherit writeShellScript nix-prefetch-git curl jq xe src formats lib; }; fetchGrammar = (v: fetchgit {inherit (v) url rev sha256 fetchSubmodules; }); @@ -43,14 +43,22 @@ let builtGrammars = let change = name: grammar: - callPackage ./library.nix {} { + callPackage ./grammar.nix {} { language = name; inherit version; source = fetchGrammar grammar; }; in - # typescript doesn't have parser.c in the same place as others - lib.mapAttrs change (removeAttrs (import ./grammars) ["typescript"]); + lib.mapAttrs change (removeAttrs (import ./grammars) [ + # TODO these don't have parser.c in the same place as others. + # They might require more elaborate builds? + # /nix/…/src/parser.c: No such file or directory + "tree-sitter-typescript" + # /nix/…/src/parser.c: No such file or directory + "tree-sitter-ocaml" + # /nix/…/src/parser.c:1:10: fatal error: tree_sitter/parser.h: No such file or directory + "tree-sitter-razor" + ]); in rustPlatform.buildRustPackage { pname = "tree-sitter"; @@ -93,6 +101,11 @@ in rustPlatform.buildRustPackage { }; inherit grammars; inherit builtGrammars; + + tests = { + # make sure all grammars build + builtGrammars = lib.recurseIntoAttrs builtGrammars; + }; }; meta = { diff --git a/pkgs/development/tools/parsing/tree-sitter/library.nix b/pkgs/development/tools/parsing/tree-sitter/grammar.nix index f78939b22ff..f92e0d79426 100644 --- a/pkgs/development/tools/parsing/tree-sitter/library.nix +++ b/pkgs/development/tools/parsing/tree-sitter/grammar.nix @@ -15,7 +15,7 @@ stdenv.mkDerivation { - pname = "tree-sitter-${language}-library"; + pname = "${language}-grammar"; inherit version; src = source; diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix b/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix index e9cb0a1db6e..e1648b84be5 100644 --- a/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix @@ -1,18 +1,31 @@ { - bash = (builtins.fromJSON (builtins.readFile ./tree-sitter-bash.json)); - c = (builtins.fromJSON (builtins.readFile ./tree-sitter-c.json)); - cpp = (builtins.fromJSON (builtins.readFile ./tree-sitter-cpp.json)); - embedded-template = (builtins.fromJSON (builtins.readFile ./tree-sitter-embedded-template.json)); - go = (builtins.fromJSON (builtins.readFile ./tree-sitter-go.json)); - html = (builtins.fromJSON (builtins.readFile ./tree-sitter-html.json)); - java = (builtins.fromJSON (builtins.readFile ./tree-sitter-java.json)); - javascript = (builtins.fromJSON (builtins.readFile ./tree-sitter-javascript.json)); - jsdoc = (builtins.fromJSON (builtins.readFile ./tree-sitter-jsdoc.json)); - json = (builtins.fromJSON (builtins.readFile ./tree-sitter-json.json)); - lua = (builtins.fromJSON (builtins.readFile ./tree-sitter-lua.json)); - php = (builtins.fromJSON (builtins.readFile ./tree-sitter-php.json)); - python = (builtins.fromJSON (builtins.readFile ./tree-sitter-python.json)); - ruby = (builtins.fromJSON (builtins.readFile ./tree-sitter-ruby.json)); - rust = (builtins.fromJSON (builtins.readFile ./tree-sitter-rust.json)); - typescript = (builtins.fromJSON (builtins.readFile ./tree-sitter-typescript.json)); + tree-sitter-agda = (builtins.fromJSON (builtins.readFile ./tree-sitter-agda.json)); + tree-sitter-bash = (builtins.fromJSON (builtins.readFile ./tree-sitter-bash.json)); + tree-sitter-c = (builtins.fromJSON (builtins.readFile ./tree-sitter-c.json)); + tree-sitter-c-sharp = (builtins.fromJSON (builtins.readFile ./tree-sitter-c-sharp.json)); + tree-sitter-cpp = (builtins.fromJSON (builtins.readFile ./tree-sitter-cpp.json)); + tree-sitter-css = (builtins.fromJSON (builtins.readFile ./tree-sitter-css.json)); + tree-sitter-embedded-template = (builtins.fromJSON (builtins.readFile ./tree-sitter-embedded-template.json)); + tree-sitter-fluent = (builtins.fromJSON (builtins.readFile ./tree-sitter-fluent.json)); + tree-sitter-go = (builtins.fromJSON (builtins.readFile ./tree-sitter-go.json)); + tree-sitter-haskell = (builtins.fromJSON (builtins.readFile ./tree-sitter-haskell.json)); + tree-sitter-html = (builtins.fromJSON (builtins.readFile ./tree-sitter-html.json)); + tree-sitter-java = (builtins.fromJSON (builtins.readFile ./tree-sitter-java.json)); + tree-sitter-javascript = (builtins.fromJSON (builtins.readFile ./tree-sitter-javascript.json)); + tree-sitter-jsdoc = (builtins.fromJSON (builtins.readFile ./tree-sitter-jsdoc.json)); + tree-sitter-json = (builtins.fromJSON (builtins.readFile ./tree-sitter-json.json)); + tree-sitter-julia = (builtins.fromJSON (builtins.readFile ./tree-sitter-julia.json)); + tree-sitter-lua = (builtins.fromJSON (builtins.readFile ./tree-sitter-lua.json)); + tree-sitter-ocaml = (builtins.fromJSON (builtins.readFile ./tree-sitter-ocaml.json)); + tree-sitter-php = (builtins.fromJSON (builtins.readFile ./tree-sitter-php.json)); + tree-sitter-python = (builtins.fromJSON (builtins.readFile ./tree-sitter-python.json)); + tree-sitter-ql = (builtins.fromJSON (builtins.readFile ./tree-sitter-ql.json)); + tree-sitter-razor = (builtins.fromJSON (builtins.readFile ./tree-sitter-razor.json)); + tree-sitter-regex = (builtins.fromJSON (builtins.readFile ./tree-sitter-regex.json)); + tree-sitter-ruby = (builtins.fromJSON (builtins.readFile ./tree-sitter-ruby.json)); + tree-sitter-rust = (builtins.fromJSON (builtins.readFile ./tree-sitter-rust.json)); + tree-sitter-scala = (builtins.fromJSON (builtins.readFile ./tree-sitter-scala.json)); + tree-sitter-swift = (builtins.fromJSON (builtins.readFile ./tree-sitter-swift.json)); + tree-sitter-typescript = (builtins.fromJSON (builtins.readFile ./tree-sitter-typescript.json)); + tree-sitter-verilog = (builtins.fromJSON (builtins.readFile ./tree-sitter-verilog.json)); } diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-agda.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-agda.json new file mode 100644 index 00000000000..f16877e9465 --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-agda.json @@ -0,0 +1,10 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-agda", + "rev": "d710ff14d15ddee3764fd73a0837c6c4c8c913e9", + "date": "2019-09-20T18:06:06+08:00", + "path": "/nix/store/wqz9v9znaiwhhqi19hgig9bn0yvl4i9s-tree-sitter-agda", + "sha256": "1wpfj47l97pxk3i9rzdylqipy849r482fnj3lmx8byhalv7z1vm6", + "fetchSubmodules": false, + "deepClone": false, + "leaveDotGit": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-c-sharp.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-c-sharp.json new file mode 100644 index 00000000000..6498e40497e --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-c-sharp.json @@ -0,0 +1,10 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-c-sharp", + "rev": "b8bff2a389ecc098dbb7e8abee492816a5eb42db", + "date": "2020-11-15T07:54:17+00:00", + "path": "/nix/store/h5p71g661hbyzcdcj6xff8y5pcsivpa4-tree-sitter-c-sharp", + "sha256": "0x78s2wgd8b6pwjzbmc9fgp0ivdmxv39wikig1m55slai6yq51wh", + "fetchSubmodules": false, + "deepClone": false, + "leaveDotGit": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json index 16aef8ee33e..a4538b1a932 100644 --- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json @@ -1,9 +1,9 @@ { "url": "https://github.com/tree-sitter/tree-sitter-cpp", - "rev": "fb8250eef8b4cf1ce104806c50dc206e388b0e72", - "date": "2020-11-02T09:46:58-08:00", - "path": "/nix/store/qmza43f0fraa111bg50vaxsgnrs5kqs1-tree-sitter-cpp", - "sha256": "1sbk5a2p1fh6798naiswsap6fpj7n7bqhwd3xdyi35xf613qpwsi", + "rev": "a35a275df92e7583df38f2de2562361f2b69987e", + "date": "2020-12-13T11:27:21-08:00", + "path": "/nix/store/l0mv4q1xdxz94ym1nl73y52i1yr9zcgi-tree-sitter-cpp", + "sha256": "130vizybkm11j3lpzmf183myz0vjxq75mpy6qz48rrkidhnrlryk", "fetchSubmodules": false, "deepClone": false, "leaveDotGit": false diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-css.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-css.json new file mode 100644 index 00000000000..b8609c0bd17 --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-css.json @@ -0,0 +1,10 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-css", + "rev": "23f2cb97d47860c517f67f03e1f4b621d5bd2085", + "date": "2020-05-14T14:44:30-07:00", + "path": "/nix/store/r5pkz9kly0mhgrmqzdzdsr6d1dpqavld-tree-sitter-css", + "sha256": "17svpf36p0p7spppzhm3fi833zpdl2l1scg34r6d4vcbv7dknrjy", + "fetchSubmodules": false, + "deepClone": false, + "leaveDotGit": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-fluent.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-fluent.json new file mode 100644 index 00000000000..f40b8465b7d --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-fluent.json @@ -0,0 +1,10 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-fluent", + "rev": "858fdd6f1e81992e00d3541bfb31bac9365d7a47", + "date": "2018-06-18T13:00:38-07:00", + "path": "/nix/store/zbj8abdlrqi9swm8qn8rhpqmjwcz145f-tree-sitter-fluent", + "sha256": "0528v9w0cs73p9048xrddb1wpdhr92sn1sw8yyqfrq5sq0danr9k", + "fetchSubmodules": false, + "deepClone": false, + "leaveDotGit": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-haskell.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-haskell.json new file mode 100644 index 00000000000..10d2dd64ac2 --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-haskell.json @@ -0,0 +1,10 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-haskell", + "rev": "2a0aa1cb5f1b787a4056a29fa0791e87846e33fb", + "date": "2018-11-03T09:56:20-07:00", + "path": "/nix/store/9xszs4xi51qr72laxr67zxnh8y2br0gy-tree-sitter-haskell", + "sha256": "0z0nfip5m0yrjgm49j5nld7jkgyjdmps101xhbng39pwfnh1av83", + "fetchSubmodules": false, + "deepClone": false, + "leaveDotGit": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json index 68008d393d1..2c563f3fbfa 100644 --- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json @@ -1,9 +1,9 @@ { "url": "https://github.com/tree-sitter/tree-sitter-javascript", - "rev": "852f11b394804ac2a8986f8bcaafe77753635667", - "date": "2020-10-27T13:43:40-04:00", - "path": "/nix/store/7a9nd3prxbv2izvilqdxf58his097ak7-tree-sitter-javascript", - "sha256": "1cdqx75fm7fvna0iymw7n03a7f6gayfg97qwqz3himi0775fz9ir", + "rev": "3f8b62f9befd3cb3b4cb0de22f6595a0aadf76ca", + "date": "2020-12-02T10:20:20-08:00", + "path": "/nix/store/c17bf7sjq95lank5ygbglv8j48i5z9w3-tree-sitter-javascript", + "sha256": "0fjq1jzrzd8c8rfxkh2s25gnqlyc19k3a8i3r1129kakisn1288k", "fetchSubmodules": false, "deepClone": false, "leaveDotGit": false diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-julia.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-julia.json new file mode 100644 index 00000000000..2cf5f06dc19 --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-julia.json @@ -0,0 +1,10 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-julia", + "rev": "6a0863f1ce3fcf6f99dc0addb7886dcbd27c5a48", + "date": "2020-09-08T19:39:52-07:00", + "path": "/nix/store/xn5nii9mi2aw7xdabyxlglha2vk12h1w-tree-sitter-julia", + "sha256": "07ds4wzgvnkgkq07izdglkk8lgpqrylvrs96afnwxi56bnzs8sbv", + "fetchSubmodules": false, + "deepClone": false, + "leaveDotGit": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-lua.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-lua.json index 267d25a52a0..804d613357e 100644 --- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-lua.json +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-lua.json @@ -1,9 +1,9 @@ { "url": "https://github.com/nvim-treesitter/tree-sitter-lua", - "rev": "278b0ea1f1f3d86afc86faaca0cbbf6c01182a54", - "date": "2020-09-08T18:29:23+02:00", - "path": "/nix/store/xkaqsw030bf6zd6pivrzrg3d50j1ka1k-tree-sitter-lua", - "sha256": "0i8jdp0bbl02h58hfhpild4v3rvy8yk5r5l90navvfxw4ad4f6f3", + "rev": "97b757ad3546d68dc1131e6ffb9b1d2a750eea29", + "date": "2020-11-25T21:23:57+01:00", + "path": "/nix/store/irz0dikk9dahkayi41chcznqq1i3wr84-tree-sitter-lua", + "sha256": "1nhg139vk3xm6ip4mhr29z0rprfg4q417z2vlvkz3m9wp8gppzls", "fetchSubmodules": false, "deepClone": false, "leaveDotGit": false diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ocaml.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ocaml.json new file mode 100644 index 00000000000..32488ebb73e --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ocaml.json @@ -0,0 +1,10 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-ocaml", + "rev": "4ec9ee414dadc2b6e7325a9f8124d02f6cd8c250", + "date": "2020-09-18T02:13:40+02:00", + "path": "/nix/store/dszjdnwnbziqxav2khs85026msm6fasz-tree-sitter-ocaml", + "sha256": "0wy85940fhmrnz7c1gk6xkipm8ixzalq5q4i7mcc6wnjiiwq60gx", + "fetchSubmodules": false, + "deepClone": false, + "leaveDotGit": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json index 402c4c087b1..86551acbee0 100644 --- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json @@ -1,9 +1,9 @@ { "url": "https://github.com/tree-sitter/tree-sitter-python", - "rev": "d245768132eb6cb74d8a394ca4d29dc57169b096", - "date": "2020-10-27T13:20:04-04:00", - "path": "/nix/store/afv7ibxpf5c2shafa89cnza88hsfb7kj-tree-sitter-python", - "sha256": "075r5i2id2rn76xm3pcrn5cpvj63dlaxcpvphig39a4c9f9hrpdx", + "rev": "65b486b42fe0188e4be344092151042bf1bc6d9a", + "date": "2020-12-15T09:33:20-08:00", + "path": "/nix/store/k9jziqzyxq2bv55pwl03jcmmca83fjyp-tree-sitter-python", + "sha256": "1yxqdlmp0jybm7vvza1ni5a320vrviqkd14pnpcrg9ilzq23mlsh", "fetchSubmodules": false, "deepClone": false, "leaveDotGit": false diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ql.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ql.json new file mode 100644 index 00000000000..27042b6ef95 --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ql.json @@ -0,0 +1,10 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-ql", + "rev": "a0d688d62dcb9cbc7c53f0d98343c458b3776b3d", + "date": "2020-09-16T12:56:09-07:00", + "path": "/nix/store/dfdaf6wg80dfw5fvdiir7n9nj6j30g3g-tree-sitter-ql", + "sha256": "0f6rfhrbvpg8czfa7mld45by3rp628bs6fyl47a8mn18w6x0n5g2", + "fetchSubmodules": false, + "deepClone": false, + "leaveDotGit": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-razor.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-razor.json new file mode 100644 index 00000000000..6905f4145aa --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-razor.json @@ -0,0 +1,10 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-razor", + "rev": "60edbd8e798e416f5226a746396efa6a8614fa9b", + "date": "2016-07-08T15:17:50-07:00", + "path": "/nix/store/2vkxw48wc0xf07awi65r4rg2m1lpqq6c-tree-sitter-razor", + "sha256": "1ldsn9nxldxkxdy7irywx88d6a56q05f6907lypzass07piyqp2i", + "fetchSubmodules": false, + "deepClone": false, + "leaveDotGit": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-regex.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-regex.json new file mode 100644 index 00000000000..d04c1bdab86 --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-regex.json @@ -0,0 +1,10 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-regex", + "rev": "be2e415b5716615530234d179dc27c32b7a1d86b", + "date": "2020-05-14T15:05:54-07:00", + "path": "/nix/store/acbcq9lgh5pf6hq3329fa253yfvaj6ji-tree-sitter-regex", + "sha256": "0qxbafbwfj64bwhj3455jgkidpdzqwrlsqm3rckq2pi75flnkv42", + "fetchSubmodules": false, + "deepClone": false, + "leaveDotGit": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json index d5a1767a23c..027478e99f8 100644 --- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json @@ -1,9 +1,9 @@ { "url": "https://github.com/tree-sitter/tree-sitter-ruby", - "rev": "f86d90cce578e28c824afdcdbdca7d031b88c0b0", - "date": "2020-11-02T20:49:49-08:00", - "path": "/nix/store/gs0hhb1dlgmv0y60zyyz7ki30yvpddfx-tree-sitter-ruby", - "sha256": "1n1irdjfmv27p5ia2vqzv8zfsq19pxi0as0wrkb7z7s1lqr1gwrb", + "rev": "bb572f60e9538bd11fbde95a54f97522073f1e06", + "date": "2020-12-20T12:25:50-08:00", + "path": "/nix/store/hkjgvxh8l4drk1z4zkhmd0wcj876x5y1-tree-sitter-ruby", + "sha256": "198sfmrb3mxvpfmlwmwfmb3qs075ih0bjmr0ffrdy4nqm4a00cb8", "fetchSubmodules": false, "deepClone": false, "leaveDotGit": false diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json index 4e0e2e2f7f4..38c9e44f2f6 100644 --- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json @@ -1,9 +1,9 @@ { "url": "https://github.com/tree-sitter/tree-sitter-rust", - "rev": "cf47ff80cbd8bdc6dd649a005c5f5d73f9c347e2", - "date": "2020-11-04T06:23:40-08:00", - "path": "/nix/store/8cvjxw8cz5jrkqwvgq88r7rfm269xwk1-tree-sitter-rust", - "sha256": "0yzcag0yy3ncxwpnb3fmsw8bxaidp7z8kbl8wwaxaj2vvxxf6sam", + "rev": "8746bd4b584b8063ee8e445bf31015e887417d33", + "date": "2020-11-23T17:16:42-08:00", + "path": "/nix/store/1zzxvza23wsdyazw47lhjvrs0za6wjpn-tree-sitter-rust", + "sha256": "0bhxfyq8ycnp90pqvr6cf3gpq1vax8a34kaq85dmbrc6ar8a7ap6", "fetchSubmodules": false, "deepClone": false, "leaveDotGit": false diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-scala.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-scala.json new file mode 100644 index 00000000000..8d9959cf4cf --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-scala.json @@ -0,0 +1,10 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-scala", + "rev": "211bb726bb5857f872247b600c7c1808e641a8d4", + "date": "2020-07-13T13:31:00-07:00", + "path": "/nix/store/6q66gzabxazr2581dgp4pp5wwyk2p6mw-tree-sitter-scala", + "sha256": "096ps5za8gxmq61gdd3xdk8cif07vb4v8asls2kdwm6jazm82777", + "fetchSubmodules": false, + "deepClone": false, + "leaveDotGit": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-swift.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-swift.json new file mode 100644 index 00000000000..8f73380e379 --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-swift.json @@ -0,0 +1,10 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-swift", + "rev": "a22fa5e19bae50098e2252ea96cba3aba43f4c58", + "date": "2019-10-24T19:04:02-06:00", + "path": "/nix/store/pk5xk8yp6vanbar75bhfrs104w0k1ph0-tree-sitter-swift", + "sha256": "14b40lmwrnyvdz2wiv684kfh4fvqfhbj1dgrx81ppmy7hsz7jcq7", + "fetchSubmodules": false, + "deepClone": false, + "leaveDotGit": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json index d31f72754bf..a875232eab6 100644 --- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json @@ -1,9 +1,9 @@ { "url": "https://github.com/tree-sitter/tree-sitter-typescript", - "rev": "73afadbd117a8e8551758af9c3a522ef46452119", - "date": "2020-10-20T12:15:52-07:00", - "path": "/nix/store/wrgp1j5l50cigv5cmlxikw693a55i6g9-tree-sitter-typescript", - "sha256": "0dsbmcvjzys9s229drp1l8anram7d6nsx1nixl5m9znw54hr4w9p", + "rev": "a3a4becef889692724bb1c2191f8fc4bce6573f9", + "date": "2020-12-15T13:34:55-08:00", + "path": "/nix/store/dra5wbhgxkkhphm1mhgv87hsfw0g18nw-tree-sitter-typescript", + "sha256": "0ir7lmzwnbf1zwbl4bfsib1jilg8pvfc1nabaq2n2rixvghlvf3d", "fetchSubmodules": false, "deepClone": false, "leaveDotGit": false diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-verilog.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-verilog.json new file mode 100644 index 00000000000..7ab79c6f2d5 --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-verilog.json @@ -0,0 +1,10 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-verilog", + "rev": "ad551aae2649da56582bc0557478a7dc979c0be3", + "date": "2020-10-13T17:40:47-07:00", + "path": "/nix/store/nfaxfqrqkxpwaq8rnk7kcp28nnj8y6m2-tree-sitter-verilog", + "sha256": "0cy29i200rnc34d237s19r6a1n5vv4d3wgwpbywxg6ahcankc34m", + "fetchSubmodules": false, + "deepClone": false, + "leaveDotGit": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/update.nix b/pkgs/development/tools/parsing/tree-sitter/update.nix index 2a3575a44d2..7b0b16b403f 100644 --- a/pkgs/development/tools/parsing/tree-sitter/update.nix +++ b/pkgs/development/tools/parsing/tree-sitter/update.nix @@ -1,66 +1,188 @@ -{ writeShellScript, nix-prefetch-git +{ writeShellScript, nix-prefetch-git, formats, lib , curl, jq, xe , src }: let - # print all the grammar names mentioned in the fetch-fixtures script - getGrammarNames = writeShellScript "get-grammars.sh" '' + # Grammars we want to fetch from the tree-sitter github orga + knownTreeSitterOrgGrammarRepos = [ + "tree-sitter-javascript" + "tree-sitter-c" + "tree-sitter-swift" + "tree-sitter-json" + "tree-sitter-cpp" + "tree-sitter-ruby" + "tree-sitter-razor" + "tree-sitter-go" + "tree-sitter-c-sharp" + "tree-sitter-python" + "tree-sitter-typescript" + "tree-sitter-rust" + "tree-sitter-bash" + "tree-sitter-php" + "tree-sitter-java" + "tree-sitter-scala" + "tree-sitter-ocaml" + "tree-sitter-julia" + "tree-sitter-agda" + "tree-sitter-fluent" + "tree-sitter-html" + "tree-sitter-haskell" + "tree-sitter-regex" + "tree-sitter-css" + "tree-sitter-verilog" + "tree-sitter-jsdoc" + "tree-sitter-ql" + "tree-sitter-embedded-template" + ]; + knownTreeSitterOrgGrammarReposJson = jsonFile "known-tree-sitter-org-grammar-repos" knownTreeSitterOrgGrammarRepos; + + # repos of the tree-sitter github orga we want to ignore (not grammars) + ignoredTreeSitterOrgRepos = [ + "tree-sitter" + "tree-sitter-cli" + # this is the haskell language bindings, tree-sitter-haskell is the grammar + "haskell-tree-sitter" + # this is the ruby language bindings, tree-sitter-ruby is the grammar + "ruby-tree-sitter" + # this is the (unmaintained) rust language bindings, tree-sitter-rust is the grammar + "rust-tree-sitter" + # this is the nodejs language bindings, tree-sitter-javascript is the grammar + "node-tree-sitter" + # this is the python language bindings, tree-sitter-python is the grammar + "py-tree-sitter" + # afl fuzzing for tree sitter + "afl-tree-sitter" + # archived + "highlight-schema" + # website + "tree-sitter.github.io" + ]; + ignoredTreeSitterOrgReposJson = jsonFile "ignored-tree-sitter-org-repos" ignoredTreeSitterOrgRepos; + + # Additional grammars that are not in the official github orga. + # If you need a grammar that already exists in the official orga, + # make sure to give it a different name. + otherGrammars = { + "tree-sitter-lua" = { + orga = "nvim-treesitter"; + repo = "tree-sitter-lua"; + }; + }; + + allGrammars = + let + treeSitterOrgaGrammars = + lib.listToAttrs (map (repo: + { name = repo; + value = { + orga = "tree-sitter"; + inherit repo; + }; + }) + knownTreeSitterOrgGrammarRepos); + + in + mergeAttrsUnique otherGrammars treeSitterOrgaGrammars; + + # TODO: move to lib + mergeAttrsUnique = left: right: + let intersect = lib.intersectLists (lib.attrNames left) (lib.attrNames right); in + assert + lib.assertMsg (intersect == []) + (lib.concatStringsSep "\n" [ + "mergeAttrsUnique: keys in attrset overlapping:" + "left: ${lib.generators.toPretty {} (lib.getAttrs intersect left)}" + "right: ${lib.generators.toPretty {} (lib.getAttrs intersect right)}" + ]); + left // right; + + + + jsonFile = name: val: (formats.json {}).generate name val; + + # check the tree-sitter orga repos + checkTreeSitterRepos = writeShellScript "get-grammars.sh" '' set -euo pipefail - sed -ne 's/^fetch_grammar \(\S*\).*$/\1/p' \ - ${src}/script/fetch-fixtures + res=$(${jq}/bin/jq \ + --slurpfile known "${knownTreeSitterOrgGrammarReposJson}" \ + --slurpfile ignore "${ignoredTreeSitterOrgReposJson}" \ + '. - ($known[0] + $ignore[0])' \ + ) + if [ ! "$res" == "[]" ]; then + echo "These repositories are neither known nor ignored:" 1>&2 + echo "$res" 1>&2 + exit 1 + fi ''; # TODO urlEscape = x: x; - # TODO - urlEscapeSh = writeShellScript "escape-url" ''printf '%s' "$1"''; # generic bash script to find the latest github release for a repo - latestGithubRelease = { owner }: writeShellScript "latest-github-release" '' + latestGithubRelease = { orga, repo }: writeShellScript "latest-github-release" '' set -euo pipefail - repo="$1" res=$(${curl}/bin/curl \ --silent \ - "https://api.github.com/repos/${urlEscape owner}/$(${urlEscapeSh} "$repo")/releases/latest") - if [[ "$(printf "%s" "$res" | ${jq}/bin/jq '.message')" =~ "rate limit" ]]; then + "https://api.github.com/repos/${urlEscape orga}/${urlEscape repo}/releases/latest") + if [[ "$(printf "%s" "$res" | ${jq}/bin/jq '.message?')" =~ "rate limit" ]]; then echo "rate limited" >&2 fi release=$(printf "%s" "$res" | ${jq}/bin/jq '.tag_name') # github sometimes returns an empty list even tough there are releases if [ "$release" = "null" ]; then - echo "uh-oh, latest for $repo is not there, using HEAD" >&2 + echo "uh-oh, latest for ${orga + "/" + repo} is not there, using HEAD" >&2 release="HEAD" fi echo "$release" ''; + # find the latest repos of a github organization + latestGithubRepos = { orga }: writeShellScript "latest-github-repos" '' + set -euo pipefail + res=$(${curl}/bin/curl \ + --silent \ + 'https://api.github.com/orgs/${urlEscape orga}/repos?per_page=100') + + if [[ "$(printf "%s" "$res" | ${jq}/bin/jq '.message?')" =~ "rate limit" ]]; then + echo "rate limited" >&2 # + fi + + printf "%s" "$res" | ${jq}/bin/jq 'map(.name)' \ + || echo "failed $res" + ''; + # update one tree-sitter grammar repo and print their nix-prefetch-git output - updateGrammar = { owner }: writeShellScript "update-grammar.sh" '' + updateGrammar = { orga, repo }: writeShellScript "update-grammar.sh" '' set -euo pipefail - repo="$1" - latest="$(${latestGithubRelease { inherit owner; }} "$repo")" - echo "Fetching latest release ($latest) of $repo …" >&2 + latest="$(${latestGithubRelease { inherit orga repo; }})" + echo "Fetching latest release ($latest) of ${repo} …" >&2 ${nix-prefetch-git}/bin/nix-prefetch-git \ --quiet \ --no-deepClone \ - --url "https://github.com/${urlEscape owner}/$(${urlEscapeSh} "$repo")" \ + --url "https://github.com/${urlEscape orga}/${urlEscape repo}" \ --rev "$latest" ''; + foreachSh = attrs: f: + lib.concatMapStringsSep "\n" f + (lib.mapAttrsToList (k: v: { name = k; } // v) attrs); + update-all-grammars = writeShellScript "update-all-grammars.sh" '' set -euo pipefail - grammarNames=$(${getGrammarNames}) + echo "fetching list of grammars" 1>&2 + treeSitterRepos=$(${latestGithubRepos { orga = "tree-sitter"; }}) + echo "checking the tree-sitter repo list against the grammars we know" 1>&2 + printf '%s' "$treeSitterRepos" | ${checkTreeSitterRepos} outputDir="${toString ./.}/grammars" + echo "writing files to $outputDir" 1>&2 mkdir -p "$outputDir" - updateCommand=$(printf \ - '${updateGrammar { owner = "tree-sitter"; }} "$1" > "%s/$1.json"' \ - "$outputDir") - printf '%s' "$grammarNames" \ - | ${xe}/bin/xe printf "tree-sitter-%s\n" {} \ - | ${xe}/bin/xe -j2 -s "$updateCommand" + ${foreachSh allGrammars + ({name, orga, repo}: ''${updateGrammar { inherit orga repo; }} > $outputDir/${name}.json'')} ( echo "{" - printf '%s' "$grammarNames" \ - | ${xe}/bin/xe -s 'printf " %s = (builtins.fromJSON (builtins.readFile ./tree-sitter-%s.json));\n" "$1" "$1"' + ${foreachSh allGrammars + ({name, ...}: '' + # indentation hack + printf " %s = (builtins.fromJSON (builtins.readFile ./%s.json));\n" "${name}" "${name}"'')} echo "}" ) \ > "$outputDir/default.nix" ''; |