Port to termwiz: compiles but no rendering yet

Fixes #1654

.cargo/config

@@ -0,0 +1,2 @@
+[alias]
+xtask = "run --package xtask --"

.envrc

@@ -3,3 +3,4 @@ watch_file flake.lock
 
 # try to use flakes, if it fails use normal nix (ie. shell.nix)
 use flake || use nix
+eval "$shellHook"

.github/ISSUE_TEMPLATE/bug_report.md

@@ -17,7 +17,8 @@ Please search on the issue tracker before creating one. -->
 ### Environment
 
 - Platform: <!--  macOS / Windows / Linux -->
-- Helix version: <!--  'hx -v' if using a release, 'git describe' if building from master -->
+- Terminal emulator: 
+- Helix version: <!--  'hx -V' if using a release, 'git describe' if building from master -->
 
 <details><summary>~/.cache/helix/helix.log</summary>
 

.github/workflows/build.yml

@@ -13,9 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
-        with:
-          submodules: true
 
       - name: Install stable toolchain
         uses: actions-rs/toolchain@v1
@@ -25,22 +23,25 @@ jobs:
           override: true
 
       - name: Cache cargo registry
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v2.1.7
         with:
           path: ~/.cargo/registry
-          key: ${{ runner.os }}-v1-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ runner.os }}-v2-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-registry-
 
       - name: Cache cargo index
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v2.1.7
         with:
           path: ~/.cargo/git
-          key: ${{ runner.os }}-v1-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ runner.os }}-v2-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-index-
 
       - name: Cache cargo target dir
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v2.1.7
         with:
           path: target
-          key: ${{ runner.os }}-v1-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ runner.os }}-v2-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-build-target-
 
       - name: Run cargo check
         uses: actions-rs/cargo@v1
@@ -52,9 +53,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
-        with:
-          submodules: true
 
       - name: Install stable toolchain
         uses: actions-rs/toolchain@v1
@@ -64,27 +63,41 @@ jobs:
           override: true
 
       - name: Cache cargo registry
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v2.1.7
         with:
           path: ~/.cargo/registry
-          key: ${{ runner.os }}-v1-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ runner.os }}-v2-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-registry-
 
       - name: Cache cargo index
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v2.1.7
         with:
           path: ~/.cargo/git
-          key: ${{ runner.os }}-v1-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ runner.os }}-v2-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-index-
 
       - name: Cache cargo target dir
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v2.1.7
         with:
           path: target
-          key: ${{ runner.os }}-v1-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ runner.os }}-v2-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-build-target-
+
+      - name: Copy minimal languages config
+        run: cp .github/workflows/languages.toml ./languages.toml
+
+      - name: Cache test tree-sitter grammar
+        uses: actions/cache@v2.1.7
+        with:
+          path: runtime/grammars
+          key: ${{ runner.os }}-v2-tree-sitter-grammars-${{ hashFiles('languages.toml') }}
+          restore-keys: ${{ runner.os }}-v2-tree-sitter-grammars-
 
       - name: Run cargo test
         uses: actions-rs/cargo@v1
         with:
           command: test
+          args: --workspace
 
     strategy:
       matrix:
@@ -96,9 +109,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
-        with:
-          submodules: true
 
       - name: Install stable toolchain
         uses: actions-rs/toolchain@v1
@@ -109,22 +120,25 @@ jobs:
           components: rustfmt, clippy
 
       - name: Cache cargo registry
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v2.1.7
         with:
           path: ~/.cargo/registry
-          key: ${{ runner.os }}-v1-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ runner.os }}-v2-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-registry-
 
       - name: Cache cargo index
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v2.1.7
         with:
           path: ~/.cargo/git
-          key: ${{ runner.os }}-v1-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ runner.os }}-v2-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-index-
 
       - name: Cache cargo target dir
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v2.1.7
         with:
           path: target
-          key: ${{ runner.os }}-v1-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ runner.os }}-v2-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-build-target-
 
       - name: Run cargo fmt
         uses: actions-rs/cargo@v1
@@ -136,4 +150,53 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: clippy
-          args: -- -D warnings
+          args: --all-targets -- -D warnings
+
+  docs:
+    name: Docs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+
+      - name: Install stable toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+
+      - name: Cache cargo registry
+        uses: actions/cache@v2.1.6
+        with:
+          path: ~/.cargo/registry
+          key: ${{ runner.os }}-v2-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-registry-
+
+      - name: Cache cargo index
+        uses: actions/cache@v2.1.6
+        with:
+          path: ~/.cargo/git
+          key: ${{ runner.os }}-v2-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-index-
+
+      - name: Cache cargo target dir
+        uses: actions/cache@v2.1.6
+        with:
+          path: target
+          key: ${{ runner.os }}-v2-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-build-target-
+
+      - name: Generate docs
+        uses: actions-rs/cargo@v1
+        with:
+          command: xtask
+          args: docgen
+
+      - name: Check uncommitted documentation changes
+        run: |
+          git diff
+          git diff-files --quiet \
+            || (echo "Run 'cargo xtask docgen', commit the changes and push again" \
+            && exit 1)
+

.github/workflows/cachix.yml

@@ -0,0 +1,26 @@
+# Publish the Nix flake outputs to Cachix
+name: Cachix
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  publish:
+    name: Publish Flake
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout sources
+      uses: actions/checkout@v3
+
+    - name: Install nix
+      uses: cachix/install-nix-action@v16
+
+    - name: Authenticate with Cachix
+      uses: cachix/cachix-action@v10
+      with:
+        name: helix
+        authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
+    - name: Build nix flake
+      run: nix build

.github/workflows/gh-pages.yml

@@ -4,12 +4,14 @@ on:
   push:
     branches:
       - master
+    tags:
+      - '*'
 
 jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Setup mdBook
         uses: peaceiris/actions-mdbook@v1
@@ -19,9 +21,21 @@ jobs:
 
       - run: mdbook build book
       
+      - name: Set output directory
+        run: |
+          OUTDIR=$(basename ${{ github.ref }})
+          echo "OUTDIR=$OUTDIR" >> $GITHUB_ENV
+
       - name: Deploy
         uses: peaceiris/actions-gh-pages@v3
-        if: github.ref == 'refs/heads/master'
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./book/book
+          destination_dir: ./${{ env.OUTDIR }}
+
+      - name: Deploy stable
+        uses: peaceiris/actions-gh-pages@v3
+        if: startswith(github.ref, 'refs/tags/')
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./book/book

.github/workflows/languages.toml

@@ -0,0 +1,26 @@
+# This languages.toml is used for testing in CI.
+
+[[language]]
+name = "rust"
+scope = "source.rust"
+injection-regex = "rust"
+file-types = ["rs"]
+comment-token = "//"
+roots = ["Cargo.toml", "Cargo.lock"]
+indent = { tab-width = 4, unit = "    " }
+
+[[grammar]]
+name = "rust"
+source = { git = "https://github.com/tree-sitter/tree-sitter-rust", rev = "a360da0a29a19c281d08295a35ecd0544d2da211" }
+
+[[language]]
+name = "nix"
+scope = "source.nix"
+injection-regex = "nix"
+file-types = ["nix"]
+shebangs = []
+roots = []
+comment-token = "#"
+
+# A grammar entry is not necessary for this language - it is only used for
+# testing TOML merging behavior.

.github/workflows/release.yml

@@ -52,9 +52,7 @@ jobs:
 
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
-        with:
-          submodules: true
 
       - name: Install ${{ matrix.rust }} toolchain
         uses: actions-rs/toolchain@v1
@@ -67,8 +65,9 @@ jobs:
       - name: Run cargo test
         uses: actions-rs/cargo@v1
         with:
+          use-cross: ${{ matrix.cross }}
           command: test
-          args: --release --locked
+          args: --release --locked --target ${{ matrix.target }} --workspace
 
       - name: Build release binary
         uses: actions-rs/cargo@v1
@@ -101,7 +100,7 @@ jobs:
           fi
           cp -r runtime dist
 
-      - uses: actions/upload-artifact@v2.2.4
+      - uses: actions/upload-artifact@v3
         with:
           name: bins-${{ matrix.build }}
           path: dist
@@ -112,15 +111,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
-        with:
-          submodules: false
 
       - uses: actions/download-artifact@v2
-        # with:
-        #   path: dist
-      # - run: ls -al ./dist
-      - run: ls -al bins-*
 
       - name: Calculate tag name
         run: |

.gitmodules

@@ -1,96 +0,0 @@
-[submodule "helix-syntax/languages/tree-sitter-cpp"]
-	path = helix-syntax/languages/tree-sitter-cpp
-	url = https://github.com/tree-sitter/tree-sitter-cpp
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-javascript"]
-	path = helix-syntax/languages/tree-sitter-javascript
-	url = https://github.com/tree-sitter/tree-sitter-javascript
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-julia"]
-	path = helix-syntax/languages/tree-sitter-julia
-	url = https://github.com/tree-sitter/tree-sitter-julia
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-python"]
-	path = helix-syntax/languages/tree-sitter-python
-	url = https://github.com/tree-sitter/tree-sitter-python
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-typescript"]
-	path = helix-syntax/languages/tree-sitter-typescript
-	url = https://github.com/tree-sitter/tree-sitter-typescript
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-agda"]
-	path = helix-syntax/languages/tree-sitter-agda
-	url = https://github.com/tree-sitter/tree-sitter-agda
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-go"]
-	path = helix-syntax/languages/tree-sitter-go
-	url = https://github.com/tree-sitter/tree-sitter-go
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-ruby"]
-	path = helix-syntax/languages/tree-sitter-ruby
-	url = https://github.com/tree-sitter/tree-sitter-ruby
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-java"]
-	path = helix-syntax/languages/tree-sitter-java
-	url = https://github.com/tree-sitter/tree-sitter-java
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-php"]
-	path = helix-syntax/languages/tree-sitter-php
-	url = https://github.com/tree-sitter/tree-sitter-php
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-html"]
-	path = helix-syntax/languages/tree-sitter-html
-	url = https://github.com/tree-sitter/tree-sitter-html
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-scala"]
-	path = helix-syntax/languages/tree-sitter-scala
-	url = https://github.com/tree-sitter/tree-sitter-scala
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-bash"]
-	path = helix-syntax/languages/tree-sitter-bash
-	url = https://github.com/tree-sitter/tree-sitter-bash
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-rust"]
-	path = helix-syntax/languages/tree-sitter-rust
-	url = https://github.com/tree-sitter/tree-sitter-rust
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-json"]
-	path = helix-syntax/languages/tree-sitter-json
-	url = https://github.com/tree-sitter/tree-sitter-json
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-css"]
-	path = helix-syntax/languages/tree-sitter-css
-	url = https://github.com/tree-sitter/tree-sitter-css
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-c-sharp"]
-	path = helix-syntax/languages/tree-sitter-c-sharp
-	url = https://github.com/tree-sitter/tree-sitter-c-sharp
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-c"]
-	path = helix-syntax/languages/tree-sitter-c
-	url = https://github.com/tree-sitter/tree-sitter-c
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-haskell"]
-	path = helix-syntax/languages/tree-sitter-haskell
-	url = https://github.com/tree-sitter/tree-sitter-haskell
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-swift"]
-	path = helix-syntax/languages/tree-sitter-swift
-	url = https://github.com/tree-sitter/tree-sitter-swift
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-toml"]
-	path = helix-syntax/languages/tree-sitter-toml
-	url = https://github.com/ikatyang/tree-sitter-toml
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-elixir"]
-	path = helix-syntax/languages/tree-sitter-elixir
-	url = https://github.com/IceDragon200/tree-sitter-elixir
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-nix"]
-	path = helix-syntax/languages/tree-sitter-nix
-	url = https://github.com/cstrahan/tree-sitter-nix
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-latex"]
-	path = helix-syntax/languages/tree-sitter-latex
-	url = https://github.com/latex-lsp/tree-sitter-latex
-	shallow = true

CHANGELOG.md

@@ -1,6 +1,214 @@
 
+# 0.6.0 (2022-01-04)
+
+Happy new year and a big shout out to all the contributors! We had 55 contributors in this release.
+
+Helix has popped up in DPorts and Fedora Linux via COPR ([#1270](https://github.com/helix-editor/helix/pull/1270))
+
+As usual the following is a brief summary, refer to the git history for a full log:
+
+Breaking changes:
+
+- fix: Normalize backtab into shift-tab 
+
+Features:
+
+- Macros ([#1234](https://github.com/helix-editor/helix/pull/1234)) 
+- Add reverse search functionality ([#958](https://github.com/helix-editor/helix/pull/958))
+- Allow keys to be mapped to sequences of commands ([#589](https://github.com/helix-editor/helix/pull/589)) 
+- Make it possible to keybind TypableCommands ([#1169](https://github.com/helix-editor/helix/pull/1169))
+- Detect workspace root using language markers ([#1370](https://github.com/helix-editor/helix/pull/1370))
+- Add WORD textobject ([#991](https://github.com/helix-editor/helix/pull/991))
+- Add LSP rename_symbol (space-r) ([#1011](https://github.com/helix-editor/helix/pull/1011))
+- Added workspace_symbol_picker ([#1041](https://github.com/helix-editor/helix/pull/1041)) 
+- Detect filetype from shebang line ([#1001](https://github.com/helix-editor/helix/pull/1001))
+- Allow piping from stdin into a buffer on startup ([#996](https://github.com/helix-editor/helix/pull/996)) 
+- Add auto pairs for same-char pairs ([#1219](https://github.com/helix-editor/helix/pull/1219)) 
+- Update settings at runtime ([#798](https://github.com/helix-editor/helix/pull/798)) 
+- Enable thin LTO (cccc194)
+
+Commands:
+- :wonly -- window only ([#1057](https://github.com/helix-editor/helix/pull/1057)) 
+- buffer-close (:bc, :bclose) ([#1035](https://github.com/helix-editor/helix/pull/1035)) 
+- Add :<line> and :goto <line> commands ([#1128](https://github.com/helix-editor/helix/pull/1128))
+- :sort command ([#1288](https://github.com/helix-editor/helix/pull/1288)) 
+- Add m textobject for pair under cursor ([#961](https://github.com/helix-editor/helix/pull/961)) 
+- Implement "Goto next buffer / Goto previous buffer" commands ([#950](https://github.com/helix-editor/helix/pull/950))
+- Implement "Goto last modification" command ([#1067](https://github.com/helix-editor/helix/pull/1067)) 
+- Add trim_selections command ([#1092](https://github.com/helix-editor/helix/pull/1092)) 
+- Add movement shortcut for history ([#1088](https://github.com/helix-editor/helix/pull/1088))
+- Add command to inc/dec number under cursor ([#1027](https://github.com/helix-editor/helix/pull/1027))
+  - Add support for dates for increment/decrement
+- Align selections (&) ([#1101](https://github.com/helix-editor/helix/pull/1101))
+- Implement no-yank delete/change ([#1099](https://github.com/helix-editor/helix/pull/1099)) 
+- Implement black hole register ([#1165](https://github.com/helix-editor/helix/pull/1165)) 
+- gf as goto_file (gf) ([#1102](https://github.com/helix-editor/helix/pull/1102))
+- Add last modified file (gm) ([#1093](https://github.com/helix-editor/helix/pull/1093))
+- ensure_selections_forward ([#1393](https://github.com/helix-editor/helix/pull/1393))
+- Readline style insert mode ([#1039](https://github.com/helix-editor/helix/pull/1039))
+
+Usability improvements and fixes:
+
+- Detect filetype on :write ([#1141](https://github.com/helix-editor/helix/pull/1141))
+- Add single and double quotes to matching pairs ([#995](https://github.com/helix-editor/helix/pull/995)) 
+- Launch with defaults upon invalid config/theme (rather than panicking) ([#982](https://github.com/helix-editor/helix/pull/982))
+- If switching away from an empty scratch buffer, remove it ([#935](https://github.com/helix-editor/helix/pull/935)) 
+- Truncate the starts of file paths instead of the ends in picker ([#951](https://github.com/helix-editor/helix/pull/951))
+- Truncate the start of file paths in the StatusLine ([#1351](https://github.com/helix-editor/helix/pull/1351)) 
+- Prevent picker from previewing binaries or large file ([#939](https://github.com/helix-editor/helix/pull/939))
+- Inform when reaching undo/redo bounds ([#981](https://github.com/helix-editor/helix/pull/981))
+- search_impl will only align cursor center when it isn't in view ([#959](https://github.com/helix-editor/helix/pull/959)) 
+- Add <C-h>, <C-u>, <C-d>, Delete in prompt mode ([#1034](https://github.com/helix-editor/helix/pull/1034)) 
+- Restore screen position when aborting search ([#1047](https://github.com/helix-editor/helix/pull/1047)) 
+- Buffer picker: show is_modifier flag ([#1020](https://github.com/helix-editor/helix/pull/1020)) 
+- Add commit hash to version info, if present ([#957](https://github.com/helix-editor/helix/pull/957))
+- Implement indent-aware delete ([#1120](https://github.com/helix-editor/helix/pull/1120))
+- Jump to end char of surrounding pair from any cursor pos ([#1121](https://github.com/helix-editor/helix/pull/1121))
+- File picker configuration ([#988](https://github.com/helix-editor/helix/pull/988))
+- Fix surround cursor position calculation ([#1183](https://github.com/helix-editor/helix/pull/1183))
+- Accept count for goto_window ([#1033](https://github.com/helix-editor/helix/pull/1033))
+- Make kill_to_line_end behave like emacs ([#1235](https://github.com/helix-editor/helix/pull/1235)) 
+- Only use a single documentation popup ([#1241](https://github.com/helix-editor/helix/pull/1241)) 
+- ui: popup: Don't allow scrolling past the end of content (3307f44c)
+- Open files with spaces in filename, allow opening multiple files ([#1231](https://github.com/helix-editor/helix/pull/1231)) 
+- Allow paste commands to take a count ([#1261](https://github.com/helix-editor/helix/pull/1261))
+- Auto pairs selection ([#1254](https://github.com/helix-editor/helix/pull/1254)) 
+- Use a fuzzy matcher for commands ([#1386](https://github.com/helix-editor/helix/pull/1386)) 
+- Add c-s to pick word under doc cursor to prompt line & search completion ([#831](https://github.com/helix-editor/helix/pull/831))
+- Fix :earlier/:later missing changeset update ([#1069](https://github.com/helix-editor/helix/pull/1069))
+- Support extend for multiple goto ([#909](https://github.com/helix-editor/helix/pull/909))
+- Add arrow-key bindings for window switching ([#933](https://github.com/helix-editor/helix/pull/933))
+- Implement key ordering for info box ([#952](https://github.com/helix-editor/helix/pull/952))
+
+LSP:
+- Implement MarkedString rendering (e128a8702)
+- Don't panic if init fails (d31bef7)
+- Configurable diagnostic severity ([#1325](https://github.com/helix-editor/helix/pull/1325))
+- Resolve completion item ([#1315](https://github.com/helix-editor/helix/pull/1315))
+- Code action command support ([#1304](https://github.com/helix-editor/helix/pull/1304))
+
+Grammars:
+
+- Adds mint language server ([#974](https://github.com/helix-editor/helix/pull/974)) 
+- Perl ([#978](https://github.com/helix-editor/helix/pull/978)) ([#1280](https://github.com/helix-editor/helix/pull/1280))
+- GLSL ([#993](https://github.com/helix-editor/helix/pull/993)) 
+- Racket ([#1143](https://github.com/helix-editor/helix/pull/1143)) 
+- WGSL ([#1166](https://github.com/helix-editor/helix/pull/1166)) 
+- LLVM ([#1167](https://github.com/helix-editor/helix/pull/1167)) ([#1388](https://github.com/helix-editor/helix/pull/1388)) ([#1409](https://github.com/helix-editor/helix/pull/1409)) ([#1398](https://github.com/helix-editor/helix/pull/1398))
+- Markdown (49e06787)
+- Scala ([#1278](https://github.com/helix-editor/helix/pull/1278))
+- Dart ([#1250](https://github.com/helix-editor/helix/pull/1250))
+- Fish ([#1308](https://github.com/helix-editor/helix/pull/1308)) 
+- Dockerfile ([#1303](https://github.com/helix-editor/helix/pull/1303))
+- Git (commit, rebase, diff) ([#1338](https://github.com/helix-editor/helix/pull/1338)) ([#1402](https://github.com/helix-editor/helix/pull/1402)) ([#1373](https://github.com/helix-editor/helix/pull/1373))
+- tree-sitter-comment ([#1300](https://github.com/helix-editor/helix/pull/1300))
+- Highlight comments in c, cpp, cmake and llvm ([#1309](https://github.com/helix-editor/helix/pull/1309))
+- Improve yaml syntax highlighting highlighting ([#1294](https://github.com/helix-editor/helix/pull/1294)) 
+- Improve rust syntax highlighting ([#1295](https://github.com/helix-editor/helix/pull/1295))
+- Add textobjects and indents to cmake ([#1307](https://github.com/helix-editor/helix/pull/1307)) 
+- Add textobjects and indents to c and cpp ([#1293](https://github.com/helix-editor/helix/pull/1293))
+
+New themes:
+
+- Solarized dark ([#999](https://github.com/helix-editor/helix/pull/999)) 
+- Solarized light ([#1010](https://github.com/helix-editor/helix/pull/1010)) 
+- Spacebones light ([#1131](https://github.com/helix-editor/helix/pull/1131))
+- Monokai Pro ([#1206](https://github.com/helix-editor/helix/pull/1206)) 
+- Base16 Light and Terminal ([#1078](https://github.com/helix-editor/helix/pull/1078))
+  - and a default 16 color theme, truecolor detection 
+- Dracula ([#1258](https://github.com/helix-editor/helix/pull/1258))
+
+# 0.5.0 (2021-10-28)
+
+A big shout out to all the contributors! We had 46 contributors in this release.
+
+Helix has popped up in [Scoop, FreeBSD Ports and Gentu GURU](https://repology.org/project/helix/versions)!
+
+The following is a quick rundown of the larger changes, there were many more
+(check the git history for more details).
+
+Breaking changes:
+
+- A couple of keymaps moved to resolve a few conflicting keybinds.
+  - Documentation popups were moved from `K` to `space+k`
+  - `K` is now `keep_selections` which filters selections to only keeps ones matching the regex
+  - `keep_primary_selection` moved from `space+space` to `,`
+  - `Alt-,` is now `remove_primary_selection` which keeps all selections except the primary one
+  - Opening files in a split moved from `C-h` to `C-s`
+- Some configuration options moved from a `[terminal]` section to `[editor]`. [Consult the documentation for more information.](https://docs.helix-editor.com/configuration.html)
+
+Features:
+
+- LSP compatibility greatly improved for some implementations (Julia, Python, Typescript)
+- Autocompletion! Completion now triggers automatically after a set idle timeout
+- Completion documentation is now displayed next to the popup ([#691](https://github.com/helix-editor/helix/pull/691))
+- Treesitter textobjects (select a function via `mf`, class via `mc`) ([#728](https://github.com/helix-editor/helix/pull/728)) 
+- Global search across entire workspace `space+/` ([#651](https://github.com/helix-editor/helix/pull/651)) 
+- Relative line number support ([#485](https://github.com/helix-editor/helix/pull/485))
+- Prompts now store a history (72cf86e)
+- `:vsplit` and `:hsplit` commands ([#639](https://github.com/helix-editor/helix/pull/639))
+- `C-w h/j/k/l` can now be used to navigate between splits ([#860](https://github.com/helix-editor/helix/pull/860))
+- `C-j` and `C-k` are now alternative keybindings to `C-n` and `C-p` in the UI ([#876](https://github.com/helix-editor/helix/pull/876))
+- Shell commands (shell-pipe, pipe-to, shell-insert-output, shell-append-output, keep-pipe) ([#547](https://github.com/helix-editor/helix/pull/547))
+- Searching now defaults to smart case search (case insensitive unless uppercase is used) ([#761](https://github.com/helix-editor/helix/pull/761))
+- The preview pane was improved to highlight and center line ranges
+- The user `languages.toml` is now merged into defaults, no longer need to copy the entire file (dc57f8dc)
+- Show hidden files in completions ([#648](https://github.com/helix-editor/helix/pull/648))
+- Grammar injections are now properly handled (dd0b15e)
+- `v` in select mode now switches back to normal mode ([#660](https://github.com/helix-editor/helix/pull/660))
+- View mode can now be triggered as a "sticky" mode ([#719](https://github.com/helix-editor/helix/pull/719))
+- `f`/`t` and object selection motions can now be repeated via `Alt-.` ([#891](https://github.com/helix-editor/helix/pull/891))
+- Statusline now displays total selection count and diagnostics counts for both errors and warnings ([#916](https://github.com/helix-editor/helix/pull/916))
+
+New grammars:
+
+- Ledger ([#572](https://github.com/helix-editor/helix/pull/572))
+- Protobuf ([#614](https://github.com/helix-editor/helix/pull/614))
+- Zig ([#631](https://github.com/helix-editor/helix/pull/631))
+- YAML ([#667](https://github.com/helix-editor/helix/pull/667))
+- Lua ([#665](https://github.com/helix-editor/helix/pull/665))
+- OCaml ([#666](https://github.com/helix-editor/helix/pull/666))
+- Svelte ([#733](https://github.com/helix-editor/helix/pull/733))
+- Vue ([#787](https://github.com/helix-editor/helix/pull/787))
+- Tree-sitter queries ([#845](https://github.com/helix-editor/helix/pull/845))
+- CMake ([#888](https://github.com/helix-editor/helix/pull/888))
+- Elixir (we switched over to the official grammar) (6c0786e)
+- Language server definitions for Nix and Elixir ([#725](https://github.com/helix-editor/helix/pull/725))
+- Python now uses `pylsp` instead of `pyls`
+- Python now supports indentation
+
+New themes:
+
+- Monokai ([#628](https://github.com/helix-editor/helix/pull/628))
+- Everforest Dark ([#760](https://github.com/helix-editor/helix/pull/760))
+- Nord ([#799](https://github.com/helix-editor/helix/pull/799))
+- Base16 Default Dark ([#833](https://github.com/helix-editor/helix/pull/833))
+- Rose Pine ([#897](https://github.com/helix-editor/helix/pull/897))
+
+Fixes:
+
+- Fix crash on empty rust file ([#592](https://github.com/helix-editor/helix/pull/592))
+- Exit select mode after toggle comment ([#598](https://github.com/helix-editor/helix/pull/598))
+- Pin popups with no positioning to the initial position (12ea3888)
+- xsel copy should not freeze the editor (6dd7dc4)
+- `*` now only sets the search register and doesn't jump to the next occurrence (3426285)
+- Goto line start/end commands extend when in select mode ([#739](https://github.com/helix-editor/helix/pull/739)) 
+- Fix documentation popups sometimes not getting fully highlighted (066367c)
+- Refactor apply_workspace_edit to remove assert (b02d872)
+- Wrap around the top of the picker menu when scrolling (c7d6e44)
+- Don't allow closing the last split if there's unsaved changes (3ff5b00)
+- Indentation used different default on hx vs hx new_file.txt (c913bad)
+
+# 0.4.1 (2021-08-14)
+
+A minor release that includes:
+- A fix for rendering glitches that would occur after editing with multiple selections.
+- CI fix for grammars not being cross-compiled for aarch64 
+
 # 0.4.0 (2021-08-13)
 
+A big shout out to all the contributors! We had 28 contributors in this release.
+
 Two months have passed, so this is another big release. A big thank you to all
 the contributors and package maintainers!
 
@@ -19,7 +227,7 @@ selections in the future as well as resolves many bugs and edge cases.
 - Selection contents rotation via `Alt-(` and `Alt-)` ([02cba2a](https://github.com/helix-editor/helix/commit/02cba2a7f403f48eccb18100fb751f7b42373dba))
 - Completion behavior improvements ([f917b5a4](https://github.com/helix-editor/helix/commit/f917b5a441ff3ae582358b6939ffbf889f4aa530), [627b899](https://github.com/helix-editor/helix/commit/627b89931576f7af86166ae8d5cbc55537877473))
 - Fixed a language server crash ([385a6b5a](https://github.com/helix-editor/helix/commit/385a6b5a1adddfc26e917982641530e1a7c7aa81))
-- Case change commands (`\``, `~`, `<a-~>`) ([#441](https://github.com/helix-editor/helix/pull/441))
+- Case change commands (`` ` ``, `~`, ``<a-`>``) ([#441](https://github.com/helix-editor/helix/pull/441))
 - File pickers (including goto) now provide a preview! ([#534](https://github.com/helix-editor/helix/pull/534))
 - Injection query support. Rust macro calls and embedded languages are now properly highlighted ([#430](https://github.com/helix-editor/helix/pull/430))
 - Formatting is now asynchronous, and the async job infrastructure has been improved ([#285](https://github.com/helix-editor/helix/pull/285))
@@ -36,7 +244,9 @@ selections in the future as well as resolves many bugs and edge cases.
 - Java support ([#448](https://github.com/helix-editor/helix/pull/448))
 - Prompts have an (in-memory) history ([63e54e30](https://github.com/helix-editor/helix/commit/63e54e30a74bb0d1d782877ddbbcf95f2817d061))
 
-# 0.4.0 (2021-06-27)
+# 0.3.0 (2021-06-27)
+
+A big shout out to all the contributors! We had 24 contributors in this release.
 
 Another big release. 
 
@@ -84,6 +294,8 @@ Includes a fix where wq/wqa could exit before file saving completed.
 
 # 0.2.0
 
+A big shout out to all the contributors! We had 18 contributors in this release.
+
 Enough has changed to bump the version. We're skipping 0.1.x because
 previously the CLI would always report version as 0.1.0, and we'd like
 to distinguish it in bug reports..

Cargo.toml

@@ -4,13 +4,26 @@ members = [
   "helix-view",
   "helix-term",
   "helix-tui",
-  "helix-syntax",
   "helix-lsp",
+  "helix-dap",
+  "helix-loader",
+  "xtask",
 ]
 
-# Build helix-syntax in release mode to make the code path faster in development.
+default-members = [
-# [profile.dev.package."helix-syntax"]
+  "helix-term"
-# opt-level = 3
+]
 
 [profile.dev]
 split-debuginfo = "unpacked"
+
+[profile.release]
+lto = "thin"
+# debug = true
+
+[profile.opt]
+inherits = "release"
+lto = "fat"
+codegen-units = 1
+# strip = "debuginfo" # TODO: or strip = true
+opt-level = 3

README.md

@@ -13,7 +13,9 @@ myself agreeing with most of kakoune's design decisions.
 For more information, see the [website](https://helix-editor.com) or
 [documentation](https://docs.helix-editor.com/).
 
-All shortcuts/keymaps can be found [in the documentation on the website](https://docs.helix-editor.com/keymap.html)
+All shortcuts/keymaps can be found [in the documentation on the website](https://docs.helix-editor.com/keymap.html).
+
+[Troubleshooting](https://github.com/helix-editor/helix/wiki/Troubleshooting)
 
 # Features
 
@@ -25,25 +27,27 @@ All shortcuts/keymaps can be found [in the documentation on the website](https:/
 It's a terminal-based editor first, but I'd like to explore a custom renderer
 (similar to emacs) in wgpu or skulpin.
 
-# Installation
-
 Note: Only certain languages have indentation definitions at the moment. Check
 `runtime/queries/<lang>/` for `indents.toml`.
 
+# Installation
+
 We provide packaging for various distributions, but here's a quick method to
 build from source.
 
 ```
-git clone --recurse-submodules --shallow-submodules -j8 https://github.com/helix-editor/helix
+git clone https://github.com/helix-editor/helix
 cd helix
 cargo install --path helix-term
+hx --grammar fetch
+hx --grammar build
 ```
 
-This will install the `hx` binary to `$HOME/.cargo/bin`.
+This will install the `hx` binary to `$HOME/.cargo/bin` and build tree-sitter grammars.
 
-Helix also needs it's runtime files so make sure to copy/symlink the `runtime/` directory into the
+Helix also needs its runtime files so make sure to copy/symlink the `runtime/` directory into the
-config directory (for example `~/.config/helix/runtime` on Linux/macOS). This location can be overriden
+config directory (for example `~/.config/helix/runtime` on Linux/macOS, or `%AppData%/helix/runtime` on Windows).
-via the `HELIX_RUNTIME` environment variable.
+This location can be overridden via the `HELIX_RUNTIME` environment variable.
 
 Packages already solve this for you by wrapping the `hx` binary with a wrapper
 that sets the variable to the install dir.
@@ -54,6 +58,7 @@ that sets the variable to the install dir.
 [![Packaging status](https://repology.org/badge/vertical-allrepos/helix.svg)](https://repology.org/project/helix/versions)
 
 ## MacOS
+
 Helix can be installed on MacOS through homebrew via:
 
 ```
@@ -63,22 +68,10 @@ brew install helix
  
 # Contributing
 
-Contributors are very welcome! **No contribution is too small and all contributions are valued.**
+Contributing guidelines can be found [here](./docs/CONTRIBUTING.md).
-
-Some suggestions to get started:
-
-- You can look at the [good first issue](https://github.com/helix-editor/helix/labels/E-easy) label on the issue tracker.
-- Help with packaging on various distributions needed!
-- To use print debugging to the `~/.cache/helix/helix.log` file, you must:
-  * Print using `log::info!`, `warn!`, or `error!`. (`log::info!("helix!")`)
-  * Pass the appropriate verbosity level option for the desired log level. (`hx -v <file>` for info, more `v`s for higher severity inclusive)
-- If your preferred language is missing, integrating a tree-sitter grammar for
-    it and defining syntax highlight queries for it is straight forward and
-    doesn't require much knowledge of the internals.
-
-We provide an [architecture.md](./docs/architecture.md) that should give you
-a good overview of the internals.
 
 # Getting help
 
+Your question might already be answered on the [FAQ](https://github.com/helix-editor/helix/wiki/FAQ).
+
 Discuss the project on the community [Matrix Space](https://matrix.to/#/#helix-community:matrix.org) (make sure to join `#helix-editor:matrix.org` if you're on a client that doesn't support Matrix Spaces yet).

TODO.md

@@ -1,51 +0,0 @@
-
-- tree sitter:
-  - lua
-  - markdown
-  - zig
-  - regex
-  - vue
-  - kotlin
-  - julia
-  - clojure
-  - erlang
-
-as you type completion!
-- [ ] use signature_help_provider and completion_provider trigger characters in
-    a hook to trigger signature help text / autocompletion
-- [ ] document.on_type provider triggers
-- [ ] completion isIncomplete support
-
-1
-- [ ] respect view fullscreen flag
-- [ ] Implement marks (superset of Selection/Range)
-
-- [ ] = for auto indent line/selection
-- [ ]  :x for closing buffers
-
-- [ ] repeat selection
-
-- [] jump to alt buffer
-
-- [ ] lsp: signature help
-- [ ] lsp: code actions
-- [ ] lsp: formatting
-
-- [ ] search: smart case by default: insensitive unless upper detected
-
-- [ ] move Compositor into tui
-
-2
-- [ ] macro recording
-- [ ] extend selection (treesitter select parent node) (replaces viw, vi(, va( etc )
-- [ ] selection align
-- [ ] store some state between restarts: file positions, prompt history
-- [ ] highlight matched characters in picker
-
-3
-- [ ] diff mode with highlighting?
-- [ ] snippet support (tab to jump between marks)
-- [ ] gamelisp/wasm scripting
-
-X
-- [ ] rendering via skulpin/skia or raw wgpu

base16_theme.toml

@@ -0,0 +1,51 @@
+# Author: NNB <nnbnh@protonmail.com>
+
+"ui.menu" = "black"
+"ui.menu.selected" = { modifiers = ["reversed"] }
+"ui.linenr" = { fg = "gray", bg = "black" }
+"ui.popup" = { modifiers = ["reversed"] }
+"ui.linenr.selected" = { fg = "white", bg = "black", modifiers = ["bold"] }
+"ui.selection" = { fg = "black", bg = "blue" }
+"ui.selection.primary" = { fg = "white", bg = "blue" }
+"comment" = { fg = "gray" }
+"ui.statusline" = { fg = "black", bg = "white" }
+"ui.statusline.inactive" = { fg = "gray", bg = "white" }
+"ui.help" = { modifiers = ["reversed"] }
+"ui.cursor" = { fg = "white", modifiers = ["reversed"] }
+"variable" = "red"
+"constant.numeric" = "yellow"
+"constant" = "yellow"
+"attributes" = "yellow"
+"type" = "yellow"
+"ui.cursor.match" = { fg = "yellow", modifiers = ["underlined"] }
+"string"  = "green"
+"variable.other.member" = "green"
+"constant.character.escape" = "cyan"
+"function" = "blue"
+"constructor" = "blue"
+"special" = "blue"
+"keyword" = "magenta"
+"label" = "magenta"
+"namespace" = "magenta"
+"ui.help" = { fg = "white", bg = "black" }
+
+"markup.heading" = "blue"
+"markup.list" = "red"
+"markup.bold" = { fg = "yellow", modifiers = ["bold"] }
+"markup.italic" = { fg = "magenta", modifiers = ["italic"] }
+"markup.link.url" = { fg = "yellow", modifiers = ["underlined"] }
+"markup.link.text" = "red"
+"markup.quote" = "cyan"
+"markup.raw" = "green"
+
+"diff.plus" = "green"
+"diff.delta" = "yellow"
+"diff.minus" = "red"
+
+"diagnostic" = { modifiers = ["underlined"] }
+"ui.gutter" = { bg = "black" }
+"info" = "blue"
+"hint" = "gray"
+"debug" = "gray"
+"warning" = "yellow"
+"error" = "red"

book/book.toml

@@ -3,8 +3,9 @@ authors = ["Blaž Hrastnik"]
 language = "en"
 multilingual = false
 src = "src"
-theme = "colibri"
 edit-url-template = "https://github.com/helix-editor/helix/tree/master/book/{path}?mode=edit"
 
 [output.html]
 cname = "docs.helix-editor.com"
+default-theme = "colibri"
+preferred-dark-theme = "colibri"

book/src/SUMMARY.md

@@ -1,9 +1,18 @@
 # Summary
 
+[Helix](./title-page.md)
+
 - [Installation](./install.md)
 - [Usage](./usage.md)
+  - [Keymap](./keymap.md)
+  - [Commands](./commands.md)
+  - [Language Support](./lang-support.md)
+- [Migrating from Vim](./from-vim.md)
 - [Configuration](./configuration.md)
   - [Themes](./themes.md)
-  - [Keymap](./keymap.md)
   - [Key Remapping](./remapping.md)
   - [Hooks](./hooks.md)
+  - [Languages](./languages.md)
+- [Guides](./guides/README.md)
+  - [Adding Languages](./guides/adding_languages.md)
+  - [Adding Textobject Queries](./guides/textobject.md)

book/src/commands.md

@@ -0,0 +1,5 @@
+# Commands
+
+Command mode can be activated by pressing `:`, similar to vim. Built-in commands:
+
+{{#include ./generated/typable-cmd.md}}

book/src/configuration.md

@@ -1,10 +1,137 @@
 # Configuration
 
-To override global configuration parameters create a `config.toml` file located in your config directory (i.e `~/.config/helix/config.toml`).
+To override global configuration parameters, create a `config.toml` file located in your config directory:
+
+* Linux and Mac: `~/.config/helix/config.toml`
+* Windows: `%AppData%\helix\config.toml`
+
+> Note: You may use `hx --edit-config` to create and edit the `config.toml` file.
+
+Example config:
+
+```toml
+theme = "onedark"
+
+[editor]
+line-number = "relative"
+mouse = false
+
+[editor.cursor-shape]
+insert = "bar"
+normal = "block"
+select = "underline"
+
+[editor.file-picker]
+hidden = false
+```
+
+## Editor
+
+### `[editor]` Section
+
+| Key | Description | Default |
+|--|--|---------|
+| `scrolloff` | Number of lines of padding around the edge of the screen when scrolling. | `3` |
+| `mouse` | Enable mouse mode. | `true` |
+| `middle-click-paste` | Middle click paste support. | `true` |
+| `scroll-lines` | Number of lines to scroll per scroll wheel step. | `3` |
+| `shell` | Shell to use when running external commands. | Unix: `["sh", "-c"]`<br/>Windows: `["cmd", "/C"]` |
+| `line-number` | Line number display: `absolute` simply shows each line's number, while `relative` shows the distance from the current line. When unfocused or in insert mode, `relative` will still show absolute line numbers. | `absolute` |
+| `auto-completion` | Enable automatic pop up of auto-completion. | `true` |
+| `idle-timeout` | Time in milliseconds since last keypress before idle timers trigger. Used for autocompletion, set to 0 for instant. | `400` |
+| `completion-trigger-len` | The min-length of word under cursor to trigger autocompletion | `2` |
+| `auto-info` | Whether to display infoboxes | `true` |
+| `true-color` | Set to `true` to override automatic detection of terminal truecolor support in the event of a false negative. | `false` |
+
+### `[editor.cursor-shape]` Section
+
+Defines the shape of cursor in each mode. Note that due to limitations
+of the terminal environment, only the primary cursor can change shape.
+
+| Key      | Description                                | Default |
+| ---      | -----------                                | ------- |
+| `normal` | Cursor shape in [normal mode][normal mode] | `block` |
+| `insert` | Cursor shape in [insert mode][insert mode] | `block` |
+| `select` | Cursor shape in [select mode][select mode] | `block` |
+
+[normal mode]: ./keymap.md#normal-mode
+[insert mode]: ./keymap.md#insert-mode
+[select mode]: ./keymap.md#select--extend-mode
+
+### `[editor.file-picker]` Section
+
+Sets options for file picker and global search. All but the last key listed in
+the default file-picker configuration below are IgnoreOptions: whether hidden
+files and files listed within ignore files are ignored by (not visible in) the
+helix file picker and global search. There is also one other key, `max-depth`
+available, which is not defined by default.
+
+| Key | Description | Default |
+|--|--|---------|
+|`hidden` | Enables ignoring hidden files. | true
+|`parents` | Enables reading ignore files from parent directories. | true
+|`ignore` | Enables reading `.ignore` files. | true
+|`git-ignore` | Enables reading `.gitignore` files. | true
+|`git-global` | Enables reading global .gitignore, whose path is specified in git's config: `core.excludefile` option. | true
+|`git-exclude` | Enables reading `.git/info/exclude` files. | true
+|`max-depth` | Set with an integer value for maximum depth to recurse. | Defaults to `None`.
+
+### `[editor.auto-pairs]` Section
+
+Enable automatic insertion of pairs to parentheses, brackets, etc. Can be
+a simple boolean value, or a specific mapping of pairs of single characters.
+
+| Key | Description |
+| --- | ----------- |
+| `false` | Completely disable auto pairing, regardless of language-specific settings
+| `true` | Use the default pairs: <code>(){}[]''""``</code>
+| Mapping of pairs | e.g. `{ "(" =  ")", "{" = "}", ... }`
+
+Example
+
+```toml
+[editor.auto-pairs]
+'(' = ')'
+'{' = '}'
+'[' = ']'
+'"' = '"'
+'`' = '`'
+'<' = '>'
+```
+
+Additionally, this setting can be used in a language config. Unless
+the editor setting is `false`, this will override the editor config in
+documents with this language.
+
+Example `languages.toml` that adds <> and removes ''
+
+```toml
+[[language]]
+name = "rust"
+
+[language.auto-pairs]
+'(' = ')'
+'{' = '}'
+'[' = ']'
+'"' = '"'
+'`' = '`'
+'<' = '>'
+```
+
+### `[editor.search]` Section
+
+Search specific options.
+
+| Key | Description | Default |
+|--|--|---------|
+| `smart-case` | Enable smart case regex searching (case insensitive unless pattern contains upper case characters) | `true` |
+| `wrap-around`| Whether the search should wrap after depleting the matches | `true` |
+
 
 ## LSP
 
 To display all language server messages in the status line add the following to your `config.toml`:
+
 ```toml
 [lsp]
 display-messages = true

book/src/from-vim.md

@@ -0,0 +1,12 @@
+# Migrating from Vim
+
+Helix's editing model is strongly inspired from vim and kakoune, and a notable
+difference from vim (and the most striking similarity to kakoune) is that Helix
+follows the `selection → action` model. This means that the whatever you are
+going to act on (a word, a paragraph, a line, etc) is selected first and the
+action itself (delete, change, yank, etc) comes second. A cursor is simply a
+single width selection.
+
+See also Kakoune's [Migrating from Vim](https://github.com/mawww/kakoune/wiki/Migrating-from-Vim).
+
+> TODO: Mention texobjects, surround, registers

book/src/generated/lang-support.md

@@ -0,0 +1,66 @@
+| Language | Syntax Highlighting | Treesitter Textobjects | Auto Indent | Default LSP |
+| --- | --- | --- | --- | --- |
+| bash | ✓ |  |  | `bash-language-server` |
+| c | ✓ | ✓ | ✓ | `clangd` |
+| c-sharp | ✓ |  |  | `OmniSharp` |
+| cmake | ✓ | ✓ | ✓ | `cmake-language-server` |
+| comment | ✓ |  |  |  |
+| cpp | ✓ | ✓ | ✓ | `clangd` |
+| css | ✓ |  |  |  |
+| dart | ✓ |  | ✓ | `dart` |
+| dockerfile | ✓ |  |  | `docker-langserver` |
+| elixir | ✓ |  |  | `elixir-ls` |
+| elm | ✓ |  |  | `elm-language-server` |
+| erlang | ✓ |  |  |  |
+| fish | ✓ | ✓ | ✓ |  |
+| git-commit | ✓ |  |  |  |
+| git-config | ✓ |  |  |  |
+| git-diff | ✓ |  |  |  |
+| git-rebase | ✓ |  |  |  |
+| glsl | ✓ |  | ✓ |  |
+| go | ✓ | ✓ | ✓ | `gopls` |
+| graphql | ✓ |  |  |  |
+| haskell | ✓ |  |  | `haskell-language-server-wrapper` |
+| hcl | ✓ |  | ✓ | `terraform-ls` |
+| html | ✓ |  |  |  |
+| iex | ✓ |  |  |  |
+| java | ✓ |  |  |  |
+| javascript | ✓ |  | ✓ | `typescript-language-server` |
+| json | ✓ |  | ✓ |  |
+| julia | ✓ |  |  | `julia` |
+| kotlin | ✓ |  |  | `kotlin-language-server` |
+| latex | ✓ |  |  |  |
+| lean | ✓ |  |  | `lean` |
+| ledger | ✓ |  |  |  |
+| llvm | ✓ | ✓ | ✓ |  |
+| llvm-mir | ✓ | ✓ | ✓ |  |
+| llvm-mir-yaml | ✓ |  | ✓ |  |
+| lua | ✓ |  | ✓ |  |
+| make | ✓ |  |  |  |
+| markdown | ✓ |  |  |  |
+| mint |  |  |  | `mint` |
+| nix | ✓ |  | ✓ | `rnix-lsp` |
+| ocaml | ✓ |  | ✓ |  |
+| ocaml-interface | ✓ |  |  |  |
+| perl | ✓ | ✓ | ✓ |  |
+| php | ✓ | ✓ | ✓ |  |
+| prolog |  |  |  | `swipl` |
+| protobuf | ✓ |  | ✓ |  |
+| python | ✓ | ✓ | ✓ | `pylsp` |
+| racket |  |  |  | `racket` |
+| regex | ✓ |  |  |  |
+| rescript | ✓ | ✓ |  | `rescript-language-server` |
+| ruby | ✓ |  | ✓ | `solargraph` |
+| rust | ✓ | ✓ | ✓ | `rust-analyzer` |
+| scala | ✓ |  | ✓ | `metals` |
+| svelte | ✓ |  | ✓ | `svelteserver` |
+| tablegen | ✓ | ✓ | ✓ |  |
+| toml | ✓ |  |  |  |
+| tsq | ✓ |  |  |  |
+| tsx | ✓ |  |  | `typescript-language-server` |
+| twig | ✓ |  |  |  |
+| typescript | ✓ |  | ✓ | `typescript-language-server` |
+| vue | ✓ |  |  |  |
+| wgsl | ✓ |  |  |  |
+| yaml | ✓ |  | ✓ |  |
+| zig | ✓ |  | ✓ | `zls` |

book/src/generated/typable-cmd.md

@@ -0,0 +1,57 @@
+| Name | Description |
+| --- | --- |
+| `:quit`, `:q` | Close the current view. |
+| `:quit!`, `:q!` | Close the current view forcefully (ignoring unsaved changes). |
+| `:open`, `:o` | Open a file from disk into the current view. |
+| `:buffer-close`, `:bc`, `:bclose` | Close the current buffer. |
+| `:buffer-close!`, `:bc!`, `:bclose!` | Close the current buffer forcefully (ignoring unsaved changes). |
+| `:buffer-close-others`, `:bco`, `:bcloseother` | Close all buffers but the currently focused one. |
+| `:buffer-close-others!`, `:bco!`, `:bcloseother!` | Close all buffers but the currently focused one. |
+| `:buffer-close-all`, `:bca`, `:bcloseall` | Close all buffers, without quiting. |
+| `:buffer-close-all!`, `:bca!`, `:bcloseall!` | Close all buffers forcefully (ignoring unsaved changes), without quiting. |
+| `:write`, `:w` | Write changes to disk. Accepts an optional path (:write some/path.txt) |
+| `:new`, `:n` | Create a new scratch buffer. |
+| `:format`, `:fmt` | Format the file using the LSP formatter. |
+| `:indent-style` | Set the indentation style for editing. ('t' for tabs or 1-8 for number of spaces.) |
+| `:line-ending` | Set the document's default line ending. Options: crlf, lf, cr, ff, nel. |
+| `:earlier`, `:ear` | Jump back to an earlier point in edit history. Accepts a number of steps or a time span. |
+| `:later`, `:lat` | Jump to a later point in edit history. Accepts a number of steps or a time span. |
+| `:write-quit`, `:wq`, `:x` | Write changes to disk and close the current view. Accepts an optional path (:wq some/path.txt) |
+| `:write-quit!`, `:wq!`, `:x!` | Write changes to disk and close the current view forcefully. Accepts an optional path (:wq! some/path.txt) |
+| `:write-all`, `:wa` | Write changes from all views to disk. |
+| `:write-quit-all`, `:wqa`, `:xa` | Write changes from all views to disk and close all views. |
+| `:write-quit-all!`, `:wqa!`, `:xa!` | Write changes from all views to disk and close all views forcefully (ignoring unsaved changes). |
+| `:quit-all`, `:qa` | Close all views. |
+| `:quit-all!`, `:qa!` | Close all views forcefully (ignoring unsaved changes). |
+| `:cquit`, `:cq` | Quit with exit code (default 1). Accepts an optional integer exit code (:cq 2). |
+| `:cquit!`, `:cq!` | Quit with exit code (default 1) forcefully (ignoring unsaved changes). Accepts an optional integer exit code (:cq! 2). |
+| `:theme` | Change the editor theme. |
+| `:clipboard-yank` | Yank main selection into system clipboard. |
+| `:clipboard-yank-join` | Yank joined selections into system clipboard. A separator can be provided as first argument. Default value is newline. |
+| `:primary-clipboard-yank` | Yank main selection into system primary clipboard. |
+| `:primary-clipboard-yank-join` | Yank joined selections into system primary clipboard. A separator can be provided as first argument. Default value is newline. |
+| `:clipboard-paste-after` | Paste system clipboard after selections. |
+| `:clipboard-paste-before` | Paste system clipboard before selections. |
+| `:clipboard-paste-replace` | Replace selections with content of system clipboard. |
+| `:primary-clipboard-paste-after` | Paste primary clipboard after selections. |
+| `:primary-clipboard-paste-before` | Paste primary clipboard before selections. |
+| `:primary-clipboard-paste-replace` | Replace selections with content of system primary clipboard. |
+| `:show-clipboard-provider` | Show clipboard provider name in status bar. |
+| `:change-current-directory`, `:cd` | Change the current working directory. |
+| `:show-directory`, `:pwd` | Show the current working directory. |
+| `:encoding` | Set encoding based on `https://encoding.spec.whatwg.org` |
+| `:reload` | Discard changes and reload from the source file. |
+| `:tree-sitter-scopes` | Display tree sitter scopes, primarily for theming and development. |
+| `:debug-start`, `:dbg` | Start a debug session from a given template with given parameters. |
+| `:debug-remote`, `:dbg-tcp` | Connect to a debug adapter by TCP address and start a debugging session from a given template with given parameters. |
+| `:debug-eval` | Evaluate expression in current debug context. |
+| `:vsplit`, `:vs` | Open the file in a vertical split. |
+| `:vsplit-new`, `:vnew` | Open a scratch buffer in a vertical split. |
+| `:hsplit`, `:hs`, `:sp` | Open the file in a horizontal split. |
+| `:hsplit-new`, `:hnew` | Open a scratch buffer in a horizontal split. |
+| `:tutor` | Open the tutorial. |
+| `:goto`, `:g` | Go to line number. |
+| `:set-option`, `:set` | Set a config option at runtime |
+| `:sort` | Sort ranges in selection. |
+| `:rsort` | Sort ranges in selection in reverse order. |
+| `:tree-sitter-subtree`, `:ts-subtree` | Display tree sitter subtree under cursor, primarily for debugging queries. |

book/src/guides/README.md

@@ -0,0 +1,4 @@
+# Guides
+
+This section contains guides for adding new language server configurations,
+tree-sitter grammers, textobject queries, etc.

book/src/guides/adding_languages.md

@@ -0,0 +1,90 @@
+# Adding languages
+
+## Language configuration
+
+To add a new language, you need to add a `language` entry to the
+[`languages.toml`][languages.toml] found in the root of the repository;
+this `languages.toml` file is included at compilation time, and is
+distinct from the `languages.toml` file in the user's [configuration
+directory](../configuration.md).
+
+```toml
+[[language]]
+name = "mylang"
+scope = "scope.mylang"
+injection-regex = "^mylang$"
+file-types = ["mylang", "myl"]
+comment-token = "#"
+indent = { tab-width = 2, unit = "  " }
+```
+
+These are the available keys and descriptions for the file.
+
+| Key                   | Description                                                   |
+| ----                  | -----------                                                   |
+| `name`                | The name of the language                                      |
+| `scope`               | A string like `source.js` that identifies the language. Currently, we strive to match the scope names used by popular TextMate grammars and by the Linguist library. Usually `source.<name>` or `text.<name>` in case of markup languages |
+| `injection-regex`     | regex pattern that will be tested against a language name in order to determine whether this language should be used for a potential [language injection][treesitter-language-injection] site. |
+| `file-types`          | The filetypes of the language, for example `["yml", "yaml"]`. Extensions and full file names are supported.  |
+| `shebangs`            | The interpreters from the shebang line, for example `["sh", "bash"]` |
+| `roots`               | A set of marker files to look for when trying to find the workspace root. For example `Cargo.lock`, `yarn.lock` |
+| `auto-format`         | Whether to autoformat this language when saving               |
+| `diagnostic-severity` | Minimal severity of diagnostic for it to be displayed. (Allowed values: `Error`, `Warning`, `Info`, `Hint`) |
+| `comment-token`       | The token to use as a comment-token                           |
+| `indent`              | The indent to use. Has sub keys `tab-width` and `unit`        |
+| `config`              | Language server configuration                                 |
+| `grammar`             | The tree-sitter grammar to use (defaults to the value of `name`) |
+
+## Grammar configuration
+
+If a tree-sitter grammar is available for the language, add a new `grammar`
+entry to `languages.toml`.
+
+```toml
+[[grammar]]
+name = "mylang"
+source = { git = "https://github.com/example/mylang", rev = "a250c4582510ff34767ec3b7dcdd3c24e8c8aa68" }
+```
+
+Grammar configuration takes these keys:
+
+| Key      | Description                                                              |
+| ---      | -----------                                                              |
+| `name`   | The name of the tree-sitter grammar                                      |
+| `source` | The method of fetching the grammar - a table with a schema defined below |
+
+Where `source` is a table with either these keys when using a grammar from a
+git repository:
+
+| Key    | Description                                               |
+| ---    | -----------                                               |
+| `git`  | A git remote URL from which the grammar should be cloned  |
+| `rev`  | The revision (commit hash or tag) which should be fetched |
+| `subpath` | A path within the grammar directory which should be built. Some grammar repositories host multiple grammars (for example `tree-sitter-typescript` and `tree-sitter-ocaml`) in subdirectories. This key is used to point `hx --grammar build` to the correct path for compilation. When omitted, the root of repository is used |
+
+Or a `path` key with an absolute path to a locally available grammar directory.
+
+## Queries
+
+For a language to have syntax-highlighting and indentation among
+other things, you have to add queries. Add a directory for your
+language with the path `runtime/queries/<name>/`. The tree-sitter
+[website](https://tree-sitter.github.io/tree-sitter/syntax-highlighting#queries)
+gives more info on how to write queries.
+
+> NOTE: When evaluating queries, the first matching query takes
+precedence, which is different from other editors like neovim where
+the last matching query supersedes the ones before it. See
+[this issue][neovim-query-precedence] for an example.
+
+## Common Issues
+
+- If you get errors when running after switching branches, you may have to update the tree-sitter grammars. Run `hx --grammar fetch` to fetch the grammars and `hx --grammar build` to build any out-of-date grammars.
+
+- If a parser is segfaulting or you want to remove the parser, make sure to remove the compiled parser in `runtime/grammar/<name>.so`
+
+- The indents query is `indents.toml`, *not* `indents.scm`. See [this](https://github.com/helix-editor/helix/issues/114) issue for more information.
+
+[treesitter-language-injection]: https://tree-sitter.github.io/tree-sitter/syntax-highlighting#language-injection
+[languages.toml]: https://github.com/helix-editor/helix/blob/master/languages.toml
+[neovim-query-precedence]: https://github.com/helix-editor/helix/pull/1170#issuecomment-997294090

book/src/guides/textobject.md

@@ -0,0 +1,47 @@
+# Adding Textobject Queries
+
+Textobjects that are language specific ([like functions, classes, etc][textobjects])
+require an accompanying tree-sitter grammar and a `textobjects.scm` query file
+to work properly. Tree-sitter allows us to query the source code syntax tree
+and capture specific parts of it. The queries are written in a lisp dialect.
+More information on how to write queries can be found in the [official tree-sitter
+documentation][tree-sitter-queries].
+
+Query files should be placed in `runtime/queries/{language}/textobjects.scm`
+when contributing. Note that to test the query files locally you should put
+them under your local runtime directory (`~/.config/helix/runtime` on Linux
+for example).
+
+The following [captures][tree-sitter-captures] are recognized:
+
+| Capture Name       |
+| ---                |
+| `function.inside`  |
+| `function.around`  |
+| `class.inside`     |
+| `class.around`     |
+| `parameter.inside` |
+| `comment.inside`   |
+| `comment.around`   |
+
+[Example query files][textobject-examples] can be found in the helix GitHub repository.
+
+## Queries for Textobject Based Navigation
+
+[Tree-sitter based navigation][textobjects-nav] is done using captures in the
+following order:
+
+- `object.movement`
+- `object.around`
+- `object.inside`
+
+For example if a `function.around` capture has been already defined for a language
+in it's `textobjects.scm` file, function navigation should also work automatically.
+`function.movement` should be defined only if the node captured by `function.around`
+doesn't make sense in a navigation context.
+
+[textobjects]: ../usage.md#textobjects
+[textobjects-nav]: ../usage.md#tree-sitter-textobject-based-navigation
+[tree-sitter-queries]: https://tree-sitter.github.io/tree-sitter/using-parsers#query-syntax
+[tree-sitter-captures]: https://tree-sitter.github.io/tree-sitter/using-parsers#capturing-nodes
+[textobject-examples]: https://github.com/search?q=repo%3Ahelix-editor%2Fhelix+filename%3Atextobjects.scm&type=Code&ref=advsearch&l=&l=

book/src/install.md

@@ -19,18 +19,32 @@ brew install helix
 
 A [flake](https://nixos.wiki/wiki/Flakes) containing the package is available in
 the project root. The flake can also be used to spin up a reproducible development
-shell for working on Helix.
+shell for working on Helix with `nix develop`.
+
+Flake outputs are cached for each push to master using
+[Cachix](https://www.cachix.org/). With Cachix
+[installed](https://docs.cachix.org/installation), `cachix use helix` will
+configure Nix to use cached outputs when possible.
 
 ### Arch Linux
 
-Binary packages are available on AUR:
+Releases are available in the `community` repository.
-- [helix-bin](https://aur.archlinux.org/packages/helix-bin/) contains the pre-built release
+
-- [helix-git](https://aur.archlinux.org/packages/helix-git/) builds the master branch
+A [helix-git](https://aur.archlinux.org/packages/helix-git/) package is also available on the AUR, which builds the master branch.
+
+### Fedora Linux
+
+You can install the COPR package for Helix via
+
+```
+sudo dnf copr enable varlad/helix
+sudo dnf install helix
+```
 
 ## Build from source
 
 ```
-git clone --recurse-submodules --shallow-submodules -j8 https://github.com/helix-editor/helix
+git clone https://github.com/helix-editor/helix
 cd helix
 cargo install --path helix-term
 ```
@@ -40,3 +54,9 @@ This will install the `hx` binary to `$HOME/.cargo/bin`.
 Helix also needs it's runtime files so make sure to copy/symlink the `runtime/` directory into the
 config directory (for example `~/.config/helix/runtime` on Linux/macOS). This location can be overriden
 via the `HELIX_RUNTIME` environment variable.
+
+## Building tree-sitter grammars
+
+Tree-sitter grammars must be fetched and compiled if not pre-packaged.
+Fetch grammars with `hx --grammar fetch` (requires `git`) and compile them
+with `hx --grammar build` (requires a C compiler).

book/src/keymap.md

@@ -1,229 +1,371 @@
 # Keymap
 
+- Mappings marked (**LSP**) require an active language server for the file.
+- Mappings marked (**TS**) require a tree-sitter grammar for the filetype.
+
 ## Normal mode
 
 ### Movement
 
-> NOTE: `f`, `F`, `t` and `T` are not confined to the current line.
+> NOTE: Unlike vim, `f`, `F`, `t` and `T` are not confined to the current line.
 
-| Key          | Description                                                                |
+| Key                   | Description                                        | Command                     |
-| -----        | -----------                                                                |
+| -----                 | -----------                                        | -------                     |
-| `h`, `Left`  | Move left                                                                  |
+| `h`, `Left`           | Move left                                          | `move_char_left`            |
-| `j`, `Down`  | Move down                                                                  |
+| `j`, `Down`           | Move down                                          | `move_line_down`            |
-| `k`, `Up`    | Move up                                                                    |
+| `k`, `Up`             | Move up                                            | `move_line_up`              |
-| `l`, `Right` | Move right                                                                 |
+| `l`, `Right`          | Move right                                         | `move_char_right`           |
-| `w`          | Move next word start                                                       |
+| `w`                   | Move next word start                               | `move_next_word_start`      |
-| `b`          | Move previous word start                                                   |
+| `b`                   | Move previous word start                           | `move_prev_word_start`      |
-| `e`          | Move next word end                                                         |
+| `e`                   | Move next word end                                 | `move_next_word_end`        |
-| `W`          | Move next WORD start                                                       |
+| `W`                   | Move next WORD start                               | `move_next_long_word_start` |
-| `B`          | Move previous WORD start                                                   |
+| `B`                   | Move previous WORD start                           | `move_prev_long_word_start` |
-| `E`          | Move next WORD end                                                         |
+| `E`                   | Move next WORD end                                 | `move_next_long_word_end`   |
-| `t`          | Find 'till next char                                                       |
+| `t`                   | Find 'till next char                               | `find_till_char`            |
-| `f`          | Find next char                                                             |
+| `f`                   | Find next char                                     | `find_next_char`            |
-| `T`          | Find 'till previous char                                                   |
+| `T`                   | Find 'till previous char                           | `till_prev_char`            |
-| `F`          | Find previous char                                                         |
+| `F`                   | Find previous char                                 | `find_prev_char`            |
-| `Home`       | Move to the start of the line                                              |
+| `G`                   | Go to line number `<n>`                            | `goto_line`                 |
-| `End`        | Move to the end of the line                                                |
+| `Alt-.`               | Repeat last motion (`f`, `t` or `m`)               | `repeat_last_motion`        |
-| `PageUp`     | Move page up                                                               |
+| `Home`                | Move to the start of the line                      | `goto_line_start`           |
-| `PageDown`   | Move page down                                                             |
+| `End`                 | Move to the end of the line                        | `goto_line_end`             |
-| `Ctrl-u`     | Move half page up                                                          |
+| `Ctrl-b`, `PageUp`    | Move page up                                       | `page_up`                   |
-| `Ctrl-d`     | Move half page down                                                        |
+| `Ctrl-f`, `PageDown`  | Move page down                                     | `page_down`                 |
-| `Ctrl-i`     | Jump forward on the jumplist TODO: conflicts tab                           |
+| `Ctrl-u`              | Move half page up                                  | `half_page_up`              |
-| `Ctrl-o`     | Jump backward on the jumplist                                              |
+| `Ctrl-d`              | Move half page down                                | `half_page_down`            |
-| `v`          | Enter [select (extend) mode](#select--extend-mode)                         |
+| `Ctrl-i`              | Jump forward on the jumplist                       | `jump_forward`              |
-| `g`          | Enter [goto mode](#goto-mode)                                              |
+| `Ctrl-o`              | Jump backward on the jumplist                      | `jump_backward`             |
-| `m`          | Enter [match mode](#match-mode)                                            |
+| `Ctrl-s`              | Save the current selection to the jumplist         | `save_selection`            |
-| `:`          | Enter command mode                                                         |
-| `z`          | Enter [view mode](#view-mode)                                              |
-| `Ctrl-w`     | Enter [window mode](#window-mode) (maybe will be remove for spc w w later) |
-| `Space`      | Enter [space mode](#space-mode)                                            |
-| `K`          | Show documentation for the item under the cursor                           |
 
 ### Changes
 
-| Key         | Description                                     |
+| Key         | Description                                                          | Command                   |
-| -----       | -----------                                     |
+| -----       | -----------                                                          | -------                   |
-| `r`         | Replace with a character                        |
+| `r`         | Replace with a character                                             | `replace`                 |
-| `R`         | Replace with yanked text                        |
+| `R`         | Replace with yanked text                                             | `replace_with_yanked`     |
-| `~`         | Switch case of the selected text                |
+| `~`         | Switch case of the selected text                                     | `switch_case`             |
-| `` ` ``     | Set the selected text to lower case             |
+| `` ` ``     | Set the selected text to lower case                                  | `switch_to_lowercase`     |
-| `` Alt-` `` | Set the selected text to upper case             |
+| `` Alt-` `` | Set the selected text to upper case                                  | `switch_to_uppercase`     |
-| `i`         | Insert before selection                         |
+| `i`         | Insert before selection                                              | `insert_mode`             |
-| `a`         | Insert after selection (append)                 |
+| `a`         | Insert after selection (append)                                      | `append_mode`             |
-| `I`         | Insert at the start of the line                 |
+| `I`         | Insert at the start of the line                                      | `prepend_to_line`         |
-| `A`         | Insert at the end of the line                   |
+| `A`         | Insert at the end of the line                                        | `append_to_line`          |
-| `o`         | Open new line below selection                   |
+| `o`         | Open new line below selection                                        | `open_below`              |
-| `O`         | Open new line above selection                   |
+| `O`         | Open new line above selection                                        | `open_above`              |
-| `u`         | Undo change                                     |
+| `.`         | Repeat last change                                                   | N/A                       |
-| `U`         | Redo change                                     |
+| `u`         | Undo change                                                          | `undo`                    |
-| `y`         | Yank selection                                  |
+| `U`         | Redo change                                                          | `redo`                    |
-| `p`         | Paste after selection                           |
+| `Alt-u`     | Move backward in history                                             | `earlier`                 |
-| `P`         | Paste before selection                          |
+| `Alt-U`     | Move forward in history                                              | `later`                   |
-| `>`         | Indent selection                                |
+| `y`         | Yank selection                                                       | `yank`                    |
-| `<`         | Unindent selection                              |
+| `p`         | Paste after selection                                                | `paste_after`             |
-| `=`         | Format selection                                |
+| `P`         | Paste before selection                                               | `paste_before`            |
-| `d`         | Delete selection                                |
+| `"` `<reg>` | Select a register to yank to or paste from                           | `select_register`         |
-| `c`         | Change selection (delete and enter insert mode) |
+| `>`         | Indent selection                                                     | `indent`                  |
+| `<`         | Unindent selection                                                   | `unindent`                |
+| `=`         | Format selection (currently nonfunctional/disabled) (**LSP**)        | `format_selections`       |
+| `d`         | Delete selection                                                     | `delete_selection`        |
+| `Alt-d`     | Delete selection, without yanking                                    | `delete_selection_noyank` |
+| `c`         | Change selection (delete and enter insert mode)                      | `change_selection`        |
+| `Alt-c`     | Change selection (delete and enter insert mode, without yanking)     | `change_selection_noyank` |
+| `Ctrl-a`    | Increment object (number) under cursor                               | `increment`               |
+| `Ctrl-x`    | Decrement object (number) under cursor                               | `decrement`               |
+| `Q`         | Start/stop macro recording to the selected register (experimental)   | `record_macro`            |
+| `q`         | Play back a recorded macro from the selected register (experimental) | `replay_macro`            |
+
+#### Shell
+
+| Key     | Description                                                                      | Command               |
+| ------  | -----------                                                                      | -------               |
+| <code>&#124;</code>     | Pipe each selection through shell command, replacing with output                 | `shell_pipe`          |
+| <code>Alt-&#124;</code> | Pipe each selection into shell command, ignoring output                          | `shell_pipe_to`       |
+| `!`     | Run shell command, inserting output before each selection                        | `shell_insert_output` |
+| `Alt-!` | Run shell command, appending output after each selection                         | `shell_append_output` |
+| `$`     | Pipe each selection into shell command, keep selections where command returned 0 | `shell_keep_pipe`     |
+
 
 ### Selection manipulation
 
-| Key      | Description                                                       |
+| Key                   | Description                                                       | Command                              |
-| -----    | -----------                                                       |
+| -----                 | -----------                                                       | -------                              |
-| `s`      | Select all regex matches inside selections                        |
+| `s`                   | Select all regex matches inside selections                        | `select_regex`                       |
-| `S`      | Split selection into subselections on regex matches               |
+| `S`                   | Split selection into subselections on regex matches               | `split_selection`                    |
-| `Alt-s`  | Split selection on newlines                                       |
+| `Alt-s`               | Split selection on newlines                                       | `split_selection_on_newline`         |
-| `;`      | Collapse selection onto a single cursor                           |
+| `&`                   | Align selection in columns                                        | `align_selections`                   |
-| `Alt-;`  | Flip selection cursor and anchor                                  |
+| `_`                   | Trim whitespace from the selection                                | `trim_selections`                    |
-| `C`      | Copy selection onto the next line                                 |
+| `;`                   | Collapse selection onto a single cursor                           | `collapse_selection`                 |
-| `Alt-C`  | Copy selection onto the previous line                             |
+| `Alt-;`               | Flip selection cursor and anchor                                  | `flip_selections`                    |
-| `(`      | Rotate main selection forward                                     |
+| `Alt-:`               | Ensures the selection is in forward direction                     | `ensure_selections_forward`          |
-| `)`      | Rotate main selection backward                                    |
+| `,`                   | Keep only the primary selection                                   | `keep_primary_selection`             |
-| `Alt-(`  | Rotate selection contents forward                                 |
+| `Alt-,`               | Remove the primary selection                                      | `remove_primary_selection`           |
-| `Alt-)`  | Rotate selection contents backward                                |
+| `C`                   | Copy selection onto the next line (Add cursor below)              | `copy_selection_on_next_line`        |
-| `%`      | Select entire file                                                |
+| `Alt-C`               | Copy selection onto the previous line (Add cursor above)          | `copy_selection_on_prev_line`        |
-| `x`      | Select current line, if already selected, extend to next line     |
+| `(`                   | Rotate main selection backward                                    | `rotate_selections_backward`         |
-| `X`      | Extend selection to line bounds (line-wise selection)             |
+| `)`                   | Rotate main selection forward                                     | `rotate_selections_forward`          |
-|          | Expand selection to parent syntax node TODO: pick a key           |
+| `Alt-(`               | Rotate selection contents backward                                | `rotate_selection_contents_backward` |
-| `J`      | Join lines inside selection                                       |
+| `Alt-)`               | Rotate selection contents forward                                 | `rotate_selection_contents_forward`  |
-| `K`      | Keep selections matching the regex TODO: overlapped by hover help |
+| `%`                   | Select entire file                                                | `select_all`                         |
-| `Space`  | Keep only the primary selection TODO: overlapped by space mode    |
+| `x`                   | Select current line, if already selected, extend to next line     | `extend_line`                        |
-| `Ctrl-c` | Comment/uncomment the selections                                  |
+| `X`                   | Extend selection to line bounds (line-wise selection)             | `extend_to_line_bounds`              |
-
+| `J`                   | Join lines inside selection                                       | `join_selections`                    |
-### Insert Mode
+| `K`                   | Keep selections matching the regex                                | `keep_selections`                    |
-
+| `Alt-K`               | Remove selections matching the regex                              | `remove_selections`                  |
-| Key      | Description           |
+| `Ctrl-c`              | Comment/uncomment the selections                                  | `toggle_comments`                    |
-| -----    | -----------           |
+| `Alt-k`, `Alt-up`     | Expand selection to parent syntax node (**TS**)                   | `expand_selection`                   |
-| `Escape` | Switch to normal mode |
+| `Alt-j`, `Alt-down`   | Shrink syntax tree object selection (**TS**)                      | `shrink_selection`                   |
-| `Ctrl-x` | Autocomplete          |
+| `Alt-h`, `Alt-left`   | Select previous sibling node in syntax tree (**TS**)              | `select_prev_sibling`                |
-| `Ctrl-w` | Delete previous word  |
+| `Alt-l`, `Alt-right`  | Select next sibling node in syntax tree (**TS**)                  | `select_next_sibling`                |
 
 ### Search
 
-> TODO: The search implementation isn't ideal yet -- we don't support searching
+Search commands all operate on the `/` register by default. Use `"<char>` to operate on a different one.
-in reverse, or searching via smartcase.
 
-| Key   | Description                                 |
+| Key   | Description                                 | Command              |
-| ----- | -----------                                 |
+| ----- | -----------                                 | -------              |
-| `/`   | Search for regex pattern                    |
+| `/`   | Search for regex pattern                    | `search`             |
-| `n`   | Select next search match                    |
+| `?`   | Search for previous pattern                 | `rsearch`            |
-| `N`   | Add next search match to selection          |
+| `n`   | Select next search match                    | `search_next`        |
-| `*`   | Use current selection as the search pattern |
+| `N`   | Select previous search match                | `search_prev`        |
+| `*`   | Use current selection as the search pattern | `search_selection`   |
 
-### Diagnostics
+### Minor modes
 
-> NOTE: `[` and `]` will likely contain more pair mappings in the style of
+These sub-modes are accessible from normal mode and typically switch back to normal mode after a command.
-> [vim-unimpaired](https://github.com/tpope/vim-unimpaired)
 
-| Key   | Description                        |
+| Key      | Description                                        | Command        |
-| ----- | -----------                        |
+| -----    | -----------                                        | -------        |
-| `[d`  | Go to previous diagnostic          |
+| `v`      | Enter [select (extend) mode](#select--extend-mode) | `select_mode`  |
-| `]d`  | Go to next diagnostic              |
+| `g`      | Enter [goto mode](#goto-mode)                      | N/A            |
-| `[D`  | Go to first diagnostic in document |
+| `m`      | Enter [match mode](#match-mode)                    | N/A            |
-| `]D`  | Go to last diagnostic in document  |
+| `:`      | Enter command mode                                 | `command_mode` |
+| `z`      | Enter [view mode](#view-mode)                      | N/A            |
+| `Z`      | Enter sticky [view mode](#view-mode)               | N/A            |
+| `Ctrl-w` | Enter [window mode](#window-mode)                  | N/A            |
+| `Space`  | Enter [space mode](#space-mode)                    | N/A            |
 
-## Select / extend mode
+#### View mode
-
-I'm still pondering whether to keep this mode or not. It changes movement
-commands to extend the existing selection instead of replacing it.
-
-> NOTE: It's a bit confusing at the moment because extend hasn't been
-> implemented for all movement commands yet.
-
-## View mode
 
 View mode is intended for scrolling and manipulating the view without changing
-the selection.
+the selection. The "sticky" variant of this mode is persistent; use the Escape
+key to return to normal mode after usage (useful when you're simply looking
+over text and not actively editing it).
 
-| Key       | Description                                               |
-| -----     | -----------                                               |
-| `z` , `c` | Vertically center the line                                |
-| `t`       | Align the line to the top of the screen                   |
-| `b`       | Align the line to the bottom of the screen                |
-| `m`       | Align the line to the middle of the screen (horizontally) |
-| `j`       | Scroll the view downwards                                 |
-| `k`       | Scroll the view upwards                                   |
 
-## Goto mode
+| Key                  | Description                                               | Command             |
+| -----                | -----------                                               | -------             |
+| `z`, `c`             | Vertically center the line                                | `align_view_center` |
+| `t`                  | Align the line to the top of the screen                   | `align_view_top`    |
+| `b`                  | Align the line to the bottom of the screen                | `align_view_bottom` |
+| `m`                  | Align the line to the middle of the screen (horizontally) | `align_view_middle` |
+| `j`, `down`          | Scroll the view downwards                                 | `scroll_down`       |
+| `k`, `up`            | Scroll the view upwards                                   | `scroll_up`         |
+| `Ctrl-f`, `PageDown` | Move page down                                            | `page_down`         |
+| `Ctrl-b`, `PageUp`   | Move page up                                              | `page_up`           |
+| `Ctrl-d`             | Move half page down                                       | `half_page_down`    |
+| `Ctrl-u`             | Move half page up                                         | `half_page_up`      |
+
+#### Goto mode
 
 Jumps to various locations.
 
-> NOTE: Some of these features are only available with the LSP present.
+| Key   | Description                                      | Command                    |
+| ----- | -----------                                      | -------                    |
+| `g`   | Go to line number `<n>` else start of file       | `goto_file_start`          |
+| `e`   | Go to the end of the file                        | `goto_last_line`           |
+| `f`   | Go to files in the selection                     | `goto_file`                |
+| `h`   | Go to the start of the line                      | `goto_line_start`          |
+| `l`   | Go to the end of the line                        | `goto_line_end`            |
+| `s`   | Go to first non-whitespace character of the line | `goto_first_nonwhitespace` |
+| `t`   | Go to the top of the screen                      | `goto_window_top`          |
+| `c`   | Go to the middle of the screen                   | `goto_window_center`       |
+| `b`   | Go to the bottom of the screen                   | `goto_window_bottom`       |
+| `d`   | Go to definition (**LSP**)                       | `goto_definition`          |
+| `y`   | Go to type definition (**LSP**)                  | `goto_type_definition`     |
+| `r`   | Go to references (**LSP**)                       | `goto_reference`           |
+| `i`   | Go to implementation (**LSP**)                   | `goto_implementation`      |
+| `a`   | Go to the last accessed/alternate file           | `goto_last_accessed_file`  |
+| `m`   | Go to the last modified/alternate file           | `goto_last_modified_file`  |
+| `n`   | Go to next buffer                                | `goto_next_buffer`         |
+| `p`   | Go to previous buffer                            | `goto_previous_buffer`     |
+| `.`   | Go to last modification in current file          | `goto_last_modification`   |
 
-| Key   | Description                                      |
+#### Match mode
-| ----- | -----------                                      |
-| `g`   | Go to the start of the file                      |
-| `e`   | Go to the end of the file                        |
-| `h`   | Go to the start of the line                      |
-| `l`   | Go to the end of the line                        |
-| `s`   | Go to first non-whitespace character of the line |
-| `t`   | Go to the top of the screen                      |
-| `m`   | Go to the middle of the screen                   |
-| `b`   | Go to the bottom of the screen                   |
-| `d`   | Go to definition                                 |
-| `y`   | Go to type definition                            |
-| `r`   | Go to references                                 |
-| `i`   | Go to implementation                             |
-| `a`   | Go to the last accessed/alternate file           |
-
-## Match mode
 
 Enter this mode using `m` from normal mode. See the relavant section
 in [Usage](./usage.md) for an explanation about [surround](./usage.md#surround)
 and [textobject](./usage.md#textobject) usage.
 
-| Key              | Description                                     |
+| Key              | Description                                     | Command                    |
-| -----            | -----------                                     |
+| -----            | -----------                                     | -------                    |
-| `m`              | Goto matching bracket                           |
+| `m`              | Goto matching bracket (**TS**)                  | `match_brackets`           |
-| `s` `<char>`     | Surround current selection with `<char>`        |
+| `s` `<char>`     | Surround current selection with `<char>`        | `surround_add`             |
-| `r` `<from><to>` | Replace surround character `<from>` with `<to>` |
+| `r` `<from><to>` | Replace surround character `<from>` with `<to>` | `surround_replace`         |
-| `d` `<char>`     | Delete surround character `<char>`              |
+| `d` `<char>`     | Delete surround character `<char>`              | `surround_delete`          |
-| `a` `<object>`   | Select around textobject                        |
+| `a` `<object>`   | Select around textobject                        | `select_textobject_around` |
-| `i` `<object>`   | Select inside textobject                        |
+| `i` `<object>`   | Select inside textobject                        | `select_textobject_inner`  |
-
-## Object mode
 
 TODO: Mappings for selecting syntax nodes (a superset of `[`).
 
-## Window mode
+#### Window mode
 
 This layer is similar to vim keybindings as kakoune does not support window.
 
-| Key           | Description             |
+| Key                    | Description                                          | Command           |
-| -----         | -------------           |
+| -----                  | -------------                                        | -------           |
-| `w`, `Ctrl-w` | Switch to next window   |
+| `w`, `Ctrl-w`          | Switch to next window                                | `rotate_view`     |
-| `v`, `Ctrl-v` | Vertical right split    |
+| `v`, `Ctrl-v`          | Vertical right split                                 | `vsplit`          |
-| `h`, `Ctrl-h` | Horizontal bottom split |
+| `s`, `Ctrl-s`          | Horizontal bottom split                              | `hsplit`          |
-| `q`, `Ctrl-q` | Close current window    |
+| `f`                    | Go to files in the selection in horizontal splits    | `goto_file`       |
+| `F`                    | Go to files in the selection in vertical splits      | `goto_file`       |
+| `h`, `Ctrl-h`, `Left`  | Move to left split                                   | `jump_view_left`  |
+| `j`, `Ctrl-j`, `Down`  | Move to split below                                  | `jump_view_down`  |
+| `k`, `Ctrl-k`, `Up`    | Move to split above                                  | `jump_view_up`    |
+| `l`, `Ctrl-l`, `Right` | Move to right split                                  | `jump_view_right` |
+| `q`, `Ctrl-q`          | Close current window                                 | `wclose`          |
+| `o`, `Ctrl-o`          | Only keep the current window, closing all the others | `wonly`           |
 
-## Space mode
+#### Space mode
 
-This layer is a kludge of mappings I had under leader key in neovim.
+This layer is a kludge of mappings, mostly pickers.
+
+
+| Key     | Description                                                             | Command                             |
+| -----   | -----------                                                             | -------                             |
+| `f`     | Open file picker                                                        | `file_picker`                       |
+| `b`     | Open buffer picker                                                      | `buffer_picker`                     |
+| `k`     | Show documentation for item under cursor in a [popup](#popup) (**LSP**) | `hover`                             |
+| `s`     | Open document symbol picker (**LSP**)                                   | `symbol_picker`                     |
+| `S`     | Open workspace symbol picker (**LSP**)                                  | `workspace_symbol_picker`           |
+| `r`     | Rename symbol (**LSP**)                                                 | `rename_symbol`                     |
+| `a`     | Apply code action  (**LSP**)                                            | `code_action`                       |
+| `'`     | Open last fuzzy picker                                                  | `last_picker`                       |
+| `w`     | Enter [window mode](#window-mode)                                       | N/A                                 |
+| `p`     | Paste system clipboard after selections                                 | `paste_clipboard_after`             |
+| `P`     | Paste system clipboard before selections                                | `paste_clipboard_before`            |
+| `y`     | Join and yank selections to clipboard                                   | `yank_joined_to_clipboard`          |
+| `Y`     | Yank main selection to clipboard                                        | `yank_main_selection_to_clipboard`  |
+| `R`     | Replace selections by clipboard contents                                | `replace_selections_with_clipboard` |
+| `/`     | Global search in workspace folder                                       | `global_search`                     |
+| `?`     | Open command palette                                                    | `command_palette`                   |
+
+> TIP: Global search displays results in a fuzzy picker, use `space + '` to bring it back up after opening a file.
+
+##### Popup
+
+Displays documentation for item under cursor.
 
 | Key      | Description |
-| -----   | -----------                                                           |
+| ----     | ----------- |
-| `f`     | Open file picker                                                      |
+| `Ctrl-u` | Scroll up   |
-| `b`     | Open buffer picker                                                    |
+| `Ctrl-d` | Scroll down |
-| `s`     | Open symbol picker (current document)                                 |
+ 
-| `a`     | Apply code action                                                     |
+#### Unimpaired
-| `'`     | Open last fuzzy picker                                                |
+
-| `w`     | Enter [window mode](#window-mode)                                     |
+Mappings in the style of [vim-unimpaired](https://github.com/tpope/vim-unimpaired).
-| `space` | Keep primary selection TODO: it's here because space mode replaced it |
+
-| `p`     | Paste system clipboard after selections                               |
+| Key      | Description                                  | Command               |
-| `P`     | Paste system clipboard before selections                              |
+| -----    | -----------                                  | -------               |
-| `y`     | Join and yank selections to clipboard                                 |
+| `[d`     | Go to previous diagnostic (**LSP**)          | `goto_prev_diag`      |
-| `Y`     | Yank main selection to clipboard                                      |
+| `]d`     | Go to next diagnostic (**LSP**)              | `goto_next_diag`      |
-| `R`     | Replace selections by clipboard contents                              |
+| `[D`     | Go to first diagnostic in document (**LSP**) | `goto_first_diag`     |
+| `]D`     | Go to last diagnostic in document (**LSP**)  | `goto_last_diag`      |
+| `]f`     | Go to next function (**TS**)                 | `goto_next_function`  |
+| `[f`     | Go to previous function (**TS**)             | `goto_prev_function`  |
+| `]c`     | Go to next class (**TS**)                    | `goto_next_class`     |
+| `[c`     | Go to previous class (**TS**)                | `goto_prev_class`     |
+| `]a`     | Go to next argument/parameter (**TS**)       | `goto_next_parameter` |
+| `[a`     | Go to previous argument/parameter (**TS**)   | `goto_prev_parameter` |
+| `]o`     | Go to next comment (**TS**)                  | `goto_next_comment`   |
+| `[o`     | Go to previous comment (**TS**)              | `goto_prev_comment`   |
+| `[space` | Add newline above                            | `add_newline_above`   |
+| `]space` | Add newline below                            | `add_newline_below`   |
+
+## Insert Mode
+
+We support many readline/emacs style bindings in insert mode for
+convenience. These can be helpful for making simple modifications
+without escaping to normal mode, but beware that you will not have an
+undo-able "save point" until you return to normal mode.
+
+| Key                       | Description                 | Command                 |
+| -----                     | -----------                 | -------                 |
+| `Escape`                  | Switch to normal mode       | `normal_mode`           |
+| `Ctrl-x`                  | Autocomplete                | `completion`            |
+| `Ctrl-r`                  | Insert a register content   | `insert_register`       |
+| `Ctrl-w`, `Alt-Backspace` | Delete previous word        | `delete_word_backward`  |
+| `Alt-d`                   | Delete next word            | `delete_word_forward`   |
+| `Alt-b`, `Alt-Left`       | Backward a word             | `move_prev_word_end`    |
+| `Ctrl-b`, `Left`          | Backward a char             | `move_char_left`        |
+| `Alt-f`, `Alt-Right`      | Forward a word              | `move_next_word_start`  |
+| `Ctrl-f`, `Right`         | Forward a char              | `move_char_right`       |
+| `Ctrl-e`, `End`           | Move to line end            | `goto_line_end_newline` |
+| `Ctrl-a`, `Home`          | Move to line start          | `goto_line_start`       |
+| `Ctrl-u`                  | Delete to start of line     | `kill_to_line_start`    |
+| `Ctrl-k`                  | Delete to end of line       | `kill_to_line_end`      |
+| `Ctrl-j`, `Enter`         | Insert new line             | `insert_newline`        |
+| `Backspace`, `Ctrl-h`     | Delete previous char        | `delete_char_backward`  |
+| `Delete`, `Ctrl-d`        | Delete previous char        | `delete_char_forward`   |
+| `Ctrl-p`, `Up`            | Move to previous line       | `move_line_up`          |
+| `Ctrl-n`, `Down`          | Move to next line           | `move_line_down`        |
+| `PageUp`                  | Move one page up            | `page_up`               |
+| `PageDown`                | Move one page down          | `page_down`             |
+| `Alt->`                   | Go to end of buffer         | `goto_file_end`         |
+| `Alt-<`                   | Go to start of buffer       | `goto_file_start`       |
+
+## Select / extend mode
+
+This mode echoes Normal mode, but changes any movements to extend
+selections rather than replace them. Goto motions are also changed to
+extend, so that `vgl` for example extends the selection to the end of
+the line.
+
+Search is also affected. By default, `n` and `N` will remove the current
+selection and select the next instance of the search term. Toggling this
+mode before pressing `n` or `N` makes it possible to keep the current
+selection. Toggling it on and off during your iterative searching allows
+you to selectively add search terms to your selections.
 
 # Picker
 
-Keys to use within picker.
+Keys to use within picker. Remapping currently not supported.
 
 | Key                          | Description       |
 | -----                        | -------------     |
-| `Up`, `Ctrl-p`     | Previous entry    |
+| `Up`, `Ctrl-k`, `Ctrl-p`     | Previous entry    |
-| `Down`, `Ctrl-n`   | Next entry        |
+| `PageUp`, `Ctrl-b`           | Page up           |
+| `Down`, `Ctrl-j`, `Ctrl-n`   | Next entry        |
+| `PageDown`, `Ctrl-f`         | Page down         |
+| `Home`                       | Go to first entry |
+| `End`                        | Go to last entry  |
 | `Ctrl-space`                 | Filter options    |
 | `Enter`                      | Open selected     |
-| `Ctrl-h`           | Open horizontally |
+| `Ctrl-s`                     | Open horizontally |
 | `Ctrl-v`                     | Open vertically   |
 | `Escape`, `Ctrl-c`           | Close picker      |
+
+# Prompt
+
+Keys to use within prompt, Remapping currently not supported.
+
+| Key                     | Description                                                             |
+| -----                   | -------------                                                           |
+| `Escape`, `Ctrl-c`      | Close prompt                                                            |
+| `Alt-b`, `Alt-Left`     | Backward a word                                                         |
+| `Ctrl-b`, `Left`        | Backward a char                                                         |
+| `Alt-f`, `Alt-Right`    | Forward a word                                                          |
+| `Ctrl-f`, `Right`       | Forward a char                                                          |
+| `Ctrl-e`, `End`         | Move prompt end                                                         |
+| `Ctrl-a`, `Home`        | Move prompt start                                                       |
+| `Ctrl-w`                | Delete previous word                                                    |
+| `Alt-d`                 | Delete next word                                                        |
+| `Ctrl-u`                | Delete to start of line                                                 |
+| `Ctrl-k`                | Delete to end of line                                                   |
+| `backspace`, `Ctrl-h`   | Delete previous char                                                    |
+| `delete`, `Ctrl-d`      | Delete next char                                                        |
+| `Ctrl-s`                | Insert a word under doc cursor, may be changed to Ctrl-r Ctrl-w later   |
+| `Ctrl-p`, `Up`          | Select previous history                                                 |
+| `Ctrl-n`, `Down`        | Select next history                                                     |
+| `Tab`                   | Select next completion item                                             |
+| `BackTab`               | Select previous completion item                                         |
+| `Enter`                 | Open selected                                                           |
+

book/src/lang-support.md

@@ -0,0 +1,10 @@
+# Language Support
+
+For more information like arguments passed to default LSP server,
+extensions assosciated with a filetype, custom LSP settings, filetype
+specific indent settings, etc see the default
+[`languages.toml`][languages.toml] file.
+
+{{#include ./generated/lang-support.md}}
+
+[languages.toml]: https://github.com/helix-editor/helix/blob/master/languages.toml

book/src/languages.md

@@ -0,0 +1,40 @@
+# Languages
+
+Language-specific settings and settings for particular language servers can be configured in a `languages.toml` file placed in your [configuration directory](./configuration.md). Helix actually uses two `languages.toml` files, the [first one](https://github.com/helix-editor/helix/blob/master/languages.toml) is in the main helix repository; it contains the default settings for each language and is included in the helix binary at compile time. Users who want to see the available settings and options can either reference the helix repo's `languages.toml` file, or consult the table in the [adding languages](./guides/adding_languages.md) section.
+
+Changes made to the `languages.toml` file in a user's [configuration directory](./configuration.md) are merged with helix's defaults on start-up, such that a user's settings will take precedence over defaults in the event of a collision. For example, the default `languages.toml` sets rust's `auto-format` to `true`. If a user wants to disable auto-format, they can change the `languages.toml` in their [configuration directory](./configuration.md) to make the rust entry read like the example below; the new key/value pair `auto-format = false` will override the default when the two sets of settings are merged on start-up:
+
+```toml
+# in <config_dir>/helix/languages.toml
+
+[[language]]
+name = "rust"
+auto-format = false
+```
+
+## Tree-sitter grammars
+
+Tree-sitter grammars can also be configured in `languages.toml`:
+
+```toml
+# in <config_dir>/helix/languages.toml
+
+[[grammar]]
+name = "rust"
+source = { git = "https://github.com/tree-sitter/tree-sitter-rust", rev = "a250c4582510ff34767ec3b7dcdd3c24e8c8aa68" }
+
+[[grammar]]
+name = "c"
+source = { path = "/path/to/tree-sitter-c" }
+```
+
+You may use a top-level `use-grammars` key to control which grammars are fetched and built.
+
+```toml
+# Note: this key must come **before** the [[language]] and [[grammar]] sections
+use-grammars = { only = [ "rust", "c", "cpp" ] }
+# or
+use-grammars = { except = [ "yaml", "json" ] }
+```
+
+When omitted, all grammars are fetched and built.

book/src/remapping.md

@@ -2,7 +2,7 @@
 
 One-way key remapping is temporarily supported via a simple TOML configuration
 file. (More powerful solutions such as rebinding via commands will be
-available in the feature).
+available in the future).
 
 To remap keys, write a `config.toml` file in your `helix` configuration
 directory (default `~/.config/helix` in Linux systems) with a structure like
@@ -11,13 +11,19 @@ this:
 ```toml
 # At most one section each of 'keys.normal', 'keys.insert' and 'keys.select'
 [keys.normal]
+C-s = ":w" # Maps the Control-s to the typable command :w which is an alias for :write (save file)
+C-o = ":open ~/.config/helix/config.toml" # Maps the Control-o to opening of the helix config file
 a = "move_char_left" # Maps the 'a' key to the move_char_left command
 w = "move_line_up" # Maps the 'w' key move_line_up
-C-S-esc = "extend_line" # Maps Control-Shift-Escape to extend_line
+"C-S-esc" = "extend_line" # Maps Control-Shift-Escape to extend_line
+g = { a = "code_action" } # Maps `ga` to show possible code actions
+"ret" = ["open_below", "normal_mode"] # Maps the enter key to open_below then re-enter normal mode
 
 [keys.insert]
-A-x = "normal_mode" # Maps Alt-X to enter normal mode
+"A-x" = "normal_mode" # Maps Alt-X to enter normal mode
+j = { k = "normal_mode" } # Maps `jk` to exit insert mode
 ```
+> NOTE: Typable commands can also be remapped, remember to keep the `:` prefix to indicate it's a typable command.
 
 Control, Shift and Alt modifiers are encoded respectively with the prefixes
 `C-`, `S-` and `A-`. Special keys are encoded as follows:
@@ -36,15 +42,18 @@ Control, Shift and Alt modifiers are encoded respectively with the prefixes
 | Left         | `"left"`       |
 | Right        | `"right"`      |
 | Up           | `"up"`         |
+| Down         | `"down"`       |
 | Home         | `"home"`       |
 | End          | `"end"`        |
-| Page         | `"pageup"`     |
+| Page Up      | `"pageup"`     |
-| Page         | `"pagedown"`   |
+| Page Down    | `"pagedown"`   |
 | Tab          | `"tab"`        |
-| Back         | `"backtab"`    |
 | Delete       | `"del"`        |
 | Insert       | `"ins"`        |
 | Null         | `"null"`       |
 | Escape       | `"esc"`        |
 
-Commands can be found in the source code at [`helix-term/src/commands.rs`](https://github.com/helix-editor/helix/blob/master/helix-term/src/commands.rs)
+Keys can be disabled by binding them to the `no_op` command.
+
+Commands can be found at [Keymap](https://docs.helix-editor.com/keymap.html) Commands.
+> Commands can also be found in the source code at [`helix-term/src/commands.rs`](https://github.com/helix-editor/helix/blob/master/helix-term/src/commands.rs) at the invocation of `static_commands!` macro and the `TypableCommandList`.

book/src/themes.md

@@ -1,14 +1,14 @@
 # Themes
 
-First you'll need to place selected themes in your `themes` directory (i.e `~/.config/helix/themes`), the directory might have to be created beforehand.
+To use a theme add `theme = "<name>"` to your [`config.toml`](./configuration.md) at the very top of the file before the first section or select it during runtime using `:theme <name>`.
-
-To use a custom theme add `theme = <name>` to your [`config.toml`](./configuration.md) or override it during runtime using `:theme <name>`.
-
-The default theme.toml can be found [here](https://github.com/helix-editor/helix/blob/master/theme.toml), and user submitted themes [here](https://github.com/helix-editor/helix/blob/master/runtime/themes). 
 
 ## Creating a theme
 
-First create a file with the name of your theme as file name (i.e `mytheme.toml`) and place it in your `themes` directory (i.e `~/.config/helix/themes`).
+Create a file with the name of your theme as file name (i.e `mytheme.toml`) and place it in your `themes` directory (i.e `~/.config/helix/themes`). The directory might have to be created beforehand.
+
+The names "default" and "base16_default" are reserved for the builtin themes and cannot be overridden by user defined themes.
+
+The default theme.toml can be found [here](https://github.com/helix-editor/helix/blob/master/theme.toml), and user submitted themes [here](https://github.com/helix-editor/helix/blob/master/runtime/themes). 
 
 Each line in the theme file is specified as below:
 
@@ -30,82 +30,9 @@ if the key contains a dot `'.'`, it must be quoted to prevent it being parsed as
 "key.key" = "#ffffff"
 ```
 
-Possible modifiers:
+### Color palettes
 
-| Modifier       |
+It's recommended define a palette of named colors, and refer to them from the
-| ---            |
-| `bold`         |
-| `dim`          |
-| `italic`       |
-| `underlined`   |
-| `slow\_blink`  |
-| `rapid\_blink` |
-| `reversed`     |
-| `hidden`       |
-| `crossed\_out` |
-
-Possible keys:
-
-| Key                      | Notes                               |
-| ---                      | ---                                 |
-| `attribute`              |                                     |
-| `keyword`                |                                     |
-| `keyword.directive`      | Preprocessor directives (\#if in C) |
-| `keyword.control`        | Control flow                        |
-| `namespace`              |                                     |
-| `punctuation`            |                                     |
-| `punctuation.delimiter`  |                                     |
-| `operator`               |                                     |
-| `special`                |                                     |
-| `property`               |                                     |
-| `variable`               |                                     |
-| `variable.parameter`     |                                     |
-| `type`                   |                                     |
-| `type.builtin`           |                                     |
-| `type.enum.variant`      | Enum variants                       |
-| `constructor`            |                                     |
-| `function`               |                                     |
-| `function.macro`         |                                     |
-| `function.builtin`       |                                     |
-| `comment`                |                                     |
-| `variable.builtin`       |                                     |
-| `constant`               |                                     |
-| `constant.builtin`       |                                     |
-| `string`                 |                                     |
-| `number`                 |                                     |
-| `escape`                 | Escaped characters                  |
-| `label`                  | For lifetimes                       |
-| `module`                 |                                     |
-| `ui.background`          |                                     |
-| `ui.cursor`              |                                     |
-| `ui.cursor.insert`       |                                     |
-| `ui.cursor.select`       |                                     |
-| `ui.cursor.match`        | Matching bracket etc.               |
-| `ui.cursor.primary`      | Cursor with primary selection       |
-| `ui.linenr`              |                                     |
-| `ui.linenr.selected`     |                                     |
-| `ui.statusline`          |                                     |
-| `ui.statusline.inactive` |                                     |
-| `ui.popup`               |                                     |
-| `ui.window`              |                                     |
-| `ui.help`                |                                     |
-| `ui.text`                |                                     |
-| `ui.text.focus`          |                                     |
-| `ui.menu.selected`       |                                     |
-| `ui.selection`           | For selections in the editing area  |
-| `ui.selection.primary`   |                                     |
-| `warning`                | LSP warning                         |
-| `error`                  | LSP error                           |
-| `info`                   | LSP info                            |
-| `hint`                   | LSP hint                            |
-
-These keys match [tree-sitter scopes](https://tree-sitter.github.io/tree-sitter/syntax-highlighting#theme). We half-follow the common scopes from [macromates language grammars](https://macromates.com/manual/en/language_grammars) with some differences.
-
-For a given highlight produced, styling will be determined based on the longest matching theme key. So it's enough to provide function to highlight `function.macro` and `function.builtin` as well, but you can use more specific scopes to highlight specific cases differently.
-
-## Color palettes
-
-You can define a palette of named colors, and refer to them from the
 configuration values in your theme. To do this, add a table called
 `palette` to your theme file:
 
@@ -120,3 +47,190 @@ black = "#000000"
 
 Remember that the `[palette]` table includes all keys after its header,
 so you should define the palette after normal theme options.
+
+The default palette uses the terminal's default 16 colors, and the colors names
+are listed below. The `[palette]` section in the config file takes precedence
+over it and is merged into the default palette.
+
+| Color Name      |
+| ---             |
+| `black`         |
+| `red`           |
+| `green`         |
+| `yellow`        |
+| `blue`          |
+| `magenta`       |
+| `cyan`          |
+| `gray`          |
+| `light-red`     |
+| `light-green`   |
+| `light-yellow`  |
+| `light-blue`    |
+| `light-magenta` |
+| `light-cyan`    |
+| `light-gray`    |
+| `white`         |
+
+### Modifiers
+
+The following values may be used as modifiers. 
+
+Less common modifiers might not be supported by your terminal emulator.
+
+| Modifier       |
+| ---            |
+| `bold`         |
+| `dim`          |
+| `italic`       |
+| `underlined`   |
+| `slow_blink`   |
+| `rapid_blink`  |
+| `reversed`     |
+| `hidden`       |
+| `crossed_out`  |
+
+### Scopes
+
+The following is a list of scopes available to use for styling.
+
+#### Syntax highlighting
+
+These keys match [tree-sitter scopes](https://tree-sitter.github.io/tree-sitter/syntax-highlighting#theme).
+
+For a given highlight produced, styling will be determined based on the longest matching theme key. For example, the highlight `function.builtin.static` would match the key `function.builtin` rather than `function`.
+
+We use a similar set of scopes as
+[SublimeText](https://www.sublimetext.com/docs/scope_naming.html). See also
+[TextMate](https://macromates.com/manual/en/language_grammars) scopes.
+
+- `type` - Types
+  - `builtin` - Primitive types provided by the language (`int`, `usize`)
+- `constructor`
+
+- `constant` (TODO: constant.other.placeholder for %v)
+  - `builtin` Special constants provided by the language (`true`, `false`, `nil` etc)
+    - `boolean`
+  - `character`
+    - `escape`
+  - `numeric` (numbers)
+    - `integer`
+    - `float`
+
+- `string` (TODO: string.quoted.{single, double}, string.raw/.unquoted)?
+  - `regexp` - Regular expressions
+  - `special`
+    - `path`
+    - `url`
+    - `symbol` - Erlang/Elixir atoms, Ruby symbols, Clojure keywords
+
+- `comment` - Code comments
+  - `line` - Single line comments (`//`)
+  - `block` - Block comments (e.g. (`/*     */`)
+    - `documentation` - Documentation comments (e.g. `///` in Rust)
+
+- `variable` - Variables
+  - `builtin` - Reserved language variables (`self`, `this`, `super`, etc)
+  - `parameter` - Function parameters
+  - `other`
+    - `member` - Fields of composite data types (e.g. structs, unions)
+  - `function` (TODO: ?)
+
+- `label`
+
+- `punctuation`
+  - `delimiter` - Commas, colons
+  - `bracket` - Parentheses, angle brackets, etc.
+
+- `keyword`
+  - `control`
+    - `conditional` - `if`, `else`
+    - `repeat` - `for`, `while`, `loop`
+    - `import` - `import`, `export`
+    - `return`
+    - `exception`
+  - `operator` - `or`, `in`
+  - `directive` - Preprocessor directives (`#if` in C) 
+  - `function` - `fn`, `func`
+
+- `operator` - `||`, `+=`, `>`
+
+- `function`
+  - `builtin`
+  - `method`
+  - `macro`
+  - `special` (preprocesor in C)
+
+- `tag` - Tags (e.g. `<body>` in HTML)
+
+- `namespace`
+
+- `markup`
+  - `heading`
+    - `marker`
+    - `1`, `2`, `3`, `4`, `5`, `6` - heading text for h1 through h6
+  - `list`
+    - `unnumbered`
+    - `numbered`
+  - `bold`
+  - `italic`
+  - `link`
+    - `url` - urls pointed to by links
+    - `label` - non-url link references
+    - `text` - url and image descriptions in links
+  - `quote`
+  - `raw`
+    - `inline`
+    - `block`
+
+- `diff` - version control changes
+  - `plus` - additions
+  - `minus` - deletions
+  - `delta` - modifications
+    - `moved` - renamed or moved files/changes
+
+#### Interface
+
+These scopes are used for theming the editor interface.
+
+- `markup`
+  - `normal`
+    - `completion` - for completion doc popup ui
+    - `hover` - for hover popup ui
+  - `heading`
+    - `completion` - for completion doc popup ui
+    - `hover` - for hover popup ui
+  - `raw`
+    - `inline`
+      - `completion` - for completion doc popup ui
+      - `hover` - for hover popup ui
+
+
+| Key                      | Notes                               |
+| ---                      | ---                                 |
+| `ui.background`          |                                     |
+| `ui.cursor`              |                                     |
+| `ui.cursor.insert`       |                                     |
+| `ui.cursor.select`       |                                     |
+| `ui.cursor.match`        | Matching bracket etc.               |
+| `ui.cursor.primary`      | Cursor with primary selection       |
+| `ui.linenr`              |                                     |
+| `ui.linenr.selected`     |                                     |
+| `ui.statusline`          | Statusline                          |
+| `ui.statusline.inactive` | Statusline (unfocused document)     |
+| `ui.popup`               |                                     |
+| `ui.popup.info`          |                                     |
+| `ui.window`              |                                     |
+| `ui.help`                |                                     |
+| `ui.text`                |                                     |
+| `ui.text.focus`          |                                     |
+| `ui.text.info`           |                                     |
+| `ui.menu`                |                                     |
+| `ui.menu.selected`       |                                     |
+| `ui.selection`           | For selections in the editing area  |
+| `ui.selection.primary`   |                                     |
+| `warning`                | Diagnostics warning (gutter)        |
+| `error`                  | Diagnostics error (gutter)          |
+| `info`                   | Diagnostics info (gutter)           |
+| `hint`                   | Diagnostics hint (gutter)           |
+| `diagnostic`             | For text in editing area            |
+

book/src/title-page.md

@@ -0,0 +1,15 @@
+# Helix
+
+Docs for bleeding edge master can be found at
+[https://docs.helix-editor.com/master](https://docs.helix-editor.com/master).
+
+See the [usage] section for a quick overview of the editor, [keymap]
+section for all available keybindings and the [configuration] section
+for defining custom keybindings, setting themes, etc.
+
+Refer the [FAQ] for common questions.
+
+[FAQ]: https://github.com/helix-editor/helix/wiki/FAQ
+[usage]: ./usage.md
+[keymap]: ./keymap.md
+[configuration]: ./configuration.md

book/src/usage.md

@@ -2,6 +2,32 @@
 
 (Currently not fully documented, see the [keymappings](./keymap.md) list for more.)
 
+See [tutor.txt](https://github.com/helix-editor/helix/blob/master/runtime/tutor.txt) (accessible via `hx --tutor` or `:tutor`) for a vimtutor-like introduction.
+
+## Registers
+
+Vim-like registers can be used to yank and store text to be pasted later. Usage is similar, with `"` being used to select a register:
+
+- `"ay` - Yank the current selection to register `a`.
+- `"op` - Paste the text in register `o` after the selection.
+
+If there is a selected register before invoking a change or delete command, the selection will be stored in the register and the action will be carried out:
+
+- `"hc` - Store the selection in register `h` and then change it (delete and enter insert mode).
+- `"md` - Store the selection in register `m` and delete it.
+
+### Special Registers
+
+| Register character | Contains              |
+| ---                | ---                   |
+| `/`                | Last search           |
+| `:`                | Last executed command |
+| `"`                | Last yanked text      |
+| `_`                | Black hole            |
+
+> There is no special register for copying to system clipboard, instead special commands and keybindings are provided. See the [keymap](keymap.md#space-mode) for the specifics.
+> The black hole register works as a no-op register, meaning no data will be written to / read from it.
+
 ## Surround
 
 Functionality similar to [vim-surround](https://github.com/tpope/vim-surround) is built into
@@ -16,7 +42,7 @@ helix. The keymappings have been inspired from [vim-sandwich](https://github.com
 `ms` acts on a selection, so select the text first and use `ms<char>`. `mr` and `md` work
 on the closest pairs found and selections are not required; use counts to act in outer pairs.
 
-It can also act on multiple seletions (yay!). For example, to change every occurance of `(use)` to `[use]`:
+It can also act on multiple selections (yay!). For example, to change every occurrence of `(use)` to `[use]`:
 
 - `%` to select the whole file
 - `s` to split the selections on a search term
@@ -27,9 +53,10 @@ Multiple characters are currently not supported, but planned.
 
 ## Textobjects
 
-Currently supported: `word`, `surround`.
+Currently supported: `word`, `surround`, `function`, `class`, `parameter`.
 
 ![textobject-demo](https://user-images.githubusercontent.com/23398472/124231131-81a4bb00-db2d-11eb-9d10-8e577ca7b177.gif)
+![textobject-treesitter-demo](https://user-images.githubusercontent.com/23398472/132537398-2a2e0a54-582b-44ab-a77f-eb818942203d.gif)
 
 - `ma` - Select around the object (`va` in vim, `<alt-a>` in kakoune)
 - `mi` - Select inside the object (`vi` in vim, `<alt-i>` in kakoune)
@@ -37,6 +64,33 @@ Currently supported: `word`, `surround`.
 | Key after `mi` or `ma` | Textobject selected      |
 | ---                    | ---                      |
 | `w`                    | Word                     |
+| `W`                    | WORD                     |
 | `(`, `[`, `'`, etc     | Specified surround pairs |
+| `f`                    | Function                 |
+| `c`                    | Class                    |
+| `a`                    | Argument/parameter       |
+| `o`                    | Comment                  |
 
-Textobjects based on treesitter, like `function`, `class`, etc are planned.
+> NOTE: `f`, `c`, etc need a tree-sitter grammar active for the current
+document and a special tree-sitter query file to work properly. [Only
+some grammars][lang-support] currently have the query file implemented.
+Contributions are welcome!
+
+## Tree-sitter Textobject Based Navigation
+
+Navigating between functions, classes, parameters, etc is made
+possible by leveraging tree-sitter and textobjects queries. For
+example to move to the next function use `]f`, to move to previous
+class use `[c`, and so on.
+
+![tree-sitter-nav-demo][tree-sitter-nav-demo]
+
+See the [unimpaired][unimpaired-keybinds] section of the keybind
+documentation for the full reference.
+
+> NOTE: This feature is dependent on tree-sitter based textobjects
+and therefore requires the corresponding query file to work properly.
+
+[lang-support]: ./lang-support.md
+[unimpaired-keybinds]: ./keymap.md#unimpaired
+[tree-sitter-nav-demo]: https://user-images.githubusercontent.com/23398472/152332550-7dfff043-36a2-4aec-b8f2-77c13eb56d6f.gif

book/theme/css/chrome.css

@@ -196,7 +196,7 @@ a > .hljs {
     border-radius: 3px;
 }
 
-:not(pre):not(a) > .hljs {
+:not(pre):not(a):not(td):not(p) > .hljs {
     color: var(--inline-code-color);
     overflow-x: initial;
 }

book/theme/css/general.css

@@ -114,6 +114,19 @@ h6:target::before {
     margin-bottom: .875em;
 }
 
+.content ul li {
+margin-bottom: .25rem;
+}
+.content ul {
+    list-style-type: square;
+}
+.content ul ul, .content ol ul {
+    margin-bottom: .5rem;
+}
+.content li p {
+    margin-bottom: .5em;
+}
+
 .content p { line-height: 1.45em; }
 .content ol { line-height: 1.45em; }
 .content ul { line-height: 1.45em; }
@@ -149,7 +162,6 @@ table thead td {
 table thead th {
     padding: .75rem;
     text-align: left;
-    color: var(--table-border-color);
     font-weight: 500;
     line-height: 1.5;
     width: auto;
@@ -215,3 +227,7 @@ blockquote *:last-child {
     margin: 5px 0px;
     font-weight: bold;
 }
+
+.result-no-output {
+    font-style: italic;
+}

book/theme/css/variables.css

@@ -13,7 +13,6 @@
 .ayu {
     --bg: hsl(210, 25%, 8%);
     --fg: #c5c5c5;
-    --heading-fg: #c5c5c5;
 
     --sidebar-bg: #14191f;
     --sidebar-fg: #c8c9db;
@@ -54,7 +53,6 @@
 .coal {
     --bg: hsl(200, 7%, 8%);
     --fg: #98a3ad;
-    --heading-fg: #98a3ad;
 
     --sidebar-bg: #292c2f;
     --sidebar-fg: #a1adb8;
@@ -69,7 +67,7 @@
 
     --links: #2b79a2;
 
-    --inline-code-color: #c5c8c6;;
+    --inline-code-color: #c5c8c6;
 
     --theme-popup-bg: #141617;
     --theme-popup-border: #43484d;
@@ -95,7 +93,6 @@
 .light {
     --bg: hsl(0, 0%, 100%);
     --fg: hsl(0, 0%, 0%);
-    --heading-fg: hsl(0, 0%, 0%);
 
     --sidebar-bg: #fafafa;
     --sidebar-fg: hsl(0, 0%, 0%);
@@ -136,7 +133,6 @@
 .navy {
     --bg: hsl(226, 23%, 11%);
     --fg: #bcbdd0;
-    --heading-fg: #bcbdd0;
 
     --sidebar-bg: #282d3f;
     --sidebar-fg: #c8c9db;
@@ -151,7 +147,7 @@
 
     --links: #2b79a2;
 
-    --inline-code-color: #c5c8c6;;
+    --inline-code-color: #c5c8c6;
 
     --theme-popup-bg: #161923;
     --theme-popup-border: #737480;
@@ -177,7 +173,6 @@
 .rust {
     --bg: hsl(60, 9%, 87%);
     --fg: #262625;
-    --heading-fg: #262625;
 
     --sidebar-bg: #3b2e2a;
     --sidebar-fg: #c8c9db;
@@ -218,8 +213,7 @@
 @media (prefers-color-scheme: dark) {
     .light.no-js {
         --bg: hsl(200, 7%, 8%);
-        --fg: #ebeafa;
+        --fg: #98a3ad;
-        --heading-fg: #ebeafa;
 
         --sidebar-bg: #292c2f;
         --sidebar-fg: #a1adb8;
@@ -234,7 +228,7 @@
 
         --links: #2b79a2;
 
-        --inline-code-color: #c5c8c6;;
+        --inline-code-color: #c5c8c6;
 
         --theme-popup-bg: #141617;
         --theme-popup-border: #43484d;
@@ -261,6 +255,7 @@
 .colibri {
     --bg: #3b224c;
     --fg: #bcbdd0;
+    --heading-fg: #fff;
 
     --sidebar-bg: #281733;
     --sidebar-fg: #c8c9db;
@@ -276,18 +271,19 @@
     /* --links: #a4a0e8; */
     --links: #ECCDBA;
 
-    --inline-code-color: #c5c8c6;;
+    --inline-code-color: hsl(48.7, 7.8%, 70%);
 
     --theme-popup-bg: #161923;
     --theme-popup-border: #737480;
     --theme-hover: rgba(0,0,0, .2);
 
-    --quote-bg: hsl(226, 15%, 17%);
+    --quote-bg: #281733;
     --quote-border: hsl(226, 15%, 22%);
 
-    --table-border-color: hsl(226, 23%, 16%);
+    --table-border-color: hsl(226, 23%, 76%);
-    --table-header-bg: hsl(226, 23%, 31%);
+    --table-header-bg: hsla(226, 23%, 31%, 0);
     --table-alternate-bg: hsl(226, 23%, 14%);
+    --table-border-line: hsla(201deg, 20%, 92%, 0.2);
 
     --searchbar-border-color: #aaa;
     --searchbar-bg: #aeaec6;
@@ -300,6 +296,7 @@
 }
 
 .colibri {
+/*
     --bg: #ffffff;
     --fg: #452859;
     --fg: #5a5977;
@@ -318,7 +315,7 @@
 
     --links: #6F44F0;
 
-    --inline-code-color: #697C81;
+    --inline-code-color: #a39e9b;
 
     --theme-popup-bg: #161923;
     --theme-popup-border: #737480;
@@ -341,4 +338,5 @@
     --searchresults-border-color: #5c5c68;
     --searchresults-li-bg: #242430;
     --search-mark-bg: #a2cff5;
+*/
 }

book/theme/highlight.css

@@ -1,83 +1,56 @@
-/*
+pre code.hljs {
- * An increased contrast highlighting scheme loosely based on the
- * "Base16 Atelier Dune Light" theme by Bram de Haan
- * (http://atelierbram.github.io/syntax-highlighting/atelier-schemes/dune)
- * Original Base16 color scheme by Chris Kempson
- * (https://github.com/chriskempson/base16)
- */
-
-/* Comment */
-.hljs-comment,
-.hljs-quote {
-  color: #575757;
-}
-
-/* Red */
-.hljs-variable,
-.hljs-template-variable,
-.hljs-attribute,
-.hljs-tag,
-.hljs-name,
-.hljs-regexp,
-.hljs-link,
-.hljs-name,
-.hljs-selector-id,
-.hljs-selector-class {
-  color: #d70025;
-}
-
-/* Orange */
-.hljs-number,
-.hljs-meta,
-.hljs-built_in,
-.hljs-builtin-name,
-.hljs-literal,
-.hljs-type,
-.hljs-params {
-  color: #b21e00;
-}
-
-/* Green */
-.hljs-string,
-.hljs-symbol,
-.hljs-bullet {
-  color: #008200;
-}
-
-/* Blue */
-.hljs-title,
-.hljs-section {
-  color: #0030f2;
-}
-
-/* Purple */
-.hljs-keyword,
-.hljs-selector-tag {
-  color: #9d00ec;
-}
-
-.hljs {
  display:block;
  overflow-x:auto;
-  background: #f6f7f6;
+ padding:1em
-  color: #000;
+}
-  padding: 0.5em;
+code.hljs {
+ padding:3px 5px
+}
+.hljs {
+ background:#2f1e2e;
+ color:#a39e9b
+}
+.hljs-comment,
+.hljs-quote {
+ color:#8d8687
+}
+.hljs-link,
+.hljs-meta,
+.hljs-name,
+.hljs-regexp,
+.hljs-selector-class,
+.hljs-selector-id,
+.hljs-tag,
+.hljs-template-variable,
+.hljs-variable {
+ color:#ef6155
+}
+.hljs-built_in,
+.hljs-deletion,
+.hljs-literal,
+.hljs-number,
+.hljs-params,
+.hljs-type {
+ color:#f99b15
+}
+.hljs-attribute,
+.hljs-section,
+.hljs-title {
+ color:#fec418
+}
+.hljs-addition,
+.hljs-bullet,
+.hljs-string,
+.hljs-symbol {
+ color:#48b685
+}
+.hljs-keyword,
+.hljs-selector-tag {
+ color:#815ba4
 }
-
 .hljs-emphasis {
-  font-style: italic;
+ font-style:italic
 }
-
 .hljs-strong {
-  font-weight: bold;
+ font-weight:700
-}
-
-.hljs-addition {
-  color: #22863a;
-  background-color: #f0fff4;
-}
-
-.hljs-deletion {
-  color: #b31d28;
-  background-color: #ffeef0;
 }

docs/CONTRIBUTING.md

@@ -0,0 +1,37 @@
+# Contributing
+
+Contributors are very welcome! **No contribution is too small and all contributions are valued.**
+
+Some suggestions to get started:
+
+- You can look at the [good first issue][good-first-issue] label on the issue tracker.
+- Help with packaging on various distributions needed!
+- To use print debugging to the [Helix log file][log-file], you must:
+  * Print using `log::info!`, `warn!`, or `error!`. (`log::info!("helix!")`)
+  * Pass the appropriate verbosity level option for the desired log level. (`hx -v <file>` for info, more `v`s for higher severity inclusive)
+- If your preferred language is missing, integrating a tree-sitter grammar for
+    it and defining syntax highlight queries for it is straight forward and
+    doesn't require much knowledge of the internals.
+
+We provide an [architecture.md][architecture.md] that should give you
+a good overview of the internals.
+
+# Auto generated documentation
+
+Some parts of [the book][docs] are autogenerated from the code itself,
+like the list of `:commands` and supported languages. To generate these
+files, run
+
+```shell
+cargo xtask docgen
+```
+
+inside the project. We use [xtask][xtask] as an ad-hoc task runner and
+thus do not require any dependencies other than `cargo` (You don't have
+to `cargo install` anything either).
+
+[good-first-issue]: https://github.com/helix-editor/helix/labels/E-easy
+[log-file]: https://github.com/helix-editor/helix/wiki/FAQ#access-the-log-file
+[architecture.md]: ./architecture.md
+[docs]: https://docs.helix-editor.com/
+[xtask]: https://github.com/matklad/cargo-xtask

docs/architecture.md

@@ -2,8 +2,9 @@
 | Crate        | Description                                                      |
 | -----------  | -----------                                                      |
 | helix-core   | Core editing primitives, functional.                             |
-| helix-syntax | Tree-sitter grammars                                   |
 | helix-lsp    | Language server client                                           |
+| helix-dap    | Debug Adapter Protocol (DAP) client                              |
+| helix-loader | Functions for building, fetching, and loading external resources |
 | helix-view   | UI abstractions for use in backends, imperative shell.           |
 | helix-term   | Terminal UI                                                      |
 | helix-tui    | TUI primitives, forked from tui-rs, inspired by Cursive          |
@@ -54,15 +55,40 @@ A `Document` ties together the `Rope`, `Selection`(s), `Syntax`, document
 file.
 
 A `View` represents an open split in the UI. It holds the currently open
-document ID and other related state.
+document ID and other related state. Views encapsulate the gutter, status line,
+diagnostics, and the inner area where the code is displayed.
 
 > NOTE: Multiple views are able to display the same document, so the document
 > contains selections for each view. To retrieve, `document.selection()` takes
 > a `ViewId`.
 
+`Info` is the autoinfo box that shows hints when awaiting another key with bindings
+like `g` and `m`. It is attached to the viewport as a whole.
+
+`Surface` is like a buffer to which widgets draw themselves to, and the
+surface is then rendered on the screen on each cycle.
+
+`Rect`s are areas (simply an x and y coordinate with the origin at the
+screen top left and then a height and width) which are part of a
+`Surface`. They can be used to limit the area to which a `Component` can
+render. For example if we wrap a `Markdown` component in a `Popup`
+(think the documentation popup with space+k), Markdown's render method
+will get a Rect that is the exact size of the popup.
+
+Widgets are called `Component`s internally, and you can see most of them
+in `helix-term/src/ui`. Some components like `Popup` and `Overlay` can take
+other components as children.
+
+`Layer`s are how multiple components are displayed, and is simply a
+`Vec<Component>`. Layers are managed by the `Compositor`. On each top
+level render call, the compositor renders each component in the order
+they were pushed into the stack. This makes multiple components "layer"
+on top of one another. Hence we get a file picker displayed over the
+editor, etc.
+
 The `Editor` holds the global state: all the open documents, a tree
-representation of all the view splits, and a registry of language servers. To
+representation of all the view splits, the configuration, and a registry of 
-open or close files, interact with the editor.
+language servers. To open or close files, interact with the editor.
 
 ## LSP
 

docs/vision.md

@@ -0,0 +1,24 @@
+The Helix project still has a ways to go before reaching its goals.  This document outlines some of those goals and the overall vision for the project.
+
+# Vision
+
+An efficient, batteries-included editor you can take anywhere and be productive... if it's your kind of thing.
+
+* **Cross-platform.**  Whether on Linux, Windows, or OSX, you should be able to take your editor with you.
+* **Terminal first.**  Not all environments have a windowing system, and you shouldn't have to abandon your preferred editor in those cases.
+* **Native.**  No Electron or HTML DOM here.  We want an efficient, native-compiled editor that can run with minimal resources when needed.  If you're working on a Raspberry Pi, your editor shouldn't consume half of your RAM.
+* **Batteries included.**  Both the default configuration and bundled features should be enough to have a good editing experience and be productive.  You shouldn't need a massive custom config or external executables and plugins for basic features and functionality.
+* **Don't try to be everything for everyone.**  There are many great editors out there to choose from.  Let's make Helix *one of* those great options, with its own take on things.
+
+# Goals
+
+Vision statements are all well and good, but are also vague and subjective.  Here is a (non-exhaustive) list of some of Helix's more concrete goals, to help give a clearer idea of the project's direction:
+
+* **Modal.**  Vim is a great idea.
+* **Selection -> Action**, not Verb -> Object.  Interaction models aren't linguistics, and "selection first" lets you see what you're doing (among other benefits).
+* **We aren't playing code golf.**  It's more important for the keymap to be consistent and easy to memorize than it is to save a key stroke or two when editing.
+* **Built-in tools** for working with code bases efficiently.  Most projects aren't a single file, and an editor should handle that as a first-class use case.  In Helix's case, this means (among other things) a fuzzy-search file navigator and LSP support.
+* **Edit anything** that comes up when coding, within reason.  Whether it's a 200 MB XML file, a megabyte of minified javascript on a single line, or Japanese text encoded in ShiftJIS, you should be able to open it and edit it without problems.  (Note: this doesn't mean handle every esoteric use case.  Sometimes you do just need a specialized tool, and Helix isn't that.)
+* **Configurable**, within reason.  Although the defaults should be good, not everyone will agree on what "good" is.  Within the bounds of Helix's core interaction models, it should be reasonably configurable so that it can be "good" for more people.  This means, for example, custom key maps among other things.
+* **Extensible**, within reason.  Although we want Helix to be productive out-of-the-box, it's not practical or desirable to cram every useful feature and use case into the core editor.  The basics should be built-in, but you should be able to extend it with additional functionality as needed.  Right now we're thinking Wasm-based plugins.
+* **Clean code base.**  Sometimes other factors (e.g. significant performance gains, important features, correctness, etc.) will trump strict readability, but we nevertheless want to keep the code base straightforward and easy to understand to the extent we can.

flake.lock

@@ -1,12 +1,35 @@
 {
   "nodes": {
-    "devshell": {
+    "crane": {
+      "flake": false,
       "locked": {
-        "lastModified": 1625086391,
+        "lastModified": 1644785799,
-        "narHash": "sha256-IpNPv1v8s4L3CoxhwcgZIitGpcrnNgnj09X7TA0QV3k=",
+        "narHash": "sha256-VpAJO1L0XeBvtCuNGK4IDKp6ENHIpTrlaZT7yfBCvwo=",
+        "owner": "ipetkov",
+        "repo": "crane",
+        "rev": "fc7a94f841347c88f2cb44217b2a3faa93e2a0b2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ipetkov",
+        "repo": "crane",
+        "type": "github"
+      }
+    },
+    "devshell": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1646667754,
+        "narHash": "sha256-LahZHvCC3UVzGQ55iWDRZkuDssXl1rYgqgScrPV9S38=",
         "owner": "numtide",
         "repo": "devshell",
-        "rev": "4b5ac7cf7d9a1cc60b965bb51b59922f2210cbc7",
+        "rev": "59fbe1dfc0de8c3332957c16998a7d16dff365d8",
         "type": "github"
       },
       "original": {
@@ -15,36 +38,104 @@
         "type": "github"
       }
     },
-    "flakeCompat": {
+    "dream2nix": {
-      "flake": false,
+      "inputs": {
+        "alejandra": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ],
+        "crane": "crane",
+        "flake-utils-pre-commit": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ],
+        "gomod2nix": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ],
+        "mach-nix": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ],
+        "nixpkgs": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ],
+        "node2nix": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ],
+        "poetry2nix": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ],
+        "pre-commit-hooks": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ]
+      },
       "locked": {
-        "lastModified": 1627913399,
+        "lastModified": 1646710334,
-        "narHash": "sha256-hY8g6H2KFL8ownSiFeMOjwPC8P0ueXpCVEbxgda3pko=",
+        "narHash": "sha256-eLBcDgcbOUfeH4k6SEW5a5v0PTp2KNCn+5ZXIoWGYww=",
-        "owner": "edolstra",
+        "owner": "nix-community",
-        "repo": "flake-compat",
+        "repo": "dream2nix",
-        "rev": "12c64ca55c1014cdc1b16ed5a804aa8576601ff2",
+        "rev": "5dcfbfd3b60ce0208b894c1bdea00e2bdf80ca6a",
         "type": "github"
       },
       "original": {
-        "owner": "edolstra",
+        "owner": "nix-community",
-        "repo": "flake-compat",
+        "ref": "main",
+        "repo": "dream2nix",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "locked": {
+        "lastModified": 1642700792,
+        "narHash": "sha256-XqHrk7hFb+zBvRg6Ghl+AZDq03ov6OshJLiSWOoX5es=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "846b2ae0fc4cc943637d3d1def4454213e203cba",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_2": {
+      "locked": {
+        "lastModified": 1637014545,
+        "narHash": "sha256-26IZAc5yzlD9FlDT54io1oqG/bBoyka+FJk5guaX4x4=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "bba5dcc8e0b20ab664967ad83d24d64cb64ec4f4",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
         "type": "github"
       }
     },
     "nixCargoIntegration": {
       "inputs": {
         "devshell": "devshell",
+        "dream2nix": "dream2nix",
         "nixpkgs": [
           "nixpkgs"
         ],
-        "rustOverlay": "rustOverlay"
+        "rustOverlay": [
+          "rust-overlay"
+        ]
       },
       "locked": {
-        "lastModified": 1628489367,
+        "lastModified": 1646766572,
-        "narHash": "sha256-ADYKHf8aPo1qTw1J+eqVprnEbH8lES0yZamD/yM7RAM=",
+        "narHash": "sha256-DV3+zxvAIKsMHsHedJKYFsracvFyLKpFQqurUBR86oY=",
         "owner": "yusdacra",
         "repo": "nix-cargo-integration",
-        "rev": "0dc8383aae5f791a48e34120edb04670b947dc0b",
+        "rev": "3a3f47f43ba486b7554164a698c8dfc5a38624ce",
         "type": "github"
       },
       "original": {
@@ -55,11 +146,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1628465643,
+        "lastModified": 1646497237,
-        "narHash": "sha256-QSNw9bDq9uGUniQQtakRuw4m21Jxugm23SXLVgEV4DM=",
+        "narHash": "sha256-Ccpot1h/rV8MgcngDp5OrdmLTMaUTbStZTR5/sI7zW0=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "6ef4f522d63f22b40004319778761040d3197390",
+        "rev": "062a0c5437b68f950b081bbfc8a699d57a4ee026",
         "type": "github"
       },
       "original": {
@@ -71,19 +162,24 @@
     },
     "root": {
       "inputs": {
-        "flakeCompat": "flakeCompat",
         "nixCargoIntegration": "nixCargoIntegration",
-        "nixpkgs": "nixpkgs"
+        "nixpkgs": "nixpkgs",
+        "rust-overlay": "rust-overlay"
       }
     },
-    "rustOverlay": {
+    "rust-overlay": {
-      "flake": false,
+      "inputs": {
+        "flake-utils": "flake-utils_2",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
       "locked": {
-        "lastModified": 1628475192,
+        "lastModified": 1646792695,
-        "narHash": "sha256-A32shcfPMCll7psCS0OBxVCkA+PKfeWvmU4y9lgNZzU=",
+        "narHash": "sha256-2drCXIKIQnJMlTZbcCfuHZAh+iPcdlRkCqtZnA6MHLY=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "56a8ddb827cbe7a914be88f4a52998a5f93ff468",
+        "rev": "7f599870402c8d2a5806086c8ee0f2d92b175c54",
         "type": "github"
       },
       "original": {

flake.nix

@@ -3,84 +3,61 @@
 
   inputs = {
     nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
+    rust-overlay = {
+      url = "github:oxalica/rust-overlay";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
     nixCargoIntegration = {
       url = "github:yusdacra/nix-cargo-integration";
       inputs.nixpkgs.follows = "nixpkgs";
-    };
+      inputs.rustOverlay.follows = "rust-overlay";
-    flakeCompat = {
-      url = "github:edolstra/flake-compat";
-      flake = false;
     };
   };
 
-  outputs = inputs@{ self, nixCargoIntegration, ... }:
+  outputs = inputs@{ nixCargoIntegration, ... }:
     nixCargoIntegration.lib.makeOutputs {
       root = ./.;
-      buildPlatform = "crate2nix";
       renameOutputs = { "helix-term" = "helix"; };
       # Set default app to hx (binary is from helix-term release build)
       # Set default package to helix-term release build
-      defaultOutputs = { app = "hx"; package = "helix"; };
+      defaultOutputs = {
+        app = "hx";
+        package = "helix";
+      };
       overrides = {
         crateOverrides = common: _: {
-          helix-term = prev: {
+          helix-term = prev:
-            # link languages and theme toml files since helix-term expects them (for tests)
-            preConfigure = "ln -s ${common.root}/{languages.toml,theme.toml} ..";
-            buildInputs = (prev.buildInputs or [ ]) ++ [ common.cCompiler.cc.lib ];
-          };
-          # link languages and theme toml files since helix-view expects them
-          helix-view = _: { preConfigure = "ln -s ${common.root}/{languages.toml,theme.toml} .."; };
-          helix-syntax = prev: {
-            src =
             let
-                pkgs = common.pkgs;
+              inherit (common) pkgs;
-                helix = pkgs.fetchgit {
+              grammars = pkgs.callPackage ./grammars.nix { };
-                  url = "https://github.com/helix-editor/helix.git";
-                  rev = "d4bd5b37669708361a0a6cd2917464b010e6b7f5";
-                  fetchSubmodules = true;
-                  sha256 = "sha256-KayR7K7UC0mT6EjHsZsCYY9IVDJzft63fGpPKGSY8nQ=";
-                };
-              in
-              pkgs.runCommand prev.src.name { } ''
-                mkdir -p $out
-                ln -s ${prev.src}/* $out
-                ln -sf ${helix}/helix-syntax/languages $out
-              '';
-            preConfigure = "mkdir -p ../runtime/grammars";
-            postInstall = "cp -r ../runtime $out/runtime";
-          };
-        };
-        mainBuild = common: prev:
-          let
-            inherit (common) pkgs lib;
-            helixSyntax = lib.buildCrate {
-              root = self;
-              memberName = "helix-syntax";
-              defaultCrateOverrides = {
-                helix-syntax = common.crateOverrides.helix-syntax;
-              };
-              release = false;
-            };
               runtimeDir = pkgs.runCommand "helix-runtime" { } ''
                 mkdir -p $out
                 ln -s ${common.root}/runtime/* $out
-              ln -sf ${helixSyntax}/runtime/grammars $out
+                rm -r $out/grammars
+                ln -s ${grammars} $out/grammars
               '';
             in
-          lib.optionalAttrs (common.memberName == "helix-term") {
+            {
+              # disable fetching and building of tree-sitter grammars in the helix-term build.rs
+              HELIX_DISABLE_AUTO_GRAMMAR_BUILD = "1";
+              # link languages and theme toml files since helix-term expects them (for tests)
+              preConfigure = "ln -s ${common.root}/{languages.toml,theme.toml,base16_theme.toml} ..";
+              buildInputs = (prev.buildInputs or [ ]) ++ [ common.cCompiler.cc.lib ];
               nativeBuildInputs = [ pkgs.makeWrapper ];
+
               postFixup = ''
                 if [ -f "$out/bin/hx" ]; then
                   wrapProgram "$out/bin/hx" --set HELIX_RUNTIME "${runtimeDir}"
                 fi
               '';
             };
+        };
         shell = common: prev: {
-          packages = prev.packages ++ (with common.pkgs; [ lld_10 lldb cargo-tarpaulin ]);
+          packages = prev.packages ++ (with common.pkgs; [ lld_13 lldb cargo-tarpaulin cargo-flamegraph ]);
           env = prev.env ++ [
             { name = "HELIX_RUNTIME"; eval = "$PWD/runtime"; }
             { name = "RUST_BACKTRACE"; value = "1"; }
-            { name = "RUSTFLAGS"; value = "-C link-arg=-fuse-ld=lld -C target-cpu=native"; }
+            { name = "RUSTFLAGS"; value = "-C link-arg=-fuse-ld=lld -C target-cpu=native -Clink-arg=-Wl,--no-rosegment"; }
           ];
         };
       };

grammars.nix

@@ -0,0 +1,89 @@
+{ stdenv, lib, runCommand, yj }:
+let
+  # HACK: nix < 2.6 has a bug in the toml parser, so we convert to JSON
+  # before parsing
+  languages-json = runCommand "languages-toml-to-json" { } ''
+    ${yj}/bin/yj -t < ${./languages.toml} > $out
+  '';
+  languagesConfig =
+    builtins.fromJSON (builtins.readFile (builtins.toPath languages-json));
+  isGitGrammar = (grammar:
+    builtins.hasAttr "source" grammar && builtins.hasAttr "git" grammar.source
+    && builtins.hasAttr "rev" grammar.source);
+  gitGrammars = builtins.filter isGitGrammar languagesConfig.grammar;
+  buildGrammar = grammar:
+    let
+      source = builtins.fetchTree {
+        type = "git";
+        url = grammar.source.git;
+        rev = grammar.source.rev;
+        ref = grammar.source.ref or "HEAD";
+        shallow = true;
+      };
+    in stdenv.mkDerivation rec {
+      # see https://github.com/NixOS/nixpkgs/blob/fbdd1a7c0bc29af5325e0d7dd70e804a972eb465/pkgs/development/tools/parsing/tree-sitter/grammar.nix
+
+      pname = "helix-tree-sitter-${grammar.name}";
+      version = grammar.source.rev;
+
+      src = if builtins.hasAttr "subpath" grammar.source then
+        "${source}/${grammar.source.subpath}"
+      else
+        source;
+
+      dontUnpack = true;
+      dontConfigure = true;
+
+      FLAGS = [
+        "-I${src}/src"
+        "-g"
+        "-O3"
+        "-fPIC"
+        "-fno-exceptions"
+        "-Wl,-z,relro,-z,now"
+      ];
+
+      NAME = grammar.name;
+
+      buildPhase = ''
+        runHook preBuild
+
+        if [[ -e "$src/src/scanner.cc" ]]; then
+          $CXX -c "$src/src/scanner.cc" -o scanner.o $FLAGS
+        elif [[ -e "$src/src/scanner.c" ]]; then
+          $CC -c "$src/src/scanner.c" -o scanner.o $FLAGS
+        fi
+
+        $CC -c "$src/src/parser.c" -o parser.o $FLAGS
+        $CXX -shared -o $NAME.so *.o
+
+        ls -al
+
+        runHook postBuild
+      '';
+
+      installPhase = ''
+        runHook preInstall
+        mkdir $out
+        mv $NAME.so $out/
+        runHook postInstall
+      '';
+
+      # Strip failed on darwin: strip: error: symbols referenced by indirect symbol table entries that can't be stripped
+      fixupPhase = lib.optionalString stdenv.isLinux ''
+        runHook preFixup
+        $STRIP $out/$NAME.so
+        runHook postFixup
+      '';
+    };
+  builtGrammars = builtins.map (grammar: {
+    inherit (grammar) name;
+    artifact = buildGrammar grammar;
+  }) gitGrammars;
+  grammarLinks = builtins.map (grammar:
+    "ln -s ${grammar.artifact}/${grammar.name}.so $out/${grammar.name}.so")
+    builtGrammars;
+in runCommand "consolidated-helix-grammars" { } ''
+  mkdir -p $out
+  ${builtins.concatStringsSep "\n" grammarLinks}
+''

helix-core/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
 name = "helix-core"
-version = "0.4.0"
+version = "0.6.0"
 authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
-edition = "2018"
+edition = "2021"
 license = "MPL-2.0"
 description = "Helix editor core editing primitives"
 categories = ["editor"]
@@ -13,24 +13,31 @@ include = ["src/**/*", "README.md"]
 [features]
 
 [dependencies]
-helix-syntax = { version = "0.4", path = "../helix-syntax" }
+helix-loader = { version = "0.6", path = "../helix-loader" }
 
 ropey = "1.3"
-smallvec = "1.4"
+smallvec = "1.8"
-tendril = "0.4.2"
+smartstring = "1.0.0"
-unicode-segmentation = "1.8"
+unicode-segmentation = "1.9"
 unicode-width = "0.1"
-unicode-general-category = "0.4"
+unicode-general-category = "0.5"
 # slab = "0.4.2"
-tree-sitter = "0.19"
+slotmap = "1.0"
-once_cell = "1.8"
+tree-sitter = "0.20"
+once_cell = "1.10"
 arc-swap = "1"
 regex = "1"
 
+log = "0.4"
 serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
 toml = "0.5"
 
-similar = "1.3"
+similar = "2.1"
+
+encoding_rs = "0.8"
+
+chrono = { version = "0.4", default-features = false, features = ["alloc", "std"] }
 
 etcetera = "0.3"
 

helix-core/src/chars.rs

@@ -1,3 +1,5 @@
+//! Utility functions to categorize a `char`.
+
 use crate::LineEnding;
 
 #[derive(Debug, Eq, PartialEq)]
@@ -89,12 +91,11 @@ mod test {
 
     #[test]
     fn test_categorize() {
-        const EOL_TEST_CASE: &'static str = "\n\r\u{000B}\u{000C}\u{0085}\u{2028}\u{2029}";
+        const EOL_TEST_CASE: &str = "\n\r\u{000B}\u{000C}\u{0085}\u{2028}\u{2029}";
-        const WORD_TEST_CASE: &'static str =
+        const WORD_TEST_CASE: &str = "_hello_world_あいうえおー1234567890１２３４５６７８９０";
-            "_hello_world_あいうえおー1234567890１２３４５６７８９０";
+        const PUNCTUATION_TEST_CASE: &str =
-        const PUNCTUATION_TEST_CASE: &'static str =
             "!\"#$%&\'()*+,-./:;<=>?@[\\]^`{|}~！”＃＄％＆’（）＊＋、。：；＜＝＞？＠「」＾｀｛｜｝～";
-        const WHITESPACE_TEST_CASE: &'static str = "  　   ";
+        const WHITESPACE_TEST_CASE: &str = "  　   ";
 
         for ch in EOL_TEST_CASE.chars() {
             assert_eq!(CharCategory::Eol, categorize_char(ch));

helix-core/src/comment.rs

@@ -1,3 +1,6 @@
+//! This module contains the functionality toggle comments on lines over the selection
+//! using the comment character defined in the user's `languages.toml`
+
 use crate::{
     find_first_non_whitespace_char, Change, Rope, RopeSlice, Selection, Tendril, Transaction,
 };
@@ -60,7 +63,7 @@ pub fn toggle_line_comments(doc: &Rope, selection: &Selection, token: Option<&st
     let token = token.unwrap_or("//");
     let comment = Tendril::from(format!("{} ", token));
 
-    let mut lines: Vec<usize> = Vec::new();
+    let mut lines: Vec<usize> = Vec::with_capacity(selection.len());
 
     let mut min_next_line = 0;
     for selection in selection {

helix-core/src/config.rs

@@ -0,0 +1,10 @@
+/// Syntax configuration loader based on built-in languages.toml.
+pub fn default_syntax_loader() -> crate::syntax::Configuration {
+    helix_loader::default_lang_config()
+        .try_into()
+        .expect("Could not serialize built-in languages.toml")
+}
+/// Syntax configuration loader based on user configured languages.toml.
+pub fn user_syntax_loader() -> Result<crate::syntax::Configuration, toml::de::Error> {
+    helix_loader::user_lang_config()?.try_into()
+}

helix-core/src/diagnostic.rs

@@ -1,18 +1,30 @@
-#[derive(Debug, Eq, PartialEq)]
+//! LSP diagnostic utility types.
+use serde::{Deserialize, Serialize};
+
+/// Describes the severity level of a [`Diagnostic`].
+#[derive(Debug, Clone, Copy, Eq, PartialEq, PartialOrd, Ord, Deserialize, Serialize)]
 pub enum Severity {
-    Error,
-    Warning,
-    Info,
     Hint,
+    Info,
+    Warning,
+    Error,
 }
 
-#[derive(Debug)]
+impl Default for Severity {
+    fn default() -> Self {
+        Self::Hint
+    }
+}
+
+/// A range of `char`s within the text.
+#[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq)]
 pub struct Range {
     pub start: usize,
     pub end: usize,
 }
 
-#[derive(Debug)]
+/// Corresponds to [`lsp_types::Diagnostic`](https://docs.rs/lsp-types/0.91.0/lsp_types/struct.Diagnostic.html)
+#[derive(Debug, Clone)]
 pub struct Diagnostic {
     pub range: Range,
     pub line: usize,

helix-core/src/diff.rs

@@ -11,10 +11,6 @@ pub fn compare_ropes(old: &Rope, new: &Rope) -> Transaction {
     // A timeout is set so after 1 seconds, the algorithm will start
     // approximating. This is especially important for big `Rope`s or
     // `Rope`s that are extremely dissimilar to each other.
-    //
-    // Note: Ignore the clippy warning, as the trait bounds of
-    // `Transaction::change()` require an iterator implementing
-    // `ExactIterator`.
     let mut config = similar::TextDiff::configure();
     config.timeout(std::time::Duration::from_secs(1));
 
@@ -62,7 +58,7 @@ mod tests {
             let mut old = Rope::from(a);
             let new = Rope::from(b);
             compare_ropes(&old, &new).apply(&mut old);
-            old.to_string() == new.to_string()
+            old == new
         }
     }
 }

helix-core/src/graphemes.rs

@@ -1,4 +1,6 @@
-// Based on https://github.com/cessen/led/blob/c4fa72405f510b7fd16052f90a598c429b3104a6/src/graphemes.rs
+//! Utility functions to traverse the unicode graphemes of a `Rope`'s text contents.
+//!
+//! Based on <https://github.com/cessen/led/blob/c4fa72405f510b7fd16052f90a598c429b3104a6/src/graphemes.rs>
 use ropey::{iter::Chunks, str_utils::byte_to_char_idx, RopeSlice};
 use unicode_segmentation::{GraphemeCursor, GraphemeIncomplete};
 use unicode_width::UnicodeWidthStr;
@@ -118,6 +120,43 @@ pub fn nth_next_grapheme_boundary(slice: RopeSlice, char_idx: usize, n: usize) -
     chunk_char_idx + tmp
 }
 
+#[must_use]
+pub fn nth_next_grapheme_boundary_byte(slice: RopeSlice, mut byte_idx: usize, n: usize) -> usize {
+    // Bounds check
+    debug_assert!(byte_idx <= slice.len_bytes());
+
+    // Get the chunk with our byte index in it.
+    let (mut chunk, mut chunk_byte_idx, mut _chunk_char_idx, _) = slice.chunk_at_byte(byte_idx);
+
+    // Set up the grapheme cursor.
+    let mut gc = GraphemeCursor::new(byte_idx, slice.len_bytes(), true);
+
+    // Find the nth next grapheme cluster boundary.
+    for _ in 0..n {
+        loop {
+            match gc.next_boundary(chunk, chunk_byte_idx) {
+                Ok(None) => return slice.len_bytes(),
+                Ok(Some(n)) => {
+                    byte_idx = n;
+                    break;
+                }
+                Err(GraphemeIncomplete::NextChunk) => {
+                    chunk_byte_idx += chunk.len();
+                    let (a, _, _c, _) = slice.chunk_at_byte(chunk_byte_idx);
+                    chunk = a;
+                    // chunk_char_idx = c;
+                }
+                Err(GraphemeIncomplete::PreContext(n)) => {
+                    let ctx_chunk = slice.chunk_at_byte(n - 1).0;
+                    gc.provide_context(ctx_chunk, n - ctx_chunk.len());
+                }
+                _ => unreachable!(),
+            }
+        }
+    }
+    byte_idx
+}
+
 /// Finds the next grapheme boundary after the given char position.
 #[must_use]
 #[inline(always)]
@@ -125,6 +164,13 @@ pub fn next_grapheme_boundary(slice: RopeSlice, char_idx: usize) -> usize {
     nth_next_grapheme_boundary(slice, char_idx, 1)
 }
 
+/// Finds the next grapheme boundary after the given byte position.
+#[must_use]
+#[inline(always)]
+pub fn next_grapheme_boundary_byte(slice: RopeSlice, byte_idx: usize) -> usize {
+    nth_next_grapheme_boundary_byte(slice, byte_idx, 1)
+}
+
 /// Returns the passed char index if it's already a grapheme boundary,
 /// or the next grapheme boundary char index if not.
 #[must_use]
@@ -149,6 +195,23 @@ pub fn ensure_grapheme_boundary_prev(slice: RopeSlice, char_idx: usize) -> usize
     }
 }
 
+/// Returns the passed byte index if it's already a grapheme boundary,
+/// or the next grapheme boundary byte index if not.
+#[must_use]
+#[inline]
+pub fn ensure_grapheme_boundary_next_byte(slice: RopeSlice, byte_idx: usize) -> usize {
+    if byte_idx == 0 {
+        byte_idx
+    } else {
+        // TODO: optimize so we're not constructing grapheme cursor twice
+        if is_grapheme_boundary_byte(slice, byte_idx) {
+            byte_idx
+        } else {
+            next_grapheme_boundary_byte(slice, byte_idx)
+        }
+    }
+}
+
 /// Returns whether the given char position is a grapheme boundary.
 #[must_use]
 pub fn is_grapheme_boundary(slice: RopeSlice, char_idx: usize) -> bool {
@@ -177,6 +240,31 @@ pub fn is_grapheme_boundary(slice: RopeSlice, char_idx: usize) -> bool {
     }
 }
 
+/// Returns whether the given byte position is a grapheme boundary.
+#[must_use]
+pub fn is_grapheme_boundary_byte(slice: RopeSlice, byte_idx: usize) -> bool {
+    // Bounds check
+    debug_assert!(byte_idx <= slice.len_bytes());
+
+    // Get the chunk with our byte index in it.
+    let (chunk, chunk_byte_idx, _, _) = slice.chunk_at_byte(byte_idx);
+
+    // Set up the grapheme cursor.
+    let mut gc = GraphemeCursor::new(byte_idx, slice.len_bytes(), true);
+
+    // Determine if the given position is a grapheme cluster boundary.
+    loop {
+        match gc.is_boundary(chunk, chunk_byte_idx) {
+            Ok(n) => return n,
+            Err(GraphemeIncomplete::PreContext(n)) => {
+                let (ctx_chunk, ctx_byte_start, _, _) = slice.chunk_at_byte(n - 1);
+                gc.provide_context(ctx_chunk, ctx_byte_start);
+            }
+            Err(_) => unreachable!(),
+        }
+    }
+}
+
 /// An iterator over the graphemes of a `RopeSlice`.
 #[derive(Clone)]
 pub struct RopeGraphemes<'a> {

helix-core/src/history.rs

@@ -1,51 +1,53 @@
-use crate::{ChangeSet, Rope, State, Transaction};
+use crate::{Assoc, ChangeSet, Range, Rope, State, Transaction};
 use once_cell::sync::Lazy;
 use regex::Regex;
 use std::num::NonZeroUsize;
 use std::time::{Duration, Instant};
 
-// Stores the history of changes to a buffer.
+/// Stores the history of changes to a buffer.
-//
+///
-// Currently the history is represented as a vector of revisions. The vector
+/// Currently the history is represented as a vector of revisions. The vector
-// always has at least one element: the empty root revision. Each revision
+/// always has at least one element: the empty root revision. Each revision
-// with the exception of the root has a parent revision, a [Transaction]
+/// with the exception of the root has a parent revision, a [Transaction]
-// that can be applied to its parent to transition from the parent to itself,
+/// that can be applied to its parent to transition from the parent to itself,
-// and an inversion of that transaction to transition from the parent to its
+/// and an inversion of that transaction to transition from the parent to its
-// latest child.
+/// latest child.
-//
+///
-// When using `u` to undo a change, an inverse of the stored transaction will
+/// When using `u` to undo a change, an inverse of the stored transaction will
-// be applied which will transition the buffer to the parent state.
+/// be applied which will transition the buffer to the parent state.
-//
+///
-// Each revision with the exception of the last in the vector also has a
+/// Each revision with the exception of the last in the vector also has a
-// last child revision. When using `U` to redo a change, the last child transaction
+/// last child revision. When using `U` to redo a change, the last child transaction
-// will be applied to the current state of the buffer.
+/// will be applied to the current state of the buffer.
-//
+///
-// The current revision is the one currently displayed in the buffer.
+/// The current revision is the one currently displayed in the buffer.
-//
+///
-// Commiting a new revision to the history will update the last child of the
+/// Commiting a new revision to the history will update the last child of the
-// current revision, and push a new revision to the end of the vector.
+/// current revision, and push a new revision to the end of the vector.
-//
+///
-// Revisions are commited with a timestamp. :earlier and :later can be used
+/// Revisions are commited with a timestamp. :earlier and :later can be used
-// to jump to the closest revision to a moment in time relative to the timestamp
+/// to jump to the closest revision to a moment in time relative to the timestamp
-// of the current revision plus (:later) or minus (:earlier) the duration
+/// of the current revision plus (:later) or minus (:earlier) the duration
-// given to the command. If a single integer is given, the editor will instead
+/// given to the command. If a single integer is given, the editor will instead
-// jump the given number of revisions in the vector.
+/// jump the given number of revisions in the vector.
-//
+///
-// Limitations:
+/// Limitations:
-//  * Changes in selections currently don't commit history changes. The selection
+///  * Changes in selections currently don't commit history changes. The selection
-//    will only be updated to the state after a commited buffer change.
+///    will only be updated to the state after a commited buffer change.
-//  * The vector of history revisions is currently unbounded. This might
+///  * The vector of history revisions is currently unbounded. This might
-//    cause the memory consumption to grow significantly large during long
+///    cause the memory consumption to grow significantly large during long
-//    editing sessions.
+///    editing sessions.
-//  * Because delete transactions currently don't store the text that they
+///  * Because delete transactions currently don't store the text that they
-//    delete, we also store an inversion of the transaction.
+///    delete, we also store an inversion of the transaction.
+///
+/// Using time to navigate the history: <https://github.com/helix-editor/helix/pull/194>
 #[derive(Debug)]
 pub struct History {
     revisions: Vec<Revision>,
     current: usize,
 }
 
-// A single point in history. See [History] for more information.
+/// A single point in history. See [History] for more information.
 #[derive(Debug)]
 struct Revision {
     parent: usize,
@@ -111,6 +113,7 @@ impl History {
         self.current == 0
     }
 
+    /// Undo the last edit.
     pub fn undo(&mut self) -> Option<&Transaction> {
         if self.at_root() {
             return None;
@@ -121,6 +124,7 @@ impl History {
         Some(&current_revision.inversion)
     }
 
+    /// Redo the last edit.
     pub fn redo(&mut self) -> Option<&Transaction> {
         let current_revision = &self.revisions[self.current];
         let last_child = current_revision.last_child?;
@@ -129,6 +133,32 @@ impl History {
         Some(&self.revisions[last_child.get()].transaction)
     }
 
+    // Get the position of last change
+    pub fn last_edit_pos(&self) -> Option<usize> {
+        if self.current == 0 {
+            return None;
+        }
+        let current_revision = &self.revisions[self.current];
+        let primary_selection = current_revision
+            .inversion
+            .selection()
+            .expect("inversion always contains a selection")
+            .primary();
+        let (_from, to, _fragment) = current_revision
+            .transaction
+            .changes_iter()
+            // find a change that matches the primary selection
+            .find(|(from, to, _fragment)| Range::new(*from, *to).overlaps(&primary_selection))
+            // or use the first change
+            .or_else(|| current_revision.transaction.changes_iter().next())
+            .unwrap();
+        let pos = current_revision
+            .transaction
+            .changes()
+            .map_pos(to, Assoc::After);
+        Some(pos)
+    }
+
     fn lowest_common_ancestor(&self, mut a: usize, mut b: usize) -> usize {
         use std::collections::HashSet;
         let mut a_path_set = HashSet::new();
@@ -147,8 +177,8 @@ impl History {
         }
     }
 
-    // List of nodes on the way from `n` to 'a`. Doesn`t include `a`.
+    /// List of nodes on the way from `n` to 'a`. Doesn`t include `a`.
-    // Includes `n` unless `a == n`. `a` must be an ancestor of `n`.
+    /// Includes `n` unless `a == n`. `a` must be an ancestor of `n`.
     fn path_up(&self, mut n: usize, a: usize) -> Vec<usize> {
         let mut path = Vec::new();
         while n != a {
@@ -158,6 +188,7 @@ impl History {
         path
     }
 
+    /// Create a [`Transaction`] that will jump to a specific revision in the history.
     fn jump_to(&mut self, to: usize) -> Vec<Transaction> {
         let lca = self.lowest_common_ancestor(self.current, to);
         let up = self.path_up(self.current, lca);
@@ -171,10 +202,12 @@ impl History {
         up_txns.chain(down_txns).collect()
     }
 
+    /// Creates a [`Transaction`] that will undo `delta` revisions.
     fn jump_backward(&mut self, delta: usize) -> Vec<Transaction> {
         self.jump_to(self.current.saturating_sub(delta))
     }
 
+    /// Creates a [`Transaction`] that will redo `delta` revisions.
     fn jump_forward(&mut self, delta: usize) -> Vec<Transaction> {
         self.jump_to(
             self.current
@@ -183,7 +216,7 @@ impl History {
         )
     }
 
-    // Helper for a binary search case below.
+    /// Helper for a binary search case below.
     fn revision_closer_to_instant(&self, i: usize, instant: Instant) -> usize {
         let dur_im1 = instant.duration_since(self.revisions[i - 1].timestamp);
         let dur_i = self.revisions[i].timestamp.duration_since(instant);
@@ -194,6 +227,8 @@ impl History {
         }
     }
 
+    /// Creates a [`Transaction`] that will match a revision created at around
+    /// `instant`.
     fn jump_instant(&mut self, instant: Instant) -> Vec<Transaction> {
         let search_result = self
             .revisions
@@ -209,6 +244,8 @@ impl History {
         self.jump_to(revision)
     }
 
+    /// Creates a [`Transaction`] that will match a revision created `duration` ago
+    /// from the timestamp of current revision.
     fn jump_duration_backward(&mut self, duration: Duration) -> Vec<Transaction> {
         match self.revisions[self.current].timestamp.checked_sub(duration) {
             Some(instant) => self.jump_instant(instant),
@@ -216,6 +253,8 @@ impl History {
         }
     }
 
+    /// Creates a [`Transaction`] that will match a revision created `duration` in
+    /// the future from the timestamp of the current revision.
     fn jump_duration_forward(&mut self, duration: Duration) -> Vec<Transaction> {
         match self.revisions[self.current].timestamp.checked_add(duration) {
             Some(instant) => self.jump_instant(instant),
@@ -223,6 +262,7 @@ impl History {
         }
     }
 
+    /// Creates an undo [`Transaction`].
     pub fn earlier(&mut self, uk: UndoKind) -> Vec<Transaction> {
         use UndoKind::*;
         match uk {
@@ -231,6 +271,7 @@ impl History {
         }
     }
 
+    /// Creates a redo [`Transaction`].
     pub fn later(&mut self, uk: UndoKind) -> Vec<Transaction> {
         use UndoKind::*;
         match uk {
@@ -240,13 +281,14 @@ impl History {
     }
 }
 
-#[derive(Debug, PartialEq)]
+/// Whether to undo by a number of edits or a duration of time.
+#[derive(Debug, PartialEq, Clone, Copy)]
 pub enum UndoKind {
     Steps(usize),
     TimePeriod(std::time::Duration),
 }
 
-// A subset of sytemd.time time span syntax units.
+/// A subset of sytemd.time time span syntax units.
 const TIME_UNITS: &[(&[&str], &str, u64)] = &[
     (&["seconds", "second", "sec", "s"], "seconds", 1),
     (&["minutes", "minute", "min", "m"], "minutes", 60),
@@ -254,11 +296,20 @@ const TIME_UNITS: &[(&[&str], &str, u64)] = &[
     (&["days", "day", "d"], "days", 24 * 60 * 60),
 ];
 
+/// Checks if the duration input can be turned into a valid duration. It must be a
+/// positive integer and denote the [unit of time.](`TIME_UNITS`)
+/// Examples of valid durations:
+///  * `5 sec`
+///  * `5 min`
+///  * `5 hr`
+///  * `5 days`
 static DURATION_VALIDATION_REGEX: Lazy<Regex> =
     Lazy::new(|| Regex::new(r"^(?:\d+\s*[a-z]+\s*)+$").unwrap());
 
+/// Captures both the number and unit as separate capture groups.
 static NUMBER_UNIT_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"(\d+)\s*([a-z]+)").unwrap());
 
+/// Parse a string (e.g. "5 sec") and try to convert it into a [`Duration`].
 fn parse_human_duration(s: &str) -> Result<Duration, String> {
     if !DURATION_VALIDATION_REGEX.is_match(s) {
         return Err("duration should be composed \
@@ -397,8 +448,8 @@ mod test {
             change: crate::transaction::Change,
             instant: Instant,
         ) {
-            let txn = Transaction::change(&state.doc, vec![change.clone()].into_iter());
+            let txn = Transaction::change(&state.doc, vec![change].into_iter());
-            history.commit_revision_at_timestamp(&txn, &state, instant);
+            history.commit_revision_at_timestamp(&txn, state, instant);
             txn.apply(&mut state.doc);
         }
 

helix-core/src/increment/date_time.rs

@@ -0,0 +1,490 @@
+use chrono::{Datelike, Duration, NaiveDate, NaiveDateTime, NaiveTime, Timelike};
+use once_cell::sync::Lazy;
+use regex::Regex;
+use ropey::RopeSlice;
+
+use std::borrow::Cow;
+use std::cmp;
+
+use super::Increment;
+use crate::{Range, Tendril};
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct DateTimeIncrementor {
+    date_time: NaiveDateTime,
+    range: Range,
+    fmt: &'static str,
+    field: DateField,
+}
+
+impl DateTimeIncrementor {
+    pub fn from_range(text: RopeSlice, range: Range) -> Option<DateTimeIncrementor> {
+        let range = if range.is_empty() {
+            if range.anchor < text.len_chars() {
+                // Treat empty range as a cursor range.
+                range.put_cursor(text, range.anchor + 1, true)
+            } else {
+                // The range is empty and at the end of the text.
+                return None;
+            }
+        } else {
+            range
+        };
+
+        FORMATS.iter().find_map(|format| {
+            let from = range.from().saturating_sub(format.max_len);
+            let to = (range.from() + format.max_len).min(text.len_chars());
+
+            let (from_in_text, to_in_text) = (range.from() - from, range.to() - from);
+            let text: Cow<str> = text.slice(from..to).into();
+
+            let captures = format.regex.captures(&text)?;
+            if captures.len() - 1 != format.fields.len() {
+                return None;
+            }
+
+            let date_time = captures.get(0)?;
+            let offset = range.from() - from_in_text;
+            let range = Range::new(date_time.start() + offset, date_time.end() + offset);
+
+            let field = captures
+                .iter()
+                .skip(1)
+                .enumerate()
+                .find_map(|(i, capture)| {
+                    let capture = capture?;
+                    let capture_range = capture.range();
+
+                    if capture_range.contains(&from_in_text)
+                        && capture_range.contains(&(to_in_text - 1))
+                    {
+                        Some(format.fields[i])
+                    } else {
+                        None
+                    }
+                })?;
+
+            let has_date = format.fields.iter().any(|f| f.unit.is_date());
+            let has_time = format.fields.iter().any(|f| f.unit.is_time());
+
+            let date_time = &text[date_time.start()..date_time.end()];
+            let date_time = match (has_date, has_time) {
+                (true, true) => NaiveDateTime::parse_from_str(date_time, format.fmt).ok()?,
+                (true, false) => {
+                    let date = NaiveDate::parse_from_str(date_time, format.fmt).ok()?;
+
+                    date.and_hms(0, 0, 0)
+                }
+                (false, true) => {
+                    let time = NaiveTime::parse_from_str(date_time, format.fmt).ok()?;
+
+                    NaiveDate::from_ymd(0, 1, 1).and_time(time)
+                }
+                (false, false) => return None,
+            };
+
+            Some(DateTimeIncrementor {
+                date_time,
+                range,
+                fmt: format.fmt,
+                field,
+            })
+        })
+    }
+}
+
+impl Increment for DateTimeIncrementor {
+    fn increment(&self, amount: i64) -> (Range, Tendril) {
+        let date_time = match self.field.unit {
+            DateUnit::Years => add_years(self.date_time, amount),
+            DateUnit::Months => add_months(self.date_time, amount),
+            DateUnit::Days => add_duration(self.date_time, Duration::days(amount)),
+            DateUnit::Hours => add_duration(self.date_time, Duration::hours(amount)),
+            DateUnit::Minutes => add_duration(self.date_time, Duration::minutes(amount)),
+            DateUnit::Seconds => add_duration(self.date_time, Duration::seconds(amount)),
+            DateUnit::AmPm => toggle_am_pm(self.date_time),
+        }
+        .unwrap_or(self.date_time);
+
+        (self.range, date_time.format(self.fmt).to_string().into())
+    }
+}
+
+static FORMATS: Lazy<Vec<Format>> = Lazy::new(|| {
+    vec![
+        Format::new("%Y-%m-%d %H:%M:%S"), // 2021-11-24 07:12:23
+        Format::new("%Y/%m/%d %H:%M:%S"), // 2021/11/24 07:12:23
+        Format::new("%Y-%m-%d %H:%M"),    // 2021-11-24 07:12
+        Format::new("%Y/%m/%d %H:%M"),    // 2021/11/24 07:12
+        Format::new("%Y-%m-%d"),          // 2021-11-24
+        Format::new("%Y/%m/%d"),          // 2021/11/24
+        Format::new("%a %b %d %Y"),       // Wed Nov 24 2021
+        Format::new("%d-%b-%Y"),          // 24-Nov-2021
+        Format::new("%Y %b %d"),          // 2021 Nov 24
+        Format::new("%b %d, %Y"),         // Nov 24, 2021
+        Format::new("%-I:%M:%S %P"),      // 7:21:53 am
+        Format::new("%-I:%M %P"),         // 7:21 am
+        Format::new("%-I:%M:%S %p"),      // 7:21:53 AM
+        Format::new("%-I:%M %p"),         // 7:21 AM
+        Format::new("%H:%M:%S"),          // 23:24:23
+        Format::new("%H:%M"),             // 23:24
+    ]
+});
+
+#[derive(Debug)]
+struct Format {
+    fmt: &'static str,
+    fields: Vec<DateField>,
+    regex: Regex,
+    max_len: usize,
+}
+
+impl Format {
+    fn new(fmt: &'static str) -> Self {
+        let mut remaining = fmt;
+        let mut fields = Vec::new();
+        let mut regex = String::new();
+        let mut max_len = 0;
+
+        while let Some(i) = remaining.find('%') {
+            let after = &remaining[i + 1..];
+            let mut chars = after.chars();
+            let c = chars.next().unwrap();
+
+            let spec_len = if c == '-' {
+                1 + chars.next().unwrap().len_utf8()
+            } else {
+                c.len_utf8()
+            };
+
+            let specifier = &after[..spec_len];
+            let field = DateField::from_specifier(specifier).unwrap();
+            fields.push(field);
+            max_len += field.max_len + remaining[..i].len();
+            regex += &remaining[..i];
+            regex += &format!("({})", field.regex);
+            remaining = &after[spec_len..];
+        }
+
+        let regex = Regex::new(&regex).unwrap();
+
+        Self {
+            fmt,
+            fields,
+            regex,
+            max_len,
+        }
+    }
+}
+
+impl PartialEq for Format {
+    fn eq(&self, other: &Self) -> bool {
+        self.fmt == other.fmt && self.fields == other.fields && self.max_len == other.max_len
+    }
+}
+
+impl Eq for Format {}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+struct DateField {
+    regex: &'static str,
+    unit: DateUnit,
+    max_len: usize,
+}
+
+impl DateField {
+    fn from_specifier(specifier: &str) -> Option<Self> {
+        match specifier {
+            "Y" => Some(Self {
+                regex: r"\d{4}",
+                unit: DateUnit::Years,
+                max_len: 5,
+            }),
+            "y" => Some(Self {
+                regex: r"\d\d",
+                unit: DateUnit::Years,
+                max_len: 2,
+            }),
+            "m" => Some(Self {
+                regex: r"[0-1]\d",
+                unit: DateUnit::Months,
+                max_len: 2,
+            }),
+            "d" => Some(Self {
+                regex: r"[0-3]\d",
+                unit: DateUnit::Days,
+                max_len: 2,
+            }),
+            "-d" => Some(Self {
+                regex: r"[1-3]?\d",
+                unit: DateUnit::Days,
+                max_len: 2,
+            }),
+            "a" => Some(Self {
+                regex: r"Sun|Mon|Tue|Wed|Thu|Fri|Sat",
+                unit: DateUnit::Days,
+                max_len: 3,
+            }),
+            "A" => Some(Self {
+                regex: r"Sunday|Monday|Tuesday|Wednesday|Thursday|Friday|Saturday",
+                unit: DateUnit::Days,
+                max_len: 9,
+            }),
+            "b" | "h" => Some(Self {
+                regex: r"Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec",
+                unit: DateUnit::Months,
+                max_len: 3,
+            }),
+            "B" => Some(Self {
+                regex: r"January|February|March|April|May|June|July|August|September|October|November|December",
+                unit: DateUnit::Months,
+                max_len: 9,
+            }),
+            "H" => Some(Self {
+                regex: r"[0-2]\d",
+                unit: DateUnit::Hours,
+                max_len: 2,
+            }),
+            "M" => Some(Self {
+                regex: r"[0-5]\d",
+                unit: DateUnit::Minutes,
+                max_len: 2,
+            }),
+            "S" => Some(Self {
+                regex: r"[0-5]\d",
+                unit: DateUnit::Seconds,
+                max_len: 2,
+            }),
+            "I" => Some(Self {
+                regex: r"[0-1]\d",
+                unit: DateUnit::Hours,
+                max_len: 2,
+            }),
+            "-I" => Some(Self {
+                regex: r"1?\d",
+                unit: DateUnit::Hours,
+                max_len: 2,
+            }),
+            "P" => Some(Self {
+                regex: r"am|pm",
+                unit: DateUnit::AmPm,
+                max_len: 2,
+            }),
+            "p" => Some(Self {
+                regex: r"AM|PM",
+                unit: DateUnit::AmPm,
+                max_len: 2,
+            }),
+            _ => None,
+        }
+    }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+enum DateUnit {
+    Years,
+    Months,
+    Days,
+    Hours,
+    Minutes,
+    Seconds,
+    AmPm,
+}
+
+impl DateUnit {
+    fn is_date(self) -> bool {
+        matches!(self, DateUnit::Years | DateUnit::Months | DateUnit::Days)
+    }
+
+    fn is_time(self) -> bool {
+        matches!(
+            self,
+            DateUnit::Hours | DateUnit::Minutes | DateUnit::Seconds
+        )
+    }
+}
+
+fn ndays_in_month(year: i32, month: u32) -> u32 {
+    // The first day of the next month...
+    let (y, m) = if month == 12 {
+        (year + 1, 1)
+    } else {
+        (year, month + 1)
+    };
+    let d = NaiveDate::from_ymd(y, m, 1);
+
+    // ...is preceded by the last day of the original month.
+    d.pred().day()
+}
+
+fn add_months(date_time: NaiveDateTime, amount: i64) -> Option<NaiveDateTime> {
+    let month = (date_time.month0() as i64).checked_add(amount)?;
+    let year = date_time.year() + i32::try_from(month / 12).ok()?;
+    let year = if month.is_negative() { year - 1 } else { year };
+
+    // Normalize month
+    let month = month % 12;
+    let month = if month.is_negative() {
+        month + 12
+    } else {
+        month
+    } as u32
+        + 1;
+
+    let day = cmp::min(date_time.day(), ndays_in_month(year, month));
+
+    Some(NaiveDate::from_ymd(year, month, day).and_time(date_time.time()))
+}
+
+fn add_years(date_time: NaiveDateTime, amount: i64) -> Option<NaiveDateTime> {
+    let year = i32::try_from((date_time.year() as i64).checked_add(amount)?).ok()?;
+    let ndays = ndays_in_month(year, date_time.month());
+
+    if date_time.day() > ndays {
+        let d = NaiveDate::from_ymd(year, date_time.month(), ndays);
+        Some(d.succ().and_time(date_time.time()))
+    } else {
+        date_time.with_year(year)
+    }
+}
+
+fn add_duration(date_time: NaiveDateTime, duration: Duration) -> Option<NaiveDateTime> {
+    date_time.checked_add_signed(duration)
+}
+
+fn toggle_am_pm(date_time: NaiveDateTime) -> Option<NaiveDateTime> {
+    if date_time.hour() < 12 {
+        add_duration(date_time, Duration::hours(12))
+    } else {
+        add_duration(date_time, Duration::hours(-12))
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::Rope;
+
+    #[test]
+    fn test_increment_date_times() {
+        let tests = [
+            // (original, cursor, amount, expected)
+            ("2020-02-28", 0, 1, "2021-02-28"),
+            ("2020-02-29", 0, 1, "2021-03-01"),
+            ("2020-01-31", 5, 1, "2020-02-29"),
+            ("2020-01-20", 5, 1, "2020-02-20"),
+            ("2021-01-01", 5, -1, "2020-12-01"),
+            ("2021-01-31", 5, -2, "2020-11-30"),
+            ("2020-02-28", 8, 1, "2020-02-29"),
+            ("2021-02-28", 8, 1, "2021-03-01"),
+            ("2021-02-28", 0, -1, "2020-02-28"),
+            ("2021-03-01", 0, -1, "2020-03-01"),
+            ("2020-02-29", 5, -1, "2020-01-29"),
+            ("2020-02-20", 5, -1, "2020-01-20"),
+            ("2020-02-29", 8, -1, "2020-02-28"),
+            ("2021-03-01", 8, -1, "2021-02-28"),
+            ("1980/12/21", 8, 100, "1981/03/31"),
+            ("1980/12/21", 8, -100, "1980/09/12"),
+            ("1980/12/21", 8, 1000, "1983/09/17"),
+            ("1980/12/21", 8, -1000, "1978/03/27"),
+            ("2021-11-24 07:12:23", 0, 1, "2022-11-24 07:12:23"),
+            ("2021-11-24 07:12:23", 5, 1, "2021-12-24 07:12:23"),
+            ("2021-11-24 07:12:23", 8, 1, "2021-11-25 07:12:23"),
+            ("2021-11-24 07:12:23", 11, 1, "2021-11-24 08:12:23"),
+            ("2021-11-24 07:12:23", 14, 1, "2021-11-24 07:13:23"),
+            ("2021-11-24 07:12:23", 17, 1, "2021-11-24 07:12:24"),
+            ("2021/11/24 07:12:23", 0, 1, "2022/11/24 07:12:23"),
+            ("2021/11/24 07:12:23", 5, 1, "2021/12/24 07:12:23"),
+            ("2021/11/24 07:12:23", 8, 1, "2021/11/25 07:12:23"),
+            ("2021/11/24 07:12:23", 11, 1, "2021/11/24 08:12:23"),
+            ("2021/11/24 07:12:23", 14, 1, "2021/11/24 07:13:23"),
+            ("2021/11/24 07:12:23", 17, 1, "2021/11/24 07:12:24"),
+            ("2021-11-24 07:12", 0, 1, "2022-11-24 07:12"),
+            ("2021-11-24 07:12", 5, 1, "2021-12-24 07:12"),
+            ("2021-11-24 07:12", 8, 1, "2021-11-25 07:12"),
+            ("2021-11-24 07:12", 11, 1, "2021-11-24 08:12"),
+            ("2021-11-24 07:12", 14, 1, "2021-11-24 07:13"),
+            ("2021/11/24 07:12", 0, 1, "2022/11/24 07:12"),
+            ("2021/11/24 07:12", 5, 1, "2021/12/24 07:12"),
+            ("2021/11/24 07:12", 8, 1, "2021/11/25 07:12"),
+            ("2021/11/24 07:12", 11, 1, "2021/11/24 08:12"),
+            ("2021/11/24 07:12", 14, 1, "2021/11/24 07:13"),
+            ("Wed Nov 24 2021", 0, 1, "Thu Nov 25 2021"),
+            ("Wed Nov 24 2021", 4, 1, "Fri Dec 24 2021"),
+            ("Wed Nov 24 2021", 8, 1, "Thu Nov 25 2021"),
+            ("Wed Nov 24 2021", 11, 1, "Thu Nov 24 2022"),
+            ("24-Nov-2021", 0, 1, "25-Nov-2021"),
+            ("24-Nov-2021", 3, 1, "24-Dec-2021"),
+            ("24-Nov-2021", 7, 1, "24-Nov-2022"),
+            ("2021 Nov 24", 0, 1, "2022 Nov 24"),
+            ("2021 Nov 24", 5, 1, "2021 Dec 24"),
+            ("2021 Nov 24", 9, 1, "2021 Nov 25"),
+            ("Nov 24, 2021", 0, 1, "Dec 24, 2021"),
+            ("Nov 24, 2021", 4, 1, "Nov 25, 2021"),
+            ("Nov 24, 2021", 8, 1, "Nov 24, 2022"),
+            ("7:21:53 am", 0, 1, "8:21:53 am"),
+            ("7:21:53 am", 3, 1, "7:22:53 am"),
+            ("7:21:53 am", 5, 1, "7:21:54 am"),
+            ("7:21:53 am", 8, 1, "7:21:53 pm"),
+            ("7:21:53 AM", 0, 1, "8:21:53 AM"),
+            ("7:21:53 AM", 3, 1, "7:22:53 AM"),
+            ("7:21:53 AM", 5, 1, "7:21:54 AM"),
+            ("7:21:53 AM", 8, 1, "7:21:53 PM"),
+            ("7:21 am", 0, 1, "8:21 am"),
+            ("7:21 am", 3, 1, "7:22 am"),
+            ("7:21 am", 5, 1, "7:21 pm"),
+            ("7:21 AM", 0, 1, "8:21 AM"),
+            ("7:21 AM", 3, 1, "7:22 AM"),
+            ("7:21 AM", 5, 1, "7:21 PM"),
+            ("23:24:23", 1, 1, "00:24:23"),
+            ("23:24:23", 3, 1, "23:25:23"),
+            ("23:24:23", 6, 1, "23:24:24"),
+            ("23:24", 1, 1, "00:24"),
+            ("23:24", 3, 1, "23:25"),
+        ];
+
+        for (original, cursor, amount, expected) in tests {
+            let rope = Rope::from_str(original);
+            let range = Range::new(cursor, cursor + 1);
+            assert_eq!(
+                DateTimeIncrementor::from_range(rope.slice(..), range)
+                    .unwrap()
+                    .increment(amount)
+                    .1,
+                Tendril::from(expected)
+            );
+        }
+    }
+
+    #[test]
+    fn test_invalid_date_times() {
+        let tests = [
+            "0000-00-00",
+            "1980-2-21",
+            "1980-12-1",
+            "12345",
+            "2020-02-30",
+            "1999-12-32",
+            "19-12-32",
+            "1-2-3",
+            "0000/00/00",
+            "1980/2/21",
+            "1980/12/1",
+            "12345",
+            "2020/02/30",
+            "1999/12/32",
+            "19/12/32",
+            "1/2/3",
+            "123:456:789",
+            "11:61",
+            "2021-55-12 08:12:54",
+        ];
+
+        for invalid in tests {
+            let rope = Rope::from_str(invalid);
+            let range = Range::new(0, 1);
+
+            assert_eq!(DateTimeIncrementor::from_range(rope.slice(..), range), None)
+        }
+    }
+}

helix-core/src/increment/mod.rs

@@ -0,0 +1,8 @@
+pub mod date_time;
+pub mod number;
+
+use crate::{Range, Tendril};
+
+pub trait Increment {
+    fn increment(&self, amount: i64) -> (Range, Tendril);
+}

helix-core/src/increment/number.rs

@@ -0,0 +1,507 @@
+use std::borrow::Cow;
+
+use ropey::RopeSlice;
+
+use super::Increment;
+
+use crate::{
+    textobject::{textobject_word, TextObject},
+    Range, Tendril,
+};
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct NumberIncrementor<'a> {
+    value: i64,
+    radix: u32,
+    range: Range,
+
+    text: RopeSlice<'a>,
+}
+
+impl<'a> NumberIncrementor<'a> {
+    /// Return information about number under rang if there is one.
+    pub fn from_range(text: RopeSlice, range: Range) -> Option<NumberIncrementor> {
+        // If the cursor is on the minus sign of a number we want to get the word textobject to the
+        // right of it.
+        let range = if range.to() < text.len_chars()
+            && range.to() - range.from() <= 1
+            && text.char(range.from()) == '-'
+        {
+            Range::new(range.from() + 1, range.to() + 1)
+        } else {
+            range
+        };
+
+        let range = textobject_word(text, range, TextObject::Inside, 1, false);
+
+        // If there is a minus sign to the left of the word object, we want to include it in the range.
+        let range = if range.from() > 0 && text.char(range.from() - 1) == '-' {
+            range.extend(range.from() - 1, range.from())
+        } else {
+            range
+        };
+
+        let word: String = text
+            .slice(range.from()..range.to())
+            .chars()
+            .filter(|&c| c != '_')
+            .collect();
+        let (radix, prefixed) = if word.starts_with("0x") {
+            (16, true)
+        } else if word.starts_with("0o") {
+            (8, true)
+        } else if word.starts_with("0b") {
+            (2, true)
+        } else {
+            (10, false)
+        };
+
+        let number = if prefixed { &word[2..] } else { &word };
+
+        let value = i128::from_str_radix(number, radix).ok()?;
+        if (value.is_positive() && value.leading_zeros() < 64)
+            || (value.is_negative() && value.leading_ones() < 64)
+        {
+            return None;
+        }
+
+        let value = value as i64;
+        Some(NumberIncrementor {
+            range,
+            value,
+            radix,
+            text,
+        })
+    }
+}
+
+impl<'a> Increment for NumberIncrementor<'a> {
+    fn increment(&self, amount: i64) -> (Range, Tendril) {
+        let old_text: Cow<str> = self.text.slice(self.range.from()..self.range.to()).into();
+        let old_length = old_text.len();
+        let new_value = self.value.wrapping_add(amount);
+
+        // Get separator indexes from right to left.
+        let separator_rtl_indexes: Vec<usize> = old_text
+            .chars()
+            .rev()
+            .enumerate()
+            .filter_map(|(i, c)| if c == '_' { Some(i) } else { None })
+            .collect();
+
+        let format_length = if self.radix == 10 {
+            match (self.value.is_negative(), new_value.is_negative()) {
+                (true, false) => old_length - 1,
+                (false, true) => old_length + 1,
+                _ => old_text.len(),
+            }
+        } else {
+            old_text.len() - 2
+        } - separator_rtl_indexes.len();
+
+        let mut new_text = match self.radix {
+            2 => format!("0b{:01$b}", new_value, format_length),
+            8 => format!("0o{:01$o}", new_value, format_length),
+            10 if old_text.starts_with('0') || old_text.starts_with("-0") => {
+                format!("{:01$}", new_value, format_length)
+            }
+            10 => format!("{}", new_value),
+            16 => {
+                let (lower_count, upper_count): (usize, usize) =
+                    old_text.chars().skip(2).fold((0, 0), |(lower, upper), c| {
+                        (
+                            lower + c.is_ascii_lowercase().then(|| 1).unwrap_or(0),
+                            upper + c.is_ascii_uppercase().then(|| 1).unwrap_or(0),
+                        )
+                    });
+                if upper_count > lower_count {
+                    format!("0x{:01$X}", new_value, format_length)
+                } else {
+                    format!("0x{:01$x}", new_value, format_length)
+                }
+            }
+            _ => unimplemented!("radix not supported: {}", self.radix),
+        };
+
+        // Add separators from original number.
+        for &rtl_index in &separator_rtl_indexes {
+            if rtl_index < new_text.len() {
+                let new_index = new_text.len() - rtl_index;
+                new_text.insert(new_index, '_');
+            }
+        }
+
+        // Add in additional separators if necessary.
+        if new_text.len() > old_length && !separator_rtl_indexes.is_empty() {
+            let spacing = match separator_rtl_indexes.as_slice() {
+                [.., b, a] => a - b - 1,
+                _ => separator_rtl_indexes[0],
+            };
+
+            let prefix_length = if self.radix == 10 { 0 } else { 2 };
+            if let Some(mut index) = new_text.find('_') {
+                while index - prefix_length > spacing {
+                    index -= spacing;
+                    new_text.insert(index, '_');
+                }
+            }
+        }
+
+        (self.range, new_text.into())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::Rope;
+
+    #[test]
+    fn test_decimal_at_point() {
+        let rope = Rope::from_str("Test text 12345 more text.");
+        let range = Range::point(12);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(10, 15),
+                value: 12345,
+                radix: 10,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_uppercase_hexadecimal_at_point() {
+        let rope = Rope::from_str("Test text 0x123ABCDEF more text.");
+        let range = Range::point(12);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(10, 21),
+                value: 0x123ABCDEF,
+                radix: 16,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_lowercase_hexadecimal_at_point() {
+        let rope = Rope::from_str("Test text 0xfa3b4e more text.");
+        let range = Range::point(12);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(10, 18),
+                value: 0xfa3b4e,
+                radix: 16,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_octal_at_point() {
+        let rope = Rope::from_str("Test text 0o1074312 more text.");
+        let range = Range::point(12);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(10, 19),
+                value: 0o1074312,
+                radix: 8,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_binary_at_point() {
+        let rope = Rope::from_str("Test text 0b10111010010101 more text.");
+        let range = Range::point(12);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(10, 26),
+                value: 0b10111010010101,
+                radix: 2,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_negative_decimal_at_point() {
+        let rope = Rope::from_str("Test text -54321 more text.");
+        let range = Range::point(12);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(10, 16),
+                value: -54321,
+                radix: 10,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_decimal_with_leading_zeroes_at_point() {
+        let rope = Rope::from_str("Test text 000045326 more text.");
+        let range = Range::point(12);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(10, 19),
+                value: 45326,
+                radix: 10,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_negative_decimal_cursor_on_minus_sign() {
+        let rope = Rope::from_str("Test text -54321 more text.");
+        let range = Range::point(10);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(10, 16),
+                value: -54321,
+                radix: 10,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_number_under_range_start_of_rope() {
+        let rope = Rope::from_str("100");
+        let range = Range::point(0);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(0, 3),
+                value: 100,
+                radix: 10,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_number_under_range_end_of_rope() {
+        let rope = Rope::from_str("100");
+        let range = Range::point(2);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(0, 3),
+                value: 100,
+                radix: 10,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_number_surrounded_by_punctuation() {
+        let rope = Rope::from_str(",100;");
+        let range = Range::point(1);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(1, 4),
+                value: 100,
+                radix: 10,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_not_a_number_point() {
+        let rope = Rope::from_str("Test text 45326 more text.");
+        let range = Range::point(6);
+        assert_eq!(NumberIncrementor::from_range(rope.slice(..), range), None);
+    }
+
+    #[test]
+    fn test_number_too_large_at_point() {
+        let rope = Rope::from_str("Test text 0xFFFFFFFFFFFFFFFFF more text.");
+        let range = Range::point(12);
+        assert_eq!(NumberIncrementor::from_range(rope.slice(..), range), None);
+    }
+
+    #[test]
+    fn test_number_cursor_one_right_of_number() {
+        let rope = Rope::from_str("100 ");
+        let range = Range::point(3);
+        assert_eq!(NumberIncrementor::from_range(rope.slice(..), range), None);
+    }
+
+    #[test]
+    fn test_number_cursor_one_left_of_number() {
+        let rope = Rope::from_str(" 100");
+        let range = Range::point(0);
+        assert_eq!(NumberIncrementor::from_range(rope.slice(..), range), None);
+    }
+
+    #[test]
+    fn test_increment_basic_decimal_numbers() {
+        let tests = [
+            ("100", 1, "101"),
+            ("100", -1, "99"),
+            ("99", 1, "100"),
+            ("100", 1000, "1100"),
+            ("100", -1000, "-900"),
+            ("-1", 1, "0"),
+            ("-1", 2, "1"),
+            ("1", -1, "0"),
+            ("1", -2, "-1"),
+        ];
+
+        for (original, amount, expected) in tests {
+            let rope = Rope::from_str(original);
+            let range = Range::point(0);
+            assert_eq!(
+                NumberIncrementor::from_range(rope.slice(..), range)
+                    .unwrap()
+                    .increment(amount)
+                    .1,
+                Tendril::from(expected)
+            );
+        }
+    }
+
+    #[test]
+    fn test_increment_basic_hexadedimal_numbers() {
+        let tests = [
+            ("0x0100", 1, "0x0101"),
+            ("0x0100", -1, "0x00ff"),
+            ("0x0001", -1, "0x0000"),
+            ("0x0000", -1, "0xffffffffffffffff"),
+            ("0xffffffffffffffff", 1, "0x0000000000000000"),
+            ("0xffffffffffffffff", 2, "0x0000000000000001"),
+            ("0xffffffffffffffff", -1, "0xfffffffffffffffe"),
+            ("0xABCDEF1234567890", 1, "0xABCDEF1234567891"),
+            ("0xabcdef1234567890", 1, "0xabcdef1234567891"),
+        ];
+
+        for (original, amount, expected) in tests {
+            let rope = Rope::from_str(original);
+            let range = Range::point(0);
+            assert_eq!(
+                NumberIncrementor::from_range(rope.slice(..), range)
+                    .unwrap()
+                    .increment(amount)
+                    .1,
+                Tendril::from(expected)
+            );
+        }
+    }
+
+    #[test]
+    fn test_increment_basic_octal_numbers() {
+        let tests = [
+            ("0o0107", 1, "0o0110"),
+            ("0o0110", -1, "0o0107"),
+            ("0o0001", -1, "0o0000"),
+            ("0o7777", 1, "0o10000"),
+            ("0o1000", -1, "0o0777"),
+            ("0o0107", 10, "0o0121"),
+            ("0o0000", -1, "0o1777777777777777777777"),
+            ("0o1777777777777777777777", 1, "0o0000000000000000000000"),
+            ("0o1777777777777777777777", 2, "0o0000000000000000000001"),
+            ("0o1777777777777777777777", -1, "0o1777777777777777777776"),
+        ];
+
+        for (original, amount, expected) in tests {
+            let rope = Rope::from_str(original);
+            let range = Range::point(0);
+            assert_eq!(
+                NumberIncrementor::from_range(rope.slice(..), range)
+                    .unwrap()
+                    .increment(amount)
+                    .1,
+                Tendril::from(expected)
+            );
+        }
+    }
+
+    #[test]
+    fn test_increment_basic_binary_numbers() {
+        let tests = [
+            ("0b00000100", 1, "0b00000101"),
+            ("0b00000100", -1, "0b00000011"),
+            ("0b00000100", 2, "0b00000110"),
+            ("0b00000100", -2, "0b00000010"),
+            ("0b00000001", -1, "0b00000000"),
+            ("0b00111111", 10, "0b01001001"),
+            ("0b11111111", 1, "0b100000000"),
+            ("0b10000000", -1, "0b01111111"),
+            (
+                "0b0000",
+                -1,
+                "0b1111111111111111111111111111111111111111111111111111111111111111",
+            ),
+            (
+                "0b1111111111111111111111111111111111111111111111111111111111111111",
+                1,
+                "0b0000000000000000000000000000000000000000000000000000000000000000",
+            ),
+            (
+                "0b1111111111111111111111111111111111111111111111111111111111111111",
+                2,
+                "0b0000000000000000000000000000000000000000000000000000000000000001",
+            ),
+            (
+                "0b1111111111111111111111111111111111111111111111111111111111111111",
+                -1,
+                "0b1111111111111111111111111111111111111111111111111111111111111110",
+            ),
+        ];
+
+        for (original, amount, expected) in tests {
+            let rope = Rope::from_str(original);
+            let range = Range::point(0);
+            assert_eq!(
+                NumberIncrementor::from_range(rope.slice(..), range)
+                    .unwrap()
+                    .increment(amount)
+                    .1,
+                Tendril::from(expected)
+            );
+        }
+    }
+
+    #[test]
+    fn test_increment_with_separators() {
+        let tests = [
+            ("999_999", 1, "1_000_000"),
+            ("1_000_000", -1, "999_999"),
+            ("-999_999", -1, "-1_000_000"),
+            ("0x0000_0000_0001", 0x1_ffff_0000, "0x0001_ffff_0001"),
+            ("0x0000_0000_0001", 0x1_ffff_0000, "0x0001_ffff_0001"),
+            ("0x0000_0000_0001", 0x1_ffff_0000, "0x0001_ffff_0001"),
+            ("0x0000_0000", -1, "0xffff_ffff_ffff_ffff"),
+            ("0x0000_0000_0000", -1, "0xffff_ffff_ffff_ffff"),
+            ("0b01111111_11111111", 1, "0b10000000_00000000"),
+            ("0b11111111_11111111", 1, "0b1_00000000_00000000"),
+        ];
+
+        for (original, amount, expected) in tests {
+            let rope = Rope::from_str(original);
+            let range = Range::point(0);
+            assert_eq!(
+                NumberIncrementor::from_range(rope.slice(..), range)
+                    .unwrap()
+                    .increment(amount)
+                    .1,
+                Tendril::from(expected)
+            );
+        }
+    }
+}

helix-core/src/indent.rs

@@ -1,6 +1,5 @@
 use crate::{
     chars::{char_is_line_ending, char_is_whitespace},
-    find_first_non_whitespace_char,
     syntax::{IndentQuery, LanguageConfiguration, Syntax},
     tree_sitter::Node,
     Rope, RopeSlice,
@@ -174,8 +173,7 @@ pub fn auto_detect_indent_style(document_text: &Rope) -> Option<IndentStyle> {
 
 /// To determine indentation of a newly inserted line, figure out the indentation at the last col
 /// of the previous line.
-#[allow(dead_code)]
+pub fn indent_level_for_line(line: RopeSlice, tab_width: usize) -> usize {
-fn indent_level_for_line(line: RopeSlice, tab_width: usize) -> usize {
     let mut len = 0;
     for ch in line.chars() {
         match ch {
@@ -194,10 +192,7 @@ fn get_highest_syntax_node_at_bytepos(syntax: &Syntax, pos: usize) -> Option<Nod
     let tree = syntax.tree();
 
     // named_descendant
-    let mut node = match tree.root_node().descendant_for_byte_range(pos, pos) {
+    let mut node = tree.root_node().descendant_for_byte_range(pos, pos)?;
-        Some(node) => node,
-        None => return None,
-    };
 
     while let Some(parent) = node.parent() {
         if parent.start_byte() == node.start_byte() {
@@ -210,10 +205,15 @@ fn get_highest_syntax_node_at_bytepos(syntax: &Syntax, pos: usize) -> Option<Nod
     Some(node)
 }
 
-fn calculate_indentation(query: &IndentQuery, node: Option<Node>, newline: bool) -> usize {
+/// Calculate the indentation at a given treesitter node.
-    // NOTE: can't use contains() on query because of comparing Vec<String> and &str
+/// If newline is false, then any "indent" nodes on the line are ignored ("outdent" still applies).
-    // https://doc.rust-lang.org/std/vec/struct.Vec.html#method.contains
+/// This is because the indentation is only increased starting at the second line of the node.
-
+fn calculate_indentation(
+    query: &IndentQuery,
+    node: Option<Node>,
+    line: usize,
+    newline: bool,
+) -> usize {
     let mut increment: isize = 0;
 
     let mut node = match node {
@@ -221,70 +221,45 @@ fn calculate_indentation(query: &IndentQuery, node: Option<Node>, newline: bool)
         None => return 0,
     };
 
-    let mut prev_start = node.start_position().row;
+    let mut current_line = line;
+    let mut consider_indent = newline;
+    let mut increment_from_line: isize = 0;
 
-    // if we're calculating indentation for a brand new line then the current node will become the
+    loop {
-    // parent node. We need to take it's indentation level into account too.
         let node_kind = node.kind();
-    if newline && query.indent.contains(node_kind) {
+        let start = node.start_position().row;
-        increment += 1;
+        if current_line != start {
-    }
+            // Indent/dedent by at most one per line:
-
-    while let Some(parent) = node.parent() {
-        let parent_kind = parent.kind();
-        let start = parent.start_position().row;
-
-        // detect deeply nested indents in the same line
             // .map(|a| {       <-- ({ is two scopes
             //     let len = 1; <-- indents one level
             // })               <-- }) is two scopes
-        let starts_same_line = start == prev_start;
+            if consider_indent || increment_from_line < 0 {
-
+                increment += increment_from_line.signum();
-        if query.outdent.contains(node.kind()) && !starts_same_line {
+            }
-            // we outdent by skipping the rules for the current level and jumping up
+            increment_from_line = 0;
-            // node = parent;
+            current_line = start;
-            increment -= 1;
+            consider_indent = true;
-            // continue;
         }
 
-        if query.indent.contains(parent_kind) // && not_first_or_last_sibling
+        if query.outdent.contains(node_kind) {
-            && !starts_same_line
+            increment_from_line -= 1;
-        {
+        }
-            // println!("is_scope {}", parent_kind);
+        if query.indent.contains(node_kind) {
-            prev_start = start;
+            increment_from_line += 1;
-            increment += 1
         }
 
-        // if last_scope && increment > 0 && ...{ ignore }
+        if let Some(parent) = node.parent() {
-
             node = parent;
+        } else {
+            break;
+        }
+    }
+    if consider_indent || increment_from_line < 0 {
+        increment += increment_from_line.signum();
     }
-
     increment.max(0) as usize
 }
 
-#[allow(dead_code)]
-fn suggested_indent_for_line(
-    language_config: &LanguageConfiguration,
-    syntax: Option<&Syntax>,
-    text: RopeSlice,
-    line_num: usize,
-    _tab_width: usize,
-) -> usize {
-    if let Some(start) = find_first_non_whitespace_char(text.line(line_num)) {
-        return suggested_indent_for_pos(
-            Some(language_config),
-            syntax,
-            text,
-            start + text.line_to_char(line_num),
-            false,
-        );
-    };
-
-    // if the line is blank, indent should be zero
-    0
-}
-
 // TODO: two usecases: if we are triggering this for a new, blank line:
 // - it should return 0 when mass indenting stuff
 // - it should look up the wrapper node and count it too when we press o/O
@@ -293,26 +268,48 @@ pub fn suggested_indent_for_pos(
     syntax: Option<&Syntax>,
     text: RopeSlice,
     pos: usize,
+    line: usize,
     new_line: bool,
-) -> usize {
+) -> Option<usize> {
     if let (Some(query), Some(syntax)) = (
         language_config.and_then(|config| config.indent_query()),
         syntax,
     ) {
         let byte_start = text.char_to_byte(pos);
         let node = get_highest_syntax_node_at_bytepos(syntax, byte_start);
-
-        // let config = load indentation query config from Syntax(should contain language_config)
-
         // TODO: special case for comments
         // TODO: if preserve_leading_whitespace
-        calculate_indentation(query, node, new_line)
+        Some(calculate_indentation(query, node, line, new_line))
     } else {
-        // TODO: heuristics for non-tree sitter grammars
+        None
-        0
     }
 }
 
+pub fn get_scopes(syntax: Option<&Syntax>, text: RopeSlice, pos: usize) -> Vec<&'static str> {
+    let mut scopes = Vec::new();
+    if let Some(syntax) = syntax {
+        let pos = text.char_to_byte(pos);
+        let mut node = match syntax
+            .tree()
+            .root_node()
+            .descendant_for_byte_range(pos, pos)
+        {
+            Some(node) => node,
+            None => return scopes,
+        };
+
+        scopes.push(node.kind());
+
+        while let Some(parent) = node.parent() {
+            scopes.push(parent.kind());
+            node = parent;
+        }
+    }
+
+    scopes.reverse();
+    scopes
+}
+
 #[cfg(test)]
 mod test {
     use super::*;
@@ -416,7 +413,8 @@ where
 ",
         );
 
-        let doc = Rope::from(doc);
+        let doc = doc;
+        use crate::diagnostic::Severity;
         use crate::syntax::{
             Configuration, IndentationConfiguration, LanguageConfiguration, Loader,
         };
@@ -425,19 +423,26 @@ where
             language: vec![LanguageConfiguration {
                 scope: "source.rust".to_string(),
                 file_types: vec!["rs".to_string()],
+                shebangs: vec![],
                 language_id: "Rust".to_string(),
                 highlight_config: OnceCell::new(),
                 config: None,
                 //
+                injection_regex: None,
                 roots: vec![],
                 comment_token: None,
                 auto_format: false,
+                diagnostic_severity: Severity::Warning,
+                grammar: None,
                 language_server: None,
                 indent: Some(IndentationConfiguration {
                     tab_width: 4,
                     unit: String::from("    "),
                 }),
                 indent_query: OnceCell::new(),
+                textobject_query: OnceCell::new(),
+                debugger: None,
+                auto_pairs: None,
             }],
         });
 
@@ -448,20 +453,29 @@ where
 
         let language_config = loader.language_config_for_scope("source.rust").unwrap();
         let highlight_config = language_config.highlight_config(&[]).unwrap();
-        let syntax = Syntax::new(&doc, highlight_config.clone());
+        let syntax = Syntax::new(&doc, highlight_config, std::sync::Arc::new(loader));
         let text = doc.slice(..);
         let tab_width = 4;
 
         for i in 0..doc.len_lines() {
             let line = text.line(i);
+            if let Some(pos) = crate::find_first_non_whitespace_char(line) {
                 let indent = indent_level_for_line(line, tab_width);
                 assert_eq!(
-                suggested_indent_for_line(&language_config, Some(&syntax), text, i, tab_width),
+                    suggested_indent_for_pos(
-                indent,
+                        Some(&language_config),
-                "line {}: {}",
+                        Some(&syntax),
+                        text,
+                        text.line_to_char(i) + pos,
+                        i,
+                        false
+                    ),
+                    Some(indent),
+                    "line {}: \"{}\"",
                     i,
                     line
                 );
             }
         }
     }
+}

helix-core/src/lib.rs

@@ -1,20 +1,26 @@
+pub use encoding_rs as encoding;
+
 pub mod auto_pairs;
 pub mod chars;
 pub mod comment;
+pub mod config;
 pub mod diagnostic;
 pub mod diff;
 pub mod graphemes;
 pub mod history;
+pub mod increment;
 pub mod indent;
 pub mod line_ending;
 pub mod macros;
 pub mod match_brackets;
 pub mod movement;
 pub mod object;
+pub mod path;
 mod position;
 pub mod register;
 pub mod search;
 pub mod selection;
+pub mod shellwords;
 mod state;
 pub mod surround;
 pub mod syntax;
@@ -27,14 +33,18 @@ pub mod unicode {
     pub use unicode_width as width;
 }
 
-static RUNTIME_DIR: once_cell::sync::Lazy<std::path::PathBuf> =
-    once_cell::sync::Lazy::new(runtime_dir);
-
 pub fn find_first_non_whitespace_char(line: RopeSlice) -> Option<usize> {
     line.chars().position(|ch| !ch.is_whitespace())
 }
 
-pub fn find_root(root: Option<&str>) -> Option<std::path::PathBuf> {
+/// Find project root.
+///
+/// Order of detection:
+/// * Top-most folder containing a root marker in current git repository
+/// * Git repostory root if no marker detected
+/// * Top-most folder containing a root marker if not git repository detected
+/// * Current working directory as fallback
+pub fn find_root(root: Option<&str>, root_markers: &[String]) -> Option<std::path::PathBuf> {
     let current_dir = std::env::current_dir().expect("unable to determine current directory");
 
     let root = match root {
@@ -46,72 +56,46 @@ pub fn find_root(root: Option<&str>) -> Option<std::path::PathBuf> {
                 current_dir.join(root)
             }
         }
-        None => current_dir,
+        None => current_dir.clone(),
     };
 
+    let mut top_marker = None;
     for ancestor in root.ancestors() {
-        // TODO: also use defined roots if git isn't found
+        for marker in root_markers {
+            if ancestor.join(marker).exists() {
+                top_marker = Some(ancestor);
+                break;
+            }
+        }
+        // don't go higher than repo
         if ancestor.join(".git").is_dir() {
-            return Some(ancestor.to_path_buf());
+            // Use workspace if detected from marker
+            return Some(top_marker.unwrap_or(ancestor).to_path_buf());
         }
     }
-    None
-}
 
-pub fn runtime_dir() -> std::path::PathBuf {
+    // In absence of git repo, use workspace if detected
-    if let Ok(dir) = std::env::var("HELIX_RUNTIME") {
+    if top_marker.is_some() {
-        return dir.into();
+        top_marker.map(|a| a.to_path_buf())
+    } else {
+        Some(current_dir)
     }
-
-    const RT_DIR: &str = "runtime";
-    let conf_dir = config_dir().join(RT_DIR);
-    if conf_dir.exists() {
-        return conf_dir;
 }
 
-    if let Ok(dir) = std::env::var("CARGO_MANIFEST_DIR") {
-        // this is the directory of the crate being run by cargo, we need the workspace path so we take the parent
-        return std::path::PathBuf::from(dir).parent().unwrap().join(RT_DIR);
-    }
-
-    // fallback to location of the executable being run
-    std::env::current_exe()
-        .ok()
-        .and_then(|path| path.parent().map(|path| path.to_path_buf().join(RT_DIR)))
-        .unwrap()
-}
-
-pub fn config_dir() -> std::path::PathBuf {
-    // TODO: allow env var override
-    let strategy = choose_base_strategy().expect("Unable to find the config directory!");
-    let mut path = strategy.config_dir();
-    path.push("helix");
-    path
-}
-
-pub fn cache_dir() -> std::path::PathBuf {
-    // TODO: allow env var override
-    let strategy = choose_base_strategy().expect("Unable to find the config directory!");
-    let mut path = strategy.cache_dir();
-    path.push("helix");
-    path
-}
-
-pub use etcetera::home_dir;
-
-use etcetera::base_strategy::{choose_base_strategy, BaseStrategy};
-
 pub use ropey::{Rope, RopeBuilder, RopeSlice};
 
-pub use tendril::StrTendril as Tendril;
+// pub use tendril::StrTendril as Tendril;
+pub use smartstring::SmartString;
+
+pub type Tendril = SmartString<smartstring::LazyCompact>;
 
 #[doc(inline)]
 pub use {regex, tree_sitter};
 
 pub use graphemes::RopeGraphemes;
-pub use position::{coords_at_pos, pos_at_coords, Position};
+pub use position::{coords_at_pos, pos_at_coords, visual_coords_at_pos, Position};
 pub use selection::{Range, Selection};
-pub use smallvec::SmallVec;
+pub use smallvec::{smallvec, SmallVec};
 pub use syntax::Syntax;
 
 pub use diagnostic::Diagnostic;

helix-core/src/line_ending.rs

@@ -20,7 +20,7 @@ pub enum LineEnding {
 
 impl LineEnding {
     #[inline]
-    pub fn len_chars(&self) -> usize {
+    pub const fn len_chars(&self) -> usize {
         match self {
             Self::Crlf => 2,
             _ => 1,
@@ -28,7 +28,7 @@ impl LineEnding {
     }
 
     #[inline]
-    pub fn as_str(&self) -> &'static str {
+    pub const fn as_str(&self) -> &'static str {
         match self {
             Self::Crlf => "\u{000D}\u{000A}",
             Self::LF => "\u{000A}",
@@ -42,7 +42,7 @@ impl LineEnding {
     }
 
     #[inline]
-    pub fn from_char(ch: char) -> Option<LineEnding> {
+    pub const fn from_char(ch: char) -> Option<LineEnding> {
         match ch {
             '\u{000A}' => Some(LineEnding::LF),
             '\u{000B}' => Some(LineEnding::VT),
@@ -250,7 +250,7 @@ mod line_ending_tests {
         assert_eq!(get_line_ending_of_str(&text[..6]), Some(LineEnding::CR));
         assert_eq!(get_line_ending_of_str(&text[..12]), Some(LineEnding::LF));
         assert_eq!(get_line_ending_of_str(&text[..17]), Some(LineEnding::Crlf));
-        assert_eq!(get_line_ending_of_str(&text[..]), None);
+        assert_eq!(get_line_ending_of_str(text), None);
     }
 
     #[test]

helix-core/src/match_brackets.rs

@@ -1,48 +1,92 @@
+use tree_sitter::Node;
+
 use crate::{Rope, Syntax};
 
-const PAIRS: &[(char, char)] = &[('(', ')'), ('{', '}'), ('[', ']'), ('<', '>')];
+const PAIRS: &[(char, char)] = &[
-// limit matching pairs to only ( ) { } [ ] < >
+    ('(', ')'),
+    ('{', '}'),
+    ('[', ']'),
+    ('<', '>'),
+    ('\'', '\''),
+    ('\"', '\"'),
+];
 
+// limit matching pairs to only ( ) { } [ ] < > ' ' " "
+
+// Returns the position of the matching bracket under cursor.
+//
+// If the cursor is one the opening bracket, the position of
+// the closing bracket is returned. If the cursor in the closing
+// bracket, the position of the opening bracket is returned.
+//
+// If the cursor is not on a bracket, `None` is returned.
 #[must_use]
-pub fn find(syntax: &Syntax, doc: &Rope, pos: usize) -> Option<usize> {
+pub fn find_matching_bracket(syntax: &Syntax, doc: &Rope, pos: usize) -> Option<usize> {
-    let tree = syntax.tree();
+    if pos >= doc.len_chars() || !is_valid_bracket(doc.char(pos)) {
-
-    let byte_pos = doc.char_to_byte(pos);
-
-    // most naive implementation: find the innermost syntax node, if we're at the edge of a node,
-    // return the other edge.
-
-    let node = match tree
-        .root_node()
-        .named_descendant_for_byte_range(byte_pos, byte_pos)
-    {
-        Some(node) => node,
-        None => return None,
-    };
-
-    if node.is_error() {
         return None;
     }
+    find_pair(syntax, doc, pos, false)
+}
 
+// Returns the position of the bracket that is closing the current scope.
+//
+// If the cursor is on an opening or closing bracket, the function
+// behaves equivalent to [`find_matching_bracket`].
+//
+// If the cursor position is within a scope, the function searches
+// for the surrounding scope that is surrounded by brackets and
+// returns the position of the closing bracket for that scope.
+//
+// If no surrounding scope is found, the function returns `None`.
+#[must_use]
+pub fn find_matching_bracket_fuzzy(syntax: &Syntax, doc: &Rope, pos: usize) -> Option<usize> {
+    find_pair(syntax, doc, pos, true)
+}
+
+fn find_pair(syntax: &Syntax, doc: &Rope, pos: usize, traverse_parents: bool) -> Option<usize> {
+    let tree = syntax.tree();
+    let pos = doc.char_to_byte(pos);
+
+    let mut node = tree.root_node().named_descendant_for_byte_range(pos, pos)?;
+
+    loop {
+        let (start_byte, end_byte) = surrounding_bytes(doc, &node)?;
+        let (start_char, end_char) = (doc.byte_to_char(start_byte), doc.byte_to_char(end_byte));
+
+        if is_valid_pair(doc, start_char, end_char) {
+            if end_byte == pos {
+                return Some(start_char);
+            }
+            // We return the end char if the cursor is either on the start char
+            // or at some arbitrary position between start and end char.
+            return Some(end_char);
+        }
+
+        if traverse_parents {
+            node = node.parent()?;
+        } else {
+            return None;
+        }
+    }
+}
+
+fn is_valid_bracket(c: char) -> bool {
+    PAIRS.iter().any(|(l, r)| *l == c || *r == c)
+}
+
+fn is_valid_pair(doc: &Rope, start_char: usize, end_char: usize) -> bool {
+    PAIRS.contains(&(doc.char(start_char), doc.char(end_char)))
+}
+
+fn surrounding_bytes(doc: &Rope, node: &Node) -> Option<(usize, usize)> {
     let len = doc.len_bytes();
+
     let start_byte = node.start_byte();
-    let end_byte = node.end_byte() - 1; // it's end exclusive
+    let end_byte = node.end_byte().saturating_sub(1);
+
     if start_byte >= len || end_byte >= len {
         return None;
     }
 
-    let start_char = doc.byte_to_char(start_byte);
+    Some((start_byte, end_byte))
-    let end_char = doc.byte_to_char(end_byte);
-
-    if PAIRS.contains(&(doc.char(start_char), doc.char(end_char))) {
-        if start_byte == byte_pos {
-            return Some(end_char);
-        }
-
-        if end_byte == byte_pos {
-            return Some(start_char);
-        }
-    }
-
-    None
 }

helix-core/src/movement.rs

@@ -1,6 +1,7 @@
 use std::iter;
 
 use ropey::iter::Chars;
+use tree_sitter::{Node, QueryCursor};
 
 use crate::{
     chars::{categorize_char, char_is_line_ending, CharCategory},
@@ -9,7 +10,10 @@ use crate::{
         next_grapheme_boundary, nth_next_grapheme_boundary, nth_prev_grapheme_boundary,
         prev_grapheme_boundary,
     },
-    pos_at_coords, Position, Range, RopeSlice,
+    pos_at_coords,
+    syntax::LanguageConfiguration,
+    textobject::TextObject,
+    Position, Range, RopeSlice,
 };
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
@@ -53,6 +57,10 @@ pub fn move_vertically(
     let pos = range.cursor(slice);
 
     // Compute the current position's 2d coordinates.
+    // TODO: switch this to use `visual_coords_at_pos` rather than
+    // `coords_at_pos` as this will cause a jerky movement when the visual
+    // position does not match, like moving from a line with tabs/CJK to
+    // a line without
     let Position { row, col } = coords_at_pos(slice, pos);
     let horiz = range.horiz.unwrap_or(col as u32);
 
@@ -164,7 +172,7 @@ pub fn backwards_skip_while<F>(slice: RopeSlice, pos: usize, fun: F) -> Option<u
 where
     F: Fn(char) -> bool,
 {
-    let mut chars_starting_from_next = slice.chars_at(pos + 1);
+    let mut chars_starting_from_next = slice.chars_at(pos);
     let mut backwards = iter::from_fn(|| chars_starting_from_next.prev()).enumerate();
     backwards.find_map(|(i, c)| {
         if !fun(c) {
@@ -301,10 +309,58 @@ fn reached_target(target: WordMotionTarget, prev_ch: char, next_ch: char) -> boo
     }
 }
 
+pub fn goto_treesitter_object(
+    slice: RopeSlice,
+    range: Range,
+    object_name: &str,
+    dir: Direction,
+    slice_tree: Node,
+    lang_config: &LanguageConfiguration,
+    _count: usize,
+) -> Range {
+    let get_range = move || -> Option<Range> {
+        let byte_pos = slice.char_to_byte(range.cursor(slice));
+
+        let cap_name = |t: TextObject| format!("{}.{}", object_name, t);
+        let mut cursor = QueryCursor::new();
+        let nodes = lang_config.textobject_query()?.capture_nodes_any(
+            &[
+                &cap_name(TextObject::Movement),
+                &cap_name(TextObject::Around),
+                &cap_name(TextObject::Inside),
+            ],
+            slice_tree,
+            slice,
+            &mut cursor,
+        )?;
+
+        let node = match dir {
+            Direction::Forward => nodes
+                .filter(|n| n.start_byte() > byte_pos)
+                .min_by_key(|n| n.start_byte())?,
+            Direction::Backward => nodes
+                .filter(|n| n.start_byte() < byte_pos)
+                .max_by_key(|n| n.start_byte())?,
+        };
+
+        let len = slice.len_bytes();
+        let start_byte = node.start_byte();
+        let end_byte = node.end_byte();
+        if start_byte >= len || end_byte >= len {
+            return None;
+        }
+
+        let start_char = slice.byte_to_char(start_byte);
+        let end_char = slice.byte_to_char(end_byte);
+
+        // head of range should be at beginning
+        Some(Range::new(end_char, start_char))
+    };
+    get_range().unwrap_or(range)
+}
+
 #[cfg(test)]
 mod test {
-    use std::array::{self, IntoIter};
-
     use ropey::Rope;
 
     use super::*;
@@ -356,7 +412,7 @@ mod test {
             ((Direction::Backward, 999usize), (0, 0)), // |This is a simple alphabetic line
         ];
 
-        for ((direction, amount), coordinates) in IntoIter::new(moves_and_expected_coordinates) {
+        for ((direction, amount), coordinates) in moves_and_expected_coordinates {
             range = move_horizontally(slice, range, direction, amount, Movement::Move);
             assert_eq!(coords_at_pos(slice, range.head), coordinates.into())
         }
@@ -370,7 +426,7 @@ mod test {
 
         let mut range = Range::point(position);
 
-        let moves_and_expected_coordinates = IntoIter::new([
+        let moves_and_expected_coordinates = [
             ((Direction::Forward, 11usize), (1, 1)), // Multiline\nt|ext sample\n...
             ((Direction::Backward, 1usize), (1, 0)), // Multiline\n|text sample\n...
             ((Direction::Backward, 5usize), (0, 5)), // Multi|line\ntext sample\n...
@@ -380,7 +436,7 @@ mod test {
             ((Direction::Backward, 0usize), (0, 3)), // Mul|tiline\ntext sample\n...
             ((Direction::Forward, 999usize), (5, 0)), // ...and whitespaced\n|
             ((Direction::Forward, 999usize), (5, 0)), // ...and whitespaced\n|
-        ]);
+        ];
 
         for ((direction, amount), coordinates) in moves_and_expected_coordinates {
             range = move_horizontally(slice, range, direction, amount, Movement::Move);
@@ -398,11 +454,11 @@ mod test {
         let mut range = Range::point(position);
         let original_anchor = range.anchor;
 
-        let moves = IntoIter::new([
+        let moves = [
             (Direction::Forward, 1usize),
             (Direction::Forward, 5usize),
             (Direction::Backward, 3usize),
-        ]);
+        ];
 
         for (direction, amount) in moves {
             range = move_horizontally(slice, range, direction, amount, Movement::Extend);
@@ -416,7 +472,7 @@ mod test {
         let slice = text.slice(..);
         let position = pos_at_coords(slice, (0, 0).into(), true);
         let mut range = Range::point(position);
-        let moves_and_expected_coordinates = IntoIter::new([
+        let moves_and_expected_coordinates = [
             ((Direction::Forward, 1usize), (1, 0)),
             ((Direction::Forward, 2usize), (3, 0)),
             ((Direction::Forward, 1usize), (4, 0)),
@@ -426,7 +482,7 @@ mod test {
             ((Direction::Backward, 0usize), (4, 0)),
             ((Direction::Forward, 5), (5, 0)),
             ((Direction::Forward, 999usize), (5, 0)),
-        ]);
+        ];
 
         for ((direction, amount), coordinates) in moves_and_expected_coordinates {
             range = move_vertically(slice, range, direction, amount, Movement::Move);
@@ -446,7 +502,7 @@ mod test {
             H,
             V,
         }
-        let moves_and_expected_coordinates = IntoIter::new([
+        let moves_and_expected_coordinates = [
             // Places cursor at the end of line
             ((Axis::H, Direction::Forward, 8usize), (0, 8)),
             // First descent preserves column as the target line is wider
@@ -459,7 +515,7 @@ mod test {
             ((Axis::V, Direction::Backward, 999usize), (0, 8)),
             ((Axis::V, Direction::Forward, 4usize), (4, 8)),
             ((Axis::V, Direction::Forward, 999usize), (5, 0)),
-        ]);
+        ];
 
         for ((axis, direction, amount), coordinates) in moves_and_expected_coordinates {
             range = match axis {
@@ -485,7 +541,7 @@ mod test {
             H,
             V,
         }
-        let moves_and_expected_coordinates = IntoIter::new([
+        let moves_and_expected_coordinates = [
             // Places cursor at the fourth kana.
             ((Axis::H, Direction::Forward, 4), (0, 4)),
             // Descent places cursor at the 4th character.
@@ -494,7 +550,7 @@ mod test {
             ((Axis::H, Direction::Backward, 1usize), (1, 3)),
             // Jumping back up 1 line.
             ((Axis::V, Direction::Backward, 1usize), (0, 3)),
-        ]);
+        ];
 
         for ((axis, direction, amount), coordinates) in moves_and_expected_coordinates {
             range = match axis {
@@ -526,7 +582,7 @@ mod test {
 
     #[test]
     fn test_behaviour_when_moving_to_start_of_next_words() {
-        let tests = array::IntoIter::new([
+        let tests = [
             ("Basic forward motion stops at the first space",
                 vec![(1, Range::new(0, 0), Range::new(0, 6))]),
             (" Starting from a boundary advances the anchor",
@@ -600,7 +656,7 @@ mod test {
                 vec![
                     (1, Range::new(0, 0), Range::new(0, 6)),
                 ]),
-        ]);
+        ];
 
         for (sample, scenario) in tests {
             for (count, begin, expected_end) in scenario.into_iter() {
@@ -612,7 +668,7 @@ mod test {
 
     #[test]
     fn test_behaviour_when_moving_to_start_of_next_long_words() {
-        let tests = array::IntoIter::new([
+        let tests = [
             ("Basic forward motion stops at the first space",
                 vec![(1, Range::new(0, 0), Range::new(0, 6))]),
             (" Starting from a boundary advances the anchor",
@@ -684,7 +740,7 @@ mod test {
                 vec![
                     (1, Range::new(0, 0), Range::new(0, 8)),
                 ]),
-        ]);
+        ];
 
         for (sample, scenario) in tests {
             for (count, begin, expected_end) in scenario.into_iter() {
@@ -696,7 +752,7 @@ mod test {
 
     #[test]
     fn test_behaviour_when_moving_to_start_of_previous_words() {
-        let tests = array::IntoIter::new([
+        let tests = [
             ("Basic backward motion from the middle of a word",
                 vec![(1, Range::new(3, 3), Range::new(4, 0))]),
 
@@ -769,7 +825,7 @@ mod test {
                 vec![
                     (1, Range::new(0, 6), Range::new(6, 0)),
                 ]),
-        ]);
+        ];
 
         for (sample, scenario) in tests {
             for (count, begin, expected_end) in scenario.into_iter() {
@@ -781,7 +837,7 @@ mod test {
 
     #[test]
     fn test_behaviour_when_moving_to_start_of_previous_long_words() {
-        let tests = array::IntoIter::new([
+        let tests = [
             (
                 "Basic backward motion from the middle of a word",
                 vec![(1, Range::new(3, 3), Range::new(4, 0))],
@@ -866,7 +922,7 @@ mod test {
                 vec![
                     (1, Range::new(0, 8), Range::new(8, 0)),
                 ]),
-        ]);
+        ];
 
         for (sample, scenario) in tests {
             for (count, begin, expected_end) in scenario.into_iter() {
@@ -878,7 +934,7 @@ mod test {
 
     #[test]
     fn test_behaviour_when_moving_to_end_of_next_words() {
-        let tests = array::IntoIter::new([
+        let tests = [
             ("Basic forward motion from the start of a word to the end of it",
                 vec![(1, Range::new(0, 0), Range::new(0, 5))]),
             ("Basic forward motion from the end of a word to the end of the next",
@@ -950,7 +1006,7 @@ mod test {
                 vec![
                     (1, Range::new(0, 0), Range::new(0, 5)),
                 ]),
-        ]);
+        ];
 
         for (sample, scenario) in tests {
             for (count, begin, expected_end) in scenario.into_iter() {
@@ -962,7 +1018,7 @@ mod test {
 
     #[test]
     fn test_behaviour_when_moving_to_end_of_previous_words() {
-        let tests = array::IntoIter::new([
+        let tests = [
             ("Basic backward motion from the middle of a word",
                 vec![(1, Range::new(9, 9), Range::new(10, 5))]),
             ("Starting from after boundary retreats the anchor",
@@ -1032,7 +1088,7 @@ mod test {
                 vec![
                     (1, Range::new(0, 10), Range::new(10, 4)),
                 ]),
-        ]);
+        ];
 
         for (sample, scenario) in tests {
             for (count, begin, expected_end) in scenario.into_iter() {
@@ -1044,7 +1100,7 @@ mod test {
 
     #[test]
     fn test_behaviour_when_moving_to_end_of_next_long_words() {
-        let tests = array::IntoIter::new([
+        let tests = [
             ("Basic forward motion from the start of a word to the end of it",
                 vec![(1, Range::new(0, 0), Range::new(0, 5))]),
             ("Basic forward motion from the end of a word to the end of the next",
@@ -1114,7 +1170,7 @@ mod test {
                 vec![
                     (1, Range::new(0, 0), Range::new(0, 7)),
                 ]),
-        ]);
+        ];
 
         for (sample, scenario) in tests {
             for (count, begin, expected_end) in scenario.into_iter() {

helix-core/src/object.rs

@@ -1,26 +1,72 @@
 use crate::{Range, RopeSlice, Selection, Syntax};
+use tree_sitter::Node;
 
-// TODO: to contract_selection we'd need to store the previous ranges before expand.
+pub fn expand_selection(syntax: &Syntax, text: RopeSlice, selection: Selection) -> Selection {
-// Maybe just contract to the first child node?
+    select_node_impl(syntax, text, selection, |descendant, from, to| {
-pub fn expand_selection(syntax: &Syntax, text: RopeSlice, selection: &Selection) -> Selection {
+        if descendant.start_byte() == from && descendant.end_byte() == to {
+            descendant.parent()
+        } else {
+            Some(descendant)
+        }
+    })
+}
+
+pub fn shrink_selection(syntax: &Syntax, text: RopeSlice, selection: Selection) -> Selection {
+    select_node_impl(syntax, text, selection, |descendant, _from, _to| {
+        descendant.child(0).or(Some(descendant))
+    })
+}
+
+pub fn select_sibling<F>(
+    syntax: &Syntax,
+    text: RopeSlice,
+    selection: Selection,
+    sibling_fn: &F,
+) -> Selection
+where
+    F: Fn(Node) -> Option<Node>,
+{
+    select_node_impl(syntax, text, selection, |descendant, _from, _to| {
+        find_sibling_recursive(descendant, sibling_fn)
+    })
+}
+
+fn find_sibling_recursive<F>(node: Node, sibling_fn: F) -> Option<Node>
+where
+    F: Fn(Node) -> Option<Node>,
+{
+    sibling_fn(node).or_else(|| {
+        node.parent()
+            .and_then(|node| find_sibling_recursive(node, sibling_fn))
+    })
+}
+
+fn select_node_impl<F>(
+    syntax: &Syntax,
+    text: RopeSlice,
+    selection: Selection,
+    select_fn: F,
+) -> Selection
+where
+    F: Fn(Node, usize, usize) -> Option<Node>,
+{
     let tree = syntax.tree();
 
-    selection.clone().transform(|range| {
+    selection.transform(|range| {
         let from = text.char_to_byte(range.from());
         let to = text.char_to_byte(range.to());
 
-        // find parent of a descendant that matches the range
+        let node = match tree
-        let parent = match tree
             .root_node()
             .descendant_for_byte_range(from, to)
-            .and_then(|node| node.parent())
+            .and_then(|node| select_fn(node, from, to))
         {
-            Some(parent) => parent,
+            Some(node) => node,
             None => return range,
         };
 
-        let from = text.byte_to_char(parent.start_byte());
+        let from = text.byte_to_char(node.start_byte());
-        let to = text.byte_to_char(parent.end_byte());
+        let to = text.byte_to_char(node.end_byte());
 
         if range.head < range.anchor {
             Range::new(to, from)

helix-core/src/path.rs

@@ -0,0 +1,93 @@
+use etcetera::home_dir;
+use std::path::{Component, Path, PathBuf};
+
+/// Replaces users home directory from `path` with tilde `~` if the directory
+/// is available, otherwise returns the path unchanged.
+pub fn fold_home_dir(path: &Path) -> PathBuf {
+    if let Ok(home) = home_dir() {
+        if path.starts_with(&home) {
+            // it's ok to unwrap, the path starts with home dir
+            return PathBuf::from("~").join(path.strip_prefix(&home).unwrap());
+        }
+    }
+
+    path.to_path_buf()
+}
+
+/// Expands tilde `~` into users home directory if avilable, otherwise returns the path
+/// unchanged. The tilde will only be expanded when present as the first component of the path
+/// and only slash follows it.
+pub fn expand_tilde(path: &Path) -> PathBuf {
+    let mut components = path.components().peekable();
+    if let Some(Component::Normal(c)) = components.peek() {
+        if c == &"~" {
+            if let Ok(home) = home_dir() {
+                // it's ok to unwrap, the path starts with `~`
+                return home.join(path.strip_prefix("~").unwrap());
+            }
+        }
+    }
+
+    path.to_path_buf()
+}
+
+/// Normalize a path, removing things like `.` and `..`.
+///
+/// CAUTION: This does not resolve symlinks (unlike
+/// [`std::fs::canonicalize`]). This may cause incorrect or surprising
+/// behavior at times. This should be used carefully. Unfortunately,
+/// [`std::fs::canonicalize`] can be hard to use correctly, since it can often
+/// fail, or on Windows returns annoying device paths. This is a problem Cargo
+/// needs to improve on.
+/// Copied from cargo: <https://github.com/rust-lang/cargo/blob/070e459c2d8b79c5b2ac5218064e7603329c92ae/crates/cargo-util/src/paths.rs#L81>
+pub fn get_normalized_path(path: &Path) -> PathBuf {
+    let mut components = path.components().peekable();
+    let mut ret = if let Some(c @ Component::Prefix(..)) = components.peek().cloned() {
+        components.next();
+        PathBuf::from(c.as_os_str())
+    } else {
+        PathBuf::new()
+    };
+
+    for component in components {
+        match component {
+            Component::Prefix(..) => unreachable!(),
+            Component::RootDir => {
+                ret.push(component.as_os_str());
+            }
+            Component::CurDir => {}
+            Component::ParentDir => {
+                ret.pop();
+            }
+            Component::Normal(c) => {
+                ret.push(c);
+            }
+        }
+    }
+    ret
+}
+
+/// Returns the canonical, absolute form of a path with all intermediate components normalized.
+///
+/// This function is used instead of `std::fs::canonicalize` because we don't want to verify
+/// here if the path exists, just normalize it's components.
+pub fn get_canonicalized_path(path: &Path) -> std::io::Result<PathBuf> {
+    let path = expand_tilde(path);
+    let path = if path.is_relative() {
+        std::env::current_dir().map(|current_dir| current_dir.join(path))?
+    } else {
+        path
+    };
+
+    Ok(get_normalized_path(path.as_path()))
+}
+
+pub fn get_relative_path(path: &Path) -> PathBuf {
+    let path = if path.is_absolute() {
+        let cwdir = std::env::current_dir().expect("couldn't determine current directory");
+        path.strip_prefix(cwdir).unwrap_or(path)
+    } else {
+        path
+    };
+    fold_home_dir(path)
+}

helix-core/src/position.rs

@@ -1,6 +1,8 @@
+use std::borrow::Cow;
+
 use crate::{
     chars::char_is_line_ending,
-    graphemes::{ensure_grapheme_boundary_prev, RopeGraphemes},
+    graphemes::{ensure_grapheme_boundary_prev, grapheme_width, RopeGraphemes},
     line_ending::line_end_char_index,
     RopeSlice,
 };
@@ -54,11 +56,8 @@ impl From<Position> for tree_sitter::Point {
 }
 /// Convert a character index to (line, column) coordinates.
 ///
-/// TODO: this should be split into two methods: one for visual
+/// column in `char` count which can be used for row:column display in
-/// row/column, and one for "objective" row/column (possibly with
+/// status line. See [`visual_coords_at_pos`] for a visual one.
-/// the column specified in `char`s).  The former would be used
-/// for cursor movement, and the latter would be used for e.g. the
-/// row:column display in the status line.
 pub fn coords_at_pos(text: RopeSlice, pos: usize) -> Position {
     let line = text.char_to_line(pos);
 
@@ -69,6 +68,31 @@ pub fn coords_at_pos(text: RopeSlice, pos: usize) -> Position {
     Position::new(line, col)
 }
 
+/// Convert a character index to (line, column) coordinates visually.
+///
+/// Takes \t, double-width characters (CJK) into account as well as text
+/// not in the document in the future.
+/// See [`coords_at_pos`] for an "objective" one.
+pub fn visual_coords_at_pos(text: RopeSlice, pos: usize, tab_width: usize) -> Position {
+    let line = text.char_to_line(pos);
+
+    let line_start = text.line_to_char(line);
+    let pos = ensure_grapheme_boundary_prev(text, pos);
+
+    let mut col = 0;
+
+    for grapheme in RopeGraphemes::new(text.slice(line_start..pos)) {
+        if grapheme == "\t" {
+            col += tab_width - (col % tab_width);
+        } else {
+            let grapheme = Cow::from(grapheme);
+            col += grapheme_width(&grapheme);
+        }
+    }
+
+    Position::new(line, col)
+}
+
 /// Convert (line, column) coordinates to a character index.
 ///
 /// If the `line` coordinate is beyond the end of the file, the EOF
@@ -89,7 +113,10 @@ pub fn coords_at_pos(text: RopeSlice, pos: usize) -> Position {
 /// TODO: this should be changed to work in terms of visual row/column, not
 /// graphemes.
 pub fn pos_at_coords(text: RopeSlice, coords: Position, limit_before_line_ending: bool) -> usize {
-    let Position { row, col } = coords;
+    let Position { mut row, col } = coords;
+    if limit_before_line_ending {
+        row = row.min(text.len_lines() - 1);
+    };
     let line_start = text.line_to_char(row);
     let line_end = if limit_before_line_ending {
         line_end_char_index(&text, row)
@@ -130,7 +157,6 @@ mod test {
         assert_eq!(coords_at_pos(slice, 10), (1, 4).into()); // position on d
 
         // Test with wide characters.
-        // TODO: account for character width.
         let text = Rope::from("今日はいい\n");
         let slice = text.slice(..);
         assert_eq!(coords_at_pos(slice, 0), (0, 0).into());
@@ -151,7 +177,6 @@ mod test {
         assert_eq!(coords_at_pos(slice, 9), (1, 0).into());
 
         // Test with wide-character grapheme clusters.
-        // TODO: account for character width.
         let text = Rope::from("किमपि\n");
         let slice = text.slice(..);
         assert_eq!(coords_at_pos(slice, 0), (0, 0).into());
@@ -161,7 +186,6 @@ mod test {
         assert_eq!(coords_at_pos(slice, 6), (1, 0).into());
 
         // Test with tabs.
-        // Todo: account for tab stops.
         let text = Rope::from("\tHello\n");
         let slice = text.slice(..);
         assert_eq!(coords_at_pos(slice, 0), (0, 0).into());
@@ -169,6 +193,54 @@ mod test {
         assert_eq!(coords_at_pos(slice, 2), (0, 2).into());
     }
 
+    #[test]
+    fn test_visual_coords_at_pos() {
+        let text = Rope::from("ḧëḷḷö\nẅöṛḷḋ");
+        let slice = text.slice(..);
+        assert_eq!(visual_coords_at_pos(slice, 0, 8), (0, 0).into());
+        assert_eq!(visual_coords_at_pos(slice, 5, 8), (0, 5).into()); // position on \n
+        assert_eq!(visual_coords_at_pos(slice, 6, 8), (1, 0).into()); // position on w
+        assert_eq!(visual_coords_at_pos(slice, 7, 8), (1, 1).into()); // position on o
+        assert_eq!(visual_coords_at_pos(slice, 10, 8), (1, 4).into()); // position on d
+
+        // Test with wide characters.
+        let text = Rope::from("今日はいい\n");
+        let slice = text.slice(..);
+        assert_eq!(visual_coords_at_pos(slice, 0, 8), (0, 0).into());
+        assert_eq!(visual_coords_at_pos(slice, 1, 8), (0, 2).into());
+        assert_eq!(visual_coords_at_pos(slice, 2, 8), (0, 4).into());
+        assert_eq!(visual_coords_at_pos(slice, 3, 8), (0, 6).into());
+        assert_eq!(visual_coords_at_pos(slice, 4, 8), (0, 8).into());
+        assert_eq!(visual_coords_at_pos(slice, 5, 8), (0, 10).into());
+        assert_eq!(visual_coords_at_pos(slice, 6, 8), (1, 0).into());
+
+        // Test with grapheme clusters.
+        let text = Rope::from("a̐éö̲\r\n");
+        let slice = text.slice(..);
+        assert_eq!(visual_coords_at_pos(slice, 0, 8), (0, 0).into());
+        assert_eq!(visual_coords_at_pos(slice, 2, 8), (0, 1).into());
+        assert_eq!(visual_coords_at_pos(slice, 4, 8), (0, 2).into());
+        assert_eq!(visual_coords_at_pos(slice, 7, 8), (0, 3).into());
+        assert_eq!(visual_coords_at_pos(slice, 9, 8), (1, 0).into());
+
+        // Test with wide-character grapheme clusters.
+        // TODO: account for cluster.
+        let text = Rope::from("किमपि\n");
+        let slice = text.slice(..);
+        assert_eq!(visual_coords_at_pos(slice, 0, 8), (0, 0).into());
+        assert_eq!(visual_coords_at_pos(slice, 2, 8), (0, 2).into());
+        assert_eq!(visual_coords_at_pos(slice, 3, 8), (0, 3).into());
+        assert_eq!(visual_coords_at_pos(slice, 5, 8), (0, 5).into());
+        assert_eq!(visual_coords_at_pos(slice, 6, 8), (1, 0).into());
+
+        // Test with tabs.
+        let text = Rope::from("\tHello\n");
+        let slice = text.slice(..);
+        assert_eq!(visual_coords_at_pos(slice, 0, 8), (0, 0).into());
+        assert_eq!(visual_coords_at_pos(slice, 1, 8), (0, 8).into());
+        assert_eq!(visual_coords_at_pos(slice, 2, 8), (0, 9).into());
+    }
+
     #[test]
     fn test_pos_at_coords() {
         let text = Rope::from("ḧëḷḷö\nẅöṛḷḋ");
@@ -225,5 +297,12 @@ mod test {
         assert_eq!(pos_at_coords(slice, (0, 0).into(), false), 0);
         assert_eq!(pos_at_coords(slice, (0, 1).into(), false), 1);
         assert_eq!(pos_at_coords(slice, (0, 2).into(), false), 2);
+
+        // Test out of bounds.
+        let text = Rope::new();
+        let slice = text.slice(..);
+        assert_eq!(pos_at_coords(slice, (10, 0).into(), true), 0);
+        assert_eq!(pos_at_coords(slice, (0, 10).into(), true), 0);
+        assert_eq!(pos_at_coords(slice, (10, 10).into(), true), 0);
     }
 }

helix-core/src/register.rs

@@ -7,7 +7,7 @@ pub struct Register {
 }
 
 impl Register {
-    pub fn new(name: char) -> Self {
+    pub const fn new(name: char) -> Self {
         Self {
             name,
             values: Vec::new(),
@@ -15,10 +15,14 @@ impl Register {
     }
 
     pub fn new_with_values(name: char, values: Vec<String>) -> Self {
+        if name == '_' {
+            Self::new(name)
+        } else {
             Self { name, values }
         }
+    }
 
-    pub fn name(&self) -> char {
+    pub const fn name(&self) -> char {
         self.name
     }
 
@@ -27,13 +31,17 @@ impl Register {
     }
 
     pub fn write(&mut self, values: Vec<String>) {
+        if self.name != '_' {
             self.values = values;
         }
+    }
 
     pub fn push(&mut self, value: String) {
+        if self.name != '_' {
             self.values.push(value);
         }
     }
+}
 
 /// Currently just wraps a `HashMap` of `Register`s
 #[derive(Debug, Default)]
@@ -60,4 +68,8 @@ impl Registers {
     pub fn read(&self, name: char) -> Option<&[String]> {
         self.get(name).map(|reg| reg.read())
     }
+
+    pub fn inner(&self) -> &HashMap<char, Register> {
+        &self.inner
+    }
 }

helix-core/src/selection.rs

@@ -7,6 +7,7 @@ use crate::{
         ensure_grapheme_boundary_next, ensure_grapheme_boundary_prev, next_grapheme_boundary,
         prev_grapheme_boundary,
     },
+    movement::Direction,
     Assoc, ChangeSet, RopeSlice,
 };
 use smallvec::{smallvec, SmallVec};
@@ -29,10 +30,10 @@ use std::borrow::Cow;
 /// "(anchor, head)", followed by example text with "[" and "]"
 /// inserted to represent the anchor and head positions:
 ///
-/// - (0, 3): [Som]e text.
+/// - (0, 3): `[Som]e text`.
-/// - (3, 0): ]Som[e text.
+/// - (3, 0): `]Som[e text`.
-/// - (2, 7): So[me te]xt.
+/// - (2, 7): `So[me te]xt`.
-/// - (1, 1): S[]ome text.
+/// - (1, 1): `S[]ome text`.
 ///
 /// Ranges are considered to be inclusive on the left and
 /// exclusive on the right, regardless of anchor-head ordering.
@@ -82,6 +83,13 @@ impl Range {
         std::cmp::max(self.anchor, self.head)
     }
 
+    /// Total length of the range.
+    #[inline]
+    #[must_use]
+    pub fn len(&self) -> usize {
+        self.to() - self.from()
+    }
+
     /// The (inclusive) range of lines that the range overlaps.
     #[inline]
     #[must_use]
@@ -102,6 +110,27 @@ impl Range {
         self.anchor == self.head
     }
 
+    /// `Direction::Backward` when head < anchor.
+    /// `Direction::Backward` otherwise.
+    #[inline]
+    #[must_use]
+    pub fn direction(&self) -> Direction {
+        if self.head < self.anchor {
+            Direction::Backward
+        } else {
+            Direction::Forward
+        }
+    }
+
+    // flips the direction of the selection
+    pub fn flip(&self) -> Self {
+        Self {
+            anchor: self.head,
+            head: self.anchor,
+            horiz: self.horiz,
+        }
+    }
+
     /// Check two ranges for overlap.
     #[must_use]
     pub fn overlaps(&self, other: &Self) -> bool {
@@ -111,6 +140,11 @@ impl Range {
         self.from() == other.from() || (self.to() > other.from() && other.to() > self.from())
     }
 
+    #[inline]
+    pub fn contains_range(&self, other: &Self) -> bool {
+        self.from() <= other.from() && self.to() >= other.to()
+    }
+
     pub fn contains(&self, pos: usize) -> bool {
         self.from() <= pos && pos < self.to()
     }
@@ -308,10 +342,10 @@ impl Range {
 }
 
 impl From<(usize, usize)> for Range {
-    fn from(tuple: (usize, usize)) -> Self {
+    fn from((anchor, head): (usize, usize)) -> Self {
         Self {
-            anchor: tuple.0,
+            anchor,
-            head: tuple.1,
+            head,
             horiz: None,
         }
     }
@@ -360,6 +394,26 @@ impl Selection {
         self.normalize()
     }
 
+    /// Removes a range from the selection.
+    pub fn remove(mut self, index: usize) -> Self {
+        assert!(
+            self.ranges.len() > 1,
+            "can't remove the last range from a selection!"
+        );
+
+        self.ranges.remove(index);
+        if index < self.primary_index || self.primary_index == self.ranges.len() {
+            self.primary_index -= 1;
+        }
+        self
+    }
+
+    /// Replace a range in the selection with a new range.
+    pub fn replace(mut self, index: usize, range: Range) -> Self {
+        self.ranges[index] = range;
+        self.normalize()
+    }
+
     /// Map selections over a set of changes. Useful for adjusting the selection position after
     /// applying changes to a document.
     pub fn map(self, changes: &ChangeSet) -> Self {
@@ -495,6 +549,39 @@ impl Selection {
     pub fn len(&self) -> usize {
         self.ranges.len()
     }
+
+    // returns true if self ⊇ other
+    pub fn contains(&self, other: &Selection) -> bool {
+        // can't contain other if it is larger
+        if other.len() > self.len() {
+            return false;
+        }
+
+        let (mut iter_self, mut iter_other) = (self.iter(), other.iter());
+        let (mut ele_self, mut ele_other) = (iter_self.next(), iter_other.next());
+
+        loop {
+            match (ele_self, ele_other) {
+                (Some(ra), Some(rb)) => {
+                    if !ra.contains_range(rb) {
+                        // `self` doesn't contain next element from `other`, advance `self`, we need to match all from `other`
+                        ele_self = iter_self.next();
+                    } else {
+                        // matched element from `other`, advance `other`
+                        ele_other = iter_other.next();
+                    };
+                }
+                (None, Some(_)) => {
+                    // exhausted `self`, we can't match the reminder of `other`
+                    return false;
+                }
+                (_, None) => {
+                    // no elements from `other` left to match, `self` contains `other`
+                    return true;
+                }
+            }
+        }
+    }
 }
 
 impl<'a> IntoIterator for &'a Selection {
@@ -508,14 +595,15 @@ impl<'a> IntoIterator for &'a Selection {
 
 // TODO: checkSelection -> check if valid for doc length && sorted
 
-pub fn keep_matches(
+pub fn keep_or_remove_matches(
     text: RopeSlice,
     selection: &Selection,
     regex: &crate::regex::Regex,
+    remove: bool,
 ) -> Option<Selection> {
     let result: SmallVec<_> = selection
         .iter()
-        .filter(|range| regex.is_match(&range.fragment(text)))
+        .filter(|range| regex.is_match(&range.fragment(text)) ^ remove)
         .copied()
         .collect();
 
@@ -678,16 +766,16 @@ mod test {
     fn test_contains() {
         let range = Range::new(10, 12);
 
-        assert_eq!(range.contains(9), false);
+        assert!(!range.contains(9));
-        assert_eq!(range.contains(10), true);
+        assert!(range.contains(10));
-        assert_eq!(range.contains(11), true);
+        assert!(range.contains(11));
-        assert_eq!(range.contains(12), false);
+        assert!(!range.contains(12));
-        assert_eq!(range.contains(13), false);
+        assert!(!range.contains(13));
 
         let range = Range::new(9, 6);
-        assert_eq!(range.contains(9), false);
+        assert!(!range.contains(9));
-        assert_eq!(range.contains(7), true);
+        assert!(range.contains(7));
-        assert_eq!(range.contains(6), true);
+        assert!(range.contains(6));
     }
 
     #[test]
@@ -932,4 +1020,30 @@ mod test {
             &["", "abcd", "efg", "rs", "xyz"]
         );
     }
+    #[test]
+    fn test_selection_contains() {
+        fn contains(a: Vec<(usize, usize)>, b: Vec<(usize, usize)>) -> bool {
+            let sela = Selection::new(a.iter().map(|a| Range::new(a.0, a.1)).collect(), 0);
+            let selb = Selection::new(b.iter().map(|b| Range::new(b.0, b.1)).collect(), 0);
+            sela.contains(&selb)
+        }
+
+        // exact match
+        assert!(contains(vec!((1, 1)), vec!((1, 1))));
+
+        // larger set contains smaller
+        assert!(contains(vec!((1, 1), (2, 2), (3, 3)), vec!((2, 2))));
+
+        // multiple matches
+        assert!(contains(vec!((1, 1), (2, 2)), vec!((1, 1), (2, 2))));
+
+        // smaller set can't contain bigger
+        assert!(!contains(vec!((1, 1)), vec!((1, 1), (2, 2))));
+
+        assert!(contains(
+            vec!((1, 1), (2, 4), (5, 6), (7, 9), (10, 13)),
+            vec!((3, 4), (7, 9))
+        ));
+        assert!(!contains(vec!((1, 1), (5, 6)), vec!((1, 6))));
+    }
 }

helix-core/src/shellwords.rs

@@ -0,0 +1,164 @@
+use std::borrow::Cow;
+
+/// Get the vec of escaped / quoted / doublequoted filenames from the input str
+pub fn shellwords(input: &str) -> Vec<Cow<'_, str>> {
+    enum State {
+        Normal,
+        NormalEscaped,
+        Quoted,
+        QuoteEscaped,
+        Dquoted,
+        DquoteEscaped,
+    }
+
+    use State::*;
+
+    let mut state = Normal;
+    let mut args: Vec<Cow<str>> = Vec::new();
+    let mut escaped = String::with_capacity(input.len());
+
+    let mut start = 0;
+    let mut end = 0;
+
+    for (i, c) in input.char_indices() {
+        state = match state {
+            Normal => match c {
+                '\\' => {
+                    escaped.push_str(&input[start..i]);
+                    start = i + 1;
+                    NormalEscaped
+                }
+                '"' => {
+                    end = i;
+                    Dquoted
+                }
+                '\'' => {
+                    end = i;
+                    Quoted
+                }
+                c if c.is_ascii_whitespace() => {
+                    end = i;
+                    Normal
+                }
+                _ => Normal,
+            },
+            NormalEscaped => Normal,
+            Quoted => match c {
+                '\\' => {
+                    escaped.push_str(&input[start..i]);
+                    start = i + 1;
+                    QuoteEscaped
+                }
+                '\'' => {
+                    end = i;
+                    Normal
+                }
+                _ => Quoted,
+            },
+            QuoteEscaped => Quoted,
+            Dquoted => match c {
+                '\\' => {
+                    escaped.push_str(&input[start..i]);
+                    start = i + 1;
+                    DquoteEscaped
+                }
+                '"' => {
+                    end = i;
+                    Normal
+                }
+                _ => Dquoted,
+            },
+            DquoteEscaped => Dquoted,
+        };
+
+        if i >= input.len() - 1 && end == 0 {
+            end = i + 1;
+        }
+
+        if end > 0 {
+            let esc_trim = escaped.trim();
+            let inp = &input[start..end];
+
+            if !(esc_trim.is_empty() && inp.trim().is_empty()) {
+                if esc_trim.is_empty() {
+                    args.push(inp.into());
+                } else {
+                    args.push([escaped, inp.into()].concat().into());
+                    escaped = "".to_string();
+                }
+            }
+            start = i + 1;
+            end = 0;
+        }
+    }
+    args
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_normal() {
+        let input = r#":o single_word twó wörds \three\ \"with\ escaping\\"#;
+        let result = shellwords(input);
+        let expected = vec![
+            Cow::from(":o"),
+            Cow::from("single_word"),
+            Cow::from("twó"),
+            Cow::from("wörds"),
+            Cow::from(r#"three "with escaping\"#),
+        ];
+        // TODO test is_owned and is_borrowed, once they get stabilized.
+        assert_eq!(expected, result);
+    }
+
+    #[test]
+    fn test_quoted() {
+        let quoted =
+            r#":o 'single_word' 'twó wörds' '' ' ''\three\' \"with\ escaping\\' 'quote incomplete"#;
+        let result = shellwords(quoted);
+        let expected = vec![
+            Cow::from(":o"),
+            Cow::from("single_word"),
+            Cow::from("twó wörds"),
+            Cow::from(r#"three' "with escaping\"#),
+            Cow::from("quote incomplete"),
+        ];
+        assert_eq!(expected, result);
+    }
+
+    #[test]
+    fn test_dquoted() {
+        let dquoted = r#":o "single_word" "twó wörds" "" "  ""\three\' \"with\ escaping\\" "dquote incomplete"#;
+        let result = shellwords(dquoted);
+        let expected = vec![
+            Cow::from(":o"),
+            Cow::from("single_word"),
+            Cow::from("twó wörds"),
+            Cow::from(r#"three' "with escaping\"#),
+            Cow::from("dquote incomplete"),
+        ];
+        assert_eq!(expected, result);
+    }
+
+    #[test]
+    fn test_mixed() {
+        let dquoted = r#":o single_word 'twó wörds' "\three\' \"with\ escaping\\""no space before"'and after' $#%^@ "%^&(%^" ')(*&^%''a\\\\\b' '"#;
+        let result = shellwords(dquoted);
+        let expected = vec![
+            Cow::from(":o"),
+            Cow::from("single_word"),
+            Cow::from("twó wörds"),
+            Cow::from("three' \"with escaping\\"),
+            Cow::from("no space before"),
+            Cow::from("and after"),
+            Cow::from("$#%^@"),
+            Cow::from("%^&(%^"),
+            Cow::from(")(*&^%"),
+            Cow::from(r#"a\\b"#),
+            //last ' just changes to quoted but since we dont have anything after it, it should be ignored
+        ];
+        assert_eq!(expected, result);
+    }
+}

helix-core/src/state.rs

@@ -1,6 +1,5 @@
 use crate::{Rope, Selection};
 
-/// A state represents the current editor state of a single buffer.
 #[derive(Debug, Clone)]
 pub struct State {
     pub doc: Rope,
@@ -15,27 +14,4 @@ impl State {
             selection: Selection::point(0),
         }
     }
-
-    // update/transact:
-    // update(desc) => transaction ?  transaction.doc() for applied doc
-    // transaction.apply(doc)
-    // doc.transact(fn -> ... end)
-
-    // replaceSelection (transaction that replaces selection)
-    // changeByRange
-    // changes
-    // slice
-    //
-    // getters:
-    // tabSize
-    // indentUnit
-    // languageDataAt()
-    //
-    // config:
-    // indentation
-    // tabSize
-    // lineUnit
-    // syntax
-    // foldable
-    // changeFilter/transactionFilter
 }

helix-core/src/surround.rs

@@ -1,4 +1,6 @@
-use crate::{search, Selection};
+use std::fmt::Display;
+
+use crate::{search, Range, Selection};
 use ropey::RopeSlice;
 
 pub const PAIRS: &[(char, char)] = &[
@@ -11,6 +13,27 @@ pub const PAIRS: &[(char, char)] = &[
     ('（', '）'),
 ];
 
+#[derive(Debug, PartialEq)]
+pub enum Error {
+    PairNotFound,
+    CursorOverlap,
+    RangeExceedsText,
+    CursorOnAmbiguousPair,
+}
+
+impl Display for Error {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(match *self {
+            Error::PairNotFound => "Surround pair not found around all cursors",
+            Error::CursorOverlap => "Cursors overlap for a single surround pair range",
+            Error::RangeExceedsText => "Cursor range exceeds text length",
+            Error::CursorOnAmbiguousPair => "Cursor on ambiguous surround pair",
+        })
+    }
+}
+
+type Result<T> = std::result::Result<T, Error>;
+
 /// Given any char in [PAIRS], return the open and closing chars. If not found in
 /// [PAIRS] return (ch, ch).
 ///
@@ -35,39 +58,38 @@ pub fn get_pair(ch: char) -> (char, char) {
 pub fn find_nth_pairs_pos(
     text: RopeSlice,
     ch: char,
-    pos: usize,
+    range: Range,
     n: usize,
-) -> Option<(usize, usize)> {
+) -> Result<(usize, usize)> {
+    if text.len_chars() < 2 {
+        return Err(Error::PairNotFound);
+    }
+    if range.to() >= text.len_chars() {
+        return Err(Error::RangeExceedsText);
+    }
+
     let (open, close) = get_pair(ch);
+    let pos = range.cursor(text);
 
-    if text.len_chars() < 2 || pos >= text.len_chars() {
+    let (open, close) = if open == close {
-        return None;
-    }
-
-    if open == close {
         if Some(open) == text.get_char(pos) {
-            // Special case: cursor is directly on a matching char.
+            // Cursor is directly on match char. We return no match
-            match pos {
+            // because there's no way to know which side of the char
-                0 => Some((pos, search::find_nth_next(text, close, pos + 1, n)?)),
+            // we should be searching on.
-                _ if (pos + 1) == text.len_chars() => {
+            return Err(Error::CursorOnAmbiguousPair);
-                    Some((search::find_nth_prev(text, open, pos, n)?, pos))
-                }
-                // We return no match because there's no way to know which
-                // side of the char we should be searching on.
-                _ => None,
         }
+        (
+            search::find_nth_prev(text, open, pos, n),
+            search::find_nth_next(text, close, pos, n),
+        )
     } else {
-            Some((
+        (
-                search::find_nth_prev(text, open, pos, n)?,
+            find_nth_open_pair(text, open, close, pos, n),
-                search::find_nth_next(text, close, pos, n)?,
+            find_nth_close_pair(text, open, close, pos, n),
-            ))
+        )
-        }
+    };
-    } else {
+
-        Some((
+    Option::zip(open, close).ok_or(Error::PairNotFound)
-            find_nth_open_pair(text, open, close, pos, n)?,
-            find_nth_close_pair(text, open, close, pos, n)?,
-        ))
-    }
 }
 
 fn find_nth_open_pair(
@@ -157,17 +179,17 @@ pub fn get_surround_pos(
     selection: &Selection,
     ch: char,
     skip: usize,
-) -> Option<Vec<usize>> {
+) -> Result<Vec<usize>> {
     let mut change_pos = Vec::new();
 
-    for range in selection {
+    for &range in selection {
-        let (open_pos, close_pos) = find_nth_pairs_pos(text, ch, range.head, skip)?;
+        let (open_pos, close_pos) = find_nth_pairs_pos(text, ch, range, skip)?;
         if change_pos.contains(&open_pos) || change_pos.contains(&close_pos) {
-            return None;
+            return Err(Error::CursorOverlap);
         }
         change_pos.extend_from_slice(&[open_pos, close_pos]);
     }
-    Some(change_pos)
+    Ok(change_pos)
 }
 
 #[cfg(test)]
@@ -178,67 +200,92 @@ mod test {
     use ropey::Rope;
     use smallvec::SmallVec;
 
-    #[test]
+    #[allow(clippy::type_complexity)]
-    fn test_find_nth_pairs_pos() {
+    fn check_find_nth_pair_pos(
-        let doc = Rope::from("some (text) here");
+        text: &str,
+        cases: Vec<(usize, char, usize, Result<(usize, usize)>)>,
+    ) {
+        let doc = Rope::from(text);
         let slice = doc.slice(..);
 
+        for (cursor_pos, ch, n, expected_range) in cases {
+            let range = find_nth_pairs_pos(slice, ch, (cursor_pos, cursor_pos + 1).into(), n);
+            assert_eq!(
+                range, expected_range,
+                "Expected {:?}, got {:?}",
+                expected_range, range
+            );
+        }
+    }
+
+    #[test]
+    fn test_find_nth_pairs_pos() {
+        check_find_nth_pair_pos(
+            "some (text) here",
+            vec![
                 // cursor on [t]ext
-        assert_eq!(find_nth_pairs_pos(slice, '(', 6, 1), Some((5, 10)));
+                (6, '(', 1, Ok((5, 10))),
-        assert_eq!(find_nth_pairs_pos(slice, ')', 6, 1), Some((5, 10)));
+                (6, ')', 1, Ok((5, 10))),
                 // cursor on so[m]e
-        assert_eq!(find_nth_pairs_pos(slice, '(', 2, 1), None);
+                (2, '(', 1, Err(Error::PairNotFound)),
                 // cursor on bracket itself
-        assert_eq!(find_nth_pairs_pos(slice, '(', 5, 1), Some((5, 10)));
+                (5, '(', 1, Ok((5, 10))),
-        assert_eq!(find_nth_pairs_pos(slice, '(', 10, 1), Some((5, 10)));
+                (10, '(', 1, Ok((5, 10))),
+            ],
+        );
     }
 
     #[test]
     fn test_find_nth_pairs_pos_skip() {
-        let doc = Rope::from("(so (many (good) text) here)");
+        check_find_nth_pair_pos(
-        let slice = doc.slice(..);
+            "(so (many (good) text) here)",
-
+            vec![
                 // cursor on go[o]d
-        assert_eq!(find_nth_pairs_pos(slice, '(', 13, 1), Some((10, 15)));
+                (13, '(', 1, Ok((10, 15))),
-        assert_eq!(find_nth_pairs_pos(slice, '(', 13, 2), Some((4, 21)));
+                (13, '(', 2, Ok((4, 21))),
-        assert_eq!(find_nth_pairs_pos(slice, '(', 13, 3), Some((0, 27)));
+                (13, '(', 3, Ok((0, 27))),
+            ],
+        );
     }
 
     #[test]
     fn test_find_nth_pairs_pos_same() {
-        let doc = Rope::from("'so 'many 'good' text' here'");
+        check_find_nth_pair_pos(
-        let slice = doc.slice(..);
+            "'so 'many 'good' text' here'",
-
+            vec![
                 // cursor on go[o]d
-        assert_eq!(find_nth_pairs_pos(slice, '\'', 13, 1), Some((10, 15)));
+                (13, '\'', 1, Ok((10, 15))),
-        assert_eq!(find_nth_pairs_pos(slice, '\'', 13, 2), Some((4, 21)));
+                (13, '\'', 2, Ok((4, 21))),
-        assert_eq!(find_nth_pairs_pos(slice, '\'', 13, 3), Some((0, 27)));
+                (13, '\'', 3, Ok((0, 27))),
                 // cursor on the quotes
-        assert_eq!(find_nth_pairs_pos(slice, '\'', 10, 1), None);
+                (10, '\'', 1, Err(Error::CursorOnAmbiguousPair)),
-        // this is the best we can do since opening and closing pairs are same
+            ],
-        assert_eq!(find_nth_pairs_pos(slice, '\'', 0, 1), Some((0, 4)));
+        )
-        assert_eq!(find_nth_pairs_pos(slice, '\'', 27, 1), Some((21, 27)));
     }
 
     #[test]
     fn test_find_nth_pairs_pos_step() {
-        let doc = Rope::from("((so)((many) good (text))(here))");
+        check_find_nth_pair_pos(
-        let slice = doc.slice(..);
+            "((so)((many) good (text))(here))",
-
+            vec![
                 // cursor on go[o]d
-        assert_eq!(find_nth_pairs_pos(slice, '(', 15, 1), Some((5, 24)));
+                (15, '(', 1, Ok((5, 24))),
-        assert_eq!(find_nth_pairs_pos(slice, '(', 15, 2), Some((0, 31)));
+                (15, '(', 2, Ok((0, 31))),
+            ],
+        )
     }
 
     #[test]
     fn test_find_nth_pairs_pos_mixed() {
-        let doc = Rope::from("(so [many {good} text] here)");
+        check_find_nth_pair_pos(
-        let slice = doc.slice(..);
+            "(so [many {good} text] here)",
-
+            vec![
                 // cursor on go[o]d
-        assert_eq!(find_nth_pairs_pos(slice, '{', 13, 1), Some((10, 15)));
+                (13, '{', 1, Ok((10, 15))),
-        assert_eq!(find_nth_pairs_pos(slice, '[', 13, 1), Some((4, 21)));
+                (13, '[', 1, Ok((4, 21))),
-        assert_eq!(find_nth_pairs_pos(slice, '(', 13, 1), Some((0, 27)));
+                (13, '(', 1, Ok((0, 27))),
+            ],
+        )
     }
 
     #[test]
@@ -266,11 +313,10 @@ mod test {
 
         let selection =
             Selection::new(SmallVec::from_slice(&[Range::point(2), Range::point(9)]), 0);
-
         // cursor on s[o]me, c[h]ars
         assert_eq!(
             get_surround_pos(slice, &selection, '(', 1),
-            None // different surround chars
+            Err(Error::PairNotFound) // different surround chars
         );
 
         let selection = Selection::new(
@@ -280,7 +326,15 @@ mod test {
         // cursor on [x]x, newli[n]e
         assert_eq!(
             get_surround_pos(slice, &selection, '(', 1),
-            None // overlapping surround chars
+            Err(Error::PairNotFound) // overlapping surround chars
+        );
+
+        let selection =
+            Selection::new(SmallVec::from_slice(&[Range::point(2), Range::point(3)]), 0);
+        // cursor on s[o][m]e
+        assert_eq!(
+            get_surround_pos(slice, &selection, '[', 1),
+            Err(Error::CursorOverlap)
         );
     }
 }

helix-core/src/textobject.rs

@@ -1,12 +1,16 @@
+use std::fmt::Display;
+
 use ropey::RopeSlice;
+use tree_sitter::{Node, QueryCursor};
 
 use crate::chars::{categorize_char, char_is_whitespace, CharCategory};
 use crate::graphemes::next_grapheme_boundary;
 use crate::movement::Direction;
 use crate::surround;
+use crate::syntax::LanguageConfiguration;
 use crate::Range;
 
-fn find_word_boundary(slice: RopeSlice, mut pos: usize, direction: Direction) -> usize {
+fn find_word_boundary(slice: RopeSlice, mut pos: usize, direction: Direction, long: bool) -> usize {
     use CharCategory::{Eol, Whitespace};
 
     let iter = match direction {
@@ -29,7 +33,7 @@ fn find_word_boundary(slice: RopeSlice, mut pos: usize, direction: Direction) ->
         match categorize_char(ch) {
             Eol | Whitespace => return pos,
             category => {
-                if category != prev_category && pos != 0 && pos != slice.len_chars() {
+                if !long && category != prev_category && pos != 0 && pos != slice.len_chars() {
                     return pos;
                 } else {
                     match direction {
@@ -49,6 +53,18 @@ fn find_word_boundary(slice: RopeSlice, mut pos: usize, direction: Direction) ->
 pub enum TextObject {
     Around,
     Inside,
+    /// Used for moving between objects.
+    Movement,
+}
+
+impl Display for TextObject {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(match self {
+            Self::Around => "around",
+            Self::Inside => "inside",
+            Self::Movement => "movement",
+        })
+    }
 }
 
 // count doesn't do anything yet
@@ -57,13 +73,14 @@ pub fn textobject_word(
     range: Range,
     textobject: TextObject,
     _count: usize,
+    long: bool,
 ) -> Range {
     let pos = range.cursor(slice);
 
-    let word_start = find_word_boundary(slice, pos, Direction::Backward);
+    let word_start = find_word_boundary(slice, pos, Direction::Backward, long);
     let word_end = match slice.get_char(pos).map(categorize_char) {
         None | Some(CharCategory::Whitespace | CharCategory::Eol) => pos,
-        _ => find_word_boundary(slice, pos + 1, Direction::Forward),
+        _ => find_word_boundary(slice, pos + 1, Direction::Forward, long),
     };
 
     // Special case.
@@ -90,6 +107,7 @@ pub fn textobject_word(
                 Range::new(word_start - whitespace_count_left, word_end)
             }
         }
+        TextObject::Movement => unreachable!(),
     }
 }
 
@@ -100,14 +118,53 @@ pub fn textobject_surround(
     ch: char,
     count: usize,
 ) -> Range {
-    surround::find_nth_pairs_pos(slice, ch, range.head, count)
+    surround::find_nth_pairs_pos(slice, ch, range, count)
         .map(|(anchor, head)| match textobject {
             TextObject::Inside => Range::new(next_grapheme_boundary(slice, anchor), head),
             TextObject::Around => Range::new(anchor, next_grapheme_boundary(slice, head)),
+            TextObject::Movement => unreachable!(),
         })
         .unwrap_or(range)
 }
 
+/// Transform the given range to select text objects based on tree-sitter.
+/// `object_name` is a query capture base name like "function", "class", etc.
+/// `slice_tree` is the tree-sitter node corresponding to given text slice.
+pub fn textobject_treesitter(
+    slice: RopeSlice,
+    range: Range,
+    textobject: TextObject,
+    object_name: &str,
+    slice_tree: Node,
+    lang_config: &LanguageConfiguration,
+    _count: usize,
+) -> Range {
+    let get_range = move || -> Option<Range> {
+        let byte_pos = slice.char_to_byte(range.cursor(slice));
+
+        let capture_name = format!("{}.{}", object_name, textobject); // eg. function.inner
+        let mut cursor = QueryCursor::new();
+        let node = lang_config
+            .textobject_query()?
+            .capture_nodes(&capture_name, slice_tree, slice, &mut cursor)?
+            .filter(|node| node.byte_range().contains(&byte_pos))
+            .min_by_key(|node| node.byte_range().len())?;
+
+        let len = slice.len_bytes();
+        let start_byte = node.start_byte();
+        let end_byte = node.end_byte();
+        if start_byte >= len || end_byte >= len {
+            return None;
+        }
+
+        let start_char = slice.byte_to_char(start_byte);
+        let end_char = slice.byte_to_char(end_byte);
+
+        Some(Range::new(start_char, end_char))
+    };
+    get_range().unwrap_or(range)
+}
+
 #[cfg(test)]
 mod test {
     use super::TextObject::*;
@@ -118,7 +175,7 @@ mod test {
 
     #[test]
     fn test_textobject_word() {
-        // (text, [(cursor position, textobject, final range), ...])
+        // (text, [(char position, textobject, final range), ...])
         let tests = &[
             (
                 "cursor at beginning of doc",
@@ -217,7 +274,9 @@ mod test {
             let slice = doc.slice(..);
             for &case in scenario {
                 let (pos, objtype, expected_range) = case;
-                let result = textobject_word(slice, Range::point(pos), objtype, 1);
+                // cursor is a single width selection
+                let range = Range::new(pos, pos + 1);
+                let result = textobject_word(slice, range, objtype, 1, false);
                 assert_eq!(
                     result,
                     expected_range.into(),
@@ -231,7 +290,7 @@ mod test {
 
     #[test]
     fn test_textobject_surround() {
-        // (text, [(cursor position, textobject, final range, count), ...])
+        // (text, [(cursor position, textobject, final range, surround char, count), ...])
         let tests = &[
             (
                 "simple (single) surround pairs",

helix-core/src/transaction.rs

@@ -21,8 +21,7 @@ pub enum Assoc {
     After,
 }
 
-// ChangeSpec = Change | ChangeSet | Vec<Change>
+#[derive(Debug, Default, Clone, PartialEq, Eq)]
-#[derive(Debug, Clone, PartialEq, Eq)]
 pub struct ChangeSet {
     pub(crate) changes: Vec<Operation>,
     /// The required document length. Will refuse to apply changes unless it matches.
@@ -30,16 +29,6 @@ pub struct ChangeSet {
     len_after: usize,
 }
 
-impl Default for ChangeSet {
-    fn default() -> Self {
-        Self {
-            changes: Vec::new(),
-            len: 0,
-            len_after: 0,
-        }
-    }
-}
-
 impl ChangeSet {
     pub fn with_capacity(capacity: usize) -> Self {
         Self {
@@ -60,7 +49,6 @@ impl ChangeSet {
     }
 
     // TODO: from iter
-    //
 
     #[doc(hidden)] // used by lsp to convert to LSP changes
     pub fn changes(&self) -> &[Operation] {
@@ -95,7 +83,7 @@ impl ChangeSet {
 
         let new_last = match self.changes.as_mut_slice() {
             [.., Insert(prev)] | [.., Insert(prev), Delete(_)] => {
-                prev.push_tendril(&fragment);
+                prev.push_str(&fragment);
                 return;
             }
             [.., last @ Delete(_)] => std::mem::replace(last, Insert(fragment)),
@@ -125,13 +113,16 @@ impl ChangeSet {
     /// In other words,  If `this` goes `docA` → `docB` and `other` represents `docB` → `docC`, the
     /// returned value will represent the change `docA` → `docC`.
     pub fn compose(self, other: Self) -> Self {
-        debug_assert!(self.len_after == other.len);
+        assert!(self.len_after == other.len);
 
         // composing fails in weird ways if one of the sets is empty
         // a: [] len: 0 len_after: 1 | b: [Insert(Tendril<UTF8>(inline: "\n")), Retain(1)] len 1
         if self.changes.is_empty() {
             return other;
         }
+        if other.changes.is_empty() {
+            return self;
+        }
 
         let len = self.changes.len();
 
@@ -196,7 +187,7 @@ impl ChangeSet {
                             // TODO: cover this with a test
                             // figure out the byte index of the truncated string end
                             let (pos, _) = s.char_indices().nth(j).unwrap();
-                            s.pop_front(pos as u32);
+                            s.replace_range(0..pos, "");
                             head_a = Some(Insert(s));
                             head_b = changes_b.next();
                         }
@@ -218,9 +209,11 @@ impl ChangeSet {
                         Ordering::Greater => {
                             // figure out the byte index of the truncated string end
                             let (pos, _) = s.char_indices().nth(j).unwrap();
-                            let pos = pos as u32;
+                            let mut before = s;
-                            changes.insert(s.subtendril(0, pos));
+                            let after = before.split_off(pos);
-                            head_a = Some(Insert(s.subtendril(pos, s.len() as u32 - pos)));
+
+                            changes.insert(before);
+                            head_a = Some(Insert(after));
                             head_b = changes_b.next();
                         }
                     }
@@ -284,7 +277,7 @@ impl ChangeSet {
                 }
                 Delete(n) => {
                     let text = Cow::from(original_doc.slice(pos..pos + *n));
-                    changes.insert(Tendril::from_slice(&text));
+                    changes.insert(Tendril::from(text.as_ref()));
                     pos += n;
                 }
                 Insert(s) => {
@@ -327,7 +320,7 @@ impl ChangeSet {
     /// `true` when the set is empty.
     #[inline]
     pub fn is_empty(&self) -> bool {
-        self.changes.is_empty()
+        self.changes.is_empty() || self.changes == [Operation::Retain(self.len)]
     }
 
     /// Map a position through the changes.
@@ -416,12 +409,10 @@ impl ChangeSet {
 
 /// Transaction represents a single undoable unit of changes. Several changes can be grouped into
 /// a single transaction.
-#[derive(Debug, Default, Clone)]
+#[derive(Debug, Default, Clone, PartialEq, Eq)]
 pub struct Transaction {
     changes: ChangeSet,
     selection: Option<Selection>,
-    // effects, annotations
-    // scroll_into_view
 }
 
 impl Transaction {
@@ -445,14 +436,12 @@ impl Transaction {
 
     /// Returns true if applied successfully.
     pub fn apply(&self, doc: &mut Rope) -> bool {
-        if !self.changes.is_empty() {
+        if self.changes.is_empty() {
-            // apply changes to the document
+            return true;
-            if !self.changes.apply(doc) {
-                return false;
-            }
         }
 
-        true
+        // apply changes to the document
+        self.changes.apply(doc)
     }
 
     /// Generate a transaction that reverts this one.
@@ -465,6 +454,13 @@ impl Transaction {
         }
     }
 
+    pub fn compose(mut self, other: Self) -> Self {
+        self.changes = self.changes.compose(other.changes);
+        // Other selection takes precedence
+        self.selection = other.selection;
+        self
+    }
+
     pub fn with_selection(mut self, selection: Selection) -> Self {
         self.selection = Some(selection);
         self
@@ -473,7 +469,7 @@ impl Transaction {
     /// Generate a transaction from a set of changes.
     pub fn change<I>(doc: &Rope, changes: I) -> Self
     where
-        I: IntoIterator<Item = Change> + Iterator,
+        I: Iterator<Item = Change>,
     {
         let len = doc.len_chars();
 
@@ -481,12 +477,11 @@ impl Transaction {
         let size = upper.unwrap_or(lower);
         let mut changeset = ChangeSet::with_capacity(2 * size + 1); // rough estimate
 
-        // TODO: verify ranges are ordered and not overlapping or change will panic.
-
-        // TODO: test for (pos, pos, None) to factor out as nothing
-
         let mut last = 0;
         for (from, to, tendril) in changes {
+            // Verify ranges are ordered and not overlapping
+            debug_assert!(last <= from);
+
             // Retain from last "to" to current "from"
             changeset.retain(from - last);
             let span = to - from;
@@ -689,40 +684,40 @@ mod test {
 
     #[test]
     fn transaction_change() {
-        let mut state = State::new("hello world!\ntest 123".into());
+        let mut doc = Rope::from("hello world!\ntest 123");
         let transaction = Transaction::change(
-            &state.doc,
+            &doc,
-            // (1, 1, None) is a useless 0-width delete
+            // (1, 1, None) is a useless 0-width delete that gets factored out
             vec![(1, 1, None), (6, 11, Some("void".into())), (12, 17, None)].into_iter(),
         );
-        transaction.apply(&mut state.doc);
+        transaction.apply(&mut doc);
-        assert_eq!(state.doc, Rope::from_str("hello void! 123"));
+        assert_eq!(doc, Rope::from_str("hello void! 123"));
     }
 
     #[test]
     fn changes_iter() {
-        let state = State::new("hello world!\ntest 123".into());
+        let doc = Rope::from("hello world!\ntest 123");
         let changes = vec![(6, 11, Some("void".into())), (12, 17, None)];
-        let transaction = Transaction::change(&state.doc, changes.clone().into_iter());
+        let transaction = Transaction::change(&doc, changes.clone().into_iter());
         assert_eq!(transaction.changes_iter().collect::<Vec<_>>(), changes);
     }
 
     #[test]
     fn optimized_composition() {
         let mut state = State::new("".into());
-        let t1 = Transaction::insert(&state.doc, &state.selection, Tendril::from_char('h'));
+        let t1 = Transaction::insert(&state.doc, &state.selection, Tendril::from("h"));
         t1.apply(&mut state.doc);
         state.selection = state.selection.clone().map(t1.changes());
-        let t2 = Transaction::insert(&state.doc, &state.selection, Tendril::from_char('e'));
+        let t2 = Transaction::insert(&state.doc, &state.selection, Tendril::from("e"));
         t2.apply(&mut state.doc);
         state.selection = state.selection.clone().map(t2.changes());
-        let t3 = Transaction::insert(&state.doc, &state.selection, Tendril::from_char('l'));
+        let t3 = Transaction::insert(&state.doc, &state.selection, Tendril::from("l"));
         t3.apply(&mut state.doc);
         state.selection = state.selection.clone().map(t3.changes());
-        let t4 = Transaction::insert(&state.doc, &state.selection, Tendril::from_char('l'));
+        let t4 = Transaction::insert(&state.doc, &state.selection, Tendril::from("l"));
         t4.apply(&mut state.doc);
         state.selection = state.selection.clone().map(t4.changes());
-        let t5 = Transaction::insert(&state.doc, &state.selection, Tendril::from_char('o'));
+        let t5 = Transaction::insert(&state.doc, &state.selection, Tendril::from("o"));
         t5.apply(&mut state.doc);
         state.selection = state.selection.clone().map(t5.changes());
 
@@ -761,7 +756,7 @@ mod test {
 
     #[test]
     fn combine_with_utf8() {
-        const TEST_CASE: &'static str = "Hello, これはヘリックスエディターです！";
+        const TEST_CASE: &str = "Hello, これはヘリックスエディターです！";
 
         let empty = Rope::from("");
         let a = ChangeSet::new(&empty);

helix-dap/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "helix-dap"
+version = "0.6.0"
+authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
+edition = "2018"
+license = "MPL-2.0"
+description = "DAP client implementation for Helix project"
+categories = ["editor"]
+repository = "https://github.com/helix-editor/helix"
+homepage = "https://helix-editor.com"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+helix-core = { version = "0.6", path = "../helix-core" }
+anyhow = "1.0"
+log = "0.4"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+thiserror = "1.0"
+tokio = { version = "1", features = ["rt", "rt-multi-thread", "io-util", "io-std", "time", "process", "macros", "fs", "parking_lot", "net", "sync"] }
+which = "4.2"
+
+[dev-dependencies]
+fern = "0.6"

helix-dap/src/client.rs

@@ -0,0 +1,480 @@
+use crate::{
+    transport::{Payload, Request, Response, Transport},
+    types::*,
+    Error, Result, ThreadId,
+};
+use helix_core::syntax::DebuggerQuirks;
+
+use serde_json::Value;
+
+use anyhow::anyhow;
+pub use log::{error, info};
+use std::{
+    collections::HashMap,
+    future::Future,
+    net::{IpAddr, Ipv4Addr, SocketAddr},
+    path::PathBuf,
+    process::Stdio,
+    sync::atomic::{AtomicU64, Ordering},
+};
+use tokio::{
+    io::{AsyncBufRead, AsyncWrite, BufReader, BufWriter},
+    net::TcpStream,
+    process::{Child, Command},
+    sync::mpsc::{channel, unbounded_channel, UnboundedReceiver, UnboundedSender},
+    time,
+};
+
+#[derive(Debug)]
+pub struct Client {
+    id: usize,
+    _process: Option<Child>,
+    server_tx: UnboundedSender<Payload>,
+    request_counter: AtomicU64,
+    pub caps: Option<DebuggerCapabilities>,
+    // thread_id -> frames
+    pub stack_frames: HashMap<ThreadId, Vec<StackFrame>>,
+    pub thread_states: HashMap<ThreadId, String>,
+    pub thread_id: Option<ThreadId>,
+    /// Currently active frame for the current thread.
+    pub active_frame: Option<usize>,
+    pub quirks: DebuggerQuirks,
+}
+
+impl Client {
+    // Spawn a process and communicate with it by either TCP or stdio
+    pub async fn process(
+        transport: &str,
+        command: &str,
+        args: Vec<&str>,
+        port_arg: Option<&str>,
+        id: usize,
+    ) -> Result<(Self, UnboundedReceiver<Payload>)> {
+        if command.is_empty() {
+            return Result::Err(Error::Other(anyhow!("Command not provided")));
+        }
+        if transport == "tcp" && port_arg.is_some() {
+            Self::tcp_process(command, args, port_arg.unwrap(), id).await
+        } else if transport == "stdio" {
+            Self::stdio(command, args, id)
+        } else {
+            Result::Err(Error::Other(anyhow!("Incorrect transport {}", transport)))
+        }
+    }
+
+    pub fn streams(
+        rx: Box<dyn AsyncBufRead + Unpin + Send>,
+        tx: Box<dyn AsyncWrite + Unpin + Send>,
+        err: Option<Box<dyn AsyncBufRead + Unpin + Send>>,
+        id: usize,
+        process: Option<Child>,
+    ) -> Result<(Self, UnboundedReceiver<Payload>)> {
+        let (server_rx, server_tx) = Transport::start(rx, tx, err, id);
+        let (client_rx, client_tx) = unbounded_channel();
+
+        let client = Self {
+            id,
+            _process: process,
+            server_tx,
+            request_counter: AtomicU64::new(0),
+            caps: None,
+            //
+            stack_frames: HashMap::new(),
+            thread_states: HashMap::new(),
+            thread_id: None,
+            active_frame: None,
+            quirks: DebuggerQuirks::default(),
+        };
+
+        tokio::spawn(Self::recv(server_rx, client_rx));
+
+        Ok((client, client_tx))
+    }
+
+    pub async fn tcp(
+        addr: std::net::SocketAddr,
+        id: usize,
+    ) -> Result<(Self, UnboundedReceiver<Payload>)> {
+        let stream = TcpStream::connect(addr).await?;
+        let (rx, tx) = stream.into_split();
+        Self::streams(Box::new(BufReader::new(rx)), Box::new(tx), None, id, None)
+    }
+
+    pub fn stdio(
+        cmd: &str,
+        args: Vec<&str>,
+        id: usize,
+    ) -> Result<(Self, UnboundedReceiver<Payload>)> {
+        // Resolve path to the binary
+        let cmd = which::which(cmd).map_err(|err| anyhow::anyhow!(err))?;
+
+        let process = Command::new(cmd)
+            .args(args)
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            // make sure the process is reaped on drop
+            .kill_on_drop(true)
+            .spawn();
+
+        let mut process = process?;
+
+        // TODO: do we need bufreader/writer here? or do we use async wrappers on unblock?
+        let writer = BufWriter::new(process.stdin.take().expect("Failed to open stdin"));
+        let reader = BufReader::new(process.stdout.take().expect("Failed to open stdout"));
+        let errors = process.stderr.take().map(BufReader::new);
+
+        Self::streams(
+            Box::new(BufReader::new(reader)),
+            Box::new(writer),
+            // errors.map(|errors| Box::new(BufReader::new(errors))),
+            match errors {
+                Some(errors) => Some(Box::new(BufReader::new(errors))),
+                None => None,
+            },
+            id,
+            Some(process),
+        )
+    }
+
+    async fn get_port() -> Option<u16> {
+        Some(
+            tokio::net::TcpListener::bind(SocketAddr::new(
+                IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)),
+                0,
+            ))
+            .await
+            .ok()?
+            .local_addr()
+            .ok()?
+            .port(),
+        )
+    }
+
+    pub async fn tcp_process(
+        cmd: &str,
+        args: Vec<&str>,
+        port_format: &str,
+        id: usize,
+    ) -> Result<(Self, UnboundedReceiver<Payload>)> {
+        let port = Self::get_port().await.unwrap();
+
+        let process = Command::new(cmd)
+            .args(args)
+            .args(port_format.replace("{}", &port.to_string()).split(' '))
+            // silence messages
+            .stdin(Stdio::null())
+            .stdout(Stdio::null())
+            .stderr(Stdio::null())
+            // Do not kill debug adapter when leaving, it should exit automatically
+            .spawn()?;
+
+        // Wait for adapter to become ready for connection
+        time::sleep(time::Duration::from_millis(500)).await;
+
+        let stream = TcpStream::connect(SocketAddr::new(
+            IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)),
+            port,
+        ))
+        .await?;
+
+        let (rx, tx) = stream.into_split();
+        Self::streams(
+            Box::new(BufReader::new(rx)),
+            Box::new(tx),
+            None,
+            id,
+            Some(process),
+        )
+    }
+
+    async fn recv(mut server_rx: UnboundedReceiver<Payload>, client_tx: UnboundedSender<Payload>) {
+        while let Some(msg) = server_rx.recv().await {
+            match msg {
+                Payload::Event(ev) => {
+                    client_tx.send(Payload::Event(ev)).expect("Failed to send");
+                }
+                Payload::Response(_) => unreachable!(),
+                Payload::Request(req) => {
+                    client_tx
+                        .send(Payload::Request(req))
+                        .expect("Failed to send");
+                }
+            }
+        }
+    }
+
+    pub fn id(&self) -> usize {
+        self.id
+    }
+
+    fn next_request_id(&self) -> u64 {
+        self.request_counter.fetch_add(1, Ordering::Relaxed)
+    }
+
+    // Internal, called by specific DAP commands when resuming
+    pub fn resume_application(&mut self) {
+        if let Some(thread_id) = self.thread_id {
+            self.thread_states.insert(thread_id, "running".to_string());
+            self.stack_frames.remove(&thread_id);
+        }
+        self.active_frame = None;
+        self.thread_id = None;
+    }
+
+    /// Execute a RPC request on the debugger.
+    pub fn call<R: crate::types::Request>(
+        &self,
+        arguments: R::Arguments,
+    ) -> impl Future<Output = Result<Value>>
+    where
+        R::Arguments: serde::Serialize,
+    {
+        let server_tx = self.server_tx.clone();
+        let id = self.next_request_id();
+
+        async move {
+            use std::time::Duration;
+            use tokio::time::timeout;
+
+            let arguments = Some(serde_json::to_value(arguments)?);
+
+            let (callback_tx, mut callback_rx) = channel(1);
+
+            let req = Request {
+                back_ch: Some(callback_tx),
+                seq: id,
+                command: R::COMMAND.to_string(),
+                arguments,
+            };
+
+            server_tx
+                .send(Payload::Request(req))
+                .map_err(|e| Error::Other(e.into()))?;
+
+            // TODO: specifiable timeout, delay other calls until initialize success
+            timeout(Duration::from_secs(20), callback_rx.recv())
+                .await
+                .map_err(|_| Error::Timeout)? // return Timeout
+                .ok_or(Error::StreamClosed)?
+                .map(|response| response.body.unwrap_or_default())
+            // TODO: check response.success
+        }
+    }
+
+    pub async fn request<R: crate::types::Request>(&self, params: R::Arguments) -> Result<R::Result>
+    where
+        R::Arguments: serde::Serialize,
+        R::Result: core::fmt::Debug, // TODO: temporary
+    {
+        // a future that resolves into the response
+        let json = self.call::<R>(params).await?;
+        let response = serde_json::from_value(json)?;
+        Ok(response)
+    }
+
+    pub fn reply(
+        &self,
+        request_seq: u64,
+        command: &str,
+        result: core::result::Result<Value, Error>,
+    ) -> impl Future<Output = Result<()>> {
+        let server_tx = self.server_tx.clone();
+        let command = command.to_string();
+
+        async move {
+            let response = match result {
+                Ok(result) => Response {
+                    request_seq,
+                    command,
+                    success: true,
+                    message: None,
+                    body: Some(result),
+                },
+                Err(error) => Response {
+                    request_seq,
+                    command,
+                    success: false,
+                    message: Some(error.to_string()),
+                    body: None,
+                },
+            };
+
+            server_tx
+                .send(Payload::Response(response))
+                .map_err(|e| Error::Other(e.into()))?;
+
+            Ok(())
+        }
+    }
+
+    pub fn capabilities(&self) -> &DebuggerCapabilities {
+        self.caps.as_ref().expect("debugger not yet initialized!")
+    }
+
+    pub async fn initialize(&mut self, adapter_id: String) -> Result<()> {
+        let args = requests::InitializeArguments {
+            client_id: Some("hx".to_owned()),
+            client_name: Some("helix".to_owned()),
+            adapter_id,
+            locale: Some("en-us".to_owned()),
+            lines_start_at_one: Some(true),
+            columns_start_at_one: Some(true),
+            path_format: Some("path".to_owned()),
+            supports_variable_type: Some(true),
+            supports_variable_paging: Some(false),
+            supports_run_in_terminal_request: Some(true),
+            supports_memory_references: Some(false),
+            supports_progress_reporting: Some(false),
+            supports_invalidated_event: Some(false),
+        };
+
+        let response = self.request::<requests::Initialize>(args).await?;
+        self.caps = Some(response);
+
+        Ok(())
+    }
+
+    pub fn disconnect(&self) -> impl Future<Output = Result<Value>> {
+        self.call::<requests::Disconnect>(())
+    }
+
+    pub fn launch(&self, args: serde_json::Value) -> impl Future<Output = Result<Value>> {
+        self.call::<requests::Launch>(args)
+    }
+
+    pub fn attach(&self, args: serde_json::Value) -> impl Future<Output = Result<Value>> {
+        self.call::<requests::Attach>(args)
+    }
+
+    pub async fn set_breakpoints(
+        &self,
+        file: PathBuf,
+        breakpoints: Vec<SourceBreakpoint>,
+    ) -> Result<Option<Vec<Breakpoint>>> {
+        let args = requests::SetBreakpointsArguments {
+            source: Source {
+                path: Some(file),
+                name: None,
+                source_reference: None,
+                presentation_hint: None,
+                origin: None,
+                sources: None,
+                adapter_data: None,
+                checksums: None,
+            },
+            breakpoints: Some(breakpoints),
+            source_modified: Some(false),
+        };
+
+        let response = self.request::<requests::SetBreakpoints>(args).await?;
+
+        Ok(response.breakpoints)
+    }
+
+    pub async fn configuration_done(&self) -> Result<()> {
+        self.request::<requests::ConfigurationDone>(()).await
+    }
+
+    pub fn continue_thread(&self, thread_id: ThreadId) -> impl Future<Output = Result<Value>> {
+        let args = requests::ContinueArguments { thread_id };
+
+        self.call::<requests::Continue>(args)
+    }
+
+    pub async fn stack_trace(
+        &self,
+        thread_id: ThreadId,
+    ) -> Result<(Vec<StackFrame>, Option<usize>)> {
+        let args = requests::StackTraceArguments {
+            thread_id,
+            start_frame: None,
+            levels: None,
+            format: None,
+        };
+
+        let response = self.request::<requests::StackTrace>(args).await?;
+        Ok((response.stack_frames, response.total_frames))
+    }
+
+    pub fn threads(&self) -> impl Future<Output = Result<Value>> {
+        self.call::<requests::Threads>(())
+    }
+
+    pub async fn scopes(&self, frame_id: usize) -> Result<Vec<Scope>> {
+        let args = requests::ScopesArguments { frame_id };
+
+        let response = self.request::<requests::Scopes>(args).await?;
+        Ok(response.scopes)
+    }
+
+    pub async fn variables(&self, variables_reference: usize) -> Result<Vec<Variable>> {
+        let args = requests::VariablesArguments {
+            variables_reference,
+            filter: None,
+            start: None,
+            count: None,
+            format: None,
+        };
+
+        let response = self.request::<requests::Variables>(args).await?;
+        Ok(response.variables)
+    }
+
+    pub fn step_in(&self, thread_id: ThreadId) -> impl Future<Output = Result<Value>> {
+        let args = requests::StepInArguments {
+            thread_id,
+            target_id: None,
+            granularity: None,
+        };
+
+        self.call::<requests::StepIn>(args)
+    }
+
+    pub fn step_out(&self, thread_id: ThreadId) -> impl Future<Output = Result<Value>> {
+        let args = requests::StepOutArguments {
+            thread_id,
+            granularity: None,
+        };
+
+        self.call::<requests::StepOut>(args)
+    }
+
+    pub fn next(&self, thread_id: ThreadId) -> impl Future<Output = Result<Value>> {
+        let args = requests::NextArguments {
+            thread_id,
+            granularity: None,
+        };
+
+        self.call::<requests::Next>(args)
+    }
+
+    pub fn pause(&self, thread_id: ThreadId) -> impl Future<Output = Result<Value>> {
+        let args = requests::PauseArguments { thread_id };
+
+        self.call::<requests::Pause>(args)
+    }
+
+    pub async fn eval(
+        &self,
+        expression: String,
+        frame_id: Option<usize>,
+    ) -> Result<requests::EvaluateResponse> {
+        let args = requests::EvaluateArguments {
+            expression,
+            frame_id,
+            context: None,
+            format: None,
+        };
+
+        self.request::<requests::Evaluate>(args).await
+    }
+
+    pub fn set_exception_breakpoints(
+        &self,
+        filters: Vec<String>,
+    ) -> impl Future<Output = Result<Value>> {
+        let args = requests::SetExceptionBreakpointsArguments { filters };
+
+        self.call::<requests::SetExceptionBreakpoints>(args)
+    }
+}

helix-dap/src/lib.rs

@@ -0,0 +1,24 @@
+mod client;
+mod transport;
+mod types;
+
+pub use client::Client;
+pub use events::Event;
+pub use transport::{Payload, Response, Transport};
+pub use types::*;
+
+use thiserror::Error;
+#[derive(Error, Debug)]
+pub enum Error {
+    #[error("failed to parse: {0}")]
+    Parse(#[from] serde_json::Error),
+    #[error("IO Error: {0}")]
+    IO(#[from] std::io::Error),
+    #[error("request timed out")]
+    Timeout,
+    #[error("server closed the stream")]
+    StreamClosed,
+    #[error(transparent)]
+    Other(#[from] anyhow::Error),
+}
+pub type Result<T> = core::result::Result<T, Error>;

helix-dap/src/transport.rs

@@ -0,0 +1,280 @@
+use crate::{Error, Event, Result};
+use anyhow::Context;
+use log::{error, info, warn};
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use std::collections::HashMap;
+use std::sync::Arc;
+use tokio::{
+    io::{AsyncBufRead, AsyncBufReadExt, AsyncReadExt, AsyncWrite, AsyncWriteExt},
+    sync::{
+        mpsc::{unbounded_channel, Sender, UnboundedReceiver, UnboundedSender},
+        Mutex,
+    },
+};
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct Request {
+    #[serde(skip)]
+    pub back_ch: Option<Sender<Result<Response>>>,
+    pub seq: u64,
+    pub command: String,
+    pub arguments: Option<Value>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+pub struct Response {
+    // seq is omitted as unused and is not sent by some implementations
+    pub request_seq: u64,
+    pub success: bool,
+    pub command: String,
+    pub message: Option<String>,
+    pub body: Option<Value>,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+#[serde(tag = "type", rename_all = "camelCase")]
+pub enum Payload {
+    // type = "event"
+    Event(Box<Event>),
+    // type = "response"
+    Response(Response),
+    // type = "request"
+    Request(Request),
+}
+
+#[derive(Debug)]
+pub struct Transport {
+    #[allow(unused)]
+    id: usize,
+    pending_requests: Mutex<HashMap<u64, Sender<Result<Response>>>>,
+}
+
+impl Transport {
+    pub fn start(
+        server_stdout: Box<dyn AsyncBufRead + Unpin + Send>,
+        server_stdin: Box<dyn AsyncWrite + Unpin + Send>,
+        server_stderr: Option<Box<dyn AsyncBufRead + Unpin + Send>>,
+        id: usize,
+    ) -> (UnboundedReceiver<Payload>, UnboundedSender<Payload>) {
+        let (client_tx, rx) = unbounded_channel();
+        let (tx, client_rx) = unbounded_channel();
+
+        let transport = Self {
+            id,
+            pending_requests: Mutex::new(HashMap::default()),
+        };
+
+        let transport = Arc::new(transport);
+
+        tokio::spawn(Self::recv(transport.clone(), server_stdout, client_tx));
+        tokio::spawn(Self::send(transport, server_stdin, client_rx));
+        if let Some(stderr) = server_stderr {
+            tokio::spawn(Self::err(stderr));
+        }
+
+        (rx, tx)
+    }
+
+    async fn recv_server_message(
+        reader: &mut Box<dyn AsyncBufRead + Unpin + Send>,
+        buffer: &mut String,
+    ) -> Result<Payload> {
+        let mut content_length = None;
+        loop {
+            buffer.truncate(0);
+            if reader.read_line(buffer).await? == 0 {
+                return Err(Error::StreamClosed);
+            };
+
+            if buffer == "\r\n" {
+                // look for an empty CRLF line
+                break;
+            }
+
+            let header = buffer.trim();
+            let parts = header.split_once(": ");
+
+            match parts {
+                Some(("Content-Length", value)) => {
+                    content_length = Some(value.parse().context("invalid content length")?);
+                }
+                Some((_, _)) => {}
+                None => {
+                    // Workaround: Some non-conformant language servers will output logging and other garbage
+                    // into the same stream as JSON-RPC messages. This can also happen from shell scripts that spawn
+                    // the server. Skip such lines and log a warning.
+
+                    // warn!("Failed to parse header: {:?}", header);
+                }
+            }
+        }
+
+        let content_length = content_length.context("missing content length")?;
+
+        //TODO: reuse vector
+        let mut content = vec![0; content_length];
+        reader.read_exact(&mut content).await?;
+        let msg = std::str::from_utf8(&content).context("invalid utf8 from server")?;
+
+        info!("<- DAP {}", msg);
+
+        // try parsing as output (server response) or call (server request)
+        let output: serde_json::Result<Payload> = serde_json::from_str(msg);
+
+        Ok(output?)
+    }
+
+    async fn recv_server_error(
+        err: &mut (impl AsyncBufRead + Unpin + Send),
+        buffer: &mut String,
+    ) -> Result<()> {
+        buffer.truncate(0);
+        if err.read_line(buffer).await? == 0 {
+            return Err(Error::StreamClosed);
+        };
+        error!("err <- {}", buffer);
+
+        Ok(())
+    }
+
+    async fn send_payload_to_server(
+        &self,
+        server_stdin: &mut Box<dyn AsyncWrite + Unpin + Send>,
+        mut payload: Payload,
+    ) -> Result<()> {
+        if let Payload::Request(request) = &mut payload {
+            if let Some(back) = request.back_ch.take() {
+                self.pending_requests.lock().await.insert(request.seq, back);
+            }
+        }
+        let json = serde_json::to_string(&payload)?;
+        self.send_string_to_server(server_stdin, json).await
+    }
+
+    async fn send_string_to_server(
+        &self,
+        server_stdin: &mut Box<dyn AsyncWrite + Unpin + Send>,
+        request: String,
+    ) -> Result<()> {
+        info!("-> DAP {}", request);
+
+        // send the headers
+        server_stdin
+            .write_all(format!("Content-Length: {}\r\n\r\n", request.len()).as_bytes())
+            .await?;
+
+        // send the body
+        server_stdin.write_all(request.as_bytes()).await?;
+
+        server_stdin.flush().await?;
+
+        Ok(())
+    }
+
+    fn process_response(res: Response) -> Result<Response> {
+        if res.success {
+            info!("<- DAP success in response to {}", res.request_seq);
+
+            Ok(res)
+        } else {
+            error!(
+                "<- DAP error {:?} ({:?}) for command #{} {}",
+                res.message, res.body, res.request_seq, res.command
+            );
+
+            Err(Error::Other(anyhow::format_err!("{:?}", res.body)))
+        }
+    }
+
+    async fn process_server_message(
+        &self,
+        client_tx: &UnboundedSender<Payload>,
+        msg: Payload,
+    ) -> Result<()> {
+        match msg {
+            Payload::Response(res) => {
+                let request_seq = res.request_seq;
+                let tx = self.pending_requests.lock().await.remove(&request_seq);
+
+                match tx {
+                    Some(tx) => match tx.send(Self::process_response(res)).await {
+                        Ok(_) => (),
+                        Err(_) => error!(
+                            "Tried sending response into a closed channel (id={:?}), original request likely timed out",
+                            request_seq
+                        ),
+                    }
+                    None => {
+                        warn!("Response to nonexistent request #{}", res.request_seq);
+                        client_tx.send(Payload::Response(res)).expect("Failed to send");
+                    }
+                }
+
+                Ok(())
+            }
+            Payload::Request(Request {
+                ref command,
+                ref seq,
+                ..
+            }) => {
+                info!("<- DAP request {} #{}", command, seq);
+                client_tx.send(msg).expect("Failed to send");
+                Ok(())
+            }
+            Payload::Event(ref event) => {
+                info!("<- DAP event {:?}", event);
+                client_tx.send(msg).expect("Failed to send");
+                Ok(())
+            }
+        }
+    }
+
+    async fn recv(
+        transport: Arc<Self>,
+        mut server_stdout: Box<dyn AsyncBufRead + Unpin + Send>,
+        client_tx: UnboundedSender<Payload>,
+    ) {
+        let mut recv_buffer = String::new();
+        loop {
+            match Self::recv_server_message(&mut server_stdout, &mut recv_buffer).await {
+                Ok(msg) => {
+                    transport
+                        .process_server_message(&client_tx, msg)
+                        .await
+                        .unwrap();
+                }
+                Err(err) => {
+                    error!("err: <- {:?}", err);
+                    break;
+                }
+            }
+        }
+    }
+
+    async fn send(
+        transport: Arc<Self>,
+        mut server_stdin: Box<dyn AsyncWrite + Unpin + Send>,
+        mut client_rx: UnboundedReceiver<Payload>,
+    ) {
+        while let Some(payload) = client_rx.recv().await {
+            transport
+                .send_payload_to_server(&mut server_stdin, payload)
+                .await
+                .unwrap()
+        }
+    }
+
+    async fn err(mut server_stderr: Box<dyn AsyncBufRead + Unpin + Send>) {
+        let mut recv_buffer = String::new();
+        loop {
+            match Self::recv_server_error(&mut server_stderr, &mut recv_buffer).await {
+                Ok(_) => {}
+                Err(err) => {
+                    error!("err: <- {:?}", err);
+                    break;
+                }
+            }
+        }
+    }
+}

helix-dap/src/types.rs

@@ -0,0 +1,707 @@
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use std::collections::HashMap;
+use std::path::PathBuf;
+
+#[derive(
+    Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize,
+)]
+pub struct ThreadId(isize);
+
+impl std::fmt::Display for ThreadId {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+pub trait Request {
+    type Arguments: serde::de::DeserializeOwned + serde::Serialize;
+    type Result: serde::de::DeserializeOwned + serde::Serialize;
+    const COMMAND: &'static str;
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ColumnDescriptor {
+    pub attribute_name: String,
+    pub label: String,
+    pub format: Option<String>,
+    #[serde(rename = "type")]
+    pub ty: Option<String>,
+    pub width: Option<usize>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ExceptionBreakpointsFilter {
+    pub filter: String,
+    pub label: String,
+    pub description: Option<String>,
+    pub default: Option<bool>,
+    pub supports_condition: Option<bool>,
+    pub condition_description: Option<String>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct DebuggerCapabilities {
+    pub supports_configuration_done_request: Option<bool>,
+    pub supports_function_breakpoints: Option<bool>,
+    pub supports_conditional_breakpoints: Option<bool>,
+    pub supports_hit_conditional_breakpoints: Option<bool>,
+    pub supports_evaluate_for_hovers: Option<bool>,
+    pub supports_step_back: Option<bool>,
+    pub supports_set_variable: Option<bool>,
+    pub supports_restart_frame: Option<bool>,
+    pub supports_goto_targets_request: Option<bool>,
+    pub supports_step_in_targets_request: Option<bool>,
+    pub supports_completions_request: Option<bool>,
+    pub supports_modules_request: Option<bool>,
+    pub supports_restart_request: Option<bool>,
+    pub supports_exception_options: Option<bool>,
+    pub supports_value_formatting_options: Option<bool>,
+    pub supports_exception_info_request: Option<bool>,
+    pub support_terminate_debuggee: Option<bool>,
+    pub support_suspend_debuggee: Option<bool>,
+    pub supports_delayed_stack_trace_loading: Option<bool>,
+    pub supports_loaded_sources_request: Option<bool>,
+    pub supports_log_points: Option<bool>,
+    pub supports_terminate_threads_request: Option<bool>,
+    pub supports_set_expression: Option<bool>,
+    pub supports_terminate_request: Option<bool>,
+    pub supports_data_breakpoints: Option<bool>,
+    pub supports_read_memory_request: Option<bool>,
+    pub supports_write_memory_request: Option<bool>,
+    pub supports_disassemble_request: Option<bool>,
+    pub supports_cancel_request: Option<bool>,
+    pub supports_breakpoint_locations_request: Option<bool>,
+    pub supports_clipboard_context: Option<bool>,
+    pub supports_stepping_granularity: Option<bool>,
+    pub supports_instruction_breakpoints: Option<bool>,
+    pub supports_exception_filter_options: Option<bool>,
+    pub exception_breakpoint_filters: Option<Vec<ExceptionBreakpointsFilter>>,
+    pub completion_trigger_characters: Option<Vec<String>>,
+    pub additional_module_columns: Option<Vec<ColumnDescriptor>>,
+    pub supported_checksum_algorithms: Option<Vec<String>>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Checksum {
+    pub algorithm: String,
+    pub checksum: String,
+}
+
+#[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Source {
+    pub name: Option<String>,
+    pub path: Option<PathBuf>,
+    pub source_reference: Option<usize>,
+    pub presentation_hint: Option<String>,
+    pub origin: Option<String>,
+    pub sources: Option<Vec<Source>>,
+    pub adapter_data: Option<Value>,
+    pub checksums: Option<Vec<Checksum>>,
+}
+
+#[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct SourceBreakpoint {
+    pub line: usize,
+    pub column: Option<usize>,
+    pub condition: Option<String>,
+    pub hit_condition: Option<String>,
+    pub log_message: Option<String>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Breakpoint {
+    pub id: Option<usize>,
+    pub verified: bool,
+    pub message: Option<String>,
+    pub source: Option<Source>,
+    pub line: Option<usize>,
+    pub column: Option<usize>,
+    pub end_line: Option<usize>,
+    pub end_column: Option<usize>,
+    pub instruction_reference: Option<String>,
+    pub offset: Option<usize>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct StackFrameFormat {
+    pub parameters: Option<bool>,
+    pub parameter_types: Option<bool>,
+    pub parameter_names: Option<bool>,
+    pub parameter_values: Option<bool>,
+    pub line: Option<bool>,
+    pub module: Option<bool>,
+    pub include_all: Option<bool>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct StackFrame {
+    pub id: usize,
+    pub name: String,
+    pub source: Option<Source>,
+    pub line: usize,
+    pub column: usize,
+    pub end_line: Option<usize>,
+    pub end_column: Option<usize>,
+    pub can_restart: Option<bool>,
+    pub instruction_pointer_reference: Option<String>,
+    pub module_id: Option<Value>,
+    pub presentation_hint: Option<String>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Thread {
+    pub id: ThreadId,
+    pub name: String,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Scope {
+    pub name: String,
+    pub presentation_hint: Option<String>,
+    pub variables_reference: usize,
+    pub named_variables: Option<usize>,
+    pub indexed_variables: Option<usize>,
+    pub expensive: bool,
+    pub source: Option<Source>,
+    pub line: Option<usize>,
+    pub column: Option<usize>,
+    pub end_line: Option<usize>,
+    pub end_column: Option<usize>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ValueFormat {
+    pub hex: Option<bool>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct VariablePresentationHint {
+    pub kind: Option<String>,
+    pub attributes: Option<Vec<String>>,
+    pub visibility: Option<String>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Variable {
+    pub name: String,
+    pub value: String,
+    #[serde(rename = "type")]
+    pub ty: Option<String>,
+    pub presentation_hint: Option<VariablePresentationHint>,
+    pub evaluate_name: Option<String>,
+    pub variables_reference: usize,
+    pub named_variables: Option<usize>,
+    pub indexed_variables: Option<usize>,
+    pub memory_reference: Option<String>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Module {
+    pub id: String, // TODO: || number
+    pub name: String,
+    pub path: Option<PathBuf>,
+    pub is_optimized: Option<bool>,
+    pub is_user_code: Option<bool>,
+    pub version: Option<String>,
+    pub symbol_status: Option<String>,
+    pub symbol_file_path: Option<String>,
+    pub date_time_stamp: Option<String>,
+    pub address_range: Option<String>,
+}
+
+pub mod requests {
+    use super::*;
+    #[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct InitializeArguments {
+        #[serde(rename = "clientID")]
+        pub client_id: Option<String>,
+        pub client_name: Option<String>,
+        #[serde(rename = "adapterID")]
+        pub adapter_id: String,
+        pub locale: Option<String>,
+        #[serde(rename = "linesStartAt1")]
+        pub lines_start_at_one: Option<bool>,
+        #[serde(rename = "columnsStartAt1")]
+        pub columns_start_at_one: Option<bool>,
+        pub path_format: Option<String>,
+        pub supports_variable_type: Option<bool>,
+        pub supports_variable_paging: Option<bool>,
+        pub supports_run_in_terminal_request: Option<bool>,
+        pub supports_memory_references: Option<bool>,
+        pub supports_progress_reporting: Option<bool>,
+        pub supports_invalidated_event: Option<bool>,
+    }
+
+    #[derive(Debug)]
+    pub enum Initialize {}
+
+    impl Request for Initialize {
+        type Arguments = InitializeArguments;
+        type Result = DebuggerCapabilities;
+        const COMMAND: &'static str = "initialize";
+    }
+
+    #[derive(Debug)]
+    pub enum Launch {}
+
+    impl Request for Launch {
+        type Arguments = Value;
+        type Result = Value;
+        const COMMAND: &'static str = "launch";
+    }
+
+    #[derive(Debug)]
+    pub enum Attach {}
+
+    impl Request for Attach {
+        type Arguments = Value;
+        type Result = Value;
+        const COMMAND: &'static str = "attach";
+    }
+
+    #[derive(Debug)]
+    pub enum Disconnect {}
+
+    impl Request for Disconnect {
+        type Arguments = ();
+        type Result = ();
+        const COMMAND: &'static str = "disconnect";
+    }
+
+    #[derive(Debug)]
+    pub enum ConfigurationDone {}
+
+    impl Request for ConfigurationDone {
+        type Arguments = ();
+        type Result = ();
+        const COMMAND: &'static str = "configurationDone";
+    }
+
+    #[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct SetBreakpointsArguments {
+        pub source: Source,
+        pub breakpoints: Option<Vec<SourceBreakpoint>>,
+        // lines is deprecated
+        pub source_modified: Option<bool>,
+    }
+
+    #[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct SetBreakpointsResponse {
+        pub breakpoints: Option<Vec<Breakpoint>>,
+    }
+
+    #[derive(Debug)]
+    pub enum SetBreakpoints {}
+
+    impl Request for SetBreakpoints {
+        type Arguments = SetBreakpointsArguments;
+        type Result = SetBreakpointsResponse;
+        const COMMAND: &'static str = "setBreakpoints";
+    }
+
+    #[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct ContinueArguments {
+        pub thread_id: ThreadId,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct ContinueResponse {
+        pub all_threads_continued: Option<bool>,
+    }
+
+    #[derive(Debug)]
+    pub enum Continue {}
+
+    impl Request for Continue {
+        type Arguments = ContinueArguments;
+        type Result = ContinueResponse;
+        const COMMAND: &'static str = "continue";
+    }
+
+    #[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct StackTraceArguments {
+        pub thread_id: ThreadId,
+        pub start_frame: Option<usize>,
+        pub levels: Option<usize>,
+        pub format: Option<StackFrameFormat>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct StackTraceResponse {
+        pub total_frames: Option<usize>,
+        pub stack_frames: Vec<StackFrame>,
+    }
+
+    #[derive(Debug)]
+    pub enum StackTrace {}
+
+    impl Request for StackTrace {
+        type Arguments = StackTraceArguments;
+        type Result = StackTraceResponse;
+        const COMMAND: &'static str = "stackTrace";
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct ThreadsResponse {
+        pub threads: Vec<Thread>,
+    }
+
+    #[derive(Debug)]
+    pub enum Threads {}
+
+    impl Request for Threads {
+        type Arguments = ();
+        type Result = ThreadsResponse;
+        const COMMAND: &'static str = "threads";
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct ScopesArguments {
+        pub frame_id: usize,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct ScopesResponse {
+        pub scopes: Vec<Scope>,
+    }
+
+    #[derive(Debug)]
+    pub enum Scopes {}
+
+    impl Request for Scopes {
+        type Arguments = ScopesArguments;
+        type Result = ScopesResponse;
+        const COMMAND: &'static str = "scopes";
+    }
+
+    #[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct VariablesArguments {
+        pub variables_reference: usize,
+        pub filter: Option<String>,
+        pub start: Option<usize>,
+        pub count: Option<usize>,
+        pub format: Option<ValueFormat>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct VariablesResponse {
+        pub variables: Vec<Variable>,
+    }
+
+    #[derive(Debug)]
+    pub enum Variables {}
+
+    impl Request for Variables {
+        type Arguments = VariablesArguments;
+        type Result = VariablesResponse;
+        const COMMAND: &'static str = "variables";
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct StepInArguments {
+        pub thread_id: ThreadId,
+        pub target_id: Option<usize>,
+        pub granularity: Option<String>,
+    }
+
+    #[derive(Debug)]
+    pub enum StepIn {}
+
+    impl Request for StepIn {
+        type Arguments = StepInArguments;
+        type Result = ();
+        const COMMAND: &'static str = "stepIn";
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct StepOutArguments {
+        pub thread_id: ThreadId,
+        pub granularity: Option<String>,
+    }
+
+    #[derive(Debug)]
+    pub enum StepOut {}
+
+    impl Request for StepOut {
+        type Arguments = StepOutArguments;
+        type Result = ();
+        const COMMAND: &'static str = "stepOut";
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct NextArguments {
+        pub thread_id: ThreadId,
+        pub granularity: Option<String>,
+    }
+
+    #[derive(Debug)]
+    pub enum Next {}
+
+    impl Request for Next {
+        type Arguments = NextArguments;
+        type Result = ();
+        const COMMAND: &'static str = "next";
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct PauseArguments {
+        pub thread_id: ThreadId,
+    }
+
+    #[derive(Debug)]
+    pub enum Pause {}
+
+    impl Request for Pause {
+        type Arguments = PauseArguments;
+        type Result = ();
+        const COMMAND: &'static str = "pause";
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct EvaluateArguments {
+        pub expression: String,
+        pub frame_id: Option<usize>,
+        pub context: Option<String>,
+        pub format: Option<ValueFormat>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct EvaluateResponse {
+        pub result: String,
+        #[serde(rename = "type")]
+        pub ty: Option<String>,
+        pub presentation_hint: Option<VariablePresentationHint>,
+        pub variables_reference: usize,
+        pub named_variables: Option<usize>,
+        pub indexed_variables: Option<usize>,
+        pub memory_reference: Option<String>,
+    }
+
+    #[derive(Debug)]
+    pub enum Evaluate {}
+
+    impl Request for Evaluate {
+        type Arguments = EvaluateArguments;
+        type Result = EvaluateResponse;
+        const COMMAND: &'static str = "evaluate";
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct SetExceptionBreakpointsArguments {
+        pub filters: Vec<String>,
+        // pub filterOptions: Option<Vec<ExceptionFilterOptions>>, // needs capability
+        // pub exceptionOptions: Option<Vec<ExceptionOptions>>, // needs capability
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct SetExceptionBreakpointsResponse {
+        pub breakpoints: Option<Vec<Breakpoint>>,
+    }
+
+    #[derive(Debug)]
+    pub enum SetExceptionBreakpoints {}
+
+    impl Request for SetExceptionBreakpoints {
+        type Arguments = SetExceptionBreakpointsArguments;
+        type Result = SetExceptionBreakpointsResponse;
+        const COMMAND: &'static str = "setExceptionBreakpoints";
+    }
+
+    // Reverse Requests
+
+    #[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct RunInTerminalResponse {
+        pub process_id: Option<u32>,
+        pub shell_process_id: Option<u32>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct RunInTerminalArguments {
+        pub kind: Option<String>,
+        pub title: Option<String>,
+        pub cwd: Option<String>,
+        pub args: Vec<String>,
+        pub env: Option<HashMap<String, Option<String>>>,
+    }
+
+    #[derive(Debug)]
+    pub enum RunInTerminal {}
+
+    impl Request for RunInTerminal {
+        type Arguments = RunInTerminalArguments;
+        type Result = RunInTerminalResponse;
+        const COMMAND: &'static str = "runInTerminal";
+    }
+}
+
+// Events
+
+pub mod events {
+    use super::*;
+
+    #[derive(Debug, Clone, Serialize, Deserialize)]
+    #[serde(rename_all = "camelCase")]
+    #[serde(tag = "event", content = "body")]
+    // seq is omitted as unused and is not sent by some implementations
+    pub enum Event {
+        Initialized,
+        Stopped(Stopped),
+        Continued(Continued),
+        Exited(Exited),
+        Terminated(Option<Terminated>),
+        Thread(Thread),
+        Output(Output),
+        Breakpoint(Breakpoint),
+        Module(Module),
+        LoadedSource(LoadedSource),
+        Process(Process),
+        Capabilities(Capabilities),
+        // ProgressStart(),
+        // ProgressUpdate(),
+        // ProgressEnd(),
+        // Invalidated(),
+        Memory(Memory),
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Stopped {
+        pub reason: String,
+        pub description: Option<String>,
+        pub thread_id: Option<ThreadId>,
+        pub preserve_focus_hint: Option<bool>,
+        pub text: Option<String>,
+        pub all_threads_stopped: Option<bool>,
+        pub hit_breakpoint_ids: Option<Vec<usize>>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Continued {
+        pub thread_id: ThreadId,
+        pub all_threads_continued: Option<bool>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Exited {
+        pub exit_code: usize,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Terminated {
+        pub restart: Option<Value>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Thread {
+        pub reason: String,
+        pub thread_id: ThreadId,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Output {
+        pub output: String,
+        pub category: Option<String>,
+        pub group: Option<String>,
+        pub line: Option<usize>,
+        pub column: Option<usize>,
+        pub variables_reference: Option<usize>,
+        pub source: Option<Source>,
+        pub data: Option<Value>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Breakpoint {
+        pub reason: String,
+        pub breakpoint: super::Breakpoint,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Module {
+        pub reason: String,
+        pub module: super::Module,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct LoadedSource {
+        pub reason: String,
+        pub source: super::Source,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Process {
+        pub name: String,
+        pub system_process_id: Option<usize>,
+        pub is_local_process: Option<bool>,
+        pub start_method: Option<String>, // TODO: use enum
+        pub pointer_size: Option<usize>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Capabilities {
+        pub capabilities: super::DebuggerCapabilities,
+    }
+
+    // #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    // #[serde(rename_all = "camelCase")]
+    // pub struct Invalidated {
+    // pub areas: Vec<InvalidatedArea>,
+    // pub thread_id: Option<ThreadId>,
+    // pub stack_frame_id: Option<usize>,
+    // }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Memory {
+        pub memory_reference: String,
+        pub offset: usize,
+        pub count: usize,
+    }
+}

helix-loader/Cargo.toml

@@ -1,21 +1,23 @@
 [package]
-name = "helix-syntax"
+name = "helix-loader"
-version = "0.4.0"
+version = "0.6.0"
+description = "A post-modern text editor."
 authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
-edition = "2018"
+edition = "2021"
 license = "MPL-2.0"
-description = "Tree-sitter grammars support"
 categories = ["editor"]
 repository = "https://github.com/helix-editor/helix"
 homepage = "https://helix-editor.com"
-include = ["src/**/*", "languages/**/*", "build.rs", "!**/docs/**/*", "!**/test/**/*", "!**/examples/**/*", "!**/build/**/*"]
 
 [dependencies]
-tree-sitter = "0.19"
-libloading = "0.7"
 anyhow = "1"
+serde = { version = "1.0", features = ["derive"] }
+toml = "0.5"
+etcetera = "0.3"
+tree-sitter = "0.20"
+libloading = "0.7"
+once_cell = "1.9"
 
-[build-dependencies]
+# cloning/compiling tree-sitter grammars
 cc = { version = "1" }
 threadpool = { version = "1.0" }
-anyhow = "1"

helix-loader/build.rs

@@ -0,0 +1,6 @@
+fn main() {
+    println!(
+        "cargo:rustc-env=BUILD_TARGET={}",
+        std::env::var("TARGET").unwrap()
+    );
+}

helix-loader/src/grammar.rs

@@ -0,0 +1,387 @@
+use anyhow::{anyhow, Context, Result};
+use libloading::{Library, Symbol};
+use serde::{Deserialize, Serialize};
+use std::fs;
+use std::time::SystemTime;
+use std::{
+    collections::HashSet,
+    path::{Path, PathBuf},
+    process::Command,
+    sync::mpsc::channel,
+};
+use tree_sitter::Language;
+
+#[cfg(unix)]
+const DYLIB_EXTENSION: &str = "so";
+
+#[cfg(windows)]
+const DYLIB_EXTENSION: &str = "dll";
+
+#[derive(Debug, Serialize, Deserialize)]
+struct Configuration {
+    #[serde(rename = "use-grammars")]
+    pub grammar_selection: Option<GrammarSelection>,
+    pub grammar: Vec<GrammarConfiguration>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase", untagged)]
+pub enum GrammarSelection {
+    Only(HashSet<String>),
+    Except(HashSet<String>),
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct GrammarConfiguration {
+    #[serde(rename = "name")]
+    pub grammar_id: String,
+    pub source: GrammarSource,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase", untagged)]
+pub enum GrammarSource {
+    Local {
+        path: String,
+    },
+    Git {
+        #[serde(rename = "git")]
+        remote: String,
+        #[serde(rename = "rev")]
+        revision: String,
+        subpath: Option<String>,
+    },
+}
+
+const BUILD_TARGET: &str = env!("BUILD_TARGET");
+const REMOTE_NAME: &str = "origin";
+
+pub fn get_language(name: &str) -> Result<Language> {
+    let name = name.to_ascii_lowercase();
+    let mut library_path = crate::runtime_dir().join("grammars").join(&name);
+    library_path.set_extension(DYLIB_EXTENSION);
+
+    let library = unsafe { Library::new(&library_path) }
+        .with_context(|| format!("Error opening dynamic library {library_path:?}"))?;
+    let language_fn_name = format!("tree_sitter_{}", name.replace('-', "_"));
+    let language = unsafe {
+        let language_fn: Symbol<unsafe extern "C" fn() -> Language> = library
+            .get(language_fn_name.as_bytes())
+            .with_context(|| format!("Failed to load symbol {language_fn_name}"))?;
+        language_fn()
+    };
+    std::mem::forget(library);
+    Ok(language)
+}
+
+pub fn fetch_grammars() -> Result<()> {
+    // We do not need to fetch local grammars.
+    let mut grammars = get_grammar_configs()?;
+    grammars.retain(|grammar| !matches!(grammar.source, GrammarSource::Local { .. }));
+
+    run_parallel(grammars, fetch_grammar, "fetch")
+}
+
+pub fn build_grammars() -> Result<()> {
+    run_parallel(get_grammar_configs()?, build_grammar, "build")
+}
+
+// Returns the set of grammar configurations the user requests.
+// Grammars are configured in the default and user `languages.toml` and are
+// merged. The `grammar_selection` key of the config is then used to filter
+// down all grammars into a subset of the user's choosing.
+fn get_grammar_configs() -> Result<Vec<GrammarConfiguration>> {
+    let config: Configuration = crate::user_lang_config()
+        .context("Could not parse languages.toml")?
+        .try_into()?;
+
+    let grammars = match config.grammar_selection {
+        Some(GrammarSelection::Only(selections)) => config
+            .grammar
+            .into_iter()
+            .filter(|grammar| selections.contains(&grammar.grammar_id))
+            .collect(),
+        Some(GrammarSelection::Except(rejections)) => config
+            .grammar
+            .into_iter()
+            .filter(|grammar| !rejections.contains(&grammar.grammar_id))
+            .collect(),
+        None => config.grammar,
+    };
+
+    Ok(grammars)
+}
+
+fn run_parallel<F>(grammars: Vec<GrammarConfiguration>, job: F, action: &'static str) -> Result<()>
+where
+    F: Fn(GrammarConfiguration) -> Result<()> + std::marker::Send + 'static + Copy,
+{
+    let pool = threadpool::Builder::new().build();
+    let (tx, rx) = channel();
+
+    for grammar in grammars {
+        let tx = tx.clone();
+
+        pool.execute(move || {
+            tx.send(job(grammar)).unwrap();
+        });
+    }
+
+    drop(tx);
+
+    // TODO: print all failures instead of the first one found.
+    rx.iter()
+        .find(|result| result.is_err())
+        .map(|err| err.with_context(|| format!("Failed to {action} some grammar(s)")))
+        .unwrap_or(Ok(()))
+}
+
+fn fetch_grammar(grammar: GrammarConfiguration) -> Result<()> {
+    if let GrammarSource::Git {
+        remote, revision, ..
+    } = grammar.source
+    {
+        let grammar_dir = crate::runtime_dir()
+            .join("grammars/sources")
+            .join(&grammar.grammar_id);
+
+        fs::create_dir_all(&grammar_dir).context(format!(
+            "Could not create grammar directory {:?}",
+            grammar_dir
+        ))?;
+
+        // create the grammar dir contains a git directory
+        if !grammar_dir.join(".git").is_dir() {
+            git(&grammar_dir, ["init"])?;
+        }
+
+        // ensure the remote matches the configured remote
+        if get_remote_url(&grammar_dir).map_or(true, |s| s != remote) {
+            set_remote(&grammar_dir, &remote)?;
+        }
+
+        // ensure the revision matches the configured revision
+        if get_revision(&grammar_dir).map_or(true, |s| s != revision) {
+            // Fetch the exact revision from the remote.
+            // Supported by server-side git since v2.5.0 (July 2015),
+            // enabled by default on major git hosts.
+            git(
+                &grammar_dir,
+                ["fetch", "--depth", "1", REMOTE_NAME, &revision],
+            )?;
+            git(&grammar_dir, ["checkout", &revision])?;
+
+            println!(
+                "Grammar '{}' checked out at '{}'.",
+                grammar.grammar_id, revision
+            );
+        } else {
+            println!("Grammar '{}' is already up to date.", grammar.grammar_id);
+        }
+    }
+
+    Ok(())
+}
+
+// Sets the remote for a repository to the given URL, creating the remote if
+// it does not yet exist.
+fn set_remote(repository_dir: &Path, remote_url: &str) -> Result<String> {
+    git(
+        repository_dir,
+        ["remote", "set-url", REMOTE_NAME, remote_url],
+    )
+    .or_else(|_| git(repository_dir, ["remote", "add", REMOTE_NAME, remote_url]))
+}
+
+fn get_remote_url(repository_dir: &Path) -> Option<String> {
+    git(repository_dir, ["remote", "get-url", REMOTE_NAME]).ok()
+}
+
+fn get_revision(repository_dir: &Path) -> Option<String> {
+    git(repository_dir, ["rev-parse", "HEAD"]).ok()
+}
+
+// A wrapper around 'git' commands which returns stdout in success and a
+// helpful error message showing the command, stdout, and stderr in error.
+fn git<I, S>(repository_dir: &Path, args: I) -> Result<String>
+where
+    I: IntoIterator<Item = S>,
+    S: AsRef<std::ffi::OsStr>,
+{
+    let output = Command::new("git")
+        .args(args)
+        .current_dir(repository_dir)
+        .output()?;
+
+    if output.status.success() {
+        Ok(String::from_utf8_lossy(&output.stdout)
+            .trim_end()
+            .to_owned())
+    } else {
+        // TODO: figure out how to display the git command using `args`
+        Err(anyhow!(
+            "Git command failed.\nStdout: {}\nStderr: {}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr),
+        ))
+    }
+}
+
+fn build_grammar(grammar: GrammarConfiguration) -> Result<()> {
+    let grammar_dir = if let GrammarSource::Local { path } = &grammar.source {
+        PathBuf::from(&path)
+    } else {
+        crate::runtime_dir()
+            .join("grammars/sources")
+            .join(&grammar.grammar_id)
+    };
+
+    let grammar_dir_entries = grammar_dir.read_dir().with_context(|| {
+        format!("Failed to read directory {grammar_dir:?}. Did you use 'hx --grammar fetch'?")
+    })?;
+
+    if grammar_dir_entries.count() == 0 {
+        return Err(anyhow!(
+            "Directory {grammar_dir:?} is empty. Did you use 'hx --grammar fetch'?"
+        ));
+    };
+
+    let path = match &grammar.source {
+        GrammarSource::Git {
+            subpath: Some(subpath),
+            ..
+        } => grammar_dir.join(subpath),
+        _ => grammar_dir,
+    }
+    .join("src");
+
+    build_tree_sitter_library(&path, grammar)
+}
+
+fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) -> Result<()> {
+    let header_path = src_path;
+    let parser_path = src_path.join("parser.c");
+    let mut scanner_path = src_path.join("scanner.c");
+
+    let scanner_path = if scanner_path.exists() {
+        Some(scanner_path)
+    } else {
+        scanner_path.set_extension("cc");
+        if scanner_path.exists() {
+            Some(scanner_path)
+        } else {
+            None
+        }
+    };
+    let parser_lib_path = crate::runtime_dir().join("grammars");
+    let mut library_path = parser_lib_path.join(&grammar.grammar_id);
+    library_path.set_extension(DYLIB_EXTENSION);
+
+    let recompile = needs_recompile(&library_path, &parser_path, &scanner_path)
+        .context("Failed to compare source and binary timestamps")?;
+
+    if !recompile {
+        println!("Grammar '{}' is already built.", grammar.grammar_id);
+        return Ok(());
+    }
+
+    println!("Building grammar '{}'", grammar.grammar_id);
+
+    let mut config = cc::Build::new();
+    config
+        .cpp(true)
+        .opt_level(3)
+        .cargo_metadata(false)
+        .host(BUILD_TARGET)
+        .target(BUILD_TARGET);
+    let compiler = config.get_compiler();
+    let mut command = Command::new(compiler.path());
+    command.current_dir(src_path);
+    for (key, value) in compiler.env() {
+        command.env(key, value);
+    }
+
+    if cfg!(windows) {
+        command
+            .args(&["/nologo", "/LD", "/I"])
+            .arg(header_path)
+            .arg("/Od")
+            .arg("/utf-8");
+        if let Some(scanner_path) = scanner_path.as_ref() {
+            command.arg(scanner_path);
+        }
+
+        command
+            .arg(parser_path)
+            .arg("/link")
+            .arg(format!("/out:{}", library_path.to_str().unwrap()));
+    } else {
+        command
+            .arg("-shared")
+            .arg("-fPIC")
+            .arg("-fno-exceptions")
+            .arg("-g")
+            .arg("-I")
+            .arg(header_path)
+            .arg("-o")
+            .arg(&library_path)
+            .arg("-O3");
+        if let Some(scanner_path) = scanner_path.as_ref() {
+            if scanner_path.extension() == Some("c".as_ref()) {
+                command.arg("-xc").arg("-std=c99").arg(scanner_path);
+            } else {
+                command.arg(scanner_path);
+            }
+        }
+        command.arg("-xc").arg(parser_path);
+        if cfg!(all(unix, not(target_os = "macos"))) {
+            command.arg("-Wl,-z,relro,-z,now");
+        }
+    }
+
+    let output = command.output().context("Failed to execute C compiler")?;
+    if !output.status.success() {
+        return Err(anyhow!(
+            "Parser compilation failed.\nStdout: {}\nStderr: {}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr)
+        ));
+    }
+
+    Ok(())
+}
+
+fn needs_recompile(
+    lib_path: &Path,
+    parser_c_path: &Path,
+    scanner_path: &Option<PathBuf>,
+) -> Result<bool> {
+    if !lib_path.exists() {
+        return Ok(true);
+    }
+    let lib_mtime = mtime(lib_path)?;
+    if mtime(parser_c_path)? > lib_mtime {
+        return Ok(true);
+    }
+    if let Some(scanner_path) = scanner_path {
+        if mtime(scanner_path)? > lib_mtime {
+            return Ok(true);
+        }
+    }
+    Ok(false)
+}
+
+fn mtime(path: &Path) -> Result<SystemTime> {
+    Ok(fs::metadata(path)?.modified()?)
+}
+
+/// Gives the contents of a file from a language's `runtime/queries/<lang>`
+/// directory
+pub fn load_runtime_file(language: &str, filename: &str) -> Result<String, std::io::Error> {
+    let path = crate::RUNTIME_DIR
+        .join("queries")
+        .join(language)
+        .join(filename);
+    std::fs::read_to_string(&path)
+}

helix-loader/src/lib.rs

@@ -0,0 +1,161 @@
+pub mod grammar;
+
+use etcetera::base_strategy::{choose_base_strategy, BaseStrategy};
+
+pub static RUNTIME_DIR: once_cell::sync::Lazy<std::path::PathBuf> =
+    once_cell::sync::Lazy::new(runtime_dir);
+
+pub fn runtime_dir() -> std::path::PathBuf {
+    if let Ok(dir) = std::env::var("HELIX_RUNTIME") {
+        return dir.into();
+    }
+
+    const RT_DIR: &str = "runtime";
+    let conf_dir = config_dir().join(RT_DIR);
+    if conf_dir.exists() {
+        return conf_dir;
+    }
+
+    if let Ok(dir) = std::env::var("CARGO_MANIFEST_DIR") {
+        // this is the directory of the crate being run by cargo, we need the workspace path so we take the parent
+        return std::path::PathBuf::from(dir).parent().unwrap().join(RT_DIR);
+    }
+
+    // fallback to location of the executable being run
+    std::env::current_exe()
+        .ok()
+        .and_then(|path| path.parent().map(|path| path.to_path_buf().join(RT_DIR)))
+        .unwrap()
+}
+
+pub fn config_dir() -> std::path::PathBuf {
+    // TODO: allow env var override
+    let strategy = choose_base_strategy().expect("Unable to find the config directory!");
+    let mut path = strategy.config_dir();
+    path.push("helix");
+    path
+}
+
+pub fn cache_dir() -> std::path::PathBuf {
+    // TODO: allow env var override
+    let strategy = choose_base_strategy().expect("Unable to find the config directory!");
+    let mut path = strategy.cache_dir();
+    path.push("helix");
+    path
+}
+
+pub fn config_file() -> std::path::PathBuf {
+    config_dir().join("config.toml")
+}
+
+pub fn lang_config_file() -> std::path::PathBuf {
+    config_dir().join("languages.toml")
+}
+
+pub fn log_file() -> std::path::PathBuf {
+    cache_dir().join("helix.log")
+}
+
+/// Default bultin-in languages.toml.
+pub fn default_lang_config() -> toml::Value {
+    toml::from_slice(include_bytes!("../../languages.toml"))
+        .expect("Could not parse bultin-in languages.toml to valid toml")
+}
+
+/// User configured languages.toml file, merged with the default config.
+pub fn user_lang_config() -> Result<toml::Value, toml::de::Error> {
+    let def_lang_conf = default_lang_config();
+    let data = std::fs::read(crate::config_dir().join("languages.toml"));
+    let user_lang_conf = match data {
+        Ok(raw) => {
+            let value = toml::from_slice(&raw)?;
+            merge_toml_values(def_lang_conf, value)
+        }
+        Err(_) => def_lang_conf,
+    };
+
+    Ok(user_lang_conf)
+}
+
+// right overrides left
+pub fn merge_toml_values(left: toml::Value, right: toml::Value) -> toml::Value {
+    use toml::Value;
+
+    fn get_name(v: &Value) -> Option<&str> {
+        v.get("name").and_then(Value::as_str)
+    }
+
+    match (left, right) {
+        (Value::Array(mut left_items), Value::Array(right_items)) => {
+            left_items.reserve(right_items.len());
+            for rvalue in right_items {
+                let lvalue = get_name(&rvalue)
+                    .and_then(|rname| left_items.iter().position(|v| get_name(v) == Some(rname)))
+                    .map(|lpos| left_items.remove(lpos));
+                let mvalue = match lvalue {
+                    Some(lvalue) => merge_toml_values(lvalue, rvalue),
+                    None => rvalue,
+                };
+                left_items.push(mvalue);
+            }
+            Value::Array(left_items)
+        }
+        (Value::Table(mut left_map), Value::Table(right_map)) => {
+            for (rname, rvalue) in right_map {
+                match left_map.remove(&rname) {
+                    Some(lvalue) => {
+                        let merged_value = merge_toml_values(lvalue, rvalue);
+                        left_map.insert(rname, merged_value);
+                    }
+                    None => {
+                        left_map.insert(rname, rvalue);
+                    }
+                }
+            }
+            Value::Table(left_map)
+        }
+        // Catch everything else we didn't handle, and use the right value
+        (_, value) => value,
+    }
+}
+
+#[cfg(test)]
+mod merge_toml_tests {
+    use super::merge_toml_values;
+
+    #[test]
+    fn language_tomls() {
+        use toml::Value;
+
+        const USER: &str = "
+        [[language]]
+        name = \"nix\"
+        test = \"bbb\"
+        indent = { tab-width = 4, unit = \"    \", test = \"aaa\" }
+        ";
+
+        let base: Value = toml::from_slice(include_bytes!("../../languages.toml"))
+            .expect("Couldn't parse built-in languages config");
+        let user: Value = toml::from_str(USER).unwrap();
+
+        let merged = merge_toml_values(base, user);
+        let languages = merged.get("language").unwrap().as_array().unwrap();
+        let nix = languages
+            .iter()
+            .find(|v| v.get("name").unwrap().as_str().unwrap() == "nix")
+            .unwrap();
+        let nix_indent = nix.get("indent").unwrap();
+
+        // We changed tab-width and unit in indent so check them if they are the new values
+        assert_eq!(
+            nix_indent.get("tab-width").unwrap().as_integer().unwrap(),
+            4
+        );
+        assert_eq!(nix_indent.get("unit").unwrap().as_str().unwrap(), "    ");
+        // We added a new keys, so check them
+        assert_eq!(nix.get("test").unwrap().as_str().unwrap(), "bbb");
+        assert_eq!(nix_indent.get("test").unwrap().as_str().unwrap(), "aaa");
+        // We didn't change comment-token so it should be same
+        assert_eq!(nix.get("comment-token").unwrap().as_str().unwrap(), "#");
+    }
+}

helix-lsp/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
 name = "helix-lsp"
-version = "0.4.0"
+version = "0.6.0"
 authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
-edition = "2018"
+edition = "2021"
 license = "MPL-2.0"
 description = "LSP client implementation for Helix project"
 categories = ["editor"]
@@ -12,16 +12,17 @@ homepage = "https://helix-editor.com"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-helix-core = { version = "0.4", path = "../helix-core" }
+helix-core = { version = "0.6", path = "../helix-core" }
 
 anyhow = "1.0"
 futures-executor = "0.3"
 futures-util = { version = "0.3", features = ["std", "async-await"], default-features = false }
 jsonrpc-core = { version = "18.0", default-features = false } # don't pull in all of futures
 log = "0.4"
-lsp-types = { version = "0.89", features = ["proposed"] }
+lsp-types = { version = "0.92", features = ["proposed"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 thiserror = "1.0"
-tokio = { version = "1.9", features = ["rt", "rt-multi-thread", "io-util", "io-std", "time", "process", "macros", "fs", "parking_lot"] }
+tokio = { version = "1.17", features = ["rt", "rt-multi-thread", "io-util", "io-std", "time", "process", "macros", "fs", "parking_lot", "sync"] }
-tokio-stream = "0.1.7"
+tokio-stream = "0.1.8"
+which = "4.2"

helix-lsp/src/client.rs

@@ -3,17 +3,23 @@ use crate::{
     Call, Error, OffsetEncoding, Result,
 };
 
-use helix_core::{chars::char_is_line_ending, find_root, ChangeSet, Rope};
+use helix_core::{find_root, ChangeSet, Rope};
 use jsonrpc_core as jsonrpc;
 use lsp_types as lsp;
 use serde_json::Value;
 use std::future::Future;
 use std::process::Stdio;
-use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{
+    atomic::{AtomicU64, Ordering},
+    Arc,
+};
 use tokio::{
     io::{BufReader, BufWriter},
     process::{Child, Command},
-    sync::mpsc::{channel, UnboundedReceiver, UnboundedSender},
+    sync::{
+        mpsc::{channel, UnboundedReceiver, UnboundedSender},
+        Notify, OnceCell,
+    },
 };
 
 #[derive(Debug)]
@@ -22,18 +28,24 @@ pub struct Client {
     _process: Child,
     server_tx: UnboundedSender<Payload>,
     request_counter: AtomicU64,
-    capabilities: Option<lsp::ServerCapabilities>,
+    pub(crate) capabilities: OnceCell<lsp::ServerCapabilities>,
     offset_encoding: OffsetEncoding,
     config: Option<Value>,
+    root_markers: Vec<String>,
 }
 
 impl Client {
+    #[allow(clippy::type_complexity)]
     pub fn start(
         cmd: &str,
         args: &[String],
         config: Option<Value>,
+        root_markers: Vec<String>,
         id: usize,
-    ) -> Result<(Self, UnboundedReceiver<(usize, Call)>)> {
+    ) -> Result<(Self, UnboundedReceiver<(usize, Call)>, Arc<Notify>)> {
+        // Resolve path to the binary
+        let cmd = which::which(cmd).map_err(|err| anyhow::anyhow!(err))?;
+
         let process = Command::new(cmd)
             .args(args)
             .stdin(Stdio::piped())
@@ -50,22 +62,21 @@ impl Client {
         let reader = BufReader::new(process.stdout.take().expect("Failed to open stdout"));
         let stderr = BufReader::new(process.stderr.take().expect("Failed to open stderr"));
 
-        let (server_rx, server_tx) = Transport::start(reader, writer, stderr, id);
+        let (server_rx, server_tx, initialize_notify) =
+            Transport::start(reader, writer, stderr, id);
 
         let client = Self {
             id,
             _process: process,
             server_tx,
             request_counter: AtomicU64::new(0),
-            capabilities: None,
+            capabilities: OnceCell::new(),
             offset_encoding: OffsetEncoding::Utf8,
             config,
+            root_markers,
         };
 
-        // TODO: async client.initialize()
+        Ok((client, server_rx, initialize_notify))
-        // maybe use an arc<atomic> flag
-
-        Ok((client, server_rx))
     }
 
     pub fn id(&self) -> usize {
@@ -88,9 +99,13 @@ impl Client {
         }
     }
 
+    pub fn is_initialized(&self) -> bool {
+        self.capabilities.get().is_some()
+    }
+
     pub fn capabilities(&self) -> &lsp::ServerCapabilities {
         self.capabilities
-            .as_ref()
+            .get()
             .expect("language server not yet initialized!")
     }
 
@@ -98,6 +113,10 @@ impl Client {
         self.offset_encoding
     }
 
+    pub fn config(&self) -> Option<&Value> {
+        self.config.as_ref()
+    }
+
     /// Execute a RPC request on the language server.
     async fn request<R: lsp::request::Request>(&self, params: R::Params) -> Result<R::Result>
     where
@@ -143,7 +162,8 @@ impl Client {
                 })
                 .map_err(|e| Error::Other(e.into()))?;
 
-            timeout(Duration::from_secs(2), rx.recv())
+            // TODO: specifiable timeout, delay other calls until initialize success
+            timeout(Duration::from_secs(20), rx.recv())
                 .await
                 .map_err(|_| Error::Timeout)? // return Timeout
                 .ok_or(Error::StreamClosed)?
@@ -151,7 +171,7 @@ impl Client {
     }
 
     /// Send a RPC notification to the language server.
-    fn notify<R: lsp::notification::Notification>(
+    pub fn notify<R: lsp::notification::Notification>(
         &self,
         params: R::Params,
     ) -> impl Future<Output = Result<()>>
@@ -192,7 +212,7 @@ impl Client {
                 Ok(result) => Output::Success(Success {
                     jsonrpc: Some(Version::V2),
                     id,
-                    result,
+                    result: serde_json::to_value(result)?,
                 }),
                 Err(error) => Output::Failure(Failure {
                     jsonrpc: Some(Version::V2),
@@ -213,9 +233,10 @@ impl Client {
     // General messages
     // -------------------------------------------------------------------------------------------
 
-    pub(crate) async fn initialize(&mut self) -> Result<()> {
+    pub(crate) async fn initialize(&self) -> Result<lsp::InitializeResult> {
         // TODO: delay any requests that are triggered prior to initialize
-        let root = find_root(None).and_then(|root| lsp::Url::from_file_path(root).ok());
+        let root = find_root(None, &self.root_markers)
+            .and_then(|root| lsp::Url::from_file_path(root).ok());
 
         if self.config.is_some() {
             log::info!("Using custom LSP config: {}", self.config.as_ref().unwrap());
@@ -229,6 +250,13 @@ impl Client {
             root_uri: root,
             initialization_options: self.config.clone(),
             capabilities: lsp::ClientCapabilities {
+                workspace: Some(lsp::WorkspaceClientCapabilities {
+                    configuration: Some(true),
+                    did_change_configuration: Some(lsp::DynamicRegistrationClientCapabilities {
+                        dynamic_registration: Some(false),
+                    }),
+                    ..Default::default()
+                }),
                 text_document: Some(lsp::TextDocumentClientCapabilities {
                     completion: Some(lsp::CompletionClientCapabilities {
                         completion_item: Some(lsp::CompletionItemCapability {
@@ -247,6 +275,12 @@ impl Client {
                         content_format: Some(vec![lsp::MarkupKind::Markdown]),
                         ..Default::default()
                     }),
+                    rename: Some(lsp::RenameClientCapabilities {
+                        dynamic_registration: Some(false),
+                        prepare_support: Some(false),
+                        prepare_support_default_behavior: None,
+                        honors_change_annotations: Some(false),
+                    }),
                     code_action: Some(lsp::CodeActionClientCapabilities {
                         code_action_literal_support: Some(lsp::CodeActionLiteralSupport {
                             code_action_kind: lsp::CodeActionKindLiteralSupport {
@@ -281,14 +315,7 @@ impl Client {
             locale: None, // TODO
         };
 
-        let response = self.request::<lsp::request::Initialize>(params).await?;
+        self.request::<lsp::request::Initialize>(params).await
-        self.capabilities = Some(response.capabilities);
-
-        // next up, notify<initialized>
-        self.notify::<lsp::notification::Initialized>(lsp::InitializedParams {})
-            .await?;
-
-        Ok(())
     }
 
     pub async fn shutdown(&self) -> Result<()> {
@@ -314,6 +341,16 @@ impl Client {
         self.exit().await
     }
 
+    // -------------------------------------------------------------------------------------------
+    // Workspace
+    // -------------------------------------------------------------------------------------------
+
+    pub fn did_change_configuration(&self, settings: Value) -> impl Future<Output = Result<()>> {
+        self.notify::<lsp::notification::DidChangeConfiguration>(
+            lsp::DidChangeConfigurationParams { settings },
+        )
+    }
+
     // -------------------------------------------------------------------------------------------
     // Text document
     // -------------------------------------------------------------------------------------------
@@ -356,7 +393,6 @@ impl Client {
         //
         // Calculation is therefore a bunch trickier.
 
-        // TODO: stolen from syntax.rs, share
         use helix_core::RopeSlice;
         fn traverse(pos: lsp::Position, text: RopeSlice) -> lsp::Position {
             let lsp::Position {
@@ -366,7 +402,12 @@ impl Client {
 
             let mut chars = text.chars().peekable();
             while let Some(ch) = chars.next() {
-                if char_is_line_ending(ch) && !(ch == '\r' && chars.peek() == Some(&'\n')) {
+                // LSP only considers \n, \r or \r\n as line endings
+                if ch == '\n' || ch == '\r' {
+                    // consume a \r\n
+                    if ch == '\r' && chars.peek() == Some(&'\n') {
+                        chars.next();
+                    }
                     line += 1;
                     character = 0;
                 } else {
@@ -421,7 +462,7 @@ impl Client {
 
                     changes.push(lsp::TextDocumentContentChangeEvent {
                         range: Some(lsp::Range::new(start, end)),
-                        text: s.into(),
+                        text: s.to_string(),
                         range_length: None,
                     });
                 }
@@ -441,7 +482,7 @@ impl Client {
     ) -> Option<impl Future<Output = Result<()>>> {
         // figure out what kind of sync the server supports
 
-        let capabilities = self.capabilities.as_ref().unwrap();
+        let capabilities = self.capabilities.get().unwrap();
 
         let sync_capabilities = match capabilities.text_document_sync {
             Some(lsp::TextDocumentSyncCapability::Kind(kind))
@@ -454,18 +495,19 @@ impl Client {
         };
 
         let changes = match sync_capabilities {
-            lsp::TextDocumentSyncKind::Full => {
+            lsp::TextDocumentSyncKind::FULL => {
                 vec![lsp::TextDocumentContentChangeEvent {
                     // range = None -> whole document
                     range: None,        //Some(Range)
                     range_length: None, // u64 apparently deprecated
-                    text: "".to_string(),
+                    text: new_text.to_string(),
                 }]
             }
-            lsp::TextDocumentSyncKind::Incremental => {
+            lsp::TextDocumentSyncKind::INCREMENTAL => {
                 Self::changeset_to_changes(old_text, new_text, changes, self.offset_encoding)
             }
-            lsp::TextDocumentSyncKind::None => return None,
+            lsp::TextDocumentSyncKind::NONE => return None,
+            kind => unimplemented!("{:?}", kind),
         };
 
         Some(self.notify::<lsp::notification::DidChangeTextDocument>(
@@ -487,12 +529,12 @@ impl Client {
 
     // will_save / will_save_wait_until
 
-    pub async fn text_document_did_save(
+    pub fn text_document_did_save(
         &self,
         text_document: lsp::TextDocumentIdentifier,
         text: &Rope,
-    ) -> Result<()> {
+    ) -> Option<impl Future<Output = Result<()>>> {
-        let capabilities = self.capabilities.as_ref().unwrap();
+        let capabilities = self.capabilities.get().unwrap();
 
         let include_text = match &capabilities.text_document_sync {
             Some(lsp::TextDocumentSyncCapability::Options(lsp::TextDocumentSyncOptions {
@@ -504,17 +546,18 @@ impl Client {
                     include_text,
                 }) => include_text.unwrap_or(false),
                 // Supported(false)
-                _ => return Ok(()),
+                _ => return None,
             },
             // unsupported
-            _ => return Ok(()),
+            _ => return None,
         };
 
-        self.notify::<lsp::notification::DidSaveTextDocument>(lsp::DidSaveTextDocumentParams {
+        Some(self.notify::<lsp::notification::DidSaveTextDocument>(
+            lsp::DidSaveTextDocumentParams {
                 text_document,
                 text: include_text.then(|| text.into()),
-        })
+            },
-        .await
+        ))
     }
 
     pub fn completion(
@@ -541,6 +584,14 @@ impl Client {
         self.call::<lsp::request::Completion>(params)
     }
 
+    pub async fn resolve_completion_item(
+        &self,
+        completion_item: lsp::CompletionItem,
+    ) -> Result<lsp::CompletionItem> {
+        self.request::<lsp::request::ResolveCompletionItem>(completion_item)
+            .await
+    }
+
     pub fn text_document_signature_help(
         &self,
         text_document: lsp::TextDocumentIdentifier,
@@ -580,19 +631,19 @@ impl Client {
 
     // formatting
 
-    pub async fn text_document_formatting(
+    pub fn text_document_formatting(
         &self,
         text_document: lsp::TextDocumentIdentifier,
         options: lsp::FormattingOptions,
         work_done_token: Option<lsp::ProgressToken>,
-    ) -> anyhow::Result<Vec<lsp::TextEdit>> {
+    ) -> Option<impl Future<Output = Result<Vec<lsp::TextEdit>>>> {
-        let capabilities = self.capabilities.as_ref().unwrap();
+        let capabilities = self.capabilities.get().unwrap();
 
         // check if we're able to format
         match capabilities.document_formatting_provider {
             Some(lsp::OneOf::Left(true)) | Some(lsp::OneOf::Right(_)) => (),
             // None | Some(false)
-            _ => return Ok(Vec::new()),
+            _ => return None,
         };
         // TODO: return err::unavailable so we can fall back to tree sitter formatting
 
@@ -602,9 +653,13 @@ impl Client {
             work_done_progress_params: lsp::WorkDoneProgressParams { work_done_token },
         };
 
-        let response = self.request::<lsp::request::Formatting>(params).await?;
+        let request = self.call::<lsp::request::Formatting>(params);
 
+        Some(async move {
+            let json = request.await?;
+            let response: Option<Vec<lsp::TextEdit>> = serde_json::from_value(json)?;
             Ok(response.unwrap_or_default())
+        })
     }
 
     pub async fn text_document_range_formatting(
@@ -614,7 +669,7 @@ impl Client {
         options: lsp::FormattingOptions,
         work_done_token: Option<lsp::ProgressToken>,
     ) -> anyhow::Result<Vec<lsp::TextEdit>> {
-        let capabilities = self.capabilities.as_ref().unwrap();
+        let capabilities = self.capabilities.get().unwrap();
 
         // check if we're able to format
         match capabilities.document_range_formatting_provider {
@@ -760,4 +815,37 @@ impl Client {
 
         self.call::<lsp::request::CodeActionRequest>(params)
     }
+
+    pub async fn rename_symbol(
+        &self,
+        text_document: lsp::TextDocumentIdentifier,
+        position: lsp::Position,
+        new_name: String,
+    ) -> anyhow::Result<lsp::WorkspaceEdit> {
+        let params = lsp::RenameParams {
+            text_document_position: lsp::TextDocumentPositionParams {
+                text_document,
+                position,
+            },
+            new_name,
+            work_done_progress_params: lsp::WorkDoneProgressParams {
+                work_done_token: None,
+            },
+        };
+
+        let response = self.request::<lsp::request::Rename>(params).await?;
+        Ok(response.unwrap_or_default())
+    }
+
+    pub fn command(&self, command: lsp::Command) -> impl Future<Output = Result<Value>> {
+        let params = lsp::ExecuteCommandParams {
+            command: command.command,
+            arguments: command.arguments.unwrap_or_default(),
+            work_done_progress_params: lsp::WorkDoneProgressParams {
+                work_done_token: None,
+            },
+        };
+
+        self.call::<lsp::request::ExecuteCommand>(params)
+    }
 }

helix-lsp/src/lib.rs

@@ -66,39 +66,26 @@ pub mod util {
         pos: lsp::Position,
         offset_encoding: OffsetEncoding,
     ) -> Option<usize> {
-        let max_line = doc.lines().count().saturating_sub(1);
         let pos_line = pos.line as usize;
-        let pos_line = if pos_line > max_line {
+        if pos_line > doc.len_lines() - 1 {
             return None;
-        } else {
+        }
-            pos_line
+
-        };
         match offset_encoding {
             OffsetEncoding::Utf8 => {
-                let max_char = doc
-                    .line_to_char(max_line)
-                    .checked_add(doc.line(max_line).len_chars())?;
                 let line = doc.line_to_char(pos_line);
                 let pos = line.checked_add(pos.character as usize)?;
-                if pos <= max_char {
+                if pos <= doc.len_chars() {
                     Some(pos)
                 } else {
                     None
                 }
             }
             OffsetEncoding::Utf16 => {
-                let max_char = doc
-                    .line_to_char(max_line)
-                    .checked_add(doc.line(max_line).len_chars())?;
-                let max_cu = doc.char_to_utf16_cu(max_char);
                 let line = doc.line_to_char(pos_line);
                 let line_start = doc.char_to_utf16_cu(line);
                 let pos = line_start.checked_add(pos.character as usize)?;
-                if pos <= max_cu {
+                doc.try_utf16_cu_to_char(pos).ok()
-                    Some(doc.utf16_cu_to_char(pos))
-                } else {
-                    None
-                }
             }
         }
     }
@@ -203,6 +190,8 @@ pub mod util {
 #[derive(Debug, PartialEq, Clone)]
 pub enum MethodCall {
     WorkDoneProgressCreate(lsp::WorkDoneProgressCreateParams),
+    ApplyWorkspaceEdit(lsp::ApplyWorkspaceEditParams),
+    WorkspaceConfiguration(lsp::ConfigurationParams),
 }
 
 impl MethodCall {
@@ -215,6 +204,18 @@ impl MethodCall {
                     .expect("Failed to parse WorkDoneCreate params");
                 Self::WorkDoneProgressCreate(params)
             }
+            lsp::request::ApplyWorkspaceEdit::METHOD => {
+                let params: lsp::ApplyWorkspaceEditParams = params
+                    .parse()
+                    .expect("Failed to parse ApplyWorkspaceEdit params");
+                Self::ApplyWorkspaceEdit(params)
+            }
+            lsp::request::WorkspaceConfiguration::METHOD => {
+                let params: lsp::ConfigurationParams = params
+                    .parse()
+                    .expect("Failed to parse WorkspaceConfiguration params");
+                Self::WorkspaceConfiguration(params)
+            }
             _ => {
                 log::warn!("unhandled lsp request: {}", method);
                 return None;
@@ -226,6 +227,8 @@ impl MethodCall {
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum Notification {
+    // we inject this notification to signal the LSP is ready
+    Initialized,
     PublishDiagnostics(lsp::PublishDiagnosticsParams),
     ShowMessage(lsp::ShowMessageParams),
     LogMessage(lsp::LogMessageParams),
@@ -237,6 +240,7 @@ impl Notification {
         use lsp::notification::Notification as _;
 
         let notification = match method {
+            lsp::notification::Initialized::METHOD => Self::Initialized,
             lsp::notification::PublishDiagnostics::METHOD => {
                 let params: lsp::PublishDiagnosticsParams = params
                     .parse()
@@ -294,7 +298,7 @@ impl Registry {
         }
     }
 
-    pub fn get_by_id(&mut self, id: usize) -> Option<&Client> {
+    pub fn get_by_id(&self, id: usize) -> Option<&Client> {
         self.inner
             .values()
             .find(|(client_id, _)| client_id == &id)
@@ -302,34 +306,57 @@ impl Registry {
     }
 
     pub fn get(&mut self, language_config: &LanguageConfiguration) -> Result<Arc<Client>> {
-        if let Some(config) = &language_config.language_server {
+        let config = match &language_config.language_server {
-            // avoid borrow issues
+            Some(config) => config,
-            let inner = &mut self.inner;
+            None => return Err(Error::LspNotDefined),
-            let s_incoming = &mut self.incoming;
+        };
 
-            match inner.entry(language_config.scope.clone()) {
+        match self.inner.entry(language_config.scope.clone()) {
             Entry::Occupied(entry) => Ok(entry.get().1.clone()),
             Entry::Vacant(entry) => {
                 // initialize a new client
                 let id = self.counter.fetch_add(1, Ordering::Relaxed);
-                    let (mut client, incoming) = Client::start(
+                let (client, incoming, initialize_notify) = Client::start(
                     &config.command,
                     &config.args,
-                        serde_json::from_str(language_config.config.as_deref().unwrap_or("")).ok(),
+                    language_config.config.clone(),
+                    language_config.roots.clone(),
                     id,
                 )?;
-                    // TODO: run this async without blocking
+                self.incoming.push(UnboundedReceiverStream::new(incoming));
-                    futures_executor::block_on(client.initialize())?;
-                    s_incoming.push(UnboundedReceiverStream::new(incoming));
                 let client = Arc::new(client);
 
+                // Initialize the client asynchronously
+                let _client = client.clone();
+                tokio::spawn(async move {
+                    use futures_util::TryFutureExt;
+                    let value = _client
+                        .capabilities
+                        .get_or_try_init(|| {
+                            _client
+                                .initialize()
+                                .map_ok(|response| response.capabilities)
+                        })
+                        .await;
+
+                    if let Err(e) = value {
+                        log::error!("failed to initialize language server: {}", e);
+                        return;
+                    }
+
+                    // next up, notify<initialized>
+                    _client
+                        .notify::<lsp::notification::Initialized>(lsp::InitializedParams {})
+                        .await
+                        .unwrap();
+
+                    initialize_notify.notify_one();
+                });
+
                 entry.insert((id, client.clone()));
                 Ok(client)
             }
         }
-        } else {
-            Err(Error::LspNotDefined)
-        }
     }
 
     pub fn iter_clients(&self) -> impl Iterator<Item = &Arc<Client>> {
@@ -415,32 +442,6 @@ impl LspProgressMap {
     }
 }
 
-// REGISTRY = HashMap<LanguageId, Lazy/OnceCell<Arc<RwLock<Client>>>
-// spawn one server per language type, need to spawn one per workspace if server doesn't support
-// workspaces
-//
-// could also be a client per root dir
-//
-// storing a copy of Option<Arc<RwLock<Client>>> on Document would make the LSP client easily
-// accessible during edit/save callbacks
-//
-// the event loop needs to process all incoming streams, maybe we can just have that be a separate
-// task that's continually running and store the state on the client, then use read lock to
-// retrieve data during render
-// -> PROBLEM: how do you trigger an update on the editor side when data updates?
-//
-// -> The data updates should pull all events until we run out so we don't frequently re-render
-//
-//
-// v2:
-//
-// there should be a registry of lsp clients, one per language type (or workspace).
-// the clients should lazy init on first access
-// the client.initialize() should be called async and we buffer any requests until that completes
-// there needs to be a way to process incoming lsp messages from all clients.
-//  -> notifications need to be dispatched to wherever
-//  -> requests need to generate a reply and travel back to the same lsp!
-
 #[cfg(test)]
 mod tests {
     use super::{lsp, util::*, OffsetEncoding};

helix-lsp/src/transport.rs

@@ -1,4 +1,4 @@
-use crate::Result;
+use crate::{Error, Result};
 use anyhow::Context;
 use jsonrpc_core as jsonrpc;
 use log::{error, info};
@@ -11,7 +11,7 @@ use tokio::{
     process::{ChildStderr, ChildStdin, ChildStdout},
     sync::{
         mpsc::{unbounded_channel, Sender, UnboundedReceiver, UnboundedSender},
-        Mutex,
+        Mutex, Notify,
     },
 };
 
@@ -51,9 +51,11 @@ impl Transport {
     ) -> (
         UnboundedReceiver<(usize, jsonrpc::Call)>,
         UnboundedSender<Payload>,
+        Arc<Notify>,
     ) {
         let (client_tx, rx) = unbounded_channel();
         let (tx, client_rx) = unbounded_channel();
+        let notify = Arc::new(Notify::new());
 
         let transport = Self {
             id,
@@ -62,11 +64,21 @@ impl Transport {
 
         let transport = Arc::new(transport);
 
-        tokio::spawn(Self::recv(transport.clone(), server_stdout, client_tx));
+        tokio::spawn(Self::recv(
+            transport.clone(),
+            server_stdout,
+            client_tx.clone(),
+        ));
         tokio::spawn(Self::err(transport.clone(), server_stderr));
-        tokio::spawn(Self::send(transport, server_stdin, client_rx));
+        tokio::spawn(Self::send(
+            transport,
+            server_stdin,
+            client_tx,
+            client_rx,
+            notify.clone(),
+        ));
 
-        (rx, tx)
+        (rx, tx, notify)
     }
 
     async fn recv_server_message(
@@ -76,26 +88,32 @@ impl Transport {
         let mut content_length = None;
         loop {
             buffer.truncate(0);
-            reader.read_line(buffer).await?;
+            if reader.read_line(buffer).await? == 0 {
-            let header = buffer.trim();
+                return Err(Error::StreamClosed);
+            };
 
-            if header.is_empty() {
+            // debug!("<- header {:?}", buffer);
+
+            if buffer == "\r\n" {
+                // look for an empty CRLF line
                 break;
             }
 
-            let mut parts = header.split(": ");
+            let header = buffer.trim();
 
-            match (parts.next(), parts.next(), parts.next()) {
+            let parts = header.split_once(": ");
-                (Some("Content-Length"), Some(value), None) => {
+
+            match parts {
+                Some(("Content-Length", value)) => {
                     content_length = Some(value.parse().context("invalid content length")?);
                 }
-                (Some(_), Some(_), None) => {}
+                Some((_, _)) => {}
-                _ => {
+                None => {
-                    return Err(std::io::Error::new(
+                    // Workaround: Some non-conformant language servers will output logging and other garbage
-                        std::io::ErrorKind::Other,
+                    // into the same stream as JSON-RPC messages. This can also happen from shell scripts that spawn
-                        "Failed to parse header",
+                    // the server. Skip such lines and log a warning.
-                    )
+
-                    .into());
+                    // warn!("Failed to parse header: {:?}", header);
                 }
             }
         }
@@ -120,8 +138,10 @@ impl Transport {
         buffer: &mut String,
     ) -> Result<()> {
         buffer.truncate(0);
-        err.read_line(buffer).await?;
+        if err.read_line(buffer).await? == 0 {
-        error!("err <- {}", buffer);
+            return Err(Error::StreamClosed);
+        };
+        error!("err <- {:?}", buffer);
 
         Ok(())
     }
@@ -222,10 +242,13 @@ impl Transport {
         loop {
             match Self::recv_server_message(&mut server_stdout, &mut recv_buffer).await {
                 Ok(msg) => {
-                    transport
+                    match transport.process_server_message(&client_tx, msg).await {
-                        .process_server_message(&client_tx, msg)
+                        Ok(_) => {}
-                        .await
+                        Err(err) => {
-                        .unwrap();
+                            error!("err: <- {:?}", err);
+                            break;
+                        }
+                    };
                 }
                 Err(err) => {
                     error!("err: <- {:?}", err);
@@ -251,13 +274,92 @@ impl Transport {
     async fn send(
         transport: Arc<Self>,
         mut server_stdin: BufWriter<ChildStdin>,
+        client_tx: UnboundedSender<(usize, jsonrpc::Call)>,
         mut client_rx: UnboundedReceiver<Payload>,
+        initialize_notify: Arc<Notify>,
     ) {
-        while let Some(msg) = client_rx.recv().await {
+        let mut pending_messages: Vec<Payload> = Vec::new();
-            transport
+        let mut is_pending = true;
-                .send_payload_to_server(&mut server_stdin, msg)
+
-                .await
+        // Determine if a message is allowed to be sent early
-                .unwrap()
+        fn is_initialize(payload: &Payload) -> bool {
+            use lsp_types::{
+                notification::{Initialized, Notification},
+                request::{Initialize, Request},
+            };
+            match payload {
+                Payload::Request {
+                    value: jsonrpc::MethodCall { method, .. },
+                    ..
+                } if method == Initialize::METHOD => true,
+                Payload::Notification(jsonrpc::Notification { method, .. })
+                    if method == Initialized::METHOD =>
+                {
+                    true
+                }
+                _ => false,
+            }
+        }
+
+        // TODO: events that use capabilities need to do the right thing
+
+        loop {
+            tokio::select! {
+                biased;
+                _ = initialize_notify.notified() => { // TODO: notified is technically not cancellation safe
+                    // server successfully initialized
+                    is_pending = false;
+
+                    use lsp_types::notification::Notification;
+                    // Hack: inject an initialized notification so we trigger code that needs to happen after init
+                    let notification = ServerMessage::Call(jsonrpc::Call::Notification(jsonrpc::Notification {
+                        jsonrpc: None,
+
+                        method: lsp_types::notification::Initialized::METHOD.to_string(),
+                        params: jsonrpc::Params::None,
+                    }));
+                    match transport.process_server_message(&client_tx, notification).await {
+                        Ok(_) => {}
+                        Err(err) => {
+                            error!("err: <- {:?}", err);
+                        }
+                    }
+
+                    // drain the pending queue and send payloads to server
+                    for msg in pending_messages.drain(..) {
+                        log::info!("Draining pending message {:?}", msg);
+                        match transport.send_payload_to_server(&mut server_stdin, msg).await {
+                            Ok(_) => {}
+                            Err(err) => {
+                                error!("err: <- {:?}", err);
+                            }
+                        }
+                    }
+                }
+                msg = client_rx.recv() => {
+                    if let Some(msg) = msg {
+                        if is_pending && !is_initialize(&msg) {
+                            // ignore notifications
+                            if let Payload::Notification(_) = msg {
+                                continue;
+                            }
+
+                            log::info!("Language server not initialized, delaying request");
+                            pending_messages.push(msg);
+                        } else {
+                            match transport.send_payload_to_server(&mut server_stdin, msg).await {
+                                Ok(_) => {}
+                                Err(err) => {
+                                    error!("err: <- {:?}", err);
+                                }
+                            }
+                        }
+                    } else {
+                        // channel closed
+                        break;
+                    }
+                }
+            }
         }
     }
 }

helix-syntax/build.rs

@@ -1,201 +0,0 @@
-use anyhow::{anyhow, Context, Result};
-use std::fs;
-use std::time::SystemTime;
-use std::{
-    path::{Path, PathBuf},
-    process::Command,
-};
-
-use std::sync::mpsc::channel;
-
-fn collect_tree_sitter_dirs(ignore: &[String]) -> Result<Vec<String>> {
-    let mut dirs = Vec::new();
-    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("languages");
-
-    for entry in fs::read_dir(path)? {
-        let entry = entry?;
-        let path = entry.path();
-
-        if !entry.file_type()?.is_dir() {
-            continue;
-        }
-
-        let dir = path.file_name().unwrap().to_str().unwrap().to_string();
-
-        // filter ignores
-        if ignore.contains(&dir) {
-            continue;
-        }
-        dirs.push(dir)
-    }
-
-    Ok(dirs)
-}
-
-#[cfg(unix)]
-const DYLIB_EXTENSION: &str = "so";
-
-#[cfg(windows)]
-const DYLIB_EXTENSION: &str = "dll";
-
-fn build_library(src_path: &Path, language: &str) -> Result<()> {
-    let header_path = src_path;
-    // let grammar_path = src_path.join("grammar.json");
-    let parser_path = src_path.join("parser.c");
-    let mut scanner_path = src_path.join("scanner.c");
-
-    let scanner_path = if scanner_path.exists() {
-        Some(scanner_path)
-    } else {
-        scanner_path.set_extension("cc");
-        if scanner_path.exists() {
-            Some(scanner_path)
-        } else {
-            None
-        }
-    };
-    let parser_lib_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../runtime/grammars");
-    let mut library_path = parser_lib_path.join(language);
-    library_path.set_extension(DYLIB_EXTENSION);
-
-    let recompile = needs_recompile(&library_path, &parser_path, &scanner_path)
-        .with_context(|| "Failed to compare source and binary timestamps")?;
-
-    if !recompile {
-        return Ok(());
-    }
-    let mut config = cc::Build::new();
-    config.cpp(true).opt_level(2).cargo_metadata(false);
-    let compiler = config.get_compiler();
-    let mut command = Command::new(compiler.path());
-    command.current_dir(src_path);
-    for (key, value) in compiler.env() {
-        command.env(key, value);
-    }
-
-    if cfg!(windows) {
-        command
-            .args(&["/nologo", "/LD", "/I"])
-            .arg(header_path)
-            .arg("/Od");
-        if let Some(scanner_path) = scanner_path.as_ref() {
-            command.arg(scanner_path);
-        }
-
-        command
-            .arg(parser_path)
-            .arg("/link")
-            .arg(format!("/out:{}", library_path.to_str().unwrap()));
-    } else {
-        command
-            .arg("-shared")
-            .arg("-fPIC")
-            .arg("-fno-exceptions")
-            .arg("-g")
-            .arg("-I")
-            .arg(header_path)
-            .arg("-o")
-            .arg(&library_path)
-            .arg("-O2");
-        if let Some(scanner_path) = scanner_path.as_ref() {
-            if scanner_path.extension() == Some("c".as_ref()) {
-                command.arg("-xc").arg("-std=c99").arg(scanner_path);
-            } else {
-                command.arg(scanner_path);
-            }
-        }
-        command.arg("-xc").arg(parser_path);
-    }
-
-    let output = command
-        .output()
-        .with_context(|| "Failed to execute C compiler")?;
-    if !output.status.success() {
-        return Err(anyhow!(
-            "Parser compilation failed.\nStdout: {}\nStderr: {}",
-            String::from_utf8_lossy(&output.stdout),
-            String::from_utf8_lossy(&output.stderr)
-        ));
-    }
-
-    Ok(())
-}
-fn needs_recompile(
-    lib_path: &Path,
-    parser_c_path: &Path,
-    scanner_path: &Option<PathBuf>,
-) -> Result<bool> {
-    if !lib_path.exists() {
-        return Ok(true);
-    }
-    let lib_mtime = mtime(lib_path)?;
-    if mtime(parser_c_path)? > lib_mtime {
-        return Ok(true);
-    }
-    if let Some(scanner_path) = scanner_path {
-        if mtime(scanner_path)? > lib_mtime {
-            return Ok(true);
-        }
-    }
-    Ok(false)
-}
-
-fn mtime(path: &Path) -> Result<SystemTime> {
-    Ok(fs::metadata(path)?.modified()?)
-}
-
-fn build_dir(dir: &str, language: &str) {
-    println!("Build language {}", language);
-    if PathBuf::from("languages")
-        .join(dir)
-        .read_dir()
-        .unwrap()
-        .next()
-        .is_none()
-    {
-        eprintln!(
-            "The directory {} is empty, did you use 'git clone --recursive'?",
-            dir
-        );
-        eprintln!("You can fix in using 'git submodule init && git submodule update --recursive'.");
-        std::process::exit(1);
-    }
-
-    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
-        .join("languages")
-        .join(dir)
-        .join("src");
-
-    build_library(&path, language).unwrap();
-}
-
-fn main() {
-    let ignore = vec![
-        "tree-sitter-typescript".to_string(),
-        "tree-sitter-haskell".to_string(), // aarch64 failures: https://github.com/tree-sitter/tree-sitter-haskell/issues/34
-    ];
-    let dirs = collect_tree_sitter_dirs(&ignore).unwrap();
-
-    let mut n_jobs = 0;
-    let pool = threadpool::Builder::new().build(); // by going through the builder, it'll use num_cpus
-    let (tx, rx) = channel();
-
-    for dir in dirs {
-        let tx = tx.clone();
-        n_jobs += 1;
-
-        pool.execute(move || {
-            let language = &dir.strip_prefix("tree-sitter-").unwrap();
-            build_dir(&dir, language);
-
-            // report progress
-            tx.send(1).unwrap();
-        });
-    }
-    pool.join();
-    // drop(tx);
-    assert_eq!(rx.try_iter().sum::<usize>(), n_jobs);
-
-    build_dir("tree-sitter-typescript/tsx", "tsx");
-    build_dir("tree-sitter-typescript/typescript", "typescript");
-}