Release 0.6

lines().count() is slow compared to len_lines()

.cargo/config

@@ -0,0 +1,2 @@
+[alias]
+xtask = "run --package xtask --"

.envrc

@@ -1,2 +1,5 @@
 watch_file shell.nix
-use flake
+watch_file flake.lock
+
+# try to use flakes, if it fails use normal nix (ie. shell.nix)
+use flake || use nix

.github/ISSUE_TEMPLATE/blank_issue.md

@@ -0,0 +1,4 @@
+---
+name: Blank Issue
+about: Create a blank issue.
+---

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,28 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: C-bug
+assignees: ''
+
+---
+
+<!-- Your issue may already be reported!
+Please search on the issue tracker before creating one. -->
+
+### Reproduction steps
+
+<!-- Ideally provide a key sequence and/or asciinema.org recording. --> 
+
+### Environment
+
+- Platform: <!--  macOS / Windows / Linux -->
+- Helix version: <!--  'hx -V' if using a release, 'git describe' if building from master -->
+
+<details><summary>~/.cache/helix/helix.log</summary>
+
+```
+please provide a copy of `~/.cache/helix/helix.log` here if possible, you may need to redact some of the lines
+```
+
+</details>

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,13 @@
+---
+name: Feature request
+about: Suggest a new feature or improvement
+title: ''
+labels: C-enhancement
+assignees: ''
+---
+
+<!-- Your feature may already be reported!
+Please search on the issue tracker before creating one. -->
+
+#### Describe your feature request
+

.github/workflows/build.yml

@@ -25,22 +25,22 @@ jobs:
           override: true
 
       - name: Cache cargo registry
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v2.1.7
         with:
           path: ~/.cargo/registry
-          key: ${{ runner.os }}-v1-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ runner.os }}-v2-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
 
       - name: Cache cargo index
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v2.1.7
         with:
           path: ~/.cargo/git
-          key: ${{ runner.os }}-v1-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ runner.os }}-v2-cargo-index-${{ hashFiles('**/Cargo.lock') }}
 
       - name: Cache cargo target dir
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v2.1.7
         with:
           path: target
-          key: ${{ runner.os }}-v1-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ runner.os }}-v2-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
 
       - name: Run cargo check
         uses: actions-rs/cargo@v1
@@ -64,22 +64,22 @@ jobs:
           override: true
 
       - name: Cache cargo registry
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v2.1.7
         with:
           path: ~/.cargo/registry
-          key: ${{ runner.os }}-v1-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ runner.os }}-v2-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
 
       - name: Cache cargo index
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v2.1.7
         with:
           path: ~/.cargo/git
-          key: ${{ runner.os }}-v1-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ runner.os }}-v2-cargo-index-${{ hashFiles('**/Cargo.lock') }}
 
       - name: Cache cargo target dir
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v2.1.7
         with:
           path: target
-          key: ${{ runner.os }}-v1-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ runner.os }}-v2-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
 
       - name: Run cargo test
         uses: actions-rs/cargo@v1
@@ -109,22 +109,22 @@ jobs:
           components: rustfmt, clippy
 
       - name: Cache cargo registry
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v2.1.7
         with:
           path: ~/.cargo/registry
-          key: ${{ runner.os }}-v1-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ runner.os }}-v2-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
 
       - name: Cache cargo index
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v2.1.7
         with:
           path: ~/.cargo/git
-          key: ${{ runner.os }}-v1-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ runner.os }}-v2-cargo-index-${{ hashFiles('**/Cargo.lock') }}
 
       - name: Cache cargo target dir
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v2.1.7
         with:
           path: target
-          key: ${{ runner.os }}-v1-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ runner.os }}-v2-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
 
       - name: Run cargo fmt
         uses: actions-rs/cargo@v1
@@ -137,3 +137,51 @@ jobs:
         with:
           command: clippy
           args: -- -D warnings
+
+  docs:
+    name: Docs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - name: Install stable toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+
+      - name: Cache cargo registry
+        uses: actions/cache@v2.1.6
+        with:
+          path: ~/.cargo/registry
+          key: ${{ runner.os }}-v2-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Cache cargo index
+        uses: actions/cache@v2.1.6
+        with:
+          path: ~/.cargo/git
+          key: ${{ runner.os }}-v2-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Cache cargo target dir
+        uses: actions/cache@v2.1.6
+        with:
+          path: target
+          key: ${{ runner.os }}-v2-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Generate docs
+        uses: actions-rs/cargo@v1
+        with:
+          command: xtask
+          args: docgen
+
+      - name: Check uncommitted documentation changes
+        run: |
+          git diff
+          git diff-files --quiet \
+            || (echo "Run 'cargo xtask docgen', commit the changes and push again" \
+            && exit 1)
+

.github/workflows/release.yml

@@ -67,8 +67,9 @@ jobs:
       - name: Run cargo test
         uses: actions-rs/cargo@v1
         with:
+          use-cross: ${{ matrix.cross }}
           command: test
-          args: --locked
+          args: --release --locked --target ${{ matrix.target }}
 
       - name: Build release binary
         uses: actions-rs/cargo@v1
@@ -99,8 +100,9 @@ jobs:
           else
             cp "target/${{ matrix.target }}/release/hx" "dist/"
           fi
+          cp -r runtime dist
 
-      - uses: actions/upload-artifact@v2.2.3
+      - uses: actions/upload-artifact@v2.3.1
         with:
           name: bins-${{ matrix.build }}
           path: dist
@@ -148,7 +150,7 @@ jobs:
               pkgname=helix-$TAG-$platform
               mkdir tmp/$pkgname
               cp LICENSE README.md tmp/$pkgname
-              cp -r runtime tmp/$pkgname/
+              mv bins-$platform/runtime tmp/$pkgname/
               mv bins-$platform/hx$exe tmp/$pkgname
               chmod +x tmp/$pkgname/hx$exe
 

.gitignore

@@ -2,3 +2,5 @@ target
 .direnv
 helix-term/rustfmt.toml
 helix-syntax/languages/
+result
+runtime/grammars

.gitmodules

@@ -84,5 +84,105 @@
 	shallow = true
 [submodule "helix-syntax/languages/tree-sitter-elixir"]
 	path = helix-syntax/languages/tree-sitter-elixir
-	url = https://github.com/IceDragon200/tree-sitter-elixir
+	url = https://github.com/elixir-lang/tree-sitter-elixir
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-nix"]
+	path = helix-syntax/languages/tree-sitter-nix
+	url = https://github.com/cstrahan/tree-sitter-nix
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-latex"]
+	path = helix-syntax/languages/tree-sitter-latex
+	url = https://github.com/latex-lsp/tree-sitter-latex
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-ledger"]
+	path = helix-syntax/languages/tree-sitter-ledger
+	url = https://github.com/cbarrete/tree-sitter-ledger
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-protobuf"]
+	path = helix-syntax/languages/tree-sitter-protobuf
+	url = https://github.com/yusdacra/tree-sitter-protobuf.git
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-ocaml"]
+	path = helix-syntax/languages/tree-sitter-ocaml
+	url = https://github.com/tree-sitter/tree-sitter-ocaml
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-lua"]
+	path = helix-syntax/languages/tree-sitter-lua
+	url = https://github.com/nvim-treesitter/tree-sitter-lua
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-yaml"]
+	path = helix-syntax/languages/tree-sitter-yaml
+	url = https://github.com/ikatyang/tree-sitter-yaml
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-zig"]
+	path = helix-syntax/languages/tree-sitter-zig
+	url = https://github.com/maxxnino/tree-sitter-zig
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-svelte"]
+	path = helix-syntax/languages/tree-sitter-svelte
+	url = https://github.com/Himujjal/tree-sitter-svelte
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-vue"]
+	path = helix-syntax/languages/tree-sitter-vue
+	url = https://github.com/ikatyang/tree-sitter-vue
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-tsq"]
+	path = helix-syntax/languages/tree-sitter-tsq
+	url = https://github.com/tree-sitter/tree-sitter-tsq
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-cmake"]
+	path = helix-syntax/languages/tree-sitter-cmake
+	url = https://github.com/uyha/tree-sitter-cmake
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-glsl"]
+	path = helix-syntax/languages/tree-sitter-glsl
+	url = https://github.com/theHamsta/tree-sitter-glsl.git
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-perl"]
+	path = helix-syntax/languages/tree-sitter-perl
+	url = https://github.com/ganezdragon/tree-sitter-perl
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-comment"]
+	path = helix-syntax/languages/tree-sitter-comment
+	url = https://github.com/stsewd/tree-sitter-comment
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-wgsl"]
+	path = helix-syntax/languages/tree-sitter-wgsl
+	url = https://github.com/szebniok/tree-sitter-wgsl
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-llvm"]
+	path = helix-syntax/languages/tree-sitter-llvm
+	url = https://github.com/benwilliamgraham/tree-sitter-llvm
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-markdown"]
+	path = helix-syntax/languages/tree-sitter-markdown
+	url = https://github.com/MDeiml/tree-sitter-markdown
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-dart"]
+	path = helix-syntax/languages/tree-sitter-dart
+	url = https://github.com/UserNobody14/tree-sitter-dart.git
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-dockerfile"]
+	path = helix-syntax/languages/tree-sitter-dockerfile
+	url = https://github.com/camdencheek/tree-sitter-dockerfile.git
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-fish"]
+	path = helix-syntax/languages/tree-sitter-fish
+	url = https://github.com/ram02z/tree-sitter-fish
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-git-commit"]
+	path = helix-syntax/languages/tree-sitter-git-commit
+	url = https://github.com/the-mikedavis/tree-sitter-git-commit.git
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-git-diff"]
+	path = helix-syntax/languages/tree-sitter-git-diff
+	url = https://github.com/the-mikedavis/tree-sitter-git-diff.git
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-tablegen"]
+	path = helix-syntax/languages/tree-sitter-tablegen
+	url = https://github.com/Flakebi/tree-sitter-tablegen
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-git-rebase"]
+	path = helix-syntax/languages/tree-sitter-git-rebase
+	url = https://github.com/the-mikedavis/tree-sitter-git-rebase.git
 	shallow = true

CHANGELOG.md

@@ -0,0 +1,337 @@
+
+# 0.6.0 (2022-01-04)
+
+Happy new year and a big shout out to all the contributors! We had 55 contributors in this release.
+
+Helix has popped up in DPorts and Fedora Linux via COPR ([#1270](https://github.com/helix-editor/helix/pull/1270))
+
+As usual the following is a brief summary, refer to the git history for a full log:
+
+Breaking changes:
+
+- fix: Normalize backtab into shift-tab 
+
+Features:
+
+- Macros ([#1234](https://github.com/helix-editor/helix/pull/1234)) 
+- Add reverse search functionality ([#958](https://github.com/helix-editor/helix/pull/958))
+- Allow keys to be mapped to sequences of commands ([#589](https://github.com/helix-editor/helix/pull/589)) 
+- Make it possible to keybind TypableCommands ([#1169](https://github.com/helix-editor/helix/pull/1169))
+- Detect workspace root using language markers ([#1370](https://github.com/helix-editor/helix/pull/1370))
+- Add WORD textobject ([#991](https://github.com/helix-editor/helix/pull/991))
+- Add LSP rename_symbol (space-r) ([#1011](https://github.com/helix-editor/helix/pull/1011))
+- Added workspace_symbol_picker ([#1041](https://github.com/helix-editor/helix/pull/1041)) 
+- Detect filetype from shebang line ([#1001](https://github.com/helix-editor/helix/pull/1001))
+- Allow piping from stdin into a buffer on startup ([#996](https://github.com/helix-editor/helix/pull/996)) 
+- Add auto pairs for same-char pairs ([#1219](https://github.com/helix-editor/helix/pull/1219)) 
+- Update settings at runtime ([#798](https://github.com/helix-editor/helix/pull/798)) 
+- Enable thin LTO (cccc194)
+
+Commands:
+- :wonly -- window only ([#1057](https://github.com/helix-editor/helix/pull/1057)) 
+- buffer-close (:bc, :bclose) ([#1035](https://github.com/helix-editor/helix/pull/1035)) 
+- Add :<line> and :goto <line> commands ([#1128](https://github.com/helix-editor/helix/pull/1128))
+- :sort command ([#1288](https://github.com/helix-editor/helix/pull/1288)) 
+- Add m textobject for pair under cursor ([#961](https://github.com/helix-editor/helix/pull/961)) 
+- Implement "Goto next buffer / Goto previous buffer" commands ([#950](https://github.com/helix-editor/helix/pull/950))
+- Implement "Goto last modification" command ([#1067](https://github.com/helix-editor/helix/pull/1067)) 
+- Add trim_selections command ([#1092](https://github.com/helix-editor/helix/pull/1092)) 
+- Add movement shortcut for history ([#1088](https://github.com/helix-editor/helix/pull/1088))
+- Add command to inc/dec number under cursor ([#1027](https://github.com/helix-editor/helix/pull/1027))
+  - Add support for dates for increment/decrement
+- Align selections (&) ([#1101](https://github.com/helix-editor/helix/pull/1101))
+- Implement no-yank delete/change ([#1099](https://github.com/helix-editor/helix/pull/1099)) 
+- Implement black hole register ([#1165](https://github.com/helix-editor/helix/pull/1165)) 
+- gf as goto_file (gf) ([#1102](https://github.com/helix-editor/helix/pull/1102))
+- Add last modified file (gm) ([#1093](https://github.com/helix-editor/helix/pull/1093))
+- ensure_selections_forward ([#1393](https://github.com/helix-editor/helix/pull/1393))
+- Readline style insert mode ([#1039](https://github.com/helix-editor/helix/pull/1039))
+
+Usability improvements and fixes:
+
+- Detect filetype on :write ([#1141](https://github.com/helix-editor/helix/pull/1141))
+- Add single and double quotes to matching pairs ([#995](https://github.com/helix-editor/helix/pull/995)) 
+- Launch with defaults upon invalid config/theme (rather than panicking) ([#982](https://github.com/helix-editor/helix/pull/982))
+- If switching away from an empty scratch buffer, remove it ([#935](https://github.com/helix-editor/helix/pull/935)) 
+- Truncate the starts of file paths instead of the ends in picker ([#951](https://github.com/helix-editor/helix/pull/951))
+- Truncate the start of file paths in the StatusLine ([#1351](https://github.com/helix-editor/helix/pull/1351)) 
+- Prevent picker from previewing binaries or large file ([#939](https://github.com/helix-editor/helix/pull/939))
+- Inform when reaching undo/redo bounds ([#981](https://github.com/helix-editor/helix/pull/981))
+- search_impl will only align cursor center when it isn't in view ([#959](https://github.com/helix-editor/helix/pull/959)) 
+- Add <C-h>, <C-u>, <C-d>, Delete in prompt mode ([#1034](https://github.com/helix-editor/helix/pull/1034)) 
+- Restore screen position when aborting search ([#1047](https://github.com/helix-editor/helix/pull/1047)) 
+- Buffer picker: show is_modifier flag ([#1020](https://github.com/helix-editor/helix/pull/1020)) 
+- Add commit hash to version info, if present ([#957](https://github.com/helix-editor/helix/pull/957))
+- Implement indent-aware delete ([#1120](https://github.com/helix-editor/helix/pull/1120))
+- Jump to end char of surrounding pair from any cursor pos ([#1121](https://github.com/helix-editor/helix/pull/1121))
+- File picker configuration ([#988](https://github.com/helix-editor/helix/pull/988))
+- Fix surround cursor position calculation ([#1183](https://github.com/helix-editor/helix/pull/1183))
+- Accept count for goto_window ([#1033](https://github.com/helix-editor/helix/pull/1033))
+- Make kill_to_line_end behave like emacs ([#1235](https://github.com/helix-editor/helix/pull/1235)) 
+- Only use a single documentation popup ([#1241](https://github.com/helix-editor/helix/pull/1241)) 
+- ui: popup: Don't allow scrolling past the end of content (3307f44c)
+- Open files with spaces in filename, allow opening multiple files ([#1231](https://github.com/helix-editor/helix/pull/1231)) 
+- Allow paste commands to take a count ([#1261](https://github.com/helix-editor/helix/pull/1261))
+- Auto pairs selection ([#1254](https://github.com/helix-editor/helix/pull/1254)) 
+- Use a fuzzy matcher for commands ([#1386](https://github.com/helix-editor/helix/pull/1386)) 
+- Add c-s to pick word under doc cursor to prompt line & search completion ([#831](https://github.com/helix-editor/helix/pull/831))
+- Fix :earlier/:later missing changeset update ([#1069](https://github.com/helix-editor/helix/pull/1069))
+- Support extend for multiple goto ([#909](https://github.com/helix-editor/helix/pull/909))
+- Add arrow-key bindings for window switching ([#933](https://github.com/helix-editor/helix/pull/933))
+- Implement key ordering for info box ([#952](https://github.com/helix-editor/helix/pull/952))
+
+LSP:
+- Implement MarkedString rendering (e128a8702)
+- Don't panic if init fails (d31bef7)
+- Configurable diagnostic severity ([#1325](https://github.com/helix-editor/helix/pull/1325))
+- Resolve completion item ([#1315](https://github.com/helix-editor/helix/pull/1315))
+- Code action command support ([#1304](https://github.com/helix-editor/helix/pull/1304))
+
+Grammars:
+
+- Adds mint language server (#974) 
+- Perl (#978) (#1280)
+- GLSL (#993) 
+- Racket (#1143) 
+- WGSL (#1166) 
+- LLVM (#1167) (#1388) (#1409) (#1398)
+- Markdown (49e06787)
+- Scala (#1278)
+- Dart (#1250)
+- Fish (#1308) 
+- Dockerfile (#1303)
+- Git (commit, rebase, diff) (#1338) (#1402) (#1373)
+- tree-sitter-comment (#1300)
+- Highlight comments in c, cpp, cmake and llvm (#1309)
+- Improve yaml syntax highlighting highlighting (#1294) 
+- Improve rust syntax highlighting (#1295)
+- Add textobjects and indents to cmake (#1307) 
+- Add textobjects and indents to c and cpp (#1293)
+
+New themes:
+
+- Solarized dark (#999) 
+- Solarized light (#1010) 
+- Spacebones light (#1131)
+- Monokai Pro (#1206) 
+- Base16 Light and Terminal (#1078)
+  - and a default 16 color theme, truecolor detection 
+- Dracula (#1258)
+
+# 0.6.0 (2021-10-28)
+
+A big shout out to all the contributors! We had 46 contributors in this release.
+
+Helix has popped up in [Scoop, FreeBSD Ports and Gentu GURU](https://repology.org/project/helix/versions)!
+
+The following is a quick rundown of the larger changes, there were many more
+(check the git history for more details).
+
+Breaking changes:
+
+- A couple of keymaps moved to resolve a few conflicting keybinds.
+  - Documentation popups were moved from `K` to `space+k`
+  - `K` is now `keep_selections` which filters selections to only keeps ones matching the regex
+  - `keep_primary_selection` moved from `space+space` to `,`
+  - `Alt-,` is now `remove_primary_selection` which keeps all selections except the primary one
+  - Opening files in a split moved from `C-h` to `C-s`
+- Some configuration options moved from a `[terminal]` section to `[editor]`. [Consult the documentation for more information.](https://docs.helix-editor.com/configuration.html)
+
+Features:
+
+- LSP compatibility greatly improved for some implementations (Julia, Python, Typescript)
+- Autocompletion! Completion now triggers automatically after a set idle timeout
+- Completion documentation is now displayed next to the popup ([#691](https://github.com/helix-editor/helix/pull/691))
+- Treesitter textobjects (select a function via `mf`, class via `mc`) ([#728](https://github.com/helix-editor/helix/pull/728)) 
+- Global search across entire workspace `space+/` ([#651](https://github.com/helix-editor/helix/pull/651)) 
+- Relative line number support ([#485](https://github.com/helix-editor/helix/pull/485))
+- Prompts now store a history (72cf86e)
+- `:vsplit` and `:hsplit` commands ([#639](https://github.com/helix-editor/helix/pull/639))
+- `C-w h/j/k/l` can now be used to navigate between splits ([#860](https://github.com/helix-editor/helix/pull/860))
+- `C-j` and `C-k` are now alternative keybindings to `C-n` and `C-p` in the UI ([#876](https://github.com/helix-editor/helix/pull/876))
+- Shell commands (shell-pipe, pipe-to, shell-insert-output, shell-append-output, keep-pipe) ([#547](https://github.com/helix-editor/helix/pull/547))
+- Searching now defaults to smart case search (case insensitive unless uppercase is used) ([#761](https://github.com/helix-editor/helix/pull/761))
+- The preview pane was improved to highlight and center line ranges
+- The user `languages.toml` is now merged into defaults, no longer need to copy the entire file (dc57f8dc)
+- Show hidden files in completions ([#648](https://github.com/helix-editor/helix/pull/648))
+- Grammar injections are now properly handled (dd0b15e)
+- `v` in select mode now switches back to normal mode ([#660](https://github.com/helix-editor/helix/pull/660))
+- View mode can now be triggered as a "sticky" mode ([#719](https://github.com/helix-editor/helix/pull/719))
+- `f`/`t` and object selection motions can now be repeated via `Alt-.` ([#891](https://github.com/helix-editor/helix/pull/891))
+- Statusline now displays total selection count and diagnostics counts for both errors and warnings ([#916](https://github.com/helix-editor/helix/pull/916))
+
+New grammars:
+
+- Ledger ([#572](https://github.com/helix-editor/helix/pull/572))
+- Protobuf ([#614](https://github.com/helix-editor/helix/pull/614))
+- Zig ([#631](https://github.com/helix-editor/helix/pull/631))
+- YAML ([#667](https://github.com/helix-editor/helix/pull/667))
+- Lua ([#665](https://github.com/helix-editor/helix/pull/665))
+- OCaml ([#666](https://github.com/helix-editor/helix/pull/666))
+- Svelte ([#733](https://github.com/helix-editor/helix/pull/733))
+- Vue ([#787](https://github.com/helix-editor/helix/pull/787))
+- Tree-sitter queries ([#845](https://github.com/helix-editor/helix/pull/845))
+- CMake ([#888](https://github.com/helix-editor/helix/pull/888))
+- Elixir (we switched over to the official grammar) (6c0786e)
+- Language server definitions for Nix and Elixir ([#725](https://github.com/helix-editor/helix/pull/725))
+- Python now uses `pylsp` instead of `pyls`
+- Python now supports indentation
+
+New themes:
+
+- Monokai ([#628](https://github.com/helix-editor/helix/pull/628))
+- Everforest Dark ([#760](https://github.com/helix-editor/helix/pull/760))
+- Nord ([#799](https://github.com/helix-editor/helix/pull/799))
+- Base16 Default Dark ([#833](https://github.com/helix-editor/helix/pull/833))
+- Rose Pine ([#897](https://github.com/helix-editor/helix/pull/897))
+
+Fixes:
+
+- Fix crash on empty rust file ([#592](https://github.com/helix-editor/helix/pull/592))
+- Exit select mode after toggle comment ([#598](https://github.com/helix-editor/helix/pull/598))
+- Pin popups with no positioning to the initial position (12ea3888)
+- xsel copy should not freeze the editor (6dd7dc4)
+- `*` now only sets the search register and doesn't jump to the next occurrence (3426285)
+- Goto line start/end commands extend when in select mode ([#739](https://github.com/helix-editor/helix/pull/739)) 
+- Fix documentation popups sometimes not getting fully highlighted (066367c)
+- Refactor apply_workspace_edit to remove assert (b02d872)
+- Wrap around the top of the picker menu when scrolling (c7d6e44)
+- Don't allow closing the last split if there's unsaved changes (3ff5b00)
+- Indentation used different default on hx vs hx new_file.txt (c913bad)
+
+# 0.4.1 (2021-08-14)
+
+A minor release that includes:
+- A fix for rendering glitches that would occur after editing with multiple selections.
+- CI fix for grammars not being cross-compiled for aarch64 
+
+# 0.4.0 (2021-08-13)
+
+A big shout out to all the contributors! We had 28 contributors in this release.
+
+Two months have passed, so this is another big release. A big thank you to all
+the contributors and package maintainers!
+
+Helix has popped up in [Arch, Manjaro, Nix, MacPorts and Parabola and Termux repositories](https://repology.org/project/helix/versions)!
+
+A [large scale refactor](https://github.com/helix-editor/helix/pull/376) landed that allows us to support zero width (empty)
+selections in the future as well as resolves many bugs and edge cases.
+
+- Multi-key remapping! Key binds now support much more complex usecases ([#454](https://github.com/helix-editor/helix/pull/454))
+- Pending keys are shown in the statusline ([#515](https://github.com/helix-editor/helix/pull/515))
+- Object selection / textobjects. `mi(` to select text inside parentheses ([#385](https://github.com/helix-editor/helix/pull/385))
+- Autoinfo: `whichkey`-like popups which show available sub-mode shortcuts ([#316](https://github.com/helix-editor/helix/pull/316))
+- Added WORD movements (W/B/E) ([#390](https://github.com/helix-editor/helix/pull/390))
+- Vertical selections (repeat selection above/below) ([#462](https://github.com/helix-editor/helix/pull/462))
+- Selection rotation via `(` and `)` ([66a90130](https://github.com/helix-editor/helix/commit/66a90130a5f99d769e9f6034025297f78ecaa3ec))
+- Selection contents rotation via `Alt-(` and `Alt-)` ([02cba2a](https://github.com/helix-editor/helix/commit/02cba2a7f403f48eccb18100fb751f7b42373dba))
+- Completion behavior improvements ([f917b5a4](https://github.com/helix-editor/helix/commit/f917b5a441ff3ae582358b6939ffbf889f4aa530), [627b899](https://github.com/helix-editor/helix/commit/627b89931576f7af86166ae8d5cbc55537877473))
+- Fixed a language server crash ([385a6b5a](https://github.com/helix-editor/helix/commit/385a6b5a1adddfc26e917982641530e1a7c7aa81))
+- Case change commands (`` ` ``, `~`, ``<a-`>``) ([#441](https://github.com/helix-editor/helix/pull/441))
+- File pickers (including goto) now provide a preview! ([#534](https://github.com/helix-editor/helix/pull/534))
+- Injection query support. Rust macro calls and embedded languages are now properly highlighted ([#430](https://github.com/helix-editor/helix/pull/430))
+- Formatting is now asynchronous, and the async job infrastructure has been improved ([#285](https://github.com/helix-editor/helix/pull/285))
+- Grammars are now compiled as separate shared libraries and loaded on-demand at runtime ([#432](https://github.com/helix-editor/helix/pull/432))
+- Code action support ([#478](https://github.com/helix-editor/helix/pull/478))
+- Mouse support ([#509](https://github.com/helix-editor/helix/pull/509), [#548](https://github.com/helix-editor/helix/pull/548))
+- Native Windows clipboard support ([#373](https://github.com/helix-editor/helix/pull/373))
+- Themes can now use color palettes ([#393](https://github.com/helix-editor/helix/pull/393))
+- `:reload` command ([#374](https://github.com/helix-editor/helix/pull/374))
+- Ctrl-z to suspend ([#464](https://github.com/helix-editor/helix/pull/464))
+- Language servers can now be configured with a custom JSON config ([#460](https://github.com/helix-editor/helix/pull/460))
+- Comment toggling now uses a language specific comment token ([#463](https://github.com/helix-editor/helix/pull/463))
+- Julia support ([#413](https://github.com/helix-editor/helix/pull/413))
+- Java support ([#448](https://github.com/helix-editor/helix/pull/448))
+- Prompts have an (in-memory) history ([63e54e30](https://github.com/helix-editor/helix/commit/63e54e30a74bb0d1d782877ddbbcf95f2817d061))
+
+# 0.3.0 (2021-06-27)
+
+A big shout out to all the contributors! We had 24 contributors in this release.
+
+Another big release. 
+
+Highlights:
+
+- Indentation is now automatically detected from file heuristics. ([#245](https://github.com/helix-editor/helix/pull/245))
+- Support for other line endings (CRLF). Significantly improved Windows support. ([#224](https://github.com/helix-editor/helix/pull/224))
+- Encodings other than UTF-8 are now supported! ([#228](https://github.com/helix-editor/helix/pull/228))
+- Key bindings can now be configured via a `config.toml` file ([#268](https://github.com/helix-editor/helix/pull/268))
+- Theme can now be configured and changed at runtime ([please feel free to contribute more themes!](https://github.com/helix-editor/helix/tree/master/runtime/themes)) ([#267](https://github.com/helix-editor/helix/pull/267))
+- System clipboard yank/paste is now supported! ([#310](https://github.com/helix-editor/helix/pull/310))
+- Surround commands were implemented ([#320](https://github.com/helix-editor/helix/pull/320))
+
+Features:
+
+- File picker can now be repeatedly filtered ([#232](https://github.com/helix-editor/helix/pull/232))
+- LSP progress is now received and rendered as a spinner ([#234](https://github.com/helix-editor/helix/pull/234))
+- Current line number can now be themed ([#260](https://github.com/helix-editor/helix/pull/260))
+- Arrow keys & home/end now work in insert mode ([#305](https://github.com/helix-editor/helix/pull/305))
+- Cursors and selections can now be themed ([#325](https://github.com/helix-editor/helix/pull/325))
+- Language servers are now gracefully shut down before `hx` exits ([#287](https://github.com/helix-editor/helix/pull/287))
+- `:show-directory`/`:change-directory` ([#335](https://github.com/helix-editor/helix/pull/335))
+- File picker is now sorted by access time (before filtering) ([#336](https://github.com/helix-editor/helix/pull/336))
+- Code is being migrated from helix-term to helix-view (prerequisite for
+  alternative frontends) ([#366](https://github.com/helix-editor/helix/pull/366))
+- `x` and `X` merged
+  ([f41688d9](https://github.com/helix-editor/helix/commit/f41688d960ef89c29c4a51c872b8406fb8f81a85))
+
+Fixes:
+
+- The IME popup is now correctly positioned ([#273](https://github.com/helix-editor/helix/pull/273))
+- A bunch of bugs regarding `o`/`O` behavior ([#281](https://github.com/helix-editor/helix/pull/281))
+- `~` expansion now works in file completion ([#284](https://github.com/helix-editor/helix/pull/284))
+- Several UI related overflow crashes ([#318](https://github.com/helix-editor/helix/pull/318))
+- Fix a test failure occuring only on `test --release` ([4f108ab1](https://github.com/helix-editor/helix/commit/4f108ab1b2197809506bd7305ad903a3525eabfa))
+- Prompts now support unicode input ([#295](https://github.com/helix-editor/helix/pull/295))
+- Completion documentation no longer overlaps the popup ([#322](https://github.com/helix-editor/helix/pull/322))
+- Fix a crash when trying to select `^` ([9c534614](https://github.com/helix-editor/helix/commit/9c53461429a3e72e3b1fb87d7ca490e168d7dee2))
+- Prompt completions are now paginated ([39dc09e6](https://github.com/helix-editor/helix/commit/39dc09e6c4172299bc79de4c1c52288d3f624bd7))
+- Goto did not work on Windows ([503ca112](https://github.com/helix-editor/helix/commit/503ca112ae57ebdf3ea323baf8940346204b46d2))
+
+# 0.2.1
+
+Includes a fix where wq/wqa could exit before file saving completed.
+
+# 0.2.0
+
+A big shout out to all the contributors! We had 18 contributors in this release.
+
+Enough has changed to bump the version. We're skipping 0.1.x because
+previously the CLI would always report version as 0.1.0, and we'd like
+to distinguish it in bug reports..
+
+- The `runtime/` directory is now properly detected on binary releases and
+  on cargo run. `~/.config/helix/runtime` can also be used.
+- Registers can now be selected via " (for example `"ay`)
+- Support for Nix files was added
+- Movement is now fully tested and matches kakoune implementation
+- A per-file LSP symbol picker was added to space+s
+- Selection can be replaced with yanked text via R
+
+- `1g` now correctly goes to line 1
+- `ctrl-i` now correctly jumps backwards in history
+- A small memory leak was fixed, where we tried to reuse tree-sitter
+  query cursors, but always allocated a new one
+- Auto-formatting is now only on for certain languages
+- The root directory is now provided in LSP initialization, fixing
+  certain language servers (typescript)
+- LSP failing to start no longer panics
+- Elixir language queries were fixed
+
+# 0.0.10
+
+Keymaps:
+- Add mappings to jump to diagnostics
+- Add gt/gm/gb mappings to jump to top/middle/bottom of screen
+- ^ and $ are now gh, gl
+
+- The runtime/ can now optionally be embedded in the binary
+- Haskell syntax added
+- Window mode (ctrl-w) added
+- Show matching bracket (vim's matchbrackets)
+- Themes now support style modifiers
+- First user contributed theme
+- Create a document if it doesn't exist yet on save
+- Detect language on a new file on save
+
+- Panic fixes, lots of them

Cargo.toml

@@ -6,6 +6,7 @@ members = [
   "helix-tui",
   "helix-syntax",
   "helix-lsp",
+  "xtask",
 ]
 
 # Build helix-syntax in release mode to make the code path faster in development.
@@ -14,3 +15,6 @@ members = [
 
 [profile.dev]
 split-debuginfo = "unpacked"
+
+[profile.release]
+lto = "thin"

README.md

@@ -13,7 +13,9 @@ myself agreeing with most of kakoune's design decisions.
 For more information, see the [website](https://helix-editor.com) or
 [documentation](https://docs.helix-editor.com/).
 
-All shortcuts/keymaps can be found [in the documentation on the website](https://docs.helix-editor.com/keymap.html)
+All shortcuts/keymaps can be found [in the documentation on the website](https://docs.helix-editor.com/keymap.html).
+
+[Troubleshooting](https://github.com/helix-editor/helix/wiki/Troubleshooting)
 
 # Features
 
@@ -25,10 +27,10 @@ All shortcuts/keymaps can be found [in the documentation on the website](https:/
 It's a terminal-based editor first, but I'd like to explore a custom renderer
 (similar to emacs) in wgpu or skulpin.
 
-# Installation
-
 Note: Only certain languages have indentation definitions at the moment. Check
-`runtime/<lang>/` for `indents.toml`.
+`runtime/queries/<lang>/` for `indents.toml`.
+
+# Installation
 
 We provide packaging for various distributions, but here's a quick method to
 build from source.
@@ -41,26 +43,17 @@ cargo install --path helix-term
 
 This will install the `hx` binary to `$HOME/.cargo/bin`.
 
-Now copy the `runtime/` directory somewhere. Helix will by default look for the
+Helix also needs its runtime files so make sure to copy/symlink the `runtime/` directory into the
-runtime inside the same folder as the executable, but that can be overriden via
+config directory (for example `~/.config/helix/runtime` on Linux/macOS, or `%AppData%/helix/runtime` on Windows).
-the `HELIX_RUNTIME` environment variable.
+This location can be overriden via the `HELIX_RUNTIME` environment variable.
 
-> NOTE: You should set this to <path to repository>/runtime in development (if
+Packages already solve this for you by wrapping the `hx` binary with a wrapper
-> running via cargo).
+that sets the variable to the install dir.
-> 
-> `export HELIX_RUNTIME=$PWD/runtime`
 
-If you want to embed the `runtime/` directory into the Helix binary you can build
+> NOTE: running via cargo also doesn't require setting explicit `HELIX_RUNTIME` path, it will automatically
-it with:
+> detect the `runtime` directory in the project root.
 
-```
+[![Packaging status](https://repology.org/badge/vertical-allrepos/helix.svg)](https://repology.org/project/helix/versions)
-cargo install --path helix-term --features "embed_runtime"
-```
-
-## Arch Linux
-There are two packages available from AUR:
- - `helix-bin`: contains prebuilt binary from GitHub releases
- - `helix-git`: builds the master branch of this repository
 
 ## MacOS
 Helix can be installed on MacOS through homebrew via:
@@ -72,19 +65,10 @@ brew install helix
  
 # Contributing
 
-Contributors are very welcome! **No contribution is too small and all contributions are valued.**
+Contributing guidelines can be found [here](./docs/CONTRIBUTING.md).
-
-Some suggestions to get started:
-
-- You can look at the [good first issue](https://github.com/helix-editor/helix/labels/good%20first%20issue) label on the issue tracker.
-- Help with packaging on various distributions needed!
-- If your preferred language is missing, integrating a tree-sitter grammar for
-    it and defining syntax highlight queries for it is straight forward and
-    doesn't require much knowledge of the internals.
-
-We provide an [architecture.md](./docs/architecture.md) that should give you
-a good overview of the internals.
 
 # Getting help
 
+Your question might already be answered on the [FAQ](https://github.com/helix-editor/helix/wiki/FAQ).
+
 Discuss the project on the community [Matrix Space](https://matrix.to/#/#helix-community:matrix.org) (make sure to join `#helix-editor:matrix.org` if you're on a client that doesn't support Matrix Spaces yet).

TODO.md

@@ -1,61 +1,27 @@
-- Refactor tree-sitter-highlight to work like the atom one, recomputing partial tree updates.
-
-------
-
-as you type completion!
 
 - tree sitter:
-  - lua
   - markdown
-  - zig
   - regex
-  - vue
   - kotlin
-  - julia
   - clojure
   - erlang
 
-- [ ] use signature_help_provider and completion_provider trigger characters in
-    a hook to trigger signature help text / autocompletion
-- [ ] document.on_type provider triggers
 - [ ] completion isIncomplete support
 
-- [ ] scroll wheel support
-- [ ] matching bracket highlight
-
 1
 - [ ] respect view fullscreen flag
 - [ ] Implement marks (superset of Selection/Range)
 
-- [ ] nixos packaging
-
 - [ ] = for auto indent line/selection
 - [ ]  :x for closing buffers
-
-- [ ] repeat selection
-
-- [] jump to alt buffer
-
 - [ ] lsp: signature help
-- [x] lsp: hover
-- [ ] lsp: document symbols (nested/vec)
-- [ ] lsp: code actions
-- [ ] lsp: formatting
-- [x] lsp: goto
-
-- [ ] search: smart case by default: insensitive unless upper detected
-
-- [ ] move Compositor into tui
 
 2
-- [ ] surround bindings (select + surround ( wraps selection in parens )
 - [ ] macro recording
 - [ ] extend selection (treesitter select parent node) (replaces viw, vi(, va( etc )
-- [x] bracket pairs
-- [x] comment block (gcc)
 - [ ] selection align
 - [ ] store some state between restarts: file positions, prompt history
-- [ ] highlight matched characters in completion
+- [ ] highlight matched characters in picker
 
 3
 - [ ] diff mode with highlighting?

base16_theme.toml

@@ -0,0 +1,42 @@
+# Author: NNB <nnbnh@protonmail.com>
+
+"ui.menu" = "black"
+"ui.menu.selected" = { modifiers = ["reversed"] }
+"ui.linenr" = { fg = "gray", bg = "black" }
+"ui.popup" = { modifiers = ["reversed"] }
+"ui.linenr.selected" = { fg = "white", bg = "black", modifiers = ["bold"] }
+"ui.selection" = { fg = "black", bg = "blue" }
+"ui.selection.primary" = { fg = "white", bg = "blue" }
+"comment" = { fg = "gray" }
+"ui.statusline" = { fg = "black", bg = "white" }
+"ui.statusline.inactive" = { fg = "gray", bg = "white" }
+"ui.help" = { modifiers = ["reversed"] }
+"ui.cursor" = { modifiers = ["reversed"] }
+"variable" = "red"
+"constant.numeric" = "yellow"
+"constant" = "yellow"
+"attributes" = "yellow"
+"type" = "yellow"
+"ui.cursor.match" = { fg = "yellow", modifiers = ["underlined"] }
+"string"  = "green"
+"variable.other.member" = "green"
+"constant.character.escape" = "cyan"
+"function" = "blue"
+"constructor" = "blue"
+"special" = "blue"
+"keyword" = "magenta"
+"label" = "magenta"
+"namespace" = "magenta"
+"ui.help" = { fg = "white", bg = "black" }
+
+"diff.plus" = "green"
+"diff.delta" = "yellow"
+"diff.minus" = "red"
+
+"diagnostic" = { modifiers = ["underlined"] }
+"ui.gutter" = { bg = "black" }
+"info" = "blue"
+"hint" = "gray"
+"debug" = "gray"
+"warning" = "yellow"
+"error" = "red"

book/book.toml

@@ -3,7 +3,9 @@ authors = ["Blaž Hrastnik"]
 language = "en"
 multilingual = false
 src = "src"
-theme = "colibri"
+edit-url-template = "https://github.com/helix-editor/helix/tree/master/book/{path}?mode=edit"
 
 [output.html]
 cname = "docs.helix-editor.com"
+default-theme = "colibri"
+preferred-dark-theme = "colibri"

book/src/SUMMARY.md

@@ -2,6 +2,15 @@
 
 - [Installation](./install.md)
 - [Usage](./usage.md)
-- [Configuration](./configuration.md)
   - [Keymap](./keymap.md)
+  - [Commands](./commands.md)
+  - [Language Support](./lang-support.md)
+- [Migrating from Vim](./from-vim.md)
+- [Configuration](./configuration.md)
+  - [Themes](./themes.md)
+  - [Key Remapping](./remapping.md)
   - [Hooks](./hooks.md)
+  - [Languages](./languages.md)
+- [Guides](./guides/README.md)
+  - [Adding Languages](./guides/adding_languages.md)
+  - [Adding Textobject Queries](./guides/textobject.md)

book/src/commands.md

@@ -0,0 +1,5 @@
+# Commands
+
+Command mode can be activated by pressing `:`, similar to vim. Built-in commands:
+
+{{#include ./generated/typable-cmd.md}}

book/src/configuration.md

@@ -1,87 +1,46 @@
 # Configuration
 
-## Theme
+To override global configuration parameters, create a `config.toml` file located in your config directory:
 
-Use a custom theme by placing a theme.toml in your config directory (i.e ~/.config/helix/theme.toml). The default theme.toml can be found [here](https://github.com/helix-editor/helix/blob/master/theme.toml), and user submitted themes [here](https://github.com/helix-editor/helix/blob/master/contrib/themes).
+* Linux and Mac: `~/.config/helix/config.toml`
+* Windows: `%AppData%\helix\config.toml`
 
-Styles in theme.toml are specified of in the form:
+## Editor
 
+`[editor]` section of the config.
+
+| Key | Description | Default |
+|--|--|---------|
+| `scrolloff` | Number of lines of padding around the edge of the screen when scrolling. | `3` |
+| `mouse` | Enable mouse mode. | `true` |
+| `middle-click-paste` | Middle click paste support. | `true` |
+| `scroll-lines` | Number of lines to scroll per scroll wheel step. | `3` |
+| `shell` | Shell to use when running external commands. | Unix: `["sh", "-c"]`<br/>Windows: `["cmd", "/C"]` |
+| `line-number` | Line number display (`absolute`, `relative`) | `absolute` |
+| `smart-case` | Enable smart case regex searching (case insensitive unless pattern contains upper case characters) | `true` |
+| `auto-pairs` | Enable automatic insertion of pairs to parenthese, brackets, etc. | `true` |
+| `auto-completion` | Enable automatic pop up of auto-completion. | `true` |
+| `idle-timeout` | Time in milliseconds since last keypress before idle timers trigger. Used for autocompletion, set to 0 for instant. | `400` |
+| `completion-trigger-len` | The min-length of word under cursor to trigger autocompletion | `2` |
+| `auto-info` | Whether to display infoboxes | `true` |
+| `true-color` | Set to `true` to override automatic detection of terminal truecolor support in the event of a false negative. | `false` |
+
+`[editor.filepicker]` section of the config. Sets options for file picker and global search. All but the last key listed in the default file-picker configuration below are IgnoreOptions: whether hidden files and files listed within ignore files are ignored by (not visible in) the helix file picker and global search. There is also one other key, `max-depth` available, which is not defined by default.
+
+| Key | Description | Default |
+|--|--|---------|
+|`hidden` | Enables ignoring hidden files. | true
+|`parents` | Enables reading ignore files from parent directories. | true
+|`ignore` | Enables reading `.ignore` files. | true
+|`git-ignore` | Enables reading `.gitignore` files. | true
+|`git-global` | Enables reading global .gitignore, whose path is specified in git's config: `core.excludefile` option. | true
+|`git-exclude` | Enables reading `.git/info/exclude` files. | true
+|`max-depth` | Set with an integer value for maximum depth to recurse. | Defaults to `None`.
+
+## LSP
+
+To display all language server messages in the status line add the following to your `config.toml`:
 ```toml
-key = { fg = "#ffffff", bg = "#000000", modifiers = ["bold", "italic"] }
+[lsp]
+display-messages = true
 ```
-
-where `name` represents what you want to style, `fg` specifies the foreground color, `bg` the background color, and `modifiers` is a list of style modifiers. `bg` and `modifiers` can be omitted to defer to the defaults.
-
-To specify only the foreground color:
-
-```toml
-key = "#ffffff"
-```
-
-if the key contains a dot `'.'`, it must be quoted to prevent it being parsed as a [dotted key](https://toml.io/en/v1.0.0#keys).
-
-```toml
-"key.key" = "#ffffff"
-```
-
-Possible modifiers:
-
-| modifier |
-| --- |
-| bold |
-| dim |
-| italic |
-| underlined |
-| slow\_blink |
-| rapid\_blink |
-| reversed |
-| hidden |
-| crossed\_out |
-
-Possible keys:
-
-| key | notes |
-| --- | --- |
-| attribute | |
-| keyword | |
-| keyword.directive | preprocessor directives (\#if in C) |
-| namespace | |
-| punctuation | |
-| punctuation.delimiter | |
-| operator | |
-| special | |
-| property | |
-| variable | |
-| variable.parameter | |
-| type | |
-| type.builtin | |
-| constructor | |
-| function | |
-| function.macro | |
-| function.builtin | |
-| comment | |
-| variable.builtin | |
-| constant | |
-| constant.builtin | |
-| string | |
-| number | |
-| escape | escaped characters |
-| label | used for lifetimes |
-| module | |
-| ui.background | |
-| ui.linenr | |
-| ui.statusline | |
-| ui.popup | |
-| ui.window | |
-| ui.help | |
-| ui.text | |
-| ui.text.focus | |
-| ui.menu.selected | |
-| warning | LSP warning |
-| error | LSP error |
-| info | LSP info |
-| hint | LSP hint |
-
-These keys match [tree-sitter scopes](https://tree-sitter.github.io/tree-sitter/syntax-highlighting#theme). We half-follow the common scopes from [macromates language grammars](https://macromates.com/manual/en/language_grammars) with some differences.
-
-For a given highlight produced, styling will be determined based on the longest matching theme key. So it's enough to provide function to highlight `function.macro` and `function.builtin` as well, but you can use more specific scopes to highlight specific cases differently.

book/src/from-vim.md

@@ -0,0 +1,12 @@
+# Migrating from Vim
+
+Helix's editing model is strongly inspired from vim and kakoune, and a notable
+difference from vim (and the most striking similarity to kakoune) is that Helix
+follows the `selection → action` model. This means that the whatever you are
+going to act on (a word, a paragraph, a line, etc) is selected first and the
+action itself (delete, change, yank, etc) comes second. A cursor is simply a
+single width selection.
+
+See also Kakoune's [Migrating from Vim](https://github.com/mawww/kakoune/wiki/Migrating-from-Vim).
+
+> TODO: Mention texobjects, surround, registers

book/src/generated/lang-support.md

@@ -0,0 +1,51 @@
+| Language | Syntax Highlighting | Treesitter Textobjects | Auto Indent | Default LSP |
+| --- | --- | --- | --- | --- |
+| bash | ✓ |  |  | `bash-language-server` |
+| c | ✓ |  |  | `clangd` |
+| c-sharp | ✓ |  |  |  |
+| cmake | ✓ | ✓ | ✓ | `cmake-language-server` |
+| comment | ✓ |  |  |  |
+| cpp | ✓ |  |  | `clangd` |
+| css | ✓ |  |  |  |
+| dart | ✓ |  | ✓ | `dart` |
+| dockerfile | ✓ |  |  | `docker-langserver` |
+| elixir | ✓ |  |  | `elixir-ls` |
+| fish | ✓ | ✓ | ✓ |  |
+| git-commit | ✓ |  |  |  |
+| git-diff | ✓ |  |  |  |
+| git-rebase | ✓ |  |  |  |
+| glsl | ✓ |  | ✓ |  |
+| go | ✓ | ✓ | ✓ | `gopls` |
+| html | ✓ |  |  |  |
+| java | ✓ |  |  |  |
+| javascript | ✓ |  | ✓ |  |
+| json | ✓ |  | ✓ |  |
+| julia | ✓ |  |  | `julia` |
+| latex | ✓ |  |  |  |
+| ledger | ✓ |  |  |  |
+| llvm | ✓ | ✓ | ✓ |  |
+| lua | ✓ |  | ✓ |  |
+| markdown | ✓ |  |  |  |
+| mint |  |  |  | `mint` |
+| nix | ✓ |  | ✓ | `rnix-lsp` |
+| ocaml | ✓ |  | ✓ |  |
+| ocaml-interface | ✓ |  |  |  |
+| perl | ✓ | ✓ | ✓ |  |
+| php | ✓ |  | ✓ |  |
+| prolog |  |  |  | `swipl` |
+| protobuf | ✓ |  | ✓ |  |
+| python | ✓ | ✓ | ✓ | `pylsp` |
+| racket |  |  |  | `racket` |
+| ruby | ✓ |  | ✓ | `solargraph` |
+| rust | ✓ | ✓ | ✓ | `rust-analyzer` |
+| scala | ✓ |  | ✓ | `metals` |
+| svelte | ✓ |  | ✓ | `svelteserver` |
+| tablegen | ✓ | ✓ | ✓ |  |
+| toml | ✓ |  |  |  |
+| tsq | ✓ |  |  |  |
+| tsx | ✓ |  |  | `typescript-language-server` |
+| typescript | ✓ |  | ✓ | `typescript-language-server` |
+| vue | ✓ |  |  |  |
+| wgsl | ✓ |  |  |  |
+| yaml | ✓ |  | ✓ |  |
+| zig | ✓ |  | ✓ | `zls` |

book/src/generated/typable-cmd.md

@@ -0,0 +1,46 @@
+| Name | Description |
+| --- | --- |
+| `:quit`, `:q` | Close the current view. |
+| `:quit!`, `:q!` | Close the current view forcefully (ignoring unsaved changes). |
+| `:open`, `:o` | Open a file from disk into the current view. |
+| `:buffer-close`, `:bc`, `:bclose` | Close the current buffer. |
+| `:buffer-close!`, `:bc!`, `:bclose!` | Close the current buffer forcefully (ignoring unsaved changes). |
+| `:write`, `:w` | Write changes to disk. Accepts an optional path (:write some/path.txt) |
+| `:new`, `:n` | Create a new scratch buffer. |
+| `:format`, `:fmt` | Format the file using the LSP formatter. |
+| `:indent-style` | Set the indentation style for editing. ('t' for tabs or 1-8 for number of spaces.) |
+| `:line-ending` | Set the document's default line ending. Options: crlf, lf, cr, ff, nel. |
+| `:earlier`, `:ear` | Jump back to an earlier point in edit history. Accepts a number of steps or a time span. |
+| `:later`, `:lat` | Jump to a later point in edit history. Accepts a number of steps or a time span. |
+| `:write-quit`, `:wq`, `:x` | Write changes to disk and close the current view. Accepts an optional path (:wq some/path.txt) |
+| `:write-quit!`, `:wq!`, `:x!` | Write changes to disk and close the current view forcefully. Accepts an optional path (:wq! some/path.txt) |
+| `:write-all`, `:wa` | Write changes from all views to disk. |
+| `:write-quit-all`, `:wqa`, `:xa` | Write changes from all views to disk and close all views. |
+| `:write-quit-all!`, `:wqa!`, `:xa!` | Write changes from all views to disk and close all views forcefully (ignoring unsaved changes). |
+| `:quit-all`, `:qa` | Close all views. |
+| `:quit-all!`, `:qa!` | Close all views forcefully (ignoring unsaved changes). |
+| `:cquit`, `:cq` | Quit with exit code (default 1). Accepts an optional integer exit code (:cq 2). |
+| `:theme` | Change the editor theme. |
+| `:clipboard-yank` | Yank main selection into system clipboard. |
+| `:clipboard-yank-join` | Yank joined selections into system clipboard. A separator can be provided as first argument. Default value is newline. |
+| `:primary-clipboard-yank` | Yank main selection into system primary clipboard. |
+| `:primary-clipboard-yank-join` | Yank joined selections into system primary clipboard. A separator can be provided as first argument. Default value is newline. |
+| `:clipboard-paste-after` | Paste system clipboard after selections. |
+| `:clipboard-paste-before` | Paste system clipboard before selections. |
+| `:clipboard-paste-replace` | Replace selections with content of system clipboard. |
+| `:primary-clipboard-paste-after` | Paste primary clipboard after selections. |
+| `:primary-clipboard-paste-before` | Paste primary clipboard before selections. |
+| `:primary-clipboard-paste-replace` | Replace selections with content of system primary clipboard. |
+| `:show-clipboard-provider` | Show clipboard provider name in status bar. |
+| `:change-current-directory`, `:cd` | Change the current working directory. |
+| `:show-directory`, `:pwd` | Show the current working directory. |
+| `:encoding` | Set encoding based on `https://encoding.spec.whatwg.org` |
+| `:reload` | Discard changes and reload from the source file. |
+| `:tree-sitter-scopes` | Display tree sitter scopes, primarily for theming and development. |
+| `:vsplit`, `:vs` | Open the file in a vertical split. |
+| `:hsplit`, `:hs`, `:sp` | Open the file in a horizontal split. |
+| `:tutor` | Open the tutorial. |
+| `:goto`, `:g` | Go to line number. |
+| `:set-option`, `:set` | Set a config option at runtime |
+| `:sort` | Sort ranges in selection. |
+| `:rsort` | Sort ranges in selection in reverse order. |

book/src/guides/README.md

@@ -0,0 +1,4 @@
+# Guides
+
+This section contains guides for adding new language server configurations,
+tree-sitter grammers, textobject queries, etc.

book/src/guides/adding_languages.md

@@ -0,0 +1,71 @@
+# Adding languages
+
+## Submodules
+
+To add a new language, you should first add a tree-sitter submodule. To do this,
+you can run the command
+```sh
+git submodule add -f <repository> helix-syntax/languages/tree-sitter-<name>
+```
+For example, to add tree-sitter-ocaml you would run
+```sh
+git submodule add -f https://github.com/tree-sitter/tree-sitter-ocaml helix-syntax/languages/tree-sitter-ocaml
+```
+Make sure the submodule is shallow by doing
+```sh
+git config -f .gitmodules submodule.helix-syntax/languages/tree-sitter-<name>.shallow true
+```
+
+or you can manually add `shallow = true` to `.gitmodules`.
+
+## languages.toml
+
+Next, you need to add the language to the [`languages.toml`][languages.toml] found in the root of
+the repository; this `languages.toml` file is included at compilation time, and
+is distinct from the `language.toml` file in the user's [configuration
+directory](../configuration.md).
+
+These are the available keys and descriptions for the file.
+
+| Key                 | Description                                                   |
+| ----                | -----------                                                   |
+| name                | The name of the language                                      |
+| scope               | A string like `source.js` that identifies the language. Currently, we strive to match the scope names used by popular TextMate grammars and by the Linguist library. Usually `source.<name>` or `text.<name>` in case of markup languages |
+| injection-regex     | regex pattern that will be tested against a language name in order to determine whether this language should be used for a potential [language injection][treesitter-language-injection] site. |
+| file-types          | The filetypes of the language, for example `["yml", "yaml"]`  |
+| shebangs            | The interpreters from the shebang line, for example `["sh", "bash"]` |
+| roots               | A set of marker files to look for when trying to find the workspace root. For example `Cargo.lock`, `yarn.lock` |
+| auto-format         | Whether to autoformat this language when saving               |
+| diagnostic-severity | Minimal severity of diagnostic for it to be displayed. (Allowed values: `Error`, `Warning`, `Info`, `Hint`) |
+| comment-token       | The token to use as a comment-token                           |
+| indent              | The indent to use. Has sub keys `tab-width` and `unit`        |
+| config              | Language server configuration                                 |
+
+## Queries
+
+For a language to have syntax-highlighting and indentation among
+other things, you have to add queries. Add a directory for your
+language with the path `runtime/queries/<name>/`. The tree-sitter
+[website](https://tree-sitter.github.io/tree-sitter/syntax-highlighting#queries)
+gives more info on how to write queries.
+
+> NOTE: When evaluating queries, the first matching query takes
+precedence, which is different from other editors like neovim where
+the last matching query supercedes the ones before it. See
+[this issue][neovim-query-precedence] for an example.
+
+## Common Issues
+
+- If you get errors when building after switching branches, you may have to remove or update tree-sitter submodules. You can update submodules by running
+    ```sh
+    git submodule sync; git submodule update --init
+    ```
+- Make sure to not use the `--remote` flag. To remove submodules look inside the `.gitmodules` and remove directories that are not present inside of it.
+
+- If a parser is segfaulting or you want to remove the parser, make sure to remove the submodule *and* the compiled parser in `runtime/grammar/<name>.so`
+
+- The indents query is `indents.toml`, *not* `indents.scm`. See [this](https://github.com/helix-editor/helix/issues/114) issue for more information.
+
+[treesitter-language-injection]: https://tree-sitter.github.io/tree-sitter/syntax-highlighting#language-injection
+[languages.toml]: https://github.com/helix-editor/helix/blob/master/languages.toml
+[neovim-query-precedence]: https://github.com/helix-editor/helix/pull/1170#issuecomment-997294090

book/src/guides/textobject.md

@@ -0,0 +1,30 @@
+# Adding Textobject Queries
+
+Textobjects that are language specific ([like functions, classes, etc][textobjects])
+require an accompanying tree-sitter grammar and a `textobjects.scm` query file
+to work properly. Tree-sitter allows us to query the source code syntax tree
+and capture specific parts of it. The queries are written in a lisp dialect.
+More information on how to write queries can be found in the [official tree-sitter
+documentation][tree-sitter-queries].
+
+Query files should be placed in `runtime/queries/{language}/textobjects.scm`
+when contributing. Note that to test the query files locally you should put
+them under your local runtime directory (`~/.config/helix/runtime` on Linux
+for example).
+
+The following [captures][tree-sitter-captures] are recognized:
+
+| Capture Name       |
+| ---                |
+| `function.inside`  |
+| `function.around`  |
+| `class.inside`     |
+| `class.around`     |
+| `parameter.inside` |
+
+[Example query files][textobject-examples] can be found in the helix GitHub repository.
+
+[textobjects]: ../usage.md#textobjects
+[tree-sitter-queries]: https://tree-sitter.github.io/tree-sitter/using-parsers#query-syntax
+[tree-sitter-captures]: https://tree-sitter.github.io/tree-sitter/using-parsers#capturing-nodes
+[textobject-examples]: https://github.com/search?q=repo%3Ahelix-editor%2Fhelix+filename%3Atextobjects.scm&type=Code&ref=advsearch&l=&l=

book/src/install.md

@@ -2,11 +2,16 @@
 
 We provide pre-built binaries on the [GitHub Releases page](https://github.com/helix-editor/helix/releases).
 
+[![Packaging status](https://repology.org/badge/vertical-allrepos/helix.svg)](https://repology.org/project/helix/versions)
+
 ## OSX
 
-TODO: brew tap
+A Homebrew tap is available:
 
-Please use a pre-built binary release for the time being.
+```
+brew tap helix-editor/helix
+brew install helix
+```
 
 ## Linux
 
@@ -18,9 +23,18 @@ shell for working on Helix.
 
 ### Arch Linux
 
-Binary packages are available on AUR:
+Releases are available in the `community` repository.
-- [helix-bin](https://aur.archlinux.org/packages/helix-bin/) contains the pre-built release
+
-- [helix-git](https://aur.archlinux.org/packages/helix-git/) builds the master branch
+A [helix-git](https://aur.archlinux.org/packages/helix-git/) package is also available on the AUR, which builds the master branch.
+
+### Fedora Linux
+
+You can install the COPR package for Helix via
+
+```
+sudo dnf copr enable varlad/helix
+sudo dnf install helix
+```
 
 ## Build from source
 
@@ -32,13 +46,6 @@ cargo install --path helix-term
 
 This will install the `hx` binary to `$HOME/.cargo/bin`.
 
-Now copy the `runtime/` directory somewhere. Helix will by default look for the
+Helix also needs it's runtime files so make sure to copy/symlink the `runtime/` directory into the
-runtime inside the same folder as the executable, but that can be overriden via
+config directory (for example `~/.config/helix/runtime` on Linux/macOS). This location can be overriden
-the `HELIX_RUNTIME` environment variable.
+via the `HELIX_RUNTIME` environment variable.
-
-If you want to embed the `runtime/` directory into the Helix binary you can build
-it with:
-
-```
-cargo install --path helix-term --features "embed_runtime"
-```

book/src/keymap.md

@@ -1,167 +1,334 @@
 # Keymap
 
+- Mappings marked (**LSP**) require an active language server for the file.
+- Mappings marked (**TS**) require a tree-sitter grammar for the filetype.
+
 ## Normal mode
 
 ### Movement
 
-| Key | Description |
+> NOTE: Unlike vim, `f`, `F`, `t` and `T` are not confined to the current line.
-|-----|-----------|
+
-| h, Left   | move left |
+| Key         | Description                                        | Command                     |
-| j, Down   | move down |
+| -----       | -----------                                        | -------                     |
-| k, Up   | move up |
+| `h`/`Left`  | Move left                                          | `move_char_left`            |
-| l, Right   | move right |
+| `j`/`Down`  | Move down                                          | `move_line_down`            |
-| w   | move next word start |
+| `k`/`Up`    | Move up                                            | `move_line_up`              |
-| b   | move previous word start |
+| `l`/`Right` | Move right                                         | `move_char_right`           |
-| e   | move next word end |
+| `w`         | Move next word start                               | `move_next_word_start`      |
-| t   | find 'till next char |
+| `b`         | Move previous word start                           | `move_prev_word_start`      |
-| f   | find next char |
+| `e`         | Move next word end                                 | `move_next_word_end`        |
-| T   | find 'till previous char |
+| `W`         | Move next WORD start                               | `move_next_long_word_start` |
-| F   | find previous char |
+| `B`         | Move previous WORD start                           | `move_prev_long_word_start` |
-| Home   | move to the start of the line |
+| `E`         | Move next WORD end                                 | `move_next_long_word_end`   |
-| End   | move to the end of the line |
+| `t`         | Find 'till next char                               | `find_till_char`            |
-| m   | Jump to matching bracket | 
+| `f`         | Find next char                                     | `find_next_char`            |
-| PageUp | Move page up |
+| `T`         | Find 'till previous char                           | `till_prev_char`            |
-| PageDown | Move page down |
+| `F`         | Find previous char                                 | `find_prev_char`            |
-| ctrl-u | Move half page up |
+| `Alt-.`     | Repeat last motion (`f`, `t` or `m`)               | `repeat_last_motion`        |
-| ctrl-d | Move half page down |
+| `Home`      | Move to the start of the line                      | `goto_line_start`           |
-| ctrl-i | Jump forward on the jumplist TODO: conflicts tab |
+| `End`       | Move to the end of the line                        | `goto_line_end`             |
-| ctrl-o | Jump backward on the jumplist |
+| `PageUp`    | Move page up                                       | `page_up`                   |
-| v   | Enter select (extend) mode |
+| `PageDown`  | Move page down                                     | `page_down`                 |
-| g   | Enter goto mode |
+| `Ctrl-u`    | Move half page up                                  | `half_page_up`              |
-| :   | Enter command mode |
+| `Ctrl-d`    | Move half page down                                | `half_page_down`            |
-| z   | Enter view mode |
+| `Ctrl-i`    | Jump forward on the jumplist                       | `jump_forward`              |
-| ctrl-w | Enter window mode (maybe will be remove for spc w w later) |
+| `Ctrl-o`    | Jump backward on the jumplist                      | `jump_backward`             |
-| space   | Enter space mode |
+| `Ctrl-s`    | Save the current selection to the jumplist         | `save_selection`            |
-| K | Show documentation for the item under the cursor |
+| `v`         | Enter [select (extend) mode](#select--extend-mode) | `select_mode`               |
+| `g`         | Enter [goto mode](#goto-mode)                      | N/A                         |
+| `m`         | Enter [match mode](#match-mode)                    | N/A                         |
+| `:`         | Enter command mode                                 | `command_mode`              |
+| `z`         | Enter [view mode](#view-mode)                      | N/A                         |
+| `Z`         | Enter sticky [view mode](#view-mode)               | N/A                         |
+| `Ctrl-w`    | Enter [window mode](#window-mode)                  | N/A                         |
+| `Space`     | Enter [space mode](#space-mode)                    | N/A                         |
 
 ### Changes
 
-| Key | Description |
+| Key         | Description                                                          | Command                   |
-|-----|-----------|
+| -----       | -----------                                                          | -------                   |
-| r   | replace (single character change) |
+| `r`         | Replace with a character                                             | `replace`                 |
-| i   | Insert before selection |
+| `R`         | Replace with yanked text                                             | `replace_with_yanked`     |
-| a   | Insert after selection (append) |
+| `~`         | Switch case of the selected text                                     | `switch_case`             |
-| I   | Insert at the start of the line | 
+| `` ` ``     | Set the selected text to lower case                                  | `switch_to_lowercase`     |
-| A   | Insert at the end of the line | 
+| `` Alt-` `` | Set the selected text to upper case                                  | `switch_to_uppercase`     |
-| o   | Open new line below selection | 
+| `i`         | Insert before selection                                              | `insert_mode`             |
-| o   | Open new line above selection | 
+| `a`         | Insert after selection (append)                                      | `append_mode`             |
-| u   | Undo change |
+| `I`         | Insert at the start of the line                                      | `prepend_to_line`         |
-| U   | Redo change |
+| `A`         | Insert at the end of the line                                        | `append_to_line`          |
-| y   | Yank selection |
+| `o`         | Open new line below selection                                        | `open_below`              |
-| p   | Paste after selection |
+| `O`         | Open new line above selection                                        | `open_above`              |
-| P   | Paste before selection |
+| `.`         | Repeat last change                                                   | N/A                       |
-| >   | Indent selection |
+| `u`         | Undo change                                                          | `undo`                    |
-| <   | Unindent selection |
+| `U`         | Redo change                                                          | `redo`                    |
-| =   | Format selection |
+| `Alt-u`     | Move backward in history                                             | `earlier`                 |
-| d   | Delete selection | 
+| `Alt-U`     | Move forward in history                                              | `later`                   |
-| c   | Change selection (delete and enter insert mode) | 
+| `y`         | Yank selection                                                       | `yank`                    |
+| `p`         | Paste after selection                                                | `paste_after`             |
+| `P`         | Paste before selection                                               | `paste_before`            |
+| `"` `<reg>` | Select a register to yank to or paste from                           | `select_register`         |
+| `>`         | Indent selection                                                     | `indent`                  |
+| `<`         | Unindent selection                                                   | `unindent`                |
+| `=`         | Format selection (currently nonfunctional/disabled) (**LSP**)        | `format_selections`       |
+| `d`         | Delete selection                                                     | `delete_selection`        |
+| `Alt-d`     | Delete selection, without yanking                                    | `delete_selection_noyank` |
+| `c`         | Change selection (delete and enter insert mode)                      | `change_selection`        |
+| `Alt-c`     | Change selection (delete and enter insert mode, without yanking)     | `change_selection_noyank` |
+| `Ctrl-a`    | Increment object (number) under cursor                               | `increment`               |
+| `Ctrl-x`    | Decrement object (number) under cursor                               | `decrement`               |
+| `Q`         | Start/stop macro recording to the selected register (experimental)   | `record_macro`            |
+| `q`         | Play back a recorded macro from the selected register (experimental) | `replay_macro`            |
+
+#### Shell
+
+| Key                     | Description                                                      | Command               |
+| ------                  | -----------                                                      | -------               |
+| <code>&#124;</code>     | Pipe each selection through shell command, replacing with output | `shell_pipe`          |
+| <code>Alt-&#124;</code> | Pipe each selection into shell command, ignoring output          | `shell_pipe_to`       |
+| `!`                     | Run shell command, inserting output before each selection        | `shell_insert_output` |
+| `Alt-!`                 | Run shell command, appending output after each selection         | `shell_append_output` |
+
 
 ### Selection manipulation
 
-| Key | Description |
+| Key      | Description                                                       | Command                              |
-|-----|-----------|
+| -----    | -----------                                                       | -------                              |
-| s   | Select all regex matches inside selections | 
+| `s`      | Select all regex matches inside selections                        | `select_regex`                       |
-| S   | Split selection into subselections on regex matches | 
+| `S`      | Split selection into subselections on regex matches               | `split_selection`                    |
-| alt-s   | Split selection on newlines | 
+| `Alt-s`  | Split selection on newlines                                       | `split_selection_on_newline`         |
-| ;   | Collapse selection onto a single cursor | 
+| `&`      | Align selection in columns                                        | `align_selections`                   |
-| alt-;   | Flip selection cursor and anchor | 
+| `_`      | Trim whitespace from the selection                                | `trim_selections`                    |
-| %   | Select entire file | 
+| `;`      | Collapse selection onto a single cursor                           | `collapse_selection`                 |
-| x   | Select current line | 
+| `Alt-;`  | Flip selection cursor and anchor                                  | `flip_selections`                    |
-| X   | Extend to next line | 
+| `,`      | Keep only the primary selection                                   | `keep_primary_selection`             |
-| [   | Expand selection to parent syntax node TODO: pick a key | 
+| `Alt-,`  | Remove the primary selection                                      | `remove_primary_selection`           |
-| J | join lines inside selection |
+| `C`      | Copy selection onto the next line (Add cursor below)              | `copy_selection_on_next_line`        |
-| K | keep selections matching the regex TODO: overlapped by hover help |
+| `Alt-C`  | Copy selection onto the previous line (Add cursor above)          | `copy_selection_on_prev_line`        |
-| space | keep only the primary selection TODO: overlapped by space mode |
+| `(`      | Rotate main selection backward                                    | `rotate_selections_backward`         |
-| ctrl-c | Comment/uncomment the selections | 
+| `)`      | Rotate main selection forward                                     | `rotate_selections_forward`          |
+| `Alt-(`  | Rotate selection contents backward                                | `rotate_selection_contents_backward` |
+| `Alt-)`  | Rotate selection contents forward                                 | `rotate_selection_contents_forward`  |
+| `%`      | Select entire file                                                | `select_all`                         |
+| `x`      | Select current line, if already selected, extend to next line     | `extend_line`                        |
+| `X`      | Extend selection to line bounds (line-wise selection)             | `extend_to_line_bounds`              |
+|          | Expand selection to parent syntax node TODO: pick a key (**TS**)  | `expand_selection`                   |
+| `J`      | Join lines inside selection                                       | `join_selections`                    |
+| `K`      | Keep selections matching the regex                                | `keep_selections`                    |
+| `Alt-K`  | Remove selections matching the regex                              | `remove_selections`                  |
+| `$`      | Pipe each selection into shell command, keep selections where command returned 0 | `shell_keep_pipe`     |
+| `Ctrl-c` | Comment/uncomment the selections                                  | `toggle_comments`                    |
 
 ### Search
 
-> TODO: The search implementation isn't ideal yet -- we don't support searching
-in reverse, or searching via smartcase.
 
-| Key | Description |
+| Key   | Description                                 | Command              |
-|-----|-----------|
+| ----- | -----------                                 | -------              |
-| /   | Search for regex pattern | 
+| `/`   | Search for regex pattern                    | `search`             |
-| n   | Select next search match | 
+| `?`   | Search for previous pattern                 | `rsearch`            |
-| N   | Add next search match to selection | 
+| `n`   | Select next search match                    | `search_next`        |
-| *   | Use current selection as the search pattern | 
+| `N`   | Select previous search match                | `search_prev`        |
+| `*`   | Use current selection as the search pattern | `search_selection`   |
 
-### Diagnostics
+### Minor modes
 
-> NOTE: `[` and `]` will likely contain more pair mappings in the style of
+These sub-modes are accessible from normal mode and typically switch back to normal mode after a command.
-> [vim-unimpaired](https://github.com/tpope/vim-unimpaired)
 
-| Key | Description |
+#### View mode
-|-----|-----------|
+
-| [d | Go to previous diagnostic |
+View mode is intended for scrolling and manipulating the view without changing
-| ]d | Go to next diagnostic |
+the selection. The "sticky" variant of this mode is persistent; use the Escape
-| [D | Go to first diagnostic in document |
+key to return to normal mode after usage (useful when you're simply looking
-| ]D | Go to last diagnostic in document |
+over text and not actively editing it).
+
+
+| Key           | Description                                               | Command             |
+| -----         | -----------                                               | -------             |
+| `z` , `c`     | Vertically center the line                                | `align_view_center` |
+| `t`           | Align the line to the top of the screen                   | `align_view_top`    |
+| `b`           | Align the line to the bottom of the screen                | `align_view_bottom` |
+| `m`           | Align the line to the middle of the screen (horizontally) | `align_view_middle` |
+| `j` , `down`  | Scroll the view downwards                                 | `scroll_down`       |
+| `k` , `up`    | Scroll the view upwards                                   | `scroll_up`         |
+| `f`           | Move page down                                            | `page_down`         |
+| `b`           | Move page up                                              | `page_up`           |
+| `d`           | Move half page down                                       | `half_page_down`    |
+| `u`           | Move half page up                                         | `half_page_up`      |
+
+#### Goto mode
+
+Jumps to various locations.
+
+| Key   | Description                                      | Command                    |
+| ----- | -----------                                      | -------                    |
+| `g`   | Go to line number `<n>` else start of file       | `goto_file_start`          |
+| `e`   | Go to the end of the file                        | `goto_last_line`           |
+| `f`   | Go to files in the selection                     | `goto_file`                |
+| `h`   | Go to the start of the line                      | `goto_line_start`          |
+| `l`   | Go to the end of the line                        | `goto_line_end`            |
+| `s`   | Go to first non-whitespace character of the line | `goto_first_nonwhitespace` |
+| `t`   | Go to the top of the screen                      | `goto_window_top`          |
+| `c`   | Go to the middle of the screen                   | `goto_window_center`       |
+| `b`   | Go to the bottom of the screen                   | `goto_window_bottom`       |
+| `d`   | Go to definition (**LSP**)                       | `goto_definition`          |
+| `y`   | Go to type definition (**LSP**)                  | `goto_type_definition`     |
+| `r`   | Go to references (**LSP**)                       | `goto_reference`           |
+| `i`   | Go to implementation (**LSP**)                   | `goto_implementation`      |
+| `a`   | Go to the last accessed/alternate file           | `goto_last_accessed_file`  |
+| `m`   | Go to the last modified/alternate file           | `goto_last_modified_file`  |
+| `n`   | Go to next buffer                                | `goto_next_buffer`         |
+| `p`   | Go to previous buffer                            | `goto_previous_buffer`     |
+| `.`   | Go to last modification in current file          | `goto_last_modification`   |
+
+#### Match mode
+
+Enter this mode using `m` from normal mode. See the relavant section
+in [Usage](./usage.md) for an explanation about [surround](./usage.md#surround)
+and [textobject](./usage.md#textobject) usage.
+
+| Key              | Description                                     | Command                    |
+| -----            | -----------                                     | -------                    |
+| `m`              | Goto matching bracket (**TS**)                  | `match_brackets`           |
+| `s` `<char>`     | Surround current selection with `<char>`        | `surround_add`             |
+| `r` `<from><to>` | Replace surround character `<from>` with `<to>` | `surround_replace`         |
+| `d` `<char>`     | Delete surround character `<char>`              | `surround_delete`          |
+| `a` `<object>`   | Select around textobject                        | `select_textobject_around` |
+| `i` `<object>`   | Select inside textobject                        | `select_textobject_inner`  |
+
+TODO: Mappings for selecting syntax nodes (a superset of `[`).
+
+#### Window mode
+
+This layer is similar to vim keybindings as kakoune does not support window.
+
+| Key                    | Description                    | Command           |
+| -----                  | -------------                  | -------           |
+| `w`, `Ctrl-w`          | Switch to next window          | `rotate_view`     |
+| `v`, `Ctrl-v`          | Vertical right split           | `vsplit`          |
+| `s`, `Ctrl-s`          | Horizontal bottom split        | `hsplit`          |
+| `h`, `Ctrl-h`, `left`  | Move to left split             | `jump_view_left`  |
+| `f`                    | Go to files in the selection in horizontal splits  | `goto_file`                |
+| `F`                    | Go to files in the selection in vertical splits    | `goto_file`                |
+| `j`, `Ctrl-j`, `down`  | Move to split below            | `jump_view_down`  |
+| `k`, `Ctrl-k`, `up`    | Move to split above            | `jump_view_up`    |
+| `l`, `Ctrl-l`, `right` | Move to right split            | `jump_view_right` |
+| `q`, `Ctrl-q`          | Close current window           | `wclose`          |
+| `o`, `Ctrl-o`          | Only keep the current window, closing all the others            | `wonly`           |
+
+#### Space mode
+
+This layer is a kludge of mappings, mostly pickers.
+
+
+| Key     | Description                                                             | Command                             |
+| -----   | -----------                                                             | -------                             |
+| `f`     | Open file picker                                                        | `file_picker`                       |
+| `b`     | Open buffer picker                                                      | `buffer_picker`                     |
+| `k`     | Show documentation for item under cursor in a [popup](#popup) (**LSP**) | `hover`                             |
+| `s`     | Open document symbol picker (**LSP**)                                   | `symbol_picker`                     |
+| `S`     | Open workspace symbol picker (**LSP**)                                  | `workspace_symbol_picker`           |
+| `r`     | Rename symbol (**LSP**)                                                 | `rename_symbol`                     |
+| `a`     | Apply code action  (**LSP**)                                            | `code_action`                       |
+| `'`     | Open last fuzzy picker                                                  | `last_picker`                       |
+| `w`     | Enter [window mode](#window-mode)                                       | N/A                                 |
+| `p`     | Paste system clipboard after selections                                 | `paste_clipboard_after`             |
+| `P`     | Paste system clipboard before selections                                | `paste_clipboard_before`            |
+| `y`     | Join and yank selections to clipboard                                   | `yank_joined_to_clipboard`          |
+| `Y`     | Yank main selection to clipboard                                        | `yank_main_selection_to_clipboard`  |
+| `R`     | Replace selections by clipboard contents                                | `replace_selections_with_clipboard` |
+| `/`     | Global search in workspace folder                                       | `global_search`                     |
+
+> TIP: Global search displays results in a fuzzy picker, use `space + '` to bring it back up after opening a file.
+
+##### Popup
+
+Displays documentation for item under cursor.
+
+| Key      | Description |
+| ----     | ----------- |
+| `Ctrl-u` | Scroll up   |
+| `Ctrl-d` | Scroll down |
+ 
+#### Unimpaired
+
+Mappings in the style of [vim-unimpaired](https://github.com/tpope/vim-unimpaired).
+
+| Key      | Description                                  | Command             |
+| -----    | -----------                                  | -------             |
+| `[d`     | Go to previous diagnostic (**LSP**)          | `goto_prev_diag`    |
+| `]d`     | Go to next diagnostic (**LSP**)              | `goto_next_diag`    |
+| `[D`     | Go to first diagnostic in document (**LSP**) | `goto_first_diag`   |
+| `]D`     | Go to last diagnostic in document (**LSP**)  | `goto_last_diag`    |
+| `[space` | Add newline above                            | `add_newline_above` |
+| `]space` | Add newline below                            | `add_newline_below` |
+
+## Insert Mode
+
+| Key                     | Description                 | Command                 |
+| -----                   | -----------                 | -------                 |
+| `Escape`                | Switch to normal mode       | `normal_mode`           |
+| `Ctrl-x`                | Autocomplete                | `completion`            |
+| `Ctrl-r`                | Insert a register content   | `insert_register`       |
+| `Ctrl-w`                | Delete previous word        | `delete_word_backward`  |
+| `Alt-d`                 | Delete next word            | `delete_word_forward`   |
+| `Alt-b`, `Alt-Left`     | Backward a word             | `move_prev_word_end`    |
+| `Ctrl-b`, `Left`        | Backward a char             | `move_char_left`        |
+| `Alt-f`, `Alt-Right`    | Forward a word              | `move_next_word_start`  |
+| `Ctrl-f`, `Right`       | Forward a char              | `move_char_right`       |
+| `Ctrl-e`, `End`         | move to line end            | `goto_line_end_newline` |
+| `Ctrl-a`, `Home`        | move to line start          | `goto_line_start`       |
+| `Ctrl-u`                | delete to start of line     | `kill_to_line_start`    |
+| `Ctrl-k`                | delete to end of line       | `kill_to_line_end`      |
+| `backspace`, `Ctrl-h`   | delete previous char        | `delete_char_backward`  |
+| `delete`, `Ctrl-d`      | delete previous char        | `delete_char_forward`   |
+| `Ctrl-p`, `Up`          | move to previous line       | `move_line_up`          |
+| `Ctrl-n`, `Down`        | move to next line           | `move_line_down`        |
 
 ## Select / extend mode
 
 I'm still pondering whether to keep this mode or not. It changes movement
-commands to extend the existing selection instead of replacing it.
+commands (including goto) to extend the existing selection instead of replacing it.
 
 > NOTE: It's a bit confusing at the moment because extend hasn't been
 > implemented for all movement commands yet.
 
-## View mode
+# Picker
 
-View mode is intended for scrolling and manipulating the view without changing
+Keys to use within picker. Remapping currently not supported.
-the selection.
 
-| Key | Description |
+| Key                          | Description       |
-|-----|-----------|
+| -----                        | -------------     |
-| z , c | Vertically center the line |
+| `Up`, `Ctrl-k`, `Ctrl-p`     | Previous entry    |
-| t   | Align the line to the top of the screen |
+| `Down`, `Ctrl-j`, `Ctrl-n`   | Next entry        |
-| b   | Align the line to the bottom of the screen |
+| `Ctrl-space`                 | Filter options    |
-| m   | Align the line to the middle of the screen (horizontally) |
+| `Enter`                      | Open selected     |
-| j   | Scroll the view downwards |
+| `Ctrl-s`                     | Open horizontally |
-| k   | Scroll the view upwards |
+| `Ctrl-v`                     | Open vertically   |
+| `Escape`, `Ctrl-c`           | Close picker      |
 
-## Goto mode
+# Prompt
 
-Jumps to various locations.
+Keys to use within prompt, Remapping currently not supported.
 
-> NOTE: Some of these features are only available with the LSP present.
+| Key                     | Description                                                             |
+| -----                   | -------------                                                           |
+| `Escape`, `Ctrl-c`      | Close prompt                                                            |
+| `Alt-b`, `Alt-Left`     | Backward a word                                                         |
+| `Ctrl-b`, `Left`        | Backward a char                                                         |
+| `Alt-f`, `Alt-Right`    | Forward a word                                                          |
+| `Ctrl-f`, `Right`       | Forward a char                                                          |
+| `Ctrl-e`, `End`         | Move prompt end                                                         |
+| `Ctrl-a`, `Home`        | Move prompt start                                                       |
+| `Ctrl-w`                | Delete previous word                                                    |
+| `Alt-d`                 | Delete next word                                                        |
+| `Ctrl-u`                | Delete to start of line                                                 |
+| `Ctrl-k`                | Delete to end of line                                                   |
+| `backspace`, `Ctrl-h`   | Delete previous char                                                    |
+| `delete`, `Ctrl-d`      | Delete next char                                                        |
+| `Ctrl-s`                | Insert a word under doc cursor, may be changed to Ctrl-r Ctrl-w later   |
+| `Ctrl-p`, `Up`          | Select previous history                                                 |
+| `Ctrl-n`, `Down`        | Select next history                                                     |
+| `Tab`                   | Select next completion item                                             |
+| `BackTab`               | Select previous completion item                                         |
+| `Enter`                 | Open selected                                                           |
 
-| Key | Description |
-|-----|-----------|
-| g   | Go to the start of the file |
-| e   | Go to the end of the file |
-| h   | Go to the start of the line |
-| l   | Go to the end of the line |
-| t   | Go to the top of the screen |
-| m   | Go to the middle of the screen |
-| b   | Go to the bottom of the screen |
-| d   | Go to definition |
-| y   | Go to type definition |
-| r   | Go to references |
-| i   | Go to implementation |
-
-## Object mode
-
-TODO: Mappings for selecting syntax nodes (a superset of `[`).
-
-## Window mode
-
-This layer is similar to vim keybindings as kakoune does not support window.
-
-| Key | Description |
-|-----|-------------|
-| w, ctrl-w | Switch to next window |
-| v, ctrl-v | Vertical right split |
-| h, ctrl-h | Horizontal bottom split |
-| q, ctrl-q | Close current window |
-
-## Space mode
-
-This layer is a kludge of mappings I had under leader key in neovim.
-
-| Key | Description |
-|-----|-----------|
-| f   | Open file picker |
-| b   | Open buffer picker |
-| w   | Enter window mode |
-| space   | Keep primary selection TODO: it's here because space mode replaced it |

book/src/lang-support.md

@@ -0,0 +1,10 @@
+# Language Support
+
+For more information like arguments passed to default LSP server,
+extensions assosciated with a filetype, custom LSP settings, filetype
+specific indent settings, etc see the default
+[`languages.toml`][languages.toml] file.
+
+{{#include ./generated/lang-support.md}}
+
+[languages.toml]: https://github.com/helix-editor/helix/blob/master/languages.toml

book/src/languages.md

@@ -0,0 +1,13 @@
+# Languages
+
+Language-specific settings and settings for particular language servers can be configured in a `languages.toml` file placed in your [configuration directory](./configuration.md). Helix actually uses two `languages.toml` files, the [first one](https://github.com/helix-editor/helix/blob/master/languages.toml) is in the main helix repository; it contains the default settings for each language and is included in the helix binary at compile time. Users who want to see the available settings and options can either reference the helix repo's `languages.toml` file, or consult the table in the [adding languages](./guides/adding_languages.md) section.
+
+Changes made to the `languages.toml` file in a user's [configuration directory](./configuration.md) are merged with helix's defaults on start-up, such that a user's settings will take precedence over defaults in the event of a collision. For example, the default `languages.toml` sets rust's `auto-format` to `true`. If a user wants to disable auto-format, they can change the `languages.toml` in their [configuration directory](./configuration.md) to make the rust entry read like the example below; the new key/value pair `auto-format = false` will override the default when the two sets of settings are merged on start-up:
+
+```
+# in <config_dir>/helix/languages.toml
+
+[[language]]
+name = "rust"
+auto-format = false
+```

book/src/remapping.md

@@ -0,0 +1,59 @@
+# Key Remapping
+
+One-way key remapping is temporarily supported via a simple TOML configuration
+file. (More powerful solutions such as rebinding via commands will be
+available in the future).
+
+To remap keys, write a `config.toml` file in your `helix` configuration
+directory (default `~/.config/helix` in Linux systems) with a structure like
+this:
+
+```toml
+# At most one section each of 'keys.normal', 'keys.insert' and 'keys.select'
+[keys.normal]
+C-s = ":w" # Maps the Control-s to the typable command :w which is an alias for :write (save file)
+C-o = ":open ~/.config/helix/config.toml" # Maps the Control-o to opening of the helix config file
+a = "move_char_left" # Maps the 'a' key to the move_char_left command
+w = "move_line_up" # Maps the 'w' key move_line_up
+"C-S-esc" = "extend_line" # Maps Control-Shift-Escape to extend_line
+g = { a = "code_action" } # Maps `ga` to show possible code actions
+"ret" = ["open_below", "normal_mode"] # Maps the enter key to open_below then re-enter normal mode
+
+[keys.insert]
+"A-x" = "normal_mode" # Maps Alt-X to enter normal mode
+j = { k = "normal_mode" } # Maps `jk` to exit insert mode
+```
+> NOTE: Typable commands can also be remapped, remember to keep the `:` prefix to indicate it's a typable command.
+
+Control, Shift and Alt modifiers are encoded respectively with the prefixes
+`C-`, `S-` and `A-`. Special keys are encoded as follows:
+
+| Key name     | Representation |
+| ---          | ---            |
+| Backspace    | `"backspace"`  |
+| Space        | `"space"`      |
+| Return/Enter | `"ret"`        |
+| <            | `"lt"`         |
+| \>           | `"gt"`         |
+| \+           | `"plus"`       |
+| \-           | `"minus"`      |
+| ;            | `"semicolon"`  |
+| %            | `"percent"`    |
+| Left         | `"left"`       |
+| Right        | `"right"`      |
+| Up           | `"up"`         |
+| Down         | `"down"`       |
+| Home         | `"home"`       |
+| End          | `"end"`        |
+| Page Up      | `"pageup"`     |
+| Page Down    | `"pagedown"`   |
+| Tab          | `"tab"`        |
+| Delete       | `"del"`        |
+| Insert       | `"ins"`        |
+| Null         | `"null"`       |
+| Escape       | `"esc"`        |
+
+Keys can be disabled by binding them to the `no_op` command.
+
+Commands can be found at [Keymap](https://docs.helix-editor.com/keymap.html) Commands.
+> Commands can also be found in the source code at [`helix-term/src/commands.rs`](https://github.com/helix-editor/helix/blob/master/helix-term/src/commands.rs) at the invocation of `static_commands!` macro and the `TypableCommandList`.

book/src/themes.md

@@ -0,0 +1,220 @@
+# Themes
+
+First you'll need to place selected themes in your `themes` directory (i.e `~/.config/helix/themes`), the directory might have to be created beforehand.
+
+To use a custom theme add `theme = <name>` to your [`config.toml`](./configuration.md) or override it during runtime using `:theme <name>`.
+
+The default theme.toml can be found [here](https://github.com/helix-editor/helix/blob/master/theme.toml), and user submitted themes [here](https://github.com/helix-editor/helix/blob/master/runtime/themes). 
+
+## Creating a theme
+
+First create a file with the name of your theme as file name (i.e `mytheme.toml`) and place it in your `themes` directory (i.e `~/.config/helix/themes`).
+
+Each line in the theme file is specified as below:
+
+```toml
+key = { fg = "#ffffff", bg = "#000000", modifiers = ["bold", "italic"] }
+```
+
+where `key` represents what you want to style, `fg` specifies the foreground color, `bg` the background color, and `modifiers` is a list of style modifiers. `bg` and `modifiers` can be omitted to defer to the defaults.
+
+To specify only the foreground color:
+
+```toml
+key = "#ffffff"
+```
+
+if the key contains a dot `'.'`, it must be quoted to prevent it being parsed as a [dotted key](https://toml.io/en/v1.0.0#keys).
+
+```toml
+"key.key" = "#ffffff"
+```
+
+### Color palettes
+
+It's recommended define a palette of named colors, and refer to them from the
+configuration values in your theme. To do this, add a table called
+`palette` to your theme file:
+
+```toml
+ui.background = "white"
+ui.text = "black"
+
+[palette]
+white = "#ffffff"
+black = "#000000"
+```
+
+Remember that the `[palette]` table includes all keys after its header,
+so you should define the palette after normal theme options.
+
+The default palette uses the terminal's default 16 colors, and the colors names
+are listed below. The `[palette]` section in the config file takes precedence
+over it and is merged into the default palette.
+
+| Color Name      |
+| ---             |
+| `black`         |
+| `red`           |
+| `green`         |
+| `yellow`        |
+| `blue`          |
+| `magenta`       |
+| `cyan`          |
+| `gray`          |
+| `light-red`     |
+| `light-green`   |
+| `light-yellow`  |
+| `light-blue`    |
+| `light-magenta` |
+| `light-cyan`    |
+| `light-gray`    |
+| `white`         |
+
+### Modifiers
+
+The following values may be used as modifiers. 
+
+Less common modifiers might not be supported by your terminal emulator.
+
+| Modifier       |
+| ---            |
+| `bold`         |
+| `dim`          |
+| `italic`       |
+| `underlined`   |
+| `slow_blink`   |
+| `rapid_blink`  |
+| `reversed`     |
+| `hidden`       |
+| `crossed_out`  |
+
+### Scopes
+
+The following is a list of scopes available to use for styling.
+
+#### Syntax highlighting
+
+These keys match [tree-sitter scopes](https://tree-sitter.github.io/tree-sitter/syntax-highlighting#theme).
+
+For a given highlight produced, styling will be determined based on the longest matching theme key. For example, the highlight `function.builtin.static` would match the key `function.builtin` rather than `function`.
+
+We use a similar set of scopes as
+[SublimeText](https://www.sublimetext.com/docs/scope_naming.html). See also
+[TextMate](https://macromates.com/manual/en/language_grammars) scopes.
+
+- `type` - Types
+  - `builtin` - Primitive types provided by the language (`int`, `usize`)
+- `constructor`
+
+- `constant` (TODO: constant.other.placeholder for %v)
+  - `builtin` Special constants provided by the language (`true`, `false`, `nil` etc)
+    - `boolean`
+  - `character`
+    - `escape`
+  - `numeric` (numbers)
+    - `integer`
+    - `float`
+
+- `string` (TODO: string.quoted.{single, double}, string.raw/.unquoted)?
+  - `regexp` - Regular expressions
+  - `special`
+    - `path`
+    - `url`
+    - `symbol` - Erlang/Elixir atoms, Ruby symbols, Clojure keywords
+
+- `comment` - Code comments
+  - `line` - Single line comments (`//`)
+  - `block` - Block comments (e.g. (`/*     */`)
+    - `documentation` - Documentation comments (e.g. `///` in Rust)
+
+- `variable` - Variables
+  - `builtin` - Reserved language variables (`self`, `this`, `super`, etc)
+  - `parameter` - Function parameters
+  - `other`
+    - `member` - Fields of composite data types (e.g. structs, unions)
+  - `function` (TODO: ?)
+
+- `label`
+
+- `punctuation`
+  - `delimiter` - Commas, colons
+  - `bracket` - Parentheses, angle brackets, etc.
+
+- `keyword`
+  - `control`
+    - `conditional` - `if`, `else`
+    - `repeat` - `for`, `while`, `loop`
+    - `import` - `import`, `export`
+    - `return`
+  - `operator` - `or`, `in`
+  - `directive` - Preprocessor directives (`#if` in C) 
+  - `function` - `fn`, `func`
+
+- `operator` - `||`, `+=`, `>`
+
+- `function`
+  - `builtin`
+  - `method`
+  - `macro`
+  - `special` (preprocesor in C)
+
+- `tag` - Tags (e.g. `<body>` in HTML)
+
+- `namespace`
+
+- `markup`
+  - `heading`
+  - `list`
+    - `unnumbered`
+    - `numbered`
+  - `bold`
+  - `italic`
+  - `link`
+    - `url`
+    - `label`
+  - `quote`
+  - `raw`
+    - `inline`
+    - `block`
+
+- `diff` - version control changes
+  - `plus` - additions
+  - `minus` - deletions
+  - `delta` - modifications
+    - `moved` - renamed or moved files/changes
+
+#### Interface
+
+These scopes are used for theming the editor interface.
+
+
+| Key                      | Notes                               |
+| ---                      | ---                                 |
+| `ui.background`          |                                     |
+| `ui.cursor`              |                                     |
+| `ui.cursor.insert`       |                                     |
+| `ui.cursor.select`       |                                     |
+| `ui.cursor.match`        | Matching bracket etc.               |
+| `ui.cursor.primary`      | Cursor with primary selection       |
+| `ui.linenr`              |                                     |
+| `ui.linenr.selected`     |                                     |
+| `ui.statusline`          | Statusline                          |
+| `ui.statusline.inactive` | Statusline (unfocused document)     |
+| `ui.popup`               |                                     |
+| `ui.window`              |                                     |
+| `ui.help`                |                                     |
+| `ui.text`                |                                     |
+| `ui.text.focus`          |                                     |
+| `ui.info`                |                                     |
+| `ui.info.text`           |                                     |
+| `ui.menu`                |                                     |
+| `ui.menu.selected`       |                                     |
+| `ui.selection`           | For selections in the editing area  |
+| `ui.selection.primary`   |                                     |
+| `warning`                | Diagnostics warning (gutter)        |
+| `error`                  | Diagnostics error (gutter)          |
+| `info`                   | Diagnostics info (gutter)           |
+| `hint`                   | Diagnostics hint (gutter)           |
+| `diagnostic`             | For text in editing area            |
+

book/src/usage.md

@@ -1 +1,76 @@
 # Usage
+
+(Currently not fully documented, see the [keymappings](./keymap.md) list for more.)
+
+See [tutor.txt](https://github.com/helix-editor/helix/blob/master/runtime/tutor.txt) (accessible via `hx --tutor` or `:tutor`) for a vimtutor-like introduction.
+
+## Registers
+
+Vim-like registers can be used to yank and store text to be pasted later. Usage is similar, with `"` being used to select a register:
+
+- `"ay` - Yank the current selection to register `a`.
+- `"op` - Paste the text in register `o` after the selection.
+
+If there is a selected register before invoking a change or delete command, the selection will be stored in the register and the action will be carried out:
+
+- `"hc` - Store the selection in register `h` and then change it (delete and enter insert mode).
+- `"md` - Store the selection in register `m` and delete it.
+
+### Special Registers
+
+| Register character | Contains              |
+| ---                | ---                   |
+| `/`                | Last search           |
+| `:`                | Last executed command |
+| `"`                | Last yanked text      |
+| `_`                | Black hole            |
+
+> There is no special register for copying to system clipboard, instead special commands and keybindings are provided. See the [keymap](keymap.md#space-mode) for the specifics.
+> The black hole register works as a no-op register, meaning no data will be written to / read from it.
+
+## Surround
+
+Functionality similar to [vim-surround](https://github.com/tpope/vim-surround) is built into
+helix. The keymappings have been inspired from [vim-sandwich](https://github.com/machakann/vim-sandwich):
+
+![surround demo](https://user-images.githubusercontent.com/23398472/122865801-97073180-d344-11eb-8142-8f43809982c6.gif)
+
+- `ms` - Add surround characters
+- `mr` - Replace surround characters
+- `md` - Delete surround characters
+
+`ms` acts on a selection, so select the text first and use `ms<char>`. `mr` and `md` work
+on the closest pairs found and selections are not required; use counts to act in outer pairs.
+
+It can also act on multiple seletions (yay!). For example, to change every occurance of `(use)` to `[use]`:
+
+- `%` to select the whole file
+- `s` to split the selections on a search term
+- Input `use` and hit Enter
+- `mr([` to replace the parens with square brackets
+
+Multiple characters are currently not supported, but planned.
+
+## Textobjects
+
+Currently supported: `word`, `surround`, `function`, `class`, `parameter`.
+
+![textobject-demo](https://user-images.githubusercontent.com/23398472/124231131-81a4bb00-db2d-11eb-9d10-8e577ca7b177.gif)
+![textobject-treesitter-demo](https://user-images.githubusercontent.com/23398472/132537398-2a2e0a54-582b-44ab-a77f-eb818942203d.gif)
+
+- `ma` - Select around the object (`va` in vim, `<alt-a>` in kakoune)
+- `mi` - Select inside the object (`vi` in vim, `<alt-i>` in kakoune)
+
+| Key after `mi` or `ma` | Textobject selected      |
+| ---                    | ---                      |
+| `w`                    | Word                     |
+| `W`                    | WORD                     |
+| `(`, `[`, `'`, etc     | Specified surround pairs |
+| `f`                    | Function                 |
+| `c`                    | Class                    |
+| `p`                    | Parameter                |
+
+Note: `f`, `c`, etc need a tree-sitter grammar active for the current
+document and a special tree-sitter query file to work properly. [Only
+some grammars](https://github.com/search?q=repo%3Ahelix-editor%2Fhelix+filename%3Atextobjects.scm&type=Code&ref=advsearch&l=&l=)
+currently have the query file implemented. Contributions are welcome !

book/theme/css/chrome.css

@@ -196,7 +196,7 @@ a > .hljs {
     border-radius: 3px;
 }
 
-:not(pre):not(a) > .hljs {
+:not(pre):not(a):not(td):not(p) > .hljs {
     color: var(--inline-code-color);
     overflow-x: initial;
 }

book/theme/css/general.css

@@ -114,6 +114,19 @@ h6:target::before {
     margin-bottom: .875em;
 }
 
+.content ul li {
+margin-bottom: .25rem;
+}
+.content ul {
+    list-style-type: square;
+}
+.content ul ul, .content ol ul {
+    margin-bottom: .5rem;
+}
+.content li p {
+    margin-bottom: .5em;
+}
+
 .content p { line-height: 1.45em; }
 .content ol { line-height: 1.45em; }
 .content ul { line-height: 1.45em; }
@@ -149,7 +162,6 @@ table thead td {
 table thead th {
     padding: .75rem;
     text-align: left;
-    color: var(--table-border-color);
     font-weight: 500;
     line-height: 1.5;
     width: auto;
@@ -215,3 +227,7 @@ blockquote *:last-child {
     margin: 5px 0px;
     font-weight: bold;
 }
+
+.result-no-output {
+    font-style: italic;
+}

book/theme/css/variables.css

@@ -13,7 +13,6 @@
 .ayu {
     --bg: hsl(210, 25%, 8%);
     --fg: #c5c5c5;
-    --heading-fg: #c5c5c5;
 
     --sidebar-bg: #14191f;
     --sidebar-fg: #c8c9db;
@@ -54,7 +53,6 @@
 .coal {
     --bg: hsl(200, 7%, 8%);
     --fg: #98a3ad;
-    --heading-fg: #98a3ad;
 
     --sidebar-bg: #292c2f;
     --sidebar-fg: #a1adb8;
@@ -69,7 +67,7 @@
 
     --links: #2b79a2;
 
-    --inline-code-color: #c5c8c6;;
+    --inline-code-color: #c5c8c6;
 
     --theme-popup-bg: #141617;
     --theme-popup-border: #43484d;
@@ -95,7 +93,6 @@
 .light {
     --bg: hsl(0, 0%, 100%);
     --fg: hsl(0, 0%, 0%);
-    --heading-fg: hsl(0, 0%, 0%);
 
     --sidebar-bg: #fafafa;
     --sidebar-fg: hsl(0, 0%, 0%);
@@ -136,7 +133,6 @@
 .navy {
     --bg: hsl(226, 23%, 11%);
     --fg: #bcbdd0;
-    --heading-fg: #bcbdd0;
 
     --sidebar-bg: #282d3f;
     --sidebar-fg: #c8c9db;
@@ -151,7 +147,7 @@
 
     --links: #2b79a2;
 
-    --inline-code-color: #c5c8c6;;
+    --inline-code-color: #c5c8c6;
 
     --theme-popup-bg: #161923;
     --theme-popup-border: #737480;
@@ -177,7 +173,6 @@
 .rust {
     --bg: hsl(60, 9%, 87%);
     --fg: #262625;
-    --heading-fg: #262625;
 
     --sidebar-bg: #3b2e2a;
     --sidebar-fg: #c8c9db;
@@ -218,8 +213,7 @@
 @media (prefers-color-scheme: dark) {
     .light.no-js {
         --bg: hsl(200, 7%, 8%);
-        --fg: #ebeafa;
+        --fg: #98a3ad;
-        --heading-fg: #ebeafa;
 
         --sidebar-bg: #292c2f;
         --sidebar-fg: #a1adb8;
@@ -234,7 +228,7 @@
 
         --links: #2b79a2;
 
-        --inline-code-color: #c5c8c6;;
+        --inline-code-color: #c5c8c6;
 
         --theme-popup-bg: #141617;
         --theme-popup-border: #43484d;
@@ -261,6 +255,7 @@
 .colibri {
     --bg: #3b224c;
     --fg: #bcbdd0;
+    --heading-fg: #fff;
 
     --sidebar-bg: #281733;
     --sidebar-fg: #c8c9db;
@@ -276,18 +271,19 @@
     /* --links: #a4a0e8; */
     --links: #ECCDBA;
 
-    --inline-code-color: #c5c8c6;;
+    --inline-code-color: hsl(48.7, 7.8%, 70%);
 
     --theme-popup-bg: #161923;
     --theme-popup-border: #737480;
     --theme-hover: rgba(0,0,0, .2);
 
-    --quote-bg: hsl(226, 15%, 17%);
+    --quote-bg: #281733;
     --quote-border: hsl(226, 15%, 22%);
 
-    --table-border-color: hsl(226, 23%, 16%);
+    --table-border-color: hsl(226, 23%, 76%);
-    --table-header-bg: hsl(226, 23%, 31%);
+    --table-header-bg: hsla(226, 23%, 31%, 0);
     --table-alternate-bg: hsl(226, 23%, 14%);
+    --table-border-line: hsla(201deg, 20%, 92%, 0.2);
 
     --searchbar-border-color: #aaa;
     --searchbar-bg: #aeaec6;
@@ -300,6 +296,7 @@
 }
 
 .colibri {
+/*
     --bg: #ffffff;
     --fg: #452859;
     --fg: #5a5977;
@@ -318,7 +315,7 @@
 
     --links: #6F44F0;
 
-    --inline-code-color: #697C81;
+    --inline-code-color: #a39e9b;
 
     --theme-popup-bg: #161923;
     --theme-popup-border: #737480;
@@ -341,4 +338,5 @@
     --searchresults-border-color: #5c5c68;
     --searchresults-li-bg: #242430;
     --search-mark-bg: #a2cff5;
+*/
 }

book/theme/highlight.css

@@ -1,83 +1,56 @@
-/*
+pre code.hljs {
- * An increased contrast highlighting scheme loosely based on the
+ display:block;
- * "Base16 Atelier Dune Light" theme by Bram de Haan
+ overflow-x:auto;
- * (http://atelierbram.github.io/syntax-highlighting/atelier-schemes/dune)
+ padding:1em
- * Original Base16 color scheme by Chris Kempson
+}
- * (https://github.com/chriskempson/base16)
+code.hljs {
- */
+ padding:3px 5px
-
+}
-/* Comment */
+.hljs {
+ background:#2f1e2e;
+ color:#a39e9b
+}
 .hljs-comment,
 .hljs-quote {
-  color: #575757;
+ color:#8d8687
 }
-
+.hljs-link,
-/* Red */
+.hljs-meta,
-.hljs-variable,
-.hljs-template-variable,
-.hljs-attribute,
-.hljs-tag,
 .hljs-name,
 .hljs-regexp,
-.hljs-link,
+.hljs-selector-class,
-.hljs-name,
 .hljs-selector-id,
-.hljs-selector-class {
+.hljs-tag,
-  color: #d70025;
+.hljs-template-variable,
+.hljs-variable {
+ color:#ef6155
 }
-
-/* Orange */
-.hljs-number,
-.hljs-meta,
 .hljs-built_in,
-.hljs-builtin-name,
+.hljs-deletion,
 .hljs-literal,
-.hljs-type,
+.hljs-number,
-.hljs-params {
+.hljs-params,
-  color: #b21e00;
+.hljs-type {
+ color:#f99b15
 }
-
+.hljs-attribute,
-/* Green */
+.hljs-section,
+.hljs-title {
+ color:#fec418
+}
+.hljs-addition,
+.hljs-bullet,
 .hljs-string,
-.hljs-symbol,
+.hljs-symbol {
-.hljs-bullet {
+ color:#48b685
-  color: #008200;
 }
-
-/* Blue */
-.hljs-title,
-.hljs-section {
-  color: #0030f2;
-}
-
-/* Purple */
 .hljs-keyword,
 .hljs-selector-tag {
-  color: #9d00ec;
+ color:#815ba4
 }
-
-.hljs {
-  display: block;
-  overflow-x: auto;
-  background: #f6f7f6;
-  color: #000;
-  padding: 0.5em;
-}
-
 .hljs-emphasis {
-  font-style: italic;
+ font-style:italic
 }
-
 .hljs-strong {
-  font-weight: bold;
+ font-weight:700
-}
-
-.hljs-addition {
-  color: #22863a;
-  background-color: #f0fff4;
-}
-
-.hljs-deletion {
-  color: #b31d28;
-  background-color: #ffeef0;
 }

contrib/themes

@@ -0,0 +1 @@
+../runtime/themes

docs/CONTRIBUTING.md

@@ -0,0 +1,37 @@
+# Contributing
+
+Contributors are very welcome! **No contribution is too small and all contributions are valued.**
+
+Some suggestions to get started:
+
+- You can look at the [good first issue][good-first-issue] label on the issue tracker.
+- Help with packaging on various distributions needed!
+- To use print debugging to the [Helix log file][log-file], you must:
+  * Print using `log::info!`, `warn!`, or `error!`. (`log::info!("helix!")`)
+  * Pass the appropriate verbosity level option for the desired log level. (`hx -v <file>` for info, more `v`s for higher severity inclusive)
+- If your preferred language is missing, integrating a tree-sitter grammar for
+    it and defining syntax highlight queries for it is straight forward and
+    doesn't require much knowledge of the internals.
+
+We provide an [architecture.md][architecture.md] that should give you
+a good overview of the internals.
+
+# Auto generated documentation
+
+Some parts of [the book][docs] are autogenerated from the code itself,
+like the list of `:commands` and supported languages. To generate these
+files, run
+
+```shell
+cargo xtask docgen
+```
+
+inside the project. We use [xtask][xtask] as an ad-hoc task runner and
+thus do not require any dependencies other than `cargo` (You don't have
+to `cargo install` anything either).
+
+[good-first-issue]: https://github.com/helix-editor/helix/labels/E-easy
+[log-file]: https://github.com/helix-editor/helix/wiki/FAQ#access-the-log-file
+[architecture.md]: ./architecture.md
+[docs]: https://docs.helix-editor.com/
+[xtask]: https://github.com/matklad/cargo-xtask

docs/vision.md

@@ -0,0 +1,24 @@
+The Helix project still has a ways to go before reaching its goals.  This document outlines some of those goals and the overall vision for the project.
+
+# Vision
+
+An efficient, batteries-included editor you can take anywhere and be productive... if it's your kind of thing.
+
+* **Cross-platform.**  Whether on Linux, Windows, or OSX, you should be able to take your editor with you.
+* **Terminal first.**  Not all environments have a windowing system, and you shouldn't have to abandon your preferred editor in those cases.
+* **Native.**  No Electron or HTML DOM here.  We want an efficient, native-compiled editor that can run with minimal resources when needed.  If you're working on a Raspberry Pi, your editor shouldn't consume half of your RAM.
+* **Batteries included.**  Both the default configuration and bundled features should be enough to have a good editing experience and be productive.  You shouldn't need a massive custom config or external executables and plugins for basic features and functionality.
+* **Don't try to be everything for everyone.**  There are many great editors out there to choose from.  Let's make Helix *one of* those great options, with its own take on things.
+
+# Goals
+
+Vision statements are all well and good, but are also vague and subjective.  Here is a (non-exhaustive) list of some of Helix's more concrete goals, to help give a clearer idea of the project's direction:
+
+* **Modal.**  Vim is a great idea.
+* **Selection -> Action**, not Verb -> Object.  Interaction models aren't linguistics, and "selection first" lets you see what you're doing (among other benefits).
+* **We aren't playing code golf.**  It's more important for the keymap to be consistent and easy to memorize than it is to save a key stroke or two when editing.
+* **Built-in tools** for working with code bases efficiently.  Most projects aren't a single file, and an editor should handle that as a first-class use case.  In Helix's case, this means (among other things) a fuzzy-search file navigator and LSP support.
+* **Edit anything** that comes up when coding, within reason.  Whether it's a 200 MB XML file, a megabyte of minified javascript on a single line, or Japanese text encoded in ShiftJIS, you should be able to open it and edit it without problems.  (Note: this doesn't mean handle every esoteric use case.  Sometimes you do just need a specialized tool, and Helix isn't that.)
+* **Configurable**, within reason.  Although the defaults should be good, not everyone will agree on what "good" is.  Within the bounds of Helix's core interaction models, it should be reasonably configurable so that it can be "good" for more people.  This means, for example, custom key maps among other things.
+* **Extensible**, within reason.  Although we want Helix to be productive out-of-the-box, it's not practical or desirable to cram every useful feature and use case into the core editor.  The basics should be built-in, but you should be able to extend it with additional functionality as needed.  Right now we're thinking Wasm-based plugins.
+* **Clean code base.**  Sometimes other factors (e.g. significant performance gains, important features, correctness, etc.) will trump strict readability, but we nevertheless want to keep the code base straightforward and easy to understand to the extent we can.

flake.lock

@@ -1,12 +1,27 @@
 {
   "nodes": {
+    "devshell": {
+      "locked": {
+        "lastModified": 1639692811,
+        "narHash": "sha256-wOOBH0fVsfNqw/5ZWRoKspyesoXBgiwEOUBH4c7JKEo=",
+        "owner": "numtide",
+        "repo": "devshell",
+        "rev": "d3a1f5bec3632b33346865b1c165bf2420bb2f52",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "devshell",
+        "type": "github"
+      }
+    },
     "flake-utils": {
       "locked": {
-        "lastModified": 1620759905,
+        "lastModified": 1637014545,
-        "narHash": "sha256-WiyWawrgmyN0EdmiHyG2V+fqReiVi8bM9cRdMaKQOFg=",
+        "narHash": "sha256-26IZAc5yzlD9FlDT54io1oqG/bBoyka+FJk5guaX4x4=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "b543720b25df6ffdfcf9227afafc5b8c1fabfae8",
+        "rev": "bba5dcc8e0b20ab664967ad83d24d64cb64ec4f4",
         "type": "github"
       },
       "original": {
@@ -15,59 +30,37 @@
         "type": "github"
       }
     },
-    "flake-utils_2": {
+    "nixCargoIntegration": {
-      "locked": {
-        "lastModified": 1614513358,
-        "narHash": "sha256-LakhOx3S1dRjnh0b5Dg3mbZyH0ToC9I8Y2wKSkBaTzU=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "5466c5bbece17adaab2d82fae80b46e807611bf3",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
-    "naersk": {
       "inputs": {
-        "nixpkgs": "nixpkgs"
+        "devshell": "devshell",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rustOverlay": [
+          "rust-overlay"
+        ]
       },
       "locked": {
-        "lastModified": 1620316130,
+        "lastModified": 1639807801,
-        "narHash": "sha256-sU0VS5oJS1FsHsZsLELAXc7G2eIelVuucRw+q5B1x9k=",
+        "narHash": "sha256-y32tMq1LTRVbMW3QN5i98iOQjQt2QSsif3ayUkD1o3g=",
-        "owner": "nmattia",
+        "owner": "yusdacra",
-        "repo": "naersk",
+        "repo": "nix-cargo-integration",
-        "rev": "a3f40fe42cc6d267ff7518fa3199e99ff1444ac4",
+        "rev": "b5bbaa4f5239e6f0619846f9a5380f07baa853d3",
         "type": "github"
       },
       "original": {
-        "owner": "nmattia",
+        "owner": "yusdacra",
-        "repo": "naersk",
+        "repo": "nix-cargo-integration",
         "type": "github"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1622059058,
+        "lastModified": 1639699734,
-        "narHash": "sha256-t1/ZMtyxClVSfcV4Pt5C1YpkeJ/UwFF3oitLD7Ch/UA=",
+        "narHash": "sha256-tlX6WebGmiHb2Hmniff+ltYp+7dRfdsBxw9YczLsP60=",
-        "path": "/nix/store/2gam4i1fa1v19k3n5rc9vgvqac1c2xj5-source",
-        "rev": "84aa23742f6c72501f9cc209f29c438766f5352d",
-        "type": "path"
-      },
-      "original": {
-        "id": "nixpkgs",
-        "type": "indirect"
-      }
-    },
-    "nixpkgs_2": {
-      "locked": {
-        "lastModified": 1622194753,
-        "narHash": "sha256-76qtvFp/vFEz46lz5iZMJ0mnsWQYmuGYlb0fHgKqqMg=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "540dccb2aeaffa9dc69bfdc41c55abd7ccc6baa3",
+        "rev": "03ec468b14067729a285c2c7cfa7b9434a04816c",
         "type": "github"
       },
       "original": {
@@ -77,40 +70,40 @@
         "type": "github"
       }
     },
-    "nixpkgs_3": {
+    "nixpkgs_2": {
       "locked": {
-        "lastModified": 1617325113,
+        "lastModified": 1637453606,
-        "narHash": "sha256-GksR0nvGxfZ79T91UUtWjjccxazv6Yh/MvEJ82v1Xmw=",
+        "narHash": "sha256-Gy6cwUswft9xqsjWxFYEnx/63/qzaFUwatcbV5GF/GQ=",
-        "owner": "nixos",
+        "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "54c1e44240d8a527a8f4892608c4bce5440c3ecb",
+        "rev": "8afc4e543663ca0a6a4f496262cd05233737e732",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
     "root": {
       "inputs": {
-        "flake-utils": "flake-utils",
+        "nixCargoIntegration": "nixCargoIntegration",
-        "naersk": "naersk",
+        "nixpkgs": "nixpkgs",
-        "nixpkgs": "nixpkgs_2",
         "rust-overlay": "rust-overlay"
       }
     },
     "rust-overlay": {
       "inputs": {
-        "flake-utils": "flake-utils_2",
+        "flake-utils": "flake-utils",
-        "nixpkgs": "nixpkgs_3"
+        "nixpkgs": "nixpkgs_2"
       },
       "locked": {
-        "lastModified": 1622257069,
+        "lastModified": 1639880499,
-        "narHash": "sha256-+QVnS/es9JCRZXphoHL0fOIUhpGqB4/wreBsXWArVck=",
+        "narHash": "sha256-/BibDmFwgWuuTUkNVO6YlvuTSWM9dpBvlZoTAPs7ORI=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "8aa5f93c0b665e5357af19c5631a3450bff4aba5",
+        "rev": "c6c83589ae048af20d93d01eb07a4176012093d0",
         "type": "github"
       },
       "original": {

flake.nix

@@ -3,31 +3,69 @@
 
   inputs = {
     nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
-    flake-utils.url = "github:numtide/flake-utils";
     rust-overlay.url = "github:oxalica/rust-overlay";
-    naersk.url = "github:nmattia/naersk";
+    nixCargoIntegration = {
+      url = "github:yusdacra/nix-cargo-integration";
+      inputs.nixpkgs.follows = "nixpkgs";
+      inputs.rustOverlay.follows = "rust-overlay";
+    };
   };
 
-  outputs = inputs@{ self, nixpkgs, naersk, rust-overlay, flake-utils, ... }:
+  outputs = inputs@{ self, nixCargoIntegration, ... }:
-    flake-utils.lib.eachDefaultSystem (system:
+    nixCargoIntegration.lib.makeOutputs {
-      let
+      root = ./.;
-        pkgs = import nixpkgs { inherit system; overlays = [ rust-overlay.overlay ]; };
+      buildPlatform = "crate2nix";
-        rust = (pkgs.rustChannelOf {
+      renameOutputs = { "helix-term" = "helix"; };
-            date = "2021-05-01";
+      # Set default app to hx (binary is from helix-term release build)
-            channel = "nightly";
+      # Set default package to helix-term release build
-        }).minimal; # cargo, rustc and rust-std
+      defaultOutputs = { app = "hx"; package = "helix"; };
-        naerskLib = naersk.lib."${system}".override {
+      overrides = {
-          # naersk can't build with stable?!
+        crateOverrides = common: _: {
-          # inherit (pkgs.rust-bin.stable.latest) rustc cargo;
+          helix-term = prev: {
-          rustc = rust;
+            # link languages and theme toml files since helix-term expects them (for tests)
-          cargo = rust;
+            preConfigure = "ln -s ${common.root}/{languages.toml,theme.toml} ..";
+            buildInputs = (prev.buildInputs or [ ]) ++ [ common.cCompiler.cc.lib ];
+          };
+          # link languages and theme toml files since helix-view expects them
+          helix-view = _: { preConfigure = "ln -s ${common.root}/{languages.toml,theme.toml} .."; };
+          helix-syntax = _prev: {
+            preConfigure = "mkdir -p ../runtime/grammars";
+            postInstall = "cp -r ../runtime $out/runtime";
+          };
         };
-      in rec {
+        mainBuild = common: prev:
-        packages.helix = naerskLib.buildPackage {
+          let
-          pname = "helix";
+            inherit (common) pkgs lib;
-          root = ./.;
+            helixSyntax = lib.buildCrate {
+              root = self;
+              memberName = "helix-syntax";
+              defaultCrateOverrides = {
+                helix-syntax = common.crateOverrides.helix-syntax;
+              };
+              release = false;
+            };
+            runtimeDir = pkgs.runCommand "helix-runtime" { } ''
+              mkdir -p $out
+              ln -s ${common.root}/runtime/* $out
+              ln -sf ${helixSyntax}/runtime/grammars $out
+            '';
+          in
+          lib.optionalAttrs (common.memberName == "helix-term") {
+            nativeBuildInputs = [ pkgs.makeWrapper ];
+            postFixup = ''
+              if [ -f "$out/bin/hx" ]; then
+                wrapProgram "$out/bin/hx" --set HELIX_RUNTIME "${runtimeDir}"
+              fi
+            '';
+          };
+        shell = common: prev: {
+          packages = prev.packages ++ (with common.pkgs; [ lld_13 lldb cargo-tarpaulin ]);
+          env = prev.env ++ [
+            { name = "HELIX_RUNTIME"; eval = "$PWD/runtime"; }
+            { name = "RUST_BACKTRACE"; value = "1"; }
+            { name = "RUSTFLAGS"; value = "-C link-arg=-fuse-ld=lld -C target-cpu=native"; }
+          ];
         };
-        defaultPackage = packages.helix;
+      };
-        devShell = pkgs.callPackage ./shell.nix {};
+    };
-      });
 }

helix-core/Cargo.toml

@@ -1,31 +1,43 @@
 [package]
 name = "helix-core"
-version = "0.0.10"
+version = "0.6.0"
 authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
-edition = "2018"
+edition = "2021"
 license = "MPL-2.0"
-
+description = "Helix editor core editing primitives"
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+categories = ["editor"]
-
+repository = "https://github.com/helix-editor/helix"
+homepage = "https://helix-editor.com"
+include = ["src/**/*", "README.md"]
 
 [features]
-embed_runtime = ["rust-embed"]
 
 [dependencies]
-helix-syntax = { path = "../helix-syntax" }
+helix-syntax = { version = "0.6", path = "../helix-syntax" }
 
-ropey = "1.2"
+ropey = "1.3"
-smallvec = "1.4"
+smallvec = "1.7"
 tendril = "0.4.2"
-unicode-segmentation = "1.6"
+unicode-segmentation = "1.8"
 unicode-width = "0.1"
+unicode-general-category = "0.4"
 # slab = "0.4.2"
-tree-sitter = "0.19"
+tree-sitter = "0.20"
-once_cell = "1.4"
+once_cell = "1.9"
+arc-swap = "1"
 regex = "1"
 
+log = "0.4"
 serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
 toml = "0.5"
 
+similar = "2.1"
+
 etcetera = "0.3"
-rust-embed = { version = "5.9.0", optional = true }
+encoding_rs = "0.8"
+
+chrono = { version = "0.4", default-features = false, features = ["alloc", "std"] }
+
+[dev-dependencies]
+quickcheck = { version = "1", default-features = false }

helix-core/src/auto_pairs.rs

@@ -1,4 +1,8 @@
-use crate::{Range, Rope, Selection, Tendril, Transaction};
+//! When typing the opening character of one of the possible pairs defined below,
+//! this module provides the functionality to insert the paired closing character.
+
+use crate::{movement::Direction, Range, Rope, Selection, Tendril, Transaction};
+use log::debug;
 use smallvec::SmallVec;
 
 // Heavily based on https://github.com/codemirror/closebrackets/
@@ -12,7 +16,9 @@ pub const PAIRS: &[(char, char)] = &[
     ('`', '`'),
 ];
 
-const CLOSE_BEFORE: &str = ")]}'\":;> \n"; // includes space and newline
+// [TODO] build this dynamically in language config. see #992
+const OPEN_BEFORE: &str = "([{'\":;,> \n\r\u{000B}\u{000C}\u{0085}\u{2028}\u{2029}";
+const CLOSE_BEFORE: &str = ")]}'\":;,> \n\r\u{000B}\u{000C}\u{0085}\u{2028}\u{2029}"; // includes space and newlines
 
 // insert hook:
 // Fn(doc, selection, char) => Option<Transaction>
@@ -22,14 +28,19 @@ const CLOSE_BEFORE: &str = ")]}'\":;> \n"; // includes space and newline
 //
 // to simplify, maybe return Option<Transaction> and just reimplement the default
 
-// TODO: delete implementation where it erases the whole bracket (|) -> |
+// [TODO]
+// * delete implementation where it erases the whole bracket (|) -> |
+// * change to multi character pairs to handle cases like placing the cursor in the
+//   middle of triple quotes, and more exotic pairs like Jinja's {% %}
 
 #[must_use]
 pub fn hook(doc: &Rope, selection: &Selection, ch: char) -> Option<Transaction> {
+    debug!("autopairs hook selection: {:#?}", selection);
+
     for &(open, close) in PAIRS {
         if open == ch {
             if open == close {
-                return handle_same(doc, selection, open);
+                return Some(handle_same(doc, selection, open, CLOSE_BEFORE, OPEN_BEFORE));
             } else {
                 return Some(handle_open(doc, selection, open, close, CLOSE_BEFORE));
             }
@@ -44,18 +55,44 @@ pub fn hook(doc: &Rope, selection: &Selection, ch: char) -> Option<Transaction>
     None
 }
 
-// TODO: special handling for lifetimes in rust: if preceeded by & or < don't auto close '
+fn prev_char(doc: &Rope, pos: usize) -> Option<char> {
-// for example "&'a mut", or "fn<'a>"
+    if pos == 0 {
-
-fn next_char(doc: &Rope, pos: usize) -> Option<char> {
-    if pos >= doc.len_chars() {
         return None;
     }
-    Some(doc.char(pos))
-}
-// TODO: selections should be extended if range, moved if point.
 
-// TODO: if not cursor but selection, wrap on both sides of selection (surround)
+    doc.get_char(pos - 1)
+}
+
+/// calculate what the resulting range should be for an auto pair insertion
+fn get_next_range(
+    start_range: &Range,
+    offset: usize,
+    typed_char: char,
+    len_inserted: usize,
+) -> Range {
+    let end_head = start_range.head + offset + typed_char.len_utf8();
+
+    let end_anchor = match (start_range.len(), start_range.direction()) {
+        // if we have a zero width cursor, it shifts to the same number
+        (0, _) => end_head,
+
+        // if we are inserting for a regular one-width cursor, the anchor
+        // moves with the head
+        (1, Direction::Forward) => end_head - 1,
+        (1, Direction::Backward) => end_head + 1,
+
+        // if we are appending, the anchor stays where it is; only offset
+        // for multiple range insertions
+        (_, Direction::Forward) => start_range.anchor + offset,
+
+        // when we are inserting in front of a selection, we need to move
+        // the anchor over by however many characters were inserted overall
+        (_, Direction::Backward) => start_range.anchor + offset + len_inserted,
+    };
+
+    Range::new(end_anchor, end_head)
+}
+
 fn handle_open(
     doc: &Rope,
     selection: &Selection,
@@ -63,97 +100,510 @@ fn handle_open(
     close: char,
     close_before: &str,
 ) -> Transaction {
-    let mut ranges = SmallVec::with_capacity(selection.len());
+    let mut end_ranges = SmallVec::with_capacity(selection.len());
-
     let mut offs = 0;
 
-    let mut transaction = Transaction::change_by_selection(doc, selection, |range| {
+    let transaction = Transaction::change_by_selection(doc, selection, |start_range| {
-        let pos = range.head;
+        let cursor = start_range.cursor(doc.slice(..));
-        let next = next_char(doc, pos);
+        let next_char = doc.get_char(cursor);
+        let len_inserted;
 
-        let head = pos + offs + open.len_utf8();
+        let change = match next_char {
-        // if selection, retain anchor, if cursor, move over
-        ranges.push(Range::new(
-            if range.is_empty() {
-                head
-            } else {
-                range.anchor + offs
-            },
-            head,
-        ));
-
-        match next {
             Some(ch) if !close_before.contains(ch) => {
-                // TODO: else return (use default handler that inserts open)
+                len_inserted = open.len_utf8();
-                (pos, pos, Some(Tendril::from_char(open)))
+                (cursor, cursor, Some(Tendril::from_char(open)))
             }
             // None | Some(ch) if close_before.contains(ch) => {}
             _ => {
                 // insert open & close
-                let mut pair = Tendril::with_capacity(2);
+                let pair = Tendril::from_iter([open, close]);
-                pair.push_char(open);
+                len_inserted = open.len_utf8() + close.len_utf8();
-                pair.push_char(close);
+                (cursor, cursor, Some(pair))
-
-                offs += 2;
-
-                (pos, pos, Some(pair))
             }
-        }
+        };
+
+        let next_range = get_next_range(start_range, offs, open, len_inserted);
+        end_ranges.push(next_range);
+        offs += len_inserted;
+
+        change
     });
 
-    transaction.with_selection(Selection::new(ranges, selection.primary_index()))
+    let t = transaction.with_selection(Selection::new(end_ranges, selection.primary_index()));
+    debug!("auto pair transaction: {:#?}", t);
+    t
 }
 
 fn handle_close(doc: &Rope, selection: &Selection, _open: char, close: char) -> Transaction {
-    let mut ranges = SmallVec::with_capacity(selection.len());
+    let mut end_ranges = SmallVec::with_capacity(selection.len());
 
     let mut offs = 0;
 
-    let mut transaction = Transaction::change_by_selection(doc, selection, |range| {
+    let transaction = Transaction::change_by_selection(doc, selection, |start_range| {
-        let pos = range.head;
+        let cursor = start_range.cursor(doc.slice(..));
-        let next = next_char(doc, pos);
+        let next_char = doc.get_char(cursor);
+        let mut len_inserted = 0;
 
-        let head = pos + offs + close.len_utf8();
+        let change = if next_char == Some(close) {
-        // if selection, retain anchor, if cursor, move over
+            // return transaction that moves past close
-        ranges.push(Range::new(
+            (cursor, cursor, None) // no-op
-            if range.is_empty() {
-                head
-            } else {
-                range.anchor + offs
-            },
-            head,
-        ));
-
-        if next == Some(close) {
-            //  return transaction that moves past close
-            (pos, pos, None) // no-op
         } else {
-            offs += close.len_utf8();
+            len_inserted += close.len_utf8();
+            (cursor, cursor, Some(Tendril::from_char(close)))
+        };
 
-            // TODO: else return (use default handler that inserts close)
+        let next_range = get_next_range(start_range, offs, close, len_inserted);
-            (pos, pos, Some(Tendril::from_char(close)))
+        end_ranges.push(next_range);
-        }
+        offs += len_inserted;
+
+        change
     });
 
-    transaction.with_selection(Selection::new(ranges, selection.primary_index()))
+    let t = transaction.with_selection(Selection::new(end_ranges, selection.primary_index()));
+    debug!("auto pair transaction: {:#?}", t);
+    t
 }
 
-// handle cases where open and close is the same, or in triples ("""docstring""")
+/// handle cases where open and close is the same, or in triples ("""docstring""")
-fn handle_same(doc: &Rope, selection: &Selection, token: char) -> Option<Transaction> {
+fn handle_same(
-    // if not cursor but selection, wrap
+    doc: &Rope,
-    // let next = next char
+    selection: &Selection,
+    token: char,
+    close_before: &str,
+    open_before: &str,
+) -> Transaction {
+    let mut end_ranges = SmallVec::with_capacity(selection.len());
 
-    // if next == bracket {
+    let mut offs = 0;
-    //   // if start of syntax node, insert token twice (new pair because node is complete)
+
-    //   // elseif colsedBracketAt
+    let transaction = Transaction::change_by_selection(doc, selection, |start_range| {
-    //      // is_triple == allow triple && next 3 is equal
+        let cursor = start_range.cursor(doc.slice(..));
-    //      // cursor jump over
+        let mut len_inserted = 0;
-    // }
+
-    //} else if allow_triple && followed by triple {
+        let next_char = doc.get_char(cursor);
-    //}
+        let prev_char = prev_char(doc, cursor);
-    //} else if next != word char && prev != bracket && prev != word char {
+
-    // // condition checks for cases like I' where you don't want I'' (or I'm)
+        let change = if next_char == Some(token) {
-    //  insert pair ("")
+            //  return transaction that moves past close
-    //}
+            (cursor, cursor, None) // no-op
-    None
+        } else {
+            let mut pair = Tendril::with_capacity(2 * token.len_utf8() as u32);
+            pair.push_char(token);
+
+            // for equal pairs, don't insert both open and close if either
+            // side has a non-pair char
+            if (next_char.is_none() || close_before.contains(next_char.unwrap()))
+                && (prev_char.is_none() || open_before.contains(prev_char.unwrap()))
+            {
+                pair.push_char(token);
+            }
+
+            len_inserted += pair.len();
+            (cursor, cursor, Some(pair))
+        };
+
+        let next_range = get_next_range(start_range, offs, token, len_inserted);
+        end_ranges.push(next_range);
+        offs += len_inserted;
+
+        change
+    });
+
+    let t = transaction.with_selection(Selection::new(end_ranges, selection.primary_index()));
+    debug!("auto pair transaction: {:#?}", t);
+    t
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use smallvec::smallvec;
+
+    fn differing_pairs() -> impl Iterator<Item = &'static (char, char)> {
+        PAIRS.iter().filter(|(open, close)| open != close)
+    }
+
+    fn matching_pairs() -> impl Iterator<Item = &'static (char, char)> {
+        PAIRS.iter().filter(|(open, close)| open == close)
+    }
+
+    fn test_hooks(
+        in_doc: &Rope,
+        in_sel: &Selection,
+        ch: char,
+        expected_doc: &Rope,
+        expected_sel: &Selection,
+    ) {
+        let trans = hook(&in_doc, &in_sel, ch).unwrap();
+        let mut actual_doc = in_doc.clone();
+        assert!(trans.apply(&mut actual_doc));
+        assert_eq!(expected_doc, &actual_doc);
+        assert_eq!(expected_sel, trans.selection().unwrap());
+    }
+
+    fn test_hooks_with_pairs<I, F, R>(
+        in_doc: &Rope,
+        in_sel: &Selection,
+        pairs: I,
+        get_expected_doc: F,
+        actual_sel: &Selection,
+    ) where
+        I: IntoIterator<Item = &'static (char, char)>,
+        F: Fn(char, char) -> R,
+        R: Into<Rope>,
+        Rope: From<R>,
+    {
+        pairs.into_iter().for_each(|(open, close)| {
+            test_hooks(
+                in_doc,
+                in_sel,
+                *open,
+                &Rope::from(get_expected_doc(*open, *close)),
+                actual_sel,
+            )
+        });
+    }
+
+    // [] indicates range
+
+    /// [] -> insert ( -> ([])
+    #[test]
+    fn test_insert_blank() {
+        test_hooks_with_pairs(
+            &Rope::new(),
+            &Selection::single(1, 0),
+            PAIRS,
+            |open, close| format!("{}{}", open, close),
+            &Selection::single(2, 1),
+        );
+    }
+
+    /// [] -> append ( -> ([])
+    #[test]
+    fn test_append_blank() {
+        test_hooks_with_pairs(
+            // this is what happens when you have a totally blank document and then append
+            &Rope::from("\n\n"),
+            &Selection::single(0, 2),
+            PAIRS,
+            |open, close| format!("\n{}{}\n", open, close),
+            &Selection::single(0, 3),
+        );
+    }
+
+    /// []              ([])
+    /// [] -> insert -> ([])
+    /// []              ([])
+    #[test]
+    fn test_insert_blank_multi_cursor() {
+        test_hooks_with_pairs(
+            &Rope::from("\n\n\n"),
+            &Selection::new(
+                smallvec!(Range::new(1, 0), Range::new(2, 1), Range::new(3, 2),),
+                0,
+            ),
+            PAIRS,
+            |open, close| {
+                format!(
+                    "{open}{close}\n{open}{close}\n{open}{close}\n",
+                    open = open,
+                    close = close
+                )
+            },
+            &Selection::new(
+                smallvec!(Range::new(2, 1), Range::new(5, 4), Range::new(8, 7),),
+                0,
+            ),
+        );
+    }
+
+    /// fo[o] -> append ( -> fo[o(])
+    #[test]
+    fn test_append() {
+        test_hooks_with_pairs(
+            &Rope::from("foo\n"),
+            &Selection::single(2, 4),
+            differing_pairs(),
+            |open, close| format!("foo{}{}\n", open, close),
+            &Selection::single(2, 5),
+        );
+    }
+
+    /// fo[o]                fo[o(])
+    /// fo[o] -> append ( -> fo[o(])
+    /// fo[o]                fo[o(])
+    #[test]
+    fn test_append_multi() {
+        test_hooks_with_pairs(
+            &Rope::from("foo\nfoo\nfoo\n"),
+            &Selection::new(
+                smallvec!(Range::new(2, 4), Range::new(6, 8), Range::new(10, 12)),
+                0,
+            ),
+            differing_pairs(),
+            |open, close| {
+                format!(
+                    "foo{open}{close}\nfoo{open}{close}\nfoo{open}{close}\n",
+                    open = open,
+                    close = close
+                )
+            },
+            &Selection::new(
+                smallvec!(Range::new(2, 5), Range::new(8, 11), Range::new(14, 17)),
+                0,
+            ),
+        );
+    }
+
+    /// ([]) -> insert ) -> ()[]
+    #[test]
+    fn test_insert_close_inside_pair() {
+        for (open, close) in PAIRS {
+            let doc = Rope::from(format!("{}{}", open, close));
+
+            test_hooks(
+                &doc,
+                &Selection::single(2, 1),
+                *close,
+                &doc,
+                &Selection::single(3, 2),
+            );
+        }
+    }
+
+    /// [(]) -> append ) -> [()]
+    #[test]
+    fn test_append_close_inside_pair() {
+        for (open, close) in PAIRS {
+            let doc = Rope::from(format!("{}{}\n", open, close));
+
+            test_hooks(
+                &doc,
+                &Selection::single(0, 2),
+                *close,
+                &doc,
+                &Selection::single(0, 3),
+            );
+        }
+    }
+
+    /// ([])                ()[]
+    /// ([]) -> insert ) -> ()[]
+    /// ([])                ()[]
+    #[test]
+    fn test_insert_close_inside_pair_multi_cursor() {
+        let sel = Selection::new(
+            smallvec!(Range::new(2, 1), Range::new(5, 4), Range::new(8, 7),),
+            0,
+        );
+
+        let expected_sel = Selection::new(
+            smallvec!(Range::new(3, 2), Range::new(6, 5), Range::new(9, 8),),
+            0,
+        );
+
+        for (open, close) in PAIRS {
+            let doc = Rope::from(format!(
+                "{open}{close}\n{open}{close}\n{open}{close}\n",
+                open = open,
+                close = close
+            ));
+
+            test_hooks(&doc, &sel, *close, &doc, &expected_sel);
+        }
+    }
+
+    /// [(])                [()]
+    /// [(]) -> append ) -> [()]
+    /// [(])                [()]
+    #[test]
+    fn test_append_close_inside_pair_multi_cursor() {
+        let sel = Selection::new(
+            smallvec!(Range::new(0, 2), Range::new(3, 5), Range::new(6, 8),),
+            0,
+        );
+
+        let expected_sel = Selection::new(
+            smallvec!(Range::new(0, 3), Range::new(3, 6), Range::new(6, 9),),
+            0,
+        );
+
+        for (open, close) in PAIRS {
+            let doc = Rope::from(format!(
+                "{open}{close}\n{open}{close}\n{open}{close}\n",
+                open = open,
+                close = close
+            ));
+
+            test_hooks(&doc, &sel, *close, &doc, &expected_sel);
+        }
+    }
+
+    /// ([]) -> insert ( -> (([]))
+    #[test]
+    fn test_insert_open_inside_pair() {
+        let sel = Selection::single(2, 1);
+        let expected_sel = Selection::single(3, 2);
+
+        for (open, close) in differing_pairs() {
+            let doc = Rope::from(format!("{}{}", open, close));
+            let expected_doc = Rope::from(format!(
+                "{open}{open}{close}{close}",
+                open = open,
+                close = close
+            ));
+
+            test_hooks(&doc, &sel, *open, &expected_doc, &expected_sel);
+        }
+    }
+
+    /// [word(]) -> append ( -> [word((]))
+    #[test]
+    fn test_append_open_inside_pair() {
+        let sel = Selection::single(0, 6);
+        let expected_sel = Selection::single(0, 7);
+
+        for (open, close) in differing_pairs() {
+            let doc = Rope::from(format!("word{}{}", open, close));
+            let expected_doc = Rope::from(format!(
+                "word{open}{open}{close}{close}",
+                open = open,
+                close = close
+            ));
+
+            test_hooks(&doc, &sel, *open, &expected_doc, &expected_sel);
+        }
+    }
+
+    /// ([]) -> insert " -> ("[]")
+    #[test]
+    fn test_insert_nested_open_inside_pair() {
+        let sel = Selection::single(2, 1);
+        let expected_sel = Selection::single(3, 2);
+
+        for (outer_open, outer_close) in differing_pairs() {
+            let doc = Rope::from(format!("{}{}", outer_open, outer_close,));
+
+            for (inner_open, inner_close) in matching_pairs() {
+                let expected_doc = Rope::from(format!(
+                    "{}{}{}{}",
+                    outer_open, inner_open, inner_close, outer_close
+                ));
+
+                test_hooks(&doc, &sel, *inner_open, &expected_doc, &expected_sel);
+            }
+        }
+    }
+
+    /// [(]) -> append " -> [("]")
+    #[test]
+    fn test_append_nested_open_inside_pair() {
+        let sel = Selection::single(0, 2);
+        let expected_sel = Selection::single(0, 3);
+
+        for (outer_open, outer_close) in differing_pairs() {
+            let doc = Rope::from(format!("{}{}", outer_open, outer_close,));
+
+            for (inner_open, inner_close) in matching_pairs() {
+                let expected_doc = Rope::from(format!(
+                    "{}{}{}{}",
+                    outer_open, inner_open, inner_close, outer_close
+                ));
+
+                test_hooks(&doc, &sel, *inner_open, &expected_doc, &expected_sel);
+            }
+        }
+    }
+
+    /// []word -> insert ( -> ([]word
+    #[test]
+    fn test_insert_open_before_non_pair() {
+        test_hooks_with_pairs(
+            &Rope::from("word"),
+            &Selection::single(1, 0),
+            PAIRS,
+            |open, _| format!("{}word", open),
+            &Selection::single(2, 1),
+        )
+    }
+
+    /// [wor]d -> insert ( -> ([wor]d
+    #[test]
+    fn test_insert_open_with_selection() {
+        test_hooks_with_pairs(
+            &Rope::from("word"),
+            &Selection::single(3, 0),
+            PAIRS,
+            |open, _| format!("{}word", open),
+            &Selection::single(4, 1),
+        )
+    }
+
+    /// [wor]d -> append ) -> [wor)]d
+    #[test]
+    fn test_append_close_inside_non_pair_with_selection() {
+        let sel = Selection::single(0, 4);
+        let expected_sel = Selection::single(0, 5);
+
+        for (_, close) in PAIRS {
+            let doc = Rope::from("word");
+            let expected_doc = Rope::from(format!("wor{}d", close));
+            test_hooks(&doc, &sel, *close, &expected_doc, &expected_sel);
+        }
+    }
+
+    /// foo[ wor]d -> insert ( -> foo([) wor]d
+    #[test]
+    fn test_insert_open_trailing_word_with_selection() {
+        test_hooks_with_pairs(
+            &Rope::from("foo word"),
+            &Selection::single(7, 3),
+            differing_pairs(),
+            |open, close| format!("foo{}{} word", open, close),
+            &Selection::single(9, 4),
+        )
+    }
+
+    /// we want pairs that are *not* the same char to be inserted after
+    /// a non-pair char, for cases like functions, but for pairs that are
+    /// the same char, we want to *not* insert a pair to handle cases like "I'm"
+    ///
+    /// word[]  -> insert ( -> word([])
+    /// word[]  -> insert ' -> word'[]
+    #[test]
+    fn test_insert_open_after_non_pair() {
+        let doc = Rope::from("word");
+        let sel = Selection::single(5, 4);
+        let expected_sel = Selection::single(6, 5);
+
+        test_hooks_with_pairs(
+            &doc,
+            &sel,
+            differing_pairs(),
+            |open, close| format!("word{}{}", open, close),
+            &expected_sel,
+        );
+
+        test_hooks_with_pairs(
+            &doc,
+            &sel,
+            matching_pairs(),
+            |open, _| format!("word{}", open),
+            &expected_sel,
+        );
+    }
+
+    /// appending with only a cursor should stay a cursor
+    ///
+    /// [] -> append to end "foo -> "foo[]"
+    #[test]
+    fn test_append_single_cursor() {
+        test_hooks_with_pairs(
+            &Rope::from("\n"),
+            &Selection::single(0, 1),
+            PAIRS,
+            |open, close| format!("{}{}\n", open, close),
+            &Selection::single(1, 2),
+        );
+    }
 }

helix-core/src/chars.rs

@@ -0,0 +1,135 @@
+//! Utility functions to categorize a `char`.
+
+use crate::LineEnding;
+
+#[derive(Debug, Eq, PartialEq)]
+pub enum CharCategory {
+    Whitespace,
+    Eol,
+    Word,
+    Punctuation,
+    Unknown,
+}
+
+#[inline]
+pub fn categorize_char(ch: char) -> CharCategory {
+    if char_is_line_ending(ch) {
+        CharCategory::Eol
+    } else if ch.is_whitespace() {
+        CharCategory::Whitespace
+    } else if char_is_word(ch) {
+        CharCategory::Word
+    } else if char_is_punctuation(ch) {
+        CharCategory::Punctuation
+    } else {
+        CharCategory::Unknown
+    }
+}
+
+/// Determine whether a character is a line ending.
+#[inline]
+pub fn char_is_line_ending(ch: char) -> bool {
+    LineEnding::from_char(ch).is_some()
+}
+
+/// Determine whether a character qualifies as (non-line-break)
+/// whitespace.
+#[inline]
+pub fn char_is_whitespace(ch: char) -> bool {
+    // TODO: this is a naive binary categorization of whitespace
+    // characters.  For display, word wrapping, etc. we'll need a better
+    // categorization based on e.g. breaking vs non-breaking spaces
+    // and whether they're zero-width or not.
+    match ch {
+        //'\u{1680}' | // Ogham Space Mark (here for completeness, but usually displayed as a dash, not as whitespace)
+        '\u{0009}' | // Character Tabulation
+        '\u{0020}' | // Space
+        '\u{00A0}' | // No-break Space
+        '\u{180E}' | // Mongolian Vowel Separator
+        '\u{202F}' | // Narrow No-break Space
+        '\u{205F}' | // Medium Mathematical Space
+        '\u{3000}' | // Ideographic Space
+        '\u{FEFF}'   // Zero Width No-break Space
+        => true,
+
+        // En Quad, Em Quad, En Space, Em Space, Three-per-em Space,
+        // Four-per-em Space, Six-per-em Space, Figure Space,
+        // Punctuation Space, Thin Space, Hair Space, Zero Width Space.
+        ch if ('\u{2000}' ..= '\u{200B}').contains(&ch) => true,
+
+        _ => false,
+    }
+}
+
+#[inline]
+pub fn char_is_punctuation(ch: char) -> bool {
+    use unicode_general_category::{get_general_category, GeneralCategory};
+
+    matches!(
+        get_general_category(ch),
+        GeneralCategory::OtherPunctuation
+            | GeneralCategory::OpenPunctuation
+            | GeneralCategory::ClosePunctuation
+            | GeneralCategory::InitialPunctuation
+            | GeneralCategory::FinalPunctuation
+            | GeneralCategory::ConnectorPunctuation
+            | GeneralCategory::DashPunctuation
+            | GeneralCategory::MathSymbol
+            | GeneralCategory::CurrencySymbol
+            | GeneralCategory::ModifierSymbol
+    )
+}
+
+#[inline]
+pub fn char_is_word(ch: char) -> bool {
+    ch.is_alphanumeric() || ch == '_'
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_categorize() {
+        const EOL_TEST_CASE: &'static str = "\n\r\u{000B}\u{000C}\u{0085}\u{2028}\u{2029}";
+        const WORD_TEST_CASE: &'static str =
+            "_hello_world_あいうえおー1234567890１２３４５６７８９０";
+        const PUNCTUATION_TEST_CASE: &'static str =
+            "!\"#$%&\'()*+,-./:;<=>?@[\\]^`{|}~！”＃＄％＆’（）＊＋、。：；＜＝＞？＠「」＾｀｛｜｝～";
+        const WHITESPACE_TEST_CASE: &'static str = "  　   ";
+
+        for ch in EOL_TEST_CASE.chars() {
+            assert_eq!(CharCategory::Eol, categorize_char(ch));
+        }
+
+        for ch in WHITESPACE_TEST_CASE.chars() {
+            assert_eq!(
+                CharCategory::Whitespace,
+                categorize_char(ch),
+                "Testing '{}', but got `{:?}` instead of `Category::Whitespace`",
+                ch,
+                categorize_char(ch)
+            );
+        }
+
+        for ch in WORD_TEST_CASE.chars() {
+            assert_eq!(
+                CharCategory::Word,
+                categorize_char(ch),
+                "Testing '{}', but got `{:?}` instead of `Category::Word`",
+                ch,
+                categorize_char(ch)
+            );
+        }
+
+        for ch in PUNCTUATION_TEST_CASE.chars() {
+            assert_eq!(
+                CharCategory::Punctuation,
+                categorize_char(ch),
+                "Testing '{}', but got `{:?}` instead of `Category::Punctuation`",
+                ch,
+                categorize_char(ch)
+            );
+        }
+    }
+}

helix-core/src/comment.rs

@@ -1,20 +1,33 @@
+//! This module contains the functionality toggle comments on lines over the selection
+//! using the comment character defined in the user's `languages.toml`
+
 use crate::{
-    find_first_non_whitespace_char2, Change, Rope, RopeSlice, Selection, Tendril, Transaction,
+    find_first_non_whitespace_char, Change, Rope, RopeSlice, Selection, Tendril, Transaction,
 };
-use core::ops::Range;
 use std::borrow::Cow;
 
+/// Given text, a comment token, and a set of line indices, returns the following:
+/// - Whether the given lines should be considered commented
+///     - If any of the lines are uncommented, all lines are considered as such.
+/// - The lines to change for toggling comments
+///     - This is all provided lines excluding blanks lines.
+/// - The column of the comment tokens
+///     - Column of existing tokens, if the lines are commented; column to place tokens at otherwise.
+/// - The margin to the right of the comment tokens
+///     - Defaults to `1`. If any existing comment token is not followed by a space, changes to `0`.
 fn find_line_comment(
     token: &str,
     text: RopeSlice,
-    lines: Range<usize>,
+    lines: impl IntoIterator<Item = usize>,
-) -> (bool, Vec<usize>, usize) {
+) -> (bool, Vec<usize>, usize, usize) {
     let mut commented = true;
-    let mut skipped = Vec::new();
+    let mut to_change = Vec::new();
     let mut min = usize::MAX; // minimum col for find_first_non_whitespace_char
+    let mut margin = 1;
+    let token_len = token.chars().count();
     for line in lines {
         let line_slice = text.line(line);
-        if let Some(pos) = find_first_non_whitespace_char2(line_slice) {
+        if let Some(pos) = find_first_non_whitespace_char(line_slice) {
             let len = line_slice.len_chars();
 
             if pos < min {
@@ -29,47 +42,55 @@ fn find_line_comment(
                 // considered uncommented.
                 commented = false;
             }
-        } else {
+
-            // blank line
+            // determine margin of 0 or 1 for uncommenting; if any comment token is not followed by a space,
-            skipped.push(line);
+            // a margin of 0 is used for all lines.
+            if matches!(line_slice.get_char(pos + token_len), Some(c) if c != ' ') {
+                margin = 0;
+            }
+
+            // blank lines don't get pushed.
+            to_change.push(line);
         }
     }
-    (commented, skipped, min)
+    (commented, to_change, min, margin)
 }
 
 #[must_use]
-pub fn toggle_line_comments(doc: &Rope, selection: &Selection) -> Transaction {
+pub fn toggle_line_comments(doc: &Rope, selection: &Selection, token: Option<&str>) -> Transaction {
     let text = doc.slice(..);
-    let mut changes: Vec<Change> = Vec::new();
 
-    let token = "//";
+    let token = token.unwrap_or("//");
     let comment = Tendril::from(format!("{} ", token));
 
+    let mut lines: Vec<usize> = Vec::with_capacity(selection.len());
+
+    let mut min_next_line = 0;
     for selection in selection {
-        let start = text.char_to_line(selection.from());
+        let (start, end) = selection.line_range(text);
-        let end = text.char_to_line(selection.to());
+        let start = start.max(min_next_line).min(text.len_lines());
-        let lines = start..end + 1;
+        let end = (end + 1).min(text.len_lines());
-        let (commented, skipped, min) = find_line_comment(token, text, lines.clone());
 
-        changes.reserve(end - start - skipped.len());
+        lines.extend(start..end);
+        min_next_line = end + 1;
+    }
 
-        for line in lines {
+    let (commented, to_change, min, margin) = find_line_comment(token, text, lines);
-            if skipped.contains(&line) {
-                continue;
-            }
 
-            let pos = text.line_to_char(line) + min;
+    let mut changes: Vec<Change> = Vec::with_capacity(to_change.len());
 
-            if !commented {
+    for line in to_change {
-                // comment line
+        let pos = text.line_to_char(line) + min;
-                changes.push((pos, pos, Some(comment.clone())))
+
-            } else {
+        if !commented {
-                // uncomment line
+            // comment line
-                let margin = 1; // TODO: margin is hardcoded 1 but could easily be 0
+            changes.push((pos, pos, Some(comment.clone())));
-                changes.push((pos, pos + token.len() + margin, None))
+        } else {
-            }
+            // uncomment line
+            changes.push((pos, pos + token.len() + margin, None));
         }
     }
+
     Transaction::change(doc, changes.into_iter())
 }
 
@@ -91,23 +112,32 @@ mod test {
         let text = state.doc.slice(..);
 
         let res = find_line_comment("//", text, 0..3);
-        // (commented = true, skipped = [line 1], min = col 2)
+        // (commented = true, to_change = [line 0, line 2], min = col 2, margin = 1)
-        assert_eq!(res, (false, vec![1], 2));
+        assert_eq!(res, (false, vec![0, 2], 2, 1));
 
         // comment
-        let transaction = toggle_line_comments(&state.doc, &state.selection);
+        let transaction = toggle_line_comments(&state.doc, &state.selection, None);
         transaction.apply(&mut state.doc);
-        state.selection = state.selection.clone().map(transaction.changes());
+        state.selection = state.selection.map(transaction.changes());
 
         assert_eq!(state.doc, "  // 1\n\n  // 2\n  // 3");
 
         // uncomment
-        let transaction = toggle_line_comments(&state.doc, &state.selection);
+        let transaction = toggle_line_comments(&state.doc, &state.selection, None);
         transaction.apply(&mut state.doc);
-        state.selection = state.selection.clone().map(transaction.changes());
+        state.selection = state.selection.map(transaction.changes());
+        assert_eq!(state.doc, "  1\n\n  2\n  3");
+
+        // 0 margin comments
+        state.doc = Rope::from("  //1\n\n  //2\n  //3");
+        // reset the selection.
+        state.selection = Selection::single(0, state.doc.len_chars() - 1);
+
+        let transaction = toggle_line_comments(&state.doc, &state.selection, None);
+        transaction.apply(&mut state.doc);
+        state.selection = state.selection.map(transaction.changes());
         assert_eq!(state.doc, "  1\n\n  2\n  3");
 
-        // TODO: account for no margin after comment
         // TODO: account for uncommenting with uneven comment indentation
     }
 }

helix-core/src/diagnostic.rs

@@ -1,15 +1,30 @@
-#[derive(Eq, PartialEq)]
+//! LSP diagnostic utility types.
+use serde::{Deserialize, Serialize};
+
+/// Describes the severity level of a [`Diagnostic`].
+#[derive(Debug, Clone, Copy, Eq, PartialEq, PartialOrd, Ord, Deserialize, Serialize)]
 pub enum Severity {
-    Error,
-    Warning,
-    Info,
     Hint,
+    Info,
+    Warning,
+    Error,
 }
 
+impl Default for Severity {
+    fn default() -> Self {
+        Self::Hint
+    }
+}
+
+/// A range of `char`s within the text.
+#[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq)]
 pub struct Range {
     pub start: usize,
     pub end: usize,
 }
+
+/// Corresponds to [`lsp_types::Diagnostic`](https://docs.rs/lsp-types/0.91.0/lsp_types/struct.Diagnostic.html)
+#[derive(Debug, Clone)]
 pub struct Diagnostic {
     pub range: Range,
     pub line: usize,

helix-core/src/diff.rs

@@ -0,0 +1,68 @@
+use crate::{Rope, Transaction};
+
+/// Compares `old` and `new` to generate a [`Transaction`] describing
+/// the steps required to get from `old` to `new`.
+pub fn compare_ropes(old: &Rope, new: &Rope) -> Transaction {
+    // `similar` only works on contiguous data, so a `Rope` has
+    // to be temporarily converted into a `String`.
+    let old_converted = old.to_string();
+    let new_converted = new.to_string();
+
+    // A timeout is set so after 1 seconds, the algorithm will start
+    // approximating. This is especially important for big `Rope`s or
+    // `Rope`s that are extremely dissimilar to each other.
+    //
+    // Note: Ignore the clippy warning, as the trait bounds of
+    // `Transaction::change()` require an iterator implementing
+    // `ExactIterator`.
+    let mut config = similar::TextDiff::configure();
+    config.timeout(std::time::Duration::from_secs(1));
+
+    let diff = config.diff_chars(&old_converted, &new_converted);
+
+    // The current position of the change needs to be tracked to
+    // construct the `Change`s.
+    let mut pos = 0;
+    Transaction::change(
+        old,
+        diff.ops()
+            .iter()
+            .map(|op| op.as_tag_tuple())
+            .filter_map(|(tag, old_range, new_range)| {
+                // `old_pos..pos` is equivalent to `start..end` for where
+                // the change should be applied.
+                let old_pos = pos;
+                pos += old_range.end - old_range.start;
+
+                match tag {
+                    // Semantically, inserts and replacements are the same thing.
+                    similar::DiffTag::Insert | similar::DiffTag::Replace => {
+                        // This is the text from the `new` rope that should be
+                        // inserted into `old`.
+                        let text: &str = {
+                            let start = new.char_to_byte(new_range.start);
+                            let end = new.char_to_byte(new_range.end);
+                            &new_converted[start..end]
+                        };
+                        Some((old_pos, pos, Some(text.into())))
+                    }
+                    similar::DiffTag::Delete => Some((old_pos, pos, None)),
+                    similar::DiffTag::Equal => None,
+                }
+            }),
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    quickcheck::quickcheck! {
+        fn test_compare_ropes(a: String, b: String) -> bool {
+            let mut old = Rope::from(a);
+            let new = Rope::from(b);
+            compare_ropes(&old, &new).apply(&mut old);
+            old.to_string() == new.to_string()
+        }
+    }
+}

helix-core/src/graphemes.rs

@@ -1,8 +1,12 @@
-// Based on https://github.com/cessen/led/blob/c4fa72405f510b7fd16052f90a598c429b3104a6/src/graphemes.rs
+//! Utility functions to traverse the unicode graphemes of a `Rope`'s text contents.
+//!
+//! Based on <https://github.com/cessen/led/blob/c4fa72405f510b7fd16052f90a598c429b3104a6/src/graphemes.rs>
 use ropey::{iter::Chunks, str_utils::byte_to_char_idx, RopeSlice};
 use unicode_segmentation::{GraphemeCursor, GraphemeIncomplete};
 use unicode_width::UnicodeWidthStr;
 
+use std::fmt;
+
 #[must_use]
 pub fn grapheme_width(g: &str) -> usize {
     if g.as_bytes()[0] <= 127 {
@@ -69,6 +73,8 @@ pub fn nth_prev_grapheme_boundary(slice: RopeSlice, char_idx: usize, n: usize) -
 }
 
 /// Finds the previous grapheme boundary before the given char position.
+#[must_use]
+#[inline(always)]
 pub fn prev_grapheme_boundary(slice: RopeSlice, char_idx: usize) -> usize {
     nth_prev_grapheme_boundary(slice, char_idx, 1)
 }
@@ -115,11 +121,38 @@ pub fn nth_next_grapheme_boundary(slice: RopeSlice, char_idx: usize, n: usize) -
 }
 
 /// Finds the next grapheme boundary after the given char position.
+#[must_use]
+#[inline(always)]
 pub fn next_grapheme_boundary(slice: RopeSlice, char_idx: usize) -> usize {
     nth_next_grapheme_boundary(slice, char_idx, 1)
 }
 
+/// Returns the passed char index if it's already a grapheme boundary,
+/// or the next grapheme boundary char index if not.
+#[must_use]
+#[inline]
+pub fn ensure_grapheme_boundary_next(slice: RopeSlice, char_idx: usize) -> usize {
+    if char_idx == 0 {
+        char_idx
+    } else {
+        next_grapheme_boundary(slice, char_idx - 1)
+    }
+}
+
+/// Returns the passed char index if it's already a grapheme boundary,
+/// or the prev grapheme boundary char index if not.
+#[must_use]
+#[inline]
+pub fn ensure_grapheme_boundary_prev(slice: RopeSlice, char_idx: usize) -> usize {
+    if char_idx == slice.len_chars() {
+        char_idx
+    } else {
+        prev_grapheme_boundary(slice, char_idx + 1)
+    }
+}
+
 /// Returns whether the given char position is a grapheme boundary.
+#[must_use]
 pub fn is_grapheme_boundary(slice: RopeSlice, char_idx: usize) -> bool {
     // Bounds check
     debug_assert!(char_idx <= slice.len_chars());
@@ -156,6 +189,18 @@ pub struct RopeGraphemes<'a> {
     cursor: GraphemeCursor,
 }
 
+impl<'a> fmt::Debug for RopeGraphemes<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("RopeGraphemes")
+            .field("text", &self.text)
+            .field("chunks", &self.chunks)
+            .field("cur_chunk", &self.cur_chunk)
+            .field("cur_chunk_start", &self.cur_chunk_start)
+            // .field("cursor", &self.cursor)
+            .finish()
+    }
+}
+
 impl<'a> RopeGraphemes<'a> {
     #[must_use]
     pub fn new(slice: RopeSlice) -> RopeGraphemes {
@@ -193,6 +238,10 @@ impl<'a> Iterator for RopeGraphemes<'a> {
                     self.cur_chunk_start += self.cur_chunk.len();
                     self.cur_chunk = self.chunks.next().unwrap_or("");
                 }
+                Err(GraphemeIncomplete::PreContext(idx)) => {
+                    let (chunk, byte_idx, _, _) = self.text.chunk_at_byte(idx.saturating_sub(1));
+                    self.cursor.provide_context(chunk, byte_idx);
+                }
                 _ => unreachable!(),
             }
         }

helix-core/src/history.rs

@@ -1,19 +1,62 @@
-use crate::{ChangeSet, Rope, State, Transaction};
+use crate::{Assoc, ChangeSet, Range, Rope, State, Transaction};
-use smallvec::{smallvec, SmallVec};
+use once_cell::sync::Lazy;
+use regex::Regex;
+use std::num::NonZeroUsize;
+use std::time::{Duration, Instant};
 
-/// Undo-tree style history store.
+/// Stores the history of changes to a buffer.
+///
+/// Currently the history is represented as a vector of revisions. The vector
+/// always has at least one element: the empty root revision. Each revision
+/// with the exception of the root has a parent revision, a [Transaction]
+/// that can be applied to its parent to transition from the parent to itself,
+/// and an inversion of that transaction to transition from the parent to its
+/// latest child.
+///
+/// When using `u` to undo a change, an inverse of the stored transaction will
+/// be applied which will transition the buffer to the parent state.
+///
+/// Each revision with the exception of the last in the vector also has a
+/// last child revision. When using `U` to redo a change, the last child transaction
+/// will be applied to the current state of the buffer.
+///
+/// The current revision is the one currently displayed in the buffer.
+///
+/// Commiting a new revision to the history will update the last child of the
+/// current revision, and push a new revision to the end of the vector.
+///
+/// Revisions are commited with a timestamp. :earlier and :later can be used
+/// to jump to the closest revision to a moment in time relative to the timestamp
+/// of the current revision plus (:later) or minus (:earlier) the duration
+/// given to the command. If a single integer is given, the editor will instead
+/// jump the given number of revisions in the vector.
+///
+/// Limitations:
+///  * Changes in selections currently don't commit history changes. The selection
+///    will only be updated to the state after a commited buffer change.
+///  * The vector of history revisions is currently unbounded. This might
+///    cause the memory consumption to grow significantly large during long
+///    editing sessions.
+///  * Because delete transactions currently don't store the text that they
+///    delete, we also store an inversion of the transaction.
+///
+/// Using time to navigate the history: <https://github.com/helix-editor/helix/pull/194>
+#[derive(Debug)]
 pub struct History {
     revisions: Vec<Revision>,
-    cursor: usize,
+    current: usize,
 }
 
+/// A single point in history. See [History] for more information.
 #[derive(Debug)]
 struct Revision {
     parent: usize,
-    children: SmallVec<[(usize, Transaction); 1]>,
+    last_child: Option<NonZeroUsize>,
-    /// The transaction to revert to previous state.
+    transaction: Transaction,
-    revert: Transaction,
+    // We need an inversion for undos because delete transactions don't store
-    // selection before, selection after?
+    // the deleted text.
+    inversion: Transaction,
+    timestamp: Instant,
 }
 
 impl Default for History {
@@ -22,72 +65,301 @@ impl Default for History {
         Self {
             revisions: vec![Revision {
                 parent: 0,
-                children: SmallVec::new(),
+                last_child: None,
-                revert: Transaction::from(ChangeSet::new(&Rope::new())),
+                transaction: Transaction::from(ChangeSet::new(&Rope::new())),
+                inversion: Transaction::from(ChangeSet::new(&Rope::new())),
+                timestamp: Instant::now(),
             }],
-            cursor: 0,
+            current: 0,
         }
     }
 }
 
 impl History {
     pub fn commit_revision(&mut self, transaction: &Transaction, original: &State) {
-        // TODO: could store a single transaction, if deletes also stored the text they delete
+        self.commit_revision_at_timestamp(transaction, original, Instant::now());
-        let revert = transaction
+    }
+
+    pub fn commit_revision_at_timestamp(
+        &mut self,
+        transaction: &Transaction,
+        original: &State,
+        timestamp: Instant,
+    ) {
+        let inversion = transaction
             .invert(&original.doc)
             // Store the current cursor position
             .with_selection(original.selection.clone());
 
-        let new_cursor = self.revisions.len();
+        let new_current = self.revisions.len();
+        self.revisions[self.current].last_child = NonZeroUsize::new(new_current);
         self.revisions.push(Revision {
-            parent: self.cursor,
+            parent: self.current,
-            children: SmallVec::new(),
+            last_child: None,
-            revert,
+            transaction: transaction.clone(),
+            inversion,
+            timestamp,
         });
-
+        self.current = new_current;
-        // add a reference to the parent
-        self.revisions
-            .get_mut(self.cursor)
-            .unwrap() // TODO: get_unchecked_mut
-            .children
-            .push((new_cursor, transaction.clone()));
-
-        self.cursor = new_cursor;
     }
 
     #[inline]
     pub fn current_revision(&self) -> usize {
-        self.cursor
+        self.current
     }
 
     #[inline]
     pub const fn at_root(&self) -> bool {
-        self.cursor == 0
+        self.current == 0
     }
 
+    /// Undo the last edit.
     pub fn undo(&mut self) -> Option<&Transaction> {
         if self.at_root() {
-            // We're at the root of undo, nothing to do.
             return None;
         }
 
-        let current_revision = &self.revisions[self.cursor];
+        let current_revision = &self.revisions[self.current];
-
+        self.current = current_revision.parent;
-        self.cursor = current_revision.parent;
+        Some(&current_revision.inversion)
-
-        Some(&current_revision.revert)
     }
 
+    /// Redo the last edit.
     pub fn redo(&mut self) -> Option<&Transaction> {
-        let current_revision = &self.revisions[self.cursor];
+        let current_revision = &self.revisions[self.current];
+        let last_child = current_revision.last_child?;
+        self.current = last_child.get();
 
-        // for now, simply pick the latest child (linear undo / redo)
+        Some(&self.revisions[last_child.get()].transaction)
-        if let Some((index, transaction)) = current_revision.children.last() {
+    }
-            self.cursor = *index;
 
-            return Some(&transaction);
+    // Get the position of last change
+    pub fn last_edit_pos(&self) -> Option<usize> {
+        if self.current == 0 {
+            return None;
+        }
+        let current_revision = &self.revisions[self.current];
+        let primary_selection = current_revision
+            .inversion
+            .selection()
+            .expect("inversion always contains a selection")
+            .primary();
+        let (_from, to, _fragment) = current_revision
+            .transaction
+            .changes_iter()
+            // find a change that matches the primary selection
+            .find(|(from, to, _fragment)| Range::new(*from, *to).overlaps(&primary_selection))
+            // or use the first change
+            .or_else(|| current_revision.transaction.changes_iter().next())
+            .unwrap();
+        let pos = current_revision
+            .transaction
+            .changes()
+            .map_pos(to, Assoc::After);
+        Some(pos)
+    }
+
+    fn lowest_common_ancestor(&self, mut a: usize, mut b: usize) -> usize {
+        use std::collections::HashSet;
+        let mut a_path_set = HashSet::new();
+        let mut b_path_set = HashSet::new();
+        loop {
+            a_path_set.insert(a);
+            b_path_set.insert(b);
+            if a_path_set.contains(&b) {
+                return b;
+            }
+            if b_path_set.contains(&a) {
+                return a;
+            }
+            a = self.revisions[a].parent; // Relies on the parent of 0 being 0.
+            b = self.revisions[b].parent; // Same as above.
+        }
+    }
+
+    /// List of nodes on the way from `n` to 'a`. Doesn`t include `a`.
+    /// Includes `n` unless `a == n`. `a` must be an ancestor of `n`.
+    fn path_up(&self, mut n: usize, a: usize) -> Vec<usize> {
+        let mut path = Vec::new();
+        while n != a {
+            path.push(n);
+            n = self.revisions[n].parent;
+        }
+        path
+    }
+
+    /// Create a [`Transaction`] that will jump to a specific revision in the history.
+    fn jump_to(&mut self, to: usize) -> Vec<Transaction> {
+        let lca = self.lowest_common_ancestor(self.current, to);
+        let up = self.path_up(self.current, lca);
+        let down = self.path_up(to, lca);
+        self.current = to;
+        let up_txns = up.iter().map(|&n| self.revisions[n].inversion.clone());
+        let down_txns = down
+            .iter()
+            .rev()
+            .map(|&n| self.revisions[n].transaction.clone());
+        up_txns.chain(down_txns).collect()
+    }
+
+    /// Creates a [`Transaction`] that will undo `delta` revisions.
+    fn jump_backward(&mut self, delta: usize) -> Vec<Transaction> {
+        self.jump_to(self.current.saturating_sub(delta))
+    }
+
+    /// Creates a [`Transaction`] that will redo `delta` revisions.
+    fn jump_forward(&mut self, delta: usize) -> Vec<Transaction> {
+        self.jump_to(
+            self.current
+                .saturating_add(delta)
+                .min(self.revisions.len() - 1),
+        )
+    }
+
+    /// Helper for a binary search case below.
+    fn revision_closer_to_instant(&self, i: usize, instant: Instant) -> usize {
+        let dur_im1 = instant.duration_since(self.revisions[i - 1].timestamp);
+        let dur_i = self.revisions[i].timestamp.duration_since(instant);
+        use std::cmp::Ordering::*;
+        match dur_im1.cmp(&dur_i) {
+            Less => i - 1,
+            Equal | Greater => i,
+        }
+    }
+
+    /// Creates a [`Transaction`] that will match a revision created at around
+    /// `instant`.
+    fn jump_instant(&mut self, instant: Instant) -> Vec<Transaction> {
+        let search_result = self
+            .revisions
+            .binary_search_by(|rev| rev.timestamp.cmp(&instant));
+        let revision = match search_result {
+            Ok(revision) => revision,
+            Err(insert_point) => match insert_point {
+                0 => 0,
+                n if n == self.revisions.len() => n - 1,
+                i => self.revision_closer_to_instant(i, instant),
+            },
+        };
+        self.jump_to(revision)
+    }
+
+    /// Creates a [`Transaction`] that will match a revision created `duration` ago
+    /// from the timestamp of current revision.
+    fn jump_duration_backward(&mut self, duration: Duration) -> Vec<Transaction> {
+        match self.revisions[self.current].timestamp.checked_sub(duration) {
+            Some(instant) => self.jump_instant(instant),
+            None => self.jump_to(0),
+        }
+    }
+
+    /// Creates a [`Transaction`] that will match a revision created `duration` in
+    /// the future from the timestamp of the current revision.
+    fn jump_duration_forward(&mut self, duration: Duration) -> Vec<Transaction> {
+        match self.revisions[self.current].timestamp.checked_add(duration) {
+            Some(instant) => self.jump_instant(instant),
+            None => self.jump_to(self.revisions.len() - 1),
+        }
+    }
+
+    /// Creates an undo [`Transaction`].
+    pub fn earlier(&mut self, uk: UndoKind) -> Vec<Transaction> {
+        use UndoKind::*;
+        match uk {
+            Steps(n) => self.jump_backward(n),
+            TimePeriod(d) => self.jump_duration_backward(d),
+        }
+    }
+
+    /// Creates a redo [`Transaction`].
+    pub fn later(&mut self, uk: UndoKind) -> Vec<Transaction> {
+        use UndoKind::*;
+        match uk {
+            Steps(n) => self.jump_forward(n),
+            TimePeriod(d) => self.jump_duration_forward(d),
+        }
+    }
+}
+
+/// Whether to undo by a number of edits or a duration of time.
+#[derive(Debug, PartialEq, Clone, Copy)]
+pub enum UndoKind {
+    Steps(usize),
+    TimePeriod(std::time::Duration),
+}
+
+/// A subset of sytemd.time time span syntax units.
+const TIME_UNITS: &[(&[&str], &str, u64)] = &[
+    (&["seconds", "second", "sec", "s"], "seconds", 1),
+    (&["minutes", "minute", "min", "m"], "minutes", 60),
+    (&["hours", "hour", "hr", "h"], "hours", 60 * 60),
+    (&["days", "day", "d"], "days", 24 * 60 * 60),
+];
+
+/// Checks if the duration input can be turned into a valid duration. It must be a
+/// positive integer and denote the [unit of time.](`TIME_UNITS`)
+/// Examples of valid durations:
+///  * `5 sec`
+///  * `5 min`
+///  * `5 hr`
+///  * `5 days`
+static DURATION_VALIDATION_REGEX: Lazy<Regex> =
+    Lazy::new(|| Regex::new(r"^(?:\d+\s*[a-z]+\s*)+$").unwrap());
+
+/// Captures both the number and unit as separate capture groups.
+static NUMBER_UNIT_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"(\d+)\s*([a-z]+)").unwrap());
+
+/// Parse a string (e.g. "5 sec") and try to convert it into a [`Duration`].
+fn parse_human_duration(s: &str) -> Result<Duration, String> {
+    if !DURATION_VALIDATION_REGEX.is_match(s) {
+        return Err("duration should be composed \
+        of positive integers followed by time units"
+            .to_string());
+    }
+
+    let mut specified = [false; TIME_UNITS.len()];
+    let mut seconds = 0u64;
+    for cap in NUMBER_UNIT_REGEX.captures_iter(s) {
+        let (n, unit_str) = (&cap[1], &cap[2]);
+
+        let n: u64 = n.parse().map_err(|_| format!("integer too large: {}", n))?;
+
+        let time_unit = TIME_UNITS
+            .iter()
+            .enumerate()
+            .find(|(_, (forms, _, _))| forms.iter().any(|f| f == &unit_str));
+
+        if let Some((i, (_, unit, mul))) = time_unit {
+            if specified[i] {
+                return Err(format!("{} specified more than once", unit));
+            }
+            specified[i] = true;
+
+            let new_seconds = n.checked_mul(*mul).and_then(|s| seconds.checked_add(s));
+            match new_seconds {
+                Some(ns) => seconds = ns,
+                None => return Err("duration too large".to_string()),
+            }
+        } else {
+            return Err(format!("incorrect time unit: {}", unit_str));
+        }
+    }
+
+    Ok(Duration::from_secs(seconds))
+}
+
+impl std::str::FromStr for UndoKind {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let s = s.trim();
+        if s.is_empty() {
+            Ok(Self::Steps(1usize))
+        } else if let Ok(n) = s.parse::<usize>() {
+            Ok(UndoKind::Steps(n))
+        } else {
+            Ok(Self::TimePeriod(parse_human_duration(s)?))
         }
-        None
     }
 }
 
@@ -143,4 +415,191 @@ mod test {
         undo(&mut history, &mut state);
         assert_eq!("hello", state.doc);
     }
+
+    #[test]
+    fn test_earlier_later() {
+        let mut history = History::default();
+        let doc = Rope::from("a\n");
+        let mut state = State::new(doc);
+
+        fn undo(history: &mut History, state: &mut State) {
+            if let Some(transaction) = history.undo() {
+                transaction.apply(&mut state.doc);
+            }
+        }
+
+        fn earlier(history: &mut History, state: &mut State, uk: UndoKind) {
+            let txns = history.earlier(uk);
+            for txn in txns {
+                txn.apply(&mut state.doc);
+            }
+        }
+
+        fn later(history: &mut History, state: &mut State, uk: UndoKind) {
+            let txns = history.later(uk);
+            for txn in txns {
+                txn.apply(&mut state.doc);
+            }
+        }
+
+        fn commit_change(
+            history: &mut History,
+            state: &mut State,
+            change: crate::transaction::Change,
+            instant: Instant,
+        ) {
+            let txn = Transaction::change(&state.doc, vec![change.clone()].into_iter());
+            history.commit_revision_at_timestamp(&txn, &state, instant);
+            txn.apply(&mut state.doc);
+        }
+
+        let t0 = Instant::now();
+        let t = |n| t0.checked_add(Duration::from_secs(n)).unwrap();
+
+        commit_change(&mut history, &mut state, (1, 1, Some(" b".into())), t(0));
+        assert_eq!("a b\n", state.doc);
+
+        commit_change(&mut history, &mut state, (3, 3, Some(" c".into())), t(10));
+        assert_eq!("a b c\n", state.doc);
+
+        commit_change(&mut history, &mut state, (5, 5, Some(" d".into())), t(20));
+        assert_eq!("a b c d\n", state.doc);
+
+        undo(&mut history, &mut state);
+        assert_eq!("a b c\n", state.doc);
+
+        commit_change(&mut history, &mut state, (5, 5, Some(" e".into())), t(30));
+        assert_eq!("a b c e\n", state.doc);
+
+        undo(&mut history, &mut state);
+        undo(&mut history, &mut state);
+        assert_eq!("a b\n", state.doc);
+
+        commit_change(&mut history, &mut state, (1, 3, None), t(40));
+        assert_eq!("a\n", state.doc);
+
+        commit_change(&mut history, &mut state, (1, 1, Some(" f".into())), t(50));
+        assert_eq!("a f\n", state.doc);
+
+        use UndoKind::*;
+
+        earlier(&mut history, &mut state, Steps(3));
+        assert_eq!("a b c d\n", state.doc);
+
+        later(&mut history, &mut state, TimePeriod(Duration::new(20, 0)));
+        assert_eq!("a\n", state.doc);
+
+        earlier(&mut history, &mut state, TimePeriod(Duration::new(19, 0)));
+        assert_eq!("a b c d\n", state.doc);
+
+        earlier(
+            &mut history,
+            &mut state,
+            TimePeriod(Duration::new(10000, 0)),
+        );
+        assert_eq!("a\n", state.doc);
+
+        later(&mut history, &mut state, Steps(50));
+        assert_eq!("a f\n", state.doc);
+
+        earlier(&mut history, &mut state, Steps(4));
+        assert_eq!("a b c\n", state.doc);
+
+        later(&mut history, &mut state, TimePeriod(Duration::new(1, 0)));
+        assert_eq!("a b c\n", state.doc);
+
+        later(&mut history, &mut state, TimePeriod(Duration::new(5, 0)));
+        assert_eq!("a b c d\n", state.doc);
+
+        later(&mut history, &mut state, TimePeriod(Duration::new(6, 0)));
+        assert_eq!("a b c e\n", state.doc);
+
+        later(&mut history, &mut state, Steps(1));
+        assert_eq!("a\n", state.doc);
+    }
+
+    #[test]
+    fn test_parse_undo_kind() {
+        use UndoKind::*;
+
+        // Default is one step.
+        assert_eq!("".parse(), Ok(Steps(1)));
+
+        // An integer means the number of steps.
+        assert_eq!("1".parse(), Ok(Steps(1)));
+        assert_eq!("  16 ".parse(), Ok(Steps(16)));
+
+        // Duration has a strict format.
+        let validation_err = Err("duration should be composed \
+         of positive integers followed by time units"
+            .to_string());
+        assert_eq!("  16 33".parse::<UndoKind>(), validation_err);
+        assert_eq!("  seconds 22  ".parse::<UndoKind>(), validation_err);
+        assert_eq!("  -4 m".parse::<UndoKind>(), validation_err);
+        assert_eq!("5s 3".parse::<UndoKind>(), validation_err);
+
+        // Units are u64.
+        assert_eq!(
+            "18446744073709551616minutes".parse::<UndoKind>(),
+            Err("integer too large: 18446744073709551616".to_string())
+        );
+
+        // Units are validated.
+        assert_eq!(
+            "1 millenium".parse::<UndoKind>(),
+            Err("incorrect time unit: millenium".to_string())
+        );
+
+        // Units can't be specified twice.
+        assert_eq!(
+            "2 seconds 6s".parse::<UndoKind>(),
+            Err("seconds specified more than once".to_string())
+        );
+
+        // Various formats are correctly handled.
+        assert_eq!(
+            "4s".parse::<UndoKind>(),
+            Ok(TimePeriod(Duration::from_secs(4)))
+        );
+        assert_eq!(
+            "2m".parse::<UndoKind>(),
+            Ok(TimePeriod(Duration::from_secs(120)))
+        );
+        assert_eq!(
+            "5h".parse::<UndoKind>(),
+            Ok(TimePeriod(Duration::from_secs(5 * 60 * 60)))
+        );
+        assert_eq!(
+            "3d".parse::<UndoKind>(),
+            Ok(TimePeriod(Duration::from_secs(3 * 24 * 60 * 60)))
+        );
+        assert_eq!(
+            "1m30s".parse::<UndoKind>(),
+            Ok(TimePeriod(Duration::from_secs(90)))
+        );
+        assert_eq!(
+            "1m 20 seconds".parse::<UndoKind>(),
+            Ok(TimePeriod(Duration::from_secs(80)))
+        );
+        assert_eq!(
+            "  2 minute 1day".parse::<UndoKind>(),
+            Ok(TimePeriod(Duration::from_secs(24 * 60 * 60 + 2 * 60)))
+        );
+        assert_eq!(
+            "3 d 2hour 5 minutes 30sec".parse::<UndoKind>(),
+            Ok(TimePeriod(Duration::from_secs(
+                3 * 24 * 60 * 60 + 2 * 60 * 60 + 5 * 60 + 30
+            )))
+        );
+
+        // Sum overflow is handled.
+        assert_eq!(
+            "18446744073709551615minutes".parse::<UndoKind>(),
+            Err("duration too large".to_string())
+        );
+        assert_eq!(
+            "1 minute 18446744073709551615 seconds".parse::<UndoKind>(),
+            Err("duration too large".to_string())
+        );
+    }
 }

helix-core/src/increment/date_time.rs

@@ -0,0 +1,490 @@
+use chrono::{Datelike, Duration, NaiveDate, NaiveDateTime, NaiveTime, Timelike};
+use once_cell::sync::Lazy;
+use regex::Regex;
+use ropey::RopeSlice;
+
+use std::borrow::Cow;
+use std::cmp;
+
+use super::Increment;
+use crate::{Range, Tendril};
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct DateTimeIncrementor {
+    date_time: NaiveDateTime,
+    range: Range,
+    fmt: &'static str,
+    field: DateField,
+}
+
+impl DateTimeIncrementor {
+    pub fn from_range(text: RopeSlice, range: Range) -> Option<DateTimeIncrementor> {
+        let range = if range.is_empty() {
+            if range.anchor < text.len_chars() {
+                // Treat empty range as a cursor range.
+                range.put_cursor(text, range.anchor + 1, true)
+            } else {
+                // The range is empty and at the end of the text.
+                return None;
+            }
+        } else {
+            range
+        };
+
+        FORMATS.iter().find_map(|format| {
+            let from = range.from().saturating_sub(format.max_len);
+            let to = (range.from() + format.max_len).min(text.len_chars());
+
+            let (from_in_text, to_in_text) = (range.from() - from, range.to() - from);
+            let text: Cow<str> = text.slice(from..to).into();
+
+            let captures = format.regex.captures(&text)?;
+            if captures.len() - 1 != format.fields.len() {
+                return None;
+            }
+
+            let date_time = captures.get(0)?;
+            let offset = range.from() - from_in_text;
+            let range = Range::new(date_time.start() + offset, date_time.end() + offset);
+
+            let field = captures
+                .iter()
+                .skip(1)
+                .enumerate()
+                .find_map(|(i, capture)| {
+                    let capture = capture?;
+                    let capture_range = capture.range();
+
+                    if capture_range.contains(&from_in_text)
+                        && capture_range.contains(&(to_in_text - 1))
+                    {
+                        Some(format.fields[i])
+                    } else {
+                        None
+                    }
+                })?;
+
+            let has_date = format.fields.iter().any(|f| f.unit.is_date());
+            let has_time = format.fields.iter().any(|f| f.unit.is_time());
+
+            let date_time = &text[date_time.start()..date_time.end()];
+            let date_time = match (has_date, has_time) {
+                (true, true) => NaiveDateTime::parse_from_str(date_time, format.fmt).ok()?,
+                (true, false) => {
+                    let date = NaiveDate::parse_from_str(date_time, format.fmt).ok()?;
+
+                    date.and_hms(0, 0, 0)
+                }
+                (false, true) => {
+                    let time = NaiveTime::parse_from_str(date_time, format.fmt).ok()?;
+
+                    NaiveDate::from_ymd(0, 1, 1).and_time(time)
+                }
+                (false, false) => return None,
+            };
+
+            Some(DateTimeIncrementor {
+                date_time,
+                range,
+                fmt: format.fmt,
+                field,
+            })
+        })
+    }
+}
+
+impl Increment for DateTimeIncrementor {
+    fn increment(&self, amount: i64) -> (Range, Tendril) {
+        let date_time = match self.field.unit {
+            DateUnit::Years => add_years(self.date_time, amount),
+            DateUnit::Months => add_months(self.date_time, amount),
+            DateUnit::Days => add_duration(self.date_time, Duration::days(amount)),
+            DateUnit::Hours => add_duration(self.date_time, Duration::hours(amount)),
+            DateUnit::Minutes => add_duration(self.date_time, Duration::minutes(amount)),
+            DateUnit::Seconds => add_duration(self.date_time, Duration::seconds(amount)),
+            DateUnit::AmPm => toggle_am_pm(self.date_time),
+        }
+        .unwrap_or(self.date_time);
+
+        (self.range, date_time.format(self.fmt).to_string().into())
+    }
+}
+
+static FORMATS: Lazy<Vec<Format>> = Lazy::new(|| {
+    vec![
+        Format::new("%Y-%m-%d %H:%M:%S"), // 2021-11-24 07:12:23
+        Format::new("%Y/%m/%d %H:%M:%S"), // 2021/11/24 07:12:23
+        Format::new("%Y-%m-%d %H:%M"),    // 2021-11-24 07:12
+        Format::new("%Y/%m/%d %H:%M"),    // 2021/11/24 07:12
+        Format::new("%Y-%m-%d"),          // 2021-11-24
+        Format::new("%Y/%m/%d"),          // 2021/11/24
+        Format::new("%a %b %d %Y"),       // Wed Nov 24 2021
+        Format::new("%d-%b-%Y"),          // 24-Nov-2021
+        Format::new("%Y %b %d"),          // 2021 Nov 24
+        Format::new("%b %d, %Y"),         // Nov 24, 2021
+        Format::new("%-I:%M:%S %P"),      // 7:21:53 am
+        Format::new("%-I:%M %P"),         // 7:21 am
+        Format::new("%-I:%M:%S %p"),      // 7:21:53 AM
+        Format::new("%-I:%M %p"),         // 7:21 AM
+        Format::new("%H:%M:%S"),          // 23:24:23
+        Format::new("%H:%M"),             // 23:24
+    ]
+});
+
+#[derive(Debug)]
+struct Format {
+    fmt: &'static str,
+    fields: Vec<DateField>,
+    regex: Regex,
+    max_len: usize,
+}
+
+impl Format {
+    fn new(fmt: &'static str) -> Self {
+        let mut remaining = fmt;
+        let mut fields = Vec::new();
+        let mut regex = String::new();
+        let mut max_len = 0;
+
+        while let Some(i) = remaining.find('%') {
+            let after = &remaining[i + 1..];
+            let mut chars = after.chars();
+            let c = chars.next().unwrap();
+
+            let spec_len = if c == '-' {
+                1 + chars.next().unwrap().len_utf8()
+            } else {
+                c.len_utf8()
+            };
+
+            let specifier = &after[..spec_len];
+            let field = DateField::from_specifier(specifier).unwrap();
+            fields.push(field);
+            max_len += field.max_len + remaining[..i].len();
+            regex += &remaining[..i];
+            regex += &format!("({})", field.regex);
+            remaining = &after[spec_len..];
+        }
+
+        let regex = Regex::new(&regex).unwrap();
+
+        Self {
+            fmt,
+            fields,
+            regex,
+            max_len,
+        }
+    }
+}
+
+impl PartialEq for Format {
+    fn eq(&self, other: &Self) -> bool {
+        self.fmt == other.fmt && self.fields == other.fields && self.max_len == other.max_len
+    }
+}
+
+impl Eq for Format {}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+struct DateField {
+    regex: &'static str,
+    unit: DateUnit,
+    max_len: usize,
+}
+
+impl DateField {
+    fn from_specifier(specifier: &str) -> Option<Self> {
+        match specifier {
+            "Y" => Some(DateField {
+                regex: r"\d{4}",
+                unit: DateUnit::Years,
+                max_len: 5,
+            }),
+            "y" => Some(DateField {
+                regex: r"\d\d",
+                unit: DateUnit::Years,
+                max_len: 2,
+            }),
+            "m" => Some(DateField {
+                regex: r"[0-1]\d",
+                unit: DateUnit::Months,
+                max_len: 2,
+            }),
+            "d" => Some(DateField {
+                regex: r"[0-3]\d",
+                unit: DateUnit::Days,
+                max_len: 2,
+            }),
+            "-d" => Some(DateField {
+                regex: r"[1-3]?\d",
+                unit: DateUnit::Days,
+                max_len: 2,
+            }),
+            "a" => Some(DateField {
+                regex: r"Sun|Mon|Tue|Wed|Thu|Fri|Sat",
+                unit: DateUnit::Days,
+                max_len: 3,
+            }),
+            "A" => Some(DateField {
+                regex: r"Sunday|Monday|Tuesday|Wednesday|Thursday|Friday|Saturday",
+                unit: DateUnit::Days,
+                max_len: 9,
+            }),
+            "b" | "h" => Some(DateField {
+                regex: r"Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec",
+                unit: DateUnit::Months,
+                max_len: 3,
+            }),
+            "B" => Some(DateField {
+                regex: r"January|February|March|April|May|June|July|August|September|October|November|December",
+                unit: DateUnit::Months,
+                max_len: 9,
+            }),
+            "H" => Some(DateField {
+                regex: r"[0-2]\d",
+                unit: DateUnit::Hours,
+                max_len: 2,
+            }),
+            "M" => Some(DateField {
+                regex: r"[0-5]\d",
+                unit: DateUnit::Minutes,
+                max_len: 2,
+            }),
+            "S" => Some(DateField {
+                regex: r"[0-5]\d",
+                unit: DateUnit::Seconds,
+                max_len: 2,
+            }),
+            "I" => Some(DateField {
+                regex: r"[0-1]\d",
+                unit: DateUnit::Hours,
+                max_len: 2,
+            }),
+            "-I" => Some(DateField {
+                regex: r"1?\d",
+                unit: DateUnit::Hours,
+                max_len: 2,
+            }),
+            "P" => Some(DateField {
+                regex: r"am|pm",
+                unit: DateUnit::AmPm,
+                max_len: 2,
+            }),
+            "p" => Some(DateField {
+                regex: r"AM|PM",
+                unit: DateUnit::AmPm,
+                max_len: 2,
+            }),
+            _ => None,
+        }
+    }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+enum DateUnit {
+    Years,
+    Months,
+    Days,
+    Hours,
+    Minutes,
+    Seconds,
+    AmPm,
+}
+
+impl DateUnit {
+    fn is_date(self) -> bool {
+        matches!(self, DateUnit::Years | DateUnit::Months | DateUnit::Days)
+    }
+
+    fn is_time(self) -> bool {
+        matches!(
+            self,
+            DateUnit::Hours | DateUnit::Minutes | DateUnit::Seconds
+        )
+    }
+}
+
+fn ndays_in_month(year: i32, month: u32) -> u32 {
+    // The first day of the next month...
+    let (y, m) = if month == 12 {
+        (year + 1, 1)
+    } else {
+        (year, month + 1)
+    };
+    let d = NaiveDate::from_ymd(y, m, 1);
+
+    // ...is preceded by the last day of the original month.
+    d.pred().day()
+}
+
+fn add_months(date_time: NaiveDateTime, amount: i64) -> Option<NaiveDateTime> {
+    let month = (date_time.month0() as i64).checked_add(amount)?;
+    let year = date_time.year() + i32::try_from(month / 12).ok()?;
+    let year = if month.is_negative() { year - 1 } else { year };
+
+    // Normalize month
+    let month = month % 12;
+    let month = if month.is_negative() {
+        month + 12
+    } else {
+        month
+    } as u32
+        + 1;
+
+    let day = cmp::min(date_time.day(), ndays_in_month(year, month));
+
+    Some(NaiveDate::from_ymd(year, month, day).and_time(date_time.time()))
+}
+
+fn add_years(date_time: NaiveDateTime, amount: i64) -> Option<NaiveDateTime> {
+    let year = i32::try_from((date_time.year() as i64).checked_add(amount)?).ok()?;
+    let ndays = ndays_in_month(year, date_time.month());
+
+    if date_time.day() > ndays {
+        let d = NaiveDate::from_ymd(year, date_time.month(), ndays);
+        Some(d.succ().and_time(date_time.time()))
+    } else {
+        date_time.with_year(year)
+    }
+}
+
+fn add_duration(date_time: NaiveDateTime, duration: Duration) -> Option<NaiveDateTime> {
+    date_time.checked_add_signed(duration)
+}
+
+fn toggle_am_pm(date_time: NaiveDateTime) -> Option<NaiveDateTime> {
+    if date_time.hour() < 12 {
+        add_duration(date_time, Duration::hours(12))
+    } else {
+        add_duration(date_time, Duration::hours(-12))
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::Rope;
+
+    #[test]
+    fn test_increment_date_times() {
+        let tests = [
+            // (original, cursor, amount, expected)
+            ("2020-02-28", 0, 1, "2021-02-28"),
+            ("2020-02-29", 0, 1, "2021-03-01"),
+            ("2020-01-31", 5, 1, "2020-02-29"),
+            ("2020-01-20", 5, 1, "2020-02-20"),
+            ("2021-01-01", 5, -1, "2020-12-01"),
+            ("2021-01-31", 5, -2, "2020-11-30"),
+            ("2020-02-28", 8, 1, "2020-02-29"),
+            ("2021-02-28", 8, 1, "2021-03-01"),
+            ("2021-02-28", 0, -1, "2020-02-28"),
+            ("2021-03-01", 0, -1, "2020-03-01"),
+            ("2020-02-29", 5, -1, "2020-01-29"),
+            ("2020-02-20", 5, -1, "2020-01-20"),
+            ("2020-02-29", 8, -1, "2020-02-28"),
+            ("2021-03-01", 8, -1, "2021-02-28"),
+            ("1980/12/21", 8, 100, "1981/03/31"),
+            ("1980/12/21", 8, -100, "1980/09/12"),
+            ("1980/12/21", 8, 1000, "1983/09/17"),
+            ("1980/12/21", 8, -1000, "1978/03/27"),
+            ("2021-11-24 07:12:23", 0, 1, "2022-11-24 07:12:23"),
+            ("2021-11-24 07:12:23", 5, 1, "2021-12-24 07:12:23"),
+            ("2021-11-24 07:12:23", 8, 1, "2021-11-25 07:12:23"),
+            ("2021-11-24 07:12:23", 11, 1, "2021-11-24 08:12:23"),
+            ("2021-11-24 07:12:23", 14, 1, "2021-11-24 07:13:23"),
+            ("2021-11-24 07:12:23", 17, 1, "2021-11-24 07:12:24"),
+            ("2021/11/24 07:12:23", 0, 1, "2022/11/24 07:12:23"),
+            ("2021/11/24 07:12:23", 5, 1, "2021/12/24 07:12:23"),
+            ("2021/11/24 07:12:23", 8, 1, "2021/11/25 07:12:23"),
+            ("2021/11/24 07:12:23", 11, 1, "2021/11/24 08:12:23"),
+            ("2021/11/24 07:12:23", 14, 1, "2021/11/24 07:13:23"),
+            ("2021/11/24 07:12:23", 17, 1, "2021/11/24 07:12:24"),
+            ("2021-11-24 07:12", 0, 1, "2022-11-24 07:12"),
+            ("2021-11-24 07:12", 5, 1, "2021-12-24 07:12"),
+            ("2021-11-24 07:12", 8, 1, "2021-11-25 07:12"),
+            ("2021-11-24 07:12", 11, 1, "2021-11-24 08:12"),
+            ("2021-11-24 07:12", 14, 1, "2021-11-24 07:13"),
+            ("2021/11/24 07:12", 0, 1, "2022/11/24 07:12"),
+            ("2021/11/24 07:12", 5, 1, "2021/12/24 07:12"),
+            ("2021/11/24 07:12", 8, 1, "2021/11/25 07:12"),
+            ("2021/11/24 07:12", 11, 1, "2021/11/24 08:12"),
+            ("2021/11/24 07:12", 14, 1, "2021/11/24 07:13"),
+            ("Wed Nov 24 2021", 0, 1, "Thu Nov 25 2021"),
+            ("Wed Nov 24 2021", 4, 1, "Fri Dec 24 2021"),
+            ("Wed Nov 24 2021", 8, 1, "Thu Nov 25 2021"),
+            ("Wed Nov 24 2021", 11, 1, "Thu Nov 24 2022"),
+            ("24-Nov-2021", 0, 1, "25-Nov-2021"),
+            ("24-Nov-2021", 3, 1, "24-Dec-2021"),
+            ("24-Nov-2021", 7, 1, "24-Nov-2022"),
+            ("2021 Nov 24", 0, 1, "2022 Nov 24"),
+            ("2021 Nov 24", 5, 1, "2021 Dec 24"),
+            ("2021 Nov 24", 9, 1, "2021 Nov 25"),
+            ("Nov 24, 2021", 0, 1, "Dec 24, 2021"),
+            ("Nov 24, 2021", 4, 1, "Nov 25, 2021"),
+            ("Nov 24, 2021", 8, 1, "Nov 24, 2022"),
+            ("7:21:53 am", 0, 1, "8:21:53 am"),
+            ("7:21:53 am", 3, 1, "7:22:53 am"),
+            ("7:21:53 am", 5, 1, "7:21:54 am"),
+            ("7:21:53 am", 8, 1, "7:21:53 pm"),
+            ("7:21:53 AM", 0, 1, "8:21:53 AM"),
+            ("7:21:53 AM", 3, 1, "7:22:53 AM"),
+            ("7:21:53 AM", 5, 1, "7:21:54 AM"),
+            ("7:21:53 AM", 8, 1, "7:21:53 PM"),
+            ("7:21 am", 0, 1, "8:21 am"),
+            ("7:21 am", 3, 1, "7:22 am"),
+            ("7:21 am", 5, 1, "7:21 pm"),
+            ("7:21 AM", 0, 1, "8:21 AM"),
+            ("7:21 AM", 3, 1, "7:22 AM"),
+            ("7:21 AM", 5, 1, "7:21 PM"),
+            ("23:24:23", 1, 1, "00:24:23"),
+            ("23:24:23", 3, 1, "23:25:23"),
+            ("23:24:23", 6, 1, "23:24:24"),
+            ("23:24", 1, 1, "00:24"),
+            ("23:24", 3, 1, "23:25"),
+        ];
+
+        for (original, cursor, amount, expected) in tests {
+            let rope = Rope::from_str(original);
+            let range = Range::new(cursor, cursor + 1);
+            assert_eq!(
+                DateTimeIncrementor::from_range(rope.slice(..), range)
+                    .unwrap()
+                    .increment(amount)
+                    .1,
+                expected.into()
+            );
+        }
+    }
+
+    #[test]
+    fn test_invalid_date_times() {
+        let tests = [
+            "0000-00-00",
+            "1980-2-21",
+            "1980-12-1",
+            "12345",
+            "2020-02-30",
+            "1999-12-32",
+            "19-12-32",
+            "1-2-3",
+            "0000/00/00",
+            "1980/2/21",
+            "1980/12/1",
+            "12345",
+            "2020/02/30",
+            "1999/12/32",
+            "19/12/32",
+            "1/2/3",
+            "123:456:789",
+            "11:61",
+            "2021-55-12 08:12:54",
+        ];
+
+        for invalid in tests {
+            let rope = Rope::from_str(invalid);
+            let range = Range::new(0, 1);
+
+            assert_eq!(DateTimeIncrementor::from_range(rope.slice(..), range), None)
+        }
+    }
+}

helix-core/src/increment/mod.rs

@@ -0,0 +1,8 @@
+pub mod date_time;
+pub mod number;
+
+use crate::{Range, Tendril};
+
+pub trait Increment {
+    fn increment(&self, amount: i64) -> (Range, Tendril);
+}

helix-core/src/increment/number.rs

@@ -0,0 +1,507 @@
+use std::borrow::Cow;
+
+use ropey::RopeSlice;
+
+use super::Increment;
+
+use crate::{
+    textobject::{textobject_word, TextObject},
+    Range, Tendril,
+};
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct NumberIncrementor<'a> {
+    value: i64,
+    radix: u32,
+    range: Range,
+
+    text: RopeSlice<'a>,
+}
+
+impl<'a> NumberIncrementor<'a> {
+    /// Return information about number under rang if there is one.
+    pub fn from_range(text: RopeSlice, range: Range) -> Option<NumberIncrementor> {
+        // If the cursor is on the minus sign of a number we want to get the word textobject to the
+        // right of it.
+        let range = if range.to() < text.len_chars()
+            && range.to() - range.from() <= 1
+            && text.char(range.from()) == '-'
+        {
+            Range::new(range.from() + 1, range.to() + 1)
+        } else {
+            range
+        };
+
+        let range = textobject_word(text, range, TextObject::Inside, 1, false);
+
+        // If there is a minus sign to the left of the word object, we want to include it in the range.
+        let range = if range.from() > 0 && text.char(range.from() - 1) == '-' {
+            range.extend(range.from() - 1, range.from())
+        } else {
+            range
+        };
+
+        let word: String = text
+            .slice(range.from()..range.to())
+            .chars()
+            .filter(|&c| c != '_')
+            .collect();
+        let (radix, prefixed) = if word.starts_with("0x") {
+            (16, true)
+        } else if word.starts_with("0o") {
+            (8, true)
+        } else if word.starts_with("0b") {
+            (2, true)
+        } else {
+            (10, false)
+        };
+
+        let number = if prefixed { &word[2..] } else { &word };
+
+        let value = i128::from_str_radix(number, radix).ok()?;
+        if (value.is_positive() && value.leading_zeros() < 64)
+            || (value.is_negative() && value.leading_ones() < 64)
+        {
+            return None;
+        }
+
+        let value = value as i64;
+        Some(NumberIncrementor {
+            range,
+            value,
+            radix,
+            text,
+        })
+    }
+}
+
+impl<'a> Increment for NumberIncrementor<'a> {
+    fn increment(&self, amount: i64) -> (Range, Tendril) {
+        let old_text: Cow<str> = self.text.slice(self.range.from()..self.range.to()).into();
+        let old_length = old_text.len();
+        let new_value = self.value.wrapping_add(amount);
+
+        // Get separator indexes from right to left.
+        let separator_rtl_indexes: Vec<usize> = old_text
+            .chars()
+            .rev()
+            .enumerate()
+            .filter_map(|(i, c)| if c == '_' { Some(i) } else { None })
+            .collect();
+
+        let format_length = if self.radix == 10 {
+            match (self.value.is_negative(), new_value.is_negative()) {
+                (true, false) => old_length - 1,
+                (false, true) => old_length + 1,
+                _ => old_text.len(),
+            }
+        } else {
+            old_text.len() - 2
+        } - separator_rtl_indexes.len();
+
+        let mut new_text = match self.radix {
+            2 => format!("0b{:01$b}", new_value, format_length),
+            8 => format!("0o{:01$o}", new_value, format_length),
+            10 if old_text.starts_with('0') || old_text.starts_with("-0") => {
+                format!("{:01$}", new_value, format_length)
+            }
+            10 => format!("{}", new_value),
+            16 => {
+                let (lower_count, upper_count): (usize, usize) =
+                    old_text.chars().skip(2).fold((0, 0), |(lower, upper), c| {
+                        (
+                            lower + c.is_ascii_lowercase().then(|| 1).unwrap_or(0),
+                            upper + c.is_ascii_uppercase().then(|| 1).unwrap_or(0),
+                        )
+                    });
+                if upper_count > lower_count {
+                    format!("0x{:01$X}", new_value, format_length)
+                } else {
+                    format!("0x{:01$x}", new_value, format_length)
+                }
+            }
+            _ => unimplemented!("radix not supported: {}", self.radix),
+        };
+
+        // Add separators from original number.
+        for &rtl_index in &separator_rtl_indexes {
+            if rtl_index < new_text.len() {
+                let new_index = new_text.len() - rtl_index;
+                new_text.insert(new_index, '_');
+            }
+        }
+
+        // Add in additional separators if necessary.
+        if new_text.len() > old_length && !separator_rtl_indexes.is_empty() {
+            let spacing = match separator_rtl_indexes.as_slice() {
+                [.., b, a] => a - b - 1,
+                _ => separator_rtl_indexes[0],
+            };
+
+            let prefix_length = if self.radix == 10 { 0 } else { 2 };
+            if let Some(mut index) = new_text.find('_') {
+                while index - prefix_length > spacing {
+                    index -= spacing;
+                    new_text.insert(index, '_');
+                }
+            }
+        }
+
+        (self.range, new_text.into())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::Rope;
+
+    #[test]
+    fn test_decimal_at_point() {
+        let rope = Rope::from_str("Test text 12345 more text.");
+        let range = Range::point(12);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(10, 15),
+                value: 12345,
+                radix: 10,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_uppercase_hexadecimal_at_point() {
+        let rope = Rope::from_str("Test text 0x123ABCDEF more text.");
+        let range = Range::point(12);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(10, 21),
+                value: 0x123ABCDEF,
+                radix: 16,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_lowercase_hexadecimal_at_point() {
+        let rope = Rope::from_str("Test text 0xfa3b4e more text.");
+        let range = Range::point(12);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(10, 18),
+                value: 0xfa3b4e,
+                radix: 16,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_octal_at_point() {
+        let rope = Rope::from_str("Test text 0o1074312 more text.");
+        let range = Range::point(12);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(10, 19),
+                value: 0o1074312,
+                radix: 8,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_binary_at_point() {
+        let rope = Rope::from_str("Test text 0b10111010010101 more text.");
+        let range = Range::point(12);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(10, 26),
+                value: 0b10111010010101,
+                radix: 2,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_negative_decimal_at_point() {
+        let rope = Rope::from_str("Test text -54321 more text.");
+        let range = Range::point(12);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(10, 16),
+                value: -54321,
+                radix: 10,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_decimal_with_leading_zeroes_at_point() {
+        let rope = Rope::from_str("Test text 000045326 more text.");
+        let range = Range::point(12);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(10, 19),
+                value: 45326,
+                radix: 10,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_negative_decimal_cursor_on_minus_sign() {
+        let rope = Rope::from_str("Test text -54321 more text.");
+        let range = Range::point(10);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(10, 16),
+                value: -54321,
+                radix: 10,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_number_under_range_start_of_rope() {
+        let rope = Rope::from_str("100");
+        let range = Range::point(0);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(0, 3),
+                value: 100,
+                radix: 10,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_number_under_range_end_of_rope() {
+        let rope = Rope::from_str("100");
+        let range = Range::point(2);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(0, 3),
+                value: 100,
+                radix: 10,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_number_surrounded_by_punctuation() {
+        let rope = Rope::from_str(",100;");
+        let range = Range::point(1);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(1, 4),
+                value: 100,
+                radix: 10,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_not_a_number_point() {
+        let rope = Rope::from_str("Test text 45326 more text.");
+        let range = Range::point(6);
+        assert_eq!(NumberIncrementor::from_range(rope.slice(..), range), None);
+    }
+
+    #[test]
+    fn test_number_too_large_at_point() {
+        let rope = Rope::from_str("Test text 0xFFFFFFFFFFFFFFFFF more text.");
+        let range = Range::point(12);
+        assert_eq!(NumberIncrementor::from_range(rope.slice(..), range), None);
+    }
+
+    #[test]
+    fn test_number_cursor_one_right_of_number() {
+        let rope = Rope::from_str("100 ");
+        let range = Range::point(3);
+        assert_eq!(NumberIncrementor::from_range(rope.slice(..), range), None);
+    }
+
+    #[test]
+    fn test_number_cursor_one_left_of_number() {
+        let rope = Rope::from_str(" 100");
+        let range = Range::point(0);
+        assert_eq!(NumberIncrementor::from_range(rope.slice(..), range), None);
+    }
+
+    #[test]
+    fn test_increment_basic_decimal_numbers() {
+        let tests = [
+            ("100", 1, "101"),
+            ("100", -1, "99"),
+            ("99", 1, "100"),
+            ("100", 1000, "1100"),
+            ("100", -1000, "-900"),
+            ("-1", 1, "0"),
+            ("-1", 2, "1"),
+            ("1", -1, "0"),
+            ("1", -2, "-1"),
+        ];
+
+        for (original, amount, expected) in tests {
+            let rope = Rope::from_str(original);
+            let range = Range::point(0);
+            assert_eq!(
+                NumberIncrementor::from_range(rope.slice(..), range)
+                    .unwrap()
+                    .increment(amount)
+                    .1,
+                expected.into()
+            );
+        }
+    }
+
+    #[test]
+    fn test_increment_basic_hexadedimal_numbers() {
+        let tests = [
+            ("0x0100", 1, "0x0101"),
+            ("0x0100", -1, "0x00ff"),
+            ("0x0001", -1, "0x0000"),
+            ("0x0000", -1, "0xffffffffffffffff"),
+            ("0xffffffffffffffff", 1, "0x0000000000000000"),
+            ("0xffffffffffffffff", 2, "0x0000000000000001"),
+            ("0xffffffffffffffff", -1, "0xfffffffffffffffe"),
+            ("0xABCDEF1234567890", 1, "0xABCDEF1234567891"),
+            ("0xabcdef1234567890", 1, "0xabcdef1234567891"),
+        ];
+
+        for (original, amount, expected) in tests {
+            let rope = Rope::from_str(original);
+            let range = Range::point(0);
+            assert_eq!(
+                NumberIncrementor::from_range(rope.slice(..), range)
+                    .unwrap()
+                    .increment(amount)
+                    .1,
+                expected.into()
+            );
+        }
+    }
+
+    #[test]
+    fn test_increment_basic_octal_numbers() {
+        let tests = [
+            ("0o0107", 1, "0o0110"),
+            ("0o0110", -1, "0o0107"),
+            ("0o0001", -1, "0o0000"),
+            ("0o7777", 1, "0o10000"),
+            ("0o1000", -1, "0o0777"),
+            ("0o0107", 10, "0o0121"),
+            ("0o0000", -1, "0o1777777777777777777777"),
+            ("0o1777777777777777777777", 1, "0o0000000000000000000000"),
+            ("0o1777777777777777777777", 2, "0o0000000000000000000001"),
+            ("0o1777777777777777777777", -1, "0o1777777777777777777776"),
+        ];
+
+        for (original, amount, expected) in tests {
+            let rope = Rope::from_str(original);
+            let range = Range::point(0);
+            assert_eq!(
+                NumberIncrementor::from_range(rope.slice(..), range)
+                    .unwrap()
+                    .increment(amount)
+                    .1,
+                expected.into()
+            );
+        }
+    }
+
+    #[test]
+    fn test_increment_basic_binary_numbers() {
+        let tests = [
+            ("0b00000100", 1, "0b00000101"),
+            ("0b00000100", -1, "0b00000011"),
+            ("0b00000100", 2, "0b00000110"),
+            ("0b00000100", -2, "0b00000010"),
+            ("0b00000001", -1, "0b00000000"),
+            ("0b00111111", 10, "0b01001001"),
+            ("0b11111111", 1, "0b100000000"),
+            ("0b10000000", -1, "0b01111111"),
+            (
+                "0b0000",
+                -1,
+                "0b1111111111111111111111111111111111111111111111111111111111111111",
+            ),
+            (
+                "0b1111111111111111111111111111111111111111111111111111111111111111",
+                1,
+                "0b0000000000000000000000000000000000000000000000000000000000000000",
+            ),
+            (
+                "0b1111111111111111111111111111111111111111111111111111111111111111",
+                2,
+                "0b0000000000000000000000000000000000000000000000000000000000000001",
+            ),
+            (
+                "0b1111111111111111111111111111111111111111111111111111111111111111",
+                -1,
+                "0b1111111111111111111111111111111111111111111111111111111111111110",
+            ),
+        ];
+
+        for (original, amount, expected) in tests {
+            let rope = Rope::from_str(original);
+            let range = Range::point(0);
+            assert_eq!(
+                NumberIncrementor::from_range(rope.slice(..), range)
+                    .unwrap()
+                    .increment(amount)
+                    .1,
+                expected.into()
+            );
+        }
+    }
+
+    #[test]
+    fn test_increment_with_separators() {
+        let tests = [
+            ("999_999", 1, "1_000_000"),
+            ("1_000_000", -1, "999_999"),
+            ("-999_999", -1, "-1_000_000"),
+            ("0x0000_0000_0001", 0x1_ffff_0000, "0x0001_ffff_0001"),
+            ("0x0000_0000_0001", 0x1_ffff_0000, "0x0001_ffff_0001"),
+            ("0x0000_0000_0001", 0x1_ffff_0000, "0x0001_ffff_0001"),
+            ("0x0000_0000", -1, "0xffff_ffff_ffff_ffff"),
+            ("0x0000_0000_0000", -1, "0xffff_ffff_ffff_ffff"),
+            ("0b01111111_11111111", 1, "0b10000000_00000000"),
+            ("0b11111111_11111111", 1, "0b1_00000000_00000000"),
+        ];
+
+        for (original, amount, expected) in tests {
+            let rope = Rope::from_str(original);
+            let range = Range::point(0);
+            assert_eq!(
+                NumberIncrementor::from_range(rope.slice(..), range)
+                    .unwrap()
+                    .increment(amount)
+                    .1,
+                expected.into()
+            );
+        }
+    }
+}

helix-core/src/indent.rs

@@ -1,14 +1,179 @@
 use crate::{
-    find_first_non_whitespace_char,
+    chars::{char_is_line_ending, char_is_whitespace},
     syntax::{IndentQuery, LanguageConfiguration, Syntax},
-    tree_sitter::{Node, Tree},
+    tree_sitter::Node,
     Rope, RopeSlice,
 };
 
+/// Enum representing indentation style.
+///
+/// Only values 1-8 are valid for the `Spaces` variant.
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub enum IndentStyle {
+    Tabs,
+    Spaces(u8),
+}
+
+impl IndentStyle {
+    /// Creates an `IndentStyle` from an indentation string.
+    ///
+    /// For example, passing `"    "` (four spaces) will create `IndentStyle::Spaces(4)`.
+    #[allow(clippy::should_implement_trait)]
+    #[inline]
+    pub fn from_str(indent: &str) -> Self {
+        // XXX: do we care about validating the input more than this?  Probably not...?
+        debug_assert!(!indent.is_empty() && indent.len() <= 8);
+
+        if indent.starts_with(' ') {
+            IndentStyle::Spaces(indent.len() as u8)
+        } else {
+            IndentStyle::Tabs
+        }
+    }
+
+    #[inline]
+    pub fn as_str(&self) -> &'static str {
+        match *self {
+            IndentStyle::Tabs => "\t",
+            IndentStyle::Spaces(1) => " ",
+            IndentStyle::Spaces(2) => "  ",
+            IndentStyle::Spaces(3) => "   ",
+            IndentStyle::Spaces(4) => "    ",
+            IndentStyle::Spaces(5) => "     ",
+            IndentStyle::Spaces(6) => "      ",
+            IndentStyle::Spaces(7) => "       ",
+            IndentStyle::Spaces(8) => "        ",
+
+            // Unsupported indentation style.  This should never happen,
+            // but just in case fall back to two spaces.
+            IndentStyle::Spaces(n) => {
+                debug_assert!(n > 0 && n <= 8); // Always triggers. `debug_panic!()` wanted.
+                "  "
+            }
+        }
+    }
+}
+
+/// Attempts to detect the indentation style used in a document.
+///
+/// Returns the indentation style if the auto-detect confidence is
+/// reasonably high, otherwise returns `None`.
+pub fn auto_detect_indent_style(document_text: &Rope) -> Option<IndentStyle> {
+    // Build a histogram of the indentation *increases* between
+    // subsequent lines, ignoring lines that are all whitespace.
+    //
+    // Index 0 is for tabs, the rest are 1-8 spaces.
+    let histogram: [usize; 9] = {
+        let mut histogram = [0; 9];
+        let mut prev_line_is_tabs = false;
+        let mut prev_line_leading_count = 0usize;
+
+        // Loop through the lines, checking for and recording indentation
+        // increases as we go.
+        'outer: for line in document_text.lines().take(1000) {
+            let mut c_iter = line.chars();
+
+            // Is first character a tab or space?
+            let is_tabs = match c_iter.next() {
+                Some('\t') => true,
+                Some(' ') => false,
+
+                // Ignore blank lines.
+                Some(c) if char_is_line_ending(c) => continue,
+
+                _ => {
+                    prev_line_is_tabs = false;
+                    prev_line_leading_count = 0;
+                    continue;
+                }
+            };
+
+            // Count the line's total leading tab/space characters.
+            let mut leading_count = 1;
+            let mut count_is_done = false;
+            for c in c_iter {
+                match c {
+                    '\t' if is_tabs && !count_is_done => leading_count += 1,
+                    ' ' if !is_tabs && !count_is_done => leading_count += 1,
+
+                    // We stop counting if we hit whitespace that doesn't
+                    // qualify as indent or doesn't match the leading
+                    // whitespace, but we don't exit the loop yet because
+                    // we still want to determine if the line is blank.
+                    c if char_is_whitespace(c) => count_is_done = true,
+
+                    // Ignore blank lines.
+                    c if char_is_line_ending(c) => continue 'outer,
+
+                    _ => break,
+                }
+
+                // Bound the worst-case execution time for weird text files.
+                if leading_count > 256 {
+                    continue 'outer;
+                }
+            }
+
+            // If there was an increase in indentation over the previous
+            // line, update the histogram with that increase.
+            if (prev_line_is_tabs == is_tabs || prev_line_leading_count == 0)
+                && prev_line_leading_count < leading_count
+            {
+                if is_tabs {
+                    histogram[0] += 1;
+                } else {
+                    let amount = leading_count - prev_line_leading_count;
+                    if amount <= 8 {
+                        histogram[amount] += 1;
+                    }
+                }
+            }
+
+            // Store this line's leading whitespace info for use with
+            // the next line.
+            prev_line_is_tabs = is_tabs;
+            prev_line_leading_count = leading_count;
+        }
+
+        // Give more weight to tabs, because their presence is a very
+        // strong indicator.
+        histogram[0] *= 2;
+
+        histogram
+    };
+
+    // Find the most frequent indent, its frequency, and the frequency of
+    // the next-most frequent indent.
+    let indent = histogram
+        .iter()
+        .enumerate()
+        .max_by_key(|kv| kv.1)
+        .unwrap()
+        .0;
+    let indent_freq = histogram[indent];
+    let indent_freq_2 = *histogram
+        .iter()
+        .enumerate()
+        .filter(|kv| kv.0 != indent)
+        .map(|kv| kv.1)
+        .max()
+        .unwrap();
+
+    // Return the the auto-detected result if we're confident enough in its
+    // accuracy, based on some heuristics.
+    if indent_freq >= 1 && (indent_freq_2 as f64 / indent_freq as f64) < 0.66 {
+        Some(match indent {
+            0 => IndentStyle::Tabs,
+            _ => IndentStyle::Spaces(indent as u8),
+        })
+    } else {
+        None
+    }
+}
+
 /// To determine indentation of a newly inserted line, figure out the indentation at the last col
 /// of the previous line.
-
+pub fn indent_level_for_line(line: RopeSlice, tab_width: usize) -> usize {
-fn indent_level_for_line(line: RopeSlice, tab_width: usize) -> usize {
     let mut len = 0;
     for ch in line.chars() {
         match ch {
@@ -43,77 +208,59 @@ fn get_highest_syntax_node_at_bytepos(syntax: &Syntax, pos: usize) -> Option<Nod
     Some(node)
 }
 
-fn calculate_indentation(query: &IndentQuery, node: Option<Node>, newline: bool) -> usize {
+/// Calculate the indentation at a given treesitter node.
-    // NOTE: can't use contains() on query because of comparing Vec<String> and &str
+/// If newline is false, then any "indent" nodes on the line are ignored ("outdent" still applies).
-    // https://doc.rust-lang.org/std/vec/struct.Vec.html#method.contains
+/// This is because the indentation is only increased starting at the second line of the node.
-
+fn calculate_indentation(
-    let mut increment: i32 = 0;
+    query: &IndentQuery,
+    node: Option<Node>,
+    line: usize,
+    newline: bool,
+) -> usize {
+    let mut increment: isize = 0;
 
     let mut node = match node {
         Some(node) => node,
         None => return 0,
     };
 
-    let mut prev_start = node.start_position().row;
+    let mut current_line = line;
+    let mut consider_indent = newline;
+    let mut increment_from_line: isize = 0;
 
-    // if we're calculating indentation for a brand new line then the current node will become the
+    loop {
-    // parent node. We need to take it's indentation level into account too.
+        let node_kind = node.kind();
-    let node_kind = node.kind();
+        let start = node.start_position().row;
-    if newline && query.indent.contains(node_kind) {
+        if current_line != start {
-        increment += 1;
+            // Indent/dedent by at most one per line:
-    }
+            // .map(|a| {       <-- ({ is two scopes
-
+            //     let len = 1; <-- indents one level
-    while let Some(parent) = node.parent() {
+            // })               <-- }) is two scopes
-        let parent_kind = parent.kind();
+            if consider_indent || increment_from_line < 0 {
-        let start = parent.start_position().row;
+                increment += increment_from_line.signum();
-
+            }
-        // detect deeply nested indents in the same line
+            increment_from_line = 0;
-        // .map(|a| {       <-- ({ is two scopes
+            current_line = start;
-        //     let len = 1; <-- indents one level
+            consider_indent = true;
-        // })               <-- }) is two scopes
-        let starts_same_line = start == prev_start;
-
-        if query.outdent.contains(node.kind()) && !starts_same_line {
-            // we outdent by skipping the rules for the current level and jumping up
-            // node = parent;
-            increment -= 1;
-            // continue;
         }
 
-        if query.indent.contains(parent_kind) // && not_first_or_last_sibling
+        if query.outdent.contains(node_kind) {
-            && !starts_same_line
+            increment_from_line -= 1;
-        {
+        }
-            // println!("is_scope {}", parent_kind);
+        if query.indent.contains(node_kind) {
-            prev_start = start;
+            increment_from_line += 1;
-            increment += 1
         }
 
-        // if last_scope && increment > 0 && ...{ ignore }
+        if let Some(parent) = node.parent() {
-
+            node = parent;
-        node = parent;
+        } else {
+            break;
+        }
     }
-
+    if consider_indent || increment_from_line < 0 {
-    assert!(increment >= 0);
+        increment += increment_from_line.signum();
-
+    }
-    increment as usize
+    increment.max(0) as usize
-}
-
-fn suggested_indent_for_line(
-    language_config: &LanguageConfiguration,
-    syntax: Option<&Syntax>,
-    text: RopeSlice,
-    line_num: usize,
-    tab_width: usize,
-) -> usize {
-    let line = text.line(line_num);
-    let current = indent_level_for_line(line, tab_width);
-
-    if let Some(start) = find_first_non_whitespace_char(text, line_num) {
-        return suggested_indent_for_pos(Some(language_config), syntax, text, start, false);
-    };
-
-    // if the line is blank, indent should be zero
-    0
 }
 
 // TODO: two usecases: if we are triggering this for a new, blank line:
@@ -124,29 +271,52 @@ pub fn suggested_indent_for_pos(
     syntax: Option<&Syntax>,
     text: RopeSlice,
     pos: usize,
+    line: usize,
     new_line: bool,
-) -> usize {
+) -> Option<usize> {
     if let (Some(query), Some(syntax)) = (
         language_config.and_then(|config| config.indent_query()),
         syntax,
     ) {
         let byte_start = text.char_to_byte(pos);
         let node = get_highest_syntax_node_at_bytepos(syntax, byte_start);
-
-        // let config = load indentation query config from Syntax(should contain language_config)
-
         // TODO: special case for comments
         // TODO: if preserve_leading_whitespace
-        calculate_indentation(query, node, new_line)
+        Some(calculate_indentation(query, node, line, new_line))
     } else {
-        // TODO: heuristics for non-tree sitter grammars
+        None
-        0
     }
 }
 
+pub fn get_scopes(syntax: Option<&Syntax>, text: RopeSlice, pos: usize) -> Vec<&'static str> {
+    let mut scopes = Vec::new();
+    if let Some(syntax) = syntax {
+        let pos = text.char_to_byte(pos);
+        let mut node = match syntax
+            .tree()
+            .root_node()
+            .descendant_for_byte_range(pos, pos)
+        {
+            Some(node) => node,
+            None => return scopes,
+        };
+
+        scopes.push(node.kind());
+
+        while let Some(parent) = node.parent() {
+            scopes.push(parent.kind());
+            node = parent;
+        }
+    }
+
+    scopes.reverse();
+    scopes
+}
+
 #[cfg(test)]
 mod test {
     use super::*;
+    use crate::Rope;
 
     #[test]
     fn test_indent_level() {
@@ -247,29 +417,34 @@ where
         );
 
         let doc = Rope::from(doc);
+        use crate::diagnostic::Severity;
         use crate::syntax::{
-            Configuration, IndentationConfiguration, Lang, LanguageConfiguration, Loader,
+            Configuration, IndentationConfiguration, LanguageConfiguration, Loader,
         };
         use once_cell::sync::OnceCell;
-        let loader = Loader::new(
+        let loader = Loader::new(Configuration {
-            Configuration {
+            language: vec![LanguageConfiguration {
-                language: vec![LanguageConfiguration {
+                scope: "source.rust".to_string(),
-                    scope: "source.rust".to_string(),
+                file_types: vec!["rs".to_string()],
-                    file_types: vec!["rs".to_string()],
+                shebangs: vec![],
-                    language_id: Lang::Rust,
+                language_id: "Rust".to_string(),
-                    highlight_config: OnceCell::new(),
+                highlight_config: OnceCell::new(),
-                    //
+                config: None,
-                    roots: vec![],
+                //
-                    language_server: None,
+                injection_regex: None,
-                    indent: Some(IndentationConfiguration {
+                roots: vec![],
-                        tab_width: 4,
+                comment_token: None,
-                        unit: String::from("    "),
+                auto_format: false,
-                    }),
+                diagnostic_severity: Severity::Warning,
-                    indent_query: OnceCell::new(),
+                language_server: None,
-                }],
+                indent: Some(IndentationConfiguration {
-            },
+                    tab_width: 4,
-            Vec::new(),
+                    unit: String::from("    "),
-        );
+                }),
+                indent_query: OnceCell::new(),
+                textobject_query: OnceCell::new(),
+            }],
+        });
 
         // set runtime path so we can find the queries
         let mut runtime = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
@@ -284,14 +459,23 @@ where
 
         for i in 0..doc.len_lines() {
             let line = text.line(i);
-            let indent = indent_level_for_line(line, tab_width);
+            if let Some(pos) = crate::find_first_non_whitespace_char(line) {
-            assert_eq!(
+                let indent = indent_level_for_line(line, tab_width);
-                suggested_indent_for_line(&language_config, Some(&syntax), text, i, tab_width),
+                assert_eq!(
-                indent,
+                    suggested_indent_for_pos(
-                "line {}: {}",
+                        Some(&language_config),
-                i,
+                        Some(&syntax),
-                line
+                        text,
-            );
+                        text.line_to_char(i) + pos,
+                        i,
+                        false
+                    ),
+                    Some(indent),
+                    "line {}: \"{}\"",
+                    i,
+                    line
+                );
+            }
         }
     }
 }

helix-core/src/lib.rs

@@ -1,61 +1,110 @@
-#![allow(unused)]
+pub use encoding_rs as encoding;
+
 pub mod auto_pairs;
+pub mod chars;
 pub mod comment;
 pub mod diagnostic;
+pub mod diff;
 pub mod graphemes;
-mod history;
+pub mod history;
+pub mod increment;
 pub mod indent;
+pub mod line_ending;
 pub mod macros;
 pub mod match_brackets;
 pub mod movement;
 pub mod object;
+pub mod path;
 mod position;
 pub mod register;
 pub mod search;
 pub mod selection;
+pub mod shellwords;
 mod state;
+pub mod surround;
 pub mod syntax;
+pub mod textobject;
 mod transaction;
-pub mod words;
 
-pub(crate) fn find_first_non_whitespace_char2(line: RopeSlice) -> Option<usize> {
+pub mod unicode {
-    // find first non-whitespace char
+    pub use unicode_general_category as category;
-    for (start, ch) in line.chars().enumerate() {
+    pub use unicode_segmentation as segmentation;
-        // TODO: could use memchr with chunks?
+    pub use unicode_width as width;
-        if ch != ' ' && ch != '\t' && ch != '\n' {
+}
-            return Some(start);
+
+static RUNTIME_DIR: once_cell::sync::Lazy<std::path::PathBuf> =
+    once_cell::sync::Lazy::new(runtime_dir);
+
+pub fn find_first_non_whitespace_char(line: RopeSlice) -> Option<usize> {
+    line.chars().position(|ch| !ch.is_whitespace())
+}
+
+/// Find project root.
+///
+/// Order of detection:
+/// * Top-most folder containing a root marker in current git repository
+/// * Git repostory root if no marker detected
+/// * Top-most folder containing a root marker if not git repository detected
+/// * Current working directory as fallback
+pub fn find_root(root: Option<&str>, root_markers: &[String]) -> Option<std::path::PathBuf> {
+    let current_dir = std::env::current_dir().expect("unable to determine current directory");
+
+    let root = match root {
+        Some(root) => {
+            let root = std::path::Path::new(root);
+            if root.is_absolute() {
+                root.to_path_buf()
+            } else {
+                current_dir.join(root)
+            }
+        }
+        None => current_dir.clone(),
+    };
+
+    let mut top_marker = None;
+    for ancestor in root.ancestors() {
+        for marker in root_markers {
+            if ancestor.join(marker).exists() {
+                top_marker = Some(ancestor);
+                break;
+            }
+        }
+        // don't go higher than repo
+        if ancestor.join(".git").is_dir() {
+            // Use workspace if detected from marker
+            return Some(top_marker.unwrap_or(ancestor).to_path_buf());
         }
     }
 
-    None
+    // In absence of git repo, use workspace if detected
-}
+    if top_marker.is_some() {
-pub(crate) fn find_first_non_whitespace_char(text: RopeSlice, line_num: usize) -> Option<usize> {
+        top_marker.map(|a| a.to_path_buf())
-    let line = text.line(line_num);
+    } else {
-    let mut start = text.line_to_char(line_num);
+        Some(current_dir)
-
-    // find first non-whitespace char
-    for ch in line.chars() {
-        // TODO: could use memchr with chunks?
-        if ch != ' ' && ch != '\t' && ch != '\n' {
-            return Some(start);
-        }
-        start += 1;
     }
-
-    None
 }
 
-#[cfg(not(embed_runtime))]
 pub fn runtime_dir() -> std::path::PathBuf {
-    // runtime env var || dir where binary is located
+    if let Ok(dir) = std::env::var("HELIX_RUNTIME") {
-    std::env::var("HELIX_RUNTIME")
+        return dir.into();
-        .map(|path| path.into())
+    }
-        .unwrap_or_else(|_| {
+
-            std::env::current_exe()
+    const RT_DIR: &str = "runtime";
-                .ok()
+    let conf_dir = config_dir().join(RT_DIR);
-                .and_then(|path| path.parent().map(|path| path.to_path_buf()))
+    if conf_dir.exists() {
-                .unwrap()
+        return conf_dir;
-        })
+    }
+
+    if let Ok(dir) = std::env::var("CARGO_MANIFEST_DIR") {
+        // this is the directory of the crate being run by cargo, we need the workspace path so we take the parent
+        return std::path::PathBuf::from(dir).parent().unwrap().join(RT_DIR);
+    }
+
+    // fallback to location of the executable being run
+    std::env::current_exe()
+        .ok()
+        .and_then(|path| path.parent().map(|path| path.to_path_buf().join(RT_DIR)))
+        .unwrap()
 }
 
 pub fn config_dir() -> std::path::PathBuf {
@@ -74,22 +123,108 @@ pub fn cache_dir() -> std::path::PathBuf {
     path
 }
 
+// right overrides left
+pub fn merge_toml_values(left: toml::Value, right: toml::Value) -> toml::Value {
+    use toml::Value;
+
+    fn get_name(v: &Value) -> Option<&str> {
+        v.get("name").and_then(Value::as_str)
+    }
+
+    match (left, right) {
+        (Value::Array(mut left_items), Value::Array(right_items)) => {
+            left_items.reserve(right_items.len());
+            for rvalue in right_items {
+                let lvalue = get_name(&rvalue)
+                    .and_then(|rname| left_items.iter().position(|v| get_name(v) == Some(rname)))
+                    .map(|lpos| left_items.remove(lpos));
+                let mvalue = match lvalue {
+                    Some(lvalue) => merge_toml_values(lvalue, rvalue),
+                    None => rvalue,
+                };
+                left_items.push(mvalue);
+            }
+            Value::Array(left_items)
+        }
+        (Value::Table(mut left_map), Value::Table(right_map)) => {
+            for (rname, rvalue) in right_map {
+                match left_map.remove(&rname) {
+                    Some(lvalue) => {
+                        let merged_value = merge_toml_values(lvalue, rvalue);
+                        left_map.insert(rname, merged_value);
+                    }
+                    None => {
+                        left_map.insert(rname, rvalue);
+                    }
+                }
+            }
+            Value::Table(left_map)
+        }
+        // Catch everything else we didn't handle, and use the right value
+        (_, value) => value,
+    }
+}
+
+#[cfg(test)]
+mod merge_toml_tests {
+    use super::merge_toml_values;
+
+    #[test]
+    fn language_tomls() {
+        use toml::Value;
+
+        const USER: &str = "
+        [[language]]
+        name = \"nix\"
+        test = \"bbb\"
+        indent = { tab-width = 4, unit = \"    \", test = \"aaa\" }
+        ";
+
+        let base: Value = toml::from_slice(include_bytes!("../../languages.toml"))
+            .expect("Couldn't parse built-in languages config");
+        let user: Value = toml::from_str(USER).unwrap();
+
+        let merged = merge_toml_values(base, user);
+        let languages = merged.get("language").unwrap().as_array().unwrap();
+        let nix = languages
+            .iter()
+            .find(|v| v.get("name").unwrap().as_str().unwrap() == "nix")
+            .unwrap();
+        let nix_indent = nix.get("indent").unwrap();
+
+        // We changed tab-width and unit in indent so check them if they are the new values
+        assert_eq!(
+            nix_indent.get("tab-width").unwrap().as_integer().unwrap(),
+            4
+        );
+        assert_eq!(nix_indent.get("unit").unwrap().as_str().unwrap(), "    ");
+        // We added a new keys, so check them
+        assert_eq!(nix.get("test").unwrap().as_str().unwrap(), "bbb");
+        assert_eq!(nix_indent.get("test").unwrap().as_str().unwrap(), "aaa");
+        // We didn't change comment-token so it should be same
+        assert_eq!(nix.get("comment-token").unwrap().as_str().unwrap(), "#");
+    }
+}
+
+pub use etcetera::home_dir;
+
 use etcetera::base_strategy::{choose_base_strategy, BaseStrategy};
 
-pub use ropey::{Rope, RopeSlice};
+pub use ropey::{Rope, RopeBuilder, RopeSlice};
 
 pub use tendril::StrTendril as Tendril;
 
 #[doc(inline)]
 pub use {regex, tree_sitter};
 
-pub use position::{coords_at_pos, pos_at_coords, Position};
+pub use graphemes::RopeGraphemes;
+pub use position::{coords_at_pos, pos_at_coords, visual_coords_at_pos, Position};
 pub use selection::{Range, Selection};
 pub use smallvec::SmallVec;
 pub use syntax::Syntax;
 
 pub use diagnostic::Diagnostic;
-pub use history::History;
 pub use state::State;
 
+pub use line_ending::{LineEnding, DEFAULT_LINE_ENDING};
 pub use transaction::{Assoc, Change, ChangeSet, Operation, Transaction};

helix-core/src/line_ending.rs

@@ -0,0 +1,265 @@
+use crate::{Rope, RopeSlice};
+
+#[cfg(target_os = "windows")]
+pub const DEFAULT_LINE_ENDING: LineEnding = LineEnding::Crlf;
+#[cfg(not(target_os = "windows"))]
+pub const DEFAULT_LINE_ENDING: LineEnding = LineEnding::LF;
+
+/// Represents one of the valid Unicode line endings.
+#[derive(PartialEq, Copy, Clone, Debug)]
+pub enum LineEnding {
+    Crlf, // CarriageReturn followed by LineFeed
+    LF,   // U+000A -- LineFeed
+    VT,   // U+000B -- VerticalTab
+    FF,   // U+000C -- FormFeed
+    CR,   // U+000D -- CarriageReturn
+    Nel,  // U+0085 -- NextLine
+    LS,   // U+2028 -- Line Separator
+    PS,   // U+2029 -- ParagraphSeparator
+}
+
+impl LineEnding {
+    #[inline]
+    pub const fn len_chars(&self) -> usize {
+        match self {
+            Self::Crlf => 2,
+            _ => 1,
+        }
+    }
+
+    #[inline]
+    pub const fn as_str(&self) -> &'static str {
+        match self {
+            Self::Crlf => "\u{000D}\u{000A}",
+            Self::LF => "\u{000A}",
+            Self::VT => "\u{000B}",
+            Self::FF => "\u{000C}",
+            Self::CR => "\u{000D}",
+            Self::Nel => "\u{0085}",
+            Self::LS => "\u{2028}",
+            Self::PS => "\u{2029}",
+        }
+    }
+
+    #[inline]
+    pub const fn from_char(ch: char) -> Option<LineEnding> {
+        match ch {
+            '\u{000A}' => Some(LineEnding::LF),
+            '\u{000B}' => Some(LineEnding::VT),
+            '\u{000C}' => Some(LineEnding::FF),
+            '\u{000D}' => Some(LineEnding::CR),
+            '\u{0085}' => Some(LineEnding::Nel),
+            '\u{2028}' => Some(LineEnding::LS),
+            '\u{2029}' => Some(LineEnding::PS),
+            // Not a line ending
+            _ => None,
+        }
+    }
+
+    // Normally we'd want to implement the FromStr trait, but in this case
+    // that would force us into a different return type than from_char or
+    // or from_rope_slice, which would be weird.
+    #[allow(clippy::should_implement_trait)]
+    #[inline]
+    pub fn from_str(g: &str) -> Option<LineEnding> {
+        match g {
+            "\u{000D}\u{000A}" => Some(LineEnding::Crlf),
+            "\u{000A}" => Some(LineEnding::LF),
+            "\u{000B}" => Some(LineEnding::VT),
+            "\u{000C}" => Some(LineEnding::FF),
+            "\u{000D}" => Some(LineEnding::CR),
+            "\u{0085}" => Some(LineEnding::Nel),
+            "\u{2028}" => Some(LineEnding::LS),
+            "\u{2029}" => Some(LineEnding::PS),
+            // Not a line ending
+            _ => None,
+        }
+    }
+
+    #[inline]
+    pub fn from_rope_slice(g: &RopeSlice) -> Option<LineEnding> {
+        if let Some(text) = g.as_str() {
+            LineEnding::from_str(text)
+        } else {
+            // Non-contiguous, so it can't be a line ending.
+            // Specifically, Ropey guarantees that CRLF is always
+            // contiguous.  And the remaining line endings are all
+            // single `char`s, and therefore trivially contiguous.
+            None
+        }
+    }
+}
+
+#[inline]
+pub fn str_is_line_ending(s: &str) -> bool {
+    LineEnding::from_str(s).is_some()
+}
+
+/// Attempts to detect what line ending the passed document uses.
+pub fn auto_detect_line_ending(doc: &Rope) -> Option<LineEnding> {
+    // Return first matched line ending. Not all possible line endings
+    // are being matched, as they might be special-use only
+    for line in doc.lines().take(100) {
+        match get_line_ending(&line) {
+            None | Some(LineEnding::VT) | Some(LineEnding::FF) | Some(LineEnding::PS) => {}
+            ending => return ending,
+        }
+    }
+    None
+}
+
+/// Returns the passed line's line ending, if any.
+pub fn get_line_ending(line: &RopeSlice) -> Option<LineEnding> {
+    // Last character as str.
+    let g1 = line
+        .slice(line.len_chars().saturating_sub(1)..)
+        .as_str()
+        .unwrap();
+
+    // Last two characters as str, or empty str if they're not contiguous.
+    // It's fine to punt on the non-contiguous case, because Ropey guarantees
+    // that CRLF is always contiguous.
+    let g2 = line
+        .slice(line.len_chars().saturating_sub(2)..)
+        .as_str()
+        .unwrap_or("");
+
+    // First check the two-character case for CRLF, then check the single-character case.
+    LineEnding::from_str(g2).or_else(|| LineEnding::from_str(g1))
+}
+
+/// Returns the passed line's line ending, if any.
+pub fn get_line_ending_of_str(line: &str) -> Option<LineEnding> {
+    if line.ends_with("\u{000D}\u{000A}") {
+        Some(LineEnding::Crlf)
+    } else if line.ends_with('\u{000A}') {
+        Some(LineEnding::LF)
+    } else if line.ends_with('\u{000B}') {
+        Some(LineEnding::VT)
+    } else if line.ends_with('\u{000C}') {
+        Some(LineEnding::FF)
+    } else if line.ends_with('\u{000D}') {
+        Some(LineEnding::CR)
+    } else if line.ends_with('\u{0085}') {
+        Some(LineEnding::Nel)
+    } else if line.ends_with('\u{2028}') {
+        Some(LineEnding::LS)
+    } else if line.ends_with('\u{2029}') {
+        Some(LineEnding::PS)
+    } else {
+        None
+    }
+}
+
+/// Returns the char index of the end of the given line, not including its line ending.
+pub fn line_end_char_index(slice: &RopeSlice, line: usize) -> usize {
+    slice.line_to_char(line + 1)
+        - get_line_ending(&slice.line(line))
+            .map(|le| le.len_chars())
+            .unwrap_or(0)
+}
+
+/// Fetches line `line_idx` from the passed rope slice, sans any line ending.
+pub fn line_without_line_ending<'a>(slice: &'a RopeSlice, line_idx: usize) -> RopeSlice<'a> {
+    let start = slice.line_to_char(line_idx);
+    let end = line_end_char_index(slice, line_idx);
+    slice.slice(start..end)
+}
+
+/// Returns the char index of the end of the given RopeSlice, not including
+/// any final line ending.
+pub fn rope_end_without_line_ending(slice: &RopeSlice) -> usize {
+    slice.len_chars() - get_line_ending(slice).map(|le| le.len_chars()).unwrap_or(0)
+}
+
+#[cfg(test)]
+mod line_ending_tests {
+    use super::*;
+
+    #[test]
+    fn line_ending_autodetect() {
+        assert_eq!(
+            auto_detect_line_ending(&Rope::from_str("\n")),
+            Some(LineEnding::LF)
+        );
+        assert_eq!(
+            auto_detect_line_ending(&Rope::from_str("\r\n")),
+            Some(LineEnding::Crlf)
+        );
+        assert_eq!(auto_detect_line_ending(&Rope::from_str("hello")), None);
+        assert_eq!(auto_detect_line_ending(&Rope::from_str("")), None);
+        assert_eq!(
+            auto_detect_line_ending(&Rope::from_str("hello\nhelix\r\n")),
+            Some(LineEnding::LF)
+        );
+        assert_eq!(
+            auto_detect_line_ending(&Rope::from_str("a formfeed\u{000C}")),
+            None
+        );
+        assert_eq!(
+            auto_detect_line_ending(&Rope::from_str("\n\u{000A}\n \u{000A}")),
+            Some(LineEnding::LF)
+        );
+        assert_eq!(
+            auto_detect_line_ending(&Rope::from_str(
+                "a formfeed\u{000C} with a\u{000C} linefeed\u{000A}"
+            )),
+            Some(LineEnding::LF)
+        );
+        assert_eq!(auto_detect_line_ending(&Rope::from_str("a formfeed\u{000C} with a\u{000C} carriage return linefeed\u{000D}\u{000A} and a linefeed\u{000A}")), Some(LineEnding::Crlf));
+    }
+
+    #[test]
+    fn str_to_line_ending() {
+        assert_eq!(LineEnding::from_str("\r"), Some(LineEnding::CR));
+        assert_eq!(LineEnding::from_str("\n"), Some(LineEnding::LF));
+        assert_eq!(LineEnding::from_str("\r\n"), Some(LineEnding::Crlf));
+        assert_eq!(LineEnding::from_str("hello\n"), None);
+    }
+
+    #[test]
+    fn rope_slice_to_line_ending() {
+        let r = Rope::from_str("hello\r\n");
+        assert_eq!(
+            LineEnding::from_rope_slice(&r.slice(5..6)),
+            Some(LineEnding::CR)
+        );
+        assert_eq!(
+            LineEnding::from_rope_slice(&r.slice(6..7)),
+            Some(LineEnding::LF)
+        );
+        assert_eq!(
+            LineEnding::from_rope_slice(&r.slice(5..7)),
+            Some(LineEnding::Crlf)
+        );
+        assert_eq!(LineEnding::from_rope_slice(&r.slice(..)), None);
+    }
+
+    #[test]
+    fn get_line_ending_rope_slice() {
+        let r = Rope::from_str("Hello\rworld\nhow\r\nare you?");
+        assert_eq!(get_line_ending(&r.slice(..6)), Some(LineEnding::CR));
+        assert_eq!(get_line_ending(&r.slice(..12)), Some(LineEnding::LF));
+        assert_eq!(get_line_ending(&r.slice(..17)), Some(LineEnding::Crlf));
+        assert_eq!(get_line_ending(&r.slice(..)), None);
+    }
+
+    #[test]
+    fn get_line_ending_str() {
+        let text = "Hello\rworld\nhow\r\nare you?";
+        assert_eq!(get_line_ending_of_str(&text[..6]), Some(LineEnding::CR));
+        assert_eq!(get_line_ending_of_str(&text[..12]), Some(LineEnding::LF));
+        assert_eq!(get_line_ending_of_str(&text[..17]), Some(LineEnding::Crlf));
+        assert_eq!(get_line_ending_of_str(&text[..]), None);
+    }
+
+    #[test]
+    fn line_end_char_index_rope_slice() {
+        let r = Rope::from_str("Hello\rworld\nhow\r\nare you?");
+        let s = &r.slice(..);
+        assert_eq!(line_end_char_index(s, 0), 5);
+        assert_eq!(line_end_char_index(s, 1), 11);
+        assert_eq!(line_end_char_index(s, 2), 15);
+        assert_eq!(line_end_char_index(s, 3), 25);
+    }
+}

helix-core/src/match_brackets.rs

@@ -1,47 +1,92 @@
-use crate::{Range, Rope, Selection, Syntax};
+use tree_sitter::Node;
 
-const PAIRS: &[(char, char)] = &[('(', ')'), ('{', '}'), ('[', ']'), ('<', '>')];
+use crate::{Rope, Syntax};
-// limit matching pairs to only ( ) { } [ ] < >
 
+const PAIRS: &[(char, char)] = &[
+    ('(', ')'),
+    ('{', '}'),
+    ('[', ']'),
+    ('<', '>'),
+    ('\'', '\''),
+    ('\"', '\"'),
+];
+
+// limit matching pairs to only ( ) { } [ ] < > ' ' " "
+
+// Returns the position of the matching bracket under cursor.
+//
+// If the cursor is one the opening bracket, the position of
+// the closing bracket is returned. If the cursor in the closing
+// bracket, the position of the opening bracket is returned.
+//
+// If the cursor is not on a bracket, `None` is returned.
 #[must_use]
-pub fn find(syntax: &Syntax, doc: &Rope, pos: usize) -> Option<usize> {
+pub fn find_matching_bracket(syntax: &Syntax, doc: &Rope, pos: usize) -> Option<usize> {
+    if pos >= doc.len_chars() || !is_valid_bracket(doc.char(pos)) {
+        return None;
+    }
+    find_pair(syntax, doc, pos, false)
+}
+
+// Returns the position of the bracket that is closing the current scope.
+//
+// If the cursor is on an opening or closing bracket, the function
+// behaves equivalent to [`find_matching_bracket`].
+//
+// If the cursor position is within a scope, the function searches
+// for the surrounding scope that is surrounded by brackets and
+// returns the position of the closing bracket for that scope.
+//
+// If no surrounding scope is found, the function returns `None`.
+#[must_use]
+pub fn find_matching_bracket_fuzzy(syntax: &Syntax, doc: &Rope, pos: usize) -> Option<usize> {
+    find_pair(syntax, doc, pos, true)
+}
+
+fn find_pair(syntax: &Syntax, doc: &Rope, pos: usize, traverse_parents: bool) -> Option<usize> {
     let tree = syntax.tree();
+    let pos = doc.char_to_byte(pos);
 
-    let byte_pos = doc.char_to_byte(pos);
+    let mut node = tree.root_node().named_descendant_for_byte_range(pos, pos)?;
 
-    // most naive implementation: find the innermost syntax node, if we're at the edge of a node,
+    loop {
-    // return the other edge.
+        let (start_byte, end_byte) = surrounding_bytes(doc, &node)?;
+        let (start_char, end_char) = (doc.byte_to_char(start_byte), doc.byte_to_char(end_byte));
 
-    let mut node = match tree
+        if is_valid_pair(doc, start_char, end_char) {
-        .root_node()
+            if end_byte == pos {
-        .named_descendant_for_byte_range(byte_pos, byte_pos)
+                return Some(start_char);
-    {
+            }
-        Some(node) => node,
+            // We return the end char if the cursor is either on the start char
-        None => return None,
+            // or at some arbitrary position between start and end char.
-    };
-
-    if node.is_error() {
-        return None;
-    }
-
-    let start_byte = node.start_byte();
-    let len = doc.len_bytes();
-    if start_byte >= len {
-        return None;
-    }
-    let end_byte = node.end_byte() - 1; // it's end exclusive
-    let start_char = doc.byte_to_char(start_byte);
-    let end_char = doc.byte_to_char(end_byte);
-
-    if PAIRS.contains(&(doc.char(start_char), doc.char(end_char))) {
-        if start_byte == byte_pos {
             return Some(end_char);
         }
 
-        if end_byte == byte_pos {
+        if traverse_parents {
-            return Some(start_char);
+            node = node.parent()?;
+        } else {
+            return None;
         }
     }
-
+}
-    None
+
+fn is_valid_bracket(c: char) -> bool {
+    PAIRS.iter().any(|(l, r)| *l == c || *r == c)
+}
+
+fn is_valid_pair(doc: &Rope, start_char: usize, end_char: usize) -> bool {
+    PAIRS.contains(&(doc.char(start_char), doc.char(end_char)))
+}
+
+fn surrounding_bytes(doc: &Rope, node: &Node) -> Option<(usize, usize)> {
+    let len = doc.len_bytes();
+
+    let start_byte = node.start_byte();
+    let end_byte = node.end_byte().saturating_sub(1);
+
+    if start_byte >= len || end_byte >= len {
+        return None;
+    }
+
+    Some((start_byte, end_byte))
 }

helix-core/src/object.rs

@@ -1,12 +1,11 @@
 use crate::{Range, RopeSlice, Selection, Syntax};
-use smallvec::smallvec;
 
 // TODO: to contract_selection we'd need to store the previous ranges before expand.
 // Maybe just contract to the first child node?
 pub fn expand_selection(syntax: &Syntax, text: RopeSlice, selection: &Selection) -> Selection {
     let tree = syntax.tree();
 
-    selection.transform(|range| {
+    selection.clone().transform(|range| {
         let from = text.char_to_byte(range.from());
         let to = text.char_to_byte(range.to());
 
@@ -14,8 +13,13 @@ pub fn expand_selection(syntax: &Syntax, text: RopeSlice, selection: &Selection)
         let parent = match tree
             .root_node()
             .descendant_for_byte_range(from, to)
-            .and_then(|node| node.parent())
+            .and_then(|node| {
-        {
+                if node.child_count() == 0 || (node.start_byte() == from && node.end_byte() == to) {
+                    node.parent()
+                } else {
+                    Some(node)
+                }
+            }) {
             Some(parent) => parent,
             None => return range,
         };

helix-core/src/path.rs

@@ -0,0 +1,92 @@
+use std::path::{Component, Path, PathBuf};
+
+/// Replaces users home directory from `path` with tilde `~` if the directory
+/// is available, otherwise returns the path unchanged.
+pub fn fold_home_dir(path: &Path) -> PathBuf {
+    if let Ok(home) = super::home_dir() {
+        if path.starts_with(&home) {
+            // it's ok to unwrap, the path starts with home dir
+            return PathBuf::from("~").join(path.strip_prefix(&home).unwrap());
+        }
+    }
+
+    path.to_path_buf()
+}
+
+/// Expands tilde `~` into users home directory if avilable, otherwise returns the path
+/// unchanged. The tilde will only be expanded when present as the first component of the path
+/// and only slash follows it.
+pub fn expand_tilde(path: &Path) -> PathBuf {
+    let mut components = path.components().peekable();
+    if let Some(Component::Normal(c)) = components.peek() {
+        if c == &"~" {
+            if let Ok(home) = super::home_dir() {
+                // it's ok to unwrap, the path starts with `~`
+                return home.join(path.strip_prefix("~").unwrap());
+            }
+        }
+    }
+
+    path.to_path_buf()
+}
+
+/// Normalize a path, removing things like `.` and `..`.
+///
+/// CAUTION: This does not resolve symlinks (unlike
+/// [`std::fs::canonicalize`]). This may cause incorrect or surprising
+/// behavior at times. This should be used carefully. Unfortunately,
+/// [`std::fs::canonicalize`] can be hard to use correctly, since it can often
+/// fail, or on Windows returns annoying device paths. This is a problem Cargo
+/// needs to improve on.
+/// Copied from cargo: <https://github.com/rust-lang/cargo/blob/070e459c2d8b79c5b2ac5218064e7603329c92ae/crates/cargo-util/src/paths.rs#L81>
+pub fn get_normalized_path(path: &Path) -> PathBuf {
+    let mut components = path.components().peekable();
+    let mut ret = if let Some(c @ Component::Prefix(..)) = components.peek().cloned() {
+        components.next();
+        PathBuf::from(c.as_os_str())
+    } else {
+        PathBuf::new()
+    };
+
+    for component in components {
+        match component {
+            Component::Prefix(..) => unreachable!(),
+            Component::RootDir => {
+                ret.push(component.as_os_str());
+            }
+            Component::CurDir => {}
+            Component::ParentDir => {
+                ret.pop();
+            }
+            Component::Normal(c) => {
+                ret.push(c);
+            }
+        }
+    }
+    ret
+}
+
+/// Returns the canonical, absolute form of a path with all intermediate components normalized.
+///
+/// This function is used instead of `std::fs::canonicalize` because we don't want to verify
+/// here if the path exists, just normalize it's components.
+pub fn get_canonicalized_path(path: &Path) -> std::io::Result<PathBuf> {
+    let path = expand_tilde(path);
+    let path = if path.is_relative() {
+        std::env::current_dir().map(|current_dir| current_dir.join(path))?
+    } else {
+        path
+    };
+
+    Ok(get_normalized_path(path.as_path()))
+}
+
+pub fn get_relative_path(path: &Path) -> PathBuf {
+    let path = if path.is_absolute() {
+        let cwdir = std::env::current_dir().expect("couldn't determine current directory");
+        path.strip_prefix(cwdir).unwrap_or(path)
+    } else {
+        path
+    };
+    fold_home_dir(path)
+}

helix-core/src/position.rs

@@ -1,6 +1,9 @@
 use crate::{
-    graphemes::{nth_next_grapheme_boundary, RopeGraphemes},
+    chars::char_is_line_ending,
-    Rope, RopeSlice,
+    graphemes::{ensure_grapheme_boundary_prev, RopeGraphemes},
+    line_ending::line_end_char_index,
+    unicode::width::UnicodeWidthChar,
+    RopeSlice,
 };
 
 /// Represents a single point in a text buffer. Zero indexed.
@@ -23,8 +26,9 @@ impl Position {
     pub fn traverse(self, text: &crate::Tendril) -> Self {
         let Self { mut row, mut col } = self;
         // TODO: there should be a better way here
-        for ch in text.chars() {
+        let mut chars = text.chars().peekable();
-            if ch == '\n' {
+        while let Some(ch) = chars.next() {
+            if char_is_line_ending(ch) && !(ch == '\r' && chars.peek() == Some(&'\n')) {
                 row += 1;
                 col = 0;
             } else {
@@ -50,24 +54,84 @@ impl From<Position> for tree_sitter::Point {
     }
 }
 /// Convert a character index to (line, column) coordinates.
+///
+/// column in `char` count which can be used for row:column display in
+/// status line. See [`visual_coords_at_pos`] for a visual one.
 pub fn coords_at_pos(text: RopeSlice, pos: usize) -> Position {
     let line = text.char_to_line(pos);
+
     let line_start = text.line_to_char(line);
+    let pos = ensure_grapheme_boundary_prev(text, pos);
     let col = RopeGraphemes::new(text.slice(line_start..pos)).count();
+
+    Position::new(line, col)
+}
+
+/// Convert a character index to (line, column) coordinates visually.
+///
+/// Takes \t, double-width characters (CJK) into account as well as text
+/// not in the document in the future.
+/// See [`coords_at_pos`] for an "objective" one.
+pub fn visual_coords_at_pos(text: RopeSlice, pos: usize, tab_width: usize) -> Position {
+    let line = text.char_to_line(pos);
+
+    let line_start = text.line_to_char(line);
+    let pos = ensure_grapheme_boundary_prev(text, pos);
+    let col = text
+        .slice(line_start..pos)
+        .chars()
+        .flat_map(|c| match c {
+            '\t' => Some(tab_width),
+            c => UnicodeWidthChar::width(c),
+        })
+        .sum();
+
     Position::new(line, col)
 }
 
 /// Convert (line, column) coordinates to a character index.
-pub fn pos_at_coords(text: RopeSlice, coords: Position) -> usize {
+///
+/// If the `line` coordinate is beyond the end of the file, the EOF
+/// position will be returned.
+///
+/// If the `column` coordinate is past the end of the given line, the
+/// line-end position will be returned.  What constitutes the "line-end
+/// position" depends on the parameter `limit_before_line_ending`.  If it's
+/// `true`, the line-end position will be just *before* the line ending
+/// character.  If `false` it will be just *after* the line ending
+/// character--on the border between the current line and the next.
+///
+/// Usually you only want `limit_before_line_ending` to be `true` if you're working
+/// with left-side block-cursor positions, as this prevents the the block cursor
+/// from jumping to the next line.  Otherwise you typically want it to be `false`,
+/// such as when dealing with raw anchor/head positions.
+///
+/// TODO: this should be changed to work in terms of visual row/column, not
+/// graphemes.
+pub fn pos_at_coords(text: RopeSlice, coords: Position, limit_before_line_ending: bool) -> usize {
     let Position { row, col } = coords;
     let line_start = text.line_to_char(row);
-    // line_start + col
+    let line_end = if limit_before_line_ending {
-    nth_next_grapheme_boundary(text, line_start, col)
+        line_end_char_index(&text, row)
+    } else {
+        text.line_to_char((row + 1).min(text.len_lines()))
+    };
+
+    let mut col_char_offset = 0;
+    for (i, g) in RopeGraphemes::new(text.slice(line_start..line_end)).enumerate() {
+        if i == col {
+            break;
+        }
+        col_char_offset += g.chars().count();
+    }
+
+    line_start + col_char_offset
 }
 
 #[cfg(test)]
 mod test {
     use super::*;
+    use crate::Rope;
 
     #[test]
     fn test_ordering() {
@@ -79,53 +143,152 @@ mod test {
     fn test_coords_at_pos() {
         let text = Rope::from("ḧëḷḷö\nẅöṛḷḋ");
         let slice = text.slice(..);
-        // assert_eq!(coords_at_pos(slice, 0), (0, 0).into());
+        assert_eq!(coords_at_pos(slice, 0), (0, 0).into());
-        // assert_eq!(coords_at_pos(slice, 5), (0, 5).into()); // position on \n
+        assert_eq!(coords_at_pos(slice, 5), (0, 5).into()); // position on \n
-        // assert_eq!(coords_at_pos(slice, 6), (1, 0).into()); // position on w
+        assert_eq!(coords_at_pos(slice, 6), (1, 0).into()); // position on w
-        // assert_eq!(coords_at_pos(slice, 7), (1, 1).into()); // position on o
+        assert_eq!(coords_at_pos(slice, 7), (1, 1).into()); // position on o
-        // assert_eq!(coords_at_pos(slice, 10), (1, 4).into()); // position on d
+        assert_eq!(coords_at_pos(slice, 10), (1, 4).into()); // position on d
 
-        // test with grapheme clusters
+        // Test with wide characters.
+        let text = Rope::from("今日はいい\n");
+        let slice = text.slice(..);
+        assert_eq!(coords_at_pos(slice, 0), (0, 0).into());
+        assert_eq!(coords_at_pos(slice, 1), (0, 1).into());
+        assert_eq!(coords_at_pos(slice, 2), (0, 2).into());
+        assert_eq!(coords_at_pos(slice, 3), (0, 3).into());
+        assert_eq!(coords_at_pos(slice, 4), (0, 4).into());
+        assert_eq!(coords_at_pos(slice, 5), (0, 5).into());
+        assert_eq!(coords_at_pos(slice, 6), (1, 0).into());
+
+        // Test with grapheme clusters.
         let text = Rope::from("a̐éö̲\r\n");
         let slice = text.slice(..);
         assert_eq!(coords_at_pos(slice, 0), (0, 0).into());
         assert_eq!(coords_at_pos(slice, 2), (0, 1).into());
         assert_eq!(coords_at_pos(slice, 4), (0, 2).into());
         assert_eq!(coords_at_pos(slice, 7), (0, 3).into());
+        assert_eq!(coords_at_pos(slice, 9), (1, 0).into());
 
-        let text = Rope::from("किमपि");
+        // Test with wide-character grapheme clusters.
+        let text = Rope::from("किमपि\n");
         let slice = text.slice(..);
         assert_eq!(coords_at_pos(slice, 0), (0, 0).into());
         assert_eq!(coords_at_pos(slice, 2), (0, 1).into());
         assert_eq!(coords_at_pos(slice, 3), (0, 2).into());
         assert_eq!(coords_at_pos(slice, 5), (0, 3).into());
+        assert_eq!(coords_at_pos(slice, 6), (1, 0).into());
+
+        // Test with tabs.
+        let text = Rope::from("\tHello\n");
+        let slice = text.slice(..);
+        assert_eq!(coords_at_pos(slice, 0), (0, 0).into());
+        assert_eq!(coords_at_pos(slice, 1), (0, 1).into());
+        assert_eq!(coords_at_pos(slice, 2), (0, 2).into());
+    }
+
+    #[test]
+    fn test_visual_coords_at_pos() {
+        let text = Rope::from("ḧëḷḷö\nẅöṛḷḋ");
+        let slice = text.slice(..);
+        assert_eq!(visual_coords_at_pos(slice, 0, 8), (0, 0).into());
+        assert_eq!(visual_coords_at_pos(slice, 5, 8), (0, 5).into()); // position on \n
+        assert_eq!(visual_coords_at_pos(slice, 6, 8), (1, 0).into()); // position on w
+        assert_eq!(visual_coords_at_pos(slice, 7, 8), (1, 1).into()); // position on o
+        assert_eq!(visual_coords_at_pos(slice, 10, 8), (1, 4).into()); // position on d
+
+        // Test with wide characters.
+        let text = Rope::from("今日はいい\n");
+        let slice = text.slice(..);
+        assert_eq!(visual_coords_at_pos(slice, 0, 8), (0, 0).into());
+        assert_eq!(visual_coords_at_pos(slice, 1, 8), (0, 2).into());
+        assert_eq!(visual_coords_at_pos(slice, 2, 8), (0, 4).into());
+        assert_eq!(visual_coords_at_pos(slice, 3, 8), (0, 6).into());
+        assert_eq!(visual_coords_at_pos(slice, 4, 8), (0, 8).into());
+        assert_eq!(visual_coords_at_pos(slice, 5, 8), (0, 10).into());
+        assert_eq!(visual_coords_at_pos(slice, 6, 8), (1, 0).into());
+
+        // Test with grapheme clusters.
+        let text = Rope::from("a̐éö̲\r\n");
+        let slice = text.slice(..);
+        assert_eq!(visual_coords_at_pos(slice, 0, 8), (0, 0).into());
+        assert_eq!(visual_coords_at_pos(slice, 2, 8), (0, 1).into());
+        assert_eq!(visual_coords_at_pos(slice, 4, 8), (0, 2).into());
+        assert_eq!(visual_coords_at_pos(slice, 7, 8), (0, 3).into());
+        assert_eq!(visual_coords_at_pos(slice, 9, 8), (1, 0).into());
+
+        // Test with wide-character grapheme clusters.
+        // TODO: account for cluster.
+        let text = Rope::from("किमपि\n");
+        let slice = text.slice(..);
+        assert_eq!(visual_coords_at_pos(slice, 0, 8), (0, 0).into());
+        assert_eq!(visual_coords_at_pos(slice, 2, 8), (0, 2).into());
+        assert_eq!(visual_coords_at_pos(slice, 3, 8), (0, 3).into());
+        assert_eq!(visual_coords_at_pos(slice, 5, 8), (0, 5).into());
+        assert_eq!(visual_coords_at_pos(slice, 6, 8), (1, 0).into());
+
+        // Test with tabs.
+        let text = Rope::from("\tHello\n");
+        let slice = text.slice(..);
+        assert_eq!(visual_coords_at_pos(slice, 0, 8), (0, 0).into());
+        assert_eq!(visual_coords_at_pos(slice, 1, 8), (0, 8).into());
+        assert_eq!(visual_coords_at_pos(slice, 2, 8), (0, 9).into());
     }
 
     #[test]
     fn test_pos_at_coords() {
         let text = Rope::from("ḧëḷḷö\nẅöṛḷḋ");
         let slice = text.slice(..);
-        assert_eq!(pos_at_coords(slice, (0, 0).into()), 0);
+        assert_eq!(pos_at_coords(slice, (0, 0).into(), false), 0);
-        assert_eq!(pos_at_coords(slice, (0, 5).into()), 5); // position on \n
+        assert_eq!(pos_at_coords(slice, (0, 5).into(), false), 5); // position on \n
-        assert_eq!(pos_at_coords(slice, (1, 0).into()), 6); // position on w
+        assert_eq!(pos_at_coords(slice, (0, 6).into(), false), 6); // position after \n
-        assert_eq!(pos_at_coords(slice, (1, 1).into()), 7); // position on o
+        assert_eq!(pos_at_coords(slice, (0, 6).into(), true), 5); // position after \n
-        assert_eq!(pos_at_coords(slice, (1, 4).into()), 10); // position on d
+        assert_eq!(pos_at_coords(slice, (1, 0).into(), false), 6); // position on w
+        assert_eq!(pos_at_coords(slice, (1, 1).into(), false), 7); // position on o
+        assert_eq!(pos_at_coords(slice, (1, 4).into(), false), 10); // position on d
 
-        // test with grapheme clusters
+        // Test with wide characters.
+        // TODO: account for character width.
+        let text = Rope::from("今日はいい\n");
+        let slice = text.slice(..);
+        assert_eq!(pos_at_coords(slice, (0, 0).into(), false), 0);
+        assert_eq!(pos_at_coords(slice, (0, 1).into(), false), 1);
+        assert_eq!(pos_at_coords(slice, (0, 2).into(), false), 2);
+        assert_eq!(pos_at_coords(slice, (0, 3).into(), false), 3);
+        assert_eq!(pos_at_coords(slice, (0, 4).into(), false), 4);
+        assert_eq!(pos_at_coords(slice, (0, 5).into(), false), 5);
+        assert_eq!(pos_at_coords(slice, (0, 6).into(), false), 6);
+        assert_eq!(pos_at_coords(slice, (0, 6).into(), true), 5);
+        assert_eq!(pos_at_coords(slice, (1, 0).into(), false), 6);
+
+        // Test with grapheme clusters.
         let text = Rope::from("a̐éö̲\r\n");
         let slice = text.slice(..);
-        assert_eq!(pos_at_coords(slice, (0, 0).into()), 0);
+        assert_eq!(pos_at_coords(slice, (0, 0).into(), false), 0);
-        assert_eq!(pos_at_coords(slice, (0, 1).into()), 2);
+        assert_eq!(pos_at_coords(slice, (0, 1).into(), false), 2);
-        assert_eq!(pos_at_coords(slice, (0, 2).into()), 4);
+        assert_eq!(pos_at_coords(slice, (0, 2).into(), false), 4);
-        assert_eq!(pos_at_coords(slice, (0, 3).into()), 7); // \r\n is one char here
+        assert_eq!(pos_at_coords(slice, (0, 3).into(), false), 7); // \r\n is one char here
-        assert_eq!(pos_at_coords(slice, (0, 4).into()), 9);
+        assert_eq!(pos_at_coords(slice, (0, 4).into(), false), 9);
+        assert_eq!(pos_at_coords(slice, (0, 4).into(), true), 7);
+        assert_eq!(pos_at_coords(slice, (1, 0).into(), false), 9);
+
+        // Test with wide-character grapheme clusters.
+        // TODO: account for character width.
         let text = Rope::from("किमपि");
         // 2 - 1 - 2 codepoints
         // TODO: delete handling as per https://news.ycombinator.com/item?id=20058454
         let slice = text.slice(..);
-        assert_eq!(pos_at_coords(slice, (0, 0).into()), 0);
+        assert_eq!(pos_at_coords(slice, (0, 0).into(), false), 0);
-        assert_eq!(pos_at_coords(slice, (0, 1).into()), 2);
+        assert_eq!(pos_at_coords(slice, (0, 1).into(), false), 2);
-        assert_eq!(pos_at_coords(slice, (0, 2).into()), 3);
+        assert_eq!(pos_at_coords(slice, (0, 2).into(), false), 3);
-        assert_eq!(pos_at_coords(slice, (0, 3).into()), 5); // eol
+        assert_eq!(pos_at_coords(slice, (0, 3).into(), false), 5);
+        assert_eq!(pos_at_coords(slice, (0, 3).into(), true), 5);
+
+        // Test with tabs.
+        // Todo: account for tab stops.
+        let text = Rope::from("\tHello\n");
+        let slice = text.slice(..);
+        assert_eq!(pos_at_coords(slice, (0, 0).into(), false), 0);
+        assert_eq!(pos_at_coords(slice, (0, 1).into(), false), 1);
+        assert_eq!(pos_at_coords(slice, (0, 2).into(), false), 2);
     }
 }

helix-core/src/register.rs

@@ -1,21 +1,71 @@
-use crate::Tendril;
+use std::collections::HashMap;
-use once_cell::sync::Lazy;
-use std::{collections::HashMap, sync::RwLock};
 
-// TODO: could be an instance on Editor
+#[derive(Debug)]
-static REGISTRY: Lazy<RwLock<HashMap<char, Vec<String>>>> =
+pub struct Register {
-    Lazy::new(|| RwLock::new(HashMap::new()));
+    name: char,
-
+    values: Vec<String>,
-pub fn get(register: char) -> Option<Vec<String>> {
-    let registry = REGISTRY.read().unwrap();
-
-    // TODO: no cloning
-    registry.get(&register).cloned()
 }
 
-// restoring: bool
+impl Register {
-pub fn set(register: char, values: Vec<String>) {
+    pub const fn new(name: char) -> Self {
-    let mut registry = REGISTRY.write().unwrap();
+        Self {
+            name,
+            values: Vec::new(),
+        }
+    }
 
-    registry.insert(register, values);
+    pub fn new_with_values(name: char, values: Vec<String>) -> Self {
+        if name == '_' {
+            Self::new(name)
+        } else {
+            Self { name, values }
+        }
+    }
+
+    pub const fn name(&self) -> char {
+        self.name
+    }
+
+    pub fn read(&self) -> &[String] {
+        &self.values
+    }
+
+    pub fn write(&mut self, values: Vec<String>) {
+        if self.name != '_' {
+            self.values = values;
+        }
+    }
+
+    pub fn push(&mut self, value: String) {
+        if self.name != '_' {
+            self.values.push(value);
+        }
+    }
+}
+
+/// Currently just wraps a `HashMap` of `Register`s
+#[derive(Debug, Default)]
+pub struct Registers {
+    inner: HashMap<char, Register>,
+}
+
+impl Registers {
+    pub fn get(&self, name: char) -> Option<&Register> {
+        self.inner.get(&name)
+    }
+
+    pub fn get_mut(&mut self, name: char) -> &mut Register {
+        self.inner
+            .entry(name)
+            .or_insert_with(|| Register::new(name))
+    }
+
+    pub fn write(&mut self, name: char, values: Vec<String>) {
+        self.inner
+            .insert(name, Register::new_with_values(name, values));
+    }
+
+    pub fn read(&self, name: char) -> Option<&[String]> {
+        self.get(name).map(|reg| reg.read())
+    }
 }

helix-core/src/search.rs

@@ -1,18 +1,11 @@
 use crate::RopeSlice;
 
-pub fn find_nth_next(
+pub fn find_nth_next(text: RopeSlice, ch: char, mut pos: usize, n: usize) -> Option<usize> {
-    text: RopeSlice,
+    if pos >= text.len_chars() || n == 0 {
-    ch: char,
-    mut pos: usize,
-    n: usize,
-    inclusive: bool,
-) -> Option<usize> {
-    if pos >= text.len_chars() {
         return None;
     }
 
-    // start searching right after pos
+    let mut chars = text.chars_at(pos);
-    let mut chars = text.chars_at(pos + 1);
 
     for _ in 0..n {
         loop {
@@ -26,28 +19,21 @@ pub fn find_nth_next(
         }
     }
 
-    if !inclusive {
+    Some(pos - 1)
-        pos -= 1;
-    }
-
-    Some(pos)
 }
 
-pub fn find_nth_prev(
+pub fn find_nth_prev(text: RopeSlice, ch: char, mut pos: usize, n: usize) -> Option<usize> {
-    text: RopeSlice,
+    if pos == 0 || n == 0 {
-    ch: char,
+        return None;
-    mut pos: usize,
+    }
-    n: usize,
+
-    inclusive: bool,
-) -> Option<usize> {
-    // start searching right before pos
     let mut chars = text.chars_at(pos);
 
     for _ in 0..n {
         loop {
             let c = chars.prev()?;
 
-            pos = pos.saturating_sub(1);
+            pos -= 1;
 
             if c == ch {
                 break;
@@ -55,9 +41,5 @@ pub fn find_nth_prev(
         }
     }
 
-    if !inclusive {
-        pos += 1;
-    }
-
     Some(pos)
 }

helix-core/src/selection.rs

@@ -1,30 +1,60 @@
-//! Selections are the primary editing construct. Even a single cursor is defined as an empty
+//! Selections are the primary editing construct. Even cursors are
-//! single selection range.
+//! defined as a selection range.
 //!
 //! All positioning is done via `char` offsets into the buffer.
-use crate::{Assoc, ChangeSet, Rope, RopeSlice};
+use crate::{
+    graphemes::{
+        ensure_grapheme_boundary_next, ensure_grapheme_boundary_prev, next_grapheme_boundary,
+        prev_grapheme_boundary,
+    },
+    movement::Direction,
+    Assoc, ChangeSet, RopeSlice,
+};
 use smallvec::{smallvec, SmallVec};
 use std::borrow::Cow;
 
-#[inline]
+/// A single selection range.
-fn abs_difference(x: usize, y: usize) -> usize {
+///
-    if x < y {
+/// A range consists of an "anchor" and "head" position in
-        y - x
+/// the text.  The head is the part that the user moves when
-    } else {
+/// directly extending a selection.  The head and anchor
-        x - y
+/// can be in any order, or even share the same position.
-    }
+///
-}
+/// The anchor and head positions use gap indexing, meaning
-
+/// that their indices represent the the gaps *between* `char`s
-/// A single selection range. Anchor-inclusive, head-exclusive.
+/// rather than the `char`s themselves. For example, 1
+/// represents the position between the first and second `char`.
+///
+/// Below are some example `Range` configurations to better
+/// illustrate.  The anchor and head indices are show as
+/// "(anchor, head)", followed by example text with "[" and "]"
+/// inserted to represent the anchor and head positions:
+///
+/// - (0, 3): `[Som]e text`.
+/// - (3, 0): `]Som[e text`.
+/// - (2, 7): `So[me te]xt`.
+/// - (1, 1): `S[]ome text`.
+///
+/// Ranges are considered to be inclusive on the left and
+/// exclusive on the right, regardless of anchor-head ordering.
+/// This means, for example, that non-zero-width ranges that
+/// are directly adjecent, sharing an edge, do not overlap.
+/// However, a zero-width range will overlap with the shared
+/// left-edge of another range.
+///
+/// By convention, user-facing ranges are considered to have
+/// a block cursor on the head-side of the range that spans a
+/// single grapheme inward from the range's edge.  There are a
+/// variety of helper methods on `Range` for working in terms of
+/// that block cursor, all of which have `cursor` in their name.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub struct Range {
-    // TODO: optimize into u32
     /// The anchor of the range: the side that doesn't move when extending.
     pub anchor: usize,
     /// The head of the range, moved when extending.
     pub head: usize,
     pub horiz: Option<u32>,
-} // TODO: might be cheaper to store normalized as from/to and an inverted flag
+}
 
 impl Range {
     pub fn new(anchor: usize, head: usize) -> Self {
@@ -35,6 +65,10 @@ impl Range {
         }
     }
 
+    pub fn point(head: usize) -> Self {
+        Self::new(head, head)
+    }
+
     /// Start of the range.
     #[inline]
     #[must_use]
@@ -49,46 +83,90 @@ impl Range {
         std::cmp::max(self.anchor, self.head)
     }
 
+    /// Total length of the range.
+    #[inline]
+    #[must_use]
+    pub fn len(&self) -> usize {
+        self.to() - self.from()
+    }
+
+    /// The (inclusive) range of lines that the range overlaps.
+    #[inline]
+    #[must_use]
+    pub fn line_range(&self, text: RopeSlice) -> (usize, usize) {
+        let from = self.from();
+        let to = if self.is_empty() {
+            self.to()
+        } else {
+            prev_grapheme_boundary(text, self.to()).max(from)
+        };
+
+        (text.char_to_line(from), text.char_to_line(to))
+    }
+
     /// `true` when head and anchor are at the same position.
     #[inline]
     pub fn is_empty(&self) -> bool {
         self.anchor == self.head
     }
 
-    /// Check two ranges for overlap.
+    /// `Direction::Backward` when head < anchor.
+    /// `Direction::Backward` otherwise.
+    #[inline]
     #[must_use]
-    pub fn overlaps(&self, other: &Self) -> bool {
+    pub fn direction(&self) -> Direction {
-        // cursor overlap is checked differently
+        if self.head < self.anchor {
-        if self.is_empty() {
+            Direction::Backward
-            let pos = self.head;
-            pos >= other.from() && other.to() >= pos
         } else {
-            self.to() > other.from() && other.to() > self.from()
+            Direction::Forward
         }
     }
 
-    pub fn contains(&self, pos: usize) -> bool {
+    // flips the direction of the selection
-        if self.is_empty() {
+    pub fn flip(&self) -> Self {
-            return false;
+        Self {
+            anchor: self.head,
+            head: self.anchor,
+            horiz: self.horiz,
         }
+    }
 
-        if self.anchor < self.head {
+    /// Check two ranges for overlap.
-            self.anchor <= pos && pos < self.head
+    #[must_use]
-        } else {
+    pub fn overlaps(&self, other: &Self) -> bool {
-            self.head < pos && pos <= self.anchor
+        // To my eye, it's non-obvious why this works, but I arrived
-        }
+        // at it after transforming the slower version that explicitly
+        // enumerated more cases.  The unit tests are thorough.
+        self.from() == other.from() || (self.to() > other.from() && other.to() > self.from())
+    }
+
+    pub fn contains(&self, pos: usize) -> bool {
+        self.from() <= pos && pos < self.to()
     }
 
     /// Map a range through a set of changes. Returns a new range representing the same position
     /// after the changes are applied.
     pub fn map(self, changes: &ChangeSet) -> Self {
-        let anchor = changes.map_pos(self.anchor, Assoc::After);
+        use std::cmp::Ordering;
-        let head = changes.map_pos(self.head, Assoc::After);
+        let (anchor, head) = match self.anchor.cmp(&self.head) {
+            Ordering::Equal => (
+                changes.map_pos(self.anchor, Assoc::After),
+                changes.map_pos(self.head, Assoc::After),
+            ),
+            Ordering::Less => (
+                changes.map_pos(self.anchor, Assoc::After),
+                changes.map_pos(self.head, Assoc::Before),
+            ),
+            Ordering::Greater => (
+                changes.map_pos(self.anchor, Assoc::Before),
+                changes.map_pos(self.head, Assoc::After),
+            ),
+        };
 
-        // TODO: possibly unnecessary
+        // We want to return a new `Range` with `horiz == None` every time,
-        if self.anchor == anchor && self.head == head {
+        // even if the anchor and head haven't changed, because we don't
-            return self;
+        // know if the *visual* position hasn't changed due to
-        }
+        // character-width or grapheme changes earlier in the text.
         Self {
             anchor,
             head,
@@ -99,22 +177,41 @@ impl Range {
     /// Extend the range to cover at least `from` `to`.
     #[must_use]
     pub fn extend(&self, from: usize, to: usize) -> Self {
-        if from <= self.anchor && to >= self.anchor {
+        debug_assert!(from <= to);
-            return Self {
-                anchor: from,
-                head: to,
-                horiz: None,
-            };
-        }
 
-        Self {
+        if self.anchor <= self.head {
-            anchor: self.anchor,
+            Self {
-            head: if abs_difference(from, self.anchor) > abs_difference(to, self.anchor) {
+                anchor: self.anchor.min(from),
-                from
+                head: self.head.max(to),
-            } else {
+                horiz: None,
-                to
+            }
-            },
+        } else {
-            horiz: None,
+            Self {
+                anchor: self.anchor.max(to),
+                head: self.head.min(from),
+                horiz: None,
+            }
+        }
+    }
+
+    /// Returns a range that encompasses both input ranges.
+    ///
+    /// This is like `extend()`, but tries to negotiate the
+    /// anchor/head ordering between the two input ranges.
+    #[must_use]
+    pub fn merge(&self, other: Self) -> Self {
+        if self.anchor > self.head && other.anchor > other.head {
+            Range {
+                anchor: self.anchor.max(other.anchor),
+                head: self.head.min(other.head),
+                horiz: None,
+            }
+        } else {
+            Range {
+                anchor: self.from().min(other.from()),
+                head: self.to().max(other.to()),
+                horiz: None,
+            }
         }
     }
 
@@ -122,7 +219,130 @@ impl Range {
 
     #[inline]
     pub fn fragment<'a, 'b: 'a>(&'a self, text: RopeSlice<'b>) -> Cow<'b, str> {
-        Cow::from(text.slice(self.from()..self.to() + 1))
+        text.slice(self.from()..self.to()).into()
+    }
+
+    //--------------------------------
+    // Alignment methods.
+
+    /// Compute a possibly new range from this range, with its ends
+    /// shifted as needed to align with grapheme boundaries.
+    ///
+    /// Zero-width ranges will always stay zero-width, and non-zero-width
+    /// ranges will never collapse to zero-width.
+    #[must_use]
+    pub fn grapheme_aligned(&self, slice: RopeSlice) -> Self {
+        use std::cmp::Ordering;
+        let (new_anchor, new_head) = match self.anchor.cmp(&self.head) {
+            Ordering::Equal => {
+                let pos = ensure_grapheme_boundary_prev(slice, self.anchor);
+                (pos, pos)
+            }
+            Ordering::Less => (
+                ensure_grapheme_boundary_prev(slice, self.anchor),
+                ensure_grapheme_boundary_next(slice, self.head),
+            ),
+            Ordering::Greater => (
+                ensure_grapheme_boundary_next(slice, self.anchor),
+                ensure_grapheme_boundary_prev(slice, self.head),
+            ),
+        };
+        Range {
+            anchor: new_anchor,
+            head: new_head,
+            horiz: if new_anchor == self.anchor {
+                self.horiz
+            } else {
+                None
+            },
+        }
+    }
+
+    /// Compute a possibly new range from this range, attempting to ensure
+    /// a minimum range width of 1 char by shifting the head in the forward
+    /// direction as needed.
+    ///
+    /// This method will never shift the anchor, and will only shift the
+    /// head in the forward direction.  Therefore, this method can fail
+    /// at ensuring the minimum width if and only if the passed range is
+    /// both zero-width and at the end of the `RopeSlice`.
+    ///
+    /// If the input range is grapheme-boundary aligned, the returned range
+    /// will also be.  Specifically, if the head needs to shift to achieve
+    /// the minimum width, it will shift to the next grapheme boundary.
+    #[must_use]
+    #[inline]
+    pub fn min_width_1(&self, slice: RopeSlice) -> Self {
+        if self.anchor == self.head {
+            Range {
+                anchor: self.anchor,
+                head: next_grapheme_boundary(slice, self.head),
+                horiz: self.horiz,
+            }
+        } else {
+            *self
+        }
+    }
+
+    //--------------------------------
+    // Block-cursor methods.
+
+    /// Gets the left-side position of the block cursor.
+    #[must_use]
+    #[inline]
+    pub fn cursor(self, text: RopeSlice) -> usize {
+        if self.head > self.anchor {
+            prev_grapheme_boundary(text, self.head)
+        } else {
+            self.head
+        }
+    }
+
+    /// Puts the left side of the block cursor at `char_idx`, optionally extending.
+    ///
+    /// This follows "1-width" semantics, and therefore does a combination of anchor
+    /// and head moves to behave as if both the front and back of the range are 1-width
+    /// blocks
+    ///
+    /// This method assumes that the range and `char_idx` are already properly
+    /// grapheme-aligned.
+    #[must_use]
+    #[inline]
+    pub fn put_cursor(self, text: RopeSlice, char_idx: usize, extend: bool) -> Range {
+        if extend {
+            let anchor = if self.head >= self.anchor && char_idx < self.anchor {
+                next_grapheme_boundary(text, self.anchor)
+            } else if self.head < self.anchor && char_idx >= self.anchor {
+                prev_grapheme_boundary(text, self.anchor)
+            } else {
+                self.anchor
+            };
+
+            if anchor <= char_idx {
+                Range::new(anchor, next_grapheme_boundary(text, char_idx))
+            } else {
+                Range::new(anchor, char_idx)
+            }
+        } else {
+            Range::point(char_idx)
+        }
+    }
+
+    /// The line number that the block-cursor is on.
+    #[inline]
+    #[must_use]
+    pub fn cursor_line(&self, text: RopeSlice) -> usize {
+        text.char_to_line(self.cursor(text))
+    }
+}
+
+impl From<(usize, usize)> for Range {
+    fn from((anchor, head): (usize, usize)) -> Self {
+        Self {
+            anchor,
+            head,
+            horiz: None,
+        }
     }
 }
 
@@ -138,14 +358,16 @@ pub struct Selection {
 impl Selection {
     // eq
 
+    #[inline]
     #[must_use]
     pub fn primary(&self) -> Range {
         self.ranges[self.primary_index]
     }
 
+    #[inline]
     #[must_use]
-    pub fn cursor(&self) -> usize {
+    pub fn primary_mut(&mut self) -> &mut Range {
-        self.primary().head
+        &mut self.ranges[self.primary_index]
     }
 
     /// Ensure selection containing only the primary selection.
@@ -160,13 +382,32 @@ impl Selection {
         }
     }
 
+    /// Adds a new range to the selection and makes it the primary range.
     pub fn push(mut self, range: Range) -> Self {
-        let index = self.ranges.len();
         self.ranges.push(range);
-
+        self.set_primary_index(self.ranges().len() - 1);
-        Self::normalize(self.ranges, index)
+        self.normalize()
+    }
+
+    /// Removes a range from the selection.
+    pub fn remove(mut self, index: usize) -> Self {
+        assert!(
+            self.ranges.len() > 1,
+            "can't remove the last range from a selection!"
+        );
+
+        self.ranges.remove(index);
+        if index < self.primary_index || self.primary_index == self.ranges.len() {
+            self.primary_index -= 1;
+        }
+        self
+    }
+
+    /// Replace a range in the selection with a new range.
+    pub fn replace(mut self, index: usize, range: Range) -> Self {
+        self.ranges[index] = range;
+        self.normalize()
     }
-    // replace_range
 
     /// Map selections over a set of changes. Useful for adjusting the selection position after
     /// applying changes to a document.
@@ -192,6 +433,11 @@ impl Selection {
         self.primary_index
     }
 
+    pub fn set_primary_index(&mut self, idx: usize) {
+        assert!(idx < self.ranges.len());
+        self.primary_index = idx;
+    }
+
     #[must_use]
     /// Constructs a selection holding a single range.
     pub fn single(anchor: usize, head: usize) -> Self {
@@ -210,77 +456,79 @@ impl Selection {
         Self::single(pos, pos)
     }
 
-    fn normalize(mut ranges: SmallVec<[Range; 1]>, mut primary_index: usize) -> Self {
+    /// Normalizes a `Selection`.
-        let primary = ranges[primary_index];
+    fn normalize(mut self) -> Self {
-        ranges.sort_unstable_by_key(Range::from);
+        let primary = self.ranges[self.primary_index];
-        primary_index = ranges.iter().position(|&range| range == primary).unwrap();
+        self.ranges.sort_unstable_by_key(Range::from);
+        self.primary_index = self
+            .ranges
+            .iter()
+            .position(|&range| range == primary)
+            .unwrap();
 
-        let mut result = SmallVec::with_capacity(ranges.len()); // approx
+        let mut prev_i = 0;
-
+        for i in 1..self.ranges.len() {
-        // TODO: we could do with one vec by removing elements as we mutate
+            if self.ranges[prev_i].overlaps(&self.ranges[i]) {
-
+                self.ranges[prev_i] = self.ranges[prev_i].merge(self.ranges[i]);
-        for (i, range) in ranges.into_iter().enumerate() {
+            } else {
-            // if previous value exists
+                prev_i += 1;
-            if let Some(prev) = result.last_mut() {
+                self.ranges[prev_i] = self.ranges[i];
-                // and we overlap it
+            }
-
+            if i == self.primary_index {
-                // TODO: we used to simply check range.from() <(=) prev.to()
+                self.primary_index = prev_i;
-                // avoiding two comparisons
-                if range.overlaps(prev) {
-                    let from = prev.from();
-                    let to = std::cmp::max(range.to(), prev.to());
-
-                    if i <= primary_index {
-                        primary_index -= 1
-                    }
-
-                    // merge into previous
-                    if range.anchor > range.head {
-                        prev.anchor = to;
-                        prev.head = from;
-                    } else {
-                        prev.anchor = from;
-                        prev.head = to;
-                    }
-                    continue;
-                }
             }
-
-            result.push(range)
         }
 
-        Self {
+        self.ranges.truncate(prev_i + 1);
-            ranges: result,
+
-            primary_index,
+        self
-        }
     }
 
     // TODO: consume an iterator or a vec to reduce allocations?
     #[must_use]
     pub fn new(ranges: SmallVec<[Range; 1]>, primary_index: usize) -> Self {
         assert!(!ranges.is_empty());
+        debug_assert!(primary_index < ranges.len());
 
-        // fast path for a single selection (cursor)
+        let mut selection = Self {
-        if ranges.len() == 1 {
+            ranges,
-            return Self {
+            primary_index,
-                ranges,
+        };
-                primary_index: 0,
+
-            };
+        if selection.ranges.len() > 1 {
+            // TODO: only normalize if needed (any ranges out of order)
+            selection = selection.normalize();
         }
 
-        // TODO: only normalize if needed (any ranges out of order)
+        selection
-        Self::normalize(ranges, primary_index)
     }
 
-    /// Takes a closure and maps each selection over the closure.
+    /// Takes a closure and maps each `Range` over the closure.
-    pub fn transform<F>(&self, f: F) -> Self
+    pub fn transform<F>(mut self, f: F) -> Self
     where
         F: Fn(Range) -> Range,
     {
-        Self::new(
+        for range in self.ranges.iter_mut() {
-            self.ranges.iter().copied().map(f).collect(),
+            *range = f(*range)
-            self.primary_index,
+        }
-        )
+        self.normalize()
+    }
+
+    // Ensures the selection adheres to the following invariants:
+    // 1. All ranges are grapheme aligned.
+    // 2. All ranges are at least 1 character wide, unless at the
+    //    very end of the document.
+    // 3. Ranges are non-overlapping.
+    // 4. Ranges are sorted by their position in the text.
+    pub fn ensure_invariants(self, text: RopeSlice) -> Self {
+        self.transform(|r| r.min_width_1(text).grapheme_aligned(text))
+            .normalize()
+    }
+
+    /// Transforms the selection into all of the left-side head positions,
+    /// using block-cursor semantics.
+    pub fn cursors(self, text: RopeSlice) -> Self {
+        self.transform(|range| Range::point(range.cursor(text)))
     }
 
     pub fn fragments<'a>(&'a self, text: RopeSlice<'a>) -> impl Iterator<Item = Cow<str>> + 'a {
@@ -309,14 +557,15 @@ impl<'a> IntoIterator for &'a Selection {
 
 // TODO: checkSelection -> check if valid for doc length && sorted
 
-pub fn keep_matches(
+pub fn keep_or_remove_matches(
     text: RopeSlice,
     selection: &Selection,
     regex: &crate::regex::Regex,
+    remove: bool,
 ) -> Option<Selection> {
     let result: SmallVec<_> = selection
         .iter()
-        .filter(|range| regex.is_match(&range.fragment(text)))
+        .filter(|range| regex.is_match(&range.fragment(text)) ^ remove)
         .copied()
         .collect();
 
@@ -338,17 +587,15 @@ pub fn select_on_matches(
         // TODO: can't avoid occasional allocations since Regex can't operate on chunks yet
         let fragment = sel.fragment(text);
 
-        let mut sel_start = sel.from();
+        let sel_start = sel.from();
-        let sel_end = sel.to();
+        let start_byte = text.char_to_byte(sel_start);
-
-        let mut start_byte = text.char_to_byte(sel_start);
 
         for mat in regex.find_iter(&fragment) {
             // TODO: retain range direction
 
             let start = text.byte_to_char(start_byte + mat.start());
             let end = text.byte_to_char(start_byte + mat.end());
-            result.push(Range::new(start, end - 1));
+            result.push(Range::new(start, end));
         }
     }
 
@@ -369,25 +616,30 @@ pub fn split_on_matches(
     let mut result = SmallVec::with_capacity(selection.len());
 
     for sel in selection {
+        // Special case: zero-width selection.
+        if sel.from() == sel.to() {
+            result.push(*sel);
+            continue;
+        }
+
         // TODO: can't avoid occasional allocations since Regex can't operate on chunks yet
         let fragment = sel.fragment(text);
 
-        let mut sel_start = sel.from();
+        let sel_start = sel.from();
         let sel_end = sel.to();
 
-        let mut start_byte = text.char_to_byte(sel_start);
+        let start_byte = text.char_to_byte(sel_start);
 
         let mut start = sel_start;
 
         for mat in regex.find_iter(&fragment) {
             // TODO: retain range direction
-
             let end = text.byte_to_char(start_byte + mat.start());
-            result.push(Range::new(start, end.saturating_sub(1)));
+            result.push(Range::new(start, end));
             start = text.byte_to_char(start_byte + mat.end());
         }
 
-        if start <= sel_end {
+        if start < sel_end {
             result.push(Range::new(start, sel_end));
         }
     }
@@ -399,11 +651,12 @@ pub fn split_on_matches(
 #[cfg(test)]
 mod test {
     use super::*;
+    use crate::Rope;
 
     #[test]
     #[should_panic]
     fn test_new_empty() {
-        let sel = Selection::new(smallvec![], 0);
+        let _ = Selection::new(smallvec![], 0);
     }
 
     #[test]
@@ -430,6 +683,22 @@ mod test {
             .join(",");
 
         assert_eq!(res, "0/6,6/7,7/8,9/13,13/14");
+
+        // it correctly calculates a new primary index
+        let sel = Selection::new(
+            smallvec![Range::new(0, 2), Range::new(1, 5), Range::new(4, 7)],
+            2,
+        );
+
+        let res = sel
+            .ranges
+            .into_iter()
+            .map(|range| format!("{}/{}", range.anchor, range.head))
+            .collect::<Vec<String>>()
+            .join(",");
+
+        assert_eq!(res, "0/7");
+        assert_eq!(sel.primary_index, 0);
     }
 
     #[test]
@@ -452,7 +721,7 @@ mod test {
             .collect::<Vec<String>>()
             .join(",");
 
-        assert_eq!(res, "8/10,10/12");
+        assert_eq!(res, "8/10,10/12,12/12");
     }
 
     #[test]
@@ -466,35 +735,251 @@ mod test {
         assert_eq!(range.contains(13), false);
 
         let range = Range::new(9, 6);
-        assert_eq!(range.contains(9), true);
+        assert_eq!(range.contains(9), false);
         assert_eq!(range.contains(7), true);
-        assert_eq!(range.contains(6), false);
+        assert_eq!(range.contains(6), true);
+    }
+
+    #[test]
+    fn test_overlaps() {
+        fn overlaps(a: (usize, usize), b: (usize, usize)) -> bool {
+            Range::new(a.0, a.1).overlaps(&Range::new(b.0, b.1))
+        }
+
+        // Two non-zero-width ranges, no overlap.
+        assert!(!overlaps((0, 3), (3, 6)));
+        assert!(!overlaps((0, 3), (6, 3)));
+        assert!(!overlaps((3, 0), (3, 6)));
+        assert!(!overlaps((3, 0), (6, 3)));
+        assert!(!overlaps((3, 6), (0, 3)));
+        assert!(!overlaps((3, 6), (3, 0)));
+        assert!(!overlaps((6, 3), (0, 3)));
+        assert!(!overlaps((6, 3), (3, 0)));
+
+        // Two non-zero-width ranges, overlap.
+        assert!(overlaps((0, 4), (3, 6)));
+        assert!(overlaps((0, 4), (6, 3)));
+        assert!(overlaps((4, 0), (3, 6)));
+        assert!(overlaps((4, 0), (6, 3)));
+        assert!(overlaps((3, 6), (0, 4)));
+        assert!(overlaps((3, 6), (4, 0)));
+        assert!(overlaps((6, 3), (0, 4)));
+        assert!(overlaps((6, 3), (4, 0)));
+
+        // Zero-width and non-zero-width range, no overlap.
+        assert!(!overlaps((0, 3), (3, 3)));
+        assert!(!overlaps((3, 0), (3, 3)));
+        assert!(!overlaps((3, 3), (0, 3)));
+        assert!(!overlaps((3, 3), (3, 0)));
+
+        // Zero-width and non-zero-width range, overlap.
+        assert!(overlaps((1, 4), (1, 1)));
+        assert!(overlaps((4, 1), (1, 1)));
+        assert!(overlaps((1, 1), (1, 4)));
+        assert!(overlaps((1, 1), (4, 1)));
+
+        assert!(overlaps((1, 4), (3, 3)));
+        assert!(overlaps((4, 1), (3, 3)));
+        assert!(overlaps((3, 3), (1, 4)));
+        assert!(overlaps((3, 3), (4, 1)));
+
+        // Two zero-width ranges, no overlap.
+        assert!(!overlaps((0, 0), (1, 1)));
+        assert!(!overlaps((1, 1), (0, 0)));
+
+        // Two zero-width ranges, overlap.
+        assert!(overlaps((1, 1), (1, 1)));
+    }
+
+    #[test]
+    fn test_graphem_aligned() {
+        let r = Rope::from_str("\r\nHi\r\n");
+        let s = r.slice(..);
+
+        // Zero-width.
+        assert_eq!(Range::new(0, 0).grapheme_aligned(s), Range::new(0, 0));
+        assert_eq!(Range::new(1, 1).grapheme_aligned(s), Range::new(0, 0));
+        assert_eq!(Range::new(2, 2).grapheme_aligned(s), Range::new(2, 2));
+        assert_eq!(Range::new(3, 3).grapheme_aligned(s), Range::new(3, 3));
+        assert_eq!(Range::new(4, 4).grapheme_aligned(s), Range::new(4, 4));
+        assert_eq!(Range::new(5, 5).grapheme_aligned(s), Range::new(4, 4));
+        assert_eq!(Range::new(6, 6).grapheme_aligned(s), Range::new(6, 6));
+
+        // Forward.
+        assert_eq!(Range::new(0, 1).grapheme_aligned(s), Range::new(0, 2));
+        assert_eq!(Range::new(1, 2).grapheme_aligned(s), Range::new(0, 2));
+        assert_eq!(Range::new(2, 3).grapheme_aligned(s), Range::new(2, 3));
+        assert_eq!(Range::new(3, 4).grapheme_aligned(s), Range::new(3, 4));
+        assert_eq!(Range::new(4, 5).grapheme_aligned(s), Range::new(4, 6));
+        assert_eq!(Range::new(5, 6).grapheme_aligned(s), Range::new(4, 6));
+
+        assert_eq!(Range::new(0, 2).grapheme_aligned(s), Range::new(0, 2));
+        assert_eq!(Range::new(1, 3).grapheme_aligned(s), Range::new(0, 3));
+        assert_eq!(Range::new(2, 4).grapheme_aligned(s), Range::new(2, 4));
+        assert_eq!(Range::new(3, 5).grapheme_aligned(s), Range::new(3, 6));
+        assert_eq!(Range::new(4, 6).grapheme_aligned(s), Range::new(4, 6));
+
+        // Reverse.
+        assert_eq!(Range::new(1, 0).grapheme_aligned(s), Range::new(2, 0));
+        assert_eq!(Range::new(2, 1).grapheme_aligned(s), Range::new(2, 0));
+        assert_eq!(Range::new(3, 2).grapheme_aligned(s), Range::new(3, 2));
+        assert_eq!(Range::new(4, 3).grapheme_aligned(s), Range::new(4, 3));
+        assert_eq!(Range::new(5, 4).grapheme_aligned(s), Range::new(6, 4));
+        assert_eq!(Range::new(6, 5).grapheme_aligned(s), Range::new(6, 4));
+
+        assert_eq!(Range::new(2, 0).grapheme_aligned(s), Range::new(2, 0));
+        assert_eq!(Range::new(3, 1).grapheme_aligned(s), Range::new(3, 0));
+        assert_eq!(Range::new(4, 2).grapheme_aligned(s), Range::new(4, 2));
+        assert_eq!(Range::new(5, 3).grapheme_aligned(s), Range::new(6, 3));
+        assert_eq!(Range::new(6, 4).grapheme_aligned(s), Range::new(6, 4));
+    }
+
+    #[test]
+    fn test_min_width_1() {
+        let r = Rope::from_str("\r\nHi\r\n");
+        let s = r.slice(..);
+
+        // Zero-width.
+        assert_eq!(Range::new(0, 0).min_width_1(s), Range::new(0, 2));
+        assert_eq!(Range::new(1, 1).min_width_1(s), Range::new(1, 2));
+        assert_eq!(Range::new(2, 2).min_width_1(s), Range::new(2, 3));
+        assert_eq!(Range::new(3, 3).min_width_1(s), Range::new(3, 4));
+        assert_eq!(Range::new(4, 4).min_width_1(s), Range::new(4, 6));
+        assert_eq!(Range::new(5, 5).min_width_1(s), Range::new(5, 6));
+        assert_eq!(Range::new(6, 6).min_width_1(s), Range::new(6, 6));
+
+        // Forward.
+        assert_eq!(Range::new(0, 1).min_width_1(s), Range::new(0, 1));
+        assert_eq!(Range::new(1, 2).min_width_1(s), Range::new(1, 2));
+        assert_eq!(Range::new(2, 3).min_width_1(s), Range::new(2, 3));
+        assert_eq!(Range::new(3, 4).min_width_1(s), Range::new(3, 4));
+        assert_eq!(Range::new(4, 5).min_width_1(s), Range::new(4, 5));
+        assert_eq!(Range::new(5, 6).min_width_1(s), Range::new(5, 6));
+
+        // Reverse.
+        assert_eq!(Range::new(1, 0).min_width_1(s), Range::new(1, 0));
+        assert_eq!(Range::new(2, 1).min_width_1(s), Range::new(2, 1));
+        assert_eq!(Range::new(3, 2).min_width_1(s), Range::new(3, 2));
+        assert_eq!(Range::new(4, 3).min_width_1(s), Range::new(4, 3));
+        assert_eq!(Range::new(5, 4).min_width_1(s), Range::new(5, 4));
+        assert_eq!(Range::new(6, 5).min_width_1(s), Range::new(6, 5));
+    }
+
+    #[test]
+    fn test_line_range() {
+        let r = Rope::from_str("\r\nHi\r\nthere!");
+        let s = r.slice(..);
+
+        // Zero-width ranges.
+        assert_eq!(Range::new(0, 0).line_range(s), (0, 0));
+        assert_eq!(Range::new(1, 1).line_range(s), (0, 0));
+        assert_eq!(Range::new(2, 2).line_range(s), (1, 1));
+        assert_eq!(Range::new(3, 3).line_range(s), (1, 1));
+
+        // Forward ranges.
+        assert_eq!(Range::new(0, 1).line_range(s), (0, 0));
+        assert_eq!(Range::new(0, 2).line_range(s), (0, 0));
+        assert_eq!(Range::new(0, 3).line_range(s), (0, 1));
+        assert_eq!(Range::new(1, 2).line_range(s), (0, 0));
+        assert_eq!(Range::new(2, 3).line_range(s), (1, 1));
+        assert_eq!(Range::new(3, 8).line_range(s), (1, 2));
+        assert_eq!(Range::new(0, 12).line_range(s), (0, 2));
+
+        // Reverse ranges.
+        assert_eq!(Range::new(1, 0).line_range(s), (0, 0));
+        assert_eq!(Range::new(2, 0).line_range(s), (0, 0));
+        assert_eq!(Range::new(3, 0).line_range(s), (0, 1));
+        assert_eq!(Range::new(2, 1).line_range(s), (0, 0));
+        assert_eq!(Range::new(3, 2).line_range(s), (1, 1));
+        assert_eq!(Range::new(8, 3).line_range(s), (1, 2));
+        assert_eq!(Range::new(12, 0).line_range(s), (0, 2));
+    }
+
+    #[test]
+    fn test_cursor() {
+        let r = Rope::from_str("\r\nHi\r\nthere!");
+        let s = r.slice(..);
+
+        // Zero-width ranges.
+        assert_eq!(Range::new(0, 0).cursor(s), 0);
+        assert_eq!(Range::new(2, 2).cursor(s), 2);
+        assert_eq!(Range::new(3, 3).cursor(s), 3);
+
+        // Forward ranges.
+        assert_eq!(Range::new(0, 2).cursor(s), 0);
+        assert_eq!(Range::new(0, 3).cursor(s), 2);
+        assert_eq!(Range::new(3, 6).cursor(s), 4);
+
+        // Reverse ranges.
+        assert_eq!(Range::new(2, 0).cursor(s), 0);
+        assert_eq!(Range::new(6, 2).cursor(s), 2);
+        assert_eq!(Range::new(6, 3).cursor(s), 3);
+    }
+
+    #[test]
+    fn test_put_cursor() {
+        let r = Rope::from_str("\r\nHi\r\nthere!");
+        let s = r.slice(..);
+
+        // Zero-width ranges.
+        assert_eq!(Range::new(0, 0).put_cursor(s, 0, true), Range::new(0, 2));
+        assert_eq!(Range::new(0, 0).put_cursor(s, 2, true), Range::new(0, 3));
+        assert_eq!(Range::new(2, 3).put_cursor(s, 4, true), Range::new(2, 6));
+        assert_eq!(Range::new(2, 8).put_cursor(s, 4, true), Range::new(2, 6));
+        assert_eq!(Range::new(8, 8).put_cursor(s, 4, true), Range::new(9, 4));
+
+        // Forward ranges.
+        assert_eq!(Range::new(3, 6).put_cursor(s, 0, true), Range::new(4, 0));
+        assert_eq!(Range::new(3, 6).put_cursor(s, 2, true), Range::new(4, 2));
+        assert_eq!(Range::new(3, 6).put_cursor(s, 3, true), Range::new(3, 4));
+        assert_eq!(Range::new(3, 6).put_cursor(s, 4, true), Range::new(3, 6));
+        assert_eq!(Range::new(3, 6).put_cursor(s, 6, true), Range::new(3, 7));
+        assert_eq!(Range::new(3, 6).put_cursor(s, 8, true), Range::new(3, 9));
+
+        // Reverse ranges.
+        assert_eq!(Range::new(6, 3).put_cursor(s, 0, true), Range::new(6, 0));
+        assert_eq!(Range::new(6, 3).put_cursor(s, 2, true), Range::new(6, 2));
+        assert_eq!(Range::new(6, 3).put_cursor(s, 3, true), Range::new(6, 3));
+        assert_eq!(Range::new(6, 3).put_cursor(s, 4, true), Range::new(6, 4));
+        assert_eq!(Range::new(6, 3).put_cursor(s, 6, true), Range::new(4, 7));
+        assert_eq!(Range::new(6, 3).put_cursor(s, 8, true), Range::new(4, 9));
     }
 
     #[test]
     fn test_split_on_matches() {
         use crate::regex::Regex;
 
-        let text = Rope::from("abcd efg wrs   xyz 123 456");
+        let text = Rope::from(" abcd efg wrs   xyz 123 456");
 
-        let selection = Selection::new(smallvec![Range::new(0, 8), Range::new(10, 19),], 0);
+        let selection = Selection::new(smallvec![Range::new(0, 9), Range::new(11, 20),], 0);
 
         let result = split_on_matches(text.slice(..), &selection, &Regex::new(r"\s+").unwrap());
 
         assert_eq!(
             result.ranges(),
             &[
-                Range::new(0, 3),
+                // TODO: rather than this behavior, maybe we want it
-                Range::new(5, 7),
+                // to be based on which side is the anchor?
-                Range::new(10, 11),
+                //
-                Range::new(15, 17),
+                // We get a leading zero-width range when there's
-                Range::new(19, 19),
+                // a leading match because ranges are inclusive on
+                // the left.  Imagine, for example, if the entire
+                // selection range were matched: you'd still want
+                // at least one range to remain after the split.
+                Range::new(0, 0),
+                Range::new(1, 5),
+                Range::new(6, 9),
+                Range::new(11, 13),
+                Range::new(16, 19),
+                // In contrast to the comment above, there is no
+                // _trailing_ zero-width range despite the trailing
+                // match, because ranges are exclusive on the right.
             ]
         );
 
         assert_eq!(
             result.fragments(text.slice(..)).collect::<Vec<_>>(),
-            &["abcd", "efg", "rs", "xyz", "1"]
+            &["", "abcd", "efg", "rs", "xyz"]
         );
     }
 }

helix-core/src/shellwords.rs

@@ -0,0 +1,164 @@
+use std::borrow::Cow;
+
+/// Get the vec of escaped / quoted / doublequoted filenames from the input str
+pub fn shellwords(input: &str) -> Vec<Cow<'_, str>> {
+    enum State {
+        Normal,
+        NormalEscaped,
+        Quoted,
+        QuoteEscaped,
+        Dquoted,
+        DquoteEscaped,
+    }
+
+    use State::*;
+
+    let mut state = Normal;
+    let mut args: Vec<Cow<str>> = Vec::new();
+    let mut escaped = String::with_capacity(input.len());
+
+    let mut start = 0;
+    let mut end = 0;
+
+    for (i, c) in input.char_indices() {
+        state = match state {
+            Normal => match c {
+                '\\' => {
+                    escaped.push_str(&input[start..i]);
+                    start = i + 1;
+                    NormalEscaped
+                }
+                '"' => {
+                    end = i;
+                    Dquoted
+                }
+                '\'' => {
+                    end = i;
+                    Quoted
+                }
+                c if c.is_ascii_whitespace() => {
+                    end = i;
+                    Normal
+                }
+                _ => Normal,
+            },
+            NormalEscaped => Normal,
+            Quoted => match c {
+                '\\' => {
+                    escaped.push_str(&input[start..i]);
+                    start = i + 1;
+                    QuoteEscaped
+                }
+                '\'' => {
+                    end = i;
+                    Normal
+                }
+                _ => Quoted,
+            },
+            QuoteEscaped => Quoted,
+            Dquoted => match c {
+                '\\' => {
+                    escaped.push_str(&input[start..i]);
+                    start = i + 1;
+                    DquoteEscaped
+                }
+                '"' => {
+                    end = i;
+                    Normal
+                }
+                _ => Dquoted,
+            },
+            DquoteEscaped => Dquoted,
+        };
+
+        if i >= input.len() - 1 && end == 0 {
+            end = i + 1;
+        }
+
+        if end > 0 {
+            let esc_trim = escaped.trim();
+            let inp = &input[start..end];
+
+            if !(esc_trim.is_empty() && inp.trim().is_empty()) {
+                if esc_trim.is_empty() {
+                    args.push(inp.into());
+                } else {
+                    args.push([escaped, inp.into()].concat().into());
+                    escaped = "".to_string();
+                }
+            }
+            start = i + 1;
+            end = 0;
+        }
+    }
+    args
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_normal() {
+        let input = r#":o single_word twó wörds \three\ \"with\ escaping\\"#;
+        let result = shellwords(input);
+        let expected = vec![
+            Cow::from(":o"),
+            Cow::from("single_word"),
+            Cow::from("twó"),
+            Cow::from("wörds"),
+            Cow::from(r#"three "with escaping\"#),
+        ];
+        // TODO test is_owned and is_borrowed, once they get stabilized.
+        assert_eq!(expected, result);
+    }
+
+    #[test]
+    fn test_quoted() {
+        let quoted =
+            r#":o 'single_word' 'twó wörds' '' ' ''\three\' \"with\ escaping\\' 'quote incomplete"#;
+        let result = shellwords(quoted);
+        let expected = vec![
+            Cow::from(":o"),
+            Cow::from("single_word"),
+            Cow::from("twó wörds"),
+            Cow::from(r#"three' "with escaping\"#),
+            Cow::from("quote incomplete"),
+        ];
+        assert_eq!(expected, result);
+    }
+
+    #[test]
+    fn test_dquoted() {
+        let dquoted = r#":o "single_word" "twó wörds" "" "  ""\three\' \"with\ escaping\\" "dquote incomplete"#;
+        let result = shellwords(dquoted);
+        let expected = vec![
+            Cow::from(":o"),
+            Cow::from("single_word"),
+            Cow::from("twó wörds"),
+            Cow::from(r#"three' "with escaping\"#),
+            Cow::from("dquote incomplete"),
+        ];
+        assert_eq!(expected, result);
+    }
+
+    #[test]
+    fn test_mixed() {
+        let dquoted = r#":o single_word 'twó wörds' "\three\' \"with\ escaping\\""no space before"'and after' $#%^@ "%^&(%^" ')(*&^%''a\\\\\b' '"#;
+        let result = shellwords(dquoted);
+        let expected = vec![
+            Cow::from(":o"),
+            Cow::from("single_word"),
+            Cow::from("twó wörds"),
+            Cow::from("three' \"with escaping\\"),
+            Cow::from("no space before"),
+            Cow::from("and after"),
+            Cow::from("$#%^@"),
+            Cow::from("%^&(%^"),
+            Cow::from(")(*&^%"),
+            Cow::from(r#"a\\b"#),
+            //last ' just changes to quoted but since we dont have anything after it, it should be ignored
+        ];
+        assert_eq!(expected, result);
+    }
+}

helix-core/src/state.rs

@@ -1,7 +1,6 @@
 use crate::{Rope, Selection};
 
-/// A state represents the current editor state of a single buffer.
+#[derive(Debug, Clone)]
-#[derive(Clone)]
 pub struct State {
     pub doc: Rope,
     pub selection: Selection,
@@ -15,27 +14,4 @@ impl State {
             selection: Selection::point(0),
         }
     }
-
-    // update/transact:
-    // update(desc) => transaction ?  transaction.doc() for applied doc
-    // transaction.apply(doc)
-    // doc.transact(fn -> ... end)
-
-    // replaceSelection (transaction that replaces selection)
-    // changeByRange
-    // changes
-    // slice
-    //
-    // getters:
-    // tabSize
-    // indentUnit
-    // languageDataAt()
-    //
-    // config:
-    // indentation
-    // tabSize
-    // lineUnit
-    // syntax
-    // foldable
-    // changeFilter/transactionFilter
 }

helix-core/src/surround.rs

@@ -0,0 +1,304 @@
+use crate::{search, Range, Selection};
+use ropey::RopeSlice;
+
+pub const PAIRS: &[(char, char)] = &[
+    ('(', ')'),
+    ('[', ']'),
+    ('{', '}'),
+    ('<', '>'),
+    ('«', '»'),
+    ('「', '」'),
+    ('（', '）'),
+];
+
+/// Given any char in [PAIRS], return the open and closing chars. If not found in
+/// [PAIRS] return (ch, ch).
+///
+/// ```
+/// use helix_core::surround::get_pair;
+///
+/// assert_eq!(get_pair('['), ('[', ']'));
+/// assert_eq!(get_pair('}'), ('{', '}'));
+/// assert_eq!(get_pair('"'), ('"', '"'));
+/// ```
+pub fn get_pair(ch: char) -> (char, char) {
+    PAIRS
+        .iter()
+        .find(|(open, close)| *open == ch || *close == ch)
+        .copied()
+        .unwrap_or((ch, ch))
+}
+
+/// Find the position of surround pairs of `ch` which can be either a closing
+/// or opening pair. `n` will skip n - 1 pairs (eg. n=2 will discard (only)
+/// the first pair found and keep looking)
+pub fn find_nth_pairs_pos(
+    text: RopeSlice,
+    ch: char,
+    range: Range,
+    n: usize,
+) -> Option<(usize, usize)> {
+    if text.len_chars() < 2 || range.to() >= text.len_chars() {
+        return None;
+    }
+
+    let (open, close) = get_pair(ch);
+    let pos = range.cursor(text);
+
+    if open == close {
+        if Some(open) == text.get_char(pos) {
+            // Cursor is directly on match char. We return no match
+            // because there's no way to know which side of the char
+            // we should be searching on.
+            return None;
+        }
+        Some((
+            search::find_nth_prev(text, open, pos, n)?,
+            search::find_nth_next(text, close, pos, n)?,
+        ))
+    } else {
+        Some((
+            find_nth_open_pair(text, open, close, pos, n)?,
+            find_nth_close_pair(text, open, close, pos, n)?,
+        ))
+    }
+}
+
+fn find_nth_open_pair(
+    text: RopeSlice,
+    open: char,
+    close: char,
+    mut pos: usize,
+    n: usize,
+) -> Option<usize> {
+    let mut chars = text.chars_at(pos + 1);
+
+    // Adjusts pos for the first iteration, and handles the case of the
+    // cursor being *on* the close character which will get falsely stepped over
+    // if not skipped here
+    if chars.prev()? == open {
+        return Some(pos);
+    }
+
+    for _ in 0..n {
+        let mut step_over: usize = 0;
+
+        loop {
+            let c = chars.prev()?;
+            pos = pos.saturating_sub(1);
+
+            // ignore other surround pairs that are enclosed *within* our search scope
+            if c == close {
+                step_over += 1;
+            } else if c == open {
+                if step_over == 0 {
+                    break;
+                }
+
+                step_over = step_over.saturating_sub(1);
+            }
+        }
+    }
+
+    Some(pos)
+}
+
+fn find_nth_close_pair(
+    text: RopeSlice,
+    open: char,
+    close: char,
+    mut pos: usize,
+    n: usize,
+) -> Option<usize> {
+    if pos >= text.len_chars() {
+        return None;
+    }
+
+    let mut chars = text.chars_at(pos);
+
+    if chars.next()? == close {
+        return Some(pos);
+    }
+
+    for _ in 0..n {
+        let mut step_over: usize = 0;
+
+        loop {
+            let c = chars.next()?;
+            pos += 1;
+
+            if c == open {
+                step_over += 1;
+            } else if c == close {
+                if step_over == 0 {
+                    break;
+                }
+
+                step_over = step_over.saturating_sub(1);
+            }
+        }
+    }
+
+    Some(pos)
+}
+
+/// Find position of surround characters around every cursor. Returns None
+/// if any positions overlap. Note that the positions are in a flat Vec.
+/// Use get_surround_pos().chunks(2) to get matching pairs of surround positions.
+/// `ch` can be either closing or opening pair.
+pub fn get_surround_pos(
+    text: RopeSlice,
+    selection: &Selection,
+    ch: char,
+    skip: usize,
+) -> Option<Vec<usize>> {
+    let mut change_pos = Vec::new();
+
+    for &range in selection {
+        let (open_pos, close_pos) = find_nth_pairs_pos(text, ch, range, skip)?;
+        if change_pos.contains(&open_pos) || change_pos.contains(&close_pos) {
+            return None;
+        }
+        change_pos.extend_from_slice(&[open_pos, close_pos]);
+    }
+    Some(change_pos)
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::Range;
+
+    use ropey::Rope;
+    use smallvec::SmallVec;
+
+    fn check_find_nth_pair_pos(
+        text: &str,
+        cases: Vec<(usize, char, usize, Option<(usize, usize)>)>,
+    ) {
+        let doc = Rope::from(text);
+        let slice = doc.slice(..);
+
+        for (cursor_pos, ch, n, expected_range) in cases {
+            let range = find_nth_pairs_pos(slice, ch, (cursor_pos, cursor_pos + 1).into(), n);
+            assert_eq!(
+                range, expected_range,
+                "Expected {:?}, got {:?}",
+                expected_range, range
+            );
+        }
+    }
+
+    #[test]
+    fn test_find_nth_pairs_pos() {
+        check_find_nth_pair_pos(
+            "some (text) here",
+            vec![
+                // cursor on [t]ext
+                (6, '(', 1, Some((5, 10))),
+                (6, ')', 1, Some((5, 10))),
+                // cursor on so[m]e
+                (2, '(', 1, None),
+                // cursor on bracket itself
+                (5, '(', 1, Some((5, 10))),
+                (10, '(', 1, Some((5, 10))),
+            ],
+        );
+    }
+
+    #[test]
+    fn test_find_nth_pairs_pos_skip() {
+        check_find_nth_pair_pos(
+            "(so (many (good) text) here)",
+            vec![
+                // cursor on go[o]d
+                (13, '(', 1, Some((10, 15))),
+                (13, '(', 2, Some((4, 21))),
+                (13, '(', 3, Some((0, 27))),
+            ],
+        );
+    }
+
+    #[test]
+    fn test_find_nth_pairs_pos_same() {
+        check_find_nth_pair_pos(
+            "'so 'many 'good' text' here'",
+            vec![
+                // cursor on go[o]d
+                (13, '\'', 1, Some((10, 15))),
+                (13, '\'', 2, Some((4, 21))),
+                (13, '\'', 3, Some((0, 27))),
+                // cursor on the quotes
+                (10, '\'', 1, None),
+            ],
+        )
+    }
+
+    #[test]
+    fn test_find_nth_pairs_pos_step() {
+        check_find_nth_pair_pos(
+            "((so)((many) good (text))(here))",
+            vec![
+                // cursor on go[o]d
+                (15, '(', 1, Some((5, 24))),
+                (15, '(', 2, Some((0, 31))),
+            ],
+        )
+    }
+
+    #[test]
+    fn test_find_nth_pairs_pos_mixed() {
+        check_find_nth_pair_pos(
+            "(so [many {good} text] here)",
+            vec![
+                // cursor on go[o]d
+                (13, '{', 1, Some((10, 15))),
+                (13, '[', 1, Some((4, 21))),
+                (13, '(', 1, Some((0, 27))),
+            ],
+        )
+    }
+
+    #[test]
+    fn test_get_surround_pos() {
+        let doc = Rope::from("(some) (chars)\n(newline)");
+        let slice = doc.slice(..);
+        let selection = Selection::new(
+            SmallVec::from_slice(&[Range::point(2), Range::point(9), Range::point(20)]),
+            0,
+        );
+
+        // cursor on s[o]me, c[h]ars, newl[i]ne
+        assert_eq!(
+            get_surround_pos(slice, &selection, '(', 1)
+                .unwrap()
+                .as_slice(),
+            &[0, 5, 7, 13, 15, 23]
+        );
+    }
+
+    #[test]
+    fn test_get_surround_pos_bail() {
+        let doc = Rope::from("[some]\n(chars)xx\n(newline)");
+        let slice = doc.slice(..);
+
+        let selection =
+            Selection::new(SmallVec::from_slice(&[Range::point(2), Range::point(9)]), 0);
+
+        // cursor on s[o]me, c[h]ars
+        assert_eq!(
+            get_surround_pos(slice, &selection, '(', 1),
+            None // different surround chars
+        );
+
+        let selection = Selection::new(
+            SmallVec::from_slice(&[Range::point(14), Range::point(24)]),
+            0,
+        );
+        // cursor on [x]x, newli[n]e
+        assert_eq!(
+            get_surround_pos(slice, &selection, '(', 1),
+            None // overlapping surround chars
+        );
+    }
+}

helix-core/src/textobject.rs

@@ -0,0 +1,375 @@
+use std::fmt::Display;
+
+use ropey::RopeSlice;
+use tree_sitter::{Node, QueryCursor};
+
+use crate::chars::{categorize_char, char_is_whitespace, CharCategory};
+use crate::graphemes::next_grapheme_boundary;
+use crate::movement::Direction;
+use crate::surround;
+use crate::syntax::LanguageConfiguration;
+use crate::Range;
+
+fn find_word_boundary(slice: RopeSlice, mut pos: usize, direction: Direction, long: bool) -> usize {
+    use CharCategory::{Eol, Whitespace};
+
+    let iter = match direction {
+        Direction::Forward => slice.chars_at(pos),
+        Direction::Backward => {
+            let mut iter = slice.chars_at(pos);
+            iter.reverse();
+            iter
+        }
+    };
+
+    let mut prev_category = match direction {
+        Direction::Forward if pos == 0 => Whitespace,
+        Direction::Forward => categorize_char(slice.char(pos - 1)),
+        Direction::Backward if pos == slice.len_chars() => Whitespace,
+        Direction::Backward => categorize_char(slice.char(pos)),
+    };
+
+    for ch in iter {
+        match categorize_char(ch) {
+            Eol | Whitespace => return pos,
+            category => {
+                if !long && category != prev_category && pos != 0 && pos != slice.len_chars() {
+                    return pos;
+                } else {
+                    match direction {
+                        Direction::Forward => pos += 1,
+                        Direction::Backward => pos = pos.saturating_sub(1),
+                    }
+                    prev_category = category;
+                }
+            }
+        }
+    }
+
+    pos
+}
+
+#[derive(Copy, Clone, PartialEq, Eq, Debug)]
+pub enum TextObject {
+    Around,
+    Inside,
+}
+
+impl Display for TextObject {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(match self {
+            Self::Around => "around",
+            Self::Inside => "inside",
+        })
+    }
+}
+
+// count doesn't do anything yet
+pub fn textobject_word(
+    slice: RopeSlice,
+    range: Range,
+    textobject: TextObject,
+    _count: usize,
+    long: bool,
+) -> Range {
+    let pos = range.cursor(slice);
+
+    let word_start = find_word_boundary(slice, pos, Direction::Backward, long);
+    let word_end = match slice.get_char(pos).map(categorize_char) {
+        None | Some(CharCategory::Whitespace | CharCategory::Eol) => pos,
+        _ => find_word_boundary(slice, pos + 1, Direction::Forward, long),
+    };
+
+    // Special case.
+    if word_start == word_end {
+        return Range::new(word_start, word_end);
+    }
+
+    match textobject {
+        TextObject::Inside => Range::new(word_start, word_end),
+        TextObject::Around => {
+            let whitespace_count_right = slice
+                .chars_at(word_end)
+                .take_while(|c| char_is_whitespace(*c))
+                .count();
+
+            if whitespace_count_right > 0 {
+                Range::new(word_start, word_end + whitespace_count_right)
+            } else {
+                let whitespace_count_left = {
+                    let mut iter = slice.chars_at(word_start);
+                    iter.reverse();
+                    iter.take_while(|c| char_is_whitespace(*c)).count()
+                };
+                Range::new(word_start - whitespace_count_left, word_end)
+            }
+        }
+    }
+}
+
+pub fn textobject_surround(
+    slice: RopeSlice,
+    range: Range,
+    textobject: TextObject,
+    ch: char,
+    count: usize,
+) -> Range {
+    surround::find_nth_pairs_pos(slice, ch, range, count)
+        .map(|(anchor, head)| match textobject {
+            TextObject::Inside => Range::new(next_grapheme_boundary(slice, anchor), head),
+            TextObject::Around => Range::new(anchor, next_grapheme_boundary(slice, head)),
+        })
+        .unwrap_or(range)
+}
+
+/// Transform the given range to select text objects based on tree-sitter.
+/// `object_name` is a query capture base name like "function", "class", etc.
+/// `slice_tree` is the tree-sitter node corresponding to given text slice.
+pub fn textobject_treesitter(
+    slice: RopeSlice,
+    range: Range,
+    textobject: TextObject,
+    object_name: &str,
+    slice_tree: Node,
+    lang_config: &LanguageConfiguration,
+    _count: usize,
+) -> Range {
+    let get_range = move || -> Option<Range> {
+        let byte_pos = slice.char_to_byte(range.cursor(slice));
+
+        let capture_name = format!("{}.{}", object_name, textobject); // eg. function.inner
+        let mut cursor = QueryCursor::new();
+        let node = lang_config
+            .textobject_query()?
+            .capture_nodes(&capture_name, slice_tree, slice, &mut cursor)?
+            .filter(|node| node.byte_range().contains(&byte_pos))
+            .min_by_key(|node| node.byte_range().len())?;
+
+        let len = slice.len_bytes();
+        let start_byte = node.start_byte();
+        let end_byte = node.end_byte();
+        if start_byte >= len || end_byte >= len {
+            return None;
+        }
+
+        let start_char = slice.byte_to_char(start_byte);
+        let end_char = slice.byte_to_char(end_byte);
+
+        Some(Range::new(start_char, end_char))
+    };
+    get_range().unwrap_or(range)
+}
+
+#[cfg(test)]
+mod test {
+    use super::TextObject::*;
+    use super::*;
+
+    use crate::Range;
+    use ropey::Rope;
+
+    #[test]
+    fn test_textobject_word() {
+        // (text, [(char position, textobject, final range), ...])
+        let tests = &[
+            (
+                "cursor at beginning of doc",
+                vec![(0, Inside, (0, 6)), (0, Around, (0, 7))],
+            ),
+            (
+                "cursor at middle of word",
+                vec![
+                    (13, Inside, (10, 16)),
+                    (10, Inside, (10, 16)),
+                    (15, Inside, (10, 16)),
+                    (13, Around, (10, 17)),
+                    (10, Around, (10, 17)),
+                    (15, Around, (10, 17)),
+                ],
+            ),
+            (
+                "cursor between word whitespace",
+                vec![(6, Inside, (6, 6)), (6, Around, (6, 6))],
+            ),
+            (
+                "cursor on word before newline\n",
+                vec![
+                    (22, Inside, (22, 29)),
+                    (28, Inside, (22, 29)),
+                    (25, Inside, (22, 29)),
+                    (22, Around, (21, 29)),
+                    (28, Around, (21, 29)),
+                    (25, Around, (21, 29)),
+                ],
+            ),
+            (
+                "cursor on newline\nnext line",
+                vec![(17, Inside, (17, 17)), (17, Around, (17, 17))],
+            ),
+            (
+                "cursor on word after newline\nnext line",
+                vec![
+                    (29, Inside, (29, 33)),
+                    (30, Inside, (29, 33)),
+                    (32, Inside, (29, 33)),
+                    (29, Around, (29, 34)),
+                    (30, Around, (29, 34)),
+                    (32, Around, (29, 34)),
+                ],
+            ),
+            (
+                "cursor on #$%:;* punctuation",
+                vec![
+                    (13, Inside, (10, 16)),
+                    (10, Inside, (10, 16)),
+                    (15, Inside, (10, 16)),
+                    (13, Around, (10, 17)),
+                    (10, Around, (10, 17)),
+                    (15, Around, (10, 17)),
+                ],
+            ),
+            (
+                "cursor on punc%^#$:;.tuation",
+                vec![
+                    (14, Inside, (14, 21)),
+                    (20, Inside, (14, 21)),
+                    (17, Inside, (14, 21)),
+                    (14, Around, (14, 21)),
+                    (20, Around, (14, 21)),
+                    (17, Around, (14, 21)),
+                ],
+            ),
+            (
+                "cursor in   extra whitespace",
+                vec![
+                    (9, Inside, (9, 9)),
+                    (10, Inside, (10, 10)),
+                    (11, Inside, (11, 11)),
+                    (9, Around, (9, 9)),
+                    (10, Around, (10, 10)),
+                    (11, Around, (11, 11)),
+                ],
+            ),
+            (
+                "cursor on word   with extra whitespace",
+                vec![(11, Inside, (10, 14)), (11, Around, (10, 17))],
+            ),
+            (
+                "cursor at end with extra   whitespace",
+                vec![(28, Inside, (27, 37)), (28, Around, (24, 37))],
+            ),
+            (
+                "cursor at end of doc",
+                vec![(19, Inside, (17, 20)), (19, Around, (16, 20))],
+            ),
+        ];
+
+        for (sample, scenario) in tests {
+            let doc = Rope::from(*sample);
+            let slice = doc.slice(..);
+            for &case in scenario {
+                let (pos, objtype, expected_range) = case;
+                // cursor is a single width selection
+                let range = Range::new(pos, pos + 1);
+                let result = textobject_word(slice, range, objtype, 1, false);
+                assert_eq!(
+                    result,
+                    expected_range.into(),
+                    "\nCase failed: {:?} - {:?}",
+                    sample,
+                    case
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn test_textobject_surround() {
+        // (text, [(cursor position, textobject, final range, surround char, count), ...])
+        let tests = &[
+            (
+                "simple (single) surround pairs",
+                vec![
+                    (3, Inside, (3, 3), '(', 1),
+                    (7, Inside, (8, 14), ')', 1),
+                    (10, Inside, (8, 14), '(', 1),
+                    (14, Inside, (8, 14), ')', 1),
+                    (3, Around, (3, 3), '(', 1),
+                    (7, Around, (7, 15), ')', 1),
+                    (10, Around, (7, 15), '(', 1),
+                    (14, Around, (7, 15), ')', 1),
+                ],
+            ),
+            (
+                "samexx 'single' surround pairs",
+                vec![
+                    (3, Inside, (3, 3), '\'', 1),
+                    (7, Inside, (7, 7), '\'', 1),
+                    (10, Inside, (8, 14), '\'', 1),
+                    (14, Inside, (14, 14), '\'', 1),
+                    (3, Around, (3, 3), '\'', 1),
+                    (7, Around, (7, 7), '\'', 1),
+                    (10, Around, (7, 15), '\'', 1),
+                    (14, Around, (14, 14), '\'', 1),
+                ],
+            ),
+            (
+                "(nested (surround (pairs)) 3 levels)",
+                vec![
+                    (0, Inside, (1, 35), '(', 1),
+                    (6, Inside, (1, 35), ')', 1),
+                    (8, Inside, (9, 25), '(', 1),
+                    (8, Inside, (9, 35), ')', 2),
+                    (20, Inside, (9, 25), '(', 2),
+                    (20, Inside, (1, 35), ')', 3),
+                    (0, Around, (0, 36), '(', 1),
+                    (6, Around, (0, 36), ')', 1),
+                    (8, Around, (8, 26), '(', 1),
+                    (8, Around, (8, 36), ')', 2),
+                    (20, Around, (8, 26), '(', 2),
+                    (20, Around, (0, 36), ')', 3),
+                ],
+            ),
+            (
+                "(mixed {surround [pair] same} line)",
+                vec![
+                    (2, Inside, (1, 34), '(', 1),
+                    (9, Inside, (8, 28), '{', 1),
+                    (18, Inside, (18, 22), '[', 1),
+                    (2, Around, (0, 35), '(', 1),
+                    (9, Around, (7, 29), '{', 1),
+                    (18, Around, (17, 23), '[', 1),
+                ],
+            ),
+            (
+                "(stepped (surround) pairs (should) skip)",
+                vec![(22, Inside, (1, 39), '(', 1), (22, Around, (0, 40), '(', 1)],
+            ),
+            (
+                "[surround pairs{\non different]\nlines}",
+                vec![
+                    (7, Inside, (1, 29), '[', 1),
+                    (15, Inside, (16, 36), '{', 1),
+                    (7, Around, (0, 30), '[', 1),
+                    (15, Around, (15, 37), '{', 1),
+                ],
+            ),
+        ];
+
+        for (sample, scenario) in tests {
+            let doc = Rope::from(*sample);
+            let slice = doc.slice(..);
+            for &case in scenario {
+                let (pos, objtype, expected_range, ch, count) = case;
+                let result = textobject_surround(slice, Range::point(pos), objtype, ch, count);
+                assert_eq!(
+                    result,
+                    expected_range.into(),
+                    "\nCase failed: {:?} - {:?}",
+                    sample,
+                    case
+                );
+            }
+        }
+    }
+}

helix-core/src/transaction.rs

@@ -1,5 +1,5 @@
-use crate::{Range, Rope, Selection, State, Tendril};
+use crate::{Range, Rope, Selection, Tendril};
-use std::{borrow::Cow, convert::TryFrom};
+use std::borrow::Cow;
 
 /// (from, to, replacement)
 pub type Change = (usize, usize, Option<Tendril>);
@@ -15,14 +15,14 @@ pub enum Operation {
     Insert(Tendril),
 }
 
-#[derive(Copy, Clone, PartialEq, Eq)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
 pub enum Assoc {
     Before,
     After,
 }
 
 // ChangeSpec = Change | ChangeSet | Vec<Change>
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Default, Clone, PartialEq, Eq)]
 pub struct ChangeSet {
     pub(crate) changes: Vec<Operation>,
     /// The required document length. Will refuse to apply changes unless it matches.
@@ -30,16 +30,6 @@ pub struct ChangeSet {
     len_after: usize,
 }
 
-impl Default for ChangeSet {
-    fn default() -> Self {
-        Self {
-            changes: Vec::new(),
-            len: 0,
-            len_after: 0,
-        }
-    }
-}
-
 impl ChangeSet {
     pub fn with_capacity(capacity: usize) -> Self {
         Self {
@@ -125,13 +115,16 @@ impl ChangeSet {
     /// In other words,  If `this` goes `docA` → `docB` and `other` represents `docB` → `docC`, the
     /// returned value will represent the change `docA` → `docC`.
     pub fn compose(self, other: Self) -> Self {
-        debug_assert!(self.len_after == other.len);
+        assert!(self.len_after == other.len);
 
         // composing fails in weird ways if one of the sets is empty
         // a: [] len: 0 len_after: 1 | b: [Insert(Tendril<UTF8>(inline: "\n")), Retain(1)] len 1
         if self.changes.is_empty() {
             return other;
         }
+        if other.changes.is_empty() {
+            return self;
+        }
 
         let len = self.changes.len();
 
@@ -163,7 +156,7 @@ impl ChangeSet {
                     head_a = a;
                     head_b = changes_b.next();
                 }
-                (None, _) | (_, None) => return unreachable!(),
+                (None, val) | (val, None) => unreachable!("({:?})", val),
                 (Some(Retain(i)), Some(Retain(j))) => match i.cmp(&j) {
                     Ordering::Less => {
                         changes.retain(i);
@@ -193,15 +186,16 @@ impl ChangeSet {
                             head_b = changes_b.next();
                         }
                         Ordering::Greater => {
+                            // TODO: cover this with a test
                             // figure out the byte index of the truncated string end
-                            let (pos, _) = s.char_indices().nth(len - j).unwrap();
+                            let (pos, _) = s.char_indices().nth(j).unwrap();
-                            s.pop_front(s.len() as u32 - pos as u32);
+                            s.pop_front(pos as u32);
                             head_a = Some(Insert(s));
                             head_b = changes_b.next();
                         }
                     }
                 }
-                (Some(Insert(mut s)), Some(Retain(j))) => {
+                (Some(Insert(s)), Some(Retain(j))) => {
                     let len = s.chars().count();
                     match len.cmp(&j) {
                         Ordering::Less => {
@@ -273,7 +267,6 @@ impl ChangeSet {
         let mut changes = Self::with_capacity(self.changes.len());
 
         let mut pos = 0;
-        let mut len = 0;
 
         for change in &self.changes {
             use Operation::*;
@@ -327,7 +320,7 @@ impl ChangeSet {
     /// `true` when the set is empty.
     #[inline]
     pub fn is_empty(&self) -> bool {
-        self.changes.is_empty()
+        self.changes.is_empty() || self.changes == [Operation::Retain(self.len)]
     }
 
     /// Map a position through the changes.
@@ -416,7 +409,7 @@ impl ChangeSet {
 
 /// Transaction represents a single undoable unit of changes. Several changes can be grouped into
 /// a single transaction.
-#[derive(Debug, Default, Clone)]
+#[derive(Debug, Default, Clone, PartialEq, Eq)]
 pub struct Transaction {
     changes: ChangeSet,
     selection: Option<Selection>,
@@ -465,6 +458,13 @@ impl Transaction {
         }
     }
 
+    pub fn compose(mut self, other: Self) -> Self {
+        self.changes = self.changes.compose(other.changes);
+        // Other selection takes precedence
+        self.selection = other.selection;
+        self
+    }
+
     pub fn with_selection(mut self, selection: Selection) -> Self {
         self.selection = Some(selection);
         self
@@ -473,11 +473,13 @@ impl Transaction {
     /// Generate a transaction from a set of changes.
     pub fn change<I>(doc: &Rope, changes: I) -> Self
     where
-        I: IntoIterator<Item = Change> + ExactSizeIterator,
+        I: IntoIterator<Item = Change> + Iterator,
     {
         let len = doc.len_chars();
 
-        let mut changeset = ChangeSet::with_capacity(2 * changes.len() + 1); // rough estimate
+        let (lower, upper) = changes.size_hint();
+        let size = upper.unwrap_or(lower);
+        let mut changeset = ChangeSet::with_capacity(2 * size + 1); // rough estimate
 
         // TODO: verify ranges are ordered and not overlapping or change will panic.
 
@@ -580,6 +582,7 @@ impl<'a> Iterator for ChangeIterator<'a> {
 #[cfg(test)]
 mod test {
     use super::*;
+    use crate::State;
 
     #[test]
     fn composition() {
@@ -598,7 +601,7 @@ mod test {
         };
 
         let b = ChangeSet {
-            changes: vec![Delete(10), Insert("world".into()), Retain(5)],
+            changes: vec![Delete(10), Insert("世orld".into()), Retain(5)],
             len: 15,
             len_after: 10,
         };
@@ -609,7 +612,7 @@ mod test {
         let composed = a.compose(b);
         assert_eq!(composed.len, 8);
         assert!(composed.apply(&mut text));
-        assert_eq!(text, "world! abc");
+        assert_eq!(text, "世orld! abc");
     }
 
     #[test]
@@ -686,21 +689,21 @@ mod test {
 
     #[test]
     fn transaction_change() {
-        let mut state = State::new("hello world!\ntest 123".into());
+        let mut doc = Rope::from("hello world!\ntest 123");
         let transaction = Transaction::change(
-            &state.doc,
+            &doc,
             // (1, 1, None) is a useless 0-width delete
             vec![(1, 1, None), (6, 11, Some("void".into())), (12, 17, None)].into_iter(),
         );
-        transaction.apply(&mut state.doc);
+        transaction.apply(&mut doc);
-        assert_eq!(state.doc, Rope::from_str("hello void! 123"));
+        assert_eq!(doc, Rope::from_str("hello void! 123"));
     }
 
     #[test]
     fn changes_iter() {
-        let mut state = State::new("hello world!\ntest 123".into());
+        let doc = Rope::from("hello world!\ntest 123");
         let changes = vec![(6, 11, Some("void".into())), (12, 17, None)];
-        let transaction = Transaction::change(&state.doc, changes.clone().into_iter());
+        let transaction = Transaction::change(&doc, changes.clone().into_iter());
         assert_eq!(transaction.changes_iter().collect::<Vec<_>>(), changes);
     }
 
@@ -730,7 +733,7 @@ mod test {
         // retain 1, e
         // retain 2, l
 
-        let mut changes = t1
+        let changes = t1
             .changes
             .compose(t2.changes)
             .compose(t3.changes)
@@ -745,7 +748,7 @@ mod test {
     #[test]
     fn combine_with_empty() {
         let empty = Rope::from("");
-        let mut a = ChangeSet::new(&empty);
+        let a = ChangeSet::new(&empty);
 
         let mut b = ChangeSet::new(&empty);
         b.insert("a".into());
@@ -758,10 +761,10 @@ mod test {
 
     #[test]
     fn combine_with_utf8() {
-        const TEST_CASE: &'static str = "Hello, これはヒレクスエディターです！";
+        const TEST_CASE: &'static str = "Hello, これはヘリックスエディターです！";
 
         let empty = Rope::from("");
-        let mut a = ChangeSet::new(&empty);
+        let a = ChangeSet::new(&empty);
 
         let mut b = ChangeSet::new(&empty);
         b.insert(TEST_CASE.into());

helix-core/src/words.rs

@@ -1,65 +0,0 @@
-use crate::movement::{categorize, is_horiz_blank, is_word, skip_over_prev};
-use ropey::RopeSlice;
-
-#[must_use]
-pub fn nth_prev_word_boundary(slice: RopeSlice, mut char_idx: usize, count: usize) -> usize {
-    let mut with_end = false;
-
-    for _ in 0..count {
-        if char_idx == 0 {
-            break;
-        }
-
-        // return if not skip while?
-        skip_over_prev(slice, &mut char_idx, |ch| ch == '\n');
-
-        with_end = skip_over_prev(slice, &mut char_idx, is_horiz_blank);
-
-        // refetch
-        let ch = slice.char(char_idx);
-
-        if is_word(ch) {
-            with_end = skip_over_prev(slice, &mut char_idx, is_word);
-        } else if ch.is_ascii_punctuation() {
-            with_end = skip_over_prev(slice, &mut char_idx, |ch| ch.is_ascii_punctuation());
-        }
-    }
-
-    if with_end {
-        char_idx
-    } else {
-        char_idx + 1
-    }
-}
-
-#[test]
-fn different_prev_word_boundary() {
-    use ropey::Rope;
-    let t = |x, y| {
-        let text = Rope::from(x);
-        let out = nth_prev_word_boundary(text.slice(..), text.len_chars() - 1, 1);
-        assert_eq!(text.slice(..out), y, r#"from "{}""#, x);
-    };
-    t("abcd\nefg\nwrs", "abcd\nefg\n");
-    t("abcd\nefg\n", "abcd\n");
-    t("abcd\n", "");
-    t("hello, world!", "hello, world");
-    t("hello, world", "hello, ");
-    t("hello, ", "hello");
-    t("hello", "");
-    t("こんにちは、世界！", "こんにちは、世界！"); // TODO: punctuation
-    t("こんにちは、世界", "こんにちは、");
-    t("こんにちは、", "こんにちは、"); // what?
-    t("こんにちは", "");
-    t("この世界。", "この世界。"); // what?
-    t("この世界", "");
-    t("お前はもう死んでいる", "");
-    t("その300円です", ""); // TODO: should stop at 300
-    t("唱k", ""); // TODO: should stop at 唱
-    t("1 + 1 = 2", "1 + 1 = ");
-    t("1 + 1 =", "1 + 1 ");
-    t("1 + 1", "1 + ");
-    t("1 + ", "1 ");
-    t("1 ", "");
-    t("1+1=2", "1+1=");
-}

helix-lsp/Cargo.toml

@@ -1,29 +1,27 @@
 [package]
 name = "helix-lsp"
-version = "0.0.10"
+version = "0.6.0"
 authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
-edition = "2018"
+edition = "2021"
 license = "MPL-2.0"
+description = "LSP client implementation for Helix project"
+categories = ["editor"]
+repository = "https://github.com/helix-editor/helix"
+homepage = "https://helix-editor.com"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-helix-core = { path = "../helix-core" }
+helix-core = { version = "0.6", path = "../helix-core" }
 
-once_cell = "1.4"
+anyhow = "1.0"
-
+futures-executor = "0.3"
-lsp-types = { version = "0.89", features = ["proposed"] }
-tokio = { version = "1", features = ["full"] }
-tokio-stream = "0.1.5"
-futures-executor = { version =  "0.3" }
-url = "2"
-pathdiff = "0.2"
-glob = "0.3"
-anyhow = "1"
-serde_json = "1.0"
-serde = { version = "1.0", features = ["derive"] }
-# jsonrpc-core = { version = "17.1", default-features = false } # don't pull in all of futures
-jsonrpc-core = { git = "https://github.com/paritytech/jsonrpc", default-features = false } # don't pull in all of futures
 futures-util = { version = "0.3", features = ["std", "async-await"], default-features = false }
+jsonrpc-core = { version = "18.0", default-features = false } # don't pull in all of futures
+log = "0.4"
+lsp-types = { version = "0.91", features = ["proposed"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
 thiserror = "1.0"
-log = "~0.4"
+tokio = { version = "1.15", features = ["rt", "rt-multi-thread", "io-util", "io-std", "time", "process", "macros", "fs", "parking_lot"] }
+tokio-stream = "0.1.8"

helix-lsp/src/client.rs

@@ -3,37 +3,46 @@ use crate::{
     Call, Error, OffsetEncoding, Result,
 };
 
-use helix_core::{ChangeSet, Rope};
+use helix_core::{find_root, ChangeSet, Rope};
-
-// use std::collections::HashMap;
-use std::future::Future;
-use std::sync::atomic::{AtomicU64, Ordering};
-
 use jsonrpc_core as jsonrpc;
 use lsp_types as lsp;
 use serde_json::Value;
-
+use std::future::Future;
 use std::process::Stdio;
+use std::sync::{
+    atomic::{AtomicU64, Ordering},
+    Arc,
+};
 use tokio::{
     io::{BufReader, BufWriter},
-    // prelude::*,
     process::{Child, Command},
-    sync::mpsc::{channel, UnboundedReceiver, UnboundedSender},
+    sync::{
+        mpsc::{channel, UnboundedReceiver, UnboundedSender},
+        Notify, OnceCell,
+    },
 };
 
+#[derive(Debug)]
 pub struct Client {
+    id: usize,
     _process: Child,
-
+    server_tx: UnboundedSender<Payload>,
-    outgoing: UnboundedSender<Payload>,
+    request_counter: AtomicU64,
-    // pub incoming: Receiver<Call>,
+    pub(crate) capabilities: OnceCell<lsp::ServerCapabilities>,
-    pub request_counter: AtomicU64,
-
-    capabilities: Option<lsp::ServerCapabilities>,
     offset_encoding: OffsetEncoding,
+    config: Option<Value>,
+    root_markers: Vec<String>,
 }
 
 impl Client {
-    pub fn start(cmd: &str, args: &[String]) -> Result<(Self, UnboundedReceiver<Call>)> {
+    #[allow(clippy::type_complexity)]
+    pub fn start(
+        cmd: &str,
+        args: &[String],
+        config: Option<Value>,
+        root_markers: Vec<String>,
+        id: usize,
+    ) -> Result<(Self, UnboundedReceiver<(usize, Call)>, Arc<Notify>)> {
         let process = Command::new(cmd)
             .args(args)
             .stdin(Stdio::piped())
@@ -43,40 +52,32 @@ impl Client {
             .kill_on_drop(true)
             .spawn();
 
-        // use std::io::ErrorKind;
+        let mut process = process?;
-        let mut process = match process {
-            Ok(process) => process,
-            Err(err) => match err.kind() {
-                // ErrorKind::NotFound | ErrorKind::PermissionDenied => {
-                //     return Err(Error::Other(err.into()))
-                // }
-                _kind => return Err(Error::Other(err.into())),
-            },
-        };
 
         // TODO: do we need bufreader/writer here? or do we use async wrappers on unblock?
         let writer = BufWriter::new(process.stdin.take().expect("Failed to open stdin"));
         let reader = BufReader::new(process.stdout.take().expect("Failed to open stdout"));
         let stderr = BufReader::new(process.stderr.take().expect("Failed to open stderr"));
 
-        let (incoming, outgoing) = Transport::start(reader, writer, stderr);
+        let (server_rx, server_tx, initialize_notify) =
+            Transport::start(reader, writer, stderr, id);
 
         let client = Self {
+            id,
             _process: process,
-
+            server_tx,
-            outgoing,
-            // incoming,
             request_counter: AtomicU64::new(0),
-
+            capabilities: OnceCell::new(),
-            capabilities: None,
-            // diagnostics: HashMap::new(),
             offset_encoding: OffsetEncoding::Utf8,
+            config,
+            root_markers,
         };
 
-        // TODO: async client.initialize()
+        Ok((client, server_rx, initialize_notify))
-        // maybe use an arc<atomic> flag
+    }
 
-        Ok((client, incoming))
+    pub fn id(&self) -> usize {
+        self.id
     }
 
     fn next_request_id(&self) -> jsonrpc::Id {
@@ -95,9 +96,13 @@ impl Client {
         }
     }
 
+    pub fn is_initialized(&self) -> bool {
+        self.capabilities.get().is_some()
+    }
+
     pub fn capabilities(&self) -> &lsp::ServerCapabilities {
         self.capabilities
-            .as_ref()
+            .get()
             .expect("language server not yet initialized!")
     }
 
@@ -106,7 +111,7 @@ impl Client {
     }
 
     /// Execute a RPC request on the language server.
-    pub async fn request<R: lsp::request::Request>(&self, params: R::Params) -> Result<R::Result>
+    async fn request<R: lsp::request::Request>(&self, params: R::Params) -> Result<R::Result>
     where
         R::Params: serde::Serialize,
         R::Result: core::fmt::Debug, // TODO: temporary
@@ -118,17 +123,20 @@ impl Client {
     }
 
     /// Execute a RPC request on the language server.
-    pub fn call<R: lsp::request::Request>(
+    fn call<R: lsp::request::Request>(
         &self,
         params: R::Params,
     ) -> impl Future<Output = Result<Value>>
     where
         R::Params: serde::Serialize,
     {
-        let outgoing = self.outgoing.clone();
+        let server_tx = self.server_tx.clone();
         let id = self.next_request_id();
 
         async move {
+            use std::time::Duration;
+            use tokio::time::timeout;
+
             let params = serde_json::to_value(params)?;
 
             let request = jsonrpc::MethodCall {
@@ -140,20 +148,18 @@ impl Client {
 
             let (tx, mut rx) = channel::<Result<Value>>(1);
 
-            outgoing
+            server_tx
                 .send(Payload::Request {
                     chan: tx,
                     value: request,
                 })
                 .map_err(|e| Error::Other(e.into()))?;
 
-            use std::time::Duration;
+            // TODO: specifiable timeout, delay other calls until initialize success
-            use tokio::time::timeout;
+            timeout(Duration::from_secs(20), rx.recv())
-
-            timeout(Duration::from_secs(2), rx.recv())
                 .await
                 .map_err(|_| Error::Timeout)? // return Timeout
-                .unwrap() // TODO: None if channel closed
+                .ok_or(Error::StreamClosed)?
         }
     }
 
@@ -165,7 +171,7 @@ impl Client {
     where
         R::Params: serde::Serialize,
     {
-        let outgoing = self.outgoing.clone();
+        let server_tx = self.server_tx.clone();
 
         async move {
             let params = serde_json::to_value(params)?;
@@ -176,7 +182,7 @@ impl Client {
                 params: Self::value_into_params(params),
             };
 
-            outgoing
+            server_tx
                 .send(Payload::Notification(notification))
                 .map_err(|e| Error::Other(e.into()))?;
 
@@ -185,47 +191,57 @@ impl Client {
     }
 
     /// Reply to a language server RPC call.
-    pub async fn reply(
+    pub fn reply(
         &self,
         id: jsonrpc::Id,
         result: core::result::Result<Value, jsonrpc::Error>,
-    ) -> Result<()> {
+    ) -> impl Future<Output = Result<()>> {
         use jsonrpc::{Failure, Output, Success, Version};
 
-        let output = match result {
+        let server_tx = self.server_tx.clone();
-            Ok(result) => Output::Success(Success {
-                jsonrpc: Some(Version::V2),
-                id,
-                result,
-            }),
-            Err(error) => Output::Failure(Failure {
-                jsonrpc: Some(Version::V2),
-                id,
-                error,
-            }),
-        };
 
-        self.outgoing
+        async move {
-            .send(Payload::Response(output))
+            let output = match result {
-            .map_err(|e| Error::Other(e.into()))?;
+                Ok(result) => Output::Success(Success {
+                    jsonrpc: Some(Version::V2),
+                    id,
+                    result: serde_json::to_value(result)?,
+                }),
+                Err(error) => Output::Failure(Failure {
+                    jsonrpc: Some(Version::V2),
+                    id,
+                    error,
+                }),
+            };
 
-        Ok(())
+            server_tx
+                .send(Payload::Response(output))
+                .map_err(|e| Error::Other(e.into()))?;
+
+            Ok(())
+        }
     }
 
     // -------------------------------------------------------------------------------------------
     // General messages
     // -------------------------------------------------------------------------------------------
 
-    pub async fn initialize(&mut self) -> Result<()> {
+    pub(crate) async fn initialize(&self) -> Result<lsp::InitializeResult> {
         // TODO: delay any requests that are triggered prior to initialize
+        let root = find_root(None, &self.root_markers)
+            .and_then(|root| lsp::Url::from_file_path(root).ok());
+
+        if self.config.is_some() {
+            log::info!("Using custom LSP config: {}", self.config.as_ref().unwrap());
+        }
 
         #[allow(deprecated)]
         let params = lsp::InitializeParams {
             process_id: Some(std::process::id()),
+            // root_path is obsolete, use root_uri
             root_path: None,
-            // root_uri: Some(lsp_types::Url::parse("file://localhost/")?),
+            root_uri: root,
-            root_uri: None, // set to project root in the future
+            initialization_options: self.config.clone(),
-            initialization_options: None,
             capabilities: lsp::ClientCapabilities {
                 text_document: Some(lsp::TextDocumentClientCapabilities {
                     completion: Some(lsp::CompletionClientCapabilities {
@@ -245,6 +261,36 @@ impl Client {
                         content_format: Some(vec![lsp::MarkupKind::Markdown]),
                         ..Default::default()
                     }),
+                    rename: Some(lsp::RenameClientCapabilities {
+                        dynamic_registration: Some(false),
+                        prepare_support: Some(false),
+                        prepare_support_default_behavior: None,
+                        honors_change_annotations: Some(false),
+                    }),
+                    code_action: Some(lsp::CodeActionClientCapabilities {
+                        code_action_literal_support: Some(lsp::CodeActionLiteralSupport {
+                            code_action_kind: lsp::CodeActionKindLiteralSupport {
+                                value_set: [
+                                    lsp::CodeActionKind::EMPTY,
+                                    lsp::CodeActionKind::QUICKFIX,
+                                    lsp::CodeActionKind::REFACTOR,
+                                    lsp::CodeActionKind::REFACTOR_EXTRACT,
+                                    lsp::CodeActionKind::REFACTOR_INLINE,
+                                    lsp::CodeActionKind::REFACTOR_REWRITE,
+                                    lsp::CodeActionKind::SOURCE,
+                                    lsp::CodeActionKind::SOURCE_ORGANIZE_IMPORTS,
+                                ]
+                                .iter()
+                                .map(|kind| kind.as_str().to_string())
+                                .collect(),
+                            },
+                        }),
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                }),
+                window: Some(lsp::WindowClientCapabilities {
+                    work_done_progress: Some(true),
                     ..Default::default()
                 }),
                 ..Default::default()
@@ -255,14 +301,7 @@ impl Client {
             locale: None, // TODO
         };
 
-        let response = self.request::<lsp::request::Initialize>(params).await?;
+        self.request::<lsp::request::Initialize>(params).await
-        self.capabilities = Some(response.capabilities);
-
-        // next up, notify<initialized>
-        self.notify::<lsp::notification::Initialized>(lsp::InitializedParams {})
-            .await?;
-
-        Ok(())
     }
 
     pub async fn shutdown(&self) -> Result<()> {
@@ -273,6 +312,21 @@ impl Client {
         self.notify::<lsp::notification::Exit>(())
     }
 
+    /// Tries to shut down the language server but returns
+    /// early if server responds with an error.
+    pub async fn shutdown_and_exit(&self) -> Result<()> {
+        self.shutdown().await?;
+        self.exit().await
+    }
+
+    /// Forcefully shuts down the language server ignoring any errors.
+    pub async fn force_shutdown(&self) -> Result<()> {
+        if let Err(e) = self.shutdown().await {
+            log::warn!("language server failed to terminate gracefully - {}", e);
+        }
+        self.exit().await
+    }
+
     // -------------------------------------------------------------------------------------------
     // Text document
     // -------------------------------------------------------------------------------------------
@@ -315,7 +369,6 @@ impl Client {
         //
         // Calculation is therefore a bunch trickier.
 
-        // TODO: stolen from syntax.rs, share
         use helix_core::RopeSlice;
         fn traverse(pos: lsp::Position, text: RopeSlice) -> lsp::Position {
             let lsp::Position {
@@ -323,8 +376,14 @@ impl Client {
                 mut character,
             } = pos;
 
-            for ch in text.chars() {
+            let mut chars = text.chars().peekable();
-                if ch == '\n' {
+            while let Some(ch) = chars.next() {
+                // LSP only considers \n, \r or \r\n as line endings
+                if ch == '\n' || ch == '\r' {
+                    // consume a \r\n
+                    if ch == '\r' && chars.peek() == Some(&'\n') {
+                        chars.next();
+                    }
                     line += 1;
                     character = 0;
                 } else {
@@ -399,7 +458,7 @@ impl Client {
     ) -> Option<impl Future<Output = Result<()>>> {
         // figure out what kind of sync the server supports
 
-        let capabilities = self.capabilities.as_ref().unwrap();
+        let capabilities = self.capabilities.get().unwrap();
 
         let sync_capabilities = match capabilities.text_document_sync {
             Some(lsp::TextDocumentSyncCapability::Kind(kind))
@@ -412,18 +471,19 @@ impl Client {
         };
 
         let changes = match sync_capabilities {
-            lsp::TextDocumentSyncKind::Full => {
+            lsp::TextDocumentSyncKind::FULL => {
                 vec![lsp::TextDocumentContentChangeEvent {
                     // range = None -> whole document
                     range: None,        //Some(Range)
                     range_length: None, // u64 apparently deprecated
-                    text: "".to_string(),
+                    text: new_text.to_string(),
                 }]
             }
-            lsp::TextDocumentSyncKind::Incremental => {
+            lsp::TextDocumentSyncKind::INCREMENTAL => {
                 Self::changeset_to_changes(old_text, new_text, changes, self.offset_encoding)
             }
-            lsp::TextDocumentSyncKind::None => return None,
+            lsp::TextDocumentSyncKind::NONE => return None,
+            kind => unimplemented!("{:?}", kind),
         };
 
         Some(self.notify::<lsp::notification::DidChangeTextDocument>(
@@ -445,12 +505,12 @@ impl Client {
 
     // will_save / will_save_wait_until
 
-    pub async fn text_document_did_save(
+    pub fn text_document_did_save(
         &self,
         text_document: lsp::TextDocumentIdentifier,
         text: &Rope,
-    ) -> Result<()> {
+    ) -> Option<impl Future<Output = Result<()>>> {
-        let capabilities = self.capabilities.as_ref().unwrap();
+        let capabilities = self.capabilities.get().unwrap();
 
         let include_text = match &capabilities.text_document_sync {
             Some(lsp::TextDocumentSyncCapability::Options(lsp::TextDocumentSyncOptions {
@@ -462,23 +522,25 @@ impl Client {
                     include_text,
                 }) => include_text.unwrap_or(false),
                 // Supported(false)
-                _ => return Ok(()),
+                _ => return None,
             },
             // unsupported
-            _ => return Ok(()),
+            _ => return None,
         };
 
-        self.notify::<lsp::notification::DidSaveTextDocument>(lsp::DidSaveTextDocumentParams {
+        Some(self.notify::<lsp::notification::DidSaveTextDocument>(
-            text_document,
+            lsp::DidSaveTextDocumentParams {
-            text: include_text.then(|| text.into()),
+                text_document,
-        })
+                text: include_text.then(|| text.into()),
-        .await
+            },
+        ))
     }
 
     pub fn completion(
         &self,
         text_document: lsp::TextDocumentIdentifier,
         position: lsp::Position,
+        work_done_token: Option<lsp::ProgressToken>,
     ) -> impl Future<Output = Result<Value>> {
         // ) -> Result<Vec<lsp::CompletionItem>> {
         let params = lsp::CompletionParams {
@@ -487,9 +549,7 @@ impl Client {
                 position,
             },
             // TODO: support these tokens by async receiving and updating the choice list
-            work_done_progress_params: lsp::WorkDoneProgressParams {
+            work_done_progress_params: lsp::WorkDoneProgressParams { work_done_token },
-                work_done_token: None,
-            },
             partial_result_params: lsp::PartialResultParams {
                 partial_result_token: None,
             },
@@ -500,19 +560,26 @@ impl Client {
         self.call::<lsp::request::Completion>(params)
     }
 
+    pub async fn resolve_completion_item(
+        &self,
+        completion_item: lsp::CompletionItem,
+    ) -> Result<lsp::CompletionItem> {
+        self.request::<lsp::request::ResolveCompletionItem>(completion_item)
+            .await
+    }
+
     pub fn text_document_signature_help(
         &self,
         text_document: lsp::TextDocumentIdentifier,
         position: lsp::Position,
+        work_done_token: Option<lsp::ProgressToken>,
     ) -> impl Future<Output = Result<Value>> {
         let params = lsp::SignatureHelpParams {
             text_document_position_params: lsp::TextDocumentPositionParams {
                 text_document,
                 position,
             },
-            work_done_progress_params: lsp::WorkDoneProgressParams {
+            work_done_progress_params: lsp::WorkDoneProgressParams { work_done_token },
-                work_done_token: None,
-            },
             context: None,
             // lsp::SignatureHelpContext
         };
@@ -524,15 +591,14 @@ impl Client {
         &self,
         text_document: lsp::TextDocumentIdentifier,
         position: lsp::Position,
+        work_done_token: Option<lsp::ProgressToken>,
     ) -> impl Future<Output = Result<Value>> {
         let params = lsp::HoverParams {
             text_document_position_params: lsp::TextDocumentPositionParams {
                 text_document,
                 position,
             },
-            work_done_progress_params: lsp::WorkDoneProgressParams {
+            work_done_progress_params: lsp::WorkDoneProgressParams { work_done_token },
-                work_done_token: None,
-            },
             // lsp::SignatureHelpContext
         };
 
@@ -541,32 +607,35 @@ impl Client {
 
     // formatting
 
-    pub async fn text_document_formatting(
+    pub fn text_document_formatting(
         &self,
         text_document: lsp::TextDocumentIdentifier,
         options: lsp::FormattingOptions,
-    ) -> anyhow::Result<Vec<lsp::TextEdit>> {
+        work_done_token: Option<lsp::ProgressToken>,
-        let capabilities = self.capabilities.as_ref().unwrap();
+    ) -> Option<impl Future<Output = Result<Vec<lsp::TextEdit>>>> {
+        let capabilities = self.capabilities.get().unwrap();
 
         // check if we're able to format
         match capabilities.document_formatting_provider {
             Some(lsp::OneOf::Left(true)) | Some(lsp::OneOf::Right(_)) => (),
             // None | Some(false)
-            _ => return Ok(Vec::new()),
+            _ => return None,
         };
         // TODO: return err::unavailable so we can fall back to tree sitter formatting
 
         let params = lsp::DocumentFormattingParams {
             text_document,
             options,
-            work_done_progress_params: lsp::WorkDoneProgressParams {
+            work_done_progress_params: lsp::WorkDoneProgressParams { work_done_token },
-                work_done_token: None,
-            },
         };
 
-        let response = self.request::<lsp::request::Formatting>(params).await?;
+        let request = self.call::<lsp::request::Formatting>(params);
 
-        Ok(response.unwrap_or_default())
+        Some(async move {
+            let json = request.await?;
+            let response: Option<Vec<lsp::TextEdit>> = serde_json::from_value(json)?;
+            Ok(response.unwrap_or_default())
+        })
     }
 
     pub async fn text_document_range_formatting(
@@ -574,8 +643,9 @@ impl Client {
         text_document: lsp::TextDocumentIdentifier,
         range: lsp::Range,
         options: lsp::FormattingOptions,
+        work_done_token: Option<lsp::ProgressToken>,
     ) -> anyhow::Result<Vec<lsp::TextEdit>> {
-        let capabilities = self.capabilities.as_ref().unwrap();
+        let capabilities = self.capabilities.get().unwrap();
 
         // check if we're able to format
         match capabilities.document_range_formatting_provider {
@@ -589,9 +659,7 @@ impl Client {
             text_document,
             range,
             options,
-            work_done_progress_params: lsp::WorkDoneProgressParams {
+            work_done_progress_params: lsp::WorkDoneProgressParams { work_done_token },
-                work_done_token: None,
-            },
         };
 
         let response = self
@@ -610,15 +678,14 @@ impl Client {
         &self,
         text_document: lsp::TextDocumentIdentifier,
         position: lsp::Position,
+        work_done_token: Option<lsp::ProgressToken>,
     ) -> impl Future<Output = Result<Value>> {
         let params = lsp::GotoDefinitionParams {
             text_document_position_params: lsp::TextDocumentPositionParams {
                 text_document,
                 position,
             },
-            work_done_progress_params: lsp::WorkDoneProgressParams {
+            work_done_progress_params: lsp::WorkDoneProgressParams { work_done_token },
-                work_done_token: None,
-            },
             partial_result_params: lsp::PartialResultParams {
                 partial_result_token: None,
             },
@@ -631,30 +698,42 @@ impl Client {
         &self,
         text_document: lsp::TextDocumentIdentifier,
         position: lsp::Position,
+        work_done_token: Option<lsp::ProgressToken>,
     ) -> impl Future<Output = Result<Value>> {
-        self.goto_request::<lsp::request::GotoDefinition>(text_document, position)
+        self.goto_request::<lsp::request::GotoDefinition>(text_document, position, work_done_token)
     }
 
     pub fn goto_type_definition(
         &self,
         text_document: lsp::TextDocumentIdentifier,
         position: lsp::Position,
+        work_done_token: Option<lsp::ProgressToken>,
     ) -> impl Future<Output = Result<Value>> {
-        self.goto_request::<lsp::request::GotoTypeDefinition>(text_document, position)
+        self.goto_request::<lsp::request::GotoTypeDefinition>(
+            text_document,
+            position,
+            work_done_token,
+        )
     }
 
     pub fn goto_implementation(
         &self,
         text_document: lsp::TextDocumentIdentifier,
         position: lsp::Position,
+        work_done_token: Option<lsp::ProgressToken>,
     ) -> impl Future<Output = Result<Value>> {
-        self.goto_request::<lsp::request::GotoImplementation>(text_document, position)
+        self.goto_request::<lsp::request::GotoImplementation>(
+            text_document,
+            position,
+            work_done_token,
+        )
     }
 
     pub fn goto_reference(
         &self,
         text_document: lsp::TextDocumentIdentifier,
         position: lsp::Position,
+        work_done_token: Option<lsp::ProgressToken>,
     ) -> impl Future<Output = Result<Value>> {
         let params = lsp::ReferenceParams {
             text_document_position: lsp::TextDocumentPositionParams {
@@ -664,9 +743,7 @@ impl Client {
             context: lsp::ReferenceContext {
                 include_declaration: true,
             },
-            work_done_progress_params: lsp::WorkDoneProgressParams {
+            work_done_progress_params: lsp::WorkDoneProgressParams { work_done_token },
-                work_done_token: None,
-            },
             partial_result_params: lsp::PartialResultParams {
                 partial_result_token: None,
             },
@@ -674,4 +751,77 @@ impl Client {
 
         self.call::<lsp::request::References>(params)
     }
+
+    pub fn document_symbols(
+        &self,
+        text_document: lsp::TextDocumentIdentifier,
+    ) -> impl Future<Output = Result<Value>> {
+        let params = lsp::DocumentSymbolParams {
+            text_document,
+            work_done_progress_params: lsp::WorkDoneProgressParams::default(),
+            partial_result_params: lsp::PartialResultParams::default(),
+        };
+
+        self.call::<lsp::request::DocumentSymbolRequest>(params)
+    }
+
+    // empty string to get all symbols
+    pub fn workspace_symbols(&self, query: String) -> impl Future<Output = Result<Value>> {
+        let params = lsp::WorkspaceSymbolParams {
+            query,
+            work_done_progress_params: lsp::WorkDoneProgressParams::default(),
+            partial_result_params: lsp::PartialResultParams::default(),
+        };
+
+        self.call::<lsp::request::WorkspaceSymbol>(params)
+    }
+
+    pub fn code_actions(
+        &self,
+        text_document: lsp::TextDocumentIdentifier,
+        range: lsp::Range,
+    ) -> impl Future<Output = Result<Value>> {
+        let params = lsp::CodeActionParams {
+            text_document,
+            range,
+            context: lsp::CodeActionContext::default(),
+            work_done_progress_params: lsp::WorkDoneProgressParams::default(),
+            partial_result_params: lsp::PartialResultParams::default(),
+        };
+
+        self.call::<lsp::request::CodeActionRequest>(params)
+    }
+
+    pub async fn rename_symbol(
+        &self,
+        text_document: lsp::TextDocumentIdentifier,
+        position: lsp::Position,
+        new_name: String,
+    ) -> anyhow::Result<lsp::WorkspaceEdit> {
+        let params = lsp::RenameParams {
+            text_document_position: lsp::TextDocumentPositionParams {
+                text_document,
+                position,
+            },
+            new_name,
+            work_done_progress_params: lsp::WorkDoneProgressParams {
+                work_done_token: None,
+            },
+        };
+
+        let response = self.request::<lsp::request::Rename>(params).await?;
+        Ok(response.unwrap_or_default())
+    }
+
+    pub fn command(&self, command: lsp::Command) -> impl Future<Output = Result<Value>> {
+        let params = lsp::ExecuteCommandParams {
+            command: command.command,
+            arguments: command.arguments.unwrap_or_default(),
+            work_done_progress_params: lsp::WorkDoneProgressParams {
+                work_done_token: None,
+            },
+        };
+
+        self.call::<lsp::request::ExecuteCommand>(params)
+    }
 }

helix-lsp/src/lib.rs

@@ -1,25 +1,30 @@
 mod client;
 mod transport;
 
+pub use client::Client;
+pub use futures_executor::block_on;
+pub use jsonrpc::Call;
 pub use jsonrpc_core as jsonrpc;
+pub use lsp::{Position, Url};
 pub use lsp_types as lsp;
 
-pub use client::Client;
+use futures_util::stream::select_all::SelectAll;
-pub use lsp::{Position, Url};
-
-pub type Result<T> = core::result::Result<T, Error>;
-
 use helix_core::syntax::LanguageConfiguration;
 
-use thiserror::Error;
+use std::{
-
+    collections::{hash_map::Entry, HashMap},
-use std::{collections::HashMap, sync::Arc};
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc,
+    },
+};
 
 use serde::{Deserialize, Serialize};
-
+use thiserror::Error;
 use tokio_stream::wrappers::UnboundedReceiverStream;
 
-pub use futures_executor::block_on;
+pub type Result<T> = core::result::Result<T, Error>;
+type LanguageId = String;
 
 #[derive(Error, Debug)]
 pub enum Error {
@@ -27,8 +32,14 @@ pub enum Error {
     Rpc(#[from] jsonrpc::Error),
     #[error("failed to parse: {0}")]
     Parse(#[from] serde_json::Error),
+    #[error("IO Error: {0}")]
+    IO(#[from] std::io::Error),
     #[error("request timed out")]
     Timeout,
+    #[error("server closed the stream")]
+    StreamClosed,
+    #[error("LSP not defined")]
+    LspNotDefined,
     #[error(transparent)]
     Other(#[from] anyhow::Error),
 }
@@ -47,23 +58,41 @@ pub mod util {
     use super::*;
     use helix_core::{Range, Rope, Transaction};
 
+    /// Converts [`lsp::Position`] to a position in the document.
+    ///
+    /// Returns `None` if position exceeds document length or an operation overflows.
     pub fn lsp_pos_to_pos(
         doc: &Rope,
         pos: lsp::Position,
         offset_encoding: OffsetEncoding,
-    ) -> usize {
+    ) -> Option<usize> {
+        let pos_line = pos.line as usize;
+        if pos_line > doc.len_lines() - 1 {
+            return None;
+        }
+
         match offset_encoding {
             OffsetEncoding::Utf8 => {
-                let line = doc.line_to_char(pos.line as usize);
+                let line = doc.line_to_char(pos_line);
-                line + pos.character as usize
+                let pos = line.checked_add(pos.character as usize)?;
+                if pos <= doc.len_chars() {
+                    Some(pos)
+                } else {
+                    None
+                }
             }
             OffsetEncoding::Utf16 => {
-                let line = doc.line_to_char(pos.line as usize);
+                let line = doc.line_to_char(pos_line);
                 let line_start = doc.char_to_utf16_cu(line);
-                doc.utf16_cu_to_char(line_start + pos.character as usize)
+                let pos = line_start.checked_add(pos.character as usize)?;
+                doc.try_utf16_cu_to_char(pos).ok()
             }
         }
     }
+
+    /// Converts position in the document to [`lsp::Position`].
+    ///
+    /// Panics when `pos` is out of `doc` bounds or operation overflows.
     pub fn pos_to_lsp_pos(
         doc: &Rope,
         pos: usize,
@@ -87,6 +116,7 @@ pub mod util {
         }
     }
 
+    /// Converts a range in the document to [`lsp::Range`].
     pub fn range_to_lsp_range(
         doc: &Rope,
         range: Range,
@@ -98,6 +128,17 @@ pub mod util {
         lsp::Range::new(start, end)
     }
 
+    pub fn lsp_range_to_range(
+        doc: &Rope,
+        range: lsp::Range,
+        offset_encoding: OffsetEncoding,
+    ) -> Option<Range> {
+        let start = lsp_pos_to_pos(doc, range.start, offset_encoding)?;
+        let end = lsp_pos_to_pos(doc, range.end, offset_encoding)?;
+
+        Some(Range::new(start, end))
+    }
+
     pub fn generate_transaction_from_edits(
         doc: &Rope,
         edits: Vec<lsp::TextEdit>,
@@ -113,21 +154,78 @@ pub mod util {
                     None
                 };
 
-                let start = lsp_pos_to_pos(doc, edit.range.start, offset_encoding);
+                let start =
-                let end = lsp_pos_to_pos(doc, edit.range.end, offset_encoding);
+                    if let Some(start) = lsp_pos_to_pos(doc, edit.range.start, offset_encoding) {
+                        start
+                    } else {
+                        return (0, 0, None);
+                    };
+                let end = if let Some(end) = lsp_pos_to_pos(doc, edit.range.end, offset_encoding) {
+                    end
+                } else {
+                    return (0, 0, None);
+                };
                 (start, end, replacement)
             }),
         )
     }
 
-    // apply_insert_replace_edit
+    /// The result of asking the language server to format the document. This can be turned into a
+    /// `Transaction`, but the advantage of not doing that straight away is that this one is
+    /// `Send` and `Sync`.
+    #[derive(Clone, Debug)]
+    pub struct LspFormatting {
+        pub doc: Rope,
+        pub edits: Vec<lsp::TextEdit>,
+        pub offset_encoding: OffsetEncoding,
+    }
+
+    impl From<LspFormatting> for Transaction {
+        fn from(fmt: LspFormatting) -> Transaction {
+            generate_transaction_from_edits(&fmt.doc, fmt.edits, fmt.offset_encoding)
+        }
+    }
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub enum MethodCall {
+    WorkDoneProgressCreate(lsp::WorkDoneProgressCreateParams),
+    ApplyWorkspaceEdit(lsp::ApplyWorkspaceEditParams),
+}
+
+impl MethodCall {
+    pub fn parse(method: &str, params: jsonrpc::Params) -> Option<MethodCall> {
+        use lsp::request::Request;
+        let request = match method {
+            lsp::request::WorkDoneProgressCreate::METHOD => {
+                let params: lsp::WorkDoneProgressCreateParams = params
+                    .parse()
+                    .expect("Failed to parse WorkDoneCreate params");
+                Self::WorkDoneProgressCreate(params)
+            }
+            lsp::request::ApplyWorkspaceEdit::METHOD => {
+                let params: lsp::ApplyWorkspaceEditParams = params
+                    .parse()
+                    .expect("Failed to parse ApplyWorkspaceEdit params");
+                Self::ApplyWorkspaceEdit(params)
+            }
+            _ => {
+                log::warn!("unhandled lsp request: {}", method);
+                return None;
+            }
+        };
+        Some(request)
+    }
 }
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum Notification {
+    // we inject this notification to signal the LSP is ready
+    Initialized,
     PublishDiagnostics(lsp::PublishDiagnosticsParams),
     ShowMessage(lsp::ShowMessageParams),
     LogMessage(lsp::LogMessageParams),
+    ProgressMessage(lsp::ProgressParams),
 }
 
 impl Notification {
@@ -135,6 +233,7 @@ impl Notification {
         use lsp::notification::Notification as _;
 
         let notification = match method {
+            lsp::notification::Initialized::METHOD => Self::Initialized,
             lsp::notification::PublishDiagnostics::METHOD => {
                 let params: lsp::PublishDiagnosticsParams = params
                     .parse()
@@ -145,17 +244,20 @@ impl Notification {
             }
 
             lsp::notification::ShowMessage::METHOD => {
-                let params: lsp::ShowMessageParams =
+                let params: lsp::ShowMessageParams = params.parse().ok()?;
-                    params.parse().expect("Failed to parse ShowMessage params");
 
                 Self::ShowMessage(params)
             }
             lsp::notification::LogMessage::METHOD => {
-                let params: lsp::LogMessageParams =
+                let params: lsp::LogMessageParams = params.parse().ok()?;
-                    params.parse().expect("Failed to parse ShowMessage params");
 
                 Self::LogMessage(params)
             }
+            lsp::notification::Progress::METHOD => {
+                let params: lsp::ProgressParams = params.parse().ok()?;
+
+                Self::ProgressMessage(params)
+            }
             _ => {
                 log::error!("unhandled LSP notification: {}", method);
                 return None;
@@ -166,16 +268,12 @@ impl Notification {
     }
 }
 
-pub use jsonrpc::Call;
+#[derive(Debug)]
-
-type LanguageId = String;
-
-use futures_util::stream::select_all::SelectAll;
-
 pub struct Registry {
-    inner: HashMap<LanguageId, Option<Arc<Client>>>,
+    inner: HashMap<LanguageId, (usize, Arc<Client>)>,
 
-    pub incoming: SelectAll<UnboundedReceiverStream<Call>>,
+    counter: AtomicUsize,
+    pub incoming: SelectAll<UnboundedReceiverStream<(usize, Call)>>,
 }
 
 impl Default for Registry {
@@ -188,64 +286,182 @@ impl Registry {
     pub fn new() -> Self {
         Self {
             inner: HashMap::new(),
+            counter: AtomicUsize::new(0),
             incoming: SelectAll::new(),
         }
     }
 
-    pub fn get(&mut self, language_config: &LanguageConfiguration) -> Option<Arc<Client>> {
+    pub fn get_by_id(&self, id: usize) -> Option<&Client> {
-        // TODO: propagate the error
+        self.inner
-        if let Some(config) = &language_config.language_server {
+            .values()
-            // avoid borrow issues
+            .find(|(client_id, _)| client_id == &id)
-            let inner = &mut self.inner;
+            .map(|(_, client)| client.as_ref())
-            let s_incoming = &mut self.incoming;
+    }
 
-            let language_server = inner
+    pub fn get(&mut self, language_config: &LanguageConfiguration) -> Result<Arc<Client>> {
-                .entry(language_config.scope.clone()) // can't use entry with Borrow keys: https://github.com/rust-lang/rfcs/pull/1769
+        let config = match &language_config.language_server {
-                .or_insert_with(|| {
+            Some(config) => config,
-                    // TODO: lookup defaults for id (name, args)
+            None => return Err(Error::LspNotDefined),
+        };
 
-                    // initialize a new client
+        match self.inner.entry(language_config.scope.clone()) {
-                    let (mut client, incoming) =
+            Entry::Occupied(entry) => Ok(entry.get().1.clone()),
-                        Client::start(&config.command, &config.args).ok()?;
+            Entry::Vacant(entry) => {
+                // initialize a new client
+                let id = self.counter.fetch_add(1, Ordering::Relaxed);
+                let (client, incoming, initialize_notify) = Client::start(
+                    &config.command,
+                    &config.args,
+                    language_config.config.clone(),
+                    language_config.roots.clone(),
+                    id,
+                )?;
+                self.incoming.push(UnboundedReceiverStream::new(incoming));
+                let client = Arc::new(client);
 
-                    // TODO: run this async without blocking
+                // Initialize the client asynchronously
-                    futures_executor::block_on(client.initialize()).unwrap();
+                let _client = client.clone();
+                tokio::spawn(async move {
+                    use futures_util::TryFutureExt;
+                    let value = _client
+                        .capabilities
+                        .get_or_try_init(|| {
+                            _client
+                                .initialize()
+                                .map_ok(|response| response.capabilities)
+                        })
+                        .await;
 
-                    s_incoming.push(UnboundedReceiverStream::new(incoming));
+                    if let Err(e) = value {
+                        log::error!("failed to initialize language server: {}", e);
+                        return;
+                    }
 
-                    Some(Arc::new(client))
+                    // next up, notify<initialized>
-                })
+                    _client
-                .clone();
+                        .notify::<lsp::notification::Initialized>(lsp::InitializedParams {})
+                        .await
+                        .unwrap();
 
-            return language_server;
+                    initialize_notify.notify_one();
+                });
+
+                entry.insert((id, client.clone()));
+                Ok(client)
+            }
         }
+    }
 
-        None
+    pub fn iter_clients(&self) -> impl Iterator<Item = &Arc<Client>> {
+        self.inner.values().map(|(_, client)| client)
     }
 }
 
-// REGISTRY = HashMap<LanguageId, Lazy/OnceCell<Arc<RwLock<Client>>>
+#[derive(Debug)]
-// spawn one server per language type, need to spawn one per workspace if server doesn't support
+pub enum ProgressStatus {
-// workspaces
+    Created,
-//
+    Started(lsp::WorkDoneProgress),
-// could also be a client per root dir
+}
-//
+
-// storing a copy of Option<Arc<RwLock<Client>>> on Document would make the LSP client easily
+impl ProgressStatus {
-// accessible during edit/save callbacks
+    pub fn progress(&self) -> Option<&lsp::WorkDoneProgress> {
-//
+        match &self {
-// the event loop needs to process all incoming streams, maybe we can just have that be a separate
+            ProgressStatus::Created => None,
-// task that's continually running and store the state on the client, then use read lock to
+            ProgressStatus::Started(progress) => Some(progress),
-// retrieve data during render
+        }
-// -> PROBLEM: how do you trigger an update on the editor side when data updates?
+    }
-//
+}
-// -> The data updates should pull all events until we run out so we don't frequently re-render
+
-//
+#[derive(Default, Debug)]
-//
+/// Acts as a container for progress reported by language servers. Each server
-// v2:
+/// has a unique id assigned at creation through [`Registry`]. This id is then used
-//
+/// to store the progress in this map.
-// there should be a registry of lsp clients, one per language type (or workspace).
+pub struct LspProgressMap(HashMap<usize, HashMap<lsp::ProgressToken, ProgressStatus>>);
-// the clients should lazy init on first access
+
-// the client.initialize() should be called async and we buffer any requests until that completes
+impl LspProgressMap {
-// there needs to be a way to process incoming lsp messages from all clients.
+    pub fn new() -> Self {
-//  -> notifications need to be dispatched to wherever
+        Self::default()
-//  -> requests need to generate a reply and travel back to the same lsp!
+    }
+
+    /// Returns a map of all tokens coresponding to the lanaguage server with `id`.
+    pub fn progress_map(&self, id: usize) -> Option<&HashMap<lsp::ProgressToken, ProgressStatus>> {
+        self.0.get(&id)
+    }
+
+    pub fn is_progressing(&self, id: usize) -> bool {
+        self.0.get(&id).map(|it| !it.is_empty()).unwrap_or_default()
+    }
+
+    /// Returns last progress status for a given server with `id` and `token`.
+    pub fn progress(&self, id: usize, token: &lsp::ProgressToken) -> Option<&ProgressStatus> {
+        self.0.get(&id).and_then(|values| values.get(token))
+    }
+
+    /// Checks if progress `token` for server with `id` is created.
+    pub fn is_created(&mut self, id: usize, token: &lsp::ProgressToken) -> bool {
+        self.0
+            .get(&id)
+            .map(|values| values.get(token).is_some())
+            .unwrap_or_default()
+    }
+
+    pub fn create(&mut self, id: usize, token: lsp::ProgressToken) {
+        self.0
+            .entry(id)
+            .or_default()
+            .insert(token, ProgressStatus::Created);
+    }
+
+    /// Ends the progress by removing the `token` from server with `id`, if removed returns the value.
+    pub fn end_progress(
+        &mut self,
+        id: usize,
+        token: &lsp::ProgressToken,
+    ) -> Option<ProgressStatus> {
+        self.0.get_mut(&id).and_then(|vals| vals.remove(token))
+    }
+
+    /// Updates the progess of `token` for server with `id` to `status`, returns the value replaced or `None`.
+    pub fn update(
+        &mut self,
+        id: usize,
+        token: lsp::ProgressToken,
+        status: lsp::WorkDoneProgress,
+    ) -> Option<ProgressStatus> {
+        self.0
+            .entry(id)
+            .or_default()
+            .insert(token, ProgressStatus::Started(status))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{lsp, util::*, OffsetEncoding};
+    use helix_core::Rope;
+
+    #[test]
+    fn converts_lsp_pos_to_pos() {
+        macro_rules! test_case {
+            ($doc:expr, ($x:expr, $y:expr) => $want:expr) => {
+                let doc = Rope::from($doc);
+                let pos = lsp::Position::new($x, $y);
+                assert_eq!($want, lsp_pos_to_pos(&doc, pos, OffsetEncoding::Utf16));
+                assert_eq!($want, lsp_pos_to_pos(&doc, pos, OffsetEncoding::Utf8))
+            };
+        }
+
+        test_case!("", (0, 0) => Some(0));
+        test_case!("", (0, 1) => None);
+        test_case!("", (1, 0) => None);
+        test_case!("\n\n", (0, 0) => Some(0));
+        test_case!("\n\n", (1, 0) => Some(1));
+        test_case!("\n\n", (1, 1) => Some(2));
+        test_case!("\n\n", (2, 0) => Some(2));
+        test_case!("\n\n", (3, 0) => None);
+        test_case!("test\n\n\n\ncase", (4, 3) => Some(11));
+        test_case!("test\n\n\n\ncase", (4, 4) => Some(12));
+        test_case!("test\n\n\n\ncase", (4, 5) => None);
+        test_case!("", (u32::MAX, u32::MAX) => None);
+    }
+}

helix-lsp/src/transport.rs

@@ -1,19 +1,18 @@
-use std::collections::HashMap;
+use crate::{Error, Result};
-use std::io;
+use anyhow::Context;
-
-use log::{error, info};
-
-use crate::Error;
-
-type Result<T> = core::result::Result<T, Error>;
-
 use jsonrpc_core as jsonrpc;
+use log::{error, info};
+use serde::{Deserialize, Serialize};
 use serde_json::Value;
-
+use std::collections::HashMap;
+use std::sync::Arc;
 use tokio::{
     io::{AsyncBufRead, AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader, BufWriter},
     process::{ChildStderr, ChildStdin, ChildStdout},
-    sync::mpsc::{unbounded_channel, Sender, UnboundedReceiver, UnboundedSender},
+    sync::{
+        mpsc::{unbounded_channel, Sender, UnboundedReceiver, UnboundedSender},
+        Mutex, Notify,
+    },
 };
 
 #[derive(Debug)]
@@ -26,153 +25,185 @@ pub enum Payload {
     Response(jsonrpc::Output),
 }
 
-use serde::{Deserialize, Serialize};
 /// A type representing all possible values sent from the server to the client.
 #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
 #[serde(deny_unknown_fields)]
 #[serde(untagged)]
-enum Message {
+enum ServerMessage {
     /// A regular JSON-RPC request output (single response).
     Output(jsonrpc::Output),
     /// A JSON-RPC request or notification.
     Call(jsonrpc::Call),
 }
 
+#[derive(Debug)]
 pub struct Transport {
-    incoming: UnboundedSender<jsonrpc::Call>,
+    id: usize,
-    outgoing: UnboundedReceiver<Payload>,
+    pending_requests: Mutex<HashMap<jsonrpc::Id, Sender<Result<Value>>>>,
-
-    pending_requests: HashMap<jsonrpc::Id, Sender<Result<Value>>>,
-    headers: HashMap<String, String>,
-
-    writer: BufWriter<ChildStdin>,
-    reader: BufReader<ChildStdout>,
-    stderr: BufReader<ChildStderr>,
 }
 
 impl Transport {
     pub fn start(
-        reader: BufReader<ChildStdout>,
+        server_stdout: BufReader<ChildStdout>,
-        writer: BufWriter<ChildStdin>,
+        server_stdin: BufWriter<ChildStdin>,
-        stderr: BufReader<ChildStderr>,
+        server_stderr: BufReader<ChildStderr>,
-    ) -> (UnboundedReceiver<jsonrpc::Call>, UnboundedSender<Payload>) {
+        id: usize,
-        let (incoming, rx) = unbounded_channel();
+    ) -> (
-        let (tx, outgoing) = unbounded_channel();
+        UnboundedReceiver<(usize, jsonrpc::Call)>,
+        UnboundedSender<Payload>,
+        Arc<Notify>,
+    ) {
+        let (client_tx, rx) = unbounded_channel();
+        let (tx, client_rx) = unbounded_channel();
+        let notify = Arc::new(Notify::new());
 
         let transport = Self {
-            reader,
+            id,
-            writer,
+            pending_requests: Mutex::new(HashMap::default()),
-            stderr,
-            incoming,
-            outgoing,
-            pending_requests: HashMap::default(),
-            headers: HashMap::default(),
         };
 
-        tokio::spawn(transport.duplex());
+        let transport = Arc::new(transport);
 
-        (rx, tx)
+        tokio::spawn(Self::recv(
+            transport.clone(),
+            server_stdout,
+            client_tx.clone(),
+        ));
+        tokio::spawn(Self::err(transport.clone(), server_stderr));
+        tokio::spawn(Self::send(
+            transport,
+            server_stdin,
+            client_tx,
+            client_rx,
+            notify.clone(),
+        ));
+
+        (rx, tx, notify)
     }
 
-    async fn recv(
+    async fn recv_server_message(
         reader: &mut (impl AsyncBufRead + Unpin + Send),
-        headers: &mut HashMap<String, String>,
+        buffer: &mut String,
-    ) -> core::result::Result<Message, std::io::Error> {
+    ) -> Result<ServerMessage> {
-        // read headers
+        let mut content_length = None;
         loop {
-            let mut header = String::new();
+            buffer.truncate(0);
-            // detect pipe closed if 0
+            if reader.read_line(buffer).await? == 0 {
-            reader.read_line(&mut header).await?;
+                return Err(Error::StreamClosed);
-            let header = header.trim();
+            };
 
-            if header.is_empty() {
+            // debug!("<- header {:?}", buffer);
+
+            if buffer == "\r\n" {
+                // look for an empty CRLF line
                 break;
             }
 
-            let parts: Vec<&str> = header.split(": ").collect();
+            let header = buffer.trim();
-            if parts.len() != 2 {
+
-                return Err(std::io::Error::new(
+            let parts = header.split_once(": ");
-                    std::io::ErrorKind::Other,
+
-                    "Failed to parse header",
+            match parts {
-                ));
+                Some(("Content-Length", value)) => {
+                    content_length = Some(value.parse().context("invalid content length")?);
+                }
+                Some((_, _)) => {}
+                None => {
+                    // Workaround: Some non-conformant language servers will output logging and other garbage
+                    // into the same stream as JSON-RPC messages. This can also happen from shell scripts that spawn
+                    // the server. Skip such lines and log a warning.
+
+                    // warn!("Failed to parse header: {:?}", header);
+                }
             }
-            headers.insert(parts[0].to_string(), parts[1].to_string());
         }
 
-        // find content-length
+        let content_length = content_length.context("missing content length")?;
-        let content_length = headers.get("Content-Length").unwrap().parse().unwrap();
 
+        //TODO: reuse vector
         let mut content = vec![0; content_length];
         reader.read_exact(&mut content).await?;
-        let msg = String::from_utf8(content).unwrap();
+        let msg = std::str::from_utf8(&content).context("invalid utf8 from server")?;
-
-        // read data
 
         info!("<- {}", msg);
 
         // try parsing as output (server response) or call (server request)
-        let output: serde_json::Result<Message> = serde_json::from_str(&msg);
+        let output: serde_json::Result<ServerMessage> = serde_json::from_str(msg);
 
         Ok(output?)
     }
 
-    async fn err(
+    async fn recv_server_error(
         err: &mut (impl AsyncBufRead + Unpin + Send),
-    ) -> core::result::Result<(), std::io::Error> {
+        buffer: &mut String,
-        let mut line = String::new();
+    ) -> Result<()> {
-        err.read_line(&mut line).await?;
+        buffer.truncate(0);
-        error!("err <- {}", line);
+        if err.read_line(buffer).await? == 0 {
+            return Err(Error::StreamClosed);
+        };
+        error!("err <- {:?}", buffer);
 
         Ok(())
     }
 
-    pub async fn send_payload(&mut self, payload: Payload) -> io::Result<()> {
+    async fn send_payload_to_server(
-        match payload {
+        &self,
+        server_stdin: &mut BufWriter<ChildStdin>,
+        payload: Payload,
+    ) -> Result<()> {
+        //TODO: reuse string
+        let json = match payload {
             Payload::Request { chan, value } => {
-                self.pending_requests.insert(value.id.clone(), chan);
+                self.pending_requests
-
+                    .lock()
-                let json = serde_json::to_string(&value)?;
+                    .await
-                self.send(json).await
+                    .insert(value.id.clone(), chan);
+                serde_json::to_string(&value)?
             }
-            Payload::Notification(value) => {
+            Payload::Notification(value) => serde_json::to_string(&value)?,
-                let json = serde_json::to_string(&value)?;
+            Payload::Response(error) => serde_json::to_string(&error)?,
-                self.send(json).await
+        };
-            }
+        self.send_string_to_server(server_stdin, json).await
-            Payload::Response(error) => {
-                let json = serde_json::to_string(&error)?;
-                self.send(json).await
-            }
-        }
     }
 
-    pub async fn send(&mut self, request: String) -> io::Result<()> {
+    async fn send_string_to_server(
+        &self,
+        server_stdin: &mut BufWriter<ChildStdin>,
+        request: String,
+    ) -> Result<()> {
         info!("-> {}", request);
 
         // send the headers
-        self.writer
+        server_stdin
             .write_all(format!("Content-Length: {}\r\n\r\n", request.len()).as_bytes())
             .await?;
 
         // send the body
-        self.writer.write_all(request.as_bytes()).await?;
+        server_stdin.write_all(request.as_bytes()).await?;
 
-        self.writer.flush().await?;
+        server_stdin.flush().await?;
 
         Ok(())
     }
 
-    async fn recv_msg(&mut self, msg: Message) -> anyhow::Result<()> {
+    async fn process_server_message(
+        &self,
+        client_tx: &UnboundedSender<(usize, jsonrpc::Call)>,
+        msg: ServerMessage,
+    ) -> Result<()> {
         match msg {
-            Message::Output(output) => self.recv_response(output).await?,
+            ServerMessage::Output(output) => self.process_request_response(output).await?,
-            Message::Call(call) => {
+            ServerMessage::Call(call) => {
-                self.incoming.send(call).unwrap();
+                client_tx
+                    .send((self.id, call))
+                    .context("failed to send a message to server")?;
                 // let notification = Notification::parse(&method, params);
             }
         };
         Ok(())
     }
 
-    async fn recv_response(&mut self, output: jsonrpc::Output) -> io::Result<()> {
+    async fn process_request_response(&self, output: jsonrpc::Output) -> Result<()> {
         let (id, result) = match output {
             jsonrpc::Output::Success(jsonrpc::Success { id, result, .. }) => {
                 info!("<- {}", result);
@@ -186,6 +217,8 @@ impl Transport {
 
         let tx = self
             .pending_requests
+            .lock()
+            .await
             .remove(&id)
             .expect("pending_request with id not found!");
 
@@ -200,29 +233,132 @@ impl Transport {
         Ok(())
     }
 
-    pub async fn duplex(mut self) {
+    async fn recv(
+        transport: Arc<Self>,
+        mut server_stdout: BufReader<ChildStdout>,
+        client_tx: UnboundedSender<(usize, jsonrpc::Call)>,
+    ) {
+        let mut recv_buffer = String::new();
+        loop {
+            match Self::recv_server_message(&mut server_stdout, &mut recv_buffer).await {
+                Ok(msg) => {
+                    match transport.process_server_message(&client_tx, msg).await {
+                        Ok(_) => {}
+                        Err(err) => {
+                            error!("err: <- {:?}", err);
+                            break;
+                        }
+                    };
+                }
+                Err(err) => {
+                    error!("err: <- {:?}", err);
+                    break;
+                }
+            }
+        }
+    }
+
+    async fn err(_transport: Arc<Self>, mut server_stderr: BufReader<ChildStderr>) {
+        let mut recv_buffer = String::new();
+        loop {
+            match Self::recv_server_error(&mut server_stderr, &mut recv_buffer).await {
+                Ok(_) => {}
+                Err(err) => {
+                    error!("err: <- {:?}", err);
+                    break;
+                }
+            }
+        }
+    }
+
+    async fn send(
+        transport: Arc<Self>,
+        mut server_stdin: BufWriter<ChildStdin>,
+        client_tx: UnboundedSender<(usize, jsonrpc::Call)>,
+        mut client_rx: UnboundedReceiver<Payload>,
+        initialize_notify: Arc<Notify>,
+    ) {
+        let mut pending_messages: Vec<Payload> = Vec::new();
+        let mut is_pending = true;
+
+        // Determine if a message is allowed to be sent early
+        fn is_initialize(payload: &Payload) -> bool {
+            use lsp_types::{
+                notification::{Initialized, Notification},
+                request::{Initialize, Request},
+            };
+            match payload {
+                Payload::Request {
+                    value: jsonrpc::MethodCall { method, .. },
+                    ..
+                } if method == Initialize::METHOD => true,
+                Payload::Notification(jsonrpc::Notification { method, .. })
+                    if method == Initialized::METHOD =>
+                {
+                    true
+                }
+                _ => false,
+            }
+        }
+
+        // TODO: events that use capabilities need to do the right thing
+
         loop {
             tokio::select! {
-                // client -> server
+                biased;
-                msg = self.outgoing.recv() => {
+                _ = initialize_notify.notified() => { // TODO: notified is technically not cancellation safe
-                    if msg.is_none() {
+                    // server successfully initialized
+                    is_pending = false;
+
+                    use lsp_types::notification::Notification;
+                    // Hack: inject an initialized notification so we trigger code that needs to happen after init
+                    let notification = ServerMessage::Call(jsonrpc::Call::Notification(jsonrpc::Notification {
+                        jsonrpc: None,
+
+                        method: lsp_types::notification::Initialized::METHOD.to_string(),
+                        params: jsonrpc::Params::None,
+                    }));
+                    match transport.process_server_message(&client_tx, notification).await {
+                        Ok(_) => {}
+                        Err(err) => {
+                            error!("err: <- {:?}", err);
+                        }
+                    }
+
+                    // drain the pending queue and send payloads to server
+                    for msg in pending_messages.drain(..) {
+                        log::info!("Draining pending message {:?}", msg);
+                        match transport.send_payload_to_server(&mut server_stdin, msg).await {
+                            Ok(_) => {}
+                            Err(err) => {
+                                error!("err: <- {:?}", err);
+                            }
+                        }
+                    }
+                }
+                msg = client_rx.recv() => {
+                    if let Some(msg) = msg {
+                        if is_pending && !is_initialize(&msg) {
+                            // ignore notifications
+                            if let Payload::Notification(_) = msg {
+                                continue;
+                            }
+
+                            log::info!("Language server not initialized, delaying request");
+                            pending_messages.push(msg);
+                        } else {
+                            match transport.send_payload_to_server(&mut server_stdin, msg).await {
+                                Ok(_) => {}
+                                Err(err) => {
+                                    error!("err: <- {:?}", err);
+                                }
+                            }
+                        }
+                    } else {
+                        // channel closed
                         break;
                     }
-                    let msg = msg.unwrap();
-
-                    self.send_payload(msg).await.unwrap();
                 }
-                // server <- client
-                msg = Self::recv(&mut self.reader, &mut self.headers) => {
-                    if msg.is_err() {
-                        error!("err: <- {:?}", msg);
-                        break;
-                    }
-                    let msg = msg.unwrap();
-
-                    self.recv_msg(msg).await.unwrap();
-                }
-                _msg = Self::err(&mut self.stderr) => {}
             }
         }
     }

helix-syntax/Cargo.toml

@@ -1,16 +1,21 @@
 [package]
 name = "helix-syntax"
-version = "0.0.10"
+version = "0.6.0"
 authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
-edition = "2018"
+edition = "2021"
 license = "MPL-2.0"
-
+description = "Tree-sitter grammars support"
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+categories = ["editor"]
+repository = "https://github.com/helix-editor/helix"
+homepage = "https://helix-editor.com"
+include = ["src/**/*", "languages/**/*", "build.rs", "!**/docs/**/*", "!**/test/**/*", "!**/examples/**/*", "!**/build/**/*"]
 
 [dependencies]
-tree-sitter = "0.19"
+tree-sitter = "0.20"
-serde = { version = "1.0", features = ["derive"] }
+libloading = "0.7"
+anyhow = "1"
 
 [build-dependencies]
-cc = { version = "1",  features = ["parallel"] }
+cc = { version = "1" }
 threadpool = { version = "1.0" }
+anyhow = "1"

helix-syntax/README.md

@@ -0,0 +1,13 @@
+helix-syntax
+============
+
+Syntax highlighting for helix, (shallow) submodules resides here.
+
+Differences from nvim-treesitter
+--------------------------------
+
+As the syntax are commonly ported from
+<https://github.com/nvim-treesitter/nvim-treesitter>.
+
+Note that we do not support the custom `#any-of` predicate which is
+supported by neovim so one needs to change it to `#match` with regex.

helix-syntax/build.rs

@@ -1,79 +1,151 @@
+use anyhow::{anyhow, Context, Result};
 use std::fs;
-use std::path::PathBuf;
+use std::time::SystemTime;
+use std::{
+    path::{Path, PathBuf},
+    process::Command,
+};
 
 use std::sync::mpsc::channel;
 
-fn collect_tree_sitter_dirs(ignore: &[String]) -> Vec<String> {
+fn collect_tree_sitter_dirs(ignore: &[String]) -> Result<Vec<String>> {
     let mut dirs = Vec::new();
-    for entry in fs::read_dir("languages").unwrap().flatten() {
+    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("languages");
-        let path = entry.path();
-        let dir = path.file_name().unwrap().to_str().unwrap().to_string();
-        if !ignore.contains(&dir) {
-            dirs.push(dir);
-        }
-    }
-    dirs
-}
 
-fn collect_src_files(dir: &str) -> (Vec<String>, Vec<String>) {
+    for entry in fs::read_dir(path)? {
-    eprintln!("Collect files for {}", dir);
+        let entry = entry?;
-
-    let mut c_files = Vec::new();
-    let mut cpp_files = Vec::new();
-    let path = PathBuf::from("languages").join(&dir).join("src");
-    for entry in fs::read_dir(path).unwrap().flatten() {
         let path = entry.path();
-        if path
+
-            .file_stem()
+        if !entry.file_type()?.is_dir() {
-            .unwrap()
-            .to_str()
-            .unwrap()
-            .starts_with("binding")
-        {
             continue;
         }
-        if let Some(ext) = path.extension() {
+
-            if ext == "c" {
+        let dir = path.file_name().unwrap().to_str().unwrap().to_string();
-                c_files.push(path.to_str().unwrap().to_string());
+
-            } else if ext == "cc" || ext == "cpp" || ext == "cxx" {
+        // filter ignores
-                cpp_files.push(path.to_str().unwrap().to_string());
+        if ignore.contains(&dir) {
+            continue;
+        }
+        dirs.push(dir)
+    }
+
+    Ok(dirs)
+}
+
+#[cfg(unix)]
+const DYLIB_EXTENSION: &str = "so";
+
+#[cfg(windows)]
+const DYLIB_EXTENSION: &str = "dll";
+
+fn build_library(src_path: &Path, language: &str) -> Result<()> {
+    let header_path = src_path;
+    // let grammar_path = src_path.join("grammar.json");
+    let parser_path = src_path.join("parser.c");
+    let mut scanner_path = src_path.join("scanner.c");
+
+    let scanner_path = if scanner_path.exists() {
+        Some(scanner_path)
+    } else {
+        scanner_path.set_extension("cc");
+        if scanner_path.exists() {
+            Some(scanner_path)
+        } else {
+            None
+        }
+    };
+    let parser_lib_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../runtime/grammars");
+    let mut library_path = parser_lib_path.join(language);
+    library_path.set_extension(DYLIB_EXTENSION);
+
+    let recompile = needs_recompile(&library_path, &parser_path, &scanner_path)
+        .with_context(|| "Failed to compare source and binary timestamps")?;
+
+    if !recompile {
+        return Ok(());
+    }
+    let mut config = cc::Build::new();
+    config.cpp(true).opt_level(2).cargo_metadata(false);
+    let compiler = config.get_compiler();
+    let mut command = Command::new(compiler.path());
+    command.current_dir(src_path);
+    for (key, value) in compiler.env() {
+        command.env(key, value);
+    }
+
+    if cfg!(windows) {
+        command
+            .args(&["/nologo", "/LD", "/I"])
+            .arg(header_path)
+            .arg("/Od")
+            .arg("/utf-8");
+        if let Some(scanner_path) = scanner_path.as_ref() {
+            command.arg(scanner_path);
+        }
+
+        command
+            .arg(parser_path)
+            .arg("/link")
+            .arg(format!("/out:{}", library_path.to_str().unwrap()));
+    } else {
+        command
+            .arg("-shared")
+            .arg("-fPIC")
+            .arg("-fno-exceptions")
+            .arg("-g")
+            .arg("-I")
+            .arg(header_path)
+            .arg("-o")
+            .arg(&library_path)
+            .arg("-O2");
+        if let Some(scanner_path) = scanner_path.as_ref() {
+            if scanner_path.extension() == Some("c".as_ref()) {
+                command.arg("-xc").arg("-std=c99").arg(scanner_path);
+            } else {
+                command.arg(scanner_path);
             }
         }
+        command.arg("-xc").arg(parser_path);
+        if cfg!(all(unix, not(target_os = "macos"))) {
+            command.arg("-Wl,-z,relro,-z,now");
+        }
     }
-    (c_files, cpp_files)
+
+    let output = command
+        .output()
+        .with_context(|| "Failed to execute C compiler")?;
+    if !output.status.success() {
+        return Err(anyhow!(
+            "Parser compilation failed.\nStdout: {}\nStderr: {}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr)
+        ));
+    }
+
+    Ok(())
+}
+fn needs_recompile(
+    lib_path: &Path,
+    parser_c_path: &Path,
+    scanner_path: &Option<PathBuf>,
+) -> Result<bool> {
+    if !lib_path.exists() {
+        return Ok(true);
+    }
+    let lib_mtime = mtime(lib_path)?;
+    if mtime(parser_c_path)? > lib_mtime {
+        return Ok(true);
+    }
+    if let Some(scanner_path) = scanner_path {
+        if mtime(scanner_path)? > lib_mtime {
+            return Ok(true);
+        }
+    }
+    Ok(false)
 }
 
-fn build_c(files: Vec<String>, language: &str) {
+fn mtime(path: &Path) -> Result<SystemTime> {
-    let mut build = cc::Build::new();
+    Ok(fs::metadata(path)?.modified()?)
-    for file in files {
-        build
-            .file(&file)
-            .include(PathBuf::from(file).parent().unwrap())
-            .pic(true)
-            .warnings(false);
-    }
-    build.compile(&format!("tree-sitter-{}-c", language));
-}
-
-fn build_cpp(files: Vec<String>, language: &str) {
-    let mut build = cc::Build::new();
-
-    let flag = if build.get_compiler().is_like_msvc() {
-        "/std:c++17"
-    } else {
-        "-std=c++14"
-    };
-
-    for file in files {
-        build
-            .file(&file)
-            .include(PathBuf::from(file).parent().unwrap())
-            .pic(true)
-            .warnings(false)
-            .cpp(true)
-            .flag_if_supported(flag);
-    }
-    build.compile(&format!("tree-sitter-{}-cpp", language));
 }
 
 fn build_dir(dir: &str, language: &str) {
@@ -86,28 +158,27 @@ fn build_dir(dir: &str, language: &str) {
         .is_none()
     {
         eprintln!(
-            "The directory {} is empty, did you use 'git clone --recursive'?",
+            "The directory {} is empty, you probably need to use 'git submodule update --init --recursive'?",
             dir
         );
-        eprintln!("You can fix in using 'git submodule init && git submodule update --recursive'.");
         std::process::exit(1);
     }
-    let (c, cpp) = collect_src_files(dir);
+
-    if !c.is_empty() {
+    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
-        build_c(c, language);
+        .join("languages")
-    }
+        .join(dir)
-    if !cpp.is_empty() {
+        .join("src");
-        build_cpp(cpp, language);
+
-    }
+    build_library(&path, language).unwrap();
 }
 
 fn main() {
     let ignore = vec![
         "tree-sitter-typescript".to_string(),
         "tree-sitter-haskell".to_string(), // aarch64 failures: https://github.com/tree-sitter/tree-sitter-haskell/issues/34
-        ".DS_Store".to_string(),
+        "tree-sitter-ocaml".to_string(),
     ];
-    let dirs = collect_tree_sitter_dirs(&ignore);
+    let dirs = collect_tree_sitter_dirs(&ignore).unwrap();
 
     let mut n_jobs = 0;
     let pool = threadpool::Builder::new().build(); // by going through the builder, it'll use num_cpus
@@ -118,7 +189,7 @@ fn main() {
         n_jobs += 1;
 
         pool.execute(move || {
-            let language = &dir[12..]; // skip tree-sitter- prefix
+            let language = &dir.strip_prefix("tree-sitter-").unwrap();
             build_dir(&dir, language);
 
             // report progress
@@ -131,4 +202,6 @@ fn main() {
 
     build_dir("tree-sitter-typescript/tsx", "tsx");
     build_dir("tree-sitter-typescript/typescript", "typescript");
+    build_dir("tree-sitter-ocaml/ocaml", "ocaml");
+    build_dir("tree-sitter-ocaml/interface", "ocaml-interface")
 }