Add 23.05 to the AppData

There was an issue with autocompletion of a path with a space in it.

Before:

:o test\ dir -> <TAB> -> test\ dirfile1

After:

:o test\ dir -> <TAB> -> test\ dir\file1

.cargo/config.toml

@@ -1,3 +1,3 @@
 [alias]
 xtask = "run --package xtask --"
-integration-test = "test --features integration --workspace --test integration"
+integration-test = "test --features integration --profile integration --workspace --test integration"

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -32,7 +32,7 @@ body:
     id: helix-log
     attributes:
       label: Helix log
-      description: See `hx -h` for log file path
+      description: See `hx -h` for log file path. If you can reproduce the issue run `RUST_BACKTRACE=1 hx -vv` to generate a more detailed log file.
       value: |
         <details><summary>~/.cache/helix/helix.log</summary>
 
@@ -61,7 +61,8 @@ body:
       label: Helix Version
       description: >
         Helix version (`hx -V` if using a release, `git describe` if building
-        from master)
-      placeholder: "helix 0.6.0 (c0dbd6dc)"
+        from master).
+        **Make sure that you are using the [latest helix release](https://github.com/helix-editor/helix/releases) or a newer master build**
+      placeholder: "helix 22.12 (5eaa6d97)"
     validations:
       required: true

.github/workflows/build.yml

@@ -4,24 +4,17 @@ on:
   push:
     branches:
       - master
+  merge_group:
   schedule:
     - cron: '00 01 * * *'
 
 jobs:
   check:
-    name: Check
+    name: Check (msrv)
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        rust: [stable, msrv]
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
-
-      - name: Use MSRV rust toolchain
-        if: matrix.rust == 'msrv'
-        run: cp .github/workflows/msrv-rust-toolchain.toml rust-toolchain.toml
-
       - name: Install stable toolchain
         uses: helix-editor/rust-toolchain@v1
         with:
@@ -44,10 +37,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Install stable toolchain
-        uses: helix-editor/rust-toolchain@v1
-        with:
-          profile: minimal
-          override: true
+        uses: dtolnay/rust-toolchain@1.65
 
       - uses: Swatinem/rust-cache@v2
 
@@ -76,10 +66,8 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Install stable toolchain
-        uses: helix-editor/rust-toolchain@v1
+        uses: dtolnay/rust-toolchain@1.65
         with:
-          profile: minimal
-          override: true
           components: rustfmt, clippy
 
       - uses: Swatinem/rust-cache@v2
@@ -103,13 +91,13 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Install stable toolchain
-        uses: helix-editor/rust-toolchain@v1
-        with:
-          profile: minimal
-          override: true
+        uses: dtolnay/rust-toolchain@1.65
 
       - uses: Swatinem/rust-cache@v2
 
+      - name: Validate queries
+        run: cargo xtask query-check
+
       - name: Generate docs
         run: cargo xtask docgen
 
@@ -120,20 +108,3 @@ jobs:
             || (echo "Run 'cargo xtask docgen', commit the changes and push again" \
             && exit 1)
 
-  queries:
-    name: Tree-sitter queries
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-
-      - name: Install stable toolchain
-        uses: helix-editor/rust-toolchain@v1
-        with:
-          profile: minimal
-          override: true
-
-      - uses: Swatinem/rust-cache@v2
-
-      - name: Generate docs
-        run: cargo xtask query-check

.github/workflows/cachix.yml

@@ -14,7 +14,7 @@ jobs:
       uses: actions/checkout@v3
 
     - name: Install nix
-      uses: cachix/install-nix-action@v18
+      uses: cachix/install-nix-action@v20
 
     - name: Authenticate with Cachix
       uses: cachix/cachix-action@v12

.github/workflows/msrv-rust-toolchain.toml

@@ -1,3 +0,0 @@
-[toolchain]
-channel = "1.61.0"
-components = ["rustfmt", "rust-src"]

.github/workflows/release.yml

@@ -26,10 +26,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Install stable toolchain
-        uses: helix-editor/rust-toolchain@v1
-        with:
-          profile: minimal
-          override: true
+        uses: dtolnay/rust-toolchain@stable
 
       - uses: Swatinem/rust-cache@v2
 
@@ -47,6 +44,16 @@ jobs:
   dist:
     name: Dist
     needs: [fetch-grammars]
+    env:
+      # For some builds, we use cross to test on 32-bit and big-endian
+      # systems.
+      CARGO: cargo
+      # When CARGO is set to CROSS, this is set to `--target matrix.target`.
+      TARGET_FLAGS:
+      # When CARGO is set to CROSS, TARGET_DIR includes matrix.target.
+      TARGET_DIR: ./target
+      # Emit backtraces on panics.
+      RUST_BACKTRACE: 1
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false # don't fail other jobs if one fails
@@ -54,17 +61,17 @@ jobs:
         build: [x86_64-linux, x86_64-macos, x86_64-windows] #, x86_64-win-gnu, win32-msvc
         include:
         - build: x86_64-linux
-          os: ubuntu-20.04
+          os: ubuntu-latest
           rust: stable
           target: x86_64-unknown-linux-gnu
           cross: false
         - build: aarch64-linux
-          os: ubuntu-20.04
+          os: ubuntu-latest
           rust: stable
           target: aarch64-unknown-linux-gnu
           cross: true
         - build: riscv64-linux
-          os: ubuntu-20.04
+          os: ubuntu-latest
           rust: stable
           target: riscv64gc-unknown-linux-gnu
           cross: true
@@ -74,10 +81,11 @@ jobs:
           target: x86_64-apple-darwin
           cross: false
         - build: x86_64-windows
-          os: windows-2019
+          os: windows-latest
           rust: stable
           target: x86_64-pc-windows-msvc
           cross: false
+        # 23.03: build issues
         - build: aarch64-macos
           os: macos-latest
           rust: stable
@@ -106,13 +114,17 @@ jobs:
           mkdir -p runtime/grammars/sources
           tar xJf grammars/grammars.tar.xz -C runtime/grammars/sources
 
+      # The rust-toolchain action ignores rust-toolchain.toml files.
+      # Removing this before building with cargo ensures that the rust-toolchain
+      # is considered the same between installation and usage.
+      - name: Remove the rust-toolchain.toml file
+        run: rm rust-toolchain.toml
+
       - name: Install ${{ matrix.rust }} toolchain
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@master
         with:
-          profile: minimal
           toolchain: ${{ matrix.rust }}
           target: ${{ matrix.target }}
-          override: true
 
       # Install a pre-release version of Cross
       # TODO: We need to pre-install Cross because we need cross-rs/cross#591 to
@@ -120,15 +132,20 @@ jobs:
       #       0.3.0, which includes cross-rs/cross#591, is released.
       - name: Install Cross
         if: "matrix.cross"
-        run: cargo install cross --git https://github.com/cross-rs/cross.git --rev 47df5c76e7cba682823a0b6aa6d95c17b31ba63a
+        run: |
+          cargo install cross --git https://github.com/cross-rs/cross.git --rev 47df5c76e7cba682823a0b6aa6d95c17b31ba63a
+          echo "CARGO=cross" >> $GITHUB_ENV
+        # echo "TARGET_FLAGS=--target ${{ matrix.target }}" >> $GITHUB_ENV
+        # echo "TARGET_DIR=./target/${{ matrix.target }}" >> $GITHUB_ENV
+
+      - name: Show command used for Cargo
+        run: |
+          echo "cargo command is: ${{ env.CARGO }}"
+          echo "target flag is: ${{ env.TARGET_FLAGS }}"
 
       - name: Run cargo test
-        uses: actions-rs/cargo@v1
         if: "!matrix.skip_tests"
-        with:
-          use-cross: ${{ matrix.cross }}
-          command: test
-          args: --release --locked --target ${{ matrix.target }} --workspace
+        run: ${{ env.CARGO }} test --release --locked --target ${{ matrix.target }} --workspace
 
       - name: Set profile.release.strip = true
         shell: bash
@@ -139,16 +156,16 @@ jobs:
           EOF
 
       - name: Build release binary
-        uses: actions-rs/cargo@v1
-        with:
-          use-cross: ${{ matrix.cross }}
-          command: build
-          args: --release --locked --target ${{ matrix.target }}
+        run: ${{ env.CARGO }} build --release --locked --target ${{ matrix.target }}
 
       - name: Build AppImage
         shell: bash
         if: matrix.build == 'aarch64-linux' || matrix.build == 'x86_64-linux'
         run: |
+          # Required as of 22.x https://github.com/AppImage/AppImageKit/wiki/FUSE
+          sudo add-apt-repository universe
+          sudo apt install libfuse2
+
           mkdir dist
 
           name=dev
@@ -238,7 +255,7 @@ jobs:
                   exe=".exe"
               fi
               pkgname=helix-$GITHUB_REF_NAME-$platform
-              mkdir $pkgname
+              mkdir -p $pkgname
               cp $source/LICENSE $source/README.md $pkgname
               mkdir $pkgname/contrib
               cp -r $source/contrib/completion $pkgname/contrib

CHANGELOG.md

@@ -1,3 +1,393 @@
+# 23.05 (2023-05-18)
+
+23.05 is a smaller release focusing on fixes. There were 88 contributors in this release. Thank you all!
+
+Features:
+
+- Add a config option to exclude declaration from LSP references request ([#6886](https://github.com/helix-editor/helix/pull/6886))
+- Enable injecting languages based on their file extension and shebang ([#3970](https://github.com/helix-editor/helix/pull/3970))
+- Sort the buffer picker by most recent access ([#2980](https://github.com/helix-editor/helix/pull/2980))
+- Perform syntax highlighting in the picker asynchronously ([#7028](https://github.com/helix-editor/helix/pull/7028))
+
+Commands:
+
+- `:update` is now aliased as `:u` ([#6835](https://github.com/helix-editor/helix/pull/6835))
+- Add `extend_to_first_nonwhitespace` which acts the same as `goto_first_nonwhitespace` but always extends ([#6837](https://github.com/helix-editor/helix/pull/6837))
+- Add `:clear-register` for clearing the given register or all registers ([#5695](https://github.com/helix-editor/helix/pull/5695))
+- Add `:write-buffer-close` and `:write-buffer-close!` ([#6947](https://github.com/helix-editor/helix/pull/6947))
+
+Fixes:
+
+- Normalize LSP workspace paths ([#6517](https://github.com/helix-editor/helix/pull/6517))
+- Robustly handle invalid LSP ranges ([#6512](https://github.com/helix-editor/helix/pull/6512))
+- Fix line number display for LSP goto pickers ([#6559](https://github.com/helix-editor/helix/pull/6559))
+- Fix toggling of `soft-wrap.enable` option ([#6656](https://github.com/helix-editor/helix/pull/6656), [58e457a](https://github.com/helix-editor/helix/commit/58e457a), [#6742](https://github.com/helix-editor/helix/pull/6742))
+- Handle `workspace/configuration` requests from stopped language servers ([#6693](https://github.com/helix-editor/helix/pull/6693))
+- Fix possible crash from opening the jumplist picker ([#6672](https://github.com/helix-editor/helix/pull/6672))
+- Fix theme preview returning to current theme on line and word deletions ([#6694](https://github.com/helix-editor/helix/pull/6694))
+- Re-run crate build scripts on changes to revision and grammar repositories ([#6743](https://github.com/helix-editor/helix/pull/6743))
+- Fix crash on opening from suspended state ([#6764](https://github.com/helix-editor/helix/pull/6764))
+- Fix unwrap bug in DAP ([#6786](https://github.com/helix-editor/helix/pull/6786))
+- Always build tree-sitter parsers with C++14 and C11 ([#6792](https://github.com/helix-editor/helix/pull/6792), [#6834](https://github.com/helix-editor/helix/pull/6834), [#6845](https://github.com/helix-editor/helix/pull/6845))
+- Exit with a non-zero statuscode when tree-sitter parser builds fail ([#6795](https://github.com/helix-editor/helix/pull/6795))
+- Flip symbol range in LSP goto commands ([#6794](https://github.com/helix-editor/helix/pull/6794))
+- Fix runtime toggling of the `mouse` option ([#6675](https://github.com/helix-editor/helix/pull/6675))
+- Fix panic in inlay hint computation when view anchor is out of bounds ([#6883](https://github.com/helix-editor/helix/pull/6883))
+- Significantly improve performance of git discovery on slow file systems ([#6890](https://github.com/helix-editor/helix/pull/6890))
+- Downgrade gix log level to info ([#6915](https://github.com/helix-editor/helix/pull/6915))
+- Conserve BOM and properly support saving UTF16 files ([#6497](https://github.com/helix-editor/helix/pull/6497))
+- Correctly handle completion re-request ([#6594](https://github.com/helix-editor/helix/pull/6594))
+- Fix offset encoding in LSP `didChange` notifications ([#6921](https://github.com/helix-editor/helix/pull/6921))
+- Change `gix` logging level to info ([#6915](https://github.com/helix-editor/helix/pull/6915))
+- Improve error message when writes fail because parent directories do not exist ([#7014](https://github.com/helix-editor/helix/pull/7014))
+- Replace DAP variables popup instead of pushing more popups ([#7034](https://github.com/helix-editor/helix/pull/7034))
+- Disable tree-sitter for files after parsing for 500ms ([#7028](https://github.com/helix-editor/helix/pull/7028))
+- Fix crash when deleting with multiple cursors ([#6024](https://github.com/helix-editor/helix/pull/6024))
+- Fix selection sliding when deleting forwards in append mode ([#6024](https://github.com/helix-editor/helix/pull/6024))
+- Fix completion on paths containing spaces ([#6779](https://github.com/helix-editor/helix/pull/6779))
+
+Themes:
+
+- Style inlay hints in `dracula` theme ([#6515](https://github.com/helix-editor/helix/pull/6515))
+- Style inlay hints in `onedark` theme ([#6503](https://github.com/helix-editor/helix/pull/6503))
+- Style inlay hints and the soft-wrap indicator in `varua` ([#6568](https://github.com/helix-editor/helix/pull/6568), [#6589](https://github.com/helix-editor/helix/pull/6589))
+- Style inlay hints in `emacs` theme ([#6569](https://github.com/helix-editor/helix/pull/6569))
+- Update `base16_transparent` and `dark_high_contrast` themes ([#6577](https://github.com/helix-editor/helix/pull/6577))
+- Style inlay hints for `mellow` and `rasmus` themes ([#6583](https://github.com/helix-editor/helix/pull/6583))
+- Dim pane divider for `base16_transparent` theme ([#6534](https://github.com/helix-editor/helix/pull/6534))
+- Style inlay hints in `zenburn` theme ([#6593](https://github.com/helix-editor/helix/pull/6593))
+- Style inlay hints in `boo_berry` theme ([#6625](https://github.com/helix-editor/helix/pull/6625))
+- Add `ferra` theme ([#6619](https://github.com/helix-editor/helix/pull/6619), [#6776](https://github.com/helix-editor/helix/pull/6776))
+- Style inlay hints in `nightfox` theme ([#6655](https://github.com/helix-editor/helix/pull/6655))
+- Fix `ayu` theme family markup code block background ([#6538](https://github.com/helix-editor/helix/pull/6538))
+- Improve whitespace and search match colors in `rose_pine` theme ([#6679](https://github.com/helix-editor/helix/pull/6679))
+- Highlight selected items in `base16_transparent` theme ([#6716](https://github.com/helix-editor/helix/pull/6716))
+- Adjust everforest to resemble original more closely ([#5866](https://github.com/helix-editor/helix/pull/5866))
+- Refactor `dracula` theme ([#6552](https://github.com/helix-editor/helix/pull/6552), [#6767](https://github.com/helix-editor/helix/pull/6767), [#6855](https://github.com/helix-editor/helix/pull/6855), [#6987](https://github.com/helix-editor/helix/pull/6987))
+- Style inlay hints in `darcula` theme ([#6732](https://github.com/helix-editor/helix/pull/6732))
+- Style inlay hints in `kanagawa` theme ([#6773](https://github.com/helix-editor/helix/pull/6773))
+- Improve `ayu_dark` theme ([#6622](https://github.com/helix-editor/helix/pull/6622))
+- Refactor `noctis` theme multiple cursor highlighting ([96720e7](https://github.com/helix-editor/helix/commit/96720e7))
+- Refactor `noctis` theme whitespace rendering and indent guides ([f2ccc03](https://github.com/helix-editor/helix/commit/f2ccc03))
+- Add `amberwood` theme ([#6924](https://github.com/helix-editor/helix/pull/6924))
+- Update `nightfox` theme ([#7061](https://github.com/helix-editor/helix/pull/7061))
+
+Language support:
+
+- R language server: use the `--no-echo` flag to silence output ([#6570](https://github.com/helix-editor/helix/pull/6570))
+- Recognize CUDA files as C++ ([#6521](https://github.com/helix-editor/helix/pull/6521))
+- Add support for Hurl ([#6450](https://github.com/helix-editor/helix/pull/6450))
+- Add textobject queries for Julia ([#6588](https://github.com/helix-editor/helix/pull/6588))
+- Update Ruby highlight queries ([#6587](https://github.com/helix-editor/helix/pull/6587))
+- Add xsd to XML file-types ([#6631](https://github.com/helix-editor/helix/pull/6631))
+- Support Robot Framework ([#6611](https://github.com/helix-editor/helix/pull/6611))
+- Update Gleam tree-sitter parser ([#6641](https://github.com/helix-editor/helix/pull/6641))
+- Update git-commit tree-sitter parser ([#6692](https://github.com/helix-editor/helix/pull/6692))
+- Update Haskell tree-sitter parser ([#6317](https://github.com/helix-editor/helix/pull/6317))
+- Add injection queries for Haskell quasiquotes ([#6474](https://github.com/helix-editor/helix/pull/6474))
+- Highlight C/C++ escape sequences ([#6724](https://github.com/helix-editor/helix/pull/6724))
+- Support Markdoc ([#6432](https://github.com/helix-editor/helix/pull/6432))
+- Support OpenCL ([#6473](https://github.com/helix-editor/helix/pull/6473))
+- Support DTD ([#6644](https://github.com/helix-editor/helix/pull/6644))
+- Fix constant highlighting in Python queries ([#6751](https://github.com/helix-editor/helix/pull/6751))
+- Support Just ([#6453](https://github.com/helix-editor/helix/pull/6453))
+- Fix Go locals query for `var_spec` identifiers ([#6763](https://github.com/helix-editor/helix/pull/6763))
+- Update Markdown tree-sitter parser ([#6785](https://github.com/helix-editor/helix/pull/6785))
+- Fix Haskell workspace root for cabal projects ([#6828](https://github.com/helix-editor/helix/pull/6828))
+- Avoid extra indentation in Go switches ([#6817](https://github.com/helix-editor/helix/pull/6817))
+- Fix Go workspace roots ([#6884](https://github.com/helix-editor/helix/pull/6884))
+- Set PerlNavigator as the default Perl language server ([#6860](https://github.com/helix-editor/helix/pull/6860))
+- Highlight more sqlx macros in Rust ([#6793](https://github.com/helix-editor/helix/pull/6793))
+- Switch Odin tree-sitter grammar ([#6766](https://github.com/helix-editor/helix/pull/6766))
+- Recognize `poetry.lock` as TOML ([#6928](https://github.com/helix-editor/helix/pull/6928))
+- Recognize Jupyter notebooks as JSON ([#6927](https://github.com/helix-editor/helix/pull/6927))
+- Add language server configuration for Crystal ([#6948](https://github.com/helix-editor/helix/pull/6948))
+- Add `build.gradle.kts` to Java and Scala roots ([#6970](https://github.com/helix-editor/helix/pull/6970))
+- Recognize `sty` and `cls` files as latex ([#6986](https://github.com/helix-editor/helix/pull/6986))
+- Update Dockerfile tree-sitter grammar ([#6895](https://github.com/helix-editor/helix/pull/6895))
+- Add comment injections for Odin ([#7027](https://github.com/helix-editor/helix/pull/7027))
+- Recognize `gml` as XML ([#7055](https://github.com/helix-editor/helix/pull/7055))
+- Recognize `geojson` as JSON ([#7054](https://github.com/helix-editor/helix/pull/7054))
+
+Packaging:
+
+- Update the Nix flake dependencies, remove a deprecated option ([#6546](https://github.com/helix-editor/helix/pull/6546))
+- Fix and re-enable aarch64-macos release binary builds ([#6504](https://github.com/helix-editor/helix/pull/6504))
+- The git dependency on `tree-sitter` has been replaced with a regular crates.io dependency ([#6608](https://github.com/helix-editor/helix/pull/6608))
+
+# 23.03 (2023-03-31)
+
+23.03 brings some long-awaited and exciting features. Thank you to everyone involved! This release saw changes from 102 contributors.
+
+For the full log, check out the [git log](https://github.com/helix-editor/helix/compare/22.12..23.03).
+Also check out the [release notes](https://helix-editor.com/news/release-23-03-highlights/) for more commentary on larger features.
+
+Breaking changes:
+
+- Select diagnostic range in `goto_*_diag` commands ([#4713](https://github.com/helix-editor/helix/pull/4713), [#5164](https://github.com/helix-editor/helix/pull/5164), [#6193](https://github.com/helix-editor/helix/pull/6193))
+- Remove jump behavior from `increment`/`decrement` ([#4123](https://github.com/helix-editor/helix/pull/4123), [#5929](https://github.com/helix-editor/helix/pull/5929))
+- Select change range in `goto_*_change` commands ([#5206](https://github.com/helix-editor/helix/pull/5206))
+- Split file modification indicator from filename statusline elements ([#4731](https://github.com/helix-editor/helix/pull/4731), [#6036](https://github.com/helix-editor/helix/pull/6036))
+- Jump to symbol ranges in LSP goto commands ([#5986](https://github.com/helix-editor/helix/pull/5986))
+- Workspace detection now stops at the first `.helix/` directory (merging multiple `.helix/languages.toml` configurations is no longer supported) ([#5748](https://github.com/helix-editor/helix/pull/5748))
+
+Features:
+
+- Dynamic workspace symbol picker ([#5055](https://github.com/helix-editor/helix/pull/5055))
+- Soft-wrap ([#5420](https://github.com/helix-editor/helix/pull/5420), [#5786](https://github.com/helix-editor/helix/pull/5786), [#5893](https://github.com/helix-editor/helix/pull/5893), [#6142](https://github.com/helix-editor/helix/pull/6142), [#6440](https://github.com/helix-editor/helix/pull/6440))
+- Initial support for LSP snippet completions ([#5864](https://github.com/helix-editor/helix/pull/5864), [b1f7528](https://github.com/helix-editor/helix/commit/b1f7528), [#6263](https://github.com/helix-editor/helix/pull/6263), [bbf4800](https://github.com/helix-editor/helix/commit/bbf4800), [90348b8](https://github.com/helix-editor/helix/commit/90348b8), [f87299f](https://github.com/helix-editor/helix/commit/f87299f), [#6371](https://github.com/helix-editor/helix/pull/6371), [9fe3adc](https://github.com/helix-editor/helix/commit/9fe3adc))
+- Add a statusline element for showing the current version control HEAD ([#5682](https://github.com/helix-editor/helix/pull/5682))
+- Display LSP type hints ([#5420](https://github.com/helix-editor/helix/pull/5420), [#5934](https://github.com/helix-editor/helix/pull/5934), [#6312](https://github.com/helix-editor/helix/pull/6312))
+- Enable the Kitty keyboard protocol on terminals with support ([#4939](https://github.com/helix-editor/helix/pull/4939), [#6170](https://github.com/helix-editor/helix/pull/6170), [#6194](https://github.com/helix-editor/helix/pull/6194), [#6438](https://github.com/helix-editor/helix/pull/6438))
+- Add a statusline element for the basename of the current file ([#5318](https://github.com/helix-editor/helix/pull/5318))
+- Add substring matching syntax for the picker ([#5658](https://github.com/helix-editor/helix/pull/5658))
+- Support LSP `textDocument/prepareRename` ([#6103](https://github.com/helix-editor/helix/pull/6103))
+- Allow multiple runtime directories with priorities ([#5411](https://github.com/helix-editor/helix/pull/5411))
+- Allow configuring whether to insert or replace completions ([#5728](https://github.com/helix-editor/helix/pull/5728))
+- Allow per-workspace config file `.helix/config.toml` ([#5748](https://github.com/helix-editor/helix/pull/5748))
+- Add `workspace-lsp-roots` config option to support multiple LSP roots for use with monorepos ([#5748](https://github.com/helix-editor/helix/pull/5748))
+
+Commands:
+
+- `:pipe-to` which pipes selections into a shell command and ignores output ([#4931](https://github.com/helix-editor/helix/pull/4931))
+- `merge_consecutive_selections` (`A-_`) combines all consecutive selections ([#5047](https://github.com/helix-editor/helix/pull/5047))
+- `rotate_view_reverse` which focuses the previous view ([#5356](https://github.com/helix-editor/helix/pull/5356))
+- `goto_declaration` (`gD`, requires LSP) which jumps to a symbol's declaration ([#5646](https://github.com/helix-editor/helix/pull/5646))
+- `file_picker_in_current_buffer_directory` ([#4666](https://github.com/helix-editor/helix/pull/4666))
+- `:character-info` which shows information about the character under the cursor ([#4000](https://github.com/helix-editor/helix/pull/4000))
+- `:toggle-option` for toggling config options at runtime ([#4085](https://github.com/helix-editor/helix/pull/4085))
+- `dap_restart` for restarting a debug session in DAP ([#5651](https://github.com/helix-editor/helix/pull/5651))
+- `:lsp-stop` to stop the language server of the current buffer ([#5964](https://github.com/helix-editor/helix/pull/5964))
+- `:reset-diff-change` for resetting a diff hunk to its original text ([#4974](https://github.com/helix-editor/helix/pull/4974))
+- `:config-open-workspace` for opening the config file local to the current workspace ([#5748](https://github.com/helix-editor/helix/pull/5748))
+
+Usability improvements:
+
+- Remove empty detail section in completion menu when LSP doesn't send details ([#4902](https://github.com/helix-editor/helix/pull/4902))
+- Pass client information on LSP initialization ([#4904](https://github.com/helix-editor/helix/pull/4904))
+- Allow specifying environment variables for language servers in language config ([#4004](https://github.com/helix-editor/helix/pull/4004))
+- Allow detached git worktrees to be recognized as root paths ([#5097](https://github.com/helix-editor/helix/pull/5097))
+- Improve error message handling for theme loading failures ([#5073](https://github.com/helix-editor/helix/pull/5073))
+- Print the names of binaries required for LSP/DAP in health-check ([#5195](https://github.com/helix-editor/helix/pull/5195))
+- Improve sorting in the picker in cases of ties ([#5169](https://github.com/helix-editor/helix/pull/5169))
+- Add theming for prompt suggestions ([#5104](https://github.com/helix-editor/helix/pull/5104))
+- Open a file picker when using `:open` on directories ([#2707](https://github.com/helix-editor/helix/pull/2707), [#5278](https://github.com/helix-editor/helix/pull/5278))
+- Reload language config with `:config-reload` ([#5239](https://github.com/helix-editor/helix/pull/5239), [#5381](https://github.com/helix-editor/helix/pull/5381), [#5431](https://github.com/helix-editor/helix/pull/5431))
+- Improve indent queries for python when the tree is errored ([#5332](https://github.com/helix-editor/helix/pull/5332))
+- Picker: Open files without closing the picker with `A-ret` ([#4435](https://github.com/helix-editor/helix/pull/4435))
+- Allow theming cursors by primary/secondary and by mode ([#5130](https://github.com/helix-editor/helix/pull/5130))
+- Allow configuration of the minimum width for the line-numbers gutter ([#4724](https://github.com/helix-editor/helix/pull/4724), [#5696](https://github.com/helix-editor/helix/pull/5696))
+- Use filename completer for `:run-shell-command` command ([#5729](https://github.com/helix-editor/helix/pull/5729))
+- Surround with line-endings with `ms<ret>` ([#4571](https://github.com/helix-editor/helix/pull/4571))
+- Hide duplicate symlinks in file pickers ([#5658](https://github.com/helix-editor/helix/pull/5658))
+- Tabulate buffer picker contents ([#5777](https://github.com/helix-editor/helix/pull/5777))
+- Add an option to disable LSP ([#4425](https://github.com/helix-editor/helix/pull/4425))
+- Short-circuit tree-sitter and word object motions ([#5851](https://github.com/helix-editor/helix/pull/5851))
+- Add exit code to failed command message ([#5898](https://github.com/helix-editor/helix/pull/5898))
+- Make `m` textobject look for pairs enclosing selections ([#3344](https://github.com/helix-editor/helix/pull/3344))
+- Negotiate LSP position encoding ([#5894](https://github.com/helix-editor/helix/pull/5894), [a48d1a4](https://github.com/helix-editor/helix/commit/a48d1a4))
+- Display deprecated LSP completions with strikethrough ([#5932](https://github.com/helix-editor/helix/pull/5932))
+- Add JSONRPC request ID to failed LSP/DAP request log messages ([#6010](https://github.com/helix-editor/helix/pull/6010), [#6018](https://github.com/helix-editor/helix/pull/6018))
+- Ignore case when filtering LSP completions ([#6008](https://github.com/helix-editor/helix/pull/6008))
+- Show current language when no arguments are passed to `:set-language` ([#5895](https://github.com/helix-editor/helix/pull/5895))
+- Refactor and rewrite all book documentation ([#5534](https://github.com/helix-editor/helix/pull/5534))
+- Separate diagnostic picker message and code ([#6095](https://github.com/helix-editor/helix/pull/6095))
+- Add a config option to bypass undercurl detection ([#6253](https://github.com/helix-editor/helix/pull/6253))
+- Only complete appropriate arguments for typed commands ([#5966](https://github.com/helix-editor/helix/pull/5966))
+- Discard outdated LSP diagnostics ([3c9d5d0](https://github.com/helix-editor/helix/commit/3c9d5d0))
+- Discard outdated LSP workspace edits ([b6a4927](https://github.com/helix-editor/helix/commit/b6a4927))
+- Run shell commands asynchronously ([#6373](https://github.com/helix-editor/helix/pull/6373))
+- Show diagnostic codes in LSP diagnostic messages ([#6378](https://github.com/helix-editor/helix/pull/6378))
+- Highlight the current line in a DAP debug session ([#5957](https://github.com/helix-editor/helix/pull/5957))
+- Hide signature help if it overlaps with the completion menu ([#5523](https://github.com/helix-editor/helix/pull/5523), [7a69c40](https://github.com/helix-editor/helix/commit/7a69c40))
+
+Fixes:
+
+- Fix behavior of `auto-completion` flag for completion-on-trigger ([#5042](https://github.com/helix-editor/helix/pull/5042))
+- Reset editor mode when changing buffers ([#5072](https://github.com/helix-editor/helix/pull/5072))
+- Respect scrolloff settings in mouse movements ([#5255](https://github.com/helix-editor/helix/pull/5255))
+- Avoid trailing `s` when only one file is opened ([#5189](https://github.com/helix-editor/helix/pull/5189))
+- Fix erroneous indent between closers of auto-pairs ([#5330](https://github.com/helix-editor/helix/pull/5330))
+- Expand `~` when parsing file paths in `:open` ([#5329](https://github.com/helix-editor/helix/pull/5329))
+- Fix theme inheritance for default themes ([#5218](https://github.com/helix-editor/helix/pull/5218))
+- Fix `extend_line` with a count when the current line(s) are selected ([#5288](https://github.com/helix-editor/helix/pull/5288))
+- Prompt: Fix autocompletion for paths containing periods ([#5175](https://github.com/helix-editor/helix/pull/5175))
+- Skip serializing JSONRPC params if params is null ([#5471](https://github.com/helix-editor/helix/pull/5471))
+- Fix interaction with the `xclip` clipboard provider ([#5426](https://github.com/helix-editor/helix/pull/5426))
+- Fix undo/redo execution from the command palette ([#5294](https://github.com/helix-editor/helix/pull/5294))
+- Fix highlighting of non-block cursors ([#5575](https://github.com/helix-editor/helix/pull/5575))
+- Fix panic when nooping in `join_selections` and `join_selections_space` ([#5423](https://github.com/helix-editor/helix/pull/5423))
+- Fix selecting a changed file in global search ([#5639](https://github.com/helix-editor/helix/pull/5639))
+- Fix initial syntax highlight layer sort order ([#5196](https://github.com/helix-editor/helix/pull/5196))
+- Fix UTF-8 length handling for shellwords ([#5738](https://github.com/helix-editor/helix/pull/5738))
+- Remove C-j and C-k bindings from the completion menu ([#5070](https://github.com/helix-editor/helix/pull/5070))
+- Always commit to history when pasting ([#5790](https://github.com/helix-editor/helix/pull/5790))
+- Properly handle LSP position encoding ([#5711](https://github.com/helix-editor/helix/pull/5711))
+- Fix infinite loop in `copy_selection_on_prev_line` ([#5888](https://github.com/helix-editor/helix/pull/5888))
+- Fix completion popup positioning ([#5842](https://github.com/helix-editor/helix/pull/5842))
+- Fix a panic when uncommenting a line with only a comment token ([#5933](https://github.com/helix-editor/helix/pull/5933))
+- Fix panic in `goto_window_center` at EOF ([#5987](https://github.com/helix-editor/helix/pull/5987))
+- Ignore invalid file URIs sent by a language server ([#6000](https://github.com/helix-editor/helix/pull/6000))
+- Decode LSP URIs for the workspace diagnostics picker ([#6016](https://github.com/helix-editor/helix/pull/6016))
+- Fix incorrect usages of `tab_width` with `indent_width` ([#5918](https://github.com/helix-editor/helix/pull/5918))
+- DAP: Send Disconnect if the Terminated event is received ([#5532](https://github.com/helix-editor/helix/pull/5532))
+- DAP: Validate key and index exist when requesting variables ([#5628](https://github.com/helix-editor/helix/pull/5628))
+- Check LSP renaming support before prompting for rename text ([#6257](https://github.com/helix-editor/helix/pull/6257))
+- Fix indent guide rendering ([#6136](https://github.com/helix-editor/helix/pull/6136))
+- Fix division by zero panic ([#6155](https://github.com/helix-editor/helix/pull/6155))
+- Fix lacking space panic ([#6109](https://github.com/helix-editor/helix/pull/6109))
+- Send error replies for malformed and unhandled LSP requests ([#6058](https://github.com/helix-editor/helix/pull/6058))
+- Fix table column calculations for dynamic pickers ([#5920](https://github.com/helix-editor/helix/pull/5920))
+- Skip adding jumplist entries for `:<n>` line number previews ([#5751](https://github.com/helix-editor/helix/pull/5751))
+- Fix completion race conditions ([#6173](https://github.com/helix-editor/helix/pull/6173))
+- Fix `shrink_selection` with multiple cursors ([#6093](https://github.com/helix-editor/helix/pull/6093))
+- Fix indentation calculation for lines with mixed tabs/spaces ([#6278](https://github.com/helix-editor/helix/pull/6278))
+- No-op `client/registerCapability` LSP requests ([#6258](https://github.com/helix-editor/helix/pull/6258))
+- Send the STOP signal to all processes in the process group ([#3546](https://github.com/helix-editor/helix/pull/3546))
+- Fix workspace edit client capabilities declaration ([7bf168d](https://github.com/helix-editor/helix/commit/7bf168d))
+- Fix highlighting in picker results with multiple columns ([#6333](https://github.com/helix-editor/helix/pull/6333))
+- Canonicalize paths before stripping the current dir as a prefix ([#6290](https://github.com/helix-editor/helix/pull/6290))
+- Fix truncation behavior for long path names in the file picker ([#6410](https://github.com/helix-editor/helix/pull/6410), [67783dd](https://github.com/helix-editor/helix/commit/67783dd))
+- Fix theme reloading behavior in `:config-reload` ([ab819d8](https://github.com/helix-editor/helix/commit/ab819d8))
+
+Themes:
+
+- Update `serika` ([#5038](https://github.com/helix-editor/helix/pull/5038), [#6344](https://github.com/helix-editor/helix/pull/6344))
+- Update `flatwhite` ([#5036](https://github.com/helix-editor/helix/pull/5036), [#6323](https://github.com/helix-editor/helix/pull/6323))
+- Update `autumn` ([#5051](https://github.com/helix-editor/helix/pull/5051), [#5397](https://github.com/helix-editor/helix/pull/5397), [#6280](https://github.com/helix-editor/helix/pull/6280), [#6316](https://github.com/helix-editor/helix/pull/6316))
+- Update `acme` ([#5019](https://github.com/helix-editor/helix/pull/5019), [#5486](https://github.com/helix-editor/helix/pull/5486), [#5488](https://github.com/helix-editor/helix/pull/5488))
+- Update `gruvbox` themes ([#5066](https://github.com/helix-editor/helix/pull/5066), [#5333](https://github.com/helix-editor/helix/pull/5333), [#5540](https://github.com/helix-editor/helix/pull/5540), [#6285](https://github.com/helix-editor/helix/pull/6285), [#6295](https://github.com/helix-editor/helix/pull/6295))
+- Update `base16_transparent` ([#5105](https://github.com/helix-editor/helix/pull/5105))
+- Update `dark_high_contrast` ([#5105](https://github.com/helix-editor/helix/pull/5105))
+- Update `dracula` ([#5236](https://github.com/helix-editor/helix/pull/5236), [#5627](https://github.com/helix-editor/helix/pull/5627), [#6414](https://github.com/helix-editor/helix/pull/6414))
+- Update `monokai_pro_spectrum` ([#5250](https://github.com/helix-editor/helix/pull/5250), [#5602](https://github.com/helix-editor/helix/pull/5602))
+- Update `rose_pine` ([#5267](https://github.com/helix-editor/helix/pull/5267), [#5489](https://github.com/helix-editor/helix/pull/5489), [#6384](https://github.com/helix-editor/helix/pull/6384))
+- Update `kanagawa` ([#5273](https://github.com/helix-editor/helix/pull/5273), [#5571](https://github.com/helix-editor/helix/pull/5571), [#6085](https://github.com/helix-editor/helix/pull/6085))
+- Update `emacs` ([#5334](https://github.com/helix-editor/helix/pull/5334))
+- Add `github` themes ([#5353](https://github.com/helix-editor/helix/pull/5353), [efeec12](https://github.com/helix-editor/helix/commit/efeec12))
+    - Dark themes: `github_dark`, `github_dark_colorblind`, `github_dark_dimmed`, `github_dark_high_contrast`, `github_dark_tritanopia`
+    - Light themes: `github_light`, `github_light_colorblind`, `github_light_dimmed`, `github_light_high_contrast`, `github_light_tritanopia`
+- Update `solarized` variants ([#5445](https://github.com/helix-editor/helix/pull/5445), [#6327](https://github.com/helix-editor/helix/pull/6327))
+- Update `catppuccin` variants ([#5404](https://github.com/helix-editor/helix/pull/5404), [#6107](https://github.com/helix-editor/helix/pull/6107), [#6269](https://github.com/helix-editor/helix/pull/6269), [#6464](https://github.com/helix-editor/helix/pull/6464))
+- Use curly underlines in built-in themes ([#5419](https://github.com/helix-editor/helix/pull/5419))
+- Update `zenburn` ([#5573](https://github.com/helix-editor/helix/pull/5573))
+- Rewrite `snazzy` ([#3971](https://github.com/helix-editor/helix/pull/3971))
+- Add `monokai_aqua` ([#5578](https://github.com/helix-editor/helix/pull/5578))
+- Add `markup.strikethrough` to existing themes ([#5619](https://github.com/helix-editor/helix/pull/5619))
+- Update `sonokai` ([#5440](https://github.com/helix-editor/helix/pull/5440))
+- Update `onedark` ([#5755](https://github.com/helix-editor/helix/pull/5755))
+- Add `ayu_evolve` ([#5638](https://github.com/helix-editor/helix/pull/5638), [#6028](https://github.com/helix-editor/helix/pull/6028), [#6225](https://github.com/helix-editor/helix/pull/6225))
+- Add `jellybeans` ([#5719](https://github.com/helix-editor/helix/pull/5719))
+- Update `fleet_dark` ([#5605](https://github.com/helix-editor/helix/pull/5605), [#6266](https://github.com/helix-editor/helix/pull/6266), [#6324](https://github.com/helix-editor/helix/pull/6324), [#6375](https://github.com/helix-editor/helix/pull/6375))
+- Add `darcula-solid` ([#5778](https://github.com/helix-editor/helix/pull/5778))
+- Remove text background from monokai themes ([#6009](https://github.com/helix-editor/helix/pull/6009))
+- Update `pop_dark` ([#5992](https://github.com/helix-editor/helix/pull/5992), [#6208](https://github.com/helix-editor/helix/pull/6208), [#6227](https://github.com/helix-editor/helix/pull/6227), [#6292](https://github.com/helix-editor/helix/pull/6292))
+- Add `everblush` ([#6086](https://github.com/helix-editor/helix/pull/6086))
+- Add `adwaita-dark` ([#6042](https://github.com/helix-editor/helix/pull/6042), [#6342](https://github.com/helix-editor/helix/pull/6342))
+- Update `papercolor` ([#6162](https://github.com/helix-editor/helix/pull/6162))
+- Update `onelight` ([#6192](https://github.com/helix-editor/helix/pull/6192), [#6276](https://github.com/helix-editor/helix/pull/6276))
+- Add `molokai` ([#6260](https://github.com/helix-editor/helix/pull/6260))
+- Update `ayu` variants ([#6329](https://github.com/helix-editor/helix/pull/6329))
+- Update `tokyonight` variants ([#6349](https://github.com/helix-editor/helix/pull/6349))
+- Update `nord` variants ([#6376](https://github.com/helix-editor/helix/pull/6376))
+
+New languages:
+
+- BibTeX ([#5064](https://github.com/helix-editor/helix/pull/5064))
+- Mermaid.js ([#5147](https://github.com/helix-editor/helix/pull/5147))
+- Crystal ([#4993](https://github.com/helix-editor/helix/pull/4993), [#5205](https://github.com/helix-editor/helix/pull/5205))
+- MATLAB/Octave ([#5192](https://github.com/helix-editor/helix/pull/5192))
+- `tfvars` (uses HCL) ([#5396](https://github.com/helix-editor/helix/pull/5396))
+- Ponylang ([#5416](https://github.com/helix-editor/helix/pull/5416))
+- DHall ([1f6809c](https://github.com/helix-editor/helix/commit/1f6809c))
+- Sagemath ([#5649](https://github.com/helix-editor/helix/pull/5649))
+- MSBuild ([#5793](https://github.com/helix-editor/helix/pull/5793))
+- pem ([#5797](https://github.com/helix-editor/helix/pull/5797))
+- passwd ([#4959](https://github.com/helix-editor/helix/pull/4959))
+- hosts ([#4950](https://github.com/helix-editor/helix/pull/4950), [#5914](https://github.com/helix-editor/helix/pull/5914))
+- uxntal ([#6047](https://github.com/helix-editor/helix/pull/6047))
+- Yuck ([#6064](https://github.com/helix-editor/helix/pull/6064), [#6242](https://github.com/helix-editor/helix/pull/6242))
+- GNU gettext PO ([#5996](https://github.com/helix-editor/helix/pull/5996))
+- Sway ([#6023](https://github.com/helix-editor/helix/pull/6023))
+- NASM ([#6068](https://github.com/helix-editor/helix/pull/6068))
+- PRQL ([#6126](https://github.com/helix-editor/helix/pull/6126))
+- reStructuredText ([#6180](https://github.com/helix-editor/helix/pull/6180))
+- Smithy ([#6370](https://github.com/helix-editor/helix/pull/6370))
+- VHDL ([#5826](https://github.com/helix-editor/helix/pull/5826))
+- Rego (OpenPolicy Agent) ([#6415](https://github.com/helix-editor/helix/pull/6415))
+- Nim ([#6123](https://github.com/helix-editor/helix/pull/6123))
+
+Updated languages and queries:
+
+- Use diff syntax for patch files ([#5085](https://github.com/helix-editor/helix/pull/5085))
+- Add Haskell textobjects ([#5061](https://github.com/helix-editor/helix/pull/5061))
+- Fix commonlisp configuration ([#5091](https://github.com/helix-editor/helix/pull/5091))
+- Update Scheme ([bae890d](https://github.com/helix-editor/helix/commit/bae890d))
+- Add indent queries for Bash ([#5149](https://github.com/helix-editor/helix/pull/5149))
+- Recognize `c++` as a C++ extension ([#5183](https://github.com/helix-editor/helix/pull/5183))
+- Enable HTTP server in `metals` (Scala) config ([#5551](https://github.com/helix-editor/helix/pull/5551))
+- Change V-lang language server to `v ls` from `vls` ([#5677](https://github.com/helix-editor/helix/pull/5677))
+- Inject comment grammar into Nix ([#5208](https://github.com/helix-editor/helix/pull/5208))
+- Update Rust highlights ([#5238](https://github.com/helix-editor/helix/pull/5238), [#5349](https://github.com/helix-editor/helix/pull/5349))
+- Fix HTML injection within Markdown ([#5265](https://github.com/helix-editor/helix/pull/5265))
+- Fix comment token for godot ([#5276](https://github.com/helix-editor/helix/pull/5276))
+- Expand injections for Vue ([#5268](https://github.com/helix-editor/helix/pull/5268))
+- Add `.bash_aliases` as a Bash file-type ([#5347](https://github.com/helix-editor/helix/pull/5347))
+- Fix comment token for sshclientconfig ([#5351](https://github.com/helix-editor/helix/pull/5351))
+- Update Prisma ([#5417](https://github.com/helix-editor/helix/pull/5417))
+- Update C++ ([#5457](https://github.com/helix-editor/helix/pull/5457))
+- Add more file-types for Python ([#5593](https://github.com/helix-editor/helix/pull/5593))
+- Update tree-sitter-scala ([#5576](https://github.com/helix-editor/helix/pull/5576))
+- Add an injection regex for Lua ([#5606](https://github.com/helix-editor/helix/pull/5606))
+- Add `build.gradle` to java roots configuration ([#5641](https://github.com/helix-editor/helix/pull/5641))
+- Add Hub PR files to markdown file-types ([#5634](https://github.com/helix-editor/helix/pull/5634))
+- Add an external formatter configuration for Cue ([#5679](https://github.com/helix-editor/helix/pull/5679))
+- Add injections for builders and writers to Nix ([#5629](https://github.com/helix-editor/helix/pull/5629))
+- Update tree-sitter-xml to fix whitespace parsing ([#5685](https://github.com/helix-editor/helix/pull/5685))
+- Add `Justfile` to the make file-types configuration ([#5687](https://github.com/helix-editor/helix/pull/5687))
+- Update tree-sitter-sql and highlight queries ([#5683](https://github.com/helix-editor/helix/pull/5683), [#5772](https://github.com/helix-editor/helix/pull/5772))
+- Use the bash grammar and queries for env language ([#5720](https://github.com/helix-editor/helix/pull/5720))
+- Add podspec files to ruby file-types ([#5811](https://github.com/helix-editor/helix/pull/5811))
+- Recognize `.C` and `.H` file-types as C++ ([#5808](https://github.com/helix-editor/helix/pull/5808))
+- Recognize plist and mobileconfig files as XML ([#5863](https://github.com/helix-editor/helix/pull/5863))
+- Fix `select` indentation in Go ([#5713](https://github.com/helix-editor/helix/pull/5713))
+- Check for external file modifications when writing ([#5805](https://github.com/helix-editor/helix/pull/5805))
+- Recognize containerfiles as dockerfile syntax ([#5873](https://github.com/helix-editor/helix/pull/5873))
+- Update godot grammar and queries ([#5944](https://github.com/helix-editor/helix/pull/5944), [#6186](https://github.com/helix-editor/helix/pull/6186))
+- Improve DHall highlights ([#5959](https://github.com/helix-editor/helix/pull/5959))
+- Recognize `.env.dist` and `source.env` as env language ([#6003](https://github.com/helix-editor/helix/pull/6003))
+- Update tree-sitter-git-rebase ([#6030](https://github.com/helix-editor/helix/pull/6030), [#6094](https://github.com/helix-editor/helix/pull/6094))
+- Improve SQL highlights ([#6041](https://github.com/helix-editor/helix/pull/6041))
+- Improve markdown highlights and inject LaTeX ([#6100](https://github.com/helix-editor/helix/pull/6100))
+- Add textobject queries for Elm ([#6084](https://github.com/helix-editor/helix/pull/6084))
+- Recognize graphql schema file type ([#6159](https://github.com/helix-editor/helix/pull/6159))
+- Improve highlighting in comments ([#6143](https://github.com/helix-editor/helix/pull/6143))
+- Improve highlighting for JavaScript/TypeScript/ECMAScript languages ([#6205](https://github.com/helix-editor/helix/pull/6205))
+- Improve PHP highlights ([#6203](https://github.com/helix-editor/helix/pull/6203), [#6250](https://github.com/helix-editor/helix/pull/6250), [#6299](https://github.com/helix-editor/helix/pull/6299))
+- Improve Go highlights ([#6204](https://github.com/helix-editor/helix/pull/6204))
+- Highlight unchecked sqlx functions as SQL in Rust ([#6256](https://github.com/helix-editor/helix/pull/6256))
+- Improve Erlang highlights ([cdd6c8d](https://github.com/helix-editor/helix/commit/cdd6c8d))
+- Improve Nix highlights ([fb4d703](https://github.com/helix-editor/helix/commit/fb4d703))
+- Improve gdscript highlights ([#6311](https://github.com/helix-editor/helix/pull/6311))
+- Improve Vlang highlights ([#6279](https://github.com/helix-editor/helix/pull/6279))
+- Improve Makefile highlights ([#6339](https://github.com/helix-editor/helix/pull/6339))
+- Remove auto-pair for `'` in OCaml ([#6381](https://github.com/helix-editor/helix/pull/6381))
+- Fix indents in switch statements in ECMA languages ([#6369](https://github.com/helix-editor/helix/pull/6369))
+- Recognize xlb and storyboard file-types as XML ([#6407](https://github.com/helix-editor/helix/pull/6407))
+- Recognize cts and mts file-types as TypeScript ([#6424](https://github.com/helix-editor/helix/pull/6424))
+- Recognize SVG file-type as XML ([#6431](https://github.com/helix-editor/helix/pull/6431))
+- Add theme scopes for (un)checked list item markup scopes ([#6434](https://github.com/helix-editor/helix/pull/6434))
+- Update git commit grammar and add the comment textobject ([#6439](https://github.com/helix-editor/helix/pull/6439), [#6493](https://github.com/helix-editor/helix/pull/6493))
+- Recognize ARB file-type as JSON ([#6452](https://github.com/helix-editor/helix/pull/6452))
+- Inject markdown into markdown strings in Julia ([#6489](https://github.com/helix-editor/helix/pull/6489))
+
+Packaging:
+
+- Fix Nix flake devShell for darwin hosts ([#5368](https://github.com/helix-editor/helix/pull/5368))
+- Add Appstream metadata file to `contrib/` ([#5643](https://github.com/helix-editor/helix/pull/5643))
+- Increase the MSRV to 1.65 ([#5570](https://github.com/helix-editor/helix/pull/5570), [#6185](https://github.com/helix-editor/helix/pull/6185))
+- Expose the Nix flake's `wrapper` ([#5994](https://github.com/helix-editor/helix/pull/5994))
+
 # 22.12 (2022-12-06)
 
 This is a great big release filled with changes from a 99 contributors. A big _thank you_ to you all!

Cargo.toml

@@ -8,6 +8,7 @@ members = [
   "helix-dap",
   "helix-loader",
   "helix-vcs",
+  "helix-parsec",
   "xtask",
 ]
 
@@ -25,3 +26,9 @@ lto = "fat"
 codegen-units = 1
 # strip = "debuginfo" # TODO: or strip = true
 opt-level = 3
+
+[profile.integration]
+inherits = "test"
+package.helix-core.opt-level = 2
+package.helix-tui.opt-level = 2
+package.helix-term.opt-level = 2

README.md

@@ -45,92 +45,10 @@ Note: Only certain languages have indentation definitions at the moment. Check
 
 # Installation
 
-Packages are available for various distributions (see [Installation docs](https://docs.helix-editor.com/install.html)).
-
-If you would like to build from source:
-
-```shell
-git clone https://github.com/helix-editor/helix
-cd helix
-cargo install --path helix-term
-```
-
-This will install the `hx` binary to `$HOME/.cargo/bin` and build tree-sitter grammars in `./runtime/grammars`.
-
-Helix needs its runtime files so make sure to copy/symlink the `runtime/` directory into the
-config directory (for example `~/.config/helix/runtime` on Linux/macOS, or `%AppData%/helix/runtime` on Windows).
-
-| OS                   | Command                                          |
-| -------------------- | ------------------------------------------------ |
-| Windows (Cmd)        | `xcopy /e /i runtime %AppData%\helix\runtime`    |
-| Windows (PowerShell) | `xcopy /e /i runtime $Env:AppData\helix\runtime` |
-| Linux / macOS        | `ln -s $PWD/runtime ~/.config/helix/runtime`     |
-
-Starting with Windows Vista you can also create symbolic links on Windows. Note that this requires
-elevated privileges - i.e. PowerShell or Cmd must be run as administrator.
-
-**PowerShell:**
-
-```powershell
-New-Item -ItemType SymbolicLink -Target "runtime" -Path "$Env:AppData\helix\runtime"
-```
-
-**Cmd:**
-
-```cmd
-cd %appdata%\helix
-mklink /D runtime "<helix-repo>\runtime"
-```
-
-The runtime location can be overridden via the `HELIX_RUNTIME` environment variable.
-
-> NOTE: if `HELIX_RUNTIME` is set prior to calling `cargo install --path helix-term`,
-> tree-sitter grammars will be built in `$HELIX_RUNTIME/grammars`.
-
-If you plan on keeping the repo locally, an alternative to copying/symlinking
-runtime files is to set `HELIX_RUNTIME=/path/to/helix/runtime`
-(`HELIX_RUNTIME=$PWD/runtime` if you're in the helix repo directory).
-
-Packages already solve this for you by wrapping the `hx` binary with a wrapper
-that sets the variable to the install dir.
-
-> NOTE: running via cargo also doesn't require setting explicit `HELIX_RUNTIME` path, it will automatically
-> detect the `runtime` directory in the project root.
-
-If you want to customize your `languages.toml` config,
-tree-sitter grammars may be manually fetched and built with `hx --grammar fetch` and `hx --grammar build`.
-
-In order to use LSP features like auto-complete, you will need to
-[install the appropriate Language Server](https://github.com/helix-editor/helix/wiki/How-to-install-the-default-language-servers)
-for a language.
+[Installation documentation](https://docs.helix-editor.com/install.html).
 
 [![Packaging status](https://repology.org/badge/vertical-allrepos/helix.svg)](https://repology.org/project/helix/versions)
 
-## Adding Helix to your desktop environment
-
-If installing from source, to use Helix in desktop environments that supports [XDG desktop menu](https://specifications.freedesktop.org/menu-spec/menu-spec-latest.html), including Gnome and KDE, copy the provided `.desktop` file to the correct folder:
-
-```bash
-cp contrib/Helix.desktop ~/.local/share/applications
-```
-
-To use another terminal than the default, you will need to modify the `.desktop` file. For example, to use `kitty`:
-
-```bash
-sed -i "s|Exec=hx %F|Exec=kitty hx %F|g" ~/.local/share/applications/Helix.desktop
-sed -i "s|Terminal=true|Terminal=false|g" ~/.local/share/applications/Helix.desktop
-```
-
-Please note: there is no icon for Helix yet, so the system default will be used.
-
-## macOS
-
-Helix can be installed on macOS through homebrew:
-
-```
-brew install helix
-```
-
 # Contributing
 
 Contributing guidelines can be found [here](./docs/CONTRIBUTING.md).

VERSION

@@ -1 +1 @@
-22.12
+23.05

base16_theme.toml

@@ -7,6 +7,7 @@
 "ui.linenr.selected" = { fg = "white", bg = "black", modifiers = ["bold"] }
 "ui.selection" = { fg = "black", bg = "blue" }
 "ui.selection.primary" = { fg = "white", bg = "blue" }
+"ui.text.inactive" = { fg = "gray" }
 "comment" = { fg = "gray" }
 "ui.statusline" = { fg = "black", bg = "white" }
 "ui.statusline.inactive" = { fg = "gray", bg = "white" }

book/src/SUMMARY.md

@@ -6,13 +6,14 @@
 - [Usage](./usage.md)
   - [Keymap](./keymap.md)
   - [Commands](./commands.md)
-  - [Language Support](./lang-support.md)
+  - [Language support](./lang-support.md)
 - [Migrating from Vim](./from-vim.md)
 - [Configuration](./configuration.md)
   - [Themes](./themes.md)
-  - [Key Remapping](./remapping.md)
+  - [Key remapping](./remapping.md)
   - [Languages](./languages.md)
 - [Guides](./guides/README.md)
-  - [Adding Languages](./guides/adding_languages.md)
-  - [Adding Textobject Queries](./guides/textobject.md)
-  - [Adding Indent Queries](./guides/indent.md)
+  - [Adding languages](./guides/adding_languages.md)
+  - [Adding textobject queries](./guides/textobject.md)
+  - [Adding indent queries](./guides/indent.md)
+  - [Adding injection queries](./guides/injection.md)

book/src/commands.md

@@ -1,5 +1,5 @@
 # Commands
 
-Command mode can be activated by pressing `:`, similar to Vim. Built-in commands:
+Command mode can be activated by pressing `:`. The built-in commands are:
 
 {{#include ./generated/typable-cmd.md}}

book/src/configuration.md

@@ -2,10 +2,10 @@
 
 To override global configuration parameters, create a `config.toml` file located in your config directory:
 
-* Linux and Mac: `~/.config/helix/config.toml`
-* Windows: `%AppData%\helix\config.toml`
+- Linux and Mac: `~/.config/helix/config.toml`
+- Windows: `%AppData%\helix\config.toml`
 
-> Hint: You can easily open the config file by typing `:config-open` within Helix normal mode.
+> 💡 You can easily open the config file by typing `:config-open` within Helix normal mode.
 
 Example config:
 
@@ -25,12 +25,13 @@ select = "underline"
 hidden = false
 ```
 
-You may also specify a file to use for configuration with the `-c` or
-`--config` CLI argument: `hx -c path/to/custom-config.toml`.
+You can use a custom configuration file by specifying it with the `-c` or
+`--config` command line argument, for example `hx -c path/to/custom-config.toml`.
+Additionally, you can reload the configuration file by sending the USR1
+signal to the Helix process on Unix operating systems, such as by using the command `pkill -USR1 hx`.
 
-It is also possible to trigger configuration file reloading by sending the `USR1`
-signal to the helix process, e.g. via `pkill -USR1 hx`. This is only supported 
-on unix operating systems.
+Finally, you can have a `config.toml` local to a project by putting it under a `.helix` directory in your repository.
+Its settings will be merged with the configuration directory `config.toml` and the built-in configuration.
 
 ## Editor
 
@@ -38,25 +39,29 @@ on unix operating systems.
 
 | Key | Description | Default |
 |--|--|---------|
-| `scrolloff` | Number of lines of padding around the edge of the screen when scrolling. | `5` |
-| `mouse` | Enable mouse mode. | `true` |
-| `middle-click-paste` | Middle click paste support. | `true` |
-| `scroll-lines` | Number of lines to scroll per scroll wheel step. | `3` |
-| `shell` | Shell to use when running external commands. | Unix: `["sh", "-c"]`<br/>Windows: `["cmd", "/C"]` |
-| `line-number` | Line number display: `absolute` simply shows each line's number, while `relative` shows the distance from the current line. When unfocused or in insert mode, `relative` will still show absolute line numbers. | `absolute` |
-| `cursorline` | Highlight all lines with a cursor. | `false` |
-| `cursorcolumn` | Highlight all columns with a cursor. | `false` |
+| `scrolloff` | Number of lines of padding around the edge of the screen when scrolling | `5` |
+| `mouse` | Enable mouse mode | `true` |
+| `middle-click-paste` | Middle click paste support | `true` |
+| `scroll-lines` | Number of lines to scroll per scroll wheel step | `3` |
+| `shell` | Shell to use when running external commands | Unix: `["sh", "-c"]`<br/>Windows: `["cmd", "/C"]` |
+| `line-number` | Line number display: `absolute` simply shows each line's number, while `relative` shows the distance from the current line. When unfocused or in insert mode, `relative` will still show absolute line numbers | `absolute` |
+| `cursorline` | Highlight all lines with a cursor | `false` |
+| `cursorcolumn` | Highlight all columns with a cursor | `false` |
 | `gutters` | Gutters to display: Available are `diagnostics` and `diff` and `line-numbers` and `spacer`, note that `diagnostics` also includes other features like breakpoints, 1-width padding will be inserted if gutters is non-empty | `["diagnostics", "spacer", "line-numbers", "spacer", "diff"]` |
-| `auto-completion` | Enable automatic pop up of auto-completion. | `true` |
-| `auto-format` | Enable automatic formatting on save. | `true` |
-| `auto-save` | Enable automatic saving on focus moving away from Helix. Requires [focus event support](https://github.com/helix-editor/helix/wiki/Terminal-Support) from your terminal. | `false` |
-| `idle-timeout` | Time in milliseconds since last keypress before idle timers trigger. Used for autocompletion, set to 0 for instant. | `400` |
+| `auto-completion` | Enable automatic pop up of auto-completion | `true` |
+| `auto-format` | Enable automatic formatting on save | `true` |
+| `auto-save` | Enable automatic saving on the focus moving away from Helix. Requires [focus event support](https://github.com/helix-editor/helix/wiki/Terminal-Support) from your terminal | `false` |
+| `idle-timeout` | Time in milliseconds since last keypress before idle timers trigger. Used for autocompletion, set to 0 for instant | `400` |
 | `completion-trigger-len` | The min-length of word under cursor to trigger autocompletion | `2` |
-| `auto-info` | Whether to display infoboxes | `true` |
-| `true-color` | Set to `true` to override automatic detection of terminal truecolor support in the event of a false negative. | `false` |
-| `rulers` | List of column positions at which to display the rulers. Can be overridden by language specific `rulers` in `languages.toml` file. | `[]` |
+| `completion-replace` | Set to `true` to make completions always replace the entire word and not just the part before the cursor | `false` |
+| `auto-info` | Whether to display info boxes | `true` |
+| `true-color` | Set to `true` to override automatic detection of terminal truecolor support in the event of a false negative | `false` |
+| `undercurl` | Set to `true` to override automatic detection of terminal undercurl support in the event of a false negative | `false` |
+| `rulers` | List of column positions at which to display the rulers. Can be overridden by language specific `rulers` in `languages.toml` file | `[]` |
 | `bufferline` | Renders a line at the top of the editor displaying open buffers. Can be `always`, `never` or `multiple` (only shown if more than one buffer is in use) | `never` |
 | `color-modes` | Whether to color the mode indicator with different colors depending on the mode itself | `false` |
+| `text-width` | Maximum line length. Used for the `:reflow` command and soft-wrapping if `soft-wrap.wrap-at-text-width` is set | `80` |
+| `workspace-lsp-roots` | Directories relative to the workspace root that are treated as LSP roots. Should only be set in `.helix/config.toml` | `[]` |
 
 ### `[editor.statusline]` Section
 
@@ -97,6 +102,8 @@ The following statusline elements can be configured:
 | `mode` | The current editor mode (`mode.normal`/`mode.insert`/`mode.select`) |
 | `spinner` | A progress spinner indicating LSP activity |
 | `file-name` | The path/name of the opened file |
+| `file-base-name` | The basename of the opened file |
+| `file-modification-indicator` | The indicator to show whether the file is modified (a `[+]` appears when there are unsaved changes) |
 | `file-encoding` | The encoding of the opened file if it differs from UTF-8 |
 | `file-line-ending` | The file line endings (CRLF or LF) |
 | `total-line-numbers` | The total line numbers of the opened file |
@@ -109,23 +116,32 @@ The following statusline elements can be configured:
 | `position-percentage` | The cursor position as a percentage of the total number of lines |
 | `separator` | The string defined in `editor.statusline.separator` (defaults to `"│"`) |
 | `spacer` | Inserts a space between elements (multiple/contiguous spacers may be specified) |
+| `version-control` | The current branch name or detached commit hash of the opened workspace |
 
 ### `[editor.lsp]` Section
 
 | Key                   | Description                                                 | Default |
 | ---                   | -----------                                                 | ------- |
+| `enable`              | Enables LSP integration. Setting to false will completely disable language servers regardless of language settings.| `true` |
 | `display-messages`    | Display LSP progress messages below statusline[^1]          | `false` |
 | `auto-signature-help` | Enable automatic popup of signature help (parameter hints)  | `true`  |
+| `display-inlay-hints` | Display inlay hints[^2]                                     | `false` |
 | `display-signature-help-docs` | Display docs under signature help popup             | `true`  |
+| `snippets`      | Enables snippet completions. Requires a server restart (`:lsp-restart`) to take effect after `:config-reload`/`:set`. | `true`  |
+| `goto-reference-include-declaration` | Include declaration in the goto references popup. | `true`  |
 
 [^1]: By default, a progress spinner is shown in the statusline beside the file path.
+[^2]: You may also have to activate them in the LSP config for them to appear, not just in Helix.
+      Inlay hints in Helix are still being improved on and may be a little bit laggy/janky under some circumstances, please report any bugs you see so we can fix them!
 
 ### `[editor.cursor-shape]` Section
 
-Defines the shape of cursor in each mode. Note that due to limitations
-of the terminal environment, only the primary cursor can change shape.
+Defines the shape of cursor in each mode.
 Valid values for these options are `block`, `bar`, `underline`, or `hidden`.
 
+> 💡 Due to limitations of the terminal environment, only the primary cursor can
+> change shape.
+
 | Key      | Description                                | Default |
 | ---      | -----------                                | ------- |
 | `normal` | Cursor shape in [normal mode][normal mode] | `block` |
@@ -138,23 +154,22 @@ Valid values for these options are `block`, `bar`, `underline`, or `hidden`.
 
 ### `[editor.file-picker]` Section
 
-Sets options for file picker and global search. All but the last key listed in
-the default file-picker configuration below are IgnoreOptions: whether hidden
-files and files listed within ignore files are ignored by (not visible in) the
-helix file picker and global search. There is also one other key, `max-depth`
-available, which is not defined by default.
+Set options for file picker and global search. Ignoring a file means it is
+not visible in the Helix file picker and global search.
 
 All git related options are only enabled in a git repository.
 
 | Key | Description | Default |
 |--|--|---------|
-|`hidden` | Enables ignoring hidden files. | true
-|`parents` | Enables reading ignore files from parent directories. | true
-|`ignore` | Enables reading `.ignore` files. | true
-|`git-ignore` | Enables reading `.gitignore` files. | true
-|`git-global` | Enables reading global .gitignore, whose path is specified in git's config: `core.excludefile` option. | true
-|`git-exclude` | Enables reading `.git/info/exclude` files. | true
-|`max-depth` | Set with an integer value for maximum depth to recurse. | Defaults to `None`.
+|`hidden` | Enables ignoring hidden files | true
+|`follow-symlinks` | Follow symlinks instead of ignoring them | true
+|`deduplicate-links` | Ignore symlinks that point at files already shown in the picker | true
+|`parents` | Enables reading ignore files from parent directories | true
+|`ignore` | Enables reading `.ignore` files | true
+|`git-ignore` | Enables reading `.gitignore` files | true
+|`git-global` | Enables reading global `.gitignore`, whose path is specified in git's config: `core.excludefile` option | true
+|`git-exclude` | Enables reading `.git/info/exclude` files | true
+|`max-depth` | Set with an integer value for maximum depth to recurse | Defaults to `None`.
 
 ### `[editor.auto-pairs]` Section
 
@@ -206,7 +221,7 @@ Search specific options.
 
 | Key | Description | Default |
 |--|--|---------|
-| `smart-case` | Enable smart case regex searching (case insensitive unless pattern contains upper case characters) | `true` |
+| `smart-case` | Enable smart case regex searching (case-insensitive unless pattern contains upper case characters) | `true` |
 | `wrap-around`| Whether the search should wrap after depleting the matches | `true` |
 
 ### `[editor.whitespace]` Section
@@ -215,7 +230,7 @@ Options for rendering whitespace with visible characters. Use `:set whitespace.r
 
 | Key | Description | Default |
 |-----|-------------|---------|
-| `render` | Whether to render whitespace. May either be `"all"` or `"none"`, or a table with sub-keys `space`, `tab`, and `newline`. | `"none"` |
+| `render` | Whether to render whitespace. May either be `"all"` or `"none"`, or a table with sub-keys `space`, `nbsp`, `tab`, and `newline` | `"none"` |
 | `characters` | Literal characters to use when rendering whitespace. Sub-keys may be any of `tab`, `space`, `nbsp`, `newline` or `tabpad` | See example below |
 
 Example
@@ -243,7 +258,7 @@ Options for rendering vertical indent guides.
 
 | Key           | Description                                             | Default |
 | ---           | ---                                                     | ---     |
-| `render`      | Whether to render indent guides.                        | `false` |
+| `render`      | Whether to render indent guides                         | `false` |
 | `character`   | Literal character to use for rendering the indent guide | `│`     |
 | `skip-levels` | Number of indent levels to skip                         | `0`     |
 
@@ -255,3 +270,77 @@ render = true
 character = "╎" # Some characters that work well: "▏", "┆", "┊", "⸽"
 skip-levels = 1
 ```
+
+### `[editor.gutters]` Section
+
+For simplicity, `editor.gutters` accepts an array of gutter types, which will
+use default settings for all gutter components.
+
+```toml
+[editor]
+gutters = ["diff", "diagnostics", "line-numbers", "spacer"]
+```
+
+To customize the behavior of gutters, the `[editor.gutters]` section must
+be used. This section contains top level settings, as well as settings for
+specific gutter components as subsections.
+
+| Key      | Description                    | Default                                                       |
+| ---      | ---                            | ---                                                           |
+| `layout` | A vector of gutters to display | `["diagnostics", "spacer", "line-numbers", "spacer", "diff"]` |
+
+Example:
+
+```toml
+[editor.gutters]
+layout = ["diff", "diagnostics", "line-numbers", "spacer"]
+```
+
+#### `[editor.gutters.line-numbers]` Section
+
+Options for the line number gutter
+
+| Key         | Description                             | Default |
+| ---         | ---                                     | ---     |
+| `min-width` | The minimum number of characters to use | `3`     |
+
+Example:
+
+```toml
+[editor.gutters.line-numbers]
+min-width = 1
+```
+
+#### `[editor.gutters.diagnostics]` Section
+
+Currently unused
+
+#### `[editor.gutters.diff]` Section
+
+Currently unused
+
+#### `[editor.gutters.spacer]` Section
+
+Currently unused
+
+### `[editor.soft-wrap]` Section
+
+Options for soft wrapping lines that exceed the view width:
+
+| Key                  | Description                                                  | Default |
+| ---                  | ---                                                          | ---     |
+| `enable`             | Whether soft wrapping is enabled.                            | `false` |
+| `max-wrap`           | Maximum free space left at the end of the line.              | `20`    |
+| `max-indent-retain`  | Maximum indentation to carry over when soft wrapping a line. | `40`    |
+| `wrap-indicator`     | Text inserted before soft wrapped lines, highlighted with `ui.virtual.wrap` | `↪ `    |
+| `wrap-at-text-width` | Soft wrap at `text-width` instead of using the full viewport size. | `false` |
+
+Example:
+
+```toml
+[editor.soft-wrap]
+enable = true
+max-wrap = 25 # increase value to reduce forced mid-word wrapping
+max-indent-retain = 0
+wrap-indicator = ""  # set wrap-indicator to "" to hide it
+```

book/src/from-vim.md

@@ -2,9 +2,9 @@
 
 Helix's editing model is strongly inspired from Vim and Kakoune, and a notable
 difference from Vim (and the most striking similarity to Kakoune) is that Helix
-follows the `selection → action` model. This means that the whatever you are
-going to act on (a word, a paragraph, a line, etc) is selected first and the
-action itself (delete, change, yank, etc) comes second. A cursor is simply a
+follows the `selection → action` model. This means that whatever you are
+going to act on (a word, a paragraph, a line, etc.) is selected first and the
+action itself (delete, change, yank, etc.) comes second. A cursor is simply a
 single width selection.
 
 See also Kakoune's [Migrating from Vim](https://github.com/mawww/kakoune/wiki/Migrating-from-Vim) and Helix's [Migrating from Vim](https://github.com/helix-editor/helix/wiki/Migrating-from-Vim).

book/src/generated/lang-support.md

@@ -2,33 +2,38 @@
 | --- | --- | --- | --- | --- |
 | astro | ✓ |  |  |  |
 | awk | ✓ | ✓ |  | `awk-language-server` |
-| bash | ✓ |  |  | `bash-language-server` |
+| bash | ✓ |  | ✓ | `bash-language-server` |
 | bass | ✓ |  |  | `bass` |
 | beancount | ✓ |  |  |  |
 | bibtex | ✓ |  |  | `texlab` |
 | bicep | ✓ |  |  | `bicep-langserver` |
 | c | ✓ | ✓ | ✓ | `clangd` |
 | c-sharp | ✓ | ✓ |  | `OmniSharp` |
+| cabal |  |  |  |  |
 | cairo | ✓ |  |  |  |
+| capnp | ✓ |  | ✓ |  |
 | clojure | ✓ |  |  | `clojure-lsp` |
 | cmake | ✓ | ✓ | ✓ | `cmake-language-server` |
 | comment | ✓ |  |  |  |
 | common-lisp | ✓ |  |  | `cl-lsp` |
 | cpon | ✓ |  | ✓ |  |
 | cpp | ✓ | ✓ | ✓ | `clangd` |
+| crystal | ✓ | ✓ |  | `crystalline` |
 | css | ✓ |  |  | `vscode-css-language-server` |
 | cue | ✓ |  |  | `cuelsp` |
 | d | ✓ | ✓ | ✓ | `serve-d` |
 | dart | ✓ |  | ✓ | `dart` |
 | devicetree | ✓ |  |  |  |
+| dhall | ✓ | ✓ |  | `dhall-lsp-server` |
 | diff | ✓ |  |  |  |
 | dockerfile | ✓ |  |  | `docker-langserver` |
 | dot | ✓ |  |  | `dot-language-server` |
+| dtd | ✓ |  |  |  |
 | edoc | ✓ |  |  |  |
 | eex | ✓ |  |  |  |
 | ejs | ✓ |  |  |  |
 | elixir | ✓ | ✓ | ✓ | `elixir-ls` |
-| elm | ✓ |  |  | `elm-language-server` |
+| elm | ✓ | ✓ |  | `elm-language-server` |
 | elvish | ✓ |  |  | `elvish` |
 | env | ✓ |  |  |  |
 | erb | ✓ |  |  |  |
@@ -36,9 +41,9 @@
 | esdl | ✓ |  |  |  |
 | fish | ✓ | ✓ | ✓ |  |
 | fortran | ✓ |  | ✓ | `fortls` |
-| gdscript | ✓ | ✓ |  |  |
+| gdscript | ✓ | ✓ | ✓ |  |
 | git-attributes | ✓ |  |  |  |
-| git-commit | ✓ |  |  |  |
+| git-commit | ✓ | ✓ |  |  |
 | git-config | ✓ |  |  |  |
 | git-ignore | ✓ |  |  |  |
 | git-rebase | ✓ |  |  |  |
@@ -54,7 +59,9 @@
 | haskell | ✓ | ✓ |  | `haskell-language-server-wrapper` |
 | hcl | ✓ |  | ✓ | `terraform-ls` |
 | heex | ✓ | ✓ |  | `elixir-ls` |
+| hosts | ✓ |  |  |  |
 | html | ✓ |  |  | `vscode-html-language-server` |
+| hurl | ✓ |  | ✓ |  |
 | idris |  |  |  | `idris2-lsp` |
 | iex | ✓ |  |  |  |
 | ini | ✓ |  |  |  |
@@ -64,7 +71,8 @@
 | json | ✓ |  | ✓ | `vscode-json-language-server` |
 | jsonnet | ✓ |  |  | `jsonnet-language-server` |
 | jsx | ✓ | ✓ | ✓ | `typescript-language-server` |
-| julia | ✓ |  |  | `julia` |
+| julia | ✓ | ✓ | ✓ | `julia` |
+| just | ✓ | ✓ | ✓ |  |
 | kdl | ✓ |  |  |  |
 | kotlin | ✓ |  |  | `kotlin-language-server` |
 | latex | ✓ | ✓ |  | `texlab` |
@@ -75,58 +83,78 @@
 | llvm-mir-yaml | ✓ |  | ✓ |  |
 | lua | ✓ | ✓ | ✓ | `lua-language-server` |
 | make | ✓ |  |  |  |
+| markdoc | ✓ |  |  | `markdoc-ls` |
 | markdown | ✓ |  |  | `marksman` |
 | markdown.inline | ✓ |  |  |  |
+| matlab | ✓ |  |  |  |
+| mermaid | ✓ |  |  |  |
 | meson | ✓ |  | ✓ |  |
 | mint |  |  |  | `mint` |
+| msbuild | ✓ |  | ✓ |  |
+| nasm | ✓ | ✓ |  |  |
 | nickel | ✓ |  | ✓ | `nls` |
+| nim | ✓ | ✓ | ✓ | `nimlangserver` |
 | nix | ✓ |  |  | `nil` |
 | nu | ✓ |  |  |  |
 | ocaml | ✓ |  | ✓ | `ocamllsp` |
 | ocaml-interface | ✓ |  |  | `ocamllsp` |
-| odin | ✓ |  |  | `ols` |
+| odin | ✓ |  | ✓ | `ols` |
+| opencl | ✓ | ✓ | ✓ | `clangd` |
 | openscad | ✓ |  |  | `openscad-lsp` |
 | org | ✓ |  |  |  |
 | pascal | ✓ | ✓ |  | `pasls` |
-| perl | ✓ | ✓ | ✓ |  |
+| passwd | ✓ |  |  |  |
+| pem | ✓ |  |  |  |
+| perl | ✓ | ✓ | ✓ | `perlnavigator` |
 | php | ✓ | ✓ | ✓ | `intelephense` |
+| po | ✓ | ✓ |  |  |
+| ponylang | ✓ | ✓ | ✓ |  |
 | prisma | ✓ |  |  | `prisma-language-server` |
 | prolog |  |  |  | `swipl` |
 | protobuf | ✓ |  | ✓ |  |
+| prql | ✓ |  |  |  |
 | purescript | ✓ |  |  | `purescript-language-server` |
 | python | ✓ | ✓ | ✓ | `pylsp` |
 | qml | ✓ |  | ✓ | `qmlls` |
 | r | ✓ |  |  | `R` |
 | racket | ✓ |  |  | `racket` |
 | regex | ✓ |  |  |  |
+| rego | ✓ |  |  | `regols` |
 | rescript | ✓ | ✓ |  | `rescript-language-server` |
 | rmarkdown | ✓ |  | ✓ | `R` |
+| robot | ✓ |  |  | `robotframework_ls` |
 | ron | ✓ |  | ✓ |  |
+| rst | ✓ |  |  |  |
 | ruby | ✓ | ✓ | ✓ | `solargraph` |
 | rust | ✓ | ✓ | ✓ | `rust-analyzer` |
+| sage | ✓ | ✓ |  |  |
 | scala | ✓ |  | ✓ | `metals` |
 | scheme | ✓ |  |  |  |
 | scss | ✓ |  |  | `vscode-css-language-server` |
 | slint | ✓ |  | ✓ | `slint-lsp` |
+| smithy | ✓ |  |  | `cs` |
 | sml | ✓ |  |  |  |
 | solidity | ✓ |  |  | `solc` |
 | sql | ✓ |  |  |  |
 | sshclientconfig | ✓ |  |  |  |
 | starlark | ✓ | ✓ |  |  |
 | svelte | ✓ |  |  | `svelteserver` |
+| sway | ✓ | ✓ | ✓ | `forc` |
 | swift | ✓ |  |  | `sourcekit-lsp` |
 | tablegen | ✓ | ✓ | ✓ |  |
 | task | ✓ |  |  |  |
-| tfvars |  |  |  | `terraform-ls` |
+| tfvars | ✓ |  | ✓ | `terraform-ls` |
 | toml | ✓ |  |  | `taplo` |
 | tsq | ✓ |  |  |  |
 | tsx | ✓ | ✓ | ✓ | `typescript-language-server` |
 | twig | ✓ |  |  |  |
 | typescript | ✓ | ✓ | ✓ | `typescript-language-server` |
 | ungrammar | ✓ |  |  |  |
-| v | ✓ |  |  | `vls` |
+| uxntal | ✓ |  |  |  |
+| v | ✓ | ✓ | ✓ | `v` |
 | vala | ✓ |  |  | `vala-language-server` |
 | verilog | ✓ | ✓ |  | `svlangserver` |
+| vhdl | ✓ |  |  | `vhdl_ls` |
 | vhs | ✓ |  |  |  |
 | vue | ✓ |  |  | `vls` |
 | wast | ✓ |  |  |  |
@@ -136,4 +164,5 @@
 | xit | ✓ |  |  |  |
 | xml | ✓ |  | ✓ |  |
 | yaml | ✓ |  | ✓ | `yaml-language-server` |
+| yuck | ✓ |  |  |  |
 | zig | ✓ | ✓ | ✓ | `zls` |

book/src/generated/typable-cmd.md

@@ -12,7 +12,9 @@
 | `:buffer-next`, `:bn`, `:bnext` | Goto next buffer. |
 | `:buffer-previous`, `:bp`, `:bprev` | Goto previous buffer. |
 | `:write`, `:w` | Write changes to disk. Accepts an optional path (:write some/path.txt) |
-| `:write!`, `:w!` | Force write changes to disk creating necessary subdirectories. Accepts an optional path (:write some/path.txt) |
+| `:write!`, `:w!` | Force write changes to disk creating necessary subdirectories. Accepts an optional path (:write! some/path.txt) |
+| `:write-buffer-close`, `:wbc` | Write changes to disk and closes the buffer. Accepts an optional path (:write-buffer-close some/path.txt) |
+| `:write-buffer-close!`, `:wbc!` | Force write changes to disk creating necessary subdirectories and closes the buffer. Accepts an optional path (:write-buffer-close! some/path.txt) |
 | `:new`, `:n` | Create a new scratch buffer. |
 | `:format`, `:fmt` | Format the file using the LSP formatter. |
 | `:indent-style` | Set the indentation style for editing. ('t' for tabs or 1-8 for number of spaces.) |
@@ -43,11 +45,13 @@
 | `:change-current-directory`, `:cd` | Change the current working directory. |
 | `:show-directory`, `:pwd` | Show the current working directory. |
 | `:encoding` | Set encoding. Based on `https://encoding.spec.whatwg.org`. |
+| `:character-info`, `:char` | Get info about the character under the primary cursor. |
 | `:reload` | Discard changes and reload from the source file. |
 | `:reload-all` | Discard changes and reload all documents from the source files. |
-| `:update` | Write changes only if the file has been modified. |
+| `:update`, `:u` | Write changes only if the file has been modified. |
 | `:lsp-workspace-command` | Open workspace command picker |
 | `:lsp-restart` | Restarts the Language Server that is in use by the current doc |
+| `:lsp-stop` | Stops the Language Server that is in use by the current doc |
 | `:tree-sitter-scopes` | Display tree sitter scopes, primarily for theming and development. |
 | `:debug-start`, `:dbg` | Start a debug session from a given template with given parameters. |
 | `:debug-remote`, `:dbg-tcp` | Connect to a debug adapter by TCP address and start a debugging session from a given template with given parameters. |
@@ -58,8 +62,9 @@
 | `:hsplit-new`, `:hnew` | Open a scratch buffer in a horizontal split. |
 | `:tutor` | Open the tutorial. |
 | `:goto`, `:g` | Goto line number. |
-| `:set-language`, `:lang` | Set the language of current buffer. |
+| `:set-language`, `:lang` | Set the language of current buffer (show current language if no value specified). |
 | `:set-option`, `:set` | Set a config option at runtime.<br>For example to disable smart case search, use `:set search.smart-case false`. |
+| `:toggle-option`, `:toggle` | Toggle a boolean config option at runtime.<br>For example to toggle smart case search, use `:toggle search.smart-case`. |
 | `:get-option`, `:get` | Get the current value of a config option. |
 | `:sort` | Sort ranges in selection. |
 | `:rsort` | Sort ranges in selection in reverse order. |
@@ -67,9 +72,12 @@
 | `:tree-sitter-subtree`, `:ts-subtree` | Display tree sitter subtree under cursor, primarily for debugging queries. |
 | `:config-reload` | Refresh user config. |
 | `:config-open` | Open the user config.toml file. |
+| `:config-open-workspace` | Open the workspace config.toml file. |
 | `:log-open` | Open the helix log file. |
 | `:insert-output` | Run shell command, inserting output before each selection. |
 | `:append-output` | Run shell command, appending output after each selection. |
 | `:pipe` | Pipe each selection to the shell command. |
 | `:pipe-to` | Pipe each selection to the shell command, ignoring output. |
 | `:run-shell-command`, `:sh` | Run a shell command |
+| `:reset-diff-change`, `:diffget`, `:diffg` | Reset the diff change at the cursor position. |
+| `:clear-register` | Clear given register. If no argument is provided, clear all registers. |

book/src/guides/README.md

@@ -1,4 +1,4 @@
 # Guides
 
 This section contains guides for adding new language server configurations,
-tree-sitter grammars, textobject queries, etc.
+tree-sitter grammars, textobject queries, and other similar items.

book/src/guides/adding_languages.md

@@ -1,45 +1,52 @@
-# Adding languages
+# Adding new languages to Helix
+
+In order to add a new language to Helix, you will need to follow the steps
+below.
 
 ## Language configuration
 
-To add a new language, you need to add a `[[language]]` entry to the
-`languages.toml` (see the [language configuration section]).
+1. Add a new `[[language]]` entry in the `languages.toml` file and provide the
+   necessary configuration for the new language. For more information on
+   language configuration, refer to the
+   [language configuration section](../languages.md) of the documentation.
+2. If you are adding a new language or updating an existing language server
+   configuration, run the command `cargo xtask docgen` to update the
+   [Language Support](../lang-support.md) documentation.
 
-When adding a new language or Language Server configuration for an existing
-language, run `cargo xtask docgen` to add the new configuration to the
-[Language Support][lang-support] docs before creating a pull request.
-When adding a Language Server configuration, be sure to update the
-[Language Server Wiki][install-lsp-wiki] with installation notes.
+> 💡 If you are adding a new Language Server configuration, make sure to update
+> the
+> [Language Server Wiki](https://github.com/helix-editor/helix/wiki/How-to-install-the-default-language-servers)
+> with the installation instructions.
 
 ## Grammar configuration
 
-If a tree-sitter grammar is available for the language, add a new `[[grammar]]`
-entry to `languages.toml`.
-
-You may use the `source.path` key rather than `source.git` with an absolute path
-to a locally available grammar for testing, but switch to `source.git` before
-submitting a pull request.
+1. If a tree-sitter grammar is available for the new language, add a new
+   `[[grammar]]` entry to the `languages.toml` file.
+2. If you are testing the grammar locally, you can use the `source.path` key
+   with an absolute path to the grammar. However, before submitting a pull
+   request, make sure to switch to using `source.git`.
 
 ## Queries
 
-For a language to have syntax-highlighting and indentation among
-other things, you have to add queries. Add a directory for your
-language with the path `runtime/queries/<name>/`. The tree-sitter
-[website](https://tree-sitter.github.io/tree-sitter/syntax-highlighting#queries)
-gives more info on how to write queries.
+1. In order to provide syntax highlighting and indentation for the new language,
+   you will need to add queries.
+2. Create a new directory for the language with the path
+   `runtime/queries/<name>/`.
+3. Refer to the
+   [tree-sitter website](https://tree-sitter.github.io/tree-sitter/syntax-highlighting#queries)
+   for more information on writing queries.
 
-> NOTE: When evaluating queries, the first matching query takes
-precedence, which is different from other editors like Neovim where
-the last matching query supersedes the ones before it. See
-[this issue][neovim-query-precedence] for an example.
+> 💡 In Helix, the first matching query takes precedence when evaluating
+> queries, which is different from other editors such as Neovim where the last
+> matching query supersedes the ones before it. See
+> [this issue](https://github.com/helix-editor/helix/pull/1170#issuecomment-997294090)
+> for an example.
 
-## Common Issues
+## Common issues
 
-- If you get errors when running after switching branches, you may have to update the tree-sitter grammars. Run `hx --grammar fetch` to fetch the grammars and `hx --grammar build` to build any out-of-date grammars.
-
-- If a parser is segfaulting or you want to remove the parser, make sure to remove the compiled parser in `runtime/grammar/<name>.so`
-
-[language configuration section]: ../languages.md
-[neovim-query-precedence]: https://github.com/helix-editor/helix/pull/1170#issuecomment-997294090
-[install-lsp-wiki]: https://github.com/helix-editor/helix/wiki/How-to-install-the-default-language-servers
-[lang-support]: ../lang-support.md
+- If you encounter errors when running Helix after switching branches, you may
+  need to update the tree-sitter grammars. Run the command `hx --grammar fetch`
+  to fetch the grammars and `hx --grammar build` to build any out-of-date
+  grammars.
+- If a parser is causing a segfault, or you want to remove it, make sure to
+  remove the compiled parser located at `runtime/grammars/<name>.so`.

book/src/guides/indent.md

@@ -1,4 +1,4 @@
-# Adding Indent Queries
+# Adding indent queries
 
 Helix uses tree-sitter to correctly indent new lines. This requires
 a tree-sitter grammar and an `indent.scm` query file placed in
@@ -36,7 +36,7 @@ changed by using a `#set!` declaration anywhere in the pattern:
   (#set! "scope" "all"))
 ```
 
-## Capture Types
+## Capture types
 
 - `@indent` (default scope `tail`):
 Increase the indent level by 1. Multiple occurrences in the same line

book/src/guides/injection.md

@@ -0,0 +1,57 @@
+# Adding Injection Queries
+
+Writing language injection queries allows one to highlight a specific node as a different language.
+In addition to the [standard](upstream-docs) language injection options used by tree-sitter, there
+are a few Helix specific extensions that allow for more control.
+
+And example of a simple query that would highlight all strings as bash in Nix:
+```scm
+((string_expression (string_fragment) @injection.content)
+  (#set! injection.language "bash"))
+```
+
+## Capture Types
+
+- `@injection.language` (standard):
+The captured node may contain the language name used to highlight the node captured by
+`@injection.content`.
+
+- `@injection.content` (standard):
+Marks the content to be highlighted as the language captured with `@injection.language` _et al_.
+
+- `@injection.filename` (extension):
+The captured node may contain a filename with a file-extension known to Helix,
+highlighting `@injection.content` as that language. This uses the language extensions defined in
+both the default languages.toml distributed with Helix, as well as user defined languages.
+
+- `@injection.shebang` (extension):
+The captured node may contain a shebang used to choose a language to highlight as. This also uses
+the shebangs defined in the default and user `languages.toml`.
+
+## Settings
+
+- `injection.combined` (standard):
+Indicates that all the matching nodes in the tree should have their content parsed as one
+nested document.
+
+- `injection.language` (standard):
+Forces the captured content to be highlighted as the given language
+
+- `injection.include-children` (standard):
+Indicates that the content node’s entire text should be re-parsed, including the text of its child
+nodes. By default, child nodes’ text will be excluded from the injected document.
+
+- `injection.include-unnamed-children` (extension):
+Same as `injection.include-children` but only for unnamed child nodes.
+
+## Predicates
+
+- `#eq?` (standard):
+The first argument (a capture) must be equal to the second argument
+(a capture or a string).
+
+- `#match?` (standard):
+The first argument (a capture) must match the regex given in the
+second argument (a string).
+
+[upstream-docs]: http://tree-sitter.github.io/tree-sitter/syntax-highlighting#language-injection

book/src/guides/textobject.md

@@ -1,14 +1,14 @@
-# Adding Textobject Queries
+# Adding textobject queries
 
-Textobjects that are language specific ([like functions, classes, etc][textobjects])
-require an accompanying tree-sitter grammar and a `textobjects.scm` query file
+Helix supports textobjects that are language specific, such as functions, classes, etc.
+These textobjects require an accompanying tree-sitter grammar and a `textobjects.scm` query file
 to work properly. Tree-sitter allows us to query the source code syntax tree
 and capture specific parts of it. The queries are written in a lisp dialect.
 More information on how to write queries can be found in the [official tree-sitter
 documentation][tree-sitter-queries].
 
 Query files should be placed in `runtime/queries/{language}/textobjects.scm`
-when contributing. Note that to test the query files locally you should put
+when contributing to Helix. Note that to test the query files locally you should put
 them under your local runtime directory (`~/.config/helix/runtime` on Linux
 for example).
 
@@ -28,9 +28,9 @@ The following [captures][tree-sitter-captures] are recognized:
 
 [Example query files][textobject-examples] can be found in the helix GitHub repository.
 
-## Queries for Textobject Based Navigation
+## Queries for textobject based navigation
 
-[Tree-sitter based navigation][textobjects-nav] is done using captures in the
+Tree-sitter based navigation in Helix is done using captures in the
 following order:
 
 - `object.movement`
@@ -38,12 +38,10 @@ following order:
 - `object.inside`
 
 For example if a `function.around` capture has been already defined for a language
-in it's `textobjects.scm` file, function navigation should also work automatically.
+in its `textobjects.scm` file, function navigation should also work automatically.
 `function.movement` should be defined only if the node captured by `function.around`
 doesn't make sense in a navigation context.
 
-[textobjects]: ../usage.md#textobjects
-[textobjects-nav]: ../usage.md#tree-sitter-textobject-based-navigation
 [tree-sitter-queries]: https://tree-sitter.github.io/tree-sitter/using-parsers#query-syntax
 [tree-sitter-captures]: https://tree-sitter.github.io/tree-sitter/using-parsers#capturing-nodes
 [textobject-examples]: https://github.com/search?q=repo%3Ahelix-editor%2Fhelix+filename%3Atextobjects.scm&type=Code&ref=advsearch&l=&l=

book/src/install.md

@@ -1,167 +1,274 @@
-# Installation
+# Installing Helix
 
-We provide pre-built binaries on the [GitHub Releases page](https://github.com/helix-editor/helix/releases).
+<!--toc:start-->
+- [Pre-built binaries](#pre-built-binaries)
+- [Linux, macOS, Windows and OpenBSD packaging status](#linux-macos-windows-and-openbsd-packaging-status)
+- [Linux](#linux)
+  - [Ubuntu](#ubuntu)
+  - [Fedora/RHEL](#fedorarhel)
+  - [Arch Linux community](#arch-linux-community)
+  - [NixOS](#nixos)
+  - [Flatpak](#flatpak)
+  - [AppImage](#appimage)
+- [macOS](#macos)
+  - [Homebrew Core](#homebrew-core)
+- [Windows](#windows)
+  - [Winget](#winget)
+  - [Scoop](#scoop)
+  - [Chocolatey](#chocolatey)
+  - [MSYS2](#msys2)
+- [Building from source](#building-from-source)
+  - [Configuring Helix's runtime files](#configuring-helixs-runtime-files)
+    - [Linux and macOS](#linux-and-macos)
+    - [Windows](#windows)
+    - [Multiple runtime directories](#multiple-runtime-directories)
+  - [Validating the installation](#validating-the-installation)
+  - [Configure the desktop shortcut](#configure-the-desktop-shortcut)
+<!--toc:end-->
+
+To install Helix, follow the instructions specific to your operating system.
+Note that:
+
+- To get the latest nightly version of Helix, you need to
+  [build from source](#building-from-source).
+
+- To take full advantage of Helix, install the language servers for your
+  preferred programming languages. See the
+  [wiki](https://github.com/helix-editor/helix/wiki/How-to-install-the-default-language-servers)
+  for instructions.
+
+## Pre-built binaries
+
+Download pre-built binaries from the
+[GitHub Releases page](https://github.com/helix-editor/helix/releases). Add the binary to your system's `$PATH` to use it from the command
+line.
+
+## Linux, macOS, Windows and OpenBSD packaging status
 
 [![Packaging status](https://repology.org/badge/vertical-allrepos/helix.svg)](https://repology.org/project/helix/versions)
 
-## OSX
-
-Helix is available in homebrew-core:
-
-```
-brew install helix
-```
-
 ## Linux
 
-### NixOS
+The following third party repositories are available:
 
-A [flake](https://nixos.wiki/wiki/Flakes) containing the package is available in
-the project root. The flake can also be used to spin up a reproducible development
-shell for working on Helix with `nix develop`.
+### Ubuntu
 
-Flake outputs are cached for each push to master using
-[Cachix](https://www.cachix.org/). The flake is configured to
-automatically make use of this cache assuming the user accepts
-the new settings on first use.
-
-If you are using a version of Nix without flakes enabled you can
-[install Cachix cli](https://docs.cachix.org/installation); `cachix use helix` will
-configure Nix to use cached outputs when possible.
-
-### Arch Linux
-
-Releases are available in the `community` repository.
-
-A [helix-git](https://aur.archlinux.org/packages/helix-git/) package is also available on the AUR, which builds the master branch.
-
-### Fedora Linux
-
-You can install the COPR package for Helix via
+Add the `PPA` for Helix:
 
+```sh
+sudo add-apt-repository ppa:maveonair/helix-editor
+sudo apt update
+sudo apt install helix
 ```
+
+### Fedora/RHEL
+
+Enable the `COPR` repository for Helix:
+
+```sh
 sudo dnf copr enable varlad/helix
 sudo dnf install helix
 ```
 
-### Void Linux
+### Arch Linux community
 
+Releases are available in the `community` repository:
+
+```sh
+sudo pacman -S helix
 ```
-sudo xbps-install helix
+Additionally, a [helix-git](https://aur.archlinux.org/packages/helix-git/) package is available
+in the AUR, which builds the master branch.
+
+### NixOS
+
+Helix is available as a [flake](https://nixos.wiki/wiki/Flakes) in the project
+root. Use `nix develop` to spin up a reproducible development shell. Outputs are
+cached for each push to master using [Cachix](https://www.cachix.org/). The
+flake is configured to automatically make use of this cache assuming the user
+accepts the new settings on first use.
+
+If you are using a version of Nix without flakes enabled,
+[install Cachix CLI](https://docs.cachix.org/installation) and use
+`cachix use helix` to configure Nix to use cached outputs when possible.
+
+### Flatpak
+
+Helix is available on [Flathub](https://flathub.org/en-GB/apps/com.helix_editor.Helix):
+
+```sh
+flatpak install flathub com.helix_editor.Helix
+flatpak run com.helix_editor.Helix
+```
+
+### AppImage
+
+Install Helix using the Linux [AppImage](https://appimage.org/) format.
+Download the official Helix AppImage from the [latest releases](https://github.com/helix-editor/helix/releases/latest) page.
+
+```sh
+chmod +x helix-*.AppImage # change permission for executable mode
+./helix-*.AppImage # run helix
+```
+ 
+## macOS
+
+### Homebrew Core
+
+```sh
+brew install helix
 ```
 
 ## Windows
 
-Helix can be installed using [Scoop](https://scoop.sh/), [Chocolatey](https://chocolatey.org/)
+Install on Windows using [Winget](https://learn.microsoft.com/en-us/windows/package-manager/winget/), [Scoop](https://scoop.sh/), [Chocolatey](https://chocolatey.org/)
 or [MSYS2](https://msys2.org/).
 
-**Scoop:**
+### Winget
+Windows Package Manager winget command-line tool is by default available on Windows 11 and modern versions of Windows 10 as a part of the App Installer.
+You can get [App Installer from the Microsoft Store](https://www.microsoft.com/p/app-installer/9nblggh4nns1#activetab=pivot:overviewtab). If it's already installed, make sure it is updated with the latest version.
 
+```sh
+winget install Helix.Helix
 ```
+
+### Scoop
+
+```sh
 scoop install helix
 ```
 
-**Chocolatey:**
+### Chocolatey
 
-```
+```sh
 choco install helix
 ```
 
-**MSYS2:**
+### MSYS2
 
-```
-pacman -S mingw-w64-i686-helix
-```
+For 64-bit Windows 8.1 or above:
 
-or
-
-```
-pacman -S mingw-w64-x86_64-helix
-```
-
-or
-
-```
+```sh
 pacman -S mingw-w64-ucrt-x86_64-helix
 ```
 
-## Build from source
+## Building from source
 
+Requirements:
+
+- The [Rust toolchain](https://www.rust-lang.org/tools/install)
+- The [Git version control system](https://git-scm.com/)
+- A c++14 compatible compiler to build the tree-sitter grammars, for example GCC or Clang
+
+If you are using the `musl-libc` standard library instead of `glibc` the following environment variable must be set during the build to ensure tree-sitter grammars can be loaded correctly:
+
+```sh
+RUSTFLAGS="-C target-feature=-crt-static"
 ```
+
+1. Clone the repository:
+
+```sh
 git clone https://github.com/helix-editor/helix
 cd helix
-cargo install --path helix-term
 ```
 
-This will install the `hx` binary to `$HOME/.cargo/bin` and build tree-sitter grammars in `./runtime/grammars`.
+2. Compile from source:
 
-Helix also needs its runtime files so make sure to copy/symlink the `runtime/` directory into the
-config directory (for example `~/.config/helix/runtime` on Linux/macOS). This location can be overridden
-via the `HELIX_RUNTIME` environment variable.
-
-| OS                   | Command                                          |
-| -------------------- | ------------------------------------------------ |
-| Windows (Cmd)        | `xcopy /e /i runtime %AppData%\helix\runtime`    |
-| Windows (PowerShell) | `xcopy /e /i runtime $Env:AppData\helix\runtime` |
-| Linux / macOS        | `ln -s $PWD/runtime ~/.config/helix/runtime`     |
-
-Starting with Windows Vista you can also create symbolic links on Windows. Note that this requires
-elevated privileges - i.e. PowerShell or Cmd must be run as administrator.
-
-**PowerShell:**
-
-```powershell
-New-Item -ItemType SymbolicLink -Target "runtime" -Path "$Env:AppData\helix\runtime"
+```sh
+cargo install --path helix-term --locked
 ```
 
-**Cmd:**
+This command will create the `hx` executable and construct the tree-sitter
+grammars in the local `runtime` folder.
 
-```cmd
-cd %appdata%\helix
-mklink /D runtime "<helix-repo>\runtime"
+> 💡 Tree-sitter grammars can be fetched and compiled if not pre-packaged. Fetch
+> grammars with `hx --grammar fetch` and compile them with
+> `hx --grammar build`. This will install them in
+> the `runtime` directory within the user's helix config directory (more
+> [details below](#multiple-runtime-directories)).
+
+### Configuring Helix's runtime files
+
+#### Linux and macOS
+
+Either set the `HELIX_RUNTIME` environment variable to point to the runtime files and add it to your `~/.bashrc` or equivalent:
+
+```sh
+HELIX_RUNTIME=/home/user-name/src/helix/runtime
 ```
 
-The runtime location can be overridden via the `HELIX_RUNTIME` environment variable.
+Or, create a symlink in `~/.config/helix` that links to the source code directory:
 
-> NOTE: if `HELIX_RUNTIME` is set prior to calling `cargo install --path helix-term`,
-> tree-sitter grammars will be built in `$HELIX_RUNTIME/grammars`.
-
-If you plan on keeping the repo locally, an alternative to copying/symlinking
-runtime files is to set `HELIX_RUNTIME=/path/to/helix/runtime`
-(`HELIX_RUNTIME=$PWD/runtime` if you're in the helix repo directory).
-
-To use Helix in desktop environments that supports [XDG desktop menu](https://specifications.freedesktop.org/menu-spec/menu-spec-latest.html), including Gnome and KDE, copy the provided `.desktop` file to the correct folder:
-
-```bash
-cp contrib/Helix.desktop ~/.local/share/applications
+```sh
+ln -s $PWD/runtime ~/.config/helix/runtime
 ```
 
-To use another terminal than the default, you will need to modify the `.desktop` file. For example, to use `kitty`:
+#### Windows
 
-```bash
-sed -i "s|Exec=hx %F|Exec=kitty hx %F|g" ~/.local/share/applications/Helix.desktop
-sed -i "s|Terminal=true|Terminal=false|g" ~/.local/share/applications/Helix.desktop
+Either set the `HELIX_RUNTIME` environment variable to point to the runtime files using the Windows setting (search for
+`Edit environment variables for your account`) or use the `setx` command in
+Cmd:
+
+```sh
+setx HELIX_RUNTIME "%userprofile%\source\repos\helix\runtime"
 ```
 
-Please note: there is no icon for Helix yet, so the system default will be used.
+> 💡 `%userprofile%` resolves to your user directory like
+> `C:\Users\Your-Name\` for example.
 
-## Finishing up the installation
+Or, create a symlink in `%appdata%\helix\` that links to the source code directory:
 
-To make sure everything is set up as expected you should finally run the helix healthcheck via
+| Method     | Command                                                                                |
+| ---------- | -------------------------------------------------------------------------------------- |
+| PowerShell | `New-Item -ItemType Junction -Target "runtime" -Path "$Env:AppData\helix\runtime"`     |
+| Cmd        | `cd %appdata%\helix` <br/> `mklink /D runtime "%userprofile%\src\helix\runtime"`       |
 
-```
+> 💡 On Windows, creating a symbolic link may require running PowerShell or
+> Cmd as an administrator.
+
+#### Multiple runtime directories
+
+When Helix finds multiple runtime directories it will search through them for files in the
+following order:
+
+1. `runtime/` sibling directory to `$CARGO_MANIFEST_DIR` directory (this is intended for
+  developing and testing helix only).
+2. `runtime/` subdirectory of OS-dependent helix user config directory.
+3. `$HELIX_RUNTIME`.
+4. `runtime/` subdirectory of path to Helix executable.
+
+This order also sets the priority for selecting which file will be used if multiple runtime
+directories have files with the same name.
+
+### Validating the installation
+
+To make sure everything is set up as expected you should run the Helix health
+check:
+
+```sh
 hx --health
 ```
 
-For more information on the information displayed in the health check results refer to [Healthcheck](https://github.com/helix-editor/helix/wiki/Healthcheck).
+For more information on the health check results refer to
+[Health check](https://github.com/helix-editor/helix/wiki/Healthcheck).
 
-### Building tree-sitter grammars
+### Configure the desktop shortcut
 
-Tree-sitter grammars must be fetched and compiled if not pre-packaged.
-Fetch grammars with `hx --grammar fetch` (requires `git`) and compile them
-with `hx --grammar build` (requires a C++ compiler).
+If your desktop environment supports the
+[XDG desktop menu](https://specifications.freedesktop.org/menu-spec/menu-spec-latest.html)
+you can configure Helix to show up in the application menu by copying the
+provided `.desktop` and icon files to their correct folders:
 
-### Installing language servers
+```sh
+cp contrib/Helix.desktop ~/.local/share/applications
+cp contrib/helix.png ~/.icons # or ~/.local/share/icons
+```
 
-Language servers can optionally be installed if you want their features (auto-complete, diagnostics etc.).
-Follow the [instructions on the wiki page](https://github.com/helix-editor/helix/wiki/How-to-install-the-default-language-servers) to add your language servers of choice.
+To use another terminal than the system default, you can modify the `.desktop`
+file. For example, to use `kitty`:
+
+```sh
+sed -i "s|Exec=hx %F|Exec=kitty hx %F|g" ~/.local/share/applications/Helix.desktop
+sed -i "s|Terminal=true|Terminal=false|g" ~/.local/share/applications/Helix.desktop
+```

book/src/keymap.md

@@ -14,14 +14,14 @@
     - [Space mode](#space-mode)
       - [Popup](#popup)
     - [Unimpaired](#unimpaired)
-- [Insert Mode](#insert-mode)
+- [Insert mode](#insert-mode)
 - [Select / extend mode](#select--extend-mode)
 - [Picker](#picker)
 - [Prompt](#prompt)
 
 > 💡 Mappings marked (**LSP**) require an active language server for the file.
 
-> 💡 Mappings marked (**TS**) require a tree-sitter grammar for the filetype.
+> 💡 Mappings marked (**TS**) require a tree-sitter grammar for the file type.
 
 ## Normal mode
 
@@ -32,8 +32,8 @@
 | Key                   | Description                                        | Command                     |
 | -----                 | -----------                                        | -------                     |
 | `h`, `Left`           | Move left                                          | `move_char_left`            |
-| `j`, `Down`           | Move down                                          | `move_line_down`            |
-| `k`, `Up`             | Move up                                            | `move_line_up`              |
+| `j`, `Down`           | Move down                                          | `move_visual_line_down`     |
+| `k`, `Up`             | Move up                                            | `move_visual_line_up`       |
 | `l`, `Right`          | Move right                                         | `move_char_right`           |
 | `w`                   | Move next word start                               | `move_next_word_start`      |
 | `b`                   | Move previous word start                           | `move_prev_word_start`      |
@@ -109,8 +109,9 @@
 | Key                   | Description                                                       | Command                              |
 | -----                 | -----------                                                       | -------                              |
 | `s`                   | Select all regex matches inside selections                        | `select_regex`                       |
-| `S`                   | Split selection into subselections on regex matches               | `split_selection`                    |
+| `S`                   | Split selection into sub selections on regex matches              | `split_selection`                    |
 | `Alt-s`               | Split selection on newlines                                       | `split_selection_on_newline`         |
+| `Alt-_ `              | Merge consecutive selections                                      | `merge_consecutive_selections`       |
 | `&`                   | Align selection in columns                                        | `align_selections`                   |
 | `_`                   | Trim whitespace from the selection                                | `trim_selections`                    |
 | `;`                   | Collapse selection onto a single cursor                           | `collapse_selection`                 |
@@ -129,7 +130,7 @@
 | `X`                   | Extend selection to line bounds (line-wise selection)             | `extend_to_line_bounds`              |
 | `Alt-x`               | Shrink selection to line bounds (line-wise selection)             | `shrink_to_line_bounds`              |
 | `J`                   | Join lines inside selection                                       | `join_selections`                    |
-| `Alt-J`               | Join lines inside selection and select space                      | `join_selections_space`              |
+| `Alt-J`               | Join lines inside selection and select the inserted space         | `join_selections_space`              |
 | `K`                   | Keep selections matching the regex                                | `keep_selections`                    |
 | `Alt-K`               | Remove selections matching the regex                              | `remove_selections`                  |
 | `Ctrl-c`              | Comment/uncomment the selections                                  | `toggle_comments`                    |
@@ -140,7 +141,7 @@
 
 ### Search
 
-Search commands all operate on the `/` register by default. Use `"<char>` to operate on a different one.
+Search commands all operate on the `/` register by default. To use a different register, use `"<char>`.
 
 | Key   | Description                                 | Command              |
 | ----- | -----------                                 | -------              |
@@ -165,15 +166,17 @@ These sub-modes are accessible from normal mode and typically switch back to nor
 | `Ctrl-w` | Enter [window mode](#window-mode)                  | N/A            |
 | `Space`  | Enter [space mode](#space-mode)                    | N/A            |
 
+These modes (except command mode) can be configured by
+[remapping keys](https://docs.helix-editor.com/remapping.html#minor-modes).
+
 #### View mode
 
 Accessed by typing `z` in [normal mode](#normal-mode).
 
 View mode is intended for scrolling and manipulating the view without changing
 the selection. The "sticky" variant of this mode (accessed by typing `Z` in
-normal mode) is persistent; use the Escape key to return to normal mode after
-usage (useful when you're simply looking over text and not actively editing
-it).
+normal mode) is persistent and can be exited using the escape key. This is
+useful when you're simply looking over text and not actively editing it.
 
 
 | Key                  | Description                                               | Command             |
@@ -215,13 +218,15 @@ Jumps to various locations.
 | `n`   | Go to next buffer                                | `goto_next_buffer`         |
 | `p`   | Go to previous buffer                            | `goto_previous_buffer`     |
 | `.`   | Go to last modification in current file          | `goto_last_modification`   |
+| `j`   | Move down textual (instead of visual) line       | `move_line_down`           |
+| `k`   | Move up textual (instead of visual) line         | `move_line_up`             |
 
 #### Match mode
 
 Accessed by typing `m` in [normal mode](#normal-mode).
 
 See the relevant section in [Usage](./usage.md) for an explanation about
-[surround](./usage.md#surround) and [textobject](./usage.md#textobjects) usage.
+[surround](./usage.md#surround) and [textobject](./usage.md#navigating-using-tree-sitter-textobjects) usage.
 
 | Key              | Description                                     | Command                    |
 | -----            | -----------                                     | -------                    |
@@ -238,7 +243,7 @@ TODO: Mappings for selecting syntax nodes (a superset of `[`).
 
 Accessed by typing `Ctrl-w` in [normal mode](#normal-mode).
 
-This layer is similar to Vim keybindings as Kakoune does not support window.
+This layer is similar to Vim keybindings as Kakoune does not support windows.
 
 | Key                    | Description                                          | Command           |
 | -----                  | -------------                                        | -------           |
@@ -264,30 +269,32 @@ Accessed by typing `Space` in [normal mode](#normal-mode).
 
 This layer is a kludge of mappings, mostly pickers.
 
-| Key     | Description                                                             | Command                             |
-| -----   | -----------                                                             | -------                             |
-| `f`     | Open file picker                                                        | `file_picker`                       |
-| `F`     | Open file picker at current working directory                           | `file_picker_in_current_directory`  |
-| `b`     | Open buffer picker                                                      | `buffer_picker`                     |
-| `j`     | Open jumplist picker                                                    | `jumplist_picker`                   |
-| `k`     | Show documentation for item under cursor in a [popup](#popup) (**LSP**) | `hover`                             |
-| `s`     | Open document symbol picker (**LSP**)                                   | `symbol_picker`                     |
-| `S`     | Open workspace symbol picker (**LSP**)                                  | `workspace_symbol_picker`           |
-| `d`     | Open document diagnostics picker (**LSP**)                              | `diagnostics_picker`                |
-| `D`     | Open workspace diagnostics picker (**LSP**)                             | `workspace_diagnostics_picker`      |
-| `r`     | Rename symbol (**LSP**)                                                 | `rename_symbol`                     |
-| `a`     | Apply code action  (**LSP**)                                            | `code_action`                       |
-| `'`     | Open last fuzzy picker                                                  | `last_picker`                       |
-| `w`     | Enter [window mode](#window-mode)                                       | N/A                                 |
-| `p`     | Paste system clipboard after selections                                 | `paste_clipboard_after`             |
-| `P`     | Paste system clipboard before selections                                | `paste_clipboard_before`            |
-| `y`     | Join and yank selections to clipboard                                   | `yank_joined_to_clipboard`          |
-| `Y`     | Yank main selection to clipboard                                        | `yank_main_selection_to_clipboard`  |
-| `R`     | Replace selections by clipboard contents                                | `replace_selections_with_clipboard` |
-| `/`     | Global search in workspace folder                                       | `global_search`                     |
-| `?`     | Open command palette                                                    | `command_palette`                   |
+| Key     | Description                                                             | Command                                    |
+| -----   | -----------                                                             | -------                                    |
+| `f`     | Open file picker                                                        | `file_picker`                              |
+| `F`     | Open file picker at current working directory                           | `file_picker_in_current_directory`         |
+| `b`     | Open buffer picker                                                      | `buffer_picker`                            |
+| `j`     | Open jumplist picker                                                    | `jumplist_picker`                          |
+| `g`     | Debug (experimental)                                                    | N/A                                        |
+| `k`     | Show documentation for item under cursor in a [popup](#popup) (**LSP**) | `hover`                                    |
+| `s`     | Open document symbol picker (**LSP**)                                   | `symbol_picker`                            |
+| `S`     | Open workspace symbol picker (**LSP**)                                  | `workspace_symbol_picker`                  |
+| `d`     | Open document diagnostics picker (**LSP**)                              | `diagnostics_picker`                       |
+| `D`     | Open workspace diagnostics picker (**LSP**)                             | `workspace_diagnostics_picker`             |
+| `r`     | Rename symbol (**LSP**)                                                 | `rename_symbol`                            |
+| `a`     | Apply code action (**LSP**)                                             | `code_action`                              |
+| `h`     | Select symbol references (**LSP**)                                      | `select_references_to_symbol_under_cursor` |
+| `'`     | Open last fuzzy picker                                                  | `last_picker`                              |
+| `w`     | Enter [window mode](#window-mode)                                       | N/A                                        |
+| `p`     | Paste system clipboard after selections                                 | `paste_clipboard_after`                    |
+| `P`     | Paste system clipboard before selections                                | `paste_clipboard_before`                   |
+| `y`     | Join and yank selections to clipboard                                   | `yank_joined_to_clipboard`                 |
+| `Y`     | Yank main selection to clipboard                                        | `yank_main_selection_to_clipboard`         |
+| `R`     | Replace selections by clipboard contents                                | `replace_selections_with_clipboard`        |
+| `/`     | Global search in workspace folder                                       | `global_search`                            |
+| `?`     | Open command palette                                                    | `command_palette`                          |
 
-> TIP: Global search displays results in a fuzzy picker, use `Space + '` to bring it back up after opening a file.
+> 💡 Global search displays results in a fuzzy picker, use `Space + '` to bring it back up after opening a file.
 
 ##### Popup
 
@@ -300,14 +307,14 @@ Displays documentation for item under cursor.
 
 #### Unimpaired
 
-Mappings in the style of [vim-unimpaired](https://github.com/tpope/vim-unimpaired).
+These mappings are in the style of [vim-unimpaired](https://github.com/tpope/vim-unimpaired).
 
 | Key      | Description                                  | Command               |
 | -----    | -----------                                  | -------               |
-| `[d`     | Go to previous diagnostic (**LSP**)          | `goto_prev_diag`      |
 | `]d`     | Go to next diagnostic (**LSP**)              | `goto_next_diag`      |
-| `[D`     | Go to first diagnostic in document (**LSP**) | `goto_first_diag`     |
+| `[d`     | Go to previous diagnostic (**LSP**)          | `goto_prev_diag`      |
 | `]D`     | Go to last diagnostic in document (**LSP**)  | `goto_last_diag`      |
+| `[D`     | Go to first diagnostic in document (**LSP**) | `goto_first_diag`     |
 | `]f`     | Go to next function (**TS**)                 | `goto_next_function`  |
 | `[f`     | Go to previous function (**TS**)             | `goto_prev_function`  |
 | `]t`     | Go to next type definition (**TS**)          | `goto_next_class`     |
@@ -317,24 +324,25 @@ Mappings in the style of [vim-unimpaired](https://github.com/tpope/vim-unimpaire
 | `]c`     | Go to next comment (**TS**)                  | `goto_next_comment`   |
 | `[c`     | Go to previous comment (**TS**)              | `goto_prev_comment`   |
 | `]T`     | Go to next test (**TS**)                     | `goto_next_test`      |
-| `]T`     | Go to previous test (**TS**)                 | `goto_prev_test`      |
+| `[T`     | Go to previous test (**TS**)                 | `goto_prev_test`      |
 | `]p`     | Go to next paragraph                         | `goto_next_paragraph` |
 | `[p`     | Go to previous paragraph                     | `goto_prev_paragraph` |
 | `]g`     | Go to next change                            | `goto_next_change`    |
 | `[g`     | Go to previous change                        | `goto_prev_change`    |
-| `]G`     | Go to first change                           | `goto_first_change`   |
-| `[G`     | Go to last change                            | `goto_last_change`    |
-| `[Space` | Add newline above                            | `add_newline_above`   |
+| `]G`     | Go to last change                            | `goto_last_change`    |
+| `[G`     | Go to first change                           | `goto_first_change`   |
 | `]Space` | Add newline below                            | `add_newline_below`   |
+| `[Space` | Add newline above                            | `add_newline_above`   |
 
 ## Insert mode
 
-Insert mode bindings are somewhat minimal by default. Helix is designed to
+Insert mode bindings are minimal by default. Helix is designed to
 be a modal editor, and this is reflected in the user experience and internal
-mechanics. For example, changes to the text are only saved for undos when
-escaping from insert mode to normal mode. For this reason, new users are
-strongly encouraged to learn the modal editing paradigm to get the smoothest
-experience.
+mechanics. Changes to the text are only saved for undos when
+escaping from insert mode to normal mode.
+
+> 💡 New users are strongly encouraged to learn the modal editing paradigm
+> to get the smoothest experience.
 
 | Key                                         | Description                 | Command                  |
 | -----                                       | -----------                 | -------                  |
@@ -346,7 +354,7 @@ experience.
 | `Alt-d`, `Alt-Delete`                       | Delete next word            | `delete_word_forward`    |
 | `Ctrl-u`                                    | Delete to start of line     | `kill_to_line_start`     |
 | `Ctrl-k`                                    | Delete to end of line       | `kill_to_line_end`       |
-| `Ctrl-h`, `Backspace`                       | Delete previous char        | `delete_char_backward`   |
+| `Ctrl-h`, `Backspace`, `Shift-Backspace`    | Delete previous char        | `delete_char_backward`   |
 | `Ctrl-d`, `Delete`                          | Delete next char            | `delete_char_forward`    |
 | `Ctrl-j`, `Enter`                           | Insert new line             | `insert_newline`         |
 
@@ -364,8 +372,8 @@ with modal editors.
 | `Home`                                      | Move to line start          | `goto_line_start`        |
 | `End`                                       | Move to line end            | `goto_line_end_newline`  |
 
-If you want to disable them in insert mode as you become more comfortable with modal editing, you can use
-the following in your `config.toml`:
+As you become more comfortable with modal editing, you may want to disable some
+insert mode bindings. You can do this by editing your `config.toml` file.
 
 ```toml
 [keys.insert]
@@ -381,7 +389,7 @@ end = "no_op"
 
 ## Select / extend mode
 
-This mode echoes Normal mode, but changes any movements to extend
+Select mode echoes Normal mode, but changes any movements to extend
 selections rather than replace them. Goto motions are also changed to
 extend, so that `vgl` for example extends the selection to the end of
 the line.
@@ -427,7 +435,7 @@ Keys to use within prompt, Remapping currently not supported.
 | `Alt-d`, `Alt-Delete`, `Ctrl-Delete`        | Delete next word                                                        |
 | `Ctrl-u`                                    | Delete to start of line                                                 |
 | `Ctrl-k`                                    | Delete to end of line                                                   |
-| `Backspace`, `Ctrl-h`                       | Delete previous char                                                    |
+| `Backspace`, `Ctrl-h`, `Shift-Backspace`    | Delete previous char                                                    |
 | `Delete`, `Ctrl-d`                          | Delete next char                                                        |
 | `Ctrl-s`                                    | Insert a word under doc cursor, may be changed to Ctrl-r Ctrl-w later   |
 | `Ctrl-p`, `Up`                              | Select previous history                                                 |

book/src/lang-support.md

@@ -1,10 +1,10 @@
 # Language Support
 
-The following languages and Language Servers are supported. In order to use
+The following languages and Language Servers are supported. To use
 Language Server features, you must first [install][lsp-install-wiki] the
 appropriate Language Server.
 
-Check the language support in your installed helix version with `hx --health`.
+You can check the language support in your installed helix version with `hx --health`.
 
 Also see the [Language Configuration][lang-config] docs and the [Adding
 Languages][adding-languages] guide for more language configuration information.

book/src/languages.md

@@ -5,13 +5,15 @@ in `languages.toml` files.
 
 ## `languages.toml` files
 
-There are three possible `languages.toml` files. The first is compiled into
-Helix and lives in the [Helix repository](https://github.com/helix-editor/helix/blob/master/languages.toml).
-This provides the default configurations for languages and language servers.
+There are three possible locations for a `languages.toml` file:
 
-You may define a `languages.toml` in your [configuration directory](./configuration.md)
-which overrides values from the built-in language configuration. For example
-to disable auto-LSP-formatting in Rust:
+1. In the Helix source code, this lives in the
+   [Helix repository](https://github.com/helix-editor/helix/blob/master/languages.toml).
+   It provides the default configurations for languages and language servers.
+
+2. In your [configuration directory](./configuration.md). This overrides values
+   from the built-in language configuration. For example to disable
+   auto-LSP-formatting in Rust:
 
 ```toml
 # in <config_dir>/helix/languages.toml
@@ -21,10 +23,10 @@ name = "rust"
 auto-format = false
 ```
 
-Language configuration may also be overridden local to a project by creating
-a `languages.toml` file under a `.helix` directory. Its settings will be merged
-with the language configuration in the configuration directory and the built-in
-configuration.
+3. In a `.helix` folder in your project. Language configuration may also be
+   overridden local to a project by creating a `languages.toml` file in a
+   `.helix` folder. Its settings will be merged with the language configuration
+   in the configuration directory and the built-in configuration.
 
 ## Language configuration
 
@@ -35,7 +37,7 @@ Each language is configured by adding a `[[language]]` section to a
 [[language]]
 name = "mylang"
 scope = "source.mylang"
-injection-regex = "^mylang$"
+injection-regex = "mylang"
 file-types = ["mylang", "myl"]
 comment-token = "#"
 indent = { tab-width = 2, unit = "  " }
@@ -56,16 +58,17 @@ These configuration keys are available:
 | `auto-format`         | Whether to autoformat this language when saving               |
 | `diagnostic-severity` | Minimal severity of diagnostic for it to be displayed. (Allowed values: `Error`, `Warning`, `Info`, `Hint`) |
 | `comment-token`       | The token to use as a comment-token                           |
-| `indent`              | The indent to use. Has sub keys `tab-width` and `unit`        |
+| `indent`              | The indent to use. Has sub keys `unit` (the text inserted into the document when indenting; usually set to N spaces or `"\t"` for tabs) and `tab-width` (the number of spaces rendered for a tab) |
 | `language-server`     | The Language Server to run. See the Language Server configuration section below. |
 | `config`              | Language Server configuration                                 |
 | `grammar`             | The tree-sitter grammar to use (defaults to the value of `name`) |
 | `formatter`           | The formatter for the language, it will take precedence over the lsp when defined. The formatter must be able to take the original file as input from stdin and write the formatted file to stdout |
-| `max-line-length`     | Maximum line length. Used for the `:reflow` command           |
+| `text-width`          |  Maximum line length. Used for the `:reflow` command and soft-wrapping if `soft-wrap.wrap-at-text-width` is set, defaults to `editor.text-width`   |
+| `workspace-lsp-roots`     | Directories relative to the workspace root that are treated as LSP roots. Should only be set in `.helix/config.toml`. Overwrites the setting of the same name in `config.toml` if set. |
 
 ### File-type detection and the `file-types` key
 
-Helix determines which language configuration to use with the `file-types` key
+Helix determines which language configuration to use based on the `file-types` key
 from the above section. `file-types` is a list of strings or tables, for
 example:
 

book/src/remapping.md

@@ -1,18 +1,18 @@
-# Key Remapping
+# Key remapping
 
-One-way key remapping is temporarily supported via a simple TOML configuration
+Helix currently supports one-way key remapping through a simple TOML configuration
 file. (More powerful solutions such as rebinding via commands will be
 available in the future).
 
-To remap keys, write a `config.toml` file in your `helix` configuration
-directory (default `~/.config/helix` in Linux systems) with a structure like
+To remap keys, create a `config.toml` file in your `helix` configuration
+directory (default `~/.config/helix` on Linux systems) with a structure like
 this:
 
 ```toml
 # At most one section each of 'keys.normal', 'keys.insert' and 'keys.select'
 [keys.normal]
-C-s = ":w" # Maps the Ctrl-s to the typable command :w which is an alias for :write (save file)
-C-o = ":open ~/.config/helix/config.toml" # Maps the Ctrl-o to opening of the helix config file
+C-s = ":w" # Maps Ctrl-s to the typable command :w which is an alias for :write (save file)
+C-o = ":open ~/.config/helix/config.toml" # Maps Ctrl-o to opening of the helix config file
 a = "move_char_left" # Maps the 'a' key to the move_char_left command
 w = "move_line_up" # Maps the 'w' key move_line_up
 "C-S-esc" = "extend_line" # Maps Ctrl-Shift-Escape to extend_line
@@ -20,10 +20,35 @@ g = { a = "code_action" } # Maps `ga` to show possible code actions
 "ret" = ["open_below", "normal_mode"] # Maps the enter key to open_below then re-enter normal mode
 
 [keys.insert]
-"A-x" = "normal_mode" # Maps Alt-X to enter normal mode
+"A-x" = "normal_mode"     # Maps Alt-X to enter normal mode
 j = { k = "normal_mode" } # Maps `jk` to exit insert mode
 ```
-> NOTE: Typable commands can also be remapped, remember to keep the `:` prefix to indicate it's a typable command.
+
+## Minor modes
+
+Minor modes are accessed by pressing a key (usually from normal mode), giving access to dedicated bindings. Bindings
+can be modified or added by nesting definitions.
+
+```toml
+[keys.insert.j]
+k = "normal_mode" # Maps `jk` to exit insert mode
+
+[keys.normal.g]
+a = "code_action" # Maps `ga` to show possible code actions
+
+# invert `j` and `k` in view mode
+[keys.normal.z]
+j = "scroll_up"
+k = "scroll_down"
+
+# create a new minor mode bound to `+`
+[keys.normal."+"]
+m = ":run-shell-command make"
+c = ":run-shell-command cargo build"
+t = ":run-shell-command cargo test"
+```
+
+## Special keys and modifiers
 
 Ctrl, Shift and Alt modifiers are encoded respectively with the prefixes
 `C-`, `S-` and `A-`. Special keys are encoded as follows:
@@ -50,5 +75,5 @@ Ctrl, Shift and Alt modifiers are encoded respectively with the prefixes
 
 Keys can be disabled by binding them to the `no_op` command.
 
-Commands can be found at [Keymap](https://docs.helix-editor.com/keymap.html) Commands.
-> Commands can also be found in the source code at [`helix-term/src/commands.rs`](https://github.com/helix-editor/helix/blob/master/helix-term/src/commands.rs) at the invocation of `static_commands!` macro and the `TypableCommandList`.
+A list of commands is available in the [Keymap](https://docs.helix-editor.com/keymap.html) documentation
+ and in the source code at [`helix-term/src/commands.rs`](https://github.com/helix-editor/helix/blob/master/helix-term/src/commands.rs) at the invocation of `static_commands!` macro and the `TypableCommandList`.

book/src/themes.md

@@ -1,14 +1,15 @@
 # Themes
 
-To use a theme add `theme = "<name>"` to your [`config.toml`](./configuration.md) at the very top of the file before the first section or select it during runtime using `:theme <name>`.
+To use a theme add `theme = "<name>"` to the top of your [`config.toml`](./configuration.md) file, or select it during runtime using `:theme <name>`.
 
 ## Creating a theme
 
-Create a file with the name of your theme as file name (i.e `mytheme.toml`) and place it in your `themes` directory (i.e `~/.config/helix/themes`). The directory might have to be created beforehand.
+Create a file with the name of your theme as the file name (i.e `mytheme.toml`) and place it in your `themes` directory (i.e `~/.config/helix/themes` or `%AppData%\helix\themes` on Windows). The directory might have to be created beforehand.
 
-The names "default" and "base16_default" are reserved for the builtin themes and cannot be overridden by user defined themes.
+> 💡 The names "default" and "base16_default" are reserved for built-in themes
+> and cannot be overridden by user-defined themes.
 
-The default theme.toml can be found [here](https://github.com/helix-editor/helix/blob/master/theme.toml), and user submitted themes [here](https://github.com/helix-editor/helix/blob/master/runtime/themes). 
+### Overview
 
 Each line in the theme file is specified as below:
 
@@ -16,7 +17,7 @@ Each line in the theme file is specified as below:
 key = { fg = "#ffffff", bg = "#000000", underline = { color = "#ff0000", style = "curl"}, modifiers = ["bold", "italic"] }
 ```
 
-where `key` represents what you want to style, `fg` specifies the foreground color, `bg` the background color, `underline` the underline `style`/`color`, and `modifiers` is a list of style modifiers. `bg`, `underline` and `modifiers` can be omitted to defer to the defaults.
+Where `key` represents what you want to style, `fg` specifies the foreground color, `bg` the background color, `underline` the underline `style`/`color`, and `modifiers` is a list of style modifiers. `bg`, `underline` and `modifiers` can be omitted to defer to the defaults.
 
 To specify only the foreground color:
 
@@ -24,15 +25,30 @@ To specify only the foreground color:
 key = "#ffffff"
 ```
 
-if the key contains a dot `'.'`, it must be quoted to prevent it being parsed as a [dotted key](https://toml.io/en/v1.0.0#keys).
+If the key contains a dot `'.'`, it must be quoted to prevent it being parsed as a [dotted key](https://toml.io/en/v1.0.0#keys).
 
 ```toml
 "key.key" = "#ffffff"
 ```
 
+For inspiration, you can find the default `theme.toml`
+[here](https://github.com/helix-editor/helix/blob/master/theme.toml) and
+user-submitted themes
+[here](https://github.com/helix-editor/helix/blob/master/runtime/themes).
+
+### Using the linter
+
+Use the supplied linting tool to check for errors and missing scopes:
+
+```sh
+cargo xtask themelint onedark # replace onedark with <name>
+```
+
+## The details of theme creation
+
 ### Color palettes
 
-It's recommended define a palette of named colors, and refer to them from the
+It's recommended to define a palette of named colors, and refer to them in the
 configuration values in your theme. To do this, add a table called
 `palette` to your theme file:
 
@@ -45,8 +61,8 @@ white = "#ffffff"
 black = "#000000"
 ```
 
-Remember that the `[palette]` table includes all keys after its header,
-so you should define the palette after normal theme options.
+Keep in mind that the `[palette]` table includes all keys after its header,
+so it should be defined after the normal theme options.
 
 The default palette uses the terminal's default 16 colors, and the colors names
 are listed below. The `[palette]` section in the config file takes precedence
@@ -73,9 +89,8 @@ over it and is merged into the default palette.
 
 ### Modifiers
 
-The following values may be used as modifiers. 
-
-Less common modifiers might not be supported by your terminal emulator.
+The following values may be used as modifier, provided they are supported by
+your terminal emulator.
 
 | Modifier             |
 | ---                  |
@@ -89,14 +104,13 @@ Less common modifiers might not be supported by your terminal emulator.
 | `hidden`             |
 | `crossed_out`        |
 
-> Note: The `underlined` modifier is deprecated and only available for backwards compatibility.
+> 💡 The `underlined` modifier is deprecated and only available for backwards compatibility.
 > Its behavior is equivalent to setting `underline.style="line"`.
 
-### Underline Style
+### Underline style
 
-One of the following values may be used as a value for `underline.style`. 
-
-Some styles might not be supported by your terminal emulator.
+One of the following values may be used as a value for `underline.style`, providing it is
+supported by your terminal emulator.
 
 | Modifier       |
 | ---            |
@@ -109,7 +123,7 @@ Some styles might not be supported by your terminal emulator.
 
 ### Inheritance
 
-Extend upon other themes by setting the `inherits` property to an existing theme.
+Extend other themes by setting the `inherits` property to an existing theme.
 
 ```toml
 inherits = "boo_berry"
@@ -124,25 +138,27 @@ berry = "#2A2A4D"
 
 ### Scopes
 
-The following is a list of scopes available to use for styling.
+The following is a list of scopes available to use for styling:
 
 #### Syntax highlighting
 
 These keys match [tree-sitter scopes](https://tree-sitter.github.io/tree-sitter/syntax-highlighting#theme).
 
-For a given highlight produced, styling will be determined based on the longest matching theme key. For example, the highlight `function.builtin.static` would match the key `function.builtin` rather than `function`.
+When determining styling for a highlight, the longest matching theme key will be used. For example, if the highlight is `function.builtin.static`, the key `function.builtin` will be used instead of `function`.
 
 We use a similar set of scopes as
-[SublimeText](https://www.sublimetext.com/docs/scope_naming.html). See also
+[Sublime Text](https://www.sublimetext.com/docs/scope_naming.html). See also
 [TextMate](https://macromates.com/manual/en/language_grammars) scopes.
 
-- `attribute` - Class attributes, html tag attributes
+- `attribute` - Class attributes, HTML tag attributes
 
 - `type` - Types
   - `builtin` - Primitive types provided by the language (`int`, `usize`)
+  - `enum`
+    - `variant`
 - `constructor`
 
-- `constant` (TODO: constant.other.placeholder for %v)
+- `constant` (TODO: constant.other.placeholder for `%v`)
   - `builtin` Special constants provided by the language (`true`, `false`, `nil` etc)
     - `boolean`
   - `character`
@@ -160,11 +176,11 @@ We use a similar set of scopes as
 
 - `comment` - Code comments
   - `line` - Single line comments (`//`)
-  - `block` - Block comments (e.g. (`/*     */`)
+  - `block` - Block comments (e.g. (`/* */`)
     - `documentation` - Documentation comments (e.g. `///` in Rust)
 
 - `variable` - Variables
-  - `builtin` - Reserved language variables (`self`, `this`, `super`, etc)
+  - `builtin` - Reserved language variables (`self`, `this`, `super`, etc.)
   - `parameter` - Function parameters
   - `other`
     - `member` - Fields of composite data types (e.g. structs, unions)
@@ -184,10 +200,10 @@ We use a similar set of scopes as
     - `return`
     - `exception`
   - `operator` - `or`, `in`
-  - `directive` - Preprocessor directives (`#if` in C) 
+  - `directive` - Preprocessor directives (`#if` in C)
   - `function` - `fn`, `func`
   - `storage` - Keywords describing how things are stored
-    - `type` - The type of something, `class`, `function`, `var`, `let`, etc. 
+    - `type` - The type of something, `class`, `function`, `var`, `let`, etc.
     - `modifier` - Storage modifiers like `static`, `mut`, `const`, `ref`, etc.
 
 - `operator` - `||`, `+=`, `>`
@@ -199,9 +215,12 @@ We use a similar set of scopes as
   - `special` (preprocessor in C)
 
 - `tag` - Tags (e.g. `<body>` in HTML)
+  - `builtin`
 
 - `namespace`
 
+- `special`
+
 - `markup`
   - `heading`
     - `marker`
@@ -209,12 +228,15 @@ We use a similar set of scopes as
   - `list`
     - `unnumbered`
     - `numbered`
+    - `checked`
+    - `unchecked`
   - `bold`
   - `italic`
+  - `strikethrough`
   - `link`
-    - `url` - urls pointed to by links
-    - `label` - non-url link references
-    - `text` - url and image descriptions in links
+    - `url` - URLs pointed to by links
+    - `label` - non-URL link references
+    - `text` - URL and image descriptions in links
   - `quote`
   - `raw`
     - `inline`
@@ -228,73 +250,80 @@ We use a similar set of scopes as
 
 #### Interface
 
-These scopes are used for theming the editor interface.
+These scopes are used for theming the editor interface:
 
 - `markup`
   - `normal`
-    - `completion` - for completion doc popup ui
-    - `hover` - for hover popup ui
+    - `completion` - for completion doc popup UI
+    - `hover` - for hover popup UI
   - `heading`
-    - `completion` - for completion doc popup ui
-    - `hover` - for hover popup ui
+    - `completion` - for completion doc popup UI
+    - `hover` - for hover popup UI
   - `raw`
     - `inline`
-      - `completion` - for completion doc popup ui
-      - `hover` - for hover popup ui
+      - `completion` - for completion doc popup UI
+      - `hover` - for hover popup UI
 
 
-| Key                         | Notes                                                                                          |
-| ---                         | ---                                                                                            |
-| `ui.background`             |                                                                                                |
-| `ui.background.separator`   | Picker separator below input line                                                              |
-| `ui.cursor`                 |                                                                                                |
-| `ui.cursor.insert`          |                                                                                                |
-| `ui.cursor.select`          |                                                                                                |
-| `ui.cursor.match`           | Matching bracket etc.                                                                          |
-| `ui.cursor.primary`         | Cursor with primary selection                                                                  |
-| `ui.gutter`                 | Gutter                                                                                         |
-| `ui.gutter.selected`        | Gutter for the line the cursor is on                                                           |
-| `ui.linenr`                 | Line numbers                                                                                   |
-| `ui.linenr.selected`        | Line number for the line the cursor is on                                                      |
-| `ui.statusline`             | Statusline                                                                                     |
-| `ui.statusline.inactive`    | Statusline (unfocused document)                                                                |
-| `ui.statusline.normal`      | Statusline mode during normal mode ([only if `editor.color-modes` is enabled][editor-section]) |
-| `ui.statusline.insert`      | Statusline mode during insert mode ([only if `editor.color-modes` is enabled][editor-section]) |
-| `ui.statusline.select`      | Statusline mode during select mode ([only if `editor.color-modes` is enabled][editor-section]) |
-| `ui.statusline.separator`   | Separator character in statusline                                                              |
-| `ui.popup`                  | Documentation popups (e.g Space + k)                                                             |
-| `ui.popup.info`             | Prompt for multiple key options                                                                |
-| `ui.window`                 | Border lines separating splits                                                                 |
-| `ui.help`                   | Description box for commands                                                                   |
-| `ui.text`                   | Command prompts, popup text, etc.                                                              |
-| `ui.text.focus`             |                                                                                                |
-| `ui.text.info`              | The key: command text in `ui.popup.info` boxes                                                 |
-| `ui.virtual.ruler`          | Ruler columns (see the [`editor.rulers` config][editor-section])                               |
-| `ui.virtual.whitespace`     | Visible whitespace characters                                                                 |
-| `ui.virtual.indent-guide`   | Vertical indent width guides                                                                   |
-| `ui.menu`                   | Code and command completion menus                                                              |
-| `ui.menu.selected`          | Selected autocomplete item                                                                     |
-| `ui.menu.scroll`            | `fg` sets thumb color, `bg` sets track color of scrollbar                                      |
-| `ui.selection`              | For selections in the editing area                                                             |
-| `ui.selection.primary`      |                                                                                                |
-| `ui.cursorline.primary`     | The line of the primary cursor ([if cursorline is enabled][editor-section])                    |
-| `ui.cursorline.secondary`   | The lines of any other cursors ([if cursorline is enabled][editor-section])                    |
-| `ui.cursorcolumn.primary`   | The column of the primary cursor ([if cursorcolumn is enabled][editor-section])                |
-| `ui.cursorcolumn.secondary` | The columns of any other cursors ([if cursorcolumn is enabled][editor-section])                |
-| `warning`                   | Diagnostics warning (gutter)                                                                   |
-| `error`                     | Diagnostics error (gutter)                                                                     |
-| `info`                      | Diagnostics info (gutter)                                                                      |
-| `hint`                      | Diagnostics hint (gutter)                                                                      |
-| `diagnostic`                | Diagnostics fallback style (editing area)                                                      |
-| `diagnostic.hint`           | Diagnostics hint (editing area)                                                                |
-| `diagnostic.info`           | Diagnostics info (editing area)                                                                |
-| `diagnostic.warning`        | Diagnostics warning (editing area)                                                             |
-| `diagnostic.error`          | Diagnostics error (editing area)                                                               |
-
-You can check compliance to spec with
-
-```shell
-cargo xtask themelint onedark  # replace onedark with <name>
-```
+| Key                               | Notes                                                                                          |
+| ---                               | ---                                                                                            |
+| `ui.background`                   |                                                                                                |
+| `ui.background.separator`         | Picker separator below input line                                                              |
+| `ui.cursor`                       |                                                                                                |
+| `ui.cursor.normal`                |                                                                                                |
+| `ui.cursor.insert`                |                                                                                                |
+| `ui.cursor.select`                |                                                                                                |
+| `ui.cursor.match`                 | Matching bracket etc.                                                                          |
+| `ui.cursor.primary`               | Cursor with primary selection                                                                  |
+| `ui.cursor.primary.normal`        |                                                                                                |
+| `ui.cursor.primary.insert`        |                                                                                                |
+| `ui.cursor.primary.select`        |                                                                                                |
+| `ui.debug.breakpoint`             | Breakpoint indicator, found in the gutter                                                      |
+| `ui.debug.active`                 | Indicator for the line at which debugging execution is paused at, found in the gutter          |
+| `ui.gutter`                       | Gutter                                                                                         |
+| `ui.gutter.selected`              | Gutter for the line the cursor is on                                                           |
+| `ui.highlight.frameline`          | Line at which debugging execution is paused at                                                 |
+| `ui.linenr`                       | Line numbers                                                                                   |
+| `ui.linenr.selected`              | Line number for the line the cursor is on                                                      |
+| `ui.statusline`                   | Statusline                                                                                     |
+| `ui.statusline.inactive`          | Statusline (unfocused document)                                                                |
+| `ui.statusline.normal`            | Statusline mode during normal mode ([only if `editor.color-modes` is enabled][editor-section]) |
+| `ui.statusline.insert`            | Statusline mode during insert mode ([only if `editor.color-modes` is enabled][editor-section]) |
+| `ui.statusline.select`            | Statusline mode during select mode ([only if `editor.color-modes` is enabled][editor-section]) |
+| `ui.statusline.separator`         | Separator character in statusline                                                              |
+| `ui.popup`                        | Documentation popups (e.g. Space + k)                                                          |
+| `ui.popup.info`                   | Prompt for multiple key options                                                                |
+| `ui.window`                       | Borderlines separating splits                                                                  |
+| `ui.help`                         | Description box for commands                                                                   |
+| `ui.text`                         | Command prompts, popup text, etc.                                                              |
+| `ui.text.focus`                   |                                                                                                |
+| `ui.text.inactive`                | Same as `ui.text` but when the text is inactive (e.g. suggestions)                             |
+| `ui.text.info`                    | The key: command text in `ui.popup.info` boxes                                                 |
+| `ui.virtual.ruler`                | Ruler columns (see the [`editor.rulers` config][editor-section])                               |
+| `ui.virtual.whitespace`           | Visible whitespace characters                                                                  |
+| `ui.virtual.indent-guide`         | Vertical indent width guides                                                                   |
+| `ui.virtual.inlay-hint`           | Default style for inlay hints of all kinds                                                     |
+| `ui.virtual.inlay-hint.parameter` | Style for inlay hints of kind `parameter` (LSPs are not required to set a kind)                |
+| `ui.virtual.inlay-hint.type`      | Style for inlay hints of kind `type` (LSPs are not required to set a kind)                     |
+| `ui.virtual.wrap`                 | Soft-wrap indicator (see the [`editor.soft-wrap` config][editor-section])                      |
+| `ui.menu`                         | Code and command completion menus                                                              |
+| `ui.menu.selected`                | Selected autocomplete item                                                                     |
+| `ui.menu.scroll`                  | `fg` sets thumb color, `bg` sets track color of scrollbar                                      |
+| `ui.selection`                    | For selections in the editing area                                                             |
+| `ui.selection.primary`            |                                                                                                |
+| `ui.highlight`                    | Highlighted lines in the picker preview                                                        |
+| `ui.cursorline.primary`           | The line of the primary cursor ([if cursorline is enabled][editor-section])                    |
+| `ui.cursorline.secondary`         | The lines of any other cursors ([if cursorline is enabled][editor-section])                    |
+| `ui.cursorcolumn.primary`         | The column of the primary cursor ([if cursorcolumn is enabled][editor-section])                |
+| `ui.cursorcolumn.secondary`       | The columns of any other cursors ([if cursorcolumn is enabled][editor-section])                |
+| `warning`                         | Diagnostics warning (gutter)                                                                   |
+| `error`                           | Diagnostics error (gutter)                                                                     |
+| `info`                            | Diagnostics info (gutter)                                                                      |
+| `hint`                            | Diagnostics hint (gutter)                                                                      |
+| `diagnostic`                      | Diagnostics fallback style (editing area)                                                      |
+| `diagnostic.hint`                 | Diagnostics hint (editing area)                                                                |
+| `diagnostic.info`                 | Diagnostics info (editing area)                                                                |
+| `diagnostic.warning`              | Diagnostics warning (editing area)                                                             |
+| `diagnostic.error`                | Diagnostics error (editing area)                                                               |
 
 [editor-section]: ./configuration.md#editor-section

book/src/usage.md

@@ -1,22 +1,43 @@
-# Usage
+# Using Helix
 
-(Currently not fully documented, see the [keymappings](./keymap.md) list for more.)
+<!--toc:start-->
+- [Registers](#registers)
+  - [User-defined registers](#user-defined-registers)
+  - [Special registers](#special-registers)
+- [Surround](#surround)
+- [Selecting and manipulating text with textobjects](#selecting-and-manipulating-text-with-textobjects)
+- [Navigating using tree-sitter textobjects](#navigating-using-tree-sitter-textobjects)
+- [Moving the selection with syntax-aware motions](#moving-the-selection-with-syntax-aware-motions)
+<!--toc:end-->
 
-See [tutor](https://github.com/helix-editor/helix/blob/master/runtime/tutor) (accessible via `hx --tutor` or `:tutor`) for a vimtutor-like introduction.
+For a full interactive introduction to Helix, refer to the
+[tutor](https://github.com/helix-editor/helix/blob/master/runtime/tutor) which
+can be accessed via the command `hx --tutor` or `:tutor`.
+
+> 💡 Currently, not all functionality is fully documented, please refer to the
+> [key mappings](./keymap.md) list.
 
 ## Registers
 
-Vim-like registers can be used to yank and store text to be pasted later. Usage is similar, with `"` being used to select a register:
+In Helix, registers are storage locations for text and other data, such as the
+result of a search. Registers can be used to cut, copy, and paste text, similar
+to the clipboard in other text editors. Usage is similar to Vim, with `"` being
+used to select a register.
+
+### User-defined registers
+
+Helix allows you to create your own named registers for storing text, for
+example:
 
 - `"ay` - Yank the current selection to register `a`.
 - `"op` - Paste the text in register `o` after the selection.
 
-If there is a selected register before invoking a change or delete command, the selection will be stored in the register and the action will be carried out:
+If a register is selected before invoking a change or delete command, the selection will be stored in the register and the action will be carried out:
 
 - `"hc` - Store the selection in register `h` and then change it (delete and enter insert mode).
 - `"md` - Store the selection in register `m` and delete it.
 
-### Special Registers
+### Special registers
 
 | Register character | Contains              |
 | ---                | ---                   |
@@ -25,41 +46,90 @@ If there is a selected register before invoking a change or delete command, the
 | `"`                | Last yanked text      |
 | `_`                | Black hole            |
 
-> There is no special register for copying to system clipboard, instead special commands and keybindings are provided. See the [keymap](keymap.md#space-mode) for the specifics.
-> The black hole register works as a no-op register, meaning no data will be written to / read from it.
+The system clipboard is not directly supported by a special register. Instead, special commands and keybindings are provided. Refer to the
+[key map](keymap.md#space-mode) for more details.
+
+The black hole register is a no-op register, meaning that no data will be read or written to it.
 
 ## Surround
 
-Functionality similar to [vim-surround](https://github.com/tpope/vim-surround) is built into
-helix. The keymappings have been inspired from [vim-sandwich](https://github.com/machakann/vim-sandwich):
+Helix includes built-in functionality similar to [vim-surround](https://github.com/tpope/vim-surround).
+The keymappings have been inspired from [vim-sandwich](https://github.com/machakann/vim-sandwich):
 
-![surround demo](https://user-images.githubusercontent.com/23398472/122865801-97073180-d344-11eb-8142-8f43809982c6.gif)
+![Surround demo](https://user-images.githubusercontent.com/23398472/122865801-97073180-d344-11eb-8142-8f43809982c6.gif)
 
-- `ms` - Add surround characters
-- `mr` - Replace surround characters
-- `md` - Delete surround characters
+| Key Sequence                      | Action                                  |
+| --------------------------------- | --------------------------------------- |
+| `ms<char>` (after selecting text) | Add surround characters to selection    |
+| `mr<char_to_replace><new_char>`   | Replace the closest surround characters |
+| `md<char_to_delete>`              | Delete the closest surround characters  |
 
-`ms` acts on a selection, so select the text first and use `ms<char>`. `mr` and `md` work
-on the closest pairs found and selections are not required; use counts to act in outer pairs.
+You can use counts to act on outer pairs.
 
-It can also act on multiple selections (yay!). For example, to change every occurrence of `(use)` to `[use]`:
+Surround can also act on multiple selections. For example, to change every occurrence of `(use)` to `[use]`:
 
-- `%` to select the whole file
-- `s` to split the selections on a search term
-- Input `use` and hit Enter
-- `mr([` to replace the parens with square brackets
+1. `%` to select the whole file
+2. `s` to split the selections on a search term
+3. Input `use` and hit Enter
+4. `mr([` to replace the parentheses with square brackets
 
-Multiple characters are currently not supported, but planned.
+Multiple characters are currently not supported, but planned for future release.
 
-## Syntax-tree Motions
+## Selecting and manipulating text with textobjects
 
-`Alt-p`, `Alt-o`, `Alt-i`, and `Alt-n` (or `Alt` and arrow keys) move the primary
-selection according to the selection's place in the syntax tree. Let's walk
-through an example to get familiar with them. Many languages have a syntax like
-so for function calls:
+In Helix, textobjects are a way to select, manipulate and operate on a piece of
+text in a structured way. They allow you to refer to blocks of text based on
+their structure or purpose, such as a word, sentence, paragraph, or even a
+function or block of code.
 
-```
-func(arg1, arg2, arg3)
+![Textobject demo](https://user-images.githubusercontent.com/23398472/124231131-81a4bb00-db2d-11eb-9d10-8e577ca7b177.gif)
+![Textobject tree-sitter demo](https://user-images.githubusercontent.com/23398472/132537398-2a2e0a54-582b-44ab-a77f-eb818942203d.gif)
+
+- `ma` - Select around the object (`va` in Vim, `<alt-a>` in Kakoune)
+- `mi` - Select inside the object (`vi` in Vim, `<alt-i>` in Kakoune)
+
+| Key after `mi` or `ma` | Textobject selected      |
+| ---                    | ---                      |
+| `w`                    | Word                     |
+| `W`                    | WORD                     |
+| `p`                    | Paragraph                |
+| `(`, `[`, `'`, etc.    | Specified surround pairs |
+| `m`                    | The closest surround pair    |
+| `f`                    | Function                 |
+| `c`                    | Class                    |
+| `a`                    | Argument/parameter       |
+| `o`                    | Comment                  |
+| `t`                    | Test                     |
+| `g`                    | Change                   |
+
+> 💡 `f`, `c`, etc. need a tree-sitter grammar active for the current
+document and a special tree-sitter query file to work properly. [Only
+some grammars][lang-support] currently have the query file implemented.
+Contributions are welcome!
+
+## Navigating using tree-sitter textobjects
+
+Navigating between functions, classes, parameters, and other elements is
+possible using tree-sitter and textobject queries. For
+example to move to the next function use `]f`, to move to previous
+class use `[c`, and so on.
+
+![Tree-sitter-nav-demo][tree-sitter-nav-demo]
+
+For the full reference see the [unimpaired][unimpaired-keybinds] section of the key bind
+documentation.
+
+> 💡 This feature relies on tree-sitter textobjects
+> and requires the corresponding query file to work properly.
+
+## Moving the selection with syntax-aware motions
+
+`Alt-p`, `Alt-o`, `Alt-i`, and `Alt-n` (or `Alt` and arrow keys) allow you to move the 
+selection according to its location in the syntax tree. For example, many languages have the
+following syntax for function calls:
+
+```js
+func(arg1, arg2, arg3);
 ```
 
 A function call might be parsed by tree-sitter into a tree like the following.
@@ -93,77 +163,29 @@ a more intuitive tree format:
    └──────────┘  └──────────┘  └──────────┘
 ```
 
-Say we have a selection that wraps `arg1`. The selection is on the `arg1` leaf
-in the tree above.
+If you have a selection that wraps `arg1` (see the tree above), and you use
+`Alt-n`, it will select the next sibling in the syntax tree: `arg2`.
 
-```
+```js
+// before
 func([arg1], arg2, arg3)
+// after
+func(arg1, [arg2], arg3);
 ```
 
-Using `Alt-n` would select the next sibling in the syntax tree: `arg2`.
+Similarly, `Alt-o` will expand the selection to the parent node, in this case, the
+arguments node.
 
-```
-func(arg1, [arg2], arg3)
-```
-
-While `Alt-o` would expand the selection to the parent node. In the tree above we
-can see that we would select the `arguments` node.
-
-```
-func[(arg1, arg2, arg3)]
+```js
+func[(arg1, arg2, arg3)];
 ```
 
 There is also some nuanced behavior that prevents you from getting stuck on a
-node with no sibling. If we have a selection on `arg1`, `Alt-p` would bring us
-to the previous child node. Since `arg1` doesn't have a sibling to its left,
-though, we climb the syntax tree and then take the previous selection. So
-`Alt-p` will move the selection over to the "func" `identifier`.
-
-```
-[func](arg1, arg2, arg3)
-```
-
-## Textobjects
-
-![textobject-demo](https://user-images.githubusercontent.com/23398472/124231131-81a4bb00-db2d-11eb-9d10-8e577ca7b177.gif)
-![textobject-treesitter-demo](https://user-images.githubusercontent.com/23398472/132537398-2a2e0a54-582b-44ab-a77f-eb818942203d.gif)
-
-- `ma` - Select around the object (`va` in Vim, `<alt-a>` in Kakoune)
-- `mi` - Select inside the object (`vi` in Vim, `<alt-i>` in Kakoune)
-
-| Key after `mi` or `ma` | Textobject selected      |
-| ---                    | ---                      |
-| `w`                    | Word                     |
-| `W`                    | WORD                     |
-| `p`                    | Paragraph                |
-| `(`, `[`, `'`, etc     | Specified surround pairs |
-| `m`                    | Closest surround pair    |
-| `f`                    | Function                 |
-| `c`                    | Class                    |
-| `a`                    | Argument/parameter       |
-| `o`                    | Comment                  |
-| `t`                    | Test                     |
-| `g`                    | Change                   |
-
-> NOTE: `f`, `c`, etc need a tree-sitter grammar active for the current
-document and a special tree-sitter query file to work properly. [Only
-some grammars][lang-support] currently have the query file implemented.
-Contributions are welcome!
-
-## Tree-sitter Textobject Based Navigation
-
-Navigating between functions, classes, parameters, etc is made
-possible by leveraging tree-sitter and textobjects queries. For
-example to move to the next function use `]f`, to move to previous
-class use `[c`, and so on.
-
-![tree-sitter-nav-demo][tree-sitter-nav-demo]
-
-See the [unimpaired][unimpaired-keybinds] section of the keybind
-documentation for the full reference.
-
-> NOTE: This feature is dependent on tree-sitter based textobjects
-and therefore requires the corresponding query file to work properly.
+node with no sibling. When using `Alt-p` with a selection on `arg1`, the previous
+child node will be selected. In the event that `arg1` does not have a previous
+sibling, the selection will move up the syntax tree and select the previous
+element. As a result, using `Alt-p` with a selection on `arg1` will move the
+selection to the "func" `identifier`.
 
 [lang-support]: ./lang-support.md
 [unimpaired-keybinds]: ./keymap.md#unimpaired

book/theme/css/variables.css

@@ -48,6 +48,18 @@
     --searchresults-border-color: #888;
     --searchresults-li-bg: #252932;
     --search-mark-bg: #e3b171;
+    --hljs-background: #191f26;
+    --hljs-color: #e6e1cf;
+    --hljs-quote: #5c6773;
+    --hljs-variable: #ff7733;
+    --hljs-type: #ffee99;
+    --hljs-title: #b8cc52;
+    --hljs-symbol: #ffb454;
+    --hljs-selector-tag: #ff7733;
+    --hljs-selector-tag: #36a3d9;
+    --hljs-selector-tag: #00568d;
+    --hljs-selector-tag: #91b362;
+    --hljs-selector-tag: #d96c75;
 }
 
 .coal {
@@ -88,6 +100,18 @@
     --searchresults-border-color: #98a3ad;
     --searchresults-li-bg: #2b2b2f;
     --search-mark-bg: #355c7d;
+    --hljs-background: #969896;
+    --hljs-color: #cc6666;
+    --hljs-quote: #de935f;
+    --hljs-variable: #f0c674;
+    --hljs-type: #b5bd68;
+    --hljs-title: #8abeb7;
+    --hljs-symbol: #81a2be;
+    --hljs-selector-tag: #b294bb;
+    --hljs-selector-tag: #1d1f21;
+    --hljs-selector-tag: #c5c8c6;
+    --hljs-selector-tag: #718c00;
+    --hljs-selector-tag: #c82829;
 }
 
 .light {
@@ -128,6 +152,14 @@
     --searchresults-border-color: #888;
     --searchresults-li-bg: #e4f2fe;
     --search-mark-bg: #a2cff5;
+    --hljs-background: #f6f7f6;
+    --hljs-color: #000;
+    --hljs-quote: #575757;
+    --hljs-variable: #d70025;
+    --hljs-type: #b21e00;
+    --hljs-title: #0030f2;
+    --hljs-symbol: #008200;
+    --hljs-selector-tag: #9d00ec;
 }
 
 .navy {
@@ -168,6 +200,19 @@
     --searchresults-border-color: #5c5c68;
     --searchresults-li-bg: #242430;
     --search-mark-bg: #a2cff5;
+
+    --hljs-background: #969896;
+    --hljs-color: #cc6666;
+    --hljs-quote: #de935f;
+    --hljs-variable: #f0c674;
+    --hljs-type: #b5bd68;
+    --hljs-title: #8abeb7;
+    --hljs-symbol: #81a2be;
+    --hljs-selector-tag: #b294bb;
+    --hljs-selector-tag: #1d1f21;
+    --hljs-selector-tag: #c5c8c6;
+    --hljs-selector-tag: #718c00;
+    --hljs-selector-tag: #c82829;
 }
 
 .rust {
@@ -208,6 +253,14 @@
     --searchresults-border-color: #888;
     --searchresults-li-bg: #dec2a2;
     --search-mark-bg: #e69f67;
+    --hljs-background: #f6f7f6;
+    --hljs-color: #000;
+    --hljs-quote: #575757;
+    --hljs-variable: #d70025;
+    --hljs-type: #b21e00;
+    --hljs-title: #0030f2;
+    --hljs-symbol: #008200;
+    --hljs-selector-tag: #9d00ec;
 }
 
 @media (prefers-color-scheme: dark) {
@@ -292,7 +345,15 @@
     --searchresults-header-fg: #5f5f71;
     --searchresults-border-color: #5c5c68;
     --searchresults-li-bg: #242430;
-    --search-mark-bg: #a2cff5;
+    --search-mark-bg: #acff5;
+    --hljs-background: #2f1e2e;
+    --hljs-color: #a39e9b;
+    --hljs-quote: #8d8687;
+    --hljs-variable: #ef6155;
+    --hljs-type: #f99b15;
+    --hljs-title: #fec418;
+    --hljs-symbol: #48b685;
+    --hljs-selector-tag: #815ba4;
 }
 
 .colibri {
@@ -338,5 +399,13 @@
     --searchresults-border-color: #5c5c68;
     --searchresults-li-bg: #242430;
     --search-mark-bg: #a2cff5;
+    --hljs-background: #TODO;
+    --hljs-color: #TODO;
+    --hljs-quote: #TODO;
+    --hljs-variable: #TODO;
+    --hljs-type: #TODO;
+    --hljs-title: #TODO;
+    --hljs-symbol: #TODO;
+    --hljs-selector-tag: #TODO;
 */
 }

book/theme/highlight.css

@@ -7,12 +7,12 @@ code.hljs {
  padding:3px 5px
 }
 .hljs {
- background:#2f1e2e;
- color:#a39e9b
+ background: var(--hljs-background);
+ color: var(--hljs-color);
 }
 .hljs-comment,
 .hljs-quote {
- color:#8d8687
+ color: var(--hljs-quote)
 }
 .hljs-link,
 .hljs-meta,
@@ -23,7 +23,7 @@ code.hljs {
 .hljs-tag,
 .hljs-template-variable,
 .hljs-variable {
- color:#ef6155
+ color: var(--hljs-variable)
 }
 .hljs-built_in,
 .hljs-deletion,
@@ -31,22 +31,22 @@ code.hljs {
 .hljs-number,
 .hljs-params,
 .hljs-type {
- color:#f99b15
+ color: var(--hljs-type)
 }
 .hljs-attribute,
 .hljs-section,
 .hljs-title {
- color:#fec418
+ color: var(--hljs-title)
 }
 .hljs-addition,
 .hljs-bullet,
 .hljs-string,
 .hljs-symbol {
- color:#48b685
+ color: var(--hljs-symbol)
 }
 .hljs-keyword,
 .hljs-selector-tag {
- color:#815ba4
+ color: var(--hljs-selector-tag)
 }
 .hljs-emphasis {
  font-style:italic

contrib/Helix.appdata.xml

@@ -0,0 +1,93 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<component type="desktop-application">
+  <id>com.helix_editor.Helix</id>
+  <metadata_license>CC0-1.0</metadata_license>
+  <project_license>MPL-2.0</project_license>
+  <name>Helix</name>
+  <summary>A post-modern text editor</summary>
+
+  <description>
+    <p>
+      Helix is a terminal-based text editor inspired by Kakoune / Neovim and written in Rust.
+    </p>
+    <ul>
+      <li>Vim-like modal editing</li>
+      <li>Multiple selections</li>
+      <li>Built-in language server support</li>
+      <li>Smart, incremental syntax highlighting and code editing via tree-sitter</li>
+    </ul>
+  </description>
+
+  <launchable type="desktop-id">Helix.desktop</launchable>
+
+  <screenshots>
+    <screenshot type="default">
+      <caption>Helix with default theme</caption>
+      <image>https://github.com/helix-editor/helix/raw/d4565b4404cabc522bd60822abd374755581d751/screenshot.png</image>
+    </screenshot>
+  </screenshots>
+
+  <url type="homepage">https://helix-editor.com/</url>
+  <url type="donation">https://opencollective.com/helix-editor</url>
+  <url type="help">https://docs.helix-editor.com/</url>
+  <url type="vcs-browser">https://github.com/helix-editor/helix</url>
+  <url type="bugtracker">https://github.com/helix-editor/helix/issues</url>
+
+  <content_rating type="oars-1.1" />
+
+  <releases>
+    <release version="23.05" date="2023-05-18">
+      <url>https://github.com/helix-editor/helix/releases/tag/23.05</url>
+    </release>
+    <release version="23.03" date="2023-03-31">
+      <url>https://helix-editor.com/news/release-23-03-highlights/</url>
+    </release>
+    <release version="22.12" date="2022-12-6">
+      <url>https://helix-editor.com/news/release-22-12-highlights/</url>
+    </release>
+    <release version="22.08" date="2022-8-31">
+      <url>https://helix-editor.com/news/release-22-08-highlights/</url>
+    </release>
+    <release version="22.05" date="2022-5-28">
+      <url>https://helix-editor.com/news/release-22-05-highlights/</url>
+    </release>
+    <release version="22.03" date="2022-3-28">
+      <url>https://helix-editor.com/news/release-22-03-highlights/</url>
+    </release>
+  </releases>
+
+  <requires>
+    <control>keyboard</control>
+  </requires>
+
+  <categories>
+    <category>Utility</category>
+    <category>TextEditor</category>
+  </categories>
+
+  <keywords>
+    <keyword>text</keyword>
+    <keyword>editor</keyword>
+    <keyword>development</keyword>
+    <keyword>programming</keyword>
+  </keywords>
+
+  <provides>
+    <binary>hx</binary>
+    <mediatype>text/english</mediatype>
+    <mediatype>text/plain</mediatype>
+    <mediatype>text/x-makefile</mediatype>
+    <mediatype>text/x-c++hdr</mediatype>
+    <mediatype>text/x-c++src</mediatype>
+    <mediatype>text/x-chdr</mediatype>
+    <mediatype>text/x-csrc</mediatype>
+    <mediatype>text/x-java</mediatype>
+    <mediatype>text/x-moc</mediatype>
+    <mediatype>text/x-pascal</mediatype>
+    <mediatype>text/x-tcl</mediatype>
+    <mediatype>text/x-tex</mediatype>
+    <mediatype>application/x-shellscript</mediatype>
+    <mediatype>text/x-c</mediatype>
+    <mediatype>text/x-c++</mediatype>
+  </provides>
+</component>

docs/CONTRIBUTING.md

@@ -8,7 +8,10 @@ Some suggestions to get started:
 - Help with packaging on various distributions needed!
 - To use print debugging to the [Helix log file][log-file], you must:
   * Print using `log::info!`, `warn!`, or `error!`. (`log::info!("helix!")`)
-  * Pass the appropriate verbosity level option for the desired log level. (`hx -v <file>` for info, more `v`s for higher severity inclusive)
+  * Pass the appropriate verbosity level option for the desired log level. (`hx -v <file>` for info, more `v`s for higher verbosity)
+  * Want to display the logs in a separate file instead of using the `:log-open` command in your compiled Helix editor? Start your debug version with `cargo run -- --log foo.log` and in a new terminal use `tail -f foo.log`
+- Instead of running a release version of Helix, while developing you may want to run in debug mode with `cargo run` which is way faster to compile
+- Looking for even faster compile times? Give a try to [mold](https://github.com/rui314/mold)
 - If your preferred language is missing, integrating a tree-sitter grammar for
     it and defining syntax highlight queries for it is straight forward and
     doesn't require much knowledge of the internals.
@@ -30,7 +33,13 @@ inside the project. We use [xtask][xtask] as an ad-hoc task runner and
 thus do not require any dependencies other than `cargo` (You don't have
 to `cargo install` anything either).
 
-# Integration tests
+# Testing
+
+## Unit tests/Documentation tests
+
+Run `cargo test --workspace` to run unit tests and documentation tests in all packages.
+
+## Integration tests
 
 Integration tests for helix-term can be run with `cargo integration-test`. Code
 contributors are strongly encouraged to write integration tests for their code.

docs/releases.md

@@ -5,6 +5,7 @@ Helix releases are versioned in the Calendar Versioning scheme:
 we'll use `<tag>` as a placeholder for the tag being published.
 
 * Merge the changelog PR
+* Add new `<release>` entry in `contrib/Helix.appdata.xml` with release information according to the [AppStream spec](https://www.freedesktop.org/software/appstream/docs/sect-Metadata-Releases.html)
 * Tag and push
     * `git tag -s -m "<tag>" -a <tag> && git push`
     * Make sure to switch to master and pull first

flake.lock

@@ -3,47 +3,34 @@
     "crane": {
       "flake": false,
       "locked": {
-        "lastModified": 1661875961,
-        "narHash": "sha256-f1h/2c6Teeu1ofAHWzrS8TwBPcnN+EEu+z1sRVmMQTk=",
+        "lastModified": 1681175776,
+        "narHash": "sha256-7SsUy9114fryHAZ8p1L6G6YSu7jjz55FddEwa2U8XZc=",
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "d9f394e4e20e97c2a60c3ad82c2b6ef99be19e24",
+        "rev": "445a3d222947632b5593112bb817850e8a9cf737",
         "type": "github"
       },
       "original": {
         "owner": "ipetkov",
+        "ref": "v0.12.1",
         "repo": "crane",
         "type": "github"
       }
     },
-    "devshell": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1667210711,
-        "narHash": "sha256-IoErjXZAkzYWHEpQqwu/DeRNJGFdR7X2OGbkhMqMrpw=",
-        "owner": "numtide",
-        "repo": "devshell",
-        "rev": "96a9dd12b8a447840cc246e17a47b81a4268bba7",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "devshell",
-        "type": "github"
-      }
-    },
     "dream2nix": {
       "inputs": {
-        "alejandra": [
-          "nci"
-        ],
         "all-cabal-json": [
           "nci"
         ],
         "crane": "crane",
         "devshell": [
+          "nci"
+        ],
+        "drv-parts": "drv-parts",
+        "flake-compat": "flake-compat",
+        "flake-parts": [
           "nci",
-          "devshell"
+          "parts"
         ],
         "flake-utils-pre-commit": [
           "nci"
@@ -57,23 +44,30 @@
         "mach-nix": [
           "nci"
         ],
+        "nix-pypi-fetcher": [
+          "nci"
+        ],
         "nixpkgs": [
           "nci",
           "nixpkgs"
         ],
+        "nixpkgsV1": "nixpkgsV1",
         "poetry2nix": [
           "nci"
         ],
         "pre-commit-hooks": [
           "nci"
+        ],
+        "pruned-racket-catalog": [
+          "nci"
         ]
       },
       "locked": {
-        "lastModified": 1668851003,
-        "narHash": "sha256-X7RCQQynbxStZR2m7HW38r/msMQwVl3afD6UXOCtvx4=",
+        "lastModified": 1683212002,
+        "narHash": "sha256-EObtqyQsv9v+inieRY5cvyCMCUI5zuU5qu+1axlJCPM=",
         "owner": "nix-community",
         "repo": "dream2nix",
-        "rev": "c77e8379d8fe01213ba072e40946cbfb7b58e628",
+        "rev": "fbfb09d2ab5ff761d822dd40b4a1def81651d096",
         "type": "github"
       },
       "original": {
@@ -82,13 +76,64 @@
         "type": "github"
       }
     },
-    "flake-utils": {
+    "drv-parts": {
+      "inputs": {
+        "flake-compat": [
+          "nci",
+          "dream2nix",
+          "flake-compat"
+        ],
+        "flake-parts": [
+          "nci",
+          "dream2nix",
+          "flake-parts"
+        ],
+        "nixpkgs": [
+          "nci",
+          "dream2nix",
+          "nixpkgs"
+        ]
+      },
       "locked": {
-        "lastModified": 1659877975,
-        "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
+        "lastModified": 1680698112,
+        "narHash": "sha256-FgnobN/DvCjEsc0UAZEAdPLkL4IZi2ZMnu2K2bUaElc=",
+        "owner": "davhau",
+        "repo": "drv-parts",
+        "rev": "e8c2ec1157dc1edb002989669a0dbd935f430201",
+        "type": "github"
+      },
+      "original": {
+        "owner": "davhau",
+        "repo": "drv-parts",
+        "type": "github"
+      }
+    },
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1673956053,
+        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1681202837,
+        "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
+        "rev": "cfacdce06f30d2b68473a46042957675eebb3401",
         "type": "github"
       },
       "original": {
@@ -97,23 +142,40 @@
         "type": "github"
       }
     },
+    "mk-naked-shell": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1681286841,
+        "narHash": "sha256-3XlJrwlR0nBiREnuogoa5i1b4+w/XPe0z8bbrJASw0g=",
+        "owner": "yusdacra",
+        "repo": "mk-naked-shell",
+        "rev": "7612f828dd6f22b7fb332cc69440e839d7ffe6bd",
+        "type": "github"
+      },
+      "original": {
+        "owner": "yusdacra",
+        "repo": "mk-naked-shell",
+        "type": "github"
+      }
+    },
     "nci": {
       "inputs": {
-        "devshell": "devshell",
         "dream2nix": "dream2nix",
+        "mk-naked-shell": "mk-naked-shell",
         "nixpkgs": [
           "nixpkgs"
         ],
+        "parts": "parts",
         "rust-overlay": [
           "rust-overlay"
         ]
       },
       "locked": {
-        "lastModified": 1669011203,
-        "narHash": "sha256-Lymj4HktNEFmVXtwI0Os7srDXHZbZW0Nzw3/+5Hf8ko=",
+        "lastModified": 1683699050,
+        "narHash": "sha256-UWKQpzVcSshB+sU2O8CCHjOSTQrNS7Kk9V3+UeBsJpg=",
         "owner": "yusdacra",
         "repo": "nix-cargo-integration",
-        "rev": "c5133b91fc1d549087c91228bd213f2518728a4b",
+        "rev": "ed27173cd1b223f598343ea3c15aacb1d140feac",
         "type": "github"
       },
       "original": {
@@ -124,11 +186,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1668905981,
-        "narHash": "sha256-RBQa/+9Uk1eFTqIOXBSBezlEbA3v5OkgP+qptQs1OxY=",
+        "lastModified": 1683408522,
+        "narHash": "sha256-9kcPh6Uxo17a3kK3XCHhcWiV1Yu1kYj22RHiymUhMkU=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "690ffff026b4e635b46f69002c0f4e81c65dfc2e",
+        "rev": "897876e4c484f1e8f92009fd11b7d988a121a4e7",
         "type": "github"
       },
       "original": {
@@ -138,10 +200,83 @@
         "type": "github"
       }
     },
+    "nixpkgs-lib": {
+      "locked": {
+        "dir": "lib",
+        "lastModified": 1682879489,
+        "narHash": "sha256-sASwo8gBt7JDnOOstnps90K1wxmVfyhsTPPNTGBPjjg=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "da45bf6ec7bbcc5d1e14d3795c025199f28e0de0",
+        "type": "github"
+      },
+      "original": {
+        "dir": "lib",
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgsV1": {
+      "locked": {
+        "lastModified": 1678500271,
+        "narHash": "sha256-tRBLElf6f02HJGG0ZR7znMNFv/Uf7b2fFInpTHiHaSE=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "5eb98948b66de29f899c7fe27ae112a47964baf8",
+        "type": "github"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "ref": "nixos-22.11",
+        "type": "indirect"
+      }
+    },
+    "parts": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "nci",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1683560683,
+        "narHash": "sha256-XAygPMN5Xnk/W2c1aW0jyEa6lfMDZWlQgiNtmHXytPc=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "006c75898cf814ef9497252b022e91c946ba8e17",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "parts_2": {
+      "inputs": {
+        "nixpkgs-lib": "nixpkgs-lib"
+      },
+      "locked": {
+        "lastModified": 1683560683,
+        "narHash": "sha256-XAygPMN5Xnk/W2c1aW0jyEa6lfMDZWlQgiNtmHXytPc=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "006c75898cf814ef9497252b022e91c946ba8e17",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
     "root": {
       "inputs": {
         "nci": "nci",
         "nixpkgs": "nixpkgs",
+        "parts": "parts_2",
         "rust-overlay": "rust-overlay"
       }
     },
@@ -153,11 +288,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1668998422,
-        "narHash": "sha256-G/BklIplCHZEeDIabaaxqgITdIXtMolRGlwxn9jG2/Q=",
+        "lastModified": 1683771545,
+        "narHash": "sha256-we0GYcKTo2jRQGmUGrzQ9VH0OYAUsJMCsK8UkF+vZUA=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "68ab029c93f8f8eed4cf3ce9a89a9fd4504b2d6e",
+        "rev": "c57e210faf68e5d5386f18f1b17ad8365d25e4ed",
         "type": "github"
       },
       "original": {
@@ -165,6 +300,21 @@
         "repo": "rust-overlay",
         "type": "github"
       }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
     }
   },
   "root": "root",

flake.nix

@@ -12,16 +12,10 @@
       inputs.nixpkgs.follows = "nixpkgs";
       inputs.rust-overlay.follows = "rust-overlay";
     };
+    parts.url = "github:hercules-ci/flake-parts";
   };
 
-  outputs = {
-    self,
-    nixpkgs,
-    nci,
-    ...
-  }: let
-    lib = nixpkgs.lib;
-    ncl = nci.lib.nci-lib;
+  outputs = inp: let
     mkRootPath = rel:
       builtins.path {
         path = "${toString ./.}/${rel}";
@@ -32,6 +26,12 @@
         ".envrc"
         ".ignore"
         ".github"
+        ".gitignore"
+        "logo.svg"
+        "logo_dark.svg"
+        "logo_light.svg"
+        "rust-toolchain.toml"
+        "rustfmt.toml"
         "runtime"
         "screenshot.png"
         "book"
@@ -46,6 +46,7 @@
         "flake.lock"
       ];
       ignorePaths = path: type: let
+        inherit (inp.nixpkgs) lib;
         # split the nix store path into its components
         components = lib.splitString "/" path;
         # drop off the `/nix/hash-source` section from the path
@@ -61,116 +62,104 @@
         # filter out unnecessary paths
         filter = ignorePaths;
       };
-    outputs = nci.lib.makeOutputs {
-      root = ./.;
-      config = common: {
-        outputs = {
-          # rename helix-term to helix since it's our main package
-          rename = {"helix-term" = "helix";};
-          # Set default app to hx (binary is from helix-term release build)
-          # Set default package to helix-term release build
-          defaults = {
-            app = "hx";
-            package = "helix";
-          };
-        };
-        cCompiler.package = with common.pkgs;
-          if stdenv.isLinux
-          then gcc
-          else clang;
-        shell = {
-          packages = with common.pkgs;
-            [lld_13 cargo-flamegraph rust-analyzer]
-            ++ (lib.optional (stdenv.isx86_64 && stdenv.isLinux) cargo-tarpaulin)
-            ++ (lib.optional stdenv.isLinux lldb);
-          env = [
-            {
-              name = "HELIX_RUNTIME";
-              eval = "$PWD/runtime";
-            }
-            {
-              name = "RUST_BACKTRACE";
-              value = "1";
-            }
-            {
-              name = "RUSTFLAGS";
-              value =
-                if common.pkgs.stdenv.isLinux
-                then "-C link-arg=-fuse-ld=lld -C target-cpu=native -Clink-arg=-Wl,--no-rosegment"
-                else "";
-            }
-          ];
-        };
-      };
-      pkgConfig = common: {
-        helix-term = {
-          # Wrap helix with runtime
-          wrapper = _: old: let
-            inherit (common) pkgs;
-            makeOverridableHelix = old: config: let
-              grammars = pkgs.callPackage ./grammars.nix config;
-              runtimeDir = pkgs.runCommand "helix-runtime" {} ''
-                mkdir -p $out
-                ln -s ${mkRootPath "runtime"}/* $out
-                rm -r $out/grammars
-                ln -s ${grammars} $out/grammars
-              '';
-              helix-wrapped =
-                common.internal.pkgsSet.utils.wrapDerivation old
-                {
-                  nativeBuildInputs = [pkgs.makeWrapper];
-                  makeWrapperArgs = config.makeWrapperArgs or [];
-                }
-                ''
-                  rm -rf $out/bin
-                  mkdir -p $out/bin
-                  ln -sf ${old}/bin/* $out/bin/
-                  wrapProgram "$out/bin/hx" ''${makeWrapperArgs[@]} --set HELIX_RUNTIME "${runtimeDir}"
-                '';
-            in
-              helix-wrapped
-              // {override = makeOverridableHelix old;};
-          in
-            makeOverridableHelix old {};
-          overrides.fix-build.overrideAttrs = prev: {
-            src = filteredSource;
-
-            # disable fetching and building of tree-sitter grammars in the helix-term build.rs
-            HELIX_DISABLE_AUTO_GRAMMAR_BUILD = "1";
-
-            buildInputs = ncl.addBuildInputs prev [common.config.cCompiler.package.cc.lib];
-
-            # link languages and theme toml files since helix-term expects them (for tests)
-            preConfigure = ''
-              ${prev.preConfigure or ""}
-              ${
-                lib.concatMapStringsSep
-                "\n"
-                (path: "ln -sf ${mkRootPath path} ..")
-                ["languages.toml" "theme.toml" "base16_theme.toml"]
-              }
-            '';
-            checkPhase = ":";
-
-            meta.mainProgram = "hx";
-          };
-        };
-      };
-    };
   in
-    outputs
-    // {
-      packages =
-        lib.mapAttrs
-        (
-          system: packages:
-            packages
-            // {
-              helix-unwrapped = packages.helix.passthru.unwrapped;
-              helix-unwrapped-dev = packages.helix-dev.passthru.unwrapped;
+    inp.parts.lib.mkFlake {inputs = inp;} {
+      imports = [inp.nci.flakeModule];
+      systems = [
+        "x86_64-linux"
+        "x86_64-darwin"
+        "aarch64-linux"
+        "aarch64-darwin"
+        "i686-linux"
+      ];
+      perSystem = {
+        config,
+        pkgs,
+        lib,
+        ...
+      }: let
+        makeOverridableHelix = old: config: let
+          grammars = pkgs.callPackage ./grammars.nix config;
+          runtimeDir = pkgs.runCommand "helix-runtime" {} ''
+            mkdir -p $out
+            ln -s ${mkRootPath "runtime"}/* $out
+            rm -r $out/grammars
+            ln -s ${grammars} $out/grammars
+          '';
+          helix-wrapped =
+            pkgs.runCommand
+            old.name
+            {
+              inherit (old) pname version;
+              meta = old.meta or {};
+              passthru =
+                (old.passthru or {})
+                // {
+                  unwrapped = old;
+                };
+              nativeBuildInputs = [pkgs.makeWrapper];
+              makeWrapperArgs = config.makeWrapperArgs or [];
             }
-        )
-        outputs.packages;
+            ''
+              cp -rs --no-preserve=mode,ownership ${old} $out
+              wrapProgram "$out/bin/hx" ''${makeWrapperArgs[@]} --set HELIX_RUNTIME "${runtimeDir}"
+            '';
+        in
+          helix-wrapped
+          // {
+            override = makeOverridableHelix old;
+            passthru =
+              helix-wrapped.passthru
+              // {
+                wrapper = old: makeOverridableHelix old config;
+              };
+          };
+        stdenv =
+          if pkgs.stdenv.isLinux
+          then pkgs.stdenv
+          else pkgs.clangStdenv;
+        rustFlagsEnv =
+          if stdenv.isLinux
+          then ''$RUSTFLAGS -C link-arg=-fuse-ld=lld -C target-cpu=native -Clink-arg=-Wl,--no-rosegment''
+          else "$RUSTFLAGS";
+      in {
+        nci.projects."helix-project".relPath = "";
+        nci.crates."helix-term" = {
+          overrides = {
+            add-meta.override = _: {meta.mainProgram = "hx";};
+            add-inputs.overrideAttrs = prev: {
+              buildInputs = (prev.buildInputs or []) ++ [stdenv.cc.cc.lib];
+            };
+            disable-grammar-builds = {
+              # disable fetching and building of tree-sitter grammars in the helix-term build.rs
+              HELIX_DISABLE_AUTO_GRAMMAR_BUILD = "1";
+            };
+            disable-tests = {checkPhase = ":";};
+            set-stdenv.override = _: {inherit stdenv;};
+            set-filtered-src.override = _: {src = filteredSource;};
+          };
+        };
+
+        packages.helix-unwrapped = config.nci.outputs."helix-term".packages.release;
+        packages.helix-unwrapped-dev = config.nci.outputs."helix-term".packages.dev;
+        packages.helix = makeOverridableHelix config.packages.helix-unwrapped {};
+        packages.helix-dev = makeOverridableHelix config.packages.helix-unwrapped-dev {};
+        packages.default = config.packages.helix;
+
+        devShells.default = config.nci.outputs."helix-project".devShell.overrideAttrs (old: {
+          nativeBuildInputs =
+            (old.nativeBuildInputs or [])
+            ++ (with pkgs; [lld_13 cargo-flamegraph rust-analyzer])
+            ++ (lib.optional (stdenv.isx86_64 && stdenv.isLinux) pkgs.cargo-tarpaulin)
+            ++ (lib.optional stdenv.isLinux pkgs.lldb)
+            ++ (lib.optional stdenv.isDarwin pkgs.darwin.apple_sdk.frameworks.CoreFoundation);
+          shellHook = ''
+            export HELIX_RUNTIME="$PWD/runtime"
+            export RUST_BACKTRACE="1"
+            export RUSTFLAGS="${rustFlagsEnv}"
+          '';
+        });
+      };
     };
 
   nixConfig = {

helix-core/Cargo.toml

@@ -17,7 +17,7 @@ integration = []
 [dependencies]
 helix-loader = { version = "0.6", path = "../helix-loader" }
 
-ropey = { version = "1.5.1-alpha", default-features = false, features = ["simd"] }
+ropey = { version = "1.6.0", default-features = false, features = ["simd"] }
 smallvec = "1.10"
 smartstring = "1.0.1"
 unicode-segmentation = "1.10"
@@ -26,17 +26,18 @@ unicode-general-category = "0.6"
 # slab = "0.4.2"
 slotmap = "1.0"
 tree-sitter = "0.20"
-once_cell = "1.16"
+once_cell = "1.17"
 arc-swap = "1"
 regex = "1"
-bitflags = "1.3"
-ahash = "0.8.2"
-hashbrown = { version = "0.13.1", features = ["raw"] }
+bitflags = "2.2"
+ahash = "0.8.3"
+hashbrown = { version = "0.13.2", features = ["raw"] }
+dunce = "1.0"
 
 log = "0.4"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
-toml = "0.5"
+toml = "0.7"
 
 imara-diff = "0.1.0"
 
@@ -44,8 +45,9 @@ encoding_rs = "0.8"
 
 chrono = { version = "0.4", default-features = false, features = ["alloc", "std"] }
 
-etcetera = "0.4"
+etcetera = "0.8"
 textwrap = "0.16.0"
 
 [dev-dependencies]
 quickcheck = { version = "1", default-features = false }
+indoc = "2.0.1"

helix-core/src/auto_pairs.rs

@@ -17,7 +17,7 @@ pub const DEFAULT_PAIRS: &[(char, char)] = &[
 ];
 
 /// The type that represents the collection of auto pairs,
-/// keyed by the opener.
+/// keyed by both opener and closer.
 #[derive(Debug, Clone)]
 pub struct AutoPairs(HashMap<char, Pair>);
 

helix-core/src/comment.rs

@@ -45,7 +45,7 @@ fn find_line_comment(
 
             // determine margin of 0 or 1 for uncommenting; if any comment token is not followed by a space,
             // a margin of 0 is used for all lines.
-            if matches!(line_slice.get_char(pos + token_len), Some(c) if c != ' ') {
+            if !matches!(line_slice.get_char(pos + token_len), Some(c) if c == ' ') {
                 margin = 0;
             }
 
@@ -68,7 +68,7 @@ pub fn toggle_line_comments(doc: &Rope, selection: &Selection, token: Option<&st
     let mut min_next_line = 0;
     for selection in selection {
         let (start, end) = selection.line_range(text);
-        let start = start.max(min_next_line).min(text.len_lines());
+        let start = start.clamp(min_next_line, text.len_lines());
         let end = (end + 1).min(text.len_lines());
 
         lines.extend(start..end);
@@ -108,8 +108,8 @@ mod test {
         let text = doc.slice(..);
 
         let res = find_line_comment("//", text, 0..3);
-        // (commented = true, to_change = [line 0, line 2], min = col 2, margin = 1)
-        assert_eq!(res, (false, vec![0, 2], 2, 1));
+        // (commented = true, to_change = [line 0, line 2], min = col 2, margin = 0)
+        assert_eq!(res, (false, vec![0, 2], 2, 0));
 
         // comment
         let transaction = toggle_line_comments(&doc, &selection, None);
@@ -136,6 +136,17 @@ mod test {
         assert_eq!(doc, "  1\n\n  2\n  3");
         assert!(selection.len() == 1); // to ignore the selection unused warning
 
+        // 0 margin comments, with no space
+        doc = Rope::from("//");
+        // reset the selection.
+        selection = Selection::single(0, doc.len_chars() - 1);
+
+        let transaction = toggle_line_comments(&doc, &selection, None);
+        transaction.apply(&mut doc);
+        selection = selection.map(transaction.changes());
+        assert_eq!(doc, "");
+        assert!(selection.len() == 1); // to ignore the selection unused warning
+
         // TODO: account for uncommenting with uneven comment indentation
     }
 }

helix-core/src/diagnostic.rs

@@ -35,7 +35,7 @@ pub enum DiagnosticTag {
     Deprecated,
 }
 
-/// Corresponds to [`lsp_types::Diagnostic`](https://docs.rs/lsp-types/0.91.0/lsp_types/struct.Diagnostic.html)
+/// Corresponds to [`lsp_types::Diagnostic`](https://docs.rs/lsp-types/0.94.0/lsp_types/struct.Diagnostic.html)
 #[derive(Debug, Clone)]
 pub struct Diagnostic {
     pub range: Range,

helix-core/src/doc_formatter.rs

@@ -0,0 +1,384 @@
+//! The `DocumentFormatter` forms the bridge between the raw document text
+//! and onscreen positioning. It yields the text graphemes as an iterator
+//! and traverses (part) of the document text. During that traversal it
+//! handles grapheme detection, softwrapping and annotations.
+//! It yields `FormattedGrapheme`s and their corresponding visual coordinates.
+//!
+//! As both virtual text and softwrapping can insert additional lines into the document
+//! it is generally not possible to find the start of the previous visual line.
+//! Instead the `DocumentFormatter` starts at the last "checkpoint" (usually a linebreak)
+//! called a "block" and the caller must advance it as needed.
+
+use std::borrow::Cow;
+use std::fmt::Debug;
+use std::mem::{replace, take};
+
+#[cfg(test)]
+mod test;
+
+use unicode_segmentation::{Graphemes, UnicodeSegmentation};
+
+use crate::graphemes::{Grapheme, GraphemeStr};
+use crate::syntax::Highlight;
+use crate::text_annotations::TextAnnotations;
+use crate::{Position, RopeGraphemes, RopeSlice};
+
+/// TODO make Highlight a u32 to reduce the size of this enum to a single word.
+#[derive(Debug, Clone, Copy)]
+pub enum GraphemeSource {
+    Document {
+        codepoints: u32,
+    },
+    /// Inline virtual text can not be highlighted with a `Highlight` iterator
+    /// because it's not part of the document. Instead the `Highlight`
+    /// is emitted right by the document formatter
+    VirtualText {
+        highlight: Option<Highlight>,
+    },
+}
+
+#[derive(Debug, Clone)]
+pub struct FormattedGrapheme<'a> {
+    pub grapheme: Grapheme<'a>,
+    pub source: GraphemeSource,
+}
+
+impl<'a> FormattedGrapheme<'a> {
+    pub fn new(
+        g: GraphemeStr<'a>,
+        visual_x: usize,
+        tab_width: u16,
+        source: GraphemeSource,
+    ) -> FormattedGrapheme<'a> {
+        FormattedGrapheme {
+            grapheme: Grapheme::new(g, visual_x, tab_width),
+            source,
+        }
+    }
+    /// Returns whether this grapheme is virtual inline text
+    pub fn is_virtual(&self) -> bool {
+        matches!(self.source, GraphemeSource::VirtualText { .. })
+    }
+
+    pub fn placeholder() -> Self {
+        FormattedGrapheme {
+            grapheme: Grapheme::Other { g: " ".into() },
+            source: GraphemeSource::Document { codepoints: 0 },
+        }
+    }
+
+    pub fn doc_chars(&self) -> usize {
+        match self.source {
+            GraphemeSource::Document { codepoints } => codepoints as usize,
+            GraphemeSource::VirtualText { .. } => 0,
+        }
+    }
+
+    pub fn is_whitespace(&self) -> bool {
+        self.grapheme.is_whitespace()
+    }
+
+    pub fn width(&self) -> usize {
+        self.grapheme.width()
+    }
+
+    pub fn is_word_boundary(&self) -> bool {
+        self.grapheme.is_word_boundary()
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct TextFormat {
+    pub soft_wrap: bool,
+    pub tab_width: u16,
+    pub max_wrap: u16,
+    pub max_indent_retain: u16,
+    pub wrap_indicator: Box<str>,
+    pub wrap_indicator_highlight: Option<Highlight>,
+    pub viewport_width: u16,
+}
+
+// test implementation is basically only used for testing or when softwrap is always disabled
+impl Default for TextFormat {
+    fn default() -> Self {
+        TextFormat {
+            soft_wrap: false,
+            tab_width: 4,
+            max_wrap: 3,
+            max_indent_retain: 4,
+            wrap_indicator: Box::from(" "),
+            viewport_width: 17,
+            wrap_indicator_highlight: None,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct DocumentFormatter<'t> {
+    text_fmt: &'t TextFormat,
+    annotations: &'t TextAnnotations,
+
+    /// The visual position at the end of the last yielded word boundary
+    visual_pos: Position,
+    graphemes: RopeGraphemes<'t>,
+    /// The character pos of the `graphemes` iter used for inserting annotations
+    char_pos: usize,
+    /// The line pos of the `graphemes` iter used for inserting annotations
+    line_pos: usize,
+    exhausted: bool,
+
+    /// Line breaks to be reserved for virtual text
+    /// at the next line break
+    virtual_lines: usize,
+    inline_anntoation_graphemes: Option<(Graphemes<'t>, Option<Highlight>)>,
+
+    // softwrap specific
+    /// The indentation of the current line
+    /// Is set to `None` if the indentation level is not yet known
+    /// because no non-whitespace graphemes have been encountered yet
+    indent_level: Option<usize>,
+    /// In case a long word needs to be split a single grapheme might need to be wrapped
+    /// while the rest of the word stays on the same line
+    peeked_grapheme: Option<(FormattedGrapheme<'t>, usize)>,
+    /// A first-in first-out (fifo) buffer for the Graphemes of any given word
+    word_buf: Vec<FormattedGrapheme<'t>>,
+    /// The index of the next grapheme that will be yielded from the `word_buf`
+    word_i: usize,
+}
+
+impl<'t> DocumentFormatter<'t> {
+    /// Creates a new formatter at the last block before `char_idx`.
+    /// A block is a chunk which always ends with a linebreak.
+    /// This is usually just a normal line break.
+    /// However very long lines are always wrapped at constant intervals that can be cheaply calculated
+    /// to avoid pathological behaviour.
+    pub fn new_at_prev_checkpoint(
+        text: RopeSlice<'t>,
+        text_fmt: &'t TextFormat,
+        annotations: &'t TextAnnotations,
+        char_idx: usize,
+    ) -> (Self, usize) {
+        // TODO divide long lines into blocks to avoid bad performance for long lines
+        let block_line_idx = text.char_to_line(char_idx.min(text.len_chars()));
+        let block_char_idx = text.line_to_char(block_line_idx);
+        annotations.reset_pos(block_char_idx);
+        (
+            DocumentFormatter {
+                text_fmt,
+                annotations,
+                visual_pos: Position { row: 0, col: 0 },
+                graphemes: RopeGraphemes::new(text.slice(block_char_idx..)),
+                char_pos: block_char_idx,
+                exhausted: false,
+                virtual_lines: 0,
+                indent_level: None,
+                peeked_grapheme: None,
+                word_buf: Vec::with_capacity(64),
+                word_i: 0,
+                line_pos: block_line_idx,
+                inline_anntoation_graphemes: None,
+            },
+            block_char_idx,
+        )
+    }
+
+    fn next_inline_annotation_grapheme(&mut self) -> Option<(&'t str, Option<Highlight>)> {
+        loop {
+            if let Some(&mut (ref mut annotation, highlight)) =
+                self.inline_anntoation_graphemes.as_mut()
+            {
+                if let Some(grapheme) = annotation.next() {
+                    return Some((grapheme, highlight));
+                }
+            }
+
+            if let Some((annotation, highlight)) =
+                self.annotations.next_inline_annotation_at(self.char_pos)
+            {
+                self.inline_anntoation_graphemes = Some((
+                    UnicodeSegmentation::graphemes(&*annotation.text, true),
+                    highlight,
+                ))
+            } else {
+                return None;
+            }
+        }
+    }
+
+    fn advance_grapheme(&mut self, col: usize) -> Option<FormattedGrapheme<'t>> {
+        let (grapheme, source) =
+            if let Some((grapheme, highlight)) = self.next_inline_annotation_grapheme() {
+                (grapheme.into(), GraphemeSource::VirtualText { highlight })
+            } else if let Some(grapheme) = self.graphemes.next() {
+                self.virtual_lines += self.annotations.annotation_lines_at(self.char_pos);
+                let codepoints = grapheme.len_chars() as u32;
+
+                let overlay = self.annotations.overlay_at(self.char_pos);
+                let grapheme = match overlay {
+                    Some((overlay, _)) => overlay.grapheme.as_str().into(),
+                    None => Cow::from(grapheme).into(),
+                };
+
+                self.char_pos += codepoints as usize;
+                (grapheme, GraphemeSource::Document { codepoints })
+            } else {
+                if self.exhausted {
+                    return None;
+                }
+                self.exhausted = true;
+                // EOF grapheme is required for rendering
+                // and correct position computations
+                return Some(FormattedGrapheme {
+                    grapheme: Grapheme::Other { g: " ".into() },
+                    source: GraphemeSource::Document { codepoints: 0 },
+                });
+            };
+
+        let grapheme = FormattedGrapheme::new(grapheme, col, self.text_fmt.tab_width, source);
+
+        Some(grapheme)
+    }
+
+    /// Move a word to the next visual line
+    fn wrap_word(&mut self, virtual_lines_before_word: usize) -> usize {
+        // softwrap this word to the next line
+        let indent_carry_over = if let Some(indent) = self.indent_level {
+            if indent as u16 <= self.text_fmt.max_indent_retain {
+                indent as u16
+            } else {
+                0
+            }
+        } else {
+            // ensure the indent stays 0
+            self.indent_level = Some(0);
+            0
+        };
+
+        self.visual_pos.col = indent_carry_over as usize;
+        self.virtual_lines -= virtual_lines_before_word;
+        self.visual_pos.row += 1 + virtual_lines_before_word;
+        let mut i = 0;
+        let mut word_width = 0;
+        let wrap_indicator = UnicodeSegmentation::graphemes(&*self.text_fmt.wrap_indicator, true)
+            .map(|g| {
+                i += 1;
+                let grapheme = FormattedGrapheme::new(
+                    g.into(),
+                    self.visual_pos.col + word_width,
+                    self.text_fmt.tab_width,
+                    GraphemeSource::VirtualText {
+                        highlight: self.text_fmt.wrap_indicator_highlight,
+                    },
+                );
+                word_width += grapheme.width();
+                grapheme
+            });
+        self.word_buf.splice(0..0, wrap_indicator);
+
+        for grapheme in &mut self.word_buf[i..] {
+            let visual_x = self.visual_pos.col + word_width;
+            grapheme
+                .grapheme
+                .change_position(visual_x, self.text_fmt.tab_width);
+            word_width += grapheme.width();
+        }
+        word_width
+    }
+
+    fn advance_to_next_word(&mut self) {
+        self.word_buf.clear();
+        let mut word_width = 0;
+        let virtual_lines_before_word = self.virtual_lines;
+        let mut virtual_lines_before_grapheme = self.virtual_lines;
+
+        loop {
+            // softwrap word if necessary
+            if word_width + self.visual_pos.col >= self.text_fmt.viewport_width as usize {
+                // wrapping this word would move too much text to the next line
+                // split the word at the line end instead
+                if word_width > self.text_fmt.max_wrap as usize {
+                    // Usually we stop accomulating graphemes as soon as softwrapping becomes necessary.
+                    // However if the last grapheme is multiple columns wide it might extend beyond the EOL.
+                    // The condition below ensures that this grapheme is not cutoff and instead wrapped to the next line
+                    if word_width + self.visual_pos.col > self.text_fmt.viewport_width as usize {
+                        self.peeked_grapheme = self.word_buf.pop().map(|grapheme| {
+                            (grapheme, self.virtual_lines - virtual_lines_before_grapheme)
+                        });
+                        self.virtual_lines = virtual_lines_before_grapheme;
+                    }
+                    return;
+                }
+
+                word_width = self.wrap_word(virtual_lines_before_word);
+            }
+
+            virtual_lines_before_grapheme = self.virtual_lines;
+
+            let grapheme = if let Some((grapheme, virtual_lines)) = self.peeked_grapheme.take() {
+                self.virtual_lines += virtual_lines;
+                grapheme
+            } else if let Some(grapheme) = self.advance_grapheme(self.visual_pos.col + word_width) {
+                grapheme
+            } else {
+                return;
+            };
+
+            // Track indentation
+            if !grapheme.is_whitespace() && self.indent_level.is_none() {
+                self.indent_level = Some(self.visual_pos.col);
+            } else if grapheme.grapheme == Grapheme::Newline {
+                self.indent_level = None;
+            }
+
+            let is_word_boundary = grapheme.is_word_boundary();
+            word_width += grapheme.width();
+            self.word_buf.push(grapheme);
+
+            if is_word_boundary {
+                return;
+            }
+        }
+    }
+
+    /// returns the document line pos of the **next** grapheme that will be yielded
+    pub fn line_pos(&self) -> usize {
+        self.line_pos
+    }
+
+    /// returns the visual pos of the **next** grapheme that will be yielded
+    pub fn visual_pos(&self) -> Position {
+        self.visual_pos
+    }
+}
+
+impl<'t> Iterator for DocumentFormatter<'t> {
+    type Item = (FormattedGrapheme<'t>, Position);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let grapheme = if self.text_fmt.soft_wrap {
+            if self.word_i >= self.word_buf.len() {
+                self.advance_to_next_word();
+                self.word_i = 0;
+            }
+            let grapheme = replace(
+                self.word_buf.get_mut(self.word_i)?,
+                FormattedGrapheme::placeholder(),
+            );
+            self.word_i += 1;
+            grapheme
+        } else {
+            self.advance_grapheme(self.visual_pos.col)?
+        };
+
+        let pos = self.visual_pos;
+        if grapheme.grapheme == Grapheme::Newline {
+            self.visual_pos.row += 1;
+            self.visual_pos.row += take(&mut self.virtual_lines);
+            self.visual_pos.col = 0;
+            self.line_pos += 1;
+        } else {
+            self.visual_pos.col += grapheme.width();
+        }
+        Some((grapheme, pos))
+    }
+}

helix-core/src/doc_formatter/test.rs

@@ -0,0 +1,182 @@
+use std::rc::Rc;
+
+use crate::doc_formatter::{DocumentFormatter, TextFormat};
+use crate::text_annotations::{InlineAnnotation, Overlay, TextAnnotations};
+
+impl TextFormat {
+    fn new_test(softwrap: bool) -> Self {
+        TextFormat {
+            soft_wrap: softwrap,
+            tab_width: 2,
+            max_wrap: 3,
+            max_indent_retain: 4,
+            wrap_indicator: ".".into(),
+            wrap_indicator_highlight: None,
+            // use a prime number to allow lining up too often with repeat
+            viewport_width: 17,
+        }
+    }
+}
+
+impl<'t> DocumentFormatter<'t> {
+    fn collect_to_str(&mut self) -> String {
+        use std::fmt::Write;
+        let mut res = String::new();
+        let viewport_width = self.text_fmt.viewport_width;
+        let mut line = 0;
+
+        for (grapheme, pos) in self {
+            if pos.row != line {
+                line += 1;
+                assert_eq!(pos.row, line);
+                write!(res, "\n{}", ".".repeat(pos.col)).unwrap();
+                assert!(
+                    pos.col <= viewport_width as usize,
+                    "softwrapped failed {}<={viewport_width}",
+                    pos.col
+                );
+            }
+            write!(res, "{}", grapheme.grapheme).unwrap();
+        }
+
+        res
+    }
+}
+
+fn softwrap_text(text: &str) -> String {
+    DocumentFormatter::new_at_prev_checkpoint(
+        text.into(),
+        &TextFormat::new_test(true),
+        &TextAnnotations::default(),
+        0,
+    )
+    .0
+    .collect_to_str()
+}
+
+#[test]
+fn basic_softwrap() {
+    assert_eq!(
+        softwrap_text(&"foo ".repeat(10)),
+        "foo foo foo foo \n.foo foo foo foo \n.foo foo  "
+    );
+    assert_eq!(
+        softwrap_text(&"fooo ".repeat(10)),
+        "fooo fooo fooo \n.fooo fooo fooo \n.fooo fooo fooo \n.fooo  "
+    );
+
+    // check that we don't wrap unnecessarily
+    assert_eq!(softwrap_text("\t\txxxx1xxxx2xx\n"), "    xxxx1xxxx2xx \n ");
+}
+
+#[test]
+fn softwrap_indentation() {
+    assert_eq!(
+        softwrap_text("\t\tfoo1 foo2 foo3 foo4 foo5 foo6\n"),
+        "    foo1 foo2 \n.....foo3 foo4 \n.....foo5 foo6 \n "
+    );
+    assert_eq!(
+        softwrap_text("\t\t\tfoo1 foo2 foo3 foo4 foo5 foo6\n"),
+        "      foo1 foo2 \n.foo3 foo4 foo5 \n.foo6 \n "
+    );
+}
+
+#[test]
+fn long_word_softwrap() {
+    assert_eq!(
+        softwrap_text("\t\txxxx1xxxx2xxxx3xxxx4xxxx5xxxx6xxxx7xxxx8xxxx9xxx\n"),
+        "    xxxx1xxxx2xxx\n.....x3xxxx4xxxx5\n.....xxxx6xxxx7xx\n.....xx8xxxx9xxx \n "
+    );
+    assert_eq!(
+        softwrap_text("xxxxxxxx1xxxx2xxx\n"),
+        "xxxxxxxx1xxxx2xxx\n. \n "
+    );
+    assert_eq!(
+        softwrap_text("\t\txxxx1xxxx 2xxxx3xxxx4xxxx5xxxx6xxxx7xxxx8xxxx9xxx\n"),
+        "    xxxx1xxxx \n.....2xxxx3xxxx4x\n.....xxx5xxxx6xxx\n.....x7xxxx8xxxx9\n.....xxx \n "
+    );
+    assert_eq!(
+        softwrap_text("\t\txxxx1xxx 2xxxx3xxxx4xxxx5xxxx6xxxx7xxxx8xxxx9xxx\n"),
+        "    xxxx1xxx 2xxx\n.....x3xxxx4xxxx5\n.....xxxx6xxxx7xx\n.....xx8xxxx9xxx \n "
+    );
+}
+
+fn overlay_text(text: &str, char_pos: usize, softwrap: bool, overlays: &[Overlay]) -> String {
+    DocumentFormatter::new_at_prev_checkpoint(
+        text.into(),
+        &TextFormat::new_test(softwrap),
+        TextAnnotations::default().add_overlay(overlays.into(), None),
+        char_pos,
+    )
+    .0
+    .collect_to_str()
+}
+
+#[test]
+fn overlay() {
+    assert_eq!(
+        overlay_text(
+            "foobar",
+            0,
+            false,
+            &[Overlay::new(0, "X"), Overlay::new(2, "\t")],
+        ),
+        "Xo  bar "
+    );
+    assert_eq!(
+        overlay_text(
+            &"foo ".repeat(10),
+            0,
+            true,
+            &[
+                Overlay::new(2, "\t"),
+                Overlay::new(5, "\t"),
+                Overlay::new(16, "X"),
+            ]
+        ),
+        "fo   f  o foo \n.foo Xoo foo foo \n.foo foo foo  "
+    );
+}
+
+fn annotate_text(text: &str, softwrap: bool, annotations: &[InlineAnnotation]) -> String {
+    DocumentFormatter::new_at_prev_checkpoint(
+        text.into(),
+        &TextFormat::new_test(softwrap),
+        TextAnnotations::default().add_inline_annotations(annotations.into(), None),
+        0,
+    )
+    .0
+    .collect_to_str()
+}
+
+#[test]
+fn annotation() {
+    assert_eq!(
+        annotate_text("bar", false, &[InlineAnnotation::new(0, "foo")]),
+        "foobar "
+    );
+    assert_eq!(
+        annotate_text(
+            &"foo ".repeat(10),
+            true,
+            &[InlineAnnotation::new(0, "foo ")]
+        ),
+        "foo foo foo foo \n.foo foo foo foo \n.foo foo foo  "
+    );
+}
+#[test]
+fn annotation_and_overlay() {
+    assert_eq!(
+        DocumentFormatter::new_at_prev_checkpoint(
+            "bbar".into(),
+            &TextFormat::new_test(false),
+            TextAnnotations::default()
+                .add_inline_annotations(Rc::new([InlineAnnotation::new(0, "fooo")]), None)
+                .add_overlay(Rc::new([Overlay::new(0, "\t")]), None),
+            0,
+        )
+        .0
+        .collect_to_str(),
+        "fooo  bar "
+    );
+}

helix-core/src/graphemes.rs

@@ -5,7 +5,88 @@ use ropey::{iter::Chunks, str_utils::byte_to_char_idx, RopeSlice};
 use unicode_segmentation::{GraphemeCursor, GraphemeIncomplete};
 use unicode_width::UnicodeWidthStr;
 
-use std::fmt;
+use std::borrow::Cow;
+use std::fmt::{self, Debug, Display};
+use std::marker::PhantomData;
+use std::ops::Deref;
+use std::ptr::NonNull;
+use std::{slice, str};
+
+use crate::chars::{char_is_whitespace, char_is_word};
+use crate::LineEnding;
+
+#[inline]
+pub fn tab_width_at(visual_x: usize, tab_width: u16) -> usize {
+    tab_width as usize - (visual_x % tab_width as usize)
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Grapheme<'a> {
+    Newline,
+    Tab { width: usize },
+    Other { g: GraphemeStr<'a> },
+}
+
+impl<'a> Grapheme<'a> {
+    pub fn new(g: GraphemeStr<'a>, visual_x: usize, tab_width: u16) -> Grapheme<'a> {
+        match g {
+            g if g == "\t" => Grapheme::Tab {
+                width: tab_width_at(visual_x, tab_width),
+            },
+            _ if LineEnding::from_str(&g).is_some() => Grapheme::Newline,
+            _ => Grapheme::Other { g },
+        }
+    }
+
+    pub fn change_position(&mut self, visual_x: usize, tab_width: u16) {
+        if let Grapheme::Tab { width } = self {
+            *width = tab_width_at(visual_x, tab_width)
+        }
+    }
+
+    /// Returns the a visual width of this grapheme,
+    #[inline]
+    pub fn width(&self) -> usize {
+        match *self {
+            // width is not cached because we are dealing with
+            // ASCII almost all the time which already has a fastpath
+            // it's okay to convert to u16 here because no codepoint has a width larger
+            // than 2 and graphemes are usually atmost two visible codepoints wide
+            Grapheme::Other { ref g } => grapheme_width(g),
+            Grapheme::Tab { width } => width,
+            Grapheme::Newline => 1,
+        }
+    }
+
+    pub fn is_whitespace(&self) -> bool {
+        !matches!(&self, Grapheme::Other { g } if !g.chars().all(char_is_whitespace))
+    }
+
+    // TODO currently word boundaries are used for softwrapping.
+    // This works best for programming languages and well for prose.
+    // This could however be improved in the future by considering unicode
+    // character classes but
+    pub fn is_word_boundary(&self) -> bool {
+        !matches!(&self, Grapheme::Other { g,.. } if g.chars().all(char_is_word))
+    }
+}
+
+impl Display for Grapheme<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match *self {
+            Grapheme::Newline => write!(f, " "),
+            Grapheme::Tab { width } => {
+                for _ in 0..width {
+                    write!(f, " ")?;
+                }
+                Ok(())
+            }
+            Grapheme::Other { ref g } => {
+                write!(f, "{g}")
+            }
+        }
+    }
+}
 
 #[must_use]
 pub fn grapheme_width(g: &str) -> usize {
@@ -27,6 +108,8 @@ pub fn grapheme_width(g: &str) -> usize {
         // We use max(1) here because all grapeheme clusters--even illformed
         // ones--should have at least some width so they can be edited
         // properly.
+        // TODO properly handle unicode width for all codepoints
+        // example of where unicode width is currently wrong: 🤦🏼‍♂️ (taken from https://hsivonen.fi/string-length/)
         UnicodeWidthStr::width(g).max(1)
     }
 }
@@ -341,3 +424,101 @@ impl<'a> Iterator for RopeGraphemes<'a> {
         }
     }
 }
+
+/// A highly compressed Cow<'a, str> that holds
+/// atmost u31::MAX bytes and is readonly
+pub struct GraphemeStr<'a> {
+    ptr: NonNull<u8>,
+    len: u32,
+    phantom: PhantomData<&'a str>,
+}
+
+impl GraphemeStr<'_> {
+    const MASK_OWNED: u32 = 1 << 31;
+
+    fn compute_len(&self) -> usize {
+        (self.len & !Self::MASK_OWNED) as usize
+    }
+}
+
+impl Deref for GraphemeStr<'_> {
+    type Target = str;
+    fn deref(&self) -> &Self::Target {
+        unsafe {
+            let bytes = slice::from_raw_parts(self.ptr.as_ptr(), self.compute_len());
+            str::from_utf8_unchecked(bytes)
+        }
+    }
+}
+
+impl Drop for GraphemeStr<'_> {
+    fn drop(&mut self) {
+        if self.len & Self::MASK_OWNED != 0 {
+            // free allocation
+            unsafe {
+                drop(Box::from_raw(slice::from_raw_parts_mut(
+                    self.ptr.as_ptr(),
+                    self.compute_len(),
+                )));
+            }
+        }
+    }
+}
+
+impl<'a> From<&'a str> for GraphemeStr<'a> {
+    fn from(g: &'a str) -> Self {
+        GraphemeStr {
+            ptr: unsafe { NonNull::new_unchecked(g.as_bytes().as_ptr() as *mut u8) },
+            len: i32::try_from(g.len()).unwrap() as u32,
+            phantom: PhantomData,
+        }
+    }
+}
+
+impl<'a> From<String> for GraphemeStr<'a> {
+    fn from(g: String) -> Self {
+        let len = g.len();
+        let ptr = Box::into_raw(g.into_bytes().into_boxed_slice()) as *mut u8;
+        GraphemeStr {
+            ptr: unsafe { NonNull::new_unchecked(ptr) },
+            len: i32::try_from(len).unwrap() as u32,
+            phantom: PhantomData,
+        }
+    }
+}
+
+impl<'a> From<Cow<'a, str>> for GraphemeStr<'a> {
+    fn from(g: Cow<'a, str>) -> Self {
+        match g {
+            Cow::Borrowed(g) => g.into(),
+            Cow::Owned(g) => g.into(),
+        }
+    }
+}
+
+impl<T: Deref<Target = str>> PartialEq<T> for GraphemeStr<'_> {
+    fn eq(&self, other: &T) -> bool {
+        self.deref() == other.deref()
+    }
+}
+impl PartialEq<str> for GraphemeStr<'_> {
+    fn eq(&self, other: &str) -> bool {
+        self.deref() == other
+    }
+}
+impl Eq for GraphemeStr<'_> {}
+impl Debug for GraphemeStr<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        Debug::fmt(self.deref(), f)
+    }
+}
+impl Display for GraphemeStr<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        Display::fmt(self.deref(), f)
+    }
+}
+impl Clone for GraphemeStr<'_> {
+    fn clone(&self) -> Self {
+        self.deref().to_owned().into()
+    }
+}

helix-core/src/increment/date_time.rs

@@ -1,114 +1,53 @@
-use chrono::{Datelike, Duration, NaiveDate, NaiveDateTime, NaiveTime, Timelike};
+use chrono::{Duration, NaiveDate, NaiveDateTime, NaiveTime};
 use once_cell::sync::Lazy;
 use regex::Regex;
-use ropey::RopeSlice;
-
-use std::borrow::Cow;
-use std::cmp;
 use std::fmt::Write;
 
-use super::Increment;
-use crate::{Range, Tendril};
-
-#[derive(Debug, PartialEq, Eq)]
-pub struct DateTimeIncrementor {
-    date_time: NaiveDateTime,
-    range: Range,
-    fmt: &'static str,
-    field: DateField,
-}
-
-impl DateTimeIncrementor {
-    pub fn from_range(text: RopeSlice, range: Range) -> Option<DateTimeIncrementor> {
-        let range = if range.is_empty() {
-            if range.anchor < text.len_chars() {
-                // Treat empty range as a cursor range.
-                range.put_cursor(text, range.anchor + 1, true)
-            } else {
-                // The range is empty and at the end of the text.
-                return None;
-            }
-        } else {
-            range
-        };
-
-        FORMATS.iter().find_map(|format| {
-            let from = range.from().saturating_sub(format.max_len);
-            let to = (range.from() + format.max_len).min(text.len_chars());
-
-            let (from_in_text, to_in_text) = (range.from() - from, range.to() - from);
-            let text: Cow<str> = text.slice(from..to).into();
-
-            let captures = format.regex.captures(&text)?;
-            if captures.len() - 1 != format.fields.len() {
-                return None;
-            }
-
-            let date_time = captures.get(0)?;
-            let offset = range.from() - from_in_text;
-            let range = Range::new(date_time.start() + offset, date_time.end() + offset);
-
-            let field = captures
-                .iter()
-                .skip(1)
-                .enumerate()
-                .find_map(|(i, capture)| {
-                    let capture = capture?;
-                    let capture_range = capture.range();
-
-                    if capture_range.contains(&from_in_text)
-                        && capture_range.contains(&(to_in_text - 1))
-                    {
-                        Some(format.fields[i])
-                    } else {
-                        None
-                    }
-                })?;
-
-            let has_date = format.fields.iter().any(|f| f.unit.is_date());
-            let has_time = format.fields.iter().any(|f| f.unit.is_time());
-
-            let date_time = &text[date_time.start()..date_time.end()];
-            let date_time = match (has_date, has_time) {
-                (true, true) => NaiveDateTime::parse_from_str(date_time, format.fmt).ok()?,
-                (true, false) => {
-                    let date = NaiveDate::parse_from_str(date_time, format.fmt).ok()?;
-
-                    date.and_hms_opt(0, 0, 0).unwrap()
-                }
-                (false, true) => {
-                    let time = NaiveTime::parse_from_str(date_time, format.fmt).ok()?;
-
-                    NaiveDate::from_ymd_opt(0, 1, 1).unwrap().and_time(time)
-                }
-                (false, false) => return None,
-            };
-
-            Some(DateTimeIncrementor {
-                date_time,
-                range,
-                fmt: format.fmt,
-                field,
-            })
-        })
+/// Increment a Date or DateTime
+///
+/// If just a Date is selected the day will be incremented.
+/// If a DateTime is selected the second will be incremented.
+pub fn increment(selected_text: &str, amount: i64) -> Option<String> {
+    if selected_text.is_empty() {
+        return None;
     }
-}
 
-impl Increment for DateTimeIncrementor {
-    fn increment(&self, amount: i64) -> (Range, Tendril) {
-        let date_time = match self.field.unit {
-            DateUnit::Years => add_years(self.date_time, amount),
-            DateUnit::Months => add_months(self.date_time, amount),
-            DateUnit::Days => add_duration(self.date_time, Duration::days(amount)),
-            DateUnit::Hours => add_duration(self.date_time, Duration::hours(amount)),
-            DateUnit::Minutes => add_duration(self.date_time, Duration::minutes(amount)),
-            DateUnit::Seconds => add_duration(self.date_time, Duration::seconds(amount)),
-            DateUnit::AmPm => toggle_am_pm(self.date_time),
+    FORMATS.iter().find_map(|format| {
+        let captures = format.regex.captures(selected_text)?;
+        if captures.len() - 1 != format.fields.len() {
+            return None;
         }
-        .unwrap_or(self.date_time);
 
-        (self.range, date_time.format(self.fmt).to_string().into())
-    }
+        let date_time = captures.get(0)?;
+        let has_date = format.fields.iter().any(|f| f.unit.is_date());
+        let has_time = format.fields.iter().any(|f| f.unit.is_time());
+        let date_time = &selected_text[date_time.start()..date_time.end()];
+        match (has_date, has_time) {
+            (true, true) => {
+                let date_time = NaiveDateTime::parse_from_str(date_time, format.fmt).ok()?;
+                Some(
+                    date_time
+                        .checked_add_signed(Duration::minutes(amount))?
+                        .format(format.fmt)
+                        .to_string(),
+                )
+            }
+            (true, false) => {
+                let date = NaiveDate::parse_from_str(date_time, format.fmt).ok()?;
+                Some(
+                    date.checked_add_signed(Duration::days(amount))?
+                        .format(format.fmt)
+                        .to_string(),
+                )
+            }
+            (false, true) => {
+                let time = NaiveTime::parse_from_str(date_time, format.fmt).ok()?;
+                let (adjusted_time, _) = time.overflowing_add_signed(Duration::minutes(amount));
+                Some(adjusted_time.format(format.fmt).to_string())
+            }
+            (false, false) => None,
+        }
+    })
 }
 
 static FORMATS: Lazy<Vec<Format>> = Lazy::new(|| {
@@ -144,7 +83,7 @@ impl Format {
     fn new(fmt: &'static str) -> Self {
         let mut remaining = fmt;
         let mut fields = Vec::new();
-        let mut regex = String::new();
+        let mut regex = "^".to_string();
         let mut max_len = 0;
 
         while let Some(i) = remaining.find('%') {
@@ -166,6 +105,7 @@ impl Format {
             write!(regex, "({})", field.regex).unwrap();
             remaining = &after[spec_len..];
         }
+        regex += "$";
 
         let regex = Regex::new(&regex).unwrap();
 
@@ -305,155 +245,47 @@ impl DateUnit {
     }
 }
 
-fn ndays_in_month(year: i32, month: u32) -> u32 {
-    // The first day of the next month...
-    let (y, m) = if month == 12 {
-        (year + 1, 1)
-    } else {
-        (year, month + 1)
-    };
-    let d = NaiveDate::from_ymd_opt(y, m, 1).unwrap();
-
-    // ...is preceded by the last day of the original month.
-    d.pred_opt().unwrap().day()
-}
-
-fn add_months(date_time: NaiveDateTime, amount: i64) -> Option<NaiveDateTime> {
-    let month = (date_time.month0() as i64).checked_add(amount)?;
-    let year = date_time.year() + i32::try_from(month / 12).ok()?;
-    let year = if month.is_negative() { year - 1 } else { year };
-
-    // Normalize month
-    let month = month % 12;
-    let month = if month.is_negative() {
-        month + 12
-    } else {
-        month
-    } as u32
-        + 1;
-
-    let day = cmp::min(date_time.day(), ndays_in_month(year, month));
-
-    NaiveDate::from_ymd_opt(year, month, day).map(|date| date.and_time(date_time.time()))
-}
-
-fn add_years(date_time: NaiveDateTime, amount: i64) -> Option<NaiveDateTime> {
-    let year = i32::try_from((date_time.year() as i64).checked_add(amount)?).ok()?;
-    let ndays = ndays_in_month(year, date_time.month());
-
-    if date_time.day() > ndays {
-        NaiveDate::from_ymd_opt(year, date_time.month(), ndays)
-            .and_then(|date| date.succ_opt().map(|date| date.and_time(date_time.time())))
-    } else {
-        date_time.with_year(year)
-    }
-}
-
-fn add_duration(date_time: NaiveDateTime, duration: Duration) -> Option<NaiveDateTime> {
-    date_time.checked_add_signed(duration)
-}
-
-fn toggle_am_pm(date_time: NaiveDateTime) -> Option<NaiveDateTime> {
-    if date_time.hour() < 12 {
-        add_duration(date_time, Duration::hours(12))
-    } else {
-        add_duration(date_time, Duration::hours(-12))
-    }
-}
-
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::Rope;
 
     #[test]
     fn test_increment_date_times() {
         let tests = [
             // (original, cursor, amount, expected)
-            ("2020-02-28", 0, 1, "2021-02-28"),
-            ("2020-02-29", 0, 1, "2021-03-01"),
-            ("2020-01-31", 5, 1, "2020-02-29"),
-            ("2020-01-20", 5, 1, "2020-02-20"),
-            ("2021-01-01", 5, -1, "2020-12-01"),
-            ("2021-01-31", 5, -2, "2020-11-30"),
-            ("2020-02-28", 8, 1, "2020-02-29"),
-            ("2021-02-28", 8, 1, "2021-03-01"),
-            ("2021-02-28", 0, -1, "2020-02-28"),
-            ("2021-03-01", 0, -1, "2020-03-01"),
-            ("2020-02-29", 5, -1, "2020-01-29"),
-            ("2020-02-20", 5, -1, "2020-01-20"),
-            ("2020-02-29", 8, -1, "2020-02-28"),
-            ("2021-03-01", 8, -1, "2021-02-28"),
-            ("1980/12/21", 8, 100, "1981/03/31"),
-            ("1980/12/21", 8, -100, "1980/09/12"),
-            ("1980/12/21", 8, 1000, "1983/09/17"),
-            ("1980/12/21", 8, -1000, "1978/03/27"),
-            ("2021-11-24 07:12:23", 0, 1, "2022-11-24 07:12:23"),
-            ("2021-11-24 07:12:23", 5, 1, "2021-12-24 07:12:23"),
-            ("2021-11-24 07:12:23", 8, 1, "2021-11-25 07:12:23"),
-            ("2021-11-24 07:12:23", 11, 1, "2021-11-24 08:12:23"),
-            ("2021-11-24 07:12:23", 14, 1, "2021-11-24 07:13:23"),
-            ("2021-11-24 07:12:23", 17, 1, "2021-11-24 07:12:24"),
-            ("2021/11/24 07:12:23", 0, 1, "2022/11/24 07:12:23"),
-            ("2021/11/24 07:12:23", 5, 1, "2021/12/24 07:12:23"),
-            ("2021/11/24 07:12:23", 8, 1, "2021/11/25 07:12:23"),
-            ("2021/11/24 07:12:23", 11, 1, "2021/11/24 08:12:23"),
-            ("2021/11/24 07:12:23", 14, 1, "2021/11/24 07:13:23"),
-            ("2021/11/24 07:12:23", 17, 1, "2021/11/24 07:12:24"),
-            ("2021-11-24 07:12", 0, 1, "2022-11-24 07:12"),
-            ("2021-11-24 07:12", 5, 1, "2021-12-24 07:12"),
-            ("2021-11-24 07:12", 8, 1, "2021-11-25 07:12"),
-            ("2021-11-24 07:12", 11, 1, "2021-11-24 08:12"),
-            ("2021-11-24 07:12", 14, 1, "2021-11-24 07:13"),
-            ("2021/11/24 07:12", 0, 1, "2022/11/24 07:12"),
-            ("2021/11/24 07:12", 5, 1, "2021/12/24 07:12"),
-            ("2021/11/24 07:12", 8, 1, "2021/11/25 07:12"),
-            ("2021/11/24 07:12", 11, 1, "2021/11/24 08:12"),
-            ("2021/11/24 07:12", 14, 1, "2021/11/24 07:13"),
-            ("Wed Nov 24 2021", 0, 1, "Thu Nov 25 2021"),
-            ("Wed Nov 24 2021", 4, 1, "Fri Dec 24 2021"),
-            ("Wed Nov 24 2021", 8, 1, "Thu Nov 25 2021"),
-            ("Wed Nov 24 2021", 11, 1, "Thu Nov 24 2022"),
-            ("24-Nov-2021", 0, 1, "25-Nov-2021"),
-            ("24-Nov-2021", 3, 1, "24-Dec-2021"),
-            ("24-Nov-2021", 7, 1, "24-Nov-2022"),
-            ("2021 Nov 24", 0, 1, "2022 Nov 24"),
-            ("2021 Nov 24", 5, 1, "2021 Dec 24"),
-            ("2021 Nov 24", 9, 1, "2021 Nov 25"),
-            ("Nov 24, 2021", 0, 1, "Dec 24, 2021"),
-            ("Nov 24, 2021", 4, 1, "Nov 25, 2021"),
-            ("Nov 24, 2021", 8, 1, "Nov 24, 2022"),
-            ("7:21:53 am", 0, 1, "8:21:53 am"),
-            ("7:21:53 am", 3, 1, "7:22:53 am"),
-            ("7:21:53 am", 5, 1, "7:21:54 am"),
-            ("7:21:53 am", 8, 1, "7:21:53 pm"),
-            ("7:21:53 AM", 0, 1, "8:21:53 AM"),
-            ("7:21:53 AM", 3, 1, "7:22:53 AM"),
-            ("7:21:53 AM", 5, 1, "7:21:54 AM"),
-            ("7:21:53 AM", 8, 1, "7:21:53 PM"),
-            ("7:21 am", 0, 1, "8:21 am"),
-            ("7:21 am", 3, 1, "7:22 am"),
-            ("7:21 am", 5, 1, "7:21 pm"),
-            ("7:21 AM", 0, 1, "8:21 AM"),
-            ("7:21 AM", 3, 1, "7:22 AM"),
-            ("7:21 AM", 5, 1, "7:21 PM"),
-            ("23:24:23", 1, 1, "00:24:23"),
-            ("23:24:23", 3, 1, "23:25:23"),
-            ("23:24:23", 6, 1, "23:24:24"),
-            ("23:24", 1, 1, "00:24"),
-            ("23:24", 3, 1, "23:25"),
+            ("2020-02-28", 1, "2020-02-29"),
+            ("2020-02-29", 1, "2020-03-01"),
+            ("2020-01-31", 1, "2020-02-01"),
+            ("2020-01-20", 1, "2020-01-21"),
+            ("2021-01-01", -1, "2020-12-31"),
+            ("2021-01-31", -2, "2021-01-29"),
+            ("2020-02-28", 1, "2020-02-29"),
+            ("2021-02-28", 1, "2021-03-01"),
+            ("2021-03-01", -1, "2021-02-28"),
+            ("2020-02-29", -1, "2020-02-28"),
+            ("2020-02-20", -1, "2020-02-19"),
+            ("2021-03-01", -1, "2021-02-28"),
+            ("1980/12/21", 100, "1981/03/31"),
+            ("1980/12/21", -100, "1980/09/12"),
+            ("1980/12/21", 1000, "1983/09/17"),
+            ("1980/12/21", -1000, "1978/03/27"),
+            ("2021-11-24 07:12:23", 1, "2021-11-24 07:13:23"),
+            ("2021-11-24 07:12", 1, "2021-11-24 07:13"),
+            ("Wed Nov 24 2021", 1, "Thu Nov 25 2021"),
+            ("24-Nov-2021", 1, "25-Nov-2021"),
+            ("2021 Nov 24", 1, "2021 Nov 25"),
+            ("Nov 24, 2021", 1, "Nov 25, 2021"),
+            ("7:21:53 am", 1, "7:22:53 am"),
+            ("7:21:53 AM", 1, "7:22:53 AM"),
+            ("7:21 am", 1, "7:22 am"),
+            ("23:24:23", 1, "23:25:23"),
+            ("23:24", 1, "23:25"),
+            ("23:59", 1, "00:00"),
+            ("23:59:59", 1, "00:00:59"),
         ];
 
-        for (original, cursor, amount, expected) in tests {
-            let rope = Rope::from_str(original);
-            let range = Range::new(cursor, cursor + 1);
-            assert_eq!(
-                DateTimeIncrementor::from_range(rope.slice(..), range)
-                    .unwrap()
-                    .increment(amount)
-                    .1,
-                Tendril::from(expected)
-            );
+        for (original, amount, expected) in tests {
+            assert_eq!(increment(original, amount).unwrap(), expected);
         }
     }
 
@@ -482,10 +314,7 @@ mod test {
         ];
 
         for invalid in tests {
-            let rope = Rope::from_str(invalid);
-            let range = Range::new(0, 1);
-
-            assert_eq!(DateTimeIncrementor::from_range(rope.slice(..), range), None)
+            assert_eq!(increment(invalid, 1), None)
         }
     }
 }

helix-core/src/increment/integer.rs

@@ -0,0 +1,235 @@
+const SEPARATOR: char = '_';
+
+/// Increment an integer.
+///
+/// Supported bases:
+///     2 with prefix 0b
+///     8 with prefix 0o
+///     10 with no prefix
+///     16 with prefix 0x
+///
+/// An integer can contain `_` as a separator but may not start or end with a separator.
+/// Base 10 integers can go negative, but bases 2, 8, and 16 cannot.
+/// All addition and subtraction is saturating.
+pub fn increment(selected_text: &str, amount: i64) -> Option<String> {
+    if selected_text.is_empty()
+        || selected_text.ends_with(SEPARATOR)
+        || selected_text.starts_with(SEPARATOR)
+    {
+        return None;
+    }
+
+    let radix = if selected_text.starts_with("0x") {
+        16
+    } else if selected_text.starts_with("0o") {
+        8
+    } else if selected_text.starts_with("0b") {
+        2
+    } else {
+        10
+    };
+
+    // Get separator indexes from right to left.
+    let separator_rtl_indexes: Vec<usize> = selected_text
+        .chars()
+        .rev()
+        .enumerate()
+        .filter_map(|(i, c)| if c == SEPARATOR { Some(i) } else { None })
+        .collect();
+
+    let word: String = selected_text.chars().filter(|&c| c != SEPARATOR).collect();
+
+    let mut new_text = if radix == 10 {
+        let number = &word;
+        let value = i128::from_str_radix(number, radix).ok()?;
+        let new_value = value.saturating_add(amount as i128);
+
+        let format_length = match (value.is_negative(), new_value.is_negative()) {
+            (true, false) => number.len() - 1,
+            (false, true) => number.len() + 1,
+            _ => number.len(),
+        } - separator_rtl_indexes.len();
+
+        if number.starts_with('0') || number.starts_with("-0") {
+            format!("{:01$}", new_value, format_length)
+        } else {
+            format!("{}", new_value)
+        }
+    } else {
+        let number = &word[2..];
+        let value = u128::from_str_radix(number, radix).ok()?;
+        let new_value = (value as i128).saturating_add(amount as i128);
+        let new_value = if new_value < 0 { 0 } else { new_value };
+        let format_length = selected_text.len() - 2 - separator_rtl_indexes.len();
+
+        match radix {
+            2 => format!("0b{:01$b}", new_value, format_length),
+            8 => format!("0o{:01$o}", new_value, format_length),
+            16 => {
+                let (lower_count, upper_count): (usize, usize) =
+                    number.chars().fold((0, 0), |(lower, upper), c| {
+                        (
+                            lower + c.is_ascii_lowercase() as usize,
+                            upper + c.is_ascii_uppercase() as usize,
+                        )
+                    });
+                if upper_count > lower_count {
+                    format!("0x{:01$X}", new_value, format_length)
+                } else {
+                    format!("0x{:01$x}", new_value, format_length)
+                }
+            }
+            _ => unimplemented!("radix not supported: {}", radix),
+        }
+    };
+
+    // Add separators from original number.
+    for &rtl_index in &separator_rtl_indexes {
+        if rtl_index < new_text.len() {
+            let new_index = new_text.len().saturating_sub(rtl_index);
+            if new_index > 0 {
+                new_text.insert(new_index, SEPARATOR);
+            }
+        }
+    }
+
+    // Add in additional separators if necessary.
+    if new_text.len() > selected_text.len() && !separator_rtl_indexes.is_empty() {
+        let spacing = match separator_rtl_indexes.as_slice() {
+            [.., b, a] => a - b - 1,
+            _ => separator_rtl_indexes[0],
+        };
+
+        let prefix_length = if radix == 10 { 0 } else { 2 };
+        if let Some(mut index) = new_text.find(SEPARATOR) {
+            while index - prefix_length > spacing {
+                index -= spacing;
+                new_text.insert(index, SEPARATOR);
+            }
+        }
+    }
+
+    Some(new_text)
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_increment_basic_decimal_numbers() {
+        let tests = [
+            ("100", 1, "101"),
+            ("100", -1, "99"),
+            ("99", 1, "100"),
+            ("100", 1000, "1100"),
+            ("100", -1000, "-900"),
+            ("-1", 1, "0"),
+            ("-1", 2, "1"),
+            ("1", -1, "0"),
+            ("1", -2, "-1"),
+        ];
+
+        for (original, amount, expected) in tests {
+            assert_eq!(increment(original, amount).unwrap(), expected);
+        }
+    }
+
+    #[test]
+    fn test_increment_basic_hexadecimal_numbers() {
+        let tests = [
+            ("0x0100", 1, "0x0101"),
+            ("0x0100", -1, "0x00ff"),
+            ("0x0001", -1, "0x0000"),
+            ("0x0000", -1, "0x0000"),
+            ("0xffffffffffffffff", 1, "0x10000000000000000"),
+            ("0xffffffffffffffff", 2, "0x10000000000000001"),
+            ("0xffffffffffffffff", -1, "0xfffffffffffffffe"),
+            ("0xABCDEF1234567890", 1, "0xABCDEF1234567891"),
+            ("0xabcdef1234567890", 1, "0xabcdef1234567891"),
+        ];
+
+        for (original, amount, expected) in tests {
+            assert_eq!(increment(original, amount).unwrap(), expected);
+        }
+    }
+
+    #[test]
+    fn test_increment_basic_octal_numbers() {
+        let tests = [
+            ("0o0107", 1, "0o0110"),
+            ("0o0110", -1, "0o0107"),
+            ("0o0001", -1, "0o0000"),
+            ("0o7777", 1, "0o10000"),
+            ("0o1000", -1, "0o0777"),
+            ("0o0107", 10, "0o0121"),
+            ("0o0000", -1, "0o0000"),
+            ("0o1777777777777777777777", 1, "0o2000000000000000000000"),
+            ("0o1777777777777777777777", 2, "0o2000000000000000000001"),
+            ("0o1777777777777777777777", -1, "0o1777777777777777777776"),
+        ];
+
+        for (original, amount, expected) in tests {
+            assert_eq!(increment(original, amount).unwrap(), expected);
+        }
+    }
+
+    #[test]
+    fn test_increment_basic_binary_numbers() {
+        let tests = [
+            ("0b00000100", 1, "0b00000101"),
+            ("0b00000100", -1, "0b00000011"),
+            ("0b00000100", 2, "0b00000110"),
+            ("0b00000100", -2, "0b00000010"),
+            ("0b00000001", -1, "0b00000000"),
+            ("0b00111111", 10, "0b01001001"),
+            ("0b11111111", 1, "0b100000000"),
+            ("0b10000000", -1, "0b01111111"),
+            ("0b0000", -1, "0b0000"),
+            (
+                "0b1111111111111111111111111111111111111111111111111111111111111111",
+                1,
+                "0b10000000000000000000000000000000000000000000000000000000000000000",
+            ),
+            (
+                "0b1111111111111111111111111111111111111111111111111111111111111111",
+                2,
+                "0b10000000000000000000000000000000000000000000000000000000000000001",
+            ),
+            (
+                "0b1111111111111111111111111111111111111111111111111111111111111111",
+                -1,
+                "0b1111111111111111111111111111111111111111111111111111111111111110",
+            ),
+        ];
+
+        for (original, amount, expected) in tests {
+            assert_eq!(increment(original, amount).unwrap(), expected);
+        }
+    }
+
+    #[test]
+    fn test_increment_with_separators() {
+        let tests = [
+            ("999_999", 1, "1_000_000"),
+            ("1_000_000", -1, "999_999"),
+            ("-999_999", -1, "-1_000_000"),
+            ("0x0000_0000_0001", 0x1_ffff_0000, "0x0001_ffff_0001"),
+            ("0x0000_0000", -1, "0x0000_0000"),
+            ("0x0000_0000_0000", -1, "0x0000_0000_0000"),
+            ("0b01111111_11111111", 1, "0b10000000_00000000"),
+            ("0b11111111_11111111", 1, "0b1_00000000_00000000"),
+        ];
+
+        for (original, amount, expected) in tests {
+            assert_eq!(increment(original, amount).unwrap(), expected);
+        }
+    }
+
+    #[test]
+    fn test_leading_and_trailing_separators_arent_a_match() {
+        assert_eq!(increment("9_", 1), None);
+        assert_eq!(increment("_9", 1), None);
+        assert_eq!(increment("_9_", 1), None);
+    }
+}

helix-core/src/increment/mod.rs

@@ -1,8 +1,10 @@
-pub mod date_time;
-pub mod number;
+mod date_time;
+mod integer;
 
-use crate::{Range, Tendril};
-
-pub trait Increment {
-    fn increment(&self, amount: i64) -> (Range, Tendril);
+pub fn integer(selected_text: &str, amount: i64) -> Option<String> {
+    integer::increment(selected_text, amount)
+}
+
+pub fn date_time(selected_text: &str, amount: i64) -> Option<String> {
+    date_time::increment(selected_text, amount)
 }

helix-core/src/increment/number.rs

@@ -1,507 +0,0 @@
-use std::borrow::Cow;
-
-use ropey::RopeSlice;
-
-use super::Increment;
-
-use crate::{
-    textobject::{textobject_word, TextObject},
-    Range, Tendril,
-};
-
-#[derive(Debug, PartialEq, Eq)]
-pub struct NumberIncrementor<'a> {
-    value: i64,
-    radix: u32,
-    range: Range,
-
-    text: RopeSlice<'a>,
-}
-
-impl<'a> NumberIncrementor<'a> {
-    /// Return information about number under rang if there is one.
-    pub fn from_range(text: RopeSlice, range: Range) -> Option<NumberIncrementor> {
-        // If the cursor is on the minus sign of a number we want to get the word textobject to the
-        // right of it.
-        let range = if range.to() < text.len_chars()
-            && range.to() - range.from() <= 1
-            && text.char(range.from()) == '-'
-        {
-            Range::new(range.from() + 1, range.to() + 1)
-        } else {
-            range
-        };
-
-        let range = textobject_word(text, range, TextObject::Inside, 1, false);
-
-        // If there is a minus sign to the left of the word object, we want to include it in the range.
-        let range = if range.from() > 0 && text.char(range.from() - 1) == '-' {
-            range.extend(range.from() - 1, range.from())
-        } else {
-            range
-        };
-
-        let word: String = text
-            .slice(range.from()..range.to())
-            .chars()
-            .filter(|&c| c != '_')
-            .collect();
-        let (radix, prefixed) = if word.starts_with("0x") {
-            (16, true)
-        } else if word.starts_with("0o") {
-            (8, true)
-        } else if word.starts_with("0b") {
-            (2, true)
-        } else {
-            (10, false)
-        };
-
-        let number = if prefixed { &word[2..] } else { &word };
-
-        let value = i128::from_str_radix(number, radix).ok()?;
-        if (value.is_positive() && value.leading_zeros() < 64)
-            || (value.is_negative() && value.leading_ones() < 64)
-        {
-            return None;
-        }
-
-        let value = value as i64;
-        Some(NumberIncrementor {
-            range,
-            value,
-            radix,
-            text,
-        })
-    }
-}
-
-impl<'a> Increment for NumberIncrementor<'a> {
-    fn increment(&self, amount: i64) -> (Range, Tendril) {
-        let old_text: Cow<str> = self.text.slice(self.range.from()..self.range.to()).into();
-        let old_length = old_text.len();
-        let new_value = self.value.wrapping_add(amount);
-
-        // Get separator indexes from right to left.
-        let separator_rtl_indexes: Vec<usize> = old_text
-            .chars()
-            .rev()
-            .enumerate()
-            .filter_map(|(i, c)| if c == '_' { Some(i) } else { None })
-            .collect();
-
-        let format_length = if self.radix == 10 {
-            match (self.value.is_negative(), new_value.is_negative()) {
-                (true, false) => old_length - 1,
-                (false, true) => old_length + 1,
-                _ => old_text.len(),
-            }
-        } else {
-            old_text.len() - 2
-        } - separator_rtl_indexes.len();
-
-        let mut new_text = match self.radix {
-            2 => format!("0b{:01$b}", new_value, format_length),
-            8 => format!("0o{:01$o}", new_value, format_length),
-            10 if old_text.starts_with('0') || old_text.starts_with("-0") => {
-                format!("{:01$}", new_value, format_length)
-            }
-            10 => format!("{}", new_value),
-            16 => {
-                let (lower_count, upper_count): (usize, usize) =
-                    old_text.chars().skip(2).fold((0, 0), |(lower, upper), c| {
-                        (
-                            lower + usize::from(c.is_ascii_lowercase()),
-                            upper + usize::from(c.is_ascii_uppercase()),
-                        )
-                    });
-                if upper_count > lower_count {
-                    format!("0x{:01$X}", new_value, format_length)
-                } else {
-                    format!("0x{:01$x}", new_value, format_length)
-                }
-            }
-            _ => unimplemented!("radix not supported: {}", self.radix),
-        };
-
-        // Add separators from original number.
-        for &rtl_index in &separator_rtl_indexes {
-            if rtl_index < new_text.len() {
-                let new_index = new_text.len() - rtl_index;
-                new_text.insert(new_index, '_');
-            }
-        }
-
-        // Add in additional separators if necessary.
-        if new_text.len() > old_length && !separator_rtl_indexes.is_empty() {
-            let spacing = match separator_rtl_indexes.as_slice() {
-                [.., b, a] => a - b - 1,
-                _ => separator_rtl_indexes[0],
-            };
-
-            let prefix_length = if self.radix == 10 { 0 } else { 2 };
-            if let Some(mut index) = new_text.find('_') {
-                while index - prefix_length > spacing {
-                    index -= spacing;
-                    new_text.insert(index, '_');
-                }
-            }
-        }
-
-        (self.range, new_text.into())
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-    use crate::Rope;
-
-    #[test]
-    fn test_decimal_at_point() {
-        let rope = Rope::from_str("Test text 12345 more text.");
-        let range = Range::point(12);
-        assert_eq!(
-            NumberIncrementor::from_range(rope.slice(..), range),
-            Some(NumberIncrementor {
-                range: Range::new(10, 15),
-                value: 12345,
-                radix: 10,
-                text: rope.slice(..),
-            })
-        );
-    }
-
-    #[test]
-    fn test_uppercase_hexadecimal_at_point() {
-        let rope = Rope::from_str("Test text 0x123ABCDEF more text.");
-        let range = Range::point(12);
-        assert_eq!(
-            NumberIncrementor::from_range(rope.slice(..), range),
-            Some(NumberIncrementor {
-                range: Range::new(10, 21),
-                value: 0x123ABCDEF,
-                radix: 16,
-                text: rope.slice(..),
-            })
-        );
-    }
-
-    #[test]
-    fn test_lowercase_hexadecimal_at_point() {
-        let rope = Rope::from_str("Test text 0xfa3b4e more text.");
-        let range = Range::point(12);
-        assert_eq!(
-            NumberIncrementor::from_range(rope.slice(..), range),
-            Some(NumberIncrementor {
-                range: Range::new(10, 18),
-                value: 0xfa3b4e,
-                radix: 16,
-                text: rope.slice(..),
-            })
-        );
-    }
-
-    #[test]
-    fn test_octal_at_point() {
-        let rope = Rope::from_str("Test text 0o1074312 more text.");
-        let range = Range::point(12);
-        assert_eq!(
-            NumberIncrementor::from_range(rope.slice(..), range),
-            Some(NumberIncrementor {
-                range: Range::new(10, 19),
-                value: 0o1074312,
-                radix: 8,
-                text: rope.slice(..),
-            })
-        );
-    }
-
-    #[test]
-    fn test_binary_at_point() {
-        let rope = Rope::from_str("Test text 0b10111010010101 more text.");
-        let range = Range::point(12);
-        assert_eq!(
-            NumberIncrementor::from_range(rope.slice(..), range),
-            Some(NumberIncrementor {
-                range: Range::new(10, 26),
-                value: 0b10111010010101,
-                radix: 2,
-                text: rope.slice(..),
-            })
-        );
-    }
-
-    #[test]
-    fn test_negative_decimal_at_point() {
-        let rope = Rope::from_str("Test text -54321 more text.");
-        let range = Range::point(12);
-        assert_eq!(
-            NumberIncrementor::from_range(rope.slice(..), range),
-            Some(NumberIncrementor {
-                range: Range::new(10, 16),
-                value: -54321,
-                radix: 10,
-                text: rope.slice(..),
-            })
-        );
-    }
-
-    #[test]
-    fn test_decimal_with_leading_zeroes_at_point() {
-        let rope = Rope::from_str("Test text 000045326 more text.");
-        let range = Range::point(12);
-        assert_eq!(
-            NumberIncrementor::from_range(rope.slice(..), range),
-            Some(NumberIncrementor {
-                range: Range::new(10, 19),
-                value: 45326,
-                radix: 10,
-                text: rope.slice(..),
-            })
-        );
-    }
-
-    #[test]
-    fn test_negative_decimal_cursor_on_minus_sign() {
-        let rope = Rope::from_str("Test text -54321 more text.");
-        let range = Range::point(10);
-        assert_eq!(
-            NumberIncrementor::from_range(rope.slice(..), range),
-            Some(NumberIncrementor {
-                range: Range::new(10, 16),
-                value: -54321,
-                radix: 10,
-                text: rope.slice(..),
-            })
-        );
-    }
-
-    #[test]
-    fn test_number_under_range_start_of_rope() {
-        let rope = Rope::from_str("100");
-        let range = Range::point(0);
-        assert_eq!(
-            NumberIncrementor::from_range(rope.slice(..), range),
-            Some(NumberIncrementor {
-                range: Range::new(0, 3),
-                value: 100,
-                radix: 10,
-                text: rope.slice(..),
-            })
-        );
-    }
-
-    #[test]
-    fn test_number_under_range_end_of_rope() {
-        let rope = Rope::from_str("100");
-        let range = Range::point(2);
-        assert_eq!(
-            NumberIncrementor::from_range(rope.slice(..), range),
-            Some(NumberIncrementor {
-                range: Range::new(0, 3),
-                value: 100,
-                radix: 10,
-                text: rope.slice(..),
-            })
-        );
-    }
-
-    #[test]
-    fn test_number_surrounded_by_punctuation() {
-        let rope = Rope::from_str(",100;");
-        let range = Range::point(1);
-        assert_eq!(
-            NumberIncrementor::from_range(rope.slice(..), range),
-            Some(NumberIncrementor {
-                range: Range::new(1, 4),
-                value: 100,
-                radix: 10,
-                text: rope.slice(..),
-            })
-        );
-    }
-
-    #[test]
-    fn test_not_a_number_point() {
-        let rope = Rope::from_str("Test text 45326 more text.");
-        let range = Range::point(6);
-        assert_eq!(NumberIncrementor::from_range(rope.slice(..), range), None);
-    }
-
-    #[test]
-    fn test_number_too_large_at_point() {
-        let rope = Rope::from_str("Test text 0xFFFFFFFFFFFFFFFFF more text.");
-        let range = Range::point(12);
-        assert_eq!(NumberIncrementor::from_range(rope.slice(..), range), None);
-    }
-
-    #[test]
-    fn test_number_cursor_one_right_of_number() {
-        let rope = Rope::from_str("100 ");
-        let range = Range::point(3);
-        assert_eq!(NumberIncrementor::from_range(rope.slice(..), range), None);
-    }
-
-    #[test]
-    fn test_number_cursor_one_left_of_number() {
-        let rope = Rope::from_str(" 100");
-        let range = Range::point(0);
-        assert_eq!(NumberIncrementor::from_range(rope.slice(..), range), None);
-    }
-
-    #[test]
-    fn test_increment_basic_decimal_numbers() {
-        let tests = [
-            ("100", 1, "101"),
-            ("100", -1, "99"),
-            ("99", 1, "100"),
-            ("100", 1000, "1100"),
-            ("100", -1000, "-900"),
-            ("-1", 1, "0"),
-            ("-1", 2, "1"),
-            ("1", -1, "0"),
-            ("1", -2, "-1"),
-        ];
-
-        for (original, amount, expected) in tests {
-            let rope = Rope::from_str(original);
-            let range = Range::point(0);
-            assert_eq!(
-                NumberIncrementor::from_range(rope.slice(..), range)
-                    .unwrap()
-                    .increment(amount)
-                    .1,
-                Tendril::from(expected)
-            );
-        }
-    }
-
-    #[test]
-    fn test_increment_basic_hexadecimal_numbers() {
-        let tests = [
-            ("0x0100", 1, "0x0101"),
-            ("0x0100", -1, "0x00ff"),
-            ("0x0001", -1, "0x0000"),
-            ("0x0000", -1, "0xffffffffffffffff"),
-            ("0xffffffffffffffff", 1, "0x0000000000000000"),
-            ("0xffffffffffffffff", 2, "0x0000000000000001"),
-            ("0xffffffffffffffff", -1, "0xfffffffffffffffe"),
-            ("0xABCDEF1234567890", 1, "0xABCDEF1234567891"),
-            ("0xabcdef1234567890", 1, "0xabcdef1234567891"),
-        ];
-
-        for (original, amount, expected) in tests {
-            let rope = Rope::from_str(original);
-            let range = Range::point(0);
-            assert_eq!(
-                NumberIncrementor::from_range(rope.slice(..), range)
-                    .unwrap()
-                    .increment(amount)
-                    .1,
-                Tendril::from(expected)
-            );
-        }
-    }
-
-    #[test]
-    fn test_increment_basic_octal_numbers() {
-        let tests = [
-            ("0o0107", 1, "0o0110"),
-            ("0o0110", -1, "0o0107"),
-            ("0o0001", -1, "0o0000"),
-            ("0o7777", 1, "0o10000"),
-            ("0o1000", -1, "0o0777"),
-            ("0o0107", 10, "0o0121"),
-            ("0o0000", -1, "0o1777777777777777777777"),
-            ("0o1777777777777777777777", 1, "0o0000000000000000000000"),
-            ("0o1777777777777777777777", 2, "0o0000000000000000000001"),
-            ("0o1777777777777777777777", -1, "0o1777777777777777777776"),
-        ];
-
-        for (original, amount, expected) in tests {
-            let rope = Rope::from_str(original);
-            let range = Range::point(0);
-            assert_eq!(
-                NumberIncrementor::from_range(rope.slice(..), range)
-                    .unwrap()
-                    .increment(amount)
-                    .1,
-                Tendril::from(expected)
-            );
-        }
-    }
-
-    #[test]
-    fn test_increment_basic_binary_numbers() {
-        let tests = [
-            ("0b00000100", 1, "0b00000101"),
-            ("0b00000100", -1, "0b00000011"),
-            ("0b00000100", 2, "0b00000110"),
-            ("0b00000100", -2, "0b00000010"),
-            ("0b00000001", -1, "0b00000000"),
-            ("0b00111111", 10, "0b01001001"),
-            ("0b11111111", 1, "0b100000000"),
-            ("0b10000000", -1, "0b01111111"),
-            (
-                "0b0000",
-                -1,
-                "0b1111111111111111111111111111111111111111111111111111111111111111",
-            ),
-            (
-                "0b1111111111111111111111111111111111111111111111111111111111111111",
-                1,
-                "0b0000000000000000000000000000000000000000000000000000000000000000",
-            ),
-            (
-                "0b1111111111111111111111111111111111111111111111111111111111111111",
-                2,
-                "0b0000000000000000000000000000000000000000000000000000000000000001",
-            ),
-            (
-                "0b1111111111111111111111111111111111111111111111111111111111111111",
-                -1,
-                "0b1111111111111111111111111111111111111111111111111111111111111110",
-            ),
-        ];
-
-        for (original, amount, expected) in tests {
-            let rope = Rope::from_str(original);
-            let range = Range::point(0);
-            assert_eq!(
-                NumberIncrementor::from_range(rope.slice(..), range)
-                    .unwrap()
-                    .increment(amount)
-                    .1,
-                Tendril::from(expected)
-            );
-        }
-    }
-
-    #[test]
-    fn test_increment_with_separators() {
-        let tests = [
-            ("999_999", 1, "1_000_000"),
-            ("1_000_000", -1, "999_999"),
-            ("-999_999", -1, "-1_000_000"),
-            ("0x0000_0000_0001", 0x1_ffff_0000, "0x0001_ffff_0001"),
-            ("0x0000_0000_0001", 0x1_ffff_0000, "0x0001_ffff_0001"),
-            ("0x0000_0000_0001", 0x1_ffff_0000, "0x0001_ffff_0001"),
-            ("0x0000_0000", -1, "0xffff_ffff_ffff_ffff"),
-            ("0x0000_0000_0000", -1, "0xffff_ffff_ffff_ffff"),
-            ("0b01111111_11111111", 1, "0b10000000_00000000"),
-            ("0b11111111_11111111", 1, "0b1_00000000_00000000"),
-        ];
-
-        for (original, amount, expected) in tests {
-            let rope = Rope::from_str(original);
-            let range = Range::point(0);
-            assert_eq!(
-                NumberIncrementor::from_range(rope.slice(..), range)
-                    .unwrap()
-                    .increment(amount)
-                    .1,
-                Tendril::from(expected)
-            );
-        }
-    }
-}

helix-core/src/indent.rs

@@ -4,6 +4,7 @@ use tree_sitter::{Query, QueryCursor, QueryPredicateArg};
 
 use crate::{
     chars::{char_is_line_ending, char_is_whitespace},
+    graphemes::tab_width_at,
     syntax::{LanguageConfiguration, RopeProvider, Syntax},
     tree_sitter::Node,
     Rope, RopeSlice,
@@ -56,6 +57,14 @@ impl IndentStyle {
             }
         }
     }
+
+    #[inline]
+    pub fn indent_width(&self, tab_width: usize) -> usize {
+        match *self {
+            IndentStyle::Tabs => tab_width,
+            IndentStyle::Spaces(width) => width as usize,
+        }
+    }
 }
 
 /// Attempts to detect the indentation style used in a document.
@@ -177,17 +186,17 @@ pub fn auto_detect_indent_style(document_text: &Rope) -> Option<IndentStyle> {
 
 /// To determine indentation of a newly inserted line, figure out the indentation at the last col
 /// of the previous line.
-pub fn indent_level_for_line(line: RopeSlice, tab_width: usize) -> usize {
+pub fn indent_level_for_line(line: RopeSlice, tab_width: usize, indent_width: usize) -> usize {
     let mut len = 0;
     for ch in line.chars() {
         match ch {
-            '\t' => len += tab_width,
+            '\t' => len += tab_width_at(len, tab_width as u16),
             ' ' => len += 1,
             _ => break,
         }
     }
 
-    len / tab_width
+    len / indent_width
 }
 
 /// Computes for node and all ancestors whether they are the first node on their line.
@@ -466,6 +475,7 @@ fn extend_nodes<'a>(
     text: RopeSlice,
     line: usize,
     tab_width: usize,
+    indent_width: usize,
 ) {
     let mut stop_extend = false;
 
@@ -490,10 +500,12 @@ fn extend_nodes<'a>(
                         if deepest_preceding.end_position().row == line {
                             extend_node = true;
                         } else {
-                            let cursor_indent = indent_level_for_line(text.line(line), tab_width);
+                            let cursor_indent =
+                                indent_level_for_line(text.line(line), tab_width, indent_width);
                             let node_indent = indent_level_for_line(
                                 text.line(deepest_preceding.start_position().row),
                                 tab_width,
+                                indent_width,
                             );
                             if cursor_indent > node_indent {
                                 extend_node = true;
@@ -562,6 +574,7 @@ pub fn treesitter_indent_for_pos(
     syntax: &Syntax,
     indent_style: &IndentStyle,
     tab_width: usize,
+    indent_width: usize,
     text: RopeSlice,
     line: usize,
     pos: usize,
@@ -604,7 +617,7 @@ pub fn treesitter_indent_for_pos(
                 &mut cursor,
                 text,
                 query_range,
-                new_line.then(|| (line, byte_pos)),
+                new_line.then_some((line, byte_pos)),
             );
             ts_parser.cursors.push(cursor);
             (query_result, deepest_preceding)
@@ -622,9 +635,10 @@ pub fn treesitter_indent_for_pos(
             text,
             line,
             tab_width,
+            indent_width,
         );
     }
-    let mut first_in_line = get_first_in_line(node, new_line.then(|| byte_pos));
+    let mut first_in_line = get_first_in_line(node, new_line.then_some(byte_pos));
 
     let mut result = Indentation::default();
     // We always keep track of all the indent changes on one line, in order to only indent once
@@ -709,6 +723,7 @@ pub fn indent_for_newline(
     line_before_end_pos: usize,
     current_line: usize,
 ) -> String {
+    let indent_width = indent_style.indent_width(tab_width);
     if let (Some(query), Some(syntax)) = (
         language_config.and_then(|config| config.indent_query()),
         syntax,
@@ -718,6 +733,7 @@ pub fn indent_for_newline(
             syntax,
             indent_style,
             tab_width,
+            indent_width,
             text,
             line_before,
             line_before_end_pos,
@@ -726,7 +742,7 @@ pub fn indent_for_newline(
             return indent;
         };
     }
-    let indent_level = indent_level_for_line(text.line(current_line), tab_width);
+    let indent_level = indent_level_for_line(text.line(current_line), tab_width, indent_width);
     indent_style.as_str().repeat(indent_level)
 }
 
@@ -763,12 +779,22 @@ mod test {
     #[test]
     fn test_indent_level() {
         let tab_width = 4;
+        let indent_width = 4;
         let line = Rope::from("        fn new"); // 8 spaces
-        assert_eq!(indent_level_for_line(line.slice(..), tab_width), 2);
+        assert_eq!(
+            indent_level_for_line(line.slice(..), tab_width, indent_width),
+            2
+        );
         let line = Rope::from("\t\t\tfn new"); // 3 tabs
-        assert_eq!(indent_level_for_line(line.slice(..), tab_width), 3);
+        assert_eq!(
+            indent_level_for_line(line.slice(..), tab_width, indent_width),
+            3
+        );
         // mixed indentation
         let line = Rope::from("\t    \tfn new"); // 1 tab, 4 spaces, tab
-        assert_eq!(indent_level_for_line(line.slice(..), tab_width), 3);
+        assert_eq!(
+            indent_level_for_line(line.slice(..), tab_width, indent_width),
+            3
+        );
     }
 }

helix-core/src/lib.rs

@@ -6,6 +6,7 @@ pub mod comment;
 pub mod config;
 pub mod diagnostic;
 pub mod diff;
+pub mod doc_formatter;
 pub mod graphemes;
 pub mod history;
 pub mod increment;
@@ -24,6 +25,7 @@ pub mod shellwords;
 pub mod surround;
 pub mod syntax;
 pub mod test;
+pub mod text_annotations;
 pub mod textobject;
 mod transaction;
 pub mod wrap;
@@ -34,55 +36,12 @@ pub mod unicode {
     pub use unicode_width as width;
 }
 
+pub use helix_loader::find_workspace;
+
 pub fn find_first_non_whitespace_char(line: RopeSlice) -> Option<usize> {
     line.chars().position(|ch| !ch.is_whitespace())
 }
 
-/// Find project root.
-///
-/// Order of detection:
-/// * Top-most folder containing a root marker in current git repository
-/// * Git repository root if no marker detected
-/// * Top-most folder containing a root marker if not git repository detected
-/// * Current working directory as fallback
-pub fn find_root(root: Option<&str>, root_markers: &[String]) -> std::path::PathBuf {
-    let current_dir = std::env::current_dir().expect("unable to determine current directory");
-
-    let root = match root {
-        Some(root) => {
-            let root = std::path::Path::new(root);
-            if root.is_absolute() {
-                root.to_path_buf()
-            } else {
-                current_dir.join(root)
-            }
-        }
-        None => current_dir.clone(),
-    };
-
-    let mut top_marker = None;
-    for ancestor in root.ancestors() {
-        if root_markers
-            .iter()
-            .any(|marker| ancestor.join(marker).exists())
-        {
-            top_marker = Some(ancestor);
-        }
-
-        if ancestor.join(".git").exists() {
-            // Top marker is repo root if not root marker was detected yet
-            if top_marker.is_none() {
-                top_marker = Some(ancestor);
-            }
-            // Don't go higher than repo if we're in one
-            break;
-        }
-    }
-
-    // Return the found top marker or the current_dir as fallback
-    top_marker.map_or(current_dir, |a| a.to_path_buf())
-}
-
 pub use ropey::{self, str_utils, Rope, RopeBuilder, RopeSlice};
 
 // pub use tendril::StrTendril as Tendril;
@@ -95,8 +54,12 @@ pub use {regex, tree_sitter};
 
 pub use graphemes::RopeGraphemes;
 pub use position::{
-    coords_at_pos, pos_at_coords, pos_at_visual_coords, visual_coords_at_pos, Position,
+    char_idx_at_visual_offset, coords_at_pos, pos_at_coords, visual_offset_from_anchor,
+    visual_offset_from_block, Position, VisualOffsetError,
 };
+#[allow(deprecated)]
+pub use position::{pos_at_visual_coords, visual_coords_at_pos};
+
 pub use selection::{Range, Selection};
 pub use smallvec::{smallvec, SmallVec};
 pub use syntax::Syntax;
@@ -104,4 +67,4 @@ pub use syntax::Syntax;
 pub use diagnostic::Diagnostic;
 
 pub use line_ending::{LineEnding, DEFAULT_LINE_ENDING};
-pub use transaction::{Assoc, Change, ChangeSet, Operation, Transaction};
+pub use transaction::{Assoc, Change, ChangeSet, Deletion, Operation, Transaction};

helix-core/src/line_ending.rs

@@ -203,6 +203,13 @@ pub fn line_end_char_index(slice: &RopeSlice, line: usize) -> usize {
             .unwrap_or(0)
 }
 
+pub fn line_end_byte_index(slice: &RopeSlice, line: usize) -> usize {
+    slice.line_to_byte(line + 1)
+        - get_line_ending(&slice.line(line))
+            .map(|le| le.as_str().len())
+            .unwrap_or(0)
+}
+
 /// Fetches line `line_idx` from the passed rope slice, sans any line ending.
 pub fn line_without_line_ending<'a>(slice: &'a RopeSlice, line_idx: usize) -> RopeSlice<'a> {
     let start = slice.line_to_char(line_idx);

helix-core/src/movement.rs

@@ -4,16 +4,19 @@ use ropey::iter::Chars;
 use tree_sitter::{Node, QueryCursor};
 
 use crate::{
+    char_idx_at_visual_offset,
     chars::{categorize_char, char_is_line_ending, CharCategory},
+    doc_formatter::TextFormat,
     graphemes::{
         next_grapheme_boundary, nth_next_grapheme_boundary, nth_prev_grapheme_boundary,
         prev_grapheme_boundary,
     },
     line_ending::rope_is_line_ending,
-    pos_at_visual_coords,
+    position::char_idx_at_visual_block_offset,
     syntax::LanguageConfiguration,
+    text_annotations::TextAnnotations,
     textobject::TextObject,
-    visual_coords_at_pos, Position, Range, RopeSlice,
+    visual_offset_from_block, Range, RopeSlice,
 };
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
@@ -34,7 +37,8 @@ pub fn move_horizontally(
     dir: Direction,
     count: usize,
     behaviour: Movement,
-    _: usize,
+    _: &TextFormat,
+    _: &mut TextAnnotations,
 ) -> Range {
     let pos = range.cursor(slice);
 
@@ -48,35 +52,116 @@ pub fn move_horizontally(
     range.put_cursor(slice, new_pos, behaviour == Movement::Extend)
 }
 
+pub fn move_vertically_visual(
+    slice: RopeSlice,
+    range: Range,
+    dir: Direction,
+    count: usize,
+    behaviour: Movement,
+    text_fmt: &TextFormat,
+    annotations: &mut TextAnnotations,
+) -> Range {
+    if !text_fmt.soft_wrap {
+        return move_vertically(slice, range, dir, count, behaviour, text_fmt, annotations);
+    }
+    annotations.clear_line_annotations();
+    let pos = range.cursor(slice);
+
+    // Compute the current position's 2d coordinates.
+    let (visual_pos, block_off) = visual_offset_from_block(slice, pos, pos, text_fmt, annotations);
+    let new_col = range
+        .old_visual_position
+        .map_or(visual_pos.col as u32, |(_, col)| col);
+
+    // Compute the new position.
+    let mut row_off = match dir {
+        Direction::Forward => count as isize,
+        Direction::Backward => -(count as isize),
+    };
+
+    // TODO how to handle inline annotations that span an entire visual line (very unlikely).
+
+    // Compute visual offset relative to block start to avoid trasversing the block twice
+    row_off += visual_pos.row as isize;
+    let new_pos = char_idx_at_visual_offset(
+        slice,
+        block_off,
+        row_off,
+        new_col as usize,
+        text_fmt,
+        annotations,
+    )
+    .0;
+
+    // Special-case to avoid moving to the end of the last non-empty line.
+    if behaviour == Movement::Extend && slice.line(slice.char_to_line(new_pos)).len_chars() == 0 {
+        return range;
+    }
+
+    let mut new_range = range.put_cursor(slice, new_pos, behaviour == Movement::Extend);
+    new_range.old_visual_position = Some((0, new_col));
+    new_range
+}
+
 pub fn move_vertically(
     slice: RopeSlice,
     range: Range,
     dir: Direction,
     count: usize,
     behaviour: Movement,
-    tab_width: usize,
+    text_fmt: &TextFormat,
+    annotations: &mut TextAnnotations,
 ) -> Range {
+    annotations.clear_line_annotations();
     let pos = range.cursor(slice);
+    let line_idx = slice.char_to_line(pos);
+    let line_start = slice.line_to_char(line_idx);
 
     // Compute the current position's 2d coordinates.
-    let Position { row, col } = visual_coords_at_pos(slice, pos, tab_width);
-    let horiz = range.horiz.unwrap_or(col as u32);
+    let visual_pos = visual_offset_from_block(slice, line_start, pos, text_fmt, annotations).0;
+    let (mut new_row, new_col) = range
+        .old_visual_position
+        .map_or((visual_pos.row as u32, visual_pos.col as u32), |pos| pos);
+    new_row = new_row.max(visual_pos.row as u32);
+    let line_idx = slice.char_to_line(pos);
 
     // Compute the new position.
-    let new_row = match dir {
-        Direction::Forward => (row + count).min(slice.len_lines().saturating_sub(1)),
-        Direction::Backward => row.saturating_sub(count),
+    let mut new_line_idx = match dir {
+        Direction::Forward => line_idx.saturating_add(count),
+        Direction::Backward => line_idx.saturating_sub(count),
     };
-    let new_col = col.max(horiz as usize);
-    let new_pos = pos_at_visual_coords(slice, Position::new(new_row, new_col), tab_width);
+
+    let line = if new_line_idx >= slice.len_lines() - 1 {
+        // there is no line terminator for the last line
+        // so the logic below is not necessary here
+        new_line_idx = slice.len_lines() - 1;
+        slice
+    } else {
+        // char_idx_at_visual_block_offset returns a one-past-the-end index
+        // in case it reaches the end of the slice
+        // to avoid moving to the nextline in that case the line terminator is removed from the line
+        let new_line_end = prev_grapheme_boundary(slice, slice.line_to_char(new_line_idx + 1));
+        slice.slice(..new_line_end)
+    };
+
+    let new_line_start = line.line_to_char(new_line_idx);
+
+    let (new_pos, _) = char_idx_at_visual_block_offset(
+        line,
+        new_line_start,
+        new_row as usize,
+        new_col as usize,
+        text_fmt,
+        annotations,
+    );
 
     // Special-case to avoid moving to the end of the last non-empty line.
-    if behaviour == Movement::Extend && slice.line(new_row).len_chars() == 0 {
+    if behaviour == Movement::Extend && slice.line(new_line_idx).len_chars() == 0 {
         return range;
     }
 
     let mut new_range = range.put_cursor(slice, new_pos, behaviour == Movement::Extend);
-    new_range.horiz = Some(horiz);
+    new_range.old_visual_position = Some((new_row, new_col));
     new_range
 }
 
@@ -142,9 +227,15 @@ fn word_move(slice: RopeSlice, range: Range, count: usize, target: WordMotionTar
     };
 
     // Do the main work.
-    (0..count).fold(start_range, |r, _| {
-        slice.chars_at(r.head).range_to_target(target, r)
-    })
+    let mut range = start_range;
+    for _ in 0..count {
+        let next_range = slice.chars_at(range.head).range_to_target(target, range);
+        if range == next_range {
+            break;
+        }
+        range = next_range;
+    }
+    range
 }
 
 pub fn move_prev_paragraph(
@@ -166,6 +257,7 @@ pub fn move_prev_paragraph(
     let mut lines = slice.lines_at(line);
     lines.reverse();
     let mut lines = lines.map(rope_is_line_ending).peekable();
+    let mut last_line = line;
     for _ in 0..count {
         while lines.next_if(|&e| e).is_some() {
             line -= 1;
@@ -173,6 +265,10 @@ pub fn move_prev_paragraph(
         while lines.next_if(|&e| !e).is_some() {
             line -= 1;
         }
+        if line == last_line {
+            break;
+        }
+        last_line = line;
     }
 
     let head = slice.line_to_char(line);
@@ -208,6 +304,7 @@ pub fn move_next_paragraph(
         line += 1;
     }
     let mut lines = slice.lines_at(line).map(rope_is_line_ending).peekable();
+    let mut last_line = line;
     for _ in 0..count {
         while lines.next_if(|&e| !e).is_some() {
             line += 1;
@@ -215,6 +312,10 @@ pub fn move_next_paragraph(
         while lines.next_if(|&e| e).is_some() {
             line += 1;
         }
+        if line == last_line {
+            break;
+        }
+        last_line = line;
     }
     let head = slice.line_to_char(line);
     let anchor = if behavior == Movement::Move {
@@ -438,7 +539,14 @@ pub fn goto_treesitter_object(
         // head of range should be at beginning
         Some(Range::new(start_char, end_char))
     };
-    (0..count).fold(range, |range, _| get_range(range).unwrap_or(range))
+    let mut last_range = range;
+    for _ in 0..count {
+        match get_range(last_range) {
+            Some(r) if r != last_range => last_range = r,
+            _ => break,
+        }
+    }
+    last_range
 }
 
 #[cfg(test)]
@@ -473,7 +581,16 @@ mod test {
         assert_eq!(
             coords_at_pos(
                 slice,
-                move_vertically(slice, range, Direction::Forward, 1, Movement::Move, 4).head
+                move_vertically_visual(
+                    slice,
+                    range,
+                    Direction::Forward,
+                    1,
+                    Movement::Move,
+                    &TextFormat::default(),
+                    &mut TextAnnotations::default(),
+                )
+                .head
             ),
             (1, 3).into()
         );
@@ -497,7 +614,15 @@ mod test {
         ];
 
         for ((direction, amount), coordinates) in moves_and_expected_coordinates {
-            range = move_horizontally(slice, range, direction, amount, Movement::Move, 0);
+            range = move_horizontally(
+                slice,
+                range,
+                direction,
+                amount,
+                Movement::Move,
+                &TextFormat::default(),
+                &mut TextAnnotations::default(),
+            );
             assert_eq!(coords_at_pos(slice, range.head), coordinates.into())
         }
     }
@@ -523,7 +648,15 @@ mod test {
         ];
 
         for ((direction, amount), coordinates) in moves_and_expected_coordinates {
-            range = move_horizontally(slice, range, direction, amount, Movement::Move, 0);
+            range = move_horizontally(
+                slice,
+                range,
+                direction,
+                amount,
+                Movement::Move,
+                &TextFormat::default(),
+                &mut TextAnnotations::default(),
+            );
             assert_eq!(coords_at_pos(slice, range.head), coordinates.into());
             assert_eq!(range.head, range.anchor);
         }
@@ -545,7 +678,15 @@ mod test {
         ];
 
         for (direction, amount) in moves {
-            range = move_horizontally(slice, range, direction, amount, Movement::Extend, 0);
+            range = move_horizontally(
+                slice,
+                range,
+                direction,
+                amount,
+                Movement::Extend,
+                &TextFormat::default(),
+                &mut TextAnnotations::default(),
+            );
             assert_eq!(range.anchor, original_anchor);
         }
     }
@@ -569,7 +710,15 @@ mod test {
         ];
 
         for ((direction, amount), coordinates) in moves_and_expected_coordinates {
-            range = move_vertically(slice, range, direction, amount, Movement::Move, 4);
+            range = move_vertically_visual(
+                slice,
+                range,
+                direction,
+                amount,
+                Movement::Move,
+                &TextFormat::default(),
+                &mut TextAnnotations::default(),
+            );
             assert_eq!(coords_at_pos(slice, range.head), coordinates.into());
             assert_eq!(range.head, range.anchor);
         }
@@ -603,8 +752,24 @@ mod test {
 
         for ((axis, direction, amount), coordinates) in moves_and_expected_coordinates {
             range = match axis {
-                Axis::H => move_horizontally(slice, range, direction, amount, Movement::Move, 0),
-                Axis::V => move_vertically(slice, range, direction, amount, Movement::Move, 4),
+                Axis::H => move_horizontally(
+                    slice,
+                    range,
+                    direction,
+                    amount,
+                    Movement::Move,
+                    &TextFormat::default(),
+                    &mut TextAnnotations::default(),
+                ),
+                Axis::V => move_vertically_visual(
+                    slice,
+                    range,
+                    direction,
+                    amount,
+                    Movement::Move,
+                    &TextFormat::default(),
+                    &mut TextAnnotations::default(),
+                ),
             };
             assert_eq!(coords_at_pos(slice, range.head), coordinates.into());
             assert_eq!(range.head, range.anchor);
@@ -638,8 +803,24 @@ mod test {
 
         for ((axis, direction, amount), coordinates) in moves_and_expected_coordinates {
             range = match axis {
-                Axis::H => move_horizontally(slice, range, direction, amount, Movement::Move, 0),
-                Axis::V => move_vertically(slice, range, direction, amount, Movement::Move, 4),
+                Axis::H => move_horizontally(
+                    slice,
+                    range,
+                    direction,
+                    amount,
+                    Movement::Move,
+                    &TextFormat::default(),
+                    &mut TextAnnotations::default(),
+                ),
+                Axis::V => move_vertically_visual(
+                    slice,
+                    range,
+                    direction,
+                    amount,
+                    Movement::Move,
+                    &TextFormat::default(),
+                    &mut TextAnnotations::default(),
+                ),
             };
             assert_eq!(coords_at_pos(slice, range.head), coordinates.into());
             assert_eq!(range.head, range.anchor);
@@ -1293,7 +1474,7 @@ mod test {
             let text = Rope::from(s.as_str());
             let selection =
                 selection.transform(|r| move_prev_paragraph(text.slice(..), r, 1, Movement::Move));
-            let actual = crate::test::plain(&s, selection);
+            let actual = crate::test::plain(s.as_ref(), &selection);
             assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
         }
     }
@@ -1316,7 +1497,7 @@ mod test {
             let text = Rope::from(s.as_str());
             let selection =
                 selection.transform(|r| move_prev_paragraph(text.slice(..), r, 2, Movement::Move));
-            let actual = crate::test::plain(&s, selection);
+            let actual = crate::test::plain(s.as_ref(), &selection);
             assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
         }
     }
@@ -1339,7 +1520,7 @@ mod test {
             let text = Rope::from(s.as_str());
             let selection = selection
                 .transform(|r| move_prev_paragraph(text.slice(..), r, 1, Movement::Extend));
-            let actual = crate::test::plain(&s, selection);
+            let actual = crate::test::plain(s.as_ref(), &selection);
             assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
         }
     }
@@ -1359,7 +1540,7 @@ mod test {
                 "a\nb\n\n#[goto\nthird\n\n|]#paragraph",
             ),
             (
-                "a\nb#[\n|]#\ngoto\nsecond\n\nparagraph",
+                "a\nb#[\n|]#\n\ngoto\nsecond\n\nparagraph",
                 "a\nb#[\n\n|]#goto\nsecond\n\nparagraph",
             ),
             (
@@ -1381,7 +1562,7 @@ mod test {
             let text = Rope::from(s.as_str());
             let selection =
                 selection.transform(|r| move_next_paragraph(text.slice(..), r, 1, Movement::Move));
-            let actual = crate::test::plain(&s, selection);
+            let actual = crate::test::plain(s.as_ref(), &selection);
             assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
         }
     }
@@ -1404,7 +1585,7 @@ mod test {
             let text = Rope::from(s.as_str());
             let selection =
                 selection.transform(|r| move_next_paragraph(text.slice(..), r, 2, Movement::Move));
-            let actual = crate::test::plain(&s, selection);
+            let actual = crate::test::plain(s.as_ref(), &selection);
             assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
         }
     }
@@ -1427,7 +1608,7 @@ mod test {
             let text = Rope::from(s.as_str());
             let selection = selection
                 .transform(|r| move_next_paragraph(text.slice(..), r, 1, Movement::Extend));
-            let actual = crate::test::plain(&s, selection);
+            let actual = crate::test::plain(s.as_ref(), &selection);
             assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
         }
     }

helix-core/src/path.rs

@@ -40,6 +40,21 @@ pub fn expand_tilde(path: &Path) -> PathBuf {
 /// needs to improve on.
 /// Copied from cargo: <https://github.com/rust-lang/cargo/blob/070e459c2d8b79c5b2ac5218064e7603329c92ae/crates/cargo-util/src/paths.rs#L81>
 pub fn get_normalized_path(path: &Path) -> PathBuf {
+    // normalization strategy is to canonicalize first ancestor path that exists (i.e., canonicalize as much as possible),
+    // then run handrolled normalization on the non-existent remainder
+    let (base, path) = path
+        .ancestors()
+        .find_map(|base| {
+            let canonicalized_base = dunce::canonicalize(base).ok()?;
+            let remainder = path.strip_prefix(base).ok()?.into();
+            Some((canonicalized_base, remainder))
+        })
+        .unwrap_or_else(|| (PathBuf::new(), PathBuf::from(path)));
+
+    if path.as_os_str().is_empty() {
+        return base;
+    }
+
     let mut components = path.components().peekable();
     let mut ret = if let Some(c @ Component::Prefix(..)) = components.peek().cloned() {
         components.next();
@@ -63,7 +78,7 @@ pub fn get_normalized_path(path: &Path) -> PathBuf {
             }
         }
     }
-    ret
+    base.join(ret)
 }
 
 /// Returns the canonical, absolute form of a path with all intermediate components normalized.
@@ -82,13 +97,19 @@ pub fn get_canonicalized_path(path: &Path) -> std::io::Result<PathBuf> {
 }
 
 pub fn get_relative_path(path: &Path) -> PathBuf {
+    let path = PathBuf::from(path);
     let path = if path.is_absolute() {
-        let cwdir = std::env::current_dir().expect("couldn't determine current directory");
-        path.strip_prefix(cwdir).unwrap_or(path)
+        let cwdir = std::env::current_dir()
+            .map(|path| get_normalized_path(&path))
+            .expect("couldn't determine current directory");
+        get_normalized_path(&path)
+            .strip_prefix(cwdir)
+            .map(PathBuf::from)
+            .unwrap_or(path)
     } else {
         path
     };
-    fold_home_dir(path)
+    fold_home_dir(&path)
 }
 
 /// Returns a truncated filepath where the basepart of the path is reduced to the first

helix-core/src/position.rs

@@ -1,9 +1,11 @@
-use std::borrow::Cow;
+use std::{borrow::Cow, cmp::Ordering};
 
 use crate::{
     chars::char_is_line_ending,
+    doc_formatter::{DocumentFormatter, TextFormat},
     graphemes::{ensure_grapheme_boundary_prev, grapheme_width, RopeGraphemes},
     line_ending::line_end_char_index,
+    text_annotations::TextAnnotations,
     RopeSlice,
 };
 
@@ -73,6 +75,13 @@ pub fn coords_at_pos(text: RopeSlice, pos: usize) -> Position {
 /// Takes \t, double-width characters (CJK) into account as well as text
 /// not in the document in the future.
 /// See [`coords_at_pos`] for an "objective" one.
+///
+/// This function should be used very rarely. Usually `visual_offset_from_anchor`
+/// or `visual_offset_from_block` is preferable. However when you want to compute the
+/// actual visual row/column in the text (not what is actually shown on screen)
+/// then you should use this function. For example aligning text should ignore virtual
+/// text and softwrap.
+#[deprecated = "Doesn't account for softwrap or decorations, use visual_offset_from_anchor instead"]
 pub fn visual_coords_at_pos(text: RopeSlice, pos: usize, tab_width: usize) -> Position {
     let line = text.char_to_line(pos);
 
@@ -93,6 +102,106 @@ pub fn visual_coords_at_pos(text: RopeSlice, pos: usize, tab_width: usize) -> Po
     Position::new(line, col)
 }
 
+/// Returns the visual offset from the start of the first visual line
+/// in the block that contains anchor.
+/// Text is always wrapped at blocks, they usually correspond to
+/// actual line breaks but for very long lines
+/// softwrapping positions are estimated with an O(1) algorithm
+/// to ensure consistent performance for large lines (currently unimplemented)
+///
+/// Usually you want to use `visual_offset_from_anchor` instead but this function
+/// can be useful (and faster) if
+/// * You already know the visual position of the block
+/// * You only care about the horizontal offset (column) and not the vertical offset (row)
+pub fn visual_offset_from_block(
+    text: RopeSlice,
+    anchor: usize,
+    pos: usize,
+    text_fmt: &TextFormat,
+    annotations: &TextAnnotations,
+) -> (Position, usize) {
+    let mut last_pos = Position::default();
+    let (formatter, block_start) =
+        DocumentFormatter::new_at_prev_checkpoint(text, text_fmt, annotations, anchor);
+    let mut char_pos = block_start;
+
+    for (grapheme, vpos) in formatter {
+        last_pos = vpos;
+        char_pos += grapheme.doc_chars();
+
+        if char_pos > pos {
+            return (last_pos, block_start);
+        }
+    }
+
+    (last_pos, block_start)
+}
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+pub enum VisualOffsetError {
+    PosBeforeAnchorRow,
+    PosAfterMaxRow,
+}
+
+/// Returns the visual offset from the start of the visual line
+/// that contains anchor.
+pub fn visual_offset_from_anchor(
+    text: RopeSlice,
+    anchor: usize,
+    pos: usize,
+    text_fmt: &TextFormat,
+    annotations: &TextAnnotations,
+    max_rows: usize,
+) -> Result<(Position, usize), VisualOffsetError> {
+    let (formatter, block_start) =
+        DocumentFormatter::new_at_prev_checkpoint(text, text_fmt, annotations, anchor);
+    let mut char_pos = block_start;
+    let mut anchor_line = None;
+    let mut found_pos = None;
+    let mut last_pos = Position::default();
+
+    if pos < block_start {
+        return Err(VisualOffsetError::PosBeforeAnchorRow);
+    }
+
+    for (grapheme, vpos) in formatter {
+        last_pos = vpos;
+        char_pos += grapheme.doc_chars();
+
+        if char_pos > pos {
+            if let Some(anchor_line) = anchor_line {
+                last_pos.row -= anchor_line;
+                return Ok((last_pos, block_start));
+            } else {
+                found_pos = Some(last_pos);
+            }
+        }
+        if char_pos > anchor && anchor_line.is_none() {
+            if let Some(mut found_pos) = found_pos {
+                return if found_pos.row == last_pos.row {
+                    found_pos.row = 0;
+                    Ok((found_pos, block_start))
+                } else {
+                    Err(VisualOffsetError::PosBeforeAnchorRow)
+                };
+            } else {
+                anchor_line = Some(last_pos.row);
+            }
+        }
+
+        if let Some(anchor_line) = anchor_line {
+            if vpos.row >= anchor_line + max_rows {
+                return Err(VisualOffsetError::PosAfterMaxRow);
+            }
+        }
+    }
+
+    let anchor_line = anchor_line.unwrap_or(last_pos.row);
+    last_pos.row -= anchor_line;
+
+    Ok((last_pos, block_start))
+}
+
 /// Convert (line, column) coordinates to a character index.
 ///
 /// If the `line` coordinate is beyond the end of the file, the EOF
@@ -140,6 +249,11 @@ pub fn pos_at_coords(text: RopeSlice, coords: Position, limit_before_line_ending
 /// If the `column` coordinate is past the end of the given line, the
 /// line-end position (in this case, just before the line ending
 /// character) will be returned.
+/// This function should be used very rarely. Usually `char_idx_at_visual_offset` is preferable.
+/// However when you want to compute a char position from the visual row/column in the text
+/// (not what is actually shown on screen) then you should use this function.
+/// For example aligning text should ignore virtual text and softwrap.
+#[deprecated = "Doesn't account for softwrap or decorations, use char_idx_at_visual_offset instead"]
 pub fn pos_at_visual_coords(text: RopeSlice, coords: Position, tab_width: usize) -> usize {
     let Position { mut row, col } = coords;
     row = row.min(text.len_lines() - 1);
@@ -169,6 +283,121 @@ pub fn pos_at_visual_coords(text: RopeSlice, coords: Position, tab_width: usize)
     line_start + col_char_offset
 }
 
+/// Returns the char index on the visual line `row_offset` below the visual line of
+/// the provided char index `anchor` that is closest to the supplied visual `column`.
+///
+/// If the targeted visual line is entirely covered by virtual text the last
+/// char position before the virtual text and a virtual offset is returned instead.
+///
+/// If no (text) grapheme starts at exactly at the specified column the
+/// start of the grapheme to the left is returned. If there is no grapheme
+/// to the left (for example if the line starts with virtual text) then the positioning
+/// of the next grapheme to the right is returned.
+///
+/// If the `line` coordinate is beyond the end of the file, the EOF
+/// position will be returned.
+///
+/// If the `column` coordinate is past the end of the given line, the
+/// line-end position (in this case, just before the line ending
+/// character) will be returned.
+///
+/// # Returns
+///
+/// `(real_char_idx, virtual_lines)`
+///
+/// The nearest character idx "closest" (see above) to the specified visual offset
+/// on the visual line is returned if the visual line contains any text:
+/// If the visual line at the specified offset is a virtual line generated by a `LineAnnotation`
+/// the previous char_index is returned, together with the remaining vertical offset (`virtual_lines`)
+pub fn char_idx_at_visual_offset(
+    text: RopeSlice,
+    mut anchor: usize,
+    mut row_offset: isize,
+    column: usize,
+    text_fmt: &TextFormat,
+    annotations: &TextAnnotations,
+) -> (usize, usize) {
+    let mut pos = anchor;
+    // convert row relative to visual line containing anchor to row relative to a block containing anchor (anchor may change)
+    loop {
+        let (visual_pos_in_block, block_char_offset) =
+            visual_offset_from_block(text, anchor, pos, text_fmt, annotations);
+        row_offset += visual_pos_in_block.row as isize;
+        anchor = block_char_offset;
+        if row_offset >= 0 {
+            break;
+        }
+
+        if block_char_offset == 0 {
+            row_offset = 0;
+            break;
+        }
+        // the row_offset is negative so we need to look at the previous block
+        // set the anchor to the last char before the current block so that we can compute
+        // the distance of this block from the start of the previous block
+        pos = anchor;
+        anchor -= 1;
+    }
+
+    char_idx_at_visual_block_offset(
+        text,
+        anchor,
+        row_offset as usize,
+        column,
+        text_fmt,
+        annotations,
+    )
+}
+
+/// This function behaves the same as `char_idx_at_visual_offset`, except that
+/// the vertical offset `row` is always computed relative to the block that contains `anchor`
+/// instead of the visual line that contains `anchor`.
+/// Usually `char_idx_at_visual_offset` is more useful but this function can be
+/// used in some situations as an optimization when `visual_offset_from_block` was used
+///
+/// # Returns
+///
+/// `(real_char_idx, virtual_lines)`
+///
+/// See `char_idx_at_visual_offset` for details
+pub fn char_idx_at_visual_block_offset(
+    text: RopeSlice,
+    anchor: usize,
+    row: usize,
+    column: usize,
+    text_fmt: &TextFormat,
+    annotations: &TextAnnotations,
+) -> (usize, usize) {
+    let (formatter, mut char_idx) =
+        DocumentFormatter::new_at_prev_checkpoint(text, text_fmt, annotations, anchor);
+    let mut last_char_idx = char_idx;
+    let mut last_char_idx_on_line = None;
+    let mut last_row = 0;
+    for (grapheme, grapheme_pos) in formatter {
+        match grapheme_pos.row.cmp(&row) {
+            Ordering::Equal => {
+                if grapheme_pos.col + grapheme.width() > column {
+                    if !grapheme.is_virtual() {
+                        return (char_idx, 0);
+                    } else if let Some(char_idx) = last_char_idx_on_line {
+                        return (char_idx, 0);
+                    }
+                } else if !grapheme.is_virtual() {
+                    last_char_idx_on_line = Some(char_idx)
+                }
+            }
+            Ordering::Greater => return (last_char_idx, row - last_row),
+            _ => (),
+        }
+
+        last_char_idx = char_idx;
+        last_row = grapheme_pos.row;
+        char_idx += grapheme.doc_chars();
+    }
+
+    (char_idx, 0)
+}
+
 #[cfg(test)]
 mod test {
     use super::*;
@@ -228,6 +457,7 @@ mod test {
     }
 
     #[test]
+    #[allow(deprecated)]
     fn test_visual_coords_at_pos() {
         let text = Rope::from("ḧëḷḷö\nẅöṛḷḋ");
         let slice = text.slice(..);
@@ -275,6 +505,130 @@ mod test {
         assert_eq!(visual_coords_at_pos(slice, 2, 8), (0, 9).into());
     }
 
+    #[test]
+    fn test_visual_off_from_block() {
+        let text = Rope::from("ḧëḷḷö\nẅöṛḷḋ");
+        let slice = text.slice(..);
+        let annot = TextAnnotations::default();
+        let text_fmt = TextFormat::default();
+        assert_eq!(
+            visual_offset_from_block(slice, 0, 0, &text_fmt, &annot).0,
+            (0, 0).into()
+        );
+        assert_eq!(
+            visual_offset_from_block(slice, 0, 5, &text_fmt, &annot).0,
+            (0, 5).into()
+        ); // position on \n
+        assert_eq!(
+            visual_offset_from_block(slice, 0, 6, &text_fmt, &annot).0,
+            (1, 0).into()
+        ); // position on w
+        assert_eq!(
+            visual_offset_from_block(slice, 0, 7, &text_fmt, &annot).0,
+            (1, 1).into()
+        ); // position on o
+        assert_eq!(
+            visual_offset_from_block(slice, 0, 10, &text_fmt, &annot).0,
+            (1, 4).into()
+        ); // position on d
+
+        // Test with wide characters.
+        let text = Rope::from("今日はいい\n");
+        let slice = text.slice(..);
+        assert_eq!(
+            visual_offset_from_block(slice, 0, 0, &text_fmt, &annot).0,
+            (0, 0).into()
+        );
+        assert_eq!(
+            visual_offset_from_block(slice, 0, 1, &text_fmt, &annot).0,
+            (0, 2).into()
+        );
+        assert_eq!(
+            visual_offset_from_block(slice, 0, 2, &text_fmt, &annot).0,
+            (0, 4).into()
+        );
+        assert_eq!(
+            visual_offset_from_block(slice, 0, 3, &text_fmt, &annot).0,
+            (0, 6).into()
+        );
+        assert_eq!(
+            visual_offset_from_block(slice, 0, 4, &text_fmt, &annot).0,
+            (0, 8).into()
+        );
+        assert_eq!(
+            visual_offset_from_block(slice, 0, 5, &text_fmt, &annot).0,
+            (0, 10).into()
+        );
+        assert_eq!(
+            visual_offset_from_block(slice, 0, 6, &text_fmt, &annot).0,
+            (1, 0).into()
+        );
+
+        // Test with grapheme clusters.
+        let text = Rope::from("a̐éö̲\r\n");
+        let slice = text.slice(..);
+        assert_eq!(
+            visual_offset_from_block(slice, 0, 0, &text_fmt, &annot).0,
+            (0, 0).into()
+        );
+        assert_eq!(
+            visual_offset_from_block(slice, 0, 2, &text_fmt, &annot).0,
+            (0, 1).into()
+        );
+        assert_eq!(
+            visual_offset_from_block(slice, 0, 4, &text_fmt, &annot).0,
+            (0, 2).into()
+        );
+        assert_eq!(
+            visual_offset_from_block(slice, 0, 7, &text_fmt, &annot).0,
+            (0, 3).into()
+        );
+        assert_eq!(
+            visual_offset_from_block(slice, 0, 9, &text_fmt, &annot).0,
+            (1, 0).into()
+        );
+
+        // Test with wide-character grapheme clusters.
+        // TODO: account for cluster.
+        let text = Rope::from("किमपि\n");
+        let slice = text.slice(..);
+        assert_eq!(
+            visual_offset_from_block(slice, 0, 0, &text_fmt, &annot).0,
+            (0, 0).into()
+        );
+        assert_eq!(
+            visual_offset_from_block(slice, 0, 2, &text_fmt, &annot).0,
+            (0, 2).into()
+        );
+        assert_eq!(
+            visual_offset_from_block(slice, 0, 3, &text_fmt, &annot).0,
+            (0, 3).into()
+        );
+        assert_eq!(
+            visual_offset_from_block(slice, 0, 5, &text_fmt, &annot).0,
+            (0, 5).into()
+        );
+        assert_eq!(
+            visual_offset_from_block(slice, 0, 6, &text_fmt, &annot).0,
+            (1, 0).into()
+        );
+
+        // Test with tabs.
+        let text = Rope::from("\tHello\n");
+        let slice = text.slice(..);
+        assert_eq!(
+            visual_offset_from_block(slice, 0, 0, &text_fmt, &annot).0,
+            (0, 0).into()
+        );
+        assert_eq!(
+            visual_offset_from_block(slice, 0, 1, &text_fmt, &annot).0,
+            (0, 4).into()
+        );
+        assert_eq!(
+            visual_offset_from_block(slice, 0, 2, &text_fmt, &annot).0,
+            (0, 5).into()
+        );
+    }
     #[test]
     fn test_pos_at_coords() {
         let text = Rope::from("ḧëḷḷö\nẅöṛḷḋ");
@@ -341,6 +695,7 @@ mod test {
     }
 
     #[test]
+    #[allow(deprecated)]
     fn test_pos_at_visual_coords() {
         let text = Rope::from("ḧëḷḷö\nẅöṛḷḋ");
         let slice = text.slice(..);
@@ -405,4 +760,100 @@ mod test {
         assert_eq!(pos_at_visual_coords(slice, (0, 10).into(), 4), 0);
         assert_eq!(pos_at_visual_coords(slice, (10, 10).into(), 4), 0);
     }
+
+    #[test]
+    fn test_char_idx_at_visual_row_offset() {
+        let text = Rope::from("ḧëḷḷö\nẅöṛḷḋ\nfoo");
+        let slice = text.slice(..);
+        let mut text_fmt = TextFormat::default();
+        for i in 0isize..3isize {
+            for j in -2isize..=2isize {
+                if !(0..3).contains(&(i + j)) {
+                    continue;
+                }
+                println!("{i} {j}");
+                assert_eq!(
+                    char_idx_at_visual_offset(
+                        slice,
+                        slice.line_to_char(i as usize),
+                        j,
+                        3,
+                        &text_fmt,
+                        &TextAnnotations::default(),
+                    )
+                    .0,
+                    slice.line_to_char((i + j) as usize) + 3
+                );
+            }
+        }
+
+        text_fmt.soft_wrap = true;
+        let mut softwrapped_text = "foo ".repeat(10);
+        softwrapped_text.push('\n');
+        let last_char = softwrapped_text.len() - 1;
+
+        let text = Rope::from(softwrapped_text.repeat(3));
+        let slice = text.slice(..);
+        assert_eq!(
+            char_idx_at_visual_offset(
+                slice,
+                last_char,
+                0,
+                0,
+                &text_fmt,
+                &TextAnnotations::default(),
+            )
+            .0,
+            32
+        );
+        assert_eq!(
+            char_idx_at_visual_offset(
+                slice,
+                last_char,
+                -1,
+                0,
+                &text_fmt,
+                &TextAnnotations::default(),
+            )
+            .0,
+            16
+        );
+        assert_eq!(
+            char_idx_at_visual_offset(
+                slice,
+                last_char,
+                -2,
+                0,
+                &text_fmt,
+                &TextAnnotations::default(),
+            )
+            .0,
+            0
+        );
+        assert_eq!(
+            char_idx_at_visual_offset(
+                slice,
+                softwrapped_text.len() + last_char,
+                -2,
+                0,
+                &text_fmt,
+                &TextAnnotations::default(),
+            )
+            .0,
+            softwrapped_text.len()
+        );
+
+        assert_eq!(
+            char_idx_at_visual_offset(
+                slice,
+                softwrapped_text.len() + last_char,
+                -5,
+                0,
+                &text_fmt,
+                &TextAnnotations::default(),
+            )
+            .0,
+            0
+        );
+    }
 }

helix-core/src/register.rs

@@ -78,4 +78,12 @@ impl Registers {
     pub fn inner(&self) -> &HashMap<char, Register> {
         &self.inner
     }
+
+    pub fn clear(&mut self) {
+        self.inner.clear();
+    }
+
+    pub fn remove(&mut self, name: char) -> Option<Register> {
+        self.inner.remove(&name)
+    }
 }

helix-core/src/selection.rs

@@ -21,14 +21,14 @@ use std::borrow::Cow;
 /// can be in any order, or even share the same position.
 ///
 /// The anchor and head positions use gap indexing, meaning
-/// that their indices represent the the gaps *between* `char`s
+/// that their indices represent the gaps *between* `char`s
 /// rather than the `char`s themselves. For example, 1
 /// represents the position between the first and second `char`.
 ///
-/// Below are some example `Range` configurations to better
-/// illustrate.  The anchor and head indices are show as
-/// "(anchor, head)", followed by example text with "[" and "]"
-/// inserted to represent the anchor and head positions:
+/// Below are some examples of `Range` configurations.
+/// The anchor and head indices are shown as "(anchor, head)"
+/// tuples, followed by example text with "[" and "]" symbols
+/// representing the anchor and head positions:
 ///
 /// - (0, 3): `[Som]e text`.
 /// - (3, 0): `]Som[e text`.
@@ -38,7 +38,7 @@ use std::borrow::Cow;
 /// Ranges are considered to be inclusive on the left and
 /// exclusive on the right, regardless of anchor-head ordering.
 /// This means, for example, that non-zero-width ranges that
-/// are directly adjecent, sharing an edge, do not overlap.
+/// are directly adjacent, sharing an edge, do not overlap.
 /// However, a zero-width range will overlap with the shared
 /// left-edge of another range.
 ///
@@ -53,7 +53,9 @@ pub struct Range {
     pub anchor: usize,
     /// The head of the range, moved when extending.
     pub head: usize,
-    pub horiz: Option<u32>,
+    /// The previous visual offset (softwrapped lines and columns) from
+    /// the start of the line
+    pub old_visual_position: Option<(u32, u32)>,
 }
 
 impl Range {
@@ -61,7 +63,7 @@ impl Range {
         Self {
             anchor,
             head,
-            horiz: None,
+            old_visual_position: None,
         }
     }
 
@@ -127,7 +129,7 @@ impl Range {
         Self {
             anchor: self.head,
             head: self.anchor,
-            horiz: self.horiz,
+            old_visual_position: self.old_visual_position,
         }
     }
 
@@ -185,7 +187,7 @@ impl Range {
         Self {
             anchor,
             head,
-            horiz: None,
+            old_visual_position: None,
         }
     }
 
@@ -198,13 +200,13 @@ impl Range {
             Self {
                 anchor: self.anchor.min(from),
                 head: self.head.max(to),
-                horiz: None,
+                old_visual_position: None,
             }
         } else {
             Self {
                 anchor: self.anchor.max(to),
                 head: self.head.min(from),
-                horiz: None,
+                old_visual_position: None,
             }
         }
     }
@@ -219,13 +221,13 @@ impl Range {
             Range {
                 anchor: self.anchor.max(other.anchor),
                 head: self.head.min(other.head),
-                horiz: None,
+                old_visual_position: None,
             }
         } else {
             Range {
                 anchor: self.from().min(other.from()),
                 head: self.to().max(other.to()),
-                horiz: None,
+                old_visual_position: None,
             }
         }
     }
@@ -279,8 +281,8 @@ impl Range {
         Range {
             anchor: new_anchor,
             head: new_head,
-            horiz: if new_anchor == self.anchor {
-                self.horiz
+            old_visual_position: if new_anchor == self.anchor {
+                self.old_visual_position
             } else {
                 None
             },
@@ -306,7 +308,7 @@ impl Range {
             Range {
                 anchor: self.anchor,
                 head: next_grapheme_boundary(slice, self.head),
-                horiz: self.horiz,
+                old_visual_position: self.old_visual_position,
             }
         } else {
             *self
@@ -378,7 +380,7 @@ impl From<(usize, usize)> for Range {
         Self {
             anchor,
             head,
-            horiz: None,
+            old_visual_position: None,
         }
     }
 }
@@ -482,7 +484,7 @@ impl Selection {
             ranges: smallvec![Range {
                 anchor,
                 head,
-                horiz: None
+                old_visual_position: None
             }],
             primary_index: 0,
         }
@@ -495,28 +497,53 @@ impl Selection {
 
     /// Normalizes a `Selection`.
     fn normalize(mut self) -> Self {
-        let primary = self.ranges[self.primary_index];
+        let mut primary = self.ranges[self.primary_index];
         self.ranges.sort_unstable_by_key(Range::from);
+
+        self.ranges.dedup_by(|curr_range, prev_range| {
+            if prev_range.overlaps(curr_range) {
+                let new_range = curr_range.merge(*prev_range);
+                if prev_range == &primary || curr_range == &primary {
+                    primary = new_range;
+                }
+                *prev_range = new_range;
+                true
+            } else {
+                false
+            }
+        });
+
         self.primary_index = self
             .ranges
             .iter()
             .position(|&range| range == primary)
             .unwrap();
 
-        let mut prev_i = 0;
-        for i in 1..self.ranges.len() {
-            if self.ranges[prev_i].overlaps(&self.ranges[i]) {
-                self.ranges[prev_i] = self.ranges[prev_i].merge(self.ranges[i]);
-            } else {
-                prev_i += 1;
-                self.ranges[prev_i] = self.ranges[i];
-            }
-            if i == self.primary_index {
-                self.primary_index = prev_i;
-            }
-        }
+        self
+    }
 
-        self.ranges.truncate(prev_i + 1);
+    // Merges all ranges that are consecutive
+    pub fn merge_consecutive_ranges(mut self) -> Self {
+        let mut primary = self.ranges[self.primary_index];
+
+        self.ranges.dedup_by(|curr_range, prev_range| {
+            if prev_range.to() == curr_range.from() {
+                let new_range = curr_range.merge(*prev_range);
+                if prev_range == &primary || curr_range == &primary {
+                    primary = new_range;
+                }
+                *prev_range = new_range;
+                true
+            } else {
+                false
+            }
+        });
+
+        self.primary_index = self
+            .ranges
+            .iter()
+            .position(|&range| range == primary)
+            .unwrap();
 
         self
     }
@@ -541,9 +568,9 @@ impl Selection {
     }
 
     /// Takes a closure and maps each `Range` over the closure.
-    pub fn transform<F>(mut self, f: F) -> Self
+    pub fn transform<F>(mut self, mut f: F) -> Self
     where
-        F: Fn(Range) -> Range,
+        F: FnMut(Range) -> Range,
     {
         for range in self.ranges.iter_mut() {
             *range = f(*range)
@@ -551,6 +578,16 @@ impl Selection {
         self.normalize()
     }
 
+    /// Takes a closure and maps each `Range` over the closure to multiple `Range`s.
+    pub fn transform_iter<F, I>(mut self, f: F) -> Self
+    where
+        F: FnMut(Range) -> I,
+        I: Iterator<Item = Range>,
+    {
+        self.ranges = self.ranges.into_iter().flat_map(f).collect();
+        self.normalize()
+    }
+
     // Ensures the selection adheres to the following invariants:
     // 1. All ranges are grapheme aligned.
     // 2. All ranges are at least 1 character wide, unless at the
@@ -588,11 +625,6 @@ impl Selection {
 
     // returns true if self ⊇ other
     pub fn contains(&self, other: &Selection) -> bool {
-        // can't contain other if it is larger
-        if other.len() > self.len() {
-            return false;
-        }
-
         let (mut iter_self, mut iter_other) = (self.iter(), other.iter());
         let (mut ele_self, mut ele_other) = (iter_self.next(), iter_other.next());
 
@@ -629,6 +661,15 @@ impl<'a> IntoIterator for &'a Selection {
     }
 }
 
+impl IntoIterator for Selection {
+    type Item = Range;
+    type IntoIter = smallvec::IntoIter<[Range; 1]>;
+
+    fn into_iter(self) -> smallvec::IntoIter<[Range; 1]> {
+        self.ranges.into_iter()
+    }
+}
+
 // TODO: checkSelection -> check if valid for doc length && sorted
 
 pub fn keep_or_remove_matches(
@@ -1132,6 +1173,52 @@ mod test {
             &["", "abcd", "efg", "rs", "xyz"]
         );
     }
+
+    #[test]
+    fn test_merge_consecutive_ranges() {
+        let selection = Selection::new(
+            smallvec![
+                Range::new(0, 1),
+                Range::new(1, 10),
+                Range::new(15, 20),
+                Range::new(25, 26),
+                Range::new(26, 30)
+            ],
+            4,
+        );
+
+        let result = selection.merge_consecutive_ranges();
+
+        assert_eq!(
+            result.ranges(),
+            &[Range::new(0, 10), Range::new(15, 20), Range::new(25, 30)]
+        );
+        assert_eq!(result.primary_index, 2);
+
+        let selection = Selection::new(smallvec![Range::new(0, 1)], 0);
+        let result = selection.merge_consecutive_ranges();
+
+        assert_eq!(result.ranges(), &[Range::new(0, 1)]);
+        assert_eq!(result.primary_index, 0);
+
+        let selection = Selection::new(
+            smallvec![
+                Range::new(0, 1),
+                Range::new(1, 5),
+                Range::new(5, 8),
+                Range::new(8, 10),
+                Range::new(10, 15),
+                Range::new(18, 25)
+            ],
+            3,
+        );
+
+        let result = selection.merge_consecutive_ranges();
+
+        assert_eq!(result.ranges(), &[Range::new(0, 15), Range::new(18, 25)]);
+        assert_eq!(result.primary_index, 0);
+    }
+
     #[test]
     fn test_selection_contains() {
         fn contains(a: Vec<(usize, usize)>, b: Vec<(usize, usize)>) -> bool {
@@ -1157,5 +1244,11 @@ mod test {
             vec!((3, 4), (7, 9))
         ));
         assert!(!contains(vec!((1, 1), (5, 6)), vec!((1, 6))));
+
+        // multiple ranges of other are all contained in some ranges of self,
+        assert!(contains(
+            vec!((1, 4), (7, 10)),
+            vec!((1, 2), (3, 4), (7, 9))
+        ));
     }
 }

helix-core/src/shellwords.rs

@@ -129,8 +129,9 @@ impl<'a> From<&'a str> for Shellwords<'a> {
                 DquoteEscaped => Dquoted,
             };
 
-            if i >= input.len() - 1 && end == 0 {
-                end = i + 1;
+            let c_len = c.len_utf8();
+            if i == input.len() - c_len && end == 0 {
+                end = i + c_len;
             }
 
             if end > 0 {
@@ -293,14 +294,14 @@ mod test {
     #[test]
     fn test_lists() {
         let input =
-            r#":set statusline.center ["file-type","file-encoding"] '["list", "in", "qoutes"]'"#;
+            r#":set statusline.center ["file-type","file-encoding"] '["list", "in", "quotes"]'"#;
         let shellwords = Shellwords::from(input);
         let result = shellwords.words().to_vec();
         let expected = vec![
             Cow::from(":set"),
             Cow::from("statusline.center"),
             Cow::from(r#"["file-type","file-encoding"]"#),
-            Cow::from(r#"["list", "in", "qoutes"]"#),
+            Cow::from(r#"["list", "in", "quotes"]"#),
         ];
         assert_eq!(expected, result);
     }
@@ -333,4 +334,17 @@ mod test {
         assert_eq!(Shellwords::from(":o a").parts(), &[":o", "a"]);
         assert_eq!(Shellwords::from(":o a\\ ").parts(), &[":o", "a\\"]);
     }
+
+    #[test]
+    fn test_multibyte_at_end() {
+        assert_eq!(Shellwords::from("𒀀").parts(), &["𒀀"]);
+        assert_eq!(
+            Shellwords::from(":sh echo 𒀀").parts(),
+            &[":sh", "echo", "𒀀"]
+        );
+        assert_eq!(
+            Shellwords::from(":sh echo 𒀀 hello world𒀀").parts(),
+            &[":sh", "echo", "𒀀", "hello", "world𒀀"]
+        );
+    }
 }

helix-core/src/surround.rs

@@ -1,6 +1,6 @@
 use std::fmt::Display;
 
-use crate::{search, Range, Selection};
+use crate::{movement::Direction, search, Range, Selection};
 use ropey::RopeSlice;
 
 pub const PAIRS: &[(char, char)] = &[
@@ -55,15 +55,18 @@ pub fn get_pair(ch: char) -> (char, char) {
 pub fn find_nth_closest_pairs_pos(
     text: RopeSlice,
     range: Range,
-    n: usize,
+    mut skip: usize,
 ) -> Result<(usize, usize)> {
     let is_open_pair = |ch| PAIRS.iter().any(|(open, _)| *open == ch);
     let is_close_pair = |ch| PAIRS.iter().any(|(_, close)| *close == ch);
 
     let mut stack = Vec::with_capacity(2);
-    let pos = range.cursor(text);
+    let pos = range.from();
+    let mut close_pos = pos.saturating_sub(1);
 
     for ch in text.chars_at(pos) {
+        close_pos += 1;
+
         if is_open_pair(ch) {
             // Track open pairs encountered so that we can step over
             // the corresponding close pairs that will come up further
@@ -71,20 +74,46 @@ pub fn find_nth_closest_pairs_pos(
             // open pair is before the cursor position.
             stack.push(ch);
             continue;
-        } else if is_close_pair(ch) {
-            let (open, _) = get_pair(ch);
-            if stack.last() == Some(&open) {
-                stack.pop();
-                continue;
-            } else {
-                // In the ideal case the stack would be empty here and the
-                // current character would be the close pair that we are
-                // looking for. It could also be the case that the pairs
-                // are unbalanced and we encounter a close pair that doesn't
-                // close the last seen open pair. In either case use this
-                // char as the auto-detected closest pair.
-                return find_nth_pairs_pos(text, ch, range, n);
+        }
+
+        if !is_close_pair(ch) {
+            // We don't care if this character isn't a brace pair item,
+            // so short circuit here.
+            continue;
+        }
+
+        let (open, close) = get_pair(ch);
+
+        if stack.last() == Some(&open) {
+            // If we are encountering the closing pair for an opener
+            // we just found while traversing, then its inside the
+            // selection and should be skipped over.
+            stack.pop();
+            continue;
+        }
+
+        match find_nth_open_pair(text, open, close, close_pos, 1) {
+            // Before we accept this pair, we want to ensure that the
+            // pair encloses the range rather than just the cursor.
+            Some(open_pos)
+                if open_pos <= pos.saturating_add(1)
+                    && close_pos >= range.to().saturating_sub(1) =>
+            {
+                // Since we have special conditions for when to
+                // accept, we can't just pass the skip parameter on
+                // through to the find_nth_*_pair methods, so we
+                // track skips manually here.
+                if skip > 1 {
+                    skip -= 1;
+                    continue;
+                }
+
+                return match range.direction() {
+                    Direction::Forward => Ok((open_pos, close_pos)),
+                    Direction::Backward => Ok((close_pos, open_pos)),
+                };
             }
+            _ => continue,
         }
     }
 
@@ -244,141 +273,135 @@ mod test {
     use ropey::Rope;
     use smallvec::SmallVec;
 
-    #[allow(clippy::type_complexity)]
-    fn check_find_nth_pair_pos(
-        text: &str,
-        cases: Vec<(usize, char, usize, Result<(usize, usize)>)>,
-    ) {
-        let doc = Rope::from(text);
-        let slice = doc.slice(..);
-
-        for (cursor_pos, ch, n, expected_range) in cases {
-            let range = find_nth_pairs_pos(slice, ch, (cursor_pos, cursor_pos + 1).into(), n);
-            assert_eq!(
-                range, expected_range,
-                "Expected {:?}, got {:?}",
-                expected_range, range
-            );
-        }
-    }
-
-    #[test]
-    fn test_find_nth_pairs_pos() {
-        check_find_nth_pair_pos(
-            "some (text) here",
-            vec![
-                // cursor on [t]ext
-                (6, '(', 1, Ok((5, 10))),
-                (6, ')', 1, Ok((5, 10))),
-                // cursor on so[m]e
-                (2, '(', 1, Err(Error::PairNotFound)),
-                // cursor on bracket itself
-                (5, '(', 1, Ok((5, 10))),
-                (10, '(', 1, Ok((5, 10))),
-            ],
-        );
-    }
-
-    #[test]
-    fn test_find_nth_pairs_pos_skip() {
-        check_find_nth_pair_pos(
-            "(so (many (good) text) here)",
-            vec![
-                // cursor on go[o]d
-                (13, '(', 1, Ok((10, 15))),
-                (13, '(', 2, Ok((4, 21))),
-                (13, '(', 3, Ok((0, 27))),
-            ],
-        );
-    }
-
-    #[test]
-    fn test_find_nth_pairs_pos_same() {
-        check_find_nth_pair_pos(
-            "'so 'many 'good' text' here'",
-            vec![
-                // cursor on go[o]d
-                (13, '\'', 1, Ok((10, 15))),
-                (13, '\'', 2, Ok((4, 21))),
-                (13, '\'', 3, Ok((0, 27))),
-                // cursor on the quotes
-                (10, '\'', 1, Err(Error::CursorOnAmbiguousPair)),
-            ],
-        )
-    }
-
-    #[test]
-    fn test_find_nth_pairs_pos_step() {
-        check_find_nth_pair_pos(
-            "((so)((many) good (text))(here))",
-            vec![
-                // cursor on go[o]d
-                (15, '(', 1, Ok((5, 24))),
-                (15, '(', 2, Ok((0, 31))),
-            ],
-        )
-    }
-
-    #[test]
-    fn test_find_nth_pairs_pos_mixed() {
-        check_find_nth_pair_pos(
-            "(so [many {good} text] here)",
-            vec![
-                // cursor on go[o]d
-                (13, '{', 1, Ok((10, 15))),
-                (13, '[', 1, Ok((4, 21))),
-                (13, '(', 1, Ok((0, 27))),
-            ],
-        )
-    }
-
     #[test]
     fn test_get_surround_pos() {
-        let doc = Rope::from("(some) (chars)\n(newline)");
-        let slice = doc.slice(..);
-        let selection = Selection::new(
-            SmallVec::from_slice(&[Range::point(2), Range::point(9), Range::point(20)]),
-            0,
-        );
+        #[rustfmt::skip]
+        let (doc, selection, expectations) =
+            rope_with_selections_and_expectations(
+                "(some) (chars)\n(newline)",
+                "_ ^  _ _ ^   _\n_    ^  _"
+            );
 
-        // cursor on s[o]me, c[h]ars, newl[i]ne
         assert_eq!(
-            get_surround_pos(slice, &selection, Some('('), 1)
-                .unwrap()
-                .as_slice(),
-            &[0, 5, 7, 13, 15, 23]
+            get_surround_pos(doc.slice(..), &selection, Some('('), 1).unwrap(),
+            expectations
         );
     }
 
     #[test]
-    fn test_get_surround_pos_bail() {
-        let doc = Rope::from("[some]\n(chars)xx\n(newline)");
-        let slice = doc.slice(..);
+    fn test_get_surround_pos_bail_different_surround_chars() {
+        #[rustfmt::skip]
+        let (doc, selection, _) =
+            rope_with_selections_and_expectations(
+                "[some]\n(chars)xx\n(newline)",
+                "  ^   \n  ^      \n         "
+            );
 
-        let selection =
-            Selection::new(SmallVec::from_slice(&[Range::point(2), Range::point(9)]), 0);
-        // cursor on s[o]me, c[h]ars
         assert_eq!(
-            get_surround_pos(slice, &selection, Some('('), 1),
-            Err(Error::PairNotFound) // different surround chars
+            get_surround_pos(doc.slice(..), &selection, Some('('), 1),
+            Err(Error::PairNotFound)
         );
+    }
+
+    #[test]
+    fn test_get_surround_pos_bail_overlapping_surround_chars() {
+        #[rustfmt::skip]
+        let (doc, selection, _) =
+            rope_with_selections_and_expectations(
+                "[some]\n(chars)xx\n(newline)",
+                "      \n       ^ \n      ^  "
+            );
 
-        let selection = Selection::new(
-            SmallVec::from_slice(&[Range::point(14), Range::point(24)]),
-            0,
-        );
-        // cursor on [x]x, newli[n]e
         assert_eq!(
-            get_surround_pos(slice, &selection, Some('('), 1),
+            get_surround_pos(doc.slice(..), &selection, Some('('), 1),
             Err(Error::PairNotFound) // overlapping surround chars
         );
+    }
+
+    #[test]
+    fn test_get_surround_pos_bail_cursor_overlap() {
+        #[rustfmt::skip]
+        let (doc, selection, _) =
+            rope_with_selections_and_expectations(
+                "[some]\n(chars)xx\n(newline)",
+                "  ^^  \n         \n         "
+            );
 
-        let selection =
-            Selection::new(SmallVec::from_slice(&[Range::point(2), Range::point(3)]), 0);
-        // cursor on s[o][m]e
         assert_eq!(
-            get_surround_pos(slice, &selection, Some('['), 1),
+            get_surround_pos(doc.slice(..), &selection, Some('['), 1),
             Err(Error::CursorOverlap)
         );
     }
+
+    #[test]
+    fn test_find_nth_pairs_pos_quote_success() {
+        #[rustfmt::skip]
+        let (doc, selection, expectations) =
+            rope_with_selections_and_expectations(
+                "some 'quoted text' on this 'line'\n'and this one'",
+                "     _        ^  _               \n              "
+            );
+
+        assert_eq!(2, expectations.len());
+        assert_eq!(
+            find_nth_pairs_pos(doc.slice(..), '\'', selection.primary(), 1)
+                .expect("find should succeed"),
+            (expectations[0], expectations[1])
+        )
+    }
+
+    #[test]
+    fn test_find_nth_pairs_pos_nested_quote_success() {
+        #[rustfmt::skip]
+        let (doc, selection, expectations) =
+            rope_with_selections_and_expectations(
+                "some 'nested 'quoted' text' on this 'line'\n'and this one'",
+                "     _           ^        _               \n              "
+            );
+
+        assert_eq!(2, expectations.len());
+        assert_eq!(
+            find_nth_pairs_pos(doc.slice(..), '\'', selection.primary(), 2)
+                .expect("find should succeed"),
+            (expectations[0], expectations[1])
+        )
+    }
+
+    #[test]
+    fn test_find_nth_pairs_pos_inside_quote_ambiguous() {
+        #[rustfmt::skip]
+        let (doc, selection, _) =
+            rope_with_selections_and_expectations(
+                "some 'nested 'quoted' text' on this 'line'\n'and this one'",
+                "                    ^                     \n              "
+            );
+
+        assert_eq!(
+            find_nth_pairs_pos(doc.slice(..), '\'', selection.primary(), 1),
+            Err(Error::CursorOnAmbiguousPair)
+        )
+    }
+
+    // Create a Rope and a matching Selection using a specification language.
+    // ^ is a single-point selection.
+    // _ is an expected index. These are returned as a Vec<usize> for use in assertions.
+    fn rope_with_selections_and_expectations(
+        text: &str,
+        spec: &str,
+    ) -> (Rope, Selection, Vec<usize>) {
+        if text.len() != spec.len() {
+            panic!("specification must match text length -- are newlines aligned?");
+        }
+
+        let rope = Rope::from(text);
+
+        let selections: SmallVec<[Range; 1]> = spec
+            .match_indices('^')
+            .map(|(i, _)| Range::point(i))
+            .collect();
+
+        let expectations: Vec<usize> = spec.match_indices('_').map(|(i, _)| i).collect();
+
+        (rope, Selection::new(selections, 0), expectations)
+    }
 }

helix-core/src/syntax.rs

@@ -20,7 +20,7 @@ use std::{
     fmt,
     hash::{Hash, Hasher},
     mem::{replace, transmute},
-    path::Path,
+    path::{Path, PathBuf},
     str::FromStr,
     sync::Arc,
 };
@@ -82,7 +82,8 @@ pub struct LanguageConfiguration {
     pub shebangs: Vec<String>, // interpreter(s) associated with language
     pub roots: Vec<String>,        // these indicate project roots <.git, Cargo.toml>
     pub comment_token: Option<String>,
-    pub max_line_length: Option<usize>,
+    pub text_width: Option<usize>,
+    pub soft_wrap: Option<SoftWrap>,
 
     #[serde(default, skip_serializing, deserialize_with = "deserialize_lsp_config")]
     pub config: Option<serde_json::Value>,
@@ -126,6 +127,10 @@ pub struct LanguageConfiguration {
     pub auto_pairs: Option<AutoPairs>,
 
     pub rulers: Option<Vec<u16>>, // if set, override editor's rulers
+
+    /// Hardcoded LSP root directories relative to the workspace root, like `examples` or `tools/fuzz`.
+    /// Falling back to the current working directory if none are configured.
+    pub workspace_lsp_roots: Option<Vec<PathBuf>>,
 }
 
 #[derive(Debug, PartialEq, Eq, Hash)]
@@ -182,9 +187,12 @@ impl<'de> Deserialize<'de> for FileType {
                 M: serde::de::MapAccess<'de>,
             {
                 match map.next_entry::<String, String>()? {
-                    Some((key, suffix)) if key == "suffix" => Ok(FileType::Suffix(
-                        suffix.replace('/', &std::path::MAIN_SEPARATOR.to_string()),
-                    )),
+                    Some((key, suffix)) if key == "suffix" => Ok(FileType::Suffix({
+                        // FIXME: use `suffix.replace('/', std::path::MAIN_SEPARATOR_STR)`
+                        //        if MSRV is updated to 1.68
+                        let mut seperator = [0; 1];
+                        suffix.replace('/', std::path::MAIN_SEPARATOR.encode_utf8(&mut seperator))
+                    })),
                     Some((key, _value)) => Err(serde::de::Error::custom(format!(
                         "unknown key in `file-types` list: {}",
                         key
@@ -427,7 +435,7 @@ impl TextObjectQuery {
                 let nodes: Vec<_> = mat
                     .captures
                     .iter()
-                    .filter_map(|cap| (cap.index == capture_idx).then(|| cap.node))
+                    .filter_map(|cap| (cap.index == capture_idx).then_some(cap.node))
                     .collect();
 
                 if nodes.len() > 1 {
@@ -546,6 +554,35 @@ impl LanguageConfiguration {
             .ok()
     }
 }
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(default, rename_all = "kebab-case", deny_unknown_fields)]
+pub struct SoftWrap {
+    /// Soft wrap lines that exceed viewport width. Default to off
+    // NOTE: Option on purpose because the struct is shared between language config and global config.
+    // By default the option is None so that the language config falls back to the global config unless explicitly set.
+    pub enable: Option<bool>,
+    /// Maximum space left free at the end of the line.
+    /// This space is used to wrap text at word boundaries. If that is not possible within this limit
+    /// the word is simply split at the end of the line.
+    ///
+    /// This is automatically hard-limited to a quarter of the viewport to ensure correct display on small views.
+    ///
+    /// Default to 20
+    pub max_wrap: Option<u16>,
+    /// Maximum number of indentation that can be carried over from the previous line when softwrapping.
+    /// If a line is indented further then this limit it is rendered at the start of the viewport instead.
+    ///
+    /// This is automatically hard-limited to a quarter of the viewport to ensure correct display on small views.
+    ///
+    /// Default to 40
+    pub max_indent_retain: Option<u16>,
+    /// Indicator placed at the beginning of softwrapped lines
+    ///
+    /// Defaults to ↪
+    pub wrap_indicator: Option<String>,
+    /// Softwrap at `text_width` instead of viewport width if it is shorter
+    pub wrap_at_text_width: Option<bool>,
+}
 
 // Expose loader as Lazy<> global since it's always static?
 
@@ -628,9 +665,8 @@ impl Loader {
 
     pub fn language_config_for_shebang(&self, source: &Rope) -> Option<Arc<LanguageConfiguration>> {
         let line = Cow::from(source.line(0));
-        static SHEBANG_REGEX: Lazy<Regex> = Lazy::new(|| {
-            Regex::new(r"^#!\s*(?:\S*[/\\](?:env\s+(?:\-\S+\s+)*)?)?([^\s\.\d]+)").unwrap()
-        });
+        static SHEBANG_REGEX: Lazy<Regex> =
+            Lazy::new(|| Regex::new(&["^", SHEBANG].concat()).unwrap());
         let configuration_id = SHEBANG_REGEX
             .captures(&line)
             .and_then(|cap| self.language_config_ids_by_shebang.get(&cap[1]));
@@ -652,15 +688,14 @@ impl Loader {
             .cloned()
     }
 
-    pub fn language_configuration_for_injection_string(
-        &self,
-        string: &str,
-    ) -> Option<Arc<LanguageConfiguration>> {
+    /// Unlike language_config_for_language_id, which only returns Some for an exact id, this
+    /// function will perform a regex match on the given string to find the closest language match.
+    pub fn language_config_for_name(&self, name: &str) -> Option<Arc<LanguageConfiguration>> {
         let mut best_match_length = 0;
         let mut best_match_position = None;
         for (i, configuration) in self.language_configs.iter().enumerate() {
             if let Some(injection_regex) = &configuration.injection_regex {
-                if let Some(mat) = injection_regex.find(string) {
+                if let Some(mat) = injection_regex.find(name) {
                     let length = mat.end() - mat.start();
                     if length > best_match_length {
                         best_match_position = Some(i);
@@ -670,11 +705,20 @@ impl Loader {
             }
         }
 
-        if let Some(i) = best_match_position {
-            let configuration = &self.language_configs[i];
-            return Some(configuration.clone());
+        best_match_position.map(|i| self.language_configs[i].clone())
+    }
+
+    pub fn language_configuration_for_injection_string(
+        &self,
+        capture: &InjectionLanguageMarker,
+    ) -> Option<Arc<LanguageConfiguration>> {
+        match capture {
+            InjectionLanguageMarker::Name(string) => self.language_config_for_name(string),
+            InjectionLanguageMarker::Filename(file) => self.language_config_for_file_name(file),
+            InjectionLanguageMarker::Shebang(shebang) => {
+                self.language_config_for_language_id(shebang)
+            }
         }
-        None
     }
 
     pub fn language_configs(&self) -> impl Iterator<Item = &Arc<LanguageConfiguration>> {
@@ -724,7 +768,11 @@ fn byte_range_to_str(range: std::ops::Range<usize>, source: RopeSlice) -> Cow<st
 }
 
 impl Syntax {
-    pub fn new(source: &Rope, config: Arc<HighlightConfiguration>, loader: Arc<Loader>) -> Self {
+    pub fn new(
+        source: &Rope,
+        config: Arc<HighlightConfiguration>,
+        loader: Arc<Loader>,
+    ) -> Option<Self> {
         let root_layer = LanguageLayer {
             tree: None,
             config,
@@ -749,11 +797,13 @@ impl Syntax {
             loader,
         };
 
-        syntax
-            .update(source, source, &ChangeSet::new(source))
-            .unwrap();
+        let res = syntax.update(source, source, &ChangeSet::new(source));
 
-        syntax
+        if res.is_err() {
+            log::error!("TS parser failed, disabeling TS for the current buffer: {res:?}");
+            return None;
+        }
+        Some(syntax)
     }
 
     pub fn update(
@@ -766,7 +816,7 @@ impl Syntax {
         queue.push_back(self.root);
 
         let scopes = self.loader.scopes.load();
-        let injection_callback = |language: &str| {
+        let injection_callback = |language: &InjectionLanguageMarker| {
             self.loader
                 .language_configuration_for_injection_string(language)
                 .and_then(|language_config| language_config.highlight_config(&scopes))
@@ -881,6 +931,7 @@ impl Syntax {
 
         PARSER.with(|ts_parser| {
             let ts_parser = &mut ts_parser.borrow_mut();
+            ts_parser.parser.set_timeout_micros(1000 * 500); // half a second is pretty generours
             let mut cursor = ts_parser.cursors.pop().unwrap_or_else(QueryCursor::new);
             // TODO: might need to set cursor range
             cursor.set_byte_range(0..usize::MAX);
@@ -927,12 +978,9 @@ impl Syntax {
                 );
                 let mut injections = Vec::new();
                 for mat in matches {
-                    let (language_name, content_node, included_children) = injection_for_match(
-                        &layer.config,
-                        &layer.config.injections_query,
-                        &mat,
-                        source_slice,
-                    );
+                    let (injection_capture, content_node, included_children) = layer
+                        .config
+                        .injection_for_match(&layer.config.injections_query, &mat, source_slice);
 
                     // Explicitly remove this match so that none of its other captures will remain
                     // in the stream of captures.
@@ -940,9 +988,10 @@ impl Syntax {
 
                     // If a language is found with the given name, then add a new language layer
                     // to the highlighted document.
-                    if let (Some(language_name), Some(content_node)) = (language_name, content_node)
+                    if let (Some(injection_capture), Some(content_node)) =
+                        (injection_capture, content_node)
                     {
-                        if let Some(config) = (injection_callback)(&language_name) {
+                        if let Some(config) = (injection_callback)(&injection_capture) {
                             let ranges =
                                 intersect_ranges(&layer.ranges, &[content_node], included_children);
 
@@ -967,14 +1016,11 @@ impl Syntax {
                     );
                     for mat in matches {
                         let entry = &mut injections_by_pattern_index[mat.pattern_index];
-                        let (language_name, content_node, included_children) = injection_for_match(
-                            &layer.config,
-                            combined_injections_query,
-                            &mat,
-                            source_slice,
-                        );
-                        if language_name.is_some() {
-                            entry.0 = language_name;
+                        let (injection_capture, content_node, included_children) = layer
+                            .config
+                            .injection_for_match(combined_injections_query, &mat, source_slice);
+                        if injection_capture.is_some() {
+                            entry.0 = injection_capture;
                         }
                         if let Some(content_node) = content_node {
                             entry.1.push(content_node);
@@ -1092,21 +1138,14 @@ impl Syntax {
                     }],
                     cursor,
                     _tree: None,
-                    captures,
+                    captures: RefCell::new(captures),
                     config: layer.config.as_ref(), // TODO: just reuse `layer`
                     depth: layer.depth,            // TODO: just reuse `layer`
-                    ranges: &layer.ranges,         // TODO: temp
                 })
             })
             .collect::<Vec<_>>();
 
-        // HAXX: arrange layers by byte range, with deeper layers positioned first
-        layers.sort_by_key(|layer| {
-            (
-                layer.ranges.first().cloned(),
-                std::cmp::Reverse(layer.depth),
-            )
-        });
+        layers.sort_unstable_by_key(|layer| layer.sort_key());
 
         let mut result = HighlightIter {
             source,
@@ -1136,6 +1175,7 @@ impl Syntax {
 bitflags! {
     /// Flags that track the status of a layer
     /// in the `Sytaxn::update` function
+    #[derive(Debug)]
     struct LayerUpdateFlags : u32{
         const MODIFIED = 0b001;
         const MOVED = 0b010;
@@ -1367,6 +1407,8 @@ pub struct HighlightConfiguration {
     non_local_variable_patterns: Vec<bool>,
     injection_content_capture_index: Option<u32>,
     injection_language_capture_index: Option<u32>,
+    injection_filename_capture_index: Option<u32>,
+    injection_shebang_capture_index: Option<u32>,
     local_scope_capture_index: Option<u32>,
     local_def_capture_index: Option<u32>,
     local_def_value_capture_index: Option<u32>,
@@ -1424,12 +1466,11 @@ impl<'a> TextProvider<'a> for RopeProvider<'a> {
 struct HighlightIterLayer<'a> {
     _tree: Option<Tree>,
     cursor: QueryCursor,
-    captures: iter::Peekable<QueryCaptures<'a, 'a, RopeProvider<'a>>>,
+    captures: RefCell<iter::Peekable<QueryCaptures<'a, 'a, RopeProvider<'a>>>>,
     config: &'a HighlightConfiguration,
     highlight_end_stack: Vec<usize>,
     scope_stack: Vec<LocalScope<'a>>,
     depth: u32,
-    ranges: &'a [Range],
 }
 
 impl<'a> fmt::Debug for HighlightIterLayer<'a> {
@@ -1511,6 +1552,8 @@ impl HighlightConfiguration {
         // Store the numeric ids for all of the special captures.
         let mut injection_content_capture_index = None;
         let mut injection_language_capture_index = None;
+        let mut injection_filename_capture_index = None;
+        let mut injection_shebang_capture_index = None;
         let mut local_def_capture_index = None;
         let mut local_def_value_capture_index = None;
         let mut local_ref_capture_index = None;
@@ -1531,6 +1574,8 @@ impl HighlightConfiguration {
             match name.as_str() {
                 "injection.content" => injection_content_capture_index = i,
                 "injection.language" => injection_language_capture_index = i,
+                "injection.filename" => injection_filename_capture_index = i,
+                "injection.shebang" => injection_shebang_capture_index = i,
                 _ => {}
             }
         }
@@ -1546,6 +1591,8 @@ impl HighlightConfiguration {
             non_local_variable_patterns,
             injection_content_capture_index,
             injection_language_capture_index,
+            injection_filename_capture_index,
+            injection_shebang_capture_index,
             local_scope_capture_index,
             local_def_capture_index,
             local_def_value_capture_index,
@@ -1604,16 +1651,101 @@ impl HighlightConfiguration {
 
         self.highlight_indices.store(Arc::new(indices));
     }
+
+    fn injection_pair<'a>(
+        &self,
+        query_match: &QueryMatch<'a, 'a>,
+        source: RopeSlice<'a>,
+    ) -> (Option<InjectionLanguageMarker<'a>>, Option<Node<'a>>) {
+        let mut injection_capture = None;
+        let mut content_node = None;
+
+        for capture in query_match.captures {
+            let index = Some(capture.index);
+            if index == self.injection_language_capture_index {
+                let name = byte_range_to_str(capture.node.byte_range(), source);
+                injection_capture = Some(InjectionLanguageMarker::Name(name));
+            } else if index == self.injection_filename_capture_index {
+                let name = byte_range_to_str(capture.node.byte_range(), source);
+                let path = Path::new(name.as_ref()).to_path_buf();
+                injection_capture = Some(InjectionLanguageMarker::Filename(path.into()));
+            } else if index == self.injection_shebang_capture_index {
+                let node_slice = source.byte_slice(capture.node.byte_range());
+
+                // some languages allow space and newlines before the actual string content
+                // so a shebang could be on either the first or second line
+                let lines = if let Ok(end) = node_slice.try_line_to_byte(2) {
+                    node_slice.byte_slice(..end)
+                } else {
+                    node_slice
+                };
+
+                static SHEBANG_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(SHEBANG).unwrap());
+
+                injection_capture = SHEBANG_REGEX
+                    .captures(&Cow::from(lines))
+                    .map(|cap| InjectionLanguageMarker::Shebang(cap[1].to_owned()))
+            } else if index == self.injection_content_capture_index {
+                content_node = Some(capture.node);
+            }
+        }
+        (injection_capture, content_node)
+    }
+
+    fn injection_for_match<'a>(
+        &self,
+        query: &'a Query,
+        query_match: &QueryMatch<'a, 'a>,
+        source: RopeSlice<'a>,
+    ) -> (
+        Option<InjectionLanguageMarker<'a>>,
+        Option<Node<'a>>,
+        IncludedChildren,
+    ) {
+        let (mut injection_capture, content_node) = self.injection_pair(query_match, source);
+
+        let mut included_children = IncludedChildren::default();
+        for prop in query.property_settings(query_match.pattern_index) {
+            match prop.key.as_ref() {
+                // In addition to specifying the language name via the text of a
+                // captured node, it can also be hard-coded via a `#set!` predicate
+                // that sets the injection.language key.
+                "injection.language" if injection_capture.is_none() => {
+                    injection_capture = prop
+                        .value
+                        .as_ref()
+                        .map(|s| InjectionLanguageMarker::Name(s.as_ref().into()));
+                }
+
+                // By default, injections do not include the *children* of an
+                // `injection.content` node - only the ranges that belong to the
+                // node itself. This can be changed using a `#set!` predicate that
+                // sets the `injection.include-children` key.
+                "injection.include-children" => included_children = IncludedChildren::All,
+
+                // Some queries might only exclude named children but include unnamed
+                // children in their `injection.content` node. This can be enabled using
+                // a `#set!` predicate that sets the `injection.include-unnamed-children` key.
+                "injection.include-unnamed-children" => {
+                    included_children = IncludedChildren::Unnamed
+                }
+                _ => {}
+            }
+        }
+
+        (injection_capture, content_node, included_children)
+    }
 }
 
 impl<'a> HighlightIterLayer<'a> {
     // First, sort scope boundaries by their byte offset in the document. At a
     // given position, emit scope endings before scope beginnings. Finally, emit
     // scope boundaries from deeper layers first.
-    fn sort_key(&mut self) -> Option<(usize, bool, isize)> {
+    fn sort_key(&self) -> Option<(usize, bool, isize)> {
         let depth = -(self.depth as isize);
         let next_start = self
             .captures
+            .borrow_mut()
             .peek()
             .map(|(m, i)| m.captures[*i].node.start_byte());
         let next_end = self.highlight_end_stack.last().cloned();
@@ -1838,7 +1970,8 @@ impl<'a> Iterator for HighlightIter<'a> {
             // Get the next capture from whichever layer has the earliest highlight boundary.
             let range;
             let layer = &mut self.layers[0];
-            if let Some((next_match, capture_index)) = layer.captures.peek() {
+            let captures = layer.captures.get_mut();
+            if let Some((next_match, capture_index)) = captures.peek() {
                 let next_capture = next_match.captures[*capture_index];
                 range = next_capture.node.byte_range();
 
@@ -1861,7 +1994,7 @@ impl<'a> Iterator for HighlightIter<'a> {
                 return self.emit_event(self.source.len_bytes(), None);
             };
 
-            let (mut match_, capture_index) = layer.captures.next().unwrap();
+            let (mut match_, capture_index) = captures.next().unwrap();
             let mut capture = match_.captures[capture_index];
 
             // Remove from the local scope stack any local scopes that have already ended.
@@ -1937,11 +2070,11 @@ impl<'a> Iterator for HighlightIter<'a> {
                 }
 
                 // Continue processing any additional matches for the same node.
-                if let Some((next_match, next_capture_index)) = layer.captures.peek() {
+                if let Some((next_match, next_capture_index)) = captures.peek() {
                     let next_capture = next_match.captures[*next_capture_index];
                     if next_capture.node == capture.node {
                         capture = next_capture;
-                        match_ = layer.captures.next().unwrap().0;
+                        match_ = captures.next().unwrap().0;
                         continue;
                     }
                 }
@@ -1964,11 +2097,11 @@ impl<'a> Iterator for HighlightIter<'a> {
             // highlighting patterns that are disabled for local variables.
             if definition_highlight.is_some() || reference_highlight.is_some() {
                 while layer.config.non_local_variable_patterns[match_.pattern_index] {
-                    if let Some((next_match, next_capture_index)) = layer.captures.peek() {
+                    if let Some((next_match, next_capture_index)) = captures.peek() {
                         let next_capture = next_match.captures[*next_capture_index];
                         if next_capture.node == capture.node {
                             capture = next_capture;
-                            match_ = layer.captures.next().unwrap().0;
+                            match_ = captures.next().unwrap().0;
                             continue;
                         }
                     }
@@ -1983,10 +2116,10 @@ impl<'a> Iterator for HighlightIter<'a> {
             // for a given node are ordered by pattern index, so these subsequent
             // captures are guaranteed to be for highlighting, not injections or
             // local variables.
-            while let Some((next_match, next_capture_index)) = layer.captures.peek() {
+            while let Some((next_match, next_capture_index)) = captures.peek() {
                 let next_capture = next_match.captures[*next_capture_index];
                 if next_capture.node == capture.node {
-                    layer.captures.next();
+                    captures.next();
                 } else {
                     break;
                 }
@@ -2013,56 +2146,15 @@ impl<'a> Iterator for HighlightIter<'a> {
     }
 }
 
-fn injection_for_match<'a>(
-    config: &HighlightConfiguration,
-    query: &'a Query,
-    query_match: &QueryMatch<'a, 'a>,
-    source: RopeSlice<'a>,
-) -> (Option<Cow<'a, str>>, Option<Node<'a>>, IncludedChildren) {
-    let content_capture_index = config.injection_content_capture_index;
-    let language_capture_index = config.injection_language_capture_index;
-
-    let mut language_name = None;
-    let mut content_node = None;
-    for capture in query_match.captures {
-        let index = Some(capture.index);
-        if index == language_capture_index {
-            let name = byte_range_to_str(capture.node.byte_range(), source);
-            language_name = Some(name);
-        } else if index == content_capture_index {
-            content_node = Some(capture.node);
-        }
-    }
-
-    let mut included_children = IncludedChildren::default();
-    for prop in query.property_settings(query_match.pattern_index) {
-        match prop.key.as_ref() {
-            // In addition to specifying the language name via the text of a
-            // captured node, it can also be hard-coded via a `#set!` predicate
-            // that sets the injection.language key.
-            "injection.language" => {
-                if language_name.is_none() {
-                    language_name = prop.value.as_ref().map(|s| s.as_ref().into())
-                }
-            }
-
-            // By default, injections do not include the *children* of an
-            // `injection.content` node - only the ranges that belong to the
-            // node itself. This can be changed using a `#set!` predicate that
-            // sets the `injection.include-children` key.
-            "injection.include-children" => included_children = IncludedChildren::All,
-
-            // Some queries might only exclude named children but include unnamed
-            // children in their `injection.content` node. This can be enabled using
-            // a `#set!` predicate that sets the `injection.include-unnamed-children` key.
-            "injection.include-unnamed-children" => included_children = IncludedChildren::Unnamed,
-            _ => {}
-        }
-    }
-
-    (language_name, content_node, included_children)
+#[derive(Debug, Clone)]
+pub enum InjectionLanguageMarker<'a> {
+    Name(Cow<'a, str>),
+    Filename(Cow<'a, Path>),
+    Shebang(String),
 }
 
+const SHEBANG: &str = r"#!\s*(?:\S*[/\\](?:env\s+(?:\-\S+\s+)*)?)?([^\s\.\d]+)";
+
 pub struct Merge<I> {
     iter: I,
     spans: Box<dyn Iterator<Item = (usize, std::ops::Range<usize>)>>,
@@ -2286,7 +2378,7 @@ mod test {
         let mut cursor = QueryCursor::new();
 
         let config = HighlightConfiguration::new(language, "", "", "").unwrap();
-        let syntax = Syntax::new(&source, Arc::new(config), Arc::new(loader));
+        let syntax = Syntax::new(&source, Arc::new(config), Arc::new(loader)).unwrap();
 
         let root = syntax.tree().root_node();
         let mut test = |capture, range| {
@@ -2357,7 +2449,7 @@ mod test {
             fn main() {}
         ",
         );
-        let syntax = Syntax::new(&source, Arc::new(config), Arc::new(loader));
+        let syntax = Syntax::new(&source, Arc::new(config), Arc::new(loader)).unwrap();
         let tree = syntax.tree();
         let root = tree.root_node();
         assert_eq!(root.kind(), "source_file");
@@ -2444,7 +2536,7 @@ mod test {
         let language = get_language(language_name).unwrap();
 
         let config = HighlightConfiguration::new(language, "", "", "").unwrap();
-        let syntax = Syntax::new(&source, Arc::new(config), Arc::new(loader));
+        let syntax = Syntax::new(&source, Arc::new(config), Arc::new(loader)).unwrap();
 
         let root = syntax
             .tree()

helix-core/src/test.rs

@@ -1,7 +1,9 @@
 //! Test helpers.
 use crate::{Range, Selection};
+use ropey::Rope;
 use smallvec::SmallVec;
 use std::cmp::Reverse;
+use unicode_segmentation::UnicodeSegmentation;
 
 /// Convert annotated test string to test string and selection.
 ///
@@ -10,6 +12,10 @@ use std::cmp::Reverse;
 /// `#[` for primary selection with head after anchor followed by `|]#`.
 /// `#(` for secondary selection with head after anchor followed by `|)#`.
 ///
+/// If the selection contains any LF or CRLF sequences, which are immediately
+/// followed by the same grapheme, then the subsequent one is removed. This is
+/// to allow representing having the cursor over the end of the line.
+///
 /// # Examples
 ///
 /// ```
@@ -30,23 +36,23 @@ use std::cmp::Reverse;
 pub fn print(s: &str) -> (String, Selection) {
     let mut primary_idx = None;
     let mut ranges = SmallVec::new();
-    let mut iter = s.chars().peekable();
+    let mut iter = UnicodeSegmentation::graphemes(s, true).peekable();
     let mut left = String::with_capacity(s.len());
 
     'outer: while let Some(c) = iter.next() {
         let start = left.chars().count();
 
-        if c != '#' {
-            left.push(c);
+        if c != "#" {
+            left.push_str(c);
             continue;
         }
 
         let (is_primary, close_pair) = match iter.next() {
-            Some('[') => (true, ']'),
-            Some('(') => (false, ')'),
+            Some("[") => (true, "]"),
+            Some("(") => (false, ")"),
             Some(ch) => {
                 left.push('#');
-                left.push(ch);
+                left.push_str(ch);
                 continue;
             }
             None => break,
@@ -56,24 +62,45 @@ pub fn print(s: &str) -> (String, Selection) {
             panic!("primary `#[` already appeared {:?} {:?}", left, s);
         }
 
-        let head_at_beg = iter.next_if_eq(&'|').is_some();
+        let head_at_beg = iter.next_if_eq(&"|").is_some();
+        let last_grapheme = |s: &str| {
+            UnicodeSegmentation::graphemes(s, true)
+                .last()
+                .map(String::from)
+        };
 
         while let Some(c) = iter.next() {
-            if !(c == close_pair && iter.peek() == Some(&'#')) {
-                left.push(c);
+            let next = iter.peek();
+            let mut prev = last_grapheme(left.as_str());
+
+            if !(c == close_pair && next == Some(&"#")) {
+                left.push_str(c);
                 continue;
             }
 
             if !head_at_beg {
-                let prev = left.pop().unwrap();
-                if prev != '|' {
-                    left.push(prev);
-                    left.push(c);
-                    continue;
+                match &prev {
+                    Some(p) if p != "|" => {
+                        left.push_str(c);
+                        continue;
+                    }
+                    Some(p) if p == "|" => {
+                        left.pop().unwrap(); // pop the |
+                        prev = last_grapheme(left.as_str());
+                    }
+                    _ => (),
                 }
             }
 
             iter.next(); // skip "#"
+            let next = iter.peek();
+
+            // skip explicit line end inside selection
+            if (prev == Some(String::from("\r\n")) || prev == Some(String::from("\n")))
+                && next.map(|n| String::from(*n)) == prev
+            {
+                iter.next();
+            }
 
             if is_primary {
                 primary_idx = Some(ranges.len());
@@ -118,14 +145,16 @@ pub fn print(s: &str) -> (String, Selection) {
 /// use smallvec::smallvec;
 ///
 /// assert_eq!(
-///     plain("abc", Selection::new(smallvec![Range::new(0, 1), Range::new(3, 2)], 0)),
+///     plain("abc", &Selection::new(smallvec![Range::new(0, 1), Range::new(3, 2)], 0)),
 ///     "#[a|]#b#(|c)#".to_owned()
 /// );
 /// ```
-pub fn plain(s: &str, selection: Selection) -> String {
+pub fn plain<R: Into<Rope>>(s: R, selection: &Selection) -> String {
+    let s = s.into();
     let primary = selection.primary_index();
-    let mut out = String::with_capacity(s.len() + 5 * selection.len());
-    out.push_str(s);
+    let mut out = String::with_capacity(s.len_bytes() + 5 * selection.len());
+    out.push_str(&s.to_string());
+
     let mut insertion: Vec<_> = selection
         .iter()
         .enumerate()
@@ -138,7 +167,9 @@ pub fn plain(s: &str, selection: Selection) -> String {
                 (false, false) => [(range.anchor, ")#"), (range.head, "#(|")],
             }
         })
+        .map(|(char_idx, marker)| (s.char_to_byte(char_idx), marker))
         .collect();
+
     // insert in reverse order
     insertion.sort_unstable_by_key(|k| Reverse(k.0));
     for (i, s) in insertion {
@@ -262,4 +293,94 @@ mod test {
             print("hello #[|👨‍👩‍👧‍👦]# goodbye")
         );
     }
+
+    #[test]
+    fn plain_single() {
+        assert_eq!("#[|h]#ello", plain("hello", &Selection::single(1, 0)));
+        assert_eq!("#[h|]#ello", plain("hello", &Selection::single(0, 1)));
+        assert_eq!("#[|hell]#o", plain("hello", &Selection::single(4, 0)));
+        assert_eq!("#[hell|]#o", plain("hello", &Selection::single(0, 4)));
+        assert_eq!("#[|hello]#", plain("hello", &Selection::single(5, 0)));
+        assert_eq!("#[hello|]#", plain("hello", &Selection::single(0, 5)));
+    }
+
+    #[test]
+    fn plain_multi() {
+        assert_eq!(
+            plain(
+                "hello",
+                &Selection::new(
+                    SmallVec::from_slice(&[Range::new(1, 0), Range::new(5, 4)]),
+                    0
+                )
+            ),
+            String::from("#[|h]#ell#(|o)#")
+        );
+        assert_eq!(
+            plain(
+                "hello",
+                &Selection::new(
+                    SmallVec::from_slice(&[Range::new(0, 1), Range::new(4, 5)]),
+                    0
+                )
+            ),
+            String::from("#[h|]#ell#(o|)#")
+        );
+        assert_eq!(
+            plain(
+                "hello",
+                &Selection::new(
+                    SmallVec::from_slice(&[Range::new(2, 0), Range::new(5, 3)]),
+                    0
+                )
+            ),
+            String::from("#[|he]#l#(|lo)#")
+        );
+        assert_eq!(
+            plain(
+                "hello\r\nhello\r\nhello\r\n",
+                &Selection::new(
+                    SmallVec::from_slice(&[
+                        Range::new(7, 5),
+                        Range::new(21, 19),
+                        Range::new(14, 12)
+                    ]),
+                    0
+                )
+            ),
+            String::from("hello#[|\r\n]#hello#(|\r\n)#hello#(|\r\n)#")
+        );
+    }
+
+    #[test]
+    fn plain_multi_byte_code_point() {
+        assert_eq!(
+            plain("„“", &Selection::single(1, 0)),
+            String::from("#[|„]#“")
+        );
+        assert_eq!(
+            plain("„“", &Selection::single(2, 1)),
+            String::from("„#[|“]#")
+        );
+        assert_eq!(
+            plain("„“", &Selection::single(0, 1)),
+            String::from("#[„|]#“")
+        );
+        assert_eq!(
+            plain("„“", &Selection::single(1, 2)),
+            String::from("„#[“|]#")
+        );
+        assert_eq!(
+            plain("they said „hello“", &Selection::single(11, 10)),
+            String::from("they said #[|„]#hello“")
+        );
+    }
+
+    #[test]
+    fn plain_multi_code_point_grapheme() {
+        assert_eq!(
+            plain("hello 👨‍👩‍👧‍👦 goodbye", &Selection::single(13, 6)),
+            String::from("hello #[|👨‍👩‍👧‍👦]# goodbye")
+        );
+    }
 }

helix-core/src/text_annotations.rs

@@ -0,0 +1,271 @@
+use std::cell::Cell;
+use std::ops::Range;
+use std::rc::Rc;
+
+use crate::syntax::Highlight;
+use crate::Tendril;
+
+/// An inline annotation is continuous text shown
+/// on the screen before the grapheme that starts at
+/// `char_idx`
+#[derive(Debug, Clone)]
+pub struct InlineAnnotation {
+    pub text: Tendril,
+    pub char_idx: usize,
+}
+
+impl InlineAnnotation {
+    pub fn new(char_idx: usize, text: impl Into<Tendril>) -> Self {
+        Self {
+            char_idx,
+            text: text.into(),
+        }
+    }
+}
+
+/// Represents a **single Grapheme** that is part of the document
+/// that start at `char_idx` that will be replaced with
+/// a different `grapheme`.
+/// If `grapheme` contains multiple graphemes the text
+/// will render incorrectly.
+/// If you want to overlay multiple graphemes simply
+/// use multiple `Overlays`.
+///
+/// # Examples
+///
+/// The following examples are valid overlays for the following text:
+///
+/// `aX͎̊͢͜͝͡bc`
+///
+/// ```
+/// use helix_core::text_annotations::Overlay;
+///
+/// // replaces a
+/// Overlay::new(0, "X");
+///
+/// // replaces X͎̊͢͜͝͡
+/// Overlay::new(1, "\t");
+///
+/// // replaces b
+/// Overlay::new(6, "X̢̢̟͖̲͌̋̇͑͝");
+/// ```
+///
+/// The following examples are invalid uses
+///
+/// ```
+/// use helix_core::text_annotations::Overlay;
+///
+/// // overlay is not aligned at grapheme boundary
+/// Overlay::new(3, "x");
+///
+/// // overlay contains multiple graphemes
+/// Overlay::new(0, "xy");
+/// ```
+#[derive(Debug, Clone)]
+pub struct Overlay {
+    pub char_idx: usize,
+    pub grapheme: Tendril,
+}
+
+impl Overlay {
+    pub fn new(char_idx: usize, grapheme: impl Into<Tendril>) -> Self {
+        Self {
+            char_idx,
+            grapheme: grapheme.into(),
+        }
+    }
+}
+
+/// Line annotations allow for virtual text between normal
+/// text lines. They cause `height` empty lines to be inserted
+/// below the document line that contains `anchor_char_idx`.
+///
+/// These lines can be filled with text in the rendering code
+/// as their contents have no effect beyond visual appearance.
+///
+/// To insert a line after a document line simply set
+/// `anchor_char_idx` to `doc.line_to_char(line_idx)`
+#[derive(Debug, Clone)]
+pub struct LineAnnotation {
+    pub anchor_char_idx: usize,
+    pub height: usize,
+}
+
+#[derive(Debug)]
+struct Layer<A, M> {
+    annotations: Rc<[A]>,
+    current_index: Cell<usize>,
+    metadata: M,
+}
+
+impl<A, M: Clone> Clone for Layer<A, M> {
+    fn clone(&self) -> Self {
+        Layer {
+            annotations: self.annotations.clone(),
+            current_index: self.current_index.clone(),
+            metadata: self.metadata.clone(),
+        }
+    }
+}
+
+impl<A, M> Layer<A, M> {
+    pub fn reset_pos(&self, char_idx: usize, get_char_idx: impl Fn(&A) -> usize) {
+        let new_index = self
+            .annotations
+            .partition_point(|annot| get_char_idx(annot) < char_idx);
+        self.current_index.set(new_index);
+    }
+
+    pub fn consume(&self, char_idx: usize, get_char_idx: impl Fn(&A) -> usize) -> Option<&A> {
+        let annot = self.annotations.get(self.current_index.get())?;
+        debug_assert!(get_char_idx(annot) >= char_idx);
+        if get_char_idx(annot) == char_idx {
+            self.current_index.set(self.current_index.get() + 1);
+            Some(annot)
+        } else {
+            None
+        }
+    }
+}
+
+impl<A, M> From<(Rc<[A]>, M)> for Layer<A, M> {
+    fn from((annotations, metadata): (Rc<[A]>, M)) -> Layer<A, M> {
+        Layer {
+            annotations,
+            current_index: Cell::new(0),
+            metadata,
+        }
+    }
+}
+
+fn reset_pos<A, M>(layers: &[Layer<A, M>], pos: usize, get_pos: impl Fn(&A) -> usize) {
+    for layer in layers {
+        layer.reset_pos(pos, &get_pos)
+    }
+}
+
+/// Annotations that change that is displayed when the document is render.
+/// Also commonly called virtual text.
+#[derive(Default, Debug, Clone)]
+pub struct TextAnnotations {
+    inline_annotations: Vec<Layer<InlineAnnotation, Option<Highlight>>>,
+    overlays: Vec<Layer<Overlay, Option<Highlight>>>,
+    line_annotations: Vec<Layer<LineAnnotation, ()>>,
+}
+
+impl TextAnnotations {
+    /// Prepare the TextAnnotations for iteration starting at char_idx
+    pub fn reset_pos(&self, char_idx: usize) {
+        reset_pos(&self.inline_annotations, char_idx, |annot| annot.char_idx);
+        reset_pos(&self.overlays, char_idx, |annot| annot.char_idx);
+        reset_pos(&self.line_annotations, char_idx, |annot| {
+            annot.anchor_char_idx
+        });
+    }
+
+    pub fn collect_overlay_highlights(
+        &self,
+        char_range: Range<usize>,
+    ) -> Vec<(usize, Range<usize>)> {
+        let mut highlights = Vec::new();
+        self.reset_pos(char_range.start);
+        for char_idx in char_range {
+            if let Some((_, Some(highlight))) = self.overlay_at(char_idx) {
+                // we don't know the number of chars the original grapheme takes
+                // however it doesn't matter as highlight boundaries are automatically
+                // aligned to grapheme boundaries in the rendering code
+                highlights.push((highlight.0, char_idx..char_idx + 1))
+            }
+        }
+
+        highlights
+    }
+
+    /// Add new inline annotations.
+    ///
+    /// The annotations grapheme will be rendered with `highlight`
+    /// patched on top of `ui.text`.
+    ///
+    /// The annotations **must be sorted** by their `char_idx`.
+    /// Multiple annotations with the same `char_idx` are allowed,
+    /// they will be display in the order that they are present in the layer.
+    ///
+    /// If multiple layers contain annotations at the same position
+    /// the annotations that belong to the layers added first will be shown first.
+    pub fn add_inline_annotations(
+        &mut self,
+        layer: Rc<[InlineAnnotation]>,
+        highlight: Option<Highlight>,
+    ) -> &mut Self {
+        self.inline_annotations.push((layer, highlight).into());
+        self
+    }
+
+    /// Add new grapheme overlays.
+    ///
+    /// The overlaid grapheme will be rendered with `highlight`
+    /// patched on top of `ui.text`.
+    ///
+    /// The overlays **must be sorted** by their `char_idx`.
+    /// Multiple overlays with the same `char_idx` **are allowed**.
+    ///
+    /// If multiple layers contain overlay at the same position
+    /// the overlay from the layer added last will be show.
+    pub fn add_overlay(&mut self, layer: Rc<[Overlay]>, highlight: Option<Highlight>) -> &mut Self {
+        self.overlays.push((layer, highlight).into());
+        self
+    }
+
+    /// Add new annotation lines.
+    ///
+    /// The line annotations **must be sorted** by their `char_idx`.
+    /// Multiple line annotations with the same `char_idx` **are not allowed**.
+    pub fn add_line_annotation(&mut self, layer: Rc<[LineAnnotation]>) -> &mut Self {
+        self.line_annotations.push((layer, ()).into());
+        self
+    }
+
+    /// Removes all line annotations, useful for vertical motions
+    /// so that virtual text lines are automatically skipped.
+    pub fn clear_line_annotations(&mut self) {
+        self.line_annotations.clear();
+    }
+
+    pub(crate) fn next_inline_annotation_at(
+        &self,
+        char_idx: usize,
+    ) -> Option<(&InlineAnnotation, Option<Highlight>)> {
+        self.inline_annotations.iter().find_map(|layer| {
+            let annotation = layer.consume(char_idx, |annot| annot.char_idx)?;
+            Some((annotation, layer.metadata))
+        })
+    }
+
+    pub(crate) fn overlay_at(&self, char_idx: usize) -> Option<(&Overlay, Option<Highlight>)> {
+        let mut overlay = None;
+        for layer in &self.overlays {
+            while let Some(new_overlay) = layer.consume(char_idx, |annot| annot.char_idx) {
+                overlay = Some((new_overlay, layer.metadata));
+            }
+        }
+        overlay
+    }
+
+    pub(crate) fn annotation_lines_at(&self, char_idx: usize) -> usize {
+        self.line_annotations
+            .iter()
+            .map(|layer| {
+                let mut lines = 0;
+                while let Some(annot) = layer.annotations.get(layer.current_index.get()) {
+                    if annot.anchor_char_idx == char_idx {
+                        layer.current_index.set(layer.current_index.get() + 1);
+                        lines += annot.height
+                    } else {
+                        break;
+                    }
+                }
+                lines
+            })
+            .sum()
+    }
+}

helix-core/src/textobject.rs

@@ -231,8 +231,20 @@ fn textobject_pair_surround_impl(
     };
     pair_pos
         .map(|(anchor, head)| match textobject {
-            TextObject::Inside => Range::new(next_grapheme_boundary(slice, anchor), head),
-            TextObject::Around => Range::new(anchor, next_grapheme_boundary(slice, head)),
+            TextObject::Inside => {
+                if anchor < head {
+                    Range::new(next_grapheme_boundary(slice, anchor), head)
+                } else {
+                    Range::new(anchor, next_grapheme_boundary(slice, head))
+                }
+            }
+            TextObject::Around => {
+                if anchor < head {
+                    Range::new(anchor, next_grapheme_boundary(slice, head))
+                } else {
+                    Range::new(next_grapheme_boundary(slice, anchor), head)
+                }
+            }
             TextObject::Movement => unreachable!(),
         })
         .unwrap_or(range)
@@ -425,7 +437,7 @@ mod test {
             let text = Rope::from(s.as_str());
             let selection = selection
                 .transform(|r| textobject_paragraph(text.slice(..), r, TextObject::Inside, 1));
-            let actual = crate::test::plain(&s, selection);
+            let actual = crate::test::plain(s.as_ref(), &selection);
             assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
         }
     }
@@ -448,7 +460,7 @@ mod test {
             let text = Rope::from(s.as_str());
             let selection = selection
                 .transform(|r| textobject_paragraph(text.slice(..), r, TextObject::Inside, 2));
-            let actual = crate::test::plain(&s, selection);
+            let actual = crate::test::plain(s.as_ref(), &selection);
             assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
         }
     }
@@ -479,7 +491,7 @@ mod test {
             let text = Rope::from(s.as_str());
             let selection = selection
                 .transform(|r| textobject_paragraph(text.slice(..), r, TextObject::Around, 1));
-            let actual = crate::test::plain(&s, selection);
+            let actual = crate::test::plain(s.as_ref(), &selection);
             assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
         }
     }

helix-core/src/transaction.rs

@@ -1,8 +1,11 @@
+use smallvec::SmallVec;
+
 use crate::{Range, Rope, Selection, Tendril};
 use std::borrow::Cow;
 
 /// (from, to, replacement)
 pub type Change = (usize, usize, Option<Tendril>);
+pub type Deletion = (usize, usize);
 
 // TODO: pub(crate)
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -466,6 +469,33 @@ impl Transaction {
         self
     }
 
+    /// Generate a transaction from a set of potentially overlapping changes. The `change_ranges`
+    /// iterator yield the range (of removed text) in the old document for each edit. If any change
+    /// overlaps with a range overlaps with a previous range then that range is ignored.
+    ///
+    /// The `process_change` callback is called for each edit that is not ignored (in the order
+    /// yielded by `changes`) and should return the new text that the associated range will be
+    /// replaced with.
+    ///
+    /// To make this function more flexible the iterator can yield additional data for each change
+    /// that is passed to `process_change`
+    pub fn change_ignore_overlapping<T>(
+        doc: &Rope,
+        change_ranges: impl Iterator<Item = (usize, usize, T)>,
+        mut process_change: impl FnMut(usize, usize, T) -> Option<Tendril>,
+    ) -> Self {
+        let mut last = 0;
+        let changes = change_ranges.filter_map(|(from, to, data)| {
+            if from < last {
+                return None;
+            }
+            let tendril = process_change(from, to, data);
+            last = to;
+            Some((from, to, tendril))
+        });
+        Self::change(doc, changes)
+    }
+
     /// Generate a transaction from a set of changes.
     pub fn change<I>(doc: &Rope, changes: I) -> Self
     where
@@ -481,6 +511,11 @@ impl Transaction {
         for (from, to, tendril) in changes {
             // Verify ranges are ordered and not overlapping
             debug_assert!(last <= from);
+            // Verify ranges are correct
+            debug_assert!(
+                from <= to,
+                "Edit end must end before it starts (should {from} <= {to})"
+            );
 
             // Retain from last "to" to current "from"
             changeset.retain(from - last);
@@ -500,6 +535,46 @@ impl Transaction {
         Self::from(changeset)
     }
 
+    /// Generate a transaction from a set of potentially overlapping deletions
+    /// by merging overlapping deletions together.
+    pub fn delete<I>(doc: &Rope, deletions: I) -> Self
+    where
+        I: Iterator<Item = Deletion>,
+    {
+        let len = doc.len_chars();
+
+        let (lower, upper) = deletions.size_hint();
+        let size = upper.unwrap_or(lower);
+        let mut changeset = ChangeSet::with_capacity(2 * size + 1); // rough estimate
+
+        let mut last = 0;
+        for (mut from, to) in deletions {
+            if last > to {
+                continue;
+            }
+            if last > from {
+                from = last
+            }
+            debug_assert!(
+                from <= to,
+                "Edit end must end before it starts (should {from} <= {to})"
+            );
+            // Retain from last "to" to current "from"
+            changeset.retain(from - last);
+            changeset.delete(to - from);
+            last = to;
+        }
+
+        changeset.retain(len - last);
+
+        Self::from(changeset)
+    }
+
+    pub fn insert_at_eof(mut self, text: Tendril) -> Transaction {
+        self.changes.insert(text);
+        self
+    }
+
     /// Generate a transaction with a change per selection range.
     pub fn change_by_selection<F>(doc: &Rope, selection: &Selection, f: F) -> Self
     where
@@ -508,6 +583,54 @@ impl Transaction {
         Self::change(doc, selection.iter().map(f))
     }
 
+    pub fn change_by_selection_ignore_overlapping(
+        doc: &Rope,
+        selection: &Selection,
+        mut change_range: impl FnMut(&Range) -> (usize, usize),
+        mut create_tendril: impl FnMut(usize, usize) -> Option<Tendril>,
+    ) -> (Transaction, Selection) {
+        let mut last_selection_idx = None;
+        let mut new_primary_idx = None;
+        let mut ranges: SmallVec<[Range; 1]> = SmallVec::new();
+        let process_change = |change_start, change_end, (idx, range): (usize, &Range)| {
+            // update the primary idx
+            if idx == selection.primary_index() {
+                new_primary_idx = Some(idx);
+            } else if new_primary_idx.is_none() {
+                if idx > selection.primary_index() {
+                    new_primary_idx = last_selection_idx;
+                } else {
+                    last_selection_idx = Some(idx);
+                }
+            }
+            ranges.push(*range);
+            create_tendril(change_start, change_end)
+        };
+        let transaction = Self::change_ignore_overlapping(
+            doc,
+            selection.iter().enumerate().map(|range| {
+                let (change_start, change_end) = change_range(range.1);
+                (change_start, change_end, range)
+            }),
+            process_change,
+        );
+
+        (
+            transaction,
+            Selection::new(ranges, new_primary_idx.unwrap_or(0)),
+        )
+    }
+
+    /// Generate a transaction with a deletion per selection range.
+    /// Compared to using `change_by_selection` directly these ranges may overlap.
+    /// In that case they are merged
+    pub fn delete_by_selection<F>(doc: &Rope, selection: &Selection, f: F) -> Self
+    where
+        F: FnMut(&Range) -> Deletion,
+    {
+        Self::delete(doc, selection.iter().map(f))
+    }
+
     /// Insert text at each selection head.
     pub fn insert(doc: &Rope, selection: &Selection, text: Tendril) -> Self {
         Self::change_by_selection(doc, selection, |range| {

helix-core/src/wrap.rs

@@ -2,6 +2,6 @@ use smartstring::{LazyCompact, SmartString};
 
 /// Given a slice of text, return the text re-wrapped to fit it
 /// within the given width.
-pub fn reflow_hard_wrap(text: &str, max_line_len: usize) -> SmartString<LazyCompact> {
-    textwrap::refill(text, max_line_len).into()
+pub fn reflow_hard_wrap(text: &str, text_width: usize) -> SmartString<LazyCompact> {
+    textwrap::refill(text, text_width).into()
 }

helix-core/tests/indent.rs

@@ -1,5 +1,5 @@
 use helix_core::{
-    indent::{treesitter_indent_for_pos, IndentStyle},
+    indent::{indent_level_for_line, treesitter_indent_for_pos, IndentStyle},
     syntax::Loader,
     Syntax,
 };
@@ -17,6 +17,39 @@ fn test_treesitter_indent_rust_2() {
     // test_treesitter_indent("commands.rs", "source.rust");
 }
 
+#[test]
+fn test_indent_level_for_line_with_spaces() {
+    let tab_width: usize = 4;
+    let indent_width: usize = 4;
+
+    let line = ropey::Rope::from_str("        Indented with 8 spaces");
+
+    let indent_level = indent_level_for_line(line.slice(0..), tab_width, indent_width);
+    assert_eq!(indent_level, 2)
+}
+
+#[test]
+fn test_indent_level_for_line_with_tabs() {
+    let tab_width: usize = 4;
+    let indent_width: usize = 4;
+
+    let line = ropey::Rope::from_str("\t\tIndented with 2 tabs");
+
+    let indent_level = indent_level_for_line(line.slice(0..), tab_width, indent_width);
+    assert_eq!(indent_level, 2)
+}
+
+#[test]
+fn test_indent_level_for_line_with_spaces_and_tabs() {
+    let tab_width: usize = 4;
+    let indent_width: usize = 4;
+
+    let line = ropey::Rope::from_str("   \t \tIndented with mix of spaces and tabs");
+
+    let indent_level = indent_level_for_line(line.slice(0..), tab_width, indent_width);
+    assert_eq!(indent_level, 2)
+}
+
 fn test_treesitter_indent(file_name: &str, lang_scope: &str) {
     let mut test_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
     test_dir.push("tests/data/indent");
@@ -28,8 +61,8 @@ fn test_treesitter_indent(file_name: &str, lang_scope: &str) {
 
     let mut config_file = test_dir;
     config_file.push("languages.toml");
-    let config = std::fs::read(config_file).unwrap();
-    let config = toml::from_slice(&config).unwrap();
+    let config = std::fs::read_to_string(config_file).unwrap();
+    let config = toml::from_str(&config).unwrap();
     let loader = Loader::new(config);
 
     // set runtime path so we can find the queries
@@ -39,18 +72,20 @@ fn test_treesitter_indent(file_name: &str, lang_scope: &str) {
 
     let language_config = loader.language_config_for_scope(lang_scope).unwrap();
     let highlight_config = language_config.highlight_config(&[]).unwrap();
-    let syntax = Syntax::new(&doc, highlight_config, std::sync::Arc::new(loader));
+    let syntax = Syntax::new(&doc, highlight_config, std::sync::Arc::new(loader)).unwrap();
     let indent_query = language_config.indent_query().unwrap();
     let text = doc.slice(..);
 
     for i in 0..doc.len_lines() {
         let line = text.line(i);
         if let Some(pos) = helix_core::find_first_non_whitespace_char(line) {
+            let tab_and_indent_width: usize = 4;
             let suggested_indent = treesitter_indent_for_pos(
                 indent_query,
                 &syntax,
-                &IndentStyle::Spaces(4),
-                4,
+                &IndentStyle::Spaces(tab_and_indent_width as u8),
+                tab_and_indent_width,
+                tab_and_indent_width,
                 text,
                 i,
                 text.line_to_char(i) + pos,

helix-dap/Cargo.toml

@@ -19,7 +19,7 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 thiserror = "1.0"
 tokio = { version = "1", features = ["rt", "rt-multi-thread", "io-util", "io-std", "time", "process", "macros", "fs", "parking_lot", "net", "sync"] }
-which = "4.2"
+which = "4.4"
 
 [dev-dependencies]
 fern = "0.6"

helix-dap/src/client.rs

@@ -1,4 +1,5 @@
 use crate::{
+    requests::DisconnectArguments,
     transport::{Payload, Request, Response, Transport},
     types::*,
     Error, Result, ThreadId,
@@ -31,6 +32,8 @@ pub struct Client {
     _process: Option<Child>,
     server_tx: UnboundedSender<Payload>,
     request_counter: AtomicU64,
+    connection_type: Option<ConnectionType>,
+    starting_request_args: Option<Value>,
     pub caps: Option<DebuggerCapabilities>,
     // thread_id -> frames
     pub stack_frames: HashMap<ThreadId, Vec<StackFrame>>,
@@ -41,6 +44,12 @@ pub struct Client {
     pub quirks: DebuggerQuirks,
 }
 
+#[derive(Clone, Copy, Debug)]
+pub enum ConnectionType {
+    Launch,
+    Attach,
+}
+
 impl Client {
     // Spawn a process and communicate with it by either TCP or stdio
     pub async fn process(
@@ -53,12 +62,10 @@ impl Client {
         if command.is_empty() {
             return Result::Err(Error::Other(anyhow!("Command not provided")));
         }
-        if transport == "tcp" && port_arg.is_some() {
-            Self::tcp_process(command, args, port_arg.unwrap(), id).await
-        } else if transport == "stdio" {
-            Self::stdio(command, args, id)
-        } else {
-            Result::Err(Error::Other(anyhow!("Incorrect transport {}", transport)))
+        match (transport, port_arg) {
+            ("tcp", Some(port_arg)) => Self::tcp_process(command, args, port_arg, id).await,
+            ("stdio", _) => Self::stdio(command, args, id),
+            _ => Result::Err(Error::Other(anyhow!("Incorrect transport {}", transport))),
         }
     }
 
@@ -70,7 +77,7 @@ impl Client {
         process: Option<Child>,
     ) -> Result<(Self, UnboundedReceiver<Payload>)> {
         let (server_rx, server_tx) = Transport::start(rx, tx, err, id);
-        let (client_rx, client_tx) = unbounded_channel();
+        let (client_tx, client_rx) = unbounded_channel();
 
         let client = Self {
             id,
@@ -78,7 +85,8 @@ impl Client {
             server_tx,
             request_counter: AtomicU64::new(0),
             caps: None,
-            //
+            connection_type: None,
+            starting_request_args: None,
             stack_frames: HashMap::new(),
             thread_states: HashMap::new(),
             thread_id: None,
@@ -86,9 +94,9 @@ impl Client {
             quirks: DebuggerQuirks::default(),
         };
 
-        tokio::spawn(Self::recv(server_rx, client_rx));
+        tokio::spawn(Self::recv(server_rx, client_tx));
 
-        Ok((client, client_tx))
+        Ok((client, client_rx))
     }
 
     pub async fn tcp(
@@ -150,6 +158,10 @@ impl Client {
         )
     }
 
+    pub fn starting_request_args(&self) -> &Option<Value> {
+        &self.starting_request_args
+    }
+
     pub async fn tcp_process(
         cmd: &str,
         args: Vec<&str>,
@@ -207,6 +219,10 @@ impl Client {
         self.id
     }
 
+    pub fn connection_type(&self) -> Option<ConnectionType> {
+        self.connection_type
+    }
+
     fn next_request_id(&self) -> u64 {
         self.request_counter.fetch_add(1, Ordering::Relaxed)
     }
@@ -254,7 +270,7 @@ impl Client {
             // TODO: specifiable timeout, delay other calls until initialize success
             timeout(Duration::from_secs(20), callback_rx.recv())
                 .await
-                .map_err(|_| Error::Timeout)? // return Timeout
+                .map_err(|_| Error::Timeout(id))? // return Timeout
                 .ok_or(Error::StreamClosed)?
                 .map(|response| response.body.unwrap_or_default())
             // TODO: check response.success
@@ -334,18 +350,35 @@ impl Client {
         Ok(())
     }
 
-    pub fn disconnect(&self) -> impl Future<Output = Result<Value>> {
-        self.call::<requests::Disconnect>(())
+    pub fn disconnect(
+        &mut self,
+        args: Option<DisconnectArguments>,
+    ) -> impl Future<Output = Result<Value>> {
+        self.connection_type = None;
+        self.call::<requests::Disconnect>(args)
     }
 
-    pub fn launch(&self, args: serde_json::Value) -> impl Future<Output = Result<Value>> {
+    pub fn launch(&mut self, args: serde_json::Value) -> impl Future<Output = Result<Value>> {
+        self.connection_type = Some(ConnectionType::Launch);
+        self.starting_request_args = Some(args.clone());
         self.call::<requests::Launch>(args)
     }
 
-    pub fn attach(&self, args: serde_json::Value) -> impl Future<Output = Result<Value>> {
+    pub fn attach(&mut self, args: serde_json::Value) -> impl Future<Output = Result<Value>> {
+        self.connection_type = Some(ConnectionType::Attach);
+        self.starting_request_args = Some(args.clone());
         self.call::<requests::Attach>(args)
     }
 
+    pub fn restart(&self) -> impl Future<Output = Result<Value>> {
+        let args = if let Some(args) = &self.starting_request_args {
+            args.clone()
+        } else {
+            Value::Null
+        };
+        self.call::<requests::Restart>(args)
+    }
+
     pub async fn set_breakpoints(
         &self,
         file: PathBuf,
@@ -477,4 +510,10 @@ impl Client {
 
         self.call::<requests::SetExceptionBreakpoints>(args)
     }
+
+    pub fn current_stack_frame(&self) -> Option<&StackFrame> {
+        self.stack_frames
+            .get(&self.thread_id?)?
+            .get(self.active_frame?)
+    }
 }

helix-dap/src/lib.rs

@@ -2,7 +2,7 @@ mod client;
 mod transport;
 mod types;
 
-pub use client::Client;
+pub use client::{Client, ConnectionType};
 pub use events::Event;
 pub use transport::{Payload, Response, Transport};
 pub use types::*;
@@ -14,8 +14,8 @@ pub enum Error {
     Parse(#[from] serde_json::Error),
     #[error("IO Error: {0}")]
     IO(#[from] std::io::Error),
-    #[error("request timed out")]
-    Timeout,
+    #[error("request {0} timed out")]
+    Timeout(u64),
     #[error("server closed the stream")]
     StreamClosed,
     #[error(transparent)]

helix-dap/src/transport.rs

@@ -230,38 +230,48 @@ impl Transport {
         }
     }
 
-    async fn recv(
+    async fn recv_inner(
         transport: Arc<Self>,
         mut server_stdout: Box<dyn AsyncBufRead + Unpin + Send>,
         client_tx: UnboundedSender<Payload>,
-    ) {
+    ) -> Result<()> {
         let mut recv_buffer = String::new();
         loop {
-            match Self::recv_server_message(&mut server_stdout, &mut recv_buffer).await {
-                Ok(msg) => {
-                    transport
-                        .process_server_message(&client_tx, msg)
-                        .await
-                        .unwrap();
-                }
-                Err(err) => {
-                    error!("err: <- {:?}", err);
-                    break;
-                }
-            }
+            let msg = Self::recv_server_message(&mut server_stdout, &mut recv_buffer).await?;
+            transport.process_server_message(&client_tx, msg).await?;
         }
     }
 
+    async fn recv(
+        transport: Arc<Self>,
+        server_stdout: Box<dyn AsyncBufRead + Unpin + Send>,
+        client_tx: UnboundedSender<Payload>,
+    ) {
+        if let Err(err) = Self::recv_inner(transport, server_stdout, client_tx).await {
+            error!("err: <- {:?}", err);
+        }
+    }
+
+    async fn send_inner(
+        transport: Arc<Self>,
+        mut server_stdin: Box<dyn AsyncWrite + Unpin + Send>,
+        mut client_rx: UnboundedReceiver<Payload>,
+    ) -> Result<()> {
+        while let Some(payload) = client_rx.recv().await {
+            transport
+                .send_payload_to_server(&mut server_stdin, payload)
+                .await?;
+        }
+        Ok(())
+    }
+
     async fn send(
         transport: Arc<Self>,
-        mut server_stdin: Box<dyn AsyncWrite + Unpin + Send>,
-        mut client_rx: UnboundedReceiver<Payload>,
+        server_stdin: Box<dyn AsyncWrite + Unpin + Send>,
+        client_rx: UnboundedReceiver<Payload>,
     ) {
-        while let Some(payload) = client_rx.recv().await {
-            transport
-                .send_payload_to_server(&mut server_stdin, payload)
-                .await
-                .unwrap()
+        if let Err(err) = Self::send_inner(transport, server_stdin, client_rx).await {
+            error!("err: <- {:?}", err);
         }
     }
 

helix-dap/src/types.rs

@@ -378,7 +378,7 @@ pub mod requests {
 
     impl Request for Launch {
         type Arguments = Value;
-        type Result = Value;
+        type Result = ();
         const COMMAND: &'static str = "launch";
     }
 
@@ -387,15 +387,35 @@ pub mod requests {
 
     impl Request for Attach {
         type Arguments = Value;
-        type Result = Value;
+        type Result = ();
         const COMMAND: &'static str = "attach";
     }
 
+    #[derive(Debug, Default, PartialEq, Eq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct DisconnectArguments {
+        #[serde(skip_serializing_if = "Option::is_none")]
+        pub restart: Option<bool>,
+        #[serde(skip_serializing_if = "Option::is_none")]
+        pub terminate_debuggee: Option<bool>,
+        #[serde(skip_serializing_if = "Option::is_none")]
+        pub suspend_debuggee: Option<bool>,
+    }
+
+    #[derive(Debug)]
+    pub enum Restart {}
+
+    impl Request for Restart {
+        type Arguments = Value;
+        type Result = ();
+        const COMMAND: &'static str = "restart";
+    }
+
     #[derive(Debug)]
     pub enum Disconnect {}
 
     impl Request for Disconnect {
-        type Arguments = ();
+        type Arguments = Option<DisconnectArguments>;
         type Result = ();
         const COMMAND: &'static str = "disconnect";
     }

helix-loader/Cargo.toml

@@ -16,10 +16,10 @@ path = "src/main.rs"
 [dependencies]
 anyhow = "1"
 serde = { version = "1.0", features = ["derive"] }
-toml = "0.5"
-etcetera = "0.4"
+toml = "0.7"
+etcetera = "0.8"
 tree-sitter = "0.20"
-once_cell = "1.16"
+once_cell = "1.17"
 log = "0.4"
 
 # TODO: these two should be on !wasm32 only
@@ -27,6 +27,7 @@ log = "0.4"
 # cloning/compiling tree-sitter grammars
 cc = { version = "1" }
 threadpool = { version = "1.0" }
+tempfile = "3.5.0"
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
-libloading = "0.7"
+libloading = "0.8"

helix-loader/build.rs

@@ -1,4 +1,5 @@
 use std::borrow::Cow;
+use std::path::Path;
 use std::process::Command;
 
 const VERSION: &str = include_str!("../VERSION");
@@ -11,7 +12,7 @@ fn main() {
         .filter(|output| output.status.success())
         .and_then(|x| String::from_utf8(x.stdout).ok());
 
-    let version: Cow<_> = match git_hash {
+    let version: Cow<_> = match &git_hash {
         Some(git_hash) => format!("{} ({})", VERSION, &git_hash[..8]).into(),
         None => VERSION.into(),
     };
@@ -23,4 +24,40 @@ fn main() {
 
     println!("cargo:rerun-if-changed=../VERSION");
     println!("cargo:rustc-env=VERSION_AND_GIT_HASH={}", version);
+
+    if git_hash.is_none() {
+        return;
+    }
+
+    // we need to revparse because the git dir could be anywhere if you are
+    // using detached worktrees but there is no good way to obtain an OsString
+    // from command output so for now we can't accept non-utf8 paths here
+    // probably rare enouch where it doesn't matter tough we could use gitoxide
+    // here but that would be make it a hard dependency and slow compile times
+    let Some(git_dir): Option<String> = Command::new("git")
+        .args(["rev-parse", "--git-dir"])
+        .output()
+        .ok()
+        .filter(|output| output.status.success())
+        .and_then(|x| String::from_utf8(x.stdout).ok())
+    else{ return; };
+    // If heads starts pointing at something else (different branch)
+    // we need to return
+    let head = Path::new(&git_dir).join("HEAD");
+    if head.exists() {
+        println!("cargo:rerun-if-changed={}", head.display());
+    }
+    // if the thing head points to (branch) itself changes
+    // we need to return
+    let Some(head_ref): Option<String> = Command::new("git")
+        .args(["symbolic-ref", "HEAD"])
+        .output()
+        .ok()
+        .filter(|output| output.status.success())
+        .and_then(|x| String::from_utf8(x.stdout).ok())
+    else{ return; };
+    let head_ref = Path::new(&git_dir).join(head_ref);
+    if head_ref.exists() {
+        println!("cargo:rerun-if-changed={}", head_ref.display());
+    }
 }

helix-loader/src/config.rs

@@ -1,42 +1,46 @@
+use std::str::from_utf8;
+
 /// Default built-in languages.toml.
 pub fn default_lang_config() -> toml::Value {
-    toml::from_slice(include_bytes!("../../languages.toml"))
+    let default_config = include_bytes!("../../languages.toml");
+    toml::from_str(from_utf8(default_config).unwrap())
         .expect("Could not parse built-in languages.toml to valid toml")
 }
 
 /// User configured languages.toml file, merged with the default config.
 pub fn user_lang_config() -> Result<toml::Value, toml::de::Error> {
-    let config = crate::local_config_dirs()
-        .into_iter()
-        .chain([crate::config_dir()].into_iter())
-        .map(|path| path.join("languages.toml"))
-        .filter_map(|file| {
-            std::fs::read(&file)
-                .map(|config| toml::from_slice(&config))
-                .ok()
-        })
-        .collect::<Result<Vec<_>, _>>()?
-        .into_iter()
-        .chain([default_lang_config()].into_iter())
-        .fold(toml::Value::Table(toml::value::Table::default()), |a, b| {
-            // combines for example
-            // b:
-            //   [[language]]
-            //   name = "toml"
-            //   language-server = { command = "taplo", args = ["lsp", "stdio"] }
-            //
-            // a:
-            //   [[language]]
-            //   language-server = { command = "/usr/bin/taplo" }
-            //
-            // into:
-            //   [[language]]
-            //   name = "toml"
-            //   language-server = { command = "/usr/bin/taplo" }
-            //
-            // thus it overrides the third depth-level of b with values of a if they exist, but otherwise merges their values
-            crate::merge_toml_values(b, a, 3)
-        });
+    let config = [
+        crate::config_dir(),
+        crate::find_workspace().0.join(".helix"),
+    ]
+    .into_iter()
+    .map(|path| path.join("languages.toml"))
+    .filter_map(|file| {
+        std::fs::read_to_string(file)
+            .map(|config| toml::from_str(&config))
+            .ok()
+    })
+    .collect::<Result<Vec<_>, _>>()?
+    .into_iter()
+    .fold(default_lang_config(), |a, b| {
+        // combines for example
+        // b:
+        //   [[language]]
+        //   name = "toml"
+        //   language-server = { command = "taplo", args = ["lsp", "stdio"] }
+        //
+        // a:
+        //   [[language]]
+        //   language-server = { command = "/usr/bin/taplo" }
+        //
+        // into:
+        //   [[language]]
+        //   name = "toml"
+        //   language-server = { command = "/usr/bin/taplo" }
+        //
+        // thus it overrides the third depth-level of b with values of a if they exist, but otherwise merges their values
+        crate::merge_toml_values(a, b, 3)
+    });
 
     Ok(config)
 }

helix-loader/src/grammar.rs

@@ -1,4 +1,4 @@
-use anyhow::{anyhow, Context, Result};
+use anyhow::{anyhow, bail, Context, Result};
 use serde::{Deserialize, Serialize};
 use std::fs;
 use std::time::SystemTime;
@@ -8,6 +8,7 @@ use std::{
     process::Command,
     sync::mpsc::channel,
 };
+use tempfile::TempPath;
 use tree_sitter::Language;
 
 #[cfg(unix)]
@@ -67,8 +68,9 @@ pub fn get_language(name: &str) -> Result<Language> {
 #[cfg(not(target_arch = "wasm32"))]
 pub fn get_language(name: &str) -> Result<Language> {
     use libloading::{Library, Symbol};
-    let mut library_path = crate::runtime_dir().join("grammars").join(name);
-    library_path.set_extension(DYLIB_EXTENSION);
+    let mut rel_library_path = PathBuf::new().join("grammars").join(name);
+    rel_library_path.set_extension(DYLIB_EXTENSION);
+    let library_path = crate::runtime_file(&rel_library_path);
 
     let library = unsafe { Library::new(&library_path) }
         .with_context(|| format!("Error opening dynamic library {:?}", library_path))?;
@@ -96,15 +98,12 @@ pub fn fetch_grammars() -> Result<()> {
     let mut git_up_to_date = 0;
     let mut non_git = Vec::new();
 
-    for res in results {
+    for (grammar_id, res) in results {
         match res {
             Ok(FetchStatus::GitUpToDate) => git_up_to_date += 1,
-            Ok(FetchStatus::GitUpdated {
-                grammar_id,
-                revision,
-            }) => git_updated.push((grammar_id, revision)),
-            Ok(FetchStatus::NonGit { grammar_id }) => non_git.push(grammar_id),
-            Err(e) => errors.push(e),
+            Ok(FetchStatus::GitUpdated { revision }) => git_updated.push((grammar_id, revision)),
+            Ok(FetchStatus::NonGit) => non_git.push(grammar_id),
+            Err(e) => errors.push((grammar_id, e)),
         }
     }
 
@@ -136,10 +135,10 @@ pub fn fetch_grammars() -> Result<()> {
 
     if !errors.is_empty() {
         let len = errors.len();
-        println!("{} grammars failed to fetch", len);
-        for (i, error) in errors.into_iter().enumerate() {
-            println!("\tFailure {}/{}: {}", i + 1, len, error);
+        for (i, (grammar, error)) in errors.into_iter().enumerate() {
+            println!("Failure {}/{len}: {grammar} {error}", i + 1);
         }
+        bail!("{len} grammars failed to fetch");
     }
 
     Ok(())
@@ -156,11 +155,11 @@ pub fn build_grammars(target: Option<String>) -> Result<()> {
     let mut already_built = 0;
     let mut built = Vec::new();
 
-    for res in results {
+    for (grammar_id, res) in results {
         match res {
             Ok(BuildStatus::AlreadyBuilt) => already_built += 1,
-            Ok(BuildStatus::Built { grammar_id }) => built.push(grammar_id),
-            Err(e) => errors.push(e),
+            Ok(BuildStatus::Built) => built.push(grammar_id),
+            Err(e) => errors.push((grammar_id, e)),
         }
     }
 
@@ -177,10 +176,10 @@ pub fn build_grammars(target: Option<String>) -> Result<()> {
 
     if !errors.is_empty() {
         let len = errors.len();
-        println!("{} grammars failed to build", len);
-        for (i, error) in errors.into_iter().enumerate() {
-            println!("\tFailure {}/{}: {}", i, len, error);
+        for (i, (grammar_id, error)) in errors.into_iter().enumerate() {
+            println!("Failure {}/{len}: {grammar_id} {error}", i + 1);
         }
+        bail!("{len} grammars failed to build");
     }
 
     Ok(())
@@ -212,7 +211,7 @@ fn get_grammar_configs() -> Result<Vec<GrammarConfiguration>> {
     Ok(grammars)
 }
 
-fn run_parallel<F, Res>(grammars: Vec<GrammarConfiguration>, job: F) -> Vec<Result<Res>>
+fn run_parallel<F, Res>(grammars: Vec<GrammarConfiguration>, job: F) -> Vec<(String, Result<Res>)>
 where
     F: Fn(GrammarConfiguration) -> Result<Res> + Send + 'static + Clone,
     Res: Send + 'static,
@@ -227,7 +226,7 @@ where
         pool.execute(move || {
             // Ignore any SendErrors, if any job in another thread has encountered an
             // error the Receiver will be closed causing this send to fail.
-            let _ = tx.send(job(grammar));
+            let _ = tx.send((grammar.grammar_id.clone(), job(grammar)));
         });
     }
 
@@ -238,13 +237,8 @@ where
 
 enum FetchStatus {
     GitUpToDate,
-    GitUpdated {
-        grammar_id: String,
-        revision: String,
-    },
-    NonGit {
-        grammar_id: String,
-    },
+    GitUpdated { revision: String },
+    NonGit,
 }
 
 fn fetch_grammar(grammar: GrammarConfiguration) -> Result<FetchStatus> {
@@ -252,7 +246,9 @@ fn fetch_grammar(grammar: GrammarConfiguration) -> Result<FetchStatus> {
         remote, revision, ..
     } = grammar.source
     {
-        let grammar_dir = crate::runtime_dir()
+        let grammar_dir = crate::runtime_dirs()
+            .first()
+            .expect("No runtime directories provided") // guaranteed by post-condition
             .join("grammars")
             .join("sources")
             .join(&grammar.grammar_id);
@@ -283,17 +279,12 @@ fn fetch_grammar(grammar: GrammarConfiguration) -> Result<FetchStatus> {
             )?;
             git(&grammar_dir, ["checkout", &revision])?;
 
-            Ok(FetchStatus::GitUpdated {
-                grammar_id: grammar.grammar_id,
-                revision,
-            })
+            Ok(FetchStatus::GitUpdated { revision })
         } else {
             Ok(FetchStatus::GitUpToDate)
         }
     } else {
-        Ok(FetchStatus::NonGit {
-            grammar_id: grammar.grammar_id,
-        })
+        Ok(FetchStatus::NonGit)
     }
 }
 
@@ -343,14 +334,16 @@ where
 
 enum BuildStatus {
     AlreadyBuilt,
-    Built { grammar_id: String },
+    Built,
 }
 
 fn build_grammar(grammar: GrammarConfiguration, target: Option<&str>) -> Result<BuildStatus> {
     let grammar_dir = if let GrammarSource::Local { path } = &grammar.source {
         PathBuf::from(&path)
     } else {
-        crate::runtime_dir()
+        crate::runtime_dirs()
+            .first()
+            .expect("No runtime directories provided") // guaranteed by post-condition
             .join("grammars")
             .join("sources")
             .join(&grammar.grammar_id)
@@ -401,10 +394,25 @@ fn build_tree_sitter_library(
             None
         }
     };
-    let parser_lib_path = crate::runtime_dir().join("grammars");
+    let parser_lib_path = crate::runtime_dirs()
+        .first()
+        .expect("No runtime directories provided") // guaranteed by post-condition
+        .join("grammars");
     let mut library_path = parser_lib_path.join(&grammar.grammar_id);
     library_path.set_extension(DYLIB_EXTENSION);
 
+    // if we are running inside a buildscript emit cargo metadata
+    // to detect if we are running from a buildscript check some env variables
+    // that cargo only sets for build scripts
+    if std::env::var("OUT_DIR").is_ok() && std::env::var("CARGO").is_ok() {
+        if let Some(scanner_path) = scanner_path.as_ref().and_then(|path| path.to_str()) {
+            println!("cargo:rerun-if-changed={scanner_path}");
+        }
+        if let Some(parser_path) = parser_path.to_str() {
+            println!("cargo:rerun-if-changed={parser_path}");
+        }
+    }
+
     let recompile = needs_recompile(&library_path, &parser_path, &scanner_path)
         .context("Failed to compare source and binary timestamps")?;
 
@@ -425,16 +433,53 @@ fn build_tree_sitter_library(
     for (key, value) in compiler.env() {
         command.env(key, value);
     }
-    command.args(compiler.args());
 
-    if cfg!(all(windows, target_env = "msvc")) {
+    command.args(compiler.args());
+    // used to delay dropping the temporary object file until after the compilation is complete
+    let _path_guard;
+
+    if compiler.is_like_msvc() {
         command
             .args(["/nologo", "/LD", "/I"])
             .arg(header_path)
             .arg("/Od")
-            .arg("/utf-8");
+            .arg("/utf-8")
+            .arg("/std:c11");
         if let Some(scanner_path) = scanner_path.as_ref() {
-            command.arg(scanner_path);
+            if scanner_path.extension() == Some("c".as_ref()) {
+                command.arg(scanner_path);
+            } else {
+                let mut cpp_command = Command::new(compiler.path());
+                cpp_command.current_dir(src_path);
+                for (key, value) in compiler.env() {
+                    cpp_command.env(key, value);
+                }
+                cpp_command.args(compiler.args());
+                let object_file =
+                    library_path.with_file_name(format!("{}_scanner.obj", &grammar.grammar_id));
+                cpp_command
+                    .args(["/nologo", "/LD", "/I"])
+                    .arg(header_path)
+                    .arg("/Od")
+                    .arg("/utf-8")
+                    .arg("/std:c++14")
+                    .arg(format!("/Fo{}", object_file.display()))
+                    .arg("/c")
+                    .arg(scanner_path);
+                let output = cpp_command
+                    .output()
+                    .context("Failed to execute C++ compiler")?;
+
+                if !output.status.success() {
+                    return Err(anyhow!(
+                        "Parser compilation failed.\nStdout: {}\nStderr: {}",
+                        String::from_utf8_lossy(&output.stdout),
+                        String::from_utf8_lossy(&output.stderr)
+                    ));
+                }
+                command.arg(&object_file);
+                _path_guard = TempPath::from_path(object_file);
+            }
         }
 
         command
@@ -446,20 +491,49 @@ fn build_tree_sitter_library(
             .arg("-shared")
             .arg("-fPIC")
             .arg("-fno-exceptions")
-            .arg("-g")
             .arg("-I")
             .arg(header_path)
             .arg("-o")
-            .arg(&library_path)
-            .arg("-O3");
+            .arg(&library_path);
+
         if let Some(scanner_path) = scanner_path.as_ref() {
             if scanner_path.extension() == Some("c".as_ref()) {
-                command.arg("-xc").arg("-std=c99").arg(scanner_path);
+                command.arg("-xc").arg("-std=c11").arg(scanner_path);
             } else {
-                command.arg(scanner_path);
+                let mut cpp_command = Command::new(compiler.path());
+                cpp_command.current_dir(src_path);
+                for (key, value) in compiler.env() {
+                    cpp_command.env(key, value);
+                }
+                cpp_command.args(compiler.args());
+                let object_file =
+                    library_path.with_file_name(format!("{}_scanner.o", &grammar.grammar_id));
+                cpp_command
+                    .arg("-fPIC")
+                    .arg("-fno-exceptions")
+                    .arg("-I")
+                    .arg(header_path)
+                    .arg("-o")
+                    .arg(&object_file)
+                    .arg("-std=c++14")
+                    .arg("-c")
+                    .arg(scanner_path);
+                let output = cpp_command
+                    .output()
+                    .context("Failed to execute C++ compiler")?;
+                if !output.status.success() {
+                    return Err(anyhow!(
+                        "Parser compilation failed.\nStdout: {}\nStderr: {}",
+                        String::from_utf8_lossy(&output.stdout),
+                        String::from_utf8_lossy(&output.stderr)
+                    ));
+                }
+
+                command.arg(&object_file);
+                _path_guard = TempPath::from_path(object_file);
             }
         }
-        command.arg("-xc").arg(parser_path);
+        command.arg("-xc").arg("-std=c11").arg(parser_path);
         if cfg!(all(
             unix,
             not(any(target_os = "macos", target_os = "illumos"))
@@ -479,9 +553,7 @@ fn build_tree_sitter_library(
         ));
     }
 
-    Ok(BuildStatus::Built {
-        grammar_id: grammar.grammar_id,
-    })
+    Ok(BuildStatus::Built)
 }
 
 fn needs_recompile(
@@ -511,9 +583,6 @@ fn mtime(path: &Path) -> Result<SystemTime> {
 /// Gives the contents of a file from a language's `runtime/queries/<lang>`
 /// directory
 pub fn load_runtime_file(language: &str, filename: &str) -> Result<String, std::io::Error> {
-    let path = crate::RUNTIME_DIR
-        .join("queries")
-        .join(language)
-        .join(filename);
-    std::fs::read_to_string(&path)
+    let path = crate::runtime_file(&PathBuf::new().join("queries").join(language).join(filename));
+    std::fs::read_to_string(path)
 }

helix-loader/src/lib.rs

@@ -2,11 +2,12 @@ pub mod config;
 pub mod grammar;
 
 use etcetera::base_strategy::{choose_base_strategy, BaseStrategy};
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 pub const VERSION_AND_GIT_HASH: &str = env!("VERSION_AND_GIT_HASH");
 
-pub static RUNTIME_DIR: once_cell::sync::Lazy<PathBuf> = once_cell::sync::Lazy::new(runtime_dir);
+static RUNTIME_DIRS: once_cell::sync::Lazy<Vec<PathBuf>> =
+    once_cell::sync::Lazy::new(prioritize_runtime_dirs);
 
 static CONFIG_FILE: once_cell::sync::OnceCell<PathBuf> = once_cell::sync::OnceCell::new();
 
@@ -25,31 +26,83 @@ pub fn initialize_config_file(specified_file: Option<PathBuf>) {
     CONFIG_FILE.set(config_file).ok();
 }
 
-pub fn runtime_dir() -> PathBuf {
-    if let Ok(dir) = std::env::var("HELIX_RUNTIME") {
-        return dir.into();
-    }
-
+/// A list of runtime directories from highest to lowest priority
+///
+/// The priority is:
+///
+/// 1. sibling directory to `CARGO_MANIFEST_DIR` (if environment variable is set)
+/// 2. subdirectory of user config directory (always included)
+/// 3. `HELIX_RUNTIME` (if environment variable is set)
+/// 4. subdirectory of path to helix executable (always included)
+///
+/// Postcondition: returns at least two paths (they might not exist).
+fn prioritize_runtime_dirs() -> Vec<PathBuf> {
+    const RT_DIR: &str = "runtime";
+    // Adding higher priority first
+    let mut rt_dirs = Vec::new();
     if let Ok(dir) = std::env::var("CARGO_MANIFEST_DIR") {
         // this is the directory of the crate being run by cargo, we need the workspace path so we take the parent
-        let path = std::path::PathBuf::from(dir).parent().unwrap().join(RT_DIR);
+        let path = PathBuf::from(dir).parent().unwrap().join(RT_DIR);
         log::debug!("runtime dir: {}", path.to_string_lossy());
-        return path;
+        rt_dirs.push(path);
     }
 
-    const RT_DIR: &str = "runtime";
-    let conf_dir = config_dir().join(RT_DIR);
-    if conf_dir.exists() {
-        return conf_dir;
+    let conf_rt_dir = config_dir().join(RT_DIR);
+    rt_dirs.push(conf_rt_dir);
+
+    if let Ok(dir) = std::env::var("HELIX_RUNTIME") {
+        rt_dirs.push(dir.into());
     }
 
     // fallback to location of the executable being run
     // canonicalize the path in case the executable is symlinked
-    std::env::current_exe()
+    let exe_rt_dir = std::env::current_exe()
         .ok()
         .and_then(|path| std::fs::canonicalize(path).ok())
         .and_then(|path| path.parent().map(|path| path.to_path_buf().join(RT_DIR)))
-        .unwrap()
+        .unwrap();
+    rt_dirs.push(exe_rt_dir);
+    rt_dirs
+}
+
+/// Runtime directories ordered from highest to lowest priority
+///
+/// All directories should be checked when looking for files.
+///
+/// Postcondition: returns at least one path (it might not exist).
+pub fn runtime_dirs() -> &'static [PathBuf] {
+    &RUNTIME_DIRS
+}
+
+/// Find file with path relative to runtime directory
+///
+/// `rel_path` should be the relative path from within the `runtime/` directory.
+/// The valid runtime directories are searched in priority order and the first
+/// file found to exist is returned, otherwise None.
+fn find_runtime_file(rel_path: &Path) -> Option<PathBuf> {
+    RUNTIME_DIRS.iter().find_map(|rt_dir| {
+        let path = rt_dir.join(rel_path);
+        if path.exists() {
+            Some(path)
+        } else {
+            None
+        }
+    })
+}
+
+/// Find file with path relative to runtime directory
+///
+/// `rel_path` should be the relative path from within the `runtime/` directory.
+/// The valid runtime directories are searched in priority order and the first
+/// file found to exist is returned, otherwise the path to the final attempt
+/// that failed.
+pub fn runtime_file(rel_path: &Path) -> PathBuf {
+    find_runtime_file(rel_path).unwrap_or_else(|| {
+        RUNTIME_DIRS
+            .last()
+            .map(|dir| dir.join(rel_path))
+            .unwrap_or_default()
+    })
 }
 
 pub fn config_dir() -> PathBuf {
@@ -60,15 +113,6 @@ pub fn config_dir() -> PathBuf {
     path
 }
 
-pub fn local_config_dirs() -> Vec<PathBuf> {
-    let directories = find_local_config_dirs()
-        .into_iter()
-        .map(|path| path.join(".helix"))
-        .collect();
-    log::debug!("Located configuration folders: {:?}", directories);
-    directories
-}
-
 pub fn cache_dir() -> PathBuf {
     // TODO: allow env var override
     let strategy = choose_base_strategy().expect("Unable to find the config directory!");
@@ -84,6 +128,10 @@ pub fn config_file() -> PathBuf {
         .unwrap_or_else(|| config_dir().join("config.toml"))
 }
 
+pub fn workspace_config_file() -> PathBuf {
+    find_workspace().0.join(".helix").join("config.toml")
+}
+
 pub fn lang_config_file() -> PathBuf {
     config_dir().join("languages.toml")
 }
@@ -92,22 +140,6 @@ pub fn log_file() -> PathBuf {
     cache_dir().join("helix.log")
 }
 
-pub fn find_local_config_dirs() -> Vec<PathBuf> {
-    let current_dir = std::env::current_dir().expect("unable to determine current directory");
-    let mut directories = Vec::new();
-
-    for ancestor in current_dir.ancestors() {
-        if ancestor.join(".git").exists() {
-            directories.push(ancestor.to_path_buf());
-            // Don't go higher than repo if we're in one
-            break;
-        } else if ancestor.join(".helix").is_dir() {
-            directories.push(ancestor.to_path_buf());
-        }
-    }
-    directories
-}
-
 /// Merge two TOML documents, merging values from `right` onto `left`
 ///
 /// When an array exists in both `left` and `right`, `right`'s array is
@@ -177,8 +209,28 @@ pub fn merge_toml_values(left: toml::Value, right: toml::Value, merge_depth: usi
     }
 }
 
+/// Finds the current workspace folder.
+/// Used as a ceiling dir for LSP root resolution, the filepicker and potentially as a future filewatching root
+///
+/// This function starts searching the FS upward from the CWD
+/// and returns the first directory that contains either `.git` or `.helix`.
+/// If no workspace was found returns (CWD, true).
+/// Otherwise (workspace, false) is returned
+pub fn find_workspace() -> (PathBuf, bool) {
+    let current_dir = std::env::current_dir().expect("unable to determine current directory");
+    for ancestor in current_dir.ancestors() {
+        if ancestor.join(".git").exists() || ancestor.join(".helix").exists() {
+            return (ancestor.to_owned(), false);
+        }
+    }
+
+    (current_dir, true)
+}
+
 #[cfg(test)]
 mod merge_toml_tests {
+    use std::str;
+
     use super::merge_toml_values;
     use toml::Value;
 
@@ -191,8 +243,9 @@ mod merge_toml_tests {
         indent = { tab-width = 4, unit = "    ", test = "aaa" }
         "#;
 
-        let base: Value = toml::from_slice(include_bytes!("../../languages.toml"))
-            .expect("Couldn't parse built-in languages config");
+        let base = include_bytes!("../../languages.toml");
+        let base = str::from_utf8(base).expect("Couldn't parse built-in languages config");
+        let base: Value = toml::from_str(base).expect("Couldn't parse built-in languages config");
         let user: Value = toml::from_str(USER).unwrap();
 
         let merged = merge_toml_values(base, user, 3);
@@ -224,8 +277,9 @@ mod merge_toml_tests {
         language-server = { command = "deno", args = ["lsp"] }
         "#;
 
-        let base: Value = toml::from_slice(include_bytes!("../../languages.toml"))
-            .expect("Couldn't parse built-in languages config");
+        let base = include_bytes!("../../languages.toml");
+        let base = str::from_utf8(base).expect("Couldn't parse built-in languages config");
+        let base: Value = toml::from_str(base).expect("Couldn't parse built-in languages config");
         let user: Value = toml::from_str(USER).unwrap();
 
         let merged = merge_toml_values(base, user, 3);

helix-lsp/Cargo.toml

@@ -14,15 +14,17 @@ homepage = "https://helix-editor.com"
 [dependencies]
 helix-core = { version = "0.6", path = "../helix-core" }
 helix-loader = { version = "0.6", path = "../helix-loader" }
+helix-parsec = { version = "0.6", path = "../helix-parsec" }
 
 anyhow = "1.0"
 futures-executor = "0.3"
 futures-util = { version = "0.3", features = ["std", "async-await"], default-features = false }
 log = "0.4"
-lsp-types = { version = "0.93", features = ["proposed"] }
+lsp-types = { version = "0.94" }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 thiserror = "1.0"
-tokio = { version = "1.23", features = ["rt", "rt-multi-thread", "io-util", "io-std", "time", "process", "macros", "fs", "parking_lot", "sync"] }
-tokio-stream = "0.1.11"
-which = "4.2"
+tokio = { version = "1.28", features = ["rt", "rt-multi-thread", "io-util", "io-std", "time", "process", "macros", "fs", "parking_lot", "sync"] }
+tokio-stream = "0.1.14"
+which = "4.4"
+parking_lot = "0.12.1"

helix-lsp/src/client.rs

@@ -1,21 +1,26 @@
 use crate::{
-    jsonrpc,
+    find_lsp_workspace, jsonrpc,
     transport::{Payload, Transport},
     Call, Error, OffsetEncoding, Result,
 };
 
-use helix_core::{find_root, ChangeSet, Rope};
+use helix_core::{find_workspace, path, ChangeSet, Rope};
 use helix_loader::{self, VERSION_AND_GIT_HASH};
+use lsp::{
+    notification::DidChangeWorkspaceFolders, DidChangeWorkspaceFoldersParams, OneOf,
+    PositionEncodingKind, WorkspaceFolder, WorkspaceFoldersChangeEvent,
+};
 use lsp_types as lsp;
+use parking_lot::Mutex;
 use serde::Deserialize;
 use serde_json::Value;
-use std::collections::HashMap;
 use std::future::Future;
 use std::process::Stdio;
 use std::sync::{
     atomic::{AtomicU64, Ordering},
     Arc,
 };
+use std::{collections::HashMap, path::PathBuf};
 use tokio::{
     io::{BufReader, BufWriter},
     process::{Child, Command},
@@ -25,6 +30,17 @@ use tokio::{
     },
 };
 
+fn workspace_for_uri(uri: lsp::Url) -> WorkspaceFolder {
+    lsp::WorkspaceFolder {
+        name: uri
+            .path_segments()
+            .and_then(|segments| segments.last())
+            .map(|basename| basename.to_string())
+            .unwrap_or_default(),
+        uri,
+    }
+}
+
 #[derive(Debug)]
 pub struct Client {
     id: usize,
@@ -32,15 +48,124 @@ pub struct Client {
     server_tx: UnboundedSender<Payload>,
     request_counter: AtomicU64,
     pub(crate) capabilities: OnceCell<lsp::ServerCapabilities>,
-    offset_encoding: OffsetEncoding,
     config: Option<Value>,
     root_path: std::path::PathBuf,
     root_uri: Option<lsp::Url>,
-    workspace_folders: Vec<lsp::WorkspaceFolder>,
+    workspace_folders: Mutex<Vec<lsp::WorkspaceFolder>>,
+    initialize_notify: Arc<Notify>,
+    /// workspace folders added while the server is still initializing
     req_timeout: u64,
 }
 
 impl Client {
+    pub fn try_add_doc(
+        self: &Arc<Self>,
+        root_markers: &[String],
+        manual_roots: &[PathBuf],
+        doc_path: Option<&std::path::PathBuf>,
+        may_support_workspace: bool,
+    ) -> bool {
+        let (workspace, workspace_is_cwd) = find_workspace();
+        let workspace = path::get_normalized_path(&workspace);
+        let root = find_lsp_workspace(
+            doc_path
+                .and_then(|x| x.parent().and_then(|x| x.to_str()))
+                .unwrap_or("."),
+            root_markers,
+            manual_roots,
+            &workspace,
+            workspace_is_cwd,
+        );
+        let root_uri = root
+            .as_ref()
+            .and_then(|root| lsp::Url::from_file_path(root).ok());
+
+        if self.root_path == root.unwrap_or(workspace)
+            || root_uri.as_ref().map_or(false, |root_uri| {
+                self.workspace_folders
+                    .lock()
+                    .iter()
+                    .any(|workspace| &workspace.uri == root_uri)
+            })
+        {
+            // workspace URI is already registered so we can use this client
+            return true;
+        }
+
+        // this server definitely doesn't support multiple workspace, no need to check capabilities
+        if !may_support_workspace {
+            return false;
+        }
+
+        let Some(capabilities) = self.capabilities.get() else {
+            let client = Arc::clone(self);
+            // initialization hasn't finished yet, deal with this new root later
+            // TODO: In the edgecase that a **new root** is added
+            // for an LSP that **doesn't support workspace_folders** before initaliation is finished
+            // the new roots are ignored.
+            // That particular edgecase would require retroactively spawning new LSP
+            // clients and therefore also require us to retroactively update the corresponding
+            // documents LSP client handle. It's doable but a pretty weird edgecase so let's
+            // wait and see if anyone ever runs into it.
+            tokio::spawn(async move {
+                client.initialize_notify.notified().await;
+                if let Some(workspace_folders_caps) = client
+                    .capabilities()
+                    .workspace
+                    .as_ref()
+                    .and_then(|cap| cap.workspace_folders.as_ref())
+                    .filter(|cap| cap.supported.unwrap_or(false))
+                {
+                    client.add_workspace_folder(
+                        root_uri,
+                        &workspace_folders_caps.change_notifications,
+                    );
+                }
+            });
+            return true;
+        };
+
+        if let Some(workspace_folders_caps) = capabilities
+            .workspace
+            .as_ref()
+            .and_then(|cap| cap.workspace_folders.as_ref())
+            .filter(|cap| cap.supported.unwrap_or(false))
+        {
+            self.add_workspace_folder(root_uri, &workspace_folders_caps.change_notifications);
+            true
+        } else {
+            // the server doesn't support multi workspaces, we need a new client
+            false
+        }
+    }
+
+    fn add_workspace_folder(
+        &self,
+        root_uri: Option<lsp::Url>,
+        change_notifications: &Option<OneOf<bool, String>>,
+    ) {
+        // root_uri is None just means that there isn't really any LSP workspace
+        // associated with this file. For servers that support multiple workspaces
+        // there is just one server so we can always just use that shared instance.
+        // No need to add a new workspace root here as there is no logical root for this file
+        // let the server deal with this
+        let Some(root_uri) = root_uri else {
+            return;
+        };
+
+        // server supports workspace folders, let's add the new root to the list
+        self.workspace_folders
+            .lock()
+            .push(workspace_for_uri(root_uri.clone()));
+        if &Some(OneOf::Left(false)) == change_notifications {
+            // server specifically opted out of DidWorkspaceChange notifications
+            // let's assume the server will request the workspace folders itself
+            // and that we can therefore reuse the client (but are done now)
+            return;
+        }
+        tokio::spawn(self.did_change_workspace(vec![workspace_for_uri(root_uri)], Vec::new()));
+    }
+
     #[allow(clippy::type_complexity)]
     #[allow(clippy::too_many_arguments)]
     pub fn start(
@@ -49,6 +174,7 @@ impl Client {
         config: Option<Value>,
         server_environment: HashMap<String, String>,
         root_markers: &[String],
+        manual_roots: &[PathBuf],
         id: usize,
         req_timeout: u64,
         doc_path: Option<&std::path::PathBuf>,
@@ -75,27 +201,26 @@ impl Client {
 
         let (server_rx, server_tx, initialize_notify) =
             Transport::start(reader, writer, stderr, id);
-
-        let root_path = find_root(
-            doc_path.and_then(|x| x.parent().and_then(|x| x.to_str())),
+        let (workspace, workspace_is_cwd) = find_workspace();
+        let workspace = path::get_normalized_path(&workspace);
+        let root = find_lsp_workspace(
+            doc_path
+                .and_then(|x| x.parent().and_then(|x| x.to_str()))
+                .unwrap_or("."),
             root_markers,
+            manual_roots,
+            &workspace,
+            workspace_is_cwd,
         );
 
-        let root_uri = lsp::Url::from_file_path(root_path.clone()).ok();
+        // `root_uri` and `workspace_folder` can be empty in case there is no workspace
+        // `root_url` can not, use `workspace` as a fallback
+        let root_path = root.clone().unwrap_or_else(|| workspace.clone());
+        let root_uri = root.and_then(|root| lsp::Url::from_file_path(root).ok());
 
-        // TODO: support multiple workspace folders
         let workspace_folders = root_uri
             .clone()
-            .map(|root| {
-                vec![lsp::WorkspaceFolder {
-                    name: root
-                        .path_segments()
-                        .and_then(|segments| segments.last())
-                        .map(|basename| basename.to_string())
-                        .unwrap_or_default(),
-                    uri: root,
-                }]
-            })
+            .map(|root| vec![workspace_for_uri(root)])
             .unwrap_or_default();
 
         let client = Self {
@@ -104,13 +229,12 @@ impl Client {
             server_tx,
             request_counter: AtomicU64::new(0),
             capabilities: OnceCell::new(),
-            offset_encoding: OffsetEncoding::Utf8,
             config,
             req_timeout,
-
             root_path,
             root_uri,
-            workspace_folders,
+            workspace_folders: Mutex::new(workspace_folders),
+            initialize_notify: initialize_notify.clone(),
         };
 
         Ok((client, server_rx, initialize_notify))
@@ -147,15 +271,29 @@ impl Client {
     }
 
     pub fn offset_encoding(&self) -> OffsetEncoding {
-        self.offset_encoding
+        self.capabilities()
+            .position_encoding
+            .as_ref()
+            .and_then(|encoding| match encoding.as_str() {
+                "utf-8" => Some(OffsetEncoding::Utf8),
+                "utf-16" => Some(OffsetEncoding::Utf16),
+                "utf-32" => Some(OffsetEncoding::Utf32),
+                encoding => {
+                    log::error!("Server provided invalid position encoding {encoding}, defaulting to utf-16");
+                    None
+                },
+            })
+            .unwrap_or_default()
     }
 
     pub fn config(&self) -> Option<&Value> {
         self.config.as_ref()
     }
 
-    pub fn workspace_folders(&self) -> &[lsp::WorkspaceFolder] {
-        &self.workspace_folders
+    pub async fn workspace_folders(
+        &self,
+    ) -> parking_lot::MutexGuard<'_, Vec<lsp::WorkspaceFolder>> {
+        self.workspace_folders.lock()
     }
 
     /// Execute a RPC request on the language server.
@@ -190,7 +328,7 @@ impl Client {
 
             let request = jsonrpc::MethodCall {
                 jsonrpc: Some(jsonrpc::Version::V2),
-                id,
+                id: id.clone(),
                 method: R::METHOD.to_string(),
                 params: Self::value_into_params(params),
             };
@@ -207,7 +345,7 @@ impl Client {
             // TODO: delay other calls until initialize success
             timeout(Duration::from_secs(timeout_secs), rx.recv())
                 .await
-                .map_err(|_| Error::Timeout)? // return Timeout
+                .map_err(|_| Error::Timeout(id))? // return Timeout
                 .ok_or(Error::StreamClosed)?
         }
     }
@@ -275,7 +413,7 @@ impl Client {
     // General messages
     // -------------------------------------------------------------------------------------------
 
-    pub(crate) async fn initialize(&self) -> Result<lsp::InitializeResult> {
+    pub(crate) async fn initialize(&self, enable_snippets: bool) -> Result<lsp::InitializeResult> {
         if let Some(config) = &self.config {
             log::info!("Using custom LSP config: {}", config);
         }
@@ -283,7 +421,7 @@ impl Client {
         #[allow(deprecated)]
         let params = lsp::InitializeParams {
             process_id: Some(std::process::id()),
-            workspace_folders: Some(self.workspace_folders.clone()),
+            workspace_folders: Some(self.workspace_folders.lock().clone()),
             // root_path is obsolete, but some clients like pyright still use it so we specify both.
             // clients will prefer _uri if possible
             root_path: self.root_path.to_str().map(|path| path.to_owned()),
@@ -304,12 +442,26 @@ impl Client {
                     execute_command: Some(lsp::DynamicRegistrationClientCapabilities {
                         dynamic_registration: Some(false),
                     }),
+                    inlay_hint: Some(lsp::InlayHintWorkspaceClientCapabilities {
+                        refresh_support: Some(false),
+                    }),
+                    workspace_edit: Some(lsp::WorkspaceEditClientCapabilities {
+                        document_changes: Some(true),
+                        resource_operations: Some(vec![
+                            lsp::ResourceOperationKind::Create,
+                            lsp::ResourceOperationKind::Rename,
+                            lsp::ResourceOperationKind::Delete,
+                        ]),
+                        failure_handling: Some(lsp::FailureHandlingKind::Abort),
+                        normalizes_line_endings: Some(false),
+                        change_annotation_support: None,
+                    }),
                     ..Default::default()
                 }),
                 text_document: Some(lsp::TextDocumentClientCapabilities {
                     completion: Some(lsp::CompletionClientCapabilities {
                         completion_item: Some(lsp::CompletionItemCapability {
-                            snippet_support: Some(false),
+                            snippet_support: Some(enable_snippets),
                             resolve_support: Some(lsp::CompletionItemCapabilityResolveSupport {
                                 properties: vec![
                                     String::from("documentation"),
@@ -318,6 +470,10 @@ impl Client {
                                 ],
                             }),
                             insert_replace_support: Some(true),
+                            deprecated_support: Some(true),
+                            tag_support: Some(lsp::TagSupport {
+                                value_set: vec![lsp::CompletionItemTag::DEPRECATED],
+                            }),
                             ..Default::default()
                         }),
                         completion_item_kind: Some(lsp::CompletionItemKindCapability {
@@ -344,7 +500,7 @@ impl Client {
                     }),
                     rename: Some(lsp::RenameClientCapabilities {
                         dynamic_registration: Some(false),
-                        prepare_support: Some(false),
+                        prepare_support: Some(true),
                         prepare_support_default_behavior: None,
                         honors_change_annotations: Some(false),
                     }),
@@ -369,14 +525,27 @@ impl Client {
                         ..Default::default()
                     }),
                     publish_diagnostics: Some(lsp::PublishDiagnosticsClientCapabilities {
+                        version_support: Some(true),
                         ..Default::default()
                     }),
+                    inlay_hint: Some(lsp::InlayHintClientCapabilities {
+                        dynamic_registration: Some(false),
+                        resolve_support: None,
+                    }),
                     ..Default::default()
                 }),
                 window: Some(lsp::WindowClientCapabilities {
                     work_done_progress: Some(true),
                     ..Default::default()
                 }),
+                general: Some(lsp::GeneralClientCapabilities {
+                    position_encodings: Some(vec![
+                        PositionEncodingKind::UTF8,
+                        PositionEncodingKind::UTF32,
+                        PositionEncodingKind::UTF16,
+                    ]),
+                    ..Default::default()
+                }),
                 ..Default::default()
             },
             trace: None,
@@ -423,6 +592,16 @@ impl Client {
         )
     }
 
+    pub fn did_change_workspace(
+        &self,
+        added: Vec<WorkspaceFolder>,
+        removed: Vec<WorkspaceFolder>,
+    ) -> impl Future<Output = Result<()>> {
+        self.notify::<DidChangeWorkspaceFolders>(DidChangeWorkspaceFoldersParams {
+            event: WorkspaceFoldersChangeEvent { added, removed },
+        })
+    }
+
     // -------------------------------------------------------------------------------------------
     // Text document
     // -------------------------------------------------------------------------------------------
@@ -466,7 +645,11 @@ impl Client {
         // Calculation is therefore a bunch trickier.
 
         use helix_core::RopeSlice;
-        fn traverse(pos: lsp::Position, text: RopeSlice) -> lsp::Position {
+        fn traverse(
+            pos: lsp::Position,
+            text: RopeSlice,
+            offset_encoding: OffsetEncoding,
+        ) -> lsp::Position {
             let lsp::Position {
                 mut line,
                 mut character,
@@ -483,7 +666,11 @@ impl Client {
                     line += 1;
                     character = 0;
                 } else {
-                    character += ch.len_utf16() as u32;
+                    character += match offset_encoding {
+                        OffsetEncoding::Utf8 => ch.len_utf8() as u32,
+                        OffsetEncoding::Utf16 => ch.len_utf16() as u32,
+                        OffsetEncoding::Utf32 => 1,
+                    };
                 }
             }
             lsp::Position { line, character }
@@ -504,7 +691,7 @@ impl Client {
                 }
                 Delete(_) => {
                     let start = pos_to_lsp_pos(new_text, new_pos, offset_encoding);
-                    let end = traverse(start, old_text.slice(old_pos..old_end));
+                    let end = traverse(start, old_text.slice(old_pos..old_end), offset_encoding);
 
                     // deletion
                     changes.push(lsp::TextDocumentContentChangeEvent {
@@ -521,7 +708,8 @@ impl Client {
                     // a subsequent delete means a replace, consume it
                     let end = if let Some(Delete(len)) = iter.peek() {
                         old_end = old_pos + len;
-                        let end = traverse(start, old_text.slice(old_pos..old_end));
+                        let end =
+                            traverse(start, old_text.slice(old_pos..old_end), offset_encoding);
 
                         iter.next();
 
@@ -577,7 +765,7 @@ impl Client {
                 }]
             }
             lsp::TextDocumentSyncKind::INCREMENTAL => {
-                Self::changeset_to_changes(old_text, new_text, changes, self.offset_encoding)
+                Self::changeset_to_changes(old_text, new_text, changes, self.offset_encoding())
             }
             lsp::TextDocumentSyncKind::NONE => return None,
             kind => unimplemented!("{:?}", kind),
@@ -628,7 +816,7 @@ impl Client {
         Some(self.notify::<lsp::notification::DidSaveTextDocument>(
             lsp::DidSaveTextDocumentParams {
                 text_document,
-                text: include_text.then(|| text.into()),
+                text: include_text.then_some(text.into()),
             },
         ))
     }
@@ -703,6 +891,31 @@ impl Client {
         Some(self.call::<lsp::request::SignatureHelpRequest>(params))
     }
 
+    pub fn text_document_range_inlay_hints(
+        &self,
+        text_document: lsp::TextDocumentIdentifier,
+        range: lsp::Range,
+        work_done_token: Option<lsp::ProgressToken>,
+    ) -> Option<impl Future<Output = Result<Value>>> {
+        let capabilities = self.capabilities.get().unwrap();
+
+        match capabilities.inlay_hint_provider {
+            Some(
+                lsp::OneOf::Left(true)
+                | lsp::OneOf::Right(lsp::InlayHintServerCapabilities::Options(_)),
+            ) => (),
+            _ => return None,
+        }
+
+        let params = lsp::InlayHintParams {
+            text_document,
+            range,
+            work_done_progress_params: lsp::WorkDoneProgressParams { work_done_token },
+        };
+
+        Some(self.call::<lsp::request::InlayHintRequest>(params))
+    }
+
     pub fn text_document_hover(
         &self,
         text_document: lsp::TextDocumentIdentifier,
@@ -886,6 +1099,31 @@ impl Client {
         ))
     }
 
+    pub fn goto_declaration(
+        &self,
+        text_document: lsp::TextDocumentIdentifier,
+        position: lsp::Position,
+        work_done_token: Option<lsp::ProgressToken>,
+    ) -> Option<impl Future<Output = Result<Value>>> {
+        let capabilities = self.capabilities.get().unwrap();
+
+        // Return early if the server does not support goto-declaration.
+        match capabilities.declaration_provider {
+            Some(
+                lsp::DeclarationCapability::Simple(true)
+                | lsp::DeclarationCapability::RegistrationOptions(_)
+                | lsp::DeclarationCapability::Options(_),
+            ) => (),
+            _ => return None,
+        }
+
+        Some(self.goto_request::<lsp::request::GotoDeclaration>(
+            text_document,
+            position,
+            work_done_token,
+        ))
+    }
+
     pub fn goto_type_definition(
         &self,
         text_document: lsp::TextDocumentIdentifier,
@@ -938,6 +1176,7 @@ impl Client {
         &self,
         text_document: lsp::TextDocumentIdentifier,
         position: lsp::Position,
+        include_declaration: bool,
         work_done_token: Option<lsp::ProgressToken>,
     ) -> Option<impl Future<Output = Result<Value>>> {
         let capabilities = self.capabilities.get().unwrap();
@@ -954,7 +1193,7 @@ impl Client {
                 position,
             },
             context: lsp::ReferenceContext {
-                include_declaration: true,
+                include_declaration,
             },
             work_done_progress_params: lsp::WorkDoneProgressParams { work_done_token },
             partial_result_params: lsp::PartialResultParams {
@@ -986,6 +1225,29 @@ impl Client {
         Some(self.call::<lsp::request::DocumentSymbolRequest>(params))
     }
 
+    pub fn prepare_rename(
+        &self,
+        text_document: lsp::TextDocumentIdentifier,
+        position: lsp::Position,
+    ) -> Option<impl Future<Output = Result<Value>>> {
+        let capabilities = self.capabilities.get().unwrap();
+
+        match capabilities.rename_provider {
+            Some(lsp::OneOf::Right(lsp::RenameOptions {
+                prepare_provider: Some(true),
+                ..
+            })) => (),
+            _ => return None,
+        }
+
+        let params = lsp::TextDocumentPositionParams {
+            text_document,
+            position,
+        };
+
+        Some(self.call::<lsp::request::PrepareRenameRequest>(params))
+    }
+
     // empty string to get all symbols
     pub fn workspace_symbols(&self, query: String) -> Option<impl Future<Output = Result<Value>>> {
         let capabilities = self.capabilities.get().unwrap();
@@ -1002,7 +1264,7 @@ impl Client {
             partial_result_params: lsp::PartialResultParams::default(),
         };
 
-        Some(self.call::<lsp::request::WorkspaceSymbol>(params))
+        Some(self.call::<lsp::request::WorkspaceSymbolRequest>(params))
     }
 
     pub fn code_actions(
@@ -1033,20 +1295,23 @@ impl Client {
         Some(self.call::<lsp::request::CodeActionRequest>(params))
     }
 
+    pub fn supports_rename(&self) -> bool {
+        let capabilities = self.capabilities.get().unwrap();
+        matches!(
+            capabilities.rename_provider,
+            Some(lsp::OneOf::Left(true) | lsp::OneOf::Right(_))
+        )
+    }
+
     pub fn rename_symbol(
         &self,
         text_document: lsp::TextDocumentIdentifier,
         position: lsp::Position,
         new_name: String,
     ) -> Option<impl Future<Output = Result<lsp::WorkspaceEdit>>> {
-        let capabilities = self.capabilities.get().unwrap();
-
-        // Return early if the language server does not support renaming.
-        match capabilities.rename_provider {
-            Some(lsp::OneOf::Left(true)) | Some(lsp::OneOf::Right(_)) => (),
-            // None | Some(false)
-            _ => return None,
-        };
+        if !self.supports_rename() {
+            return None;
+        }
 
         let params = lsp::RenameParams {
             text_document_position: lsp::TextDocumentPositionParams {

helix-lsp/src/jsonrpc.rs

@@ -108,6 +108,16 @@ pub enum Id {
     Str(String),
 }
 
+impl std::fmt::Display for Id {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Id::Null => f.write_str("null"),
+            Id::Num(num) => write!(f, "{}", num),
+            Id::Str(string) => f.write_str(string),
+        }
+    }
+}
+
 /// Protocol Version
 #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
 pub enum Version {
@@ -170,6 +180,10 @@ impl Params {
         serde_json::from_value(value)
             .map_err(|err| Error::invalid_params(format!("Invalid params: {}.", err)))
     }
+
+    pub fn is_none(&self) -> bool {
+        self == &Params::None
+    }
 }
 
 impl From<Params> for Value {
@@ -187,7 +201,7 @@ impl From<Params> for Value {
 pub struct MethodCall {
     pub jsonrpc: Option<Version>,
     pub method: String,
-    #[serde(default = "default_params")]
+    #[serde(default = "default_params", skip_serializing_if = "Params::is_none")]
     pub params: Params,
     pub id: Id,
 }
@@ -197,7 +211,7 @@ pub struct MethodCall {
 pub struct Notification {
     pub jsonrpc: Option<Version>,
     pub method: String,
-    #[serde(default = "default_params")]
+    #[serde(default = "default_params", skip_serializing_if = "Params::is_none")]
     pub params: Params,
 }
 
@@ -334,6 +348,33 @@ fn notification_serialize() {
     );
 }
 
+#[test]
+fn serialize_skip_none_params() {
+    use serde_json;
+
+    let m = MethodCall {
+        jsonrpc: Some(Version::V2),
+        method: "shutdown".to_owned(),
+        params: Params::None,
+        id: Id::Num(1),
+    };
+
+    let serialized = serde_json::to_string(&m).unwrap();
+    assert_eq!(
+        serialized,
+        r#"{"jsonrpc":"2.0","method":"shutdown","id":1}"#
+    );
+
+    let n = Notification {
+        jsonrpc: Some(Version::V2),
+        method: "exit".to_owned(),
+        params: Params::None,
+    };
+
+    let serialized = serde_json::to_string(&n).unwrap();
+    assert_eq!(serialized, r#"{"jsonrpc":"2.0","method":"exit"}"#);
+}
+
 #[test]
 fn success_output_deserialize() {
     use serde_json;

helix-lsp/src/lib.rs

@@ -1,5 +1,6 @@
 mod client;
 pub mod jsonrpc;
+pub mod snippet;
 mod transport;
 
 pub use client::Client;
@@ -9,18 +10,21 @@ pub use lsp::{Position, Url};
 pub use lsp_types as lsp;
 
 use futures_util::stream::select_all::SelectAll;
-use helix_core::syntax::{LanguageConfiguration, LanguageServerConfiguration};
+use helix_core::{
+    path,
+    syntax::{LanguageConfiguration, LanguageServerConfiguration},
+};
 use tokio::sync::mpsc::UnboundedReceiver;
 
 use std::{
     collections::{hash_map::Entry, HashMap},
+    path::{Path, PathBuf},
     sync::{
         atomic::{AtomicUsize, Ordering},
         Arc,
     },
 };
 
-use serde::{Deserialize, Serialize};
 use thiserror::Error;
 use tokio_stream::wrappers::UnboundedReceiverStream;
 
@@ -35,8 +39,8 @@ pub enum Error {
     Parse(#[from] serde_json::Error),
     #[error("IO Error: {0}")]
     IO(#[from] std::io::Error),
-    #[error("request timed out")]
-    Timeout,
+    #[error("request {0} timed out")]
+    Timeout(jsonrpc::Id),
     #[error("server closed the stream")]
     StreamClosed,
     #[error("Unhandled")]
@@ -45,18 +49,21 @@ pub enum Error {
     Other(#[from] anyhow::Error),
 }
 
-#[derive(Clone, Copy, Debug, Serialize, Deserialize)]
+#[derive(Clone, Copy, Debug, Default)]
 pub enum OffsetEncoding {
     /// UTF-8 code units aka bytes
-    #[serde(rename = "utf-8")]
     Utf8,
+    /// UTF-32 code units aka chars
+    Utf32,
     /// UTF-16 code units
-    #[serde(rename = "utf-16")]
+    #[default]
     Utf16,
 }
 
 pub mod util {
     use super::*;
+    use helix_core::line_ending::{line_end_byte_index, line_end_char_index};
+    use helix_core::{chars, RopeSlice, SmallVec};
     use helix_core::{diagnostic::NumberOrString, Range, Rope, Selection, Tendril, Transaction};
 
     /// Converts a diagnostic in the document to [`lsp::Diagnostic`].
@@ -117,7 +124,7 @@ pub mod util {
 
     /// Converts [`lsp::Position`] to a position in the document.
     ///
-    /// Returns `None` if position exceeds document length or an operation overflows.
+    /// Returns `None` if position.line is out of bounds or an overflow occurs
     pub fn lsp_pos_to_pos(
         doc: &Rope,
         pos: lsp::Position,
@@ -125,25 +132,70 @@ pub mod util {
     ) -> Option<usize> {
         let pos_line = pos.line as usize;
         if pos_line > doc.len_lines() - 1 {
-            return None;
+            // If it extends past the end, truncate it to the end. This is because the
+            // way the LSP describes the range including the last newline is by
+            // specifying a line number after what we would call the last line.
+            log::warn!("LSP position {pos:?} out of range assuming EOF");
+            return Some(doc.len_chars());
         }
 
-        match offset_encoding {
+        // We need to be careful here to fully comply ith the LSP spec.
+        // Two relevant quotes from the spec:
+        //
+        // https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#position
+        // > If the character value is greater than the line length it defaults back
+        // >  to the line length.
+        //
+        // https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocuments
+        // > To ensure that both client and server split the string into the same
+        // > line representation the protocol specifies the following end-of-line sequences:
+        // > ‘\n’, ‘\r\n’ and ‘\r’. Positions are line end character agnostic.
+        // > So you can not specify a position that denotes \r|\n or \n| where | represents the character offset.
+        //
+        // This means that while the line must be in bounds the `character`
+        // must be capped to the end of the line.
+        // Note that the end of the line here is **before** the line terminator
+        // so we must use `line_end_char_index` instead of `doc.line_to_char(pos_line + 1)`
+        //
+        // FIXME: Helix does not fully comply with the LSP spec for line terminators.
+        // The LSP standard requires that line terminators are ['\n', '\r\n', '\r'].
+        // Without the unicode-linebreak feature disabled, the `\r` terminator is not handled by helix.
+        // With the unicode-linebreak feature, helix recognizes multiple extra line break chars
+        // which means that positions will be decoded/encoded incorrectly in their presence
+
+        let line = match offset_encoding {
             OffsetEncoding::Utf8 => {
-                let line = doc.line_to_char(pos_line);
-                let pos = line.checked_add(pos.character as usize)?;
-                if pos <= doc.len_chars() {
-                    Some(pos)
-                } else {
-                    None
-                }
+                let line_start = doc.line_to_byte(pos_line);
+                let line_end = line_end_byte_index(&doc.slice(..), pos_line);
+                line_start..line_end
             }
             OffsetEncoding::Utf16 => {
-                let line = doc.line_to_char(pos_line);
-                let line_start = doc.char_to_utf16_cu(line);
-                let pos = line_start.checked_add(pos.character as usize)?;
-                doc.try_utf16_cu_to_char(pos).ok()
+                // TODO directly translate line index to char-idx
+                // ropey can do this just as easily as utf-8 byte translation
+                // but the functions are just missing.
+                // Translate to char first and then utf-16 as a workaround
+                let line_start = doc.line_to_char(pos_line);
+                let line_end = line_end_char_index(&doc.slice(..), pos_line);
+                doc.char_to_utf16_cu(line_start)..doc.char_to_utf16_cu(line_end)
             }
+            OffsetEncoding::Utf32 => {
+                let line_start = doc.line_to_char(pos_line);
+                let line_end = line_end_char_index(&doc.slice(..), pos_line);
+                line_start..line_end
+            }
+        };
+
+        // The LSP spec demands that the offset is capped to the end of the line
+        let pos = line
+            .start
+            .checked_add(pos.character as usize)
+            .unwrap_or(line.end)
+            .min(line.end);
+
+        match offset_encoding {
+            OffsetEncoding::Utf8 => doc.try_byte_to_char(pos).ok(),
+            OffsetEncoding::Utf16 => doc.try_utf16_cu_to_char(pos).ok(),
+            OffsetEncoding::Utf32 => Some(pos),
         }
     }
 
@@ -158,8 +210,8 @@ pub mod util {
         match offset_encoding {
             OffsetEncoding::Utf8 => {
                 let line = doc.char_to_line(pos);
-                let line_start = doc.line_to_char(line);
-                let col = pos - line_start;
+                let line_start = doc.line_to_byte(line);
+                let col = doc.char_to_byte(pos) - line_start;
 
                 lsp::Position::new(line as u32, col as u32)
             }
@@ -168,6 +220,13 @@ pub mod util {
                 let line_start = doc.char_to_utf16_cu(doc.line_to_char(line));
                 let col = doc.char_to_utf16_cu(pos) - line_start;
 
+                lsp::Position::new(line as u32, col as u32)
+            }
+            OffsetEncoding::Utf32 => {
+                let line = doc.char_to_line(pos);
+                let line_start = doc.line_to_char(line);
+                let col = pos - line_start;
+
                 lsp::Position::new(line as u32, col as u32)
             }
         }
@@ -187,49 +246,247 @@ pub mod util {
 
     pub fn lsp_range_to_range(
         doc: &Rope,
-        range: lsp::Range,
+        mut range: lsp::Range,
         offset_encoding: OffsetEncoding,
     ) -> Option<Range> {
+        // This is sort of an edgecase. It's not clear from the spec how to deal with
+        // ranges where end < start. They don't make much sense but vscode simply caps start to end
+        // and because it's not specified quite a few LS rely on this as a result (for example the TS server)
+        if range.start > range.end {
+            log::error!(
+                "Invalid LSP range start {:?} > end {:?}, using an empty range at the end instead",
+                range.start,
+                range.end
+            );
+            range.start = range.end;
+        }
         let start = lsp_pos_to_pos(doc, range.start, offset_encoding)?;
         let end = lsp_pos_to_pos(doc, range.end, offset_encoding)?;
 
         Some(Range::new(start, end))
     }
 
+    /// If the LS did not provide a range for the completion or the range of the
+    /// primary cursor can not be used for the secondary cursor, this function
+    /// can be used to find the completion range for a cursor
+    fn find_completion_range(text: RopeSlice, replace_mode: bool, cursor: usize) -> (usize, usize) {
+        let start = cursor
+            - text
+                .chars_at(cursor)
+                .reversed()
+                .take_while(|ch| chars::char_is_word(*ch))
+                .count();
+        let mut end = cursor;
+        if replace_mode {
+            end += text
+                .chars_at(cursor)
+                .skip(1)
+                .take_while(|ch| chars::char_is_word(*ch))
+                .count();
+        }
+        (start, end)
+    }
+    fn completion_range(
+        text: RopeSlice,
+        edit_offset: Option<(i128, i128)>,
+        replace_mode: bool,
+        cursor: usize,
+    ) -> Option<(usize, usize)> {
+        let res = match edit_offset {
+            Some((start_offset, end_offset)) => {
+                let start_offset = cursor as i128 + start_offset;
+                if start_offset < 0 {
+                    return None;
+                }
+                let end_offset = cursor as i128 + end_offset;
+                if end_offset > text.len_chars() as i128 {
+                    return None;
+                }
+                (start_offset as usize, end_offset as usize)
+            }
+            None => find_completion_range(text, replace_mode, cursor),
+        };
+        Some(res)
+    }
+
     /// Creates a [Transaction] from the [lsp::TextEdit] in a completion response.
     /// The transaction applies the edit to all cursors.
     pub fn generate_transaction_from_completion_edit(
         doc: &Rope,
         selection: &Selection,
-        edit: lsp::TextEdit,
-        offset_encoding: OffsetEncoding,
+        edit_offset: Option<(i128, i128)>,
+        replace_mode: bool,
+        new_text: String,
     ) -> Transaction {
-        let replacement: Option<Tendril> = if edit.new_text.is_empty() {
+        let replacement: Option<Tendril> = if new_text.is_empty() {
             None
         } else {
-            Some(edit.new_text.into())
+            Some(new_text.into())
         };
 
         let text = doc.slice(..);
-        let primary_cursor = selection.primary().cursor(text);
+        let (removed_start, removed_end) = completion_range(
+            text,
+            edit_offset,
+            replace_mode,
+            selection.primary().cursor(text),
+        )
+        .expect("transaction must be valid for primary selection");
+        let removed_text = text.slice(removed_start..removed_end);
 
-        let start_offset = match lsp_pos_to_pos(doc, edit.range.start, offset_encoding) {
-            Some(start) => start as i128 - primary_cursor as i128,
-            None => return Transaction::new(doc),
-        };
-        let end_offset = match lsp_pos_to_pos(doc, edit.range.end, offset_encoding) {
-            Some(end) => end as i128 - primary_cursor as i128,
-            None => return Transaction::new(doc),
-        };
+        let (transaction, mut selection) = Transaction::change_by_selection_ignore_overlapping(
+            doc,
+            selection,
+            |range| {
+                let cursor = range.cursor(text);
+                completion_range(text, edit_offset, replace_mode, cursor)
+                    .filter(|(start, end)| text.slice(start..end) == removed_text)
+                    .unwrap_or_else(|| find_completion_range(text, replace_mode, cursor))
+            },
+            |_, _| replacement.clone(),
+        );
+        if transaction.changes().is_empty() {
+            return transaction;
+        }
+        selection = selection.map(transaction.changes());
+        transaction.with_selection(selection)
+    }
 
-        Transaction::change_by_selection(doc, selection, |range| {
-            let cursor = range.cursor(text);
-            (
-                (cursor as i128 + start_offset) as usize,
-                (cursor as i128 + end_offset) as usize,
-                replacement.clone(),
-            )
-        })
+    /// Creates a [Transaction] from the [snippet::Snippet] in a completion response.
+    /// The transaction applies the edit to all cursors.
+    #[allow(clippy::too_many_arguments)]
+    pub fn generate_transaction_from_snippet(
+        doc: &Rope,
+        selection: &Selection,
+        edit_offset: Option<(i128, i128)>,
+        replace_mode: bool,
+        snippet: snippet::Snippet,
+        line_ending: &str,
+        include_placeholder: bool,
+        tab_width: usize,
+        indent_width: usize,
+    ) -> Transaction {
+        let text = doc.slice(..);
+
+        let mut off = 0i128;
+        let mut mapped_doc = doc.clone();
+        let mut selection_tabstops: SmallVec<[_; 1]> = SmallVec::new();
+        let (removed_start, removed_end) = completion_range(
+            text,
+            edit_offset,
+            replace_mode,
+            selection.primary().cursor(text),
+        )
+        .expect("transaction must be valid for primary selection");
+        let removed_text = text.slice(removed_start..removed_end);
+
+        let (transaction, selection) = Transaction::change_by_selection_ignore_overlapping(
+            doc,
+            selection,
+            |range| {
+                let cursor = range.cursor(text);
+                completion_range(text, edit_offset, replace_mode, cursor)
+                    .filter(|(start, end)| text.slice(start..end) == removed_text)
+                    .unwrap_or_else(|| find_completion_range(text, replace_mode, cursor))
+            },
+            |replacement_start, replacement_end| {
+                let mapped_replacement_start = (replacement_start as i128 + off) as usize;
+                let mapped_replacement_end = (replacement_end as i128 + off) as usize;
+
+                let line_idx = mapped_doc.char_to_line(mapped_replacement_start);
+                let indent_level = helix_core::indent::indent_level_for_line(
+                    mapped_doc.line(line_idx),
+                    tab_width,
+                    indent_width,
+                ) * indent_width;
+
+                let newline_with_offset = format!(
+                    "{line_ending}{blank:indent_level$}",
+                    line_ending = line_ending,
+                    blank = ""
+                );
+
+                let (replacement, tabstops) =
+                    snippet::render(&snippet, &newline_with_offset, include_placeholder);
+                selection_tabstops.push((mapped_replacement_start, tabstops));
+                mapped_doc.remove(mapped_replacement_start..mapped_replacement_end);
+                mapped_doc.insert(mapped_replacement_start, &replacement);
+                off +=
+                    replacement_start as i128 - replacement_end as i128 + replacement.len() as i128;
+
+                Some(replacement)
+            },
+        );
+
+        let changes = transaction.changes();
+        if changes.is_empty() {
+            return transaction;
+        }
+
+        let mut mapped_selection = SmallVec::with_capacity(selection.len());
+        let mut mapped_primary_idx = 0;
+        let primary_range = selection.primary();
+        for (range, (tabstop_anchor, tabstops)) in selection.into_iter().zip(selection_tabstops) {
+            if range == primary_range {
+                mapped_primary_idx = mapped_selection.len()
+            }
+
+            let range = range.map(changes);
+            let tabstops = tabstops.first().filter(|tabstops| !tabstops.is_empty());
+            let Some(tabstops) = tabstops else{
+                // no tabstop normal mapping
+                mapped_selection.push(range);
+                continue;
+            };
+
+            // expand the selection to cover the tabstop to retain the helix selection semantic
+            // the tabstop closest to the range simply replaces `head` while anchor remains in place
+            // the remaining tabstops receive their own single-width cursor
+            if range.head < range.anchor {
+                let first_tabstop = tabstop_anchor + tabstops[0].1;
+
+                // if selection is forward but was moved to the right it is
+                // contained entirely in the replacement text, just do a point
+                // selection (fallback below)
+                if range.anchor >= first_tabstop {
+                    let range = Range::new(range.anchor, first_tabstop);
+                    mapped_selection.push(range);
+                    let rem_tabstops = tabstops[1..]
+                        .iter()
+                        .map(|tabstop| Range::point(tabstop_anchor + tabstop.1));
+                    mapped_selection.extend(rem_tabstops);
+                    continue;
+                }
+            } else {
+                let last_idx = tabstops.len() - 1;
+                let last_tabstop = tabstop_anchor + tabstops[last_idx].1;
+
+                // if selection is forward but was moved to the right it is
+                // contained entirely in the replacement text, just do a point
+                // selection (fallback below)
+                if range.anchor <= last_tabstop {
+                    // we can't properly compute the the next grapheme
+                    // here because the transaction hasn't been applied yet
+                    // that is not a problem because the range gets grapheme aligned anyway
+                    // tough so just adding one will always cause head to be grapheme
+                    // aligned correctly when applied to the document
+                    let range = Range::new(range.anchor, last_tabstop + 1);
+                    mapped_selection.push(range);
+                    let rem_tabstops = tabstops[..last_idx]
+                        .iter()
+                        .map(|tabstop| Range::point(tabstop_anchor + tabstop.0));
+                    mapped_selection.extend(rem_tabstops);
+                    continue;
+                }
+            };
+
+            let tabstops = tabstops
+                .iter()
+                .map(|tabstop| Range::point(tabstop_anchor + tabstop.0));
+            mapped_selection.extend(tabstops);
+        }
+
+        transaction.with_selection(Selection::new(mapped_selection, mapped_primary_idx))
     }
 
     pub fn generate_transaction_from_edits(
@@ -288,6 +545,7 @@ pub enum MethodCall {
     ApplyWorkspaceEdit(lsp::ApplyWorkspaceEditParams),
     WorkspaceFolders,
     WorkspaceConfiguration(lsp::ConfigurationParams),
+    RegisterCapability(lsp::RegistrationParams),
 }
 
 impl MethodCall {
@@ -307,6 +565,10 @@ impl MethodCall {
                 let params: lsp::ConfigurationParams = params.parse()?;
                 Self::WorkspaceConfiguration(params)
             }
+            lsp::request::RegisterCapability::METHOD => {
+                let params: lsp::RegistrationParams = params.parse()?;
+                Self::RegisterCapability(params)
+            }
             _ => {
                 return Err(Error::Unhandled);
             }
@@ -362,7 +624,7 @@ impl Notification {
 
 #[derive(Debug)]
 pub struct Registry {
-    inner: HashMap<LanguageId, (usize, Arc<Client>)>,
+    inner: HashMap<LanguageId, Vec<(usize, Arc<Client>)>>,
 
     counter: AtomicUsize,
     pub incoming: SelectAll<UnboundedReceiverStream<(usize, Call)>>,
@@ -386,18 +648,24 @@ impl Registry {
     pub fn get_by_id(&self, id: usize) -> Option<&Client> {
         self.inner
             .values()
+            .flatten()
             .find(|(client_id, _)| client_id == &id)
             .map(|(_, client)| client.as_ref())
     }
 
     pub fn remove_by_id(&mut self, id: usize) {
-        self.inner.retain(|_, (client_id, _)| client_id != &id)
+        self.inner.retain(|_, clients| {
+            clients.retain(|&(client_id, _)| client_id != id);
+            !clients.is_empty()
+        })
     }
 
     pub fn restart(
         &mut self,
         language_config: &LanguageConfiguration,
         doc_path: Option<&std::path::PathBuf>,
+        root_dirs: &[PathBuf],
+        enable_snippets: bool,
     ) -> Result<Option<Arc<Client>>> {
         let config = match &language_config.language_server {
             Some(config) => config,
@@ -412,49 +680,78 @@ impl Registry {
                 // initialize a new client
                 let id = self.counter.fetch_add(1, Ordering::Relaxed);
 
-                let NewClientResult(client, incoming) =
-                    start_client(id, language_config, config, doc_path)?;
+                let NewClientResult(client, incoming) = start_client(
+                    id,
+                    language_config,
+                    config,
+                    doc_path,
+                    root_dirs,
+                    enable_snippets,
+                )?;
                 self.incoming.push(UnboundedReceiverStream::new(incoming));
 
-                let (_, old_client) = entry.insert((id, client.clone()));
+                let old_clients = entry.insert(vec![(id, client.clone())]);
 
-                tokio::spawn(async move {
-                    let _ = old_client.force_shutdown().await;
-                });
+                for (_, old_client) in old_clients {
+                    tokio::spawn(async move {
+                        let _ = old_client.force_shutdown().await;
+                    });
+                }
 
                 Ok(Some(client))
             }
         }
     }
 
+    pub fn stop(&mut self, language_config: &LanguageConfiguration) {
+        let scope = language_config.scope.clone();
+
+        if let Some(clients) = self.inner.remove(&scope) {
+            for (_, client) in clients {
+                tokio::spawn(async move {
+                    let _ = client.force_shutdown().await;
+                });
+            }
+        }
+    }
+
     pub fn get(
         &mut self,
         language_config: &LanguageConfiguration,
         doc_path: Option<&std::path::PathBuf>,
+        root_dirs: &[PathBuf],
+        enable_snippets: bool,
     ) -> Result<Option<Arc<Client>>> {
         let config = match &language_config.language_server {
             Some(config) => config,
             None => return Ok(None),
         };
 
-        match self.inner.entry(language_config.scope.clone()) {
-            Entry::Occupied(entry) => Ok(Some(entry.get().1.clone())),
-            Entry::Vacant(entry) => {
-                // initialize a new client
-                let id = self.counter.fetch_add(1, Ordering::Relaxed);
-
-                let NewClientResult(client, incoming) =
-                    start_client(id, language_config, config, doc_path)?;
-                self.incoming.push(UnboundedReceiverStream::new(incoming));
-
-                entry.insert((id, client.clone()));
-                Ok(Some(client))
-            }
+        let clients = self.inner.entry(language_config.scope.clone()).or_default();
+        // check if we already have a client for this documents root that we can reuse
+        if let Some((_, client)) = clients.iter_mut().enumerate().find(|(i, (_, client))| {
+            client.try_add_doc(&language_config.roots, root_dirs, doc_path, *i == 0)
+        }) {
+            return Ok(Some(client.1.clone()));
         }
+        // initialize a new client
+        let id = self.counter.fetch_add(1, Ordering::Relaxed);
+
+        let NewClientResult(client, incoming) = start_client(
+            id,
+            language_config,
+            config,
+            doc_path,
+            root_dirs,
+            enable_snippets,
+        )?;
+        clients.push((id, client.clone()));
+        self.incoming.push(UnboundedReceiverStream::new(incoming));
+        Ok(Some(client))
     }
 
     pub fn iter_clients(&self) -> impl Iterator<Item = &Arc<Client>> {
-        self.inner.values().map(|(_, client)| client)
+        self.inner.values().flatten().map(|(_, client)| client)
     }
 }
 
@@ -545,6 +842,8 @@ fn start_client(
     config: &LanguageConfiguration,
     ls_config: &LanguageServerConfiguration,
     doc_path: Option<&std::path::PathBuf>,
+    root_dirs: &[PathBuf],
+    enable_snippets: bool,
 ) -> Result<NewClientResult> {
     let (client, incoming, initialize_notify) = Client::start(
         &ls_config.command,
@@ -552,6 +851,7 @@ fn start_client(
         config.config.clone(),
         ls_config.environment.clone(),
         &config.roots,
+        config.workspace_lsp_roots.as_deref().unwrap_or(root_dirs),
         id,
         ls_config.timeout,
         doc_path,
@@ -567,7 +867,7 @@ fn start_client(
             .capabilities
             .get_or_try_init(|| {
                 _client
-                    .initialize()
+                    .initialize(enable_snippets)
                     .map_ok(|response| response.capabilities)
             })
             .await;
@@ -589,6 +889,65 @@ fn start_client(
     Ok(NewClientResult(client, incoming))
 }
 
+/// Find an LSP workspace of a file using the following mechanism:
+/// * if the file is outside `workspace` return `None`
+/// * start at `file` and search the file tree upward
+/// * stop the search at the first `root_dirs` entry that contains `file`
+/// * if no `root_dirs` matches `file` stop at workspace
+/// * Returns the top most directory that contains a `root_marker`
+/// * If no root marker and we stopped at a `root_dirs` entry, return the directory we stopped at
+/// * If we stopped at `workspace` instead and `workspace_is_cwd == false` return `None`
+/// * If we stopped at `workspace` instead and `workspace_is_cwd == true` return `workspace`
+pub fn find_lsp_workspace(
+    file: &str,
+    root_markers: &[String],
+    root_dirs: &[PathBuf],
+    workspace: &Path,
+    workspace_is_cwd: bool,
+) -> Option<PathBuf> {
+    let file = std::path::Path::new(file);
+    let mut file = if file.is_absolute() {
+        file.to_path_buf()
+    } else {
+        let current_dir = std::env::current_dir().expect("unable to determine current directory");
+        current_dir.join(file)
+    };
+    file = path::get_normalized_path(&file);
+
+    if !file.starts_with(workspace) {
+        return None;
+    }
+
+    let mut top_marker = None;
+    for ancestor in file.ancestors() {
+        if root_markers
+            .iter()
+            .any(|marker| ancestor.join(marker).exists())
+        {
+            top_marker = Some(ancestor);
+        }
+
+        if root_dirs
+            .iter()
+            .any(|root_dir| path::get_normalized_path(&workspace.join(root_dir)) == ancestor)
+        {
+            // if the worskapce is the cwd do not search any higher for workspaces
+            // but specify
+            return Some(top_marker.unwrap_or(workspace).to_owned());
+        }
+        if ancestor == workspace {
+            // if the workspace is the CWD, let the LSP decide what the workspace
+            // is
+            return top_marker
+                .or_else(|| (!workspace_is_cwd).then_some(workspace))
+                .map(Path::to_owned);
+        }
+    }
+
+    debug_assert!(false, "workspace must be an ancestor of <file>");
+    None
+}
+
 #[cfg(test)]
 mod tests {
     use super::{lsp, util::*, OffsetEncoding};
@@ -606,16 +965,55 @@ mod tests {
         }
 
         test_case!("", (0, 0) => Some(0));
-        test_case!("", (0, 1) => None);
-        test_case!("", (1, 0) => None);
+        test_case!("", (0, 1) => Some(0));
+        test_case!("", (1, 0) => Some(0));
         test_case!("\n\n", (0, 0) => Some(0));
         test_case!("\n\n", (1, 0) => Some(1));
-        test_case!("\n\n", (1, 1) => Some(2));
+        test_case!("\n\n", (1, 1) => Some(1));
         test_case!("\n\n", (2, 0) => Some(2));
-        test_case!("\n\n", (3, 0) => None);
+        test_case!("\n\n", (3, 0) => Some(2));
         test_case!("test\n\n\n\ncase", (4, 3) => Some(11));
         test_case!("test\n\n\n\ncase", (4, 4) => Some(12));
-        test_case!("test\n\n\n\ncase", (4, 5) => None);
-        test_case!("", (u32::MAX, u32::MAX) => None);
+        test_case!("test\n\n\n\ncase", (4, 5) => Some(12));
+        test_case!("", (u32::MAX, u32::MAX) => Some(0));
+    }
+
+    #[test]
+    fn emoji_format_gh_4791() {
+        use lsp_types::{Position, Range, TextEdit};
+
+        let edits = vec![
+            TextEdit {
+                range: Range {
+                    start: Position {
+                        line: 0,
+                        character: 1,
+                    },
+                    end: Position {
+                        line: 1,
+                        character: 0,
+                    },
+                },
+                new_text: "\n  ".to_string(),
+            },
+            TextEdit {
+                range: Range {
+                    start: Position {
+                        line: 1,
+                        character: 7,
+                    },
+                    end: Position {
+                        line: 2,
+                        character: 0,
+                    },
+                },
+                new_text: "\n  ".to_string(),
+            },
+        ];
+
+        let mut source = Rope::from_str("[\n\"🇺🇸\",\n\"🎄\",\n]");
+
+        let transaction = generate_transaction_from_edits(&source, edits, OffsetEncoding::Utf8);
+        assert!(transaction.apply(&mut source));
     }
 }

helix-parsec/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "helix-parsec"
+version = "0.6.0"
+authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
+edition = "2021"
+license = "MPL-2.0"
+description = "Parser combinators for Helix"
+categories = ["editor"]
+repository = "https://github.com/helix-editor/helix"
+homepage = "https://helix-editor.com"
+include = ["src/**/*", "README.md"]
+
+[dependencies]

helix-parsec/src/lib.rs

@@ -0,0 +1,574 @@
+//! Parser-combinator functions
+//!
+//! This module provides parsers and parser combinators which can be used
+//! together to build parsers by functional composition.
+
+// This module implements parser combinators following https://bodil.lol/parser-combinators/.
+// `sym` (trait implementation for `&'static str`), `map`, `pred` (filter), `one_or_more`,
+// `zero_or_more`, as well as the `Parser` trait originate mostly from that post.
+// The remaining parsers and parser combinators are either based on
+// https://github.com/archseer/snippets.nvim/blob/a583da6ef130d2a4888510afd8c4e5ffd62d0dce/lua/snippet/parser.lua#L5-L138
+// or are novel.
+
+// When a parser matches the input successfully, it returns `Ok((next_input, some_value))`
+// where the type of the returned value depends on the parser. If the parser fails to match,
+// it returns `Err(input)`.
+type ParseResult<'a, Output> = Result<(&'a str, Output), &'a str>;
+
+/// A parser or parser-combinator.
+///
+/// Parser-combinators compose multiple parsers together to parse input.
+/// For example, two basic parsers (`&'static str`s) may be combined with
+/// a parser-combinator like [or] to produce a new parser.
+///
+/// ```
+/// use helix_parsec::{or, Parser};
+/// let foo = "foo"; // matches "foo" literally
+/// let bar = "bar"; // matches "bar" literally
+/// let foo_or_bar = or(foo, bar); // matches either "foo" or "bar"
+/// assert_eq!(Ok(("", "foo")), foo_or_bar.parse("foo"));
+/// assert_eq!(Ok(("", "bar")), foo_or_bar.parse("bar"));
+/// assert_eq!(Err("baz"), foo_or_bar.parse("baz"));
+/// ```
+pub trait Parser<'a> {
+    type Output;
+
+    fn parse(&self, input: &'a str) -> ParseResult<'a, Self::Output>;
+}
+
+// Most parser-combinators are written as higher-order functions which take some
+// parser(s) as input and return a new parser: a function that takes input and returns
+// a parse result. The underlying implementation of [Parser::parse] for these functions
+// is simply application.
+#[doc(hidden)]
+impl<'a, F, T> Parser<'a> for F
+where
+    F: Fn(&'a str) -> ParseResult<T>,
+{
+    type Output = T;
+
+    fn parse(&self, input: &'a str) -> ParseResult<'a, Self::Output> {
+        self(input)
+    }
+}
+
+/// A parser which matches the string literal exactly.
+///
+/// This parser succeeds if the next characters in the input are equal to the given
+/// string literal.
+///
+/// Note that [str::parse] interferes with calling [Parser::parse] on string literals
+/// directly; this trait implementation works when used within any parser combinator
+/// but does not work on its own. To call [Parser::parse] on a parser for a string
+/// literal, use the [token] parser.
+///
+/// # Examples
+///
+/// ```
+/// use helix_parsec::{or, Parser};
+/// let parser = or("foo", "bar");
+/// assert_eq!(Ok(("", "foo")), parser.parse("foo"));
+/// assert_eq!(Ok(("", "bar")), parser.parse("bar"));
+/// assert_eq!(Err("baz"), parser.parse("baz"));
+/// ```
+impl<'a> Parser<'a> for &'static str {
+    type Output = &'a str;
+
+    fn parse(&self, input: &'a str) -> ParseResult<'a, Self::Output> {
+        match input.get(0..self.len()) {
+            Some(actual) if actual == *self => Ok((&input[self.len()..], &input[0..self.len()])),
+            _ => Err(input),
+        }
+    }
+}
+
+// Parsers
+
+/// A parser which matches the given string literally.
+///
+/// This function is a convenience for interpreting string literals as parsers
+/// and is only necessary to avoid conflict with [str::parse]. See the documentation
+/// for the `&'static str` implementation of [Parser].
+///
+/// # Examples
+///
+/// ```
+/// use helix_parsec::{token, Parser};
+/// let parser = token("foo");
+/// assert_eq!(Ok(("", "foo")), parser.parse("foo"));
+/// assert_eq!(Err("bar"), parser.parse("bar"));
+/// ```
+pub fn token<'a>(literal: &'static str) -> impl Parser<'a, Output = &'a str> {
+    literal
+}
+
+/// A parser which matches all values until the specified pattern is found.
+///
+/// If the pattern is not found, this parser does not match. The input up to the
+/// character which returns `true` is returned but not that character itself.
+///
+/// If the pattern function returns true on the first input character, this
+/// parser fails.
+///
+/// # Examples
+///
+/// ```
+/// use helix_parsec::{take_until, Parser};
+/// let parser = take_until(|c| c == '.');
+/// assert_eq!(Ok((".bar", "foo")), parser.parse("foo.bar"));
+/// assert_eq!(Err(".foo"), parser.parse(".foo"));
+/// assert_eq!(Err("foo"), parser.parse("foo"));
+/// ```
+pub fn take_until<'a, F>(pattern: F) -> impl Parser<'a, Output = &'a str>
+where
+    F: Fn(char) -> bool,
+{
+    move |input: &'a str| match input.find(&pattern) {
+        Some(index) if index != 0 => Ok((&input[index..], &input[0..index])),
+        _ => Err(input),
+    }
+}
+
+/// A parser which matches all values until the specified pattern no longer match.
+///
+/// This parser only ever fails if the input has a length of zero.
+///
+/// # Examples
+///
+/// ```
+/// use helix_parsec::{take_while, Parser};
+/// let parser = take_while(|c| c == '1');
+/// assert_eq!(Ok(("2", "11")), parser.parse("112"));
+/// assert_eq!(Err("22"), parser.parse("22"));
+/// ```
+pub fn take_while<'a, F>(pattern: F) -> impl Parser<'a, Output = &'a str>
+where
+    F: Fn(char) -> bool,
+{
+    move |input: &'a str| match input
+        .char_indices()
+        .take_while(|(_p, c)| pattern(*c))
+        .last()
+    {
+        Some((index, c)) => {
+            let index = index + c.len_utf8();
+            Ok((&input[index..], &input[0..index]))
+        }
+        _ => Err(input),
+    }
+}
+
+// Variadic parser combinators
+
+/// A parser combinator which matches a sequence of parsers in an all-or-nothing fashion.
+///
+/// The returned value is a tuple containing the outputs of all parsers in order. Each
+/// parser in the sequence may be typed differently.
+///
+/// # Examples
+///
+/// ```
+/// use helix_parsec::{seq, Parser};
+/// let parser = seq!("<", "a", ">");
+/// assert_eq!(Ok(("", ("<", "a", ">"))), parser.parse("<a>"));
+/// assert_eq!(Err("<b>"), parser.parse("<b>"));
+/// ```
+#[macro_export]
+macro_rules! seq {
+    ($($parsers: expr),+ $(,)?) => {
+        ($($parsers),+)
+    }
+}
+
+// Seq is implemented using trait-implementations of Parser for various size tuples.
+// This allows sequences to be typed heterogeneously.
+macro_rules! seq_impl {
+    ($($parser:ident),+) => {
+        #[allow(non_snake_case)]
+        impl<'a, $($parser),+> Parser<'a> for ($($parser),+)
+        where
+            $($parser: Parser<'a>),+
+        {
+            type Output = ($($parser::Output),+);
+
+            fn parse(&self, input: &'a str) -> ParseResult<'a, Self::Output> {
+                let ($($parser),+) = self;
+                seq_body_impl!(input, input, $($parser),+ ; )
+            }
+        }
+    }
+}
+
+macro_rules! seq_body_impl {
+    ($input:expr, $next_input:expr, $head:ident, $($tail:ident),+ ; $(,)? $($acc:ident),*) => {
+        match $head.parse($next_input) {
+            Ok((next_input, $head)) => seq_body_impl!($input, next_input, $($tail),+ ; $($acc),*, $head),
+            Err(_) => Err($input),
+        }
+    };
+    ($input:expr, $next_input:expr, $last:ident ; $(,)? $($acc:ident),*) => {
+        match $last.parse($next_input) {
+            Ok((next_input, last)) => Ok((next_input, ($($acc),+, last))),
+            Err(_) => Err($input),
+        }
+    }
+}
+
+seq_impl!(A, B);
+seq_impl!(A, B, C);
+seq_impl!(A, B, C, D);
+seq_impl!(A, B, C, D, E);
+seq_impl!(A, B, C, D, E, F);
+seq_impl!(A, B, C, D, E, F, G);
+seq_impl!(A, B, C, D, E, F, G, H);
+seq_impl!(A, B, C, D, E, F, G, H, I);
+seq_impl!(A, B, C, D, E, F, G, H, I, J);
+
+/// A parser combinator which chooses the first of the input parsers which matches
+/// successfully.
+///
+/// All input parsers must have the same output type. This is a variadic form for [or].
+///
+/// # Examples
+///
+/// ```
+/// use helix_parsec::{choice, or, Parser};
+/// let parser = choice!("foo", "bar", "baz");
+/// assert_eq!(Ok(("", "foo")), parser.parse("foo"));
+/// assert_eq!(Ok(("", "bar")), parser.parse("bar"));
+/// assert_eq!(Err("quiz"), parser.parse("quiz"));
+/// ```
+#[macro_export]
+macro_rules! choice {
+    ($parser: expr $(,)?) => {
+        $parser
+    };
+    ($parser: expr, $($rest: expr),+ $(,)?) => {
+        or($parser, choice!($($rest),+))
+    }
+}
+
+// Ordinary parser combinators
+
+/// A parser combinator which takes a parser as input and maps the output using the
+/// given transformation function.
+///
+/// This corresponds to [Result::map]. The value is only mapped if the input parser
+/// matches against input.
+///
+/// # Examples
+///
+/// ```
+/// use helix_parsec::{map, Parser};
+/// let parser = map("123", |s| s.parse::<i32>().unwrap());
+/// assert_eq!(Ok(("", 123)), parser.parse("123"));
+/// assert_eq!(Err("abc"), parser.parse("abc"));
+/// ```
+pub fn map<'a, P, F, T>(parser: P, map_fn: F) -> impl Parser<'a, Output = T>
+where
+    P: Parser<'a>,
+    F: Fn(P::Output) -> T,
+{
+    move |input| {
+        parser
+            .parse(input)
+            .map(|(next_input, result)| (next_input, map_fn(result)))
+    }
+}
+
+/// A parser combinator which succeeds if the given parser matches the input and
+/// the given `filter_map_fn` returns `Some`.
+///
+/// # Examples
+///
+/// ```
+/// use helix_parsec::{filter_map, take_until, Parser};
+/// let parser = filter_map(take_until(|c| c == '.'), |s| s.parse::<i32>().ok());
+/// assert_eq!(Ok((".456", 123)), parser.parse("123.456"));
+/// assert_eq!(Err("abc.def"), parser.parse("abc.def"));
+/// ```
+pub fn filter_map<'a, P, F, T>(parser: P, filter_map_fn: F) -> impl Parser<'a, Output = T>
+where
+    P: Parser<'a>,
+    F: Fn(P::Output) -> Option<T>,
+{
+    move |input| match parser.parse(input) {
+        Ok((next_input, value)) => match filter_map_fn(value) {
+            Some(value) => Ok((next_input, value)),
+            None => Err(input),
+        },
+        Err(_) => Err(input),
+    }
+}
+
+/// A parser combinator which succeeds if the first given parser matches the input and
+/// the second given parse also matches.
+///
+/// # Examples
+///
+/// ```
+/// use helix_parsec::{reparse_as, take_until, one_or_more, Parser};
+/// let parser = reparse_as(take_until(|c| c == '/'), one_or_more("a"));
+/// assert_eq!(Ok(("/bb", vec!["a", "a"])), parser.parse("aa/bb"));
+/// ```
+pub fn reparse_as<'a, P1, P2, T>(parser1: P1, parser2: P2) -> impl Parser<'a, Output = T>
+where
+    P1: Parser<'a, Output = &'a str>,
+    P2: Parser<'a, Output = T>,
+{
+    filter_map(parser1, move |str| {
+        parser2.parse(str).map(|(_, value)| value).ok()
+    })
+}
+
+/// A parser combinator which only matches the input when the predicate function
+/// returns true.
+///
+/// # Examples
+///
+/// ```
+/// use helix_parsec::{filter, take_until, Parser};
+/// let parser = filter(take_until(|c| c == '.'), |s| s == &"123");
+/// assert_eq!(Ok((".456", "123")), parser.parse("123.456"));
+/// assert_eq!(Err("456.123"), parser.parse("456.123"));
+/// ```
+pub fn filter<'a, P, F, T>(parser: P, pred_fn: F) -> impl Parser<'a, Output = T>
+where
+    P: Parser<'a, Output = T>,
+    F: Fn(&P::Output) -> bool,
+{
+    move |input| {
+        if let Ok((next_input, value)) = parser.parse(input) {
+            if pred_fn(&value) {
+                return Ok((next_input, value));
+            }
+        }
+        Err(input)
+    }
+}
+
+/// A parser combinator which matches either of the input parsers.
+///
+/// Both parsers must have the same output type. For a variadic form which
+/// can take any number of parsers, use `choice!`.
+///
+/// # Examples
+///
+/// ```
+/// use helix_parsec::{or, Parser};
+/// let parser = or("foo", "bar");
+/// assert_eq!(Ok(("", "foo")), parser.parse("foo"));
+/// assert_eq!(Ok(("", "bar")), parser.parse("bar"));
+/// assert_eq!(Err("baz"), parser.parse("baz"));
+/// ```
+pub fn or<'a, P1, P2, T>(parser1: P1, parser2: P2) -> impl Parser<'a, Output = T>
+where
+    P1: Parser<'a, Output = T>,
+    P2: Parser<'a, Output = T>,
+{
+    move |input| match parser1.parse(input) {
+        ok @ Ok(_) => ok,
+        Err(_) => parser2.parse(input),
+    }
+}
+
+/// A parser combinator which attempts to match the given parser, returning a
+/// `None` output value if the parser does not match.
+///
+/// The parser produced with this combinator always succeeds. If the given parser
+/// succeeds, `Some(value)` is returned where `value` is the output of the given
+/// parser. Otherwise, `None`.
+///
+/// # Examples
+///
+/// ```
+/// use helix_parsec::{optional, Parser};
+/// let parser = optional("foo");
+/// assert_eq!(Ok(("bar", Some("foo"))), parser.parse("foobar"));
+/// assert_eq!(Ok(("bar", None)), parser.parse("bar"));
+/// ```
+pub fn optional<'a, P, T>(parser: P) -> impl Parser<'a, Output = Option<T>>
+where
+    P: Parser<'a, Output = T>,
+{
+    move |input| match parser.parse(input) {
+        Ok((next_input, value)) => Ok((next_input, Some(value))),
+        Err(_) => Ok((input, None)),
+    }
+}
+
+/// A parser combinator which runs the given parsers in sequence and returns the
+/// value of `left` if both are matched.
+///
+/// This is useful for two-element sequences in which you only want the output
+/// value of the `left` parser.
+///
+/// # Examples
+///
+/// ```
+/// use helix_parsec::{left, Parser};
+/// let parser = left("foo", "bar");
+/// assert_eq!(Ok(("", "foo")), parser.parse("foobar"));
+/// ```
+pub fn left<'a, L, R, T>(left: L, right: R) -> impl Parser<'a, Output = T>
+where
+    L: Parser<'a, Output = T>,
+    R: Parser<'a>,
+{
+    map(seq!(left, right), |(left_value, _)| left_value)
+}
+
+/// A parser combinator which runs the given parsers in sequence and returns the
+/// value of `right` if both are matched.
+///
+/// This is useful for two-element sequences in which you only want the output
+/// value of the `right` parser.
+///
+/// # Examples
+///
+/// ```
+/// use helix_parsec::{right, Parser};
+/// let parser = right("foo", "bar");
+/// assert_eq!(Ok(("", "bar")), parser.parse("foobar"));
+/// ```
+pub fn right<'a, L, R, T>(left: L, right: R) -> impl Parser<'a, Output = T>
+where
+    L: Parser<'a>,
+    R: Parser<'a, Output = T>,
+{
+    map(seq!(left, right), |(_, right_value)| right_value)
+}
+
+/// A parser combinator which matches the given parser against the input zero or
+/// more times.
+///
+/// This parser always succeeds and returns the empty Vec when it matched zero
+/// times.
+///
+/// # Examples
+///
+/// ```
+/// use helix_parsec::{zero_or_more, Parser};
+/// let parser = zero_or_more("a");
+/// assert_eq!(Ok(("", vec![])), parser.parse(""));
+/// assert_eq!(Ok(("", vec!["a"])), parser.parse("a"));
+/// assert_eq!(Ok(("", vec!["a", "a"])), parser.parse("aa"));
+/// assert_eq!(Ok(("bb", vec![])), parser.parse("bb"));
+/// ```
+pub fn zero_or_more<'a, P, T>(parser: P) -> impl Parser<'a, Output = Vec<T>>
+where
+    P: Parser<'a, Output = T>,
+{
+    let parser = non_empty(parser);
+    move |mut input| {
+        let mut values = Vec::new();
+
+        while let Ok((next_input, value)) = parser.parse(input) {
+            input = next_input;
+            values.push(value);
+        }
+
+        Ok((input, values))
+    }
+}
+
+/// A parser combinator which matches the given parser against the input one or
+/// more times.
+///
+/// This parser combinator acts the same as [zero_or_more] but must match at
+/// least once.
+///
+/// # Examples
+///
+/// ```
+/// use helix_parsec::{one_or_more, Parser};
+/// let parser = one_or_more("a");
+/// assert_eq!(Err(""), parser.parse(""));
+/// assert_eq!(Ok(("", vec!["a"])), parser.parse("a"));
+/// assert_eq!(Ok(("", vec!["a", "a"])), parser.parse("aa"));
+/// assert_eq!(Err("bb"), parser.parse("bb"));
+/// ```
+pub fn one_or_more<'a, P, T>(parser: P) -> impl Parser<'a, Output = Vec<T>>
+where
+    P: Parser<'a, Output = T>,
+{
+    let parser = non_empty(parser);
+    move |mut input| {
+        let mut values = Vec::new();
+
+        match parser.parse(input) {
+            Ok((next_input, value)) => {
+                input = next_input;
+                values.push(value);
+            }
+            Err(err) => return Err(err),
+        }
+
+        while let Ok((next_input, value)) = parser.parse(input) {
+            input = next_input;
+            values.push(value);
+        }
+
+        Ok((input, values))
+    }
+}
+
+/// A parser combinator which matches one or more instances of the given parser
+/// interspersed with the separator parser.
+///
+/// Output values of the separator parser are discarded.
+///
+/// This is typically used to parse function arguments or list items.
+///
+/// # Examples
+///
+/// ```rust
+/// use helix_parsec::{sep, Parser};
+/// let parser = sep("a", ",");
+/// assert_eq!(Ok(("", vec!["a", "a", "a"])), parser.parse("a,a,a"));
+/// ```
+pub fn sep<'a, P, S, T>(parser: P, separator: S) -> impl Parser<'a, Output = Vec<T>>
+where
+    P: Parser<'a, Output = T>,
+    S: Parser<'a>,
+{
+    move |mut input| {
+        let mut values = Vec::new();
+
+        match parser.parse(input) {
+            Ok((next_input, value)) => {
+                input = next_input;
+                values.push(value);
+            }
+            Err(err) => return Err(err),
+        }
+
+        loop {
+            match separator.parse(input) {
+                Ok((next_input, _)) => input = next_input,
+                Err(_) => break,
+            }
+
+            match parser.parse(input) {
+                Ok((next_input, value)) => {
+                    input = next_input;
+                    values.push(value);
+                }
+                Err(_) => break,
+            }
+        }
+
+        Ok((input, values))
+    }
+}
+
+pub fn non_empty<'a, T>(p: impl Parser<'a, Output = T>) -> impl Parser<'a, Output = T> {
+    move |input| {
+        let (new_input, res) = p.parse(input)?;
+        if new_input.len() == input.len() {
+            Err(input)
+        } else {
+            Ok((new_input, res))
+        }
+    }
+}

helix-term/Cargo.toml

@@ -10,11 +10,7 @@ repository = "https://github.com/helix-editor/helix"
 homepage = "https://helix-editor.com"
 include = ["src/**/*", "README.md"]
 default-run = "hx"
-rust-version = "1.57"
-
-[package.metadata.nix]
-build = true
-app = true
+rust-version = "1.65"
 
 [features]
 default = ["git"]
@@ -35,17 +31,17 @@ helix-vcs = { version = "0.6", path = "../helix-vcs" }
 helix-loader = { version = "0.6", path = "../helix-loader" }
 
 anyhow = "1"
-once_cell = "1.16"
+once_cell = "1.17"
 
-which = "4.2"
+which = "4.4"
 
 tokio = { version = "1", features = ["rt", "rt-multi-thread", "io-util", "io-std", "time", "process", "macros", "fs", "parking_lot"] }
 tui = { path = "../helix-tui", package = "helix-tui", default-features = false, features = ["crossterm"] }
-crossterm = { version = "0.25", features = ["event-stream"] }
+crossterm = { version = "0.26", features = ["event-stream"] }
 signal-hook = "0.3"
 tokio-stream = "0.1"
 futures-util = { version = "0.3", features = ["std", "async-await"], default-features = false }
-arc-swap = { version = "1.5.1" }
+arc-swap = { version = "1.6.0" }
 
 # Logging
 fern = "0.6"
@@ -61,22 +57,23 @@ pulldown-cmark = { version = "0.9", default-features = false }
 content_inspector = "0.2.4"
 
 # config
-toml = "0.5"
+toml = "0.7"
 
 serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
 
 # ripgrep for global search
-grep-regex = "0.1.10"
-grep-searcher = "0.1.10"
+grep-regex = "0.1.11"
+grep-searcher = "0.1.11"
 
 [target.'cfg(not(windows))'.dependencies]  # https://github.com/vorner/signal-hook/issues/100
 signal-hook-tokio = { version = "0.3", features = ["futures-v0_3"] }
+libc = "0.2.144"
 
 [build-dependencies]
 helix-loader = { version = "0.6", path = "../helix-loader" }
 
 [dev-dependencies]
 smallvec = "1.10"
-indoc = "1.0.6"
-tempfile = "3.3.0"
+indoc = "2.0.1"
+tempfile = "3.4.0"

helix-term/src/application.rs

@@ -25,31 +25,22 @@ use crate::{
     config::Config,
     job::Jobs,
     keymap::Keymaps,
-    ui::{self, overlay::overlayed},
+    ui::{self, overlay::overlaid},
 };
 
 use log::{debug, error, warn};
 use std::{
-    io::{stdin, stdout, Write},
+    io::{stdin, stdout},
+    path::Path,
     sync::Arc,
     time::{Duration, Instant},
 };
 
 use anyhow::{Context, Error};
 
-use crossterm::{
-    event::{
-        DisableBracketedPaste, DisableFocusChange, DisableMouseCapture, EnableBracketedPaste,
-        EnableFocusChange, EnableMouseCapture, Event as CrosstermEvent,
-    },
-    execute, terminal,
-    tty::IsTty,
-};
+use crossterm::{event::Event as CrosstermEvent, tty::IsTty};
 #[cfg(not(windows))]
-use {
-    signal_hook::{consts::signal, low_level},
-    signal_hook_tokio::Signals,
-};
+use {signal_hook::consts::signal, signal_hook_tokio::Signals};
 #[cfg(windows)]
 type Signals = futures_util::stream::Empty<()>;
 
@@ -62,10 +53,12 @@ use tui::backend::CrosstermBackend;
 use tui::backend::TestBackend;
 
 #[cfg(not(feature = "integration"))]
-type Terminal = tui::terminal::Terminal<CrosstermBackend<std::io::Stdout>>;
+type TerminalBackend = CrosstermBackend<std::io::Stdout>;
 
 #[cfg(feature = "integration")]
-type Terminal = tui::terminal::Terminal<TestBackend>;
+type TerminalBackend = TestBackend;
+
+type Terminal = tui::terminal::Terminal<TerminalBackend>;
 
 pub struct Application {
     compositor: Compositor,
@@ -107,23 +100,6 @@ fn setup_integration_logging() {
         .apply();
 }
 
-fn restore_term() -> Result<(), Error> {
-    let mut stdout = stdout();
-    // reset cursor shape
-    write!(stdout, "\x1B[0 q")?;
-    // Ignore errors on disabling, this might trigger on windows if we call
-    // disable without calling enable previously
-    let _ = execute!(stdout, DisableMouseCapture);
-    execute!(
-        stdout,
-        DisableBracketedPaste,
-        DisableFocusChange,
-        terminal::LeaveAlternateScreen
-    )?;
-    terminal::disable_raw_mode()?;
-    Ok(())
-}
-
 impl Application {
     pub fn new(
         args: Args,
@@ -135,10 +111,9 @@ impl Application {
 
         use helix_view::editor::Action;
 
-        let theme_loader = std::sync::Arc::new(theme::Loader::new(
-            &helix_loader::config_dir(),
-            &helix_loader::runtime_dir(),
-        ));
+        let mut theme_parent_dirs = vec![helix_loader::config_dir()];
+        theme_parent_dirs.extend(helix_loader::runtime_dirs().iter().cloned());
+        let theme_loader = std::sync::Arc::new(theme::Loader::new(&theme_parent_dirs));
 
         let true_color = config.editor.true_color || crate::true_color();
         let theme = config
@@ -159,7 +134,7 @@ impl Application {
         let syn_loader = std::sync::Arc::new(syntax::Loader::new(syn_loader_conf));
 
         #[cfg(not(feature = "integration"))]
-        let backend = CrosstermBackend::new(stdout());
+        let backend = CrosstermBackend::new(stdout(), &config.editor);
 
         #[cfg(feature = "integration")]
         let backend = TestBackend::new(120, 150);
@@ -172,7 +147,7 @@ impl Application {
             area,
             theme_loader.clone(),
             syn_loader.clone(),
-            Box::new(Map::new(Arc::clone(&config), |config: &Config| {
+            Arc::new(Map::new(Arc::clone(&config), |config: &Config| {
                 &config.editor
             })),
         );
@@ -184,7 +159,7 @@ impl Application {
         compositor.push(editor_view);
 
         if args.load_tutor {
-            let path = helix_loader::runtime_dir().join("tutor");
+            let path = helix_loader::runtime_file(Path::new("tutor"));
             editor.open(&path, Action::VerticalSplit)?;
             // Unset path to prevent accidentally saving to the original tutor file.
             doc_mut!(editor).set_path(None)?;
@@ -194,7 +169,7 @@ impl Application {
                 std::env::set_current_dir(first).context("set current dir")?;
                 editor.new_file(Action::VerticalSplit);
                 let picker = ui::file_picker(".".into(), &config.load().editor);
-                compositor.push(Box::new(overlayed(picker)));
+                compositor.push(Box::new(overlaid(picker)));
             } else {
                 let nr_of_files = args.files.len();
                 for (i, (file, pos)) in args.files.into_iter().enumerate() {
@@ -227,7 +202,11 @@ impl Application {
                         doc.set_selection(view_id, pos);
                     }
                 }
-                editor.set_status(format!("Loaded {} files.", nr_of_files));
+                editor.set_status(format!(
+                    "Loaded {} file{}.",
+                    nr_of_files,
+                    if nr_of_files == 1 { "" } else { "s" } // avoid "Loaded 1 files." grammo
+                ));
                 // align the view to center after all files are loaded,
                 // does not affect views without pos since it is at the top
                 let (view, doc) = current!(editor);
@@ -273,10 +252,6 @@ impl Application {
         Ok(app)
     }
 
-    #[cfg(feature = "integration")]
-    async fn render(&mut self) {}
-
-    #[cfg(not(feature = "integration"))]
     async fn render(&mut self) {
         let mut cx = crate::compositor::Context {
             editor: &mut self.editor,
@@ -305,8 +280,10 @@ impl Application {
         let surface = self.terminal.current_buffer_mut();
 
         self.compositor.render(area, surface, &mut cx);
-
         let (pos, kind) = self.compositor.cursor(area, &self.editor);
+        // reset cursor cache
+        self.editor.cursor_cache.set(None);
+
         let pos = pos.map(|pos| (pos.col as u16, pos.row as u16));
         self.terminal.draw(pos, kind).unwrap();
     }
@@ -339,12 +316,12 @@ impl Application {
             tokio::select! {
                 biased;
 
-                Some(event) = input_stream.next() => {
-                    self.handle_terminal_events(event).await;
-                }
                 Some(signal) = self.signals.next() => {
                     self.handle_signals(signal).await;
                 }
+                Some(event) = input_stream.next() => {
+                    self.handle_terminal_events(event).await;
+                }
                 Some(callback) = self.jobs.futures.next() => {
                     self.jobs.handle_callback(&mut self.editor, &mut self.compositor, callback);
                     self.render().await;
@@ -384,6 +361,9 @@ impl Application {
             ConfigEvent::Update(editor_config) => {
                 let mut app_config = (*self.config.load().clone()).clone();
                 app_config.editor = *editor_config;
+                if let Err(err) = self.terminal.reconfigure(app_config.editor.clone().into()) {
+                    self.editor.set_error(err.to_string());
+                };
                 self.config.store(Arc::new(app_config));
             }
         }
@@ -391,36 +371,67 @@ impl Application {
         // Update all the relevant members in the editor after updating
         // the configuration.
         self.editor.refresh_config();
-    }
 
-    /// Refresh theme after config change
-    fn refresh_theme(&mut self, config: &Config) {
-        if let Some(theme) = config.theme.clone() {
-            let true_color = self.true_color();
-            match self.theme_loader.load(&theme) {
-                Ok(theme) => {
-                    if true_color || theme.is_16_color() {
-                        self.editor.set_theme(theme);
-                    } else {
-                        self.editor
-                            .set_error("theme requires truecolor support, which is not available");
-                    }
-                }
-                Err(err) => {
-                    let err_string = format!("failed to load theme `{}` - {}", theme, err);
-                    self.editor.set_error(err_string);
-                }
-            }
+        // reset view position in case softwrap was enabled/disabled
+        let scrolloff = self.editor.config().scrolloff;
+        for (view, _) in self.editor.tree.views_mut() {
+            let doc = &self.editor.documents[&view.doc];
+            view.ensure_cursor_in_view(doc, scrolloff)
         }
     }
 
-    fn refresh_config(&mut self) {
-        match Config::load_default() {
-            Ok(config) => {
-                self.refresh_theme(&config);
+    /// refresh language config after config change
+    fn refresh_language_config(&mut self) -> Result<(), Error> {
+        let syntax_config = helix_core::config::user_syntax_loader()
+            .map_err(|err| anyhow::anyhow!("Failed to load language config: {}", err))?;
 
-                // Store new config
-                self.config.store(Arc::new(config));
+        self.syn_loader = std::sync::Arc::new(syntax::Loader::new(syntax_config));
+        self.editor.syn_loader = self.syn_loader.clone();
+        for document in self.editor.documents.values_mut() {
+            document.detect_language(self.syn_loader.clone());
+        }
+
+        Ok(())
+    }
+
+    /// Refresh theme after config change
+    fn refresh_theme(&mut self, config: &Config) -> Result<(), Error> {
+        let true_color = config.editor.true_color || crate::true_color();
+        let theme = config
+            .theme
+            .as_ref()
+            .and_then(|theme| {
+                self.theme_loader
+                    .load(theme)
+                    .map_err(|e| {
+                        log::warn!("failed to load theme `{}` - {}", theme, e);
+                        e
+                    })
+                    .ok()
+                    .filter(|theme| (true_color || theme.is_16_color()))
+            })
+            .unwrap_or_else(|| self.theme_loader.default_theme(true_color));
+
+        self.editor.set_theme(theme);
+        Ok(())
+    }
+
+    fn refresh_config(&mut self) {
+        let mut refresh_config = || -> Result<(), Error> {
+            let default_config = Config::load_default()
+                .map_err(|err| anyhow::anyhow!("Failed to load config: {}", err))?;
+            self.refresh_language_config()?;
+            self.refresh_theme(&default_config)?;
+            self.terminal
+                .reconfigure(default_config.editor.clone().into())?;
+            // Store new config
+            self.config.store(Arc::new(default_config));
+            Ok(())
+        };
+
+        match refresh_config() {
+            Ok(_) => {
+                self.editor.set_status("Config refreshed");
             }
             Err(err) => {
                 self.editor.set_error(err.to_string());
@@ -428,10 +439,6 @@ impl Application {
         }
     }
 
-    fn true_color(&self) -> bool {
-        self.config.load().editor.true_color || crate::true_color()
-    }
-
     #[cfg(windows)]
     // no signal handling available on windows
     pub async fn handle_signals(&mut self, _signal: ()) {}
@@ -440,17 +447,46 @@ impl Application {
     pub async fn handle_signals(&mut self, signal: i32) {
         match signal {
             signal::SIGTSTP => {
-                // restore cursor
-                use helix_view::graphics::CursorKind;
-                self.terminal
-                    .backend_mut()
-                    .show_cursor(CursorKind::Block)
-                    .ok();
-                restore_term().unwrap();
-                low_level::emulate_default_handler(signal::SIGTSTP).unwrap();
+                self.restore_term().unwrap();
+
+                // SAFETY:
+                //
+                // - helix must have permissions to send signals to all processes in its signal
+                //   group, either by already having the requisite permission, or by having the
+                //   user's UID / EUID / SUID match that of the receiving process(es).
+                let res = unsafe {
+                    // A pid of 0 sends the signal to the entire process group, allowing the user to
+                    // regain control of their terminal if the editor was spawned under another process
+                    // (e.g. when running `git commit`).
+                    //
+                    // We have to send SIGSTOP (not SIGTSTP) to the entire process group, because,
+                    // as mentioned above, the terminal will get stuck if `helix` was spawned from
+                    // an external process and that process waits for `helix` to complete. This may
+                    // be an issue with signal-hook-tokio, but the author of signal-hook believes it
+                    // could be a tokio issue instead:
+                    // https://github.com/vorner/signal-hook/issues/132
+                    libc::kill(0, signal::SIGSTOP)
+                };
+
+                if res != 0 {
+                    let err = std::io::Error::last_os_error();
+                    eprintln!("{}", err);
+                    let res = err.raw_os_error().unwrap_or(1);
+                    std::process::exit(res);
+                }
             }
             signal::SIGCONT => {
-                self.claim_term().await.unwrap();
+                // Copy/Paste from same issue from neovim:
+                // https://github.com/neovim/neovim/issues/12322
+                // https://github.com/neovim/neovim/pull/13084
+                for retries in 1..=10 {
+                    match self.claim_term().await {
+                        Ok(()) => break,
+                        Err(err) if retries == 10 => panic!("Failed to claim terminal: {}", err),
+                        Err(_) => continue,
+                    }
+                }
+
                 // redraw the terminal
                 let area = self.terminal.size().expect("couldn't get terminal size");
                 self.compositor.resize(area);
@@ -606,6 +642,11 @@ impl Application {
                 self.compositor
                     .handle_event(&Event::Resize(width, height), &mut cx)
             }
+            // Ignore keyboard release events.
+            CrosstermEvent::Key(crossterm::event::KeyEvent {
+                kind: crossterm::event::KeyEventKind::Release,
+                ..
+            }) => false,
             event => self.compositor.handle_event(&event.into(), &mut cx),
         };
 
@@ -675,8 +716,23 @@ impl Application {
                         }
                     }
                     Notification::PublishDiagnostics(mut params) => {
-                        let path = params.uri.to_file_path().unwrap();
-                        let doc = self.editor.document_by_path_mut(&path);
+                        let path = match params.uri.to_file_path() {
+                            Ok(path) => path,
+                            Err(_) => {
+                                log::error!("Unsupported file URI: {}", params.uri);
+                                return;
+                            }
+                        };
+                        let doc = self.editor.document_by_path_mut(&path).filter(|doc| {
+                            if let Some(version) = params.version {
+                                if version != doc.version() {
+                                    log::info!("Version ({version}) is out of date for {path:?} (expected ({}), dropping PublishDiagnostic notification", doc.version());
+                                    return false;
+                                }
+                            }
+
+                            true
+                        });
 
                         if let Some(doc) = doc {
                             let lang_conf = doc.language_config();
@@ -917,24 +973,32 @@ impl Application {
             Call::MethodCall(helix_lsp::jsonrpc::MethodCall {
                 method, params, id, ..
             }) => {
-                let call = match MethodCall::parse(&method, params) {
-                    Ok(call) => call,
+                let reply = match MethodCall::parse(&method, params) {
                     Err(helix_lsp::Error::Unhandled) => {
-                        error!("Language Server: Method not found {}", method);
-                        return;
+                        error!(
+                            "Language Server: Method {} not found in request {}",
+                            method, id
+                        );
+                        Err(helix_lsp::jsonrpc::Error {
+                            code: helix_lsp::jsonrpc::ErrorCode::MethodNotFound,
+                            message: format!("Method not found: {}", method),
+                            data: None,
+                        })
                     }
                     Err(err) => {
                         log::error!(
-                            "received malformed method call from Language Server: {}: {}",
+                            "Language Server: Received malformed method call {} in request {}: {}",
                             method,
+                            id,
                             err
                         );
-                        return;
+                        Err(helix_lsp::jsonrpc::Error {
+                            code: helix_lsp::jsonrpc::ErrorCode::ParseError,
+                            message: format!("Malformed method call: {}", method),
+                            data: None,
+                        })
                     }
-                };
-
-                let reply = match call {
-                    MethodCall::WorkDoneProgressCreate(params) => {
+                    Ok(MethodCall::WorkDoneProgressCreate(params)) => {
                         self.lsp_progress.create(server_id, params.token);
 
                         let editor_view = self
@@ -948,26 +1012,29 @@ impl Application {
 
                         Ok(serde_json::Value::Null)
                     }
-                    MethodCall::ApplyWorkspaceEdit(params) => {
-                        apply_workspace_edit(
+                    Ok(MethodCall::ApplyWorkspaceEdit(params)) => {
+                        let res = apply_workspace_edit(
                             &mut self.editor,
                             helix_lsp::OffsetEncoding::Utf8,
                             &params.edit,
                         );
 
                         Ok(json!(lsp::ApplyWorkspaceEditResponse {
-                            applied: true,
-                            failure_reason: None,
-                            failed_change: None,
+                            applied: res.is_ok(),
+                            failure_reason: res.as_ref().err().map(|err| err.kind.to_string()),
+                            failed_change: res
+                                .as_ref()
+                                .err()
+                                .map(|err| err.failed_change_idx as u32),
                         }))
                     }
-                    MethodCall::WorkspaceFolders => {
+                    Ok(MethodCall::WorkspaceFolders) => {
                         let language_server =
                             self.editor.language_servers.get_by_id(server_id).unwrap();
 
-                        Ok(json!(language_server.workspace_folders()))
+                        Ok(json!(&*language_server.workspace_folders().await))
                     }
-                    MethodCall::WorkspaceConfiguration(params) => {
+                    Ok(MethodCall::WorkspaceConfiguration(params)) => {
                         let result: Vec<_> = params
                             .items
                             .iter()
@@ -981,8 +1048,7 @@ impl Application {
                                     None => self
                                         .editor
                                         .language_servers
-                                        .get_by_id(server_id)
-                                        .unwrap()
+                                        .get_by_id(server_id)?
                                         .config()?,
                                 };
                                 if let Some(section) = item.section.as_ref() {
@@ -995,6 +1061,17 @@ impl Application {
                             .collect();
                         Ok(json!(result))
                     }
+                    Ok(MethodCall::RegisterCapability(_params)) => {
+                        log::warn!("Ignoring a client/registerCapability request because dynamic capability registration is not enabled. Please report this upstream to the language server");
+                        // Language Servers based on the `vscode-languageserver-node` library often send
+                        // client/registerCapability even though we do not enable dynamic registration
+                        // for any capabilities. We should send a MethodNotFound JSONRPC error in this
+                        // case but that rejects the registration promise in the server which causes an
+                        // exit. So we work around this by ignoring the request and sending back an OK
+                        // response.
+
+                        Ok(serde_json::Value::Null)
+                    }
                 };
 
                 let language_server = match self.editor.language_servers.get_by_id(server_id) {
@@ -1011,24 +1088,19 @@ impl Application {
         }
     }
 
-    async fn claim_term(&mut self) -> Result<(), Error> {
+    async fn claim_term(&mut self) -> std::io::Result<()> {
+        let terminal_config = self.config.load().editor.clone().into();
+        self.terminal.claim(terminal_config)
+    }
+
+    fn restore_term(&mut self) -> std::io::Result<()> {
+        let terminal_config = self.config.load().editor.clone().into();
         use helix_view::graphics::CursorKind;
-        terminal::enable_raw_mode()?;
-        if self.terminal.cursor_kind() == CursorKind::Hidden {
-            self.terminal.backend_mut().hide_cursor().ok();
-        }
-        let mut stdout = stdout();
-        execute!(
-            stdout,
-            terminal::EnterAlternateScreen,
-            EnableBracketedPaste,
-            EnableFocusChange
-        )?;
-        execute!(stdout, terminal::Clear(terminal::ClearType::All))?;
-        if self.config.load().editor.mouse {
-            execute!(stdout, EnableMouseCapture)?;
-        }
-        Ok(())
+        self.terminal
+            .backend_mut()
+            .show_cursor(CursorKind::Block)
+            .ok();
+        self.terminal.restore(terminal_config)
     }
 
     pub async fn run<S>(&mut self, input_stream: &mut S) -> Result<i32, Error>
@@ -1043,7 +1115,7 @@ impl Application {
             // We can't handle errors properly inside this closure.  And it's
             // probably not a good idea to `unwrap()` inside a panic handler.
             // So we just ignore the `Result`.
-            let _ = restore_term();
+            let _ = TerminalBackend::force_restore();
             hook(info);
         }));
 
@@ -1051,13 +1123,7 @@ impl Application {
 
         let close_errs = self.close().await;
 
-        // restore cursor
-        use helix_view::graphics::CursorKind;
-        self.terminal
-            .backend_mut()
-            .show_cursor(CursorKind::Block)
-            .ok();
-        restore_term()?;
+        self.restore_term()?;
 
         for err in close_errs {
             self.editor.exit_code = 1;

helix-term/src/commands/dap.rs

@@ -2,7 +2,7 @@ use super::{Context, Editor};
 use crate::{
     compositor::{self, Compositor},
     job::{Callback, Jobs},
-    ui::{self, overlay::overlayed, FilePicker, Picker, Popup, Prompt, PromptEvent, Text},
+    ui::{self, overlay::overlaid, FilePicker, Picker, Popup, Prompt, PromptEvent, Text},
 };
 use dap::{StackFrame, Thread, ThreadStates};
 use helix_core::syntax::{DebugArgumentValue, DebugConfigCompletion, DebugTemplate};
@@ -12,7 +12,7 @@ use helix_view::editor::Breakpoint;
 
 use serde_json::{to_value, Value};
 use tokio_stream::wrappers::UnboundedReceiverStream;
-use tui::text::Spans;
+use tui::{text::Spans, widgets::Row};
 
 use std::collections::HashMap;
 use std::future::Future;
@@ -25,7 +25,7 @@ use helix_view::handlers::dap::{breakpoints_changed, jump_to_stack_frame, select
 impl ui::menu::Item for StackFrame {
     type Data = ();
 
-    fn label(&self, _data: &Self::Data) -> Spans {
+    fn format(&self, _data: &Self::Data) -> Row {
         self.name.as_str().into() // TODO: include thread_states in the label
     }
 }
@@ -33,7 +33,7 @@ impl ui::menu::Item for StackFrame {
 impl ui::menu::Item for DebugTemplate {
     type Data = ();
 
-    fn label(&self, _data: &Self::Data) -> Spans {
+    fn format(&self, _data: &Self::Data) -> Row {
         self.name.as_str().into()
     }
 }
@@ -41,7 +41,7 @@ impl ui::menu::Item for DebugTemplate {
 impl ui::menu::Item for Thread {
     type Data = ThreadStates;
 
-    fn label(&self, thread_states: &Self::Data) -> Spans {
+    fn format(&self, thread_states: &Self::Data) -> Row {
         format!(
             "{} ({})",
             self.name,
@@ -270,7 +270,7 @@ pub fn dap_launch(cx: &mut Context) {
 
     let templates = config.templates.clone();
 
-    cx.push_layer(Box::new(overlayed(Picker::new(
+    cx.push_layer(Box::new(overlaid(Picker::new(
         templates,
         (),
         |cx, template, _action| {
@@ -289,6 +289,36 @@ pub fn dap_launch(cx: &mut Context) {
     ))));
 }
 
+pub fn dap_restart(cx: &mut Context) {
+    let debugger = match &cx.editor.debugger {
+        Some(debugger) => debugger,
+        None => {
+            cx.editor.set_error("Debugger is not running");
+            return;
+        }
+    };
+    if !debugger
+        .capabilities()
+        .supports_restart_request
+        .unwrap_or(false)
+    {
+        cx.editor
+            .set_error("Debugger does not support session restarts");
+        return;
+    }
+    if debugger.starting_request_args().is_none() {
+        cx.editor
+            .set_error("No arguments found with which to restart the sessions");
+        return;
+    }
+
+    dap_callback(
+        cx.jobs,
+        debugger.restart(),
+        |editor, _compositor, _resp: ()| editor.set_status("Debugging session restarted"),
+    );
+}
+
 fn debug_parameter_prompt(
     completions: Vec<DebugConfigCompletion>,
     config_name: String,
@@ -475,19 +505,36 @@ pub fn dap_variables(cx: &mut Context) {
 
     if debugger.thread_id.is_none() {
         cx.editor
-            .set_status("Cannot access variables while target is running");
+            .set_status("Cannot access variables while target is running.");
         return;
     }
     let (frame, thread_id) = match (debugger.active_frame, debugger.thread_id) {
         (Some(frame), Some(thread_id)) => (frame, thread_id),
         _ => {
             cx.editor
-                .set_status("Cannot find current stack frame to access variables");
+                .set_status("Cannot find current stack frame to access variables.");
             return;
         }
     };
 
-    let frame_id = debugger.stack_frames[&thread_id][frame].id;
+    let thread_frame = match debugger.stack_frames.get(&thread_id) {
+        Some(thread_frame) => thread_frame,
+        None => {
+            cx.editor
+                .set_error("Failed to get stack frame for thread: {thread_id}");
+            return;
+        }
+    };
+    let stack_frame = match thread_frame.get(frame) {
+        Some(stack_frame) => stack_frame,
+        None => {
+            cx.editor
+                .set_error("Failed to get stack frame for thread {thread_id} and frame {frame}.");
+            return;
+        }
+    };
+
+    let frame_id = stack_frame.id;
     let scopes = match block_on(debugger.scopes(frame_id)) {
         Ok(s) => s,
         Err(e) => {
@@ -533,13 +580,13 @@ pub fn dap_variables(cx: &mut Context) {
 
     let contents = Text::from(tui::text::Text::from(variables));
     let popup = Popup::new("dap-variables", contents);
-    cx.push_layer(Box::new(popup));
+    cx.replace_or_push_layer("dap-variables", popup);
 }
 
 pub fn dap_terminate(cx: &mut Context) {
     let debugger = debugger!(cx.editor);
 
-    let request = debugger.disconnect();
+    let request = debugger.disconnect(None);
     dap_callback(cx.jobs, request, |editor, _compositor, _response: ()| {
         // editor.set_error(format!("Failed to disconnect: {}", e));
         editor.debugger = None;

helix-term/src/commands/lsp.rs

@@ -1,25 +1,38 @@
+use futures_util::FutureExt;
 use helix_lsp::{
     block_on,
-    lsp::{self, CodeAction, CodeActionOrCommand, DiagnosticSeverity, NumberOrString},
-    util::{diagnostic_to_lsp_diagnostic, lsp_pos_to_pos, lsp_range_to_range, range_to_lsp_range},
+    lsp::{
+        self, CodeAction, CodeActionOrCommand, CodeActionTriggerKind, DiagnosticSeverity,
+        NumberOrString,
+    },
+    util::{diagnostic_to_lsp_diagnostic, lsp_range_to_range, range_to_lsp_range},
     OffsetEncoding,
 };
-use tui::text::{Span, Spans};
+use tui::{
+    text::{Span, Spans},
+    widgets::Row,
+};
 
 use super::{align_view, push_jump, Align, Context, Editor, Open};
 
-use helix_core::{path, Selection};
-use helix_view::{apply_transaction, document::Mode, editor::Action, theme::Style};
+use helix_core::{path, text_annotations::InlineAnnotation, Selection};
+use helix_view::{
+    document::{DocumentInlayHints, DocumentInlayHintsId, Mode},
+    editor::Action,
+    theme::Style,
+    Document, View,
+};
 
 use crate::{
     compositor::{self, Compositor},
     ui::{
-        self, lsp::SignatureHelp, overlay::overlayed, FileLocation, FilePicker, Popup, PromptEvent,
+        self, lsp::SignatureHelp, overlay::overlaid, DynamicPicker, FileLocation, FilePicker,
+        Popup, PromptEvent,
     },
 };
 
 use std::{
-    borrow::Cow, cmp::Ordering, collections::BTreeMap, fmt::Write, path::PathBuf, sync::Arc,
+    cmp::Ordering, collections::BTreeMap, fmt::Write, future::Future, path::PathBuf, sync::Arc,
 };
 
 /// Gets the language server that is attached to a document, and
@@ -44,7 +57,7 @@ impl ui::menu::Item for lsp::Location {
     /// Current working directory.
     type Data = PathBuf;
 
-    fn label(&self, cwdir: &Self::Data) -> Spans {
+    fn format(&self, cwdir: &Self::Data) -> Row {
         // The preallocation here will overallocate a few characters since it will account for the
         // URL's scheme, which is not used most of the time since that scheme will be "file://".
         // Those extra chars will be used to avoid allocating when writing the line number (in the
@@ -68,7 +81,7 @@ impl ui::menu::Item for lsp::Location {
 
         // Most commonly, this will not allocate, especially on Unix systems where the root prefix
         // is a simple `/` and not `C:\` (with whatever drive letter)
-        write!(&mut res, ":{}", self.range.start.line)
+        write!(&mut res, ":{}", self.range.start.line + 1)
             .expect("Will only failed if allocating fail");
         res.into()
     }
@@ -78,7 +91,7 @@ impl ui::menu::Item for lsp::SymbolInformation {
     /// Path to currently focussed document
     type Data = Option<lsp::Url>;
 
-    fn label(&self, current_doc_path: &Self::Data) -> Spans {
+    fn format(&self, current_doc_path: &Self::Data) -> Row {
         if current_doc_path.as_ref() == Some(&self.location.uri) {
             self.name.as_str().into()
         } else {
@@ -108,7 +121,7 @@ struct PickerDiagnostic {
 impl ui::menu::Item for PickerDiagnostic {
     type Data = (DiagnosticStyles, DiagnosticsFormat);
 
-    fn label(&self, (styles, format): &Self::Data) -> Spans {
+    fn format(&self, (styles, format): &Self::Data) -> Row {
         let mut style = self
             .diag
             .severity
@@ -124,20 +137,17 @@ impl ui::menu::Item for PickerDiagnostic {
         // remove background as it is distracting in the picker list
         style.bg = None;
 
-        let code: Cow<'_, str> = self
-            .diag
-            .code
-            .as_ref()
-            .map(|c| match c {
-                NumberOrString::Number(n) => n.to_string().into(),
-                NumberOrString::String(s) => s.as_str().into(),
-            })
-            .unwrap_or_default();
+        let code = match self.diag.code.as_ref() {
+            Some(NumberOrString::Number(n)) => format!(" ({n})"),
+            Some(NumberOrString::String(s)) => format!(" ({s})"),
+            None => String::new(),
+        };
 
         let path = match format {
             DiagnosticsFormat::HideSourcePath => String::new(),
             DiagnosticsFormat::ShowSourcePath => {
-                let path = path::get_truncated_path(self.url.path());
+                let file_path = self.url.to_file_path().unwrap();
+                let path = path::get_truncated_path(file_path);
                 format!("{}: ", path.to_string_lossy())
             }
         };
@@ -147,6 +157,7 @@ impl ui::menu::Item for PickerDiagnostic {
             Span::styled(&self.diag.message, style),
             Span::styled(code, style),
         ])
+        .into()
     }
 }
 
@@ -186,15 +197,17 @@ fn jump_to_location(
         }
     }
     let (view, doc) = current!(editor);
-    let definition_pos = location.range.start;
     // TODO: convert inside server
-    let new_pos = if let Some(new_pos) = lsp_pos_to_pos(doc.text(), definition_pos, offset_encoding)
-    {
-        new_pos
-    } else {
-        return;
-    };
-    doc.set_selection(view.id, Selection::point(new_pos));
+    let new_range =
+        if let Some(new_range) = lsp_range_to_range(doc.text(), location.range, offset_encoding) {
+            new_range
+        } else {
+            log::warn!("lsp position out of bounds - {:?}", location.range);
+            return;
+        };
+    // we flip the range so that the cursor sits on the start of the symbol
+    // (for example start of the function).
+    doc.set_selection(view.id, Selection::single(new_range.head, new_range.anchor));
     align_view(doc, view, Align::Center);
 }
 
@@ -361,7 +374,7 @@ pub fn symbol_picker(cx: &mut Context) {
                 };
 
                 let picker = sym_picker(symbols, current_url, offset_encoding);
-                compositor.push(Box::new(overlayed(picker)))
+                compositor.push(Box::new(overlaid(picker)))
             }
         },
     )
@@ -384,10 +397,43 @@ pub fn workspace_symbol_picker(cx: &mut Context) {
     cx.callback(
         future,
         move |_editor, compositor, response: Option<Vec<lsp::SymbolInformation>>| {
-            if let Some(symbols) = response {
-                let picker = sym_picker(symbols, current_url, offset_encoding);
-                compositor.push(Box::new(overlayed(picker)))
-            }
+            let symbols = response.unwrap_or_default();
+            let picker = sym_picker(symbols, current_url, offset_encoding);
+            let get_symbols = |query: String, editor: &mut Editor| {
+                let doc = doc!(editor);
+                let language_server = match doc.language_server() {
+                    Some(s) => s,
+                    None => {
+                        // This should not generally happen since the picker will not
+                        // even open in the first place if there is no server.
+                        return async move { Err(anyhow::anyhow!("LSP not active")) }.boxed();
+                    }
+                };
+                let symbol_request = match language_server.workspace_symbols(query) {
+                    Some(future) => future,
+                    None => {
+                        // This should also not happen since the language server must have
+                        // supported workspace symbols before to reach this block.
+                        return async move {
+                            Err(anyhow::anyhow!(
+                                "Language server does not support workspace symbols"
+                            ))
+                        }
+                        .boxed();
+                    }
+                };
+
+                let future = async move {
+                    let json = symbol_request.await?;
+                    let response: Option<Vec<lsp::SymbolInformation>> =
+                        serde_json::from_value(json)?;
+
+                    Ok(response.unwrap_or_default())
+                };
+                future.boxed()
+            };
+            let dyn_picker = DynamicPicker::new(picker, Box::new(get_symbols));
+            compositor.push(Box::new(overlaid(dyn_picker)))
         },
     )
 }
@@ -410,7 +456,7 @@ pub fn diagnostics_picker(cx: &mut Context) {
             DiagnosticsFormat::HideSourcePath,
             offset_encoding,
         );
-        cx.push_layer(Box::new(overlayed(picker)));
+        cx.push_layer(Box::new(overlaid(picker)));
     }
 }
 
@@ -427,12 +473,12 @@ pub fn workspace_diagnostics_picker(cx: &mut Context) {
         DiagnosticsFormat::ShowSourcePath,
         offset_encoding,
     );
-    cx.push_layer(Box::new(overlayed(picker)));
+    cx.push_layer(Box::new(overlaid(picker)));
 }
 
 impl ui::menu::Item for lsp::CodeActionOrCommand {
     type Data = ();
-    fn label(&self, _data: &Self::Data) -> Spans {
+    fn format(&self, _data: &Self::Data) -> Row {
         match self {
             lsp::CodeActionOrCommand::CodeAction(action) => action.title.as_str().into(),
             lsp::CodeActionOrCommand::Command(command) => command.title.as_str().into(),
@@ -447,7 +493,7 @@ impl ui::menu::Item for lsp::CodeActionOrCommand {
 ///
 /// While the `kind` field is defined as open ended in the LSP spec (any value may be used)
 /// in practice a closed set of common values (mostly suggested in the LSP spec) are used.
-/// VSCode displays each of these categories seperatly (seperated by a heading in the codeactions picker)
+/// VSCode displays each of these categories separately (separated by a heading in the codeactions picker)
 /// to make them easier to navigate. Helix does not display these  headings to the user.
 /// However it does sort code actions by their categories to achieve the same order as the VScode picker,
 /// just without the headings.
@@ -477,7 +523,7 @@ fn action_category(action: &CodeActionOrCommand) -> u32 {
     }
 }
 
-fn action_prefered(action: &CodeActionOrCommand) -> bool {
+fn action_preferred(action: &CodeActionOrCommand) -> bool {
     matches!(
         action,
         CodeActionOrCommand::CodeAction(CodeAction {
@@ -522,6 +568,7 @@ pub fn code_action(cx: &mut Context) {
                 .map(|diag| diagnostic_to_lsp_diagnostic(doc.text(), diag, offset_encoding))
                 .collect(),
             only: None,
+            trigger_kind: Some(CodeActionTriggerKind::INVOKED),
         },
     ) {
         Some(future) => future,
@@ -555,12 +602,12 @@ pub fn code_action(cx: &mut Context) {
             }
 
             // Sort codeactions into a useful order. This behaviour is only partially described in the LSP spec.
-            // Many details are modeled after vscode because langauge servers are usually tested against it.
+            // Many details are modeled after vscode because language servers are usually tested against it.
             // VScode sorts the codeaction two times:
             //
             // First the codeactions that fix some diagnostics are moved to the front.
             // If both codeactions fix some diagnostics (or both fix none) the codeaction
-            // that is marked with `is_preffered` is shown first. The codeactions are then shown in seperate
+            // that is marked with `is_preferred` is shown first. The codeactions are then shown in separate
             // submenus that only contain a certain category (see `action_category`) of actions.
             //
             // Below this done in in a single sorting step
@@ -582,10 +629,10 @@ pub fn code_action(cx: &mut Context) {
                     return order;
                 }
 
-                // if one of the codeactions is marked as prefered show it first
+                // if one of the codeactions is marked as preferred show it first
                 // otherwise keep the original LSP sorting
-                action_prefered(action1)
-                    .cmp(&action_prefered(action2))
+                action_preferred(action1)
+                    .cmp(&action_preferred(action2))
                     .reverse()
             });
 
@@ -606,7 +653,7 @@ pub fn code_action(cx: &mut Context) {
                         log::debug!("code action: {:?}", code_action);
                         if let Some(ref workspace_edit) = code_action.edit {
                             log::debug!("edit: {:?}", workspace_edit);
-                            apply_workspace_edit(editor, offset_encoding, workspace_edit);
+                            let _ = apply_workspace_edit(editor, offset_encoding, workspace_edit);
                         }
 
                         // if code action provides both edit and command first the edit
@@ -627,7 +674,7 @@ pub fn code_action(cx: &mut Context) {
 
 impl ui::menu::Item for lsp::Command {
     type Data = ();
-    fn label(&self, _data: &Self::Data) -> Spans {
+    fn format(&self, _data: &Self::Data) -> Row {
         self.title.as_str().into()
     }
 }
@@ -712,19 +759,50 @@ pub fn apply_document_resource_op(op: &lsp::ResourceOp) -> std::io::Result<()> {
     }
 }
 
+#[derive(Debug)]
+pub struct ApplyEditError {
+    pub kind: ApplyEditErrorKind,
+    pub failed_change_idx: usize,
+}
+
+#[derive(Debug)]
+pub enum ApplyEditErrorKind {
+    DocumentChanged,
+    FileNotFound,
+    UnknownURISchema,
+    IoError(std::io::Error),
+    // TODO: check edits before applying and propagate failure
+    // InvalidEdit,
+}
+
+impl ToString for ApplyEditErrorKind {
+    fn to_string(&self) -> String {
+        match self {
+            ApplyEditErrorKind::DocumentChanged => "document has changed".to_string(),
+            ApplyEditErrorKind::FileNotFound => "file not found".to_string(),
+            ApplyEditErrorKind::UnknownURISchema => "URI schema not supported".to_string(),
+            ApplyEditErrorKind::IoError(err) => err.to_string(),
+        }
+    }
+}
+
+///TODO make this transactional (and set failureMode to transactional)
 pub fn apply_workspace_edit(
     editor: &mut Editor,
     offset_encoding: OffsetEncoding,
     workspace_edit: &lsp::WorkspaceEdit,
-) {
-    let mut apply_edits = |uri: &helix_lsp::Url, text_edits: Vec<lsp::TextEdit>| {
+) -> Result<(), ApplyEditError> {
+    let mut apply_edits = |uri: &helix_lsp::Url,
+                           version: Option<i32>,
+                           text_edits: Vec<lsp::TextEdit>|
+     -> Result<(), ApplyEditErrorKind> {
         let path = match uri.to_file_path() {
             Ok(path) => path,
             Err(_) => {
                 let err = format!("unable to convert URI to filepath: {}", uri);
                 log::error!("{}", err);
                 editor.set_error(err);
-                return;
+                return Err(ApplyEditErrorKind::UnknownURISchema);
             }
         };
 
@@ -735,11 +813,19 @@ pub fn apply_workspace_edit(
                 let err = format!("failed to open document: {}: {}", uri, err);
                 log::error!("{}", err);
                 editor.set_error(err);
-                return;
+                return Err(ApplyEditErrorKind::FileNotFound);
             }
         };
 
         let doc = doc_mut!(editor, &doc_id);
+        if let Some(version) = version {
+            if version != doc.version() {
+                let err = format!("outdated workspace edit for {path:?}");
+                log::error!("{err}, expected {} but got {version}", doc.version());
+                editor.set_error(err);
+                return Err(ApplyEditErrorKind::DocumentChanged);
+            }
+        }
 
         // Need to determine a view for apply/append_changes_to_history
         let selections = doc.selections();
@@ -761,33 +847,15 @@ pub fn apply_workspace_edit(
             offset_encoding,
         );
         let view = view_mut!(editor, view_id);
-        apply_transaction(&transaction, doc, view);
+        doc.apply(&transaction, view.id);
         doc.append_changes_to_history(view);
+        Ok(())
     };
 
-    if let Some(ref changes) = workspace_edit.changes {
-        log::debug!("workspace changes: {:?}", changes);
-        for (uri, text_edits) in changes {
-            let text_edits = text_edits.to_vec();
-            apply_edits(uri, text_edits)
-        }
-        return;
-        // Not sure if it works properly, it'll be safer to just panic here to avoid breaking some parts of code on which code actions will be used
-        // TODO: find some example that uses workspace changes, and test it
-        // for (url, edits) in changes.iter() {
-        //     let file_path = url.origin().ascii_serialization();
-        //     let file_path = std::path::PathBuf::from(file_path);
-        //     let file = std::fs::File::open(file_path).unwrap();
-        //     let mut text = Rope::from_reader(file).unwrap();
-        //     let transaction = edits_to_changes(&text, edits);
-        //     transaction.apply(&mut text);
-        // }
-    }
-
     if let Some(ref document_changes) = workspace_edit.document_changes {
         match document_changes {
             lsp::DocumentChanges::Edits(document_edits) => {
-                for document_edit in document_edits {
+                for (i, document_edit) in document_edits.iter().enumerate() {
                     let edits = document_edit
                         .edits
                         .iter()
@@ -799,15 +867,26 @@ pub fn apply_workspace_edit(
                         })
                         .cloned()
                         .collect();
-                    apply_edits(&document_edit.text_document.uri, edits);
+                    apply_edits(
+                        &document_edit.text_document.uri,
+                        document_edit.text_document.version,
+                        edits,
+                    )
+                    .map_err(|kind| ApplyEditError {
+                        kind,
+                        failed_change_idx: i,
+                    })?;
                 }
             }
             lsp::DocumentChanges::Operations(operations) => {
                 log::debug!("document changes - operations: {:?}", operations);
-                for operation in operations {
+                for (i, operation) in operations.iter().enumerate() {
                     match operation {
                         lsp::DocumentChangeOperation::Op(op) => {
-                            apply_document_resource_op(op).unwrap();
+                            apply_document_resource_op(op).map_err(|io| ApplyEditError {
+                                kind: ApplyEditErrorKind::IoError(io),
+                                failed_change_idx: i,
+                            })?;
                         }
 
                         lsp::DocumentChangeOperation::Edit(document_edit) => {
@@ -822,13 +901,36 @@ pub fn apply_workspace_edit(
                                 })
                                 .cloned()
                                 .collect();
-                            apply_edits(&document_edit.text_document.uri, edits);
+                            apply_edits(
+                                &document_edit.text_document.uri,
+                                document_edit.text_document.version,
+                                edits,
+                            )
+                            .map_err(|kind| ApplyEditError {
+                                kind,
+                                failed_change_idx: i,
+                            })?;
                         }
                     }
                 }
             }
         }
+
+        return Ok(());
     }
+
+    if let Some(ref changes) = workspace_edit.changes {
+        log::debug!("workspace changes: {:?}", changes);
+        for (i, (uri, text_edits)) in changes.iter().enumerate() {
+            let text_edits = text_edits.to_vec();
+            apply_edits(uri, None, text_edits).map_err(|kind| ApplyEditError {
+                kind,
+                failed_change_idx: i,
+            })?;
+        }
+    }
+
+    Ok(())
 }
 
 fn goto_impl(
@@ -855,7 +957,7 @@ fn goto_impl(
                 },
                 move |_editor, location| Some(location_to_file_location(location)),
             );
-            compositor.push(Box::new(overlayed(picker)));
+            compositor.push(Box::new(overlaid(picker)));
         }
     }
 }
@@ -875,6 +977,31 @@ fn to_locations(definitions: Option<lsp::GotoDefinitionResponse>) -> Vec<lsp::Lo
     }
 }
 
+pub fn goto_declaration(cx: &mut Context) {
+    let (view, doc) = current!(cx.editor);
+    let language_server = language_server!(cx.editor, doc);
+    let offset_encoding = language_server.offset_encoding();
+
+    let pos = doc.position(view.id, offset_encoding);
+
+    let future = match language_server.goto_declaration(doc.identifier(), pos, None) {
+        Some(future) => future,
+        None => {
+            cx.editor
+                .set_error("Language server does not support goto-declaration");
+            return;
+        }
+    };
+
+    cx.callback(
+        future,
+        move |editor, compositor, response: Option<lsp::GotoDefinitionResponse>| {
+            let items = to_locations(response);
+            goto_impl(editor, compositor, items, offset_encoding);
+        },
+    );
+}
+
 pub fn goto_definition(cx: &mut Context) {
     let (view, doc) = current!(cx.editor);
     let language_server = language_server!(cx.editor, doc);
@@ -951,13 +1078,19 @@ pub fn goto_implementation(cx: &mut Context) {
 }
 
 pub fn goto_reference(cx: &mut Context) {
+    let config = cx.editor.config();
     let (view, doc) = current!(cx.editor);
     let language_server = language_server!(cx.editor, doc);
     let offset_encoding = language_server.offset_encoding();
 
     let pos = doc.position(view.id, offset_encoding);
 
-    let future = match language_server.goto_reference(doc.identifier(), pos, None) {
+    let future = match language_server.goto_reference(
+        doc.identifier(),
+        pos,
+        config.lsp.goto_reference_include_declaration,
+        None,
+    ) {
         Some(future) => future,
         None => {
             cx.editor
@@ -1096,10 +1229,25 @@ pub fn signature_help_impl(cx: &mut Context, invoked: SignatureHelpInvoked) {
             contents.set_active_param_range(active_param_range());
 
             let old_popup = compositor.find_id::<Popup<SignatureHelp>>(SignatureHelp::ID);
-            let popup = Popup::new(SignatureHelp::ID, contents)
+            let mut popup = Popup::new(SignatureHelp::ID, contents)
                 .position(old_popup.and_then(|p| p.get_position()))
                 .position_bias(Open::Above)
                 .ignore_escape_key(true);
+
+            // Don't create a popup if it intersects the auto-complete menu.
+            let size = compositor.size();
+            if compositor
+                .find::<ui::EditorView>()
+                .unwrap()
+                .completion
+                .as_mut()
+                .map(|completion| completion.area(size, editor))
+                .filter(|area| area.intersects(popup.area(size, editor)))
+                .is_some()
+            {
+                return;
+            }
+
             compositor.replace_or_push(SignatureHelp::ID, popup);
         },
     );
@@ -1163,49 +1311,123 @@ pub fn hover(cx: &mut Context) {
 }
 
 pub fn rename_symbol(cx: &mut Context) {
-    let (view, doc) = current_ref!(cx.editor);
-    let text = doc.text().slice(..);
-    let primary_selection = doc.selection(view.id).primary();
-    let prefill = if primary_selection.len() > 1 {
-        primary_selection
-    } else {
-        use helix_core::textobject::{textobject_word, TextObject};
-        textobject_word(text, primary_selection, TextObject::Inside, 1, false)
+    fn get_prefill_from_word_boundary(editor: &Editor) -> String {
+        let (view, doc) = current_ref!(editor);
+        let text = doc.text().slice(..);
+        let primary_selection = doc.selection(view.id).primary();
+        if primary_selection.len() > 1 {
+            primary_selection
+        } else {
+            use helix_core::textobject::{textobject_word, TextObject};
+            textobject_word(text, primary_selection, TextObject::Inside, 1, false)
+        }
+        .fragment(text)
+        .into()
     }
-    .fragment(text)
-    .into();
-    ui::prompt_with_input(
-        cx,
-        "rename-to:".into(),
-        prefill,
-        None,
-        ui::completers::none,
-        move |cx: &mut compositor::Context, input: &str, event: PromptEvent| {
-            if event != PromptEvent::Validate {
-                return;
+
+    fn get_prefill_from_lsp_response(
+        editor: &Editor,
+        offset_encoding: OffsetEncoding,
+        response: Option<lsp::PrepareRenameResponse>,
+    ) -> Result<String, &'static str> {
+        match response {
+            Some(lsp::PrepareRenameResponse::Range(range)) => {
+                let text = doc!(editor).text();
+
+                Ok(lsp_range_to_range(text, range, offset_encoding)
+                    .ok_or("lsp sent invalid selection range for rename")?
+                    .fragment(text.slice(..))
+                    .into())
             }
+            Some(lsp::PrepareRenameResponse::RangeWithPlaceholder { placeholder, .. }) => {
+                Ok(placeholder)
+            }
+            Some(lsp::PrepareRenameResponse::DefaultBehavior { .. }) => {
+                Ok(get_prefill_from_word_boundary(editor))
+            }
+            None => Err("lsp did not respond to prepare rename request"),
+        }
+    }
 
-            let (view, doc) = current!(cx.editor);
-            let language_server = language_server!(cx.editor, doc);
-            let offset_encoding = language_server.offset_encoding();
+    fn create_rename_prompt(editor: &Editor, prefill: String) -> Box<ui::Prompt> {
+        let prompt = ui::Prompt::new(
+            "rename-to:".into(),
+            None,
+            ui::completers::none,
+            move |cx: &mut compositor::Context, input: &str, event: PromptEvent| {
+                if event != PromptEvent::Validate {
+                    return;
+                }
 
-            let pos = doc.position(view.id, offset_encoding);
+                let (view, doc) = current!(cx.editor);
+                let language_server = language_server!(cx.editor, doc);
+                let offset_encoding = language_server.offset_encoding();
 
-            let future =
-                match language_server.rename_symbol(doc.identifier(), pos, input.to_string()) {
-                    Some(future) => future,
-                    None => {
-                        cx.editor
-                            .set_error("Language server does not support symbol renaming");
+                let pos = doc.position(view.id, offset_encoding);
+
+                let future =
+                    match language_server.rename_symbol(doc.identifier(), pos, input.to_string()) {
+                        Some(future) => future,
+                        None => {
+                            cx.editor
+                                .set_error("Language server does not support symbol renaming");
+                            return;
+                        }
+                    };
+                match block_on(future) {
+                    Ok(edits) => {
+                        let _ = apply_workspace_edit(cx.editor, offset_encoding, &edits);
+                    }
+                    Err(err) => cx.editor.set_error(err.to_string()),
+                }
+            },
+        )
+        .with_line(prefill, editor);
+
+        Box::new(prompt)
+    }
+
+    let (view, doc) = current!(cx.editor);
+    let language_server = language_server!(cx.editor, doc);
+    let offset_encoding = language_server.offset_encoding();
+
+    if !language_server.supports_rename() {
+        cx.editor
+            .set_error("Language server does not support symbol renaming");
+        return;
+    }
+
+    let pos = doc.position(view.id, offset_encoding);
+
+    match language_server.prepare_rename(doc.identifier(), pos) {
+        // Language server supports textDocument/prepareRename, use it.
+        Some(future) => cx.callback(
+            future,
+            move |editor, compositor, response: Option<lsp::PrepareRenameResponse>| {
+                let prefill = match get_prefill_from_lsp_response(editor, offset_encoding, response)
+                {
+                    Ok(p) => p,
+                    Err(e) => {
+                        editor.set_error(e);
                         return;
                     }
                 };
-            match block_on(future) {
-                Ok(edits) => apply_workspace_edit(cx.editor, offset_encoding, &edits),
-                Err(err) => cx.editor.set_error(err.to_string()),
-            }
-        },
-    );
+
+                let prompt = create_rename_prompt(editor, prefill);
+
+                compositor.push(prompt);
+            },
+        ),
+        // Language server does not support textDocument/prepareRename, fall back
+        // to word boundary selection.
+        None => {
+            let prefill = get_prefill_from_word_boundary(cx.editor);
+
+            let prompt = create_rename_prompt(cx.editor, prefill);
+
+            cx.push_layer(prompt);
+        }
+    };
 }
 
 pub fn select_references_to_symbol_under_cursor(cx: &mut Context) {
@@ -1256,3 +1478,175 @@ pub fn select_references_to_symbol_under_cursor(cx: &mut Context) {
         },
     );
 }
+
+pub fn compute_inlay_hints_for_all_views(editor: &mut Editor, jobs: &mut crate::job::Jobs) {
+    if !editor.config().lsp.display_inlay_hints {
+        return;
+    }
+
+    for (view, _) in editor.tree.views() {
+        let doc = match editor.documents.get(&view.doc) {
+            Some(doc) => doc,
+            None => continue,
+        };
+        if let Some(callback) = compute_inlay_hints_for_view(view, doc) {
+            jobs.callback(callback);
+        }
+    }
+}
+
+fn compute_inlay_hints_for_view(
+    view: &View,
+    doc: &Document,
+) -> Option<std::pin::Pin<Box<impl Future<Output = Result<crate::job::Callback, anyhow::Error>>>>> {
+    let view_id = view.id;
+    let doc_id = view.doc;
+
+    let language_server = doc.language_server()?;
+
+    let capabilities = language_server.capabilities();
+
+    let (future, new_doc_inlay_hints_id) = match capabilities.inlay_hint_provider {
+        Some(
+            lsp::OneOf::Left(true)
+            | lsp::OneOf::Right(lsp::InlayHintServerCapabilities::Options(_)),
+        ) => {
+            let doc_text = doc.text();
+            let len_lines = doc_text.len_lines();
+
+            // Compute ~3 times the current view height of inlay hints, that way some scrolling
+            // will not show half the view with hints and half without while still being faster
+            // than computing all the hints for the full file (which could be dozens of time
+            // longer than the view is).
+            let view_height = view.inner_height();
+            let first_visible_line =
+                doc_text.char_to_line(view.offset.anchor.min(doc_text.len_chars()));
+            let first_line = first_visible_line.saturating_sub(view_height);
+            let last_line = first_visible_line
+                .saturating_add(view_height.saturating_mul(2))
+                .min(len_lines);
+
+            let new_doc_inlay_hint_id = DocumentInlayHintsId {
+                first_line,
+                last_line,
+            };
+            // Don't recompute the annotations in case nothing has changed about the view
+            if !doc.inlay_hints_oudated
+                && doc
+                    .inlay_hints(view_id)
+                    .map_or(false, |dih| dih.id == new_doc_inlay_hint_id)
+            {
+                return None;
+            }
+
+            let doc_slice = doc_text.slice(..);
+            let first_char_in_range = doc_slice.line_to_char(first_line);
+            let last_char_in_range = doc_slice.line_to_char(last_line);
+
+            let range = helix_lsp::util::range_to_lsp_range(
+                doc_text,
+                helix_core::Range::new(first_char_in_range, last_char_in_range),
+                language_server.offset_encoding(),
+            );
+
+            (
+                language_server.text_document_range_inlay_hints(doc.identifier(), range, None),
+                new_doc_inlay_hint_id,
+            )
+        }
+        _ => return None,
+    };
+
+    let callback = super::make_job_callback(
+        future?,
+        move |editor, _compositor, response: Option<Vec<lsp::InlayHint>>| {
+            // The config was modified or the window was closed while the request was in flight
+            if !editor.config().lsp.display_inlay_hints || editor.tree.try_get(view_id).is_none() {
+                return;
+            }
+
+            // Add annotations to relevant document, not the current one (it may have changed in between)
+            let doc = match editor.documents.get_mut(&doc_id) {
+                Some(doc) => doc,
+                None => return,
+            };
+
+            // If we have neither hints nor an LSP, empty the inlay hints since they're now oudated
+            let (mut hints, offset_encoding) = match (response, doc.language_server()) {
+                (Some(h), Some(ls)) if !h.is_empty() => (h, ls.offset_encoding()),
+                _ => {
+                    doc.set_inlay_hints(
+                        view_id,
+                        DocumentInlayHints::empty_with_id(new_doc_inlay_hints_id),
+                    );
+                    doc.inlay_hints_oudated = false;
+                    return;
+                }
+            };
+
+            // Most language servers will already send them sorted but ensure this is the case to
+            // avoid errors on our end.
+            hints.sort_unstable_by_key(|inlay_hint| inlay_hint.position);
+
+            let mut padding_before_inlay_hints = Vec::new();
+            let mut type_inlay_hints = Vec::new();
+            let mut parameter_inlay_hints = Vec::new();
+            let mut other_inlay_hints = Vec::new();
+            let mut padding_after_inlay_hints = Vec::new();
+
+            let doc_text = doc.text();
+
+            for hint in hints {
+                let char_idx =
+                    match helix_lsp::util::lsp_pos_to_pos(doc_text, hint.position, offset_encoding)
+                    {
+                        Some(pos) => pos,
+                        // Skip inlay hints that have no "real" position
+                        None => continue,
+                    };
+
+                let label = match hint.label {
+                    lsp::InlayHintLabel::String(s) => s,
+                    lsp::InlayHintLabel::LabelParts(parts) => parts
+                        .into_iter()
+                        .map(|p| p.value)
+                        .collect::<Vec<_>>()
+                        .join(""),
+                };
+
+                let inlay_hints_vec = match hint.kind {
+                    Some(lsp::InlayHintKind::TYPE) => &mut type_inlay_hints,
+                    Some(lsp::InlayHintKind::PARAMETER) => &mut parameter_inlay_hints,
+                    // We can't warn on unknown kind here since LSPs are free to set it or not, for
+                    // example Rust Analyzer does not: every kind will be `None`.
+                    _ => &mut other_inlay_hints,
+                };
+
+                if let Some(true) = hint.padding_left {
+                    padding_before_inlay_hints.push(InlineAnnotation::new(char_idx, " "));
+                }
+
+                inlay_hints_vec.push(InlineAnnotation::new(char_idx, label));
+
+                if let Some(true) = hint.padding_right {
+                    padding_after_inlay_hints.push(InlineAnnotation::new(char_idx, " "));
+                }
+            }
+
+            doc.set_inlay_hints(
+                view_id,
+                DocumentInlayHints {
+                    id: new_doc_inlay_hints_id,
+                    type_inlay_hints: type_inlay_hints.into(),
+                    parameter_inlay_hints: parameter_inlay_hints.into(),
+                    other_inlay_hints: other_inlay_hints.into(),
+                    padding_before_inlay_hints: padding_before_inlay_hints.into(),
+                    padding_after_inlay_hints: padding_after_inlay_hints.into(),
+                },
+            );
+            doc.inlay_hints_oudated = false;
+        },
+    );
+
+    Some(callback)
+}

helix-term/src/compositor.rs

@@ -1,4 +1,4 @@
-// Each component declares it's own size constraints and gets fitted based on it's parent.
+// Each component declares its own size constraints and gets fitted based on its parent.
 // Q: how does this work with popups?
 // cursive does compositor.screen_mut().add_layer_at(pos::absolute(x, y), <component>)
 use helix_core::Position;
@@ -7,6 +7,7 @@ use helix_view::graphics::{CursorKind, Rect};
 use tui::buffer::Buffer as Surface;
 
 pub type Callback = Box<dyn FnOnce(&mut Compositor, &mut Context)>;
+pub type SyncCallback = Box<dyn FnOnce(&mut Compositor, &mut Context) + Sync>;
 
 // Cursive-inspired
 pub enum EventResult {
@@ -97,6 +98,7 @@ impl Compositor {
         self.area = area;
     }
 
+    /// Add a layer to be rendered in front of all existing layers.
     pub fn push(&mut self, mut layer: Box<dyn Component>) {
         let size = self.size();
         // trigger required_size on init
@@ -136,7 +138,8 @@ impl Compositor {
         let mut consumed = false;
 
         // propagate events through the layers until we either find a layer that consumes it or we
-        // run out of layers (event bubbling)
+        // run out of layers (event bubbling), starting at the front layer and then moving to the
+        // background.
         for layer in self.layers.iter_mut().rev() {
             match layer.handle_event(event, cx) {
                 EventResult::Consumed(Some(callback)) => {

helix-term/src/config.rs

@@ -1,27 +1,34 @@
-use crate::keymap::{default::default, merge_keys, Keymap};
+use crate::keymap;
+use crate::keymap::{merge_keys, Keymap};
+use helix_loader::merge_toml_values;
 use helix_view::document::Mode;
 use serde::Deserialize;
 use std::collections::HashMap;
 use std::fmt::Display;
+use std::fs;
 use std::io::Error as IOError;
-use std::path::PathBuf;
 use toml::de::Error as TomlError;
 
+#[derive(Debug, Clone, PartialEq)]
+pub struct Config {
+    pub theme: Option<String>,
+    pub keys: HashMap<Mode, Keymap>,
+    pub editor: helix_view::editor::Config,
+}
+
 #[derive(Debug, Clone, PartialEq, Deserialize)]
 #[serde(deny_unknown_fields)]
-pub struct Config {
+pub struct ConfigRaw {
     pub theme: Option<String>,
-    #[serde(default = "default")]
-    pub keys: HashMap<Mode, Keymap>,
-    #[serde(default)]
-    pub editor: helix_view::editor::Config,
+    pub keys: Option<HashMap<Mode, Keymap>>,
+    pub editor: Option<toml::Value>,
 }
 
 impl Default for Config {
     fn default() -> Config {
         Config {
             theme: None,
-            keys: default(),
+            keys: keymap::default(),
             editor: helix_view::editor::Config::default(),
         }
     }
@@ -33,6 +40,12 @@ pub enum ConfigLoadError {
     Error(IOError),
 }
 
+impl Default for ConfigLoadError {
+    fn default() -> Self {
+        ConfigLoadError::Error(IOError::new(std::io::ErrorKind::NotFound, "place holder"))
+    }
+}
+
 impl Display for ConfigLoadError {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
@@ -43,17 +56,72 @@ impl Display for ConfigLoadError {
 }
 
 impl Config {
-    pub fn load(config_path: PathBuf) -> Result<Config, ConfigLoadError> {
-        match std::fs::read_to_string(config_path) {
-            Ok(config) => toml::from_str(&config)
-                .map(merge_keys)
-                .map_err(ConfigLoadError::BadConfig),
-            Err(err) => Err(ConfigLoadError::Error(err)),
-        }
+    pub fn load(
+        global: Result<String, ConfigLoadError>,
+        local: Result<String, ConfigLoadError>,
+    ) -> Result<Config, ConfigLoadError> {
+        let global_config: Result<ConfigRaw, ConfigLoadError> =
+            global.and_then(|file| toml::from_str(&file).map_err(ConfigLoadError::BadConfig));
+        let local_config: Result<ConfigRaw, ConfigLoadError> =
+            local.and_then(|file| toml::from_str(&file).map_err(ConfigLoadError::BadConfig));
+        let res = match (global_config, local_config) {
+            (Ok(global), Ok(local)) => {
+                let mut keys = keymap::default();
+                if let Some(global_keys) = global.keys {
+                    merge_keys(&mut keys, global_keys)
+                }
+                if let Some(local_keys) = local.keys {
+                    merge_keys(&mut keys, local_keys)
+                }
+
+                let editor = match (global.editor, local.editor) {
+                    (None, None) => helix_view::editor::Config::default(),
+                    (None, Some(val)) | (Some(val), None) => {
+                        val.try_into().map_err(ConfigLoadError::BadConfig)?
+                    }
+                    (Some(global), Some(local)) => merge_toml_values(global, local, 3)
+                        .try_into()
+                        .map_err(ConfigLoadError::BadConfig)?,
+                };
+
+                Config {
+                    theme: local.theme.or(global.theme),
+                    keys,
+                    editor,
+                }
+            }
+            // if any configs are invalid return that first
+            (_, Err(ConfigLoadError::BadConfig(err)))
+            | (Err(ConfigLoadError::BadConfig(err)), _) => {
+                return Err(ConfigLoadError::BadConfig(err))
+            }
+            (Ok(config), Err(_)) | (Err(_), Ok(config)) => {
+                let mut keys = keymap::default();
+                if let Some(keymap) = config.keys {
+                    merge_keys(&mut keys, keymap);
+                }
+                Config {
+                    theme: config.theme,
+                    keys,
+                    editor: config.editor.map_or_else(
+                        || Ok(helix_view::editor::Config::default()),
+                        |val| val.try_into().map_err(ConfigLoadError::BadConfig),
+                    )?,
+                }
+            }
+            // these are just two io errors return the one for the global config
+            (Err(err), Err(_)) => return Err(err),
+        };
+
+        Ok(res)
     }
 
     pub fn load_default() -> Result<Config, ConfigLoadError> {
-        Config::load(helix_loader::config_file())
+        let global_config =
+            fs::read_to_string(helix_loader::config_file()).map_err(ConfigLoadError::Error);
+        let local_config = fs::read_to_string(helix_loader::workspace_config_file())
+            .map_err(ConfigLoadError::Error);
+        Config::load(global_config, local_config)
     }
 }
 
@@ -61,6 +129,12 @@ impl Config {
 mod tests {
     use super::*;
 
+    impl Config {
+        fn load_test(config: &str) -> Config {
+            Config::load(Ok(config.to_owned()), Err(ConfigLoadError::default())).unwrap()
+        }
+    }
+
     #[test]
     fn parsing_keymaps_config_file() {
         use crate::keymap;
@@ -77,18 +151,24 @@ mod tests {
             A-F12 = "move_next_word_end"
         "#;
 
+        let mut keys = keymap::default();
+        merge_keys(
+            &mut keys,
+            hashmap! {
+                Mode::Insert => Keymap::new(keymap!({ "Insert mode"
+                    "y" => move_line_down,
+                    "S-C-a" => delete_selection,
+                })),
+                Mode::Normal => Keymap::new(keymap!({ "Normal mode"
+                    "A-F12" => move_next_word_end,
+                })),
+            },
+        );
+
         assert_eq!(
-            toml::from_str::<Config>(sample_keymaps).unwrap(),
+            Config::load_test(sample_keymaps),
             Config {
-                keys: hashmap! {
-                    Mode::Insert => Keymap::new(keymap!({ "Insert mode"
-                        "y" => move_line_down,
-                        "S-C-a" => delete_selection,
-                    })),
-                    Mode::Normal => Keymap::new(keymap!({ "Normal mode"
-                        "A-F12" => move_next_word_end,
-                    })),
-                },
+                keys,
                 ..Default::default()
             }
         );
@@ -97,11 +177,11 @@ mod tests {
     #[test]
     fn keys_resolve_to_correct_defaults() {
         // From serde default
-        let default_keys = toml::from_str::<Config>("").unwrap().keys;
-        assert_eq!(default_keys, default());
+        let default_keys = Config::load_test("").keys;
+        assert_eq!(default_keys, keymap::default());
 
         // From the Default trait
         let default_keys = Config::default().keys;
-        assert_eq!(default_keys, default());
+        assert_eq!(default_keys, keymap::default());
     }
 }

helix-term/src/health.rs

@@ -52,7 +52,7 @@ pub fn general() -> std::io::Result<()> {
     let config_file = helix_loader::config_file();
     let lang_file = helix_loader::lang_config_file();
     let log_file = helix_loader::log_file();
-    let rt_dir = helix_loader::runtime_dir();
+    let rt_dirs = helix_loader::runtime_dirs();
     let clipboard_provider = get_clipboard_provider();
 
     if config_file.exists() {
@@ -66,17 +66,31 @@ pub fn general() -> std::io::Result<()> {
         writeln!(stdout, "Language file: default")?;
     }
     writeln!(stdout, "Log file: {}", log_file.display())?;
-    writeln!(stdout, "Runtime directory: {}", rt_dir.display())?;
-
-    if let Ok(path) = std::fs::read_link(&rt_dir) {
-        let msg = format!("Runtime directory is symlinked to {}", path.display());
-        writeln!(stdout, "{}", msg.yellow())?;
-    }
-    if !rt_dir.exists() {
-        writeln!(stdout, "{}", "Runtime directory does not exist.".red())?;
-    }
-    if rt_dir.read_dir().ok().map(|it| it.count()) == Some(0) {
-        writeln!(stdout, "{}", "Runtime directory is empty.".red())?;
+    writeln!(
+        stdout,
+        "Runtime directories: {}",
+        rt_dirs
+            .iter()
+            .map(|d| d.to_string_lossy())
+            .collect::<Vec<_>>()
+            .join(";")
+    )?;
+    for rt_dir in rt_dirs.iter() {
+        if let Ok(path) = std::fs::read_link(rt_dir) {
+            let msg = format!(
+                "Runtime directory {} is symlinked to: {}",
+                rt_dir.display(),
+                path.display()
+            );
+            writeln!(stdout, "{}", msg.yellow())?;
+        }
+        if !rt_dir.exists() {
+            let msg = format!("Runtime directory does not exist: {}", rt_dir.display());
+            writeln!(stdout, "{}", msg.yellow())?;
+        } else if rt_dir.read_dir().ok().map(|it| it.count()) == Some(0) {
+            let msg = format!("Runtime directory is empty: {}", rt_dir.display());
+            writeln!(stdout, "{}", msg.yellow())?;
+        }
     }
     writeln!(stdout, "Clipboard provider: {}", clipboard_provider.name())?;
 
@@ -281,9 +295,9 @@ fn probe_protocol(protocol_name: &str, server_cmd: Option<String>) -> std::io::R
     writeln!(stdout, "Configured {}: {}", protocol_name, cmd_name)?;
 
     if let Some(cmd) = server_cmd {
-        let path = match which::which(cmd) {
+        let path = match which::which(&cmd) {
             Ok(path) => path.display().to_string().green(),
-            Err(_) => "Not found in $PATH".to_string().red(),
+            Err(_) => format!("'{}' not found in $PATH", cmd).red(),
         };
         writeln!(stdout, "Binary for {}: {}", protocol_name, path)?;
     }

helix-term/src/job.rs

@@ -5,9 +5,12 @@ use crate::compositor::Compositor;
 use futures_util::future::{BoxFuture, Future, FutureExt};
 use futures_util::stream::{FuturesUnordered, StreamExt};
 
+pub type EditorCompositorCallback = Box<dyn FnOnce(&mut Editor, &mut Compositor) + Send>;
+pub type EditorCallback = Box<dyn FnOnce(&mut Editor) + Send>;
+
 pub enum Callback {
-    EditorCompositor(Box<dyn FnOnce(&mut Editor, &mut Compositor) + Send>),
-    Editor(Box<dyn FnOnce(&mut Editor) + Send>),
+    EditorCompositor(EditorCompositorCallback),
+    Editor(EditorCallback),
 }
 
 pub type JobFuture = BoxFuture<'static, anyhow::Result<Option<Callback>>>;

helix-term/src/keymap.rs

@@ -2,7 +2,6 @@ pub mod default;
 pub mod macros;
 
 pub use crate::commands::MappableCommand;
-use crate::config::Config;
 use arc_swap::{
     access::{DynAccess, DynGuard},
     ArcSwap,
@@ -16,7 +15,7 @@ use std::{
     sync::Arc,
 };
 
-use default::default;
+pub use default::default;
 use macros::key;
 
 #[derive(Debug, Clone)]
@@ -184,7 +183,7 @@ impl<'de> serde::de::Visitor<'de> for KeyTrieVisitor {
         S: serde::de::SeqAccess<'de>,
     {
         let mut commands = Vec::new();
-        while let Some(command) = seq.next_element::<&str>()? {
+        while let Some(command) = seq.next_element::<String>()? {
             commands.push(
                 command
                     .parse::<MappableCommand>()
@@ -417,12 +416,10 @@ impl Default for Keymaps {
 }
 
 /// Merge default config keys with user overwritten keys for custom user config.
-pub fn merge_keys(mut config: Config) -> Config {
-    let mut delta = std::mem::replace(&mut config.keys, default());
-    for (mode, keys) in &mut config.keys {
+pub fn merge_keys(dst: &mut HashMap<Mode, Keymap>, mut delta: HashMap<Mode, Keymap>) {
+    for (mode, keys) in dst {
         keys.merge(delta.remove(mode).unwrap_or_default())
     }
-    config
 }
 
 #[cfg(test)]
@@ -449,26 +446,24 @@ mod tests {
 
     #[test]
     fn merge_partial_keys() {
-        let config = Config {
-            keys: hashmap! {
-                Mode::Normal => Keymap::new(
-                    keymap!({ "Normal mode"
-                        "i" => normal_mode,
-                        "无" => insert_mode,
-                        "z" => jump_backward,
-                        "g" => { "Merge into goto mode"
-                            "$" => goto_line_end,
-                            "g" => delete_char_forward,
-                        },
-                    })
-                )
-            },
-            ..Default::default()
+        let keymap = hashmap! {
+            Mode::Normal => Keymap::new(
+                keymap!({ "Normal mode"
+                    "i" => normal_mode,
+                    "无" => insert_mode,
+                    "z" => jump_backward,
+                    "g" => { "Merge into goto mode"
+                        "$" => goto_line_end,
+                        "g" => delete_char_forward,
+                    },
+                })
+            )
         };
-        let mut merged_config = merge_keys(config.clone());
-        assert_ne!(config, merged_config);
+        let mut merged_keyamp = default();
+        merge_keys(&mut merged_keyamp, keymap.clone());
+        assert_ne!(keymap, merged_keyamp);
 
-        let mut keymap = Keymaps::new(Box::new(Constant(merged_config.keys.clone())));
+        let mut keymap = Keymaps::new(Box::new(Constant(merged_keyamp.clone())));
         assert_eq!(
             keymap.get(Mode::Normal, key!('i')),
             KeymapResult::Matched(MappableCommand::normal_mode),
@@ -486,7 +481,7 @@ mod tests {
             "Leaf should replace node"
         );
 
-        let keymap = merged_config.keys.get_mut(&Mode::Normal).unwrap();
+        let keymap = merged_keyamp.get_mut(&Mode::Normal).unwrap();
         // Assumes that `g` is a node in default keymap
         assert_eq!(
             keymap.root().search(&[key!('g'), key!('$')]).unwrap(),
@@ -506,30 +501,28 @@ mod tests {
             "Old leaves in subnode should be present in merged node"
         );
 
-        assert!(merged_config.keys.get(&Mode::Normal).unwrap().len() > 1);
-        assert!(merged_config.keys.get(&Mode::Insert).unwrap().len() > 0);
+        assert!(merged_keyamp.get(&Mode::Normal).unwrap().len() > 1);
+        assert!(merged_keyamp.get(&Mode::Insert).unwrap().len() > 0);
     }
 
     #[test]
     fn order_should_be_set() {
-        let config = Config {
-            keys: hashmap! {
-                Mode::Normal => Keymap::new(
-                    keymap!({ "Normal mode"
-                        "space" => { ""
-                            "s" => { ""
-                                "v" => vsplit,
-                                "c" => hsplit,
-                            },
+        let keymap = hashmap! {
+            Mode::Normal => Keymap::new(
+                keymap!({ "Normal mode"
+                    "space" => { ""
+                        "s" => { ""
+                            "v" => vsplit,
+                            "c" => hsplit,
                         },
-                    })
-                )
-            },
-            ..Default::default()
+                    },
+                })
+            )
         };
-        let mut merged_config = merge_keys(config.clone());
-        assert_ne!(config, merged_config);
-        let keymap = merged_config.keys.get_mut(&Mode::Normal).unwrap();
+        let mut merged_keyamp = default();
+        merge_keys(&mut merged_keyamp, keymap.clone());
+        assert_ne!(keymap, merged_keyamp);
+        let keymap = merged_keyamp.get_mut(&Mode::Normal).unwrap();
         // Make sure mapping works
         assert_eq!(
             keymap
@@ -600,4 +593,43 @@ mod tests {
             "Mismatch"
         )
     }
+
+    #[test]
+    fn escaped_keymap() {
+        use crate::commands::MappableCommand;
+        use helix_view::input::{KeyCode, KeyEvent, KeyModifiers};
+
+        let keys = r#"
+"+" = [
+    "select_all",
+    ":pipe sed -E 's/\\s+$//g'",
+]
+        "#;
+
+        let key = KeyEvent {
+            code: KeyCode::Char('+'),
+            modifiers: KeyModifiers::NONE,
+        };
+
+        let expectation = Keymap::new(KeyTrie::Node(KeyTrieNode::new(
+            "",
+            hashmap! {
+                key => KeyTrie::Sequence(vec!{
+                    MappableCommand::select_all,
+                    MappableCommand::Typable {
+                        name: "pipe".to_string(),
+                        args: vec!{
+                            "sed".to_string(),
+                            "-E".to_string(),
+                            "'s/\\s+$//g'".to_string()
+                        },
+                        doc: "".to_string(),
+                    },
+                })
+            },
+            vec![key],
+        )));
+
+        assert_eq!(toml::from_str(keys), Ok(expectation));
+    }
 }

helix-term/src/keymap/default.rs

@@ -7,8 +7,8 @@ use helix_core::hashmap;
 pub fn default() -> HashMap<Mode, Keymap> {
     let normal = keymap!({ "Normal mode"
         "h" | "left" => move_char_left,
-        "j" | "down" => move_line_down,
-        "k" | "up" => move_line_up,
+        "j" | "down" => move_visual_line_down,
+        "k" | "up" => move_visual_line_up,
         "l" | "right" => move_char_right,
 
         "t" => find_till_char,
@@ -44,6 +44,7 @@ pub fn default() -> HashMap<Mode, Keymap> {
             "l" => goto_line_end,
             "s" => goto_first_nonwhitespace,
             "d" => goto_definition,
+            "D" => goto_declaration,
             "y" => goto_type_definition,
             "r" => goto_reference,
             "i" => goto_implementation,
@@ -54,6 +55,8 @@ pub fn default() -> HashMap<Mode, Keymap> {
             "m" => goto_last_modified_file,
             "n" => goto_next_buffer,
             "p" => goto_previous_buffer,
+            "k" => move_line_up,
+            "j" => move_line_down,
             "." => goto_last_modification,
         },
         ":" => command_mode,
@@ -76,6 +79,7 @@ pub fn default() -> HashMap<Mode, Keymap> {
 
         "s" => select_regex,
         "A-s" => split_selection_on_newline,
+        "A-_" => merge_consecutive_selections,
         "S" => split_selection,
         ";" => collapse_selection,
         "A-;" => flip_selections,
@@ -219,6 +223,7 @@ pub fn default() -> HashMap<Mode, Keymap> {
             "'" => last_picker,
             "g" => { "Debug (experimental)" sticky=true
                 "l" => dap_launch,
+                "r" => dap_restart,
                 "b" => dap_toggle_breakpoint,
                 "c" => dap_continue,
                 "h" => dap_pause,
@@ -319,8 +324,8 @@ pub fn default() -> HashMap<Mode, Keymap> {
     let mut select = normal.clone();
     select.merge_nodes(keymap!({ "Select mode"
         "h" | "left" => extend_char_left,
-        "j" | "down" => extend_line_down,
-        "k" | "up" => extend_line_up,
+        "j" | "down" => extend_visual_line_down,
+        "k" | "up" => extend_visual_line_up,
         "l" | "right" => extend_char_right,
 
         "w" => extend_next_word_start,
@@ -343,6 +348,10 @@ pub fn default() -> HashMap<Mode, Keymap> {
         "esc" => exit_select_mode,
 
         "v" => normal_mode,
+        "g" => { "Goto"
+            "k" => extend_line_up,
+            "j" => extend_line_down,
+        },
     }));
     let insert = keymap!({ "Insert mode"
         "esc" => normal_mode,
@@ -355,13 +364,13 @@ pub fn default() -> HashMap<Mode, Keymap> {
         "A-d" | "A-del" => delete_word_forward,
         "C-u" => kill_to_line_start,
         "C-k" => kill_to_line_end,
-        "C-h" | "backspace" => delete_char_backward,
+        "C-h" | "backspace" | "S-backspace" => delete_char_backward,
         "C-d" | "del" => delete_char_forward,
         "C-j" | "ret" => insert_newline,
         "tab" => insert_tab,
 
-        "up" => move_line_up,
-        "down" => move_line_down,
+        "up" => move_visual_line_up,
+        "down" => move_visual_line_down,
         "left" => move_char_left,
         "right" => move_char_right,
         "pageup" => page_up,

helix-term/src/lib.rs

@@ -10,6 +10,9 @@ pub mod health;
 pub mod job;
 pub mod keymap;
 pub mod ui;
+use std::path::Path;
+
+use ignore::DirEntry;
 pub use keymap::macros::*;
 
 #[cfg(not(windows))]
@@ -22,3 +25,25 @@ fn true_color() -> bool {
 fn true_color() -> bool {
     true
 }
+
+/// Function used for filtering dir entries in the various file pickers.
+fn filter_picker_entry(entry: &DirEntry, root: &Path, dedup_symlinks: bool) -> bool {
+    // We always want to ignore the .git directory, otherwise if
+    // `ignore` is turned off, we end up with a lot of noise
+    // in our picker.
+    if entry.file_name() == ".git" {
+        return false;
+    }
+
+    // We also ignore symlinks that point inside the current directory
+    // if `dedup_links` is enabled.
+    if dedup_symlinks && entry.path_is_symlink() {
+        return entry
+            .path()
+            .canonicalize()
+            .ok()
+            .map_or(false, |path| !path.starts_with(root));
+    }
+
+    true
+}

helix-term/src/main.rs

@@ -3,7 +3,7 @@ use crossterm::event::EventStream;
 use helix_loader::VERSION_AND_GIT_HASH;
 use helix_term::application::Application;
 use helix_term::args::Args;
-use helix_term::config::Config;
+use helix_term::config::{Config, ConfigLoadError};
 use std::path::PathBuf;
 
 fn setup_logging(logpath: PathBuf, verbosity: u64) -> Result<()> {
@@ -126,18 +126,19 @@ FLAGS:
 
     helix_loader::initialize_config_file(args.config_file.clone());
 
-    let config = match std::fs::read_to_string(helix_loader::config_file()) {
-        Ok(config) => toml::from_str(&config)
-            .map(helix_term::keymap::merge_keys)
-            .unwrap_or_else(|err| {
-                eprintln!("Bad config: {}", err);
-                eprintln!("Press <ENTER> to continue with default config");
-                use std::io::Read;
-                let _ = std::io::stdin().read(&mut []);
-                Config::default()
-            }),
-        Err(err) if err.kind() == std::io::ErrorKind::NotFound => Config::default(),
-        Err(err) => return Err(Error::new(err)),
+    let config = match Config::load_default() {
+        Ok(config) => config,
+        Err(ConfigLoadError::Error(err)) if err.kind() == std::io::ErrorKind::NotFound => {
+            Config::default()
+        }
+        Err(ConfigLoadError::Error(err)) => return Err(Error::new(err)),
+        Err(ConfigLoadError::BadConfig(err)) => {
+            eprintln!("Bad config: {}", err);
+            eprintln!("Press <ENTER> to continue with default config");
+            use std::io::Read;
+            let _ = std::io::stdin().read(&mut []);
+            Config::default()
+        }
     };
 
     let syn_loader_conf = helix_core::config::user_syntax_loader().unwrap_or_else(|err| {

helix-term/src/ui/completion.rs

@@ -1,22 +1,21 @@
 use crate::compositor::{Component, Context, Event, EventResult};
-use helix_view::{apply_transaction, editor::CompleteAction, ViewId};
-use tui::buffer::Buffer as Surface;
-use tui::text::Spans;
+use helix_view::{
+    document::SavePoint,
+    editor::CompleteAction,
+    theme::{Modifier, Style},
+    ViewId,
+};
+use tui::{buffer::Buffer as Surface, text::Span};
 
-use std::borrow::Cow;
+use std::{borrow::Cow, sync::Arc};
 
 use helix_core::{Change, Transaction};
-use helix_view::{
-    graphics::Rect,
-    input::{KeyCode, KeyEvent},
-    Document, Editor,
-};
+use helix_view::{graphics::Rect, Document, Editor};
 
 use crate::commands;
 use crate::ui::{menu, Markdown, Menu, Popup, PromptEvent};
 
 use helix_lsp::{lsp, util};
-use lsp::CompletionItem;
 
 impl menu::Item for CompletionItem {
     type Data = ();
@@ -26,21 +25,29 @@ impl menu::Item for CompletionItem {
 
     #[inline]
     fn filter_text(&self, _data: &Self::Data) -> Cow<str> {
-        self.filter_text
+        self.item
+            .filter_text
             .as_ref()
-            .unwrap_or(&self.label)
+            .unwrap_or(&self.item.label)
             .as_str()
             .into()
     }
 
-    fn label(&self, _data: &Self::Data) -> Spans {
-        self.label.as_str().into()
-    }
-
-    fn row(&self, _data: &Self::Data) -> menu::Row {
+    fn format(&self, _data: &Self::Data) -> menu::Row {
+        let deprecated = self.item.deprecated.unwrap_or_default()
+            || self.item.tags.as_ref().map_or(false, |tags| {
+                tags.contains(&lsp::CompletionItemTag::DEPRECATED)
+            });
         menu::Row::new(vec![
-            menu::Cell::from(self.label.as_str()),
-            menu::Cell::from(match self.kind {
+            menu::Cell::from(Span::styled(
+                self.item.label.as_str(),
+                if deprecated {
+                    Style::default().add_modifier(Modifier::CROSSED_OUT)
+                } else {
+                    Style::default()
+                },
+            )),
+            menu::Cell::from(match self.item.kind {
                 Some(lsp::CompletionItemKind::TEXT) => "text",
                 Some(lsp::CompletionItemKind::METHOD) => "method",
                 Some(lsp::CompletionItemKind::FUNCTION) => "function",
@@ -81,6 +88,12 @@ impl menu::Item for CompletionItem {
     }
 }
 
+#[derive(Debug, PartialEq, Default, Clone)]
+struct CompletionItem {
+    item: lsp::CompletionItem,
+    resolved: bool,
+}
+
 /// Wraps a Menu.
 pub struct Completion {
     popup: Popup<Menu<CompletionItem>>,
@@ -95,13 +108,22 @@ impl Completion {
 
     pub fn new(
         editor: &Editor,
-        mut items: Vec<CompletionItem>,
+        savepoint: Arc<SavePoint>,
+        mut items: Vec<lsp::CompletionItem>,
         offset_encoding: helix_lsp::OffsetEncoding,
         start_offset: usize,
         trigger_offset: usize,
     ) -> Self {
+        let replace_mode = editor.config().completion_replace;
         // Sort completion items according to their preselect status (given by the LSP server)
         items.sort_by_key(|item| !item.preselect.unwrap_or(false));
+        let items = items
+            .into_iter()
+            .map(|item| CompletionItem {
+                item,
+                resolved: false,
+            })
+            .collect();
 
         // Then create the menu
         let menu = Menu::new(items, (), move |editor: &mut Editor, item, event| {
@@ -110,50 +132,86 @@ impl Completion {
                 view_id: ViewId,
                 item: &CompletionItem,
                 offset_encoding: helix_lsp::OffsetEncoding,
-                start_offset: usize,
                 trigger_offset: usize,
+                include_placeholder: bool,
+                replace_mode: bool,
             ) -> Transaction {
-                let transaction = if let Some(edit) = &item.text_edit {
+                use helix_lsp::snippet;
+                let selection = doc.selection(view_id);
+                let text = doc.text().slice(..);
+                let primary_cursor = selection.primary().cursor(text);
+
+                let (edit_offset, new_text) = if let Some(edit) = &item.item.text_edit {
                     let edit = match edit {
                         lsp::CompletionTextEdit::Edit(edit) => edit.clone(),
                         lsp::CompletionTextEdit::InsertAndReplace(item) => {
-                            // TODO: support using "insert" instead of "replace" via user config
-                            lsp::TextEdit::new(item.replace, item.new_text.clone())
+                            let range = if replace_mode {
+                                item.replace
+                            } else {
+                                item.insert
+                            };
+                            lsp::TextEdit::new(range, item.new_text.clone())
                         }
                     };
 
-                    util::generate_transaction_from_completion_edit(
-                        doc.text(),
-                        doc.selection(view_id),
-                        edit,
-                        offset_encoding, // TODO: should probably transcode in Client
-                    )
+                    let Some(range) = util::lsp_range_to_range(doc.text(), edit.range, offset_encoding) else{
+                        return Transaction::new(doc.text());
+                    };
+
+                    let start_offset = range.anchor as i128 - primary_cursor as i128;
+                    let end_offset = range.head as i128 - primary_cursor as i128;
+
+                    (Some((start_offset, end_offset)), edit.new_text)
                 } else {
-                    let text = item.insert_text.as_ref().unwrap_or(&item.label);
-                    // Some LSPs just give you an insertText with no offset ¯\_(ツ)_/¯
-                    // in these cases we need to check for a common prefix and remove it
-                    let prefix = Cow::from(doc.text().slice(start_offset..trigger_offset));
-                    let text = text.trim_start_matches::<&str>(&prefix);
-
-                    // TODO: this needs to be true for the numbers to work out correctly
-                    // in the closure below. It's passed in to a callback as this same
-                    // formula, but can the value change between the LSP request and
-                    // response? If it does, can we recover?
-                    debug_assert!(
-                        doc.selection(view_id)
-                            .primary()
-                            .cursor(doc.text().slice(..))
-                            == trigger_offset
-                    );
-
-                    Transaction::change_by_selection(doc.text(), doc.selection(view_id), |range| {
-                        let cursor = range.cursor(doc.text().slice(..));
-
-                        (cursor, cursor, Some(text.into()))
-                    })
+                    let new_text = item
+                        .item
+                        .insert_text
+                        .clone()
+                        .unwrap_or_else(|| item.item.label.clone());
+                    // check that we are still at the correct savepoint
+                    // we can still generate a transaction regardless but if the
+                    // document changed (and not just the selection) then we will
+                    // likely delete the wrong text (same if we applied an edit sent by the LS)
+                    debug_assert!(primary_cursor == trigger_offset);
+                    (None, new_text)
                 };
 
-                transaction
+                if matches!(item.item.kind, Some(lsp::CompletionItemKind::SNIPPET))
+                    || matches!(
+                        item.item.insert_text_format,
+                        Some(lsp::InsertTextFormat::SNIPPET)
+                    )
+                {
+                    match snippet::parse(&new_text) {
+                        Ok(snippet) => util::generate_transaction_from_snippet(
+                            doc.text(),
+                            selection,
+                            edit_offset,
+                            replace_mode,
+                            snippet,
+                            doc.line_ending.as_str(),
+                            include_placeholder,
+                            doc.tab_width(),
+                            doc.indent_width(),
+                        ),
+                        Err(err) => {
+                            log::error!(
+                                "Failed to parse snippet: {:?}, remaining output: {}",
+                                &new_text,
+                                err
+                            );
+                            Transaction::new(doc.text())
+                        }
+                    }
+                } else {
+                    util::generate_transaction_from_completion_edit(
+                        doc.text(),
+                        selection,
+                        edit_offset,
+                        replace_mode,
+                        new_text,
+                    )
+                }
             }
 
             fn completion_changes(transaction: &Transaction, trigger_offset: usize) -> Vec<Change> {
@@ -165,15 +223,27 @@ impl Completion {
 
             let (view, doc) = current!(editor);
 
-            // if more text was entered, remove it
-            doc.restore(view);
-
             match event {
-                PromptEvent::Abort => {
-                    doc.restore(view);
-                    editor.last_completion = None;
-                }
+                PromptEvent::Abort => {}
                 PromptEvent::Update => {
+                    // Update creates "ghost" transactions which are not sent to the
+                    // lsp server to avoid messing up re-requesting completions. Once a
+                    // completion has been selected (with tab, c-n or c-p) it's always accepted whenever anything
+                    // is typed. The only way to avoid that is to explicitly abort the completion
+                    // with c-c. This will remove the "ghost" transaction.
+                    //
+                    // The ghost transaction is modeled with a transaction that is not sent to the LS.
+                    // (apply_temporary) and a savepoint. It's extremely important this savepoint is restored
+                    // (also without sending the transaction to the LS) *before any further transaction is applied*.
+                    // Otherwise incremental sync breaks (since the state of the LS doesn't match the state the transaction
+                    // is applied to).
+                    if editor.last_completion.is_none() {
+                        editor.last_completion = Some(CompleteAction::Selected {
+                            savepoint: doc.savepoint(view),
+                        })
+                    }
+                    // if more text was entered, remove it
+                    doc.restore(view, &savepoint, false);
                     // always present here
                     let item = item.unwrap();
 
@@ -182,69 +252,64 @@ impl Completion {
                         view.id,
                         item,
                         offset_encoding,
-                        start_offset,
                         trigger_offset,
+                        true,
+                        replace_mode,
                     );
-
-                    // initialize a savepoint
-                    doc.savepoint();
-                    apply_transaction(&transaction, doc, view);
-
-                    editor.last_completion = Some(CompleteAction {
-                        trigger_offset,
-                        changes: completion_changes(&transaction, trigger_offset),
-                    });
+                    doc.apply_temporary(&transaction, view.id);
                 }
                 PromptEvent::Validate => {
+                    if let Some(CompleteAction::Selected { savepoint }) =
+                        editor.last_completion.take()
+                    {
+                        doc.restore(view, &savepoint, false);
+                    }
                     // always present here
-                    let item = item.unwrap();
+                    let mut item = item.unwrap().clone();
 
+                    // resolve item if not yet resolved
+                    if !item.resolved {
+                        if let Some(resolved) =
+                            Self::resolve_completion_item(doc, item.item.clone())
+                        {
+                            item.item = resolved;
+                        }
+                    };
+                    // if more text was entered, remove it
+                    doc.restore(view, &savepoint, true);
                     let transaction = item_to_transaction(
                         doc,
                         view.id,
-                        item,
+                        &item,
                         offset_encoding,
-                        start_offset,
                         trigger_offset,
+                        false,
+                        replace_mode,
                     );
+                    doc.apply(&transaction, view.id);
 
-                    apply_transaction(&transaction, doc, view);
-
-                    editor.last_completion = Some(CompleteAction {
+                    editor.last_completion = Some(CompleteAction::Applied {
                         trigger_offset,
                         changes: completion_changes(&transaction, trigger_offset),
                     });
 
-                    // apply additional edits, mostly used to auto import unqualified types
-                    let resolved_item = if item
-                        .additional_text_edits
-                        .as_ref()
-                        .map(|edits| !edits.is_empty())
-                        .unwrap_or(false)
-                    {
-                        None
-                    } else {
-                        Self::resolve_completion_item(doc, item.clone())
-                    };
-
-                    if let Some(additional_edits) = resolved_item
-                        .as_ref()
-                        .and_then(|item| item.additional_text_edits.as_ref())
-                        .or(item.additional_text_edits.as_ref())
-                    {
+                    // TODO: add additional _edits to completion_changes?
+                    if let Some(additional_edits) = item.item.additional_text_edits {
                         if !additional_edits.is_empty() {
                             let transaction = util::generate_transaction_from_edits(
                                 doc.text(),
-                                additional_edits.clone(),
+                                additional_edits,
                                 offset_encoding, // TODO: should probably transcode in Client
                             );
-                            apply_transaction(&transaction, doc, view);
+                            doc.apply(&transaction, view.id);
                         }
                     }
                 }
             };
         });
-        let popup = Popup::new(Self::ID, menu).with_scrollbar(false);
+        let popup = Popup::new(Self::ID, menu)
+            .with_scrollbar(false)
+            .ignore_escape_key(true);
         let mut completion = Self {
             popup,
             start_offset,
@@ -260,7 +325,7 @@ impl Completion {
     fn resolve_completion_item(
         doc: &Document,
         completion_item: lsp::CompletionItem,
-    ) -> Option<CompletionItem> {
+    ) -> Option<lsp::CompletionItem> {
         let language_server = doc.language_server()?;
 
         let future = language_server.resolve_completion_item(completion_item)?;
@@ -313,7 +378,7 @@ impl Completion {
         self.popup.contents().is_empty()
     }
 
-    fn replace_item(&mut self, old_item: lsp::CompletionItem, new_item: lsp::CompletionItem) {
+    fn replace_item(&mut self, old_item: CompletionItem, new_item: CompletionItem) {
         self.popup.contents_mut().replace_option(old_item, new_item);
     }
 
@@ -329,7 +394,7 @@ impl Completion {
         // > The returned completion item should have the documentation property filled in.
         // https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_completion
         let current_item = match self.popup.contents().selection() {
-            Some(item) if item.documentation.is_none() => item.clone(),
+            Some(item) if !item.resolved => item.clone(),
             _ => return false,
         };
 
@@ -339,7 +404,7 @@ impl Completion {
         };
 
         // This method should not block the compositor so we handle the response asynchronously.
-        let future = match language_server.resolve_completion_item(current_item.clone()) {
+        let future = match language_server.resolve_completion_item(current_item.item.clone()) {
             Some(future) => future,
             None => return false,
         };
@@ -357,24 +422,27 @@ impl Completion {
                     .unwrap()
                     .completion
                 {
-                    completion.replace_item(current_item, resolved_item);
+                    completion.replace_item(
+                        current_item,
+                        CompletionItem {
+                            item: resolved_item,
+                            resolved: true,
+                        },
+                    );
                 }
             },
         );
 
         true
     }
+
+    pub fn area(&mut self, viewport: Rect, editor: &Editor) -> Rect {
+        self.popup.area(viewport, editor)
+    }
 }
 
 impl Component for Completion {
     fn handle_event(&mut self, event: &Event, cx: &mut Context) -> EventResult {
-        // let the Editor handle Esc instead
-        if let Event::Key(KeyEvent {
-            code: KeyCode::Esc, ..
-        }) = event
-        {
-            return EventResult::Ignored(None);
-        }
         self.popup.handle_event(event, cx)
     }
 
@@ -386,102 +454,102 @@ impl Component for Completion {
         self.popup.render(area, surface, cx);
 
         // if we have a selection, render a markdown popup on top/below with info
-        if let Some(option) = self.popup.contents().selection() {
-            // need to render:
-            // option.detail
-            // ---
-            // option.documentation
+        let option = match self.popup.contents().selection() {
+            Some(option) => option,
+            None => return,
+        };
+        // need to render:
+        // option.detail
+        // ---
+        // option.documentation
 
-            let (view, doc) = current!(cx.editor);
-            let language = doc.language_name().unwrap_or("");
-            let text = doc.text().slice(..);
-            let cursor_pos = doc.selection(view.id).primary().cursor(text);
-            let coords = helix_core::visual_coords_at_pos(text, cursor_pos, doc.tab_width());
-            let cursor_pos = (coords.row - view.offset.row) as u16;
+        let (view, doc) = current!(cx.editor);
+        let language = doc.language_name().unwrap_or("");
+        let text = doc.text().slice(..);
+        let cursor_pos = doc.selection(view.id).primary().cursor(text);
+        let coords = view
+            .screen_coords_at_pos(doc, text, cursor_pos)
+            .expect("cursor must be in view");
+        let cursor_pos = coords.row as u16;
 
-            let mut markdown_doc = match &option.documentation {
-                Some(lsp::Documentation::String(contents))
-                | Some(lsp::Documentation::MarkupContent(lsp::MarkupContent {
-                    kind: lsp::MarkupKind::PlainText,
-                    value: contents,
-                })) => {
-                    // TODO: convert to wrapped text
-                    Markdown::new(
-                        format!(
-                            "```{}\n{}\n```\n{}",
-                            language,
-                            option.detail.as_deref().unwrap_or_default(),
-                            contents
-                        ),
-                        cx.editor.syn_loader.clone(),
-                    )
-                }
-                Some(lsp::Documentation::MarkupContent(lsp::MarkupContent {
-                    kind: lsp::MarkupKind::Markdown,
-                    value: contents,
-                })) => {
-                    // TODO: set language based on doc scope
-                    if let Some(detail) = &option.detail.as_deref() {
-                        Markdown::new(
-                            format!("```{}\n{}\n```\n{}", language, detail, contents),
-                            cx.editor.syn_loader.clone(),
-                        )
-                    } else {
-                        Markdown::new(contents.to_string(), cx.editor.syn_loader.clone())
-                    }
-                }
-                None if option.detail.is_some() => {
-                    // TODO: copied from above
-
-                    // TODO: set language based on doc scope
-                    Markdown::new(
-                        format!(
-                            "```{}\n{}\n```",
-                            language,
-                            option.detail.as_deref().unwrap_or_default(),
-                        ),
-                        cx.editor.syn_loader.clone(),
-                    )
-                }
-                None => return,
+        let markdowned = |lang: &str, detail: Option<&str>, doc: Option<&str>| {
+            let md = match (detail, doc) {
+                (Some(detail), Some(doc)) => format!("```{lang}\n{detail}\n```\n{doc}"),
+                (Some(detail), None) => format!("```{lang}\n{detail}\n```"),
+                (None, Some(doc)) => doc.to_string(),
+                (None, None) => String::new(),
             };
+            Markdown::new(md, cx.editor.syn_loader.clone())
+        };
 
-            let (popup_x, popup_y) = self.popup.get_rel_position(area, cx);
-            let (popup_width, _popup_height) = self.popup.get_size();
-            let mut width = area
-                .width
-                .saturating_sub(popup_x)
-                .saturating_sub(popup_width);
-            let area = if width > 30 {
-                let mut height = area.height.saturating_sub(popup_y);
-                let x = popup_x + popup_width;
-                let y = popup_y;
+        let mut markdown_doc = match &option.item.documentation {
+            Some(lsp::Documentation::String(contents))
+            | Some(lsp::Documentation::MarkupContent(lsp::MarkupContent {
+                kind: lsp::MarkupKind::PlainText,
+                value: contents,
+            })) => {
+                // TODO: convert to wrapped text
+                markdowned(language, option.item.detail.as_deref(), Some(contents))
+            }
+            Some(lsp::Documentation::MarkupContent(lsp::MarkupContent {
+                kind: lsp::MarkupKind::Markdown,
+                value: contents,
+            })) => {
+                // TODO: set language based on doc scope
+                markdowned(language, option.item.detail.as_deref(), Some(contents))
+            }
+            None if option.item.detail.is_some() => {
+                // TODO: set language based on doc scope
+                markdowned(language, option.item.detail.as_deref(), None)
+            }
+            None => return,
+        };
 
-                if let Some((rel_width, rel_height)) = markdown_doc.required_size((width, height)) {
-                    width = rel_width.min(width);
-                    height = rel_height.min(height);
-                }
-                Rect::new(x, y, width, height)
+        let popup_area = {
+            let (popup_x, popup_y) = self.popup.get_rel_position(area, cx.editor);
+            let (popup_width, popup_height) = self.popup.get_size();
+            Rect::new(popup_x, popup_y, popup_width, popup_height)
+        };
+
+        let doc_width_available = area.width.saturating_sub(popup_area.right());
+        let doc_area = if doc_width_available > 30 {
+            let mut doc_width = doc_width_available;
+            let mut doc_height = area.height.saturating_sub(popup_area.top());
+            let x = popup_area.right();
+            let y = popup_area.top();
+
+            if let Some((rel_width, rel_height)) =
+                markdown_doc.required_size((doc_width, doc_height))
+            {
+                doc_width = rel_width.min(doc_width);
+                doc_height = rel_height.min(doc_height);
+            }
+            Rect::new(x, y, doc_width, doc_height)
+        } else {
+            // Documentation should not cover the cursor or the completion popup
+            // Completion popup could be above or below the current line
+            let avail_height_above = cursor_pos.min(popup_area.top()).saturating_sub(1);
+            let avail_height_below = area
+                .height
+                .saturating_sub(cursor_pos.max(popup_area.bottom()) + 1 /* padding */);
+            let (y, avail_height) = if avail_height_below >= avail_height_above {
+                (
+                    area.height.saturating_sub(avail_height_below),
+                    avail_height_below,
+                )
             } else {
-                let half = area.height / 2;
-                let height = 15.min(half);
-                // we want to make sure the cursor is visible (not hidden behind the documentation)
-                let y = if cursor_pos + area.y
-                    >= (cx.editor.tree.area().height - height - 2/* statusline + commandline */)
-                {
-                    0
-                } else {
-                    // -2 to subtract command line + statusline. a bit of a hack, because of splits.
-                    area.height.saturating_sub(height).saturating_sub(2)
-                };
-
-                Rect::new(0, y, area.width, height)
+                (0, avail_height_above)
             };
+            if avail_height <= 1 {
+                return;
+            }
 
-            // clear area
-            let background = cx.editor.theme.get("ui.popup");
-            surface.clear_with(area, background);
-            markdown_doc.render(area, surface, cx);
-        }
+            Rect::new(0, y, area.width, avail_height.min(15))
+        };
+
+        // clear area
+        let background = cx.editor.theme.get("ui.popup");
+        surface.clear_with(doc_area, background);
+        markdown_doc.render(doc_area, surface, cx);
     }
 }

helix-term/src/ui/document.rs

@@ -0,0 +1,484 @@
+use std::cmp::min;
+
+use helix_core::doc_formatter::{DocumentFormatter, GraphemeSource, TextFormat};
+use helix_core::graphemes::Grapheme;
+use helix_core::str_utils::char_to_byte_idx;
+use helix_core::syntax::Highlight;
+use helix_core::syntax::HighlightEvent;
+use helix_core::text_annotations::TextAnnotations;
+use helix_core::{visual_offset_from_block, Position, RopeSlice};
+use helix_view::editor::{WhitespaceConfig, WhitespaceRenderValue};
+use helix_view::graphics::Rect;
+use helix_view::theme::Style;
+use helix_view::view::ViewPosition;
+use helix_view::Document;
+use helix_view::Theme;
+use tui::buffer::Buffer as Surface;
+
+pub trait LineDecoration {
+    fn render_background(&mut self, _renderer: &mut TextRenderer, _pos: LinePos) {}
+    fn render_foreground(
+        &mut self,
+        _renderer: &mut TextRenderer,
+        _pos: LinePos,
+        _end_char_idx: usize,
+    ) {
+    }
+}
+
+impl<F: FnMut(&mut TextRenderer, LinePos)> LineDecoration for F {
+    fn render_background(&mut self, renderer: &mut TextRenderer, pos: LinePos) {
+        self(renderer, pos)
+    }
+}
+
+/// A wrapper around a HighlightIterator
+/// that merges the layered highlights to create the final text style
+/// and yields the active text style and the char_idx where the active
+/// style will have to be recomputed.
+struct StyleIter<'a, H: Iterator<Item = HighlightEvent>> {
+    text_style: Style,
+    active_highlights: Vec<Highlight>,
+    highlight_iter: H,
+    theme: &'a Theme,
+}
+
+impl<H: Iterator<Item = HighlightEvent>> Iterator for StyleIter<'_, H> {
+    type Item = (Style, usize);
+    fn next(&mut self) -> Option<(Style, usize)> {
+        while let Some(event) = self.highlight_iter.next() {
+            match event {
+                HighlightEvent::HighlightStart(highlights) => {
+                    self.active_highlights.push(highlights)
+                }
+                HighlightEvent::HighlightEnd => {
+                    self.active_highlights.pop();
+                }
+                HighlightEvent::Source { start, end } => {
+                    if start == end {
+                        continue;
+                    }
+                    let style = self
+                        .active_highlights
+                        .iter()
+                        .fold(self.text_style, |acc, span| {
+                            acc.patch(self.theme.highlight(span.0))
+                        });
+                    return Some((style, end));
+                }
+            }
+        }
+        None
+    }
+}
+
+#[derive(Debug, PartialEq, Eq, Copy, Clone)]
+pub struct LinePos {
+    /// Indicates whether the given visual line
+    /// is the first visual line of the given document line
+    pub first_visual_line: bool,
+    /// The line index of the document line that contains the given visual line
+    pub doc_line: usize,
+    /// Vertical offset from the top of the inner view area
+    pub visual_line: u16,
+    /// The first char index of this visual line.
+    /// Note that if the visual line is entirely filled by
+    /// a very long inline virtual text then this index will point
+    /// at the next (non-virtual) char after this visual line
+    pub start_char_idx: usize,
+}
+
+pub type TranslatedPosition<'a> = (usize, Box<dyn FnMut(&mut TextRenderer, Position) + 'a>);
+
+#[allow(clippy::too_many_arguments)]
+pub fn render_document(
+    surface: &mut Surface,
+    viewport: Rect,
+    doc: &Document,
+    offset: ViewPosition,
+    doc_annotations: &TextAnnotations,
+    highlight_iter: impl Iterator<Item = HighlightEvent>,
+    theme: &Theme,
+    line_decoration: &mut [Box<dyn LineDecoration + '_>],
+    translated_positions: &mut [TranslatedPosition],
+) {
+    let mut renderer = TextRenderer::new(surface, doc, theme, offset.horizontal_offset, viewport);
+    render_text(
+        &mut renderer,
+        doc.text().slice(..),
+        offset,
+        &doc.text_format(viewport.width, Some(theme)),
+        doc_annotations,
+        highlight_iter,
+        theme,
+        line_decoration,
+        translated_positions,
+    )
+}
+
+fn translate_positions(
+    char_pos: usize,
+    first_visible_char_idx: usize,
+    translated_positions: &mut [TranslatedPosition],
+    text_fmt: &TextFormat,
+    renderer: &mut TextRenderer,
+    pos: Position,
+) {
+    // check if any positions translated on the fly (like cursor) has been reached
+    for (char_idx, callback) in &mut *translated_positions {
+        if *char_idx < char_pos && *char_idx >= first_visible_char_idx {
+            // by replacing the char_index with usize::MAX large number we ensure
+            // that the same position is only translated once
+            // text will never reach usize::MAX as rust memory allocations are limited
+            // to isize::MAX
+            *char_idx = usize::MAX;
+
+            if text_fmt.soft_wrap {
+                callback(renderer, pos)
+            } else if pos.col >= renderer.col_offset
+                && pos.col - renderer.col_offset < renderer.viewport.width as usize
+            {
+                callback(
+                    renderer,
+                    Position {
+                        row: pos.row,
+                        col: pos.col - renderer.col_offset,
+                    },
+                )
+            }
+        }
+    }
+}
+
+#[allow(clippy::too_many_arguments)]
+pub fn render_text<'t>(
+    renderer: &mut TextRenderer,
+    text: RopeSlice<'t>,
+    offset: ViewPosition,
+    text_fmt: &TextFormat,
+    text_annotations: &TextAnnotations,
+    highlight_iter: impl Iterator<Item = HighlightEvent>,
+    theme: &Theme,
+    line_decorations: &mut [Box<dyn LineDecoration + '_>],
+    translated_positions: &mut [TranslatedPosition],
+) {
+    let (
+        Position {
+            row: mut row_off, ..
+        },
+        mut char_pos,
+    ) = visual_offset_from_block(
+        text,
+        offset.anchor,
+        offset.anchor,
+        text_fmt,
+        text_annotations,
+    );
+    row_off += offset.vertical_offset;
+
+    let (mut formatter, mut first_visible_char_idx) =
+        DocumentFormatter::new_at_prev_checkpoint(text, text_fmt, text_annotations, offset.anchor);
+    let mut styles = StyleIter {
+        text_style: renderer.text_style,
+        active_highlights: Vec::with_capacity(64),
+        highlight_iter,
+        theme,
+    };
+
+    let mut last_line_pos = LinePos {
+        first_visual_line: false,
+        doc_line: usize::MAX,
+        visual_line: u16::MAX,
+        start_char_idx: usize::MAX,
+    };
+    let mut is_in_indent_area = true;
+    let mut last_line_indent_level = 0;
+    let mut style_span = styles
+        .next()
+        .unwrap_or_else(|| (Style::default(), usize::MAX));
+
+    loop {
+        // formattter.line_pos returns to line index of the next grapheme
+        // so it must be called before formatter.next
+        let doc_line = formatter.line_pos();
+        let Some((grapheme, mut pos)) = formatter.next() else {
+            let mut last_pos = formatter.visual_pos();
+            if last_pos.row >= row_off {
+                last_pos.col -= 1;
+                last_pos.row -= row_off;
+                // check if any positions translated on the fly (like cursor) are at the EOF
+                translate_positions(
+                    char_pos + 1,
+                    first_visible_char_idx,
+                    translated_positions,
+                    text_fmt,
+                    renderer,
+                    last_pos,
+                );
+            }
+            break;
+        };
+
+        // skip any graphemes on visual lines before the block start
+        if pos.row < row_off {
+            if char_pos >= style_span.1 {
+                style_span = if let Some(style_span) = styles.next() {
+                    style_span
+                } else {
+                    break;
+                }
+            }
+            char_pos += grapheme.doc_chars();
+            first_visible_char_idx = char_pos + 1;
+            continue;
+        }
+        pos.row -= row_off;
+
+        // if the end of the viewport is reached stop rendering
+        if pos.row as u16 >= renderer.viewport.height {
+            break;
+        }
+
+        // apply decorations before rendering a new line
+        if pos.row as u16 != last_line_pos.visual_line {
+            if pos.row > 0 {
+                renderer.draw_indent_guides(last_line_indent_level, last_line_pos.visual_line);
+                is_in_indent_area = true;
+                for line_decoration in &mut *line_decorations {
+                    line_decoration.render_foreground(renderer, last_line_pos, char_pos);
+                }
+            }
+            last_line_pos = LinePos {
+                first_visual_line: doc_line != last_line_pos.doc_line,
+                doc_line,
+                visual_line: pos.row as u16,
+                start_char_idx: char_pos,
+            };
+            for line_decoration in &mut *line_decorations {
+                line_decoration.render_background(renderer, last_line_pos);
+            }
+        }
+
+        // acquire the correct grapheme style
+        if char_pos >= style_span.1 {
+            style_span = styles.next().unwrap_or((Style::default(), usize::MAX));
+        }
+        char_pos += grapheme.doc_chars();
+
+        // check if any positions translated on the fly (like cursor) has been reached
+        translate_positions(
+            char_pos,
+            first_visible_char_idx,
+            translated_positions,
+            text_fmt,
+            renderer,
+            pos,
+        );
+
+        let grapheme_style = if let GraphemeSource::VirtualText { highlight } = grapheme.source {
+            let style = renderer.text_style;
+            if let Some(highlight) = highlight {
+                style.patch(theme.highlight(highlight.0))
+            } else {
+                style
+            }
+        } else {
+            style_span.0
+        };
+
+        let virt = grapheme.is_virtual();
+        renderer.draw_grapheme(
+            grapheme.grapheme,
+            grapheme_style,
+            virt,
+            &mut last_line_indent_level,
+            &mut is_in_indent_area,
+            pos,
+        );
+    }
+
+    renderer.draw_indent_guides(last_line_indent_level, last_line_pos.visual_line);
+    for line_decoration in &mut *line_decorations {
+        line_decoration.render_foreground(renderer, last_line_pos, char_pos);
+    }
+}
+
+#[derive(Debug)]
+pub struct TextRenderer<'a> {
+    pub surface: &'a mut Surface,
+    pub text_style: Style,
+    pub whitespace_style: Style,
+    pub indent_guide_char: String,
+    pub indent_guide_style: Style,
+    pub newline: String,
+    pub nbsp: String,
+    pub space: String,
+    pub tab: String,
+    pub virtual_tab: String,
+    pub indent_width: u16,
+    pub starting_indent: usize,
+    pub draw_indent_guides: bool,
+    pub col_offset: usize,
+    pub viewport: Rect,
+}
+
+impl<'a> TextRenderer<'a> {
+    pub fn new(
+        surface: &'a mut Surface,
+        doc: &Document,
+        theme: &Theme,
+        col_offset: usize,
+        viewport: Rect,
+    ) -> TextRenderer<'a> {
+        let editor_config = doc.config.load();
+        let WhitespaceConfig {
+            render: ws_render,
+            characters: ws_chars,
+        } = &editor_config.whitespace;
+
+        let tab_width = doc.tab_width();
+        let tab = if ws_render.tab() == WhitespaceRenderValue::All {
+            std::iter::once(ws_chars.tab)
+                .chain(std::iter::repeat(ws_chars.tabpad).take(tab_width - 1))
+                .collect()
+        } else {
+            " ".repeat(tab_width)
+        };
+        let virtual_tab = " ".repeat(tab_width);
+        let newline = if ws_render.newline() == WhitespaceRenderValue::All {
+            ws_chars.newline.into()
+        } else {
+            " ".to_owned()
+        };
+
+        let space = if ws_render.space() == WhitespaceRenderValue::All {
+            ws_chars.space.into()
+        } else {
+            " ".to_owned()
+        };
+        let nbsp = if ws_render.nbsp() == WhitespaceRenderValue::All {
+            ws_chars.nbsp.into()
+        } else {
+            " ".to_owned()
+        };
+
+        let text_style = theme.get("ui.text");
+
+        let indent_width = doc.indent_style.indent_width(tab_width) as u16;
+
+        TextRenderer {
+            surface,
+            indent_guide_char: editor_config.indent_guides.character.into(),
+            newline,
+            nbsp,
+            space,
+            tab,
+            virtual_tab,
+            whitespace_style: theme.get("ui.virtual.whitespace"),
+            indent_width,
+            starting_indent: col_offset / indent_width as usize
+                + (col_offset % indent_width as usize != 0) as usize
+                + editor_config.indent_guides.skip_levels as usize,
+            indent_guide_style: text_style.patch(
+                theme
+                    .try_get("ui.virtual.indent-guide")
+                    .unwrap_or_else(|| theme.get("ui.virtual.whitespace")),
+            ),
+            text_style,
+            draw_indent_guides: editor_config.indent_guides.render,
+            viewport,
+            col_offset,
+        }
+    }
+
+    /// Draws a single `grapheme` at the current render position with a specified `style`.
+    pub fn draw_grapheme(
+        &mut self,
+        grapheme: Grapheme,
+        mut style: Style,
+        is_virtual: bool,
+        last_indent_level: &mut usize,
+        is_in_indent_area: &mut bool,
+        position: Position,
+    ) {
+        let cut_off_start = self.col_offset.saturating_sub(position.col);
+        let is_whitespace = grapheme.is_whitespace();
+
+        // TODO is it correct to apply the whitespace style to all unicode white spaces?
+        if is_whitespace {
+            style = style.patch(self.whitespace_style);
+        }
+
+        let width = grapheme.width();
+        let space = if is_virtual { " " } else { &self.space };
+        let nbsp = if is_virtual { " " } else { &self.nbsp };
+        let tab = if is_virtual {
+            &self.virtual_tab
+        } else {
+            &self.tab
+        };
+        let grapheme = match grapheme {
+            Grapheme::Tab { width } => {
+                let grapheme_tab_width = char_to_byte_idx(tab, width);
+                &tab[..grapheme_tab_width]
+            }
+            // TODO special rendering for other whitespaces?
+            Grapheme::Other { ref g } if g == " " => space,
+            Grapheme::Other { ref g } if g == "\u{00A0}" => nbsp,
+            Grapheme::Other { ref g } => g,
+            Grapheme::Newline => &self.newline,
+        };
+
+        let in_bounds = self.col_offset <= position.col
+            && position.col < self.viewport.width as usize + self.col_offset;
+
+        if in_bounds {
+            self.surface.set_string(
+                self.viewport.x + (position.col - self.col_offset) as u16,
+                self.viewport.y + position.row as u16,
+                grapheme,
+                style,
+            );
+        } else if cut_off_start != 0 && cut_off_start < width {
+            // partially on screen
+            let rect = Rect::new(
+                self.viewport.x,
+                self.viewport.y + position.row as u16,
+                (width - cut_off_start) as u16,
+                1,
+            );
+            self.surface.set_style(rect, style);
+        }
+
+        if *is_in_indent_area && !is_whitespace {
+            *last_indent_level = position.col;
+            *is_in_indent_area = false;
+        }
+    }
+
+    /// Overlay indentation guides ontop of a rendered line
+    /// The indentation level is computed in `draw_lines`.
+    /// Therefore this function must always be called afterwards.
+    pub fn draw_indent_guides(&mut self, indent_level: usize, row: u16) {
+        if !self.draw_indent_guides {
+            return;
+        }
+
+        // Don't draw indent guides outside of view
+        let end_indent = min(
+            indent_level,
+            // Add indent_width - 1 to round up, since the first visible
+            // indent might be a bit after offset.col
+            self.col_offset + self.viewport.width as usize + (self.indent_width as usize - 1),
+        ) / self.indent_width as usize;
+
+        for i in self.starting_indent..end_indent {
+            let x = (self.viewport.x as usize + (i * self.indent_width as usize) - self.col_offset)
+                as u16;
+            let y = self.viewport.y + row;
+            debug_assert!(self.surface.in_bounds(x, y));
+            self.surface
+                .set_string(x, y, &self.indent_guide_char, self.indent_guide_style);
+        }
+    }
+}

helix-term/src/ui/fuzzy_match.rs

@@ -4,71 +4,236 @@ use fuzzy_matcher::FuzzyMatcher;
 #[cfg(test)]
 mod test;
 
+struct QueryAtom {
+    kind: QueryAtomKind,
+    atom: String,
+    ignore_case: bool,
+    inverse: bool,
+}
+impl QueryAtom {
+    fn new(atom: &str) -> Option<QueryAtom> {
+        let mut atom = atom.to_string();
+        let inverse = atom.starts_with('!');
+        if inverse {
+            atom.remove(0);
+        }
+
+        let mut kind = match atom.chars().next() {
+            Some('^') => QueryAtomKind::Prefix,
+            Some('\'') => QueryAtomKind::Substring,
+            _ if inverse => QueryAtomKind::Substring,
+            _ => QueryAtomKind::Fuzzy,
+        };
+
+        if atom.starts_with(['^', '\'']) {
+            atom.remove(0);
+        }
+
+        if atom.is_empty() {
+            return None;
+        }
+
+        if atom.ends_with('$') && !atom.ends_with("\\$") {
+            atom.pop();
+            kind = if kind == QueryAtomKind::Prefix {
+                QueryAtomKind::Exact
+            } else {
+                QueryAtomKind::Postfix
+            }
+        }
+
+        Some(QueryAtom {
+            kind,
+            atom: atom.replace('\\', ""),
+            // not ideal but fuzzy_matches only knows ascii uppercase so more consistent
+            // to behave the same
+            ignore_case: kind != QueryAtomKind::Fuzzy
+                && atom.chars().all(|c| c.is_ascii_lowercase()),
+            inverse,
+        })
+    }
+
+    fn indices(&self, matcher: &Matcher, item: &str, indices: &mut Vec<usize>) -> bool {
+        // for inverse there are no indices to return
+        // just return whether we matched
+        if self.inverse {
+            return self.matches(matcher, item);
+        }
+        let buf;
+        let item = if self.ignore_case {
+            buf = item.to_ascii_lowercase();
+            &buf
+        } else {
+            item
+        };
+        let off = match self.kind {
+            QueryAtomKind::Fuzzy => {
+                if let Some((_, fuzzy_indices)) = matcher.fuzzy_indices(item, &self.atom) {
+                    indices.extend_from_slice(&fuzzy_indices);
+                    return true;
+                } else {
+                    return false;
+                }
+            }
+            QueryAtomKind::Substring => {
+                if let Some(off) = item.find(&self.atom) {
+                    off
+                } else {
+                    return false;
+                }
+            }
+            QueryAtomKind::Prefix if item.starts_with(&self.atom) => 0,
+            QueryAtomKind::Postfix if item.ends_with(&self.atom) => item.len() - self.atom.len(),
+            QueryAtomKind::Exact if item == self.atom => 0,
+            _ => return false,
+        };
+
+        indices.extend(off..(off + self.atom.len()));
+        true
+    }
+
+    fn matches(&self, matcher: &Matcher, item: &str) -> bool {
+        let buf;
+        let item = if self.ignore_case {
+            buf = item.to_ascii_lowercase();
+            &buf
+        } else {
+            item
+        };
+        let mut res = match self.kind {
+            QueryAtomKind::Fuzzy => matcher.fuzzy_match(item, &self.atom).is_some(),
+            QueryAtomKind::Substring => item.contains(&self.atom),
+            QueryAtomKind::Prefix => item.starts_with(&self.atom),
+            QueryAtomKind::Postfix => item.ends_with(&self.atom),
+            QueryAtomKind::Exact => item == self.atom,
+        };
+        if self.inverse {
+            res = !res;
+        }
+        res
+    }
+}
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+enum QueryAtomKind {
+    /// Item is a fuzzy match of this behaviour
+    ///
+    /// Usage: `foo`
+    Fuzzy,
+    /// Item contains query atom as a continuous substring
+    ///
+    /// Usage `'foo`
+    Substring,
+    /// Item starts with query atom
+    ///
+    /// Usage: `^foo`
+    Prefix,
+    /// Item ends with query atom
+    ///
+    /// Usage: `foo$`
+    Postfix,
+    /// Item is equal to query atom
+    ///
+    /// Usage `^foo$`
+    Exact,
+}
+
+#[derive(Default)]
 pub struct FuzzyQuery {
-    queries: Vec<String>,
+    first_fuzzy_atom: Option<String>,
+    query_atoms: Vec<QueryAtom>,
+}
+
+fn query_atoms(query: &str) -> impl Iterator<Item = &str> + '_ {
+    let mut saw_backslash = false;
+    query.split(move |c| {
+        saw_backslash = match c {
+            ' ' if !saw_backslash => return true,
+            '\\' => true,
+            _ => false,
+        };
+        false
+    })
 }
 
 impl FuzzyQuery {
+    pub fn refine(&self, query: &str, old_query: &str) -> (FuzzyQuery, bool) {
+        // TODO: we could be a lot smarter about this
+        let new_query = Self::new(query);
+        let mut is_refinement = query.starts_with(old_query);
+
+        // if the last atom is an inverse atom adding more text to it
+        // will actually increase the number of matches and we can not refine
+        // the matches.
+        if is_refinement && !self.query_atoms.is_empty() {
+            let last_idx = self.query_atoms.len() - 1;
+            if self.query_atoms[last_idx].inverse
+                && self.query_atoms[last_idx].atom != new_query.query_atoms[last_idx].atom
+            {
+                is_refinement = false;
+            }
+        }
+
+        (new_query, is_refinement)
+    }
+
     pub fn new(query: &str) -> FuzzyQuery {
-        let mut saw_backslash = false;
-        let queries = query
-            .split(|c| {
-                saw_backslash = match c {
-                    ' ' if !saw_backslash => return true,
-                    '\\' => true,
-                    _ => false,
-                };
-                false
-            })
-            .filter_map(|query| {
-                if query.is_empty() {
+        let mut first_fuzzy_query = None;
+        let query_atoms = query_atoms(query)
+            .filter_map(|atom| {
+                let atom = QueryAtom::new(atom)?;
+                if atom.kind == QueryAtomKind::Fuzzy && first_fuzzy_query.is_none() {
+                    first_fuzzy_query = Some(atom.atom);
                     None
                 } else {
-                    Some(query.replace("\\ ", " "))
+                    Some(atom)
                 }
             })
             .collect();
-        FuzzyQuery { queries }
+        FuzzyQuery {
+            first_fuzzy_atom: first_fuzzy_query,
+            query_atoms,
+        }
     }
 
     pub fn fuzzy_match(&self, item: &str, matcher: &Matcher) -> Option<i64> {
-        // use the rank of the first query for the rank, because merging ranks is not really possible
+        // use the rank of the first fuzzzy query for the rank, because merging ranks is not really possible
         // this behaviour matches fzf and skim
-        let score = matcher.fuzzy_match(item, self.queries.get(0)?)?;
+        let score = self
+            .first_fuzzy_atom
+            .as_ref()
+            .map_or(Some(0), |atom| matcher.fuzzy_match(item, atom))?;
         if self
-            .queries
+            .query_atoms
             .iter()
-            .any(|query| matcher.fuzzy_match(item, query).is_none())
+            .any(|atom| !atom.matches(matcher, item))
         {
             return None;
         }
         Some(score)
     }
 
-    pub fn fuzzy_indicies(&self, item: &str, matcher: &Matcher) -> Option<(i64, Vec<usize>)> {
-        if self.queries.len() == 1 {
-            return matcher.fuzzy_indices(item, &self.queries[0]);
+    pub fn fuzzy_indices(&self, item: &str, matcher: &Matcher) -> Option<(i64, Vec<usize>)> {
+        let (score, mut indices) = self.first_fuzzy_atom.as_ref().map_or_else(
+            || Some((0, Vec::new())),
+            |atom| matcher.fuzzy_indices(item, atom),
+        )?;
+
+        // fast path for the common case of just a single atom
+        if self.query_atoms.is_empty() {
+            return Some((score, indices));
         }
 
-        // use the rank of the first query for the rank, because merging ranks is not really possible
-        // this behaviour matches fzf and skim
-        let (score, mut indicies) = matcher.fuzzy_indices(item, self.queries.get(0)?)?;
-
-        // fast path for the common case of not using a space
-        // during matching this branch should be free thanks to branch prediction
-        if self.queries.len() == 1 {
-            return Some((score, indicies));
-        }
-
-        for query in &self.queries[1..] {
-            let (_, matched_indicies) = matcher.fuzzy_indices(item, query)?;
-            indicies.extend_from_slice(&matched_indicies);
+        for atom in &self.query_atoms {
+            if !atom.indices(matcher, item, &mut indices) {
+                return None;
+            }
         }
 
         // deadup and remove duplicate matches
-        indicies.sort_unstable();
-        indicies.dedup();
+        indices.sort_unstable();
+        indices.dedup();
 
-        Some((score, indicies))
+        Some((score, indices))
     }
 }