fix build

When fiddling with paths in a :o prompt, one usually would want Ctrl-W to erase a path segment
rather than the whole path. This is how Ctrl-W works in e.g. (neo)vim out of the box.

.cargo/config

@@ -0,0 +1,2 @@
+[alias]
+xtask = "run --package xtask --"

.envrc

@@ -3,3 +3,4 @@ watch_file flake.lock
 
 # try to use flakes, if it fails use normal nix (ie. shell.nix)
 use flake || use nix
+eval "$shellHook"

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,28 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: C-bug
-assignees: ''
-
----
-
-<!-- Your issue may already be reported!
-Please search on the issue tracker before creating one. -->
-
-### Reproduction steps
-
-<!-- Ideally provide a key sequence and/or asciinema.org recording. --> 
-
-### Environment
-
-- Platform: <!--  macOS / Windows / Linux -->
-- Helix version: <!--  'hx -v' if using a release, 'git describe' if building from master -->
-
-<details><summary>~/.cache/helix/helix.log</summary>
-
-```
-please provide a copy of `~/.cache/helix/helix.log` here if possible, you may need to redact some of the lines
-```
-
-</details>

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -0,0 +1,67 @@
+name: Bug Report
+description: Create a report to help us improve
+labels: C-bug
+body:
+  - type: markdown
+    attributes:
+      value: Thank you for filing a bug report! 🐛
+  - type: textarea
+    id: problem
+    attributes:
+      label: Summary
+      description: >
+        Please provide a short summary of the bug, along with any information
+        you feel relevant to replicate the bug.
+    validations:
+      required: true
+  - type: textarea
+    id: reproduction-steps
+    attributes:
+      label: Reproduction Steps
+      value: |
+        <!-- Ideally provide a key sequence and/or asciinema.org recording. --> 
+
+        I tried this:
+
+        1. `hx`
+
+        I expected this to happen:
+
+        Instead, this happened:
+  - type: textarea
+    id: helix-log
+    attributes:
+      label: Helix log
+      description: See `hx -h` for log file path
+      value: |
+        <details><summary>~/.cache/helix/helix.log</summary>
+
+        ```
+        please provide a copy of `~/.cache/helix/helix.log` here if possible, you may need to redact some of the lines
+        ```
+
+        </details>
+  - type: input
+    id: platform
+    attributes:
+      label: Platform
+      placeholder: Linux / macOS / Windows
+    validations:
+      required: true
+  - type: input
+    id: terminal-emulator
+    attributes:
+      label: Terminal Emulator
+      placeholder: wezterm 20220101-133340-7edc5b5a
+    validations:
+      required: true
+  - type: input
+    id: helix-version
+    attributes:
+      label: Helix Version
+      description: >
+        Helix version (`hx -V` if using a release, `git describe` if building
+        from master)
+      placeholder: "helix 0.6.0 (c0dbd6dc)"
+    validations:
+      required: true

.github/workflows/build.yml

@@ -13,9 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
-        with:
-          submodules: true
+        uses: actions/checkout@v3
 
       - name: Install stable toolchain
         uses: actions-rs/toolchain@v1
@@ -25,22 +23,25 @@ jobs:
           override: true
 
       - name: Cache cargo registry
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v3
         with:
           path: ~/.cargo/registry
           key: ${{ runner.os }}-v2-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-registry-
 
       - name: Cache cargo index
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v3
         with:
           path: ~/.cargo/git
           key: ${{ runner.os }}-v2-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-index-
 
       - name: Cache cargo target dir
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v3
         with:
           path: target
           key: ${{ runner.os }}-v2-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-build-target-
 
       - name: Run cargo check
         uses: actions-rs/cargo@v1
@@ -52,9 +53,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
-        with:
-          submodules: true
+        uses: actions/checkout@v3
 
       - name: Install stable toolchain
         uses: actions-rs/toolchain@v1
@@ -64,27 +63,41 @@ jobs:
           override: true
 
       - name: Cache cargo registry
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v3
         with:
           path: ~/.cargo/registry
           key: ${{ runner.os }}-v2-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-registry-
 
       - name: Cache cargo index
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v3
         with:
           path: ~/.cargo/git
           key: ${{ runner.os }}-v2-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-index-
 
       - name: Cache cargo target dir
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v3
         with:
           path: target
           key: ${{ runner.os }}-v2-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-build-target-
+
+      - name: Copy minimal languages config
+        run: cp .github/workflows/languages.toml ./languages.toml
+
+      - name: Cache test tree-sitter grammar
+        uses: actions/cache@v3
+        with:
+          path: runtime/grammars
+          key: ${{ runner.os }}-v2-tree-sitter-grammars-${{ hashFiles('languages.toml') }}
+          restore-keys: ${{ runner.os }}-v2-tree-sitter-grammars-
 
       - name: Run cargo test
         uses: actions-rs/cargo@v1
         with:
           command: test
+          args: --workspace
 
     strategy:
       matrix:
@@ -96,9 +109,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
-        with:
-          submodules: true
+        uses: actions/checkout@v3
 
       - name: Install stable toolchain
         uses: actions-rs/toolchain@v1
@@ -109,22 +120,25 @@ jobs:
           components: rustfmt, clippy
 
       - name: Cache cargo registry
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v3
         with:
           path: ~/.cargo/registry
           key: ${{ runner.os }}-v2-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-registry-
 
       - name: Cache cargo index
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v3
         with:
           path: ~/.cargo/git
           key: ${{ runner.os }}-v2-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-index-
 
       - name: Cache cargo target dir
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v3
         with:
           path: target
           key: ${{ runner.os }}-v2-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-build-target-
 
       - name: Run cargo fmt
         uses: actions-rs/cargo@v1
@@ -136,4 +150,53 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: clippy
-          args: -- -D warnings
+          args: --all-targets -- -D warnings
+
+  docs:
+    name: Docs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+
+      - name: Install stable toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+
+      - name: Cache cargo registry
+        uses: actions/cache@v3
+        with:
+          path: ~/.cargo/registry
+          key: ${{ runner.os }}-v2-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-registry-
+
+      - name: Cache cargo index
+        uses: actions/cache@v3
+        with:
+          path: ~/.cargo/git
+          key: ${{ runner.os }}-v2-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-index-
+
+      - name: Cache cargo target dir
+        uses: actions/cache@v3
+        with:
+          path: target
+          key: ${{ runner.os }}-v2-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-build-target-
+
+      - name: Generate docs
+        uses: actions-rs/cargo@v1
+        with:
+          command: xtask
+          args: docgen
+
+      - name: Check uncommitted documentation changes
+        run: |
+          git diff
+          git diff-files --quiet \
+            || (echo "Run 'cargo xtask docgen', commit the changes and push again" \
+            && exit 1)
+

.github/workflows/cachix.yml

@@ -0,0 +1,26 @@
+# Publish the Nix flake outputs to Cachix
+name: Cachix
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  publish:
+    name: Publish Flake
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout sources
+      uses: actions/checkout@v3
+
+    - name: Install nix
+      uses: cachix/install-nix-action@v17
+
+    - name: Authenticate with Cachix
+      uses: cachix/cachix-action@v10
+      with:
+        name: helix
+        authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
+    - name: Build nix flake
+      run: nix build

.github/workflows/gh-pages.yml

@@ -4,12 +4,14 @@ on:
   push:
     branches:
       - master
+    tags:
+      - '*'
 
 jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Setup mdBook
         uses: peaceiris/actions-mdbook@v1
@@ -19,9 +21,21 @@ jobs:
 
       - run: mdbook build book
       
+      - name: Set output directory
+        run: |
+          OUTDIR=$(basename ${{ github.ref }})
+          echo "OUTDIR=$OUTDIR" >> $GITHUB_ENV
+
       - name: Deploy
         uses: peaceiris/actions-gh-pages@v3
-        if: github.ref == 'refs/heads/master'
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./book/book
+          destination_dir: ./${{ env.OUTDIR }}
+
+      - name: Deploy stable
+        uses: peaceiris/actions-gh-pages@v3
+        if: startswith(github.ref, 'refs/tags/')
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./book/book

.github/workflows/languages.toml

@@ -0,0 +1,26 @@
+# This languages.toml is used for testing in CI.
+
+[[language]]
+name = "rust"
+scope = "source.rust"
+injection-regex = "rust"
+file-types = ["rs"]
+comment-token = "//"
+roots = ["Cargo.toml", "Cargo.lock"]
+indent = { tab-width = 4, unit = "    " }
+
+[[grammar]]
+name = "rust"
+source = { git = "https://github.com/tree-sitter/tree-sitter-rust", rev = "a360da0a29a19c281d08295a35ecd0544d2da211" }
+
+[[language]]
+name = "nix"
+scope = "source.nix"
+injection-regex = "nix"
+file-types = ["nix"]
+shebangs = []
+roots = []
+comment-token = "#"
+
+# A grammar entry is not necessary for this language - it is only used for
+# testing TOML merging behavior.

.github/workflows/release.yml

@@ -1,32 +1,81 @@
 name: Release
 on:
-  # schedule:
-  # - cron: '0 0 * * *' # midnight UTC
-
   push:
     tags:
-    - 'v[0-9]+.[0-9]+.[0-9]+'
-    ## - release
+    - '[0-9]+.[0-9]+'
+    - '[0-9]+.[0-9]+.[0-9]+'
 
 jobs:
+  fetch-grammars:
+    name: Fetch Grammars
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+
+      - name: Install stable toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+
+      - name: Cache cargo registry
+        uses: actions/cache@v3
+        with:
+          path: ~/.cargo/registry
+          key: ${{ runner.os }}-v2-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-registry-
+
+      - name: Cache cargo index
+        uses: actions/cache@v3
+        with:
+          path: ~/.cargo/git
+          key: ${{ runner.os }}-v2-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-index-
+
+      - name: Cache cargo target dir
+        uses: actions/cache@v3
+        with:
+          path: target
+          key: ${{ runner.os }}-v2-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-build-target-
+
+      - name: Fetch tree-sitter grammars
+        uses: actions-rs/cargo@v1
+        env:
+          HELIX_DISABLE_AUTO_GRAMMAR_BUILD: yes
+        with:
+          command: run
+          args: -- --grammar fetch
+
+      - name: Bundle grammars
+        run: tar cJf grammars.tar.xz -C runtime/grammars/sources .
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: grammars
+          path: grammars.tar.xz
+
   dist:
     name: Dist
+    needs: [fetch-grammars]
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false # don't fail other jobs if one fails
       matrix:
-        build: [x86_64-linux, aarch64-linux, x86_64-macos, x86_64-windows] #, x86_64-win-gnu, win32-msvc
+        build: [x86_64-linux, x86_64-macos, x86_64-windows] #, x86_64-win-gnu, win32-msvc
         include:
         - build: x86_64-linux
           os: ubuntu-20.04
           rust: stable
           target: x86_64-unknown-linux-gnu
           cross: false
-        - build: aarch64-linux
-          os: ubuntu-20.04
-          rust: stable
-          target: aarch64-unknown-linux-gnu
-          cross: true
+        # - build: aarch64-linux
+        #   os: ubuntu-20.04
+        #   rust: stable
+        #   target: aarch64-unknown-linux-gnu
+        #   cross: true
         - build: x86_64-macos
           os: macos-latest
           rust: stable
@@ -52,9 +101,16 @@ jobs:
 
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
-        with:
-          submodules: true
+        uses: actions/checkout@v3
+
+      - name: Download grammars
+        uses: actions/download-artifact@v3
+
+      - name: Move grammars under runtime
+        if: "!startsWith(matrix.os, 'windows')"
+        run: |
+          mkdir -p runtime/grammars/sources
+          tar xJf grammars/grammars.tar.xz -C runtime/grammars/sources
 
       - name: Install ${{ matrix.rust }} toolchain
         uses: actions-rs/toolchain@v1
@@ -69,7 +125,7 @@ jobs:
         with:
           use-cross: ${{ matrix.cross }}
           command: test
-          args: --release --locked --target ${{ matrix.target }}
+          args: --release --locked --target ${{ matrix.target }} --workspace
 
       - name: Build release binary
         uses: actions-rs/cargo@v1
@@ -91,18 +147,63 @@ jobs:
             aarch64-linux-gnu-strip \
             /target/${{ matrix.target }}/release/hx
 
+      - name: Build AppImage
+        shell: bash
+        if: matrix.build == 'x86_64-linux'
+        run: |
+          mkdir dist
+
+          name=dev
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            name=${GITHUB_REF:10}
+          fi
+
+          export VERSION="$name"
+          export ARCH=x86_64
+          export APP=helix
+          export OUTPUT="helix-$VERSION-$ARCH.AppImage"
+          export UPDATE_INFORMATION="gh-releases-zsync|$GITHUB_REPOSITORY_OWNER|helix|latest|$APP-*-$ARCH.AppImage.zsync"
+
+          mkdir -p "$APP.AppDir"/usr/{bin,lib/helix}
+
+          cp "target/${{ matrix.target }}/release/hx" "$APP.AppDir/usr/bin/hx"
+          rm -rf runtime/grammars/sources
+          cp -r runtime "$APP.AppDir/usr/lib/helix/runtime"
+
+          cat << 'EOF' > "$APP.AppDir/AppRun"
+          #!/bin/sh
+
+          APPDIR="$(dirname "$(readlink -f "${0}")")"
+          HELIX_RUNTIME="$APPDIR/usr/lib/helix/runtime" exec "$APPDIR/usr/bin/hx" "$@"
+          EOF
+          chmod 755 "$APP.AppDir/AppRun"
+
+          curl -Lo linuxdeploy-x86_64.AppImage \
+              https://github.com/linuxdeploy/linuxdeploy/releases/download/continuous/linuxdeploy-x86_64.AppImage
+          chmod +x linuxdeploy-x86_64.AppImage
+
+          ./linuxdeploy-x86_64.AppImage \
+              --appdir "$APP.AppDir" -d contrib/Helix.desktop \
+              -i contrib/helix.png --output appimage
+
+          mv "$APP-$VERSION-$ARCH.AppImage" \
+              "$APP-$VERSION-$ARCH.AppImage.zsync" dist
+
       - name: Build archive
         shell: bash
         run: |
-          mkdir dist
+          mkdir -p dist
           if [ "${{ matrix.os }}" = "windows-2019" ]; then
             cp "target/${{ matrix.target }}/release/hx.exe" "dist/"
           else
             cp "target/${{ matrix.target }}/release/hx" "dist/"
           fi
+          if [ -d runtime/grammars/sources ]; then
+            rm -rf runtime/grammars/sources
+          fi
           cp -r runtime dist
 
-      - uses: actions/upload-artifact@v2.2.4
+      - uses: actions/upload-artifact@v3
         with:
           name: bins-${{ matrix.build }}
           path: dist
@@ -113,20 +214,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
-        with:
-          submodules: false
+        uses: actions/checkout@v3
 
-      - uses: actions/download-artifact@v2
-        # with:
-        #   path: dist
-      # - run: ls -al ./dist
-      - run: ls -al bins-*
+      - uses: actions/download-artifact@v3
 
       - name: Calculate tag name
         run: |
           name=dev
-          if [[ $GITHUB_REF == refs/tags/v* ]]; then
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
             name=${GITHUB_REF:10}
           fi
           echo ::set-output name=val::$name
@@ -138,8 +233,13 @@ jobs:
         run: |
           set -ex
 
-          rm -rf tmp
-          mkdir tmp
+          source="$(pwd)"
+          mkdir -p runtime/grammars/sources
+          tar xJf grammars/grammars.tar.xz -C runtime/grammars/sources
+          rm -rf grammars
+
+          cd "$(mktemp -d)"
+          mv $source/bins-* .
           mkdir dist
 
           for dir in bins-* ; do
@@ -148,19 +248,28 @@ jobs:
                   exe=".exe"
               fi
               pkgname=helix-$TAG-$platform
-              mkdir tmp/$pkgname
-              cp LICENSE README.md tmp/$pkgname
-              mv bins-$platform/runtime tmp/$pkgname/
-              mv bins-$platform/hx$exe tmp/$pkgname
-              chmod +x tmp/$pkgname/hx$exe
+              mkdir $pkgname
+              cp $source/LICENSE $source/README.md $pkgname
+              mkdir $pkgname/contrib
+              cp -r $source/contrib/completion $pkgname/contrib
+              mv bins-$platform/runtime $pkgname/
+              mv bins-$platform/hx$exe $pkgname
+              chmod +x $pkgname/hx$exe
+
+              if [[ "$platform" = "x86_64-linux" ]]; then
+                  mv bins-$platform/helix-*.AppImage* dist/
+              fi
 
               if [ "$exe" = "" ]; then
-                  tar cJf dist/$pkgname.tar.xz -C tmp $pkgname
+                  tar cJf dist/$pkgname.tar.xz $pkgname
               else
-                  (cd tmp && 7z a -r ../dist/$pkgname.zip $pkgname)
+                  7z a -r dist/$pkgname.zip $pkgname
               fi
           done
 
+          tar cJf dist/helix-$TAG-source.tar.xz -C $source .
+          mv dist $source/
+
       - name: Upload binaries to release
         uses: svenstaro/upload-release-action@v2
         with:

.gitmodules

@@ -1,136 +0,0 @@
-[submodule "helix-syntax/languages/tree-sitter-cpp"]
-	path = helix-syntax/languages/tree-sitter-cpp
-	url = https://github.com/tree-sitter/tree-sitter-cpp
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-javascript"]
-	path = helix-syntax/languages/tree-sitter-javascript
-	url = https://github.com/tree-sitter/tree-sitter-javascript
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-julia"]
-	path = helix-syntax/languages/tree-sitter-julia
-	url = https://github.com/tree-sitter/tree-sitter-julia
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-python"]
-	path = helix-syntax/languages/tree-sitter-python
-	url = https://github.com/tree-sitter/tree-sitter-python
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-typescript"]
-	path = helix-syntax/languages/tree-sitter-typescript
-	url = https://github.com/tree-sitter/tree-sitter-typescript
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-agda"]
-	path = helix-syntax/languages/tree-sitter-agda
-	url = https://github.com/tree-sitter/tree-sitter-agda
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-go"]
-	path = helix-syntax/languages/tree-sitter-go
-	url = https://github.com/tree-sitter/tree-sitter-go
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-ruby"]
-	path = helix-syntax/languages/tree-sitter-ruby
-	url = https://github.com/tree-sitter/tree-sitter-ruby
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-java"]
-	path = helix-syntax/languages/tree-sitter-java
-	url = https://github.com/tree-sitter/tree-sitter-java
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-php"]
-	path = helix-syntax/languages/tree-sitter-php
-	url = https://github.com/tree-sitter/tree-sitter-php
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-html"]
-	path = helix-syntax/languages/tree-sitter-html
-	url = https://github.com/tree-sitter/tree-sitter-html
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-scala"]
-	path = helix-syntax/languages/tree-sitter-scala
-	url = https://github.com/tree-sitter/tree-sitter-scala
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-bash"]
-	path = helix-syntax/languages/tree-sitter-bash
-	url = https://github.com/tree-sitter/tree-sitter-bash
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-rust"]
-	path = helix-syntax/languages/tree-sitter-rust
-	url = https://github.com/tree-sitter/tree-sitter-rust
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-json"]
-	path = helix-syntax/languages/tree-sitter-json
-	url = https://github.com/tree-sitter/tree-sitter-json
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-css"]
-	path = helix-syntax/languages/tree-sitter-css
-	url = https://github.com/tree-sitter/tree-sitter-css
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-c-sharp"]
-	path = helix-syntax/languages/tree-sitter-c-sharp
-	url = https://github.com/tree-sitter/tree-sitter-c-sharp
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-c"]
-	path = helix-syntax/languages/tree-sitter-c
-	url = https://github.com/tree-sitter/tree-sitter-c
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-haskell"]
-	path = helix-syntax/languages/tree-sitter-haskell
-	url = https://github.com/tree-sitter/tree-sitter-haskell
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-swift"]
-	path = helix-syntax/languages/tree-sitter-swift
-	url = https://github.com/tree-sitter/tree-sitter-swift
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-toml"]
-	path = helix-syntax/languages/tree-sitter-toml
-	url = https://github.com/ikatyang/tree-sitter-toml
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-elixir"]
-	path = helix-syntax/languages/tree-sitter-elixir
-	url = https://github.com/elixir-lang/tree-sitter-elixir
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-nix"]
-	path = helix-syntax/languages/tree-sitter-nix
-	url = https://github.com/cstrahan/tree-sitter-nix
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-latex"]
-	path = helix-syntax/languages/tree-sitter-latex
-	url = https://github.com/latex-lsp/tree-sitter-latex
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-ledger"]
-	path = helix-syntax/languages/tree-sitter-ledger
-	url = https://github.com/cbarrete/tree-sitter-ledger
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-protobuf"]
-	path = helix-syntax/languages/tree-sitter-protobuf
-	url = https://github.com/yusdacra/tree-sitter-protobuf.git
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-ocaml"]
-	path = helix-syntax/languages/tree-sitter-ocaml
-	url = https://github.com/tree-sitter/tree-sitter-ocaml
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-lua"]
-	path = helix-syntax/languages/tree-sitter-lua
-	url = https://github.com/nvim-treesitter/tree-sitter-lua
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-yaml"]
-	path = helix-syntax/languages/tree-sitter-yaml
-	url = https://github.com/ikatyang/tree-sitter-yaml
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-zig"]
-	path = helix-syntax/languages/tree-sitter-zig
-	url = https://github.com/maxxnino/tree-sitter-zig
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-svelte"]
-	path = helix-syntax/languages/tree-sitter-svelte
-	url = https://github.com/Himujjal/tree-sitter-svelte
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-vue"]
-	path = helix-syntax/languages/tree-sitter-vue
-	url = https://github.com/ikatyang/tree-sitter-vue
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-tsq"]
-	path = helix-syntax/languages/tree-sitter-tsq
-	url = https://github.com/tree-sitter/tree-sitter-tsq
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-cmake"]
-	path = helix-syntax/languages/tree-sitter-cmake
-	url = https://github.com/uyha/tree-sitter-cmake
-	shallow = true

CHANGELOG.md

@@ -1,5 +1,287 @@
+# 22.03 (2022-03-28)
 
-# 0.5.0 (2021-11-28)
+A big shout out to all the contributors! We had 51 contributors in this release.
+
+This release is particularly large and featureful. Check out some of the
+highlights in the [news section](https://helix-editor.com/news/release-22-03-highlights/).
+
+As usual, the following is a summary of each of the changes since the last release.
+For the full log, check out the [git log](https://github.com/helix-editor/helix/compare/v0.6.0..22.03).
+
+Breaking changes:
+
+- LSP config now lives under `editor.lsp` ([#1868](https://github.com/helix-editor/helix/pull/1868))
+- Expand-selection was moved from `]o` to `Alt-h` ([#1495](https://github.com/helix-editor/helix/pull/1495))
+
+Features:
+
+- Experimental Debug Adapter Protocol (DAP) support ([#574](https://github.com/helix-editor/helix/pull/574))
+- Primary cursor shape may now be customized per mode ([#1154](https://github.com/helix-editor/helix/pull/1154))
+- Overhaul incremental highlights and enable combined injections ([`6728344..4080341`](https://github.com/helix-editor/helix/compare/6728344..4080341))
+- Allow specifying file start position ([#445](https://github.com/helix-editor/helix/pull/445), [#1676](https://github.com/helix-editor/helix/pull/1676))
+- Dynamic line numbers ([#1522](https://github.com/helix-editor/helix/pull/1522))
+- Show an info box with the contents of registers ([#980](https://github.com/helix-editor/helix/pull/980))
+- Wrap-around behavior during search is now configurable ([#1516](https://github.com/helix-editor/helix/pull/1516))
+- Tree-sitter textobjects motions for classes, functions, and parameters ([#1619](https://github.com/helix-editor/helix/pull/1619), [#1708](https://github.com/helix-editor/helix/pull/1708), [#1805](https://github.com/helix-editor/helix/pull/1805))
+- Command palette: a picker for available commands ([#1400](https://github.com/helix-editor/helix/pull/1400))
+- LSP `workspace/configuration` and `workspace/didChangeConfiguration` support ([#1684](https://github.com/helix-editor/helix/pull/1684))
+- `hx --health [LANG]` command ([#1669](https://github.com/helix-editor/helix/pull/1669))
+- Refactor of the tree-sitter grammar system ([#1659](https://github.com/helix-editor/helix/pull/1659))
+  - All submodules have been removed
+  - New `hx --grammar {fetch|build}` flags for fetching and building tree-sitter grammars
+  - A custom grammar selection may now be declared with the `use-grammars` key in `languages.toml`
+
+Commands:
+
+- `:cquit!` - quit forcefully with a non-zero exit-code ([#1414](https://github.com/helix-editor/helix/pull/1414))
+- `shrink_selection` - shrink the selection to a child tree-sitter node (`Alt-j`, [#1340](https://github.com/helix-editor/helix/pull/1340))
+- `:tree-sitter-subtree` - show the tree-sitter subtree under the primary selection ([#1453](https://github.com/helix-editor/helix/pull/1453), [#1524](https://github.com/helix-editor/helix/pull/1524))
+- Add `Alt-Backspace`, `Alt-<`, `Alt->`, and `Ctrl-j` to insert mode ([#1441](https://github.com/helix-editor/helix/pull/1441))
+- `select_next_sibling`, `select_prev_sibling` - select next and previous tree-sitter nodes (`Alt-l` and `Alt-h`, [#1495](https://github.com/helix-editor/helix/pull/1495))
+- `:buffer-close-all`, `:buffer-close-all!`, `:buffer-close-others`, and `:buffer-close-others!` ([#1677](https://github.com/helix-editor/helix/pull/1677))
+- `:vsplit-new` and `:hsplit-new` - open vertical and horizontal splits with new scratch buffers ([#1763](https://github.com/helix-editor/helix/pull/1763))
+- `:open-config` to open the config file and `:refresh-config` to refresh config after changes ([#1771](https://github.com/helix-editor/helix/pull/1771), [#1803](https://github.com/helix-editor/helix/pull/1803))
+
+Usability improvements and fixes:
+
+- Prevent `:cquit` from ignoring unsaved changes ([#1414](https://github.com/helix-editor/helix/pull/1414))
+- Scrolling view keeps selections ([#1420](https://github.com/helix-editor/helix/pull/1420))
+- Only use shellwords parsing on unix platforms ([`7767703`](https://github.com/helix-editor/helix/commit/7767703))
+- Fix slash in search selector status message ([#1449](https://github.com/helix-editor/helix/pull/1449))
+- Use `std::path::MAIN_SEPARATOR` to determine completion ([`3e4f815`](https://github.com/helix-editor/helix/commit/3e4f815))
+- Expand to current node with `expand_selection` when the node has no children ([#1454](https://github.com/helix-editor/helix/pull/1454))
+- Add vertical and horizontal splits to the buffer picker ([#1502](https://github.com/helix-editor/helix/pull/1502))
+- Use the correct language ID for JavaScript & TypeScript LSP ([#1466](https://github.com/helix-editor/helix/pull/1466))
+- Run format command for all buffers being written ([#1444](https://github.com/helix-editor/helix/pull/1444))
+- Fix panics during resizing ([#1408](https://github.com/helix-editor/helix/pull/1408))
+- Fix auto-pairs with CRLF ([#1470](https://github.com/helix-editor/helix/pull/1470))
+- Fix picker scrolling when the bottom is reached ([#1567](https://github.com/helix-editor/helix/pull/1567))
+- Use markup themes for the markdown component ([#1363](https://github.com/helix-editor/helix/pull/1363))
+- Automatically commit changes to history if not in insert mode ([`2a7ae96`](https://github.com/helix-editor/helix/commit/2a7ae96))
+- Render code-actions as a menu and add padding to popup ([`094a0aa`](https://github.com/helix-editor/helix/commit/094a0aa))
+- Only render menu scrollbar if the menu doesn't fit ([`f10a06f`](https://github.com/helix-editor/helix/commit/f10a06f), [`36b975c`](https://github.com/helix-editor/helix/commit/36b975c))
+- Parse git revision instead of tag for version ([`d3221b0`](https://github.com/helix-editor/helix/commit/d3221b0), [#1674](https://github.com/helix-editor/helix/pull/1674))
+- Fix incorrect last modified buffer ([#1621](https://github.com/helix-editor/helix/pull/1621))
+- Add `PageUp`, `PageDown`, `Ctrl-u`, `Ctrl-d`, `Home`, `End` bindings to the file picker ([#1612](https://github.com/helix-editor/helix/pull/1612))
+- Display buffer IDs in the buffer picker ([#1134](https://github.com/helix-editor/helix/pull/1134))
+- Allow multi-line prompt documentation ([`2af0432`](https://github.com/helix-editor/helix/commit/2af0432))
+- Ignore the `.git` directory from the file picker ([#1604](https://github.com/helix-editor/helix/pull/1604))
+- Allow separate styling for markup heading levels ([#1618](https://github.com/helix-editor/helix/pull/1618))
+- Automatically close popups ([#1285](https://github.com/helix-editor/helix/pull/1285))
+- Allow auto-pairs tokens to be configured ([#1624](https://github.com/helix-editor/helix/pull/1624))
+- Don't indent empty lines in `indent` command ([#1653](https://github.com/helix-editor/helix/pull/1653))
+- Ignore `Enter` keypress when a menu has no selection ([#1704](https://github.com/helix-editor/helix/pull/1704))
+- Show errors when surround deletions and replacements fail ([#1709](https://github.com/helix-editor/helix/pull/1709))
+- Show infobox hints for `mi` and `ma` ([#1686](https://github.com/helix-editor/helix/pull/1686))
+- Highlight matching text in file picker suggestions ([#1635](https://github.com/helix-editor/helix/pull/1635))
+- Allow capturing multiple nodes in textobject queries ([#1611](https://github.com/helix-editor/helix/pull/1611))
+- Make repeat operator work with completion edits ([#1640](https://github.com/helix-editor/helix/pull/1640))
+- Save to the jumplist when searching ([#1718](https://github.com/helix-editor/helix/pull/1718))
+- Fix bug with auto-replacement of components in compositor ([#1711](https://github.com/helix-editor/helix/pull/1711))
+- Use Kakoune logic for `align_selection` ([#1675](https://github.com/helix-editor/helix/pull/1675))
+- Fix `follows` for `nixpkgs` in `flake.nix` ([#1729](https://github.com/helix-editor/helix/pull/1729))
+- Performance improvements for the picker ([`78fba86`](https://github.com/helix-editor/helix/commit/78fba86))
+- Rename infobox theme scopes ([#1741](https://github.com/helix-editor/helix/pull/1741))
+- Fallback to broader scopes if a theme scope is not found ([#1714](https://github.com/helix-editor/helix/pull/1714))
+- Add arrow-keys bindings for tree-sitter sibling selection commands ([#1724](https://github.com/helix-editor/helix/pull/1724))
+- Fix a bug in LSP when creating a file in a folder that does not exist ([#1775](https://github.com/helix-editor/helix/pull/1775))
+- Use `^` and `$` regex location assertions for search ([#1793](https://github.com/helix-editor/helix/pull/1793))
+- Fix register names in `insert_register` command ([#1751](https://github.com/helix-editor/helix/pull/1751))
+- Perform extend line for all selections ([#1804](https://github.com/helix-editor/helix/pull/1804))
+- Prevent panic when moving in an empty picker ([#1786](https://github.com/helix-editor/helix/pull/1786))
+- Fix line number calculations for non CR/CRLF line breaks ([`b4a282f`](https://github.com/helix-editor/helix/commit/b4a282f), [`0b96201`](https://github.com/helix-editor/helix/commit/0b96201))
+- Deploy documentation for `master` builds separately from release docs ([#1783](https://github.com/helix-editor/helix/pull/1783))
+
+Themes:
+
+- Add everforest_light ([#1412](https://github.com/helix-editor/helix/pull/1412))
+- Add gruvbox_light ([#1509](https://github.com/helix-editor/helix/pull/1509))
+- Add modified background to dracula popup ([#1434](https://github.com/helix-editor/helix/pull/1434))
+- Markup support for monokai pro themes ([#1553](https://github.com/helix-editor/helix/pull/1553))
+- Markup support for dracula theme ([#1554](https://github.com/helix-editor/helix/pull/1554))
+- Add `tag` to gruvbox theme ([#1555](https://github.com/helix-editor/helix/pull/1555))
+- Markup support for remaining themes ([#1525](https://github.com/helix-editor/helix/pull/1525))
+- Serika light and dark ([#1566](https://github.com/helix-editor/helix/pull/1566))
+- Fix rose_pine and rose_pine_dawn popup background color ([#1606](https://github.com/helix-editor/helix/pull/1606))
+- Fix hover menu item text color in base16 themes ([#1668](https://github.com/helix-editor/helix/pull/1668))
+- Update markup heading styles for everforest ([#1687](https://github.com/helix-editor/helix/pull/1687))
+- Update markup heading styles for rose_pine themes ([#1706](https://github.com/helix-editor/helix/pull/1706))
+- Style bogster cursors ([`6a6a9ab`](https://github.com/helix-editor/helix/commit/6a6a9ab))
+- Fix `ui.selection` in rose_pine themes ([#1716](https://github.com/helix-editor/helix/pull/1716))
+- Use distinct colors for cursor and matched pair in gruvbox ([#1791](https://github.com/helix-editor/helix/pull/1791))
+- Improve colors for `ui.cursor.match` capture in some themes ([#1862](https://github.com/helix-editor/helix/pull/1862))
+
+LSP:
+
+- Add default language server for JavaScript ([#1457](https://github.com/helix-editor/helix/pull/1457))
+- Add `pom.xml` as maven root directory marker ([#1496](https://github.com/helix-editor/helix/pull/1496))
+- Haskell LSP ([#1556](https://github.com/helix-editor/helix/pull/1556))
+- C-sharp LSP support ([#1788](https://github.com/helix-editor/helix/pull/1788))
+- Clean up Julia LSP config ([#1811](https://github.com/helix-editor/helix/pull/1811))
+
+New Languages:
+
+- llvm-mir ([#1398](https://github.com/helix-editor/helix/pull/1398))
+- regex ([#1362](https://github.com/helix-editor/helix/pull/1362))
+- Make ([#1433](https://github.com/helix-editor/helix/pull/1433), [#1661](https://github.com/helix-editor/helix/pull/1661))
+- git-config ([#1426](https://github.com/helix-editor/helix/pull/1426))
+- Lean ([#1422](https://github.com/helix-editor/helix/pull/1422))
+- Elm ([#1514](https://github.com/helix-editor/helix/pull/1514))
+- GraphQL ([#1515](https://github.com/helix-editor/helix/pull/1515))
+- Twig ([#1602](https://github.com/helix-editor/helix/pull/1602))
+- Rescript ([#1616](https://github.com/helix-editor/helix/pull/1616), [#1863](https://github.com/helix-editor/helix/pull/1863))
+- Erlang ([#1657](https://github.com/helix-editor/helix/pull/1657))
+- Kotlin ([#1689](https://github.com/helix-editor/helix/pull/1689))
+- HCL ([#1705](https://github.com/helix-editor/helix/pull/1705), [#1726](https://github.com/helix-editor/helix/pull/1726))
+- Org ([#1845](https://github.com/helix-editor/helix/pull/1845))
+- Solidity ([#1848](https://github.com/helix-editor/helix/pull/1848), [#1854](https://github.com/helix-editor/helix/pull/1854))
+
+Updated Languages and Queries:
+
+- Textobject and indent queries for c and cpp ([#1293](https://github.com/helix-editor/helix/pull/1293))
+- Fix null and boolean constant highlights for nix ([#1428](https://github.com/helix-editor/helix/pull/1428))
+- Capture markdown link text as `markup.link.text` ([#1456](https://github.com/helix-editor/helix/pull/1456))
+- Update and re-enable Haskell ([#1417](https://github.com/helix-editor/helix/pull/1417), [#1520](https://github.com/helix-editor/helix/pull/1520))
+- Update Go with generics support ([`ddbf036`](https://github.com/helix-editor/helix/commit/ddbf036))
+- Use `tree-sitter-css` for SCSS files ([#1507](https://github.com/helix-editor/helix/pull/1507))
+- Update Zig ([#1501](https://github.com/helix-editor/helix/pull/1501))
+- Update PHP ([#1521](https://github.com/helix-editor/helix/pull/1521))
+- Expand language support for comment injections ([#1527](https://github.com/helix-editor/helix/pull/1527))
+- Use tree-sitter-bash for `.zshrc` and `.bashrc` ([`7d51042`](https://github.com/helix-editor/helix/commit/7d51042))
+- Use tree-sitter-bash for `.bash_profile` ([#1571](https://github.com/helix-editor/helix/pull/1571))
+- Use tree-sitter-bash for `.zshenv` and ZSH files ([#1574](https://github.com/helix-editor/helix/pull/1574))
+- IEx ([#1576](https://github.com/helix-editor/helix/pull/1576))
+- Textobject queries for PHP ([#1601](https://github.com/helix-editor/helix/pull/1601))
+- C-sharp highlight query improvements ([#1795](https://github.com/helix-editor/helix/pull/1795))
+- Git commit performance has been improved on large verbose commits ([#1838](https://github.com/helix-editor/helix/pull/1838))
+
+Packaging:
+
+- The submodules system has been replaced with command-line flags for fetching and building tree-sitter grammars ([#1659](https://github.com/helix-editor/helix/pull/1659))
+- Flake outputs are pushed to Cachix on each push to `master` ([#1721](https://github.com/helix-editor/helix/pull/1721))
+- Update flake's `nix-cargo-integration` to depend on `dream2nix` ([#1758](https://github.com/helix-editor/helix/pull/1758))
+
+# 0.6.0 (2022-01-04)
+
+Happy new year and a big shout out to all the contributors! We had 55 contributors in this release.
+
+Helix has popped up in DPorts and Fedora Linux via COPR ([#1270](https://github.com/helix-editor/helix/pull/1270))
+
+As usual the following is a brief summary, refer to the git history for a full log:
+
+Breaking changes:
+
+- fix: Normalize backtab into shift-tab
+
+Features:
+
+- Macros ([#1234](https://github.com/helix-editor/helix/pull/1234))
+- Add reverse search functionality ([#958](https://github.com/helix-editor/helix/pull/958))
+- Allow keys to be mapped to sequences of commands ([#589](https://github.com/helix-editor/helix/pull/589))
+- Make it possible to keybind TypableCommands ([#1169](https://github.com/helix-editor/helix/pull/1169))
+- Detect workspace root using language markers ([#1370](https://github.com/helix-editor/helix/pull/1370))
+- Add WORD textobject ([#991](https://github.com/helix-editor/helix/pull/991))
+- Add LSP rename_symbol (`space-r`) ([#1011](https://github.com/helix-editor/helix/pull/1011))
+- Added workspace_symbol_picker ([#1041](https://github.com/helix-editor/helix/pull/1041))
+- Detect filetype from shebang line ([#1001](https://github.com/helix-editor/helix/pull/1001))
+- Allow piping from stdin into a buffer on startup ([#996](https://github.com/helix-editor/helix/pull/996))
+- Add auto pairs for same-char pairs ([#1219](https://github.com/helix-editor/helix/pull/1219))
+- Update settings at runtime ([#798](https://github.com/helix-editor/helix/pull/798))
+- Enable thin LTO ([`cccc194`](https://github.com/helix-editor/helix/commit/cccc194))
+
+Commands:
+
+- `:wonly` -- window only ([#1057](https://github.com/helix-editor/helix/pull/1057))
+- buffer-close (`:bc`, `:bclose`) ([#1035](https://github.com/helix-editor/helix/pull/1035))
+- Add `:<line>` and `:goto <line>` commands ([#1128](https://github.com/helix-editor/helix/pull/1128))
+- `:sort` command ([#1288](https://github.com/helix-editor/helix/pull/1288))
+- Add m textobject for pair under cursor ([#961](https://github.com/helix-editor/helix/pull/961))
+- Implement "Goto next buffer / Goto previous buffer" commands ([#950](https://github.com/helix-editor/helix/pull/950))
+- Implement "Goto last modification" command ([#1067](https://github.com/helix-editor/helix/pull/1067))
+- Add trim_selections command ([#1092](https://github.com/helix-editor/helix/pull/1092))
+- Add movement shortcut for history ([#1088](https://github.com/helix-editor/helix/pull/1088))
+- Add command to inc/dec number under cursor ([#1027](https://github.com/helix-editor/helix/pull/1027))
+  - Add support for dates for increment/decrement
+- Align selections (`&`) ([#1101](https://github.com/helix-editor/helix/pull/1101))
+- Implement no-yank delete/change ([#1099](https://github.com/helix-editor/helix/pull/1099))
+- Implement black hole register ([#1165](https://github.com/helix-editor/helix/pull/1165))
+- `gf` as goto_file (`gf`) ([#1102](https://github.com/helix-editor/helix/pull/1102))
+- Add last modified file (`gm`) ([#1093](https://github.com/helix-editor/helix/pull/1093))
+- ensure_selections_forward ([#1393](https://github.com/helix-editor/helix/pull/1393))
+- Readline style insert mode ([#1039](https://github.com/helix-editor/helix/pull/1039))
+
+Usability improvements and fixes:
+
+- Detect filetype on `:write` ([#1141](https://github.com/helix-editor/helix/pull/1141))
+- Add single and double quotes to matching pairs ([#995](https://github.com/helix-editor/helix/pull/995))
+- Launch with defaults upon invalid config/theme (rather than panicking) ([#982](https://github.com/helix-editor/helix/pull/982))
+- If switching away from an empty scratch buffer, remove it ([#935](https://github.com/helix-editor/helix/pull/935))
+- Truncate the starts of file paths instead of the ends in picker ([#951](https://github.com/helix-editor/helix/pull/951))
+- Truncate the start of file paths in the StatusLine ([#1351](https://github.com/helix-editor/helix/pull/1351))
+- Prevent picker from previewing binaries or large file ([#939](https://github.com/helix-editor/helix/pull/939))
+- Inform when reaching undo/redo bounds ([#981](https://github.com/helix-editor/helix/pull/981))
+- search_impl will only align cursor center when it isn't in view ([#959](https://github.com/helix-editor/helix/pull/959))
+- Add `<C-h>`, `<C-u>`, `<C-d>`, Delete in prompt mode ([#1034](https://github.com/helix-editor/helix/pull/1034))
+- Restore screen position when aborting search ([#1047](https://github.com/helix-editor/helix/pull/1047))
+- Buffer picker: show is_modifier flag ([#1020](https://github.com/helix-editor/helix/pull/1020))
+- Add commit hash to version info, if present ([#957](https://github.com/helix-editor/helix/pull/957))
+- Implement indent-aware delete ([#1120](https://github.com/helix-editor/helix/pull/1120))
+- Jump to end char of surrounding pair from any cursor pos ([#1121](https://github.com/helix-editor/helix/pull/1121))
+- File picker configuration ([#988](https://github.com/helix-editor/helix/pull/988))
+- Fix surround cursor position calculation ([#1183](https://github.com/helix-editor/helix/pull/1183))
+- Accept count for goto_window ([#1033](https://github.com/helix-editor/helix/pull/1033))
+- Make kill_to_line_end behave like emacs ([#1235](https://github.com/helix-editor/helix/pull/1235))
+- Only use a single documentation popup ([#1241](https://github.com/helix-editor/helix/pull/1241))
+- ui: popup: Don't allow scrolling past the end of content ([`3307f44c`](https://github.com/helix-editor/helix/commit/3307f44c))
+- Open files with spaces in filename, allow opening multiple files ([#1231](https://github.com/helix-editor/helix/pull/1231))
+- Allow paste commands to take a count ([#1261](https://github.com/helix-editor/helix/pull/1261))
+- Auto pairs selection ([#1254](https://github.com/helix-editor/helix/pull/1254))
+- Use a fuzzy matcher for commands ([#1386](https://github.com/helix-editor/helix/pull/1386))
+- Add `<C-s>` to pick word under doc cursor to prompt line & search completion ([#831](https://github.com/helix-editor/helix/pull/831))
+- Fix `:earlier`/`:later` missing changeset update ([#1069](https://github.com/helix-editor/helix/pull/1069))
+- Support extend for multiple goto ([#909](https://github.com/helix-editor/helix/pull/909))
+- Add arrow-key bindings for window switching ([#933](https://github.com/helix-editor/helix/pull/933))
+- Implement key ordering for info box ([#952](https://github.com/helix-editor/helix/pull/952))
+
+LSP:
+- Implement MarkedString rendering ([`e128a8702`](https://github.com/helix-editor/helix/commit/e128a8702))
+- Don't panic if init fails ([`d31bef7`](https://github.com/helix-editor/helix/commit/d31bef7))
+- Configurable diagnostic severity ([#1325](https://github.com/helix-editor/helix/pull/1325))
+- Resolve completion item ([#1315](https://github.com/helix-editor/helix/pull/1315))
+- Code action command support ([#1304](https://github.com/helix-editor/helix/pull/1304))
+
+Grammars:
+
+- Adds mint language server ([#974](https://github.com/helix-editor/helix/pull/974))
+- Perl ([#978](https://github.com/helix-editor/helix/pull/978)) ([#1280](https://github.com/helix-editor/helix/pull/1280))
+- GLSL ([#993](https://github.com/helix-editor/helix/pull/993))
+- Racket ([#1143](https://github.com/helix-editor/helix/pull/1143))
+- WGSL ([#1166](https://github.com/helix-editor/helix/pull/1166))
+- LLVM ([#1167](https://github.com/helix-editor/helix/pull/1167)) ([#1388](https://github.com/helix-editor/helix/pull/1388)) ([#1409](https://github.com/helix-editor/helix/pull/1409)) ([#1398](https://github.com/helix-editor/helix/pull/1398))
+- Markdown ([`49e06787`](https://github.com/helix-editor/helix/commit/49e06787))
+- Scala ([#1278](https://github.com/helix-editor/helix/pull/1278))
+- Dart ([#1250](https://github.com/helix-editor/helix/pull/1250))
+- Fish ([#1308](https://github.com/helix-editor/helix/pull/1308))
+- Dockerfile ([#1303](https://github.com/helix-editor/helix/pull/1303))
+- Git (commit, rebase, diff) ([#1338](https://github.com/helix-editor/helix/pull/1338)) ([#1402](https://github.com/helix-editor/helix/pull/1402)) ([#1373](https://github.com/helix-editor/helix/pull/1373))
+- tree-sitter-comment ([#1300](https://github.com/helix-editor/helix/pull/1300))
+- Highlight comments in c, cpp, cmake and llvm ([#1309](https://github.com/helix-editor/helix/pull/1309))
+- Improve yaml syntax highlighting highlighting ([#1294](https://github.com/helix-editor/helix/pull/1294))
+- Improve rust syntax highlighting ([#1295](https://github.com/helix-editor/helix/pull/1295))
+- Add textobjects and indents to cmake ([#1307](https://github.com/helix-editor/helix/pull/1307))
+- Add textobjects and indents to c and cpp ([#1293](https://github.com/helix-editor/helix/pull/1293))
+
+New themes:
+
+- Solarized dark ([#999](https://github.com/helix-editor/helix/pull/999))
+- Solarized light ([#1010](https://github.com/helix-editor/helix/pull/1010))
+- Spacebones light ([#1131](https://github.com/helix-editor/helix/pull/1131))
+- Monokai Pro ([#1206](https://github.com/helix-editor/helix/pull/1206))
+- Base16 Light and Terminal ([#1078](https://github.com/helix-editor/helix/pull/1078))
+  - and a default 16 color theme, truecolor detection
+- Dracula ([#1258](https://github.com/helix-editor/helix/pull/1258))
+
+# 0.5.0 (2021-10-28)
 
 A big shout out to all the contributors! We had 46 contributors in this release.
 
@@ -16,72 +298,74 @@ Breaking changes:
   - `keep_primary_selection` moved from `space+space` to `,`
   - `Alt-,` is now `remove_primary_selection` which keeps all selections except the primary one
   - Opening files in a split moved from `C-h` to `C-s`
-- Some configuration options moved from a `[terminal]` section to `[editor]`. [Consult the documentation for more information.](https://docs.helix-editor.com/configuration.html).
+- Some configuration options moved from a `[terminal]` section to `[editor]`. [Consult the documentation for more information.](https://docs.helix-editor.com/configuration.html)
 
 Features:
 
 - LSP compatibility greatly improved for some implementations (Julia, Python, Typescript)
 - Autocompletion! Completion now triggers automatically after a set idle timeout
-- Completion documentation is now displayed next to the popup (#691)
-- Treesitter textobjects (select a function via `mf`, class via `mc`) (#728) 
-- Global search across entire workspace `space+/` (#651) 
-- Relative line number support (#485)
-- Prompts now store a history (72cf86e)
-- `:vsplit` and `:hsplit` commands (#639)
-- `C-w h/j/k/l` can now be used to navigate between splits (#860)
-- `C-j` and `C-k` are now alternative keybindings to `C-n` and `C-p` in the UI (#876)
-- Shell commands (shell-pipe, pipe-to, shell-insert-output, shell-append-output, keep-pipe) (#547)
-- Searching now defaults to smart case search (case insensitive unless uppercase is used) (#761)
+- Completion documentation is now displayed next to the popup ([#691](https://github.com/helix-editor/helix/pull/691))
+- Treesitter textobjects (select a function via `mf`, class via `mc`) ([#728](https://github.com/helix-editor/helix/pull/728))
+- Global search across entire workspace `space+/` ([#651](https://github.com/helix-editor/helix/pull/651))
+- Relative line number support ([#485](https://github.com/helix-editor/helix/pull/485))
+- Prompts now store a history ([`72cf86e`](https://github.com/helix-editor/helix/commit/72cf86e))
+- `:vsplit` and `:hsplit` commands ([#639](https://github.com/helix-editor/helix/pull/639))
+- `C-w h/j/k/l` can now be used to navigate between splits ([#860](https://github.com/helix-editor/helix/pull/860))
+- `C-j` and `C-k` are now alternative keybindings to `C-n` and `C-p` in the UI ([#876](https://github.com/helix-editor/helix/pull/876))
+- Shell commands (shell-pipe, pipe-to, shell-insert-output, shell-append-output, keep-pipe) ([#547](https://github.com/helix-editor/helix/pull/547))
+- Searching now defaults to smart case search (case insensitive unless uppercase is used) ([#761](https://github.com/helix-editor/helix/pull/761))
 - The preview pane was improved to highlight and center line ranges
-- The user `languages.toml` is now merged into defaults, no longer need to copy the entire file (dc57f8dc)
-- Show hidden files in completions (#648)
-- Grammar injections are now properly handled (dd0b15e)
-- `v` in select mode now switches back to normal mode (#660)
-- View mode can now be triggered as a "sticky" mode (#719)
-- `f`/`t` and object selection motions can now be repeated via `Alt-.` (#891)
-- Statusline now displays total selection count and diagnostics counts for both errors and warnings (#916)
+- The user `languages.toml` is now merged into defaults, no longer need to copy the entire file ([`dc57f8dc`](https://github.com/helix-editor/helix/commit/dc57f8dc))
+- Show hidden files in completions ([#648](https://github.com/helix-editor/helix/pull/648))
+- Grammar injections are now properly handled ([`dd0b15e`](https://github.com/helix-editor/helix/commit/dd0b15e))
+- `v` in select mode now switches back to normal mode ([#660](https://github.com/helix-editor/helix/pull/660))
+- View mode can now be triggered as a "sticky" mode ([#719](https://github.com/helix-editor/helix/pull/719))
+- `f`/`t` and object selection motions can now be repeated via `Alt-.` ([#891](https://github.com/helix-editor/helix/pull/891))
+- Statusline now displays total selection count and diagnostics counts for both errors and warnings ([#916](https://github.com/helix-editor/helix/pull/916))
 
 New grammars:
 
-- Ledger (#572)
-- Protobuf (#614)
-- Zig (#631)
-- YAML (#667)
-- Lua (#665)
-- OCaml (#666)
-- Svelte (#733)
-- Vue (#787)
-- Tree-sitter queries (#845)
-- Elixir (we switched over to the official grammar) (6c0786e)
-- Language server definitions for Nix and Elixir (#725)
+- Ledger ([#572](https://github.com/helix-editor/helix/pull/572))
+- Protobuf ([#614](https://github.com/helix-editor/helix/pull/614))
+- Zig ([#631](https://github.com/helix-editor/helix/pull/631))
+- YAML ([#667](https://github.com/helix-editor/helix/pull/667))
+- Lua ([#665](https://github.com/helix-editor/helix/pull/665))
+- OCaml ([#666](https://github.com/helix-editor/helix/pull/666))
+- Svelte ([#733](https://github.com/helix-editor/helix/pull/733))
+- Vue ([#787](https://github.com/helix-editor/helix/pull/787))
+- Tree-sitter queries ([#845](https://github.com/helix-editor/helix/pull/845))
+- CMake ([#888](https://github.com/helix-editor/helix/pull/888))
+- Elixir (we switched over to the official grammar) ([`6c0786e`](https://github.com/helix-editor/helix/commit/6c0786e))
+- Language server definitions for Nix and Elixir ([#725](https://github.com/helix-editor/helix/pull/725))
 - Python now uses `pylsp` instead of `pyls`
 - Python now supports indentation
 
 New themes:
 
-- Monokai (#628)
-- Everforest Dark (#760)
-- Nord (#799)
-- Base16 Default Dark (#833)
-- Rose Pine (#897)
+- Monokai ([#628](https://github.com/helix-editor/helix/pull/628))
+- Everforest Dark ([#760](https://github.com/helix-editor/helix/pull/760))
+- Nord ([#799](https://github.com/helix-editor/helix/pull/799))
+- Base16 Default Dark ([#833](https://github.com/helix-editor/helix/pull/833))
+- Rose Pine ([#897](https://github.com/helix-editor/helix/pull/897))
 
 Fixes:
 
-- Fix crash on empty rust file (#592)
-- Exit select mode after toggle comment (#598)
-- Pin popups with no positioning to the initial position (12ea3888)
-- xsel copy should not freeze the editor (6dd7dc4)
-- `*` now only sets the search register and doesn't jump to the next occurrence (3426285)
-- Goto line start/end commands extend when in select mode (#739) 
-- Fix documentation popups sometimes not getting fully highlighted (066367c)
-- Refactor apply_workspace_edit to remove assert (b02d872)
-- Wrap around the top of the picker menu when scrolling (c7d6e44)
-- Don't allow closing the last split if there's unsaved changes (3ff5b00)
-- Indentation used different default on hx vs hx new_file.txt (c913bad)
+- Fix crash on empty rust file ([#592](https://github.com/helix-editor/helix/pull/592))
+- Exit select mode after toggle comment ([#598](https://github.com/helix-editor/helix/pull/598))
+- Pin popups with no positioning to the initial position ([`12ea3888`](https://github.com/helix-editor/helix/commit/12ea3888))
+- xsel copy should not freeze the editor ([`6dd7dc4`](https://github.com/helix-editor/helix/commit/6dd7dc4))
+- `*` now only sets the search register and doesn't jump to the next occurrence ([`3426285`](https://github.com/helix-editor/helix/commit/3426285))
+- Goto line start/end commands extend when in select mode ([#739](https://github.com/helix-editor/helix/pull/739)) 
+- Fix documentation popups sometimes not getting fully highlighted ([`066367c`](https://github.com/helix-editor/helix/commit/066367c))
+- Refactor apply_workspace_edit to remove assert ([`b02d872`](https://github.com/helix-editor/helix/commit/b02d872))
+- Wrap around the top of the picker menu when scrolling ([`c7d6e44`](https://github.com/helix-editor/helix/commit/c7d6e44))
+- Don't allow closing the last split if there's unsaved changes ([`3ff5b00`](https://github.com/helix-editor/helix/commit/3ff5b00))
+- Indentation used different default on hx vs hx new_file.txt ([`c913bad`](https://github.com/helix-editor/helix/commit/c913bad))
 
 # 0.4.1 (2021-08-14)
 
 A minor release that includes:
+
 - A fix for rendering glitches that would occur after editing with multiple selections.
 - CI fix for grammars not being cross-compiled for aarch64
 
@@ -103,10 +387,10 @@ selections in the future as well as resolves many bugs and edge cases.
 - Autoinfo: `whichkey`-like popups which show available sub-mode shortcuts ([#316](https://github.com/helix-editor/helix/pull/316))
 - Added WORD movements (W/B/E) ([#390](https://github.com/helix-editor/helix/pull/390))
 - Vertical selections (repeat selection above/below) ([#462](https://github.com/helix-editor/helix/pull/462))
-- Selection rotation via `(` and `)` ([66a90130](https://github.com/helix-editor/helix/commit/66a90130a5f99d769e9f6034025297f78ecaa3ec))
-- Selection contents rotation via `Alt-(` and `Alt-)` ([02cba2a](https://github.com/helix-editor/helix/commit/02cba2a7f403f48eccb18100fb751f7b42373dba))
-- Completion behavior improvements ([f917b5a4](https://github.com/helix-editor/helix/commit/f917b5a441ff3ae582358b6939ffbf889f4aa530), [627b899](https://github.com/helix-editor/helix/commit/627b89931576f7af86166ae8d5cbc55537877473))
-- Fixed a language server crash ([385a6b5a](https://github.com/helix-editor/helix/commit/385a6b5a1adddfc26e917982641530e1a7c7aa81))
+- Selection rotation via `(` and `)` ([`66a90130`](https://github.com/helix-editor/helix/commit/66a90130a5f99d769e9f6034025297f78ecaa3ec))
+- Selection contents rotation via `Alt-(` and `Alt-)` ([`02cba2a`](https://github.com/helix-editor/helix/commit/02cba2a7f403f48eccb18100fb751f7b42373dba))
+- Completion behavior improvements ([`f917b5a4`](https://github.com/helix-editor/helix/commit/f917b5a441ff3ae582358b6939ffbf889f4aa530), [`627b899`](https://github.com/helix-editor/helix/commit/627b89931576f7af86166ae8d5cbc55537877473))
+- Fixed a language server crash ([`385a6b5a`](https://github.com/helix-editor/helix/commit/385a6b5a1adddfc26e917982641530e1a7c7aa81))
 - Case change commands (`` ` ``, `~`, ``<a-`>``) ([#441](https://github.com/helix-editor/helix/pull/441))
 - File pickers (including goto) now provide a preview! ([#534](https://github.com/helix-editor/helix/pull/534))
 - Injection query support. Rust macro calls and embedded languages are now properly highlighted ([#430](https://github.com/helix-editor/helix/pull/430))
@@ -122,7 +406,7 @@ selections in the future as well as resolves many bugs and edge cases.
 - Comment toggling now uses a language specific comment token ([#463](https://github.com/helix-editor/helix/pull/463))
 - Julia support ([#413](https://github.com/helix-editor/helix/pull/413))
 - Java support ([#448](https://github.com/helix-editor/helix/pull/448))
-- Prompts have an (in-memory) history ([63e54e30](https://github.com/helix-editor/helix/commit/63e54e30a74bb0d1d782877ddbbcf95f2817d061))
+- Prompts have an (in-memory) history ([`63e54e30`](https://github.com/helix-editor/helix/commit/63e54e30a74bb0d1d782877ddbbcf95f2817d061))
 
 # 0.3.0 (2021-06-27)
 
@@ -136,7 +420,7 @@ Highlights:
 - Support for other line endings (CRLF). Significantly improved Windows support. ([#224](https://github.com/helix-editor/helix/pull/224))
 - Encodings other than UTF-8 are now supported! ([#228](https://github.com/helix-editor/helix/pull/228))
 - Key bindings can now be configured via a `config.toml` file ([#268](https://github.com/helix-editor/helix/pull/268))
-- Theme can now be configured and changed at runtime ([please feel free to contribute more themes!](https://github.com/helix-editor/helix/tree/master/runtime/themes)) ([#267](https://github.com/helix-editor/helix/pull/267))
+- Theme can now be configured and changed at runtime. ([Please feel free to contribute more themes!](https://github.com/helix-editor/helix/tree/master/runtime/themes)) ([#267](https://github.com/helix-editor/helix/pull/267))
 - System clipboard yank/paste is now supported! ([#310](https://github.com/helix-editor/helix/pull/310))
 - Surround commands were implemented ([#320](https://github.com/helix-editor/helix/pull/320))
 
@@ -153,7 +437,7 @@ Features:
 - Code is being migrated from helix-term to helix-view (prerequisite for
   alternative frontends) ([#366](https://github.com/helix-editor/helix/pull/366))
 - `x` and `X` merged
-  ([f41688d9](https://github.com/helix-editor/helix/commit/f41688d960ef89c29c4a51c872b8406fb8f81a85))
+  ([`f41688d9`](https://github.com/helix-editor/helix/commit/f41688d960ef89c29c4a51c872b8406fb8f81a85))
 
 Fixes:
 
@@ -161,12 +445,12 @@ Fixes:
 - A bunch of bugs regarding `o`/`O` behavior ([#281](https://github.com/helix-editor/helix/pull/281))
 - `~` expansion now works in file completion ([#284](https://github.com/helix-editor/helix/pull/284))
 - Several UI related overflow crashes ([#318](https://github.com/helix-editor/helix/pull/318))
-- Fix a test failure occuring only on `test --release` ([4f108ab1](https://github.com/helix-editor/helix/commit/4f108ab1b2197809506bd7305ad903a3525eabfa))
+- Fix a test failure occurring only on `test --release` ([`4f108ab1`](https://github.com/helix-editor/helix/commit/4f108ab1b2197809506bd7305ad903a3525eabfa))
 - Prompts now support unicode input ([#295](https://github.com/helix-editor/helix/pull/295))
 - Completion documentation no longer overlaps the popup ([#322](https://github.com/helix-editor/helix/pull/322))
-- Fix a crash when trying to select `^` ([9c534614](https://github.com/helix-editor/helix/commit/9c53461429a3e72e3b1fb87d7ca490e168d7dee2))
-- Prompt completions are now paginated ([39dc09e6](https://github.com/helix-editor/helix/commit/39dc09e6c4172299bc79de4c1c52288d3f624bd7))
-- Goto did not work on Windows ([503ca112](https://github.com/helix-editor/helix/commit/503ca112ae57ebdf3ea323baf8940346204b46d2))
+- Fix a crash when trying to select `^` ([`9c534614`](https://github.com/helix-editor/helix/commit/9c53461429a3e72e3b1fb87d7ca490e168d7dee2))
+- Prompt completions are now paginated ([`39dc09e6`](https://github.com/helix-editor/helix/commit/39dc09e6c4172299bc79de4c1c52288d3f624bd7))
+- Goto did not work on Windows ([`503ca112`](https://github.com/helix-editor/helix/commit/503ca112ae57ebdf3ea323baf8940346204b46d2))
 
 # 0.2.1
 

Cargo.toml

@@ -4,13 +4,31 @@ members = [
   "helix-view",
   "helix-term",
   "helix-tui",
-  "helix-syntax",
+  "helix-graphics",
+  "helix-ui",
   "helix-lsp",
+  "helix-dap",
+  "helix-loader",
+  "xtask",
 ]
 
-# Build helix-syntax in release mode to make the code path faster in development.
-# [profile.dev.package."helix-syntax"]
-# opt-level = 3
+default-members = [
+  "helix-term",
+  "helix-ui"
+]
+
+resolver = "2"
 
 [profile.dev]
 split-debuginfo = "unpacked"
+
+[profile.release]
+lto = "thin"
+# debug = true
+
+[profile.opt]
+inherits = "release"
+lto = "fat"
+codegen-units = 1
+# strip = "debuginfo" # TODO: or strip = true
+opt-level = 3

README.md

@@ -27,25 +27,34 @@ All shortcuts/keymaps can be found [in the documentation on the website](https:/
 It's a terminal-based editor first, but I'd like to explore a custom renderer
 (similar to emacs) in wgpu or skulpin.
 
+Note: Only certain languages have indentation definitions at the moment. Check
+`runtime/queries/<lang>/` for `indents.scm`.
+
 # Installation
 
-Note: Only certain languages have indentation definitions at the moment. Check
-`runtime/queries/<lang>/` for `indents.toml`.
+Packages are available for various distributions (see [Installation docs](https://docs.helix-editor.com/install.html)).
 
-We provide packaging for various distributions, but here's a quick method to
-build from source.
+If you would like to build from source:
 
-```
-git clone --recurse-submodules --shallow-submodules -j8 https://github.com/helix-editor/helix
+```shell
+git clone https://github.com/helix-editor/helix
 cd helix
 cargo install --path helix-term
+hx --grammar fetch
+hx --grammar build
 ```
 
-This will install the `hx` binary to `$HOME/.cargo/bin`.
+This will install the `hx` binary to `$HOME/.cargo/bin` and build tree-sitter grammars.
 
 Helix also needs its runtime files so make sure to copy/symlink the `runtime/` directory into the
-config directory (for example `~/.config/helix/runtime` on Linux/macOS). This location can be overriden
-via the `HELIX_RUNTIME` environment variable.
+config directory (for example `~/.config/helix/runtime` on Linux/macOS, or `%AppData%/helix/runtime` on Windows).
+
+| OS        | command   |
+|-----------|-----------|
+|windows    |`xcopy runtime %AppData%/helix/runtime`|
+|linux/macos|`ln -s $PWD/runtime ~/.config/helix/runtime`
+
+This location can be overridden via the `HELIX_RUNTIME` environment variable.
 
 Packages already solve this for you by wrapping the `hx` binary with a wrapper
 that sets the variable to the install dir.
@@ -53,9 +62,14 @@ that sets the variable to the install dir.
 > NOTE: running via cargo also doesn't require setting explicit `HELIX_RUNTIME` path, it will automatically
 > detect the `runtime` directory in the project root.
 
+In order to use LSP features like auto-complete, you will need to
+[install the appropriate Language Server](https://github.com/helix-editor/helix/wiki/How-to-install-the-default-language-servers)
+for a language.
+
 [![Packaging status](https://repology.org/badge/vertical-allrepos/helix.svg)](https://repology.org/project/helix/versions)
 
 ## MacOS
+
 Helix can be installed on MacOS through homebrew via:
 
 ```
@@ -65,21 +79,7 @@ brew install helix
  
 # Contributing
 
-Contributors are very welcome! **No contribution is too small and all contributions are valued.**
-
-Some suggestions to get started:
-
-- You can look at the [good first issue](https://github.com/helix-editor/helix/labels/E-easy) label on the issue tracker.
-- Help with packaging on various distributions needed!
-- To use print debugging to the `~/.cache/helix/helix.log` file, you must:
-  * Print using `log::info!`, `warn!`, or `error!`. (`log::info!("helix!")`)
-  * Pass the appropriate verbosity level option for the desired log level. (`hx -v <file>` for info, more `v`s for higher severity inclusive)
-- If your preferred language is missing, integrating a tree-sitter grammar for
-    it and defining syntax highlight queries for it is straight forward and
-    doesn't require much knowledge of the internals.
-
-We provide an [architecture.md](./docs/architecture.md) that should give you
-a good overview of the internals.
+Contributing guidelines can be found [here](./docs/CONTRIBUTING.md).
 
 # Getting help
 

TODO.md

@@ -1,32 +0,0 @@
-
-- tree sitter:
-  - markdown
-  - regex
-  - kotlin
-  - clojure
-  - erlang
-
-- [ ] completion isIncomplete support
-
-1
-- [ ] respect view fullscreen flag
-- [ ] Implement marks (superset of Selection/Range)
-
-- [ ] = for auto indent line/selection
-- [ ]  :x for closing buffers
-- [ ] lsp: signature help
-
-2
-- [ ] macro recording
-- [ ] extend selection (treesitter select parent node) (replaces viw, vi(, va( etc )
-- [ ] selection align
-- [ ] store some state between restarts: file positions, prompt history
-- [ ] highlight matched characters in picker
-
-3
-- [ ] diff mode with highlighting?
-- [ ] snippet support (tab to jump between marks)
-- [ ] gamelisp/wasm scripting
-
-X
-- [ ] rendering via skulpin/skia or raw wgpu

VERSION

@@ -0,0 +1 @@
+22.05-dev

base16_theme.toml

@@ -0,0 +1,51 @@
+# Author: NNB <nnbnh@protonmail.com>
+
+"ui.menu" = "black"
+"ui.menu.selected" = { modifiers = ["reversed"] }
+"ui.linenr" = { fg = "gray", bg = "black" }
+"ui.popup" = { modifiers = ["reversed"] }
+"ui.linenr.selected" = { fg = "white", bg = "black", modifiers = ["bold"] }
+"ui.selection" = { fg = "black", bg = "blue" }
+"ui.selection.primary" = { fg = "white", bg = "blue" }
+"comment" = { fg = "gray" }
+"ui.statusline" = { fg = "black", bg = "white" }
+"ui.statusline.inactive" = { fg = "gray", bg = "white" }
+"ui.help" = { modifiers = ["reversed"] }
+"ui.cursor" = { fg = "white", modifiers = ["reversed"] }
+"variable" = "red"
+"constant.numeric" = "yellow"
+"constant" = "yellow"
+"attributes" = "yellow"
+"type" = "yellow"
+"ui.cursor.match" = { fg = "yellow", modifiers = ["underlined"] }
+"string"  = "green"
+"variable.other.member" = "green"
+"constant.character.escape" = "cyan"
+"function" = "blue"
+"constructor" = "blue"
+"special" = "blue"
+"keyword" = "magenta"
+"label" = "magenta"
+"namespace" = "magenta"
+"ui.help" = { fg = "white", bg = "black" }
+
+"markup.heading" = "blue"
+"markup.list" = "red"
+"markup.bold" = { fg = "yellow", modifiers = ["bold"] }
+"markup.italic" = { fg = "magenta", modifiers = ["italic"] }
+"markup.link.url" = { fg = "yellow", modifiers = ["underlined"] }
+"markup.link.text" = "red"
+"markup.quote" = "cyan"
+"markup.raw" = "green"
+
+"diff.plus" = "green"
+"diff.delta" = "yellow"
+"diff.minus" = "red"
+
+"diagnostic" = { modifiers = ["underlined"] }
+"ui.gutter" = { bg = "black" }
+"info" = "blue"
+"hint" = "gray"
+"debug" = "gray"
+"warning" = "yellow"
+"error" = "red"

book/src/SUMMARY.md

@@ -1,12 +1,19 @@
 # Summary
 
+[Helix](./title-page.md)
+
 - [Installation](./install.md)
 - [Usage](./usage.md)
+  - [Keymap](./keymap.md)
+  - [Commands](./commands.md)
+  - [Language Support](./lang-support.md)
 - [Migrating from Vim](./from-vim.md)
 - [Configuration](./configuration.md)
   - [Themes](./themes.md)
-  - [Keymap](./keymap.md)
   - [Key Remapping](./remapping.md)
   - [Hooks](./hooks.md)
+  - [Languages](./languages.md)
 - [Guides](./guides/README.md)
+  - [Adding Languages](./guides/adding_languages.md)
   - [Adding Textobject Queries](./guides/textobject.md)
+  - [Adding Indent Queries](./guides/indent.md)

book/src/commands.md

@@ -0,0 +1,5 @@
+# Commands
+
+Command mode can be activated by pressing `:`, similar to vim. Built-in commands:
+
+{{#include ./generated/typable-cmd.md}}

book/src/configuration.md

@@ -5,9 +5,29 @@ To override global configuration parameters, create a `config.toml` file located
 * Linux and Mac: `~/.config/helix/config.toml`
 * Windows: `%AppData%\helix\config.toml`
 
+> Hint: You can easily open the config file by typing `:config-open` within Helix normal mode.
+
+Example config:
+
+```toml
+theme = "onedark"
+
+[editor]
+line-number = "relative"
+mouse = false
+
+[editor.cursor-shape]
+insert = "bar"
+normal = "block"
+select = "underline"
+
+[editor.file-picker]
+hidden = false
+```
+
 ## Editor
 
-`[editor]` section of the config.
+### `[editor]` Section
 
 | Key | Description | Default |
 |--|--|---------|
@@ -16,17 +36,131 @@ To override global configuration parameters, create a `config.toml` file located
 | `middle-click-paste` | Middle click paste support. | `true` |
 | `scroll-lines` | Number of lines to scroll per scroll wheel step. | `3` |
 | `shell` | Shell to use when running external commands. | Unix: `["sh", "-c"]`<br/>Windows: `["cmd", "/C"]` |
-| `line-number` | Line number display (`absolute`, `relative`) | `absolute` |
-| `smart-case` | Enable smart case regex searching (case insensitive unless pattern contains upper case characters) | `true` |
-| `auto-pairs` | Enable automatic insertion of pairs to parenthese, brackets, etc. | `true` |
+| `line-number` | Line number display: `absolute` simply shows each line's number, while `relative` shows the distance from the current line. When unfocused or in insert mode, `relative` will still show absolute line numbers. | `absolute` |
+| `gutters` | Gutters to display: Available are `diagnostics` and `line-numbers`, note that `diagnostics` also includes other features like breakpoints | `["diagnostics", "line-numbers"]` |
 | `auto-completion` | Enable automatic pop up of auto-completion. | `true` |
 | `idle-timeout` | Time in milliseconds since last keypress before idle timers trigger. Used for autocompletion, set to 0 for instant. | `400` |
 | `completion-trigger-len` | The min-length of word under cursor to trigger autocompletion | `2` |
+| `auto-info` | Whether to display infoboxes | `true` |
+| `true-color` | Set to `true` to override automatic detection of terminal truecolor support in the event of a false negative. | `false` |
+| `rulers` | List of column positions at which to display the rulers. Can be overridden by language specific `rulers` in `languages.toml` file. | `[]` |
 
-## LSP
+### `[editor.lsp]` Section
+
+| Key                | Description                                 | Default |
+| ---                | -----------                                 | ------- |
+| `display-messages` | Display LSP progress messages below statusline[^1] | `false` |
+
+[^1]: A progress spinner is always shown in the statusline beside the file path.
+
+### `[editor.cursor-shape]` Section
+
+Defines the shape of cursor in each mode. Note that due to limitations
+of the terminal environment, only the primary cursor can change shape.
+Valid values for these options are `block`, `bar`, `underline`, or `none`.
+
+| Key      | Description                                | Default |
+| ---      | -----------                                | ------- |
+| `normal` | Cursor shape in [normal mode][normal mode] | `block` |
+| `insert` | Cursor shape in [insert mode][insert mode] | `block` |
+| `select` | Cursor shape in [select mode][select mode] | `block` |
+
+[normal mode]: ./keymap.md#normal-mode
+[insert mode]: ./keymap.md#insert-mode
+[select mode]: ./keymap.md#select--extend-mode
+
+### `[editor.file-picker]` Section
+
+Sets options for file picker and global search. All but the last key listed in
+the default file-picker configuration below are IgnoreOptions: whether hidden
+files and files listed within ignore files are ignored by (not visible in) the
+helix file picker and global search. There is also one other key, `max-depth`
+available, which is not defined by default.
+
+| Key | Description | Default |
+|--|--|---------|
+|`hidden` | Enables ignoring hidden files. | true
+|`parents` | Enables reading ignore files from parent directories. | true
+|`ignore` | Enables reading `.ignore` files. | true
+|`git-ignore` | Enables reading `.gitignore` files. | true
+|`git-global` | Enables reading global .gitignore, whose path is specified in git's config: `core.excludefile` option. | true
+|`git-exclude` | Enables reading `.git/info/exclude` files. | true
+|`max-depth` | Set with an integer value for maximum depth to recurse. | Defaults to `None`.
+
+### `[editor.auto-pairs]` Section
+
+Enable automatic insertion of pairs to parentheses, brackets, etc. Can be
+a simple boolean value, or a specific mapping of pairs of single characters.
+
+| Key | Description |
+| --- | ----------- |
+| `false` | Completely disable auto pairing, regardless of language-specific settings
+| `true` | Use the default pairs: <code>(){}[]''""``</code>
+| Mapping of pairs | e.g. `{ "(" =  ")", "{" = "}", ... }`
+
+Example
 
-To display all language server messages in the status line add the following to your `config.toml`:
 ```toml
-[lsp]
-display-messages = true
+[editor.auto-pairs]
+'(' = ')'
+'{' = '}'
+'[' = ']'
+'"' = '"'
+'`' = '`'
+'<' = '>'
+```
+
+Additionally, this setting can be used in a language config. Unless
+the editor setting is `false`, this will override the editor config in
+documents with this language.
+
+Example `languages.toml` that adds <> and removes ''
+
+```toml
+[[language]]
+name = "rust"
+
+[language.auto-pairs]
+'(' = ')'
+'{' = '}'
+'[' = ']'
+'"' = '"'
+'`' = '`'
+'<' = '>'
+```
+
+### `[editor.search]` Section
+
+Search specific options.
+
+| Key | Description | Default |
+|--|--|---------|
+| `smart-case` | Enable smart case regex searching (case insensitive unless pattern contains upper case characters) | `true` |
+| `wrap-around`| Whether the search should wrap after depleting the matches | `true` |
+
+### `[editor.whitespace]` Section
+
+Options for rendering whitespace with visible characters. Use `:set whitespace.render all` to temporarily enable visible whitespace.
+
+| Key | Description | Default |
+|-----|-------------|---------|
+| `render` | Whether to render whitespace. May either be `"all"` or `"none"`, or a table with sub-keys `space`, `tab`, and `newline`. | `"none"` |
+| `characters` | Literal characters to use when rendering whitespace. Sub-keys may be any of `tab`, `space`, `nbsp` or `newline` | See example below |
+
+Example
+
+```toml
+[editor.whitespace]
+render = "all"
+# or control each character
+[editor.whitespace.render]
+space = "all"
+tab = "all"
+newline = "none"
+
+[editor.whitespace.characters]
+space = "·"
+nbsp = "⍽"
+tab = "→"
+newline = "⏎"
 ```

book/src/generated/lang-support.md

@@ -0,0 +1,89 @@
+| Language | Syntax Highlighting | Treesitter Textobjects | Auto Indent | Default LSP |
+| --- | --- | --- | --- | --- |
+| bash | ✓ |  |  | `bash-language-server` |
+| c | ✓ | ✓ | ✓ | `clangd` |
+| c-sharp | ✓ |  |  | `OmniSharp` |
+| cairo | ✓ |  |  |  |
+| cmake | ✓ | ✓ | ✓ | `cmake-language-server` |
+| comment | ✓ |  |  |  |
+| cpp | ✓ | ✓ | ✓ | `clangd` |
+| css | ✓ |  |  | `vscode-css-language-server` |
+| dart | ✓ |  | ✓ | `dart` |
+| devicetree | ✓ |  | ✓ |  |
+| dockerfile | ✓ |  |  | `docker-langserver` |
+| eex | ✓ |  |  |  |
+| ejs | ✓ |  |  |  |
+| elixir | ✓ |  |  | `elixir-ls` |
+| elm | ✓ |  |  | `elm-language-server` |
+| erb | ✓ |  |  |  |
+| erlang | ✓ |  |  | `erlang_ls` |
+| fish | ✓ | ✓ | ✓ |  |
+| gdscript | ✓ |  | ✓ |  |
+| git-commit | ✓ |  |  |  |
+| git-config | ✓ |  |  |  |
+| git-diff | ✓ |  |  |  |
+| git-rebase | ✓ |  |  |  |
+| gleam | ✓ |  |  |  |
+| glsl | ✓ |  | ✓ |  |
+| go | ✓ | ✓ | ✓ | `gopls` |
+| gomod | ✓ |  |  | `gopls` |
+| gowork | ✓ |  |  | `gopls` |
+| graphql | ✓ |  |  |  |
+| hare | ✓ |  | ✓ |  |
+| haskell | ✓ |  |  | `haskell-language-server-wrapper` |
+| hcl | ✓ |  | ✓ | `terraform-ls` |
+| heex | ✓ |  |  |  |
+| html | ✓ |  |  | `vscode-html-language-server` |
+| iex | ✓ |  |  |  |
+| java | ✓ |  |  |  |
+| javascript | ✓ |  | ✓ | `typescript-language-server` |
+| json | ✓ |  | ✓ | `vscode-json-language-server` |
+| jsx | ✓ |  | ✓ | `typescript-language-server` |
+| julia | ✓ |  |  | `julia` |
+| kotlin | ✓ |  |  | `kotlin-language-server` |
+| latex | ✓ |  |  | `texlab` |
+| lean | ✓ |  |  | `lean` |
+| ledger | ✓ |  |  |  |
+| llvm | ✓ | ✓ | ✓ |  |
+| llvm-mir | ✓ | ✓ | ✓ |  |
+| llvm-mir-yaml | ✓ |  | ✓ |  |
+| lua | ✓ |  | ✓ |  |
+| make | ✓ |  |  |  |
+| markdown | ✓ |  |  |  |
+| mint |  |  |  | `mint` |
+| nickel | ✓ |  | ✓ | `nls` |
+| nix | ✓ |  | ✓ | `rnix-lsp` |
+| nu | ✓ |  |  |  |
+| ocaml | ✓ |  | ✓ | `ocamllsp` |
+| ocaml-interface | ✓ |  |  | `ocamllsp` |
+| org | ✓ |  |  |  |
+| perl | ✓ | ✓ | ✓ |  |
+| php | ✓ | ✓ | ✓ | `intelephense` |
+| prolog |  |  |  | `swipl` |
+| protobuf | ✓ |  | ✓ |  |
+| python | ✓ | ✓ | ✓ | `pylsp` |
+| r | ✓ |  |  | `R` |
+| racket |  |  |  | `racket` |
+| regex | ✓ |  |  |  |
+| rescript | ✓ | ✓ |  | `rescript-language-server` |
+| rmarkdown | ✓ |  | ✓ | `R` |
+| ron | ✓ |  | ✓ |  |
+| ruby | ✓ | ✓ | ✓ | `solargraph` |
+| rust | ✓ | ✓ | ✓ | `rust-analyzer` |
+| scala | ✓ |  | ✓ | `metals` |
+| solidity | ✓ |  |  | `solc` |
+| sql | ✓ |  |  |  |
+| svelte | ✓ |  | ✓ | `svelteserver` |
+| swift | ✓ |  |  | `sourcekit-lsp` |
+| tablegen | ✓ | ✓ | ✓ |  |
+| tfvars |  |  |  | `terraform-ls` |
+| toml | ✓ |  |  | `taplo` |
+| tsq | ✓ |  |  |  |
+| tsx | ✓ |  |  | `typescript-language-server` |
+| twig | ✓ |  |  |  |
+| typescript | ✓ |  | ✓ | `typescript-language-server` |
+| vala | ✓ |  |  | `vala-language-server` |
+| vue | ✓ |  |  | `vls` |
+| wgsl | ✓ |  |  |  |
+| yaml | ✓ |  | ✓ | `yaml-language-server` |
+| zig | ✓ |  | ✓ | `zls` |

book/src/generated/typable-cmd.md

@@ -0,0 +1,65 @@
+| Name | Description |
+| --- | --- |
+| `:quit`, `:q` | Close the current view. |
+| `:quit!`, `:q!` | Close the current view forcefully (ignoring unsaved changes). |
+| `:open`, `:o` | Open a file from disk into the current view. |
+| `:buffer-close`, `:bc`, `:bclose` | Close the current buffer. |
+| `:buffer-close!`, `:bc!`, `:bclose!` | Close the current buffer forcefully (ignoring unsaved changes). |
+| `:buffer-close-others`, `:bco`, `:bcloseother` | Close all buffers but the currently focused one. |
+| `:buffer-close-others!`, `:bco!`, `:bcloseother!` | Close all buffers but the currently focused one. |
+| `:buffer-close-all`, `:bca`, `:bcloseall` | Close all buffers, without quitting. |
+| `:buffer-close-all!`, `:bca!`, `:bcloseall!` | Close all buffers forcefully (ignoring unsaved changes), without quitting. |
+| `:buffer-next`, `:bn`, `:bnext` | Go to next buffer. |
+| `:buffer-previous`, `:bp`, `:bprev` | Go to previous buffer. |
+| `:write`, `:w` | Write changes to disk. Accepts an optional path (:write some/path.txt) |
+| `:write!`, `:w!` | Write changes to disk forcefully (creating necessary subdirectories). Accepts an optional path (:write some/path.txt) |
+| `:new`, `:n` | Create a new scratch buffer. |
+| `:format`, `:fmt` | Format the file using the LSP formatter. |
+| `:indent-style` | Set the indentation style for editing. ('t' for tabs or 1-8 for number of spaces.) |
+| `:line-ending` | Set the document's default line ending. Options: crlf, lf. |
+| `:earlier`, `:ear` | Jump back to an earlier point in edit history. Accepts a number of steps or a time span. |
+| `:later`, `:lat` | Jump to a later point in edit history. Accepts a number of steps or a time span. |
+| `:write-quit`, `:wq`, `:x` | Write changes to disk and close the current view. Accepts an optional path (:wq some/path.txt) |
+| `:write-quit!`, `:wq!`, `:x!` | Write changes to disk and close the current view forcefully. Accepts an optional path (:wq! some/path.txt) |
+| `:write-all`, `:wa` | Write changes from all views to disk. |
+| `:write-quit-all`, `:wqa`, `:xa` | Write changes from all views to disk and close all views. |
+| `:write-quit-all!`, `:wqa!`, `:xa!` | Write changes from all views to disk and close all views forcefully (ignoring unsaved changes). |
+| `:quit-all`, `:qa` | Close all views. |
+| `:quit-all!`, `:qa!` | Close all views forcefully (ignoring unsaved changes). |
+| `:cquit`, `:cq` | Quit with exit code (default 1). Accepts an optional integer exit code (:cq 2). |
+| `:cquit!`, `:cq!` | Quit with exit code (default 1) forcefully (ignoring unsaved changes). Accepts an optional integer exit code (:cq! 2). |
+| `:theme` | Change the editor theme. |
+| `:clipboard-yank` | Yank main selection into system clipboard. |
+| `:clipboard-yank-join` | Yank joined selections into system clipboard. A separator can be provided as first argument. Default value is newline. |
+| `:primary-clipboard-yank` | Yank main selection into system primary clipboard. |
+| `:primary-clipboard-yank-join` | Yank joined selections into system primary clipboard. A separator can be provided as first argument. Default value is newline. |
+| `:clipboard-paste-after` | Paste system clipboard after selections. |
+| `:clipboard-paste-before` | Paste system clipboard before selections. |
+| `:clipboard-paste-replace` | Replace selections with content of system clipboard. |
+| `:primary-clipboard-paste-after` | Paste primary clipboard after selections. |
+| `:primary-clipboard-paste-before` | Paste primary clipboard before selections. |
+| `:primary-clipboard-paste-replace` | Replace selections with content of system primary clipboard. |
+| `:show-clipboard-provider` | Show clipboard provider name in status bar. |
+| `:change-current-directory`, `:cd` | Change the current working directory. |
+| `:show-directory`, `:pwd` | Show the current working directory. |
+| `:encoding` | Set encoding based on `https://encoding.spec.whatwg.org` |
+| `:reload` | Discard changes and reload from the source file. |
+| `:tree-sitter-scopes` | Display tree sitter scopes, primarily for theming and development. |
+| `:debug-start`, `:dbg` | Start a debug session from a given template with given parameters. |
+| `:debug-remote`, `:dbg-tcp` | Connect to a debug adapter by TCP address and start a debugging session from a given template with given parameters. |
+| `:debug-eval` | Evaluate expression in current debug context. |
+| `:vsplit`, `:vs` | Open the file in a vertical split. |
+| `:vsplit-new`, `:vnew` | Open a scratch buffer in a vertical split. |
+| `:hsplit`, `:hs`, `:sp` | Open the file in a horizontal split. |
+| `:hsplit-new`, `:hnew` | Open a scratch buffer in a horizontal split. |
+| `:tutor` | Open the tutorial. |
+| `:goto`, `:g` | Go to line number. |
+| `:set-language`, `:lang` | Set the language of current buffer. |
+| `:set-option`, `:set` | Set a config option at runtime. |
+| `:get-option`, `:get` | Get the current value of a config option. |
+| `:sort` | Sort ranges in selection. |
+| `:rsort` | Sort ranges in selection in reverse order. |
+| `:tree-sitter-subtree`, `:ts-subtree` | Display tree sitter subtree under cursor, primarily for debugging queries. |
+| `:config-reload` | Refreshes helix's config. |
+| `:config-open` | Open the helix config.toml file. |
+| `:pipe` | Pipe each selection to the shell command. |

book/src/guides/README.md

@@ -1,4 +1,4 @@
 # Guides
 
 This section contains guides for adding new language server configurations,
-tree-sitter grammers, textobject queries, etc.
+tree-sitter grammars, textobject queries, etc.

book/src/guides/adding_languages.md

@@ -0,0 +1,98 @@
+# Adding languages
+
+## Language configuration
+
+To add a new language, you need to add a `language` entry to the
+[`languages.toml`][languages.toml] found in the root of the repository;
+this `languages.toml` file is included at compilation time, and is
+distinct from the `languages.toml` file in the user's [configuration
+directory](../configuration.md).
+
+```toml
+[[language]]
+name = "mylang"
+scope = "scope.mylang"
+injection-regex = "^mylang$"
+file-types = ["mylang", "myl"]
+comment-token = "#"
+indent = { tab-width = 2, unit = "  " }
+language-server = { command = "mylang-lsp", args = ["--stdio"] }
+```
+
+These are the available keys and descriptions for the file.
+
+| Key                   | Description                                                   |
+| ----                  | -----------                                                   |
+| `name`                | The name of the language                                      |
+| `scope`               | A string like `source.js` that identifies the language. Currently, we strive to match the scope names used by popular TextMate grammars and by the Linguist library. Usually `source.<name>` or `text.<name>` in case of markup languages |
+| `injection-regex`     | regex pattern that will be tested against a language name in order to determine whether this language should be used for a potential [language injection][treesitter-language-injection] site. |
+| `file-types`          | The filetypes of the language, for example `["yml", "yaml"]`. Extensions and full file names are supported.  |
+| `shebangs`            | The interpreters from the shebang line, for example `["sh", "bash"]` |
+| `roots`               | A set of marker files to look for when trying to find the workspace root. For example `Cargo.lock`, `yarn.lock` |
+| `auto-format`         | Whether to autoformat this language when saving               |
+| `diagnostic-severity` | Minimal severity of diagnostic for it to be displayed. (Allowed values: `Error`, `Warning`, `Info`, `Hint`) |
+| `comment-token`       | The token to use as a comment-token                           |
+| `indent`              | The indent to use. Has sub keys `tab-width` and `unit`        |
+| `language-server`     | The Language Server to run. Has sub keys `command` and `args` |
+| `config`              | Language Server configuration                                 |
+| `grammar`             | The tree-sitter grammar to use (defaults to the value of `name`) |
+
+When adding a new language or Language Server configuration for an existing
+language, run `cargo xtask docgen` to add the new configuration to the
+[Language Support][lang-support] docs before creating a pull request.
+When adding a Language Server configuration, be sure to update the
+[Language Server Wiki][install-lsp-wiki] with installation notes.
+
+## Grammar configuration
+
+If a tree-sitter grammar is available for the language, add a new `grammar`
+entry to `languages.toml`.
+
+```toml
+[[grammar]]
+name = "mylang"
+source = { git = "https://github.com/example/mylang", rev = "a250c4582510ff34767ec3b7dcdd3c24e8c8aa68" }
+```
+
+Grammar configuration takes these keys:
+
+| Key      | Description                                                              |
+| ---      | -----------                                                              |
+| `name`   | The name of the tree-sitter grammar                                      |
+| `source` | The method of fetching the grammar - a table with a schema defined below |
+
+Where `source` is a table with either these keys when using a grammar from a
+git repository:
+
+| Key    | Description                                               |
+| ---    | -----------                                               |
+| `git`  | A git remote URL from which the grammar should be cloned  |
+| `rev`  | The revision (commit hash or tag) which should be fetched |
+| `subpath` | A path within the grammar directory which should be built. Some grammar repositories host multiple grammars (for example `tree-sitter-typescript` and `tree-sitter-ocaml`) in subdirectories. This key is used to point `hx --grammar build` to the correct path for compilation. When omitted, the root of repository is used |
+
+Or a `path` key with an absolute path to a locally available grammar directory.
+
+## Queries
+
+For a language to have syntax-highlighting and indentation among
+other things, you have to add queries. Add a directory for your
+language with the path `runtime/queries/<name>/`. The tree-sitter
+[website](https://tree-sitter.github.io/tree-sitter/syntax-highlighting#queries)
+gives more info on how to write queries.
+
+> NOTE: When evaluating queries, the first matching query takes
+precedence, which is different from other editors like neovim where
+the last matching query supersedes the ones before it. See
+[this issue][neovim-query-precedence] for an example.
+
+## Common Issues
+
+- If you get errors when running after switching branches, you may have to update the tree-sitter grammars. Run `hx --grammar fetch` to fetch the grammars and `hx --grammar build` to build any out-of-date grammars.
+
+- If a parser is segfaulting or you want to remove the parser, make sure to remove the compiled parser in `runtime/grammar/<name>.so`
+
+[treesitter-language-injection]: https://tree-sitter.github.io/tree-sitter/syntax-highlighting#language-injection
+[languages.toml]: https://github.com/helix-editor/helix/blob/master/languages.toml
+[neovim-query-precedence]: https://github.com/helix-editor/helix/pull/1170#issuecomment-997294090
+[install-lsp-wiki]: https://github.com/helix-editor/helix/wiki/How-to-install-the-default-language-servers
+[lang-support]: ../lang-support.md

book/src/guides/indent.md

@@ -0,0 +1,79 @@
+# Adding Indent Queries
+
+Helix uses tree-sitter to correctly indent new lines. This requires
+a tree-sitter grammar and an `indent.scm` query file placed in
+`runtime/queries/{language}/indents.scm`. The indentation for a line
+is calculated by traversing the syntax tree from the lowest node at the
+beginning of the new line. Each of these nodes contributes to the total
+indent when it is captured by the query (in what way depends on the name
+of the capture).
+
+Note that it matters where these added indents begin. For example,
+multiple indent level increases that start on the same line only increase
+the total indent level by 1.
+
+## Scopes
+
+Added indents don't always apply to the whole node. For example, in most
+cases when a node should be indented, we actually only want everything
+except for its first line to be indented. For this, there are several
+scopes (more scopes may be added in the future if required):
+
+- `all`:
+This scope applies to the whole captured node. This is only different from
+`tail` when the captured node is the first node on its line.
+
+- `tail`:
+This scope applies to everything except for the first line of the
+captured node.
+
+Every capture type has a default scope which should do the right thing
+in most situations. When a different scope is required, this can be
+changed by using a `#set!` declaration anywhere in the pattern:
+```scm
+(assignment_expression
+  right: (_) @indent
+  (#set! "scope" "all"))
+```
+
+## Capture Types
+
+- `@indent` (default scope `tail`):
+Increase the indent level by 1. Multiple occurrences in the same line
+don't stack. If there is at least one `@indent` and one `@outdent`
+capture on the same line, the indent level isn't changed at all.
+
+- `@outdent` (default scope `all`):
+Decrease the indent level by 1. The same rules as for `@indent` apply.
+
+## Predicates
+
+In some cases, an S-expression cannot express exactly what pattern should be matched.
+For that, tree-sitter allows for predicates to appear anywhere within a pattern,
+similar to how `#set!` declarations work:
+```scm
+(some_kind
+  (child_kind) @indent
+  (#predicate? arg1 arg2 ...)
+)
+```
+The number of arguments depends on the predicate that's used.
+Each argument is either a capture (`@name`) or a string (`"some string"`).
+The following predicates are supported by tree-sitter:
+
+- `#eq?`/`#not-eq?`:
+The first argument (a capture) must/must not be equal to the second argument
+(a capture or a string).
+
+- `#match?`/`#not-match?`:
+The first argument (a capture) must/must not match the regex given in the
+second argument (a string).
+
+Additionally, we support some custom predicates for indent queries:
+
+- `#not-kind-eq?`:
+The kind of the first argument (a capture) must not be equal to the second
+argument (a string).
+
+- `#same-line?`/`#not-same-line?`:
+The captures given by the 2 arguments must/must not start on the same line.

book/src/guides/textobject.md

@@ -5,7 +5,7 @@ require an accompanying tree-sitter grammar and a `textobjects.scm` query file
 to work properly. Tree-sitter allows us to query the source code syntax tree
 and capture specific parts of it. The queries are written in a lisp dialect.
 More information on how to write queries can be found in the [official tree-sitter
-documentation](tree-sitter-queries).
+documentation][tree-sitter-queries].
 
 Query files should be placed in `runtime/queries/{language}/textobjects.scm`
 when contributing. Note that to test the query files locally you should put
@@ -21,10 +21,27 @@ The following [captures][tree-sitter-captures] are recognized:
 | `class.inside`     |
 | `class.around`     |
 | `parameter.inside` |
+| `comment.inside`   |
+| `comment.around`   |
 
 [Example query files][textobject-examples] can be found in the helix GitHub repository.
 
+## Queries for Textobject Based Navigation
+
+[Tree-sitter based navigation][textobjects-nav] is done using captures in the
+following order:
+
+- `object.movement`
+- `object.around`
+- `object.inside`
+
+For example if a `function.around` capture has been already defined for a language
+in it's `textobjects.scm` file, function navigation should also work automatically.
+`function.movement` should be defined only if the node captured by `function.around`
+doesn't make sense in a navigation context.
+
 [textobjects]: ../usage.md#textobjects
+[textobjects-nav]: ../usage.md#tree-sitter-textobject-based-navigation
 [tree-sitter-queries]: https://tree-sitter.github.io/tree-sitter/using-parsers#query-syntax
 [tree-sitter-captures]: https://tree-sitter.github.io/tree-sitter/using-parsers#capturing-nodes
 [textobject-examples]: https://github.com/search?q=repo%3Ahelix-editor%2Fhelix+filename%3Atextobjects.scm&type=Code&ref=advsearch&l=&l=

book/src/install.md

@@ -19,20 +19,38 @@ brew install helix
 
 A [flake](https://nixos.wiki/wiki/Flakes) containing the package is available in
 the project root. The flake can also be used to spin up a reproducible development
-shell for working on Helix.
+shell for working on Helix with `nix develop`.
+
+Flake outputs are cached for each push to master using
+[Cachix](https://www.cachix.org/). With Cachix
+[installed](https://docs.cachix.org/installation), `cachix use helix` will
+configure Nix to use cached outputs when possible.
 
 ### Arch Linux
 
 Releases are available in the `community` repository.
 
-Packages are also available on AUR:
-- [helix-bin](https://aur.archlinux.org/packages/helix-bin/) contains the pre-built release
-- [helix-git](https://aur.archlinux.org/packages/helix-git/) builds the master branch
+A [helix-git](https://aur.archlinux.org/packages/helix-git/) package is also available on the AUR, which builds the master branch.
+
+### Fedora Linux
+
+You can install the COPR package for Helix via
+
+```
+sudo dnf copr enable varlad/helix
+sudo dnf install helix
+```
+
+### Void Linux
+
+```
+sudo xbps-install helix
+```
 
 ## Build from source
 
 ```
-git clone --recurse-submodules --shallow-submodules -j8 https://github.com/helix-editor/helix
+git clone https://github.com/helix-editor/helix
 cd helix
 cargo install --path helix-term
 ```
@@ -40,5 +58,16 @@ cargo install --path helix-term
 This will install the `hx` binary to `$HOME/.cargo/bin`.
 
 Helix also needs it's runtime files so make sure to copy/symlink the `runtime/` directory into the
-config directory (for example `~/.config/helix/runtime` on Linux/macOS). This location can be overriden
+config directory (for example `~/.config/helix/runtime` on Linux/macOS). This location can be overridden
 via the `HELIX_RUNTIME` environment variable.
+
+| OS        | command   |
+|-----------|-----------|
+|windows    |`xcopy runtime %AppData%/helix/runtime`|
+|linux/macos|`ln -s $PWD/runtime ~/.config/helix/runtime`
+
+## Building tree-sitter grammars
+
+Tree-sitter grammars must be fetched and compiled if not pre-packaged.
+Fetch grammars with `hx --grammar fetch` (requires `git`) and compile them
+with `hx --grammar build` (requires a C compiler).

book/src/keymap.md

@@ -1,124 +1,148 @@
 # Keymap
 
+- Mappings marked (**LSP**) require an active language server for the file.
+- Mappings marked (**TS**) require a tree-sitter grammar for the filetype.
+
 ## Normal mode
 
 ### Movement
 
 > NOTE: Unlike vim, `f`, `F`, `t` and `T` are not confined to the current line.
 
-| Key         | Description                                        | Command                     |
-| -----       | -----------                                        | -------                     |
-| `h`/`Left`  | Move left                                          | `move_char_left`            |
-| `j`/`Down`  | Move down                                          | `move_line_down`            |
-| `k`/`Up`    | Move up                                            | `move_line_up`              |
-| `l`/`Right` | Move right                                         | `move_char_right`           |
-| `w`         | Move next word start                               | `move_next_word_start`      |
-| `b`         | Move previous word start                           | `move_prev_word_start`      |
-| `e`         | Move next word end                                 | `move_next_word_end`        |
-| `W`         | Move next WORD start                               | `move_next_long_word_start` |
-| `B`         | Move previous WORD start                           | `move_prev_long_word_start` |
-| `E`         | Move next WORD end                                 | `move_next_long_word_end`   |
-| `t`         | Find 'till next char                               | `find_till_char`            |
-| `f`         | Find next char                                     | `find_next_char`            |
-| `T`         | Find 'till previous char                           | `till_prev_char`            |
-| `F`         | Find previous char                                 | `find_prev_char`            |
-| `Alt-.`     | Repeat last motion (`f`, `t` or `m`)               | `repeat_last_motion`        |
-| `Home`      | Move to the start of the line                      | `goto_line_start`           |
-| `End`       | Move to the end of the line                        | `goto_line_end`             |
-| `PageUp`    | Move page up                                       | `page_up`                   |
-| `PageDown`  | Move page down                                     | `page_down`                 |
-| `Ctrl-u`    | Move half page up                                  | `half_page_up`              |
-| `Ctrl-d`    | Move half page down                                | `half_page_down`            |
-| `Ctrl-i`    | Jump forward on the jumplist                       | `jump_forward`              |
-| `Ctrl-o`    | Jump backward on the jumplist                      | `jump_backward`             |
-| `v`         | Enter [select (extend) mode](#select--extend-mode) | `select_mode`               |
-| `g`         | Enter [goto mode](#goto-mode)                      | N/A                         |
-| `m`         | Enter [match mode](#match-mode)                    | N/A                         |
-| `:`         | Enter command mode                                 | `command_mode`              |
-| `z`         | Enter [view mode](#view-mode)                      | N/A                         |
-| `Z`         | Enter sticky [view mode](#view-mode)               | N/A                         |
-| `Ctrl-w`    | Enter [window mode](#window-mode)                  | N/A                         |
-| `Space`     | Enter [space mode](#space-mode)                    | N/A                         |
+| Key                   | Description                                        | Command                     |
+| -----                 | -----------                                        | -------                     |
+| `h`, `Left`           | Move left                                          | `move_char_left`            |
+| `j`, `Down`           | Move down                                          | `move_line_down`            |
+| `k`, `Up`             | Move up                                            | `move_line_up`              |
+| `l`, `Right`          | Move right                                         | `move_char_right`           |
+| `w`                   | Move next word start                               | `move_next_word_start`      |
+| `b`                   | Move previous word start                           | `move_prev_word_start`      |
+| `e`                   | Move next word end                                 | `move_next_word_end`        |
+| `W`                   | Move next WORD start                               | `move_next_long_word_start` |
+| `B`                   | Move previous WORD start                           | `move_prev_long_word_start` |
+| `E`                   | Move next WORD end                                 | `move_next_long_word_end`   |
+| `t`                   | Find 'till next char                               | `find_till_char`            |
+| `f`                   | Find next char                                     | `find_next_char`            |
+| `T`                   | Find 'till previous char                           | `till_prev_char`            |
+| `F`                   | Find previous char                                 | `find_prev_char`            |
+| `G`                   | Go to line number `<n>`                            | `goto_line`                 |
+| `Alt-.`               | Repeat last motion (`f`, `t` or `m`)               | `repeat_last_motion`        |
+| `Home`                | Move to the start of the line                      | `goto_line_start`           |
+| `End`                 | Move to the end of the line                        | `goto_line_end`             |
+| `Ctrl-b`, `PageUp`    | Move page up                                       | `page_up`                   |
+| `Ctrl-f`, `PageDown`  | Move page down                                     | `page_down`                 |
+| `Ctrl-u`              | Move half page up                                  | `half_page_up`              |
+| `Ctrl-d`              | Move half page down                                | `half_page_down`            |
+| `Ctrl-i`              | Jump forward on the jumplist                       | `jump_forward`              |
+| `Ctrl-o`              | Jump backward on the jumplist                      | `jump_backward`             |
+| `Ctrl-s`              | Save the current selection to the jumplist         | `save_selection`            |
 
 ### Changes
 
-| Key         | Description                                     | Command               |
-| -----       | -----------                                     | -------               |
-| `r`         | Replace with a character                        | `replace`             |
-| `R`         | Replace with yanked text                        | `replace_with_yanked` |
-| `~`         | Switch case of the selected text                | `switch_case`         |
-| `` ` ``     | Set the selected text to lower case             | `switch_to_lowercase` |
-| `` Alt-` `` | Set the selected text to upper case             | `switch_to_uppercase` |
-| `i`         | Insert before selection                         | `insert_mode`         |
-| `a`         | Insert after selection (append)                 | `append_mode`         |
-| `I`         | Insert at the start of the line                 | `prepend_to_line`     |
-| `A`         | Insert at the end of the line                   | `append_to_line`      |
-| `o`         | Open new line below selection                   | `open_below`          |
-| `O`         | Open new line above selection                   | `open_above`          |
-| `.`         | Repeat last change                              | N/A                   |
-| `u`         | Undo change                                     | `undo`                |
-| `U`         | Redo change                                     | `redo`                |
-| `y`         | Yank selection                                  | `yank`                |
-| `p`         | Paste after selection                           | `paste_after`         |
-| `P`         | Paste before selection                          | `paste_before`        |
-| `"` `<reg>` | Select a register to yank to or paste from      | `select_register`     |
-| `>`         | Indent selection                                | `indent`              |
-| `<`         | Unindent selection                              | `unindent`            |
-| `=`         | Format selection                                | `format_selections`   |
-| `d`         | Delete selection                                | `delete_selection`    |
-| `c`         | Change selection (delete and enter insert mode) | `change_selection`    |
+| Key         | Description                                                          | Command                   |
+| -----       | -----------                                                          | -------                   |
+| `r`         | Replace with a character                                             | `replace`                 |
+| `R`         | Replace with yanked text                                             | `replace_with_yanked`     |
+| `~`         | Switch case of the selected text                                     | `switch_case`             |
+| `` ` ``     | Set the selected text to lower case                                  | `switch_to_lowercase`     |
+| `` Alt-` `` | Set the selected text to upper case                                  | `switch_to_uppercase`     |
+| `i`         | Insert before selection                                              | `insert_mode`             |
+| `a`         | Insert after selection (append)                                      | `append_mode`             |
+| `I`         | Insert at the start of the line                                      | `prepend_to_line`         |
+| `A`         | Insert at the end of the line                                        | `append_to_line`          |
+| `o`         | Open new line below selection                                        | `open_below`              |
+| `O`         | Open new line above selection                                        | `open_above`              |
+| `.`         | Repeat last change                                                   | N/A                       |
+| `u`         | Undo change                                                          | `undo`                    |
+| `U`         | Redo change                                                          | `redo`                    |
+| `Alt-u`     | Move backward in history                                             | `earlier`                 |
+| `Alt-U`     | Move forward in history                                              | `later`                   |
+| `y`         | Yank selection                                                       | `yank`                    |
+| `p`         | Paste after selection                                                | `paste_after`             |
+| `P`         | Paste before selection                                               | `paste_before`            |
+| `"` `<reg>` | Select a register to yank to or paste from                           | `select_register`         |
+| `>`         | Indent selection                                                     | `indent`                  |
+| `<`         | Unindent selection                                                   | `unindent`                |
+| `=`         | Format selection (currently nonfunctional/disabled) (**LSP**)        | `format_selections`       |
+| `d`         | Delete selection                                                     | `delete_selection`        |
+| `Alt-d`     | Delete selection, without yanking                                    | `delete_selection_noyank` |
+| `c`         | Change selection (delete and enter insert mode)                      | `change_selection`        |
+| `Alt-c`     | Change selection (delete and enter insert mode, without yanking)     | `change_selection_noyank` |
+| `Ctrl-a`    | Increment object (number) under cursor                               | `increment`               |
+| `Ctrl-x`    | Decrement object (number) under cursor                               | `decrement`               |
+| `Q`         | Start/stop macro recording to the selected register (experimental)   | `record_macro`            |
+| `q`         | Play back a recorded macro from the selected register (experimental) | `replay_macro`            |
 
 #### Shell
 
-| Key                   | Description                                                                      | Command               |
-| ------                | -----------                                                                      | -------               |
-| <code>&#124;</code>   | Pipe each selection through shell command, replacing with output                 | `shell_pipe`          |
-| <code>A-&#124;</code> | Pipe each selection into shell command, ignoring output                          | `shell_pipe_to`       |
-| `!`                   | Run shell command, inserting output before each selection                        | `shell_insert_output` |
-| `A-!`                 | Run shell command, appending output after each selection                         | `shell_append_output` |
+| Key     | Description                                                                      | Command               |
+| ------  | -----------                                                                      | -------               |
+| <code>&#124;</code>     | Pipe each selection through shell command, replacing with output                 | `shell_pipe`          |
+| <code>Alt-&#124;</code> | Pipe each selection into shell command, ignoring output                          | `shell_pipe_to`       |
+| `!`     | Run shell command, inserting output before each selection                        | `shell_insert_output` |
+| `Alt-!` | Run shell command, appending output after each selection                         | `shell_append_output` |
+| `$`     | Pipe each selection into shell command, keep selections where command returned 0 | `shell_keep_pipe`     |
 
 
 ### Selection manipulation
 
-| Key      | Description                                                       | Command                              |
-| -----    | -----------                                                       | -------                              |
-| `s`      | Select all regex matches inside selections                        | `select_regex`                       |
-| `S`      | Split selection into subselections on regex matches               | `split_selection`                    |
-| `Alt-s`  | Split selection on newlines                                       | `split_selection_on_newline`         |
-| `;`      | Collapse selection onto a single cursor                           | `collapse_selection`                 |
-| `Alt-;`  | Flip selection cursor and anchor                                  | `flip_selections`                    |
-| `,`      | Keep only the primary selection                                   | `keep_primary_selection`             |
-| `Alt-,`  | Remove the primary selection                                      | `remove_primary_selection`           |
-| `C`      | Copy selection onto the next line                                 | `copy_selection_on_next_line`        |
-| `Alt-C`  | Copy selection onto the previous line                             | `copy_selection_on_prev_line`        |
-| `(`      | Rotate main selection backward                                    | `rotate_selections_backward`         |
-| `)`      | Rotate main selection forward                                     | `rotate_selections_forward`          |
-| `Alt-(`  | Rotate selection contents backward                                | `rotate_selection_contents_backward` |
-| `Alt-)`  | Rotate selection contents forward                                 | `rotate_selection_contents_forward`  |
-| `%`      | Select entire file                                                | `select_all`                         |
-| `x`      | Select current line, if already selected, extend to next line     | `extend_line`                        |
-| `X`      | Extend selection to line bounds (line-wise selection)             | `extend_to_line_bounds`              |
-|          | Expand selection to parent syntax node TODO: pick a key           | `expand_selection`                   |
-| `J`      | Join lines inside selection                                       | `join_selections`                    |
-| `K`      | Keep selections matching the regex                                | `keep_selections`                    |
-| `$`      | Pipe each selection into shell command, keep selections where command returned 0 | `shell_keep_pipe`     |
-| `Ctrl-c` | Comment/uncomment the selections                                  | `toggle_comments`                    |
+| Key                   | Description                                                       | Command                              |
+| -----                 | -----------                                                       | -------                              |
+| `s`                   | Select all regex matches inside selections                        | `select_regex`                       |
+| `S`                   | Split selection into subselections on regex matches               | `split_selection`                    |
+| `Alt-s`               | Split selection on newlines                                       | `split_selection_on_newline`         |
+| `&`                   | Align selection in columns                                        | `align_selections`                   |
+| `_`                   | Trim whitespace from the selection                                | `trim_selections`                    |
+| `;`                   | Collapse selection onto a single cursor                           | `collapse_selection`                 |
+| `Alt-;`               | Flip selection cursor and anchor                                  | `flip_selections`                    |
+| `Alt-:`               | Ensures the selection is in forward direction                     | `ensure_selections_forward`          |
+| `,`                   | Keep only the primary selection                                   | `keep_primary_selection`             |
+| `Alt-,`               | Remove the primary selection                                      | `remove_primary_selection`           |
+| `C`                   | Copy selection onto the next line (Add cursor below)              | `copy_selection_on_next_line`        |
+| `Alt-C`               | Copy selection onto the previous line (Add cursor above)          | `copy_selection_on_prev_line`        |
+| `(`                   | Rotate main selection backward                                    | `rotate_selections_backward`         |
+| `)`                   | Rotate main selection forward                                     | `rotate_selections_forward`          |
+| `Alt-(`               | Rotate selection contents backward                                | `rotate_selection_contents_backward` |
+| `Alt-)`               | Rotate selection contents forward                                 | `rotate_selection_contents_forward`  |
+| `%`                   | Select entire file                                                | `select_all`                         |
+| `x`                   | Select current line, if already selected, extend to next line     | `extend_line`                        |
+| `X`                   | Extend selection to line bounds (line-wise selection)             | `extend_to_line_bounds`              |
+| `J`                   | Join lines inside selection                                       | `join_selections`                    |
+| `K`                   | Keep selections matching the regex                                | `keep_selections`                    |
+| `Alt-K`               | Remove selections matching the regex                              | `remove_selections`                  |
+| `Ctrl-c`              | Comment/uncomment the selections                                  | `toggle_comments`                    |
+| `Alt-o`, `Alt-up`     | Expand selection to parent syntax node (**TS**)                   | `expand_selection`                   |
+| `Alt-i`, `Alt-down`   | Shrink syntax tree object selection (**TS**)                      | `shrink_selection`                   |
+| `Alt-p`, `Alt-left`   | Select previous sibling node in syntax tree (**TS**)              | `select_prev_sibling`                |
+| `Alt-n`, `Alt-right`  | Select next sibling node in syntax tree (**TS**)                  | `select_next_sibling`                |
 
 ### Search
 
-> TODO: The search implementation isn't ideal yet -- we don't support searching in reverse.
+Search commands all operate on the `/` register by default. Use `"<char>` to operate on a different one.
 
 | Key   | Description                                 | Command              |
 | ----- | -----------                                 | -------              |
 | `/`   | Search for regex pattern                    | `search`             |
+| `?`   | Search for previous pattern                 | `rsearch`            |
 | `n`   | Select next search match                    | `search_next`        |
-| `N`   | Add next search match to selection          | `extend_search_next` |
+| `N`   | Select previous search match                | `search_prev`        |
 | `*`   | Use current selection as the search pattern | `search_selection`   |
 
 ### Minor modes
 
 These sub-modes are accessible from normal mode and typically switch back to normal mode after a command.
 
+| Key      | Description                                        | Command        |
+| -----    | -----------                                        | -------        |
+| `v`      | Enter [select (extend) mode](#select--extend-mode) | `select_mode`  |
+| `g`      | Enter [goto mode](#goto-mode)                      | N/A            |
+| `m`      | Enter [match mode](#match-mode)                    | N/A            |
+| `:`      | Enter command mode                                 | `command_mode` |
+| `z`      | Enter [view mode](#view-mode)                      | N/A            |
+| `Z`      | Enter sticky [view mode](#view-mode)               | N/A            |
+| `Ctrl-w` | Enter [window mode](#window-mode)                  | N/A            |
+| `Space`  | Enter [space mode](#space-mode)                    | N/A            |
+
 #### View mode
 
 View mode is intended for scrolling and manipulating the view without changing
@@ -127,50 +151,53 @@ key to return to normal mode after usage (useful when you're simply looking
 over text and not actively editing it).
 
 
-| Key       | Description                                               | Command             |
-| -----     | -----------                                               | -------             |
-| `z` , `c` | Vertically center the line                                | `align_view_center` |
-| `t`       | Align the line to the top of the screen                   | `align_view_top`    |
-| `b`       | Align the line to the bottom of the screen                | `align_view_bottom` |
-| `m`       | Align the line to the middle of the screen (horizontally) | `align_view_middle` |
-| `j`       | Scroll the view downwards                                 | `scroll_down`       |
-| `k`       | Scroll the view upwards                                   | `scroll_up`         |
-| `f`       | Move page down                                            | `page_down`         |
-| `b`       | Move page up                                              | `page_up`           |
-| `d`       | Move half page down                                       | `half_page_down`    |
-| `u`       | Move half page up                                         | `half_page_up`      |
+| Key                  | Description                                               | Command             |
+| -----                | -----------                                               | -------             |
+| `z`, `c`             | Vertically center the line                                | `align_view_center` |
+| `t`                  | Align the line to the top of the screen                   | `align_view_top`    |
+| `b`                  | Align the line to the bottom of the screen                | `align_view_bottom` |
+| `m`                  | Align the line to the middle of the screen (horizontally) | `align_view_middle` |
+| `j`, `down`          | Scroll the view downwards                                 | `scroll_down`       |
+| `k`, `up`            | Scroll the view upwards                                   | `scroll_up`         |
+| `Ctrl-f`, `PageDown` | Move page down                                            | `page_down`         |
+| `Ctrl-b`, `PageUp`   | Move page up                                              | `page_up`           |
+| `Ctrl-d`             | Move half page down                                       | `half_page_down`    |
+| `Ctrl-u`             | Move half page up                                         | `half_page_up`      |
 
 #### Goto mode
 
 Jumps to various locations.
 
-> NOTE: Some of these features are only available with the LSP present.
-
 | Key   | Description                                      | Command                    |
 | ----- | -----------                                      | -------                    |
-| `g`   | Go to the start of the file                      | `goto_file_start`          |
+| `g`   | Go to line number `<n>` else start of file       | `goto_file_start`          |
 | `e`   | Go to the end of the file                        | `goto_last_line`           |
+| `f`   | Go to files in the selection                     | `goto_file`                |
 | `h`   | Go to the start of the line                      | `goto_line_start`          |
 | `l`   | Go to the end of the line                        | `goto_line_end`            |
 | `s`   | Go to first non-whitespace character of the line | `goto_first_nonwhitespace` |
 | `t`   | Go to the top of the screen                      | `goto_window_top`          |
-| `m`   | Go to the middle of the screen                   | `goto_window_middle`       |
+| `c`   | Go to the middle of the screen                   | `goto_window_center`       |
 | `b`   | Go to the bottom of the screen                   | `goto_window_bottom`       |
-| `d`   | Go to definition                                 | `goto_definition`          |
-| `y`   | Go to type definition                            | `goto_type_definition`     |
-| `r`   | Go to references                                 | `goto_reference`           |
-| `i`   | Go to implementation                             | `goto_implementation`      |
+| `d`   | Go to definition (**LSP**)                       | `goto_definition`          |
+| `y`   | Go to type definition (**LSP**)                  | `goto_type_definition`     |
+| `r`   | Go to references (**LSP**)                       | `goto_reference`           |
+| `i`   | Go to implementation (**LSP**)                   | `goto_implementation`      |
 | `a`   | Go to the last accessed/alternate file           | `goto_last_accessed_file`  |
+| `m`   | Go to the last modified/alternate file           | `goto_last_modified_file`  |
+| `n`   | Go to next buffer                                | `goto_next_buffer`         |
+| `p`   | Go to previous buffer                            | `goto_previous_buffer`     |
+| `.`   | Go to last modification in current file          | `goto_last_modification`   |
 
 #### Match mode
 
-Enter this mode using `m` from normal mode. See the relavant section
+Enter this mode using `m` from normal mode. See the relevant section
 in [Usage](./usage.md) for an explanation about [surround](./usage.md#surround)
 and [textobject](./usage.md#textobject) usage.
 
 | Key              | Description                                     | Command                    |
 | -----            | -----------                                     | -------                    |
-| `m`              | Goto matching bracket                           | `match_brackets`           |
+| `m`              | Goto matching bracket (**TS**)                  | `match_brackets`           |
 | `s` `<char>`     | Surround current selection with `<char>`        | `surround_add`             |
 | `r` `<from><to>` | Replace surround character `<from>` with `<to>` | `surround_replace`         |
 | `d` `<char>`     | Delete surround character `<char>`              | `surround_delete`          |
@@ -183,67 +210,122 @@ TODO: Mappings for selecting syntax nodes (a superset of `[`).
 
 This layer is similar to vim keybindings as kakoune does not support window.
 
-| Key           | Description             | Command           |
-| -----         | -------------           | -------           |
-| `w`, `Ctrl-w` | Switch to next window   | `rotate_view`     |
-| `v`, `Ctrl-v` | Vertical right split    | `vsplit`          |
-| `s`, `Ctrl-s` | Horizontal bottom split | `hsplit`          |
-| `h`, `Ctrl-h` | Move to left split      | `jump_view_left`  |
-| `j`, `Ctrl-j` | Move to split below     | `jump_view_down`  |
-| `k`, `Ctrl-k` | Move to split above     | `jump_view_up`    |
-| `l`, `Ctrl-l` | Move to right split     | `jump_view_right` |
-| `q`, `Ctrl-q` | Close current window    | `wclose`          |
+| Key                    | Description                                          | Command           |
+| -----                  | -------------                                        | -------           |
+| `w`, `Ctrl-w`          | Switch to next window                                | `rotate_view`     |
+| `v`, `Ctrl-v`          | Vertical right split                                 | `vsplit`          |
+| `s`, `Ctrl-s`          | Horizontal bottom split                              | `hsplit`          |
+| `f`                    | Go to files in the selection in horizontal splits    | `goto_file`       |
+| `F`                    | Go to files in the selection in vertical splits      | `goto_file`       |
+| `h`, `Ctrl-h`, `Left`  | Move to left split                                   | `jump_view_left`  |
+| `j`, `Ctrl-j`, `Down`  | Move to split below                                  | `jump_view_down`  |
+| `k`, `Ctrl-k`, `Up`    | Move to split above                                  | `jump_view_up`    |
+| `l`, `Ctrl-l`, `Right` | Move to right split                                  | `jump_view_right` |
+| `q`, `Ctrl-q`          | Close current window                                 | `wclose`          |
+| `o`, `Ctrl-o`          | Only keep the current window, closing all the others | `wonly`           |
 
 #### Space mode
 
 This layer is a kludge of mappings, mostly pickers.
 
-| Key     | Description                                                           | Command                             |
-| -----   | -----------                                                           | -------                             |
-| `k`     | Show documentation for the item under the cursor                      | `hover`                             |
-| `f`     | Open file picker                                                      | `file_picker`                       |
-| `b`     | Open buffer picker                                                    | `buffer_picker`                     |
-| `s`     | Open symbol picker (current document)                                 | `symbol_picker`                     |
-| `a`     | Apply code action                                                     | `code_action`                       |
-| `'`     | Open last fuzzy picker                                                | `last_picker`                       |
-| `w`     | Enter [window mode](#window-mode)                                     | N/A                                 |
-| `p`     | Paste system clipboard after selections                               | `paste_clipboard_after`             |
-| `P`     | Paste system clipboard before selections                              | `paste_clipboard_before`            |
-| `y`     | Join and yank selections to clipboard                                 | `yank_joined_to_clipboard`          |
-| `Y`     | Yank main selection to clipboard                                      | `yank_main_selection_to_clipboard`  |
-| `R`     | Replace selections by clipboard contents                              | `replace_selections_with_clipboard` |
-| `/`     | Global search in workspace folder                                     | `global_search`                     |
 
-> NOTE: Global search display results in a fuzzy picker, use `space + '` to bring it back up after opening a file.
+| Key     | Description                                                             | Command                             |
+| -----   | -----------                                                             | -------                             |
+| `f`     | Open file picker                                                        | `file_picker`                       |
+| `b`     | Open buffer picker                                                      | `buffer_picker`                     |
+| `k`     | Show documentation for item under cursor in a [popup](#popup) (**LSP**) | `hover`                             |
+| `s`     | Open document symbol picker (**LSP**)                                   | `symbol_picker`                     |
+| `S`     | Open workspace symbol picker (**LSP**)                                  | `workspace_symbol_picker`           |
+| `r`     | Rename symbol (**LSP**)                                                 | `rename_symbol`                     |
+| `a`     | Apply code action  (**LSP**)                                            | `code_action`                       |
+| `'`     | Open last fuzzy picker                                                  | `last_picker`                       |
+| `w`     | Enter [window mode](#window-mode)                                       | N/A                                 |
+| `p`     | Paste system clipboard after selections                                 | `paste_clipboard_after`             |
+| `P`     | Paste system clipboard before selections                                | `paste_clipboard_before`            |
+| `y`     | Join and yank selections to clipboard                                   | `yank_joined_to_clipboard`          |
+| `Y`     | Yank main selection to clipboard                                        | `yank_main_selection_to_clipboard`  |
+| `R`     | Replace selections by clipboard contents                                | `replace_selections_with_clipboard` |
+| `/`     | Global search in workspace folder                                       | `global_search`                     |
+| `?`     | Open command palette                                                    | `command_palette`                   |
+
+> TIP: Global search displays results in a fuzzy picker, use `space + '` to bring it back up after opening a file.
+
+##### Popup
+
+Displays documentation for item under cursor.
+
+| Key      | Description |
+| ----     | ----------- |
+| `Ctrl-u` | Scroll up   |
+| `Ctrl-d` | Scroll down |
  
 #### Unimpaired
 
 Mappings in the style of [vim-unimpaired](https://github.com/tpope/vim-unimpaired).
 
-| Key       | Description                        | Command           |
-| -----     | -----------                        | -------           |
-| `[d`      | Go to previous diagnostic          | `goto_prev_diag`  |
-| `]d`      | Go to next diagnostic              | `goto_next_diag`  |
-| `[D`      | Go to first diagnostic in document | `goto_first_diag` |
-| `]D`      | Go to last diagnostic in document  | `goto_last_diag`  |
-| `[space`  | Add newline above                  | `add_newline_above` |
-| `]space`  | Add newline below                  | `add_newline_below` |
+| Key      | Description                                  | Command               |
+| -----    | -----------                                  | -------               |
+| `[d`     | Go to previous diagnostic (**LSP**)          | `goto_prev_diag`      |
+| `]d`     | Go to next diagnostic (**LSP**)              | `goto_next_diag`      |
+| `[D`     | Go to first diagnostic in document (**LSP**) | `goto_first_diag`     |
+| `]D`     | Go to last diagnostic in document (**LSP**)  | `goto_last_diag`      |
+| `]f`     | Go to next function (**TS**)                 | `goto_next_function`  |
+| `[f`     | Go to previous function (**TS**)             | `goto_prev_function`  |
+| `]c`     | Go to next class (**TS**)                    | `goto_next_class`     |
+| `[c`     | Go to previous class (**TS**)                | `goto_prev_class`     |
+| `]a`     | Go to next argument/parameter (**TS**)       | `goto_next_parameter` |
+| `[a`     | Go to previous argument/parameter (**TS**)   | `goto_prev_parameter` |
+| `]o`     | Go to next comment (**TS**)                  | `goto_next_comment`   |
+| `[o`     | Go to previous comment (**TS**)              | `goto_prev_comment`   |
+| `]p`     | Go to next paragraph                         | `goto_next_paragraph` |
+| `[p`     | Go to previous paragraph                     | `goto_prev_paragraph` |
+| `[space` | Add newline above                            | `add_newline_above`   |
+| `]space` | Add newline below                            | `add_newline_below`   |
 
 ## Insert Mode
 
-| Key      | Description           | Command                |
-| -----    | -----------           | -------                |
-| `Escape` | Switch to normal mode | `normal_mode`          |
-| `Ctrl-x` | Autocomplete          | `completion`           |
-| `Ctrl-w` | Delete previous word  | `delete_word_backward` |
+We support many readline/emacs style bindings in insert mode for
+convenience. These can be helpful for making simple modifications
+without escaping to normal mode, but beware that you will not have an
+undo-able "save point" until you return to normal mode.
+
+| Key                       | Description                 | Command                 |
+| -----                     | -----------                 | -------                 |
+| `Escape`                  | Switch to normal mode       | `normal_mode`           |
+| `Ctrl-x`                  | Autocomplete                | `completion`            |
+| `Ctrl-r`                  | Insert a register content   | `insert_register`       |
+| `Ctrl-w`, `Alt-Backspace` | Delete previous word        | `delete_word_backward`  |
+| `Alt-d`                   | Delete next word            | `delete_word_forward`   |
+| `Alt-b`, `Ctrl-Left`      | Backward a word             | `move_prev_word_end`    |
+| `Ctrl-b`, `Left`          | Backward a char             | `move_char_left`        |
+| `Alt-f`, `Ctrl-Right`     | Forward a word              | `move_next_word_start`  |
+| `Ctrl-f`, `Right`         | Forward a char              | `move_char_right`       |
+| `Ctrl-e`, `End`           | Move to line end            | `goto_line_end_newline` |
+| `Ctrl-a`, `Home`          | Move to line start          | `goto_line_start`       |
+| `Ctrl-u`                  | Delete to start of line     | `kill_to_line_start`    |
+| `Ctrl-k`                  | Delete to end of line       | `kill_to_line_end`      |
+| `Ctrl-j`, `Enter`         | Insert new line             | `insert_newline`        |
+| `Backspace`, `Ctrl-h`     | Delete previous char        | `delete_char_backward`  |
+| `Delete`, `Ctrl-d`        | Delete next char            | `delete_char_forward`   |
+| `Ctrl-p`, `Up`            | Move to previous line       | `move_line_up`          |
+| `Ctrl-n`, `Down`          | Move to next line           | `move_line_down`        |
+| `PageUp`                  | Move one page up            | `page_up`               |
+| `PageDown`                | Move one page down          | `page_down`             |
+| `Alt->`                   | Go to end of buffer         | `goto_file_end`         |
+| `Alt-<`                   | Go to start of buffer       | `goto_file_start`       |
 
 ## Select / extend mode
 
-I'm still pondering whether to keep this mode or not. It changes movement
-commands (including goto) to extend the existing selection instead of replacing it.
+This mode echoes Normal mode, but changes any movements to extend
+selections rather than replace them. Goto motions are also changed to
+extend, so that `vgl` for example extends the selection to the end of
+the line.
 
-> NOTE: It's a bit confusing at the moment because extend hasn't been
-> implemented for all movement commands yet.
+Search is also affected. By default, `n` and `N` will remove the current
+selection and select the next instance of the search term. Toggling this
+mode before pressing `n` or `N` makes it possible to keep the current
+selection. Toggling it on and off during your iterative searching allows
+you to selectively add search terms to your selections.
 
 # Picker
 
@@ -251,10 +333,41 @@ Keys to use within picker. Remapping currently not supported.
 
 | Key                          | Description       |
 | -----                        | -------------     |
-| `Up`, `Ctrl-k`, `Ctrl-p`     | Previous entry    |
-| `Down`, `Ctrl-j`, `Ctrl-n`   | Next entry        |
+| `Up`, `Ctrl-p`               | Previous entry    |
+| `PageUp`, `Ctrl-u`           | Page up           |
+| `Down`, `Ctrl-n`             | Next entry        |
+| `PageDown`, `Ctrl-d`         | Page down         |
+| `Home`                       | Go to first entry |
+| `End`                        | Go to last entry  |
 | `Ctrl-space`                 | Filter options    |
 | `Enter`                      | Open selected     |
 | `Ctrl-s`                     | Open horizontally |
 | `Ctrl-v`                     | Open vertically   |
 | `Escape`, `Ctrl-c`           | Close picker      |
+
+# Prompt
+
+Keys to use within prompt, Remapping currently not supported.
+
+| Key                     | Description                                                             |
+| -----                   | -------------                                                           |
+| `Escape`, `Ctrl-c`      | Close prompt                                                            |
+| `Alt-b`, `Alt-Left`     | Backward a word                                                         |
+| `Ctrl-b`, `Left`        | Backward a char                                                         |
+| `Alt-f`, `Alt-Right`    | Forward a word                                                          |
+| `Ctrl-f`, `Right`       | Forward a char                                                          |
+| `Ctrl-e`, `End`         | Move prompt end                                                         |
+| `Ctrl-a`, `Home`        | Move prompt start                                                       |
+| `Ctrl-w`                | Delete previous word                                                    |
+| `Alt-d`                 | Delete next word                                                        |
+| `Ctrl-u`                | Delete to start of line                                                 |
+| `Ctrl-k`                | Delete to end of line                                                   |
+| `backspace`, `Ctrl-h`   | Delete previous char                                                    |
+| `delete`, `Ctrl-d`      | Delete next char                                                        |
+| `Ctrl-s`                | Insert a word under doc cursor, may be changed to Ctrl-r Ctrl-w later   |
+| `Ctrl-p`, `Up`          | Select previous history                                                 |
+| `Ctrl-n`, `Down`        | Select next history                                                     |
+| `Tab`                   | Select next completion item                                             |
+| `BackTab`               | Select previous completion item                                         |
+| `Enter`                 | Open selected                                                           |
+

book/src/lang-support.md

@@ -0,0 +1,16 @@
+# Language Support
+
+The following languages and Language Servers are supported. In order to use
+Language Server features, you must first [install][lsp-install-wiki] the
+appropriate Language Server.
+
+Check the language support in your installed helix version with `hx --health`.
+
+Also see the [Language Configuration][lang-config] docs and the [Adding
+Languages][adding-languages] guide for more language configuration information.
+
+{{#include ./generated/lang-support.md}}
+
+[lsp-install-wiki]: https://github.com/helix-editor/helix/wiki/How-to-install-the-default-language-servers
+[lang-config]: ./languages.md
+[adding-languages]: ./guides/adding_languages.md

book/src/languages.md

@@ -0,0 +1,42 @@
+# Languages
+
+Language-specific settings and settings for particular language servers can be configured in a `languages.toml` file placed in your [configuration directory](./configuration.md). Helix actually uses two `languages.toml` files, the [first one](https://github.com/helix-editor/helix/blob/master/languages.toml) is in the main helix repository; it contains the default settings for each language and is included in the helix binary at compile time. Users who want to see the available settings and options can either reference the helix repo's `languages.toml` file, or consult the table in the [adding languages](./guides/adding_languages.md) section.
+
+A local `languages.toml` can be created within a `.helix` directory. Its settings will be merged with both the global and default configs.
+
+Changes made to the `languages.toml` file in a user's [configuration directory](./configuration.md) are merged with helix's defaults on start-up, such that a user's settings will take precedence over defaults in the event of a collision. For example, the default `languages.toml` sets rust's `auto-format` to `true`. If a user wants to disable auto-format, they can change the `languages.toml` in their [configuration directory](./configuration.md) to make the rust entry read like the example below; the new key/value pair `auto-format = false` will override the default when the two sets of settings are merged on start-up:
+
+```toml
+# in <config_dir>/helix/languages.toml
+
+[[language]]
+name = "rust"
+auto-format = false
+```
+
+## Tree-sitter grammars
+
+Tree-sitter grammars can also be configured in `languages.toml`:
+
+```toml
+# in <config_dir>/helix/languages.toml
+
+[[grammar]]
+name = "rust"
+source = { git = "https://github.com/tree-sitter/tree-sitter-rust", rev = "a250c4582510ff34767ec3b7dcdd3c24e8c8aa68" }
+
+[[grammar]]
+name = "c"
+source = { path = "/path/to/tree-sitter-c" }
+```
+
+You may use a top-level `use-grammars` key to control which grammars are fetched and built.
+
+```toml
+# Note: this key must come **before** the [[language]] and [[grammar]] sections
+use-grammars = { only = [ "rust", "c", "cpp" ] }
+# or
+use-grammars = { except = [ "yaml", "json" ] }
+```
+
+When omitted, all grammars are fetched and built.

book/src/remapping.md

@@ -11,15 +11,19 @@ this:
 ```toml
 # At most one section each of 'keys.normal', 'keys.insert' and 'keys.select'
 [keys.normal]
+C-s = ":w" # Maps the Control-s to the typable command :w which is an alias for :write (save file)
+C-o = ":open ~/.config/helix/config.toml" # Maps the Control-o to opening of the helix config file
 a = "move_char_left" # Maps the 'a' key to the move_char_left command
 w = "move_line_up" # Maps the 'w' key move_line_up
 "C-S-esc" = "extend_line" # Maps Control-Shift-Escape to extend_line
 g = { a = "code_action" } # Maps `ga` to show possible code actions
+"ret" = ["open_below", "normal_mode"] # Maps the enter key to open_below then re-enter normal mode
 
 [keys.insert]
 "A-x" = "normal_mode" # Maps Alt-X to enter normal mode
 j = { k = "normal_mode" } # Maps `jk` to exit insert mode
 ```
+> NOTE: Typable commands can also be remapped, remember to keep the `:` prefix to indicate it's a typable command.
 
 Control, Shift and Alt modifiers are encoded respectively with the prefixes
 `C-`, `S-` and `A-`. Special keys are encoded as follows:
@@ -38,12 +42,12 @@ Control, Shift and Alt modifiers are encoded respectively with the prefixes
 | Left         | `"left"`       |
 | Right        | `"right"`      |
 | Up           | `"up"`         |
+| Down         | `"down"`       |
 | Home         | `"home"`       |
 | End          | `"end"`        |
-| Page         | `"pageup"`     |
-| Page         | `"pagedown"`   |
+| Page Up      | `"pageup"`     |
+| Page Down    | `"pagedown"`   |
 | Tab          | `"tab"`        |
-| Back         | `"backtab"`    |
 | Delete       | `"del"`        |
 | Insert       | `"ins"`        |
 | Null         | `"null"`       |
@@ -51,4 +55,5 @@ Control, Shift and Alt modifiers are encoded respectively with the prefixes
 
 Keys can be disabled by binding them to the `no_op` command.
 
-Commands can be found in the source code at [`helix-term/src/commands.rs`](https://github.com/helix-editor/helix/blob/master/helix-term/src/commands.rs)
+Commands can be found at [Keymap](https://docs.helix-editor.com/keymap.html) Commands.
+> Commands can also be found in the source code at [`helix-term/src/commands.rs`](https://github.com/helix-editor/helix/blob/master/helix-term/src/commands.rs) at the invocation of `static_commands!` macro and the `TypableCommandList`.

book/src/themes.md

@@ -1,14 +1,14 @@
 # Themes
 
-First you'll need to place selected themes in your `themes` directory (i.e `~/.config/helix/themes`), the directory might have to be created beforehand.
-
-To use a custom theme add `theme = <name>` to your [`config.toml`](./configuration.md) or override it during runtime using `:theme <name>`.
-
-The default theme.toml can be found [here](https://github.com/helix-editor/helix/blob/master/theme.toml), and user submitted themes [here](https://github.com/helix-editor/helix/blob/master/runtime/themes). 
+To use a theme add `theme = "<name>"` to your [`config.toml`](./configuration.md) at the very top of the file before the first section or select it during runtime using `:theme <name>`.
 
 ## Creating a theme
 
-First create a file with the name of your theme as file name (i.e `mytheme.toml`) and place it in your `themes` directory (i.e `~/.config/helix/themes`).
+Create a file with the name of your theme as file name (i.e `mytheme.toml`) and place it in your `themes` directory (i.e `~/.config/helix/themes`). The directory might have to be created beforehand.
+
+The names "default" and "base16_default" are reserved for the builtin themes and cannot be overridden by user defined themes.
+
+The default theme.toml can be found [here](https://github.com/helix-editor/helix/blob/master/theme.toml), and user submitted themes [here](https://github.com/helix-editor/helix/blob/master/runtime/themes). 
 
 Each line in the theme file is specified as below:
 
@@ -37,8 +37,8 @@ configuration values in your theme. To do this, add a table called
 `palette` to your theme file:
 
 ```toml
-ui.background = "white"
-ui.text = "black"
+"ui.background" = "white"
+"ui.text" = "black"
 
 [palette]
 white = "#ffffff"
@@ -103,17 +103,19 @@ We use a similar set of scopes as
 [SublimeText](https://www.sublimetext.com/docs/scope_naming.html). See also
 [TextMate](https://macromates.com/manual/en/language_grammars) scopes.
 
-- `escape` (TODO: rename to (constant).character.escape)
-
 - `type` - Types
   - `builtin` - Primitive types provided by the language (`int`, `usize`)
+- `constructor`
 
 - `constant` (TODO: constant.other.placeholder for %v)
   - `builtin` Special constants provided by the language (`true`, `false`, `nil` etc)
     - `boolean`
   - `character`
+    - `escape`
+  - `numeric` (numbers)
+    - `integer`
+    - `float`
 
-- `number` (TODO: rename to constant.number/.numeric.{integer, float, complex})
 - `string` (TODO: string.quoted.{single, double}, string.raw/.unquoted)?
   - `regexp` - Regular expressions
   - `special`
@@ -129,7 +131,8 @@ We use a similar set of scopes as
 - `variable` - Variables
   - `builtin` - Reserved language variables (`self`, `this`, `super`, etc)
   - `parameter` - Function parameters
-  - `property`
+  - `other`
+    - `member` - Fields of composite data types (e.g. structs, unions)
   - `function` (TODO: ?)
 
 - `label`
@@ -143,53 +146,94 @@ We use a similar set of scopes as
     - `conditional` - `if`, `else`
     - `repeat` - `for`, `while`, `loop`
     - `import` - `import`, `export`
-    - (TODO: return?)
+    - `return`
+    - `exception`
+  - `operator` - `or`, `in`
   - `directive` - Preprocessor directives (`#if` in C) 
   - `function` - `fn`, `func`
 
-- `operator` - `||`, `+=`, `>`, `or`
+- `operator` - `||`, `+=`, `>`
 
 - `function`
   - `builtin`
   - `method`
   - `macro`
-  - `special` (preprocesor in C)
+  - `special` (preprocessor in C)
 
 - `tag` - Tags (e.g. `<body>` in HTML)
 
 - `namespace`
 
+- `markup`
+  - `heading`
+    - `marker`
+    - `1`, `2`, `3`, `4`, `5`, `6` - heading text for h1 through h6
+  - `list`
+    - `unnumbered`
+    - `numbered`
+  - `bold`
+  - `italic`
+  - `link`
+    - `url` - urls pointed to by links
+    - `label` - non-url link references
+    - `text` - url and image descriptions in links
+  - `quote`
+  - `raw`
+    - `inline`
+    - `block`
+
+- `diff` - version control changes
+  - `plus` - additions
+  - `minus` - deletions
+  - `delta` - modifications
+    - `moved` - renamed or moved files/changes
+
 #### Interface
 
 These scopes are used for theming the editor interface.
 
+- `markup`
+  - `normal`
+    - `completion` - for completion doc popup ui
+    - `hover` - for hover popup ui
+  - `heading`
+    - `completion` - for completion doc popup ui
+    - `hover` - for hover popup ui
+  - `raw`
+    - `inline`
+      - `completion` - for completion doc popup ui
+      - `hover` - for hover popup ui
 
-| Key                      | Notes                               |
-| ---                      | ---                                 |
-| `ui.background`          |                                     |
-| `ui.cursor`              |                                     |
-| `ui.cursor.insert`       |                                     |
-| `ui.cursor.select`       |                                     |
-| `ui.cursor.match`        | Matching bracket etc.               |
-| `ui.cursor.primary`      | Cursor with primary selection       |
-| `ui.linenr`              |                                     |
-| `ui.linenr.selected`     |                                     |
-| `ui.statusline`          | Statusline                          |
-| `ui.statusline.inactive` | Statusline (unfocused document)     |
-| `ui.popup`               |                                     |
-| `ui.window`              |                                     |
-| `ui.help`                |                                     |
-| `ui.text`                |                                     |
-| `ui.text.focus`          |                                     |
-| `ui.info`                |                                     |
-| `ui.info.text`           |                                     |
-| `ui.menu`                |                                     |
-| `ui.menu.selected`       |                                     |
-| `ui.selection`           | For selections in the editing area  |
-| `ui.selection.primary`   |                                     |
-| `warning`                | Diagnostics warning (gutter)        |
-| `error`                  | Diagnostics error (gutter)          |
-| `info`                   | Diagnostics info (gutter)           |
-| `hint`                   | Diagnostics hint (gutter)           |
-| `diagnostic`             | For text in editing area            |
 
+| Key                      | Notes                                          |
+| ---                      | ---                                            |
+| `ui.background`          |                                                |
+| `ui.cursor`              |                                                |
+| `ui.cursor.insert`       |                                                |
+| `ui.cursor.select`       |                                                |
+| `ui.cursor.match`        | Matching bracket etc.                          |
+| `ui.cursor.primary`      | Cursor with primary selection                  |
+| `ui.linenr`              | Line numbers                                   |
+| `ui.linenr.selected`     | Line number for the line the cursor is on      |
+| `ui.statusline`          | Statusline                                     |
+| `ui.statusline.inactive` | Statusline (unfocused document)                |
+| `ui.popup`               | Documentation popups (e.g space-k)             |
+| `ui.popup.info`          | Prompt for multiple key options                |
+| `ui.window`              | Border lines separating splits                 |
+| `ui.help`                | Description box for commands                   |
+| `ui.text`                | Command prompts, popup text, etc.              |
+| `ui.text.focus`          |                                                |
+| `ui.text.info`           | The key: command text in `ui.popup.info` boxes |
+| `ui.virtual.ruler`       | Ruler columns (see the [`editor.rulers` config][rulers-config])|
+| `ui.virtual.whitespace`  | Visible white-space characters                 |
+| `ui.menu`                | Code and command completion menus              |
+| `ui.menu.selected`       | Selected autocomplete item                     |
+| `ui.selection`           | For selections in the editing area             |
+| `ui.selection.primary`   |                                                |
+| `warning`                | Diagnostics warning (gutter)                   |
+| `error`                  | Diagnostics error (gutter)                     |
+| `info`                   | Diagnostics info (gutter)                      |
+| `hint`                   | Diagnostics hint (gutter)                      |
+| `diagnostic`             | For text in editing area                       |
+
+[rulers-config]: ./configuration.md#editor-section

book/src/title-page.md

@@ -0,0 +1,15 @@
+# Helix
+
+Docs for bleeding edge master can be found at
+[https://docs.helix-editor.com/master](https://docs.helix-editor.com/master).
+
+See the [usage] section for a quick overview of the editor, [keymap]
+section for all available keybindings and the [configuration] section
+for defining custom keybindings, setting themes, etc.
+
+Refer the [FAQ] for common questions.
+
+[FAQ]: https://github.com/helix-editor/helix/wiki/FAQ
+[usage]: ./usage.md
+[keymap]: ./keymap.md
+[configuration]: ./configuration.md

book/src/usage.md

@@ -23,8 +23,10 @@ If there is a selected register before invoking a change or delete command, the
 | `/`                | Last search           |
 | `:`                | Last executed command |
 | `"`                | Last yanked text      |
+| `_`                | Black hole            |
 
 > There is no special register for copying to system clipboard, instead special commands and keybindings are provided. See the [keymap](keymap.md#space-mode) for the specifics.
+> The black hole register works as a no-op register, meaning no data will be written to / read from it.
 
 ## Surround
 
@@ -40,7 +42,7 @@ helix. The keymappings have been inspired from [vim-sandwich](https://github.com
 `ms` acts on a selection, so select the text first and use `ms<char>`. `mr` and `md` work
 on the closest pairs found and selections are not required; use counts to act in outer pairs.
 
-It can also act on multiple seletions (yay!). For example, to change every occurance of `(use)` to `[use]`:
+It can also act on multiple selections (yay!). For example, to change every occurrence of `(use)` to `[use]`:
 
 - `%` to select the whole file
 - `s` to split the selections on a search term
@@ -62,12 +64,34 @@ Currently supported: `word`, `surround`, `function`, `class`, `parameter`.
 | Key after `mi` or `ma` | Textobject selected      |
 | ---                    | ---                      |
 | `w`                    | Word                     |
+| `W`                    | WORD                     |
 | `(`, `[`, `'`, etc     | Specified surround pairs |
+| `m`                    | Closest surround pair    |
 | `f`                    | Function                 |
 | `c`                    | Class                    |
-| `p`                    | Parameter                |
+| `a`                    | Argument/parameter       |
+| `o`                    | Comment                  |
 
-Note: `f`, `c`, etc need a tree-sitter grammar active for the current
+> NOTE: `f`, `c`, etc need a tree-sitter grammar active for the current
 document and a special tree-sitter query file to work properly. [Only
-some grammars](https://github.com/search?q=repo%3Ahelix-editor%2Fhelix+filename%3Atextobjects.scm&type=Code&ref=advsearch&l=&l=)
-currently have the query file implemented. Contributions are welcome !
+some grammars][lang-support] currently have the query file implemented.
+Contributions are welcome!
+
+## Tree-sitter Textobject Based Navigation
+
+Navigating between functions, classes, parameters, etc is made
+possible by leveraging tree-sitter and textobjects queries. For
+example to move to the next function use `]f`, to move to previous
+class use `[c`, and so on.
+
+![tree-sitter-nav-demo][tree-sitter-nav-demo]
+
+See the [unimpaired][unimpaired-keybinds] section of the keybind
+documentation for the full reference.
+
+> NOTE: This feature is dependent on tree-sitter based textobjects
+and therefore requires the corresponding query file to work properly.
+
+[lang-support]: ./lang-support.md
+[unimpaired-keybinds]: ./keymap.md#unimpaired
+[tree-sitter-nav-demo]: https://user-images.githubusercontent.com/23398472/152332550-7dfff043-36a2-4aec-b8f2-77c13eb56d6f.gif

book/theme/book.js

@@ -601,10 +601,10 @@ function playground_text(playground) {
     });
 })();
 
-(function controllMenu() {
+(function controlMenu() {
     var menu = document.getElementById('menu-bar');
 
-    (function controllPosition() {
+    (function controlPosition() {
         var scrollTop = document.scrollingElement.scrollTop;
         var prevScrollTop = scrollTop;
         var minMenuY = -menu.clientHeight - 50;
@@ -647,7 +647,7 @@ function playground_text(playground) {
             prevScrollTop = scrollTop;
         }, { passive: true });
     })();
-    (function controllBorder() {
+    (function controlBorder() {
         menu.classList.remove('bordered');
         document.addEventListener('scroll', function () {
             if (menu.offsetTop === 0) {

book/theme/css/chrome.css

@@ -390,7 +390,7 @@ ul#searchresults span.teaser em {
 
 .chapter li {
     display: flex;
-    color: var(--sidebar-non-existant);
+    color: var(--sidebar-non-existent);
 }
 .chapter li a {
     display: block;

book/theme/css/variables.css

@@ -16,7 +16,7 @@
 
     --sidebar-bg: #14191f;
     --sidebar-fg: #c8c9db;
-    --sidebar-non-existant: #5c6773;
+    --sidebar-non-existent: #5c6773;
     --sidebar-active: #ffb454;
     --sidebar-spacer: #2d334f;
 
@@ -56,7 +56,7 @@
 
     --sidebar-bg: #292c2f;
     --sidebar-fg: #a1adb8;
-    --sidebar-non-existant: #505254;
+    --sidebar-non-existent: #505254;
     --sidebar-active: #3473ad;
     --sidebar-spacer: #393939;
 
@@ -96,7 +96,7 @@
 
     --sidebar-bg: #fafafa;
     --sidebar-fg: hsl(0, 0%, 0%);
-    --sidebar-non-existant: #aaaaaa;
+    --sidebar-non-existent: #aaaaaa;
     --sidebar-active: #1f1fff;
     --sidebar-spacer: #f4f4f4;
 
@@ -136,7 +136,7 @@
 
     --sidebar-bg: #282d3f;
     --sidebar-fg: #c8c9db;
-    --sidebar-non-existant: #505274;
+    --sidebar-non-existent: #505274;
     --sidebar-active: #2b79a2;
     --sidebar-spacer: #2d334f;
 
@@ -176,7 +176,7 @@
 
     --sidebar-bg: #3b2e2a;
     --sidebar-fg: #c8c9db;
-    --sidebar-non-existant: #505254;
+    --sidebar-non-existent: #505254;
     --sidebar-active: #e69f67;
     --sidebar-spacer: #45373a;
 
@@ -217,7 +217,7 @@
 
         --sidebar-bg: #292c2f;
         --sidebar-fg: #a1adb8;
-        --sidebar-non-existant: #505254;
+        --sidebar-non-existent: #505254;
         --sidebar-active: #3473ad;
         --sidebar-spacer: #393939;
 
@@ -259,7 +259,7 @@
 
     --sidebar-bg: #281733;
     --sidebar-fg: #c8c9db;
-    --sidebar-non-existant: #505274;
+    --sidebar-non-existent: #505274;
     --sidebar-active: #a4a0e8;
     --sidebar-spacer: #2d334f;
 
@@ -304,7 +304,7 @@
 
     --sidebar-bg: #281733;
     --sidebar-fg: #c8c9db;
-    --sidebar-non-existant: #505274;
+    --sidebar-non-existent: #505274;
     --sidebar-active: #a4a0e8;
     --sidebar-spacer: #2d334f;
 

contrib/Helix.desktop

@@ -0,0 +1,89 @@
+[Desktop Entry]
+Name=Helix
+GenericName=Text Editor
+GenericName[de]=Texteditor
+GenericName[fr]=Éditeur de texte
+GenericName[ru]=Текстовый редактор
+GenericName[sr]=Едитор текст
+GenericName[tr]=Metin Düzenleyici
+Comment=Edit text files
+Comment[af]=Redigeer tekslêers
+Comment[am]=የጽሑፍ ፋይሎች ያስተካክሉ
+Comment[ar]=حرّر ملفات نصية
+Comment[az]=Mətn fayllarını redaktə edin
+Comment[be]=Рэдагаваньне тэкставых файлаў
+Comment[bg]=Редактиране на текстови файлове
+Comment[bn]=টেক্স্ট ফাইল এডিট করুন
+Comment[bs]=Izmijeni tekstualne datoteke
+Comment[ca]=Edita fitxers de text
+Comment[cs]=Úprava textových souborů
+Comment[cy]=Golygu ffeiliau testun
+Comment[da]=Redigér tekstfiler
+Comment[de]=Textdateien bearbeiten
+Comment[el]=Επεξεργασία αρχείων κειμένου
+Comment[en_CA]=Edit text files
+Comment[en_GB]=Edit text files
+Comment[es]=Edita archivos de texto
+Comment[et]=Redigeeri tekstifaile
+Comment[eu]=Editatu testu-fitxategiak
+Comment[fa]=ویرایش پرونده‌های متنی
+Comment[fi]=Muokkaa tekstitiedostoja
+Comment[fr]=Éditer des fichiers texte
+Comment[ga]=Eagar comhad Téacs
+Comment[gu]=લખાણ ફાઇલોમાં ફેરફાર કરો
+Comment[he]=ערוך קבצי טקסט
+Comment[hi]=पाठ फ़ाइलें संपादित करें
+Comment[hr]=Uređivanje tekstualne datoteke
+Comment[hu]=Szövegfájlok szerkesztése
+Comment[id]=Edit file teks
+Comment[it]=Modifica file di testo
+Comment[ja]=テキストファイルを編集します
+Comment[kn]=ಪಠ್ಯ ಕಡತಗಳನ್ನು ಸಂಪಾದಿಸು
+Comment[ko]=텍스트 파일을 편집합니다
+Comment[lt]=Redaguoti tekstines bylas
+Comment[lv]=Rediģēt teksta failus
+Comment[mk]=Уреди текстуални фајлови
+Comment[ml]=വാചക രചനകള് തിരുത്തുക
+Comment[mn]=Текст файл боловсруулах
+Comment[mr]=गद्य फाइल संपादित करा
+Comment[ms]=Edit fail teks
+Comment[nb]=Rediger tekstfiler
+Comment[ne]=पाठ फाइललाई संशोधन गर्नुहोस्
+Comment[nl]=Tekstbestanden bewerken
+Comment[nn]=Rediger tekstfiler
+Comment[no]=Rediger tekstfiler
+Comment[or]=ପାଠ୍ଯ ଫାଇଲଗୁଡ଼ିକୁ ସମ୍ପାଦନ କରନ୍ତୁ
+Comment[pa]=ਪਾਠ ਫਾਇਲਾਂ ਸੰਪਾਦਨ
+Comment[pl]=Edytor plików tekstowych
+Comment[pt]=Editar ficheiros de texto
+Comment[pt_BR]=Edite arquivos de texto
+Comment[ro]=Editare fişiere text
+Comment[ru]=Редактирование текстовых файлов
+Comment[sk]=Úprava textových súborov
+Comment[sl]=Urejanje datotek z besedili
+Comment[sq]=Përpuno files teksti
+Comment[sr]=Уређујте текст фајлове
+Comment[sr@Latn]=Izmeni tekstualne datoteke
+Comment[sv]=Redigera textfiler
+Comment[ta]=உரை கோப்புகளை தொகுக்கவும்
+Comment[th]=แก้ไขแฟ้มข้อความ
+Comment[tk]=Metin faýllary editle
+Comment[tr]=Metin dosyaları düzenleyin
+Comment[uk]=Редактор текстових файлів
+Comment[vi]=Soạn thảo tập tin văn bản
+Comment[wa]=Asspougnî des fitchîs tecses
+Comment[zh_CN]=编辑文本文件
+Comment[zh_TW]=編輯文字檔
+TryExec=hx
+Exec=hx %F
+Terminal=true
+Type=Application
+Keywords=Text;editor;
+Keywords[fr]=Texte;éditeur;
+Keywords[ru]=текст;текстовый редактор;
+Keywords[sr]=Текст;едитор;
+Keywords[tr]=Metin;düzenleyici;
+Icon=helix
+Categories=Utility;TextEditor;
+StartupNotify=false
+MimeType=text/english;text/plain;text/x-makefile;text/x-c++hdr;text/x-c++src;text/x-chdr;text/x-csrc;text/x-java;text/x-moc;text/x-pascal;text/x-tcl;text/x-tex;application/x-shellscript;text/x-c;text/x-c++;

contrib/completion/hx.bash

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Bash completion script for Helix editor
+
+_hx() {
+	# $1 command name
+	# $2 word being completed
+	# $3 word preceding
+	COMPREPLY=()
+
+	case "$3" in
+	-g | --grammar)
+		COMPREPLY=($(compgen -W "fetch build" -- $2))
+		;;
+	--health)
+		local languages=$(hx --health |tail -n '+7' |awk '{print $1}' |sed 's/\x1b\[[0-9;]*m//g')
+		COMPREPLY=($(compgen -W "$languages" -- $2))
+		;;
+	*)
+		COMPREPLY=($(compgen -fd -W "-h --help --tutor -V --version -v -vv -vvv --health -g --grammar" -- $2))
+		;;
+	esac
+} && complete -F _hx hx
+

contrib/completion/hx.fish

@@ -0,0 +1,12 @@
+#!/usr/bin/env fish
+# Fish completion script for Helix editor
+
+set -l langs (hx --health |tail -n '+7' |awk '{print $1}' |sed 's/\x1b\[[0-9;]*m//g')
+
+complete -c hx -s h -l help -d "Prints help information"
+complete -c hx -l tutor -d "Loads the tutorial"
+complete -c hx -l health -x -a "$langs" -d "Checks for errors in editor setup"
+complete -c hx -s g -l grammar -x -a "fetch build" -d "Fetches or builds tree-sitter grammars"
+complete -c hx -s v -o vv -o vvv -d "Increases logging verbosity"
+complete -c hx -s V -l version -d "Prints version information"
+

contrib/completion/hx.zsh

@@ -0,0 +1,29 @@
+#compdef _hx hx
+# Zsh completion script for Helix editor
+
+_hx() {
+	_arguments -C \
+		"-h[Prints help information]" \
+		"--help[Prints help information]" \
+		"-v[Increase logging verbosity]" \
+		"-vv[Increase logging verbosity]" \
+		"-vvv[Increase logging verbosity]" \
+		"-V[Prints version information]" \
+		"--version[Prints version information]" \
+		"--tutor[Loads the tutorial]" \
+		"--health[Checks for errors in editor setup]:language:->health" \
+		"-g[Fetches or builds tree-sitter grammars]:action:->grammar" \
+		"--grammar[Fetches or builds tree-sitter grammars]:action:->grammar" \
+		"*:file:_files"
+
+	case "$state" in
+	health)
+		local languages=($(hx --health |tail -n '+7' |awk '{print $1}' |sed 's/\x1b\[[0-9;]*m//g'))
+		_values 'language' $languages
+		;;
+	grammar)
+		_values 'action' fetch build
+		;;
+	esac
+}
+

docs/CONTRIBUTING.md

@@ -0,0 +1,37 @@
+# Contributing
+
+Contributors are very welcome! **No contribution is too small and all contributions are valued.**
+
+Some suggestions to get started:
+
+- You can look at the [good first issue][good-first-issue] label on the issue tracker.
+- Help with packaging on various distributions needed!
+- To use print debugging to the [Helix log file][log-file], you must:
+  * Print using `log::info!`, `warn!`, or `error!`. (`log::info!("helix!")`)
+  * Pass the appropriate verbosity level option for the desired log level. (`hx -v <file>` for info, more `v`s for higher severity inclusive)
+- If your preferred language is missing, integrating a tree-sitter grammar for
+    it and defining syntax highlight queries for it is straight forward and
+    doesn't require much knowledge of the internals.
+
+We provide an [architecture.md][architecture.md] that should give you
+a good overview of the internals.
+
+# Auto generated documentation
+
+Some parts of [the book][docs] are autogenerated from the code itself,
+like the list of `:commands` and supported languages. To generate these
+files, run
+
+```shell
+cargo xtask docgen
+```
+
+inside the project. We use [xtask][xtask] as an ad-hoc task runner and
+thus do not require any dependencies other than `cargo` (You don't have
+to `cargo install` anything either).
+
+[good-first-issue]: https://github.com/helix-editor/helix/labels/E-easy
+[log-file]: https://github.com/helix-editor/helix/wiki/FAQ#access-the-log-file
+[architecture.md]: ./architecture.md
+[docs]: https://docs.helix-editor.com/
+[xtask]: https://github.com/matklad/cargo-xtask

docs/architecture.md

@@ -1,12 +1,13 @@
 
-| Crate        | Description                                            |
-| -----------  | -----------                                            |
-| helix-core   | Core editing primitives, functional.                   |
-| helix-syntax | Tree-sitter grammars                                   |
-| helix-lsp    | Language server client                                 |
-| helix-view   | UI abstractions for use in backends, imperative shell. |
-| helix-term   | Terminal UI                                            |
-| helix-tui    | TUI primitives, forked from tui-rs, inspired by Cursive |
+| Crate        | Description                                                      |
+| -----------  | -----------                                                      |
+| helix-core   | Core editing primitives, functional.                             |
+| helix-lsp    | Language server client                                           |
+| helix-dap    | Debug Adapter Protocol (DAP) client                              |
+| helix-loader | Functions for building, fetching, and loading external resources |
+| helix-view   | UI abstractions for use in backends, imperative shell.           |
+| helix-term   | Terminal UI                                                      |
+| helix-tui    | TUI primitives, forked from tui-rs, inspired by Cursive          |
 
 
 This document contains a high-level overview of Helix internals.
@@ -41,7 +42,7 @@ applying the transaction.
 > interface used to generate text edits.
 
 `Syntax` is the interface used to interact with tree-sitter ASTs for syntax
-highling and other features.
+highlighting and other features.
 
 ## View
 
@@ -54,15 +55,40 @@ A `Document` ties together the `Rope`, `Selection`(s), `Syntax`, document
 file.
 
 A `View` represents an open split in the UI. It holds the currently open
-document ID and other related state.
+document ID and other related state. Views encapsulate the gutter, status line,
+diagnostics, and the inner area where the code is displayed.
 
 > NOTE: Multiple views are able to display the same document, so the document
 > contains selections for each view. To retrieve, `document.selection()` takes
 > a `ViewId`.
 
+`Info` is the autoinfo box that shows hints when awaiting another key with bindings
+like `g` and `m`. It is attached to the viewport as a whole.
+
+`Surface` is like a buffer to which widgets draw themselves to, and the
+surface is then rendered on the screen on each cycle.
+
+`Rect`s are areas (simply an x and y coordinate with the origin at the
+screen top left and then a height and width) which are part of a
+`Surface`. They can be used to limit the area to which a `Component` can
+render. For example if we wrap a `Markdown` component in a `Popup`
+(think the documentation popup with space+k), Markdown's render method
+will get a Rect that is the exact size of the popup.
+
+Widgets are called `Component`s internally, and you can see most of them
+in `helix-term/src/ui`. Some components like `Popup` and `Overlay` can take
+other components as children.
+
+`Layer`s are how multiple components are displayed, and is simply a
+`Vec<Component>`. Layers are managed by the `Compositor`. On each top
+level render call, the compositor renders each component in the order
+they were pushed into the stack. This makes multiple components "layer"
+on top of one another. Hence we get a file picker displayed over the
+editor, etc.
+
 The `Editor` holds the global state: all the open documents, a tree
-representation of all the view splits, and a registry of language servers. To
-open or close files, interact with the editor.
+representation of all the view splits, the configuration, and a registry of 
+language servers. To open or close files, interact with the editor.
 
 ## LSP
 

flake.lock

@@ -1,12 +1,35 @@
 {
   "nodes": {
-    "devshell": {
+    "crane": {
+      "flake": false,
       "locked": {
-        "lastModified": 1632436039,
-        "narHash": "sha256-OtITeVWcKXn1SpVEnImpTGH91FycCskGBPqmlxiykv4=",
+        "lastModified": 1644785799,
+        "narHash": "sha256-VpAJO1L0XeBvtCuNGK4IDKp6ENHIpTrlaZT7yfBCvwo=",
+        "owner": "ipetkov",
+        "repo": "crane",
+        "rev": "fc7a94f841347c88f2cb44217b2a3faa93e2a0b2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ipetkov",
+        "repo": "crane",
+        "type": "github"
+      }
+    },
+    "devshell": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1650201426,
+        "narHash": "sha256-u43Xf03ImFJWKLHddtjOpCJCHbuM0SQbb6FKR5NuFhk=",
         "owner": "numtide",
         "repo": "devshell",
-        "rev": "7a7a7aa0adebe5488e5abaec688fd9ae0f8ea9c6",
+        "rev": "cb76bc75a0ee81f2d8676fe681f268c48dbd380e",
         "type": "github"
       },
       "original": {
@@ -15,13 +38,63 @@
         "type": "github"
       }
     },
+    "dream2nix": {
+      "inputs": {
+        "alejandra": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ],
+        "crane": "crane",
+        "flake-utils-pre-commit": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ],
+        "gomod2nix": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ],
+        "mach-nix": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ],
+        "nixpkgs": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ],
+        "node2nix": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ],
+        "poetry2nix": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ],
+        "pre-commit-hooks": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1650199368,
+        "narHash": "sha256-35wZQDyohH6NI8ifT124p+TPDiwfRnQZkbUXc1yjGIE=",
+        "owner": "nix-community",
+        "repo": "dream2nix",
+        "rev": "feb7eabd0c1b24d47ea6ecd38b737319e75fd513",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "dream2nix",
+        "type": "github"
+      }
+    },
     "flake-utils": {
       "locked": {
-        "lastModified": 1623875721,
-        "narHash": "sha256-A8BU7bjS5GirpAUv4QA+QnJ4CceLHkcXdRp4xITDB0s=",
+        "lastModified": 1642700792,
+        "narHash": "sha256-XqHrk7hFb+zBvRg6Ghl+AZDq03ov6OshJLiSWOoX5es=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "f7e004a55b120c02ecb6219596820fcd32ca8772",
+        "rev": "846b2ae0fc4cc943637d3d1def4454213e203cba",
         "type": "github"
       },
       "original": {
@@ -30,25 +103,25 @@
         "type": "github"
       }
     },
-    "flakeCompat": {
-      "flake": false,
+    "flake-utils_2": {
       "locked": {
-        "lastModified": 1627913399,
-        "narHash": "sha256-hY8g6H2KFL8ownSiFeMOjwPC8P0ueXpCVEbxgda3pko=",
-        "owner": "edolstra",
-        "repo": "flake-compat",
-        "rev": "12c64ca55c1014cdc1b16ed5a804aa8576601ff2",
+        "lastModified": 1637014545,
+        "narHash": "sha256-26IZAc5yzlD9FlDT54io1oqG/bBoyka+FJk5guaX4x4=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "bba5dcc8e0b20ab664967ad83d24d64cb64ec4f4",
         "type": "github"
       },
       "original": {
-        "owner": "edolstra",
-        "repo": "flake-compat",
+        "owner": "numtide",
+        "repo": "flake-utils",
         "type": "github"
       }
     },
     "nixCargoIntegration": {
       "inputs": {
         "devshell": "devshell",
+        "dream2nix": "dream2nix",
         "nixpkgs": [
           "nixpkgs"
         ],
@@ -57,11 +130,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1634796585,
-        "narHash": "sha256-CW4yx6omk5qCXUIwXHp/sztA7u0SpyLq9NEACPnkiz8=",
+        "lastModified": 1650262356,
+        "narHash": "sha256-1ko5YhiLeFt9SjKvzq31IGrBJkHITNJc+08rod4P6lk=",
         "owner": "yusdacra",
         "repo": "nix-cargo-integration",
-        "rev": "a84a2137a396f303978f1d48341e0390b0e16a8b",
+        "rev": "924aa1e39bbf1182cdf0dd6dfdb1d7b4a047ef6f",
         "type": "github"
       },
       "original": {
@@ -72,11 +145,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1634782485,
-        "narHash": "sha256-psfh4OQSokGXG0lpq3zKFbhOo3QfoeudRcaUnwMRkQo=",
+        "lastModified": 1650161686,
+        "narHash": "sha256-70ZWAlOQ9nAZ08OU6WY7n4Ij2kOO199dLfNlvO/+pf8=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "34ad3ffe08adfca17fcb4e4a47bb5f3b113687be",
+        "rev": "1ffba9f2f683063c2b14c9f4d12c55ad5f4ed887",
         "type": "github"
       },
       "original": {
@@ -86,24 +159,8 @@
         "type": "github"
       }
     },
-    "nixpkgs_2": {
-      "locked": {
-        "lastModified": 1628186154,
-        "narHash": "sha256-r2d0wvywFnL9z4iptztdFMhaUIAaGzrSs7kSok0PgmE=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "06552b72346632b6943c8032e57e702ea12413bf",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
     "root": {
       "inputs": {
-        "flakeCompat": "flakeCompat",
         "nixCargoIntegration": "nixCargoIntegration",
         "nixpkgs": "nixpkgs",
         "rust-overlay": "rust-overlay"
@@ -111,15 +168,17 @@
     },
     "rust-overlay": {
       "inputs": {
-        "flake-utils": "flake-utils",
-        "nixpkgs": "nixpkgs_2"
+        "flake-utils": "flake-utils_2",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
       },
       "locked": {
-        "lastModified": 1634869268,
-        "narHash": "sha256-RVAcEFlFU3877Mm4q/nbXGEYTDg/wQNhzmXGMTV6wBs=",
+        "lastModified": 1650336226,
+        "narHash": "sha256-A68t/BM3JPXUDFx9JGBk24euXvsaIZuPL28+hX5TmwA=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "c02c2d86354327317546501af001886fbb53d374",
+        "rev": "8cd3024e5b011218308eeee35c4839601af458f8",
         "type": "github"
       },
       "original": {

flake.nix

@@ -3,71 +3,66 @@
 
   inputs = {
     nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
-    rust-overlay.url = "github:oxalica/rust-overlay";
+    rust-overlay = {
+      url = "github:oxalica/rust-overlay";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
     nixCargoIntegration = {
       url = "github:yusdacra/nix-cargo-integration";
       inputs.nixpkgs.follows = "nixpkgs";
       inputs.rustOverlay.follows = "rust-overlay";
     };
-    flakeCompat = {
-      url = "github:edolstra/flake-compat";
-      flake = false;
-    };
   };
 
-  outputs = inputs@{ self, nixCargoIntegration, ... }:
+  outputs = inputs@{ nixpkgs, nixCargoIntegration, ... }:
     nixCargoIntegration.lib.makeOutputs {
       root = ./.;
-      buildPlatform = "crate2nix";
       renameOutputs = { "helix-term" = "helix"; };
       # Set default app to hx (binary is from helix-term release build)
       # Set default package to helix-term release build
-      defaultOutputs = { app = "hx"; package = "helix"; };
+      defaultOutputs = {
+        app = "hx";
+        package = "helix";
+      };
       overrides = {
         crateOverrides = common: _: {
-          helix-term = prev: {
-            # link languages and theme toml files since helix-term expects them (for tests)
-            preConfigure = "ln -s ${common.root}/{languages.toml,theme.toml} ..";
-            buildInputs = (prev.buildInputs or [ ]) ++ [ common.cCompiler.cc.lib ];
-          };
-          # link languages and theme toml files since helix-view expects them
-          helix-view = _: { preConfigure = "ln -s ${common.root}/{languages.toml,theme.toml} .."; };
-          helix-syntax = _prev: {
-            preConfigure = "mkdir -p ../runtime/grammars";
-            postInstall = "cp -r ../runtime $out/runtime";
-          };
-        };
-        mainBuild = common: prev:
-          let
-            inherit (common) pkgs lib;
-            helixSyntax = lib.buildCrate {
-              root = self;
-              memberName = "helix-syntax";
-              defaultCrateOverrides = {
-                helix-syntax = common.crateOverrides.helix-syntax;
-              };
-              release = false;
+          helix-term = prev:
+            let
+              inherit (common) pkgs;
+              grammars = pkgs.callPackage ./grammars.nix { };
+              runtimeDir = pkgs.runCommand "helix-runtime" { } ''
+                mkdir -p $out
+                ln -s ${common.root}/runtime/* $out
+                rm -r $out/grammars
+                ln -s ${grammars} $out/grammars
+              '';
+            in
+            {
+              # disable fetching and building of tree-sitter grammars in the helix-term build.rs
+              HELIX_DISABLE_AUTO_GRAMMAR_BUILD = "1";
+              # link languages and theme toml files since helix-term expects them (for tests)
+              preConfigure = "ln -s ${common.root}/{languages.toml,theme.toml,base16_theme.toml} ..";
+              buildInputs = (prev.buildInputs or [ ]) ++ [ common.cCompiler.cc.lib ];
+              nativeBuildInputs = [ pkgs.makeWrapper ];
+
+              postFixup = ''
+                if [ -f "$out/bin/hx" ]; then
+                  wrapProgram "$out/bin/hx" --set HELIX_RUNTIME "${runtimeDir}"
+                fi
+              '';
             };
-            runtimeDir = pkgs.runCommand "helix-runtime" { } ''
-              mkdir -p $out
-              ln -s ${common.root}/runtime/* $out
-              ln -sf ${helixSyntax}/runtime/grammars $out
-            '';
-          in
-          lib.optionalAttrs (common.memberName == "helix-term") {
-            nativeBuildInputs = [ pkgs.makeWrapper ];
-            postFixup = ''
-              if [ -f "$out/bin/hx" ]; then
-                wrapProgram "$out/bin/hx" --set HELIX_RUNTIME "${runtimeDir}"
-              fi
-            '';
-          };
+        };
         shell = common: prev: {
-          packages = prev.packages ++ (with common.pkgs; [ lld_12 lldb cargo-tarpaulin ]);
+          packages = prev.packages ++ (with common.pkgs; [ lld_13 lldb cargo-tarpaulin cargo-flamegraph vulkan-tools ]);
           env = prev.env ++ [
             { name = "HELIX_RUNTIME"; eval = "$PWD/runtime"; }
             { name = "RUST_BACKTRACE"; value = "1"; }
-            { name = "RUSTFLAGS"; value = "-C link-arg=-fuse-ld=lld -C target-cpu=native"; }
+            # { name = "RUSTFLAGS"; value = "-C link-arg=-fuse-ld=lld -C target-cpu=native -Clink-arg=-Wl,--no-rosegment"; }
+            { name = "LD_LIBRARY_PATH"; value = nixpkgs.lib.makeLibraryPath (with common.pkgs; [
+              wayland libxkbcommon xorg.libxcb
+              vulkan-loader # vulkan
+              # libGL # GLES instead of vulkan
+            ]); }
           ];
         };
       };

grammars.nix

@@ -0,0 +1,106 @@
+{ stdenv, lib, runCommand, yj }:
+let
+  # HACK: nix < 2.6 has a bug in the toml parser, so we convert to JSON
+  # before parsing
+  languages-json = runCommand "languages-toml-to-json" { } ''
+    ${yj}/bin/yj -t < ${./languages.toml} > $out
+  '';
+  languagesConfig = if lib.versionAtLeast builtins.nixVersion "2.6.0" then
+      builtins.fromTOML (builtins.readFile ./languages.toml)
+    else
+      builtins.fromJSON (builtins.readFile (builtins.toPath languages-json));
+  isGitGrammar = (grammar:
+    builtins.hasAttr "source" grammar && builtins.hasAttr "git" grammar.source
+    && builtins.hasAttr "rev" grammar.source);
+  isGitHubGrammar = grammar: lib.hasPrefix "https://github.com" grammar.source.git;
+  toGitHubFetcher = url: let
+    match = builtins.match "https://github\.com/([^/]*)/([^/]*)/?" url;
+  in {
+    owner = builtins.elemAt match 0;
+    repo = builtins.elemAt match 1;
+  };
+  gitGrammars = builtins.filter isGitGrammar languagesConfig.grammar;
+  buildGrammar = grammar:
+    let
+      gh = toGitHubFetcher grammar.source.git;
+      sourceGit = builtins.fetchTree {
+        type = "git";
+        url = grammar.source.git;
+        rev = grammar.source.rev;
+        ref = grammar.source.ref or "HEAD";
+        shallow = true;
+      };
+      sourceGitHub = builtins.fetchTree {
+        type = "github";
+        owner = gh.owner;
+        repo = gh.repo;
+        inherit (grammar.source) rev;
+      };
+      source = if isGitHubGrammar grammar then sourceGitHub else sourceGit;
+    in stdenv.mkDerivation rec {
+      # see https://github.com/NixOS/nixpkgs/blob/fbdd1a7c0bc29af5325e0d7dd70e804a972eb465/pkgs/development/tools/parsing/tree-sitter/grammar.nix
+
+      pname = "helix-tree-sitter-${grammar.name}";
+      version = grammar.source.rev;
+
+      src = if builtins.hasAttr "subpath" grammar.source then
+        "${source}/${grammar.source.subpath}"
+      else
+        source;
+
+      dontUnpack = true;
+      dontConfigure = true;
+
+      FLAGS = [
+        "-I${src}/src"
+        "-g"
+        "-O3"
+        "-fPIC"
+        "-fno-exceptions"
+        "-Wl,-z,relro,-z,now"
+      ];
+
+      NAME = grammar.name;
+
+      buildPhase = ''
+        runHook preBuild
+
+        if [[ -e "$src/src/scanner.cc" ]]; then
+          $CXX -c "$src/src/scanner.cc" -o scanner.o $FLAGS
+        elif [[ -e "$src/src/scanner.c" ]]; then
+          $CC -c "$src/src/scanner.c" -o scanner.o $FLAGS
+        fi
+
+        $CC -c "$src/src/parser.c" -o parser.o $FLAGS
+        $CXX -shared -o $NAME.so *.o
+
+        ls -al
+
+        runHook postBuild
+      '';
+
+      installPhase = ''
+        runHook preInstall
+        mkdir $out
+        mv $NAME.so $out/
+        runHook postInstall
+      '';
+
+      # Strip failed on darwin: strip: error: symbols referenced by indirect symbol table entries that can't be stripped
+      fixupPhase = lib.optionalString stdenv.isLinux ''
+        runHook preFixup
+        $STRIP $out/$NAME.so
+        runHook postFixup
+      '';
+    };
+  builtGrammars = builtins.map (grammar: {
+    inherit (grammar) name;
+    artifact = buildGrammar grammar;
+  }) gitGrammars;
+  grammarLinks = builtins.map (grammar:
+    "ln -s ${grammar.artifact}/${grammar.name}.so $out/${grammar.name}.so")
+    builtGrammars;
+in runCommand "consolidated-helix-grammars" { } ''
+  mkdir -p $out
+  ${builtins.concatStringsSep "\n" grammarLinks}
+''

helix-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "helix-core"
-version = "0.5.0"
+version = "0.6.0"
 authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
 edition = "2021"
 license = "MPL-2.0"
@@ -11,19 +11,21 @@ homepage = "https://helix-editor.com"
 include = ["src/**/*", "README.md"]
 
 [features]
+unicode-lines = ["ropey/unicode_lines"]
 
 [dependencies]
-helix-syntax = { version = "0.5", path = "../helix-syntax" }
+helix-loader = { version = "0.6", path = "../helix-loader" }
 
-ropey = "1.3"
-smallvec = "1.7"
-tendril = "0.4.2"
-unicode-segmentation = "1.8"
+ropey = { version = "1.4", default-features = false }
+smallvec = "1.8"
+smartstring = "1.0.1"
+unicode-segmentation = "1.9"
 unicode-width = "0.1"
-unicode-general-category = "0.4"
+unicode-general-category = "0.5"
 # slab = "0.4.2"
-tree-sitter = "0.20"
-once_cell = "1.8"
+slotmap = "1.0"
+tree-sitter = "0.20" # TODO: use tree-sitter-facade / web-tree-sitter-sys on wasm32
+once_cell = "1.10"
 arc-swap = "1"
 regex = "1"
 
@@ -34,6 +36,10 @@ toml = "0.5"
 
 similar = "2.1"
 
+encoding_rs = "0.8"
+
+chrono = { version = "0.4", default-features = false, features = ["alloc", "std"] }
+
 etcetera = "0.3"
 
 [dev-dependencies]

helix-core/src/chars.rs

@@ -91,12 +91,14 @@ mod test {
 
     #[test]
     fn test_categorize() {
-        const EOL_TEST_CASE: &'static str = "\n\r\u{000B}\u{000C}\u{0085}\u{2028}\u{2029}";
-        const WORD_TEST_CASE: &'static str =
-            "_hello_world_あいうえおー1234567890１２３４５６７８９０";
-        const PUNCTUATION_TEST_CASE: &'static str =
+        #[cfg(not(feature = "unicode-lines"))]
+        const EOL_TEST_CASE: &str = "\n";
+        #[cfg(feature = "unicode-lines")]
+        const EOL_TEST_CASE: &str = "\n\u{000B}\u{000C}\u{0085}\u{2028}\u{2029}";
+        const WORD_TEST_CASE: &str = "_hello_world_あいうえおー1234567890１２３４５６７８９０";
+        const PUNCTUATION_TEST_CASE: &str =
             "!\"#$%&\'()*+,-./:;<=>?@[\\]^`{|}~！”＃＄％＆’（）＊＋、。：；＜＝＞？＠「」＾｀｛｜｝～";
-        const WHITESPACE_TEST_CASE: &'static str = "  　   ";
+        const WHITESPACE_TEST_CASE: &str = "  　   ";
 
         for ch in EOL_TEST_CASE.chars() {
             assert_eq!(CharCategory::Eol, categorize_char(ch));

helix-core/src/comment.rs

@@ -63,7 +63,7 @@ pub fn toggle_line_comments(doc: &Rope, selection: &Selection, token: Option<&st
     let token = token.unwrap_or("//");
     let comment = Tendril::from(format!("{} ", token));
 
-    let mut lines: Vec<usize> = Vec::new();
+    let mut lines: Vec<usize> = Vec::with_capacity(selection.len());
 
     let mut min_next_line = 0;
     for selection in selection {
@@ -72,7 +72,7 @@ pub fn toggle_line_comments(doc: &Rope, selection: &Selection, token: Option<&st
         let end = (end + 1).min(text.len_lines());
 
         lines.extend(start..end);
-        min_next_line = end + 1;
+        min_next_line = end;
     }
 
     let (commented, to_change, min, margin) = find_line_comment(token, text, lines);

helix-core/src/config.rs

@@ -0,0 +1,10 @@
+/// Syntax configuration loader based on built-in languages.toml.
+pub fn default_syntax_loader() -> crate::syntax::Configuration {
+    helix_loader::config::default_lang_config()
+        .try_into()
+        .expect("Could not serialize built-in languages.toml")
+}
+/// Syntax configuration loader based on user configured languages.toml.
+pub fn user_syntax_loader() -> Result<crate::syntax::Configuration, toml::de::Error> {
+    helix_loader::config::user_lang_config()?.try_into()
+}

helix-core/src/diagnostic.rs

@@ -1,23 +1,30 @@
 //! LSP diagnostic utility types.
+use serde::{Deserialize, Serialize};
 
 /// Describes the severity level of a [`Diagnostic`].
-#[derive(Debug, Eq, PartialEq)]
+#[derive(Debug, Clone, Copy, Eq, PartialEq, PartialOrd, Ord, Deserialize, Serialize)]
 pub enum Severity {
-    Error,
-    Warning,
-    Info,
     Hint,
+    Info,
+    Warning,
+    Error,
+}
+
+impl Default for Severity {
+    fn default() -> Self {
+        Self::Hint
+    }
 }
 
 /// A range of `char`s within the text.
-#[derive(Debug)]
+#[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq)]
 pub struct Range {
     pub start: usize,
     pub end: usize,
 }
 
 /// Corresponds to [`lsp_types::Diagnostic`](https://docs.rs/lsp-types/0.91.0/lsp_types/struct.Diagnostic.html)
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct Diagnostic {
     pub range: Range,
     pub line: usize,

helix-core/src/diff.rs

@@ -11,10 +11,6 @@ pub fn compare_ropes(old: &Rope, new: &Rope) -> Transaction {
     // A timeout is set so after 1 seconds, the algorithm will start
     // approximating. This is especially important for big `Rope`s or
     // `Rope`s that are extremely dissimilar to each other.
-    //
-    // Note: Ignore the clippy warning, as the trait bounds of
-    // `Transaction::change()` require an iterator implementing
-    // `ExactIterator`.
     let mut config = similar::TextDiff::configure();
     config.timeout(std::time::Duration::from_secs(1));
 
@@ -62,7 +58,7 @@ mod tests {
             let mut old = Rope::from(a);
             let new = Rope::from(b);
             compare_ropes(&old, &new).apply(&mut old);
-            old.to_string() == new.to_string()
+            old == new
         }
     }
 }

helix-core/src/graphemes.rs

@@ -1,6 +1,6 @@
 //! Utility functions to traverse the unicode graphemes of a `Rope`'s text contents.
 //!
-//! Based on https://github.com/cessen/led/blob/c4fa72405f510b7fd16052f90a598c429b3104a6/src/graphemes.rs
+//! Based on <https://github.com/cessen/led/blob/c4fa72405f510b7fd16052f90a598c429b3104a6/src/graphemes.rs>
 use ropey::{iter::Chunks, str_utils::byte_to_char_idx, RopeSlice};
 use unicode_segmentation::{GraphemeCursor, GraphemeIncomplete};
 use unicode_width::UnicodeWidthStr;
@@ -120,6 +120,43 @@ pub fn nth_next_grapheme_boundary(slice: RopeSlice, char_idx: usize, n: usize) -
     chunk_char_idx + tmp
 }
 
+#[must_use]
+pub fn nth_next_grapheme_boundary_byte(slice: RopeSlice, mut byte_idx: usize, n: usize) -> usize {
+    // Bounds check
+    debug_assert!(byte_idx <= slice.len_bytes());
+
+    // Get the chunk with our byte index in it.
+    let (mut chunk, mut chunk_byte_idx, mut _chunk_char_idx, _) = slice.chunk_at_byte(byte_idx);
+
+    // Set up the grapheme cursor.
+    let mut gc = GraphemeCursor::new(byte_idx, slice.len_bytes(), true);
+
+    // Find the nth next grapheme cluster boundary.
+    for _ in 0..n {
+        loop {
+            match gc.next_boundary(chunk, chunk_byte_idx) {
+                Ok(None) => return slice.len_bytes(),
+                Ok(Some(n)) => {
+                    byte_idx = n;
+                    break;
+                }
+                Err(GraphemeIncomplete::NextChunk) => {
+                    chunk_byte_idx += chunk.len();
+                    let (a, _, _c, _) = slice.chunk_at_byte(chunk_byte_idx);
+                    chunk = a;
+                    // chunk_char_idx = c;
+                }
+                Err(GraphemeIncomplete::PreContext(n)) => {
+                    let ctx_chunk = slice.chunk_at_byte(n - 1).0;
+                    gc.provide_context(ctx_chunk, n - ctx_chunk.len());
+                }
+                _ => unreachable!(),
+            }
+        }
+    }
+    byte_idx
+}
+
 /// Finds the next grapheme boundary after the given char position.
 #[must_use]
 #[inline(always)]
@@ -127,6 +164,13 @@ pub fn next_grapheme_boundary(slice: RopeSlice, char_idx: usize) -> usize {
     nth_next_grapheme_boundary(slice, char_idx, 1)
 }
 
+/// Finds the next grapheme boundary after the given byte position.
+#[must_use]
+#[inline(always)]
+pub fn next_grapheme_boundary_byte(slice: RopeSlice, byte_idx: usize) -> usize {
+    nth_next_grapheme_boundary_byte(slice, byte_idx, 1)
+}
+
 /// Returns the passed char index if it's already a grapheme boundary,
 /// or the next grapheme boundary char index if not.
 #[must_use]
@@ -151,6 +195,23 @@ pub fn ensure_grapheme_boundary_prev(slice: RopeSlice, char_idx: usize) -> usize
     }
 }
 
+/// Returns the passed byte index if it's already a grapheme boundary,
+/// or the next grapheme boundary byte index if not.
+#[must_use]
+#[inline]
+pub fn ensure_grapheme_boundary_next_byte(slice: RopeSlice, byte_idx: usize) -> usize {
+    if byte_idx == 0 {
+        byte_idx
+    } else {
+        // TODO: optimize so we're not constructing grapheme cursor twice
+        if is_grapheme_boundary_byte(slice, byte_idx) {
+            byte_idx
+        } else {
+            next_grapheme_boundary_byte(slice, byte_idx)
+        }
+    }
+}
+
 /// Returns whether the given char position is a grapheme boundary.
 #[must_use]
 pub fn is_grapheme_boundary(slice: RopeSlice, char_idx: usize) -> bool {
@@ -179,6 +240,31 @@ pub fn is_grapheme_boundary(slice: RopeSlice, char_idx: usize) -> bool {
     }
 }
 
+/// Returns whether the given byte position is a grapheme boundary.
+#[must_use]
+pub fn is_grapheme_boundary_byte(slice: RopeSlice, byte_idx: usize) -> bool {
+    // Bounds check
+    debug_assert!(byte_idx <= slice.len_bytes());
+
+    // Get the chunk with our byte index in it.
+    let (chunk, chunk_byte_idx, _, _) = slice.chunk_at_byte(byte_idx);
+
+    // Set up the grapheme cursor.
+    let mut gc = GraphemeCursor::new(byte_idx, slice.len_bytes(), true);
+
+    // Determine if the given position is a grapheme cluster boundary.
+    loop {
+        match gc.is_boundary(chunk, chunk_byte_idx) {
+            Ok(n) => return n,
+            Err(GraphemeIncomplete::PreContext(n)) => {
+                let (ctx_chunk, ctx_byte_start, _, _) = slice.chunk_at_byte(n - 1);
+                gc.provide_context(ctx_chunk, ctx_byte_start);
+            }
+            Err(_) => unreachable!(),
+        }
+    }
+}
+
 /// An iterator over the graphemes of a `RopeSlice`.
 #[derive(Clone)]
 pub struct RopeGraphemes<'a> {
@@ -247,10 +333,7 @@ impl<'a> Iterator for RopeGraphemes<'a> {
         }
 
         if a < self.cur_chunk_start {
-            let a_char = self.text.byte_to_char(a);
-            let b_char = self.text.byte_to_char(b);
-
-            Some(self.text.slice(a_char..b_char))
+            Some(self.text.byte_slice(a..b))
         } else {
             let a2 = a - self.cur_chunk_start;
             let b2 = b - self.cur_chunk_start;

helix-core/src/history.rs

@@ -1,4 +1,4 @@
-use crate::{ChangeSet, Rope, State, Transaction};
+use crate::{Assoc, ChangeSet, Range, Rope, State, Transaction};
 use once_cell::sync::Lazy;
 use regex::Regex;
 use std::num::NonZeroUsize;
@@ -22,10 +22,10 @@ use std::time::{Duration, Instant};
 ///
 /// The current revision is the one currently displayed in the buffer.
 ///
-/// Commiting a new revision to the history will update the last child of the
+/// Committing a new revision to the history will update the last child of the
 /// current revision, and push a new revision to the end of the vector.
 ///
-/// Revisions are commited with a timestamp. :earlier and :later can be used
+/// Revisions are committed with a timestamp. :earlier and :later can be used
 /// to jump to the closest revision to a moment in time relative to the timestamp
 /// of the current revision plus (:later) or minus (:earlier) the duration
 /// given to the command. If a single integer is given, the editor will instead
@@ -33,14 +33,14 @@ use std::time::{Duration, Instant};
 ///
 /// Limitations:
 ///  * Changes in selections currently don't commit history changes. The selection
-///    will only be updated to the state after a commited buffer change.
+///    will only be updated to the state after a committed buffer change.
 ///  * The vector of history revisions is currently unbounded. This might
 ///    cause the memory consumption to grow significantly large during long
 ///    editing sessions.
 ///  * Because delete transactions currently don't store the text that they
 ///    delete, we also store an inversion of the transaction.
 ///
-/// Using time to navigate the history: https://github.com/helix-editor/helix/pull/194
+/// Using time to navigate the history: <https://github.com/helix-editor/helix/pull/194>
 #[derive(Debug)]
 pub struct History {
     revisions: Vec<Revision>,
@@ -133,6 +133,32 @@ impl History {
         Some(&self.revisions[last_child.get()].transaction)
     }
 
+    // Get the position of last change
+    pub fn last_edit_pos(&self) -> Option<usize> {
+        if self.current == 0 {
+            return None;
+        }
+        let current_revision = &self.revisions[self.current];
+        let primary_selection = current_revision
+            .inversion
+            .selection()
+            .expect("inversion always contains a selection")
+            .primary();
+        let (_from, to, _fragment) = current_revision
+            .transaction
+            .changes_iter()
+            // find a change that matches the primary selection
+            .find(|(from, to, _fragment)| Range::new(*from, *to).overlaps(&primary_selection))
+            // or use the first change
+            .or_else(|| current_revision.transaction.changes_iter().next())
+            .unwrap();
+        let pos = current_revision
+            .transaction
+            .changes()
+            .map_pos(to, Assoc::After);
+        Some(pos)
+    }
+
     fn lowest_common_ancestor(&self, mut a: usize, mut b: usize) -> usize {
         use std::collections::HashSet;
         let mut a_path_set = HashSet::new();
@@ -256,13 +282,13 @@ impl History {
 }
 
 /// Whether to undo by a number of edits or a duration of time.
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone, Copy)]
 pub enum UndoKind {
     Steps(usize),
     TimePeriod(std::time::Duration),
 }
 
-/// A subset of sytemd.time time span syntax units.
+/// A subset of systemd.time time span syntax units.
 const TIME_UNITS: &[(&[&str], &str, u64)] = &[
     (&["seconds", "second", "sec", "s"], "seconds", 1),
     (&["minutes", "minute", "min", "m"], "minutes", 60),
@@ -422,8 +448,8 @@ mod test {
             change: crate::transaction::Change,
             instant: Instant,
         ) {
-            let txn = Transaction::change(&state.doc, vec![change.clone()].into_iter());
-            history.commit_revision_at_timestamp(&txn, &state, instant);
+            let txn = Transaction::change(&state.doc, vec![change].into_iter());
+            history.commit_revision_at_timestamp(&txn, state, instant);
             txn.apply(&mut state.doc);
         }
 

helix-core/src/increment/date_time.rs

@@ -0,0 +1,490 @@
+use chrono::{Datelike, Duration, NaiveDate, NaiveDateTime, NaiveTime, Timelike};
+use once_cell::sync::Lazy;
+use regex::Regex;
+use ropey::RopeSlice;
+
+use std::borrow::Cow;
+use std::cmp;
+
+use super::Increment;
+use crate::{Range, Tendril};
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct DateTimeIncrementor {
+    date_time: NaiveDateTime,
+    range: Range,
+    fmt: &'static str,
+    field: DateField,
+}
+
+impl DateTimeIncrementor {
+    pub fn from_range(text: RopeSlice, range: Range) -> Option<DateTimeIncrementor> {
+        let range = if range.is_empty() {
+            if range.anchor < text.len_chars() {
+                // Treat empty range as a cursor range.
+                range.put_cursor(text, range.anchor + 1, true)
+            } else {
+                // The range is empty and at the end of the text.
+                return None;
+            }
+        } else {
+            range
+        };
+
+        FORMATS.iter().find_map(|format| {
+            let from = range.from().saturating_sub(format.max_len);
+            let to = (range.from() + format.max_len).min(text.len_chars());
+
+            let (from_in_text, to_in_text) = (range.from() - from, range.to() - from);
+            let text: Cow<str> = text.slice(from..to).into();
+
+            let captures = format.regex.captures(&text)?;
+            if captures.len() - 1 != format.fields.len() {
+                return None;
+            }
+
+            let date_time = captures.get(0)?;
+            let offset = range.from() - from_in_text;
+            let range = Range::new(date_time.start() + offset, date_time.end() + offset);
+
+            let field = captures
+                .iter()
+                .skip(1)
+                .enumerate()
+                .find_map(|(i, capture)| {
+                    let capture = capture?;
+                    let capture_range = capture.range();
+
+                    if capture_range.contains(&from_in_text)
+                        && capture_range.contains(&(to_in_text - 1))
+                    {
+                        Some(format.fields[i])
+                    } else {
+                        None
+                    }
+                })?;
+
+            let has_date = format.fields.iter().any(|f| f.unit.is_date());
+            let has_time = format.fields.iter().any(|f| f.unit.is_time());
+
+            let date_time = &text[date_time.start()..date_time.end()];
+            let date_time = match (has_date, has_time) {
+                (true, true) => NaiveDateTime::parse_from_str(date_time, format.fmt).ok()?,
+                (true, false) => {
+                    let date = NaiveDate::parse_from_str(date_time, format.fmt).ok()?;
+
+                    date.and_hms(0, 0, 0)
+                }
+                (false, true) => {
+                    let time = NaiveTime::parse_from_str(date_time, format.fmt).ok()?;
+
+                    NaiveDate::from_ymd(0, 1, 1).and_time(time)
+                }
+                (false, false) => return None,
+            };
+
+            Some(DateTimeIncrementor {
+                date_time,
+                range,
+                fmt: format.fmt,
+                field,
+            })
+        })
+    }
+}
+
+impl Increment for DateTimeIncrementor {
+    fn increment(&self, amount: i64) -> (Range, Tendril) {
+        let date_time = match self.field.unit {
+            DateUnit::Years => add_years(self.date_time, amount),
+            DateUnit::Months => add_months(self.date_time, amount),
+            DateUnit::Days => add_duration(self.date_time, Duration::days(amount)),
+            DateUnit::Hours => add_duration(self.date_time, Duration::hours(amount)),
+            DateUnit::Minutes => add_duration(self.date_time, Duration::minutes(amount)),
+            DateUnit::Seconds => add_duration(self.date_time, Duration::seconds(amount)),
+            DateUnit::AmPm => toggle_am_pm(self.date_time),
+        }
+        .unwrap_or(self.date_time);
+
+        (self.range, date_time.format(self.fmt).to_string().into())
+    }
+}
+
+static FORMATS: Lazy<Vec<Format>> = Lazy::new(|| {
+    vec![
+        Format::new("%Y-%m-%d %H:%M:%S"), // 2021-11-24 07:12:23
+        Format::new("%Y/%m/%d %H:%M:%S"), // 2021/11/24 07:12:23
+        Format::new("%Y-%m-%d %H:%M"),    // 2021-11-24 07:12
+        Format::new("%Y/%m/%d %H:%M"),    // 2021/11/24 07:12
+        Format::new("%Y-%m-%d"),          // 2021-11-24
+        Format::new("%Y/%m/%d"),          // 2021/11/24
+        Format::new("%a %b %d %Y"),       // Wed Nov 24 2021
+        Format::new("%d-%b-%Y"),          // 24-Nov-2021
+        Format::new("%Y %b %d"),          // 2021 Nov 24
+        Format::new("%b %d, %Y"),         // Nov 24, 2021
+        Format::new("%-I:%M:%S %P"),      // 7:21:53 am
+        Format::new("%-I:%M %P"),         // 7:21 am
+        Format::new("%-I:%M:%S %p"),      // 7:21:53 AM
+        Format::new("%-I:%M %p"),         // 7:21 AM
+        Format::new("%H:%M:%S"),          // 23:24:23
+        Format::new("%H:%M"),             // 23:24
+    ]
+});
+
+#[derive(Debug)]
+struct Format {
+    fmt: &'static str,
+    fields: Vec<DateField>,
+    regex: Regex,
+    max_len: usize,
+}
+
+impl Format {
+    fn new(fmt: &'static str) -> Self {
+        let mut remaining = fmt;
+        let mut fields = Vec::new();
+        let mut regex = String::new();
+        let mut max_len = 0;
+
+        while let Some(i) = remaining.find('%') {
+            let after = &remaining[i + 1..];
+            let mut chars = after.chars();
+            let c = chars.next().unwrap();
+
+            let spec_len = if c == '-' {
+                1 + chars.next().unwrap().len_utf8()
+            } else {
+                c.len_utf8()
+            };
+
+            let specifier = &after[..spec_len];
+            let field = DateField::from_specifier(specifier).unwrap();
+            fields.push(field);
+            max_len += field.max_len + remaining[..i].len();
+            regex += &remaining[..i];
+            regex += &format!("({})", field.regex);
+            remaining = &after[spec_len..];
+        }
+
+        let regex = Regex::new(&regex).unwrap();
+
+        Self {
+            fmt,
+            fields,
+            regex,
+            max_len,
+        }
+    }
+}
+
+impl PartialEq for Format {
+    fn eq(&self, other: &Self) -> bool {
+        self.fmt == other.fmt && self.fields == other.fields && self.max_len == other.max_len
+    }
+}
+
+impl Eq for Format {}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+struct DateField {
+    regex: &'static str,
+    unit: DateUnit,
+    max_len: usize,
+}
+
+impl DateField {
+    fn from_specifier(specifier: &str) -> Option<Self> {
+        match specifier {
+            "Y" => Some(Self {
+                regex: r"\d{4}",
+                unit: DateUnit::Years,
+                max_len: 5,
+            }),
+            "y" => Some(Self {
+                regex: r"\d\d",
+                unit: DateUnit::Years,
+                max_len: 2,
+            }),
+            "m" => Some(Self {
+                regex: r"[0-1]\d",
+                unit: DateUnit::Months,
+                max_len: 2,
+            }),
+            "d" => Some(Self {
+                regex: r"[0-3]\d",
+                unit: DateUnit::Days,
+                max_len: 2,
+            }),
+            "-d" => Some(Self {
+                regex: r"[1-3]?\d",
+                unit: DateUnit::Days,
+                max_len: 2,
+            }),
+            "a" => Some(Self {
+                regex: r"Sun|Mon|Tue|Wed|Thu|Fri|Sat",
+                unit: DateUnit::Days,
+                max_len: 3,
+            }),
+            "A" => Some(Self {
+                regex: r"Sunday|Monday|Tuesday|Wednesday|Thursday|Friday|Saturday",
+                unit: DateUnit::Days,
+                max_len: 9,
+            }),
+            "b" | "h" => Some(Self {
+                regex: r"Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec",
+                unit: DateUnit::Months,
+                max_len: 3,
+            }),
+            "B" => Some(Self {
+                regex: r"January|February|March|April|May|June|July|August|September|October|November|December",
+                unit: DateUnit::Months,
+                max_len: 9,
+            }),
+            "H" => Some(Self {
+                regex: r"[0-2]\d",
+                unit: DateUnit::Hours,
+                max_len: 2,
+            }),
+            "M" => Some(Self {
+                regex: r"[0-5]\d",
+                unit: DateUnit::Minutes,
+                max_len: 2,
+            }),
+            "S" => Some(Self {
+                regex: r"[0-5]\d",
+                unit: DateUnit::Seconds,
+                max_len: 2,
+            }),
+            "I" => Some(Self {
+                regex: r"[0-1]\d",
+                unit: DateUnit::Hours,
+                max_len: 2,
+            }),
+            "-I" => Some(Self {
+                regex: r"1?\d",
+                unit: DateUnit::Hours,
+                max_len: 2,
+            }),
+            "P" => Some(Self {
+                regex: r"am|pm",
+                unit: DateUnit::AmPm,
+                max_len: 2,
+            }),
+            "p" => Some(Self {
+                regex: r"AM|PM",
+                unit: DateUnit::AmPm,
+                max_len: 2,
+            }),
+            _ => None,
+        }
+    }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+enum DateUnit {
+    Years,
+    Months,
+    Days,
+    Hours,
+    Minutes,
+    Seconds,
+    AmPm,
+}
+
+impl DateUnit {
+    fn is_date(self) -> bool {
+        matches!(self, DateUnit::Years | DateUnit::Months | DateUnit::Days)
+    }
+
+    fn is_time(self) -> bool {
+        matches!(
+            self,
+            DateUnit::Hours | DateUnit::Minutes | DateUnit::Seconds
+        )
+    }
+}
+
+fn ndays_in_month(year: i32, month: u32) -> u32 {
+    // The first day of the next month...
+    let (y, m) = if month == 12 {
+        (year + 1, 1)
+    } else {
+        (year, month + 1)
+    };
+    let d = NaiveDate::from_ymd(y, m, 1);
+
+    // ...is preceded by the last day of the original month.
+    d.pred().day()
+}
+
+fn add_months(date_time: NaiveDateTime, amount: i64) -> Option<NaiveDateTime> {
+    let month = (date_time.month0() as i64).checked_add(amount)?;
+    let year = date_time.year() + i32::try_from(month / 12).ok()?;
+    let year = if month.is_negative() { year - 1 } else { year };
+
+    // Normalize month
+    let month = month % 12;
+    let month = if month.is_negative() {
+        month + 12
+    } else {
+        month
+    } as u32
+        + 1;
+
+    let day = cmp::min(date_time.day(), ndays_in_month(year, month));
+
+    Some(NaiveDate::from_ymd(year, month, day).and_time(date_time.time()))
+}
+
+fn add_years(date_time: NaiveDateTime, amount: i64) -> Option<NaiveDateTime> {
+    let year = i32::try_from((date_time.year() as i64).checked_add(amount)?).ok()?;
+    let ndays = ndays_in_month(year, date_time.month());
+
+    if date_time.day() > ndays {
+        let d = NaiveDate::from_ymd(year, date_time.month(), ndays);
+        Some(d.succ().and_time(date_time.time()))
+    } else {
+        date_time.with_year(year)
+    }
+}
+
+fn add_duration(date_time: NaiveDateTime, duration: Duration) -> Option<NaiveDateTime> {
+    date_time.checked_add_signed(duration)
+}
+
+fn toggle_am_pm(date_time: NaiveDateTime) -> Option<NaiveDateTime> {
+    if date_time.hour() < 12 {
+        add_duration(date_time, Duration::hours(12))
+    } else {
+        add_duration(date_time, Duration::hours(-12))
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::Rope;
+
+    #[test]
+    fn test_increment_date_times() {
+        let tests = [
+            // (original, cursor, amount, expected)
+            ("2020-02-28", 0, 1, "2021-02-28"),
+            ("2020-02-29", 0, 1, "2021-03-01"),
+            ("2020-01-31", 5, 1, "2020-02-29"),
+            ("2020-01-20", 5, 1, "2020-02-20"),
+            ("2021-01-01", 5, -1, "2020-12-01"),
+            ("2021-01-31", 5, -2, "2020-11-30"),
+            ("2020-02-28", 8, 1, "2020-02-29"),
+            ("2021-02-28", 8, 1, "2021-03-01"),
+            ("2021-02-28", 0, -1, "2020-02-28"),
+            ("2021-03-01", 0, -1, "2020-03-01"),
+            ("2020-02-29", 5, -1, "2020-01-29"),
+            ("2020-02-20", 5, -1, "2020-01-20"),
+            ("2020-02-29", 8, -1, "2020-02-28"),
+            ("2021-03-01", 8, -1, "2021-02-28"),
+            ("1980/12/21", 8, 100, "1981/03/31"),
+            ("1980/12/21", 8, -100, "1980/09/12"),
+            ("1980/12/21", 8, 1000, "1983/09/17"),
+            ("1980/12/21", 8, -1000, "1978/03/27"),
+            ("2021-11-24 07:12:23", 0, 1, "2022-11-24 07:12:23"),
+            ("2021-11-24 07:12:23", 5, 1, "2021-12-24 07:12:23"),
+            ("2021-11-24 07:12:23", 8, 1, "2021-11-25 07:12:23"),
+            ("2021-11-24 07:12:23", 11, 1, "2021-11-24 08:12:23"),
+            ("2021-11-24 07:12:23", 14, 1, "2021-11-24 07:13:23"),
+            ("2021-11-24 07:12:23", 17, 1, "2021-11-24 07:12:24"),
+            ("2021/11/24 07:12:23", 0, 1, "2022/11/24 07:12:23"),
+            ("2021/11/24 07:12:23", 5, 1, "2021/12/24 07:12:23"),
+            ("2021/11/24 07:12:23", 8, 1, "2021/11/25 07:12:23"),
+            ("2021/11/24 07:12:23", 11, 1, "2021/11/24 08:12:23"),
+            ("2021/11/24 07:12:23", 14, 1, "2021/11/24 07:13:23"),
+            ("2021/11/24 07:12:23", 17, 1, "2021/11/24 07:12:24"),
+            ("2021-11-24 07:12", 0, 1, "2022-11-24 07:12"),
+            ("2021-11-24 07:12", 5, 1, "2021-12-24 07:12"),
+            ("2021-11-24 07:12", 8, 1, "2021-11-25 07:12"),
+            ("2021-11-24 07:12", 11, 1, "2021-11-24 08:12"),
+            ("2021-11-24 07:12", 14, 1, "2021-11-24 07:13"),
+            ("2021/11/24 07:12", 0, 1, "2022/11/24 07:12"),
+            ("2021/11/24 07:12", 5, 1, "2021/12/24 07:12"),
+            ("2021/11/24 07:12", 8, 1, "2021/11/25 07:12"),
+            ("2021/11/24 07:12", 11, 1, "2021/11/24 08:12"),
+            ("2021/11/24 07:12", 14, 1, "2021/11/24 07:13"),
+            ("Wed Nov 24 2021", 0, 1, "Thu Nov 25 2021"),
+            ("Wed Nov 24 2021", 4, 1, "Fri Dec 24 2021"),
+            ("Wed Nov 24 2021", 8, 1, "Thu Nov 25 2021"),
+            ("Wed Nov 24 2021", 11, 1, "Thu Nov 24 2022"),
+            ("24-Nov-2021", 0, 1, "25-Nov-2021"),
+            ("24-Nov-2021", 3, 1, "24-Dec-2021"),
+            ("24-Nov-2021", 7, 1, "24-Nov-2022"),
+            ("2021 Nov 24", 0, 1, "2022 Nov 24"),
+            ("2021 Nov 24", 5, 1, "2021 Dec 24"),
+            ("2021 Nov 24", 9, 1, "2021 Nov 25"),
+            ("Nov 24, 2021", 0, 1, "Dec 24, 2021"),
+            ("Nov 24, 2021", 4, 1, "Nov 25, 2021"),
+            ("Nov 24, 2021", 8, 1, "Nov 24, 2022"),
+            ("7:21:53 am", 0, 1, "8:21:53 am"),
+            ("7:21:53 am", 3, 1, "7:22:53 am"),
+            ("7:21:53 am", 5, 1, "7:21:54 am"),
+            ("7:21:53 am", 8, 1, "7:21:53 pm"),
+            ("7:21:53 AM", 0, 1, "8:21:53 AM"),
+            ("7:21:53 AM", 3, 1, "7:22:53 AM"),
+            ("7:21:53 AM", 5, 1, "7:21:54 AM"),
+            ("7:21:53 AM", 8, 1, "7:21:53 PM"),
+            ("7:21 am", 0, 1, "8:21 am"),
+            ("7:21 am", 3, 1, "7:22 am"),
+            ("7:21 am", 5, 1, "7:21 pm"),
+            ("7:21 AM", 0, 1, "8:21 AM"),
+            ("7:21 AM", 3, 1, "7:22 AM"),
+            ("7:21 AM", 5, 1, "7:21 PM"),
+            ("23:24:23", 1, 1, "00:24:23"),
+            ("23:24:23", 3, 1, "23:25:23"),
+            ("23:24:23", 6, 1, "23:24:24"),
+            ("23:24", 1, 1, "00:24"),
+            ("23:24", 3, 1, "23:25"),
+        ];
+
+        for (original, cursor, amount, expected) in tests {
+            let rope = Rope::from_str(original);
+            let range = Range::new(cursor, cursor + 1);
+            assert_eq!(
+                DateTimeIncrementor::from_range(rope.slice(..), range)
+                    .unwrap()
+                    .increment(amount)
+                    .1,
+                Tendril::from(expected)
+            );
+        }
+    }
+
+    #[test]
+    fn test_invalid_date_times() {
+        let tests = [
+            "0000-00-00",
+            "1980-2-21",
+            "1980-12-1",
+            "12345",
+            "2020-02-30",
+            "1999-12-32",
+            "19-12-32",
+            "1-2-3",
+            "0000/00/00",
+            "1980/2/21",
+            "1980/12/1",
+            "12345",
+            "2020/02/30",
+            "1999/12/32",
+            "19/12/32",
+            "1/2/3",
+            "123:456:789",
+            "11:61",
+            "2021-55-12 08:12:54",
+        ];
+
+        for invalid in tests {
+            let rope = Rope::from_str(invalid);
+            let range = Range::new(0, 1);
+
+            assert_eq!(DateTimeIncrementor::from_range(rope.slice(..), range), None)
+        }
+    }
+}

helix-core/src/increment/mod.rs

@@ -0,0 +1,8 @@
+pub mod date_time;
+pub mod number;
+
+use crate::{Range, Tendril};
+
+pub trait Increment {
+    fn increment(&self, amount: i64) -> (Range, Tendril);
+}

helix-core/src/increment/number.rs

@@ -0,0 +1,507 @@
+use std::borrow::Cow;
+
+use ropey::RopeSlice;
+
+use super::Increment;
+
+use crate::{
+    textobject::{textobject_word, TextObject},
+    Range, Tendril,
+};
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct NumberIncrementor<'a> {
+    value: i64,
+    radix: u32,
+    range: Range,
+
+    text: RopeSlice<'a>,
+}
+
+impl<'a> NumberIncrementor<'a> {
+    /// Return information about number under rang if there is one.
+    pub fn from_range(text: RopeSlice, range: Range) -> Option<NumberIncrementor> {
+        // If the cursor is on the minus sign of a number we want to get the word textobject to the
+        // right of it.
+        let range = if range.to() < text.len_chars()
+            && range.to() - range.from() <= 1
+            && text.char(range.from()) == '-'
+        {
+            Range::new(range.from() + 1, range.to() + 1)
+        } else {
+            range
+        };
+
+        let range = textobject_word(text, range, TextObject::Inside, 1, false);
+
+        // If there is a minus sign to the left of the word object, we want to include it in the range.
+        let range = if range.from() > 0 && text.char(range.from() - 1) == '-' {
+            range.extend(range.from() - 1, range.from())
+        } else {
+            range
+        };
+
+        let word: String = text
+            .slice(range.from()..range.to())
+            .chars()
+            .filter(|&c| c != '_')
+            .collect();
+        let (radix, prefixed) = if word.starts_with("0x") {
+            (16, true)
+        } else if word.starts_with("0o") {
+            (8, true)
+        } else if word.starts_with("0b") {
+            (2, true)
+        } else {
+            (10, false)
+        };
+
+        let number = if prefixed { &word[2..] } else { &word };
+
+        let value = i128::from_str_radix(number, radix).ok()?;
+        if (value.is_positive() && value.leading_zeros() < 64)
+            || (value.is_negative() && value.leading_ones() < 64)
+        {
+            return None;
+        }
+
+        let value = value as i64;
+        Some(NumberIncrementor {
+            range,
+            value,
+            radix,
+            text,
+        })
+    }
+}
+
+impl<'a> Increment for NumberIncrementor<'a> {
+    fn increment(&self, amount: i64) -> (Range, Tendril) {
+        let old_text: Cow<str> = self.text.slice(self.range.from()..self.range.to()).into();
+        let old_length = old_text.len();
+        let new_value = self.value.wrapping_add(amount);
+
+        // Get separator indexes from right to left.
+        let separator_rtl_indexes: Vec<usize> = old_text
+            .chars()
+            .rev()
+            .enumerate()
+            .filter_map(|(i, c)| if c == '_' { Some(i) } else { None })
+            .collect();
+
+        let format_length = if self.radix == 10 {
+            match (self.value.is_negative(), new_value.is_negative()) {
+                (true, false) => old_length - 1,
+                (false, true) => old_length + 1,
+                _ => old_text.len(),
+            }
+        } else {
+            old_text.len() - 2
+        } - separator_rtl_indexes.len();
+
+        let mut new_text = match self.radix {
+            2 => format!("0b{:01$b}", new_value, format_length),
+            8 => format!("0o{:01$o}", new_value, format_length),
+            10 if old_text.starts_with('0') || old_text.starts_with("-0") => {
+                format!("{:01$}", new_value, format_length)
+            }
+            10 => format!("{}", new_value),
+            16 => {
+                let (lower_count, upper_count): (usize, usize) =
+                    old_text.chars().skip(2).fold((0, 0), |(lower, upper), c| {
+                        (
+                            lower + c.is_ascii_lowercase().then(|| 1).unwrap_or(0),
+                            upper + c.is_ascii_uppercase().then(|| 1).unwrap_or(0),
+                        )
+                    });
+                if upper_count > lower_count {
+                    format!("0x{:01$X}", new_value, format_length)
+                } else {
+                    format!("0x{:01$x}", new_value, format_length)
+                }
+            }
+            _ => unimplemented!("radix not supported: {}", self.radix),
+        };
+
+        // Add separators from original number.
+        for &rtl_index in &separator_rtl_indexes {
+            if rtl_index < new_text.len() {
+                let new_index = new_text.len() - rtl_index;
+                new_text.insert(new_index, '_');
+            }
+        }
+
+        // Add in additional separators if necessary.
+        if new_text.len() > old_length && !separator_rtl_indexes.is_empty() {
+            let spacing = match separator_rtl_indexes.as_slice() {
+                [.., b, a] => a - b - 1,
+                _ => separator_rtl_indexes[0],
+            };
+
+            let prefix_length = if self.radix == 10 { 0 } else { 2 };
+            if let Some(mut index) = new_text.find('_') {
+                while index - prefix_length > spacing {
+                    index -= spacing;
+                    new_text.insert(index, '_');
+                }
+            }
+        }
+
+        (self.range, new_text.into())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::Rope;
+
+    #[test]
+    fn test_decimal_at_point() {
+        let rope = Rope::from_str("Test text 12345 more text.");
+        let range = Range::point(12);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(10, 15),
+                value: 12345,
+                radix: 10,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_uppercase_hexadecimal_at_point() {
+        let rope = Rope::from_str("Test text 0x123ABCDEF more text.");
+        let range = Range::point(12);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(10, 21),
+                value: 0x123ABCDEF,
+                radix: 16,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_lowercase_hexadecimal_at_point() {
+        let rope = Rope::from_str("Test text 0xfa3b4e more text.");
+        let range = Range::point(12);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(10, 18),
+                value: 0xfa3b4e,
+                radix: 16,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_octal_at_point() {
+        let rope = Rope::from_str("Test text 0o1074312 more text.");
+        let range = Range::point(12);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(10, 19),
+                value: 0o1074312,
+                radix: 8,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_binary_at_point() {
+        let rope = Rope::from_str("Test text 0b10111010010101 more text.");
+        let range = Range::point(12);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(10, 26),
+                value: 0b10111010010101,
+                radix: 2,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_negative_decimal_at_point() {
+        let rope = Rope::from_str("Test text -54321 more text.");
+        let range = Range::point(12);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(10, 16),
+                value: -54321,
+                radix: 10,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_decimal_with_leading_zeroes_at_point() {
+        let rope = Rope::from_str("Test text 000045326 more text.");
+        let range = Range::point(12);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(10, 19),
+                value: 45326,
+                radix: 10,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_negative_decimal_cursor_on_minus_sign() {
+        let rope = Rope::from_str("Test text -54321 more text.");
+        let range = Range::point(10);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(10, 16),
+                value: -54321,
+                radix: 10,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_number_under_range_start_of_rope() {
+        let rope = Rope::from_str("100");
+        let range = Range::point(0);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(0, 3),
+                value: 100,
+                radix: 10,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_number_under_range_end_of_rope() {
+        let rope = Rope::from_str("100");
+        let range = Range::point(2);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(0, 3),
+                value: 100,
+                radix: 10,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_number_surrounded_by_punctuation() {
+        let rope = Rope::from_str(",100;");
+        let range = Range::point(1);
+        assert_eq!(
+            NumberIncrementor::from_range(rope.slice(..), range),
+            Some(NumberIncrementor {
+                range: Range::new(1, 4),
+                value: 100,
+                radix: 10,
+                text: rope.slice(..),
+            })
+        );
+    }
+
+    #[test]
+    fn test_not_a_number_point() {
+        let rope = Rope::from_str("Test text 45326 more text.");
+        let range = Range::point(6);
+        assert_eq!(NumberIncrementor::from_range(rope.slice(..), range), None);
+    }
+
+    #[test]
+    fn test_number_too_large_at_point() {
+        let rope = Rope::from_str("Test text 0xFFFFFFFFFFFFFFFFF more text.");
+        let range = Range::point(12);
+        assert_eq!(NumberIncrementor::from_range(rope.slice(..), range), None);
+    }
+
+    #[test]
+    fn test_number_cursor_one_right_of_number() {
+        let rope = Rope::from_str("100 ");
+        let range = Range::point(3);
+        assert_eq!(NumberIncrementor::from_range(rope.slice(..), range), None);
+    }
+
+    #[test]
+    fn test_number_cursor_one_left_of_number() {
+        let rope = Rope::from_str(" 100");
+        let range = Range::point(0);
+        assert_eq!(NumberIncrementor::from_range(rope.slice(..), range), None);
+    }
+
+    #[test]
+    fn test_increment_basic_decimal_numbers() {
+        let tests = [
+            ("100", 1, "101"),
+            ("100", -1, "99"),
+            ("99", 1, "100"),
+            ("100", 1000, "1100"),
+            ("100", -1000, "-900"),
+            ("-1", 1, "0"),
+            ("-1", 2, "1"),
+            ("1", -1, "0"),
+            ("1", -2, "-1"),
+        ];
+
+        for (original, amount, expected) in tests {
+            let rope = Rope::from_str(original);
+            let range = Range::point(0);
+            assert_eq!(
+                NumberIncrementor::from_range(rope.slice(..), range)
+                    .unwrap()
+                    .increment(amount)
+                    .1,
+                Tendril::from(expected)
+            );
+        }
+    }
+
+    #[test]
+    fn test_increment_basic_hexadedimal_numbers() {
+        let tests = [
+            ("0x0100", 1, "0x0101"),
+            ("0x0100", -1, "0x00ff"),
+            ("0x0001", -1, "0x0000"),
+            ("0x0000", -1, "0xffffffffffffffff"),
+            ("0xffffffffffffffff", 1, "0x0000000000000000"),
+            ("0xffffffffffffffff", 2, "0x0000000000000001"),
+            ("0xffffffffffffffff", -1, "0xfffffffffffffffe"),
+            ("0xABCDEF1234567890", 1, "0xABCDEF1234567891"),
+            ("0xabcdef1234567890", 1, "0xabcdef1234567891"),
+        ];
+
+        for (original, amount, expected) in tests {
+            let rope = Rope::from_str(original);
+            let range = Range::point(0);
+            assert_eq!(
+                NumberIncrementor::from_range(rope.slice(..), range)
+                    .unwrap()
+                    .increment(amount)
+                    .1,
+                Tendril::from(expected)
+            );
+        }
+    }
+
+    #[test]
+    fn test_increment_basic_octal_numbers() {
+        let tests = [
+            ("0o0107", 1, "0o0110"),
+            ("0o0110", -1, "0o0107"),
+            ("0o0001", -1, "0o0000"),
+            ("0o7777", 1, "0o10000"),
+            ("0o1000", -1, "0o0777"),
+            ("0o0107", 10, "0o0121"),
+            ("0o0000", -1, "0o1777777777777777777777"),
+            ("0o1777777777777777777777", 1, "0o0000000000000000000000"),
+            ("0o1777777777777777777777", 2, "0o0000000000000000000001"),
+            ("0o1777777777777777777777", -1, "0o1777777777777777777776"),
+        ];
+
+        for (original, amount, expected) in tests {
+            let rope = Rope::from_str(original);
+            let range = Range::point(0);
+            assert_eq!(
+                NumberIncrementor::from_range(rope.slice(..), range)
+                    .unwrap()
+                    .increment(amount)
+                    .1,
+                Tendril::from(expected)
+            );
+        }
+    }
+
+    #[test]
+    fn test_increment_basic_binary_numbers() {
+        let tests = [
+            ("0b00000100", 1, "0b00000101"),
+            ("0b00000100", -1, "0b00000011"),
+            ("0b00000100", 2, "0b00000110"),
+            ("0b00000100", -2, "0b00000010"),
+            ("0b00000001", -1, "0b00000000"),
+            ("0b00111111", 10, "0b01001001"),
+            ("0b11111111", 1, "0b100000000"),
+            ("0b10000000", -1, "0b01111111"),
+            (
+                "0b0000",
+                -1,
+                "0b1111111111111111111111111111111111111111111111111111111111111111",
+            ),
+            (
+                "0b1111111111111111111111111111111111111111111111111111111111111111",
+                1,
+                "0b0000000000000000000000000000000000000000000000000000000000000000",
+            ),
+            (
+                "0b1111111111111111111111111111111111111111111111111111111111111111",
+                2,
+                "0b0000000000000000000000000000000000000000000000000000000000000001",
+            ),
+            (
+                "0b1111111111111111111111111111111111111111111111111111111111111111",
+                -1,
+                "0b1111111111111111111111111111111111111111111111111111111111111110",
+            ),
+        ];
+
+        for (original, amount, expected) in tests {
+            let rope = Rope::from_str(original);
+            let range = Range::point(0);
+            assert_eq!(
+                NumberIncrementor::from_range(rope.slice(..), range)
+                    .unwrap()
+                    .increment(amount)
+                    .1,
+                Tendril::from(expected)
+            );
+        }
+    }
+
+    #[test]
+    fn test_increment_with_separators() {
+        let tests = [
+            ("999_999", 1, "1_000_000"),
+            ("1_000_000", -1, "999_999"),
+            ("-999_999", -1, "-1_000_000"),
+            ("0x0000_0000_0001", 0x1_ffff_0000, "0x0001_ffff_0001"),
+            ("0x0000_0000_0001", 0x1_ffff_0000, "0x0001_ffff_0001"),
+            ("0x0000_0000_0001", 0x1_ffff_0000, "0x0001_ffff_0001"),
+            ("0x0000_0000", -1, "0xffff_ffff_ffff_ffff"),
+            ("0x0000_0000_0000", -1, "0xffff_ffff_ffff_ffff"),
+            ("0b01111111_11111111", 1, "0b10000000_00000000"),
+            ("0b11111111_11111111", 1, "0b1_00000000_00000000"),
+        ];
+
+        for (original, amount, expected) in tests {
+            let rope = Rope::from_str(original);
+            let range = Range::point(0);
+            assert_eq!(
+                NumberIncrementor::from_range(rope.slice(..), range)
+                    .unwrap()
+                    .increment(amount)
+                    .1,
+                Tendril::from(expected)
+            );
+        }
+    }
+}

helix-core/src/indent.rs

@@ -1,7 +1,10 @@
+use std::collections::HashMap;
+
+use tree_sitter::{Query, QueryCursor, QueryPredicateArg};
+
 use crate::{
     chars::{char_is_line_ending, char_is_whitespace},
-    find_first_non_whitespace_char,
-    syntax::{IndentQuery, LanguageConfiguration, Syntax},
+    syntax::{LanguageConfiguration, RopeProvider, Syntax},
     tree_sitter::Node,
     Rope, RopeSlice,
 };
@@ -174,8 +177,7 @@ pub fn auto_detect_indent_style(document_text: &Rope) -> Option<IndentStyle> {
 
 /// To determine indentation of a newly inserted line, figure out the indentation at the last col
 /// of the previous line.
-#[allow(dead_code)]
-fn indent_level_for_line(line: RopeSlice, tab_width: usize) -> usize {
+pub fn indent_level_for_line(line: RopeSlice, tab_width: usize) -> usize {
     let mut len = 0;
     for ch in line.chars() {
         match ch {
@@ -188,129 +190,405 @@ fn indent_level_for_line(line: RopeSlice, tab_width: usize) -> usize {
     len / tab_width
 }
 
-/// Find the highest syntax node at position.
-/// This is to identify the column where this node (e.g., an HTML closing tag) ends.
-fn get_highest_syntax_node_at_bytepos(syntax: &Syntax, pos: usize) -> Option<Node> {
-    let tree = syntax.tree();
-
-    // named_descendant
-    let mut node = match tree.root_node().descendant_for_byte_range(pos, pos) {
-        Some(node) => node,
-        None => return None,
-    };
-
-    while let Some(parent) = node.parent() {
-        if parent.start_byte() == node.start_byte() {
-            node = parent
+/// Computes for node and all ancestors whether they are the first node on their line.
+/// The first entry in the return value represents the root node, the last one the node itself
+fn get_first_in_line(mut node: Node, byte_pos: usize, new_line: bool) -> Vec<bool> {
+    let mut first_in_line = Vec::new();
+    loop {
+        if let Some(prev) = node.prev_sibling() {
+            // If we insert a new line, the first node at/after the cursor is considered to be the first in its line
+            let first = prev.end_position().row != node.start_position().row
+                || (new_line && node.start_byte() >= byte_pos && prev.start_byte() < byte_pos);
+            first_in_line.push(Some(first));
+        } else {
+            // Nodes that have no previous siblings are first in their line if and only if their parent is
+            // (which we don't know yet)
+            first_in_line.push(None);
+        }
+        if let Some(parent) = node.parent() {
+            node = parent;
         } else {
             break;
         }
     }
 
-    Some(node)
-}
-
-fn calculate_indentation(query: &IndentQuery, node: Option<Node>, newline: bool) -> usize {
-    // NOTE: can't use contains() on query because of comparing Vec<String> and &str
-    // https://doc.rust-lang.org/std/vec/struct.Vec.html#method.contains
-
-    let mut increment: isize = 0;
-
-    let mut node = match node {
-        Some(node) => node,
-        None => return 0,
-    };
-
-    let mut prev_start = node.start_position().row;
-
-    // if we're calculating indentation for a brand new line then the current node will become the
-    // parent node. We need to take it's indentation level into account too.
-    let node_kind = node.kind();
-    if newline && query.indent.contains(node_kind) {
-        increment += 1;
-    }
-
-    while let Some(parent) = node.parent() {
-        let parent_kind = parent.kind();
-        let start = parent.start_position().row;
-
-        // detect deeply nested indents in the same line
-        // .map(|a| {       <-- ({ is two scopes
-        //     let len = 1; <-- indents one level
-        // })               <-- }) is two scopes
-        let starts_same_line = start == prev_start;
-
-        if query.outdent.contains(node.kind()) && !starts_same_line {
-            // we outdent by skipping the rules for the current level and jumping up
-            // node = parent;
-            increment -= 1;
-            // continue;
+    let mut result = Vec::with_capacity(first_in_line.len());
+    let mut parent_is_first = true; // The root node is by definition the first node in its line
+    for first in first_in_line.into_iter().rev() {
+        if let Some(first) = first {
+            result.push(first);
+            parent_is_first = first;
+        } else {
+            result.push(parent_is_first);
         }
-
-        if query.indent.contains(parent_kind) // && not_first_or_last_sibling
-            && !starts_same_line
-        {
-            // println!("is_scope {}", parent_kind);
-            prev_start = start;
-            increment += 1
-        }
-
-        // if last_scope && increment > 0 && ...{ ignore }
-
-        node = parent;
     }
-
-    increment.max(0) as usize
+    result
 }
 
-#[allow(dead_code)]
-fn suggested_indent_for_line(
-    language_config: &LanguageConfiguration,
-    syntax: Option<&Syntax>,
-    text: RopeSlice,
-    line_num: usize,
-    _tab_width: usize,
-) -> usize {
-    if let Some(start) = find_first_non_whitespace_char(text.line(line_num)) {
-        return suggested_indent_for_pos(
-            Some(language_config),
-            syntax,
-            text,
-            start + text.line_to_char(line_num),
-            false,
-        );
-    };
-
-    // if the line is blank, indent should be zero
-    0
+/// The total indent for some line of code.
+/// This is usually constructed in one of 2 ways:
+/// - Successively add indent captures to get the (added) indent from a single line
+/// - Successively add the indent results for each line
+#[derive(Default)]
+struct Indentation {
+    /// The total indent (the number of indent levels) is defined as max(0, indent-outdent).
+    /// The string that this results in depends on the indent style (spaces or tabs, etc.)
+    indent: usize,
+    outdent: usize,
+}
+impl Indentation {
+    /// Add some other [IndentResult] to this.
+    /// The added indent should be the total added indent from one line
+    fn add_line(&mut self, added: &Indentation) {
+        if added.indent > 0 && added.outdent == 0 {
+            self.indent += 1;
+        } else if added.outdent > 0 && added.indent == 0 {
+            self.outdent += 1;
+        }
+    }
+    /// Add an indent capture to this indent.
+    /// All the captures that are added in this way should be on the same line.
+    fn add_capture(&mut self, added: IndentCaptureType) {
+        match added {
+            IndentCaptureType::Indent => {
+                self.indent = 1;
+            }
+            IndentCaptureType::Outdent => {
+                self.outdent = 1;
+            }
+        }
+    }
+    fn as_string(&self, indent_style: &IndentStyle) -> String {
+        let indent_level = if self.indent >= self.outdent {
+            self.indent - self.outdent
+        } else {
+            log::warn!("Encountered more outdent than indent nodes while calculating indentation: {} outdent, {} indent", self.outdent, self.indent);
+            0
+        };
+        indent_style.as_str().repeat(indent_level)
+    }
 }
 
-// TODO: two usecases: if we are triggering this for a new, blank line:
-// - it should return 0 when mass indenting stuff
-// - it should look up the wrapper node and count it too when we press o/O
-pub fn suggested_indent_for_pos(
-    language_config: Option<&LanguageConfiguration>,
-    syntax: Option<&Syntax>,
+/// An indent definition which corresponds to a capture from the indent query
+struct IndentCapture {
+    capture_type: IndentCaptureType,
+    scope: IndentScope,
+}
+#[derive(Clone, Copy)]
+enum IndentCaptureType {
+    Indent,
+    Outdent,
+}
+impl IndentCaptureType {
+    fn default_scope(&self) -> IndentScope {
+        match self {
+            IndentCaptureType::Indent => IndentScope::Tail,
+            IndentCaptureType::Outdent => IndentScope::All,
+        }
+    }
+}
+/// This defines which part of a node an [IndentCapture] applies to.
+/// Each [IndentCaptureType] has a default scope, but the scope can be changed
+/// with `#set!` property declarations.
+#[derive(Clone, Copy)]
+enum IndentScope {
+    /// The indent applies to the whole node
+    All,
+    /// The indent applies to everything except for the first line of the node
+    Tail,
+}
+
+/// Execute the indent query.
+/// Returns for each node (identified by its id) a list of indent captures for that node.
+fn query_indents(
+    query: &Query,
+    syntax: &Syntax,
+    cursor: &mut QueryCursor,
     text: RopeSlice,
+    range: std::ops::Range<usize>,
+    // Position of the (optional) newly inserted line break.
+    // Given as (line, byte_pos)
+    new_line_break: Option<(usize, usize)>,
+) -> HashMap<usize, Vec<IndentCapture>> {
+    let mut indent_captures: HashMap<usize, Vec<IndentCapture>> = HashMap::new();
+    cursor.set_byte_range(range);
+    // Iterate over all captures from the query
+    for m in cursor.matches(query, syntax.tree().root_node(), RopeProvider(text)) {
+        // Skip matches where not all custom predicates are fulfilled
+        if !query.general_predicates(m.pattern_index).iter().all(|pred| {
+            match pred.operator.as_ref() {
+                "not-kind-eq?" => match (pred.args.get(0), pred.args.get(1)) {
+                    (
+                        Some(QueryPredicateArg::Capture(capture_idx)),
+                        Some(QueryPredicateArg::String(kind)),
+                    ) => {
+                        let node = m.nodes_for_capture_index(*capture_idx).next();
+                        match node {
+                            Some(node) => node.kind()!=kind.as_ref(),
+                            _ => true,
+                        }
+                    }
+                    _ => {
+                        panic!("Invalid indent query: Arguments to \"not-kind-eq?\" must be a capture and a string");
+                    }
+                },
+                "same-line?" | "not-same-line?" => {
+                    match (pred.args.get(0), pred.args.get(1)) {
+                        (
+                            Some(QueryPredicateArg::Capture(capt1)),
+                            Some(QueryPredicateArg::Capture(capt2))
+                        ) => {
+                            let get_line_num = |node: Node| {
+                                let mut node_line = node.start_position().row;
+                                // Adjust for the new line that will be inserted
+                                if let Some((line, byte)) = new_line_break {
+                                    if node_line==line && node.start_byte()>=byte {
+                                        node_line += 1;
+                                    }
+                                }
+                                node_line
+                            };
+                            let n1 = m.nodes_for_capture_index(*capt1).next();
+                            let n2 = m.nodes_for_capture_index(*capt2).next();
+                            match (n1, n2) {
+                                (Some(n1), Some(n2)) => {
+                                    let same_line = get_line_num(n1)==get_line_num(n2);
+                                    same_line==(pred.operator.as_ref()=="same-line?")
+                                }
+                                _ => true,
+                            }
+                        }
+                        _ => {
+                            panic!("Invalid indent query: Arguments to \"{}\" must be 2 captures", pred.operator);
+                        }
+                    }
+                }
+                _ => {
+                    panic!(
+                        "Invalid indent query: Unknown predicate (\"{}\")",
+                        pred.operator
+                    );
+                }
+            }
+        }) {
+            continue;
+        }
+        for capture in m.captures {
+            let capture_type = query.capture_names()[capture.index as usize].as_str();
+            let capture_type = match capture_type {
+                "indent" => IndentCaptureType::Indent,
+                "outdent" => IndentCaptureType::Outdent,
+                _ => {
+                    // Ignore any unknown captures (these may be needed for predicates such as #match?)
+                    continue;
+                }
+            };
+            let scope = capture_type.default_scope();
+            let mut indent_capture = IndentCapture {
+                capture_type,
+                scope,
+            };
+            // Apply additional settings for this capture
+            for property in query.property_settings(m.pattern_index) {
+                match property.key.as_ref() {
+                    "scope" => {
+                        indent_capture.scope = match property.value.as_deref() {
+                            Some("all") => IndentScope::All,
+                            Some("tail") => IndentScope::Tail,
+                            Some(s) => {
+                                panic!("Invalid indent query: Unknown value for \"scope\" property (\"{}\")", s);
+                            }
+                            None => {
+                                panic!(
+                                    "Invalid indent query: Missing value for \"scope\" property"
+                                );
+                            }
+                        }
+                    }
+                    _ => {
+                        panic!(
+                            "Invalid indent query: Unknown property \"{}\"",
+                            property.key
+                        );
+                    }
+                }
+            }
+            indent_captures
+                .entry(capture.node.id())
+                // Most entries only need to contain a single IndentCapture
+                .or_insert_with(|| Vec::with_capacity(1))
+                .push(indent_capture);
+        }
+    }
+    indent_captures
+}
+
+/// Use the syntax tree to determine the indentation for a given position.
+/// This can be used in 2 ways:
+///
+/// - To get the correct indentation for an existing line (new_line=false), not necessarily equal to the current indentation.
+///   - In this case, pos should be inside the first tree-sitter node on that line.
+///     In most cases, this can just be the first non-whitespace on that line.
+///   - To get the indentation for a new line (new_line=true). This behaves like the first usecase if the part of the current line
+///     after pos were moved to a new line.
+///
+/// The indentation is determined by traversing all the tree-sitter nodes containing the position.
+/// Each of these nodes produces some [AddedIndent] for:
+///
+/// - The line of the (beginning of the) node. This is defined by the scope `all` if this is the first node on its line.
+/// - The line after the node. This is defined by:
+///   - The scope `tail`.
+///   - The scope `all` if this node is not the first node on its line.
+/// Intuitively, `all` applies to everything contained in this node while `tail` applies to everything except for the first line of the node.
+/// The indents from different nodes for the same line are then combined.
+/// The [IndentResult] is simply the sum of the [AddedIndent] for all lines.
+///
+/// Specifying which line exactly an [AddedIndent] applies to is important because indents on the same line combine differently than indents on different lines:
+/// ```ignore
+/// some_function(|| {
+///     // Both the function parameters as well as the contained block should be indented.
+///     // Because they are on the same line, this only yields one indent level
+/// });
+/// ```
+///
+/// ```ignore
+/// some_function(
+///     parm1,
+///     || {
+///         // Here we get 2 indent levels because the 'parameters' and the 'block' node begin on different lines
+///     },
+/// );
+/// ```
+pub fn treesitter_indent_for_pos(
+    query: &Query,
+    syntax: &Syntax,
+    indent_style: &IndentStyle,
+    text: RopeSlice,
+    line: usize,
     pos: usize,
     new_line: bool,
-) -> usize {
+) -> Option<String> {
+    let byte_pos = text.char_to_byte(pos);
+    let mut node = syntax
+        .tree()
+        .root_node()
+        .descendant_for_byte_range(byte_pos, byte_pos)?;
+    let mut first_in_line = get_first_in_line(node, byte_pos, new_line);
+    let new_line_break = if new_line {
+        Some((line, byte_pos))
+    } else {
+        None
+    };
+    let query_result = crate::syntax::PARSER.with(|ts_parser| {
+        let mut ts_parser = ts_parser.borrow_mut();
+        let mut cursor = ts_parser.cursors.pop().unwrap_or_else(QueryCursor::new);
+        let query_result = query_indents(
+            query,
+            syntax,
+            &mut cursor,
+            text,
+            byte_pos..byte_pos + 1,
+            new_line_break,
+        );
+        ts_parser.cursors.push(cursor);
+        query_result
+    });
+
+    let mut result = Indentation::default();
+    // We always keep track of all the indent changes on one line, in order to only indent once
+    // even if there are multiple "indent" nodes on the same line
+    let mut indent_for_line = Indentation::default();
+    let mut indent_for_line_below = Indentation::default();
+    loop {
+        // This can safely be unwrapped because `first_in_line` contains
+        // one entry for each ancestor of the node (which is what we iterate over)
+        let is_first = *first_in_line.last().unwrap();
+        // Apply all indent definitions for this node
+        if let Some(definitions) = query_result.get(&node.id()) {
+            for definition in definitions {
+                match definition.scope {
+                    IndentScope::All => {
+                        if is_first {
+                            indent_for_line.add_capture(definition.capture_type);
+                        } else {
+                            indent_for_line_below.add_capture(definition.capture_type);
+                        }
+                    }
+                    IndentScope::Tail => {
+                        indent_for_line_below.add_capture(definition.capture_type);
+                    }
+                }
+            }
+        }
+
+        if let Some(parent) = node.parent() {
+            let mut node_line = node.start_position().row;
+            let mut parent_line = parent.start_position().row;
+            if node_line == line && new_line {
+                // Also consider the line that will be inserted
+                if node.start_byte() >= byte_pos {
+                    node_line += 1;
+                }
+                if parent.start_byte() >= byte_pos {
+                    parent_line += 1;
+                }
+            };
+            if node_line != parent_line {
+                if node_line < line + (new_line as usize) {
+                    // Don't add indent for the line below the line of the query
+                    result.add_line(&indent_for_line_below);
+                }
+                if node_line == parent_line + 1 {
+                    indent_for_line_below = indent_for_line;
+                } else {
+                    result.add_line(&indent_for_line);
+                    indent_for_line_below = Indentation::default();
+                }
+                indent_for_line = Indentation::default();
+            }
+
+            node = parent;
+            first_in_line.pop();
+        } else {
+            result.add_line(&indent_for_line_below);
+            result.add_line(&indent_for_line);
+            break;
+        }
+    }
+    Some(result.as_string(indent_style))
+}
+
+/// Returns the indentation for a new line.
+/// This is done either using treesitter, or if that's not available by copying the indentation from the current line
+#[allow(clippy::too_many_arguments)]
+pub fn indent_for_newline(
+    language_config: Option<&LanguageConfiguration>,
+    syntax: Option<&Syntax>,
+    indent_style: &IndentStyle,
+    tab_width: usize,
+    text: RopeSlice,
+    line_before: usize,
+    line_before_end_pos: usize,
+    current_line: usize,
+) -> String {
     if let (Some(query), Some(syntax)) = (
         language_config.and_then(|config| config.indent_query()),
         syntax,
     ) {
-        let byte_start = text.char_to_byte(pos);
-        let node = get_highest_syntax_node_at_bytepos(syntax, byte_start);
-
-        // let config = load indentation query config from Syntax(should contain language_config)
-
-        // TODO: special case for comments
-        // TODO: if preserve_leading_whitespace
-        calculate_indentation(query, node, new_line)
-    } else {
-        // TODO: heuristics for non-tree sitter grammars
-        0
+        if let Some(indent) = treesitter_indent_for_pos(
+            query,
+            syntax,
+            indent_style,
+            text,
+            line_before,
+            line_before_end_pos,
+            true,
+        ) {
+            return indent;
+        };
     }
+    let indent_level = indent_level_for_line(text.line(current_line), tab_width);
+    indent_style.as_str().repeat(indent_level)
 }
 
 pub fn get_scopes(syntax: Option<&Syntax>, text: RopeSlice, pos: usize) -> Vec<&'static str> {
@@ -354,141 +632,4 @@ mod test {
         let line = Rope::from("\t    \tfn new"); // 1 tab, 4 spaces, tab
         assert_eq!(indent_level_for_line(line.slice(..), tab_width), 3);
     }
-
-    #[test]
-    fn test_suggested_indent_for_line() {
-        let doc = Rope::from(
-            "
-use std::{
-    io::{self, stdout, Stdout, Write},
-    path::PathBuf,
-    sync::Arc,
-    time::Duration,
-}
-mod test {
-    fn hello_world() {
-        1 + 1;
-
-        let does_indentation_work = 1;
-
-        let test_function = function_with_param(this_param,
-            that_param
-        );
-
-        let test_function = function_with_param(
-            this_param,
-            that_param
-        );
-
-        let test_function = function_with_proper_indent(param1,
-            param2,
-        );
-
-        let selection = Selection::new(
-            changes
-                .clone()
-                .map(|(start, end, text): (usize, usize, Option<Tendril>)| {
-                    let len = text.map(|text| text.len()).unwrap() - 1; // minus newline
-                    let pos = start + len;
-                    Range::new(pos, pos)
-                })
-                .collect(),
-            0,
-        );
-
-        return;
-    }
-}
-
-impl<A, D> MyTrait<A, D> for YourType
-where
-    A: TraitB + TraitC,
-    D: TraitE + TraitF,
-{
-
-}
-#[test]
-//
-match test {
-    Some(a) => 1,
-    None => {
-        unimplemented!()
-    }
-}
-std::panic::set_hook(Box::new(move |info| {
-    hook(info);
-}));
-
-{ { {
-    1
-}}}
-
-pub fn change<I>(document: &Document, changes: I) -> Self
-where
-    I: IntoIterator<Item = Change> + ExactSizeIterator,
-{
-    [
-        1,
-        2,
-        3,
-    ];
-    (
-        1,
-        2
-    );
-    true
-}
-",
-        );
-
-        let doc = Rope::from(doc);
-        use crate::syntax::{
-            Configuration, IndentationConfiguration, LanguageConfiguration, Loader,
-        };
-        use once_cell::sync::OnceCell;
-        let loader = Loader::new(Configuration {
-            language: vec![LanguageConfiguration {
-                scope: "source.rust".to_string(),
-                file_types: vec!["rs".to_string()],
-                language_id: "Rust".to_string(),
-                highlight_config: OnceCell::new(),
-                config: None,
-                //
-                injection_regex: None,
-                roots: vec![],
-                comment_token: None,
-                auto_format: false,
-                language_server: None,
-                indent: Some(IndentationConfiguration {
-                    tab_width: 4,
-                    unit: String::from("    "),
-                }),
-                indent_query: OnceCell::new(),
-                textobject_query: OnceCell::new(),
-            }],
-        });
-
-        // set runtime path so we can find the queries
-        let mut runtime = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
-        runtime.push("../runtime");
-        std::env::set_var("HELIX_RUNTIME", runtime.to_str().unwrap());
-
-        let language_config = loader.language_config_for_scope("source.rust").unwrap();
-        let highlight_config = language_config.highlight_config(&[]).unwrap();
-        let syntax = Syntax::new(&doc, highlight_config.clone());
-        let text = doc.slice(..);
-        let tab_width = 4;
-
-        for i in 0..doc.len_lines() {
-            let line = text.line(i);
-            let indent = indent_level_for_line(line, tab_width);
-            assert_eq!(
-                suggested_indent_for_line(&language_config, Some(&syntax), text, i, tab_width),
-                indent,
-                "line {}: {}",
-                i,
-                line
-            );
-        }
-    }
 }

helix-core/src/lib.rs

@@ -1,10 +1,14 @@
+pub use encoding_rs as encoding;
+
 pub mod auto_pairs;
 pub mod chars;
 pub mod comment;
+pub mod config;
 pub mod diagnostic;
 pub mod diff;
 pub mod graphemes;
 pub mod history;
+pub mod increment;
 pub mod indent;
 pub mod line_ending;
 pub mod macros;
@@ -16,9 +20,11 @@ mod position;
 pub mod register;
 pub mod search;
 pub mod selection;
+pub mod shellwords;
 mod state;
 pub mod surround;
 pub mod syntax;
+pub mod test;
 pub mod textobject;
 mod transaction;
 
@@ -28,174 +34,37 @@ pub mod unicode {
     pub use unicode_width as width;
 }
 
-static RUNTIME_DIR: once_cell::sync::Lazy<std::path::PathBuf> =
-    once_cell::sync::Lazy::new(runtime_dir);
-
 pub fn find_first_non_whitespace_char(line: RopeSlice) -> Option<usize> {
     line.chars().position(|ch| !ch.is_whitespace())
 }
 
-pub fn find_root(root: Option<&str>) -> Option<std::path::PathBuf> {
-    let current_dir = std::env::current_dir().expect("unable to determine current directory");
-
-    let root = match root {
-        Some(root) => {
-            let root = std::path::Path::new(root);
-            if root.is_absolute() {
-                root.to_path_buf()
-            } else {
-                current_dir.join(root)
-            }
-        }
-        None => current_dir,
-    };
-
-    for ancestor in root.ancestors() {
-        // TODO: also use defined roots if git isn't found
-        if ancestor.join(".git").is_dir() {
-            return Some(ancestor.to_path_buf());
-        }
-    }
-    None
+/// Find project root.
+///
+/// Order of detection:
+/// * Top-most folder containing a root marker in current git repository
+/// * Git repostory root if no marker detected
+/// * Top-most folder containing a root marker if not git repository detected
+/// * Current working directory as fallback
+pub fn find_root(root: Option<&str>, root_markers: &[String]) -> Option<std::path::PathBuf> {
+    helix_loader::find_root_impl(root, root_markers)
+        .first()
+        .cloned()
 }
 
-pub fn runtime_dir() -> std::path::PathBuf {
-    if let Ok(dir) = std::env::var("HELIX_RUNTIME") {
-        return dir.into();
-    }
+pub use ropey::{str_utils, Rope, RopeBuilder, RopeSlice};
 
-    const RT_DIR: &str = "runtime";
-    let conf_dir = config_dir().join(RT_DIR);
-    if conf_dir.exists() {
-        return conf_dir;
-    }
+// pub use tendril::StrTendril as Tendril;
+pub use smartstring::SmartString;
 
-    if let Ok(dir) = std::env::var("CARGO_MANIFEST_DIR") {
-        // this is the directory of the crate being run by cargo, we need the workspace path so we take the parent
-        return std::path::PathBuf::from(dir).parent().unwrap().join(RT_DIR);
-    }
-
-    // fallback to location of the executable being run
-    std::env::current_exe()
-        .ok()
-        .and_then(|path| path.parent().map(|path| path.to_path_buf().join(RT_DIR)))
-        .unwrap()
-}
-
-pub fn config_dir() -> std::path::PathBuf {
-    // TODO: allow env var override
-    let strategy = choose_base_strategy().expect("Unable to find the config directory!");
-    let mut path = strategy.config_dir();
-    path.push("helix");
-    path
-}
-
-pub fn cache_dir() -> std::path::PathBuf {
-    // TODO: allow env var override
-    let strategy = choose_base_strategy().expect("Unable to find the config directory!");
-    let mut path = strategy.cache_dir();
-    path.push("helix");
-    path
-}
-
-// right overrides left
-pub fn merge_toml_values(left: toml::Value, right: toml::Value) -> toml::Value {
-    use toml::Value;
-
-    fn get_name(v: &Value) -> Option<&str> {
-        v.get("name").and_then(Value::as_str)
-    }
-
-    match (left, right) {
-        (Value::Array(mut left_items), Value::Array(right_items)) => {
-            left_items.reserve(right_items.len());
-            for rvalue in right_items {
-                let lvalue = get_name(&rvalue)
-                    .and_then(|rname| left_items.iter().position(|v| get_name(v) == Some(rname)))
-                    .map(|lpos| left_items.remove(lpos));
-                let mvalue = match lvalue {
-                    Some(lvalue) => merge_toml_values(lvalue, rvalue),
-                    None => rvalue,
-                };
-                left_items.push(mvalue);
-            }
-            Value::Array(left_items)
-        }
-        (Value::Table(mut left_map), Value::Table(right_map)) => {
-            for (rname, rvalue) in right_map {
-                match left_map.remove(&rname) {
-                    Some(lvalue) => {
-                        let merged_value = merge_toml_values(lvalue, rvalue);
-                        left_map.insert(rname, merged_value);
-                    }
-                    None => {
-                        left_map.insert(rname, rvalue);
-                    }
-                }
-            }
-            Value::Table(left_map)
-        }
-        // Catch everything else we didn't handle, and use the right value
-        (_, value) => value,
-    }
-}
-
-#[cfg(test)]
-mod merge_toml_tests {
-    use super::merge_toml_values;
-
-    #[test]
-    fn language_tomls() {
-        use toml::Value;
-
-        const USER: &str = "
-        [[language]]
-        name = \"nix\"
-        test = \"bbb\"
-        indent = { tab-width = 4, unit = \"    \", test = \"aaa\" }
-        ";
-
-        let base: Value = toml::from_slice(include_bytes!("../../languages.toml"))
-            .expect("Couldn't parse built-in langauges config");
-        let user: Value = toml::from_str(USER).unwrap();
-
-        let merged = merge_toml_values(base, user);
-        let languages = merged.get("language").unwrap().as_array().unwrap();
-        let nix = languages
-            .iter()
-            .find(|v| v.get("name").unwrap().as_str().unwrap() == "nix")
-            .unwrap();
-        let nix_indent = nix.get("indent").unwrap();
-
-        // We changed tab-width and unit in indent so check them if they are the new values
-        assert_eq!(
-            nix_indent.get("tab-width").unwrap().as_integer().unwrap(),
-            4
-        );
-        assert_eq!(nix_indent.get("unit").unwrap().as_str().unwrap(), "    ");
-        // We added a new keys, so check them
-        assert_eq!(nix.get("test").unwrap().as_str().unwrap(), "bbb");
-        assert_eq!(nix_indent.get("test").unwrap().as_str().unwrap(), "aaa");
-        // We didn't change comment-token so it should be same
-        assert_eq!(nix.get("comment-token").unwrap().as_str().unwrap(), "#");
-    }
-}
-
-pub use etcetera::home_dir;
-
-use etcetera::base_strategy::{choose_base_strategy, BaseStrategy};
-
-pub use ropey::{Rope, RopeBuilder, RopeSlice};
-
-pub use tendril::StrTendril as Tendril;
+pub type Tendril = SmartString<smartstring::LazyCompact>;
 
 #[doc(inline)]
 pub use {regex, tree_sitter};
 
 pub use graphemes::RopeGraphemes;
-pub use position::{coords_at_pos, pos_at_coords, Position};
+pub use position::{coords_at_pos, pos_at_coords, visual_coords_at_pos, Position};
 pub use selection::{Range, Selection};
-pub use smallvec::SmallVec;
+pub use smallvec::{smallvec, SmallVec};
 pub use syntax::Syntax;
 
 pub use diagnostic::Diagnostic;

helix-core/src/line_ending.rs

@@ -10,12 +10,18 @@ pub const DEFAULT_LINE_ENDING: LineEnding = LineEnding::LF;
 pub enum LineEnding {
     Crlf, // CarriageReturn followed by LineFeed
     LF,   // U+000A -- LineFeed
-    VT,   // U+000B -- VerticalTab
-    FF,   // U+000C -- FormFeed
-    CR,   // U+000D -- CarriageReturn
-    Nel,  // U+0085 -- NextLine
-    LS,   // U+2028 -- Line Separator
-    PS,   // U+2029 -- ParagraphSeparator
+    #[cfg(feature = "unicode-lines")]
+    VT, // U+000B -- VerticalTab
+    #[cfg(feature = "unicode-lines")]
+    FF, // U+000C -- FormFeed
+    #[cfg(feature = "unicode-lines")]
+    CR, // U+000D -- CarriageReturn
+    #[cfg(feature = "unicode-lines")]
+    Nel, // U+0085 -- NextLine
+    #[cfg(feature = "unicode-lines")]
+    LS, // U+2028 -- Line Separator
+    #[cfg(feature = "unicode-lines")]
+    PS, // U+2029 -- ParagraphSeparator
 }
 
 impl LineEnding {
@@ -32,11 +38,17 @@ impl LineEnding {
         match self {
             Self::Crlf => "\u{000D}\u{000A}",
             Self::LF => "\u{000A}",
+            #[cfg(feature = "unicode-lines")]
             Self::VT => "\u{000B}",
+            #[cfg(feature = "unicode-lines")]
             Self::FF => "\u{000C}",
+            #[cfg(feature = "unicode-lines")]
             Self::CR => "\u{000D}",
+            #[cfg(feature = "unicode-lines")]
             Self::Nel => "\u{0085}",
+            #[cfg(feature = "unicode-lines")]
             Self::LS => "\u{2028}",
+            #[cfg(feature = "unicode-lines")]
             Self::PS => "\u{2029}",
         }
     }
@@ -45,11 +57,17 @@ impl LineEnding {
     pub const fn from_char(ch: char) -> Option<LineEnding> {
         match ch {
             '\u{000A}' => Some(LineEnding::LF),
+            #[cfg(feature = "unicode-lines")]
             '\u{000B}' => Some(LineEnding::VT),
+            #[cfg(feature = "unicode-lines")]
             '\u{000C}' => Some(LineEnding::FF),
+            #[cfg(feature = "unicode-lines")]
             '\u{000D}' => Some(LineEnding::CR),
+            #[cfg(feature = "unicode-lines")]
             '\u{0085}' => Some(LineEnding::Nel),
+            #[cfg(feature = "unicode-lines")]
             '\u{2028}' => Some(LineEnding::LS),
+            #[cfg(feature = "unicode-lines")]
             '\u{2029}' => Some(LineEnding::PS),
             // Not a line ending
             _ => None,
@@ -65,11 +83,17 @@ impl LineEnding {
         match g {
             "\u{000D}\u{000A}" => Some(LineEnding::Crlf),
             "\u{000A}" => Some(LineEnding::LF),
+            #[cfg(feature = "unicode-lines")]
             "\u{000B}" => Some(LineEnding::VT),
+            #[cfg(feature = "unicode-lines")]
             "\u{000C}" => Some(LineEnding::FF),
+            #[cfg(feature = "unicode-lines")]
             "\u{000D}" => Some(LineEnding::CR),
+            #[cfg(feature = "unicode-lines")]
             "\u{0085}" => Some(LineEnding::Nel),
+            #[cfg(feature = "unicode-lines")]
             "\u{2028}" => Some(LineEnding::LS),
+            #[cfg(feature = "unicode-lines")]
             "\u{2029}" => Some(LineEnding::PS),
             // Not a line ending
             _ => None,
@@ -95,13 +119,20 @@ pub fn str_is_line_ending(s: &str) -> bool {
     LineEnding::from_str(s).is_some()
 }
 
+#[inline]
+pub fn rope_is_line_ending(r: RopeSlice) -> bool {
+    r.chunks().all(str_is_line_ending)
+}
+
 /// Attempts to detect what line ending the passed document uses.
 pub fn auto_detect_line_ending(doc: &Rope) -> Option<LineEnding> {
     // Return first matched line ending. Not all possible line endings
     // are being matched, as they might be special-use only
     for line in doc.lines().take(100) {
         match get_line_ending(&line) {
-            None | Some(LineEnding::VT) | Some(LineEnding::FF) | Some(LineEnding::PS) => {}
+            None => {}
+            #[cfg(feature = "unicode-lines")]
+            Some(LineEnding::VT) | Some(LineEnding::FF) | Some(LineEnding::PS) => {}
             ending => return ending,
         }
     }
@@ -128,6 +159,19 @@ pub fn get_line_ending(line: &RopeSlice) -> Option<LineEnding> {
     LineEnding::from_str(g2).or_else(|| LineEnding::from_str(g1))
 }
 
+#[cfg(not(feature = "unicode-lines"))]
+/// Returns the passed line's line ending, if any.
+pub fn get_line_ending_of_str(line: &str) -> Option<LineEnding> {
+    if line.ends_with("\u{000D}\u{000A}") {
+        Some(LineEnding::Crlf)
+    } else if line.ends_with('\u{000A}') {
+        Some(LineEnding::LF)
+    } else {
+        None
+    }
+}
+
+#[cfg(feature = "unicode-lines")]
 /// Returns the passed line's line ending, if any.
 pub fn get_line_ending_of_str(line: &str) -> Option<LineEnding> {
     if line.ends_with("\u{000D}\u{000A}") {
@@ -211,6 +255,7 @@ mod line_ending_tests {
 
     #[test]
     fn str_to_line_ending() {
+        #[cfg(feature = "unicode-lines")]
         assert_eq!(LineEnding::from_str("\r"), Some(LineEnding::CR));
         assert_eq!(LineEnding::from_str("\n"), Some(LineEnding::LF));
         assert_eq!(LineEnding::from_str("\r\n"), Some(LineEnding::Crlf));
@@ -220,6 +265,7 @@ mod line_ending_tests {
     #[test]
     fn rope_slice_to_line_ending() {
         let r = Rope::from_str("hello\r\n");
+        #[cfg(feature = "unicode-lines")]
         assert_eq!(
             LineEnding::from_rope_slice(&r.slice(5..6)),
             Some(LineEnding::CR)
@@ -238,6 +284,7 @@ mod line_ending_tests {
     #[test]
     fn get_line_ending_rope_slice() {
         let r = Rope::from_str("Hello\rworld\nhow\r\nare you?");
+        #[cfg(feature = "unicode-lines")]
         assert_eq!(get_line_ending(&r.slice(..6)), Some(LineEnding::CR));
         assert_eq!(get_line_ending(&r.slice(..12)), Some(LineEnding::LF));
         assert_eq!(get_line_ending(&r.slice(..17)), Some(LineEnding::Crlf));
@@ -247,19 +294,19 @@ mod line_ending_tests {
     #[test]
     fn get_line_ending_str() {
         let text = "Hello\rworld\nhow\r\nare you?";
+        #[cfg(feature = "unicode-lines")]
         assert_eq!(get_line_ending_of_str(&text[..6]), Some(LineEnding::CR));
         assert_eq!(get_line_ending_of_str(&text[..12]), Some(LineEnding::LF));
         assert_eq!(get_line_ending_of_str(&text[..17]), Some(LineEnding::Crlf));
-        assert_eq!(get_line_ending_of_str(&text[..]), None);
+        assert_eq!(get_line_ending_of_str(text), None);
     }
 
     #[test]
     fn line_end_char_index_rope_slice() {
         let r = Rope::from_str("Hello\rworld\nhow\r\nare you?");
         let s = &r.slice(..);
-        assert_eq!(line_end_char_index(s, 0), 5);
-        assert_eq!(line_end_char_index(s, 1), 11);
-        assert_eq!(line_end_char_index(s, 2), 15);
-        assert_eq!(line_end_char_index(s, 3), 25);
+        assert_eq!(line_end_char_index(s, 0), 11);
+        assert_eq!(line_end_char_index(s, 1), 15);
+        assert_eq!(line_end_char_index(s, 2), 25);
     }
 }

helix-core/src/match_brackets.rs

@@ -1,48 +1,92 @@
+use tree_sitter::Node;
+
 use crate::{Rope, Syntax};
 
-const PAIRS: &[(char, char)] = &[('(', ')'), ('{', '}'), ('[', ']'), ('<', '>')];
-// limit matching pairs to only ( ) { } [ ] < >
+const PAIRS: &[(char, char)] = &[
+    ('(', ')'),
+    ('{', '}'),
+    ('[', ']'),
+    ('<', '>'),
+    ('\'', '\''),
+    ('\"', '\"'),
+];
 
+// limit matching pairs to only ( ) { } [ ] < > ' ' " "
+
+// Returns the position of the matching bracket under cursor.
+//
+// If the cursor is one the opening bracket, the position of
+// the closing bracket is returned. If the cursor in the closing
+// bracket, the position of the opening bracket is returned.
+//
+// If the cursor is not on a bracket, `None` is returned.
 #[must_use]
-pub fn find(syntax: &Syntax, doc: &Rope, pos: usize) -> Option<usize> {
-    let tree = syntax.tree();
-
-    let byte_pos = doc.char_to_byte(pos);
-
-    // most naive implementation: find the innermost syntax node, if we're at the edge of a node,
-    // return the other edge.
-
-    let node = match tree
-        .root_node()
-        .named_descendant_for_byte_range(byte_pos, byte_pos)
-    {
-        Some(node) => node,
-        None => return None,
-    };
-
-    if node.is_error() {
+pub fn find_matching_bracket(syntax: &Syntax, doc: &Rope, pos: usize) -> Option<usize> {
+    if pos >= doc.len_chars() || !is_valid_bracket(doc.char(pos)) {
         return None;
     }
+    find_pair(syntax, doc, pos, false)
+}
 
+// Returns the position of the bracket that is closing the current scope.
+//
+// If the cursor is on an opening or closing bracket, the function
+// behaves equivalent to [`find_matching_bracket`].
+//
+// If the cursor position is within a scope, the function searches
+// for the surrounding scope that is surrounded by brackets and
+// returns the position of the closing bracket for that scope.
+//
+// If no surrounding scope is found, the function returns `None`.
+#[must_use]
+pub fn find_matching_bracket_fuzzy(syntax: &Syntax, doc: &Rope, pos: usize) -> Option<usize> {
+    find_pair(syntax, doc, pos, true)
+}
+
+fn find_pair(syntax: &Syntax, doc: &Rope, pos: usize, traverse_parents: bool) -> Option<usize> {
+    let tree = syntax.tree();
+    let pos = doc.char_to_byte(pos);
+
+    let mut node = tree.root_node().named_descendant_for_byte_range(pos, pos)?;
+
+    loop {
+        let (start_byte, end_byte) = surrounding_bytes(doc, &node)?;
+        let (start_char, end_char) = (doc.byte_to_char(start_byte), doc.byte_to_char(end_byte));
+
+        if is_valid_pair(doc, start_char, end_char) {
+            if end_byte == pos {
+                return Some(start_char);
+            }
+            // We return the end char if the cursor is either on the start char
+            // or at some arbitrary position between start and end char.
+            return Some(end_char);
+        }
+
+        if traverse_parents {
+            node = node.parent()?;
+        } else {
+            return None;
+        }
+    }
+}
+
+fn is_valid_bracket(c: char) -> bool {
+    PAIRS.iter().any(|(l, r)| *l == c || *r == c)
+}
+
+fn is_valid_pair(doc: &Rope, start_char: usize, end_char: usize) -> bool {
+    PAIRS.contains(&(doc.char(start_char), doc.char(end_char)))
+}
+
+fn surrounding_bytes(doc: &Rope, node: &Node) -> Option<(usize, usize)> {
     let len = doc.len_bytes();
+
     let start_byte = node.start_byte();
-    let end_byte = node.end_byte().saturating_sub(1); // it's end exclusive
+    let end_byte = node.end_byte().saturating_sub(1);
+
     if start_byte >= len || end_byte >= len {
         return None;
     }
 
-    let start_char = doc.byte_to_char(start_byte);
-    let end_char = doc.byte_to_char(end_byte);
-
-    if PAIRS.contains(&(doc.char(start_char), doc.char(end_char))) {
-        if start_byte == byte_pos {
-            return Some(end_char);
-        }
-
-        if end_byte == byte_pos {
-            return Some(start_char);
-        }
-    }
-
-    None
+    Some((start_byte, end_byte))
 }

helix-core/src/movement.rs

@@ -1,6 +1,7 @@
 use std::iter;
 
 use ropey::iter::Chars;
+use tree_sitter::{Node, QueryCursor};
 
 use crate::{
     chars::{categorize_char, char_is_line_ending, CharCategory},
@@ -9,7 +10,11 @@ use crate::{
         next_grapheme_boundary, nth_next_grapheme_boundary, nth_prev_grapheme_boundary,
         prev_grapheme_boundary,
     },
-    pos_at_coords, Position, Range, RopeSlice,
+    line_ending::rope_is_line_ending,
+    pos_at_coords,
+    syntax::LanguageConfiguration,
+    textobject::TextObject,
+    Position, Range, RopeSlice,
 };
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
@@ -53,6 +58,10 @@ pub fn move_vertically(
     let pos = range.cursor(slice);
 
     // Compute the current position's 2d coordinates.
+    // TODO: switch this to use `visual_coords_at_pos` rather than
+    // `coords_at_pos` as this will cause a jerky movement when the visual
+    // position does not match, like moving from a line with tabs/CJK to
+    // a line without
     let Position { row, col } = coords_at_pos(slice, pos);
     let horiz = range.horiz.unwrap_or(col as u32);
 
@@ -141,6 +150,88 @@ fn word_move(slice: RopeSlice, range: Range, count: usize, target: WordMotionTar
     })
 }
 
+pub fn move_prev_paragraph(
+    slice: RopeSlice,
+    range: Range,
+    count: usize,
+    behavior: Movement,
+) -> Range {
+    let mut line = range.cursor_line(slice);
+    let first_char = slice.line_to_char(line) == range.cursor(slice);
+    let prev_line_empty = rope_is_line_ending(slice.line(line.saturating_sub(1)));
+    let curr_line_empty = rope_is_line_ending(slice.line(line));
+    let prev_empty_to_line = prev_line_empty && !curr_line_empty;
+
+    // skip character before paragraph boundary
+    if prev_empty_to_line && !first_char {
+        line += 1;
+    }
+    let mut lines = slice.lines_at(line);
+    lines.reverse();
+    let mut lines = lines.map(rope_is_line_ending).peekable();
+    for _ in 0..count {
+        while lines.next_if(|&e| e).is_some() {
+            line -= 1;
+        }
+        while lines.next_if(|&e| !e).is_some() {
+            line -= 1;
+        }
+    }
+
+    let head = slice.line_to_char(line);
+    let anchor = if behavior == Movement::Move {
+        // exclude first character after paragraph boundary
+        if prev_empty_to_line && first_char {
+            range.cursor(slice)
+        } else {
+            range.head
+        }
+    } else {
+        range.put_cursor(slice, head, true).anchor
+    };
+    Range::new(anchor, head)
+}
+
+pub fn move_next_paragraph(
+    slice: RopeSlice,
+    range: Range,
+    count: usize,
+    behavior: Movement,
+) -> Range {
+    let mut line = range.cursor_line(slice);
+    let last_char =
+        prev_grapheme_boundary(slice, slice.line_to_char(line + 1)) == range.cursor(slice);
+    let curr_line_empty = rope_is_line_ending(slice.line(line));
+    let next_line_empty =
+        rope_is_line_ending(slice.line(slice.len_lines().saturating_sub(1).min(line + 1)));
+    let curr_empty_to_line = curr_line_empty && !next_line_empty;
+
+    // skip character after paragraph boundary
+    if curr_empty_to_line && last_char {
+        line += 1;
+    }
+    let mut lines = slice.lines_at(line).map(rope_is_line_ending).peekable();
+    for _ in 0..count {
+        while lines.next_if(|&e| !e).is_some() {
+            line += 1;
+        }
+        while lines.next_if(|&e| e).is_some() {
+            line += 1;
+        }
+    }
+    let head = slice.line_to_char(line);
+    let anchor = if behavior == Movement::Move {
+        if curr_empty_to_line && last_char {
+            range.head
+        } else {
+            range.cursor(slice)
+        }
+    } else {
+        range.put_cursor(slice, head, true).anchor
+    };
+    Range::new(anchor, head)
+}
+
 // ---- util ------------
 
 #[inline]
@@ -164,7 +255,7 @@ pub fn backwards_skip_while<F>(slice: RopeSlice, pos: usize, fun: F) -> Option<u
 where
     F: Fn(char) -> bool,
 {
-    let mut chars_starting_from_next = slice.chars_at(pos + 1);
+    let mut chars_starting_from_next = slice.chars_at(pos);
     let mut backwards = iter::from_fn(|| chars_starting_from_next.prev()).enumerate();
     backwards.find_map(|(i, c)| {
         if !fun(c) {
@@ -301,10 +392,58 @@ fn reached_target(target: WordMotionTarget, prev_ch: char, next_ch: char) -> boo
     }
 }
 
+pub fn goto_treesitter_object(
+    slice: RopeSlice,
+    range: Range,
+    object_name: &str,
+    dir: Direction,
+    slice_tree: Node,
+    lang_config: &LanguageConfiguration,
+    _count: usize,
+) -> Range {
+    let get_range = move || -> Option<Range> {
+        let byte_pos = slice.char_to_byte(range.cursor(slice));
+
+        let cap_name = |t: TextObject| format!("{}.{}", object_name, t);
+        let mut cursor = QueryCursor::new();
+        let nodes = lang_config.textobject_query()?.capture_nodes_any(
+            &[
+                &cap_name(TextObject::Movement),
+                &cap_name(TextObject::Around),
+                &cap_name(TextObject::Inside),
+            ],
+            slice_tree,
+            slice,
+            &mut cursor,
+        )?;
+
+        let node = match dir {
+            Direction::Forward => nodes
+                .filter(|n| n.start_byte() > byte_pos)
+                .min_by_key(|n| n.start_byte())?,
+            Direction::Backward => nodes
+                .filter(|n| n.start_byte() < byte_pos)
+                .max_by_key(|n| n.start_byte())?,
+        };
+
+        let len = slice.len_bytes();
+        let start_byte = node.start_byte();
+        let end_byte = node.end_byte();
+        if start_byte >= len || end_byte >= len {
+            return None;
+        }
+
+        let start_char = slice.byte_to_char(start_byte);
+        let end_char = slice.byte_to_char(end_byte);
+
+        // head of range should be at beginning
+        Some(Range::new(end_char, start_char))
+    };
+    get_range().unwrap_or(range)
+}
+
 #[cfg(test)]
 mod test {
-    use std::array::{self, IntoIter};
-
     use ropey::Rope;
 
     use super::*;
@@ -356,7 +495,7 @@ mod test {
             ((Direction::Backward, 999usize), (0, 0)), // |This is a simple alphabetic line
         ];
 
-        for ((direction, amount), coordinates) in IntoIter::new(moves_and_expected_coordinates) {
+        for ((direction, amount), coordinates) in moves_and_expected_coordinates {
             range = move_horizontally(slice, range, direction, amount, Movement::Move);
             assert_eq!(coords_at_pos(slice, range.head), coordinates.into())
         }
@@ -370,7 +509,7 @@ mod test {
 
         let mut range = Range::point(position);
 
-        let moves_and_expected_coordinates = IntoIter::new([
+        let moves_and_expected_coordinates = [
             ((Direction::Forward, 11usize), (1, 1)), // Multiline\nt|ext sample\n...
             ((Direction::Backward, 1usize), (1, 0)), // Multiline\n|text sample\n...
             ((Direction::Backward, 5usize), (0, 5)), // Multi|line\ntext sample\n...
@@ -380,7 +519,7 @@ mod test {
             ((Direction::Backward, 0usize), (0, 3)), // Mul|tiline\ntext sample\n...
             ((Direction::Forward, 999usize), (5, 0)), // ...and whitespaced\n|
             ((Direction::Forward, 999usize), (5, 0)), // ...and whitespaced\n|
-        ]);
+        ];
 
         for ((direction, amount), coordinates) in moves_and_expected_coordinates {
             range = move_horizontally(slice, range, direction, amount, Movement::Move);
@@ -398,11 +537,11 @@ mod test {
         let mut range = Range::point(position);
         let original_anchor = range.anchor;
 
-        let moves = IntoIter::new([
+        let moves = [
             (Direction::Forward, 1usize),
             (Direction::Forward, 5usize),
             (Direction::Backward, 3usize),
-        ]);
+        ];
 
         for (direction, amount) in moves {
             range = move_horizontally(slice, range, direction, amount, Movement::Extend);
@@ -416,7 +555,7 @@ mod test {
         let slice = text.slice(..);
         let position = pos_at_coords(slice, (0, 0).into(), true);
         let mut range = Range::point(position);
-        let moves_and_expected_coordinates = IntoIter::new([
+        let moves_and_expected_coordinates = [
             ((Direction::Forward, 1usize), (1, 0)),
             ((Direction::Forward, 2usize), (3, 0)),
             ((Direction::Forward, 1usize), (4, 0)),
@@ -426,7 +565,7 @@ mod test {
             ((Direction::Backward, 0usize), (4, 0)),
             ((Direction::Forward, 5), (5, 0)),
             ((Direction::Forward, 999usize), (5, 0)),
-        ]);
+        ];
 
         for ((direction, amount), coordinates) in moves_and_expected_coordinates {
             range = move_vertically(slice, range, direction, amount, Movement::Move);
@@ -446,7 +585,7 @@ mod test {
             H,
             V,
         }
-        let moves_and_expected_coordinates = IntoIter::new([
+        let moves_and_expected_coordinates = [
             // Places cursor at the end of line
             ((Axis::H, Direction::Forward, 8usize), (0, 8)),
             // First descent preserves column as the target line is wider
@@ -459,7 +598,7 @@ mod test {
             ((Axis::V, Direction::Backward, 999usize), (0, 8)),
             ((Axis::V, Direction::Forward, 4usize), (4, 8)),
             ((Axis::V, Direction::Forward, 999usize), (5, 0)),
-        ]);
+        ];
 
         for ((axis, direction, amount), coordinates) in moves_and_expected_coordinates {
             range = match axis {
@@ -485,7 +624,7 @@ mod test {
             H,
             V,
         }
-        let moves_and_expected_coordinates = IntoIter::new([
+        let moves_and_expected_coordinates = [
             // Places cursor at the fourth kana.
             ((Axis::H, Direction::Forward, 4), (0, 4)),
             // Descent places cursor at the 4th character.
@@ -494,7 +633,7 @@ mod test {
             ((Axis::H, Direction::Backward, 1usize), (1, 3)),
             // Jumping back up 1 line.
             ((Axis::V, Direction::Backward, 1usize), (0, 3)),
-        ]);
+        ];
 
         for ((axis, direction, amount), coordinates) in moves_and_expected_coordinates {
             range = match axis {
@@ -526,7 +665,7 @@ mod test {
 
     #[test]
     fn test_behaviour_when_moving_to_start_of_next_words() {
-        let tests = array::IntoIter::new([
+        let tests = [
             ("Basic forward motion stops at the first space",
                 vec![(1, Range::new(0, 0), Range::new(0, 6))]),
             (" Starting from a boundary advances the anchor",
@@ -600,7 +739,7 @@ mod test {
                 vec![
                     (1, Range::new(0, 0), Range::new(0, 6)),
                 ]),
-        ]);
+        ];
 
         for (sample, scenario) in tests {
             for (count, begin, expected_end) in scenario.into_iter() {
@@ -612,7 +751,7 @@ mod test {
 
     #[test]
     fn test_behaviour_when_moving_to_start_of_next_long_words() {
-        let tests = array::IntoIter::new([
+        let tests = [
             ("Basic forward motion stops at the first space",
                 vec![(1, Range::new(0, 0), Range::new(0, 6))]),
             (" Starting from a boundary advances the anchor",
@@ -684,7 +823,7 @@ mod test {
                 vec![
                     (1, Range::new(0, 0), Range::new(0, 8)),
                 ]),
-        ]);
+        ];
 
         for (sample, scenario) in tests {
             for (count, begin, expected_end) in scenario.into_iter() {
@@ -696,7 +835,7 @@ mod test {
 
     #[test]
     fn test_behaviour_when_moving_to_start_of_previous_words() {
-        let tests = array::IntoIter::new([
+        let tests = [
             ("Basic backward motion from the middle of a word",
                 vec![(1, Range::new(3, 3), Range::new(4, 0))]),
 
@@ -769,7 +908,7 @@ mod test {
                 vec![
                     (1, Range::new(0, 6), Range::new(6, 0)),
                 ]),
-        ]);
+        ];
 
         for (sample, scenario) in tests {
             for (count, begin, expected_end) in scenario.into_iter() {
@@ -781,7 +920,7 @@ mod test {
 
     #[test]
     fn test_behaviour_when_moving_to_start_of_previous_long_words() {
-        let tests = array::IntoIter::new([
+        let tests = [
             (
                 "Basic backward motion from the middle of a word",
                 vec![(1, Range::new(3, 3), Range::new(4, 0))],
@@ -866,7 +1005,7 @@ mod test {
                 vec![
                     (1, Range::new(0, 8), Range::new(8, 0)),
                 ]),
-        ]);
+        ];
 
         for (sample, scenario) in tests {
             for (count, begin, expected_end) in scenario.into_iter() {
@@ -878,7 +1017,7 @@ mod test {
 
     #[test]
     fn test_behaviour_when_moving_to_end_of_next_words() {
-        let tests = array::IntoIter::new([
+        let tests = [
             ("Basic forward motion from the start of a word to the end of it",
                 vec![(1, Range::new(0, 0), Range::new(0, 5))]),
             ("Basic forward motion from the end of a word to the end of the next",
@@ -950,7 +1089,7 @@ mod test {
                 vec![
                     (1, Range::new(0, 0), Range::new(0, 5)),
                 ]),
-        ]);
+        ];
 
         for (sample, scenario) in tests {
             for (count, begin, expected_end) in scenario.into_iter() {
@@ -962,7 +1101,7 @@ mod test {
 
     #[test]
     fn test_behaviour_when_moving_to_end_of_previous_words() {
-        let tests = array::IntoIter::new([
+        let tests = [
             ("Basic backward motion from the middle of a word",
                 vec![(1, Range::new(9, 9), Range::new(10, 5))]),
             ("Starting from after boundary retreats the anchor",
@@ -1032,7 +1171,7 @@ mod test {
                 vec![
                     (1, Range::new(0, 10), Range::new(10, 4)),
                 ]),
-        ]);
+        ];
 
         for (sample, scenario) in tests {
             for (count, begin, expected_end) in scenario.into_iter() {
@@ -1044,7 +1183,7 @@ mod test {
 
     #[test]
     fn test_behaviour_when_moving_to_end_of_next_long_words() {
-        let tests = array::IntoIter::new([
+        let tests = [
             ("Basic forward motion from the start of a word to the end of it",
                 vec![(1, Range::new(0, 0), Range::new(0, 5))]),
             ("Basic forward motion from the end of a word to the end of the next",
@@ -1114,7 +1253,7 @@ mod test {
                 vec![
                     (1, Range::new(0, 0), Range::new(0, 7)),
                 ]),
-        ]);
+        ];
 
         for (sample, scenario) in tests {
             for (count, begin, expected_end) in scenario.into_iter() {
@@ -1123,4 +1262,172 @@ mod test {
             }
         }
     }
+
+    #[test]
+    fn test_behaviour_when_moving_to_prev_paragraph_single() {
+        let tests = [
+            ("#[|]#", "#[|]#"),
+            ("#[s|]#tart at\nfirst char\n", "#[|s]#tart at\nfirst char\n"),
+            ("start at\nlast char#[\n|]#", "#[|start at\nlast char\n]#"),
+            (
+                "goto\nfirst\n\n#[p|]#aragraph",
+                "#[|goto\nfirst\n\n]#paragraph",
+            ),
+            (
+                "goto\nfirst\n#[\n|]#paragraph",
+                "#[|goto\nfirst\n\n]#paragraph",
+            ),
+            (
+                "goto\nsecond\n\np#[a|]#ragraph",
+                "goto\nsecond\n\n#[|pa]#ragraph",
+            ),
+            (
+                "here\n\nhave\nmultiple\nparagraph\n\n\n\n\n#[|]#",
+                "here\n\n#[|have\nmultiple\nparagraph\n\n\n\n\n]#",
+            ),
+        ];
+
+        for (before, expected) in tests {
+            let (s, selection) = crate::test::print(before);
+            let text = Rope::from(s.as_str());
+            let selection =
+                selection.transform(|r| move_prev_paragraph(text.slice(..), r, 1, Movement::Move));
+            let actual = crate::test::plain(&s, selection);
+            assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
+        }
+    }
+
+    #[test]
+    fn test_behaviour_when_moving_to_prev_paragraph_double() {
+        let tests = [
+            (
+                "on#[e|]#\n\ntwo\n\nthree\n\n",
+                "#[|one]#\n\ntwo\n\nthree\n\n",
+            ),
+            (
+                "one\n\ntwo\n\nth#[r|]#ee\n\n",
+                "one\n\n#[|two\n\nthr]#ee\n\n",
+            ),
+        ];
+
+        for (before, expected) in tests {
+            let (s, selection) = crate::test::print(before);
+            let text = Rope::from(s.as_str());
+            let selection =
+                selection.transform(|r| move_prev_paragraph(text.slice(..), r, 2, Movement::Move));
+            let actual = crate::test::plain(&s, selection);
+            assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
+        }
+    }
+
+    #[test]
+    fn test_behaviour_when_moving_to_prev_paragraph_extend() {
+        let tests = [
+            (
+                "one\n\n#[|two\n\n]#three\n\n",
+                "#[|one\n\ntwo\n\n]#three\n\n",
+            ),
+            (
+                "#[|one\n\ntwo\n\n]#three\n\n",
+                "#[|one\n\ntwo\n\n]#three\n\n",
+            ),
+        ];
+
+        for (before, expected) in tests {
+            let (s, selection) = crate::test::print(before);
+            let text = Rope::from(s.as_str());
+            let selection = selection
+                .transform(|r| move_prev_paragraph(text.slice(..), r, 1, Movement::Extend));
+            let actual = crate::test::plain(&s, selection);
+            assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
+        }
+    }
+
+    #[test]
+    fn test_behaviour_when_moving_to_next_paragraph_single() {
+        let tests = [
+            ("#[|]#", "#[|]#"),
+            ("#[s|]#tart at\nfirst char\n", "#[start at\nfirst char\n|]#"),
+            ("start at\nlast char#[\n|]#", "start at\nlast char#[\n|]#"),
+            (
+                "a\nb\n\n#[g|]#oto\nthird\n\nparagraph",
+                "a\nb\n\n#[goto\nthird\n\n|]#paragraph",
+            ),
+            (
+                "a\nb\n#[\n|]#goto\nthird\n\nparagraph",
+                "a\nb\n\n#[goto\nthird\n\n|]#paragraph",
+            ),
+            (
+                "a\nb#[\n|]#\ngoto\nsecond\n\nparagraph",
+                "a\nb#[\n\n|]#goto\nsecond\n\nparagraph",
+            ),
+            (
+                "here\n\nhave\n#[m|]#ultiple\nparagraph\n\n\n\n\n",
+                "here\n\nhave\n#[multiple\nparagraph\n\n\n\n\n|]#",
+            ),
+            (
+                "#[t|]#ext\n\n\nafter two blank lines\n\nmore text\n",
+                "#[text\n\n\n|]#after two blank lines\n\nmore text\n",
+            ),
+            (
+                "#[text\n\n\n|]#after two blank lines\n\nmore text\n",
+                "text\n\n\n#[after two blank lines\n\n|]#more text\n",
+            ),
+        ];
+
+        for (before, expected) in tests {
+            let (s, selection) = crate::test::print(before);
+            let text = Rope::from(s.as_str());
+            let selection =
+                selection.transform(|r| move_next_paragraph(text.slice(..), r, 1, Movement::Move));
+            let actual = crate::test::plain(&s, selection);
+            assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
+        }
+    }
+
+    #[test]
+    fn test_behaviour_when_moving_to_next_paragraph_double() {
+        let tests = [
+            (
+                "one\n\ntwo\n\nth#[r|]#ee\n\n",
+                "one\n\ntwo\n\nth#[ree\n\n|]#",
+            ),
+            (
+                "on#[e|]#\n\ntwo\n\nthree\n\n",
+                "on#[e\n\ntwo\n\n|]#three\n\n",
+            ),
+        ];
+
+        for (before, expected) in tests {
+            let (s, selection) = crate::test::print(before);
+            let text = Rope::from(s.as_str());
+            let selection =
+                selection.transform(|r| move_next_paragraph(text.slice(..), r, 2, Movement::Move));
+            let actual = crate::test::plain(&s, selection);
+            assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
+        }
+    }
+
+    #[test]
+    fn test_behaviour_when_moving_to_next_paragraph_extend() {
+        let tests = [
+            (
+                "one\n\n#[two\n\n|]#three\n\n",
+                "one\n\n#[two\n\nthree\n\n|]#",
+            ),
+            (
+                "one\n\n#[two\n\nthree\n\n|]#",
+                "one\n\n#[two\n\nthree\n\n|]#",
+            ),
+        ];
+
+        for (before, expected) in tests {
+            let (s, selection) = crate::test::print(before);
+            let text = Rope::from(s.as_str());
+            let selection = selection
+                .transform(|r| move_next_paragraph(text.slice(..), r, 1, Movement::Extend));
+            let actual = crate::test::plain(&s, selection);
+            assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
+        }
+    }
 }

helix-core/src/object.rs

@@ -1,26 +1,72 @@
 use crate::{Range, RopeSlice, Selection, Syntax};
+use tree_sitter::Node;
 
-// TODO: to contract_selection we'd need to store the previous ranges before expand.
-// Maybe just contract to the first child node?
-pub fn expand_selection(syntax: &Syntax, text: RopeSlice, selection: &Selection) -> Selection {
+pub fn expand_selection(syntax: &Syntax, text: RopeSlice, selection: Selection) -> Selection {
+    select_node_impl(syntax, text, selection, |descendant, from, to| {
+        if descendant.start_byte() == from && descendant.end_byte() == to {
+            descendant.parent()
+        } else {
+            Some(descendant)
+        }
+    })
+}
+
+pub fn shrink_selection(syntax: &Syntax, text: RopeSlice, selection: Selection) -> Selection {
+    select_node_impl(syntax, text, selection, |descendant, _from, _to| {
+        descendant.child(0).or(Some(descendant))
+    })
+}
+
+pub fn select_sibling<F>(
+    syntax: &Syntax,
+    text: RopeSlice,
+    selection: Selection,
+    sibling_fn: &F,
+) -> Selection
+where
+    F: Fn(Node) -> Option<Node>,
+{
+    select_node_impl(syntax, text, selection, |descendant, _from, _to| {
+        find_sibling_recursive(descendant, sibling_fn)
+    })
+}
+
+fn find_sibling_recursive<F>(node: Node, sibling_fn: F) -> Option<Node>
+where
+    F: Fn(Node) -> Option<Node>,
+{
+    sibling_fn(node).or_else(|| {
+        node.parent()
+            .and_then(|node| find_sibling_recursive(node, sibling_fn))
+    })
+}
+
+fn select_node_impl<F>(
+    syntax: &Syntax,
+    text: RopeSlice,
+    selection: Selection,
+    select_fn: F,
+) -> Selection
+where
+    F: Fn(Node, usize, usize) -> Option<Node>,
+{
     let tree = syntax.tree();
 
-    selection.clone().transform(|range| {
+    selection.transform(|range| {
         let from = text.char_to_byte(range.from());
         let to = text.char_to_byte(range.to());
 
-        // find parent of a descendant that matches the range
-        let parent = match tree
+        let node = match tree
             .root_node()
             .descendant_for_byte_range(from, to)
-            .and_then(|node| node.parent())
+            .and_then(|node| select_fn(node, from, to))
         {
-            Some(parent) => parent,
+            Some(node) => node,
             None => return range,
         };
 
-        let from = text.byte_to_char(parent.start_byte());
-        let to = text.byte_to_char(parent.end_byte());
+        let from = text.byte_to_char(node.start_byte());
+        let to = text.byte_to_char(node.end_byte());
 
         if range.head < range.anchor {
             Range::new(to, from)

helix-core/src/path.rs

@@ -1,9 +1,10 @@
+use etcetera::home_dir;
 use std::path::{Component, Path, PathBuf};
 
 /// Replaces users home directory from `path` with tilde `~` if the directory
 /// is available, otherwise returns the path unchanged.
 pub fn fold_home_dir(path: &Path) -> PathBuf {
-    if let Ok(home) = super::home_dir() {
+    if let Ok(home) = home_dir() {
         if path.starts_with(&home) {
             // it's ok to unwrap, the path starts with home dir
             return PathBuf::from("~").join(path.strip_prefix(&home).unwrap());
@@ -13,14 +14,14 @@ pub fn fold_home_dir(path: &Path) -> PathBuf {
     path.to_path_buf()
 }
 
-/// Expands tilde `~` into users home directory if avilable, otherwise returns the path
+/// Expands tilde `~` into users home directory if available, otherwise returns the path
 /// unchanged. The tilde will only be expanded when present as the first component of the path
 /// and only slash follows it.
 pub fn expand_tilde(path: &Path) -> PathBuf {
     let mut components = path.components().peekable();
     if let Some(Component::Normal(c)) = components.peek() {
         if c == &"~" {
-            if let Ok(home) = super::home_dir() {
+            if let Ok(home) = home_dir() {
                 // it's ok to unwrap, the path starts with `~`
                 return home.join(path.strip_prefix("~").unwrap());
             }
@@ -40,7 +41,6 @@ pub fn expand_tilde(path: &Path) -> PathBuf {
 /// needs to improve on.
 /// Copied from cargo: <https://github.com/rust-lang/cargo/blob/070e459c2d8b79c5b2ac5218064e7603329c92ae/crates/cargo-util/src/paths.rs#L81>
 pub fn get_normalized_path(path: &Path) -> PathBuf {
-    let path = expand_tilde(path);
     let mut components = path.components().peekable();
     let mut ret = if let Some(c @ Component::Prefix(..)) = components.peek().cloned() {
         components.next();
@@ -72,10 +72,11 @@ pub fn get_normalized_path(path: &Path) -> PathBuf {
 /// This function is used instead of `std::fs::canonicalize` because we don't want to verify
 /// here if the path exists, just normalize it's components.
 pub fn get_canonicalized_path(path: &Path) -> std::io::Result<PathBuf> {
+    let path = expand_tilde(path);
     let path = if path.is_relative() {
         std::env::current_dir().map(|current_dir| current_dir.join(path))?
     } else {
-        path.to_path_buf()
+        path
     };
 
     Ok(get_normalized_path(path.as_path()))

helix-core/src/position.rs

@@ -1,6 +1,8 @@
+use std::borrow::Cow;
+
 use crate::{
     chars::char_is_line_ending,
-    graphemes::{ensure_grapheme_boundary_prev, RopeGraphemes},
+    graphemes::{ensure_grapheme_boundary_prev, grapheme_width, RopeGraphemes},
     line_ending::line_end_char_index,
     RopeSlice,
 };
@@ -54,11 +56,8 @@ impl From<Position> for tree_sitter::Point {
 }
 /// Convert a character index to (line, column) coordinates.
 ///
-/// TODO: this should be split into two methods: one for visual
-/// row/column, and one for "objective" row/column (possibly with
-/// the column specified in `char`s).  The former would be used
-/// for cursor movement, and the latter would be used for e.g. the
-/// row:column display in the status line.
+/// column in `char` count which can be used for row:column display in
+/// status line. See [`visual_coords_at_pos`] for a visual one.
 pub fn coords_at_pos(text: RopeSlice, pos: usize) -> Position {
     let line = text.char_to_line(pos);
 
@@ -69,6 +68,31 @@ pub fn coords_at_pos(text: RopeSlice, pos: usize) -> Position {
     Position::new(line, col)
 }
 
+/// Convert a character index to (line, column) coordinates visually.
+///
+/// Takes \t, double-width characters (CJK) into account as well as text
+/// not in the document in the future.
+/// See [`coords_at_pos`] for an "objective" one.
+pub fn visual_coords_at_pos(text: RopeSlice, pos: usize, tab_width: usize) -> Position {
+    let line = text.char_to_line(pos);
+
+    let line_start = text.line_to_char(line);
+    let pos = ensure_grapheme_boundary_prev(text, pos);
+
+    let mut col = 0;
+
+    for grapheme in RopeGraphemes::new(text.slice(line_start..pos)) {
+        if grapheme == "\t" {
+            col += tab_width - (col % tab_width);
+        } else {
+            let grapheme = Cow::from(grapheme);
+            col += grapheme_width(&grapheme);
+        }
+    }
+
+    Position::new(line, col)
+}
+
 /// Convert (line, column) coordinates to a character index.
 ///
 /// If the `line` coordinate is beyond the end of the file, the EOF
@@ -89,7 +113,10 @@ pub fn coords_at_pos(text: RopeSlice, pos: usize) -> Position {
 /// TODO: this should be changed to work in terms of visual row/column, not
 /// graphemes.
 pub fn pos_at_coords(text: RopeSlice, coords: Position, limit_before_line_ending: bool) -> usize {
-    let Position { row, col } = coords;
+    let Position { mut row, col } = coords;
+    if limit_before_line_ending {
+        row = row.min(text.len_lines() - 1);
+    };
     let line_start = text.line_to_char(row);
     let line_end = if limit_before_line_ending {
         line_end_char_index(&text, row)
@@ -130,7 +157,6 @@ mod test {
         assert_eq!(coords_at_pos(slice, 10), (1, 4).into()); // position on d
 
         // Test with wide characters.
-        // TODO: account for character width.
         let text = Rope::from("今日はいい\n");
         let slice = text.slice(..);
         assert_eq!(coords_at_pos(slice, 0), (0, 0).into());
@@ -151,7 +177,6 @@ mod test {
         assert_eq!(coords_at_pos(slice, 9), (1, 0).into());
 
         // Test with wide-character grapheme clusters.
-        // TODO: account for character width.
         let text = Rope::from("किमपि\n");
         let slice = text.slice(..);
         assert_eq!(coords_at_pos(slice, 0), (0, 0).into());
@@ -161,7 +186,6 @@ mod test {
         assert_eq!(coords_at_pos(slice, 6), (1, 0).into());
 
         // Test with tabs.
-        // Todo: account for tab stops.
         let text = Rope::from("\tHello\n");
         let slice = text.slice(..);
         assert_eq!(coords_at_pos(slice, 0), (0, 0).into());
@@ -169,6 +193,54 @@ mod test {
         assert_eq!(coords_at_pos(slice, 2), (0, 2).into());
     }
 
+    #[test]
+    fn test_visual_coords_at_pos() {
+        let text = Rope::from("ḧëḷḷö\nẅöṛḷḋ");
+        let slice = text.slice(..);
+        assert_eq!(visual_coords_at_pos(slice, 0, 8), (0, 0).into());
+        assert_eq!(visual_coords_at_pos(slice, 5, 8), (0, 5).into()); // position on \n
+        assert_eq!(visual_coords_at_pos(slice, 6, 8), (1, 0).into()); // position on w
+        assert_eq!(visual_coords_at_pos(slice, 7, 8), (1, 1).into()); // position on o
+        assert_eq!(visual_coords_at_pos(slice, 10, 8), (1, 4).into()); // position on d
+
+        // Test with wide characters.
+        let text = Rope::from("今日はいい\n");
+        let slice = text.slice(..);
+        assert_eq!(visual_coords_at_pos(slice, 0, 8), (0, 0).into());
+        assert_eq!(visual_coords_at_pos(slice, 1, 8), (0, 2).into());
+        assert_eq!(visual_coords_at_pos(slice, 2, 8), (0, 4).into());
+        assert_eq!(visual_coords_at_pos(slice, 3, 8), (0, 6).into());
+        assert_eq!(visual_coords_at_pos(slice, 4, 8), (0, 8).into());
+        assert_eq!(visual_coords_at_pos(slice, 5, 8), (0, 10).into());
+        assert_eq!(visual_coords_at_pos(slice, 6, 8), (1, 0).into());
+
+        // Test with grapheme clusters.
+        let text = Rope::from("a̐éö̲\r\n");
+        let slice = text.slice(..);
+        assert_eq!(visual_coords_at_pos(slice, 0, 8), (0, 0).into());
+        assert_eq!(visual_coords_at_pos(slice, 2, 8), (0, 1).into());
+        assert_eq!(visual_coords_at_pos(slice, 4, 8), (0, 2).into());
+        assert_eq!(visual_coords_at_pos(slice, 7, 8), (0, 3).into());
+        assert_eq!(visual_coords_at_pos(slice, 9, 8), (1, 0).into());
+
+        // Test with wide-character grapheme clusters.
+        // TODO: account for cluster.
+        let text = Rope::from("किमपि\n");
+        let slice = text.slice(..);
+        assert_eq!(visual_coords_at_pos(slice, 0, 8), (0, 0).into());
+        assert_eq!(visual_coords_at_pos(slice, 2, 8), (0, 2).into());
+        assert_eq!(visual_coords_at_pos(slice, 3, 8), (0, 3).into());
+        assert_eq!(visual_coords_at_pos(slice, 5, 8), (0, 5).into());
+        assert_eq!(visual_coords_at_pos(slice, 6, 8), (1, 0).into());
+
+        // Test with tabs.
+        let text = Rope::from("\tHello\n");
+        let slice = text.slice(..);
+        assert_eq!(visual_coords_at_pos(slice, 0, 8), (0, 0).into());
+        assert_eq!(visual_coords_at_pos(slice, 1, 8), (0, 8).into());
+        assert_eq!(visual_coords_at_pos(slice, 2, 8), (0, 9).into());
+    }
+
     #[test]
     fn test_pos_at_coords() {
         let text = Rope::from("ḧëḷḷö\nẅöṛḷḋ");
@@ -225,5 +297,12 @@ mod test {
         assert_eq!(pos_at_coords(slice, (0, 0).into(), false), 0);
         assert_eq!(pos_at_coords(slice, (0, 1).into(), false), 1);
         assert_eq!(pos_at_coords(slice, (0, 2).into(), false), 2);
+
+        // Test out of bounds.
+        let text = Rope::new();
+        let slice = text.slice(..);
+        assert_eq!(pos_at_coords(slice, (10, 0).into(), true), 0);
+        assert_eq!(pos_at_coords(slice, (0, 10).into(), true), 0);
+        assert_eq!(pos_at_coords(slice, (10, 10).into(), true), 0);
     }
 }

helix-core/src/register.rs

@@ -15,7 +15,11 @@ impl Register {
     }
 
     pub fn new_with_values(name: char, values: Vec<String>) -> Self {
-        Self { name, values }
+        if name == '_' {
+            Self::new(name)
+        } else {
+            Self { name, values }
+        }
     }
 
     pub const fn name(&self) -> char {
@@ -27,11 +31,15 @@ impl Register {
     }
 
     pub fn write(&mut self, values: Vec<String>) {
-        self.values = values;
+        if self.name != '_' {
+            self.values = values;
+        }
     }
 
     pub fn push(&mut self, value: String) {
-        self.values.push(value);
+        if self.name != '_' {
+            self.values.push(value);
+        }
     }
 }
 
@@ -60,4 +68,8 @@ impl Registers {
     pub fn read(&self, name: char) -> Option<&[String]> {
         self.get(name).map(|reg| reg.read())
     }
+
+    pub fn inner(&self) -> &HashMap<char, Register> {
+        &self.inner
+    }
 }

helix-core/src/search.rs

@@ -1,6 +1,28 @@
 use crate::RopeSlice;
 
-pub fn find_nth_next(text: RopeSlice, ch: char, mut pos: usize, n: usize) -> Option<usize> {
+// TODO: switch to std::str::Pattern when it is stable.
+pub trait CharMatcher {
+    fn char_match(&self, ch: char) -> bool;
+}
+
+impl CharMatcher for char {
+    fn char_match(&self, ch: char) -> bool {
+        *self == ch
+    }
+}
+
+impl<F: Fn(&char) -> bool> CharMatcher for F {
+    fn char_match(&self, ch: char) -> bool {
+        (*self)(&ch)
+    }
+}
+
+pub fn find_nth_next<M: CharMatcher>(
+    text: RopeSlice,
+    char_matcher: M,
+    mut pos: usize,
+    n: usize,
+) -> Option<usize> {
     if pos >= text.len_chars() || n == 0 {
         return None;
     }
@@ -13,7 +35,7 @@ pub fn find_nth_next(text: RopeSlice, ch: char, mut pos: usize, n: usize) -> Opt
 
             pos += 1;
 
-            if c == ch {
+            if char_matcher.char_match(c) {
                 break;
             }
         }

helix-core/src/selection.rs

@@ -7,6 +7,7 @@ use crate::{
         ensure_grapheme_boundary_next, ensure_grapheme_boundary_prev, next_grapheme_boundary,
         prev_grapheme_boundary,
     },
+    movement::Direction,
     Assoc, ChangeSet, RopeSlice,
 };
 use smallvec::{smallvec, SmallVec};
@@ -82,6 +83,13 @@ impl Range {
         std::cmp::max(self.anchor, self.head)
     }
 
+    /// Total length of the range.
+    #[inline]
+    #[must_use]
+    pub fn len(&self) -> usize {
+        self.to() - self.from()
+    }
+
     /// The (inclusive) range of lines that the range overlaps.
     #[inline]
     #[must_use]
@@ -102,6 +110,27 @@ impl Range {
         self.anchor == self.head
     }
 
+    /// `Direction::Backward` when head < anchor.
+    /// `Direction::Backward` otherwise.
+    #[inline]
+    #[must_use]
+    pub fn direction(&self) -> Direction {
+        if self.head < self.anchor {
+            Direction::Backward
+        } else {
+            Direction::Forward
+        }
+    }
+
+    // flips the direction of the selection
+    pub fn flip(&self) -> Self {
+        Self {
+            anchor: self.head,
+            head: self.anchor,
+            horiz: self.horiz,
+        }
+    }
+
     /// Check two ranges for overlap.
     #[must_use]
     pub fn overlaps(&self, other: &Self) -> bool {
@@ -111,6 +140,11 @@ impl Range {
         self.from() == other.from() || (self.to() > other.from() && other.to() > self.from())
     }
 
+    #[inline]
+    pub fn contains_range(&self, other: &Self) -> bool {
+        self.from() <= other.from() && self.to() >= other.to()
+    }
+
     pub fn contains(&self, pos: usize) -> bool {
         self.from() <= pos && pos < self.to()
     }
@@ -308,10 +342,10 @@ impl Range {
 }
 
 impl From<(usize, usize)> for Range {
-    fn from(tuple: (usize, usize)) -> Self {
+    fn from((anchor, head): (usize, usize)) -> Self {
         Self {
-            anchor: tuple.0,
-            head: tuple.1,
+            anchor,
+            head,
             horiz: None,
         }
     }
@@ -360,8 +394,13 @@ impl Selection {
         self.normalize()
     }
 
-    /// Adds a new range to the selection and makes it the primary range.
+    /// Removes a range from the selection.
     pub fn remove(mut self, index: usize) -> Self {
+        assert!(
+            self.ranges.len() > 1,
+            "can't remove the last range from a selection!"
+        );
+
         self.ranges.remove(index);
         if index < self.primary_index || self.primary_index == self.ranges.len() {
             self.primary_index -= 1;
@@ -369,6 +408,12 @@ impl Selection {
         self
     }
 
+    /// Replace a range in the selection with a new range.
+    pub fn replace(mut self, index: usize, range: Range) -> Self {
+        self.ranges[index] = range;
+        self.normalize()
+    }
+
     /// Map selections over a set of changes. Useful for adjusting the selection position after
     /// applying changes to a document.
     pub fn map(self, changes: &ChangeSet) -> Self {
@@ -504,6 +549,39 @@ impl Selection {
     pub fn len(&self) -> usize {
         self.ranges.len()
     }
+
+    // returns true if self ⊇ other
+    pub fn contains(&self, other: &Selection) -> bool {
+        // can't contain other if it is larger
+        if other.len() > self.len() {
+            return false;
+        }
+
+        let (mut iter_self, mut iter_other) = (self.iter(), other.iter());
+        let (mut ele_self, mut ele_other) = (iter_self.next(), iter_other.next());
+
+        loop {
+            match (ele_self, ele_other) {
+                (Some(ra), Some(rb)) => {
+                    if !ra.contains_range(rb) {
+                        // `self` doesn't contain next element from `other`, advance `self`, we need to match all from `other`
+                        ele_self = iter_self.next();
+                    } else {
+                        // matched element from `other`, advance `other`
+                        ele_other = iter_other.next();
+                    };
+                }
+                (None, Some(_)) => {
+                    // exhausted `self`, we can't match the reminder of `other`
+                    return false;
+                }
+                (_, None) => {
+                    // no elements from `other` left to match, `self` contains `other`
+                    return true;
+                }
+            }
+        }
+    }
 }
 
 impl<'a> IntoIterator for &'a Selection {
@@ -517,14 +595,15 @@ impl<'a> IntoIterator for &'a Selection {
 
 // TODO: checkSelection -> check if valid for doc length && sorted
 
-pub fn keep_matches(
+pub fn keep_or_remove_matches(
     text: RopeSlice,
     selection: &Selection,
     regex: &crate::regex::Regex,
+    remove: bool,
 ) -> Option<Selection> {
     let result: SmallVec<_> = selection
         .iter()
-        .filter(|range| regex.is_match(&range.fragment(text)))
+        .filter(|range| regex.is_match(&range.fragment(text)) ^ remove)
         .copied()
         .collect();
 
@@ -687,16 +766,16 @@ mod test {
     fn test_contains() {
         let range = Range::new(10, 12);
 
-        assert_eq!(range.contains(9), false);
-        assert_eq!(range.contains(10), true);
-        assert_eq!(range.contains(11), true);
-        assert_eq!(range.contains(12), false);
-        assert_eq!(range.contains(13), false);
+        assert!(!range.contains(9));
+        assert!(range.contains(10));
+        assert!(range.contains(11));
+        assert!(!range.contains(12));
+        assert!(!range.contains(13));
 
         let range = Range::new(9, 6);
-        assert_eq!(range.contains(9), false);
-        assert_eq!(range.contains(7), true);
-        assert_eq!(range.contains(6), true);
+        assert!(!range.contains(9));
+        assert!(range.contains(7));
+        assert!(range.contains(6));
     }
 
     #[test]
@@ -941,4 +1020,30 @@ mod test {
             &["", "abcd", "efg", "rs", "xyz"]
         );
     }
+    #[test]
+    fn test_selection_contains() {
+        fn contains(a: Vec<(usize, usize)>, b: Vec<(usize, usize)>) -> bool {
+            let sela = Selection::new(a.iter().map(|a| Range::new(a.0, a.1)).collect(), 0);
+            let selb = Selection::new(b.iter().map(|b| Range::new(b.0, b.1)).collect(), 0);
+            sela.contains(&selb)
+        }
+
+        // exact match
+        assert!(contains(vec!((1, 1)), vec!((1, 1))));
+
+        // larger set contains smaller
+        assert!(contains(vec!((1, 1), (2, 2), (3, 3)), vec!((2, 2))));
+
+        // multiple matches
+        assert!(contains(vec!((1, 1), (2, 2)), vec!((1, 1), (2, 2))));
+
+        // smaller set can't contain bigger
+        assert!(!contains(vec!((1, 1)), vec!((1, 1), (2, 2))));
+
+        assert!(contains(
+            vec!((1, 1), (2, 4), (5, 6), (7, 9), (10, 13)),
+            vec!((3, 4), (7, 9))
+        ));
+        assert!(!contains(vec!((1, 1), (5, 6)), vec!((1, 6))));
+    }
 }

helix-core/src/shellwords.rs

@@ -0,0 +1,164 @@
+use std::borrow::Cow;
+
+/// Get the vec of escaped / quoted / doublequoted filenames from the input str
+pub fn shellwords(input: &str) -> Vec<Cow<'_, str>> {
+    enum State {
+        Normal,
+        NormalEscaped,
+        Quoted,
+        QuoteEscaped,
+        Dquoted,
+        DquoteEscaped,
+    }
+
+    use State::*;
+
+    let mut state = Normal;
+    let mut args: Vec<Cow<str>> = Vec::new();
+    let mut escaped = String::with_capacity(input.len());
+
+    let mut start = 0;
+    let mut end = 0;
+
+    for (i, c) in input.char_indices() {
+        state = match state {
+            Normal => match c {
+                '\\' => {
+                    escaped.push_str(&input[start..i]);
+                    start = i + 1;
+                    NormalEscaped
+                }
+                '"' => {
+                    end = i;
+                    Dquoted
+                }
+                '\'' => {
+                    end = i;
+                    Quoted
+                }
+                c if c.is_ascii_whitespace() => {
+                    end = i;
+                    Normal
+                }
+                _ => Normal,
+            },
+            NormalEscaped => Normal,
+            Quoted => match c {
+                '\\' => {
+                    escaped.push_str(&input[start..i]);
+                    start = i + 1;
+                    QuoteEscaped
+                }
+                '\'' => {
+                    end = i;
+                    Normal
+                }
+                _ => Quoted,
+            },
+            QuoteEscaped => Quoted,
+            Dquoted => match c {
+                '\\' => {
+                    escaped.push_str(&input[start..i]);
+                    start = i + 1;
+                    DquoteEscaped
+                }
+                '"' => {
+                    end = i;
+                    Normal
+                }
+                _ => Dquoted,
+            },
+            DquoteEscaped => Dquoted,
+        };
+
+        if i >= input.len() - 1 && end == 0 {
+            end = i + 1;
+        }
+
+        if end > 0 {
+            let esc_trim = escaped.trim();
+            let inp = &input[start..end];
+
+            if !(esc_trim.is_empty() && inp.trim().is_empty()) {
+                if esc_trim.is_empty() {
+                    args.push(inp.into());
+                } else {
+                    args.push([escaped, inp.into()].concat().into());
+                    escaped = "".to_string();
+                }
+            }
+            start = i + 1;
+            end = 0;
+        }
+    }
+    args
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_normal() {
+        let input = r#":o single_word twó wörds \three\ \"with\ escaping\\"#;
+        let result = shellwords(input);
+        let expected = vec![
+            Cow::from(":o"),
+            Cow::from("single_word"),
+            Cow::from("twó"),
+            Cow::from("wörds"),
+            Cow::from(r#"three "with escaping\"#),
+        ];
+        // TODO test is_owned and is_borrowed, once they get stabilized.
+        assert_eq!(expected, result);
+    }
+
+    #[test]
+    fn test_quoted() {
+        let quoted =
+            r#":o 'single_word' 'twó wörds' '' ' ''\three\' \"with\ escaping\\' 'quote incomplete"#;
+        let result = shellwords(quoted);
+        let expected = vec![
+            Cow::from(":o"),
+            Cow::from("single_word"),
+            Cow::from("twó wörds"),
+            Cow::from(r#"three' "with escaping\"#),
+            Cow::from("quote incomplete"),
+        ];
+        assert_eq!(expected, result);
+    }
+
+    #[test]
+    fn test_dquoted() {
+        let dquoted = r#":o "single_word" "twó wörds" "" "  ""\three\' \"with\ escaping\\" "dquote incomplete"#;
+        let result = shellwords(dquoted);
+        let expected = vec![
+            Cow::from(":o"),
+            Cow::from("single_word"),
+            Cow::from("twó wörds"),
+            Cow::from(r#"three' "with escaping\"#),
+            Cow::from("dquote incomplete"),
+        ];
+        assert_eq!(expected, result);
+    }
+
+    #[test]
+    fn test_mixed() {
+        let dquoted = r#":o single_word 'twó wörds' "\three\' \"with\ escaping\\""no space before"'and after' $#%^@ "%^&(%^" ')(*&^%''a\\\\\b' '"#;
+        let result = shellwords(dquoted);
+        let expected = vec![
+            Cow::from(":o"),
+            Cow::from("single_word"),
+            Cow::from("twó wörds"),
+            Cow::from("three' \"with escaping\\"),
+            Cow::from("no space before"),
+            Cow::from("and after"),
+            Cow::from("$#%^@"),
+            Cow::from("%^&(%^"),
+            Cow::from(")(*&^%"),
+            Cow::from(r#"a\\b"#),
+            //last ' just changes to quoted but since we dont have anything after it, it should be ignored
+        ];
+        assert_eq!(expected, result);
+    }
+}

helix-core/src/surround.rs

@@ -1,4 +1,6 @@
-use crate::{search, Selection};
+use std::fmt::Display;
+
+use crate::{search, Range, Selection};
 use ropey::RopeSlice;
 
 pub const PAIRS: &[(char, char)] = &[
@@ -11,6 +13,27 @@ pub const PAIRS: &[(char, char)] = &[
     ('（', '）'),
 ];
 
+#[derive(Debug, PartialEq)]
+pub enum Error {
+    PairNotFound,
+    CursorOverlap,
+    RangeExceedsText,
+    CursorOnAmbiguousPair,
+}
+
+impl Display for Error {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(match *self {
+            Error::PairNotFound => "Surround pair not found around all cursors",
+            Error::CursorOverlap => "Cursors overlap for a single surround pair range",
+            Error::RangeExceedsText => "Cursor range exceeds text length",
+            Error::CursorOnAmbiguousPair => "Cursor on ambiguous surround pair",
+        })
+    }
+}
+
+type Result<T> = std::result::Result<T, Error>;
+
 /// Given any char in [PAIRS], return the open and closing chars. If not found in
 /// [PAIRS] return (ch, ch).
 ///
@@ -29,45 +52,83 @@ pub fn get_pair(ch: char) -> (char, char) {
         .unwrap_or((ch, ch))
 }
 
+pub fn find_nth_closest_pairs_pos(
+    text: RopeSlice,
+    range: Range,
+    n: usize,
+) -> Result<(usize, usize)> {
+    let is_open_pair = |ch| PAIRS.iter().any(|(open, _)| *open == ch);
+    let is_close_pair = |ch| PAIRS.iter().any(|(_, close)| *close == ch);
+
+    let mut stack = Vec::with_capacity(2);
+    let pos = range.cursor(text);
+
+    for ch in text.chars_at(pos) {
+        if is_open_pair(ch) {
+            // Track open pairs encountered so that we can step over
+            // the correspoding close pairs that will come up further
+            // down the loop. We want to find a lone close pair whose
+            // open pair is before the cursor position.
+            stack.push(ch);
+            continue;
+        } else if is_close_pair(ch) {
+            let (open, _) = get_pair(ch);
+            if stack.last() == Some(&open) {
+                stack.pop();
+                continue;
+            } else {
+                // In the ideal case the stack would be empty here and the
+                // current character would be the close pair that we are
+                // looking for. It could also be the case that the pairs
+                // are unbalanced and we encounter a close pair that doesn't
+                // close the last seen open pair. In either case use this
+                // char as the auto-detected closest pair.
+                return find_nth_pairs_pos(text, ch, range, n);
+            }
+        }
+    }
+
+    Err(Error::PairNotFound)
+}
+
 /// Find the position of surround pairs of `ch` which can be either a closing
 /// or opening pair. `n` will skip n - 1 pairs (eg. n=2 will discard (only)
 /// the first pair found and keep looking)
 pub fn find_nth_pairs_pos(
     text: RopeSlice,
     ch: char,
-    pos: usize,
+    range: Range,
     n: usize,
-) -> Option<(usize, usize)> {
+) -> Result<(usize, usize)> {
+    if text.len_chars() < 2 {
+        return Err(Error::PairNotFound);
+    }
+    if range.to() >= text.len_chars() {
+        return Err(Error::RangeExceedsText);
+    }
+
     let (open, close) = get_pair(ch);
+    let pos = range.cursor(text);
 
-    if text.len_chars() < 2 || pos >= text.len_chars() {
-        return None;
-    }
-
-    if open == close {
+    let (open, close) = if open == close {
         if Some(open) == text.get_char(pos) {
-            // Special case: cursor is directly on a matching char.
-            match pos {
-                0 => Some((pos, search::find_nth_next(text, close, pos + 1, n)?)),
-                _ if (pos + 1) == text.len_chars() => {
-                    Some((search::find_nth_prev(text, open, pos, n)?, pos))
-                }
-                // We return no match because there's no way to know which
-                // side of the char we should be searching on.
-                _ => None,
-            }
-        } else {
-            Some((
-                search::find_nth_prev(text, open, pos, n)?,
-                search::find_nth_next(text, close, pos, n)?,
-            ))
+            // Cursor is directly on match char. We return no match
+            // because there's no way to know which side of the char
+            // we should be searching on.
+            return Err(Error::CursorOnAmbiguousPair);
         }
+        (
+            search::find_nth_prev(text, open, pos, n),
+            search::find_nth_next(text, close, pos, n),
+        )
     } else {
-        Some((
-            find_nth_open_pair(text, open, close, pos, n)?,
-            find_nth_close_pair(text, open, close, pos, n)?,
-        ))
-    }
+        (
+            find_nth_open_pair(text, open, close, pos, n),
+            find_nth_close_pair(text, open, close, pos, n),
+        )
+    };
+
+    Option::zip(open, close).ok_or(Error::PairNotFound)
 }
 
 fn find_nth_open_pair(
@@ -151,23 +212,28 @@ fn find_nth_close_pair(
 /// Find position of surround characters around every cursor. Returns None
 /// if any positions overlap. Note that the positions are in a flat Vec.
 /// Use get_surround_pos().chunks(2) to get matching pairs of surround positions.
-/// `ch` can be either closing or opening pair.
+/// `ch` can be either closing or opening pair. If `ch` is None, surround pairs
+/// are automatically detected around each cursor (note that this may result
+/// in them selecting different surround characters for each selection).
 pub fn get_surround_pos(
     text: RopeSlice,
     selection: &Selection,
-    ch: char,
+    ch: Option<char>,
     skip: usize,
-) -> Option<Vec<usize>> {
+) -> Result<Vec<usize>> {
     let mut change_pos = Vec::new();
 
-    for range in selection {
-        let (open_pos, close_pos) = find_nth_pairs_pos(text, ch, range.head, skip)?;
+    for &range in selection {
+        let (open_pos, close_pos) = match ch {
+            Some(ch) => find_nth_pairs_pos(text, ch, range, skip)?,
+            None => find_nth_closest_pairs_pos(text, range, skip)?,
+        };
         if change_pos.contains(&open_pos) || change_pos.contains(&close_pos) {
-            return None;
+            return Err(Error::CursorOverlap);
         }
         change_pos.extend_from_slice(&[open_pos, close_pos]);
     }
-    Some(change_pos)
+    Ok(change_pos)
 }
 
 #[cfg(test)]
@@ -178,67 +244,92 @@ mod test {
     use ropey::Rope;
     use smallvec::SmallVec;
 
-    #[test]
-    fn test_find_nth_pairs_pos() {
-        let doc = Rope::from("some (text) here");
+    #[allow(clippy::type_complexity)]
+    fn check_find_nth_pair_pos(
+        text: &str,
+        cases: Vec<(usize, char, usize, Result<(usize, usize)>)>,
+    ) {
+        let doc = Rope::from(text);
         let slice = doc.slice(..);
 
-        // cursor on [t]ext
-        assert_eq!(find_nth_pairs_pos(slice, '(', 6, 1), Some((5, 10)));
-        assert_eq!(find_nth_pairs_pos(slice, ')', 6, 1), Some((5, 10)));
-        // cursor on so[m]e
-        assert_eq!(find_nth_pairs_pos(slice, '(', 2, 1), None);
-        // cursor on bracket itself
-        assert_eq!(find_nth_pairs_pos(slice, '(', 5, 1), Some((5, 10)));
-        assert_eq!(find_nth_pairs_pos(slice, '(', 10, 1), Some((5, 10)));
+        for (cursor_pos, ch, n, expected_range) in cases {
+            let range = find_nth_pairs_pos(slice, ch, (cursor_pos, cursor_pos + 1).into(), n);
+            assert_eq!(
+                range, expected_range,
+                "Expected {:?}, got {:?}",
+                expected_range, range
+            );
+        }
+    }
+
+    #[test]
+    fn test_find_nth_pairs_pos() {
+        check_find_nth_pair_pos(
+            "some (text) here",
+            vec![
+                // cursor on [t]ext
+                (6, '(', 1, Ok((5, 10))),
+                (6, ')', 1, Ok((5, 10))),
+                // cursor on so[m]e
+                (2, '(', 1, Err(Error::PairNotFound)),
+                // cursor on bracket itself
+                (5, '(', 1, Ok((5, 10))),
+                (10, '(', 1, Ok((5, 10))),
+            ],
+        );
     }
 
     #[test]
     fn test_find_nth_pairs_pos_skip() {
-        let doc = Rope::from("(so (many (good) text) here)");
-        let slice = doc.slice(..);
-
-        // cursor on go[o]d
-        assert_eq!(find_nth_pairs_pos(slice, '(', 13, 1), Some((10, 15)));
-        assert_eq!(find_nth_pairs_pos(slice, '(', 13, 2), Some((4, 21)));
-        assert_eq!(find_nth_pairs_pos(slice, '(', 13, 3), Some((0, 27)));
+        check_find_nth_pair_pos(
+            "(so (many (good) text) here)",
+            vec![
+                // cursor on go[o]d
+                (13, '(', 1, Ok((10, 15))),
+                (13, '(', 2, Ok((4, 21))),
+                (13, '(', 3, Ok((0, 27))),
+            ],
+        );
     }
 
     #[test]
     fn test_find_nth_pairs_pos_same() {
-        let doc = Rope::from("'so 'many 'good' text' here'");
-        let slice = doc.slice(..);
-
-        // cursor on go[o]d
-        assert_eq!(find_nth_pairs_pos(slice, '\'', 13, 1), Some((10, 15)));
-        assert_eq!(find_nth_pairs_pos(slice, '\'', 13, 2), Some((4, 21)));
-        assert_eq!(find_nth_pairs_pos(slice, '\'', 13, 3), Some((0, 27)));
-        // cursor on the quotes
-        assert_eq!(find_nth_pairs_pos(slice, '\'', 10, 1), None);
-        // this is the best we can do since opening and closing pairs are same
-        assert_eq!(find_nth_pairs_pos(slice, '\'', 0, 1), Some((0, 4)));
-        assert_eq!(find_nth_pairs_pos(slice, '\'', 27, 1), Some((21, 27)));
+        check_find_nth_pair_pos(
+            "'so 'many 'good' text' here'",
+            vec![
+                // cursor on go[o]d
+                (13, '\'', 1, Ok((10, 15))),
+                (13, '\'', 2, Ok((4, 21))),
+                (13, '\'', 3, Ok((0, 27))),
+                // cursor on the quotes
+                (10, '\'', 1, Err(Error::CursorOnAmbiguousPair)),
+            ],
+        )
     }
 
     #[test]
     fn test_find_nth_pairs_pos_step() {
-        let doc = Rope::from("((so)((many) good (text))(here))");
-        let slice = doc.slice(..);
-
-        // cursor on go[o]d
-        assert_eq!(find_nth_pairs_pos(slice, '(', 15, 1), Some((5, 24)));
-        assert_eq!(find_nth_pairs_pos(slice, '(', 15, 2), Some((0, 31)));
+        check_find_nth_pair_pos(
+            "((so)((many) good (text))(here))",
+            vec![
+                // cursor on go[o]d
+                (15, '(', 1, Ok((5, 24))),
+                (15, '(', 2, Ok((0, 31))),
+            ],
+        )
     }
 
     #[test]
     fn test_find_nth_pairs_pos_mixed() {
-        let doc = Rope::from("(so [many {good} text] here)");
-        let slice = doc.slice(..);
-
-        // cursor on go[o]d
-        assert_eq!(find_nth_pairs_pos(slice, '{', 13, 1), Some((10, 15)));
-        assert_eq!(find_nth_pairs_pos(slice, '[', 13, 1), Some((4, 21)));
-        assert_eq!(find_nth_pairs_pos(slice, '(', 13, 1), Some((0, 27)));
+        check_find_nth_pair_pos(
+            "(so [many {good} text] here)",
+            vec![
+                // cursor on go[o]d
+                (13, '{', 1, Ok((10, 15))),
+                (13, '[', 1, Ok((4, 21))),
+                (13, '(', 1, Ok((0, 27))),
+            ],
+        )
     }
 
     #[test]
@@ -252,7 +343,7 @@ mod test {
 
         // cursor on s[o]me, c[h]ars, newl[i]ne
         assert_eq!(
-            get_surround_pos(slice, &selection, '(', 1)
+            get_surround_pos(slice, &selection, Some('('), 1)
                 .unwrap()
                 .as_slice(),
             &[0, 5, 7, 13, 15, 23]
@@ -266,11 +357,10 @@ mod test {
 
         let selection =
             Selection::new(SmallVec::from_slice(&[Range::point(2), Range::point(9)]), 0);
-
         // cursor on s[o]me, c[h]ars
         assert_eq!(
-            get_surround_pos(slice, &selection, '(', 1),
-            None // different surround chars
+            get_surround_pos(slice, &selection, Some('('), 1),
+            Err(Error::PairNotFound) // different surround chars
         );
 
         let selection = Selection::new(
@@ -279,8 +369,16 @@ mod test {
         );
         // cursor on [x]x, newli[n]e
         assert_eq!(
-            get_surround_pos(slice, &selection, '(', 1),
-            None // overlapping surround chars
+            get_surround_pos(slice, &selection, Some('('), 1),
+            Err(Error::PairNotFound) // overlapping surround chars
+        );
+
+        let selection =
+            Selection::new(SmallVec::from_slice(&[Range::point(2), Range::point(3)]), 0);
+        // cursor on s[o][m]e
+        assert_eq!(
+            get_surround_pos(slice, &selection, Some('['), 1),
+            Err(Error::CursorOverlap)
         );
     }
 }

helix-core/src/test.rs

@@ -0,0 +1,143 @@
+//! Test helpers.
+use crate::{Range, Selection};
+use smallvec::SmallVec;
+use std::cmp::Reverse;
+
+/// Convert annotated test string to test string and selection.
+///
+/// `#[|` for primary selection with head before anchor followed by `]#`.
+/// `#(|` for secondary selection with head before anchor followed by `)#`.
+/// `#[` for primary selection with head after anchor followed by `|]#`.
+/// `#(` for secondary selection with head after anchor followed by `|)#`.
+///
+/// # Examples
+///
+/// ```
+/// use helix_core::{Range, Selection, test::print};
+/// use smallvec::smallvec;
+///
+/// assert_eq!(
+///     print("#[a|]#b#(|c)#"),
+///     ("abc".to_owned(), Selection::new(smallvec![Range::new(0, 1), Range::new(3, 2)], 0))
+/// );
+/// ```
+///
+/// # Panics
+///
+/// Panics when missing primary or appeared more than once.
+/// Panics when missing head or anchor.
+/// Panics when head come after head or anchor come after anchor.
+pub fn print(s: &str) -> (String, Selection) {
+    let mut primary_idx = None;
+    let mut ranges = SmallVec::new();
+    let mut iter = s.chars().peekable();
+    let mut left = String::with_capacity(s.len());
+
+    'outer: while let Some(c) = iter.next() {
+        let start = left.len();
+
+        if c != '#' {
+            left.push(c);
+            continue;
+        }
+
+        let (is_primary, close_pair) = match iter.next() {
+            Some('[') => (true, ']'),
+            Some('(') => (false, ')'),
+            Some(ch) => {
+                left.push('#');
+                left.push(ch);
+                continue;
+            }
+            None => break,
+        };
+
+        if is_primary && primary_idx.is_some() {
+            panic!("primary `#[` already appeared {:?} {:?}", left, s);
+        }
+
+        let head_at_beg = iter.next_if_eq(&'|').is_some();
+
+        while let Some(c) = iter.next() {
+            if !(c == close_pair && iter.peek() == Some(&'#')) {
+                left.push(c);
+                continue;
+            }
+            if !head_at_beg {
+                let prev = left.pop().unwrap();
+                if prev != '|' {
+                    left.push(prev);
+                    left.push(c);
+                    continue;
+                }
+            }
+            iter.next(); // skip "#"
+
+            if is_primary {
+                primary_idx = Some(ranges.len());
+            }
+            let (anchor, head) = match head_at_beg {
+                true => (left.len(), start),
+                false => (start, left.len()),
+            };
+            ranges.push(Range::new(anchor, head));
+            continue 'outer;
+        }
+
+        if head_at_beg {
+            panic!("missing end `{}#` {:?} {:?}", close_pair, left, s);
+        } else {
+            panic!("missing end `|{}#` {:?} {:?}", close_pair, left, s);
+        }
+    }
+
+    let primary = match primary_idx {
+        Some(i) => i,
+        None => panic!("missing primary `#[|]#` {:?}", s),
+    };
+    let selection = Selection::new(ranges, primary);
+    (left, selection)
+}
+
+/// Convert test string and selection to annotated test string.
+///
+/// `#[|` for primary selection with head before anchor followed by `]#`.
+/// `#(|` for secondary selection with head before anchor followed by `)#`.
+/// `#[` for primary selection with head after anchor followed by `|]#`.
+/// `#(` for secondary selection with head after anchor followed by `|)#`.
+///
+/// # Examples
+///
+/// ```
+/// use helix_core::{Range, Selection, test::plain};
+/// use smallvec::smallvec;
+///
+/// assert_eq!(
+///     plain("abc", Selection::new(smallvec![Range::new(0, 1), Range::new(3, 2)], 0)),
+///     "#[a|]#b#(|c)#".to_owned()
+/// );
+/// ```
+pub fn plain(s: &str, selection: Selection) -> String {
+    let primary = selection.primary_index();
+    let mut out = String::with_capacity(s.len() + 5 * selection.len());
+    out.push_str(s);
+    let mut insertion: Vec<_> = selection
+        .iter()
+        .enumerate()
+        .flat_map(|(i, range)| {
+            // sort like this before reversed so anchor < head later
+            match (range.anchor < range.head, i == primary) {
+                (true, true) => [(range.anchor, "#["), (range.head, "|]#")],
+                (true, false) => [(range.anchor, "#("), (range.head, "|)#")],
+                (false, true) => [(range.anchor, "]#"), (range.head, "#[|")],
+                (false, false) => [(range.anchor, ")#"), (range.head, "#(|")],
+            }
+        })
+        .collect();
+    // insert in reverse order
+    insertion.sort_unstable_by_key(|k| Reverse(k.0));
+    for (i, s) in insertion {
+        out.insert_str(i, s);
+    }
+    out
+}

helix-core/src/textobject.rs

@@ -4,13 +4,14 @@ use ropey::RopeSlice;
 use tree_sitter::{Node, QueryCursor};
 
 use crate::chars::{categorize_char, char_is_whitespace, CharCategory};
-use crate::graphemes::next_grapheme_boundary;
+use crate::graphemes::{next_grapheme_boundary, prev_grapheme_boundary};
+use crate::line_ending::rope_is_line_ending;
 use crate::movement::Direction;
 use crate::surround;
 use crate::syntax::LanguageConfiguration;
 use crate::Range;
 
-fn find_word_boundary(slice: RopeSlice, mut pos: usize, direction: Direction) -> usize {
+fn find_word_boundary(slice: RopeSlice, mut pos: usize, direction: Direction, long: bool) -> usize {
     use CharCategory::{Eol, Whitespace};
 
     let iter = match direction {
@@ -33,7 +34,7 @@ fn find_word_boundary(slice: RopeSlice, mut pos: usize, direction: Direction) ->
         match categorize_char(ch) {
             Eol | Whitespace => return pos,
             category => {
-                if category != prev_category && pos != 0 && pos != slice.len_chars() {
+                if !long && category != prev_category && pos != 0 && pos != slice.len_chars() {
                     return pos;
                 } else {
                     match direction {
@@ -53,6 +54,8 @@ fn find_word_boundary(slice: RopeSlice, mut pos: usize, direction: Direction) ->
 pub enum TextObject {
     Around,
     Inside,
+    /// Used for moving between objects.
+    Movement,
 }
 
 impl Display for TextObject {
@@ -60,6 +63,7 @@ impl Display for TextObject {
         f.write_str(match self {
             Self::Around => "around",
             Self::Inside => "inside",
+            Self::Movement => "movement",
         })
     }
 }
@@ -70,13 +74,14 @@ pub fn textobject_word(
     range: Range,
     textobject: TextObject,
     _count: usize,
+    long: bool,
 ) -> Range {
     let pos = range.cursor(slice);
 
-    let word_start = find_word_boundary(slice, pos, Direction::Backward);
+    let word_start = find_word_boundary(slice, pos, Direction::Backward, long);
     let word_end = match slice.get_char(pos).map(categorize_char) {
         None | Some(CharCategory::Whitespace | CharCategory::Eol) => pos,
-        _ => find_word_boundary(slice, pos + 1, Direction::Forward),
+        _ => find_word_boundary(slice, pos + 1, Direction::Forward, long),
     };
 
     // Special case.
@@ -103,9 +108,96 @@ pub fn textobject_word(
                 Range::new(word_start - whitespace_count_left, word_end)
             }
         }
+        TextObject::Movement => unreachable!(),
     }
 }
 
+pub fn textobject_paragraph(
+    slice: RopeSlice,
+    range: Range,
+    textobject: TextObject,
+    count: usize,
+) -> Range {
+    let mut line = range.cursor_line(slice);
+    let prev_line_empty = rope_is_line_ending(slice.line(line.saturating_sub(1)));
+    let curr_line_empty = rope_is_line_ending(slice.line(line));
+    let next_line_empty = rope_is_line_ending(slice.line(line.saturating_sub(1)));
+    let last_char =
+        prev_grapheme_boundary(slice, slice.line_to_char(line + 1)) == range.cursor(slice);
+    let prev_empty_to_line = prev_line_empty && !curr_line_empty;
+    let curr_empty_to_line = curr_line_empty && !next_line_empty;
+
+    // skip character before paragraph boundary
+    let mut line_back = line; // line but backwards
+    if prev_empty_to_line || curr_empty_to_line {
+        line_back += 1;
+    }
+    // do not include current paragraph on paragraph end (include next)
+    if !(curr_empty_to_line && last_char) {
+        let mut lines = slice.lines_at(line_back);
+        lines.reverse();
+        let mut lines = lines.map(rope_is_line_ending).peekable();
+        while lines.next_if(|&e| e).is_some() {
+            line_back -= 1;
+        }
+        while lines.next_if(|&e| !e).is_some() {
+            line_back -= 1;
+        }
+    }
+
+    // skip character after paragraph boundary
+    if curr_empty_to_line && last_char {
+        line += 1;
+    }
+    let mut lines = slice.lines_at(line).map(rope_is_line_ending).peekable();
+    let mut count_done = 0; // count how many non-whitespace paragraphs done
+    for _ in 0..count {
+        let mut done = false;
+        while lines.next_if(|&e| !e).is_some() {
+            line += 1;
+            done = true;
+        }
+        while lines.next_if(|&e| e).is_some() {
+            line += 1;
+        }
+        count_done += done as usize;
+    }
+
+    // search one paragraph backwards for last paragraph
+    // makes `map` at the end of the paragraph with trailing newlines useful
+    let last_paragraph = count_done != count && lines.peek().is_none();
+    if last_paragraph {
+        let mut lines = slice.lines_at(line_back);
+        lines.reverse();
+        let mut lines = lines.map(rope_is_line_ending).peekable();
+        while lines.next_if(|&e| e).is_some() {
+            line_back -= 1;
+        }
+        while lines.next_if(|&e| !e).is_some() {
+            line_back -= 1;
+        }
+    }
+
+    // handle last whitespaces part separately depending on textobject
+    match textobject {
+        TextObject::Around => {}
+        TextObject::Inside => {
+            // remove last whitespace paragraph
+            let mut lines = slice.lines_at(line);
+            lines.reverse();
+            let mut lines = lines.map(rope_is_line_ending).peekable();
+            while lines.next_if(|&e| e).is_some() {
+                line -= 1;
+            }
+        }
+        TextObject::Movement => unreachable!(),
+    }
+
+    let anchor = slice.line_to_char(line_back);
+    let head = slice.line_to_char(line);
+    Range::new(anchor, head)
+}
+
 pub fn textobject_surround(
     slice: RopeSlice,
     range: Range,
@@ -113,10 +205,35 @@ pub fn textobject_surround(
     ch: char,
     count: usize,
 ) -> Range {
-    surround::find_nth_pairs_pos(slice, ch, range.head, count)
+    textobject_surround_impl(slice, range, textobject, Some(ch), count)
+}
+
+pub fn textobject_surround_closest(
+    slice: RopeSlice,
+    range: Range,
+    textobject: TextObject,
+    count: usize,
+) -> Range {
+    textobject_surround_impl(slice, range, textobject, None, count)
+}
+
+fn textobject_surround_impl(
+    slice: RopeSlice,
+    range: Range,
+    textobject: TextObject,
+    ch: Option<char>,
+    count: usize,
+) -> Range {
+    let pair_pos = match ch {
+        Some(ch) => surround::find_nth_pairs_pos(slice, ch, range, count),
+        // Automatically find the closest surround pairs
+        None => surround::find_nth_closest_pairs_pos(slice, range, count),
+    };
+    pair_pos
         .map(|(anchor, head)| match textobject {
             TextObject::Inside => Range::new(next_grapheme_boundary(slice, anchor), head),
             TextObject::Around => Range::new(anchor, next_grapheme_boundary(slice, head)),
+            TextObject::Movement => unreachable!(),
         })
         .unwrap_or(range)
 }
@@ -169,7 +286,7 @@ mod test {
 
     #[test]
     fn test_textobject_word() {
-        // (text, [(cursor position, textobject, final range), ...])
+        // (text, [(char position, textobject, final range), ...])
         let tests = &[
             (
                 "cursor at beginning of doc",
@@ -268,7 +385,9 @@ mod test {
             let slice = doc.slice(..);
             for &case in scenario {
                 let (pos, objtype, expected_range) = case;
-                let result = textobject_word(slice, Range::point(pos), objtype, 1);
+                // cursor is a single width selection
+                let range = Range::new(pos, pos + 1);
+                let result = textobject_word(slice, range, objtype, 1, false);
                 assert_eq!(
                     result,
                     expected_range.into(),
@@ -280,9 +399,94 @@ mod test {
         }
     }
 
+    #[test]
+    fn test_textobject_paragraph_inside_single() {
+        let tests = [
+            ("#[|]#", "#[|]#"),
+            ("firs#[t|]#\n\nparagraph\n\n", "#[first\n|]#\nparagraph\n\n"),
+            (
+                "second\n\npa#[r|]#agraph\n\n",
+                "second\n\n#[paragraph\n|]#\n",
+            ),
+            ("#[f|]#irst char\n\n", "#[first char\n|]#\n"),
+            ("last char\n#[\n|]#", "#[last char\n|]#\n"),
+            (
+                "empty to line\n#[\n|]#paragraph boundary\n\n",
+                "empty to line\n\n#[paragraph boundary\n|]#\n",
+            ),
+            (
+                "line to empty\n\n#[p|]#aragraph boundary\n\n",
+                "line to empty\n\n#[paragraph boundary\n|]#\n",
+            ),
+        ];
+
+        for (before, expected) in tests {
+            let (s, selection) = crate::test::print(before);
+            let text = Rope::from(s.as_str());
+            let selection = selection
+                .transform(|r| textobject_paragraph(text.slice(..), r, TextObject::Inside, 1));
+            let actual = crate::test::plain(&s, selection);
+            assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
+        }
+    }
+
+    #[test]
+    fn test_textobject_paragraph_inside_double() {
+        let tests = [
+            (
+                "last two\n\n#[p|]#aragraph\n\nwithout whitespaces\n\n",
+                "last two\n\n#[paragraph\n\nwithout whitespaces\n|]#\n",
+            ),
+            (
+                "last two\n#[\n|]#paragraph\n\nwithout whitespaces\n\n",
+                "last two\n\n#[paragraph\n\nwithout whitespaces\n|]#\n",
+            ),
+        ];
+
+        for (before, expected) in tests {
+            let (s, selection) = crate::test::print(before);
+            let text = Rope::from(s.as_str());
+            let selection = selection
+                .transform(|r| textobject_paragraph(text.slice(..), r, TextObject::Inside, 2));
+            let actual = crate::test::plain(&s, selection);
+            assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
+        }
+    }
+
+    #[test]
+    fn test_textobject_paragraph_around_single() {
+        let tests = [
+            ("#[|]#", "#[|]#"),
+            ("firs#[t|]#\n\nparagraph\n\n", "#[first\n\n|]#paragraph\n\n"),
+            (
+                "second\n\npa#[r|]#agraph\n\n",
+                "second\n\n#[paragraph\n\n|]#",
+            ),
+            ("#[f|]#irst char\n\n", "#[first char\n\n|]#"),
+            ("last char\n#[\n|]#", "#[last char\n\n|]#"),
+            (
+                "empty to line\n#[\n|]#paragraph boundary\n\n",
+                "empty to line\n\n#[paragraph boundary\n\n|]#",
+            ),
+            (
+                "line to empty\n\n#[p|]#aragraph boundary\n\n",
+                "line to empty\n\n#[paragraph boundary\n\n|]#",
+            ),
+        ];
+
+        for (before, expected) in tests {
+            let (s, selection) = crate::test::print(before);
+            let text = Rope::from(s.as_str());
+            let selection = selection
+                .transform(|r| textobject_paragraph(text.slice(..), r, TextObject::Around, 1));
+            let actual = crate::test::plain(&s, selection);
+            assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
+        }
+    }
+
     #[test]
     fn test_textobject_surround() {
-        // (text, [(cursor position, textobject, final range, count), ...])
+        // (text, [(cursor position, textobject, final range, surround char, count), ...])
         let tests = &[
             (
                 "simple (single) surround pairs",

helix-core/src/transaction.rs

@@ -21,8 +21,7 @@ pub enum Assoc {
     After,
 }
 
-// ChangeSpec = Change | ChangeSet | Vec<Change>
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Default, Clone, PartialEq, Eq)]
 pub struct ChangeSet {
     pub(crate) changes: Vec<Operation>,
     /// The required document length. Will refuse to apply changes unless it matches.
@@ -30,16 +29,6 @@ pub struct ChangeSet {
     len_after: usize,
 }
 
-impl Default for ChangeSet {
-    fn default() -> Self {
-        Self {
-            changes: Vec::new(),
-            len: 0,
-            len_after: 0,
-        }
-    }
-}
-
 impl ChangeSet {
     pub fn with_capacity(capacity: usize) -> Self {
         Self {
@@ -60,7 +49,6 @@ impl ChangeSet {
     }
 
     // TODO: from iter
-    //
 
     #[doc(hidden)] // used by lsp to convert to LSP changes
     pub fn changes(&self) -> &[Operation] {
@@ -95,7 +83,7 @@ impl ChangeSet {
 
         let new_last = match self.changes.as_mut_slice() {
             [.., Insert(prev)] | [.., Insert(prev), Delete(_)] => {
-                prev.push_tendril(&fragment);
+                prev.push_str(&fragment);
                 return;
             }
             [.., last @ Delete(_)] => std::mem::replace(last, Insert(fragment)),
@@ -199,7 +187,7 @@ impl ChangeSet {
                             // TODO: cover this with a test
                             // figure out the byte index of the truncated string end
                             let (pos, _) = s.char_indices().nth(j).unwrap();
-                            s.pop_front(pos as u32);
+                            s.replace_range(0..pos, "");
                             head_a = Some(Insert(s));
                             head_b = changes_b.next();
                         }
@@ -221,9 +209,11 @@ impl ChangeSet {
                         Ordering::Greater => {
                             // figure out the byte index of the truncated string end
                             let (pos, _) = s.char_indices().nth(j).unwrap();
-                            let pos = pos as u32;
-                            changes.insert(s.subtendril(0, pos));
-                            head_a = Some(Insert(s.subtendril(pos, s.len() as u32 - pos)));
+                            let mut before = s;
+                            let after = before.split_off(pos);
+
+                            changes.insert(before);
+                            head_a = Some(Insert(after));
                             head_b = changes_b.next();
                         }
                     }
@@ -287,7 +277,7 @@ impl ChangeSet {
                 }
                 Delete(n) => {
                     let text = Cow::from(original_doc.slice(pos..pos + *n));
-                    changes.insert(Tendril::from_slice(&text));
+                    changes.insert(Tendril::from(text.as_ref()));
                     pos += n;
                 }
                 Insert(s) => {
@@ -330,7 +320,7 @@ impl ChangeSet {
     /// `true` when the set is empty.
     #[inline]
     pub fn is_empty(&self) -> bool {
-        self.changes.is_empty()
+        self.changes.is_empty() || self.changes == [Operation::Retain(self.len)]
     }
 
     /// Map a position through the changes.
@@ -419,12 +409,10 @@ impl ChangeSet {
 
 /// Transaction represents a single undoable unit of changes. Several changes can be grouped into
 /// a single transaction.
-#[derive(Debug, Default, Clone)]
+#[derive(Debug, Default, Clone, PartialEq, Eq)]
 pub struct Transaction {
     changes: ChangeSet,
     selection: Option<Selection>,
-    // effects, annotations
-    // scroll_into_view
 }
 
 impl Transaction {
@@ -448,14 +436,12 @@ impl Transaction {
 
     /// Returns true if applied successfully.
     pub fn apply(&self, doc: &mut Rope) -> bool {
-        if !self.changes.is_empty() {
-            // apply changes to the document
-            if !self.changes.apply(doc) {
-                return false;
-            }
+        if self.changes.is_empty() {
+            return true;
         }
 
-        true
+        // apply changes to the document
+        self.changes.apply(doc)
     }
 
     /// Generate a transaction that reverts this one.
@@ -483,7 +469,7 @@ impl Transaction {
     /// Generate a transaction from a set of changes.
     pub fn change<I>(doc: &Rope, changes: I) -> Self
     where
-        I: IntoIterator<Item = Change> + Iterator,
+        I: Iterator<Item = Change>,
     {
         let len = doc.len_chars();
 
@@ -491,12 +477,11 @@ impl Transaction {
         let size = upper.unwrap_or(lower);
         let mut changeset = ChangeSet::with_capacity(2 * size + 1); // rough estimate
 
-        // TODO: verify ranges are ordered and not overlapping or change will panic.
-
-        // TODO: test for (pos, pos, None) to factor out as nothing
-
         let mut last = 0;
         for (from, to, tendril) in changes {
+            // Verify ranges are ordered and not overlapping
+            debug_assert!(last <= from);
+
             // Retain from last "to" to current "from"
             changeset.retain(from - last);
             let span = to - from;
@@ -702,7 +687,7 @@ mod test {
         let mut doc = Rope::from("hello world!\ntest 123");
         let transaction = Transaction::change(
             &doc,
-            // (1, 1, None) is a useless 0-width delete
+            // (1, 1, None) is a useless 0-width delete that gets factored out
             vec![(1, 1, None), (6, 11, Some("void".into())), (12, 17, None)].into_iter(),
         );
         transaction.apply(&mut doc);
@@ -720,19 +705,19 @@ mod test {
     #[test]
     fn optimized_composition() {
         let mut state = State::new("".into());
-        let t1 = Transaction::insert(&state.doc, &state.selection, Tendril::from_char('h'));
+        let t1 = Transaction::insert(&state.doc, &state.selection, Tendril::from("h"));
         t1.apply(&mut state.doc);
         state.selection = state.selection.clone().map(t1.changes());
-        let t2 = Transaction::insert(&state.doc, &state.selection, Tendril::from_char('e'));
+        let t2 = Transaction::insert(&state.doc, &state.selection, Tendril::from("e"));
         t2.apply(&mut state.doc);
         state.selection = state.selection.clone().map(t2.changes());
-        let t3 = Transaction::insert(&state.doc, &state.selection, Tendril::from_char('l'));
+        let t3 = Transaction::insert(&state.doc, &state.selection, Tendril::from("l"));
         t3.apply(&mut state.doc);
         state.selection = state.selection.clone().map(t3.changes());
-        let t4 = Transaction::insert(&state.doc, &state.selection, Tendril::from_char('l'));
+        let t4 = Transaction::insert(&state.doc, &state.selection, Tendril::from("l"));
         t4.apply(&mut state.doc);
         state.selection = state.selection.clone().map(t4.changes());
-        let t5 = Transaction::insert(&state.doc, &state.selection, Tendril::from_char('o'));
+        let t5 = Transaction::insert(&state.doc, &state.selection, Tendril::from("o"));
         t5.apply(&mut state.doc);
         state.selection = state.selection.clone().map(t5.changes());
 
@@ -771,7 +756,7 @@ mod test {
 
     #[test]
     fn combine_with_utf8() {
-        const TEST_CASE: &'static str = "Hello, これはヘリックスエディターです！";
+        const TEST_CASE: &str = "Hello, これはヘリックスエディターです！";
 
         let empty = Rope::from("");
         let a = ChangeSet::new(&empty);

helix-core/tests/data/indent/indent.rs

@@ -0,0 +1 @@
+../../../src/indent.rs

helix-core/tests/data/indent/languages.toml

@@ -0,0 +1,13 @@
+# This languages.toml should contain definitions for all languages for which we have indent tests
+[[language]]
+name = "rust"
+scope = "source.rust"
+injection-regex = "rust"
+file-types = ["rs"]
+comment-token = "//"
+roots = ["Cargo.toml", "Cargo.lock"]
+indent = { tab-width = 4, unit = "    " }
+
+[[grammar]]
+name = "rust"
+source = { git = "https://github.com/tree-sitter/tree-sitter-rust", rev = "a360da0a29a19c281d08295a35ecd0544d2da211" }

helix-core/tests/data/indent/rust.rs

@@ -0,0 +1,105 @@
+use std::{
+    io::{self, stdout, Stdout, Write},
+    path::PathBuf,
+    sync::Arc,
+    time::Duration,
+};
+mod test {
+    fn hello_world() {
+        1 + 1;
+
+        let does_indentation_work = 1;
+
+        let mut really_long_variable_name_using_up_the_line =
+            really_long_fn_that_should_definitely_go_on_the_next_line();
+        really_long_variable_name_using_up_the_line =
+            really_long_fn_that_should_definitely_go_on_the_next_line();
+        really_long_variable_name_using_up_the_line |=
+            really_long_fn_that_should_definitely_go_on_the_next_line();
+
+        let (
+            a_long_variable_name_in_this_tuple,
+            b_long_variable_name_in_this_tuple,
+            c_long_variable_name_in_this_tuple,
+            d_long_variable_name_in_this_tuple,
+            e_long_variable_name_in_this_tuple,
+        ): (usize, usize, usize, usize, usize) =
+            if really_long_fn_that_should_definitely_go_on_the_next_line() {
+                (
+                    03294239434,
+                    1213412342314,
+                    21231234134,
+                    834534234549898789,
+                    9879234234543853457,
+                )
+            } else {
+                (0, 1, 2, 3, 4)
+            };
+
+        let test_function = function_with_param(this_param,
+            that_param
+        );
+
+        let test_function = function_with_param(
+            this_param,
+            that_param
+        );
+
+        let test_function = function_with_proper_indent(param1,
+            param2,
+        );
+
+        let selection = Selection::new(
+            changes
+                .clone()
+                .map(|(start, end, text): (usize, usize, Option<Tendril>)| {
+                    let len = text.map(|text| text.len()).unwrap() - 1; // minus newline
+                    let pos = start + len;
+                    Range::new(pos, pos)
+                })
+                .collect(),
+            0,
+        );
+
+        return;
+    }
+}
+
+impl<A, D> MyTrait<A, D> for YourType
+where
+    A: TraitB + TraitC,
+    D: TraitE + TraitF,
+{
+
+}
+#[test]
+//
+match test {
+    Some(a) => 1,
+    None => {
+        unimplemented!()
+    }
+}
+std::panic::set_hook(Box::new(move |info| {
+    hook(info);
+}));
+
+{ { {
+    1
+}}}
+
+pub fn change<I>(document: &Document, changes: I) -> Self
+where
+    I: IntoIterator<Item = Change> + ExactSizeIterator,
+{
+    [
+        1,
+        2,
+        3,
+    ];
+    (
+        1,
+        2
+    );
+    true
+}

helix-core/tests/indent.rs

@@ -0,0 +1,68 @@
+use helix_core::{
+    indent::{treesitter_indent_for_pos, IndentStyle},
+    syntax::Loader,
+    Syntax,
+};
+use std::path::PathBuf;
+
+#[test]
+fn test_treesitter_indent_rust() {
+    test_treesitter_indent("rust.rs", "source.rust");
+}
+#[test]
+fn test_treesitter_indent_rust_2() {
+    test_treesitter_indent("indent.rs", "source.rust");
+    // TODO Use commands.rs as indentation test.
+    // Currently this fails because we can't align the parameters of a closure yet
+    // test_treesitter_indent("commands.rs", "source.rust");
+}
+
+fn test_treesitter_indent(file_name: &str, lang_scope: &str) {
+    let mut test_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    test_dir.push("tests/data/indent");
+
+    let mut test_file = test_dir.clone();
+    test_file.push(file_name);
+    let test_file = std::fs::File::open(test_file).unwrap();
+    let doc = ropey::Rope::from_reader(test_file).unwrap();
+
+    let mut config_file = test_dir;
+    config_file.push("languages.toml");
+    let config = std::fs::read(config_file).unwrap();
+    let config = toml::from_slice(&config).unwrap();
+    let loader = Loader::new(config);
+
+    // set runtime path so we can find the queries
+    let mut runtime = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    runtime.push("../runtime");
+    std::env::set_var("HELIX_RUNTIME", runtime.to_str().unwrap());
+
+    let language_config = loader.language_config_for_scope(lang_scope).unwrap();
+    let highlight_config = language_config.highlight_config(&[]).unwrap();
+    let syntax = Syntax::new(&doc, highlight_config, std::sync::Arc::new(loader));
+    let indent_query = language_config.indent_query().unwrap();
+    let text = doc.slice(..);
+
+    for i in 0..doc.len_lines() {
+        let line = text.line(i);
+        if let Some(pos) = helix_core::find_first_non_whitespace_char(line) {
+            let suggested_indent = treesitter_indent_for_pos(
+                indent_query,
+                &syntax,
+                &IndentStyle::Spaces(4),
+                text,
+                i,
+                text.line_to_char(i) + pos,
+                false,
+            )
+            .unwrap();
+            assert!(
+                line.get_slice(..pos).map_or(false, |s| s == suggested_indent),
+                "Wrong indentation on line {}:\n\"{}\" (original line)\n\"{}\" (suggested indentation)\n",
+                i+1,
+                line.slice(..line.len_chars()-1),
+                suggested_indent,
+            );
+        }
+    }
+}

helix-dap/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "helix-dap"
+version = "0.6.0"
+authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
+edition = "2018"
+license = "MPL-2.0"
+description = "DAP client implementation for Helix project"
+categories = ["editor"]
+repository = "https://github.com/helix-editor/helix"
+homepage = "https://helix-editor.com"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+helix-core = { version = "0.6", path = "../helix-core" }
+anyhow = "1.0"
+log = "0.4"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+thiserror = "1.0"
+tokio = { version = "1", features = ["rt", "rt-multi-thread", "io-util", "io-std", "time", "process", "macros", "fs", "parking_lot", "net", "sync"] }
+which = "4.2"
+
+[dev-dependencies]
+fern = "0.6"

helix-dap/src/client.rs

@@ -0,0 +1,480 @@
+use crate::{
+    transport::{Payload, Request, Response, Transport},
+    types::*,
+    Error, Result, ThreadId,
+};
+use helix_core::syntax::DebuggerQuirks;
+
+use serde_json::Value;
+
+use anyhow::anyhow;
+pub use log::{error, info};
+use std::{
+    collections::HashMap,
+    future::Future,
+    net::{IpAddr, Ipv4Addr, SocketAddr},
+    path::PathBuf,
+    process::Stdio,
+    sync::atomic::{AtomicU64, Ordering},
+};
+use tokio::{
+    io::{AsyncBufRead, AsyncWrite, BufReader, BufWriter},
+    net::TcpStream,
+    process::{Child, Command},
+    sync::mpsc::{channel, unbounded_channel, UnboundedReceiver, UnboundedSender},
+    time,
+};
+
+#[derive(Debug)]
+pub struct Client {
+    id: usize,
+    _process: Option<Child>,
+    server_tx: UnboundedSender<Payload>,
+    request_counter: AtomicU64,
+    pub caps: Option<DebuggerCapabilities>,
+    // thread_id -> frames
+    pub stack_frames: HashMap<ThreadId, Vec<StackFrame>>,
+    pub thread_states: HashMap<ThreadId, String>,
+    pub thread_id: Option<ThreadId>,
+    /// Currently active frame for the current thread.
+    pub active_frame: Option<usize>,
+    pub quirks: DebuggerQuirks,
+}
+
+impl Client {
+    // Spawn a process and communicate with it by either TCP or stdio
+    pub async fn process(
+        transport: &str,
+        command: &str,
+        args: Vec<&str>,
+        port_arg: Option<&str>,
+        id: usize,
+    ) -> Result<(Self, UnboundedReceiver<Payload>)> {
+        if command.is_empty() {
+            return Result::Err(Error::Other(anyhow!("Command not provided")));
+        }
+        if transport == "tcp" && port_arg.is_some() {
+            Self::tcp_process(command, args, port_arg.unwrap(), id).await
+        } else if transport == "stdio" {
+            Self::stdio(command, args, id)
+        } else {
+            Result::Err(Error::Other(anyhow!("Incorrect transport {}", transport)))
+        }
+    }
+
+    pub fn streams(
+        rx: Box<dyn AsyncBufRead + Unpin + Send>,
+        tx: Box<dyn AsyncWrite + Unpin + Send>,
+        err: Option<Box<dyn AsyncBufRead + Unpin + Send>>,
+        id: usize,
+        process: Option<Child>,
+    ) -> Result<(Self, UnboundedReceiver<Payload>)> {
+        let (server_rx, server_tx) = Transport::start(rx, tx, err, id);
+        let (client_rx, client_tx) = unbounded_channel();
+
+        let client = Self {
+            id,
+            _process: process,
+            server_tx,
+            request_counter: AtomicU64::new(0),
+            caps: None,
+            //
+            stack_frames: HashMap::new(),
+            thread_states: HashMap::new(),
+            thread_id: None,
+            active_frame: None,
+            quirks: DebuggerQuirks::default(),
+        };
+
+        tokio::spawn(Self::recv(server_rx, client_rx));
+
+        Ok((client, client_tx))
+    }
+
+    pub async fn tcp(
+        addr: std::net::SocketAddr,
+        id: usize,
+    ) -> Result<(Self, UnboundedReceiver<Payload>)> {
+        let stream = TcpStream::connect(addr).await?;
+        let (rx, tx) = stream.into_split();
+        Self::streams(Box::new(BufReader::new(rx)), Box::new(tx), None, id, None)
+    }
+
+    pub fn stdio(
+        cmd: &str,
+        args: Vec<&str>,
+        id: usize,
+    ) -> Result<(Self, UnboundedReceiver<Payload>)> {
+        // Resolve path to the binary
+        let cmd = which::which(cmd).map_err(|err| anyhow::anyhow!(err))?;
+
+        let process = Command::new(cmd)
+            .args(args)
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            // make sure the process is reaped on drop
+            .kill_on_drop(true)
+            .spawn();
+
+        let mut process = process?;
+
+        // TODO: do we need bufreader/writer here? or do we use async wrappers on unblock?
+        let writer = BufWriter::new(process.stdin.take().expect("Failed to open stdin"));
+        let reader = BufReader::new(process.stdout.take().expect("Failed to open stdout"));
+        let errors = process.stderr.take().map(BufReader::new);
+
+        Self::streams(
+            Box::new(BufReader::new(reader)),
+            Box::new(writer),
+            // errors.map(|errors| Box::new(BufReader::new(errors))),
+            match errors {
+                Some(errors) => Some(Box::new(BufReader::new(errors))),
+                None => None,
+            },
+            id,
+            Some(process),
+        )
+    }
+
+    async fn get_port() -> Option<u16> {
+        Some(
+            tokio::net::TcpListener::bind(SocketAddr::new(
+                IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)),
+                0,
+            ))
+            .await
+            .ok()?
+            .local_addr()
+            .ok()?
+            .port(),
+        )
+    }
+
+    pub async fn tcp_process(
+        cmd: &str,
+        args: Vec<&str>,
+        port_format: &str,
+        id: usize,
+    ) -> Result<(Self, UnboundedReceiver<Payload>)> {
+        let port = Self::get_port().await.unwrap();
+
+        let process = Command::new(cmd)
+            .args(args)
+            .args(port_format.replace("{}", &port.to_string()).split(' '))
+            // silence messages
+            .stdin(Stdio::null())
+            .stdout(Stdio::null())
+            .stderr(Stdio::null())
+            // Do not kill debug adapter when leaving, it should exit automatically
+            .spawn()?;
+
+        // Wait for adapter to become ready for connection
+        time::sleep(time::Duration::from_millis(500)).await;
+
+        let stream = TcpStream::connect(SocketAddr::new(
+            IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)),
+            port,
+        ))
+        .await?;
+
+        let (rx, tx) = stream.into_split();
+        Self::streams(
+            Box::new(BufReader::new(rx)),
+            Box::new(tx),
+            None,
+            id,
+            Some(process),
+        )
+    }
+
+    async fn recv(mut server_rx: UnboundedReceiver<Payload>, client_tx: UnboundedSender<Payload>) {
+        while let Some(msg) = server_rx.recv().await {
+            match msg {
+                Payload::Event(ev) => {
+                    client_tx.send(Payload::Event(ev)).expect("Failed to send");
+                }
+                Payload::Response(_) => unreachable!(),
+                Payload::Request(req) => {
+                    client_tx
+                        .send(Payload::Request(req))
+                        .expect("Failed to send");
+                }
+            }
+        }
+    }
+
+    pub fn id(&self) -> usize {
+        self.id
+    }
+
+    fn next_request_id(&self) -> u64 {
+        self.request_counter.fetch_add(1, Ordering::Relaxed)
+    }
+
+    // Internal, called by specific DAP commands when resuming
+    pub fn resume_application(&mut self) {
+        if let Some(thread_id) = self.thread_id {
+            self.thread_states.insert(thread_id, "running".to_string());
+            self.stack_frames.remove(&thread_id);
+        }
+        self.active_frame = None;
+        self.thread_id = None;
+    }
+
+    /// Execute a RPC request on the debugger.
+    pub fn call<R: crate::types::Request>(
+        &self,
+        arguments: R::Arguments,
+    ) -> impl Future<Output = Result<Value>>
+    where
+        R::Arguments: serde::Serialize,
+    {
+        let server_tx = self.server_tx.clone();
+        let id = self.next_request_id();
+
+        async move {
+            use std::time::Duration;
+            use tokio::time::timeout;
+
+            let arguments = Some(serde_json::to_value(arguments)?);
+
+            let (callback_tx, mut callback_rx) = channel(1);
+
+            let req = Request {
+                back_ch: Some(callback_tx),
+                seq: id,
+                command: R::COMMAND.to_string(),
+                arguments,
+            };
+
+            server_tx
+                .send(Payload::Request(req))
+                .map_err(|e| Error::Other(e.into()))?;
+
+            // TODO: specifiable timeout, delay other calls until initialize success
+            timeout(Duration::from_secs(20), callback_rx.recv())
+                .await
+                .map_err(|_| Error::Timeout)? // return Timeout
+                .ok_or(Error::StreamClosed)?
+                .map(|response| response.body.unwrap_or_default())
+            // TODO: check response.success
+        }
+    }
+
+    pub async fn request<R: crate::types::Request>(&self, params: R::Arguments) -> Result<R::Result>
+    where
+        R::Arguments: serde::Serialize,
+        R::Result: core::fmt::Debug, // TODO: temporary
+    {
+        // a future that resolves into the response
+        let json = self.call::<R>(params).await?;
+        let response = serde_json::from_value(json)?;
+        Ok(response)
+    }
+
+    pub fn reply(
+        &self,
+        request_seq: u64,
+        command: &str,
+        result: core::result::Result<Value, Error>,
+    ) -> impl Future<Output = Result<()>> {
+        let server_tx = self.server_tx.clone();
+        let command = command.to_string();
+
+        async move {
+            let response = match result {
+                Ok(result) => Response {
+                    request_seq,
+                    command,
+                    success: true,
+                    message: None,
+                    body: Some(result),
+                },
+                Err(error) => Response {
+                    request_seq,
+                    command,
+                    success: false,
+                    message: Some(error.to_string()),
+                    body: None,
+                },
+            };
+
+            server_tx
+                .send(Payload::Response(response))
+                .map_err(|e| Error::Other(e.into()))?;
+
+            Ok(())
+        }
+    }
+
+    pub fn capabilities(&self) -> &DebuggerCapabilities {
+        self.caps.as_ref().expect("debugger not yet initialized!")
+    }
+
+    pub async fn initialize(&mut self, adapter_id: String) -> Result<()> {
+        let args = requests::InitializeArguments {
+            client_id: Some("hx".to_owned()),
+            client_name: Some("helix".to_owned()),
+            adapter_id,
+            locale: Some("en-us".to_owned()),
+            lines_start_at_one: Some(true),
+            columns_start_at_one: Some(true),
+            path_format: Some("path".to_owned()),
+            supports_variable_type: Some(true),
+            supports_variable_paging: Some(false),
+            supports_run_in_terminal_request: Some(true),
+            supports_memory_references: Some(false),
+            supports_progress_reporting: Some(false),
+            supports_invalidated_event: Some(false),
+        };
+
+        let response = self.request::<requests::Initialize>(args).await?;
+        self.caps = Some(response);
+
+        Ok(())
+    }
+
+    pub fn disconnect(&self) -> impl Future<Output = Result<Value>> {
+        self.call::<requests::Disconnect>(())
+    }
+
+    pub fn launch(&self, args: serde_json::Value) -> impl Future<Output = Result<Value>> {
+        self.call::<requests::Launch>(args)
+    }
+
+    pub fn attach(&self, args: serde_json::Value) -> impl Future<Output = Result<Value>> {
+        self.call::<requests::Attach>(args)
+    }
+
+    pub async fn set_breakpoints(
+        &self,
+        file: PathBuf,
+        breakpoints: Vec<SourceBreakpoint>,
+    ) -> Result<Option<Vec<Breakpoint>>> {
+        let args = requests::SetBreakpointsArguments {
+            source: Source {
+                path: Some(file),
+                name: None,
+                source_reference: None,
+                presentation_hint: None,
+                origin: None,
+                sources: None,
+                adapter_data: None,
+                checksums: None,
+            },
+            breakpoints: Some(breakpoints),
+            source_modified: Some(false),
+        };
+
+        let response = self.request::<requests::SetBreakpoints>(args).await?;
+
+        Ok(response.breakpoints)
+    }
+
+    pub async fn configuration_done(&self) -> Result<()> {
+        self.request::<requests::ConfigurationDone>(()).await
+    }
+
+    pub fn continue_thread(&self, thread_id: ThreadId) -> impl Future<Output = Result<Value>> {
+        let args = requests::ContinueArguments { thread_id };
+
+        self.call::<requests::Continue>(args)
+    }
+
+    pub async fn stack_trace(
+        &self,
+        thread_id: ThreadId,
+    ) -> Result<(Vec<StackFrame>, Option<usize>)> {
+        let args = requests::StackTraceArguments {
+            thread_id,
+            start_frame: None,
+            levels: None,
+            format: None,
+        };
+
+        let response = self.request::<requests::StackTrace>(args).await?;
+        Ok((response.stack_frames, response.total_frames))
+    }
+
+    pub fn threads(&self) -> impl Future<Output = Result<Value>> {
+        self.call::<requests::Threads>(())
+    }
+
+    pub async fn scopes(&self, frame_id: usize) -> Result<Vec<Scope>> {
+        let args = requests::ScopesArguments { frame_id };
+
+        let response = self.request::<requests::Scopes>(args).await?;
+        Ok(response.scopes)
+    }
+
+    pub async fn variables(&self, variables_reference: usize) -> Result<Vec<Variable>> {
+        let args = requests::VariablesArguments {
+            variables_reference,
+            filter: None,
+            start: None,
+            count: None,
+            format: None,
+        };
+
+        let response = self.request::<requests::Variables>(args).await?;
+        Ok(response.variables)
+    }
+
+    pub fn step_in(&self, thread_id: ThreadId) -> impl Future<Output = Result<Value>> {
+        let args = requests::StepInArguments {
+            thread_id,
+            target_id: None,
+            granularity: None,
+        };
+
+        self.call::<requests::StepIn>(args)
+    }
+
+    pub fn step_out(&self, thread_id: ThreadId) -> impl Future<Output = Result<Value>> {
+        let args = requests::StepOutArguments {
+            thread_id,
+            granularity: None,
+        };
+
+        self.call::<requests::StepOut>(args)
+    }
+
+    pub fn next(&self, thread_id: ThreadId) -> impl Future<Output = Result<Value>> {
+        let args = requests::NextArguments {
+            thread_id,
+            granularity: None,
+        };
+
+        self.call::<requests::Next>(args)
+    }
+
+    pub fn pause(&self, thread_id: ThreadId) -> impl Future<Output = Result<Value>> {
+        let args = requests::PauseArguments { thread_id };
+
+        self.call::<requests::Pause>(args)
+    }
+
+    pub async fn eval(
+        &self,
+        expression: String,
+        frame_id: Option<usize>,
+    ) -> Result<requests::EvaluateResponse> {
+        let args = requests::EvaluateArguments {
+            expression,
+            frame_id,
+            context: None,
+            format: None,
+        };
+
+        self.request::<requests::Evaluate>(args).await
+    }
+
+    pub fn set_exception_breakpoints(
+        &self,
+        filters: Vec<String>,
+    ) -> impl Future<Output = Result<Value>> {
+        let args = requests::SetExceptionBreakpointsArguments { filters };
+
+        self.call::<requests::SetExceptionBreakpoints>(args)
+    }
+}

helix-dap/src/lib.rs

@@ -0,0 +1,24 @@
+mod client;
+mod transport;
+mod types;
+
+pub use client::Client;
+pub use events::Event;
+pub use transport::{Payload, Response, Transport};
+pub use types::*;
+
+use thiserror::Error;
+#[derive(Error, Debug)]
+pub enum Error {
+    #[error("failed to parse: {0}")]
+    Parse(#[from] serde_json::Error),
+    #[error("IO Error: {0}")]
+    IO(#[from] std::io::Error),
+    #[error("request timed out")]
+    Timeout,
+    #[error("server closed the stream")]
+    StreamClosed,
+    #[error(transparent)]
+    Other(#[from] anyhow::Error),
+}
+pub type Result<T> = core::result::Result<T, Error>;

helix-dap/src/transport.rs

@@ -0,0 +1,280 @@
+use crate::{Error, Event, Result};
+use anyhow::Context;
+use log::{error, info, warn};
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use std::collections::HashMap;
+use std::sync::Arc;
+use tokio::{
+    io::{AsyncBufRead, AsyncBufReadExt, AsyncReadExt, AsyncWrite, AsyncWriteExt},
+    sync::{
+        mpsc::{unbounded_channel, Sender, UnboundedReceiver, UnboundedSender},
+        Mutex,
+    },
+};
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct Request {
+    #[serde(skip)]
+    pub back_ch: Option<Sender<Result<Response>>>,
+    pub seq: u64,
+    pub command: String,
+    pub arguments: Option<Value>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+pub struct Response {
+    // seq is omitted as unused and is not sent by some implementations
+    pub request_seq: u64,
+    pub success: bool,
+    pub command: String,
+    pub message: Option<String>,
+    pub body: Option<Value>,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+#[serde(tag = "type", rename_all = "camelCase")]
+pub enum Payload {
+    // type = "event"
+    Event(Box<Event>),
+    // type = "response"
+    Response(Response),
+    // type = "request"
+    Request(Request),
+}
+
+#[derive(Debug)]
+pub struct Transport {
+    #[allow(unused)]
+    id: usize,
+    pending_requests: Mutex<HashMap<u64, Sender<Result<Response>>>>,
+}
+
+impl Transport {
+    pub fn start(
+        server_stdout: Box<dyn AsyncBufRead + Unpin + Send>,
+        server_stdin: Box<dyn AsyncWrite + Unpin + Send>,
+        server_stderr: Option<Box<dyn AsyncBufRead + Unpin + Send>>,
+        id: usize,
+    ) -> (UnboundedReceiver<Payload>, UnboundedSender<Payload>) {
+        let (client_tx, rx) = unbounded_channel();
+        let (tx, client_rx) = unbounded_channel();
+
+        let transport = Self {
+            id,
+            pending_requests: Mutex::new(HashMap::default()),
+        };
+
+        let transport = Arc::new(transport);
+
+        tokio::spawn(Self::recv(transport.clone(), server_stdout, client_tx));
+        tokio::spawn(Self::send(transport, server_stdin, client_rx));
+        if let Some(stderr) = server_stderr {
+            tokio::spawn(Self::err(stderr));
+        }
+
+        (rx, tx)
+    }
+
+    async fn recv_server_message(
+        reader: &mut Box<dyn AsyncBufRead + Unpin + Send>,
+        buffer: &mut String,
+    ) -> Result<Payload> {
+        let mut content_length = None;
+        loop {
+            buffer.truncate(0);
+            if reader.read_line(buffer).await? == 0 {
+                return Err(Error::StreamClosed);
+            };
+
+            if buffer == "\r\n" {
+                // look for an empty CRLF line
+                break;
+            }
+
+            let header = buffer.trim();
+            let parts = header.split_once(": ");
+
+            match parts {
+                Some(("Content-Length", value)) => {
+                    content_length = Some(value.parse().context("invalid content length")?);
+                }
+                Some((_, _)) => {}
+                None => {
+                    // Workaround: Some non-conformant language servers will output logging and other garbage
+                    // into the same stream as JSON-RPC messages. This can also happen from shell scripts that spawn
+                    // the server. Skip such lines and log a warning.
+
+                    // warn!("Failed to parse header: {:?}", header);
+                }
+            }
+        }
+
+        let content_length = content_length.context("missing content length")?;
+
+        //TODO: reuse vector
+        let mut content = vec![0; content_length];
+        reader.read_exact(&mut content).await?;
+        let msg = std::str::from_utf8(&content).context("invalid utf8 from server")?;
+
+        info!("<- DAP {}", msg);
+
+        // try parsing as output (server response) or call (server request)
+        let output: serde_json::Result<Payload> = serde_json::from_str(msg);
+
+        Ok(output?)
+    }
+
+    async fn recv_server_error(
+        err: &mut (impl AsyncBufRead + Unpin + Send),
+        buffer: &mut String,
+    ) -> Result<()> {
+        buffer.truncate(0);
+        if err.read_line(buffer).await? == 0 {
+            return Err(Error::StreamClosed);
+        };
+        error!("err <- {}", buffer);
+
+        Ok(())
+    }
+
+    async fn send_payload_to_server(
+        &self,
+        server_stdin: &mut Box<dyn AsyncWrite + Unpin + Send>,
+        mut payload: Payload,
+    ) -> Result<()> {
+        if let Payload::Request(request) = &mut payload {
+            if let Some(back) = request.back_ch.take() {
+                self.pending_requests.lock().await.insert(request.seq, back);
+            }
+        }
+        let json = serde_json::to_string(&payload)?;
+        self.send_string_to_server(server_stdin, json).await
+    }
+
+    async fn send_string_to_server(
+        &self,
+        server_stdin: &mut Box<dyn AsyncWrite + Unpin + Send>,
+        request: String,
+    ) -> Result<()> {
+        info!("-> DAP {}", request);
+
+        // send the headers
+        server_stdin
+            .write_all(format!("Content-Length: {}\r\n\r\n", request.len()).as_bytes())
+            .await?;
+
+        // send the body
+        server_stdin.write_all(request.as_bytes()).await?;
+
+        server_stdin.flush().await?;
+
+        Ok(())
+    }
+
+    fn process_response(res: Response) -> Result<Response> {
+        if res.success {
+            info!("<- DAP success in response to {}", res.request_seq);
+
+            Ok(res)
+        } else {
+            error!(
+                "<- DAP error {:?} ({:?}) for command #{} {}",
+                res.message, res.body, res.request_seq, res.command
+            );
+
+            Err(Error::Other(anyhow::format_err!("{:?}", res.body)))
+        }
+    }
+
+    async fn process_server_message(
+        &self,
+        client_tx: &UnboundedSender<Payload>,
+        msg: Payload,
+    ) -> Result<()> {
+        match msg {
+            Payload::Response(res) => {
+                let request_seq = res.request_seq;
+                let tx = self.pending_requests.lock().await.remove(&request_seq);
+
+                match tx {
+                    Some(tx) => match tx.send(Self::process_response(res)).await {
+                        Ok(_) => (),
+                        Err(_) => error!(
+                            "Tried sending response into a closed channel (id={:?}), original request likely timed out",
+                            request_seq
+                        ),
+                    }
+                    None => {
+                        warn!("Response to nonexistent request #{}", res.request_seq);
+                        client_tx.send(Payload::Response(res)).expect("Failed to send");
+                    }
+                }
+
+                Ok(())
+            }
+            Payload::Request(Request {
+                ref command,
+                ref seq,
+                ..
+            }) => {
+                info!("<- DAP request {} #{}", command, seq);
+                client_tx.send(msg).expect("Failed to send");
+                Ok(())
+            }
+            Payload::Event(ref event) => {
+                info!("<- DAP event {:?}", event);
+                client_tx.send(msg).expect("Failed to send");
+                Ok(())
+            }
+        }
+    }
+
+    async fn recv(
+        transport: Arc<Self>,
+        mut server_stdout: Box<dyn AsyncBufRead + Unpin + Send>,
+        client_tx: UnboundedSender<Payload>,
+    ) {
+        let mut recv_buffer = String::new();
+        loop {
+            match Self::recv_server_message(&mut server_stdout, &mut recv_buffer).await {
+                Ok(msg) => {
+                    transport
+                        .process_server_message(&client_tx, msg)
+                        .await
+                        .unwrap();
+                }
+                Err(err) => {
+                    error!("err: <- {:?}", err);
+                    break;
+                }
+            }
+        }
+    }
+
+    async fn send(
+        transport: Arc<Self>,
+        mut server_stdin: Box<dyn AsyncWrite + Unpin + Send>,
+        mut client_rx: UnboundedReceiver<Payload>,
+    ) {
+        while let Some(payload) = client_rx.recv().await {
+            transport
+                .send_payload_to_server(&mut server_stdin, payload)
+                .await
+                .unwrap()
+        }
+    }
+
+    async fn err(mut server_stderr: Box<dyn AsyncBufRead + Unpin + Send>) {
+        let mut recv_buffer = String::new();
+        loop {
+            match Self::recv_server_error(&mut server_stderr, &mut recv_buffer).await {
+                Ok(_) => {}
+                Err(err) => {
+                    error!("err: <- {:?}", err);
+                    break;
+                }
+            }
+        }
+    }
+}

helix-dap/src/types.rs

@@ -0,0 +1,707 @@
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use std::collections::HashMap;
+use std::path::PathBuf;
+
+#[derive(
+    Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize,
+)]
+pub struct ThreadId(isize);
+
+impl std::fmt::Display for ThreadId {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+pub trait Request {
+    type Arguments: serde::de::DeserializeOwned + serde::Serialize;
+    type Result: serde::de::DeserializeOwned + serde::Serialize;
+    const COMMAND: &'static str;
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ColumnDescriptor {
+    pub attribute_name: String,
+    pub label: String,
+    pub format: Option<String>,
+    #[serde(rename = "type")]
+    pub ty: Option<String>,
+    pub width: Option<usize>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ExceptionBreakpointsFilter {
+    pub filter: String,
+    pub label: String,
+    pub description: Option<String>,
+    pub default: Option<bool>,
+    pub supports_condition: Option<bool>,
+    pub condition_description: Option<String>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct DebuggerCapabilities {
+    pub supports_configuration_done_request: Option<bool>,
+    pub supports_function_breakpoints: Option<bool>,
+    pub supports_conditional_breakpoints: Option<bool>,
+    pub supports_hit_conditional_breakpoints: Option<bool>,
+    pub supports_evaluate_for_hovers: Option<bool>,
+    pub supports_step_back: Option<bool>,
+    pub supports_set_variable: Option<bool>,
+    pub supports_restart_frame: Option<bool>,
+    pub supports_goto_targets_request: Option<bool>,
+    pub supports_step_in_targets_request: Option<bool>,
+    pub supports_completions_request: Option<bool>,
+    pub supports_modules_request: Option<bool>,
+    pub supports_restart_request: Option<bool>,
+    pub supports_exception_options: Option<bool>,
+    pub supports_value_formatting_options: Option<bool>,
+    pub supports_exception_info_request: Option<bool>,
+    pub support_terminate_debuggee: Option<bool>,
+    pub support_suspend_debuggee: Option<bool>,
+    pub supports_delayed_stack_trace_loading: Option<bool>,
+    pub supports_loaded_sources_request: Option<bool>,
+    pub supports_log_points: Option<bool>,
+    pub supports_terminate_threads_request: Option<bool>,
+    pub supports_set_expression: Option<bool>,
+    pub supports_terminate_request: Option<bool>,
+    pub supports_data_breakpoints: Option<bool>,
+    pub supports_read_memory_request: Option<bool>,
+    pub supports_write_memory_request: Option<bool>,
+    pub supports_disassemble_request: Option<bool>,
+    pub supports_cancel_request: Option<bool>,
+    pub supports_breakpoint_locations_request: Option<bool>,
+    pub supports_clipboard_context: Option<bool>,
+    pub supports_stepping_granularity: Option<bool>,
+    pub supports_instruction_breakpoints: Option<bool>,
+    pub supports_exception_filter_options: Option<bool>,
+    pub exception_breakpoint_filters: Option<Vec<ExceptionBreakpointsFilter>>,
+    pub completion_trigger_characters: Option<Vec<String>>,
+    pub additional_module_columns: Option<Vec<ColumnDescriptor>>,
+    pub supported_checksum_algorithms: Option<Vec<String>>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Checksum {
+    pub algorithm: String,
+    pub checksum: String,
+}
+
+#[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Source {
+    pub name: Option<String>,
+    pub path: Option<PathBuf>,
+    pub source_reference: Option<usize>,
+    pub presentation_hint: Option<String>,
+    pub origin: Option<String>,
+    pub sources: Option<Vec<Source>>,
+    pub adapter_data: Option<Value>,
+    pub checksums: Option<Vec<Checksum>>,
+}
+
+#[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct SourceBreakpoint {
+    pub line: usize,
+    pub column: Option<usize>,
+    pub condition: Option<String>,
+    pub hit_condition: Option<String>,
+    pub log_message: Option<String>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Breakpoint {
+    pub id: Option<usize>,
+    pub verified: bool,
+    pub message: Option<String>,
+    pub source: Option<Source>,
+    pub line: Option<usize>,
+    pub column: Option<usize>,
+    pub end_line: Option<usize>,
+    pub end_column: Option<usize>,
+    pub instruction_reference: Option<String>,
+    pub offset: Option<usize>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct StackFrameFormat {
+    pub parameters: Option<bool>,
+    pub parameter_types: Option<bool>,
+    pub parameter_names: Option<bool>,
+    pub parameter_values: Option<bool>,
+    pub line: Option<bool>,
+    pub module: Option<bool>,
+    pub include_all: Option<bool>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct StackFrame {
+    pub id: usize,
+    pub name: String,
+    pub source: Option<Source>,
+    pub line: usize,
+    pub column: usize,
+    pub end_line: Option<usize>,
+    pub end_column: Option<usize>,
+    pub can_restart: Option<bool>,
+    pub instruction_pointer_reference: Option<String>,
+    pub module_id: Option<Value>,
+    pub presentation_hint: Option<String>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Thread {
+    pub id: ThreadId,
+    pub name: String,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Scope {
+    pub name: String,
+    pub presentation_hint: Option<String>,
+    pub variables_reference: usize,
+    pub named_variables: Option<usize>,
+    pub indexed_variables: Option<usize>,
+    pub expensive: bool,
+    pub source: Option<Source>,
+    pub line: Option<usize>,
+    pub column: Option<usize>,
+    pub end_line: Option<usize>,
+    pub end_column: Option<usize>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ValueFormat {
+    pub hex: Option<bool>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct VariablePresentationHint {
+    pub kind: Option<String>,
+    pub attributes: Option<Vec<String>>,
+    pub visibility: Option<String>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Variable {
+    pub name: String,
+    pub value: String,
+    #[serde(rename = "type")]
+    pub ty: Option<String>,
+    pub presentation_hint: Option<VariablePresentationHint>,
+    pub evaluate_name: Option<String>,
+    pub variables_reference: usize,
+    pub named_variables: Option<usize>,
+    pub indexed_variables: Option<usize>,
+    pub memory_reference: Option<String>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Module {
+    pub id: String, // TODO: || number
+    pub name: String,
+    pub path: Option<PathBuf>,
+    pub is_optimized: Option<bool>,
+    pub is_user_code: Option<bool>,
+    pub version: Option<String>,
+    pub symbol_status: Option<String>,
+    pub symbol_file_path: Option<String>,
+    pub date_time_stamp: Option<String>,
+    pub address_range: Option<String>,
+}
+
+pub mod requests {
+    use super::*;
+    #[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct InitializeArguments {
+        #[serde(rename = "clientID")]
+        pub client_id: Option<String>,
+        pub client_name: Option<String>,
+        #[serde(rename = "adapterID")]
+        pub adapter_id: String,
+        pub locale: Option<String>,
+        #[serde(rename = "linesStartAt1")]
+        pub lines_start_at_one: Option<bool>,
+        #[serde(rename = "columnsStartAt1")]
+        pub columns_start_at_one: Option<bool>,
+        pub path_format: Option<String>,
+        pub supports_variable_type: Option<bool>,
+        pub supports_variable_paging: Option<bool>,
+        pub supports_run_in_terminal_request: Option<bool>,
+        pub supports_memory_references: Option<bool>,
+        pub supports_progress_reporting: Option<bool>,
+        pub supports_invalidated_event: Option<bool>,
+    }
+
+    #[derive(Debug)]
+    pub enum Initialize {}
+
+    impl Request for Initialize {
+        type Arguments = InitializeArguments;
+        type Result = DebuggerCapabilities;
+        const COMMAND: &'static str = "initialize";
+    }
+
+    #[derive(Debug)]
+    pub enum Launch {}
+
+    impl Request for Launch {
+        type Arguments = Value;
+        type Result = Value;
+        const COMMAND: &'static str = "launch";
+    }
+
+    #[derive(Debug)]
+    pub enum Attach {}
+
+    impl Request for Attach {
+        type Arguments = Value;
+        type Result = Value;
+        const COMMAND: &'static str = "attach";
+    }
+
+    #[derive(Debug)]
+    pub enum Disconnect {}
+
+    impl Request for Disconnect {
+        type Arguments = ();
+        type Result = ();
+        const COMMAND: &'static str = "disconnect";
+    }
+
+    #[derive(Debug)]
+    pub enum ConfigurationDone {}
+
+    impl Request for ConfigurationDone {
+        type Arguments = ();
+        type Result = ();
+        const COMMAND: &'static str = "configurationDone";
+    }
+
+    #[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct SetBreakpointsArguments {
+        pub source: Source,
+        pub breakpoints: Option<Vec<SourceBreakpoint>>,
+        // lines is deprecated
+        pub source_modified: Option<bool>,
+    }
+
+    #[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct SetBreakpointsResponse {
+        pub breakpoints: Option<Vec<Breakpoint>>,
+    }
+
+    #[derive(Debug)]
+    pub enum SetBreakpoints {}
+
+    impl Request for SetBreakpoints {
+        type Arguments = SetBreakpointsArguments;
+        type Result = SetBreakpointsResponse;
+        const COMMAND: &'static str = "setBreakpoints";
+    }
+
+    #[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct ContinueArguments {
+        pub thread_id: ThreadId,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct ContinueResponse {
+        pub all_threads_continued: Option<bool>,
+    }
+
+    #[derive(Debug)]
+    pub enum Continue {}
+
+    impl Request for Continue {
+        type Arguments = ContinueArguments;
+        type Result = ContinueResponse;
+        const COMMAND: &'static str = "continue";
+    }
+
+    #[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct StackTraceArguments {
+        pub thread_id: ThreadId,
+        pub start_frame: Option<usize>,
+        pub levels: Option<usize>,
+        pub format: Option<StackFrameFormat>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct StackTraceResponse {
+        pub total_frames: Option<usize>,
+        pub stack_frames: Vec<StackFrame>,
+    }
+
+    #[derive(Debug)]
+    pub enum StackTrace {}
+
+    impl Request for StackTrace {
+        type Arguments = StackTraceArguments;
+        type Result = StackTraceResponse;
+        const COMMAND: &'static str = "stackTrace";
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct ThreadsResponse {
+        pub threads: Vec<Thread>,
+    }
+
+    #[derive(Debug)]
+    pub enum Threads {}
+
+    impl Request for Threads {
+        type Arguments = ();
+        type Result = ThreadsResponse;
+        const COMMAND: &'static str = "threads";
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct ScopesArguments {
+        pub frame_id: usize,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct ScopesResponse {
+        pub scopes: Vec<Scope>,
+    }
+
+    #[derive(Debug)]
+    pub enum Scopes {}
+
+    impl Request for Scopes {
+        type Arguments = ScopesArguments;
+        type Result = ScopesResponse;
+        const COMMAND: &'static str = "scopes";
+    }
+
+    #[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct VariablesArguments {
+        pub variables_reference: usize,
+        pub filter: Option<String>,
+        pub start: Option<usize>,
+        pub count: Option<usize>,
+        pub format: Option<ValueFormat>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct VariablesResponse {
+        pub variables: Vec<Variable>,
+    }
+
+    #[derive(Debug)]
+    pub enum Variables {}
+
+    impl Request for Variables {
+        type Arguments = VariablesArguments;
+        type Result = VariablesResponse;
+        const COMMAND: &'static str = "variables";
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct StepInArguments {
+        pub thread_id: ThreadId,
+        pub target_id: Option<usize>,
+        pub granularity: Option<String>,
+    }
+
+    #[derive(Debug)]
+    pub enum StepIn {}
+
+    impl Request for StepIn {
+        type Arguments = StepInArguments;
+        type Result = ();
+        const COMMAND: &'static str = "stepIn";
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct StepOutArguments {
+        pub thread_id: ThreadId,
+        pub granularity: Option<String>,
+    }
+
+    #[derive(Debug)]
+    pub enum StepOut {}
+
+    impl Request for StepOut {
+        type Arguments = StepOutArguments;
+        type Result = ();
+        const COMMAND: &'static str = "stepOut";
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct NextArguments {
+        pub thread_id: ThreadId,
+        pub granularity: Option<String>,
+    }
+
+    #[derive(Debug)]
+    pub enum Next {}
+
+    impl Request for Next {
+        type Arguments = NextArguments;
+        type Result = ();
+        const COMMAND: &'static str = "next";
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct PauseArguments {
+        pub thread_id: ThreadId,
+    }
+
+    #[derive(Debug)]
+    pub enum Pause {}
+
+    impl Request for Pause {
+        type Arguments = PauseArguments;
+        type Result = ();
+        const COMMAND: &'static str = "pause";
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct EvaluateArguments {
+        pub expression: String,
+        pub frame_id: Option<usize>,
+        pub context: Option<String>,
+        pub format: Option<ValueFormat>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct EvaluateResponse {
+        pub result: String,
+        #[serde(rename = "type")]
+        pub ty: Option<String>,
+        pub presentation_hint: Option<VariablePresentationHint>,
+        pub variables_reference: usize,
+        pub named_variables: Option<usize>,
+        pub indexed_variables: Option<usize>,
+        pub memory_reference: Option<String>,
+    }
+
+    #[derive(Debug)]
+    pub enum Evaluate {}
+
+    impl Request for Evaluate {
+        type Arguments = EvaluateArguments;
+        type Result = EvaluateResponse;
+        const COMMAND: &'static str = "evaluate";
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct SetExceptionBreakpointsArguments {
+        pub filters: Vec<String>,
+        // pub filterOptions: Option<Vec<ExceptionFilterOptions>>, // needs capability
+        // pub exceptionOptions: Option<Vec<ExceptionOptions>>, // needs capability
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct SetExceptionBreakpointsResponse {
+        pub breakpoints: Option<Vec<Breakpoint>>,
+    }
+
+    #[derive(Debug)]
+    pub enum SetExceptionBreakpoints {}
+
+    impl Request for SetExceptionBreakpoints {
+        type Arguments = SetExceptionBreakpointsArguments;
+        type Result = SetExceptionBreakpointsResponse;
+        const COMMAND: &'static str = "setExceptionBreakpoints";
+    }
+
+    // Reverse Requests
+
+    #[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct RunInTerminalResponse {
+        pub process_id: Option<u32>,
+        pub shell_process_id: Option<u32>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct RunInTerminalArguments {
+        pub kind: Option<String>,
+        pub title: Option<String>,
+        pub cwd: Option<String>,
+        pub args: Vec<String>,
+        pub env: Option<HashMap<String, Option<String>>>,
+    }
+
+    #[derive(Debug)]
+    pub enum RunInTerminal {}
+
+    impl Request for RunInTerminal {
+        type Arguments = RunInTerminalArguments;
+        type Result = RunInTerminalResponse;
+        const COMMAND: &'static str = "runInTerminal";
+    }
+}
+
+// Events
+
+pub mod events {
+    use super::*;
+
+    #[derive(Debug, Clone, Serialize, Deserialize)]
+    #[serde(rename_all = "camelCase")]
+    #[serde(tag = "event", content = "body")]
+    // seq is omitted as unused and is not sent by some implementations
+    pub enum Event {
+        Initialized,
+        Stopped(Stopped),
+        Continued(Continued),
+        Exited(Exited),
+        Terminated(Option<Terminated>),
+        Thread(Thread),
+        Output(Output),
+        Breakpoint(Breakpoint),
+        Module(Module),
+        LoadedSource(LoadedSource),
+        Process(Process),
+        Capabilities(Capabilities),
+        // ProgressStart(),
+        // ProgressUpdate(),
+        // ProgressEnd(),
+        // Invalidated(),
+        Memory(Memory),
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Stopped {
+        pub reason: String,
+        pub description: Option<String>,
+        pub thread_id: Option<ThreadId>,
+        pub preserve_focus_hint: Option<bool>,
+        pub text: Option<String>,
+        pub all_threads_stopped: Option<bool>,
+        pub hit_breakpoint_ids: Option<Vec<usize>>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Continued {
+        pub thread_id: ThreadId,
+        pub all_threads_continued: Option<bool>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Exited {
+        pub exit_code: usize,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Terminated {
+        pub restart: Option<Value>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Thread {
+        pub reason: String,
+        pub thread_id: ThreadId,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Output {
+        pub output: String,
+        pub category: Option<String>,
+        pub group: Option<String>,
+        pub line: Option<usize>,
+        pub column: Option<usize>,
+        pub variables_reference: Option<usize>,
+        pub source: Option<Source>,
+        pub data: Option<Value>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Breakpoint {
+        pub reason: String,
+        pub breakpoint: super::Breakpoint,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Module {
+        pub reason: String,
+        pub module: super::Module,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct LoadedSource {
+        pub reason: String,
+        pub source: super::Source,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Process {
+        pub name: String,
+        pub system_process_id: Option<usize>,
+        pub is_local_process: Option<bool>,
+        pub start_method: Option<String>, // TODO: use enum
+        pub pointer_size: Option<usize>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Capabilities {
+        pub capabilities: super::DebuggerCapabilities,
+    }
+
+    // #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    // #[serde(rename_all = "camelCase")]
+    // pub struct Invalidated {
+    // pub areas: Vec<InvalidatedArea>,
+    // pub thread_id: Option<ThreadId>,
+    // pub stack_frame_id: Option<usize>,
+    // }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Memory {
+        pub memory_reference: String,
+        pub offset: usize,
+        pub count: usize,
+    }
+}

helix-graphics/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "helix-graphics"
+version = "0.6.0"
+authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
+edition = "2021"
+license = "MPL-2.0"
+repository = "https://github.com/helix-editor/helix"
+homepage = "https://helix-editor.com"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/
+
+[features]
+term = ["crossterm"]
+
+[dependencies]
+bitflags = "1.3"
+serde = { version = "1.0", features = ["derive"] }
+
+crossterm = { version = "0.23", optional = true }
+
+# TODO: graphics.rs tests rely on this, but we should remove that
+# [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
+# helix-tui = { path = "../helix-tui" }

helix-graphics/src/lib.rs

@@ -1,10 +1,12 @@
 use bitflags::bitflags;
+use serde::{Deserialize, Serialize};
 use std::{
     cmp::{max, min},
     str::FromStr,
 };
 
-#[derive(Debug, Clone, Copy, PartialEq)]
+#[derive(Debug, Clone, Copy, PartialEq, Deserialize, Serialize)]
+#[serde(rename_all = "lowercase")]
 /// UNSTABLE
 pub enum CursorKind {
     /// █
@@ -17,6 +19,12 @@ pub enum CursorKind {
     Hidden,
 }
 
+impl Default for CursorKind {
+    fn default() -> Self {
+        Self::Block
+    }
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct Margin {
     pub vertical: u16,
@@ -25,7 +33,7 @@ pub struct Margin {
 
 /// A simple rectangle used in the computation of the layout and to give widgets an hint about the
 /// area they are supposed to render to. (x, y) = (0, 0) is at the top left corner of the screen.
-#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
+#[derive(Debug, Default, Clone, Copy, Hash, PartialEq, Eq)]
 pub struct Rect {
     pub x: u16,
     pub y: u16,
@@ -33,17 +41,6 @@ pub struct Rect {
     pub height: u16,
 }
 
-impl Default for Rect {
-    fn default() -> Rect {
-        Rect {
-            x: 0,
-            y: 0,
-            width: 0,
-            height: 0,
-        }
-    }
-}
-
 impl Rect {
     /// Creates a new rect, with width and height limited to keep the area under max u16.
     /// If clipped, aspect ratio will be preserved.
@@ -334,7 +331,7 @@ impl FromStr for Modifier {
 /// ];
 /// let mut buffer = Buffer::empty(Rect::new(0, 0, 1, 1));
 /// for style in &styles {
-///   buffer.get_mut(0, 0).set_style(*style);
+///   buffer[(0, 0)].set_style(*style);
 /// }
 /// assert_eq!(
 ///     Style {
@@ -343,7 +340,7 @@ impl FromStr for Modifier {
 ///         add_modifier: Modifier::BOLD,
 ///         sub_modifier: Modifier::empty(),
 ///     },
-///     buffer.get(0, 0).style(),
+///     buffer[(0, 0)].style(),
 /// );
 /// ```
 ///
@@ -359,7 +356,7 @@ impl FromStr for Modifier {
 /// ];
 /// let mut buffer = Buffer::empty(Rect::new(0, 0, 1, 1));
 /// for style in &styles {
-///   buffer.get_mut(0, 0).set_style(*style);
+///   buffer[(0, 0)].set_style(*style);
 /// }
 /// assert_eq!(
 ///     Style {
@@ -368,7 +365,7 @@ impl FromStr for Modifier {
 ///         add_modifier: Modifier::empty(),
 ///         sub_modifier: Modifier::empty(),
 ///     },
-///     buffer.get(0, 0).style(),
+///     buffer[(0, 0)].style(),
 /// );
 /// ```
 #[derive(Debug, Clone, Copy, PartialEq)]

helix-loader/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+name = "helix-loader"
+version = "0.6.0"
+description = "A post-modern text editor."
+authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
+edition = "2021"
+license = "MPL-2.0"
+categories = ["editor"]
+repository = "https://github.com/helix-editor/helix"
+homepage = "https://helix-editor.com"
+
+[dependencies]
+anyhow = "1"
+serde = { version = "1.0", features = ["derive"] }
+toml = "0.5"
+etcetera = "0.3"
+tree-sitter = "0.20"
+once_cell = "1.9"
+
+log = "0.4"
+
+# TODO: these two should be on !wasm32 only
+
+# cloning/compiling tree-sitter grammars
+cc = { version = "1" }
+threadpool = { version = "1.0" }
+
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+libloading = "0.7"
+

helix-loader/build.rs

@@ -0,0 +1,6 @@
+fn main() {
+    println!(
+        "cargo:rustc-env=BUILD_TARGET={}",
+        std::env::var("TARGET").unwrap()
+    );
+}

helix-loader/src/config.rs

@@ -0,0 +1,26 @@
+/// Default built-in languages.toml.
+pub fn default_lang_config() -> toml::Value {
+    toml::from_slice(include_bytes!("../../languages.toml"))
+        .expect("Could not parse built-in languages.toml to valid toml")
+}
+
+/// User configured languages.toml file, merged with the default config.
+pub fn user_lang_config() -> Result<toml::Value, toml::de::Error> {
+    let config = crate::local_config_dirs()
+        .into_iter()
+        .chain([crate::config_dir()].into_iter())
+        .map(|path| path.join("languages.toml"))
+        .filter_map(|file| {
+            std::fs::read(&file)
+                .map(|config| toml::from_slice(&config))
+                .ok()
+        })
+        .collect::<Result<Vec<_>, _>>()?
+        .into_iter()
+        .chain([default_lang_config()].into_iter())
+        .fold(toml::Value::Table(toml::value::Table::default()), |a, b| {
+            crate::merge_toml_values(b, a, true)
+        });
+
+    Ok(config)
+}

helix-loader/src/grammar.rs

@@ -0,0 +1,402 @@
+use anyhow::{anyhow, Context, Result};
+use serde::{Deserialize, Serialize};
+use std::fs;
+use std::time::SystemTime;
+use std::{
+    collections::HashSet,
+    path::{Path, PathBuf},
+    process::Command,
+    sync::mpsc::channel,
+};
+use tree_sitter::Language;
+
+#[cfg(unix)]
+const DYLIB_EXTENSION: &str = "so";
+
+#[cfg(windows)]
+const DYLIB_EXTENSION: &str = "dll";
+
+#[cfg(target_arch = "wasm32")]
+const DYLIB_EXTENSION: &str = "wasm";
+
+#[derive(Debug, Serialize, Deserialize)]
+struct Configuration {
+    #[serde(rename = "use-grammars")]
+    pub grammar_selection: Option<GrammarSelection>,
+    pub grammar: Vec<GrammarConfiguration>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase", untagged)]
+pub enum GrammarSelection {
+    Only { only: HashSet<String> },
+    Except { except: HashSet<String> },
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct GrammarConfiguration {
+    #[serde(rename = "name")]
+    pub grammar_id: String,
+    pub source: GrammarSource,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase", untagged)]
+pub enum GrammarSource {
+    Local {
+        path: String,
+    },
+    Git {
+        #[serde(rename = "git")]
+        remote: String,
+        #[serde(rename = "rev")]
+        revision: String,
+        subpath: Option<String>,
+    },
+}
+
+const BUILD_TARGET: &str = env!("BUILD_TARGET");
+const REMOTE_NAME: &str = "origin";
+
+#[cfg(target_arch = "wasm32")]
+pub fn get_language(name: &str) -> Result<Language> {
+    unimplemented!()
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+pub fn get_language(name: &str) -> Result<Language> {
+    use libloading::{Library, Symbol};
+    let name = name.to_ascii_lowercase();
+    let mut library_path = crate::runtime_dir().join("grammars").join(&name);
+    library_path.set_extension(DYLIB_EXTENSION);
+
+    let library = unsafe { Library::new(&library_path) }
+        .with_context(|| format!("Error opening dynamic library {:?}", library_path))?;
+    let language_fn_name = format!("tree_sitter_{}", name.replace('-', "_"));
+    let language = unsafe {
+        let language_fn: Symbol<unsafe extern "C" fn() -> Language> = library
+            .get(language_fn_name.as_bytes())
+            .with_context(|| format!("Failed to load symbol {}", language_fn_name))?;
+        language_fn()
+    };
+    std::mem::forget(library);
+    Ok(language)
+}
+
+pub fn fetch_grammars() -> Result<()> {
+    // We do not need to fetch local grammars.
+    let mut grammars = get_grammar_configs()?;
+    grammars.retain(|grammar| !matches!(grammar.source, GrammarSource::Local { .. }));
+
+    run_parallel(grammars, fetch_grammar, "fetch")
+}
+
+pub fn build_grammars() -> Result<()> {
+    run_parallel(get_grammar_configs()?, build_grammar, "build")
+}
+
+// Returns the set of grammar configurations the user requests.
+// Grammars are configured in the default and user `languages.toml` and are
+// merged. The `grammar_selection` key of the config is then used to filter
+// down all grammars into a subset of the user's choosing.
+fn get_grammar_configs() -> Result<Vec<GrammarConfiguration>> {
+    let config: Configuration = crate::config::user_lang_config()
+        .context("Could not parse languages.toml")?
+        .try_into()?;
+
+    let grammars = match config.grammar_selection {
+        Some(GrammarSelection::Only { only: selections }) => config
+            .grammar
+            .into_iter()
+            .filter(|grammar| selections.contains(&grammar.grammar_id))
+            .collect(),
+        Some(GrammarSelection::Except { except: rejections }) => config
+            .grammar
+            .into_iter()
+            .filter(|grammar| !rejections.contains(&grammar.grammar_id))
+            .collect(),
+        None => config.grammar,
+    };
+
+    Ok(grammars)
+}
+
+fn run_parallel<F>(grammars: Vec<GrammarConfiguration>, job: F, action: &'static str) -> Result<()>
+where
+    F: Fn(GrammarConfiguration) -> Result<()> + std::marker::Send + 'static + Copy,
+{
+    let pool = threadpool::Builder::new().build();
+    let (tx, rx) = channel();
+
+    for grammar in grammars {
+        let tx = tx.clone();
+
+        pool.execute(move || {
+            tx.send(job(grammar)).unwrap();
+        });
+    }
+
+    drop(tx);
+
+    // TODO: print all failures instead of the first one found.
+    rx.iter()
+        .find(|result| result.is_err())
+        .map(|err| err.with_context(|| format!("Failed to {} some grammar(s)", action)))
+        .unwrap_or(Ok(()))
+}
+
+fn fetch_grammar(grammar: GrammarConfiguration) -> Result<()> {
+    if let GrammarSource::Git {
+        remote, revision, ..
+    } = grammar.source
+    {
+        let grammar_dir = crate::runtime_dir()
+            .join("grammars")
+            .join("sources")
+            .join(&grammar.grammar_id);
+
+        fs::create_dir_all(&grammar_dir).context(format!(
+            "Could not create grammar directory {:?}",
+            grammar_dir
+        ))?;
+
+        // create the grammar dir contains a git directory
+        if !grammar_dir.join(".git").is_dir() {
+            git(&grammar_dir, ["init"])?;
+        }
+
+        // ensure the remote matches the configured remote
+        if get_remote_url(&grammar_dir).map_or(true, |s| s != remote) {
+            set_remote(&grammar_dir, &remote)?;
+        }
+
+        // ensure the revision matches the configured revision
+        if get_revision(&grammar_dir).map_or(true, |s| s != revision) {
+            // Fetch the exact revision from the remote.
+            // Supported by server-side git since v2.5.0 (July 2015),
+            // enabled by default on major git hosts.
+            git(
+                &grammar_dir,
+                ["fetch", "--depth", "1", REMOTE_NAME, &revision],
+            )?;
+            git(&grammar_dir, ["checkout", &revision])?;
+
+            println!(
+                "Grammar '{}' checked out at '{}'.",
+                grammar.grammar_id, revision
+            );
+        } else {
+            println!("Grammar '{}' is already up to date.", grammar.grammar_id);
+        }
+    }
+
+    Ok(())
+}
+
+// Sets the remote for a repository to the given URL, creating the remote if
+// it does not yet exist.
+fn set_remote(repository_dir: &Path, remote_url: &str) -> Result<String> {
+    git(
+        repository_dir,
+        ["remote", "set-url", REMOTE_NAME, remote_url],
+    )
+    .or_else(|_| git(repository_dir, ["remote", "add", REMOTE_NAME, remote_url]))
+}
+
+fn get_remote_url(repository_dir: &Path) -> Option<String> {
+    git(repository_dir, ["remote", "get-url", REMOTE_NAME]).ok()
+}
+
+fn get_revision(repository_dir: &Path) -> Option<String> {
+    git(repository_dir, ["rev-parse", "HEAD"]).ok()
+}
+
+// A wrapper around 'git' commands which returns stdout in success and a
+// helpful error message showing the command, stdout, and stderr in error.
+fn git<I, S>(repository_dir: &Path, args: I) -> Result<String>
+where
+    I: IntoIterator<Item = S>,
+    S: AsRef<std::ffi::OsStr>,
+{
+    let output = Command::new("git")
+        .args(args)
+        .current_dir(repository_dir)
+        .output()?;
+
+    if output.status.success() {
+        Ok(String::from_utf8_lossy(&output.stdout)
+            .trim_end()
+            .to_owned())
+    } else {
+        // TODO: figure out how to display the git command using `args`
+        Err(anyhow!(
+            "Git command failed.\nStdout: {}\nStderr: {}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr),
+        ))
+    }
+}
+
+fn build_grammar(grammar: GrammarConfiguration) -> Result<()> {
+    let grammar_dir = if let GrammarSource::Local { path } = &grammar.source {
+        PathBuf::from(&path)
+    } else {
+        crate::runtime_dir()
+            .join("grammars")
+            .join("sources")
+            .join(&grammar.grammar_id)
+    };
+
+    let grammar_dir_entries = grammar_dir.read_dir().with_context(|| {
+        format!(
+            "Failed to read directory {:?}. Did you use 'hx --grammar fetch'?",
+            grammar_dir
+        )
+    })?;
+
+    if grammar_dir_entries.count() == 0 {
+        return Err(anyhow!(
+            "Directory {:?} is empty. Did you use 'hx --grammar fetch'?",
+            grammar_dir
+        ));
+    };
+
+    let path = match &grammar.source {
+        GrammarSource::Git {
+            subpath: Some(subpath),
+            ..
+        } => grammar_dir.join(subpath),
+        _ => grammar_dir,
+    }
+    .join("src");
+
+    build_tree_sitter_library(&path, grammar)
+}
+
+fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) -> Result<()> {
+    let header_path = src_path;
+    let parser_path = src_path.join("parser.c");
+    let mut scanner_path = src_path.join("scanner.c");
+
+    let scanner_path = if scanner_path.exists() {
+        Some(scanner_path)
+    } else {
+        scanner_path.set_extension("cc");
+        if scanner_path.exists() {
+            Some(scanner_path)
+        } else {
+            None
+        }
+    };
+    let parser_lib_path = crate::runtime_dir().join("grammars");
+    let mut library_path = parser_lib_path.join(&grammar.grammar_id);
+    library_path.set_extension(DYLIB_EXTENSION);
+
+    let recompile = needs_recompile(&library_path, &parser_path, &scanner_path)
+        .context("Failed to compare source and binary timestamps")?;
+
+    if !recompile {
+        println!("Grammar '{}' is already built.", grammar.grammar_id);
+        return Ok(());
+    }
+
+    println!("Building grammar '{}'", grammar.grammar_id);
+
+    let mut config = cc::Build::new();
+    config
+        .cpp(true)
+        .opt_level(3)
+        .cargo_metadata(false)
+        .host(BUILD_TARGET)
+        .target(BUILD_TARGET);
+    let compiler = config.get_compiler();
+    let mut command = Command::new(compiler.path());
+    command.current_dir(src_path);
+    for (key, value) in compiler.env() {
+        command.env(key, value);
+    }
+
+    if cfg!(windows) {
+        command
+            .args(&["/nologo", "/LD", "/I"])
+            .arg(header_path)
+            .arg("/Od")
+            .arg("/utf-8");
+        if let Some(scanner_path) = scanner_path.as_ref() {
+            command.arg(scanner_path);
+        }
+
+        command
+            .arg(parser_path)
+            .arg("/link")
+            .arg(format!("/out:{}", library_path.to_str().unwrap()));
+    } else {
+        command
+            .arg("-shared")
+            .arg("-fPIC")
+            .arg("-fno-exceptions")
+            .arg("-g")
+            .arg("-I")
+            .arg(header_path)
+            .arg("-o")
+            .arg(&library_path)
+            .arg("-O3");
+        if let Some(scanner_path) = scanner_path.as_ref() {
+            if scanner_path.extension() == Some("c".as_ref()) {
+                command.arg("-xc").arg("-std=c99").arg(scanner_path);
+            } else {
+                command.arg(scanner_path);
+            }
+        }
+        command.arg("-xc").arg(parser_path);
+        if cfg!(all(unix, not(target_os = "macos"))) {
+            command.arg("-Wl,-z,relro,-z,now");
+        }
+    }
+
+    let output = command.output().context("Failed to execute C compiler")?;
+    if !output.status.success() {
+        return Err(anyhow!(
+            "Parser compilation failed.\nStdout: {}\nStderr: {}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr)
+        ));
+    }
+
+    Ok(())
+}
+
+fn needs_recompile(
+    lib_path: &Path,
+    parser_c_path: &Path,
+    scanner_path: &Option<PathBuf>,
+) -> Result<bool> {
+    if !lib_path.exists() {
+        return Ok(true);
+    }
+    let lib_mtime = mtime(lib_path)?;
+    if mtime(parser_c_path)? > lib_mtime {
+        return Ok(true);
+    }
+    if let Some(scanner_path) = scanner_path {
+        if mtime(scanner_path)? > lib_mtime {
+            return Ok(true);
+        }
+    }
+    Ok(false)
+}
+
+fn mtime(path: &Path) -> Result<SystemTime> {
+    Ok(fs::metadata(path)?.modified()?)
+}
+
+/// Gives the contents of a file from a language's `runtime/queries/<lang>`
+/// directory
+pub fn load_runtime_file(language: &str, filename: &str) -> Result<String, std::io::Error> {
+    let path = crate::RUNTIME_DIR
+        .join("queries")
+        .join(language)
+        .join(filename);
+    std::fs::read_to_string(&path)
+}

helix-loader/src/lib.rs

@@ -0,0 +1,237 @@
+pub mod config;
+pub mod grammar;
+
+use etcetera::base_strategy::{choose_base_strategy, BaseStrategy};
+
+pub static RUNTIME_DIR: once_cell::sync::Lazy<std::path::PathBuf> =
+    once_cell::sync::Lazy::new(runtime_dir);
+
+pub fn runtime_dir() -> std::path::PathBuf {
+    if let Ok(dir) = std::env::var("HELIX_RUNTIME") {
+        return dir.into();
+    }
+
+    const RT_DIR: &str = "runtime";
+    let conf_dir = config_dir().join(RT_DIR);
+    if conf_dir.exists() {
+        return conf_dir;
+    }
+
+    if let Ok(dir) = std::env::var("CARGO_MANIFEST_DIR") {
+        // this is the directory of the crate being run by cargo, we need the workspace path so we take the parent
+        return std::path::PathBuf::from(dir).parent().unwrap().join(RT_DIR);
+    }
+
+    // fallback to location of the executable being run
+    std::env::current_exe()
+        .ok()
+        .and_then(|path| path.parent().map(|path| path.to_path_buf().join(RT_DIR)))
+        .unwrap()
+}
+
+pub fn config_dir() -> std::path::PathBuf {
+    // TODO: allow env var override
+    let strategy = choose_base_strategy().expect("Unable to find the config directory!");
+    let mut path = strategy.config_dir();
+    path.push("helix");
+    path
+}
+
+pub fn local_config_dirs() -> Vec<std::path::PathBuf> {
+    let directories = find_root_impl(None, &[".helix".to_string()])
+        .into_iter()
+        .map(|path| path.join(".helix"))
+        .collect();
+    log::debug!("Located configuration folders: {:?}", directories);
+    directories
+}
+
+pub fn cache_dir() -> std::path::PathBuf {
+    // TODO: allow env var override
+    let strategy = choose_base_strategy().expect("Unable to find the config directory!");
+    let mut path = strategy.cache_dir();
+    path.push("helix");
+    path
+}
+
+pub fn config_file() -> std::path::PathBuf {
+    config_dir().join("config.toml")
+}
+
+pub fn lang_config_file() -> std::path::PathBuf {
+    config_dir().join("languages.toml")
+}
+
+pub fn log_file() -> std::path::PathBuf {
+    cache_dir().join("helix.log")
+}
+
+pub fn find_root_impl(root: Option<&str>, root_markers: &[String]) -> Vec<std::path::PathBuf> {
+    let current_dir = std::env::current_dir().expect("unable to determine current directory");
+    let mut directories = Vec::new();
+
+    let root = match root {
+        Some(root) => {
+            let root = std::path::Path::new(root);
+            if root.is_absolute() {
+                root.to_path_buf()
+            } else {
+                current_dir.join(root)
+            }
+        }
+        None => current_dir,
+    };
+
+    for ancestor in root.ancestors() {
+        // don't go higher than repo
+        if ancestor.join(".git").is_dir() {
+            // Use workspace if detected from marker
+            directories.push(ancestor.to_path_buf());
+            break;
+        } else if root_markers
+            .iter()
+            .any(|marker| ancestor.join(marker).exists())
+        {
+            directories.push(ancestor.to_path_buf());
+        }
+    }
+    directories
+}
+
+/// Merge two TOML documents, merging values from `right` onto `left`
+///
+/// When an array exists in both `left` and `right`, `right`'s array is
+/// used. When a table exists in both `left` and `right`, the merged table
+/// consists of all keys in `left`'s table unioned with all keys in `right`
+/// with the values of `right` being merged recursively onto values of
+/// `left`.
+///
+/// `merge_toplevel_arrays` controls whether a top-level array in the TOML
+/// document is merged instead of overridden. This is useful for TOML
+/// documents that use a top-level array of values like the `languages.toml`,
+/// where one usually wants to override or add to the array instead of
+/// replacing it altogether.
+pub fn merge_toml_values(
+    left: toml::Value,
+    right: toml::Value,
+    merge_toplevel_arrays: bool,
+) -> toml::Value {
+    use toml::Value;
+
+    fn get_name(v: &Value) -> Option<&str> {
+        v.get("name").and_then(Value::as_str)
+    }
+
+    match (left, right) {
+        (Value::Array(mut left_items), Value::Array(right_items)) => {
+            // The top-level arrays should be merged but nested arrays should
+            // act as overrides. For the `languages.toml` config, this means
+            // that you can specify a sub-set of languages in an overriding
+            // `languages.toml` but that nested arrays like Language Server
+            // arguments are replaced instead of merged.
+            if merge_toplevel_arrays {
+                left_items.reserve(right_items.len());
+                for rvalue in right_items {
+                    let lvalue = get_name(&rvalue)
+                        .and_then(|rname| {
+                            left_items.iter().position(|v| get_name(v) == Some(rname))
+                        })
+                        .map(|lpos| left_items.remove(lpos));
+                    let mvalue = match lvalue {
+                        Some(lvalue) => merge_toml_values(lvalue, rvalue, false),
+                        None => rvalue,
+                    };
+                    left_items.push(mvalue);
+                }
+                Value::Array(left_items)
+            } else {
+                Value::Array(right_items)
+            }
+        }
+        (Value::Table(mut left_map), Value::Table(right_map)) => {
+            for (rname, rvalue) in right_map {
+                match left_map.remove(&rname) {
+                    Some(lvalue) => {
+                        let merged_value = merge_toml_values(lvalue, rvalue, merge_toplevel_arrays);
+                        left_map.insert(rname, merged_value);
+                    }
+                    None => {
+                        left_map.insert(rname, rvalue);
+                    }
+                }
+            }
+            Value::Table(left_map)
+        }
+        // Catch everything else we didn't handle, and use the right value
+        (_, value) => value,
+    }
+}
+
+#[cfg(test)]
+mod merge_toml_tests {
+    use super::merge_toml_values;
+    use toml::Value;
+
+    #[test]
+    fn language_toml_map_merges() {
+        const USER: &str = r#"
+        [[language]]
+        name = "nix"
+        test = "bbb"
+        indent = { tab-width = 4, unit = "    ", test = "aaa" }
+        "#;
+
+        let base: Value = toml::from_slice(include_bytes!("../../languages.toml"))
+            .expect("Couldn't parse built-in languages config");
+        let user: Value = toml::from_str(USER).unwrap();
+
+        let merged = merge_toml_values(base, user, true);
+        let languages = merged.get("language").unwrap().as_array().unwrap();
+        let nix = languages
+            .iter()
+            .find(|v| v.get("name").unwrap().as_str().unwrap() == "nix")
+            .unwrap();
+        let nix_indent = nix.get("indent").unwrap();
+
+        // We changed tab-width and unit in indent so check them if they are the new values
+        assert_eq!(
+            nix_indent.get("tab-width").unwrap().as_integer().unwrap(),
+            4
+        );
+        assert_eq!(nix_indent.get("unit").unwrap().as_str().unwrap(), "    ");
+        // We added a new keys, so check them
+        assert_eq!(nix.get("test").unwrap().as_str().unwrap(), "bbb");
+        assert_eq!(nix_indent.get("test").unwrap().as_str().unwrap(), "aaa");
+        // We didn't change comment-token so it should be same
+        assert_eq!(nix.get("comment-token").unwrap().as_str().unwrap(), "#");
+    }
+
+    #[test]
+    fn language_toml_nested_array_merges() {
+        const USER: &str = r#"
+        [[language]]
+        name = "typescript"
+        language-server = { command = "deno", args = ["lsp"] }
+        "#;
+
+        let base: Value = toml::from_slice(include_bytes!("../../languages.toml"))
+            .expect("Couldn't parse built-in languages config");
+        let user: Value = toml::from_str(USER).unwrap();
+
+        let merged = merge_toml_values(base, user, true);
+        let languages = merged.get("language").unwrap().as_array().unwrap();
+        let ts = languages
+            .iter()
+            .find(|v| v.get("name").unwrap().as_str().unwrap() == "typescript")
+            .unwrap();
+        assert_eq!(
+            ts.get("language-server")
+                .unwrap()
+                .get("args")
+                .unwrap()
+                .as_array()
+                .unwrap(),
+            &vec![Value::String("lsp".into())]
+        )
+    }
+}

helix-lsp/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "helix-lsp"
-version = "0.5.0"
+version = "0.6.0"
 authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
 edition = "2021"
 license = "MPL-2.0"
@@ -12,16 +12,17 @@ homepage = "https://helix-editor.com"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-helix-core = { version = "0.5", path = "../helix-core" }
+helix-core = { version = "0.6", path = "../helix-core" }
 
 anyhow = "1.0"
 futures-executor = "0.3"
 futures-util = { version = "0.3", features = ["std", "async-await"], default-features = false }
 jsonrpc-core = { version = "18.0", default-features = false } # don't pull in all of futures
 log = "0.4"
-lsp-types = { version = "0.90", features = ["proposed"] }
+lsp-types = { version = "0.93", features = ["proposed"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 thiserror = "1.0"
-tokio = { version = "1.12", features = ["rt", "rt-multi-thread", "io-util", "io-std", "time", "process", "macros", "fs", "parking_lot"] }
-tokio-stream = "0.1.7"
+tokio = { version = "1.17", features = ["rt", "rt-multi-thread", "io-util", "io-std", "time", "process", "macros", "fs", "parking_lot", "sync"] }
+tokio-stream = "0.1.8"
+which = "4.2"

helix-lsp/src/client.rs

@@ -3,6 +3,7 @@ use crate::{
     Call, Error, OffsetEncoding, Result,
 };
 
+use anyhow::anyhow;
 use helix_core::{find_root, ChangeSet, Rope};
 use jsonrpc_core as jsonrpc;
 use lsp_types as lsp;
@@ -31,6 +32,9 @@ pub struct Client {
     pub(crate) capabilities: OnceCell<lsp::ServerCapabilities>,
     offset_encoding: OffsetEncoding,
     config: Option<Value>,
+    root_path: Option<std::path::PathBuf>,
+    root_uri: Option<lsp::Url>,
+    workspace_folders: Vec<lsp::WorkspaceFolder>,
 }
 
 impl Client {
@@ -39,8 +43,12 @@ impl Client {
         cmd: &str,
         args: &[String],
         config: Option<Value>,
+        root_markers: &[String],
         id: usize,
     ) -> Result<(Self, UnboundedReceiver<(usize, Call)>, Arc<Notify>)> {
+        // Resolve path to the binary
+        let cmd = which::which(cmd).map_err(|err| anyhow::anyhow!(err))?;
+
         let process = Command::new(cmd)
             .args(args)
             .stdin(Stdio::piped())
@@ -60,6 +68,27 @@ impl Client {
         let (server_rx, server_tx, initialize_notify) =
             Transport::start(reader, writer, stderr, id);
 
+        let root_path = find_root(None, root_markers);
+
+        let root_uri = root_path
+            .clone()
+            .and_then(|root| lsp::Url::from_file_path(root).ok());
+
+        // TODO: support multiple workspace folders
+        let workspace_folders = root_uri
+            .clone()
+            .map(|root| {
+                vec![lsp::WorkspaceFolder {
+                    name: root
+                        .path_segments()
+                        .and_then(|segments| segments.last())
+                        .map(|basename| basename.to_string())
+                        .unwrap_or_default(),
+                    uri: root,
+                }]
+            })
+            .unwrap_or_default();
+
         let client = Self {
             id,
             _process: process,
@@ -68,6 +97,10 @@ impl Client {
             capabilities: OnceCell::new(),
             offset_encoding: OffsetEncoding::Utf8,
             config,
+
+            root_path,
+            root_uri,
+            workspace_folders,
         };
 
         Ok((client, server_rx, initialize_notify))
@@ -107,6 +140,14 @@ impl Client {
         self.offset_encoding
     }
 
+    pub fn config(&self) -> Option<&Value> {
+        self.config.as_ref()
+    }
+
+    pub fn workspace_folders(&self) -> &[lsp::WorkspaceFolder] {
+        &self.workspace_folders
+    }
+
     /// Execute a RPC request on the language server.
     async fn request<R: lsp::request::Request>(&self, params: R::Params) -> Result<R::Result>
     where
@@ -202,7 +243,7 @@ impl Client {
                 Ok(result) => Output::Success(Success {
                     jsonrpc: Some(Version::V2),
                     id,
-                    result,
+                    result: serde_json::to_value(result)?,
                 }),
                 Err(error) => Output::Failure(Failure {
                     jsonrpc: Some(Version::V2),
@@ -224,9 +265,6 @@ impl Client {
     // -------------------------------------------------------------------------------------------
 
     pub(crate) async fn initialize(&self) -> Result<lsp::InitializeResult> {
-        // TODO: delay any requests that are triggered prior to initialize
-        let root = find_root(None).and_then(|root| lsp::Url::from_file_path(root).ok());
-
         if self.config.is_some() {
             log::info!("Using custom LSP config: {}", self.config.as_ref().unwrap());
         }
@@ -234,15 +272,35 @@ impl Client {
         #[allow(deprecated)]
         let params = lsp::InitializeParams {
             process_id: Some(std::process::id()),
-            // root_path is obsolete, use root_uri
-            root_path: None,
-            root_uri: root,
+            workspace_folders: Some(self.workspace_folders.clone()),
+            // root_path is obsolete, but some clients like pyright still use it so we specify both.
+            // clients will prefer _uri if possible
+            root_path: self
+                .root_path
+                .clone()
+                .and_then(|path| path.to_str().map(|path| path.to_owned())),
+            root_uri: self.root_uri.clone(),
             initialization_options: self.config.clone(),
             capabilities: lsp::ClientCapabilities {
+                workspace: Some(lsp::WorkspaceClientCapabilities {
+                    configuration: Some(true),
+                    did_change_configuration: Some(lsp::DynamicRegistrationClientCapabilities {
+                        dynamic_registration: Some(false),
+                    }),
+                    workspace_folders: Some(true),
+                    ..Default::default()
+                }),
                 text_document: Some(lsp::TextDocumentClientCapabilities {
                     completion: Some(lsp::CompletionClientCapabilities {
                         completion_item: Some(lsp::CompletionItemCapability {
                             snippet_support: Some(false),
+                            resolve_support: Some(lsp::CompletionItemCapabilityResolveSupport {
+                                properties: vec![
+                                    String::from("documentation"),
+                                    String::from("detail"),
+                                    String::from("additionalTextEdits"),
+                                ],
+                            }),
                             ..Default::default()
                         }),
                         completion_item_kind: Some(lsp::CompletionItemKindCapability {
@@ -257,6 +315,12 @@ impl Client {
                         content_format: Some(vec![lsp::MarkupKind::Markdown]),
                         ..Default::default()
                     }),
+                    rename: Some(lsp::RenameClientCapabilities {
+                        dynamic_registration: Some(false),
+                        prepare_support: Some(false),
+                        prepare_support_default_behavior: None,
+                        honors_change_annotations: Some(false),
+                    }),
                     code_action: Some(lsp::CodeActionClientCapabilities {
                         code_action_literal_support: Some(lsp::CodeActionLiteralSupport {
                             code_action_kind: lsp::CodeActionKindLiteralSupport {
@@ -277,6 +341,9 @@ impl Client {
                         }),
                         ..Default::default()
                     }),
+                    publish_diagnostics: Some(lsp::PublishDiagnosticsClientCapabilities {
+                        ..Default::default()
+                    }),
                     ..Default::default()
                 }),
                 window: Some(lsp::WindowClientCapabilities {
@@ -286,7 +353,6 @@ impl Client {
                 ..Default::default()
             },
             trace: None,
-            workspace_folders: None,
             client_info: None,
             locale: None, // TODO
         };
@@ -317,6 +383,16 @@ impl Client {
         self.exit().await
     }
 
+    // -------------------------------------------------------------------------------------------
+    // Workspace
+    // -------------------------------------------------------------------------------------------
+
+    pub fn did_change_configuration(&self, settings: Value) -> impl Future<Output = Result<()>> {
+        self.notify::<lsp::notification::DidChangeConfiguration>(
+            lsp::DidChangeConfigurationParams { settings },
+        )
+    }
+
     // -------------------------------------------------------------------------------------------
     // Text document
     // -------------------------------------------------------------------------------------------
@@ -428,7 +504,7 @@ impl Client {
 
                     changes.push(lsp::TextDocumentContentChangeEvent {
                         range: Some(lsp::Range::new(start, end)),
-                        text: s.into(),
+                        text: s.to_string(),
                         range_length: None,
                     });
                 }
@@ -461,7 +537,7 @@ impl Client {
         };
 
         let changes = match sync_capabilities {
-            lsp::TextDocumentSyncKind::Full => {
+            lsp::TextDocumentSyncKind::FULL => {
                 vec![lsp::TextDocumentContentChangeEvent {
                     // range = None -> whole document
                     range: None,        //Some(Range)
@@ -469,10 +545,11 @@ impl Client {
                     text: new_text.to_string(),
                 }]
             }
-            lsp::TextDocumentSyncKind::Incremental => {
+            lsp::TextDocumentSyncKind::INCREMENTAL => {
                 Self::changeset_to_changes(old_text, new_text, changes, self.offset_encoding)
             }
-            lsp::TextDocumentSyncKind::None => return None,
+            lsp::TextDocumentSyncKind::NONE => return None,
+            kind => unimplemented!("{:?}", kind),
         };
 
         Some(self.notify::<lsp::notification::DidChangeTextDocument>(
@@ -549,6 +626,14 @@ impl Client {
         self.call::<lsp::request::Completion>(params)
     }
 
+    pub async fn resolve_completion_item(
+        &self,
+        completion_item: lsp::CompletionItem,
+    ) -> Result<lsp::CompletionItem> {
+        self.request::<lsp::request::ResolveCompletionItem>(completion_item)
+            .await
+    }
+
     pub fn text_document_signature_help(
         &self,
         text_document: lsp::TextDocumentIdentifier,
@@ -761,15 +846,62 @@ impl Client {
         &self,
         text_document: lsp::TextDocumentIdentifier,
         range: lsp::Range,
+        context: lsp::CodeActionContext,
     ) -> impl Future<Output = Result<Value>> {
         let params = lsp::CodeActionParams {
             text_document,
             range,
-            context: lsp::CodeActionContext::default(),
+            context,
             work_done_progress_params: lsp::WorkDoneProgressParams::default(),
             partial_result_params: lsp::PartialResultParams::default(),
         };
 
         self.call::<lsp::request::CodeActionRequest>(params)
     }
+
+    pub async fn rename_symbol(
+        &self,
+        text_document: lsp::TextDocumentIdentifier,
+        position: lsp::Position,
+        new_name: String,
+    ) -> anyhow::Result<lsp::WorkspaceEdit> {
+        let capabilities = self.capabilities.get().unwrap();
+
+        // check if we're able to rename
+        match capabilities.rename_provider {
+            Some(lsp::OneOf::Left(true)) | Some(lsp::OneOf::Right(_)) => (),
+            // None | Some(false)
+            _ => {
+                let err = "The server does not support rename";
+                log::warn!("rename_symbol failed: {}", err);
+                return Err(anyhow!(err));
+            }
+        };
+
+        let params = lsp::RenameParams {
+            text_document_position: lsp::TextDocumentPositionParams {
+                text_document,
+                position,
+            },
+            new_name,
+            work_done_progress_params: lsp::WorkDoneProgressParams {
+                work_done_token: None,
+            },
+        };
+
+        let response = self.request::<lsp::request::Rename>(params).await?;
+        Ok(response.unwrap_or_default())
+    }
+
+    pub fn command(&self, command: lsp::Command) -> impl Future<Output = Result<Value>> {
+        let params = lsp::ExecuteCommandParams {
+            command: command.command,
+            arguments: command.arguments.unwrap_or_default(),
+            work_done_progress_params: lsp::WorkDoneProgressParams {
+                work_done_token: None,
+            },
+        };
+
+        self.call::<lsp::request::ExecuteCommand>(params)
+    }
 }

helix-lsp/src/lib.rs

@@ -58,6 +58,36 @@ pub mod util {
     use super::*;
     use helix_core::{Range, Rope, Transaction};
 
+    /// Converts a diagnostic in the document to [`lsp::Diagnostic`].
+    ///
+    /// Panics when [`pos_to_lsp_pos`] would for an invalid range on the diagnostic.
+    pub fn diagnostic_to_lsp_diagnostic(
+        doc: &Rope,
+        diag: &helix_core::diagnostic::Diagnostic,
+        offset_encoding: OffsetEncoding,
+    ) -> lsp::Diagnostic {
+        use helix_core::diagnostic::Severity::*;
+
+        let range = Range::new(diag.range.start, diag.range.end);
+        let severity = diag.severity.map(|s| match s {
+            Hint => lsp::DiagnosticSeverity::HINT,
+            Info => lsp::DiagnosticSeverity::INFORMATION,
+            Warning => lsp::DiagnosticSeverity::WARNING,
+            Error => lsp::DiagnosticSeverity::ERROR,
+        });
+
+        // TODO: add support for Diagnostic.data
+        lsp::Diagnostic::new(
+            range_to_lsp_range(doc, range, offset_encoding),
+            severity,
+            None,
+            None,
+            diag.message.to_owned(),
+            None,
+            None,
+        )
+    }
+
     /// Converts [`lsp::Position`] to a position in the document.
     ///
     /// Returns `None` if position exceeds document length or an operation overflows.
@@ -66,39 +96,26 @@ pub mod util {
         pos: lsp::Position,
         offset_encoding: OffsetEncoding,
     ) -> Option<usize> {
-        let max_line = doc.lines().count().saturating_sub(1);
         let pos_line = pos.line as usize;
-        let pos_line = if pos_line > max_line {
+        if pos_line > doc.len_lines() - 1 {
             return None;
-        } else {
-            pos_line
-        };
+        }
+
         match offset_encoding {
             OffsetEncoding::Utf8 => {
-                let max_char = doc
-                    .line_to_char(max_line)
-                    .checked_add(doc.line(max_line).len_chars())?;
                 let line = doc.line_to_char(pos_line);
                 let pos = line.checked_add(pos.character as usize)?;
-                if pos <= max_char {
+                if pos <= doc.len_chars() {
                     Some(pos)
                 } else {
                     None
                 }
             }
             OffsetEncoding::Utf16 => {
-                let max_char = doc
-                    .line_to_char(max_line)
-                    .checked_add(doc.line(max_line).len_chars())?;
-                let max_cu = doc.char_to_utf16_cu(max_char);
                 let line = doc.line_to_char(pos_line);
                 let line_start = doc.char_to_utf16_cu(line);
                 let pos = line_start.checked_add(pos.character as usize)?;
-                if pos <= max_cu {
-                    Some(doc.utf16_cu_to_char(pos))
-                } else {
-                    None
-                }
+                doc.try_utf16_cu_to_char(pos).ok()
             }
         }
     }
@@ -182,27 +199,14 @@ pub mod util {
             }),
         )
     }
-
-    /// The result of asking the language server to format the document. This can be turned into a
-    /// `Transaction`, but the advantage of not doing that straight away is that this one is
-    /// `Send` and `Sync`.
-    #[derive(Clone, Debug)]
-    pub struct LspFormatting {
-        pub doc: Rope,
-        pub edits: Vec<lsp::TextEdit>,
-        pub offset_encoding: OffsetEncoding,
-    }
-
-    impl From<LspFormatting> for Transaction {
-        fn from(fmt: LspFormatting) -> Transaction {
-            generate_transaction_from_edits(&fmt.doc, fmt.edits, fmt.offset_encoding)
-        }
-    }
 }
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum MethodCall {
     WorkDoneProgressCreate(lsp::WorkDoneProgressCreateParams),
+    ApplyWorkspaceEdit(lsp::ApplyWorkspaceEditParams),
+    WorkspaceFolders,
+    WorkspaceConfiguration(lsp::ConfigurationParams),
 }
 
 impl MethodCall {
@@ -215,6 +219,19 @@ impl MethodCall {
                     .expect("Failed to parse WorkDoneCreate params");
                 Self::WorkDoneProgressCreate(params)
             }
+            lsp::request::ApplyWorkspaceEdit::METHOD => {
+                let params: lsp::ApplyWorkspaceEditParams = params
+                    .parse()
+                    .expect("Failed to parse ApplyWorkspaceEdit params");
+                Self::ApplyWorkspaceEdit(params)
+            }
+            lsp::request::WorkspaceFoldersRequest::METHOD => Self::WorkspaceFolders,
+            lsp::request::WorkspaceConfiguration::METHOD => {
+                let params: lsp::ConfigurationParams = params
+                    .parse()
+                    .expect("Failed to parse WorkspaceConfiguration params");
+                Self::WorkspaceConfiguration(params)
+            }
             _ => {
                 log::warn!("unhandled lsp request: {}", method);
                 return None;
@@ -243,7 +260,13 @@ impl Notification {
             lsp::notification::PublishDiagnostics::METHOD => {
                 let params: lsp::PublishDiagnosticsParams = params
                     .parse()
-                    .expect("Failed to parse PublishDiagnostics params");
+                    .map_err(|err| {
+                        log::error!(
+                            "received malformed PublishDiagnostic from Language Server: {}",
+                            err
+                        )
+                    })
+                    .ok()?;
 
                 // TODO: need to loop over diagnostics and distinguish them by URI
                 Self::PublishDiagnostics(params)
@@ -319,6 +342,7 @@ impl Registry {
                     &config.command,
                     &config.args,
                     language_config.config.clone(),
+                    &language_config.roots,
                     id,
                 )?;
                 self.incoming.push(UnboundedReceiverStream::new(incoming));
@@ -337,7 +361,10 @@ impl Registry {
                         })
                         .await;
 
-                    value.expect("failed to initialize capabilities");
+                    if let Err(e) = value {
+                        log::error!("failed to initialize language server: {}", e);
+                        return;
+                    }
 
                     // next up, notify<initialized>
                     _client
@@ -385,7 +412,7 @@ impl LspProgressMap {
         Self::default()
     }
 
-    /// Returns a map of all tokens coresponding to the lanaguage server with `id`.
+    /// Returns a map of all tokens corresponding to the language server with `id`.
     pub fn progress_map(&self, id: usize) -> Option<&HashMap<lsp::ProgressToken, ProgressStatus>> {
         self.0.get(&id)
     }
@@ -423,7 +450,7 @@ impl LspProgressMap {
         self.0.get_mut(&id).and_then(|vals| vals.remove(token))
     }
 
-    /// Updates the progess of `token` for server with `id` to `status`, returns the value replaced or `None`.
+    /// Updates the progress of `token` for server with `id` to `status`, returns the value replaced or `None`.
     pub fn update(
         &mut self,
         id: usize,

helix-syntax/Cargo.toml

@@ -1,21 +0,0 @@
-[package]
-name = "helix-syntax"
-version = "0.5.0"
-authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
-edition = "2021"
-license = "MPL-2.0"
-description = "Tree-sitter grammars support"
-categories = ["editor"]
-repository = "https://github.com/helix-editor/helix"
-homepage = "https://helix-editor.com"
-include = ["src/**/*", "languages/**/*", "build.rs", "!**/docs/**/*", "!**/test/**/*", "!**/examples/**/*", "!**/build/**/*"]
-
-[dependencies]
-tree-sitter = "0.20"
-libloading = "0.7"
-anyhow = "1"
-
-[build-dependencies]
-cc = { version = "1" }
-threadpool = { version = "1.0" }
-anyhow = "1"

helix-syntax/build.rs

@@ -1,207 +0,0 @@
-use anyhow::{anyhow, Context, Result};
-use std::fs;
-use std::time::SystemTime;
-use std::{
-    path::{Path, PathBuf},
-    process::Command,
-};
-
-use std::sync::mpsc::channel;
-
-fn collect_tree_sitter_dirs(ignore: &[String]) -> Result<Vec<String>> {
-    let mut dirs = Vec::new();
-    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("languages");
-
-    for entry in fs::read_dir(path)? {
-        let entry = entry?;
-        let path = entry.path();
-
-        if !entry.file_type()?.is_dir() {
-            continue;
-        }
-
-        let dir = path.file_name().unwrap().to_str().unwrap().to_string();
-
-        // filter ignores
-        if ignore.contains(&dir) {
-            continue;
-        }
-        dirs.push(dir)
-    }
-
-    Ok(dirs)
-}
-
-#[cfg(unix)]
-const DYLIB_EXTENSION: &str = "so";
-
-#[cfg(windows)]
-const DYLIB_EXTENSION: &str = "dll";
-
-fn build_library(src_path: &Path, language: &str) -> Result<()> {
-    let header_path = src_path;
-    // let grammar_path = src_path.join("grammar.json");
-    let parser_path = src_path.join("parser.c");
-    let mut scanner_path = src_path.join("scanner.c");
-
-    let scanner_path = if scanner_path.exists() {
-        Some(scanner_path)
-    } else {
-        scanner_path.set_extension("cc");
-        if scanner_path.exists() {
-            Some(scanner_path)
-        } else {
-            None
-        }
-    };
-    let parser_lib_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../runtime/grammars");
-    let mut library_path = parser_lib_path.join(language);
-    library_path.set_extension(DYLIB_EXTENSION);
-
-    let recompile = needs_recompile(&library_path, &parser_path, &scanner_path)
-        .with_context(|| "Failed to compare source and binary timestamps")?;
-
-    if !recompile {
-        return Ok(());
-    }
-    let mut config = cc::Build::new();
-    config.cpp(true).opt_level(2).cargo_metadata(false);
-    let compiler = config.get_compiler();
-    let mut command = Command::new(compiler.path());
-    command.current_dir(src_path);
-    for (key, value) in compiler.env() {
-        command.env(key, value);
-    }
-
-    if cfg!(windows) {
-        command
-            .args(&["/nologo", "/LD", "/I"])
-            .arg(header_path)
-            .arg("/Od")
-            .arg("/utf-8");
-        if let Some(scanner_path) = scanner_path.as_ref() {
-            command.arg(scanner_path);
-        }
-
-        command
-            .arg(parser_path)
-            .arg("/link")
-            .arg(format!("/out:{}", library_path.to_str().unwrap()));
-    } else {
-        command
-            .arg("-shared")
-            .arg("-fPIC")
-            .arg("-fno-exceptions")
-            .arg("-g")
-            .arg("-I")
-            .arg(header_path)
-            .arg("-o")
-            .arg(&library_path)
-            .arg("-O2");
-        if let Some(scanner_path) = scanner_path.as_ref() {
-            if scanner_path.extension() == Some("c".as_ref()) {
-                command.arg("-xc").arg("-std=c99").arg(scanner_path);
-            } else {
-                command.arg(scanner_path);
-            }
-        }
-        command.arg("-xc").arg(parser_path);
-        if cfg!(all(unix, not(target_os = "macos"))) {
-            command.arg("-Wl,-z,relro,-z,now");
-        }
-    }
-
-    let output = command
-        .output()
-        .with_context(|| "Failed to execute C compiler")?;
-    if !output.status.success() {
-        return Err(anyhow!(
-            "Parser compilation failed.\nStdout: {}\nStderr: {}",
-            String::from_utf8_lossy(&output.stdout),
-            String::from_utf8_lossy(&output.stderr)
-        ));
-    }
-
-    Ok(())
-}
-fn needs_recompile(
-    lib_path: &Path,
-    parser_c_path: &Path,
-    scanner_path: &Option<PathBuf>,
-) -> Result<bool> {
-    if !lib_path.exists() {
-        return Ok(true);
-    }
-    let lib_mtime = mtime(lib_path)?;
-    if mtime(parser_c_path)? > lib_mtime {
-        return Ok(true);
-    }
-    if let Some(scanner_path) = scanner_path {
-        if mtime(scanner_path)? > lib_mtime {
-            return Ok(true);
-        }
-    }
-    Ok(false)
-}
-
-fn mtime(path: &Path) -> Result<SystemTime> {
-    Ok(fs::metadata(path)?.modified()?)
-}
-
-fn build_dir(dir: &str, language: &str) {
-    println!("Build language {}", language);
-    if PathBuf::from("languages")
-        .join(dir)
-        .read_dir()
-        .unwrap()
-        .next()
-        .is_none()
-    {
-        eprintln!(
-            "The directory {} is empty, you probably need to use 'git submodule update --init --recursive'?",
-            dir
-        );
-        std::process::exit(1);
-    }
-
-    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
-        .join("languages")
-        .join(dir)
-        .join("src");
-
-    build_library(&path, language).unwrap();
-}
-
-fn main() {
-    let ignore = vec![
-        "tree-sitter-typescript".to_string(),
-        "tree-sitter-haskell".to_string(), // aarch64 failures: https://github.com/tree-sitter/tree-sitter-haskell/issues/34
-        "tree-sitter-ocaml".to_string(),
-    ];
-    let dirs = collect_tree_sitter_dirs(&ignore).unwrap();
-
-    let mut n_jobs = 0;
-    let pool = threadpool::Builder::new().build(); // by going through the builder, it'll use num_cpus
-    let (tx, rx) = channel();
-
-    for dir in dirs {
-        let tx = tx.clone();
-        n_jobs += 1;
-
-        pool.execute(move || {
-            let language = &dir.strip_prefix("tree-sitter-").unwrap();
-            build_dir(&dir, language);
-
-            // report progress
-            tx.send(1).unwrap();
-        });
-    }
-    pool.join();
-    // drop(tx);
-    assert_eq!(rx.try_iter().sum::<usize>(), n_jobs);
-
-    build_dir("tree-sitter-typescript/tsx", "tsx");
-    build_dir("tree-sitter-typescript/typescript", "typescript");
-    build_dir("tree-sitter-ocaml/ocaml", "ocaml");
-    build_dir("tree-sitter-ocaml/interface", "ocaml-interface")
-}