fix build

When fiddling with paths in a :o prompt, one usually would want Ctrl-W to erase a path segment
rather than the whole path. This is how Ctrl-W works in e.g. (neo)vim out of the box.

.envrc

@@ -3,3 +3,4 @@ watch_file flake.lock
 
 # try to use flakes, if it fails use normal nix (ie. shell.nix)
 use flake || use nix
+eval "$shellHook"

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,28 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: C-bug
-assignees: ''
-
----
-
-<!-- Your issue may already be reported!
-Please search on the issue tracker before creating one. -->
-
-### Reproduction steps
-
-<!-- Ideally provide a key sequence and/or asciinema.org recording. --> 
-
-### Environment
-
-- Platform: <!--  macOS / Windows / Linux -->
-- Helix version: <!--  'hx -V' if using a release, 'git describe' if building from master -->
-
-<details><summary>~/.cache/helix/helix.log</summary>
-
-```
-please provide a copy of `~/.cache/helix/helix.log` here if possible, you may need to redact some of the lines
-```
-
-</details>

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -0,0 +1,67 @@
+name: Bug Report
+description: Create a report to help us improve
+labels: C-bug
+body:
+  - type: markdown
+    attributes:
+      value: Thank you for filing a bug report! 🐛
+  - type: textarea
+    id: problem
+    attributes:
+      label: Summary
+      description: >
+        Please provide a short summary of the bug, along with any information
+        you feel relevant to replicate the bug.
+    validations:
+      required: true
+  - type: textarea
+    id: reproduction-steps
+    attributes:
+      label: Reproduction Steps
+      value: |
+        <!-- Ideally provide a key sequence and/or asciinema.org recording. --> 
+
+        I tried this:
+
+        1. `hx`
+
+        I expected this to happen:
+
+        Instead, this happened:
+  - type: textarea
+    id: helix-log
+    attributes:
+      label: Helix log
+      description: See `hx -h` for log file path
+      value: |
+        <details><summary>~/.cache/helix/helix.log</summary>
+
+        ```
+        please provide a copy of `~/.cache/helix/helix.log` here if possible, you may need to redact some of the lines
+        ```
+
+        </details>
+  - type: input
+    id: platform
+    attributes:
+      label: Platform
+      placeholder: Linux / macOS / Windows
+    validations:
+      required: true
+  - type: input
+    id: terminal-emulator
+    attributes:
+      label: Terminal Emulator
+      placeholder: wezterm 20220101-133340-7edc5b5a
+    validations:
+      required: true
+  - type: input
+    id: helix-version
+    attributes:
+      label: Helix Version
+      description: >
+        Helix version (`hx -V` if using a release, `git describe` if building
+        from master)
+      placeholder: "helix 0.6.0 (c0dbd6dc)"
+    validations:
+      required: true

.github/workflows/build.yml

@@ -13,9 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
-        with:
-          submodules: true
 
       - name: Install stable toolchain
         uses: actions-rs/toolchain@v1
@@ -25,22 +23,25 @@ jobs:
           override: true
 
       - name: Cache cargo registry
-        uses: actions/cache@v2.1.7
+        uses: actions/cache@v3
         with:
           path: ~/.cargo/registry
           key: ${{ runner.os }}-v2-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-registry-
 
       - name: Cache cargo index
-        uses: actions/cache@v2.1.7
+        uses: actions/cache@v3
         with:
           path: ~/.cargo/git
           key: ${{ runner.os }}-v2-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-index-
 
       - name: Cache cargo target dir
-        uses: actions/cache@v2.1.7
+        uses: actions/cache@v3
         with:
           path: target
           key: ${{ runner.os }}-v2-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-build-target-
 
       - name: Run cargo check
         uses: actions-rs/cargo@v1
@@ -52,9 +53,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
-        with:
-          submodules: true
 
       - name: Install stable toolchain
         uses: actions-rs/toolchain@v1
@@ -64,27 +63,41 @@ jobs:
           override: true
 
       - name: Cache cargo registry
-        uses: actions/cache@v2.1.7
+        uses: actions/cache@v3
         with:
           path: ~/.cargo/registry
           key: ${{ runner.os }}-v2-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-registry-
 
       - name: Cache cargo index
-        uses: actions/cache@v2.1.7
+        uses: actions/cache@v3
         with:
           path: ~/.cargo/git
           key: ${{ runner.os }}-v2-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-index-
 
       - name: Cache cargo target dir
-        uses: actions/cache@v2.1.7
+        uses: actions/cache@v3
         with:
           path: target
           key: ${{ runner.os }}-v2-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-build-target-
+
+      - name: Copy minimal languages config
+        run: cp .github/workflows/languages.toml ./languages.toml
+
+      - name: Cache test tree-sitter grammar
+        uses: actions/cache@v3
+        with:
+          path: runtime/grammars
+          key: ${{ runner.os }}-v2-tree-sitter-grammars-${{ hashFiles('languages.toml') }}
+          restore-keys: ${{ runner.os }}-v2-tree-sitter-grammars-
 
       - name: Run cargo test
         uses: actions-rs/cargo@v1
         with:
           command: test
+          args: --workspace
 
     strategy:
       matrix:
@@ -96,9 +109,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
-        with:
-          submodules: true
 
       - name: Install stable toolchain
         uses: actions-rs/toolchain@v1
@@ -109,22 +120,25 @@ jobs:
           components: rustfmt, clippy
 
       - name: Cache cargo registry
-        uses: actions/cache@v2.1.7
+        uses: actions/cache@v3
         with:
           path: ~/.cargo/registry
           key: ${{ runner.os }}-v2-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-registry-
 
       - name: Cache cargo index
-        uses: actions/cache@v2.1.7
+        uses: actions/cache@v3
         with:
           path: ~/.cargo/git
           key: ${{ runner.os }}-v2-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-index-
 
       - name: Cache cargo target dir
-        uses: actions/cache@v2.1.7
+        uses: actions/cache@v3
         with:
           path: target
           key: ${{ runner.os }}-v2-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-build-target-
 
       - name: Run cargo fmt
         uses: actions-rs/cargo@v1
@@ -136,16 +150,14 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: clippy
-          args: -- -D warnings
+          args: --all-targets -- -D warnings
 
   docs:
     name: Docs
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
-        with:
-          submodules: true
 
       - name: Install stable toolchain
         uses: actions-rs/toolchain@v1
@@ -155,22 +167,25 @@ jobs:
           override: true
 
       - name: Cache cargo registry
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v3
         with:
           path: ~/.cargo/registry
           key: ${{ runner.os }}-v2-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-registry-
 
       - name: Cache cargo index
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v3
         with:
           path: ~/.cargo/git
           key: ${{ runner.os }}-v2-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-index-
 
       - name: Cache cargo target dir
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v3
         with:
           path: target
           key: ${{ runner.os }}-v2-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-build-target-
 
       - name: Generate docs
         uses: actions-rs/cargo@v1

.github/workflows/cachix.yml

@@ -0,0 +1,26 @@
+# Publish the Nix flake outputs to Cachix
+name: Cachix
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  publish:
+    name: Publish Flake
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout sources
+      uses: actions/checkout@v3
+
+    - name: Install nix
+      uses: cachix/install-nix-action@v17
+
+    - name: Authenticate with Cachix
+      uses: cachix/cachix-action@v10
+      with:
+        name: helix
+        authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
+    - name: Build nix flake
+      run: nix build

.github/workflows/gh-pages.yml

@@ -4,12 +4,14 @@ on:
   push:
     branches:
       - master
+    tags:
+      - '*'
 
 jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Setup mdBook
         uses: peaceiris/actions-mdbook@v1
@@ -18,10 +20,22 @@ jobs:
           # mdbook-version: '0.4.8'
 
       - run: mdbook build book
+      
+      - name: Set output directory
+        run: |
+          OUTDIR=$(basename ${{ github.ref }})
+          echo "OUTDIR=$OUTDIR" >> $GITHUB_ENV
 
       - name: Deploy
         uses: peaceiris/actions-gh-pages@v3
-        if: github.ref == 'refs/heads/master'
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./book/book
+          destination_dir: ./${{ env.OUTDIR }}
+
+      - name: Deploy stable
+        uses: peaceiris/actions-gh-pages@v3
+        if: startswith(github.ref, 'refs/tags/')
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./book/book

.github/workflows/languages.toml

@@ -0,0 +1,26 @@
+# This languages.toml is used for testing in CI.
+
+[[language]]
+name = "rust"
+scope = "source.rust"
+injection-regex = "rust"
+file-types = ["rs"]
+comment-token = "//"
+roots = ["Cargo.toml", "Cargo.lock"]
+indent = { tab-width = 4, unit = "    " }
+
+[[grammar]]
+name = "rust"
+source = { git = "https://github.com/tree-sitter/tree-sitter-rust", rev = "a360da0a29a19c281d08295a35ecd0544d2da211" }
+
+[[language]]
+name = "nix"
+scope = "source.nix"
+injection-regex = "nix"
+file-types = ["nix"]
+shebangs = []
+roots = []
+comment-token = "#"
+
+# A grammar entry is not necessary for this language - it is only used for
+# testing TOML merging behavior.

.github/workflows/release.yml

@@ -1,32 +1,81 @@
 name: Release
 on:
-  # schedule:
-  # - cron: '0 0 * * *' # midnight UTC
-
   push:
     tags:
-    - 'v[0-9]+.[0-9]+.[0-9]+'
+    - '[0-9]+.[0-9]+'
-    ## - release
+    - '[0-9]+.[0-9]+.[0-9]+'
 
 jobs:
+  fetch-grammars:
+    name: Fetch Grammars
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+
+      - name: Install stable toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+
+      - name: Cache cargo registry
+        uses: actions/cache@v3
+        with:
+          path: ~/.cargo/registry
+          key: ${{ runner.os }}-v2-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-registry-
+
+      - name: Cache cargo index
+        uses: actions/cache@v3
+        with:
+          path: ~/.cargo/git
+          key: ${{ runner.os }}-v2-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-index-
+
+      - name: Cache cargo target dir
+        uses: actions/cache@v3
+        with:
+          path: target
+          key: ${{ runner.os }}-v2-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-v2-cargo-build-target-
+
+      - name: Fetch tree-sitter grammars
+        uses: actions-rs/cargo@v1
+        env:
+          HELIX_DISABLE_AUTO_GRAMMAR_BUILD: yes
+        with:
+          command: run
+          args: -- --grammar fetch
+
+      - name: Bundle grammars
+        run: tar cJf grammars.tar.xz -C runtime/grammars/sources .
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: grammars
+          path: grammars.tar.xz
+
   dist:
     name: Dist
+    needs: [fetch-grammars]
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false # don't fail other jobs if one fails
       matrix:
-        build: [x86_64-linux, aarch64-linux, x86_64-macos, x86_64-windows] #, x86_64-win-gnu, win32-msvc
+        build: [x86_64-linux, x86_64-macos, x86_64-windows] #, x86_64-win-gnu, win32-msvc
         include:
         - build: x86_64-linux
           os: ubuntu-20.04
           rust: stable
           target: x86_64-unknown-linux-gnu
           cross: false
-        - build: aarch64-linux
+        # - build: aarch64-linux
-          os: ubuntu-20.04
+        #   os: ubuntu-20.04
-          rust: stable
+        #   rust: stable
-          target: aarch64-unknown-linux-gnu
+        #   target: aarch64-unknown-linux-gnu
-          cross: true
+        #   cross: true
         - build: x86_64-macos
           os: macos-latest
           rust: stable
@@ -52,9 +101,16 @@ jobs:
 
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
-        with:
+
-          submodules: true
+      - name: Download grammars
+        uses: actions/download-artifact@v3
+
+      - name: Move grammars under runtime
+        if: "!startsWith(matrix.os, 'windows')"
+        run: |
+          mkdir -p runtime/grammars/sources
+          tar xJf grammars/grammars.tar.xz -C runtime/grammars/sources
 
       - name: Install ${{ matrix.rust }} toolchain
         uses: actions-rs/toolchain@v1
@@ -69,7 +125,7 @@ jobs:
         with:
           use-cross: ${{ matrix.cross }}
           command: test
-          args: --release --locked --target ${{ matrix.target }}
+          args: --release --locked --target ${{ matrix.target }} --workspace
 
       - name: Build release binary
         uses: actions-rs/cargo@v1
@@ -91,18 +147,63 @@ jobs:
             aarch64-linux-gnu-strip \
             /target/${{ matrix.target }}/release/hx
 
+      - name: Build AppImage
+        shell: bash
+        if: matrix.build == 'x86_64-linux'
+        run: |
+          mkdir dist
+
+          name=dev
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            name=${GITHUB_REF:10}
+          fi
+
+          export VERSION="$name"
+          export ARCH=x86_64
+          export APP=helix
+          export OUTPUT="helix-$VERSION-$ARCH.AppImage"
+          export UPDATE_INFORMATION="gh-releases-zsync|$GITHUB_REPOSITORY_OWNER|helix|latest|$APP-*-$ARCH.AppImage.zsync"
+
+          mkdir -p "$APP.AppDir"/usr/{bin,lib/helix}
+
+          cp "target/${{ matrix.target }}/release/hx" "$APP.AppDir/usr/bin/hx"
+          rm -rf runtime/grammars/sources
+          cp -r runtime "$APP.AppDir/usr/lib/helix/runtime"
+
+          cat << 'EOF' > "$APP.AppDir/AppRun"
+          #!/bin/sh
+
+          APPDIR="$(dirname "$(readlink -f "${0}")")"
+          HELIX_RUNTIME="$APPDIR/usr/lib/helix/runtime" exec "$APPDIR/usr/bin/hx" "$@"
+          EOF
+          chmod 755 "$APP.AppDir/AppRun"
+
+          curl -Lo linuxdeploy-x86_64.AppImage \
+              https://github.com/linuxdeploy/linuxdeploy/releases/download/continuous/linuxdeploy-x86_64.AppImage
+          chmod +x linuxdeploy-x86_64.AppImage
+
+          ./linuxdeploy-x86_64.AppImage \
+              --appdir "$APP.AppDir" -d contrib/Helix.desktop \
+              -i contrib/helix.png --output appimage
+
+          mv "$APP-$VERSION-$ARCH.AppImage" \
+              "$APP-$VERSION-$ARCH.AppImage.zsync" dist
+
       - name: Build archive
         shell: bash
         run: |
-          mkdir dist
+          mkdir -p dist
           if [ "${{ matrix.os }}" = "windows-2019" ]; then
             cp "target/${{ matrix.target }}/release/hx.exe" "dist/"
           else
             cp "target/${{ matrix.target }}/release/hx" "dist/"
           fi
+          if [ -d runtime/grammars/sources ]; then
+            rm -rf runtime/grammars/sources
+          fi
           cp -r runtime dist
 
-      - uses: actions/upload-artifact@v2.3.1
+      - uses: actions/upload-artifact@v3
         with:
           name: bins-${{ matrix.build }}
           path: dist
@@ -113,20 +214,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
-        with:
-          submodules: false
 
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v3
-        # with:
-        #   path: dist
-      # - run: ls -al ./dist
-      - run: ls -al bins-*
 
       - name: Calculate tag name
         run: |
           name=dev
-          if [[ $GITHUB_REF == refs/tags/v* ]]; then
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
             name=${GITHUB_REF:10}
           fi
           echo ::set-output name=val::$name
@@ -138,8 +233,13 @@ jobs:
         run: |
           set -ex
 
-          rm -rf tmp
+          source="$(pwd)"
-          mkdir tmp
+          mkdir -p runtime/grammars/sources
+          tar xJf grammars/grammars.tar.xz -C runtime/grammars/sources
+          rm -rf grammars
+
+          cd "$(mktemp -d)"
+          mv $source/bins-* .
           mkdir dist
 
           for dir in bins-* ; do
@@ -148,19 +248,28 @@ jobs:
                   exe=".exe"
               fi
               pkgname=helix-$TAG-$platform
-              mkdir tmp/$pkgname
+              mkdir $pkgname
-              cp LICENSE README.md tmp/$pkgname
+              cp $source/LICENSE $source/README.md $pkgname
-              mv bins-$platform/runtime tmp/$pkgname/
+              mkdir $pkgname/contrib
-              mv bins-$platform/hx$exe tmp/$pkgname
+              cp -r $source/contrib/completion $pkgname/contrib
-              chmod +x tmp/$pkgname/hx$exe
+              mv bins-$platform/runtime $pkgname/
+              mv bins-$platform/hx$exe $pkgname
+              chmod +x $pkgname/hx$exe
+
+              if [[ "$platform" = "x86_64-linux" ]]; then
+                  mv bins-$platform/helix-*.AppImage* dist/
+              fi
 
               if [ "$exe" = "" ]; then
-                  tar cJf dist/$pkgname.tar.xz -C tmp $pkgname
+                  tar cJf dist/$pkgname.tar.xz $pkgname
               else
-                  (cd tmp && 7z a -r ../dist/$pkgname.zip $pkgname)
+                  7z a -r dist/$pkgname.zip $pkgname
               fi
           done
 
+          tar cJf dist/helix-$TAG-source.tar.xz -C $source .
+          mv dist $source/
+
       - name: Upload binaries to release
         uses: svenstaro/upload-release-action@v2
         with:

.gitmodules

@@ -1,188 +0,0 @@
-[submodule "helix-syntax/languages/tree-sitter-cpp"]
-	path = helix-syntax/languages/tree-sitter-cpp
-	url = https://github.com/tree-sitter/tree-sitter-cpp
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-javascript"]
-	path = helix-syntax/languages/tree-sitter-javascript
-	url = https://github.com/tree-sitter/tree-sitter-javascript
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-julia"]
-	path = helix-syntax/languages/tree-sitter-julia
-	url = https://github.com/tree-sitter/tree-sitter-julia
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-python"]
-	path = helix-syntax/languages/tree-sitter-python
-	url = https://github.com/tree-sitter/tree-sitter-python
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-typescript"]
-	path = helix-syntax/languages/tree-sitter-typescript
-	url = https://github.com/tree-sitter/tree-sitter-typescript
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-agda"]
-	path = helix-syntax/languages/tree-sitter-agda
-	url = https://github.com/tree-sitter/tree-sitter-agda
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-go"]
-	path = helix-syntax/languages/tree-sitter-go
-	url = https://github.com/tree-sitter/tree-sitter-go
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-ruby"]
-	path = helix-syntax/languages/tree-sitter-ruby
-	url = https://github.com/tree-sitter/tree-sitter-ruby
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-java"]
-	path = helix-syntax/languages/tree-sitter-java
-	url = https://github.com/tree-sitter/tree-sitter-java
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-php"]
-	path = helix-syntax/languages/tree-sitter-php
-	url = https://github.com/tree-sitter/tree-sitter-php
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-html"]
-	path = helix-syntax/languages/tree-sitter-html
-	url = https://github.com/tree-sitter/tree-sitter-html
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-scala"]
-	path = helix-syntax/languages/tree-sitter-scala
-	url = https://github.com/tree-sitter/tree-sitter-scala
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-bash"]
-	path = helix-syntax/languages/tree-sitter-bash
-	url = https://github.com/tree-sitter/tree-sitter-bash
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-rust"]
-	path = helix-syntax/languages/tree-sitter-rust
-	url = https://github.com/tree-sitter/tree-sitter-rust
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-json"]
-	path = helix-syntax/languages/tree-sitter-json
-	url = https://github.com/tree-sitter/tree-sitter-json
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-css"]
-	path = helix-syntax/languages/tree-sitter-css
-	url = https://github.com/tree-sitter/tree-sitter-css
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-c-sharp"]
-	path = helix-syntax/languages/tree-sitter-c-sharp
-	url = https://github.com/tree-sitter/tree-sitter-c-sharp
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-c"]
-	path = helix-syntax/languages/tree-sitter-c
-	url = https://github.com/tree-sitter/tree-sitter-c
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-haskell"]
-	path = helix-syntax/languages/tree-sitter-haskell
-	url = https://github.com/tree-sitter/tree-sitter-haskell
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-swift"]
-	path = helix-syntax/languages/tree-sitter-swift
-	url = https://github.com/tree-sitter/tree-sitter-swift
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-toml"]
-	path = helix-syntax/languages/tree-sitter-toml
-	url = https://github.com/ikatyang/tree-sitter-toml
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-elixir"]
-	path = helix-syntax/languages/tree-sitter-elixir
-	url = https://github.com/elixir-lang/tree-sitter-elixir
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-nix"]
-	path = helix-syntax/languages/tree-sitter-nix
-	url = https://github.com/cstrahan/tree-sitter-nix
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-latex"]
-	path = helix-syntax/languages/tree-sitter-latex
-	url = https://github.com/latex-lsp/tree-sitter-latex
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-ledger"]
-	path = helix-syntax/languages/tree-sitter-ledger
-	url = https://github.com/cbarrete/tree-sitter-ledger
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-protobuf"]
-	path = helix-syntax/languages/tree-sitter-protobuf
-	url = https://github.com/yusdacra/tree-sitter-protobuf.git
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-ocaml"]
-	path = helix-syntax/languages/tree-sitter-ocaml
-	url = https://github.com/tree-sitter/tree-sitter-ocaml
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-lua"]
-	path = helix-syntax/languages/tree-sitter-lua
-	url = https://github.com/nvim-treesitter/tree-sitter-lua
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-yaml"]
-	path = helix-syntax/languages/tree-sitter-yaml
-	url = https://github.com/ikatyang/tree-sitter-yaml
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-zig"]
-	path = helix-syntax/languages/tree-sitter-zig
-	url = https://github.com/maxxnino/tree-sitter-zig
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-svelte"]
-	path = helix-syntax/languages/tree-sitter-svelte
-	url = https://github.com/Himujjal/tree-sitter-svelte
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-vue"]
-	path = helix-syntax/languages/tree-sitter-vue
-	url = https://github.com/ikatyang/tree-sitter-vue
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-tsq"]
-	path = helix-syntax/languages/tree-sitter-tsq
-	url = https://github.com/tree-sitter/tree-sitter-tsq
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-cmake"]
-	path = helix-syntax/languages/tree-sitter-cmake
-	url = https://github.com/uyha/tree-sitter-cmake
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-glsl"]
-	path = helix-syntax/languages/tree-sitter-glsl
-	url = https://github.com/theHamsta/tree-sitter-glsl.git
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-perl"]
-	path = helix-syntax/languages/tree-sitter-perl
-	url = https://github.com/ganezdragon/tree-sitter-perl
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-comment"]
-	path = helix-syntax/languages/tree-sitter-comment
-	url = https://github.com/stsewd/tree-sitter-comment
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-wgsl"]
-	path = helix-syntax/languages/tree-sitter-wgsl
-	url = https://github.com/szebniok/tree-sitter-wgsl
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-llvm"]
-	path = helix-syntax/languages/tree-sitter-llvm
-	url = https://github.com/benwilliamgraham/tree-sitter-llvm
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-markdown"]
-	path = helix-syntax/languages/tree-sitter-markdown
-	url = https://github.com/MDeiml/tree-sitter-markdown
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-dart"]
-	path = helix-syntax/languages/tree-sitter-dart
-	url = https://github.com/UserNobody14/tree-sitter-dart.git
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-dockerfile"]
-	path = helix-syntax/languages/tree-sitter-dockerfile
-	url = https://github.com/camdencheek/tree-sitter-dockerfile.git
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-fish"]
-	path = helix-syntax/languages/tree-sitter-fish
-	url = https://github.com/ram02z/tree-sitter-fish
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-git-commit"]
-	path = helix-syntax/languages/tree-sitter-git-commit
-	url = https://github.com/the-mikedavis/tree-sitter-git-commit.git
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-git-diff"]
-	path = helix-syntax/languages/tree-sitter-git-diff
-	url = https://github.com/the-mikedavis/tree-sitter-git-diff.git
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-tablegen"]
-	path = helix-syntax/languages/tree-sitter-tablegen
-	url = https://github.com/Flakebi/tree-sitter-tablegen
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-git-rebase"]
-	path = helix-syntax/languages/tree-sitter-git-rebase
-	url = https://github.com/the-mikedavis/tree-sitter-git-rebase.git
-	shallow = true

CHANGELOG.md

@@ -1,3 +1,285 @@
+# 22.03 (2022-03-28)
+
+A big shout out to all the contributors! We had 51 contributors in this release.
+
+This release is particularly large and featureful. Check out some of the
+highlights in the [news section](https://helix-editor.com/news/release-22-03-highlights/).
+
+As usual, the following is a summary of each of the changes since the last release.
+For the full log, check out the [git log](https://github.com/helix-editor/helix/compare/v0.6.0..22.03).
+
+Breaking changes:
+
+- LSP config now lives under `editor.lsp` ([#1868](https://github.com/helix-editor/helix/pull/1868))
+- Expand-selection was moved from `]o` to `Alt-h` ([#1495](https://github.com/helix-editor/helix/pull/1495))
+
+Features:
+
+- Experimental Debug Adapter Protocol (DAP) support ([#574](https://github.com/helix-editor/helix/pull/574))
+- Primary cursor shape may now be customized per mode ([#1154](https://github.com/helix-editor/helix/pull/1154))
+- Overhaul incremental highlights and enable combined injections ([`6728344..4080341`](https://github.com/helix-editor/helix/compare/6728344..4080341))
+- Allow specifying file start position ([#445](https://github.com/helix-editor/helix/pull/445), [#1676](https://github.com/helix-editor/helix/pull/1676))
+- Dynamic line numbers ([#1522](https://github.com/helix-editor/helix/pull/1522))
+- Show an info box with the contents of registers ([#980](https://github.com/helix-editor/helix/pull/980))
+- Wrap-around behavior during search is now configurable ([#1516](https://github.com/helix-editor/helix/pull/1516))
+- Tree-sitter textobjects motions for classes, functions, and parameters ([#1619](https://github.com/helix-editor/helix/pull/1619), [#1708](https://github.com/helix-editor/helix/pull/1708), [#1805](https://github.com/helix-editor/helix/pull/1805))
+- Command palette: a picker for available commands ([#1400](https://github.com/helix-editor/helix/pull/1400))
+- LSP `workspace/configuration` and `workspace/didChangeConfiguration` support ([#1684](https://github.com/helix-editor/helix/pull/1684))
+- `hx --health [LANG]` command ([#1669](https://github.com/helix-editor/helix/pull/1669))
+- Refactor of the tree-sitter grammar system ([#1659](https://github.com/helix-editor/helix/pull/1659))
+  - All submodules have been removed
+  - New `hx --grammar {fetch|build}` flags for fetching and building tree-sitter grammars
+  - A custom grammar selection may now be declared with the `use-grammars` key in `languages.toml`
+
+Commands:
+
+- `:cquit!` - quit forcefully with a non-zero exit-code ([#1414](https://github.com/helix-editor/helix/pull/1414))
+- `shrink_selection` - shrink the selection to a child tree-sitter node (`Alt-j`, [#1340](https://github.com/helix-editor/helix/pull/1340))
+- `:tree-sitter-subtree` - show the tree-sitter subtree under the primary selection ([#1453](https://github.com/helix-editor/helix/pull/1453), [#1524](https://github.com/helix-editor/helix/pull/1524))
+- Add `Alt-Backspace`, `Alt-<`, `Alt->`, and `Ctrl-j` to insert mode ([#1441](https://github.com/helix-editor/helix/pull/1441))
+- `select_next_sibling`, `select_prev_sibling` - select next and previous tree-sitter nodes (`Alt-l` and `Alt-h`, [#1495](https://github.com/helix-editor/helix/pull/1495))
+- `:buffer-close-all`, `:buffer-close-all!`, `:buffer-close-others`, and `:buffer-close-others!` ([#1677](https://github.com/helix-editor/helix/pull/1677))
+- `:vsplit-new` and `:hsplit-new` - open vertical and horizontal splits with new scratch buffers ([#1763](https://github.com/helix-editor/helix/pull/1763))
+- `:open-config` to open the config file and `:refresh-config` to refresh config after changes ([#1771](https://github.com/helix-editor/helix/pull/1771), [#1803](https://github.com/helix-editor/helix/pull/1803))
+
+Usability improvements and fixes:
+
+- Prevent `:cquit` from ignoring unsaved changes ([#1414](https://github.com/helix-editor/helix/pull/1414))
+- Scrolling view keeps selections ([#1420](https://github.com/helix-editor/helix/pull/1420))
+- Only use shellwords parsing on unix platforms ([`7767703`](https://github.com/helix-editor/helix/commit/7767703))
+- Fix slash in search selector status message ([#1449](https://github.com/helix-editor/helix/pull/1449))
+- Use `std::path::MAIN_SEPARATOR` to determine completion ([`3e4f815`](https://github.com/helix-editor/helix/commit/3e4f815))
+- Expand to current node with `expand_selection` when the node has no children ([#1454](https://github.com/helix-editor/helix/pull/1454))
+- Add vertical and horizontal splits to the buffer picker ([#1502](https://github.com/helix-editor/helix/pull/1502))
+- Use the correct language ID for JavaScript & TypeScript LSP ([#1466](https://github.com/helix-editor/helix/pull/1466))
+- Run format command for all buffers being written ([#1444](https://github.com/helix-editor/helix/pull/1444))
+- Fix panics during resizing ([#1408](https://github.com/helix-editor/helix/pull/1408))
+- Fix auto-pairs with CRLF ([#1470](https://github.com/helix-editor/helix/pull/1470))
+- Fix picker scrolling when the bottom is reached ([#1567](https://github.com/helix-editor/helix/pull/1567))
+- Use markup themes for the markdown component ([#1363](https://github.com/helix-editor/helix/pull/1363))
+- Automatically commit changes to history if not in insert mode ([`2a7ae96`](https://github.com/helix-editor/helix/commit/2a7ae96))
+- Render code-actions as a menu and add padding to popup ([`094a0aa`](https://github.com/helix-editor/helix/commit/094a0aa))
+- Only render menu scrollbar if the menu doesn't fit ([`f10a06f`](https://github.com/helix-editor/helix/commit/f10a06f), [`36b975c`](https://github.com/helix-editor/helix/commit/36b975c))
+- Parse git revision instead of tag for version ([`d3221b0`](https://github.com/helix-editor/helix/commit/d3221b0), [#1674](https://github.com/helix-editor/helix/pull/1674))
+- Fix incorrect last modified buffer ([#1621](https://github.com/helix-editor/helix/pull/1621))
+- Add `PageUp`, `PageDown`, `Ctrl-u`, `Ctrl-d`, `Home`, `End` bindings to the file picker ([#1612](https://github.com/helix-editor/helix/pull/1612))
+- Display buffer IDs in the buffer picker ([#1134](https://github.com/helix-editor/helix/pull/1134))
+- Allow multi-line prompt documentation ([`2af0432`](https://github.com/helix-editor/helix/commit/2af0432))
+- Ignore the `.git` directory from the file picker ([#1604](https://github.com/helix-editor/helix/pull/1604))
+- Allow separate styling for markup heading levels ([#1618](https://github.com/helix-editor/helix/pull/1618))
+- Automatically close popups ([#1285](https://github.com/helix-editor/helix/pull/1285))
+- Allow auto-pairs tokens to be configured ([#1624](https://github.com/helix-editor/helix/pull/1624))
+- Don't indent empty lines in `indent` command ([#1653](https://github.com/helix-editor/helix/pull/1653))
+- Ignore `Enter` keypress when a menu has no selection ([#1704](https://github.com/helix-editor/helix/pull/1704))
+- Show errors when surround deletions and replacements fail ([#1709](https://github.com/helix-editor/helix/pull/1709))
+- Show infobox hints for `mi` and `ma` ([#1686](https://github.com/helix-editor/helix/pull/1686))
+- Highlight matching text in file picker suggestions ([#1635](https://github.com/helix-editor/helix/pull/1635))
+- Allow capturing multiple nodes in textobject queries ([#1611](https://github.com/helix-editor/helix/pull/1611))
+- Make repeat operator work with completion edits ([#1640](https://github.com/helix-editor/helix/pull/1640))
+- Save to the jumplist when searching ([#1718](https://github.com/helix-editor/helix/pull/1718))
+- Fix bug with auto-replacement of components in compositor ([#1711](https://github.com/helix-editor/helix/pull/1711))
+- Use Kakoune logic for `align_selection` ([#1675](https://github.com/helix-editor/helix/pull/1675))
+- Fix `follows` for `nixpkgs` in `flake.nix` ([#1729](https://github.com/helix-editor/helix/pull/1729))
+- Performance improvements for the picker ([`78fba86`](https://github.com/helix-editor/helix/commit/78fba86))
+- Rename infobox theme scopes ([#1741](https://github.com/helix-editor/helix/pull/1741))
+- Fallback to broader scopes if a theme scope is not found ([#1714](https://github.com/helix-editor/helix/pull/1714))
+- Add arrow-keys bindings for tree-sitter sibling selection commands ([#1724](https://github.com/helix-editor/helix/pull/1724))
+- Fix a bug in LSP when creating a file in a folder that does not exist ([#1775](https://github.com/helix-editor/helix/pull/1775))
+- Use `^` and `$` regex location assertions for search ([#1793](https://github.com/helix-editor/helix/pull/1793))
+- Fix register names in `insert_register` command ([#1751](https://github.com/helix-editor/helix/pull/1751))
+- Perform extend line for all selections ([#1804](https://github.com/helix-editor/helix/pull/1804))
+- Prevent panic when moving in an empty picker ([#1786](https://github.com/helix-editor/helix/pull/1786))
+- Fix line number calculations for non CR/CRLF line breaks ([`b4a282f`](https://github.com/helix-editor/helix/commit/b4a282f), [`0b96201`](https://github.com/helix-editor/helix/commit/0b96201))
+- Deploy documentation for `master` builds separately from release docs ([#1783](https://github.com/helix-editor/helix/pull/1783))
+
+Themes:
+
+- Add everforest_light ([#1412](https://github.com/helix-editor/helix/pull/1412))
+- Add gruvbox_light ([#1509](https://github.com/helix-editor/helix/pull/1509))
+- Add modified background to dracula popup ([#1434](https://github.com/helix-editor/helix/pull/1434))
+- Markup support for monokai pro themes ([#1553](https://github.com/helix-editor/helix/pull/1553))
+- Markup support for dracula theme ([#1554](https://github.com/helix-editor/helix/pull/1554))
+- Add `tag` to gruvbox theme ([#1555](https://github.com/helix-editor/helix/pull/1555))
+- Markup support for remaining themes ([#1525](https://github.com/helix-editor/helix/pull/1525))
+- Serika light and dark ([#1566](https://github.com/helix-editor/helix/pull/1566))
+- Fix rose_pine and rose_pine_dawn popup background color ([#1606](https://github.com/helix-editor/helix/pull/1606))
+- Fix hover menu item text color in base16 themes ([#1668](https://github.com/helix-editor/helix/pull/1668))
+- Update markup heading styles for everforest ([#1687](https://github.com/helix-editor/helix/pull/1687))
+- Update markup heading styles for rose_pine themes ([#1706](https://github.com/helix-editor/helix/pull/1706))
+- Style bogster cursors ([`6a6a9ab`](https://github.com/helix-editor/helix/commit/6a6a9ab))
+- Fix `ui.selection` in rose_pine themes ([#1716](https://github.com/helix-editor/helix/pull/1716))
+- Use distinct colors for cursor and matched pair in gruvbox ([#1791](https://github.com/helix-editor/helix/pull/1791))
+- Improve colors for `ui.cursor.match` capture in some themes ([#1862](https://github.com/helix-editor/helix/pull/1862))
+
+LSP:
+
+- Add default language server for JavaScript ([#1457](https://github.com/helix-editor/helix/pull/1457))
+- Add `pom.xml` as maven root directory marker ([#1496](https://github.com/helix-editor/helix/pull/1496))
+- Haskell LSP ([#1556](https://github.com/helix-editor/helix/pull/1556))
+- C-sharp LSP support ([#1788](https://github.com/helix-editor/helix/pull/1788))
+- Clean up Julia LSP config ([#1811](https://github.com/helix-editor/helix/pull/1811))
+
+New Languages:
+
+- llvm-mir ([#1398](https://github.com/helix-editor/helix/pull/1398))
+- regex ([#1362](https://github.com/helix-editor/helix/pull/1362))
+- Make ([#1433](https://github.com/helix-editor/helix/pull/1433), [#1661](https://github.com/helix-editor/helix/pull/1661))
+- git-config ([#1426](https://github.com/helix-editor/helix/pull/1426))
+- Lean ([#1422](https://github.com/helix-editor/helix/pull/1422))
+- Elm ([#1514](https://github.com/helix-editor/helix/pull/1514))
+- GraphQL ([#1515](https://github.com/helix-editor/helix/pull/1515))
+- Twig ([#1602](https://github.com/helix-editor/helix/pull/1602))
+- Rescript ([#1616](https://github.com/helix-editor/helix/pull/1616), [#1863](https://github.com/helix-editor/helix/pull/1863))
+- Erlang ([#1657](https://github.com/helix-editor/helix/pull/1657))
+- Kotlin ([#1689](https://github.com/helix-editor/helix/pull/1689))
+- HCL ([#1705](https://github.com/helix-editor/helix/pull/1705), [#1726](https://github.com/helix-editor/helix/pull/1726))
+- Org ([#1845](https://github.com/helix-editor/helix/pull/1845))
+- Solidity ([#1848](https://github.com/helix-editor/helix/pull/1848), [#1854](https://github.com/helix-editor/helix/pull/1854))
+
+Updated Languages and Queries:
+
+- Textobject and indent queries for c and cpp ([#1293](https://github.com/helix-editor/helix/pull/1293))
+- Fix null and boolean constant highlights for nix ([#1428](https://github.com/helix-editor/helix/pull/1428))
+- Capture markdown link text as `markup.link.text` ([#1456](https://github.com/helix-editor/helix/pull/1456))
+- Update and re-enable Haskell ([#1417](https://github.com/helix-editor/helix/pull/1417), [#1520](https://github.com/helix-editor/helix/pull/1520))
+- Update Go with generics support ([`ddbf036`](https://github.com/helix-editor/helix/commit/ddbf036))
+- Use `tree-sitter-css` for SCSS files ([#1507](https://github.com/helix-editor/helix/pull/1507))
+- Update Zig ([#1501](https://github.com/helix-editor/helix/pull/1501))
+- Update PHP ([#1521](https://github.com/helix-editor/helix/pull/1521))
+- Expand language support for comment injections ([#1527](https://github.com/helix-editor/helix/pull/1527))
+- Use tree-sitter-bash for `.zshrc` and `.bashrc` ([`7d51042`](https://github.com/helix-editor/helix/commit/7d51042))
+- Use tree-sitter-bash for `.bash_profile` ([#1571](https://github.com/helix-editor/helix/pull/1571))
+- Use tree-sitter-bash for `.zshenv` and ZSH files ([#1574](https://github.com/helix-editor/helix/pull/1574))
+- IEx ([#1576](https://github.com/helix-editor/helix/pull/1576))
+- Textobject queries for PHP ([#1601](https://github.com/helix-editor/helix/pull/1601))
+- C-sharp highlight query improvements ([#1795](https://github.com/helix-editor/helix/pull/1795))
+- Git commit performance has been improved on large verbose commits ([#1838](https://github.com/helix-editor/helix/pull/1838))
+
+Packaging:
+
+- The submodules system has been replaced with command-line flags for fetching and building tree-sitter grammars ([#1659](https://github.com/helix-editor/helix/pull/1659))
+- Flake outputs are pushed to Cachix on each push to `master` ([#1721](https://github.com/helix-editor/helix/pull/1721))
+- Update flake's `nix-cargo-integration` to depend on `dream2nix` ([#1758](https://github.com/helix-editor/helix/pull/1758))
+
+# 0.6.0 (2022-01-04)
+
+Happy new year and a big shout out to all the contributors! We had 55 contributors in this release.
+
+Helix has popped up in DPorts and Fedora Linux via COPR ([#1270](https://github.com/helix-editor/helix/pull/1270))
+
+As usual the following is a brief summary, refer to the git history for a full log:
+
+Breaking changes:
+
+- fix: Normalize backtab into shift-tab
+
+Features:
+
+- Macros ([#1234](https://github.com/helix-editor/helix/pull/1234))
+- Add reverse search functionality ([#958](https://github.com/helix-editor/helix/pull/958))
+- Allow keys to be mapped to sequences of commands ([#589](https://github.com/helix-editor/helix/pull/589))
+- Make it possible to keybind TypableCommands ([#1169](https://github.com/helix-editor/helix/pull/1169))
+- Detect workspace root using language markers ([#1370](https://github.com/helix-editor/helix/pull/1370))
+- Add WORD textobject ([#991](https://github.com/helix-editor/helix/pull/991))
+- Add LSP rename_symbol (`space-r`) ([#1011](https://github.com/helix-editor/helix/pull/1011))
+- Added workspace_symbol_picker ([#1041](https://github.com/helix-editor/helix/pull/1041))
+- Detect filetype from shebang line ([#1001](https://github.com/helix-editor/helix/pull/1001))
+- Allow piping from stdin into a buffer on startup ([#996](https://github.com/helix-editor/helix/pull/996))
+- Add auto pairs for same-char pairs ([#1219](https://github.com/helix-editor/helix/pull/1219))
+- Update settings at runtime ([#798](https://github.com/helix-editor/helix/pull/798))
+- Enable thin LTO ([`cccc194`](https://github.com/helix-editor/helix/commit/cccc194))
+
+Commands:
+
+- `:wonly` -- window only ([#1057](https://github.com/helix-editor/helix/pull/1057))
+- buffer-close (`:bc`, `:bclose`) ([#1035](https://github.com/helix-editor/helix/pull/1035))
+- Add `:<line>` and `:goto <line>` commands ([#1128](https://github.com/helix-editor/helix/pull/1128))
+- `:sort` command ([#1288](https://github.com/helix-editor/helix/pull/1288))
+- Add m textobject for pair under cursor ([#961](https://github.com/helix-editor/helix/pull/961))
+- Implement "Goto next buffer / Goto previous buffer" commands ([#950](https://github.com/helix-editor/helix/pull/950))
+- Implement "Goto last modification" command ([#1067](https://github.com/helix-editor/helix/pull/1067))
+- Add trim_selections command ([#1092](https://github.com/helix-editor/helix/pull/1092))
+- Add movement shortcut for history ([#1088](https://github.com/helix-editor/helix/pull/1088))
+- Add command to inc/dec number under cursor ([#1027](https://github.com/helix-editor/helix/pull/1027))
+  - Add support for dates for increment/decrement
+- Align selections (`&`) ([#1101](https://github.com/helix-editor/helix/pull/1101))
+- Implement no-yank delete/change ([#1099](https://github.com/helix-editor/helix/pull/1099))
+- Implement black hole register ([#1165](https://github.com/helix-editor/helix/pull/1165))
+- `gf` as goto_file (`gf`) ([#1102](https://github.com/helix-editor/helix/pull/1102))
+- Add last modified file (`gm`) ([#1093](https://github.com/helix-editor/helix/pull/1093))
+- ensure_selections_forward ([#1393](https://github.com/helix-editor/helix/pull/1393))
+- Readline style insert mode ([#1039](https://github.com/helix-editor/helix/pull/1039))
+
+Usability improvements and fixes:
+
+- Detect filetype on `:write` ([#1141](https://github.com/helix-editor/helix/pull/1141))
+- Add single and double quotes to matching pairs ([#995](https://github.com/helix-editor/helix/pull/995))
+- Launch with defaults upon invalid config/theme (rather than panicking) ([#982](https://github.com/helix-editor/helix/pull/982))
+- If switching away from an empty scratch buffer, remove it ([#935](https://github.com/helix-editor/helix/pull/935))
+- Truncate the starts of file paths instead of the ends in picker ([#951](https://github.com/helix-editor/helix/pull/951))
+- Truncate the start of file paths in the StatusLine ([#1351](https://github.com/helix-editor/helix/pull/1351))
+- Prevent picker from previewing binaries or large file ([#939](https://github.com/helix-editor/helix/pull/939))
+- Inform when reaching undo/redo bounds ([#981](https://github.com/helix-editor/helix/pull/981))
+- search_impl will only align cursor center when it isn't in view ([#959](https://github.com/helix-editor/helix/pull/959))
+- Add `<C-h>`, `<C-u>`, `<C-d>`, Delete in prompt mode ([#1034](https://github.com/helix-editor/helix/pull/1034))
+- Restore screen position when aborting search ([#1047](https://github.com/helix-editor/helix/pull/1047))
+- Buffer picker: show is_modifier flag ([#1020](https://github.com/helix-editor/helix/pull/1020))
+- Add commit hash to version info, if present ([#957](https://github.com/helix-editor/helix/pull/957))
+- Implement indent-aware delete ([#1120](https://github.com/helix-editor/helix/pull/1120))
+- Jump to end char of surrounding pair from any cursor pos ([#1121](https://github.com/helix-editor/helix/pull/1121))
+- File picker configuration ([#988](https://github.com/helix-editor/helix/pull/988))
+- Fix surround cursor position calculation ([#1183](https://github.com/helix-editor/helix/pull/1183))
+- Accept count for goto_window ([#1033](https://github.com/helix-editor/helix/pull/1033))
+- Make kill_to_line_end behave like emacs ([#1235](https://github.com/helix-editor/helix/pull/1235))
+- Only use a single documentation popup ([#1241](https://github.com/helix-editor/helix/pull/1241))
+- ui: popup: Don't allow scrolling past the end of content ([`3307f44c`](https://github.com/helix-editor/helix/commit/3307f44c))
+- Open files with spaces in filename, allow opening multiple files ([#1231](https://github.com/helix-editor/helix/pull/1231))
+- Allow paste commands to take a count ([#1261](https://github.com/helix-editor/helix/pull/1261))
+- Auto pairs selection ([#1254](https://github.com/helix-editor/helix/pull/1254))
+- Use a fuzzy matcher for commands ([#1386](https://github.com/helix-editor/helix/pull/1386))
+- Add `<C-s>` to pick word under doc cursor to prompt line & search completion ([#831](https://github.com/helix-editor/helix/pull/831))
+- Fix `:earlier`/`:later` missing changeset update ([#1069](https://github.com/helix-editor/helix/pull/1069))
+- Support extend for multiple goto ([#909](https://github.com/helix-editor/helix/pull/909))
+- Add arrow-key bindings for window switching ([#933](https://github.com/helix-editor/helix/pull/933))
+- Implement key ordering for info box ([#952](https://github.com/helix-editor/helix/pull/952))
+
+LSP:
+- Implement MarkedString rendering ([`e128a8702`](https://github.com/helix-editor/helix/commit/e128a8702))
+- Don't panic if init fails ([`d31bef7`](https://github.com/helix-editor/helix/commit/d31bef7))
+- Configurable diagnostic severity ([#1325](https://github.com/helix-editor/helix/pull/1325))
+- Resolve completion item ([#1315](https://github.com/helix-editor/helix/pull/1315))
+- Code action command support ([#1304](https://github.com/helix-editor/helix/pull/1304))
+
+Grammars:
+
+- Adds mint language server ([#974](https://github.com/helix-editor/helix/pull/974))
+- Perl ([#978](https://github.com/helix-editor/helix/pull/978)) ([#1280](https://github.com/helix-editor/helix/pull/1280))
+- GLSL ([#993](https://github.com/helix-editor/helix/pull/993))
+- Racket ([#1143](https://github.com/helix-editor/helix/pull/1143))
+- WGSL ([#1166](https://github.com/helix-editor/helix/pull/1166))
+- LLVM ([#1167](https://github.com/helix-editor/helix/pull/1167)) ([#1388](https://github.com/helix-editor/helix/pull/1388)) ([#1409](https://github.com/helix-editor/helix/pull/1409)) ([#1398](https://github.com/helix-editor/helix/pull/1398))
+- Markdown ([`49e06787`](https://github.com/helix-editor/helix/commit/49e06787))
+- Scala ([#1278](https://github.com/helix-editor/helix/pull/1278))
+- Dart ([#1250](https://github.com/helix-editor/helix/pull/1250))
+- Fish ([#1308](https://github.com/helix-editor/helix/pull/1308))
+- Dockerfile ([#1303](https://github.com/helix-editor/helix/pull/1303))
+- Git (commit, rebase, diff) ([#1338](https://github.com/helix-editor/helix/pull/1338)) ([#1402](https://github.com/helix-editor/helix/pull/1402)) ([#1373](https://github.com/helix-editor/helix/pull/1373))
+- tree-sitter-comment ([#1300](https://github.com/helix-editor/helix/pull/1300))
+- Highlight comments in c, cpp, cmake and llvm ([#1309](https://github.com/helix-editor/helix/pull/1309))
+- Improve yaml syntax highlighting highlighting ([#1294](https://github.com/helix-editor/helix/pull/1294))
+- Improve rust syntax highlighting ([#1295](https://github.com/helix-editor/helix/pull/1295))
+- Add textobjects and indents to cmake ([#1307](https://github.com/helix-editor/helix/pull/1307))
+- Add textobjects and indents to c and cpp ([#1293](https://github.com/helix-editor/helix/pull/1293))
+
+New themes:
+
+- Solarized dark ([#999](https://github.com/helix-editor/helix/pull/999))
+- Solarized light ([#1010](https://github.com/helix-editor/helix/pull/1010))
+- Spacebones light ([#1131](https://github.com/helix-editor/helix/pull/1131))
+- Monokai Pro ([#1206](https://github.com/helix-editor/helix/pull/1206))
+- Base16 Light and Terminal ([#1078](https://github.com/helix-editor/helix/pull/1078))
+  - and a default 16 color theme, truecolor detection
+- Dracula ([#1258](https://github.com/helix-editor/helix/pull/1258))
 
 # 0.5.0 (2021-10-28)
 
@@ -23,19 +305,19 @@ Features:
 - LSP compatibility greatly improved for some implementations (Julia, Python, Typescript)
 - Autocompletion! Completion now triggers automatically after a set idle timeout
 - Completion documentation is now displayed next to the popup ([#691](https://github.com/helix-editor/helix/pull/691))
-- Treesitter textobjects (select a function via `mf`, class via `mc`) ([#728](https://github.com/helix-editor/helix/pull/728)) 
+- Treesitter textobjects (select a function via `mf`, class via `mc`) ([#728](https://github.com/helix-editor/helix/pull/728))
-- Global search across entire workspace `space+/` ([#651](https://github.com/helix-editor/helix/pull/651)) 
+- Global search across entire workspace `space+/` ([#651](https://github.com/helix-editor/helix/pull/651))
 - Relative line number support ([#485](https://github.com/helix-editor/helix/pull/485))
-- Prompts now store a history (72cf86e)
+- Prompts now store a history ([`72cf86e`](https://github.com/helix-editor/helix/commit/72cf86e))
 - `:vsplit` and `:hsplit` commands ([#639](https://github.com/helix-editor/helix/pull/639))
 - `C-w h/j/k/l` can now be used to navigate between splits ([#860](https://github.com/helix-editor/helix/pull/860))
 - `C-j` and `C-k` are now alternative keybindings to `C-n` and `C-p` in the UI ([#876](https://github.com/helix-editor/helix/pull/876))
 - Shell commands (shell-pipe, pipe-to, shell-insert-output, shell-append-output, keep-pipe) ([#547](https://github.com/helix-editor/helix/pull/547))
 - Searching now defaults to smart case search (case insensitive unless uppercase is used) ([#761](https://github.com/helix-editor/helix/pull/761))
 - The preview pane was improved to highlight and center line ranges
-- The user `languages.toml` is now merged into defaults, no longer need to copy the entire file (dc57f8dc)
+- The user `languages.toml` is now merged into defaults, no longer need to copy the entire file ([`dc57f8dc`](https://github.com/helix-editor/helix/commit/dc57f8dc))
 - Show hidden files in completions ([#648](https://github.com/helix-editor/helix/pull/648))
-- Grammar injections are now properly handled (dd0b15e)
+- Grammar injections are now properly handled ([`dd0b15e`](https://github.com/helix-editor/helix/commit/dd0b15e))
 - `v` in select mode now switches back to normal mode ([#660](https://github.com/helix-editor/helix/pull/660))
 - View mode can now be triggered as a "sticky" mode ([#719](https://github.com/helix-editor/helix/pull/719))
 - `f`/`t` and object selection motions can now be repeated via `Alt-.` ([#891](https://github.com/helix-editor/helix/pull/891))
@@ -53,7 +335,7 @@ New grammars:
 - Vue ([#787](https://github.com/helix-editor/helix/pull/787))
 - Tree-sitter queries ([#845](https://github.com/helix-editor/helix/pull/845))
 - CMake ([#888](https://github.com/helix-editor/helix/pull/888))
-- Elixir (we switched over to the official grammar) (6c0786e)
+- Elixir (we switched over to the official grammar) ([`6c0786e`](https://github.com/helix-editor/helix/commit/6c0786e))
 - Language server definitions for Nix and Elixir ([#725](https://github.com/helix-editor/helix/pull/725))
 - Python now uses `pylsp` instead of `pyls`
 - Python now supports indentation
@@ -70,21 +352,22 @@ Fixes:
 
 - Fix crash on empty rust file ([#592](https://github.com/helix-editor/helix/pull/592))
 - Exit select mode after toggle comment ([#598](https://github.com/helix-editor/helix/pull/598))
-- Pin popups with no positioning to the initial position (12ea3888)
+- Pin popups with no positioning to the initial position ([`12ea3888`](https://github.com/helix-editor/helix/commit/12ea3888))
-- xsel copy should not freeze the editor (6dd7dc4)
+- xsel copy should not freeze the editor ([`6dd7dc4`](https://github.com/helix-editor/helix/commit/6dd7dc4))
-- `*` now only sets the search register and doesn't jump to the next occurrence (3426285)
+- `*` now only sets the search register and doesn't jump to the next occurrence ([`3426285`](https://github.com/helix-editor/helix/commit/3426285))
 - Goto line start/end commands extend when in select mode ([#739](https://github.com/helix-editor/helix/pull/739)) 
-- Fix documentation popups sometimes not getting fully highlighted (066367c)
+- Fix documentation popups sometimes not getting fully highlighted ([`066367c`](https://github.com/helix-editor/helix/commit/066367c))
-- Refactor apply_workspace_edit to remove assert (b02d872)
+- Refactor apply_workspace_edit to remove assert ([`b02d872`](https://github.com/helix-editor/helix/commit/b02d872))
-- Wrap around the top of the picker menu when scrolling (c7d6e44)
+- Wrap around the top of the picker menu when scrolling ([`c7d6e44`](https://github.com/helix-editor/helix/commit/c7d6e44))
-- Don't allow closing the last split if there's unsaved changes (3ff5b00)
+- Don't allow closing the last split if there's unsaved changes ([`3ff5b00`](https://github.com/helix-editor/helix/commit/3ff5b00))
-- Indentation used different default on hx vs hx new_file.txt (c913bad)
+- Indentation used different default on hx vs hx new_file.txt ([`c913bad`](https://github.com/helix-editor/helix/commit/c913bad))
 
 # 0.4.1 (2021-08-14)
 
 A minor release that includes:
+
 - A fix for rendering glitches that would occur after editing with multiple selections.
-- CI fix for grammars not being cross-compiled for aarch64 
+- CI fix for grammars not being cross-compiled for aarch64
 
 # 0.4.0 (2021-08-13)
 
@@ -104,10 +387,10 @@ selections in the future as well as resolves many bugs and edge cases.
 - Autoinfo: `whichkey`-like popups which show available sub-mode shortcuts ([#316](https://github.com/helix-editor/helix/pull/316))
 - Added WORD movements (W/B/E) ([#390](https://github.com/helix-editor/helix/pull/390))
 - Vertical selections (repeat selection above/below) ([#462](https://github.com/helix-editor/helix/pull/462))
-- Selection rotation via `(` and `)` ([66a90130](https://github.com/helix-editor/helix/commit/66a90130a5f99d769e9f6034025297f78ecaa3ec))
+- Selection rotation via `(` and `)` ([`66a90130`](https://github.com/helix-editor/helix/commit/66a90130a5f99d769e9f6034025297f78ecaa3ec))
-- Selection contents rotation via `Alt-(` and `Alt-)` ([02cba2a](https://github.com/helix-editor/helix/commit/02cba2a7f403f48eccb18100fb751f7b42373dba))
+- Selection contents rotation via `Alt-(` and `Alt-)` ([`02cba2a`](https://github.com/helix-editor/helix/commit/02cba2a7f403f48eccb18100fb751f7b42373dba))
-- Completion behavior improvements ([f917b5a4](https://github.com/helix-editor/helix/commit/f917b5a441ff3ae582358b6939ffbf889f4aa530), [627b899](https://github.com/helix-editor/helix/commit/627b89931576f7af86166ae8d5cbc55537877473))
+- Completion behavior improvements ([`f917b5a4`](https://github.com/helix-editor/helix/commit/f917b5a441ff3ae582358b6939ffbf889f4aa530), [`627b899`](https://github.com/helix-editor/helix/commit/627b89931576f7af86166ae8d5cbc55537877473))
-- Fixed a language server crash ([385a6b5a](https://github.com/helix-editor/helix/commit/385a6b5a1adddfc26e917982641530e1a7c7aa81))
+- Fixed a language server crash ([`385a6b5a`](https://github.com/helix-editor/helix/commit/385a6b5a1adddfc26e917982641530e1a7c7aa81))
 - Case change commands (`` ` ``, `~`, ``<a-`>``) ([#441](https://github.com/helix-editor/helix/pull/441))
 - File pickers (including goto) now provide a preview! ([#534](https://github.com/helix-editor/helix/pull/534))
 - Injection query support. Rust macro calls and embedded languages are now properly highlighted ([#430](https://github.com/helix-editor/helix/pull/430))
@@ -123,7 +406,7 @@ selections in the future as well as resolves many bugs and edge cases.
 - Comment toggling now uses a language specific comment token ([#463](https://github.com/helix-editor/helix/pull/463))
 - Julia support ([#413](https://github.com/helix-editor/helix/pull/413))
 - Java support ([#448](https://github.com/helix-editor/helix/pull/448))
-- Prompts have an (in-memory) history ([63e54e30](https://github.com/helix-editor/helix/commit/63e54e30a74bb0d1d782877ddbbcf95f2817d061))
+- Prompts have an (in-memory) history ([`63e54e30`](https://github.com/helix-editor/helix/commit/63e54e30a74bb0d1d782877ddbbcf95f2817d061))
 
 # 0.3.0 (2021-06-27)
 
@@ -137,7 +420,7 @@ Highlights:
 - Support for other line endings (CRLF). Significantly improved Windows support. ([#224](https://github.com/helix-editor/helix/pull/224))
 - Encodings other than UTF-8 are now supported! ([#228](https://github.com/helix-editor/helix/pull/228))
 - Key bindings can now be configured via a `config.toml` file ([#268](https://github.com/helix-editor/helix/pull/268))
-- Theme can now be configured and changed at runtime ([please feel free to contribute more themes!](https://github.com/helix-editor/helix/tree/master/runtime/themes)) ([#267](https://github.com/helix-editor/helix/pull/267))
+- Theme can now be configured and changed at runtime. ([Please feel free to contribute more themes!](https://github.com/helix-editor/helix/tree/master/runtime/themes)) ([#267](https://github.com/helix-editor/helix/pull/267))
 - System clipboard yank/paste is now supported! ([#310](https://github.com/helix-editor/helix/pull/310))
 - Surround commands were implemented ([#320](https://github.com/helix-editor/helix/pull/320))
 
@@ -154,7 +437,7 @@ Features:
 - Code is being migrated from helix-term to helix-view (prerequisite for
   alternative frontends) ([#366](https://github.com/helix-editor/helix/pull/366))
 - `x` and `X` merged
-  ([f41688d9](https://github.com/helix-editor/helix/commit/f41688d960ef89c29c4a51c872b8406fb8f81a85))
+  ([`f41688d9`](https://github.com/helix-editor/helix/commit/f41688d960ef89c29c4a51c872b8406fb8f81a85))
 
 Fixes:
 
@@ -162,12 +445,12 @@ Fixes:
 - A bunch of bugs regarding `o`/`O` behavior ([#281](https://github.com/helix-editor/helix/pull/281))
 - `~` expansion now works in file completion ([#284](https://github.com/helix-editor/helix/pull/284))
 - Several UI related overflow crashes ([#318](https://github.com/helix-editor/helix/pull/318))
-- Fix a test failure occuring only on `test --release` ([4f108ab1](https://github.com/helix-editor/helix/commit/4f108ab1b2197809506bd7305ad903a3525eabfa))
+- Fix a test failure occurring only on `test --release` ([`4f108ab1`](https://github.com/helix-editor/helix/commit/4f108ab1b2197809506bd7305ad903a3525eabfa))
 - Prompts now support unicode input ([#295](https://github.com/helix-editor/helix/pull/295))
 - Completion documentation no longer overlaps the popup ([#322](https://github.com/helix-editor/helix/pull/322))
-- Fix a crash when trying to select `^` ([9c534614](https://github.com/helix-editor/helix/commit/9c53461429a3e72e3b1fb87d7ca490e168d7dee2))
+- Fix a crash when trying to select `^` ([`9c534614`](https://github.com/helix-editor/helix/commit/9c53461429a3e72e3b1fb87d7ca490e168d7dee2))
-- Prompt completions are now paginated ([39dc09e6](https://github.com/helix-editor/helix/commit/39dc09e6c4172299bc79de4c1c52288d3f624bd7))
+- Prompt completions are now paginated ([`39dc09e6`](https://github.com/helix-editor/helix/commit/39dc09e6c4172299bc79de4c1c52288d3f624bd7))
-- Goto did not work on Windows ([503ca112](https://github.com/helix-editor/helix/commit/503ca112ae57ebdf3ea323baf8940346204b46d2))
+- Goto did not work on Windows ([`503ca112`](https://github.com/helix-editor/helix/commit/503ca112ae57ebdf3ea323baf8940346204b46d2))
 
 # 0.2.1
 

Cargo.toml

@@ -4,17 +4,31 @@ members = [
   "helix-view",
   "helix-term",
   "helix-tui",
-  "helix-syntax",
+  "helix-graphics",
+  "helix-ui",
   "helix-lsp",
+  "helix-dap",
+  "helix-loader",
   "xtask",
 ]
 
-# Build helix-syntax in release mode to make the code path faster in development.
+default-members = [
-# [profile.dev.package."helix-syntax"]
+  "helix-term",
-# opt-level = 3
+  "helix-ui"
+]
+
+resolver = "2"
 
 [profile.dev]
 split-debuginfo = "unpacked"
 
 [profile.release]
 lto = "thin"
+# debug = true
+
+[profile.opt]
+inherits = "release"
+lto = "fat"
+codegen-units = 1
+# strip = "debuginfo" # TODO: or strip = true
+opt-level = 3

README.md

@@ -28,24 +28,33 @@ It's a terminal-based editor first, but I'd like to explore a custom renderer
 (similar to emacs) in wgpu or skulpin.
 
 Note: Only certain languages have indentation definitions at the moment. Check
-`runtime/queries/<lang>/` for `indents.toml`.
+`runtime/queries/<lang>/` for `indents.scm`.
 
 # Installation
 
-We provide packaging for various distributions, but here's a quick method to
+Packages are available for various distributions (see [Installation docs](https://docs.helix-editor.com/install.html)).
-build from source.
 
-```
+If you would like to build from source:
-git clone --recurse-submodules --shallow-submodules -j8 https://github.com/helix-editor/helix
+
+```shell
+git clone https://github.com/helix-editor/helix
 cd helix
 cargo install --path helix-term
+hx --grammar fetch
+hx --grammar build
 ```
 
-This will install the `hx` binary to `$HOME/.cargo/bin`.
+This will install the `hx` binary to `$HOME/.cargo/bin` and build tree-sitter grammars.
 
 Helix also needs its runtime files so make sure to copy/symlink the `runtime/` directory into the
 config directory (for example `~/.config/helix/runtime` on Linux/macOS, or `%AppData%/helix/runtime` on Windows).
-This location can be overriden via the `HELIX_RUNTIME` environment variable.
+
+| OS        | command   |
+|-----------|-----------|
+|windows    |`xcopy runtime %AppData%/helix/runtime`|
+|linux/macos|`ln -s $PWD/runtime ~/.config/helix/runtime`
+
+This location can be overridden via the `HELIX_RUNTIME` environment variable.
 
 Packages already solve this for you by wrapping the `hx` binary with a wrapper
 that sets the variable to the install dir.
@@ -53,9 +62,14 @@ that sets the variable to the install dir.
 > NOTE: running via cargo also doesn't require setting explicit `HELIX_RUNTIME` path, it will automatically
 > detect the `runtime` directory in the project root.
 
+In order to use LSP features like auto-complete, you will need to
+[install the appropriate Language Server](https://github.com/helix-editor/helix/wiki/How-to-install-the-default-language-servers)
+for a language.
+
 [![Packaging status](https://repology.org/badge/vertical-allrepos/helix.svg)](https://repology.org/project/helix/versions)
 
 ## MacOS
+
 Helix can be installed on MacOS through homebrew via:
 
 ```

TODO.md

@@ -1,32 +0,0 @@
-
-- tree sitter:
-  - markdown
-  - regex
-  - kotlin
-  - clojure
-  - erlang
-
-- [ ] completion isIncomplete support
-
-1
-- [ ] respect view fullscreen flag
-- [ ] Implement marks (superset of Selection/Range)
-
-- [ ] = for auto indent line/selection
-- [ ]  :x for closing buffers
-- [ ] lsp: signature help
-
-2
-- [ ] macro recording
-- [ ] extend selection (treesitter select parent node) (replaces viw, vi(, va( etc )
-- [ ] selection align
-- [ ] store some state between restarts: file positions, prompt history
-- [ ] highlight matched characters in picker
-
-3
-- [ ] diff mode with highlighting?
-- [ ] snippet support (tab to jump between marks)
-- [ ] gamelisp/wasm scripting
-
-X
-- [ ] rendering via skulpin/skia or raw wgpu

VERSION

@@ -0,0 +1 @@
+22.05-dev

base16_theme.toml

@@ -11,7 +11,7 @@
 "ui.statusline" = { fg = "black", bg = "white" }
 "ui.statusline.inactive" = { fg = "gray", bg = "white" }
 "ui.help" = { modifiers = ["reversed"] }
-"ui.cursor" = { modifiers = ["reversed"] }
+"ui.cursor" = { fg = "white", modifiers = ["reversed"] }
 "variable" = "red"
 "constant.numeric" = "yellow"
 "constant" = "yellow"
@@ -29,6 +29,15 @@
 "namespace" = "magenta"
 "ui.help" = { fg = "white", bg = "black" }
 
+"markup.heading" = "blue"
+"markup.list" = "red"
+"markup.bold" = { fg = "yellow", modifiers = ["bold"] }
+"markup.italic" = { fg = "magenta", modifiers = ["italic"] }
+"markup.link.url" = { fg = "yellow", modifiers = ["underlined"] }
+"markup.link.text" = "red"
+"markup.quote" = "cyan"
+"markup.raw" = "green"
+
 "diff.plus" = "green"
 "diff.delta" = "yellow"
 "diff.minus" = "red"

book/src/SUMMARY.md

@@ -1,5 +1,7 @@
 # Summary
 
+[Helix](./title-page.md)
+
 - [Installation](./install.md)
 - [Usage](./usage.md)
   - [Keymap](./keymap.md)
@@ -14,3 +16,4 @@
 - [Guides](./guides/README.md)
   - [Adding Languages](./guides/adding_languages.md)
   - [Adding Textobject Queries](./guides/textobject.md)
+  - [Adding Indent Queries](./guides/indent.md)

book/src/configuration.md

@@ -5,9 +5,29 @@ To override global configuration parameters, create a `config.toml` file located
 * Linux and Mac: `~/.config/helix/config.toml`
 * Windows: `%AppData%\helix\config.toml`
 
+> Hint: You can easily open the config file by typing `:config-open` within Helix normal mode.
+
+Example config:
+
+```toml
+theme = "onedark"
+
+[editor]
+line-number = "relative"
+mouse = false
+
+[editor.cursor-shape]
+insert = "bar"
+normal = "block"
+select = "underline"
+
+[editor.file-picker]
+hidden = false
+```
+
 ## Editor
 
-`[editor]` section of the config.
+### `[editor]` Section
 
 | Key | Description | Default |
 |--|--|---------|
@@ -16,16 +36,46 @@ To override global configuration parameters, create a `config.toml` file located
 | `middle-click-paste` | Middle click paste support. | `true` |
 | `scroll-lines` | Number of lines to scroll per scroll wheel step. | `3` |
 | `shell` | Shell to use when running external commands. | Unix: `["sh", "-c"]`<br/>Windows: `["cmd", "/C"]` |
-| `line-number` | Line number display (`absolute`, `relative`) | `absolute` |
+| `line-number` | Line number display: `absolute` simply shows each line's number, while `relative` shows the distance from the current line. When unfocused or in insert mode, `relative` will still show absolute line numbers. | `absolute` |
-| `smart-case` | Enable smart case regex searching (case insensitive unless pattern contains upper case characters) | `true` |
+| `gutters` | Gutters to display: Available are `diagnostics` and `line-numbers`, note that `diagnostics` also includes other features like breakpoints | `["diagnostics", "line-numbers"]` |
-| `auto-pairs` | Enable automatic insertion of pairs to parenthese, brackets, etc. | `true` |
 | `auto-completion` | Enable automatic pop up of auto-completion. | `true` |
 | `idle-timeout` | Time in milliseconds since last keypress before idle timers trigger. Used for autocompletion, set to 0 for instant. | `400` |
 | `completion-trigger-len` | The min-length of word under cursor to trigger autocompletion | `2` |
 | `auto-info` | Whether to display infoboxes | `true` |
 | `true-color` | Set to `true` to override automatic detection of terminal truecolor support in the event of a false negative. | `false` |
+| `rulers` | List of column positions at which to display the rulers. Can be overridden by language specific `rulers` in `languages.toml` file. | `[]` |
 
-`[editor.filepicker]` section of the config. Sets options for file picker and global search. All but the last key listed in the default file-picker configuration below are IgnoreOptions: whether hidden files and files listed within ignore files are ignored by (not visible in) the helix file picker and global search. There is also one other key, `max-depth` available, which is not defined by default.
+### `[editor.lsp]` Section
+
+| Key                | Description                                 | Default |
+| ---                | -----------                                 | ------- |
+| `display-messages` | Display LSP progress messages below statusline[^1] | `false` |
+
+[^1]: A progress spinner is always shown in the statusline beside the file path.
+
+### `[editor.cursor-shape]` Section
+
+Defines the shape of cursor in each mode. Note that due to limitations
+of the terminal environment, only the primary cursor can change shape.
+Valid values for these options are `block`, `bar`, `underline`, or `none`.
+
+| Key      | Description                                | Default |
+| ---      | -----------                                | ------- |
+| `normal` | Cursor shape in [normal mode][normal mode] | `block` |
+| `insert` | Cursor shape in [insert mode][insert mode] | `block` |
+| `select` | Cursor shape in [select mode][select mode] | `block` |
+
+[normal mode]: ./keymap.md#normal-mode
+[insert mode]: ./keymap.md#insert-mode
+[select mode]: ./keymap.md#select--extend-mode
+
+### `[editor.file-picker]` Section
+
+Sets options for file picker and global search. All but the last key listed in
+the default file-picker configuration below are IgnoreOptions: whether hidden
+files and files listed within ignore files are ignored by (not visible in) the
+helix file picker and global search. There is also one other key, `max-depth`
+available, which is not defined by default.
 
 | Key | Description | Default |
 |--|--|---------|
@@ -37,10 +87,80 @@ To override global configuration parameters, create a `config.toml` file located
 |`git-exclude` | Enables reading `.git/info/exclude` files. | true
 |`max-depth` | Set with an integer value for maximum depth to recurse. | Defaults to `None`.
 
-## LSP
+### `[editor.auto-pairs]` Section
+
+Enable automatic insertion of pairs to parentheses, brackets, etc. Can be
+a simple boolean value, or a specific mapping of pairs of single characters.
+
+| Key | Description |
+| --- | ----------- |
+| `false` | Completely disable auto pairing, regardless of language-specific settings
+| `true` | Use the default pairs: <code>(){}[]''""``</code>
+| Mapping of pairs | e.g. `{ "(" =  ")", "{" = "}", ... }`
+
+Example
 
-To display all language server messages in the status line add the following to your `config.toml`:
 ```toml
-[lsp]
+[editor.auto-pairs]
-display-messages = true
+'(' = ')'
+'{' = '}'
+'[' = ']'
+'"' = '"'
+'`' = '`'
+'<' = '>'
+```
+
+Additionally, this setting can be used in a language config. Unless
+the editor setting is `false`, this will override the editor config in
+documents with this language.
+
+Example `languages.toml` that adds <> and removes ''
+
+```toml
+[[language]]
+name = "rust"
+
+[language.auto-pairs]
+'(' = ')'
+'{' = '}'
+'[' = ']'
+'"' = '"'
+'`' = '`'
+'<' = '>'
+```
+
+### `[editor.search]` Section
+
+Search specific options.
+
+| Key | Description | Default |
+|--|--|---------|
+| `smart-case` | Enable smart case regex searching (case insensitive unless pattern contains upper case characters) | `true` |
+| `wrap-around`| Whether the search should wrap after depleting the matches | `true` |
+
+### `[editor.whitespace]` Section
+
+Options for rendering whitespace with visible characters. Use `:set whitespace.render all` to temporarily enable visible whitespace.
+
+| Key | Description | Default |
+|-----|-------------|---------|
+| `render` | Whether to render whitespace. May either be `"all"` or `"none"`, or a table with sub-keys `space`, `tab`, and `newline`. | `"none"` |
+| `characters` | Literal characters to use when rendering whitespace. Sub-keys may be any of `tab`, `space`, `nbsp` or `newline` | See example below |
+
+Example
+
+```toml
+[editor.whitespace]
+render = "all"
+# or control each character
+[editor.whitespace.render]
+space = "all"
+tab = "all"
+newline = "none"
+
+[editor.whitespace.characters]
+space = "·"
+nbsp = "⍽"
+tab = "→"
+newline = "⏎"
 ```

book/src/generated/lang-support.md

@@ -1,51 +1,89 @@
 | Language | Syntax Highlighting | Treesitter Textobjects | Auto Indent | Default LSP |
 | --- | --- | --- | --- | --- |
 | bash | ✓ |  |  | `bash-language-server` |
-| c | ✓ |  |  | `clangd` |
+| c | ✓ | ✓ | ✓ | `clangd` |
-| c-sharp | ✓ |  |  |  |
+| c-sharp | ✓ |  |  | `OmniSharp` |
+| cairo | ✓ |  |  |  |
 | cmake | ✓ | ✓ | ✓ | `cmake-language-server` |
 | comment | ✓ |  |  |  |
-| cpp | ✓ |  |  | `clangd` |
+| cpp | ✓ | ✓ | ✓ | `clangd` |
-| css | ✓ |  |  |  |
+| css | ✓ |  |  | `vscode-css-language-server` |
 | dart | ✓ |  | ✓ | `dart` |
+| devicetree | ✓ |  | ✓ |  |
 | dockerfile | ✓ |  |  | `docker-langserver` |
+| eex | ✓ |  |  |  |
+| ejs | ✓ |  |  |  |
 | elixir | ✓ |  |  | `elixir-ls` |
+| elm | ✓ |  |  | `elm-language-server` |
+| erb | ✓ |  |  |  |
+| erlang | ✓ |  |  | `erlang_ls` |
 | fish | ✓ | ✓ | ✓ |  |
+| gdscript | ✓ |  | ✓ |  |
 | git-commit | ✓ |  |  |  |
+| git-config | ✓ |  |  |  |
 | git-diff | ✓ |  |  |  |
 | git-rebase | ✓ |  |  |  |
+| gleam | ✓ |  |  |  |
 | glsl | ✓ |  | ✓ |  |
 | go | ✓ | ✓ | ✓ | `gopls` |
-| html | ✓ |  |  |  |
+| gomod | ✓ |  |  | `gopls` |
+| gowork | ✓ |  |  | `gopls` |
+| graphql | ✓ |  |  |  |
+| hare | ✓ |  | ✓ |  |
+| haskell | ✓ |  |  | `haskell-language-server-wrapper` |
+| hcl | ✓ |  | ✓ | `terraform-ls` |
+| heex | ✓ |  |  |  |
+| html | ✓ |  |  | `vscode-html-language-server` |
+| iex | ✓ |  |  |  |
 | java | ✓ |  |  |  |
-| javascript | ✓ |  | ✓ |  |
+| javascript | ✓ |  | ✓ | `typescript-language-server` |
-| json | ✓ |  | ✓ |  |
+| json | ✓ |  | ✓ | `vscode-json-language-server` |
+| jsx | ✓ |  | ✓ | `typescript-language-server` |
 | julia | ✓ |  |  | `julia` |
-| latex | ✓ |  |  |  |
+| kotlin | ✓ |  |  | `kotlin-language-server` |
+| latex | ✓ |  |  | `texlab` |
+| lean | ✓ |  |  | `lean` |
 | ledger | ✓ |  |  |  |
 | llvm | ✓ | ✓ | ✓ |  |
+| llvm-mir | ✓ | ✓ | ✓ |  |
+| llvm-mir-yaml | ✓ |  | ✓ |  |
 | lua | ✓ |  | ✓ |  |
+| make | ✓ |  |  |  |
 | markdown | ✓ |  |  |  |
 | mint |  |  |  | `mint` |
+| nickel | ✓ |  | ✓ | `nls` |
 | nix | ✓ |  | ✓ | `rnix-lsp` |
-| ocaml | ✓ |  | ✓ |  |
+| nu | ✓ |  |  |  |
-| ocaml-interface | ✓ |  |  |  |
+| ocaml | ✓ |  | ✓ | `ocamllsp` |
+| ocaml-interface | ✓ |  |  | `ocamllsp` |
+| org | ✓ |  |  |  |
 | perl | ✓ | ✓ | ✓ |  |
-| php | ✓ |  | ✓ |  |
+| php | ✓ | ✓ | ✓ | `intelephense` |
 | prolog |  |  |  | `swipl` |
 | protobuf | ✓ |  | ✓ |  |
 | python | ✓ | ✓ | ✓ | `pylsp` |
+| r | ✓ |  |  | `R` |
 | racket |  |  |  | `racket` |
-| ruby | ✓ |  | ✓ | `solargraph` |
+| regex | ✓ |  |  |  |
+| rescript | ✓ | ✓ |  | `rescript-language-server` |
+| rmarkdown | ✓ |  | ✓ | `R` |
+| ron | ✓ |  | ✓ |  |
+| ruby | ✓ | ✓ | ✓ | `solargraph` |
 | rust | ✓ | ✓ | ✓ | `rust-analyzer` |
 | scala | ✓ |  | ✓ | `metals` |
+| solidity | ✓ |  |  | `solc` |
+| sql | ✓ |  |  |  |
 | svelte | ✓ |  | ✓ | `svelteserver` |
+| swift | ✓ |  |  | `sourcekit-lsp` |
 | tablegen | ✓ | ✓ | ✓ |  |
-| toml | ✓ |  |  |  |
+| tfvars |  |  |  | `terraform-ls` |
+| toml | ✓ |  |  | `taplo` |
 | tsq | ✓ |  |  |  |
 | tsx | ✓ |  |  | `typescript-language-server` |
+| twig | ✓ |  |  |  |
 | typescript | ✓ |  | ✓ | `typescript-language-server` |
-| vue | ✓ |  |  |  |
+| vala | ✓ |  |  | `vala-language-server` |
+| vue | ✓ |  |  | `vls` |
 | wgsl | ✓ |  |  |  |
-| yaml | ✓ |  | ✓ |  |
+| yaml | ✓ |  | ✓ | `yaml-language-server` |
 | zig | ✓ |  | ✓ | `zls` |

book/src/generated/typable-cmd.md

@@ -5,11 +5,18 @@
 | `:open`, `:o` | Open a file from disk into the current view. |
 | `:buffer-close`, `:bc`, `:bclose` | Close the current buffer. |
 | `:buffer-close!`, `:bc!`, `:bclose!` | Close the current buffer forcefully (ignoring unsaved changes). |
+| `:buffer-close-others`, `:bco`, `:bcloseother` | Close all buffers but the currently focused one. |
+| `:buffer-close-others!`, `:bco!`, `:bcloseother!` | Close all buffers but the currently focused one. |
+| `:buffer-close-all`, `:bca`, `:bcloseall` | Close all buffers, without quitting. |
+| `:buffer-close-all!`, `:bca!`, `:bcloseall!` | Close all buffers forcefully (ignoring unsaved changes), without quitting. |
+| `:buffer-next`, `:bn`, `:bnext` | Go to next buffer. |
+| `:buffer-previous`, `:bp`, `:bprev` | Go to previous buffer. |
 | `:write`, `:w` | Write changes to disk. Accepts an optional path (:write some/path.txt) |
+| `:write!`, `:w!` | Write changes to disk forcefully (creating necessary subdirectories). Accepts an optional path (:write some/path.txt) |
 | `:new`, `:n` | Create a new scratch buffer. |
 | `:format`, `:fmt` | Format the file using the LSP formatter. |
 | `:indent-style` | Set the indentation style for editing. ('t' for tabs or 1-8 for number of spaces.) |
-| `:line-ending` | Set the document's default line ending. Options: crlf, lf, cr, ff, nel. |
+| `:line-ending` | Set the document's default line ending. Options: crlf, lf. |
 | `:earlier`, `:ear` | Jump back to an earlier point in edit history. Accepts a number of steps or a time span. |
 | `:later`, `:lat` | Jump to a later point in edit history. Accepts a number of steps or a time span. |
 | `:write-quit`, `:wq`, `:x` | Write changes to disk and close the current view. Accepts an optional path (:wq some/path.txt) |
@@ -20,6 +27,7 @@
 | `:quit-all`, `:qa` | Close all views. |
 | `:quit-all!`, `:qa!` | Close all views forcefully (ignoring unsaved changes). |
 | `:cquit`, `:cq` | Quit with exit code (default 1). Accepts an optional integer exit code (:cq 2). |
+| `:cquit!`, `:cq!` | Quit with exit code (default 1) forcefully (ignoring unsaved changes). Accepts an optional integer exit code (:cq! 2). |
 | `:theme` | Change the editor theme. |
 | `:clipboard-yank` | Yank main selection into system clipboard. |
 | `:clipboard-yank-join` | Yank joined selections into system clipboard. A separator can be provided as first argument. Default value is newline. |
@@ -37,10 +45,21 @@
 | `:encoding` | Set encoding based on `https://encoding.spec.whatwg.org` |
 | `:reload` | Discard changes and reload from the source file. |
 | `:tree-sitter-scopes` | Display tree sitter scopes, primarily for theming and development. |
+| `:debug-start`, `:dbg` | Start a debug session from a given template with given parameters. |
+| `:debug-remote`, `:dbg-tcp` | Connect to a debug adapter by TCP address and start a debugging session from a given template with given parameters. |
+| `:debug-eval` | Evaluate expression in current debug context. |
 | `:vsplit`, `:vs` | Open the file in a vertical split. |
+| `:vsplit-new`, `:vnew` | Open a scratch buffer in a vertical split. |
 | `:hsplit`, `:hs`, `:sp` | Open the file in a horizontal split. |
+| `:hsplit-new`, `:hnew` | Open a scratch buffer in a horizontal split. |
 | `:tutor` | Open the tutorial. |
 | `:goto`, `:g` | Go to line number. |
-| `:set-option`, `:set` | Set a config option at runtime |
+| `:set-language`, `:lang` | Set the language of current buffer. |
+| `:set-option`, `:set` | Set a config option at runtime. |
+| `:get-option`, `:get` | Get the current value of a config option. |
 | `:sort` | Sort ranges in selection. |
 | `:rsort` | Sort ranges in selection in reverse order. |
+| `:tree-sitter-subtree`, `:ts-subtree` | Display tree sitter subtree under cursor, primarily for debugging queries. |
+| `:config-reload` | Refreshes helix's config. |
+| `:config-open` | Open the helix config.toml file. |
+| `:pipe` | Pipe each selection to the shell command. |

book/src/guides/README.md

@@ -1,4 +1,4 @@
 # Guides
 
 This section contains guides for adding new language server configurations,
-tree-sitter grammers, textobject queries, etc.
+tree-sitter grammars, textobject queries, etc.

book/src/guides/adding_languages.md

@@ -1,45 +1,76 @@
 # Adding languages
 
-## Submodules
+## Language configuration
 
-To add a new language, you should first add a tree-sitter submodule. To do this,
+To add a new language, you need to add a `language` entry to the
-you can run the command
+[`languages.toml`][languages.toml] found in the root of the repository;
-```sh
+this `languages.toml` file is included at compilation time, and is
-git submodule add -f <repository> helix-syntax/languages/tree-sitter-<name>
+distinct from the `languages.toml` file in the user's [configuration
-```
-For example, to add tree-sitter-ocaml you would run
-```sh
-git submodule add -f https://github.com/tree-sitter/tree-sitter-ocaml helix-syntax/languages/tree-sitter-ocaml
-```
-Make sure the submodule is shallow by doing
-```sh
-git config -f .gitmodules submodule.helix-syntax/languages/tree-sitter-<name>.shallow true
-```
-
-or you can manually add `shallow = true` to `.gitmodules`.
-
-## languages.toml
-
-Next, you need to add the language to the [`languages.toml`][languages.toml] found in the root of
-the repository; this `languages.toml` file is included at compilation time, and
-is distinct from the `language.toml` file in the user's [configuration
 directory](../configuration.md).
 
+```toml
+[[language]]
+name = "mylang"
+scope = "scope.mylang"
+injection-regex = "^mylang$"
+file-types = ["mylang", "myl"]
+comment-token = "#"
+indent = { tab-width = 2, unit = "  " }
+language-server = { command = "mylang-lsp", args = ["--stdio"] }
+```
+
 These are the available keys and descriptions for the file.
 
-| Key                 | Description                                                   |
+| Key                   | Description                                                   |
-| ----                | -----------                                                   |
+| ----                  | -----------                                                   |
-| name                | The name of the language                                      |
+| `name`                | The name of the language                                      |
-| scope               | A string like `source.js` that identifies the language. Currently, we strive to match the scope names used by popular TextMate grammars and by the Linguist library. Usually `source.<name>` or `text.<name>` in case of markup languages |
+| `scope`               | A string like `source.js` that identifies the language. Currently, we strive to match the scope names used by popular TextMate grammars and by the Linguist library. Usually `source.<name>` or `text.<name>` in case of markup languages |
-| injection-regex     | regex pattern that will be tested against a language name in order to determine whether this language should be used for a potential [language injection][treesitter-language-injection] site. |
+| `injection-regex`     | regex pattern that will be tested against a language name in order to determine whether this language should be used for a potential [language injection][treesitter-language-injection] site. |
-| file-types          | The filetypes of the language, for example `["yml", "yaml"]`  |
+| `file-types`          | The filetypes of the language, for example `["yml", "yaml"]`. Extensions and full file names are supported.  |
-| shebangs            | The interpreters from the shebang line, for example `["sh", "bash"]` |
+| `shebangs`            | The interpreters from the shebang line, for example `["sh", "bash"]` |
-| roots               | A set of marker files to look for when trying to find the workspace root. For example `Cargo.lock`, `yarn.lock` |
+| `roots`               | A set of marker files to look for when trying to find the workspace root. For example `Cargo.lock`, `yarn.lock` |
-| auto-format         | Whether to autoformat this language when saving               |
+| `auto-format`         | Whether to autoformat this language when saving               |
-| diagnostic-severity | Minimal severity of diagnostic for it to be displayed. (Allowed values: `Error`, `Warning`, `Info`, `Hint`) |
+| `diagnostic-severity` | Minimal severity of diagnostic for it to be displayed. (Allowed values: `Error`, `Warning`, `Info`, `Hint`) |
-| comment-token       | The token to use as a comment-token                           |
+| `comment-token`       | The token to use as a comment-token                           |
-| indent              | The indent to use. Has sub keys `tab-width` and `unit`        |
+| `indent`              | The indent to use. Has sub keys `tab-width` and `unit`        |
-| config              | Language server configuration                                 |
+| `language-server`     | The Language Server to run. Has sub keys `command` and `args` |
+| `config`              | Language Server configuration                                 |
+| `grammar`             | The tree-sitter grammar to use (defaults to the value of `name`) |
+
+When adding a new language or Language Server configuration for an existing
+language, run `cargo xtask docgen` to add the new configuration to the
+[Language Support][lang-support] docs before creating a pull request.
+When adding a Language Server configuration, be sure to update the
+[Language Server Wiki][install-lsp-wiki] with installation notes.
+
+## Grammar configuration
+
+If a tree-sitter grammar is available for the language, add a new `grammar`
+entry to `languages.toml`.
+
+```toml
+[[grammar]]
+name = "mylang"
+source = { git = "https://github.com/example/mylang", rev = "a250c4582510ff34767ec3b7dcdd3c24e8c8aa68" }
+```
+
+Grammar configuration takes these keys:
+
+| Key      | Description                                                              |
+| ---      | -----------                                                              |
+| `name`   | The name of the tree-sitter grammar                                      |
+| `source` | The method of fetching the grammar - a table with a schema defined below |
+
+Where `source` is a table with either these keys when using a grammar from a
+git repository:
+
+| Key    | Description                                               |
+| ---    | -----------                                               |
+| `git`  | A git remote URL from which the grammar should be cloned  |
+| `rev`  | The revision (commit hash or tag) which should be fetched |
+| `subpath` | A path within the grammar directory which should be built. Some grammar repositories host multiple grammars (for example `tree-sitter-typescript` and `tree-sitter-ocaml`) in subdirectories. This key is used to point `hx --grammar build` to the correct path for compilation. When omitted, the root of repository is used |
+
+Or a `path` key with an absolute path to a locally available grammar directory.
 
 ## Queries
 
@@ -51,21 +82,17 @@ gives more info on how to write queries.
 
 > NOTE: When evaluating queries, the first matching query takes
 precedence, which is different from other editors like neovim where
-the last matching query supercedes the ones before it. See
+the last matching query supersedes the ones before it. See
 [this issue][neovim-query-precedence] for an example.
 
 ## Common Issues
 
-- If you get errors when building after switching branches, you may have to remove or update tree-sitter submodules. You can update submodules by running
+- If you get errors when running after switching branches, you may have to update the tree-sitter grammars. Run `hx --grammar fetch` to fetch the grammars and `hx --grammar build` to build any out-of-date grammars.
-    ```sh
-    git submodule sync; git submodule update --init
-    ```
-- Make sure to not use the `--remote` flag. To remove submodules look inside the `.gitmodules` and remove directories that are not present inside of it.
 
-- If a parser is segfaulting or you want to remove the parser, make sure to remove the submodule *and* the compiled parser in `runtime/grammar/<name>.so`
+- If a parser is segfaulting or you want to remove the parser, make sure to remove the compiled parser in `runtime/grammar/<name>.so`
-
-- The indents query is `indents.toml`, *not* `indents.scm`. See [this](https://github.com/helix-editor/helix/issues/114) issue for more information.
 
 [treesitter-language-injection]: https://tree-sitter.github.io/tree-sitter/syntax-highlighting#language-injection
 [languages.toml]: https://github.com/helix-editor/helix/blob/master/languages.toml
 [neovim-query-precedence]: https://github.com/helix-editor/helix/pull/1170#issuecomment-997294090
+[install-lsp-wiki]: https://github.com/helix-editor/helix/wiki/How-to-install-the-default-language-servers
+[lang-support]: ../lang-support.md

book/src/guides/indent.md

@@ -0,0 +1,79 @@
+# Adding Indent Queries
+
+Helix uses tree-sitter to correctly indent new lines. This requires
+a tree-sitter grammar and an `indent.scm` query file placed in
+`runtime/queries/{language}/indents.scm`. The indentation for a line
+is calculated by traversing the syntax tree from the lowest node at the
+beginning of the new line. Each of these nodes contributes to the total
+indent when it is captured by the query (in what way depends on the name
+of the capture).
+
+Note that it matters where these added indents begin. For example,
+multiple indent level increases that start on the same line only increase
+the total indent level by 1.
+
+## Scopes
+
+Added indents don't always apply to the whole node. For example, in most
+cases when a node should be indented, we actually only want everything
+except for its first line to be indented. For this, there are several
+scopes (more scopes may be added in the future if required):
+
+- `all`:
+This scope applies to the whole captured node. This is only different from
+`tail` when the captured node is the first node on its line.
+
+- `tail`:
+This scope applies to everything except for the first line of the
+captured node.
+
+Every capture type has a default scope which should do the right thing
+in most situations. When a different scope is required, this can be
+changed by using a `#set!` declaration anywhere in the pattern:
+```scm
+(assignment_expression
+  right: (_) @indent
+  (#set! "scope" "all"))
+```
+
+## Capture Types
+
+- `@indent` (default scope `tail`):
+Increase the indent level by 1. Multiple occurrences in the same line
+don't stack. If there is at least one `@indent` and one `@outdent`
+capture on the same line, the indent level isn't changed at all.
+
+- `@outdent` (default scope `all`):
+Decrease the indent level by 1. The same rules as for `@indent` apply.
+
+## Predicates
+
+In some cases, an S-expression cannot express exactly what pattern should be matched.
+For that, tree-sitter allows for predicates to appear anywhere within a pattern,
+similar to how `#set!` declarations work:
+```scm
+(some_kind
+  (child_kind) @indent
+  (#predicate? arg1 arg2 ...)
+)
+```
+The number of arguments depends on the predicate that's used.
+Each argument is either a capture (`@name`) or a string (`"some string"`).
+The following predicates are supported by tree-sitter:
+
+- `#eq?`/`#not-eq?`:
+The first argument (a capture) must/must not be equal to the second argument
+(a capture or a string).
+
+- `#match?`/`#not-match?`:
+The first argument (a capture) must/must not match the regex given in the
+second argument (a string).
+
+Additionally, we support some custom predicates for indent queries:
+
+- `#not-kind-eq?`:
+The kind of the first argument (a capture) must not be equal to the second
+argument (a string).
+
+- `#same-line?`/`#not-same-line?`:
+The captures given by the 2 arguments must/must not start on the same line.

book/src/guides/textobject.md

@@ -21,10 +21,27 @@ The following [captures][tree-sitter-captures] are recognized:
 | `class.inside`     |
 | `class.around`     |
 | `parameter.inside` |
+| `comment.inside`   |
+| `comment.around`   |
 
 [Example query files][textobject-examples] can be found in the helix GitHub repository.
 
+## Queries for Textobject Based Navigation
+
+[Tree-sitter based navigation][textobjects-nav] is done using captures in the
+following order:
+
+- `object.movement`
+- `object.around`
+- `object.inside`
+
+For example if a `function.around` capture has been already defined for a language
+in it's `textobjects.scm` file, function navigation should also work automatically.
+`function.movement` should be defined only if the node captured by `function.around`
+doesn't make sense in a navigation context.
+
 [textobjects]: ../usage.md#textobjects
+[textobjects-nav]: ../usage.md#tree-sitter-textobject-based-navigation
 [tree-sitter-queries]: https://tree-sitter.github.io/tree-sitter/using-parsers#query-syntax
 [tree-sitter-captures]: https://tree-sitter.github.io/tree-sitter/using-parsers#capturing-nodes
 [textobject-examples]: https://github.com/search?q=repo%3Ahelix-editor%2Fhelix+filename%3Atextobjects.scm&type=Code&ref=advsearch&l=&l=

book/src/install.md

@@ -19,7 +19,12 @@ brew install helix
 
 A [flake](https://nixos.wiki/wiki/Flakes) containing the package is available in
 the project root. The flake can also be used to spin up a reproducible development
-shell for working on Helix.
+shell for working on Helix with `nix develop`.
+
+Flake outputs are cached for each push to master using
+[Cachix](https://www.cachix.org/). With Cachix
+[installed](https://docs.cachix.org/installation), `cachix use helix` will
+configure Nix to use cached outputs when possible.
 
 ### Arch Linux
 
@@ -36,10 +41,16 @@ sudo dnf copr enable varlad/helix
 sudo dnf install helix
 ```
 
+### Void Linux
+
+```
+sudo xbps-install helix
+```
+
 ## Build from source
 
 ```
-git clone --recurse-submodules --shallow-submodules -j8 https://github.com/helix-editor/helix
+git clone https://github.com/helix-editor/helix
 cd helix
 cargo install --path helix-term
 ```
@@ -47,5 +58,16 @@ cargo install --path helix-term
 This will install the `hx` binary to `$HOME/.cargo/bin`.
 
 Helix also needs it's runtime files so make sure to copy/symlink the `runtime/` directory into the
-config directory (for example `~/.config/helix/runtime` on Linux/macOS). This location can be overriden
+config directory (for example `~/.config/helix/runtime` on Linux/macOS). This location can be overridden
 via the `HELIX_RUNTIME` environment variable.
+
+| OS        | command   |
+|-----------|-----------|
+|windows    |`xcopy runtime %AppData%/helix/runtime`|
+|linux/macos|`ln -s $PWD/runtime ~/.config/helix/runtime`
+
+## Building tree-sitter grammars
+
+Tree-sitter grammars must be fetched and compiled if not pre-packaged.
+Fetch grammars with `hx --grammar fetch` (requires `git`) and compile them
+with `hx --grammar build` (requires a C compiler).

book/src/keymap.md

@@ -9,40 +9,33 @@
 
 > NOTE: Unlike vim, `f`, `F`, `t` and `T` are not confined to the current line.
 
-| Key         | Description                                        | Command                     |
+| Key                   | Description                                        | Command                     |
-| -----       | -----------                                        | -------                     |
+| -----                 | -----------                                        | -------                     |
-| `h`/`Left`  | Move left                                          | `move_char_left`            |
+| `h`, `Left`           | Move left                                          | `move_char_left`            |
-| `j`/`Down`  | Move down                                          | `move_line_down`            |
+| `j`, `Down`           | Move down                                          | `move_line_down`            |
-| `k`/`Up`    | Move up                                            | `move_line_up`              |
+| `k`, `Up`             | Move up                                            | `move_line_up`              |
-| `l`/`Right` | Move right                                         | `move_char_right`           |
+| `l`, `Right`          | Move right                                         | `move_char_right`           |
-| `w`         | Move next word start                               | `move_next_word_start`      |
+| `w`                   | Move next word start                               | `move_next_word_start`      |
-| `b`         | Move previous word start                           | `move_prev_word_start`      |
+| `b`                   | Move previous word start                           | `move_prev_word_start`      |
-| `e`         | Move next word end                                 | `move_next_word_end`        |
+| `e`                   | Move next word end                                 | `move_next_word_end`        |
-| `W`         | Move next WORD start                               | `move_next_long_word_start` |
+| `W`                   | Move next WORD start                               | `move_next_long_word_start` |
-| `B`         | Move previous WORD start                           | `move_prev_long_word_start` |
+| `B`                   | Move previous WORD start                           | `move_prev_long_word_start` |
-| `E`         | Move next WORD end                                 | `move_next_long_word_end`   |
+| `E`                   | Move next WORD end                                 | `move_next_long_word_end`   |
-| `t`         | Find 'till next char                               | `find_till_char`            |
+| `t`                   | Find 'till next char                               | `find_till_char`            |
-| `f`         | Find next char                                     | `find_next_char`            |
+| `f`                   | Find next char                                     | `find_next_char`            |
-| `T`         | Find 'till previous char                           | `till_prev_char`            |
+| `T`                   | Find 'till previous char                           | `till_prev_char`            |
-| `F`         | Find previous char                                 | `find_prev_char`            |
+| `F`                   | Find previous char                                 | `find_prev_char`            |
-| `Alt-.`     | Repeat last motion (`f`, `t` or `m`)               | `repeat_last_motion`        |
+| `G`                   | Go to line number `<n>`                            | `goto_line`                 |
-| `Home`      | Move to the start of the line                      | `goto_line_start`           |
+| `Alt-.`               | Repeat last motion (`f`, `t` or `m`)               | `repeat_last_motion`        |
-| `End`       | Move to the end of the line                        | `goto_line_end`             |
+| `Home`                | Move to the start of the line                      | `goto_line_start`           |
-| `PageUp`    | Move page up                                       | `page_up`                   |
+| `End`                 | Move to the end of the line                        | `goto_line_end`             |
-| `PageDown`  | Move page down                                     | `page_down`                 |
+| `Ctrl-b`, `PageUp`    | Move page up                                       | `page_up`                   |
-| `Ctrl-u`    | Move half page up                                  | `half_page_up`              |
+| `Ctrl-f`, `PageDown`  | Move page down                                     | `page_down`                 |
-| `Ctrl-d`    | Move half page down                                | `half_page_down`            |
+| `Ctrl-u`              | Move half page up                                  | `half_page_up`              |
-| `Ctrl-i`    | Jump forward on the jumplist                       | `jump_forward`              |
+| `Ctrl-d`              | Move half page down                                | `half_page_down`            |
-| `Ctrl-o`    | Jump backward on the jumplist                      | `jump_backward`             |
+| `Ctrl-i`              | Jump forward on the jumplist                       | `jump_forward`              |
-| `Ctrl-s`    | Save the current selection to the jumplist         | `save_selection`            |
+| `Ctrl-o`              | Jump backward on the jumplist                      | `jump_backward`             |
-| `v`         | Enter [select (extend) mode](#select--extend-mode) | `select_mode`               |
+| `Ctrl-s`              | Save the current selection to the jumplist         | `save_selection`            |
-| `g`         | Enter [goto mode](#goto-mode)                      | N/A                         |
-| `m`         | Enter [match mode](#match-mode)                    | N/A                         |
-| `:`         | Enter command mode                                 | `command_mode`              |
-| `z`         | Enter [view mode](#view-mode)                      | N/A                         |
-| `Z`         | Enter sticky [view mode](#view-mode)               | N/A                         |
-| `Ctrl-w`    | Enter [window mode](#window-mode)                  | N/A                         |
-| `Space`     | Enter [space mode](#space-mode)                    | N/A                         |
 
 ### Changes
 
@@ -82,45 +75,50 @@
 
 #### Shell
 
-| Key                     | Description                                                      | Command               |
+| Key     | Description                                                                      | Command               |
-| ------                  | -----------                                                      | -------               |
+| ------  | -----------                                                                      | -------               |
-| <code>&#124;</code>     | Pipe each selection through shell command, replacing with output | `shell_pipe`          |
+| <code>&#124;</code>     | Pipe each selection through shell command, replacing with output                 | `shell_pipe`          |
-| <code>Alt-&#124;</code> | Pipe each selection into shell command, ignoring output          | `shell_pipe_to`       |
+| <code>Alt-&#124;</code> | Pipe each selection into shell command, ignoring output                          | `shell_pipe_to`       |
-| `!`                     | Run shell command, inserting output before each selection        | `shell_insert_output` |
+| `!`     | Run shell command, inserting output before each selection                        | `shell_insert_output` |
-| `Alt-!`                 | Run shell command, appending output after each selection         | `shell_append_output` |
+| `Alt-!` | Run shell command, appending output after each selection                         | `shell_append_output` |
+| `$`     | Pipe each selection into shell command, keep selections where command returned 0 | `shell_keep_pipe`     |
 
 
 ### Selection manipulation
 
-| Key      | Description                                                       | Command                              |
+| Key                   | Description                                                       | Command                              |
-| -----    | -----------                                                       | -------                              |
+| -----                 | -----------                                                       | -------                              |
-| `s`      | Select all regex matches inside selections                        | `select_regex`                       |
+| `s`                   | Select all regex matches inside selections                        | `select_regex`                       |
-| `S`      | Split selection into subselections on regex matches               | `split_selection`                    |
+| `S`                   | Split selection into subselections on regex matches               | `split_selection`                    |
-| `Alt-s`  | Split selection on newlines                                       | `split_selection_on_newline`         |
+| `Alt-s`               | Split selection on newlines                                       | `split_selection_on_newline`         |
-| `&`      | Align selection in columns                                        | `align_selections`                   |
+| `&`                   | Align selection in columns                                        | `align_selections`                   |
-| `_`      | Trim whitespace from the selection                                | `trim_selections`                    |
+| `_`                   | Trim whitespace from the selection                                | `trim_selections`                    |
-| `;`      | Collapse selection onto a single cursor                           | `collapse_selection`                 |
+| `;`                   | Collapse selection onto a single cursor                           | `collapse_selection`                 |
-| `Alt-;`  | Flip selection cursor and anchor                                  | `flip_selections`                    |
+| `Alt-;`               | Flip selection cursor and anchor                                  | `flip_selections`                    |
-| `,`      | Keep only the primary selection                                   | `keep_primary_selection`             |
+| `Alt-:`               | Ensures the selection is in forward direction                     | `ensure_selections_forward`          |
-| `Alt-,`  | Remove the primary selection                                      | `remove_primary_selection`           |
+| `,`                   | Keep only the primary selection                                   | `keep_primary_selection`             |
-| `C`      | Copy selection onto the next line (Add cursor below)              | `copy_selection_on_next_line`        |
+| `Alt-,`               | Remove the primary selection                                      | `remove_primary_selection`           |
-| `Alt-C`  | Copy selection onto the previous line (Add cursor above)          | `copy_selection_on_prev_line`        |
+| `C`                   | Copy selection onto the next line (Add cursor below)              | `copy_selection_on_next_line`        |
-| `(`      | Rotate main selection backward                                    | `rotate_selections_backward`         |
+| `Alt-C`               | Copy selection onto the previous line (Add cursor above)          | `copy_selection_on_prev_line`        |
-| `)`      | Rotate main selection forward                                     | `rotate_selections_forward`          |
+| `(`                   | Rotate main selection backward                                    | `rotate_selections_backward`         |
-| `Alt-(`  | Rotate selection contents backward                                | `rotate_selection_contents_backward` |
+| `)`                   | Rotate main selection forward                                     | `rotate_selections_forward`          |
-| `Alt-)`  | Rotate selection contents forward                                 | `rotate_selection_contents_forward`  |
+| `Alt-(`               | Rotate selection contents backward                                | `rotate_selection_contents_backward` |
-| `%`      | Select entire file                                                | `select_all`                         |
+| `Alt-)`               | Rotate selection contents forward                                 | `rotate_selection_contents_forward`  |
-| `x`      | Select current line, if already selected, extend to next line     | `extend_line`                        |
+| `%`                   | Select entire file                                                | `select_all`                         |
-| `X`      | Extend selection to line bounds (line-wise selection)             | `extend_to_line_bounds`              |
+| `x`                   | Select current line, if already selected, extend to next line     | `extend_line`                        |
-|          | Expand selection to parent syntax node TODO: pick a key (**TS**)  | `expand_selection`                   |
+| `X`                   | Extend selection to line bounds (line-wise selection)             | `extend_to_line_bounds`              |
-| `J`      | Join lines inside selection                                       | `join_selections`                    |
+| `J`                   | Join lines inside selection                                       | `join_selections`                    |
-| `K`      | Keep selections matching the regex                                | `keep_selections`                    |
+| `K`                   | Keep selections matching the regex                                | `keep_selections`                    |
-| `Alt-K`  | Remove selections matching the regex                              | `remove_selections`                  |
+| `Alt-K`               | Remove selections matching the regex                              | `remove_selections`                  |
-| `$`      | Pipe each selection into shell command, keep selections where command returned 0 | `shell_keep_pipe`     |
+| `Ctrl-c`              | Comment/uncomment the selections                                  | `toggle_comments`                    |
-| `Ctrl-c` | Comment/uncomment the selections                                  | `toggle_comments`                    |
+| `Alt-o`, `Alt-up`     | Expand selection to parent syntax node (**TS**)                   | `expand_selection`                   |
+| `Alt-i`, `Alt-down`   | Shrink syntax tree object selection (**TS**)                      | `shrink_selection`                   |
+| `Alt-p`, `Alt-left`   | Select previous sibling node in syntax tree (**TS**)              | `select_prev_sibling`                |
+| `Alt-n`, `Alt-right`  | Select next sibling node in syntax tree (**TS**)                  | `select_next_sibling`                |
 
 ### Search
 
+Search commands all operate on the `/` register by default. Use `"<char>` to operate on a different one.
 
 | Key   | Description                                 | Command              |
 | ----- | -----------                                 | -------              |
@@ -134,6 +132,17 @@
 
 These sub-modes are accessible from normal mode and typically switch back to normal mode after a command.
 
+| Key      | Description                                        | Command        |
+| -----    | -----------                                        | -------        |
+| `v`      | Enter [select (extend) mode](#select--extend-mode) | `select_mode`  |
+| `g`      | Enter [goto mode](#goto-mode)                      | N/A            |
+| `m`      | Enter [match mode](#match-mode)                    | N/A            |
+| `:`      | Enter command mode                                 | `command_mode` |
+| `z`      | Enter [view mode](#view-mode)                      | N/A            |
+| `Z`      | Enter sticky [view mode](#view-mode)               | N/A            |
+| `Ctrl-w` | Enter [window mode](#window-mode)                  | N/A            |
+| `Space`  | Enter [space mode](#space-mode)                    | N/A            |
+
 #### View mode
 
 View mode is intended for scrolling and manipulating the view without changing
@@ -142,18 +151,18 @@ key to return to normal mode after usage (useful when you're simply looking
 over text and not actively editing it).
 
 
-| Key           | Description                                               | Command             |
+| Key                  | Description                                               | Command             |
-| -----         | -----------                                               | -------             |
+| -----                | -----------                                               | -------             |
-| `z` , `c`     | Vertically center the line                                | `align_view_center` |
+| `z`, `c`             | Vertically center the line                                | `align_view_center` |
-| `t`           | Align the line to the top of the screen                   | `align_view_top`    |
+| `t`                  | Align the line to the top of the screen                   | `align_view_top`    |
-| `b`           | Align the line to the bottom of the screen                | `align_view_bottom` |
+| `b`                  | Align the line to the bottom of the screen                | `align_view_bottom` |
-| `m`           | Align the line to the middle of the screen (horizontally) | `align_view_middle` |
+| `m`                  | Align the line to the middle of the screen (horizontally) | `align_view_middle` |
-| `j` , `down`  | Scroll the view downwards                                 | `scroll_down`       |
+| `j`, `down`          | Scroll the view downwards                                 | `scroll_down`       |
-| `k` , `up`    | Scroll the view upwards                                   | `scroll_up`         |
+| `k`, `up`            | Scroll the view upwards                                   | `scroll_up`         |
-| `f`           | Move page down                                            | `page_down`         |
+| `Ctrl-f`, `PageDown` | Move page down                                            | `page_down`         |
-| `b`           | Move page up                                              | `page_up`           |
+| `Ctrl-b`, `PageUp`   | Move page up                                              | `page_up`           |
-| `d`           | Move half page down                                       | `half_page_down`    |
+| `Ctrl-d`             | Move half page down                                       | `half_page_down`    |
-| `u`           | Move half page up                                         | `half_page_up`      |
+| `Ctrl-u`             | Move half page up                                         | `half_page_up`      |
 
 #### Goto mode
 
@@ -182,7 +191,7 @@ Jumps to various locations.
 
 #### Match mode
 
-Enter this mode using `m` from normal mode. See the relavant section
+Enter this mode using `m` from normal mode. See the relevant section
 in [Usage](./usage.md) for an explanation about [surround](./usage.md#surround)
 and [textobject](./usage.md#textobject) usage.
 
@@ -201,19 +210,19 @@ TODO: Mappings for selecting syntax nodes (a superset of `[`).
 
 This layer is similar to vim keybindings as kakoune does not support window.
 
-| Key                    | Description                    | Command           |
+| Key                    | Description                                          | Command           |
-| -----                  | -------------                  | -------           |
+| -----                  | -------------                                        | -------           |
-| `w`, `Ctrl-w`          | Switch to next window          | `rotate_view`     |
+| `w`, `Ctrl-w`          | Switch to next window                                | `rotate_view`     |
-| `v`, `Ctrl-v`          | Vertical right split           | `vsplit`          |
+| `v`, `Ctrl-v`          | Vertical right split                                 | `vsplit`          |
-| `s`, `Ctrl-s`          | Horizontal bottom split        | `hsplit`          |
+| `s`, `Ctrl-s`          | Horizontal bottom split                              | `hsplit`          |
-| `h`, `Ctrl-h`, `left`  | Move to left split             | `jump_view_left`  |
+| `f`                    | Go to files in the selection in horizontal splits    | `goto_file`       |
-| `f`                    | Go to files in the selection in horizontal splits  | `goto_file`                |
+| `F`                    | Go to files in the selection in vertical splits      | `goto_file`       |
-| `F`                    | Go to files in the selection in vertical splits    | `goto_file`                |
+| `h`, `Ctrl-h`, `Left`  | Move to left split                                   | `jump_view_left`  |
-| `j`, `Ctrl-j`, `down`  | Move to split below            | `jump_view_down`  |
+| `j`, `Ctrl-j`, `Down`  | Move to split below                                  | `jump_view_down`  |
-| `k`, `Ctrl-k`, `up`    | Move to split above            | `jump_view_up`    |
+| `k`, `Ctrl-k`, `Up`    | Move to split above                                  | `jump_view_up`    |
-| `l`, `Ctrl-l`, `right` | Move to right split            | `jump_view_right` |
+| `l`, `Ctrl-l`, `Right` | Move to right split                                  | `jump_view_right` |
-| `q`, `Ctrl-q`          | Close current window           | `wclose`          |
+| `q`, `Ctrl-q`          | Close current window                                 | `wclose`          |
-| `o`, `Ctrl-o`          | Only keep the current window, closing all the others            | `wonly`           |
+| `o`, `Ctrl-o`          | Only keep the current window, closing all the others | `wonly`           |
 
 #### Space mode
 
@@ -237,6 +246,7 @@ This layer is a kludge of mappings, mostly pickers.
 | `Y`     | Yank main selection to clipboard                                        | `yank_main_selection_to_clipboard`  |
 | `R`     | Replace selections by clipboard contents                                | `replace_selections_with_clipboard` |
 | `/`     | Global search in workspace folder                                       | `global_search`                     |
+| `?`     | Open command palette                                                    | `command_palette`                   |
 
 > TIP: Global search displays results in a fuzzy picker, use `space + '` to bring it back up after opening a file.
 
@@ -253,44 +263,69 @@ Displays documentation for item under cursor.
 
 Mappings in the style of [vim-unimpaired](https://github.com/tpope/vim-unimpaired).
 
-| Key      | Description                                  | Command             |
+| Key      | Description                                  | Command               |
-| -----    | -----------                                  | -------             |
+| -----    | -----------                                  | -------               |
-| `[d`     | Go to previous diagnostic (**LSP**)          | `goto_prev_diag`    |
+| `[d`     | Go to previous diagnostic (**LSP**)          | `goto_prev_diag`      |
-| `]d`     | Go to next diagnostic (**LSP**)              | `goto_next_diag`    |
+| `]d`     | Go to next diagnostic (**LSP**)              | `goto_next_diag`      |
-| `[D`     | Go to first diagnostic in document (**LSP**) | `goto_first_diag`   |
+| `[D`     | Go to first diagnostic in document (**LSP**) | `goto_first_diag`     |
-| `]D`     | Go to last diagnostic in document (**LSP**)  | `goto_last_diag`    |
+| `]D`     | Go to last diagnostic in document (**LSP**)  | `goto_last_diag`      |
-| `[space` | Add newline above                            | `add_newline_above` |
+| `]f`     | Go to next function (**TS**)                 | `goto_next_function`  |
-| `]space` | Add newline below                            | `add_newline_below` |
+| `[f`     | Go to previous function (**TS**)             | `goto_prev_function`  |
+| `]c`     | Go to next class (**TS**)                    | `goto_next_class`     |
+| `[c`     | Go to previous class (**TS**)                | `goto_prev_class`     |
+| `]a`     | Go to next argument/parameter (**TS**)       | `goto_next_parameter` |
+| `[a`     | Go to previous argument/parameter (**TS**)   | `goto_prev_parameter` |
+| `]o`     | Go to next comment (**TS**)                  | `goto_next_comment`   |
+| `[o`     | Go to previous comment (**TS**)              | `goto_prev_comment`   |
+| `]p`     | Go to next paragraph                         | `goto_next_paragraph` |
+| `[p`     | Go to previous paragraph                     | `goto_prev_paragraph` |
+| `[space` | Add newline above                            | `add_newline_above`   |
+| `]space` | Add newline below                            | `add_newline_below`   |
 
 ## Insert Mode
 
-| Key                     | Description                 | Command                 |
+We support many readline/emacs style bindings in insert mode for
-| -----                   | -----------                 | -------                 |
+convenience. These can be helpful for making simple modifications
-| `Escape`                | Switch to normal mode       | `normal_mode`           |
+without escaping to normal mode, but beware that you will not have an
-| `Ctrl-x`                | Autocomplete                | `completion`            |
+undo-able "save point" until you return to normal mode.
-| `Ctrl-r`                | Insert a register content   | `insert_register`       |
+
-| `Ctrl-w`                | Delete previous word        | `delete_word_backward`  |
+| Key                       | Description                 | Command                 |
-| `Alt-d`                 | Delete next word            | `delete_word_forward`   |
+| -----                     | -----------                 | -------                 |
-| `Alt-b`, `Alt-Left`     | Backward a word             | `move_prev_word_end`    |
+| `Escape`                  | Switch to normal mode       | `normal_mode`           |
-| `Ctrl-b`, `Left`        | Backward a char             | `move_char_left`        |
+| `Ctrl-x`                  | Autocomplete                | `completion`            |
-| `Alt-f`, `Alt-Right`    | Forward a word              | `move_next_word_start`  |
+| `Ctrl-r`                  | Insert a register content   | `insert_register`       |
-| `Ctrl-f`, `Right`       | Forward a char              | `move_char_right`       |
+| `Ctrl-w`, `Alt-Backspace` | Delete previous word        | `delete_word_backward`  |
-| `Ctrl-e`, `End`         | move to line end            | `goto_line_end_newline` |
+| `Alt-d`                   | Delete next word            | `delete_word_forward`   |
-| `Ctrl-a`, `Home`        | move to line start          | `goto_line_start`       |
+| `Alt-b`, `Ctrl-Left`      | Backward a word             | `move_prev_word_end`    |
-| `Ctrl-u`                | delete to start of line     | `kill_to_line_start`    |
+| `Ctrl-b`, `Left`          | Backward a char             | `move_char_left`        |
-| `Ctrl-k`                | delete to end of line       | `kill_to_line_end`      |
+| `Alt-f`, `Ctrl-Right`     | Forward a word              | `move_next_word_start`  |
-| `backspace`, `Ctrl-h`   | delete previous char        | `delete_char_backward`  |
+| `Ctrl-f`, `Right`         | Forward a char              | `move_char_right`       |
-| `delete`, `Ctrl-d`      | delete previous char        | `delete_char_forward`   |
+| `Ctrl-e`, `End`           | Move to line end            | `goto_line_end_newline` |
-| `Ctrl-p`, `Up`          | move to previous line       | `move_line_up`          |
+| `Ctrl-a`, `Home`          | Move to line start          | `goto_line_start`       |
-| `Ctrl-n`, `Down`        | move to next line           | `move_line_down`        |
+| `Ctrl-u`                  | Delete to start of line     | `kill_to_line_start`    |
+| `Ctrl-k`                  | Delete to end of line       | `kill_to_line_end`      |
+| `Ctrl-j`, `Enter`         | Insert new line             | `insert_newline`        |
+| `Backspace`, `Ctrl-h`     | Delete previous char        | `delete_char_backward`  |
+| `Delete`, `Ctrl-d`        | Delete next char            | `delete_char_forward`   |
+| `Ctrl-p`, `Up`            | Move to previous line       | `move_line_up`          |
+| `Ctrl-n`, `Down`          | Move to next line           | `move_line_down`        |
+| `PageUp`                  | Move one page up            | `page_up`               |
+| `PageDown`                | Move one page down          | `page_down`             |
+| `Alt->`                   | Go to end of buffer         | `goto_file_end`         |
+| `Alt-<`                   | Go to start of buffer       | `goto_file_start`       |
 
 ## Select / extend mode
 
-I'm still pondering whether to keep this mode or not. It changes movement
+This mode echoes Normal mode, but changes any movements to extend
-commands (including goto) to extend the existing selection instead of replacing it.
+selections rather than replace them. Goto motions are also changed to
+extend, so that `vgl` for example extends the selection to the end of
+the line.
 
-> NOTE: It's a bit confusing at the moment because extend hasn't been
+Search is also affected. By default, `n` and `N` will remove the current
-> implemented for all movement commands yet.
+selection and select the next instance of the search term. Toggling this
+mode before pressing `n` or `N` makes it possible to keep the current
+selection. Toggling it on and off during your iterative searching allows
+you to selectively add search terms to your selections.
 
 # Picker
 
@@ -298,8 +333,12 @@ Keys to use within picker. Remapping currently not supported.
 
 | Key                          | Description       |
 | -----                        | -------------     |
-| `Up`, `Ctrl-k`, `Ctrl-p`     | Previous entry    |
+| `Up`, `Ctrl-p`               | Previous entry    |
-| `Down`, `Ctrl-j`, `Ctrl-n`   | Next entry        |
+| `PageUp`, `Ctrl-u`           | Page up           |
+| `Down`, `Ctrl-n`             | Next entry        |
+| `PageDown`, `Ctrl-d`         | Page down         |
+| `Home`                       | Go to first entry |
+| `End`                        | Go to last entry  |
 | `Ctrl-space`                 | Filter options    |
 | `Enter`                      | Open selected     |
 | `Ctrl-s`                     | Open horizontally |

book/src/lang-support.md

@@ -1,10 +1,16 @@
 # Language Support
 
-For more information like arguments passed to default LSP server,
+The following languages and Language Servers are supported. In order to use
-extensions assosciated with a filetype, custom LSP settings, filetype
+Language Server features, you must first [install][lsp-install-wiki] the
-specific indent settings, etc see the default
+appropriate Language Server.
-[`languages.toml`][languages.toml] file.
+
+Check the language support in your installed helix version with `hx --health`.
+
+Also see the [Language Configuration][lang-config] docs and the [Adding
+Languages][adding-languages] guide for more language configuration information.
 
 {{#include ./generated/lang-support.md}}
 
-[languages.toml]: https://github.com/helix-editor/helix/blob/master/languages.toml
+[lsp-install-wiki]: https://github.com/helix-editor/helix/wiki/How-to-install-the-default-language-servers
+[lang-config]: ./languages.md
+[adding-languages]: ./guides/adding_languages.md

book/src/languages.md

@@ -2,12 +2,41 @@
 
 Language-specific settings and settings for particular language servers can be configured in a `languages.toml` file placed in your [configuration directory](./configuration.md). Helix actually uses two `languages.toml` files, the [first one](https://github.com/helix-editor/helix/blob/master/languages.toml) is in the main helix repository; it contains the default settings for each language and is included in the helix binary at compile time. Users who want to see the available settings and options can either reference the helix repo's `languages.toml` file, or consult the table in the [adding languages](./guides/adding_languages.md) section.
 
+A local `languages.toml` can be created within a `.helix` directory. Its settings will be merged with both the global and default configs.
+
 Changes made to the `languages.toml` file in a user's [configuration directory](./configuration.md) are merged with helix's defaults on start-up, such that a user's settings will take precedence over defaults in the event of a collision. For example, the default `languages.toml` sets rust's `auto-format` to `true`. If a user wants to disable auto-format, they can change the `languages.toml` in their [configuration directory](./configuration.md) to make the rust entry read like the example below; the new key/value pair `auto-format = false` will override the default when the two sets of settings are merged on start-up:
 
-```
+```toml
 # in <config_dir>/helix/languages.toml
 
 [[language]]
 name = "rust"
 auto-format = false
 ```
+
+## Tree-sitter grammars
+
+Tree-sitter grammars can also be configured in `languages.toml`:
+
+```toml
+# in <config_dir>/helix/languages.toml
+
+[[grammar]]
+name = "rust"
+source = { git = "https://github.com/tree-sitter/tree-sitter-rust", rev = "a250c4582510ff34767ec3b7dcdd3c24e8c8aa68" }
+
+[[grammar]]
+name = "c"
+source = { path = "/path/to/tree-sitter-c" }
+```
+
+You may use a top-level `use-grammars` key to control which grammars are fetched and built.
+
+```toml
+# Note: this key must come **before** the [[language]] and [[grammar]] sections
+use-grammars = { only = [ "rust", "c", "cpp" ] }
+# or
+use-grammars = { except = [ "yaml", "json" ] }
+```
+
+When omitted, all grammars are fetched and built.

book/src/themes.md

@@ -1,14 +1,14 @@
 # Themes
 
-First you'll need to place selected themes in your `themes` directory (i.e `~/.config/helix/themes`), the directory might have to be created beforehand.
+To use a theme add `theme = "<name>"` to your [`config.toml`](./configuration.md) at the very top of the file before the first section or select it during runtime using `:theme <name>`.
-
-To use a custom theme add `theme = <name>` to your [`config.toml`](./configuration.md) or override it during runtime using `:theme <name>`.
-
-The default theme.toml can be found [here](https://github.com/helix-editor/helix/blob/master/theme.toml), and user submitted themes [here](https://github.com/helix-editor/helix/blob/master/runtime/themes). 
 
 ## Creating a theme
 
-First create a file with the name of your theme as file name (i.e `mytheme.toml`) and place it in your `themes` directory (i.e `~/.config/helix/themes`).
+Create a file with the name of your theme as file name (i.e `mytheme.toml`) and place it in your `themes` directory (i.e `~/.config/helix/themes`). The directory might have to be created beforehand.
+
+The names "default" and "base16_default" are reserved for the builtin themes and cannot be overridden by user defined themes.
+
+The default theme.toml can be found [here](https://github.com/helix-editor/helix/blob/master/theme.toml), and user submitted themes [here](https://github.com/helix-editor/helix/blob/master/runtime/themes). 
 
 Each line in the theme file is specified as below:
 
@@ -37,8 +37,8 @@ configuration values in your theme. To do this, add a table called
 `palette` to your theme file:
 
 ```toml
-ui.background = "white"
+"ui.background" = "white"
-ui.text = "black"
+"ui.text" = "black"
 
 [palette]
 white = "#ffffff"
@@ -147,6 +147,7 @@ We use a similar set of scopes as
     - `repeat` - `for`, `while`, `loop`
     - `import` - `import`, `export`
     - `return`
+    - `exception`
   - `operator` - `or`, `in`
   - `directive` - Preprocessor directives (`#if` in C) 
   - `function` - `fn`, `func`
@@ -157,7 +158,7 @@ We use a similar set of scopes as
   - `builtin`
   - `method`
   - `macro`
-  - `special` (preprocesor in C)
+  - `special` (preprocessor in C)
 
 - `tag` - Tags (e.g. `<body>` in HTML)
 
@@ -165,14 +166,17 @@ We use a similar set of scopes as
 
 - `markup`
   - `heading`
+    - `marker`
+    - `1`, `2`, `3`, `4`, `5`, `6` - heading text for h1 through h6
   - `list`
     - `unnumbered`
     - `numbered`
   - `bold`
   - `italic`
   - `link`
-    - `url`
+    - `url` - urls pointed to by links
-    - `label`
+    - `label` - non-url link references
+    - `text` - url and image descriptions in links
   - `quote`
   - `raw`
     - `inline`
@@ -188,33 +192,48 @@ We use a similar set of scopes as
 
 These scopes are used for theming the editor interface.
 
+- `markup`
+  - `normal`
+    - `completion` - for completion doc popup ui
+    - `hover` - for hover popup ui
+  - `heading`
+    - `completion` - for completion doc popup ui
+    - `hover` - for hover popup ui
+  - `raw`
+    - `inline`
+      - `completion` - for completion doc popup ui
+      - `hover` - for hover popup ui
 
-| Key                      | Notes                               |
-| ---                      | ---                                 |
-| `ui.background`          |                                     |
-| `ui.cursor`              |                                     |
-| `ui.cursor.insert`       |                                     |
-| `ui.cursor.select`       |                                     |
-| `ui.cursor.match`        | Matching bracket etc.               |
-| `ui.cursor.primary`      | Cursor with primary selection       |
-| `ui.linenr`              |                                     |
-| `ui.linenr.selected`     |                                     |
-| `ui.statusline`          | Statusline                          |
-| `ui.statusline.inactive` | Statusline (unfocused document)     |
-| `ui.popup`               |                                     |
-| `ui.window`              |                                     |
-| `ui.help`                |                                     |
-| `ui.text`                |                                     |
-| `ui.text.focus`          |                                     |
-| `ui.info`                |                                     |
-| `ui.info.text`           |                                     |
-| `ui.menu`                |                                     |
-| `ui.menu.selected`       |                                     |
-| `ui.selection`           | For selections in the editing area  |
-| `ui.selection.primary`   |                                     |
-| `warning`                | Diagnostics warning (gutter)        |
-| `error`                  | Diagnostics error (gutter)          |
-| `info`                   | Diagnostics info (gutter)           |
-| `hint`                   | Diagnostics hint (gutter)           |
-| `diagnostic`             | For text in editing area            |
 
+| Key                      | Notes                                          |
+| ---                      | ---                                            |
+| `ui.background`          |                                                |
+| `ui.cursor`              |                                                |
+| `ui.cursor.insert`       |                                                |
+| `ui.cursor.select`       |                                                |
+| `ui.cursor.match`        | Matching bracket etc.                          |
+| `ui.cursor.primary`      | Cursor with primary selection                  |
+| `ui.linenr`              | Line numbers                                   |
+| `ui.linenr.selected`     | Line number for the line the cursor is on      |
+| `ui.statusline`          | Statusline                                     |
+| `ui.statusline.inactive` | Statusline (unfocused document)                |
+| `ui.popup`               | Documentation popups (e.g space-k)             |
+| `ui.popup.info`          | Prompt for multiple key options                |
+| `ui.window`              | Border lines separating splits                 |
+| `ui.help`                | Description box for commands                   |
+| `ui.text`                | Command prompts, popup text, etc.              |
+| `ui.text.focus`          |                                                |
+| `ui.text.info`           | The key: command text in `ui.popup.info` boxes |
+| `ui.virtual.ruler`       | Ruler columns (see the [`editor.rulers` config][rulers-config])|
+| `ui.virtual.whitespace`  | Visible white-space characters                 |
+| `ui.menu`                | Code and command completion menus              |
+| `ui.menu.selected`       | Selected autocomplete item                     |
+| `ui.selection`           | For selections in the editing area             |
+| `ui.selection.primary`   |                                                |
+| `warning`                | Diagnostics warning (gutter)                   |
+| `error`                  | Diagnostics error (gutter)                     |
+| `info`                   | Diagnostics info (gutter)                      |
+| `hint`                   | Diagnostics hint (gutter)                      |
+| `diagnostic`             | For text in editing area                       |
+
+[rulers-config]: ./configuration.md#editor-section

book/src/title-page.md

@@ -0,0 +1,15 @@
+# Helix
+
+Docs for bleeding edge master can be found at
+[https://docs.helix-editor.com/master](https://docs.helix-editor.com/master).
+
+See the [usage] section for a quick overview of the editor, [keymap]
+section for all available keybindings and the [configuration] section
+for defining custom keybindings, setting themes, etc.
+
+Refer the [FAQ] for common questions.
+
+[FAQ]: https://github.com/helix-editor/helix/wiki/FAQ
+[usage]: ./usage.md
+[keymap]: ./keymap.md
+[configuration]: ./configuration.md

book/src/usage.md

@@ -42,7 +42,7 @@ helix. The keymappings have been inspired from [vim-sandwich](https://github.com
 `ms` acts on a selection, so select the text first and use `ms<char>`. `mr` and `md` work
 on the closest pairs found and selections are not required; use counts to act in outer pairs.
 
-It can also act on multiple seletions (yay!). For example, to change every occurance of `(use)` to `[use]`:
+It can also act on multiple selections (yay!). For example, to change every occurrence of `(use)` to `[use]`:
 
 - `%` to select the whole file
 - `s` to split the selections on a search term
@@ -66,11 +66,32 @@ Currently supported: `word`, `surround`, `function`, `class`, `parameter`.
 | `w`                    | Word                     |
 | `W`                    | WORD                     |
 | `(`, `[`, `'`, etc     | Specified surround pairs |
+| `m`                    | Closest surround pair    |
 | `f`                    | Function                 |
 | `c`                    | Class                    |
-| `p`                    | Parameter                |
+| `a`                    | Argument/parameter       |
+| `o`                    | Comment                  |
 
-Note: `f`, `c`, etc need a tree-sitter grammar active for the current
+> NOTE: `f`, `c`, etc need a tree-sitter grammar active for the current
 document and a special tree-sitter query file to work properly. [Only
-some grammars](https://github.com/search?q=repo%3Ahelix-editor%2Fhelix+filename%3Atextobjects.scm&type=Code&ref=advsearch&l=&l=)
+some grammars][lang-support] currently have the query file implemented.
-currently have the query file implemented. Contributions are welcome !
+Contributions are welcome!
+
+## Tree-sitter Textobject Based Navigation
+
+Navigating between functions, classes, parameters, etc is made
+possible by leveraging tree-sitter and textobjects queries. For
+example to move to the next function use `]f`, to move to previous
+class use `[c`, and so on.
+
+![tree-sitter-nav-demo][tree-sitter-nav-demo]
+
+See the [unimpaired][unimpaired-keybinds] section of the keybind
+documentation for the full reference.
+
+> NOTE: This feature is dependent on tree-sitter based textobjects
+and therefore requires the corresponding query file to work properly.
+
+[lang-support]: ./lang-support.md
+[unimpaired-keybinds]: ./keymap.md#unimpaired
+[tree-sitter-nav-demo]: https://user-images.githubusercontent.com/23398472/152332550-7dfff043-36a2-4aec-b8f2-77c13eb56d6f.gif

book/theme/book.js

@@ -601,10 +601,10 @@ function playground_text(playground) {
     });
 })();
 
-(function controllMenu() {
+(function controlMenu() {
     var menu = document.getElementById('menu-bar');
 
-    (function controllPosition() {
+    (function controlPosition() {
         var scrollTop = document.scrollingElement.scrollTop;
         var prevScrollTop = scrollTop;
         var minMenuY = -menu.clientHeight - 50;
@@ -647,7 +647,7 @@ function playground_text(playground) {
             prevScrollTop = scrollTop;
         }, { passive: true });
     })();
-    (function controllBorder() {
+    (function controlBorder() {
         menu.classList.remove('bordered');
         document.addEventListener('scroll', function () {
             if (menu.offsetTop === 0) {

book/theme/css/chrome.css

@@ -390,7 +390,7 @@ ul#searchresults span.teaser em {
 
 .chapter li {
     display: flex;
-    color: var(--sidebar-non-existant);
+    color: var(--sidebar-non-existent);
 }
 .chapter li a {
     display: block;

book/theme/css/variables.css

@@ -16,7 +16,7 @@
 
     --sidebar-bg: #14191f;
     --sidebar-fg: #c8c9db;
-    --sidebar-non-existant: #5c6773;
+    --sidebar-non-existent: #5c6773;
     --sidebar-active: #ffb454;
     --sidebar-spacer: #2d334f;
 
@@ -56,7 +56,7 @@
 
     --sidebar-bg: #292c2f;
     --sidebar-fg: #a1adb8;
-    --sidebar-non-existant: #505254;
+    --sidebar-non-existent: #505254;
     --sidebar-active: #3473ad;
     --sidebar-spacer: #393939;
 
@@ -96,7 +96,7 @@
 
     --sidebar-bg: #fafafa;
     --sidebar-fg: hsl(0, 0%, 0%);
-    --sidebar-non-existant: #aaaaaa;
+    --sidebar-non-existent: #aaaaaa;
     --sidebar-active: #1f1fff;
     --sidebar-spacer: #f4f4f4;
 
@@ -136,7 +136,7 @@
 
     --sidebar-bg: #282d3f;
     --sidebar-fg: #c8c9db;
-    --sidebar-non-existant: #505274;
+    --sidebar-non-existent: #505274;
     --sidebar-active: #2b79a2;
     --sidebar-spacer: #2d334f;
 
@@ -176,7 +176,7 @@
 
     --sidebar-bg: #3b2e2a;
     --sidebar-fg: #c8c9db;
-    --sidebar-non-existant: #505254;
+    --sidebar-non-existent: #505254;
     --sidebar-active: #e69f67;
     --sidebar-spacer: #45373a;
 
@@ -217,7 +217,7 @@
 
         --sidebar-bg: #292c2f;
         --sidebar-fg: #a1adb8;
-        --sidebar-non-existant: #505254;
+        --sidebar-non-existent: #505254;
         --sidebar-active: #3473ad;
         --sidebar-spacer: #393939;
 
@@ -259,7 +259,7 @@
 
     --sidebar-bg: #281733;
     --sidebar-fg: #c8c9db;
-    --sidebar-non-existant: #505274;
+    --sidebar-non-existent: #505274;
     --sidebar-active: #a4a0e8;
     --sidebar-spacer: #2d334f;
 
@@ -304,7 +304,7 @@
 
     --sidebar-bg: #281733;
     --sidebar-fg: #c8c9db;
-    --sidebar-non-existant: #505274;
+    --sidebar-non-existent: #505274;
     --sidebar-active: #a4a0e8;
     --sidebar-spacer: #2d334f;
 

contrib/Helix.desktop

@@ -0,0 +1,89 @@
+[Desktop Entry]
+Name=Helix
+GenericName=Text Editor
+GenericName[de]=Texteditor
+GenericName[fr]=Éditeur de texte
+GenericName[ru]=Текстовый редактор
+GenericName[sr]=Едитор текст
+GenericName[tr]=Metin Düzenleyici
+Comment=Edit text files
+Comment[af]=Redigeer tekslêers
+Comment[am]=የጽሑፍ ፋይሎች ያስተካክሉ
+Comment[ar]=حرّر ملفات نصية
+Comment[az]=Mətn fayllarını redaktə edin
+Comment[be]=Рэдагаваньне тэкставых файлаў
+Comment[bg]=Редактиране на текстови файлове
+Comment[bn]=টেক্স্ট ফাইল এডিট করুন
+Comment[bs]=Izmijeni tekstualne datoteke
+Comment[ca]=Edita fitxers de text
+Comment[cs]=Úprava textových souborů
+Comment[cy]=Golygu ffeiliau testun
+Comment[da]=Redigér tekstfiler
+Comment[de]=Textdateien bearbeiten
+Comment[el]=Επεξεργασία αρχείων κειμένου
+Comment[en_CA]=Edit text files
+Comment[en_GB]=Edit text files
+Comment[es]=Edita archivos de texto
+Comment[et]=Redigeeri tekstifaile
+Comment[eu]=Editatu testu-fitxategiak
+Comment[fa]=ویرایش پرونده‌های متنی
+Comment[fi]=Muokkaa tekstitiedostoja
+Comment[fr]=Éditer des fichiers texte
+Comment[ga]=Eagar comhad Téacs
+Comment[gu]=લખાણ ફાઇલોમાં ફેરફાર કરો
+Comment[he]=ערוך קבצי טקסט
+Comment[hi]=पाठ फ़ाइलें संपादित करें
+Comment[hr]=Uređivanje tekstualne datoteke
+Comment[hu]=Szövegfájlok szerkesztése
+Comment[id]=Edit file teks
+Comment[it]=Modifica file di testo
+Comment[ja]=テキストファイルを編集します
+Comment[kn]=ಪಠ್ಯ ಕಡತಗಳನ್ನು ಸಂಪಾದಿಸು
+Comment[ko]=텍스트 파일을 편집합니다
+Comment[lt]=Redaguoti tekstines bylas
+Comment[lv]=Rediģēt teksta failus
+Comment[mk]=Уреди текстуални фајлови
+Comment[ml]=വാചക രചനകള് തിരുത്തുക
+Comment[mn]=Текст файл боловсруулах
+Comment[mr]=गद्य फाइल संपादित करा
+Comment[ms]=Edit fail teks
+Comment[nb]=Rediger tekstfiler
+Comment[ne]=पाठ फाइललाई संशोधन गर्नुहोस्
+Comment[nl]=Tekstbestanden bewerken
+Comment[nn]=Rediger tekstfiler
+Comment[no]=Rediger tekstfiler
+Comment[or]=ପାଠ୍ଯ ଫାଇଲଗୁଡ଼ିକୁ ସମ୍ପାଦନ କରନ୍ତୁ
+Comment[pa]=ਪਾਠ ਫਾਇਲਾਂ ਸੰਪਾਦਨ
+Comment[pl]=Edytor plików tekstowych
+Comment[pt]=Editar ficheiros de texto
+Comment[pt_BR]=Edite arquivos de texto
+Comment[ro]=Editare fişiere text
+Comment[ru]=Редактирование текстовых файлов
+Comment[sk]=Úprava textových súborov
+Comment[sl]=Urejanje datotek z besedili
+Comment[sq]=Përpuno files teksti
+Comment[sr]=Уређујте текст фајлове
+Comment[sr@Latn]=Izmeni tekstualne datoteke
+Comment[sv]=Redigera textfiler
+Comment[ta]=உரை கோப்புகளை தொகுக்கவும்
+Comment[th]=แก้ไขแฟ้มข้อความ
+Comment[tk]=Metin faýllary editle
+Comment[tr]=Metin dosyaları düzenleyin
+Comment[uk]=Редактор текстових файлів
+Comment[vi]=Soạn thảo tập tin văn bản
+Comment[wa]=Asspougnî des fitchîs tecses
+Comment[zh_CN]=编辑文本文件
+Comment[zh_TW]=編輯文字檔
+TryExec=hx
+Exec=hx %F
+Terminal=true
+Type=Application
+Keywords=Text;editor;
+Keywords[fr]=Texte;éditeur;
+Keywords[ru]=текст;текстовый редактор;
+Keywords[sr]=Текст;едитор;
+Keywords[tr]=Metin;düzenleyici;
+Icon=helix
+Categories=Utility;TextEditor;
+StartupNotify=false
+MimeType=text/english;text/plain;text/x-makefile;text/x-c++hdr;text/x-c++src;text/x-chdr;text/x-csrc;text/x-java;text/x-moc;text/x-pascal;text/x-tcl;text/x-tex;application/x-shellscript;text/x-c;text/x-c++;

contrib/completion/hx.bash

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Bash completion script for Helix editor
+
+_hx() {
+	# $1 command name
+	# $2 word being completed
+	# $3 word preceding
+	COMPREPLY=()
+
+	case "$3" in
+	-g | --grammar)
+		COMPREPLY=($(compgen -W "fetch build" -- $2))
+		;;
+	--health)
+		local languages=$(hx --health |tail -n '+7' |awk '{print $1}' |sed 's/\x1b\[[0-9;]*m//g')
+		COMPREPLY=($(compgen -W "$languages" -- $2))
+		;;
+	*)
+		COMPREPLY=($(compgen -fd -W "-h --help --tutor -V --version -v -vv -vvv --health -g --grammar" -- $2))
+		;;
+	esac
+} && complete -F _hx hx
+

contrib/completion/hx.fish

@@ -0,0 +1,12 @@
+#!/usr/bin/env fish
+# Fish completion script for Helix editor
+
+set -l langs (hx --health |tail -n '+7' |awk '{print $1}' |sed 's/\x1b\[[0-9;]*m//g')
+
+complete -c hx -s h -l help -d "Prints help information"
+complete -c hx -l tutor -d "Loads the tutorial"
+complete -c hx -l health -x -a "$langs" -d "Checks for errors in editor setup"
+complete -c hx -s g -l grammar -x -a "fetch build" -d "Fetches or builds tree-sitter grammars"
+complete -c hx -s v -o vv -o vvv -d "Increases logging verbosity"
+complete -c hx -s V -l version -d "Prints version information"
+

contrib/completion/hx.zsh

@@ -0,0 +1,29 @@
+#compdef _hx hx
+# Zsh completion script for Helix editor
+
+_hx() {
+	_arguments -C \
+		"-h[Prints help information]" \
+		"--help[Prints help information]" \
+		"-v[Increase logging verbosity]" \
+		"-vv[Increase logging verbosity]" \
+		"-vvv[Increase logging verbosity]" \
+		"-V[Prints version information]" \
+		"--version[Prints version information]" \
+		"--tutor[Loads the tutorial]" \
+		"--health[Checks for errors in editor setup]:language:->health" \
+		"-g[Fetches or builds tree-sitter grammars]:action:->grammar" \
+		"--grammar[Fetches or builds tree-sitter grammars]:action:->grammar" \
+		"*:file:_files"
+
+	case "$state" in
+	health)
+		local languages=($(hx --health |tail -n '+7' |awk '{print $1}' |sed 's/\x1b\[[0-9;]*m//g'))
+		_values 'language' $languages
+		;;
+	grammar)
+		_values 'action' fetch build
+		;;
+	esac
+}
+

docs/architecture.md

@@ -1,12 +1,13 @@
 
-| Crate        | Description                                            |
+| Crate        | Description                                                      |
-| -----------  | -----------                                            |
+| -----------  | -----------                                                      |
-| helix-core   | Core editing primitives, functional.                   |
+| helix-core   | Core editing primitives, functional.                             |
-| helix-syntax | Tree-sitter grammars                                   |
+| helix-lsp    | Language server client                                           |
-| helix-lsp    | Language server client                                 |
+| helix-dap    | Debug Adapter Protocol (DAP) client                              |
-| helix-view   | UI abstractions for use in backends, imperative shell. |
+| helix-loader | Functions for building, fetching, and loading external resources |
-| helix-term   | Terminal UI                                            |
+| helix-view   | UI abstractions for use in backends, imperative shell.           |
-| helix-tui    | TUI primitives, forked from tui-rs, inspired by Cursive |
+| helix-term   | Terminal UI                                                      |
+| helix-tui    | TUI primitives, forked from tui-rs, inspired by Cursive          |
 
 
 This document contains a high-level overview of Helix internals.
@@ -41,7 +42,7 @@ applying the transaction.
 > interface used to generate text edits.
 
 `Syntax` is the interface used to interact with tree-sitter ASTs for syntax
-highling and other features.
+highlighting and other features.
 
 ## View
 
@@ -54,15 +55,40 @@ A `Document` ties together the `Rope`, `Selection`(s), `Syntax`, document
 file.
 
 A `View` represents an open split in the UI. It holds the currently open
-document ID and other related state.
+document ID and other related state. Views encapsulate the gutter, status line,
+diagnostics, and the inner area where the code is displayed.
 
 > NOTE: Multiple views are able to display the same document, so the document
 > contains selections for each view. To retrieve, `document.selection()` takes
 > a `ViewId`.
 
+`Info` is the autoinfo box that shows hints when awaiting another key with bindings
+like `g` and `m`. It is attached to the viewport as a whole.
+
+`Surface` is like a buffer to which widgets draw themselves to, and the
+surface is then rendered on the screen on each cycle.
+
+`Rect`s are areas (simply an x and y coordinate with the origin at the
+screen top left and then a height and width) which are part of a
+`Surface`. They can be used to limit the area to which a `Component` can
+render. For example if we wrap a `Markdown` component in a `Popup`
+(think the documentation popup with space+k), Markdown's render method
+will get a Rect that is the exact size of the popup.
+
+Widgets are called `Component`s internally, and you can see most of them
+in `helix-term/src/ui`. Some components like `Popup` and `Overlay` can take
+other components as children.
+
+`Layer`s are how multiple components are displayed, and is simply a
+`Vec<Component>`. Layers are managed by the `Compositor`. On each top
+level render call, the compositor renders each component in the order
+they were pushed into the stack. This makes multiple components "layer"
+on top of one another. Hence we get a file picker displayed over the
+editor, etc.
+
 The `Editor` holds the global state: all the open documents, a tree
-representation of all the view splits, and a registry of language servers. To
+representation of all the view splits, the configuration, and a registry of 
-open or close files, interact with the editor.
+language servers. To open or close files, interact with the editor.
 
 ## LSP
 

flake.lock

@@ -1,12 +1,35 @@
 {
   "nodes": {
-    "devshell": {
+    "crane": {
+      "flake": false,
       "locked": {
-        "lastModified": 1639692811,
+        "lastModified": 1644785799,
-        "narHash": "sha256-wOOBH0fVsfNqw/5ZWRoKspyesoXBgiwEOUBH4c7JKEo=",
+        "narHash": "sha256-VpAJO1L0XeBvtCuNGK4IDKp6ENHIpTrlaZT7yfBCvwo=",
+        "owner": "ipetkov",
+        "repo": "crane",
+        "rev": "fc7a94f841347c88f2cb44217b2a3faa93e2a0b2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ipetkov",
+        "repo": "crane",
+        "type": "github"
+      }
+    },
+    "devshell": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1650201426,
+        "narHash": "sha256-u43Xf03ImFJWKLHddtjOpCJCHbuM0SQbb6FKR5NuFhk=",
         "owner": "numtide",
         "repo": "devshell",
-        "rev": "d3a1f5bec3632b33346865b1c165bf2420bb2f52",
+        "rev": "cb76bc75a0ee81f2d8676fe681f268c48dbd380e",
         "type": "github"
       },
       "original": {
@@ -15,7 +38,72 @@
         "type": "github"
       }
     },
+    "dream2nix": {
+      "inputs": {
+        "alejandra": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ],
+        "crane": "crane",
+        "flake-utils-pre-commit": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ],
+        "gomod2nix": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ],
+        "mach-nix": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ],
+        "nixpkgs": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ],
+        "node2nix": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ],
+        "poetry2nix": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ],
+        "pre-commit-hooks": [
+          "nixCargoIntegration",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1650199368,
+        "narHash": "sha256-35wZQDyohH6NI8ifT124p+TPDiwfRnQZkbUXc1yjGIE=",
+        "owner": "nix-community",
+        "repo": "dream2nix",
+        "rev": "feb7eabd0c1b24d47ea6ecd38b737319e75fd513",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "dream2nix",
+        "type": "github"
+      }
+    },
     "flake-utils": {
+      "locked": {
+        "lastModified": 1642700792,
+        "narHash": "sha256-XqHrk7hFb+zBvRg6Ghl+AZDq03ov6OshJLiSWOoX5es=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "846b2ae0fc4cc943637d3d1def4454213e203cba",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_2": {
       "locked": {
         "lastModified": 1637014545,
         "narHash": "sha256-26IZAc5yzlD9FlDT54io1oqG/bBoyka+FJk5guaX4x4=",
@@ -33,6 +121,7 @@
     "nixCargoIntegration": {
       "inputs": {
         "devshell": "devshell",
+        "dream2nix": "dream2nix",
         "nixpkgs": [
           "nixpkgs"
         ],
@@ -41,11 +130,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1639807801,
+        "lastModified": 1650262356,
-        "narHash": "sha256-y32tMq1LTRVbMW3QN5i98iOQjQt2QSsif3ayUkD1o3g=",
+        "narHash": "sha256-1ko5YhiLeFt9SjKvzq31IGrBJkHITNJc+08rod4P6lk=",
         "owner": "yusdacra",
         "repo": "nix-cargo-integration",
-        "rev": "b5bbaa4f5239e6f0619846f9a5380f07baa853d3",
+        "rev": "924aa1e39bbf1182cdf0dd6dfdb1d7b4a047ef6f",
         "type": "github"
       },
       "original": {
@@ -56,11 +145,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1639699734,
+        "lastModified": 1650161686,
-        "narHash": "sha256-tlX6WebGmiHb2Hmniff+ltYp+7dRfdsBxw9YczLsP60=",
+        "narHash": "sha256-70ZWAlOQ9nAZ08OU6WY7n4Ij2kOO199dLfNlvO/+pf8=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "03ec468b14067729a285c2c7cfa7b9434a04816c",
+        "rev": "1ffba9f2f683063c2b14c9f4d12c55ad5f4ed887",
         "type": "github"
       },
       "original": {
@@ -70,22 +159,6 @@
         "type": "github"
       }
     },
-    "nixpkgs_2": {
-      "locked": {
-        "lastModified": 1637453606,
-        "narHash": "sha256-Gy6cwUswft9xqsjWxFYEnx/63/qzaFUwatcbV5GF/GQ=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "8afc4e543663ca0a6a4f496262cd05233737e732",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixpkgs-unstable",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
     "root": {
       "inputs": {
         "nixCargoIntegration": "nixCargoIntegration",
@@ -95,15 +168,17 @@
     },
     "rust-overlay": {
       "inputs": {
-        "flake-utils": "flake-utils",
+        "flake-utils": "flake-utils_2",
-        "nixpkgs": "nixpkgs_2"
+        "nixpkgs": [
+          "nixpkgs"
+        ]
       },
       "locked": {
-        "lastModified": 1639880499,
+        "lastModified": 1650336226,
-        "narHash": "sha256-/BibDmFwgWuuTUkNVO6YlvuTSWM9dpBvlZoTAPs7ORI=",
+        "narHash": "sha256-A68t/BM3JPXUDFx9JGBk24euXvsaIZuPL28+hX5TmwA=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "c6c83589ae048af20d93d01eb07a4176012093d0",
+        "rev": "8cd3024e5b011218308eeee35c4839601af458f8",
         "type": "github"
       },
       "original": {

flake.nix

@@ -3,7 +3,10 @@
 
   inputs = {
     nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
-    rust-overlay.url = "github:oxalica/rust-overlay";
+    rust-overlay = {
+      url = "github:oxalica/rust-overlay";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
     nixCargoIntegration = {
       url = "github:yusdacra/nix-cargo-integration";
       inputs.nixpkgs.follows = "nixpkgs";
@@ -11,59 +14,55 @@
     };
   };
 
-  outputs = inputs@{ self, nixCargoIntegration, ... }:
+  outputs = inputs@{ nixpkgs, nixCargoIntegration, ... }:
     nixCargoIntegration.lib.makeOutputs {
       root = ./.;
-      buildPlatform = "crate2nix";
       renameOutputs = { "helix-term" = "helix"; };
       # Set default app to hx (binary is from helix-term release build)
       # Set default package to helix-term release build
-      defaultOutputs = { app = "hx"; package = "helix"; };
+      defaultOutputs = {
+        app = "hx";
+        package = "helix";
+      };
       overrides = {
         crateOverrides = common: _: {
-          helix-term = prev: {
+          helix-term = prev:
-            # link languages and theme toml files since helix-term expects them (for tests)
+            let
-            preConfigure = "ln -s ${common.root}/{languages.toml,theme.toml} ..";
+              inherit (common) pkgs;
-            buildInputs = (prev.buildInputs or [ ]) ++ [ common.cCompiler.cc.lib ];
+              grammars = pkgs.callPackage ./grammars.nix { };
-          };
+              runtimeDir = pkgs.runCommand "helix-runtime" { } ''
-          # link languages and theme toml files since helix-view expects them
+                mkdir -p $out
-          helix-view = _: { preConfigure = "ln -s ${common.root}/{languages.toml,theme.toml} .."; };
+                ln -s ${common.root}/runtime/* $out
-          helix-syntax = _prev: {
+                rm -r $out/grammars
-            preConfigure = "mkdir -p ../runtime/grammars";
+                ln -s ${grammars} $out/grammars
-            postInstall = "cp -r ../runtime $out/runtime";
+              '';
-          };
+            in
-        };
+            {
-        mainBuild = common: prev:
+              # disable fetching and building of tree-sitter grammars in the helix-term build.rs
-          let
+              HELIX_DISABLE_AUTO_GRAMMAR_BUILD = "1";
-            inherit (common) pkgs lib;
+              # link languages and theme toml files since helix-term expects them (for tests)
-            helixSyntax = lib.buildCrate {
+              preConfigure = "ln -s ${common.root}/{languages.toml,theme.toml,base16_theme.toml} ..";
-              root = self;
+              buildInputs = (prev.buildInputs or [ ]) ++ [ common.cCompiler.cc.lib ];
-              memberName = "helix-syntax";
+              nativeBuildInputs = [ pkgs.makeWrapper ];
-              defaultCrateOverrides = {
+
-                helix-syntax = common.crateOverrides.helix-syntax;
+              postFixup = ''
-              };
+                if [ -f "$out/bin/hx" ]; then
-              release = false;
+                  wrapProgram "$out/bin/hx" --set HELIX_RUNTIME "${runtimeDir}"
+                fi
+              '';
             };
-            runtimeDir = pkgs.runCommand "helix-runtime" { } ''
+        };
-              mkdir -p $out
-              ln -s ${common.root}/runtime/* $out
-              ln -sf ${helixSyntax}/runtime/grammars $out
-            '';
-          in
-          lib.optionalAttrs (common.memberName == "helix-term") {
-            nativeBuildInputs = [ pkgs.makeWrapper ];
-            postFixup = ''
-              if [ -f "$out/bin/hx" ]; then
-                wrapProgram "$out/bin/hx" --set HELIX_RUNTIME "${runtimeDir}"
-              fi
-            '';
-          };
         shell = common: prev: {
-          packages = prev.packages ++ (with common.pkgs; [ lld_13 lldb cargo-tarpaulin ]);
+          packages = prev.packages ++ (with common.pkgs; [ lld_13 lldb cargo-tarpaulin cargo-flamegraph vulkan-tools ]);
           env = prev.env ++ [
             { name = "HELIX_RUNTIME"; eval = "$PWD/runtime"; }
             { name = "RUST_BACKTRACE"; value = "1"; }
-            { name = "RUSTFLAGS"; value = "-C link-arg=-fuse-ld=lld -C target-cpu=native"; }
+            # { name = "RUSTFLAGS"; value = "-C link-arg=-fuse-ld=lld -C target-cpu=native -Clink-arg=-Wl,--no-rosegment"; }
+            { name = "LD_LIBRARY_PATH"; value = nixpkgs.lib.makeLibraryPath (with common.pkgs; [
+              wayland libxkbcommon xorg.libxcb
+              vulkan-loader # vulkan
+              # libGL # GLES instead of vulkan
+            ]); }
           ];
         };
       };

grammars.nix

@@ -0,0 +1,106 @@
+{ stdenv, lib, runCommand, yj }:
+let
+  # HACK: nix < 2.6 has a bug in the toml parser, so we convert to JSON
+  # before parsing
+  languages-json = runCommand "languages-toml-to-json" { } ''
+    ${yj}/bin/yj -t < ${./languages.toml} > $out
+  '';
+  languagesConfig = if lib.versionAtLeast builtins.nixVersion "2.6.0" then
+      builtins.fromTOML (builtins.readFile ./languages.toml)
+    else
+      builtins.fromJSON (builtins.readFile (builtins.toPath languages-json));
+  isGitGrammar = (grammar:
+    builtins.hasAttr "source" grammar && builtins.hasAttr "git" grammar.source
+    && builtins.hasAttr "rev" grammar.source);
+  isGitHubGrammar = grammar: lib.hasPrefix "https://github.com" grammar.source.git;
+  toGitHubFetcher = url: let
+    match = builtins.match "https://github\.com/([^/]*)/([^/]*)/?" url;
+  in {
+    owner = builtins.elemAt match 0;
+    repo = builtins.elemAt match 1;
+  };
+  gitGrammars = builtins.filter isGitGrammar languagesConfig.grammar;
+  buildGrammar = grammar:
+    let
+      gh = toGitHubFetcher grammar.source.git;
+      sourceGit = builtins.fetchTree {
+        type = "git";
+        url = grammar.source.git;
+        rev = grammar.source.rev;
+        ref = grammar.source.ref or "HEAD";
+        shallow = true;
+      };
+      sourceGitHub = builtins.fetchTree {
+        type = "github";
+        owner = gh.owner;
+        repo = gh.repo;
+        inherit (grammar.source) rev;
+      };
+      source = if isGitHubGrammar grammar then sourceGitHub else sourceGit;
+    in stdenv.mkDerivation rec {
+      # see https://github.com/NixOS/nixpkgs/blob/fbdd1a7c0bc29af5325e0d7dd70e804a972eb465/pkgs/development/tools/parsing/tree-sitter/grammar.nix
+
+      pname = "helix-tree-sitter-${grammar.name}";
+      version = grammar.source.rev;
+
+      src = if builtins.hasAttr "subpath" grammar.source then
+        "${source}/${grammar.source.subpath}"
+      else
+        source;
+
+      dontUnpack = true;
+      dontConfigure = true;
+
+      FLAGS = [
+        "-I${src}/src"
+        "-g"
+        "-O3"
+        "-fPIC"
+        "-fno-exceptions"
+        "-Wl,-z,relro,-z,now"
+      ];
+
+      NAME = grammar.name;
+
+      buildPhase = ''
+        runHook preBuild
+
+        if [[ -e "$src/src/scanner.cc" ]]; then
+          $CXX -c "$src/src/scanner.cc" -o scanner.o $FLAGS
+        elif [[ -e "$src/src/scanner.c" ]]; then
+          $CC -c "$src/src/scanner.c" -o scanner.o $FLAGS
+        fi
+
+        $CC -c "$src/src/parser.c" -o parser.o $FLAGS
+        $CXX -shared -o $NAME.so *.o
+
+        ls -al
+
+        runHook postBuild
+      '';
+
+      installPhase = ''
+        runHook preInstall
+        mkdir $out
+        mv $NAME.so $out/
+        runHook postInstall
+      '';
+
+      # Strip failed on darwin: strip: error: symbols referenced by indirect symbol table entries that can't be stripped
+      fixupPhase = lib.optionalString stdenv.isLinux ''
+        runHook preFixup
+        $STRIP $out/$NAME.so
+        runHook postFixup
+      '';
+    };
+  builtGrammars = builtins.map (grammar: {
+    inherit (grammar) name;
+    artifact = buildGrammar grammar;
+  }) gitGrammars;
+  grammarLinks = builtins.map (grammar:
+    "ln -s ${grammar.artifact}/${grammar.name}.so $out/${grammar.name}.so")
+    builtGrammars;
+in runCommand "consolidated-helix-grammars" { } ''
+  mkdir -p $out
+  ${builtins.concatStringsSep "\n" grammarLinks}
+''

helix-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "helix-core"
-version = "0.5.0"
+version = "0.6.0"
 authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
 edition = "2021"
 license = "MPL-2.0"
@@ -11,19 +11,21 @@ homepage = "https://helix-editor.com"
 include = ["src/**/*", "README.md"]
 
 [features]
+unicode-lines = ["ropey/unicode_lines"]
 
 [dependencies]
-helix-syntax = { version = "0.5", path = "../helix-syntax" }
+helix-loader = { version = "0.6", path = "../helix-loader" }
 
-ropey = "1.3"
+ropey = { version = "1.4", default-features = false }
-smallvec = "1.7"
+smallvec = "1.8"
-tendril = "0.4.2"
+smartstring = "1.0.1"
-unicode-segmentation = "1.8"
+unicode-segmentation = "1.9"
 unicode-width = "0.1"
-unicode-general-category = "0.4"
+unicode-general-category = "0.5"
 # slab = "0.4.2"
-tree-sitter = "0.20"
+slotmap = "1.0"
-once_cell = "1.9"
+tree-sitter = "0.20" # TODO: use tree-sitter-facade / web-tree-sitter-sys on wasm32
+once_cell = "1.10"
 arc-swap = "1"
 regex = "1"
 
@@ -34,10 +36,11 @@ toml = "0.5"
 
 similar = "2.1"
 
-etcetera = "0.3"
 encoding_rs = "0.8"
 
 chrono = { version = "0.4", default-features = false, features = ["alloc", "std"] }
 
+etcetera = "0.3"
+
 [dev-dependencies]
 quickcheck = { version = "1", default-features = false }

helix-core/src/auto_pairs.rs

@@ -1,13 +1,17 @@
 //! When typing the opening character of one of the possible pairs defined below,
 //! this module provides the functionality to insert the paired closing character.
 
-use crate::{movement::Direction, Range, Rope, Selection, Tendril, Transaction};
+use crate::{
+    graphemes, movement::Direction, Range, Rope, RopeGraphemes, Selection, Tendril, Transaction,
+};
+use std::collections::HashMap;
+
 use log::debug;
 use smallvec::SmallVec;
 
 // Heavily based on https://github.com/codemirror/closebrackets/
 
-pub const PAIRS: &[(char, char)] = &[
+pub const DEFAULT_PAIRS: &[(char, char)] = &[
     ('(', ')'),
     ('{', '}'),
     ('[', ']'),
@@ -16,9 +20,95 @@ pub const PAIRS: &[(char, char)] = &[
     ('`', '`'),
 ];
 
-// [TODO] build this dynamically in language config. see #992
+/// The type that represents the collection of auto pairs,
-const OPEN_BEFORE: &str = "([{'\":;,> \n\r\u{000B}\u{000C}\u{0085}\u{2028}\u{2029}";
+/// keyed by the opener.
-const CLOSE_BEFORE: &str = ")]}'\":;,> \n\r\u{000B}\u{000C}\u{0085}\u{2028}\u{2029}"; // includes space and newlines
+#[derive(Debug, Clone)]
+pub struct AutoPairs(HashMap<char, Pair>);
+
+/// Represents the config for a particular pairing.
+#[derive(Debug, Clone, Copy)]
+pub struct Pair {
+    pub open: char,
+    pub close: char,
+}
+
+impl Pair {
+    /// true if open == close
+    pub fn same(&self) -> bool {
+        self.open == self.close
+    }
+
+    /// true if all of the pair's conditions hold for the given document and range
+    pub fn should_close(&self, doc: &Rope, range: &Range) -> bool {
+        let mut should_close = Self::next_is_not_alpha(doc, range);
+
+        if self.same() {
+            should_close &= Self::prev_is_not_alpha(doc, range);
+        }
+
+        should_close
+    }
+
+    pub fn next_is_not_alpha(doc: &Rope, range: &Range) -> bool {
+        let cursor = range.cursor(doc.slice(..));
+        let next_char = doc.get_char(cursor);
+        next_char.map(|c| !c.is_alphanumeric()).unwrap_or(true)
+    }
+
+    pub fn prev_is_not_alpha(doc: &Rope, range: &Range) -> bool {
+        let cursor = range.cursor(doc.slice(..));
+        let prev_char = prev_char(doc, cursor);
+        prev_char.map(|c| !c.is_alphanumeric()).unwrap_or(true)
+    }
+}
+
+impl From<&(char, char)> for Pair {
+    fn from(&(open, close): &(char, char)) -> Self {
+        Self { open, close }
+    }
+}
+
+impl From<(&char, &char)> for Pair {
+    fn from((open, close): (&char, &char)) -> Self {
+        Self {
+            open: *open,
+            close: *close,
+        }
+    }
+}
+
+impl AutoPairs {
+    /// Make a new AutoPairs set with the given pairs and default conditions.
+    pub fn new<'a, V: 'a, A>(pairs: V) -> Self
+    where
+        V: IntoIterator<Item = A>,
+        A: Into<Pair>,
+    {
+        let mut auto_pairs = HashMap::new();
+
+        for pair in pairs.into_iter() {
+            let auto_pair = pair.into();
+
+            auto_pairs.insert(auto_pair.open, auto_pair);
+
+            if auto_pair.open != auto_pair.close {
+                auto_pairs.insert(auto_pair.close, auto_pair);
+            }
+        }
+
+        Self(auto_pairs)
+    }
+
+    pub fn get(&self, ch: char) -> Option<&Pair> {
+        self.0.get(&ch)
+    }
+}
+
+impl Default for AutoPairs {
+    fn default() -> Self {
+        AutoPairs::new(DEFAULT_PAIRS.iter())
+    }
+}
 
 // insert hook:
 // Fn(doc, selection, char) => Option<Transaction>
@@ -34,21 +124,17 @@ const CLOSE_BEFORE: &str = ")]}'\":;,> \n\r\u{000B}\u{000C}\u{0085}\u{2028}\u{20
 //   middle of triple quotes, and more exotic pairs like Jinja's {% %}
 
 #[must_use]
-pub fn hook(doc: &Rope, selection: &Selection, ch: char) -> Option<Transaction> {
+pub fn hook(doc: &Rope, selection: &Selection, ch: char, pairs: &AutoPairs) -> Option<Transaction> {
     debug!("autopairs hook selection: {:#?}", selection);
 
-    for &(open, close) in PAIRS {
+    if let Some(pair) = pairs.get(ch) {
-        if open == ch {
+        if pair.same() {
-            if open == close {
+            return Some(handle_same(doc, selection, pair));
-                return Some(handle_same(doc, selection, open, CLOSE_BEFORE, OPEN_BEFORE));
+        } else if pair.open == ch {
-            } else {
+            return Some(handle_open(doc, selection, pair));
-                return Some(handle_open(doc, selection, open, close, CLOSE_BEFORE));
+        } else if pair.close == ch {
-            }
-        }
-
-        if close == ch {
             // && char_at pos == close
-            return Some(handle_close(doc, selection, open, close));
+            return Some(handle_close(doc, selection, pair));
         }
     }
 
@@ -63,43 +149,138 @@ fn prev_char(doc: &Rope, pos: usize) -> Option<char> {
     doc.get_char(pos - 1)
 }
 
+fn is_single_grapheme(doc: &Rope, range: &Range) -> bool {
+    let mut graphemes = RopeGraphemes::new(doc.slice(range.from()..range.to()));
+    let first = graphemes.next();
+    let second = graphemes.next();
+    debug!("first: {:#?}, second: {:#?}", first, second);
+    first.is_some() && second.is_none()
+}
+
 /// calculate what the resulting range should be for an auto pair insertion
 fn get_next_range(
+    doc: &Rope,
     start_range: &Range,
     offset: usize,
     typed_char: char,
     len_inserted: usize,
 ) -> Range {
-    let end_head = start_range.head + offset + typed_char.len_utf8();
+    // When the character under the cursor changes due to complete pair
+    // insertion, we must look backward a grapheme and then add the length
+    // of the insertion to put the resulting cursor in the right place, e.g.
+    //
+    // foo[\r\n] - anchor: 3, head: 5
+    // foo([)]\r\n - anchor: 4, head: 5
+    //
+    // foo[\r\n] - anchor: 3, head: 5
+    // foo'[\r\n] - anchor: 4, head: 6
+    //
+    // foo([)]\r\n - anchor: 4, head: 5
+    // foo()[\r\n] - anchor: 5, head: 7
+    //
+    // [foo]\r\n - anchor: 0, head: 3
+    // [foo(])\r\n - anchor: 0, head: 5
+
+    // inserting at the very end of the document after the last newline
+    if start_range.head == doc.len_chars() && start_range.anchor == doc.len_chars() {
+        return Range::new(
+            start_range.anchor + offset + typed_char.len_utf8(),
+            start_range.head + offset + typed_char.len_utf8(),
+        );
+    }
+
+    let single_grapheme = is_single_grapheme(doc, start_range);
+    let doc_slice = doc.slice(..);
+
+    // just skip over graphemes
+    if len_inserted == 0 {
+        let end_anchor = if single_grapheme {
+            graphemes::next_grapheme_boundary(doc_slice, start_range.anchor) + offset
+
+        // even for backward inserts with multiple grapheme selections,
+        // we want the anchor to stay where it is so that the relative
+        // selection does not change, e.g.:
+        //
+        // foo([) wor]d -> insert ) -> foo()[ wor]d
+        } else {
+            start_range.anchor + offset
+        };
+
+        return Range::new(
+            end_anchor,
+            graphemes::next_grapheme_boundary(doc_slice, start_range.head) + offset,
+        );
+    }
+
+    // trivial case: only inserted a single-char opener, just move the selection
+    if len_inserted == 1 {
+        let end_anchor = if single_grapheme || start_range.direction() == Direction::Backward {
+            start_range.anchor + offset + typed_char.len_utf8()
+        } else {
+            start_range.anchor + offset
+        };
+
+        return Range::new(
+            end_anchor,
+            start_range.head + offset + typed_char.len_utf8(),
+        );
+    }
+
+    // If the head = 0, then we must be in insert mode with a backward
+    // cursor, which implies the head will just move
+    let end_head = if start_range.head == 0 || start_range.direction() == Direction::Backward {
+        start_range.head + offset + typed_char.len_utf8()
+    } else {
+        // We must have a forward cursor, which means we must move to the
+        // other end of the grapheme to get to where the new characters
+        // are inserted, then move the head to where it should be
+        let prev_bound = graphemes::prev_grapheme_boundary(doc_slice, start_range.head);
+        debug!(
+            "prev_bound: {}, offset: {}, len_inserted: {}",
+            prev_bound, offset, len_inserted
+        );
+        prev_bound + offset + len_inserted
+    };
 
     let end_anchor = match (start_range.len(), start_range.direction()) {
         // if we have a zero width cursor, it shifts to the same number
         (0, _) => end_head,
 
-        // if we are inserting for a regular one-width cursor, the anchor
+        // If we are inserting for a regular one-width cursor, the anchor
-        // moves with the head
+        // moves with the head. This is the fast path for ASCII.
         (1, Direction::Forward) => end_head - 1,
         (1, Direction::Backward) => end_head + 1,
 
-        // if we are appending, the anchor stays where it is; only offset
+        (_, Direction::Forward) => {
-        // for multiple range insertions
+            if single_grapheme {
-        (_, Direction::Forward) => start_range.anchor + offset,
+                graphemes::prev_grapheme_boundary(doc.slice(..), start_range.head)
+                    + typed_char.len_utf8()
 
-        // when we are inserting in front of a selection, we need to move
+            // if we are appending, the anchor stays where it is; only offset
-        // the anchor over by however many characters were inserted overall
+            // for multiple range insertions
-        (_, Direction::Backward) => start_range.anchor + offset + len_inserted,
+            } else {
+                start_range.anchor + offset
+            }
+        }
+
+        (_, Direction::Backward) => {
+            if single_grapheme {
+                // if we're backward, then the head is at the first char
+                // of the typed char, so we need to add the length of
+                // the closing char
+                graphemes::prev_grapheme_boundary(doc.slice(..), start_range.anchor) + len_inserted
+            } else {
+                // when we are inserting in front of a selection, we need to move
+                // the anchor over by however many characters were inserted overall
+                start_range.anchor + offset + len_inserted
+            }
+        }
     };
 
     Range::new(end_anchor, end_head)
 }
 
-fn handle_open(
+fn handle_open(doc: &Rope, selection: &Selection, pair: &Pair) -> Transaction {
-    doc: &Rope,
-    selection: &Selection,
-    open: char,
-    close: char,
-    close_before: &str,
-) -> Transaction {
     let mut end_ranges = SmallVec::with_capacity(selection.len());
     let mut offs = 0;
 
@@ -109,20 +290,21 @@ fn handle_open(
         let len_inserted;
 
         let change = match next_char {
-            Some(ch) if !close_before.contains(ch) => {
+            Some(_) if !pair.should_close(doc, start_range) => {
-                len_inserted = open.len_utf8();
+                len_inserted = pair.open.len_utf8();
-                (cursor, cursor, Some(Tendril::from_char(open)))
+                let mut tendril = Tendril::new();
+                tendril.push(pair.open);
+                (cursor, cursor, Some(tendril))
             }
-            // None | Some(ch) if close_before.contains(ch) => {}
             _ => {
                 // insert open & close
-                let pair = Tendril::from_iter([open, close]);
+                let pair_str = Tendril::from_iter([pair.open, pair.close]);
-                len_inserted = open.len_utf8() + close.len_utf8();
+                len_inserted = pair.open.len_utf8() + pair.close.len_utf8();
-                (cursor, cursor, Some(pair))
+                (cursor, cursor, Some(pair_str))
             }
         };
 
-        let next_range = get_next_range(start_range, offs, open, len_inserted);
+        let next_range = get_next_range(doc, start_range, offs, pair.open, len_inserted);
         end_ranges.push(next_range);
         offs += len_inserted;
 
@@ -134,7 +316,7 @@ fn handle_open(
     t
 }
 
-fn handle_close(doc: &Rope, selection: &Selection, _open: char, close: char) -> Transaction {
+fn handle_close(doc: &Rope, selection: &Selection, pair: &Pair) -> Transaction {
     let mut end_ranges = SmallVec::with_capacity(selection.len());
 
     let mut offs = 0;
@@ -144,15 +326,17 @@ fn handle_close(doc: &Rope, selection: &Selection, _open: char, close: char) ->
         let next_char = doc.get_char(cursor);
         let mut len_inserted = 0;
 
-        let change = if next_char == Some(close) {
+        let change = if next_char == Some(pair.close) {
             // return transaction that moves past close
             (cursor, cursor, None) // no-op
         } else {
-            len_inserted += close.len_utf8();
+            len_inserted += pair.close.len_utf8();
-            (cursor, cursor, Some(Tendril::from_char(close)))
+            let mut tendril = Tendril::new();
+            tendril.push(pair.close);
+            (cursor, cursor, Some(tendril))
         };
 
-        let next_range = get_next_range(start_range, offs, close, len_inserted);
+        let next_range = get_next_range(doc, start_range, offs, pair.close, len_inserted);
         end_ranges.push(next_range);
         offs += len_inserted;
 
@@ -165,13 +349,7 @@ fn handle_close(doc: &Rope, selection: &Selection, _open: char, close: char) ->
 }
 
 /// handle cases where open and close is the same, or in triples ("""docstring""")
-fn handle_same(
+fn handle_same(doc: &Rope, selection: &Selection, pair: &Pair) -> Transaction {
-    doc: &Rope,
-    selection: &Selection,
-    token: char,
-    close_before: &str,
-    open_before: &str,
-) -> Transaction {
     let mut end_ranges = SmallVec::with_capacity(selection.len());
 
     let mut offs = 0;
@@ -179,30 +357,26 @@ fn handle_same(
     let transaction = Transaction::change_by_selection(doc, selection, |start_range| {
         let cursor = start_range.cursor(doc.slice(..));
         let mut len_inserted = 0;
-
         let next_char = doc.get_char(cursor);
-        let prev_char = prev_char(doc, cursor);
 
-        let change = if next_char == Some(token) {
+        let change = if next_char == Some(pair.open) {
             //  return transaction that moves past close
             (cursor, cursor, None) // no-op
         } else {
-            let mut pair = Tendril::with_capacity(2 * token.len_utf8() as u32);
+            let mut pair_str = Tendril::new();
-            pair.push_char(token);
+            pair_str.push(pair.open);
 
             // for equal pairs, don't insert both open and close if either
             // side has a non-pair char
-            if (next_char.is_none() || close_before.contains(next_char.unwrap()))
+            if pair.should_close(doc, start_range) {
-                && (prev_char.is_none() || open_before.contains(prev_char.unwrap()))
+                pair_str.push(pair.close);
-            {
-                pair.push_char(token);
             }
 
-            len_inserted += pair.len();
+            len_inserted += pair_str.len();
-            (cursor, cursor, Some(pair))
+            (cursor, cursor, Some(pair_str))
         };
 
-        let next_range = get_next_range(start_range, offs, token, len_inserted);
+        let next_range = get_next_range(doc, start_range, offs, pair.open, len_inserted);
         end_ranges.push(next_range);
         offs += len_inserted;
 
@@ -219,22 +393,26 @@ mod test {
     use super::*;
     use smallvec::smallvec;
 
+    const LINE_END: &str = crate::DEFAULT_LINE_ENDING.as_str();
+
     fn differing_pairs() -> impl Iterator<Item = &'static (char, char)> {
-        PAIRS.iter().filter(|(open, close)| open != close)
+        DEFAULT_PAIRS.iter().filter(|(open, close)| open != close)
     }
 
     fn matching_pairs() -> impl Iterator<Item = &'static (char, char)> {
-        PAIRS.iter().filter(|(open, close)| open == close)
+        DEFAULT_PAIRS.iter().filter(|(open, close)| open == close)
     }
 
     fn test_hooks(
         in_doc: &Rope,
         in_sel: &Selection,
         ch: char,
+        pairs: &[(char, char)],
         expected_doc: &Rope,
         expected_sel: &Selection,
     ) {
-        let trans = hook(&in_doc, &in_sel, ch).unwrap();
+        let pairs = AutoPairs::new(pairs.iter());
+        let trans = hook(in_doc, in_sel, ch, &pairs).unwrap();
         let mut actual_doc = in_doc.clone();
         assert!(trans.apply(&mut actual_doc));
         assert_eq!(expected_doc, &actual_doc);
@@ -244,7 +422,8 @@ mod test {
     fn test_hooks_with_pairs<I, F, R>(
         in_doc: &Rope,
         in_sel: &Selection,
-        pairs: I,
+        test_pairs: I,
+        pairs: &[(char, char)],
         get_expected_doc: F,
         actual_sel: &Selection,
     ) where
@@ -253,11 +432,12 @@ mod test {
         R: Into<Rope>,
         Rope: From<R>,
     {
-        pairs.into_iter().for_each(|(open, close)| {
+        test_pairs.into_iter().for_each(|(open, close)| {
             test_hooks(
                 in_doc,
                 in_sel,
                 *open,
+                pairs,
                 &Rope::from(get_expected_doc(*open, *close)),
                 actual_sel,
             )
@@ -270,12 +450,66 @@ mod test {
     #[test]
     fn test_insert_blank() {
         test_hooks_with_pairs(
-            &Rope::new(),
+            &Rope::from(LINE_END),
             &Selection::single(1, 0),
-            PAIRS,
+            DEFAULT_PAIRS,
-            |open, close| format!("{}{}", open, close),
+            DEFAULT_PAIRS,
+            |open, close| format!("{}{}{}", open, close, LINE_END),
             &Selection::single(2, 1),
         );
+
+        let empty_doc = Rope::from(format!("{line_end}{line_end}", line_end = LINE_END));
+
+        test_hooks_with_pairs(
+            &empty_doc,
+            &Selection::single(empty_doc.len_chars(), LINE_END.len()),
+            DEFAULT_PAIRS,
+            DEFAULT_PAIRS,
+            |open, close| {
+                format!(
+                    "{line_end}{open}{close}{line_end}",
+                    open = open,
+                    close = close,
+                    line_end = LINE_END
+                )
+            },
+            &Selection::single(LINE_END.len() + 2, LINE_END.len() + 1),
+        );
+    }
+
+    #[test]
+    fn test_insert_before_multi_code_point_graphemes() {
+        for (_, close) in differing_pairs() {
+            test_hooks(
+                &Rope::from(format!("hello 👨‍👩‍👧‍👦 goodbye{}", LINE_END)),
+                &Selection::single(13, 6),
+                *close,
+                DEFAULT_PAIRS,
+                &Rope::from(format!("hello {}👨‍👩‍👧‍👦 goodbye{}", close, LINE_END)),
+                &Selection::single(14, 7),
+            );
+        }
+    }
+
+    #[test]
+    fn test_insert_at_end_of_document() {
+        test_hooks_with_pairs(
+            &Rope::from(LINE_END),
+            &Selection::single(LINE_END.len(), LINE_END.len()),
+            DEFAULT_PAIRS,
+            DEFAULT_PAIRS,
+            |open, close| format!("{}{}{}", LINE_END, open, close),
+            &Selection::single(LINE_END.len() + 1, LINE_END.len() + 1),
+        );
+
+        test_hooks_with_pairs(
+            &Rope::from(format!("foo{}", LINE_END)),
+            &Selection::single(3 + LINE_END.len(), 3 + LINE_END.len()),
+            DEFAULT_PAIRS,
+            DEFAULT_PAIRS,
+            |open, close| format!("foo{}{}{}", LINE_END, open, close),
+            &Selection::single(LINE_END.len() + 4, LINE_END.len() + 4),
+        );
     }
 
     /// [] -> append ( -> ([])
@@ -283,11 +517,21 @@ mod test {
     fn test_append_blank() {
         test_hooks_with_pairs(
             // this is what happens when you have a totally blank document and then append
-            &Rope::from("\n\n"),
+            &Rope::from(format!("{line_end}{line_end}", line_end = LINE_END)),
-            &Selection::single(0, 2),
+            // before inserting the pair, the cursor covers all of both empty lines
-            PAIRS,
+            &Selection::single(0, LINE_END.len() * 2),
-            |open, close| format!("\n{}{}\n", open, close),
+            DEFAULT_PAIRS,
-            &Selection::single(0, 3),
+            DEFAULT_PAIRS,
+            |open, close| {
+                format!(
+                    "{line_end}{open}{close}{line_end}",
+                    line_end = LINE_END,
+                    open = open,
+                    close = close
+                )
+            },
+            // after inserting pair, the cursor covers the first new line and the open char
+            &Selection::single(0, LINE_END.len() + 2),
         );
     }
 
@@ -302,7 +546,8 @@ mod test {
                 smallvec!(Range::new(1, 0), Range::new(2, 1), Range::new(3, 2),),
                 0,
             ),
-            PAIRS,
+            DEFAULT_PAIRS,
+            DEFAULT_PAIRS,
             |open, close| {
                 format!(
                     "{open}{close}\n{open}{close}\n{open}{close}\n",
@@ -324,11 +569,25 @@ mod test {
             &Rope::from("foo\n"),
             &Selection::single(2, 4),
             differing_pairs(),
+            DEFAULT_PAIRS,
             |open, close| format!("foo{}{}\n", open, close),
             &Selection::single(2, 5),
         );
     }
 
+    /// foo[] -> append to end of line ( -> foo([])
+    #[test]
+    fn test_append_single_cursor() {
+        test_hooks_with_pairs(
+            &Rope::from(format!("foo{}", LINE_END)),
+            &Selection::single(3, 3 + LINE_END.len()),
+            differing_pairs(),
+            DEFAULT_PAIRS,
+            |open, close| format!("foo{}{}{}", open, close, LINE_END),
+            &Selection::single(4, 5),
+        );
+    }
+
     /// fo[o]                fo[o(])
     /// fo[o] -> append ( -> fo[o(])
     /// fo[o]                fo[o(])
@@ -341,6 +600,7 @@ mod test {
                 0,
             ),
             differing_pairs(),
+            DEFAULT_PAIRS,
             |open, close| {
                 format!(
                     "foo{open}{close}\nfoo{open}{close}\nfoo{open}{close}\n",
@@ -355,18 +615,19 @@ mod test {
         );
     }
 
-    /// ([]) -> insert ) -> ()[]
+    /// ([)] -> insert ) -> ()[]
     #[test]
     fn test_insert_close_inside_pair() {
-        for (open, close) in PAIRS {
+        for (open, close) in DEFAULT_PAIRS {
-            let doc = Rope::from(format!("{}{}", open, close));
+            let doc = Rope::from(format!("{}{}{}", open, close, LINE_END));
 
             test_hooks(
                 &doc,
                 &Selection::single(2, 1),
                 *close,
+                DEFAULT_PAIRS,
                 &doc,
-                &Selection::single(3, 2),
+                &Selection::single(2 + LINE_END.len(), 2),
             );
         }
     }
@@ -374,15 +635,16 @@ mod test {
     /// [(]) -> append ) -> [()]
     #[test]
     fn test_append_close_inside_pair() {
-        for (open, close) in PAIRS {
+        for (open, close) in DEFAULT_PAIRS {
-            let doc = Rope::from(format!("{}{}\n", open, close));
+            let doc = Rope::from(format!("{}{}{}", open, close, LINE_END));
 
             test_hooks(
                 &doc,
                 &Selection::single(0, 2),
                 *close,
+                DEFAULT_PAIRS,
                 &doc,
-                &Selection::single(0, 3),
+                &Selection::single(0, 2 + LINE_END.len()),
             );
         }
     }
@@ -402,14 +664,14 @@ mod test {
             0,
         );
 
-        for (open, close) in PAIRS {
+        for (open, close) in DEFAULT_PAIRS {
             let doc = Rope::from(format!(
                 "{open}{close}\n{open}{close}\n{open}{close}\n",
                 open = open,
                 close = close
             ));
 
-            test_hooks(&doc, &sel, *close, &doc, &expected_sel);
+            test_hooks(&doc, &sel, *close, DEFAULT_PAIRS, &doc, &expected_sel);
         }
     }
 
@@ -428,14 +690,14 @@ mod test {
             0,
         );
 
-        for (open, close) in PAIRS {
+        for (open, close) in DEFAULT_PAIRS {
             let doc = Rope::from(format!(
                 "{open}{close}\n{open}{close}\n{open}{close}\n",
                 open = open,
                 close = close
             ));
 
-            test_hooks(&doc, &sel, *close, &doc, &expected_sel);
+            test_hooks(&doc, &sel, *close, DEFAULT_PAIRS, &doc, &expected_sel);
         }
     }
 
@@ -453,7 +715,14 @@ mod test {
                 close = close
             ));
 
-            test_hooks(&doc, &sel, *open, &expected_doc, &expected_sel);
+            test_hooks(
+                &doc,
+                &sel,
+                *open,
+                DEFAULT_PAIRS,
+                &expected_doc,
+                &expected_sel,
+            );
         }
     }
 
@@ -471,7 +740,14 @@ mod test {
                 close = close
             ));
 
-            test_hooks(&doc, &sel, *open, &expected_doc, &expected_sel);
+            test_hooks(
+                &doc,
+                &sel,
+                *open,
+                DEFAULT_PAIRS,
+                &expected_doc,
+                &expected_sel,
+            );
         }
     }
 
@@ -490,7 +766,14 @@ mod test {
                     outer_open, inner_open, inner_close, outer_close
                 ));
 
-                test_hooks(&doc, &sel, *inner_open, &expected_doc, &expected_sel);
+                test_hooks(
+                    &doc,
+                    &sel,
+                    *inner_open,
+                    DEFAULT_PAIRS,
+                    &expected_doc,
+                    &expected_sel,
+                );
             }
         }
     }
@@ -510,7 +793,14 @@ mod test {
                     outer_open, inner_open, inner_close, outer_close
                 ));
 
-                test_hooks(&doc, &sel, *inner_open, &expected_doc, &expected_sel);
+                test_hooks(
+                    &doc,
+                    &sel,
+                    *inner_open,
+                    DEFAULT_PAIRS,
+                    &expected_doc,
+                    &expected_sel,
+                );
             }
         }
     }
@@ -521,7 +811,8 @@ mod test {
         test_hooks_with_pairs(
             &Rope::from("word"),
             &Selection::single(1, 0),
-            PAIRS,
+            DEFAULT_PAIRS,
+            DEFAULT_PAIRS,
             |open, _| format!("{}word", open),
             &Selection::single(2, 1),
         )
@@ -533,7 +824,8 @@ mod test {
         test_hooks_with_pairs(
             &Rope::from("word"),
             &Selection::single(3, 0),
-            PAIRS,
+            DEFAULT_PAIRS,
+            DEFAULT_PAIRS,
             |open, _| format!("{}word", open),
             &Selection::single(4, 1),
         )
@@ -545,10 +837,17 @@ mod test {
         let sel = Selection::single(0, 4);
         let expected_sel = Selection::single(0, 5);
 
-        for (_, close) in PAIRS {
+        for (_, close) in DEFAULT_PAIRS {
             let doc = Rope::from("word");
             let expected_doc = Rope::from(format!("wor{}d", close));
-            test_hooks(&doc, &sel, *close, &expected_doc, &expected_sel);
+            test_hooks(
+                &doc,
+                &sel,
+                *close,
+                DEFAULT_PAIRS,
+                &expected_doc,
+                &expected_sel,
+            );
         }
     }
 
@@ -559,11 +858,27 @@ mod test {
             &Rope::from("foo word"),
             &Selection::single(7, 3),
             differing_pairs(),
+            DEFAULT_PAIRS,
             |open, close| format!("foo{}{} word", open, close),
             &Selection::single(9, 4),
         )
     }
 
+    /// foo([) wor]d -> insert ) -> foo()[ wor]d
+    #[test]
+    fn test_insert_close_inside_pair_trailing_word_with_selection() {
+        for (open, close) in differing_pairs() {
+            test_hooks(
+                &Rope::from(format!("foo{}{} word{}", open, close, LINE_END)),
+                &Selection::single(9, 4),
+                *close,
+                DEFAULT_PAIRS,
+                &Rope::from(format!("foo{}{} word{}", open, close, LINE_END)),
+                &Selection::single(9, 5),
+            )
+        }
+    }
+
     /// we want pairs that are *not* the same char to be inserted after
     /// a non-pair char, for cases like functions, but for pairs that are
     /// the same char, we want to *not* insert a pair to handle cases like "I'm"
@@ -572,7 +887,7 @@ mod test {
     /// word[]  -> insert ' -> word'[]
     #[test]
     fn test_insert_open_after_non_pair() {
-        let doc = Rope::from("word");
+        let doc = Rope::from(format!("word{}", LINE_END));
         let sel = Selection::single(5, 4);
         let expected_sel = Selection::single(6, 5);
 
@@ -580,7 +895,8 @@ mod test {
             &doc,
             &sel,
             differing_pairs(),
-            |open, close| format!("word{}{}", open, close),
+            DEFAULT_PAIRS,
+            |open, close| format!("word{}{}{}", open, close, LINE_END),
             &expected_sel,
         );
 
@@ -588,22 +904,34 @@ mod test {
             &doc,
             &sel,
             matching_pairs(),
-            |open, _| format!("word{}", open),
+            DEFAULT_PAIRS,
+            |open, _| format!("word{}{}", open, LINE_END),
             &expected_sel,
         );
     }
 
-    /// appending with only a cursor should stay a cursor
-    ///
-    /// [] -> append to end "foo -> "foo[]"
     #[test]
-    fn test_append_single_cursor() {
+    fn test_configured_pairs() {
+        let test_pairs = &[('`', ':'), ('+', '-')];
+
         test_hooks_with_pairs(
-            &Rope::from("\n"),
+            &Rope::from(LINE_END),
-            &Selection::single(0, 1),
+            &Selection::single(1, 0),
-            PAIRS,
+            test_pairs,
-            |open, close| format!("{}{}\n", open, close),
+            test_pairs,
-            &Selection::single(1, 2),
+            |open, close| format!("{}{}{}", open, close, LINE_END),
+            &Selection::single(2, 1),
         );
+
+        let doc = Rope::from(format!("foo`: word{}", LINE_END));
+
+        test_hooks(
+            &doc,
+            &Selection::single(9, 4),
+            ':',
+            test_pairs,
+            &doc,
+            &Selection::single(9, 5),
+        )
     }
 }

helix-core/src/chars.rs

@@ -91,12 +91,14 @@ mod test {
 
     #[test]
     fn test_categorize() {
-        const EOL_TEST_CASE: &'static str = "\n\r\u{000B}\u{000C}\u{0085}\u{2028}\u{2029}";
+        #[cfg(not(feature = "unicode-lines"))]
-        const WORD_TEST_CASE: &'static str =
+        const EOL_TEST_CASE: &str = "\n";
-            "_hello_world_あいうえおー1234567890１２３４５６７８９０";
+        #[cfg(feature = "unicode-lines")]
-        const PUNCTUATION_TEST_CASE: &'static str =
+        const EOL_TEST_CASE: &str = "\n\u{000B}\u{000C}\u{0085}\u{2028}\u{2029}";
+        const WORD_TEST_CASE: &str = "_hello_world_あいうえおー1234567890１２３４５６７８９０";
+        const PUNCTUATION_TEST_CASE: &str =
             "!\"#$%&\'()*+,-./:;<=>?@[\\]^`{|}~！”＃＄％＆’（）＊＋、。：；＜＝＞？＠「」＾｀｛｜｝～";
-        const WHITESPACE_TEST_CASE: &'static str = "  　   ";
+        const WHITESPACE_TEST_CASE: &str = "  　   ";
 
         for ch in EOL_TEST_CASE.chars() {
             assert_eq!(CharCategory::Eol, categorize_char(ch));

helix-core/src/comment.rs

@@ -72,7 +72,7 @@ pub fn toggle_line_comments(doc: &Rope, selection: &Selection, token: Option<&st
         let end = (end + 1).min(text.len_lines());
 
         lines.extend(start..end);
-        min_next_line = end + 1;
+        min_next_line = end;
     }
 
     let (commented, to_change, min, margin) = find_line_comment(token, text, lines);

helix-core/src/config.rs

@@ -0,0 +1,10 @@
+/// Syntax configuration loader based on built-in languages.toml.
+pub fn default_syntax_loader() -> crate::syntax::Configuration {
+    helix_loader::config::default_lang_config()
+        .try_into()
+        .expect("Could not serialize built-in languages.toml")
+}
+/// Syntax configuration loader based on user configured languages.toml.
+pub fn user_syntax_loader() -> Result<crate::syntax::Configuration, toml::de::Error> {
+    helix_loader::config::user_lang_config()?.try_into()
+}

helix-core/src/diff.rs

@@ -11,10 +11,6 @@ pub fn compare_ropes(old: &Rope, new: &Rope) -> Transaction {
     // A timeout is set so after 1 seconds, the algorithm will start
     // approximating. This is especially important for big `Rope`s or
     // `Rope`s that are extremely dissimilar to each other.
-    //
-    // Note: Ignore the clippy warning, as the trait bounds of
-    // `Transaction::change()` require an iterator implementing
-    // `ExactIterator`.
     let mut config = similar::TextDiff::configure();
     config.timeout(std::time::Duration::from_secs(1));
 
@@ -62,7 +58,7 @@ mod tests {
             let mut old = Rope::from(a);
             let new = Rope::from(b);
             compare_ropes(&old, &new).apply(&mut old);
-            old.to_string() == new.to_string()
+            old == new
         }
     }
 }

helix-core/src/graphemes.rs

@@ -120,6 +120,43 @@ pub fn nth_next_grapheme_boundary(slice: RopeSlice, char_idx: usize, n: usize) -
     chunk_char_idx + tmp
 }
 
+#[must_use]
+pub fn nth_next_grapheme_boundary_byte(slice: RopeSlice, mut byte_idx: usize, n: usize) -> usize {
+    // Bounds check
+    debug_assert!(byte_idx <= slice.len_bytes());
+
+    // Get the chunk with our byte index in it.
+    let (mut chunk, mut chunk_byte_idx, mut _chunk_char_idx, _) = slice.chunk_at_byte(byte_idx);
+
+    // Set up the grapheme cursor.
+    let mut gc = GraphemeCursor::new(byte_idx, slice.len_bytes(), true);
+
+    // Find the nth next grapheme cluster boundary.
+    for _ in 0..n {
+        loop {
+            match gc.next_boundary(chunk, chunk_byte_idx) {
+                Ok(None) => return slice.len_bytes(),
+                Ok(Some(n)) => {
+                    byte_idx = n;
+                    break;
+                }
+                Err(GraphemeIncomplete::NextChunk) => {
+                    chunk_byte_idx += chunk.len();
+                    let (a, _, _c, _) = slice.chunk_at_byte(chunk_byte_idx);
+                    chunk = a;
+                    // chunk_char_idx = c;
+                }
+                Err(GraphemeIncomplete::PreContext(n)) => {
+                    let ctx_chunk = slice.chunk_at_byte(n - 1).0;
+                    gc.provide_context(ctx_chunk, n - ctx_chunk.len());
+                }
+                _ => unreachable!(),
+            }
+        }
+    }
+    byte_idx
+}
+
 /// Finds the next grapheme boundary after the given char position.
 #[must_use]
 #[inline(always)]
@@ -127,6 +164,13 @@ pub fn next_grapheme_boundary(slice: RopeSlice, char_idx: usize) -> usize {
     nth_next_grapheme_boundary(slice, char_idx, 1)
 }
 
+/// Finds the next grapheme boundary after the given byte position.
+#[must_use]
+#[inline(always)]
+pub fn next_grapheme_boundary_byte(slice: RopeSlice, byte_idx: usize) -> usize {
+    nth_next_grapheme_boundary_byte(slice, byte_idx, 1)
+}
+
 /// Returns the passed char index if it's already a grapheme boundary,
 /// or the next grapheme boundary char index if not.
 #[must_use]
@@ -151,6 +195,23 @@ pub fn ensure_grapheme_boundary_prev(slice: RopeSlice, char_idx: usize) -> usize
     }
 }
 
+/// Returns the passed byte index if it's already a grapheme boundary,
+/// or the next grapheme boundary byte index if not.
+#[must_use]
+#[inline]
+pub fn ensure_grapheme_boundary_next_byte(slice: RopeSlice, byte_idx: usize) -> usize {
+    if byte_idx == 0 {
+        byte_idx
+    } else {
+        // TODO: optimize so we're not constructing grapheme cursor twice
+        if is_grapheme_boundary_byte(slice, byte_idx) {
+            byte_idx
+        } else {
+            next_grapheme_boundary_byte(slice, byte_idx)
+        }
+    }
+}
+
 /// Returns whether the given char position is a grapheme boundary.
 #[must_use]
 pub fn is_grapheme_boundary(slice: RopeSlice, char_idx: usize) -> bool {
@@ -179,6 +240,31 @@ pub fn is_grapheme_boundary(slice: RopeSlice, char_idx: usize) -> bool {
     }
 }
 
+/// Returns whether the given byte position is a grapheme boundary.
+#[must_use]
+pub fn is_grapheme_boundary_byte(slice: RopeSlice, byte_idx: usize) -> bool {
+    // Bounds check
+    debug_assert!(byte_idx <= slice.len_bytes());
+
+    // Get the chunk with our byte index in it.
+    let (chunk, chunk_byte_idx, _, _) = slice.chunk_at_byte(byte_idx);
+
+    // Set up the grapheme cursor.
+    let mut gc = GraphemeCursor::new(byte_idx, slice.len_bytes(), true);
+
+    // Determine if the given position is a grapheme cluster boundary.
+    loop {
+        match gc.is_boundary(chunk, chunk_byte_idx) {
+            Ok(n) => return n,
+            Err(GraphemeIncomplete::PreContext(n)) => {
+                let (ctx_chunk, ctx_byte_start, _, _) = slice.chunk_at_byte(n - 1);
+                gc.provide_context(ctx_chunk, ctx_byte_start);
+            }
+            Err(_) => unreachable!(),
+        }
+    }
+}
+
 /// An iterator over the graphemes of a `RopeSlice`.
 #[derive(Clone)]
 pub struct RopeGraphemes<'a> {
@@ -247,10 +333,7 @@ impl<'a> Iterator for RopeGraphemes<'a> {
         }
 
         if a < self.cur_chunk_start {
-            let a_char = self.text.byte_to_char(a);
+            Some(self.text.byte_slice(a..b))
-            let b_char = self.text.byte_to_char(b);
-
-            Some(self.text.slice(a_char..b_char))
         } else {
             let a2 = a - self.cur_chunk_start;
             let b2 = b - self.cur_chunk_start;

helix-core/src/history.rs

@@ -22,10 +22,10 @@ use std::time::{Duration, Instant};
 ///
 /// The current revision is the one currently displayed in the buffer.
 ///
-/// Commiting a new revision to the history will update the last child of the
+/// Committing a new revision to the history will update the last child of the
 /// current revision, and push a new revision to the end of the vector.
 ///
-/// Revisions are commited with a timestamp. :earlier and :later can be used
+/// Revisions are committed with a timestamp. :earlier and :later can be used
 /// to jump to the closest revision to a moment in time relative to the timestamp
 /// of the current revision plus (:later) or minus (:earlier) the duration
 /// given to the command. If a single integer is given, the editor will instead
@@ -33,7 +33,7 @@ use std::time::{Duration, Instant};
 ///
 /// Limitations:
 ///  * Changes in selections currently don't commit history changes. The selection
-///    will only be updated to the state after a commited buffer change.
+///    will only be updated to the state after a committed buffer change.
 ///  * The vector of history revisions is currently unbounded. This might
 ///    cause the memory consumption to grow significantly large during long
 ///    editing sessions.
@@ -288,7 +288,7 @@ pub enum UndoKind {
     TimePeriod(std::time::Duration),
 }
 
-/// A subset of sytemd.time time span syntax units.
+/// A subset of systemd.time time span syntax units.
 const TIME_UNITS: &[(&[&str], &str, u64)] = &[
     (&["seconds", "second", "sec", "s"], "seconds", 1),
     (&["minutes", "minute", "min", "m"], "minutes", 60),
@@ -448,8 +448,8 @@ mod test {
             change: crate::transaction::Change,
             instant: Instant,
         ) {
-            let txn = Transaction::change(&state.doc, vec![change.clone()].into_iter());
+            let txn = Transaction::change(&state.doc, vec![change].into_iter());
-            history.commit_revision_at_timestamp(&txn, &state, instant);
+            history.commit_revision_at_timestamp(&txn, state, instant);
             txn.apply(&mut state.doc);
         }
 

helix-core/src/increment/date_time.rs

@@ -195,82 +195,82 @@ struct DateField {
 impl DateField {
     fn from_specifier(specifier: &str) -> Option<Self> {
         match specifier {
-            "Y" => Some(DateField {
+            "Y" => Some(Self {
                 regex: r"\d{4}",
                 unit: DateUnit::Years,
                 max_len: 5,
             }),
-            "y" => Some(DateField {
+            "y" => Some(Self {
                 regex: r"\d\d",
                 unit: DateUnit::Years,
                 max_len: 2,
             }),
-            "m" => Some(DateField {
+            "m" => Some(Self {
                 regex: r"[0-1]\d",
                 unit: DateUnit::Months,
                 max_len: 2,
             }),
-            "d" => Some(DateField {
+            "d" => Some(Self {
                 regex: r"[0-3]\d",
                 unit: DateUnit::Days,
                 max_len: 2,
             }),
-            "-d" => Some(DateField {
+            "-d" => Some(Self {
                 regex: r"[1-3]?\d",
                 unit: DateUnit::Days,
                 max_len: 2,
             }),
-            "a" => Some(DateField {
+            "a" => Some(Self {
                 regex: r"Sun|Mon|Tue|Wed|Thu|Fri|Sat",
                 unit: DateUnit::Days,
                 max_len: 3,
             }),
-            "A" => Some(DateField {
+            "A" => Some(Self {
                 regex: r"Sunday|Monday|Tuesday|Wednesday|Thursday|Friday|Saturday",
                 unit: DateUnit::Days,
                 max_len: 9,
             }),
-            "b" | "h" => Some(DateField {
+            "b" | "h" => Some(Self {
                 regex: r"Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec",
                 unit: DateUnit::Months,
                 max_len: 3,
             }),
-            "B" => Some(DateField {
+            "B" => Some(Self {
                 regex: r"January|February|March|April|May|June|July|August|September|October|November|December",
                 unit: DateUnit::Months,
                 max_len: 9,
             }),
-            "H" => Some(DateField {
+            "H" => Some(Self {
                 regex: r"[0-2]\d",
                 unit: DateUnit::Hours,
                 max_len: 2,
             }),
-            "M" => Some(DateField {
+            "M" => Some(Self {
                 regex: r"[0-5]\d",
                 unit: DateUnit::Minutes,
                 max_len: 2,
             }),
-            "S" => Some(DateField {
+            "S" => Some(Self {
                 regex: r"[0-5]\d",
                 unit: DateUnit::Seconds,
                 max_len: 2,
             }),
-            "I" => Some(DateField {
+            "I" => Some(Self {
                 regex: r"[0-1]\d",
                 unit: DateUnit::Hours,
                 max_len: 2,
             }),
-            "-I" => Some(DateField {
+            "-I" => Some(Self {
                 regex: r"1?\d",
                 unit: DateUnit::Hours,
                 max_len: 2,
             }),
-            "P" => Some(DateField {
+            "P" => Some(Self {
                 regex: r"am|pm",
                 unit: DateUnit::AmPm,
                 max_len: 2,
             }),
-            "p" => Some(DateField {
+            "p" => Some(Self {
                 regex: r"AM|PM",
                 unit: DateUnit::AmPm,
                 max_len: 2,
@@ -451,7 +451,7 @@ mod test {
                     .unwrap()
                     .increment(amount)
                     .1,
-                expected.into()
+                Tendril::from(expected)
             );
         }
     }

helix-core/src/increment/number.rs

@@ -371,7 +371,7 @@ mod test {
                     .unwrap()
                     .increment(amount)
                     .1,
-                expected.into()
+                Tendril::from(expected)
             );
         }
     }
@@ -398,7 +398,7 @@ mod test {
                     .unwrap()
                     .increment(amount)
                     .1,
-                expected.into()
+                Tendril::from(expected)
             );
         }
     }
@@ -426,7 +426,7 @@ mod test {
                     .unwrap()
                     .increment(amount)
                     .1,
-                expected.into()
+                Tendril::from(expected)
             );
         }
     }
@@ -472,7 +472,7 @@ mod test {
                     .unwrap()
                     .increment(amount)
                     .1,
-                expected.into()
+                Tendril::from(expected)
             );
         }
     }
@@ -500,7 +500,7 @@ mod test {
                     .unwrap()
                     .increment(amount)
                     .1,
-                expected.into()
+                Tendril::from(expected)
             );
         }
     }

helix-core/src/indent.rs

@@ -1,6 +1,10 @@
+use std::collections::HashMap;
+
+use tree_sitter::{Query, QueryCursor, QueryPredicateArg};
+
 use crate::{
     chars::{char_is_line_ending, char_is_whitespace},
-    syntax::{IndentQuery, LanguageConfiguration, Syntax},
+    syntax::{LanguageConfiguration, RopeProvider, Syntax},
     tree_sitter::Node,
     Rope, RopeSlice,
 };
@@ -186,106 +190,405 @@ pub fn indent_level_for_line(line: RopeSlice, tab_width: usize) -> usize {
     len / tab_width
 }
 
-/// Find the highest syntax node at position.
+/// Computes for node and all ancestors whether they are the first node on their line.
-/// This is to identify the column where this node (e.g., an HTML closing tag) ends.
+/// The first entry in the return value represents the root node, the last one the node itself
-fn get_highest_syntax_node_at_bytepos(syntax: &Syntax, pos: usize) -> Option<Node> {
+fn get_first_in_line(mut node: Node, byte_pos: usize, new_line: bool) -> Vec<bool> {
-    let tree = syntax.tree();
+    let mut first_in_line = Vec::new();
-
-    // named_descendant
-    let mut node = match tree.root_node().descendant_for_byte_range(pos, pos) {
-        Some(node) => node,
-        None => return None,
-    };
-
-    while let Some(parent) = node.parent() {
-        if parent.start_byte() == node.start_byte() {
-            node = parent
-        } else {
-            break;
-        }
-    }
-
-    Some(node)
-}
-
-/// Calculate the indentation at a given treesitter node.
-/// If newline is false, then any "indent" nodes on the line are ignored ("outdent" still applies).
-/// This is because the indentation is only increased starting at the second line of the node.
-fn calculate_indentation(
-    query: &IndentQuery,
-    node: Option<Node>,
-    line: usize,
-    newline: bool,
-) -> usize {
-    let mut increment: isize = 0;
-
-    let mut node = match node {
-        Some(node) => node,
-        None => return 0,
-    };
-
-    let mut current_line = line;
-    let mut consider_indent = newline;
-    let mut increment_from_line: isize = 0;
-
     loop {
-        let node_kind = node.kind();
+        if let Some(prev) = node.prev_sibling() {
-        let start = node.start_position().row;
+            // If we insert a new line, the first node at/after the cursor is considered to be the first in its line
-        if current_line != start {
+            let first = prev.end_position().row != node.start_position().row
-            // Indent/dedent by at most one per line:
+                || (new_line && node.start_byte() >= byte_pos && prev.start_byte() < byte_pos);
-            // .map(|a| {       <-- ({ is two scopes
+            first_in_line.push(Some(first));
-            //     let len = 1; <-- indents one level
+        } else {
-            // })               <-- }) is two scopes
+            // Nodes that have no previous siblings are first in their line if and only if their parent is
-            if consider_indent || increment_from_line < 0 {
+            // (which we don't know yet)
-                increment += increment_from_line.signum();
+            first_in_line.push(None);
-            }
-            increment_from_line = 0;
-            current_line = start;
-            consider_indent = true;
         }
-
-        if query.outdent.contains(node_kind) {
-            increment_from_line -= 1;
-        }
-        if query.indent.contains(node_kind) {
-            increment_from_line += 1;
-        }
-
         if let Some(parent) = node.parent() {
             node = parent;
         } else {
             break;
         }
     }
-    if consider_indent || increment_from_line < 0 {
+
-        increment += increment_from_line.signum();
+    let mut result = Vec::with_capacity(first_in_line.len());
+    let mut parent_is_first = true; // The root node is by definition the first node in its line
+    for first in first_in_line.into_iter().rev() {
+        if let Some(first) = first {
+            result.push(first);
+            parent_is_first = first;
+        } else {
+            result.push(parent_is_first);
+        }
     }
-    increment.max(0) as usize
+    result
 }
 
-// TODO: two usecases: if we are triggering this for a new, blank line:
+/// The total indent for some line of code.
-// - it should return 0 when mass indenting stuff
+/// This is usually constructed in one of 2 ways:
-// - it should look up the wrapper node and count it too when we press o/O
+/// - Successively add indent captures to get the (added) indent from a single line
-pub fn suggested_indent_for_pos(
+/// - Successively add the indent results for each line
+#[derive(Default)]
+struct Indentation {
+    /// The total indent (the number of indent levels) is defined as max(0, indent-outdent).
+    /// The string that this results in depends on the indent style (spaces or tabs, etc.)
+    indent: usize,
+    outdent: usize,
+}
+impl Indentation {
+    /// Add some other [IndentResult] to this.
+    /// The added indent should be the total added indent from one line
+    fn add_line(&mut self, added: &Indentation) {
+        if added.indent > 0 && added.outdent == 0 {
+            self.indent += 1;
+        } else if added.outdent > 0 && added.indent == 0 {
+            self.outdent += 1;
+        }
+    }
+    /// Add an indent capture to this indent.
+    /// All the captures that are added in this way should be on the same line.
+    fn add_capture(&mut self, added: IndentCaptureType) {
+        match added {
+            IndentCaptureType::Indent => {
+                self.indent = 1;
+            }
+            IndentCaptureType::Outdent => {
+                self.outdent = 1;
+            }
+        }
+    }
+    fn as_string(&self, indent_style: &IndentStyle) -> String {
+        let indent_level = if self.indent >= self.outdent {
+            self.indent - self.outdent
+        } else {
+            log::warn!("Encountered more outdent than indent nodes while calculating indentation: {} outdent, {} indent", self.outdent, self.indent);
+            0
+        };
+        indent_style.as_str().repeat(indent_level)
+    }
+}
+
+/// An indent definition which corresponds to a capture from the indent query
+struct IndentCapture {
+    capture_type: IndentCaptureType,
+    scope: IndentScope,
+}
+#[derive(Clone, Copy)]
+enum IndentCaptureType {
+    Indent,
+    Outdent,
+}
+impl IndentCaptureType {
+    fn default_scope(&self) -> IndentScope {
+        match self {
+            IndentCaptureType::Indent => IndentScope::Tail,
+            IndentCaptureType::Outdent => IndentScope::All,
+        }
+    }
+}
+/// This defines which part of a node an [IndentCapture] applies to.
+/// Each [IndentCaptureType] has a default scope, but the scope can be changed
+/// with `#set!` property declarations.
+#[derive(Clone, Copy)]
+enum IndentScope {
+    /// The indent applies to the whole node
+    All,
+    /// The indent applies to everything except for the first line of the node
+    Tail,
+}
+
+/// Execute the indent query.
+/// Returns for each node (identified by its id) a list of indent captures for that node.
+fn query_indents(
+    query: &Query,
+    syntax: &Syntax,
+    cursor: &mut QueryCursor,
+    text: RopeSlice,
+    range: std::ops::Range<usize>,
+    // Position of the (optional) newly inserted line break.
+    // Given as (line, byte_pos)
+    new_line_break: Option<(usize, usize)>,
+) -> HashMap<usize, Vec<IndentCapture>> {
+    let mut indent_captures: HashMap<usize, Vec<IndentCapture>> = HashMap::new();
+    cursor.set_byte_range(range);
+    // Iterate over all captures from the query
+    for m in cursor.matches(query, syntax.tree().root_node(), RopeProvider(text)) {
+        // Skip matches where not all custom predicates are fulfilled
+        if !query.general_predicates(m.pattern_index).iter().all(|pred| {
+            match pred.operator.as_ref() {
+                "not-kind-eq?" => match (pred.args.get(0), pred.args.get(1)) {
+                    (
+                        Some(QueryPredicateArg::Capture(capture_idx)),
+                        Some(QueryPredicateArg::String(kind)),
+                    ) => {
+                        let node = m.nodes_for_capture_index(*capture_idx).next();
+                        match node {
+                            Some(node) => node.kind()!=kind.as_ref(),
+                            _ => true,
+                        }
+                    }
+                    _ => {
+                        panic!("Invalid indent query: Arguments to \"not-kind-eq?\" must be a capture and a string");
+                    }
+                },
+                "same-line?" | "not-same-line?" => {
+                    match (pred.args.get(0), pred.args.get(1)) {
+                        (
+                            Some(QueryPredicateArg::Capture(capt1)),
+                            Some(QueryPredicateArg::Capture(capt2))
+                        ) => {
+                            let get_line_num = |node: Node| {
+                                let mut node_line = node.start_position().row;
+                                // Adjust for the new line that will be inserted
+                                if let Some((line, byte)) = new_line_break {
+                                    if node_line==line && node.start_byte()>=byte {
+                                        node_line += 1;
+                                    }
+                                }
+                                node_line
+                            };
+                            let n1 = m.nodes_for_capture_index(*capt1).next();
+                            let n2 = m.nodes_for_capture_index(*capt2).next();
+                            match (n1, n2) {
+                                (Some(n1), Some(n2)) => {
+                                    let same_line = get_line_num(n1)==get_line_num(n2);
+                                    same_line==(pred.operator.as_ref()=="same-line?")
+                                }
+                                _ => true,
+                            }
+                        }
+                        _ => {
+                            panic!("Invalid indent query: Arguments to \"{}\" must be 2 captures", pred.operator);
+                        }
+                    }
+                }
+                _ => {
+                    panic!(
+                        "Invalid indent query: Unknown predicate (\"{}\")",
+                        pred.operator
+                    );
+                }
+            }
+        }) {
+            continue;
+        }
+        for capture in m.captures {
+            let capture_type = query.capture_names()[capture.index as usize].as_str();
+            let capture_type = match capture_type {
+                "indent" => IndentCaptureType::Indent,
+                "outdent" => IndentCaptureType::Outdent,
+                _ => {
+                    // Ignore any unknown captures (these may be needed for predicates such as #match?)
+                    continue;
+                }
+            };
+            let scope = capture_type.default_scope();
+            let mut indent_capture = IndentCapture {
+                capture_type,
+                scope,
+            };
+            // Apply additional settings for this capture
+            for property in query.property_settings(m.pattern_index) {
+                match property.key.as_ref() {
+                    "scope" => {
+                        indent_capture.scope = match property.value.as_deref() {
+                            Some("all") => IndentScope::All,
+                            Some("tail") => IndentScope::Tail,
+                            Some(s) => {
+                                panic!("Invalid indent query: Unknown value for \"scope\" property (\"{}\")", s);
+                            }
+                            None => {
+                                panic!(
+                                    "Invalid indent query: Missing value for \"scope\" property"
+                                );
+                            }
+                        }
+                    }
+                    _ => {
+                        panic!(
+                            "Invalid indent query: Unknown property \"{}\"",
+                            property.key
+                        );
+                    }
+                }
+            }
+            indent_captures
+                .entry(capture.node.id())
+                // Most entries only need to contain a single IndentCapture
+                .or_insert_with(|| Vec::with_capacity(1))
+                .push(indent_capture);
+        }
+    }
+    indent_captures
+}
+
+/// Use the syntax tree to determine the indentation for a given position.
+/// This can be used in 2 ways:
+///
+/// - To get the correct indentation for an existing line (new_line=false), not necessarily equal to the current indentation.
+///   - In this case, pos should be inside the first tree-sitter node on that line.
+///     In most cases, this can just be the first non-whitespace on that line.
+///   - To get the indentation for a new line (new_line=true). This behaves like the first usecase if the part of the current line
+///     after pos were moved to a new line.
+///
+/// The indentation is determined by traversing all the tree-sitter nodes containing the position.
+/// Each of these nodes produces some [AddedIndent] for:
+///
+/// - The line of the (beginning of the) node. This is defined by the scope `all` if this is the first node on its line.
+/// - The line after the node. This is defined by:
+///   - The scope `tail`.
+///   - The scope `all` if this node is not the first node on its line.
+/// Intuitively, `all` applies to everything contained in this node while `tail` applies to everything except for the first line of the node.
+/// The indents from different nodes for the same line are then combined.
+/// The [IndentResult] is simply the sum of the [AddedIndent] for all lines.
+///
+/// Specifying which line exactly an [AddedIndent] applies to is important because indents on the same line combine differently than indents on different lines:
+/// ```ignore
+/// some_function(|| {
+///     // Both the function parameters as well as the contained block should be indented.
+///     // Because they are on the same line, this only yields one indent level
+/// });
+/// ```
+///
+/// ```ignore
+/// some_function(
+///     parm1,
+///     || {
+///         // Here we get 2 indent levels because the 'parameters' and the 'block' node begin on different lines
+///     },
+/// );
+/// ```
+pub fn treesitter_indent_for_pos(
+    query: &Query,
+    syntax: &Syntax,
+    indent_style: &IndentStyle,
+    text: RopeSlice,
+    line: usize,
+    pos: usize,
+    new_line: bool,
+) -> Option<String> {
+    let byte_pos = text.char_to_byte(pos);
+    let mut node = syntax
+        .tree()
+        .root_node()
+        .descendant_for_byte_range(byte_pos, byte_pos)?;
+    let mut first_in_line = get_first_in_line(node, byte_pos, new_line);
+    let new_line_break = if new_line {
+        Some((line, byte_pos))
+    } else {
+        None
+    };
+    let query_result = crate::syntax::PARSER.with(|ts_parser| {
+        let mut ts_parser = ts_parser.borrow_mut();
+        let mut cursor = ts_parser.cursors.pop().unwrap_or_else(QueryCursor::new);
+        let query_result = query_indents(
+            query,
+            syntax,
+            &mut cursor,
+            text,
+            byte_pos..byte_pos + 1,
+            new_line_break,
+        );
+        ts_parser.cursors.push(cursor);
+        query_result
+    });
+
+    let mut result = Indentation::default();
+    // We always keep track of all the indent changes on one line, in order to only indent once
+    // even if there are multiple "indent" nodes on the same line
+    let mut indent_for_line = Indentation::default();
+    let mut indent_for_line_below = Indentation::default();
+    loop {
+        // This can safely be unwrapped because `first_in_line` contains
+        // one entry for each ancestor of the node (which is what we iterate over)
+        let is_first = *first_in_line.last().unwrap();
+        // Apply all indent definitions for this node
+        if let Some(definitions) = query_result.get(&node.id()) {
+            for definition in definitions {
+                match definition.scope {
+                    IndentScope::All => {
+                        if is_first {
+                            indent_for_line.add_capture(definition.capture_type);
+                        } else {
+                            indent_for_line_below.add_capture(definition.capture_type);
+                        }
+                    }
+                    IndentScope::Tail => {
+                        indent_for_line_below.add_capture(definition.capture_type);
+                    }
+                }
+            }
+        }
+
+        if let Some(parent) = node.parent() {
+            let mut node_line = node.start_position().row;
+            let mut parent_line = parent.start_position().row;
+            if node_line == line && new_line {
+                // Also consider the line that will be inserted
+                if node.start_byte() >= byte_pos {
+                    node_line += 1;
+                }
+                if parent.start_byte() >= byte_pos {
+                    parent_line += 1;
+                }
+            };
+            if node_line != parent_line {
+                if node_line < line + (new_line as usize) {
+                    // Don't add indent for the line below the line of the query
+                    result.add_line(&indent_for_line_below);
+                }
+                if node_line == parent_line + 1 {
+                    indent_for_line_below = indent_for_line;
+                } else {
+                    result.add_line(&indent_for_line);
+                    indent_for_line_below = Indentation::default();
+                }
+                indent_for_line = Indentation::default();
+            }
+
+            node = parent;
+            first_in_line.pop();
+        } else {
+            result.add_line(&indent_for_line_below);
+            result.add_line(&indent_for_line);
+            break;
+        }
+    }
+    Some(result.as_string(indent_style))
+}
+
+/// Returns the indentation for a new line.
+/// This is done either using treesitter, or if that's not available by copying the indentation from the current line
+#[allow(clippy::too_many_arguments)]
+pub fn indent_for_newline(
     language_config: Option<&LanguageConfiguration>,
     syntax: Option<&Syntax>,
+    indent_style: &IndentStyle,
+    tab_width: usize,
     text: RopeSlice,
-    pos: usize,
+    line_before: usize,
-    line: usize,
+    line_before_end_pos: usize,
-    new_line: bool,
+    current_line: usize,
-) -> Option<usize> {
+) -> String {
     if let (Some(query), Some(syntax)) = (
         language_config.and_then(|config| config.indent_query()),
         syntax,
     ) {
-        let byte_start = text.char_to_byte(pos);
+        if let Some(indent) = treesitter_indent_for_pos(
-        let node = get_highest_syntax_node_at_bytepos(syntax, byte_start);
+            query,
-        // TODO: special case for comments
+            syntax,
-        // TODO: if preserve_leading_whitespace
+            indent_style,
-        Some(calculate_indentation(query, node, line, new_line))
+            text,
-    } else {
+            line_before,
-        None
+            line_before_end_pos,
+            true,
+        ) {
+            return indent;
+        };
     }
+    let indent_level = indent_level_for_line(text.line(current_line), tab_width);
+    indent_style.as_str().repeat(indent_level)
 }
 
 pub fn get_scopes(syntax: Option<&Syntax>, text: RopeSlice, pos: usize) -> Vec<&'static str> {
@@ -329,153 +632,4 @@ mod test {
         let line = Rope::from("\t    \tfn new"); // 1 tab, 4 spaces, tab
         assert_eq!(indent_level_for_line(line.slice(..), tab_width), 3);
     }
-
-    #[test]
-    fn test_suggested_indent_for_line() {
-        let doc = Rope::from(
-            "
-use std::{
-    io::{self, stdout, Stdout, Write},
-    path::PathBuf,
-    sync::Arc,
-    time::Duration,
-}
-mod test {
-    fn hello_world() {
-        1 + 1;
-
-        let does_indentation_work = 1;
-
-        let test_function = function_with_param(this_param,
-            that_param
-        );
-
-        let test_function = function_with_param(
-            this_param,
-            that_param
-        );
-
-        let test_function = function_with_proper_indent(param1,
-            param2,
-        );
-
-        let selection = Selection::new(
-            changes
-                .clone()
-                .map(|(start, end, text): (usize, usize, Option<Tendril>)| {
-                    let len = text.map(|text| text.len()).unwrap() - 1; // minus newline
-                    let pos = start + len;
-                    Range::new(pos, pos)
-                })
-                .collect(),
-            0,
-        );
-
-        return;
-    }
-}
-
-impl<A, D> MyTrait<A, D> for YourType
-where
-    A: TraitB + TraitC,
-    D: TraitE + TraitF,
-{
-
-}
-#[test]
-//
-match test {
-    Some(a) => 1,
-    None => {
-        unimplemented!()
-    }
-}
-std::panic::set_hook(Box::new(move |info| {
-    hook(info);
-}));
-
-{ { {
-    1
-}}}
-
-pub fn change<I>(document: &Document, changes: I) -> Self
-where
-    I: IntoIterator<Item = Change> + ExactSizeIterator,
-{
-    [
-        1,
-        2,
-        3,
-    ];
-    (
-        1,
-        2
-    );
-    true
-}
-",
-        );
-
-        let doc = Rope::from(doc);
-        use crate::diagnostic::Severity;
-        use crate::syntax::{
-            Configuration, IndentationConfiguration, LanguageConfiguration, Loader,
-        };
-        use once_cell::sync::OnceCell;
-        let loader = Loader::new(Configuration {
-            language: vec![LanguageConfiguration {
-                scope: "source.rust".to_string(),
-                file_types: vec!["rs".to_string()],
-                shebangs: vec![],
-                language_id: "Rust".to_string(),
-                highlight_config: OnceCell::new(),
-                config: None,
-                //
-                injection_regex: None,
-                roots: vec![],
-                comment_token: None,
-                auto_format: false,
-                diagnostic_severity: Severity::Warning,
-                language_server: None,
-                indent: Some(IndentationConfiguration {
-                    tab_width: 4,
-                    unit: String::from("    "),
-                }),
-                indent_query: OnceCell::new(),
-                textobject_query: OnceCell::new(),
-            }],
-        });
-
-        // set runtime path so we can find the queries
-        let mut runtime = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
-        runtime.push("../runtime");
-        std::env::set_var("HELIX_RUNTIME", runtime.to_str().unwrap());
-
-        let language_config = loader.language_config_for_scope("source.rust").unwrap();
-        let highlight_config = language_config.highlight_config(&[]).unwrap();
-        let syntax = Syntax::new(&doc, highlight_config.clone());
-        let text = doc.slice(..);
-        let tab_width = 4;
-
-        for i in 0..doc.len_lines() {
-            let line = text.line(i);
-            if let Some(pos) = crate::find_first_non_whitespace_char(line) {
-                let indent = indent_level_for_line(line, tab_width);
-                assert_eq!(
-                    suggested_indent_for_pos(
-                        Some(&language_config),
-                        Some(&syntax),
-                        text,
-                        text.line_to_char(i) + pos,
-                        i,
-                        false
-                    ),
-                    Some(indent),
-                    "line {}: \"{}\"",
-                    i,
-                    line
-                );
-            }
-        }
-    }
 }

helix-core/src/lib.rs

@@ -3,6 +3,7 @@ pub use encoding_rs as encoding;
 pub mod auto_pairs;
 pub mod chars;
 pub mod comment;
+pub mod config;
 pub mod diagnostic;
 pub mod diff;
 pub mod graphemes;
@@ -23,6 +24,7 @@ pub mod shellwords;
 mod state;
 pub mod surround;
 pub mod syntax;
+pub mod test;
 pub mod textobject;
 mod transaction;
 
@@ -32,9 +34,6 @@ pub mod unicode {
     pub use unicode_width as width;
 }
 
-static RUNTIME_DIR: once_cell::sync::Lazy<std::path::PathBuf> =
-    once_cell::sync::Lazy::new(runtime_dir);
-
 pub fn find_first_non_whitespace_char(line: RopeSlice) -> Option<usize> {
     line.chars().position(|ch| !ch.is_whitespace())
 }
@@ -47,172 +46,17 @@ pub fn find_first_non_whitespace_char(line: RopeSlice) -> Option<usize> {
 /// * Top-most folder containing a root marker if not git repository detected
 /// * Current working directory as fallback
 pub fn find_root(root: Option<&str>, root_markers: &[String]) -> Option<std::path::PathBuf> {
-    let current_dir = std::env::current_dir().expect("unable to determine current directory");
+    helix_loader::find_root_impl(root, root_markers)
-
+        .first()
-    let root = match root {
+        .cloned()
-        Some(root) => {
-            let root = std::path::Path::new(root);
-            if root.is_absolute() {
-                root.to_path_buf()
-            } else {
-                current_dir.join(root)
-            }
-        }
-        None => current_dir.clone(),
-    };
-
-    let mut top_marker = None;
-    for ancestor in root.ancestors() {
-        for marker in root_markers {
-            if ancestor.join(marker).exists() {
-                top_marker = Some(ancestor);
-                break;
-            }
-        }
-        // don't go higher than repo
-        if ancestor.join(".git").is_dir() {
-            // Use workspace if detected from marker
-            return Some(top_marker.unwrap_or(ancestor).to_path_buf());
-        }
-    }
-
-    // In absence of git repo, use workspace if detected
-    if top_marker.is_some() {
-        top_marker.map(|a| a.to_path_buf())
-    } else {
-        Some(current_dir)
-    }
 }
 
-pub fn runtime_dir() -> std::path::PathBuf {
+pub use ropey::{str_utils, Rope, RopeBuilder, RopeSlice};
-    if let Ok(dir) = std::env::var("HELIX_RUNTIME") {
-        return dir.into();
-    }
 
-    const RT_DIR: &str = "runtime";
+// pub use tendril::StrTendril as Tendril;
-    let conf_dir = config_dir().join(RT_DIR);
+pub use smartstring::SmartString;
-    if conf_dir.exists() {
-        return conf_dir;
-    }
 
-    if let Ok(dir) = std::env::var("CARGO_MANIFEST_DIR") {
+pub type Tendril = SmartString<smartstring::LazyCompact>;
-        // this is the directory of the crate being run by cargo, we need the workspace path so we take the parent
-        return std::path::PathBuf::from(dir).parent().unwrap().join(RT_DIR);
-    }
-
-    // fallback to location of the executable being run
-    std::env::current_exe()
-        .ok()
-        .and_then(|path| path.parent().map(|path| path.to_path_buf().join(RT_DIR)))
-        .unwrap()
-}
-
-pub fn config_dir() -> std::path::PathBuf {
-    // TODO: allow env var override
-    let strategy = choose_base_strategy().expect("Unable to find the config directory!");
-    let mut path = strategy.config_dir();
-    path.push("helix");
-    path
-}
-
-pub fn cache_dir() -> std::path::PathBuf {
-    // TODO: allow env var override
-    let strategy = choose_base_strategy().expect("Unable to find the config directory!");
-    let mut path = strategy.cache_dir();
-    path.push("helix");
-    path
-}
-
-// right overrides left
-pub fn merge_toml_values(left: toml::Value, right: toml::Value) -> toml::Value {
-    use toml::Value;
-
-    fn get_name(v: &Value) -> Option<&str> {
-        v.get("name").and_then(Value::as_str)
-    }
-
-    match (left, right) {
-        (Value::Array(mut left_items), Value::Array(right_items)) => {
-            left_items.reserve(right_items.len());
-            for rvalue in right_items {
-                let lvalue = get_name(&rvalue)
-                    .and_then(|rname| left_items.iter().position(|v| get_name(v) == Some(rname)))
-                    .map(|lpos| left_items.remove(lpos));
-                let mvalue = match lvalue {
-                    Some(lvalue) => merge_toml_values(lvalue, rvalue),
-                    None => rvalue,
-                };
-                left_items.push(mvalue);
-            }
-            Value::Array(left_items)
-        }
-        (Value::Table(mut left_map), Value::Table(right_map)) => {
-            for (rname, rvalue) in right_map {
-                match left_map.remove(&rname) {
-                    Some(lvalue) => {
-                        let merged_value = merge_toml_values(lvalue, rvalue);
-                        left_map.insert(rname, merged_value);
-                    }
-                    None => {
-                        left_map.insert(rname, rvalue);
-                    }
-                }
-            }
-            Value::Table(left_map)
-        }
-        // Catch everything else we didn't handle, and use the right value
-        (_, value) => value,
-    }
-}
-
-#[cfg(test)]
-mod merge_toml_tests {
-    use super::merge_toml_values;
-
-    #[test]
-    fn language_tomls() {
-        use toml::Value;
-
-        const USER: &str = "
-        [[language]]
-        name = \"nix\"
-        test = \"bbb\"
-        indent = { tab-width = 4, unit = \"    \", test = \"aaa\" }
-        ";
-
-        let base: Value = toml::from_slice(include_bytes!("../../languages.toml"))
-            .expect("Couldn't parse built-in languages config");
-        let user: Value = toml::from_str(USER).unwrap();
-
-        let merged = merge_toml_values(base, user);
-        let languages = merged.get("language").unwrap().as_array().unwrap();
-        let nix = languages
-            .iter()
-            .find(|v| v.get("name").unwrap().as_str().unwrap() == "nix")
-            .unwrap();
-        let nix_indent = nix.get("indent").unwrap();
-
-        // We changed tab-width and unit in indent so check them if they are the new values
-        assert_eq!(
-            nix_indent.get("tab-width").unwrap().as_integer().unwrap(),
-            4
-        );
-        assert_eq!(nix_indent.get("unit").unwrap().as_str().unwrap(), "    ");
-        // We added a new keys, so check them
-        assert_eq!(nix.get("test").unwrap().as_str().unwrap(), "bbb");
-        assert_eq!(nix_indent.get("test").unwrap().as_str().unwrap(), "aaa");
-        // We didn't change comment-token so it should be same
-        assert_eq!(nix.get("comment-token").unwrap().as_str().unwrap(), "#");
-    }
-}
-
-pub use etcetera::home_dir;
-
-use etcetera::base_strategy::{choose_base_strategy, BaseStrategy};
-
-pub use ropey::{Rope, RopeBuilder, RopeSlice};
-
-pub use tendril::StrTendril as Tendril;
 
 #[doc(inline)]
 pub use {regex, tree_sitter};
@@ -220,7 +64,7 @@ pub use {regex, tree_sitter};
 pub use graphemes::RopeGraphemes;
 pub use position::{coords_at_pos, pos_at_coords, visual_coords_at_pos, Position};
 pub use selection::{Range, Selection};
-pub use smallvec::SmallVec;
+pub use smallvec::{smallvec, SmallVec};
 pub use syntax::Syntax;
 
 pub use diagnostic::Diagnostic;

helix-core/src/line_ending.rs

@@ -10,12 +10,18 @@ pub const DEFAULT_LINE_ENDING: LineEnding = LineEnding::LF;
 pub enum LineEnding {
     Crlf, // CarriageReturn followed by LineFeed
     LF,   // U+000A -- LineFeed
-    VT,   // U+000B -- VerticalTab
+    #[cfg(feature = "unicode-lines")]
-    FF,   // U+000C -- FormFeed
+    VT, // U+000B -- VerticalTab
-    CR,   // U+000D -- CarriageReturn
+    #[cfg(feature = "unicode-lines")]
-    Nel,  // U+0085 -- NextLine
+    FF, // U+000C -- FormFeed
-    LS,   // U+2028 -- Line Separator
+    #[cfg(feature = "unicode-lines")]
-    PS,   // U+2029 -- ParagraphSeparator
+    CR, // U+000D -- CarriageReturn
+    #[cfg(feature = "unicode-lines")]
+    Nel, // U+0085 -- NextLine
+    #[cfg(feature = "unicode-lines")]
+    LS, // U+2028 -- Line Separator
+    #[cfg(feature = "unicode-lines")]
+    PS, // U+2029 -- ParagraphSeparator
 }
 
 impl LineEnding {
@@ -32,11 +38,17 @@ impl LineEnding {
         match self {
             Self::Crlf => "\u{000D}\u{000A}",
             Self::LF => "\u{000A}",
+            #[cfg(feature = "unicode-lines")]
             Self::VT => "\u{000B}",
+            #[cfg(feature = "unicode-lines")]
             Self::FF => "\u{000C}",
+            #[cfg(feature = "unicode-lines")]
             Self::CR => "\u{000D}",
+            #[cfg(feature = "unicode-lines")]
             Self::Nel => "\u{0085}",
+            #[cfg(feature = "unicode-lines")]
             Self::LS => "\u{2028}",
+            #[cfg(feature = "unicode-lines")]
             Self::PS => "\u{2029}",
         }
     }
@@ -45,11 +57,17 @@ impl LineEnding {
     pub const fn from_char(ch: char) -> Option<LineEnding> {
         match ch {
             '\u{000A}' => Some(LineEnding::LF),
+            #[cfg(feature = "unicode-lines")]
             '\u{000B}' => Some(LineEnding::VT),
+            #[cfg(feature = "unicode-lines")]
             '\u{000C}' => Some(LineEnding::FF),
+            #[cfg(feature = "unicode-lines")]
             '\u{000D}' => Some(LineEnding::CR),
+            #[cfg(feature = "unicode-lines")]
             '\u{0085}' => Some(LineEnding::Nel),
+            #[cfg(feature = "unicode-lines")]
             '\u{2028}' => Some(LineEnding::LS),
+            #[cfg(feature = "unicode-lines")]
             '\u{2029}' => Some(LineEnding::PS),
             // Not a line ending
             _ => None,
@@ -65,11 +83,17 @@ impl LineEnding {
         match g {
             "\u{000D}\u{000A}" => Some(LineEnding::Crlf),
             "\u{000A}" => Some(LineEnding::LF),
+            #[cfg(feature = "unicode-lines")]
             "\u{000B}" => Some(LineEnding::VT),
+            #[cfg(feature = "unicode-lines")]
             "\u{000C}" => Some(LineEnding::FF),
+            #[cfg(feature = "unicode-lines")]
             "\u{000D}" => Some(LineEnding::CR),
+            #[cfg(feature = "unicode-lines")]
             "\u{0085}" => Some(LineEnding::Nel),
+            #[cfg(feature = "unicode-lines")]
             "\u{2028}" => Some(LineEnding::LS),
+            #[cfg(feature = "unicode-lines")]
             "\u{2029}" => Some(LineEnding::PS),
             // Not a line ending
             _ => None,
@@ -95,13 +119,20 @@ pub fn str_is_line_ending(s: &str) -> bool {
     LineEnding::from_str(s).is_some()
 }
 
+#[inline]
+pub fn rope_is_line_ending(r: RopeSlice) -> bool {
+    r.chunks().all(str_is_line_ending)
+}
+
 /// Attempts to detect what line ending the passed document uses.
 pub fn auto_detect_line_ending(doc: &Rope) -> Option<LineEnding> {
     // Return first matched line ending. Not all possible line endings
     // are being matched, as they might be special-use only
     for line in doc.lines().take(100) {
         match get_line_ending(&line) {
-            None | Some(LineEnding::VT) | Some(LineEnding::FF) | Some(LineEnding::PS) => {}
+            None => {}
+            #[cfg(feature = "unicode-lines")]
+            Some(LineEnding::VT) | Some(LineEnding::FF) | Some(LineEnding::PS) => {}
             ending => return ending,
         }
     }
@@ -128,6 +159,19 @@ pub fn get_line_ending(line: &RopeSlice) -> Option<LineEnding> {
     LineEnding::from_str(g2).or_else(|| LineEnding::from_str(g1))
 }
 
+#[cfg(not(feature = "unicode-lines"))]
+/// Returns the passed line's line ending, if any.
+pub fn get_line_ending_of_str(line: &str) -> Option<LineEnding> {
+    if line.ends_with("\u{000D}\u{000A}") {
+        Some(LineEnding::Crlf)
+    } else if line.ends_with('\u{000A}') {
+        Some(LineEnding::LF)
+    } else {
+        None
+    }
+}
+
+#[cfg(feature = "unicode-lines")]
 /// Returns the passed line's line ending, if any.
 pub fn get_line_ending_of_str(line: &str) -> Option<LineEnding> {
     if line.ends_with("\u{000D}\u{000A}") {
@@ -211,6 +255,7 @@ mod line_ending_tests {
 
     #[test]
     fn str_to_line_ending() {
+        #[cfg(feature = "unicode-lines")]
         assert_eq!(LineEnding::from_str("\r"), Some(LineEnding::CR));
         assert_eq!(LineEnding::from_str("\n"), Some(LineEnding::LF));
         assert_eq!(LineEnding::from_str("\r\n"), Some(LineEnding::Crlf));
@@ -220,6 +265,7 @@ mod line_ending_tests {
     #[test]
     fn rope_slice_to_line_ending() {
         let r = Rope::from_str("hello\r\n");
+        #[cfg(feature = "unicode-lines")]
         assert_eq!(
             LineEnding::from_rope_slice(&r.slice(5..6)),
             Some(LineEnding::CR)
@@ -238,6 +284,7 @@ mod line_ending_tests {
     #[test]
     fn get_line_ending_rope_slice() {
         let r = Rope::from_str("Hello\rworld\nhow\r\nare you?");
+        #[cfg(feature = "unicode-lines")]
         assert_eq!(get_line_ending(&r.slice(..6)), Some(LineEnding::CR));
         assert_eq!(get_line_ending(&r.slice(..12)), Some(LineEnding::LF));
         assert_eq!(get_line_ending(&r.slice(..17)), Some(LineEnding::Crlf));
@@ -247,19 +294,19 @@ mod line_ending_tests {
     #[test]
     fn get_line_ending_str() {
         let text = "Hello\rworld\nhow\r\nare you?";
+        #[cfg(feature = "unicode-lines")]
         assert_eq!(get_line_ending_of_str(&text[..6]), Some(LineEnding::CR));
         assert_eq!(get_line_ending_of_str(&text[..12]), Some(LineEnding::LF));
         assert_eq!(get_line_ending_of_str(&text[..17]), Some(LineEnding::Crlf));
-        assert_eq!(get_line_ending_of_str(&text[..]), None);
+        assert_eq!(get_line_ending_of_str(text), None);
     }
 
     #[test]
     fn line_end_char_index_rope_slice() {
         let r = Rope::from_str("Hello\rworld\nhow\r\nare you?");
         let s = &r.slice(..);
-        assert_eq!(line_end_char_index(s, 0), 5);
+        assert_eq!(line_end_char_index(s, 0), 11);
-        assert_eq!(line_end_char_index(s, 1), 11);
+        assert_eq!(line_end_char_index(s, 1), 15);
-        assert_eq!(line_end_char_index(s, 2), 15);
+        assert_eq!(line_end_char_index(s, 2), 25);
-        assert_eq!(line_end_char_index(s, 3), 25);
     }
 }

helix-core/src/movement.rs

@@ -1,6 +1,7 @@
 use std::iter;
 
 use ropey::iter::Chars;
+use tree_sitter::{Node, QueryCursor};
 
 use crate::{
     chars::{categorize_char, char_is_line_ending, CharCategory},
@@ -9,7 +10,11 @@ use crate::{
         next_grapheme_boundary, nth_next_grapheme_boundary, nth_prev_grapheme_boundary,
         prev_grapheme_boundary,
     },
-    pos_at_coords, Position, Range, RopeSlice,
+    line_ending::rope_is_line_ending,
+    pos_at_coords,
+    syntax::LanguageConfiguration,
+    textobject::TextObject,
+    Position, Range, RopeSlice,
 };
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
@@ -145,6 +150,88 @@ fn word_move(slice: RopeSlice, range: Range, count: usize, target: WordMotionTar
     })
 }
 
+pub fn move_prev_paragraph(
+    slice: RopeSlice,
+    range: Range,
+    count: usize,
+    behavior: Movement,
+) -> Range {
+    let mut line = range.cursor_line(slice);
+    let first_char = slice.line_to_char(line) == range.cursor(slice);
+    let prev_line_empty = rope_is_line_ending(slice.line(line.saturating_sub(1)));
+    let curr_line_empty = rope_is_line_ending(slice.line(line));
+    let prev_empty_to_line = prev_line_empty && !curr_line_empty;
+
+    // skip character before paragraph boundary
+    if prev_empty_to_line && !first_char {
+        line += 1;
+    }
+    let mut lines = slice.lines_at(line);
+    lines.reverse();
+    let mut lines = lines.map(rope_is_line_ending).peekable();
+    for _ in 0..count {
+        while lines.next_if(|&e| e).is_some() {
+            line -= 1;
+        }
+        while lines.next_if(|&e| !e).is_some() {
+            line -= 1;
+        }
+    }
+
+    let head = slice.line_to_char(line);
+    let anchor = if behavior == Movement::Move {
+        // exclude first character after paragraph boundary
+        if prev_empty_to_line && first_char {
+            range.cursor(slice)
+        } else {
+            range.head
+        }
+    } else {
+        range.put_cursor(slice, head, true).anchor
+    };
+    Range::new(anchor, head)
+}
+
+pub fn move_next_paragraph(
+    slice: RopeSlice,
+    range: Range,
+    count: usize,
+    behavior: Movement,
+) -> Range {
+    let mut line = range.cursor_line(slice);
+    let last_char =
+        prev_grapheme_boundary(slice, slice.line_to_char(line + 1)) == range.cursor(slice);
+    let curr_line_empty = rope_is_line_ending(slice.line(line));
+    let next_line_empty =
+        rope_is_line_ending(slice.line(slice.len_lines().saturating_sub(1).min(line + 1)));
+    let curr_empty_to_line = curr_line_empty && !next_line_empty;
+
+    // skip character after paragraph boundary
+    if curr_empty_to_line && last_char {
+        line += 1;
+    }
+    let mut lines = slice.lines_at(line).map(rope_is_line_ending).peekable();
+    for _ in 0..count {
+        while lines.next_if(|&e| !e).is_some() {
+            line += 1;
+        }
+        while lines.next_if(|&e| e).is_some() {
+            line += 1;
+        }
+    }
+    let head = slice.line_to_char(line);
+    let anchor = if behavior == Movement::Move {
+        if curr_empty_to_line && last_char {
+            range.head
+        } else {
+            range.cursor(slice)
+        }
+    } else {
+        range.put_cursor(slice, head, true).anchor
+    };
+    Range::new(anchor, head)
+}
+
 // ---- util ------------
 
 #[inline]
@@ -305,6 +392,56 @@ fn reached_target(target: WordMotionTarget, prev_ch: char, next_ch: char) -> boo
     }
 }
 
+pub fn goto_treesitter_object(
+    slice: RopeSlice,
+    range: Range,
+    object_name: &str,
+    dir: Direction,
+    slice_tree: Node,
+    lang_config: &LanguageConfiguration,
+    _count: usize,
+) -> Range {
+    let get_range = move || -> Option<Range> {
+        let byte_pos = slice.char_to_byte(range.cursor(slice));
+
+        let cap_name = |t: TextObject| format!("{}.{}", object_name, t);
+        let mut cursor = QueryCursor::new();
+        let nodes = lang_config.textobject_query()?.capture_nodes_any(
+            &[
+                &cap_name(TextObject::Movement),
+                &cap_name(TextObject::Around),
+                &cap_name(TextObject::Inside),
+            ],
+            slice_tree,
+            slice,
+            &mut cursor,
+        )?;
+
+        let node = match dir {
+            Direction::Forward => nodes
+                .filter(|n| n.start_byte() > byte_pos)
+                .min_by_key(|n| n.start_byte())?,
+            Direction::Backward => nodes
+                .filter(|n| n.start_byte() < byte_pos)
+                .max_by_key(|n| n.start_byte())?,
+        };
+
+        let len = slice.len_bytes();
+        let start_byte = node.start_byte();
+        let end_byte = node.end_byte();
+        if start_byte >= len || end_byte >= len {
+            return None;
+        }
+
+        let start_char = slice.byte_to_char(start_byte);
+        let end_char = slice.byte_to_char(end_byte);
+
+        // head of range should be at beginning
+        Some(Range::new(end_char, start_char))
+    };
+    get_range().unwrap_or(range)
+}
+
 #[cfg(test)]
 mod test {
     use ropey::Rope;
@@ -1125,4 +1262,172 @@ mod test {
             }
         }
     }
+
+    #[test]
+    fn test_behaviour_when_moving_to_prev_paragraph_single() {
+        let tests = [
+            ("#[|]#", "#[|]#"),
+            ("#[s|]#tart at\nfirst char\n", "#[|s]#tart at\nfirst char\n"),
+            ("start at\nlast char#[\n|]#", "#[|start at\nlast char\n]#"),
+            (
+                "goto\nfirst\n\n#[p|]#aragraph",
+                "#[|goto\nfirst\n\n]#paragraph",
+            ),
+            (
+                "goto\nfirst\n#[\n|]#paragraph",
+                "#[|goto\nfirst\n\n]#paragraph",
+            ),
+            (
+                "goto\nsecond\n\np#[a|]#ragraph",
+                "goto\nsecond\n\n#[|pa]#ragraph",
+            ),
+            (
+                "here\n\nhave\nmultiple\nparagraph\n\n\n\n\n#[|]#",
+                "here\n\n#[|have\nmultiple\nparagraph\n\n\n\n\n]#",
+            ),
+        ];
+
+        for (before, expected) in tests {
+            let (s, selection) = crate::test::print(before);
+            let text = Rope::from(s.as_str());
+            let selection =
+                selection.transform(|r| move_prev_paragraph(text.slice(..), r, 1, Movement::Move));
+            let actual = crate::test::plain(&s, selection);
+            assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
+        }
+    }
+
+    #[test]
+    fn test_behaviour_when_moving_to_prev_paragraph_double() {
+        let tests = [
+            (
+                "on#[e|]#\n\ntwo\n\nthree\n\n",
+                "#[|one]#\n\ntwo\n\nthree\n\n",
+            ),
+            (
+                "one\n\ntwo\n\nth#[r|]#ee\n\n",
+                "one\n\n#[|two\n\nthr]#ee\n\n",
+            ),
+        ];
+
+        for (before, expected) in tests {
+            let (s, selection) = crate::test::print(before);
+            let text = Rope::from(s.as_str());
+            let selection =
+                selection.transform(|r| move_prev_paragraph(text.slice(..), r, 2, Movement::Move));
+            let actual = crate::test::plain(&s, selection);
+            assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
+        }
+    }
+
+    #[test]
+    fn test_behaviour_when_moving_to_prev_paragraph_extend() {
+        let tests = [
+            (
+                "one\n\n#[|two\n\n]#three\n\n",
+                "#[|one\n\ntwo\n\n]#three\n\n",
+            ),
+            (
+                "#[|one\n\ntwo\n\n]#three\n\n",
+                "#[|one\n\ntwo\n\n]#three\n\n",
+            ),
+        ];
+
+        for (before, expected) in tests {
+            let (s, selection) = crate::test::print(before);
+            let text = Rope::from(s.as_str());
+            let selection = selection
+                .transform(|r| move_prev_paragraph(text.slice(..), r, 1, Movement::Extend));
+            let actual = crate::test::plain(&s, selection);
+            assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
+        }
+    }
+
+    #[test]
+    fn test_behaviour_when_moving_to_next_paragraph_single() {
+        let tests = [
+            ("#[|]#", "#[|]#"),
+            ("#[s|]#tart at\nfirst char\n", "#[start at\nfirst char\n|]#"),
+            ("start at\nlast char#[\n|]#", "start at\nlast char#[\n|]#"),
+            (
+                "a\nb\n\n#[g|]#oto\nthird\n\nparagraph",
+                "a\nb\n\n#[goto\nthird\n\n|]#paragraph",
+            ),
+            (
+                "a\nb\n#[\n|]#goto\nthird\n\nparagraph",
+                "a\nb\n\n#[goto\nthird\n\n|]#paragraph",
+            ),
+            (
+                "a\nb#[\n|]#\ngoto\nsecond\n\nparagraph",
+                "a\nb#[\n\n|]#goto\nsecond\n\nparagraph",
+            ),
+            (
+                "here\n\nhave\n#[m|]#ultiple\nparagraph\n\n\n\n\n",
+                "here\n\nhave\n#[multiple\nparagraph\n\n\n\n\n|]#",
+            ),
+            (
+                "#[t|]#ext\n\n\nafter two blank lines\n\nmore text\n",
+                "#[text\n\n\n|]#after two blank lines\n\nmore text\n",
+            ),
+            (
+                "#[text\n\n\n|]#after two blank lines\n\nmore text\n",
+                "text\n\n\n#[after two blank lines\n\n|]#more text\n",
+            ),
+        ];
+
+        for (before, expected) in tests {
+            let (s, selection) = crate::test::print(before);
+            let text = Rope::from(s.as_str());
+            let selection =
+                selection.transform(|r| move_next_paragraph(text.slice(..), r, 1, Movement::Move));
+            let actual = crate::test::plain(&s, selection);
+            assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
+        }
+    }
+
+    #[test]
+    fn test_behaviour_when_moving_to_next_paragraph_double() {
+        let tests = [
+            (
+                "one\n\ntwo\n\nth#[r|]#ee\n\n",
+                "one\n\ntwo\n\nth#[ree\n\n|]#",
+            ),
+            (
+                "on#[e|]#\n\ntwo\n\nthree\n\n",
+                "on#[e\n\ntwo\n\n|]#three\n\n",
+            ),
+        ];
+
+        for (before, expected) in tests {
+            let (s, selection) = crate::test::print(before);
+            let text = Rope::from(s.as_str());
+            let selection =
+                selection.transform(|r| move_next_paragraph(text.slice(..), r, 2, Movement::Move));
+            let actual = crate::test::plain(&s, selection);
+            assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
+        }
+    }
+
+    #[test]
+    fn test_behaviour_when_moving_to_next_paragraph_extend() {
+        let tests = [
+            (
+                "one\n\n#[two\n\n|]#three\n\n",
+                "one\n\n#[two\n\nthree\n\n|]#",
+            ),
+            (
+                "one\n\n#[two\n\nthree\n\n|]#",
+                "one\n\n#[two\n\nthree\n\n|]#",
+            ),
+        ];
+
+        for (before, expected) in tests {
+            let (s, selection) = crate::test::print(before);
+            let text = Rope::from(s.as_str());
+            let selection = selection
+                .transform(|r| move_next_paragraph(text.slice(..), r, 1, Movement::Extend));
+            let actual = crate::test::plain(&s, selection);
+            assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
+        }
+    }
 }

helix-core/src/object.rs

@@ -1,31 +1,72 @@
 use crate::{Range, RopeSlice, Selection, Syntax};
+use tree_sitter::Node;
 
-// TODO: to contract_selection we'd need to store the previous ranges before expand.
+pub fn expand_selection(syntax: &Syntax, text: RopeSlice, selection: Selection) -> Selection {
-// Maybe just contract to the first child node?
+    select_node_impl(syntax, text, selection, |descendant, from, to| {
-pub fn expand_selection(syntax: &Syntax, text: RopeSlice, selection: &Selection) -> Selection {
+        if descendant.start_byte() == from && descendant.end_byte() == to {
+            descendant.parent()
+        } else {
+            Some(descendant)
+        }
+    })
+}
+
+pub fn shrink_selection(syntax: &Syntax, text: RopeSlice, selection: Selection) -> Selection {
+    select_node_impl(syntax, text, selection, |descendant, _from, _to| {
+        descendant.child(0).or(Some(descendant))
+    })
+}
+
+pub fn select_sibling<F>(
+    syntax: &Syntax,
+    text: RopeSlice,
+    selection: Selection,
+    sibling_fn: &F,
+) -> Selection
+where
+    F: Fn(Node) -> Option<Node>,
+{
+    select_node_impl(syntax, text, selection, |descendant, _from, _to| {
+        find_sibling_recursive(descendant, sibling_fn)
+    })
+}
+
+fn find_sibling_recursive<F>(node: Node, sibling_fn: F) -> Option<Node>
+where
+    F: Fn(Node) -> Option<Node>,
+{
+    sibling_fn(node).or_else(|| {
+        node.parent()
+            .and_then(|node| find_sibling_recursive(node, sibling_fn))
+    })
+}
+
+fn select_node_impl<F>(
+    syntax: &Syntax,
+    text: RopeSlice,
+    selection: Selection,
+    select_fn: F,
+) -> Selection
+where
+    F: Fn(Node, usize, usize) -> Option<Node>,
+{
     let tree = syntax.tree();
 
-    selection.clone().transform(|range| {
+    selection.transform(|range| {
         let from = text.char_to_byte(range.from());
         let to = text.char_to_byte(range.to());
 
-        // find parent of a descendant that matches the range
+        let node = match tree
-        let parent = match tree
             .root_node()
             .descendant_for_byte_range(from, to)
-            .and_then(|node| {
+            .and_then(|node| select_fn(node, from, to))
-                if node.child_count() == 0 || (node.start_byte() == from && node.end_byte() == to) {
+        {
-                    node.parent()
+            Some(node) => node,
-                } else {
-                    Some(node)
-                }
-            }) {
-            Some(parent) => parent,
             None => return range,
         };
 
-        let from = text.byte_to_char(parent.start_byte());
+        let from = text.byte_to_char(node.start_byte());
-        let to = text.byte_to_char(parent.end_byte());
+        let to = text.byte_to_char(node.end_byte());
 
         if range.head < range.anchor {
             Range::new(to, from)

helix-core/src/path.rs

@@ -1,9 +1,10 @@
+use etcetera::home_dir;
 use std::path::{Component, Path, PathBuf};
 
 /// Replaces users home directory from `path` with tilde `~` if the directory
 /// is available, otherwise returns the path unchanged.
 pub fn fold_home_dir(path: &Path) -> PathBuf {
-    if let Ok(home) = super::home_dir() {
+    if let Ok(home) = home_dir() {
         if path.starts_with(&home) {
             // it's ok to unwrap, the path starts with home dir
             return PathBuf::from("~").join(path.strip_prefix(&home).unwrap());
@@ -13,14 +14,14 @@ pub fn fold_home_dir(path: &Path) -> PathBuf {
     path.to_path_buf()
 }
 
-/// Expands tilde `~` into users home directory if avilable, otherwise returns the path
+/// Expands tilde `~` into users home directory if available, otherwise returns the path
 /// unchanged. The tilde will only be expanded when present as the first component of the path
 /// and only slash follows it.
 pub fn expand_tilde(path: &Path) -> PathBuf {
     let mut components = path.components().peekable();
     if let Some(Component::Normal(c)) = components.peek() {
         if c == &"~" {
-            if let Ok(home) = super::home_dir() {
+            if let Ok(home) = home_dir() {
                 // it's ok to unwrap, the path starts with `~`
                 return home.join(path.strip_prefix("~").unwrap());
             }

helix-core/src/position.rs

@@ -1,8 +1,9 @@
+use std::borrow::Cow;
+
 use crate::{
     chars::char_is_line_ending,
-    graphemes::{ensure_grapheme_boundary_prev, RopeGraphemes},
+    graphemes::{ensure_grapheme_boundary_prev, grapheme_width, RopeGraphemes},
     line_ending::line_end_char_index,
-    unicode::width::UnicodeWidthChar,
     RopeSlice,
 };
 
@@ -77,14 +78,17 @@ pub fn visual_coords_at_pos(text: RopeSlice, pos: usize, tab_width: usize) -> Po
 
     let line_start = text.line_to_char(line);
     let pos = ensure_grapheme_boundary_prev(text, pos);
-    let col = text
+
-        .slice(line_start..pos)
+    let mut col = 0;
-        .chars()
+
-        .flat_map(|c| match c {
+    for grapheme in RopeGraphemes::new(text.slice(line_start..pos)) {
-            '\t' => Some(tab_width),
+        if grapheme == "\t" {
-            c => UnicodeWidthChar::width(c),
+            col += tab_width - (col % tab_width);
-        })
+        } else {
-        .sum();
+            let grapheme = Cow::from(grapheme);
+            col += grapheme_width(&grapheme);
+        }
+    }
 
     Position::new(line, col)
 }
@@ -109,7 +113,10 @@ pub fn visual_coords_at_pos(text: RopeSlice, pos: usize, tab_width: usize) -> Po
 /// TODO: this should be changed to work in terms of visual row/column, not
 /// graphemes.
 pub fn pos_at_coords(text: RopeSlice, coords: Position, limit_before_line_ending: bool) -> usize {
-    let Position { row, col } = coords;
+    let Position { mut row, col } = coords;
+    if limit_before_line_ending {
+        row = row.min(text.len_lines() - 1);
+    };
     let line_start = text.line_to_char(row);
     let line_end = if limit_before_line_ending {
         line_end_char_index(&text, row)
@@ -290,5 +297,12 @@ mod test {
         assert_eq!(pos_at_coords(slice, (0, 0).into(), false), 0);
         assert_eq!(pos_at_coords(slice, (0, 1).into(), false), 1);
         assert_eq!(pos_at_coords(slice, (0, 2).into(), false), 2);
+
+        // Test out of bounds.
+        let text = Rope::new();
+        let slice = text.slice(..);
+        assert_eq!(pos_at_coords(slice, (10, 0).into(), true), 0);
+        assert_eq!(pos_at_coords(slice, (0, 10).into(), true), 0);
+        assert_eq!(pos_at_coords(slice, (10, 10).into(), true), 0);
     }
 }

helix-core/src/register.rs

@@ -68,4 +68,8 @@ impl Registers {
     pub fn read(&self, name: char) -> Option<&[String]> {
         self.get(name).map(|reg| reg.read())
     }
+
+    pub fn inner(&self) -> &HashMap<char, Register> {
+        &self.inner
+    }
 }

helix-core/src/search.rs

@@ -1,6 +1,28 @@
 use crate::RopeSlice;
 
-pub fn find_nth_next(text: RopeSlice, ch: char, mut pos: usize, n: usize) -> Option<usize> {
+// TODO: switch to std::str::Pattern when it is stable.
+pub trait CharMatcher {
+    fn char_match(&self, ch: char) -> bool;
+}
+
+impl CharMatcher for char {
+    fn char_match(&self, ch: char) -> bool {
+        *self == ch
+    }
+}
+
+impl<F: Fn(&char) -> bool> CharMatcher for F {
+    fn char_match(&self, ch: char) -> bool {
+        (*self)(&ch)
+    }
+}
+
+pub fn find_nth_next<M: CharMatcher>(
+    text: RopeSlice,
+    char_matcher: M,
+    mut pos: usize,
+    n: usize,
+) -> Option<usize> {
     if pos >= text.len_chars() || n == 0 {
         return None;
     }
@@ -13,7 +35,7 @@ pub fn find_nth_next(text: RopeSlice, ch: char, mut pos: usize, n: usize) -> Opt
 
             pos += 1;
 
-            if c == ch {
+            if char_matcher.char_match(c) {
                 break;
             }
         }

helix-core/src/selection.rs

@@ -140,6 +140,11 @@ impl Range {
         self.from() == other.from() || (self.to() > other.from() && other.to() > self.from())
     }
 
+    #[inline]
+    pub fn contains_range(&self, other: &Self) -> bool {
+        self.from() <= other.from() && self.to() >= other.to()
+    }
+
     pub fn contains(&self, pos: usize) -> bool {
         self.from() <= pos && pos < self.to()
     }
@@ -544,6 +549,39 @@ impl Selection {
     pub fn len(&self) -> usize {
         self.ranges.len()
     }
+
+    // returns true if self ⊇ other
+    pub fn contains(&self, other: &Selection) -> bool {
+        // can't contain other if it is larger
+        if other.len() > self.len() {
+            return false;
+        }
+
+        let (mut iter_self, mut iter_other) = (self.iter(), other.iter());
+        let (mut ele_self, mut ele_other) = (iter_self.next(), iter_other.next());
+
+        loop {
+            match (ele_self, ele_other) {
+                (Some(ra), Some(rb)) => {
+                    if !ra.contains_range(rb) {
+                        // `self` doesn't contain next element from `other`, advance `self`, we need to match all from `other`
+                        ele_self = iter_self.next();
+                    } else {
+                        // matched element from `other`, advance `other`
+                        ele_other = iter_other.next();
+                    };
+                }
+                (None, Some(_)) => {
+                    // exhausted `self`, we can't match the reminder of `other`
+                    return false;
+                }
+                (_, None) => {
+                    // no elements from `other` left to match, `self` contains `other`
+                    return true;
+                }
+            }
+        }
+    }
 }
 
 impl<'a> IntoIterator for &'a Selection {
@@ -728,16 +766,16 @@ mod test {
     fn test_contains() {
         let range = Range::new(10, 12);
 
-        assert_eq!(range.contains(9), false);
+        assert!(!range.contains(9));
-        assert_eq!(range.contains(10), true);
+        assert!(range.contains(10));
-        assert_eq!(range.contains(11), true);
+        assert!(range.contains(11));
-        assert_eq!(range.contains(12), false);
+        assert!(!range.contains(12));
-        assert_eq!(range.contains(13), false);
+        assert!(!range.contains(13));
 
         let range = Range::new(9, 6);
-        assert_eq!(range.contains(9), false);
+        assert!(!range.contains(9));
-        assert_eq!(range.contains(7), true);
+        assert!(range.contains(7));
-        assert_eq!(range.contains(6), true);
+        assert!(range.contains(6));
     }
 
     #[test]
@@ -982,4 +1020,30 @@ mod test {
             &["", "abcd", "efg", "rs", "xyz"]
         );
     }
+    #[test]
+    fn test_selection_contains() {
+        fn contains(a: Vec<(usize, usize)>, b: Vec<(usize, usize)>) -> bool {
+            let sela = Selection::new(a.iter().map(|a| Range::new(a.0, a.1)).collect(), 0);
+            let selb = Selection::new(b.iter().map(|b| Range::new(b.0, b.1)).collect(), 0);
+            sela.contains(&selb)
+        }
+
+        // exact match
+        assert!(contains(vec!((1, 1)), vec!((1, 1))));
+
+        // larger set contains smaller
+        assert!(contains(vec!((1, 1), (2, 2), (3, 3)), vec!((2, 2))));
+
+        // multiple matches
+        assert!(contains(vec!((1, 1), (2, 2)), vec!((1, 1), (2, 2))));
+
+        // smaller set can't contain bigger
+        assert!(!contains(vec!((1, 1)), vec!((1, 1), (2, 2))));
+
+        assert!(contains(
+            vec!((1, 1), (2, 4), (5, 6), (7, 9), (10, 13)),
+            vec!((3, 4), (7, 9))
+        ));
+        assert!(!contains(vec!((1, 1), (5, 6)), vec!((1, 6))));
+    }
 }

helix-core/src/surround.rs

@@ -1,3 +1,5 @@
+use std::fmt::Display;
+
 use crate::{search, Range, Selection};
 use ropey::RopeSlice;
 
@@ -11,6 +13,27 @@ pub const PAIRS: &[(char, char)] = &[
     ('（', '）'),
 ];
 
+#[derive(Debug, PartialEq)]
+pub enum Error {
+    PairNotFound,
+    CursorOverlap,
+    RangeExceedsText,
+    CursorOnAmbiguousPair,
+}
+
+impl Display for Error {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(match *self {
+            Error::PairNotFound => "Surround pair not found around all cursors",
+            Error::CursorOverlap => "Cursors overlap for a single surround pair range",
+            Error::RangeExceedsText => "Cursor range exceeds text length",
+            Error::CursorOnAmbiguousPair => "Cursor on ambiguous surround pair",
+        })
+    }
+}
+
+type Result<T> = std::result::Result<T, Error>;
+
 /// Given any char in [PAIRS], return the open and closing chars. If not found in
 /// [PAIRS] return (ch, ch).
 ///
@@ -29,6 +52,45 @@ pub fn get_pair(ch: char) -> (char, char) {
         .unwrap_or((ch, ch))
 }
 
+pub fn find_nth_closest_pairs_pos(
+    text: RopeSlice,
+    range: Range,
+    n: usize,
+) -> Result<(usize, usize)> {
+    let is_open_pair = |ch| PAIRS.iter().any(|(open, _)| *open == ch);
+    let is_close_pair = |ch| PAIRS.iter().any(|(_, close)| *close == ch);
+
+    let mut stack = Vec::with_capacity(2);
+    let pos = range.cursor(text);
+
+    for ch in text.chars_at(pos) {
+        if is_open_pair(ch) {
+            // Track open pairs encountered so that we can step over
+            // the correspoding close pairs that will come up further
+            // down the loop. We want to find a lone close pair whose
+            // open pair is before the cursor position.
+            stack.push(ch);
+            continue;
+        } else if is_close_pair(ch) {
+            let (open, _) = get_pair(ch);
+            if stack.last() == Some(&open) {
+                stack.pop();
+                continue;
+            } else {
+                // In the ideal case the stack would be empty here and the
+                // current character would be the close pair that we are
+                // looking for. It could also be the case that the pairs
+                // are unbalanced and we encounter a close pair that doesn't
+                // close the last seen open pair. In either case use this
+                // char as the auto-detected closest pair.
+                return find_nth_pairs_pos(text, ch, range, n);
+            }
+        }
+    }
+
+    Err(Error::PairNotFound)
+}
+
 /// Find the position of surround pairs of `ch` which can be either a closing
 /// or opening pair. `n` will skip n - 1 pairs (eg. n=2 will discard (only)
 /// the first pair found and keep looking)
@@ -37,31 +99,36 @@ pub fn find_nth_pairs_pos(
     ch: char,
     range: Range,
     n: usize,
-) -> Option<(usize, usize)> {
+) -> Result<(usize, usize)> {
-    if text.len_chars() < 2 || range.to() >= text.len_chars() {
+    if text.len_chars() < 2 {
-        return None;
+        return Err(Error::PairNotFound);
+    }
+    if range.to() >= text.len_chars() {
+        return Err(Error::RangeExceedsText);
     }
 
     let (open, close) = get_pair(ch);
     let pos = range.cursor(text);
 
-    if open == close {
+    let (open, close) = if open == close {
         if Some(open) == text.get_char(pos) {
             // Cursor is directly on match char. We return no match
             // because there's no way to know which side of the char
             // we should be searching on.
-            return None;
+            return Err(Error::CursorOnAmbiguousPair);
         }
-        Some((
+        (
-            search::find_nth_prev(text, open, pos, n)?,
+            search::find_nth_prev(text, open, pos, n),
-            search::find_nth_next(text, close, pos, n)?,
+            search::find_nth_next(text, close, pos, n),
-        ))
+        )
     } else {
-        Some((
+        (
-            find_nth_open_pair(text, open, close, pos, n)?,
+            find_nth_open_pair(text, open, close, pos, n),
-            find_nth_close_pair(text, open, close, pos, n)?,
+            find_nth_close_pair(text, open, close, pos, n),
-        ))
+        )
-    }
+    };
+
+    Option::zip(open, close).ok_or(Error::PairNotFound)
 }
 
 fn find_nth_open_pair(
@@ -145,23 +212,28 @@ fn find_nth_close_pair(
 /// Find position of surround characters around every cursor. Returns None
 /// if any positions overlap. Note that the positions are in a flat Vec.
 /// Use get_surround_pos().chunks(2) to get matching pairs of surround positions.
-/// `ch` can be either closing or opening pair.
+/// `ch` can be either closing or opening pair. If `ch` is None, surround pairs
+/// are automatically detected around each cursor (note that this may result
+/// in them selecting different surround characters for each selection).
 pub fn get_surround_pos(
     text: RopeSlice,
     selection: &Selection,
-    ch: char,
+    ch: Option<char>,
     skip: usize,
-) -> Option<Vec<usize>> {
+) -> Result<Vec<usize>> {
     let mut change_pos = Vec::new();
 
     for &range in selection {
-        let (open_pos, close_pos) = find_nth_pairs_pos(text, ch, range, skip)?;
+        let (open_pos, close_pos) = match ch {
+            Some(ch) => find_nth_pairs_pos(text, ch, range, skip)?,
+            None => find_nth_closest_pairs_pos(text, range, skip)?,
+        };
         if change_pos.contains(&open_pos) || change_pos.contains(&close_pos) {
-            return None;
+            return Err(Error::CursorOverlap);
         }
         change_pos.extend_from_slice(&[open_pos, close_pos]);
     }
-    Some(change_pos)
+    Ok(change_pos)
 }
 
 #[cfg(test)]
@@ -172,9 +244,10 @@ mod test {
     use ropey::Rope;
     use smallvec::SmallVec;
 
+    #[allow(clippy::type_complexity)]
     fn check_find_nth_pair_pos(
         text: &str,
-        cases: Vec<(usize, char, usize, Option<(usize, usize)>)>,
+        cases: Vec<(usize, char, usize, Result<(usize, usize)>)>,
     ) {
         let doc = Rope::from(text);
         let slice = doc.slice(..);
@@ -195,13 +268,13 @@ mod test {
             "some (text) here",
             vec![
                 // cursor on [t]ext
-                (6, '(', 1, Some((5, 10))),
+                (6, '(', 1, Ok((5, 10))),
-                (6, ')', 1, Some((5, 10))),
+                (6, ')', 1, Ok((5, 10))),
                 // cursor on so[m]e
-                (2, '(', 1, None),
+                (2, '(', 1, Err(Error::PairNotFound)),
                 // cursor on bracket itself
-                (5, '(', 1, Some((5, 10))),
+                (5, '(', 1, Ok((5, 10))),
-                (10, '(', 1, Some((5, 10))),
+                (10, '(', 1, Ok((5, 10))),
             ],
         );
     }
@@ -212,9 +285,9 @@ mod test {
             "(so (many (good) text) here)",
             vec![
                 // cursor on go[o]d
-                (13, '(', 1, Some((10, 15))),
+                (13, '(', 1, Ok((10, 15))),
-                (13, '(', 2, Some((4, 21))),
+                (13, '(', 2, Ok((4, 21))),
-                (13, '(', 3, Some((0, 27))),
+                (13, '(', 3, Ok((0, 27))),
             ],
         );
     }
@@ -225,11 +298,11 @@ mod test {
             "'so 'many 'good' text' here'",
             vec![
                 // cursor on go[o]d
-                (13, '\'', 1, Some((10, 15))),
+                (13, '\'', 1, Ok((10, 15))),
-                (13, '\'', 2, Some((4, 21))),
+                (13, '\'', 2, Ok((4, 21))),
-                (13, '\'', 3, Some((0, 27))),
+                (13, '\'', 3, Ok((0, 27))),
                 // cursor on the quotes
-                (10, '\'', 1, None),
+                (10, '\'', 1, Err(Error::CursorOnAmbiguousPair)),
             ],
         )
     }
@@ -240,8 +313,8 @@ mod test {
             "((so)((many) good (text))(here))",
             vec![
                 // cursor on go[o]d
-                (15, '(', 1, Some((5, 24))),
+                (15, '(', 1, Ok((5, 24))),
-                (15, '(', 2, Some((0, 31))),
+                (15, '(', 2, Ok((0, 31))),
             ],
         )
     }
@@ -252,9 +325,9 @@ mod test {
             "(so [many {good} text] here)",
             vec![
                 // cursor on go[o]d
-                (13, '{', 1, Some((10, 15))),
+                (13, '{', 1, Ok((10, 15))),
-                (13, '[', 1, Some((4, 21))),
+                (13, '[', 1, Ok((4, 21))),
-                (13, '(', 1, Some((0, 27))),
+                (13, '(', 1, Ok((0, 27))),
             ],
         )
     }
@@ -270,7 +343,7 @@ mod test {
 
         // cursor on s[o]me, c[h]ars, newl[i]ne
         assert_eq!(
-            get_surround_pos(slice, &selection, '(', 1)
+            get_surround_pos(slice, &selection, Some('('), 1)
                 .unwrap()
                 .as_slice(),
             &[0, 5, 7, 13, 15, 23]
@@ -284,11 +357,10 @@ mod test {
 
         let selection =
             Selection::new(SmallVec::from_slice(&[Range::point(2), Range::point(9)]), 0);
-
         // cursor on s[o]me, c[h]ars
         assert_eq!(
-            get_surround_pos(slice, &selection, '(', 1),
+            get_surround_pos(slice, &selection, Some('('), 1),
-            None // different surround chars
+            Err(Error::PairNotFound) // different surround chars
         );
 
         let selection = Selection::new(
@@ -297,8 +369,16 @@ mod test {
         );
         // cursor on [x]x, newli[n]e
         assert_eq!(
-            get_surround_pos(slice, &selection, '(', 1),
+            get_surround_pos(slice, &selection, Some('('), 1),
-            None // overlapping surround chars
+            Err(Error::PairNotFound) // overlapping surround chars
+        );
+
+        let selection =
+            Selection::new(SmallVec::from_slice(&[Range::point(2), Range::point(3)]), 0);
+        // cursor on s[o][m]e
+        assert_eq!(
+            get_surround_pos(slice, &selection, Some('['), 1),
+            Err(Error::CursorOverlap)
         );
     }
 }

helix-core/src/test.rs

@@ -0,0 +1,143 @@
+//! Test helpers.
+use crate::{Range, Selection};
+use smallvec::SmallVec;
+use std::cmp::Reverse;
+
+/// Convert annotated test string to test string and selection.
+///
+/// `#[|` for primary selection with head before anchor followed by `]#`.
+/// `#(|` for secondary selection with head before anchor followed by `)#`.
+/// `#[` for primary selection with head after anchor followed by `|]#`.
+/// `#(` for secondary selection with head after anchor followed by `|)#`.
+///
+/// # Examples
+///
+/// ```
+/// use helix_core::{Range, Selection, test::print};
+/// use smallvec::smallvec;
+///
+/// assert_eq!(
+///     print("#[a|]#b#(|c)#"),
+///     ("abc".to_owned(), Selection::new(smallvec![Range::new(0, 1), Range::new(3, 2)], 0))
+/// );
+/// ```
+///
+/// # Panics
+///
+/// Panics when missing primary or appeared more than once.
+/// Panics when missing head or anchor.
+/// Panics when head come after head or anchor come after anchor.
+pub fn print(s: &str) -> (String, Selection) {
+    let mut primary_idx = None;
+    let mut ranges = SmallVec::new();
+    let mut iter = s.chars().peekable();
+    let mut left = String::with_capacity(s.len());
+
+    'outer: while let Some(c) = iter.next() {
+        let start = left.len();
+
+        if c != '#' {
+            left.push(c);
+            continue;
+        }
+
+        let (is_primary, close_pair) = match iter.next() {
+            Some('[') => (true, ']'),
+            Some('(') => (false, ')'),
+            Some(ch) => {
+                left.push('#');
+                left.push(ch);
+                continue;
+            }
+            None => break,
+        };
+
+        if is_primary && primary_idx.is_some() {
+            panic!("primary `#[` already appeared {:?} {:?}", left, s);
+        }
+
+        let head_at_beg = iter.next_if_eq(&'|').is_some();
+
+        while let Some(c) = iter.next() {
+            if !(c == close_pair && iter.peek() == Some(&'#')) {
+                left.push(c);
+                continue;
+            }
+            if !head_at_beg {
+                let prev = left.pop().unwrap();
+                if prev != '|' {
+                    left.push(prev);
+                    left.push(c);
+                    continue;
+                }
+            }
+            iter.next(); // skip "#"
+
+            if is_primary {
+                primary_idx = Some(ranges.len());
+            }
+            let (anchor, head) = match head_at_beg {
+                true => (left.len(), start),
+                false => (start, left.len()),
+            };
+            ranges.push(Range::new(anchor, head));
+            continue 'outer;
+        }
+
+        if head_at_beg {
+            panic!("missing end `{}#` {:?} {:?}", close_pair, left, s);
+        } else {
+            panic!("missing end `|{}#` {:?} {:?}", close_pair, left, s);
+        }
+    }
+
+    let primary = match primary_idx {
+        Some(i) => i,
+        None => panic!("missing primary `#[|]#` {:?}", s),
+    };
+    let selection = Selection::new(ranges, primary);
+    (left, selection)
+}
+
+/// Convert test string and selection to annotated test string.
+///
+/// `#[|` for primary selection with head before anchor followed by `]#`.
+/// `#(|` for secondary selection with head before anchor followed by `)#`.
+/// `#[` for primary selection with head after anchor followed by `|]#`.
+/// `#(` for secondary selection with head after anchor followed by `|)#`.
+///
+/// # Examples
+///
+/// ```
+/// use helix_core::{Range, Selection, test::plain};
+/// use smallvec::smallvec;
+///
+/// assert_eq!(
+///     plain("abc", Selection::new(smallvec![Range::new(0, 1), Range::new(3, 2)], 0)),
+///     "#[a|]#b#(|c)#".to_owned()
+/// );
+/// ```
+pub fn plain(s: &str, selection: Selection) -> String {
+    let primary = selection.primary_index();
+    let mut out = String::with_capacity(s.len() + 5 * selection.len());
+    out.push_str(s);
+    let mut insertion: Vec<_> = selection
+        .iter()
+        .enumerate()
+        .flat_map(|(i, range)| {
+            // sort like this before reversed so anchor < head later
+            match (range.anchor < range.head, i == primary) {
+                (true, true) => [(range.anchor, "#["), (range.head, "|]#")],
+                (true, false) => [(range.anchor, "#("), (range.head, "|)#")],
+                (false, true) => [(range.anchor, "]#"), (range.head, "#[|")],
+                (false, false) => [(range.anchor, ")#"), (range.head, "#(|")],
+            }
+        })
+        .collect();
+    // insert in reverse order
+    insertion.sort_unstable_by_key(|k| Reverse(k.0));
+    for (i, s) in insertion {
+        out.insert_str(i, s);
+    }
+    out
+}

helix-core/src/textobject.rs

@@ -4,7 +4,8 @@ use ropey::RopeSlice;
 use tree_sitter::{Node, QueryCursor};
 
 use crate::chars::{categorize_char, char_is_whitespace, CharCategory};
-use crate::graphemes::next_grapheme_boundary;
+use crate::graphemes::{next_grapheme_boundary, prev_grapheme_boundary};
+use crate::line_ending::rope_is_line_ending;
 use crate::movement::Direction;
 use crate::surround;
 use crate::syntax::LanguageConfiguration;
@@ -53,6 +54,8 @@ fn find_word_boundary(slice: RopeSlice, mut pos: usize, direction: Direction, lo
 pub enum TextObject {
     Around,
     Inside,
+    /// Used for moving between objects.
+    Movement,
 }
 
 impl Display for TextObject {
@@ -60,6 +63,7 @@ impl Display for TextObject {
         f.write_str(match self {
             Self::Around => "around",
             Self::Inside => "inside",
+            Self::Movement => "movement",
         })
     }
 }
@@ -104,9 +108,96 @@ pub fn textobject_word(
                 Range::new(word_start - whitespace_count_left, word_end)
             }
         }
+        TextObject::Movement => unreachable!(),
     }
 }
 
+pub fn textobject_paragraph(
+    slice: RopeSlice,
+    range: Range,
+    textobject: TextObject,
+    count: usize,
+) -> Range {
+    let mut line = range.cursor_line(slice);
+    let prev_line_empty = rope_is_line_ending(slice.line(line.saturating_sub(1)));
+    let curr_line_empty = rope_is_line_ending(slice.line(line));
+    let next_line_empty = rope_is_line_ending(slice.line(line.saturating_sub(1)));
+    let last_char =
+        prev_grapheme_boundary(slice, slice.line_to_char(line + 1)) == range.cursor(slice);
+    let prev_empty_to_line = prev_line_empty && !curr_line_empty;
+    let curr_empty_to_line = curr_line_empty && !next_line_empty;
+
+    // skip character before paragraph boundary
+    let mut line_back = line; // line but backwards
+    if prev_empty_to_line || curr_empty_to_line {
+        line_back += 1;
+    }
+    // do not include current paragraph on paragraph end (include next)
+    if !(curr_empty_to_line && last_char) {
+        let mut lines = slice.lines_at(line_back);
+        lines.reverse();
+        let mut lines = lines.map(rope_is_line_ending).peekable();
+        while lines.next_if(|&e| e).is_some() {
+            line_back -= 1;
+        }
+        while lines.next_if(|&e| !e).is_some() {
+            line_back -= 1;
+        }
+    }
+
+    // skip character after paragraph boundary
+    if curr_empty_to_line && last_char {
+        line += 1;
+    }
+    let mut lines = slice.lines_at(line).map(rope_is_line_ending).peekable();
+    let mut count_done = 0; // count how many non-whitespace paragraphs done
+    for _ in 0..count {
+        let mut done = false;
+        while lines.next_if(|&e| !e).is_some() {
+            line += 1;
+            done = true;
+        }
+        while lines.next_if(|&e| e).is_some() {
+            line += 1;
+        }
+        count_done += done as usize;
+    }
+
+    // search one paragraph backwards for last paragraph
+    // makes `map` at the end of the paragraph with trailing newlines useful
+    let last_paragraph = count_done != count && lines.peek().is_none();
+    if last_paragraph {
+        let mut lines = slice.lines_at(line_back);
+        lines.reverse();
+        let mut lines = lines.map(rope_is_line_ending).peekable();
+        while lines.next_if(|&e| e).is_some() {
+            line_back -= 1;
+        }
+        while lines.next_if(|&e| !e).is_some() {
+            line_back -= 1;
+        }
+    }
+
+    // handle last whitespaces part separately depending on textobject
+    match textobject {
+        TextObject::Around => {}
+        TextObject::Inside => {
+            // remove last whitespace paragraph
+            let mut lines = slice.lines_at(line);
+            lines.reverse();
+            let mut lines = lines.map(rope_is_line_ending).peekable();
+            while lines.next_if(|&e| e).is_some() {
+                line -= 1;
+            }
+        }
+        TextObject::Movement => unreachable!(),
+    }
+
+    let anchor = slice.line_to_char(line_back);
+    let head = slice.line_to_char(line);
+    Range::new(anchor, head)
+}
+
 pub fn textobject_surround(
     slice: RopeSlice,
     range: Range,
@@ -114,10 +205,35 @@ pub fn textobject_surround(
     ch: char,
     count: usize,
 ) -> Range {
-    surround::find_nth_pairs_pos(slice, ch, range, count)
+    textobject_surround_impl(slice, range, textobject, Some(ch), count)
+}
+
+pub fn textobject_surround_closest(
+    slice: RopeSlice,
+    range: Range,
+    textobject: TextObject,
+    count: usize,
+) -> Range {
+    textobject_surround_impl(slice, range, textobject, None, count)
+}
+
+fn textobject_surround_impl(
+    slice: RopeSlice,
+    range: Range,
+    textobject: TextObject,
+    ch: Option<char>,
+    count: usize,
+) -> Range {
+    let pair_pos = match ch {
+        Some(ch) => surround::find_nth_pairs_pos(slice, ch, range, count),
+        // Automatically find the closest surround pairs
+        None => surround::find_nth_closest_pairs_pos(slice, range, count),
+    };
+    pair_pos
         .map(|(anchor, head)| match textobject {
             TextObject::Inside => Range::new(next_grapheme_boundary(slice, anchor), head),
             TextObject::Around => Range::new(anchor, next_grapheme_boundary(slice, head)),
+            TextObject::Movement => unreachable!(),
         })
         .unwrap_or(range)
 }
@@ -283,6 +399,91 @@ mod test {
         }
     }
 
+    #[test]
+    fn test_textobject_paragraph_inside_single() {
+        let tests = [
+            ("#[|]#", "#[|]#"),
+            ("firs#[t|]#\n\nparagraph\n\n", "#[first\n|]#\nparagraph\n\n"),
+            (
+                "second\n\npa#[r|]#agraph\n\n",
+                "second\n\n#[paragraph\n|]#\n",
+            ),
+            ("#[f|]#irst char\n\n", "#[first char\n|]#\n"),
+            ("last char\n#[\n|]#", "#[last char\n|]#\n"),
+            (
+                "empty to line\n#[\n|]#paragraph boundary\n\n",
+                "empty to line\n\n#[paragraph boundary\n|]#\n",
+            ),
+            (
+                "line to empty\n\n#[p|]#aragraph boundary\n\n",
+                "line to empty\n\n#[paragraph boundary\n|]#\n",
+            ),
+        ];
+
+        for (before, expected) in tests {
+            let (s, selection) = crate::test::print(before);
+            let text = Rope::from(s.as_str());
+            let selection = selection
+                .transform(|r| textobject_paragraph(text.slice(..), r, TextObject::Inside, 1));
+            let actual = crate::test::plain(&s, selection);
+            assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
+        }
+    }
+
+    #[test]
+    fn test_textobject_paragraph_inside_double() {
+        let tests = [
+            (
+                "last two\n\n#[p|]#aragraph\n\nwithout whitespaces\n\n",
+                "last two\n\n#[paragraph\n\nwithout whitespaces\n|]#\n",
+            ),
+            (
+                "last two\n#[\n|]#paragraph\n\nwithout whitespaces\n\n",
+                "last two\n\n#[paragraph\n\nwithout whitespaces\n|]#\n",
+            ),
+        ];
+
+        for (before, expected) in tests {
+            let (s, selection) = crate::test::print(before);
+            let text = Rope::from(s.as_str());
+            let selection = selection
+                .transform(|r| textobject_paragraph(text.slice(..), r, TextObject::Inside, 2));
+            let actual = crate::test::plain(&s, selection);
+            assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
+        }
+    }
+
+    #[test]
+    fn test_textobject_paragraph_around_single() {
+        let tests = [
+            ("#[|]#", "#[|]#"),
+            ("firs#[t|]#\n\nparagraph\n\n", "#[first\n\n|]#paragraph\n\n"),
+            (
+                "second\n\npa#[r|]#agraph\n\n",
+                "second\n\n#[paragraph\n\n|]#",
+            ),
+            ("#[f|]#irst char\n\n", "#[first char\n\n|]#"),
+            ("last char\n#[\n|]#", "#[last char\n\n|]#"),
+            (
+                "empty to line\n#[\n|]#paragraph boundary\n\n",
+                "empty to line\n\n#[paragraph boundary\n\n|]#",
+            ),
+            (
+                "line to empty\n\n#[p|]#aragraph boundary\n\n",
+                "line to empty\n\n#[paragraph boundary\n\n|]#",
+            ),
+        ];
+
+        for (before, expected) in tests {
+            let (s, selection) = crate::test::print(before);
+            let text = Rope::from(s.as_str());
+            let selection = selection
+                .transform(|r| textobject_paragraph(text.slice(..), r, TextObject::Around, 1));
+            let actual = crate::test::plain(&s, selection);
+            assert_eq!(actual, expected, "\nbefore: `{:?}`", before);
+        }
+    }
+
     #[test]
     fn test_textobject_surround() {
         // (text, [(cursor position, textobject, final range, surround char, count), ...])

helix-core/src/transaction.rs

@@ -21,7 +21,6 @@ pub enum Assoc {
     After,
 }
 
-// ChangeSpec = Change | ChangeSet | Vec<Change>
 #[derive(Debug, Default, Clone, PartialEq, Eq)]
 pub struct ChangeSet {
     pub(crate) changes: Vec<Operation>,
@@ -50,7 +49,6 @@ impl ChangeSet {
     }
 
     // TODO: from iter
-    //
 
     #[doc(hidden)] // used by lsp to convert to LSP changes
     pub fn changes(&self) -> &[Operation] {
@@ -85,7 +83,7 @@ impl ChangeSet {
 
         let new_last = match self.changes.as_mut_slice() {
             [.., Insert(prev)] | [.., Insert(prev), Delete(_)] => {
-                prev.push_tendril(&fragment);
+                prev.push_str(&fragment);
                 return;
             }
             [.., last @ Delete(_)] => std::mem::replace(last, Insert(fragment)),
@@ -189,7 +187,7 @@ impl ChangeSet {
                             // TODO: cover this with a test
                             // figure out the byte index of the truncated string end
                             let (pos, _) = s.char_indices().nth(j).unwrap();
-                            s.pop_front(pos as u32);
+                            s.replace_range(0..pos, "");
                             head_a = Some(Insert(s));
                             head_b = changes_b.next();
                         }
@@ -211,9 +209,11 @@ impl ChangeSet {
                         Ordering::Greater => {
                             // figure out the byte index of the truncated string end
                             let (pos, _) = s.char_indices().nth(j).unwrap();
-                            let pos = pos as u32;
+                            let mut before = s;
-                            changes.insert(s.subtendril(0, pos));
+                            let after = before.split_off(pos);
-                            head_a = Some(Insert(s.subtendril(pos, s.len() as u32 - pos)));
+
+                            changes.insert(before);
+                            head_a = Some(Insert(after));
                             head_b = changes_b.next();
                         }
                     }
@@ -277,7 +277,7 @@ impl ChangeSet {
                 }
                 Delete(n) => {
                     let text = Cow::from(original_doc.slice(pos..pos + *n));
-                    changes.insert(Tendril::from_slice(&text));
+                    changes.insert(Tendril::from(text.as_ref()));
                     pos += n;
                 }
                 Insert(s) => {
@@ -413,8 +413,6 @@ impl ChangeSet {
 pub struct Transaction {
     changes: ChangeSet,
     selection: Option<Selection>,
-    // effects, annotations
-    // scroll_into_view
 }
 
 impl Transaction {
@@ -438,14 +436,12 @@ impl Transaction {
 
     /// Returns true if applied successfully.
     pub fn apply(&self, doc: &mut Rope) -> bool {
-        if !self.changes.is_empty() {
+        if self.changes.is_empty() {
-            // apply changes to the document
+            return true;
-            if !self.changes.apply(doc) {
-                return false;
-            }
         }
 
-        true
+        // apply changes to the document
+        self.changes.apply(doc)
     }
 
     /// Generate a transaction that reverts this one.
@@ -473,7 +469,7 @@ impl Transaction {
     /// Generate a transaction from a set of changes.
     pub fn change<I>(doc: &Rope, changes: I) -> Self
     where
-        I: IntoIterator<Item = Change> + Iterator,
+        I: Iterator<Item = Change>,
     {
         let len = doc.len_chars();
 
@@ -481,12 +477,11 @@ impl Transaction {
         let size = upper.unwrap_or(lower);
         let mut changeset = ChangeSet::with_capacity(2 * size + 1); // rough estimate
 
-        // TODO: verify ranges are ordered and not overlapping or change will panic.
-
-        // TODO: test for (pos, pos, None) to factor out as nothing
-
         let mut last = 0;
         for (from, to, tendril) in changes {
+            // Verify ranges are ordered and not overlapping
+            debug_assert!(last <= from);
+
             // Retain from last "to" to current "from"
             changeset.retain(from - last);
             let span = to - from;
@@ -692,7 +687,7 @@ mod test {
         let mut doc = Rope::from("hello world!\ntest 123");
         let transaction = Transaction::change(
             &doc,
-            // (1, 1, None) is a useless 0-width delete
+            // (1, 1, None) is a useless 0-width delete that gets factored out
             vec![(1, 1, None), (6, 11, Some("void".into())), (12, 17, None)].into_iter(),
         );
         transaction.apply(&mut doc);
@@ -710,19 +705,19 @@ mod test {
     #[test]
     fn optimized_composition() {
         let mut state = State::new("".into());
-        let t1 = Transaction::insert(&state.doc, &state.selection, Tendril::from_char('h'));
+        let t1 = Transaction::insert(&state.doc, &state.selection, Tendril::from("h"));
         t1.apply(&mut state.doc);
         state.selection = state.selection.clone().map(t1.changes());
-        let t2 = Transaction::insert(&state.doc, &state.selection, Tendril::from_char('e'));
+        let t2 = Transaction::insert(&state.doc, &state.selection, Tendril::from("e"));
         t2.apply(&mut state.doc);
         state.selection = state.selection.clone().map(t2.changes());
-        let t3 = Transaction::insert(&state.doc, &state.selection, Tendril::from_char('l'));
+        let t3 = Transaction::insert(&state.doc, &state.selection, Tendril::from("l"));
         t3.apply(&mut state.doc);
         state.selection = state.selection.clone().map(t3.changes());
-        let t4 = Transaction::insert(&state.doc, &state.selection, Tendril::from_char('l'));
+        let t4 = Transaction::insert(&state.doc, &state.selection, Tendril::from("l"));
         t4.apply(&mut state.doc);
         state.selection = state.selection.clone().map(t4.changes());
-        let t5 = Transaction::insert(&state.doc, &state.selection, Tendril::from_char('o'));
+        let t5 = Transaction::insert(&state.doc, &state.selection, Tendril::from("o"));
         t5.apply(&mut state.doc);
         state.selection = state.selection.clone().map(t5.changes());
 
@@ -761,7 +756,7 @@ mod test {
 
     #[test]
     fn combine_with_utf8() {
-        const TEST_CASE: &'static str = "Hello, これはヘリックスエディターです！";
+        const TEST_CASE: &str = "Hello, これはヘリックスエディターです！";
 
         let empty = Rope::from("");
         let a = ChangeSet::new(&empty);

helix-core/tests/data/indent/indent.rs

@@ -0,0 +1 @@
+../../../src/indent.rs

helix-core/tests/data/indent/languages.toml

@@ -0,0 +1,13 @@
+# This languages.toml should contain definitions for all languages for which we have indent tests
+[[language]]
+name = "rust"
+scope = "source.rust"
+injection-regex = "rust"
+file-types = ["rs"]
+comment-token = "//"
+roots = ["Cargo.toml", "Cargo.lock"]
+indent = { tab-width = 4, unit = "    " }
+
+[[grammar]]
+name = "rust"
+source = { git = "https://github.com/tree-sitter/tree-sitter-rust", rev = "a360da0a29a19c281d08295a35ecd0544d2da211" }

helix-core/tests/data/indent/rust.rs

@@ -0,0 +1,105 @@
+use std::{
+    io::{self, stdout, Stdout, Write},
+    path::PathBuf,
+    sync::Arc,
+    time::Duration,
+};
+mod test {
+    fn hello_world() {
+        1 + 1;
+
+        let does_indentation_work = 1;
+
+        let mut really_long_variable_name_using_up_the_line =
+            really_long_fn_that_should_definitely_go_on_the_next_line();
+        really_long_variable_name_using_up_the_line =
+            really_long_fn_that_should_definitely_go_on_the_next_line();
+        really_long_variable_name_using_up_the_line |=
+            really_long_fn_that_should_definitely_go_on_the_next_line();
+
+        let (
+            a_long_variable_name_in_this_tuple,
+            b_long_variable_name_in_this_tuple,
+            c_long_variable_name_in_this_tuple,
+            d_long_variable_name_in_this_tuple,
+            e_long_variable_name_in_this_tuple,
+        ): (usize, usize, usize, usize, usize) =
+            if really_long_fn_that_should_definitely_go_on_the_next_line() {
+                (
+                    03294239434,
+                    1213412342314,
+                    21231234134,
+                    834534234549898789,
+                    9879234234543853457,
+                )
+            } else {
+                (0, 1, 2, 3, 4)
+            };
+
+        let test_function = function_with_param(this_param,
+            that_param
+        );
+
+        let test_function = function_with_param(
+            this_param,
+            that_param
+        );
+
+        let test_function = function_with_proper_indent(param1,
+            param2,
+        );
+
+        let selection = Selection::new(
+            changes
+                .clone()
+                .map(|(start, end, text): (usize, usize, Option<Tendril>)| {
+                    let len = text.map(|text| text.len()).unwrap() - 1; // minus newline
+                    let pos = start + len;
+                    Range::new(pos, pos)
+                })
+                .collect(),
+            0,
+        );
+
+        return;
+    }
+}
+
+impl<A, D> MyTrait<A, D> for YourType
+where
+    A: TraitB + TraitC,
+    D: TraitE + TraitF,
+{
+
+}
+#[test]
+//
+match test {
+    Some(a) => 1,
+    None => {
+        unimplemented!()
+    }
+}
+std::panic::set_hook(Box::new(move |info| {
+    hook(info);
+}));
+
+{ { {
+    1
+}}}
+
+pub fn change<I>(document: &Document, changes: I) -> Self
+where
+    I: IntoIterator<Item = Change> + ExactSizeIterator,
+{
+    [
+        1,
+        2,
+        3,
+    ];
+    (
+        1,
+        2
+    );
+    true
+}

helix-core/tests/indent.rs

@@ -0,0 +1,68 @@
+use helix_core::{
+    indent::{treesitter_indent_for_pos, IndentStyle},
+    syntax::Loader,
+    Syntax,
+};
+use std::path::PathBuf;
+
+#[test]
+fn test_treesitter_indent_rust() {
+    test_treesitter_indent("rust.rs", "source.rust");
+}
+#[test]
+fn test_treesitter_indent_rust_2() {
+    test_treesitter_indent("indent.rs", "source.rust");
+    // TODO Use commands.rs as indentation test.
+    // Currently this fails because we can't align the parameters of a closure yet
+    // test_treesitter_indent("commands.rs", "source.rust");
+}
+
+fn test_treesitter_indent(file_name: &str, lang_scope: &str) {
+    let mut test_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    test_dir.push("tests/data/indent");
+
+    let mut test_file = test_dir.clone();
+    test_file.push(file_name);
+    let test_file = std::fs::File::open(test_file).unwrap();
+    let doc = ropey::Rope::from_reader(test_file).unwrap();
+
+    let mut config_file = test_dir;
+    config_file.push("languages.toml");
+    let config = std::fs::read(config_file).unwrap();
+    let config = toml::from_slice(&config).unwrap();
+    let loader = Loader::new(config);
+
+    // set runtime path so we can find the queries
+    let mut runtime = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    runtime.push("../runtime");
+    std::env::set_var("HELIX_RUNTIME", runtime.to_str().unwrap());
+
+    let language_config = loader.language_config_for_scope(lang_scope).unwrap();
+    let highlight_config = language_config.highlight_config(&[]).unwrap();
+    let syntax = Syntax::new(&doc, highlight_config, std::sync::Arc::new(loader));
+    let indent_query = language_config.indent_query().unwrap();
+    let text = doc.slice(..);
+
+    for i in 0..doc.len_lines() {
+        let line = text.line(i);
+        if let Some(pos) = helix_core::find_first_non_whitespace_char(line) {
+            let suggested_indent = treesitter_indent_for_pos(
+                indent_query,
+                &syntax,
+                &IndentStyle::Spaces(4),
+                text,
+                i,
+                text.line_to_char(i) + pos,
+                false,
+            )
+            .unwrap();
+            assert!(
+                line.get_slice(..pos).map_or(false, |s| s == suggested_indent),
+                "Wrong indentation on line {}:\n\"{}\" (original line)\n\"{}\" (suggested indentation)\n",
+                i+1,
+                line.slice(..line.len_chars()-1),
+                suggested_indent,
+            );
+        }
+    }
+}

helix-dap/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "helix-dap"
+version = "0.6.0"
+authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
+edition = "2018"
+license = "MPL-2.0"
+description = "DAP client implementation for Helix project"
+categories = ["editor"]
+repository = "https://github.com/helix-editor/helix"
+homepage = "https://helix-editor.com"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+helix-core = { version = "0.6", path = "../helix-core" }
+anyhow = "1.0"
+log = "0.4"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+thiserror = "1.0"
+tokio = { version = "1", features = ["rt", "rt-multi-thread", "io-util", "io-std", "time", "process", "macros", "fs", "parking_lot", "net", "sync"] }
+which = "4.2"
+
+[dev-dependencies]
+fern = "0.6"

helix-dap/src/client.rs

@@ -0,0 +1,480 @@
+use crate::{
+    transport::{Payload, Request, Response, Transport},
+    types::*,
+    Error, Result, ThreadId,
+};
+use helix_core::syntax::DebuggerQuirks;
+
+use serde_json::Value;
+
+use anyhow::anyhow;
+pub use log::{error, info};
+use std::{
+    collections::HashMap,
+    future::Future,
+    net::{IpAddr, Ipv4Addr, SocketAddr},
+    path::PathBuf,
+    process::Stdio,
+    sync::atomic::{AtomicU64, Ordering},
+};
+use tokio::{
+    io::{AsyncBufRead, AsyncWrite, BufReader, BufWriter},
+    net::TcpStream,
+    process::{Child, Command},
+    sync::mpsc::{channel, unbounded_channel, UnboundedReceiver, UnboundedSender},
+    time,
+};
+
+#[derive(Debug)]
+pub struct Client {
+    id: usize,
+    _process: Option<Child>,
+    server_tx: UnboundedSender<Payload>,
+    request_counter: AtomicU64,
+    pub caps: Option<DebuggerCapabilities>,
+    // thread_id -> frames
+    pub stack_frames: HashMap<ThreadId, Vec<StackFrame>>,
+    pub thread_states: HashMap<ThreadId, String>,
+    pub thread_id: Option<ThreadId>,
+    /// Currently active frame for the current thread.
+    pub active_frame: Option<usize>,
+    pub quirks: DebuggerQuirks,
+}
+
+impl Client {
+    // Spawn a process and communicate with it by either TCP or stdio
+    pub async fn process(
+        transport: &str,
+        command: &str,
+        args: Vec<&str>,
+        port_arg: Option<&str>,
+        id: usize,
+    ) -> Result<(Self, UnboundedReceiver<Payload>)> {
+        if command.is_empty() {
+            return Result::Err(Error::Other(anyhow!("Command not provided")));
+        }
+        if transport == "tcp" && port_arg.is_some() {
+            Self::tcp_process(command, args, port_arg.unwrap(), id).await
+        } else if transport == "stdio" {
+            Self::stdio(command, args, id)
+        } else {
+            Result::Err(Error::Other(anyhow!("Incorrect transport {}", transport)))
+        }
+    }
+
+    pub fn streams(
+        rx: Box<dyn AsyncBufRead + Unpin + Send>,
+        tx: Box<dyn AsyncWrite + Unpin + Send>,
+        err: Option<Box<dyn AsyncBufRead + Unpin + Send>>,
+        id: usize,
+        process: Option<Child>,
+    ) -> Result<(Self, UnboundedReceiver<Payload>)> {
+        let (server_rx, server_tx) = Transport::start(rx, tx, err, id);
+        let (client_rx, client_tx) = unbounded_channel();
+
+        let client = Self {
+            id,
+            _process: process,
+            server_tx,
+            request_counter: AtomicU64::new(0),
+            caps: None,
+            //
+            stack_frames: HashMap::new(),
+            thread_states: HashMap::new(),
+            thread_id: None,
+            active_frame: None,
+            quirks: DebuggerQuirks::default(),
+        };
+
+        tokio::spawn(Self::recv(server_rx, client_rx));
+
+        Ok((client, client_tx))
+    }
+
+    pub async fn tcp(
+        addr: std::net::SocketAddr,
+        id: usize,
+    ) -> Result<(Self, UnboundedReceiver<Payload>)> {
+        let stream = TcpStream::connect(addr).await?;
+        let (rx, tx) = stream.into_split();
+        Self::streams(Box::new(BufReader::new(rx)), Box::new(tx), None, id, None)
+    }
+
+    pub fn stdio(
+        cmd: &str,
+        args: Vec<&str>,
+        id: usize,
+    ) -> Result<(Self, UnboundedReceiver<Payload>)> {
+        // Resolve path to the binary
+        let cmd = which::which(cmd).map_err(|err| anyhow::anyhow!(err))?;
+
+        let process = Command::new(cmd)
+            .args(args)
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            // make sure the process is reaped on drop
+            .kill_on_drop(true)
+            .spawn();
+
+        let mut process = process?;
+
+        // TODO: do we need bufreader/writer here? or do we use async wrappers on unblock?
+        let writer = BufWriter::new(process.stdin.take().expect("Failed to open stdin"));
+        let reader = BufReader::new(process.stdout.take().expect("Failed to open stdout"));
+        let errors = process.stderr.take().map(BufReader::new);
+
+        Self::streams(
+            Box::new(BufReader::new(reader)),
+            Box::new(writer),
+            // errors.map(|errors| Box::new(BufReader::new(errors))),
+            match errors {
+                Some(errors) => Some(Box::new(BufReader::new(errors))),
+                None => None,
+            },
+            id,
+            Some(process),
+        )
+    }
+
+    async fn get_port() -> Option<u16> {
+        Some(
+            tokio::net::TcpListener::bind(SocketAddr::new(
+                IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)),
+                0,
+            ))
+            .await
+            .ok()?
+            .local_addr()
+            .ok()?
+            .port(),
+        )
+    }
+
+    pub async fn tcp_process(
+        cmd: &str,
+        args: Vec<&str>,
+        port_format: &str,
+        id: usize,
+    ) -> Result<(Self, UnboundedReceiver<Payload>)> {
+        let port = Self::get_port().await.unwrap();
+
+        let process = Command::new(cmd)
+            .args(args)
+            .args(port_format.replace("{}", &port.to_string()).split(' '))
+            // silence messages
+            .stdin(Stdio::null())
+            .stdout(Stdio::null())
+            .stderr(Stdio::null())
+            // Do not kill debug adapter when leaving, it should exit automatically
+            .spawn()?;
+
+        // Wait for adapter to become ready for connection
+        time::sleep(time::Duration::from_millis(500)).await;
+
+        let stream = TcpStream::connect(SocketAddr::new(
+            IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)),
+            port,
+        ))
+        .await?;
+
+        let (rx, tx) = stream.into_split();
+        Self::streams(
+            Box::new(BufReader::new(rx)),
+            Box::new(tx),
+            None,
+            id,
+            Some(process),
+        )
+    }
+
+    async fn recv(mut server_rx: UnboundedReceiver<Payload>, client_tx: UnboundedSender<Payload>) {
+        while let Some(msg) = server_rx.recv().await {
+            match msg {
+                Payload::Event(ev) => {
+                    client_tx.send(Payload::Event(ev)).expect("Failed to send");
+                }
+                Payload::Response(_) => unreachable!(),
+                Payload::Request(req) => {
+                    client_tx
+                        .send(Payload::Request(req))
+                        .expect("Failed to send");
+                }
+            }
+        }
+    }
+
+    pub fn id(&self) -> usize {
+        self.id
+    }
+
+    fn next_request_id(&self) -> u64 {
+        self.request_counter.fetch_add(1, Ordering::Relaxed)
+    }
+
+    // Internal, called by specific DAP commands when resuming
+    pub fn resume_application(&mut self) {
+        if let Some(thread_id) = self.thread_id {
+            self.thread_states.insert(thread_id, "running".to_string());
+            self.stack_frames.remove(&thread_id);
+        }
+        self.active_frame = None;
+        self.thread_id = None;
+    }
+
+    /// Execute a RPC request on the debugger.
+    pub fn call<R: crate::types::Request>(
+        &self,
+        arguments: R::Arguments,
+    ) -> impl Future<Output = Result<Value>>
+    where
+        R::Arguments: serde::Serialize,
+    {
+        let server_tx = self.server_tx.clone();
+        let id = self.next_request_id();
+
+        async move {
+            use std::time::Duration;
+            use tokio::time::timeout;
+
+            let arguments = Some(serde_json::to_value(arguments)?);
+
+            let (callback_tx, mut callback_rx) = channel(1);
+
+            let req = Request {
+                back_ch: Some(callback_tx),
+                seq: id,
+                command: R::COMMAND.to_string(),
+                arguments,
+            };
+
+            server_tx
+                .send(Payload::Request(req))
+                .map_err(|e| Error::Other(e.into()))?;
+
+            // TODO: specifiable timeout, delay other calls until initialize success
+            timeout(Duration::from_secs(20), callback_rx.recv())
+                .await
+                .map_err(|_| Error::Timeout)? // return Timeout
+                .ok_or(Error::StreamClosed)?
+                .map(|response| response.body.unwrap_or_default())
+            // TODO: check response.success
+        }
+    }
+
+    pub async fn request<R: crate::types::Request>(&self, params: R::Arguments) -> Result<R::Result>
+    where
+        R::Arguments: serde::Serialize,
+        R::Result: core::fmt::Debug, // TODO: temporary
+    {
+        // a future that resolves into the response
+        let json = self.call::<R>(params).await?;
+        let response = serde_json::from_value(json)?;
+        Ok(response)
+    }
+
+    pub fn reply(
+        &self,
+        request_seq: u64,
+        command: &str,
+        result: core::result::Result<Value, Error>,
+    ) -> impl Future<Output = Result<()>> {
+        let server_tx = self.server_tx.clone();
+        let command = command.to_string();
+
+        async move {
+            let response = match result {
+                Ok(result) => Response {
+                    request_seq,
+                    command,
+                    success: true,
+                    message: None,
+                    body: Some(result),
+                },
+                Err(error) => Response {
+                    request_seq,
+                    command,
+                    success: false,
+                    message: Some(error.to_string()),
+                    body: None,
+                },
+            };
+
+            server_tx
+                .send(Payload::Response(response))
+                .map_err(|e| Error::Other(e.into()))?;
+
+            Ok(())
+        }
+    }
+
+    pub fn capabilities(&self) -> &DebuggerCapabilities {
+        self.caps.as_ref().expect("debugger not yet initialized!")
+    }
+
+    pub async fn initialize(&mut self, adapter_id: String) -> Result<()> {
+        let args = requests::InitializeArguments {
+            client_id: Some("hx".to_owned()),
+            client_name: Some("helix".to_owned()),
+            adapter_id,
+            locale: Some("en-us".to_owned()),
+            lines_start_at_one: Some(true),
+            columns_start_at_one: Some(true),
+            path_format: Some("path".to_owned()),
+            supports_variable_type: Some(true),
+            supports_variable_paging: Some(false),
+            supports_run_in_terminal_request: Some(true),
+            supports_memory_references: Some(false),
+            supports_progress_reporting: Some(false),
+            supports_invalidated_event: Some(false),
+        };
+
+        let response = self.request::<requests::Initialize>(args).await?;
+        self.caps = Some(response);
+
+        Ok(())
+    }
+
+    pub fn disconnect(&self) -> impl Future<Output = Result<Value>> {
+        self.call::<requests::Disconnect>(())
+    }
+
+    pub fn launch(&self, args: serde_json::Value) -> impl Future<Output = Result<Value>> {
+        self.call::<requests::Launch>(args)
+    }
+
+    pub fn attach(&self, args: serde_json::Value) -> impl Future<Output = Result<Value>> {
+        self.call::<requests::Attach>(args)
+    }
+
+    pub async fn set_breakpoints(
+        &self,
+        file: PathBuf,
+        breakpoints: Vec<SourceBreakpoint>,
+    ) -> Result<Option<Vec<Breakpoint>>> {
+        let args = requests::SetBreakpointsArguments {
+            source: Source {
+                path: Some(file),
+                name: None,
+                source_reference: None,
+                presentation_hint: None,
+                origin: None,
+                sources: None,
+                adapter_data: None,
+                checksums: None,
+            },
+            breakpoints: Some(breakpoints),
+            source_modified: Some(false),
+        };
+
+        let response = self.request::<requests::SetBreakpoints>(args).await?;
+
+        Ok(response.breakpoints)
+    }
+
+    pub async fn configuration_done(&self) -> Result<()> {
+        self.request::<requests::ConfigurationDone>(()).await
+    }
+
+    pub fn continue_thread(&self, thread_id: ThreadId) -> impl Future<Output = Result<Value>> {
+        let args = requests::ContinueArguments { thread_id };
+
+        self.call::<requests::Continue>(args)
+    }
+
+    pub async fn stack_trace(
+        &self,
+        thread_id: ThreadId,
+    ) -> Result<(Vec<StackFrame>, Option<usize>)> {
+        let args = requests::StackTraceArguments {
+            thread_id,
+            start_frame: None,
+            levels: None,
+            format: None,
+        };
+
+        let response = self.request::<requests::StackTrace>(args).await?;
+        Ok((response.stack_frames, response.total_frames))
+    }
+
+    pub fn threads(&self) -> impl Future<Output = Result<Value>> {
+        self.call::<requests::Threads>(())
+    }
+
+    pub async fn scopes(&self, frame_id: usize) -> Result<Vec<Scope>> {
+        let args = requests::ScopesArguments { frame_id };
+
+        let response = self.request::<requests::Scopes>(args).await?;
+        Ok(response.scopes)
+    }
+
+    pub async fn variables(&self, variables_reference: usize) -> Result<Vec<Variable>> {
+        let args = requests::VariablesArguments {
+            variables_reference,
+            filter: None,
+            start: None,
+            count: None,
+            format: None,
+        };
+
+        let response = self.request::<requests::Variables>(args).await?;
+        Ok(response.variables)
+    }
+
+    pub fn step_in(&self, thread_id: ThreadId) -> impl Future<Output = Result<Value>> {
+        let args = requests::StepInArguments {
+            thread_id,
+            target_id: None,
+            granularity: None,
+        };
+
+        self.call::<requests::StepIn>(args)
+    }
+
+    pub fn step_out(&self, thread_id: ThreadId) -> impl Future<Output = Result<Value>> {
+        let args = requests::StepOutArguments {
+            thread_id,
+            granularity: None,
+        };
+
+        self.call::<requests::StepOut>(args)
+    }
+
+    pub fn next(&self, thread_id: ThreadId) -> impl Future<Output = Result<Value>> {
+        let args = requests::NextArguments {
+            thread_id,
+            granularity: None,
+        };
+
+        self.call::<requests::Next>(args)
+    }
+
+    pub fn pause(&self, thread_id: ThreadId) -> impl Future<Output = Result<Value>> {
+        let args = requests::PauseArguments { thread_id };
+
+        self.call::<requests::Pause>(args)
+    }
+
+    pub async fn eval(
+        &self,
+        expression: String,
+        frame_id: Option<usize>,
+    ) -> Result<requests::EvaluateResponse> {
+        let args = requests::EvaluateArguments {
+            expression,
+            frame_id,
+            context: None,
+            format: None,
+        };
+
+        self.request::<requests::Evaluate>(args).await
+    }
+
+    pub fn set_exception_breakpoints(
+        &self,
+        filters: Vec<String>,
+    ) -> impl Future<Output = Result<Value>> {
+        let args = requests::SetExceptionBreakpointsArguments { filters };
+
+        self.call::<requests::SetExceptionBreakpoints>(args)
+    }
+}

helix-dap/src/lib.rs

@@ -0,0 +1,24 @@
+mod client;
+mod transport;
+mod types;
+
+pub use client::Client;
+pub use events::Event;
+pub use transport::{Payload, Response, Transport};
+pub use types::*;
+
+use thiserror::Error;
+#[derive(Error, Debug)]
+pub enum Error {
+    #[error("failed to parse: {0}")]
+    Parse(#[from] serde_json::Error),
+    #[error("IO Error: {0}")]
+    IO(#[from] std::io::Error),
+    #[error("request timed out")]
+    Timeout,
+    #[error("server closed the stream")]
+    StreamClosed,
+    #[error(transparent)]
+    Other(#[from] anyhow::Error),
+}
+pub type Result<T> = core::result::Result<T, Error>;

helix-dap/src/transport.rs

@@ -0,0 +1,280 @@
+use crate::{Error, Event, Result};
+use anyhow::Context;
+use log::{error, info, warn};
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use std::collections::HashMap;
+use std::sync::Arc;
+use tokio::{
+    io::{AsyncBufRead, AsyncBufReadExt, AsyncReadExt, AsyncWrite, AsyncWriteExt},
+    sync::{
+        mpsc::{unbounded_channel, Sender, UnboundedReceiver, UnboundedSender},
+        Mutex,
+    },
+};
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct Request {
+    #[serde(skip)]
+    pub back_ch: Option<Sender<Result<Response>>>,
+    pub seq: u64,
+    pub command: String,
+    pub arguments: Option<Value>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+pub struct Response {
+    // seq is omitted as unused and is not sent by some implementations
+    pub request_seq: u64,
+    pub success: bool,
+    pub command: String,
+    pub message: Option<String>,
+    pub body: Option<Value>,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+#[serde(tag = "type", rename_all = "camelCase")]
+pub enum Payload {
+    // type = "event"
+    Event(Box<Event>),
+    // type = "response"
+    Response(Response),
+    // type = "request"
+    Request(Request),
+}
+
+#[derive(Debug)]
+pub struct Transport {
+    #[allow(unused)]
+    id: usize,
+    pending_requests: Mutex<HashMap<u64, Sender<Result<Response>>>>,
+}
+
+impl Transport {
+    pub fn start(
+        server_stdout: Box<dyn AsyncBufRead + Unpin + Send>,
+        server_stdin: Box<dyn AsyncWrite + Unpin + Send>,
+        server_stderr: Option<Box<dyn AsyncBufRead + Unpin + Send>>,
+        id: usize,
+    ) -> (UnboundedReceiver<Payload>, UnboundedSender<Payload>) {
+        let (client_tx, rx) = unbounded_channel();
+        let (tx, client_rx) = unbounded_channel();
+
+        let transport = Self {
+            id,
+            pending_requests: Mutex::new(HashMap::default()),
+        };
+
+        let transport = Arc::new(transport);
+
+        tokio::spawn(Self::recv(transport.clone(), server_stdout, client_tx));
+        tokio::spawn(Self::send(transport, server_stdin, client_rx));
+        if let Some(stderr) = server_stderr {
+            tokio::spawn(Self::err(stderr));
+        }
+
+        (rx, tx)
+    }
+
+    async fn recv_server_message(
+        reader: &mut Box<dyn AsyncBufRead + Unpin + Send>,
+        buffer: &mut String,
+    ) -> Result<Payload> {
+        let mut content_length = None;
+        loop {
+            buffer.truncate(0);
+            if reader.read_line(buffer).await? == 0 {
+                return Err(Error::StreamClosed);
+            };
+
+            if buffer == "\r\n" {
+                // look for an empty CRLF line
+                break;
+            }
+
+            let header = buffer.trim();
+            let parts = header.split_once(": ");
+
+            match parts {
+                Some(("Content-Length", value)) => {
+                    content_length = Some(value.parse().context("invalid content length")?);
+                }
+                Some((_, _)) => {}
+                None => {
+                    // Workaround: Some non-conformant language servers will output logging and other garbage
+                    // into the same stream as JSON-RPC messages. This can also happen from shell scripts that spawn
+                    // the server. Skip such lines and log a warning.
+
+                    // warn!("Failed to parse header: {:?}", header);
+                }
+            }
+        }
+
+        let content_length = content_length.context("missing content length")?;
+
+        //TODO: reuse vector
+        let mut content = vec![0; content_length];
+        reader.read_exact(&mut content).await?;
+        let msg = std::str::from_utf8(&content).context("invalid utf8 from server")?;
+
+        info!("<- DAP {}", msg);
+
+        // try parsing as output (server response) or call (server request)
+        let output: serde_json::Result<Payload> = serde_json::from_str(msg);
+
+        Ok(output?)
+    }
+
+    async fn recv_server_error(
+        err: &mut (impl AsyncBufRead + Unpin + Send),
+        buffer: &mut String,
+    ) -> Result<()> {
+        buffer.truncate(0);
+        if err.read_line(buffer).await? == 0 {
+            return Err(Error::StreamClosed);
+        };
+        error!("err <- {}", buffer);
+
+        Ok(())
+    }
+
+    async fn send_payload_to_server(
+        &self,
+        server_stdin: &mut Box<dyn AsyncWrite + Unpin + Send>,
+        mut payload: Payload,
+    ) -> Result<()> {
+        if let Payload::Request(request) = &mut payload {
+            if let Some(back) = request.back_ch.take() {
+                self.pending_requests.lock().await.insert(request.seq, back);
+            }
+        }
+        let json = serde_json::to_string(&payload)?;
+        self.send_string_to_server(server_stdin, json).await
+    }
+
+    async fn send_string_to_server(
+        &self,
+        server_stdin: &mut Box<dyn AsyncWrite + Unpin + Send>,
+        request: String,
+    ) -> Result<()> {
+        info!("-> DAP {}", request);
+
+        // send the headers
+        server_stdin
+            .write_all(format!("Content-Length: {}\r\n\r\n", request.len()).as_bytes())
+            .await?;
+
+        // send the body
+        server_stdin.write_all(request.as_bytes()).await?;
+
+        server_stdin.flush().await?;
+
+        Ok(())
+    }
+
+    fn process_response(res: Response) -> Result<Response> {
+        if res.success {
+            info!("<- DAP success in response to {}", res.request_seq);
+
+            Ok(res)
+        } else {
+            error!(
+                "<- DAP error {:?} ({:?}) for command #{} {}",
+                res.message, res.body, res.request_seq, res.command
+            );
+
+            Err(Error::Other(anyhow::format_err!("{:?}", res.body)))
+        }
+    }
+
+    async fn process_server_message(
+        &self,
+        client_tx: &UnboundedSender<Payload>,
+        msg: Payload,
+    ) -> Result<()> {
+        match msg {
+            Payload::Response(res) => {
+                let request_seq = res.request_seq;
+                let tx = self.pending_requests.lock().await.remove(&request_seq);
+
+                match tx {
+                    Some(tx) => match tx.send(Self::process_response(res)).await {
+                        Ok(_) => (),
+                        Err(_) => error!(
+                            "Tried sending response into a closed channel (id={:?}), original request likely timed out",
+                            request_seq
+                        ),
+                    }
+                    None => {
+                        warn!("Response to nonexistent request #{}", res.request_seq);
+                        client_tx.send(Payload::Response(res)).expect("Failed to send");
+                    }
+                }
+
+                Ok(())
+            }
+            Payload::Request(Request {
+                ref command,
+                ref seq,
+                ..
+            }) => {
+                info!("<- DAP request {} #{}", command, seq);
+                client_tx.send(msg).expect("Failed to send");
+                Ok(())
+            }
+            Payload::Event(ref event) => {
+                info!("<- DAP event {:?}", event);
+                client_tx.send(msg).expect("Failed to send");
+                Ok(())
+            }
+        }
+    }
+
+    async fn recv(
+        transport: Arc<Self>,
+        mut server_stdout: Box<dyn AsyncBufRead + Unpin + Send>,
+        client_tx: UnboundedSender<Payload>,
+    ) {
+        let mut recv_buffer = String::new();
+        loop {
+            match Self::recv_server_message(&mut server_stdout, &mut recv_buffer).await {
+                Ok(msg) => {
+                    transport
+                        .process_server_message(&client_tx, msg)
+                        .await
+                        .unwrap();
+                }
+                Err(err) => {
+                    error!("err: <- {:?}", err);
+                    break;
+                }
+            }
+        }
+    }
+
+    async fn send(
+        transport: Arc<Self>,
+        mut server_stdin: Box<dyn AsyncWrite + Unpin + Send>,
+        mut client_rx: UnboundedReceiver<Payload>,
+    ) {
+        while let Some(payload) = client_rx.recv().await {
+            transport
+                .send_payload_to_server(&mut server_stdin, payload)
+                .await
+                .unwrap()
+        }
+    }
+
+    async fn err(mut server_stderr: Box<dyn AsyncBufRead + Unpin + Send>) {
+        let mut recv_buffer = String::new();
+        loop {
+            match Self::recv_server_error(&mut server_stderr, &mut recv_buffer).await {
+                Ok(_) => {}
+                Err(err) => {
+                    error!("err: <- {:?}", err);
+                    break;
+                }
+            }
+        }
+    }
+}

helix-dap/src/types.rs

@@ -0,0 +1,707 @@
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use std::collections::HashMap;
+use std::path::PathBuf;
+
+#[derive(
+    Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize,
+)]
+pub struct ThreadId(isize);
+
+impl std::fmt::Display for ThreadId {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+pub trait Request {
+    type Arguments: serde::de::DeserializeOwned + serde::Serialize;
+    type Result: serde::de::DeserializeOwned + serde::Serialize;
+    const COMMAND: &'static str;
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ColumnDescriptor {
+    pub attribute_name: String,
+    pub label: String,
+    pub format: Option<String>,
+    #[serde(rename = "type")]
+    pub ty: Option<String>,
+    pub width: Option<usize>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ExceptionBreakpointsFilter {
+    pub filter: String,
+    pub label: String,
+    pub description: Option<String>,
+    pub default: Option<bool>,
+    pub supports_condition: Option<bool>,
+    pub condition_description: Option<String>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct DebuggerCapabilities {
+    pub supports_configuration_done_request: Option<bool>,
+    pub supports_function_breakpoints: Option<bool>,
+    pub supports_conditional_breakpoints: Option<bool>,
+    pub supports_hit_conditional_breakpoints: Option<bool>,
+    pub supports_evaluate_for_hovers: Option<bool>,
+    pub supports_step_back: Option<bool>,
+    pub supports_set_variable: Option<bool>,
+    pub supports_restart_frame: Option<bool>,
+    pub supports_goto_targets_request: Option<bool>,
+    pub supports_step_in_targets_request: Option<bool>,
+    pub supports_completions_request: Option<bool>,
+    pub supports_modules_request: Option<bool>,
+    pub supports_restart_request: Option<bool>,
+    pub supports_exception_options: Option<bool>,
+    pub supports_value_formatting_options: Option<bool>,
+    pub supports_exception_info_request: Option<bool>,
+    pub support_terminate_debuggee: Option<bool>,
+    pub support_suspend_debuggee: Option<bool>,
+    pub supports_delayed_stack_trace_loading: Option<bool>,
+    pub supports_loaded_sources_request: Option<bool>,
+    pub supports_log_points: Option<bool>,
+    pub supports_terminate_threads_request: Option<bool>,
+    pub supports_set_expression: Option<bool>,
+    pub supports_terminate_request: Option<bool>,
+    pub supports_data_breakpoints: Option<bool>,
+    pub supports_read_memory_request: Option<bool>,
+    pub supports_write_memory_request: Option<bool>,
+    pub supports_disassemble_request: Option<bool>,
+    pub supports_cancel_request: Option<bool>,
+    pub supports_breakpoint_locations_request: Option<bool>,
+    pub supports_clipboard_context: Option<bool>,
+    pub supports_stepping_granularity: Option<bool>,
+    pub supports_instruction_breakpoints: Option<bool>,
+    pub supports_exception_filter_options: Option<bool>,
+    pub exception_breakpoint_filters: Option<Vec<ExceptionBreakpointsFilter>>,
+    pub completion_trigger_characters: Option<Vec<String>>,
+    pub additional_module_columns: Option<Vec<ColumnDescriptor>>,
+    pub supported_checksum_algorithms: Option<Vec<String>>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Checksum {
+    pub algorithm: String,
+    pub checksum: String,
+}
+
+#[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Source {
+    pub name: Option<String>,
+    pub path: Option<PathBuf>,
+    pub source_reference: Option<usize>,
+    pub presentation_hint: Option<String>,
+    pub origin: Option<String>,
+    pub sources: Option<Vec<Source>>,
+    pub adapter_data: Option<Value>,
+    pub checksums: Option<Vec<Checksum>>,
+}
+
+#[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct SourceBreakpoint {
+    pub line: usize,
+    pub column: Option<usize>,
+    pub condition: Option<String>,
+    pub hit_condition: Option<String>,
+    pub log_message: Option<String>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Breakpoint {
+    pub id: Option<usize>,
+    pub verified: bool,
+    pub message: Option<String>,
+    pub source: Option<Source>,
+    pub line: Option<usize>,
+    pub column: Option<usize>,
+    pub end_line: Option<usize>,
+    pub end_column: Option<usize>,
+    pub instruction_reference: Option<String>,
+    pub offset: Option<usize>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct StackFrameFormat {
+    pub parameters: Option<bool>,
+    pub parameter_types: Option<bool>,
+    pub parameter_names: Option<bool>,
+    pub parameter_values: Option<bool>,
+    pub line: Option<bool>,
+    pub module: Option<bool>,
+    pub include_all: Option<bool>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct StackFrame {
+    pub id: usize,
+    pub name: String,
+    pub source: Option<Source>,
+    pub line: usize,
+    pub column: usize,
+    pub end_line: Option<usize>,
+    pub end_column: Option<usize>,
+    pub can_restart: Option<bool>,
+    pub instruction_pointer_reference: Option<String>,
+    pub module_id: Option<Value>,
+    pub presentation_hint: Option<String>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Thread {
+    pub id: ThreadId,
+    pub name: String,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Scope {
+    pub name: String,
+    pub presentation_hint: Option<String>,
+    pub variables_reference: usize,
+    pub named_variables: Option<usize>,
+    pub indexed_variables: Option<usize>,
+    pub expensive: bool,
+    pub source: Option<Source>,
+    pub line: Option<usize>,
+    pub column: Option<usize>,
+    pub end_line: Option<usize>,
+    pub end_column: Option<usize>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ValueFormat {
+    pub hex: Option<bool>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct VariablePresentationHint {
+    pub kind: Option<String>,
+    pub attributes: Option<Vec<String>>,
+    pub visibility: Option<String>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Variable {
+    pub name: String,
+    pub value: String,
+    #[serde(rename = "type")]
+    pub ty: Option<String>,
+    pub presentation_hint: Option<VariablePresentationHint>,
+    pub evaluate_name: Option<String>,
+    pub variables_reference: usize,
+    pub named_variables: Option<usize>,
+    pub indexed_variables: Option<usize>,
+    pub memory_reference: Option<String>,
+}
+
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Module {
+    pub id: String, // TODO: || number
+    pub name: String,
+    pub path: Option<PathBuf>,
+    pub is_optimized: Option<bool>,
+    pub is_user_code: Option<bool>,
+    pub version: Option<String>,
+    pub symbol_status: Option<String>,
+    pub symbol_file_path: Option<String>,
+    pub date_time_stamp: Option<String>,
+    pub address_range: Option<String>,
+}
+
+pub mod requests {
+    use super::*;
+    #[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct InitializeArguments {
+        #[serde(rename = "clientID")]
+        pub client_id: Option<String>,
+        pub client_name: Option<String>,
+        #[serde(rename = "adapterID")]
+        pub adapter_id: String,
+        pub locale: Option<String>,
+        #[serde(rename = "linesStartAt1")]
+        pub lines_start_at_one: Option<bool>,
+        #[serde(rename = "columnsStartAt1")]
+        pub columns_start_at_one: Option<bool>,
+        pub path_format: Option<String>,
+        pub supports_variable_type: Option<bool>,
+        pub supports_variable_paging: Option<bool>,
+        pub supports_run_in_terminal_request: Option<bool>,
+        pub supports_memory_references: Option<bool>,
+        pub supports_progress_reporting: Option<bool>,
+        pub supports_invalidated_event: Option<bool>,
+    }
+
+    #[derive(Debug)]
+    pub enum Initialize {}
+
+    impl Request for Initialize {
+        type Arguments = InitializeArguments;
+        type Result = DebuggerCapabilities;
+        const COMMAND: &'static str = "initialize";
+    }
+
+    #[derive(Debug)]
+    pub enum Launch {}
+
+    impl Request for Launch {
+        type Arguments = Value;
+        type Result = Value;
+        const COMMAND: &'static str = "launch";
+    }
+
+    #[derive(Debug)]
+    pub enum Attach {}
+
+    impl Request for Attach {
+        type Arguments = Value;
+        type Result = Value;
+        const COMMAND: &'static str = "attach";
+    }
+
+    #[derive(Debug)]
+    pub enum Disconnect {}
+
+    impl Request for Disconnect {
+        type Arguments = ();
+        type Result = ();
+        const COMMAND: &'static str = "disconnect";
+    }
+
+    #[derive(Debug)]
+    pub enum ConfigurationDone {}
+
+    impl Request for ConfigurationDone {
+        type Arguments = ();
+        type Result = ();
+        const COMMAND: &'static str = "configurationDone";
+    }
+
+    #[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct SetBreakpointsArguments {
+        pub source: Source,
+        pub breakpoints: Option<Vec<SourceBreakpoint>>,
+        // lines is deprecated
+        pub source_modified: Option<bool>,
+    }
+
+    #[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct SetBreakpointsResponse {
+        pub breakpoints: Option<Vec<Breakpoint>>,
+    }
+
+    #[derive(Debug)]
+    pub enum SetBreakpoints {}
+
+    impl Request for SetBreakpoints {
+        type Arguments = SetBreakpointsArguments;
+        type Result = SetBreakpointsResponse;
+        const COMMAND: &'static str = "setBreakpoints";
+    }
+
+    #[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct ContinueArguments {
+        pub thread_id: ThreadId,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct ContinueResponse {
+        pub all_threads_continued: Option<bool>,
+    }
+
+    #[derive(Debug)]
+    pub enum Continue {}
+
+    impl Request for Continue {
+        type Arguments = ContinueArguments;
+        type Result = ContinueResponse;
+        const COMMAND: &'static str = "continue";
+    }
+
+    #[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct StackTraceArguments {
+        pub thread_id: ThreadId,
+        pub start_frame: Option<usize>,
+        pub levels: Option<usize>,
+        pub format: Option<StackFrameFormat>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct StackTraceResponse {
+        pub total_frames: Option<usize>,
+        pub stack_frames: Vec<StackFrame>,
+    }
+
+    #[derive(Debug)]
+    pub enum StackTrace {}
+
+    impl Request for StackTrace {
+        type Arguments = StackTraceArguments;
+        type Result = StackTraceResponse;
+        const COMMAND: &'static str = "stackTrace";
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct ThreadsResponse {
+        pub threads: Vec<Thread>,
+    }
+
+    #[derive(Debug)]
+    pub enum Threads {}
+
+    impl Request for Threads {
+        type Arguments = ();
+        type Result = ThreadsResponse;
+        const COMMAND: &'static str = "threads";
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct ScopesArguments {
+        pub frame_id: usize,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct ScopesResponse {
+        pub scopes: Vec<Scope>,
+    }
+
+    #[derive(Debug)]
+    pub enum Scopes {}
+
+    impl Request for Scopes {
+        type Arguments = ScopesArguments;
+        type Result = ScopesResponse;
+        const COMMAND: &'static str = "scopes";
+    }
+
+    #[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct VariablesArguments {
+        pub variables_reference: usize,
+        pub filter: Option<String>,
+        pub start: Option<usize>,
+        pub count: Option<usize>,
+        pub format: Option<ValueFormat>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct VariablesResponse {
+        pub variables: Vec<Variable>,
+    }
+
+    #[derive(Debug)]
+    pub enum Variables {}
+
+    impl Request for Variables {
+        type Arguments = VariablesArguments;
+        type Result = VariablesResponse;
+        const COMMAND: &'static str = "variables";
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct StepInArguments {
+        pub thread_id: ThreadId,
+        pub target_id: Option<usize>,
+        pub granularity: Option<String>,
+    }
+
+    #[derive(Debug)]
+    pub enum StepIn {}
+
+    impl Request for StepIn {
+        type Arguments = StepInArguments;
+        type Result = ();
+        const COMMAND: &'static str = "stepIn";
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct StepOutArguments {
+        pub thread_id: ThreadId,
+        pub granularity: Option<String>,
+    }
+
+    #[derive(Debug)]
+    pub enum StepOut {}
+
+    impl Request for StepOut {
+        type Arguments = StepOutArguments;
+        type Result = ();
+        const COMMAND: &'static str = "stepOut";
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct NextArguments {
+        pub thread_id: ThreadId,
+        pub granularity: Option<String>,
+    }
+
+    #[derive(Debug)]
+    pub enum Next {}
+
+    impl Request for Next {
+        type Arguments = NextArguments;
+        type Result = ();
+        const COMMAND: &'static str = "next";
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct PauseArguments {
+        pub thread_id: ThreadId,
+    }
+
+    #[derive(Debug)]
+    pub enum Pause {}
+
+    impl Request for Pause {
+        type Arguments = PauseArguments;
+        type Result = ();
+        const COMMAND: &'static str = "pause";
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct EvaluateArguments {
+        pub expression: String,
+        pub frame_id: Option<usize>,
+        pub context: Option<String>,
+        pub format: Option<ValueFormat>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct EvaluateResponse {
+        pub result: String,
+        #[serde(rename = "type")]
+        pub ty: Option<String>,
+        pub presentation_hint: Option<VariablePresentationHint>,
+        pub variables_reference: usize,
+        pub named_variables: Option<usize>,
+        pub indexed_variables: Option<usize>,
+        pub memory_reference: Option<String>,
+    }
+
+    #[derive(Debug)]
+    pub enum Evaluate {}
+
+    impl Request for Evaluate {
+        type Arguments = EvaluateArguments;
+        type Result = EvaluateResponse;
+        const COMMAND: &'static str = "evaluate";
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct SetExceptionBreakpointsArguments {
+        pub filters: Vec<String>,
+        // pub filterOptions: Option<Vec<ExceptionFilterOptions>>, // needs capability
+        // pub exceptionOptions: Option<Vec<ExceptionOptions>>, // needs capability
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct SetExceptionBreakpointsResponse {
+        pub breakpoints: Option<Vec<Breakpoint>>,
+    }
+
+    #[derive(Debug)]
+    pub enum SetExceptionBreakpoints {}
+
+    impl Request for SetExceptionBreakpoints {
+        type Arguments = SetExceptionBreakpointsArguments;
+        type Result = SetExceptionBreakpointsResponse;
+        const COMMAND: &'static str = "setExceptionBreakpoints";
+    }
+
+    // Reverse Requests
+
+    #[derive(Debug, Default, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct RunInTerminalResponse {
+        pub process_id: Option<u32>,
+        pub shell_process_id: Option<u32>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct RunInTerminalArguments {
+        pub kind: Option<String>,
+        pub title: Option<String>,
+        pub cwd: Option<String>,
+        pub args: Vec<String>,
+        pub env: Option<HashMap<String, Option<String>>>,
+    }
+
+    #[derive(Debug)]
+    pub enum RunInTerminal {}
+
+    impl Request for RunInTerminal {
+        type Arguments = RunInTerminalArguments;
+        type Result = RunInTerminalResponse;
+        const COMMAND: &'static str = "runInTerminal";
+    }
+}
+
+// Events
+
+pub mod events {
+    use super::*;
+
+    #[derive(Debug, Clone, Serialize, Deserialize)]
+    #[serde(rename_all = "camelCase")]
+    #[serde(tag = "event", content = "body")]
+    // seq is omitted as unused and is not sent by some implementations
+    pub enum Event {
+        Initialized,
+        Stopped(Stopped),
+        Continued(Continued),
+        Exited(Exited),
+        Terminated(Option<Terminated>),
+        Thread(Thread),
+        Output(Output),
+        Breakpoint(Breakpoint),
+        Module(Module),
+        LoadedSource(LoadedSource),
+        Process(Process),
+        Capabilities(Capabilities),
+        // ProgressStart(),
+        // ProgressUpdate(),
+        // ProgressEnd(),
+        // Invalidated(),
+        Memory(Memory),
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Stopped {
+        pub reason: String,
+        pub description: Option<String>,
+        pub thread_id: Option<ThreadId>,
+        pub preserve_focus_hint: Option<bool>,
+        pub text: Option<String>,
+        pub all_threads_stopped: Option<bool>,
+        pub hit_breakpoint_ids: Option<Vec<usize>>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Continued {
+        pub thread_id: ThreadId,
+        pub all_threads_continued: Option<bool>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Exited {
+        pub exit_code: usize,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Terminated {
+        pub restart: Option<Value>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Thread {
+        pub reason: String,
+        pub thread_id: ThreadId,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Output {
+        pub output: String,
+        pub category: Option<String>,
+        pub group: Option<String>,
+        pub line: Option<usize>,
+        pub column: Option<usize>,
+        pub variables_reference: Option<usize>,
+        pub source: Option<Source>,
+        pub data: Option<Value>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Breakpoint {
+        pub reason: String,
+        pub breakpoint: super::Breakpoint,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Module {
+        pub reason: String,
+        pub module: super::Module,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct LoadedSource {
+        pub reason: String,
+        pub source: super::Source,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Process {
+        pub name: String,
+        pub system_process_id: Option<usize>,
+        pub is_local_process: Option<bool>,
+        pub start_method: Option<String>, // TODO: use enum
+        pub pointer_size: Option<usize>,
+    }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Capabilities {
+        pub capabilities: super::DebuggerCapabilities,
+    }
+
+    // #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    // #[serde(rename_all = "camelCase")]
+    // pub struct Invalidated {
+    // pub areas: Vec<InvalidatedArea>,
+    // pub thread_id: Option<ThreadId>,
+    // pub stack_frame_id: Option<usize>,
+    // }
+
+    #[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Memory {
+        pub memory_reference: String,
+        pub offset: usize,
+        pub count: usize,
+    }
+}

helix-graphics/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "helix-graphics"
+version = "0.6.0"
+authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
+edition = "2021"
+license = "MPL-2.0"
+repository = "https://github.com/helix-editor/helix"
+homepage = "https://helix-editor.com"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/
+
+[features]
+term = ["crossterm"]
+
+[dependencies]
+bitflags = "1.3"
+serde = { version = "1.0", features = ["derive"] }
+
+crossterm = { version = "0.23", optional = true }
+
+# TODO: graphics.rs tests rely on this, but we should remove that
+# [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
+# helix-tui = { path = "../helix-tui" }

helix-graphics/src/lib.rs

@@ -1,10 +1,12 @@
 use bitflags::bitflags;
+use serde::{Deserialize, Serialize};
 use std::{
     cmp::{max, min},
     str::FromStr,
 };
 
-#[derive(Debug, Clone, Copy, PartialEq)]
+#[derive(Debug, Clone, Copy, PartialEq, Deserialize, Serialize)]
+#[serde(rename_all = "lowercase")]
 /// UNSTABLE
 pub enum CursorKind {
     /// █
@@ -17,6 +19,12 @@ pub enum CursorKind {
     Hidden,
 }
 
+impl Default for CursorKind {
+    fn default() -> Self {
+        Self::Block
+    }
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct Margin {
     pub vertical: u16,
@@ -323,7 +331,7 @@ impl FromStr for Modifier {
 /// ];
 /// let mut buffer = Buffer::empty(Rect::new(0, 0, 1, 1));
 /// for style in &styles {
-///   buffer.get_mut(0, 0).set_style(*style);
+///   buffer[(0, 0)].set_style(*style);
 /// }
 /// assert_eq!(
 ///     Style {
@@ -332,7 +340,7 @@ impl FromStr for Modifier {
 ///         add_modifier: Modifier::BOLD,
 ///         sub_modifier: Modifier::empty(),
 ///     },
-///     buffer.get(0, 0).style(),
+///     buffer[(0, 0)].style(),
 /// );
 /// ```
 ///
@@ -348,7 +356,7 @@ impl FromStr for Modifier {
 /// ];
 /// let mut buffer = Buffer::empty(Rect::new(0, 0, 1, 1));
 /// for style in &styles {
-///   buffer.get_mut(0, 0).set_style(*style);
+///   buffer[(0, 0)].set_style(*style);
 /// }
 /// assert_eq!(
 ///     Style {
@@ -357,7 +365,7 @@ impl FromStr for Modifier {
 ///         add_modifier: Modifier::empty(),
 ///         sub_modifier: Modifier::empty(),
 ///     },
-///     buffer.get(0, 0).style(),
+///     buffer[(0, 0)].style(),
 /// );
 /// ```
 #[derive(Debug, Clone, Copy, PartialEq)]

helix-loader/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+name = "helix-loader"
+version = "0.6.0"
+description = "A post-modern text editor."
+authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
+edition = "2021"
+license = "MPL-2.0"
+categories = ["editor"]
+repository = "https://github.com/helix-editor/helix"
+homepage = "https://helix-editor.com"
+
+[dependencies]
+anyhow = "1"
+serde = { version = "1.0", features = ["derive"] }
+toml = "0.5"
+etcetera = "0.3"
+tree-sitter = "0.20"
+once_cell = "1.9"
+
+log = "0.4"
+
+# TODO: these two should be on !wasm32 only
+
+# cloning/compiling tree-sitter grammars
+cc = { version = "1" }
+threadpool = { version = "1.0" }
+
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+libloading = "0.7"
+

helix-loader/build.rs

@@ -0,0 +1,6 @@
+fn main() {
+    println!(
+        "cargo:rustc-env=BUILD_TARGET={}",
+        std::env::var("TARGET").unwrap()
+    );
+}

helix-loader/src/config.rs

@@ -0,0 +1,26 @@
+/// Default built-in languages.toml.
+pub fn default_lang_config() -> toml::Value {
+    toml::from_slice(include_bytes!("../../languages.toml"))
+        .expect("Could not parse built-in languages.toml to valid toml")
+}
+
+/// User configured languages.toml file, merged with the default config.
+pub fn user_lang_config() -> Result<toml::Value, toml::de::Error> {
+    let config = crate::local_config_dirs()
+        .into_iter()
+        .chain([crate::config_dir()].into_iter())
+        .map(|path| path.join("languages.toml"))
+        .filter_map(|file| {
+            std::fs::read(&file)
+                .map(|config| toml::from_slice(&config))
+                .ok()
+        })
+        .collect::<Result<Vec<_>, _>>()?
+        .into_iter()
+        .chain([default_lang_config()].into_iter())
+        .fold(toml::Value::Table(toml::value::Table::default()), |a, b| {
+            crate::merge_toml_values(b, a, true)
+        });
+
+    Ok(config)
+}

helix-loader/src/grammar.rs

@@ -0,0 +1,402 @@
+use anyhow::{anyhow, Context, Result};
+use serde::{Deserialize, Serialize};
+use std::fs;
+use std::time::SystemTime;
+use std::{
+    collections::HashSet,
+    path::{Path, PathBuf},
+    process::Command,
+    sync::mpsc::channel,
+};
+use tree_sitter::Language;
+
+#[cfg(unix)]
+const DYLIB_EXTENSION: &str = "so";
+
+#[cfg(windows)]
+const DYLIB_EXTENSION: &str = "dll";
+
+#[cfg(target_arch = "wasm32")]
+const DYLIB_EXTENSION: &str = "wasm";
+
+#[derive(Debug, Serialize, Deserialize)]
+struct Configuration {
+    #[serde(rename = "use-grammars")]
+    pub grammar_selection: Option<GrammarSelection>,
+    pub grammar: Vec<GrammarConfiguration>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase", untagged)]
+pub enum GrammarSelection {
+    Only { only: HashSet<String> },
+    Except { except: HashSet<String> },
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct GrammarConfiguration {
+    #[serde(rename = "name")]
+    pub grammar_id: String,
+    pub source: GrammarSource,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase", untagged)]
+pub enum GrammarSource {
+    Local {
+        path: String,
+    },
+    Git {
+        #[serde(rename = "git")]
+        remote: String,
+        #[serde(rename = "rev")]
+        revision: String,
+        subpath: Option<String>,
+    },
+}
+
+const BUILD_TARGET: &str = env!("BUILD_TARGET");
+const REMOTE_NAME: &str = "origin";
+
+#[cfg(target_arch = "wasm32")]
+pub fn get_language(name: &str) -> Result<Language> {
+    unimplemented!()
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+pub fn get_language(name: &str) -> Result<Language> {
+    use libloading::{Library, Symbol};
+    let name = name.to_ascii_lowercase();
+    let mut library_path = crate::runtime_dir().join("grammars").join(&name);
+    library_path.set_extension(DYLIB_EXTENSION);
+
+    let library = unsafe { Library::new(&library_path) }
+        .with_context(|| format!("Error opening dynamic library {:?}", library_path))?;
+    let language_fn_name = format!("tree_sitter_{}", name.replace('-', "_"));
+    let language = unsafe {
+        let language_fn: Symbol<unsafe extern "C" fn() -> Language> = library
+            .get(language_fn_name.as_bytes())
+            .with_context(|| format!("Failed to load symbol {}", language_fn_name))?;
+        language_fn()
+    };
+    std::mem::forget(library);
+    Ok(language)
+}
+
+pub fn fetch_grammars() -> Result<()> {
+    // We do not need to fetch local grammars.
+    let mut grammars = get_grammar_configs()?;
+    grammars.retain(|grammar| !matches!(grammar.source, GrammarSource::Local { .. }));
+
+    run_parallel(grammars, fetch_grammar, "fetch")
+}
+
+pub fn build_grammars() -> Result<()> {
+    run_parallel(get_grammar_configs()?, build_grammar, "build")
+}
+
+// Returns the set of grammar configurations the user requests.
+// Grammars are configured in the default and user `languages.toml` and are
+// merged. The `grammar_selection` key of the config is then used to filter
+// down all grammars into a subset of the user's choosing.
+fn get_grammar_configs() -> Result<Vec<GrammarConfiguration>> {
+    let config: Configuration = crate::config::user_lang_config()
+        .context("Could not parse languages.toml")?
+        .try_into()?;
+
+    let grammars = match config.grammar_selection {
+        Some(GrammarSelection::Only { only: selections }) => config
+            .grammar
+            .into_iter()
+            .filter(|grammar| selections.contains(&grammar.grammar_id))
+            .collect(),
+        Some(GrammarSelection::Except { except: rejections }) => config
+            .grammar
+            .into_iter()
+            .filter(|grammar| !rejections.contains(&grammar.grammar_id))
+            .collect(),
+        None => config.grammar,
+    };
+
+    Ok(grammars)
+}
+
+fn run_parallel<F>(grammars: Vec<GrammarConfiguration>, job: F, action: &'static str) -> Result<()>
+where
+    F: Fn(GrammarConfiguration) -> Result<()> + std::marker::Send + 'static + Copy,
+{
+    let pool = threadpool::Builder::new().build();
+    let (tx, rx) = channel();
+
+    for grammar in grammars {
+        let tx = tx.clone();
+
+        pool.execute(move || {
+            tx.send(job(grammar)).unwrap();
+        });
+    }
+
+    drop(tx);
+
+    // TODO: print all failures instead of the first one found.
+    rx.iter()
+        .find(|result| result.is_err())
+        .map(|err| err.with_context(|| format!("Failed to {} some grammar(s)", action)))
+        .unwrap_or(Ok(()))
+}
+
+fn fetch_grammar(grammar: GrammarConfiguration) -> Result<()> {
+    if let GrammarSource::Git {
+        remote, revision, ..
+    } = grammar.source
+    {
+        let grammar_dir = crate::runtime_dir()
+            .join("grammars")
+            .join("sources")
+            .join(&grammar.grammar_id);
+
+        fs::create_dir_all(&grammar_dir).context(format!(
+            "Could not create grammar directory {:?}",
+            grammar_dir
+        ))?;
+
+        // create the grammar dir contains a git directory
+        if !grammar_dir.join(".git").is_dir() {
+            git(&grammar_dir, ["init"])?;
+        }
+
+        // ensure the remote matches the configured remote
+        if get_remote_url(&grammar_dir).map_or(true, |s| s != remote) {
+            set_remote(&grammar_dir, &remote)?;
+        }
+
+        // ensure the revision matches the configured revision
+        if get_revision(&grammar_dir).map_or(true, |s| s != revision) {
+            // Fetch the exact revision from the remote.
+            // Supported by server-side git since v2.5.0 (July 2015),
+            // enabled by default on major git hosts.
+            git(
+                &grammar_dir,
+                ["fetch", "--depth", "1", REMOTE_NAME, &revision],
+            )?;
+            git(&grammar_dir, ["checkout", &revision])?;
+
+            println!(
+                "Grammar '{}' checked out at '{}'.",
+                grammar.grammar_id, revision
+            );
+        } else {
+            println!("Grammar '{}' is already up to date.", grammar.grammar_id);
+        }
+    }
+
+    Ok(())
+}
+
+// Sets the remote for a repository to the given URL, creating the remote if
+// it does not yet exist.
+fn set_remote(repository_dir: &Path, remote_url: &str) -> Result<String> {
+    git(
+        repository_dir,
+        ["remote", "set-url", REMOTE_NAME, remote_url],
+    )
+    .or_else(|_| git(repository_dir, ["remote", "add", REMOTE_NAME, remote_url]))
+}
+
+fn get_remote_url(repository_dir: &Path) -> Option<String> {
+    git(repository_dir, ["remote", "get-url", REMOTE_NAME]).ok()
+}
+
+fn get_revision(repository_dir: &Path) -> Option<String> {
+    git(repository_dir, ["rev-parse", "HEAD"]).ok()
+}
+
+// A wrapper around 'git' commands which returns stdout in success and a
+// helpful error message showing the command, stdout, and stderr in error.
+fn git<I, S>(repository_dir: &Path, args: I) -> Result<String>
+where
+    I: IntoIterator<Item = S>,
+    S: AsRef<std::ffi::OsStr>,
+{
+    let output = Command::new("git")
+        .args(args)
+        .current_dir(repository_dir)
+        .output()?;
+
+    if output.status.success() {
+        Ok(String::from_utf8_lossy(&output.stdout)
+            .trim_end()
+            .to_owned())
+    } else {
+        // TODO: figure out how to display the git command using `args`
+        Err(anyhow!(
+            "Git command failed.\nStdout: {}\nStderr: {}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr),
+        ))
+    }
+}
+
+fn build_grammar(grammar: GrammarConfiguration) -> Result<()> {
+    let grammar_dir = if let GrammarSource::Local { path } = &grammar.source {
+        PathBuf::from(&path)
+    } else {
+        crate::runtime_dir()
+            .join("grammars")
+            .join("sources")
+            .join(&grammar.grammar_id)
+    };
+
+    let grammar_dir_entries = grammar_dir.read_dir().with_context(|| {
+        format!(
+            "Failed to read directory {:?}. Did you use 'hx --grammar fetch'?",
+            grammar_dir
+        )
+    })?;
+
+    if grammar_dir_entries.count() == 0 {
+        return Err(anyhow!(
+            "Directory {:?} is empty. Did you use 'hx --grammar fetch'?",
+            grammar_dir
+        ));
+    };
+
+    let path = match &grammar.source {
+        GrammarSource::Git {
+            subpath: Some(subpath),
+            ..
+        } => grammar_dir.join(subpath),
+        _ => grammar_dir,
+    }
+    .join("src");
+
+    build_tree_sitter_library(&path, grammar)
+}
+
+fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) -> Result<()> {
+    let header_path = src_path;
+    let parser_path = src_path.join("parser.c");
+    let mut scanner_path = src_path.join("scanner.c");
+
+    let scanner_path = if scanner_path.exists() {
+        Some(scanner_path)
+    } else {
+        scanner_path.set_extension("cc");
+        if scanner_path.exists() {
+            Some(scanner_path)
+        } else {
+            None
+        }
+    };
+    let parser_lib_path = crate::runtime_dir().join("grammars");
+    let mut library_path = parser_lib_path.join(&grammar.grammar_id);
+    library_path.set_extension(DYLIB_EXTENSION);
+
+    let recompile = needs_recompile(&library_path, &parser_path, &scanner_path)
+        .context("Failed to compare source and binary timestamps")?;
+
+    if !recompile {
+        println!("Grammar '{}' is already built.", grammar.grammar_id);
+        return Ok(());
+    }
+
+    println!("Building grammar '{}'", grammar.grammar_id);
+
+    let mut config = cc::Build::new();
+    config
+        .cpp(true)
+        .opt_level(3)
+        .cargo_metadata(false)
+        .host(BUILD_TARGET)
+        .target(BUILD_TARGET);
+    let compiler = config.get_compiler();
+    let mut command = Command::new(compiler.path());
+    command.current_dir(src_path);
+    for (key, value) in compiler.env() {
+        command.env(key, value);
+    }
+
+    if cfg!(windows) {
+        command
+            .args(&["/nologo", "/LD", "/I"])
+            .arg(header_path)
+            .arg("/Od")
+            .arg("/utf-8");
+        if let Some(scanner_path) = scanner_path.as_ref() {
+            command.arg(scanner_path);
+        }
+
+        command
+            .arg(parser_path)
+            .arg("/link")
+            .arg(format!("/out:{}", library_path.to_str().unwrap()));
+    } else {
+        command
+            .arg("-shared")
+            .arg("-fPIC")
+            .arg("-fno-exceptions")
+            .arg("-g")
+            .arg("-I")
+            .arg(header_path)
+            .arg("-o")
+            .arg(&library_path)
+            .arg("-O3");
+        if let Some(scanner_path) = scanner_path.as_ref() {
+            if scanner_path.extension() == Some("c".as_ref()) {
+                command.arg("-xc").arg("-std=c99").arg(scanner_path);
+            } else {
+                command.arg(scanner_path);
+            }
+        }
+        command.arg("-xc").arg(parser_path);
+        if cfg!(all(unix, not(target_os = "macos"))) {
+            command.arg("-Wl,-z,relro,-z,now");
+        }
+    }
+
+    let output = command.output().context("Failed to execute C compiler")?;
+    if !output.status.success() {
+        return Err(anyhow!(
+            "Parser compilation failed.\nStdout: {}\nStderr: {}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr)
+        ));
+    }
+
+    Ok(())
+}
+
+fn needs_recompile(
+    lib_path: &Path,
+    parser_c_path: &Path,
+    scanner_path: &Option<PathBuf>,
+) -> Result<bool> {
+    if !lib_path.exists() {
+        return Ok(true);
+    }
+    let lib_mtime = mtime(lib_path)?;
+    if mtime(parser_c_path)? > lib_mtime {
+        return Ok(true);
+    }
+    if let Some(scanner_path) = scanner_path {
+        if mtime(scanner_path)? > lib_mtime {
+            return Ok(true);
+        }
+    }
+    Ok(false)
+}
+
+fn mtime(path: &Path) -> Result<SystemTime> {
+    Ok(fs::metadata(path)?.modified()?)
+}
+
+/// Gives the contents of a file from a language's `runtime/queries/<lang>`
+/// directory
+pub fn load_runtime_file(language: &str, filename: &str) -> Result<String, std::io::Error> {
+    let path = crate::RUNTIME_DIR
+        .join("queries")
+        .join(language)
+        .join(filename);
+    std::fs::read_to_string(&path)
+}

helix-loader/src/lib.rs

@@ -0,0 +1,237 @@
+pub mod config;
+pub mod grammar;
+
+use etcetera::base_strategy::{choose_base_strategy, BaseStrategy};
+
+pub static RUNTIME_DIR: once_cell::sync::Lazy<std::path::PathBuf> =
+    once_cell::sync::Lazy::new(runtime_dir);
+
+pub fn runtime_dir() -> std::path::PathBuf {
+    if let Ok(dir) = std::env::var("HELIX_RUNTIME") {
+        return dir.into();
+    }
+
+    const RT_DIR: &str = "runtime";
+    let conf_dir = config_dir().join(RT_DIR);
+    if conf_dir.exists() {
+        return conf_dir;
+    }
+
+    if let Ok(dir) = std::env::var("CARGO_MANIFEST_DIR") {
+        // this is the directory of the crate being run by cargo, we need the workspace path so we take the parent
+        return std::path::PathBuf::from(dir).parent().unwrap().join(RT_DIR);
+    }
+
+    // fallback to location of the executable being run
+    std::env::current_exe()
+        .ok()
+        .and_then(|path| path.parent().map(|path| path.to_path_buf().join(RT_DIR)))
+        .unwrap()
+}
+
+pub fn config_dir() -> std::path::PathBuf {
+    // TODO: allow env var override
+    let strategy = choose_base_strategy().expect("Unable to find the config directory!");
+    let mut path = strategy.config_dir();
+    path.push("helix");
+    path
+}
+
+pub fn local_config_dirs() -> Vec<std::path::PathBuf> {
+    let directories = find_root_impl(None, &[".helix".to_string()])
+        .into_iter()
+        .map(|path| path.join(".helix"))
+        .collect();
+    log::debug!("Located configuration folders: {:?}", directories);
+    directories
+}
+
+pub fn cache_dir() -> std::path::PathBuf {
+    // TODO: allow env var override
+    let strategy = choose_base_strategy().expect("Unable to find the config directory!");
+    let mut path = strategy.cache_dir();
+    path.push("helix");
+    path
+}
+
+pub fn config_file() -> std::path::PathBuf {
+    config_dir().join("config.toml")
+}
+
+pub fn lang_config_file() -> std::path::PathBuf {
+    config_dir().join("languages.toml")
+}
+
+pub fn log_file() -> std::path::PathBuf {
+    cache_dir().join("helix.log")
+}
+
+pub fn find_root_impl(root: Option<&str>, root_markers: &[String]) -> Vec<std::path::PathBuf> {
+    let current_dir = std::env::current_dir().expect("unable to determine current directory");
+    let mut directories = Vec::new();
+
+    let root = match root {
+        Some(root) => {
+            let root = std::path::Path::new(root);
+            if root.is_absolute() {
+                root.to_path_buf()
+            } else {
+                current_dir.join(root)
+            }
+        }
+        None => current_dir,
+    };
+
+    for ancestor in root.ancestors() {
+        // don't go higher than repo
+        if ancestor.join(".git").is_dir() {
+            // Use workspace if detected from marker
+            directories.push(ancestor.to_path_buf());
+            break;
+        } else if root_markers
+            .iter()
+            .any(|marker| ancestor.join(marker).exists())
+        {
+            directories.push(ancestor.to_path_buf());
+        }
+    }
+    directories
+}
+
+/// Merge two TOML documents, merging values from `right` onto `left`
+///
+/// When an array exists in both `left` and `right`, `right`'s array is
+/// used. When a table exists in both `left` and `right`, the merged table
+/// consists of all keys in `left`'s table unioned with all keys in `right`
+/// with the values of `right` being merged recursively onto values of
+/// `left`.
+///
+/// `merge_toplevel_arrays` controls whether a top-level array in the TOML
+/// document is merged instead of overridden. This is useful for TOML
+/// documents that use a top-level array of values like the `languages.toml`,
+/// where one usually wants to override or add to the array instead of
+/// replacing it altogether.
+pub fn merge_toml_values(
+    left: toml::Value,
+    right: toml::Value,
+    merge_toplevel_arrays: bool,
+) -> toml::Value {
+    use toml::Value;
+
+    fn get_name(v: &Value) -> Option<&str> {
+        v.get("name").and_then(Value::as_str)
+    }
+
+    match (left, right) {
+        (Value::Array(mut left_items), Value::Array(right_items)) => {
+            // The top-level arrays should be merged but nested arrays should
+            // act as overrides. For the `languages.toml` config, this means
+            // that you can specify a sub-set of languages in an overriding
+            // `languages.toml` but that nested arrays like Language Server
+            // arguments are replaced instead of merged.
+            if merge_toplevel_arrays {
+                left_items.reserve(right_items.len());
+                for rvalue in right_items {
+                    let lvalue = get_name(&rvalue)
+                        .and_then(|rname| {
+                            left_items.iter().position(|v| get_name(v) == Some(rname))
+                        })
+                        .map(|lpos| left_items.remove(lpos));
+                    let mvalue = match lvalue {
+                        Some(lvalue) => merge_toml_values(lvalue, rvalue, false),
+                        None => rvalue,
+                    };
+                    left_items.push(mvalue);
+                }
+                Value::Array(left_items)
+            } else {
+                Value::Array(right_items)
+            }
+        }
+        (Value::Table(mut left_map), Value::Table(right_map)) => {
+            for (rname, rvalue) in right_map {
+                match left_map.remove(&rname) {
+                    Some(lvalue) => {
+                        let merged_value = merge_toml_values(lvalue, rvalue, merge_toplevel_arrays);
+                        left_map.insert(rname, merged_value);
+                    }
+                    None => {
+                        left_map.insert(rname, rvalue);
+                    }
+                }
+            }
+            Value::Table(left_map)
+        }
+        // Catch everything else we didn't handle, and use the right value
+        (_, value) => value,
+    }
+}
+
+#[cfg(test)]
+mod merge_toml_tests {
+    use super::merge_toml_values;
+    use toml::Value;
+
+    #[test]
+    fn language_toml_map_merges() {
+        const USER: &str = r#"
+        [[language]]
+        name = "nix"
+        test = "bbb"
+        indent = { tab-width = 4, unit = "    ", test = "aaa" }
+        "#;
+
+        let base: Value = toml::from_slice(include_bytes!("../../languages.toml"))
+            .expect("Couldn't parse built-in languages config");
+        let user: Value = toml::from_str(USER).unwrap();
+
+        let merged = merge_toml_values(base, user, true);
+        let languages = merged.get("language").unwrap().as_array().unwrap();
+        let nix = languages
+            .iter()
+            .find(|v| v.get("name").unwrap().as_str().unwrap() == "nix")
+            .unwrap();
+        let nix_indent = nix.get("indent").unwrap();
+
+        // We changed tab-width and unit in indent so check them if they are the new values
+        assert_eq!(
+            nix_indent.get("tab-width").unwrap().as_integer().unwrap(),
+            4
+        );
+        assert_eq!(nix_indent.get("unit").unwrap().as_str().unwrap(), "    ");
+        // We added a new keys, so check them
+        assert_eq!(nix.get("test").unwrap().as_str().unwrap(), "bbb");
+        assert_eq!(nix_indent.get("test").unwrap().as_str().unwrap(), "aaa");
+        // We didn't change comment-token so it should be same
+        assert_eq!(nix.get("comment-token").unwrap().as_str().unwrap(), "#");
+    }
+
+    #[test]
+    fn language_toml_nested_array_merges() {
+        const USER: &str = r#"
+        [[language]]
+        name = "typescript"
+        language-server = { command = "deno", args = ["lsp"] }
+        "#;
+
+        let base: Value = toml::from_slice(include_bytes!("../../languages.toml"))
+            .expect("Couldn't parse built-in languages config");
+        let user: Value = toml::from_str(USER).unwrap();
+
+        let merged = merge_toml_values(base, user, true);
+        let languages = merged.get("language").unwrap().as_array().unwrap();
+        let ts = languages
+            .iter()
+            .find(|v| v.get("name").unwrap().as_str().unwrap() == "typescript")
+            .unwrap();
+        assert_eq!(
+            ts.get("language-server")
+                .unwrap()
+                .get("args")
+                .unwrap()
+                .as_array()
+                .unwrap(),
+            &vec![Value::String("lsp".into())]
+        )
+    }
+}

helix-lsp/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "helix-lsp"
-version = "0.5.0"
+version = "0.6.0"
 authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
 edition = "2021"
 license = "MPL-2.0"
@@ -12,16 +12,17 @@ homepage = "https://helix-editor.com"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-helix-core = { version = "0.5", path = "../helix-core" }
+helix-core = { version = "0.6", path = "../helix-core" }
 
 anyhow = "1.0"
 futures-executor = "0.3"
 futures-util = { version = "0.3", features = ["std", "async-await"], default-features = false }
 jsonrpc-core = { version = "18.0", default-features = false } # don't pull in all of futures
 log = "0.4"
-lsp-types = { version = "0.91", features = ["proposed"] }
+lsp-types = { version = "0.93", features = ["proposed"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 thiserror = "1.0"
-tokio = { version = "1.15", features = ["rt", "rt-multi-thread", "io-util", "io-std", "time", "process", "macros", "fs", "parking_lot"] }
+tokio = { version = "1.17", features = ["rt", "rt-multi-thread", "io-util", "io-std", "time", "process", "macros", "fs", "parking_lot", "sync"] }
 tokio-stream = "0.1.8"
+which = "4.2"

helix-lsp/src/client.rs

@@ -3,6 +3,7 @@ use crate::{
     Call, Error, OffsetEncoding, Result,
 };
 
+use anyhow::anyhow;
 use helix_core::{find_root, ChangeSet, Rope};
 use jsonrpc_core as jsonrpc;
 use lsp_types as lsp;
@@ -31,7 +32,9 @@ pub struct Client {
     pub(crate) capabilities: OnceCell<lsp::ServerCapabilities>,
     offset_encoding: OffsetEncoding,
     config: Option<Value>,
-    root_markers: Vec<String>,
+    root_path: Option<std::path::PathBuf>,
+    root_uri: Option<lsp::Url>,
+    workspace_folders: Vec<lsp::WorkspaceFolder>,
 }
 
 impl Client {
@@ -40,9 +43,12 @@ impl Client {
         cmd: &str,
         args: &[String],
         config: Option<Value>,
-        root_markers: Vec<String>,
+        root_markers: &[String],
         id: usize,
     ) -> Result<(Self, UnboundedReceiver<(usize, Call)>, Arc<Notify>)> {
+        // Resolve path to the binary
+        let cmd = which::which(cmd).map_err(|err| anyhow::anyhow!(err))?;
+
         let process = Command::new(cmd)
             .args(args)
             .stdin(Stdio::piped())
@@ -62,6 +68,27 @@ impl Client {
         let (server_rx, server_tx, initialize_notify) =
             Transport::start(reader, writer, stderr, id);
 
+        let root_path = find_root(None, root_markers);
+
+        let root_uri = root_path
+            .clone()
+            .and_then(|root| lsp::Url::from_file_path(root).ok());
+
+        // TODO: support multiple workspace folders
+        let workspace_folders = root_uri
+            .clone()
+            .map(|root| {
+                vec![lsp::WorkspaceFolder {
+                    name: root
+                        .path_segments()
+                        .and_then(|segments| segments.last())
+                        .map(|basename| basename.to_string())
+                        .unwrap_or_default(),
+                    uri: root,
+                }]
+            })
+            .unwrap_or_default();
+
         let client = Self {
             id,
             _process: process,
@@ -70,7 +97,10 @@ impl Client {
             capabilities: OnceCell::new(),
             offset_encoding: OffsetEncoding::Utf8,
             config,
-            root_markers,
+
+            root_path,
+            root_uri,
+            workspace_folders,
         };
 
         Ok((client, server_rx, initialize_notify))
@@ -110,6 +140,14 @@ impl Client {
         self.offset_encoding
     }
 
+    pub fn config(&self) -> Option<&Value> {
+        self.config.as_ref()
+    }
+
+    pub fn workspace_folders(&self) -> &[lsp::WorkspaceFolder] {
+        &self.workspace_folders
+    }
+
     /// Execute a RPC request on the language server.
     async fn request<R: lsp::request::Request>(&self, params: R::Params) -> Result<R::Result>
     where
@@ -227,10 +265,6 @@ impl Client {
     // -------------------------------------------------------------------------------------------
 
     pub(crate) async fn initialize(&self) -> Result<lsp::InitializeResult> {
-        // TODO: delay any requests that are triggered prior to initialize
-        let root = find_root(None, &self.root_markers)
-            .and_then(|root| lsp::Url::from_file_path(root).ok());
-
         if self.config.is_some() {
             log::info!("Using custom LSP config: {}", self.config.as_ref().unwrap());
         }
@@ -238,15 +272,35 @@ impl Client {
         #[allow(deprecated)]
         let params = lsp::InitializeParams {
             process_id: Some(std::process::id()),
-            // root_path is obsolete, use root_uri
+            workspace_folders: Some(self.workspace_folders.clone()),
-            root_path: None,
+            // root_path is obsolete, but some clients like pyright still use it so we specify both.
-            root_uri: root,
+            // clients will prefer _uri if possible
+            root_path: self
+                .root_path
+                .clone()
+                .and_then(|path| path.to_str().map(|path| path.to_owned())),
+            root_uri: self.root_uri.clone(),
             initialization_options: self.config.clone(),
             capabilities: lsp::ClientCapabilities {
+                workspace: Some(lsp::WorkspaceClientCapabilities {
+                    configuration: Some(true),
+                    did_change_configuration: Some(lsp::DynamicRegistrationClientCapabilities {
+                        dynamic_registration: Some(false),
+                    }),
+                    workspace_folders: Some(true),
+                    ..Default::default()
+                }),
                 text_document: Some(lsp::TextDocumentClientCapabilities {
                     completion: Some(lsp::CompletionClientCapabilities {
                         completion_item: Some(lsp::CompletionItemCapability {
                             snippet_support: Some(false),
+                            resolve_support: Some(lsp::CompletionItemCapabilityResolveSupport {
+                                properties: vec![
+                                    String::from("documentation"),
+                                    String::from("detail"),
+                                    String::from("additionalTextEdits"),
+                                ],
+                            }),
                             ..Default::default()
                         }),
                         completion_item_kind: Some(lsp::CompletionItemKindCapability {
@@ -287,6 +341,9 @@ impl Client {
                         }),
                         ..Default::default()
                     }),
+                    publish_diagnostics: Some(lsp::PublishDiagnosticsClientCapabilities {
+                        ..Default::default()
+                    }),
                     ..Default::default()
                 }),
                 window: Some(lsp::WindowClientCapabilities {
@@ -296,7 +353,6 @@ impl Client {
                 ..Default::default()
             },
             trace: None,
-            workspace_folders: None,
             client_info: None,
             locale: None, // TODO
         };
@@ -327,6 +383,16 @@ impl Client {
         self.exit().await
     }
 
+    // -------------------------------------------------------------------------------------------
+    // Workspace
+    // -------------------------------------------------------------------------------------------
+
+    pub fn did_change_configuration(&self, settings: Value) -> impl Future<Output = Result<()>> {
+        self.notify::<lsp::notification::DidChangeConfiguration>(
+            lsp::DidChangeConfigurationParams { settings },
+        )
+    }
+
     // -------------------------------------------------------------------------------------------
     // Text document
     // -------------------------------------------------------------------------------------------
@@ -438,7 +504,7 @@ impl Client {
 
                     changes.push(lsp::TextDocumentContentChangeEvent {
                         range: Some(lsp::Range::new(start, end)),
-                        text: s.into(),
+                        text: s.to_string(),
                         range_length: None,
                     });
                 }
@@ -780,11 +846,12 @@ impl Client {
         &self,
         text_document: lsp::TextDocumentIdentifier,
         range: lsp::Range,
+        context: lsp::CodeActionContext,
     ) -> impl Future<Output = Result<Value>> {
         let params = lsp::CodeActionParams {
             text_document,
             range,
-            context: lsp::CodeActionContext::default(),
+            context,
             work_done_progress_params: lsp::WorkDoneProgressParams::default(),
             partial_result_params: lsp::PartialResultParams::default(),
         };
@@ -798,6 +865,19 @@ impl Client {
         position: lsp::Position,
         new_name: String,
     ) -> anyhow::Result<lsp::WorkspaceEdit> {
+        let capabilities = self.capabilities.get().unwrap();
+
+        // check if we're able to rename
+        match capabilities.rename_provider {
+            Some(lsp::OneOf::Left(true)) | Some(lsp::OneOf::Right(_)) => (),
+            // None | Some(false)
+            _ => {
+                let err = "The server does not support rename";
+                log::warn!("rename_symbol failed: {}", err);
+                return Err(anyhow!(err));
+            }
+        };
+
         let params = lsp::RenameParams {
             text_document_position: lsp::TextDocumentPositionParams {
                 text_document,

helix-lsp/src/lib.rs

@@ -58,6 +58,36 @@ pub mod util {
     use super::*;
     use helix_core::{Range, Rope, Transaction};
 
+    /// Converts a diagnostic in the document to [`lsp::Diagnostic`].
+    ///
+    /// Panics when [`pos_to_lsp_pos`] would for an invalid range on the diagnostic.
+    pub fn diagnostic_to_lsp_diagnostic(
+        doc: &Rope,
+        diag: &helix_core::diagnostic::Diagnostic,
+        offset_encoding: OffsetEncoding,
+    ) -> lsp::Diagnostic {
+        use helix_core::diagnostic::Severity::*;
+
+        let range = Range::new(diag.range.start, diag.range.end);
+        let severity = diag.severity.map(|s| match s {
+            Hint => lsp::DiagnosticSeverity::HINT,
+            Info => lsp::DiagnosticSeverity::INFORMATION,
+            Warning => lsp::DiagnosticSeverity::WARNING,
+            Error => lsp::DiagnosticSeverity::ERROR,
+        });
+
+        // TODO: add support for Diagnostic.data
+        lsp::Diagnostic::new(
+            range_to_lsp_range(doc, range, offset_encoding),
+            severity,
+            None,
+            None,
+            diag.message.to_owned(),
+            None,
+            None,
+        )
+    }
+
     /// Converts [`lsp::Position`] to a position in the document.
     ///
     /// Returns `None` if position exceeds document length or an operation overflows.
@@ -169,28 +199,14 @@ pub mod util {
             }),
         )
     }
-
-    /// The result of asking the language server to format the document. This can be turned into a
-    /// `Transaction`, but the advantage of not doing that straight away is that this one is
-    /// `Send` and `Sync`.
-    #[derive(Clone, Debug)]
-    pub struct LspFormatting {
-        pub doc: Rope,
-        pub edits: Vec<lsp::TextEdit>,
-        pub offset_encoding: OffsetEncoding,
-    }
-
-    impl From<LspFormatting> for Transaction {
-        fn from(fmt: LspFormatting) -> Transaction {
-            generate_transaction_from_edits(&fmt.doc, fmt.edits, fmt.offset_encoding)
-        }
-    }
 }
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum MethodCall {
     WorkDoneProgressCreate(lsp::WorkDoneProgressCreateParams),
     ApplyWorkspaceEdit(lsp::ApplyWorkspaceEditParams),
+    WorkspaceFolders,
+    WorkspaceConfiguration(lsp::ConfigurationParams),
 }
 
 impl MethodCall {
@@ -209,6 +225,13 @@ impl MethodCall {
                     .expect("Failed to parse ApplyWorkspaceEdit params");
                 Self::ApplyWorkspaceEdit(params)
             }
+            lsp::request::WorkspaceFoldersRequest::METHOD => Self::WorkspaceFolders,
+            lsp::request::WorkspaceConfiguration::METHOD => {
+                let params: lsp::ConfigurationParams = params
+                    .parse()
+                    .expect("Failed to parse WorkspaceConfiguration params");
+                Self::WorkspaceConfiguration(params)
+            }
             _ => {
                 log::warn!("unhandled lsp request: {}", method);
                 return None;
@@ -237,7 +260,13 @@ impl Notification {
             lsp::notification::PublishDiagnostics::METHOD => {
                 let params: lsp::PublishDiagnosticsParams = params
                     .parse()
-                    .expect("Failed to parse PublishDiagnostics params");
+                    .map_err(|err| {
+                        log::error!(
+                            "received malformed PublishDiagnostic from Language Server: {}",
+                            err
+                        )
+                    })
+                    .ok()?;
 
                 // TODO: need to loop over diagnostics and distinguish them by URI
                 Self::PublishDiagnostics(params)
@@ -313,7 +342,7 @@ impl Registry {
                     &config.command,
                     &config.args,
                     language_config.config.clone(),
-                    language_config.roots.clone(),
+                    &language_config.roots,
                     id,
                 )?;
                 self.incoming.push(UnboundedReceiverStream::new(incoming));
@@ -383,7 +412,7 @@ impl LspProgressMap {
         Self::default()
     }
 
-    /// Returns a map of all tokens coresponding to the lanaguage server with `id`.
+    /// Returns a map of all tokens corresponding to the language server with `id`.
     pub fn progress_map(&self, id: usize) -> Option<&HashMap<lsp::ProgressToken, ProgressStatus>> {
         self.0.get(&id)
     }
@@ -421,7 +450,7 @@ impl LspProgressMap {
         self.0.get_mut(&id).and_then(|vals| vals.remove(token))
     }
 
-    /// Updates the progess of `token` for server with `id` to `status`, returns the value replaced or `None`.
+    /// Updates the progress of `token` for server with `id` to `status`, returns the value replaced or `None`.
     pub fn update(
         &mut self,
         id: usize,

helix-syntax/Cargo.toml

@@ -1,21 +0,0 @@
-[package]
-name = "helix-syntax"
-version = "0.5.0"
-authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
-edition = "2021"
-license = "MPL-2.0"
-description = "Tree-sitter grammars support"
-categories = ["editor"]
-repository = "https://github.com/helix-editor/helix"
-homepage = "https://helix-editor.com"
-include = ["src/**/*", "languages/**/*", "build.rs", "!**/docs/**/*", "!**/test/**/*", "!**/examples/**/*", "!**/build/**/*"]
-
-[dependencies]
-tree-sitter = "0.20"
-libloading = "0.7"
-anyhow = "1"
-
-[build-dependencies]
-cc = { version = "1" }
-threadpool = { version = "1.0" }
-anyhow = "1"

helix-syntax/README.md

@@ -1,13 +0,0 @@
-helix-syntax
-============
-
-Syntax highlighting for helix, (shallow) submodules resides here.
-
-Differences from nvim-treesitter
---------------------------------
-
-As the syntax are commonly ported from
-<https://github.com/nvim-treesitter/nvim-treesitter>.
-
-Note that we do not support the custom `#any-of` predicate which is
-supported by neovim so one needs to change it to `#match` with regex.

helix-syntax/build.rs

@@ -1,207 +0,0 @@
-use anyhow::{anyhow, Context, Result};
-use std::fs;
-use std::time::SystemTime;
-use std::{
-    path::{Path, PathBuf},
-    process::Command,
-};
-
-use std::sync::mpsc::channel;
-
-fn collect_tree_sitter_dirs(ignore: &[String]) -> Result<Vec<String>> {
-    let mut dirs = Vec::new();
-    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("languages");
-
-    for entry in fs::read_dir(path)? {
-        let entry = entry?;
-        let path = entry.path();
-
-        if !entry.file_type()?.is_dir() {
-            continue;
-        }
-
-        let dir = path.file_name().unwrap().to_str().unwrap().to_string();
-
-        // filter ignores
-        if ignore.contains(&dir) {
-            continue;
-        }
-        dirs.push(dir)
-    }
-
-    Ok(dirs)
-}
-
-#[cfg(unix)]
-const DYLIB_EXTENSION: &str = "so";
-
-#[cfg(windows)]
-const DYLIB_EXTENSION: &str = "dll";
-
-fn build_library(src_path: &Path, language: &str) -> Result<()> {
-    let header_path = src_path;
-    // let grammar_path = src_path.join("grammar.json");
-    let parser_path = src_path.join("parser.c");
-    let mut scanner_path = src_path.join("scanner.c");
-
-    let scanner_path = if scanner_path.exists() {
-        Some(scanner_path)
-    } else {
-        scanner_path.set_extension("cc");
-        if scanner_path.exists() {
-            Some(scanner_path)
-        } else {
-            None
-        }
-    };
-    let parser_lib_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../runtime/grammars");
-    let mut library_path = parser_lib_path.join(language);
-    library_path.set_extension(DYLIB_EXTENSION);
-
-    let recompile = needs_recompile(&library_path, &parser_path, &scanner_path)
-        .with_context(|| "Failed to compare source and binary timestamps")?;
-
-    if !recompile {
-        return Ok(());
-    }
-    let mut config = cc::Build::new();
-    config.cpp(true).opt_level(2).cargo_metadata(false);
-    let compiler = config.get_compiler();
-    let mut command = Command::new(compiler.path());
-    command.current_dir(src_path);
-    for (key, value) in compiler.env() {
-        command.env(key, value);
-    }
-
-    if cfg!(windows) {
-        command
-            .args(&["/nologo", "/LD", "/I"])
-            .arg(header_path)
-            .arg("/Od")
-            .arg("/utf-8");
-        if let Some(scanner_path) = scanner_path.as_ref() {
-            command.arg(scanner_path);
-        }
-
-        command
-            .arg(parser_path)
-            .arg("/link")
-            .arg(format!("/out:{}", library_path.to_str().unwrap()));
-    } else {
-        command
-            .arg("-shared")
-            .arg("-fPIC")
-            .arg("-fno-exceptions")
-            .arg("-g")
-            .arg("-I")
-            .arg(header_path)
-            .arg("-o")
-            .arg(&library_path)
-            .arg("-O2");
-        if let Some(scanner_path) = scanner_path.as_ref() {
-            if scanner_path.extension() == Some("c".as_ref()) {
-                command.arg("-xc").arg("-std=c99").arg(scanner_path);
-            } else {
-                command.arg(scanner_path);
-            }
-        }
-        command.arg("-xc").arg(parser_path);
-        if cfg!(all(unix, not(target_os = "macos"))) {
-            command.arg("-Wl,-z,relro,-z,now");
-        }
-    }
-
-    let output = command
-        .output()
-        .with_context(|| "Failed to execute C compiler")?;
-    if !output.status.success() {
-        return Err(anyhow!(
-            "Parser compilation failed.\nStdout: {}\nStderr: {}",
-            String::from_utf8_lossy(&output.stdout),
-            String::from_utf8_lossy(&output.stderr)
-        ));
-    }
-
-    Ok(())
-}
-fn needs_recompile(
-    lib_path: &Path,
-    parser_c_path: &Path,
-    scanner_path: &Option<PathBuf>,
-) -> Result<bool> {
-    if !lib_path.exists() {
-        return Ok(true);
-    }
-    let lib_mtime = mtime(lib_path)?;
-    if mtime(parser_c_path)? > lib_mtime {
-        return Ok(true);
-    }
-    if let Some(scanner_path) = scanner_path {
-        if mtime(scanner_path)? > lib_mtime {
-            return Ok(true);
-        }
-    }
-    Ok(false)
-}
-
-fn mtime(path: &Path) -> Result<SystemTime> {
-    Ok(fs::metadata(path)?.modified()?)
-}
-
-fn build_dir(dir: &str, language: &str) {
-    println!("Build language {}", language);
-    if PathBuf::from("languages")
-        .join(dir)
-        .read_dir()
-        .unwrap()
-        .next()
-        .is_none()
-    {
-        eprintln!(
-            "The directory {} is empty, you probably need to use 'git submodule update --init --recursive'?",
-            dir
-        );
-        std::process::exit(1);
-    }
-
-    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
-        .join("languages")
-        .join(dir)
-        .join("src");
-
-    build_library(&path, language).unwrap();
-}
-
-fn main() {
-    let ignore = vec![
-        "tree-sitter-typescript".to_string(),
-        "tree-sitter-haskell".to_string(), // aarch64 failures: https://github.com/tree-sitter/tree-sitter-haskell/issues/34
-        "tree-sitter-ocaml".to_string(),
-    ];
-    let dirs = collect_tree_sitter_dirs(&ignore).unwrap();
-
-    let mut n_jobs = 0;
-    let pool = threadpool::Builder::new().build(); // by going through the builder, it'll use num_cpus
-    let (tx, rx) = channel();
-
-    for dir in dirs {
-        let tx = tx.clone();
-        n_jobs += 1;
-
-        pool.execute(move || {
-            let language = &dir.strip_prefix("tree-sitter-").unwrap();
-            build_dir(&dir, language);
-
-            // report progress
-            tx.send(1).unwrap();
-        });
-    }
-    pool.join();
-    // drop(tx);
-    assert_eq!(rx.try_iter().sum::<usize>(), n_jobs);
-
-    build_dir("tree-sitter-typescript/tsx", "tsx");
-    build_dir("tree-sitter-typescript/typescript", "typescript");
-    build_dir("tree-sitter-ocaml/ocaml", "ocaml");
-    build_dir("tree-sitter-ocaml/interface", "ocaml-interface")
-}