Release v0.4.1

We ran the tests first, but did not cross compile them. This step would
also compile all the grammar libraries (but for the host machine). On
the actual release build, the editor would get built for the target, but
the grammar libraries would be detected as present and wouldn't
recompile.

Refs #577

.github/ISSUE_TEMPLATE/blank_issue.md

@@ -0,0 +1,4 @@
+---
+name: Blank Issue
+about: Create a blank issue.
+---

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,13 @@
+---
+name: Feature request
+about: Suggest a new feature or improvement
+title: ''
+labels: C-enhancement
+assignees: ''
+---
+
+<!-- Your feature may already be reported!
+Please search on the issue tracker before creating one. -->
+
+#### Describe your feature request
+

.github/workflows/release.yml

@@ -67,8 +67,9 @@ jobs:
       - name: Run cargo test
         uses: actions-rs/cargo@v1
         with:
+          use-cross: ${{ matrix.cross }}
           command: test
-          args: --locked
+          args: --release --locked --target ${{ matrix.target }}
 
       - name: Build release binary
         uses: actions-rs/cargo@v1
@@ -99,8 +100,9 @@ jobs:
           else
             cp "target/${{ matrix.target }}/release/hx" "dist/"
           fi
+          cp -r runtime dist
 
-      - uses: actions/upload-artifact@v2.2.3
+      - uses: actions/upload-artifact@v2.2.4
         with:
           name: bins-${{ matrix.build }}
           path: dist
@@ -148,7 +150,7 @@ jobs:
               pkgname=helix-$TAG-$platform
               mkdir tmp/$pkgname
               cp LICENSE README.md tmp/$pkgname
-              cp -r runtime tmp/$pkgname/
+              mv bins-$platform/runtime tmp/$pkgname/
               mv bins-$platform/hx$exe tmp/$pkgname
               chmod +x tmp/$pkgname/hx$exe
 

.gitignore

@@ -3,3 +3,4 @@ target
 helix-term/rustfmt.toml
 helix-syntax/languages/
 result
+runtime/grammars

.gitmodules

@@ -89,3 +89,8 @@
 [submodule "helix-syntax/languages/tree-sitter-nix"]
 	path = helix-syntax/languages/tree-sitter-nix
 	url = https://github.com/cstrahan/tree-sitter-nix
+	shallow = true
+[submodule "helix-syntax/languages/tree-sitter-latex"]
+	path = helix-syntax/languages/tree-sitter-latex
+	url = https://github.com/latex-lsp/tree-sitter-latex
+	shallow = true

CHANGELOG.md

@@ -0,0 +1,131 @@
+
+# 0.4.1 (2021-08-14)
+
+A minor release that includes:
+- A fix for rendering glitches that would occur after editing with multiple selections.
+- CI fix for grammars not being cross-compiled for aarch64 
+
+# 0.4.1 (2021-08-13)
+
+Two months have passed, so this is another big release. A big thank you to all
+the contributors and package maintainers!
+
+Helix has popped up in [Arch, Manjaro, Nix, MacPorts and Parabola and Termux repositories](https://repology.org/project/helix/versions)!
+
+A [large scale refactor](https://github.com/helix-editor/helix/pull/376) landed that allows us to support zero width (empty)
+selections in the future as well as resolves many bugs and edge cases.
+
+- Multi-key remapping! Key binds now support much more complex usecases ([#454](https://github.com/helix-editor/helix/pull/454))
+- Pending keys are shown in the statusline ([#515](https://github.com/helix-editor/helix/pull/515))
+- Object selection / textobjects. `mi(` to select text inside parentheses ([#385](https://github.com/helix-editor/helix/pull/385))
+- Autoinfo: `whichkey`-like popups which show available sub-mode shortcuts ([#316](https://github.com/helix-editor/helix/pull/316))
+- Added WORD movements (W/B/E) ([#390](https://github.com/helix-editor/helix/pull/390))
+- Vertical selections (repeat selection above/below) ([#462](https://github.com/helix-editor/helix/pull/462))
+- Selection rotation via `(` and `)` ([66a90130](https://github.com/helix-editor/helix/commit/66a90130a5f99d769e9f6034025297f78ecaa3ec))
+- Selection contents rotation via `Alt-(` and `Alt-)` ([02cba2a](https://github.com/helix-editor/helix/commit/02cba2a7f403f48eccb18100fb751f7b42373dba))
+- Completion behavior improvements ([f917b5a4](https://github.com/helix-editor/helix/commit/f917b5a441ff3ae582358b6939ffbf889f4aa530), [627b899](https://github.com/helix-editor/helix/commit/627b89931576f7af86166ae8d5cbc55537877473))
+- Fixed a language server crash ([385a6b5a](https://github.com/helix-editor/helix/commit/385a6b5a1adddfc26e917982641530e1a7c7aa81))
+- Case change commands (`\``, `~`, `<a-~>`) ([#441](https://github.com/helix-editor/helix/pull/441))
+- File pickers (including goto) now provide a preview! ([#534](https://github.com/helix-editor/helix/pull/534))
+- Injection query support. Rust macro calls and embedded languages are now properly highlighted ([#430](https://github.com/helix-editor/helix/pull/430))
+- Formatting is now asynchronous, and the async job infrastructure has been improved ([#285](https://github.com/helix-editor/helix/pull/285))
+- Grammars are now compiled as separate shared libraries and loaded on-demand at runtime ([#432](https://github.com/helix-editor/helix/pull/432))
+- Code action support ([#478](https://github.com/helix-editor/helix/pull/478))
+- Mouse support ([#509](https://github.com/helix-editor/helix/pull/509), [#548](https://github.com/helix-editor/helix/pull/548))
+- Native Windows clipboard support ([#373](https://github.com/helix-editor/helix/pull/373))
+- Themes can now use color palettes ([#393](https://github.com/helix-editor/helix/pull/393))
+- `:reload` command ([#374](https://github.com/helix-editor/helix/pull/374))
+- Ctrl-z to suspend ([#464](https://github.com/helix-editor/helix/pull/464))
+- Language servers can now be configured with a custom JSON config ([#460](https://github.com/helix-editor/helix/pull/460))
+- Comment toggling now uses a language specific comment token ([#463](https://github.com/helix-editor/helix/pull/463))
+- Julia support ([#413](https://github.com/helix-editor/helix/pull/413))
+- Java support ([#448](https://github.com/helix-editor/helix/pull/448))
+- Prompts have an (in-memory) history ([63e54e30](https://github.com/helix-editor/helix/commit/63e54e30a74bb0d1d782877ddbbcf95f2817d061))
+
+# 0.3.0 (2021-06-27)
+
+Another big release. 
+
+Highlights:
+
+- Indentation is now automatically detected from file heuristics. ([#245](https://github.com/helix-editor/helix/pull/245))
+- Support for other line endings (CRLF). Significantly improved Windows support. ([#224](https://github.com/helix-editor/helix/pull/224))
+- Encodings other than UTF-8 are now supported! ([#228](https://github.com/helix-editor/helix/pull/228))
+- Key bindings can now be configured via a `config.toml` file ([#268](https://github.com/helix-editor/helix/pull/268))
+- Theme can now be configured and changed at runtime ([please feel free to contribute more themes!](https://github.com/helix-editor/helix/tree/master/runtime/themes)) ([#267](https://github.com/helix-editor/helix/pull/267))
+- System clipboard yank/paste is now supported! ([#310](https://github.com/helix-editor/helix/pull/310))
+- Surround commands were implemented ([#320](https://github.com/helix-editor/helix/pull/320))
+
+Features:
+
+- File picker can now be repeatedly filtered ([#232](https://github.com/helix-editor/helix/pull/232))
+- LSP progress is now received and rendered as a spinner ([#234](https://github.com/helix-editor/helix/pull/234))
+- Current line number can now be themed ([#260](https://github.com/helix-editor/helix/pull/260))
+- Arrow keys & home/end now work in insert mode ([#305](https://github.com/helix-editor/helix/pull/305))
+- Cursors and selections can now be themed ([#325](https://github.com/helix-editor/helix/pull/325))
+- Language servers are now gracefully shut down before `hx` exits ([#287](https://github.com/helix-editor/helix/pull/287))
+- `:show-directory`/`:change-directory` ([#335](https://github.com/helix-editor/helix/pull/335))
+- File picker is now sorted by access time (before filtering) ([#336](https://github.com/helix-editor/helix/pull/336))
+- Code is being migrated from helix-term to helix-view (prerequisite for
+  alternative frontends) ([#366](https://github.com/helix-editor/helix/pull/366))
+- `x` and `X` merged
+  ([f41688d9](https://github.com/helix-editor/helix/commit/f41688d960ef89c29c4a51c872b8406fb8f81a85))
+
+Fixes:
+
+- The IME popup is now correctly positioned ([#273](https://github.com/helix-editor/helix/pull/273))
+- A bunch of bugs regarding `o`/`O` behavior ([#281](https://github.com/helix-editor/helix/pull/281))
+- `~` expansion now works in file completion ([#284](https://github.com/helix-editor/helix/pull/284))
+- Several UI related overflow crashes ([#318](https://github.com/helix-editor/helix/pull/318))
+- Fix a test failure occuring only on `test --release` ([4f108ab1](https://github.com/helix-editor/helix/commit/4f108ab1b2197809506bd7305ad903a3525eabfa))
+- Prompts now support unicode input ([#295](https://github.com/helix-editor/helix/pull/295))
+- Completion documentation no longer overlaps the popup ([#322](https://github.com/helix-editor/helix/pull/322))
+- Fix a crash when trying to select `^` ([9c534614](https://github.com/helix-editor/helix/commit/9c53461429a3e72e3b1fb87d7ca490e168d7dee2))
+- Prompt completions are now paginated ([39dc09e6](https://github.com/helix-editor/helix/commit/39dc09e6c4172299bc79de4c1c52288d3f624bd7))
+- Goto did not work on Windows ([503ca112](https://github.com/helix-editor/helix/commit/503ca112ae57ebdf3ea323baf8940346204b46d2))
+
+# 0.2.1
+
+Includes a fix where wq/wqa could exit before file saving completed.
+
+# 0.2.0
+
+Enough has changed to bump the version. We're skipping 0.1.x because
+previously the CLI would always report version as 0.1.0, and we'd like
+to distinguish it in bug reports..
+
+- The `runtime/` directory is now properly detected on binary releases and
+  on cargo run. `~/.config/helix/runtime` can also be used.
+- Registers can now be selected via " (for example `"ay`)
+- Support for Nix files was added
+- Movement is now fully tested and matches kakoune implementation
+- A per-file LSP symbol picker was added to space+s
+- Selection can be replaced with yanked text via R
+
+- `1g` now correctly goes to line 1
+- `ctrl-i` now correctly jumps backwards in history
+- A small memory leak was fixed, where we tried to reuse tree-sitter
+  query cursors, but always allocated a new one
+- Auto-formatting is now only on for certain languages
+- The root directory is now provided in LSP initialization, fixing
+  certain language servers (typescript)
+- LSP failing to start no longer panics
+- Elixir language queries were fixed
+
+# 0.0.10
+
+Keymaps:
+- Add mappings to jump to diagnostics
+- Add gt/gm/gb mappings to jump to top/middle/bottom of screen
+- ^ and $ are now gh, gl
+
+- The runtime/ can now optionally be embedded in the binary
+- Haskell syntax added
+- Window mode (ctrl-w) added
+- Show matching bracket (vim's matchbrackets)
+- Themes now support style modifiers
+- First user contributed theme
+- Create a document if it doesn't exist yet on save
+- Detect language on a new file on save
+
+- Panic fixes, lots of them

Cargo.lock

@@ -13,9 +13,15 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.41"
+version = "1.0.42"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15af2628f6890fe2609a3b91bef4c83450512802e59489f9c1cb1fa5df064a61"
+checksum = "595d3cfa7a60d4555cb5067b99f07142a08ea778de5cf993f7b75c7d8fabc486"
+
+[[package]]
+name = "arc-swap"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e906254e445520903e7fc9da4f709886c84ae4bc4ddaf0e093188d66df4dc820"
 
 [[package]]
 name = "autocfg"
@@ -25,9 +31,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
 
 [[package]]
 name = "bitflags"
-version = "1.2.1"
+version = "1.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
+checksum = "2da1976d75adbe5fbc88130ecd119529cf1cc6a93ae1546d8696ee66f0d21af1"
 
 [[package]]
 name = "bstr"
@@ -52,12 +58,9 @@ checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53"
 
 [[package]]
 name = "cc"
-version = "1.0.68"
+version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a72c244c1ff497a746a7e1fb3d14bd08420ecda70c8f25c7112f2781652d787"
-dependencies = [
- "jobserver",
-]
+checksum = "e70cc2f62c6ce1868963827bd677764c62d07c3d9a3e1fb1177ee1a9ab199eb2"
 
 [[package]]
 name = "cfg-if"
@@ -65,6 +68,17 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "chardetng"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36a5a2ca47925d19fb6835f53b3e70dec0d25659211c8ee5cc784f1fd6838f9c"
+dependencies = [
+ "cfg-if",
+ "encoding_rs",
+ "memchr",
+]
+
 [[package]]
 name = "chrono"
 version = "0.4.19"
@@ -77,6 +91,17 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "clipboard-win"
+version = "4.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e4ea1881992efc993e4dc50a324cdbd03216e41bdc8385720ff47efc9bd2ca8"
+dependencies = [
+ "error-code",
+ "str-buf",
+ "winapi",
+]
+
 [[package]]
 name = "crossbeam-utils"
 version = "0.8.5"
@@ -134,6 +159,31 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "either"
+version = "1.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
+
+[[package]]
+name = "encoding_rs"
+version = "0.8.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "error-code"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b5115567ac25674e0043e472be13d14e537f37ea8aa4bdc4aef0c89add1db1ff"
+dependencies = [
+ "libc",
+ "str-buf",
+]
+
 [[package]]
 name = "etcetera"
 version = "0.3.2"
@@ -182,15 +232,15 @@ dependencies = [
 
 [[package]]
 name = "futures-core"
-version = "0.3.15"
+version = "0.3.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0402f765d8a89a26043b889b26ce3c4679d268fa6bb22cd7c6aad98340e179d1"
+checksum = "af51b1b4a7fdff033703db39de8802c673eb91855f2e0d47dcf3bf2c0ef01f99"
 
 [[package]]
 name = "futures-executor"
-version = "0.3.15"
+version = "0.3.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "badaa6a909fac9e7236d0620a2f57f7664640c56575b71a7552fbd68deafab79"
+checksum = "4d0d535a57b87e1ae31437b892713aee90cd2d7b0ee48727cd11fc72ef54761c"
 dependencies = [
  "futures-core",
  "futures-task",
@@ -199,15 +249,15 @@ dependencies = [
 
 [[package]]
 name = "futures-task"
-version = "0.3.15"
+version = "0.3.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a16bef9fc1a4dddb5bee51c989e3fbba26569cbb0e31f5b303c184e3dd33dae"
+checksum = "bbe54a98670017f3be909561f6ad13e810d9a51f3f061b902062ca3da80799f2"
 
 [[package]]
 name = "futures-util"
-version = "0.3.15"
+version = "0.3.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "feb5c238d27e2bf94ffdfd27b2c29e3df4a68c4193bb6427384259e2bf191967"
+checksum = "67eb846bfd58e44a8481a00049e82c43e0ccb5d61f8dc071057cb19249dd4d78"
 dependencies = [
  "autocfg",
  "futures-core",
@@ -239,9 +289,9 @@ dependencies = [
 
 [[package]]
 name = "globset"
-version = "0.4.7"
+version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0fc1b9fa0e64ffb1aa5b95daa0f0f167734fd528b7c02eabc581d9d843649b1"
+checksum = "10463d9ff00a2a068db14231982f5132edebad0d7660cd956a1c30292dbcbfbd"
 dependencies = [
  "aho-corasick",
  "bstr",
@@ -252,15 +302,17 @@ dependencies = [
 
 [[package]]
 name = "helix-core"
-version = "0.2.0"
+version = "0.4.1"
 dependencies = [
+ "arc-swap",
  "etcetera",
  "helix-syntax",
  "once_cell",
+ "quickcheck",
  "regex",
  "ropey",
- "rust-embed",
  "serde",
+ "similar",
  "smallvec",
  "tendril",
  "toml",
@@ -272,7 +324,7 @@ dependencies = [
 
 [[package]]
 name = "helix-lsp"
-version = "0.2.0"
+version = "0.4.1"
 dependencies = [
  "anyhow",
  "futures-executor",
@@ -290,22 +342,22 @@ dependencies = [
 
 [[package]]
 name = "helix-syntax"
-version = "0.2.0"
+version = "0.4.1"
 dependencies = [
+ "anyhow",
  "cc",
- "serde",
+ "libloading",
  "threadpool",
  "tree-sitter",
 ]
 
 [[package]]
 name = "helix-term"
-version = "0.2.0"
+version = "0.4.1"
 dependencies = [
  "anyhow",
  "chrono",
  "crossterm",
- "dirs-next",
  "fern",
  "futures-util",
  "fuzzy-matcher",
@@ -320,28 +372,36 @@ dependencies = [
  "pulldown-cmark",
  "serde",
  "serde_json",
+ "signal-hook",
+ "signal-hook-tokio",
  "tokio",
  "toml",
 ]
 
 [[package]]
 name = "helix-tui"
-version = "0.2.0"
+version = "0.4.1"
 dependencies = [
  "bitflags",
  "cassowary",
  "crossterm",
+ "helix-core",
+ "helix-view",
  "serde",
  "unicode-segmentation",
- "unicode-width",
 ]
 
 [[package]]
 name = "helix-view"
-version = "0.2.0"
+version = "0.4.1"
 dependencies = [
  "anyhow",
+ "bitflags",
+ "chardetng",
+ "clipboard-win",
  "crossterm",
+ "encoding_rs",
+ "futures-util",
  "helix-core",
  "helix-lsp",
  "helix-tui",
@@ -352,13 +412,14 @@ dependencies = [
  "tokio",
  "toml",
  "url",
+ "which",
 ]
 
 [[package]]
 name = "hermit-abi"
-version = "0.1.18"
+version = "0.1.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c"
+checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
 dependencies = [
  "libc",
 ]
@@ -394,9 +455,9 @@ dependencies = [
 
 [[package]]
 name = "instant"
-version = "0.1.9"
+version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec"
+checksum = "bee0328b1209d157ef001c94dd85b4f8f64139adb0eac2659f4b08382b2f474d"
 dependencies = [
  "cfg-if",
 ]
@@ -407,20 +468,11 @@ version = "0.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736"
 
-[[package]]
-name = "jobserver"
-version = "0.1.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "972f5ae5d1cb9c6ae417789196c803205313edde988685da5e3aae0827b9e7fd"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "jsonrpc-core"
-version = "17.1.0"
+version = "18.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4467ab6dfa369b69e52bd0692e480c4d117410538526a57a304a0f2250fd95e"
+checksum = "14f7f76aef2d054868398427f6c54943cf3d1caa9a7ec7d0c38d69df97a965eb"
 dependencies = [
  "futures-util",
  "log",
@@ -437,9 +489,19 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.95"
+version = "0.2.99"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "789da6d93f1b866ffe175afc5322a4d76c038605a1c3319bb57b06967ca98a36"
+checksum = "a7f823d141fe0a24df1e23b4af4e3c7ba9e5966ec514ea068c93024aa7deb765"
+
+[[package]]
+name = "libloading"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f84d96438c15fcd6c3f244c8fce01d1e2b9c6b5623e9c711dc9286d8fc92d6a"
+dependencies = [
+ "cfg-if",
+ "winapi",
+]
 
 [[package]]
 name = "lock_api"
@@ -480,9 +542,9 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
 
 [[package]]
 name = "matches"
-version = "0.1.8"
+version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
+checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f"
 
 [[package]]
 name = "memchr"
@@ -492,9 +554,9 @@ checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
 
 [[package]]
 name = "mio"
-version = "0.7.11"
+version = "0.7.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf80d3e903b34e0bd7282b218398aec54e082c840d9baf8339e0080a0c542956"
+checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16"
 dependencies = [
  "libc",
  "log",
@@ -595,9 +657,9 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
 
 [[package]]
 name = "pin-project-lite"
-version = "0.2.6"
+version = "0.2.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905"
+checksum = "8d31d11c69a6b52a174b42bdc0c30e5e11670f90788b2c471c31c1d17d449443"
 
 [[package]]
 name = "pin-utils"
@@ -607,9 +669,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.27"
+version = "1.0.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038"
+checksum = "5c7ed8b8c7b886ea3ed7dde405212185f423ab44682667c8c6dd14aa1d9f6612"
 dependencies = [
  "unicode-xid",
 ]
@@ -625,6 +687,15 @@ dependencies = [
  "unicase",
 ]
 
+[[package]]
+name = "quickcheck"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6"
+dependencies = [
+ "rand",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.9"
@@ -635,10 +706,28 @@ dependencies = [
 ]
 
 [[package]]
-name = "redox_syscall"
-version = "0.2.8"
+name = "rand"
+version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc"
+checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8"
+dependencies = [
+ "rand_core",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
+dependencies = [
+ "getrandom",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.2.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff"
 dependencies = [
  "bitflags",
 ]
@@ -672,46 +761,13 @@ checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
 
 [[package]]
 name = "ropey"
-version = "1.2.0"
+version = "1.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0f3ef16589fdbb3e8fbce3dca944c08e61f39c7f16064b21a257d68ea911a83"
+checksum = "9150aff6deb25b20ed110889f070a678bcd1033e46e5e9d6fb1abeab17947f28"
 dependencies = [
  "smallvec",
 ]
 
-[[package]]
-name = "rust-embed"
-version = "5.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fe1fe6aac5d6bb9e1ffd81002340363272a7648234ec7bdfac5ee202cb65523"
-dependencies = [
- "rust-embed-impl",
- "rust-embed-utils",
- "walkdir",
-]
-
-[[package]]
-name = "rust-embed-impl"
-version = "5.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ed91c41c42ef7bf687384439c312e75e0da9c149b0390889b94de3c7d9d9e66"
-dependencies = [
- "proc-macro2",
- "quote",
- "rust-embed-utils",
- "syn",
- "walkdir",
-]
-
-[[package]]
-name = "rust-embed-utils"
-version = "5.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a512219132473ab0a77b52077059f1c47ce4af7fbdc94503e9862a34422876d"
-dependencies = [
- "walkdir",
-]
-
 [[package]]
 name = "ryu"
 version = "1.0.5"
@@ -735,18 +791,18 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
 [[package]]
 name = "serde"
-version = "1.0.126"
+version = "1.0.127"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03"
+checksum = "f03b9878abf6d14e6779d3f24f07b2cfa90352cfec4acc5aab8f1ac7f146fae8"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.126"
+version = "1.0.127"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43"
+checksum = "a024926d3432516606328597e0f224a51355a493b49fdd67e9209187cbe55ecc"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -755,9 +811,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.64"
+version = "1.0.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79"
+checksum = "336b10da19a12ad094b59d870ebde26a45402e5b470add4b5fd03c5048a32127"
 dependencies = [
  "itoa",
  "ryu",
@@ -806,16 +862,34 @@ dependencies = [
 ]
 
 [[package]]
-name = "slab"
-version = "0.4.3"
+name = "signal-hook-tokio"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527"
+checksum = "f6c5d32165ff8b94e68e7b3bdecb1b082e958c22434b363482cfb89dcd6f3ff8"
+dependencies = [
+ "futures-core",
+ "libc",
+ "signal-hook",
+ "tokio",
+]
+
+[[package]]
+name = "similar"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ad1d488a557b235fc46dae55512ffbfc429d2482b08b4d9435ab07384ca8aec"
+
+[[package]]
+name = "slab"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c307a32c1c5c437f38c7fd45d753050587732ba8628319fbdf12a7e289ccc590"
 
 [[package]]
 name = "slotmap"
-version = "1.0.3"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "585cd5dffe4e9e06f6dfdf66708b70aca3f781bed561f4f667b2d9c0d4559e36"
+checksum = "a952280edbecfb1d4bd3cf2dbc309dc6ab523e53487c438ae21a6df09fe84bc4"
 dependencies = [
  "version_check",
 ]
@@ -827,10 +901,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
 
 [[package]]
-name = "syn"
-version = "1.0.72"
+name = "str-buf"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82"
+checksum = "d44a3643b4ff9caf57abcee9c2c621d6c03d9135e0d8b589bd9afb5992cb176a"
+
+[[package]]
+name = "syn"
+version = "1.0.74"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1873d832550d4588c3dbc20f01361ab00bfe741048f71e3fecf145a7cc18b29c"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -850,18 +930,18 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.25"
+version = "1.0.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa6f76457f59514c7eeb4e59d891395fab0b2fd1d40723ae737d64153392e9c6"
+checksum = "93119e4feac1cbe6c798c34d3a53ea0026b0b1de6a120deef895137c0529bfe2"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.25"
+version = "1.0.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a36768c0fbf1bb15eca10defa29526bda730a2376c2ab4393ccfa16fb1a318d"
+checksum = "060d69a0afe7796bf42e9e2ff91f5ee691fb15c53d38b4b62a9a53eb23164745"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -888,9 +968,9 @@ dependencies = [
 
 [[package]]
 name = "tinyvec"
-version = "1.2.0"
+version = "1.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b5220f05bb7de7f3f53c7c065e1199b3172696fe2db9f9c4d8ad9b4ee74c342"
+checksum = "848a1e1181b9f6753b5e96a092749e29b11d19ede67dfbbd6c7dc7e0f49b5338"
 dependencies = [
  "tinyvec_macros",
 ]
@@ -903,9 +983,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
 
 [[package]]
 name = "tokio"
-version = "1.6.1"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a38d31d7831c6ed7aad00aa4c12d9375fd225a6dd77da1d25b707346319a975"
+checksum = "01cf844b23c6131f624accf65ce0e4e9956a8bb329400ea5bcc26ae3a5c20b0b"
 dependencies = [
  "autocfg",
  "bytes",
@@ -923,9 +1003,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-macros"
-version = "1.2.0"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c49e3df43841dafb86046472506755d8501c5615673955f6aa17181125d13c37"
+checksum = "54473be61f4ebe4efd09cec9bd5d16fa51d70ea0192213d754d2d500457db110"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -934,9 +1014,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-stream"
-version = "0.1.6"
+version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8864d706fdb3cc0843a49647ac892720dac98a6eeb818b77190592cf4994066"
+checksum = "7b2f3f698253f03119ac0102beaa64f67a67e08074d03a22d18784104543727f"
 dependencies = [
  "futures-core",
  "pin-project-lite",
@@ -973,12 +1053,9 @@ dependencies = [
 
 [[package]]
 name = "unicode-bidi"
-version = "0.3.5"
+version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0"
-dependencies = [
- "matches",
-]
+checksum = "246f4c42e67e7a4e3c6106ff716a5d067d4132a642840b242e357e468a2a0085"
 
 [[package]]
 name = "unicode-general-category"
@@ -997,9 +1074,9 @@ dependencies = [
 
 [[package]]
 name = "unicode-segmentation"
-version = "1.7.1"
+version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796"
+checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b"
 
 [[package]]
 name = "unicode-width"
@@ -1055,6 +1132,17 @@ version = "0.10.2+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
 
+[[package]]
+name = "which"
+version = "4.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea187a8ef279bc014ec368c27a920da2024d2a711109bfbe3440585d5cf27ad9"
+dependencies = [
+ "either",
+ "lazy_static",
+ "libc",
+]
+
 [[package]]
 name = "winapi"
 version = "0.3.9"

README.md

@@ -13,7 +13,9 @@ myself agreeing with most of kakoune's design decisions.
 For more information, see the [website](https://helix-editor.com) or
 [documentation](https://docs.helix-editor.com/).
 
-All shortcuts/keymaps can be found [in the documentation on the website](https://docs.helix-editor.com/keymap.html)
+All shortcuts/keymaps can be found [in the documentation on the website](https://docs.helix-editor.com/keymap.html).
+
+[Troubleshooting](https://github.com/helix-editor/helix/wiki/Troubleshooting)
 
 # Features
 
@@ -28,7 +30,7 @@ It's a terminal-based editor first, but I'd like to explore a custom renderer
 # Installation
 
 Note: Only certain languages have indentation definitions at the moment. Check
-`runtime/<lang>/` for `indents.toml`.
+`runtime/queries/<lang>/` for `indents.toml`.
 
 We provide packaging for various distributions, but here's a quick method to
 build from source.
@@ -41,24 +43,17 @@ cargo install --path helix-term
 
 This will install the `hx` binary to `$HOME/.cargo/bin`.
 
-Now copy the `runtime/` directory somewhere. Helix will by default look for the runtime
-inside the config directory or the same directory as executable, but that can be overriden
+Helix also needs it's runtime files so make sure to copy/symlink the `runtime/` directory into the
+config directory (for example `~/.config/helix/runtime` on Linux/macOS). This location can be overriden
 via the `HELIX_RUNTIME` environment variable.
 
-> NOTE: running via cargo doesn't require setting explicit `HELIX_RUNTIME` path, it will automatically
+Packages already solve this for you by wrapping the `hx` binary with a wrapper
+that sets the variable to the install dir.
+
+> NOTE: running via cargo also doesn't require setting explicit `HELIX_RUNTIME` path, it will automatically
 > detect the `runtime` directory in the project root.
 
-If you want to embed the `runtime/` directory into the Helix binary you can build
-it with:
-
-```
-cargo install --path helix-term --features "embed_runtime"
-```
-
-## Arch Linux
-There are two packages available from AUR:
- - `helix-bin`: contains prebuilt binary from GitHub releases
- - `helix-git`: builds the master branch of this repository
+[![Packaging status](https://repology.org/badge/vertical-allrepos/helix.svg)](https://repology.org/project/helix/versions)
 
 ## MacOS
 Helix can be installed on MacOS through homebrew via:
@@ -74,7 +69,7 @@ Contributors are very welcome! **No contribution is too small and all contributi
 
 Some suggestions to get started:
 
-- You can look at the [good first issue](https://github.com/helix-editor/helix/labels/good%20first%20issue) label on the issue tracker.
+- You can look at the [good first issue](https://github.com/helix-editor/helix/labels/E-easy) label on the issue tracker.
 - Help with packaging on various distributions needed!
 - To use print debugging to the `~/.cache/helix/helix.log` file, you must:
   * Print using `log::info!`, `warn!`, or `error!`. (`log::info!("helix!")`)

TODO.md

@@ -1,8 +1,3 @@
-- Refactor tree-sitter-highlight to work like the atom one, recomputing partial tree updates.
-
-------
-
-as you type completion!
 
 - tree sitter:
   - lua
@@ -15,47 +10,30 @@ as you type completion!
   - clojure
   - erlang
 
+as you type completion!
 - [ ] use signature_help_provider and completion_provider trigger characters in
     a hook to trigger signature help text / autocompletion
 - [ ] document.on_type provider triggers
 - [ ] completion isIncomplete support
 
-- [ ] scroll wheel support
-- [ ] matching bracket highlight
-
 1
 - [ ] respect view fullscreen flag
 - [ ] Implement marks (superset of Selection/Range)
 
-- [ ] nixos packaging
-
 - [ ] = for auto indent line/selection
 - [ ]  :x for closing buffers
-
 - [ ] repeat selection
 
-- [] jump to alt buffer
-
 - [ ] lsp: signature help
-- [x] lsp: hover
-- [ ] lsp: document symbols (nested/vec)
-- [ ] lsp: code actions
-- [ ] lsp: formatting
-- [x] lsp: goto
 
 - [ ] search: smart case by default: insensitive unless upper detected
 
-- [ ] move Compositor into tui
-
 2
-- [ ] surround bindings (select + surround ( wraps selection in parens )
 - [ ] macro recording
 - [ ] extend selection (treesitter select parent node) (replaces viw, vi(, va( etc )
-- [x] bracket pairs
-- [x] comment block (gcc)
 - [ ] selection align
 - [ ] store some state between restarts: file positions, prompt history
-- [ ] highlight matched characters in completion
+- [ ] highlight matched characters in picker
 
 3
 - [ ] diff mode with highlighting?

book/book.toml

@@ -4,6 +4,7 @@ language = "en"
 multilingual = false
 src = "src"
 theme = "colibri"
+edit-url-template = "https://github.com/helix-editor/helix/tree/master/book/{path}?mode=edit"
 
 [output.html]
 cname = "docs.helix-editor.com"

book/src/SUMMARY.md

@@ -3,5 +3,7 @@
 - [Installation](./install.md)
 - [Usage](./usage.md)
 - [Configuration](./configuration.md)
+  - [Themes](./themes.md)
   - [Keymap](./keymap.md)
+  - [Key Remapping](./remapping.md)
   - [Hooks](./hooks.md)

book/src/configuration.md

@@ -1,88 +1,11 @@
 # Configuration
 
-## Theme
+To override global configuration parameters create a `config.toml` file located in your config directory (i.e `~/.config/helix/config.toml`).
 
-Use a custom theme by placing a theme.toml in your config directory (i.e ~/.config/helix/theme.toml). The default theme.toml can be found [here](https://github.com/helix-editor/helix/blob/master/theme.toml), and user submitted themes [here](https://github.com/helix-editor/helix/blob/master/contrib/themes).
-
-Styles in theme.toml are specified of in the form:
+## LSP
 
+To display all language server messages in the status line add the following to your `config.toml`:
 ```toml
-key = { fg = "#ffffff", bg = "#000000", modifiers = ["bold", "italic"] }
+[lsp]
+display-messages = true
 ```
-
-where `name` represents what you want to style, `fg` specifies the foreground color, `bg` the background color, and `modifiers` is a list of style modifiers. `bg` and `modifiers` can be omitted to defer to the defaults.
-
-To specify only the foreground color:
-
-```toml
-key = "#ffffff"
-```
-
-if the key contains a dot `'.'`, it must be quoted to prevent it being parsed as a [dotted key](https://toml.io/en/v1.0.0#keys).
-
-```toml
-"key.key" = "#ffffff"
-```
-
-Possible modifiers:
-
-| modifier |
-| --- |
-| bold |
-| dim |
-| italic |
-| underlined |
-| slow\_blink |
-| rapid\_blink |
-| reversed |
-| hidden |
-| crossed\_out |
-
-Possible keys:
-
-| key | notes |
-| --- | --- |
-| attribute | |
-| keyword | |
-| keyword.directive | preprocessor directives (\#if in C) |
-| namespace | |
-| punctuation | |
-| punctuation.delimiter | |
-| operator | |
-| special | |
-| property | |
-| variable | |
-| variable.parameter | |
-| type | |
-| type.builtin | |
-| constructor | |
-| function | |
-| function.macro | |
-| function.builtin | |
-| comment | |
-| variable.builtin | |
-| constant | |
-| constant.builtin | |
-| string | |
-| number | |
-| escape | escaped characters |
-| label | used for lifetimes |
-| module | |
-| ui.background | |
-| ui.linenr | |
-| ui.statusline | |
-| ui.popup | |
-| ui.window | |
-| ui.help | |
-| ui.text | |
-| ui.text.focus | |
-| ui.menu.selected | |
-| ui.selection | for selections in the editing area |
-| warning | LSP warning |
-| error | LSP error |
-| info | LSP info |
-| hint | LSP hint |
-
-These keys match [tree-sitter scopes](https://tree-sitter.github.io/tree-sitter/syntax-highlighting#theme). We half-follow the common scopes from [macromates language grammars](https://macromates.com/manual/en/language_grammars) with some differences.
-
-For a given highlight produced, styling will be determined based on the longest matching theme key. So it's enough to provide function to highlight `function.macro` and `function.builtin` as well, but you can use more specific scopes to highlight specific cases differently.

book/src/install.md

@@ -2,11 +2,16 @@
 
 We provide pre-built binaries on the [GitHub Releases page](https://github.com/helix-editor/helix/releases).
 
+[![Packaging status](https://repology.org/badge/vertical-allrepos/helix.svg)](https://repology.org/project/helix/versions)
+
 ## OSX
 
-TODO: brew tap
+A Homebrew tap is available:
 
-Please use a pre-built binary release for the time being.
+```
+brew tap helix-editor/helix
+brew install helix
+```
 
 ## Linux
 
@@ -32,13 +37,6 @@ cargo install --path helix-term
 
 This will install the `hx` binary to `$HOME/.cargo/bin`.
 
-Now copy the `runtime/` directory somewhere. Helix will by default look for the
-runtime inside the same folder as the executable, but that can be overriden via
-the `HELIX_RUNTIME` environment variable.
-
-If you want to embed the `runtime/` directory into the Helix binary you can build
-it with:
-
-```
-cargo install --path helix-term --features "embed_runtime"
-```
+Helix also needs it's runtime files so make sure to copy/symlink the `runtime/` directory into the
+config directory (for example `~/.config/helix/runtime` on Linux/macOS). This location can be overriden
+via the `HELIX_RUNTIME` environment variable.

book/src/keymap.md

@@ -4,76 +4,98 @@
 
 ### Movement
 
+> NOTE: `f`, `F`, `t` and `T` are not confined to the current line.
+
 | Key          | Description                                                                |
 | -----        | -----------                                                                |
-| h, Left   | move left |
-| j, Down   | move down |
-| k, Up   | move up |
-| l, Right   | move right |
-| w   | move next word start |
-| b   | move previous word start |
-| e   | move next word end |
-| t   | find 'till next char |
-| f   | find next char |
-| T   | find 'till previous char |
-| F   | find previous char |
-| Home   | move to the start of the line |
-| End   | move to the end of the line |
-| m   | Jump to matching bracket |
-| PageUp | Move page up |
-| PageDown | Move page down |
-| ctrl-u | Move half page up |
-| ctrl-d | Move half page down |
-| ctrl-i | Jump forward on the jumplist TODO: conflicts tab |
-| ctrl-o | Jump backward on the jumplist |
-| v   | Enter select (extend) mode |
-| g   | Enter goto mode |
-| :   | Enter command mode |
-| z   | Enter view mode |
-| ctrl-w | Enter window mode (maybe will be remove for spc w w later) |
-| space   | Enter space mode |
-| K | Show documentation for the item under the cursor |
+| `h`, `Left`  | Move left                                                                  |
+| `j`, `Down`  | Move down                                                                  |
+| `k`, `Up`    | Move up                                                                    |
+| `l`, `Right` | Move right                                                                 |
+| `w`          | Move next word start                                                       |
+| `b`          | Move previous word start                                                   |
+| `e`          | Move next word end                                                         |
+| `W`          | Move next WORD start                                                       |
+| `B`          | Move previous WORD start                                                   |
+| `E`          | Move next WORD end                                                         |
+| `t`          | Find 'till next char                                                       |
+| `f`          | Find next char                                                             |
+| `T`          | Find 'till previous char                                                   |
+| `F`          | Find previous char                                                         |
+| `Home`       | Move to the start of the line                                              |
+| `End`        | Move to the end of the line                                                |
+| `PageUp`     | Move page up                                                               |
+| `PageDown`   | Move page down                                                             |
+| `Ctrl-u`     | Move half page up                                                          |
+| `Ctrl-d`     | Move half page down                                                        |
+| `Ctrl-i`     | Jump forward on the jumplist TODO: conflicts tab                           |
+| `Ctrl-o`     | Jump backward on the jumplist                                              |
+| `v`          | Enter [select (extend) mode](#select--extend-mode)                         |
+| `g`          | Enter [goto mode](#goto-mode)                                              |
+| `m`          | Enter [match mode](#match-mode)                                            |
+| `:`          | Enter command mode                                                         |
+| `z`          | Enter [view mode](#view-mode)                                              |
+| `Ctrl-w`     | Enter [window mode](#window-mode) (maybe will be remove for spc w w later) |
+| `Space`      | Enter [space mode](#space-mode)                                            |
+| `K`          | Show documentation for the item under the cursor                           |
 
 ### Changes
 
 | Key         | Description                                     |
 | -----       | -----------                                     |
-| r   | replace with a character |
-| R   | replace with yanked text |
-| i   | Insert before selection |
-| a   | Insert after selection (append) |
-| I   | Insert at the start of the line |
-| A   | Insert at the end of the line |
-| o   | Open new line below selection |
-| o   | Open new line above selection |
-| u   | Undo change |
-| U   | Redo change |
-| y   | Yank selection |
-| p   | Paste after selection |
-| P   | Paste before selection |
-| >   | Indent selection |
-| <   | Unindent selection |
-| =   | Format selection |
-| d   | Delete selection |
-| c   | Change selection (delete and enter insert mode) |
+| `r`         | Replace with a character                        |
+| `R`         | Replace with yanked text                        |
+| `~`         | Switch case of the selected text                |
+| `` ` ``     | Set the selected text to lower case             |
+| `` Alt-` `` | Set the selected text to upper case             |
+| `i`         | Insert before selection                         |
+| `a`         | Insert after selection (append)                 |
+| `I`         | Insert at the start of the line                 |
+| `A`         | Insert at the end of the line                   |
+| `o`         | Open new line below selection                   |
+| `O`         | Open new line above selection                   |
+| `u`         | Undo change                                     |
+| `U`         | Redo change                                     |
+| `y`         | Yank selection                                  |
+| `p`         | Paste after selection                           |
+| `P`         | Paste before selection                          |
+| `>`         | Indent selection                                |
+| `<`         | Unindent selection                              |
+| `=`         | Format selection                                |
+| `d`         | Delete selection                                |
+| `c`         | Change selection (delete and enter insert mode) |
 
 ### Selection manipulation
 
 | Key      | Description                                                       |
 | -----    | -----------                                                       |
-| s   | Select all regex matches inside selections |
-| S   | Split selection into subselections on regex matches |
-| alt-s   | Split selection on newlines |
-| ;   | Collapse selection onto a single cursor |
-| alt-;   | Flip selection cursor and anchor |
-| %   | Select entire file |
-| x   | Select current line |
-| X   | Extend to next line |
-| [   | Expand selection to parent syntax node TODO: pick a key |
-| J | join lines inside selection |
-| K | keep selections matching the regex TODO: overlapped by hover help |
-| space | keep only the primary selection TODO: overlapped by space mode |
-| ctrl-c | Comment/uncomment the selections |
+| `s`      | Select all regex matches inside selections                        |
+| `S`      | Split selection into subselections on regex matches               |
+| `Alt-s`  | Split selection on newlines                                       |
+| `;`      | Collapse selection onto a single cursor                           |
+| `Alt-;`  | Flip selection cursor and anchor                                  |
+| `C`      | Copy selection onto the next line                                 |
+| `Alt-C`  | Copy selection onto the previous line                             |
+| `(`      | Rotate main selection forward                                     |
+| `)`      | Rotate main selection backward                                    |
+| `Alt-(`  | Rotate selection contents forward                                 |
+| `Alt-)`  | Rotate selection contents backward                                |
+| `%`      | Select entire file                                                |
+| `x`      | Select current line, if already selected, extend to next line     |
+| `X`      | Extend selection to line bounds (line-wise selection)             |
+|          | Expand selection to parent syntax node TODO: pick a key           |
+| `J`      | Join lines inside selection                                       |
+| `K`      | Keep selections matching the regex TODO: overlapped by hover help |
+| `Space`  | Keep only the primary selection TODO: overlapped by space mode    |
+| `Ctrl-c` | Comment/uncomment the selections                                  |
+
+### Insert Mode
+
+| Key      | Description           |
+| -----    | -----------           |
+| `Escape` | Switch to normal mode |
+| `Ctrl-x` | Autocomplete          |
+| `Ctrl-w` | Delete previous word  |
 
 ### Search
 
@@ -82,10 +104,10 @@ in reverse, or searching via smartcase.
 
 | Key   | Description                                 |
 | ----- | -----------                                 |
-| /   | Search for regex pattern |
-| n   | Select next search match |
-| N   | Add next search match to selection |
-| *   | Use current selection as the search pattern |
+| `/`   | Search for regex pattern                    |
+| `n`   | Select next search match                    |
+| `N`   | Add next search match to selection          |
+| `*`   | Use current selection as the search pattern |
 
 ### Diagnostics
 
@@ -94,10 +116,10 @@ in reverse, or searching via smartcase.
 
 | Key   | Description                        |
 | ----- | -----------                        |
-| [d | Go to previous diagnostic |
-| ]d | Go to next diagnostic |
-| [D | Go to first diagnostic in document |
-| ]D | Go to last diagnostic in document |
+| `[d`  | Go to previous diagnostic          |
+| `]d`  | Go to next diagnostic              |
+| `[D`  | Go to first diagnostic in document |
+| `]D`  | Go to last diagnostic in document  |
 
 ## Select / extend mode
 
@@ -114,12 +136,12 @@ the selection.
 
 | Key       | Description                                               |
 | -----     | -----------                                               |
-| z , c | Vertically center the line |
-| t   | Align the line to the top of the screen |
-| b   | Align the line to the bottom of the screen |
-| m   | Align the line to the middle of the screen (horizontally) |
-| j   | Scroll the view downwards |
-| k   | Scroll the view upwards |
+| `z` , `c` | Vertically center the line                                |
+| `t`       | Align the line to the top of the screen                   |
+| `b`       | Align the line to the bottom of the screen                |
+| `m`       | Align the line to the middle of the screen (horizontally) |
+| `j`       | Scroll the view downwards                                 |
+| `k`       | Scroll the view upwards                                   |
 
 ## Goto mode
 
@@ -129,19 +151,34 @@ Jumps to various locations.
 
 | Key   | Description                                      |
 | ----- | -----------                                      |
-| g   | Go to the start of the file |
-| e   | Go to the end of the file |
-| h   | Go to the start of the line |
-| l   | Go to the end of the line |
-| s   | Go to first non-whitespace character of the line |
-| t   | Go to the top of the screen |
-| m   | Go to the middle of the screen |
-| b   | Go to the bottom of the screen |
-| d   | Go to definition |
-| y   | Go to type definition |
-| r   | Go to references |
-| i   | Go to implementation |
-| a   | Go to the last accessed/alternate file |
+| `g`   | Go to the start of the file                      |
+| `e`   | Go to the end of the file                        |
+| `h`   | Go to the start of the line                      |
+| `l`   | Go to the end of the line                        |
+| `s`   | Go to first non-whitespace character of the line |
+| `t`   | Go to the top of the screen                      |
+| `m`   | Go to the middle of the screen                   |
+| `b`   | Go to the bottom of the screen                   |
+| `d`   | Go to definition                                 |
+| `y`   | Go to type definition                            |
+| `r`   | Go to references                                 |
+| `i`   | Go to implementation                             |
+| `a`   | Go to the last accessed/alternate file           |
+
+## Match mode
+
+Enter this mode using `m` from normal mode. See the relavant section
+in [Usage](./usage.md) for an explanation about [surround](./usage.md#surround)
+and [textobject](./usage.md#textobject) usage.
+
+| Key              | Description                                     |
+| -----            | -----------                                     |
+| `m`              | Goto matching bracket                           |
+| `s` `<char>`     | Surround current selection with `<char>`        |
+| `r` `<from><to>` | Replace surround character `<from>` with `<to>` |
+| `d` `<char>`     | Delete surround character `<char>`              |
+| `a` `<object>`   | Select around textobject                        |
+| `i` `<object>`   | Select inside textobject                        |
 
 ## Object mode
 
@@ -153,10 +190,10 @@ This layer is similar to vim keybindings as kakoune does not support window.
 
 | Key           | Description             |
 | -----         | -------------           |
-| w, ctrl-w | Switch to next window |
-| v, ctrl-v | Vertical right split |
-| h, ctrl-h | Horizontal bottom split |
-| q, ctrl-q | Close current window |
+| `w`, `Ctrl-w` | Switch to next window   |
+| `v`, `Ctrl-v` | Vertical right split    |
+| `h`, `Ctrl-h` | Horizontal bottom split |
+| `q`, `Ctrl-q` | Close current window    |
 
 ## Space mode
 
@@ -164,8 +201,29 @@ This layer is a kludge of mappings I had under leader key in neovim.
 
 | Key     | Description                                                           |
 | -----   | -----------                                                           |
-| f   | Open file picker |
-| b   | Open buffer picker |
-| s   | Open symbol picker (current document)|
-| w   | Enter window mode |
-| space   | Keep primary selection TODO: it's here because space mode replaced it |
+| `f`     | Open file picker                                                      |
+| `b`     | Open buffer picker                                                    |
+| `s`     | Open symbol picker (current document)                                 |
+| `a`     | Apply code action                                                     |
+| `'`     | Open last fuzzy picker                                                |
+| `w`     | Enter [window mode](#window-mode)                                     |
+| `space` | Keep primary selection TODO: it's here because space mode replaced it |
+| `p`     | Paste system clipboard after selections                               |
+| `P`     | Paste system clipboard before selections                              |
+| `y`     | Join and yank selections to clipboard                                 |
+| `Y`     | Yank main selection to clipboard                                      |
+| `R`     | Replace selections by clipboard contents                              |
+
+# Picker
+
+Keys to use within picker.
+
+| Key                | Description       |
+| -----              | -------------     |
+| `Up`, `Ctrl-p`     | Previous entry    |
+| `Down`, `Ctrl-n`   | Next entry        |
+| `Ctrl-space`       | Filter options    |
+| `Enter`            | Open selected     |
+| `Ctrl-h`           | Open horizontally |
+| `Ctrl-v`           | Open vertically   |
+| `Escape`, `Ctrl-c` | Close picker      |

book/src/remapping.md

@@ -0,0 +1,50 @@
+# Key Remapping
+
+One-way key remapping is temporarily supported via a simple TOML configuration
+file. (More powerful solutions such as rebinding via commands will be
+available in the feature).
+
+To remap keys, write a `config.toml` file in your `helix` configuration
+directory (default `~/.config/helix` in Linux systems) with a structure like
+this:
+
+```toml
+# At most one section each of 'keys.normal', 'keys.insert' and 'keys.select'
+[keys.normal]
+a = "move_char_left" # Maps the 'a' key to the move_char_left command
+w = "move_line_up" # Maps the 'w' key move_line_up
+C-S-esc = "extend_line" # Maps Control-Shift-Escape to extend_line
+
+[keys.insert]
+A-x = "normal_mode" # Maps Alt-X to enter normal mode
+```
+
+Control, Shift and Alt modifiers are encoded respectively with the prefixes
+`C-`, `S-` and `A-`. Special keys are encoded as follows:
+
+| Key name     | Representation |
+| ---          | ---            |
+| Backspace    | `"backspace"`  |
+| Space        | `"space"`      |
+| Return/Enter | `"ret"`        |
+| <            | `"lt"`         |
+| \>           | `"gt"`         |
+| \+           | `"plus"`       |
+| \-           | `"minus"`      |
+| ;            | `"semicolon"`  |
+| %            | `"percent"`    |
+| Left         | `"left"`       |
+| Right        | `"right"`      |
+| Up           | `"up"`         |
+| Home         | `"home"`       |
+| End          | `"end"`        |
+| Page         | `"pageup"`     |
+| Page         | `"pagedown"`   |
+| Tab          | `"tab"`        |
+| Back         | `"backtab"`    |
+| Delete       | `"del"`        |
+| Insert       | `"ins"`        |
+| Null         | `"null"`       |
+| Escape       | `"esc"`        |
+
+Commands can be found in the source code at [`helix-term/src/commands.rs`](https://github.com/helix-editor/helix/blob/master/helix-term/src/commands.rs)

book/src/themes.md

@@ -0,0 +1,122 @@
+# Themes
+
+First you'll need to place selected themes in your `themes` directory (i.e `~/.config/helix/themes`), the directory might have to be created beforehand.
+
+To use a custom theme add `theme = <name>` to your [`config.toml`](./configuration.md) or override it during runtime using `:theme <name>`.
+
+The default theme.toml can be found [here](https://github.com/helix-editor/helix/blob/master/theme.toml), and user submitted themes [here](https://github.com/helix-editor/helix/blob/master/runtime/themes). 
+
+## Creating a theme
+
+First create a file with the name of your theme as file name (i.e `mytheme.toml`) and place it in your `themes` directory (i.e `~/.config/helix/themes`).
+
+Each line in the theme file is specified as below:
+
+```toml
+key = { fg = "#ffffff", bg = "#000000", modifiers = ["bold", "italic"] }
+```
+
+where `key` represents what you want to style, `fg` specifies the foreground color, `bg` the background color, and `modifiers` is a list of style modifiers. `bg` and `modifiers` can be omitted to defer to the defaults.
+
+To specify only the foreground color:
+
+```toml
+key = "#ffffff"
+```
+
+if the key contains a dot `'.'`, it must be quoted to prevent it being parsed as a [dotted key](https://toml.io/en/v1.0.0#keys).
+
+```toml
+"key.key" = "#ffffff"
+```
+
+Possible modifiers:
+
+| Modifier       |
+| ---            |
+| `bold`         |
+| `dim`          |
+| `italic`       |
+| `underlined`   |
+| `slow\_blink`  |
+| `rapid\_blink` |
+| `reversed`     |
+| `hidden`       |
+| `crossed\_out` |
+
+Possible keys:
+
+| Key                      | Notes                               |
+| ---                      | ---                                 |
+| `attribute`              |                                     |
+| `keyword`                |                                     |
+| `keyword.directive`      | Preprocessor directives (\#if in C) |
+| `keyword.control`        | Control flow                        |
+| `namespace`              |                                     |
+| `punctuation`            |                                     |
+| `punctuation.delimiter`  |                                     |
+| `operator`               |                                     |
+| `special`                |                                     |
+| `property`               |                                     |
+| `variable`               |                                     |
+| `variable.parameter`     |                                     |
+| `type`                   |                                     |
+| `type.builtin`           |                                     |
+| `type.enum.variant`      | Enum variants                       |
+| `constructor`            |                                     |
+| `function`               |                                     |
+| `function.macro`         |                                     |
+| `function.builtin`       |                                     |
+| `comment`                |                                     |
+| `variable.builtin`       |                                     |
+| `constant`               |                                     |
+| `constant.builtin`       |                                     |
+| `string`                 |                                     |
+| `number`                 |                                     |
+| `escape`                 | Escaped characters                  |
+| `label`                  | For lifetimes                       |
+| `module`                 |                                     |
+| `ui.background`          |                                     |
+| `ui.cursor`              |                                     |
+| `ui.cursor.insert`       |                                     |
+| `ui.cursor.select`       |                                     |
+| `ui.cursor.match`        | Matching bracket etc.               |
+| `ui.cursor.primary`      | Cursor with primary selection       |
+| `ui.linenr`              |                                     |
+| `ui.linenr.selected`     |                                     |
+| `ui.statusline`          |                                     |
+| `ui.statusline.inactive` |                                     |
+| `ui.popup`               |                                     |
+| `ui.window`              |                                     |
+| `ui.help`                |                                     |
+| `ui.text`                |                                     |
+| `ui.text.focus`          |                                     |
+| `ui.menu.selected`       |                                     |
+| `ui.selection`           | For selections in the editing area  |
+| `ui.selection.primary`   |                                     |
+| `warning`                | LSP warning                         |
+| `error`                  | LSP error                           |
+| `info`                   | LSP info                            |
+| `hint`                   | LSP hint                            |
+
+These keys match [tree-sitter scopes](https://tree-sitter.github.io/tree-sitter/syntax-highlighting#theme). We half-follow the common scopes from [macromates language grammars](https://macromates.com/manual/en/language_grammars) with some differences.
+
+For a given highlight produced, styling will be determined based on the longest matching theme key. So it's enough to provide function to highlight `function.macro` and `function.builtin` as well, but you can use more specific scopes to highlight specific cases differently.
+
+## Color palettes
+
+You can define a palette of named colors, and refer to them from the
+configuration values in your theme. To do this, add a table called
+`palette` to your theme file:
+
+```toml
+ui.background = "white"
+ui.text = "black"
+
+[palette]
+white = "#ffffff"
+black = "#000000"
+```
+
+Remember that the `[palette]` table includes all keys after its header,
+so you should define the palette after normal theme options.

book/src/usage.md

@@ -1 +1,42 @@
 # Usage
+
+(Currently not fully documented, see the [keymappings](./keymap.md) list for more.)
+
+## Surround
+
+Functionality similar to [vim-surround](https://github.com/tpope/vim-surround) is built into
+helix. The keymappings have been inspired from [vim-sandwich](https://github.com/machakann/vim-sandwich):
+
+![surround demo](https://user-images.githubusercontent.com/23398472/122865801-97073180-d344-11eb-8142-8f43809982c6.gif)
+
+- `ms` - Add surround characters
+- `mr` - Replace surround characters
+- `md` - Delete surround characters
+
+`ms` acts on a selection, so select the text first and use `ms<char>`. `mr` and `md` work
+on the closest pairs found and selections are not required; use counts to act in outer pairs.
+
+It can also act on multiple seletions (yay!). For example, to change every occurance of `(use)` to `[use]`:
+
+- `%` to select the whole file
+- `s` to split the selections on a search term
+- Input `use` and hit Enter
+- `mr([` to replace the parens with square brackets
+
+Multiple characters are currently not supported, but planned.
+
+## Textobjects
+
+Currently supported: `word`, `surround`.
+
+![textobject-demo](https://user-images.githubusercontent.com/23398472/124231131-81a4bb00-db2d-11eb-9d10-8e577ca7b177.gif)
+
+- `ma` - Select around the object (`va` in vim, `<alt-a>` in kakoune)
+- `mi` - Select inside the object (`vi` in vim, `<alt-i>` in kakoune)
+
+| Key after `mi` or `ma` | Textobject selected      |
+| ---                    | ---                      |
+| `w`                    | Word                     |
+| `(`, `[`, `'`, etc     | Specified surround pairs |
+
+Textobjects based on treesitter, like `function`, `class`, etc are planned.

contrib/themes

@@ -0,0 +1 @@
+../runtime/themes

flake.lock

@@ -2,11 +2,11 @@
   "nodes": {
     "devshell": {
       "locked": {
-        "lastModified": 1622711433,
-        "narHash": "sha256-rGjXz7FA7HImAT3TtoqwecByLO5yhVPSwPdaYPBFRQw=",
+        "lastModified": 1625086391,
+        "narHash": "sha256-IpNPv1v8s4L3CoxhwcgZIitGpcrnNgnj09X7TA0QV3k=",
         "owner": "numtide",
         "repo": "devshell",
-        "rev": "1f4fb67b662b65fa7cfe696fc003fcc1e8f7cc36",
+        "rev": "4b5ac7cf7d9a1cc60b965bb51b59922f2210cbc7",
         "type": "github"
       },
       "original": {
@@ -18,11 +18,11 @@
     "flakeCompat": {
       "flake": false,
       "locked": {
-        "lastModified": 1606424373,
-        "narHash": "sha256-oq8d4//CJOrVj+EcOaSXvMebvuTkmBJuT5tzlfewUnQ=",
+        "lastModified": 1627913399,
+        "narHash": "sha256-hY8g6H2KFL8ownSiFeMOjwPC8P0ueXpCVEbxgda3pko=",
         "owner": "edolstra",
         "repo": "flake-compat",
-        "rev": "99f1c2157fba4bfe6211a321fd0ee43199025dbf",
+        "rev": "12c64ca55c1014cdc1b16ed5a804aa8576601ff2",
         "type": "github"
       },
       "original": {
@@ -31,24 +31,6 @@
         "type": "github"
       }
     },
-    "helix": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1623545930,
-        "narHash": "sha256-14ASoYbxXHU/qPGctiUymb4fMRCoih9c7YujjxqEkdU=",
-        "ref": "master",
-        "rev": "9640ed1425f2db904fb42cd0c54dc6fbc05ca292",
-        "revCount": 821,
-        "submodules": true,
-        "type": "git",
-        "url": "https://github.com/helix-editor/helix.git"
-      },
-      "original": {
-        "submodules": true,
-        "type": "git",
-        "url": "https://github.com/helix-editor/helix.git"
-      }
-    },
     "nixCargoIntegration": {
       "inputs": {
         "devshell": "devshell",
@@ -58,11 +40,11 @@
         "rustOverlay": "rustOverlay"
       },
       "locked": {
-        "lastModified": 1623591988,
-        "narHash": "sha256-a8E5LYKxYjHmBWZsFxKnCBVGsWHFEWrKjeAJkplWrfI=",
+        "lastModified": 1628489367,
+        "narHash": "sha256-ADYKHf8aPo1qTw1J+eqVprnEbH8lES0yZamD/yM7RAM=",
         "owner": "yusdacra",
         "repo": "nix-cargo-integration",
-        "rev": "8254b71eddd4e85173eddc189174b873fad85360",
+        "rev": "0dc8383aae5f791a48e34120edb04670b947dc0b",
         "type": "github"
       },
       "original": {
@@ -73,11 +55,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1623324058,
-        "narHash": "sha256-Jm9GUTXdjXz56gWDKy++EpFfjrBaxqXlLvTLfgEi8lo=",
+        "lastModified": 1628465643,
+        "narHash": "sha256-QSNw9bDq9uGUniQQtakRuw4m21Jxugm23SXLVgEV4DM=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "432fc2d9a67f92e05438dff5fdc2b39d33f77997",
+        "rev": "6ef4f522d63f22b40004319778761040d3197390",
         "type": "github"
       },
       "original": {
@@ -90,7 +72,6 @@
     "root": {
       "inputs": {
         "flakeCompat": "flakeCompat",
-        "helix": "helix",
         "nixCargoIntegration": "nixCargoIntegration",
         "nixpkgs": "nixpkgs"
       }
@@ -98,11 +79,11 @@
     "rustOverlay": {
       "flake": false,
       "locked": {
-        "lastModified": 1623550815,
-        "narHash": "sha256-RumRrkE6OTJDndHV4qZNZv8kUGnzwRHZQSyzx29r6/g=",
+        "lastModified": 1628475192,
+        "narHash": "sha256-A32shcfPMCll7psCS0OBxVCkA+PKfeWvmU4y9lgNZzU=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "9824f142cbd7bc3e2a92eefbb79addfff8704cd3",
+        "rev": "56a8ddb827cbe7a914be88f4a52998a5f93ff468",
         "type": "github"
       },
       "original": {

flake.nix

@@ -11,15 +11,9 @@
       url = "github:edolstra/flake-compat";
       flake = false;
     };
-    helix = {
-      url = "https://github.com/helix-editor/helix.git";
-      type = "git";
-      flake = false;
-      submodules = true;
-    };
   };
 
-  outputs = inputs@{ nixCargoIntegration, helix, ... }:
+  outputs = inputs@{ self, nixCargoIntegration, ... }:
     nixCargoIntegration.lib.makeOutputs {
       root = ./.;
       buildPlatform = "crate2nix";
@@ -29,28 +23,66 @@
       defaultOutputs = { app = "hx"; package = "helix"; };
       overrides = {
         crateOverrides = common: _: {
-          helix-term = prev: { buildInputs = (prev.buildInputs or [ ]) ++ [ common.cCompiler.cc.lib ]; };
-          # link runtime since helix-core expects it because of embed_runtime feature
-          helix-core = _: { preConfigure = "ln -s ${common.root + "/runtime"} ../runtime"; };
+          helix-term = prev: {
+            # link languages and theme toml files since helix-term expects them (for tests)
+            preConfigure = "ln -s ${common.root}/{languages.toml,theme.toml} ..";
+            buildInputs = (prev.buildInputs or [ ]) ++ [ common.cCompiler.cc.lib ];
+          };
           # link languages and theme toml files since helix-view expects them
           helix-view = _: { preConfigure = "ln -s ${common.root}/{languages.toml,theme.toml} .."; };
           helix-syntax = prev: {
-            src = common.pkgs.runCommand prev.src.name { } ''
+            src =
+              let
+                pkgs = common.pkgs;
+                helix = pkgs.fetchgit {
+                  url = "https://github.com/helix-editor/helix.git";
+                  rev = "d4bd5b37669708361a0a6cd2917464b010e6b7f5";
+                  fetchSubmodules = true;
+                  sha256 = "sha256-KayR7K7UC0mT6EjHsZsCYY9IVDJzft63fGpPKGSY8nQ=";
+                };
+              in
+              pkgs.runCommand prev.src.name { } ''
                 mkdir -p $out
                 ln -s ${prev.src}/* $out
                 ln -sf ${helix}/helix-syntax/languages $out
               '';
+            preConfigure = "mkdir -p ../runtime/grammars";
+            postInstall = "cp -r ../runtime $out/runtime";
           };
         };
+        mainBuild = common: prev:
+          let
+            inherit (common) pkgs lib;
+            helixSyntax = lib.buildCrate {
+              root = self;
+              memberName = "helix-syntax";
+              defaultCrateOverrides = {
+                helix-syntax = common.crateOverrides.helix-syntax;
+              };
+              release = false;
+            };
+            runtimeDir = pkgs.runCommand "helix-runtime" { } ''
+              mkdir -p $out
+              ln -s ${common.root}/runtime/* $out
+              ln -sf ${helixSyntax}/runtime/grammars $out
+            '';
+          in
+          lib.optionalAttrs (common.memberName == "helix-term") {
+            nativeBuildInputs = [ pkgs.makeWrapper ];
+            postFixup = ''
+              if [ -f "$out/bin/hx" ]; then
+                wrapProgram "$out/bin/hx" --set HELIX_RUNTIME "${runtimeDir}"
+              fi
+            '';
+          };
         shell = common: prev: {
-          packages = prev.packages ++ (with common.pkgs; [ lld_10 lldb ]);
+          packages = prev.packages ++ (with common.pkgs; [ lld_10 lldb cargo-tarpaulin ]);
           env = prev.env ++ [
             { name = "HELIX_RUNTIME"; eval = "$PWD/runtime"; }
             { name = "RUST_BACKTRACE"; value = "1"; }
             { name = "RUSTFLAGS"; value = "-C link-arg=-fuse-ld=lld -C target-cpu=native"; }
           ];
         };
-        build = _: prev: { rootFeatures = prev.rootFeatures ++ [ "embed_runtime" ]; };
       };
     };
 }

helix-core/Cargo.toml

@@ -1,32 +1,38 @@
 [package]
 name = "helix-core"
-version = "0.2.0"
+version = "0.4.1"
 authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
 edition = "2018"
 license = "MPL-2.0"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
+description = "Helix editor core editing primitives"
+categories = ["editor"]
+repository = "https://github.com/helix-editor/helix"
+homepage = "https://helix-editor.com"
+include = ["src/**/*", "README.md"]
 
 [features]
-embed_runtime = ["rust-embed"]
 
 [dependencies]
-helix-syntax = { path = "../helix-syntax" }
+helix-syntax = { version = "0.4", path = "../helix-syntax" }
 
-ropey = "1.2"
+ropey = "1.3"
 smallvec = "1.4"
 tendril = "0.4.2"
-unicode-segmentation = "1.7.1"
+unicode-segmentation = "1.8"
 unicode-width = "0.1"
-unicode-general-category = "0.4.0"
+unicode-general-category = "0.4"
 # slab = "0.4.2"
 tree-sitter = "0.19"
 once_cell = "1.8"
+arc-swap = "1"
 regex = "1"
 
 serde = { version = "1.0", features = ["derive"] }
 toml = "0.5"
 
+similar = "1.3"
+
 etcetera = "0.3"
-rust-embed = { version = "5.9.0", optional = true }
+
+[dev-dependencies]
+quickcheck = { version = "1", default-features = false }

helix-core/src/auto_pairs.rs

@@ -12,7 +12,7 @@ pub const PAIRS: &[(char, char)] = &[
     ('`', '`'),
 ];
 
-const CLOSE_BEFORE: &str = ")]}'\":;> \n"; // includes space and newline
+const CLOSE_BEFORE: &str = ")]}'\":;> \n\r\u{000B}\u{000C}\u{0085}\u{2028}\u{2029}"; // includes space and newlines
 
 // insert hook:
 // Fn(doc, selection, char) => Option<Transaction>
@@ -139,7 +139,7 @@ fn handle_close(doc: &Rope, selection: &Selection, _open: char, close: char) ->
 }
 
 // handle cases where open and close is the same, or in triples ("""docstring""")
-fn handle_same(doc: &Rope, selection: &Selection, token: char) -> Option<Transaction> {
+fn handle_same(_doc: &Rope, _selection: &Selection, _token: char) -> Option<Transaction> {
     // if not cursor but selection, wrap
     // let next = next char
 

helix-core/src/chars.rs

@@ -0,0 +1,133 @@
+use crate::LineEnding;
+
+#[derive(Debug, Eq, PartialEq)]
+pub enum CharCategory {
+    Whitespace,
+    Eol,
+    Word,
+    Punctuation,
+    Unknown,
+}
+
+#[inline]
+pub fn categorize_char(ch: char) -> CharCategory {
+    if char_is_line_ending(ch) {
+        CharCategory::Eol
+    } else if ch.is_whitespace() {
+        CharCategory::Whitespace
+    } else if char_is_word(ch) {
+        CharCategory::Word
+    } else if char_is_punctuation(ch) {
+        CharCategory::Punctuation
+    } else {
+        CharCategory::Unknown
+    }
+}
+
+/// Determine whether a character is a line ending.
+#[inline]
+pub fn char_is_line_ending(ch: char) -> bool {
+    LineEnding::from_char(ch).is_some()
+}
+
+/// Determine whether a character qualifies as (non-line-break)
+/// whitespace.
+#[inline]
+pub fn char_is_whitespace(ch: char) -> bool {
+    // TODO: this is a naive binary categorization of whitespace
+    // characters.  For display, word wrapping, etc. we'll need a better
+    // categorization based on e.g. breaking vs non-breaking spaces
+    // and whether they're zero-width or not.
+    match ch {
+        //'\u{1680}' | // Ogham Space Mark (here for completeness, but usually displayed as a dash, not as whitespace)
+        '\u{0009}' | // Character Tabulation
+        '\u{0020}' | // Space
+        '\u{00A0}' | // No-break Space
+        '\u{180E}' | // Mongolian Vowel Separator
+        '\u{202F}' | // Narrow No-break Space
+        '\u{205F}' | // Medium Mathematical Space
+        '\u{3000}' | // Ideographic Space
+        '\u{FEFF}'   // Zero Width No-break Space
+        => true,
+
+        // En Quad, Em Quad, En Space, Em Space, Three-per-em Space,
+        // Four-per-em Space, Six-per-em Space, Figure Space,
+        // Punctuation Space, Thin Space, Hair Space, Zero Width Space.
+        ch if ('\u{2000}' ..= '\u{200B}').contains(&ch) => true,
+
+        _ => false,
+    }
+}
+
+#[inline]
+pub fn char_is_punctuation(ch: char) -> bool {
+    use unicode_general_category::{get_general_category, GeneralCategory};
+
+    matches!(
+        get_general_category(ch),
+        GeneralCategory::OtherPunctuation
+            | GeneralCategory::OpenPunctuation
+            | GeneralCategory::ClosePunctuation
+            | GeneralCategory::InitialPunctuation
+            | GeneralCategory::FinalPunctuation
+            | GeneralCategory::ConnectorPunctuation
+            | GeneralCategory::DashPunctuation
+            | GeneralCategory::MathSymbol
+            | GeneralCategory::CurrencySymbol
+            | GeneralCategory::ModifierSymbol
+    )
+}
+
+#[inline]
+pub fn char_is_word(ch: char) -> bool {
+    ch.is_alphanumeric() || ch == '_'
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_categorize() {
+        const EOL_TEST_CASE: &'static str = "\n\r\u{000B}\u{000C}\u{0085}\u{2028}\u{2029}";
+        const WORD_TEST_CASE: &'static str =
+            "_hello_world_あいうえおー1234567890１２３４５６７８９０";
+        const PUNCTUATION_TEST_CASE: &'static str =
+            "!\"#$%&\'()*+,-./:;<=>?@[\\]^`{|}~！”＃＄％＆’（）＊＋、。：；＜＝＞？＠「」＾｀｛｜｝～";
+        const WHITESPACE_TEST_CASE: &'static str = "  　   ";
+
+        for ch in EOL_TEST_CASE.chars() {
+            assert_eq!(CharCategory::Eol, categorize_char(ch));
+        }
+
+        for ch in WHITESPACE_TEST_CASE.chars() {
+            assert_eq!(
+                CharCategory::Whitespace,
+                categorize_char(ch),
+                "Testing '{}', but got `{:?}` instead of `Category::Whitespace`",
+                ch,
+                categorize_char(ch)
+            );
+        }
+
+        for ch in WORD_TEST_CASE.chars() {
+            assert_eq!(
+                CharCategory::Word,
+                categorize_char(ch),
+                "Testing '{}', but got `{:?}` instead of `Category::Word`",
+                ch,
+                categorize_char(ch)
+            );
+        }
+
+        for ch in PUNCTUATION_TEST_CASE.chars() {
+            assert_eq!(
+                CharCategory::Punctuation,
+                categorize_char(ch),
+                "Testing '{}', but got `{:?}` instead of `Category::Punctuation`",
+                ch,
+                categorize_char(ch)
+            );
+        }
+    }
+}

helix-core/src/comment.rs

@@ -1,17 +1,27 @@
 use crate::{
     find_first_non_whitespace_char, Change, Rope, RopeSlice, Selection, Tendril, Transaction,
 };
-use core::ops::Range;
 use std::borrow::Cow;
 
+/// Given text, a comment token, and a set of line indices, returns the following:
+/// - Whether the given lines should be considered commented
+///     - If any of the lines are uncommented, all lines are considered as such.
+/// - The lines to change for toggling comments
+///     - This is all provided lines excluding blanks lines.
+/// - The column of the comment tokens
+///     - Column of existing tokens, if the lines are commented; column to place tokens at otherwise.
+/// - The margin to the right of the comment tokens
+///     - Defaults to `1`. If any existing comment token is not followed by a space, changes to `0`.
 fn find_line_comment(
     token: &str,
     text: RopeSlice,
-    lines: Range<usize>,
-) -> (bool, Vec<usize>, usize) {
+    lines: impl IntoIterator<Item = usize>,
+) -> (bool, Vec<usize>, usize, usize) {
     let mut commented = true;
-    let mut skipped = Vec::new();
+    let mut to_change = Vec::new();
     let mut min = usize::MAX; // minimum col for find_first_non_whitespace_char
+    let mut margin = 1;
+    let token_len = token.chars().count();
     for line in lines {
         let line_slice = text.line(line);
         if let Some(pos) = find_first_non_whitespace_char(line_slice) {
@@ -29,47 +39,55 @@ fn find_line_comment(
                 // considered uncommented.
                 commented = false;
             }
-        } else {
-            // blank line
-            skipped.push(line);
+
+            // determine margin of 0 or 1 for uncommenting; if any comment token is not followed by a space,
+            // a margin of 0 is used for all lines.
+            if matches!(line_slice.get_char(pos + token_len), Some(c) if c != ' ') {
+                margin = 0;
+            }
+
+            // blank lines don't get pushed.
+            to_change.push(line);
         }
     }
-    (commented, skipped, min)
+    (commented, to_change, min, margin)
 }
 
 #[must_use]
-pub fn toggle_line_comments(doc: &Rope, selection: &Selection) -> Transaction {
+pub fn toggle_line_comments(doc: &Rope, selection: &Selection, token: Option<&str>) -> Transaction {
     let text = doc.slice(..);
-    let mut changes: Vec<Change> = Vec::new();
 
-    let token = "//";
+    let token = token.unwrap_or("//");
     let comment = Tendril::from(format!("{} ", token));
 
+    let mut lines: Vec<usize> = Vec::new();
+
+    let mut min_next_line = 0;
     for selection in selection {
-        let start = text.char_to_line(selection.from());
-        let end = text.char_to_line(selection.to());
-        let lines = start..end + 1;
-        let (commented, skipped, min) = find_line_comment(token, text, lines.clone());
+        let (start, end) = selection.line_range(text);
+        let start = start.max(min_next_line).min(text.len_lines());
+        let end = (end + 1).min(text.len_lines());
 
-        changes.reserve(end - start - skipped.len());
-
-        for line in lines {
-            if skipped.contains(&line) {
-                continue;
+        lines.extend(start..end);
+        min_next_line = end + 1;
     }
 
+    let (commented, to_change, min, margin) = find_line_comment(token, text, lines);
+
+    let mut changes: Vec<Change> = Vec::with_capacity(to_change.len());
+
+    for line in to_change {
         let pos = text.line_to_char(line) + min;
 
         if !commented {
             // comment line
-                changes.push((pos, pos, Some(comment.clone())))
+            changes.push((pos, pos, Some(comment.clone())));
         } else {
             // uncomment line
-                let margin = 1; // TODO: margin is hardcoded 1 but could easily be 0
-                changes.push((pos, pos + token.len() + margin, None))
-            }
+            changes.push((pos, pos + token.len() + margin, None));
         }
     }
+
     Transaction::change(doc, changes.into_iter())
 }
 
@@ -91,23 +109,32 @@ mod test {
         let text = state.doc.slice(..);
 
         let res = find_line_comment("//", text, 0..3);
-        // (commented = true, skipped = [line 1], min = col 2)
-        assert_eq!(res, (false, vec![1], 2));
+        // (commented = true, to_change = [line 0, line 2], min = col 2, margin = 1)
+        assert_eq!(res, (false, vec![0, 2], 2, 1));
 
         // comment
-        let transaction = toggle_line_comments(&state.doc, &state.selection);
+        let transaction = toggle_line_comments(&state.doc, &state.selection, None);
         transaction.apply(&mut state.doc);
-        state.selection = state.selection.clone().map(transaction.changes());
+        state.selection = state.selection.map(transaction.changes());
 
         assert_eq!(state.doc, "  // 1\n\n  // 2\n  // 3");
 
         // uncomment
-        let transaction = toggle_line_comments(&state.doc, &state.selection);
+        let transaction = toggle_line_comments(&state.doc, &state.selection, None);
         transaction.apply(&mut state.doc);
-        state.selection = state.selection.clone().map(transaction.changes());
+        state.selection = state.selection.map(transaction.changes());
+        assert_eq!(state.doc, "  1\n\n  2\n  3");
+
+        // 0 margin comments
+        state.doc = Rope::from("  //1\n\n  //2\n  //3");
+        // reset the selection.
+        state.selection = Selection::single(0, state.doc.len_chars() - 1);
+
+        let transaction = toggle_line_comments(&state.doc, &state.selection, None);
+        transaction.apply(&mut state.doc);
+        state.selection = state.selection.map(transaction.changes());
         assert_eq!(state.doc, "  1\n\n  2\n  3");
 
-        // TODO: account for no margin after comment
         // TODO: account for uncommenting with uneven comment indentation
     }
 }

helix-core/src/diff.rs

@@ -0,0 +1,68 @@
+use crate::{Rope, Transaction};
+
+/// Compares `old` and `new` to generate a [`Transaction`] describing
+/// the steps required to get from `old` to `new`.
+pub fn compare_ropes(old: &Rope, new: &Rope) -> Transaction {
+    // `similar` only works on contiguous data, so a `Rope` has
+    // to be temporarily converted into a `String`.
+    let old_converted = old.to_string();
+    let new_converted = new.to_string();
+
+    // A timeout is set so after 1 seconds, the algorithm will start
+    // approximating. This is especially important for big `Rope`s or
+    // `Rope`s that are extremely dissimilar to each other.
+    //
+    // Note: Ignore the clippy warning, as the trait bounds of
+    // `Transaction::change()` require an iterator implementing
+    // `ExactIterator`.
+    let mut config = similar::TextDiff::configure();
+    config.timeout(std::time::Duration::from_secs(1));
+
+    let diff = config.diff_chars(&old_converted, &new_converted);
+
+    // The current position of the change needs to be tracked to
+    // construct the `Change`s.
+    let mut pos = 0;
+    Transaction::change(
+        old,
+        diff.ops()
+            .iter()
+            .map(|op| op.as_tag_tuple())
+            .filter_map(|(tag, old_range, new_range)| {
+                // `old_pos..pos` is equivalent to `start..end` for where
+                // the change should be applied.
+                let old_pos = pos;
+                pos += old_range.end - old_range.start;
+
+                match tag {
+                    // Semantically, inserts and replacements are the same thing.
+                    similar::DiffTag::Insert | similar::DiffTag::Replace => {
+                        // This is the text from the `new` rope that should be
+                        // inserted into `old`.
+                        let text: &str = {
+                            let start = new.char_to_byte(new_range.start);
+                            let end = new.char_to_byte(new_range.end);
+                            &new_converted[start..end]
+                        };
+                        Some((old_pos, pos, Some(text.into())))
+                    }
+                    similar::DiffTag::Delete => Some((old_pos, pos, None)),
+                    similar::DiffTag::Equal => None,
+                }
+            }),
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    quickcheck::quickcheck! {
+        fn test_compare_ropes(a: String, b: String) -> bool {
+            let mut old = Rope::from(a);
+            let new = Rope::from(b);
+            compare_ropes(&old, &new).apply(&mut old);
+            old.to_string() == new.to_string()
+        }
+    }
+}

helix-core/src/graphemes.rs

@@ -71,6 +71,8 @@ pub fn nth_prev_grapheme_boundary(slice: RopeSlice, char_idx: usize, n: usize) -
 }
 
 /// Finds the previous grapheme boundary before the given char position.
+#[must_use]
+#[inline(always)]
 pub fn prev_grapheme_boundary(slice: RopeSlice, char_idx: usize) -> usize {
     nth_prev_grapheme_boundary(slice, char_idx, 1)
 }
@@ -117,11 +119,38 @@ pub fn nth_next_grapheme_boundary(slice: RopeSlice, char_idx: usize, n: usize) -
 }
 
 /// Finds the next grapheme boundary after the given char position.
+#[must_use]
+#[inline(always)]
 pub fn next_grapheme_boundary(slice: RopeSlice, char_idx: usize) -> usize {
     nth_next_grapheme_boundary(slice, char_idx, 1)
 }
 
+/// Returns the passed char index if it's already a grapheme boundary,
+/// or the next grapheme boundary char index if not.
+#[must_use]
+#[inline]
+pub fn ensure_grapheme_boundary_next(slice: RopeSlice, char_idx: usize) -> usize {
+    if char_idx == 0 {
+        char_idx
+    } else {
+        next_grapheme_boundary(slice, char_idx - 1)
+    }
+}
+
+/// Returns the passed char index if it's already a grapheme boundary,
+/// or the prev grapheme boundary char index if not.
+#[must_use]
+#[inline]
+pub fn ensure_grapheme_boundary_prev(slice: RopeSlice, char_idx: usize) -> usize {
+    if char_idx == slice.len_chars() {
+        char_idx
+    } else {
+        prev_grapheme_boundary(slice, char_idx + 1)
+    }
+}
+
 /// Returns whether the given char position is a grapheme boundary.
+#[must_use]
 pub fn is_grapheme_boundary(slice: RopeSlice, char_idx: usize) -> bool {
     // Bounds check
     debug_assert!(char_idx <= slice.len_chars());
@@ -207,6 +236,10 @@ impl<'a> Iterator for RopeGraphemes<'a> {
                     self.cur_chunk_start += self.cur_chunk.len();
                     self.cur_chunk = self.chunks.next().unwrap_or("");
                 }
+                Err(GraphemeIncomplete::PreContext(idx)) => {
+                    let (chunk, byte_idx, _, _) = self.text.chunk_at_byte(idx.saturating_sub(1));
+                    self.cursor.provide_context(chunk, byte_idx);
+                }
                 _ => unreachable!(),
             }
         }

helix-core/src/history.rs

@@ -1,7 +1,6 @@
 use crate::{ChangeSet, Rope, State, Transaction};
 use once_cell::sync::Lazy;
 use regex::Regex;
-use smallvec::{smallvec, SmallVec};
 use std::num::NonZeroUsize;
 use std::time::{Duration, Instant};
 
@@ -127,7 +126,6 @@ impl History {
         let last_child = current_revision.last_child?;
         self.current = last_child.get();
 
-        let last_child_revision = &self.revisions[last_child.get()];
         Some(&self.revisions[last_child.get()].transaction)
     }
 
@@ -377,21 +375,21 @@ mod test {
             if let Some(transaction) = history.undo() {
                 transaction.apply(&mut state.doc);
             }
-        };
+        }
 
         fn earlier(history: &mut History, state: &mut State, uk: UndoKind) {
             let txns = history.earlier(uk);
             for txn in txns {
                 txn.apply(&mut state.doc);
             }
-        };
+        }
 
         fn later(history: &mut History, state: &mut State, uk: UndoKind) {
             let txns = history.later(uk);
             for txn in txns {
                 txn.apply(&mut state.doc);
             }
-        };
+        }
 
         fn commit_change(
             history: &mut History,
@@ -402,7 +400,7 @@ mod test {
             let txn = Transaction::change(&state.doc, vec![change.clone()].into_iter());
             history.commit_revision_at_timestamp(&txn, &state, instant);
             txn.apply(&mut state.doc);
-        };
+        }
 
         let t0 = Instant::now();
         let t = |n| t0.checked_add(Duration::from_secs(n)).unwrap();

helix-core/src/indent.rs

@@ -1,13 +1,180 @@
 use crate::{
+    chars::{char_is_line_ending, char_is_whitespace},
     find_first_non_whitespace_char,
     syntax::{IndentQuery, LanguageConfiguration, Syntax},
-    tree_sitter::{Node, Tree},
+    tree_sitter::Node,
     Rope, RopeSlice,
 };
 
+/// Enum representing indentation style.
+///
+/// Only values 1-8 are valid for the `Spaces` variant.
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub enum IndentStyle {
+    Tabs,
+    Spaces(u8),
+}
+
+impl IndentStyle {
+    /// Creates an `IndentStyle` from an indentation string.
+    ///
+    /// For example, passing `"    "` (four spaces) will create `IndentStyle::Spaces(4)`.
+    #[allow(clippy::should_implement_trait)]
+    #[inline]
+    pub fn from_str(indent: &str) -> Self {
+        // XXX: do we care about validating the input more than this?  Probably not...?
+        debug_assert!(!indent.is_empty() && indent.len() <= 8);
+
+        if indent.starts_with(' ') {
+            IndentStyle::Spaces(indent.len() as u8)
+        } else {
+            IndentStyle::Tabs
+        }
+    }
+
+    #[inline]
+    pub fn as_str(&self) -> &'static str {
+        match *self {
+            IndentStyle::Tabs => "\t",
+            IndentStyle::Spaces(1) => " ",
+            IndentStyle::Spaces(2) => "  ",
+            IndentStyle::Spaces(3) => "   ",
+            IndentStyle::Spaces(4) => "    ",
+            IndentStyle::Spaces(5) => "     ",
+            IndentStyle::Spaces(6) => "      ",
+            IndentStyle::Spaces(7) => "       ",
+            IndentStyle::Spaces(8) => "        ",
+
+            // Unsupported indentation style.  This should never happen,
+            // but just in case fall back to two spaces.
+            IndentStyle::Spaces(n) => {
+                debug_assert!(n > 0 && n <= 8); // Always triggers. `debug_panic!()` wanted.
+                "  "
+            }
+        }
+    }
+}
+
+/// Attempts to detect the indentation style used in a document.
+///
+/// Returns the indentation style if the auto-detect confidence is
+/// reasonably high, otherwise returns `None`.
+pub fn auto_detect_indent_style(document_text: &Rope) -> Option<IndentStyle> {
+    // Build a histogram of the indentation *increases* between
+    // subsequent lines, ignoring lines that are all whitespace.
+    //
+    // Index 0 is for tabs, the rest are 1-8 spaces.
+    let histogram: [usize; 9] = {
+        let mut histogram = [0; 9];
+        let mut prev_line_is_tabs = false;
+        let mut prev_line_leading_count = 0usize;
+
+        // Loop through the lines, checking for and recording indentation
+        // increases as we go.
+        'outer: for line in document_text.lines().take(1000) {
+            let mut c_iter = line.chars();
+
+            // Is first character a tab or space?
+            let is_tabs = match c_iter.next() {
+                Some('\t') => true,
+                Some(' ') => false,
+
+                // Ignore blank lines.
+                Some(c) if char_is_line_ending(c) => continue,
+
+                _ => {
+                    prev_line_is_tabs = false;
+                    prev_line_leading_count = 0;
+                    continue;
+                }
+            };
+
+            // Count the line's total leading tab/space characters.
+            let mut leading_count = 1;
+            let mut count_is_done = false;
+            for c in c_iter {
+                match c {
+                    '\t' if is_tabs && !count_is_done => leading_count += 1,
+                    ' ' if !is_tabs && !count_is_done => leading_count += 1,
+
+                    // We stop counting if we hit whitespace that doesn't
+                    // qualify as indent or doesn't match the leading
+                    // whitespace, but we don't exit the loop yet because
+                    // we still want to determine if the line is blank.
+                    c if char_is_whitespace(c) => count_is_done = true,
+
+                    // Ignore blank lines.
+                    c if char_is_line_ending(c) => continue 'outer,
+
+                    _ => break,
+                }
+
+                // Bound the worst-case execution time for weird text files.
+                if leading_count > 256 {
+                    continue 'outer;
+                }
+            }
+
+            // If there was an increase in indentation over the previous
+            // line, update the histogram with that increase.
+            if (prev_line_is_tabs == is_tabs || prev_line_leading_count == 0)
+                && prev_line_leading_count < leading_count
+            {
+                if is_tabs {
+                    histogram[0] += 1;
+                } else {
+                    let amount = leading_count - prev_line_leading_count;
+                    if amount <= 8 {
+                        histogram[amount] += 1;
+                    }
+                }
+            }
+
+            // Store this line's leading whitespace info for use with
+            // the next line.
+            prev_line_is_tabs = is_tabs;
+            prev_line_leading_count = leading_count;
+        }
+
+        // Give more weight to tabs, because their presence is a very
+        // strong indicator.
+        histogram[0] *= 2;
+
+        histogram
+    };
+
+    // Find the most frequent indent, its frequency, and the frequency of
+    // the next-most frequent indent.
+    let indent = histogram
+        .iter()
+        .enumerate()
+        .max_by_key(|kv| kv.1)
+        .unwrap()
+        .0;
+    let indent_freq = histogram[indent];
+    let indent_freq_2 = *histogram
+        .iter()
+        .enumerate()
+        .filter(|kv| kv.0 != indent)
+        .map(|kv| kv.1)
+        .max()
+        .unwrap();
+
+    // Return the the auto-detected result if we're confident enough in its
+    // accuracy, based on some heuristics.
+    if indent_freq >= 1 && (indent_freq_2 as f64 / indent_freq as f64) < 0.66 {
+        Some(match indent {
+            0 => IndentStyle::Tabs,
+            _ => IndentStyle::Spaces(indent as u8),
+        })
+    } else {
+        None
+    }
+}
+
 /// To determine indentation of a newly inserted line, figure out the indentation at the last col
 /// of the previous line.
-
+#[allow(dead_code)]
 fn indent_level_for_line(line: RopeSlice, tab_width: usize) -> usize {
     let mut len = 0;
     for ch in line.chars() {
@@ -47,7 +214,7 @@ fn calculate_indentation(query: &IndentQuery, node: Option<Node>, newline: bool)
     // NOTE: can't use contains() on query because of comparing Vec<String> and &str
     // https://doc.rust-lang.org/std/vec/struct.Vec.html#method.contains
 
-    let mut increment: i32 = 0;
+    let mut increment: isize = 0;
 
     let mut node = match node {
         Some(node) => node,
@@ -93,17 +260,16 @@ fn calculate_indentation(query: &IndentQuery, node: Option<Node>, newline: bool)
         node = parent;
     }
 
-    assert!(increment >= 0);
-
-    increment as usize
+    increment.max(0) as usize
 }
 
+#[allow(dead_code)]
 fn suggested_indent_for_line(
     language_config: &LanguageConfiguration,
     syntax: Option<&Syntax>,
     text: RopeSlice,
     line_num: usize,
-    tab_width: usize,
+    _tab_width: usize,
 ) -> usize {
     if let Some(start) = find_first_non_whitespace_char(text.line(line_num)) {
         return suggested_indent_for_pos(
@@ -147,9 +313,30 @@ pub fn suggested_indent_for_pos(
     }
 }
 
+pub fn get_scopes(syntax: Option<&Syntax>, text: RopeSlice, pos: usize) -> Vec<&'static str> {
+    let mut scopes = Vec::new();
+    if let Some(syntax) = syntax {
+        let byte_start = text.char_to_byte(pos);
+        let node = match get_highest_syntax_node_at_bytepos(syntax, byte_start) {
+            Some(node) => node,
+            None => return scopes,
+        };
+
+        scopes.push(node.kind());
+
+        while let Some(parent) = node.parent() {
+            scopes.push(parent.kind())
+        }
+    }
+
+    scopes.reverse();
+    scopes
+}
+
 #[cfg(test)]
 mod test {
     use super::*;
+    use crate::Rope;
 
     #[test]
     fn test_indent_level() {
@@ -251,18 +438,19 @@ where
 
         let doc = Rope::from(doc);
         use crate::syntax::{
-            Configuration, IndentationConfiguration, Lang, LanguageConfiguration, Loader,
+            Configuration, IndentationConfiguration, LanguageConfiguration, Loader,
         };
         use once_cell::sync::OnceCell;
-        let loader = Loader::new(
-            Configuration {
+        let loader = Loader::new(Configuration {
             language: vec![LanguageConfiguration {
                 scope: "source.rust".to_string(),
                 file_types: vec!["rs".to_string()],
-                    language_id: Lang::Rust,
+                language_id: "Rust".to_string(),
                 highlight_config: OnceCell::new(),
+                config: None,
                 //
                 roots: vec![],
+                comment_token: None,
                 auto_format: false,
                 language_server: None,
                 indent: Some(IndentationConfiguration {
@@ -271,9 +459,7 @@ where
                 }),
                 indent_query: OnceCell::new(),
             }],
-            },
-            Vec::new(),
-        );
+        });
 
         // set runtime path so we can find the queries
         let mut runtime = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));

helix-core/src/lib.rs

@@ -1,10 +1,12 @@
-#![allow(unused)]
 pub mod auto_pairs;
+pub mod chars;
 pub mod comment;
 pub mod diagnostic;
+pub mod diff;
 pub mod graphemes;
 pub mod history;
 pub mod indent;
+pub mod line_ending;
 pub mod macros;
 pub mod match_brackets;
 pub mod movement;
@@ -14,9 +16,17 @@ pub mod register;
 pub mod search;
 pub mod selection;
 mod state;
+pub mod surround;
 pub mod syntax;
+pub mod textobject;
 mod transaction;
 
+pub mod unicode {
+    pub use unicode_general_category as category;
+    pub use unicode_segmentation as segmentation;
+    pub use unicode_width as width;
+}
+
 static RUNTIME_DIR: once_cell::sync::Lazy<std::path::PathBuf> =
     once_cell::sync::Lazy::new(runtime_dir);
 
@@ -48,8 +58,7 @@ pub fn find_root(root: Option<&str>) -> Option<std::path::PathBuf> {
     None
 }
 
-#[cfg(not(embed_runtime))]
-fn runtime_dir() -> std::path::PathBuf {
+pub fn runtime_dir() -> std::path::PathBuf {
     if let Ok(dir) = std::env::var("HELIX_RUNTIME") {
         return dir.into();
     }
@@ -88,17 +97,18 @@ pub fn cache_dir() -> std::path::PathBuf {
     path
 }
 
+pub use etcetera::home_dir;
+
 use etcetera::base_strategy::{choose_base_strategy, BaseStrategy};
 
-pub use ropey::{Rope, RopeSlice};
+pub use ropey::{Rope, RopeBuilder, RopeSlice};
 
 pub use tendril::StrTendril as Tendril;
 
-pub use unicode_general_category::get_general_category;
-
 #[doc(inline)]
 pub use {regex, tree_sitter};
 
+pub use graphemes::RopeGraphemes;
 pub use position::{coords_at_pos, pos_at_coords, Position};
 pub use selection::{Range, Selection};
 pub use smallvec::SmallVec;
@@ -107,4 +117,5 @@ pub use syntax::Syntax;
 pub use diagnostic::Diagnostic;
 pub use state::State;
 
+pub use line_ending::{LineEnding, DEFAULT_LINE_ENDING};
 pub use transaction::{Assoc, Change, ChangeSet, Operation, Transaction};

helix-core/src/line_ending.rs

@@ -0,0 +1,265 @@
+use crate::{Rope, RopeSlice};
+
+#[cfg(target_os = "windows")]
+pub const DEFAULT_LINE_ENDING: LineEnding = LineEnding::Crlf;
+#[cfg(not(target_os = "windows"))]
+pub const DEFAULT_LINE_ENDING: LineEnding = LineEnding::LF;
+
+/// Represents one of the valid Unicode line endings.
+#[derive(PartialEq, Copy, Clone, Debug)]
+pub enum LineEnding {
+    Crlf, // CarriageReturn followed by LineFeed
+    LF,   // U+000A -- LineFeed
+    VT,   // U+000B -- VerticalTab
+    FF,   // U+000C -- FormFeed
+    CR,   // U+000D -- CarriageReturn
+    Nel,  // U+0085 -- NextLine
+    LS,   // U+2028 -- Line Separator
+    PS,   // U+2029 -- ParagraphSeparator
+}
+
+impl LineEnding {
+    #[inline]
+    pub fn len_chars(&self) -> usize {
+        match self {
+            Self::Crlf => 2,
+            _ => 1,
+        }
+    }
+
+    #[inline]
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            Self::Crlf => "\u{000D}\u{000A}",
+            Self::LF => "\u{000A}",
+            Self::VT => "\u{000B}",
+            Self::FF => "\u{000C}",
+            Self::CR => "\u{000D}",
+            Self::Nel => "\u{0085}",
+            Self::LS => "\u{2028}",
+            Self::PS => "\u{2029}",
+        }
+    }
+
+    #[inline]
+    pub fn from_char(ch: char) -> Option<LineEnding> {
+        match ch {
+            '\u{000A}' => Some(LineEnding::LF),
+            '\u{000B}' => Some(LineEnding::VT),
+            '\u{000C}' => Some(LineEnding::FF),
+            '\u{000D}' => Some(LineEnding::CR),
+            '\u{0085}' => Some(LineEnding::Nel),
+            '\u{2028}' => Some(LineEnding::LS),
+            '\u{2029}' => Some(LineEnding::PS),
+            // Not a line ending
+            _ => None,
+        }
+    }
+
+    // Normally we'd want to implement the FromStr trait, but in this case
+    // that would force us into a different return type than from_char or
+    // or from_rope_slice, which would be weird.
+    #[allow(clippy::should_implement_trait)]
+    #[inline]
+    pub fn from_str(g: &str) -> Option<LineEnding> {
+        match g {
+            "\u{000D}\u{000A}" => Some(LineEnding::Crlf),
+            "\u{000A}" => Some(LineEnding::LF),
+            "\u{000B}" => Some(LineEnding::VT),
+            "\u{000C}" => Some(LineEnding::FF),
+            "\u{000D}" => Some(LineEnding::CR),
+            "\u{0085}" => Some(LineEnding::Nel),
+            "\u{2028}" => Some(LineEnding::LS),
+            "\u{2029}" => Some(LineEnding::PS),
+            // Not a line ending
+            _ => None,
+        }
+    }
+
+    #[inline]
+    pub fn from_rope_slice(g: &RopeSlice) -> Option<LineEnding> {
+        if let Some(text) = g.as_str() {
+            LineEnding::from_str(text)
+        } else {
+            // Non-contiguous, so it can't be a line ending.
+            // Specifically, Ropey guarantees that CRLF is always
+            // contiguous.  And the remaining line endings are all
+            // single `char`s, and therefore trivially contiguous.
+            None
+        }
+    }
+}
+
+#[inline]
+pub fn str_is_line_ending(s: &str) -> bool {
+    LineEnding::from_str(s).is_some()
+}
+
+/// Attempts to detect what line ending the passed document uses.
+pub fn auto_detect_line_ending(doc: &Rope) -> Option<LineEnding> {
+    // Return first matched line ending. Not all possible line endings
+    // are being matched, as they might be special-use only
+    for line in doc.lines().take(100) {
+        match get_line_ending(&line) {
+            None | Some(LineEnding::VT) | Some(LineEnding::FF) | Some(LineEnding::PS) => {}
+            ending => return ending,
+        }
+    }
+    None
+}
+
+/// Returns the passed line's line ending, if any.
+pub fn get_line_ending(line: &RopeSlice) -> Option<LineEnding> {
+    // Last character as str.
+    let g1 = line
+        .slice(line.len_chars().saturating_sub(1)..)
+        .as_str()
+        .unwrap();
+
+    // Last two characters as str, or empty str if they're not contiguous.
+    // It's fine to punt on the non-contiguous case, because Ropey guarantees
+    // that CRLF is always contiguous.
+    let g2 = line
+        .slice(line.len_chars().saturating_sub(2)..)
+        .as_str()
+        .unwrap_or("");
+
+    // First check the two-character case for CRLF, then check the single-character case.
+    LineEnding::from_str(g2).or_else(|| LineEnding::from_str(g1))
+}
+
+/// Returns the passed line's line ending, if any.
+pub fn get_line_ending_of_str(line: &str) -> Option<LineEnding> {
+    if line.ends_with("\u{000D}\u{000A}") {
+        Some(LineEnding::Crlf)
+    } else if line.ends_with('\u{000A}') {
+        Some(LineEnding::LF)
+    } else if line.ends_with('\u{000B}') {
+        Some(LineEnding::VT)
+    } else if line.ends_with('\u{000C}') {
+        Some(LineEnding::FF)
+    } else if line.ends_with('\u{000D}') {
+        Some(LineEnding::CR)
+    } else if line.ends_with('\u{0085}') {
+        Some(LineEnding::Nel)
+    } else if line.ends_with('\u{2028}') {
+        Some(LineEnding::LS)
+    } else if line.ends_with('\u{2029}') {
+        Some(LineEnding::PS)
+    } else {
+        None
+    }
+}
+
+/// Returns the char index of the end of the given line, not including its line ending.
+pub fn line_end_char_index(slice: &RopeSlice, line: usize) -> usize {
+    slice.line_to_char(line + 1)
+        - get_line_ending(&slice.line(line))
+            .map(|le| le.len_chars())
+            .unwrap_or(0)
+}
+
+/// Fetches line `line_idx` from the passed rope slice, sans any line ending.
+pub fn line_without_line_ending<'a>(slice: &'a RopeSlice, line_idx: usize) -> RopeSlice<'a> {
+    let start = slice.line_to_char(line_idx);
+    let end = line_end_char_index(slice, line_idx);
+    slice.slice(start..end)
+}
+
+/// Returns the char index of the end of the given RopeSlice, not including
+/// any final line ending.
+pub fn rope_end_without_line_ending(slice: &RopeSlice) -> usize {
+    slice.len_chars() - get_line_ending(slice).map(|le| le.len_chars()).unwrap_or(0)
+}
+
+#[cfg(test)]
+mod line_ending_tests {
+    use super::*;
+
+    #[test]
+    fn line_ending_autodetect() {
+        assert_eq!(
+            auto_detect_line_ending(&Rope::from_str("\n")),
+            Some(LineEnding::LF)
+        );
+        assert_eq!(
+            auto_detect_line_ending(&Rope::from_str("\r\n")),
+            Some(LineEnding::Crlf)
+        );
+        assert_eq!(auto_detect_line_ending(&Rope::from_str("hello")), None);
+        assert_eq!(auto_detect_line_ending(&Rope::from_str("")), None);
+        assert_eq!(
+            auto_detect_line_ending(&Rope::from_str("hello\nhelix\r\n")),
+            Some(LineEnding::LF)
+        );
+        assert_eq!(
+            auto_detect_line_ending(&Rope::from_str("a formfeed\u{000C}")),
+            None
+        );
+        assert_eq!(
+            auto_detect_line_ending(&Rope::from_str("\n\u{000A}\n \u{000A}")),
+            Some(LineEnding::LF)
+        );
+        assert_eq!(
+            auto_detect_line_ending(&Rope::from_str(
+                "a formfeed\u{000C} with a\u{000C} linefeed\u{000A}"
+            )),
+            Some(LineEnding::LF)
+        );
+        assert_eq!(auto_detect_line_ending(&Rope::from_str("a formfeed\u{000C} with a\u{000C} carriage return linefeed\u{000D}\u{000A} and a linefeed\u{000A}")), Some(LineEnding::Crlf));
+    }
+
+    #[test]
+    fn str_to_line_ending() {
+        assert_eq!(LineEnding::from_str("\r"), Some(LineEnding::CR));
+        assert_eq!(LineEnding::from_str("\n"), Some(LineEnding::LF));
+        assert_eq!(LineEnding::from_str("\r\n"), Some(LineEnding::Crlf));
+        assert_eq!(LineEnding::from_str("hello\n"), None);
+    }
+
+    #[test]
+    fn rope_slice_to_line_ending() {
+        let r = Rope::from_str("hello\r\n");
+        assert_eq!(
+            LineEnding::from_rope_slice(&r.slice(5..6)),
+            Some(LineEnding::CR)
+        );
+        assert_eq!(
+            LineEnding::from_rope_slice(&r.slice(6..7)),
+            Some(LineEnding::LF)
+        );
+        assert_eq!(
+            LineEnding::from_rope_slice(&r.slice(5..7)),
+            Some(LineEnding::Crlf)
+        );
+        assert_eq!(LineEnding::from_rope_slice(&r.slice(..)), None);
+    }
+
+    #[test]
+    fn get_line_ending_rope_slice() {
+        let r = Rope::from_str("Hello\rworld\nhow\r\nare you?");
+        assert_eq!(get_line_ending(&r.slice(..6)), Some(LineEnding::CR));
+        assert_eq!(get_line_ending(&r.slice(..12)), Some(LineEnding::LF));
+        assert_eq!(get_line_ending(&r.slice(..17)), Some(LineEnding::Crlf));
+        assert_eq!(get_line_ending(&r.slice(..)), None);
+    }
+
+    #[test]
+    fn get_line_ending_str() {
+        let text = "Hello\rworld\nhow\r\nare you?";
+        assert_eq!(get_line_ending_of_str(&text[..6]), Some(LineEnding::CR));
+        assert_eq!(get_line_ending_of_str(&text[..12]), Some(LineEnding::LF));
+        assert_eq!(get_line_ending_of_str(&text[..17]), Some(LineEnding::Crlf));
+        assert_eq!(get_line_ending_of_str(&text[..]), None);
+    }
+
+    #[test]
+    fn line_end_char_index_rope_slice() {
+        let r = Rope::from_str("Hello\rworld\nhow\r\nare you?");
+        let s = &r.slice(..);
+        assert_eq!(line_end_char_index(s, 0), 5);
+        assert_eq!(line_end_char_index(s, 1), 11);
+        assert_eq!(line_end_char_index(s, 2), 15);
+        assert_eq!(line_end_char_index(s, 3), 25);
+    }
+}

helix-core/src/match_brackets.rs

@@ -1,4 +1,4 @@
-use crate::{Range, Rope, Selection, Syntax};
+use crate::{Rope, Syntax};
 
 const PAIRS: &[(char, char)] = &[('(', ')'), ('{', '}'), ('[', ']'), ('<', '>')];
 // limit matching pairs to only ( ) { } [ ] < >
@@ -12,7 +12,7 @@ pub fn find(syntax: &Syntax, doc: &Rope, pos: usize) -> Option<usize> {
     // most naive implementation: find the innermost syntax node, if we're at the edge of a node,
     // return the other edge.
 
-    let mut node = match tree
+    let node = match tree
         .root_node()
         .named_descendant_for_byte_range(byte_pos, byte_pos)
     {
@@ -24,12 +24,13 @@ pub fn find(syntax: &Syntax, doc: &Rope, pos: usize) -> Option<usize> {
         return None;
     }
 
-    let start_byte = node.start_byte();
     let len = doc.len_bytes();
-    if start_byte >= len {
+    let start_byte = node.start_byte();
+    let end_byte = node.end_byte() - 1; // it's end exclusive
+    if start_byte >= len || end_byte >= len {
         return None;
     }
-    let end_byte = node.end_byte() - 1; // it's end exclusive
+
     let start_char = doc.byte_to_char(start_byte);
     let end_char = doc.byte_to_char(end_byte);
 

helix-core/src/object.rs

@@ -1,12 +1,11 @@
 use crate::{Range, RopeSlice, Selection, Syntax};
-use smallvec::smallvec;
 
 // TODO: to contract_selection we'd need to store the previous ranges before expand.
 // Maybe just contract to the first child node?
 pub fn expand_selection(syntax: &Syntax, text: RopeSlice, selection: &Selection) -> Selection {
     let tree = syntax.tree();
 
-    selection.transform(|range| {
+    selection.clone().transform(|range| {
         let from = text.char_to_byte(range.from());
         let to = text.char_to_byte(range.to());
 

helix-core/src/position.rs

@@ -1,6 +1,8 @@
 use crate::{
-    graphemes::{nth_next_grapheme_boundary, RopeGraphemes},
-    Rope, RopeSlice,
+    chars::char_is_line_ending,
+    graphemes::{ensure_grapheme_boundary_prev, RopeGraphemes},
+    line_ending::line_end_char_index,
+    RopeSlice,
 };
 
 /// Represents a single point in a text buffer. Zero indexed.
@@ -23,8 +25,9 @@ impl Position {
     pub fn traverse(self, text: &crate::Tendril) -> Self {
         let Self { mut row, mut col } = self;
         // TODO: there should be a better way here
-        for ch in text.chars() {
-            if ch == '\n' {
+        let mut chars = text.chars().peekable();
+        while let Some(ch) = chars.next() {
+            if char_is_line_ending(ch) && !(ch == '\r' && chars.peek() == Some(&'\n')) {
                 row += 1;
                 col = 0;
             } else {
@@ -50,24 +53,65 @@ impl From<Position> for tree_sitter::Point {
     }
 }
 /// Convert a character index to (line, column) coordinates.
+///
+/// TODO: this should be split into two methods: one for visual
+/// row/column, and one for "objective" row/column (possibly with
+/// the column specified in `char`s).  The former would be used
+/// for cursor movement, and the latter would be used for e.g. the
+/// row:column display in the status line.
 pub fn coords_at_pos(text: RopeSlice, pos: usize) -> Position {
     let line = text.char_to_line(pos);
+
     let line_start = text.line_to_char(line);
+    let pos = ensure_grapheme_boundary_prev(text, pos);
     let col = RopeGraphemes::new(text.slice(line_start..pos)).count();
+
     Position::new(line, col)
 }
 
 /// Convert (line, column) coordinates to a character index.
-pub fn pos_at_coords(text: RopeSlice, coords: Position) -> usize {
+///
+/// If the `line` coordinate is beyond the end of the file, the EOF
+/// position will be returned.
+///
+/// If the `column` coordinate is past the end of the given line, the
+/// line-end position will be returned.  What constitutes the "line-end
+/// position" depends on the parameter `limit_before_line_ending`.  If it's
+/// `true`, the line-end position will be just *before* the line ending
+/// character.  If `false` it will be just *after* the line ending
+/// character--on the border between the current line and the next.
+///
+/// Usually you only want `limit_before_line_ending` to be `true` if you're working
+/// with left-side block-cursor positions, as this prevents the the block cursor
+/// from jumping to the next line.  Otherwise you typically want it to be `false`,
+/// such as when dealing with raw anchor/head positions.
+///
+/// TODO: this should be changed to work in terms of visual row/column, not
+/// graphemes.
+pub fn pos_at_coords(text: RopeSlice, coords: Position, limit_before_line_ending: bool) -> usize {
     let Position { row, col } = coords;
     let line_start = text.line_to_char(row);
-    // line_start + col
-    nth_next_grapheme_boundary(text, line_start, col)
+    let line_end = if limit_before_line_ending {
+        line_end_char_index(&text, row)
+    } else {
+        text.line_to_char((row + 1).min(text.len_lines()))
+    };
+
+    let mut col_char_offset = 0;
+    for (i, g) in RopeGraphemes::new(text.slice(line_start..line_end)).enumerate() {
+        if i == col {
+            break;
+        }
+        col_char_offset += g.chars().count();
+    }
+
+    line_start + col_char_offset
 }
 
 #[cfg(test)]
 mod test {
     use super::*;
+    use crate::Rope;
 
     #[test]
     fn test_ordering() {
@@ -79,53 +123,107 @@ mod test {
     fn test_coords_at_pos() {
         let text = Rope::from("ḧëḷḷö\nẅöṛḷḋ");
         let slice = text.slice(..);
-        // assert_eq!(coords_at_pos(slice, 0), (0, 0).into());
-        // assert_eq!(coords_at_pos(slice, 5), (0, 5).into()); // position on \n
-        // assert_eq!(coords_at_pos(slice, 6), (1, 0).into()); // position on w
-        // assert_eq!(coords_at_pos(slice, 7), (1, 1).into()); // position on o
-        // assert_eq!(coords_at_pos(slice, 10), (1, 4).into()); // position on d
+        assert_eq!(coords_at_pos(slice, 0), (0, 0).into());
+        assert_eq!(coords_at_pos(slice, 5), (0, 5).into()); // position on \n
+        assert_eq!(coords_at_pos(slice, 6), (1, 0).into()); // position on w
+        assert_eq!(coords_at_pos(slice, 7), (1, 1).into()); // position on o
+        assert_eq!(coords_at_pos(slice, 10), (1, 4).into()); // position on d
 
-        // test with grapheme clusters
+        // Test with wide characters.
+        // TODO: account for character width.
+        let text = Rope::from("今日はいい\n");
+        let slice = text.slice(..);
+        assert_eq!(coords_at_pos(slice, 0), (0, 0).into());
+        assert_eq!(coords_at_pos(slice, 1), (0, 1).into());
+        assert_eq!(coords_at_pos(slice, 2), (0, 2).into());
+        assert_eq!(coords_at_pos(slice, 3), (0, 3).into());
+        assert_eq!(coords_at_pos(slice, 4), (0, 4).into());
+        assert_eq!(coords_at_pos(slice, 5), (0, 5).into());
+        assert_eq!(coords_at_pos(slice, 6), (1, 0).into());
+
+        // Test with grapheme clusters.
         let text = Rope::from("a̐éö̲\r\n");
         let slice = text.slice(..);
         assert_eq!(coords_at_pos(slice, 0), (0, 0).into());
         assert_eq!(coords_at_pos(slice, 2), (0, 1).into());
         assert_eq!(coords_at_pos(slice, 4), (0, 2).into());
         assert_eq!(coords_at_pos(slice, 7), (0, 3).into());
+        assert_eq!(coords_at_pos(slice, 9), (1, 0).into());
 
-        let text = Rope::from("किमपि");
+        // Test with wide-character grapheme clusters.
+        // TODO: account for character width.
+        let text = Rope::from("किमपि\n");
         let slice = text.slice(..);
         assert_eq!(coords_at_pos(slice, 0), (0, 0).into());
         assert_eq!(coords_at_pos(slice, 2), (0, 1).into());
         assert_eq!(coords_at_pos(slice, 3), (0, 2).into());
         assert_eq!(coords_at_pos(slice, 5), (0, 3).into());
+        assert_eq!(coords_at_pos(slice, 6), (1, 0).into());
+
+        // Test with tabs.
+        // Todo: account for tab stops.
+        let text = Rope::from("\tHello\n");
+        let slice = text.slice(..);
+        assert_eq!(coords_at_pos(slice, 0), (0, 0).into());
+        assert_eq!(coords_at_pos(slice, 1), (0, 1).into());
+        assert_eq!(coords_at_pos(slice, 2), (0, 2).into());
     }
 
     #[test]
     fn test_pos_at_coords() {
         let text = Rope::from("ḧëḷḷö\nẅöṛḷḋ");
         let slice = text.slice(..);
-        assert_eq!(pos_at_coords(slice, (0, 0).into()), 0);
-        assert_eq!(pos_at_coords(slice, (0, 5).into()), 5); // position on \n
-        assert_eq!(pos_at_coords(slice, (1, 0).into()), 6); // position on w
-        assert_eq!(pos_at_coords(slice, (1, 1).into()), 7); // position on o
-        assert_eq!(pos_at_coords(slice, (1, 4).into()), 10); // position on d
+        assert_eq!(pos_at_coords(slice, (0, 0).into(), false), 0);
+        assert_eq!(pos_at_coords(slice, (0, 5).into(), false), 5); // position on \n
+        assert_eq!(pos_at_coords(slice, (0, 6).into(), false), 6); // position after \n
+        assert_eq!(pos_at_coords(slice, (0, 6).into(), true), 5); // position after \n
+        assert_eq!(pos_at_coords(slice, (1, 0).into(), false), 6); // position on w
+        assert_eq!(pos_at_coords(slice, (1, 1).into(), false), 7); // position on o
+        assert_eq!(pos_at_coords(slice, (1, 4).into(), false), 10); // position on d
 
-        // test with grapheme clusters
+        // Test with wide characters.
+        // TODO: account for character width.
+        let text = Rope::from("今日はいい\n");
+        let slice = text.slice(..);
+        assert_eq!(pos_at_coords(slice, (0, 0).into(), false), 0);
+        assert_eq!(pos_at_coords(slice, (0, 1).into(), false), 1);
+        assert_eq!(pos_at_coords(slice, (0, 2).into(), false), 2);
+        assert_eq!(pos_at_coords(slice, (0, 3).into(), false), 3);
+        assert_eq!(pos_at_coords(slice, (0, 4).into(), false), 4);
+        assert_eq!(pos_at_coords(slice, (0, 5).into(), false), 5);
+        assert_eq!(pos_at_coords(slice, (0, 6).into(), false), 6);
+        assert_eq!(pos_at_coords(slice, (0, 6).into(), true), 5);
+        assert_eq!(pos_at_coords(slice, (1, 0).into(), false), 6);
+
+        // Test with grapheme clusters.
         let text = Rope::from("a̐éö̲\r\n");
         let slice = text.slice(..);
-        assert_eq!(pos_at_coords(slice, (0, 0).into()), 0);
-        assert_eq!(pos_at_coords(slice, (0, 1).into()), 2);
-        assert_eq!(pos_at_coords(slice, (0, 2).into()), 4);
-        assert_eq!(pos_at_coords(slice, (0, 3).into()), 7); // \r\n is one char here
-        assert_eq!(pos_at_coords(slice, (0, 4).into()), 9);
+        assert_eq!(pos_at_coords(slice, (0, 0).into(), false), 0);
+        assert_eq!(pos_at_coords(slice, (0, 1).into(), false), 2);
+        assert_eq!(pos_at_coords(slice, (0, 2).into(), false), 4);
+        assert_eq!(pos_at_coords(slice, (0, 3).into(), false), 7); // \r\n is one char here
+        assert_eq!(pos_at_coords(slice, (0, 4).into(), false), 9);
+        assert_eq!(pos_at_coords(slice, (0, 4).into(), true), 7);
+        assert_eq!(pos_at_coords(slice, (1, 0).into(), false), 9);
+
+        // Test with wide-character grapheme clusters.
+        // TODO: account for character width.
         let text = Rope::from("किमपि");
         // 2 - 1 - 2 codepoints
         // TODO: delete handling as per https://news.ycombinator.com/item?id=20058454
         let slice = text.slice(..);
-        assert_eq!(pos_at_coords(slice, (0, 0).into()), 0);
-        assert_eq!(pos_at_coords(slice, (0, 1).into()), 2);
-        assert_eq!(pos_at_coords(slice, (0, 2).into()), 3);
-        assert_eq!(pos_at_coords(slice, (0, 3).into()), 5); // eol
+        assert_eq!(pos_at_coords(slice, (0, 0).into(), false), 0);
+        assert_eq!(pos_at_coords(slice, (0, 1).into(), false), 2);
+        assert_eq!(pos_at_coords(slice, (0, 2).into(), false), 3);
+        assert_eq!(pos_at_coords(slice, (0, 3).into(), false), 5);
+        assert_eq!(pos_at_coords(slice, (0, 3).into(), true), 5);
+
+        // Test with tabs.
+        // Todo: account for tab stops.
+        let text = Rope::from("\tHello\n");
+        let slice = text.slice(..);
+        assert_eq!(pos_at_coords(slice, (0, 0).into(), false), 0);
+        assert_eq!(pos_at_coords(slice, (0, 1).into(), false), 1);
+        assert_eq!(pos_at_coords(slice, (0, 2).into(), false), 2);
     }
 }

helix-core/src/register.rs

@@ -1,20 +1,63 @@
-use crate::Tendril;
-use once_cell::sync::Lazy;
-use std::{collections::HashMap, sync::RwLock};
+use std::collections::HashMap;
 
-// TODO: could be an instance on Editor
-static REGISTRY: Lazy<RwLock<HashMap<char, Vec<String>>>> =
-    Lazy::new(|| RwLock::new(HashMap::new()));
-
-/// Read register values.
-pub fn get(register_name: char) -> Option<Vec<String>> {
-    let registry = REGISTRY.read().unwrap();
-    registry.get(&register_name).cloned() // TODO: no cloning
+#[derive(Debug)]
+pub struct Register {
+    name: char,
+    values: Vec<String>,
 }
 
-/// Read register values.
-// restoring: bool
-pub fn set(register_name: char, values: Vec<String>) {
-    let mut registry = REGISTRY.write().unwrap();
-    registry.insert(register_name, values);
+impl Register {
+    pub fn new(name: char) -> Self {
+        Self {
+            name,
+            values: Vec::new(),
+        }
+    }
+
+    pub fn new_with_values(name: char, values: Vec<String>) -> Self {
+        Self { name, values }
+    }
+
+    pub fn name(&self) -> char {
+        self.name
+    }
+
+    pub fn read(&self) -> &[String] {
+        &self.values
+    }
+
+    pub fn write(&mut self, values: Vec<String>) {
+        self.values = values;
+    }
+
+    pub fn push(&mut self, value: String) {
+        self.values.push(value);
+    }
+}
+
+/// Currently just wraps a `HashMap` of `Register`s
+#[derive(Debug, Default)]
+pub struct Registers {
+    inner: HashMap<char, Register>,
+}
+
+impl Registers {
+    pub fn get(&self, name: char) -> Option<&Register> {
+        self.inner.get(&name)
+    }
+
+    pub fn get_mut(&mut self, name: char) -> &mut Register {
+        self.inner
+            .entry(name)
+            .or_insert_with(|| Register::new(name))
+    }
+
+    pub fn write(&mut self, name: char, values: Vec<String>) {
+        self.inner
+            .insert(name, Register::new_with_values(name, values));
+    }
+
+    pub fn read(&self, name: char) -> Option<&[String]> {
+        self.get(name).map(|reg| reg.read())
+    }
 }

helix-core/src/search.rs

@@ -1,18 +1,11 @@
 use crate::RopeSlice;
 
-pub fn find_nth_next(
-    text: RopeSlice,
-    ch: char,
-    mut pos: usize,
-    n: usize,
-    inclusive: bool,
-) -> Option<usize> {
-    if pos >= text.len_chars() {
+pub fn find_nth_next(text: RopeSlice, ch: char, mut pos: usize, n: usize) -> Option<usize> {
+    if pos >= text.len_chars() || n == 0 {
         return None;
     }
 
-    // start searching right after pos
-    let mut chars = text.chars_at(pos + 1);
+    let mut chars = text.chars_at(pos);
 
     for _ in 0..n {
         loop {
@@ -26,28 +19,21 @@ pub fn find_nth_next(
         }
     }
 
-    if !inclusive {
-        pos -= 1;
+    Some(pos - 1)
 }
 
-    Some(pos)
+pub fn find_nth_prev(text: RopeSlice, ch: char, mut pos: usize, n: usize) -> Option<usize> {
+    if pos == 0 || n == 0 {
+        return None;
     }
 
-pub fn find_nth_prev(
-    text: RopeSlice,
-    ch: char,
-    mut pos: usize,
-    n: usize,
-    inclusive: bool,
-) -> Option<usize> {
-    // start searching right before pos
     let mut chars = text.chars_at(pos);
 
     for _ in 0..n {
         loop {
             let c = chars.prev()?;
 
-            pos = pos.saturating_sub(1);
+            pos -= 1;
 
             if c == ch {
                 break;
@@ -55,9 +41,5 @@ pub fn find_nth_prev(
         }
     }
 
-    if !inclusive {
-        pos += 1;
-    }
-
     Some(pos)
 }

helix-core/src/selection.rs

@@ -1,30 +1,59 @@
-//! Selections are the primary editing construct. Even a single cursor is defined as an empty
-//! single selection range.
+//! Selections are the primary editing construct. Even cursors are
+//! defined as a selection range.
 //!
 //! All positioning is done via `char` offsets into the buffer.
-use crate::{Assoc, ChangeSet, Rope, RopeSlice};
+use crate::{
+    graphemes::{
+        ensure_grapheme_boundary_next, ensure_grapheme_boundary_prev, next_grapheme_boundary,
+        prev_grapheme_boundary,
+    },
+    Assoc, ChangeSet, RopeSlice,
+};
 use smallvec::{smallvec, SmallVec};
 use std::borrow::Cow;
 
-#[inline]
-fn abs_difference(x: usize, y: usize) -> usize {
-    if x < y {
-        y - x
-    } else {
-        x - y
-    }
-}
-
-/// A single selection range. Anchor-inclusive, head-exclusive.
+/// A single selection range.
+///
+/// A range consists of an "anchor" and "head" position in
+/// the text.  The head is the part that the user moves when
+/// directly extending a selection.  The head and anchor
+/// can be in any order, or even share the same position.
+///
+/// The anchor and head positions use gap indexing, meaning
+/// that their indices represent the the gaps *between* `char`s
+/// rather than the `char`s themselves. For example, 1
+/// represents the position between the first and second `char`.
+///
+/// Below are some example `Range` configurations to better
+/// illustrate.  The anchor and head indices are show as
+/// "(anchor, head)", followed by example text with "[" and "]"
+/// inserted to represent the anchor and head positions:
+///
+/// - (0, 3): [Som]e text.
+/// - (3, 0): ]Som[e text.
+/// - (2, 7): So[me te]xt.
+/// - (1, 1): S[]ome text.
+///
+/// Ranges are considered to be inclusive on the left and
+/// exclusive on the right, regardless of anchor-head ordering.
+/// This means, for example, that non-zero-width ranges that
+/// are directly adjecent, sharing an edge, do not overlap.
+/// However, a zero-width range will overlap with the shared
+/// left-edge of another range.
+///
+/// By convention, user-facing ranges are considered to have
+/// a block cursor on the head-side of the range that spans a
+/// single grapheme inward from the range's edge.  There are a
+/// variety of helper methods on `Range` for working in terms of
+/// that block cursor, all of which have `cursor` in their name.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub struct Range {
-    // TODO: optimize into u32
     /// The anchor of the range: the side that doesn't move when extending.
     pub anchor: usize,
     /// The head of the range, moved when extending.
     pub head: usize,
     pub horiz: Option<u32>,
-} // TODO: might be cheaper to store normalized as from/to and an inverted flag
+}
 
 impl Range {
     pub fn new(anchor: usize, head: usize) -> Self {
@@ -53,6 +82,20 @@ impl Range {
         std::cmp::max(self.anchor, self.head)
     }
 
+    /// The (inclusive) range of lines that the range overlaps.
+    #[inline]
+    #[must_use]
+    pub fn line_range(&self, text: RopeSlice) -> (usize, usize) {
+        let from = self.from();
+        let to = if self.is_empty() {
+            self.to()
+        } else {
+            prev_grapheme_boundary(text, self.to()).max(from)
+        };
+
+        (text.char_to_line(from), text.char_to_line(to))
+    }
+
     /// `true` when head and anchor are at the same position.
     #[inline]
     pub fn is_empty(&self) -> bool {
@@ -62,37 +105,39 @@ impl Range {
     /// Check two ranges for overlap.
     #[must_use]
     pub fn overlaps(&self, other: &Self) -> bool {
-        // cursor overlap is checked differently
-        if self.is_empty() {
-            let pos = self.head;
-            pos >= other.from() && other.to() >= pos
-        } else {
-            self.to() > other.from() && other.to() > self.from()
-        }
+        // To my eye, it's non-obvious why this works, but I arrived
+        // at it after transforming the slower version that explicitly
+        // enumerated more cases.  The unit tests are thorough.
+        self.from() == other.from() || (self.to() > other.from() && other.to() > self.from())
     }
 
     pub fn contains(&self, pos: usize) -> bool {
-        if self.is_empty() {
-            return false;
-        }
-
-        if self.anchor < self.head {
-            self.anchor <= pos && pos < self.head
-        } else {
-            self.head < pos && pos <= self.anchor
-        }
+        self.from() <= pos && pos < self.to()
     }
 
     /// Map a range through a set of changes. Returns a new range representing the same position
     /// after the changes are applied.
     pub fn map(self, changes: &ChangeSet) -> Self {
-        let anchor = changes.map_pos(self.anchor, Assoc::After);
-        let head = changes.map_pos(self.head, Assoc::After);
+        use std::cmp::Ordering;
+        let (anchor, head) = match self.anchor.cmp(&self.head) {
+            Ordering::Equal => (
+                changes.map_pos(self.anchor, Assoc::After),
+                changes.map_pos(self.head, Assoc::After),
+            ),
+            Ordering::Less => (
+                changes.map_pos(self.anchor, Assoc::After),
+                changes.map_pos(self.head, Assoc::Before),
+            ),
+            Ordering::Greater => (
+                changes.map_pos(self.anchor, Assoc::Before),
+                changes.map_pos(self.head, Assoc::After),
+            ),
+        };
 
-        // TODO: possibly unnecessary
-        if self.anchor == anchor && self.head == head {
-            return self;
-        }
+        // We want to return a new `Range` with `horiz == None` every time,
+        // even if the anchor and head haven't changed, because we don't
+        // know if the *visual* position hasn't changed due to
+        // character-width or grapheme changes earlier in the text.
         Self {
             anchor,
             head,
@@ -103,30 +148,172 @@ impl Range {
     /// Extend the range to cover at least `from` `to`.
     #[must_use]
     pub fn extend(&self, from: usize, to: usize) -> Self {
-        if from <= self.anchor && to >= self.anchor {
-            return Self {
-                anchor: from,
-                head: to,
+        debug_assert!(from <= to);
+
+        if self.anchor <= self.head {
+            Self {
+                anchor: self.anchor.min(from),
+                head: self.head.max(to),
                 horiz: None,
-            };
+            }
+        } else {
+            Self {
+                anchor: self.anchor.max(to),
+                head: self.head.min(from),
+                horiz: None,
+            }
+        }
     }
 
-        Self {
-            anchor: self.anchor,
-            head: if abs_difference(from, self.anchor) > abs_difference(to, self.anchor) {
-                from
-            } else {
-                to
-            },
+    /// Returns a range that encompasses both input ranges.
+    ///
+    /// This is like `extend()`, but tries to negotiate the
+    /// anchor/head ordering between the two input ranges.
+    #[must_use]
+    pub fn merge(&self, other: Self) -> Self {
+        if self.anchor > self.head && other.anchor > other.head {
+            Range {
+                anchor: self.anchor.max(other.anchor),
+                head: self.head.min(other.head),
                 horiz: None,
             }
+        } else {
+            Range {
+                anchor: self.from().min(other.from()),
+                head: self.to().max(other.to()),
+                horiz: None,
+            }
+        }
     }
 
     // groupAt
 
     #[inline]
     pub fn fragment<'a, 'b: 'a>(&'a self, text: RopeSlice<'b>) -> Cow<'b, str> {
-        Cow::from(text.slice(self.from()..self.to() + 1))
+        text.slice(self.from()..self.to()).into()
+    }
+
+    //--------------------------------
+    // Alignment methods.
+
+    /// Compute a possibly new range from this range, with its ends
+    /// shifted as needed to align with grapheme boundaries.
+    ///
+    /// Zero-width ranges will always stay zero-width, and non-zero-width
+    /// ranges will never collapse to zero-width.
+    #[must_use]
+    pub fn grapheme_aligned(&self, slice: RopeSlice) -> Self {
+        use std::cmp::Ordering;
+        let (new_anchor, new_head) = match self.anchor.cmp(&self.head) {
+            Ordering::Equal => {
+                let pos = ensure_grapheme_boundary_prev(slice, self.anchor);
+                (pos, pos)
+            }
+            Ordering::Less => (
+                ensure_grapheme_boundary_prev(slice, self.anchor),
+                ensure_grapheme_boundary_next(slice, self.head),
+            ),
+            Ordering::Greater => (
+                ensure_grapheme_boundary_next(slice, self.anchor),
+                ensure_grapheme_boundary_prev(slice, self.head),
+            ),
+        };
+        Range {
+            anchor: new_anchor,
+            head: new_head,
+            horiz: if new_anchor == self.anchor {
+                self.horiz
+            } else {
+                None
+            },
+        }
+    }
+
+    /// Compute a possibly new range from this range, attempting to ensure
+    /// a minimum range width of 1 char by shifting the head in the forward
+    /// direction as needed.
+    ///
+    /// This method will never shift the anchor, and will only shift the
+    /// head in the forward direction.  Therefore, this method can fail
+    /// at ensuring the minimum width if and only if the passed range is
+    /// both zero-width and at the end of the `RopeSlice`.
+    ///
+    /// If the input range is grapheme-boundary aligned, the returned range
+    /// will also be.  Specifically, if the head needs to shift to achieve
+    /// the minimum width, it will shift to the next grapheme boundary.
+    #[must_use]
+    #[inline]
+    pub fn min_width_1(&self, slice: RopeSlice) -> Self {
+        if self.anchor == self.head {
+            Range {
+                anchor: self.anchor,
+                head: next_grapheme_boundary(slice, self.head),
+                horiz: self.horiz,
+            }
+        } else {
+            *self
+        }
+    }
+
+    //--------------------------------
+    // Block-cursor methods.
+
+    /// Gets the left-side position of the block cursor.
+    #[must_use]
+    #[inline]
+    pub fn cursor(self, text: RopeSlice) -> usize {
+        if self.head > self.anchor {
+            prev_grapheme_boundary(text, self.head)
+        } else {
+            self.head
+        }
+    }
+
+    /// Puts the left side of the block cursor at `char_idx`, optionally extending.
+    ///
+    /// This follows "1-width" semantics, and therefore does a combination of anchor
+    /// and head moves to behave as if both the front and back of the range are 1-width
+    /// blocks
+    ///
+    /// This method assumes that the range and `char_idx` are already properly
+    /// grapheme-aligned.
+    #[must_use]
+    #[inline]
+    pub fn put_cursor(self, text: RopeSlice, char_idx: usize, extend: bool) -> Range {
+        if extend {
+            let anchor = if self.head >= self.anchor && char_idx < self.anchor {
+                next_grapheme_boundary(text, self.anchor)
+            } else if self.head < self.anchor && char_idx >= self.anchor {
+                prev_grapheme_boundary(text, self.anchor)
+            } else {
+                self.anchor
+            };
+
+            if anchor <= char_idx {
+                Range::new(anchor, next_grapheme_boundary(text, char_idx))
+            } else {
+                Range::new(anchor, char_idx)
+            }
+        } else {
+            Range::point(char_idx)
+        }
+    }
+
+    /// The line number that the block-cursor is on.
+    #[inline]
+    #[must_use]
+    pub fn cursor_line(&self, text: RopeSlice) -> usize {
+        text.char_to_line(self.cursor(text))
+    }
+}
+
+impl From<(usize, usize)> for Range {
+    fn from(tuple: (usize, usize)) -> Self {
+        Self {
+            anchor: tuple.0,
+            head: tuple.1,
+            horiz: None,
+        }
     }
 }
 
@@ -142,14 +329,16 @@ pub struct Selection {
 impl Selection {
     // eq
 
+    #[inline]
     #[must_use]
     pub fn primary(&self) -> Range {
         self.ranges[self.primary_index]
     }
 
+    #[inline]
     #[must_use]
-    pub fn cursor(&self) -> usize {
-        self.primary().head
+    pub fn primary_mut(&mut self) -> &mut Range {
+        &mut self.ranges[self.primary_index]
     }
 
     /// Ensure selection containing only the primary selection.
@@ -164,13 +353,12 @@ impl Selection {
         }
     }
 
+    /// Adds a new range to the selection and makes it the primary range.
     pub fn push(mut self, range: Range) -> Self {
-        let index = self.ranges.len();
         self.ranges.push(range);
-
-        Self::normalize(self.ranges, index)
+        self.set_primary_index(self.ranges().len() - 1);
+        self.normalize()
     }
-    // replace_range
 
     /// Map selections over a set of changes. Useful for adjusting the selection position after
     /// applying changes to a document.
@@ -196,6 +384,11 @@ impl Selection {
         self.primary_index
     }
 
+    pub fn set_primary_index(&mut self, idx: usize) {
+        assert!(idx < self.ranges.len());
+        self.primary_index = idx;
+    }
+
     #[must_use]
     /// Constructs a selection holding a single range.
     pub fn single(anchor: usize, head: usize) -> Self {
@@ -214,77 +407,79 @@ impl Selection {
         Self::single(pos, pos)
     }
 
-    fn normalize(mut ranges: SmallVec<[Range; 1]>, mut primary_index: usize) -> Self {
-        let primary = ranges[primary_index];
-        ranges.sort_unstable_by_key(Range::from);
-        primary_index = ranges.iter().position(|&range| range == primary).unwrap();
+    /// Normalizes a `Selection`.
+    fn normalize(mut self) -> Self {
+        let primary = self.ranges[self.primary_index];
+        self.ranges.sort_unstable_by_key(Range::from);
+        self.primary_index = self
+            .ranges
+            .iter()
+            .position(|&range| range == primary)
+            .unwrap();
 
-        let mut result = SmallVec::with_capacity(ranges.len()); // approx
-
-        // TODO: we could do with one vec by removing elements as we mutate
-
-        for (i, range) in ranges.into_iter().enumerate() {
-            // if previous value exists
-            if let Some(prev) = result.last_mut() {
-                // and we overlap it
-
-                // TODO: we used to simply check range.from() <(=) prev.to()
-                // avoiding two comparisons
-                if range.overlaps(prev) {
-                    let from = prev.from();
-                    let to = std::cmp::max(range.to(), prev.to());
-
-                    if i <= primary_index {
-                        primary_index -= 1
-                    }
-
-                    // merge into previous
-                    if range.anchor > range.head {
-                        prev.anchor = to;
-                        prev.head = from;
+        let mut prev_i = 0;
+        for i in 1..self.ranges.len() {
+            if self.ranges[prev_i].overlaps(&self.ranges[i]) {
+                self.ranges[prev_i] = self.ranges[prev_i].merge(self.ranges[i]);
             } else {
-                        prev.anchor = from;
-                        prev.head = to;
+                prev_i += 1;
+                self.ranges[prev_i] = self.ranges[i];
             }
-                    continue;
+            if i == self.primary_index {
+                self.primary_index = prev_i;
             }
         }
 
-            result.push(range)
-        }
+        self.ranges.truncate(prev_i + 1);
 
-        Self {
-            ranges: result,
-            primary_index,
-        }
+        self
     }
 
     // TODO: consume an iterator or a vec to reduce allocations?
     #[must_use]
     pub fn new(ranges: SmallVec<[Range; 1]>, primary_index: usize) -> Self {
         assert!(!ranges.is_empty());
+        debug_assert!(primary_index < ranges.len());
 
-        // fast path for a single selection (cursor)
-        if ranges.len() == 1 {
-            return Self {
+        let mut selection = Self {
             ranges,
-                primary_index: 0,
+            primary_index,
         };
-        }
 
+        if selection.ranges.len() > 1 {
             // TODO: only normalize if needed (any ranges out of order)
-        Self::normalize(ranges, primary_index)
+            selection = selection.normalize();
         }
 
-    /// Takes a closure and maps each selection over the closure.
-    pub fn transform<F>(&self, f: F) -> Self
+        selection
+    }
+
+    /// Takes a closure and maps each `Range` over the closure.
+    pub fn transform<F>(mut self, f: F) -> Self
     where
         F: Fn(Range) -> Range,
     {
-        Self::new(
-            self.ranges.iter().copied().map(f).collect(),
-            self.primary_index,
-        )
+        for range in self.ranges.iter_mut() {
+            *range = f(*range)
+        }
+        self.normalize()
+    }
+
+    // Ensures the selection adheres to the following invariants:
+    // 1. All ranges are grapheme aligned.
+    // 2. All ranges are at least 1 character wide, unless at the
+    //    very end of the document.
+    // 3. Ranges are non-overlapping.
+    // 4. Ranges are sorted by their position in the text.
+    pub fn ensure_invariants(self, text: RopeSlice) -> Self {
+        self.transform(|r| r.min_width_1(text).grapheme_aligned(text))
+            .normalize()
+    }
+
+    /// Transforms the selection into all of the left-side head positions,
+    /// using block-cursor semantics.
+    pub fn cursors(self, text: RopeSlice) -> Self {
+        self.transform(|range| Range::point(range.cursor(text)))
     }
 
     pub fn fragments<'a>(&'a self, text: RopeSlice<'a>) -> impl Iterator<Item = Cow<str>> + 'a {
@@ -342,17 +537,15 @@ pub fn select_on_matches(
         // TODO: can't avoid occasional allocations since Regex can't operate on chunks yet
         let fragment = sel.fragment(text);
 
-        let mut sel_start = sel.from();
-        let sel_end = sel.to();
-
-        let mut start_byte = text.char_to_byte(sel_start);
+        let sel_start = sel.from();
+        let start_byte = text.char_to_byte(sel_start);
 
         for mat in regex.find_iter(&fragment) {
             // TODO: retain range direction
 
             let start = text.byte_to_char(start_byte + mat.start());
             let end = text.byte_to_char(start_byte + mat.end());
-            result.push(Range::new(start, end - 1));
+            result.push(Range::new(start, end));
         }
     }
 
@@ -373,25 +566,30 @@ pub fn split_on_matches(
     let mut result = SmallVec::with_capacity(selection.len());
 
     for sel in selection {
+        // Special case: zero-width selection.
+        if sel.from() == sel.to() {
+            result.push(*sel);
+            continue;
+        }
+
         // TODO: can't avoid occasional allocations since Regex can't operate on chunks yet
         let fragment = sel.fragment(text);
 
-        let mut sel_start = sel.from();
+        let sel_start = sel.from();
         let sel_end = sel.to();
 
-        let mut start_byte = text.char_to_byte(sel_start);
+        let start_byte = text.char_to_byte(sel_start);
 
         let mut start = sel_start;
 
         for mat in regex.find_iter(&fragment) {
             // TODO: retain range direction
-
             let end = text.byte_to_char(start_byte + mat.start());
-            result.push(Range::new(start, end.saturating_sub(1)));
+            result.push(Range::new(start, end));
             start = text.byte_to_char(start_byte + mat.end());
         }
 
-        if start <= sel_end {
+        if start < sel_end {
             result.push(Range::new(start, sel_end));
         }
     }
@@ -403,11 +601,12 @@ pub fn split_on_matches(
 #[cfg(test)]
 mod test {
     use super::*;
+    use crate::Rope;
 
     #[test]
     #[should_panic]
     fn test_new_empty() {
-        let sel = Selection::new(smallvec![], 0);
+        let _ = Selection::new(smallvec![], 0);
     }
 
     #[test]
@@ -434,6 +633,22 @@ mod test {
             .join(",");
 
         assert_eq!(res, "0/6,6/7,7/8,9/13,13/14");
+
+        // it correctly calculates a new primary index
+        let sel = Selection::new(
+            smallvec![Range::new(0, 2), Range::new(1, 5), Range::new(4, 7)],
+            2,
+        );
+
+        let res = sel
+            .ranges
+            .into_iter()
+            .map(|range| format!("{}/{}", range.anchor, range.head))
+            .collect::<Vec<String>>()
+            .join(",");
+
+        assert_eq!(res, "0/7");
+        assert_eq!(sel.primary_index, 0);
     }
 
     #[test]
@@ -456,7 +671,7 @@ mod test {
             .collect::<Vec<String>>()
             .join(",");
 
-        assert_eq!(res, "8/10,10/12");
+        assert_eq!(res, "8/10,10/12,12/12");
     }
 
     #[test]
@@ -470,9 +685,214 @@ mod test {
         assert_eq!(range.contains(13), false);
 
         let range = Range::new(9, 6);
-        assert_eq!(range.contains(9), true);
+        assert_eq!(range.contains(9), false);
         assert_eq!(range.contains(7), true);
-        assert_eq!(range.contains(6), false);
+        assert_eq!(range.contains(6), true);
+    }
+
+    #[test]
+    fn test_overlaps() {
+        fn overlaps(a: (usize, usize), b: (usize, usize)) -> bool {
+            Range::new(a.0, a.1).overlaps(&Range::new(b.0, b.1))
+        }
+
+        // Two non-zero-width ranges, no overlap.
+        assert!(!overlaps((0, 3), (3, 6)));
+        assert!(!overlaps((0, 3), (6, 3)));
+        assert!(!overlaps((3, 0), (3, 6)));
+        assert!(!overlaps((3, 0), (6, 3)));
+        assert!(!overlaps((3, 6), (0, 3)));
+        assert!(!overlaps((3, 6), (3, 0)));
+        assert!(!overlaps((6, 3), (0, 3)));
+        assert!(!overlaps((6, 3), (3, 0)));
+
+        // Two non-zero-width ranges, overlap.
+        assert!(overlaps((0, 4), (3, 6)));
+        assert!(overlaps((0, 4), (6, 3)));
+        assert!(overlaps((4, 0), (3, 6)));
+        assert!(overlaps((4, 0), (6, 3)));
+        assert!(overlaps((3, 6), (0, 4)));
+        assert!(overlaps((3, 6), (4, 0)));
+        assert!(overlaps((6, 3), (0, 4)));
+        assert!(overlaps((6, 3), (4, 0)));
+
+        // Zero-width and non-zero-width range, no overlap.
+        assert!(!overlaps((0, 3), (3, 3)));
+        assert!(!overlaps((3, 0), (3, 3)));
+        assert!(!overlaps((3, 3), (0, 3)));
+        assert!(!overlaps((3, 3), (3, 0)));
+
+        // Zero-width and non-zero-width range, overlap.
+        assert!(overlaps((1, 4), (1, 1)));
+        assert!(overlaps((4, 1), (1, 1)));
+        assert!(overlaps((1, 1), (1, 4)));
+        assert!(overlaps((1, 1), (4, 1)));
+
+        assert!(overlaps((1, 4), (3, 3)));
+        assert!(overlaps((4, 1), (3, 3)));
+        assert!(overlaps((3, 3), (1, 4)));
+        assert!(overlaps((3, 3), (4, 1)));
+
+        // Two zero-width ranges, no overlap.
+        assert!(!overlaps((0, 0), (1, 1)));
+        assert!(!overlaps((1, 1), (0, 0)));
+
+        // Two zero-width ranges, overlap.
+        assert!(overlaps((1, 1), (1, 1)));
+    }
+
+    #[test]
+    fn test_graphem_aligned() {
+        let r = Rope::from_str("\r\nHi\r\n");
+        let s = r.slice(..);
+
+        // Zero-width.
+        assert_eq!(Range::new(0, 0).grapheme_aligned(s), Range::new(0, 0));
+        assert_eq!(Range::new(1, 1).grapheme_aligned(s), Range::new(0, 0));
+        assert_eq!(Range::new(2, 2).grapheme_aligned(s), Range::new(2, 2));
+        assert_eq!(Range::new(3, 3).grapheme_aligned(s), Range::new(3, 3));
+        assert_eq!(Range::new(4, 4).grapheme_aligned(s), Range::new(4, 4));
+        assert_eq!(Range::new(5, 5).grapheme_aligned(s), Range::new(4, 4));
+        assert_eq!(Range::new(6, 6).grapheme_aligned(s), Range::new(6, 6));
+
+        // Forward.
+        assert_eq!(Range::new(0, 1).grapheme_aligned(s), Range::new(0, 2));
+        assert_eq!(Range::new(1, 2).grapheme_aligned(s), Range::new(0, 2));
+        assert_eq!(Range::new(2, 3).grapheme_aligned(s), Range::new(2, 3));
+        assert_eq!(Range::new(3, 4).grapheme_aligned(s), Range::new(3, 4));
+        assert_eq!(Range::new(4, 5).grapheme_aligned(s), Range::new(4, 6));
+        assert_eq!(Range::new(5, 6).grapheme_aligned(s), Range::new(4, 6));
+
+        assert_eq!(Range::new(0, 2).grapheme_aligned(s), Range::new(0, 2));
+        assert_eq!(Range::new(1, 3).grapheme_aligned(s), Range::new(0, 3));
+        assert_eq!(Range::new(2, 4).grapheme_aligned(s), Range::new(2, 4));
+        assert_eq!(Range::new(3, 5).grapheme_aligned(s), Range::new(3, 6));
+        assert_eq!(Range::new(4, 6).grapheme_aligned(s), Range::new(4, 6));
+
+        // Reverse.
+        assert_eq!(Range::new(1, 0).grapheme_aligned(s), Range::new(2, 0));
+        assert_eq!(Range::new(2, 1).grapheme_aligned(s), Range::new(2, 0));
+        assert_eq!(Range::new(3, 2).grapheme_aligned(s), Range::new(3, 2));
+        assert_eq!(Range::new(4, 3).grapheme_aligned(s), Range::new(4, 3));
+        assert_eq!(Range::new(5, 4).grapheme_aligned(s), Range::new(6, 4));
+        assert_eq!(Range::new(6, 5).grapheme_aligned(s), Range::new(6, 4));
+
+        assert_eq!(Range::new(2, 0).grapheme_aligned(s), Range::new(2, 0));
+        assert_eq!(Range::new(3, 1).grapheme_aligned(s), Range::new(3, 0));
+        assert_eq!(Range::new(4, 2).grapheme_aligned(s), Range::new(4, 2));
+        assert_eq!(Range::new(5, 3).grapheme_aligned(s), Range::new(6, 3));
+        assert_eq!(Range::new(6, 4).grapheme_aligned(s), Range::new(6, 4));
+    }
+
+    #[test]
+    fn test_min_width_1() {
+        let r = Rope::from_str("\r\nHi\r\n");
+        let s = r.slice(..);
+
+        // Zero-width.
+        assert_eq!(Range::new(0, 0).min_width_1(s), Range::new(0, 2));
+        assert_eq!(Range::new(1, 1).min_width_1(s), Range::new(1, 2));
+        assert_eq!(Range::new(2, 2).min_width_1(s), Range::new(2, 3));
+        assert_eq!(Range::new(3, 3).min_width_1(s), Range::new(3, 4));
+        assert_eq!(Range::new(4, 4).min_width_1(s), Range::new(4, 6));
+        assert_eq!(Range::new(5, 5).min_width_1(s), Range::new(5, 6));
+        assert_eq!(Range::new(6, 6).min_width_1(s), Range::new(6, 6));
+
+        // Forward.
+        assert_eq!(Range::new(0, 1).min_width_1(s), Range::new(0, 1));
+        assert_eq!(Range::new(1, 2).min_width_1(s), Range::new(1, 2));
+        assert_eq!(Range::new(2, 3).min_width_1(s), Range::new(2, 3));
+        assert_eq!(Range::new(3, 4).min_width_1(s), Range::new(3, 4));
+        assert_eq!(Range::new(4, 5).min_width_1(s), Range::new(4, 5));
+        assert_eq!(Range::new(5, 6).min_width_1(s), Range::new(5, 6));
+
+        // Reverse.
+        assert_eq!(Range::new(1, 0).min_width_1(s), Range::new(1, 0));
+        assert_eq!(Range::new(2, 1).min_width_1(s), Range::new(2, 1));
+        assert_eq!(Range::new(3, 2).min_width_1(s), Range::new(3, 2));
+        assert_eq!(Range::new(4, 3).min_width_1(s), Range::new(4, 3));
+        assert_eq!(Range::new(5, 4).min_width_1(s), Range::new(5, 4));
+        assert_eq!(Range::new(6, 5).min_width_1(s), Range::new(6, 5));
+    }
+
+    #[test]
+    fn test_line_range() {
+        let r = Rope::from_str("\r\nHi\r\nthere!");
+        let s = r.slice(..);
+
+        // Zero-width ranges.
+        assert_eq!(Range::new(0, 0).line_range(s), (0, 0));
+        assert_eq!(Range::new(1, 1).line_range(s), (0, 0));
+        assert_eq!(Range::new(2, 2).line_range(s), (1, 1));
+        assert_eq!(Range::new(3, 3).line_range(s), (1, 1));
+
+        // Forward ranges.
+        assert_eq!(Range::new(0, 1).line_range(s), (0, 0));
+        assert_eq!(Range::new(0, 2).line_range(s), (0, 0));
+        assert_eq!(Range::new(0, 3).line_range(s), (0, 1));
+        assert_eq!(Range::new(1, 2).line_range(s), (0, 0));
+        assert_eq!(Range::new(2, 3).line_range(s), (1, 1));
+        assert_eq!(Range::new(3, 8).line_range(s), (1, 2));
+        assert_eq!(Range::new(0, 12).line_range(s), (0, 2));
+
+        // Reverse ranges.
+        assert_eq!(Range::new(1, 0).line_range(s), (0, 0));
+        assert_eq!(Range::new(2, 0).line_range(s), (0, 0));
+        assert_eq!(Range::new(3, 0).line_range(s), (0, 1));
+        assert_eq!(Range::new(2, 1).line_range(s), (0, 0));
+        assert_eq!(Range::new(3, 2).line_range(s), (1, 1));
+        assert_eq!(Range::new(8, 3).line_range(s), (1, 2));
+        assert_eq!(Range::new(12, 0).line_range(s), (0, 2));
+    }
+
+    #[test]
+    fn test_cursor() {
+        let r = Rope::from_str("\r\nHi\r\nthere!");
+        let s = r.slice(..);
+
+        // Zero-width ranges.
+        assert_eq!(Range::new(0, 0).cursor(s), 0);
+        assert_eq!(Range::new(2, 2).cursor(s), 2);
+        assert_eq!(Range::new(3, 3).cursor(s), 3);
+
+        // Forward ranges.
+        assert_eq!(Range::new(0, 2).cursor(s), 0);
+        assert_eq!(Range::new(0, 3).cursor(s), 2);
+        assert_eq!(Range::new(3, 6).cursor(s), 4);
+
+        // Reverse ranges.
+        assert_eq!(Range::new(2, 0).cursor(s), 0);
+        assert_eq!(Range::new(6, 2).cursor(s), 2);
+        assert_eq!(Range::new(6, 3).cursor(s), 3);
+    }
+
+    #[test]
+    fn test_put_cursor() {
+        let r = Rope::from_str("\r\nHi\r\nthere!");
+        let s = r.slice(..);
+
+        // Zero-width ranges.
+        assert_eq!(Range::new(0, 0).put_cursor(s, 0, true), Range::new(0, 2));
+        assert_eq!(Range::new(0, 0).put_cursor(s, 2, true), Range::new(0, 3));
+        assert_eq!(Range::new(2, 3).put_cursor(s, 4, true), Range::new(2, 6));
+        assert_eq!(Range::new(2, 8).put_cursor(s, 4, true), Range::new(2, 6));
+        assert_eq!(Range::new(8, 8).put_cursor(s, 4, true), Range::new(9, 4));
+
+        // Forward ranges.
+        assert_eq!(Range::new(3, 6).put_cursor(s, 0, true), Range::new(4, 0));
+        assert_eq!(Range::new(3, 6).put_cursor(s, 2, true), Range::new(4, 2));
+        assert_eq!(Range::new(3, 6).put_cursor(s, 3, true), Range::new(3, 4));
+        assert_eq!(Range::new(3, 6).put_cursor(s, 4, true), Range::new(3, 6));
+        assert_eq!(Range::new(3, 6).put_cursor(s, 6, true), Range::new(3, 7));
+        assert_eq!(Range::new(3, 6).put_cursor(s, 8, true), Range::new(3, 9));
+
+        // Reverse ranges.
+        assert_eq!(Range::new(6, 3).put_cursor(s, 0, true), Range::new(6, 0));
+        assert_eq!(Range::new(6, 3).put_cursor(s, 2, true), Range::new(6, 2));
+        assert_eq!(Range::new(6, 3).put_cursor(s, 3, true), Range::new(6, 3));
+        assert_eq!(Range::new(6, 3).put_cursor(s, 4, true), Range::new(6, 4));
+        assert_eq!(Range::new(6, 3).put_cursor(s, 6, true), Range::new(4, 7));
+        assert_eq!(Range::new(6, 3).put_cursor(s, 8, true), Range::new(4, 9));
     }
 
     #[test]
@@ -481,24 +901,35 @@ mod test {
 
         let text = Rope::from(" abcd efg wrs   xyz 123 456");
 
-        let selection = Selection::new(smallvec![Range::new(0, 8), Range::new(10, 19),], 0);
+        let selection = Selection::new(smallvec![Range::new(0, 9), Range::new(11, 20),], 0);
 
         let result = split_on_matches(text.slice(..), &selection, &Regex::new(r"\s+").unwrap());
 
         assert_eq!(
             result.ranges(),
             &[
-                Range::new(0, 3),
-                Range::new(5, 7),
-                Range::new(10, 11),
-                Range::new(15, 17),
-                Range::new(19, 19),
+                // TODO: rather than this behavior, maybe we want it
+                // to be based on which side is the anchor?
+                //
+                // We get a leading zero-width range when there's
+                // a leading match because ranges are inclusive on
+                // the left.  Imagine, for example, if the entire
+                // selection range were matched: you'd still want
+                // at least one range to remain after the split.
+                Range::new(0, 0),
+                Range::new(1, 5),
+                Range::new(6, 9),
+                Range::new(11, 13),
+                Range::new(16, 19),
+                // In contrast to the comment above, there is no
+                // _trailing_ zero-width range despite the trailing
+                // match, because ranges are exclusive on the right.
             ]
         );
 
         assert_eq!(
             result.fragments(text.slice(..)).collect::<Vec<_>>(),
-            &["abcd", "efg", "rs", "xyz", "1"]
+            &["", "abcd", "efg", "rs", "xyz"]
         );
     }
 }

helix-core/src/surround.rs

@@ -0,0 +1,286 @@
+use crate::{search, Selection};
+use ropey::RopeSlice;
+
+pub const PAIRS: &[(char, char)] = &[
+    ('(', ')'),
+    ('[', ']'),
+    ('{', '}'),
+    ('<', '>'),
+    ('«', '»'),
+    ('「', '」'),
+    ('（', '）'),
+];
+
+/// Given any char in [PAIRS], return the open and closing chars. If not found in
+/// [PAIRS] return (ch, ch).
+///
+/// ```
+/// use helix_core::surround::get_pair;
+///
+/// assert_eq!(get_pair('['), ('[', ']'));
+/// assert_eq!(get_pair('}'), ('{', '}'));
+/// assert_eq!(get_pair('"'), ('"', '"'));
+/// ```
+pub fn get_pair(ch: char) -> (char, char) {
+    PAIRS
+        .iter()
+        .find(|(open, close)| *open == ch || *close == ch)
+        .copied()
+        .unwrap_or((ch, ch))
+}
+
+/// Find the position of surround pairs of `ch` which can be either a closing
+/// or opening pair. `n` will skip n - 1 pairs (eg. n=2 will discard (only)
+/// the first pair found and keep looking)
+pub fn find_nth_pairs_pos(
+    text: RopeSlice,
+    ch: char,
+    pos: usize,
+    n: usize,
+) -> Option<(usize, usize)> {
+    let (open, close) = get_pair(ch);
+
+    if text.len_chars() < 2 || pos >= text.len_chars() {
+        return None;
+    }
+
+    if open == close {
+        if Some(open) == text.get_char(pos) {
+            // Special case: cursor is directly on a matching char.
+            match pos {
+                0 => Some((pos, search::find_nth_next(text, close, pos + 1, n)?)),
+                _ if (pos + 1) == text.len_chars() => {
+                    Some((search::find_nth_prev(text, open, pos, n)?, pos))
+                }
+                // We return no match because there's no way to know which
+                // side of the char we should be searching on.
+                _ => None,
+            }
+        } else {
+            Some((
+                search::find_nth_prev(text, open, pos, n)?,
+                search::find_nth_next(text, close, pos, n)?,
+            ))
+        }
+    } else {
+        Some((
+            find_nth_open_pair(text, open, close, pos, n)?,
+            find_nth_close_pair(text, open, close, pos, n)?,
+        ))
+    }
+}
+
+fn find_nth_open_pair(
+    text: RopeSlice,
+    open: char,
+    close: char,
+    mut pos: usize,
+    n: usize,
+) -> Option<usize> {
+    let mut chars = text.chars_at(pos + 1);
+
+    // Adjusts pos for the first iteration, and handles the case of the
+    // cursor being *on* the close character which will get falsely stepped over
+    // if not skipped here
+    if chars.prev()? == open {
+        return Some(pos);
+    }
+
+    for _ in 0..n {
+        let mut step_over: usize = 0;
+
+        loop {
+            let c = chars.prev()?;
+            pos = pos.saturating_sub(1);
+
+            // ignore other surround pairs that are enclosed *within* our search scope
+            if c == close {
+                step_over += 1;
+            } else if c == open {
+                if step_over == 0 {
+                    break;
+                }
+
+                step_over = step_over.saturating_sub(1);
+            }
+        }
+    }
+
+    Some(pos)
+}
+
+fn find_nth_close_pair(
+    text: RopeSlice,
+    open: char,
+    close: char,
+    mut pos: usize,
+    n: usize,
+) -> Option<usize> {
+    if pos >= text.len_chars() {
+        return None;
+    }
+
+    let mut chars = text.chars_at(pos);
+
+    if chars.next()? == close {
+        return Some(pos);
+    }
+
+    for _ in 0..n {
+        let mut step_over: usize = 0;
+
+        loop {
+            let c = chars.next()?;
+            pos += 1;
+
+            if c == open {
+                step_over += 1;
+            } else if c == close {
+                if step_over == 0 {
+                    break;
+                }
+
+                step_over = step_over.saturating_sub(1);
+            }
+        }
+    }
+
+    Some(pos)
+}
+
+/// Find position of surround characters around every cursor. Returns None
+/// if any positions overlap. Note that the positions are in a flat Vec.
+/// Use get_surround_pos().chunks(2) to get matching pairs of surround positions.
+/// `ch` can be either closing or opening pair.
+pub fn get_surround_pos(
+    text: RopeSlice,
+    selection: &Selection,
+    ch: char,
+    skip: usize,
+) -> Option<Vec<usize>> {
+    let mut change_pos = Vec::new();
+
+    for range in selection {
+        let (open_pos, close_pos) = find_nth_pairs_pos(text, ch, range.head, skip)?;
+        if change_pos.contains(&open_pos) || change_pos.contains(&close_pos) {
+            return None;
+        }
+        change_pos.extend_from_slice(&[open_pos, close_pos]);
+    }
+    Some(change_pos)
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::Range;
+
+    use ropey::Rope;
+    use smallvec::SmallVec;
+
+    #[test]
+    fn test_find_nth_pairs_pos() {
+        let doc = Rope::from("some (text) here");
+        let slice = doc.slice(..);
+
+        // cursor on [t]ext
+        assert_eq!(find_nth_pairs_pos(slice, '(', 6, 1), Some((5, 10)));
+        assert_eq!(find_nth_pairs_pos(slice, ')', 6, 1), Some((5, 10)));
+        // cursor on so[m]e
+        assert_eq!(find_nth_pairs_pos(slice, '(', 2, 1), None);
+        // cursor on bracket itself
+        assert_eq!(find_nth_pairs_pos(slice, '(', 5, 1), Some((5, 10)));
+        assert_eq!(find_nth_pairs_pos(slice, '(', 10, 1), Some((5, 10)));
+    }
+
+    #[test]
+    fn test_find_nth_pairs_pos_skip() {
+        let doc = Rope::from("(so (many (good) text) here)");
+        let slice = doc.slice(..);
+
+        // cursor on go[o]d
+        assert_eq!(find_nth_pairs_pos(slice, '(', 13, 1), Some((10, 15)));
+        assert_eq!(find_nth_pairs_pos(slice, '(', 13, 2), Some((4, 21)));
+        assert_eq!(find_nth_pairs_pos(slice, '(', 13, 3), Some((0, 27)));
+    }
+
+    #[test]
+    fn test_find_nth_pairs_pos_same() {
+        let doc = Rope::from("'so 'many 'good' text' here'");
+        let slice = doc.slice(..);
+
+        // cursor on go[o]d
+        assert_eq!(find_nth_pairs_pos(slice, '\'', 13, 1), Some((10, 15)));
+        assert_eq!(find_nth_pairs_pos(slice, '\'', 13, 2), Some((4, 21)));
+        assert_eq!(find_nth_pairs_pos(slice, '\'', 13, 3), Some((0, 27)));
+        // cursor on the quotes
+        assert_eq!(find_nth_pairs_pos(slice, '\'', 10, 1), None);
+        // this is the best we can do since opening and closing pairs are same
+        assert_eq!(find_nth_pairs_pos(slice, '\'', 0, 1), Some((0, 4)));
+        assert_eq!(find_nth_pairs_pos(slice, '\'', 27, 1), Some((21, 27)));
+    }
+
+    #[test]
+    fn test_find_nth_pairs_pos_step() {
+        let doc = Rope::from("((so)((many) good (text))(here))");
+        let slice = doc.slice(..);
+
+        // cursor on go[o]d
+        assert_eq!(find_nth_pairs_pos(slice, '(', 15, 1), Some((5, 24)));
+        assert_eq!(find_nth_pairs_pos(slice, '(', 15, 2), Some((0, 31)));
+    }
+
+    #[test]
+    fn test_find_nth_pairs_pos_mixed() {
+        let doc = Rope::from("(so [many {good} text] here)");
+        let slice = doc.slice(..);
+
+        // cursor on go[o]d
+        assert_eq!(find_nth_pairs_pos(slice, '{', 13, 1), Some((10, 15)));
+        assert_eq!(find_nth_pairs_pos(slice, '[', 13, 1), Some((4, 21)));
+        assert_eq!(find_nth_pairs_pos(slice, '(', 13, 1), Some((0, 27)));
+    }
+
+    #[test]
+    fn test_get_surround_pos() {
+        let doc = Rope::from("(some) (chars)\n(newline)");
+        let slice = doc.slice(..);
+        let selection = Selection::new(
+            SmallVec::from_slice(&[Range::point(2), Range::point(9), Range::point(20)]),
+            0,
+        );
+
+        // cursor on s[o]me, c[h]ars, newl[i]ne
+        assert_eq!(
+            get_surround_pos(slice, &selection, '(', 1)
+                .unwrap()
+                .as_slice(),
+            &[0, 5, 7, 13, 15, 23]
+        );
+    }
+
+    #[test]
+    fn test_get_surround_pos_bail() {
+        let doc = Rope::from("[some]\n(chars)xx\n(newline)");
+        let slice = doc.slice(..);
+
+        let selection =
+            Selection::new(SmallVec::from_slice(&[Range::point(2), Range::point(9)]), 0);
+
+        // cursor on s[o]me, c[h]ars
+        assert_eq!(
+            get_surround_pos(slice, &selection, '(', 1),
+            None // different surround chars
+        );
+
+        let selection = Selection::new(
+            SmallVec::from_slice(&[Range::point(14), Range::point(24)]),
+            0,
+        );
+        // cursor on [x]x, newli[n]e
+        assert_eq!(
+            get_surround_pos(slice, &selection, '(', 1),
+            None // overlapping surround chars
+        );
+    }
+}

helix-core/src/syntax.rs

@@ -1,12 +1,20 @@
-use crate::{regex::Regex, Change, Rope, RopeSlice, Transaction};
-pub use helix_syntax::{get_language, get_language_name, Lang};
+use crate::{
+    chars::char_is_line_ending,
+    regex::Regex,
+    transaction::{ChangeSet, Operation},
+    Rope, RopeSlice, Tendril,
+};
+
+pub use helix_syntax::get_language;
+
+use arc_swap::ArcSwap;
 
 use std::{
     borrow::Cow,
     cell::RefCell,
     collections::{HashMap, HashSet},
     fmt,
-    path::{Path, PathBuf},
+    path::Path,
     sync::Arc,
 };
 
@@ -23,10 +31,12 @@ pub struct Configuration {
 #[serde(rename_all = "kebab-case")]
 pub struct LanguageConfiguration {
     #[serde(rename = "name")]
-    pub(crate) language_id: Lang,
+    pub(crate) language_id: String,
     pub scope: String,           // source.rust
     pub file_types: Vec<String>, // filename ends_with? <Gemfile, rb, etc>
     pub roots: Vec<String>,      // these indicate project roots <.git, Cargo.toml>
+    pub comment_token: Option<String>,
+    pub config: Option<String>,
 
     #[serde(default)]
     pub auto_format: bool,
@@ -74,7 +84,6 @@ pub struct IndentQuery {
     pub outdent: HashSet<String>,
 }
 
-#[cfg(not(feature = "embed_runtime"))]
 fn load_runtime_file(language: &str, filename: &str) -> Result<String, std::io::Error> {
     let path = crate::RUNTIME_DIR
         .join("queries")
@@ -83,34 +92,6 @@ fn load_runtime_file(language: &str, filename: &str) -> Result<String, std::io::
     std::fs::read_to_string(&path)
 }
 
-#[cfg(feature = "embed_runtime")]
-fn load_runtime_file(language: &str, filename: &str) -> Result<String, Box<dyn std::error::Error>> {
-    use std::fmt;
-
-    #[derive(rust_embed::RustEmbed)]
-    #[folder = "../runtime/"]
-    struct Runtime;
-
-    #[derive(Debug)]
-    struct EmbeddedFileNotFoundError {
-        path: PathBuf,
-    }
-    impl std::error::Error for EmbeddedFileNotFoundError {}
-    impl fmt::Display for EmbeddedFileNotFoundError {
-        fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-            write!(f, "failed to load embedded file {}", self.path.display())
-        }
-    }
-
-    let path = PathBuf::from("queries").join(language).join(filename);
-
-    if let Some(query_bytes) = Runtime::get(&path.display().to_string()) {
-        String::from_utf8(query_bytes.to_vec()).map_err(|err| err.into())
-    } else {
-        Err(Box::new(EmbeddedFileNotFoundError { path }))
-    }
-}
-
 fn read_query(language: &str, filename: &str) -> String {
     static INHERITS_REGEX: Lazy<Regex> =
         Lazy::new(|| Regex::new(r";+\s*inherits\s*:?\s*([a-z_,()]+)\s*").unwrap());
@@ -143,10 +124,8 @@ fn read_query(language: &str, filename: &str) -> String {
 }
 
 impl LanguageConfiguration {
-    pub fn highlight_config(&self, scopes: &[String]) -> Option<Arc<HighlightConfiguration>> {
-        self.highlight_config
-            .get_or_init(|| {
-                let language = get_language_name(self.language_id).to_ascii_lowercase();
+    fn initialize_highlight(&self, scopes: &[String]) -> Option<Arc<HighlightConfiguration>> {
+        let language = self.language_id.to_ascii_lowercase();
 
         let highlights_query = read_query(&language, "highlights.scm");
         // always highlight syntax errors
@@ -154,33 +133,47 @@ impl LanguageConfiguration {
 
         let injections_query = read_query(&language, "injections.scm");
 
-                let locals_query = "";
+        let locals_query = read_query(&language, "locals.scm");
 
         if highlights_query.is_empty() {
             None
         } else {
-                    let language = get_language(self.language_id);
-                    let mut config = HighlightConfiguration::new(
+            let language = get_language(&crate::RUNTIME_DIR, &self.language_id).ok()?;
+            let config = HighlightConfiguration::new(
                 language,
                 &highlights_query,
                 &injections_query,
-                        locals_query,
+                &locals_query,
             )
             .unwrap(); // TODO: no unwrap
             config.configure(scopes);
             Some(Arc::new(config))
         }
-            })
+    }
+
+    pub fn reconfigure(&self, scopes: &[String]) {
+        if let Some(Some(config)) = self.highlight_config.get() {
+            config.configure(scopes);
+        }
+    }
+
+    pub fn highlight_config(&self, scopes: &[String]) -> Option<Arc<HighlightConfiguration>> {
+        self.highlight_config
+            .get_or_init(|| self.initialize_highlight(scopes))
             .clone()
     }
 
+    pub fn is_highlight_initialized(&self) -> bool {
+        self.highlight_config.get().is_some()
+    }
+
     pub fn indent_query(&self) -> Option<&IndentQuery> {
         self.indent_query
             .get_or_init(|| {
-                let language = get_language_name(self.language_id).to_ascii_lowercase();
+                let language = self.language_id.to_ascii_lowercase();
 
                 let toml = load_runtime_file(&language, "indents.toml").ok()?;
-                toml::from_slice(&toml.as_bytes()).ok()
+                toml::from_slice(toml.as_bytes()).ok()
             })
             .as_ref()
     }
@@ -190,22 +183,18 @@ impl LanguageConfiguration {
     }
 }
 
-pub static LOADER: OnceCell<Loader> = OnceCell::new();
-
 #[derive(Debug)]
 pub struct Loader {
     // highlight_names ?
     language_configs: Vec<Arc<LanguageConfiguration>>,
     language_config_ids_by_file_type: HashMap<String, usize>, // Vec<usize>
-    scopes: Vec<String>,
 }
 
 impl Loader {
-    pub fn new(config: Configuration, scopes: Vec<String>) -> Self {
+    pub fn new(config: Configuration) -> Self {
         let mut loader = Self {
             language_configs: Vec::new(),
             language_config_ids_by_file_type: HashMap::new(),
-            scopes,
         };
 
         for config in config.language {
@@ -225,10 +214,6 @@ impl Loader {
         loader
     }
 
-    pub fn scopes(&self) -> &[String] {
-        &self.scopes
-    }
-
     pub fn language_config_for_file_name(&self, path: &Path) -> Option<Arc<LanguageConfiguration>> {
         // Find all the language configurations that match this file name
         // or a suffix of the file name.
@@ -253,6 +238,10 @@ impl Loader {
             .find(|config| config.scope == scope)
             .cloned()
     }
+
+    pub fn language_configs_iter(&self) -> impl Iterator<Item = &Arc<LanguageConfiguration>> {
+        self.language_configs.iter()
+    }
 }
 
 pub struct TsParser {
@@ -316,7 +305,8 @@ impl Syntax {
 
         // update root layer
         PARSER.with(|ts_parser| {
-            syntax.root_layer.parse(
+            // TODO: handle the returned `Result` properly.
+            let _ = syntax.root_layer.parse(
                 &mut ts_parser.borrow_mut(),
                 &syntax.config,
                 source,
@@ -371,7 +361,7 @@ impl Syntax {
         source: RopeSlice<'a>,
         range: Option<std::ops::Range<usize>>,
         cancellation_flag: Option<&'a AtomicUsize>,
-        mut injection_callback: impl FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
+        injection_callback: impl FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
     ) -> impl Iterator<Item = Result<HighlightEvent, Error>> + 'a {
         // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which
         // prevents them from being moved. But both of these values are really just
@@ -463,12 +453,6 @@ pub struct LanguageLayer {
     pub(crate) tree: Option<Tree>,
 }
 
-use crate::{
-    coords_at_pos,
-    transaction::{ChangeSet, Operation},
-    Tendril,
-};
-
 impl LanguageLayer {
     // pub fn new() -> Self {
     //     Self { tree: None }
@@ -484,8 +468,8 @@ impl LanguageLayer {
         ts_parser: &mut TsParser,
         config: &HighlightConfiguration,
         source: &Rope,
-        mut depth: usize,
-        mut ranges: Vec<Range>,
+        _depth: usize,
+        ranges: Vec<Range>,
     ) -> Result<(), Error> {
         if ts_parser.parser.set_included_ranges(&ranges).is_ok() {
             ts_parser
@@ -556,7 +540,6 @@ impl LanguageLayer {
     ) -> Vec<tree_sitter::InputEdit> {
         use Operation::*;
         let mut old_pos = 0;
-        let mut new_pos = 0;
 
         let mut edits = Vec::new();
 
@@ -579,9 +562,10 @@ impl LanguageLayer {
                 mut column,
             } = point;
 
-            // TODO: there should be a better way here
-            for ch in text.bytes() {
-                if ch == b'\n' {
+            // TODO: there should be a better way here.
+            let mut chars = text.chars().peekable();
+            while let Some(ch) = chars.next() {
+                if char_is_line_ending(ch) && !(ch == '\r' && chars.peek() == Some(&'\n')) {
                     row += 1;
                     column = 0;
                 } else {
@@ -599,9 +583,7 @@ impl LanguageLayer {
             let mut old_end = old_pos + len;
 
             match change {
-                Retain(_) => {
-                    new_pos += len;
-                }
+                Retain(_) => {}
                 Delete(_) => {
                     let (start_byte, start_position) = point_at_pos(old_text, old_pos);
                     let (old_end_byte, old_end_position) = point_at_pos(old_text, old_end);
@@ -625,8 +607,6 @@ impl LanguageLayer {
                 Insert(s) => {
                     let (start_byte, start_position) = point_at_pos(old_text, old_pos);
 
-                    let ins = s.chars().count();
-
                     // a subsequent delete means a replace, consume it
                     if let Some(Delete(len)) = iter.peek() {
                         old_end = old_pos + len;
@@ -654,8 +634,6 @@ impl LanguageLayer {
                             new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)
                         });
                     }
-
-                    new_pos += ins;
                 }
             }
             old_pos = old_end;
@@ -678,8 +656,10 @@ impl LanguageLayer {
         let edits = Self::generate_edits(old_source.slice(..), changeset);
 
         // Notify the tree about all the changes
-        for edit in edits {
-            self.tree.as_mut().unwrap().edit(&edit);
+        for edit in edits.iter().rev() {
+            // apply the edits in reverse. If we applied them in order then edit 1 would disrupt
+            // the positioning of edit 2
+            self.tree.as_mut().unwrap().edit(edit);
         }
 
         self.parse(
@@ -771,7 +751,7 @@ pub struct HighlightConfiguration {
     combined_injections_query: Option<Query>,
     locals_pattern_index: usize,
     highlights_pattern_index: usize,
-    highlight_indices: Vec<Option<Highlight>>,
+    highlight_indices: ArcSwap<Vec<Option<Highlight>>>,
     non_local_variable_patterns: Vec<bool>,
     injection_content_capture_index: Option<u32>,
     injection_language_capture_index: Option<u32>,
@@ -923,7 +903,7 @@ impl HighlightConfiguration {
             }
         }
 
-        let highlight_indices = vec![None; query.capture_names().len()];
+        let highlight_indices = ArcSwap::from_pointee(vec![None; query.capture_names().len()]);
         Ok(Self {
             language,
             query,
@@ -956,17 +936,20 @@ impl HighlightConfiguration {
     ///
     /// When highlighting, results are returned as `Highlight` values, which contain the index
     /// of the matched highlight this list of highlight names.
-    pub fn configure(&mut self, recognized_names: &[String]) {
+    pub fn configure(&self, recognized_names: &[String]) {
         let mut capture_parts = Vec::new();
-        self.highlight_indices.clear();
-        self.highlight_indices
-            .extend(self.query.capture_names().iter().map(move |capture_name| {
+        let indices: Vec<_> = self
+            .query
+            .capture_names()
+            .iter()
+            .map(move |capture_name| {
                 capture_parts.clear();
                 capture_parts.extend(capture_name.split('.'));
 
                 let mut best_index = None;
                 let mut best_match_len = 0;
                 for (i, recognized_name) in recognized_names.iter().enumerate() {
+                    let recognized_name = recognized_name;
                     let mut len = 0;
                     let mut matches = true;
                     for part in recognized_name.split('.') {
@@ -982,7 +965,10 @@ impl HighlightConfiguration {
                     }
                 }
                 best_index.map(Highlight)
-            }));
+            })
+            .collect();
+
+        self.highlight_indices.store(Arc::new(indices));
     }
 }
 
@@ -1463,7 +1449,6 @@ where
                 // local scope at the top of the scope stack.
                 else if Some(capture.index) == layer.config.local_def_capture_index {
                     reference_highlight = None;
-                    definition_highlight = None;
                     let scope = layer.scope_stack.last_mut().unwrap();
 
                     let mut value_range = 0..0;
@@ -1561,7 +1546,7 @@ where
                 }
             }
 
-            let current_highlight = layer.config.highlight_indices[capture.index as usize];
+            let current_highlight = layer.config.highlight_indices.load()[capture.index as usize];
 
             // If this node represents a local definition, then store the current
             // highlight value on the local scope entry representing this node.
@@ -1627,14 +1612,153 @@ fn injection_for_match<'a>(
     (language_name, content_node, include_children)
 }
 
-fn shrink_and_clear<T>(vec: &mut Vec<T>, capacity: usize) {
-    if vec.len() > capacity {
-        vec.truncate(capacity);
-        vec.shrink_to_fit();
+// fn shrink_and_clear<T>(vec: &mut Vec<T>, capacity: usize) {
+//     if vec.len() > capacity {
+//         vec.truncate(capacity);
+//         vec.shrink_to_fit();
+//     }
+//     vec.clear();
+// }
+
+pub struct Merge<I> {
+    iter: I,
+    spans: Box<dyn Iterator<Item = (usize, std::ops::Range<usize>)>>,
+
+    next_event: Option<HighlightEvent>,
+    next_span: Option<(usize, std::ops::Range<usize>)>,
+
+    queue: Vec<HighlightEvent>,
 }
-    vec.clear();
+
+/// Merge a list of spans into the highlight event stream.
+pub fn merge<I: Iterator<Item = HighlightEvent>>(
+    iter: I,
+    spans: Vec<(usize, std::ops::Range<usize>)>,
+) -> Merge<I> {
+    let spans = Box::new(spans.into_iter());
+    let mut merge = Merge {
+        iter,
+        spans,
+        next_event: None,
+        next_span: None,
+        queue: Vec::new(),
+    };
+    merge.next_event = merge.iter.next();
+    merge.next_span = merge.spans.next();
+    merge
 }
 
+impl<I: Iterator<Item = HighlightEvent>> Iterator for Merge<I> {
+    type Item = HighlightEvent;
+    fn next(&mut self) -> Option<Self::Item> {
+        use HighlightEvent::*;
+        if let Some(event) = self.queue.pop() {
+            return Some(event);
+        }
+
+        loop {
+            match (self.next_event, &self.next_span) {
+                // this happens when range is partially or fully offscreen
+                (Some(Source { start, .. }), Some((span, range))) if start > range.start => {
+                    if start > range.end {
+                        self.next_span = self.spans.next();
+                    } else {
+                        self.next_span = Some((*span, start..range.end));
+                    };
+                }
+                _ => break,
+            }
+        }
+
+        match (self.next_event, &self.next_span) {
+            (Some(HighlightStart(i)), _) => {
+                self.next_event = self.iter.next();
+                Some(HighlightStart(i))
+            }
+            (Some(HighlightEnd), _) => {
+                self.next_event = self.iter.next();
+                Some(HighlightEnd)
+            }
+            (Some(Source { start, end }), Some((_, range))) if start < range.start => {
+                let intersect = range.start.min(end);
+                let event = Source {
+                    start,
+                    end: intersect,
+                };
+
+                if end == intersect {
+                    // the event is complete
+                    self.next_event = self.iter.next();
+                } else {
+                    // subslice the event
+                    self.next_event = Some(Source {
+                        start: intersect,
+                        end,
+                    });
+                };
+
+                Some(event)
+            }
+            (Some(Source { start, end }), Some((span, range))) if start == range.start => {
+                let intersect = range.end.min(end);
+                let event = HighlightStart(Highlight(*span));
+
+                // enqueue in reverse order
+                self.queue.push(HighlightEnd);
+                self.queue.push(Source {
+                    start,
+                    end: intersect,
+                });
+
+                if end == intersect {
+                    // the event is complete
+                    self.next_event = self.iter.next();
+                } else {
+                    // subslice the event
+                    self.next_event = Some(Source {
+                        start: intersect,
+                        end,
+                    });
+                };
+
+                if intersect == range.end {
+                    self.next_span = self.spans.next();
+                } else {
+                    self.next_span = Some((*span, intersect..range.end));
+                }
+
+                Some(event)
+            }
+            (Some(event), None) => {
+                self.next_event = self.iter.next();
+                Some(event)
+            }
+            // Can happen if cursor at EOF and/or diagnostic reaches past the end.
+            // We need to actually emit events for the cursor-at-EOF situation,
+            // even though the range is past the end of the text.  This needs to be
+            // handled appropriately by the drawing code by not assuming that
+            // all `Source` events point to valid indices in the rope.
+            (None, Some((span, range))) => {
+                let event = HighlightStart(Highlight(*span));
+                self.queue.push(HighlightEnd);
+                self.queue.push(Source {
+                    start: range.start,
+                    end: range.end,
+                });
+                self.next_span = self.spans.next();
+                Some(event)
+            }
+            (None, None) => None,
+            e => unreachable!("{:?}", e),
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::{Rope, Transaction};
+
     #[test]
     fn test_parser() {
         let highlight_names: Vec<String> = [
@@ -1662,8 +1786,8 @@ fn test_parser() {
         .map(String::from)
         .collect();
 
-    let language = get_language(Lang::Rust);
-    let mut config = HighlightConfiguration::new(
+        let language = get_language(&crate::RUNTIME_DIR, "Rust").unwrap();
+        let config = HighlightConfiguration::new(
             language,
             &std::fs::read_to_string(
                 "../helix-syntax/languages/tree-sitter-rust/queries/highlights.scm",
@@ -1707,7 +1831,7 @@ fn test_input_edits() {
         use crate::State;
         use tree_sitter::InputEdit;
 
-    let mut state = State::new("hello world!\ntest 123".into());
+        let state = State::new("hello world!\ntest 123".into());
         let transaction = Transaction::change(
             &state.doc,
             vec![(6, 11, Some("test".into())), (12, 17, None)].into_iter(),
@@ -1767,3 +1891,4 @@ fn test_load_runtime_file() {
         let results = load_runtime_file("rust", "does-not-exist");
         assert!(results.is_err());
     }
+}

helix-core/src/textobject.rs

@@ -0,0 +1,321 @@
+use ropey::RopeSlice;
+
+use crate::chars::{categorize_char, char_is_whitespace, CharCategory};
+use crate::graphemes::next_grapheme_boundary;
+use crate::movement::Direction;
+use crate::surround;
+use crate::Range;
+
+fn find_word_boundary(slice: RopeSlice, mut pos: usize, direction: Direction) -> usize {
+    use CharCategory::{Eol, Whitespace};
+
+    let iter = match direction {
+        Direction::Forward => slice.chars_at(pos),
+        Direction::Backward => {
+            let mut iter = slice.chars_at(pos);
+            iter.reverse();
+            iter
+        }
+    };
+
+    let mut prev_category = match direction {
+        Direction::Forward if pos == 0 => Whitespace,
+        Direction::Forward => categorize_char(slice.char(pos - 1)),
+        Direction::Backward if pos == slice.len_chars() => Whitespace,
+        Direction::Backward => categorize_char(slice.char(pos)),
+    };
+
+    for ch in iter {
+        match categorize_char(ch) {
+            Eol | Whitespace => return pos,
+            category => {
+                if category != prev_category && pos != 0 && pos != slice.len_chars() {
+                    return pos;
+                } else {
+                    match direction {
+                        Direction::Forward => pos += 1,
+                        Direction::Backward => pos = pos.saturating_sub(1),
+                    }
+                    prev_category = category;
+                }
+            }
+        }
+    }
+
+    pos
+}
+
+#[derive(Copy, Clone, PartialEq, Eq, Debug)]
+pub enum TextObject {
+    Around,
+    Inside,
+}
+
+// count doesn't do anything yet
+pub fn textobject_word(
+    slice: RopeSlice,
+    range: Range,
+    textobject: TextObject,
+    _count: usize,
+) -> Range {
+    let pos = range.cursor(slice);
+
+    let word_start = find_word_boundary(slice, pos, Direction::Backward);
+    let word_end = match slice.get_char(pos).map(categorize_char) {
+        None | Some(CharCategory::Whitespace | CharCategory::Eol) => pos,
+        _ => find_word_boundary(slice, pos + 1, Direction::Forward),
+    };
+
+    // Special case.
+    if word_start == word_end {
+        return Range::new(word_start, word_end);
+    }
+
+    match textobject {
+        TextObject::Inside => Range::new(word_start, word_end),
+        TextObject::Around => {
+            let whitespace_count_right = slice
+                .chars_at(word_end)
+                .take_while(|c| char_is_whitespace(*c))
+                .count();
+
+            if whitespace_count_right > 0 {
+                Range::new(word_start, word_end + whitespace_count_right)
+            } else {
+                let whitespace_count_left = {
+                    let mut iter = slice.chars_at(word_start);
+                    iter.reverse();
+                    iter.take_while(|c| char_is_whitespace(*c)).count()
+                };
+                Range::new(word_start - whitespace_count_left, word_end)
+            }
+        }
+    }
+}
+
+pub fn textobject_surround(
+    slice: RopeSlice,
+    range: Range,
+    textobject: TextObject,
+    ch: char,
+    count: usize,
+) -> Range {
+    surround::find_nth_pairs_pos(slice, ch, range.head, count)
+        .map(|(anchor, head)| match textobject {
+            TextObject::Inside => Range::new(next_grapheme_boundary(slice, anchor), head),
+            TextObject::Around => Range::new(anchor, next_grapheme_boundary(slice, head)),
+        })
+        .unwrap_or(range)
+}
+
+#[cfg(test)]
+mod test {
+    use super::TextObject::*;
+    use super::*;
+
+    use crate::Range;
+    use ropey::Rope;
+
+    #[test]
+    fn test_textobject_word() {
+        // (text, [(cursor position, textobject, final range), ...])
+        let tests = &[
+            (
+                "cursor at beginning of doc",
+                vec![(0, Inside, (0, 6)), (0, Around, (0, 7))],
+            ),
+            (
+                "cursor at middle of word",
+                vec![
+                    (13, Inside, (10, 16)),
+                    (10, Inside, (10, 16)),
+                    (15, Inside, (10, 16)),
+                    (13, Around, (10, 17)),
+                    (10, Around, (10, 17)),
+                    (15, Around, (10, 17)),
+                ],
+            ),
+            (
+                "cursor between word whitespace",
+                vec![(6, Inside, (6, 6)), (6, Around, (6, 6))],
+            ),
+            (
+                "cursor on word before newline\n",
+                vec![
+                    (22, Inside, (22, 29)),
+                    (28, Inside, (22, 29)),
+                    (25, Inside, (22, 29)),
+                    (22, Around, (21, 29)),
+                    (28, Around, (21, 29)),
+                    (25, Around, (21, 29)),
+                ],
+            ),
+            (
+                "cursor on newline\nnext line",
+                vec![(17, Inside, (17, 17)), (17, Around, (17, 17))],
+            ),
+            (
+                "cursor on word after newline\nnext line",
+                vec![
+                    (29, Inside, (29, 33)),
+                    (30, Inside, (29, 33)),
+                    (32, Inside, (29, 33)),
+                    (29, Around, (29, 34)),
+                    (30, Around, (29, 34)),
+                    (32, Around, (29, 34)),
+                ],
+            ),
+            (
+                "cursor on #$%:;* punctuation",
+                vec![
+                    (13, Inside, (10, 16)),
+                    (10, Inside, (10, 16)),
+                    (15, Inside, (10, 16)),
+                    (13, Around, (10, 17)),
+                    (10, Around, (10, 17)),
+                    (15, Around, (10, 17)),
+                ],
+            ),
+            (
+                "cursor on punc%^#$:;.tuation",
+                vec![
+                    (14, Inside, (14, 21)),
+                    (20, Inside, (14, 21)),
+                    (17, Inside, (14, 21)),
+                    (14, Around, (14, 21)),
+                    (20, Around, (14, 21)),
+                    (17, Around, (14, 21)),
+                ],
+            ),
+            (
+                "cursor in   extra whitespace",
+                vec![
+                    (9, Inside, (9, 9)),
+                    (10, Inside, (10, 10)),
+                    (11, Inside, (11, 11)),
+                    (9, Around, (9, 9)),
+                    (10, Around, (10, 10)),
+                    (11, Around, (11, 11)),
+                ],
+            ),
+            (
+                "cursor on word   with extra whitespace",
+                vec![(11, Inside, (10, 14)), (11, Around, (10, 17))],
+            ),
+            (
+                "cursor at end with extra   whitespace",
+                vec![(28, Inside, (27, 37)), (28, Around, (24, 37))],
+            ),
+            (
+                "cursor at end of doc",
+                vec![(19, Inside, (17, 20)), (19, Around, (16, 20))],
+            ),
+        ];
+
+        for (sample, scenario) in tests {
+            let doc = Rope::from(*sample);
+            let slice = doc.slice(..);
+            for &case in scenario {
+                let (pos, objtype, expected_range) = case;
+                let result = textobject_word(slice, Range::point(pos), objtype, 1);
+                assert_eq!(
+                    result,
+                    expected_range.into(),
+                    "\nCase failed: {:?} - {:?}",
+                    sample,
+                    case
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn test_textobject_surround() {
+        // (text, [(cursor position, textobject, final range, count), ...])
+        let tests = &[
+            (
+                "simple (single) surround pairs",
+                vec![
+                    (3, Inside, (3, 3), '(', 1),
+                    (7, Inside, (8, 14), ')', 1),
+                    (10, Inside, (8, 14), '(', 1),
+                    (14, Inside, (8, 14), ')', 1),
+                    (3, Around, (3, 3), '(', 1),
+                    (7, Around, (7, 15), ')', 1),
+                    (10, Around, (7, 15), '(', 1),
+                    (14, Around, (7, 15), ')', 1),
+                ],
+            ),
+            (
+                "samexx 'single' surround pairs",
+                vec![
+                    (3, Inside, (3, 3), '\'', 1),
+                    (7, Inside, (7, 7), '\'', 1),
+                    (10, Inside, (8, 14), '\'', 1),
+                    (14, Inside, (14, 14), '\'', 1),
+                    (3, Around, (3, 3), '\'', 1),
+                    (7, Around, (7, 7), '\'', 1),
+                    (10, Around, (7, 15), '\'', 1),
+                    (14, Around, (14, 14), '\'', 1),
+                ],
+            ),
+            (
+                "(nested (surround (pairs)) 3 levels)",
+                vec![
+                    (0, Inside, (1, 35), '(', 1),
+                    (6, Inside, (1, 35), ')', 1),
+                    (8, Inside, (9, 25), '(', 1),
+                    (8, Inside, (9, 35), ')', 2),
+                    (20, Inside, (9, 25), '(', 2),
+                    (20, Inside, (1, 35), ')', 3),
+                    (0, Around, (0, 36), '(', 1),
+                    (6, Around, (0, 36), ')', 1),
+                    (8, Around, (8, 26), '(', 1),
+                    (8, Around, (8, 36), ')', 2),
+                    (20, Around, (8, 26), '(', 2),
+                    (20, Around, (0, 36), ')', 3),
+                ],
+            ),
+            (
+                "(mixed {surround [pair] same} line)",
+                vec![
+                    (2, Inside, (1, 34), '(', 1),
+                    (9, Inside, (8, 28), '{', 1),
+                    (18, Inside, (18, 22), '[', 1),
+                    (2, Around, (0, 35), '(', 1),
+                    (9, Around, (7, 29), '{', 1),
+                    (18, Around, (17, 23), '[', 1),
+                ],
+            ),
+            (
+                "(stepped (surround) pairs (should) skip)",
+                vec![(22, Inside, (1, 39), '(', 1), (22, Around, (0, 40), '(', 1)],
+            ),
+            (
+                "[surround pairs{\non different]\nlines}",
+                vec![
+                    (7, Inside, (1, 29), '[', 1),
+                    (15, Inside, (16, 36), '{', 1),
+                    (7, Around, (0, 30), '[', 1),
+                    (15, Around, (15, 37), '{', 1),
+                ],
+            ),
+        ];
+
+        for (sample, scenario) in tests {
+            let doc = Rope::from(*sample);
+            let slice = doc.slice(..);
+            for &case in scenario {
+                let (pos, objtype, expected_range, ch, count) = case;
+                let result = textobject_surround(slice, Range::point(pos), objtype, ch, count);
+                assert_eq!(
+                    result,
+                    expected_range.into(),
+                    "\nCase failed: {:?} - {:?}",
+                    sample,
+                    case
+                );
+            }
+        }
+    }
+}

helix-core/src/transaction.rs

@@ -1,5 +1,5 @@
-use crate::{Range, Rope, Selection, State, Tendril};
-use std::{borrow::Cow, convert::TryFrom};
+use crate::{Range, Rope, Selection, Tendril};
+use std::borrow::Cow;
 
 /// (from, to, replacement)
 pub type Change = (usize, usize, Option<Tendril>);
@@ -163,7 +163,7 @@ impl ChangeSet {
                     head_a = a;
                     head_b = changes_b.next();
                 }
-                (None, _) | (_, None) => return unreachable!(),
+                (None, val) | (val, None) => unreachable!("({:?})", val),
                 (Some(Retain(i)), Some(Retain(j))) => match i.cmp(&j) {
                     Ordering::Less => {
                         changes.retain(i);
@@ -193,15 +193,16 @@ impl ChangeSet {
                             head_b = changes_b.next();
                         }
                         Ordering::Greater => {
+                            // TODO: cover this with a test
                             // figure out the byte index of the truncated string end
-                            let (pos, _) = s.char_indices().nth(len - j).unwrap();
-                            s.pop_front(s.len() as u32 - pos as u32);
+                            let (pos, _) = s.char_indices().nth(j).unwrap();
+                            s.pop_front(pos as u32);
                             head_a = Some(Insert(s));
                             head_b = changes_b.next();
                         }
                     }
                 }
-                (Some(Insert(mut s)), Some(Retain(j))) => {
+                (Some(Insert(s)), Some(Retain(j))) => {
                     let len = s.chars().count();
                     match len.cmp(&j) {
                         Ordering::Less => {
@@ -273,7 +274,6 @@ impl ChangeSet {
         let mut changes = Self::with_capacity(self.changes.len());
 
         let mut pos = 0;
-        let mut len = 0;
 
         for change in &self.changes {
             use Operation::*;
@@ -473,11 +473,13 @@ impl Transaction {
     /// Generate a transaction from a set of changes.
     pub fn change<I>(doc: &Rope, changes: I) -> Self
     where
-        I: IntoIterator<Item = Change> + ExactSizeIterator,
+        I: IntoIterator<Item = Change> + Iterator,
     {
         let len = doc.len_chars();
 
-        let mut changeset = ChangeSet::with_capacity(2 * changes.len() + 1); // rough estimate
+        let (lower, upper) = changes.size_hint();
+        let size = upper.unwrap_or(lower);
+        let mut changeset = ChangeSet::with_capacity(2 * size + 1); // rough estimate
 
         // TODO: verify ranges are ordered and not overlapping or change will panic.
 
@@ -580,6 +582,7 @@ impl<'a> Iterator for ChangeIterator<'a> {
 #[cfg(test)]
 mod test {
     use super::*;
+    use crate::State;
 
     #[test]
     fn composition() {
@@ -598,7 +601,7 @@ mod test {
         };
 
         let b = ChangeSet {
-            changes: vec![Delete(10), Insert("world".into()), Retain(5)],
+            changes: vec![Delete(10), Insert("世orld".into()), Retain(5)],
             len: 15,
             len_after: 10,
         };
@@ -609,7 +612,7 @@ mod test {
         let composed = a.compose(b);
         assert_eq!(composed.len, 8);
         assert!(composed.apply(&mut text));
-        assert_eq!(text, "world! abc");
+        assert_eq!(text, "世orld! abc");
     }
 
     #[test]
@@ -698,7 +701,7 @@ mod test {
 
     #[test]
     fn changes_iter() {
-        let mut state = State::new("hello world!\ntest 123".into());
+        let state = State::new("hello world!\ntest 123".into());
         let changes = vec![(6, 11, Some("void".into())), (12, 17, None)];
         let transaction = Transaction::change(&state.doc, changes.clone().into_iter());
         assert_eq!(transaction.changes_iter().collect::<Vec<_>>(), changes);
@@ -730,7 +733,7 @@ mod test {
         // retain 1, e
         // retain 2, l
 
-        let mut changes = t1
+        let changes = t1
             .changes
             .compose(t2.changes)
             .compose(t3.changes)
@@ -745,7 +748,7 @@ mod test {
     #[test]
     fn combine_with_empty() {
         let empty = Rope::from("");
-        let mut a = ChangeSet::new(&empty);
+        let a = ChangeSet::new(&empty);
 
         let mut b = ChangeSet::new(&empty);
         b.insert("a".into());
@@ -761,7 +764,7 @@ mod test {
         const TEST_CASE: &'static str = "Hello, これはヘリックスエディターです！";
 
         let empty = Rope::from("");
-        let mut a = ChangeSet::new(&empty);
+        let a = ChangeSet::new(&empty);
 
         let mut b = ChangeSet::new(&empty);
         b.insert(TEST_CASE.into());

helix-lsp/Cargo.toml

@@ -1,23 +1,27 @@
 [package]
 name = "helix-lsp"
-version = "0.2.0"
+version = "0.4.1"
 authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
 edition = "2018"
 license = "MPL-2.0"
+description = "LSP client implementation for Helix project"
+categories = ["editor"]
+repository = "https://github.com/helix-editor/helix"
+homepage = "https://helix-editor.com"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-helix-core = { path = "../helix-core" }
+helix-core = { version = "0.4", path = "../helix-core" }
 
 anyhow = "1.0"
 futures-executor = "0.3"
 futures-util = { version = "0.3", features = ["std", "async-await"], default-features = false }
-jsonrpc-core = { version = "17.1", default-features = false } # don't pull in all of futures
+jsonrpc-core = { version = "18.0", default-features = false } # don't pull in all of futures
 log = "0.4"
 lsp-types = { version = "0.89", features = ["proposed"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 thiserror = "1.0"
-tokio = { version = "1.6", features = ["full"] }
-tokio-stream = "0.1.6"
+tokio = { version = "1.9", features = ["rt", "rt-multi-thread", "io-util", "io-std", "time", "process", "macros", "fs", "parking_lot"] }
+tokio-stream = "0.1.7"

helix-lsp/src/client.rs

@@ -3,7 +3,7 @@ use crate::{
     Call, Error, OffsetEncoding, Result,
 };
 
-use helix_core::{find_root, ChangeSet, Rope};
+use helix_core::{chars::char_is_line_ending, find_root, ChangeSet, Rope};
 use jsonrpc_core as jsonrpc;
 use lsp_types as lsp;
 use serde_json::Value;
@@ -18,15 +18,22 @@ use tokio::{
 
 #[derive(Debug)]
 pub struct Client {
+    id: usize,
     _process: Child,
     server_tx: UnboundedSender<Payload>,
     request_counter: AtomicU64,
     capabilities: Option<lsp::ServerCapabilities>,
     offset_encoding: OffsetEncoding,
+    config: Option<Value>,
 }
 
 impl Client {
-    pub fn start(cmd: &str, args: &[String]) -> Result<(Self, UnboundedReceiver<Call>)> {
+    pub fn start(
+        cmd: &str,
+        args: &[String],
+        config: Option<Value>,
+        id: usize,
+    ) -> Result<(Self, UnboundedReceiver<(usize, Call)>)> {
         let process = Command::new(cmd)
             .args(args)
             .stdin(Stdio::piped())
@@ -43,14 +50,16 @@ impl Client {
         let reader = BufReader::new(process.stdout.take().expect("Failed to open stdout"));
         let stderr = BufReader::new(process.stderr.take().expect("Failed to open stderr"));
 
-        let (server_rx, server_tx) = Transport::start(reader, writer, stderr);
+        let (server_rx, server_tx) = Transport::start(reader, writer, stderr, id);
 
         let client = Self {
+            id,
             _process: process,
             server_tx,
             request_counter: AtomicU64::new(0),
             capabilities: None,
             offset_encoding: OffsetEncoding::Utf8,
+            config,
         };
 
         // TODO: async client.initialize()
@@ -59,6 +68,10 @@ impl Client {
         Ok((client, server_rx))
     }
 
+    pub fn id(&self) -> usize {
+        self.id
+    }
+
     fn next_request_id(&self) -> jsonrpc::Id {
         let id = self.request_counter.fetch_add(1, Ordering::Relaxed);
         jsonrpc::Id::Num(id)
@@ -165,13 +178,16 @@ impl Client {
     }
 
     /// Reply to a language server RPC call.
-    pub async fn reply(
+    pub fn reply(
         &self,
         id: jsonrpc::Id,
         result: core::result::Result<Value, jsonrpc::Error>,
-    ) -> Result<()> {
+    ) -> impl Future<Output = Result<()>> {
         use jsonrpc::{Failure, Output, Success, Version};
 
+        let server_tx = self.server_tx.clone();
+
+        async move {
             let output = match result {
                 Ok(result) => Output::Success(Success {
                     jsonrpc: Some(Version::V2),
@@ -185,12 +201,13 @@ impl Client {
                 }),
             };
 
-        self.server_tx
+            server_tx
                 .send(Payload::Response(output))
                 .map_err(|e| Error::Other(e.into()))?;
 
             Ok(())
         }
+    }
 
     // -------------------------------------------------------------------------------------------
     // General messages
@@ -200,13 +217,17 @@ impl Client {
         // TODO: delay any requests that are triggered prior to initialize
         let root = find_root(None).and_then(|root| lsp::Url::from_file_path(root).ok());
 
+        if self.config.is_some() {
+            log::info!("Using custom LSP config: {}", self.config.as_ref().unwrap());
+        }
+
         #[allow(deprecated)]
         let params = lsp::InitializeParams {
             process_id: Some(std::process::id()),
             // root_path is obsolete, use root_uri
             root_path: None,
             root_uri: root,
-            initialization_options: None,
+            initialization_options: self.config.clone(),
             capabilities: lsp::ClientCapabilities {
                 text_document: Some(lsp::TextDocumentClientCapabilities {
                     completion: Some(lsp::CompletionClientCapabilities {
@@ -226,11 +247,30 @@ impl Client {
                         content_format: Some(vec![lsp::MarkupKind::Markdown]),
                         ..Default::default()
                     }),
+                    code_action: Some(lsp::CodeActionClientCapabilities {
+                        code_action_literal_support: Some(lsp::CodeActionLiteralSupport {
+                            code_action_kind: lsp::CodeActionKindLiteralSupport {
+                                value_set: [
+                                    lsp::CodeActionKind::EMPTY,
+                                    lsp::CodeActionKind::QUICKFIX,
+                                    lsp::CodeActionKind::REFACTOR,
+                                    lsp::CodeActionKind::REFACTOR_EXTRACT,
+                                    lsp::CodeActionKind::REFACTOR_INLINE,
+                                    lsp::CodeActionKind::REFACTOR_REWRITE,
+                                    lsp::CodeActionKind::SOURCE,
+                                    lsp::CodeActionKind::SOURCE_ORGANIZE_IMPORTS,
+                                ]
+                                .iter()
+                                .map(|kind| kind.as_str().to_string())
+                                .collect(),
+                            },
+                        }),
+                        ..Default::default()
+                    }),
                     ..Default::default()
                 }),
                 window: Some(lsp::WindowClientCapabilities {
-                    // TODO: temporarily disabled until we implement handling for window/workDoneProgress/create
-                    // work_done_progress: Some(true),
+                    work_done_progress: Some(true),
                     ..Default::default()
                 }),
                 ..Default::default()
@@ -259,6 +299,21 @@ impl Client {
         self.notify::<lsp::notification::Exit>(())
     }
 
+    /// Tries to shut down the language server but returns
+    /// early if server responds with an error.
+    pub async fn shutdown_and_exit(&self) -> Result<()> {
+        self.shutdown().await?;
+        self.exit().await
+    }
+
+    /// Forcefully shuts down the language server ignoring any errors.
+    pub async fn force_shutdown(&self) -> Result<()> {
+        if let Err(e) = self.shutdown().await {
+            log::warn!("language server failed to terminate gracefully - {}", e);
+        }
+        self.exit().await
+    }
+
     // -------------------------------------------------------------------------------------------
     // Text document
     // -------------------------------------------------------------------------------------------
@@ -309,8 +364,9 @@ impl Client {
                 mut character,
             } = pos;
 
-            for ch in text.chars() {
-                if ch == '\n' {
+            let mut chars = text.chars().peekable();
+            while let Some(ch) = chars.next() {
+                if char_is_line_ending(ch) && !(ch == '\r' && chars.peek() == Some(&'\n')) {
                     line += 1;
                     character = 0;
                 } else {
@@ -465,6 +521,7 @@ impl Client {
         &self,
         text_document: lsp::TextDocumentIdentifier,
         position: lsp::Position,
+        work_done_token: Option<lsp::ProgressToken>,
     ) -> impl Future<Output = Result<Value>> {
         // ) -> Result<Vec<lsp::CompletionItem>> {
         let params = lsp::CompletionParams {
@@ -473,9 +530,7 @@ impl Client {
                 position,
             },
             // TODO: support these tokens by async receiving and updating the choice list
-            work_done_progress_params: lsp::WorkDoneProgressParams {
-                work_done_token: None,
-            },
+            work_done_progress_params: lsp::WorkDoneProgressParams { work_done_token },
             partial_result_params: lsp::PartialResultParams {
                 partial_result_token: None,
             },
@@ -490,15 +545,14 @@ impl Client {
         &self,
         text_document: lsp::TextDocumentIdentifier,
         position: lsp::Position,
+        work_done_token: Option<lsp::ProgressToken>,
     ) -> impl Future<Output = Result<Value>> {
         let params = lsp::SignatureHelpParams {
             text_document_position_params: lsp::TextDocumentPositionParams {
                 text_document,
                 position,
             },
-            work_done_progress_params: lsp::WorkDoneProgressParams {
-                work_done_token: None,
-            },
+            work_done_progress_params: lsp::WorkDoneProgressParams { work_done_token },
             context: None,
             // lsp::SignatureHelpContext
         };
@@ -510,15 +564,14 @@ impl Client {
         &self,
         text_document: lsp::TextDocumentIdentifier,
         position: lsp::Position,
+        work_done_token: Option<lsp::ProgressToken>,
     ) -> impl Future<Output = Result<Value>> {
         let params = lsp::HoverParams {
             text_document_position_params: lsp::TextDocumentPositionParams {
                 text_document,
                 position,
             },
-            work_done_progress_params: lsp::WorkDoneProgressParams {
-                work_done_token: None,
-            },
+            work_done_progress_params: lsp::WorkDoneProgressParams { work_done_token },
             // lsp::SignatureHelpContext
         };
 
@@ -531,6 +584,7 @@ impl Client {
         &self,
         text_document: lsp::TextDocumentIdentifier,
         options: lsp::FormattingOptions,
+        work_done_token: Option<lsp::ProgressToken>,
     ) -> anyhow::Result<Vec<lsp::TextEdit>> {
         let capabilities = self.capabilities.as_ref().unwrap();
 
@@ -545,9 +599,7 @@ impl Client {
         let params = lsp::DocumentFormattingParams {
             text_document,
             options,
-            work_done_progress_params: lsp::WorkDoneProgressParams {
-                work_done_token: None,
-            },
+            work_done_progress_params: lsp::WorkDoneProgressParams { work_done_token },
         };
 
         let response = self.request::<lsp::request::Formatting>(params).await?;
@@ -560,6 +612,7 @@ impl Client {
         text_document: lsp::TextDocumentIdentifier,
         range: lsp::Range,
         options: lsp::FormattingOptions,
+        work_done_token: Option<lsp::ProgressToken>,
     ) -> anyhow::Result<Vec<lsp::TextEdit>> {
         let capabilities = self.capabilities.as_ref().unwrap();
 
@@ -575,9 +628,7 @@ impl Client {
             text_document,
             range,
             options,
-            work_done_progress_params: lsp::WorkDoneProgressParams {
-                work_done_token: None,
-            },
+            work_done_progress_params: lsp::WorkDoneProgressParams { work_done_token },
         };
 
         let response = self
@@ -596,15 +647,14 @@ impl Client {
         &self,
         text_document: lsp::TextDocumentIdentifier,
         position: lsp::Position,
+        work_done_token: Option<lsp::ProgressToken>,
     ) -> impl Future<Output = Result<Value>> {
         let params = lsp::GotoDefinitionParams {
             text_document_position_params: lsp::TextDocumentPositionParams {
                 text_document,
                 position,
             },
-            work_done_progress_params: lsp::WorkDoneProgressParams {
-                work_done_token: None,
-            },
+            work_done_progress_params: lsp::WorkDoneProgressParams { work_done_token },
             partial_result_params: lsp::PartialResultParams {
                 partial_result_token: None,
             },
@@ -617,30 +667,42 @@ impl Client {
         &self,
         text_document: lsp::TextDocumentIdentifier,
         position: lsp::Position,
+        work_done_token: Option<lsp::ProgressToken>,
     ) -> impl Future<Output = Result<Value>> {
-        self.goto_request::<lsp::request::GotoDefinition>(text_document, position)
+        self.goto_request::<lsp::request::GotoDefinition>(text_document, position, work_done_token)
     }
 
     pub fn goto_type_definition(
         &self,
         text_document: lsp::TextDocumentIdentifier,
         position: lsp::Position,
+        work_done_token: Option<lsp::ProgressToken>,
     ) -> impl Future<Output = Result<Value>> {
-        self.goto_request::<lsp::request::GotoTypeDefinition>(text_document, position)
+        self.goto_request::<lsp::request::GotoTypeDefinition>(
+            text_document,
+            position,
+            work_done_token,
+        )
     }
 
     pub fn goto_implementation(
         &self,
         text_document: lsp::TextDocumentIdentifier,
         position: lsp::Position,
+        work_done_token: Option<lsp::ProgressToken>,
     ) -> impl Future<Output = Result<Value>> {
-        self.goto_request::<lsp::request::GotoImplementation>(text_document, position)
+        self.goto_request::<lsp::request::GotoImplementation>(
+            text_document,
+            position,
+            work_done_token,
+        )
     }
 
     pub fn goto_reference(
         &self,
         text_document: lsp::TextDocumentIdentifier,
         position: lsp::Position,
+        work_done_token: Option<lsp::ProgressToken>,
     ) -> impl Future<Output = Result<Value>> {
         let params = lsp::ReferenceParams {
             text_document_position: lsp::TextDocumentPositionParams {
@@ -650,9 +712,7 @@ impl Client {
             context: lsp::ReferenceContext {
                 include_declaration: true,
             },
-            work_done_progress_params: lsp::WorkDoneProgressParams {
-                work_done_token: None,
-            },
+            work_done_progress_params: lsp::WorkDoneProgressParams { work_done_token },
             partial_result_params: lsp::PartialResultParams {
                 partial_result_token: None,
             },
@@ -673,4 +733,31 @@ impl Client {
 
         self.call::<lsp::request::DocumentSymbolRequest>(params)
     }
+
+    // empty string to get all symbols
+    pub fn workspace_symbols(&self, query: String) -> impl Future<Output = Result<Value>> {
+        let params = lsp::WorkspaceSymbolParams {
+            query,
+            work_done_progress_params: lsp::WorkDoneProgressParams::default(),
+            partial_result_params: lsp::PartialResultParams::default(),
+        };
+
+        self.call::<lsp::request::WorkspaceSymbol>(params)
+    }
+
+    pub fn code_actions(
+        &self,
+        text_document: lsp::TextDocumentIdentifier,
+        range: lsp::Range,
+    ) -> impl Future<Output = Result<Value>> {
+        let params = lsp::CodeActionParams {
+            text_document,
+            range,
+            context: lsp::CodeActionContext::default(),
+            work_done_progress_params: lsp::WorkDoneProgressParams::default(),
+            partial_result_params: lsp::PartialResultParams::default(),
+        };
+
+        self.call::<lsp::request::CodeActionRequest>(params)
+    }
 }

helix-lsp/src/lib.rs

@@ -13,7 +13,10 @@ use helix_core::syntax::LanguageConfiguration;
 
 use std::{
     collections::{hash_map::Entry, HashMap},
-    sync::Arc,
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc,
+    },
 };
 
 use serde::{Deserialize, Serialize};
@@ -179,6 +182,46 @@ pub mod util {
             }),
         )
     }
+
+    /// The result of asking the language server to format the document. This can be turned into a
+    /// `Transaction`, but the advantage of not doing that straight away is that this one is
+    /// `Send` and `Sync`.
+    #[derive(Clone, Debug)]
+    pub struct LspFormatting {
+        pub doc: Rope,
+        pub edits: Vec<lsp::TextEdit>,
+        pub offset_encoding: OffsetEncoding,
+    }
+
+    impl From<LspFormatting> for Transaction {
+        fn from(fmt: LspFormatting) -> Transaction {
+            generate_transaction_from_edits(&fmt.doc, fmt.edits, fmt.offset_encoding)
+        }
+    }
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub enum MethodCall {
+    WorkDoneProgressCreate(lsp::WorkDoneProgressCreateParams),
+}
+
+impl MethodCall {
+    pub fn parse(method: &str, params: jsonrpc::Params) -> Option<MethodCall> {
+        use lsp::request::Request;
+        let request = match method {
+            lsp::request::WorkDoneProgressCreate::METHOD => {
+                let params: lsp::WorkDoneProgressCreateParams = params
+                    .parse()
+                    .expect("Failed to parse WorkDoneCreate params");
+                Self::WorkDoneProgressCreate(params)
+            }
+            _ => {
+                log::warn!("unhandled lsp request: {}", method);
+                return None;
+            }
+        };
+        Some(request)
+    }
 }
 
 #[derive(Debug, PartialEq, Clone)]
@@ -230,9 +273,10 @@ impl Notification {
 
 #[derive(Debug)]
 pub struct Registry {
-    inner: HashMap<LanguageId, Arc<Client>>,
+    inner: HashMap<LanguageId, (usize, Arc<Client>)>,
 
-    pub incoming: SelectAll<UnboundedReceiverStream<Call>>,
+    counter: AtomicUsize,
+    pub incoming: SelectAll<UnboundedReceiverStream<(usize, Call)>>,
 }
 
 impl Default for Registry {
@@ -245,10 +289,18 @@ impl Registry {
     pub fn new() -> Self {
         Self {
             inner: HashMap::new(),
+            counter: AtomicUsize::new(0),
             incoming: SelectAll::new(),
         }
     }
 
+    pub fn get_by_id(&mut self, id: usize) -> Option<&Client> {
+        self.inner
+            .values()
+            .find(|(client_id, _)| client_id == &id)
+            .map(|(_, client)| client.as_ref())
+    }
+
     pub fn get(&mut self, language_config: &LanguageConfiguration) -> Result<Arc<Client>> {
         if let Some(config) = &language_config.language_server {
             // avoid borrow issues
@@ -256,16 +308,22 @@ impl Registry {
             let s_incoming = &mut self.incoming;
 
             match inner.entry(language_config.scope.clone()) {
-                Entry::Occupied(language_server) => Ok(language_server.get().clone()),
+                Entry::Occupied(entry) => Ok(entry.get().1.clone()),
                 Entry::Vacant(entry) => {
                     // initialize a new client
-                    let (mut client, incoming) = Client::start(&config.command, &config.args)?;
+                    let id = self.counter.fetch_add(1, Ordering::Relaxed);
+                    let (mut client, incoming) = Client::start(
+                        &config.command,
+                        &config.args,
+                        serde_json::from_str(language_config.config.as_deref().unwrap_or("")).ok(),
+                        id,
+                    )?;
                     // TODO: run this async without blocking
                     futures_executor::block_on(client.initialize())?;
                     s_incoming.push(UnboundedReceiverStream::new(incoming));
                     let client = Arc::new(client);
 
-                    entry.insert(client.clone());
+                    entry.insert((id, client.clone()));
                     Ok(client)
                 }
             }
@@ -273,6 +331,88 @@ impl Registry {
             Err(Error::LspNotDefined)
         }
     }
+
+    pub fn iter_clients(&self) -> impl Iterator<Item = &Arc<Client>> {
+        self.inner.values().map(|(_, client)| client)
+    }
+}
+
+#[derive(Debug)]
+pub enum ProgressStatus {
+    Created,
+    Started(lsp::WorkDoneProgress),
+}
+
+impl ProgressStatus {
+    pub fn progress(&self) -> Option<&lsp::WorkDoneProgress> {
+        match &self {
+            ProgressStatus::Created => None,
+            ProgressStatus::Started(progress) => Some(progress),
+        }
+    }
+}
+
+#[derive(Default, Debug)]
+/// Acts as a container for progress reported by language servers. Each server
+/// has a unique id assigned at creation through [`Registry`]. This id is then used
+/// to store the progress in this map.
+pub struct LspProgressMap(HashMap<usize, HashMap<lsp::ProgressToken, ProgressStatus>>);
+
+impl LspProgressMap {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Returns a map of all tokens coresponding to the lanaguage server with `id`.
+    pub fn progress_map(&self, id: usize) -> Option<&HashMap<lsp::ProgressToken, ProgressStatus>> {
+        self.0.get(&id)
+    }
+
+    pub fn is_progressing(&self, id: usize) -> bool {
+        self.0.get(&id).map(|it| !it.is_empty()).unwrap_or_default()
+    }
+
+    /// Returns last progress status for a given server with `id` and `token`.
+    pub fn progress(&self, id: usize, token: &lsp::ProgressToken) -> Option<&ProgressStatus> {
+        self.0.get(&id).and_then(|values| values.get(token))
+    }
+
+    /// Checks if progress `token` for server with `id` is created.
+    pub fn is_created(&mut self, id: usize, token: &lsp::ProgressToken) -> bool {
+        self.0
+            .get(&id)
+            .map(|values| values.get(token).is_some())
+            .unwrap_or_default()
+    }
+
+    pub fn create(&mut self, id: usize, token: lsp::ProgressToken) {
+        self.0
+            .entry(id)
+            .or_default()
+            .insert(token, ProgressStatus::Created);
+    }
+
+    /// Ends the progress by removing the `token` from server with `id`, if removed returns the value.
+    pub fn end_progress(
+        &mut self,
+        id: usize,
+        token: &lsp::ProgressToken,
+    ) -> Option<ProgressStatus> {
+        self.0.get_mut(&id).and_then(|vals| vals.remove(token))
+    }
+
+    /// Updates the progess of `token` for server with `id` to `status`, returns the value replaced or `None`.
+    pub fn update(
+        &mut self,
+        id: usize,
+        token: lsp::ProgressToken,
+        status: lsp::WorkDoneProgress,
+    ) -> Option<ProgressStatus> {
+        self.0
+            .entry(id)
+            .or_default()
+            .insert(token, ProgressStatus::Started(status))
+    }
 }
 
 // REGISTRY = HashMap<LanguageId, Lazy/OnceCell<Arc<RwLock<Client>>>

helix-lsp/src/transport.rs

@@ -1,13 +1,18 @@
 use crate::Result;
+use anyhow::Context;
 use jsonrpc_core as jsonrpc;
 use log::{error, info};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use std::collections::HashMap;
+use std::sync::Arc;
 use tokio::{
     io::{AsyncBufRead, AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader, BufWriter},
     process::{ChildStderr, ChildStdin, ChildStdout},
-    sync::mpsc::{unbounded_channel, Sender, UnboundedReceiver, UnboundedSender},
+    sync::{
+        mpsc::{unbounded_channel, Sender, UnboundedReceiver, UnboundedSender},
+        Mutex,
+    },
 };
 
 #[derive(Debug)]
@@ -33,14 +38,8 @@ enum ServerMessage {
 
 #[derive(Debug)]
 pub struct Transport {
-    client_tx: UnboundedSender<jsonrpc::Call>,
-    client_rx: UnboundedReceiver<Payload>,
-
-    pending_requests: HashMap<jsonrpc::Id, Sender<Result<Value>>>,
-
-    server_stdin: BufWriter<ChildStdin>,
-    server_stdout: BufReader<ChildStdout>,
-    server_stderr: BufReader<ChildStderr>,
+    id: usize,
+    pending_requests: Mutex<HashMap<jsonrpc::Id, Sender<Result<Value>>>>,
 }
 
 impl Transport {
@@ -48,20 +47,24 @@ impl Transport {
         server_stdout: BufReader<ChildStdout>,
         server_stdin: BufWriter<ChildStdin>,
         server_stderr: BufReader<ChildStderr>,
-    ) -> (UnboundedReceiver<jsonrpc::Call>, UnboundedSender<Payload>) {
+        id: usize,
+    ) -> (
+        UnboundedReceiver<(usize, jsonrpc::Call)>,
+        UnboundedSender<Payload>,
+    ) {
         let (client_tx, rx) = unbounded_channel();
         let (tx, client_rx) = unbounded_channel();
 
         let transport = Self {
-            server_stdout,
-            server_stdin,
-            server_stderr,
-            client_tx,
-            client_rx,
-            pending_requests: HashMap::default(),
+            id,
+            pending_requests: Mutex::new(HashMap::default()),
         };
 
-        tokio::spawn(transport.duplex());
+        let transport = Arc::new(transport);
+
+        tokio::spawn(Self::recv(transport.clone(), server_stdout, client_tx));
+        tokio::spawn(Self::err(transport.clone(), server_stderr));
+        tokio::spawn(Self::send(transport, server_stdin, client_rx));
 
         (rx, tx)
     }
@@ -84,7 +87,7 @@ impl Transport {
 
             match (parts.next(), parts.next(), parts.next()) {
                 (Some("Content-Length"), Some(value), None) => {
-                    content_length = Some(value.parse().unwrap());
+                    content_length = Some(value.parse().context("invalid content length")?);
                 }
                 (Some(_), Some(_), None) => {}
                 _ => {
@@ -97,17 +100,17 @@ impl Transport {
             }
         }
 
-        let content_length = content_length.unwrap();
+        let content_length = content_length.context("missing content length")?;
 
         //TODO: reuse vector
         let mut content = vec![0; content_length];
         reader.read_exact(&mut content).await?;
-        let msg = String::from_utf8(content).unwrap();
+        let msg = std::str::from_utf8(&content).context("invalid utf8 from server")?;
 
         info!("<- {}", msg);
 
         // try parsing as output (server response) or call (server request)
-        let output: serde_json::Result<ServerMessage> = serde_json::from_str(&msg);
+        let output: serde_json::Result<ServerMessage> = serde_json::from_str(msg);
 
         Ok(output?)
     }
@@ -123,47 +126,64 @@ impl Transport {
         Ok(())
     }
 
-    async fn send_payload_to_server(&mut self, payload: Payload) -> Result<()> {
+    async fn send_payload_to_server(
+        &self,
+        server_stdin: &mut BufWriter<ChildStdin>,
+        payload: Payload,
+    ) -> Result<()> {
         //TODO: reuse string
         let json = match payload {
             Payload::Request { chan, value } => {
-                self.pending_requests.insert(value.id.clone(), chan);
+                self.pending_requests
+                    .lock()
+                    .await
+                    .insert(value.id.clone(), chan);
                 serde_json::to_string(&value)?
             }
             Payload::Notification(value) => serde_json::to_string(&value)?,
             Payload::Response(error) => serde_json::to_string(&error)?,
         };
-        self.send_string_to_server(json).await
+        self.send_string_to_server(server_stdin, json).await
     }
 
-    async fn send_string_to_server(&mut self, request: String) -> Result<()> {
+    async fn send_string_to_server(
+        &self,
+        server_stdin: &mut BufWriter<ChildStdin>,
+        request: String,
+    ) -> Result<()> {
         info!("-> {}", request);
 
         // send the headers
-        self.server_stdin
+        server_stdin
             .write_all(format!("Content-Length: {}\r\n\r\n", request.len()).as_bytes())
             .await?;
 
         // send the body
-        self.server_stdin.write_all(request.as_bytes()).await?;
+        server_stdin.write_all(request.as_bytes()).await?;
 
-        self.server_stdin.flush().await?;
+        server_stdin.flush().await?;
 
         Ok(())
     }
 
-    async fn process_server_message(&mut self, msg: ServerMessage) -> Result<()> {
+    async fn process_server_message(
+        &self,
+        client_tx: &UnboundedSender<(usize, jsonrpc::Call)>,
+        msg: ServerMessage,
+    ) -> Result<()> {
         match msg {
             ServerMessage::Output(output) => self.process_request_response(output).await?,
             ServerMessage::Call(call) => {
-                self.client_tx.send(call).unwrap();
+                client_tx
+                    .send((self.id, call))
+                    .context("failed to send a message to server")?;
                 // let notification = Notification::parse(&method, params);
             }
         };
         Ok(())
     }
 
-    async fn process_request_response(&mut self, output: jsonrpc::Output) -> Result<()> {
+    async fn process_request_response(&self, output: jsonrpc::Output) -> Result<()> {
         let (id, result) = match output {
             jsonrpc::Output::Success(jsonrpc::Success { id, result, .. }) => {
                 info!("<- {}", result);
@@ -177,6 +197,8 @@ impl Transport {
 
         let tx = self
             .pending_requests
+            .lock()
+            .await
             .remove(&id)
             .expect("pending_request with id not found!");
 
@@ -191,34 +213,51 @@ impl Transport {
         Ok(())
     }
 
-    async fn duplex(mut self) {
+    async fn recv(
+        transport: Arc<Self>,
+        mut server_stdout: BufReader<ChildStdout>,
+        client_tx: UnboundedSender<(usize, jsonrpc::Call)>,
+    ) {
         let mut recv_buffer = String::new();
-        let mut err_buffer = String::new();
         loop {
-            tokio::select! {
-                // client -> server
-                msg = self.client_rx.recv() => {
-                    match msg {
-                        Some(msg) => {
-                            self.send_payload_to_server(msg).await.unwrap()
-                        },
-                        None => break
-                    }
-                }
-                // server -> client
-                msg = Self::recv_server_message(&mut self.server_stdout, &mut recv_buffer) => {
-                    match msg {
+            match Self::recv_server_message(&mut server_stdout, &mut recv_buffer).await {
                 Ok(msg) => {
-                            self.process_server_message(msg).await.unwrap();
+                    transport
+                        .process_server_message(&client_tx, msg)
+                        .await
+                        .unwrap();
                 }
-                        Err(_) => {
-                            error!("err: <- {:?}", msg);
+                Err(err) => {
+                    error!("err: <- {:?}", err);
                     break;
-                        },
-                    }
-                }
-                _msg = Self::recv_server_error(&mut self.server_stderr, &mut err_buffer) => {}
                 }
             }
         }
     }
+
+    async fn err(_transport: Arc<Self>, mut server_stderr: BufReader<ChildStderr>) {
+        let mut recv_buffer = String::new();
+        loop {
+            match Self::recv_server_error(&mut server_stderr, &mut recv_buffer).await {
+                Ok(_) => {}
+                Err(err) => {
+                    error!("err: <- {:?}", err);
+                    break;
+                }
+            }
+        }
+    }
+
+    async fn send(
+        transport: Arc<Self>,
+        mut server_stdin: BufWriter<ChildStdin>,
+        mut client_rx: UnboundedReceiver<Payload>,
+    ) {
+        while let Some(msg) = client_rx.recv().await {
+            transport
+                .send_payload_to_server(&mut server_stdin, msg)
+                .await
+                .unwrap()
+        }
+    }
+}

helix-syntax/Cargo.toml

@@ -1,16 +1,21 @@
 [package]
 name = "helix-syntax"
-version = "0.2.0"
+version = "0.4.1"
 authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
 edition = "2018"
 license = "MPL-2.0"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+description = "Tree-sitter grammars support"
+categories = ["editor"]
+repository = "https://github.com/helix-editor/helix"
+homepage = "https://helix-editor.com"
+include = ["src/**/*", "languages/**/*", "build.rs", "!**/docs/**/*", "!**/test/**/*", "!**/examples/**/*", "!**/build/**/*"]
 
 [dependencies]
 tree-sitter = "0.19"
-serde = { version = "1.0", features = ["derive"] }
+libloading = "0.7"
+anyhow = "1"
 
 [build-dependencies]
-cc = { version = "1",  features = ["parallel"] }
+cc = { version = "1" }
 threadpool = { version = "1.0" }
+anyhow = "1"

helix-syntax/build.rs

@@ -1,79 +1,147 @@
+use anyhow::{anyhow, Context, Result};
 use std::fs;
-use std::path::PathBuf;
+use std::time::SystemTime;
+use std::{
+    path::{Path, PathBuf},
+    process::Command,
+};
 
 use std::sync::mpsc::channel;
 
-fn collect_tree_sitter_dirs(ignore: &[String]) -> Vec<String> {
+fn collect_tree_sitter_dirs(ignore: &[String]) -> Result<Vec<String>> {
     let mut dirs = Vec::new();
-    for entry in fs::read_dir("languages").unwrap().flatten() {
-        let path = entry.path();
-        let dir = path.file_name().unwrap().to_str().unwrap().to_string();
-        if !ignore.contains(&dir) {
-            dirs.push(dir);
-        }
-    }
-    dirs
-}
+    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("languages");
 
-fn collect_src_files(dir: &str) -> (Vec<String>, Vec<String>) {
-    eprintln!("Collect files for {}", dir);
-
-    let mut c_files = Vec::new();
-    let mut cpp_files = Vec::new();
-    let path = PathBuf::from("languages").join(&dir).join("src");
-    for entry in fs::read_dir(path).unwrap().flatten() {
+    for entry in fs::read_dir(path)? {
+        let entry = entry?;
         let path = entry.path();
-        if path
-            .file_stem()
-            .unwrap()
-            .to_str()
-            .unwrap()
-            .starts_with("binding")
-        {
+
+        if !entry.file_type()?.is_dir() {
             continue;
         }
-        if let Some(ext) = path.extension() {
-            if ext == "c" {
-                c_files.push(path.to_str().unwrap().to_string());
-            } else if ext == "cc" || ext == "cpp" || ext == "cxx" {
-                cpp_files.push(path.to_str().unwrap().to_string());
+
+        let dir = path.file_name().unwrap().to_str().unwrap().to_string();
+
+        // filter ignores
+        if ignore.contains(&dir) {
+            continue;
         }
-        }
-    }
-    (c_files, cpp_files)
+        dirs.push(dir)
     }
 
-fn build_c(files: Vec<String>, language: &str) {
-    let mut build = cc::Build::new();
-    for file in files {
-        build
-            .file(&file)
-            .include(PathBuf::from(file).parent().unwrap())
-            .pic(true)
-            .warnings(false);
-    }
-    build.compile(&format!("tree-sitter-{}-c", language));
+    Ok(dirs)
 }
 
-fn build_cpp(files: Vec<String>, language: &str) {
-    let mut build = cc::Build::new();
+#[cfg(unix)]
+const DYLIB_EXTENSION: &str = "so";
 
-    let flag = if build.get_compiler().is_like_msvc() {
-        "/std:c++17"
+#[cfg(windows)]
+const DYLIB_EXTENSION: &str = "dll";
+
+fn build_library(src_path: &Path, language: &str) -> Result<()> {
+    let header_path = src_path;
+    // let grammar_path = src_path.join("grammar.json");
+    let parser_path = src_path.join("parser.c");
+    let mut scanner_path = src_path.join("scanner.c");
+
+    let scanner_path = if scanner_path.exists() {
+        Some(scanner_path)
     } else {
-        "-std=c++14"
-    };
-
-    for file in files {
-        build
-            .file(&file)
-            .include(PathBuf::from(file).parent().unwrap())
-            .pic(true)
-            .warnings(false)
-            .cpp(true)
-            .flag_if_supported(flag);
+        scanner_path.set_extension("cc");
+        if scanner_path.exists() {
+            Some(scanner_path)
+        } else {
+            None
         }
-    build.compile(&format!("tree-sitter-{}-cpp", language));
+    };
+    let parser_lib_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../runtime/grammars");
+    let mut library_path = parser_lib_path.join(language);
+    library_path.set_extension(DYLIB_EXTENSION);
+
+    let recompile = needs_recompile(&library_path, &parser_path, &scanner_path)
+        .with_context(|| "Failed to compare source and binary timestamps")?;
+
+    if !recompile {
+        return Ok(());
+    }
+    let mut config = cc::Build::new();
+    config.cpp(true).opt_level(2).cargo_metadata(false);
+    let compiler = config.get_compiler();
+    let mut command = Command::new(compiler.path());
+    command.current_dir(src_path);
+    for (key, value) in compiler.env() {
+        command.env(key, value);
+    }
+
+    if cfg!(windows) {
+        command
+            .args(&["/nologo", "/LD", "/I"])
+            .arg(header_path)
+            .arg("/Od");
+        if let Some(scanner_path) = scanner_path.as_ref() {
+            command.arg(scanner_path);
+        }
+
+        command
+            .arg(parser_path)
+            .arg("/link")
+            .arg(format!("/out:{}", library_path.to_str().unwrap()));
+    } else {
+        command
+            .arg("-shared")
+            .arg("-fPIC")
+            .arg("-fno-exceptions")
+            .arg("-g")
+            .arg("-I")
+            .arg(header_path)
+            .arg("-o")
+            .arg(&library_path)
+            .arg("-O2");
+        if let Some(scanner_path) = scanner_path.as_ref() {
+            if scanner_path.extension() == Some("c".as_ref()) {
+                command.arg("-xc").arg("-std=c99").arg(scanner_path);
+            } else {
+                command.arg(scanner_path);
+            }
+        }
+        command.arg("-xc").arg(parser_path);
+    }
+
+    let output = command
+        .output()
+        .with_context(|| "Failed to execute C compiler")?;
+    if !output.status.success() {
+        return Err(anyhow!(
+            "Parser compilation failed.\nStdout: {}\nStderr: {}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr)
+        ));
+    }
+
+    Ok(())
+}
+fn needs_recompile(
+    lib_path: &Path,
+    parser_c_path: &Path,
+    scanner_path: &Option<PathBuf>,
+) -> Result<bool> {
+    if !lib_path.exists() {
+        return Ok(true);
+    }
+    let lib_mtime = mtime(lib_path)?;
+    if mtime(parser_c_path)? > lib_mtime {
+        return Ok(true);
+    }
+    if let Some(scanner_path) = scanner_path {
+        if mtime(scanner_path)? > lib_mtime {
+            return Ok(true);
+        }
+    }
+    Ok(false)
+}
+
+fn mtime(path: &Path) -> Result<SystemTime> {
+    Ok(fs::metadata(path)?.modified()?)
 }
 
 fn build_dir(dir: &str, language: &str) {
@@ -92,22 +160,21 @@ fn build_dir(dir: &str, language: &str) {
         eprintln!("You can fix in using 'git submodule init && git submodule update --recursive'.");
         std::process::exit(1);
     }
-    let (c, cpp) = collect_src_files(dir);
-    if !c.is_empty() {
-        build_c(c, language);
-    }
-    if !cpp.is_empty() {
-        build_cpp(cpp, language);
-    }
+
+    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("languages")
+        .join(dir)
+        .join("src");
+
+    build_library(&path, language).unwrap();
 }
 
 fn main() {
     let ignore = vec![
         "tree-sitter-typescript".to_string(),
         "tree-sitter-haskell".to_string(), // aarch64 failures: https://github.com/tree-sitter/tree-sitter-haskell/issues/34
-        ".DS_Store".to_string(),
     ];
-    let dirs = collect_tree_sitter_dirs(&ignore);
+    let dirs = collect_tree_sitter_dirs(&ignore).unwrap();
 
     let mut n_jobs = 0;
     let pool = threadpool::Builder::new().build(); // by going through the builder, it'll use num_cpus
@@ -118,7 +185,7 @@ fn main() {
         n_jobs += 1;
 
         pool.execute(move || {
-            let language = &dir[12..]; // skip tree-sitter- prefix
+            let language = &dir.strip_prefix("tree-sitter-").unwrap();
             build_dir(&dir, language);
 
             // report progress

helix-syntax/src/lib.rs

@@ -1,93 +1,31 @@
-use serde::{Deserialize, Serialize};
+use anyhow::{Context, Result};
+use libloading::{Library, Symbol};
 use tree_sitter::Language;
 
-#[macro_export]
-macro_rules! mk_extern {
-    ( $( $name:ident ),* ) => {
-        $(
-            extern "C" { pub fn $name() -> Language; }
-        )*
-    };
+fn replace_dashes_with_underscores(name: &str) -> String {
+    name.replace('-', "_")
 }
+#[cfg(unix)]
+const DYLIB_EXTENSION: &str = "so";
 
-#[macro_export]
-macro_rules! mk_enum {
-    ( $( $camel:ident ),* ) => {
-        #[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
-        #[serde(rename_all = "lowercase")]
-        pub enum Lang {
-            $(
-                $camel,
-            )*
-        }
-    };
-}
+#[cfg(windows)]
+const DYLIB_EXTENSION: &str = "dll";
 
-#[macro_export]
-macro_rules! mk_get_language {
-    ( $( ($camel:ident, $name:ident) ),* ) => {
-        #[must_use]
-        pub fn get_language(lang: Lang) -> Language {
-            unsafe {
-                match lang {
-                    $(
-                        Lang::$camel => $name(),
-                    )*
-                }
-            }
-        }
-    };
-}
+pub fn get_language(runtime_path: &std::path::Path, name: &str) -> Result<Language> {
+    let name = name.to_ascii_lowercase();
+    let mut library_path = runtime_path.join("grammars").join(&name);
+    // TODO: duplicated under build
+    library_path.set_extension(DYLIB_EXTENSION);
 
-#[macro_export]
-macro_rules! mk_get_language_name {
-    ( $( $camel:ident ),* ) => {
-        #[must_use]
-        pub const fn get_language_name(lang: Lang) -> &'static str {
-            match lang {
-                $(
-                    Lang::$camel => stringify!($camel),
-                )*
-            }
-        }
+    let library = unsafe { Library::new(&library_path) }
+        .with_context(|| format!("Error opening dynamic library {:?}", &library_path))?;
+    let language_fn_name = format!("tree_sitter_{}", replace_dashes_with_underscores(&name));
+    let language = unsafe {
+        let language_fn: Symbol<unsafe extern "C" fn() -> Language> = library
+            .get(language_fn_name.as_bytes())
+            .with_context(|| format!("Failed to load symbol {}", language_fn_name))?;
+        language_fn()
     };
+    std::mem::forget(library);
+    Ok(language)
 }
-
-#[macro_export]
-macro_rules! mk_langs {
-    ( $( ($camel:ident, $name:ident) ),* ) => {
-        mk_extern!($( $name ),*);
-        mk_enum!($( $camel ),*);
-        mk_get_language!($( ($camel, $name) ),*);
-        mk_get_language_name!($( $camel ),*);
-    };
-}
-
-mk_langs!(
-    // 1) Name for enum
-    // 2) tree-sitter function to call to get a Language
-    (Agda, tree_sitter_agda),
-    (Bash, tree_sitter_bash),
-    (Cpp, tree_sitter_cpp),
-    (CSharp, tree_sitter_c_sharp),
-    (Css, tree_sitter_css),
-    (C, tree_sitter_c),
-    (Elixir, tree_sitter_elixir),
-    (Go, tree_sitter_go),
-    // (Haskell, tree_sitter_haskell),
-    (Html, tree_sitter_html),
-    (Javascript, tree_sitter_javascript),
-    (Java, tree_sitter_java),
-    (Json, tree_sitter_json),
-    (Julia, tree_sitter_julia),
-    (Nix, tree_sitter_nix),
-    (Php, tree_sitter_php),
-    (Python, tree_sitter_python),
-    (Ruby, tree_sitter_ruby),
-    (Rust, tree_sitter_rust),
-    (Scala, tree_sitter_scala),
-    (Swift, tree_sitter_swift),
-    (Toml, tree_sitter_toml),
-    (Tsx, tree_sitter_tsx),
-    (Typescript, tree_sitter_typescript)
-);

helix-term/Cargo.toml

@@ -1,34 +1,38 @@
 [package]
 name = "helix-term"
-version = "0.2.0"
+version = "0.4.1"
 description = "A post-modern text editor."
 authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
 edition = "2018"
 license = "MPL-2.0"
+categories = ["editor", "command-line-utilities"]
+repository = "https://github.com/helix-editor/helix"
+homepage = "https://helix-editor.com"
+include = ["src/**/*", "README.md"]
 
 [package.metadata.nix]
 build = true
 app = true
 
 [features]
-embed_runtime = ["helix-core/embed_runtime"]
 
 [[bin]]
 name = "hx"
 path = "src/main.rs"
 
 [dependencies]
-helix-core = { path = "../helix-core" }
-helix-view = { path = "../helix-view", features = ["term"]}
-helix-lsp = { path = "../helix-lsp"}
+helix-core = { version = "0.4", path = "../helix-core" }
+helix-view = { version = "0.4", path = "../helix-view" }
+helix-lsp = { version = "0.4", path = "../helix-lsp" }
 
 anyhow = "1"
 once_cell = "1.8"
 
-tokio = { version = "1", features = ["full"] }
+tokio = { version = "1", features = ["rt", "rt-multi-thread", "io-util", "io-std", "time", "process", "macros", "fs", "parking_lot"] }
 num_cpus = "1"
 tui = { path = "../helix-tui", package = "helix-tui", default-features = false, features = ["crossterm"] }
 crossterm = { version = "0.20", features = ["event-stream"] }
+signal-hook = "0.3"
 
 futures-util = { version = "0.3", features = ["std", "async-await"], default-features = false }
 
@@ -41,7 +45,7 @@ log = "0.4"
 fuzzy-matcher = "0.3"
 ignore = "0.4"
 # shellexpand = "2.1"
-dirs-next = "2.0"
+# dirs-next = "2.0"
 # markdown doc rendering
 pulldown-cmark = { version = "0.8", default-features = false }
 
@@ -50,3 +54,6 @@ toml = "0.5"
 
 serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
+
+[target.'cfg(not(windows))'.dependencies]  # https://github.com/vorner/signal-hook/issues/100
+signal-hook-tokio = { version = "0.3", features = ["futures-v0_3"] }

helix-term/README.md

@@ -1,7 +0,0 @@
-
-window -> buffer -> text
-\-> contains "view", a viewport into the buffer
-
-view
-\-> selections etc
--> cursor

helix-term/src/application.rs

@@ -1,52 +1,94 @@
-use helix_lsp::lsp;
-use helix_view::{document::Mode, Document, Editor, Theme, View};
+use helix_core::syntax;
+use helix_lsp::{lsp, util::lsp_pos_to_pos, LspProgressMap};
+use helix_view::{theme, Editor};
 
-use crate::{args::Args, compositor::Compositor, ui};
+use crate::{args::Args, compositor::Compositor, config::Config, job::Jobs, ui};
 
-use log::{error, info};
+use log::error;
 
 use std::{
-    future::Future,
-    io::{self, stdout, Stdout, Write},
-    path::PathBuf,
+    io::{stdout, Write},
     sync::Arc,
-    time::Duration,
+    time::{Duration, Instant},
 };
 
 use anyhow::Error;
 
 use crossterm::{
-    event::{Event, EventStream},
+    event::{DisableMouseCapture, EnableMouseCapture, Event, EventStream},
     execute, terminal,
 };
-
-use tui::layout::Rect;
-
-use futures_util::stream::FuturesUnordered;
-use std::pin::Pin;
-
-type BoxFuture<T> = Pin<Box<dyn Future<Output = T> + Send>>;
-pub type LspCallback =
-    BoxFuture<Result<Box<dyn FnOnce(&mut Editor, &mut Compositor) + Send>, anyhow::Error>>;
-
-pub type LspCallbacks = FuturesUnordered<LspCallback>;
-pub type LspCallbackWrapper = Box<dyn FnOnce(&mut Editor, &mut Compositor) + Send>;
+#[cfg(not(windows))]
+use {
+    signal_hook::{consts::signal, low_level},
+    signal_hook_tokio::Signals,
+};
+#[cfg(windows)]
+type Signals = futures_util::stream::Empty<()>;
 
 pub struct Application {
     compositor: Compositor,
     editor: Editor,
 
-    callbacks: LspCallbacks,
+    // TODO should be separate to take only part of the config
+    config: Config,
+
+    // Currently never read from.  Remove the `allow(dead_code)` when
+    // that changes.
+    #[allow(dead_code)]
+    theme_loader: Arc<theme::Loader>,
+
+    // Currently never read from.  Remove the `allow(dead_code)` when
+    // that changes.
+    #[allow(dead_code)]
+    syn_loader: Arc<syntax::Loader>,
+
+    signals: Signals,
+    jobs: Jobs,
+    lsp_progress: LspProgressMap,
 }
 
 impl Application {
-    pub fn new(mut args: Args) -> Result<Self, Error> {
+    pub fn new(args: Args, mut config: Config) -> Result<Self, Error> {
         use helix_view::editor::Action;
         let mut compositor = Compositor::new()?;
         let size = compositor.size();
-        let mut editor = Editor::new(size);
 
-        compositor.push(Box::new(ui::EditorView::new()));
+        let conf_dir = helix_core::config_dir();
+
+        let theme_loader =
+            std::sync::Arc::new(theme::Loader::new(&conf_dir, &helix_core::runtime_dir()));
+
+        // load $HOME/.config/helix/languages.toml, fallback to default config
+        let lang_conf = std::fs::read(conf_dir.join("languages.toml"));
+        let lang_conf = lang_conf
+            .as_deref()
+            .unwrap_or(include_bytes!("../../languages.toml"));
+
+        let theme = if let Some(theme) = &config.theme {
+            match theme_loader.load(theme) {
+                Ok(theme) => theme,
+                Err(e) => {
+                    log::warn!("failed to load theme `{}` - {}", theme, e);
+                    theme_loader.default()
+                }
+            }
+        } else {
+            theme_loader.default()
+        };
+
+        let syn_loader_conf = toml::from_slice(lang_conf).expect("Could not parse languages.toml");
+        let syn_loader = std::sync::Arc::new(syntax::Loader::new(syn_loader_conf));
+
+        let mut editor = Editor::new(
+            size,
+            theme_loader.clone(),
+            syn_loader.clone(),
+            config.editor.clone(),
+        );
+
+        let editor_view = Box::new(ui::EditorView::new(std::mem::take(&mut config.keys)));
+        compositor.push(editor_view);
 
         if !args.files.is_empty() {
             let first = &args.files[0]; // we know it's not empty
@@ -54,25 +96,42 @@ impl Application {
                 editor.new_file(Action::VerticalSplit);
                 compositor.push(Box::new(ui::file_picker(first.clone())));
             } else {
+                let nr_of_files = args.files.len();
+                editor.open(first.to_path_buf(), Action::VerticalSplit)?;
                 for file in args.files {
                     if file.is_dir() {
                         return Err(anyhow::anyhow!(
                             "expected a path to file, found a directory. (to open a directory pass it as first argument)"
                         ));
                     } else {
-                        editor.open(file, Action::VerticalSplit)?;
+                        editor.open(file.to_path_buf(), Action::Load)?;
                     }
                 }
+                editor.set_status(format!("Loaded {} files.", nr_of_files));
             }
         } else {
             editor.new_file(Action::VerticalSplit);
         }
 
-        let mut app = Self {
+        editor.set_theme(theme);
+
+        #[cfg(windows)]
+        let signals = futures_util::stream::empty();
+        #[cfg(not(windows))]
+        let signals = Signals::new(&[signal::SIGTSTP, signal::SIGCONT])?;
+
+        let app = Self {
             compositor,
             editor,
 
-            callbacks: FuturesUnordered::new(),
+            config,
+
+            theme_loader,
+            syn_loader,
+
+            signals,
+            jobs: Jobs::new(),
+            lsp_progress: LspProgressMap::new(),
         };
 
         Ok(app)
@@ -81,11 +140,11 @@ impl Application {
     fn render(&mut self) {
         let editor = &mut self.editor;
         let compositor = &mut self.compositor;
-        let callbacks = &mut self.callbacks;
+        let jobs = &mut self.jobs;
 
         let mut cx = crate::compositor::Context {
             editor,
-            callbacks,
+            jobs,
             scroll: None,
         };
 
@@ -94,44 +153,78 @@ impl Application {
 
     pub async fn event_loop(&mut self) {
         let mut reader = EventStream::new();
+        let mut last_render = Instant::now();
+        let deadline = Duration::from_secs(1) / 60;
 
         self.render();
 
         loop {
             if self.editor.should_close() {
+                self.jobs.finish();
                 break;
             }
 
             use futures_util::StreamExt;
 
             tokio::select! {
+                biased;
+
                 event = reader.next() => {
                     self.handle_terminal_events(event)
                 }
-                Some(call) = self.editor.language_servers.incoming.next() => {
-                    self.handle_language_server_message(call).await
+                Some(signal) = self.signals.next() => {
+                    self.handle_signals(signal).await;
                 }
-                Some(callback) = &mut self.callbacks.next() => {
-                    self.handle_language_server_callback(callback)
-                }
-            }
-        }
-    }
-    pub fn handle_language_server_callback(
-        &mut self,
-        callback: Result<LspCallbackWrapper, anyhow::Error>,
-    ) {
-        if let Ok(callback) = callback {
-            // TODO: handle Err()
-            callback(&mut self.editor, &mut self.compositor);
+                Some((id, call)) = self.editor.language_servers.incoming.next() => {
+                    self.handle_language_server_message(call, id).await;
+                    // limit render calls for fast language server messages
+                    let last = self.editor.language_servers.incoming.is_empty();
+                    if last || last_render.elapsed() > deadline {
                         self.render();
+                        last_render = Instant::now();
+                    }
+                }
+                Some(callback) = self.jobs.futures.next() => {
+                    self.jobs.handle_callback(&mut self.editor, &mut self.compositor, callback);
+                    self.render();
+                }
+                Some(callback) = self.jobs.wait_futures.next() => {
+                    self.jobs.handle_callback(&mut self.editor, &mut self.compositor, callback);
+                    self.render();
+                }
+            }
+        }
+    }
+
+    #[cfg(windows)]
+    // no signal handling available on windows
+    pub async fn handle_signals(&mut self, _signal: ()) {}
+
+    #[cfg(not(windows))]
+    pub async fn handle_signals(&mut self, signal: i32) {
+        use helix_view::graphics::Rect;
+        match signal {
+            signal::SIGTSTP => {
+                self.compositor.save_cursor();
+                self.restore_term().unwrap();
+                low_level::emulate_default_handler(signal::SIGTSTP).unwrap();
+            }
+            signal::SIGCONT => {
+                self.claim_term().await.unwrap();
+                // redraw the terminal
+                let Rect { width, height, .. } = self.compositor.size();
+                self.compositor.resize(width, height);
+                self.compositor.load_cursor();
+                self.render();
+            }
+            _ => unreachable!(),
         }
     }
 
     pub fn handle_terminal_events(&mut self, event: Option<Result<Event, crossterm::ErrorKind>>) {
         let mut cx = crate::compositor::Context {
             editor: &mut self.editor,
-            callbacks: &mut self.callbacks,
+            jobs: &mut self.jobs,
             scroll: None,
         };
         // Handle key events
@@ -152,8 +245,21 @@ impl Application {
         }
     }
 
-    pub async fn handle_language_server_message(&mut self, call: helix_lsp::Call) {
-        use helix_lsp::{Call, Notification};
+    pub async fn handle_language_server_message(
+        &mut self,
+        call: helix_lsp::Call,
+        server_id: usize,
+    ) {
+        use helix_lsp::{Call, MethodCall, Notification};
+        let editor_view = self
+            .compositor
+            .find(std::any::type_name::<ui::EditorView>())
+            .expect("expected at least one EditorView");
+        let editor_view = editor_view
+            .as_any_mut()
+            .downcast_mut::<ui::EditorView>()
+            .unwrap();
+
         match call {
             Call::Notification(helix_lsp::jsonrpc::Notification { method, params, .. }) => {
                 let notification = match Notification::parse(&method, params) {
@@ -161,7 +267,6 @@ impl Application {
                     None => return,
                 };
 
-                // TODO: parse should return Result/Option
                 match notification {
                     Notification::PublishDiagnostics(params) => {
                         let path = Some(params.uri.to_file_path().unwrap());
@@ -180,10 +285,9 @@ impl Application {
                                 .into_iter()
                                 .filter_map(|diagnostic| {
                                     use helix_core::{
-                                        diagnostic::{Range, Severity, Severity::*},
+                                        diagnostic::{Range, Severity::*},
                                         Diagnostic,
                                     };
-                                    use helix_lsp::{lsp, util::lsp_pos_to_pos};
                                     use lsp::DiagnosticSeverity;
 
                                     let language_server = doc.language_server().unwrap();
@@ -230,8 +334,6 @@ impl Application {
                                 .collect();
 
                             doc.set_diagnostics(diagnostics);
-                            // TODO: we want to process all the events in queue, then render. publishDiagnostic tends to send a whole bunch of events
-                            self.render();
                         }
                     }
                     Notification::ShowMessage(params) => {
@@ -241,64 +343,139 @@ impl Application {
                         log::warn!("unhandled window/logMessage: {:?}", params);
                     }
                     Notification::ProgressMessage(params) => {
-                        let token = match params.token {
-                            lsp::NumberOrString::Number(n) => n.to_string(),
-                            lsp::NumberOrString::String(s) => s,
-                        };
-                        let msg = {
-                            let lsp::ProgressParamsValue::WorkDone(work) = params.value;
-                            let parts = match work {
+                        let lsp::ProgressParams { token, value } = params;
+
+                        let lsp::ProgressParamsValue::WorkDone(work) = value;
+                        let parts = match &work {
                             lsp::WorkDoneProgress::Begin(lsp::WorkDoneProgressBegin {
                                 title,
                                 message,
                                 percentage,
                                 ..
-                                }) => (Some(title), message, percentage.map(|n| n.to_string())),
+                            }) => (Some(title), message, percentage),
                             lsp::WorkDoneProgress::Report(lsp::WorkDoneProgressReport {
                                 message,
                                 percentage,
                                 ..
-                                }) => (None, message, percentage.map(|n| n.to_string())),
-                                lsp::WorkDoneProgress::End(lsp::WorkDoneProgressEnd {
-                                    message,
-                                }) => {
-                                    if let Some(message) = message {
-                                        (None, Some(message), None)
+                            }) => (None, message, percentage),
+                            lsp::WorkDoneProgress::End(lsp::WorkDoneProgressEnd { message }) => {
+                                if message.is_some() {
+                                    (None, message, &None)
                                 } else {
+                                    self.lsp_progress.end_progress(server_id, &token);
+                                    if !self.lsp_progress.is_progressing(server_id) {
+                                        editor_view.spinners_mut().get_or_create(server_id).stop();
+                                    }
                                     self.editor.clear_status();
+
+                                    // we want to render to clear any leftover spinners or messages
                                     return;
                                 }
                             }
                         };
-                            match parts {
+
+                        let token_d: &dyn std::fmt::Display = match &token {
+                            lsp::NumberOrString::Number(n) => n,
+                            lsp::NumberOrString::String(s) => s,
+                        };
+
+                        let status = match parts {
                             (Some(title), Some(message), Some(percentage)) => {
-                                    format!("{}% {} - {}", percentage, title, message)
+                                format!("[{}] {}% {} - {}", token_d, percentage, title, message)
                             }
                             (Some(title), None, Some(percentage)) => {
-                                    format!("{}% {}", percentage, title)
+                                format!("[{}] {}% {}", token_d, percentage, title)
                             }
                             (Some(title), Some(message), None) => {
-                                    format!("{} - {}", title, message)
+                                format!("[{}] {} - {}", token_d, title, message)
                             }
                             (None, Some(message), Some(percentage)) => {
-                                    format!("{}% {}", percentage, message)
+                                format!("[{}] {}% {}", token_d, percentage, message)
                             }
-                                (Some(title), None, None) => title,
-                                (None, Some(message), None) => message,
-                                (None, None, Some(percentage)) => format!("{}%", percentage),
-                                (None, None, None) => "".into(),
+                            (Some(title), None, None) => {
+                                format!("[{}] {}", token_d, title)
+                            }
+                            (None, Some(message), None) => {
+                                format!("[{}] {}", token_d, message)
+                            }
+                            (None, None, Some(percentage)) => {
+                                format!("[{}] {}%", token_d, percentage)
+                            }
+                            (None, None, None) => format!("[{}]", token_d),
+                        };
+
+                        if let lsp::WorkDoneProgress::End(_) = work {
+                            self.lsp_progress.end_progress(server_id, &token);
+                            if !self.lsp_progress.is_progressing(server_id) {
+                                editor_view.spinners_mut().get_or_create(server_id).stop();
+                            }
+                        } else {
+                            self.lsp_progress.update(server_id, token, work);
+                        }
+
+                        if self.config.lsp.display_messages {
+                            self.editor.set_status(status);
+                        }
+                    }
+                }
+            }
+            Call::MethodCall(helix_lsp::jsonrpc::MethodCall {
+                method, params, id, ..
+            }) => {
+                let call = match MethodCall::parse(&method, params) {
+                    Some(call) => call,
+                    None => {
+                        error!("Method not found {}", method);
+                        return;
                     }
                 };
-                        let status = format!("[{}] {}", token, msg);
-                        self.editor.set_status(status);
-                        self.render();
-                    }
-                    _ => unreachable!(),
-                }
-            }
-            Call::MethodCall(call) => {
-                error!("Method not found {}", call.method);
 
+                match call {
+                    MethodCall::WorkDoneProgressCreate(params) => {
+                        self.lsp_progress.create(server_id, params.token);
+
+                        let spinner = editor_view.spinners_mut().get_or_create(server_id);
+                        if spinner.is_stopped() {
+                            spinner.start();
+                        }
+
+                        let doc = self.editor.documents().find(|doc| {
+                            doc.language_server()
+                                .map(|server| server.id() == server_id)
+                                .unwrap_or_default()
+                        });
+                        match doc {
+                            Some(doc) => {
+                                // it's ok to unwrap, we check for the language server before
+                                let server = doc.language_server().unwrap();
+                                tokio::spawn(server.reply(id, Ok(serde_json::Value::Null)));
+                            }
+                            None => {
+                                if let Some(server) =
+                                    self.editor.language_servers.get_by_id(server_id)
+                                {
+                                    log::warn!(
+                                        "missing document with language server id `{}`",
+                                        server_id
+                                    );
+                                    tokio::spawn(server.reply(
+                                        id,
+                                        Err(helix_lsp::jsonrpc::Error {
+                                            code: helix_lsp::jsonrpc::ErrorCode::InternalError,
+                                            message: "document missing".to_string(),
+                                            data: None,
+                                        }),
+                                    ));
+                                } else {
+                                    log::warn!(
+                                        "can't find language server with id `{}`",
+                                        server_id
+                                    );
+                                }
+                            }
+                        }
+                    }
+                }
                 // self.language_server.reply(
                 //     call.id,
                 //     // TODO: make a Into trait that can cast to Err(jsonrpc::Error)
@@ -313,29 +490,48 @@ impl Application {
         }
     }
 
-    pub async fn run(&mut self) -> Result<(), Error> {
+    async fn claim_term(&mut self) -> Result<(), Error> {
         terminal::enable_raw_mode()?;
-
         let mut stdout = stdout();
-
         execute!(stdout, terminal::EnterAlternateScreen)?;
+        if self.config.editor.mouse {
+            execute!(stdout, EnableMouseCapture)?;
+        }
+        Ok(())
+    }
+
+    fn restore_term(&mut self) -> Result<(), Error> {
+        let mut stdout = stdout();
+        // reset cursor shape
+        write!(stdout, "\x1B[2 q")?;
+        execute!(stdout, DisableMouseCapture)?;
+        execute!(stdout, terminal::LeaveAlternateScreen)?;
+        terminal::disable_raw_mode()?;
+        Ok(())
+    }
+
+    pub async fn run(&mut self) -> Result<(), Error> {
+        self.claim_term().await?;
 
         // Exit the alternate screen and disable raw mode before panicking
         let hook = std::panic::take_hook();
         std::panic::set_hook(Box::new(move |info| {
-            execute!(std::io::stdout(), terminal::LeaveAlternateScreen);
-            terminal::disable_raw_mode();
+            // We can't handle errors properly inside this closure.  And it's
+            // probably not a good idea to `unwrap()` inside a panic handler.
+            // So we just ignore the `Result`s.
+            let _ = execute!(std::io::stdout(), DisableMouseCapture);
+            let _ = execute!(std::io::stdout(), terminal::LeaveAlternateScreen);
+            let _ = terminal::disable_raw_mode();
             hook(info);
         }));
 
         self.event_loop().await;
 
-        // reset cursor shape
-        write!(stdout, "\x1B[2 q");
+        if self.editor.close_language_servers(None).await.is_err() {
+            log::error!("Timed out waiting for language servers to shutdown");
+        };
 
-        execute!(stdout, terminal::LeaveAlternateScreen)?;
-
-        terminal::disable_raw_mode()?;
+        self.restore_term()?;
 
         Ok(())
     }

helix-term/src/args.rs

@@ -17,7 +17,7 @@ impl Args {
 
         iter.next(); // skip the program, we don't care about that
 
-        while let Some(arg) = iter.next() {
+        for arg in &mut iter {
             match arg.as_str() {
                 "--" => break, // stop parsing at this point treat the remaining as files
                 "--version" => args.display_version = true,

helix-term/src/compositor.rs

@@ -1,10 +1,11 @@
 // Each component declares it's own size constraints and gets fitted based on it's parent.
 // Q: how does this work with popups?
 // cursive does compositor.screen_mut().add_layer_at(pos::absolute(x, y), <component>)
+use helix_core::Position;
+use helix_view::graphics::{CursorKind, Rect};
 
 use crossterm::event::Event;
-use helix_core::Position;
-use tui::{buffer::Buffer as Surface, layout::Rect};
+use tui::buffer::Buffer as Surface;
 
 pub type Callback = Box<dyn FnOnce(&mut Compositor)>;
 
@@ -24,17 +25,17 @@ pub enum EventResult {
 
 use helix_view::Editor;
 
-use crate::application::LspCallbacks;
+use crate::job::Jobs;
 
 pub struct Context<'a> {
     pub editor: &'a mut Editor,
     pub scroll: Option<usize>,
-    pub callbacks: &'a mut LspCallbacks,
+    pub jobs: &'a mut Jobs,
 }
 
 pub trait Component: Any + AnyComponent {
     /// Process input events, return true if handled.
-    fn handle_event(&mut self, event: Event, ctx: &mut Context) -> EventResult {
+    fn handle_event(&mut self, _event: Event, _ctx: &mut Context) -> EventResult {
         EventResult::Ignored
     }
     // , args: ()
@@ -45,15 +46,16 @@ pub trait Component: Any + AnyComponent {
     }
 
     /// Render the component onto the provided surface.
-    fn render(&self, area: Rect, frame: &mut Surface, ctx: &mut Context);
+    fn render(&mut self, area: Rect, frame: &mut Surface, ctx: &mut Context);
 
-    fn cursor_position(&self, area: Rect, ctx: &Editor) -> Option<Position> {
-        None
+    /// Get cursor position and cursor kind.
+    fn cursor(&self, _area: Rect, _ctx: &Editor) -> (Option<Position>, CursorKind) {
+        (None, CursorKind::Hidden)
     }
 
     /// May be used by the parent component to compute the child area.
     /// viewport is the maximum allowed area, and the child should stay within those bounds.
-    fn required_size(&mut self, viewport: (u16, u16)) -> Option<(u16, u16)> {
+    fn required_size(&mut self, _viewport: (u16, u16)) -> Option<(u16, u16)> {
         // TODO: for scrolling, the scroll wrapper should place a size + offset on the Context
         // that way render can use it
         None
@@ -66,21 +68,24 @@ pub trait Component: Any + AnyComponent {
 
 use anyhow::Error;
 use std::io::stdout;
-use tui::backend::CrosstermBackend;
+use tui::backend::{Backend, CrosstermBackend};
 type Terminal = tui::terminal::Terminal<CrosstermBackend<std::io::Stdout>>;
 
 pub struct Compositor {
     layers: Vec<Box<dyn Component>>,
     terminal: Terminal,
+
+    pub(crate) last_picker: Option<Box<dyn Component>>,
 }
 
 impl Compositor {
     pub fn new() -> Result<Self, Error> {
         let backend = CrosstermBackend::new(stdout());
-        let mut terminal = Terminal::new(backend)?;
+        let terminal = Terminal::new(backend)?;
         Ok(Self {
             layers: Vec::new(),
             terminal,
+            last_picker: None,
         })
     }
 
@@ -94,6 +99,21 @@ impl Compositor {
             .expect("Unable to resize terminal")
     }
 
+    pub fn save_cursor(&mut self) {
+        if self.terminal.cursor_kind() == CursorKind::Hidden {
+            self.terminal
+                .backend_mut()
+                .show_cursor(CursorKind::Block)
+                .ok();
+        }
+    }
+
+    pub fn load_cursor(&mut self) {
+        if self.terminal.cursor_kind() == CursorKind::Hidden {
+            self.terminal.backend_mut().hide_cursor().ok();
+        }
+    }
+
     pub fn push(&mut self, mut layer: Box<dyn Component>) {
         let size = self.size();
         // trigger required_size on init
@@ -101,8 +121,8 @@ impl Compositor {
         self.layers.push(layer);
     }
 
-    pub fn pop(&mut self) {
-        self.layers.pop();
+    pub fn pop(&mut self) -> Option<Box<dyn Component>> {
+        self.layers.pop()
     }
 
     pub fn handle_event(&mut self, event: Event, cx: &mut Context) -> bool {
@@ -122,8 +142,7 @@ impl Compositor {
     }
 
     pub fn render(&mut self, cx: &mut Context) {
-        let area = self
-            .terminal
+        self.terminal
             .autoresize()
             .expect("Unable to determine terminal size");
 
@@ -133,24 +152,23 @@ impl Compositor {
 
         let area = *surface.area();
 
-        for layer in &self.layers {
-            layer.render(area, surface, cx)
+        for layer in &mut self.layers {
+            layer.render(area, surface, cx);
         }
 
-        let pos = self
-            .cursor_position(area, cx.editor)
-            .map(|pos| (pos.col as u16, pos.row as u16));
+        let (pos, kind) = self.cursor(area, cx.editor);
+        let pos = pos.map(|pos| (pos.col as u16, pos.row as u16));
 
-        self.terminal.draw(pos);
+        self.terminal.draw(pos, kind).unwrap();
     }
 
-    pub fn cursor_position(&self, area: Rect, editor: &Editor) -> Option<Position> {
+    pub fn cursor(&self, area: Rect, editor: &Editor) -> (Option<Position>, CursorKind) {
         for layer in self.layers.iter().rev() {
-            if let Some(pos) = layer.cursor_position(area, editor) {
-                return Some(pos);
+            if let (Some(pos), kind) = layer.cursor(area, editor) {
+                return (Some(pos), kind);
             }
         }
-        None
+        (None, CursorKind::Hidden)
     }
 
     pub fn find(&mut self, type_name: &str) -> Option<&mut dyn Component> {
@@ -182,8 +200,9 @@ pub trait AnyComponent {
     /// # Examples
     ///
     /// ```rust
-    /// // let boxed: Box<Component> = Box::new(TextComponent::new("text"));
-    /// // let text: Box<TextComponent> = boxed.as_boxed_any().downcast().unwrap();
+    /// use helix_term::{ui::Text, compositor::Component};
+    /// let boxed: Box<Component> = Box::new(Text::new("text".to_string()));
+    /// let text: Box<Text> = boxed.as_boxed_any().downcast().unwrap();
     /// ```
     fn as_boxed_any(self: Box<Self>) -> Box<dyn Any>;
 }

helix-term/src/config.rs

@@ -0,0 +1,53 @@
+use serde::Deserialize;
+
+use crate::keymap::Keymaps;
+
+#[derive(Debug, Default, Clone, PartialEq, Deserialize)]
+pub struct Config {
+    pub theme: Option<String>,
+    #[serde(default)]
+    pub lsp: LspConfig,
+    #[serde(default)]
+    pub keys: Keymaps,
+    #[serde(default)]
+    pub editor: helix_view::editor::Config,
+}
+
+#[derive(Debug, Default, Clone, PartialEq, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct LspConfig {
+    pub display_messages: bool,
+}
+
+#[test]
+fn parsing_keymaps_config_file() {
+    use crate::keymap;
+    use crate::keymap::Keymap;
+    use helix_core::hashmap;
+    use helix_view::document::Mode;
+
+    let sample_keymaps = r#"
+            [keys.insert]
+            y = "move_line_down"
+            S-C-a = "delete_selection"
+
+            [keys.normal]
+            A-F12 = "move_next_word_end"
+        "#;
+
+    assert_eq!(
+        toml::from_str::<Config>(sample_keymaps).unwrap(),
+        Config {
+            keys: Keymaps(hashmap! {
+                Mode::Insert => Keymap::new(keymap!({ "Insert mode"
+                    "y" => move_line_down,
+                    "S-C-a" => delete_selection,
+                })),
+                Mode::Normal => Keymap::new(keymap!({ "Normal mode"
+                    "A-F12" => move_next_word_end,
+                })),
+            }),
+            ..Default::default()
+        }
+    );
+}

helix-term/src/job.rs

@@ -0,0 +1,100 @@
+use helix_view::Editor;
+
+use crate::compositor::Compositor;
+
+use futures_util::future::{self, BoxFuture, Future, FutureExt};
+use futures_util::stream::{FuturesUnordered, StreamExt};
+
+pub type Callback = Box<dyn FnOnce(&mut Editor, &mut Compositor) + Send>;
+pub type JobFuture = BoxFuture<'static, anyhow::Result<Option<Callback>>>;
+
+pub struct Job {
+    pub future: BoxFuture<'static, anyhow::Result<Option<Callback>>>,
+    /// Do we need to wait for this job to finish before exiting?
+    pub wait: bool,
+}
+
+#[derive(Default)]
+pub struct Jobs {
+    pub futures: FuturesUnordered<JobFuture>,
+    /// These are the ones that need to complete before we exit.
+    pub wait_futures: FuturesUnordered<JobFuture>,
+}
+
+impl Job {
+    pub fn new<F: Future<Output = anyhow::Result<()>> + Send + 'static>(f: F) -> Job {
+        Job {
+            future: f.map(|r| r.map(|()| None)).boxed(),
+            wait: false,
+        }
+    }
+
+    pub fn with_callback<F: Future<Output = anyhow::Result<Callback>> + Send + 'static>(
+        f: F,
+    ) -> Job {
+        Job {
+            future: f.map(|r| r.map(Some)).boxed(),
+            wait: false,
+        }
+    }
+
+    pub fn wait_before_exiting(mut self) -> Job {
+        self.wait = true;
+        self
+    }
+}
+
+impl Jobs {
+    pub fn new() -> Jobs {
+        Jobs::default()
+    }
+
+    pub fn spawn<F: Future<Output = anyhow::Result<()>> + Send + 'static>(&mut self, f: F) {
+        self.add(Job::new(f));
+    }
+
+    pub fn callback<F: Future<Output = anyhow::Result<Callback>> + Send + 'static>(
+        &mut self,
+        f: F,
+    ) {
+        self.add(Job::with_callback(f));
+    }
+
+    pub fn handle_callback(
+        &mut self,
+        editor: &mut Editor,
+        compositor: &mut Compositor,
+        call: anyhow::Result<Option<Callback>>,
+    ) {
+        match call {
+            Ok(None) => {}
+            Ok(Some(call)) => {
+                call(editor, compositor);
+            }
+            Err(e) => {
+                editor.set_error(format!("Async job failed: {}", e));
+            }
+        }
+    }
+
+    pub async fn next_job(&mut self) -> Option<anyhow::Result<Option<Callback>>> {
+        tokio::select! {
+            event = self.futures.next() => {  event }
+            event = self.wait_futures.next() => { event }
+        }
+    }
+
+    pub fn add(&mut self, j: Job) {
+        if j.wait {
+            self.wait_futures.push(j.future);
+        } else {
+            self.futures.push(j.future);
+        }
+    }
+
+    /// Blocks until all the jobs that need to be waited on are done.
+    pub fn finish(&mut self) {
+        let wait_futures = std::mem::take(&mut self.wait_futures);
+        helix_lsp::block_on(wait_futures.for_each(|_| future::ready(())));
+    }
+}

helix-term/src/keymap.rs

@@ -1,319 +1,629 @@
-use crate::commands::{self, Command};
+pub use crate::commands::Command;
+use crate::config::Config;
 use helix_core::hashmap;
-use helix_view::document::Mode;
-use std::collections::HashMap;
-
-// Kakoune-inspired:
-// mode = {
-//      normal = {
-//          q = record_macro
-//          w = (next) word
-//          W = next WORD
-//          e = end of word
-//          E = end of WORD
-//          r = replace
-//          R = replace with yanked
-//          t = 'till char
-//          y = yank
-//          u = undo
-//          U = redo
-//          i = insert
-//          I = INSERT (start of line)
-//          o = open below (insert on new line below)
-//          O = open above (insert on new line above)
-//          p = paste (before cursor)
-//          P = PASTE (after cursor)
-//          ` =
-//          [ = select to text object start (alt = select whole object)
-//          ] = select to text object end
-//          { = extend to inner object start
-//          } = extend to inner object end
-//          a = append
-//          A = APPEND (end of line)
-//          s = split
-//          S = select
-//          d = delete()
-//          f = find_char()
-//          g = goto (gg, G, gc, gd, etc)
-//
-//          h = move_char_left(n)   || arrow-left  = move_char_left(n)
-//          j = move_line_down(n)   || arrow-down  = move_line_down(n)
-//          k = move_line_up(n)     || arrow_up    = move_line_up(n)
-//          l = move_char_right(n)  || arrow-right = move_char_right(n)
-//          : = command line
-//          ; = collapse selection to cursor
-//          " = use register
-//          ` = convert case? (to lower) (alt = swap case)
-//          ~ = convert to upper case
-//          . = repeat last command
-//          \ = disable hook?
-//          / = search
-//          > = indent
-//          < = deindent
-//          % = select whole buffer (in vim = jump to matching bracket)
-//          * = search pattern in selection
-//          ( = rotate main selection backward
-//          ) = rotate main selection forward
-//          - = trim selections? (alt = merge contiguous sel together)
-//          @ = convert tabs to spaces
-//          & = align cursor
-//          ? = extend to next given regex match (alt = to prev)
-//
-//          in kakoune these are alt-h alt-l / gh gl
-//                              select from curs to begin end / move curs to begin end
-//          0 = start of line
-//          ^ = start of line(first non blank char) || Home  = start of line(first non blank char)
-//          $ = end of line                         || End   = end of line
-//
-//          z = save selections
-//          Z = restore selections
-//          x = select line
-//          X = extend line
-//          c = change selected text
-//          C = copy selection?
-//          v = view menu (viewport manipulation)
-//          b = select to previous word start
-//          B = select to previous WORD start
-//
-//
-//
-//
-//
-//
-//          = = align?
-//          + =
-//      }
-//
-//      gd = goto definition
-//      gr = goto reference
-//      [d = previous diagnostic
-//      d] = next diagnostic
-//      [D = first diagnostic
-//      D] = last diagnostic
-// }
-
-// #[cfg(feature = "term")]
-pub use crossterm::event::{KeyCode, KeyEvent, KeyModifiers};
-
-pub type Keymap = HashMap<KeyEvent, Command>;
-pub type Keymaps = HashMap<Mode, Keymap>;
+use helix_view::{document::Mode, info::Info, input::KeyEvent};
+use serde::Deserialize;
+use std::{
+    collections::HashMap,
+    ops::{Deref, DerefMut},
+};
 
 #[macro_export]
 macro_rules! key {
     ($key:ident) => {
         KeyEvent {
-            code: KeyCode::$key,
-            modifiers: KeyModifiers::NONE,
+            code: ::helix_view::keyboard::KeyCode::$key,
+            modifiers: ::helix_view::keyboard::KeyModifiers::NONE,
         }
     };
     ($($ch:tt)*) => {
         KeyEvent {
-            code: KeyCode::Char($($ch)*),
-            modifiers: KeyModifiers::NONE,
+            code: ::helix_view::keyboard::KeyCode::Char($($ch)*),
+            modifiers: ::helix_view::keyboard::KeyModifiers::NONE,
         }
     };
 }
 
-macro_rules! ctrl {
-    ($($ch:tt)*) => {
-        KeyEvent {
-            code: KeyCode::Char($($ch)*),
-            modifiers: KeyModifiers::CONTROL,
+/// Macro for defining the root of a `Keymap` object. Example:
+///
+/// ```
+/// # use helix_core::hashmap;
+/// # use helix_term::keymap;
+/// # use helix_term::keymap::Keymap;
+/// let normal_mode = keymap!({ "Normal mode"
+///     "i" => insert_mode,
+///     "g" => { "Goto"
+///         "g" => goto_file_start,
+///         "e" => goto_file_end,
+///     },
+///     "j" | "down" => move_line_down,
+/// });
+/// let keymap = Keymap::new(normal_mode);
+/// ```
+#[macro_export]
+macro_rules! keymap {
+    (@trie $cmd:ident) => {
+        $crate::keymap::KeyTrie::Leaf($crate::commands::Command::$cmd)
+    };
+
+    (@trie
+        { $label:literal $($($key:literal)|+ => $value:tt,)+ }
+    ) => {
+        keymap!({ $label $($($key)|+ => $value,)+ })
+    };
+
+    (
+        { $label:literal $($($key:literal)|+ => $value:tt,)+ }
+    ) => {
+        // modified from the hashmap! macro
+        {
+            let _cap = hashmap!(@count $($($key),+),*);
+            let mut _map = ::std::collections::HashMap::with_capacity(_cap);
+            let mut _order = ::std::vec::Vec::with_capacity(_cap);
+            $(
+                $(
+                    let _key = $key.parse::<::helix_view::input::KeyEvent>().unwrap();
+                    _map.insert(
+                        _key,
+                        keymap!(@trie $value)
+                    );
+                    _order.push(_key);
+                )+
+            )*
+            $crate::keymap::KeyTrie::Node($crate::keymap::KeyTrieNode::new($label, _map, _order))
         }
     };
 }
 
-macro_rules! alt {
-    ($($ch:tt)*) => {
-        KeyEvent {
-            code: KeyCode::Char($($ch)*),
-            modifiers: KeyModifiers::ALT,
+#[derive(Debug, Clone, Deserialize)]
+pub struct KeyTrieNode {
+    /// A label for keys coming under this node, like "Goto mode"
+    #[serde(skip)]
+    name: String,
+    #[serde(flatten)]
+    map: HashMap<KeyEvent, KeyTrie>,
+    #[serde(skip)]
+    order: Vec<KeyEvent>,
 }
+
+impl KeyTrieNode {
+    pub fn new(name: &str, map: HashMap<KeyEvent, KeyTrie>, order: Vec<KeyEvent>) -> Self {
+        Self {
+            name: name.to_string(),
+            map,
+            order,
+        }
+    }
+
+    pub fn name(&self) -> &str {
+        &self.name
+    }
+
+    /// Merge another Node in. Leaves and subnodes from the other node replace
+    /// corresponding keyevent in self, except when both other and self have
+    /// subnodes for same key. In that case the merge is recursive.
+    pub fn merge(&mut self, mut other: Self) {
+        for (key, trie) in std::mem::take(&mut other.map) {
+            if let Some(KeyTrie::Node(node)) = self.map.get_mut(&key) {
+                if let KeyTrie::Node(other_node) = trie {
+                    node.merge(other_node);
+                    continue;
+                }
+            }
+            self.map.insert(key, trie);
+        }
+
+        for &key in self.map.keys() {
+            if !self.order.contains(&key) {
+                self.order.push(key);
+            }
+        }
+    }
+}
+
+impl From<KeyTrieNode> for Info {
+    fn from(node: KeyTrieNode) -> Self {
+        let mut body: Vec<(&str, Vec<KeyEvent>)> = Vec::with_capacity(node.len());
+        for (&key, trie) in node.iter() {
+            let desc = match trie {
+                KeyTrie::Leaf(cmd) => cmd.doc(),
+                KeyTrie::Node(n) => n.name(),
             };
+            match body.iter().position(|(d, _)| d == &desc) {
+                // FIXME: multiple keys are ordered randomly (use BTreeSet)
+                Some(pos) => body[pos].1.push(key),
+                None => body.push((desc, vec![key])),
+            }
+        }
+        body.sort_unstable_by_key(|(_, keys)| {
+            node.order.iter().position(|&k| k == keys[0]).unwrap()
+        });
+        let prefix = format!("{} ", node.name());
+        if body.iter().all(|(desc, _)| desc.starts_with(&prefix)) {
+            body = body
+                .into_iter()
+                .map(|(desc, keys)| (desc.strip_prefix(&prefix).unwrap(), keys))
+                .collect();
+        }
+        Info::key(node.name(), body)
+    }
 }
 
-pub fn default() -> Keymaps {
-    let normal = hashmap!(
-        key!('h') => commands::move_char_left as Command,
-        key!('j') => commands::move_line_down,
-        key!('k') => commands::move_line_up,
-        key!('l') => commands::move_char_right,
+impl Default for KeyTrieNode {
+    fn default() -> Self {
+        Self::new("", HashMap::new(), Vec::new())
+    }
+}
 
-        key!(Left) => commands::move_char_left,
-        key!(Down) => commands::move_line_down,
-        key!(Up) => commands::move_line_up,
-        key!(Right) => commands::move_char_right,
+impl PartialEq for KeyTrieNode {
+    fn eq(&self, other: &Self) -> bool {
+        self.map == other.map
+    }
+}
 
-        key!('t') => commands::find_till_char,
-        key!('f') => commands::find_next_char,
-        key!('T') => commands::till_prev_char,
-        key!('F') => commands::find_prev_char,
-        // and matching set for select mode (extend)
-        //
-        key!('r') => commands::replace,
-        key!('R') => commands::replace_with_yanked,
+impl Deref for KeyTrieNode {
+    type Target = HashMap<KeyEvent, KeyTrie>;
 
-        key!(Home) => commands::move_line_start,
-        key!(End) => commands::move_line_end,
+    fn deref(&self) -> &Self::Target {
+        &self.map
+    }
+}
 
-        key!('w') => commands::move_next_word_start,
-        key!('b') => commands::move_prev_word_start,
-        key!('e') => commands::move_next_word_end,
+impl DerefMut for KeyTrieNode {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.map
+    }
+}
 
-        key!('v') => commands::select_mode,
-        key!('g') => commands::goto_mode,
-        key!(':') => commands::command_mode,
+#[derive(Debug, Clone, PartialEq, Deserialize)]
+#[serde(untagged)]
+pub enum KeyTrie {
+    Leaf(Command),
+    Node(KeyTrieNode),
+}
 
-        key!('i') => commands::insert_mode,
-        key!('I') => commands::prepend_to_line,
-        key!('a') => commands::append_mode,
-        key!('A') => commands::append_to_line,
-        key!('o') => commands::open_below,
-        key!('O') => commands::open_above,
+impl KeyTrie {
+    pub fn node(&self) -> Option<&KeyTrieNode> {
+        match *self {
+            KeyTrie::Node(ref node) => Some(node),
+            KeyTrie::Leaf(_) => None,
+        }
+    }
+
+    pub fn node_mut(&mut self) -> Option<&mut KeyTrieNode> {
+        match *self {
+            KeyTrie::Node(ref mut node) => Some(node),
+            KeyTrie::Leaf(_) => None,
+        }
+    }
+
+    /// Merge another KeyTrie in, assuming that this KeyTrie and the other
+    /// are both Nodes. Panics otherwise.
+    pub fn merge_nodes(&mut self, mut other: Self) {
+        let node = std::mem::take(other.node_mut().unwrap());
+        self.node_mut().unwrap().merge(node);
+    }
+
+    pub fn search(&self, keys: &[KeyEvent]) -> Option<&KeyTrie> {
+        let mut trie = self;
+        for key in keys {
+            trie = match trie {
+                KeyTrie::Node(map) => map.get(key),
+                // leaf encountered while keys left to process
+                KeyTrie::Leaf(_) => None,
+            }?
+        }
+        Some(trie)
+    }
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum KeymapResult {
+    /// Needs more keys to execute a command. Contains valid keys for next keystroke.
+    Pending(KeyTrieNode),
+    Matched(Command),
+    /// Key was not found in the root keymap
+    NotFound,
+    /// Key is invalid in combination with previous keys. Contains keys leading upto
+    /// and including current (invalid) key.
+    Cancelled(Vec<KeyEvent>),
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize)]
+pub struct Keymap {
+    /// Always a Node
+    #[serde(flatten)]
+    root: KeyTrie,
+    /// Stores pending keys waiting for the next key
+    #[serde(skip)]
+    state: Vec<KeyEvent>,
+}
+
+impl Keymap {
+    pub fn new(root: KeyTrie) -> Self {
+        Keymap {
+            root,
+            state: Vec::new(),
+        }
+    }
+
+    pub fn root(&self) -> &KeyTrie {
+        &self.root
+    }
+
+    /// Returns list of keys waiting to be disambiguated.
+    pub fn pending(&self) -> &[KeyEvent] {
+        &self.state
+    }
+
+    /// Lookup `key` in the keymap to try and find a command to execute
+    pub fn get(&mut self, key: KeyEvent) -> KeymapResult {
+        let &first = self.state.get(0).unwrap_or(&key);
+        let trie = match self.root.search(&[first]) {
+            Some(&KeyTrie::Leaf(cmd)) => return KeymapResult::Matched(cmd),
+            None => return KeymapResult::NotFound,
+            Some(t) => t,
+        };
+        self.state.push(key);
+        match trie.search(&self.state[1..]) {
+            Some(&KeyTrie::Node(ref map)) => KeymapResult::Pending(map.clone()),
+            Some(&KeyTrie::Leaf(command)) => {
+                self.state.clear();
+                KeymapResult::Matched(command)
+            }
+            None => KeymapResult::Cancelled(self.state.drain(..).collect()),
+        }
+    }
+
+    pub fn merge(&mut self, other: Self) {
+        self.root.merge_nodes(other.root);
+    }
+}
+
+impl Deref for Keymap {
+    type Target = KeyTrieNode;
+
+    fn deref(&self) -> &Self::Target {
+        self.root.node().unwrap()
+    }
+}
+
+impl Default for Keymap {
+    fn default() -> Self {
+        Self::new(KeyTrie::Node(KeyTrieNode::default()))
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize)]
+#[serde(transparent)]
+pub struct Keymaps(pub HashMap<Mode, Keymap>);
+
+impl Keymaps {
+    /// Returns list of keys waiting to be disambiguated in current mode.
+    pub fn pending(&self) -> &[KeyEvent] {
+        self.0
+            .values()
+            .find_map(|keymap| match keymap.pending().is_empty() {
+                true => None,
+                false => Some(keymap.pending()),
+            })
+            .unwrap_or_default()
+    }
+}
+
+impl Deref for Keymaps {
+    type Target = HashMap<Mode, Keymap>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl DerefMut for Keymaps {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+
+impl Default for Keymaps {
+    fn default() -> Keymaps {
+        let normal = keymap!({ "Normal mode"
+            "h" | "left" => move_char_left,
+            "j" | "down" => move_line_down,
+            "k" | "up" => move_line_up,
+            "l" | "right" => move_char_right,
+
+            "t" => find_till_char,
+            "f" => find_next_char,
+            "T" => till_prev_char,
+            "F" => find_prev_char,
+            "r" => replace,
+            "R" => replace_with_yanked,
+
+            "~" => switch_case,
+            "`" => switch_to_lowercase,
+            "A-`" => switch_to_uppercase,
+
+            "home" => goto_line_start,
+            "end" => goto_line_end,
+
+            "w" => move_next_word_start,
+            "b" => move_prev_word_start,
+            "e" => move_next_word_end,
+
+            "W" => move_next_long_word_start,
+            "B" => move_prev_long_word_start,
+            "E" => move_next_long_word_end,
+
+            "v" => select_mode,
+            "G" => goto_line,
+            "g" => { "Goto"
+                "g" => goto_file_start,
+                "e" => goto_last_line,
+                "h" => goto_line_start,
+                "l" => goto_line_end,
+                "s" => goto_first_nonwhitespace,
+                "d" => goto_definition,
+                "y" => goto_type_definition,
+                "r" => goto_reference,
+                "i" => goto_implementation,
+                "t" => goto_window_top,
+                "m" => goto_window_middle,
+                "b" => goto_window_bottom,
+                "a" => goto_last_accessed_file,
+            },
+            ":" => command_mode,
+
+            "i" => insert_mode,
+            "I" => prepend_to_line,
+            "a" => append_mode,
+            "A" => append_to_line,
+            "o" => open_below,
+            "O" => open_above,
             // [<space>  ]<space> equivalents too (add blank new line, no edit)
 
-
-        key!('d') => commands::delete_selection,
+            "d" => delete_selection,
             // TODO: also delete without yanking
-        key!('c') => commands::change_selection,
+            "c" => change_selection,
             // TODO: also change delete without yanking
 
-        // key!('r') => commands::replace_with_char,
-
-        key!('s') => commands::select_regex,
-        alt!('s') => commands::split_selection_on_newline,
-        key!('S') => commands::split_selection,
-        key!(';') => commands::collapse_selection,
-        alt!(';') => commands::flip_selections,
-        key!('%') => commands::select_all,
-        key!('x') => commands::select_line,
-        key!('X') => commands::extend_line,
-        // or select mode X?
-        // extend_to_whole_line, crop_to_whole_line
+            "C" => copy_selection_on_next_line,
+            "A-C" => copy_selection_on_prev_line,
 
 
-        key!('m') => commands::match_brackets,
-        // TODO: refactor into
-        // key!('m') => commands::select_to_matching,
-        // key!('M') => commands::back_select_to_matching,
-        // select mode extend equivalents
+            "s" => select_regex,
+            "A-s" => split_selection_on_newline,
+            "S" => split_selection,
+            ";" => collapse_selection,
+            "A-;" => flip_selections,
+            "%" => select_all,
+            "x" => extend_line,
+            "X" => extend_to_line_bounds,
+            // crop_to_whole_line
 
-        // key!('.') => commands::repeat_insert,
-        // repeat_select
+            "m" => { "Match"
+                "m" => match_brackets,
+                "s" => surround_add,
+                "r" => surround_replace,
+                "d" => surround_delete,
+                "a" => select_textobject_around,
+                "i" => select_textobject_inner,
+            },
+            "[" => { "Left bracket"
+                "d" => goto_prev_diag,
+                "D" => goto_first_diag,
+            },
+            "]" => { "Right bracket"
+                "d" => goto_next_diag,
+                "D" => goto_last_diag,
+            },
 
-        // TODO: figure out what key to use
-        // key!('[') => commands::expand_selection, ??
-        key!('[') => commands::left_bracket_mode,
-        key!(']') => commands::right_bracket_mode,
-
-        key!('/') => commands::search,
+            "/" => search,
             // ? for search_reverse
-        key!('n') => commands::search_next,
-        key!('N') => commands::extend_search_next,
+            "n" => search_next,
+            "N" => extend_search_next,
             // N for search_prev
-        key!('*') => commands::search_selection,
+            "*" => search_selection,
 
-        key!('u') => commands::undo,
-        key!('U') => commands::redo,
+            "u" => undo,
+            "U" => redo,
 
-        key!('y') => commands::yank,
+            "y" => yank,
             // yank_all
-        key!('p') => commands::paste_after,
+            "p" => paste_after,
             // paste_all
-        key!('P') => commands::paste_before,
+            "P" => paste_before,
 
-        key!('>') => commands::indent,
-        key!('<') => commands::unindent,
-        key!('=') => commands::format_selections,
-        key!('J') => commands::join_selections,
+            ">" => indent,
+            "<" => unindent,
+            "=" => format_selections,
+            "J" => join_selections,
             // TODO: conflicts hover/doc
-        key!('K') => commands::keep_selections,
+            "K" => keep_selections,
             // TODO: and another method for inverse
 
             // TODO: clashes with space mode
-        key!(' ') => commands::keep_primary_selection,
+            "space" => keep_primary_selection,
 
-        // key!('q') => commands::record_macro,
-        // key!('Q') => commands::replay_macro,
+            // "q" => record_macro,
+            // "Q" => replay_macro,
 
-        // ~ / apostrophe => change case
             // & align selections
             // _ trim selections
 
-        // C / altC = copy (repeat) selections on prev/next lines
+            "(" => rotate_selections_backward,
+            ")" => rotate_selections_forward,
+            "A-(" => rotate_selection_contents_backward,
+            "A-)" => rotate_selection_contents_forward,
 
-        key!(Esc) => commands::normal_mode,
-        key!(PageUp) => commands::page_up,
-        key!(PageDown) => commands::page_down,
-        ctrl!('b') => commands::page_up,
-        ctrl!('f') => commands::page_down,
-        ctrl!('u') => commands::half_page_up,
-        ctrl!('d') => commands::half_page_down,
+            "esc" => normal_mode,
+            "C-b" | "pageup" => page_up,
+            "C-f" | "pagedown" => page_down,
+            "C-u" => half_page_up,
+            "C-d" => half_page_down,
 
-        ctrl!('w') => commands::window_mode,
+            "C-w" => { "Window"
+                "C-w" | "w" => rotate_view,
+                "C-h" | "h" => hsplit,
+                "C-v" | "v" => vsplit,
+                "C-q" | "q" => wclose,
+            },
 
             // move under <space>c
-        ctrl!('c') => commands::toggle_comments,
-        key!('K') => commands::hover,
+            "C-c" => toggle_comments,
+            "K" => hover,
 
             // z family for save/restore/combine from/to sels from register
 
-        // supposedly ctrl!('i') but did not work
-        key!(Tab) => commands::jump_forward,
-        ctrl!('o') => commands::jump_backward,
-        // ctrl!('s') => commands::save_selection,
+            "tab" => jump_forward, // tab == <C-i>
+            "C-o" => jump_backward,
+            // "C-s" => save_selection,
 
-        key!(' ') => commands::space_mode,
-        key!('z') => commands::view_mode,
+            "space" => { "Space"
+                "f" => file_picker,
+                "b" => buffer_picker,
+                "s" => symbol_picker,
+                "a" => code_action,
+                "'" => last_picker,
+                "w" => { "Window"
+                    "C-w" | "w" => rotate_view,
+                    "C-h" | "h" => hsplit,
+                    "C-v" | "v" => vsplit,
+                    "C-q" | "q" => wclose,
+                },
+                "y" => yank_joined_to_clipboard,
+                "Y" => yank_main_selection_to_clipboard,
+                "p" => paste_clipboard_after,
+                "P" => paste_clipboard_before,
+                "R" => replace_selections_with_clipboard,
+                "space" => keep_primary_selection,
+            },
+            "z" => { "View"
+                "z" | "c" => align_view_center,
+                "t" => align_view_top,
+                "b" => align_view_bottom,
+                "m" => align_view_middle,
+                "k" => scroll_up,
+                "j" => scroll_down,
+            },
 
-        key!('"') => commands::select_register,
-    );
-    // TODO: decide whether we want normal mode to also be select mode (kakoune-like), or whether
-    // we keep this separate select mode. More keys can fit into normal mode then, but it's weird
-    // because some selection operations can now be done from normal mode, some from select mode.
+            "\"" => select_register,
+            "C-z" => suspend,
+        });
         let mut select = normal.clone();
-    select.extend(
-        hashmap!(
-            key!('h') => commands::extend_char_left as Command,
-            key!('j') => commands::extend_line_down,
-            key!('k') => commands::extend_line_up,
-            key!('l') => commands::extend_char_right,
+        select.merge_nodes(keymap!({ "Select mode"
+            "h" | "left" => extend_char_left,
+            "j" | "down" => extend_line_down,
+            "k" | "up" => extend_line_up,
+            "l" | "right" => extend_char_right,
 
-            key!(Left) => commands::extend_char_left,
-            key!(Down) => commands::extend_line_down,
-            key!(Up) => commands::extend_line_up,
-            key!(Right) => commands::extend_char_right,
+            "w" => extend_next_word_start,
+            "b" => extend_prev_word_start,
+            "e" => extend_next_word_end,
 
-            key!('w') => commands::extend_next_word_start,
-            key!('b') => commands::extend_prev_word_start,
-            key!('e') => commands::extend_next_word_end,
+            "t" => extend_till_char,
+            "f" => extend_next_char,
+            "T" => extend_till_prev_char,
+            "F" => extend_prev_char,
 
-            key!('t') => commands::extend_till_char,
-            key!('f') => commands::extend_next_char,
+            "home" => goto_line_start,
+            "end" => goto_line_end,
+            "esc" => exit_select_mode,
+        }));
+        let insert = keymap!({ "Insert mode"
+            "esc" => normal_mode,
 
-            key!('T') => commands::extend_till_prev_char,
-            key!('F') => commands::extend_prev_char,
-            key!(Home) => commands::extend_line_start,
-            key!(End) => commands::extend_line_end,
-            key!(Esc) => commands::exit_select_mode,
+            "backspace" => delete_char_backward,
+            "del" => delete_char_forward,
+            "ret" => insert_newline,
+            "tab" => insert_tab,
+            "C-w" => delete_word_backward,
+
+            "left" => move_char_left,
+            "down" => move_line_down,
+            "up" => move_line_up,
+            "right" => move_char_right,
+            "pageup" => page_up,
+            "pagedown" => page_down,
+            "home" => goto_line_start,
+            "end" => goto_line_end_newline,
+
+            "C-x" => completion,
+        });
+        Keymaps(hashmap!(
+            Mode::Normal => Keymap::new(normal),
+            Mode::Select => Keymap::new(select),
+            Mode::Insert => Keymap::new(insert),
+        ))
+    }
+}
+
+/// Merge default config keys with user overwritten keys for custom user config.
+pub fn merge_keys(mut config: Config) -> Config {
+    let mut delta = std::mem::take(&mut config.keys);
+    for (mode, keys) in &mut *config.keys {
+        keys.merge(delta.remove(mode).unwrap_or_default())
+    }
+    config
+}
+
+#[test]
+fn merge_partial_keys() {
+    let config = Config {
+        keys: Keymaps(hashmap! {
+            Mode::Normal => Keymap::new(
+                keymap!({ "Normal mode"
+                    "i" => normal_mode,
+                    "无" => insert_mode,
+                    "z" => jump_backward,
+                    "g" => { "Merge into goto mode"
+                        "$" => goto_line_end,
+                        "g" => delete_char_forward,
+                    },
+                })
             )
-        .into_iter(),
+        }),
+        ..Default::default()
+    };
+    let mut merged_config = merge_keys(config.clone());
+    assert_ne!(config, merged_config);
+
+    let keymap = merged_config.keys.0.get_mut(&Mode::Normal).unwrap();
+    assert_eq!(
+        keymap.get(key!('i')),
+        KeymapResult::Matched(Command::normal_mode),
+        "Leaf should replace leaf"
+    );
+    assert_eq!(
+        keymap.get(key!('无')),
+        KeymapResult::Matched(Command::insert_mode),
+        "New leaf should be present in merged keymap"
+    );
+    // Assumes that z is a node in the default keymap
+    assert_eq!(
+        keymap.get(key!('z')),
+        KeymapResult::Matched(Command::jump_backward),
+        "Leaf should replace node"
+    );
+    // Assumes that `g` is a node in default keymap
+    assert_eq!(
+        keymap.root().search(&[key!('g'), key!('$')]).unwrap(),
+        &KeyTrie::Leaf(Command::goto_line_end),
+        "Leaf should be present in merged subnode"
+    );
+    // Assumes that `gg` is in default keymap
+    assert_eq!(
+        keymap.root().search(&[key!('g'), key!('g')]).unwrap(),
+        &KeyTrie::Leaf(Command::delete_char_forward),
+        "Leaf should replace old leaf in merged subnode"
+    );
+    // Assumes that `ge` is in default keymap
+    assert_eq!(
+        keymap.root().search(&[key!('g'), key!('e')]).unwrap(),
+        &KeyTrie::Leaf(Command::goto_last_line),
+        "Old leaves in subnode should be present in merged node"
     );
 
-    hashmap!(
-        // as long as you cast the first item, rust is able to infer the other cases
-        // TODO: select could be normal mode with some bindings merged over
-        Mode::Normal => normal,
-        Mode::Select => select,
-        Mode::Insert => hashmap!(
-            key!(Esc) => commands::normal_mode as Command,
-            key!(Backspace) => commands::insert::delete_char_backward,
-            key!(Delete) => commands::insert::delete_char_forward,
-            key!(Enter) => commands::insert::insert_newline,
-            key!(Tab) => commands::insert::insert_tab,
-
-            ctrl!('x') => commands::completion,
-            ctrl!('w') => commands::insert::delete_word_backward,
-        ),
-    )
+    assert!(merged_config.keys.0.get(&Mode::Normal).unwrap().len() > 1);
+    assert!(merged_config.keys.0.get(&Mode::Insert).unwrap().len() > 0);
 }

helix-term/src/lib.rs

@@ -1,8 +1,11 @@
-#![allow(unused)]
+#[macro_use]
+extern crate helix_view;
 
 pub mod application;
 pub mod args;
 pub mod commands;
 pub mod compositor;
+pub mod config;
+pub mod job;
 pub mod keymap;
 pub mod ui;

helix-term/src/main.rs

@@ -1,18 +1,13 @@
+use anyhow::{Context, Error, Result};
 use helix_term::application::Application;
 use helix_term::args::Args;
-
+use helix_term::config::Config;
+use helix_term::keymap::merge_keys;
 use std::path::PathBuf;
 
-use anyhow::{Context, Result};
-
 fn setup_logging(logpath: PathBuf, verbosity: u64) -> Result<()> {
     let mut base_config = fern::Dispatch::new();
 
-    // Let's say we depend on something which whose "info" level messages are too
-    // verbose to include in end-user output. If we don't need them,
-    // let's not include them.
-    // .level_for("overly-verbose-target", log::LevelFilter::Warn)
-
     base_config = match verbosity {
         0 => base_config.level(log::LevelFilter::Warn),
         1 => base_config.level(log::LevelFilter::Info),
@@ -89,10 +84,16 @@ FLAGS:
         std::fs::create_dir_all(&conf_dir).ok();
     }
 
+    let config = match std::fs::read_to_string(conf_dir.join("config.toml")) {
+        Ok(config) => merge_keys(toml::from_str(&config)?),
+        Err(err) if err.kind() == std::io::ErrorKind::NotFound => Config::default(),
+        Err(err) => return Err(Error::new(err)),
+    };
+
     setup_logging(logpath, args.verbosity).context("failed to initialize logging")?;
 
     // TODO: use the thread local executor to spawn the application task separately from the work pool
-    let mut app = Application::new(args).context("unable to create new appliction")?;
+    let mut app = Application::new(args, config).context("unable to create new application")?;
     app.run().await.unwrap();
 
     Ok(())

helix-term/src/ui/completion.rs

@@ -1,23 +1,23 @@
-use crate::compositor::{Component, Compositor, Context, EventResult};
-use crossterm::event::{Event, KeyCode, KeyEvent, KeyModifiers};
-use tui::{
-    buffer::Buffer as Surface,
-    layout::Rect,
-    style::{Color, Style},
-};
+use crate::compositor::{Component, Context, EventResult};
+use crossterm::event::{Event, KeyCode, KeyEvent};
+use tui::buffer::Buffer as Surface;
 
 use std::borrow::Cow;
 
-use helix_core::{Position, Transaction};
-use helix_view::Editor;
+use helix_core::Transaction;
+use helix_view::{graphics::Rect, Document, Editor, View};
 
 use crate::commands;
 use crate::ui::{menu, Markdown, Menu, Popup, PromptEvent};
 
-use helix_lsp::lsp;
+use helix_lsp::{lsp, util};
 use lsp::CompletionItem;
 
 impl menu::Item for CompletionItem {
+    fn sort_text(&self) -> &str {
+        self.filter_text.as_ref().unwrap_or(&self.label).as_str()
+    }
+
     fn filter_text(&self) -> &str {
         self.filter_text.as_ref().unwrap_or(&self.label).as_str()
     }
@@ -68,7 +68,7 @@ impl menu::Item for CompletionItem {
 
 /// Wraps a Menu.
 pub struct Completion {
-    popup: Popup<Menu<CompletionItem>>, // TODO: Popup<Menu> need to be able to access contents.
+    popup: Popup<Menu<CompletionItem>>,
     trigger_offset: usize,
     // TODO: maintain a completioncontext with trigger kind & trigger char
 }
@@ -80,47 +80,14 @@ impl Completion {
         trigger_offset: usize,
     ) -> Self {
         // let items: Vec<CompletionItem> = Vec::new();
-        let mut menu = Menu::new(items, move |editor: &mut Editor, item, event| {
-            match event {
-                PromptEvent::Abort => {
-                    // revert state
-                    // let id = editor.view().doc;
-                    // let doc = &mut editor.documents[id];
-                    // doc.state = snapshot.clone();
-                }
-                PromptEvent::Validate => {
-                    let (view, doc) = editor.current();
-
-                    // revert state to what it was before the last update
-                    // doc.state = snapshot.clone();
-
-                    // extract as fn(doc, item):
-
-                    // TODO: need to apply without composing state...
-                    // TODO: need to update lsp on accept/cancel by diffing the snapshot with
-                    // the final state?
-                    // -> on update simply update the snapshot, then on accept redo the call,
-                    // finally updating doc.changes + notifying lsp.
-                    //
-                    // or we could simply use doc.undo + apply when changing between options
-
-                    // always present here
-                    let item = item.unwrap();
-
-                    use helix_lsp::{lsp, util};
-
-                    // if more text was entered, remove it
-                    let cursor = doc.selection(view.id).cursor();
-                    if trigger_offset < cursor {
-                        let remove = Transaction::change(
-                            doc.text(),
-                            vec![(trigger_offset, cursor, None)].into_iter(),
-                        );
-                        doc.apply(&remove, view.id);
-                    }
-
-                    use helix_lsp::OffsetEncoding;
-                    let transaction = if let Some(edit) = &item.text_edit {
+        let menu = Menu::new(items, move |editor: &mut Editor, item, event| {
+            fn item_to_transaction(
+                doc: &Document,
+                view: &View,
+                item: &CompletionItem,
+                offset_encoding: helix_lsp::OffsetEncoding,
+            ) -> Transaction {
+                if let Some(edit) = &item.text_edit {
                     let edit = match edit {
                         lsp::CompletionTextEdit::Edit(edit) => edit.clone(),
                         lsp::CompletionTextEdit::InsertAndReplace(item) => {
@@ -134,16 +101,65 @@ impl Completion {
                     )
                 } else {
                     let text = item.insert_text.as_ref().unwrap_or(&item.label);
-                        let cursor = doc.selection(view.id).cursor();
+                    let cursor = doc
+                        .selection(view.id)
+                        .primary()
+                        .cursor(doc.text().slice(..));
                     Transaction::change(
                         doc.text(),
                         vec![(cursor, cursor, Some(text.as_str().into()))].into_iter(),
                     )
-                    };
+                }
+            }
 
+            match event {
+                PromptEvent::Abort => {}
+                PromptEvent::Update => {
+                    let (view, doc) = current!(editor);
+
+                    // always present here
+                    let item = item.unwrap();
+
+                    // if more text was entered, remove it
+                    // TODO: ideally to undo we should keep the last completion tx revert, and map it over new changes
+                    let cursor = doc
+                        .selection(view.id)
+                        .primary()
+                        .cursor(doc.text().slice(..));
+                    if trigger_offset < cursor {
+                        let remove = Transaction::change(
+                            doc.text(),
+                            vec![(trigger_offset, cursor, None)].into_iter(),
+                        );
+                        doc.apply(&remove, view.id);
+                    }
+
+                    let transaction = item_to_transaction(doc, view, item, offset_encoding);
+                    doc.apply(&transaction, view.id);
+                }
+                PromptEvent::Validate => {
+                    let (view, doc) = current!(editor);
+
+                    // always present here
+                    let item = item.unwrap();
+
+                    // if more text was entered, remove it
+                    // TODO: ideally to undo we should keep the last completion tx revert, and map it over new changes
+                    let cursor = doc
+                        .selection(view.id)
+                        .primary()
+                        .cursor(doc.text().slice(..));
+                    if trigger_offset < cursor {
+                        let remove = Transaction::change(
+                            doc.text(),
+                            vec![(trigger_offset, cursor, None)].into_iter(),
+                        );
+                        doc.apply(&remove, view.id);
+                    }
+
+                    let transaction = item_to_transaction(doc, view, item, offset_encoding);
                     doc.apply(&transaction, view.id);
 
-                    // TODO: merge edit with additional_text_edits
                     if let Some(additional_edits) = &item.additional_text_edits {
                         // gopls uses this to add extra imports
                         if !additional_edits.is_empty() {
@@ -156,7 +172,6 @@ impl Completion {
                         }
                     }
                 }
-                _ => (),
             };
         });
         let popup = Popup::new(menu);
@@ -169,7 +184,7 @@ impl Completion {
     pub fn update(&mut self, cx: &mut commands::Context) {
         // recompute menu based on matches
         let menu = self.popup.contents_mut();
-        let (view, doc) = cx.editor.current();
+        let (view, doc) = current!(cx.editor);
 
         // cx.hooks()
         // cx.add_hook(enum type,  ||)
@@ -181,7 +196,10 @@ impl Completion {
         // TODO: hooks should get processed immediately so maybe do it after select!(), before
         // looping?
 
-        let cursor = doc.selection(view.id).cursor();
+        let cursor = doc
+            .selection(view.id)
+            .primary()
+            .cursor(doc.text().slice(..));
         if self.trigger_offset <= cursor {
             let fragment = doc.text().slice(self.trigger_offset..cursor);
             let text = Cow::from(fragment);
@@ -223,65 +241,89 @@ impl Component for Completion {
         self.popup.required_size(viewport)
     }
 
-    fn render(&self, area: Rect, surface: &mut Surface, cx: &mut Context) {
+    fn render(&mut self, area: Rect, surface: &mut Surface, cx: &mut Context) {
         self.popup.render(area, surface, cx);
 
-        // TODO: if we have a selection, render a markdown popup on top/below with info
+        // if we have a selection, render a markdown popup on top/below with info
         if let Some(option) = self.popup.contents().selection() {
             // need to render:
             // option.detail
             // ---
             // option.documentation
 
-            let (view, doc) = cx.editor.current();
+            let (view, doc) = current!(cx.editor);
             let language = doc
                 .language()
                 .and_then(|scope| scope.strip_prefix("source."))
                 .unwrap_or("");
+            let cursor_pos = doc
+                .selection(view.id)
+                .primary()
+                .cursor(doc.text().slice(..));
+            let cursor_pos = (helix_core::coords_at_pos(doc.text().slice(..), cursor_pos).row
+                - view.first_line) as u16;
 
-            let doc = match &option.documentation {
+            let mut doc = match &option.documentation {
                 Some(lsp::Documentation::String(contents))
                 | Some(lsp::Documentation::MarkupContent(lsp::MarkupContent {
                     kind: lsp::MarkupKind::PlainText,
                     value: contents,
                 })) => {
                     // TODO: convert to wrapped text
-                    Markdown::new(format!(
+                    Markdown::new(
+                        format!(
                             "```{}\n{}\n```\n{}",
                             language,
                             option.detail.as_deref().unwrap_or_default(),
                             contents.clone()
-                    ))
+                        ),
+                        cx.editor.syn_loader.clone(),
+                    )
                 }
                 Some(lsp::Documentation::MarkupContent(lsp::MarkupContent {
                     kind: lsp::MarkupKind::Markdown,
                     value: contents,
                 })) => {
                     // TODO: set language based on doc scope
-                    Markdown::new(format!(
+                    Markdown::new(
+                        format!(
                             "```{}\n{}\n```\n{}",
                             language,
                             option.detail.as_deref().unwrap_or_default(),
                             contents.clone()
-                    ))
+                        ),
+                        cx.editor.syn_loader.clone(),
+                    )
                 }
                 None if option.detail.is_some() => {
                     // TODO: copied from above
 
                     // TODO: set language based on doc scope
-                    Markdown::new(format!(
+                    Markdown::new(
+                        format!(
                             "```{}\n{}\n```",
                             language,
                             option.detail.as_deref().unwrap_or_default(),
-                    ))
+                        ),
+                        cx.editor.syn_loader.clone(),
+                    )
                 }
                 None => return,
             };
 
             let half = area.height / 2;
             let height = 15.min(half);
+            // we want to make sure the cursor is visible (not hidden behind the documentation)
+            let y = if cursor_pos + view.area.y
+                >= (cx.editor.tree.area().height - height - 2/* statusline + commandline */)
+            {
+                0
+            } else {
                 // -2 to subtract command line + statusline. a bit of a hack, because of splits.
-            let area = Rect::new(0, area.height - height - 2, area.width, height);
+                area.height.saturating_sub(height).saturating_sub(2)
+            };
+
+            let area = Rect::new(0, y, area.width, height);
 
             // clear area
             let background = cx.editor.theme.get("ui.popup");

helix-term/src/ui/info.rs

@@ -0,0 +1,38 @@
+use crate::compositor::{Component, Context};
+use helix_view::graphics::Rect;
+use helix_view::info::Info;
+use tui::buffer::Buffer as Surface;
+use tui::widgets::{Block, Borders, Widget};
+
+impl Component for Info {
+    fn render(&mut self, viewport: Rect, surface: &mut Surface, cx: &mut Context) {
+        let style = cx.editor.theme.get("ui.popup");
+
+        // Calculate the area of the terminal to modify. Because we want to
+        // render at the bottom right, we use the viewport's width and height
+        // which evaluate to the most bottom right coordinate.
+        let (width, height) = (self.width + 2, self.height + 2);
+        let area = viewport.intersection(Rect::new(
+            viewport.width.saturating_sub(width),
+            viewport.height.saturating_sub(height + 2),
+            width,
+            height,
+        ));
+        surface.clear_with(area, style);
+
+        let block = Block::default()
+            .title(self.title.as_str())
+            .borders(Borders::ALL)
+            .border_style(style);
+        let inner = block.inner(area);
+        block.render(area, surface);
+
+        // Only write as many lines as there are rows available.
+        for (y, line) in (inner.y..)
+            .zip(self.text.lines())
+            .take(inner.height as usize)
+        {
+            surface.set_string(inner.x, y, line, style);
+        }
+    }
+}

helix-term/src/ui/markdown.rs

@@ -1,36 +1,47 @@
-use crate::compositor::{Component, Compositor, Context, EventResult};
-use crossterm::event::{Event, KeyCode, KeyEvent, KeyModifiers};
+use crate::compositor::{Component, Context};
 use tui::{
     buffer::Buffer as Surface,
-    layout::Rect,
-    style::{Color, Style},
-    text::Text,
+    text::{Span, Spans, Text},
 };
 
-use std::borrow::Cow;
+use std::sync::Arc;
 
-use helix_core::Position;
-use helix_view::{Editor, Theme};
+use pulldown_cmark::{CodeBlockKind, CowStr, Event, Options, Parser, Tag};
+
+use helix_core::{
+    syntax::{self, HighlightEvent, Syntax},
+    Rope,
+};
+use helix_view::{
+    graphics::{Color, Rect, Style},
+    Theme,
+};
 
 pub struct Markdown {
     contents: String,
+
+    config_loader: Arc<syntax::Loader>,
 }
 
 // TODO: pre-render and self reference via Pin
 // better yet, just use Tendril + subtendril for references
 
 impl Markdown {
-    pub fn new(contents: String) -> Self {
-        Self { contents }
+    pub fn new(contents: String, config_loader: Arc<syntax::Loader>) -> Self {
+        Self {
+            contents,
+            config_loader,
+        }
     }
 }
 
-fn parse<'a>(contents: &'a str, theme: Option<&Theme>) -> tui::text::Text<'a> {
-    use pulldown_cmark::{CodeBlockKind, CowStr, Event, Options, Parser, Tag};
-    use tui::text::{Span, Spans, Text};
-
-    // also 2021-03-04T16:33:58.553 helix_lsp::transport [INFO] <- {"contents":{"kind":"markdown","value":"\n```rust\ncore::num\n```\n\n```rust\npub const fn saturating_sub(self, rhs:Self) ->Self\n```\n\n---\n\n```rust\n```"},"range":{"end":{"character":61,"line":101},"start":{"character":47,"line":101}}}
-    let text = "\n```rust\ncore::iter::traits::iterator::Iterator\n```\n\n```rust\nfn collect<B: FromIterator<Self::Item>>(self) -> B\nwhere\n        Self: Sized,\n```\n\n---\n\nTransforms an iterator into a collection.\n\n`collect()` can take anything iterable, and turn it into a relevant\ncollection. This is one of the more powerful methods in the standard\nlibrary, used in a variety of contexts.\n\nThe most basic pattern in which `collect()` is used is to turn one\ncollection into another. You take a collection, call [`iter`](https://doc.rust-lang.org/nightly/core/iter/traits/iterator/trait.Iterator.html) on it,\ndo a bunch of transformations, and then `collect()` at the end.\n\n`collect()` can also create instances of types that are not typical\ncollections. For example, a [`String`](https://doc.rust-lang.org/nightly/core/iter/std/string/struct.String.html) can be built from [`char`](type@char)s,\nand an iterator of [`Result<T, E>`](https://doc.rust-lang.org/nightly/core/result/enum.Result.html) items can be collected\ninto `Result<Collection<T>, E>`. See the examples below for more.\n\nBecause `collect()` is so general, it can cause problems with type\ninference. As such, `collect()` is one of the few times you'll see\nthe syntax affectionately known as the 'turbofish': `::<>`. This\nhelps the inference algorithm understand specifically which collection\nyou're trying to collect into.\n\n# Examples\n\nBasic usage:\n\n```rust\nlet a = [1, 2, 3];\n\nlet doubled: Vec<i32> = a.iter()\n                         .map(|&x| x * 2)\n                         .collect();\n\nassert_eq!(vec![2, 4, 6], doubled);\n```\n\nNote that we needed the `: Vec<i32>` on the left-hand side. This is because\nwe could collect into, for example, a [`VecDeque<T>`](https://doc.rust-lang.org/nightly/core/iter/std/collections/struct.VecDeque.html) instead:\n\n```rust\nuse std::collections::VecDeque;\n\nlet a = [1, 2, 3];\n\nlet doubled: VecDeque<i32> = a.iter().map(|&x| x * 2).collect();\n\nassert_eq!(2, doubled[0]);\nassert_eq!(4, doubled[1]);\nassert_eq!(6, doubled[2]);\n```\n\nUsing the 'turbofish' instead of annotating `doubled`:\n\n```rust\nlet a = [1, 2, 3];\n\nlet doubled = a.iter().map(|x| x * 2).collect::<Vec<i32>>();\n\nassert_eq!(vec![2, 4, 6], doubled);\n```\n\nBecause `collect()` only cares about what you're collecting into, you can\nstill use a partial type hint, `_`, with the turbofish:\n\n```rust\nlet a = [1, 2, 3];\n\nlet doubled = a.iter().map(|x| x * 2).collect::<Vec<_>>();\n\nassert_eq!(vec![2, 4, 6], doubled);\n```\n\nUsing `collect()` to make a [`String`](https://doc.rust-lang.org/nightly/core/iter/std/string/struct.String.html):\n\n```rust\nlet chars = ['g', 'd', 'k', 'k', 'n'];\n\nlet hello: String = chars.iter()\n    .map(|&x| x as u8)\n    .map(|x| (x + 1) as char)\n    .collect();\n\nassert_eq!(\"hello\", hello);\n```\n\nIf you have a list of [`Result<T, E>`](https://doc.rust-lang.org/nightly/core/result/enum.Result.html)s, you can use `collect()` to\nsee if any of them failed:\n\n```rust\nlet results = [Ok(1), Err(\"nope\"), Ok(3), Err(\"bad\")];\n\nlet result: Result<Vec<_>, &str> = results.iter().cloned().collect();\n\n// gives us the first error\nassert_eq!(Err(\"nope\"), result);\n\nlet results = [Ok(1), Ok(3)];\n\nlet result: Result<Vec<_>, &str> = results.iter().cloned().collect();\n\n// gives us the list of answers\nassert_eq!(Ok(vec![1, 3]), result);\n```";
+fn parse<'a>(
+    contents: &'a str,
+    theme: Option<&Theme>,
+    loader: &syntax::Loader,
+) -> tui::text::Text<'a> {
+    // // also 2021-03-04T16:33:58.553 helix_lsp::transport [INFO] <- {"contents":{"kind":"markdown","value":"\n```rust\ncore::num\n```\n\n```rust\npub const fn saturating_sub(self, rhs:Self) ->Self\n```\n\n---\n\n```rust\n```"},"range":{"end":{"character":61,"line":101},"start":{"character":47,"line":101}}}
+    // let text = "\n```rust\ncore::iter::traits::iterator::Iterator\n```\n\n```rust\nfn collect<B: FromIterator<Self::Item>>(self) -> B\nwhere\n        Self: Sized,\n```\n\n---\n\nTransforms an iterator into a collection.\n\n`collect()` can take anything iterable, and turn it into a relevant\ncollection. This is one of the more powerful methods in the standard\nlibrary, used in a variety of contexts.\n\nThe most basic pattern in which `collect()` is used is to turn one\ncollection into another. You take a collection, call [`iter`](https://doc.rust-lang.org/nightly/core/iter/traits/iterator/trait.Iterator.html) on it,\ndo a bunch of transformations, and then `collect()` at the end.\n\n`collect()` can also create instances of types that are not typical\ncollections. For example, a [`String`](https://doc.rust-lang.org/nightly/core/iter/std/string/struct.String.html) can be built from [`char`](type@char)s,\nand an iterator of [`Result<T, E>`](https://doc.rust-lang.org/nightly/core/result/enum.Result.html) items can be collected\ninto `Result<Collection<T>, E>`. See the examples below for more.\n\nBecause `collect()` is so general, it can cause problems with type\ninference. As such, `collect()` is one of the few times you'll see\nthe syntax affectionately known as the 'turbofish': `::<>`. This\nhelps the inference algorithm understand specifically which collection\nyou're trying to collect into.\n\n# Examples\n\nBasic usage:\n\n```rust\nlet a = [1, 2, 3];\n\nlet doubled: Vec<i32> = a.iter()\n                         .map(|&x| x * 2)\n                         .collect();\n\nassert_eq!(vec![2, 4, 6], doubled);\n```\n\nNote that we needed the `: Vec<i32>` on the left-hand side. This is because\nwe could collect into, for example, a [`VecDeque<T>`](https://doc.rust-lang.org/nightly/core/iter/std/collections/struct.VecDeque.html) instead:\n\n```rust\nuse std::collections::VecDeque;\n\nlet a = [1, 2, 3];\n\nlet doubled: VecDeque<i32> = a.iter().map(|&x| x * 2).collect();\n\nassert_eq!(2, doubled[0]);\nassert_eq!(4, doubled[1]);\nassert_eq!(6, doubled[2]);\n```\n\nUsing the 'turbofish' instead of annotating `doubled`:\n\n```rust\nlet a = [1, 2, 3];\n\nlet doubled = a.iter().map(|x| x * 2).collect::<Vec<i32>>();\n\nassert_eq!(vec![2, 4, 6], doubled);\n```\n\nBecause `collect()` only cares about what you're collecting into, you can\nstill use a partial type hint, `_`, with the turbofish:\n\n```rust\nlet a = [1, 2, 3];\n\nlet doubled = a.iter().map(|x| x * 2).collect::<Vec<_>>();\n\nassert_eq!(vec![2, 4, 6], doubled);\n```\n\nUsing `collect()` to make a [`String`](https://doc.rust-lang.org/nightly/core/iter/std/string/struct.String.html):\n\n```rust\nlet chars = ['g', 'd', 'k', 'k', 'n'];\n\nlet hello: String = chars.iter()\n    .map(|&x| x as u8)\n    .map(|x| (x + 1) as char)\n    .collect();\n\nassert_eq!(\"hello\", hello);\n```\n\nIf you have a list of [`Result<T, E>`](https://doc.rust-lang.org/nightly/core/result/enum.Result.html)s, you can use `collect()` to\nsee if any of them failed:\n\n```rust\nlet results = [Ok(1), Err(\"nope\"), Ok(3), Err(\"bad\")];\n\nlet result: Result<Vec<_>, &str> = results.iter().cloned().collect();\n\n// gives us the first error\nassert_eq!(Err(\"nope\"), result);\n\nlet results = [Ok(1), Ok(3)];\n\nlet result: Result<Vec<_>, &str> = results.iter().cloned().collect();\n\n// gives us the list of answers\nassert_eq!(Ok(vec![1, 3]), result);\n```";
 
     let mut options = Options::empty();
     options.insert(Options::ENABLE_STRIKETHROUGH);
@@ -75,18 +86,13 @@ fn parse<'a>(contents: &'a str, theme: Option<&Theme>) -> tui::text::Text<'a> {
                 // TODO: temp workaround
                 if let Some(Tag::CodeBlock(CodeBlockKind::Fenced(language))) = tags.last() {
                     if let Some(theme) = theme {
-                        use helix_core::syntax::{self, HighlightEvent, Syntax};
-                        use helix_core::Rope;
-
                         let rope = Rope::from(text.as_ref());
-                        let syntax = syntax::LOADER
-                            .get()
-                            .unwrap()
+                        let syntax = loader
                             .language_config_for_scope(&format!("source.{}", language))
                             .and_then(|config| config.highlight_config(theme.scopes()))
                             .map(|config| Syntax::new(&rope, config));
 
-                        if let Some(mut syntax) = syntax {
+                        if let Some(syntax) = syntax {
                             // if we have a syntax available, highlight_iter and generate spans
                             let mut highlights = Vec::new();
 
@@ -101,15 +107,15 @@ fn parse<'a>(contents: &'a str, theme: Option<&Theme>) -> tui::text::Text<'a> {
                                     }
                                     HighlightEvent::Source { start, end } => {
                                         let style = match highlights.first() {
-                                            Some(span) => {
-                                                theme.get(theme.scopes()[span.0].as_str())
-                                            }
+                                            Some(span) => theme.get(&theme.scopes()[span.0]),
                                             None => text_style,
                                         };
 
                                         // TODO: replace tabs with indentation
 
                                         let mut slice = &text[start..end];
+                                        // TODO: do we need to handle all unicode line endings
+                                        // here, or is just '\n' okay?
                                         while let Some(end) = slice.find('\n') {
                                             // emit span up to newline
                                             let text = &slice[..end];
@@ -136,13 +142,13 @@ fn parse<'a>(contents: &'a str, theme: Option<&Theme>) -> tui::text::Text<'a> {
                             }
                         } else {
                             for line in text.lines() {
-                                let mut span = Span::styled(line.to_string(), code_style);
+                                let span = Span::styled(line.to_string(), code_style);
                                 lines.push(Spans::from(span));
                             }
                         }
                     } else {
                         for line in text.lines() {
-                            let mut span = Span::styled(line.to_string(), code_style);
+                            let span = Span::styled(line.to_string(), code_style);
                             lines.push(Spans::from(span));
                         }
                     }
@@ -157,7 +163,6 @@ fn parse<'a>(contents: &'a str, theme: Option<&Theme>) -> tui::text::Text<'a> {
                 }
             }
             Event::Code(text) | Event::Html(text) => {
-                log::warn!("code {:?}", text);
                 let mut span = to_span(text);
                 span.style = code_style;
                 spans.push(span);
@@ -193,10 +198,10 @@ fn parse<'a>(contents: &'a str, theme: Option<&Theme>) -> tui::text::Text<'a> {
     Text::from(lines)
 }
 impl Component for Markdown {
-    fn render(&self, area: Rect, surface: &mut Surface, cx: &mut Context) {
+    fn render(&mut self, area: Rect, surface: &mut Surface, cx: &mut Context) {
         use tui::widgets::{Paragraph, Widget, Wrap};
 
-        let text = parse(&self.contents, Some(&cx.editor.theme));
+        let text = parse(&self.contents, Some(&cx.editor.theme), &self.config_loader);
 
         let par = Paragraph::new(text)
             .wrap(Wrap { trim: false })
@@ -207,7 +212,7 @@ impl Component for Markdown {
     }
 
     fn required_size(&mut self, viewport: (u16, u16)) -> Option<(u16, u16)> {
-        let contents = parse(&self.contents, None);
+        let contents = parse(&self.contents, None, &self.config_loader);
         let padding = 2;
         let width = std::cmp::min(contents.width() as u16 + padding, viewport.0);
         let height = std::cmp::min(contents.height() as u16 + padding, viewport.1);

helix-term/src/ui/menu.rs

@@ -1,24 +1,17 @@
 use crate::compositor::{Component, Compositor, Context, EventResult};
 use crossterm::event::{Event, KeyCode, KeyEvent, KeyModifiers};
-use tui::{
-    buffer::Buffer as Surface,
-    layout::Rect,
-    style::{Color, Style},
-    widgets::Table,
-};
+use tui::{buffer::Buffer as Surface, widgets::Table};
 
 pub use tui::widgets::{Cell, Row};
 
-use std::borrow::Cow;
-
 use fuzzy_matcher::skim::SkimMatcherV2 as Matcher;
 use fuzzy_matcher::FuzzyMatcher;
 
-use helix_core::Position;
-use helix_view::Editor;
+use helix_view::{graphics::Rect, Editor};
+use tui::layout::Constraint;
 
 pub trait Item {
-    // TODO: sort_text
+    fn sort_text(&self) -> &str;
     fn filter_text(&self) -> &str;
 
     fn label(&self) -> &str;
@@ -34,6 +27,8 @@ pub struct Menu<T: Item> {
     /// (index, score)
     matches: Vec<(usize, i64)>,
 
+    widths: Vec<Constraint>,
+
     callback_fn: Box<dyn Fn(&mut Editor, Option<&T>, MenuEvent)>,
 
     scroll: usize,
@@ -52,6 +47,7 @@ impl<T: Item> Menu<T> {
             matcher: Box::new(Matcher::default()),
             matches: Vec::new(),
             cursor: None,
+            widths: Vec::new(),
             callback_fn: Box::new(callback_fn),
             scroll: 0,
             size: (0, 0),
@@ -66,27 +62,23 @@ impl<T: Item> Menu<T> {
     pub fn score(&mut self, pattern: &str) {
         // need to borrow via pattern match otherwise it complains about simultaneous borrow
         let Self {
-            ref mut options,
             ref mut matcher,
             ref mut matches,
+            ref options,
             ..
         } = *self;
 
         // reuse the matches allocation
         matches.clear();
-        matches.extend(
-            self.options
-                .iter()
-                .enumerate()
-                .filter_map(|(index, option)| {
+        matches.extend(options.iter().enumerate().filter_map(|(index, option)| {
             let text = option.filter_text();
             // TODO: using fuzzy_indices could give us the char idx for match highlighting
             matcher
-                        .fuzzy_match(&text, pattern)
+                .fuzzy_match(text, pattern)
                 .map(|score| (index, score))
-                }),
-        );
-        matches.sort_unstable_by_key(|(_, score)| -score);
+        }));
+        // matches.sort_unstable_by_key(|(_, score)| -score);
+        matches.sort_unstable_by_key(|(index, _score)| options[*index].sort_text());
 
         // reset cursor position
         self.cursor = None;
@@ -94,14 +86,15 @@ impl<T: Item> Menu<T> {
     }
 
     pub fn move_up(&mut self) {
-        // TODO: wrap around to end
-        let pos = self.cursor.map_or(0, |i| i.saturating_sub(1)) % self.options.len();
+        let len = self.matches.len();
+        let pos = self.cursor.map_or(0, |i| (i + len.saturating_sub(1)) % len) % len;
         self.cursor = Some(pos);
         self.adjust_scroll();
     }
 
     pub fn move_down(&mut self) {
-        let pos = self.cursor.map_or(0, |i| i + 1) % self.options.len();
+        let len = self.matches.len();
+        let pos = self.cursor.map_or(0, |i| i + 1) % len;
         self.cursor = Some(pos);
         self.adjust_scroll();
     }
@@ -228,11 +221,32 @@ impl<T: Item + 'static> Component for Menu<T> {
     }
 
     fn required_size(&mut self, viewport: (u16, u16)) -> Option<(u16, u16)> {
-        let width = std::cmp::min(30, viewport.0);
+        let n = self
+            .options
+            .first()
+            .map(|option| option.row().cells.len())
+            .unwrap_or_default();
+        let max_lens = self.options.iter().fold(vec![0; n], |mut acc, option| {
+            let row = option.row();
+            // maintain max for each column
+            for (acc, cell) in acc.iter_mut().zip(row.cells.iter()) {
+                let width = cell.content.width();
+                if width > *acc {
+                    *acc = width;
+                }
+            }
 
-        const MAX: usize = 10;
-        let height = std::cmp::min(self.options.len(), MAX);
-        let height = std::cmp::min(height, viewport.1 as usize);
+            acc
+        });
+        let len = max_lens.iter().sum::<usize>() + n + 1; // +1: reserve some space for scrollbar
+        let width = len.min(viewport.0 as usize);
+
+        self.widths = max_lens
+            .into_iter()
+            .map(|len| Constraint::Length(len as u16))
+            .collect();
+
+        let height = self.options.len().min(10).min(viewport.1 as usize);
 
         self.size = (width as u16, height as u16);
 
@@ -244,7 +258,7 @@ impl<T: Item + 'static> Component for Menu<T> {
 
     // TODO: required size should re-trigger when we filter items so we can draw a smaller menu
 
-    fn render(&self, area: Rect, surface: &mut Surface, cx: &mut Context) {
+    fn render(&mut self, area: Rect, surface: &mut Surface, cx: &mut Context) {
         let style = cx.editor.theme.get("ui.text");
         let selected = cx.editor.theme.get("ui.menu.selected");
 
@@ -272,13 +286,12 @@ impl<T: Item + 'static> Component for Menu<T> {
         let scroll_line = (win_height - scroll_height) * scroll
             / std::cmp::max(1, len.saturating_sub(win_height));
 
-        use tui::layout::Constraint;
         let rows = options.iter().map(|option| option.row());
         let table = Table::new(rows)
             .style(style)
             .highlight_style(selected)
             .column_spacing(1)
-            .widths(&[Constraint::Percentage(50), Constraint::Percentage(50)]);
+            .widths(&self.widths);
 
         use tui::widgets::TableState;
 
@@ -299,7 +312,7 @@ impl<T: Item + 'static> Component for Menu<T> {
         //     )
         // }
 
-        for (i, option) in (scroll..(scroll + win_height).min(len)).enumerate() {
+        for (i, _) in (scroll..(scroll + win_height).min(len)).enumerate() {
             let is_marked = i >= scroll_line && i < scroll_line + scroll_height;
 
             if is_marked {

helix-term/src/ui/mod.rs

@@ -1,45 +1,47 @@
 mod completion;
-mod editor;
+pub(crate) mod editor;
+mod info;
 mod markdown;
 mod menu;
 mod picker;
 mod popup;
 mod prompt;
+mod spinner;
 mod text;
 
 pub use completion::Completion;
 pub use editor::EditorView;
 pub use markdown::Markdown;
 pub use menu::Menu;
-pub use picker::Picker;
+pub use picker::{FilePicker, Picker};
 pub use popup::Popup;
 pub use prompt::{Prompt, PromptEvent};
+pub use spinner::{ProgressSpinners, Spinner};
 pub use text::Text;
 
-pub use tui::layout::Rect;
-pub use tui::style::{Color, Modifier, Style};
-
 use helix_core::regex::Regex;
+use helix_core::register::Registers;
 use helix_view::{Document, Editor, View};
 
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 
 pub fn regex_prompt(
     cx: &mut crate::commands::Context,
     prompt: String,
-    fun: impl Fn(&mut View, &mut Document, Regex) + 'static,
+    fun: impl Fn(&mut View, &mut Document, &mut Registers, Regex) + 'static,
 ) -> Prompt {
-    let view_id = cx.view().id;
-    let snapshot = cx.doc().selection(view_id).clone();
+    let (view, doc) = current!(cx.editor);
+    let view_id = view.id;
+    let snapshot = doc.selection(view_id).clone();
 
     Prompt::new(
         prompt,
-        |input: &str| Vec::new(), // this is fine because Vec::new() doesn't allocate
-        move |editor: &mut Editor, input: &str, event: PromptEvent| {
+        None,
+        |_input: &str| Vec::new(), // this is fine because Vec::new() doesn't allocate
+        move |cx: &mut crate::compositor::Context, input: &str, event: PromptEvent| {
             match event {
                 PromptEvent::Abort => {
-                    // TODO: also revert text
-                    let (view, doc) = editor.current();
+                    let (view, doc) = current!(cx.editor);
                     doc.set_selection(view.id, snapshot.clone());
                 }
                 PromptEvent::Validate => {
@@ -53,15 +55,15 @@ pub fn regex_prompt(
 
                     match Regex::new(input) {
                         Ok(regex) => {
-                            let (view, doc) = editor.current();
+                            let (view, doc) = current!(cx.editor);
+                            let registers = &mut cx.editor.registers;
 
                             // revert state to what it was before the last update
-                            // TODO: also revert text
                             doc.set_selection(view.id, snapshot.clone());
 
-                            fun(view, doc, regex);
+                            fun(view, doc, registers, regex);
 
-                            view.ensure_cursor_in_view(doc);
+                            view.ensure_cursor_in_view(doc, cx.editor.config.scrolloff);
                         }
                         Err(_err) => (), // TODO: mark command line as error
                     }
@@ -71,28 +73,40 @@ pub fn regex_prompt(
     )
 }
 
-pub fn file_picker(root: PathBuf) -> Picker<PathBuf> {
+pub fn file_picker(root: PathBuf) -> FilePicker<PathBuf> {
     use ignore::Walk;
-    let files = Walk::new(root.clone()).filter_map(|entry| match entry {
-        Ok(entry) => {
-            // filter dirs, but we might need special handling for symlinks!
-            if !entry.file_type().map_or(false, |entry| entry.is_dir()) {
-                Some(entry.into_path())
-            } else {
-                None
+    use std::time;
+    let files = Walk::new(root.clone()).filter_map(|entry| {
+        let entry = entry.ok()?;
+        // Path::is_dir() traverses symlinks, so we use it over DirEntry::is_dir
+        if entry.path().is_dir() {
+            // Will give a false positive if metadata cannot be read (eg. permission error)
+            return None;
         }
-        }
-        Err(_err) => None,
+
+        let time = entry.metadata().map_or(time::UNIX_EPOCH, |metadata| {
+            metadata
+                .accessed()
+                .or_else(|_| metadata.modified())
+                .or_else(|_| metadata.created())
+                .unwrap_or(time::UNIX_EPOCH)
         });
 
-    let files = if root.join(".git").is_dir() {
+        Some((entry.into_path(), time))
+    });
+
+    let mut files: Vec<_> = if root.join(".git").is_dir() {
         files.collect()
     } else {
         const MAX: usize = 8192;
         files.take(MAX).collect()
     };
 
-    Picker::new(
+    files.sort_by_key(|file| std::cmp::Reverse(file.1));
+
+    let files = files.into_iter().map(|(path, _)| path).collect();
+
+    FilePicker::new(
         files,
         move |path: &PathBuf| {
             // format_fn
@@ -103,30 +117,101 @@ pub fn file_picker(root: PathBuf) -> Picker<PathBuf> {
                 .into()
         },
         move |editor: &mut Editor, path: &PathBuf, action| {
-            let document_id = editor
+            editor
                 .open(path.into(), action)
                 .expect("editor.open failed");
         },
+        |_editor, path| Some((path.clone(), None)),
     )
 }
 
 pub mod completers {
     use crate::ui::prompt::Completion;
+    use fuzzy_matcher::skim::SkimMatcherV2 as Matcher;
+    use fuzzy_matcher::FuzzyMatcher;
+    use helix_view::theme;
     use std::borrow::Cow;
+    use std::cmp::Reverse;
 
     pub type Completer = fn(&str) -> Vec<Completion>;
 
-    // TODO: we could return an iter/lazy thing so it can fetch as many as it needs.
+    pub fn theme(input: &str) -> Vec<Completion> {
+        let mut names = theme::Loader::read_names(&helix_core::runtime_dir().join("themes"));
+        names.extend(theme::Loader::read_names(
+            &helix_core::config_dir().join("themes"),
+        ));
+        names.push("default".into());
+
+        let mut names: Vec<_> = names
+            .into_iter()
+            .map(|name| ((0..), Cow::from(name)))
+            .collect();
+
+        let matcher = Matcher::default();
+
+        let mut matches: Vec<_> = names
+            .into_iter()
+            .filter_map(|(_range, name)| {
+                matcher.fuzzy_match(&name, input).map(|score| (name, score))
+            })
+            .collect();
+
+        matches.sort_unstable_by_key(|(_file, score)| Reverse(*score));
+        names = matches.into_iter().map(|(name, _)| ((0..), name)).collect();
+
+        names
+    }
+
     pub fn filename(input: &str) -> Vec<Completion> {
+        filename_impl(input, |entry| {
+            let is_dir = entry.file_type().map_or(false, |entry| entry.is_dir());
+
+            if is_dir {
+                FileMatch::AcceptIncomplete
+            } else {
+                FileMatch::Accept
+            }
+        })
+    }
+
+    pub fn directory(input: &str) -> Vec<Completion> {
+        filename_impl(input, |entry| {
+            let is_dir = entry.file_type().map_or(false, |entry| entry.is_dir());
+
+            if is_dir {
+                FileMatch::Accept
+            } else {
+                FileMatch::Reject
+            }
+        })
+    }
+
+    #[derive(Copy, Clone, PartialEq, Eq)]
+    enum FileMatch {
+        /// Entry should be ignored
+        Reject,
+        /// Entry is usable but can't be the end (for instance if the entry is a directory and we
+        /// try to match a file)
+        AcceptIncomplete,
+        /// Entry is usable and can be the end of the match
+        Accept,
+    }
+
+    // TODO: we could return an iter/lazy thing so it can fetch as many as it needs.
+    fn filename_impl<F>(input: &str, filter_fn: F) -> Vec<Completion>
+    where
+        F: Fn(&ignore::DirEntry) -> FileMatch,
+    {
         // Rust's filename handling is really annoying.
 
         use ignore::WalkBuilder;
-        use std::path::{Path, PathBuf};
+        use std::path::Path;
 
-        let path = Path::new(input);
+        let is_tilde = input.starts_with('~') && input.len() == 1;
+        let path = helix_view::document::expand_tilde(Path::new(input));
 
         let (dir, file_name) = if input.ends_with('/') {
-            (path.into(), None)
+            (path, None)
         } else {
             let file_name = path
                 .file_name()
@@ -141,23 +226,39 @@ pub mod completers {
             (path, file_name)
         };
 
-        let end = (input.len()..);
+        let end = input.len()..;
 
         let mut files: Vec<_> = WalkBuilder::new(dir.clone())
             .max_depth(Some(1))
             .build()
             .filter_map(|file| {
-                file.ok().map(|entry| {
-                    let is_dir = entry.file_type().map_or(false, |entry| entry.is_dir());
+                file.ok().and_then(|entry| {
+                    let fmatch = filter_fn(&entry);
+
+                    if fmatch == FileMatch::Reject {
+                        return None;
+                    }
+
+                    //let is_dir = entry.file_type().map_or(false, |entry| entry.is_dir());
 
                     let path = entry.path();
-                    let mut path = path.strip_prefix(&dir).unwrap_or(path).to_path_buf();
+                    let mut path = if is_tilde {
+                        // if it's a single tilde an absolute path is displayed so that when `TAB` is pressed on
+                        // one of the directories the tilde will be replaced with a valid path not with a relative
+                        // home directory name.
+                        // ~ -> <TAB> -> /home/user
+                        // ~/ -> <TAB> -> ~/first_entry
+                        path.to_path_buf()
+                    } else {
+                        path.strip_prefix(&dir).unwrap_or(path).to_path_buf()
+                    };
 
-                    if is_dir {
+                    if fmatch == FileMatch::AcceptIncomplete {
                         path.push("");
                     }
+
                     let path = path.to_str().unwrap().to_owned();
-                    (end.clone(), Cow::from(path))
+                    Some((end.clone(), Cow::from(path)))
                 })
             }) // TODO: unwrap or skip
             .filter(|(_, path)| !path.is_empty()) // TODO
@@ -165,23 +266,19 @@ pub mod completers {
 
         // if empty, return a list of dirs and files in current dir
         if let Some(file_name) = file_name {
-            use fuzzy_matcher::skim::SkimMatcherV2 as Matcher;
-            use fuzzy_matcher::FuzzyMatcher;
-            use std::cmp::Reverse;
-
             let matcher = Matcher::default();
 
             // inefficient, but we need to calculate the scores, filter out None, then sort.
             let mut matches: Vec<_> = files
                 .into_iter()
-                .filter_map(|(range, file)| {
+                .filter_map(|(_range, file)| {
                     matcher
                         .fuzzy_match(&file, &file_name)
                         .map(|score| (file, score))
                 })
                 .collect();
 
-            let range = ((input.len() - file_name.len())..);
+            let range = (input.len().saturating_sub(file_name.len()))..;
 
             matches.sort_unstable_by_key(|(_file, score)| Reverse(*score));
             files = matches

helix-term/src/ui/picker.rs

@@ -1,21 +1,161 @@
-use crate::compositor::{Component, Compositor, Context, EventResult};
+use crate::{
+    compositor::{Component, Compositor, Context, EventResult},
+    ui::EditorView,
+};
 use crossterm::event::{Event, KeyCode, KeyEvent, KeyModifiers};
 use tui::{
     buffer::Buffer as Surface,
-    layout::Rect,
-    style::{Color, Style},
     widgets::{Block, BorderType, Borders},
 };
 
 use fuzzy_matcher::skim::SkimMatcherV2 as Matcher;
 use fuzzy_matcher::FuzzyMatcher;
+use tui::widgets::Widget;
 
-use std::borrow::Cow;
+use std::{borrow::Cow, collections::HashMap, path::PathBuf};
 
 use crate::ui::{Prompt, PromptEvent};
 use helix_core::Position;
-use helix_view::editor::Action;
-use helix_view::Editor;
+use helix_view::{
+    document::canonicalize_path,
+    editor::Action,
+    graphics::{Color, CursorKind, Rect, Style},
+    Document, Editor,
+};
+
+pub const MIN_SCREEN_WIDTH_FOR_PREVIEW: u16 = 80;
+
+/// File path and line number (used to align and highlight a line)
+type FileLocation = (PathBuf, Option<usize>);
+
+pub struct FilePicker<T> {
+    picker: Picker<T>,
+    /// Caches paths to documents
+    preview_cache: HashMap<PathBuf, Document>,
+    /// Given an item in the picker, return the file path and line number to display.
+    file_fn: Box<dyn Fn(&Editor, &T) -> Option<FileLocation>>,
+}
+
+impl<T> FilePicker<T> {
+    pub fn new(
+        options: Vec<T>,
+        format_fn: impl Fn(&T) -> Cow<str> + 'static,
+        callback_fn: impl Fn(&mut Editor, &T, Action) + 'static,
+        preview_fn: impl Fn(&Editor, &T) -> Option<FileLocation> + 'static,
+    ) -> Self {
+        Self {
+            picker: Picker::new(false, options, format_fn, callback_fn),
+            preview_cache: HashMap::new(),
+            file_fn: Box::new(preview_fn),
+        }
+    }
+
+    fn current_file(&self, editor: &Editor) -> Option<FileLocation> {
+        self.picker
+            .selection()
+            .and_then(|current| (self.file_fn)(editor, current))
+            .and_then(|(path, line)| canonicalize_path(&path).ok().zip(Some(line)))
+    }
+
+    fn calculate_preview(&mut self, editor: &Editor) {
+        if let Some((path, _line)) = self.current_file(editor) {
+            if !self.preview_cache.contains_key(&path) && editor.document_by_path(&path).is_none() {
+                // TODO: enable syntax highlighting; blocked by async rendering
+                let doc = Document::open(&path, None, Some(&editor.theme), None).unwrap();
+                self.preview_cache.insert(path, doc);
+            }
+        }
+    }
+}
+
+impl<T: 'static> Component for FilePicker<T> {
+    fn render(&mut self, area: Rect, surface: &mut Surface, cx: &mut Context) {
+        // +---------+ +---------+
+        // |prompt   | |preview  |
+        // +---------+ |         |
+        // |picker   | |         |
+        // |         | |         |
+        // +---------+ +---------+
+        self.calculate_preview(cx.editor);
+        let render_preview = area.width > MIN_SCREEN_WIDTH_FOR_PREVIEW;
+        let area = inner_rect(area);
+        // -- Render the frame:
+        // clear area
+        let background = cx.editor.theme.get("ui.background");
+        surface.clear_with(area, background);
+
+        let picker_width = if render_preview {
+            area.width / 2
+        } else {
+            area.width
+        };
+
+        let picker_area = Rect::new(area.x, area.y, picker_width, area.height);
+        self.picker.render(picker_area, surface, cx);
+
+        if !render_preview {
+            return;
+        }
+
+        let preview_area = Rect::new(area.x + picker_width, area.y, area.width / 2, area.height);
+
+        // don't like this but the lifetime sucks
+        let block = Block::default().borders(Borders::ALL);
+
+        // calculate the inner area inside the box
+        let mut inner = block.inner(preview_area);
+        // 1 column gap on either side
+        inner.x += 1;
+        inner.width = inner.width.saturating_sub(2);
+
+        block.render(preview_area, surface);
+
+        if let Some((doc, line)) = self.current_file(cx.editor).and_then(|(path, line)| {
+            cx.editor
+                .document_by_path(&path)
+                .or_else(|| self.preview_cache.get(&path))
+                .zip(Some(line))
+        }) {
+            // align to middle
+            let first_line = line.unwrap_or(0).saturating_sub(inner.height as usize / 2);
+            let offset = Position::new(first_line, 0);
+
+            let highlights = EditorView::doc_syntax_highlights(
+                doc,
+                offset,
+                area.height,
+                &cx.editor.theme,
+                &cx.editor.syn_loader,
+            );
+            EditorView::render_text_highlights(
+                doc,
+                offset,
+                inner,
+                surface,
+                &cx.editor.theme,
+                highlights,
+            );
+
+            // highlight the line
+            if let Some(line) = line {
+                for x in inner.left()..inner.right() {
+                    surface
+                        .get_mut(x, inner.y + line.saturating_sub(first_line) as u16)
+                        .set_style(cx.editor.theme.get("ui.selection"));
+                }
+            }
+        }
+    }
+
+    fn handle_event(&mut self, event: Event, ctx: &mut Context) -> EventResult {
+        // TODO: keybinds for scrolling preview
+        self.picker.handle_event(event, ctx)
+    }
+
+    fn cursor(&self, area: Rect, ctx: &Editor) -> (Option<Position>, CursorKind) {
+        self.picker.cursor(area, ctx)
+    }
+}
 
 pub struct Picker<T> {
     options: Vec<T>,
@@ -23,10 +163,14 @@ pub struct Picker<T> {
     matcher: Box<Matcher>,
     /// (index, score)
     matches: Vec<(usize, i64)>,
+    /// Filter over original options.
+    filters: Vec<usize>, // could be optimized into bit but not worth it now
 
     cursor: usize,
     // pattern: String,
     prompt: Prompt,
+    /// Whether to render in the middle of the area
+    render_centered: bool,
 
     format_fn: Box<dyn Fn(&T) -> Cow<str>>,
     callback_fn: Box<dyn Fn(&mut Editor, &T, Action)>,
@@ -34,14 +178,16 @@ pub struct Picker<T> {
 
 impl<T> Picker<T> {
     pub fn new(
+        render_centered: bool,
         options: Vec<T>,
         format_fn: impl Fn(&T) -> Cow<str> + 'static,
         callback_fn: impl Fn(&mut Editor, &T, Action) + 'static,
     ) -> Self {
         let prompt = Prompt::new(
             "".to_string(),
-            |pattern: &str| Vec::new(),
-            |editor: &mut Editor, pattern: &str, event: PromptEvent| {
+            None,
+            |_pattern: &str| Vec::new(),
+            |_editor: &mut Context, _pattern: &str, _event: PromptEvent| {
                 //
             },
         );
@@ -50,8 +196,10 @@ impl<T> Picker<T> {
             options,
             matcher: Box::new(Matcher::default()),
             matches: Vec::new(),
+            filters: Vec::new(),
             cursor: 0,
             prompt,
+            render_centered,
             format_fn: Box::new(format_fn),
             callback_fn: Box::new(callback_fn),
         };
@@ -65,9 +213,9 @@ impl<T> Picker<T> {
     pub fn score(&mut self) {
         // need to borrow via pattern match otherwise it complains about simultaneous borrow
         let Self {
-            ref mut options,
             ref mut matcher,
             ref mut matches,
+            ref filters,
             ref format_fn,
             ..
         } = *self;
@@ -81,6 +229,10 @@ impl<T> Picker<T> {
                 .iter()
                 .enumerate()
                 .filter_map(|(index, option)| {
+                    // filter options first before matching
+                    if !filters.is_empty() {
+                        filters.binary_search(&index).ok()?;
+                    }
                     // TODO: maybe using format_fn isn't the best idea here
                     let text = (format_fn)(option);
                     // TODO: using fuzzy_indices could give us the char idx for match highlighting
@@ -114,6 +266,14 @@ impl<T> Picker<T> {
             .get(self.cursor)
             .map(|(index, _score)| &self.options[*index])
     }
+
+    pub fn save_filter(&mut self) {
+        self.filters.clear();
+        self.filters
+            .extend(self.matches.iter().map(|(index, _)| *index));
+        self.filters.sort_unstable(); // used for binary search later
+        self.prompt.clear();
+    }
 }
 
 // process:
@@ -122,8 +282,8 @@ impl<T> Picker<T> {
 //  - score all the names in relation to input
 
 fn inner_rect(area: Rect) -> Rect {
-    let padding_vertical = area.height * 20 / 100;
-    let padding_horizontal = area.width * 20 / 100;
+    let padding_vertical = area.height * 10 / 100;
+    let padding_horizontal = area.width * 10 / 100;
 
     Rect::new(
         area.x + padding_horizontal,
@@ -143,7 +303,7 @@ impl<T: 'static> Component for Picker<T> {
 
         let close_fn = EventResult::Consumed(Some(Box::new(|compositor: &mut Compositor| {
             // remove the layer
-            compositor.pop();
+            compositor.last_picker = compositor.pop();
         })));
 
         match key_event {
@@ -157,7 +317,9 @@ impl<T: 'static> Component for Picker<T> {
             | KeyEvent {
                 code: KeyCode::Char('p'),
                 modifiers: KeyModifiers::CONTROL,
-            } => self.move_up(),
+            } => {
+                self.move_up();
+            }
             KeyEvent {
                 code: KeyCode::Down,
                 ..
@@ -168,7 +330,9 @@ impl<T: 'static> Component for Picker<T> {
             | KeyEvent {
                 code: KeyCode::Char('n'),
                 modifiers: KeyModifiers::CONTROL,
-            } => self.move_down(),
+            } => {
+                self.move_down();
+            }
             KeyEvent {
                 code: KeyCode::Esc, ..
             }
@@ -205,6 +369,12 @@ impl<T: 'static> Component for Picker<T> {
                 }
                 return close_fn;
             }
+            KeyEvent {
+                code: KeyCode::Char(' '),
+                modifiers: KeyModifiers::CONTROL,
+            } => {
+                self.save_filter();
+            }
             _ => {
                 if let EventResult::Consumed(_) = self.prompt.handle_event(event, cx) {
                     // TODO: recalculate only if pattern changed
@@ -216,16 +386,20 @@ impl<T: 'static> Component for Picker<T> {
         EventResult::Consumed(None)
     }
 
-    fn render(&self, area: Rect, surface: &mut Surface, cx: &mut Context) {
-        let area = inner_rect(area);
+    fn render(&mut self, area: Rect, surface: &mut Surface, cx: &mut Context) {
+        let area = if self.render_centered {
+            inner_rect(area)
+        } else {
+            area
+        };
+
+        let text_style = cx.editor.theme.get("ui.text");
 
         // -- Render the frame:
-
         // clear area
         let background = cx.editor.theme.get("ui.background");
         surface.clear_with(area, background);
 
-        use tui::widgets::Widget;
         // don't like this but the lifetime sucks
         let block = Block::default().borders(Borders::ALL);
 
@@ -233,30 +407,39 @@ impl<T: 'static> Component for Picker<T> {
         let inner = block.inner(area);
 
         block.render(area, surface);
-        // TODO: abstract into a clear(area) fn
-        // surface.set_style(inner, Style::default().bg(Color::Rgb(150, 50, 0)));
 
         // -- Render the input bar:
 
         let area = Rect::new(inner.x + 1, inner.y, inner.width - 1, 1);
+
+        let count = format!("{}/{}", self.matches.len(), self.options.len());
+        surface.set_stringn(
+            (area.x + area.width).saturating_sub(count.len() as u16 + 1),
+            area.y,
+            &count,
+            (count.len()).min(area.width as usize),
+            text_style,
+        );
+
         self.prompt.render(area, surface, cx);
 
         // -- Separator
-        let style = Style::default().fg(Color::Rgb(90, 89, 119));
-        let symbols = BorderType::line_symbols(BorderType::Plain);
+        let sep_style = Style::default().fg(Color::Rgb(90, 89, 119));
+        let borders = BorderType::line_symbols(BorderType::Plain);
         for x in inner.left()..inner.right() {
             surface
                 .get_mut(x, inner.y + 1)
-                .set_symbol(symbols.horizontal)
-                .set_style(style);
+                .set_symbol(borders.horizontal)
+                .set_style(sep_style);
         }
 
         // -- Render the contents:
+        // subtract the area of the prompt (-2) and current item marker " > " (-3)
+        let inner = Rect::new(inner.x + 3, inner.y + 2, inner.width - 3, inner.height - 2);
 
-        let style = cx.editor.theme.get("ui.text");
         let selected = Style::default().fg(Color::Rgb(255, 255, 255));
 
-        let rows = inner.height - 2; // -1 for search bar
+        let rows = inner.height;
         let offset = self.cursor / (rows as usize) * (rows as usize);
 
         let files = self.matches.iter().skip(offset).map(|(index, _score)| {
@@ -265,24 +448,25 @@ impl<T: 'static> Component for Picker<T> {
 
         for (i, (_index, option)) in files.take(rows as usize).enumerate() {
             if i == (self.cursor - offset) {
-                surface.set_string(inner.x + 1, inner.y + 2 + i as u16, ">", selected);
+                surface.set_string(inner.x - 2, inner.y + i as u16, ">", selected);
             }
 
-            surface.set_stringn(
-                inner.x + 3,
-                inner.y + 2 + i as u16,
+            surface.set_string_truncated(
+                inner.x,
+                inner.y + i as u16,
                 (self.format_fn)(option),
-                inner.width as usize - 1,
+                inner.width as usize,
                 if i == (self.cursor - offset) {
                     selected
                 } else {
-                    style
+                    text_style
                 },
+                true,
             );
         }
     }
 
-    fn cursor_position(&self, area: Rect, editor: &Editor) -> Option<Position> {
+    fn cursor(&self, area: Rect, editor: &Editor) -> (Option<Position>, CursorKind) {
         // TODO: this is mostly duplicate code
         let area = inner_rect(area);
         let block = Block::default().borders(Borders::ALL);
@@ -292,6 +476,6 @@ impl<T: 'static> Component for Picker<T> {
         // prompt area
         let area = Rect::new(inner.x + 1, inner.y, inner.width - 1, 1);
 
-        self.prompt.cursor_position(area, editor)
+        self.prompt.cursor(area, editor)
     }
 }

helix-term/src/ui/popup.rs

@@ -1,15 +1,9 @@
 use crate::compositor::{Component, Compositor, Context, EventResult};
 use crossterm::event::{Event, KeyCode, KeyEvent, KeyModifiers};
-use tui::{
-    buffer::Buffer as Surface,
-    layout::Rect,
-    style::{Color, Style},
-};
-
-use std::borrow::Cow;
+use tui::buffer::Buffer as Surface;
 
 use helix_core::Position;
-use helix_view::Editor;
+use helix_view::graphics::Rect;
 
 // TODO: share logic with Menu, it's essentially Popup(render_fn), but render fn needs to return
 // a width/height hint. maybe Popup(Box<Component>)
@@ -58,7 +52,7 @@ impl<T: Component> Component for Popup<T> {
     fn handle_event(&mut self, event: Event, cx: &mut Context) -> EventResult {
         let key = match event {
             Event::Key(event) => event,
-            Event::Resize(width, height) => {
+            Event::Resize(_, _) => {
                 // TODO: calculate inner area, call component's handle_event with that area
                 return EventResult::Ignored;
             }
@@ -100,7 +94,7 @@ impl<T: Component> Component for Popup<T> {
         // tab/enter/ctrl-k or whatever will confirm the selection/ ctrl-n/ctrl-p for scroll.
     }
 
-    fn required_size(&mut self, viewport: (u16, u16)) -> Option<(u16, u16)> {
+    fn required_size(&mut self, _viewport: (u16, u16)) -> Option<(u16, u16)> {
         let (width, height) = self
             .contents
             .required_size((120, 26)) // max width, max height
@@ -111,12 +105,12 @@ impl<T: Component> Component for Popup<T> {
         Some(self.size)
     }
 
-    fn render(&self, viewport: Rect, surface: &mut Surface, cx: &mut Context) {
+    fn render(&mut self, viewport: Rect, surface: &mut Surface, cx: &mut Context) {
         cx.scroll = Some(self.scroll);
 
         let position = self
             .position
-            .or_else(|| cx.editor.cursor_position())
+            .or_else(|| cx.editor.cursor().0)
             .unwrap_or_default();
 
         let (width, height) = self.size;
@@ -136,7 +130,6 @@ impl<T: Component> Component for Popup<T> {
             rel_y += 1 // position below point
         }
 
-        let area = Rect::new(rel_x, rel_y, width, height);
         // clip to viewport
         let area = viewport.intersection(Rect::new(rel_x, rel_y, width, height));
 

helix-term/src/ui/prompt.rs

@@ -1,9 +1,16 @@
 use crate::compositor::{Component, Compositor, Context, EventResult};
 use crate::ui;
 use crossterm::event::{Event, KeyCode, KeyEvent, KeyModifiers};
-use helix_core::Position;
-use helix_view::{Editor, Theme};
 use std::{borrow::Cow, ops::RangeFrom};
+use tui::buffer::Buffer as Surface;
+
+use helix_core::{
+    unicode::segmentation::GraphemeCursor, unicode::width::UnicodeWidthStr, Position,
+};
+use helix_view::{
+    graphics::{CursorKind, Margin, Rect},
+    Editor,
+};
 
 pub type Completion = (RangeFrom<usize>, Cow<'static, str>);
 
@@ -13,8 +20,10 @@ pub struct Prompt {
     cursor: usize,
     completion: Vec<Completion>,
     selection: Option<usize>,
+    history_register: Option<char>,
+    history_pos: Option<usize>,
     completion_fn: Box<dyn FnMut(&str) -> Vec<Completion>>,
-    callback_fn: Box<dyn FnMut(&mut Editor, &str, PromptEvent)>,
+    callback_fn: Box<dyn FnMut(&mut Context, &str, PromptEvent)>,
     pub doc_fn: Box<dyn Fn(&str) -> Option<&'static str>>,
 }
 
@@ -33,11 +42,23 @@ pub enum CompletionDirection {
     Backward,
 }
 
+#[derive(Debug, Clone, Copy)]
+pub enum Movement {
+    BackwardChar(usize),
+    BackwardWord(usize),
+    ForwardChar(usize),
+    ForwardWord(usize),
+    StartOfLine,
+    EndOfLine,
+    None,
+}
+
 impl Prompt {
     pub fn new(
         prompt: String,
+        history_register: Option<char>,
         mut completion_fn: impl FnMut(&str) -> Vec<Completion> + 'static,
-        callback_fn: impl FnMut(&mut Editor, &str, PromptEvent) + 'static,
+        callback_fn: impl FnMut(&mut Context, &str, PromptEvent) + 'static,
     ) -> Self {
         Self {
             prompt,
@@ -45,27 +66,128 @@ impl Prompt {
             cursor: 0,
             completion: completion_fn(""),
             selection: None,
+            history_register,
+            history_pos: None,
             completion_fn: Box::new(completion_fn),
             callback_fn: Box::new(callback_fn),
             doc_fn: Box::new(|_| None),
         }
     }
 
+    /// Compute the cursor position after applying movement
+    /// Taken from: https://github.com/wez/wezterm/blob/e0b62d07ca9bf8ce69a61e30a3c20e7abc48ce7e/termwiz/src/lineedit/mod.rs#L516-L611
+    fn eval_movement(&self, movement: Movement) -> usize {
+        match movement {
+            Movement::BackwardChar(rep) => {
+                let mut position = self.cursor;
+                for _ in 0..rep {
+                    let mut cursor = GraphemeCursor::new(position, self.line.len(), false);
+                    if let Ok(Some(pos)) = cursor.prev_boundary(&self.line, 0) {
+                        position = pos;
+                    } else {
+                        break;
+                    }
+                }
+                position
+            }
+            Movement::BackwardWord(rep) => {
+                let char_indices: Vec<(usize, char)> = self.line.char_indices().collect();
+                if char_indices.is_empty() {
+                    return self.cursor;
+                }
+                let mut char_position = char_indices
+                    .iter()
+                    .position(|(idx, _)| *idx == self.cursor)
+                    .unwrap_or(char_indices.len() - 1);
+
+                for _ in 0..rep {
+                    if char_position == 0 {
+                        break;
+                    }
+
+                    let mut found = None;
+                    for prev in (0..char_position - 1).rev() {
+                        if char_indices[prev].1.is_whitespace() {
+                            found = Some(prev + 1);
+                            break;
+                        }
+                    }
+
+                    char_position = found.unwrap_or(0);
+                }
+                char_indices[char_position].0
+            }
+            Movement::ForwardWord(rep) => {
+                let char_indices: Vec<(usize, char)> = self.line.char_indices().collect();
+                if char_indices.is_empty() {
+                    return self.cursor;
+                }
+                let mut char_position = char_indices
+                    .iter()
+                    .position(|(idx, _)| *idx == self.cursor)
+                    .unwrap_or_else(|| char_indices.len());
+
+                for _ in 0..rep {
+                    // Skip any non-whitespace characters
+                    while char_position < char_indices.len()
+                        && !char_indices[char_position].1.is_whitespace()
+                    {
+                        char_position += 1;
+                    }
+
+                    // Skip any whitespace characters
+                    while char_position < char_indices.len()
+                        && char_indices[char_position].1.is_whitespace()
+                    {
+                        char_position += 1;
+                    }
+
+                    // We are now on the start of the next word
+                }
+                char_indices
+                    .get(char_position)
+                    .map(|(i, _)| *i)
+                    .unwrap_or_else(|| self.line.len())
+            }
+            Movement::ForwardChar(rep) => {
+                let mut position = self.cursor;
+                for _ in 0..rep {
+                    let mut cursor = GraphemeCursor::new(position, self.line.len(), false);
+                    if let Ok(Some(pos)) = cursor.next_boundary(&self.line, 0) {
+                        position = pos;
+                    } else {
+                        break;
+                    }
+                }
+                position
+            }
+            Movement::StartOfLine => 0,
+            Movement::EndOfLine => {
+                let mut cursor =
+                    GraphemeCursor::new(self.line.len().saturating_sub(1), self.line.len(), false);
+                if let Ok(Some(pos)) = cursor.next_boundary(&self.line, 0) {
+                    pos
+                } else {
+                    self.cursor
+                }
+            }
+            Movement::None => self.cursor,
+        }
+    }
+
     pub fn insert_char(&mut self, c: char) {
         self.line.insert(self.cursor, c);
-        self.cursor += 1;
+        let mut cursor = GraphemeCursor::new(self.cursor, self.line.len(), false);
+        if let Ok(Some(pos)) = cursor.next_boundary(&self.line, 0) {
+            self.cursor = pos;
+        }
         self.completion = (self.completion_fn)(&self.line);
         self.exit_selection();
     }
 
-    pub fn move_char_left(&mut self) {
-        self.cursor = self.cursor.saturating_sub(1)
-    }
-
-    pub fn move_char_right(&mut self) {
-        if self.cursor < self.line.len() {
-            self.cursor += 1;
-        }
+    pub fn move_cursor(&mut self, movement: Movement) {
+        let pos = self.eval_movement(movement);
+        self.cursor = pos
     }
 
     pub fn move_start(&mut self) {
@@ -77,35 +199,60 @@ impl Prompt {
     }
 
     pub fn delete_char_backwards(&mut self) {
-        if self.cursor > 0 {
-            self.line.remove(self.cursor - 1);
-            self.cursor -= 1;
-            self.completion = (self.completion_fn)(&self.line);
-        }
+        let pos = self.eval_movement(Movement::BackwardChar(1));
+        self.line.replace_range(pos..self.cursor, "");
+        self.cursor = pos;
+
         self.exit_selection();
+        self.completion = (self.completion_fn)(&self.line);
     }
 
     pub fn delete_word_backwards(&mut self) {
-        use helix_core::get_general_category;
-        let mut chars = self.line.char_indices().rev();
-        // TODO add skipping whitespace logic here
-        let (mut i, cat) = match chars.next() {
-            Some((i, c)) => (i, get_general_category(c)),
-            None => return,
-        };
-        self.cursor -= 1;
-        for (nn, nc) in chars {
-            if get_general_category(nc) != cat {
-                break;
+        let pos = self.eval_movement(Movement::BackwardWord(1));
+        self.line.replace_range(pos..self.cursor, "");
+        self.cursor = pos;
+
+        self.exit_selection();
+        self.completion = (self.completion_fn)(&self.line);
     }
-            i = nn;
-            self.cursor -= 1;
+
+    pub fn kill_to_end_of_line(&mut self) {
+        let pos = self.eval_movement(Movement::EndOfLine);
+        self.line.replace_range(self.cursor..pos, "");
+
+        self.exit_selection();
+        self.completion = (self.completion_fn)(&self.line);
     }
-        self.line.drain(i..);
+
+    pub fn clear(&mut self) {
+        self.line.clear();
+        self.cursor = 0;
         self.completion = (self.completion_fn)(&self.line);
         self.exit_selection();
     }
 
+    pub fn change_history(&mut self, register: &[String], direction: CompletionDirection) {
+        if register.is_empty() {
+            return;
+        }
+
+        let end = register.len().saturating_sub(1);
+
+        let index = match direction {
+            CompletionDirection::Forward => self.history_pos.map_or(0, |i| i + 1),
+            CompletionDirection::Backward => {
+                self.history_pos.unwrap_or(register.len()).saturating_sub(1)
+            }
+        }
+        .min(end);
+
+        self.line = register[index].clone();
+
+        self.history_pos = Some(index);
+
+        self.move_end();
+    }
+
     pub fn change_completion_selection(&mut self, direction: CompletionDirection) {
         if self.completion.is_empty() {
             return;
@@ -132,12 +279,6 @@ impl Prompt {
     }
 }
 
-use tui::{
-    buffer::Buffer as Surface,
-    layout::Rect,
-    style::{Color, Modifier, Style},
-};
-
 const BASE_WIDTH: u16 = 30;
 
 impl Prompt {
@@ -160,7 +301,7 @@ impl Prompt {
 
         let height = ((self.completion.len() as u16 + cols - 1) / cols)
             .min(10) // at most 10 rows (or less)
-            .min(area.height);
+            .min(area.height.saturating_sub(1));
 
         let completion_area = Rect::new(
             area.x,
@@ -173,17 +314,21 @@ impl Prompt {
             let area = completion_area;
             let background = theme.get("ui.statusline");
 
+            let items = height as usize * cols as usize;
+
+            let offset = self
+                .selection
+                .map(|selection| selection / items * items)
+                .unwrap_or_default();
+
             surface.clear_with(area, background);
 
             let mut row = 0;
             let mut col = 0;
 
             // TODO: paginate
-            for (i, (_range, completion)) in self
-                .completion
-                .iter()
-                .enumerate()
-                .take(height as usize * cols as usize)
+            for (i, (_range, completion)) in
+                self.completion.iter().enumerate().skip(offset).take(items)
             {
                 let color = if Some(i) == self.selection {
                     // Style::default().bg(Color::Rgb(104, 60, 232))
@@ -207,12 +352,12 @@ impl Prompt {
         }
 
         if let Some(doc) = (self.doc_fn)(&self.line) {
-            let text = ui::Text::new(doc.to_string());
+            let mut text = ui::Text::new(doc.to_string());
 
             let viewport = area;
             let area = viewport.intersection(Rect::new(
                 completion_area.x,
-                completion_area.y - 3,
+                completion_area.y.saturating_sub(3),
                 BASE_WIDTH * 3,
                 3,
             ));
@@ -220,7 +365,6 @@ impl Prompt {
             let background = theme.get("ui.help");
             surface.clear_with(area, background);
 
-            use tui::layout::Margin;
             text.render(
                 area.inner(&Margin {
                     vertical: 1,
@@ -267,27 +411,63 @@ impl Component for Prompt {
                 modifiers: KeyModifiers::SHIFT,
             } => {
                 self.insert_char(c);
-                (self.callback_fn)(cx.editor, &self.line, PromptEvent::Update);
+                (self.callback_fn)(cx, &self.line, PromptEvent::Update);
             }
             KeyEvent {
+                code: KeyCode::Char('c'),
+                modifiers: KeyModifiers::CONTROL,
+            }
+            | KeyEvent {
                 code: KeyCode::Esc, ..
             } => {
-                (self.callback_fn)(cx.editor, &self.line, PromptEvent::Abort);
+                (self.callback_fn)(cx, &self.line, PromptEvent::Abort);
                 return close_fn;
             }
             KeyEvent {
+                code: KeyCode::Left,
+                modifiers: KeyModifiers::ALT,
+            }
+            | KeyEvent {
+                code: KeyCode::Char('b'),
+                modifiers: KeyModifiers::ALT,
+            } => self.move_cursor(Movement::BackwardWord(1)),
+            KeyEvent {
+                code: KeyCode::Right,
+                modifiers: KeyModifiers::ALT,
+            }
+            | KeyEvent {
+                code: KeyCode::Char('f'),
+                modifiers: KeyModifiers::ALT,
+            } => self.move_cursor(Movement::ForwardWord(1)),
+            KeyEvent {
+                code: KeyCode::Char('f'),
+                modifiers: KeyModifiers::CONTROL,
+            }
+            | KeyEvent {
                 code: KeyCode::Right,
                 ..
-            } => self.move_char_right(),
+            } => self.move_cursor(Movement::ForwardChar(1)),
             KeyEvent {
+                code: KeyCode::Char('b'),
+                modifiers: KeyModifiers::CONTROL,
+            }
+            | KeyEvent {
                 code: KeyCode::Left,
                 ..
-            } => self.move_char_left(),
+            } => self.move_cursor(Movement::BackwardChar(1)),
             KeyEvent {
+                code: KeyCode::End,
+                modifiers: KeyModifiers::NONE,
+            }
+            | KeyEvent {
                 code: KeyCode::Char('e'),
                 modifiers: KeyModifiers::CONTROL,
             } => self.move_end(),
             KeyEvent {
+                code: KeyCode::Home,
+                modifiers: KeyModifiers::NONE,
+            }
+            | KeyEvent {
                 code: KeyCode::Char('a'),
                 modifiers: KeyModifiers::CONTROL,
             } => self.move_start(),
@@ -295,25 +475,60 @@ impl Component for Prompt {
                 code: KeyCode::Char('w'),
                 modifiers: KeyModifiers::CONTROL,
             } => self.delete_word_backwards(),
+            KeyEvent {
+                code: KeyCode::Char('k'),
+                modifiers: KeyModifiers::CONTROL,
+            } => self.kill_to_end_of_line(),
             KeyEvent {
                 code: KeyCode::Backspace,
                 modifiers: KeyModifiers::NONE,
             } => {
                 self.delete_char_backwards();
-                (self.callback_fn)(cx.editor, &self.line, PromptEvent::Update);
+                (self.callback_fn)(cx, &self.line, PromptEvent::Update);
             }
             KeyEvent {
                 code: KeyCode::Enter,
                 ..
             } => {
-                if self.line.ends_with('/') {
+                if self.selection.is_some() && self.line.ends_with('/') {
                     self.completion = (self.completion_fn)(&self.line);
                     self.exit_selection();
                 } else {
-                    (self.callback_fn)(cx.editor, &self.line, PromptEvent::Validate);
+                    (self.callback_fn)(cx, &self.line, PromptEvent::Validate);
+
+                    if let Some(register) = self.history_register {
+                        // store in history
+                        let register = cx.editor.registers.get_mut(register);
+                        register.push(self.line.clone());
+                    }
                     return close_fn;
                 }
             }
+            KeyEvent {
+                code: KeyCode::Char('p'),
+                modifiers: KeyModifiers::CONTROL,
+            }
+            | KeyEvent {
+                code: KeyCode::Up, ..
+            } => {
+                if let Some(register) = self.history_register {
+                    let register = cx.editor.registers.get_mut(register);
+                    self.change_history(register.read(), CompletionDirection::Backward);
+                }
+            }
+            KeyEvent {
+                code: KeyCode::Char('n'),
+                modifiers: KeyModifiers::CONTROL,
+            }
+            | KeyEvent {
+                code: KeyCode::Down,
+                ..
+            } => {
+                if let Some(register) = self.history_register {
+                    let register = cx.editor.registers.get_mut(register);
+                    self.change_history(register.read(), CompletionDirection::Forward);
+                }
+            }
             KeyEvent {
                 code: KeyCode::Tab, ..
             } => self.change_completion_selection(CompletionDirection::Forward),
@@ -331,15 +546,20 @@ impl Component for Prompt {
         EventResult::Consumed(None)
     }
 
-    fn render(&self, area: Rect, surface: &mut Surface, cx: &mut Context) {
+    fn render(&mut self, area: Rect, surface: &mut Surface, cx: &mut Context) {
         self.render_prompt(area, surface, cx)
     }
 
-    fn cursor_position(&self, area: Rect, editor: &Editor) -> Option<Position> {
+    fn cursor(&self, area: Rect, _editor: &Editor) -> (Option<Position>, CursorKind) {
         let line = area.height as usize - 1;
+        (
             Some(Position::new(
                 area.y as usize + line,
-            area.x as usize + self.prompt.len() + self.cursor,
-        ))
+                area.x as usize
+                    + self.prompt.len()
+                    + UnicodeWidthStr::width(&self.line[..self.cursor]),
+            )),
+            CursorKind::Block,
+        )
     }
 }

helix-term/src/ui/spinner.rs

@@ -0,0 +1,75 @@
+use std::{collections::HashMap, time::SystemTime};
+
+#[derive(Default, Debug)]
+pub struct ProgressSpinners {
+    inner: HashMap<usize, Spinner>,
+}
+
+impl ProgressSpinners {
+    pub fn get(&self, id: usize) -> Option<&Spinner> {
+        self.inner.get(&id)
+    }
+
+    pub fn get_or_create(&mut self, id: usize) -> &mut Spinner {
+        self.inner.entry(id).or_insert_with(Spinner::default)
+    }
+}
+
+impl Default for Spinner {
+    fn default() -> Self {
+        Self::dots(80)
+    }
+}
+
+#[derive(Debug)]
+pub struct Spinner {
+    frames: Vec<&'static str>,
+    count: usize,
+    start: Option<SystemTime>,
+    interval: u64,
+}
+
+impl Spinner {
+    /// Creates a new spinner with `frames` and `interval`.
+    /// Expects the frames count and interval to be greater than 0.
+    pub fn new(frames: Vec<&'static str>, interval: u64) -> Self {
+        let count = frames.len();
+        assert!(count > 0);
+        assert!(interval > 0);
+
+        Self {
+            frames,
+            count,
+            interval,
+            start: None,
+        }
+    }
+
+    pub fn dots(interval: u64) -> Self {
+        Self::new(vec!["⣾", "⣽", "⣻", "⢿", "⡿", "⣟", "⣯", "⣷"], interval)
+    }
+
+    pub fn start(&mut self) {
+        self.start = Some(SystemTime::now());
+    }
+
+    pub fn frame(&self) -> Option<&str> {
+        let idx = (self
+            .start
+            .map(|time| SystemTime::now().duration_since(time))?
+            .ok()?
+            .as_millis()
+            / self.interval as u128) as usize
+            % self.count;
+
+        self.frames.get(idx).copied()
+    }
+
+    pub fn stop(&mut self) {
+        self.start = None;
+    }
+
+    pub fn is_stopped(&self) -> bool {
+        self.start.is_none()
+    }
+}

helix-term/src/ui/text.rs

@@ -1,15 +1,7 @@
-use crate::compositor::{Component, Compositor, Context, EventResult};
-use crossterm::event::{Event, KeyCode, KeyEvent, KeyModifiers};
-use tui::{
-    buffer::Buffer as Surface,
-    layout::Rect,
-    style::{Color, Style},
-};
+use crate::compositor::{Component, Context};
+use tui::buffer::Buffer as Surface;
 
-use std::borrow::Cow;
-
-use helix_core::Position;
-use helix_view::Editor;
+use helix_view::graphics::Rect;
 
 pub struct Text {
     contents: String,
@@ -21,12 +13,10 @@ impl Text {
     }
 }
 impl Component for Text {
-    fn render(&self, area: Rect, surface: &mut Surface, cx: &mut Context) {
+    fn render(&mut self, area: Rect, surface: &mut Surface, _cx: &mut Context) {
         use tui::widgets::{Paragraph, Widget, Wrap};
         let contents = tui::text::Text::from(self.contents.clone());
 
-        let style = cx.editor.theme.get("ui.text");
-
         let par = Paragraph::new(contents).wrap(Wrap { trim: false });
         // .scroll(x, y) offsets
 

helix-tui/Cargo.toml

@@ -1,12 +1,16 @@
 [package]
 name = "helix-tui"
-version = "0.2.0"
+version = "0.4.1"
 authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
 description = """
 A library to build rich terminal user interfaces or dashboards
 """
 edition = "2018"
 license = "MPL-2.0"
+categories = ["editor"]
+repository = "https://github.com/helix-editor/helix"
+homepage = "https://helix-editor.com"
+include = ["src/**/*", "README.md"]
 
 [features]
 default = ["crossterm"]
@@ -14,7 +18,8 @@ default = ["crossterm"]
 [dependencies]
 bitflags = "1.0"
 cassowary = "0.3"
-unicode-segmentation = "1.2"
-unicode-width = "0.1"
+unicode-segmentation = "1.8"
 crossterm = { version = "0.20", optional = true }
 serde = { version = "1", "optional" = true, features = ["derive"]}
+helix-view = { version = "0.4", path = "../helix-view", features = ["term"] }
+helix-core = { version = "0.4", path = "../helix-core" }

helix-tui/src/backend/crossterm.rs

@@ -1,11 +1,6 @@
-use crate::{
-    backend::Backend,
-    buffer::Cell,
-    layout::Rect,
-    style::{Color, Modifier},
-};
+use crate::{backend::Backend, buffer::Cell};
 use crossterm::{
-    cursor::{Hide, MoveTo, Show},
+    cursor::{CursorShape, Hide, MoveTo, SetCursorShape, Show},
     execute, queue,
     style::{
         Attribute as CAttribute, Color as CColor, Print, SetAttribute, SetBackgroundColor,
@@ -13,6 +8,7 @@ use crossterm::{
     },
     terminal::{self, Clear, ClearType},
 };
+use helix_view::graphics::{Color, CursorKind, Modifier, Rect};
 use std::io::{self, Write};
 
 pub struct CrosstermBackend<W: Write> {
@@ -93,8 +89,14 @@ where
         map_error(execute!(self.buffer, Hide))
     }
 
-    fn show_cursor(&mut self) -> io::Result<()> {
-        map_error(execute!(self.buffer, Show))
+    fn show_cursor(&mut self, kind: CursorKind) -> io::Result<()> {
+        let shape = match kind {
+            CursorKind::Block => CursorShape::Block,
+            CursorKind::Bar => CursorShape::Line,
+            CursorKind::Underline => CursorShape::UnderScore,
+            CursorKind::Hidden => unreachable!(),
+        };
+        map_error(execute!(self.buffer, Show, SetCursorShape(shape)))
     }
 
     fn get_cursor(&mut self) -> io::Result<(u16, u16)> {
@@ -126,32 +128,6 @@ fn map_error(error: crossterm::Result<()>) -> io::Result<()> {
     error.map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))
 }
 
-impl From<Color> for CColor {
-    fn from(color: Color) -> Self {
-        match color {
-            Color::Reset => CColor::Reset,
-            Color::Black => CColor::Black,
-            Color::Red => CColor::DarkRed,
-            Color::Green => CColor::DarkGreen,
-            Color::Yellow => CColor::DarkYellow,
-            Color::Blue => CColor::DarkBlue,
-            Color::Magenta => CColor::DarkMagenta,
-            Color::Cyan => CColor::DarkCyan,
-            Color::Gray => CColor::Grey,
-            Color::DarkGray => CColor::DarkGrey,
-            Color::LightRed => CColor::Red,
-            Color::LightGreen => CColor::Green,
-            Color::LightBlue => CColor::Blue,
-            Color::LightYellow => CColor::Yellow,
-            Color::LightMagenta => CColor::Magenta,
-            Color::LightCyan => CColor::Cyan,
-            Color::White => CColor::White,
-            Color::Indexed(i) => CColor::AnsiValue(i),
-            Color::Rgb(r, g, b) => CColor::Rgb { r, g, b },
-        }
-    }
-}
-
 #[derive(Debug)]
 struct ModifierDiff {
     pub from: Modifier,

helix-tui/src/backend/mod.rs

@@ -1,7 +1,8 @@
 use std::io;
 
 use crate::buffer::Cell;
-use crate::layout::Rect;
+
+use helix_view::graphics::{CursorKind, Rect};
 
 #[cfg(feature = "crossterm")]
 mod crossterm;
@@ -16,7 +17,7 @@ pub trait Backend {
     where
         I: Iterator<Item = (u16, u16, &'a Cell)>;
     fn hide_cursor(&mut self) -> Result<(), io::Error>;
-    fn show_cursor(&mut self) -> Result<(), io::Error>;
+    fn show_cursor(&mut self, kind: CursorKind) -> Result<(), io::Error>;
     fn get_cursor(&mut self) -> Result<(u16, u16), io::Error>;
     fn set_cursor(&mut self, x: u16, y: u16) -> Result<(), io::Error>;
     fn clear(&mut self) -> Result<(), io::Error>;

helix-tui/src/backend/test.rs

@@ -1,10 +1,10 @@
 use crate::{
     backend::Backend,
     buffer::{Buffer, Cell},
-    layout::Rect,
 };
+use helix_core::unicode::width::UnicodeWidthStr;
+use helix_view::graphics::{CursorKind, Rect};
 use std::{fmt::Write, io};
-use unicode_width::UnicodeWidthStr;
 
 /// A backend used for the integration tests.
 #[derive(Debug)]
@@ -122,7 +122,7 @@ impl Backend for TestBackend {
         Ok(())
     }
 
-    fn show_cursor(&mut self) -> Result<(), io::Error> {
+    fn show_cursor(&mut self, _kind: CursorKind) -> Result<(), io::Error> {
         self.cursor = true;
         Ok(())
     }

helix-tui/src/buffer.rs

@@ -1,11 +1,9 @@
-use crate::{
-    layout::Rect,
-    style::{Color, Modifier, Style},
-    text::{Span, Spans},
-};
+use crate::text::{Span, Spans};
+use helix_core::unicode::width::UnicodeWidthStr;
 use std::cmp::min;
 use unicode_segmentation::UnicodeSegmentation;
-use unicode_width::UnicodeWidthStr;
+
+use helix_view::graphics::{Color, Modifier, Rect, Style};
 
 /// A buffer cell
 #[derive(Debug, Clone, PartialEq)]
@@ -89,8 +87,7 @@ impl Default for Cell {
 ///
 /// ```
 /// use helix_tui::buffer::{Buffer, Cell};
-/// use helix_tui::layout::Rect;
-/// use helix_tui::style::{Color, Style, Modifier};
+/// use helix_view::graphics::{Rect, Color, Style, Modifier};
 ///
 /// let mut buf = Buffer::empty(Rect{x: 0, y: 0, width: 10, height: 5});
 /// buf.get_mut(0, 2).set_symbol("x");
@@ -193,7 +190,7 @@ impl Buffer {
     ///
     /// ```
     /// # use helix_tui::buffer::Buffer;
-    /// # use helix_tui::layout::Rect;
+    /// # use helix_view::graphics::Rect;
     /// let rect = Rect::new(200, 100, 10, 10);
     /// let buffer = Buffer::empty(rect);
     /// // Global coordinates to the top corner of this buffer's area
@@ -203,16 +200,6 @@ impl Buffer {
     /// # Panics
     ///
     /// Panics when given an coordinate that is outside of this Buffer's area.
-    ///
-    /// ```should_panic
-    /// # use helix_tui::buffer::Buffer;
-    /// # use helix_tui::layout::Rect;
-    /// let rect = Rect::new(200, 100, 10, 10);
-    /// let buffer = Buffer::empty(rect);
-    /// // Top coordinate is outside of the buffer in global coordinate space, as the Buffer's area
-    /// // starts at (200, 100).
-    /// buffer.index_of(0, 0); // Panics
-    /// ```
     pub fn index_of(&self, x: u16, y: u16) -> usize {
         debug_assert!(
             x >= self.area.left()
@@ -235,7 +222,7 @@ impl Buffer {
     ///
     /// ```
     /// # use helix_tui::buffer::Buffer;
-    /// # use helix_tui::layout::Rect;
+    /// # use helix_view::graphics::Rect;
     /// let rect = Rect::new(200, 100, 10, 10);
     /// let buffer = Buffer::empty(rect);
     /// assert_eq!(buffer.pos_of(0), (200, 100));
@@ -245,15 +232,6 @@ impl Buffer {
     /// # Panics
     ///
     /// Panics when given an index that is outside the Buffer's content.
-    ///
-    /// ```should_panic
-    /// # use helix_tui::buffer::Buffer;
-    /// # use helix_tui::layout::Rect;
-    /// let rect = Rect::new(0, 0, 10, 10); // 100 cells in total
-    /// let buffer = Buffer::empty(rect);
-    /// // Index 100 is the 101th cell, which lies outside of the area of this Buffer.
-    /// buffer.pos_of(100); // Panics
-    /// ```
     pub fn pos_of(&self, i: usize) -> (u16, u16) {
         debug_assert!(
             i < self.content.len(),
@@ -285,11 +263,30 @@ impl Buffer {
         width: usize,
         style: Style,
     ) -> (u16, u16)
+    where
+        S: AsRef<str>,
+    {
+        self.set_string_truncated(x, y, string, width, style, false)
+    }
+
+    /// Print at most the first `width` characters of a string if enough space is available
+    /// until the end of the line. If `markend` is true appends a `…` at the end of
+    /// truncated lines.
+    pub fn set_string_truncated<S>(
+        &mut self,
+        x: u16,
+        y: u16,
+        string: S,
+        width: usize,
+        style: Style,
+        ellipsis: bool,
+    ) -> (u16, u16)
     where
         S: AsRef<str>,
     {
         let mut index = self.index_of(x, y);
         let mut x_offset = x as usize;
+        let width = if ellipsis { width - 1 } else { width };
         let graphemes = UnicodeSegmentation::graphemes(string.as_ref(), true);
         let max_offset = min(self.area.right() as usize, width.saturating_add(x as usize));
         for s in graphemes {
@@ -312,6 +309,9 @@ impl Buffer {
             index += width;
             x_offset += width;
         }
+        if ellipsis && x_offset - (x as usize) < string.as_ref().width() {
+            self.content[index].set_symbol("…");
+        }
         (x_offset as u16, y)
     }
 
@@ -510,6 +510,7 @@ mod tests {
 
     #[test]
     #[should_panic(expected = "outside the buffer")]
+    #[cfg(debug_assertions)]
     fn pos_of_panics_on_out_of_bounds() {
         let rect = Rect::new(0, 0, 10, 10);
         let buf = Buffer::empty(rect);
@@ -520,6 +521,7 @@ mod tests {
 
     #[test]
     #[should_panic(expected = "outside the buffer")]
+    #[cfg(debug_assertions)]
     fn index_of_panics_on_out_of_bounds() {
         let rect = Rect::new(0, 0, 10, 10);
         let buf = Buffer::empty(rect);

helix-tui/src/layout.rs

@@ -1,11 +1,12 @@
 use std::cell::RefCell;
-use std::cmp::{max, min};
 use std::collections::HashMap;
 
 use cassowary::strength::{REQUIRED, WEAK};
 use cassowary::WeightedRelation::*;
 use cassowary::{Constraint as CassowaryConstraint, Expression, Solver, Variable};
 
+use helix_view::graphics::{Margin, Rect};
+
 #[derive(Debug, Hash, Clone, Copy, PartialEq, Eq)]
 pub enum Corner {
     TopLeft,
@@ -45,12 +46,6 @@ impl Constraint {
     }
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct Margin {
-    pub vertical: u16,
-    pub horizontal: u16,
-}
-
 #[derive(Debug, Clone, Copy, PartialEq)]
 pub enum Alignment {
     Left,
@@ -119,7 +114,8 @@ impl Layout {
     ///
     /// # Examples
     /// ```
-    /// # use helix_tui::layout::{Rect, Constraint, Direction, Layout};
+    /// # use helix_tui::layout::{Constraint, Direction, Layout};
+    /// # use helix_view::graphics::Rect;
     /// let chunks = Layout::default()
     ///     .direction(Direction::Vertical)
     ///     .constraints([Constraint::Length(5), Constraint::Min(0)].as_ref())
@@ -348,117 +344,6 @@ impl Element {
     }
 }
 
-/// A simple rectangle used in the computation of the layout and to give widgets an hint about the
-/// area they are supposed to render to.
-#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
-pub struct Rect {
-    pub x: u16,
-    pub y: u16,
-    pub width: u16,
-    pub height: u16,
-}
-
-impl Default for Rect {
-    fn default() -> Rect {
-        Rect {
-            x: 0,
-            y: 0,
-            width: 0,
-            height: 0,
-        }
-    }
-}
-
-impl Rect {
-    /// Creates a new rect, with width and height limited to keep the area under max u16.
-    /// If clipped, aspect ratio will be preserved.
-    pub fn new(x: u16, y: u16, width: u16, height: u16) -> Rect {
-        let max_area = u16::max_value();
-        let (clipped_width, clipped_height) =
-            if u32::from(width) * u32::from(height) > u32::from(max_area) {
-                let aspect_ratio = f64::from(width) / f64::from(height);
-                let max_area_f = f64::from(max_area);
-                let height_f = (max_area_f / aspect_ratio).sqrt();
-                let width_f = height_f * aspect_ratio;
-                (width_f as u16, height_f as u16)
-            } else {
-                (width, height)
-            };
-        Rect {
-            x,
-            y,
-            width: clipped_width,
-            height: clipped_height,
-        }
-    }
-
-    pub fn area(self) -> u16 {
-        self.width * self.height
-    }
-
-    pub fn left(self) -> u16 {
-        self.x
-    }
-
-    pub fn right(self) -> u16 {
-        self.x.saturating_add(self.width)
-    }
-
-    pub fn top(self) -> u16 {
-        self.y
-    }
-
-    pub fn bottom(self) -> u16 {
-        self.y.saturating_add(self.height)
-    }
-
-    pub fn inner(self, margin: &Margin) -> Rect {
-        if self.width < 2 * margin.horizontal || self.height < 2 * margin.vertical {
-            Rect::default()
-        } else {
-            Rect {
-                x: self.x + margin.horizontal,
-                y: self.y + margin.vertical,
-                width: self.width - 2 * margin.horizontal,
-                height: self.height - 2 * margin.vertical,
-            }
-        }
-    }
-
-    pub fn union(self, other: Rect) -> Rect {
-        let x1 = min(self.x, other.x);
-        let y1 = min(self.y, other.y);
-        let x2 = max(self.x + self.width, other.x + other.width);
-        let y2 = max(self.y + self.height, other.y + other.height);
-        Rect {
-            x: x1,
-            y: y1,
-            width: x2 - x1,
-            height: y2 - y1,
-        }
-    }
-
-    pub fn intersection(self, other: Rect) -> Rect {
-        let x1 = max(self.x, other.x);
-        let y1 = max(self.y, other.y);
-        let x2 = min(self.x + self.width, other.x + other.width);
-        let y2 = min(self.y + self.height, other.y + other.height);
-        Rect {
-            x: x1,
-            y: y1,
-            width: x2 - x1,
-            height: y2 - y1,
-        }
-    }
-
-    pub fn intersects(self, other: Rect) -> bool {
-        self.x < other.x + other.width
-            && self.x + self.width > other.x
-            && self.y < other.y + other.height
-            && self.y + self.height > other.y
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -487,48 +372,4 @@ mod tests {
         assert_eq!(target.height, chunks.iter().map(|r| r.height).sum::<u16>());
         chunks.windows(2).for_each(|w| assert!(w[0].y <= w[1].y));
     }
-
-    #[test]
-    fn test_rect_size_truncation() {
-        for width in 256u16..300u16 {
-            for height in 256u16..300u16 {
-                let rect = Rect::new(0, 0, width, height);
-                rect.area(); // Should not panic.
-                assert!(rect.width < width || rect.height < height);
-                // The target dimensions are rounded down so the math will not be too precise
-                // but let's make sure the ratios don't diverge crazily.
-                assert!(
-                    (f64::from(rect.width) / f64::from(rect.height)
-                        - f64::from(width) / f64::from(height))
-                    .abs()
-                        < 1.0
-                )
-            }
-        }
-
-        // One dimension below 255, one above. Area above max u16.
-        let width = 900;
-        let height = 100;
-        let rect = Rect::new(0, 0, width, height);
-        assert_ne!(rect.width, 900);
-        assert_ne!(rect.height, 100);
-        assert!(rect.width < width || rect.height < height);
-    }
-
-    #[test]
-    fn test_rect_size_preservation() {
-        for width in 0..256u16 {
-            for height in 0..256u16 {
-                let rect = Rect::new(0, 0, width, height);
-                rect.area(); // Should not panic.
-                assert_eq!(rect.width, width);
-                assert_eq!(rect.height, height);
-            }
-        }
-
-        // One dimension below 255, one above. Area below max u16.
-        let rect = Rect::new(0, 0, 300, 100);
-        assert_eq!(rect.width, 300);
-        assert_eq!(rect.height, 100);
-    }
 }

helix-tui/src/lib.rs

@@ -44,7 +44,7 @@
 //! implement your own.
 //!
 //! Each widget follows a builder pattern API providing a default configuration along with methods
-//! to customize them. The widget is then rendered using the [`Frame::render_widget`] which take
+//! to customize them. The widget is then rendered using the `Frame::render_widget` which take
 //! your widget instance an area to draw to.
 //!
 //! The following example renders a block of the size of the terminal:
@@ -125,7 +125,6 @@
 pub mod backend;
 pub mod buffer;
 pub mod layout;
-pub mod style;
 pub mod symbols;
 pub mod terminal;
 pub mod text;

helix-tui/src/style.rs

@@ -1,281 +0,0 @@
-//! `style` contains the primitives used to control how your user interface will look.
-
-use bitflags::bitflags;
-
-#[derive(Debug, Clone, Copy, PartialEq)]
-#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
-pub enum Color {
-    Reset,
-    Black,
-    Red,
-    Green,
-    Yellow,
-    Blue,
-    Magenta,
-    Cyan,
-    Gray,
-    DarkGray,
-    LightRed,
-    LightGreen,
-    LightYellow,
-    LightBlue,
-    LightMagenta,
-    LightCyan,
-    White,
-    Rgb(u8, u8, u8),
-    Indexed(u8),
-}
-
-bitflags! {
-    /// Modifier changes the way a piece of text is displayed.
-    ///
-    /// They are bitflags so they can easily be composed.
-    ///
-    /// ## Examples
-    ///
-    /// ```rust
-    /// # use helix_tui::style::Modifier;
-    ///
-    /// let m = Modifier::BOLD | Modifier::ITALIC;
-    /// ```
-    #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
-    pub struct Modifier: u16 {
-        const BOLD              = 0b0000_0000_0001;
-        const DIM               = 0b0000_0000_0010;
-        const ITALIC            = 0b0000_0000_0100;
-        const UNDERLINED        = 0b0000_0000_1000;
-        const SLOW_BLINK        = 0b0000_0001_0000;
-        const RAPID_BLINK       = 0b0000_0010_0000;
-        const REVERSED          = 0b0000_0100_0000;
-        const HIDDEN            = 0b0000_1000_0000;
-        const CROSSED_OUT       = 0b0001_0000_0000;
-    }
-}
-
-/// Style let you control the main characteristics of the displayed elements.
-///
-/// ```rust
-/// # use helix_tui::style::{Color, Modifier, Style};
-/// Style::default()
-///     .fg(Color::Black)
-///     .bg(Color::Green)
-///     .add_modifier(Modifier::ITALIC | Modifier::BOLD);
-/// ```
-///
-/// It represents an incremental change. If you apply the styles S1, S2, S3 to a cell of the
-/// terminal buffer, the style of this cell will be the result of the merge of S1, S2 and S3, not
-/// just S3.
-///
-/// ```rust
-/// # use helix_tui::style::{Color, Modifier, Style};
-/// # use helix_tui::buffer::Buffer;
-/// # use helix_tui::layout::Rect;
-/// let styles = [
-///     Style::default().fg(Color::Blue).add_modifier(Modifier::BOLD | Modifier::ITALIC),
-///     Style::default().bg(Color::Red),
-///     Style::default().fg(Color::Yellow).remove_modifier(Modifier::ITALIC),
-/// ];
-/// let mut buffer = Buffer::empty(Rect::new(0, 0, 1, 1));
-/// for style in &styles {
-///   buffer.get_mut(0, 0).set_style(*style);
-/// }
-/// assert_eq!(
-///     Style {
-///         fg: Some(Color::Yellow),
-///         bg: Some(Color::Red),
-///         add_modifier: Modifier::BOLD,
-///         sub_modifier: Modifier::empty(),
-///     },
-///     buffer.get(0, 0).style(),
-/// );
-/// ```
-///
-/// The default implementation returns a `Style` that does not modify anything. If you wish to
-/// reset all properties until that point use [`Style::reset`].
-///
-/// ```
-/// # use helix_tui::style::{Color, Modifier, Style};
-/// # use helix_tui::buffer::Buffer;
-/// # use helix_tui::layout::Rect;
-/// let styles = [
-///     Style::default().fg(Color::Blue).add_modifier(Modifier::BOLD | Modifier::ITALIC),
-///     Style::reset().fg(Color::Yellow),
-/// ];
-/// let mut buffer = Buffer::empty(Rect::new(0, 0, 1, 1));
-/// for style in &styles {
-///   buffer.get_mut(0, 0).set_style(*style);
-/// }
-/// assert_eq!(
-///     Style {
-///         fg: Some(Color::Yellow),
-///         bg: Some(Color::Reset),
-///         add_modifier: Modifier::empty(),
-///         sub_modifier: Modifier::empty(),
-///     },
-///     buffer.get(0, 0).style(),
-/// );
-/// ```
-#[derive(Debug, Clone, Copy, PartialEq)]
-#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
-pub struct Style {
-    pub fg: Option<Color>,
-    pub bg: Option<Color>,
-    pub add_modifier: Modifier,
-    pub sub_modifier: Modifier,
-}
-
-impl Default for Style {
-    fn default() -> Style {
-        Style {
-            fg: None,
-            bg: None,
-            add_modifier: Modifier::empty(),
-            sub_modifier: Modifier::empty(),
-        }
-    }
-}
-
-impl Style {
-    /// Returns a `Style` resetting all properties.
-    pub fn reset() -> Style {
-        Style {
-            fg: Some(Color::Reset),
-            bg: Some(Color::Reset),
-            add_modifier: Modifier::empty(),
-            sub_modifier: Modifier::all(),
-        }
-    }
-
-    /// Changes the foreground color.
-    ///
-    /// ## Examples
-    ///
-    /// ```rust
-    /// # use helix_tui::style::{Color, Style};
-    /// let style = Style::default().fg(Color::Blue);
-    /// let diff = Style::default().fg(Color::Red);
-    /// assert_eq!(style.patch(diff), Style::default().fg(Color::Red));
-    /// ```
-    pub fn fg(mut self, color: Color) -> Style {
-        self.fg = Some(color);
-        self
-    }
-
-    /// Changes the background color.
-    ///
-    /// ## Examples
-    ///
-    /// ```rust
-    /// # use helix_tui::style::{Color, Style};
-    /// let style = Style::default().bg(Color::Blue);
-    /// let diff = Style::default().bg(Color::Red);
-    /// assert_eq!(style.patch(diff), Style::default().bg(Color::Red));
-    /// ```
-    pub fn bg(mut self, color: Color) -> Style {
-        self.bg = Some(color);
-        self
-    }
-
-    /// Changes the text emphasis.
-    ///
-    /// When applied, it adds the given modifier to the `Style` modifiers.
-    ///
-    /// ## Examples
-    ///
-    /// ```rust
-    /// # use helix_tui::style::{Color, Modifier, Style};
-    /// let style = Style::default().add_modifier(Modifier::BOLD);
-    /// let diff = Style::default().add_modifier(Modifier::ITALIC);
-    /// let patched = style.patch(diff);
-    /// assert_eq!(patched.add_modifier, Modifier::BOLD | Modifier::ITALIC);
-    /// assert_eq!(patched.sub_modifier, Modifier::empty());
-    /// ```
-    pub fn add_modifier(mut self, modifier: Modifier) -> Style {
-        self.sub_modifier.remove(modifier);
-        self.add_modifier.insert(modifier);
-        self
-    }
-
-    /// Changes the text emphasis.
-    ///
-    /// When applied, it removes the given modifier from the `Style` modifiers.
-    ///
-    /// ## Examples
-    ///
-    /// ```rust
-    /// # use helix_tui::style::{Color, Modifier, Style};
-    /// let style = Style::default().add_modifier(Modifier::BOLD | Modifier::ITALIC);
-    /// let diff = Style::default().remove_modifier(Modifier::ITALIC);
-    /// let patched = style.patch(diff);
-    /// assert_eq!(patched.add_modifier, Modifier::BOLD);
-    /// assert_eq!(patched.sub_modifier, Modifier::ITALIC);
-    /// ```
-    pub fn remove_modifier(mut self, modifier: Modifier) -> Style {
-        self.add_modifier.remove(modifier);
-        self.sub_modifier.insert(modifier);
-        self
-    }
-
-    /// Results in a combined style that is equivalent to applying the two individual styles to
-    /// a style one after the other.
-    ///
-    /// ## Examples
-    /// ```
-    /// # use helix_tui::style::{Color, Modifier, Style};
-    /// let style_1 = Style::default().fg(Color::Yellow);
-    /// let style_2 = Style::default().bg(Color::Red);
-    /// let combined = style_1.patch(style_2);
-    /// assert_eq!(
-    ///     Style::default().patch(style_1).patch(style_2),
-    ///     Style::default().patch(combined));
-    /// ```
-    pub fn patch(mut self, other: Style) -> Style {
-        self.fg = other.fg.or(self.fg);
-        self.bg = other.bg.or(self.bg);
-
-        self.add_modifier.remove(other.sub_modifier);
-        self.add_modifier.insert(other.add_modifier);
-        self.sub_modifier.remove(other.add_modifier);
-        self.sub_modifier.insert(other.sub_modifier);
-
-        self
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    fn styles() -> Vec<Style> {
-        vec![
-            Style::default(),
-            Style::default().fg(Color::Yellow),
-            Style::default().bg(Color::Yellow),
-            Style::default().add_modifier(Modifier::BOLD),
-            Style::default().remove_modifier(Modifier::BOLD),
-            Style::default().add_modifier(Modifier::ITALIC),
-            Style::default().remove_modifier(Modifier::ITALIC),
-            Style::default().add_modifier(Modifier::ITALIC | Modifier::BOLD),
-            Style::default().remove_modifier(Modifier::ITALIC | Modifier::BOLD),
-        ]
-    }
-
-    #[test]
-    fn combined_patch_gives_same_result_as_individual_patch() {
-        let styles = styles();
-        for &a in &styles {
-            for &b in &styles {
-                for &c in &styles {
-                    for &d in &styles {
-                        let combined = a.patch(b.patch(c.patch(d)));
-
-                        assert_eq!(
-                            Style::default().patch(a).patch(b).patch(c).patch(d),
-                            Style::default().patch(combined)
-                        );
-                    }
-                }
-            }
-        }
-    }
-}

helix-tui/src/terminal.rs

@@ -1,4 +1,5 @@
-use crate::{backend::Backend, buffer::Buffer, layout::Rect};
+use crate::{backend::Backend, buffer::Buffer};
+use helix_view::graphics::{CursorKind, Rect};
 use std::io;
 
 #[derive(Debug, Clone, PartialEq)]
@@ -44,8 +45,8 @@ where
     buffers: [Buffer; 2],
     /// Index of the current buffer in the previous array
     current: usize,
-    /// Whether the cursor is currently hidden
-    hidden_cursor: bool,
+    /// Kind of cursor (hidden or others)
+    cursor_kind: CursorKind,
     /// Viewport
     viewport: Viewport,
 }
@@ -56,8 +57,8 @@ where
 {
     fn drop(&mut self) {
         // Attempt to restore the cursor state
-        if self.hidden_cursor {
-            if let Err(err) = self.show_cursor() {
+        if self.cursor_kind == CursorKind::Hidden {
+            if let Err(err) = self.show_cursor(CursorKind::Block) {
                 eprintln!("Failed to show the cursor: {}", err);
             }
         }
@@ -92,7 +93,7 @@ where
                 Buffer::empty(options.viewport.area),
             ],
             current: 0,
-            hidden_cursor: false,
+            cursor_kind: CursorKind::Block,
             viewport: options.viewport,
         })
     }
@@ -147,7 +148,11 @@ where
 
     /// Synchronizes terminal size, calls the rendering closure, flushes the current internal state
     /// and prepares for the next draw call.
-    pub fn draw(&mut self, cursor_position: Option<(u16, u16)>) -> io::Result<()> {
+    pub fn draw(
+        &mut self,
+        cursor_position: Option<(u16, u16)>,
+        cursor_kind: CursorKind,
+    ) -> io::Result<()> {
         // // Autoresize - otherwise we get glitches if shrinking or potential desync between widgets
         // // and the terminal (if growing), which may OOB.
         // self.autoresize()?;
@@ -162,12 +167,13 @@ where
         // Draw to stdout
         self.flush()?;
 
-        match cursor_position {
-            None => self.hide_cursor()?,
-            Some((x, y)) => {
-                self.show_cursor()?;
+        if let Some((x, y)) = cursor_position {
             self.set_cursor(x, y)?;
         }
+
+        match cursor_kind {
+            CursorKind::Hidden => self.hide_cursor()?,
+            kind => self.show_cursor(kind)?,
         }
 
         // Swap buffers
@@ -179,15 +185,20 @@ where
         Ok(())
     }
 
+    #[inline]
+    pub fn cursor_kind(&self) -> CursorKind {
+        self.cursor_kind
+    }
+
     pub fn hide_cursor(&mut self) -> io::Result<()> {
         self.backend.hide_cursor()?;
-        self.hidden_cursor = true;
+        self.cursor_kind = CursorKind::Hidden;
         Ok(())
     }
 
-    pub fn show_cursor(&mut self) -> io::Result<()> {
-        self.backend.show_cursor()?;
-        self.hidden_cursor = false;
+    pub fn show_cursor(&mut self, kind: CursorKind) -> io::Result<()> {
+        self.backend.show_cursor(kind)?;
+        self.cursor_kind = kind;
         Ok(())
     }
 

helix-tui/src/text.rs

@@ -21,7 +21,7 @@
 //! ```rust
 //! # use helix_tui::widgets::Block;
 //! # use helix_tui::text::{Span, Spans};
-//! # use helix_tui::style::{Color, Style};
+//! # use helix_view::graphics::{Color, Style};
 //! // A simple string with no styling.
 //! // Converted to Spans(vec![
 //! //   Span { content: Cow::Borrowed("My title"), style: Style { .. } }
@@ -46,10 +46,11 @@
 //!     Span::raw(" title"),
 //! ]);
 //! ```
-use crate::style::Style;
+use helix_core::line_ending::str_is_line_ending;
+use helix_core::unicode::width::UnicodeWidthStr;
+use helix_view::graphics::Style;
 use std::borrow::Cow;
 use unicode_segmentation::UnicodeSegmentation;
-use unicode_width::UnicodeWidthStr;
 
 /// A grapheme associated to a style.
 #[derive(Debug, Clone, PartialEq)]
@@ -91,7 +92,7 @@ impl<'a> Span<'a> {
     ///
     /// ```rust
     /// # use helix_tui::text::Span;
-    /// # use helix_tui::style::{Color, Modifier, Style};
+    /// # use helix_view::graphics::{Color, Modifier, Style};
     /// let style = Style::default().fg(Color::Yellow).add_modifier(Modifier::ITALIC);
     /// Span::styled("My text", style);
     /// Span::styled(String::from("My text"), style);
@@ -120,7 +121,7 @@ impl<'a> Span<'a> {
     ///
     /// ```rust
     /// # use helix_tui::text::{Span, StyledGrapheme};
-    /// # use helix_tui::style::{Color, Modifier, Style};
+    /// # use helix_view::graphics::{Color, Modifier, Style};
     /// # use std::iter::Iterator;
     /// let style = Style::default().fg(Color::Yellow);
     /// let span = Span::styled("Text", style);
@@ -177,7 +178,7 @@ impl<'a> Span<'a> {
                 symbol: g,
                 style: base_style.patch(self.style),
             })
-            .filter(|s| s.symbol != "\n")
+            .filter(|s| !str_is_line_ending(s.symbol))
     }
 }
 
@@ -210,7 +211,7 @@ impl<'a> Spans<'a> {
     ///
     /// ```rust
     /// # use helix_tui::text::{Span, Spans};
-    /// # use helix_tui::style::{Color, Style};
+    /// # use helix_view::graphics::{Color, Style};
     /// let spans = Spans::from(vec![
     ///     Span::styled("My", Style::default().fg(Color::Yellow)),
     ///     Span::raw(" text"),
@@ -264,7 +265,7 @@ impl<'a> From<Spans<'a>> for String {
 ///
 /// ```rust
 /// # use helix_tui::text::Text;
-/// # use helix_tui::style::{Color, Modifier, Style};
+/// # use helix_view::graphics::{Color, Modifier, Style};
 /// let style = Style::default().fg(Color::Yellow).add_modifier(Modifier::ITALIC);
 ///
 /// // An initial two lines of `Text` built from a `&str`
@@ -318,7 +319,7 @@ impl<'a> Text<'a> {
     ///
     /// ```rust
     /// # use helix_tui::text::Text;
-    /// # use helix_tui::style::{Color, Modifier, Style};
+    /// # use helix_view::graphics::{Color, Modifier, Style};
     /// let style = Style::default().fg(Color::Yellow).add_modifier(Modifier::ITALIC);
     /// Text::styled("The first line\nThe second line", style);
     /// Text::styled(String::from("The first line\nThe second line"), style);
@@ -368,7 +369,7 @@ impl<'a> Text<'a> {
     ///
     /// ```rust
     /// # use helix_tui::text::Text;
-    /// # use helix_tui::style::{Color, Modifier, Style};
+    /// # use helix_view::graphics::{Color, Modifier, Style};
     /// let style = Style::default().fg(Color::Yellow).add_modifier(Modifier::ITALIC);
     /// let mut raw_text = Text::raw("The first line\nThe second line");
     /// let styled_text = Text::styled(String::from("The first line\nThe second line"), style);

helix-tui/src/widgets/block.rs

@@ -1,11 +1,10 @@
 use crate::{
     buffer::Buffer,
-    layout::Rect,
-    style::Style,
     symbols::line,
     text::{Span, Spans},
     widgets::{Borders, Widget},
 };
+use helix_view::graphics::{Rect, Style};
 
 #[derive(Debug, Clone, Copy, PartialEq)]
 pub enum BorderType {
@@ -33,7 +32,7 @@ impl BorderType {
 ///
 /// ```
 /// # use helix_tui::widgets::{Block, BorderType, Borders};
-/// # use helix_tui::style::{Style, Color};
+/// # use helix_view::graphics::{Style, Color};
 /// Block::default()
 ///     .title("Block")
 ///     .borders(Borders::LEFT | Borders::RIGHT)
@@ -212,7 +211,6 @@ impl<'a> Widget for Block<'a> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::layout::Rect;
 
     #[test]
     fn inner_takes_into_account_the_borders() {

helix-tui/src/widgets/mod.rs

@@ -1,4 +1,4 @@
-//! `widgets` is a collection of types that implement [`Widget`] or [`StatefulWidget`] or both.
+//! `widgets` is a collection of types that implement [`Widget`].
 //!
 //! All widgets are implemented using the builder pattern and are consumable objects. They are not
 //! meant to be stored but used as *commands* to draw common figures in the UI.
@@ -20,9 +20,11 @@ pub use self::block::{Block, BorderType};
 pub use self::paragraph::{Paragraph, Wrap};
 pub use self::table::{Cell, Row, Table, TableState};
 
-use crate::{buffer::Buffer, layout::Rect};
+use crate::buffer::Buffer;
 use bitflags::bitflags;
 
+use helix_view::graphics::Rect;
+
 bitflags! {
     /// Bitflags that can be composed to set the visible borders essentially on the block widget.
     pub struct Borders: u32 {

helix-tui/src/widgets/paragraph.rs

@@ -1,15 +1,15 @@
 use crate::{
     buffer::Buffer,
-    layout::{Alignment, Rect},
-    style::Style,
+    layout::Alignment,
     text::{StyledGrapheme, Text},
     widgets::{
         reflow::{LineComposer, LineTruncator, WordWrapper},
         Block, Widget,
     },
 };
+use helix_core::unicode::width::UnicodeWidthStr;
+use helix_view::graphics::{Rect, Style};
 use std::iter;
-use unicode_width::UnicodeWidthStr;
 
 fn get_line_offset(line_width: u16, text_area_width: u16, alignment: Alignment) -> u16 {
     match alignment {
@@ -26,8 +26,8 @@ fn get_line_offset(line_width: u16, text_area_width: u16, alignment: Alignment)
 /// ```
 /// # use helix_tui::text::{Text, Spans, Span};
 /// # use helix_tui::widgets::{Block, Borders, Paragraph, Wrap};
-/// # use helix_tui::style::{Style, Color, Modifier};
 /// # use helix_tui::layout::{Alignment};
+/// # use helix_view::graphics::{Style, Color, Modifier};
 /// let text = vec![
 ///     Spans::from(vec![
 ///         Span::raw("First"),

helix-tui/src/widgets/reflow.rs

@@ -1,6 +1,7 @@
 use crate::text::StyledGrapheme;
+use helix_core::line_ending::str_is_line_ending;
+use helix_core::unicode::width::UnicodeWidthStr;
 use unicode_segmentation::UnicodeSegmentation;
-use unicode_width::UnicodeWidthStr;
 
 const NBSP: &str = "\u{00a0}";
 
@@ -62,13 +63,13 @@ impl<'a, 'b> LineComposer<'a> for WordWrapper<'a, 'b> {
             // Ignore characters wider that the total max width.
             if symbol.width() as u16 > self.max_line_width
                 // Skip leading whitespace when trim is enabled.
-                || self.trim && symbol_whitespace && symbol != "\n" && current_line_width == 0
+                || self.trim && symbol_whitespace && !str_is_line_ending(symbol) && current_line_width == 0
             {
                 continue;
             }
 
             // Break on newline and discard it.
-            if symbol == "\n" {
+            if str_is_line_ending(symbol) {
                 if prev_whitespace {
                     current_line_width = width_to_last_word_end;
                     self.current_line.truncate(symbols_to_last_word_end);
@@ -170,7 +171,7 @@ impl<'a, 'b> LineComposer<'a> for LineTruncator<'a, 'b> {
             }
 
             // Break on newline and discard it.
-            if symbol == "\n" {
+            if str_is_line_ending(symbol) {
                 break;
             }
 
@@ -199,7 +200,7 @@ impl<'a, 'b> LineComposer<'a> for LineTruncator<'a, 'b> {
 
         if skip_rest {
             for StyledGrapheme { symbol, .. } in &mut self.symbols {
-                if symbol == "\n" {
+                if str_is_line_ending(symbol) {
                     break;
                 }
             }

helix-tui/src/widgets/table.rs

@@ -1,7 +1,6 @@
 use crate::{
     buffer::Buffer,
-    layout::{Constraint, Rect},
-    style::Style,
+    layout::Constraint,
     text::Text,
     widgets::{Block, Widget},
 };
@@ -10,19 +9,17 @@ use cassowary::{
     WeightedRelation::*,
     {Expression, Solver},
 };
-use std::{
-    collections::HashMap,
-    iter::{self, Iterator},
-};
-use unicode_width::UnicodeWidthStr;
+use helix_core::unicode::width::UnicodeWidthStr;
+use helix_view::graphics::{Rect, Style};
+use std::collections::HashMap;
 
 /// A [`Cell`] contains the [`Text`] to be displayed in a [`Row`] of a [`Table`].
 ///
 /// It can be created from anything that can be converted to a [`Text`].
 /// ```rust
 /// # use helix_tui::widgets::Cell;
-/// # use helix_tui::style::{Style, Modifier};
 /// # use helix_tui::text::{Span, Spans, Text};
+/// # use helix_view::graphics::{Style, Modifier};
 /// Cell::from("simple string");
 ///
 /// Cell::from(Span::from("span"));
@@ -39,7 +36,7 @@ use unicode_width::UnicodeWidthStr;
 /// capabilities of [`Text`].
 #[derive(Debug, Clone, PartialEq, Default)]
 pub struct Cell<'a> {
-    content: Text<'a>,
+    pub content: Text<'a>,
     style: Style,
 }
 
@@ -74,7 +71,7 @@ where
 /// But if you need a bit more control over individual cells, you can explicity create [`Cell`]s:
 /// ```rust
 /// # use helix_tui::widgets::{Row, Cell};
-/// # use helix_tui::style::{Style, Color};
+/// # use helix_view::graphics::{Style, Color};
 /// Row::new(vec![
 ///     Cell::from("Cell1"),
 ///     Cell::from("Cell2").style(Style::default().fg(Color::Yellow)),
@@ -84,7 +81,7 @@ where
 /// By default, a row has a height of 1 but you can change this using [`Row::height`].
 #[derive(Debug, Clone, PartialEq, Default)]
 pub struct Row<'a> {
-    cells: Vec<Cell<'a>>,
+    pub cells: Vec<Cell<'a>>,
     height: u16,
     style: Style,
     bottom_margin: u16,
@@ -137,7 +134,7 @@ impl<'a> Row<'a> {
 /// ```rust
 /// # use helix_tui::widgets::{Block, Borders, Table, Row, Cell};
 /// # use helix_tui::layout::Constraint;
-/// # use helix_tui::style::{Style, Color, Modifier};
+/// # use helix_view::graphics::{Style, Color, Modifier};
 /// # use helix_tui::text::{Text, Spans, Span};
 /// Table::new(vec![
 ///     // Row can be created from simple strings.
@@ -415,9 +412,7 @@ impl<'a> Table<'a> {
         let has_selection = state.selected.is_some();
         let columns_widths = self.get_columns_widths(table_area.width, has_selection);
         let highlight_symbol = self.highlight_symbol.unwrap_or("");
-        let blank_symbol = iter::repeat(" ")
-            .take(highlight_symbol.width())
-            .collect::<String>();
+        let blank_symbol = " ".repeat(highlight_symbol.width());
         let mut current_height = 0;
         let mut rows_height = table_area.height;
 

helix-view/Cargo.toml

@@ -1,31 +1,45 @@
 [package]
 name = "helix-view"
-version = "0.2.0"
+version = "0.4.1"
 authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
 edition = "2018"
 license = "MPL-2.0"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+description = "UI abstractions for use in backends"
+categories = ["editor"]
+repository = "https://github.com/helix-editor/helix"
+homepage = "https://helix-editor.com"
 
 [features]
-term = ["tui", "crossterm"]
-default = ["term"]
+default = []
+term = ["crossterm"]
 
 [dependencies]
+bitflags = "1.0"
 anyhow = "1"
-helix-core = { path = "../helix-core" }
-helix-lsp = { path = "../helix-lsp"}
+helix-core = { version = "0.4", path = "../helix-core" }
+helix-lsp = { version = "0.4", path = "../helix-lsp"}
+crossterm = { version = "0.20", optional = true }
 
 # Conversion traits
-tui = { path = "../helix-tui", package = "helix-tui", default-features = false, features = ["crossterm"], optional = true }
-crossterm = { version = "0.20", features = ["event-stream"], optional = true }
 once_cell = "1.8"
 url = "2"
 
-tokio = { version = "1", features = ["full"] }
+tokio = { version = "1", features = ["rt", "rt-multi-thread", "io-util", "io-std", "time", "process", "macros", "fs", "parking_lot"] }
+futures-util = { version = "0.3", features = ["std", "async-await"], default-features = false }
 
 slotmap = "1"
 
+encoding_rs = "0.8"
+chardetng = "0.1"
+
 serde = { version = "1.0", features = ["derive"] }
 toml = "0.5"
 log = "~0.4"
+
+which = "4.2"
+
+[target.'cfg(windows)'.dependencies]
+clipboard-win = { version = "4.2", features = ["std"] }
+
+[dev-dependencies]
+helix-tui = { path = "../helix-tui" }

helix-view/src/clipboard.rs

@@ -0,0 +1,318 @@
+// Implementation reference: https://github.com/neovim/neovim/blob/f2906a4669a2eef6d7bf86a29648793d63c98949/runtime/autoload/provider/clipboard.vim#L68-L152
+
+use anyhow::Result;
+use std::borrow::Cow;
+
+pub enum ClipboardType {
+    Clipboard,
+    Selection,
+}
+
+pub trait ClipboardProvider: std::fmt::Debug {
+    fn name(&self) -> Cow<str>;
+    fn get_contents(&self, clipboard_type: ClipboardType) -> Result<String>;
+    fn set_contents(&mut self, contents: String, clipboard_type: ClipboardType) -> Result<()>;
+}
+
+macro_rules! command_provider {
+    (paste => $get_prg:literal $( , $get_arg:literal )* ; copy => $set_prg:literal $( , $set_arg:literal )* ; ) => {{
+        Box::new(provider::CommandProvider {
+            get_cmd: provider::CommandConfig {
+                prg: $get_prg,
+                args: &[ $( $get_arg ),* ],
+            },
+            set_cmd: provider::CommandConfig {
+                prg: $set_prg,
+                args: &[ $( $set_arg ),* ],
+            },
+            get_primary_cmd: None,
+            set_primary_cmd: None,
+        })
+    }};
+
+    (paste => $get_prg:literal $( , $get_arg:literal )* ;
+     copy => $set_prg:literal $( , $set_arg:literal )* ;
+     primary_paste => $pr_get_prg:literal $( , $pr_get_arg:literal )* ;
+     primary_copy => $pr_set_prg:literal $( , $pr_set_arg:literal )* ;
+    ) => {{
+        Box::new(provider::CommandProvider {
+            get_cmd: provider::CommandConfig {
+                prg: $get_prg,
+                args: &[ $( $get_arg ),* ],
+            },
+            set_cmd: provider::CommandConfig {
+                prg: $set_prg,
+                args: &[ $( $set_arg ),* ],
+            },
+            get_primary_cmd: Some(provider::CommandConfig {
+                prg: $pr_get_prg,
+                args: &[ $( $pr_get_arg ),* ],
+            }),
+            set_primary_cmd: Some(provider::CommandConfig {
+                prg: $pr_set_prg,
+                args: &[ $( $pr_set_arg ),* ],
+            }),
+        })
+    }};
+}
+
+pub fn get_clipboard_provider() -> Box<dyn ClipboardProvider> {
+    // TODO: support for user-defined provider, probably when we have plugin support by setting a
+    // variable?
+
+    if exists("pbcopy") && exists("pbpaste") {
+        command_provider! {
+            paste => "pbpaste";
+            copy => "pbcopy";
+        }
+    } else if env_var_is_set("WAYLAND_DISPLAY") && exists("wl-copy") && exists("wl-paste") {
+        command_provider! {
+            paste => "wl-paste", "--no-newline";
+            copy => "wl-copy", "--type", "text/plain";
+            primary_paste => "wl-paste", "-p", "--no-newline";
+            primary_copy => "wl-copy", "-p", "--type", "text/plain";
+        }
+    } else if env_var_is_set("DISPLAY") && exists("xclip") {
+        command_provider! {
+            paste => "xclip", "-o", "-selection", "clipboard";
+            copy => "xclip", "-i", "-selection", "clipboard";
+            primary_paste => "xclip", "-o";
+            primary_copy => "xclip", "-i";
+        }
+    } else if env_var_is_set("DISPLAY") && exists("xsel") && is_exit_success("xsel", &["-o", "-b"])
+    {
+        // FIXME: check performance of is_exit_success
+        command_provider! {
+            paste => "xsel", "-o", "-b";
+            copy => "xsel", "--nodetach", "-i", "-b";
+            primary_paste => "xsel", "-o";
+            primary_copy => "xsel", "-i";
+        }
+    } else if exists("lemonade") {
+        command_provider! {
+            paste => "lemonade", "paste";
+            copy => "lemonade", "copy";
+        }
+    } else if exists("doitclient") {
+        command_provider! {
+            paste => "doitclient", "wclip", "-r";
+            copy => "doitclient", "wclip";
+        }
+    } else if exists("win32yank.exe") {
+        // FIXME: does it work within WSL?
+        command_provider! {
+            paste => "win32yank.exe", "-o", "--lf";
+            copy => "win32yank.exe", "-i", "--crlf";
+        }
+    } else if exists("termux-clipboard-set") && exists("termux-clipboard-get") {
+        command_provider! {
+            paste => "termux-clipboard-get";
+            copy => "termux-clipboard-set";
+        }
+    } else if env_var_is_set("TMUX") && exists("tmux") {
+        command_provider! {
+            paste => "tmux", "save-buffer", "-";
+            copy => "tmux", "load-buffer", "-";
+        }
+    } else {
+        #[cfg(target_os = "windows")]
+        return Box::new(provider::WindowsProvider::new());
+
+        #[cfg(not(target_os = "windows"))]
+        return Box::new(provider::NopProvider::new());
+    }
+}
+
+fn exists(executable_name: &str) -> bool {
+    which::which(executable_name).is_ok()
+}
+
+fn env_var_is_set(env_var_name: &str) -> bool {
+    std::env::var_os(env_var_name).is_some()
+}
+
+fn is_exit_success(program: &str, args: &[&str]) -> bool {
+    std::process::Command::new(program)
+        .args(args)
+        .output()
+        .ok()
+        .and_then(|out| out.status.success().then(|| ())) // TODO: use then_some when stabilized
+        .is_some()
+}
+
+mod provider {
+    use super::{ClipboardProvider, ClipboardType};
+    use anyhow::{bail, Context as _, Result};
+    use std::borrow::Cow;
+
+    #[derive(Debug)]
+    pub struct NopProvider {
+        buf: String,
+        primary_buf: String,
+    }
+
+    impl NopProvider {
+        pub fn new() -> Self {
+            Self {
+                buf: String::new(),
+                primary_buf: String::new(),
+            }
+        }
+    }
+
+    impl ClipboardProvider for NopProvider {
+        fn name(&self) -> Cow<str> {
+            Cow::Borrowed("none")
+        }
+
+        fn get_contents(&self, clipboard_type: ClipboardType) -> Result<String> {
+            let value = match clipboard_type {
+                ClipboardType::Clipboard => self.buf.clone(),
+                ClipboardType::Selection => self.primary_buf.clone(),
+            };
+
+            Ok(value)
+        }
+
+        fn set_contents(&mut self, content: String, clipboard_type: ClipboardType) -> Result<()> {
+            match clipboard_type {
+                ClipboardType::Clipboard => self.buf = content,
+                ClipboardType::Selection => self.primary_buf = content,
+            }
+            Ok(())
+        }
+    }
+
+    #[cfg(target_os = "windows")]
+    #[derive(Debug)]
+    pub struct WindowsProvider {
+        selection_buf: String,
+    }
+
+    #[cfg(target_os = "windows")]
+    impl WindowsProvider {
+        pub fn new() -> Self {
+            Self {
+                selection_buf: String::new(),
+            }
+        }
+    }
+
+    #[cfg(target_os = "windows")]
+    impl ClipboardProvider for WindowsProvider {
+        fn name(&self) -> Cow<str> {
+            Cow::Borrowed("clipboard-win")
+        }
+
+        fn get_contents(&self, clipboard_type: ClipboardType) -> Result<String> {
+            match clipboard_type {
+                ClipboardType::Clipboard => {
+                    let contents = clipboard_win::get_clipboard(clipboard_win::formats::Unicode)?;
+                    Ok(contents)
+                }
+                ClipboardType::Selection => Ok(String::new()),
+            }
+        }
+
+        fn set_contents(&mut self, contents: String, clipboard_type: ClipboardType) -> Result<()> {
+            match clipboard_type {
+                ClipboardType::Clipboard => {
+                    clipboard_win::set_clipboard(clipboard_win::formats::Unicode, contents);
+                }
+                ClipboardType::Selection => {}
+            };
+            Ok(())
+        }
+    }
+
+    #[derive(Debug)]
+    pub struct CommandConfig {
+        pub prg: &'static str,
+        pub args: &'static [&'static str],
+    }
+
+    impl CommandConfig {
+        fn execute(&self, input: Option<&str>, pipe_output: bool) -> Result<Option<String>> {
+            use std::io::Write;
+            use std::process::{Command, Stdio};
+
+            let stdin = input.map(|_| Stdio::piped()).unwrap_or_else(Stdio::null);
+            let stdout = pipe_output.then(Stdio::piped).unwrap_or_else(Stdio::null);
+
+            let mut child = Command::new(self.prg)
+                .args(self.args)
+                .stdin(stdin)
+                .stdout(stdout)
+                .stderr(Stdio::null())
+                .spawn()?;
+
+            if let Some(input) = input {
+                let mut stdin = child.stdin.take().context("stdin is missing")?;
+                stdin
+                    .write_all(input.as_bytes())
+                    .context("couldn't write in stdin")?;
+            }
+
+            // TODO: add timer?
+            let output = child.wait_with_output()?;
+
+            if !output.status.success() {
+                bail!("clipboard provider {} failed", self.prg);
+            }
+
+            if pipe_output {
+                Ok(Some(String::from_utf8(output.stdout)?))
+            } else {
+                Ok(None)
+            }
+        }
+    }
+
+    #[derive(Debug)]
+    pub struct CommandProvider {
+        pub get_cmd: CommandConfig,
+        pub set_cmd: CommandConfig,
+        pub get_primary_cmd: Option<CommandConfig>,
+        pub set_primary_cmd: Option<CommandConfig>,
+    }
+
+    impl ClipboardProvider for CommandProvider {
+        fn name(&self) -> Cow<str> {
+            if self.get_cmd.prg != self.set_cmd.prg {
+                Cow::Owned(format!("{}+{}", self.get_cmd.prg, self.set_cmd.prg))
+            } else {
+                Cow::Borrowed(self.get_cmd.prg)
+            }
+        }
+
+        fn get_contents(&self, clipboard_type: ClipboardType) -> Result<String> {
+            match clipboard_type {
+                ClipboardType::Clipboard => Ok(self
+                    .get_cmd
+                    .execute(None, true)?
+                    .context("output is missing")?),
+                ClipboardType::Selection => {
+                    if let Some(cmd) = &self.get_primary_cmd {
+                        return cmd.execute(None, true)?.context("output is missing");
+                    }
+
+                    Ok(String::new())
+                }
+            }
+        }
+
+        fn set_contents(&mut self, value: String, clipboard_type: ClipboardType) -> Result<()> {
+            let cmd = match clipboard_type {
+                ClipboardType::Clipboard => &self.set_cmd,
+                ClipboardType::Selection => {
+                    if let Some(cmd) = &self.set_primary_cmd {
+                        cmd
+                    } else {
+                        return Ok(());
+                    }
+                }
+            };
+            cmd.execute(Some(&value), false).map(|_| ())
+        }
+    }
+}

helix-view/src/document.rs

@@ -1,18 +1,26 @@
-use anyhow::{Context, Error};
+use anyhow::{anyhow, Context, Error};
+use serde::de::{self, Deserialize, Deserializer};
 use std::cell::Cell;
+use std::collections::HashMap;
+use std::fmt::Display;
 use std::future::Future;
 use std::path::{Component, Path, PathBuf};
+use std::str::FromStr;
 use std::sync::Arc;
 
 use helix_core::{
     history::History,
-    syntax::{LanguageConfiguration, LOADER},
-    ChangeSet, Diagnostic, Rope, Selection, State, Syntax, Transaction,
+    indent::{auto_detect_indent_style, IndentStyle},
+    line_ending::auto_detect_line_ending,
+    syntax::{self, LanguageConfiguration},
+    ChangeSet, Diagnostic, LineEnding, Rope, RopeBuilder, Selection, State, Syntax, Transaction,
+    DEFAULT_LINE_ENDING,
 };
+use helix_lsp::util::LspFormatting;
 
-use crate::{DocumentId, ViewId};
+use crate::{DocumentId, Theme, ViewId};
 
-use std::collections::HashMap;
+const BUF_SIZE: usize = 8192;
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 pub enum Mode {
@@ -21,18 +29,58 @@ pub enum Mode {
     Insert,
 }
 
+impl Display for Mode {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Mode::Normal => f.write_str("normal"),
+            Mode::Select => f.write_str("select"),
+            Mode::Insert => f.write_str("insert"),
+        }
+    }
+}
+
+impl FromStr for Mode {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "normal" => Ok(Mode::Normal),
+            "select" => Ok(Mode::Select),
+            "insert" => Ok(Mode::Insert),
+            _ => Err(anyhow!("Invalid mode '{}'", s)),
+        }
+    }
+}
+
+// toml deserializer doesn't seem to recognize string as enum
+impl<'de> Deserialize<'de> for Mode {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        s.parse().map_err(de::Error::custom)
+    }
+}
+
 pub struct Document {
-    // rope + selection
     pub(crate) id: DocumentId,
     text: Rope,
     pub(crate) selections: HashMap<ViewId, Selection>,
 
     path: Option<PathBuf>,
+    encoding: &'static encoding_rs::Encoding,
 
     /// Current editing mode.
     pub mode: Mode,
     pub restore_cursor: bool,
 
+    /// Current indent style.
+    pub indent_style: IndentStyle,
+
+    /// The document's default line ending.
+    pub line_ending: LineEnding,
+
     syntax: Option<Syntax>,
     // /// Corresponding language scope name. Usually `source.<lang>`.
     pub(crate) language: Option<Arc<LanguageConfiguration>>,
@@ -61,6 +109,7 @@ impl fmt::Debug for Document {
             .field("text", &self.text)
             .field("selections", &self.selections)
             .field("path", &self.path)
+            .field("encoding", &self.encoding)
             .field("mode", &self.mode)
             .field("restore_cursor", &self.restore_cursor)
             .field("syntax", &self.syntax)
@@ -76,6 +125,181 @@ impl fmt::Debug for Document {
     }
 }
 
+// The documentation and implementation of this function should be up-to-date with
+// its sibling function, `to_writer()`.
+//
+/// Decodes a stream of bytes into UTF-8, returning a `Rope` and the
+/// encoding it was decoded as. The optional `encoding` parameter can
+/// be used to override encoding auto-detection.
+pub fn from_reader<R: std::io::Read + ?Sized>(
+    reader: &mut R,
+    encoding: Option<&'static encoding_rs::Encoding>,
+) -> Result<(Rope, &'static encoding_rs::Encoding), Error> {
+    // These two buffers are 8192 bytes in size each and are used as
+    // intermediaries during the decoding process. Text read into `buf`
+    // from `reader` is decoded into `buf_out` as UTF-8. Once either
+    // `buf_out` is full or the end of the reader was reached, the
+    // contents are appended to `builder`.
+    let mut buf = [0u8; BUF_SIZE];
+    let mut buf_out = [0u8; BUF_SIZE];
+    let mut builder = RopeBuilder::new();
+
+    // By default, the encoding of the text is auto-detected via the
+    // `chardetng` crate which requires sample data from the reader.
+    // As a manual override to this auto-detection is possible, the
+    // same data is read into `buf` to ensure symmetry in the upcoming
+    // loop.
+    let (encoding, mut decoder, mut slice, mut is_empty) = {
+        let read = reader.read(&mut buf)?;
+        let is_empty = read == 0;
+        let encoding = encoding.unwrap_or_else(|| {
+            let mut encoding_detector = chardetng::EncodingDetector::new();
+            encoding_detector.feed(&buf, is_empty);
+            encoding_detector.guess(None, true)
+        });
+        let decoder = encoding.new_decoder();
+
+        // If the amount of bytes read from the reader is less than
+        // `buf.len()`, it is undesirable to read the bytes afterwards.
+        let slice = &buf[..read];
+        (encoding, decoder, slice, is_empty)
+    };
+
+    // `RopeBuilder::append()` expects a `&str`, so this is the "real"
+    // output buffer. When decoding, the number of bytes in the output
+    // buffer will often exceed the number of bytes in the input buffer.
+    // The `result` returned by `decode_to_str()` will state whether or
+    // not that happened. The contents of `buf_str` is appended to
+    // `builder` and it is reused for the next iteration of the decoding
+    // loop.
+    //
+    // As it is possible to read less than the buffer's maximum from `read()`
+    // even when the end of the reader has yet to be reached, the end of
+    // the reader is determined only when a `read()` call returns `0`.
+    //
+    // SAFETY: `buf_out` is a zero-initialized array, thus it will always
+    // contain valid UTF-8.
+    let buf_str = unsafe { std::str::from_utf8_unchecked_mut(&mut buf_out[..]) };
+    let mut total_written = 0usize;
+    loop {
+        let mut total_read = 0usize;
+
+        // An inner loop is necessary as it is possible that the input buffer
+        // may not be completely decoded on the first `decode_to_str()` call
+        // which would happen in cases where the output buffer is filled to
+        // capacity.
+        loop {
+            let (result, read, written, ..) = decoder.decode_to_str(
+                &slice[total_read..],
+                &mut buf_str[total_written..],
+                is_empty,
+            );
+
+            // These variables act as the read and write cursors of `buf` and `buf_str` respectively.
+            // They are necessary in case the output buffer fills before decoding of the entire input
+            // loop is complete. Otherwise, the loop would endlessly iterate over the same `buf` and
+            // the data inside the output buffer would be overwritten.
+            total_read += read;
+            total_written += written;
+            match result {
+                encoding_rs::CoderResult::InputEmpty => {
+                    debug_assert_eq!(slice.len(), total_read);
+                    break;
+                }
+                encoding_rs::CoderResult::OutputFull => {
+                    debug_assert!(slice.len() > total_read);
+                    builder.append(&buf_str[..total_written]);
+                    total_written = 0;
+                }
+            }
+        }
+        // Once the end of the stream is reached, the output buffer is
+        // flushed and the loop terminates.
+        if is_empty {
+            debug_assert_eq!(reader.read(&mut buf)?, 0);
+            builder.append(&buf_str[..total_written]);
+            break;
+        }
+
+        // Once the previous input has been processed and decoded, the next set of
+        // data is fetched from the reader. The end of the reader is determined to
+        // be when exactly `0` bytes were read from the reader, as per the invariants
+        // of the `Read` trait.
+        let read = reader.read(&mut buf)?;
+        slice = &buf[..read];
+        is_empty = read == 0;
+    }
+    let rope = builder.finish();
+    Ok((rope, encoding))
+}
+
+// The documentation and implementation of this function should be up-to-date with
+// its sibling function, `from_reader()`.
+//
+/// Encodes the text inside `rope` into the given `encoding` and writes the
+/// encoded output into `writer.` As a `Rope` can only contain valid UTF-8,
+/// replacement characters may appear in the encoded text.
+pub async fn to_writer<'a, W: tokio::io::AsyncWriteExt + Unpin + ?Sized>(
+    writer: &'a mut W,
+    encoding: &'static encoding_rs::Encoding,
+    rope: &'a Rope,
+) -> Result<(), Error> {
+    // Text inside a `Rope` is stored as non-contiguous blocks of data called
+    // chunks. The absolute size of each chunk is unknown, thus it is impossible
+    // to predict the end of the chunk iterator ahead of time. Instead, it is
+    // determined by filtering the iterator to remove all empty chunks and then
+    // appending an empty chunk to it. This is valuable for detecting when all
+    // chunks in the `Rope` have been iterated over in the subsequent loop.
+    let iter = rope
+        .chunks()
+        .filter(|c| !c.is_empty())
+        .chain(std::iter::once(""));
+    let mut buf = [0u8; BUF_SIZE];
+    let mut encoder = encoding.new_encoder();
+    let mut total_written = 0usize;
+    for chunk in iter {
+        let is_empty = chunk.is_empty();
+        let mut total_read = 0usize;
+
+        // An inner loop is necessary as it is possible that the input buffer
+        // may not be completely encoded on the first `encode_from_utf8()` call
+        // which would happen in cases where the output buffer is filled to
+        // capacity.
+        loop {
+            let (result, read, written, ..) =
+                encoder.encode_from_utf8(&chunk[total_read..], &mut buf[total_written..], is_empty);
+
+            // These variables act as the read and write cursors of `chunk` and `buf` respectively.
+            // They are necessary in case the output buffer fills before encoding of the entire input
+            // loop is complete. Otherwise, the loop would endlessly iterate over the same `chunk` and
+            // the data inside the output buffer would be overwritten.
+            total_read += read;
+            total_written += written;
+            match result {
+                encoding_rs::CoderResult::InputEmpty => {
+                    debug_assert_eq!(chunk.len(), total_read);
+                    debug_assert!(buf.len() >= total_written);
+                    break;
+                }
+                encoding_rs::CoderResult::OutputFull => {
+                    debug_assert!(chunk.len() > total_read);
+                    writer.write_all(&buf[..total_written]).await?;
+                    total_written = 0;
+                }
+            }
+        }
+
+        // Once the end of the iterator is reached, the output buffer is
+        // flushed and the outer loop terminates.
+        if is_empty {
+            writer.write_all(&buf[..total_written]).await?;
+            writer.flush().await?;
+            break;
+        }
+    }
+    Ok(())
+}
+
 /// Like std::mem::replace() except it allows the replacement value to be mapped from the
 /// original value.
 fn take_with<T, F>(mut_ref: &mut T, closure: F)
@@ -92,6 +316,36 @@ where
     }
 }
 
+/// Expands tilde `~` into users home directory if avilable, otherwise returns the path
+/// unchanged. The tilde will only be expanded when present as the first component of the path
+/// and only slash follows it.
+pub fn expand_tilde(path: &Path) -> PathBuf {
+    let mut components = path.components().peekable();
+    if let Some(Component::Normal(c)) = components.peek() {
+        if c == &"~" {
+            if let Ok(home) = helix_core::home_dir() {
+                // it's ok to unwrap, the path starts with `~`
+                return home.join(path.strip_prefix("~").unwrap());
+            }
+        }
+    }
+
+    path.to_path_buf()
+}
+
+/// Replaces users home directory from `path` with tilde `~` if the directory
+/// is available, otherwise returns the path unchanged.
+pub fn fold_home_dir(path: &Path) -> PathBuf {
+    if let Ok(home) = helix_core::home_dir() {
+        if path.starts_with(&home) {
+            // it's ok to unwrap, the path starts with home dir
+            return PathBuf::from("~").join(path.strip_prefix(&home).unwrap());
+        }
+    }
+
+    path.to_path_buf()
+}
+
 /// Normalize a path, removing things like `.` and `..`.
 ///
 /// CAUTION: This does not resolve symlinks (unlike
@@ -100,8 +354,9 @@ where
 /// [`std::fs::canonicalize`] can be hard to use correctly, since it can often
 /// fail, or on Windows returns annoying device paths. This is a problem Cargo
 /// needs to improve on.
-/// Copied from cargo: https://github.com/rust-lang/cargo/blob/070e459c2d8b79c5b2ac5218064e7603329c92ae/crates/cargo-util/src/paths.rs#L81
+/// Copied from cargo: <https://github.com/rust-lang/cargo/blob/070e459c2d8b79c5b2ac5218064e7603329c92ae/crates/cargo-util/src/paths.rs#L81>
 pub fn normalize_path(path: &Path) -> PathBuf {
+    let path = expand_tilde(path);
     let mut components = path.components().peekable();
     let mut ret = if let Some(c @ Component::Prefix(..)) = components.peek().cloned() {
         components.next();
@@ -128,27 +383,36 @@ pub fn normalize_path(path: &Path) -> PathBuf {
     ret
 }
 
-// Returns the canonical, absolute form of a path with all intermediate components normalized.
-//
-// This function is used instead of `std::fs::canonicalize` because we don't want to verify
-// here if the path exists, just normalize it's components.
+/// Returns the canonical, absolute form of a path with all intermediate components normalized.
+///
+/// This function is used instead of `std::fs::canonicalize` because we don't want to verify
+/// here if the path exists, just normalize it's components.
 pub fn canonicalize_path(path: &Path) -> std::io::Result<PathBuf> {
-    std::env::current_dir().map(|current_dir| normalize_path(&current_dir.join(path)))
+    let path = if path.is_relative() {
+        std::env::current_dir().map(|current_dir| current_dir.join(path))?
+    } else {
+        path.to_path_buf()
+    };
+
+    Ok(normalize_path(&path))
 }
 
 use helix_lsp::lsp;
 use url::Url;
 
 impl Document {
-    pub fn new(text: Rope) -> Self {
+    pub fn from(text: Rope, encoding: Option<&'static encoding_rs::Encoding>) -> Self {
+        let encoding = encoding.unwrap_or(encoding_rs::UTF_8);
         let changes = ChangeSet::new(&text);
         let old_state = None;
 
         Self {
             id: DocumentId::default(),
             path: None,
+            encoding,
             text,
             selections: HashMap::default(),
+            indent_style: IndentStyle::Spaces(4),
             mode: Mode::Normal,
             restore_cursor: false,
             syntax: None,
@@ -160,76 +424,115 @@ impl Document {
             history: Cell::new(History::default()),
             last_saved_revision: 0,
             language_server: None,
+            line_ending: DEFAULT_LINE_ENDING,
         }
     }
 
     // TODO: async fn?
-    pub fn load(path: PathBuf) -> Result<Self, Error> {
-        use std::{fs::File, io::BufReader};
-
-        let doc = if !path.exists() {
-            Rope::from("\n")
+    /// Create a new document from `path`. Encoding is auto-detected, but it can be manually
+    /// overwritten with the `encoding` parameter.
+    pub fn open(
+        path: &Path,
+        encoding: Option<&'static encoding_rs::Encoding>,
+        theme: Option<&Theme>,
+        config_loader: Option<&syntax::Loader>,
+    ) -> Result<Self, Error> {
+        let (rope, encoding) = if path.exists() {
+            let mut file =
+                std::fs::File::open(path).context(format!("unable to open {:?}", path))?;
+            from_reader(&mut file, encoding)?
         } else {
-            let file = File::open(&path).context(format!("unable to open {:?}", path))?;
-            let mut doc = Rope::from_reader(BufReader::new(file))?;
-            // add missing newline at the end of file
-            if doc.len_bytes() == 0 || doc.byte(doc.len_bytes() - 1) != b'\n' {
-                doc.insert_char(doc.len_chars(), '\n');
-            }
-            doc
+            let encoding = encoding.unwrap_or(encoding_rs::UTF_8);
+            (Rope::from(DEFAULT_LINE_ENDING.as_str()), encoding)
         };
 
-        let mut doc = Self::new(doc);
+        let mut doc = Self::from(rope, Some(encoding));
+
         // set the path and try detecting the language
-        doc.set_path(&path)?;
+        doc.set_path(path)?;
+        if let Some(loader) = config_loader {
+            doc.detect_language(theme, loader);
+        }
+
+        doc.detect_indent_and_line_ending();
 
         Ok(doc)
     }
 
-    // TODO: remove view_id dependency here
-    pub fn format(&mut self, view_id: ViewId) {
-        if let Some(language_server) = self.language_server() {
-            // TODO: await, no blocking
-            let transaction = helix_lsp::block_on(
-                language_server
-                    .text_document_formatting(self.identifier(), lsp::FormattingOptions::default()),
-            )
-            .map(|edits| {
-                helix_lsp::util::generate_transaction_from_edits(
-                    self.text(),
-                    edits,
-                    language_server.offset_encoding(),
-                )
-            });
+    /// The same as [`format`], but only returns formatting changes if auto-formatting
+    /// is configured.
+    pub fn auto_format(&self) -> Option<impl Future<Output = LspFormatting> + 'static> {
+        if self.language_config().map(|c| c.auto_format) == Some(true) {
+            self.format()
+        } else {
+            None
+        }
+    }
 
-            if let Ok(transaction) = transaction {
-                self.apply(&transaction, view_id);
-                self.append_changes_to_history(view_id);
+    /// If supported, returns the changes that should be applied to this document in order
+    /// to format it nicely.
+    pub fn format(&self) -> Option<impl Future<Output = LspFormatting> + 'static> {
+        if let Some(language_server) = self.language_server.clone() {
+            let text = self.text.clone();
+            let id = self.identifier();
+            let fut = async move {
+                let edits = language_server
+                    .text_document_formatting(id, lsp::FormattingOptions::default(), None)
+                    .await
+                    .unwrap_or_else(|e| {
+                        log::warn!("LSP formatting failed: {}", e);
+                        Default::default()
+                    });
+                LspFormatting {
+                    doc: text,
+                    edits,
+                    offset_encoding: language_server.offset_encoding(),
+                }
+            };
+            Some(fut)
+        } else {
+            None
         }
     }
+
+    pub fn save(&mut self) -> impl Future<Output = Result<(), anyhow::Error>> {
+        self.save_impl::<futures_util::future::Ready<_>>(None)
+    }
+
+    pub fn format_and_save(
+        &mut self,
+        formatting: Option<impl Future<Output = LspFormatting>>,
+    ) -> impl Future<Output = anyhow::Result<()>> {
+        self.save_impl(formatting)
     }
 
     // TODO: do we need some way of ensuring two save operations on the same doc can't run at once?
     // or is that handled by the OS/async layer
-    pub fn save(&mut self) -> impl Future<Output = Result<(), anyhow::Error>> {
+    /// The `Document`'s text is encoded according to its encoding and written to the file located
+    /// at its `path()`.
+    ///
+    /// If `formatting` is present, it supplies some changes that we apply to the text before saving.
+    fn save_impl<F: Future<Output = LspFormatting>>(
+        &mut self,
+        formatting: Option<F>,
+    ) -> impl Future<Output = Result<(), anyhow::Error>> {
         // we clone and move text + path into the future so that we asynchronously save the current
         // state without blocking any further edits.
 
-        let text = self.text().clone();
-        let path = self.path.clone().expect("Can't save with no path set!"); // TODO: handle no path
+        let mut text = self.text().clone();
+        let path = self.path.clone().expect("Can't save with no path set!");
         let identifier = self.identifier();
 
-        // TODO: mark changes up to now as saved
-
         let language_server = self.language_server.clone();
 
-        // reset the modified flag
-        let history = self.history.take();
-        self.last_saved_revision = history.current_revision();
-        self.history.set(history);
+        // mark changes up to now as saved
+        self.reset_modified();
 
+        let encoding = self.encoding;
+
+        // We encode the file according to the `Document`'s encoding.
         async move {
-            use tokio::{fs::File, io::AsyncWriteExt};
+            use tokio::fs::File;
             if let Some(parent) = path.parent() {
                 // TODO: display a prompt asking the user if the directories should be created
                 if !parent.exists() {
@@ -238,13 +541,18 @@ impl Document {
                     ));
                 }
             }
-            let mut file = File::create(path).await?;
 
-            // write all the rope chunks to file
-            for chunk in text.chunks() {
-                file.write_all(chunk.as_bytes()).await?;
+            if let Some(fmt) = formatting {
+                let success = Transaction::from(fmt.await).changes().apply(&mut text);
+                if !success {
+                    // This shouldn't happen, because the transaction changes were generated
+                    // from the same text we're saving.
+                    log::error!("failed to apply format changes before saving");
                 }
-            // TODO: flush?
+            }
+
+            let mut file = File::create(path).await?;
+            to_writer(&mut file, encoding, &text).await?;
 
             if let Some(language_server) = language_server {
                 language_server
@@ -256,15 +564,62 @@ impl Document {
         }
     }
 
-    fn detect_language(&mut self) {
-        if let Some(path) = self.path() {
-            let loader = LOADER.get().unwrap();
-            let language_config = loader.language_config_for_file_name(path);
-            let scopes = loader.scopes();
-            self.set_language(language_config, scopes);
+    pub fn detect_language(&mut self, theme: Option<&Theme>, config_loader: &syntax::Loader) {
+        if let Some(path) = &self.path {
+            let language_config = config_loader.language_config_for_file_name(path);
+            self.set_language(theme, language_config);
         }
     }
 
+    pub fn detect_indent_and_line_ending(&mut self) {
+        self.indent_style = auto_detect_indent_style(&self.text).unwrap_or_else(|| {
+            IndentStyle::from_str(
+                self.language
+                    .as_ref()
+                    .and_then(|config| config.indent.as_ref())
+                    .map_or("  ", |config| config.unit.as_str()), // Fallback to 2 spaces.
+            )
+        });
+        self.line_ending = auto_detect_line_ending(&self.text).unwrap_or(DEFAULT_LINE_ENDING);
+    }
+
+    /// Reload the document from its path.
+    pub fn reload(&mut self, view_id: ViewId) -> Result<(), Error> {
+        let encoding = &self.encoding;
+        let path = self.path().filter(|path| path.exists());
+
+        // If there is no path or the path no longer exists.
+        if path.is_none() {
+            return Err(anyhow!("can't find file to reload from"));
+        }
+
+        let mut file = std::fs::File::open(path.unwrap())?;
+        let (rope, ..) = from_reader(&mut file, Some(encoding))?;
+
+        let transaction = helix_core::diff::compare_ropes(self.text(), &rope);
+        self.apply(&transaction, view_id);
+        self.append_changes_to_history(view_id);
+        self.reset_modified();
+
+        self.detect_indent_and_line_ending();
+
+        Ok(())
+    }
+
+    /// Sets the [`Document`]'s encoding with the encoding correspondent to `label`.
+    pub fn set_encoding(&mut self, label: &str) -> Result<(), Error> {
+        match encoding_rs::Encoding::for_label(label.as_bytes()) {
+            Some(encoding) => self.encoding = encoding,
+            None => return Err(anyhow::anyhow!("unknown encoding")),
+        }
+        Ok(())
+    }
+
+    /// Returns the [`Document`]'s current encoding.
+    pub fn encoding(&self) -> &'static encoding_rs::Encoding {
+        self.encoding
+    }
+
     pub fn set_path(&mut self, path: &Path) -> Result<(), std::io::Error> {
         let path = canonicalize_path(path)?;
 
@@ -272,18 +627,16 @@ impl Document {
         // and error out when document is saved
         self.path = Some(path);
 
-        // try detecting the language based on filepath
-        self.detect_language();
-
         Ok(())
     }
 
     pub fn set_language(
         &mut self,
+        theme: Option<&Theme>,
         language_config: Option<Arc<helix_core::syntax::LanguageConfiguration>>,
-        scopes: &[String],
     ) {
         if let Some(language_config) = language_config {
+            let scopes = theme.map(|theme| theme.scopes()).unwrap_or(&[]);
             if let Some(highlight_config) = language_config.highlight_config(scopes) {
                 let syntax = Syntax::new(&self.text, highlight_config);
                 self.syntax = Some(syntax);
@@ -297,12 +650,15 @@ impl Document {
         };
     }
 
-    pub fn set_language2(&mut self, scope: &str) {
-        let loader = LOADER.get().unwrap();
-        let language_config = loader.language_config_for_scope(scope);
-        let scopes = loader.scopes();
+    pub fn set_language2(
+        &mut self,
+        scope: &str,
+        theme: Option<&Theme>,
+        config_loader: Arc<syntax::Loader>,
+    ) {
+        let language_config = config_loader.language_config_for_scope(scope);
 
-        self.set_language(language_config, scopes);
+        self.set_language(theme, language_config);
     }
 
     pub fn set_language_server(&mut self, language_server: Option<Arc<helix_lsp::Client>>) {
@@ -311,22 +667,32 @@ impl Document {
 
     pub fn set_selection(&mut self, view_id: ViewId, selection: Selection) {
         // TODO: use a transaction?
-        self.selections.insert(view_id, selection);
+        self.selections
+            .insert(view_id, selection.ensure_invariants(self.text().slice(..)));
     }
 
-    fn _apply(&mut self, transaction: &Transaction, view_id: ViewId) -> bool {
+    fn apply_impl(&mut self, transaction: &Transaction, view_id: ViewId) -> bool {
         let old_doc = self.text().clone();
 
         let success = transaction.changes().apply(&mut self.text);
 
         if success {
-            // update the selection: either take the selection specified in the transaction, or map the
-            // current selection through changes.
-            let selection = transaction
-                .selection()
-                .cloned()
-                .unwrap_or_else(|| self.selection(view_id).clone().map(transaction.changes()));
-            self.set_selection(view_id, selection);
+            for selection in self.selections.values_mut() {
+                *selection = selection
+                    .clone()
+                    // Map through changes
+                    .map(transaction.changes())
+                    // Ensure all selections accross all views still adhere to invariants.
+                    .ensure_invariants(self.text.slice(..));
+            }
+
+            // if specified, the current selection should instead be replaced by transaction.selection
+            if let Some(selection) = transaction.selection() {
+                self.selections.insert(
+                    view_id,
+                    selection.clone().ensure_invariants(self.text.slice(..)),
+                );
+            }
         }
 
         if !transaction.changes().is_empty() {
@@ -377,7 +743,7 @@ impl Document {
             });
         }
 
-        let success = self._apply(transaction, view_id);
+        let success = self.apply_impl(transaction, view_id);
 
         if !transaction.changes().is_empty() {
             // Compose this transaction with the previous one
@@ -391,7 +757,7 @@ impl Document {
     pub fn undo(&mut self, view_id: ViewId) {
         let mut history = self.history.take();
         let success = if let Some(transaction) = history.undo() {
-            self._apply(&transaction, view_id)
+            self.apply_impl(transaction, view_id)
         } else {
             false
         };
@@ -406,7 +772,7 @@ impl Document {
     pub fn redo(&mut self, view_id: ViewId) {
         let mut history = self.history.take();
         let success = if let Some(transaction) = history.redo() {
-            self._apply(&transaction, view_id)
+            self.apply_impl(transaction, view_id)
         } else {
             false
         };
@@ -421,17 +787,18 @@ impl Document {
     pub fn earlier(&mut self, view_id: ViewId, uk: helix_core::history::UndoKind) {
         let txns = self.history.get_mut().earlier(uk);
         for txn in txns {
-            self._apply(&txn, view_id);
+            self.apply_impl(&txn, view_id);
         }
     }
 
     pub fn later(&mut self, view_id: ViewId, uk: helix_core::history::UndoKind) {
         let txns = self.history.get_mut().later(uk);
         for txn in txns {
-            self._apply(&txn, view_id);
+            self.apply_impl(&txn, view_id);
         }
     }
 
+    /// Commit pending changes to history
     pub fn append_changes_to_history(&mut self, view_id: ViewId) {
         if self.changes.is_empty() {
             return;
@@ -452,12 +819,10 @@ impl Document {
         self.history.set(history);
     }
 
-    #[inline]
     pub fn id(&self) -> DocumentId {
         self.id
     }
 
-    #[inline]
     pub fn is_modified(&self) -> bool {
         let history = self.history.take();
         let current_revision = history.current_revision();
@@ -465,12 +830,17 @@ impl Document {
         current_revision != self.last_saved_revision || !self.changes.is_empty()
     }
 
-    #[inline]
+    pub fn reset_modified(&mut self) {
+        let history = self.history.take();
+        let current_revision = history.current_revision();
+        self.history.set(history);
+        self.last_saved_revision = current_revision;
+    }
+
     pub fn mode(&self) -> Mode {
         self.mode
     }
 
-    #[inline]
     /// Corresponding language scope name. Usually `source.<lang>`.
     pub fn language(&self) -> Option<&str> {
         self.language
@@ -478,21 +848,21 @@ impl Document {
             .map(|language| language.scope.as_str())
     }
 
-    #[inline]
     pub fn language_config(&self) -> Option<&LanguageConfiguration> {
         self.language.as_deref()
     }
 
-    #[inline]
     /// Current document version, incremented at each change.
     pub fn version(&self) -> i32 {
         self.version
     }
 
+    #[inline]
     pub fn language_server(&self) -> Option<&helix_lsp::Client> {
         self.language_server.as_deref()
     }
 
+    #[inline]
     /// Tree-sitter AST tree
     pub fn syntax(&self) -> Option<&Syntax> {
         self.syntax.as_ref()
@@ -507,13 +877,11 @@ impl Document {
     }
 
     /// Returns a string containing a single level of indentation.
-    pub fn indent_unit(&self) -> &str {
-        self.language
-            .as_ref()
-            .and_then(|config| config.indent.as_ref())
-            .map_or("  ", |config| config.unit.as_str()) // fallback to 2 spaces
-
-        // " ".repeat(TAB_WIDTH)
+    ///
+    /// TODO: we might not need this function anymore, since the information
+    /// is conveniently available in `Document::indent_style` now.
+    pub fn indent_unit(&self) -> &'static str {
+        self.indent_style.as_str()
     }
 
     #[inline]
@@ -526,20 +894,30 @@ impl Document {
         self.path().map(|path| Url::from_file_path(path).unwrap())
     }
 
+    #[inline]
     pub fn text(&self) -> &Rope {
         &self.text
     }
 
+    #[inline]
     pub fn selection(&self, view_id: ViewId) -> &Selection {
         &self.selections[&view_id]
     }
 
-    pub fn relative_path(&self) -> Option<&Path> {
+    pub fn selections(&self) -> &HashMap<ViewId, Selection> {
+        &self.selections
+    }
+
+    pub fn relative_path(&self) -> Option<PathBuf> {
         let cwdir = std::env::current_dir().expect("couldn't determine current directory");
 
-        self.path
-            .as_ref()
-            .map(|path| path.strip_prefix(cwdir).unwrap_or(path))
+        self.path.as_ref().map(|path| {
+            let mut path = path.as_path();
+            if path.is_absolute() {
+                path = path.strip_prefix(cwdir).unwrap_or(path)
+            };
+            fold_home_dir(path)
+        })
     }
 
     // pub fn slice<R>(&self, range: R) -> RopeSlice where R: RangeBounds {
@@ -550,6 +928,7 @@ impl Document {
 
     // -- LSP methods
 
+    #[inline]
     pub fn identifier(&self) -> lsp::TextDocumentIdentifier {
         lsp::TextDocumentIdentifier::new(self.url().unwrap())
     }
@@ -558,6 +937,7 @@ impl Document {
         lsp::VersionedTextDocumentIdentifier::new(self.url().unwrap(), self.version)
     }
 
+    #[inline]
     pub fn diagnostics(&self) -> &[Diagnostic] {
         &self.diagnostics
     }
@@ -567,6 +947,13 @@ impl Document {
     }
 }
 
+impl Default for Document {
+    fn default() -> Self {
+        let text = Rope::from(DEFAULT_LINE_ENDING.as_str());
+        Self::from(text, None)
+    }
+}
+
 #[cfg(test)]
 mod test {
     use super::*;
@@ -575,7 +962,7 @@ mod test {
     fn changeset_to_changes() {
         use helix_lsp::{lsp, Client, OffsetEncoding};
         let text = Rope::from("hello");
-        let mut doc = Document::new(text);
+        let mut doc = Document::from(text, None);
         let view = ViewId::default();
         doc.set_selection(view, Selection::single(5, 5));
 
@@ -684,4 +1071,93 @@ mod test {
             ]
         );
     }
+
+    #[test]
+    fn test_line_ending() {
+        assert_eq!(
+            Document::default().text().to_string(),
+            DEFAULT_LINE_ENDING.as_str()
+        );
+    }
+
+    macro_rules! test_decode {
+        ($label:expr, $label_override:expr) => {
+            let encoding = encoding_rs::Encoding::for_label($label_override.as_bytes()).unwrap();
+            let base_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/encoding");
+            let path = base_path.join(format!("{}_in.txt", $label));
+            let ref_path = base_path.join(format!("{}_in_ref.txt", $label));
+            assert!(path.exists());
+            assert!(ref_path.exists());
+
+            let mut file = std::fs::File::open(path).unwrap();
+            let text = from_reader(&mut file, Some(encoding))
+                .unwrap()
+                .0
+                .to_string();
+            let expectation = std::fs::read_to_string(ref_path).unwrap();
+            assert_eq!(text[..], expectation[..]);
+        };
+    }
+
+    macro_rules! test_encode {
+        ($label:expr, $label_override:expr) => {
+            let encoding = encoding_rs::Encoding::for_label($label_override.as_bytes()).unwrap();
+            let base_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/encoding");
+            let path = base_path.join(format!("{}_out.txt", $label));
+            let ref_path = base_path.join(format!("{}_out_ref.txt", $label));
+            assert!(path.exists());
+            assert!(ref_path.exists());
+
+            let text = Rope::from_str(&std::fs::read_to_string(path).unwrap());
+            let mut buf: Vec<u8> = Vec::new();
+            helix_lsp::block_on(to_writer(&mut buf, encoding, &text)).unwrap();
+
+            let expectation = std::fs::read(ref_path).unwrap();
+            assert_eq!(buf, expectation);
+        };
+    }
+
+    macro_rules! test_decode_fn {
+        ($name:ident, $label:expr, $label_override:expr) => {
+            #[test]
+            fn $name() {
+                test_decode!($label, $label_override);
+            }
+        };
+        ($name:ident, $label:expr) => {
+            #[test]
+            fn $name() {
+                test_decode!($label, $label);
+            }
+        };
+    }
+
+    macro_rules! test_encode_fn {
+        ($name:ident, $label:expr, $label_override:expr) => {
+            #[test]
+            fn $name() {
+                test_encode!($label, $label_override);
+            }
+        };
+        ($name:ident, $label:expr) => {
+            #[test]
+            fn $name() {
+                test_encode!($label, $label);
+            }
+        };
+    }
+
+    test_decode_fn!(test_big5_decode, "big5");
+    test_encode_fn!(test_big5_encode, "big5");
+    test_decode_fn!(test_euc_kr_decode, "euc_kr", "EUC-KR");
+    test_encode_fn!(test_euc_kr_encode, "euc_kr", "EUC-KR");
+    test_decode_fn!(test_gb18030_decode, "gb18030");
+    test_encode_fn!(test_gb18030_encode, "gb18030");
+    test_decode_fn!(test_iso_2022_jp_decode, "iso_2022_jp", "ISO-2022-JP");
+    test_encode_fn!(test_iso_2022_jp_encode, "iso_2022_jp", "ISO-2022-JP");
+    test_decode_fn!(test_jis0208_decode, "jis0208", "EUC-JP");
+    test_encode_fn!(test_jis0208_encode, "jis0208", "EUC-JP");
+    test_decode_fn!(test_jis0212_decode, "jis0212", "EUC-JP");
+    test_decode_fn!(test_shift_jis_decode, "shift_jis");
+    test_encode_fn!(test_shift_jis_encode, "shift_jis");
 }

helix-view/src/editor.rs

@@ -1,55 +1,87 @@
-use crate::{theme::Theme, tree::Tree, Document, DocumentId, RegisterSelection, View, ViewId};
-use tui::layout::Rect;
+use crate::{
+    clipboard::{get_clipboard_provider, ClipboardProvider},
+    graphics::{CursorKind, Rect},
+    theme::{self, Theme},
+    tree::Tree,
+    Document, DocumentId, RegisterSelection, View, ViewId,
+};
 
-use std::path::PathBuf;
+use futures_util::future;
+use std::{
+    path::{Path, PathBuf},
+    sync::Arc,
+    time::Duration,
+};
 
 use slotmap::SlotMap;
 
 use anyhow::Error;
 
 pub use helix_core::diagnostic::Severity;
+pub use helix_core::register::Registers;
+use helix_core::syntax;
+use helix_core::Position;
+
+use serde::Deserialize;
+
+#[derive(Debug, Clone, PartialEq, Deserialize)]
+#[serde(rename_all = "kebab-case", default)]
+pub struct Config {
+    /// Padding to keep between the edge of the screen and the cursor when scrolling. Defaults to 5.
+    pub scrolloff: usize,
+    /// Number of lines to scroll at once. Defaults to 3
+    pub scroll_lines: isize,
+    /// Mouse support. Defaults to true.
+    pub mouse: bool,
+    /// Middle click paste support. Defaults to true
+    pub middle_click_paste: bool,
+}
+
+impl Default for Config {
+    fn default() -> Self {
+        Self {
+            scrolloff: 5,
+            scroll_lines: 3,
+            mouse: true,
+            middle_click_paste: true,
+        }
+    }
+}
 
 #[derive(Debug)]
 pub struct Editor {
     pub tree: Tree,
     pub documents: SlotMap<DocumentId, Document>,
     pub count: Option<std::num::NonZeroUsize>,
-    pub register: RegisterSelection,
+    pub selected_register: RegisterSelection,
+    pub registers: Registers,
     pub theme: Theme,
     pub language_servers: helix_lsp::Registry,
+    pub clipboard_provider: Box<dyn ClipboardProvider>,
+
+    pub syn_loader: Arc<syntax::Loader>,
+    pub theme_loader: Arc<theme::Loader>,
 
     pub status_msg: Option<(String, Severity)>,
+
+    pub config: Config,
 }
 
 #[derive(Debug, Copy, Clone)]
 pub enum Action {
+    Load,
     Replace,
     HorizontalSplit,
     VerticalSplit,
 }
 
 impl Editor {
-    pub fn new(mut area: tui::layout::Rect) -> Self {
-        use helix_core::config_dir;
-        let config = std::fs::read(config_dir().join("theme.toml"));
-        // load $HOME/.config/helix/theme.toml, fallback to default config
-        let toml = config
-            .as_deref()
-            .unwrap_or(include_bytes!("../../theme.toml"));
-        let theme: Theme = toml::from_slice(toml).expect("failed to parse theme.toml");
-
-        // initialize language registry
-        use helix_core::syntax::{Loader, LOADER};
-
-        // load $HOME/.config/helix/languages.toml, fallback to default config
-        let config = std::fs::read(helix_core::config_dir().join("languages.toml"));
-        let toml = config
-            .as_deref()
-            .unwrap_or(include_bytes!("../../languages.toml"));
-
-        let config = toml::from_slice(toml).expect("Could not parse languages.toml");
-        LOADER.get_or_init(|| Loader::new(config, theme.scopes().to_vec()));
-
+    pub fn new(
+        mut area: Rect,
+        themes: Arc<theme::Loader>,
+        config_loader: Arc<syntax::Loader>,
+        config: Config,
+    ) -> Self {
         let language_servers = helix_lsp::Registry::new();
 
         // HAXX: offset the render area height by 1 to account for prompt/commandline
@@ -59,10 +91,15 @@ impl Editor {
             tree: Tree::new(area),
             documents: SlotMap::with_key(),
             count: None,
-            register: RegisterSelection::default(),
-            theme,
+            selected_register: RegisterSelection::default(),
+            theme: themes.default(),
             language_servers,
+            syn_loader: config_loader,
+            theme_loader: themes,
+            registers: Registers::default(),
+            clipboard_provider: get_clipboard_provider(),
             status_msg: None,
+            config,
         }
     }
 
@@ -78,10 +115,34 @@ impl Editor {
         self.status_msg = Some((error, Severity::Error));
     }
 
+    pub fn set_theme(&mut self, theme: Theme) {
+        let scopes = theme.scopes();
+        for config in self
+            .syn_loader
+            .language_configs_iter()
+            .filter(|cfg| cfg.is_highlight_initialized())
+        {
+            config.reconfigure(scopes);
+        }
+
+        self.theme = theme;
+        self._refresh();
+    }
+
+    pub fn set_theme_from_name(&mut self, theme: &str) -> anyhow::Result<()> {
+        use anyhow::Context;
+        let theme = self
+            .theme_loader
+            .load(theme.as_ref())
+            .with_context(|| format!("failed setting theme `{}`", theme))?;
+        self.set_theme(theme);
+        Ok(())
+    }
+
     fn _refresh(&mut self) {
         for (view, _) in self.tree.views_mut() {
             let doc = &self.documents[view.doc];
-            view.ensure_cursor_in_view(doc)
+            view.ensure_cursor_in_view(doc, self.config.scrolloff)
         }
     }
 
@@ -96,32 +157,37 @@ impl Editor {
 
         match action {
             Action::Replace => {
-                let view = self.view();
+                let view = view!(self);
                 let jump = (
                     view.doc,
                     self.documents[view.doc].selection(view.id).clone(),
                 );
 
-                let view = self.view_mut();
+                let view = view_mut!(self);
                 view.jumps.push(jump);
                 view.last_accessed_doc = Some(view.doc);
                 view.doc = id;
                 view.first_line = 0;
 
-                let (view, doc) = self.current();
+                let (view, doc) = current!(self);
 
                 // initialize selection for view
-                let selection = doc
-                    .selections
+                doc.selections
                     .entry(view.id)
                     .or_insert_with(|| Selection::point(0));
                 // TODO: reuse align_view
-                let pos = selection.cursor();
+                let pos = doc
+                    .selection(view.id)
+                    .primary()
+                    .cursor(doc.text().slice(..));
                 let line = doc.text().char_to_line(pos);
                 view.first_line = line.saturating_sub(view.area.height as usize / 2);
 
                 return;
             }
+            Action::Load => {
+                return;
+            }
             Action::HorizontalSplit => {
                 let view = View::new(id);
                 let view_id = self.tree.split(view, Layout::Horizontal);
@@ -142,8 +208,7 @@ impl Editor {
     }
 
     pub fn new_file(&mut self, action: Action) -> DocumentId {
-        use helix_core::Rope;
-        let doc = Document::new(Rope::from("\n"));
+        let doc = Document::default();
         let id = self.documents.insert(doc);
         self.documents[id].id = id;
         self.switch(id, action);
@@ -161,7 +226,7 @@ impl Editor {
         let id = if let Some(id) = id {
             id
         } else {
-            let mut doc = Document::load(path)?;
+            let mut doc = Document::open(&path, None, Some(&self.theme), Some(&self.syn_loader))?;
 
             // try to find a language server based on the language name
             let language_server = doc
@@ -233,49 +298,70 @@ impl Editor {
         self.tree.is_empty()
     }
 
-    pub fn current(&mut self) -> (&mut View, &mut Document) {
-        let view = self.tree.get_mut(self.tree.focus);
-        let doc = &mut self.documents[view.doc];
-        (view, doc)
-    }
-
-    pub fn view(&self) -> &View {
-        self.tree.get(self.tree.focus)
-    }
-
-    pub fn view_mut(&mut self) -> &mut View {
-        self.tree.get_mut(self.tree.focus)
-    }
-
     pub fn ensure_cursor_in_view(&mut self, id: ViewId) {
         let view = self.tree.get_mut(id);
         let doc = &self.documents[view.doc];
-        view.ensure_cursor_in_view(doc)
+        view.ensure_cursor_in_view(doc, self.config.scrolloff)
     }
 
     pub fn document(&self, id: DocumentId) -> Option<&Document> {
         self.documents.get(id)
     }
 
+    pub fn document_mut(&mut self, id: DocumentId) -> Option<&mut Document> {
+        self.documents.get_mut(id)
+    }
+
     pub fn documents(&self) -> impl Iterator<Item = &Document> {
         self.documents.iter().map(|(_id, doc)| doc)
     }
 
+    pub fn documents_mut(&mut self) -> impl Iterator<Item = &mut Document> {
+        self.documents.iter_mut().map(|(_id, doc)| doc)
+    }
+
+    pub fn document_by_path<P: AsRef<Path>>(&self, path: P) -> Option<&Document> {
+        self.documents()
+            .find(|doc| doc.path().map(|p| p == path.as_ref()).unwrap_or(false))
+    }
+
     // pub fn current_document(&self) -> Document {
     //     let id = self.view().doc;
     //     let doc = &mut editor.documents[id];
     // }
 
-    pub fn cursor_position(&self) -> Option<helix_core::Position> {
+    pub fn cursor(&self) -> (Option<Position>, CursorKind) {
         const OFFSET: u16 = 7; // 1 diagnostic + 5 linenr + 1 gutter
-        let view = self.view();
+        let view = view!(self);
         let doc = &self.documents[view.doc];
-        let cursor = doc.selection(view.id).cursor();
+        let cursor = doc
+            .selection(view.id)
+            .primary()
+            .cursor(doc.text().slice(..));
         if let Some(mut pos) = view.screen_coords_at_pos(doc, doc.text().slice(..), cursor) {
             pos.col += view.area.x as usize + OFFSET as usize;
             pos.row += view.area.y as usize;
-            return Some(pos);
-        }
-        None
+            (Some(pos), CursorKind::Hidden)
+        } else {
+            (None, CursorKind::Hidden)
+        }
+    }
+
+    /// Closes language servers with timeout. The default timeout is 500 ms, use
+    /// `timeout` parameter to override this.
+    pub async fn close_language_servers(
+        &self,
+        timeout: Option<u64>,
+    ) -> Result<(), tokio::time::error::Elapsed> {
+        tokio::time::timeout(
+            Duration::from_millis(timeout.unwrap_or(500)),
+            future::join_all(
+                self.language_servers
+                    .iter_clients()
+                    .map(|client| client.force_shutdown()),
+            ),
+        )
+        .await
+        .map(|_| ())
     }
 }

helix-view/src/graphics.rs

@@ -0,0 +1,530 @@
+use bitflags::bitflags;
+use std::{
+    cmp::{max, min},
+    str::FromStr,
+};
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+/// UNSTABLE
+pub enum CursorKind {
+    /// █
+    Block,
+    /// |
+    Bar,
+    /// _
+    Underline,
+    /// Hidden cursor, can set cursor position with this to let IME have correct cursor position.
+    Hidden,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct Margin {
+    pub vertical: u16,
+    pub horizontal: u16,
+}
+
+/// A simple rectangle used in the computation of the layout and to give widgets an hint about the
+/// area they are supposed to render to.
+#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
+pub struct Rect {
+    pub x: u16,
+    pub y: u16,
+    pub width: u16,
+    pub height: u16,
+}
+
+impl Default for Rect {
+    fn default() -> Rect {
+        Rect {
+            x: 0,
+            y: 0,
+            width: 0,
+            height: 0,
+        }
+    }
+}
+
+impl Rect {
+    /// Creates a new rect, with width and height limited to keep the area under max u16.
+    /// If clipped, aspect ratio will be preserved.
+    pub fn new(x: u16, y: u16, width: u16, height: u16) -> Rect {
+        let max_area = u16::max_value();
+        let (clipped_width, clipped_height) =
+            if u32::from(width) * u32::from(height) > u32::from(max_area) {
+                let aspect_ratio = f64::from(width) / f64::from(height);
+                let max_area_f = f64::from(max_area);
+                let height_f = (max_area_f / aspect_ratio).sqrt();
+                let width_f = height_f * aspect_ratio;
+                (width_f as u16, height_f as u16)
+            } else {
+                (width, height)
+            };
+        Rect {
+            x,
+            y,
+            width: clipped_width,
+            height: clipped_height,
+        }
+    }
+
+    #[inline]
+    pub fn area(self) -> u16 {
+        self.width * self.height
+    }
+
+    #[inline]
+    pub fn left(self) -> u16 {
+        self.x
+    }
+
+    #[inline]
+    pub fn right(self) -> u16 {
+        self.x.saturating_add(self.width)
+    }
+
+    #[inline]
+    pub fn top(self) -> u16 {
+        self.y
+    }
+
+    #[inline]
+    pub fn bottom(self) -> u16 {
+        self.y.saturating_add(self.height)
+    }
+
+    pub fn inner(self, margin: &Margin) -> Rect {
+        if self.width < 2 * margin.horizontal || self.height < 2 * margin.vertical {
+            Rect::default()
+        } else {
+            Rect {
+                x: self.x + margin.horizontal,
+                y: self.y + margin.vertical,
+                width: self.width - 2 * margin.horizontal,
+                height: self.height - 2 * margin.vertical,
+            }
+        }
+    }
+
+    /// Calculate the union between two [`Rect`]s.
+    pub fn union(self, other: Rect) -> Rect {
+        // Example:
+        //
+        // If `Rect` A is positioned at `(0, 0)` with a width and height of `5`,
+        // and `Rect` B is positioned at `(5, 0)` with a width and height of `2`,
+        // then this is the resulting union:
+        //
+        // x1 = min(0, 5) => x1 = 0
+        // y1 = min(0, 0) => y1 = 0
+        // x2 = max(0 + 5, 5 + 2) => x2 = 7
+        // y2 = max(0 + 5, 0 + 2) => y2 = 5
+        let x1 = min(self.x, other.x);
+        let y1 = min(self.y, other.y);
+        let x2 = max(self.x + self.width, other.x + other.width);
+        let y2 = max(self.y + self.height, other.y + other.height);
+        Rect {
+            x: x1,
+            y: y1,
+            width: x2 - x1,
+            height: y2 - y1,
+        }
+    }
+
+    /// Calculate the intersection between two [`Rect`]s.
+    pub fn intersection(self, other: Rect) -> Rect {
+        // Example:
+        //
+        // If `Rect` A is positioned at `(0, 0)` with a width and height of `5`,
+        // and `Rect` B is positioned at `(5, 0)` with a width and height of `2`,
+        // then this is the resulting intersection:
+        //
+        // x1 = max(0, 5) => x1 = 5
+        // y1 = max(0, 0) => y1 = 0
+        // x2 = min(0 + 5, 5 + 2) => x2 = 5
+        // y2 = min(0 + 5, 0 + 2) => y2 = 2
+        let x1 = max(self.x, other.x);
+        let y1 = max(self.y, other.y);
+        let x2 = min(self.x + self.width, other.x + other.width);
+        let y2 = min(self.y + self.height, other.y + other.height);
+        Rect {
+            x: x1,
+            y: y1,
+            width: x2 - x1,
+            height: y2 - y1,
+        }
+    }
+
+    pub fn intersects(self, other: Rect) -> bool {
+        self.x < other.x + other.width
+            && self.x + self.width > other.x
+            && self.y < other.y + other.height
+            && self.y + self.height > other.y
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+pub enum Color {
+    Reset,
+    Black,
+    Red,
+    Green,
+    Yellow,
+    Blue,
+    Magenta,
+    Cyan,
+    Gray,
+    DarkGray,
+    LightRed,
+    LightGreen,
+    LightYellow,
+    LightBlue,
+    LightMagenta,
+    LightCyan,
+    White,
+    Rgb(u8, u8, u8),
+    Indexed(u8),
+}
+
+#[cfg(feature = "term")]
+impl From<Color> for crossterm::style::Color {
+    fn from(color: Color) -> Self {
+        use crossterm::style::Color as CColor;
+
+        match color {
+            Color::Reset => CColor::Reset,
+            Color::Black => CColor::Black,
+            Color::Red => CColor::DarkRed,
+            Color::Green => CColor::DarkGreen,
+            Color::Yellow => CColor::DarkYellow,
+            Color::Blue => CColor::DarkBlue,
+            Color::Magenta => CColor::DarkMagenta,
+            Color::Cyan => CColor::DarkCyan,
+            Color::Gray => CColor::Grey,
+            Color::DarkGray => CColor::DarkGrey,
+            Color::LightRed => CColor::Red,
+            Color::LightGreen => CColor::Green,
+            Color::LightBlue => CColor::Blue,
+            Color::LightYellow => CColor::Yellow,
+            Color::LightMagenta => CColor::Magenta,
+            Color::LightCyan => CColor::Cyan,
+            Color::White => CColor::White,
+            Color::Indexed(i) => CColor::AnsiValue(i),
+            Color::Rgb(r, g, b) => CColor::Rgb { r, g, b },
+        }
+    }
+}
+
+bitflags! {
+    /// Modifier changes the way a piece of text is displayed.
+    ///
+    /// They are bitflags so they can easily be composed.
+    ///
+    /// ## Examples
+    ///
+    /// ```rust
+    /// # use helix_view::graphics::Modifier;
+    ///
+    /// let m = Modifier::BOLD | Modifier::ITALIC;
+    /// ```
+    #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+    pub struct Modifier: u16 {
+        const BOLD              = 0b0000_0000_0001;
+        const DIM               = 0b0000_0000_0010;
+        const ITALIC            = 0b0000_0000_0100;
+        const UNDERLINED        = 0b0000_0000_1000;
+        const SLOW_BLINK        = 0b0000_0001_0000;
+        const RAPID_BLINK       = 0b0000_0010_0000;
+        const REVERSED          = 0b0000_0100_0000;
+        const HIDDEN            = 0b0000_1000_0000;
+        const CROSSED_OUT       = 0b0001_0000_0000;
+    }
+}
+
+impl FromStr for Modifier {
+    type Err = &'static str;
+
+    fn from_str(modifier: &str) -> Result<Self, Self::Err> {
+        match modifier {
+            "bold" => Ok(Self::BOLD),
+            "dim" => Ok(Self::DIM),
+            "italic" => Ok(Self::ITALIC),
+            "underlined" => Ok(Self::UNDERLINED),
+            "slow_blink" => Ok(Self::SLOW_BLINK),
+            "rapid_blink" => Ok(Self::RAPID_BLINK),
+            "reversed" => Ok(Self::REVERSED),
+            "hidden" => Ok(Self::HIDDEN),
+            "crossed_out" => Ok(Self::CROSSED_OUT),
+            _ => Err("Invalid modifier"),
+        }
+    }
+}
+
+/// Style let you control the main characteristics of the displayed elements.
+///
+/// ```rust
+/// # use helix_view::graphics::{Color, Modifier, Style};
+/// Style::default()
+///     .fg(Color::Black)
+///     .bg(Color::Green)
+///     .add_modifier(Modifier::ITALIC | Modifier::BOLD);
+/// ```
+///
+/// It represents an incremental change. If you apply the styles S1, S2, S3 to a cell of the
+/// terminal buffer, the style of this cell will be the result of the merge of S1, S2 and S3, not
+/// just S3.
+///
+/// ```rust
+/// # use helix_view::graphics::{Rect, Color, Modifier, Style};
+/// # use helix_tui::buffer::Buffer;
+/// let styles = [
+///     Style::default().fg(Color::Blue).add_modifier(Modifier::BOLD | Modifier::ITALIC),
+///     Style::default().bg(Color::Red),
+///     Style::default().fg(Color::Yellow).remove_modifier(Modifier::ITALIC),
+/// ];
+/// let mut buffer = Buffer::empty(Rect::new(0, 0, 1, 1));
+/// for style in &styles {
+///   buffer.get_mut(0, 0).set_style(*style);
+/// }
+/// assert_eq!(
+///     Style {
+///         fg: Some(Color::Yellow),
+///         bg: Some(Color::Red),
+///         add_modifier: Modifier::BOLD,
+///         sub_modifier: Modifier::empty(),
+///     },
+///     buffer.get(0, 0).style(),
+/// );
+/// ```
+///
+/// The default implementation returns a `Style` that does not modify anything. If you wish to
+/// reset all properties until that point use [`Style::reset`].
+///
+/// ```
+/// # use helix_view::graphics::{Rect, Color, Modifier, Style};
+/// # use helix_tui::buffer::Buffer;
+/// let styles = [
+///     Style::default().fg(Color::Blue).add_modifier(Modifier::BOLD | Modifier::ITALIC),
+///     Style::reset().fg(Color::Yellow),
+/// ];
+/// let mut buffer = Buffer::empty(Rect::new(0, 0, 1, 1));
+/// for style in &styles {
+///   buffer.get_mut(0, 0).set_style(*style);
+/// }
+/// assert_eq!(
+///     Style {
+///         fg: Some(Color::Yellow),
+///         bg: Some(Color::Reset),
+///         add_modifier: Modifier::empty(),
+///         sub_modifier: Modifier::empty(),
+///     },
+///     buffer.get(0, 0).style(),
+/// );
+/// ```
+#[derive(Debug, Clone, Copy, PartialEq)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+pub struct Style {
+    pub fg: Option<Color>,
+    pub bg: Option<Color>,
+    pub add_modifier: Modifier,
+    pub sub_modifier: Modifier,
+}
+
+impl Default for Style {
+    fn default() -> Style {
+        Style {
+            fg: None,
+            bg: None,
+            add_modifier: Modifier::empty(),
+            sub_modifier: Modifier::empty(),
+        }
+    }
+}
+
+impl Style {
+    /// Returns a `Style` resetting all properties.
+    pub fn reset() -> Style {
+        Style {
+            fg: Some(Color::Reset),
+            bg: Some(Color::Reset),
+            add_modifier: Modifier::empty(),
+            sub_modifier: Modifier::all(),
+        }
+    }
+
+    /// Changes the foreground color.
+    ///
+    /// ## Examples
+    ///
+    /// ```rust
+    /// # use helix_view::graphics::{Color, Style};
+    /// let style = Style::default().fg(Color::Blue);
+    /// let diff = Style::default().fg(Color::Red);
+    /// assert_eq!(style.patch(diff), Style::default().fg(Color::Red));
+    /// ```
+    pub fn fg(mut self, color: Color) -> Style {
+        self.fg = Some(color);
+        self
+    }
+
+    /// Changes the background color.
+    ///
+    /// ## Examples
+    ///
+    /// ```rust
+    /// # use helix_view::graphics::{Color, Style};
+    /// let style = Style::default().bg(Color::Blue);
+    /// let diff = Style::default().bg(Color::Red);
+    /// assert_eq!(style.patch(diff), Style::default().bg(Color::Red));
+    /// ```
+    pub fn bg(mut self, color: Color) -> Style {
+        self.bg = Some(color);
+        self
+    }
+
+    /// Changes the text emphasis.
+    ///
+    /// When applied, it adds the given modifier to the `Style` modifiers.
+    ///
+    /// ## Examples
+    ///
+    /// ```rust
+    /// # use helix_view::graphics::{Color, Modifier, Style};
+    /// let style = Style::default().add_modifier(Modifier::BOLD);
+    /// let diff = Style::default().add_modifier(Modifier::ITALIC);
+    /// let patched = style.patch(diff);
+    /// assert_eq!(patched.add_modifier, Modifier::BOLD | Modifier::ITALIC);
+    /// assert_eq!(patched.sub_modifier, Modifier::empty());
+    /// ```
+    pub fn add_modifier(mut self, modifier: Modifier) -> Style {
+        self.sub_modifier.remove(modifier);
+        self.add_modifier.insert(modifier);
+        self
+    }
+
+    /// Changes the text emphasis.
+    ///
+    /// When applied, it removes the given modifier from the `Style` modifiers.
+    ///
+    /// ## Examples
+    ///
+    /// ```rust
+    /// # use helix_view::graphics::{Color, Modifier, Style};
+    /// let style = Style::default().add_modifier(Modifier::BOLD | Modifier::ITALIC);
+    /// let diff = Style::default().remove_modifier(Modifier::ITALIC);
+    /// let patched = style.patch(diff);
+    /// assert_eq!(patched.add_modifier, Modifier::BOLD);
+    /// assert_eq!(patched.sub_modifier, Modifier::ITALIC);
+    /// ```
+    pub fn remove_modifier(mut self, modifier: Modifier) -> Style {
+        self.add_modifier.remove(modifier);
+        self.sub_modifier.insert(modifier);
+        self
+    }
+
+    /// Results in a combined style that is equivalent to applying the two individual styles to
+    /// a style one after the other.
+    ///
+    /// ## Examples
+    /// ```
+    /// # use helix_view::graphics::{Color, Modifier, Style};
+    /// let style_1 = Style::default().fg(Color::Yellow);
+    /// let style_2 = Style::default().bg(Color::Red);
+    /// let combined = style_1.patch(style_2);
+    /// assert_eq!(
+    ///     Style::default().patch(style_1).patch(style_2),
+    ///     Style::default().patch(combined));
+    /// ```
+    pub fn patch(mut self, other: Style) -> Style {
+        self.fg = other.fg.or(self.fg);
+        self.bg = other.bg.or(self.bg);
+
+        self.add_modifier.remove(other.sub_modifier);
+        self.add_modifier.insert(other.add_modifier);
+        self.sub_modifier.remove(other.add_modifier);
+        self.sub_modifier.insert(other.sub_modifier);
+
+        self
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_rect_size_truncation() {
+        for width in 256u16..300u16 {
+            for height in 256u16..300u16 {
+                let rect = Rect::new(0, 0, width, height);
+                rect.area(); // Should not panic.
+                assert!(rect.width < width || rect.height < height);
+                // The target dimensions are rounded down so the math will not be too precise
+                // but let's make sure the ratios don't diverge crazily.
+                assert!(
+                    (f64::from(rect.width) / f64::from(rect.height)
+                        - f64::from(width) / f64::from(height))
+                    .abs()
+                        < 1.0
+                )
+            }
+        }
+
+        // One dimension below 255, one above. Area above max u16.
+        let width = 900;
+        let height = 100;
+        let rect = Rect::new(0, 0, width, height);
+        assert_ne!(rect.width, 900);
+        assert_ne!(rect.height, 100);
+        assert!(rect.width < width || rect.height < height);
+    }
+
+    #[test]
+    fn test_rect_size_preservation() {
+        for width in 0..256u16 {
+            for height in 0..256u16 {
+                let rect = Rect::new(0, 0, width, height);
+                rect.area(); // Should not panic.
+                assert_eq!(rect.width, width);
+                assert_eq!(rect.height, height);
+            }
+        }
+
+        // One dimension below 255, one above. Area below max u16.
+        let rect = Rect::new(0, 0, 300, 100);
+        assert_eq!(rect.width, 300);
+        assert_eq!(rect.height, 100);
+    }
+
+    fn styles() -> Vec<Style> {
+        vec![
+            Style::default(),
+            Style::default().fg(Color::Yellow),
+            Style::default().bg(Color::Yellow),
+            Style::default().add_modifier(Modifier::BOLD),
+            Style::default().remove_modifier(Modifier::BOLD),
+            Style::default().add_modifier(Modifier::ITALIC),
+            Style::default().remove_modifier(Modifier::ITALIC),
+            Style::default().add_modifier(Modifier::ITALIC | Modifier::BOLD),
+            Style::default().remove_modifier(Modifier::ITALIC | Modifier::BOLD),
+        ]
+    }
+
+    #[test]
+    fn combined_patch_gives_same_result_as_individual_patch() {
+        let styles = styles();
+        for &a in &styles {
+            for &b in &styles {
+                for &c in &styles {
+                    for &d in &styles {
+                        let combined = a.patch(b.patch(c.patch(d)));
+
+                        assert_eq!(
+                            Style::default().patch(a).patch(b).patch(c).patch(d),
+                            Style::default().patch(combined)
+                        );
+                    }
+                }
+            }
+        }
+    }
+}

helix-view/src/info.rs

@@ -0,0 +1,60 @@
+use crate::input::KeyEvent;
+use helix_core::unicode::width::UnicodeWidthStr;
+use std::fmt::Write;
+
+#[derive(Debug)]
+/// Info box used in editor. Rendering logic will be in other crate.
+pub struct Info {
+    /// Title shown at top.
+    pub title: String,
+    /// Text body, should contain newlines.
+    pub text: String,
+    /// Body width.
+    pub width: u16,
+    /// Body height.
+    pub height: u16,
+}
+
+impl Info {
+    // body is a BTreeMap instead of a HashMap because keymaps are represented
+    // with nested hashmaps with no ordering, and each invocation of infobox would
+    // show different orders of items
+    pub fn key(title: &str, body: Vec<(&str, Vec<KeyEvent>)>) -> Info {
+        let (lpad, mpad, rpad) = (1, 2, 1);
+        let keymaps_width: u16 = body
+            .iter()
+            .map(|r| r.1.iter().map(|e| e.width() as u16 + 2).sum::<u16>() - 2)
+            .max()
+            .unwrap();
+        let mut text = String::new();
+        let mut width = 0;
+        let height = body.len() as u16;
+        for (desc, keyevents) in body {
+            let keyevent = keyevents[0];
+            let mut left = keymaps_width - keyevent.width() as u16;
+            for _ in 0..lpad {
+                text.push(' ');
+            }
+            write!(text, "{}", keyevent).ok();
+            for keyevent in &keyevents[1..] {
+                write!(text, ", {}", keyevent).ok();
+                left -= 2 + keyevent.width() as u16;
+            }
+            for _ in 0..left + mpad {
+                text.push(' ');
+            }
+            let desc = desc.trim();
+            let w = lpad + keymaps_width + mpad + (desc.width() as u16) + rpad;
+            if w > width {
+                width = w;
+            }
+            writeln!(text, "{}", desc).ok();
+        }
+        Info {
+            title: title.to_string(),
+            text,
+            width,
+            height,
+        }
+    }
+}

helix-view/src/input.rs

@@ -0,0 +1,318 @@
+//! Input event handling, currently backed by crossterm.
+use anyhow::{anyhow, Error};
+use helix_core::unicode::width::UnicodeWidthStr;
+use serde::de::{self, Deserialize, Deserializer};
+use std::fmt;
+
+use crate::keyboard::{KeyCode, KeyModifiers};
+
+/// Represents a key event.
+// We use a newtype here because we want to customize Deserialize and Display.
+#[derive(Debug, PartialEq, Eq, PartialOrd, Clone, Copy, Hash)]
+pub struct KeyEvent {
+    pub code: KeyCode,
+    pub modifiers: KeyModifiers,
+}
+
+impl KeyEvent {
+    /// If a character was pressed (without modifiers), return it.
+    pub fn char(&self) -> Option<char> {
+        match self.code {
+            KeyCode::Char(ch) if self.modifiers.is_empty() => Some(ch),
+            _ => None,
+        }
+    }
+}
+
+pub(crate) mod keys {
+    pub(crate) const BACKSPACE: &str = "backspace";
+    pub(crate) const ENTER: &str = "ret";
+    pub(crate) const LEFT: &str = "left";
+    pub(crate) const RIGHT: &str = "right";
+    pub(crate) const UP: &str = "up";
+    pub(crate) const DOWN: &str = "down";
+    pub(crate) const HOME: &str = "home";
+    pub(crate) const END: &str = "end";
+    pub(crate) const PAGEUP: &str = "pageup";
+    pub(crate) const PAGEDOWN: &str = "pagedown";
+    pub(crate) const TAB: &str = "tab";
+    pub(crate) const BACKTAB: &str = "backtab";
+    pub(crate) const DELETE: &str = "del";
+    pub(crate) const INSERT: &str = "ins";
+    pub(crate) const NULL: &str = "null";
+    pub(crate) const ESC: &str = "esc";
+    pub(crate) const SPACE: &str = "space";
+    pub(crate) const LESS_THAN: &str = "lt";
+    pub(crate) const GREATER_THAN: &str = "gt";
+    pub(crate) const PLUS: &str = "plus";
+    pub(crate) const MINUS: &str = "minus";
+    pub(crate) const SEMICOLON: &str = "semicolon";
+    pub(crate) const PERCENT: &str = "percent";
+}
+
+impl fmt::Display for KeyEvent {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_fmt(format_args!(
+            "{}{}{}",
+            if self.modifiers.contains(KeyModifiers::SHIFT) {
+                "S-"
+            } else {
+                ""
+            },
+            if self.modifiers.contains(KeyModifiers::ALT) {
+                "A-"
+            } else {
+                ""
+            },
+            if self.modifiers.contains(KeyModifiers::CONTROL) {
+                "C-"
+            } else {
+                ""
+            },
+        ))?;
+        match self.code {
+            KeyCode::Backspace => f.write_str(keys::BACKSPACE)?,
+            KeyCode::Enter => f.write_str(keys::ENTER)?,
+            KeyCode::Left => f.write_str(keys::LEFT)?,
+            KeyCode::Right => f.write_str(keys::RIGHT)?,
+            KeyCode::Up => f.write_str(keys::UP)?,
+            KeyCode::Down => f.write_str(keys::DOWN)?,
+            KeyCode::Home => f.write_str(keys::HOME)?,
+            KeyCode::End => f.write_str(keys::END)?,
+            KeyCode::PageUp => f.write_str(keys::PAGEUP)?,
+            KeyCode::PageDown => f.write_str(keys::PAGEDOWN)?,
+            KeyCode::Tab => f.write_str(keys::TAB)?,
+            KeyCode::BackTab => f.write_str(keys::BACKTAB)?,
+            KeyCode::Delete => f.write_str(keys::DELETE)?,
+            KeyCode::Insert => f.write_str(keys::INSERT)?,
+            KeyCode::Null => f.write_str(keys::NULL)?,
+            KeyCode::Esc => f.write_str(keys::ESC)?,
+            KeyCode::Char(' ') => f.write_str(keys::SPACE)?,
+            KeyCode::Char('<') => f.write_str(keys::LESS_THAN)?,
+            KeyCode::Char('>') => f.write_str(keys::GREATER_THAN)?,
+            KeyCode::Char('+') => f.write_str(keys::PLUS)?,
+            KeyCode::Char('-') => f.write_str(keys::MINUS)?,
+            KeyCode::Char(';') => f.write_str(keys::SEMICOLON)?,
+            KeyCode::Char('%') => f.write_str(keys::PERCENT)?,
+            KeyCode::F(i) => f.write_fmt(format_args!("F{}", i))?,
+            KeyCode::Char(c) => f.write_fmt(format_args!("{}", c))?,
+        };
+        Ok(())
+    }
+}
+
+impl UnicodeWidthStr for KeyEvent {
+    fn width(&self) -> usize {
+        use helix_core::unicode::width::UnicodeWidthChar;
+        let mut width = match self.code {
+            KeyCode::Backspace => keys::BACKSPACE.len(),
+            KeyCode::Enter => keys::ENTER.len(),
+            KeyCode::Left => keys::LEFT.len(),
+            KeyCode::Right => keys::RIGHT.len(),
+            KeyCode::Up => keys::UP.len(),
+            KeyCode::Down => keys::DOWN.len(),
+            KeyCode::Home => keys::HOME.len(),
+            KeyCode::End => keys::END.len(),
+            KeyCode::PageUp => keys::PAGEUP.len(),
+            KeyCode::PageDown => keys::PAGEDOWN.len(),
+            KeyCode::Tab => keys::TAB.len(),
+            KeyCode::BackTab => keys::BACKTAB.len(),
+            KeyCode::Delete => keys::DELETE.len(),
+            KeyCode::Insert => keys::INSERT.len(),
+            KeyCode::Null => keys::NULL.len(),
+            KeyCode::Esc => keys::ESC.len(),
+            KeyCode::Char(' ') => keys::SPACE.len(),
+            KeyCode::Char('<') => keys::LESS_THAN.len(),
+            KeyCode::Char('>') => keys::GREATER_THAN.len(),
+            KeyCode::Char('+') => keys::PLUS.len(),
+            KeyCode::Char('-') => keys::MINUS.len(),
+            KeyCode::Char(';') => keys::SEMICOLON.len(),
+            KeyCode::Char('%') => keys::PERCENT.len(),
+            KeyCode::F(1..=9) => 2,
+            KeyCode::F(_) => 3,
+            KeyCode::Char(c) => c.width().unwrap_or(0),
+        };
+        if self.modifiers.contains(KeyModifiers::SHIFT) {
+            width += 2;
+        }
+        if self.modifiers.contains(KeyModifiers::ALT) {
+            width += 2;
+        }
+        if self.modifiers.contains(KeyModifiers::CONTROL) {
+            width += 2;
+        }
+        width
+    }
+
+    fn width_cjk(&self) -> usize {
+        self.width()
+    }
+}
+
+impl std::str::FromStr for KeyEvent {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let mut tokens: Vec<_> = s.split('-').collect();
+        let code = match tokens.pop().ok_or_else(|| anyhow!("Missing key code"))? {
+            keys::BACKSPACE => KeyCode::Backspace,
+            keys::ENTER => KeyCode::Enter,
+            keys::LEFT => KeyCode::Left,
+            keys::RIGHT => KeyCode::Right,
+            keys::UP => KeyCode::Up,
+            keys::DOWN => KeyCode::Down,
+            keys::HOME => KeyCode::Home,
+            keys::END => KeyCode::End,
+            keys::PAGEUP => KeyCode::PageUp,
+            keys::PAGEDOWN => KeyCode::PageDown,
+            keys::TAB => KeyCode::Tab,
+            keys::BACKTAB => KeyCode::BackTab,
+            keys::DELETE => KeyCode::Delete,
+            keys::INSERT => KeyCode::Insert,
+            keys::NULL => KeyCode::Null,
+            keys::ESC => KeyCode::Esc,
+            keys::SPACE => KeyCode::Char(' '),
+            keys::LESS_THAN => KeyCode::Char('<'),
+            keys::GREATER_THAN => KeyCode::Char('>'),
+            keys::PLUS => KeyCode::Char('+'),
+            keys::MINUS => KeyCode::Char('-'),
+            keys::SEMICOLON => KeyCode::Char(';'),
+            keys::PERCENT => KeyCode::Char('%'),
+            single if single.chars().count() == 1 => KeyCode::Char(single.chars().next().unwrap()),
+            function if function.len() > 1 && function.starts_with('F') => {
+                let function: String = function.chars().skip(1).collect();
+                let function = str::parse::<u8>(&function)?;
+                (function > 0 && function < 13)
+                    .then(|| KeyCode::F(function))
+                    .ok_or_else(|| anyhow!("Invalid function key '{}'", function))?
+            }
+            invalid => return Err(anyhow!("Invalid key code '{}'", invalid)),
+        };
+
+        let mut modifiers = KeyModifiers::empty();
+        for token in tokens {
+            let flag = match token {
+                "S" => KeyModifiers::SHIFT,
+                "A" => KeyModifiers::ALT,
+                "C" => KeyModifiers::CONTROL,
+                _ => return Err(anyhow!("Invalid key modifier '{}-'", token)),
+            };
+
+            if modifiers.contains(flag) {
+                return Err(anyhow!("Repeated key modifier '{}-'", token));
+            }
+            modifiers.insert(flag);
+        }
+
+        Ok(KeyEvent { code, modifiers })
+    }
+}
+
+impl<'de> Deserialize<'de> for KeyEvent {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        s.parse().map_err(de::Error::custom)
+    }
+}
+
+#[cfg(feature = "term")]
+impl From<crossterm::event::KeyEvent> for KeyEvent {
+    fn from(
+        crossterm::event::KeyEvent { code, modifiers }: crossterm::event::KeyEvent,
+    ) -> KeyEvent {
+        KeyEvent {
+            code: code.into(),
+            modifiers: modifiers.into(),
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn parsing_unmodified_keys() {
+        assert_eq!(
+            str::parse::<KeyEvent>("backspace").unwrap(),
+            KeyEvent {
+                code: KeyCode::Backspace,
+                modifiers: KeyModifiers::NONE
+            }
+        );
+
+        assert_eq!(
+            str::parse::<KeyEvent>("left").unwrap(),
+            KeyEvent {
+                code: KeyCode::Left,
+                modifiers: KeyModifiers::NONE
+            }
+        );
+
+        assert_eq!(
+            str::parse::<KeyEvent>(",").unwrap(),
+            KeyEvent {
+                code: KeyCode::Char(','),
+                modifiers: KeyModifiers::NONE
+            }
+        );
+
+        assert_eq!(
+            str::parse::<KeyEvent>("w").unwrap(),
+            KeyEvent {
+                code: KeyCode::Char('w'),
+                modifiers: KeyModifiers::NONE
+            }
+        );
+
+        assert_eq!(
+            str::parse::<KeyEvent>("F12").unwrap(),
+            KeyEvent {
+                code: KeyCode::F(12),
+                modifiers: KeyModifiers::NONE
+            }
+        );
+    }
+
+    #[test]
+    fn parsing_modified_keys() {
+        assert_eq!(
+            str::parse::<KeyEvent>("S-minus").unwrap(),
+            KeyEvent {
+                code: KeyCode::Char('-'),
+                modifiers: KeyModifiers::SHIFT
+            }
+        );
+
+        assert_eq!(
+            str::parse::<KeyEvent>("C-A-S-F12").unwrap(),
+            KeyEvent {
+                code: KeyCode::F(12),
+                modifiers: KeyModifiers::SHIFT | KeyModifiers::CONTROL | KeyModifiers::ALT
+            }
+        );
+
+        assert_eq!(
+            str::parse::<KeyEvent>("S-C-2").unwrap(),
+            KeyEvent {
+                code: KeyCode::Char('2'),
+                modifiers: KeyModifiers::SHIFT | KeyModifiers::CONTROL
+            }
+        );
+    }
+
+    #[test]
+    fn parsing_nonsensical_keys_fails() {
+        assert!(str::parse::<KeyEvent>("F13").is_err());
+        assert!(str::parse::<KeyEvent>("F0").is_err());
+        assert!(str::parse::<KeyEvent>("aaa").is_err());
+        assert!(str::parse::<KeyEvent>("S-S-a").is_err());
+        assert!(str::parse::<KeyEvent>("C-A-S-C-1").is_err());
+        assert!(str::parse::<KeyEvent>("FU").is_err());
+        assert!(str::parse::<KeyEvent>("123").is_err());
+        assert!(str::parse::<KeyEvent>("S--").is_err());
+    }
+}

helix-view/src/keyboard.rs

@@ -0,0 +1,156 @@
+use bitflags::bitflags;
+
+bitflags! {
+    /// Represents key modifiers (shift, control, alt).
+    #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+    pub struct KeyModifiers: u8 {
+        const SHIFT = 0b0000_0001;
+        const CONTROL = 0b0000_0010;
+        const ALT = 0b0000_0100;
+        const NONE = 0b0000_0000;
+    }
+}
+
+#[cfg(feature = "term")]
+impl From<KeyModifiers> for crossterm::event::KeyModifiers {
+    fn from(key_modifiers: KeyModifiers) -> Self {
+        use crossterm::event::KeyModifiers as CKeyModifiers;
+
+        let mut result = CKeyModifiers::NONE;
+
+        if key_modifiers.contains(KeyModifiers::SHIFT) {
+            result.insert(CKeyModifiers::SHIFT);
+        }
+        if key_modifiers.contains(KeyModifiers::CONTROL) {
+            result.insert(CKeyModifiers::CONTROL);
+        }
+        if key_modifiers.contains(KeyModifiers::ALT) {
+            result.insert(CKeyModifiers::ALT);
+        }
+
+        result
+    }
+}
+
+#[cfg(feature = "term")]
+impl From<crossterm::event::KeyModifiers> for KeyModifiers {
+    fn from(val: crossterm::event::KeyModifiers) -> Self {
+        use crossterm::event::KeyModifiers as CKeyModifiers;
+
+        let mut result = KeyModifiers::NONE;
+
+        if val.contains(CKeyModifiers::SHIFT) {
+            result.insert(KeyModifiers::SHIFT);
+        }
+        if val.contains(CKeyModifiers::CONTROL) {
+            result.insert(KeyModifiers::CONTROL);
+        }
+        if val.contains(CKeyModifiers::ALT) {
+            result.insert(KeyModifiers::ALT);
+        }
+
+        result
+    }
+}
+
+/// Represents a key.
+#[derive(Debug, PartialOrd, PartialEq, Eq, Clone, Copy, Hash)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+pub enum KeyCode {
+    /// Backspace key.
+    Backspace,
+    /// Enter key.
+    Enter,
+    /// Left arrow key.
+    Left,
+    /// Right arrow key.
+    Right,
+    /// Up arrow key.
+    Up,
+    /// Down arrow key.
+    Down,
+    /// Home key.
+    Home,
+    /// End key.
+    End,
+    /// Page up key.
+    PageUp,
+    /// Page down key.
+    PageDown,
+    /// Tab key.
+    Tab,
+    /// Shift + Tab key.
+    BackTab,
+    /// Delete key.
+    Delete,
+    /// Insert key.
+    Insert,
+    /// F key.
+    ///
+    /// `KeyCode::F(1)` represents F1 key, etc.
+    F(u8),
+    /// A character.
+    ///
+    /// `KeyCode::Char('c')` represents `c` character, etc.
+    Char(char),
+    /// Null.
+    Null,
+    /// Escape key.
+    Esc,
+}
+
+#[cfg(feature = "term")]
+impl From<KeyCode> for crossterm::event::KeyCode {
+    fn from(key_code: KeyCode) -> Self {
+        use crossterm::event::KeyCode as CKeyCode;
+
+        match key_code {
+            KeyCode::Backspace => CKeyCode::Backspace,
+            KeyCode::Enter => CKeyCode::Enter,
+            KeyCode::Left => CKeyCode::Left,
+            KeyCode::Right => CKeyCode::Right,
+            KeyCode::Up => CKeyCode::Up,
+            KeyCode::Down => CKeyCode::Down,
+            KeyCode::Home => CKeyCode::Home,
+            KeyCode::End => CKeyCode::End,
+            KeyCode::PageUp => CKeyCode::PageUp,
+            KeyCode::PageDown => CKeyCode::PageDown,
+            KeyCode::Tab => CKeyCode::Tab,
+            KeyCode::BackTab => CKeyCode::BackTab,
+            KeyCode::Delete => CKeyCode::Delete,
+            KeyCode::Insert => CKeyCode::Insert,
+            KeyCode::F(f_number) => CKeyCode::F(f_number),
+            KeyCode::Char(character) => CKeyCode::Char(character),
+            KeyCode::Null => CKeyCode::Null,
+            KeyCode::Esc => CKeyCode::Esc,
+        }
+    }
+}
+
+#[cfg(feature = "term")]
+impl From<crossterm::event::KeyCode> for KeyCode {
+    fn from(val: crossterm::event::KeyCode) -> Self {
+        use crossterm::event::KeyCode as CKeyCode;
+
+        match val {
+            CKeyCode::Backspace => KeyCode::Backspace,
+            CKeyCode::Enter => KeyCode::Enter,
+            CKeyCode::Left => KeyCode::Left,
+            CKeyCode::Right => KeyCode::Right,
+            CKeyCode::Up => KeyCode::Up,
+            CKeyCode::Down => KeyCode::Down,
+            CKeyCode::Home => KeyCode::Home,
+            CKeyCode::End => KeyCode::End,
+            CKeyCode::PageUp => KeyCode::PageUp,
+            CKeyCode::PageDown => KeyCode::PageDown,
+            CKeyCode::Tab => KeyCode::Tab,
+            CKeyCode::BackTab => KeyCode::BackTab,
+            CKeyCode::Delete => KeyCode::Delete,
+            CKeyCode::Insert => KeyCode::Insert,
+            CKeyCode::F(f_number) => KeyCode::F(f_number),
+            CKeyCode::Char(character) => KeyCode::Char(character),
+            CKeyCode::Null => KeyCode::Null,
+            CKeyCode::Esc => KeyCode::Esc,
+        }
+    }
+}

helix-view/src/lib.rs

@@ -1,13 +1,22 @@
+#[macro_use]
+pub mod macros;
+
+pub mod clipboard;
 pub mod document;
 pub mod editor;
+pub mod graphics;
+pub mod info;
+pub mod input;
+pub mod keyboard;
 pub mod register_selection;
 pub mod theme;
 pub mod tree;
 pub mod view;
 
-use slotmap::new_key_type;
-new_key_type! { pub struct DocumentId; }
-new_key_type! { pub struct ViewId; }
+slotmap::new_key_type! {
+    pub struct DocumentId;
+    pub struct ViewId;
+}
 
 pub use document::Document;
 pub use editor::Editor;

helix-view/src/macros.rs

@@ -0,0 +1,29 @@
+#[macro_export]
+macro_rules! current {
+    ( $( $editor:ident ).+ ) => {{
+        let view = $crate::view_mut!( $( $editor ).+ );
+        let doc = &mut $( $editor ).+ .documents[view.doc];
+        (view, doc)
+    }};
+}
+
+#[macro_export]
+macro_rules! doc_mut {
+    ( $( $editor:ident ).+ ) => {{
+        $crate::current!( $( $editor ).+ ).1
+    }};
+}
+
+#[macro_export]
+macro_rules! view_mut {
+    ( $( $editor:ident ).+ ) => {{
+        $( $editor ).+ .tree.get_mut($( $editor ).+ .tree.focus)
+    }};
+}
+
+#[macro_export]
+macro_rules! view {
+    ( $( $editor:ident ).+ ) => {{
+        $( $editor ).+ .tree.get($( $editor ).+ .tree.focus)
+    }};
+}

helix-view/src/theme.rs

@@ -1,92 +1,82 @@
-use std::collections::HashMap;
+use std::{
+    collections::HashMap,
+    convert::TryFrom,
+    path::{Path, PathBuf},
+};
 
+use anyhow::Context;
 use log::warn;
+use once_cell::sync::Lazy;
 use serde::{Deserialize, Deserializer};
 use toml::Value;
 
-#[cfg(feature = "term")]
-pub use tui::style::{Color, Modifier, Style};
+pub use crate::graphics::{Color, Modifier, Style};
 
-// #[derive(Clone, Copy, PartialEq, Eq, Default, Hash)]
-// pub struct Color {
-//     pub r: u8,
-//     pub g: u8,
-//     pub b: u8,
-// }
+pub static DEFAULT_THEME: Lazy<Theme> = Lazy::new(|| {
+    toml::from_slice(include_bytes!("../../theme.toml")).expect("Failed to parse default theme")
+});
 
-// impl Color {
-//     pub fn new(r: u8, g: u8, b: u8) -> Self {
-//         Self { r, g, b }
-//     }
-// }
+#[derive(Clone, Debug)]
+pub struct Loader {
+    user_dir: PathBuf,
+    default_dir: PathBuf,
+}
+impl Loader {
+    /// Creates a new loader that can load themes from two directories.
+    pub fn new<P: AsRef<Path>>(user_dir: P, default_dir: P) -> Self {
+        Self {
+            user_dir: user_dir.as_ref().join("themes"),
+            default_dir: default_dir.as_ref().join("themes"),
+        }
+    }
 
-// #[cfg(feature = "term")]
-// impl Into<tui::style::Color> for Color {
-//     fn into(self) -> tui::style::Color {
-//         tui::style::Color::Rgb(self.r, self.g, self.b)
-//     }
-// }
+    /// Loads a theme first looking in the `user_dir` then in `default_dir`
+    pub fn load(&self, name: &str) -> Result<Theme, anyhow::Error> {
+        if name == "default" {
+            return Ok(self.default());
+        }
+        let filename = format!("{}.toml", name);
 
-// impl std::str::FromStr for Color {
-//     type Err = ();
+        let user_path = self.user_dir.join(&filename);
+        let path = if user_path.exists() {
+            user_path
+        } else {
+            self.default_dir.join(filename)
+        };
 
-//     /// Tries to parse a string (`'#FFFFFF'` or `'FFFFFF'`) into RGB.
-//     fn from_str(input: &str) -> Result<Self, Self::Err> {
-//         let input = input.trim();
-//         let input = match (input.chars().next(), input.len()) {
-//             (Some('#'), 7) => &input[1..],
-//             (_, 6) => input,
-//             _ => return Err(()),
-//         };
+        let data = std::fs::read(&path)?;
+        toml::from_slice(data.as_slice()).context("Failed to deserialize theme")
+    }
 
-//         u32::from_str_radix(&input, 16)
-//             .map(|s| Color {
-//                 r: ((s >> 16) & 0xFF) as u8,
-//                 g: ((s >> 8) & 0xFF) as u8,
-//                 b: (s & 0xFF) as u8,
-//             })
-//             .map_err(|_| ())
-//     }
-// }
+    pub fn read_names(path: &Path) -> Vec<String> {
+        std::fs::read_dir(path)
+            .map(|entries| {
+                entries
+                    .filter_map(|entry| {
+                        let entry = entry.ok()?;
+                        let path = entry.path();
+                        (path.extension()? == "toml")
+                            .then(|| path.file_stem().unwrap().to_string_lossy().into_owned())
+                    })
+                    .collect()
+            })
+            .unwrap_or_default()
+    }
 
-// #[derive(Clone, Copy, PartialEq, Eq, Default, Hash)]
-// pub struct Style {
-//     pub fg: Option<Color>,
-//     pub bg: Option<Color>,
-//     // TODO: modifiers (bold, underline, italic, etc)
-// }
+    /// Lists all theme names available in default and user directory
+    pub fn names(&self) -> Vec<String> {
+        let mut names = Self::read_names(&self.user_dir);
+        names.extend(Self::read_names(&self.default_dir));
+        names
+    }
 
-// impl Style {
-//     pub fn fg(mut self, fg: Color) -> Self {
-//         self.fg = Some(fg);
-//         self
-//     }
+    /// Returns the default theme
+    pub fn default(&self) -> Theme {
+        DEFAULT_THEME.clone()
+    }
+}
 
-//     pub fn bg(mut self, bg: Color) -> Self {
-//         self.bg = Some(bg);
-//         self
-//     }
-// }
-
-// #[cfg(feature = "term")]
-// impl Into<tui::style::Style> for Style {
-//     fn into(self) -> tui::style::Style {
-//         let style = tui::style::Style::default();
-
-//         if let Some(fg) = self.fg {
-//             style.fg(fg.into());
-//         }
-
-//         if let Some(bg) = self.bg {
-//             style.bg(bg.into());
-//         }
-
-//         style
-//     }
-// }
-
-/// Color theme for syntax highlighting.
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 pub struct Theme {
     scopes: Vec<String>,
     styles: HashMap<String, Style>,
@@ -99,13 +89,24 @@ impl<'de> Deserialize<'de> for Theme {
     {
         let mut styles = HashMap::new();
 
-        if let Ok(colors) = HashMap::<String, Value>::deserialize(deserializer) {
-            // scopes.reserve(colors.len());
+        if let Ok(mut colors) = HashMap::<String, Value>::deserialize(deserializer) {
+            // TODO: alert user of parsing failures in editor
+            let palette = colors
+                .remove("palette")
+                .map(|value| {
+                    ThemePalette::try_from(value).unwrap_or_else(|err| {
+                        warn!("{}", err);
+                        ThemePalette::default()
+                    })
+                })
+                .unwrap_or_default();
+
             styles.reserve(colors.len());
             for (name, style_value) in colors {
                 let mut style = Style::default();
-                parse_style(&mut style, style_value);
-                // scopes.push(name);
+                if let Err(err) = palette.parse_style(&mut style, style_value) {
+                    warn!("{}", err);
+                }
                 styles.insert(name, style);
             }
         }
@@ -115,100 +116,120 @@ impl<'de> Deserialize<'de> for Theme {
     }
 }
 
-fn parse_style(style: &mut Style, value: Value) {
-    //TODO: alert user of parsing failures
-    if let Value::Table(entries) = value {
-        for (name, value) in entries {
-            match name.as_str() {
-                "fg" => {
-                    if let Some(color) = parse_color(value) {
-                        *style = style.fg(color);
+impl Theme {
+    pub fn get(&self, scope: &str) -> Style {
+        self.try_get(scope)
+            .unwrap_or_else(|| Style::default().fg(Color::Rgb(0, 0, 255)))
     }
+
+    pub fn try_get(&self, scope: &str) -> Option<Style> {
+        self.styles.get(scope).copied()
     }
-                "bg" => {
-                    if let Some(color) = parse_color(value) {
-                        *style = style.bg(color);
+
+    #[inline]
+    pub fn scopes(&self) -> &[String] {
+        &self.scopes
     }
-                }
-                "modifiers" => {
-                    if let Value::Array(arr) = value {
-                        for modifier in arr.iter().filter_map(parse_modifier) {
-                            *style = style.add_modifier(modifier);
-                        }
-                    }
-                }
-                _ => (),
-            }
-        }
-    } else if let Some(color) = parse_color(value) {
-        *style = style.fg(color);
+
+    pub fn find_scope_index(&self, scope: &str) -> Option<usize> {
+        self.scopes().iter().position(|s| s == scope)
     }
 }
 
-fn hex_string_to_rgb(s: &str) -> Option<(u8, u8, u8)> {
+struct ThemePalette {
+    palette: HashMap<String, Color>,
+}
+
+impl Default for ThemePalette {
+    fn default() -> Self {
+        Self::new(HashMap::new())
+    }
+}
+
+impl ThemePalette {
+    pub fn new(palette: HashMap<String, Color>) -> Self {
+        Self { palette }
+    }
+
+    pub fn hex_string_to_rgb(s: &str) -> Result<Color, String> {
         if s.starts_with('#') && s.len() >= 7 {
             if let (Ok(red), Ok(green), Ok(blue)) = (
                 u8::from_str_radix(&s[1..3], 16),
                 u8::from_str_radix(&s[3..5], 16),
                 u8::from_str_radix(&s[5..7], 16),
             ) {
-            Some((red, green, blue))
-        } else {
-            None
-        }
-    } else {
-        None
+                return Ok(Color::Rgb(red, green, blue));
             }
         }
 
-fn parse_color(value: Value) -> Option<Color> {
-    if let Value::String(s) = value {
-        if let Some((red, green, blue)) = hex_string_to_rgb(&s) {
-            Some(Color::Rgb(red, green, blue))
-        } else {
-            warn!("malformed hexcode in theme: {}", s);
-            None
-        }
-    } else {
-        warn!("unrecognized value in theme: {}", value);
-        None
-    }
+        Err(format!("Theme: malformed hexcode: {}", s))
     }
 
-fn parse_modifier(value: &Value) -> Option<Modifier> {
-    if let Value::String(s) = value {
-        match s.as_str() {
-            "bold" => Some(Modifier::BOLD),
-            "dim" => Some(Modifier::DIM),
-            "italic" => Some(Modifier::ITALIC),
-            "underlined" => Some(Modifier::UNDERLINED),
-            "slow_blink" => Some(Modifier::SLOW_BLINK),
-            "rapid_blink" => Some(Modifier::RAPID_BLINK),
-            "reversed" => Some(Modifier::REVERSED),
-            "hidden" => Some(Modifier::HIDDEN),
-            "crossed_out" => Some(Modifier::CROSSED_OUT),
-            _ => {
-                warn!("unrecognized modifier in theme: {}", s);
-                None
-            }
-        }
-    } else {
-        warn!("unrecognized modifier in theme: {}", value);
-        None
-    }
+    fn parse_value_as_str(value: &Value) -> Result<&str, String> {
+        value
+            .as_str()
+            .ok_or(format!("Theme: unrecognized value: {}", value))
     }
 
-impl Theme {
-    pub fn get(&self, scope: &str) -> Style {
-        self.styles
-            .get(scope)
+    pub fn parse_color(&self, value: Value) -> Result<Color, String> {
+        let value = Self::parse_value_as_str(&value)?;
+
+        self.palette
+            .get(value)
             .copied()
-            .unwrap_or_else(|| Style::default().fg(Color::Rgb(0, 0, 255)))
+            .ok_or("")
+            .or_else(|_| Self::hex_string_to_rgb(value))
     }
 
-    #[inline]
-    pub fn scopes(&self) -> &[String] {
-        &self.scopes
+    pub fn parse_modifier(value: &Value) -> Result<Modifier, String> {
+        value
+            .as_str()
+            .and_then(|s| s.parse().ok())
+            .ok_or(format!("Theme: invalid modifier: {}", value))
+    }
+
+    pub fn parse_style(&self, style: &mut Style, value: Value) -> Result<(), String> {
+        if let Value::Table(entries) = value {
+            for (name, value) in entries {
+                match name.as_str() {
+                    "fg" => *style = style.fg(self.parse_color(value)?),
+                    "bg" => *style = style.bg(self.parse_color(value)?),
+                    "modifiers" => {
+                        let modifiers = value
+                            .as_array()
+                            .ok_or("Theme: modifiers should be an array")?;
+
+                        for modifier in modifiers {
+                            *style = style.add_modifier(Self::parse_modifier(modifier)?);
+                        }
+                    }
+                    _ => return Err(format!("Theme: invalid style attribute: {}", name)),
+                }
+            }
+        } else {
+            *style = style.fg(self.parse_color(value)?);
+        }
+        Ok(())
+    }
+}
+
+impl TryFrom<Value> for ThemePalette {
+    type Error = String;
+
+    fn try_from(value: Value) -> Result<Self, Self::Error> {
+        let map = match value {
+            Value::Table(entries) => entries,
+            _ => return Ok(Self::default()),
+        };
+
+        let mut palette = HashMap::with_capacity(map.len());
+        for (name, value) in map {
+            let value = Self::parse_value_as_str(&value)?;
+            let color = Self::hex_string_to_rgb(value)?;
+            palette.insert(name, color);
+        }
+
+        Ok(Self::new(palette))
     }
 }
 
@@ -217,7 +238,21 @@ fn test_parse_style_string() {
     let fg = Value::String("#ffffff".to_string());
 
     let mut style = Style::default();
-    parse_style(&mut style, fg);
+    let palette = ThemePalette::default();
+    palette.parse_style(&mut style, fg).unwrap();
+
+    assert_eq!(style, Style::default().fg(Color::Rgb(255, 255, 255)));
+}
+
+#[test]
+fn test_palette() {
+    use helix_core::hashmap;
+    let fg = Value::String("my_color".to_string());
+
+    let mut style = Style::default();
+    let palette =
+        ThemePalette::new(hashmap! { "my_color".to_string() => Color::Rgb(255, 255, 255) });
+    palette.parse_style(&mut style, fg).unwrap();
 
     assert_eq!(style, Style::default().fg(Color::Rgb(255, 255, 255)));
 }
@@ -233,9 +268,10 @@ fn test_parse_style_table() {
     };
 
     let mut style = Style::default();
+    let palette = ThemePalette::default();
     if let Value::Table(entries) = table {
         for (_name, value) in entries {
-            parse_style(&mut style, value);
+            palette.parse_style(&mut style, value).unwrap();
         }
     }
 

helix-view/src/tree.rs

@@ -1,8 +1,7 @@
-use crate::{View, ViewId};
+use crate::{graphics::Rect, View, ViewId};
 use slotmap::HopSlotMap;
-use tui::layout::Rect;
 
-// the dimensions are recomputed on windo resize/tree change.
+// the dimensions are recomputed on window resize/tree change.
 //
 #[derive(Debug)]
 pub struct Tree {
@@ -434,6 +433,10 @@ impl Tree {
             self.focus = key;
         }
     }
+
+    pub fn area(&self) -> Rect {
+        self.area
+    }
 }
 
 #[derive(Debug)]

helix-view/src/view.rs

@@ -1,18 +1,16 @@
 use std::borrow::Cow;
 
-use crate::{Document, DocumentId, ViewId};
+use crate::{graphics::Rect, Document, DocumentId, ViewId};
 use helix_core::{
     coords_at_pos,
     graphemes::{grapheme_width, RopeGraphemes},
+    line_ending::line_end_char_index,
     Position, RopeSlice, Selection,
 };
-use tui::layout::Rect;
-
-pub const PADDING: usize = 5;
 
 type Jump = (DocumentId, Selection);
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct JumpList {
     jumps: Vec<Jump>,
     current: usize,
@@ -84,19 +82,25 @@ impl View {
         }
     }
 
-    pub fn ensure_cursor_in_view(&mut self, doc: &Document) {
-        let cursor = doc.selection(self.id).cursor();
+    pub fn ensure_cursor_in_view(&mut self, doc: &Document, scrolloff: usize) {
+        let cursor = doc
+            .selection(self.id)
+            .primary()
+            .cursor(doc.text().slice(..));
         let pos = coords_at_pos(doc.text().slice(..), cursor);
         let line = pos.row;
         let col = pos.col;
         let height = self.area.height.saturating_sub(1); // - 1 for statusline
-        let last_line = self.first_line + height as usize;
+        let last_line = (self.first_line + height as usize).saturating_sub(1);
 
-        let scrolloff = PADDING.min(self.area.height as usize / 2); // TODO: user pref
+        // - 1 so we have at least one gap in the middle.
+        // a height of 6 with padding of 3 on each side will keep shifting the view back and forth
+        // as we type
+        let scrolloff = scrolloff.min(height.saturating_sub(1) as usize / 2);
 
         // TODO: not ideal
         const OFFSET: usize = 7; // 1 diagnostic + 5 linenr + 1 gutter
-        let last_col = self.first_col + (self.area.width as usize - OFFSET);
+        let last_col = (self.first_col + self.area.width as usize).saturating_sub(OFFSET + 1);
 
         if line > last_line.saturating_sub(scrolloff) {
             // scroll down
@@ -120,8 +124,9 @@ impl View {
     pub fn last_line(&self, doc: &Document) -> usize {
         let height = self.area.height.saturating_sub(1); // - 1 for statusline
         std::cmp::min(
-            self.first_line + height as usize,
-            doc.text().len_lines() - 1,
+            // Saturating subs to make it inclusive zero indexing.
+            (self.first_line + height as usize).saturating_sub(1),
+            doc.text().len_lines().saturating_sub(1),
         )
     }
 
@@ -162,6 +167,59 @@ impl View {
         Some(Position::new(row, col))
     }
 
+    pub fn text_pos_at_screen_coords(
+        &self,
+        text: &RopeSlice,
+        row: u16,
+        column: u16,
+        tab_width: usize,
+    ) -> Option<usize> {
+        // TODO: not ideal
+        const OFFSET: u16 = 7; // 1 diagnostic + 5 linenr + 1 gutter
+
+        // 2 for status
+        if row < self.area.top() || row > self.area.bottom().saturating_sub(2) {
+            return None;
+        }
+
+        if column < self.area.left() + OFFSET || column > self.area.right() {
+            return None;
+        }
+
+        let line_number = (row - self.area.y) as usize + self.first_line;
+
+        if line_number > text.len_lines() - 1 {
+            return Some(text.len_chars());
+        }
+
+        let mut pos = text.line_to_char(line_number);
+
+        let current_line = text.line(line_number);
+
+        let target = (column - OFFSET - self.area.x) as usize + self.first_col;
+        let mut selected = 0;
+
+        for grapheme in RopeGraphemes::new(current_line) {
+            if selected >= target {
+                break;
+            }
+            if grapheme == "\t" {
+                selected += tab_width;
+            } else {
+                let width = grapheme_width(&Cow::from(grapheme));
+                selected += width;
+            }
+            pos += grapheme.chars().count();
+        }
+
+        Some(pos.min(line_end_char_index(&text.slice(..), line_number)))
+    }
+
+    /// Translates a screen position to position in the text document.
+    /// Returns a usize typed position in bounds of the text if found in this view, None if out of view.
+    pub fn pos_at_screen_coords(&self, doc: &Document, row: u16, column: u16) -> Option<usize> {
+        self.text_pos_at_screen_coords(&doc.text().slice(..), row, column, doc.tab_width())
+    }
     // pub fn traverse<F>(&self, text: RopeSlice, start: usize, end: usize, fun: F)
     // where
     //     F: Fn(usize, usize),
@@ -183,3 +241,122 @@ impl View {
     //     }
     // }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use helix_core::Rope;
+    const OFFSET: u16 = 7; // 1 diagnostic + 5 linenr + 1 gutter
+
+    #[test]
+    fn test_text_pos_at_screen_coords() {
+        let mut view = View::new(DocumentId::default());
+        view.area = Rect::new(40, 40, 40, 40);
+        let rope = Rope::from_str("abc\n\tdef");
+        let text = rope.slice(..);
+
+        assert_eq!(view.text_pos_at_screen_coords(&text, 40, 2, 4), None);
+
+        assert_eq!(view.text_pos_at_screen_coords(&text, 40, 41, 4), None);
+
+        assert_eq!(view.text_pos_at_screen_coords(&text, 0, 2, 4), None);
+
+        assert_eq!(view.text_pos_at_screen_coords(&text, 0, 49, 4), None);
+
+        assert_eq!(view.text_pos_at_screen_coords(&text, 0, 41, 4), None);
+
+        assert_eq!(view.text_pos_at_screen_coords(&text, 40, 81, 4), None);
+
+        assert_eq!(view.text_pos_at_screen_coords(&text, 78, 41, 4), None);
+
+        assert_eq!(
+            view.text_pos_at_screen_coords(&text, 40, 40 + OFFSET + 3, 4),
+            Some(3)
+        );
+
+        assert_eq!(view.text_pos_at_screen_coords(&text, 40, 80, 4), Some(3));
+
+        assert_eq!(
+            view.text_pos_at_screen_coords(&text, 41, 40 + OFFSET + 1, 4),
+            Some(5)
+        );
+
+        assert_eq!(
+            view.text_pos_at_screen_coords(&text, 41, 40 + OFFSET + 4, 4),
+            Some(5)
+        );
+
+        assert_eq!(
+            view.text_pos_at_screen_coords(&text, 41, 40 + OFFSET + 7, 4),
+            Some(8)
+        );
+
+        assert_eq!(view.text_pos_at_screen_coords(&text, 41, 80, 4), Some(8));
+    }
+
+    #[test]
+    fn test_text_pos_at_screen_coords_cjk() {
+        let mut view = View::new(DocumentId::default());
+        view.area = Rect::new(40, 40, 40, 40);
+        let rope = Rope::from_str("Hi! こんにちは皆さん");
+        let text = rope.slice(..);
+
+        assert_eq!(
+            view.text_pos_at_screen_coords(&text, 40, 40 + OFFSET + 0, 4),
+            Some(0)
+        );
+
+        assert_eq!(
+            view.text_pos_at_screen_coords(&text, 40, 40 + OFFSET + 5, 4),
+            Some(5)
+        );
+
+        assert_eq!(
+            view.text_pos_at_screen_coords(&text, 40, 40 + OFFSET + 6, 4),
+            Some(5)
+        );
+
+        assert_eq!(
+            view.text_pos_at_screen_coords(&text, 40, 40 + OFFSET + 7, 4),
+            Some(6)
+        );
+
+        assert_eq!(
+            view.text_pos_at_screen_coords(&text, 40, 40 + OFFSET + 8, 4),
+            Some(6)
+        );
+    }
+
+    #[test]
+    fn test_text_pos_at_screen_coords_graphemes() {
+        let mut view = View::new(DocumentId::default());
+        view.area = Rect::new(40, 40, 40, 40);
+        let rope = Rope::from_str("Hèl̀l̀ò world!");
+        let text = rope.slice(..);
+
+        assert_eq!(
+            view.text_pos_at_screen_coords(&text, 40, 40 + OFFSET + 0, 4),
+            Some(0)
+        );
+
+        assert_eq!(
+            view.text_pos_at_screen_coords(&text, 40, 40 + OFFSET + 1, 4),
+            Some(1)
+        );
+
+        assert_eq!(
+            view.text_pos_at_screen_coords(&text, 40, 40 + OFFSET + 2, 4),
+            Some(3)
+        );
+
+        assert_eq!(
+            view.text_pos_at_screen_coords(&text, 40, 40 + OFFSET + 3, 4),
+            Some(5)
+        );
+
+        assert_eq!(
+            view.text_pos_at_screen_coords(&text, 40, 40 + OFFSET + 4, 4),
+            Some(7)
+        );
+    }
+}