skidder prototype

2025-10-05 16:02:44 +02:00 · 2024-08-09 02:54:16 +02:00
parent 79d1529845
commit 9ab700f07c
10 changed files with 584 additions and 10 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -74,6 +74,19 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"

+[[package]]
+name = "console"
+version = "0.15.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb"
+dependencies = [
+ "encode_unicode",
+ "lazy_static",
+ "libc",
+ "unicode-width",
+ "windows-sys",
+]
+
 [[package]]
 name = "cpufeatures"
 version = "0.2.12"
@@ -103,6 +116,12 @@ dependencies = [
 "crypto-common",
 ]

+[[package]]
+name = "encode_unicode"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
+
 [[package]]
 name = "errno"
 version = "0.3.9"
@@ -153,6 +172,40 @@ dependencies = [
 "twig-sitter",
 ]

+[[package]]
+name = "indicatif"
+version = "0.17.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3"
+dependencies = [
+ "console",
+ "instant",
+ "number_prefix",
+ "portable-atomic",
+ "unicode-width",
+]
+
+[[package]]
+name = "instant"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
+
+[[package]]
+name = "lazy_static"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
+
 [[package]]
 name = "libc"
 version = "0.2.155"
@@ -187,12 +240,24 @@ version = "2.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"

+[[package]]
+name = "number_prefix"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
+
 [[package]]
 name = "once_cell"
 version = "1.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"

+[[package]]
+name = "portable-atomic"
+version = "1.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265"
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.86"
@@ -276,6 +341,53 @@ dependencies = [
 "windows-sys",
 ]

+[[package]]
+name = "ryu"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
+
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.205"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e33aedb1a7135da52b7c21791455563facbbcc43d0f0f66165b42c21b3dfb150"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.205"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "692d6f5ac90220161d6774db30c662202721e64aed9058d2c394f451261420c1"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.122"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da"
+dependencies = [
+ "itoa",
+ "memchr",
+ "ryu",
+ "serde",
+]
+
 [[package]]
 name = "sha1"
 version = "0.10.6"
@@ -293,10 +405,25 @@ version = "0.1.0"
 dependencies = [
 "anyhow",
 "cc",
+ "indicatif",
+ "serde",
+ "serde_json",
 "sha1",
 "tempfile",
 ]

+[[package]]
+name = "skidder-cli"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "serde",
+ "serde_json",
+ "skidder",
+ "walkdir",
+ "xflags",
+]
+
 [[package]]
 name = "slab"
 version = "0.4.9"
@@ -384,12 +511,37 @@ version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"

+[[package]]
+name = "unicode-width"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d"
+
 [[package]]
 name = "version_check"
 version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"

+[[package]]
+name = "walkdir"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
+dependencies = [
+ "same-file",
+ "winapi-util",
+]
+
+[[package]]
+name = "winapi-util"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b"
+dependencies = [
+ "windows-sys",
+]
+
 [[package]]
 name = "windows-sys"
 version = "0.52.0"
@@ -463,6 +615,21 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"

+[[package]]
+name = "xflags"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d9e15fbb3de55454b0106e314b28e671279009b363e6f1d8e39fdc3bf048944"
+dependencies = [
+ "xflags-macros",
+]
+
+[[package]]
+name = "xflags-macros"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "672423d4fea7ffa2f6c25ba60031ea13dc6258070556f125cc4d790007d4a155"
+
 [[package]]
 name = "zerocopy"
 version = "0.7.35"
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,3 +1,3 @@
 [workspace]
 resolver = "2"
-members = ["bindings", "highlighter", "skidder"]
+members = ["bindings", "cli", "highlighter", "skidder"]
--- a/cli/Cargo.toml
+++ b/cli/Cargo.toml
@@ -0,0 +1,20 @@
+[package]
+name = "skidder-cli"
+version = "0.1.0"
+edition = "2021"
+description = "A package mangager for tree-sitter"
+authors = ["Pascal Kuthe <pascalkuthe@pm.me>"]
+license = "MPL-2.0"
+repository = "https://github.com/helix-editor/sappling"
+readme = "../README.md"
+rust-version = "1.76.0"
+
+[dependencies]
+anyhow = "1.0.86"
+serde = "1.0.205"
+serde_json = "1.0.122"
+walkdir = "2.5.0"
+xflags = "0.3.2"
+
+skidder = { path = "../skidder" }
+
--- a/cli/import.sh
+++ b/cli/import.sh
@@ -0,0 +1,16 @@
+#!/bin/env bash
+
+set -e
+cargo build
+../target/debug/skidder-cli import --metadata -r ../../../tree-sitter-grammars/ ../../../master/runtime/grammars/sources/*
+../target/debug/skidder-cli import --metadata -r ../../../tree-sitter-grammars ../../../master/runtime/grammars/sources/markdown/tree-sitter-markdown:markdown
+../target/debug/skidder-cli import --metadata -r ../../../tree-sitter-grammars ../../../master/runtime/grammars/sources/markdown/tree-sitter-markdown-inline:markdown-inline
+../target/debug/skidder-cli import --metadata -r ../../../tree-sitter-grammars ../../../master/runtime/grammars/sources/v/tree_sitter_v:v
+../target/debug/skidder-cli import --metadata -r ../../../tree-sitter-grammars ../../../master/runtime/grammars/sources/wat/wat
+../target/debug/skidder-cli import --metadata -r ../../../tree-sitter-grammars ../../../master/runtime/grammars/sources/wat/wast
+../target/debug/skidder-cli import --metadata -r ../../../tree-sitter-grammars ../../../master/runtime/grammars/sources/typescript/typescript
+../target/debug/skidder-cli import --metadata -r ../../../tree-sitter-grammars ../../../master/runtime/grammars/sources/typescript/tsx
+../target/debug/skidder-cli import --metadata -r ../../../tree-sitter-grammars ../../../master/runtime/grammars/sources/php_only/php_only
+../target/debug/skidder-cli import --metadata -r ../../../tree-sitter-grammars ../../../master/runtime/grammars/sources/php-only/php_only:php-only
+../target/debug/skidder-cli import --metadata -r ../../../tree-sitter-grammars ../../../master/runtime/grammars/sources/ocaml/ocaml
+../target/debug/skidder-cli import --metadata -r ../../../tree-sitter-grammars ../../../master/runtime/grammars/sources/ocaml/interface:ocaml-interface
--- a/cli/src/flags.rs
+++ b/cli/src/flags.rs
@@ -0,0 +1,80 @@
+use std::ffi::OsString;
+use std::path::PathBuf;
+
+xflags::xflags! {
+    src "./src/flags.rs"
+
+    cmd skidder {
+        cmd import {
+            /// Wether to import queries
+            optional --import-queries
+            /// Wether to (re)generate metadata
+            optional --metadata
+            /// The repository/diretocy where repos are copied into
+            /// Defaults to the current working directory
+            optional -r,--repo repo: PathBuf
+            /// the path of the grammars to import the name of the directory
+            /// will be used as the grammar name. To overwrite you can append
+            /// the grammar name with a colon
+            repeated path: PathBuf
+        }
+        cmd build {
+            optional --verbose
+            optional -j, --threads threads: usize
+            optional -f, --force
+            required repo: PathBuf
+            optional grammar: String
+        }
+
+    }
+}
+// generated start
+// The following code is generated by `xflags` macro.
+// Run `env UPDATE_XFLAGS=1 cargo build` to regenerate.
+#[derive(Debug)]
+pub struct Skidder {
+    pub subcommand: SkidderCmd,
+}
+
+#[derive(Debug)]
+pub enum SkidderCmd {
+    Import(Import),
+    Build(Build),
+}
+
+#[derive(Debug)]
+pub struct Import {
+    pub path: Vec<PathBuf>,
+
+    pub import_queries: bool,
+    pub metadata: bool,
+    pub repo: Option<PathBuf>,
+}
+
+#[derive(Debug)]
+pub struct Build {
+    pub repo: PathBuf,
+    pub grammar: Option<String>,
+
+    pub verbose: bool,
+    pub threads: Option<usize>,
+    pub force: bool,
+}
+
+impl Skidder {
+    #[allow(dead_code)]
+    pub fn from_env_or_exit() -> Self {
+        Self::from_env_or_exit_()
+    }
+
+    #[allow(dead_code)]
+    pub fn from_env() -> xflags::Result<Self> {
+        Self::from_env_()
+    }
+
+    #[allow(dead_code)]
+    pub fn from_vec(args: Vec<std::ffi::OsString>) -> xflags::Result<Self> {
+        Self::from_vec_(args)
+    }
+}
+// generated end
--- a/cli/src/import.rs
+++ b/cli/src/import.rs
@@ -0,0 +1,194 @@
+use std::env::current_dir;
+use std::io::Write;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+use std::{fs, io};
+
+use anyhow::{bail, Context, Result};
+use serde::Deserialize;
+use skidder::Metadata;
+use walkdir::WalkDir;
+
+use crate::flags::Import;
+const LICENSE_FILE_NAMES: &[&str] = &["LICENSE", "LICENSE.txt", "LICENCE", "LICENCE"];
+const LICENSE_SEARCH: &[(&str, &str)] = &[
+    ("unlicense", "unlicense"),
+    ("EUROPEAN UNION PUBLIC LICENCE v. 1.2", "EUPL-1.2"),
+    ("The Artistic License 2.0", "Artistic-2.0"),
+    ("Apache License", "Apache-2.0"),
+    ("GNU GENERAL PUBLIC LICENSE", "GPL-3.0"),
+    ("MIT License", "MIT"),
+    ("DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE", "WTFPL"),
+];
+
+impl Import {
+    fn repo(&self) -> Result<PathBuf> {
+        match &self.repo {
+            Some(path) => Ok(path.clone()),
+            None => Ok(current_dir()?),
+        }
+    }
+
+    pub fn run(self) -> Result<()> {
+        let repo = self.repo()?;
+        for path in &self.path {
+            let Some(dir_name) = path.file_name().and_then(|file_name| file_name.to_str()) else {
+                bail!("invalid path {path:?}");
+            };
+            let mut src_path = path.to_owned();
+            let grammar_name = match dir_name.rsplit_once(':') {
+                Some((dir_name, grammar_name)) => {
+                    src_path.set_file_name(dir_name);
+                    grammar_name
+                }
+                None => dir_name,
+            };
+            src_path.push("src");
+            let dst_path = repo.join(grammar_name);
+            fs::create_dir_all(&dst_path)
+                .with_context(|| format!("failed to create {}", dst_path.display()))?;
+            if !src_path.join("parser.c").exists() {
+                eprintln!(
+                    "skipping grammar {grammar_name}: no parser.c found at {}!",
+                    src_path.display()
+                );
+                continue;
+            }
+            println!("importing {grammar_name}");
+            for file in WalkDir::new(&src_path) {
+                let file = file?;
+                if !file.file_type().is_file() {
+                    continue;
+                }
+                let Some(file_name) = file.file_name().to_str() else {
+                    continue;
+                };
+                let Some((_, extension)) = file_name.rsplit_once('.') else {
+                    continue;
+                };
+                if !(matches!(extension, "h" | "c" | "cc")
+                    || extension == "scm" && self.import_queries)
+                {
+                    continue;
+                }
+                let relative_path = file.path().strip_prefix(&src_path).unwrap();
+                let dst_path = dst_path.join(relative_path);
+                fs::create_dir_all(dst_path.parent().unwrap()).with_context(|| {
+                    format!("failed to create {}", dst_path.parent().unwrap().display())
+                })?;
+                fs::copy(file.path(), &dst_path).with_context(|| {
+                    format!(
+                        "failed to copy {} to {}",
+                        src_path.display(),
+                        dst_path.display()
+                    )
+                })?;
+            }
+            src_path.pop();
+            let license_file = LICENSE_FILE_NAMES
+                .iter()
+                .map(|name| src_path.join(name))
+                .find(|src_path| src_path.exists());
+            let mut license = None;
+            if let Some(license_file) = license_file {
+                let license_file_content = fs::read_to_string(&license_file)
+                    .with_context(|| format!("failed to read {}", license_file.display()))?;
+                fs::write(dst_path.join("LICENSE"), &license_file_content).with_context(|| {
+                    format!("failed to wirte {}", dst_path.join("LICENSE").display())
+                })?;
+                license = LICENSE_SEARCH
+                    .iter()
+                    .find(|(needle, _)| license_file_content.contains(needle))
+                    .map(|(_, license)| (*license).to_owned());
+                if license.is_none() {
+                    eprintln!("failed to identify license in {}", license_file.display());
+                }
+            } else {
+                eprintln!("warning: {grammar_name} does not have a LICENSE file!");
+            }
+            if self.metadata {
+                let metadata_path = dst_path.join("metadata.json");
+                let rev =
+                    git_output(&["rev-parse", "HEAD"], &src_path, false).with_context(|| {
+                        format!("failed to obtain git revision at {}", src_path.display())
+                    })?;
+                let repo = git_output(&["remote", "get-url", "origin"], &src_path, false)
+                    .with_context(|| {
+                        format!("failed to obtain git remote at {}", src_path.display())
+                    })?;
+                let package_metadata: Option<PackageJson> =
+                    fs::read_to_string(src_path.join("package.json"))
+                        .ok()
+                        .and_then(|json| serde_json::from_str(&json).ok());
+                if let Some(package_metada) = package_metadata {
+                    match &license {
+                        Some(license) if license != &package_metada.license => eprintln!("warning: license in package identifier differs from detected license {license} != {}", &package_metada.license),
+                        _ => license = Some(package_metada.license),
+                    }
+                }
+
+                let old_metadata = Metadata::read(&metadata_path)
+                    .ok()
+                    .filter(|old_meta| old_meta.repo == repo && !old_meta.license.is_empty());
+
+                if let Some(old_metadata) = &old_metadata {
+                    match &license {
+                        Some(license) => {
+                            if license != &old_metadata.license {
+                                eprintln!(
+                                    "warning: license has changed {} => {license}",
+                                    old_metadata.license
+                                );
+                            }
+                        }
+                        None => {
+                            eprintln!(
+                                "warning: couldn't determine license for {grammar_name}, keeping {:?}",
+                                old_metadata.license
+                            );
+                            license = Some(old_metadata.license.clone())
+                        }
+                    }
+                }
+                if license.is_none() {
+                    eprintln!("warning: couldn't import determine license for {grammar_name}",);
+                }
+
+                let metadata = Metadata {
+                    repo,
+                    rev,
+                    license: license.unwrap_or_default(),
+                    new_prescedence: old_metadata
+                        .map_or(false, |old_metadata| old_metadata.new_prescedence),
+                };
+                metadata.write(&metadata_path).with_context(|| {
+                    format!(
+                        "failed to write metadata.json to {}",
+                        metadata_path.display()
+                    )
+                })?
+            }
+        }
+        Ok(())
+    }
+}
+
+#[derive(Deserialize)]
+struct PackageJson {
+    license: String,
+}
+
+fn git_output(args: &[&str], dir: &Path, verbose: bool) -> Result<String> {
+    let mut cmd = Command::new("git");
+    cmd.args(args).current_dir(dir);
+    if verbose {
+        println!("{}: git {}", dir.display(), args.join(" "))
+    }
+    let res = cmd.output().context("failed to invoke git")?;
+    if !res.status.success() {
+        let _ = io::stdout().write_all(&res.stdout);
+        let _ = io::stderr().write_all(&res.stderr);
+        bail!("git returned non-zero exit-code: {}", res.status);
+    }
+    String::from_utf8(res.stdout).context("git returned invalid utf8")
+}
--- a/cli/src/main.rs
+++ b/cli/src/main.rs
@@ -0,0 +1,45 @@
+use std::num::NonZeroUsize;
+use std::path::PathBuf;
+use std::process::exit;
+
+use anyhow::Context;
+
+mod flags;
+mod import;
+
+fn wrapped_main() -> anyhow::Result<()> {
+    let flags = flags::Skidder::from_env_or_exit();
+    match flags.subcommand {
+        flags::SkidderCmd::Import(import_cmd) => import_cmd.run(),
+        flags::SkidderCmd::Build(build_command) => {
+            let repo = build_command
+                .repo
+                .canonicalize()
+                .with_context(|| format!("failed to access {}", build_command.repo.display()))?;
+            let config = skidder::Config {
+                repos: vec![skidder::Repo::Local { path: repo }],
+                index: PathBuf::new(),
+                verbose: build_command.verbose,
+            };
+            if let Some(grammar) = build_command.grammar {
+                skidder::build_grammar(&config, &grammar, build_command.force)?;
+            } else {
+                skidder::build_all_grammars(
+                    &config,
+                    build_command.force,
+                    build_command.threads.and_then(NonZeroUsize::new),
+                )?;
+            }
+            Ok(())
+        }
+    }
+}
+
+pub fn main() {
+    if let Err(err) = wrapped_main() {
+        for error in err.chain() {
+            eprintln!("error: {error}")
+        }
+        exit(1)
+    }
+}
--- a/skidder/Cargo.toml
+++ b/skidder/Cargo.toml
@@ -12,6 +12,9 @@ rust-version = "1.76.0"
 [dependencies]
 anyhow = "1.0.86"
 cc = "1.1.7"
+indicatif = "0.17.8"
+serde = { version = "1.0.205", features = ["derive"] }
+serde_json = "1.0.122"
 sha1 = "0.10.6"
 tempfile = "3.10.1"

--- a/skidder/src/build.rs
+++ b/skidder/src/build.rs
@@ -23,7 +23,7 @@ fn is_fresh(grammar_dir: &Path, files: &[&str], force: bool) -> Result<(Checksum
            .with_context(|| format!("failed to read {}", path.display()))?;
        hasher.update(file);
        // paddding bytes
-        hasher.update(&[0, 0, 0, 0]);
+        hasher.update([0, 0, 0, 0]);
    }
    let checksum = hasher.finalize();
    if force {
@@ -32,7 +32,7 @@ fn is_fresh(grammar_dir: &Path, files: &[&str], force: bool) -> Result<(Checksum
    let Ok(prev_checksum) = fs::read(cookie) else {
        return Ok((checksum.into(), false));
    };
-    return Ok((checksum.into(), prev_checksum == checksum[..]));
+    Ok((checksum.into(), prev_checksum == checksum[..]))
 }

 const BUILD_TARGET: &str = env!("BUILD_TARGET");
@@ -102,7 +102,7 @@ pub fn build_grammar(grammar_name: &str, grammar_dir: &Path, force: bool) -> Res
    }
    ensure!(
        grammar_dir.join("parser.c").exists(),
-        "fialed to compile {grammar_name}: parser.c not found!"
+        "failed to compile {grammar_name}: parser.c not found!"
    );
    let build_dir = TempDir::new().context("fialed to create temporary build dierctory")?;
    let (mut cmd, output_path) = compiler_command(
@@ -124,6 +124,6 @@ pub fn build_grammar(grammar_name: &str, grammar_dir: &Path, force: bool) -> Res
        grammar_dir.join(grammar_name).with_extension(LIB_EXTENSION),
    )
    .context("failed to create library")?;
-    let _ = fs::write(grammar_dir.join(".BUILD_COOKIE"), &hash);
+    let _ = fs::write(grammar_dir.join(".BUILD_COOKIE"), hash);
    Ok(())
 }
--- a/skidder/src/lib.rs
+++ b/skidder/src/lib.rs
@@ -3,10 +3,13 @@ use std::num::NonZeroUsize;
 use std::path::{Path, PathBuf};
 use std::process::{Command, Stdio};
 use std::sync::atomic::{self, AtomicUsize};
-use std::sync::{mpsc, Mutex};
+use std::sync::Mutex;
+use std::time::Duration;
 use std::{fs, io, thread};

 use anyhow::{bail, ensure, Context, Result};
+use indicatif::{ProgressBar, ProgressStyle};
+use serde::{Deserialize, Serialize};

 #[cfg(not(windows))]
 const LIB_EXTENSION: &str = "so";
@@ -100,7 +103,15 @@ impl Repo {
    }

    pub fn has_grammar(&self, config: &Config, grammar: &str) -> bool {
-        self.dir(config).join(grammar).join("parser.c").exists()
+        self.dir(config)
+            .join(grammar)
+            .join("metadata.json")
+            .exists()
+    }
+
+    pub fn read_metadata(&self, config: &Config, grammar: &str) -> Result<Metadata> {
+        let path = self.dir(config).join(grammar).join("metadata.json");
+        Metadata::read(&path)
    }

    pub fn list_grammars(&self, config: &Config) -> Result<Vec<PathBuf>> {
@@ -112,7 +123,7 @@ impl Repo {
                    return Ok(None);
                }
                let path = dent.path();
-                if !path.join("parser.c").exists() {
+                if !path.join("metadata.json").exists() {
                    return Ok(None);
                }
                Ok(Some(dent.path()))
@@ -185,6 +196,12 @@ pub fn build_all_grammars(
    concurrency: Option<NonZeroUsize>,
 ) -> Result<usize> {
    let grammars = list_grammars(config)?;
+    let bar = ProgressBar::new(grammars.len() as u64).with_style(
+        ProgressStyle::with_template("{spinner} {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}")
+            .unwrap(),
+    );
+    bar.set_message("Compiling");
+    bar.enable_steady_tick(Duration::from_millis(100));
    let i = AtomicUsize::new(0);
    let concurrency = concurrency
        .or_else(|| thread::available_parallelism().ok())
@@ -198,15 +215,47 @@ pub fn build_all_grammars(
                };
                let name = grammar.file_name().unwrap().to_str().unwrap();
                if let Err(err) = build::build_grammar(name, grammar, force_rebuild) {
-                    eprintln!("{err}");
+                    for err in err.chain() {
+                        bar.println(format!("error: {err}"))
+                    }
                    failed.lock().unwrap().push(name.to_owned())
                }
+                bar.inc(1);
            });
        }
    });
    let failed = failed.into_inner().unwrap();
-    if failed.is_empty() {
+    if !failed.is_empty() {
        bail!("failed to build grammars {failed:?}")
    }
    Ok(grammars.len())
 }
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case", deny_unknown_fields)]
+pub struct Metadata {
+    /// The git remote of the query upstreama
+    pub repo: String,
+    /// The git remote of the query
+    pub rev: String,
+    /// The SPDX license identifier
+    #[serde(default)]
+    pub license: String,
+    /// Wether to use the new query precedence
+    /// where later matches take priority.
+    #[serde(default)]
+    pub new_prescedence: bool,
+}
+
+impl Metadata {
+    pub fn read(path: &Path) -> Result<Metadata> {
+        let json = fs::read_to_string(path)
+            .with_context(|| format!("couldn't read {}", path.display()))?;
+        serde_json::from_str(&json)
+            .with_context(|| format!("invalid metadata.json file at {}", path.display()))
+    }
+    pub fn write(&self, path: &Path) -> Result<()> {
+        let json = serde_json::to_string_pretty(&self).unwrap();
+        fs::write(path, json).with_context(|| format!("failed to write {}", path.display()))
+    }
+}