From 4fb76b7e0c5132b53f5270cabc66e59113de541f Mon Sep 17 00:00:00 2001 From: Josef Zoller Date: Thu, 20 Jun 2024 04:37:16 +0200 Subject: [PATCH] [WIP] Implement AST dump as JSON --- Cargo.lock | 85 ++++++++++++++++++++++++++++++ core/Cargo.toml | 2 + core/src/dump.rs | 53 ++++++++++++++----- driver/Cargo.toml | 1 + driver/src/cli.rs | 65 +++++++++++++++++++---- driver/src/driver/main/mod.rs | 10 ++-- driver/src/driver/main/task/mod.rs | 11 ++-- frontend/Cargo.toml | 2 + frontend/src/diag/diagnostic.rs | 2 + frontend/src/error.rs | 3 ++ frontend/src/runner.rs | 20 +++++-- 11 files changed, 217 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f4f05fd..8e9b317 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -329,12 +329,44 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" +[[package]] +name = "color-print" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ee543c60ff3888934877a5671f45494dd27ed4ba25c6670b9a7576b7ed7a8c0" +dependencies = [ + "color-print-proc-macro", +] + +[[package]] +name = "color-print-proc-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ff1a80c5f3cb1ca7c06ffdd71b6a6dd6d8f896c42141fbd43f50ed28dcdb93" +dependencies = [ + "nom", + "proc-macro2", + "quote", + "syn 2.0.52", +] + [[package]] name = "colorchoice" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "colored_json" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e35980a1b846f8e3e359fd18099172a0857140ba9230affc4f71348081e039b6" +dependencies = [ + "serde", + "serde_json", + "yansi", +] + [[package]] name = "concurrent-queue" version = "2.4.0" @@ -545,6 +577,12 @@ dependencies = [ "either", ] +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + [[package]] name = "juice-core" version = "0.1.0" @@ -554,6 +592,8 @@ dependencies = [ "derive_more", "in_definite", "itertools", + "serde", + "serde-tuple-vec-map", "thousands", ] @@ -565,6 +605,7 @@ dependencies = [ "async-process", "cfg-if", "clap", + "color-print", "derive_more", "itertools", "juice-core", @@ -580,6 +621,7 @@ version = "0.1.0" dependencies = [ "ariadne", "chumsky", + "colored_json", "derive-where", "derive_more", "itertools", @@ -587,6 +629,7 @@ dependencies = [ "juice-macros", "lasso", "num-bigint", + "serde_json", ] [[package]] @@ -640,6 +683,12 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.7.2" @@ -660,6 +709,16 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "num-bigint" version = "0.4.4" @@ -855,6 +914,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + [[package]] name = "scopeguard" version = "1.2.0" @@ -876,6 +941,15 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-tuple-vec-map" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a04d0ebe0de77d7d445bb729a895dcb0a288854b267ca85f030ce51cdc578c82" +dependencies = [ + "serde", +] + [[package]] name = "serde_derive" version = "1.0.203" @@ -887,6 +961,17 @@ dependencies = [ "syn 2.0.52", ] +[[package]] +name = "serde_json" +version = "1.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +dependencies = [ + "itoa", + "ryu", + "serde", +] + [[package]] name = "signal-hook-registry" version = "1.4.1" diff --git a/core/Cargo.toml b/core/Cargo.toml index 1ec65d9..f177663 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -12,4 +12,6 @@ chumsky = { workspace = true } derive_more = { workspace = true } in_definite = "1.0.0" itertools = { workspace = true } +serde = { version = "1.0.203", features = ["derive", "rc"] } +serde-tuple-vec-map = "1.0.1" thousands = "0.2.0" diff --git a/core/src/dump.rs b/core/src/dump.rs index 5f754e8..57f0bf2 100644 --- a/core/src/dump.rs +++ b/core/src/dump.rs @@ -3,6 +3,7 @@ use std::{io, sync::Arc}; use ariadne::{Color, Fmt as _}; use derive_more::From; use itertools::Itertools as _; +use serde::Serialize; use thousands::{digits::ASCII_HEXADECIMAL, Separable as _, SeparatorPolicy}; use crate::{ @@ -16,16 +17,45 @@ const UNDERSCORE_HEX_SEPARATOR: SeparatorPolicy = SeparatorPolicy { digits: ASCII_HEXADECIMAL, }; -#[derive(Debug, Clone, From)] +fn format_bigint(v: &[u64]) -> String { + let (first, rest) = v.split_last().unwrap(); + + let first = format!("{first:x}").separate_by_policy(UNDERSCORE_HEX_SEPARATOR); + + let rest = rest.iter().rev().format_with("_", |part, f| { + f(&format!("{part:016x}").separate_by_policy(UNDERSCORE_HEX_SEPARATOR)) + }); + + format!("0x{first}_{rest}") +} + +fn serialize_bigint(v: &[u64], serializer: S) -> Result +where + S: serde::Serializer, +{ + serializer.serialize_str(&format_bigint(v)) +} + +#[derive(Debug, Clone, From, Serialize)] +#[serde(tag = "type", content = "value")] pub enum DumpField<'src> { + #[serde(rename = "bigint", serialize_with = "serialize_bigint")] + BigInt(Arc<[u64]>), + #[serde(untagged)] Bool(bool), + #[serde(untagged)] Int(u64), - BigInt(Arc<[u64]>), + #[serde(untagged)] Float(f64), + #[serde(untagged)] Char(char), + #[serde(untagged)] String(&'src str), + #[serde(untagged)] ArcStr(Arc), + #[serde(untagged)] List(Vec>), + #[serde(untagged)] Dump(Dump<'src>), } @@ -38,21 +68,13 @@ impl DumpField<'_> { let indent_str = " ".repeat(indentation); match self { - Self::Bool(v) => write!(stream, "{v}"), - Self::Int(v) => write!(stream, "{v}"), Self::BigInt(v) => { - let (first, rest) = v.split_last().unwrap(); - - let first = format!("{first:x}").separate_by_policy(UNDERSCORE_HEX_SEPARATOR); - - let rest = rest.iter().rev().format_with("_", |part, f| { - f(&format!("{part:016x}").separate_by_policy(UNDERSCORE_HEX_SEPARATOR)) - }); - - write!(stream, "0x{first}_{rest}")?; + write!(stream, "{}", format_bigint(v))?; Ok(()) } + Self::Bool(v) => write!(stream, "{v}"), + Self::Int(v) => write!(stream, "{v}"), Self::Float(v) => write!(stream, "{v}"), Self::Char(c) => write!(stream, "{c:?}"), Self::String(s) => write!(stream, "{s:?}"), @@ -75,10 +97,13 @@ impl DumpField<'_> { } } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize)] pub struct Dump<'src> { + #[serde(rename = "node")] pub name: &'static str, + #[serde(rename = "error", skip_serializing_if = "std::ops::Not::not")] pub is_error: bool, + #[serde(flatten, with = "tuple_vec_map")] pub fields: Vec<(&'static str, DumpField<'src>)>, } diff --git a/driver/Cargo.toml b/driver/Cargo.toml index bb3d1b0..82993a9 100644 --- a/driver/Cargo.toml +++ b/driver/Cargo.toml @@ -11,6 +11,7 @@ async-once-cell = "0.5.3" async-process = "2.1.0" cfg-if = "1.0.0" clap = { version = "4.5.1", features = ["derive"] } +color-print = "0.3.6" derive_more = { workspace = true } juice-core = { workspace = true } juice-frontend = { workspace = true } diff --git a/driver/src/cli.rs b/driver/src/cli.rs index c0e4f8d..9ab8444 100644 --- a/driver/src/cli.rs +++ b/driver/src/cli.rs @@ -57,7 +57,7 @@ mod private { use super::*; #[derive(Debug, Args)] - #[clap(group = clap::ArgGroup::new("frontend-action").multiple(false))] + #[clap(group = clap::ArgGroup::new("frontend-action").multiple(false).required(true))] pub struct FrontendArgs { /// The input file to compile #[arg(long = "input-file", value_parser = absolute_path_parser, value_name = "FILE")] @@ -77,6 +77,21 @@ mod private { /// Emit an object file #[arg(long, group = "frontend-action", help_heading = "Actions")] pub emit_object: bool, + /// Dump the ast as JSON (can only be used together with the `dump-parse` and `dump-ast` actions) + #[arg(long, help_heading = "Actions")] + pub json: bool, + } + + impl FrontendArgs { + pub fn validate(&self, mut cmd: clap::Command) { + if self.json && !(self.dump_parse || self.dump_ast) { + const MESSAGE: &str = color_print::cstr!( + "--json can only be used together with \ + --dump-parse or --dump-ast" + ); + cmd.error(ErrorKind::ArgumentConflict, MESSAGE).exit(); + } + } } fn absolute_path_parser(s: &str) -> Result { @@ -106,6 +121,15 @@ mod private { Repl, } + impl Command { + pub fn validate(&self, cmd: clap::Command) { + match self { + Self::Frontend(args) => args.validate(cmd), + Self::Repl => {} + } + } + } + #[derive(Debug, Args)] #[clap(group = clap::ArgGroup::new("main-action").multiple(false))] pub struct MainArgs { @@ -136,6 +160,21 @@ mod private { /// Emit an executable (default) #[arg(long, group = "main-action", help_heading = "Actions")] pub emit_executable: bool, + /// Dump the ast as JSON (can only be used together with the `dump-parse` and `dump-ast` actions) + #[arg(long, help_heading = "Actions")] + pub json: bool, + } + + impl MainArgs { + pub fn validate(&self, mut cmd: clap::Command) { + if self.json && !(self.dump_parse || self.dump_ast) { + const MESSAGE: &str = color_print::cstr!( + "--json can only be used together with \ + --dump-parse or --dump-ast" + ); + cmd.error(ErrorKind::ArgumentConflict, MESSAGE).exit(); + } + } } fn output_file_path_parser(s: &str) -> Result { @@ -155,9 +194,9 @@ impl TryFrom for FrontendArgs { let output_stream = args.output_filepath.try_into_stream()?; let action = if args.dump_parse { - FrontendAction::DumpParse + FrontendAction::DumpParse { json: args.json } } else if args.dump_ast { - FrontendAction::DumpAst + FrontendAction::DumpAst { json: args.json } } else if args.emit_ir { FrontendAction::EmitIr } else { @@ -179,9 +218,9 @@ impl TryFrom for MainArgs { let input_filepath = path::absolute(&args.input_filename)?; let action = if args.dump_parse { - MainAction::DumpParse + MainAction::DumpParse { json: args.json } } else if args.dump_ast { - MainAction::DumpAst + MainAction::DumpAst { json: args.json } } else if args.dump_ir { MainAction::DumpIr } else if args.emit_ir { @@ -223,18 +262,22 @@ impl TryFrom for Command { impl Cli { pub fn command(self) -> DriverResult { + let mut cmd = ::command(); if let Some(command) = self.command { + command.validate(cmd); + command.try_into() } else { let Some(main_args) = self.main_args else { - ::command() - .error( - ErrorKind::MissingRequiredArgument, - "either a subcommand or main arguments are required", - ) - .exit(); + cmd.error( + ErrorKind::MissingRequiredArgument, + "either a subcommand or main arguments are required", + ) + .exit(); }; + main_args.validate(cmd); + main_args.try_into().map(Command::Main) } } diff --git a/driver/src/driver/main/mod.rs b/driver/src/driver/main/mod.rs index 128cef3..ea887f4 100644 --- a/driver/src/driver/main/mod.rs +++ b/driver/src/driver/main/mod.rs @@ -11,8 +11,12 @@ use crate::cli::OutputFilePath; #[derive(Debug, Default, PartialEq, Eq, Clone, Copy)] pub enum Action { - DumpParse, - DumpAst, + DumpParse { + json: bool, + }, + DumpAst { + json: bool, + }, DumpIr, EmitIr, EmitObject, @@ -31,7 +35,7 @@ impl Action { Some(OutputFilePath::File(filename)) => path::absolute(filename).map(OutputFilePath::File), None => { let extension = match self { - Self::DumpParse | Self::DumpAst | Self::DumpIr => return Ok(OutputFilePath::Stdout), + Self::DumpParse { .. } | Self::DumpAst { .. } | Self::DumpIr => return Ok(OutputFilePath::Stdout), Self::EmitIr => "ll", Self::EmitObject => "o", Self::EmitExecutable => "", diff --git a/driver/src/driver/main/task/mod.rs b/driver/src/driver/main/task/mod.rs index 3ef0320..aac277a 100644 --- a/driver/src/driver/main/task/mod.rs +++ b/driver/src/driver/main/task/mod.rs @@ -226,11 +226,11 @@ impl CompilationTask { ) -> DriverResult { let executable_path = env::current_exe()?; - let action_string = match action { - Action::DumpParse => "--dump-parse", - Action::DumpAst => "--dump-ast", - Action::DumpIr | Action::EmitIr => "--emit-ir", - Action::EmitObject | Action::EmitExecutable => "--emit-object", + let (action_string, json) = match action { + Action::DumpParse { json } => ("--dump-parse", json.then_some("--json")), + Action::DumpAst { json } => ("--dump-ast", json.then_some("--json")), + Action::DumpIr | Action::EmitIr => ("--emit-ir", None), + Action::EmitObject | Action::EmitExecutable => ("--emit-object", None), }; let arguments = vec![ @@ -242,6 +242,7 @@ impl CompilationTask { output_path.as_os_str(), ] .into_iter() + .chain(json.map(OsStr::new)) .map(OsString::from) .collect(); diff --git a/frontend/Cargo.toml b/frontend/Cargo.toml index 0cd23eb..de49356 100644 --- a/frontend/Cargo.toml +++ b/frontend/Cargo.toml @@ -8,6 +8,7 @@ edition = "2021" [dependencies] ariadne = { workspace = true } chumsky = { workspace = true } +colored_json = "5.0.0" derive-where = "1.2.7" derive_more = { workspace = true } itertools = { workspace = true } @@ -15,3 +16,4 @@ juice-core = { workspace = true } juice-macros = { workspace = true } lasso = "0.7.2" num-bigint = "0.4.4" +serde_json = "1.0.117" diff --git a/frontend/src/diag/diagnostic.rs b/frontend/src/diag/diagnostic.rs index 28ee1e3..8234fce 100644 --- a/frontend/src/diag/diagnostic.rs +++ b/frontend/src/diag/diagnostic.rs @@ -8,6 +8,8 @@ use juice_macros::Diagnostic; pub enum StaticDiagnostic { #[diag(error = "Error while doing IO: {}")] IoError(#[diag(into)] Colored), + #[diag(error = "Error while serializing JSON: {}")] + JsonError(#[diag(into)] Colored), } #[derive(Debug, Clone, Diagnostic)] diff --git a/frontend/src/error.rs b/frontend/src/error.rs index f7d06af..63c073d 100644 --- a/frontend/src/error.rs +++ b/frontend/src/error.rs @@ -8,12 +8,15 @@ use crate::diag::{StaticDiagnostic, StaticDiagnosticReport}; pub enum Error { #[from] Io(io::Error), + #[from] + Json(serde_json::Error), } impl Error { pub fn diagnose(self) { let diagnostic = match self { Self::Io(err) => StaticDiagnostic::io_error(err.to_string()), + Self::Json(err) => StaticDiagnostic::json_error(err.to_string()), }; StaticDiagnosticReport::new(diagnostic).diagnose(); diff --git a/frontend/src/runner.rs b/frontend/src/runner.rs index 24a547d..2f0241c 100644 --- a/frontend/src/runner.rs +++ b/frontend/src/runner.rs @@ -3,6 +3,7 @@ use std::{ path::PathBuf, }; +use colored_json::write_colored_json; use juice_core::{dump::ToDump, OutputStream}; use crate::{ @@ -22,8 +23,8 @@ macro_rules! check_error { #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum Action { - DumpParse, - DumpAst, + DumpParse { json: bool }, + DumpAst { json: bool }, EmitIr, EmitObject, } @@ -78,8 +79,19 @@ impl Runner { let stmts = parser.parse_stmt_list(&diagnostics)?; if let Some(stmts) = stmts { - if self.args.action == Action::DumpParse { - stmts.to_dump().write(self.args.output_stream.as_mut())?; + if let Action::DumpParse { json } = self.args.action { + if json { + let dump = stmts.to_dump(); + + if self.args.output_stream.is_terminal() { + write_colored_json(&dump, &mut self.args.output_stream)?; + writeln!(self.args.output_stream)?; + } else { + serde_json::to_writer(self.args.output_stream.as_mut(), &dump)?; + } + } else { + stmts.to_dump().write(self.args.output_stream.as_mut())?; + } check_error!(diagnostics);