feat: Highlight in prqlc

prqlc/prqlc-macros/src/lib.rs

@@ -1,7 +1,7 @@
 //! Macros for PRQL compilation at build time.
 //!
 //! ```
-//! use prql_compiler_macros::prql_to_sql;
+//! use prqlc_macros::prql_to_sql;
 //!
 //! let sql: &str = prql_to_sql!("from albums | select {title, artist_id}");
 //! assert_eq!(sql, "SELECT title, artist_id FROM albums");

prqlc/prqlc-parser/src/parser/pr/stmt.rs

@@ -25,8 +25,6 @@ pub enum VarDefKind {
     Main,
 }
 
-// The following code is tested by the tests_misc crate to match stmt.rs in prql_compiler.
-
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
 pub struct Stmt {
     #[serde(flatten)]

prqlc/prqlc/src/cli/highlight.rs

@@ -0,0 +1,163 @@
+use color_eyre::owo_colors::OwoColorize;
+use prqlc::{
+    lr::{TokenKind, Tokens},
+    pr::Literal,
+};
+
+/// Highlight PRQL code printed to the terminal.
+pub(crate) fn highlight(tokens: &Tokens) -> String {
+    let mut output = String::new();
+    let mut last = 0;
+
+    for token in &tokens.0 {
+        let diff = token.span.start - last;
+        last = token.span.end;
+        output.push_str(&" ".repeat(diff));
+        output.push_str(&highlight_token_kind(&token.kind));
+    }
+
+    output
+}
+
+fn highlight_token_kind(token: &TokenKind) -> String {
+    // LineWrap is recursive with TokenKind, so we needed to split this function
+    // out from the one above (otherwise would have it as a single func)
+    let mut output = String::new();
+    match &token {
+        TokenKind::NewLine => output.push('\n'),
+        TokenKind::Ident(ident) => {
+            if is_transform(ident) {
+                output.push_str(&ident.green().to_string())
+            } else {
+                output.push_str(&ident.to_string())
+            }
+        }
+        TokenKind::Keyword(keyword) => output.push_str(&keyword.blue().to_string()),
+        TokenKind::Literal(literal) => output.push_str(&match literal {
+            Literal::Null => literal.green().bold().to_string(),
+            Literal::Integer(_) => literal.green().to_string(),
+            Literal::Float(_) => literal.green().to_string(),
+            Literal::Boolean(_) => literal.green().bold().to_string(),
+            Literal::String(_) => literal.yellow().to_string(),
+            _ => literal.to_string(),
+        }),
+        TokenKind::Param(param) => output.push_str(&param.purple().to_string()),
+        TokenKind::Range {
+            bind_left: _,
+            bind_right: _,
+        } => output.push_str(".."),
+        TokenKind::Interpolation(_, _) => output.push_str(&format!("{}", token.yellow())),
+        TokenKind::Control(char) => output.push(*char),
+        TokenKind::ArrowThin
+        | TokenKind::ArrowFat
+        | TokenKind::Eq
+        | TokenKind::Ne
+        | TokenKind::Gte
+        | TokenKind::Lte
+        | TokenKind::RegexSearch => output.push_str(&format!("{}", token)),
+        TokenKind::And | TokenKind::Or => {
+            output.push_str(&format!("{}", token).purple().to_string())
+        }
+        TokenKind::Coalesce | TokenKind::DivInt | TokenKind::Pow | TokenKind::Annotate => {
+            output.push_str(&format!("{}", token))
+        }
+        TokenKind::Comment(comment) => output.push_str(
+            &format!("#{comment}")
+                .truecolor(95, 135, 135)
+                .italic()
+                .to_string(),
+        ),
+        TokenKind::DocComment(comment) => output.push_str(
+            &format!("#!{comment}")
+                .truecolor(95, 135, 135)
+                .italic()
+                .to_string(),
+        ),
+        TokenKind::LineWrap(inner_tokens) => {
+            output.push_str("\n\\");
+            for t in inner_tokens {
+                output.push_str(&highlight_token_kind(t));
+            }
+        }
+        TokenKind::Start => {}
+    }
+    output
+}
+
+fn is_transform(ident: &str) -> bool {
+    // TODO: Could we instead source these from the standard library?
+    // We could also use the semantic understanding from later compiler stages?
+    matches!(
+        ident,
+        "from"
+            | "derive"
+            | "select"
+            | "filter"
+            | "sort"
+            | "join"
+            | "take"
+            | "group"
+            | "aggregate"
+            | "window"
+            | "loop"
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use std::process::Command;
+
+    use insta_cmd::assert_cmd_snapshot;
+    use insta_cmd::get_cargo_bin;
+
+    #[test]
+    fn highlight() {
+        // (Colors don't show because they're disabled; we could have a test
+        // that forces them to show?)
+        assert_cmd_snapshot!(prqlc_command().args(["experimental", "highlight"]).pass_stdin(r#"
+        from tracks
+        filter artist == "Bob Marley"                 # Each line transforms the previous result
+        aggregate {                                   # `aggregate` reduces each column to a value
+          plays    = sum plays,
+          longest  = max length,
+          shortest = min length,                      # Trailing commas are allowed
+        }
+
+        "#), @r###"
+        success: true
+        exit_code: 0
+        ----- stdout -----
+
+                from tracks
+                filter artist == "Bob Marley"                 # Each line transforms the previous result
+                aggregate {                                   # `aggregate` reduces each column to a value
+                  plays    = sum plays,
+                  longest  = max length,
+                  shortest = min length,                      # Trailing commas are allowed
+                }
+
+
+        ----- stderr -----
+        "###);
+    }
+
+    // TODO: import from existing location, need to adjust visibility
+    fn prqlc_command() -> Command {
+        let mut cmd = Command::new(get_cargo_bin("prqlc"));
+        normalize_prqlc(&mut cmd);
+        cmd
+    }
+
+    fn normalize_prqlc(cmd: &mut Command) -> &mut Command {
+        cmd
+            // We set `CLICOLOR_FORCE` in CI to force color output, but we don't want `prqlc` to
+            // output color for our snapshot tests. And it seems to override the
+            // `--color=never` flag.
+            .env_remove("CLICOLOR_FORCE")
+            .env("NO_COLOR", "1")
+            .args(["--color=never"])
+            // We don't want the tests to be affected by the user's `RUST_BACKTRACE` setting.
+            .env_remove("RUST_BACKTRACE")
+            .env_remove("RUST_LOG")
+    }
+}

prqlc/prqlc/src/cli/mod.rs

@@ -1,9 +1,5 @@
 #![cfg(not(target_family = "wasm"))]
 
-mod docs_generator;
-mod jinja;
-mod watch;
-
 use std::collections::HashMap;
 use std::env;
 use std::fs::File;
@@ -32,9 +28,17 @@
 use prqlc::pr;
 use prqlc::semantic;
 use prqlc::semantic::reporting::FrameCollector;
+use prqlc::utils::maybe_strip_colors;
 use prqlc::{pl_to_prql, pl_to_rq_tree, prql_to_pl, prql_to_pl_tree, prql_to_tokens, rq_to_sql};
 use prqlc::{Options, SourceTree, Target};
 
+mod docs_generator;
+mod highlight;
+mod jinja;
+#[cfg(test)]
+mod test;
+mod watch;
+
 /// Entrypoint called by [`crate::main`]
 pub fn main() -> color_eyre::eyre::Result<()> {
     let mut cli = Cli::parse();
@@ -222,6 +226,10 @@
     /// Generate Markdown documentation
     #[command(name = "doc")]
     GenerateDocs(IoArgs),
+
+    /// Syntax highlight
+    #[command(name = "highlight")]
+    Highlight(IoArgs),
 }
 
 #[derive(clap::Args, Default, Debug, Clone)]
@@ -435,6 +443,15 @@
 
                 docs_generator::generate_markdown_docs(module_ref.stmts).into_bytes()
             }
+            Command::Experimental(ExperimentalCommand::Highlight(_)) => {
+                let s = sources.sources.values().exactly_one().or_else(|_| {
+                    // TODO: allow multiple sources
+                    bail!("Currently `highlight` only works with a single source, but found multiple sources")
+                })?;
+                let tokens = prql_to_tokens(s)?;
+
+                maybe_strip_colors(&highlight::highlight(&tokens)).into_bytes()
+            }
             Command::Compile {
                 signature_comment,
                 format,
@@ -483,6 +500,7 @@
                 io_args
             }
             Experimental(ExperimentalCommand::GenerateDocs(io_args)) => io_args,
+            Experimental(ExperimentalCommand::Highlight(io_args)) => io_args,
             _ => unreachable!(),
         };
         let input = &mut io_args.input;
@@ -518,6 +536,7 @@
                 io_args.output.clone()
             }
             Experimental(ExperimentalCommand::GenerateDocs(io_args)) => io_args.output.clone(),
+            Experimental(ExperimentalCommand::Highlight(io_args)) => io_args.output.clone(),
             _ => unreachable!(),
         };
         output.write_all(data)

prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion-2.snap → prqlc/prqlc/src/cli/snapshots/prqlc__cli__test__shell_completion-2.snap

@@ -1,5 +1,4 @@
 ---
-source: prqlc/prqlc/tests/integration/cli.rs
 info:
   program: prqlc
   args:
@@ -81,18 +80,24 @@ complete -c prqlc -n "__fish_seen_subcommand_from debug; and __fish_seen_subcomm
 complete -c prqlc -n "__fish_seen_subcommand_from debug; and __fish_seen_subcommand_from help; and not __fish_seen_subcommand_from annotate; and not __fish_seen_subcommand_from lineage; and not __fish_seen_subcommand_from ast; and not __fish_seen_subcommand_from json-schema; and not __fish_seen_subcommand_from help" -f -a "ast" -d 'Print info about the AST data structure'
 complete -c prqlc -n "__fish_seen_subcommand_from debug; and __fish_seen_subcommand_from help; and not __fish_seen_subcommand_from annotate; and not __fish_seen_subcommand_from lineage; and not __fish_seen_subcommand_from ast; and not __fish_seen_subcommand_from json-schema; and not __fish_seen_subcommand_from help" -f -a "json-schema" -d 'Print JSON Schema'
 complete -c prqlc -n "__fish_seen_subcommand_from debug; and __fish_seen_subcommand_from help; and not __fish_seen_subcommand_from annotate; and not __fish_seen_subcommand_from lineage; and not __fish_seen_subcommand_from ast; and not __fish_seen_subcommand_from json-schema; and not __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
-complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from help" -l color -d 'Controls when to use color' -r -f -a "{auto	'',always	'',never	''}"
-complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from help" -s v -l verbose -d 'Increase logging verbosity'
-complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from help" -s q -l quiet -d 'Silences logging output'
-complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from help" -s h -l help -d 'Print help (see more with \'--help\')'
-complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from help" -f -a "doc" -d 'Generate Markdown documentation'
-complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from highlight; and not __fish_seen_subcommand_from help" -l color -d 'Controls when to use color' -r -f -a "{auto	'',always	'',never	''}"
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from highlight; and not __fish_seen_subcommand_from help" -s v -l verbose -d 'Increase logging verbosity'
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from highlight; and not __fish_seen_subcommand_from help" -s q -l quiet -d 'Silences logging output'
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from highlight; and not __fish_seen_subcommand_from help" -s h -l help -d 'Print help (see more with \'--help\')'
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from highlight; and not __fish_seen_subcommand_from help" -f -a "doc" -d 'Generate Markdown documentation'
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from highlight; and not __fish_seen_subcommand_from help" -f -a "highlight" -d 'Syntax highlight'
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from highlight; and not __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
 complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from doc" -l color -d 'Controls when to use color' -r -f -a "{auto	'',always	'',never	''}"
 complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from doc" -s v -l verbose -d 'Increase logging verbosity'
 complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from doc" -s q -l quiet -d 'Silences logging output'
 complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from doc" -s h -l help -d 'Print help (see more with \'--help\')'
-complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from help; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from help" -f -a "doc" -d 'Generate Markdown documentation'
-complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from help; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from highlight" -l color -d 'Controls when to use color' -r -f -a "{auto	'',always	'',never	''}"
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from highlight" -s v -l verbose -d 'Increase logging verbosity'
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from highlight" -s q -l quiet -d 'Silences logging output'
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from highlight" -s h -l help -d 'Print help (see more with \'--help\')'
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from help; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from highlight; and not __fish_seen_subcommand_from help" -f -a "doc" -d 'Generate Markdown documentation'
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from help; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from highlight; and not __fish_seen_subcommand_from help" -f -a "highlight" -d 'Syntax highlight'
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from help; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from highlight; and not __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
 complete -c prqlc -n "__fish_seen_subcommand_from compile" -s t -l target -d 'Target to compile to' -r
 complete -c prqlc -n "__fish_seen_subcommand_from compile" -l debug-log -d 'File path into which to write the debug log to' -r -F
 complete -c prqlc -n "__fish_seen_subcommand_from compile" -l color -d 'Controls when to use color' -r -f -a "{auto	'',always	'',never	''}"
@@ -130,6 +135,7 @@ complete -c prqlc -n "__fish_seen_subcommand_from help; and __fish_seen_subcomma
 complete -c prqlc -n "__fish_seen_subcommand_from help; and __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from annotate; and not __fish_seen_subcommand_from lineage; and not __fish_seen_subcommand_from ast; and not __fish_seen_subcommand_from json-schema" -f -a "lineage" -d 'Output column-level lineage graph'
 complete -c prqlc -n "__fish_seen_subcommand_from help; and __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from annotate; and not __fish_seen_subcommand_from lineage; and not __fish_seen_subcommand_from ast; and not __fish_seen_subcommand_from json-schema" -f -a "ast" -d 'Print info about the AST data structure'
 complete -c prqlc -n "__fish_seen_subcommand_from help; and __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from annotate; and not __fish_seen_subcommand_from lineage; and not __fish_seen_subcommand_from ast; and not __fish_seen_subcommand_from json-schema" -f -a "json-schema" -d 'Print JSON Schema'
-complete -c prqlc -n "__fish_seen_subcommand_from help; and __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc" -f -a "doc" -d 'Generate Markdown documentation'
+complete -c prqlc -n "__fish_seen_subcommand_from help; and __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from highlight" -f -a "doc" -d 'Generate Markdown documentation'
+complete -c prqlc -n "__fish_seen_subcommand_from help; and __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from highlight" -f -a "highlight" -d 'Syntax highlight'
 
 ----- stderr -----