From 7d3da09b68f5ef1fb5e192597664de2bfc2ca2c6 Mon Sep 17 00:00:00 2001
From: Jonathan <vanillajonathan@users.noreply.github.com>
Date: Fri, 19 Jul 2024 23:27:02 +0200
Subject: [PATCH 01/10] feat: Highlight in prqlc

---
 prqlc/prqlc/src/cli/highlight.rs | 128 +++++++++++++++++++++++++++++++
 prqlc/prqlc/src/cli/mod.rs       |  16 ++++
 2 files changed, 144 insertions(+)
 create mode 100644 prqlc/prqlc/src/cli/highlight.rs

diff --git a/prqlc/prqlc/src/cli/highlight.rs b/prqlc/prqlc/src/cli/highlight.rs
new file mode 100644
index 000000000000..1bc835b03552
--- /dev/null
+++ b/prqlc/prqlc/src/cli/highlight.rs
@@ -0,0 +1,128 @@
+use color_eyre::owo_colors::OwoColorize;
+use prqlc::{
+    lr::{TokenKind, Tokens},
+    pr::Literal,
+};
+
+/// Highlight PRQL code printed to the terminal.
+pub fn highlight(tokens: &Tokens) -> String {
+    let mut output = String::new();
+    let mut last = 0;
+
+    for token in &tokens.0 {
+        let diff = token.span.start - last;
+        last = token.span.end;
+        output.push_str(&" ".repeat(diff));
+
+        match &token.kind {
+            TokenKind::NewLine => output.push('\n'),
+            TokenKind::Ident(ident) => {
+                if is_transform(ident) {
+                    output.push_str(&ident.green().to_string())
+                } else {
+                    output.push_str(&ident.to_string())
+                }
+            }
+            TokenKind::Keyword(keyword) => output.push_str(&keyword.blue().to_string()),
+            TokenKind::Literal(literal) => output.push_str(&match literal {
+                Literal::Null => literal.green().bold().to_string(),
+                Literal::Integer(_) => literal.green().to_string(),
+                Literal::Float(_) => literal.green().to_string(),
+                Literal::Boolean(_) => literal.green().bold().to_string(),
+                Literal::String(_) => literal.yellow().to_string(),
+                _ => literal.to_string(),
+            }),
+            TokenKind::Param(param) => output.push_str(&param.purple().to_string()),
+            TokenKind::Range {
+                bind_left: _,
+                bind_right: _,
+            } => output.push_str(".."),
+            TokenKind::Interpolation(_, _) => output.push_str(&format!("{}", token.kind.yellow())),
+            TokenKind::Control(char) => output.push(*char),
+            TokenKind::ArrowThin
+            | TokenKind::ArrowFat
+            | TokenKind::Eq
+            | TokenKind::Ne
+            | TokenKind::Gte
+            | TokenKind::Lte
+            | TokenKind::RegexSearch => output.push_str(&format!("{}", token.kind)),
+            TokenKind::And | TokenKind::Or => {
+                output.push_str(&format!("{}", token.kind).purple().to_string())
+            }
+            TokenKind::Coalesce | TokenKind::DivInt | TokenKind::Pow | TokenKind::Annotate => {
+                output.push_str(&format!("{}", token.kind))
+            }
+            TokenKind::Comment(comment) => output.push_str(
+                &format!("#{comment}")
+                    .truecolor(95, 135, 135)
+                    .italic()
+                    .to_string(),
+            ),
+            TokenKind::DocComment(comment) => output.push_str(
+                &format!("#!{comment}")
+                    .truecolor(95, 135, 135)
+                    .italic()
+                    .to_string(),
+            ),
+            TokenKind::LineWrap(_) => todo!(),
+            TokenKind::Start => {}
+        }
+    }
+
+    output
+}
+
+fn is_transform(ident: &str) -> bool {
+    match ident {
+        "from" => true,
+        "derive" | "select" | "filter" | "sort" | "join" | "take" | "group" | "aggregate"
+        | "window" | "lopo" => true,
+        _ => false,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::process::Command;
+
+    use insta_cmd::assert_cmd_snapshot;
+    use insta_cmd::get_cargo_bin;
+
+    #[test]
+    fn highlight() {
+        let input = r"
+foo
+
+        ";
+
+        assert_cmd_snapshot!(prqlc_command().args(["experimental", "highlight"]).pass_stdin(input), @r###"
+        success: true
+        exit_code: 0
+        ----- stdout -----
+
+        foo
+
+
+        ----- stderr -----
+        "###);
+    }
+
+    fn prqlc_command() -> Command {
+        let mut cmd = Command::new(get_cargo_bin("prqlc"));
+        normalize_prqlc(&mut cmd);
+        cmd
+    }
+    
+    fn normalize_prqlc(cmd: &mut Command) -> &mut Command {
+        cmd
+            // We set `CLICOLOR_FORCE` in CI to force color output, but we don't want `prqlc` to
+            // output color for our snapshot tests. And it seems to override the
+            // `--color=never` flag.
+            .env_remove("CLICOLOR_FORCE")
+            .env("NO_COLOR", "1")
+            .args(["--color=never"])
+            // We don't want the tests to be affected by the user's `RUST_BACKTRACE` setting.
+            .env_remove("RUST_BACKTRACE")
+            .env_remove("RUST_LOG")
+    }
+}
diff --git a/prqlc/prqlc/src/cli/mod.rs b/prqlc/prqlc/src/cli/mod.rs
index 24f034cf2415..e95e15dde10a 100644
--- a/prqlc/prqlc/src/cli/mod.rs
+++ b/prqlc/prqlc/src/cli/mod.rs
@@ -1,5 +1,6 @@
 #![cfg(not(target_family = "wasm"))]
 
+mod highlight;
 mod docs_generator;
 mod jinja;
 mod watch;
@@ -222,6 +223,10 @@ pub enum ExperimentalCommand {
     /// Generate Markdown documentation
     #[command(name = "doc")]
     GenerateDocs(IoArgs),
+
+    /// Syntax highlight
+    #[command(name = "highlight")]
+    Highlight(IoArgs),
 }
 
 #[derive(clap::Args, Default, Debug, Clone)]
@@ -435,6 +440,15 @@ impl Command {
 
                 docs_generator::generate_markdown_docs(module_ref.stmts).into_bytes()
             }
+            Command::Experimental(ExperimentalCommand::Highlight(_)) => {
+                let s = sources.sources.values().exactly_one().or_else(|_| {
+                    // TODO: allow multiple sources
+                    bail!("Currently `highlight` only works with a single source, but found multiple sources")
+                })?;
+                let tokens = prql_to_tokens(s)?;
+
+                highlight::highlight(&tokens).into_bytes()
+            }
             Command::Compile {
                 signature_comment,
                 format,
@@ -483,6 +497,7 @@ impl Command {
                 io_args
             }
             Experimental(ExperimentalCommand::GenerateDocs(io_args)) => io_args,
+            Experimental(ExperimentalCommand::Highlight(io_args)) => io_args,
             _ => unreachable!(),
         };
         let input = &mut io_args.input;
@@ -518,6 +533,7 @@ impl Command {
                 io_args.output.clone()
             }
             Experimental(ExperimentalCommand::GenerateDocs(io_args)) => io_args.output.clone(),
+            Experimental(ExperimentalCommand::Highlight(io_args)) => io_args.output.clone(),
             _ => unreachable!(),
         };
         output.write_all(data)

From 0675a73d04e0be6b2f57244ce527edc8518ca992 Mon Sep 17 00:00:00 2001
From: Jonathan <vanillajonathan@users.noreply.github.com>
Date: Fri, 19 Jul 2024 23:32:55 +0200
Subject: [PATCH 02/10] Sort imports

---
 prqlc/prqlc/src/cli/mod.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/prqlc/prqlc/src/cli/mod.rs b/prqlc/prqlc/src/cli/mod.rs
index e95e15dde10a..c09a688ea155 100644
--- a/prqlc/prqlc/src/cli/mod.rs
+++ b/prqlc/prqlc/src/cli/mod.rs
@@ -1,7 +1,7 @@
 #![cfg(not(target_family = "wasm"))]
 
-mod highlight;
 mod docs_generator;
+mod highlight;
 mod jinja;
 mod watch;
 

From 3974ca0c04a54460dae90d2c546e212b37ae4d23 Mon Sep 17 00:00:00 2001
From: Jonathan <vanillajonathan@users.noreply.github.com>
Date: Sat, 20 Jul 2024 22:22:10 +0200
Subject: [PATCH 03/10] Apply suggestions from code review

Co-authored-by: Maximilian Roos <5635139+max-sixty@users.noreply.github.com>
---
 prqlc/prqlc/src/cli/highlight.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/prqlc/prqlc/src/cli/highlight.rs b/prqlc/prqlc/src/cli/highlight.rs
index 1bc835b03552..950c5f5e03ff 100644
--- a/prqlc/prqlc/src/cli/highlight.rs
+++ b/prqlc/prqlc/src/cli/highlight.rs
@@ -73,10 +73,12 @@ pub fn highlight(tokens: &Tokens) -> String {
 }
 
 fn is_transform(ident: &str) -> bool {
+// TODO: Could we instead source these from the standard library?
+// We could also use the semantic understanding from later compiler stages?
     match ident {
         "from" => true,
         "derive" | "select" | "filter" | "sort" | "join" | "take" | "group" | "aggregate"
-        | "window" | "lopo" => true,
+        | "window" | "loop" => true,
         _ => false,
     }
 }

From 81f078f7c90e4e884ceb82020c9dbcabf8b58204 Mon Sep 17 00:00:00 2001
From: Jonathan <vanillajonathan@users.noreply.github.com>
Date: Sat, 20 Jul 2024 23:04:32 +0200
Subject: [PATCH 04/10] Add broken test

---
 prqlc/prqlc/src/cli/highlight.rs | 18 +++++-------------
 1 file changed, 5 insertions(+), 13 deletions(-)

diff --git a/prqlc/prqlc/src/cli/highlight.rs b/prqlc/prqlc/src/cli/highlight.rs
index 950c5f5e03ff..bc4e642d0461 100644
--- a/prqlc/prqlc/src/cli/highlight.rs
+++ b/prqlc/prqlc/src/cli/highlight.rs
@@ -73,8 +73,8 @@ pub fn highlight(tokens: &Tokens) -> String {
 }
 
 fn is_transform(ident: &str) -> bool {
-// TODO: Could we instead source these from the standard library?
-// We could also use the semantic understanding from later compiler stages?
+    // TODO: Could we instead source these from the standard library?
+    // We could also use the semantic understanding from later compiler stages?
     match ident {
         "from" => true,
         "derive" | "select" | "filter" | "sort" | "join" | "take" | "group" | "aggregate"
@@ -92,19 +92,11 @@ mod tests {
 
     #[test]
     fn highlight() {
-        let input = r"
-foo
-
-        ";
-
-        assert_cmd_snapshot!(prqlc_command().args(["experimental", "highlight"]).pass_stdin(input), @r###"
+        assert_cmd_snapshot!(prqlc_command().args(["experimental", "highlight"]).pass_stdin("from tracks"), @r###"
         success: true
         exit_code: 0
         ----- stdout -----
-
-        foo
-
-
+        from tracks
         ----- stderr -----
         "###);
     }
@@ -114,7 +106,7 @@ foo
         normalize_prqlc(&mut cmd);
         cmd
     }
-    
+
     fn normalize_prqlc(cmd: &mut Command) -> &mut Command {
         cmd
             // We set `CLICOLOR_FORCE` in CI to force color output, but we don't want `prqlc` to

From c1f01865623b0a6315b1784b036b872a280e50b8 Mon Sep 17 00:00:00 2001
From: Maximilian Roos <m@maxroos.com>
Date: Sat, 20 Jul 2024 16:19:39 -0700
Subject: [PATCH 05/10] small code changes

---
 prqlc/prqlc/src/cli/highlight.rs | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/prqlc/prqlc/src/cli/highlight.rs b/prqlc/prqlc/src/cli/highlight.rs
index bc4e642d0461..d98ea316ac48 100644
--- a/prqlc/prqlc/src/cli/highlight.rs
+++ b/prqlc/prqlc/src/cli/highlight.rs
@@ -5,7 +5,7 @@ use prqlc::{
 };
 
 /// Highlight PRQL code printed to the terminal.
-pub fn highlight(tokens: &Tokens) -> String {
+pub(crate) fn highlight(tokens: &Tokens) -> String {
     let mut output = String::new();
     let mut last = 0;
 
@@ -75,12 +75,20 @@ pub fn highlight(tokens: &Tokens) -> String {
 fn is_transform(ident: &str) -> bool {
     // TODO: Could we instead source these from the standard library?
     // We could also use the semantic understanding from later compiler stages?
-    match ident {
-        "from" => true,
-        "derive" | "select" | "filter" | "sort" | "join" | "take" | "group" | "aggregate"
-        | "window" | "loop" => true,
-        _ => false,
-    }
+    matches!(
+        ident,
+        "from"
+            | "derive"
+            | "select"
+            | "filter"
+            | "sort"
+            | "join"
+            | "take"
+            | "group"
+            | "aggregate"
+            | "window"
+            | "loop"
+    )
 }
 
 #[cfg(test)]

From d4fff07a27133c30de72b77486f4c7905b5e8cbc Mon Sep 17 00:00:00 2001
From: Maximilian Roos <m@maxroos.com>
Date: Sat, 20 Jul 2024 16:41:18 -0700
Subject: [PATCH 06/10] Move items, works with lib

---
 prqlc/prqlc/src/error_message.rs | 25 ++-----------------------
 prqlc/prqlc/src/lib.rs           |  5 ++++-
 prqlc/prqlc/src/utils/mod.rs     | 23 ++++++++++++++++++++++-
 3 files changed, 28 insertions(+), 25 deletions(-)

diff --git a/prqlc/prqlc/src/error_message.rs b/prqlc/prqlc/src/error_message.rs
index 3d50517d0983..8671304bcd7b 100644
--- a/prqlc/prqlc/src/error_message.rs
+++ b/prqlc/prqlc/src/error_message.rs
@@ -1,10 +1,9 @@
+use std::collections::HashMap;
 use std::error::Error as StdError;
 use std::fmt::{self, Debug, Display, Formatter};
 use std::ops::Range;
 use std::path::PathBuf;
-use std::{collections::HashMap, io::stderr};
 
-use anstream::adapter::strip_str;
 use ariadne::{Cache, Config, Label, Report, ReportKind, Source};
 use serde::Serialize;
 
@@ -193,7 +192,7 @@ impl ErrorMessage {
         report.finish().write(cache, &mut out).ok()?;
         String::from_utf8(out)
             .ok()
-            .map(|x| maybe_strip_colors(x.as_str()))
+            .map(|x| crate::utils::maybe_strip_colors(x.as_str()))
     }
 
     fn compose_location(&self, source: &Source) -> Option<SourceLocation> {
@@ -208,26 +207,6 @@ impl ErrorMessage {
     }
 }
 
-fn should_use_color() -> bool {
-    match anstream::AutoStream::choice(&stderr()) {
-        anstream::ColorChoice::Auto => true,
-        anstream::ColorChoice::Always => true,
-        anstream::ColorChoice::AlwaysAnsi => true,
-        anstream::ColorChoice::Never => false,
-    }
-}
-
-/// Strip colors, for external libraries which don't yet strip themselves, and
-/// for insta snapshot tests. This will respond to environment variables such as
-/// `CLI_COLOR`.
-pub(crate) fn maybe_strip_colors(s: &str) -> String {
-    if !should_use_color() {
-        strip_str(s).to_string()
-    } else {
-        s.to_string()
-    }
-}
-
 struct FileTreeCache<'a> {
     file_tree: &'a SourceTree,
     cache: HashMap<PathBuf, Source>,
diff --git a/prqlc/prqlc/src/lib.rs b/prqlc/prqlc/src/lib.rs
index 48b132629be2..e043943288cb 100644
--- a/prqlc/prqlc/src/lib.rs
+++ b/prqlc/prqlc/src/lib.rs
@@ -107,6 +107,8 @@ use semver::Version;
 use serde::{Deserialize, Serialize};
 use strum::VariantNames;
 
+// pub(crate) use crate::utils;
+
 pub use error_message::{ErrorMessage, ErrorMessages, SourceLocation};
 pub use prqlc_parser::error::{Error, ErrorSource, Errors, MessageKind, Reason, WithErrorInfo};
 pub use prqlc_parser::lexer::lr;
@@ -120,7 +122,8 @@ pub mod ir;
 pub mod parser;
 pub mod semantic;
 pub mod sql;
-mod utils;
+pub(crate) mod utils;
+// pub mod utils;
 
 pub type Result<T, E = Error> = core::result::Result<T, E>;
 
diff --git a/prqlc/prqlc/src/utils/mod.rs b/prqlc/prqlc/src/utils/mod.rs
index 1a1aa9af6c56..b7afaadbc1b5 100644
--- a/prqlc/prqlc/src/utils/mod.rs
+++ b/prqlc/prqlc/src/utils/mod.rs
@@ -1,8 +1,9 @@
 mod id_gen;
 mod toposort;
 
-use std::sync::OnceLock;
+use std::{io::stderr, sync::OnceLock};
 
+use anstream::adapter::strip_str;
 pub use id_gen::{IdGenerator, NameGenerator};
 use itertools::Itertools;
 use regex::Regex;
@@ -91,6 +92,26 @@ pub(crate) fn valid_ident() -> &'static Regex {
     })
 }
 
+fn should_use_color() -> bool {
+    match anstream::AutoStream::choice(&stderr()) {
+        anstream::ColorChoice::Auto => true,
+        anstream::ColorChoice::Always => true,
+        anstream::ColorChoice::AlwaysAnsi => true,
+        anstream::ColorChoice::Never => false,
+    }
+}
+
+/// Strip colors, for external libraries which don't yet strip themselves, and
+/// for insta snapshot tests. This will respond to environment variables such as
+/// `CLI_COLOR`.
+pub(crate) fn maybe_strip_colors(s: &str) -> String {
+    if !should_use_color() {
+        strip_str(s).to_string()
+    } else {
+        s.to_string()
+    }
+}
+
 #[test]
 fn test_write_ident_part() {
     assert!(!valid_ident().is_match(""));

From 45bf15e585b44849e0d3f453172202c21ebb3f8b Mon Sep 17 00:00:00 2001
From: Maximilian Roos <m@maxroos.com>
Date: Sat, 20 Jul 2024 17:01:12 -0700
Subject: [PATCH 07/10] fix visibility

---
 prqlc/prqlc/src/cli/highlight.rs | 2 +-
 prqlc/prqlc/src/cli/mod.rs       | 3 ++-
 prqlc/prqlc/src/lib.rs           | 6 +++---
 prqlc/prqlc/src/utils/mod.rs     | 2 +-
 4 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/prqlc/prqlc/src/cli/highlight.rs b/prqlc/prqlc/src/cli/highlight.rs
index d98ea316ac48..e8550859488d 100644
--- a/prqlc/prqlc/src/cli/highlight.rs
+++ b/prqlc/prqlc/src/cli/highlight.rs
@@ -104,7 +104,7 @@ mod tests {
         success: true
         exit_code: 0
         ----- stdout -----
-        from tracks
+        from tracks
         ----- stderr -----
         "###);
     }
diff --git a/prqlc/prqlc/src/cli/mod.rs b/prqlc/prqlc/src/cli/mod.rs
index c09a688ea155..70904c9eb46b 100644
--- a/prqlc/prqlc/src/cli/mod.rs
+++ b/prqlc/prqlc/src/cli/mod.rs
@@ -33,6 +33,7 @@ use prqlc::ir::{pl, rq};
 use prqlc::pr;
 use prqlc::semantic;
 use prqlc::semantic::reporting::FrameCollector;
+use prqlc::utils::maybe_strip_colors;
 use prqlc::{pl_to_prql, pl_to_rq_tree, prql_to_pl, prql_to_pl_tree, prql_to_tokens, rq_to_sql};
 use prqlc::{Options, SourceTree, Target};
 
@@ -447,7 +448,7 @@ impl Command {
                 })?;
                 let tokens = prql_to_tokens(s)?;
 
-                highlight::highlight(&tokens).into_bytes()
+                maybe_strip_colors(&highlight::highlight(&tokens)).into_bytes()
             }
             Command::Compile {
                 signature_comment,
diff --git a/prqlc/prqlc/src/lib.rs b/prqlc/prqlc/src/lib.rs
index e043943288cb..a71a22a07cca 100644
--- a/prqlc/prqlc/src/lib.rs
+++ b/prqlc/prqlc/src/lib.rs
@@ -107,8 +107,6 @@ use semver::Version;
 use serde::{Deserialize, Serialize};
 use strum::VariantNames;
 
-// pub(crate) use crate::utils;
-
 pub use error_message::{ErrorMessage, ErrorMessages, SourceLocation};
 pub use prqlc_parser::error::{Error, ErrorSource, Errors, MessageKind, Reason, WithErrorInfo};
 pub use prqlc_parser::lexer::lr;
@@ -122,8 +120,10 @@ pub mod ir;
 pub mod parser;
 pub mod semantic;
 pub mod sql;
+#[cfg(feature = "cli")]
+pub mod utils;
+#[cfg(not(feature = "cli"))]
 pub(crate) mod utils;
-// pub mod utils;
 
 pub type Result<T, E = Error> = core::result::Result<T, E>;
 
diff --git a/prqlc/prqlc/src/utils/mod.rs b/prqlc/prqlc/src/utils/mod.rs
index b7afaadbc1b5..83ff7b2e5139 100644
--- a/prqlc/prqlc/src/utils/mod.rs
+++ b/prqlc/prqlc/src/utils/mod.rs
@@ -104,7 +104,7 @@ fn should_use_color() -> bool {
 /// Strip colors, for external libraries which don't yet strip themselves, and
 /// for insta snapshot tests. This will respond to environment variables such as
 /// `CLI_COLOR`.
-pub(crate) fn maybe_strip_colors(s: &str) -> String {
+pub fn maybe_strip_colors(s: &str) -> String {
     if !should_use_color() {
         strip_str(s).to_string()
     } else {

From db0118fda2486f21b4dcab1f81feb1e6ff1128dd Mon Sep 17 00:00:00 2001
From: Maximilian Roos <m@maxroos.com>
Date: Sat, 20 Jul 2024 17:13:36 -0700
Subject: [PATCH 08/10]

---
 prqlc/prqlc-macros/src/lib.rs                 |  2 +-
 prqlc/prqlc-parser/src/parser/pr/stmt.rs      |  2 -
 prqlc/prqlc/src/cli/highlight.rs              | 24 ++++++-
 prqlc/prqlc/src/cli/mod.rs                    | 12 ++--
 ...prqlc__cli__test__shell_completion-2.snap} | 26 +++++---
 ...prqlc__cli__test__shell_completion-3.snap} | 20 +++++-
 ...prqlc__cli__test__shell_completion-4.snap} | 41 +++++++++++-
 .../prqlc__cli__test__shell_completion.snap}  | 62 +++++++++++++++++--
 .../integration/cli.rs => src/cli/test.rs}    |  2 -
 prqlc/prqlc/tests/integration/main.rs         |  1 -
 10 files changed, 163 insertions(+), 29 deletions(-)
 rename prqlc/prqlc/{tests/integration/snapshots/integration__cli__shell_completion-2.snap => src/cli/snapshots/prqlc__cli__test__shell_completion-2.snap} (90%)
 rename prqlc/prqlc/{tests/integration/snapshots/integration__cli__shell_completion-3.snap => src/cli/snapshots/prqlc__cli__test__shell_completion-3.snap} (94%)
 rename prqlc/prqlc/{tests/integration/snapshots/integration__cli__shell_completion-4.snap => src/cli/snapshots/prqlc__cli__test__shell_completion-4.snap} (94%)
 rename prqlc/prqlc/{tests/integration/snapshots/integration__cli__shell_completion.snap => src/cli/snapshots/prqlc__cli__test__shell_completion.snap} (93%)
 rename prqlc/prqlc/{tests/integration/cli.rs => src/cli/test.rs} (99%)

diff --git a/prqlc/prqlc-macros/src/lib.rs b/prqlc/prqlc-macros/src/lib.rs
index 05d885f084e0..1ef668617b95 100644
--- a/prqlc/prqlc-macros/src/lib.rs
+++ b/prqlc/prqlc-macros/src/lib.rs
@@ -1,7 +1,7 @@
 //! Macros for PRQL compilation at build time.
 //!
 //! ```
-//! use prql_compiler_macros::prql_to_sql;
+//! use prqlc_macros::prql_to_sql;
 //!
 //! let sql: &str = prql_to_sql!("from albums | select {title, artist_id}");
 //! assert_eq!(sql, "SELECT title, artist_id FROM albums");
diff --git a/prqlc/prqlc-parser/src/parser/pr/stmt.rs b/prqlc/prqlc-parser/src/parser/pr/stmt.rs
index 77ff24c888bc..df1da2569347 100644
--- a/prqlc/prqlc-parser/src/parser/pr/stmt.rs
+++ b/prqlc/prqlc-parser/src/parser/pr/stmt.rs
@@ -25,8 +25,6 @@ pub enum VarDefKind {
     Main,
 }
 
-// The following code is tested by the tests_misc crate to match stmt.rs in prql_compiler.
-
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
 pub struct Stmt {
     #[serde(flatten)]
diff --git a/prqlc/prqlc/src/cli/highlight.rs b/prqlc/prqlc/src/cli/highlight.rs
index e8550859488d..11f7f107ce08 100644
--- a/prqlc/prqlc/src/cli/highlight.rs
+++ b/prqlc/prqlc/src/cli/highlight.rs
@@ -100,11 +100,31 @@ mod tests {
 
     #[test]
     fn highlight() {
-        assert_cmd_snapshot!(prqlc_command().args(["experimental", "highlight"]).pass_stdin("from tracks"), @r###"
+        // (Colors don't show because they're disabled; we could have a test
+        // that forces them to show?)
+        assert_cmd_snapshot!(prqlc_command().args(["experimental", "highlight"]).pass_stdin(r#"
+        from tracks
+        filter artist == "Bob Marley"                 # Each line transforms the previous result
+        aggregate {                                   # `aggregate` reduces each column to a value
+          plays    = sum plays,
+          longest  = max length,
+          shortest = min length,                      # Trailing commas are allowed
+        }
+
+        "#), @r###"
         success: true
         exit_code: 0
         ----- stdout -----
-        from tracks
+
+                from tracks
+                filter artist == "Bob Marley"                 # Each line transforms the previous result
+                aggregate {                                   # `aggregate` reduces each column to a value
+                  plays    = sum plays,
+                  longest  = max length,
+                  shortest = min length,                      # Trailing commas are allowed
+                }
+
+
         ----- stderr -----
         "###);
     }
diff --git a/prqlc/prqlc/src/cli/mod.rs b/prqlc/prqlc/src/cli/mod.rs
index 70904c9eb46b..f955256289d1 100644
--- a/prqlc/prqlc/src/cli/mod.rs
+++ b/prqlc/prqlc/src/cli/mod.rs
@@ -1,10 +1,5 @@
 #![cfg(not(target_family = "wasm"))]
 
-mod docs_generator;
-mod highlight;
-mod jinja;
-mod watch;
-
 use std::collections::HashMap;
 use std::env;
 use std::fs::File;
@@ -37,6 +32,13 @@ use prqlc::utils::maybe_strip_colors;
 use prqlc::{pl_to_prql, pl_to_rq_tree, prql_to_pl, prql_to_pl_tree, prql_to_tokens, rq_to_sql};
 use prqlc::{Options, SourceTree, Target};
 
+mod docs_generator;
+mod highlight;
+mod jinja;
+#[cfg(test)]
+mod test;
+mod watch;
+
 /// Entrypoint called by [`crate::main`]
 pub fn main() -> color_eyre::eyre::Result<()> {
     let mut cli = Cli::parse();
diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion-2.snap b/prqlc/prqlc/src/cli/snapshots/prqlc__cli__test__shell_completion-2.snap
similarity index 90%
rename from prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion-2.snap
rename to prqlc/prqlc/src/cli/snapshots/prqlc__cli__test__shell_completion-2.snap
index cdd6db7eca17..ec692fc82acf 100644
--- a/prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion-2.snap
+++ b/prqlc/prqlc/src/cli/snapshots/prqlc__cli__test__shell_completion-2.snap
@@ -1,5 +1,4 @@
 ---
-source: prqlc/prqlc/tests/integration/cli.rs
 info:
   program: prqlc
   args:
@@ -81,18 +80,24 @@ complete -c prqlc -n "__fish_seen_subcommand_from debug; and __fish_seen_subcomm
 complete -c prqlc -n "__fish_seen_subcommand_from debug; and __fish_seen_subcommand_from help; and not __fish_seen_subcommand_from annotate; and not __fish_seen_subcommand_from lineage; and not __fish_seen_subcommand_from ast; and not __fish_seen_subcommand_from json-schema; and not __fish_seen_subcommand_from help" -f -a "ast" -d 'Print info about the AST data structure'
 complete -c prqlc -n "__fish_seen_subcommand_from debug; and __fish_seen_subcommand_from help; and not __fish_seen_subcommand_from annotate; and not __fish_seen_subcommand_from lineage; and not __fish_seen_subcommand_from ast; and not __fish_seen_subcommand_from json-schema; and not __fish_seen_subcommand_from help" -f -a "json-schema" -d 'Print JSON Schema'
 complete -c prqlc -n "__fish_seen_subcommand_from debug; and __fish_seen_subcommand_from help; and not __fish_seen_subcommand_from annotate; and not __fish_seen_subcommand_from lineage; and not __fish_seen_subcommand_from ast; and not __fish_seen_subcommand_from json-schema; and not __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
-complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from help" -l color -d 'Controls when to use color' -r -f -a "{auto	'',always	'',never	''}"
-complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from help" -s v -l verbose -d 'Increase logging verbosity'
-complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from help" -s q -l quiet -d 'Silences logging output'
-complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from help" -s h -l help -d 'Print help (see more with \'--help\')'
-complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from help" -f -a "doc" -d 'Generate Markdown documentation'
-complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from highlight; and not __fish_seen_subcommand_from help" -l color -d 'Controls when to use color' -r -f -a "{auto	'',always	'',never	''}"
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from highlight; and not __fish_seen_subcommand_from help" -s v -l verbose -d 'Increase logging verbosity'
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from highlight; and not __fish_seen_subcommand_from help" -s q -l quiet -d 'Silences logging output'
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from highlight; and not __fish_seen_subcommand_from help" -s h -l help -d 'Print help (see more with \'--help\')'
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from highlight; and not __fish_seen_subcommand_from help" -f -a "doc" -d 'Generate Markdown documentation'
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from highlight; and not __fish_seen_subcommand_from help" -f -a "highlight" -d 'Syntax highlight'
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from highlight; and not __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
 complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from doc" -l color -d 'Controls when to use color' -r -f -a "{auto	'',always	'',never	''}"
 complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from doc" -s v -l verbose -d 'Increase logging verbosity'
 complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from doc" -s q -l quiet -d 'Silences logging output'
 complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from doc" -s h -l help -d 'Print help (see more with \'--help\')'
-complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from help; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from help" -f -a "doc" -d 'Generate Markdown documentation'
-complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from help; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from highlight" -l color -d 'Controls when to use color' -r -f -a "{auto	'',always	'',never	''}"
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from highlight" -s v -l verbose -d 'Increase logging verbosity'
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from highlight" -s q -l quiet -d 'Silences logging output'
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from highlight" -s h -l help -d 'Print help (see more with \'--help\')'
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from help; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from highlight; and not __fish_seen_subcommand_from help" -f -a "doc" -d 'Generate Markdown documentation'
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from help; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from highlight; and not __fish_seen_subcommand_from help" -f -a "highlight" -d 'Syntax highlight'
+complete -c prqlc -n "__fish_seen_subcommand_from experimental; and __fish_seen_subcommand_from help; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from highlight; and not __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
 complete -c prqlc -n "__fish_seen_subcommand_from compile" -s t -l target -d 'Target to compile to' -r
 complete -c prqlc -n "__fish_seen_subcommand_from compile" -l debug-log -d 'File path into which to write the debug log to' -r -F
 complete -c prqlc -n "__fish_seen_subcommand_from compile" -l color -d 'Controls when to use color' -r -f -a "{auto	'',always	'',never	''}"
@@ -130,6 +135,7 @@ complete -c prqlc -n "__fish_seen_subcommand_from help; and __fish_seen_subcomma
 complete -c prqlc -n "__fish_seen_subcommand_from help; and __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from annotate; and not __fish_seen_subcommand_from lineage; and not __fish_seen_subcommand_from ast; and not __fish_seen_subcommand_from json-schema" -f -a "lineage" -d 'Output column-level lineage graph'
 complete -c prqlc -n "__fish_seen_subcommand_from help; and __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from annotate; and not __fish_seen_subcommand_from lineage; and not __fish_seen_subcommand_from ast; and not __fish_seen_subcommand_from json-schema" -f -a "ast" -d 'Print info about the AST data structure'
 complete -c prqlc -n "__fish_seen_subcommand_from help; and __fish_seen_subcommand_from debug; and not __fish_seen_subcommand_from annotate; and not __fish_seen_subcommand_from lineage; and not __fish_seen_subcommand_from ast; and not __fish_seen_subcommand_from json-schema" -f -a "json-schema" -d 'Print JSON Schema'
-complete -c prqlc -n "__fish_seen_subcommand_from help; and __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc" -f -a "doc" -d 'Generate Markdown documentation'
+complete -c prqlc -n "__fish_seen_subcommand_from help; and __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from highlight" -f -a "doc" -d 'Generate Markdown documentation'
+complete -c prqlc -n "__fish_seen_subcommand_from help; and __fish_seen_subcommand_from experimental; and not __fish_seen_subcommand_from doc; and not __fish_seen_subcommand_from highlight" -f -a "highlight" -d 'Syntax highlight'
 
 ----- stderr -----
diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion-3.snap b/prqlc/prqlc/src/cli/snapshots/prqlc__cli__test__shell_completion-3.snap
similarity index 94%
rename from prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion-3.snap
rename to prqlc/prqlc/src/cli/snapshots/prqlc__cli__test__shell_completion-3.snap
index c1e11e354ccb..8d515d6929eb 100644
--- a/prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion-3.snap
+++ b/prqlc/prqlc/src/cli/snapshots/prqlc__cli__test__shell_completion-3.snap
@@ -1,5 +1,4 @@
 ---
-source: prqlc/prqlc/tests/integration/cli.rs
 info:
   program: prqlc
   args:
@@ -191,6 +190,7 @@ Register-ArgumentCompleter -Native -CommandName 'prqlc' -ScriptBlock {
             [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             [CompletionResult]::new('doc', 'doc', [CompletionResultType]::ParameterValue, 'Generate Markdown documentation')
+            [CompletionResult]::new('highlight', 'highlight', [CompletionResultType]::ParameterValue, 'Syntax highlight')
             [CompletionResult]::new('help', 'help', [CompletionResultType]::ParameterValue, 'Print this message or the help of the given subcommand(s)')
             break
         }
@@ -204,14 +204,28 @@ Register-ArgumentCompleter -Native -CommandName 'prqlc' -ScriptBlock {
             [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             break
         }
+        'prqlc;experimental;highlight' {
+            [CompletionResult]::new('--color', 'color', [CompletionResultType]::ParameterName, 'Controls when to use color')
+            [CompletionResult]::new('-v', 'v', [CompletionResultType]::ParameterName, 'Increase logging verbosity')
+            [CompletionResult]::new('--verbose', 'verbose', [CompletionResultType]::ParameterName, 'Increase logging verbosity')
+            [CompletionResult]::new('-q', 'q', [CompletionResultType]::ParameterName, 'Silences logging output')
+            [CompletionResult]::new('--quiet', 'quiet', [CompletionResultType]::ParameterName, 'Silences logging output')
+            [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
+            [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
+            break
+        }
         'prqlc;experimental;help' {
             [CompletionResult]::new('doc', 'doc', [CompletionResultType]::ParameterValue, 'Generate Markdown documentation')
+            [CompletionResult]::new('highlight', 'highlight', [CompletionResultType]::ParameterValue, 'Syntax highlight')
             [CompletionResult]::new('help', 'help', [CompletionResultType]::ParameterValue, 'Print this message or the help of the given subcommand(s)')
             break
         }
         'prqlc;experimental;help;doc' {
             break
         }
+        'prqlc;experimental;help;highlight' {
+            break
+        }
         'prqlc;experimental;help;help' {
             break
         }
@@ -309,11 +323,15 @@ Register-ArgumentCompleter -Native -CommandName 'prqlc' -ScriptBlock {
         }
         'prqlc;help;experimental' {
             [CompletionResult]::new('doc', 'doc', [CompletionResultType]::ParameterValue, 'Generate Markdown documentation')
+            [CompletionResult]::new('highlight', 'highlight', [CompletionResultType]::ParameterValue, 'Syntax highlight')
             break
         }
         'prqlc;help;experimental;doc' {
             break
         }
+        'prqlc;help;experimental;highlight' {
+            break
+        }
         'prqlc;help;compile' {
             break
         }
diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion-4.snap b/prqlc/prqlc/src/cli/snapshots/prqlc__cli__test__shell_completion-4.snap
similarity index 94%
rename from prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion-4.snap
rename to prqlc/prqlc/src/cli/snapshots/prqlc__cli__test__shell_completion-4.snap
index 7025de96b48f..227d1ace0568 100644
--- a/prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion-4.snap
+++ b/prqlc/prqlc/src/cli/snapshots/prqlc__cli__test__shell_completion-4.snap
@@ -1,5 +1,4 @@
 ---
-source: prqlc/prqlc/tests/integration/cli.rs
 info:
   program: prqlc
   args:
@@ -250,6 +249,20 @@ _arguments "${_arguments_options[@]}" \
 '::main_path -- Identifier of the main pipeline:' \
 && ret=0
 ;;
+(highlight)
+_arguments "${_arguments_options[@]}" \
+'--color=[Controls when to use color]:WHEN:(auto always never)' \
+'*-v[Increase logging verbosity]' \
+'*--verbose[Increase logging verbosity]' \
+'(-v --verbose)*-q[Silences logging output]' \
+'(-v --verbose)*--quiet[Silences logging output]' \
+'-h[Print help (see more with '\''--help'\'')]' \
+'--help[Print help (see more with '\''--help'\'')]' \
+'::input:_files' \
+'::output:_files' \
+'::main_path -- Identifier of the main pipeline:' \
+&& ret=0
+;;
 (help)
 _arguments "${_arguments_options[@]}" \
 ":: :_prqlc__experimental__help_commands" \
@@ -266,6 +279,10 @@ _arguments "${_arguments_options[@]}" \
 _arguments "${_arguments_options[@]}" \
 && ret=0
 ;;
+(highlight)
+_arguments "${_arguments_options[@]}" \
+&& ret=0
+;;
 (help)
 _arguments "${_arguments_options[@]}" \
 && ret=0
@@ -409,6 +426,10 @@ _arguments "${_arguments_options[@]}" \
             (doc)
 _arguments "${_arguments_options[@]}" \
 && ret=0
+;;
+(highlight)
+_arguments "${_arguments_options[@]}" \
+&& ret=0
 ;;
         esac
     ;;
@@ -550,6 +571,7 @@ _prqlc__help__experimental__doc_commands() {
 _prqlc__experimental_commands() {
     local commands; commands=(
 'doc:Generate Markdown documentation' \
+'highlight:Syntax highlight' \
 'help:Print this message or the help of the given subcommand(s)' \
     )
     _describe -t commands 'prqlc experimental commands' commands "$@"
@@ -558,6 +580,7 @@ _prqlc__experimental_commands() {
 _prqlc__help__experimental_commands() {
     local commands; commands=(
 'doc:Generate Markdown documentation' \
+'highlight:Syntax highlight' \
     )
     _describe -t commands 'prqlc help experimental commands' commands "$@"
 }
@@ -591,6 +614,7 @@ _prqlc__debug__help__help_commands() {
 _prqlc__experimental__help_commands() {
     local commands; commands=(
 'doc:Generate Markdown documentation' \
+'highlight:Syntax highlight' \
 'help:Print this message or the help of the given subcommand(s)' \
     )
     _describe -t commands 'prqlc experimental help commands' commands "$@"
@@ -622,6 +646,21 @@ _prqlc__help__help_commands() {
     local commands; commands=()
     _describe -t commands 'prqlc help help commands' commands "$@"
 }
+(( $+functions[_prqlc__experimental__help__highlight_commands] )) ||
+_prqlc__experimental__help__highlight_commands() {
+    local commands; commands=()
+    _describe -t commands 'prqlc experimental help highlight commands' commands "$@"
+}
+(( $+functions[_prqlc__experimental__highlight_commands] )) ||
+_prqlc__experimental__highlight_commands() {
+    local commands; commands=()
+    _describe -t commands 'prqlc experimental highlight commands' commands "$@"
+}
+(( $+functions[_prqlc__help__experimental__highlight_commands] )) ||
+_prqlc__help__experimental__highlight_commands() {
+    local commands; commands=()
+    _describe -t commands 'prqlc help experimental highlight commands' commands "$@"
+}
 (( $+functions[_prqlc__debug__help__json-schema_commands] )) ||
 _prqlc__debug__help__json-schema_commands() {
     local commands; commands=()
diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion.snap b/prqlc/prqlc/src/cli/snapshots/prqlc__cli__test__shell_completion.snap
similarity index 93%
rename from prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion.snap
rename to prqlc/prqlc/src/cli/snapshots/prqlc__cli__test__shell_completion.snap
index 22e14b2ce6ff..f017cf0475ae 100644
--- a/prqlc/prqlc/tests/integration/snapshots/integration__cli__shell_completion.snap
+++ b/prqlc/prqlc/src/cli/snapshots/prqlc__cli__test__shell_completion.snap
@@ -1,5 +1,4 @@
 ---
-source: prqlc/prqlc/tests/integration/cli.rs
 info:
   program: prqlc
   args:
@@ -98,12 +97,18 @@ _prqlc() {
             prqlc__experimental,help)
                 cmd="prqlc__experimental__help"
                 ;;
+            prqlc__experimental,highlight)
+                cmd="prqlc__experimental__highlight"
+                ;;
             prqlc__experimental__help,doc)
                 cmd="prqlc__experimental__help__doc"
                 ;;
             prqlc__experimental__help,help)
                 cmd="prqlc__experimental__help__help"
                 ;;
+            prqlc__experimental__help,highlight)
+                cmd="prqlc__experimental__help__highlight"
+                ;;
             prqlc__help,collect)
                 cmd="prqlc__help__collect"
                 ;;
@@ -152,6 +157,9 @@ _prqlc() {
             prqlc__help__experimental,doc)
                 cmd="prqlc__help__experimental__doc"
                 ;;
+            prqlc__help__experimental,highlight)
+                cmd="prqlc__help__experimental__highlight"
+                ;;
             *)
                 ;;
         esac
@@ -407,7 +415,7 @@ _prqlc() {
             return 0
             ;;
         prqlc__experimental)
-            opts="-v -q -h --color --verbose --quiet --help doc help"
+            opts="-v -q -h --color --verbose --quiet --help doc highlight help"
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -443,7 +451,7 @@ _prqlc() {
             return 0
             ;;
         prqlc__experimental__help)
-            opts="doc help"
+            opts="doc highlight help"
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -484,6 +492,38 @@ _prqlc() {
             COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
             return 0
             ;;
+        prqlc__experimental__help__highlight)
+            opts=""
+            if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then
+                COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+                return 0
+            fi
+            case "${prev}" in
+                *)
+                    COMPREPLY=()
+                    ;;
+            esac
+            COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+            return 0
+            ;;
+        prqlc__experimental__highlight)
+            opts="-v -q -h --color --verbose --quiet --help [INPUT] [OUTPUT] [MAIN_PATH]"
+            if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then
+                COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+                return 0
+            fi
+            case "${prev}" in
+                --color)
+                    COMPREPLY=($(compgen -W "auto always never" -- "${cur}"))
+                    return 0
+                    ;;
+                *)
+                    COMPREPLY=()
+                    ;;
+            esac
+            COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+            return 0
+            ;;
         prqlc__fmt)
             opts="-v -q -h --color --verbose --quiet --help [INPUT]"
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
@@ -615,7 +655,7 @@ _prqlc() {
             return 0
             ;;
         prqlc__help__experimental)
-            opts="doc"
+            opts="doc highlight"
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -642,6 +682,20 @@ _prqlc() {
             COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
             return 0
             ;;
+        prqlc__help__experimental__highlight)
+            opts=""
+            if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then
+                COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+                return 0
+            fi
+            case "${prev}" in
+                *)
+                    COMPREPLY=()
+                    ;;
+            esac
+            COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+            return 0
+            ;;
         prqlc__help__fmt)
             opts=""
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then
diff --git a/prqlc/prqlc/tests/integration/cli.rs b/prqlc/prqlc/src/cli/test.rs
similarity index 99%
rename from prqlc/prqlc/tests/integration/cli.rs
rename to prqlc/prqlc/src/cli/test.rs
index ae39dc933b95..2a40c19635a1 100644
--- a/prqlc/prqlc/tests/integration/cli.rs
+++ b/prqlc/prqlc/src/cli/test.rs
@@ -1,5 +1,3 @@
-#![cfg(all(not(target_family = "wasm"), feature = "cli"))]
-
 use std::env::current_dir;
 use std::fs;
 use std::path::Path;
diff --git a/prqlc/prqlc/tests/integration/main.rs b/prqlc/prqlc/tests/integration/main.rs
index 8646ef80c217..e076ee1fa6a9 100644
--- a/prqlc/prqlc/tests/integration/main.rs
+++ b/prqlc/prqlc/tests/integration/main.rs
@@ -1,5 +1,4 @@
 mod bad_error_messages;
-mod cli;
 mod dbs;
 mod error_messages;
 mod queries;

From c23813e4dec749144342382669bee7e3b24ec7aa Mon Sep 17 00:00:00 2001
From: Maximilian Roos <m@maxroos.com>
Date: Sat, 20 Jul 2024 17:21:31 -0700
Subject: [PATCH 09/10]

---
 prqlc/prqlc/src/cli/highlight.rs | 114 +++++++++++++++++--------------
 1 file changed, 63 insertions(+), 51 deletions(-)

diff --git a/prqlc/prqlc/src/cli/highlight.rs b/prqlc/prqlc/src/cli/highlight.rs
index 11f7f107ce08..3e06d5c31c80 100644
--- a/prqlc/prqlc/src/cli/highlight.rs
+++ b/prqlc/prqlc/src/cli/highlight.rs
@@ -13,62 +13,74 @@ pub(crate) fn highlight(tokens: &Tokens) -> String {
         let diff = token.span.start - last;
         last = token.span.end;
         output.push_str(&" ".repeat(diff));
+        output.push_str(&highlight_token_kind(&token.kind));
+    }
 
-        match &token.kind {
-            TokenKind::NewLine => output.push('\n'),
-            TokenKind::Ident(ident) => {
-                if is_transform(ident) {
-                    output.push_str(&ident.green().to_string())
-                } else {
-                    output.push_str(&ident.to_string())
-                }
-            }
-            TokenKind::Keyword(keyword) => output.push_str(&keyword.blue().to_string()),
-            TokenKind::Literal(literal) => output.push_str(&match literal {
-                Literal::Null => literal.green().bold().to_string(),
-                Literal::Integer(_) => literal.green().to_string(),
-                Literal::Float(_) => literal.green().to_string(),
-                Literal::Boolean(_) => literal.green().bold().to_string(),
-                Literal::String(_) => literal.yellow().to_string(),
-                _ => literal.to_string(),
-            }),
-            TokenKind::Param(param) => output.push_str(&param.purple().to_string()),
-            TokenKind::Range {
-                bind_left: _,
-                bind_right: _,
-            } => output.push_str(".."),
-            TokenKind::Interpolation(_, _) => output.push_str(&format!("{}", token.kind.yellow())),
-            TokenKind::Control(char) => output.push(*char),
-            TokenKind::ArrowThin
-            | TokenKind::ArrowFat
-            | TokenKind::Eq
-            | TokenKind::Ne
-            | TokenKind::Gte
-            | TokenKind::Lte
-            | TokenKind::RegexSearch => output.push_str(&format!("{}", token.kind)),
-            TokenKind::And | TokenKind::Or => {
-                output.push_str(&format!("{}", token.kind).purple().to_string())
+    output
+}
+
+fn highlight_token_kind(token: &TokenKind) -> String {
+    // LineWrap is recursive with TokenKind, so we needed to split this function
+    // out from the one above (otherwise would have it as a single func)
+    let mut output = String::new();
+    match &token {
+        TokenKind::NewLine => output.push('\n'),
+        TokenKind::Ident(ident) => {
+            if is_transform(ident) {
+                output.push_str(&ident.green().to_string())
+            } else {
+                output.push_str(&ident.to_string())
             }
-            TokenKind::Coalesce | TokenKind::DivInt | TokenKind::Pow | TokenKind::Annotate => {
-                output.push_str(&format!("{}", token.kind))
+        }
+        TokenKind::Keyword(keyword) => output.push_str(&keyword.blue().to_string()),
+        TokenKind::Literal(literal) => output.push_str(&match literal {
+            Literal::Null => literal.green().bold().to_string(),
+            Literal::Integer(_) => literal.green().to_string(),
+            Literal::Float(_) => literal.green().to_string(),
+            Literal::Boolean(_) => literal.green().bold().to_string(),
+            Literal::String(_) => literal.yellow().to_string(),
+            _ => literal.to_string(),
+        }),
+        TokenKind::Param(param) => output.push_str(&param.purple().to_string()),
+        TokenKind::Range {
+            bind_left: _,
+            bind_right: _,
+        } => output.push_str(".."),
+        TokenKind::Interpolation(_, _) => output.push_str(&format!("{}", token.yellow())),
+        TokenKind::Control(char) => output.push(*char),
+        TokenKind::ArrowThin
+        | TokenKind::ArrowFat
+        | TokenKind::Eq
+        | TokenKind::Ne
+        | TokenKind::Gte
+        | TokenKind::Lte
+        | TokenKind::RegexSearch => output.push_str(&format!("{}", token)),
+        TokenKind::And | TokenKind::Or => {
+            output.push_str(&format!("{}", token).purple().to_string())
+        }
+        TokenKind::Coalesce | TokenKind::DivInt | TokenKind::Pow | TokenKind::Annotate => {
+            output.push_str(&format!("{}", token))
+        }
+        TokenKind::Comment(comment) => output.push_str(
+            &format!("#{comment}")
+                .truecolor(95, 135, 135)
+                .italic()
+                .to_string(),
+        ),
+        TokenKind::DocComment(comment) => output.push_str(
+            &format!("#!{comment}")
+                .truecolor(95, 135, 135)
+                .italic()
+                .to_string(),
+        ),
+        TokenKind::LineWrap(inner_tokens) => {
+            output.push_str("\n\\");
+            for t in inner_tokens {
+                output.push_str(&highlight_token_kind(t));
             }
-            TokenKind::Comment(comment) => output.push_str(
-                &format!("#{comment}")
-                    .truecolor(95, 135, 135)
-                    .italic()
-                    .to_string(),
-            ),
-            TokenKind::DocComment(comment) => output.push_str(
-                &format!("#!{comment}")
-                    .truecolor(95, 135, 135)
-                    .italic()
-                    .to_string(),
-            ),
-            TokenKind::LineWrap(_) => todo!(),
-            TokenKind::Start => {}
         }
+        TokenKind::Start => {}
     }
-
     output
 }
 

From 7c4fcbbca6319dbcb1526ac563d1a4a3316e9283 Mon Sep 17 00:00:00 2001
From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com>
Date: Sun, 21 Jul 2024 12:15:38 -0700
Subject: [PATCH 10/10] Update prqlc/prqlc/src/cli/highlight.rs

---
 prqlc/prqlc/src/cli/highlight.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/prqlc/prqlc/src/cli/highlight.rs b/prqlc/prqlc/src/cli/highlight.rs
index 3e06d5c31c80..096f0de36770 100644
--- a/prqlc/prqlc/src/cli/highlight.rs
+++ b/prqlc/prqlc/src/cli/highlight.rs
@@ -141,6 +141,7 @@ mod tests {
         "###);
     }
 
+    // TODO: import from existing location, need to adjust visibility
     fn prqlc_command() -> Command {
         let mut cmd = Command::new(get_cargo_bin("prqlc"));
         normalize_prqlc(&mut cmd);