diff --git a/SUPPORTED_LANGUAGES.md b/SUPPORTED_LANGUAGES.md index 2b9b710..14bf356 100644 --- a/SUPPORTED_LANGUAGES.md +++ b/SUPPORTED_LANGUAGES.md @@ -2,51 +2,51 @@ 46 languages are currently supported. -| File type | Extension | Supported comments | -| ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------- | -| Assembly | `.asm`, `.a51`, `.i`, `.inc`, `.nas`, `.nasm` | `;`, `/* */` | -| C | `.c`, `.cats`, `.h`, `.idc` | `//`, `/* */` | -| C# | `.cs`, `.cake`, `.cs.pp`, `.csx`, `.linq` | `//`, `/* */` | -| C++ | `.cpp`, `.c++`, `.cc`, `.cp`, `.cppm`, `.cxx`, `.h`, `.h++`, `.hh`, `.hpp`, `.hxx`, `.inc`, `.inl`, `.ino`, `.ipp`, `.ixx`, `.re`, `.tcc`, `.tpp`, `.txx` | `//`, `/* */` | -| Clojure | `.clj`, `.bb`, `.boot`, `.cl2`, `.cljc`, `.cljs`, `.cljs.hl`, `.cljscm`, `.cljx`, `.hic` | `;` | -| CoffeeScript | `.coffee`, `._coffee`, `.cake`, `.cjsx`, `.iced` | `#`, `### ###` | -| Dockerfile | `.dockerfile` | `#` | -| Elixir | `.ex`, `.exs` | `#`, `@doc """ """` | -| Emacs Lisp | `.el`, `.emacs`, `.emacs.desktop` | `;` | -| Erlang | `.erl`, `.app`, `.app.src`, `.es`, `.escript`, `.hrl`, `.xrl`, `.yrl` | `%` | -| Fortran | `.f`, `.f77`, `.for`, `.fpp` | `!` | -| Fortran Free Form | `.f90`, `.f03`, `.f08`, `.f95` | `!` | -| Go | `.go` | `//`, `/* */` | -| Go Module | | `//` | -| Groovy | `.groovy`, `.grt`, `.gtpl`, `.gvy` | `//`, `/* */` | -| HTML | `.html`, `.hta`, `.htm`, `.html.hl`, `.inc`, `.xht`, `.xhtml` | `` | -| Haskell | `.hs`, `.hs-boot`, `.hsc` | `--`, `{- -}` | -| JSON | `.json`, `.4DForm`, `.4DProject`, `.avsc`, `.geojson`, `.gltf`, `.har`, `.ice`, `.JSON-tmLanguage`, `.jsonl`, `.mcmeta`, `.sarif`, `.tfstate`, `.tfstate.backup`, `.topojson`, `.webapp`, `.webmanifest`, `.yy`, `.yyp` | `//`, `#`, `/* */` | -| Java | `.java`, `.jav`, `.jsh` | `//`, `/* */` | -| JavaScript | `.js`, `._js`, `.bones`, `.cjs`, `.es`, `.es6`, `.frag`, `.gs`, `.jake`, `.javascript`, `.jsb`, `.jscad`, `.jsfl`, `.jslib`, `.jsm`, `.jspre`, `.jss`, `.jsx`, `.mjs`, `.njs`, `.pac`, `.sjs`, `.ssjs`, `.xsjs`, `.xsjslib` | `//`, `/* */` | -| Kotlin | `.kt`, `.ktm`, `.kts` | `//`, `/* */` | -| Lua | `.lua`, `.fcgi`, `.nse`, `.p8`, `.pd_lua`, `.rbxs`, `.rockspec`, `.wlua` | `--`, `--[[ --]]` | -| MATLAB | `.matlab`, `.m` | `%`, `%{ }%` | -| Makefile | `.mak`, `.d`, `.make`, `.makefile`, `.mk`, `.mkfile` | `#` | -| Objective-C | `.m`, `.h` | `//`, `/* */` | -| PHP | `.php`, `.aw`, `.ctp`, `.fcgi`, `.inc`, `.php3`, `.php4`, `.php5`, `.phps`, `.phpt` | `#`, `//`, `/* */` | -| Perl | `.pl`, `.al`, `.cgi`, `.fcgi`, `.perl`, `.ph`, `.plx`, `.pm`, `.psgi`, `.t` | `#`, `= =cut` | -| PowerShell | `.ps1`, `.psd1`, `.psm1` | `#`, `<# #>` | -| Puppet | `.pp` | `#` | -| Python | `.py`, `.cgi`, `.fcgi`, `.gyp`, `.gypi`, `.lmi`, `.py3`, `.pyde`, `.pyi`, `.pyp`, `.pyt`, `.pyw`, `.rpy`, `.spec`, `.tac`, `.wsgi`, `.xpy` | `#`, `""" """` | -| R | `.r`, `.rd`, `.rsx` | `#` | -| Ruby | `.rb`, `.builder`, `.eye`, `.fcgi`, `.gemspec`, `.god`, `.jbuilder`, `.mspec`, `.pluginspec`, `.podspec`, `.prawn`, `.rabl`, `.rake`, `.rbi`, `.rbuild`, `.rbw`, `.rbx`, `.ru`, `.ruby`, `.spec`, `.thor`, `.watchr` | `#`, `=begin =end` | -| Rust | `.rs`, `.rs.in` | `//`, `/* */` | -| SQL | `.sql`, `.cql`, `.ddl`, `.inc`, `.mysql`, `.prc`, `.tab`, `.udf`, `.viw` | `--`, `/* */` | -| Scala | `.scala`, `.kojo`, `.sbt`, `.sc` | `//`, `/* */` | -| Shell | `.sh`, `.bash`, `.bats`, `.cgi`, `.command`, `.fcgi`, `.ksh`, `.sh.in`, `.tmux`, `.tool`, `.trigger`, `.zsh`, `.zsh-theme` | `#` | -| Swift | `.swift` | `//`, `/* */` | -| TOML | `.toml` | `#` | -| TeX | `.tex`, `.aux`, `.bbx`, `.cbx`, `.cls`, `.dtx`, `.ins`, `.lbx`, `.ltx`, `.mkii`, `.mkiv`, `.mkvi`, `.sty`, `.toc` | `%` | -| TypeScript | `.ts`, `.cts`, `.mts` | `//`, `/* */` | -| Unix Assembly | `.s`, `.ms` | `;`, `/* */` | -| VBA | `.bas`, `.cls`, `.frm`, `.vba` | `'` | -| Vim Script | `.vim`, `.vba`, `.vimrc`, `.vmb` | `"` | -| Visual Basic .NET | `.vb`, `.vbhtml` | `'` | -| XML | `.xml`, `.adml`, `.admx`, `.ant`, `.axaml`, `.axml`, `.builds`, `.ccproj`, `.ccxml`, `.clixml`, `.cproject`, `.cscfg`, `.csdef`, `.csl`, `.csproj`, `.ct`, `.depproj`, `.dita`, `.ditamap`, `.ditaval`, `.dll.config`, `.dotsettings`, `.filters`, `.fsproj`, `.fxml`, `.glade`, `.gml`, `.gmx`, `.grxml`, `.gst`, `.hzp`, `.iml`, `.ivy`, `.jelly`, `.jsproj`, `.kml`, `.launch`, `.mdpolicy`, `.mjml`, `.mm`, `.mod`, `.mojo`, `.mxml`, `.natvis`, `.ncl`, `.ndproj`, `.nproj`, `.nuspec`, `.odd`, `.osm`, `.pkgproj`, `.pluginspec`, `.proj`, `.props`, `.ps1xml`, `.psc1`, `.pt`, `.qhelp`, `.rdf`, `.res`, `.resx`, `.rs`, `.rss`, `.sch`, `.scxml`, `.sfproj`, `.shproj`, `.srdf`, `.storyboard`, `.sublime-snippet`, `.sw`, `.targets`, `.tml`, `.ts`, `.tsx`, `.typ`, `.ui`, `.urdf`, `.ux`, `.vbproj`, `.vcxproj`, `.vsixmanifest`, `.vssettings`, `.vstemplate`, `.vxml`, `.wixproj`, `.workflow`, `.wsdl`, `.wsf`, `.wxi`, `.wxl`, `.wxs`, `.x3d`, `.xacro`, `.xaml`, `.xib`, `.xlf`, `.xliff`, `.xmi`, `.xml.dist`, `.xmp`, `.xproj`, `.xsd`, `.xspec`, `.xul`, `.zcml` | `` | -| YAML | `.yml`, `.mir`, `.reek`, `.rviz`, `.sublime-syntax`, `.syntax`, `.yaml`, `.yaml-tmlanguage`, `.yaml.sed`, `.yml.mysql` | `#` | +| File type | Extension | Supported comments | +| ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------- | +| Assembly | `.asm`, `.a51`, `.i`, `.inc`, `.nas`, `.nasm` | `;`, `/* */` | +| C | `.c`, `.cats`, `.h`, `.idc` | `//`, `/* */` | +| C# | `.cs`, `.cake`, `.cs.pp`, `.csx`, `.linq` | `//`, `/* */` | +| C++ | `.cpp`, `.c++`, `.cc`, `.cp`, `.cppm`, `.cxx`, `.h`, `.h++`, `.hh`, `.hpp`, `.hxx`, `.inc`, `.inl`, `.ino`, `.ipp`, `.ixx`, `.re`, `.tcc`, `.tpp`, `.txx` | `//`, `/* */` | +| Clojure | `.clj`, `.bb`, `.boot`, `.cl2`, `.cljc`, `.cljs`, `.cljs.hl`, `.cljscm`, `.cljx`, `.hic` | `;` | +| CoffeeScript | `.coffee`, `._coffee`, `.cake`, `.cjsx`, `.iced` | `#`, `### ###` | +| Dockerfile | `.dockerfile` | `#` | +| Elixir | `.ex`, `.exs` | `#`, `@moduledoc """ """`, `@doc """ """` | +| Emacs Lisp | `.el`, `.emacs`, `.emacs.desktop` | `;` | +| Erlang | `.erl`, `.app`, `.app.src`, `.es`, `.escript`, `.hrl`, `.xrl`, `.yrl` | `%` | +| Fortran | `.f`, `.f77`, `.for`, `.fpp` | `!` | +| Fortran Free Form | `.f90`, `.f03`, `.f08`, `.f95` | `!` | +| Go | `.go` | `//`, `/* */` | +| Go Module | | `//` | +| Groovy | `.groovy`, `.grt`, `.gtpl`, `.gvy` | `//`, `/* */` | +| HTML | `.html`, `.hta`, `.htm`, `.html.hl`, `.inc`, `.xht`, `.xhtml` | `` | +| Haskell | `.hs`, `.hs-boot`, `.hsc` | `--`, `{- -}` | +| JSON | `.json`, `.4DForm`, `.4DProject`, `.avsc`, `.geojson`, `.gltf`, `.har`, `.ice`, `.JSON-tmLanguage`, `.jsonl`, `.mcmeta`, `.sarif`, `.tfstate`, `.tfstate.backup`, `.topojson`, `.webapp`, `.webmanifest`, `.yy`, `.yyp` | `//`, `#`, `/* */` | +| Java | `.java`, `.jav`, `.jsh` | `//`, `/* */` | +| JavaScript | `.js`, `._js`, `.bones`, `.cjs`, `.es`, `.es6`, `.frag`, `.gs`, `.jake`, `.javascript`, `.jsb`, `.jscad`, `.jsfl`, `.jslib`, `.jsm`, `.jspre`, `.jss`, `.jsx`, `.mjs`, `.njs`, `.pac`, `.sjs`, `.ssjs`, `.xsjs`, `.xsjslib` | `//`, `/* */` | +| Kotlin | `.kt`, `.ktm`, `.kts` | `//`, `/* */` | +| Lua | `.lua`, `.fcgi`, `.nse`, `.p8`, `.pd_lua`, `.rbxs`, `.rockspec`, `.wlua` | `--`, `--[[ --]]` | +| MATLAB | `.matlab`, `.m` | `%`, `%{ }%` | +| Makefile | `.mak`, `.d`, `.make`, `.makefile`, `.mk`, `.mkfile` | `#` | +| Objective-C | `.m`, `.h` | `//`, `/* */` | +| PHP | `.php`, `.aw`, `.ctp`, `.fcgi`, `.inc`, `.php3`, `.php4`, `.php5`, `.phps`, `.phpt` | `#`, `//`, `/* */` | +| Perl | `.pl`, `.al`, `.cgi`, `.fcgi`, `.perl`, `.ph`, `.plx`, `.pm`, `.psgi`, `.t` | `#`, `= =cut` | +| PowerShell | `.ps1`, `.psd1`, `.psm1` | `#`, `<# #>` | +| Puppet | `.pp` | `#` | +| Python | `.py`, `.cgi`, `.fcgi`, `.gyp`, `.gypi`, `.lmi`, `.py3`, `.pyde`, `.pyi`, `.pyp`, `.pyt`, `.pyw`, `.rpy`, `.spec`, `.tac`, `.wsgi`, `.xpy` | `#`, `""" """` | +| R | `.r`, `.rd`, `.rsx` | `#` | +| Ruby | `.rb`, `.builder`, `.eye`, `.fcgi`, `.gemspec`, `.god`, `.jbuilder`, `.mspec`, `.pluginspec`, `.podspec`, `.prawn`, `.rabl`, `.rake`, `.rbi`, `.rbuild`, `.rbw`, `.rbx`, `.ru`, `.ruby`, `.spec`, `.thor`, `.watchr` | `#`, `=begin =end` | +| Rust | `.rs`, `.rs.in` | `//`, `/* */` | +| SQL | `.sql`, `.cql`, `.ddl`, `.inc`, `.mysql`, `.prc`, `.tab`, `.udf`, `.viw` | `--`, `/* */` | +| Scala | `.scala`, `.kojo`, `.sbt`, `.sc` | `//`, `/* */` | +| Shell | `.sh`, `.bash`, `.bats`, `.cgi`, `.command`, `.fcgi`, `.ksh`, `.sh.in`, `.tmux`, `.tool`, `.trigger`, `.zsh`, `.zsh-theme` | `#` | +| Swift | `.swift` | `//`, `/* */` | +| TOML | `.toml` | `#` | +| TeX | `.tex`, `.aux`, `.bbx`, `.cbx`, `.cls`, `.dtx`, `.ins`, `.lbx`, `.ltx`, `.mkii`, `.mkiv`, `.mkvi`, `.sty`, `.toc` | `%` | +| TypeScript | `.ts`, `.cts`, `.mts` | `//`, `/* */` | +| Unix Assembly | `.s`, `.ms` | `;`, `/* */` | +| VBA | `.bas`, `.cls`, `.frm`, `.vba` | `'` | +| Vim Script | `.vim`, `.vba`, `.vimrc`, `.vmb` | `"` | +| Visual Basic .NET | `.vb`, `.vbhtml` | `'` | +| XML | `.xml`, `.adml`, `.admx`, `.ant`, `.axaml`, `.axml`, `.builds`, `.ccproj`, `.ccxml`, `.clixml`, `.cproject`, `.cscfg`, `.csdef`, `.csl`, `.csproj`, `.ct`, `.depproj`, `.dita`, `.ditamap`, `.ditaval`, `.dll.config`, `.dotsettings`, `.filters`, `.fsproj`, `.fxml`, `.glade`, `.gml`, `.gmx`, `.grxml`, `.gst`, `.hzp`, `.iml`, `.ivy`, `.jelly`, `.jsproj`, `.kml`, `.launch`, `.mdpolicy`, `.mjml`, `.mm`, `.mod`, `.mojo`, `.mxml`, `.natvis`, `.ncl`, `.ndproj`, `.nproj`, `.nuspec`, `.odd`, `.osm`, `.pkgproj`, `.pluginspec`, `.proj`, `.props`, `.ps1xml`, `.psc1`, `.pt`, `.qhelp`, `.rdf`, `.res`, `.resx`, `.rs`, `.rss`, `.sch`, `.scxml`, `.sfproj`, `.shproj`, `.srdf`, `.storyboard`, `.sublime-snippet`, `.sw`, `.targets`, `.tml`, `.ts`, `.tsx`, `.typ`, `.ui`, `.urdf`, `.ux`, `.vbproj`, `.vcxproj`, `.vsixmanifest`, `.vssettings`, `.vstemplate`, `.vxml`, `.wixproj`, `.workflow`, `.wsdl`, `.wsf`, `.wxi`, `.wxl`, `.wxs`, `.x3d`, `.xacro`, `.xaml`, `.xib`, `.xlf`, `.xliff`, `.xmi`, `.xml.dist`, `.xmp`, `.xproj`, `.xsd`, `.xspec`, `.xul`, `.zcml` | `` | +| YAML | `.yml`, `.mir`, `.reek`, `.rviz`, `.sublime-syntax`, `.syntax`, `.yaml`, `.yaml-tmlanguage`, `.yaml.sed`, `.yml.mysql` | `#` | diff --git a/internal/cmd/genlangdocs/main.go b/internal/cmd/genlangdocs/main.go index 536cf83..4541e96 100644 --- a/internal/cmd/genlangdocs/main.go +++ b/internal/cmd/genlangdocs/main.go @@ -71,11 +71,11 @@ func main() { for _, l := range langs { var supported []string - for _, c := range l.config.LineCommentStart { - supported = append(supported, fmt.Sprintf("`%s`", string(c))) + for _, c := range l.config.LineComments { + supported = append(supported, fmt.Sprintf("`%s`", string(c.Start))) } - if string(l.config.MultilineCommentStart) != "" { - s := fmt.Sprintf("`%s %s`", string(l.config.MultilineCommentStart), string(l.config.MultilineCommentEnd)) + for _, c := range l.config.MultilineComments { + s := fmt.Sprintf("`%s %s`", string(c.Start), string(c.End)) supported = append(supported, s) } diff --git a/internal/scanner/languages.go b/internal/scanner/languages.go index ca05a5d..774c0df 100644 --- a/internal/scanner/languages.go +++ b/internal/scanner/languages.go @@ -14,80 +14,111 @@ package scanner +// Common config. + +var ( + // sh-style languages. + + // hashLineComments are sh-style line comments. + hashLineComments = []LineCommentConfig{ + { + Start: []rune("#"), + }, + } + + // C-style languages. + + // cLineComments are C-style line comments. + cLineComments = []LineCommentConfig{ + { + Start: []rune("//"), + }, + } + + // cBlockComments are C-style block comments. + cBlockComments = []MultilineCommentConfig{ + { + Start: []rune("/*"), + End: []rune("*/"), + AtLineStart: false, + }, + } + + // cStrings are C-style strings. + cStrings = []StringConfig{ + // Strings + { + Start: []rune{'"'}, + End: []rune{'"'}, + EscapeFunc: CharEscape('\\'), + }, + // Characters + { + Start: []rune{'\''}, + End: []rune{'\''}, + EscapeFunc: CharEscape('\\'), + }, + } + + // XML-style languages. + + // xmlBlockComments are XML-style block comments. + xmlBlockComments = []MultilineCommentConfig{ + { + Start: []rune(""), + AtLineStart: false, + }, + } +) + var LanguagesConfig = map[string]*Config{ "Assembly": { - LineCommentStart: [][]rune{{';'}}, - MultilineCommentStart: []rune("/*"), - MultilineCommentEnd: []rune("*/"), - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ + LineComments: []LineCommentConfig{ { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: NoEscape, - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: NoEscape, + Start: []rune{';'}, }, }, - }, - "C": { - LineCommentStart: [][]rune{[]rune("//")}, - MultilineCommentStart: []rune("/*"), - MultilineCommentEnd: []rune("*/"), - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ + MultilineComments: []MultilineCommentConfig{ { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), + Start: []rune("/*"), + End: []rune("*/"), + AtLineStart: false, }, }, - }, - "C#": { - LineCommentStart: [][]rune{[]rune("//")}, - MultilineCommentStart: []rune("/*"), - MultilineCommentEnd: []rune("*/"), - MultilineCommentAtLineStart: false, + // NOTE: Assembly doesn't have string escape characters. Strings: []StringConfig{ { Start: []rune{'"'}, End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), + EscapeFunc: NoEscape, }, { Start: []rune{'\''}, End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), + EscapeFunc: NoEscape, }, }, }, + "C": { + LineComments: cLineComments, + MultilineComments: cBlockComments, + Strings: cStrings, + }, + "C#": { + LineComments: cLineComments, + MultilineComments: cBlockComments, + Strings: cStrings, + }, "C++": { - LineCommentStart: [][]rune{[]rune("//")}, - MultilineCommentStart: []rune("/*"), - MultilineCommentEnd: []rune("*/"), - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ - { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), - }, - }, + LineComments: cLineComments, + MultilineComments: cBlockComments, + Strings: cStrings, }, "Clojure": { - LineCommentStart: [][]rune{{';'}}, - MultilineCommentStart: nil, - MultilineCommentEnd: nil, - MultilineCommentAtLineStart: false, + LineComments: []LineCommentConfig{ + {Start: []rune{';'}}, + }, + MultilineComments: nil, Strings: []StringConfig{ { Start: []rune{'"'}, @@ -97,46 +128,33 @@ var LanguagesConfig = map[string]*Config{ }, }, "CoffeeScript": { - LineCommentStart: [][]rune{{'#'}}, - MultilineCommentStart: []rune("###"), - MultilineCommentEnd: []rune("###"), - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ - { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), - }, + LineComments: hashLineComments, + MultilineComments: []MultilineCommentConfig{ + {Start: []rune("###"), End: []rune("###"), AtLineStart: false}, }, + Strings: cStrings, }, "Dockerfile": { - LineCommentStart: [][]rune{{'#'}}, - MultilineCommentStart: nil, - MultilineCommentEnd: nil, - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ - { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), - }, - }, + LineComments: hashLineComments, + MultilineComments: nil, + Strings: cStrings, }, "Elixir": { - LineCommentStart: [][]rune{{'#'}}, + LineComments: hashLineComments, // Support function documentation. // TODO(#1546): Support @moduledoc - MultilineCommentStart: []rune("@doc \"\"\""), - MultilineCommentEnd: []rune("\"\"\""), - MultilineCommentAtLineStart: false, + MultilineComments: []MultilineCommentConfig{ + { + Start: []rune("@moduledoc \"\"\""), + End: []rune("\"\"\""), + AtLineStart: false, + }, + { + Start: []rune("@doc \"\"\""), + End: []rune("\"\"\""), + AtLineStart: false, + }, + }, Strings: []StringConfig{ { Start: []rune{'"'}, @@ -158,10 +176,10 @@ var LanguagesConfig = map[string]*Config{ }, }, "Emacs Lisp": { - LineCommentStart: [][]rune{{';'}}, - MultilineCommentStart: nil, - MultilineCommentEnd: nil, - MultilineCommentAtLineStart: false, + LineComments: []LineCommentConfig{ + {Start: []rune{';'}}, + }, + MultilineComments: nil, Strings: []StringConfig{ { Start: []rune{'"'}, @@ -171,27 +189,17 @@ var LanguagesConfig = map[string]*Config{ }, }, "Erlang": { - LineCommentStart: [][]rune{{'%'}}, - MultilineCommentStart: nil, - MultilineCommentEnd: nil, - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ - { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), - }, + LineComments: []LineCommentConfig{ + {Start: []rune{'%'}}, }, + MultilineComments: nil, + Strings: cStrings, }, "Fortran": { - LineCommentStart: [][]rune{{'!'}}, - MultilineCommentStart: nil, - MultilineCommentEnd: nil, - MultilineCommentAtLineStart: false, + LineComments: []LineCommentConfig{ + {Start: []rune{'!'}}, + }, + MultilineComments: nil, Strings: []StringConfig{ { Start: []rune{'"'}, @@ -205,10 +213,10 @@ var LanguagesConfig = map[string]*Config{ }, }, "Fortran Free Form": { - LineCommentStart: [][]rune{{'!'}}, - MultilineCommentStart: nil, - MultilineCommentEnd: nil, - MultilineCommentAtLineStart: false, + LineComments: []LineCommentConfig{ + {Start: []rune{'!'}}, + }, + MultilineComments: nil, Strings: []StringConfig{ { Start: []rune{'"'}, @@ -222,20 +230,21 @@ var LanguagesConfig = map[string]*Config{ }, }, "Go": { - LineCommentStart: [][]rune{[]rune("//")}, - MultilineCommentStart: []rune("/*"), - MultilineCommentEnd: []rune("*/"), - MultilineCommentAtLineStart: false, + LineComments: cLineComments, + MultilineComments: cBlockComments, Strings: []StringConfig{ { Start: []rune{'"'}, End: []rune{'"'}, EscapeFunc: CharEscape('\\'), - }, { + }, + { Start: []rune{'\''}, End: []rune{'\''}, EscapeFunc: CharEscape('\\'), - }, { + }, + // Go raw strings + { Start: []rune{'`'}, End: []rune{'`'}, EscapeFunc: NoEscape, @@ -243,16 +252,17 @@ var LanguagesConfig = map[string]*Config{ }, }, "Go Module": { - LineCommentStart: [][]rune{[]rune("//")}, - MultilineCommentStart: nil, - MultilineCommentEnd: nil, - MultilineCommentAtLineStart: false, + LineComments: cLineComments, + MultilineComments: nil, Strings: []StringConfig{ { Start: []rune{'"'}, End: []rune{'"'}, EscapeFunc: CharEscape('\\'), - }, { + }, + // NOTE: Characters are not supported. + // Go raw strings + { Start: []rune{'`'}, End: []rune{'`'}, EscapeFunc: NoEscape, @@ -260,10 +270,8 @@ var LanguagesConfig = map[string]*Config{ }, }, "Groovy": { - LineCommentStart: [][]rune{[]rune("//")}, - MultilineCommentStart: []rune("/*"), - MultilineCommentEnd: []rune("*/"), - MultilineCommentAtLineStart: false, + LineComments: cLineComments, + MultilineComments: cBlockComments, Strings: []StringConfig{ { Start: []rune{'"'}, @@ -281,220 +289,126 @@ var LanguagesConfig = map[string]*Config{ }, }, "HTML": { - LineCommentStart: nil, - MultilineCommentStart: []rune(""), - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ + LineComments: nil, + MultilineComments: xmlBlockComments, + Strings: cStrings, + }, + "Haskell": { + LineComments: []LineCommentConfig{ { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), + Start: []rune("--"), }, }, - }, - "Haskell": { - LineCommentStart: [][]rune{[]rune("--")}, - MultilineCommentStart: []rune("{-"), - MultilineCommentEnd: []rune("-}"), - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ + MultilineComments: []MultilineCommentConfig{ { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), + Start: []rune("{-"), + End: []rune("-}"), + AtLineStart: false, }, }, + Strings: cStrings, }, "JSON": { - LineCommentStart: [][]rune{ - []rune("//"), - {'#'}, - }, - MultilineCommentStart: []rune("/*"), - MultilineCommentEnd: []rune("*/"), - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ + // NOTE: Some JSON parsers support comments. + LineComments: []LineCommentConfig{ { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), + Start: []rune("//"), }, - }, - }, - "Java": { - LineCommentStart: [][]rune{[]rune("//")}, - MultilineCommentStart: []rune("/*"), - MultilineCommentEnd: []rune("*/"), - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), + Start: []rune{'#'}, }, }, + MultilineComments: cBlockComments, + Strings: cStrings, + }, + "Java": { + LineComments: cLineComments, + MultilineComments: cBlockComments, + Strings: cStrings, }, "JavaScript": { - LineCommentStart: [][]rune{[]rune("//")}, - MultilineCommentStart: []rune("/*"), - MultilineCommentEnd: []rune("*/"), - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ - { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), - }, - }, + LineComments: cLineComments, + MultilineComments: cBlockComments, + Strings: cStrings, }, "Kotlin": { - LineCommentStart: [][]rune{[]rune("//")}, - MultilineCommentStart: []rune("/*"), - MultilineCommentEnd: []rune("*/"), - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ + LineComments: cLineComments, + MultilineComments: cBlockComments, + Strings: cStrings, + }, + "Lua": { + LineComments: []LineCommentConfig{ { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), + Start: []rune("--"), }, }, - }, - "Lua": { - LineCommentStart: [][]rune{[]rune("--")}, - MultilineCommentStart: []rune("--[["), - MultilineCommentEnd: []rune("--]]"), - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ + MultilineComments: []MultilineCommentConfig{ { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), + Start: []rune("--[["), + End: []rune("--]]"), + AtLineStart: false, }, }, + Strings: cStrings, }, "MATLAB": { - LineCommentStart: [][]rune{{'%'}}, - MultilineCommentStart: []rune("%{"), - MultilineCommentEnd: []rune("}%"), - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ + LineComments: []LineCommentConfig{ { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), + Start: []rune{'%'}, }, }, - }, - "Makefile": { - LineCommentStart: [][]rune{{'#'}}, - MultilineCommentStart: nil, - MultilineCommentEnd: nil, - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ + MultilineComments: []MultilineCommentConfig{ { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), + Start: []rune("%{"), + End: []rune("}%"), + AtLineStart: false, }, }, + Strings: cStrings, + }, + "Makefile": { + LineComments: hashLineComments, + MultilineComments: nil, + Strings: cStrings, }, "Objective-C": { - LineCommentStart: [][]rune{[]rune("//")}, - MultilineCommentStart: []rune("/*"), - MultilineCommentEnd: []rune("*/"), - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ - { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), - }, - }, + LineComments: cLineComments, + MultilineComments: cBlockComments, + Strings: cStrings, }, "PHP": { - LineCommentStart: [][]rune{ - {'#'}, - []rune("//"), - }, - MultilineCommentStart: []rune("/*"), - MultilineCommentEnd: []rune("*/"), - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ + LineComments: []LineCommentConfig{ { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), + Start: []rune{'#'}, + }, + { + Start: []rune("//"), }, }, + MultilineComments: cBlockComments, + Strings: cStrings, }, "Perl": { - LineCommentStart: [][]rune{{'#'}}, - MultilineCommentStart: []rune{'='}, - MultilineCommentEnd: []rune("=cut"), - MultilineCommentAtLineStart: true, - Strings: []StringConfig{ + LineComments: hashLineComments, + MultilineComments: []MultilineCommentConfig{ { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), + Start: []rune{'='}, + End: []rune("=cut"), + AtLineStart: true, }, }, + Strings: cStrings, }, "PowerShell": { - LineCommentStart: [][]rune{{'#'}}, - MultilineCommentStart: []rune("<#"), - MultilineCommentEnd: []rune("#>"), - MultilineCommentAtLineStart: false, + LineComments: hashLineComments, + MultilineComments: []MultilineCommentConfig{ + { + Start: []rune("<#"), + End: []rune("#>"), + AtLineStart: false, + }, + }, + // NOTE: Powershell escape character is the grave character (`) Strings: []StringConfig{ { Start: []rune{'"'}, @@ -508,61 +422,35 @@ var LanguagesConfig = map[string]*Config{ }, }, "Puppet": { - LineCommentStart: [][]rune{{'#'}}, - MultilineCommentStart: nil, - MultilineCommentEnd: nil, - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ - { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), - }, - }, + LineComments: hashLineComments, + MultilineComments: nil, + Strings: cStrings, }, "Python": { - LineCommentStart: [][]rune{{'#'}}, - MultilineCommentStart: []rune("\"\"\""), - MultilineCommentEnd: []rune("\"\"\""), - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ + LineComments: hashLineComments, + MultilineComments: []MultilineCommentConfig{ { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), + Start: []rune("\"\"\""), + End: []rune("\"\"\""), + AtLineStart: false, }, }, + Strings: cStrings, }, "R": { - LineCommentStart: [][]rune{{'#'}}, - MultilineCommentStart: nil, - MultilineCommentEnd: nil, - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ + LineComments: hashLineComments, + MultilineComments: nil, + Strings: cStrings, + }, + "Ruby": { + LineComments: hashLineComments, + MultilineComments: []MultilineCommentConfig{ { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), + Start: []rune("=begin"), + End: []rune("=end"), + AtLineStart: true, }, }, - }, - "Ruby": { - LineCommentStart: [][]rune{{'#'}}, - MultilineCommentStart: []rune("=begin"), - MultilineCommentEnd: []rune("=end"), - MultilineCommentAtLineStart: true, Strings: []StringConfig{ { Start: []rune{'"'}, @@ -580,27 +468,17 @@ var LanguagesConfig = map[string]*Config{ }, }, "Rust": { - LineCommentStart: [][]rune{[]rune("//")}, - MultilineCommentStart: []rune("/*"), - MultilineCommentEnd: []rune("*/"), - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ + LineComments: cLineComments, + MultilineComments: cBlockComments, + Strings: cStrings, + }, + "SQL": { + LineComments: []LineCommentConfig{ { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), + Start: []rune("--"), }, }, - }, - "SQL": { - LineCommentStart: [][]rune{[]rune("--")}, - MultilineCommentStart: []rune("/*"), - MultilineCommentEnd: []rune("*/"), - MultilineCommentAtLineStart: false, + MultilineComments: cBlockComments, Strings: []StringConfig{ { Start: []rune{'"'}, @@ -614,44 +492,18 @@ var LanguagesConfig = map[string]*Config{ }, }, "Scala": { - LineCommentStart: [][]rune{[]rune("//")}, - MultilineCommentStart: []rune("/*"), - MultilineCommentEnd: []rune("*/"), - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ - { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), - }, - }, + LineComments: cLineComments, + MultilineComments: cBlockComments, + Strings: cStrings, }, "Shell": { - LineCommentStart: [][]rune{{'#'}}, - MultilineCommentStart: nil, - MultilineCommentEnd: nil, - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ - { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), - }, - }, + LineComments: hashLineComments, + MultilineComments: nil, + Strings: cStrings, }, "Swift": { - LineCommentStart: [][]rune{[]rune("//")}, - MultilineCommentStart: []rune("/*"), - MultilineCommentEnd: []rune("*/"), - MultilineCommentAtLineStart: false, + LineComments: cLineComments, + MultilineComments: cBlockComments, Strings: []StringConfig{ { Start: []rune{'"'}, @@ -661,51 +513,32 @@ var LanguagesConfig = map[string]*Config{ }, }, "TOML": { - LineCommentStart: [][]rune{{'#'}}, - MultilineCommentStart: nil, - MultilineCommentEnd: nil, - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ + LineComments: hashLineComments, + MultilineComments: nil, + Strings: cStrings, + }, + "TeX": { + LineComments: []LineCommentConfig{ { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), + Start: []rune{'%'}, }, }, - }, - "TeX": { - LineCommentStart: [][]rune{{'%'}}, - MultilineCommentStart: nil, - MultilineCommentEnd: nil, - MultilineCommentAtLineStart: false, - Strings: nil, + MultilineComments: nil, + Strings: nil, }, "TypeScript": { - LineCommentStart: [][]rune{[]rune("//")}, - MultilineCommentStart: []rune("/*"), - MultilineCommentEnd: []rune("*/"), - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ + LineComments: cLineComments, + MultilineComments: cBlockComments, + Strings: cStrings, + }, + "Unix Assembly": { + LineComments: []LineCommentConfig{ { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), + Start: []rune{';'}, }, }, - }, - "Unix Assembly": { - LineCommentStart: [][]rune{{';'}}, - MultilineCommentStart: []rune("/*"), - MultilineCommentEnd: []rune("*/"), - MultilineCommentAtLineStart: false, + MultilineComments: cBlockComments, + // NOTE: Assembly doesn't have string escape characters. Strings: []StringConfig{ { Start: []rune{'"'}, @@ -719,10 +552,12 @@ var LanguagesConfig = map[string]*Config{ }, }, "VBA": { - LineCommentStart: [][]rune{{'\''}}, - MultilineCommentStart: nil, - MultilineCommentEnd: nil, - MultilineCommentAtLineStart: false, + LineComments: []LineCommentConfig{ + { + Start: []rune{'\''}, + }, + }, + MultilineComments: nil, Strings: []StringConfig{ { Start: []rune{'"'}, @@ -732,67 +567,37 @@ var LanguagesConfig = map[string]*Config{ }, }, "Vim Script": { - LineCommentStart: [][]rune{{'"'}}, - MultilineCommentStart: nil, - MultilineCommentEnd: nil, - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ + LineComments: []LineCommentConfig{ { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), + Start: []rune{'"'}, }, }, + MultilineComments: nil, + Strings: cStrings, }, "Visual Basic .NET": { - LineCommentStart: [][]rune{{'\''}}, - MultilineCommentStart: nil, - MultilineCommentEnd: nil, - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ + LineComments: []LineCommentConfig{ { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), + Start: []rune{'\''}, }, }, - }, - "XML": { - LineCommentStart: nil, - MultilineCommentStart: []rune(""), - MultilineCommentAtLineStart: false, + MultilineComments: nil, Strings: []StringConfig{ { Start: []rune{'"'}, End: []rune{'"'}, EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), }, }, }, + "XML": { + LineComments: nil, + MultilineComments: xmlBlockComments, + Strings: cStrings, + }, "YAML": { - LineCommentStart: [][]rune{{'#'}}, - MultilineCommentStart: nil, - MultilineCommentEnd: nil, - MultilineCommentAtLineStart: false, - Strings: []StringConfig{ - { - Start: []rune{'"'}, - End: []rune{'"'}, - EscapeFunc: CharEscape('\\'), - }, { - Start: []rune{'\''}, - End: []rune{'\''}, - EscapeFunc: CharEscape('\\'), - }, - }, + LineComments: hashLineComments, + MultilineComments: nil, + Strings: cStrings, }, } diff --git a/internal/scanner/scanner.go b/internal/scanner/scanner.go index 612b999..f1e194c 100644 --- a/internal/scanner/scanner.go +++ b/internal/scanner/scanner.go @@ -45,13 +45,28 @@ type StringConfig struct { EscapeFunc EscapeFunc } +type LineCommentConfig struct { + // Start is the starting sequence for the line comment. + Start []rune +} + +type MultilineCommentConfig struct { + // Start is the starting sequence for the multiline comment. + Start []rune + + // End is the ending sequence for the multiline comment. + End []rune + + // AtLineStart indicates that the multiline comment must start at the + // beginning of a line. + AtLineStart bool +} + // Config is configuration for a generic comment scanner. type Config struct { - LineCommentStart [][]rune - MultilineCommentStart []rune - MultilineCommentEnd []rune - MultilineCommentAtLineStart bool - Strings []StringConfig + LineComments []LineCommentConfig + MultilineComments []MultilineCommentConfig + Strings []StringConfig } // FromFile returns an appropriate CommentScanner for the given file. The @@ -215,15 +230,18 @@ func (s *CommentScanner) processCode(st *stateCode) (state, error) { return st, err } - mm, err := s.multiLineMatch() + mmIndex, mm, err := s.multiLineMatch() if err != nil { return st, err } - if len(m) > 0 || len(mm) > 0 { - if len(m) >= len(mm) { + // Check for line comments. + if m != nil { + // If both line comments and multi-line comments match, chose the + // one with the longest start sequence. + if mm == nil || len(m.Start) >= len(mm.Start) { for i, stringStart := range s.config.Strings { - if string(stringStart.Start) == string(m) { + if string(stringStart.Start) == string(m.Start) { return &stateLineCommentOrString{ index: i, }, nil @@ -232,15 +250,19 @@ func (s *CommentScanner) processCode(st *stateCode) (state, error) { return &stateLineComment{}, nil } + } - if !s.config.MultilineCommentAtLineStart || s.atLineStart { + // Check for multi-line comments. + if mm != nil { + if !mm.AtLineStart || s.atLineStart { return &stateMultilineComment{ - line: s.line, + line: s.line, + index: mmIndex, }, nil } } - // Check for string + // Check for strings. for i, strs := range s.config.Strings { eq, err := s.peekEqual(strs.Start) if err != nil { @@ -260,30 +282,30 @@ func (s *CommentScanner) processCode(st *stateCode) (state, error) { } } -func (s *CommentScanner) lineMatch() ([]rune, error) { +func (s *CommentScanner) lineMatch() (*LineCommentConfig, error) { // Check for line comment - for _, start := range s.config.LineCommentStart { - eq, err := s.peekEqual(start) + for _, m := range s.config.LineComments { + eq, err := s.peekEqual(m.Start) if err != nil { return nil, err } if eq { - return start, nil + return &m, nil } } return nil, nil } -func (s *CommentScanner) multiLineMatch() ([]rune, error) { - // Check for line comment - if len(s.config.MultilineCommentStart) != 0 { - if eq, err := s.peekEqual(s.config.MultilineCommentStart); err == nil && eq { - return s.config.MultilineCommentStart, nil +func (s *CommentScanner) multiLineMatch() (int, *MultilineCommentConfig, error) { + // Check for multiline comment + for i, mlConfig := range s.config.MultilineComments { + if eq, err := s.peekEqual(mlConfig.Start); err == nil && eq { + return i, &mlConfig, nil } else if err != nil { - return nil, err + return 0, nil, err } } - return nil, nil + return 0, nil, nil } // processString processes strings and returns the next state. @@ -430,26 +452,29 @@ func (s *CommentScanner) processLineCommentOrString(st *stateLineCommentOrString // processMultilineComment processes multi-line comments and returns the next state. func (s *CommentScanner) processMultilineComment(st *stateMultilineComment) (state, error) { + mm := s.config.MultilineComments[st.index] + // Discard the opening since we don't want to parse it. It could be the same as the closing. - if _, errDiscard := s.reader.Discard(len(s.config.MultilineCommentStart)); errDiscard != nil { + if _, errDiscard := s.reader.Discard(len(mm.Start)); errDiscard != nil { return st, fmt.Errorf("parsing code: %w", errDiscard) } var b strings.Builder + // Add the opening to the builder since we want it in the output. - b.WriteString(string(s.config.MultilineCommentStart)) + b.WriteString(string(mm.Start)) for { // Look for the end of the comment. - mlEnd, err := s.peekEqual(s.config.MultilineCommentEnd) + mlEnd, err := s.peekEqual(mm.End) if err != nil { return st, err } - if mlEnd && (!s.config.MultilineCommentAtLineStart || s.atLineStart) { - if _, errDiscard := s.reader.Discard(len(s.config.MultilineCommentEnd)); errDiscard != nil { + if mlEnd && (!mm.AtLineStart || s.atLineStart) { + if _, errDiscard := s.reader.Discard(len(mm.End)); errDiscard != nil { return st, fmt.Errorf("parsing multi-line comment: %w", errDiscard) } // Add the ending to the builder. - b.WriteString(string(s.config.MultilineCommentEnd)) + b.WriteString(string(mm.End)) s.next = &Comment{ Text: b.String(), Line: st.line, diff --git a/internal/scanner/scanner_test.go b/internal/scanner/scanner_test.go index bc312ea..2fb1659 100644 --- a/internal/scanner/scanner_test.go +++ b/internal/scanner/scanner_test.go @@ -458,6 +458,27 @@ var scannerTestCases = []*struct { // NOTE: No '# Random comment' }, }, + { + name: "moduledoc_comment.ex", + src: `@moduledoc """ + module comment + """ + defmodule Math do + def TODO(a, b) do + a + b + end + end`, + config: "Elixir", + comments: []struct { + text string + line int + }{ + { + text: "@moduledoc \"\"\"\n\t\t\tmodule comment\n\t\t\t\"\"\"", + line: 1, + }, + }, + }, { name: "doc_comment.ex", src: `# module comment @@ -2514,7 +2535,7 @@ var scannerRegressionTestCases = []*struct { name: "last_line.go", src: `// last line`, config: &Config{ - LineCommentStart: [][]rune{[]rune("//")}, + LineComments: cLineComments, }, expectedComments: []*Comment{ { @@ -2527,7 +2548,11 @@ var scannerRegressionTestCases = []*struct { name: "double_escape_1538.foo", src: `x = '''''' % foo''`, config: &Config{ - LineCommentStart: [][]rune{[]rune("%")}, + LineComments: []LineCommentConfig{ + { + Start: []rune("%"), + }, + }, Strings: []StringConfig{ { Start: []rune("''"), diff --git a/internal/scanner/state.go b/internal/scanner/state.go index 4f2df7e..7127855 100644 --- a/internal/scanner/state.go +++ b/internal/scanner/state.go @@ -29,6 +29,9 @@ func (s *stateLineComment) stateMustImplement() {} type stateMultilineComment struct { // line is the line of the start of the multi-line comment. line int + + // index is the index for the type of multiline comment. + index int } func (s *stateMultilineComment) stateMustImplement() {} diff --git a/internal/todos/todos.go b/internal/todos/todos.go index 273682a..abf08f6 100644 --- a/internal/todos/todos.go +++ b/internal/todos/todos.go @@ -99,12 +99,16 @@ func NewTODOScanner(s CommentScanner, config *Config) *TODOScanner { sConfig := s.Config() var commentStarts []string - for _, lineCommentStart := range sConfig.LineCommentStart { - commentStarts = append(commentStarts, "(?:"+regexp.QuoteMeta(string(lineCommentStart))+")+") + for _, c := range sConfig.LineComments { + commentStarts = append(commentStarts, "(?:"+regexp.QuoteMeta(string(c.Start))+")+") } commentStartMatch := strings.Join(commentStarts, "|") - multiStartMatch := regexp.QuoteMeta(string(sConfig.MultilineCommentStart)) + var multilineStarts []string + for _, c := range sConfig.MultilineComments { + multilineStarts = append(multilineStarts, "(?:"+regexp.QuoteMeta(string(c.Start))+")+") + } + multiStartMatch := strings.Join(multilineStarts, "|") if config == nil { config = &Config{ diff --git a/internal/todos/todos_test.go b/internal/todos/todos_test.go index c8421dd..efaf389 100644 --- a/internal/todos/todos_test.go +++ b/internal/todos/todos_test.go @@ -30,8 +30,16 @@ type testScanner struct { func (s *testScanner) Config() *scanner.Config { return &scanner.Config{ - LineCommentStart: [][]rune{[]rune("//")}, - MultilineCommentStart: []rune("/*"), + LineComments: []scanner.LineCommentConfig{ + { + Start: []rune("//"), + }, + }, + MultilineComments: []scanner.MultilineCommentConfig{ + { + Start: []rune("/*"), + }, + }, } }