Skip to content

Commit

Permalink
parse: test all filtering scenarios, with and without context
Browse files Browse the repository at this point in the history
  • Loading branch information
jrockway committed Jun 23, 2022
1 parent bf78e5e commit 7ed8fe3
Show file tree
Hide file tree
Showing 8 changed files with 675 additions and 157 deletions.
22 changes: 6 additions & 16 deletions cmd/jlog/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"io"
"os"
"os/signal"
"regexp"
"runtime/debug"
"runtime/pprof"
"strings"
Expand Down Expand Up @@ -172,23 +171,14 @@ func main() {
fmt.Fprintf(os.Stderr, "cannot have both a non-empty MatchRegex and a non-empty NoMatchRegex\n")
os.Exit(1)
}
if rx := gen.MatchRegex; rx != "" {
regex, err := regexp.Compile(rx)
if err != nil {
fmt.Fprintf(os.Stderr, "problem compiling MatchRegex: %v\n", err)
os.Exit(1)
}
fsch.MatchRegex = regex
if err := fsch.AddMatchRegex(gen.MatchRegex); err != nil {
fmt.Fprintf(os.Stderr, "problem compiling MatchRegex: %v\n", err)
os.Exit(1)
}
if rx := gen.NoMatchRegex; rx != "" {
regex, err := regexp.Compile(rx)
if err != nil {
fmt.Fprintf(os.Stderr, "problem compiling NoMatchRegex: %v\n", err)
os.Exit(1)
}
fsch.NoMatchRegex = regex
if err := fsch.AddNoMatchRegex(gen.NoMatchRegex); err != nil {
fmt.Fprintf(os.Stderr, "problem compiling NoMatchRegex: %v\n", err)
os.Exit(1)
}

if err := fsch.AddJQ(gen.JQ); err != nil {
fmt.Fprintf(os.Stderr, "problem %v\n", err)
os.Exit(1)
Expand Down
3 changes: 1 addition & 2 deletions pkg/parse/context.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,8 @@ func (c *context) Print(msg *line, selected bool) []*line {
c.lastPrint != 0 &&
// suppress separator if we are no-op context
(c.After != 0 || c.Before != 0) &&
// suppress separator if end of after is contigious with the start of before
// suppress separator if end of after is contiguous with the start of before
c.line-len(c.lines)-c.lastPrint > 1 {

result = append(result, &line{isSeparator: true})
}
for _, l := range c.lines {
Expand Down
4 changes: 2 additions & 2 deletions pkg/parse/context_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,8 @@ func TestContext(t *testing.T) {
l.reset()
l.msg = msg
selected := test.match.MatchString(msg)
print := ctx.Print(&l, selected)
for _, x := range print {
toEmit := ctx.Print(&l, selected)
for _, x := range toEmit {
if x.isSeparator {
out.WriteString("---")
} else {
Expand Down
1 change: 1 addition & 0 deletions pkg/parse/default_parsers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ func TestLevelParsers(t *testing.T) {
}

func TestNoopParsers(t *testing.T) {
// nolint: errcheck
testData := []func(){func() { NoopTimeParser(1) }, func() { NoopLevelParser("info") }}
for _, test := range testData {
ok := func() (ok bool) {
Expand Down
64 changes: 55 additions & 9 deletions pkg/parse/filter.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,6 @@ func (f *FilterScheme) runJQ(l *line) (bool, error) {
}
} else {
filtered = true
l.fields = make(map[string]interface{})
}
return filtered, nil
}
Expand All @@ -120,6 +119,7 @@ func runRegexp(rx *regexp.Regexp, l *line, scope regexpScope) bool {
case regexpScopeMessage:
input = l.msg
}

fields := rx.FindStringSubmatch(input)
if len(fields) == 0 {
return false
Expand All @@ -139,21 +139,67 @@ func runRegexp(rx *regexp.Regexp, l *line, scope regexpScope) bool {
// Run runs all the filters defined in this FilterScheme against the provided line. The return
// value is true if the line should be removed from the output ("filtered").
func (f *FilterScheme) Run(l *line) (bool, error) {
rxFiltered := false
if rx := f.NoMatchRegex; rx != nil {
found := runRegexp(rx, l, regexpScopeMessage)
if found {
return true, nil
if found := runRegexp(rx, l, regexpScopeMessage); found {
rxFiltered = true
}
}
if rx := f.MatchRegex; rx != nil {
found := runRegexp(rx, l, regexpScopeMessage)
if !found {
return true, nil
if found := runRegexp(rx, l, regexpScopeMessage); !found {
rxFiltered = true
}
}
filtered, err := f.runJQ(l)
jqFiltered, err := f.runJQ(l)
if err != nil {
return false, fmt.Errorf("jq: %w", err)
}
return filtered, nil
return rxFiltered || jqFiltered, nil
}

var (
ErrAlreadyAdded = errors.New("regex already added")
ErrConflict = errors.New("attempt to add regex when a conflicting regex has already been added")
)

// Add a MatchRegex to this filter scheme. A MatchRegex filters out all lines that do not match it.
// An empty string is a no-op. This method may only be called with a non-empty string once, and
// returns an ErrConflict if a NoMatchRegex is set.
func (f *FilterScheme) AddMatchRegex(rx string) error {
if rx == "" {
return nil
}
if f.MatchRegex != nil {
return ErrAlreadyAdded
}
if f.NoMatchRegex != nil {
return ErrConflict
}
var err error
f.MatchRegex, err = regexp.Compile(rx)
if err != nil {
return fmt.Errorf("compile regex: %w", err)
}
return nil
}

// Add a NoMatchRegex to this filter scheme. A NoMatchRegex filters out all lines that match it.
// An empty string is a no-op. This method may only be called with a non-empty string once, and
// returns an ErrConflict if a MatchRegex is set.
func (f *FilterScheme) AddNoMatchRegex(rx string) error {
if rx == "" {
return nil
}
if f.NoMatchRegex != nil {
return ErrAlreadyAdded
}
if f.MatchRegex != nil {
return ErrConflict
}
var err error
f.NoMatchRegex, err = regexp.Compile(rx)
if err != nil {
return fmt.Errorf("compile: %w", err)
}
return nil
}
221 changes: 221 additions & 0 deletions pkg/parse/filter_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,221 @@
package parse

import (
"testing"

"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
)

func TestJQ(t *testing.T) {
referenceLine := func() *line { return &line{msg: "foo", fields: map[string]interface{}{"foo": 42, "bar": "hi"}} }
testData := []struct {
jq string
l *line
wantLine *line
wantFiltered bool
wantErr error
}{
{
jq: ".",
l: referenceLine(),
wantLine: referenceLine(),
wantFiltered: false,
wantErr: nil,
},
{
jq: "",
l: referenceLine(),
wantLine: referenceLine(),
wantFiltered: false,
wantErr: nil,
},
{
jq: `error("goodbye")`,
l: referenceLine(),
wantLine: referenceLine(),
wantFiltered: false,
wantErr: Match("goodbye"),
},
{
jq: "null",
l: referenceLine(),
wantLine: referenceLine(),
wantFiltered: false,
wantErr: Match("unexpected nil result"),
},
{
jq: "3.141592",
l: referenceLine(),
wantLine: referenceLine(),
wantFiltered: false,
wantErr: Match("unexpected result type float64\\(3.1"),
},
{
jq: "1 > 2",
l: referenceLine(),
wantLine: referenceLine(),
wantFiltered: false,
wantErr: Match("unexpected boolean output"),
},
{
jq: "{}",
l: referenceLine(),
wantLine: &line{msg: "foo"},
wantFiltered: false,
wantErr: nil,
},
{
jq: "{}, {}",
l: referenceLine(),
wantLine: &line{msg: "foo"},
wantFiltered: false,
wantErr: Match("unexpectedly produced more than 1 output"),
},
{
jq: "empty",
l: referenceLine(),
wantLine: referenceLine(),
wantFiltered: true,
wantErr: nil,
},
}
for _, test := range testData {
fs := new(FilterScheme)
if err := fs.AddJQ(test.jq); err != nil {
t.Fatal(err)
}
gotFiltered, gotErr := fs.runJQ(test.l)
if diff := cmp.Diff(test.l, test.wantLine, cmp.AllowUnexported(line{}), cmpopts.EquateEmpty()); diff != "" {
t.Errorf("line: %s", diff)
}
if got, want := gotFiltered, test.wantFiltered; got != want {
t.Errorf("filtered:\n got: %v\n want: %v", got, want)
}
if got, want := gotErr, test.wantErr; !comperror(got, want) {
t.Errorf("error:\n got: %v\n want: %v", got, want)
}
}
}

func TestAdds(t *testing.T) {
testData := []struct {
name string
match, matchagain, nomatch, jq []string
want []error
}{
{
name: "empty",
},
{
name: "valid match",
match: []string{"foo"},
},
{
name: "valid nomatch",
nomatch: []string{"foo"},
},
{
name: "valid jq",
jq: []string{"{}"},
},
{
name: "valid match and jq",
match: []string{"foo"},
jq: []string{"{}"},
},
{
name: "valid nomatch and jq",
nomatch: []string{"foo"},
jq: []string{"{}"},
},
{
name: "unparseable jq",
jq: []string{"{"},
want: []error{Match("EOF")},
},
{
name: "uncompilable jq",
jq: []string{"$INVALID"},
want: []error{Match("variable not defined")},
},
{
name: "double jq",
jq: []string{".", "."},
want: []error{Match("already added")},
},
{
name: "invalid match",
match: []string{"["},
want: []error{Match("missing closing ]")},
},
{
name: "invalid nomatch",
nomatch: []string{"["},
want: []error{Match("missing closing ]")},
},
{
name: "invalid regexes",
match: []string{"["},
nomatch: []string{"["},
want: []error{Match("missing closing ]"), Match("missing closing ]")},
},
{
name: "match and nomatch",
match: []string{".*"},
nomatch: []string{".*"},
want: []error{ErrConflict},
},
{
name: "nomatch and match",
nomatch: []string{".*"},
matchagain: []string{".*"},
want: []error{ErrConflict},
},
{
name: "double match",
match: []string{"a", "b"},
want: []error{ErrAlreadyAdded},
},
{
name: "double nomatch",
nomatch: []string{"a", "b"},
want: []error{ErrAlreadyAdded},
},
}

for _, test := range testData {
t.Run(test.name, func(t *testing.T) {
f := new(FilterScheme)
var errs []error
for _, jq := range test.jq {
if err := f.AddJQ(jq); err != nil {
errs = append(errs, err)
}
}
for _, rx := range test.match {
if err := f.AddMatchRegex(rx); err != nil {
errs = append(errs, err)
}
}
for _, rx := range test.nomatch {
if err := f.AddNoMatchRegex(rx); err != nil {
errs = append(errs, err)
}
}
for _, rx := range test.matchagain {
if err := f.AddMatchRegex(rx); err != nil {
errs = append(errs, err)
}
}
if diff := cmp.Diff(errs, test.want, cmp.Comparer(comperror)); diff != "" {
t.Errorf("error:\n%s", diff)
}
if len(errs) > 0 {
if _, err := f.Run(&line{}); err != nil {
t.Errorf("run: %v", err)
}
}
})
}
}
Loading

0 comments on commit 7ed8fe3

Please sign in to comment.