parse: handle lines up to 1MiB long

pkg/parse/parse.go

@@ -37,6 +37,9 @@ const (
 	LevelFatal
 )
 
+// LineBufferSize is the longest we're willing to look for a newline in the input.
+const LineBufferSize = 1 * 1024 * 1024 // 1 MiB
+
 // InputSchema controls the interpretation of incoming log lines.
 type InputSchema struct {
 	TimeKey     string      // The name of the key that holds the timestamp.
@@ -193,6 +196,7 @@ func runJQ(jq *gojq.Code, l *line) (bool, error) {
 // on the reader, are returned.
 func ReadLog(r io.Reader, w io.Writer, ins *InputSchema, outs *OutputSchema, jq *gojq.Code) (Summary, error) {
 	s := bufio.NewScanner(r)
+	s.Buffer(make([]byte, 0, LineBufferSize), LineBufferSize)
 	var l line
 	outs.state = State{
 		lastFields: make(map[string][]byte),

pkg/parse/parse_fuzz_test.go

@@ -4,7 +4,9 @@
 package parse
 
 import (
+	"bufio"
 	"bytes"
+	"errors"
 	"testing"
 
 	"github.com/logrusorgru/aurora/v3"
@@ -38,6 +40,12 @@ func FuzzReadLog(f *testing.F) {
 		}
 		outbuf := new(bytes.Buffer)
 		if _, err := ReadLog(inbuf, outbuf, ins, outs, jq); err != nil {
+			if errors.Is(err, bufio.ErrTooLong) {
+				// This is a known limit and the fuzzer likes to produce very long
+				// garbage lines.  The tests convinced me to increase this limit,
+				// but it has to be limited somewhere.
+				t.SkipNow()
+			}
 			t.Fatal(err)
 		}
 		if got, want := bytes.Count(outbuf.Bytes(), []byte("\n")), bytes.Count(in, []byte("\n")); got < want {