diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 60f100fe..923c83bc 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -3,6 +3,7 @@ package httpx import ( "crypto/tls" "fmt" + "io" "io/ioutil" "net/http" "net/url" @@ -156,7 +157,7 @@ get_response: // websockets don't have a readable body if httpresp.StatusCode != http.StatusSwitchingProtocols { var err error - respbody, err = ioutil.ReadAll(httpresp.Body) + respbody, err = ioutil.ReadAll(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead)) if err != nil { return nil, err } diff --git a/common/httpx/option.go b/common/httpx/option.go index 5c00faab..8ea37de8 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -25,13 +25,15 @@ type Options struct { Unsafe bool TLSGrab bool // VHOSTs options - VHostIgnoreStatusCode bool - VHostIgnoreContentLength bool - VHostIgnoreNumberOfWords bool - VHostIgnoreNumberOfLines bool - VHostStripHTML bool - Allow []string - Deny []string + VHostIgnoreStatusCode bool + VHostIgnoreContentLength bool + VHostIgnoreNumberOfWords bool + VHostIgnoreNumberOfLines bool + VHostStripHTML bool + Allow []string + Deny []string + MaxResponseBodySizeToSave int64 + MaxResponseBodySizeToRead int64 } // DefaultOptions contains the default options diff --git a/runner/options.go b/runner/options.go index 8c26422b..6af84702 100644 --- a/runner/options.go +++ b/runner/options.go @@ -22,77 +22,80 @@ const ( ) type scanOptions struct { - Methods []string - StoreResponseDirectory string - RequestURI string - RequestBody string - VHost bool - OutputTitle bool - OutputStatusCode bool - OutputLocation bool - OutputContentLength bool - StoreResponse bool - OutputServerHeader bool - OutputWebSocket bool - OutputWithNoColor bool - OutputMethod bool - ResponseInStdout bool - ChainInStdout bool - TLSProbe bool - CSPProbe bool - VHostInput bool - OutputContentType bool - Unsafe bool - Pipeline bool - HTTP2Probe bool - OutputIP bool - OutputCName bool - OutputCDN bool - OutputResponseTime bool - PreferHTTPS bool - NoFallback bool - NoFallbackScheme bool - TechDetect bool - StoreChain bool - MaxResponseBodySize int - OutputExtractRegex string - extractRegex *regexp.Regexp + Methods []string + StoreResponseDirectory string + RequestURI string + RequestBody string + VHost bool + OutputTitle bool + OutputStatusCode bool + OutputLocation bool + OutputContentLength bool + StoreResponse bool + OutputServerHeader bool + OutputWebSocket bool + OutputWithNoColor bool + OutputMethod bool + ResponseInStdout bool + ChainInStdout bool + TLSProbe bool + CSPProbe bool + VHostInput bool + OutputContentType bool + Unsafe bool + Pipeline bool + HTTP2Probe bool + OutputIP bool + OutputCName bool + OutputCDN bool + OutputResponseTime bool + PreferHTTPS bool + NoFallback bool + NoFallbackScheme bool + TechDetect bool + StoreChain bool + MaxResponseBodySizeToSave int + MaxResponseBodySizeToRead int + OutputExtractRegex string + extractRegex *regexp.Regexp } func (s *scanOptions) Clone() *scanOptions { return &scanOptions{ - Methods: s.Methods, - StoreResponseDirectory: s.StoreResponseDirectory, - RequestURI: s.RequestURI, - RequestBody: s.RequestBody, - VHost: s.VHost, - OutputTitle: s.OutputTitle, - OutputStatusCode: s.OutputStatusCode, - OutputLocation: s.OutputLocation, - OutputContentLength: s.OutputContentLength, - StoreResponse: s.StoreResponse, - OutputServerHeader: s.OutputServerHeader, - OutputWebSocket: s.OutputWebSocket, - OutputWithNoColor: s.OutputWithNoColor, - OutputMethod: s.OutputMethod, - ResponseInStdout: s.ResponseInStdout, - ChainInStdout: s.ChainInStdout, - TLSProbe: s.TLSProbe, - CSPProbe: s.CSPProbe, - OutputContentType: s.OutputContentType, - Unsafe: s.Unsafe, - Pipeline: s.Pipeline, - HTTP2Probe: s.HTTP2Probe, - OutputIP: s.OutputIP, - OutputCName: s.OutputCName, - OutputCDN: s.OutputCDN, - OutputResponseTime: s.OutputResponseTime, - PreferHTTPS: s.PreferHTTPS, - NoFallback: s.NoFallback, - NoFallbackScheme: s.NoFallbackScheme, - TechDetect: s.TechDetect, - StoreChain: s.StoreChain, - OutputExtractRegex: s.OutputExtractRegex, + Methods: s.Methods, + StoreResponseDirectory: s.StoreResponseDirectory, + RequestURI: s.RequestURI, + RequestBody: s.RequestBody, + VHost: s.VHost, + OutputTitle: s.OutputTitle, + OutputStatusCode: s.OutputStatusCode, + OutputLocation: s.OutputLocation, + OutputContentLength: s.OutputContentLength, + StoreResponse: s.StoreResponse, + OutputServerHeader: s.OutputServerHeader, + OutputWebSocket: s.OutputWebSocket, + OutputWithNoColor: s.OutputWithNoColor, + OutputMethod: s.OutputMethod, + ResponseInStdout: s.ResponseInStdout, + ChainInStdout: s.ChainInStdout, + TLSProbe: s.TLSProbe, + CSPProbe: s.CSPProbe, + OutputContentType: s.OutputContentType, + Unsafe: s.Unsafe, + Pipeline: s.Pipeline, + HTTP2Probe: s.HTTP2Probe, + OutputIP: s.OutputIP, + OutputCName: s.OutputCName, + OutputCDN: s.OutputCDN, + OutputResponseTime: s.OutputResponseTime, + PreferHTTPS: s.PreferHTTPS, + NoFallback: s.NoFallback, + NoFallbackScheme: s.NoFallbackScheme, + TechDetect: s.TechDetect, + StoreChain: s.StoreChain, + OutputExtractRegex: s.OutputExtractRegex, + MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave, + MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead, } } @@ -170,7 +173,8 @@ type Options struct { StoreChain bool Deny customlist.CustomList Allow customlist.CustomList - MaxResponseBodySize int + MaxResponseBodySizeToSave int + MaxResponseBodySizeToRead int OutputExtractRegex string RateLimit int Probe bool @@ -180,6 +184,7 @@ type Options struct { func ParseOptions() *Options { options := &Options{} + flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) flag.BoolVar(&options.TLSGrab, "tls-grab", false, "Perform TLS data grabbing") flag.BoolVar(&options.TechDetect, "tech-detect", false, "Perform wappalyzer based technology detection") flag.IntVar(&options.Threads, "threads", 50, "Number of threads") @@ -242,7 +247,8 @@ func ParseOptions() *Options { flag.BoolVar(&options.StoreChain, "store-chain", false, "Save chain to file (default 'output')") flag.Var(&options.Allow, "allow", "Allowlist ip/cidr") flag.Var(&options.Deny, "deny", "Denylist ip/cidr") - flag.IntVar(&options.MaxResponseBodySize, "max-response-body-size", math.MaxInt32, "Maximum response body size") + flag.IntVar(&options.MaxResponseBodySizeToSave, "response-size-to-save", math.MaxInt32, "Max response size to save in bytes (default - unlimited)") + flag.IntVar(&options.MaxResponseBodySizeToRead, "response-size-to-read", math.MaxInt32, "Max response size to read in bytes (default - unlimited)") flag.StringVar(&options.OutputExtractRegex, "extract-regex", "", "Extract Regex") flag.IntVar(&options.RateLimit, "rate-limit", 150, "Maximum requests to send per second") flag.BoolVar(&options.Probe, "probe", false, "Display probe status") diff --git a/runner/runner.go b/runner/runner.go index 5e10684f..085f7217 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -86,6 +86,12 @@ func New(options *Options) (*Runner, error) { httpxOptions.RandomAgent = options.RandomAgent httpxOptions.Deny = options.Deny httpxOptions.Allow = options.Allow + httpxOptions.MaxResponseBodySizeToSave = int64(options.MaxResponseBodySizeToSave) + httpxOptions.MaxResponseBodySizeToRead = int64(options.MaxResponseBodySizeToRead) + // adjust response size saved according to the max one read by the server + if httpxOptions.MaxResponseBodySizeToSave > httpxOptions.MaxResponseBodySizeToRead { + httpxOptions.MaxResponseBodySizeToSave = httpxOptions.MaxResponseBodySizeToRead + } var key, value string httpxOptions.CustomHeaders = make(map[string]string) @@ -184,7 +190,8 @@ func New(options *Options) (*Runner, error) { scanopts.NoFallbackScheme = options.NoFallbackScheme scanopts.TechDetect = options.TechDetect scanopts.StoreChain = options.StoreChain - scanopts.MaxResponseBodySize = options.MaxResponseBodySize + scanopts.MaxResponseBodySizeToSave = options.MaxResponseBodySizeToSave + scanopts.MaxResponseBodySizeToRead = options.MaxResponseBodySizeToRead if options.OutputExtractRegex != "" { if scanopts.extractRegex, err = regexp.Compile(options.OutputExtractRegex); err != nil { return nil, err @@ -887,8 +894,8 @@ retry: // store response responsePath := path.Join(scanopts.StoreResponseDirectory, domainFile) respRaw := resp.Raw - if len(respRaw) > scanopts.MaxResponseBodySize { - respRaw = respRaw[:scanopts.MaxResponseBodySize] + if len(respRaw) > scanopts.MaxResponseBodySizeToSave { + respRaw = respRaw[:scanopts.MaxResponseBodySizeToSave] } writeErr := ioutil.WriteFile(responsePath, []byte(respRaw), 0644) if writeErr != nil { @@ -1022,8 +1029,8 @@ type Result struct { // JSON the result func (r Result) JSON(scanopts *scanOptions) string { //nolint - if scanopts != nil && len(r.ResponseBody) > scanopts.MaxResponseBodySize { - r.ResponseBody = r.ResponseBody[:scanopts.MaxResponseBodySize] + if scanopts != nil && len(r.ResponseBody) > scanopts.MaxResponseBodySizeToSave { + r.ResponseBody = r.ResponseBody[:scanopts.MaxResponseBodySizeToSave] } if js, err := json.Marshal(r); err == nil {