Skip to content

Commit

Permalink
Merge pull request #346 from projectdiscovery/issue-325-max-response-…
Browse files Browse the repository at this point in the history 
…body-size

issue-325-max-response-body-size
  • Loading branch information
@Mzack9999
Mzack9999 committed Aug 7, 2021
2 parents 3181440 + d9ccc52 commit 89505f6
Show file tree
Hide file tree
Showing 4 changed files with 98 additions and 82 deletions.
3 changes: 2 additions & 1 deletion common/httpx/httpx.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package httpx
import (
"crypto/tls"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
Expand Down Expand Up @@ -156,7 +157,7 @@ get_response:
// websockets don't have a readable body
if httpresp.StatusCode != http.StatusSwitchingProtocols {
var err error
respbody, err = ioutil.ReadAll(httpresp.Body)
respbody, err = ioutil.ReadAll(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead))
if err != nil {
return nil, err
}
Expand Down
16 changes: 9 additions & 7 deletions common/httpx/option.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,15 @@ type Options struct {
Unsafe bool
TLSGrab bool
// VHOSTs options
VHostIgnoreStatusCode bool
VHostIgnoreContentLength bool
VHostIgnoreNumberOfWords bool
VHostIgnoreNumberOfLines bool
VHostStripHTML bool
Allow []string
Deny []string
VHostIgnoreStatusCode bool
VHostIgnoreContentLength bool
VHostIgnoreNumberOfWords bool
VHostIgnoreNumberOfLines bool
VHostStripHTML bool
Allow []string
Deny []string
MaxResponseBodySizeToSave int64
MaxResponseBodySizeToRead int64
}

// DefaultOptions contains the default options
Expand Down
144 changes: 75 additions & 69 deletions runner/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,77 +22,80 @@ const (
)

type scanOptions struct {
Methods []string
StoreResponseDirectory string
RequestURI string
RequestBody string
VHost bool
OutputTitle bool
OutputStatusCode bool
OutputLocation bool
OutputContentLength bool
StoreResponse bool
OutputServerHeader bool
OutputWebSocket bool
OutputWithNoColor bool
OutputMethod bool
ResponseInStdout bool
ChainInStdout bool
TLSProbe bool
CSPProbe bool
VHostInput bool
OutputContentType bool
Unsafe bool
Pipeline bool
HTTP2Probe bool
OutputIP bool
OutputCName bool
OutputCDN bool
OutputResponseTime bool
PreferHTTPS bool
NoFallback bool
NoFallbackScheme bool
TechDetect bool
StoreChain bool
MaxResponseBodySize int
OutputExtractRegex string
extractRegex *regexp.Regexp
Methods []string
StoreResponseDirectory string
RequestURI string
RequestBody string
VHost bool
OutputTitle bool
OutputStatusCode bool
OutputLocation bool
OutputContentLength bool
StoreResponse bool
OutputServerHeader bool
OutputWebSocket bool
OutputWithNoColor bool
OutputMethod bool
ResponseInStdout bool
ChainInStdout bool
TLSProbe bool
CSPProbe bool
VHostInput bool
OutputContentType bool
Unsafe bool
Pipeline bool
HTTP2Probe bool
OutputIP bool
OutputCName bool
OutputCDN bool
OutputResponseTime bool
PreferHTTPS bool
NoFallback bool
NoFallbackScheme bool
TechDetect bool
StoreChain bool
MaxResponseBodySizeToSave int
MaxResponseBodySizeToRead int
OutputExtractRegex string
extractRegex *regexp.Regexp
}

func (s *scanOptions) Clone() *scanOptions {
return &scanOptions{
Methods: s.Methods,
StoreResponseDirectory: s.StoreResponseDirectory,
RequestURI: s.RequestURI,
RequestBody: s.RequestBody,
VHost: s.VHost,
OutputTitle: s.OutputTitle,
OutputStatusCode: s.OutputStatusCode,
OutputLocation: s.OutputLocation,
OutputContentLength: s.OutputContentLength,
StoreResponse: s.StoreResponse,
OutputServerHeader: s.OutputServerHeader,
OutputWebSocket: s.OutputWebSocket,
OutputWithNoColor: s.OutputWithNoColor,
OutputMethod: s.OutputMethod,
ResponseInStdout: s.ResponseInStdout,
ChainInStdout: s.ChainInStdout,
TLSProbe: s.TLSProbe,
CSPProbe: s.CSPProbe,
OutputContentType: s.OutputContentType,
Unsafe: s.Unsafe,
Pipeline: s.Pipeline,
HTTP2Probe: s.HTTP2Probe,
OutputIP: s.OutputIP,
OutputCName: s.OutputCName,
OutputCDN: s.OutputCDN,
OutputResponseTime: s.OutputResponseTime,
PreferHTTPS: s.PreferHTTPS,
NoFallback: s.NoFallback,
NoFallbackScheme: s.NoFallbackScheme,
TechDetect: s.TechDetect,
StoreChain: s.StoreChain,
OutputExtractRegex: s.OutputExtractRegex,
Methods: s.Methods,
StoreResponseDirectory: s.StoreResponseDirectory,
RequestURI: s.RequestURI,
RequestBody: s.RequestBody,
VHost: s.VHost,
OutputTitle: s.OutputTitle,
OutputStatusCode: s.OutputStatusCode,
OutputLocation: s.OutputLocation,
OutputContentLength: s.OutputContentLength,
StoreResponse: s.StoreResponse,
OutputServerHeader: s.OutputServerHeader,
OutputWebSocket: s.OutputWebSocket,
OutputWithNoColor: s.OutputWithNoColor,
OutputMethod: s.OutputMethod,
ResponseInStdout: s.ResponseInStdout,
ChainInStdout: s.ChainInStdout,
TLSProbe: s.TLSProbe,
CSPProbe: s.CSPProbe,
OutputContentType: s.OutputContentType,
Unsafe: s.Unsafe,
Pipeline: s.Pipeline,
HTTP2Probe: s.HTTP2Probe,
OutputIP: s.OutputIP,
OutputCName: s.OutputCName,
OutputCDN: s.OutputCDN,
OutputResponseTime: s.OutputResponseTime,
PreferHTTPS: s.PreferHTTPS,
NoFallback: s.NoFallback,
NoFallbackScheme: s.NoFallbackScheme,
TechDetect: s.TechDetect,
StoreChain: s.StoreChain,
OutputExtractRegex: s.OutputExtractRegex,
MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave,
MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead,
}
}

Expand Down Expand Up @@ -170,7 +173,8 @@ type Options struct {
StoreChain bool
Deny customlist.CustomList
Allow customlist.CustomList
MaxResponseBodySize int
MaxResponseBodySizeToSave int
MaxResponseBodySizeToRead int
OutputExtractRegex string
RateLimit int
Probe bool
Expand All @@ -180,6 +184,7 @@ type Options struct {
func ParseOptions() *Options {
options := &Options{}

flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
flag.BoolVar(&options.TLSGrab, "tls-grab", false, "Perform TLS data grabbing")
flag.BoolVar(&options.TechDetect, "tech-detect", false, "Perform wappalyzer based technology detection")
flag.IntVar(&options.Threads, "threads", 50, "Number of threads")
Expand Down Expand Up @@ -242,7 +247,8 @@ func ParseOptions() *Options {
flag.BoolVar(&options.StoreChain, "store-chain", false, "Save chain to file (default 'output')")
flag.Var(&options.Allow, "allow", "Allowlist ip/cidr")
flag.Var(&options.Deny, "deny", "Denylist ip/cidr")
flag.IntVar(&options.MaxResponseBodySize, "max-response-body-size", math.MaxInt32, "Maximum response body size")
flag.IntVar(&options.MaxResponseBodySizeToSave, "response-size-to-save", math.MaxInt32, "Max response size to save in bytes (default - unlimited)")
flag.IntVar(&options.MaxResponseBodySizeToRead, "response-size-to-read", math.MaxInt32, "Max response size to read in bytes (default - unlimited)")
flag.StringVar(&options.OutputExtractRegex, "extract-regex", "", "Extract Regex")
flag.IntVar(&options.RateLimit, "rate-limit", 150, "Maximum requests to send per second")
flag.BoolVar(&options.Probe, "probe", false, "Display probe status")
Expand Down
17 changes: 12 additions & 5 deletions runner/runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,12 @@ func New(options *Options) (*Runner, error) {
httpxOptions.RandomAgent = options.RandomAgent
httpxOptions.Deny = options.Deny
httpxOptions.Allow = options.Allow
httpxOptions.MaxResponseBodySizeToSave = int64(options.MaxResponseBodySizeToSave)
httpxOptions.MaxResponseBodySizeToRead = int64(options.MaxResponseBodySizeToRead)
// adjust response size saved according to the max one read by the server
if httpxOptions.MaxResponseBodySizeToSave > httpxOptions.MaxResponseBodySizeToRead {
httpxOptions.MaxResponseBodySizeToSave = httpxOptions.MaxResponseBodySizeToRead
}

var key, value string
httpxOptions.CustomHeaders = make(map[string]string)
Expand Down Expand Up @@ -184,7 +190,8 @@ func New(options *Options) (*Runner, error) {
scanopts.NoFallbackScheme = options.NoFallbackScheme
scanopts.TechDetect = options.TechDetect
scanopts.StoreChain = options.StoreChain
scanopts.MaxResponseBodySize = options.MaxResponseBodySize
scanopts.MaxResponseBodySizeToSave = options.MaxResponseBodySizeToSave
scanopts.MaxResponseBodySizeToRead = options.MaxResponseBodySizeToRead
if options.OutputExtractRegex != "" {
if scanopts.extractRegex, err = regexp.Compile(options.OutputExtractRegex); err != nil {
return nil, err
Expand Down Expand Up @@ -887,8 +894,8 @@ retry:
// store response
responsePath := path.Join(scanopts.StoreResponseDirectory, domainFile)
respRaw := resp.Raw
if len(respRaw) > scanopts.MaxResponseBodySize {
respRaw = respRaw[:scanopts.MaxResponseBodySize]
if len(respRaw) > scanopts.MaxResponseBodySizeToSave {
respRaw = respRaw[:scanopts.MaxResponseBodySizeToSave]
}
writeErr := ioutil.WriteFile(responsePath, []byte(respRaw), 0644)
if writeErr != nil {
Expand Down Expand Up @@ -1022,8 +1029,8 @@ type Result struct {

// JSON the result
func (r Result) JSON(scanopts *scanOptions) string { //nolint
if scanopts != nil && len(r.ResponseBody) > scanopts.MaxResponseBodySize {
r.ResponseBody = r.ResponseBody[:scanopts.MaxResponseBodySize]
if scanopts != nil && len(r.ResponseBody) > scanopts.MaxResponseBodySizeToSave {
r.ResponseBody = r.ResponseBody[:scanopts.MaxResponseBodySizeToSave]
}

if js, err := json.Marshal(r); err == nil {
Expand Down

0 comments on commit 89505f6

Please sign in to comment.