From 89f77e50d202f4f0d8dfdeacdc83ad204e75fb54 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 16 Aug 2021 14:25:30 +0200 Subject: [PATCH 1/3] Adding support for maximum redirects number --- common/httpx/httpx.go | 20 +++++++++++++++----- common/httpx/option.go | 16 +++++++++------- runner/options.go | 2 ++ runner/runner.go | 1 + 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 33b3852e..9d21771f 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -59,20 +59,30 @@ func New(options *Options) (*HTTPX, error) { } if httpx.Options.FollowRedirects { - // Follow redirects - redirectFunc = nil + // Follow redirects up to a maximum number + redirectFunc = func(redirectedRequest *http.Request, previousRequests []*http.Request) error { + if len(previousRequests) >= options.MaxRedirects { + // https://github.com/golang/go/issues/10069 + return http.ErrUseLastResponse + } + return nil + } } if httpx.Options.FollowHostRedirects { - // Only follow redirects on the same host - redirectFunc = func(redirectedRequest *http.Request, previousRequest []*http.Request) error { + // Only follow redirects on the same host up to a maximum number + redirectFunc = func(redirectedRequest *http.Request, previousRequests []*http.Request) error { // Check if we get a redirect to a differen host var newHost = redirectedRequest.URL.Host - var oldHost = previousRequest[0].URL.Host + var oldHost = previousRequests[0].URL.Host if newHost != oldHost { // Tell the http client to not follow redirect return http.ErrUseLastResponse } + if len(previousRequests) >= options.MaxRedirects { + // https://github.com/golang/go/issues/10069 + return http.ErrUseLastResponse + } return nil } } diff --git a/common/httpx/option.go b/common/httpx/option.go index 840a4ac4..1106e9c6 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -23,6 +23,7 @@ type Options struct { VHostSimilarityRatio int FollowRedirects bool FollowHostRedirects bool + MaxRedirects int Unsafe bool TLSGrab bool // VHOSTs options @@ -39,13 +40,14 @@ type Options struct { // DefaultOptions contains the default options var DefaultOptions = Options{ - RandomAgent: true, - Threads: 25, - Timeout: 30 * time.Second, - RetryMax: 5, - Unsafe: false, - CdnCheck: true, - ExcludeCdn: false, + RandomAgent: true, + Threads: 25, + Timeout: 30 * time.Second, + RetryMax: 5, + MaxRedirects: 10, + Unsafe: false, + CdnCheck: true, + ExcludeCdn: false, // VHOSTs options VHostIgnoreStatusCode: false, VHostIgnoreContentLength: true, diff --git a/runner/options.go b/runner/options.go index 9d42c6a6..1b6bcba5 100644 --- a/runner/options.go +++ b/runner/options.go @@ -154,6 +154,7 @@ type Options struct { responseInStdout bool chainInStdout bool FollowHostRedirects bool + MaxRedirects int OutputMethod bool TLSProbe bool CSPProbe bool @@ -209,6 +210,7 @@ func ParseOptions() *Options { flag.StringVar(&options.StoreResponseDir, "srd", "output", "Save response directory") flag.BoolVar(&options.FollowRedirects, "follow-redirects", false, "Follow Redirects") flag.BoolVar(&options.FollowHostRedirects, "follow-host-redirects", false, "Only follow redirects on the same host") + flag.IntVar(&options.MaxRedirects, "max-redirects", 10, "Max number of redirects to follow") flag.StringVar(&options.HTTPProxy, "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080") flag.BoolVar(&options.JSONOutput, "json", false, "JSON Output") flag.StringVar(&options.InputFile, "l", "", "File containing domains") diff --git a/runner/runner.go b/runner/runner.go index e0798f48..577ba589 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -81,6 +81,7 @@ func New(options *Options) (*Runner, error) { httpxOptions.RetryMax = options.Retries httpxOptions.FollowRedirects = options.FollowRedirects httpxOptions.FollowHostRedirects = options.FollowHostRedirects + httpxOptions.MaxRedirects = options.MaxRedirects httpxOptions.HTTPProxy = options.HTTPProxy httpxOptions.Unsafe = options.Unsafe httpxOptions.RequestOverride = httpx.RequestOverride{URIPath: options.RequestURI} From 0f4f9f2cac38619994885f4784ff249794362e67 Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 16 Aug 2021 23:52:52 +0530 Subject: [PATCH 2/3] minor update --- runner/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/options.go b/runner/options.go index 1b6bcba5..97615134 100644 --- a/runner/options.go +++ b/runner/options.go @@ -210,7 +210,7 @@ func ParseOptions() *Options { flag.StringVar(&options.StoreResponseDir, "srd", "output", "Save response directory") flag.BoolVar(&options.FollowRedirects, "follow-redirects", false, "Follow Redirects") flag.BoolVar(&options.FollowHostRedirects, "follow-host-redirects", false, "Only follow redirects on the same host") - flag.IntVar(&options.MaxRedirects, "max-redirects", 10, "Max number of redirects to follow") + flag.IntVar(&options.MaxRedirects, "max-redirects", 10, "Max number of redirects to follow per host") flag.StringVar(&options.HTTPProxy, "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080") flag.BoolVar(&options.JSONOutput, "json", false, "JSON Output") flag.StringVar(&options.InputFile, "l", "", "File containing domains") From 6bc6359f3dbdc821a183e214e563f8b74d1d00c9 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 16 Aug 2021 22:23:03 +0200 Subject: [PATCH 3/3] small name refactor --- runner/runner.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index ed83520e..b2b4cbbb 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -54,14 +54,14 @@ const ( // Runner is a client for running the enumeration process. type Runner struct { - options *Options - hp *httpx.HTTPX - wappalyzer *wappalyzer.Wappalyze - scanopts scanOptions - hm *hybrid.HybridMap - stats clistats.StatisticsClient - ratelimiter ratelimit.Limiter - FailedTargets gcache.Cache + options *Options + hp *httpx.HTTPX + wappalyzer *wappalyzer.Wappalyze + scanopts scanOptions + hm *hybrid.HybridMap + stats clistats.StatisticsClient + ratelimiter ratelimit.Limiter + HostErrorsCache gcache.Cache } // New creates a new client for running enumeration process. @@ -237,7 +237,7 @@ func New(options *Options) (*Runner, error) { gc := gcache.New(1000). ARC(). Build() - runner.FailedTargets = gc + runner.HostErrorsCache = gc } return runner, nil @@ -391,7 +391,7 @@ func (r *Runner) Close() { r.hm.Close() r.hp.Dialer.Close() if r.options.HostMaxErrors >= 0 { - r.FailedTargets.Purge() + r.HostErrorsCache.Purge() } } @@ -629,8 +629,8 @@ retry: // check if we have to skip the host:port as a result of a previous failure hostPort := net.JoinHostPort(URL.Host, URL.Port) - if r.options.HostMaxErrors >= 0 && r.FailedTargets.Has(hostPort) { - numberOfErrors, err := r.FailedTargets.GetIFPresent(hostPort) + if r.options.HostMaxErrors >= 0 && r.HostErrorsCache.Has(hostPort) { + numberOfErrors, err := r.HostErrorsCache.GetIFPresent(hostPort) if err == nil && numberOfErrors.(int) >= r.options.HostMaxErrors { return Result{URL: domain, err: errors.New("skipping as previously unresponsive")} } @@ -751,11 +751,11 @@ retry: // mark the host:port as failed to avoid further checks if r.options.HostMaxErrors >= 0 { - errorCount, err := r.FailedTargets.GetIFPresent(hostPort) + errorCount, err := r.HostErrorsCache.GetIFPresent(hostPort) if err != nil || errorCount == nil { - _ = r.FailedTargets.Set(hostPort, 1) + _ = r.HostErrorsCache.Set(hostPort, 1) } else if errorCount != nil { - _ = r.FailedTargets.Set(hostPort, errorCount.(int)+1) + _ = r.HostErrorsCache.Set(hostPort, errorCount.(int)+1) } }