Merge pull request #365 from projectdiscovery/363-bugfix-follow-redir…

…ects

Adding support for maximum redirects number

common/httpx/httpx.go

@@ -59,20 +59,30 @@ func New(options *Options) (*HTTPX, error) {
 	}
 
 	if httpx.Options.FollowRedirects {
-		// Follow redirects
-		redirectFunc = nil
+		// Follow redirects up to a maximum number
+		redirectFunc = func(redirectedRequest *http.Request, previousRequests []*http.Request) error {
+			if len(previousRequests) >= options.MaxRedirects {
+				// https://github.com/golang/go/issues/10069
+				return http.ErrUseLastResponse
+			}
+			return nil
+		}
 	}
 
 	if httpx.Options.FollowHostRedirects {
-		// Only follow redirects on the same host
-		redirectFunc = func(redirectedRequest *http.Request, previousRequest []*http.Request) error {
+		// Only follow redirects on the same host up to a maximum number
+		redirectFunc = func(redirectedRequest *http.Request, previousRequests []*http.Request) error {
 			// Check if we get a redirect to a differen host
 			var newHost = redirectedRequest.URL.Host
-			var oldHost = previousRequest[0].URL.Host
+			var oldHost = previousRequests[0].URL.Host
 			if newHost != oldHost {
 				// Tell the http client to not follow redirect
 				return http.ErrUseLastResponse
 			}
+			if len(previousRequests) >= options.MaxRedirects {
+				// https://github.com/golang/go/issues/10069
+				return http.ErrUseLastResponse
+			}
 			return nil
 		}
 	}

common/httpx/option.go

@@ -23,6 +23,7 @@ type Options struct {
 	VHostSimilarityRatio int
 	FollowRedirects      bool
 	FollowHostRedirects  bool
+	MaxRedirects         int
 	Unsafe               bool
 	TLSGrab              bool
 	// VHOSTs options
@@ -39,13 +40,14 @@ type Options struct {
 
 // DefaultOptions contains the default options
 var DefaultOptions = Options{
-	RandomAgent: true,
-	Threads:     25,
-	Timeout:     30 * time.Second,
-	RetryMax:    5,
-	Unsafe:      false,
-	CdnCheck:    true,
-	ExcludeCdn:  false,
+	RandomAgent:  true,
+	Threads:      25,
+	Timeout:      30 * time.Second,
+	RetryMax:     5,
+	MaxRedirects: 10,
+	Unsafe:       false,
+	CdnCheck:     true,
+	ExcludeCdn:   false,
 	// VHOSTs options
 	VHostIgnoreStatusCode:    false,
 	VHostIgnoreContentLength: true,

runner/options.go

@@ -156,6 +156,7 @@ type Options struct {
 	responseInStdout          bool
 	chainInStdout             bool
 	FollowHostRedirects       bool
+	MaxRedirects              int
 	OutputMethod              bool
 	TLSProbe                  bool
 	CSPProbe                  bool
@@ -212,6 +213,7 @@ func ParseOptions() *Options {
 	flag.StringVar(&options.StoreResponseDir, "srd", "output", "Save response directory")
 	flag.BoolVar(&options.FollowRedirects, "follow-redirects", false, "Follow Redirects")
 	flag.BoolVar(&options.FollowHostRedirects, "follow-host-redirects", false, "Only follow redirects on the same host")
+	flag.IntVar(&options.MaxRedirects, "max-redirects", 10, "Max number of redirects to follow per host")
 	flag.StringVar(&options.HTTPProxy, "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080")
 	flag.BoolVar(&options.JSONOutput, "json", false, "JSON Output")
 	flag.StringVar(&options.InputFile, "l", "", "File containing domains")

runner/runner.go

@@ -54,14 +54,14 @@ const (
 
 // Runner is a client for running the enumeration process.
 type Runner struct {
-	options       *Options
-	hp            *httpx.HTTPX
-	wappalyzer    *wappalyzer.Wappalyze
-	scanopts      scanOptions
-	hm            *hybrid.HybridMap
-	stats         clistats.StatisticsClient
-	ratelimiter   ratelimit.Limiter
-	FailedTargets gcache.Cache
+	options         *Options
+	hp              *httpx.HTTPX
+	wappalyzer      *wappalyzer.Wappalyze
+	scanopts        scanOptions
+	hm              *hybrid.HybridMap
+	stats           clistats.StatisticsClient
+	ratelimiter     ratelimit.Limiter
+	HostErrorsCache gcache.Cache
 }
 
 // New creates a new client for running enumeration process.
@@ -84,6 +84,7 @@ func New(options *Options) (*Runner, error) {
 	httpxOptions.RetryMax = options.Retries
 	httpxOptions.FollowRedirects = options.FollowRedirects
 	httpxOptions.FollowHostRedirects = options.FollowHostRedirects
+	httpxOptions.MaxRedirects = options.MaxRedirects
 	httpxOptions.HTTPProxy = options.HTTPProxy
 	httpxOptions.Unsafe = options.Unsafe
 	httpxOptions.RequestOverride = httpx.RequestOverride{URIPath: options.RequestURI}
@@ -236,7 +237,7 @@ func New(options *Options) (*Runner, error) {
 		gc := gcache.New(1000).
 			ARC().
 			Build()
-		runner.FailedTargets = gc
+		runner.HostErrorsCache = gc
 	}
 
 	return runner, nil
@@ -390,7 +391,7 @@ func (r *Runner) Close() {
 	r.hm.Close()
 	r.hp.Dialer.Close()
 	if r.options.HostMaxErrors >= 0 {
-		r.FailedTargets.Purge()
+		r.HostErrorsCache.Purge()
 	}
 }
 
@@ -628,8 +629,8 @@ retry:
 
 	// check if we have to skip the host:port as a result of a previous failure
 	hostPort := net.JoinHostPort(URL.Host, URL.Port)
-	if r.options.HostMaxErrors >= 0 && r.FailedTargets.Has(hostPort) {
-		numberOfErrors, err := r.FailedTargets.GetIFPresent(hostPort)
+	if r.options.HostMaxErrors >= 0 && r.HostErrorsCache.Has(hostPort) {
+		numberOfErrors, err := r.HostErrorsCache.GetIFPresent(hostPort)
 		if err == nil && numberOfErrors.(int) >= r.options.HostMaxErrors {
 			return Result{URL: domain, err: errors.New("skipping as previously unresponsive")}
 		}
@@ -750,11 +751,11 @@ retry:
 
 		// mark the host:port as failed to avoid further checks
 		if r.options.HostMaxErrors >= 0 {
-			errorCount, err := r.FailedTargets.GetIFPresent(hostPort)
+			errorCount, err := r.HostErrorsCache.GetIFPresent(hostPort)
 			if err != nil || errorCount == nil {
-				_ = r.FailedTargets.Set(hostPort, 1)
+				_ = r.HostErrorsCache.Set(hostPort, 1)
 			} else if errorCount != nil {
-				_ = r.FailedTargets.Set(hostPort, errorCount.(int)+1)
+				_ = r.HostErrorsCache.Set(hostPort, errorCount.(int)+1)
 			}
 		}