From d89b3c14fbc8a70eded5c45faacd3f7d15477b90 Mon Sep 17 00:00:00 2001 From: mzack Date: Fri, 13 Aug 2021 20:01:48 +0200 Subject: [PATCH 01/28] Adding option to skip remaining paths on unresponsive host --- go.mod | 1 + go.sum | 2 ++ runner/options.go | 4 ++++ runner/runner.go | 40 +++++++++++++++++++++++++++++++++------- 4 files changed, 40 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 3d163c1b..22dbf61d 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.14 require ( github.com/akrylysov/pogreb v0.10.1 // indirect + github.com/bluele/gcache v0.0.2 // indirect github.com/corpix/uarand v0.1.1 github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect diff --git a/go.sum b/go.sum index 418df5c5..52ef7f50 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= +github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= diff --git a/runner/options.go b/runner/options.go index 9d42c6a6..24055dd8 100644 --- a/runner/options.go +++ b/runner/options.go @@ -61,6 +61,7 @@ type scanOptions struct { OutputExtractRegex string extractRegex *regexp.Regexp ExcludeCDN bool + SkipRemainingPathsOnError bool } func (s *scanOptions) Clone() *scanOptions { @@ -99,6 +100,7 @@ func (s *scanOptions) Clone() *scanOptions { OutputExtractRegex: s.OutputExtractRegex, MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave, MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead, + SkipRemainingPathsOnError: s.SkipRemainingPathsOnError, } } @@ -184,6 +186,7 @@ type Options struct { Resume bool resumeCfg *ResumeCfg ExcludeCDN bool + SkipRemainingPathsOnError bool } // ParseOptions parses the command line options for application @@ -260,6 +263,7 @@ func ParseOptions() *Options { flag.BoolVar(&options.Probe, "probe", false, "Display probe status") flag.BoolVar(&options.Resume, "resume", false, "Resume scan using resume.cfg") flag.BoolVar(&options.ExcludeCDN, "exclude-cdn", false, "Skip full port scans for CDNs (only checks for 80,443)") + flag.BoolVar(&options.SkipRemainingPathsOnError, "skip-paths-on-error", false, "Skip remaining paths on unresponsive servers") flag.Parse() diff --git a/runner/runner.go b/runner/runner.go index e0798f48..1d4a3a5c 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -8,6 +8,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net" "net/http" "net/http/httputil" "net/url" @@ -19,6 +20,7 @@ import ( "strings" "time" + "github.com/bluele/gcache" "github.com/logrusorgru/aurora" "github.com/pkg/errors" "github.com/projectdiscovery/clistats" @@ -52,13 +54,14 @@ const ( // Runner is a client for running the enumeration process. type Runner struct { - options *Options - hp *httpx.HTTPX - wappalyzer *wappalyzer.Wappalyze - scanopts scanOptions - hm *hybrid.HybridMap - stats clistats.StatisticsClient - ratelimiter ratelimit.Limiter + options *Options + hp *httpx.HTTPX + wappalyzer *wappalyzer.Wappalyze + scanopts scanOptions + hm *hybrid.HybridMap + stats clistats.StatisticsClient + ratelimiter ratelimit.Limiter + FailedTargets gcache.Cache } // New creates a new client for running enumeration process. @@ -207,6 +210,7 @@ func New(options *Options) (*Runner, error) { } scanopts.ExcludeCDN = options.ExcludeCDN + scanopts.SkipRemainingPathsOnError = options.SkipRemainingPathsOnError runner.scanopts = scanopts if options.ShowStatistics { @@ -228,6 +232,13 @@ func New(options *Options) (*Runner, error) { runner.ratelimiter = ratelimit.NewUnlimited() } + if options.SkipRemainingPathsOnError { + gc := gcache.New(1000). + ARC(). + Build() + runner.FailedTargets = gc + } + return runner, nil } @@ -378,6 +389,9 @@ func (r *Runner) Close() { // nolint:errcheck // ignore r.hm.Close() r.hp.Dialer.Close() + if r.options.SkipRemainingPathsOnError { + r.FailedTargets.Purge() + } } // RunEnumeration on targets for httpx client @@ -612,6 +626,12 @@ retry: return Result{URL: domain, err: err} } + // check if we have to skip the host:port as a result of a previous failure + hostPort := net.JoinHostPort(URL.Host, URL.Port) + if r.options.SkipRemainingPathsOnError && r.FailedTargets.Has(hostPort) { + return Result{URL: domain, err: errors.New("skipping as previously unresponsive")} + } + // check if the combination host:port should be skipped if belonging to a cdn if r.skipCDNPort(URL.Host, URL.Port) { gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port) @@ -724,6 +744,12 @@ retry: retried = true goto retry } + + // mark the host:port as failed to avoid further checks + if r.options.SkipRemainingPathsOnError { + _ = r.FailedTargets.Set(hostPort, nil) + } + if r.options.Probe { return Result{URL: URL.String(), Input: domain, Timestamp: time.Now(), err: err, Failed: err != nil, Error: errString, str: builder.String()} } else { From ae0c4501de0a3e4071a8dacc8d098083afe124d6 Mon Sep 17 00:00:00 2001 From: mzack Date: Sat, 14 Aug 2021 20:19:53 +0200 Subject: [PATCH 02/28] Improving rawhttp timeout handling --- common/httpx/httpx.go | 4 +++- go.mod | 6 +++--- go.sum | 8 ++++++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index dd1fa55a..33b3852e 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -226,7 +226,9 @@ func (h *HTTPX) doUnsafe(req *retryablehttp.Request) (*http.Response, error) { headers := req.Header targetURL := req.URL.String() body := req.Body - return rawhttp.DoRaw(method, targetURL, h.RequestOverride.URIPath, headers, body) + options := rawhttp.DefaultOptions + options.Timeout = h.Options.Timeout + return rawhttp.DoRawWithOptions(method, targetURL, h.RequestOverride.URIPath, headers, body, options) } // Verify the http calls and apply-cascade all the filters, as soon as one matches it returns true diff --git a/go.mod b/go.mod index 3d163c1b..fb4e4a43 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d github.com/projectdiscovery/iputil v0.0.0-20210705072957-5a968407979b github.com/projectdiscovery/mapcidr v0.0.8 - github.com/projectdiscovery/rawhttp v0.0.7 + github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e github.com/projectdiscovery/retryabledns v1.0.12 // indirect github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7dc github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d @@ -35,8 +35,8 @@ require ( go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/ratelimit v0.2.0 - golang.org/x/net v0.0.0-20210716203947-853a461950ff + golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect - golang.org/x/text v0.3.6 + golang.org/x/text v0.3.7 google.golang.org/protobuf v1.27.1 // indirect ) diff --git a/go.sum b/go.sum index 418df5c5..02063082 100644 --- a/go.sum +++ b/go.sum @@ -166,6 +166,10 @@ github.com/projectdiscovery/networkpolicy v0.0.1 h1:RGRuPlxE8WLFF9tdKSjTsYiTIKHN github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg= github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= +github.com/projectdiscovery/rawhttp v0.0.8-0.20210814173504-555bdf13a5d5 h1:k/bPIQbpkBn0FjjRVT9NptSM4lnn+UYo96iu4Vp60j0= +github.com/projectdiscovery/rawhttp v0.0.8-0.20210814173504-555bdf13a5d5/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= +github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e h1:hcpGb5/gSn+kNUmzgodV1+sHDmFybuGhsuhrTqFebQY= +github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f h1:HR3R/nhELwLXufUlO1ZkKVqrZl4lN1cWFBdN8RcMuLo= github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f/go.mod h1:3L0WfNIcVWXIDur8k+gKDLZLWY2F+rs0SQXtcn/3AYU= github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4= @@ -244,6 +248,8 @@ golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210716203947-853a461950ff h1:j2EK/QoxYNBsXI4R7fQkkRUk8y6wnOBI+6hgPdP/6Ds= golang.org/x/net v0.0.0-20210716203947-853a461950ff/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -278,6 +284,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= From 207640370ffc045ff5b60f054a44a290e8ded695 Mon Sep 17 00:00:00 2001 From: mzack Date: Sat, 14 Aug 2021 20:37:50 +0200 Subject: [PATCH 03/28] Upgrading fastdialer with better error message on denied ips --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 3d163c1b..d910acd3 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/projectdiscovery/cdncheck v0.0.2 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 - github.com/projectdiscovery/fastdialer v0.0.12 + github.com/projectdiscovery/fastdialer v0.0.13-0.20210814183457-4084195d84b2 github.com/projectdiscovery/fdmax v0.0.3 github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c github.com/projectdiscovery/gologger v1.1.4 diff --git a/go.sum b/go.sum index 418df5c5..899c0539 100644 --- a/go.sum +++ b/go.sum @@ -140,6 +140,8 @@ github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6 github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= github.com/projectdiscovery/fastdialer v0.0.12 h1:TjvM41UfR+A7YsxQZoTvI6C5nVe1d+fvRqtcDNbSwz8= github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= +github.com/projectdiscovery/fastdialer v0.0.13-0.20210814183457-4084195d84b2 h1:nnzfSmfCGLcdkDtMMReY5axGqxne21GDfWZ9p76kLqs= +github.com/projectdiscovery/fastdialer v0.0.13-0.20210814183457-4084195d84b2/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg= github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= From ac4a184b752faaaa3234265417a7bb2af937231e Mon Sep 17 00:00:00 2001 From: sandeep Date: Sun, 15 Aug 2021 15:56:17 +0530 Subject: [PATCH 04/28] go mod update --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index d910acd3..4e36a16f 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/projectdiscovery/cdncheck v0.0.2 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 - github.com/projectdiscovery/fastdialer v0.0.13-0.20210814183457-4084195d84b2 + github.com/projectdiscovery/fastdialer v0.0.13-0.20210815100514-360f851a5b80 github.com/projectdiscovery/fdmax v0.0.3 github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c github.com/projectdiscovery/gologger v1.1.4 diff --git a/go.sum b/go.sum index 899c0539..0f87d8c3 100644 --- a/go.sum +++ b/go.sum @@ -142,6 +142,8 @@ github.com/projectdiscovery/fastdialer v0.0.12 h1:TjvM41UfR+A7YsxQZoTvI6C5nVe1d+ github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= github.com/projectdiscovery/fastdialer v0.0.13-0.20210814183457-4084195d84b2 h1:nnzfSmfCGLcdkDtMMReY5axGqxne21GDfWZ9p76kLqs= github.com/projectdiscovery/fastdialer v0.0.13-0.20210814183457-4084195d84b2/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= +github.com/projectdiscovery/fastdialer v0.0.13-0.20210815100514-360f851a5b80 h1:Hyt1AcK5hAgE7LuaNyRD3mFc8zaS7LP4vTk6Yp83f8E= +github.com/projectdiscovery/fastdialer v0.0.13-0.20210815100514-360f851a5b80/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg= github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= From 800362e1dfcbd95793560ef8d3fb4778eecc1cc5 Mon Sep 17 00:00:00 2001 From: mzack Date: Sun, 15 Aug 2021 16:04:24 +0200 Subject: [PATCH 05/28] adding host max errors support --- runner/options.go | 8 ++++---- runner/runner.go | 22 +++++++++++++++------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/runner/options.go b/runner/options.go index 24055dd8..82d94cc2 100644 --- a/runner/options.go +++ b/runner/options.go @@ -61,7 +61,7 @@ type scanOptions struct { OutputExtractRegex string extractRegex *regexp.Regexp ExcludeCDN bool - SkipRemainingPathsOnError bool + HostMaxErrors int } func (s *scanOptions) Clone() *scanOptions { @@ -100,7 +100,7 @@ func (s *scanOptions) Clone() *scanOptions { OutputExtractRegex: s.OutputExtractRegex, MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave, MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead, - SkipRemainingPathsOnError: s.SkipRemainingPathsOnError, + HostMaxErrors: s.HostMaxErrors, } } @@ -186,7 +186,7 @@ type Options struct { Resume bool resumeCfg *ResumeCfg ExcludeCDN bool - SkipRemainingPathsOnError bool + HostMaxErrors int } // ParseOptions parses the command line options for application @@ -263,7 +263,7 @@ func ParseOptions() *Options { flag.BoolVar(&options.Probe, "probe", false, "Display probe status") flag.BoolVar(&options.Resume, "resume", false, "Resume scan using resume.cfg") flag.BoolVar(&options.ExcludeCDN, "exclude-cdn", false, "Skip full port scans for CDNs (only checks for 80,443)") - flag.BoolVar(&options.SkipRemainingPathsOnError, "skip-paths-on-error", false, "Skip remaining paths on unresponsive servers") + flag.IntVar(&options.HostMaxErrors, "host-max-errors", -1, "Allowed error count per host before skipping remaining path/s") flag.Parse() diff --git a/runner/runner.go b/runner/runner.go index 1d4a3a5c..44dd9db3 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -210,7 +210,7 @@ func New(options *Options) (*Runner, error) { } scanopts.ExcludeCDN = options.ExcludeCDN - scanopts.SkipRemainingPathsOnError = options.SkipRemainingPathsOnError + scanopts.HostMaxErrors = options.HostMaxErrors runner.scanopts = scanopts if options.ShowStatistics { @@ -232,7 +232,7 @@ func New(options *Options) (*Runner, error) { runner.ratelimiter = ratelimit.NewUnlimited() } - if options.SkipRemainingPathsOnError { + if options.HostMaxErrors >= 0 { gc := gcache.New(1000). ARC(). Build() @@ -389,7 +389,7 @@ func (r *Runner) Close() { // nolint:errcheck // ignore r.hm.Close() r.hp.Dialer.Close() - if r.options.SkipRemainingPathsOnError { + if r.options.HostMaxErrors >= 0 { r.FailedTargets.Purge() } } @@ -628,8 +628,11 @@ retry: // check if we have to skip the host:port as a result of a previous failure hostPort := net.JoinHostPort(URL.Host, URL.Port) - if r.options.SkipRemainingPathsOnError && r.FailedTargets.Has(hostPort) { - return Result{URL: domain, err: errors.New("skipping as previously unresponsive")} + if r.options.HostMaxErrors >= 0 && r.FailedTargets.Has(hostPort) { + numberOfErrors, err := r.FailedTargets.GetIFPresent(hostPort) + if err == nil && numberOfErrors.(int) >= r.options.HostMaxErrors { + return Result{URL: domain, err: errors.New("skipping as previously unresponsive")} + } } // check if the combination host:port should be skipped if belonging to a cdn @@ -746,8 +749,13 @@ retry: } // mark the host:port as failed to avoid further checks - if r.options.SkipRemainingPathsOnError { - _ = r.FailedTargets.Set(hostPort, nil) + if r.options.HostMaxErrors >= 0 { + errorCount, err := r.FailedTargets.GetIFPresent(hostPort) + if err != nil || errorCount == nil { + _ = r.FailedTargets.Set(hostPort, 1) + } else if errorCount != nil { + _ = r.FailedTargets.Set(hostPort, errorCount.(int)+1) + } } if r.options.Probe { From 0dd804c13f75fb495575f0682c876f5f576f1c6d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Aug 2021 10:08:37 +0000 Subject: [PATCH 06/28] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.9 to 0.0.10. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.9...v0.0.10) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 15 ++------------- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index dc615185..e5ab7bbb 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7dc github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.9 + github.com/projectdiscovery/wappalyzergo v0.0.10 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 go.etcd.io/bbolt v1.3.6 // indirect diff --git a/go.sum b/go.sum index 932aac50..98b32acd 100644 --- a/go.sum +++ b/go.sum @@ -138,10 +138,6 @@ github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKS github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= -github.com/projectdiscovery/fastdialer v0.0.12 h1:TjvM41UfR+A7YsxQZoTvI6C5nVe1d+fvRqtcDNbSwz8= -github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= -github.com/projectdiscovery/fastdialer v0.0.13-0.20210814183457-4084195d84b2 h1:nnzfSmfCGLcdkDtMMReY5axGqxne21GDfWZ9p76kLqs= -github.com/projectdiscovery/fastdialer v0.0.13-0.20210814183457-4084195d84b2/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= github.com/projectdiscovery/fastdialer v0.0.13-0.20210815100514-360f851a5b80 h1:Hyt1AcK5hAgE7LuaNyRD3mFc8zaS7LP4vTk6Yp83f8E= github.com/projectdiscovery/fastdialer v0.0.13-0.20210815100514-360f851a5b80/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg= @@ -168,10 +164,6 @@ github.com/projectdiscovery/mapcidr v0.0.8 h1:16U05F2x3o/jSTsxSCY2hCuCs9xOSwVxjo github.com/projectdiscovery/mapcidr v0.0.8/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= github.com/projectdiscovery/networkpolicy v0.0.1 h1:RGRuPlxE8WLFF9tdKSjTsYiTIKHNHW20Kl0nGGiRb1I= github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= -github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg= -github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= -github.com/projectdiscovery/rawhttp v0.0.8-0.20210814173504-555bdf13a5d5 h1:k/bPIQbpkBn0FjjRVT9NptSM4lnn+UYo96iu4Vp60j0= -github.com/projectdiscovery/rawhttp v0.0.8-0.20210814173504-555bdf13a5d5/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e h1:hcpGb5/gSn+kNUmzgodV1+sHDmFybuGhsuhrTqFebQY= github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f h1:HR3R/nhELwLXufUlO1ZkKVqrZl4lN1cWFBdN8RcMuLo= @@ -187,8 +179,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d h1:nl github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.9 h1:MR2A7ZmF61bT1y8A9+YVe5bsrPAiFDEQ7Ix4GbqccwQ= -github.com/projectdiscovery/wappalyzergo v0.0.9/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.10 h1:bmAjI2V99mq5mz5QikkpHQ0tXENEtaNJ5//Opr8vb5g= +github.com/projectdiscovery/wappalyzergo v0.0.10/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= @@ -250,8 +242,6 @@ golang.org/x/net v0.0.0-20210414194228-064579744ee0/go.mod h1:9tjilg8BloeKEkVJvy golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210716203947-853a461950ff h1:j2EK/QoxYNBsXI4R7fQkkRUk8y6wnOBI+6hgPdP/6Ds= -golang.org/x/net v0.0.0-20210716203947-853a461950ff/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -286,7 +276,6 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= From 89f77e50d202f4f0d8dfdeacdc83ad204e75fb54 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 16 Aug 2021 14:25:30 +0200 Subject: [PATCH 07/28] Adding support for maximum redirects number --- common/httpx/httpx.go | 20 +++++++++++++++----- common/httpx/option.go | 16 +++++++++------- runner/options.go | 2 ++ runner/runner.go | 1 + 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 33b3852e..9d21771f 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -59,20 +59,30 @@ func New(options *Options) (*HTTPX, error) { } if httpx.Options.FollowRedirects { - // Follow redirects - redirectFunc = nil + // Follow redirects up to a maximum number + redirectFunc = func(redirectedRequest *http.Request, previousRequests []*http.Request) error { + if len(previousRequests) >= options.MaxRedirects { + // https://github.com/golang/go/issues/10069 + return http.ErrUseLastResponse + } + return nil + } } if httpx.Options.FollowHostRedirects { - // Only follow redirects on the same host - redirectFunc = func(redirectedRequest *http.Request, previousRequest []*http.Request) error { + // Only follow redirects on the same host up to a maximum number + redirectFunc = func(redirectedRequest *http.Request, previousRequests []*http.Request) error { // Check if we get a redirect to a differen host var newHost = redirectedRequest.URL.Host - var oldHost = previousRequest[0].URL.Host + var oldHost = previousRequests[0].URL.Host if newHost != oldHost { // Tell the http client to not follow redirect return http.ErrUseLastResponse } + if len(previousRequests) >= options.MaxRedirects { + // https://github.com/golang/go/issues/10069 + return http.ErrUseLastResponse + } return nil } } diff --git a/common/httpx/option.go b/common/httpx/option.go index 840a4ac4..1106e9c6 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -23,6 +23,7 @@ type Options struct { VHostSimilarityRatio int FollowRedirects bool FollowHostRedirects bool + MaxRedirects int Unsafe bool TLSGrab bool // VHOSTs options @@ -39,13 +40,14 @@ type Options struct { // DefaultOptions contains the default options var DefaultOptions = Options{ - RandomAgent: true, - Threads: 25, - Timeout: 30 * time.Second, - RetryMax: 5, - Unsafe: false, - CdnCheck: true, - ExcludeCdn: false, + RandomAgent: true, + Threads: 25, + Timeout: 30 * time.Second, + RetryMax: 5, + MaxRedirects: 10, + Unsafe: false, + CdnCheck: true, + ExcludeCdn: false, // VHOSTs options VHostIgnoreStatusCode: false, VHostIgnoreContentLength: true, diff --git a/runner/options.go b/runner/options.go index 9d42c6a6..1b6bcba5 100644 --- a/runner/options.go +++ b/runner/options.go @@ -154,6 +154,7 @@ type Options struct { responseInStdout bool chainInStdout bool FollowHostRedirects bool + MaxRedirects int OutputMethod bool TLSProbe bool CSPProbe bool @@ -209,6 +210,7 @@ func ParseOptions() *Options { flag.StringVar(&options.StoreResponseDir, "srd", "output", "Save response directory") flag.BoolVar(&options.FollowRedirects, "follow-redirects", false, "Follow Redirects") flag.BoolVar(&options.FollowHostRedirects, "follow-host-redirects", false, "Only follow redirects on the same host") + flag.IntVar(&options.MaxRedirects, "max-redirects", 10, "Max number of redirects to follow") flag.StringVar(&options.HTTPProxy, "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080") flag.BoolVar(&options.JSONOutput, "json", false, "JSON Output") flag.StringVar(&options.InputFile, "l", "", "File containing domains") diff --git a/runner/runner.go b/runner/runner.go index e0798f48..577ba589 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -81,6 +81,7 @@ func New(options *Options) (*Runner, error) { httpxOptions.RetryMax = options.Retries httpxOptions.FollowRedirects = options.FollowRedirects httpxOptions.FollowHostRedirects = options.FollowHostRedirects + httpxOptions.MaxRedirects = options.MaxRedirects httpxOptions.HTTPProxy = options.HTTPProxy httpxOptions.Unsafe = options.Unsafe httpxOptions.RequestOverride = httpx.RequestOverride{URIPath: options.RequestURI} From 5931b811fbc8d5681b0800fb535691450b1870ab Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 16 Aug 2021 15:08:04 +0200 Subject: [PATCH 08/28] Fixing inconsistent "input" field behavior --- runner/runner.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index e0798f48..9ed1946b 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -513,7 +513,7 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. wg.Add() go func(target, method, protocol string) { defer wg.Done() - result := r.analyze(hp, protocol, target, method, scanopts) + result := r.analyze(hp, protocol, target, method, t, scanopts) output <- result if scanopts.TLSProbe && result.TLSData != nil { scanopts.TLSProbe = false @@ -541,7 +541,7 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. go func(port int, method, protocol string) { defer wg.Done() h, _ := urlutil.ChangePort(target, fmt.Sprint(port)) - result := r.analyze(hp, protocol, h, method, scanopts) + result := r.analyze(hp, protocol, h, method, t, scanopts) output <- result if scanopts.TLSProbe && result.TLSData != nil { scanopts.TLSProbe = false @@ -590,7 +590,7 @@ func targets(target string) chan string { return results } -func (r *Runner) analyze(hp *httpx.HTTPX, protocol, domain, method string, scanopts *scanOptions) Result { +func (r *Runner) analyze(hp *httpx.HTTPX, protocol, domain, method, origInput string, scanopts *scanOptions) Result { origProtocol := protocol if protocol == httpx.HTTPorHTTPS { protocol = httpx.HTTPS @@ -602,20 +602,20 @@ retry: parts := strings.Split(domain, ",") //nolint:gomnd // not a magic number if len(parts) != 2 { - return Result{} + return Result{Input: origInput} } domain = parts[0] customHost = parts[1] } URL, err := urlutil.Parse(domain) if err != nil { - return Result{URL: domain, err: err} + return Result{URL: domain, Input: origInput, err: err} } // check if the combination host:port should be skipped if belonging to a cdn if r.skipCDNPort(URL.Host, URL.Port) { gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port) - return Result{URL: domain, err: errors.New("cdn target only allows ports 80 and 443")} + return Result{URL: domain, Input: origInput, err: errors.New("cdn target only allows ports 80 and 443")} } URL.Scheme = protocol @@ -629,7 +629,7 @@ retry: } req, err := hp.NewRequest(method, URL.String()) if err != nil { - return Result{URL: URL.String(), err: err} + return Result{URL: URL.String(), Input: origInput, err: err} } if customHost != "" { req.Host = customHost @@ -659,7 +659,7 @@ retry: var errDump error requestDump, errDump = rawhttp.DumpRequestRaw(req.Method, req.URL.String(), reqURI, req.Header, req.Body, rawhttp.DefaultOptions) if errDump != nil { - return Result{URL: URL.String(), err: errDump} + return Result{URL: URL.String(), Input: origInput, err: errDump} } } else { // Create a copy on the fly of the request body @@ -668,7 +668,7 @@ retry: var errDump error requestDump, errDump = httputil.DumpRequestOut(req.Request, true) if errDump != nil { - return Result{URL: URL.String(), err: errDump} + return Result{URL: URL.String(), Input: origInput, err: errDump} } // The original req.Body gets modified indirectly by httputil.DumpRequestOut so we set it again to nil if it was empty // Otherwise redirects like 307/308 would fail (as they require the body to be sent along) @@ -725,9 +725,9 @@ retry: goto retry } if r.options.Probe { - return Result{URL: URL.String(), Input: domain, Timestamp: time.Now(), err: err, Failed: err != nil, Error: errString, str: builder.String()} + return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), err: err, Failed: err != nil, Error: errString, str: builder.String()} } else { - return Result{URL: URL.String(), Input: domain, Timestamp: time.Now(), err: err} + return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), err: err} } } @@ -972,7 +972,7 @@ retry: parsed, err := urlutil.Parse(fullURL) if err != nil { - return Result{URL: fullURL, err: errors.Wrap(err, "could not parse url")} + return Result{URL: fullURL, Input: origInput, err: errors.Wrap(err, "could not parse url")} } finalPort := parsed.Port @@ -1016,7 +1016,7 @@ retry: HeaderSHA256: headersSha, raw: resp.Raw, URL: fullURL, - Input: domain, + Input: origInput, ContentLength: resp.ContentLength, ChainStatusCodes: chainStatusCodes, Chain: chainItems, From 5919040a93ec3c2e0792a174bfd193a7ed83c8b6 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 16 Aug 2021 15:53:28 +0200 Subject: [PATCH 09/28] Fixing inconsistent behavior for custom ports with schemes --- common/customports/customport.go | 13 ++++++++++ common/httpx/http2.go | 4 ++- runner/runner.go | 44 ++++++++++++++++++-------------- 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/common/customports/customport.go b/common/customports/customport.go index 8a671869..49a53545 100644 --- a/common/customports/customport.go +++ b/common/customports/customport.go @@ -43,12 +43,20 @@ func (c *CustomPorts) Set(value string) error { } else if strings.HasPrefix(potentialPort, httpx.HTTPS+":") { potentialPort = strings.TrimPrefix(potentialPort, httpx.HTTPS+":") protocol = httpx.HTTPS + } else if strings.HasPrefix(potentialPort, httpx.HTTPandHTTPS+":") { + potentialPort = strings.TrimPrefix(potentialPort, httpx.HTTPandHTTPS+":") + protocol = httpx.HTTPandHTTPS } potentialRange := strings.Split(potentialPort, "-") // it's a single port? if len(potentialRange) < portRangeParts { if p, err := strconv.Atoi(potentialPort); err == nil { + if existingProtocol, ok := Ports[p]; ok { + if existingProtocol == httpx.HTTP && protocol == httpx.HTTPS || existingProtocol == httpx.HTTPS && protocol == httpx.HTTP { + protocol = httpx.HTTPandHTTPS + } + } Ports[p] = protocol } else { gologger.Warning().Msgf("Could not cast port to integer, your value: %s, resulting error %s. Skipping it\n", @@ -79,6 +87,11 @@ func (c *CustomPorts) Set(value string) error { } for i := lowP; i <= highP; i++ { + if existingProtocol, ok := Ports[i]; ok { + if existingProtocol == httpx.HTTP && protocol == httpx.HTTPS || existingProtocol == httpx.HTTPS && protocol == httpx.HTTP { + protocol = httpx.HTTPandHTTPS + } + } Ports[i] = protocol } } diff --git a/common/httpx/http2.go b/common/httpx/http2.go index 83f827c6..a1f0bb33 100644 --- a/common/httpx/http2.go +++ b/common/httpx/http2.go @@ -16,8 +16,10 @@ const ( HTTP = "http" // HTTPS defines the secure http scheme HTTPS = "https" - // HTTPorHTTPS defines the both http and https scheme + // HTTPorHTTPS defines both http and https scheme in mutual exclusion HTTPorHTTPS = "http|https" + // HTTPandHTTPS defines both http and https scheme + HTTPandHTTPS = "http&https" ) // SupportHTTP2 checks if the target host supports HTTP2 diff --git a/runner/runner.go b/runner/runner.go index e0798f48..73f0ef71 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -501,7 +501,7 @@ func (r *Runner) RunEnumeration() { func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.HTTPX, protocol string, scanopts *scanOptions, output chan Result) { protocols := []string{protocol} - if scanopts.NoFallback { + if scanopts.NoFallback || protocol == httpx.HTTPandHTTPS { protocols = []string{httpx.HTTPS, httpx.HTTP} } @@ -535,24 +535,30 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. } } - for port, wantedProtocol := range customport.Ports { - for _, method := range scanopts.Methods { - wg.Add() - go func(port int, method, protocol string) { - defer wg.Done() - h, _ := urlutil.ChangePort(target, fmt.Sprint(port)) - result := r.analyze(hp, protocol, h, method, scanopts) - output <- result - if scanopts.TLSProbe && result.TLSData != nil { - scanopts.TLSProbe = false - for _, tt := range result.TLSData.DNSNames { - r.process(tt, wg, hp, protocol, scanopts, output) - } - for _, tt := range result.TLSData.CommonName { - r.process(tt, wg, hp, protocol, scanopts, output) + for port, wantedProtocolForPort := range customport.Ports { + wantedProtocols := []string{wantedProtocolForPort} + if wantedProtocolForPort == httpx.HTTPandHTTPS { + wantedProtocols = []string{httpx.HTTPS, httpx.HTTP} + } + for _, wantedProtocol := range wantedProtocols { + for _, method := range scanopts.Methods { + wg.Add() + go func(port int, method, protocol string) { + defer wg.Done() + h, _ := urlutil.ChangePort(target, fmt.Sprint(port)) + result := r.analyze(hp, protocol, h, method, scanopts) + output <- result + if scanopts.TLSProbe && result.TLSData != nil { + scanopts.TLSProbe = false + for _, tt := range result.TLSData.DNSNames { + r.process(tt, wg, hp, protocol, scanopts, output) + } + for _, tt := range result.TLSData.CommonName { + r.process(tt, wg, hp, protocol, scanopts, output) + } } - } - }(port, method, wantedProtocol) + }(port, method, wantedProtocol) + } } } if r.options.ShowStatistics { @@ -592,7 +598,7 @@ func targets(target string) chan string { func (r *Runner) analyze(hp *httpx.HTTPX, protocol, domain, method string, scanopts *scanOptions) Result { origProtocol := protocol - if protocol == httpx.HTTPorHTTPS { + if protocol == httpx.HTTPorHTTPS || protocol == httpx.HTTPandHTTPS { protocol = httpx.HTTPS } retried := false From 38d56d5de94a130aa9b823f45d7ae0e64e4cc2f2 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 16 Aug 2021 16:09:35 +0200 Subject: [PATCH 10/28] Improving case behavior with methods CLI option --- runner/options.go | 2 +- runner/runner.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/runner/options.go b/runner/options.go index 9d42c6a6..83c23ccf 100644 --- a/runner/options.go +++ b/runner/options.go @@ -212,7 +212,7 @@ func ParseOptions() *Options { flag.StringVar(&options.HTTPProxy, "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080") flag.BoolVar(&options.JSONOutput, "json", false, "JSON Output") flag.StringVar(&options.InputFile, "l", "", "File containing domains") - flag.StringVar(&options.Methods, "x", "", "Request Methods, use ALL to check all verbs ()") + flag.StringVar(&options.Methods, "x", "", "Request Methods, use ALL to check all verbs (GET, POST, PUT, PATCH, DELETE, CONNECT, OPTIONS and TRACE)") flag.BoolVar(&options.OutputMethod, "method", false, "Display request method") flag.BoolVar(&options.Silent, "silent", false, "Silent mode") flag.BoolVar(&options.Version, "version", false, "Show version of httpx") diff --git a/runner/runner.go b/runner/runner.go index e0798f48..25c0e9ad 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -155,6 +155,10 @@ func New(options *Options) (*Runner, error) { if strings.EqualFold(options.Methods, "all") { scanopts.Methods = pdhttputil.AllHTTPMethods() } else if options.Methods != "" { + // if unsafe is specified then converts the methods to uppercase + if !options.Unsafe { + options.Methods = strings.ToUpper(options.Methods) + } scanopts.Methods = append(scanopts.Methods, stringz.SplitByCharAndTrimSpace(options.Methods, ",")...) } if len(scanopts.Methods) == 0 { From d529a41c8d31cfb141d6636bac435fd00d1b13ae Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 16 Aug 2021 21:08:03 +0530 Subject: [PATCH 11/28] Update options.go --- runner/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/options.go b/runner/options.go index 82d94cc2..ac119cba 100644 --- a/runner/options.go +++ b/runner/options.go @@ -263,7 +263,7 @@ func ParseOptions() *Options { flag.BoolVar(&options.Probe, "probe", false, "Display probe status") flag.BoolVar(&options.Resume, "resume", false, "Resume scan using resume.cfg") flag.BoolVar(&options.ExcludeCDN, "exclude-cdn", false, "Skip full port scans for CDNs (only checks for 80,443)") - flag.IntVar(&options.HostMaxErrors, "host-max-errors", -1, "Allowed error count per host before skipping remaining path/s") + flag.IntVar(&options.HostMaxErrors, "max-host-error", -1, "Max error count per host before skipping remaining path/s") flag.Parse() From 1e2e2d54e913b61e934d97f78c454364a4e7c5ed Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 16 Aug 2021 19:10:12 +0200 Subject: [PATCH 12/28] Handling edge case with head method and unresponsive server --- common/httpx/httpx.go | 31 +++++++++++++++++++++++++------ go.mod | 2 +- go.sum | 2 ++ 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 33b3852e..b3873718 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "net/http" "net/url" + "strconv" "strings" "time" "unicode/utf8" @@ -18,6 +19,7 @@ import ( pdhttputil "github.com/projectdiscovery/httputil" "github.com/projectdiscovery/rawhttp" retryablehttp "github.com/projectdiscovery/retryablehttp-go" + "github.com/projectdiscovery/stringsutil" "golang.org/x/net/http2" ) @@ -134,6 +136,13 @@ get_response: return nil, err } + var shouldIgnoreErrors, shouldIgnoreBodyErrors bool + switch { + case h.Options.Unsafe && req.Method == http.MethodHead && !stringsutil.ContainsAny("i/o timeout"): + shouldIgnoreErrors = true + shouldIgnoreBodyErrors = true + } + var resp Response resp.Headers = httpresp.Header.Clone() @@ -148,23 +157,25 @@ get_response: req.Header.Set("Accept-Encoding", "identity") goto get_response } - return nil, err + if !shouldIgnoreErrors { + return nil, err + } } - resp.Raw = rawResp - resp.RawHeaders = headers + resp.Raw = string(rawResp) + resp.RawHeaders = string(headers) var respbody []byte // websockets don't have a readable body if httpresp.StatusCode != http.StatusSwitchingProtocols { var err error respbody, err = ioutil.ReadAll(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead)) - if err != nil { + if err != nil && !shouldIgnoreBodyErrors { return nil, err } } closeErr := httpresp.Body.Close() - if closeErr != nil { + if closeErr != nil && !shouldIgnoreBodyErrors { return nil, closeErr } @@ -175,7 +186,15 @@ get_response: respbodystr = h.htmlPolicy.Sanitize(respbodystr) } - resp.ContentLength = utf8.RuneCountInString(respbodystr) + if contentLength, ok := resp.Headers["Content-Length"]; ok { + contentLengthInt, err := strconv.Atoi(strings.Join(contentLength, "")) + if err != nil { + resp.ContentLength = utf8.RuneCountInString(respbodystr) + } else { + resp.ContentLength = contentLengthInt + } + } + resp.Data = respbody // fill metrics diff --git a/go.mod b/go.mod index e5ab7bbb..cfd6dde2 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa - github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d + github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 github.com/projectdiscovery/iputil v0.0.0-20210705072957-5a968407979b github.com/projectdiscovery/mapcidr v0.0.8 github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e diff --git a/go.sum b/go.sum index 98b32acd..e5851680 100644 --- a/go.sum +++ b/go.sum @@ -153,6 +153,8 @@ github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa h1:KeN6/bZ github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8= github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d h1:IdBTOSGaPrZ8+FK0uYMQIva9dYIR5F55PLFWYtBBKc0= github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d/go.mod h1:Vm2DY4NwUV5yA6TNzJOOjTYGjTcVfuEN8m9Y5dAksLQ= +github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 h1:GzruqQhb+sj1rEuHRFLhWX8gH/tJ+sj1udRjOy9VCJo= +github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5/go.mod h1:BueJPSPWAX11IFS6bdAqTkekiIz5Fgco5LVc1kqO9L4= github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= From 0f4f9f2cac38619994885f4784ff249794362e67 Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 16 Aug 2021 23:52:52 +0530 Subject: [PATCH 13/28] minor update --- runner/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/options.go b/runner/options.go index 1b6bcba5..97615134 100644 --- a/runner/options.go +++ b/runner/options.go @@ -210,7 +210,7 @@ func ParseOptions() *Options { flag.StringVar(&options.StoreResponseDir, "srd", "output", "Save response directory") flag.BoolVar(&options.FollowRedirects, "follow-redirects", false, "Follow Redirects") flag.BoolVar(&options.FollowHostRedirects, "follow-host-redirects", false, "Only follow redirects on the same host") - flag.IntVar(&options.MaxRedirects, "max-redirects", 10, "Max number of redirects to follow") + flag.IntVar(&options.MaxRedirects, "max-redirects", 10, "Max number of redirects to follow per host") flag.StringVar(&options.HTTPProxy, "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080") flag.BoolVar(&options.JSONOutput, "json", false, "JSON Output") flag.StringVar(&options.InputFile, "l", "", "File containing domains") From 6bc6359f3dbdc821a183e214e563f8b74d1d00c9 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 16 Aug 2021 22:23:03 +0200 Subject: [PATCH 14/28] small name refactor --- runner/runner.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index ed83520e..b2b4cbbb 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -54,14 +54,14 @@ const ( // Runner is a client for running the enumeration process. type Runner struct { - options *Options - hp *httpx.HTTPX - wappalyzer *wappalyzer.Wappalyze - scanopts scanOptions - hm *hybrid.HybridMap - stats clistats.StatisticsClient - ratelimiter ratelimit.Limiter - FailedTargets gcache.Cache + options *Options + hp *httpx.HTTPX + wappalyzer *wappalyzer.Wappalyze + scanopts scanOptions + hm *hybrid.HybridMap + stats clistats.StatisticsClient + ratelimiter ratelimit.Limiter + HostErrorsCache gcache.Cache } // New creates a new client for running enumeration process. @@ -237,7 +237,7 @@ func New(options *Options) (*Runner, error) { gc := gcache.New(1000). ARC(). Build() - runner.FailedTargets = gc + runner.HostErrorsCache = gc } return runner, nil @@ -391,7 +391,7 @@ func (r *Runner) Close() { r.hm.Close() r.hp.Dialer.Close() if r.options.HostMaxErrors >= 0 { - r.FailedTargets.Purge() + r.HostErrorsCache.Purge() } } @@ -629,8 +629,8 @@ retry: // check if we have to skip the host:port as a result of a previous failure hostPort := net.JoinHostPort(URL.Host, URL.Port) - if r.options.HostMaxErrors >= 0 && r.FailedTargets.Has(hostPort) { - numberOfErrors, err := r.FailedTargets.GetIFPresent(hostPort) + if r.options.HostMaxErrors >= 0 && r.HostErrorsCache.Has(hostPort) { + numberOfErrors, err := r.HostErrorsCache.GetIFPresent(hostPort) if err == nil && numberOfErrors.(int) >= r.options.HostMaxErrors { return Result{URL: domain, err: errors.New("skipping as previously unresponsive")} } @@ -751,11 +751,11 @@ retry: // mark the host:port as failed to avoid further checks if r.options.HostMaxErrors >= 0 { - errorCount, err := r.FailedTargets.GetIFPresent(hostPort) + errorCount, err := r.HostErrorsCache.GetIFPresent(hostPort) if err != nil || errorCount == nil { - _ = r.FailedTargets.Set(hostPort, 1) + _ = r.HostErrorsCache.Set(hostPort, 1) } else if errorCount != nil { - _ = r.FailedTargets.Set(hostPort, errorCount.(int)+1) + _ = r.HostErrorsCache.Set(hostPort, errorCount.(int)+1) } } From b7d1d1794905ea9483c5838f467bae0f0fdcaa05 Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 17 Aug 2021 17:24:23 +0530 Subject: [PATCH 15/28] version update --- runner/banner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runner/banner.go b/runner/banner.go index 713adc0a..2beadea9 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -8,11 +8,11 @@ const banner = ` / __ \/ __/ __/ __ \| / / / / / /_/ /_/ /_/ / | /_/ /_/\__/\__/ .___/_/|_| - /_/ v1.1.1 + /_/ v1.1.2 ` // Version is the current version of httpx -const Version = `v1.1.1` +const Version = `v1.1.2` // showBanner is used to show the banner to the user func showBanner() { From b07da83b5ec5fae6f6ee683218b67e2a7f08eda8 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 17 Aug 2021 14:42:14 +0200 Subject: [PATCH 16/28] Adding cli options dedupe for ports --- common/customports/customport.go | 3 ++- go.mod | 3 ++- go.sum | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/common/customports/customport.go b/common/customports/customport.go index 49a53545..7728b78d 100644 --- a/common/customports/customport.go +++ b/common/customports/customport.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/sliceutil" "github.com/projectdiscovery/httpx/common/httpx" ) @@ -31,7 +32,7 @@ func (c *CustomPorts) String() string { func (c *CustomPorts) Set(value string) error { // ports can be like nmap -p [https|http:]start-end,[https|http:]port1,[https|http:]port2,[https|http:]port3 // splits on comma - potentialPorts := strings.Split(value, ",") + potentialPorts := sliceutil.Dedupe(strings.Split(value, ",")) // check if port is a single integer value or needs to be expanded further for _, potentialPort := range potentialPorts { diff --git a/go.mod b/go.mod index cfb893cb..62a745b4 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.14 require ( github.com/akrylysov/pogreb v0.10.1 // indirect - github.com/bluele/gcache v0.0.2 // indirect + github.com/bluele/gcache v0.0.2 github.com/corpix/uarand v0.1.1 github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect @@ -28,6 +28,7 @@ require ( github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e github.com/projectdiscovery/retryabledns v1.0.12 // indirect github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7dc + github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 github.com/projectdiscovery/wappalyzergo v0.0.10 diff --git a/go.sum b/go.sum index 21554c9b..82f509bb 100644 --- a/go.sum +++ b/go.sum @@ -153,8 +153,6 @@ github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8q github.com/projectdiscovery/hmap v0.0.2-0.20210616215655-7b78e7f33d1f/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8= github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa h1:KeN6/bZOVxtS4XkgzRvYxpXWZSZt+AoGP5Myyr3/Duk= github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8= -github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d h1:IdBTOSGaPrZ8+FK0uYMQIva9dYIR5F55PLFWYtBBKc0= -github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d/go.mod h1:Vm2DY4NwUV5yA6TNzJOOjTYGjTcVfuEN8m9Y5dAksLQ= github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 h1:GzruqQhb+sj1rEuHRFLhWX8gH/tJ+sj1udRjOy9VCJo= github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5/go.mod h1:BueJPSPWAX11IFS6bdAqTkekiIz5Fgco5LVc1kqO9L4= github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= @@ -178,6 +176,8 @@ github.com/projectdiscovery/retryabledns v1.0.12/go.mod h1:4sMC8HZyF01HXukRleSQY github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7dc h1:769c7sQOl9BP8dhE8uv0mQX9WmcXo6Jzv//Nm+qAf50= github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7dc/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= +github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea h1:S+DC2tmKG93Om42cnTqrBfIv699pwSIhafqZvip+RIA= +github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea/go.mod h1:QHXvznfPfA5f0AZUIBkbLapoUJJlsIDgUlkKva6dOr4= github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4= github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d h1:nlOAex7twmrEqD5i6WLnugF9uO3DQ6jDEKN9gevrTAk= github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4= From 57c1244a165b9c59e603a163d3ab3231c014e5d2 Mon Sep 17 00:00:00 2001 From: mzack Date: Sun, 22 Aug 2021 19:26:28 +0200 Subject: [PATCH 17/28] Fixing input URLs handling with non-rfc paths --- common/httpx/httpx.go | 36 +++++++++++++++++------------------- common/httpx/option.go | 2 +- common/httpx/virtualhost.go | 6 +++--- common/stringz/stringz.go | 11 +++++++++++ runner/runner.go | 29 ++++++++++++++++++++++------- 5 files changed, 54 insertions(+), 30 deletions(-) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index e95e3200..ac0986fe 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -25,15 +25,14 @@ import ( // HTTPX represent an instance of the library client type HTTPX struct { - client *retryablehttp.Client - client2 *http.Client - Filters []Filter - Options *Options - htmlPolicy *bluemonday.Policy - CustomHeaders map[string]string - RequestOverride *RequestOverride - cdn *cdncheck.Client - Dialer *fastdialer.Dialer + client *retryablehttp.Client + client2 *http.Client + Filters []Filter + Options *Options + htmlPolicy *bluemonday.Policy + CustomHeaders map[string]string + cdn *cdncheck.Client + Dialer *fastdialer.Dialer } // New httpx instance @@ -124,7 +123,6 @@ func New(options *Options) (*HTTPX, error) { httpx.htmlPolicy = bluemonday.NewPolicy() httpx.CustomHeaders = httpx.Options.CustomHeaders - httpx.RequestOverride = &options.RequestOverride if options.CdnCheck || options.ExcludeCdn { httpx.cdn, err = cdncheck.NewWithCache() if err != nil { @@ -136,12 +134,12 @@ func New(options *Options) (*HTTPX, error) { } // Do http request -func (h *HTTPX) Do(req *retryablehttp.Request) (*Response, error) { +func (h *HTTPX) Do(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (*Response, error) { timeStart := time.Now() var gzipRetry bool get_response: - httpresp, err := h.getResponse(req) + httpresp, err := h.getResponse(req, unsafeOptions) if err != nil { return nil, err } @@ -236,33 +234,33 @@ get_response: } // RequestOverride contains the URI path to override the request -type RequestOverride struct { +type UnsafeOptions struct { URIPath string } // getResponse returns response from safe / unsafe request -func (h *HTTPX) getResponse(req *retryablehttp.Request) (*http.Response, error) { +func (h *HTTPX) getResponse(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (*http.Response, error) { if h.Options.Unsafe { - return h.doUnsafe(req) + return h.doUnsafeWithOptions(req, unsafeOptions) } return h.client.Do(req) } // doUnsafe does an unsafe http request -func (h *HTTPX) doUnsafe(req *retryablehttp.Request) (*http.Response, error) { +func (h *HTTPX) doUnsafeWithOptions(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (*http.Response, error) { method := req.Method headers := req.Header targetURL := req.URL.String() body := req.Body options := rawhttp.DefaultOptions options.Timeout = h.Options.Timeout - return rawhttp.DoRawWithOptions(method, targetURL, h.RequestOverride.URIPath, headers, body, options) + return rawhttp.DoRawWithOptions(method, targetURL, unsafeOptions.URIPath, headers, body, options) } // Verify the http calls and apply-cascade all the filters, as soon as one matches it returns true -func (h *HTTPX) Verify(req *retryablehttp.Request) (bool, error) { - resp, err := h.Do(req) +func (h *HTTPX) Verify(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (bool, error) { + resp, err := h.Do(req, unsafeOptions) if err != nil { return false, err } diff --git a/common/httpx/option.go b/common/httpx/option.go index 1106e9c6..46d6070f 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -8,7 +8,6 @@ import ( type Options struct { RandomAgent bool DefaultUserAgent string - RequestOverride RequestOverride HTTPProxy string SocksProxy string Threads int @@ -36,6 +35,7 @@ type Options struct { Deny []string MaxResponseBodySizeToSave int64 MaxResponseBodySizeToRead int64 + UnsafeURI string } // DefaultOptions contains the default options diff --git a/common/httpx/virtualhost.go b/common/httpx/virtualhost.go index 0a4eee53..77e3d479 100644 --- a/common/httpx/virtualhost.go +++ b/common/httpx/virtualhost.go @@ -11,8 +11,8 @@ import ( const simMultiplier = 100 // IsVirtualHost checks if the target endpoint is a virtual host -func (h *HTTPX) IsVirtualHost(req *retryablehttp.Request) (bool, error) { - httpresp1, err := h.Do(req) +func (h *HTTPX) IsVirtualHost(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (bool, error) { + httpresp1, err := h.Do(req, unsafeOptions) if err != nil { return false, err } @@ -20,7 +20,7 @@ func (h *HTTPX) IsVirtualHost(req *retryablehttp.Request) (bool, error) { // request a non-existing endpoint req.Host = fmt.Sprintf("%s.%s", xid.New().String(), req.Host) - httpresp2, err := h.Do(req) + httpresp2, err := h.Do(req, unsafeOptions) if err != nil { return false, err } diff --git a/common/stringz/stringz.go b/common/stringz/stringz.go index 293b9f85..34eb790c 100644 --- a/common/stringz/stringz.go +++ b/common/stringz/stringz.go @@ -1,6 +1,7 @@ package stringz import ( + "net/url" "strconv" "strings" @@ -73,3 +74,13 @@ func RemoveURLDefaultPort(rawURL string) string { } return u.String() } + +func GetInvalidURI(rawURL string) (bool, string) { + if _, err := url.Parse(rawURL); err != nil { + if u, err := urlutil.Parse(rawURL); err == nil { + return true, u.RequestURI + } + } + + return false, "" +} diff --git a/runner/runner.go b/runner/runner.go index e6eb8045..048dbdc2 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -87,7 +87,7 @@ func New(options *Options) (*Runner, error) { httpxOptions.MaxRedirects = options.MaxRedirects httpxOptions.HTTPProxy = options.HTTPProxy httpxOptions.Unsafe = options.Unsafe - httpxOptions.RequestOverride = httpx.RequestOverride{URIPath: options.RequestURI} + httpxOptions.UnsafeURI = options.RequestURI httpxOptions.CdnCheck = options.OutputCDN httpxOptions.ExcludeCdn = options.ExcludeCDN httpxOptions.RandomAgent = options.RandomAgent @@ -658,19 +658,25 @@ retry: URL.Port = "" } - if !scanopts.Unsafe { + var reqURI string + // retry with unsafe + if scanopts.Unsafe { + reqURI = URL.RequestURI + scanopts.RequestURI + // then create a base request without it to avoid go errors + URL.RequestURI = "" + } else { + // in case of standard requests append the new path to the existing one URL.RequestURI += scanopts.RequestURI } req, err := hp.NewRequest(method, URL.String()) if err != nil { return Result{URL: URL.String(), Input: origInput, err: err} } + if customHost != "" { req.Host = customHost } - reqURI := req.URL.RequestURI() - hp.SetCustomHeaders(req, hp.CustomHeaders) // We set content-length even if zero to allow net/http to follow 307/308 redirects (it fails on unknown size) if scanopts.RequestBody != "" { @@ -683,7 +689,11 @@ retry: r.ratelimiter.Take() - resp, err := hp.Do(req) + // with rawhttp we should say to the server to close the connection, otherwise it will remain open + if scanopts.Unsafe { + req.Header.Add("Connection", "close") + } + resp, err := hp.Do(req, httpx.UnsafeOptions{URIPath: reqURI}) if r.options.ShowStatistics { r.stats.IncrementCounter("requests", 1) } @@ -719,7 +729,12 @@ retry: // if the full url doesn't end with the custom path we pick the original input value if !stringsutil.HasSuffixAny(fullURL, scanopts.RequestURI) { parsedURL, _ := urlutil.Parse(fullURL) - parsedURL.RequestURI = scanopts.RequestURI + if scanopts.Unsafe { + parsedURL.RequestURI = reqURI + } else { + parsedURL.RequestURI = scanopts.RequestURI + } + fullURL = parsedURL.String() } builder.WriteString(stringz.RemoveURLDefaultPort(fullURL)) @@ -870,7 +885,7 @@ retry: isvhost := false if scanopts.VHost { r.ratelimiter.Take() - isvhost, _ = hp.IsVirtualHost(req) + isvhost, _ = hp.IsVirtualHost(req, httpx.UnsafeOptions{}) if isvhost { builder.WriteString(" [vhost]") } From 8d47e9466ecec2a8e7281152d52b9767d7eaaeea Mon Sep 17 00:00:00 2001 From: mzack Date: Sun, 22 Aug 2021 19:54:52 +0200 Subject: [PATCH 18/28] fixing output url path --- runner/runner.go | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 048dbdc2..38c0577d 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -722,21 +722,18 @@ retry: } } + // fix the final output url fullURL := req.URL.String() + parsedURL, _ := urlutil.Parse(fullURL) + if r.options.Unsafe { + parsedURL.RequestURI = reqURI + // if the full url doesn't end with the custom path we pick the original input value + } else if !stringsutil.HasSuffixAny(fullURL, scanopts.RequestURI) { + parsedURL.RequestURI = scanopts.RequestURI + } + fullURL = parsedURL.String() builder := &strings.Builder{} - - // if the full url doesn't end with the custom path we pick the original input value - if !stringsutil.HasSuffixAny(fullURL, scanopts.RequestURI) { - parsedURL, _ := urlutil.Parse(fullURL) - if scanopts.Unsafe { - parsedURL.RequestURI = reqURI - } else { - parsedURL.RequestURI = scanopts.RequestURI - } - - fullURL = parsedURL.String() - } builder.WriteString(stringz.RemoveURLDefaultPort(fullURL)) if r.options.Probe { From 7a59d6702bfd58ae2f69795e83945720426fa0d4 Mon Sep 17 00:00:00 2001 From: mzack Date: Sun, 22 Aug 2021 21:07:23 +0200 Subject: [PATCH 19/28] Improving cli parsing for Allow/Deny option --- common/customlist/customlist.go | 5 ++++- common/fileutil/fileutil.go | 23 +++++++++++++++++++++++ go.mod | 1 + go.sum | 4 ++++ runner/options.go | 5 +++-- runner/runner.go | 7 ++++--- 6 files changed, 39 insertions(+), 6 deletions(-) diff --git a/common/customlist/customlist.go b/common/customlist/customlist.go index 8c7fa45b..37abe39c 100644 --- a/common/customlist/customlist.go +++ b/common/customlist/customlist.go @@ -1,5 +1,7 @@ package customlist +import "github.com/projectdiscovery/httpx/common/fileutil" + // CustomList for fastdialer type CustomList []string @@ -10,6 +12,7 @@ func (c *CustomList) String() string { // Set a new global header func (c *CustomList) Set(value string) error { - *c = append(*c, value) + values := fileutil.LoadCidrsFromSliceOrFile(value) + *c = append(*c, values...) return nil } diff --git a/common/fileutil/fileutil.go b/common/fileutil/fileutil.go index f5814865..28046461 100644 --- a/common/fileutil/fileutil.go +++ b/common/fileutil/fileutil.go @@ -3,9 +3,14 @@ package fileutil import ( "bufio" "errors" + "io/ioutil" + "net" "os" "path/filepath" "regexp" + + "github.com/projectdiscovery/fileutil" + "github.com/projectdiscovery/httpx/common/stringz" ) // FileExists checks if a file exists and is not a directory @@ -70,3 +75,21 @@ func FileNameIsGlob(pattern string) bool { _, err := regexp.Compile(pattern) return err == nil } + +func LoadCidrsFromSliceOrFile(option string) (networkList []string) { + items := stringz.SplitByCharAndTrimSpace(option, ",") + for _, item := range items { + // ip + if net.ParseIP(item) != nil { + networkList = append(networkList, item) + } else if _, _, err := net.ParseCIDR(item); err == nil { + networkList = append(networkList, item) + } else if fileutil.FileExists(item) { + if filedata, err := ioutil.ReadFile(item); err == nil && len(filedata) > 0 { + networkList = append(networkList, LoadCidrsFromSliceOrFile(string(filedata))...) + } + } + } + + return networkList +} diff --git a/go.mod b/go.mod index 62a745b4..a4e5fd41 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 github.com/projectdiscovery/fastdialer v0.0.13-0.20210815100514-360f851a5b80 github.com/projectdiscovery/fdmax v0.0.3 + github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa diff --git a/go.sum b/go.sum index 82f509bb..c887bb7d 100644 --- a/go.sum +++ b/go.sum @@ -87,6 +87,8 @@ github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMW github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= +github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -144,6 +146,8 @@ github.com/projectdiscovery/fastdialer v0.0.13-0.20210815100514-360f851a5b80 h1: github.com/projectdiscovery/fastdialer v0.0.13-0.20210815100514-360f851a5b80/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg= github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI= +github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca h1:xT//ApxoeQRbt9GgL/122688bhGy9hur8YG0Qh69k5I= +github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= diff --git a/runner/options.go b/runner/options.go index 89995470..ce800869 100644 --- a/runner/options.go +++ b/runner/options.go @@ -6,6 +6,7 @@ import ( "os" "regexp" + "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/goconfig" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" @@ -13,7 +14,7 @@ import ( "github.com/projectdiscovery/httpx/common/customheader" "github.com/projectdiscovery/httpx/common/customlist" customport "github.com/projectdiscovery/httpx/common/customports" - "github.com/projectdiscovery/httpx/common/fileutil" + fileutilz "github.com/projectdiscovery/httpx/common/fileutil" "github.com/projectdiscovery/httpx/common/stringz" ) @@ -290,7 +291,7 @@ func ParseOptions() *Options { } func (options *Options) validateOptions() { - if options.InputFile != "" && !fileutil.FileNameIsGlob(options.InputFile) && !fileutil.FileExists(options.InputFile) { + if options.InputFile != "" && !fileutilz.FileNameIsGlob(options.InputFile) && !fileutil.FileExists(options.InputFile) { gologger.Fatal().Msgf("File %s does not exist!\n", options.InputFile) } diff --git a/runner/runner.go b/runner/runner.go index e6eb8045..9bf61701 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -31,11 +31,12 @@ import ( // automatic fd max increase if running as root _ "github.com/projectdiscovery/fdmax/autofdmax" + "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/hmap/store/hybrid" pdhttputil "github.com/projectdiscovery/httputil" customport "github.com/projectdiscovery/httpx/common/customports" - "github.com/projectdiscovery/httpx/common/fileutil" + fileutilz "github.com/projectdiscovery/httpx/common/fileutil" "github.com/projectdiscovery/httpx/common/httputilz" "github.com/projectdiscovery/httpx/common/httpx" "github.com/projectdiscovery/httpx/common/slice" @@ -250,7 +251,7 @@ func New(options *Options) (*Runner, error) { func (r *Runner) prepareInput() { // Check if the user requested multiple paths if fileutil.FileExists(r.options.RequestURIs) { - r.options.requestURIs = fileutil.LoadFile(r.options.RequestURIs) + r.options.requestURIs = fileutilz.LoadFile(r.options.RequestURIs) } else if r.options.RequestURIs != "" { r.options.requestURIs = strings.Split(r.options.RequestURIs, ",") } @@ -267,7 +268,7 @@ func (r *Runner) prepareInput() { gologger.Fatal().Msgf("Could read input file '%s': %s\n", r.options.InputFile, err) } } else if r.options.InputFile != "" { - files, err := fileutil.ListFilesWithPattern(r.options.InputFile) + files, err := fileutilz.ListFilesWithPattern(r.options.InputFile) if err != nil { gologger.Fatal().Msgf("No input provided: %s", err) } From 6804cda5e4769a206669fc1a78afa4a56aa426c8 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 23 Aug 2021 01:22:56 +0200 Subject: [PATCH 20/28] using new line as split token for data from file --- common/customlist/customlist.go | 2 +- common/fileutil/fileutil.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/common/customlist/customlist.go b/common/customlist/customlist.go index 37abe39c..4bf05f37 100644 --- a/common/customlist/customlist.go +++ b/common/customlist/customlist.go @@ -12,7 +12,7 @@ func (c *CustomList) String() string { // Set a new global header func (c *CustomList) Set(value string) error { - values := fileutil.LoadCidrsFromSliceOrFile(value) + values := fileutil.LoadCidrsFromSliceOrFile(value, ",") *c = append(*c, values...) return nil } diff --git a/common/fileutil/fileutil.go b/common/fileutil/fileutil.go index 28046461..df5ce289 100644 --- a/common/fileutil/fileutil.go +++ b/common/fileutil/fileutil.go @@ -76,8 +76,8 @@ func FileNameIsGlob(pattern string) bool { return err == nil } -func LoadCidrsFromSliceOrFile(option string) (networkList []string) { - items := stringz.SplitByCharAndTrimSpace(option, ",") +func LoadCidrsFromSliceOrFile(option string, splitchar string) (networkList []string) { + items := stringz.SplitByCharAndTrimSpace(option, splitchar) for _, item := range items { // ip if net.ParseIP(item) != nil { @@ -86,7 +86,7 @@ func LoadCidrsFromSliceOrFile(option string) (networkList []string) { networkList = append(networkList, item) } else if fileutil.FileExists(item) { if filedata, err := ioutil.ReadFile(item); err == nil && len(filedata) > 0 { - networkList = append(networkList, LoadCidrsFromSliceOrFile(string(filedata))...) + networkList = append(networkList, LoadCidrsFromSliceOrFile(string(filedata), "\n")...) } } } From 596936b7ee7aa5d8a44a1dbea57b3467a46f462d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Aug 2021 10:04:22 +0000 Subject: [PATCH 21/28] chore(deps): bump golang from 1.16.7-alpine to 1.17.0-alpine Bumps golang from 1.16.7-alpine to 1.17.0-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3e63374c..2928e1c0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.16.7-alpine AS builder +FROM golang:1.17.0-alpine AS builder RUN apk add --no-cache git RUN GO111MODULE=on go get -v github.com/projectdiscovery/httpx/cmd/httpx From bd28fb02b6b9c1ae8d680c5046a9b2fa7b9134ed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Aug 2021 10:07:58 +0000 Subject: [PATCH 22/28] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.10 to 0.0.11. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.10...v0.0.11) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 62a745b4..2dac1712 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.10 + github.com/projectdiscovery/wappalyzergo v0.0.11 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 go.etcd.io/bbolt v1.3.6 // indirect diff --git a/go.sum b/go.sum index 82f509bb..acfa046f 100644 --- a/go.sum +++ b/go.sum @@ -183,8 +183,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d h1:nl github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.10 h1:bmAjI2V99mq5mz5QikkpHQ0tXENEtaNJ5//Opr8vb5g= -github.com/projectdiscovery/wappalyzergo v0.0.10/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.11 h1:KKHZq5PKk2Xq23AuybveeizFz2jYrRbsDscNxkcuKCg= +github.com/projectdiscovery/wappalyzergo v0.0.11/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= From 020b719e4d6a38601723909bdff1fe1edec62202 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 23 Aug 2021 12:50:37 +0200 Subject: [PATCH 23/28] Adding integration/regression tests --- .github/workflows/build-test.yml | 7 +- .gitignore | 2 + cmd/integration-test/http.go | 138 +++++++++++++++++++++++ cmd/integration-test/integration-test.go | 56 +++++++++ go.mod | 4 +- go.sum | 9 +- integration_tests/run.sh | 18 +++ internal/testutils/integration.go | 42 +++++++ 8 files changed, 270 insertions(+), 6 deletions(-) create mode 100644 cmd/integration-test/http.go create mode 100644 cmd/integration-test/integration-test.go create mode 100755 integration_tests/run.sh create mode 100644 internal/testutils/integration.go diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 54498e57..88045ab2 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -19,8 +19,11 @@ jobs: uses: actions/checkout@v2 - name: Test - run: go test . - working-directory: cmd/httpx/ + run: go test ./... + + - name: Integration Tests + run: bash run.sh + working-directory: integration_tests/ - name: Build run: go build . diff --git a/.gitignore b/.gitignore index a929c580..3b6ed0bc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ cmd/httpx/httpx +integration_tests/httpx +integration_tests/integration-test \ No newline at end of file diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go new file mode 100644 index 00000000..d027d105 --- /dev/null +++ b/cmd/integration-test/http.go @@ -0,0 +1,138 @@ +package main + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + + "github.com/julienschmidt/httprouter" + "github.com/projectdiscovery/httpx/internal/testutils" +) + +var httpTestcases = map[string]testutils.TestCase{ + "Standard HTTP GET Request": &standardHttpGet{}, + "Standard HTTPS GET Request": &standardHttpGet{tls: true}, + "Regression test for: https://github.com/projectdiscovery/httpx/issues/276": &issue276{}, + "Regression test for: https://github.com/projectdiscovery/httpx/issues/277": &issue277{}, + "Regression test for: https://github.com/projectdiscovery/httpx/issues/303": &issue303{}, +} + +type standardHttpGet struct { + tls bool +} + +func (h *standardHttpGet) Execute() error { + router := httprouter.New() + router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + fmt.Fprintf(w, "This is a test") + })) + var ts *httptest.Server + if h.tls { + ts = httptest.NewTLSServer(router) + } else { + ts = httptest.NewServer(router) + } + defer ts.Close() + + results, err := testutils.RunHttpxAndGetResults(ts.URL, debug) + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} + +type issue276 struct{} + +func (h *issue276) Execute() error { + var ts *httptest.Server + router := httprouter.New() + router.GET("/redirect", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.Header().Add("Location", ts.URL+"/redirect") + w.WriteHeader(302) + fmt.Fprintf(w, "Object moved") + })) + ts = httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunHttpxAndGetResults(ts.URL+"/redirect", debug, "-status-code", "-title", "-no-color") + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + // check if we have all the items on the cli + // full url with port + // status code + // title + expected := ts.URL + "/redirect" + " [302] [Object moved]" + // log.Fatal(results[0], expected) + if !strings.EqualFold(results[0], expected) { + return errIncorrectResult(results[0], expected) + } + return nil +} + +type issue277 struct{} + +func (h *issue277) Execute() error { + var ts *httptest.Server + router := httprouter.New() + router.GET("/hpp", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + if p.ByName("pp") == `%22%3E%3Ch1%3Easdasd%3C%2Fh1%3E` { + w.WriteHeader(http.StatusOK) + } + })) + ts = httptest.NewServer(router) + defer ts.Close() + uripath := "/hpp/?pp=%22%3E%3Ch1%3Easdasd%3C%2Fh1%3E" + results, err := testutils.RunHttpxAndGetResults(ts.URL+uripath, debug) + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + // check if we have all the items on the cli + // full url with port + // status code + // title + expected := ts.URL + uripath + if !strings.EqualFold(results[0], expected) { + return errIncorrectResult(results[0], expected) + } + return nil +} + +type issue303 struct{} + +func (h *issue303) Execute() error { + var ts *httptest.Server + router := httprouter.New() + router.GET("/hpp", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + // mimic a misconfigured web server behavior declaring gzip body + w.Header().Add("Content-Encoding", "gzip") + // but sending it uncompressed + fmt.Fprint(w, "This is a test") + })) + ts = httptest.NewServer(router) + defer ts.Close() + results, err := testutils.RunHttpxAndGetResults(ts.URL, debug) + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + // check if we have all the items on the cli + // full url with port + expected := ts.URL + if !strings.EqualFold(results[0], expected) { + return errIncorrectResult(results[0], expected) + } + return nil +} diff --git a/cmd/integration-test/integration-test.go b/cmd/integration-test/integration-test.go new file mode 100644 index 00000000..0b5a57e1 --- /dev/null +++ b/cmd/integration-test/integration-test.go @@ -0,0 +1,56 @@ +package main + +import ( + "fmt" + "os" + "strings" + + "github.com/logrusorgru/aurora" + "github.com/projectdiscovery/httpx/internal/testutils" +) + +var ( + debug = os.Getenv("DEBUG") == "true" + customTest = os.Getenv("TEST") + protocol = os.Getenv("PROTO") + + errored = false +) + +func main() { + success := aurora.Green("[✓]").String() + failed := aurora.Red("[✘]").String() + + tests := map[string]map[string]testutils.TestCase{ + "http": httpTestcases, + } + for proto, tests := range tests { + if protocol == "" || protocol == proto { + fmt.Printf("Running test cases for \"%s\"\n", aurora.Blue(proto)) + + for name, test := range tests { + if customTest != "" && !strings.Contains(name, customTest) { + continue // only run tests user asked + } + err := test.Execute() + if err != nil { + fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, name, err) + errored = true + } else { + fmt.Printf("%s Test \"%s\" passed!\n", success, name) + } + } + } + } + if errored { + os.Exit(1) + } +} + +func errIncorrectResultsCount(results []string) error { + return fmt.Errorf("incorrect number of results %s", strings.Join(results, "\n\t")) +} + +func errIncorrectResult(expected, got string) error { + return fmt.Errorf("incorrect result: expected \"%s\" got \"%s\"", expected, got) +} diff --git a/go.mod b/go.mod index 62a745b4..339dfefb 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf + github.com/julienschmidt/httprouter v1.3.0 github.com/logrusorgru/aurora v2.0.3+incompatible github.com/microcosm-cc/bluemonday v1.0.15 github.com/miekg/dns v1.1.43 // indirect @@ -29,11 +30,12 @@ require ( github.com/projectdiscovery/retryabledns v1.0.12 // indirect github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7dc github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea - github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d + github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 github.com/projectdiscovery/wappalyzergo v0.0.10 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 + github.com/smartystreets/assertions v1.0.0 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/ratelimit v0.2.0 diff --git a/go.sum b/go.sum index 82f509bb..95c13c45 100644 --- a/go.sum +++ b/go.sum @@ -87,6 +87,8 @@ github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMW github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -179,8 +181,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7d github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea h1:S+DC2tmKG93Om42cnTqrBfIv699pwSIhafqZvip+RIA= github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea/go.mod h1:QHXvznfPfA5f0AZUIBkbLapoUJJlsIDgUlkKva6dOr4= github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4= -github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d h1:nlOAex7twmrEqD5i6WLnugF9uO3DQ6jDEKN9gevrTAk= -github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4= +github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe h1:tQTgf5XLBgZbkJDPtnV3SfdP9tzz5ZWeDBwv8WhnH9Q= +github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= github.com/projectdiscovery/wappalyzergo v0.0.10 h1:bmAjI2V99mq5mz5QikkpHQ0tXENEtaNJ5//Opr8vb5g= @@ -190,8 +192,9 @@ github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNC github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= +github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= diff --git a/integration_tests/run.sh b/integration_tests/run.sh new file mode 100755 index 00000000..38d83015 --- /dev/null +++ b/integration_tests/run.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +rm integration-test httpx 2>/dev/null +cd ../cmd/httpx +go build +mv httpx ../../integration_tests/httpx +cd ../integration-test +go build +mv integration-test ../../integration_tests/integration-test +cd ../../integration_tests +./integration-test +rm integration-test httpx 2>/dev/null +if [ $? -eq 0 ] +then + exit 0 +else + exit 1 +fi diff --git a/internal/testutils/integration.go b/internal/testutils/integration.go new file mode 100644 index 00000000..c3297a89 --- /dev/null +++ b/internal/testutils/integration.go @@ -0,0 +1,42 @@ +package testutils + +import ( + "os" + "os/exec" + "strings" +) + +// RunNucleiAndGetResults returns a list of results for a template +func RunHttpxAndGetResults(url string, debug bool, extra ...string) ([]string, error) { + sh := os.Getenv("SHELL") + cmd := exec.Command(sh, "-c") + cmdLine := `echo ` + url + ` | ./httpx ` + cmdLine += strings.Join(extra, " ") + if debug { + cmdLine += " -debug" + cmd.Stderr = os.Stderr + } else { + cmdLine += " -silent" + } + + cmd.Args = append(cmd.Args, cmdLine) + + data, err := cmd.Output() + if err != nil { + return nil, err + } + parts := []string{} + items := strings.Split(string(data), "\n") + for _, i := range items { + if i != "" { + parts = append(parts, i) + } + } + return parts, nil +} + +// TestCase is a single integration test case +type TestCase interface { + // Execute executes a test case and returns any errors if occurred + Execute() error +} From 8f7aebef36d648f5bd04b34a3bb4d971e071a274 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 23 Aug 2021 14:10:31 +0200 Subject: [PATCH 24/28] fixing bash name --- internal/testutils/integration.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/testutils/integration.go b/internal/testutils/integration.go index c3297a89..474b6d70 100644 --- a/internal/testutils/integration.go +++ b/internal/testutils/integration.go @@ -8,8 +8,7 @@ import ( // RunNucleiAndGetResults returns a list of results for a template func RunHttpxAndGetResults(url string, debug bool, extra ...string) ([]string, error) { - sh := os.Getenv("SHELL") - cmd := exec.Command(sh, "-c") + cmd := exec.Command("bash", "-c") cmdLine := `echo ` + url + ` | ./httpx ` cmdLine += strings.Join(extra, " ") if debug { From 249e94bb6457c0e316bfdae56921c897f1e623ed Mon Sep 17 00:00:00 2001 From: Qui Date: Mon, 23 Aug 2021 09:31:42 -0400 Subject: [PATCH 25/28] Fixing 'ml' option --- runner/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/options.go b/runner/options.go index 9d42c6a6..479b5e94 100644 --- a/runner/options.go +++ b/runner/options.go @@ -229,7 +229,7 @@ func ParseOptions() *Options { flag.StringVar(&options.RequestURIs, "paths", "", "Command separated paths or file containing one path per line (example '/api/v1,/apiv2')") flag.BoolVar(&options.OutputContentType, "content-type", false, "Extracts content-type") flag.StringVar(&options.OutputMatchStatusCode, "mc", "", "Match status code") - flag.StringVar(&options.OutputMatchStatusCode, "ml", "", "Match content length") + flag.StringVar(&options.OutputMatchContentLength, "ml", "", "Match content length") flag.StringVar(&options.OutputFilterStatusCode, "fc", "", "Filter status code") flag.StringVar(&options.OutputFilterContentLength, "fl", "", "Filter content length") flag.StringVar(&options.InputRawRequest, "request", "", "File containing raw request") From 4a2a6d31d12d390eedf27caed00ae4cc9b03ae0f Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 23 Aug 2021 15:37:46 +0200 Subject: [PATCH 26/28] adding more tests --- cmd/integration-test/http.go | 64 +++++++++++++++++++++++++++---- integration_tests/run.sh | 1 - internal/testutils/integration.go | 2 +- 3 files changed, 57 insertions(+), 10 deletions(-) diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go index d027d105..17103bfb 100644 --- a/cmd/integration-test/http.go +++ b/cmd/integration-test/http.go @@ -11,21 +11,30 @@ import ( ) var httpTestcases = map[string]testutils.TestCase{ - "Standard HTTP GET Request": &standardHttpGet{}, - "Standard HTTPS GET Request": &standardHttpGet{tls: true}, - "Regression test for: https://github.com/projectdiscovery/httpx/issues/276": &issue276{}, - "Regression test for: https://github.com/projectdiscovery/httpx/issues/277": &issue277{}, - "Regression test for: https://github.com/projectdiscovery/httpx/issues/303": &issue303{}, + "Standard HTTP GET Request": &standardHttpGet{}, + "Standard HTTPS GET Request": &standardHttpGet{tls: true}, + "Raw HTTP GET Request": &standardHttpGet{unsafe: true}, + "Raw request with non standard rfc path via stdin": &standardHttpGet{unsafe: true, stdinPath: "/%invalid"}, + "Raw request with non standard rfc path via cli flag": &standardHttpGet{unsafe: true, path: "/%invalid"}, + "Regression test for: https://github.com/projectdiscovery/httpx/issues/363": &issue363{}, // infinite redirect + "Regression test for: https://github.com/projectdiscovery/httpx/issues/276": &issue276{}, // full path with port in output + "Regression test for: https://github.com/projectdiscovery/httpx/issues/277": &issue277{}, // scheme://host:port via stdin + "Regression test for: https://github.com/projectdiscovery/httpx/issues/303": &issue303{}, // misconfigured gzip header with uncompressed body } type standardHttpGet struct { - tls bool + tls bool + unsafe bool + stdinPath string + path string + expectedOutput string } func (h *standardHttpGet) Execute() error { router := httprouter.New() router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { fmt.Fprintf(w, "This is a test") + r.Close = true })) var ts *httptest.Server if h.tls { @@ -34,14 +43,31 @@ func (h *standardHttpGet) Execute() error { ts = httptest.NewServer(router) } defer ts.Close() + var extra []string + if h.unsafe { + extra = append(extra, "-unsafe") + } + if h.path != "" { + extra = append(extra, "-path", "\""+h.path+"\"") + } - results, err := testutils.RunHttpxAndGetResults(ts.URL, debug) + URL := ts.URL + if h.stdinPath != "" { + URL += h.stdinPath + } + + results, err := testutils.RunHttpxAndGetResults(URL, debug, extra...) if err != nil { return err } if len(results) != 1 { return errIncorrectResultsCount(results) } + + if h.expectedOutput != "" && !strings.EqualFold(results[0], h.expectedOutput) { + return errIncorrectResult(results[0], h.expectedOutput) + } + return nil } @@ -70,7 +96,6 @@ func (h *issue276) Execute() error { // status code // title expected := ts.URL + "/redirect" + " [302] [Object moved]" - // log.Fatal(results[0], expected) if !strings.EqualFold(results[0], expected) { return errIncorrectResult(results[0], expected) } @@ -136,3 +161,26 @@ func (h *issue303) Execute() error { } return nil } + +type issue363 struct{} + +func (h *issue363) Execute() error { + var ts *httptest.Server + router := httprouter.New() + router.GET("/redirect", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.Header().Add("Location", ts.URL+"/redirect") + w.WriteHeader(302) + fmt.Fprintf(w, "Object moved") + })) + ts = httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunHttpxAndGetResults(ts.URL+"/redirect", debug, "-no-color", "-follow-redirects") + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} diff --git a/integration_tests/run.sh b/integration_tests/run.sh index 38d83015..cb9cbf31 100755 --- a/integration_tests/run.sh +++ b/integration_tests/run.sh @@ -9,7 +9,6 @@ go build mv integration-test ../../integration_tests/integration-test cd ../../integration_tests ./integration-test -rm integration-test httpx 2>/dev/null if [ $? -eq 0 ] then exit 0 diff --git a/internal/testutils/integration.go b/internal/testutils/integration.go index 474b6d70..6f7b28b0 100644 --- a/internal/testutils/integration.go +++ b/internal/testutils/integration.go @@ -9,7 +9,7 @@ import ( // RunNucleiAndGetResults returns a list of results for a template func RunHttpxAndGetResults(url string, debug bool, extra ...string) ([]string, error) { cmd := exec.Command("bash", "-c") - cmdLine := `echo ` + url + ` | ./httpx ` + cmdLine := `echo "` + url + `" | ./httpx ` cmdLine += strings.Join(extra, " ") if debug { cmdLine += " -debug" From 73c33a7b9f856d8d553c4d881f8edaf0eecd290e Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 24 Aug 2021 20:28:35 +0530 Subject: [PATCH 27/28] help menu update --- README.md | 4 ++-- runner/options.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ddf81751..35b9c347 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ This will display help for the tool. Here are all the switches it supports. -H value Custom Header -allow value - Allowlist ip/cidr + Allow list of IP/CIDR's (file or comma separated) -body string Content to send in body with HTTP request -cdn @@ -96,7 +96,7 @@ This will display help for the tool. Here are all the switches it supports. -debug Debug mode -deny value - Denylist ip/cidr + Deny list of IP/CIDR's to process (file or comma separated) -exclude-cdn Skip full port scans for CDNs (only checks for 80,443) -extract-regex string diff --git a/runner/options.go b/runner/options.go index ce800869..2b8ca916 100644 --- a/runner/options.go +++ b/runner/options.go @@ -257,8 +257,8 @@ func ParseOptions() *Options { flag.BoolVar(&options.ShowStatistics, "stats", false, "Enable statistic on keypress (terminal may become unresponsive till the end)") flag.BoolVar(&options.RandomAgent, "random-agent", true, "Use randomly selected HTTP User-Agent header value") flag.BoolVar(&options.StoreChain, "store-chain", false, "Save chain to file (default 'output')") - flag.Var(&options.Allow, "allow", "Allowlist ip/cidr") - flag.Var(&options.Deny, "deny", "Denylist ip/cidr") + flag.Var(&options.Allow, "allow", "Allow list of IP/CIDR's to process (file or comma separated)") + flag.Var(&options.Deny, "deny", "Deny list of IP/CIDR's to process (file or comma separated)") flag.IntVar(&options.MaxResponseBodySizeToSave, "response-size-to-save", math.MaxInt32, "Max response size to save in bytes (default - unlimited)") flag.IntVar(&options.MaxResponseBodySizeToRead, "response-size-to-read", math.MaxInt32, "Max response size to read in bytes (default - unlimited)") flag.StringVar(&options.OutputExtractRegex, "extract-regex", "", "Extract Regex") From 9a9b43d94c5fdc8cd1d845b7eef3d5e8aa5a912a Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 26 Aug 2021 20:04:41 +0530 Subject: [PATCH 28/28] misc updates --- runner/options.go | 74 +++++++++++++++++++++++------------------------ 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/runner/options.go b/runner/options.go index ac2e5d91..f0678b5c 100644 --- a/runner/options.go +++ b/runner/options.go @@ -196,7 +196,7 @@ func ParseOptions() *Options { options := &Options{} flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) - flag.BoolVar(&options.TLSGrab, "tls-grab", false, "Perform TLS data grabbing") + flag.BoolVar(&options.TLSGrab, "tls-grab", false, "Perform TLS(SSL) data grabbing") flag.BoolVar(&options.TechDetect, "tech-detect", false, "Perform wappalyzer based technology detection") flag.IntVar(&options.Threads, "threads", 50, "Number of threads") flag.IntVar(&options.Retries, "retries", 0, "Number of retries") @@ -204,56 +204,56 @@ func ParseOptions() *Options { flag.StringVar(&options.Output, "o", "", "File to write output to (optional)") flag.BoolVar(&options.VHost, "vhost", false, "Check for VHOSTs") flag.BoolVar(&options.VHostInput, "vhost-input", false, "Get a list of vhosts as input") - flag.BoolVar(&options.ExtractTitle, "title", false, "Extracts title") - flag.BoolVar(&options.StatusCode, "status-code", false, "Extracts status code") - flag.BoolVar(&options.Location, "location", false, "Extracts location header") - flag.Var(&options.CustomHeaders, "H", "Custom Header") - flag.Var(&options.CustomPorts, "ports", "ports range (nmap syntax: eg 1,2-10,11)") - flag.BoolVar(&options.ContentLength, "content-length", false, "Extracts content length") - flag.BoolVar(&options.StoreResponse, "sr", false, "Save response to file (default 'output')") - flag.StringVar(&options.StoreResponseDir, "srd", "output", "Save response directory") - flag.BoolVar(&options.FollowRedirects, "follow-redirects", false, "Follow Redirects") - flag.BoolVar(&options.FollowHostRedirects, "follow-host-redirects", false, "Only follow redirects on the same host") + flag.BoolVar(&options.ExtractTitle, "title", false, "Display page title") + flag.BoolVar(&options.StatusCode, "status-code", false, "Display HTTP response status code") + flag.BoolVar(&options.Location, "location", false, "Display location header") + flag.Var(&options.CustomHeaders, "H", "Custom Header to send with request") + flag.Var(&options.CustomPorts, "ports", "Port ranges to scan (nmap syntax: eg 1,2-10,11)") + flag.BoolVar(&options.ContentLength, "content-length", false, "Display HTTP response content length") + flag.BoolVar(&options.StoreResponse, "sr", false, "Store HTTP response to directoy (default 'output')") + flag.StringVar(&options.StoreResponseDir, "srd", "output", "Custom directory to store HTTP responses") + flag.BoolVar(&options.FollowRedirects, "follow-redirects", false, "Follow HTTP Redirects") + flag.BoolVar(&options.FollowHostRedirects, "follow-host-redirects", false, "Only Follow redirects on the same host") flag.IntVar(&options.MaxRedirects, "max-redirects", 10, "Max number of redirects to follow per host") flag.StringVar(&options.HTTPProxy, "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080") - flag.BoolVar(&options.JSONOutput, "json", false, "JSON Output") - flag.StringVar(&options.InputFile, "l", "", "File containing domains") - flag.StringVar(&options.Methods, "x", "", "Request Methods, use ALL to check all verbs (GET, POST, PUT, PATCH, DELETE, CONNECT, OPTIONS and TRACE)") + flag.BoolVar(&options.JSONOutput, "json", false, "Display output in JSON format") + flag.StringVar(&options.InputFile, "l", "", "Input file containing list of hosts to process") + flag.StringVar(&options.Methods, "x", "", "Request Methods to use, use 'all' to probe all HTTP methods") flag.BoolVar(&options.OutputMethod, "method", false, "Display request method") flag.BoolVar(&options.Silent, "silent", false, "Silent mode") flag.BoolVar(&options.Version, "version", false, "Show version of httpx") flag.BoolVar(&options.Verbose, "verbose", false, "Verbose Mode") - flag.BoolVar(&options.NoColor, "no-color", false, "No Color") - flag.BoolVar(&options.OutputServerHeader, "web-server", false, "Extracts server header") - flag.BoolVar(&options.OutputWebSocket, "websocket", false, "Prints out if the server exposes a websocket") - flag.BoolVar(&options.responseInStdout, "response-in-json", false, "Show Raw HTTP Response In Output (-json only) (deprecated)") - flag.BoolVar(&options.responseInStdout, "include-response", false, "Show Raw HTTP Response In Output (-json only)") + flag.BoolVar(&options.NoColor, "no-color", false, "Disable colored output") + flag.BoolVar(&options.OutputServerHeader, "web-server", false, "Display server header") + flag.BoolVar(&options.OutputWebSocket, "websocket", false, "Display server using websocket") + flag.BoolVar(&options.responseInStdout, "response-in-json", false, "Show Raw HTTP response In Output (-json only) (deprecated)") + flag.BoolVar(&options.responseInStdout, "include-response", false, "Show Raw HTTP response In Output (-json only)") flag.BoolVar(&options.chainInStdout, "include-chain", false, "Show Raw HTTP Chain In Output (-json only)") flag.BoolVar(&options.TLSProbe, "tls-probe", false, "Send HTTP probes on the extracted TLS domains") flag.BoolVar(&options.CSPProbe, "csp-probe", false, "Send HTTP probes on the extracted CSP domains") flag.StringVar(&options.RequestURI, "path", "", "Request path/file (example '/api')") flag.StringVar(&options.RequestURIs, "paths", "", "Command separated paths or file containing one path per line (example '/api/v1,/apiv2')") - flag.BoolVar(&options.OutputContentType, "content-type", false, "Extracts content-type") - flag.StringVar(&options.OutputMatchStatusCode, "mc", "", "Match status code") - flag.StringVar(&options.OutputMatchContentLength, "ml", "", "Match content length") - flag.StringVar(&options.OutputFilterStatusCode, "fc", "", "Filter status code") - flag.StringVar(&options.OutputFilterContentLength, "fl", "", "Filter content length") + flag.BoolVar(&options.OutputContentType, "content-type", false, "Display content-type header") + flag.StringVar(&options.OutputMatchStatusCode, "mc", "", "Match response with specific status code (-mc 200,302)") + flag.StringVar(&options.OutputMatchContentLength, "ml", "", "Match response with specific content length (-ml 102)") + flag.StringVar(&options.OutputFilterStatusCode, "fc", "", "Filter response with specific status code (-fc 403,401)") + flag.StringVar(&options.OutputFilterContentLength, "fl", "", "Filter response with specific content length (-fl 23)") flag.StringVar(&options.InputRawRequest, "request", "", "File containing raw request") flag.BoolVar(&options.Unsafe, "unsafe", false, "Send raw requests skipping golang normalization") flag.StringVar(&options.RequestBody, "body", "", "Content to send in body with HTTP request") flag.BoolVar(&options.Debug, "debug", false, "Debug mode") - flag.BoolVar(&options.Pipeline, "pipeline", false, "HTTP1.1 Pipeline") + flag.BoolVar(&options.Pipeline, "pipeline", false, "HTTP1.1 Pipeline probe") flag.BoolVar(&options.HTTP2Probe, "http2", false, "HTTP2 probe") - flag.BoolVar(&options.OutputIP, "ip", false, "Output target ip") - flag.StringVar(&options.OutputFilterString, "filter-string", "", "Filter String") - flag.StringVar(&options.OutputMatchString, "match-string", "", "Match string") - flag.StringVar(&options.OutputFilterRegex, "filter-regex", "", "Filter Regex") - flag.StringVar(&options.OutputMatchRegex, "match-regex", "", "Match Regex") - flag.BoolVar(&options.OutputCName, "cname", false, "Output first cname") - flag.BoolVar(&options.OutputCDN, "cdn", false, "Check if domain's ip belongs to known CDN (akamai, cloudflare, ..)") - flag.BoolVar(&options.OutputResponseTime, "response-time", false, "Output the response time") - flag.BoolVar(&options.NoFallback, "no-fallback", false, "If HTTPS on port 443 is successful on default configuration, probes also port 80 for HTTP") - flag.BoolVar(&options.NoFallbackScheme, "no-fallback-scheme", false, "The tool will respect and attempt the scheme specified in the url (if HTTPS is specified no HTTP is attempted)") + flag.BoolVar(&options.OutputIP, "ip", false, "Display Host IP") + flag.StringVar(&options.OutputFilterString, "filter-string", "", "Filter response with specific string") + flag.StringVar(&options.OutputMatchString, "match-string", "", "Match response with specific string") + flag.StringVar(&options.OutputFilterRegex, "filter-regex", "", "Filter response with specific regex") + flag.StringVar(&options.OutputMatchRegex, "match-regex", "", "Match response with specific regex") + flag.BoolVar(&options.OutputCName, "cname", false, "Display Host cname") + flag.BoolVar(&options.OutputCDN, "cdn", false, "Diplay CDN") + flag.BoolVar(&options.OutputResponseTime, "response-time", false, "Display the response time") + flag.BoolVar(&options.NoFallback, "no-fallback", false, "Probe both protocol (HTTPS and HTTP)") + flag.BoolVar(&options.NoFallbackScheme, "no-fallback-scheme", false, "Probe with input protocol scheme") flag.BoolVar(&options.ShowStatistics, "stats", false, "Enable statistic on keypress (terminal may become unresponsive till the end)") flag.BoolVar(&options.RandomAgent, "random-agent", true, "Use randomly selected HTTP User-Agent header value") flag.BoolVar(&options.StoreChain, "store-chain", false, "Save chain to file (default 'output')") @@ -261,12 +261,12 @@ func ParseOptions() *Options { flag.Var(&options.Deny, "deny", "Deny list of IP/CIDR's to process (file or comma separated)") flag.IntVar(&options.MaxResponseBodySizeToSave, "response-size-to-save", math.MaxInt32, "Max response size to save in bytes (default - unlimited)") flag.IntVar(&options.MaxResponseBodySizeToRead, "response-size-to-read", math.MaxInt32, "Max response size to read in bytes (default - unlimited)") - flag.StringVar(&options.OutputExtractRegex, "extract-regex", "", "Extract Regex") + flag.StringVar(&options.OutputExtractRegex, "extract-regex", "", "Display response content with matched regex") flag.IntVar(&options.RateLimit, "rate-limit", 150, "Maximum requests to send per second") flag.BoolVar(&options.Probe, "probe", false, "Display probe status") flag.BoolVar(&options.Resume, "resume", false, "Resume scan using resume.cfg") flag.BoolVar(&options.ExcludeCDN, "exclude-cdn", false, "Skip full port scans for CDNs (only checks for 80,443)") - flag.IntVar(&options.HostMaxErrors, "max-host-error", -1, "Max error count per host before skipping remaining path/s") + flag.IntVar(&options.HostMaxErrors, "max-host-error", 30, "Max error count per host before skipping remaining path/s") flag.Parse()