diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
index 54498e57..88045ab2 100644
--- a/.github/workflows/build-test.yml
+++ b/.github/workflows/build-test.yml
@@ -19,8 +19,11 @@ jobs:
uses: actions/checkout@v2
- name: Test
- run: go test .
- working-directory: cmd/httpx/
+ run: go test ./...
+
+ - name: Integration Tests
+ run: bash run.sh
+ working-directory: integration_tests/
- name: Build
run: go build .
diff --git a/.gitignore b/.gitignore
index a929c580..3b6ed0bc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,4 @@
cmd/httpx/httpx
+integration_tests/httpx
+integration_tests/integration-test
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
index 3e63374c..2928e1c0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.16.7-alpine AS builder
+FROM golang:1.17.0-alpine AS builder
RUN apk add --no-cache git
RUN GO111MODULE=on go get -v github.com/projectdiscovery/httpx/cmd/httpx
diff --git a/README.md b/README.md
index ddf81751..35b9c347 100644
--- a/README.md
+++ b/README.md
@@ -80,7 +80,7 @@ This will display help for the tool. Here are all the switches it supports.
-H value
Custom Header
-allow value
- Allowlist ip/cidr
+ Allow list of IP/CIDR's (file or comma separated)
-body string
Content to send in body with HTTP request
-cdn
@@ -96,7 +96,7 @@ This will display help for the tool. Here are all the switches it supports.
-debug
Debug mode
-deny value
- Denylist ip/cidr
+ Deny list of IP/CIDR's to process (file or comma separated)
-exclude-cdn
Skip full port scans for CDNs (only checks for 80,443)
-extract-regex string
diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go
new file mode 100644
index 00000000..17103bfb
--- /dev/null
+++ b/cmd/integration-test/http.go
@@ -0,0 +1,186 @@
+package main
+
+import (
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+
+ "github.com/julienschmidt/httprouter"
+ "github.com/projectdiscovery/httpx/internal/testutils"
+)
+
+var httpTestcases = map[string]testutils.TestCase{
+ "Standard HTTP GET Request": &standardHttpGet{},
+ "Standard HTTPS GET Request": &standardHttpGet{tls: true},
+ "Raw HTTP GET Request": &standardHttpGet{unsafe: true},
+ "Raw request with non standard rfc path via stdin": &standardHttpGet{unsafe: true, stdinPath: "/%invalid"},
+ "Raw request with non standard rfc path via cli flag": &standardHttpGet{unsafe: true, path: "/%invalid"},
+ "Regression test for: https://github.com/projectdiscovery/httpx/issues/363": &issue363{}, // infinite redirect
+ "Regression test for: https://github.com/projectdiscovery/httpx/issues/276": &issue276{}, // full path with port in output
+ "Regression test for: https://github.com/projectdiscovery/httpx/issues/277": &issue277{}, // scheme://host:port via stdin
+ "Regression test for: https://github.com/projectdiscovery/httpx/issues/303": &issue303{}, // misconfigured gzip header with uncompressed body
+}
+
+type standardHttpGet struct {
+ tls bool
+ unsafe bool
+ stdinPath string
+ path string
+ expectedOutput string
+}
+
+func (h *standardHttpGet) Execute() error {
+ router := httprouter.New()
+ router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+ fmt.Fprintf(w, "This is a test")
+ r.Close = true
+ }))
+ var ts *httptest.Server
+ if h.tls {
+ ts = httptest.NewTLSServer(router)
+ } else {
+ ts = httptest.NewServer(router)
+ }
+ defer ts.Close()
+ var extra []string
+ if h.unsafe {
+ extra = append(extra, "-unsafe")
+ }
+ if h.path != "" {
+ extra = append(extra, "-path", "\""+h.path+"\"")
+ }
+
+ URL := ts.URL
+ if h.stdinPath != "" {
+ URL += h.stdinPath
+ }
+
+ results, err := testutils.RunHttpxAndGetResults(URL, debug, extra...)
+ if err != nil {
+ return err
+ }
+ if len(results) != 1 {
+ return errIncorrectResultsCount(results)
+ }
+
+ if h.expectedOutput != "" && !strings.EqualFold(results[0], h.expectedOutput) {
+ return errIncorrectResult(results[0], h.expectedOutput)
+ }
+
+ return nil
+}
+
+type issue276 struct{}
+
+func (h *issue276) Execute() error {
+ var ts *httptest.Server
+ router := httprouter.New()
+ router.GET("/redirect", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+ w.Header().Add("Location", ts.URL+"/redirect")
+ w.WriteHeader(302)
+ fmt.Fprintf(w, "Object moved")
+ }))
+ ts = httptest.NewServer(router)
+ defer ts.Close()
+
+ results, err := testutils.RunHttpxAndGetResults(ts.URL+"/redirect", debug, "-status-code", "-title", "-no-color")
+ if err != nil {
+ return err
+ }
+ if len(results) != 1 {
+ return errIncorrectResultsCount(results)
+ }
+ // check if we have all the items on the cli
+ // full url with port
+ // status code
+ // title
+ expected := ts.URL + "/redirect" + " [302] [Object moved]"
+ if !strings.EqualFold(results[0], expected) {
+ return errIncorrectResult(results[0], expected)
+ }
+ return nil
+}
+
+type issue277 struct{}
+
+func (h *issue277) Execute() error {
+ var ts *httptest.Server
+ router := httprouter.New()
+ router.GET("/hpp", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
+ if p.ByName("pp") == `%22%3E%3Ch1%3Easdasd%3C%2Fh1%3E` {
+ w.WriteHeader(http.StatusOK)
+ }
+ }))
+ ts = httptest.NewServer(router)
+ defer ts.Close()
+ uripath := "/hpp/?pp=%22%3E%3Ch1%3Easdasd%3C%2Fh1%3E"
+ results, err := testutils.RunHttpxAndGetResults(ts.URL+uripath, debug)
+ if err != nil {
+ return err
+ }
+ if len(results) != 1 {
+ return errIncorrectResultsCount(results)
+ }
+ // check if we have all the items on the cli
+ // full url with port
+ // status code
+ // title
+ expected := ts.URL + uripath
+ if !strings.EqualFold(results[0], expected) {
+ return errIncorrectResult(results[0], expected)
+ }
+ return nil
+}
+
+type issue303 struct{}
+
+func (h *issue303) Execute() error {
+ var ts *httptest.Server
+ router := httprouter.New()
+ router.GET("/hpp", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
+ // mimic a misconfigured web server behavior declaring gzip body
+ w.Header().Add("Content-Encoding", "gzip")
+ // but sending it uncompressed
+ fmt.Fprint(w, "This is a test")
+ }))
+ ts = httptest.NewServer(router)
+ defer ts.Close()
+ results, err := testutils.RunHttpxAndGetResults(ts.URL, debug)
+ if err != nil {
+ return err
+ }
+ if len(results) != 1 {
+ return errIncorrectResultsCount(results)
+ }
+ // check if we have all the items on the cli
+ // full url with port
+ expected := ts.URL
+ if !strings.EqualFold(results[0], expected) {
+ return errIncorrectResult(results[0], expected)
+ }
+ return nil
+}
+
+type issue363 struct{}
+
+func (h *issue363) Execute() error {
+ var ts *httptest.Server
+ router := httprouter.New()
+ router.GET("/redirect", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+ w.Header().Add("Location", ts.URL+"/redirect")
+ w.WriteHeader(302)
+ fmt.Fprintf(w, "Object moved")
+ }))
+ ts = httptest.NewServer(router)
+ defer ts.Close()
+
+ results, err := testutils.RunHttpxAndGetResults(ts.URL+"/redirect", debug, "-no-color", "-follow-redirects")
+ if err != nil {
+ return err
+ }
+ if len(results) != 1 {
+ return errIncorrectResultsCount(results)
+ }
+ return nil
+}
diff --git a/cmd/integration-test/integration-test.go b/cmd/integration-test/integration-test.go
new file mode 100644
index 00000000..0b5a57e1
--- /dev/null
+++ b/cmd/integration-test/integration-test.go
@@ -0,0 +1,56 @@
+package main
+
+import (
+ "fmt"
+ "os"
+ "strings"
+
+ "github.com/logrusorgru/aurora"
+ "github.com/projectdiscovery/httpx/internal/testutils"
+)
+
+var (
+ debug = os.Getenv("DEBUG") == "true"
+ customTest = os.Getenv("TEST")
+ protocol = os.Getenv("PROTO")
+
+ errored = false
+)
+
+func main() {
+ success := aurora.Green("[✓]").String()
+ failed := aurora.Red("[✘]").String()
+
+ tests := map[string]map[string]testutils.TestCase{
+ "http": httpTestcases,
+ }
+ for proto, tests := range tests {
+ if protocol == "" || protocol == proto {
+ fmt.Printf("Running test cases for \"%s\"\n", aurora.Blue(proto))
+
+ for name, test := range tests {
+ if customTest != "" && !strings.Contains(name, customTest) {
+ continue // only run tests user asked
+ }
+ err := test.Execute()
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, name, err)
+ errored = true
+ } else {
+ fmt.Printf("%s Test \"%s\" passed!\n", success, name)
+ }
+ }
+ }
+ }
+ if errored {
+ os.Exit(1)
+ }
+}
+
+func errIncorrectResultsCount(results []string) error {
+ return fmt.Errorf("incorrect number of results %s", strings.Join(results, "\n\t"))
+}
+
+func errIncorrectResult(expected, got string) error {
+ return fmt.Errorf("incorrect result: expected \"%s\" got \"%s\"", expected, got)
+}
diff --git a/common/customlist/customlist.go b/common/customlist/customlist.go
index 8c7fa45b..4bf05f37 100644
--- a/common/customlist/customlist.go
+++ b/common/customlist/customlist.go
@@ -1,5 +1,7 @@
package customlist
+import "github.com/projectdiscovery/httpx/common/fileutil"
+
// CustomList for fastdialer
type CustomList []string
@@ -10,6 +12,7 @@ func (c *CustomList) String() string {
// Set a new global header
func (c *CustomList) Set(value string) error {
- *c = append(*c, value)
+ values := fileutil.LoadCidrsFromSliceOrFile(value, ",")
+ *c = append(*c, values...)
return nil
}
diff --git a/common/customports/customport.go b/common/customports/customport.go
index 8a671869..7728b78d 100644
--- a/common/customports/customport.go
+++ b/common/customports/customport.go
@@ -5,6 +5,7 @@ import (
"strings"
"github.com/projectdiscovery/gologger"
+ "github.com/projectdiscovery/sliceutil"
"github.com/projectdiscovery/httpx/common/httpx"
)
@@ -31,7 +32,7 @@ func (c *CustomPorts) String() string {
func (c *CustomPorts) Set(value string) error {
// ports can be like nmap -p [https|http:]start-end,[https|http:]port1,[https|http:]port2,[https|http:]port3
// splits on comma
- potentialPorts := strings.Split(value, ",")
+ potentialPorts := sliceutil.Dedupe(strings.Split(value, ","))
// check if port is a single integer value or needs to be expanded further
for _, potentialPort := range potentialPorts {
@@ -43,12 +44,20 @@ func (c *CustomPorts) Set(value string) error {
} else if strings.HasPrefix(potentialPort, httpx.HTTPS+":") {
potentialPort = strings.TrimPrefix(potentialPort, httpx.HTTPS+":")
protocol = httpx.HTTPS
+ } else if strings.HasPrefix(potentialPort, httpx.HTTPandHTTPS+":") {
+ potentialPort = strings.TrimPrefix(potentialPort, httpx.HTTPandHTTPS+":")
+ protocol = httpx.HTTPandHTTPS
}
potentialRange := strings.Split(potentialPort, "-")
// it's a single port?
if len(potentialRange) < portRangeParts {
if p, err := strconv.Atoi(potentialPort); err == nil {
+ if existingProtocol, ok := Ports[p]; ok {
+ if existingProtocol == httpx.HTTP && protocol == httpx.HTTPS || existingProtocol == httpx.HTTPS && protocol == httpx.HTTP {
+ protocol = httpx.HTTPandHTTPS
+ }
+ }
Ports[p] = protocol
} else {
gologger.Warning().Msgf("Could not cast port to integer, your value: %s, resulting error %s. Skipping it\n",
@@ -79,6 +88,11 @@ func (c *CustomPorts) Set(value string) error {
}
for i := lowP; i <= highP; i++ {
+ if existingProtocol, ok := Ports[i]; ok {
+ if existingProtocol == httpx.HTTP && protocol == httpx.HTTPS || existingProtocol == httpx.HTTPS && protocol == httpx.HTTP {
+ protocol = httpx.HTTPandHTTPS
+ }
+ }
Ports[i] = protocol
}
}
diff --git a/common/fileutil/fileutil.go b/common/fileutil/fileutil.go
index f5814865..df5ce289 100644
--- a/common/fileutil/fileutil.go
+++ b/common/fileutil/fileutil.go
@@ -3,9 +3,14 @@ package fileutil
import (
"bufio"
"errors"
+ "io/ioutil"
+ "net"
"os"
"path/filepath"
"regexp"
+
+ "github.com/projectdiscovery/fileutil"
+ "github.com/projectdiscovery/httpx/common/stringz"
)
// FileExists checks if a file exists and is not a directory
@@ -70,3 +75,21 @@ func FileNameIsGlob(pattern string) bool {
_, err := regexp.Compile(pattern)
return err == nil
}
+
+func LoadCidrsFromSliceOrFile(option string, splitchar string) (networkList []string) {
+ items := stringz.SplitByCharAndTrimSpace(option, splitchar)
+ for _, item := range items {
+ // ip
+ if net.ParseIP(item) != nil {
+ networkList = append(networkList, item)
+ } else if _, _, err := net.ParseCIDR(item); err == nil {
+ networkList = append(networkList, item)
+ } else if fileutil.FileExists(item) {
+ if filedata, err := ioutil.ReadFile(item); err == nil && len(filedata) > 0 {
+ networkList = append(networkList, LoadCidrsFromSliceOrFile(string(filedata), "\n")...)
+ }
+ }
+ }
+
+ return networkList
+}
diff --git a/common/httpx/http2.go b/common/httpx/http2.go
index 83f827c6..a1f0bb33 100644
--- a/common/httpx/http2.go
+++ b/common/httpx/http2.go
@@ -16,8 +16,10 @@ const (
HTTP = "http"
// HTTPS defines the secure http scheme
HTTPS = "https"
- // HTTPorHTTPS defines the both http and https scheme
+ // HTTPorHTTPS defines both http and https scheme in mutual exclusion
HTTPorHTTPS = "http|https"
+ // HTTPandHTTPS defines both http and https scheme
+ HTTPandHTTPS = "http&https"
)
// SupportHTTP2 checks if the target host supports HTTP2
diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go
index dd1fa55a..ac0986fe 100644
--- a/common/httpx/httpx.go
+++ b/common/httpx/httpx.go
@@ -7,6 +7,7 @@ import (
"io/ioutil"
"net/http"
"net/url"
+ "strconv"
"strings"
"time"
"unicode/utf8"
@@ -18,20 +19,20 @@ import (
pdhttputil "github.com/projectdiscovery/httputil"
"github.com/projectdiscovery/rawhttp"
retryablehttp "github.com/projectdiscovery/retryablehttp-go"
+ "github.com/projectdiscovery/stringsutil"
"golang.org/x/net/http2"
)
// HTTPX represent an instance of the library client
type HTTPX struct {
- client *retryablehttp.Client
- client2 *http.Client
- Filters []Filter
- Options *Options
- htmlPolicy *bluemonday.Policy
- CustomHeaders map[string]string
- RequestOverride *RequestOverride
- cdn *cdncheck.Client
- Dialer *fastdialer.Dialer
+ client *retryablehttp.Client
+ client2 *http.Client
+ Filters []Filter
+ Options *Options
+ htmlPolicy *bluemonday.Policy
+ CustomHeaders map[string]string
+ cdn *cdncheck.Client
+ Dialer *fastdialer.Dialer
}
// New httpx instance
@@ -59,20 +60,30 @@ func New(options *Options) (*HTTPX, error) {
}
if httpx.Options.FollowRedirects {
- // Follow redirects
- redirectFunc = nil
+ // Follow redirects up to a maximum number
+ redirectFunc = func(redirectedRequest *http.Request, previousRequests []*http.Request) error {
+ if len(previousRequests) >= options.MaxRedirects {
+ // https://github.com/golang/go/issues/10069
+ return http.ErrUseLastResponse
+ }
+ return nil
+ }
}
if httpx.Options.FollowHostRedirects {
- // Only follow redirects on the same host
- redirectFunc = func(redirectedRequest *http.Request, previousRequest []*http.Request) error {
+ // Only follow redirects on the same host up to a maximum number
+ redirectFunc = func(redirectedRequest *http.Request, previousRequests []*http.Request) error {
// Check if we get a redirect to a differen host
var newHost = redirectedRequest.URL.Host
- var oldHost = previousRequest[0].URL.Host
+ var oldHost = previousRequests[0].URL.Host
if newHost != oldHost {
// Tell the http client to not follow redirect
return http.ErrUseLastResponse
}
+ if len(previousRequests) >= options.MaxRedirects {
+ // https://github.com/golang/go/issues/10069
+ return http.ErrUseLastResponse
+ }
return nil
}
}
@@ -112,7 +123,6 @@ func New(options *Options) (*HTTPX, error) {
httpx.htmlPolicy = bluemonday.NewPolicy()
httpx.CustomHeaders = httpx.Options.CustomHeaders
- httpx.RequestOverride = &options.RequestOverride
if options.CdnCheck || options.ExcludeCdn {
httpx.cdn, err = cdncheck.NewWithCache()
if err != nil {
@@ -124,16 +134,23 @@ func New(options *Options) (*HTTPX, error) {
}
// Do http request
-func (h *HTTPX) Do(req *retryablehttp.Request) (*Response, error) {
+func (h *HTTPX) Do(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (*Response, error) {
timeStart := time.Now()
var gzipRetry bool
get_response:
- httpresp, err := h.getResponse(req)
+ httpresp, err := h.getResponse(req, unsafeOptions)
if err != nil {
return nil, err
}
+ var shouldIgnoreErrors, shouldIgnoreBodyErrors bool
+ switch {
+ case h.Options.Unsafe && req.Method == http.MethodHead && !stringsutil.ContainsAny("i/o timeout"):
+ shouldIgnoreErrors = true
+ shouldIgnoreBodyErrors = true
+ }
+
var resp Response
resp.Headers = httpresp.Header.Clone()
@@ -148,23 +165,25 @@ get_response:
req.Header.Set("Accept-Encoding", "identity")
goto get_response
}
- return nil, err
+ if !shouldIgnoreErrors {
+ return nil, err
+ }
}
- resp.Raw = rawResp
- resp.RawHeaders = headers
+ resp.Raw = string(rawResp)
+ resp.RawHeaders = string(headers)
var respbody []byte
// websockets don't have a readable body
if httpresp.StatusCode != http.StatusSwitchingProtocols {
var err error
respbody, err = ioutil.ReadAll(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead))
- if err != nil {
+ if err != nil && !shouldIgnoreBodyErrors {
return nil, err
}
}
closeErr := httpresp.Body.Close()
- if closeErr != nil {
+ if closeErr != nil && !shouldIgnoreBodyErrors {
return nil, closeErr
}
@@ -175,7 +194,15 @@ get_response:
respbodystr = h.htmlPolicy.Sanitize(respbodystr)
}
- resp.ContentLength = utf8.RuneCountInString(respbodystr)
+ if contentLength, ok := resp.Headers["Content-Length"]; ok {
+ contentLengthInt, err := strconv.Atoi(strings.Join(contentLength, ""))
+ if err != nil {
+ resp.ContentLength = utf8.RuneCountInString(respbodystr)
+ } else {
+ resp.ContentLength = contentLengthInt
+ }
+ }
+
resp.Data = respbody
// fill metrics
@@ -207,31 +234,33 @@ get_response:
}
// RequestOverride contains the URI path to override the request
-type RequestOverride struct {
+type UnsafeOptions struct {
URIPath string
}
// getResponse returns response from safe / unsafe request
-func (h *HTTPX) getResponse(req *retryablehttp.Request) (*http.Response, error) {
+func (h *HTTPX) getResponse(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (*http.Response, error) {
if h.Options.Unsafe {
- return h.doUnsafe(req)
+ return h.doUnsafeWithOptions(req, unsafeOptions)
}
return h.client.Do(req)
}
// doUnsafe does an unsafe http request
-func (h *HTTPX) doUnsafe(req *retryablehttp.Request) (*http.Response, error) {
+func (h *HTTPX) doUnsafeWithOptions(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (*http.Response, error) {
method := req.Method
headers := req.Header
targetURL := req.URL.String()
body := req.Body
- return rawhttp.DoRaw(method, targetURL, h.RequestOverride.URIPath, headers, body)
+ options := rawhttp.DefaultOptions
+ options.Timeout = h.Options.Timeout
+ return rawhttp.DoRawWithOptions(method, targetURL, unsafeOptions.URIPath, headers, body, options)
}
// Verify the http calls and apply-cascade all the filters, as soon as one matches it returns true
-func (h *HTTPX) Verify(req *retryablehttp.Request) (bool, error) {
- resp, err := h.Do(req)
+func (h *HTTPX) Verify(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (bool, error) {
+ resp, err := h.Do(req, unsafeOptions)
if err != nil {
return false, err
}
diff --git a/common/httpx/option.go b/common/httpx/option.go
index 840a4ac4..46d6070f 100644
--- a/common/httpx/option.go
+++ b/common/httpx/option.go
@@ -8,7 +8,6 @@ import (
type Options struct {
RandomAgent bool
DefaultUserAgent string
- RequestOverride RequestOverride
HTTPProxy string
SocksProxy string
Threads int
@@ -23,6 +22,7 @@ type Options struct {
VHostSimilarityRatio int
FollowRedirects bool
FollowHostRedirects bool
+ MaxRedirects int
Unsafe bool
TLSGrab bool
// VHOSTs options
@@ -35,17 +35,19 @@ type Options struct {
Deny []string
MaxResponseBodySizeToSave int64
MaxResponseBodySizeToRead int64
+ UnsafeURI string
}
// DefaultOptions contains the default options
var DefaultOptions = Options{
- RandomAgent: true,
- Threads: 25,
- Timeout: 30 * time.Second,
- RetryMax: 5,
- Unsafe: false,
- CdnCheck: true,
- ExcludeCdn: false,
+ RandomAgent: true,
+ Threads: 25,
+ Timeout: 30 * time.Second,
+ RetryMax: 5,
+ MaxRedirects: 10,
+ Unsafe: false,
+ CdnCheck: true,
+ ExcludeCdn: false,
// VHOSTs options
VHostIgnoreStatusCode: false,
VHostIgnoreContentLength: true,
diff --git a/common/httpx/virtualhost.go b/common/httpx/virtualhost.go
index 0a4eee53..77e3d479 100644
--- a/common/httpx/virtualhost.go
+++ b/common/httpx/virtualhost.go
@@ -11,8 +11,8 @@ import (
const simMultiplier = 100
// IsVirtualHost checks if the target endpoint is a virtual host
-func (h *HTTPX) IsVirtualHost(req *retryablehttp.Request) (bool, error) {
- httpresp1, err := h.Do(req)
+func (h *HTTPX) IsVirtualHost(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (bool, error) {
+ httpresp1, err := h.Do(req, unsafeOptions)
if err != nil {
return false, err
}
@@ -20,7 +20,7 @@ func (h *HTTPX) IsVirtualHost(req *retryablehttp.Request) (bool, error) {
// request a non-existing endpoint
req.Host = fmt.Sprintf("%s.%s", xid.New().String(), req.Host)
- httpresp2, err := h.Do(req)
+ httpresp2, err := h.Do(req, unsafeOptions)
if err != nil {
return false, err
}
diff --git a/common/stringz/stringz.go b/common/stringz/stringz.go
index 293b9f85..34eb790c 100644
--- a/common/stringz/stringz.go
+++ b/common/stringz/stringz.go
@@ -1,6 +1,7 @@
package stringz
import (
+ "net/url"
"strconv"
"strings"
@@ -73,3 +74,13 @@ func RemoveURLDefaultPort(rawURL string) string {
}
return u.String()
}
+
+func GetInvalidURI(rawURL string) (bool, string) {
+ if _, err := url.Parse(rawURL); err != nil {
+ if u, err := urlutil.Parse(rawURL); err == nil {
+ return true, u.RequestURI
+ }
+ }
+
+ return false, ""
+}
diff --git a/go.mod b/go.mod
index 3d163c1b..a550e3fe 100644
--- a/go.mod
+++ b/go.mod
@@ -4,11 +4,13 @@ go 1.14
require (
github.com/akrylysov/pogreb v0.10.1 // indirect
+ github.com/bluele/gcache v0.0.2
github.com/corpix/uarand v0.1.1
github.com/dgraph-io/ristretto v0.1.0 // indirect
github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf
+ github.com/julienschmidt/httprouter v1.3.0
github.com/logrusorgru/aurora v2.0.3+incompatible
github.com/microcosm-cc/bluemonday v1.0.15
github.com/miekg/dns v1.1.43 // indirect
@@ -16,27 +18,30 @@ require (
github.com/projectdiscovery/cdncheck v0.0.2
github.com/projectdiscovery/clistats v0.0.8
github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345
- github.com/projectdiscovery/fastdialer v0.0.12
+ github.com/projectdiscovery/fastdialer v0.0.13-0.20210815100514-360f851a5b80
github.com/projectdiscovery/fdmax v0.0.3
+ github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca
github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c
github.com/projectdiscovery/gologger v1.1.4
github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa
- github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d
+ github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5
github.com/projectdiscovery/iputil v0.0.0-20210705072957-5a968407979b
github.com/projectdiscovery/mapcidr v0.0.8
- github.com/projectdiscovery/rawhttp v0.0.7
+ github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e
github.com/projectdiscovery/retryabledns v1.0.12 // indirect
github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7dc
- github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d
+ github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea
+ github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe
github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1
- github.com/projectdiscovery/wappalyzergo v0.0.9
+ github.com/projectdiscovery/wappalyzergo v0.0.11
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.3.0
+ github.com/smartystreets/assertions v1.0.0 // indirect
go.etcd.io/bbolt v1.3.6 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/ratelimit v0.2.0
- golang.org/x/net v0.0.0-20210716203947-853a461950ff
+ golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
- golang.org/x/text v0.3.6
+ golang.org/x/text v0.3.7
google.golang.org/protobuf v1.27.1 // indirect
)
diff --git a/go.sum b/go.sum
index 418df5c5..8e022c33 100644
--- a/go.sum
+++ b/go.sum
@@ -15,6 +15,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
+github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
+github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
@@ -85,6 +87,10 @@ github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMW
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
+github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -138,10 +144,12 @@ github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKS
github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U=
github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc=
-github.com/projectdiscovery/fastdialer v0.0.12 h1:TjvM41UfR+A7YsxQZoTvI6C5nVe1d+fvRqtcDNbSwz8=
-github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0=
+github.com/projectdiscovery/fastdialer v0.0.13-0.20210815100514-360f851a5b80 h1:Hyt1AcK5hAgE7LuaNyRD3mFc8zaS7LP4vTk6Yp83f8E=
+github.com/projectdiscovery/fastdialer v0.0.13-0.20210815100514-360f851a5b80/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0=
github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg=
github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI=
+github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca h1:xT//ApxoeQRbt9GgL/122688bhGy9hur8YG0Qh69k5I=
+github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0=
github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k=
github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk=
github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
@@ -151,8 +159,8 @@ github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8q
github.com/projectdiscovery/hmap v0.0.2-0.20210616215655-7b78e7f33d1f/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8=
github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa h1:KeN6/bZOVxtS4XkgzRvYxpXWZSZt+AoGP5Myyr3/Duk=
github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8=
-github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d h1:IdBTOSGaPrZ8+FK0uYMQIva9dYIR5F55PLFWYtBBKc0=
-github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d/go.mod h1:Vm2DY4NwUV5yA6TNzJOOjTYGjTcVfuEN8m9Y5dAksLQ=
+github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 h1:GzruqQhb+sj1rEuHRFLhWX8gH/tJ+sj1udRjOy9VCJo=
+github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5/go.mod h1:BueJPSPWAX11IFS6bdAqTkekiIz5Fgco5LVc1kqO9L4=
github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE=
github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
@@ -164,8 +172,8 @@ github.com/projectdiscovery/mapcidr v0.0.8 h1:16U05F2x3o/jSTsxSCY2hCuCs9xOSwVxjo
github.com/projectdiscovery/mapcidr v0.0.8/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00=
github.com/projectdiscovery/networkpolicy v0.0.1 h1:RGRuPlxE8WLFF9tdKSjTsYiTIKHNHW20Kl0nGGiRb1I=
github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs=
-github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg=
-github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
+github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e h1:hcpGb5/gSn+kNUmzgodV1+sHDmFybuGhsuhrTqFebQY=
+github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f h1:HR3R/nhELwLXufUlO1ZkKVqrZl4lN1cWFBdN8RcMuLo=
github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f/go.mod h1:3L0WfNIcVWXIDur8k+gKDLZLWY2F+rs0SQXtcn/3AYU=
github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
@@ -174,20 +182,23 @@ github.com/projectdiscovery/retryabledns v1.0.12/go.mod h1:4sMC8HZyF01HXukRleSQY
github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7dc h1:769c7sQOl9BP8dhE8uv0mQX9WmcXo6Jzv//Nm+qAf50=
github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7dc/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI=
+github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea h1:S+DC2tmKG93Om42cnTqrBfIv699pwSIhafqZvip+RIA=
+github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea/go.mod h1:QHXvznfPfA5f0AZUIBkbLapoUJJlsIDgUlkKva6dOr4=
github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
-github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d h1:nlOAex7twmrEqD5i6WLnugF9uO3DQ6jDEKN9gevrTAk=
-github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
+github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe h1:tQTgf5XLBgZbkJDPtnV3SfdP9tzz5ZWeDBwv8WhnH9Q=
+github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4=
github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo=
-github.com/projectdiscovery/wappalyzergo v0.0.9 h1:MR2A7ZmF61bT1y8A9+YVe5bsrPAiFDEQ7Ix4GbqccwQ=
-github.com/projectdiscovery/wappalyzergo v0.0.9/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM=
+github.com/projectdiscovery/wappalyzergo v0.0.11 h1:KKHZq5PKk2Xq23AuybveeizFz2jYrRbsDscNxkcuKCg=
+github.com/projectdiscovery/wappalyzergo v0.0.11/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM=
github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8=
+github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
@@ -242,8 +253,8 @@ golang.org/x/net v0.0.0-20210414194228-064579744ee0/go.mod h1:9tjilg8BloeKEkVJvy
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210716203947-853a461950ff h1:j2EK/QoxYNBsXI4R7fQkkRUk8y6wnOBI+6hgPdP/6Ds=
-golang.org/x/net v0.0.0-20210716203947-853a461950ff/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c=
+golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -276,8 +287,9 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
diff --git a/integration_tests/run.sh b/integration_tests/run.sh
new file mode 100755
index 00000000..cb9cbf31
--- /dev/null
+++ b/integration_tests/run.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+rm integration-test httpx 2>/dev/null
+cd ../cmd/httpx
+go build
+mv httpx ../../integration_tests/httpx
+cd ../integration-test
+go build
+mv integration-test ../../integration_tests/integration-test
+cd ../../integration_tests
+./integration-test
+if [ $? -eq 0 ]
+then
+ exit 0
+else
+ exit 1
+fi
diff --git a/internal/testutils/integration.go b/internal/testutils/integration.go
new file mode 100644
index 00000000..6f7b28b0
--- /dev/null
+++ b/internal/testutils/integration.go
@@ -0,0 +1,41 @@
+package testutils
+
+import (
+ "os"
+ "os/exec"
+ "strings"
+)
+
+// RunNucleiAndGetResults returns a list of results for a template
+func RunHttpxAndGetResults(url string, debug bool, extra ...string) ([]string, error) {
+ cmd := exec.Command("bash", "-c")
+ cmdLine := `echo "` + url + `" | ./httpx `
+ cmdLine += strings.Join(extra, " ")
+ if debug {
+ cmdLine += " -debug"
+ cmd.Stderr = os.Stderr
+ } else {
+ cmdLine += " -silent"
+ }
+
+ cmd.Args = append(cmd.Args, cmdLine)
+
+ data, err := cmd.Output()
+ if err != nil {
+ return nil, err
+ }
+ parts := []string{}
+ items := strings.Split(string(data), "\n")
+ for _, i := range items {
+ if i != "" {
+ parts = append(parts, i)
+ }
+ }
+ return parts, nil
+}
+
+// TestCase is a single integration test case
+type TestCase interface {
+ // Execute executes a test case and returns any errors if occurred
+ Execute() error
+}
diff --git a/runner/banner.go b/runner/banner.go
index 713adc0a..2beadea9 100644
--- a/runner/banner.go
+++ b/runner/banner.go
@@ -8,11 +8,11 @@ const banner = `
/ __ \/ __/ __/ __ \| /
/ / / / /_/ /_/ /_/ / |
/_/ /_/\__/\__/ .___/_/|_|
- /_/ v1.1.1
+ /_/ v1.1.2
`
// Version is the current version of httpx
-const Version = `v1.1.1`
+const Version = `v1.1.2`
// showBanner is used to show the banner to the user
func showBanner() {
diff --git a/runner/options.go b/runner/options.go
index 9d42c6a6..f0678b5c 100644
--- a/runner/options.go
+++ b/runner/options.go
@@ -6,6 +6,7 @@ import (
"os"
"regexp"
+ "github.com/projectdiscovery/fileutil"
"github.com/projectdiscovery/goconfig"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/gologger/formatter"
@@ -13,7 +14,7 @@ import (
"github.com/projectdiscovery/httpx/common/customheader"
"github.com/projectdiscovery/httpx/common/customlist"
customport "github.com/projectdiscovery/httpx/common/customports"
- "github.com/projectdiscovery/httpx/common/fileutil"
+ fileutilz "github.com/projectdiscovery/httpx/common/fileutil"
"github.com/projectdiscovery/httpx/common/stringz"
)
@@ -61,6 +62,7 @@ type scanOptions struct {
OutputExtractRegex string
extractRegex *regexp.Regexp
ExcludeCDN bool
+ HostMaxErrors int
}
func (s *scanOptions) Clone() *scanOptions {
@@ -99,6 +101,7 @@ func (s *scanOptions) Clone() *scanOptions {
OutputExtractRegex: s.OutputExtractRegex,
MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave,
MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead,
+ HostMaxErrors: s.HostMaxErrors,
}
}
@@ -154,6 +157,7 @@ type Options struct {
responseInStdout bool
chainInStdout bool
FollowHostRedirects bool
+ MaxRedirects int
OutputMethod bool
TLSProbe bool
CSPProbe bool
@@ -184,6 +188,7 @@ type Options struct {
Resume bool
resumeCfg *ResumeCfg
ExcludeCDN bool
+ HostMaxErrors int
}
// ParseOptions parses the command line options for application
@@ -191,7 +196,7 @@ func ParseOptions() *Options {
options := &Options{}
flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
- flag.BoolVar(&options.TLSGrab, "tls-grab", false, "Perform TLS data grabbing")
+ flag.BoolVar(&options.TLSGrab, "tls-grab", false, "Perform TLS(SSL) data grabbing")
flag.BoolVar(&options.TechDetect, "tech-detect", false, "Perform wappalyzer based technology detection")
flag.IntVar(&options.Threads, "threads", 50, "Number of threads")
flag.IntVar(&options.Retries, "retries", 0, "Number of retries")
@@ -199,67 +204,69 @@ func ParseOptions() *Options {
flag.StringVar(&options.Output, "o", "", "File to write output to (optional)")
flag.BoolVar(&options.VHost, "vhost", false, "Check for VHOSTs")
flag.BoolVar(&options.VHostInput, "vhost-input", false, "Get a list of vhosts as input")
- flag.BoolVar(&options.ExtractTitle, "title", false, "Extracts title")
- flag.BoolVar(&options.StatusCode, "status-code", false, "Extracts status code")
- flag.BoolVar(&options.Location, "location", false, "Extracts location header")
- flag.Var(&options.CustomHeaders, "H", "Custom Header")
- flag.Var(&options.CustomPorts, "ports", "ports range (nmap syntax: eg 1,2-10,11)")
- flag.BoolVar(&options.ContentLength, "content-length", false, "Extracts content length")
- flag.BoolVar(&options.StoreResponse, "sr", false, "Save response to file (default 'output')")
- flag.StringVar(&options.StoreResponseDir, "srd", "output", "Save response directory")
- flag.BoolVar(&options.FollowRedirects, "follow-redirects", false, "Follow Redirects")
- flag.BoolVar(&options.FollowHostRedirects, "follow-host-redirects", false, "Only follow redirects on the same host")
+ flag.BoolVar(&options.ExtractTitle, "title", false, "Display page title")
+ flag.BoolVar(&options.StatusCode, "status-code", false, "Display HTTP response status code")
+ flag.BoolVar(&options.Location, "location", false, "Display location header")
+ flag.Var(&options.CustomHeaders, "H", "Custom Header to send with request")
+ flag.Var(&options.CustomPorts, "ports", "Port ranges to scan (nmap syntax: eg 1,2-10,11)")
+ flag.BoolVar(&options.ContentLength, "content-length", false, "Display HTTP response content length")
+ flag.BoolVar(&options.StoreResponse, "sr", false, "Store HTTP response to directoy (default 'output')")
+ flag.StringVar(&options.StoreResponseDir, "srd", "output", "Custom directory to store HTTP responses")
+ flag.BoolVar(&options.FollowRedirects, "follow-redirects", false, "Follow HTTP Redirects")
+ flag.BoolVar(&options.FollowHostRedirects, "follow-host-redirects", false, "Only Follow redirects on the same host")
+ flag.IntVar(&options.MaxRedirects, "max-redirects", 10, "Max number of redirects to follow per host")
flag.StringVar(&options.HTTPProxy, "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080")
- flag.BoolVar(&options.JSONOutput, "json", false, "JSON Output")
- flag.StringVar(&options.InputFile, "l", "", "File containing domains")
- flag.StringVar(&options.Methods, "x", "", "Request Methods, use ALL to check all verbs ()")
+ flag.BoolVar(&options.JSONOutput, "json", false, "Display output in JSON format")
+ flag.StringVar(&options.InputFile, "l", "", "Input file containing list of hosts to process")
+ flag.StringVar(&options.Methods, "x", "", "Request Methods to use, use 'all' to probe all HTTP methods")
flag.BoolVar(&options.OutputMethod, "method", false, "Display request method")
flag.BoolVar(&options.Silent, "silent", false, "Silent mode")
flag.BoolVar(&options.Version, "version", false, "Show version of httpx")
flag.BoolVar(&options.Verbose, "verbose", false, "Verbose Mode")
- flag.BoolVar(&options.NoColor, "no-color", false, "No Color")
- flag.BoolVar(&options.OutputServerHeader, "web-server", false, "Extracts server header")
- flag.BoolVar(&options.OutputWebSocket, "websocket", false, "Prints out if the server exposes a websocket")
- flag.BoolVar(&options.responseInStdout, "response-in-json", false, "Show Raw HTTP Response In Output (-json only) (deprecated)")
- flag.BoolVar(&options.responseInStdout, "include-response", false, "Show Raw HTTP Response In Output (-json only)")
+ flag.BoolVar(&options.NoColor, "no-color", false, "Disable colored output")
+ flag.BoolVar(&options.OutputServerHeader, "web-server", false, "Display server header")
+ flag.BoolVar(&options.OutputWebSocket, "websocket", false, "Display server using websocket")
+ flag.BoolVar(&options.responseInStdout, "response-in-json", false, "Show Raw HTTP response In Output (-json only) (deprecated)")
+ flag.BoolVar(&options.responseInStdout, "include-response", false, "Show Raw HTTP response In Output (-json only)")
flag.BoolVar(&options.chainInStdout, "include-chain", false, "Show Raw HTTP Chain In Output (-json only)")
flag.BoolVar(&options.TLSProbe, "tls-probe", false, "Send HTTP probes on the extracted TLS domains")
flag.BoolVar(&options.CSPProbe, "csp-probe", false, "Send HTTP probes on the extracted CSP domains")
flag.StringVar(&options.RequestURI, "path", "", "Request path/file (example '/api')")
flag.StringVar(&options.RequestURIs, "paths", "", "Command separated paths or file containing one path per line (example '/api/v1,/apiv2')")
- flag.BoolVar(&options.OutputContentType, "content-type", false, "Extracts content-type")
- flag.StringVar(&options.OutputMatchStatusCode, "mc", "", "Match status code")
- flag.StringVar(&options.OutputMatchStatusCode, "ml", "", "Match content length")
- flag.StringVar(&options.OutputFilterStatusCode, "fc", "", "Filter status code")
- flag.StringVar(&options.OutputFilterContentLength, "fl", "", "Filter content length")
+ flag.BoolVar(&options.OutputContentType, "content-type", false, "Display content-type header")
+ flag.StringVar(&options.OutputMatchStatusCode, "mc", "", "Match response with specific status code (-mc 200,302)")
+ flag.StringVar(&options.OutputMatchContentLength, "ml", "", "Match response with specific content length (-ml 102)")
+ flag.StringVar(&options.OutputFilterStatusCode, "fc", "", "Filter response with specific status code (-fc 403,401)")
+ flag.StringVar(&options.OutputFilterContentLength, "fl", "", "Filter response with specific content length (-fl 23)")
flag.StringVar(&options.InputRawRequest, "request", "", "File containing raw request")
flag.BoolVar(&options.Unsafe, "unsafe", false, "Send raw requests skipping golang normalization")
flag.StringVar(&options.RequestBody, "body", "", "Content to send in body with HTTP request")
flag.BoolVar(&options.Debug, "debug", false, "Debug mode")
- flag.BoolVar(&options.Pipeline, "pipeline", false, "HTTP1.1 Pipeline")
+ flag.BoolVar(&options.Pipeline, "pipeline", false, "HTTP1.1 Pipeline probe")
flag.BoolVar(&options.HTTP2Probe, "http2", false, "HTTP2 probe")
- flag.BoolVar(&options.OutputIP, "ip", false, "Output target ip")
- flag.StringVar(&options.OutputFilterString, "filter-string", "", "Filter String")
- flag.StringVar(&options.OutputMatchString, "match-string", "", "Match string")
- flag.StringVar(&options.OutputFilterRegex, "filter-regex", "", "Filter Regex")
- flag.StringVar(&options.OutputMatchRegex, "match-regex", "", "Match Regex")
- flag.BoolVar(&options.OutputCName, "cname", false, "Output first cname")
- flag.BoolVar(&options.OutputCDN, "cdn", false, "Check if domain's ip belongs to known CDN (akamai, cloudflare, ..)")
- flag.BoolVar(&options.OutputResponseTime, "response-time", false, "Output the response time")
- flag.BoolVar(&options.NoFallback, "no-fallback", false, "If HTTPS on port 443 is successful on default configuration, probes also port 80 for HTTP")
- flag.BoolVar(&options.NoFallbackScheme, "no-fallback-scheme", false, "The tool will respect and attempt the scheme specified in the url (if HTTPS is specified no HTTP is attempted)")
+ flag.BoolVar(&options.OutputIP, "ip", false, "Display Host IP")
+ flag.StringVar(&options.OutputFilterString, "filter-string", "", "Filter response with specific string")
+ flag.StringVar(&options.OutputMatchString, "match-string", "", "Match response with specific string")
+ flag.StringVar(&options.OutputFilterRegex, "filter-regex", "", "Filter response with specific regex")
+ flag.StringVar(&options.OutputMatchRegex, "match-regex", "", "Match response with specific regex")
+ flag.BoolVar(&options.OutputCName, "cname", false, "Display Host cname")
+ flag.BoolVar(&options.OutputCDN, "cdn", false, "Diplay CDN")
+ flag.BoolVar(&options.OutputResponseTime, "response-time", false, "Display the response time")
+ flag.BoolVar(&options.NoFallback, "no-fallback", false, "Probe both protocol (HTTPS and HTTP)")
+ flag.BoolVar(&options.NoFallbackScheme, "no-fallback-scheme", false, "Probe with input protocol scheme")
flag.BoolVar(&options.ShowStatistics, "stats", false, "Enable statistic on keypress (terminal may become unresponsive till the end)")
flag.BoolVar(&options.RandomAgent, "random-agent", true, "Use randomly selected HTTP User-Agent header value")
flag.BoolVar(&options.StoreChain, "store-chain", false, "Save chain to file (default 'output')")
- flag.Var(&options.Allow, "allow", "Allowlist ip/cidr")
- flag.Var(&options.Deny, "deny", "Denylist ip/cidr")
+ flag.Var(&options.Allow, "allow", "Allow list of IP/CIDR's to process (file or comma separated)")
+ flag.Var(&options.Deny, "deny", "Deny list of IP/CIDR's to process (file or comma separated)")
flag.IntVar(&options.MaxResponseBodySizeToSave, "response-size-to-save", math.MaxInt32, "Max response size to save in bytes (default - unlimited)")
flag.IntVar(&options.MaxResponseBodySizeToRead, "response-size-to-read", math.MaxInt32, "Max response size to read in bytes (default - unlimited)")
- flag.StringVar(&options.OutputExtractRegex, "extract-regex", "", "Extract Regex")
+ flag.StringVar(&options.OutputExtractRegex, "extract-regex", "", "Display response content with matched regex")
flag.IntVar(&options.RateLimit, "rate-limit", 150, "Maximum requests to send per second")
flag.BoolVar(&options.Probe, "probe", false, "Display probe status")
flag.BoolVar(&options.Resume, "resume", false, "Resume scan using resume.cfg")
flag.BoolVar(&options.ExcludeCDN, "exclude-cdn", false, "Skip full port scans for CDNs (only checks for 80,443)")
+ flag.IntVar(&options.HostMaxErrors, "max-host-error", 30, "Max error count per host before skipping remaining path/s")
flag.Parse()
@@ -284,7 +291,7 @@ func ParseOptions() *Options {
}
func (options *Options) validateOptions() {
- if options.InputFile != "" && !fileutil.FileNameIsGlob(options.InputFile) && !fileutil.FileExists(options.InputFile) {
+ if options.InputFile != "" && !fileutilz.FileNameIsGlob(options.InputFile) && !fileutil.FileExists(options.InputFile) {
gologger.Fatal().Msgf("File %s does not exist!\n", options.InputFile)
}
diff --git a/runner/runner.go b/runner/runner.go
index e0798f48..55caab28 100644
--- a/runner/runner.go
+++ b/runner/runner.go
@@ -8,6 +8,7 @@ import (
"encoding/json"
"fmt"
"io/ioutil"
+ "net"
"net/http"
"net/http/httputil"
"net/url"
@@ -19,6 +20,7 @@ import (
"strings"
"time"
+ "github.com/bluele/gcache"
"github.com/logrusorgru/aurora"
"github.com/pkg/errors"
"github.com/projectdiscovery/clistats"
@@ -29,11 +31,12 @@ import (
// automatic fd max increase if running as root
_ "github.com/projectdiscovery/fdmax/autofdmax"
+ "github.com/projectdiscovery/fileutil"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/hmap/store/hybrid"
pdhttputil "github.com/projectdiscovery/httputil"
customport "github.com/projectdiscovery/httpx/common/customports"
- "github.com/projectdiscovery/httpx/common/fileutil"
+ fileutilz "github.com/projectdiscovery/httpx/common/fileutil"
"github.com/projectdiscovery/httpx/common/httputilz"
"github.com/projectdiscovery/httpx/common/httpx"
"github.com/projectdiscovery/httpx/common/slice"
@@ -52,13 +55,14 @@ const (
// Runner is a client for running the enumeration process.
type Runner struct {
- options *Options
- hp *httpx.HTTPX
- wappalyzer *wappalyzer.Wappalyze
- scanopts scanOptions
- hm *hybrid.HybridMap
- stats clistats.StatisticsClient
- ratelimiter ratelimit.Limiter
+ options *Options
+ hp *httpx.HTTPX
+ wappalyzer *wappalyzer.Wappalyze
+ scanopts scanOptions
+ hm *hybrid.HybridMap
+ stats clistats.StatisticsClient
+ ratelimiter ratelimit.Limiter
+ HostErrorsCache gcache.Cache
}
// New creates a new client for running enumeration process.
@@ -81,9 +85,10 @@ func New(options *Options) (*Runner, error) {
httpxOptions.RetryMax = options.Retries
httpxOptions.FollowRedirects = options.FollowRedirects
httpxOptions.FollowHostRedirects = options.FollowHostRedirects
+ httpxOptions.MaxRedirects = options.MaxRedirects
httpxOptions.HTTPProxy = options.HTTPProxy
httpxOptions.Unsafe = options.Unsafe
- httpxOptions.RequestOverride = httpx.RequestOverride{URIPath: options.RequestURI}
+ httpxOptions.UnsafeURI = options.RequestURI
httpxOptions.CdnCheck = options.OutputCDN
httpxOptions.ExcludeCdn = options.ExcludeCDN
httpxOptions.RandomAgent = options.RandomAgent
@@ -155,6 +160,10 @@ func New(options *Options) (*Runner, error) {
if strings.EqualFold(options.Methods, "all") {
scanopts.Methods = pdhttputil.AllHTTPMethods()
} else if options.Methods != "" {
+ // if unsafe is specified then converts the methods to uppercase
+ if !options.Unsafe {
+ options.Methods = strings.ToUpper(options.Methods)
+ }
scanopts.Methods = append(scanopts.Methods, stringz.SplitByCharAndTrimSpace(options.Methods, ",")...)
}
if len(scanopts.Methods) == 0 {
@@ -207,6 +216,7 @@ func New(options *Options) (*Runner, error) {
}
scanopts.ExcludeCDN = options.ExcludeCDN
+ scanopts.HostMaxErrors = options.HostMaxErrors
runner.scanopts = scanopts
if options.ShowStatistics {
@@ -228,13 +238,20 @@ func New(options *Options) (*Runner, error) {
runner.ratelimiter = ratelimit.NewUnlimited()
}
+ if options.HostMaxErrors >= 0 {
+ gc := gcache.New(1000).
+ ARC().
+ Build()
+ runner.HostErrorsCache = gc
+ }
+
return runner, nil
}
func (r *Runner) prepareInput() {
// Check if the user requested multiple paths
if fileutil.FileExists(r.options.RequestURIs) {
- r.options.requestURIs = fileutil.LoadFile(r.options.RequestURIs)
+ r.options.requestURIs = fileutilz.LoadFile(r.options.RequestURIs)
} else if r.options.RequestURIs != "" {
r.options.requestURIs = strings.Split(r.options.RequestURIs, ",")
}
@@ -251,7 +268,7 @@ func (r *Runner) prepareInput() {
gologger.Fatal().Msgf("Could read input file '%s': %s\n", r.options.InputFile, err)
}
} else if r.options.InputFile != "" {
- files, err := fileutil.ListFilesWithPattern(r.options.InputFile)
+ files, err := fileutilz.ListFilesWithPattern(r.options.InputFile)
if err != nil {
gologger.Fatal().Msgf("No input provided: %s", err)
}
@@ -378,6 +395,9 @@ func (r *Runner) Close() {
// nolint:errcheck // ignore
r.hm.Close()
r.hp.Dialer.Close()
+ if r.options.HostMaxErrors >= 0 {
+ r.HostErrorsCache.Purge()
+ }
}
// RunEnumeration on targets for httpx client
@@ -501,7 +521,7 @@ func (r *Runner) RunEnumeration() {
func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.HTTPX, protocol string, scanopts *scanOptions, output chan Result) {
protocols := []string{protocol}
- if scanopts.NoFallback {
+ if scanopts.NoFallback || protocol == httpx.HTTPandHTTPS {
protocols = []string{httpx.HTTPS, httpx.HTTP}
}
@@ -513,7 +533,7 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.
wg.Add()
go func(target, method, protocol string) {
defer wg.Done()
- result := r.analyze(hp, protocol, target, method, scanopts)
+ result := r.analyze(hp, protocol, target, method, t, scanopts)
output <- result
if scanopts.TLSProbe && result.TLSData != nil {
scanopts.TLSProbe = false
@@ -535,24 +555,30 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.
}
}
- for port, wantedProtocol := range customport.Ports {
- for _, method := range scanopts.Methods {
- wg.Add()
- go func(port int, method, protocol string) {
- defer wg.Done()
- h, _ := urlutil.ChangePort(target, fmt.Sprint(port))
- result := r.analyze(hp, protocol, h, method, scanopts)
- output <- result
- if scanopts.TLSProbe && result.TLSData != nil {
- scanopts.TLSProbe = false
- for _, tt := range result.TLSData.DNSNames {
- r.process(tt, wg, hp, protocol, scanopts, output)
- }
- for _, tt := range result.TLSData.CommonName {
- r.process(tt, wg, hp, protocol, scanopts, output)
+ for port, wantedProtocolForPort := range customport.Ports {
+ wantedProtocols := []string{wantedProtocolForPort}
+ if wantedProtocolForPort == httpx.HTTPandHTTPS {
+ wantedProtocols = []string{httpx.HTTPS, httpx.HTTP}
+ }
+ for _, wantedProtocol := range wantedProtocols {
+ for _, method := range scanopts.Methods {
+ wg.Add()
+ go func(port int, method, protocol string) {
+ defer wg.Done()
+ h, _ := urlutil.ChangePort(target, fmt.Sprint(port))
+ result := r.analyze(hp, protocol, h, method, t, scanopts)
+ output <- result
+ if scanopts.TLSProbe && result.TLSData != nil {
+ scanopts.TLSProbe = false
+ for _, tt := range result.TLSData.DNSNames {
+ r.process(tt, wg, hp, protocol, scanopts, output)
+ }
+ for _, tt := range result.TLSData.CommonName {
+ r.process(tt, wg, hp, protocol, scanopts, output)
+ }
}
- }
- }(port, method, wantedProtocol)
+ }(port, method, wantedProtocol)
+ }
}
}
if r.options.ShowStatistics {
@@ -590,9 +616,9 @@ func targets(target string) chan string {
return results
}
-func (r *Runner) analyze(hp *httpx.HTTPX, protocol, domain, method string, scanopts *scanOptions) Result {
+func (r *Runner) analyze(hp *httpx.HTTPX, protocol, domain, method, origInput string, scanopts *scanOptions) Result {
origProtocol := protocol
- if protocol == httpx.HTTPorHTTPS {
+ if protocol == httpx.HTTPorHTTPS || protocol == httpx.HTTPandHTTPS {
protocol = httpx.HTTPS
}
retried := false
@@ -602,20 +628,29 @@ retry:
parts := strings.Split(domain, ",")
//nolint:gomnd // not a magic number
if len(parts) != 2 {
- return Result{}
+ return Result{Input: origInput}
}
domain = parts[0]
customHost = parts[1]
}
URL, err := urlutil.Parse(domain)
if err != nil {
- return Result{URL: domain, err: err}
+ return Result{URL: domain, Input: origInput, err: err}
+ }
+
+ // check if we have to skip the host:port as a result of a previous failure
+ hostPort := net.JoinHostPort(URL.Host, URL.Port)
+ if r.options.HostMaxErrors >= 0 && r.HostErrorsCache.Has(hostPort) {
+ numberOfErrors, err := r.HostErrorsCache.GetIFPresent(hostPort)
+ if err == nil && numberOfErrors.(int) >= r.options.HostMaxErrors {
+ return Result{URL: domain, err: errors.New("skipping as previously unresponsive")}
+ }
}
// check if the combination host:port should be skipped if belonging to a cdn
if r.skipCDNPort(URL.Host, URL.Port) {
gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port)
- return Result{URL: domain, err: errors.New("cdn target only allows ports 80 and 443")}
+ return Result{URL: domain, Input: origInput, err: errors.New("cdn target only allows ports 80 and 443")}
}
URL.Scheme = protocol
@@ -624,19 +659,25 @@ retry:
URL.Port = ""
}
- if !scanopts.Unsafe {
+ var reqURI string
+ // retry with unsafe
+ if scanopts.Unsafe {
+ reqURI = URL.RequestURI + scanopts.RequestURI
+ // then create a base request without it to avoid go errors
+ URL.RequestURI = ""
+ } else {
+ // in case of standard requests append the new path to the existing one
URL.RequestURI += scanopts.RequestURI
}
req, err := hp.NewRequest(method, URL.String())
if err != nil {
- return Result{URL: URL.String(), err: err}
+ return Result{URL: URL.String(), Input: origInput, err: err}
}
+
if customHost != "" {
req.Host = customHost
}
- reqURI := req.URL.RequestURI()
-
hp.SetCustomHeaders(req, hp.CustomHeaders)
// We set content-length even if zero to allow net/http to follow 307/308 redirects (it fails on unknown size)
if scanopts.RequestBody != "" {
@@ -649,7 +690,11 @@ retry:
r.ratelimiter.Take()
- resp, err := hp.Do(req)
+ // with rawhttp we should say to the server to close the connection, otherwise it will remain open
+ if scanopts.Unsafe {
+ req.Header.Add("Connection", "close")
+ }
+ resp, err := hp.Do(req, httpx.UnsafeOptions{URIPath: reqURI})
if r.options.ShowStatistics {
r.stats.IncrementCounter("requests", 1)
}
@@ -659,7 +704,7 @@ retry:
var errDump error
requestDump, errDump = rawhttp.DumpRequestRaw(req.Method, req.URL.String(), reqURI, req.Header, req.Body, rawhttp.DefaultOptions)
if errDump != nil {
- return Result{URL: URL.String(), err: errDump}
+ return Result{URL: URL.String(), Input: origInput, err: errDump}
}
} else {
// Create a copy on the fly of the request body
@@ -668,7 +713,7 @@ retry:
var errDump error
requestDump, errDump = httputil.DumpRequestOut(req.Request, true)
if errDump != nil {
- return Result{URL: URL.String(), err: errDump}
+ return Result{URL: URL.String(), Input: origInput, err: errDump}
}
// The original req.Body gets modified indirectly by httputil.DumpRequestOut so we set it again to nil if it was empty
// Otherwise redirects like 307/308 would fail (as they require the body to be sent along)
@@ -678,16 +723,18 @@ retry:
}
}
+ // fix the final output url
fullURL := req.URL.String()
-
- builder := &strings.Builder{}
-
- // if the full url doesn't end with the custom path we pick the original input value
- if !stringsutil.HasSuffixAny(fullURL, scanopts.RequestURI) {
- parsedURL, _ := urlutil.Parse(fullURL)
+ parsedURL, _ := urlutil.Parse(fullURL)
+ if r.options.Unsafe {
+ parsedURL.RequestURI = reqURI
+ // if the full url doesn't end with the custom path we pick the original input value
+ } else if !stringsutil.HasSuffixAny(fullURL, scanopts.RequestURI) {
parsedURL.RequestURI = scanopts.RequestURI
- fullURL = parsedURL.String()
}
+ fullURL = parsedURL.String()
+
+ builder := &strings.Builder{}
builder.WriteString(stringz.RemoveURLDefaultPort(fullURL))
if r.options.Probe {
@@ -724,10 +771,21 @@ retry:
retried = true
goto retry
}
+
+ // mark the host:port as failed to avoid further checks
+ if r.options.HostMaxErrors >= 0 {
+ errorCount, err := r.HostErrorsCache.GetIFPresent(hostPort)
+ if err != nil || errorCount == nil {
+ _ = r.HostErrorsCache.Set(hostPort, 1)
+ } else if errorCount != nil {
+ _ = r.HostErrorsCache.Set(hostPort, errorCount.(int)+1)
+ }
+ }
+
if r.options.Probe {
- return Result{URL: URL.String(), Input: domain, Timestamp: time.Now(), err: err, Failed: err != nil, Error: errString, str: builder.String()}
+ return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), err: err, Failed: err != nil, Error: errString, str: builder.String()}
} else {
- return Result{URL: URL.String(), Input: domain, Timestamp: time.Now(), err: err}
+ return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), err: err}
}
}
@@ -825,7 +883,7 @@ retry:
isvhost := false
if scanopts.VHost {
r.ratelimiter.Take()
- isvhost, _ = hp.IsVirtualHost(req)
+ isvhost, _ = hp.IsVirtualHost(req, httpx.UnsafeOptions{})
if isvhost {
builder.WriteString(" [vhost]")
}
@@ -972,7 +1030,7 @@ retry:
parsed, err := urlutil.Parse(fullURL)
if err != nil {
- return Result{URL: fullURL, err: errors.Wrap(err, "could not parse url")}
+ return Result{URL: fullURL, Input: origInput, err: errors.Wrap(err, "could not parse url")}
}
finalPort := parsed.Port
@@ -1016,7 +1074,7 @@ retry:
HeaderSHA256: headersSha,
raw: resp.Raw,
URL: fullURL,
- Input: domain,
+ Input: origInput,
ContentLength: resp.ContentLength,
ChainStatusCodes: chainStatusCodes,
Chain: chainItems,