Merge pull request #375 from projectdiscovery/373-bugfix-non-rfc-urls

Fixing input URLs handling with non-rfc paths

common/httpx/httpx.go

@@ -25,15 +25,14 @@ import (
 
 // HTTPX represent an instance of the library client
 type HTTPX struct {
-	client          *retryablehttp.Client
-	client2         *http.Client
-	Filters         []Filter
-	Options         *Options
-	htmlPolicy      *bluemonday.Policy
-	CustomHeaders   map[string]string
-	RequestOverride *RequestOverride
-	cdn             *cdncheck.Client
-	Dialer          *fastdialer.Dialer
+	client        *retryablehttp.Client
+	client2       *http.Client
+	Filters       []Filter
+	Options       *Options
+	htmlPolicy    *bluemonday.Policy
+	CustomHeaders map[string]string
+	cdn           *cdncheck.Client
+	Dialer        *fastdialer.Dialer
 }
 
 // New httpx instance
@@ -124,7 +123,6 @@ func New(options *Options) (*HTTPX, error) {
 
 	httpx.htmlPolicy = bluemonday.NewPolicy()
 	httpx.CustomHeaders = httpx.Options.CustomHeaders
-	httpx.RequestOverride = &options.RequestOverride
 	if options.CdnCheck || options.ExcludeCdn {
 		httpx.cdn, err = cdncheck.NewWithCache()
 		if err != nil {
@@ -136,12 +134,12 @@ func New(options *Options) (*HTTPX, error) {
 }
 
 // Do http request
-func (h *HTTPX) Do(req *retryablehttp.Request) (*Response, error) {
+func (h *HTTPX) Do(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (*Response, error) {
 	timeStart := time.Now()
 
 	var gzipRetry bool
 get_response:
-	httpresp, err := h.getResponse(req)
+	httpresp, err := h.getResponse(req, unsafeOptions)
 	if err != nil {
 		return nil, err
 	}
@@ -236,33 +234,33 @@ get_response:
 }
 
 // RequestOverride contains the URI path to override the request
-type RequestOverride struct {
+type UnsafeOptions struct {
 	URIPath string
 }
 
 // getResponse returns response from safe / unsafe request
-func (h *HTTPX) getResponse(req *retryablehttp.Request) (*http.Response, error) {
+func (h *HTTPX) getResponse(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (*http.Response, error) {
 	if h.Options.Unsafe {
-		return h.doUnsafe(req)
+		return h.doUnsafeWithOptions(req, unsafeOptions)
 	}
 
 	return h.client.Do(req)
 }
 
 // doUnsafe does an unsafe http request
-func (h *HTTPX) doUnsafe(req *retryablehttp.Request) (*http.Response, error) {
+func (h *HTTPX) doUnsafeWithOptions(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (*http.Response, error) {
 	method := req.Method
 	headers := req.Header
 	targetURL := req.URL.String()
 	body := req.Body
 	options := rawhttp.DefaultOptions
 	options.Timeout = h.Options.Timeout
-	return rawhttp.DoRawWithOptions(method, targetURL, h.RequestOverride.URIPath, headers, body, options)
+	return rawhttp.DoRawWithOptions(method, targetURL, unsafeOptions.URIPath, headers, body, options)
 }
 
 // Verify the http calls and apply-cascade all the filters, as soon as one matches it returns true
-func (h *HTTPX) Verify(req *retryablehttp.Request) (bool, error) {
-	resp, err := h.Do(req)
+func (h *HTTPX) Verify(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (bool, error) {
+	resp, err := h.Do(req, unsafeOptions)
 	if err != nil {
 		return false, err
 	}

common/httpx/option.go

@@ -8,7 +8,6 @@ import (
 type Options struct {
 	RandomAgent      bool
 	DefaultUserAgent string
-	RequestOverride  RequestOverride
 	HTTPProxy        string
 	SocksProxy       string
 	Threads          int
@@ -36,6 +35,7 @@ type Options struct {
 	Deny                      []string
 	MaxResponseBodySizeToSave int64
 	MaxResponseBodySizeToRead int64
+	UnsafeURI                 string
 }
 
 // DefaultOptions contains the default options

common/httpx/virtualhost.go

@@ -11,16 +11,16 @@ import (
 const simMultiplier = 100
 
 // IsVirtualHost checks if the target endpoint is a virtual host
-func (h *HTTPX) IsVirtualHost(req *retryablehttp.Request) (bool, error) {
-	httpresp1, err := h.Do(req)
+func (h *HTTPX) IsVirtualHost(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (bool, error) {
+	httpresp1, err := h.Do(req, unsafeOptions)
 	if err != nil {
 		return false, err
 	}
 
 	// request a non-existing endpoint
 	req.Host = fmt.Sprintf("%s.%s", xid.New().String(), req.Host)
 
-	httpresp2, err := h.Do(req)
+	httpresp2, err := h.Do(req, unsafeOptions)
 	if err != nil {
 		return false, err
 	}

common/stringz/stringz.go

@@ -1,6 +1,7 @@
 package stringz
 
 import (
+	"net/url"
 	"strconv"
 	"strings"
 
@@ -73,3 +74,13 @@ func RemoveURLDefaultPort(rawURL string) string {
 	}
 	return u.String()
 }
+
+func GetInvalidURI(rawURL string) (bool, string) {
+	if _, err := url.Parse(rawURL); err != nil {
+		if u, err := urlutil.Parse(rawURL); err == nil {
+			return true, u.RequestURI
+		}
+	}
+
+	return false, ""
+}

runner/runner.go

@@ -88,7 +88,7 @@ func New(options *Options) (*Runner, error) {
 	httpxOptions.MaxRedirects = options.MaxRedirects
 	httpxOptions.HTTPProxy = options.HTTPProxy
 	httpxOptions.Unsafe = options.Unsafe
-	httpxOptions.RequestOverride = httpx.RequestOverride{URIPath: options.RequestURI}
+	httpxOptions.UnsafeURI = options.RequestURI
 	httpxOptions.CdnCheck = options.OutputCDN
 	httpxOptions.ExcludeCdn = options.ExcludeCDN
 	httpxOptions.RandomAgent = options.RandomAgent
@@ -659,19 +659,25 @@ retry:
 		URL.Port = ""
 	}
 
-	if !scanopts.Unsafe {
+	var reqURI string
+	// retry with unsafe
+	if scanopts.Unsafe {
+		reqURI = URL.RequestURI + scanopts.RequestURI
+		// then create a base request without it to avoid go errors
+		URL.RequestURI = ""
+	} else {
+		// in case of standard requests append the new path to the existing one
 		URL.RequestURI += scanopts.RequestURI
 	}
 	req, err := hp.NewRequest(method, URL.String())
 	if err != nil {
 		return Result{URL: URL.String(), Input: origInput, err: err}
 	}
+
 	if customHost != "" {
 		req.Host = customHost
 	}
 
-	reqURI := req.URL.RequestURI()
-
 	hp.SetCustomHeaders(req, hp.CustomHeaders)
 	// We set content-length even if zero to allow net/http to follow 307/308 redirects (it fails on unknown size)
 	if scanopts.RequestBody != "" {
@@ -684,7 +690,11 @@ retry:
 
 	r.ratelimiter.Take()
 
-	resp, err := hp.Do(req)
+	// with rawhttp we should say to the server to close the connection, otherwise it will remain open
+	if scanopts.Unsafe {
+		req.Header.Add("Connection", "close")
+	}
+	resp, err := hp.Do(req, httpx.UnsafeOptions{URIPath: reqURI})
 	if r.options.ShowStatistics {
 		r.stats.IncrementCounter("requests", 1)
 	}
@@ -713,16 +723,18 @@ retry:
 		}
 	}
 
+	// fix the final output url
 	fullURL := req.URL.String()
-
-	builder := &strings.Builder{}
-
-	// if the full url doesn't end with the custom path we pick the original input value
-	if !stringsutil.HasSuffixAny(fullURL, scanopts.RequestURI) {
-		parsedURL, _ := urlutil.Parse(fullURL)
+	parsedURL, _ := urlutil.Parse(fullURL)
+	if r.options.Unsafe {
+		parsedURL.RequestURI = reqURI
+		// if the full url doesn't end with the custom path we pick the original input value
+	} else if !stringsutil.HasSuffixAny(fullURL, scanopts.RequestURI) {
 		parsedURL.RequestURI = scanopts.RequestURI
-		fullURL = parsedURL.String()
 	}
+	fullURL = parsedURL.String()
+
+	builder := &strings.Builder{}
 	builder.WriteString(stringz.RemoveURLDefaultPort(fullURL))
 
 	if r.options.Probe {
@@ -871,7 +883,7 @@ retry:
 	isvhost := false
 	if scanopts.VHost {
 		r.ratelimiter.Take()
-		isvhost, _ = hp.IsVirtualHost(req)
+		isvhost, _ = hp.IsVirtualHost(req, httpx.UnsafeOptions{})
 		if isvhost {
 			builder.WriteString(" [vhost]")
 		}