Merge pull request #340 from projectdiscovery/331-bugfix-non-ascii-ch…

…aracters-in-path

Adding support for non-ascii characters in path for unsafe requests

go.mod

@@ -25,8 +25,8 @@ require (
 	github.com/projectdiscovery/rawhttp v0.0.7
 	github.com/projectdiscovery/retryabledns v1.0.12 // indirect
 	github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7dc
-	github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d // indirect
-	github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921
+	github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d
+	github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1
 	github.com/projectdiscovery/wappalyzergo v0.0.7
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.3.0

go.sum

@@ -167,8 +167,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7d
 github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
 github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d h1:nlOAex7twmrEqD5i6WLnugF9uO3DQ6jDEKN9gevrTAk=
 github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
-github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921 h1:EgaxpJm7+lKppfAHkFHs+S+II0lodp4Gu3leZCCkWlc=
-github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo=
+github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4=
+github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo=
 github.com/projectdiscovery/wappalyzergo v0.0.7 h1:MvlienkiFUbO3nDvlc5mNy1C5XiHzD2EklLDgnG9Zv4=
 github.com/projectdiscovery/wappalyzergo v0.0.7/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM=
 github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=

runner/runner.go

@@ -22,6 +22,7 @@ import (
 	"github.com/logrusorgru/aurora"
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/clistats"
+	"github.com/projectdiscovery/stringsutil"
 	"github.com/projectdiscovery/urlutil"
 
 	// automatic fd max increase if running as root
@@ -618,7 +619,15 @@ retry:
 	resp, err := hp.Do(req)
 
 	fullURL := req.URL.String()
+
 	builder := &strings.Builder{}
+
+	// if the full url doesn't end with the custom path we pick the original input value
+	if !stringsutil.HasSuffixAny(fullURL, scanopts.RequestURI) {
+		parsedURL, _ := urlutil.Parse(fullURL)
+		parsedURL.RequestURI = scanopts.RequestURI
+		fullURL = parsedURL.String()
+	}
 	builder.WriteString(stringz.RemoveURLDefaultPort(fullURL))
 
 	if r.options.Probe {