Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Adding support for lines/words matchers/filters #468

Merged
merged 2 commits into from
Jan 4, 2022
Merged

Adding support for lines/words matchers/filters #468

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
47bbacf
Adding support for lines/words matchers/filters
Mzack9999 Dec 31, 2021
20254fd
readme updates
ehsandeep Dec 31, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 19 additions & 12 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,18 +43,19 @@ httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers usi

### Supported probes:-

| Probes | Default check | Probes | Default check |
|--------------------|-----------------|--------------------|-----------------|
| URL | true | IP | true |
| Title | true | CNAME | true |
| Status Code | true | Raw HTTP | false |
| Content Length | true | HTTP2 | false |
| TLS Certificate | true | HTTP 1.1 Pipeline | false |
| CSP Header | true | Virtual host | false |
| Location Header | true | CDN | false |
| Web Server | true | Path | false |
| Web Socket | true | Ports | false |
| Response Time | true | Request method | false |
| Probes | Default check | Probes | Default check |
| --------------- | ------------- | ----------------- | ------------- |
| URL | true | IP | true |
| Title | true | CNAME | true |
| Status Code | true | Raw HTTP | false |
| Content Length | true | HTTP2 | false |
| Line Count | true | Word Count | true |
| TLS Certificate | true | HTTP 1.1 Pipeline | false |
| CSP Header | true | Virtual host | false |
| Location Header | true | CDN | false |
| Web Server | true | Paths | false |
| Web Socket | true | Ports | false |
| Response Time | true | Request Method | false |


# Installation Instructions
Expand Down Expand Up @@ -87,6 +88,8 @@ PROBES:
-sc, -status-code Display Status Code
-td, -tech-detect Display wappalyzer based technology detection
-cl, -content-length Display Content-Length
-lc, -line-count Display Response body line count
-wc, -word-count Display Response body word count
-server, -web-server Display Server header
-ct, -content-type Display Content-Type header
-rt, -response-time Display the response time
Expand All @@ -105,12 +108,16 @@ MATCHERS:
-ms, -match-string string Match response with given string
-mr, -match-regex string Match response with specific regex
-er, -extract-regex string Display response content with matched regex
-mlc, -match-line-count string Match Response body line count
-mwc, -match-word-count string Match Response body word count

FILTERS:
-fc, -filter-code string Filter response with given status code (-fc 403,401)
-fl, -filter-length string Filter response with given content length (-fl 23,33)
-fs, -filter-string string Filter response with specific string
-fe, -filter-regex string Filter response with specific regex
-flc, -filter-line-count string Filter Response body line count
-fwc, -filter-word-count string Filter Response body word count

RATE-LIMIT:
-t, -threads int Number of threads (default 50)
Expand Down
34 changes: 33 additions & 1 deletion runner/options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ type scanOptions struct {
ExcludeCDN bool
HostMaxErrors int
ProbeAllIPS bool
OutputLinesCount bool
OutputWordsCount bool
}

func (s *scanOptions) Clone() *scanOptions {
Expand Down Expand Up @@ -105,6 +107,8 @@ func (s *scanOptions) Clone() *scanOptions {
MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave,
MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead,
HostMaxErrors: s.HostMaxErrors,
OutputLinesCount: s.OutputLinesCount,
OutputWordsCount: s.OutputWordsCount,
}
}

Expand Down Expand Up @@ -199,6 +203,16 @@ type Options struct {
SkipDedupe bool
ProbeAllIPS bool
Resolvers goflags.NormalizedStringSlice
OutputLinesCount bool
OutputMatchLinesCount string
matchLinesCount []int
OutputFilterLinesCount string
filterLinesCount []int
OutputWordsCount bool
OutputMatchWordsCount string
matchWordsCount []int
OutputFilterWordsCount string
filterWordsCount []int
}

// ParseOptions parses the command line options for application
Expand All @@ -219,6 +233,8 @@ func ParseOptions() *Options {
flagSet.BoolVarP(&options.ContentLength, "content-length", "cl", false, "Display Content-Length"),
flagSet.BoolVarP(&options.OutputServerHeader, "web-server", "server", false, "Display Server header"),
flagSet.BoolVarP(&options.OutputContentType, "content-type", "ct", false, "Display Content-Type header"),
flagSet.BoolVarP(&options.OutputLinesCount, "line-count", "lc", false, "Display Response body line count"),
flagSet.BoolVarP(&options.OutputWordsCount, "word-count", "wc", false, "Display Response body word count"),
flagSet.BoolVarP(&options.OutputResponseTime, "response-time", "rt", false, "Display the response time"),
flagSet.BoolVar(&options.ExtractTitle, "title", false, "Display page title"),
flagSet.BoolVar(&options.Location, "location", false, "Display Location header"),
Expand All @@ -228,21 +244,25 @@ func ParseOptions() *Options {
flagSet.BoolVar(&options.OutputCName, "cname", false, "Display Host cname"),
flagSet.BoolVar(&options.OutputCDN, "cdn", false, "Display if CDN in use"),
flagSet.BoolVar(&options.Probe, "probe", false, "Display probe status"),
flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "Display response content with matched regex"),
)

createGroup(flagSet, "matchers", "Matchers",
flagSet.StringVarP(&options.OutputMatchStatusCode, "match-code", "mc", "", "Match response with given status code (-mc 200,302)"),
flagSet.StringVarP(&options.OutputMatchContentLength, "match-length", "ml", "", "Match response with given content length (-ml 100,102)"),
flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "Match response with given string"),
flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "Match response with specific regex"),
flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "Display response content with matched regex"),
flagSet.StringVarP(&options.OutputMatchLinesCount, "match-line-count", "mlc", "", "Match Response body line count"),
flagSet.StringVarP(&options.OutputMatchWordsCount, "match-word-count", "mwc", "", "Match Response body word count"),
)

createGroup(flagSet, "filters", "Filters",
flagSet.StringVarP(&options.OutputFilterStatusCode, "filter-code", "fc", "", "Filter response with given status code (-fc 403,401)"),
flagSet.StringVarP(&options.OutputFilterContentLength, "filter-length", "fl", "", "Filter response with given content length (-fl 23,33)"),
flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "Filter response with specific string"),
flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "Filter response with specific regex"),
flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "Filter Response body line count"),
flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-word-count", "fwc", "", "Filter Response body word count"),
)

createGroup(flagSet, "rate-limit", "Rate-Limit",
Expand Down Expand Up @@ -373,6 +393,18 @@ func (options *Options) validateOptions() {
gologger.Fatal().Msgf("Invalid value for match regex option: %s\n", err)
}
}
if options.matchLinesCount, err = stringz.StringToSliceInt(options.OutputMatchLinesCount); err != nil {
gologger.Fatal().Msgf("Invalid value for match lines count option: %s\n", err)
}
if options.matchWordsCount, err = stringz.StringToSliceInt(options.OutputMatchWordsCount); err != nil {
gologger.Fatal().Msgf("Invalid value for match words count option: %s\n", err)
}
if options.filterLinesCount, err = stringz.StringToSliceInt(options.OutputFilterLinesCount); err != nil {
gologger.Fatal().Msgf("Invalid value for filter lines count option: %s\n", err)
}
if options.filterWordsCount, err = stringz.StringToSliceInt(options.OutputFilterWordsCount); err != nil {
gologger.Fatal().Msgf("Invalid value for filter words count option: %s\n", err)
}

var resolvers []string
for _, resolver := range options.Resolvers {
Expand Down
38 changes: 38 additions & 0 deletions runner/runner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,8 @@ func New(options *Options) (*Runner, error) {
scanopts.ExcludeCDN = options.ExcludeCDN
scanopts.HostMaxErrors = options.HostMaxErrors
scanopts.ProbeAllIPS = options.ProbeAllIPS
scanopts.OutputLinesCount = options.OutputLinesCount
scanopts.OutputWordsCount = options.OutputWordsCount
runner.scanopts = scanopts

if options.ShowStatistics {
Expand Down Expand Up @@ -552,6 +554,12 @@ func (r *Runner) RunEnumeration() {
if len(r.options.filterContentLength) > 0 && slice.IntSliceContains(r.options.filterContentLength, resp.ContentLength) {
continue
}
if len(r.options.filterLinesCount) > 0 && slice.IntSliceContains(r.options.filterLinesCount, resp.Lines) {
continue
}
if len(r.options.filterWordsCount) > 0 && slice.IntSliceContains(r.options.filterWordsCount, resp.Words) {
continue
}
if r.options.filterRegex != nil && r.options.filterRegex.MatchString(resp.raw) {
continue
}
Expand All @@ -570,6 +578,12 @@ func (r *Runner) RunEnumeration() {
if r.options.OutputMatchString != "" && !strings.Contains(strings.ToLower(resp.raw), strings.ToLower(r.options.OutputMatchString)) {
continue
}
if len(r.options.matchLinesCount) > 0 && !slice.IntSliceContains(r.options.matchLinesCount, resp.Lines) {
continue
}
if len(r.options.matchWordsCount) > 0 && !slice.IntSliceContains(r.options.matchWordsCount, resp.Words) {
continue
}

row := resp.str
if r.options.JSONOutput {
Expand Down Expand Up @@ -1168,6 +1182,26 @@ retry:
builder.WriteRune(']')
}

if scanopts.OutputLinesCount {
builder.WriteString(" [")
if !scanopts.OutputWithNoColor {
builder.WriteString(aurora.Magenta(resp.Lines).String())
} else {
builder.WriteString(fmt.Sprint(resp.Lines))
}
builder.WriteRune(']')
}

if scanopts.OutputWordsCount {
builder.WriteString(" [")
if !scanopts.OutputWithNoColor {
builder.WriteString(aurora.Magenta(resp.Words).String())
} else {
builder.WriteString(fmt.Sprint(resp.Words))
}
builder.WriteRune(']')
}

// store responses or chain in directory
if scanopts.StoreResponse || scanopts.StoreChain {
domainFile := strings.ReplaceAll(urlutil.TrimScheme(URL.String()), ":", ".")
Expand Down Expand Up @@ -1271,6 +1305,8 @@ retry:
ResponseTime: resp.Duration.String(),
Technologies: technologies,
FinalURL: finalURL,
Lines: resp.Lines,
Words: resp.Words,
}
}

Expand Down Expand Up @@ -1322,6 +1358,8 @@ type Result struct {
Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"`
FinalURL string `json:"final-url,omitempty" csv:"final-url"`
Failed bool `json:"failed" csv:"failed"`
Lines int `json:"lines" csv:"lines"`
Words int `json:"words" csv:"words"`
}

// JSON the result
Expand Down