search query for specific search engine (#66)

* search query for specific search engine

* misc updates

* added deduplication logic

* misc updates

* changed "shodanidb" to "shodan-idb"

* adding missing "HUNTER" search engine

* updated readme file

Co-authored-by: sandeep <sandeep@projectdiscovery.io>

README.md

@@ -61,15 +61,23 @@ Usage:
 
 Flags:
 INPUT:
-   -q, -query string[]   search query or list (file or comma separated or stdin)
-   -e, -engine string[]  search engine to query (shodan,shodan-idb,fofa,censys,hunter) (default shodan)
+   -q, -query string[]   search query, supports: stdin,file,config input (example: -q 'example query', -q 'query.txt')
+   -e, -engine string[]  search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter) (default shodan)
+
+SEARCH-ENGINE:
+   -s, -shodan string[]       search query for shodan (example: -shodan 'query.txt')
+   -sd, -shodan-idb string[]  search query for shodan-idb (example: -shodan-idb 'query.txt')
+   -ff, -fofa string[]        search query for fofa (example: -fofa 'query.txt')
+   -cs, -censys string[]      search query for censys (example: -censys 'query.txt')
+   -qk, -quake string[]       search query for quake (example: -quake 'query.txt')
+   -ht, -hunter string[]      search query for hunter (example: -hunter 'query.txt')
 
 CONFIG:
    -pc, -provider string  provider configuration file (default "$HOME/.config/uncover/provider-config.yaml")
    -config string         flag configuration file (default "$HOME/.config/uncover/config.yaml")
    -timeout int           timeout in seconds (default 30)
    -delay int             delay between requests in seconds (0 to disable) (default 1)
-   -retries int           number of times to retry a failed request
+   -retry int             number of times to retry a failed request (default 2)
 
 OUTPUT:
    -o, -output string  output file to write found results

runner/options.go

@@ -41,6 +41,12 @@ type Options struct {
 	delay        time.Duration
 	Provider     *Provider
 	Retries      int
+	Shodan       goflags.StringSlice
+	ShodanIdb    goflags.StringSlice
+	Fofa         goflags.StringSlice
+	Censys       goflags.StringSlice
+	Quake        goflags.StringSlice
+	Hunter       goflags.StringSlice
 }
 
 // ParseOptions parses the command line flags provided by a user
@@ -55,12 +61,21 @@ func ParseOptions() *Options {
 		flagSet.StringSliceVarP(&options.Engine, "engine", "e", nil, "search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter) (default shodan)", goflags.FileNormalizedStringSliceOptions),
 	)
 
+	flagSet.CreateGroup("search-engine", "Search-Engine",
+		flagSet.StringSliceVarP(&options.Shodan, "shodan", "s", nil, "search query for shodan (example: -shodan 'query.txt')", goflags.FileStringSliceOptions),
+		flagSet.StringSliceVarP(&options.ShodanIdb, "shodan-idb", "sd", nil, "search query for shodan-idb (example: -shodan-idb 'query.txt')", goflags.FileStringSliceOptions),
+		flagSet.StringSliceVarP(&options.Fofa, "fofa", "ff", nil, "search query for fofa (example: -fofa 'query.txt')", goflags.FileStringSliceOptions),
+		flagSet.StringSliceVarP(&options.Censys, "censys", "cs", nil, "search query for censys (example: -censys 'query.txt')", goflags.FileStringSliceOptions),
+		flagSet.StringSliceVarP(&options.Quake, "quake", "qk", nil, "search query for quake (example: -quake 'query.txt')", goflags.FileStringSliceOptions),
+		flagSet.StringSliceVarP(&options.Hunter, "hunter", "ht", nil, "search query for hunter (example: -hunter 'query.txt')", goflags.FileStringSliceOptions),
+	)
+
 	flagSet.CreateGroup("config", "Config",
 		flagSet.StringVarP(&options.ProviderFile, "provider", "pc", defaultProviderConfigLocation, "provider configuration file"),
 		flagSet.StringVar(&options.ConfigFile, "config", defaultConfigLocation, "flag configuration file"),
 		flagSet.IntVar(&options.Timeout, "timeout", 30, "timeout in seconds"),
 		flagSet.IntVar(&options.Delay, "delay", 1, "delay between requests in seconds (0 to disable)"),
-		flagSet.IntVar(&options.Retries, "retries", 1, "number of times to retry a failed request"),
+		flagSet.IntVar(&options.Retries, "retry", 2, "number of times to retry a failed request"),
 	)
 
 	flagSet.CreateGroup("output", "Output",
@@ -185,7 +200,7 @@ func (options *Options) loadProvidersFromEnv() error {
 func (options *Options) validateOptions() error {
 	// Check if domain, list of domains, or stdin info was provided.
 	// If none was provided, then return.
-	if len(options.Query) == 0 {
+	if len(options.Query) == 0 && len(options.Shodan) == 0 && len(options.Censys) == 0 && len(options.Quake) == 0 && len(options.Fofa) == 0 && len(options.ShodanIdb) == 0 && len(options.Hunter) == 0 {
 		return errors.New("no query provided")
 	}
 

runner/output_writer.go

@@ -2,10 +2,12 @@ package runner
 
 import (
 	"crypto/sha1"
+	"fmt"
 	"io"
 	"sync"
 
 	lru "github.com/hashicorp/golang-lru"
+	"github.com/projectdiscovery/uncover/uncover"
 )
 
 type OutputWriter struct {
@@ -30,19 +32,30 @@ func (o *OutputWriter) Write(data []byte) {
 	o.Lock()
 	defer o.Unlock()
 
-	// skip duplicates in the last 2048 printed items
-	itemHash := sha1.Sum(data)
-	if o.cache.Contains(itemHash) {
-		return
-	}
-	o.cache.Add(itemHash, struct{}{})
-
 	for _, writer := range o.writers {
 		_, _ = writer.Write(data)
 		_, _ = writer.Write([]byte("\n"))
 	}
 }
+func (o *OutputWriter) findDuplicate(data string) bool {
+	// check if we've already printed this data
+	itemHash := sha1.Sum([]byte(data))
+	if o.cache.Contains(itemHash) {
+		return true
+	}
+	o.cache.Add(itemHash, struct{}{})
+	return false
+}
 
 func (o *OutputWriter) WriteString(data string) {
+	if o.findDuplicate(data) {
+		return
+	}
 	o.Write([]byte(data))
 }
+func (o *OutputWriter) WriteJsonData(data uncover.Result) {
+	if o.findDuplicate(fmt.Sprintf("%s:%d", data.IP, data.Port)) {
+		return
+	}
+	o.Write([]byte(data.JSON()))
+}

runner/runner.go

@@ -2,7 +2,6 @@ package runner
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"math/rand"
 	"os"
@@ -66,6 +65,31 @@ func (r *Runner) Run(ctx context.Context, query ...string) error {
 	}
 
 	var agents []uncover.Agent
+	if len(r.options.Shodan) > 0 {
+		r.options.Engine = append(r.options.Engine, "shodan")
+		query = append(query, r.options.Shodan...)
+	}
+	if len(r.options.ShodanIdb) > 0 {
+		r.options.Engine = append(r.options.Engine, "shodan-idb")
+		query = append(query, r.options.ShodanIdb...)
+	}
+	if len(r.options.Fofa) > 0 {
+		r.options.Engine = append(r.options.Engine, "fofa")
+		query = append(query, r.options.Fofa...)
+	}
+	if len(r.options.Censys) > 0 {
+		r.options.Engine = append(r.options.Engine, "censys")
+		query = append(query, r.options.Censys...)
+	}
+	if len(r.options.Quake) > 0 {
+		r.options.Engine = append(r.options.Engine, "quake")
+		query = append(query, r.options.Quake...)
+	}
+	if len(r.options.Hunter) > 0 {
+		r.options.Engine = append(r.options.Engine, "hunter")
+		query = append(query, r.options.Hunter...)
+	}
+
 	// declare clients
 	for _, engine := range r.options.Engine {
 		var (
@@ -145,12 +169,8 @@ func (r *Runner) Run(ctx context.Context, query ...string) error {
 					case result.Error != nil:
 						gologger.Warning().Label(agent.Name()).Msgf("%s\n", result.Error.Error())
 					case r.options.JSON:
-						data, err := json.Marshal(result)
-						if err != nil {
-							continue
-						}
-						gologger.Verbose().Label(agent.Name()).Msgf("%s\n", string(data))
-						outputWriter.Write(data)
+						gologger.Verbose().Label(agent.Name()).Msgf("%s\n", result.JSON())
+						outputWriter.WriteJsonData(result)
 					case r.options.Raw:
 						gologger.Verbose().Label(agent.Name()).Msgf("%s\n", result.RawData())
 						outputWriter.WriteString(result.RawData())