diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index d1e17471f7..52751ee196 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -13,7 +13,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.16 + go-version: 1.17 - name: Check out code uses: actions/checkout@v2 diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index 9c74852552..ec14f76d75 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -13,7 +13,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.15 + go-version: 1.17 - name: Check out code uses: actions/checkout@v2 diff --git a/.github/workflows/publish-docs.yaml b/.github/workflows/publish-docs.yaml index 54be79802c..6a96a98f34 100644 --- a/.github/workflows/publish-docs.yaml +++ b/.github/workflows/publish-docs.yaml @@ -1,6 +1,8 @@ name: ⏰ Publish Docs on: + push: + pull_request: workflow_dispatch: jobs: @@ -17,7 +19,7 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v2 with: - go-version: 1.16 + go-version: 1.17 - name: Generate YAML Syntax Documentation id: generate-docs @@ -28,7 +30,7 @@ jobs: fi go generate pkg/templates/templates.go go build -o "cmd/docgen/docgen" cmd/docgen/docgen.go - ./cmd/docgen/docgen syntax-reference.md nuclei-jsonschema.json + ./cmd/docgen/docgen ../SYNTAX-REFERENCE.md ../nuclei-jsonschema.json echo "::set-output name=changes::$(git status -s | wc -l)" working-directory: v2 @@ -37,7 +39,7 @@ jobs: run: | git config --local user.email "action@github.com" git config --local user.name "GitHub Action" - git add v2/syntax-reference.md v2/nuclei-jsonschema.json + git add SYNTAX-REFERENCE.md nuclei-jsonschema.json git commit -m "Auto Generate Syntax Docs + JSONSchema [$(date)] :robot:" -a - name: Push changes diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index 7bc3cc13c3..c0748e870a 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -18,7 +18,7 @@ jobs: name: "Set up Go" uses: actions/setup-go@v2 with: - go-version: 1.16 + go-version: 1.17 - env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/README.md b/README.md index fbb2c6928a..b1ee8b943f 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ Nuclei is a fast, template based vulnerability scanner focusing on extensive configurability, massive extensibility and ease of use. Usage: - ./nuclei [flags] + nuclei [flags] Flags: TARGET: @@ -100,7 +100,7 @@ FILTERING: -etags, -exclude-tags string[] exclude templates with the provided tags -include-templates string[] templates to be executed even if they are excluded either by default or configuration -exclude-templates, -exclude string[] template or template directory paths to exclude - -severity, -impact string[] execute templates that match the provided severities only + -severity, -impact value[] Templates to run based on severity. Possible values: info, low, medium, high, critical -author string[] execute templates that are (co-)created by the specified authors OUTPUT: @@ -111,7 +111,8 @@ OUTPUT: -nc, -no-color disable output content coloring (ANSI escape codes) -json write output in JSONL(ines) format -irr, -include-rr include request/response pairs in the JSONL output (for findings only) - -nm, -no-meta don't display match metadata + -nm, -no-meta don't display match metadata in CLI output + -nts, -no-timestamp don't display timestamp metadata in CLI output -rdb, -report-db string local nuclei reporting database (always use this to persist report data) -me, -markdown-export string directory to export results in markdown format -se, -sarif-export string file to export results in SARIF format @@ -124,7 +125,7 @@ CONFIGURATIONS: -r, -resolvers string file containing resolver list for nuclei -system-resolvers use system DNS resolving as error fallback -passive enable passive HTTP response processing mode - -env-vars Enable environment variables support + -env-vars enable environment variables support INTERACTSH: -no-interactsh do not use interactsh server for blind interaction polling @@ -143,8 +144,9 @@ RATE-LIMIT: OPTIMIZATIONS: -timeout int time to wait in seconds before timeout (default 5) -retries int number of times to retry a failed request (default 1) + -max-host-error int max errors for a host before skipping from scan (default 30) -project use a project folder to avoid sending same request multiple times - -project-path string set a specific project path (default "/var/folders/ml/m31ysb5x73l1s3kjlyn5g4180000gn/T/") + -project-path string set a specific project path (default "$TMPDIR/") -spm, -stop-at-first-path stop processing HTTP requests after the first match (may break template/workflow logic) HEADLESS: @@ -165,7 +167,7 @@ DEBUG: UPDATE: -update update nuclei to the latest released version -ut, -update-templates update the community templates to latest released version - -nut, -no-update-templates Do not check for nuclei-templates updates + -nut, -no-update-templates do not check for nuclei-templates updates -ud, -update-directory string overwrite the default nuclei-templates directory (default "$HOME/nuclei-templates") STATISTICS: diff --git a/v2/cmd/nuclei/issue-tracker-config.yaml b/v2/cmd/nuclei/issue-tracker-config.yaml index 27fe7a688e..7faee60517 100644 --- a/v2/cmd/nuclei/issue-tracker-config.yaml +++ b/v2/cmd/nuclei/issue-tracker-config.yaml @@ -33,8 +33,10 @@ # jira contains configuration options for jira issue tracker #jira: -# # Cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used +# # cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used # cloud: true +# # update-existing is the boolean which tells if the existing, opened issue should be updated or new one should be created +# update-existing: false # # URL is the jira application url # url: "" # # account-id is the account-id of the jira user or username in case of on-prem Jira diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index a82fc83e4e..7bd84689b1 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -127,7 +127,7 @@ on extensive configurability, massive extensibility and ease of use.`) createGroup(flagSet, "optimization", "Optimizations", flagSet.IntVar(&options.Timeout, "timeout", 5, "time to wait in seconds before timeout"), flagSet.IntVar(&options.Retries, "retries", 1, "number of times to retry a failed request"), - flagSet.IntVar(&options.HostMaxErrors, "host-max-error", 30, "max errors for a host before skipping from scan"), + flagSet.IntVar(&options.MaxHostError, "max-host-error", 30, "max errors for a host before skipping from scan"), flagSet.BoolVar(&options.Project, "project", false, "use a project folder to avoid sending same request multiple times"), flagSet.StringVar(&options.ProjectPath, "project-path", os.TempDir(), "set a specific project path"), diff --git a/v2/go.mod b/v2/go.mod index 75072f8bb8..a509f4c29a 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -13,6 +13,7 @@ require ( github.com/bluele/gcache v0.0.2 github.com/c4milo/unpackit v0.1.0 // indirect github.com/corpix/uarand v0.1.1 + github.com/davecgh/go-spew v1.1.1 github.com/go-rod/rod v0.91.1 github.com/google/go-github v17.0.0+incompatible github.com/gosuri/uilive v0.0.4 // indirect @@ -23,6 +24,7 @@ require ( github.com/karlseguin/ccache v2.0.3+incompatible github.com/karrick/godirwalk v1.16.1 github.com/logrusorgru/aurora v2.0.3+incompatible + github.com/mattn/go-runewidth v0.0.13 // indirect github.com/miekg/dns v1.1.43 github.com/olekukonko/tablewriter v0.0.5 github.com/owenrumney/go-sarif v1.0.11 @@ -58,5 +60,7 @@ require ( go.uber.org/ratelimit v0.2.0 golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab + golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf // indirect + google.golang.org/appengine v1.6.7 // indirect gopkg.in/yaml.v2 v2.4.0 ) diff --git a/v2/go.sum b/v2/go.sum index 8a195a898a..97e87b5d94 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -96,6 +96,7 @@ github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U= github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY= github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= @@ -228,6 +229,7 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 h1:i462o439ZjprVSFSZLZxcsoAe592sZB1rci2Z8j4wdk= github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA= github.com/itchyny/gojq v0.12.4 h1:8zgOZWMejEWCLjbF/1mWY7hY7QEARm7dtuhC6Bp4R8o= @@ -274,6 +276,7 @@ github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczG github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -281,6 +284,8 @@ github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1y github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= @@ -359,6 +364,8 @@ github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6n github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -501,6 +508,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -608,6 +616,8 @@ golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf h1:2ucpDCmfkl8Bd/FsLtiD653Wf96cW37s+iGx93zsu4k= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -666,6 +676,7 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -696,6 +707,8 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -779,6 +792,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +mvdan.cc/gofumpt v0.1.1 h1:bi/1aS/5W00E2ny5q65w9SnKpWEF/UIOqDYBILpo9rA= mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= diff --git a/v2/internal/runner/banner.go b/v2/internal/runner/banner.go index 36196349d7..eed5277d8e 100644 --- a/v2/internal/runner/banner.go +++ b/v2/internal/runner/banner.go @@ -20,6 +20,6 @@ func showBanner() { gologger.Print().Msgf("%s\n", banner) gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n") - gologger.Warning().Msgf("Use with caution. You are responsible for your actions\n") - gologger.Warning().Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n") + gologger.Error().Label("WRN").Msgf("Use with caution. You are responsible for your actions\n") + gologger.Error().Label("WRN").Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n") } diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 7f286da454..136bb91df7 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -39,6 +39,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/utils" + "github.com/projectdiscovery/nuclei/v2/pkg/utils/stats" ) // Runner is a client for running the enumeration process. @@ -71,6 +72,9 @@ func New(options *types.Options) (*Runner, error) { } return nil, nil } + if options.Validate { + parsers.ShouldValidate = true + } if err := runner.updateTemplates(); err != nil { gologger.Warning().Msgf("Could not update templates: %s\n", err) } @@ -297,8 +301,8 @@ func (r *Runner) RunEnumeration() error { r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...) var cache *hosterrorscache.Cache - if r.options.HostMaxErrors > 0 { - cache = hosterrorscache.New(r.options.HostMaxErrors, hosterrorscache.DefaultMaxHostsCount).SetVerbose(r.options.Verbose) + if r.options.MaxHostError > 0 { + cache = hosterrorscache.New(r.options.MaxHostError, hosterrorscache.DefaultMaxHostsCount).SetVerbose(r.options.Verbose) } r.hostErrors = cache executerOpts := protocols.ExecuterOptions{ @@ -339,14 +343,23 @@ func (r *Runner) RunEnumeration() error { if err != nil { return errors.Wrap(err, "could not load templates from config") } + store.Load() + if r.options.Validate { if err := store.ValidateTemplates(r.options.Templates, r.options.Workflows); err != nil { return err } - gologger.Info().Msgf("All templates validated successfully\n") + if stats.GetValue(parsers.SyntaxErrorStats) == 0 && stats.GetValue(parsers.SyntaxWarningStats) == 0 { + gologger.Info().Msgf("All templates validated successfully\n") + } else { + return errors.New("encountered errors while performing template validation") + } return nil // exit } - store.Load() + + // Display stats for any loaded templates syntax warnings or errors + stats.Display(parsers.SyntaxWarningStats) + stats.Display(parsers.SyntaxErrorStats) builder := &strings.Builder{} if r.templatesConfig != nil && r.templatesConfig.NucleiLatestVersion != "" { @@ -386,10 +399,11 @@ func (r *Runner) RunEnumeration() error { gologger.Info().Msgf("Using Interactsh Server %s", r.options.InteractshURL) } if len(store.Templates()) > 0 { - gologger.Info().Msgf("Templates loaded: %d (New: %d)", len(store.Templates()), r.countNewTemplates()) + gologger.Info().Msgf("Templates added in last update: %d", r.countNewTemplates()) + gologger.Info().Msgf("Templates loaded for scan: %d", len(store.Templates())) } if len(store.Workflows()) > 0 { - gologger.Info().Msgf("Workflows loaded: %d", len(store.Workflows())) + gologger.Info().Msgf("Workflows loaded for scan: %d", len(store.Workflows())) } // pre-parse all the templates, apply filters diff --git a/v2/nuclei-jsonschema.json b/v2/nuclei-jsonschema.json index 5eb5c9889c..6092a125a2 100755 --- a/v2/nuclei-jsonschema.json +++ b/v2/nuclei-jsonschema.json @@ -309,12 +309,12 @@ }, "class": { "enum": [ - "INET", - "CSNET", - "CHAOS", - "HESIOD", - "NONE", - "ANY" + "inet", + "csnet", + "chaos", + "hesiod", + "none", + "any" ], "type": "string", "title": "class of DNS request", @@ -662,6 +662,11 @@ "type": "boolean", "title": "preserve request history", "description": "Automatically assigns numbers to requests and preserves their history" + }, + "stop-at-first-match": { + "type": "boolean", + "title": "stop at first match", + "description": "Stop the execution after a match is found" } }, "additionalProperties": false, diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index 0d18cac571..f9d2e412f7 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -28,7 +28,7 @@ type Config struct { const nucleiConfigFilename = ".templates-config.json" // Version is the current version of nuclei -const Version = `2.4.4-dev` +const Version = `2.5.0` func getConfigDetails() (string, error) { homeDir, err := os.UserHomeDir() diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index 8d54f4af5d..354ea56150 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -107,18 +107,18 @@ func (store *Store) ValidateTemplates(templatesList, workflowsList []string) err } func areWorkflowsValid(store *Store, filteredWorkflowPaths map[string]struct{}) bool { - return areWorkflowOrTemplatesValid(store, filteredWorkflowPaths, func(templatePath string, tagFilter *filter.TagFilter) (bool, error) { + return areWorkflowOrTemplatesValid(store, filteredWorkflowPaths, true, func(templatePath string, tagFilter *filter.TagFilter) (bool, error) { return parsers.LoadWorkflow(templatePath) }) } func areTemplatesValid(store *Store, filteredTemplatePaths map[string]struct{}) bool { - return areWorkflowOrTemplatesValid(store, filteredTemplatePaths, func(templatePath string, tagFilter *filter.TagFilter) (bool, error) { + return areWorkflowOrTemplatesValid(store, filteredTemplatePaths, false, func(templatePath string, tagFilter *filter.TagFilter) (bool, error) { return parsers.LoadTemplate(templatePath, store.tagFilter, nil) }) } -func areWorkflowOrTemplatesValid(store *Store, filteredTemplatePaths map[string]struct{}, load func(templatePath string, tagFilter *filter.TagFilter) (bool, error)) bool { +func areWorkflowOrTemplatesValid(store *Store, filteredTemplatePaths map[string]struct{}, isWorkflow bool, load func(templatePath string, tagFilter *filter.TagFilter) (bool, error)) bool { areTemplatesValid := true for templatePath := range filteredTemplatePaths { if _, err := load(templatePath, store.tagFilter); err != nil { @@ -128,10 +128,15 @@ func areWorkflowOrTemplatesValid(store *Store, filteredTemplatePaths map[string] } } - if _, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions); err != nil { + template, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions) + if err != nil { if isParsingError("Error occurred parsing template %s: %s\n", templatePath, err) { areTemplatesValid = false } + } else { + if !isWorkflow && len(template.Workflows) > 0 { + return true + } } } return areTemplatesValid diff --git a/v2/pkg/model/model.go b/v2/pkg/model/model.go index 54a99f541c..46f426646b 100644 --- a/v2/pkg/model/model.go +++ b/v2/pkg/model/model.go @@ -22,6 +22,7 @@ type Info struct { // description: | // Author of the template. // + // Multiple values can also be specified separated by commas. // examples: // - value: "\"\"" Authors StringSlice `json:"author,omitempty" yaml:"author,omitempty" jsonschema:"title=author of the template,description=Author is the author of the template,example=username"` diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index f54b18758b..056f4ace7b 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -17,6 +17,7 @@ import ( "time" "github.com/Knetic/govaluate" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/deserialization" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/spaolacci/murmur3" @@ -241,6 +242,11 @@ var functions = map[string]govaluate.ExpressionFunction{ data := deserialization.GenerateJavaGadget(gadget, cmd, encoding) return data, nil }, + // for debug purposes + "print_debug": func(args ...interface{}) (interface{}, error) { + gologger.Info().Msgf("print_debug value: %s", fmt.Sprint(args)) + return true, nil + }, } // HelperFunctions returns the dsl helper functions diff --git a/v2/pkg/operators/extractors/extractors.go b/v2/pkg/operators/extractors/extractors.go index 383424146d..624c33006f 100644 --- a/v2/pkg/operators/extractors/extractors.go +++ b/v2/pkg/operators/extractors/extractors.go @@ -10,7 +10,7 @@ import ( type Extractor struct { // description: | // Name of the extractor. Name should be lowercase and must not contain - // spaces or dashes (-). + // spaces or underscores (_). // examples: // - value: "\"cookie-extractor\"" Name string `yaml:"name,omitempty" jsonschema:"title=name of the extractor,description=Name of the extractor"` @@ -48,18 +48,22 @@ type Extractor struct { regexCompiled []*regexp.Regexp // description: | - // kval contains the key-value pairs required in the response. + // kval contains the key-value pairs present in the HTTP response header. + // kval extractor can be used to extract HTTP response header and cookie key-value pairs. + // kval extractor inputs are case insensitive, and does not support dash (-) in input which can replaced with underscores (_) + // For example, Content-Type should be replaced with content_type // - // Each protocol exposes a lot of different data in response. The kval - // extractor can be used to extract those key-value pairs. A list of - // supported parts is available in docs for request types. + // A list of supported parts is available in docs for request types. // examples: // - name: Extract Server Header From HTTP Response // value: > - // []string{"Server"} + // []string{"server"} // - name: Extracting value of PHPSESSID Cookie // value: > - // []string{"PHPSESSID"} + // []string{"phpsessid"} + // - name: Extracting value of Content-Type Cookie + // value: > + // []string{"content_type"} KVal []string `yaml:"kval,omitempty" jsonschema:"title=kval pairs to extract from response,description=Kval pairs to extract from response"` // description: | @@ -77,8 +81,6 @@ type Extractor struct { // examples: // - value: > // []string{"/html/body/div/p[2]/a"} - // - value: > - // []string{".batters | .batter | .[] | .id"} XPath []string `yaml:"xpath,omitempty" jsonschema:"title=html xpath expressions to extract data,description=XPath allows using xpath expressions to extract items from html response"` // description: | // Attribute is an optional attribute to extract from response XPath. diff --git a/v2/pkg/operators/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go index 19cb3c8e26..9d9f64393b 100644 --- a/v2/pkg/operators/matchers/matchers.go +++ b/v2/pkg/operators/matchers/matchers.go @@ -43,7 +43,7 @@ type Matcher struct { // description: | // Name of the matcher. Name should be lowercase and must not contain - // spaces or dashes (-). + // spaces or underscores (_). // examples: // - value: "\"cookie-matcher\"" Name string `yaml:"name,omitempty" jsonschema:"title=name of the matcher,description=Name of the matcher"` diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index 90033fdbee..b65cb79e22 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -4,14 +4,17 @@ import ( "fmt" "io/ioutil" "os" + "regexp" "gopkg.in/yaml.v2" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/templates/cache" "github.com/projectdiscovery/nuclei/v2/pkg/utils" + "github.com/projectdiscovery/nuclei/v2/pkg/utils/stats" ) const mandatoryFieldMissingTemplate = "mandatory '%s' field is missing" @@ -83,10 +86,23 @@ func validateMandatoryInfoFields(info *model.Info) error { return nil } -var parsedTemplatesCache *cache.Templates +var ( + parsedTemplatesCache *cache.Templates + ShouldValidate bool + fieldErrorRegexp = regexp.MustCompile(`not found in`) +) + +const ( + SyntaxWarningStats = "syntax-warnings" + SyntaxErrorStats = "syntax-errors" +) func init() { + parsedTemplatesCache = cache.New() + + stats.NewEntry(SyntaxWarningStats, "Found %d templates with syntax warning (use -validate flag for further examination)") + stats.NewEntry(SyntaxErrorStats, "Found %d templates with syntax error (use -validate flag for further examination)") } // ParseTemplate parses a template and returns a *templates.Template structure @@ -107,8 +123,18 @@ func ParseTemplate(templatePath string) (*templates.Template, error) { } template := &templates.Template{} - if err := yaml.Unmarshal(data, template); err != nil { - return nil, err + if err := yaml.UnmarshalStrict(data, template); err != nil { + errString := err.Error() + if !fieldErrorRegexp.MatchString(errString) { + stats.Increment(SyntaxErrorStats) + return nil, err + } + stats.Increment(SyntaxWarningStats) + if ShouldValidate { + gologger.Error().Msgf("Syntax warnings for template %s: %s", templatePath, err) + } else { + gologger.Warning().Msgf("Syntax warnings for template %s: %s", templatePath, err) + } } parsedTemplatesCache.Store(templatePath, template, nil) return template, nil diff --git a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go index ae6f5bc2d8..14aefff8fa 100644 --- a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go +++ b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go @@ -16,7 +16,7 @@ import ( // It uses an LRU cache internally for skipping unresponsive hosts // that remain so for a duration. type Cache struct { - hostMaxErrors int + MaxHostError int verbose bool failedTargets gcache.Cache } @@ -24,11 +24,11 @@ type Cache struct { const DefaultMaxHostsCount = 10000 // New returns a new host max errors cache -func New(hostMaxErrors, maxHostsCount int) *Cache { +func New(MaxHostError, maxHostsCount int) *Cache { gc := gcache.New(maxHostsCount). ARC(). Build() - return &Cache{failedTargets: gc, hostMaxErrors: hostMaxErrors} + return &Cache{failedTargets: gc, MaxHostError: MaxHostError} } // SetVerbose sets the cache to log at verbose level @@ -88,7 +88,7 @@ func (c *Cache) Check(value string) bool { if numberOfErrors == -1 { return true } - if numberOfErrorsValue >= c.hostMaxErrors { + if numberOfErrorsValue >= c.MaxHostError { _ = c.failedTargets.Set(finalValue, -1) if c.verbose { gologger.Verbose().Msgf("Skipping %s as previously unresponsive %d times", finalValue, numberOfErrorsValue) diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 7812ead1ef..858f36bbd0 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -18,7 +18,7 @@ type Request struct { // Operators for the current request go here. operators.Operators `yaml:",inline"` - // ID is the ID of the request + // ID is the the optional id of the request ID string `yaml:"id,omitempty" jsonschema:"title=id of the dns request,description=ID is the optional ID of the DNS Request"` // description: | diff --git a/v2/pkg/protocols/file/file.go b/v2/pkg/protocols/file/file.go index 8072e1e00b..3f0b3b9497 100644 --- a/v2/pkg/protocols/file/file.go +++ b/v2/pkg/protocols/file/file.go @@ -26,7 +26,7 @@ type Request struct { // - value: '[]string{".avi", ".mov", ".mp3"}' ExtensionDenylist []string `yaml:"denylist,omitempty" jsonschema:"title=extensions to deny match,description=List of file extensions to deny during matching"` - // ID is the ID of the request + // ID is the the optional id of the request ID string `yaml:"id,omitempty" jsonschema:"title=id of the request,description=ID is the optional ID for the request"` // description: | diff --git a/v2/pkg/protocols/headless/headless.go b/v2/pkg/protocols/headless/headless.go index b95615abaf..b4bb984518 100644 --- a/v2/pkg/protocols/headless/headless.go +++ b/v2/pkg/protocols/headless/headless.go @@ -9,7 +9,7 @@ import ( // Request contains a Headless protocol request to be made from a template type Request struct { - // ID is the ID of the request + // ID is the the optional id of the request ID string `yaml:"id,omitempty" jsonschema:"title=id of the request,description=Optional ID of the headless request"` // description: | diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 09e2a4692a..d3fa89dab6 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -31,7 +31,7 @@ type Request struct { // value: | // []string{"GET /etc/passwd HTTP/1.1\nHost:\nContent-Length: 4", "POST /.%0d./.%0d./.%0d./.%0d./bin/sh HTTP/1.1\nHost: {{Hostname}}\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:71.0) Gecko/20100101 Firefox/71.0\nContent-Length: 1\nConnection: close\n\necho\necho\ncat /etc/passwd 2>&1"} Raw []string `yaml:"raw,omitempty" jsonschema:"http requests in raw format,description=HTTP Requests in Raw Format"` - // ID is the ID of the request + // ID is the the optional id of the request ID string `yaml:"id,omitempty" jsonschema:"title=id for the http request,description=ID for the HTTP Request"` // description: | // Name is the optional name of the request. @@ -162,7 +162,7 @@ type Request struct { // This allows matching on them later for multi-request conditions. ReqCondition bool `yaml:"req-condition,omitempty" jsonschema:"title=preserve request history,description=Automatically assigns numbers to requests and preserves their history"` // description: | - // StopAtFirstMatch stops the execution of the requests as soon as a match is found. + // StopAtFirstMatch stops the execution of the requests and template as soon as a match is found. StopAtFirstMatch bool `yaml:"stop-at-first-match,omitempty" jsonschema:"title=stop at first match,description=Stop the execution after a match is found"` } diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index e4ea521b80..1178d8f7b7 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -15,7 +15,7 @@ import ( // Request contains a Network protocol request to be made from a template type Request struct { - // ID is the ID of the request + // ID is the the optional id of the request ID string `yaml:"id,omitempty" jsonschema:"title=id of the request,description=ID of the network request"` // description: | diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index 02dcb13b5a..a3d50db555 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -7,7 +7,7 @@ import ( "strings" "github.com/andygrunwald/go-jira" - + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" "github.com/projectdiscovery/nuclei/v2/pkg/types" @@ -23,6 +23,8 @@ type Integration struct { type Options struct { // Cloud value is set to true when Jira cloud is used Cloud bool `yaml:"cloud"` + // UpdateExisting value if true, the existing opened issue is updated + UpdateExisting bool `yaml:"update-existing"` // URL is the URL of the jira server URL string `yaml:"url"` // AccountID is the accountID of the jira user. @@ -54,8 +56,8 @@ func New(options *Options) (*Integration, error) { return &Integration{jira: jiraClient, options: options}, nil } -// CreateIssue creates an issue in the tracker -func (i *Integration) CreateIssue(event *output.ResultEvent) error { +// CreateNewIssue creates a new issue in the tracker +func (i *Integration) CreateNewIssue(event *output.ResultEvent) error { summary := format.Summary(event) fields := &jira.IssueFields{ @@ -92,6 +94,52 @@ func (i *Integration) CreateIssue(event *output.ResultEvent) error { return nil } +// CreateIssue creates an issue in the tracker or updates the existing one +func (i *Integration) CreateIssue(event *output.ResultEvent) error { + if i.options.UpdateExisting { + issueID, err := i.FindExistingIssue(event) + if err != nil { + return err + } else if issueID != "" { + _, _, err = i.jira.Issue.AddComment(issueID, &jira.Comment{ + Body: jiraFormatDescription(event), + }) + return err + } + } + return i.CreateNewIssue(event) +} + +// FindExistingIssue checks if the issue already exists and returns its ID +func (i *Integration) FindExistingIssue(event *output.ResultEvent) (string, error) { + template := format.GetMatchedTemplate(event) + jql := fmt.Sprintf("summary ~ \"%s\" AND summary ~ \"%s\" AND status = \"Open\"", template, event.Host) + + searchOptions := &jira.SearchOptions{ + MaxResults: 1, // if any issue exists, then we won't create a new one + } + + chunk, resp, err := i.jira.Issue.Search(jql, searchOptions) + if err != nil { + var data string + if resp != nil && resp.Body != nil { + d, _ := ioutil.ReadAll(resp.Body) + data = string(d) + } + return "", fmt.Errorf("%s => %s", err, data) + } + + switch resp.Total { + case 0: + return "", nil + case 1: + return chunk[0].ID, nil + default: + gologger.Warning().Msgf("Discovered multiple opened issues %s for the host %s: The issue [%s] will be updated.", template, event.Host, chunk[0].ID) + return chunk[0].ID, nil + } +} + // jiraFormatDescription formats a short description of the generated // event by the nuclei scanner in Jira format. func jiraFormatDescription(event *output.ResultEvent) string { // TODO remove the code duplication: format.go <-> jira.go diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 1622b1f0d8..d7f545bc36 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -4,13 +4,11 @@ import ( "fmt" "io/ioutil" "os" - "regexp" "strings" "github.com/pkg/errors" "gopkg.in/yaml.v2" - "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/executer" @@ -21,7 +19,6 @@ import ( var ( ErrCreateTemplateExecutor = errors.New("cannot create template executer") - fieldErrorRegexp = regexp.MustCompile(`not found in`) ) var parsedTemplatesCache *cache.Templates @@ -56,11 +53,8 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute data = preprocessor.Process(data) } - if err := yaml.UnmarshalStrict(data, template); err != nil { - if !fieldErrorRegexp.MatchString(err.Error()) { - return nil, err - } - gologger.Warning().Msgf("Unrecognized fields in template %s: %s", filePath, err) + if err := yaml.Unmarshal(data, template); err != nil { + return nil, err } if utils.IsBlank(template.Info.Name) { diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index 88fa782202..5e65c59e54 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -108,7 +108,7 @@ func init() { MODELInfoDoc.Fields[1].Name = "author" MODELInfoDoc.Fields[1].Type = "StringSlice" MODELInfoDoc.Fields[1].Note = "" - MODELInfoDoc.Fields[1].Description = "Author of the template." + MODELInfoDoc.Fields[1].Description = "Author of the template.\n\nMultiple values can also be specified separated by commas." MODELInfoDoc.Fields[1].Comments[encoder.LineComment] = "Author of the template." MODELInfoDoc.Fields[1].AddExample("", "") @@ -188,7 +188,7 @@ func init() { FieldName: "requests", }, } - HTTPRequestDoc.Fields = make([]encoder.Doc, 24) + HTTPRequestDoc.Fields = make([]encoder.Doc, 25) HTTPRequestDoc.Fields[0].Name = "matchers" HTTPRequestDoc.Fields[0].Type = "[]matchers.Matcher" HTTPRequestDoc.Fields[0].Note = "" @@ -225,8 +225,8 @@ func init() { HTTPRequestDoc.Fields[5].Name = "id" HTTPRequestDoc.Fields[5].Type = "string" HTTPRequestDoc.Fields[5].Note = "" - HTTPRequestDoc.Fields[5].Description = "ID is the ID of the request" - HTTPRequestDoc.Fields[5].Comments[encoder.LineComment] = " ID is the ID of the request" + HTTPRequestDoc.Fields[5].Description = "ID is the the optional id of the request" + HTTPRequestDoc.Fields[5].Comments[encoder.LineComment] = " ID is the the optional id of the request" HTTPRequestDoc.Fields[6].Name = "name" HTTPRequestDoc.Fields[6].Type = "string" HTTPRequestDoc.Fields[6].Note = "" @@ -349,6 +349,11 @@ func init() { HTTPRequestDoc.Fields[23].Note = "" HTTPRequestDoc.Fields[23].Description = "ReqCondition automatically assigns numbers to requests and preserves their history.\n\nThis allows matching on them later for multi-request conditions." HTTPRequestDoc.Fields[23].Comments[encoder.LineComment] = "ReqCondition automatically assigns numbers to requests and preserves their history." + HTTPRequestDoc.Fields[24].Name = "stop-at-first-match" + HTTPRequestDoc.Fields[24].Type = "bool" + HTTPRequestDoc.Fields[24].Note = "" + HTTPRequestDoc.Fields[24].Description = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found." + HTTPRequestDoc.Fields[24].Comments[encoder.LineComment] = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found." MATCHERSMatcherDoc.Type = "matchers.Matcher" MATCHERSMatcherDoc.Comments[encoder.LineComment] = " Matcher is used to match a part in the output from a protocol." @@ -415,7 +420,7 @@ func init() { MATCHERSMatcherDoc.Fields[4].Name = "name" MATCHERSMatcherDoc.Fields[4].Type = "string" MATCHERSMatcherDoc.Fields[4].Note = "" - MATCHERSMatcherDoc.Fields[4].Description = "Name of the matcher. Name should be lowercase and must not contain\nspaces or dashes (-)." + MATCHERSMatcherDoc.Fields[4].Description = "Name of the matcher. Name should be lowercase and must not contain\nspaces or underscores (_)." MATCHERSMatcherDoc.Fields[4].Comments[encoder.LineComment] = "Name of the matcher. Name should be lowercase and must not contain" MATCHERSMatcherDoc.Fields[4].AddExample("", "cookie-matcher") @@ -507,7 +512,7 @@ func init() { EXTRACTORSExtractorDoc.Fields[0].Name = "name" EXTRACTORSExtractorDoc.Fields[0].Type = "string" EXTRACTORSExtractorDoc.Fields[0].Note = "" - EXTRACTORSExtractorDoc.Fields[0].Description = "Name of the extractor. Name should be lowercase and must not contain\nspaces or dashes (-)." + EXTRACTORSExtractorDoc.Fields[0].Description = "Name of the extractor. Name should be lowercase and must not contain\nspaces or underscores (_)." EXTRACTORSExtractorDoc.Fields[0].Comments[encoder.LineComment] = "Name of the extractor. Name should be lowercase and must not contain" EXTRACTORSExtractorDoc.Fields[0].AddExample("", "cookie-extractor") @@ -525,8 +530,8 @@ func init() { EXTRACTORSExtractorDoc.Fields[2].Name = "regex" EXTRACTORSExtractorDoc.Fields[2].Type = "[]string" EXTRACTORSExtractorDoc.Fields[2].Note = "" - EXTRACTORSExtractorDoc.Fields[2].Description = "Regex contains the regular expression patterns to exract from a part.\n\nGo regex engine does not supports lookaheads or lookbehinds, so as a result\nthey are also not supported in nuclei." - EXTRACTORSExtractorDoc.Fields[2].Comments[encoder.LineComment] = "Regex contains the regular expression patterns to exract from a part." + EXTRACTORSExtractorDoc.Fields[2].Description = "Regex contains the regular expression patterns to extract from a part.\n\nGo regex engine does not support lookaheads or lookbehinds, so as a result\nthey are also not supported in nuclei." + EXTRACTORSExtractorDoc.Fields[2].Comments[encoder.LineComment] = "Regex contains the regular expression patterns to extract from a part." EXTRACTORSExtractorDoc.Fields[2].AddExample("Braintree Access Token Regex", []string{"access_token\\$production\\$[0-9a-z]{16}\\$[0-9a-f]{32}"}) @@ -541,12 +546,8 @@ func init() { EXTRACTORSExtractorDoc.Fields[4].Name = "kval" EXTRACTORSExtractorDoc.Fields[4].Type = "[]string" EXTRACTORSExtractorDoc.Fields[4].Note = "" - EXTRACTORSExtractorDoc.Fields[4].Description = "kval contains the key-value pairs required in the response.\n\nEach protocol exposes a lot of different data in response. The kval\nextractor can be used to extract those key-value pairs. A list of\nsupported parts is available in docs for request types." - EXTRACTORSExtractorDoc.Fields[4].Comments[encoder.LineComment] = "kval contains the key-value pairs required in the response." - - EXTRACTORSExtractorDoc.Fields[4].AddExample("Extract Server Header From HTTP Response", []string{"Server"}) - - EXTRACTORSExtractorDoc.Fields[4].AddExample("Extracting value of PHPSESSID Cookie", []string{"PHPSESSID"}) + EXTRACTORSExtractorDoc.Fields[4].Description = "description: |\n kval contains the key-value pairs present in the HTTP response header.\n kval extractor can be used to extract HTTP response header and cookie key-value pairs.\n kval extractor inputs are case insensitive, and does not support dash (-) in input which can replaced with underscores (_)\n For example, Content-Type should be replaced with content_type\n\n A list of supported parts is available in docs for request types.\n examples:\n - name: Extract Server Header From HTTP Response\n value: >\n []string{\"server\"}\n - name: Extracting value of PHPSESSID Cookie\n value: >\n []string{\"phpsessid\"}\n - name: Extracting value of Content-Type Cookie\n value: >\n []string{\"content_type\"}" + EXTRACTORSExtractorDoc.Fields[4].Comments[encoder.LineComment] = " description: |" EXTRACTORSExtractorDoc.Fields[5].Name = "json" EXTRACTORSExtractorDoc.Fields[5].Type = "[]string" EXTRACTORSExtractorDoc.Fields[5].Note = "" @@ -563,8 +564,6 @@ func init() { EXTRACTORSExtractorDoc.Fields[6].Comments[encoder.LineComment] = "XPath allows using xpath expressions to extract items from html response" EXTRACTORSExtractorDoc.Fields[6].AddExample("", []string{"/html/body/div/p[2]/a"}) - - EXTRACTORSExtractorDoc.Fields[6].AddExample("", []string{".batters | .batter | .[] | .id"}) EXTRACTORSExtractorDoc.Fields[7].Name = "attribute" EXTRACTORSExtractorDoc.Fields[7].Type = "string" EXTRACTORSExtractorDoc.Fields[7].Note = "" @@ -621,8 +620,8 @@ func init() { DNSRequestDoc.Fields[3].Name = "id" DNSRequestDoc.Fields[3].Type = "string" DNSRequestDoc.Fields[3].Note = "" - DNSRequestDoc.Fields[3].Description = "ID is the ID of the request" - DNSRequestDoc.Fields[3].Comments[encoder.LineComment] = " ID is the ID of the request" + DNSRequestDoc.Fields[3].Description = "ID is the the optional id of the request" + DNSRequestDoc.Fields[3].Comments[encoder.LineComment] = " ID is the the optional id of the request" DNSRequestDoc.Fields[4].Name = "name" DNSRequestDoc.Fields[4].Type = "string" DNSRequestDoc.Fields[4].Note = "" @@ -652,12 +651,12 @@ func init() { DNSRequestDoc.Fields[6].Description = "Class is the class of the DNS request.\n\nUsually it's enough to just leave it as INET." DNSRequestDoc.Fields[6].Comments[encoder.LineComment] = "Class is the class of the DNS request." DNSRequestDoc.Fields[6].Values = []string{ - "INET", - "CSNET", - "CHAOS", - "HESIOD", - "NONE", - "ANY", + "inet", + "csnet", + "chaos", + "hesiod", + "none", + "any", } DNSRequestDoc.Fields[7].Name = "retries" DNSRequestDoc.Fields[7].Type = "int" @@ -720,8 +719,8 @@ func init() { FILERequestDoc.Fields[5].Name = "id" FILERequestDoc.Fields[5].Type = "string" FILERequestDoc.Fields[5].Note = "" - FILERequestDoc.Fields[5].Description = "ID is the ID of the request" - FILERequestDoc.Fields[5].Comments[encoder.LineComment] = " ID is the ID of the request" + FILERequestDoc.Fields[5].Description = "ID is the the optional id of the request" + FILERequestDoc.Fields[5].Comments[encoder.LineComment] = " ID is the the optional id of the request" FILERequestDoc.Fields[6].Name = "max-size" FILERequestDoc.Fields[6].Type = "int" FILERequestDoc.Fields[6].Note = "" @@ -750,8 +749,8 @@ func init() { NETWORKRequestDoc.Fields[0].Name = "id" NETWORKRequestDoc.Fields[0].Type = "string" NETWORKRequestDoc.Fields[0].Note = "" - NETWORKRequestDoc.Fields[0].Description = "ID is the ID of the request" - NETWORKRequestDoc.Fields[0].Comments[encoder.LineComment] = " ID is the ID of the request" + NETWORKRequestDoc.Fields[0].Description = "ID is the the optional id of the request" + NETWORKRequestDoc.Fields[0].Comments[encoder.LineComment] = " ID is the the optional id of the request" NETWORKRequestDoc.Fields[1].Name = "host" NETWORKRequestDoc.Fields[1].Type = "[]string" NETWORKRequestDoc.Fields[1].Note = "" @@ -862,8 +861,8 @@ func init() { HEADLESSRequestDoc.Fields[0].Name = "id" HEADLESSRequestDoc.Fields[0].Type = "string" HEADLESSRequestDoc.Fields[0].Note = "" - HEADLESSRequestDoc.Fields[0].Description = "ID is the ID of the request" - HEADLESSRequestDoc.Fields[0].Comments[encoder.LineComment] = " ID is the ID of the request" + HEADLESSRequestDoc.Fields[0].Description = "ID is the the optional id of the request" + HEADLESSRequestDoc.Fields[0].Comments[encoder.LineComment] = " ID is the the optional id of the request" HEADLESSRequestDoc.Fields[1].Name = "steps" HEADLESSRequestDoc.Fields[1].Type = "[]engine.Action" HEADLESSRequestDoc.Fields[1].Note = "" diff --git a/v2/pkg/templates/templates_doc_examples.go b/v2/pkg/templates/templates_doc_examples.go index 61d0a2a327..6f13cea3d4 100644 --- a/v2/pkg/templates/templates_doc_examples.go +++ b/v2/pkg/templates/templates_doc_examples.go @@ -16,10 +16,10 @@ import ( var ( exampleInfoStructure = model.Info{ Name: "Argument Injection in Ruby Dragonfly", - Authors: model.StringSlice{[]string{"0xspara"}}, + Authors: model.StringSlice{"0xspara"}, SeverityHolder: severity.SeverityHolder{severity.High}, Reference: model.StringSlice{"https://zxsecurity.co.nz/research/argunment-injection-ruby-dragonfly/"}, - Tags: model.StringSlice{[]string{"cve,cve2021,rce,ruby"}}, + Tags: model.StringSlice{"cve,cve2021,rce,ruby"}, } exampleNormalHTTPRequest = &http.Request{ Method: "GET", diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 45441bcbdf..d10199ff1a 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -65,8 +65,8 @@ type Options struct { StatsInterval int // MetricsPort is the port to show metrics on MetricsPort int - // HostMaxErrors is the maximum number of errors allowed for a host - HostMaxErrors int + // MaxHostError is the maximum number of errors allowed for a host + MaxHostError int // BulkSize is the of targets analyzed in parallel for each template BulkSize int // TemplateThreads is the number of templates executed in parallel diff --git a/v2/pkg/utils/stats/doc.go b/v2/pkg/utils/stats/doc.go new file mode 100644 index 0000000000..5105b1708e --- /dev/null +++ b/v2/pkg/utils/stats/doc.go @@ -0,0 +1,3 @@ +// Package stats provides a storage mechanism for storing +// and display vital statistics of the engine at various durations. +package stats diff --git a/v2/pkg/utils/stats/stats.go b/v2/pkg/utils/stats/stats.go new file mode 100644 index 0000000000..d25d143344 --- /dev/null +++ b/v2/pkg/utils/stats/stats.go @@ -0,0 +1,99 @@ +package stats + +import ( + "sync" + "sync/atomic" + + "github.com/projectdiscovery/gologger" +) + +// Storage is a storage for storing statistics information +// about the nuclei engine displaying it at user-defined intervals. +type Storage struct { + data map[string]*storageDataItem + mutex *sync.RWMutex +} + +type storageDataItem struct { + description string + value int64 +} + +var Default *Storage + +func init() { + Default = New() +} + +// NewEntry creates a new entry in the storage object +func NewEntry(name, description string) { + Default.NewEntry(name, description) +} + +// Increment incrmements the value for a name string +func Increment(name string) { + Default.Increment(name) +} + +// Display displays the stats for a name +func Display(name string) { + Default.Display(name) +} + +// GetValue returns the value for a set variable +func GetValue(name string) int64 { + return Default.GetValue(name) +} + +// New creates a new storage object +func New() *Storage { + return &Storage{data: make(map[string]*storageDataItem), mutex: &sync.RWMutex{}} +} + +// NewEntry creates a new entry in the storage object +func (s *Storage) NewEntry(name, description string) { + s.mutex.Lock() + s.data[name] = &storageDataItem{description: description, value: 0} + s.mutex.Unlock() +} + +// Increment incrmements the value for a name string +func (s *Storage) Increment(name string) { + s.mutex.RLock() + data, ok := s.data[name] + s.mutex.RUnlock() + if !ok { + return + } + + atomic.AddInt64(&data.value, 1) +} + +// Display displays the stats for a name +func (s *Storage) Display(name string) { + s.mutex.RLock() + data, ok := s.data[name] + s.mutex.RUnlock() + if !ok { + return + } + + dataValue := atomic.LoadInt64(&data.value) + if dataValue == 0 { + return // don't show for nil stats + } + gologger.Error().Label("WRN").Msgf(data.description, dataValue) +} + +// GetValue returns the value for a set variable +func (s *Storage) GetValue(name string) int64 { + s.mutex.RLock() + data, ok := s.data[name] + s.mutex.RUnlock() + if !ok { + return 0 + } + + dataValue := atomic.LoadInt64(&data.value) + return dataValue +} diff --git a/v2/syntax-reference.md b/v2/syntax-reference.md index bbe6ebe6bc..006885fb99 100755 --- a/v2/syntax-reference.md +++ b/v2/syntax-reference.md @@ -60,10 +60,8 @@ Examples: ```yaml info: name: Argument Injection in Ruby Dragonfly - author: - - 0xspara - tags: - - cve,cve2021,rce,ruby + author: 0xspara + tags: cve,cve2021,rce,ruby reference: https://zxsecurity.co.nz/research/argunment-injection-ruby-dragonfly/ severity: high ``` @@ -248,10 +246,8 @@ Appears in: ```yaml name: Argument Injection in Ruby Dragonfly -author: - - 0xspara -tags: - - cve,cve2021,rce,ruby +author: 0xspara +tags: cve,cve2021,rce,ruby reference: https://zxsecurity.co.nz/research/argunment-injection-ruby-dragonfly/ severity: high ``` @@ -294,6 +290,8 @@ name: Nagios Default Credentials Check Author of the template. +Multiple values can also be specified separated by commas. + Examples: @@ -620,7 +618,7 @@ raw:
-ID is the ID of the request +ID is the the optional id of the request
@@ -1005,6 +1003,19 @@ This allows matching on them later for multi-request conditions.
+
+ +stop-at-first-match bool + +
+
+ +StopAtFirstMatch stops the execution of the requests and template as soon as a match is found. + +
+ +
+ @@ -1130,7 +1141,7 @@ It will only match if the condition is not true.
Name of the matcher. Name should be lowercase and must not contain -spaces or dashes (-). +spaces or underscores (_). @@ -1373,7 +1384,7 @@ Appears in:
Name of the extractor. Name should be lowercase and must not contain -spaces or dashes (-). +spaces or underscores (_). @@ -1420,9 +1431,9 @@ Valid values:
-Regex contains the regular expression patterns to exract from a part. +Regex contains the regular expression patterns to extract from a part. -Go regex engine does not supports lookaheads or lookbehinds, so as a result +Go regex engine does not support lookaheads or lookbehinds, so as a result they are also not supported in nuclei. @@ -1478,29 +1489,23 @@ group: 1
-kval contains the key-value pairs required in the response. - -Each protocol exposes a lot of different data in response. The kval -extractor can be used to extract those key-value pairs. A list of -supported parts is available in docs for request types. - - - -Examples: - - -```yaml -# Extract Server Header From HTTP Response -kval: - - Server -``` - -```yaml -# Extracting value of PHPSESSID Cookie -kval: - - PHPSESSID -``` +description: | + kval contains the key-value pairs present in the HTTP response header. + kval extractor can be used to extract HTTP response header and cookie key-value pairs. + kval extractor inputs are case insensitive, and does not support dash (-) in input which can replaced with underscores (_) + For example, Content-Type should be replaced with content_type + A list of supported parts is available in docs for request types. + examples: + - name: Extract Server Header From HTTP Response + value: > + []string{"server"} + - name: Extracting value of PHPSESSID Cookie + value: > + []string{"phpsessid"} + - name: Extracting value of Content-Type Cookie + value: > + []string{"content_type"}
@@ -1554,11 +1559,6 @@ xpath: - /html/body/div/p[2]/a ``` -```yaml -xpath: - - .batters | .batter | .[] | .id -``` -
@@ -1718,7 +1718,7 @@ Valid values:
-ID is the ID of the request +ID is the the optional id of the request
@@ -1798,17 +1798,17 @@ Usually it's enough to just leave it as INET. Valid values: - - INET + - inet - - CSNET + - csnet - - CHAOS + - chaos - - HESIOD + - hesiod - - NONE + - none - - ANY + - any
@@ -1988,7 +1988,7 @@ denylist:
-ID is the ID of the request +ID is the the optional id of the request
@@ -2068,7 +2068,7 @@ matchers:
-ID is the ID of the request +ID is the the optional id of the request
@@ -2374,7 +2374,7 @@ Appears in:
-ID is the ID of the request +ID is the the optional id of the request