From 02fadbcf775a0a2c8b0ee6eb3f4796c32e2c64ea Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Mon, 19 Aug 2024 05:58:30 +0700 Subject: [PATCH 01/38] chore: migrate issue template to issue form (#5538) Signed-off-by: Dwi Siswanto --- .github/ISSUE_TEMPLATE/bug-report.yml | 72 ++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature-request.yml | 37 +++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 21 ------- .github/ISSUE_TEMPLATE/issue-report.md | 41 ------------ 4 files changed, 109 insertions(+), 62 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug-report.yml create mode 100644 .github/ISSUE_TEMPLATE/feature-request.yml delete mode 100644 .github/ISSUE_TEMPLATE/feature_request.md delete mode 100644 .github/ISSUE_TEMPLATE/issue-report.md diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 0000000000..d2053744c7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,72 @@ +name: Bug Report +description: Create a report to help us to improve the Nuclei. +title: "[BUG] ..." +labels: ["Type: Bug"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report! + + For support requests, FAQs or "How to" questions, please use the [GitHub Discussions](https://github.com/projectdiscovery/nuclei/discussions) section instead or join our [Discord server](https://discord.gg/projectdiscovery) to discuss the idea on the **#nuclei** channel. + + :warning: **Issues missing important information may be closed without further investigation.** + - type: checkboxes + attributes: + label: Is there an existing issue for this? + description: Please search to see if an issue already exists for the bug you encountered. + options: + - label: I have searched the existing issues. + required: true + - type: textarea + attributes: + label: Current Behavior + description: A concise description of what you're experiencing. + validations: + required: true + - type: textarea + attributes: + label: Expected Behavior + description: A concise description of what you expected to happen. + validations: + required: true + - type: textarea + attributes: + label: Steps To Reproduce + description: | + Steps to reproduce the behavior, for example, commands to run Nuclei. + + :warning: **Please redact any literal target hosts/URLs or other sensitive information.** + placeholder: | + 1. Run `nuclei -t ...` + validations: + required: true + - type: textarea + attributes: + label: Relevant log output + description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. + render: shell + - type: textarea + attributes: + label: Environment + description: | + Examples: + - **OS**: Ubuntu 20.04 + - **Nuclei** (`nuclei -version`): v3.3.1 + - **Go** (`go version`): go1.22.0 _(only if you've installed it via the `go install` command)_ + value: | + - OS: + - Nuclei: + - Go: + render: markdown + validations: + required: true + - type: textarea + attributes: + label: Anything else? + description: | + Links? References? Templates? Anything that will give us more context about the issue you are encountering! + + Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml new file mode 100644 index 0000000000..ccc1b86298 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -0,0 +1,37 @@ +name: Feature Request +description: Request feature to implement in the Nuclei. +title: "[FEATURE] ..." +labels: ["Type: Enhancement"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this feature request! + + Please make sure to provide a detailed description with all the relevant information that might be required to start working on this feature. In case you are not sure about your request or whether the particular feature is already supported or not, please [start a discussion](https://github.com/projectdiscovery/nuclei/discussions/categories/ideas) instead. + + Join our [Discord server](https://discord.gg/projectdiscovery) to discuss the idea on the **#nuclei** channel. + - type: textarea + attributes: + label: Describe your feature request + description: A clear and concise description of feature to implement. + validations: + required: true + - type: textarea + attributes: + label: Describe the use case of the feature + description: A clear and concise description of the feature request's motivation and the use-cases in which it could be useful. + validations: + required: true + - type: textarea + attributes: + label: Describe alternatives you've considered + description: A clear and concise description of any alternative solutions or features you've considered. + validations: + required: false + - type: textarea + attributes: + label: Additional context + description: Add any other context about the feature request here. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index 8b41d50c2a..0000000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -name: Feature request -about: Request feature to implement in this project -title: "" -labels: 'Type: Enhancement' -assignees: '' - ---- - - - -### Please describe your feature request: - - -### Describe the use case of this feature: - diff --git a/.github/ISSUE_TEMPLATE/issue-report.md b/.github/ISSUE_TEMPLATE/issue-report.md deleted file mode 100644 index 3aa563b195..0000000000 --- a/.github/ISSUE_TEMPLATE/issue-report.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -name: Issue report -about: Create a report to help us to improve the project -labels: 'Type: Bug' - ---- - - - - - -### Nuclei version: - - - - -## Nuclei command: - - - - -### Current Behavior: - - -### Expected Behavior: - - -### Steps To Reproduce: - - - -### Anything else: - From 80b3cc0627482adc17ce2822ae9368dd9d22267d Mon Sep 17 00:00:00 2001 From: Ice3man Date: Mon, 19 Aug 2024 06:20:12 +0530 Subject: [PATCH 02/38] feat: jira accept issue-type and project ids (#5537) * feat: jira accept issue-type and project ids * remove validation for project name --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- pkg/reporting/trackers/jira/jira.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pkg/reporting/trackers/jira/jira.go b/pkg/reporting/trackers/jira/jira.go index 5499445c87..6cb976c44e 100644 --- a/pkg/reporting/trackers/jira/jira.go +++ b/pkg/reporting/trackers/jira/jira.go @@ -70,9 +70,13 @@ type Options struct { // Token is the token for jira instance. Token string `yaml:"token" json:"token" validate:"required"` // ProjectName is the name of the project. - ProjectName string `yaml:"project-name" json:"project_name" validate:"required"` + ProjectName string `yaml:"project-name" json:"project_name"` + // ProjectID is the ID of the project (optional) + ProjectID string `yaml:"project-id" json:"project_id"` // IssueType (optional) is the name of the created issue type IssueType string `yaml:"issue-type" json:"issue_type"` + // IssueTypeID (optional) is the ID of the created issue type + IssueTypeID string `yaml:"issue-type-id" json:"issue_type_id"` // SeverityAsLabel (optional) sends the severity as the label of the created // issue. SeverityAsLabel bool `yaml:"severity-as-label" json:"severity_as_label"` @@ -183,6 +187,7 @@ func (i *Integration) CreateNewIssue(event *output.ResultEvent) (*filters.Create Project: jira.Project{Key: i.options.ProjectName}, Summary: summary, } + // On-prem version of Jira server does not use AccountID if !i.options.Cloud { fields = &jira.IssueFields{ @@ -195,6 +200,12 @@ func (i *Integration) CreateNewIssue(event *output.ResultEvent) (*filters.Create Unknowns: customFields, } } + if i.options.IssueTypeID != "" { + fields.Type = jira.IssueType{ID: i.options.IssueTypeID} + } + if i.options.ProjectID != "" { + fields.Project = jira.Project{ID: i.options.ProjectID} + } issueData := &jira.Issue{ Fields: fields, From c6bafd88eb763b0e0787e3ff692e34f40df5fb36 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 05:12:33 +0000 Subject: [PATCH 03/38] chore(deps): bump github.com/projectdiscovery/clistats Bumps [github.com/projectdiscovery/clistats](https://github.com/projectdiscovery/clistats) from 0.0.20 to 0.1.0. - [Release notes](https://github.com/projectdiscovery/clistats/releases) - [Commits](https://github.com/projectdiscovery/clistats/compare/v0.0.20...v0.1.0) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/clistats dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f2053614c1..c52eb896a3 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/miekg/dns v1.1.59 github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/clistats v0.0.20 + github.com/projectdiscovery/clistats v0.1.0 github.com/projectdiscovery/fastdialer v0.2.3 github.com/projectdiscovery/hmap v0.0.54 github.com/projectdiscovery/interactsh v1.2.0 diff --git a/go.sum b/go.sum index dadb721a3f..c18b6f1f31 100644 --- a/go.sum +++ b/go.sum @@ -830,8 +830,8 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/cdncheck v1.1.0 h1:qDITidmJsejzpk3rMkauCh6sjI2GH9hW/snk0cQ3kXE= github.com/projectdiscovery/cdncheck v1.1.0/go.mod h1:sZ8U4MjHSsyaTVjBbYWHT1cwUVvUYwDX1W+WvWRicIc= -github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPooH+DGMgoWq4= -github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= +github.com/projectdiscovery/clistats v0.1.0 h1:b+LF1w0xhNd7cneKWMXb+/yUmF1n5szawxP4XGpmbxs= +github.com/projectdiscovery/clistats v0.1.0/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= github.com/projectdiscovery/dsl v0.1.8 h1:ulz+o097XsYgqP4QXaAQhVSkoeD2ZPWd29wX3CTodUA= github.com/projectdiscovery/dsl v0.1.8/go.mod h1:AYJS2WQ/q0smr2v4pEJTg4DPe6k56KFKR7UFXvzNz/4= github.com/projectdiscovery/fastdialer v0.2.3 h1:K03x5XEXGyVWxS2rtSR104E9kHF0aphN7kOCzbh8zv0= From e16350cde610e11509ebed808f1277ac83e75492 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 05:39:12 +0000 Subject: [PATCH 04/38] chore(deps): bump github.com/projectdiscovery/hmap from 0.0.54 to 0.0.55 Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.54 to 0.0.55. - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.54...v0.0.55) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/hmap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index c52eb896a3..c7628b7b47 100644 --- a/go.mod +++ b/go.mod @@ -21,11 +21,11 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.1.0 github.com/projectdiscovery/fastdialer v0.2.3 - github.com/projectdiscovery/hmap v0.0.54 + github.com/projectdiscovery/hmap v0.0.55 github.com/projectdiscovery/interactsh v1.2.0 github.com/projectdiscovery/rawhttp v0.1.60 - github.com/projectdiscovery/retryabledns v1.0.69 - github.com/projectdiscovery/retryablehttp-go v1.0.71 + github.com/projectdiscovery/retryabledns v1.0.70 + github.com/projectdiscovery/retryablehttp-go v1.0.72 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 @@ -96,7 +96,7 @@ require ( github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/uncover v1.0.9 github.com/projectdiscovery/useragent v0.0.65 - github.com/projectdiscovery/utils v0.2.4 + github.com/projectdiscovery/utils v0.2.5 github.com/projectdiscovery/wappalyzergo v0.1.14 github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 diff --git a/go.sum b/go.sum index c18b6f1f31..ecf4846ac7 100644 --- a/go.sum +++ b/go.sum @@ -850,8 +850,8 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E= github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8= -github.com/projectdiscovery/hmap v0.0.54 h1:b3pdQZwCw4is3xiL2jBx7SJZcYaf/7vtozY7bjUzO/s= -github.com/projectdiscovery/hmap v0.0.54/go.mod h1:j0oakxYOWEfk29wRq5gQgrCv1JnfAfzGaMsRWwEas80= +github.com/projectdiscovery/hmap v0.0.55 h1:By0qSeQRE/ybeITUEL6oURN+j7qX7FVjCq92duwsqcA= +github.com/projectdiscovery/hmap v0.0.55/go.mod h1:uZOF/W8yOGDb/t9omqATirhVwiboa5F/c00OVhDeE0E= github.com/projectdiscovery/httpx v1.6.7 h1:luJAtQ+iKpiyKKDpcWw1/fF1MaRX6JZ4R3P+ol5Uikk= github.com/projectdiscovery/httpx v1.6.7/go.mod h1:rqcuexBrb4v36ZyX0BHcHaovTurOJn2P65Tdt9cYdfE= github.com/projectdiscovery/interactsh v1.2.0 h1:Al6jHiR+Usl9egYJDLJaWNHOcH8Rugk8gWMasc8Cmw8= @@ -872,10 +872,10 @@ github.com/projectdiscovery/rawhttp v0.1.60 h1:L57kAATGRuCfOp8Fs0ly3j2wlip7R5pco github.com/projectdiscovery/rawhttp v0.1.60/go.mod h1:M97w6lsphTHKeqLSio4Lw9K4KLO2riJMYAOqVMIZ3ck= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= -github.com/projectdiscovery/retryabledns v1.0.69 h1:6vhhlfxPQoJI0j1enYSzyagyAZJNfGSg9TNloVsxEBQ= -github.com/projectdiscovery/retryabledns v1.0.69/go.mod h1:5SDuGtXgL4W1z/9V+xnmHFYg03Fi1Ud/F2kqlPlAgfE= -github.com/projectdiscovery/retryablehttp-go v1.0.71 h1:yXPNShCOwoTz7dBSJsBhBh4g4ujX62XS/BrH/fL1VyE= -github.com/projectdiscovery/retryablehttp-go v1.0.71/go.mod h1:wY3T89EwcCKAw6iyMDvwzGPyL3d8TaBU80hnDErEKgM= +github.com/projectdiscovery/retryabledns v1.0.70 h1:2yFMqQ4v3tgI9ORjlPH60h5QIs2EXxHVGCaxrOA1ZlI= +github.com/projectdiscovery/retryabledns v1.0.70/go.mod h1:Ld/RLVsG7d+wlNcye9xcuPTjGHLF9XO8w34GLRKsNis= +github.com/projectdiscovery/retryablehttp-go v1.0.72 h1:3m+9aRwC4KOMoQiOF3lVsDnPm/1+OR5r0UCHt5Edz5k= +github.com/projectdiscovery/retryablehttp-go v1.0.72/go.mod h1:tDPEpm0PlDOMB4yqHFPoLJZaK3uo+Auj/QIiUlzS+6Y= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= @@ -886,8 +886,8 @@ github.com/projectdiscovery/uncover v1.0.9 h1:s5RbkD/V4r8QcPkys4gTTqMuRSgXq0Jpre github.com/projectdiscovery/uncover v1.0.9/go.mod h1:2PUF3SpB5QNIJ8epaB2xbRzkPaxEAWRDm3Ir2ijt81U= github.com/projectdiscovery/useragent v0.0.65 h1:x78ZwWdqpzokOHxLITUXvq+ljkTKc19z3ILGtoV1N70= github.com/projectdiscovery/useragent v0.0.65/go.mod h1:deOP8YLJU6SCzM8k+K8PjkcOF4Ux0spqyO4ODZGIT4A= -github.com/projectdiscovery/utils v0.2.4 h1:CHnlt2la4jr8TeL7ZK7UhQItHY7DDXqIuLnnxyAJLDY= -github.com/projectdiscovery/utils v0.2.4/go.mod h1:2Vx7geSrBfCPqknZywqbChQm8SE30mcyrlB5YsxEnhA= +github.com/projectdiscovery/utils v0.2.5 h1:puiw3/gK1WTEj9ZKnW00tAS0ex+ETqbLJ8juRuejA98= +github.com/projectdiscovery/utils v0.2.5/go.mod h1:fv3WC6m/+bVT+wH+fMB+JsKviEC/UAuDSeKxWPlK3kk= github.com/projectdiscovery/wappalyzergo v0.1.14 h1:nt1IM4RUmqeymsXk4h6BsZbKDoS2hjFvPkT2GaI1rz4= github.com/projectdiscovery/wappalyzergo v0.1.14/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= From c94d226a496a9b08026c5f39540356d772d48aba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 06:07:14 +0000 Subject: [PATCH 05/38] chore(deps): bump github.com/projectdiscovery/fastdialer Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.2.3 to 0.2.5. - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.2.3...v0.2.5) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fastdialer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index c7628b7b47..171d1bc523 100644 --- a/go.mod +++ b/go.mod @@ -20,11 +20,11 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.1.0 - github.com/projectdiscovery/fastdialer v0.2.3 + github.com/projectdiscovery/fastdialer v0.2.5 github.com/projectdiscovery/hmap v0.0.55 github.com/projectdiscovery/interactsh v1.2.0 github.com/projectdiscovery/rawhttp v0.1.60 - github.com/projectdiscovery/retryabledns v1.0.70 + github.com/projectdiscovery/retryabledns v1.0.71 github.com/projectdiscovery/retryablehttp-go v1.0.72 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 diff --git a/go.sum b/go.sum index ecf4846ac7..9bddf93345 100644 --- a/go.sum +++ b/go.sum @@ -834,8 +834,8 @@ github.com/projectdiscovery/clistats v0.1.0 h1:b+LF1w0xhNd7cneKWMXb+/yUmF1n5szaw github.com/projectdiscovery/clistats v0.1.0/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= github.com/projectdiscovery/dsl v0.1.8 h1:ulz+o097XsYgqP4QXaAQhVSkoeD2ZPWd29wX3CTodUA= github.com/projectdiscovery/dsl v0.1.8/go.mod h1:AYJS2WQ/q0smr2v4pEJTg4DPe6k56KFKR7UFXvzNz/4= -github.com/projectdiscovery/fastdialer v0.2.3 h1:K03x5XEXGyVWxS2rtSR104E9kHF0aphN7kOCzbh8zv0= -github.com/projectdiscovery/fastdialer v0.2.3/go.mod h1:a0BKvETrO1EAohUYp9gwtbbce0hKD1qGrTegyAUqyRo= +github.com/projectdiscovery/fastdialer v0.2.5 h1:OZg31LFIBBHbXVnOEcFNtD0gBM80zpY68QvYI0xbw80= +github.com/projectdiscovery/fastdialer v0.2.5/go.mod h1:E1vmos7AeD/KZ7rsDeImjLp5H3jjas+2utVdUxGulvY= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= @@ -872,8 +872,8 @@ github.com/projectdiscovery/rawhttp v0.1.60 h1:L57kAATGRuCfOp8Fs0ly3j2wlip7R5pco github.com/projectdiscovery/rawhttp v0.1.60/go.mod h1:M97w6lsphTHKeqLSio4Lw9K4KLO2riJMYAOqVMIZ3ck= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= -github.com/projectdiscovery/retryabledns v1.0.70 h1:2yFMqQ4v3tgI9ORjlPH60h5QIs2EXxHVGCaxrOA1ZlI= -github.com/projectdiscovery/retryabledns v1.0.70/go.mod h1:Ld/RLVsG7d+wlNcye9xcuPTjGHLF9XO8w34GLRKsNis= +github.com/projectdiscovery/retryabledns v1.0.71 h1:hZnTR7C4G7XDLxQEvdZVREHEJToIfeObNwR+zpmZ1vs= +github.com/projectdiscovery/retryabledns v1.0.71/go.mod h1:P3GAytlIf2c/H+PbHGwLoPrc8pSsCK+i8gt3sO/VynU= github.com/projectdiscovery/retryablehttp-go v1.0.72 h1:3m+9aRwC4KOMoQiOF3lVsDnPm/1+OR5r0UCHt5Edz5k= github.com/projectdiscovery/retryablehttp-go v1.0.72/go.mod h1:tDPEpm0PlDOMB4yqHFPoLJZaK3uo+Auj/QIiUlzS+6Y= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= From 091ca32b1cd9f1190847e5506b086d3c35b1e0e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 06:33:12 +0000 Subject: [PATCH 06/38] chore(deps): bump github.com/projectdiscovery/retryablehttp-go Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.71 to 1.0.74. - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.71...v1.0.74) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 171d1bc523..c178fb5456 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/projectdiscovery/interactsh v1.2.0 github.com/projectdiscovery/rawhttp v0.1.60 github.com/projectdiscovery/retryabledns v1.0.71 - github.com/projectdiscovery/retryablehttp-go v1.0.72 + github.com/projectdiscovery/retryablehttp-go v1.0.74 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 diff --git a/go.sum b/go.sum index 9bddf93345..4e74f6e6aa 100644 --- a/go.sum +++ b/go.sum @@ -874,8 +874,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= github.com/projectdiscovery/retryabledns v1.0.71 h1:hZnTR7C4G7XDLxQEvdZVREHEJToIfeObNwR+zpmZ1vs= github.com/projectdiscovery/retryabledns v1.0.71/go.mod h1:P3GAytlIf2c/H+PbHGwLoPrc8pSsCK+i8gt3sO/VynU= -github.com/projectdiscovery/retryablehttp-go v1.0.72 h1:3m+9aRwC4KOMoQiOF3lVsDnPm/1+OR5r0UCHt5Edz5k= -github.com/projectdiscovery/retryablehttp-go v1.0.72/go.mod h1:tDPEpm0PlDOMB4yqHFPoLJZaK3uo+Auj/QIiUlzS+6Y= +github.com/projectdiscovery/retryablehttp-go v1.0.74 h1:jXZsg3nNDuDjGNeL0+hiEx/dqUqPyZW1zsotWW1y70c= +github.com/projectdiscovery/retryablehttp-go v1.0.74/go.mod h1:fVRxa2fVMbomaPiOFA05kM2WrjV8wPFHW7J84eenVmQ= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= From 8a3818230dcc074eba34dfab1234b5a117b4f31c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 06:59:19 +0000 Subject: [PATCH 07/38] chore(deps): bump github.com/projectdiscovery/retryabledns Bumps [github.com/projectdiscovery/retryabledns](https://github.com/projectdiscovery/retryabledns) from 1.0.69 to 1.0.72. - [Release notes](https://github.com/projectdiscovery/retryabledns/releases) - [Commits](https://github.com/projectdiscovery/retryabledns/compare/v1.0.69...v1.0.72) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryabledns dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c178fb5456..855b9f3c77 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/projectdiscovery/hmap v0.0.55 github.com/projectdiscovery/interactsh v1.2.0 github.com/projectdiscovery/rawhttp v0.1.60 - github.com/projectdiscovery/retryabledns v1.0.71 + github.com/projectdiscovery/retryabledns v1.0.72 github.com/projectdiscovery/retryablehttp-go v1.0.74 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 diff --git a/go.sum b/go.sum index 4e74f6e6aa..43326744d8 100644 --- a/go.sum +++ b/go.sum @@ -872,8 +872,8 @@ github.com/projectdiscovery/rawhttp v0.1.60 h1:L57kAATGRuCfOp8Fs0ly3j2wlip7R5pco github.com/projectdiscovery/rawhttp v0.1.60/go.mod h1:M97w6lsphTHKeqLSio4Lw9K4KLO2riJMYAOqVMIZ3ck= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= -github.com/projectdiscovery/retryabledns v1.0.71 h1:hZnTR7C4G7XDLxQEvdZVREHEJToIfeObNwR+zpmZ1vs= -github.com/projectdiscovery/retryabledns v1.0.71/go.mod h1:P3GAytlIf2c/H+PbHGwLoPrc8pSsCK+i8gt3sO/VynU= +github.com/projectdiscovery/retryabledns v1.0.72 h1:+k9JRYBMpZIyEQZJWVHoCHx1TEdpiuAS7iEru2bhs3A= +github.com/projectdiscovery/retryabledns v1.0.72/go.mod h1:BDhYnqSH0Os2G37eb4L4vnKrUcik6Q6pR02Cq5PbT+0= github.com/projectdiscovery/retryablehttp-go v1.0.74 h1:jXZsg3nNDuDjGNeL0+hiEx/dqUqPyZW1zsotWW1y70c= github.com/projectdiscovery/retryablehttp-go v1.0.74/go.mod h1:fVRxa2fVMbomaPiOFA05kM2WrjV8wPFHW7J84eenVmQ= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= From 0da993afe6d41b4b1b814e8fad23a2acba13c60a Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 19 Aug 2024 14:32:54 +0200 Subject: [PATCH 08/38] Merge commit from fork MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix template signature verification * fix signature pattern check * add tests * remove signature count constraint, check for lines len * Add more tests * Centralize signature extraction logic in signer package * Move signature handling in Sign function to beginning * Remove comment * Revert `NewTemplateSigVerifier` * update tests * use ExtractSignatureAndContent func * Allow signing code templates * Remove unused const --------- Co-authored-by: Doğan Can Bakır Co-authored-by: Guy Goldenberg --- pkg/templates/signer/tmpl_signer.go | 57 +++++----- pkg/templates/signer/tmpl_signer_test.go | 126 +++++++++++++++++++++++ pkg/templates/template_sign.go | 3 +- 3 files changed, 161 insertions(+), 25 deletions(-) create mode 100644 pkg/templates/signer/tmpl_signer_test.go diff --git a/pkg/templates/signer/tmpl_signer.go b/pkg/templates/signer/tmpl_signer.go index c4378fd406..a530da1fe3 100644 --- a/pkg/templates/signer/tmpl_signer.go +++ b/pkg/templates/signer/tmpl_signer.go @@ -11,7 +11,6 @@ import ( "errors" "fmt" "os" - "regexp" "strings" "sync" @@ -21,18 +20,21 @@ import ( ) var ( - ReDigest = regexp.MustCompile(`(?m)^#\sdigest:\s.+$`) ErrUnknownAlgorithm = errors.New("unknown algorithm") SignaturePattern = "# digest: " SignatureFmt = SignaturePattern + "%x" + ":%v" // `#digest: :` ) -func RemoveSignatureFromData(data []byte) []byte { - return bytes.Trim(ReDigest.ReplaceAll(data, []byte("")), "\n") -} - -func GetSignatureFromData(data []byte) []byte { - return ReDigest.Find(data) +// ExtractSignatureAndContent extracts the signature (if present) and returns the content without the signature +func ExtractSignatureAndContent(data []byte) (signature, content []byte) { + dataStr := string(data) + if idx := strings.LastIndex(dataStr, SignaturePattern); idx != -1 { + signature = []byte(strings.TrimSpace(dataStr[idx:])) + content = []byte(strings.TrimSpace(dataStr[:idx])) + } else { + content = data + } + return } // SignableTemplate is a template that can be signed @@ -69,26 +71,29 @@ func (t *TemplateSigner) GetUserFragment() string { // Sign signs the given template with the template signer and returns the signature func (t *TemplateSigner) Sign(data []byte, tmpl SignableTemplate) (string, error) { + existingSignature, content := ExtractSignatureAndContent(data) + // while re-signing template check if it has a code protocol // if it does then verify that it is signed by current signer // if not then return error if tmpl.HasCodeProtocol() { - sig := GetSignatureFromData(data) - arr := strings.SplitN(string(sig), ":", 3) - if len(arr) == 2 { - // signature has no fragment - return "", errorutil.NewWithTag("signer", "re-signing code templates are not allowed for security reasons.") - } - if len(arr) == 3 { - // signature has fragment verify if it is equal to current fragment - fragment := t.GetUserFragment() - if fragment != arr[2] { + if len(existingSignature) > 0 { + arr := strings.SplitN(string(existingSignature), ":", 3) + if len(arr) == 2 { + // signature has no fragment return "", errorutil.NewWithTag("signer", "re-signing code templates are not allowed for security reasons.") } + if len(arr) == 3 { + // signature has fragment verify if it is equal to current fragment + fragment := t.GetUserFragment() + if fragment != arr[2] { + return "", errorutil.NewWithTag("signer", "re-signing code templates are not allowed for security reasons.") + } + } } } - buff := bytes.NewBuffer(RemoveSignatureFromData(data)) + buff := bytes.NewBuffer(content) // if file has any imports process them for _, file := range tmpl.GetFileImports() { bin, err := os.ReadFile(file) @@ -123,12 +128,16 @@ func (t *TemplateSigner) sign(data []byte) (string, error) { // Verify verifies the given template with the template signer func (t *TemplateSigner) Verify(data []byte, tmpl SignableTemplate) (bool, error) { - digestData := ReDigest.Find(data) - if len(digestData) == 0 { - return false, errors.New("digest not found") + signature, content := ExtractSignatureAndContent(data) + if len(signature) == 0 { + return false, errors.New("no signature found") + } + + if !bytes.HasPrefix(signature, []byte(SignaturePattern)) { + return false, errors.New("signature must be at the end of the template") } - digestData = bytes.TrimSpace(bytes.TrimPrefix(digestData, []byte(SignaturePattern))) + digestData := bytes.TrimSpace(bytes.TrimPrefix(signature, []byte(SignaturePattern))) // remove fragment from digest as it is used for re-signing purposes only digestString := strings.TrimSuffix(string(digestData), ":"+t.GetUserFragment()) digest, err := hex.DecodeString(digestString) @@ -136,7 +145,7 @@ func (t *TemplateSigner) Verify(data []byte, tmpl SignableTemplate) (bool, error return false, err } - buff := bytes.NewBuffer(RemoveSignatureFromData(data)) + buff := bytes.NewBuffer(content) // if file has any imports process them for _, file := range tmpl.GetFileImports() { bin, err := os.ReadFile(file) diff --git a/pkg/templates/signer/tmpl_signer_test.go b/pkg/templates/signer/tmpl_signer_test.go new file mode 100644 index 0000000000..832dcb0e05 --- /dev/null +++ b/pkg/templates/signer/tmpl_signer_test.go @@ -0,0 +1,126 @@ +package signer + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + testCertFile = "../../../integration_tests/protocols/keys/ci.crt" + testKeyFile = "../../../integration_tests/protocols/keys/ci-private-key.pem" +) + +type mockSignableTemplate struct { + imports []string + hasCode bool +} + +func (m *mockSignableTemplate) GetFileImports() []string { + return m.imports +} + +func (m *mockSignableTemplate) HasCodeProtocol() bool { + return m.hasCode +} + +var signer, _ = NewTemplateSignerFromFiles(testCertFile, testKeyFile) + +func TestTemplateSignerSignAndVerify(t *testing.T) { + tempDir := t.TempDir() + + tests := []struct { + name string + data []byte + tmpl SignableTemplate + wantSignErr bool + wantVerifyErr bool + wantVerified bool + modifyAfterSign func([]byte) []byte + }{ + { + name: "Simple template", + data: []byte("id: test-template\ninfo:\n name: Test Template"), + tmpl: &mockSignableTemplate{}, + wantVerified: true, + }, + { + name: "Template with imports", + data: []byte("id: test-template\ninfo:\n name: Test Template"), + tmpl: &mockSignableTemplate{imports: []string{ + filepath.Join(tempDir, "import1.yaml"), + filepath.Join(tempDir, "import2.yaml"), + }}, + wantVerified: true, + }, + { + name: "Template with code protocol", + data: []byte("id: test-template\ninfo:\n name: Test Template\n\ncode:\n - engine: bash\n source: echo 'Hello, World!'"), + tmpl: &mockSignableTemplate{hasCode: true}, + wantSignErr: false, + wantVerified: true, + }, + { + name: "Tampered template", + data: []byte("id: test-template\ninfo:\n name: Test Template"), + tmpl: &mockSignableTemplate{}, + modifyAfterSign: func(data []byte) []byte { + signatureIndex := bytes.LastIndex(data, []byte(SignaturePattern)) + if signatureIndex == -1 { + return data + } + return append(data[:signatureIndex], append([]byte("# Tampered content\n"), data[signatureIndex:]...)...) + }, + wantVerified: false, + }, + { + name: "Invalid signature", + data: []byte("id: test-template\ninfo:\n name: Test Template"), + tmpl: &mockSignableTemplate{}, + modifyAfterSign: func(data []byte) []byte { + return append(bytes.TrimSuffix(data, []byte("\n")), []byte("\n# digest: invalid_signature:fragment")...) + }, + wantVerifyErr: true, + wantVerified: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create import files if needed + for _, imp := range tt.tmpl.GetFileImports() { + err := os.WriteFile(imp, []byte("imported content"), 0644) + require.NoError(t, err, "Failed to create import file") + } + + // Sign the template + signature, err := signer.Sign(tt.data, tt.tmpl) + if tt.wantSignErr { + assert.Error(t, err, "Expected an error during signing") + return + } + require.NoError(t, err, "Failed to sign template") + + // Append signature to the template data + signedData := append(tt.data, []byte("\n"+signature)...) + + // Apply any modifications after signing if specified + if tt.modifyAfterSign != nil { + signedData = tt.modifyAfterSign(signedData) + } + + // Verify the signature + verified, err := signer.Verify(signedData, tt.tmpl) + if tt.wantVerifyErr { + assert.Error(t, err, "Expected an error during verification") + } else { + assert.NoError(t, err, "Unexpected error during verification") + } + assert.Equal(t, tt.wantVerified, verified, "Unexpected verification result") + }) + } +} diff --git a/pkg/templates/template_sign.go b/pkg/templates/template_sign.go index 24ae41c5e0..1eb09a447c 100644 --- a/pkg/templates/template_sign.go +++ b/pkg/templates/template_sign.go @@ -75,11 +75,12 @@ func SignTemplate(templateSigner *signer.TemplateSigner, templatePath string) er return ErrNotATemplate } if !template.Verified { + _, content := signer.ExtractSignatureAndContent(bin) signatureData, err := templateSigner.Sign(bin, template) if err != nil { return err } - buff := bytes.NewBuffer(signer.RemoveSignatureFromData(bin)) + buff := bytes.NewBuffer(content) buff.WriteString("\n" + signatureData) return os.WriteFile(templatePath, buff.Bytes(), 0644) } From 3064788d35a8ec8623eb3a54eca920ea289f1259 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 19 Aug 2024 23:02:27 +0300 Subject: [PATCH 09/38] fix race condition --- .../http/httpclientpool/clientpool.go | 20 +++++++++++++++++++ pkg/protocols/http/request.go | 5 +++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/pkg/protocols/http/httpclientpool/clientpool.go b/pkg/protocols/http/httpclientpool/clientpool.go index 2c244556ae..da7d5467e2 100644 --- a/pkg/protocols/http/httpclientpool/clientpool.go +++ b/pkg/protocols/http/httpclientpool/clientpool.go @@ -32,6 +32,7 @@ var ( forceMaxRedirects int normalClient *retryablehttp.Client clientPool *mapsutil.SyncLockMap[string, *retryablehttp.Client] + rawHttpClientMu sync.Mutex ) // Init initializes the clientpool implementation @@ -102,6 +103,22 @@ type Configuration struct { ResponseHeaderTimeout time.Duration } +func (c *Configuration) Clone() *Configuration { + clone := *c + if c.Connection != nil { + cloneConnection := &ConnectionConfiguration{ + DisableKeepAlive: c.Connection.DisableKeepAlive, + } + if c.Connection.HasCookieJar() { + cookiejar := *c.Connection.GetCookieJar() + cloneConnection.SetCookieJar(&cookiejar) + } + clone.Connection = cloneConnection + } + + return &clone +} + // Hash returns the hash of the configuration to allow client pooling func (c *Configuration) Hash() string { builder := &strings.Builder{} @@ -131,6 +148,9 @@ func (c *Configuration) HasStandardOptions() bool { // GetRawHTTP returns the rawhttp request client func GetRawHTTP(options *protocols.ExecutorOptions) *rawhttp.Client { + rawHttpClientMu.Lock() + defer rawHttpClientMu.Unlock() + if rawHttpClient == nil { rawHttpOptions := rawhttp.DefaultOptions if types.ProxyURL != "" { diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index 98e1932477..3955325599 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -770,7 +770,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ // check for cookie related configuration if input.CookieJar != nil { - connConfiguration := request.connConfiguration + connConfiguration := request.connConfiguration.Clone() connConfiguration.Connection.SetCookieJar(input.CookieJar) modifiedConfig = connConfiguration } @@ -778,7 +778,8 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ updatedTimeout, ok := generatedRequest.request.Context().Value(httpclientpool.WithCustomTimeout{}).(httpclientpool.WithCustomTimeout) if ok { if modifiedConfig == nil { - modifiedConfig = request.connConfiguration + connConfiguration := request.connConfiguration.Clone() + modifiedConfig = connConfiguration } modifiedConfig.ResponseHeaderTimeout = updatedTimeout.Timeout } From 50023428d0ae90f5ed09ebe4392364e06129ecb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 19 Aug 2024 23:09:07 +0300 Subject: [PATCH 10/38] fix lint --- lib/sdk.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/sdk.go b/lib/sdk.go index 04b084f244..daeb68c14b 100644 --- a/lib/sdk.go +++ b/lib/sdk.go @@ -178,7 +178,8 @@ func (e *NucleiEngine) SignTemplate(tmplSigner *signer.TemplateSigner, data []by if err != nil { return data, err } - buff := bytes.NewBuffer(signer.RemoveSignatureFromData(data)) + _, content := signer.ExtractSignatureAndContent(data) + buff := bytes.NewBuffer(content) buff.WriteString("\n" + signatureData) return buff.Bytes(), err } From d84aea93cc2844132af1ae364567124a48b93a87 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Tue, 20 Aug 2024 15:04:01 +0700 Subject: [PATCH 11/38] chore: sorts gitignore (#5548) Signed-off-by: Dwi Siswanto --- .gitignore | 73 +++++++++++++++++++++++++++--------------------------- 1 file changed, 36 insertions(+), 37 deletions(-) diff --git a/.gitignore b/.gitignore index b9e561e3a6..5a50568910 100644 --- a/.gitignore +++ b/.gitignore @@ -1,43 +1,42 @@ +# Misc +**/*-cache +**/*-config +**/.cache +*.DS_Store +*.exe +.devcontainer +.gitignore .idea .vscode -.devcontainer -vendor + +# Binaries +**/bindgen +**/debug-* +**/docgen +**/functional-test +**/fuzzplayground +**/integration-test +**/jsdocgen +**/main +**/memogen +**/nuclei +**/nuclei-stats* +**/nuclei_dev +**/nuclei_main +**/scan-charts +**/scrapefunc +**/scrapefuncs +**/tsgen + +# Templates +/*.yaml + +# Generated docs +/dsl.md + +# Junk dist -integration_tests/nuclei -integration_tests/.cache -integration_tests/integration-test -cmd/nuclei/main -cmd/nuclei/nuclei -cmd/integration-test/nuclei -cmd/functional-test/nuclei_dev -cmd/functional-test/nuclei_main -cmd/functional-test/functional-test -cmd/docgen/docgen pkg/protocols/common/helpers/deserialization/testdata/Deserialize.class pkg/protocols/common/helpers/deserialization/testdata/ValueObject.class pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser -*.exe -.gitignore -pkg/js/devtools/bindgen/cmd/bindgen/bindgen -pkg/js/devtools/jsdocgen/jsdocgen -pkg/js/devtools/tsgen/tsgen -pkg/js/devtools/tsgen/cmd/tsgen/tsgen -*.DS_Store -pkg/protocols/headless/engine/.cache -/nuclei -/bindgen -/jsdocgen -/tsgen -/scrapefuncs -/integration_tests/.cache/ -/*.yaml -**/*-config -**/*-cache -/fuzzplayground -integration_tests/fuzzplayground -/dsl.md -/nuclei-stats -/nuclei-stats-* -/scan-charts -/**/debug-* - +vendor \ No newline at end of file From b1152efd422363fd00843cc1ff94b19837e54392 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Tue, 20 Aug 2024 15:06:39 +0700 Subject: [PATCH 12/38] chore(make): make it DRY (#5549) Signed-off-by: Dwi Siswanto --- Makefile | 141 ++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 102 insertions(+), 39 deletions(-) diff --git a/Makefile b/Makefile index a3f60a64b3..3fa6a033ef 100644 --- a/Makefile +++ b/Makefile @@ -1,64 +1,127 @@ # Go parameters -GOCMD=go -GOBUILD=$(GOCMD) build -GOMOD=$(GOCMD) mod -GOTEST=$(GOCMD) test +GOCMD := go +GOBUILD := $(GOCMD) build +GOBUILD_OUTPUT := +GOBUILD_PACKAGES := +GOBUILD_ADDITIONAL_ARGS := +GOMOD := $(GOCMD) mod +GOTEST := $(GOCMD) test GOFLAGS := -v # This should be disabled if the binary uses pprof LDFLAGS := -s -w ifneq ($(shell go env GOOS),darwin) -LDFLAGS := -extldflags "-static" + LDFLAGS = -extldflags "-static" endif -.PHONY: all build build-stats scan-charts docs test integration functional tidy verify devtools jsupdate ts fuzzplayground memogen dsl-docs +.PHONY: all build build-stats clean devtools-all devtools-bindgen devtools-scrapefuncs +.PHONY: devtools-tsgen docs dsl-docs functional fuzzplayground go-build integration +.PHONY: jsupdate-all jsupdate-bindgen jsupdate-tsgen memogen scan-charts test tidy ts verify all: build -build: - rm -f nuclei 2>/dev/null - $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "nuclei" cmd/nuclei/main.go -build-stats: - rm -f nuclei-stats 2>/dev/null - $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -tags=stats -o "nuclei-stats" cmd/nuclei/main.go -scan-charts: - rm -f scan-charts 2>/dev/null - $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "scan-charts" cmd/scan-charts/main.go + +clean: + rm -f '${GOBUILD_OUTPUT}' 2>/dev/null + +go-build: clean +go-build: + $(GOBUILD) $(GOFLAGS) -ldflags '${LDFLAGS}' $(GOBUILD_ADDITIONAL_ARGS) \ + -o '${GOBUILD_OUTPUT}' $(GOBUILD_PACKAGES) + +build: GOBUILD_OUTPUT = nuclei +build: GOBUILD_PACKAGES = cmd/nuclei/main.go +build: go-build + +build-stats: GOBUILD_OUTPUT = nuclei-stats +build-stats: GOBUILD_PACKAGES = cmd/nuclei/main.go +build-stats: GOBUILD_ADDITIONAL_ARGS = -tags=stats +build-stats: go-build + +scan-charts: GOBUILD_OUTPUT = scan-charts +scan-charts: GOBUILD_PACKAGES = cmd/scan-charts/main.go +scan-charts: go-build + +docs: GOBUILD_OUTPUT = cmd/docgen/docgen +docs: GOBUILD_PACKAGES = cmd/docgen/docgen.go +docs: bin = dstdocgen docs: - if ! which dstdocgen > /dev/null; then - echo -e "Command not found! Install? (y/n) \c" - go get -v github.com/projectdiscovery/yamldoc-go/cmd/docgen/dstdocgen + @if ! which $(bin) >/dev/null; then \ + read -p "${bin} not found. Do you want to install it? (y/n) " answer; \ + if [ "$$answer" = "y" ]; then \ + echo "Installing ${bin}..."; \ + go get -v github.com/projectdiscovery/yamldoc-go/cmd/docgen/$(bin); \ + go install -v github.com/projectdiscovery/yamldoc-go/cmd/docgen/$(bin); \ + else \ + echo "Please install ${bin} manually."; \ + exit 1; \ + fi \ fi + + # TODO: Handle the panic, so that we just need to run `go install $(bin)@latest` (line 51-52) $(GOCMD) generate pkg/templates/templates.go - $(GOBUILD) -o "cmd/docgen/docgen" cmd/docgen/docgen.go - ./cmd/docgen/docgen docs.md nuclei-jsonschema.json + + $(GOBUILD) -o "${GOBUILD_OUTPUT}" $(GOBUILD_PACKAGES) + ./$(GOBUILD_OUTPUT) docs.md nuclei-jsonschema.json + + git reset --hard # line 59 + test: $(GOTEST) $(GOFLAGS) ./... + integration: cd integration_tests; bash run.sh + functional: cd cmd/functional-test; bash run.sh + tidy: $(GOMOD) tidy + verify: tidy $(GOMOD) verify -devtools: - $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "bindgen" pkg/js/devtools/bindgen/cmd/bindgen/main.go - $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "tsgen" pkg/js/devtools/tsgen/cmd/tsgen/main.go - $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "scrapefuncs" pkg/js/devtools/scrapefuncs/main.go -jsupdate: - $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "bindgen" pkg/js/devtools/bindgen/cmd/bindgen/main.go - $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "tsgen" pkg/js/devtools/tsgen/cmd/tsgen/main.go - ./bindgen -dir pkg/js/libs -out pkg/js/generated - ./tsgen -dir pkg/js/libs -out pkg/js/generated/ts -ts: - $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "tsgen" pkg/js/devtools/tsgen/cmd/tsgen/main.go - ./tsgen -dir pkg/js/libs -out pkg/js/generated/ts -fuzzplayground: - $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "fuzzplayground" cmd/tools/fuzzplayground/main.go + +devtools-bindgen: GOBUILD_OUTPUT = bindgen +devtools-bindgen: GOBUILD_PACKAGES = pkg/js/devtools/bindgen/cmd/bindgen/main.go +devtools-bindgen: go-build + +devtools-tsgen: GOBUILD_OUTPUT = tsgen +devtools-tsgen: GOBUILD_PACKAGES = pkg/js/devtools/tsgen/cmd/tsgen/main.go +devtools-tsgen: go-build + +devtools-scrapefuncs: GOBUILD_OUTPUT = scrapefuncs +devtools-scrapefuncs: GOBUILD_PACKAGES = pkg/js/devtools/scrapefuncs/main.go +devtools-scrapefuncs: go-build + +devtools-all: devtools-bindgen devtools-tsgen devtools-scrapefuncs + +jsupdate-bindgen: GOBUILD_OUTPUT = bindgen +jsupdate-bindgen: GOBUILD_PACKAGES = pkg/js/devtools/bindgen/cmd/bindgen/main.go +jsupdate-bindgen: go-build +jsupdate-bindgen: + ./$(GOBUILD_OUTPUT) -dir pkg/js/libs -out pkg/js/generated + +jsupdate-tsgen: GOBUILD_OUTPUT = tsgen +jsupdate-tsgen: GOBUILD_PACKAGES = pkg/js/devtools/tsgen/cmd/tsgen/main.go +jsupdate-tsgen: go-build +jsupdate-tsgen: + ./$(GOBUILD_OUTPUT) -dir pkg/js/libs -out pkg/js/generated/ts + +jsupdate-all: jsupdate-bindgen jsupdate-tsgen + +ts: jsupdate-tsgen + +fuzzplayground: GOBUILD_OUTPUT = fuzzplayground +fuzzplayground: GOBUILD_PACKAGES = cmd/tools/fuzzplayground/main.go +fuzzplayground: LDFLAGS = -s -w +fuzzplayground: go-build + +memogen: GOBUILD_OUTPUT = memogen +memogen: GOBUILD_PACKAGES = cmd/memogen/memogen.go +memogen: go-build memogen: - $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "memogen" cmd/memogen/memogen.go - ./memogen -src pkg/js/libs -tpl cmd/memogen/function.tpl + ./$(GOBUILD_OUTPUT) -src pkg/js/libs -tpl cmd/memogen/function.tpl + +dsl-docs: GOBUILD_OUTPUT = scrapefuncs +dsl-docs: GOBUILD_PACKAGES = pkg/js/devtools/scrapefuncs/main.go dsl-docs: - rm -f dsl.md scrapefuncs 2>/dev/null - $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "scrapefuncs" pkg/js/devtools/scrapefuncs/main.go - ./scrapefuncs -out dsl.md + ./$(GOBUILD_OUTPUT) -out dsl.md From 46782ff90c29d2bc04ca130763ffbf95e95c5b37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Wed, 21 Aug 2024 11:26:17 +0300 Subject: [PATCH 13/38] use sync.Once --- pkg/protocols/http/httpclientpool/clientpool.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/protocols/http/httpclientpool/clientpool.go b/pkg/protocols/http/httpclientpool/clientpool.go index da7d5467e2..5750d5aead 100644 --- a/pkg/protocols/http/httpclientpool/clientpool.go +++ b/pkg/protocols/http/httpclientpool/clientpool.go @@ -29,10 +29,10 @@ import ( var ( rawHttpClient *rawhttp.Client + rawHttpClientOnce sync.Once forceMaxRedirects int normalClient *retryablehttp.Client clientPool *mapsutil.SyncLockMap[string, *retryablehttp.Client] - rawHttpClientMu sync.Mutex ) // Init initializes the clientpool implementation @@ -148,10 +148,7 @@ func (c *Configuration) HasStandardOptions() bool { // GetRawHTTP returns the rawhttp request client func GetRawHTTP(options *protocols.ExecutorOptions) *rawhttp.Client { - rawHttpClientMu.Lock() - defer rawHttpClientMu.Unlock() - - if rawHttpClient == nil { + rawHttpClientOnce.Do(func() { rawHttpOptions := rawhttp.DefaultOptions if types.ProxyURL != "" { rawHttpOptions.Proxy = types.ProxyURL @@ -162,7 +159,7 @@ func GetRawHTTP(options *protocols.ExecutorOptions) *rawhttp.Client { } rawHttpOptions.Timeout = options.Options.GetTimeouts().HttpTimeout rawHttpClient = rawhttp.NewClient(rawHttpOptions) - } + }) return rawHttpClient } From 35a0d673ad8e12b11e90e8e0090feb26ea042b46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Wed, 21 Aug 2024 11:36:33 +0300 Subject: [PATCH 14/38] remove redundant code --- pkg/protocols/common/protocolstate/state.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go index b7c7796fa8..b21a1be9e5 100644 --- a/pkg/protocols/common/protocolstate/state.go +++ b/pkg/protocols/common/protocolstate/state.go @@ -214,6 +214,5 @@ func Close() { Dialer.Close() Dialer = nil } - Dialer = nil StopActiveMemGuardian() } From 87c8cd8e0133d03f31ecad27f18f738c04174cb0 Mon Sep 17 00:00:00 2001 From: Roy Reznik Date: Wed, 21 Aug 2024 11:44:01 +0100 Subject: [PATCH 15/38] Upgrade gitlab --- go.mod | 10 ++++------ go.sum | 19 ++++++++----------- pkg/reporting/trackers/gitlab/gitlab.go | 2 +- 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 855b9f3c77..026a96169e 100644 --- a/go.mod +++ b/go.mod @@ -36,10 +36,10 @@ require ( github.com/syndtr/goleveldb v1.0.0 github.com/valyala/fasttemplate v1.2.2 github.com/weppos/publicsuffix-go v0.30.2 - github.com/xanzy/go-gitlab v0.84.0 + github.com/xanzy/go-gitlab v0.107.0 go.uber.org/multierr v1.11.0 golang.org/x/net v0.26.0 - golang.org/x/oauth2 v0.18.0 + golang.org/x/oauth2 v0.22.0 golang.org/x/text v0.16.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -262,7 +262,6 @@ require ( github.com/gobwas/pool v0.2.1 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.6.0 // indirect @@ -306,10 +305,9 @@ require ( golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/mod v0.17.0 // indirect golang.org/x/sys v0.21.0 // indirect - golang.org/x/time v0.5.0 // indirect + golang.org/x/time v0.6.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d - google.golang.org/appengine v1.6.8 // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index 43326744d8..22a4618330 100644 --- a/go.sum +++ b/go.sum @@ -453,8 +453,6 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= @@ -1068,8 +1066,8 @@ github.com/weppos/publicsuffix-go v0.30.2/go.mod h1:/hGscit36Yt+wammfBBwdMdxBT8b github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220927085643-dc0d00c92642/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= -github.com/xanzy/go-gitlab v0.84.0 h1:PdpCaskQSgcVDsx21c6ikf8Rfyo7SNtFAJwP9PrbCFE= -github.com/xanzy/go-gitlab v0.84.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw= +github.com/xanzy/go-gitlab v0.107.0 h1:P2CT9Uy9yN9lJo3FLxpMZ4xj6uWcpnigXsjvqJ6nd2Y= +github.com/xanzy/go-gitlab v0.107.0/go.mod h1:wKNKh3GkYDMOsGmnfuX+ITCmDuSDWFO0G+C4AygL9RY= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -1291,8 +1289,8 @@ golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= -golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= -golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= +golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= +golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1421,8 +1419,8 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1505,8 +1503,6 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1569,8 +1565,9 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/reporting/trackers/gitlab/gitlab.go b/pkg/reporting/trackers/gitlab/gitlab.go index d816a26af6..fdb2b948f1 100644 --- a/pkg/reporting/trackers/gitlab/gitlab.go +++ b/pkg/reporting/trackers/gitlab/gitlab.go @@ -78,7 +78,7 @@ func (i *Integration) CreateIssue(event *output.ResultEvent) (*filters.CreateIss if label := i.options.IssueLabel; label != "" { labels = append(labels, label) } - customLabels := gitlab.Labels(labels) + customLabels := gitlab.LabelOptions(labels) assigneeIDs := []int{i.userID} if i.options.DuplicateIssueCheck { searchIn := "title" From d1f4c98cd76f82e556b4a9c9406c45bcec32ae7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Wed, 21 Aug 2024 15:03:41 +0300 Subject: [PATCH 16/38] Revert "remove redundant code" This reverts commit 35a0d673ad8e12b11e90e8e0090feb26ea042b46. --- pkg/protocols/common/protocolstate/state.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go index b21a1be9e5..b7c7796fa8 100644 --- a/pkg/protocols/common/protocolstate/state.go +++ b/pkg/protocols/common/protocolstate/state.go @@ -214,5 +214,6 @@ func Close() { Dialer.Close() Dialer = nil } + Dialer = nil StopActiveMemGuardian() } From 5e102b782ba28386288460f7e6f1fec08ea514fa Mon Sep 17 00:00:00 2001 From: mzack9999 Date: Wed, 21 Aug 2024 16:09:47 +0200 Subject: [PATCH 17/38] fixing race + nil crash --- pkg/protocols/common/protocolstate/state.go | 15 +++++++++++++-- pkg/protocols/http/httpclientpool/clientpool.go | 4 ++-- pkg/protocols/http/request.go | 6 +++++- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go index b7c7796fa8..5c56fdf77e 100644 --- a/pkg/protocols/common/protocolstate/state.go +++ b/pkg/protocols/common/protocolstate/state.go @@ -5,6 +5,7 @@ import ( "fmt" "net" "net/url" + "sync" "github.com/go-sql-driver/mysql" "github.com/pkg/errors" @@ -19,9 +20,17 @@ import ( // Dialer is a shared fastdialer instance for host DNS resolution var ( - Dialer *fastdialer.Dialer + muDialer sync.RWMutex + Dialer *fastdialer.Dialer ) +func GetDialer() *fastdialer.Dialer { + muDialer.RLock() + defer muDialer.RUnlock() + + return Dialer +} + func ShouldInit() bool { return Dialer == nil } @@ -210,10 +219,12 @@ func interfaceAddresses(interfaceName string) ([]net.Addr, error) { // Close closes the global shared fastdialer func Close() { + muDialer.Lock() + defer muDialer.Unlock() + if Dialer != nil { Dialer.Close() Dialer = nil } - Dialer = nil StopActiveMemGuardian() } diff --git a/pkg/protocols/http/httpclientpool/clientpool.go b/pkg/protocols/http/httpclientpool/clientpool.go index 5750d5aead..65879818f6 100644 --- a/pkg/protocols/http/httpclientpool/clientpool.go +++ b/pkg/protocols/http/httpclientpool/clientpool.go @@ -250,7 +250,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl transport := &http.Transport{ ForceAttemptHTTP2: options.ForceAttemptHTTP2, - DialContext: protocolstate.Dialer.Dial, + DialContext: protocolstate.GetDialer().Dial, DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) { if options.TlsImpersonate { return protocolstate.Dialer.DialTLSWithConfigImpersonate(ctx, network, addr, tlsConfig, impersonate.Random, nil) @@ -258,7 +258,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl if options.HasClientCertificates() || options.ForceAttemptHTTP2 { return protocolstate.Dialer.DialTLSWithConfig(ctx, network, addr, tlsConfig) } - return protocolstate.Dialer.DialTLS(ctx, network, addr) + return protocolstate.GetDialer().DialTLS(ctx, network, addr) }, MaxIdleConns: maxIdleConns, MaxIdleConnsPerHost: maxIdleConnsPerHost, diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index 3955325599..5a1219b83f 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -942,7 +942,11 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ if input.MetaInput.CustomIP != "" { outputEvent["ip"] = input.MetaInput.CustomIP } else { - outputEvent["ip"] = protocolstate.Dialer.GetDialedIP(hostname) + dialer := protocolstate.GetDialer() + if dialer != nil { + outputEvent["ip"] = dialer.GetDialedIP(hostname) + } + // try getting cname request.addCNameIfAvailable(hostname, outputEvent) } From ddf9b2fbec27224a536e38ee0ecde5874f7b2c5b Mon Sep 17 00:00:00 2001 From: mzack9999 Date: Wed, 21 Aug 2024 16:29:00 +0200 Subject: [PATCH 18/38] fix --- lib/sdk.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/sdk.go b/lib/sdk.go index 04b084f244..daeb68c14b 100644 --- a/lib/sdk.go +++ b/lib/sdk.go @@ -178,7 +178,8 @@ func (e *NucleiEngine) SignTemplate(tmplSigner *signer.TemplateSigner, data []by if err != nil { return data, err } - buff := bytes.NewBuffer(signer.RemoveSignatureFromData(data)) + _, content := signer.ExtractSignatureAndContent(data) + buff := bytes.NewBuffer(content) buff.WriteString("\n" + signatureData) return buff.Bytes(), err } From bfdc507c6cef6db636ae8b3e693002271ed4ddab Mon Sep 17 00:00:00 2001 From: mzack9999 Date: Wed, 21 Aug 2024 16:34:53 +0200 Subject: [PATCH 19/38] lint --- pkg/external/customtemplates/gitlab.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/external/customtemplates/gitlab.go b/pkg/external/customtemplates/gitlab.go index 40f191423a..fbb9a61cef 100644 --- a/pkg/external/customtemplates/gitlab.go +++ b/pkg/external/customtemplates/gitlab.go @@ -83,8 +83,8 @@ func (bk *customTemplateGitLabRepo) Download(_ context.Context) { // Get the directory listing for the files in the project tree, _, err := bk.gitLabClient.Repositories.ListTree(projectID, &gitlab.ListTreeOptions{ - Ref: gitlab.String(project.DefaultBranch), - Recursive: gitlab.Bool(true), + Ref: gitlab.Ptr(project.DefaultBranch), + Recursive: gitlab.Ptr(true), }) if err != nil { gologger.Error().Msgf("error retrieving files from GitLab project: %s (%d) %s", project.Name, projectID, err) @@ -95,7 +95,7 @@ func (bk *customTemplateGitLabRepo) Download(_ context.Context) { // If the object is not a file or file extension is not .yaml, skip it if file.Type == "blob" && filepath.Ext(file.Path) == ".yaml" { gf := &gitlab.GetFileOptions{ - Ref: gitlab.String(project.DefaultBranch), + Ref: gitlab.Ptr(project.DefaultBranch), } f, _, err := bk.gitLabClient.RepositoryFiles.GetFile(projectID, file.Path, gf) if err != nil { From 56af26a502692774dc879f5fe5f10d3caf5f0e6e Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Thu, 22 Aug 2024 15:49:07 +0300 Subject: [PATCH 20/38] Add auto assign (#5555) * add auto assign * commit to last commit --- .github/auto_assign.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .github/auto_assign.yml diff --git a/.github/auto_assign.yml b/.github/auto_assign.yml new file mode 100644 index 0000000000..127400e160 --- /dev/null +++ b/.github/auto_assign.yml @@ -0,0 +1,8 @@ +addReviewers: true +reviewers: + - dogancanbakir + +numberOfReviewers: 1 +# A list of keywords to be skipped the process that add reviewers if pull requests include it +# skipKeywords: +# - wip \ No newline at end of file From 710f1ad240a1fedbdd0e91573c633f218e9fb409 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2024 05:02:29 +0000 Subject: [PATCH 21/38] chore(deps): bump github.com/projectdiscovery/gologger Bumps [github.com/projectdiscovery/gologger](https://github.com/projectdiscovery/gologger) from 1.1.19 to 1.1.21. - [Release notes](https://github.com/projectdiscovery/gologger/releases) - [Commits](https://github.com/projectdiscovery/gologger/compare/v1.1.19...v1.1.21) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/gologger dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 026a96169e..05ddf348f1 100644 --- a/go.mod +++ b/go.mod @@ -84,7 +84,7 @@ require ( github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb github.com/projectdiscovery/goflags v0.1.63 - github.com/projectdiscovery/gologger v1.1.19 + github.com/projectdiscovery/gologger v1.1.21 github.com/projectdiscovery/gostruct v0.0.2 github.com/projectdiscovery/gozero v0.0.2 github.com/projectdiscovery/httpx v1.6.7 @@ -96,7 +96,7 @@ require ( github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/uncover v1.0.9 github.com/projectdiscovery/useragent v0.0.65 - github.com/projectdiscovery/utils v0.2.5 + github.com/projectdiscovery/utils v0.2.6 github.com/projectdiscovery/wappalyzergo v0.1.14 github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 diff --git a/go.sum b/go.sum index 22a4618330..a5d836f324 100644 --- a/go.sum +++ b/go.sum @@ -842,8 +842,8 @@ github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG90 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY= github.com/projectdiscovery/goflags v0.1.63 h1:xy3nqpzmSFEmG0LKTv3xLcp9LMbvbFkLZcjcnFe3xR4= github.com/projectdiscovery/goflags v0.1.63/go.mod h1:ptFbCm+5RmSHTSG4oniwBRRHJlguBTOOfMUfezeGbZ0= -github.com/projectdiscovery/gologger v1.1.19 h1:b7cU32XuDrDiwhr7hlDeE6mfj/nENBtHEohe51txJCE= -github.com/projectdiscovery/gologger v1.1.19/go.mod h1:DbeKwx9IEfcvnclImX5gBlhIKUuOZwOM5itdpYXl+54= +github.com/projectdiscovery/gologger v1.1.21 h1:FD9MGE9adY/y2IxDpQgPZIvZAkQIx3bvcuY+d1FLQUo= +github.com/projectdiscovery/gologger v1.1.21/go.mod h1:eyvlcHiSuzJQtwmh+fcfwZVaqVgSGtEP1oL1gJFgP3c= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E= @@ -884,8 +884,8 @@ github.com/projectdiscovery/uncover v1.0.9 h1:s5RbkD/V4r8QcPkys4gTTqMuRSgXq0Jpre github.com/projectdiscovery/uncover v1.0.9/go.mod h1:2PUF3SpB5QNIJ8epaB2xbRzkPaxEAWRDm3Ir2ijt81U= github.com/projectdiscovery/useragent v0.0.65 h1:x78ZwWdqpzokOHxLITUXvq+ljkTKc19z3ILGtoV1N70= github.com/projectdiscovery/useragent v0.0.65/go.mod h1:deOP8YLJU6SCzM8k+K8PjkcOF4Ux0spqyO4ODZGIT4A= -github.com/projectdiscovery/utils v0.2.5 h1:puiw3/gK1WTEj9ZKnW00tAS0ex+ETqbLJ8juRuejA98= -github.com/projectdiscovery/utils v0.2.5/go.mod h1:fv3WC6m/+bVT+wH+fMB+JsKviEC/UAuDSeKxWPlK3kk= +github.com/projectdiscovery/utils v0.2.6 h1:7ZZ+tK8u3vO/X7i0HXwybNusFBtYnM363bxDxJkVnso= +github.com/projectdiscovery/utils v0.2.6/go.mod h1:XFy8Ldt537TGkj3snH0zXfF9LigRfmHPGCNfDolSdEQ= github.com/projectdiscovery/wappalyzergo v0.1.14 h1:nt1IM4RUmqeymsXk4h6BsZbKDoS2hjFvPkT2GaI1rz4= github.com/projectdiscovery/wappalyzergo v0.1.14/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= From 4a39fdb8d6553793feb9c7310d9ee348ff927a1b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2024 05:26:22 +0000 Subject: [PATCH 22/38] chore(deps): bump github.com/projectdiscovery/utils from 0.2.5 to 0.2.7 Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.2.5 to 0.2.7. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.2.5...v0.2.7) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 05ddf348f1..e7c5efd226 100644 --- a/go.mod +++ b/go.mod @@ -96,7 +96,7 @@ require ( github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/uncover v1.0.9 github.com/projectdiscovery/useragent v0.0.65 - github.com/projectdiscovery/utils v0.2.6 + github.com/projectdiscovery/utils v0.2.7 github.com/projectdiscovery/wappalyzergo v0.1.14 github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 diff --git a/go.sum b/go.sum index a5d836f324..28af644eed 100644 --- a/go.sum +++ b/go.sum @@ -884,8 +884,8 @@ github.com/projectdiscovery/uncover v1.0.9 h1:s5RbkD/V4r8QcPkys4gTTqMuRSgXq0Jpre github.com/projectdiscovery/uncover v1.0.9/go.mod h1:2PUF3SpB5QNIJ8epaB2xbRzkPaxEAWRDm3Ir2ijt81U= github.com/projectdiscovery/useragent v0.0.65 h1:x78ZwWdqpzokOHxLITUXvq+ljkTKc19z3ILGtoV1N70= github.com/projectdiscovery/useragent v0.0.65/go.mod h1:deOP8YLJU6SCzM8k+K8PjkcOF4Ux0spqyO4ODZGIT4A= -github.com/projectdiscovery/utils v0.2.6 h1:7ZZ+tK8u3vO/X7i0HXwybNusFBtYnM363bxDxJkVnso= -github.com/projectdiscovery/utils v0.2.6/go.mod h1:XFy8Ldt537TGkj3snH0zXfF9LigRfmHPGCNfDolSdEQ= +github.com/projectdiscovery/utils v0.2.7 h1:XWdz7SscL++jqsnQ9ecHzSZE0RK33tyPcnqcXw+vmKs= +github.com/projectdiscovery/utils v0.2.7/go.mod h1:N0N7tbdNFPegd9NpJ3onCPClaBrERcOIB88yww6UCF8= github.com/projectdiscovery/wappalyzergo v0.1.14 h1:nt1IM4RUmqeymsXk4h6BsZbKDoS2hjFvPkT2GaI1rz4= github.com/projectdiscovery/wappalyzergo v0.1.14/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= From 253fbc36190aec186bdf8c32e1fea34350ad8d8e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2024 05:26:23 +0000 Subject: [PATCH 23/38] chore(deps): bump github.com/projectdiscovery/goflags Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.63 to 0.1.64. - [Release notes](https://github.com/projectdiscovery/goflags/releases) - [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.63...v0.1.64) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/goflags dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 05ddf348f1..851fc631ee 100644 --- a/go.mod +++ b/go.mod @@ -83,7 +83,7 @@ require ( github.com/projectdiscovery/dsl v0.1.8 github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb - github.com/projectdiscovery/goflags v0.1.63 + github.com/projectdiscovery/goflags v0.1.64 github.com/projectdiscovery/gologger v1.1.21 github.com/projectdiscovery/gostruct v0.0.2 github.com/projectdiscovery/gozero v0.0.2 diff --git a/go.sum b/go.sum index a5d836f324..fe4f540f3d 100644 --- a/go.sum +++ b/go.sum @@ -840,8 +840,8 @@ github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvm github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE= github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG906Drtbpz4DwU5mhGIeOhRcktDH4cGQitGUMAsg= github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY= -github.com/projectdiscovery/goflags v0.1.63 h1:xy3nqpzmSFEmG0LKTv3xLcp9LMbvbFkLZcjcnFe3xR4= -github.com/projectdiscovery/goflags v0.1.63/go.mod h1:ptFbCm+5RmSHTSG4oniwBRRHJlguBTOOfMUfezeGbZ0= +github.com/projectdiscovery/goflags v0.1.64 h1:FDfwdt9N97Hi8OuhbkDlKtVttpc/CRMIWQVa08VsHsI= +github.com/projectdiscovery/goflags v0.1.64/go.mod h1:3FyHIVQtnycNOc1LE3O1jj/XR5XuMdF9QfHd0ujhnX4= github.com/projectdiscovery/gologger v1.1.21 h1:FD9MGE9adY/y2IxDpQgPZIvZAkQIx3bvcuY+d1FLQUo= github.com/projectdiscovery/gologger v1.1.21/go.mod h1:eyvlcHiSuzJQtwmh+fcfwZVaqVgSGtEP1oL1gJFgP3c= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= From 2670b6865f6590cb4fee720bcaf1a51b90fe256f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2024 05:53:48 +0000 Subject: [PATCH 24/38] chore(deps): bump github.com/projectdiscovery/tlsx from 1.1.6 to 1.1.7 Bumps [github.com/projectdiscovery/tlsx](https://github.com/projectdiscovery/tlsx) from 1.1.6 to 1.1.7. - [Release notes](https://github.com/projectdiscovery/tlsx/releases) - [Changelog](https://github.com/projectdiscovery/tlsx/blob/main/.goreleaser.yml) - [Commits](https://github.com/projectdiscovery/tlsx/compare/v1.1.6...v1.1.7) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/tlsx dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e7c5efd226..4832777901 100644 --- a/go.mod +++ b/go.mod @@ -93,7 +93,7 @@ require ( github.com/projectdiscovery/ratelimit v0.0.49 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 github.com/projectdiscovery/sarif v0.0.1 - github.com/projectdiscovery/tlsx v1.1.6 + github.com/projectdiscovery/tlsx v1.1.7 github.com/projectdiscovery/uncover v1.0.9 github.com/projectdiscovery/useragent v0.0.65 github.com/projectdiscovery/utils v0.2.7 diff --git a/go.sum b/go.sum index 28af644eed..35385e3eb6 100644 --- a/go.sum +++ b/go.sum @@ -878,8 +878,8 @@ github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGL github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1UzcNsC77A= -github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo= +github.com/projectdiscovery/tlsx v1.1.7 h1:eSsl/SmTDL/z2CMeSrbssk4f/9oOotMP1SgXl3yynSM= +github.com/projectdiscovery/tlsx v1.1.7/go.mod h1:g66QQ4/y4tLVjoGbzWIv+Q6xwFzxfJbEDx86Y1dYHDM= github.com/projectdiscovery/uncover v1.0.9 h1:s5RbkD/V4r8QcPkys4gTTqMuRSgXq0JprejqLSopN9Y= github.com/projectdiscovery/uncover v1.0.9/go.mod h1:2PUF3SpB5QNIJ8epaB2xbRzkPaxEAWRDm3Ir2ijt81U= github.com/projectdiscovery/useragent v0.0.65 h1:x78ZwWdqpzokOHxLITUXvq+ljkTKc19z3ILGtoV1N70= From 4a85e73fbb717109f98dd3e8f96bc524b8cda26b Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Mon, 26 Aug 2024 19:48:45 +0300 Subject: [PATCH 25/38] skip dependabot prs (#5575) --- .github/auto_assign.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/auto_assign.yml b/.github/auto_assign.yml index 127400e160..0c65e536ca 100644 --- a/.github/auto_assign.yml +++ b/.github/auto_assign.yml @@ -3,6 +3,5 @@ reviewers: - dogancanbakir numberOfReviewers: 1 -# A list of keywords to be skipped the process that add reviewers if pull requests include it -# skipKeywords: -# - wip \ No newline at end of file +skipKeywords: + - '@dependabot' \ No newline at end of file From aac1af1308da2187d1f6bb9aa73c8fe226c4fe6b Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 28 Aug 2024 14:11:02 +0700 Subject: [PATCH 26/38] refactor(fuzz): use `mapsutil.Map` type (#5558) * refactor(fuzz): use `mapsutil.Map` type Signed-off-by: Dwi Siswanto * fix(headless): handle empty `key` in `*Value.SetParsedValue` Signed-off-by: Dwi Siswanto * feat(fuzz): add type assertion checks Signed-off-by: Dwi Siswanto --------- Signed-off-by: Dwi Siswanto --- pkg/fuzz/component/path.go | 21 ++++++++++++++++++--- pkg/fuzz/component/value.go | 12 +++++++++++- pkg/fuzz/dataformat/kv.go | 2 +- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/pkg/fuzz/component/path.go b/pkg/fuzz/component/path.go index b9aebd61af..a81955167a 100644 --- a/pkg/fuzz/component/path.go +++ b/pkg/fuzz/component/path.go @@ -7,6 +7,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/dataformat" "github.com/projectdiscovery/retryablehttp-go" + "github.com/projectdiscovery/utils/maps" urlutil "github.com/projectdiscovery/utils/url" ) @@ -81,7 +82,7 @@ func (q *Path) Delete(key string) error { // Rebuild returns a new request with the // component rebuilt func (q *Path) Rebuild() (*retryablehttp.Request, error) { - originalValues := make(map[string]interface{}) + originalValues := mapsutil.Map[string, any]{} splitted := strings.Split(q.req.URL.Path, "/") for i := range splitted { pathTillNow := strings.Join(splitted[:i+1], "/") @@ -95,8 +96,22 @@ func (q *Path) Rebuild() (*retryablehttp.Request, error) { lengthSplitted := len(q.value.parsed.Map) for i := lengthSplitted; i > 0; i-- { key := strconv.Itoa(i) - original := originalValues[key].(string) - new := q.value.parsed.Map[key].(string) + + original, ok := originalValues.GetOrDefault(key, "").(string) + if !ok { + continue + } + + new, ok := q.value.parsed.Map.GetOrDefault(key, "").(string) + if !ok { + continue + } + + if new == original { + // no need to replace + continue + } + originalPath = strings.Replace(originalPath, original, new, 1) } diff --git a/pkg/fuzz/component/value.go b/pkg/fuzz/component/value.go index ad2044cf1b..030c5194bf 100644 --- a/pkg/fuzz/component/value.go +++ b/pkg/fuzz/component/value.go @@ -73,12 +73,22 @@ func (v *Value) SetParsed(data dataformat.KV, dataFormat string) { // SetParsedValue sets the parsed value for a key // in the parsed map -func (v *Value) SetParsedValue(key string, value string) bool { +func (v *Value) SetParsedValue(key, value string) bool { + if key == "" { + return false + } + origValue := v.parsed.Get(key) if origValue == nil { v.parsed.Set(key, value) return true } + + // TODO(dwisiswant0): I'm sure that this can be simplified because + // `dataformat.KV.*` is a type of `mapsutil.*` where the value is `any`. So, + // it looks like we won't type conversion here or even have its own methods + // inside `dataformat.KV`. + // If the value is a list, append to it // otherwise replace it switch v := origValue.(type) { diff --git a/pkg/fuzz/dataformat/kv.go b/pkg/fuzz/dataformat/kv.go index 72bd0da62e..3036a1c76c 100644 --- a/pkg/fuzz/dataformat/kv.go +++ b/pkg/fuzz/dataformat/kv.go @@ -13,7 +13,7 @@ import ( // if it's not important/significant (ex: json,xml) we use map // this also allows us to iteratively implement ordered map type KV struct { - Map map[string]interface{} + Map mapsutil.Map[string, any] OrderedMap *mapsutil.OrderedMap[string, any] } From bf58b4d756ce8e2ba595024c54e87b4cdc2ad1c3 Mon Sep 17 00:00:00 2001 From: Peter Dave Hello Date: Wed, 28 Aug 2024 18:54:15 +0800 Subject: [PATCH 27/38] Remove conflicting `-U` option for `apk` in Dockerfile (#5578) The `--no-cache` option updates the index and avoids temporary files, making `-U` unnecessary, consistent with the subsequent `apk add`. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9dde3d7e27..45f92d4485 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN go build ./cmd/nuclei # Release FROM alpine:3.18.6 -RUN apk -U upgrade --no-cache \ +RUN apk upgrade --no-cache \ && apk add --no-cache bind-tools chromium ca-certificates COPY --from=build-env /app/nuclei /usr/local/bin/ From 6b71af448a995d76d3069aa1326f1c85946c346b Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Wed, 28 Aug 2024 13:57:43 +0300 Subject: [PATCH 28/38] Fixed issue with `-ms` option to scan non accessible host (#5576) * fail if OnResult callback is not called * generate error message from error logs * try..parse.. * fix lint * add error message to last matcher event * fix network protocol error logging * log returned log from ExecuteWithResults * add back specific logging * clean up the msg * minor * init integration test for -ms * add tests for http,network,js,ws protocols * fix lint * fix network test * return err for dns protocol * add integration test for dns protocol --- cmd/integration-test/integration-test.go | 1 + cmd/integration-test/matcher-status.go | 119 +++++++++++++++++++++++ pkg/protocols/dns/request.go | 4 +- pkg/protocols/network/request.go | 4 +- pkg/protocols/network/request_test.go | 2 +- pkg/scan/scan_context.go | 6 +- pkg/tmplexec/exec.go | 57 +++++++++++ 7 files changed, 187 insertions(+), 6 deletions(-) create mode 100644 cmd/integration-test/matcher-status.go diff --git a/cmd/integration-test/integration-test.go b/cmd/integration-test/integration-test.go index 180efcfc6e..84ec6790f8 100644 --- a/cmd/integration-test/integration-test.go +++ b/cmd/integration-test/integration-test.go @@ -55,6 +55,7 @@ var ( "dsl": dslTestcases, "flow": flowTestcases, "javascript": jsTestcases, + "matcher-status": matcherStatusTestcases, } // flakyTests are run with a retry count of 3 flakyTests = map[string]bool{ diff --git a/cmd/integration-test/matcher-status.go b/cmd/integration-test/matcher-status.go new file mode 100644 index 0000000000..b88763720a --- /dev/null +++ b/cmd/integration-test/matcher-status.go @@ -0,0 +1,119 @@ +package main + +import ( + "encoding/json" + "fmt" + + "github.com/projectdiscovery/nuclei/v3/pkg/output" + "github.com/projectdiscovery/nuclei/v3/pkg/testutils" +) + +var matcherStatusTestcases = []TestCaseInfo{ + {Path: "protocols/http/get.yaml", TestCase: &httpNoAccess{}}, + {Path: "protocols/network/net-https.yaml", TestCase: &networkNoAccess{}}, + {Path: "protocols/headless/headless-basic.yaml", TestCase: &headlessNoAccess{}}, + {Path: "protocols/javascript/net-https.yaml", TestCase: &javascriptNoAccess{}}, + {Path: "protocols/websocket/basic.yaml", TestCase: &websocketNoAccess{}}, + {Path: "protocols/dns/a.yaml", TestCase: &dnsNoAccess{}}, +} + +type httpNoAccess struct{} + +func (h *httpNoAccess) Execute(filePath string) error { + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "trust_me_bro.real", debug, "-ms", "-j") + if err != nil { + return err + } + event := &output.ResultEvent{} + _ = json.Unmarshal([]byte(results[0]), event) + + if event.Error != "no address found for host" { + return fmt.Errorf("unexpected result: expecting \"no address found for host\" error but got none") + } + return nil +} + +type networkNoAccess struct{} + +// Execute executes a test case and returns an error if occurred +func (h *networkNoAccess) Execute(filePath string) error { + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "trust_me_bro.real", debug, "-ms", "-j") + if err != nil { + return err + } + event := &output.ResultEvent{} + _ = json.Unmarshal([]byte(results[0]), event) + + if event.Error != "no address found for host" { + return fmt.Errorf("unexpected result: expecting \"no address found for host\" error but got \"%s\"", event.Error) + } + return nil +} + +type headlessNoAccess struct{} + +// Execute executes a test case and returns an error if occurred +func (h *headlessNoAccess) Execute(filePath string) error { + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "trust_me_bro.real", debug, "-headless", "-ms", "-j") + if err != nil { + return err + } + event := &output.ResultEvent{} + _ = json.Unmarshal([]byte(results[0]), event) + + if event.Error == "" { + return fmt.Errorf("unexpected result: expecting an error but got \"%s\"", event.Error) + } + return nil +} + +type javascriptNoAccess struct{} + +// Execute executes a test case and returns an error if occurred +func (h *javascriptNoAccess) Execute(filePath string) error { + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "trust_me_bro.real", debug, "-ms", "-j") + if err != nil { + return err + } + event := &output.ResultEvent{} + _ = json.Unmarshal([]byte(results[0]), event) + + if event.Error == "" { + return fmt.Errorf("unexpected result: expecting an error but got \"%s\"", event.Error) + } + return nil +} + +type websocketNoAccess struct{} + +// Execute executes a test case and returns an error if occurred +func (h *websocketNoAccess) Execute(filePath string) error { + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "ws://trust_me_bro.real", debug, "-ms", "-j") + if err != nil { + return err + } + event := &output.ResultEvent{} + _ = json.Unmarshal([]byte(results[0]), event) + + if event.Error == "" { + return fmt.Errorf("unexpected result: expecting an error but got \"%s\"", event.Error) + } + return nil +} + +type dnsNoAccess struct{} + +// Execute executes a test case and returns an error if occurred +func (h *dnsNoAccess) Execute(filePath string) error { + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "trust_me_bro.real", debug, "-ms", "-j") + if err != nil { + return err + } + event := &output.ResultEvent{} + _ = json.Unmarshal([]byte(results[0]), event) + + if event.Error == "" { + return fmt.Errorf("unexpected result: expecting an error but got \"%s\"", event.Error) + } + return nil +} diff --git a/pkg/protocols/dns/request.go b/pkg/protocols/dns/request.go index a16c2af88b..9457845270 100644 --- a/pkg/protocols/dns/request.go +++ b/pkg/protocols/dns/request.go @@ -106,7 +106,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, } func (request *Request) execute(input *contextargs.Context, domain string, metadata, previous output.InternalEvent, vars map[string]interface{}, callback protocols.OutputEventCallback) error { - + var err error if vardump.EnableVarDump { gologger.Debug().Msgf("DNS Protocol request variables: \n%s\n", vardump.DumpVariables(vars)) } @@ -199,7 +199,7 @@ func (request *Request) execute(input *contextargs.Context, domain string, metad } callback(event) - return nil + return err } func (request *Request) parseDNSInput(host string) (string, error) { diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go index 90390e53c3..5fa8609d51 100644 --- a/pkg/protocols/network/request.go +++ b/pkg/protocols/network/request.go @@ -155,14 +155,14 @@ func (request *Request) executeOnTarget(input *contextargs.Context, visited maps } visited.Set(actualAddress, struct{}{}) - if err := request.executeAddress(variables, actualAddress, address, input, kv.tls, previous, callback); err != nil { + if err = request.executeAddress(variables, actualAddress, address, input, kv.tls, previous, callback); err != nil { outputEvent := request.responseToDSLMap("", "", "", address, "") callback(&output.InternalWrappedEvent{InternalEvent: outputEvent}) gologger.Warning().Msgf("[%v] Could not make network request for (%s) : %s\n", request.options.TemplateID, actualAddress, err) continue } } - return nil + return err } // executeAddress executes the request for an address diff --git a/pkg/protocols/network/request_test.go b/pkg/protocols/network/request_test.go index 1945888e9b..7ff0f4882b 100644 --- a/pkg/protocols/network/request_test.go +++ b/pkg/protocols/network/request_test.go @@ -86,7 +86,7 @@ func TestNetworkExecuteWithResults(t *testing.T) { err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) { finalEvent = event }) - require.Nil(t, err, "could not execute network request") + require.NotNil(t, err, "could not execute network request") }) require.Nil(t, finalEvent.Results, "could not get event output from request") diff --git a/pkg/scan/scan_context.go b/pkg/scan/scan_context.go index b8f59ac7d4..45456ddcac 100644 --- a/pkg/scan/scan_context.go +++ b/pkg/scan/scan_context.go @@ -52,6 +52,10 @@ func (s *ScanContext) Context() context.Context { return s.ctx } +func (s *ScanContext) GenerateErrorMessage() string { + return joinErrors(s.errors) +} + // GenerateResult returns final results slice from all events func (s *ScanContext) GenerateResult() []*output.ResultEvent { s.m.Lock() @@ -96,7 +100,7 @@ func (s *ScanContext) LogError(err error) { } s.errors = append(s.errors, err) - errorMessage := joinErrors(s.errors) + errorMessage := s.GenerateErrorMessage() for _, result := range s.results { result.Error = errorMessage diff --git a/pkg/tmplexec/exec.go b/pkg/tmplexec/exec.go index 3d09f5e7a0..4ca9badf70 100644 --- a/pkg/tmplexec/exec.go +++ b/pkg/tmplexec/exec.go @@ -10,6 +10,7 @@ import ( "github.com/dop251/goja" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/js/compiler" + "github.com/projectdiscovery/nuclei/v3/pkg/operators" "github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" @@ -19,6 +20,8 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/generic" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/multiproto" + "github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr" + "github.com/projectdiscovery/utils/errkit" ) // TemplateExecutor is an executor for a template @@ -126,6 +129,8 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { executed := &atomic.Bool{} // matched in this case means something was exported / written to output matched := &atomic.Bool{} + // callbackCalled tracks if the callback was called or not + callbackCalled := &atomic.Bool{} defer func() { // it is essential to remove template context of `Scan i.e template x input pair` // since it is of no use after scan is completed (regardless of success or failure) @@ -143,6 +148,7 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { } ctx.OnResult = func(event *output.InternalWrappedEvent) { + callbackCalled.Store(true) if event == nil { // something went wrong return @@ -198,13 +204,64 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { } else { errx = e.engine.ExecuteWithResults(ctx) } + ctx.LogError(errx) if lastMatcherEvent != nil { + lastMatcherEvent.InternalEvent["error"] = tryParseCause(fmt.Errorf("%s", ctx.GenerateErrorMessage())) writeFailureCallback(lastMatcherEvent, e.options.Options.MatcherStatus) } + + //TODO: this is a hacky way to handle the case where the callback is not called and matcher-status is true. + // This is a workaround and needs to be refactored. + // Check if callback was never called and matcher-status is true + if !callbackCalled.Load() && e.options.Options.MatcherStatus { + fakeEvent := &output.InternalWrappedEvent{ + Results: []*output.ResultEvent{ + { + TemplateID: e.options.TemplateID, + Info: e.options.TemplateInfo, + Type: e.getTemplateType(), + Host: ctx.Input.MetaInput.Input, + Error: tryParseCause(fmt.Errorf("%s", ctx.GenerateErrorMessage())), + }, + }, + OperatorsResult: &operators.Result{ + Matched: false, + }, + } + writeFailureCallback(fakeEvent, e.options.Options.MatcherStatus) + } + return executed.Load() || matched.Load(), errx } +// tryParseCause tries to parse the cause of given error +// this is legacy support due to use of errorutil in existing libraries +// but this should not be required once all libraries are updated +func tryParseCause(err error) string { + errStr := "" + errX := errkit.FromError(err) + if errX != nil { + var errCause error + + if len(errX.Errors()) > 1 { + errCause = errX.Errors()[0] + } + if errCause == nil { + errCause = errX + } + + msg := strings.Trim(errCause.Error(), "{} ") + parts := strings.Split(msg, ":") + errCause = errkit.New("%s", parts[len(parts)-1]) + errKind := errkit.GetErrorKind(err, nucleierr.ErrTemplateLogic).String() + errStr = errCause.Error() + errStr = strings.TrimSpace(strings.Replace(errStr, "errKind="+errKind, "", -1)) + } + + return errStr +} + // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (e *TemplateExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.ResultEvent, error) { var errx error From e0b254286806d4517ea8ce69b6cb25a044b5a1ef Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 28 Aug 2024 19:27:45 +0700 Subject: [PATCH 29/38] feat: conditionally panic-recover (#5553) * feat: conditionally panic-recover As discussed with @Mzack9999, we should avoid overusing panic-recover. We need to review the RCA first to determine whether this is an exceptional situation or if it's a higher-level function meant to recover from a panic. This approach will help us establish a robust error-handling strategy. The implementation of panic-recover should be conditional and NOT applied when running in a CI environment AND IS temporary. Once we've caught all errors and made the necessary corrections, we can remove the deferred recover function. Signed-off-by: Dwi Siswanto * chore(deps): bump `go-ci` to v1.0.2 Signed-off-by: Dwi Siswanto * chore(make): add `-race` to `GOFLAGS` in `test` Signed-off-by: Dwi Siswanto --------- Signed-off-by: Dwi Siswanto --- Makefile | 1 + go.mod | 1 + go.sum | 4 ++++ pkg/js/compiler/compiler.go | 7 +++++++ pkg/js/compiler/pool.go | 8 ++++++++ pkg/protocols/headless/engine/page_actions.go | 6 ++++++ pkg/tmplexec/flow/flow_executor.go | 7 +++++++ 7 files changed, 34 insertions(+) diff --git a/Makefile b/Makefile index 3fa6a033ef..8614ef82c9 100644 --- a/Makefile +++ b/Makefile @@ -65,6 +65,7 @@ docs: git reset --hard # line 59 +test: GOFLAGS = -race -v test: $(GOTEST) $(GOFLAGS) ./... diff --git a/go.mod b/go.mod index a0cd13984a..858a21bca3 100644 --- a/go.mod +++ b/go.mod @@ -177,6 +177,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/kataras/jwt v0.1.10 // indirect + github.com/kitabisa/go-ci v1.0.2 // indirect github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/kylelemons/godebug v1.1.0 // indirect diff --git a/go.sum b/go.sum index 310bd42ef8..66c91ae901 100644 --- a/go.sum +++ b/go.sum @@ -625,6 +625,10 @@ github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kitabisa/go-ci v1.0.1 h1://FHQzlDqYW+3qr0judsOE9X2ZrdRlRc66sCaVrLFGc= +github.com/kitabisa/go-ci v1.0.1/go.mod h1:4MWu+kf/+tvd0vLWSJA689Kn+hrYkZiymmZYT5BGT4g= +github.com/kitabisa/go-ci v1.0.2 h1:rqHf8KEbQOxVb998TbqGRo70Z7ol44io7/jLYJUvKp8= +github.com/kitabisa/go-ci v1.0.2/go.mod h1:e3wBSzaJbcifXrr/Gw2ZBLn44MmeqP5WySwXyHlCK/U= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= diff --git a/pkg/js/compiler/compiler.go b/pkg/js/compiler/compiler.go index c2dc15d2cb..b13e7f9ecb 100644 --- a/pkg/js/compiler/compiler.go +++ b/pkg/js/compiler/compiler.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/dop251/goja" + "github.com/kitabisa/go-ci" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v3/pkg/types" @@ -120,11 +121,17 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs, defer cancel() // execute the script results, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (val goja.Value, err error) { + // TODO(dwisiswant0): remove this once we get the RCA. defer func() { + if ci.IsCI() { + return + } + if r := recover(); r != nil { err = fmt.Errorf("panic: %v", r) } }() + return ExecuteProgram(program, args, opts) }) if err != nil { diff --git a/pkg/js/compiler/pool.go b/pkg/js/compiler/pool.go index 3407d97369..31e2e5378f 100644 --- a/pkg/js/compiler/pool.go +++ b/pkg/js/compiler/pool.go @@ -11,6 +11,7 @@ import ( "github.com/dop251/goja" "github.com/dop251/goja_nodejs/console" "github.com/dop251/goja_nodejs/require" + "github.com/kitabisa/go-ci" "github.com/projectdiscovery/gologger" _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libbytes" _ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libfs" @@ -84,11 +85,18 @@ func executeWithRuntime(runtime *goja.Runtime, p *goja.Program, args *ExecuteArg opts.Cleanup(runtime) } }() + + // TODO(dwisiswant0): remove this once we get the RCA. defer func() { + if ci.IsCI() { + return + } + if r := recover(); r != nil { err = fmt.Errorf("panic: %s", r) } }() + // set template ctx _ = runtime.Set("template", args.TemplateCtx) // set args diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go index 57cc371093..576b6134fa 100644 --- a/pkg/protocols/headless/engine/page_actions.go +++ b/pkg/protocols/headless/engine/page_actions.go @@ -14,6 +14,7 @@ import ( "github.com/go-rod/rod/lib/input" "github.com/go-rod/rod/lib/proto" "github.com/go-rod/rod/lib/utils" + "github.com/kitabisa/go-ci" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" @@ -54,7 +55,12 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, var waitFuncs := make([]func() error, 0) // avoid any future panics caused due to go-rod library + // TODO(dwisiswant0): remove this once we get the RCA. defer func() { + if ci.IsCI() { + return + } + if r := recover(); r != nil { err = errorutil.New("panic on headless action: %v", r) } diff --git a/pkg/tmplexec/flow/flow_executor.go b/pkg/tmplexec/flow/flow_executor.go index 6a1813efd9..6e71cf8407 100644 --- a/pkg/tmplexec/flow/flow_executor.go +++ b/pkg/tmplexec/flow/flow_executor.go @@ -13,6 +13,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/scan" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" + "github.com/kitabisa/go-ci" "github.com/projectdiscovery/nuclei/v3/pkg/types" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" @@ -201,7 +202,13 @@ func (f *FlowExecutor) ExecuteWithResults(ctx *scan.ScanContext) error { } }() + + // TODO(dwisiswant0): remove this once we get the RCA. defer func() { + if ci.IsCI() { + return + } + if r := recover(); r != nil { f.ctx.LogError(fmt.Errorf("panic occurred while executing flow: %v", r)) } From 88b281ae476bb7f93e84c501d86bb4fa31e0454a Mon Sep 17 00:00:00 2001 From: linchizhen Date: Sun, 1 Sep 2024 16:41:01 +0800 Subject: [PATCH 30/38] chore: fix some function names in comment (#5586) Signed-off-by: linchizhen --- cmd/tmc/main.go | 2 +- pkg/input/formats/openapi/generator.go | 2 +- pkg/input/provider/interface.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/tmc/main.go b/cmd/tmc/main.go index 5cb38afbee..a5971ca193 100644 --- a/cmd/tmc/main.go +++ b/cmd/tmc/main.go @@ -300,7 +300,7 @@ func formatTemplate(data string) (string, bool, error) { return data, false, errorutil.New("template format failed") } -// lintTemplateData lints template data using templateman lint api +// lintTemplate lints template data using templateman lint api func lintTemplate(data string) (bool, error) { resp, err := retryablehttp.DefaultClient().Post(fmt.Sprintf("%s/lint", tmBaseUrl), "application/x-yaml", strings.NewReader(data)) if err != nil { diff --git a/pkg/input/formats/openapi/generator.go b/pkg/input/formats/openapi/generator.go index 4027d76fc7..63fef7e290 100644 --- a/pkg/input/formats/openapi/generator.go +++ b/pkg/input/formats/openapi/generator.go @@ -416,7 +416,7 @@ schemaLabel: return globalParams, nil } -// generateExampleFromSchema generates an example from a schema object +// GenerateParameterFromSecurityScheme generates an example from a schema object func GenerateParameterFromSecurityScheme(scheme *openapi3.SecuritySchemeRef) (*openapi3.Parameter, error) { if !generic.EqualsAny(scheme.Value.Type, "http", "apiKey") { return nil, errorutil.NewWithTag("openapi", "unsupported security scheme type (%s) found in openapi file", scheme.Value.Type) diff --git a/pkg/input/provider/interface.go b/pkg/input/provider/interface.go index b7ad4b4480..e6d5da14a5 100644 --- a/pkg/input/provider/interface.go +++ b/pkg/input/provider/interface.go @@ -121,7 +121,7 @@ func NewInputProvider(opts InputOptions) (InputProvider, error) { } } -// SupportedFormats returns all supported input formats of nuclei +// SupportedInputFormats returns all supported input formats of nuclei func SupportedInputFormats() string { return "list, " + http.SupportedFormats() } From fdb8e0a8c68f0aaa6669848e035400ccae47a0fe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 05:11:44 +0000 Subject: [PATCH 31/38] chore(deps): bump github.com/projectdiscovery/fastdialer Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.2.5 to 0.2.7. - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.2.5...v0.2.7) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fastdialer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 8 ++++---- go.sum | 14 ++++++-------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 858a21bca3..7b3b596b96 100644 --- a/go.mod +++ b/go.mod @@ -20,11 +20,11 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.1.0 - github.com/projectdiscovery/fastdialer v0.2.5 - github.com/projectdiscovery/hmap v0.0.55 + github.com/projectdiscovery/fastdialer v0.2.7 + github.com/projectdiscovery/hmap v0.0.56 github.com/projectdiscovery/interactsh v1.2.0 github.com/projectdiscovery/rawhttp v0.1.60 - github.com/projectdiscovery/retryabledns v1.0.72 + github.com/projectdiscovery/retryabledns v1.0.73 github.com/projectdiscovery/retryablehttp-go v1.0.74 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 @@ -72,6 +72,7 @@ require ( github.com/go-sql-driver/mysql v1.7.1 github.com/h2non/filetype v1.1.3 github.com/invopop/yaml v0.3.1 + github.com/kitabisa/go-ci v1.0.2 github.com/labstack/echo/v4 v4.10.2 github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa github.com/lib/pq v1.10.9 @@ -177,7 +178,6 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/kataras/jwt v0.1.10 // indirect - github.com/kitabisa/go-ci v1.0.2 // indirect github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/kylelemons/godebug v1.1.0 // indirect diff --git a/go.sum b/go.sum index 66c91ae901..2fedffa27a 100644 --- a/go.sum +++ b/go.sum @@ -625,8 +625,6 @@ github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kitabisa/go-ci v1.0.1 h1://FHQzlDqYW+3qr0judsOE9X2ZrdRlRc66sCaVrLFGc= -github.com/kitabisa/go-ci v1.0.1/go.mod h1:4MWu+kf/+tvd0vLWSJA689Kn+hrYkZiymmZYT5BGT4g= github.com/kitabisa/go-ci v1.0.2 h1:rqHf8KEbQOxVb998TbqGRo70Z7ol44io7/jLYJUvKp8= github.com/kitabisa/go-ci v1.0.2/go.mod h1:e3wBSzaJbcifXrr/Gw2ZBLn44MmeqP5WySwXyHlCK/U= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= @@ -836,8 +834,8 @@ github.com/projectdiscovery/clistats v0.1.0 h1:b+LF1w0xhNd7cneKWMXb+/yUmF1n5szaw github.com/projectdiscovery/clistats v0.1.0/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= github.com/projectdiscovery/dsl v0.1.8 h1:ulz+o097XsYgqP4QXaAQhVSkoeD2ZPWd29wX3CTodUA= github.com/projectdiscovery/dsl v0.1.8/go.mod h1:AYJS2WQ/q0smr2v4pEJTg4DPe6k56KFKR7UFXvzNz/4= -github.com/projectdiscovery/fastdialer v0.2.5 h1:OZg31LFIBBHbXVnOEcFNtD0gBM80zpY68QvYI0xbw80= -github.com/projectdiscovery/fastdialer v0.2.5/go.mod h1:E1vmos7AeD/KZ7rsDeImjLp5H3jjas+2utVdUxGulvY= +github.com/projectdiscovery/fastdialer v0.2.7 h1:x7bEf/kDyVOuG9kHfJLMhWO3h7J+6fMDgLWkGYL8EFg= +github.com/projectdiscovery/fastdialer v0.2.7/go.mod h1:7/BGdaw7sCekuBh6lLZqPuoO5N0Qaizwvex/PjO+V8o= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= @@ -852,8 +850,8 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E= github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8= -github.com/projectdiscovery/hmap v0.0.55 h1:By0qSeQRE/ybeITUEL6oURN+j7qX7FVjCq92duwsqcA= -github.com/projectdiscovery/hmap v0.0.55/go.mod h1:uZOF/W8yOGDb/t9omqATirhVwiboa5F/c00OVhDeE0E= +github.com/projectdiscovery/hmap v0.0.56 h1:QLIuFhCc7jrBH9WYpxNBDooSoH+/S8cCm7VO5rMt3lU= +github.com/projectdiscovery/hmap v0.0.56/go.mod h1:XyGKMO93nLOzpaTTCDBSKEGx/XWXjSpoyZJxT92Qd/Q= github.com/projectdiscovery/httpx v1.6.7 h1:luJAtQ+iKpiyKKDpcWw1/fF1MaRX6JZ4R3P+ol5Uikk= github.com/projectdiscovery/httpx v1.6.7/go.mod h1:rqcuexBrb4v36ZyX0BHcHaovTurOJn2P65Tdt9cYdfE= github.com/projectdiscovery/interactsh v1.2.0 h1:Al6jHiR+Usl9egYJDLJaWNHOcH8Rugk8gWMasc8Cmw8= @@ -874,8 +872,8 @@ github.com/projectdiscovery/rawhttp v0.1.60 h1:L57kAATGRuCfOp8Fs0ly3j2wlip7R5pco github.com/projectdiscovery/rawhttp v0.1.60/go.mod h1:M97w6lsphTHKeqLSio4Lw9K4KLO2riJMYAOqVMIZ3ck= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= -github.com/projectdiscovery/retryabledns v1.0.72 h1:+k9JRYBMpZIyEQZJWVHoCHx1TEdpiuAS7iEru2bhs3A= -github.com/projectdiscovery/retryabledns v1.0.72/go.mod h1:BDhYnqSH0Os2G37eb4L4vnKrUcik6Q6pR02Cq5PbT+0= +github.com/projectdiscovery/retryabledns v1.0.73 h1:CLqtgoI6+YVcUE2jnZPFVvwj59CHDE4HNImLf62sSuw= +github.com/projectdiscovery/retryabledns v1.0.73/go.mod h1:Wo5tQHKtx6XcYYhMypRg8dr0GGnqIb4CzoTSEw2lo7I= github.com/projectdiscovery/retryablehttp-go v1.0.74 h1:jXZsg3nNDuDjGNeL0+hiEx/dqUqPyZW1zsotWW1y70c= github.com/projectdiscovery/retryablehttp-go v1.0.74/go.mod h1:fVRxa2fVMbomaPiOFA05kM2WrjV8wPFHW7J84eenVmQ= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= From 841d8913e574b75dac3feb7c5e418d9d959a84eb Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Mon, 2 Sep 2024 16:59:52 +0700 Subject: [PATCH 32/38] feat(headless): add `ActionWaitDialog` type (#5545) * feat(headless): add `dialog` action type also implement it Signed-off-by: Dwi Siswanto * refactor(headless): add `ActionData` for action output datas Signed-off-by: Dwi Siswanto * refactor(headless): rm `value` arg for `*Page.HandleDialog` also: * expose `err` from \*proto.PageHandleJavaScriptDialog` * conditional ActionData assignment based on Signed-off-by: Dwi Siswanto * refactor(headless): rename to `ActionWaitDialog` Signed-off-by: Dwi Siswanto * test(headless): fix mismatch assertion of `src` output of `ActionGetResource` Signed-off-by: Dwi Siswanto * test(headless): add TestActionWaitDialog test case Signed-off-by: Dwi Siswanto * feat(headless): add `GetActionDataWithDefault` generic func Signed-off-by: Dwi Siswanto * feat(headless): implement `GetActionDataWithDefault` to `header` & `status_code` Signed-off-by: Dwi Siswanto * refactor(headless): use `mapsutil.Map` instead Signed-off-by: Dwi Siswanto * Revert "feat(headless): add `GetActionDataWithDefault` generic func" This reverts commit fa12e0d6a221c8a7bf62200f69814ee27681f08f. --------- Signed-off-by: Dwi Siswanto --- pkg/protocols/headless/engine/action_types.go | 9 ++ pkg/protocols/headless/engine/page.go | 2 +- pkg/protocols/headless/engine/page_actions.go | 87 +++++++++---- .../headless/engine/page_actions_test.go | 118 ++++++++++++++---- pkg/protocols/headless/request.go | 8 +- 5 files changed, 171 insertions(+), 53 deletions(-) diff --git a/pkg/protocols/headless/engine/action_types.go b/pkg/protocols/headless/engine/action_types.go index c9cd39ad2a..d61e377431 100644 --- a/pkg/protocols/headless/engine/action_types.go +++ b/pkg/protocols/headless/engine/action_types.go @@ -6,11 +6,15 @@ import ( "strings" "github.com/invopop/jsonschema" + mapsutil "github.com/projectdiscovery/utils/maps" ) // ActionType defines the action type for a browser action type ActionType int8 +// ActionData stores the action output data +type ActionData = mapsutil.Map[string, any] + // Types to be executed by the user. // name:ActionType const ( @@ -68,6 +72,9 @@ const ( // ActionWaitEvent waits for a specific event. // name:waitevent ActionWaitEvent + // ActionWaitDialog waits for JavaScript dialog (alert, confirm, prompt, or onbeforeunload). + // name:dialog + ActionWaitDialog // ActionKeyboard performs a keyboard action event on a page. // name:keyboard ActionKeyboard @@ -104,6 +111,7 @@ var ActionStringToAction = map[string]ActionType{ "deleteheader": ActionDeleteHeader, "setbody": ActionSetBody, "waitevent": ActionWaitEvent, + "waitdialog": ActionWaitDialog, "keyboard": ActionKeyboard, "debug": ActionDebug, "sleep": ActionSleep, @@ -130,6 +138,7 @@ var ActionToActionString = map[ActionType]string{ ActionDeleteHeader: "deleteheader", ActionSetBody: "setbody", ActionWaitEvent: "waitevent", + ActionWaitDialog: "waitdialog", ActionKeyboard: "keyboard", ActionDebug: "debug", ActionSleep: "sleep", diff --git a/pkg/protocols/headless/engine/page.go b/pkg/protocols/headless/engine/page.go index 7e2bb37bbe..1b417ec402 100644 --- a/pkg/protocols/headless/engine/page.go +++ b/pkg/protocols/headless/engine/page.go @@ -45,7 +45,7 @@ type Options struct { } // Run runs a list of actions by creating a new page in the browser. -func (i *Instance) Run(input *contextargs.Context, actions []*Action, payloads map[string]interface{}, options *Options) (map[string]string, *Page, error) { +func (i *Instance) Run(input *contextargs.Context, actions []*Action, payloads map[string]interface{}, options *Options) (ActionData, *Page, error) { page, err := i.engine.Page(proto.TargetCreateTarget{}) if err != nil { return nil, nil, err diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go index 576b6134fa..b34ed18b94 100644 --- a/pkg/protocols/headless/engine/page_actions.go +++ b/pkg/protocols/headless/engine/page_actions.go @@ -48,8 +48,8 @@ const ( ) // ExecuteActions executes a list of actions on a page. -func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, variables map[string]interface{}) (outData map[string]string, err error) { - outData = make(map[string]string) +func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, variables map[string]interface{}) (outData ActionData, err error) { + outData = make(ActionData) // waitFuncs are function that needs to be executed after navigation // typically used for waitEvent waitFuncs := make([]func() error, 0) @@ -106,6 +106,8 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, var if waitFunc != nil { waitFuncs = append(waitFuncs, waitFunc) } + case ActionWaitDialog: + err = p.HandleDialog(act, outData) case ActionFilesInput: if p.options.Options.AllowLocalFileAccess { err = p.FilesInput(act, outData) @@ -148,7 +150,7 @@ type rule struct { } // WaitVisible waits until an element appears. -func (p *Page) WaitVisible(act *Action, out map[string]string) error { +func (p *Page) WaitVisible(act *Action, out ActionData) error { timeout, err := getTimeout(p, act) if err != nil { return errors.Wrap(err, "Wrong timeout given") @@ -223,7 +225,7 @@ func geTimeParameter(p *Page, act *Action, parameterName string, defaultValue ti } // ActionAddHeader executes a AddHeader action. -func (p *Page) ActionAddHeader(act *Action, out map[string]string) error { +func (p *Page) ActionAddHeader(act *Action, out ActionData) error { in := p.getActionArgWithDefaultValues(act, "part") args := make(map[string]string) @@ -234,7 +236,7 @@ func (p *Page) ActionAddHeader(act *Action, out map[string]string) error { } // ActionSetHeader executes a SetHeader action. -func (p *Page) ActionSetHeader(act *Action, out map[string]string) error { +func (p *Page) ActionSetHeader(act *Action, out ActionData) error { in := p.getActionArgWithDefaultValues(act, "part") args := make(map[string]string) @@ -245,7 +247,7 @@ func (p *Page) ActionSetHeader(act *Action, out map[string]string) error { } // ActionDeleteHeader executes a DeleteHeader action. -func (p *Page) ActionDeleteHeader(act *Action, out map[string]string) error { +func (p *Page) ActionDeleteHeader(act *Action, out ActionData) error { in := p.getActionArgWithDefaultValues(act, "part") args := make(map[string]string) @@ -255,7 +257,7 @@ func (p *Page) ActionDeleteHeader(act *Action, out map[string]string) error { } // ActionSetBody executes a SetBody action. -func (p *Page) ActionSetBody(act *Action, out map[string]string) error { +func (p *Page) ActionSetBody(act *Action, out ActionData) error { in := p.getActionArgWithDefaultValues(act, "part") args := make(map[string]string) @@ -265,7 +267,7 @@ func (p *Page) ActionSetBody(act *Action, out map[string]string) error { } // ActionSetMethod executes an SetMethod action. -func (p *Page) ActionSetMethod(act *Action, out map[string]string) error { +func (p *Page) ActionSetMethod(act *Action, out ActionData) error { in := p.getActionArgWithDefaultValues(act, "part") args := make(map[string]string) @@ -275,7 +277,7 @@ func (p *Page) ActionSetMethod(act *Action, out map[string]string) error { } // NavigateURL executes an ActionLoadURL actions loading a URL for the page. -func (p *Page) NavigateURL(action *Action, out map[string]string, allvars map[string]interface{}) error { +func (p *Page) NavigateURL(action *Action, out ActionData, allvars map[string]interface{}) error { // input <- is input url from cli // target <- is the url from template (ex: {{BaseURL}}/test) input, err := urlutil.Parse(p.input.MetaInput.Input) @@ -331,7 +333,7 @@ func (p *Page) NavigateURL(action *Action, out map[string]string, allvars map[st } // RunScript runs a script on the loaded page -func (p *Page) RunScript(action *Action, out map[string]string) error { +func (p *Page) RunScript(action *Action, out ActionData) error { code := p.getActionArgWithDefaultValues(action, "code") if code == "" { return errinvalidArguments @@ -352,7 +354,7 @@ func (p *Page) RunScript(action *Action, out map[string]string) error { } // ClickElement executes click actions for an element. -func (p *Page) ClickElement(act *Action, out map[string]string) error { +func (p *Page) ClickElement(act *Action, out ActionData) error { element, err := p.pageElementBy(act.Data) if err != nil { return errors.Wrap(err, errCouldNotGetElement) @@ -367,12 +369,12 @@ func (p *Page) ClickElement(act *Action, out map[string]string) error { } // KeyboardAction executes a keyboard action on the page. -func (p *Page) KeyboardAction(act *Action, out map[string]string) error { +func (p *Page) KeyboardAction(act *Action, out ActionData) error { return p.page.Keyboard.Type([]input.Key(p.getActionArgWithDefaultValues(act, "keys"))...) } // RightClickElement executes right click actions for an element. -func (p *Page) RightClickElement(act *Action, out map[string]string) error { +func (p *Page) RightClickElement(act *Action, out ActionData) error { element, err := p.pageElementBy(act.Data) if err != nil { return errors.Wrap(err, errCouldNotGetElement) @@ -387,7 +389,7 @@ func (p *Page) RightClickElement(act *Action, out map[string]string) error { } // Screenshot executes screenshot action on a page -func (p *Page) Screenshot(act *Action, out map[string]string) error { +func (p *Page) Screenshot(act *Action, out ActionData) error { to := p.getActionArgWithDefaultValues(act, "to") if to == "" { to = ksuid.New().String() @@ -450,7 +452,7 @@ func (p *Page) Screenshot(act *Action, out map[string]string) error { } // InputElement executes input element actions for an element. -func (p *Page) InputElement(act *Action, out map[string]string) error { +func (p *Page) InputElement(act *Action, out ActionData) error { value := p.getActionArgWithDefaultValues(act, "value") if value == "" { return errinvalidArguments @@ -469,7 +471,7 @@ func (p *Page) InputElement(act *Action, out map[string]string) error { } // TimeInputElement executes time input on an element -func (p *Page) TimeInputElement(act *Action, out map[string]string) error { +func (p *Page) TimeInputElement(act *Action, out ActionData) error { value := p.getActionArgWithDefaultValues(act, "value") if value == "" { return errinvalidArguments @@ -492,7 +494,7 @@ func (p *Page) TimeInputElement(act *Action, out map[string]string) error { } // SelectInputElement executes select input statement action on a element -func (p *Page) SelectInputElement(act *Action, out map[string]string) error { +func (p *Page) SelectInputElement(act *Action, out ActionData) error { value := p.getActionArgWithDefaultValues(act, "value") if value == "" { return errinvalidArguments @@ -517,7 +519,7 @@ func (p *Page) SelectInputElement(act *Action, out map[string]string) error { } // WaitLoad waits for the page to load -func (p *Page) WaitLoad(act *Action, out map[string]string) error { +func (p *Page) WaitLoad(act *Action, out ActionData) error { p.page.Timeout(2 * time.Second).WaitNavigation(proto.PageLifecycleEventNameFirstMeaningfulPaint)() // Wait for the window.onload event and also wait for the network requests @@ -531,7 +533,7 @@ func (p *Page) WaitLoad(act *Action, out map[string]string) error { } // GetResource gets a resource from an element from page. -func (p *Page) GetResource(act *Action, out map[string]string) error { +func (p *Page) GetResource(act *Action, out ActionData) error { element, err := p.pageElementBy(act.Data) if err != nil { return errors.Wrap(err, errCouldNotGetElement) @@ -547,7 +549,7 @@ func (p *Page) GetResource(act *Action, out map[string]string) error { } // FilesInput acts with a file input element on page -func (p *Page) FilesInput(act *Action, out map[string]string) error { +func (p *Page) FilesInput(act *Action, out ActionData) error { element, err := p.pageElementBy(act.Data) if err != nil { return errors.Wrap(err, errCouldNotGetElement) @@ -564,7 +566,7 @@ func (p *Page) FilesInput(act *Action, out map[string]string) error { } // ExtractElement extracts from an element on the page. -func (p *Page) ExtractElement(act *Action, out map[string]string) error { +func (p *Page) ExtractElement(act *Action, out ActionData) error { element, err := p.pageElementBy(act.Data) if err != nil { return errors.Wrap(err, errCouldNotGetElement) @@ -598,7 +600,7 @@ func (p *Page) ExtractElement(act *Action, out map[string]string) error { } // WaitEvent waits for an event to happen on the page. -func (p *Page) WaitEvent(act *Action, out map[string]string) (func() error, error) { +func (p *Page) WaitEvent(act *Action, out ActionData) (func() error, error) { event := p.getActionArgWithDefaultValues(act, "event") if event == "" { return nil, errors.New("event not recognized") @@ -636,6 +638,43 @@ func (p *Page) WaitEvent(act *Action, out map[string]string) (func() error, erro return waitFunc, nil } +// HandleDialog handles JavaScript dialog (alert, confirm, prompt, or onbeforeunload). +func (p *Page) HandleDialog(act *Action, out ActionData) error { + maxDuration := 10 * time.Second + + if dur := p.getActionArgWithDefaultValues(act, "max-duration"); dur != "" { + var err error + + maxDuration, err = time.ParseDuration(dur) + if err != nil { + return errorutil.NewWithErr(err).Msgf("could not parse max-duration") + } + } + + ctx, cancel := context.WithTimeout(context.Background(), maxDuration) + defer cancel() + + wait, handle := p.page.HandleDialog() + fn := func() (*proto.PageJavascriptDialogOpening, error) { + dialog := wait() + err := handle(&proto.PageHandleJavaScriptDialog{ + Accept: true, + PromptText: "", + }) + + return dialog, err + } + + dialog, err := contextutil.ExecFuncWithTwoReturns(ctx, fn) + if err == nil && act.Name != "" { + out[act.Name] = true + out[act.Name+"_type"] = string(dialog.Type) + out[act.Name+"_message"] = dialog.Message + } + + return nil +} + // pageElementBy returns a page element from a variety of inputs. // // Supported values for by: r -> selector & regex, x -> xpath, js -> eval js, @@ -670,14 +709,14 @@ func (p *Page) pageElementBy(data map[string]string) (*rod.Element, error) { } // DebugAction enables debug action on a page. -func (p *Page) DebugAction(act *Action, out map[string]string) error { +func (p *Page) DebugAction(act *Action, out ActionData) error { p.instance.browser.engine.SlowMotion(5 * time.Second) p.instance.browser.engine.Trace(true) return nil } // SleepAction sleeps on the page for a specified duration -func (p *Page) SleepAction(act *Action, out map[string]string) error { +func (p *Page) SleepAction(act *Action, out ActionData) error { seconds := act.Data["duration"] if seconds == "" { seconds = "5" diff --git a/pkg/protocols/headless/engine/page_actions_test.go b/pkg/protocols/headless/engine/page_actions_test.go index e6699d6408..3620bb2bb4 100644 --- a/pkg/protocols/headless/engine/page_actions_test.go +++ b/pkg/protocols/headless/engine/page_actions_test.go @@ -38,7 +38,7 @@ func TestActionNavigate(t *testing.T) { actions := []*Action{{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}} - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) { require.Nilf(t, err, "could not run page actions") require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") }) @@ -63,7 +63,7 @@ func TestActionScript(t *testing.T) { {ActionType: ActionTypeHolder{ActionType: ActionScript}, Name: "test", Data: map[string]string{"code": "() => window.test"}}, } - testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) { + testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out ActionData) { require.Nil(t, err, "could not run page actions") require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") require.Equal(t, "some-data", out["test"], "could not run js and get results correctly") @@ -77,7 +77,7 @@ func TestActionScript(t *testing.T) { {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, {ActionType: ActionTypeHolder{ActionType: ActionScript}, Name: "test", Data: map[string]string{"code": "() => window.test"}}, } - testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) { + testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out ActionData) { require.Nil(t, err, "could not run page actions") require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") require.Equal(t, "some-data", out["test"], "could not run js and get results correctly with js hook") @@ -101,7 +101,7 @@ func TestActionClick(t *testing.T) { {ActionType: ActionTypeHolder{ActionType: ActionClick}, Data: map[string]string{"selector": "button"}}, // Use css selector for clicking } - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) { require.Nil(t, err, "could not run page actions") require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") el := page.Page().MustElement("button") @@ -134,7 +134,7 @@ func TestActionRightClick(t *testing.T) { {ActionType: ActionTypeHolder{ActionType: ActionRightClick}, Data: map[string]string{"selector": "button"}}, // Use css selector for clicking } - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) { require.Nil(t, err, "could not run page actions") require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") el := page.Page().MustElement("button") @@ -159,7 +159,7 @@ func TestActionTextInput(t *testing.T) { {ActionType: ActionTypeHolder{ActionType: ActionTextInput}, Data: map[string]string{"selector": "input", "value": "test"}}, } - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) { require.Nil(t, err, "could not run page actions") require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") el := page.Page().MustElement("input") @@ -182,7 +182,7 @@ func TestActionHeadersChange(t *testing.T) { } } - testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) { + testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out ActionData) { require.Nil(t, err, "could not run page actions") require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") }) @@ -205,7 +205,7 @@ func TestActionScreenshot(t *testing.T) { {ActionType: ActionTypeHolder{ActionType: ActionScreenshot}, Data: map[string]string{"to": filePath}}, } - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) { require.Nil(t, err, "could not run page actions") require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") _ = page.Page() @@ -233,7 +233,7 @@ func TestActionScreenshotToDir(t *testing.T) { {ActionType: ActionTypeHolder{ActionType: ActionScreenshot}, Data: map[string]string{"to": filePath, "mkdir": "true"}}, } - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) { require.Nil(t, err, "could not run page actions") require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") _ = page.Page() @@ -260,7 +260,7 @@ func TestActionTimeInput(t *testing.T) { {ActionType: ActionTypeHolder{ActionType: ActionTimeInput}, Data: map[string]string{"selector": "input", "value": "2006-01-02T15:04:05Z"}}, } - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) { require.Nil(t, err, "could not run page actions") require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") el := page.Page().MustElement("input") @@ -288,7 +288,7 @@ func TestActionSelectInput(t *testing.T) { {ActionType: ActionTypeHolder{ActionType: ActionSelectInput}, Data: map[string]string{"by": "x", "xpath": "//select[@id='test']", "value": "Test2", "selected": "true"}}, } - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) { require.Nil(t, err, "could not run page actions") el := page.Page().MustElement("select") require.Equal(t, "Test2", el.MustText(), "could not get input change value") @@ -311,7 +311,7 @@ func TestActionFilesInput(t *testing.T) { {ActionType: ActionTypeHolder{ActionType: ActionFilesInput}, Data: map[string]string{"selector": "input", "value": "test1.pdf"}}, } - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) { require.Nil(t, err, "could not run page actions") require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") el := page.Page().MustElement("input") @@ -337,7 +337,7 @@ func TestActionFilesInputNegative(t *testing.T) { } t.Setenv("LOCAL_FILE_ACCESS", "false") - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) { require.ErrorContains(t, err, ErrLFAccessDenied.Error(), "got file access when -lfa is false") }) } @@ -359,7 +359,7 @@ func TestActionWaitLoad(t *testing.T) { {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, } - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) { require.Nil(t, err, "could not run page actions") el := page.Page().MustElement("button") style, attributeErr := el.Attribute("style") @@ -384,9 +384,12 @@ func TestActionGetResource(t *testing.T) { {ActionType: ActionTypeHolder{ActionType: ActionGetResource}, Data: map[string]string{"by": "x", "xpath": "//img[@id='test']"}, Name: "src"}, } - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) { require.Nil(t, err, "could not run page actions") - require.Equal(t, len(out["src"]), 121808, "could not find resource") + + src, ok := out["src"].(string) + require.True(t, ok, "could not assert src to string") + require.Equal(t, len(src), 121808, "could not find resource") }) } @@ -404,7 +407,7 @@ func TestActionExtract(t *testing.T) { {ActionType: ActionTypeHolder{ActionType: ActionExtract}, Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}, Name: "extract"}, } - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) { require.Nil(t, err, "could not run page actions") require.Equal(t, "Wait for me!", out["extract"], "could not extract text") }) @@ -423,7 +426,7 @@ func TestActionSetMethod(t *testing.T) { {ActionType: ActionTypeHolder{ActionType: ActionSetMethod}, Data: map[string]string{"part": "x", "method": "SET"}}, } - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) { require.Nil(t, err, "could not run page actions") require.Equal(t, "SET", page.rules[0].Args["method"], "could not find resource") }) @@ -442,7 +445,7 @@ func TestActionAddHeader(t *testing.T) { } } - testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) { + testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out ActionData) { require.Nil(t, err, "could not run page actions") require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") }) @@ -463,7 +466,7 @@ func TestActionDeleteHeader(t *testing.T) { } } - testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) { + testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out ActionData) { require.Nil(t, err, "could not run page actions") require.Equal(t, "header deleted", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not delete header correctly") }) @@ -481,7 +484,7 @@ func TestActionSetBody(t *testing.T) { _, _ = fmt.Fprintln(w, string(body)) } - testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) { + testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out ActionData) { require.Nil(t, err, "could not run page actions") require.Equal(t, "hello", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") }) @@ -505,7 +508,7 @@ func TestActionKeyboard(t *testing.T) { {ActionType: ActionTypeHolder{ActionType: ActionKeyboard}, Data: map[string]string{"keys": "Test2"}}, } - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) { require.Nil(t, err, "could not run page actions") el := page.Page().MustElement("input") require.Equal(t, "Test2", el.MustText(), "could not get input change value") @@ -529,7 +532,7 @@ func TestActionSleep(t *testing.T) { {ActionType: ActionTypeHolder{ActionType: ActionSleep}, Data: map[string]string{"duration": "2"}}, } - testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) { require.Nil(t, err, "could not run page actions") require.True(t, page.Page().MustElement("button").MustVisible(), "could not get button") }) @@ -553,7 +556,7 @@ func TestActionWaitVisible(t *testing.T) { } t.Run("wait for an element being visible", func(t *testing.T) { - testHeadlessSimpleResponse(t, response, actions, 2*time.Second, func(page *Page, err error, out map[string]string) { + testHeadlessSimpleResponse(t, response, actions, 2*time.Second, func(page *Page, err error, out ActionData) { require.Nil(t, err, "could not run page actions") page.Page().MustElement("button").MustVisible() @@ -562,21 +565,82 @@ func TestActionWaitVisible(t *testing.T) { t.Run("timeout because of element not visible", func(t *testing.T) { // increased timeout from time.Second/2 to time.Second due to random fails (probably due to overhead and system) - testHeadlessSimpleResponse(t, response, actions, time.Second, func(page *Page, err error, out map[string]string) { + testHeadlessSimpleResponse(t, response, actions, time.Second, func(page *Page, err error, out ActionData) { require.Error(t, err) require.Contains(t, err.Error(), "Element did not appear in the given amount of time") }) }) } -func testHeadlessSimpleResponse(t *testing.T, response string, actions []*Action, timeout time.Duration, assert func(page *Page, pageErr error, out map[string]string)) { +func TestActionWaitDialog(t *testing.T) { + response := ` + + Nuclei Test Page + + + + + ` + + t.Run("Triggered", func(t *testing.T) { + actions := []*Action{ + { + ActionType: ActionTypeHolder{ActionType: ActionNavigate}, + Data: map[string]string{"url": "{{BaseURL}}/?script=alert%281%29"}, + }, + { + ActionType: ActionTypeHolder{ActionType: ActionWaitDialog}, + Name: "test", + }, + } + + testHeadlessSimpleResponse(t, response, actions, 1*time.Second, func(page *Page, err error, out ActionData) { + require.Nil(t, err, "could not run page actions") + + test, ok := out["test"].(bool) + require.True(t, ok, "could not assert test to bool") + require.True(t, test, "could not find test") + }) + }) + + t.Run("Invalid", func(t *testing.T) { + actions := []*Action{ + { + ActionType: ActionTypeHolder{ActionType: ActionNavigate}, + Data: map[string]string{"url": "{{BaseURL}}/?script=foo"}, + }, + { + ActionType: ActionTypeHolder{ActionType: ActionWaitDialog}, + Name: "test", + }, + } + + testHeadlessSimpleResponse(t, response, actions, 1*time.Second, func(page *Page, err error, out ActionData) { + require.Nil(t, err, "could not run page actions") + + _, ok := out["test"].(bool) + require.False(t, ok, "output assertion is success") + }) + }) +} + +func testHeadlessSimpleResponse(t *testing.T, response string, actions []*Action, timeout time.Duration, assert func(page *Page, pageErr error, out ActionData)) { t.Helper() testHeadless(t, actions, timeout, func(w http.ResponseWriter, r *http.Request) { _, _ = fmt.Fprintln(w, response) }, assert) } -func testHeadless(t *testing.T, actions []*Action, timeout time.Duration, handler func(w http.ResponseWriter, r *http.Request), assert func(page *Page, pageErr error, extractedData map[string]string)) { +func testHeadless(t *testing.T, actions []*Action, timeout time.Duration, handler func(w http.ResponseWriter, r *http.Request), assert func(page *Page, pageErr error, extractedData ActionData)) { t.Helper() lfa := getBoolFromEnv("LOCAL_FILE_ACCESS", true) diff --git a/pkg/protocols/headless/request.go b/pkg/protocols/headless/request.go index c6fd28a5de..5f9b53174e 100644 --- a/pkg/protocols/headless/request.go +++ b/pkg/protocols/headless/request.go @@ -183,7 +183,13 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p responseBody, _ = html.HTML() } - outputEvent := request.responseToDSLMap(responseBody, out["header"], out["status_code"], reqBuilder.String(), input.MetaInput.Input, navigatedURL, page.DumpHistory()) + header := out.GetOrDefault("header", "").(string) + + // NOTE(dwisiswant0): `status_code` key should be an integer type. + // Ref: https://github.com/projectdiscovery/nuclei/pull/5545#discussion_r1721291013 + statusCode := out.GetOrDefault("status_code", "").(string) + + outputEvent := request.responseToDSLMap(responseBody, header, statusCode, reqBuilder.String(), input.MetaInput.Input, navigatedURL, page.DumpHistory()) // add response fields to template context and merge templatectx variables to output event request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, outputEvent) if request.options.HasTemplateCtx(input.MetaInput) { From cb05f55251691eaff8e8d821cfa4cfc0e85cbb40 Mon Sep 17 00:00:00 2001 From: ghost Date: Mon, 2 Sep 2024 10:01:02 +0000 Subject: [PATCH 33/38] Auto Generate Syntax Docs + JSONSchema [Mon Sep 2 10:01:02 UTC 2024] :robot: --- SYNTAX-REFERENCE.md | 2 ++ nuclei-jsonschema.json | 1 + pkg/templates/templates_doc.go | 1 + 3 files changed, 4 insertions(+) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 28bb5bdf24..c757321494 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -3682,6 +3682,8 @@ Enum Values: - waitevent + - dialog + - keyboard - debug diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index cd8f8fa0b0..993a744502 100644 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -256,6 +256,7 @@ "deleteheader", "setbody", "waitevent", + "waitdialog", "keyboard", "debug", "sleep", diff --git a/pkg/templates/templates_doc.go b/pkg/templates/templates_doc.go index 5171e2a88f..b484dad565 100644 --- a/pkg/templates/templates_doc.go +++ b/pkg/templates/templates_doc.go @@ -1612,6 +1612,7 @@ func init() { "deleteheader", "setbody", "waitevent", + "dialog", "keyboard", "debug", "sleep", From 0c4c0af4e9a6612d8dae300b08b58efdaf25f8a4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 15:36:39 +0530 Subject: [PATCH 34/38] chore(deps): bump github.com/projectdiscovery/dsl from 0.1.8 to 0.2.1 (#5587) Bumps [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) from 0.1.8 to 0.2.1. - [Release notes](https://github.com/projectdiscovery/dsl/releases) - [Commits](https://github.com/projectdiscovery/dsl/compare/v0.1.8...v0.2.1) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/dsl dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7b3b596b96..075c1f0f76 100644 --- a/go.mod +++ b/go.mod @@ -81,7 +81,7 @@ require ( github.com/microsoft/go-mssqldb v1.6.0 github.com/ory/dockertest/v3 v3.10.0 github.com/praetorian-inc/fingerprintx v1.1.9 - github.com/projectdiscovery/dsl v0.1.8 + github.com/projectdiscovery/dsl v0.2.1 github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb github.com/projectdiscovery/goflags v0.1.64 diff --git a/go.sum b/go.sum index 2fedffa27a..fed643d2ab 100644 --- a/go.sum +++ b/go.sum @@ -832,8 +832,8 @@ github.com/projectdiscovery/cdncheck v1.1.0 h1:qDITidmJsejzpk3rMkauCh6sjI2GH9hW/ github.com/projectdiscovery/cdncheck v1.1.0/go.mod h1:sZ8U4MjHSsyaTVjBbYWHT1cwUVvUYwDX1W+WvWRicIc= github.com/projectdiscovery/clistats v0.1.0 h1:b+LF1w0xhNd7cneKWMXb+/yUmF1n5szawxP4XGpmbxs= github.com/projectdiscovery/clistats v0.1.0/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= -github.com/projectdiscovery/dsl v0.1.8 h1:ulz+o097XsYgqP4QXaAQhVSkoeD2ZPWd29wX3CTodUA= -github.com/projectdiscovery/dsl v0.1.8/go.mod h1:AYJS2WQ/q0smr2v4pEJTg4DPe6k56KFKR7UFXvzNz/4= +github.com/projectdiscovery/dsl v0.2.1 h1:TK3KD4jsg4YbvY7WJqnz1QyH4AOvAwezeBFOX97Evgk= +github.com/projectdiscovery/dsl v0.2.1/go.mod h1:IRQXsmi5/g1dDZ79//A9t2vrRtxm4frRSd5t8CZVSbI= github.com/projectdiscovery/fastdialer v0.2.7 h1:x7bEf/kDyVOuG9kHfJLMhWO3h7J+6fMDgLWkGYL8EFg= github.com/projectdiscovery/fastdialer v0.2.7/go.mod h1:7/BGdaw7sCekuBh6lLZqPuoO5N0Qaizwvex/PjO+V8o= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= From 04a61597130de20e730c928f68b086dfd67fbd0d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 15:46:16 +0530 Subject: [PATCH 35/38] chore(deps): bump github.com/projectdiscovery/retryablehttp-go (#5591) Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.74 to 1.0.76. - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.74...v1.0.76) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 075c1f0f76..4a382aa7a4 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/projectdiscovery/interactsh v1.2.0 github.com/projectdiscovery/rawhttp v0.1.60 github.com/projectdiscovery/retryabledns v1.0.73 - github.com/projectdiscovery/retryablehttp-go v1.0.74 + github.com/projectdiscovery/retryablehttp-go v1.0.76 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 diff --git a/go.sum b/go.sum index fed643d2ab..69164e9995 100644 --- a/go.sum +++ b/go.sum @@ -874,8 +874,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= github.com/projectdiscovery/retryabledns v1.0.73 h1:CLqtgoI6+YVcUE2jnZPFVvwj59CHDE4HNImLf62sSuw= github.com/projectdiscovery/retryabledns v1.0.73/go.mod h1:Wo5tQHKtx6XcYYhMypRg8dr0GGnqIb4CzoTSEw2lo7I= -github.com/projectdiscovery/retryablehttp-go v1.0.74 h1:jXZsg3nNDuDjGNeL0+hiEx/dqUqPyZW1zsotWW1y70c= -github.com/projectdiscovery/retryablehttp-go v1.0.74/go.mod h1:fVRxa2fVMbomaPiOFA05kM2WrjV8wPFHW7J84eenVmQ= +github.com/projectdiscovery/retryablehttp-go v1.0.76 h1:2+4QpWnXUZFBkoeCEBXtUjSeBTdgKtaGwP8LRtKAC3g= +github.com/projectdiscovery/retryablehttp-go v1.0.76/go.mod h1:0qIv7Ld8WWSKWYXMsNAWRYU5gsxnxpBza3SjDgt6nVc= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= From 7bc257d7a57932816f0319de402f8ed8d71aa04a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 15:47:35 +0530 Subject: [PATCH 36/38] chore(deps): bump github.com/projectdiscovery/ratelimit (#5588) Bumps [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) from 0.0.49 to 0.0.53. - [Release notes](https://github.com/projectdiscovery/ratelimit/releases) - [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.49...v0.0.53) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/ratelimit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4a382aa7a4..f3b56a408c 100644 --- a/go.mod +++ b/go.mod @@ -91,7 +91,7 @@ require ( github.com/projectdiscovery/httpx v1.6.7 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 - github.com/projectdiscovery/ratelimit v0.0.49 + github.com/projectdiscovery/ratelimit v0.0.53 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.1.7 diff --git a/go.sum b/go.sum index 69164e9995..d50806fa64 100644 --- a/go.sum +++ b/go.sum @@ -866,8 +866,8 @@ github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc= github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk= github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M= -github.com/projectdiscovery/ratelimit v0.0.49 h1:PYatMp8g5OuoFsZOA90e48nLd2vB6a4Tw0FZ8h9zqkQ= -github.com/projectdiscovery/ratelimit v0.0.49/go.mod h1:Xi0LTMHg4HQlmCZFzRBIhRW6N+QW5RxQ8V/Qs+Vta4k= +github.com/projectdiscovery/ratelimit v0.0.53 h1:1ci2zaVCtJ9daDi9IXjNwpDxl2InYq375role/xqUR4= +github.com/projectdiscovery/ratelimit v0.0.53/go.mod h1:4kNYTL7xKLo1qMYHTREUCBqHyvykFF+XBWutTh9fkwg= github.com/projectdiscovery/rawhttp v0.1.60 h1:L57kAATGRuCfOp8Fs0ly3j2wlip7R5pco2RVfwqh/II= github.com/projectdiscovery/rawhttp v0.1.60/go.mod h1:M97w6lsphTHKeqLSio4Lw9K4KLO2riJMYAOqVMIZ3ck= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= From 5bb730849ce124aca3a2e7177ce80d94217c493b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 15:55:34 +0530 Subject: [PATCH 37/38] chore(deps): bump github.com/projectdiscovery/httpx from 1.6.7 to 1.6.8 (#5590) Bumps [github.com/projectdiscovery/httpx](https://github.com/projectdiscovery/httpx) from 1.6.7 to 1.6.8. - [Release notes](https://github.com/projectdiscovery/httpx/releases) - [Changelog](https://github.com/projectdiscovery/httpx/blob/main/.goreleaser.yml) - [Commits](https://github.com/projectdiscovery/httpx/compare/v1.6.7...v1.6.8) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/httpx dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index f3b56a408c..1a478943b8 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/projectdiscovery/fastdialer v0.2.7 github.com/projectdiscovery/hmap v0.0.56 github.com/projectdiscovery/interactsh v1.2.0 - github.com/projectdiscovery/rawhttp v0.1.60 + github.com/projectdiscovery/rawhttp v0.1.61 github.com/projectdiscovery/retryabledns v1.0.73 github.com/projectdiscovery/retryablehttp-go v1.0.76 github.com/projectdiscovery/yamldoc-go v1.0.4 @@ -88,7 +88,7 @@ require ( github.com/projectdiscovery/gologger v1.1.21 github.com/projectdiscovery/gostruct v0.0.2 github.com/projectdiscovery/gozero v0.0.2 - github.com/projectdiscovery/httpx v1.6.7 + github.com/projectdiscovery/httpx v1.6.8 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 github.com/projectdiscovery/ratelimit v0.0.53 diff --git a/go.sum b/go.sum index d50806fa64..031530e8a4 100644 --- a/go.sum +++ b/go.sum @@ -852,8 +852,8 @@ github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3c github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8= github.com/projectdiscovery/hmap v0.0.56 h1:QLIuFhCc7jrBH9WYpxNBDooSoH+/S8cCm7VO5rMt3lU= github.com/projectdiscovery/hmap v0.0.56/go.mod h1:XyGKMO93nLOzpaTTCDBSKEGx/XWXjSpoyZJxT92Qd/Q= -github.com/projectdiscovery/httpx v1.6.7 h1:luJAtQ+iKpiyKKDpcWw1/fF1MaRX6JZ4R3P+ol5Uikk= -github.com/projectdiscovery/httpx v1.6.7/go.mod h1:rqcuexBrb4v36ZyX0BHcHaovTurOJn2P65Tdt9cYdfE= +github.com/projectdiscovery/httpx v1.6.8 h1:k0Y5g3ue/7QbDP0+LykIxp/VhPDLfau3UEUyuxtP7qE= +github.com/projectdiscovery/httpx v1.6.8/go.mod h1:7BIsDxyRwkBjthqFmEajXrA5f3yb4tlVfLmpNdf0ZXA= github.com/projectdiscovery/interactsh v1.2.0 h1:Al6jHiR+Usl9egYJDLJaWNHOcH8Rugk8gWMasc8Cmw8= github.com/projectdiscovery/interactsh v1.2.0/go.mod h1:Wxt0fnzxsfrAZQQlpVrf3xMatP4OXZaZbjuDkIQKdYY= github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb h1:MGtI4oE12ruWv11ZlPXXd7hl/uAaQZrFvrIDYDeVMd8= @@ -868,8 +868,8 @@ github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+ github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M= github.com/projectdiscovery/ratelimit v0.0.53 h1:1ci2zaVCtJ9daDi9IXjNwpDxl2InYq375role/xqUR4= github.com/projectdiscovery/ratelimit v0.0.53/go.mod h1:4kNYTL7xKLo1qMYHTREUCBqHyvykFF+XBWutTh9fkwg= -github.com/projectdiscovery/rawhttp v0.1.60 h1:L57kAATGRuCfOp8Fs0ly3j2wlip7R5pco2RVfwqh/II= -github.com/projectdiscovery/rawhttp v0.1.60/go.mod h1:M97w6lsphTHKeqLSio4Lw9K4KLO2riJMYAOqVMIZ3ck= +github.com/projectdiscovery/rawhttp v0.1.61 h1:EbskCj6kkDSG31sO5zEUFTqHp9ltccG1DdcI+MCdahQ= +github.com/projectdiscovery/rawhttp v0.1.61/go.mod h1:5XmDAKph9pLVnh87zjL+vXDpfG5W8Gz5N2BAtYTkuVU= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= github.com/projectdiscovery/retryabledns v1.0.73 h1:CLqtgoI6+YVcUE2jnZPFVvwj59CHDE4HNImLf62sSuw= From 042b33de3dec66ab92c392e7450cc25c0b76859b Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Mon, 2 Sep 2024 17:38:48 +0530 Subject: [PATCH 38/38] version update --- pkg/catalog/config/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/catalog/config/constants.go b/pkg/catalog/config/constants.go index cd0e5673c7..ff54ce9bc0 100644 --- a/pkg/catalog/config/constants.go +++ b/pkg/catalog/config/constants.go @@ -31,7 +31,7 @@ const ( CLIConfigFileName = "config.yaml" ReportingConfigFilename = "reporting-config.yaml" // Version is the current version of nuclei - Version = `v3.3.1` + Version = `v3.3.2` // Directory Names of custom templates CustomS3TemplatesDirName = "s3" CustomGitHubTemplatesDirName = "github"