Merge pull request #169 from divinerites/ignore-local-urls

Add support for ignoring local paths via `IgnoreInternalURLs` config param.
wjdp · Aug 16, 2021 · 2a7c4c5 · 2a7c4c5
2 parents 4a440fa + 2b38f62
commit 2a7c4c5
Show file tree

Hide file tree

Showing 7 changed files with 86 additions and 14 deletions.
diff --git a/README.md b/README.md
@@ -49,16 +49,16 @@ You can also build from sources by cloning this repo and running `sh build.sh`,
 
 ### ![fedora](https://user-images.githubusercontent.com/1690934/106691082-ce11bc00-65ca-11eb-8af5-92253e6ff7a2.png) Fedora
 
-htmltest is in the [Fedora repositories](https://src.fedoraproject.org/rpms/htmltest), note this is not a first party build and may lag the latest version. 
+htmltest is in the [Fedora repositories](https://src.fedoraproject.org/rpms/htmltest), note this is not a first party build and may lag the latest version.
 
 ```dnf install htmltest```
 
 ### :whale: Docker
 
-```docker run -v $(pwd):/test --rm wjdp/htmltest```  
+```docker run -v $(pwd):/test --rm wjdp/htmltest```
 Mount your directory with html files into the container and test them.
 
-If you need more arguments to the test run it like this:  
+If you need more arguments to the test run it like this:
 ```docker run -v $(pwd):/test --rm wjdp/htmltest -l 3 -s```
 
 ### Notes
@@ -67,7 +67,7 @@ We store temporary files in `tmp/.htmltest` by default. You probably want to ign
 
 ## :computer: Usage
 
-```
+```txt
 htmltest - Test generated HTML for problems
            https://github.com/wjdp/htmltest
 
@@ -158,6 +158,7 @@ htmltest uses a YAML configuration file. Put `.htmltest.yml` in the same directo
 | `EnforceHTML5` | Fails when the doctype isn't `<!DOCTYPE html>`. | `false` |
 | `EnforceHTTPS` | Fails when encountering an `http://` link. Useful to prevent mixed content errors when serving over HTTPS. | `false` |
 | `IgnoreURLs` | Array of regexs of URLs to ignore. | empty |
+| `IgnoreInternalURLs` | Array of strings of internal URLs to ignore. | empty |
 | `IgnoreDirs` | Array of regexs of directories to ignore when scanning for HTML files. | empty |
 | `IgnoreInternalEmptyHash` | When true prevents raising an error for links with `href="#"`. | `false` |
 | `IgnoreEmptyHref` | When true prevents raising an error for links with `href=""`. | `false` |
@@ -188,6 +189,8 @@ DirectoryPath: "_site"
 EnforceHTTPS: true
 IgnoreURLs:
 - "example.com"
+IgnoreInternalURLs:
+- "/misc/js/script.js"
 IgnoreDirs:
 - "lib"
 CacheExpires: "6h"

diff --git a/htmltest/check-link.go b/htmltest/check-link.go
@@ -3,16 +3,17 @@ package htmltest
 import (
 	"crypto/x509"
 	"fmt"
-	"github.com/badoux/checkmail"
-	"github.com/wjdp/htmltest/htmldoc"
-	"github.com/wjdp/htmltest/issues"
-	"github.com/wjdp/htmltest/output"
-	"golang.org/x/net/html"
 	"net/http"
 	"net/url"
 	"os"
 	"path"
 	"strings"
+
+	"github.com/badoux/checkmail"
+	"github.com/wjdp/htmltest/htmldoc"
+	"github.com/wjdp/htmltest/issues"
+	"github.com/wjdp/htmltest/output"
+	"golang.org/x/net/html"
 )
 
 func (hT *HTMLTest) checkLink(document *htmldoc.Document, node *html.Node) {
@@ -276,6 +277,14 @@ func (hT *HTMLTest) checkInternal(ref *htmldoc.Reference) {
 		return
 	}
 
+	// Solve #168
+	urlStr := ref.URLString()
+
+	// Does this internal url match a internal url ignore rule?
+	if hT.opts.isInternalURLIgnored(urlStr) {
+		return
+	}
+
 	// First lookup in document store,
 	refDoc, refExists := hT.documentStore.ResolveRef(ref)
 

diff --git a/htmltest/check-link_test.go b/htmltest/check-link_test.go
@@ -1,8 +1,9 @@
 package htmltest
 
 import (
-	"github.com/wjdp/htmltest/issues"
 	"testing"
+
+	"github.com/wjdp/htmltest/issues"
 )
 
 // Spec tests
@@ -492,6 +493,20 @@ func TestAnchorInternalHashWeird(t *testing.T) {
 	tExpectIssueCount(t, hT, 0)
 }
 
+func TestAnchorInternalUrl(t *testing.T) {
+	// fails for internal linking writen not in IgnoreInternalURLs (#168)
+	hT := tTestFile("fixtures/links/link_directory_internal_invalid.html")
+	tExpectIssueCount(t, hT, 1)
+	tExpectIssue(t, hT, "target does not exist", 1)
+}
+
+func TestAnchorInternalUrlOption(t *testing.T) {
+	// passes for internal linking writen in IgnoreInternalURLs option (#168)
+	hT := tTestFileOpts("fixtures/links/link_directory_internal_valid.html",
+		map[string]interface{}{"IgnoreInternalURLs": []interface{}{"/misc/js/script.js"}})
+	tExpectIssueCount(t, hT, 0)
+}
+
 func TestAnchorMultipleProblems(t *testing.T) {
 	// finds a mix of broken and unbroken links
 	t.Skip("Only single problem, and an hash which is not yet supported.")

diff --git a/htmltest/fixtures/links/link_directory_internal_invalid.html b/htmltest/fixtures/links/link_directory_internal_invalid.html
@@ -0,0 +1,9 @@
+<html>
+
+<body>
+
+  <p>Blah blah blah. <a href="misc/js/script.js">Unknown internal URL</a> </p>
+
+</body>
+
+</html>
diff --git a/htmltest/fixtures/links/link_directory_internal_valid.html b/htmltest/fixtures/links/link_directory_internal_valid.html
@@ -0,0 +1,9 @@
+<html>
+
+<body>
+
+  <p>Blah blah blah. <a href="/misc/js/script.js">Known internal URL</a> </p>
+
+</body>
+
+</html>
diff --git a/htmltest/options.go b/htmltest/options.go
@@ -38,8 +38,9 @@ type Options struct {
 	EnforceHTML5 bool
 	EnforceHTTPS bool
 
-	IgnoreURLs []interface{}
-	IgnoreDirs []interface{}
+	IgnoreURLs         []interface{}
+	IgnoreInternalURLs []interface{}
+	IgnoreDirs         []interface{}
 
 	IgnoreInternalEmptyHash             bool
 	IgnoreEmptyHref                     bool
@@ -102,8 +103,9 @@ func DefaultOptions() map[string]interface{} {
 		"EnforceHTML5": false,
 		"EnforceHTTPS": false,
 
-		"IgnoreURLs": []interface{}{},
-		"IgnoreDirs": []interface{}{},
+		"IgnoreURLs":         []interface{}{},
+		"IgnoreInternalURLs": []interface{}{},
+		"IgnoreDirs":         []interface{}{},
 
 		"IgnoreInternalEmptyHash":             false,
 		"IgnoreEmptyHref":                     false,
@@ -182,3 +184,14 @@ func (opts *Options) isURLIgnored(url string) bool {
 	}
 	return false
 }
+
+// Solve #168
+// Is the given local URL ignored by the current configuration
+func (opts *Options) isInternalURLIgnored(url string) bool {
+	for _, item := range opts.IgnoreInternalURLs {
+		if item.(string) == url {
+			return true
+		}
+	}
+	return false
+}
diff --git a/htmltest/options_test.go b/htmltest/options_test.go
@@ -59,6 +59,20 @@ func TestIsURLIgnored(t *testing.T) {
 	assert.IsFalse(t, "url left alone", hT.opts.isURLIgnored("http://assetstore.info/lib/test.js"))
 }
 
+func TestIsInternalURLIgnored(t *testing.T) {
+	userOpts := map[string]interface{}{
+		"IgnoreInternalURLs": []interface{}{"/misc/js/script.js"},
+		"NoRun":              true,
+	}
+
+	hT, err := Test(userOpts)
+	output.CheckErrorPanic(err)
+
+	assert.IsTrue(t, "url ignored", hT.opts.isInternalURLIgnored("/misc/js/script.js"))
+	assert.IsFalse(t, "url left alone", hT.opts.isInternalURLIgnored("misc/js/script.js"))
+	assert.IsFalse(t, "url left alone", hT.opts.isInternalURLIgnored("/misc/js/script"))
+}
+
 func TestMergeHTTPHeaders(t *testing.T) {
 	userOpts := map[string]interface{}{
 		"HTTPHeaders": map[interface{}]interface{}{