From 0319fcf00bfd8ae3e6c434df3eb6fd64bacf4293 Mon Sep 17 00:00:00 2001 From: crackcomm Date: Sun, 17 Jan 2016 11:59:56 +0100 Subject: [PATCH] bot: check for in user agent --- bot.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bot.go b/bot.go index df0b6cc..8780bf4 100644 --- a/bot.go +++ b/bot.go @@ -75,6 +75,8 @@ func (p *UserAgent) fixOther(sections []section) { } } +var botRegex = regexp.MustCompile("(?i)(bot|crawler|sp(i|y)der|search|worm|fetch|nutch)") + // Check if we're dealing with a bot or with some weird browser. If that is the // case, the receiver will be modified accordingly. func (p *UserAgent) checkBot(sections []section) { @@ -83,9 +85,8 @@ func (p *UserAgent) checkBot(sections []section) { if len(sections) == 1 && sections[0].name != "Mozilla" { p.mozilla = "" - // Check whether the name has some suspicious "bot" in his name. - reg, _ := regexp.Compile("(?i)bot") - if reg.Match([]byte(sections[0].name)) { + // Check whether the name has some suspicious "bot" or "crawler" in his name. + if botRegex.Match([]byte(sections[0].name)) { p.setSimple(sections[0].name, "", true) return }