[full-ci] enhancement: add support for natural language kql date ranges

changelog/unreleased/kql-search-query-language.md

@@ -2,16 +2,31 @@ Enhancement: Keyword Query Language (KQL) search syntax
 
 We've introduced support for [KQL](https://learn.microsoft.com/en-us/sharepoint/dev/general-development/keyword-query-language-kql-syntax-reference) as the default oCIS search query language.
 
-Some examples of a valid KQL query are:
-
-* `Tag`: `tag:golden tag:"silver"`
-* `Filename`: `name:file.txt name:"file.docx"`
-* `Content`: `content:ahab content:"captain aha*"`
+Simple queries:
+
+* `tag:golden tag:"silver"`
+* `name:file.txt name:"file.docx"`
+* `content:ahab content:"captain aha*"`
+
+Date/-range queries
+
+* `Mtime:"2023-09-05T08:42:11.23554+02:00"`
+* `Mtime>"2023-09-05T08:42:11.23554+02:00"`
+* `Mtime>="2023-09-05T08:42:11.23554+02:00"`
+* `Mtime<"2023-09-05T08:42:11.23554+02:00"`
+* `Mtime<="2023-09-05T08:42:11.23554+02:00"`
+* `Mtime:today` - range: start of today till end of today
+* `Mtime:yesterday` - range: start of yesterday till end of yesterday
+* `Mtime:"this week"` - range: start of this week till end of this week
+* `Mtime:"this month"` - range: start of this month till end of this month
+* `Mtime:"last month"` - range: start of last month till end of last month
+* `Mtime:"this year"` - range: start of this year till end of this year
+* `Mtime:"last year"` - range: start of last year till end of last year
 
 Conjunctive normal form queries:
 
-* `Boolean`: `tag:golden AND tag:"silver`, `tag:golden OR tag:"silver`, `tag:golden NOT tag:"silver`
-* `Group`: `(tag:book content:ahab*)`, `tag:(book pdf)`
+* `tag:golden AND tag:"silver`, `tag:golden OR tag:"silver`, `tag:golden NOT tag:"silver`
+* `(tag:book content:ahab*)`, `tag:(book pdf)`
 
 Complex queries:
 
@@ -22,6 +37,7 @@ https://github.com/owncloud/ocis/pull/7043
 https://github.com/owncloud/ocis/pull/7247
 https://github.com/owncloud/ocis/pull/7248
 https://github.com/owncloud/ocis/pull/7254
+https://github.com/owncloud/ocis/pull/7262
 https://github.com/owncloud/web/pull/9653
 https://github.com/owncloud/web/pull/9672
 https://github.com/owncloud/ocis/issues/7042

go.mod

@@ -8,7 +8,6 @@ require (
 	github.com/Masterminds/semver v1.5.0
 	github.com/MicahParks/keyfunc v1.9.0
 	github.com/Nerzal/gocloak/v13 v13.8.0
-	github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
 	github.com/bbalet/stopwords v1.0.0
 	github.com/blevesearch/bleve/v2 v2.3.9
 	github.com/coreos/go-oidc v2.2.1+incompatible
@@ -50,6 +49,7 @@ require (
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.17.1
 	github.com/jellydator/ttlcache/v2 v2.11.1
 	github.com/jellydator/ttlcache/v3 v3.1.0
+	github.com/jinzhu/now v1.1.5
 	github.com/justinas/alice v1.2.0
 	github.com/leonelquinteros/gotext v1.5.3-0.20230317130943-71a59c05b2c1
 	github.com/libregraph/idm v0.4.1-0.20230221143410-3503963047a5

go.sum

@@ -859,8 +859,6 @@ github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4x
 github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
-github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA=
-github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
@@ -1525,6 +1523,8 @@ github.com/jellydator/ttlcache/v3 v3.1.0/go.mod h1:hi7MGFdMAwZna5n2tuvh63DvFLzVK
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
 github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE=
+github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
+github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
@@ -1642,7 +1642,6 @@ github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
 github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
@@ -1866,7 +1865,6 @@ github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qq
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/riandyrn/otelchi v0.5.1 h1:0/45omeqpP7f/cvdL16GddQBfAEmZvUyl2QzLSE6uYo=
 github.com/riandyrn/otelchi v0.5.1/go.mod h1:ZxVxNEl+jQ9uHseRYIxKWRb3OY8YXFEu+EkNiiSNUEA=
-github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.2 h1:YwD0ulJSJytLpiaWua0sBDusfsCZohxjxzVTYjwxfV8=
 github.com/rivo/uniseg v0.4.2/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -1898,7 +1896,6 @@ github.com/sacloud/libsacloud v1.36.2/go.mod h1:P7YAOVmnIn3DKHqCZcUKYUXmSwGBm3yS
 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f/go.mod h1:CJJ5VAbozOl0yEw7nHB9+7BXTJbIn6h7W+f6Gau5IP8=
 github.com/sciencemesh/meshdirectory-web v1.0.4 h1:1YSctF6PAXhoHUYCaeRTj7rHaF7b3rYrZf2R0VXBIbo=
 github.com/sciencemesh/meshdirectory-web v1.0.4/go.mod h1:fJSThTS3xf+sTdL0iXQoaQJssLI7tn7DetHMHUl4SRk=
-github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c=

services/search/pkg/query/kql/cast.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/araddon/dateparse"
+	"github.com/jinzhu/now"
 
 	"github.com/owncloud/ocis/v2/services/search/pkg/query/ast"
 )
@@ -21,21 +21,23 @@ func toNode[T ast.Node](in interface{}) (T, error) {
 
 func toNodes[T ast.Node](in interface{}) ([]T, error) {
 	switch v := in.(type) {
-	case []T:
-		return v, nil
 	case T:
 		return []T{v}, nil
+	case []T:
+		return v, nil
+	case []*ast.OperatorNode, []*ast.DateTimeNode:
+		return toNodes[T](v)
 	case []interface{}:
-		var ts []T
-		for _, inter := range v {
-			n, err := toNodes[T](inter)
+		var nodes []T
+		for _, el := range v {
+			node, err := toNodes[T](el)
 			if err != nil {
 				return nil, err
 			}
 
-			ts = append(ts, n...)
+			nodes = append(nodes, node...)
 		}
-		return ts, nil
+		return nodes, nil
 	case nil:
 		return nil, nil
 	default:
@@ -74,5 +76,53 @@ func toTime(in interface{}) (time.Time, error) {
 		return time.Time{}, err
 	}
 
-	return dateparse.ParseLocal(ts)
+	return now.Parse(ts)
+}
+
+func toTimeRange(in interface{}) (*time.Time, *time.Time, error) {
+	var from, to time.Time
+
+	value, err := toString(in)
+	if err != nil {
+		return &from, &to, UnsupportedTimeRangeError{}
+	}
+
+	c := &now.Config{
+		WeekStartDay: time.Monday,
+	}
+
+	n := c.With(timeNow())
+
+	switch value {
+	case "today":
+		from = n.BeginningOfDay()
+		to = n.EndOfDay()
+	case "yesterday":
+		yesterday := n.With(n.AddDate(0, 0, -1))
+		from = yesterday.BeginningOfDay()
+		to = yesterday.EndOfDay()
+	case "this week":
+		from = n.BeginningOfWeek()
+		to = n.EndOfWeek()
+	case "this month":
+		from = n.BeginningOfMonth()
+		to = n.EndOfMonth()
+	case "last month":
+		lastMonth := n.With(n.AddDate(0, -1, 0))
+		from = lastMonth.BeginningOfMonth()
+		to = lastMonth.EndOfMonth()
+	case "this year":
+		from = n.BeginningOfYear()
+		to = n.EndOfYear()
+	case "last year":
+		lastYear := n.With(n.AddDate(-1, 0, 0))
+		from = lastYear.BeginningOfYear()
+		to = lastYear.EndOfYear()
+	}
+
+	if from.IsZero() || to.IsZero() {
+		return nil, nil, UnsupportedTimeRangeError{}
+	}
+
+	return &from, &to, nil
 }

services/search/pkg/query/kql/dictionary.peg

@@ -61,6 +61,12 @@ DateTimeRestrictionNode <-
         FullTime
     ) '"'? {
         return buildDateTimeNode(k, o, v, c.text, c.pos)
+    } /
+    k:Char+ (
+        OperatorEqualNode /
+        OperatorColonNode
+    ) '"'? v:NaturalLanguageDateTime '"'? {
+        return buildNaturalLanguageDateTimeNodes(k, v, c.text, c.pos)
     }
 
 TextPropertyRestrictionNode <-
@@ -185,11 +191,22 @@ FullTime <-
         return c.text, nil
     }
 
-DateTime
-  = FullDate "T" FullTime {
+DateTime  <-
+    FullDate "T" FullTime {
       return c.text, nil
     }
 
+NaturalLanguageDateTime  <-
+    "today" /
+    "yesterday" /
+    "this week" /
+    "this month" /
+    "last month" /
+    "this year" /
+    "last year"  {
+        return c.text, nil
+    }
+
 ////////////////////////////////////////////////////////
 // misc
 ////////////////////////////////////////////////////////