Date Time Ops (#1110)

* Added datetime operators

* Added date subtract support

README.md

@@ -124,7 +124,6 @@ rm /etc/myfile.tmp
 ```
 
 ### Run with Docker or Podman
-
 #### Oneshot use:
 
 ```bash
@@ -194,7 +193,7 @@ Or, in your Dockerfile:
 FROM mikefarah/yq
 
 USER root
-RUN apk add bash
+RUN apk add --no-cache bash
 USER yq
 ```
 

pkg/yqlib/context.go

@@ -3,6 +3,7 @@ package yqlib
 import (
 	"container/list"
 	"fmt"
+	"time"
 
 	"github.com/jinzhu/copier"
 	logging "gopkg.in/op/go-logging.v1"
@@ -12,6 +13,7 @@ type Context struct {
 	MatchingNodes  *list.List
 	Variables      map[string]*list.List
 	DontAutoCreate bool
+	datetimeLayout string
 }
 
 func (n *Context) SingleReadonlyChildContext(candidate *CandidateNode) Context {
@@ -28,6 +30,17 @@ func (n *Context) SingleChildContext(candidate *CandidateNode) Context {
 	return n.ChildContext(list)
 }
 
+func (n *Context) SetDateTimeLayout(newDateTimeLayout string) {
+	n.datetimeLayout = newDateTimeLayout
+}
+
+func (n *Context) GetDateTimeLayout() string {
+	if n.datetimeLayout != "" {
+		return n.datetimeLayout
+	}
+	return time.RFC3339
+}
+
 func (n *Context) GetVariable(name string) *list.List {
 	if n.Variables == nil {
 		return nil
@@ -43,7 +56,7 @@ func (n *Context) SetVariable(name string, value *list.List) {
 }
 
 func (n *Context) ChildContext(results *list.List) Context {
-	clone := Context{DontAutoCreate: n.DontAutoCreate}
+	clone := Context{DontAutoCreate: n.DontAutoCreate, datetimeLayout: n.datetimeLayout}
 	clone.Variables = make(map[string]*list.List)
 	if len(n.Variables) > 0 {
 		err := copier.Copy(&clone.Variables, n.Variables)

pkg/yqlib/doc/operators/add.md

@@ -206,6 +206,38 @@ a: 4
 b: 6
 ```
 
+## Date addition
+You can add durations to dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.
+
+Given a sample.yml file of:
+```yaml
+a: 2021-01-01T00:00:00Z
+```
+then
+```bash
+yq '.a += "3h10m"' sample.yml
+```
+will output
+```yaml
+a: 2021-01-01T03:10:00Z
+```
+
+## Date addition - custom format
+You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.
+
+Given a sample.yml file of:
+```yaml
+a: Saturday, 15-Dec-01 at 2:59AM GMT
+```
+then
+```bash
+yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")' sample.yml
+```
+will output
+```yaml
+a: Saturday, 15-Dec-01 at 6:00AM GMT
+```
+
 ## Add to null
 Adding to null simply returns the rhs
 

pkg/yqlib/doc/operators/datetime.md

@@ -0,0 +1,203 @@
+# Date Time
+
+Various operators for parsing and manipulating dates. 
+
+## Date time formattings
+This uses the golangs built in time library for parsing and formatting date times.
+
+When not specified, the RFC3339 standard is assumed `2006-01-02T15:04:05Z07:00` for parsing.
+
+To specify a custom parsing format, use the `with_dtf` operator. The first parameter sets the datetime parsing format for the expression in the second parameter. The expression can be any valid `yq` expression tree.
+
+```bash
+yq 'with_dtf("myformat"; .a + "3h" | tz("Australia/Melbourne"))'
+```
+
+See https://pkg.go.dev/time#pkg-constants for examples of formatting options.
+
+
+## Timezones
+This uses golangs built in LoadLocation function to parse timezones strings. See https://pkg.go.dev/time#LoadLocation for more details.
+
+
+## Durations
+Durations are parsed using golangs built in [ParseDuration](https://pkg.go.dev/time#ParseDuration) function.
+
+You can durations to time using the `+` operator.
+
+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
+
+`yq e <exp> <file>`
+{% endhint %}
+
+## Format: from standard RFC3339 format
+Providing a single parameter assumes a standard RFC3339 datetime format. If the target format is not a valid yaml datetime format, the result will be a string tagged node.
+
+Given a sample.yml file of:
+```yaml
+a: 2001-12-15T02:59:43.1Z
+```
+then
+```bash
+yq '.a |= format_datetime("Monday, 02-Jan-06 at 3:04PM")' sample.yml
+```
+will output
+```yaml
+a: Saturday, 15-Dec-01 at 2:59AM
+```
+
+## Format: from custom date time
+Use with_dtf to set a custom datetime format for parsing.
+
+Given a sample.yml file of:
+```yaml
+a: Saturday, 15-Dec-01 at 2:59AM
+```
+then
+```bash
+yq '.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM"; format_datetime("2006-01-02"))' sample.yml
+```
+will output
+```yaml
+a: 2001-12-15
+```
+
+## Format: get the day of the week
+Given a sample.yml file of:
+```yaml
+a: 2001-12-15T02:59:43.1Z
+```
+then
+```bash
+yq '.a | format_datetime("Monday")' sample.yml
+```
+will output
+```yaml
+Saturday
+```
+
+## Now
+Given a sample.yml file of:
+```yaml
+a: cool
+```
+then
+```bash
+yq '.updated = now' sample.yml
+```
+will output
+```yaml
+a: cool
+updated: 2021-05-19T01:02:03Z
+```
+
+## Timezone: from standard RFC3339 format
+Returns a new datetime in the specified timezone. Specify standard IANA Time Zone format or 'utc', 'local'. When given a single parameter, this assumes the datetime is in RFC3339 format.
+
+Given a sample.yml file of:
+```yaml
+a: cool
+```
+then
+```bash
+yq '.updated = (now | tz("Australia/Sydney"))' sample.yml
+```
+will output
+```yaml
+a: cool
+updated: 2021-05-19T11:02:03+10:00
+```
+
+## Timezone: with custom format
+Specify standard IANA Time Zone format or 'utc', 'local'
+
+Given a sample.yml file of:
+```yaml
+a: Saturday, 15-Dec-01 at 2:59AM GMT
+```
+then
+```bash
+yq '.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))' sample.yml
+```
+will output
+```yaml
+a: Saturday, 15-Dec-01 at 1:59PM AEDT
+```
+
+## Add and tz custom format
+Specify standard IANA Time Zone format or 'utc', 'local'
+
+Given a sample.yml file of:
+```yaml
+a: Saturday, 15-Dec-01 at 2:59AM GMT
+```
+then
+```bash
+yq '.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))' sample.yml
+```
+will output
+```yaml
+a: Saturday, 15-Dec-01 at 1:59PM AEDT
+```
+
+## Date addition
+Given a sample.yml file of:
+```yaml
+a: 2021-01-01T00:00:00Z
+```
+then
+```bash
+yq '.a += "3h10m"' sample.yml
+```
+will output
+```yaml
+a: 2021-01-01T03:10:00Z
+```
+
+## Date subtraction
+You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.
+
+Given a sample.yml file of:
+```yaml
+a: 2021-01-01T03:10:00Z
+```
+then
+```bash
+yq '.a -= "3h10m"' sample.yml
+```
+will output
+```yaml
+a: 2021-01-01T00:00:00Z
+```
+
+## Date addition - custom format
+Given a sample.yml file of:
+```yaml
+a: Saturday, 15-Dec-01 at 2:59AM GMT
+```
+then
+```bash
+yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a += "3h1m")' sample.yml
+```
+will output
+```yaml
+a: Saturday, 15-Dec-01 at 6:00AM GMT
+```
+
+## Date script with custom format
+You can embed full expressions in with_dtf if needed.
+
+Given a sample.yml file of:
+```yaml
+a: Saturday, 15-Dec-01 at 2:59AM GMT
+```
+then
+```bash
+yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a = (.a + "3h1m" | tz("Australia/Perth")))' sample.yml
+```
+will output
+```yaml
+a: Saturday, 15-Dec-01 at 2:00PM AWST
+```
+

pkg/yqlib/doc/operators/headers/datetime.md

@@ -0,0 +1,26 @@
+# Date Time
+
+Various operators for parsing and manipulating dates. 
+
+## Date time formattings
+This uses the golangs built in time library for parsing and formatting date times.
+
+When not specified, the RFC3339 standard is assumed `2006-01-02T15:04:05Z07:00` for parsing.
+
+To specify a custom parsing format, use the `with_dtf` operator. The first parameter sets the datetime parsing format for the expression in the second parameter. The expression can be any valid `yq` expression tree.
+
+```bash
+yq 'with_dtf("myformat"; .a + "3h" | tz("Australia/Melbourne"))'
+```
+
+See https://pkg.go.dev/time#pkg-constants for examples of formatting options.
+
+
+## Timezones
+This uses golangs built in LoadLocation function to parse timezones strings. See https://pkg.go.dev/time#LoadLocation for more details.
+
+
+## Durations
+Durations are parsed using golangs built in [ParseDuration](https://pkg.go.dev/time#ParseDuration) function.
+
+You can durations to time using the `+` operator.

pkg/yqlib/doc/operators/subtract.md

@@ -117,6 +117,38 @@ a: 2
 b: 4
 ```
 
+## Date subtraction
+You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.
+
+Given a sample.yml file of:
+```yaml
+a: 2021-01-01T03:10:00Z
+```
+then
+```bash
+yq '.a -= "3h10m"' sample.yml
+```
+will output
+```yaml
+a: 2021-01-01T00:00:00Z
+```
+
+## Date subtraction - custom format
+Use with_dtf to specify your datetime format. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.
+
+Given a sample.yml file of:
+```yaml
+a: Saturday, 15-Dec-01 at 6:00AM GMT
+```
+then
+```bash
+yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a -= "3h1m")' sample.yml
+```
+will output
+```yaml
+a: Saturday, 15-Dec-01 at 2:59AM GMT
+```
+
 ## Custom types: that are really numbers
 When custom tags are encountered, yq will try to decode the underlying type.
 

pkg/yqlib/expression_tokeniser.go

@@ -324,6 +324,11 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`flatten\([0-9]+\)`), flattenWithDepth())
 	lexer.Add([]byte(`flatten`), opTokenWithPrefs(flattenOpType, nil, flattenPreferences{depth: -1}))
 
+	lexer.Add([]byte(`format_datetime`), opToken(formatDateTimeOpType))
+	lexer.Add([]byte(`now`), opToken(nowOpType))
+	lexer.Add([]byte(`tz`), opToken(tzOpType))
+	lexer.Add([]byte(`with_dtf`), opToken(withDtFormatOpType))
+
 	lexer.Add([]byte(`toyaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat))
 	lexer.Add([]byte(`to_yaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat))
 

pkg/yqlib/lib.go

@@ -84,6 +84,12 @@ var collectOpType = &operationType{Type: "COLLECT", NumArgs: 1, Precedence: 50,
 var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 50, Handler: mapOperator}
 var evalOpType = &operationType{Type: "EVAL", NumArgs: 1, Precedence: 50, Handler: evalOperator}
 var mapValuesOpType = &operationType{Type: "MAP_VALUES", NumArgs: 1, Precedence: 50, Handler: mapValuesOperator}
+
+var formatDateTimeOpType = &operationType{Type: "FORMAT_DATE_TIME", NumArgs: 1, Precedence: 50, Handler: formatDateTime}
+var withDtFormatOpType = &operationType{Type: "WITH_DATE_TIME_FORMAT", NumArgs: 1, Precedence: 50, Handler: withDateTimeFormat}
+var nowOpType = &operationType{Type: "NOW", NumArgs: 0, Precedence: 50, Handler: nowOp}
+var tzOpType = &operationType{Type: "TIMEZONE", NumArgs: 1, Precedence: 50, Handler: tzOp}
+
 var encodeOpType = &operationType{Type: "ENCODE", NumArgs: 0, Precedence: 50, Handler: encodeOperator}
 var decodeOpType = &operationType{Type: "DECODE", NumArgs: 0, Precedence: 50, Handler: decodeOperator}