Skip to content

Commit

Permalink
Feat: add cronjob source (kubevela#42)
Browse files Browse the repository at this point in the history
* Refactor: allow the use of singleton for one Source, while also allowing non-singleton Sources, making it more extensible

Signed-off-by: Charlie Chiang <[email protected]>

* Feat: add cronjob source

Signed-off-by: Charlie Chiang <[email protected]>

* Chore: update cron lib

Signed-off-by: Charlie Chiang <[email protected]>

* Feat: do not exit if config contains invalid sources

Signed-off-by: Charlie Chiang <[email protected]>

* Chore: add tests and fix linter

Signed-off-by: Charlie Chiang <[email protected]>

* Chore: go mod tidy

Signed-off-by: Charlie Chiang <[email protected]>

* Docs: add cronjob example

Signed-off-by: Charlie Chiang <[email protected]>

* fix comments

Signed-off-by: Charlie Chiang <[email protected]>

* Chore: force lint to use the exact version

Signed-off-by: Charlie Chiang <[email protected]>

* remove unnecessary dep

Signed-off-by: Charlie Chiang <[email protected]>

* fix comments

Signed-off-by: Charlie Chiang <[email protected]>

* use codecov token

Signed-off-by: Charlie Chiang <[email protected]>

* update lint script

Signed-off-by: Charlie Chiang <[email protected]>

---------

Signed-off-by: Charlie Chiang <[email protected]>
Signed-off-by: Amit Singh <[email protected]>
  • Loading branch information
charlie0129 authored and semmet95 committed May 11, 2023
1 parent e214be5 commit 46f5975
Show file tree
Hide file tree
Showing 22 changed files with 424 additions and 62 deletions.
6 changes: 4 additions & 2 deletions .github/workflows/go-checks.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ on:

env:
GO_VERSION: '1.19'
GOLANGCI_VERSION: 'v1.49.0'
# Keep this in sync with build/lint.sh
GOLANGCI_VERSION: '1.49.0'
USE_BUILD_CONTAINER: '1'

jobs:
Expand Down Expand Up @@ -51,10 +52,11 @@ jobs:
with:
go-version: ${{ env.GO_VERSION }}

# Use this action instead of running golangci directly because it can comment on pr.
- name: Lint
uses: golangci/golangci-lint-action@v3
with:
version: ${{ env.GOLANGCI_VERSION }}
version: v${{ env.GOLANGCI_VERSION }}

- name: Check Diff
run: make checkdiff
Expand Down
4 changes: 1 addition & 3 deletions .github/workflows/unit-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ on:

env:
GO_VERSION: '1.19'
GOLANGCI_VERSION: 'v1.47.2'

jobs:
detect-noop:
Expand Down Expand Up @@ -74,8 +73,7 @@ jobs:
- name: Upload coverage report
uses: codecov/codecov-action@v3
with:
# This is a public repo, so token is not required.
# token: ${{ secrets.CODECOV_TOKEN }}
token: ${{ secrets.CODECOV_TOKEN }}
files: ./cover.out
flags: unittests
name: codecov-umbrella
Expand Down
3 changes: 3 additions & 0 deletions .golangci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ run:
- "e2e"
- "bin"

skip-dirs-use-default: false


output:
# colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
format: colored-line-number
Expand Down
59 changes: 19 additions & 40 deletions build/lint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,57 +26,36 @@ if [ -f "bin/golangci-lint" ]; then
GOLANGCI="bin/golangci-lint"
fi

function print_download_help() {
echo "You can install golangci-lint v${GOLANGCI_VERSION} by running:" 1>&2
echo " curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(pwd)/bin v${GOLANGCI_VERSION}" 1>&2
echo "By default, it will be installed in ./bin/golangci-lint so that it won't interfere with other versions (if any)." 1>&2
function print_install_help() {
echo "Automatic installation failed, you can install golangci-lint v${GOLANGCI_VERSION} manually by running:"
echo " curl -sSfL \"https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh\" | sh -s -- -b \"$(pwd)/bin\" v${GOLANGCI_VERSION}"
echo "It will be installed to \"$(pwd)/bin/golangci-lint\" so that it won't interfere with existing versions (if any)."
exit 1
}

function install_golangci() {
echo "Installing golangci-lint v${GOLANGCI_VERSION} ..."
echo "It will be installed to \"$(pwd)/bin/golangci-lint\" so that it won't interfere with existing versions (if any)."
curl -sSfL "https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh" |
sh -s -- -b "$(pwd)/bin" v${GOLANGCI_VERSION} || print_install_help
}

if ! ${GOLANGCI} version >/dev/null 2>&1; then
echo "You don't have golangci-lint installed." 2>&1
print_download_help
exit 1
install_golangci
$0 "$@"
exit
fi

CURRENT_GOLANGCI_VERSION="$(${GOLANGCI} version 2>&1)"
CURRENT_GOLANGCI_VERSION="${CURRENT_GOLANGCI_VERSION#*version }"
CURRENT_GOLANGCI_VERSION="${CURRENT_GOLANGCI_VERSION% built*}"

function greaterver() {
if [[ $1 == $2 ]]; then
return 0
fi
local IFS=.
local i ver1=($1) ver2=($2)
# fill empty fields in ver1 with zeros
for ((i = ${#ver1[@]}; i < ${#ver2[@]}; i++)); do
ver1[i]=0
done
for ((i = 0; i < ${#ver1[@]}; i++)); do
if [[ -z ${ver2[i]} ]]; then
# fill empty fields in ver2 with zeros
ver2[i]=0
fi
if ((10#${ver1[i]} > 10#${ver2[i]})); then
return 0
fi
if ((10#${ver1[i]} < 10#${ver2[i]})); then
return 2
fi
done
return 0
}

if ! greaterver "${CURRENT_GOLANGCI_VERSION}" "${GOLANGCI_VERSION}"; then
echo "golangci-lint version is too low." 1>&2
echo "You have v${CURRENT_GOLANGCI_VERSION}, but we need at least v${GOLANGCI_VERSION}" 1>&2
print_download_help
exit 1
fi

if [ "${CURRENT_GOLANGCI_VERSION}" != "${GOLANGCI_VERSION}" ]; then
echo "Warning: you have golangci-lint v${CURRENT_GOLANGCI_VERSION}, but we want v${GOLANGCI_VERSION}" 1>&2
print_download_help
echo "You have golangci-lint v${CURRENT_GOLANGCI_VERSION} installed, but we want v${GOLANGCI_VERSION}" 1>&2
install_golangci
$0 "$@"
exit
fi

echo "# Running golangci-lint v${CURRENT_GOLANGCI_VERSION}..."
Expand Down
9 changes: 9 additions & 0 deletions examples/conf-cronjob.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
triggers:
- source:
type: cronjob
properties:
schedule: "* * * * *"
timeZone: "Asia/Shanghai" # Optional
filter: ""
action:
# TODO: add your action here
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@ require (
github.com/kubevela/pkg v1.8.1-0.20230411071527-ac5fa22727f7
github.com/mitchellh/hashstructure/v2 v2.0.2
github.com/pkg/errors v0.9.1
github.com/robfig/cron/v3 v3.0.1
github.com/sirupsen/logrus v1.9.0
github.com/spf13/cobra v1.6.1
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.8.0
golang.org/x/sync v0.1.0
golang.org/x/time v0.3.0
k8s.io/api v0.26.3
k8s.io/apimachinery v0.26.3
Expand Down Expand Up @@ -93,6 +93,7 @@ require (
golang.org/x/crypto v0.4.0 // indirect
golang.org/x/net v0.8.0 // indirect
golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/sys v0.6.0 // indirect
golang.org/x/term v0.6.0 // indirect
golang.org/x/text v0.8.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
github.com/protocolbuffers/txtpbfmt v0.0.0-20220428173112-74888fd59c2b h1:zd/2RNzIRkoGGMjE+YIsZ85CnDIz672JK2F3Zl4vux4=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
Expand Down
17 changes: 11 additions & 6 deletions pkg/cmd/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -159,11 +159,15 @@ func runCli(cmd *cobra.Command, args []string) error {
// Make this Source type exists.
s, ok := sourceReg.Get(w.Source.Type)
if !ok {
return fmt.Errorf("source type %s does not exist", w.Source.Type)
logger.Errorf("source type %s does not exist", w.Source.Type)
continue
}

source := s.New()
if s, ok := instances[w.Source.Type]; ok {
source = s
if s.Singleton() {
if s, ok := instances[w.Source.Type]; ok {
source = s
}
}

// Create a EventHandler
Expand All @@ -172,7 +176,8 @@ func runCli(cmd *cobra.Command, args []string) error {
// Initialize Source, with user-provided prop and event handler
err = source.Init(w.Source.Properties, eh)
if err != nil {
return errors.Wrapf(err, "failed to initialize source %s", source.Type())
logger.Errorf("failed to initialize source %s: %s", source.Type(), err)
continue
}

instances[w.Source.Type] = source
Expand All @@ -181,8 +186,8 @@ func runCli(cmd *cobra.Command, args []string) error {
for _, instance := range instances {
err := instance.Run(ctx)
if err != nil {
logger.Fatalf("source %s failed to run: %v", instance.Type(), err)
return err
logger.Errorf("source %s failed to run: %v", instance.Type(), err)
continue
}
}

Expand Down
3 changes: 2 additions & 1 deletion pkg/cmd/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,9 @@ import (
"fmt"
"time"

"github.com/kubevela/kube-trigger/pkg/executor"
"github.com/sirupsen/logrus"

"github.com/kubevela/kube-trigger/pkg/executor"
)

type option struct {
Expand Down
3 changes: 2 additions & 1 deletion pkg/config/parser_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,10 @@ limitations under the License.
package config

import (
"github.com/stretchr/testify/assert"
"reflect"
"testing"

"github.com/stretchr/testify/assert"
)

func TestNewFromFileOrDir(t *testing.T) {
Expand Down
7 changes: 4 additions & 3 deletions pkg/eventhandler/event_handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,10 @@ func NewFromConfig(ctx context.Context, cli client.Client, actionMeta v1alpha1.A
// TODO: use handler to handle
// Apply filters
context := map[string]interface{}{
"event": event,
"data": data,
"timestamp": time.Now().Format(time.RFC3339),
"sourceType": sourceType,
"event": event,
"data": data,
"timestamp": time.Now().Format(time.RFC3339),
}
kept, err := filter.ApplyFilter(ctx, context, filterMeta)
if err != nil {
Expand Down
3 changes: 1 addition & 2 deletions pkg/executor/executor.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import (
"time"

"github.com/sirupsen/logrus"
"golang.org/x/sync/syncmap"
"golang.org/x/time/rate"
"k8s.io/client-go/util/workqueue"
)
Expand Down Expand Up @@ -89,7 +88,7 @@ func New(c Config) (*Executor, error) {
e.maxRetries = c.MaxJobRetries
e.allowRetries = c.RetryJobAfterFailure
e.wg = sync.WaitGroup{}
e.runningJobs = syncmap.Map{}
e.runningJobs = sync.Map{}
// Create a rate limited queue, with a token bucket for overall limiting,
// and exponential failure for per-item limiting.
e.queue = workqueue.NewRateLimitingQueue(
Expand Down
54 changes: 54 additions & 0 deletions pkg/source/builtin/cronjob/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
Copyright 2023 The KubeVela Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cronjob

import (
"fmt"
"strings"
)

// Config is the config for CronJob.
type Config struct {
Schedule string `json:"schedule"`
TimeZone string `json:"timeZone"`
}

func (c *Config) String() string {
// When TZ is set in schedule, ignore timeZone, just use schedule as is.
// This is not the intended use case, but we want to support it.
if strings.Contains(c.Schedule, "TZ") {
return c.Schedule
}

if c.TimeZone != "" {
// We don't check if the timezone is valid here.
// cron lib will do it.
return fmt.Sprintf("TZ=%s %s", c.TimeZone, c.Schedule)
}

return c.Schedule
}

func formatSchedule(c Config) string {
// When TZ is set in schedule, warn the user. This is not the intended use case.
// However, it should still work, so we can continue.
if strings.Contains(c.Schedule, "TZ") {
logger.Warnf("do NOT set 'TZ' in schedule, setting 'timeZone' is the preferred way. With 'TZ' set, any 'timeZone' setting will be ignored.")
}

return c.String()
}
58 changes: 58 additions & 0 deletions pkg/source/builtin/cronjob/config_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
/*
Copyright 2023 The KubeVela Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cronjob

import "testing"

func TestFormatSchedule(t *testing.T) {
tests := []struct {
name string
schedule string
timeZone string
want string
}{
{
name: "schedule_without_timezone",
schedule: "* * * * *",
timeZone: "",
want: "* * * * *",
},
{
name: "schedule_with_timezone",
schedule: "* * * * *",
timeZone: "Asia/Shanghai",
want: "TZ=Asia/Shanghai * * * * *",
},
{
name: "schedule_with_timezone_prefixed_unsupported_but_will_work",
schedule: "TZ=Asia/Shanghai * * * * *",
timeZone: "",
want: "TZ=Asia/Shanghai * * * * *",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := &Config{
Schedule: tt.schedule,
TimeZone: tt.timeZone,
}
if got := formatSchedule(*c); got != tt.want {
t.Errorf("String() = %v, want %v", got, tt.want)
}
})
}
}
Loading

0 comments on commit 46f5975

Please sign in to comment.