From cc1deb2f398cbfb936e8648d63dd544642ca875e Mon Sep 17 00:00:00 2001 From: Daniel Pacak Date: Wed, 13 May 2020 18:10:24 +0200 Subject: [PATCH] feat: Save historical reports of running kube-bench (#5) Signed-off-by: Daniel Pacak --- pkg/cmd/kube_bench.go | 11 ++- pkg/kube/workload.go | 15 +++- pkg/kubebench/converter.go | 48 +++++++++++ pkg/kubebench/converter_test.go | 1 + pkg/kubebench/crd/writer.go | 141 +++++++++++++++++++++++++++----- pkg/kubebench/model.go | 38 --------- pkg/kubebench/scanner.go | 20 ++--- pkg/kubebench/writer.go | 5 +- 8 files changed, 201 insertions(+), 78 deletions(-) create mode 100644 pkg/kubebench/converter.go create mode 100644 pkg/kubebench/converter_test.go delete mode 100644 pkg/kubebench/model.go diff --git a/pkg/cmd/kube_bench.go b/pkg/cmd/kube_bench.go index 92cf44447..64c335fc3 100644 --- a/pkg/cmd/kube_bench.go +++ b/pkg/cmd/kube_bench.go @@ -1,10 +1,13 @@ package cmd import ( + "github.com/aquasecurity/starboard/pkg/ext" + starboard "github.com/aquasecurity/starboard/pkg/generated/clientset/versioned" "github.com/aquasecurity/starboard/pkg/kubebench" "github.com/aquasecurity/starboard/pkg/kubebench/crd" "github.com/spf13/cobra" "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/client-go/kubernetes" ) func GetKubeBenchCmd(cf *genericclioptions.ConfigFlags) *cobra.Command { @@ -16,19 +19,19 @@ func GetKubeBenchCmd(cf *genericclioptions.ConfigFlags) *cobra.Command { if err != nil { return } - scanner, err := kubebench.NewScanner(config) + kubernetesClientset, err := kubernetes.NewForConfig(config) if err != nil { return } - report, node, err := scanner.Scan() + report, node, err := kubebench.NewScanner(kubernetesClientset).Scan() if err != nil { return } - writer, err := crd.NewWriter(config) + starboardClientset, err := starboard.NewForConfig(config) if err != nil { return } - err = writer.Write(report, node) + err = crd.NewWriter(ext.NewSystemClock(), starboardClientset).Write(report, node) return }, } diff --git a/pkg/kube/workload.go b/pkg/kube/workload.go index 6100c542e..d44720f27 100644 --- a/pkg/kube/workload.go +++ b/pkg/kube/workload.go @@ -5,9 +5,22 @@ import ( ) const ( - LabelWorkloadKind = "starboard.workload.kind" + // Deprecated use LabelResourceKind instead, which is more generic + LabelWorkloadKind = "starboard.workload.kind" + // Deprecated use LabelResourceName instead, which is more generic LabelWorkloadName = "starboard.workload.name" LabelContainerName = "starboard.container.name" + LabelResourceKind = "starboard.resource.kind" + LabelResourceName = "starboard.resource.name" + + LabelScannerName = "starboard.scanner.name" + LabelScannerVendor = "starboard.scanner.vendor" + + LabelHistoryLatest = "starboard.history.latest" +) + +const ( + AnnotationHistoryLimit = "starboard.history.limit" ) // WorkloadKind is an enum defining the different kinds of Kubernetes workloads. diff --git a/pkg/kubebench/converter.go b/pkg/kubebench/converter.go new file mode 100644 index 000000000..fc0bc73b1 --- /dev/null +++ b/pkg/kubebench/converter.go @@ -0,0 +1,48 @@ +package kubebench + +import ( + "encoding/json" + starboard "github.com/aquasecurity/starboard/pkg/apis/aquasecurity/v1alpha1" + "github.com/aquasecurity/starboard/pkg/ext" + "io" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type Converter interface { + Convert(reader io.Reader) (report starboard.CISKubernetesBenchmarkReport, err error) +} + +var DefaultConverter Converter = &converter{ + clock: ext.NewSystemClock(), +} + +type converter struct { + clock ext.Clock +} + +func (c *converter) Convert(reader io.Reader) (report starboard.CISKubernetesBenchmarkReport, err error) { + decoder := json.NewDecoder(reader) + report = starboard.CISKubernetesBenchmarkReport{ + GeneratedAt: meta.NewTime(c.clock.Now()), + Scanner: starboard.Scanner{ + Name: "kube-bench", + Vendor: "Aqua Security", + Version: "latest", + }, + Sections: []starboard.CISKubernetesBenchmarkSection{}, + } + + for { + var section starboard.CISKubernetesBenchmarkSection + de := decoder.Decode(§ion) + if de == io.EOF { + break + } + if de != nil { + err = de + break + } + report.Sections = append(report.Sections, section) + } + return +} diff --git a/pkg/kubebench/converter_test.go b/pkg/kubebench/converter_test.go new file mode 100644 index 000000000..7cf438b57 --- /dev/null +++ b/pkg/kubebench/converter_test.go @@ -0,0 +1 @@ +package kubebench diff --git a/pkg/kubebench/crd/writer.go b/pkg/kubebench/crd/writer.go index 814a45f4f..0ec5cf3d5 100644 --- a/pkg/kubebench/crd/writer.go +++ b/pkg/kubebench/crd/writer.go @@ -1,44 +1,143 @@ package crd import ( - "errors" - "strings" + "fmt" + "strconv" - sec "github.com/aquasecurity/starboard/pkg/apis/aquasecurity/v1alpha1" - secapi "github.com/aquasecurity/starboard/pkg/generated/clientset/versioned" + "github.com/aquasecurity/starboard/pkg/ext" + "github.com/aquasecurity/starboard/pkg/kube" + + "k8s.io/klog" + + core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/utils/pointer" + + starboard "github.com/aquasecurity/starboard/pkg/apis/aquasecurity/v1alpha1" + starboardapi "github.com/aquasecurity/starboard/pkg/generated/clientset/versioned" "github.com/aquasecurity/starboard/pkg/kubebench" meta "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/rest" +) + +const ( + defaultHistoryLimit = 10 ) type writer struct { - client *secapi.Clientset + clock ext.Clock + clientset starboardapi.Interface } -func NewWriter(config *rest.Config) (w kubebench.Writer, err error) { - client, err := secapi.NewForConfig(config) +func NewWriter(clock ext.Clock, clientset starboardapi.Interface) kubebench.Writer { + return &writer{ + clock: clock, + clientset: clientset, + } +} + +func (w *writer) Write(report starboard.CISKubernetesBenchmarkReport, node *core.Node) (err error) { + reports, err := w.getReportsByNodeName(node.GetName()) if err != nil { return } - w = &writer{ - client: client, + err = w.removeHistoryLatestLabel(reports) + if err != nil { + return } - return -} - -func (w *writer) Write(report sec.CISKubernetesBenchmarkReport, node string) (err error) { - if strings.TrimSpace(node) == "" { - err = errors.New("node name must not be blank") + err = w.removeReportsWithHistoryLimitExceeded(reports) + if err != nil { return } - // TODO Check if an instance of the report with the given name already exists. - // TODO If exists just update it, create new instance otherwise - _, err = w.client.AquasecurityV1alpha1().CISKubernetesBenchmarks().Create(&sec.CISKubernetesBenchmark{ + + _, err = w.clientset.AquasecurityV1alpha1().CISKubernetesBenchmarks().Create(&starboard.CISKubernetesBenchmark{ ObjectMeta: meta.ObjectMeta{ - Name: node, - Labels: map[string]string{}, + Name: fmt.Sprintf("%s-%d", node.Name, w.clock.Now().Unix()), + Labels: map[string]string{ + kube.LabelResourceKind: "Node", // TODO Why node.Kind is nil? + kube.LabelResourceName: node.Name, + kube.LabelScannerName: "kube-bench", + kube.LabelScannerVendor: "aqua", + kube.LabelHistoryLatest: "true", + }, + Annotations: map[string]string{ + // TODO Make this history limit configurable somehow, e.g. $ starboard kube-bench --history-limit=7 + kube.AnnotationHistoryLimit: strconv.Itoa(10), + }, + OwnerReferences: []meta.OwnerReference{ + { + APIVersion: "v1", // TODO Why node.APIVersion is nil? + Kind: "Node", // TODO Why node.Kind is nil? + Name: node.Name, + UID: node.UID, + Controller: pointer.BoolPtr(false), + }, + }, }, Report: report, }) return } + +func (w *writer) getReportsByNodeName(name string) (reports []starboard.CISKubernetesBenchmark, err error) { + list, err := w.clientset.AquasecurityV1alpha1().CISKubernetesBenchmarks().List(meta.ListOptions{ + LabelSelector: labels.Set{ + kube.LabelResourceKind: "Node", + kube.LabelResourceName: name, + }.String(), + }) + if err != nil { + return + } + reports = list.Items + return +} + +func (w *writer) removeHistoryLatestLabel(reports []starboard.CISKubernetesBenchmark) (err error) { + for _, report := range reports { + if value, ok := report.Labels[kube.LabelHistoryLatest]; !ok || value != "true" { + continue + } + clone := report.DeepCopy() + delete(clone.Labels, kube.LabelHistoryLatest) + klog.V(3).Infof("Removing %s label from %s report", kube.LabelHistoryLatest, clone.Name) + _, err = w.clientset.AquasecurityV1alpha1().CISKubernetesBenchmarks().Update(clone) + if err != nil { + return + } + } + return +} + +func (w *writer) removeReportsWithHistoryLimitExceeded(reports []starboard.CISKubernetesBenchmark) (err error) { + limit := w.getHistoryLimit(reports) + diff := len(reports) - limit + if diff < 0 { + return + } + for _, r := range reports[0 : diff+1] { + klog.V(3).Infof("Removing %s report which exceeded history limit of %d", r.GetName(), limit) + err = w.clientset.AquasecurityV1alpha1().CISKubernetesBenchmarks().Delete(r.GetName(), &meta.DeleteOptions{ + GracePeriodSeconds: pointer.Int64Ptr(0), + }) + if err != nil { + return + } + } + return +} + +func (w *writer) getHistoryLimit(reports []starboard.CISKubernetesBenchmark) int { + if len(reports) == 0 { + return defaultHistoryLimit + } + latestReport := reports[len(reports)-1] + if value, ok := latestReport.Annotations[kube.AnnotationHistoryLimit]; ok { + limit, err := strconv.Atoi(value) + if err != nil { + klog.V(3).Infof("Error while parsing value %s of %s annotation", value, kube.AnnotationHistoryLimit) + return defaultHistoryLimit + } + return limit + } + return defaultHistoryLimit +} diff --git a/pkg/kubebench/model.go b/pkg/kubebench/model.go deleted file mode 100644 index 2ab8d172b..000000000 --- a/pkg/kubebench/model.go +++ /dev/null @@ -1,38 +0,0 @@ -package kubebench - -import ( - "encoding/json" - "io" - "time" - - meta "k8s.io/apimachinery/pkg/apis/meta/v1" - - sec "github.com/aquasecurity/starboard/pkg/apis/aquasecurity/v1alpha1" -) - -func CISBenchmarkReportFrom(reader io.Reader) (report sec.CISKubernetesBenchmarkReport, err error) { - decoder := json.NewDecoder(reader) - report = sec.CISKubernetesBenchmarkReport{ - GeneratedAt: meta.NewTime(time.Now()), - Scanner: sec.Scanner{ - Name: "kube-bench", - Vendor: "Aqua Security", - Version: "latest", - }, - Sections: []sec.CISKubernetesBenchmarkSection{}, - } - - for { - var section sec.CISKubernetesBenchmarkSection - de := decoder.Decode(§ion) - if de == io.EOF { - break - } - if de != nil { - err = de - break - } - report.Sections = append(report.Sections, section) - } - return -} diff --git a/pkg/kubebench/scanner.go b/pkg/kubebench/scanner.go index a7e34b443..71b09d216 100644 --- a/pkg/kubebench/scanner.go +++ b/pkg/kubebench/scanner.go @@ -5,7 +5,7 @@ import ( "k8s.io/klog" - sec "github.com/aquasecurity/starboard/pkg/apis/aquasecurity/v1alpha1" + starboard "github.com/aquasecurity/starboard/pkg/apis/aquasecurity/v1alpha1" "github.com/aquasecurity/starboard/pkg/kube" "github.com/aquasecurity/starboard/pkg/kube/pod" @@ -19,7 +19,6 @@ import ( "time" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" ) const ( @@ -34,20 +33,18 @@ var ( type Scanner struct { clientset kubernetes.Interface pods *pod.Manager + converter Converter } -func NewScanner(config *rest.Config) (*Scanner, error) { - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - return nil, err - } +func NewScanner(clientset kubernetes.Interface) *Scanner { return &Scanner{ clientset: clientset, pods: pod.NewPodManager(clientset), - }, nil + converter: DefaultConverter, + } } -func (s *Scanner) Scan() (report sec.CISKubernetesBenchmarkReport, node string, err error) { +func (s *Scanner) Scan() (report starboard.CISKubernetesBenchmarkReport, node *core.Node, err error) { // 1. Prepare descriptor for the Kubernetes Job which will run kube-bench kubeBenchJob := s.prepareKubeBenchJob() @@ -75,8 +72,6 @@ func (s *Scanner) Scan() (report sec.CISKubernetesBenchmarkReport, node string, return } - node = kubeBenchPod.Spec.NodeName - // 4. Get kube-bench JSON output from the kube-bench Pod klog.V(3).Infof("Getting logs for %s container in job: %s/%s", kubeBenchContainerName, kubeBenchJob.Namespace, kubeBenchJob.Name) @@ -90,12 +85,13 @@ func (s *Scanner) Scan() (report sec.CISKubernetesBenchmarkReport, node string, }() // 5. Parse the CISBenchmarkReport from the logs Reader - report, err = CISBenchmarkReportFrom(logsReader) + report, err = s.converter.Convert(logsReader) if err != nil { err = fmt.Errorf("parsing CIS benchmark report: %w", err) return } + node, err = s.clientset.CoreV1().Nodes().Get(kubeBenchPod.Spec.NodeName, meta.GetOptions{}) return } diff --git a/pkg/kubebench/writer.go b/pkg/kubebench/writer.go index 36f67afa3..01ba9a18e 100644 --- a/pkg/kubebench/writer.go +++ b/pkg/kubebench/writer.go @@ -1,9 +1,10 @@ package kubebench import ( - sec "github.com/aquasecurity/starboard/pkg/apis/aquasecurity/v1alpha1" + starboard "github.com/aquasecurity/starboard/pkg/apis/aquasecurity/v1alpha1" + core "k8s.io/api/core/v1" ) type Writer interface { - Write(report sec.CISKubernetesBenchmarkReport, node string) error + Write(report starboard.CISKubernetesBenchmarkReport, node *core.Node) error }