Skip to content

Commit

Permalink
Modify clickhouse statistic related func for theia-manager
Browse files Browse the repository at this point in the history
Signed-off-by: Yun-Tang Hsu <[email protected]>
  • Loading branch information
yuntanghsu committed Oct 25, 2022
1 parent 1d6528d commit 4017672
Show file tree
Hide file tree
Showing 30 changed files with 1,213 additions and 686 deletions.
6 changes: 6 additions & 0 deletions build/charts/theia/templates/theia-cli/clusterrole.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,10 @@ rules:
- list
- create
- delete
- apiGroups:
- stats.theia.antrea.io
resources:
- clickhouse
verbs:
- get
{{- end }}
1 change: 1 addition & 0 deletions build/charts/theia/templates/theia-manager/service.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ metadata:
spec:
ports:
- port: {{ .Values.theiaManager.apiServer.apiPort }}
name: tcp
protocol: TCP
targetPort: theia-api-http
selector:
Expand Down
2 changes: 1 addition & 1 deletion ci/jenkins/test-vmc.sh
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,7 @@ function deliver_antrea {

${GIT_CHECKOUT_DIR}/hack/generate-manifest.sh --ch-size 100Mi --ch-monitor-threshold 0.1 > ${GIT_CHECKOUT_DIR}/build/yamls/flow-visibility.yml
${GIT_CHECKOUT_DIR}/hack/generate-manifest.sh --no-grafana --spark-operator --theia-manager > ${GIT_CHECKOUT_DIR}/build/yamls/flow-visibility-with-spark.yml
${GIT_CHECKOUT_DIR}/hack/generate-manifest.sh --no-grafana > ${GIT_CHECKOUT_DIR}/build/yamls/flow-visibility-ch-only.yml
${GIT_CHECKOUT_DIR}/hack/generate-manifest.sh --no-grafana --theia-manager > ${GIT_CHECKOUT_DIR}/build/yamls/flow-visibility-ch-only.yml

# policy/v1beta1 is deprecated in v1.21+, unavailable in v1.25+, while policy/v1 is available in v1.21+
sed -i -e "s|apiVersion: policy/v1|apiVersion: policy/v1beta1|g" ${GIT_CHECKOUT_DIR}/build/yamls/flow-visibility.yml
Expand Down
2 changes: 1 addition & 1 deletion ci/kind/test-e2e-kind.sh
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ TESTBED_CMD=$(dirname $0)"/kind-setup.sh"
YML_DIR=$(dirname $0)"/../../build/yamls"
FLOW_VISIBILITY_CMD=$(dirname $0)"/../../hack/generate-manifest.sh --ch-size 100Mi --ch-monitor-threshold 0.1"
FLOW_VISIBILITY_WITH_SPARK_CMD=$(dirname $0)"/../../hack/generate-manifest.sh --no-grafana --spark-operator --theia-manager"
FLOW_VISIBILITY_CH_ONLY_CMD=$(dirname $0)"/../../hack/generate-manifest.sh --no-grafana"
FLOW_VISIBILITY_CH_ONLY_CMD=$(dirname $0)"/../../hack/generate-manifest.sh --no-grafana --theia-manager"
CH_OPERATOR_YML=$(dirname $0)"/../../build/charts/theia/crds/clickhouse-operator-install-bundle.yaml"

make theia-linux
Expand Down
12 changes: 9 additions & 3 deletions cmd/theia-manager/theia-manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import (
crdclientset "antrea.io/theia/pkg/client/clientset/versioned"
crdinformers "antrea.io/theia/pkg/client/informers/externalversions"
"antrea.io/theia/pkg/controller/networkpolicyrecommendation"
"antrea.io/theia/pkg/controller/stats"
"antrea.io/theia/pkg/querier"
)

Expand All @@ -50,7 +51,9 @@ func createAPIServerConfig(
bindPort int,
cipherSuites []uint16,
tlsMinVersion uint16,
nprq querier.NPRecommendationQuerier) (*apiserver.Config, error) {
nprq querier.NPRecommendationQuerier,
chq querier.ClickHouseStatusQuerier,
) (*apiserver.Config, error) {
secureServing := genericoptions.NewSecureServingOptions().WithLoopback()
authentication := genericoptions.NewDelegatingAuthenticationOptions()
authorization := genericoptions.NewDelegatingAuthorizationOptions()
Expand Down Expand Up @@ -90,7 +93,8 @@ func createAPIServerConfig(
serverConfig,
client,
caCertController,
nprq), nil
nprq,
chq), nil
}

func run(o *Options) error {
Expand Down Expand Up @@ -123,6 +127,7 @@ func run(o *Options) error {
npRecommendationInformer := crdInformerFactory.Crd().V1alpha1().NetworkPolicyRecommendations()
recommendedNPInformer := crdInformerFactory.Crd().V1alpha1().RecommendedNetworkPolicies()
npRecoController := networkpolicyrecommendation.NewNPRecommendationController(crdClient, kubeClient, npRecommendationInformer, recommendedNPInformer)
clickHouseController := stats.NewStatsController(kubeClient)

cipherSuites, err := cipher.GenerateCipherSuitesList(o.config.APIServer.TLSCipherSuites)
if err != nil {
Expand All @@ -135,7 +140,8 @@ func run(o *Options) error {
o.config.APIServer.APIPort,
cipherSuites,
cipher.TLSVersionMap[o.config.APIServer.TLSMinVersion],
npRecoController)
npRecoController,
clickHouseController)
if err != nil {
return fmt.Errorf("error creating API server config: %v", err)
}
Expand Down
1 change: 1 addition & 0 deletions hack/update-codegen-dockerized.sh
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ $GOPATH/bin/informer-gen \
$GOPATH/bin/deepcopy-gen \
--input-dirs "${THEIA_PKG}/pkg/apis/intelligence/v1alpha1" \
--input-dirs "${THEIA_PKG}/pkg/apis/crd/v1alpha1" \
--input-dirs "${THEIA_PKG}/pkg/apis/stats/v1alpha1" \
-O zz_generated.deepcopy \
--go-header-file hack/boilerplate/license_header.go.txt

Expand Down
Original file line number Diff line number Diff line change
@@ -1,35 +1,28 @@
// Copyright 2022 Antrea Authors.
// Copyright 2022 Antrea Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package policyrecommendation
package install

import (
"fmt"
"strings"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"

"github.com/google/uuid"
"antrea.io/theia/pkg/apis/stats/v1alpha1"
)

func ParseRecommendationName(npName string) error {
if !strings.HasPrefix(npName, "pr-") {
return fmt.Errorf("input name %s is not a valid policy recommendation job name", npName)

}
id := npName[3:]
_, err := uuid.Parse(id)
if err != nil {
return fmt.Errorf("input name %s does not contain a valid UUID, parsing error: %v", npName, err)
}
return nil
// Install registers the API group and adds types to a scheme
func Install(scheme *runtime.Scheme) {
utilruntime.Must(v1alpha1.AddToScheme(scheme))
utilruntime.Must(scheme.SetVersionPriority(v1alpha1.SchemeGroupVersion))
}
20 changes: 20 additions & 0 deletions pkg/apis/stats/v1alpha1/doc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
// Copyright 2022 Antrea Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +k8s:openapi-gen=true
// +k8s:deepcopy-gen=package
// +k8s:defaulter-gen=TypeMeta
// +groupName=intelligence.theia.antrea.io

package v1alpha1
58 changes: 58 additions & 0 deletions pkg/apis/stats/v1alpha1/register.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
// Copyright 2022 Antrea Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v1alpha1

import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
)

const GroupName = "stats.theia.antrea.io"

var (
SchemeGroupVersion = schema.GroupVersion{
Group: GroupName,
Version: "v1alpha1"}

StatusResource = schema.GroupVersionResource{
Group: SchemeGroupVersion.Group,
Version: SchemeGroupVersion.Version,
Resource: "clickhouse"}
)

var (
SchemeBuilder runtime.SchemeBuilder
localSchemeBuilder = &SchemeBuilder
AddToScheme = localSchemeBuilder.AddToScheme
)

func init() {
localSchemeBuilder.Register(addKnownTypes)
}

func Resource(resource string) schema.GroupResource {
return SchemeGroupVersion.WithResource(resource).GroupResource()
}

func addKnownTypes(scheme *runtime.Scheme) error {
scheme.AddKnownTypes(
SchemeGroupVersion,
&ClickHouseStats{},
)

metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
return nil
}
28 changes: 28 additions & 0 deletions pkg/apis/stats/v1alpha1/types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
// Copyright 2022 Antrea Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v1alpha1

import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

// +genclient
// +genclient:nonNamespaced
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

type ClickHouseStats struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`

Result [][]string `json:"jobType,omitempty"`
}
50 changes: 50 additions & 0 deletions pkg/apis/stats/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

25 changes: 22 additions & 3 deletions pkg/apiserver/apiserver.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,11 @@ import (

intelligenceinstall "antrea.io/theia/pkg/apis/intelligence/install"
intelligence "antrea.io/theia/pkg/apis/intelligence/v1alpha1"
statsinstall "antrea.io/theia/pkg/apis/stats/install"
apistats "antrea.io/theia/pkg/apis/stats/v1alpha1"
"antrea.io/theia/pkg/apiserver/certificate"
"antrea.io/theia/pkg/apiserver/registry/intelligence/networkpolicyrecommendation"
clickhouseStatus "antrea.io/theia/pkg/apiserver/registry/stats/clickhouse"
"antrea.io/theia/pkg/querier"
)

Expand Down Expand Up @@ -60,6 +63,7 @@ var (

func init() {
intelligenceinstall.Install(scheme)
statsinstall.Install(scheme)
metav1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"})
}

Expand All @@ -68,6 +72,7 @@ type ExtraConfig struct {
k8sClient kubernetes.Interface
caCertController *certificate.CACertController
npRecommendationQuerier querier.NPRecommendationQuerier
clickHouseStatusQuerier querier.ClickHouseStatusQuerier
}

// Config defines the config for Theia manager apiserver.
Expand All @@ -80,6 +85,7 @@ type TheiaManagerAPIServer struct {
GenericAPIServer *genericapiserver.GenericAPIServer
caCertController *certificate.CACertController
NPRecommendationQuerier querier.NPRecommendationQuerier
ClickHouseStatusQuerier querier.ClickHouseStatusQuerier
}

func (s *TheiaManagerAPIServer) Run(ctx context.Context) error {
Expand All @@ -95,27 +101,38 @@ func NewConfig(
genericConfig *genericapiserver.Config,
k8sClient kubernetes.Interface,
caCertController *certificate.CACertController,
npRecommendationQuerier querier.NPRecommendationQuerier) *Config {
npRecommendationQuerier querier.NPRecommendationQuerier,
clickHouseStatusQuerier querier.ClickHouseStatusQuerier,
) *Config {
return &Config{
genericConfig: genericConfig,
extraConfig: ExtraConfig{
k8sClient: k8sClient,
caCertController: caCertController,
npRecommendationQuerier: npRecommendationQuerier,
clickHouseStatusQuerier: clickHouseStatusQuerier,
},
}
}

func installAPIGroup(s *TheiaManagerAPIServer) error {
npRecommendationStorage := networkpolicyrecommendation.NewREST(s.NPRecommendationQuerier)
clickhouseStatusStorage := clickhouseStatus.NewREST(s.ClickHouseStatusQuerier)

intelligenceGroup := genericapiserver.NewDefaultAPIGroupInfo(intelligence.GroupName, scheme, parameterCodec, Codecs)
v1alpha1Storage := map[string]rest.Storage{}
v1alpha1Storage["networkpolicyrecommendations"] = npRecommendationStorage
intelligenceGroup.VersionedResourcesStorageMap["v1alpha1"] = v1alpha1Storage

groups := []*genericapiserver.APIGroupInfo{&intelligenceGroup}
statsGroup := genericapiserver.NewDefaultAPIGroupInfo(apistats.GroupName, scheme, parameterCodec, Codecs)
statsStorage := map[string]rest.Storage{}
statsStorage["clickhouse"] = clickhouseStatusStorage
statsGroup.VersionedResourcesStorageMap["v1alpha1"] = statsStorage

groups := []*genericapiserver.APIGroupInfo{&intelligenceGroup, &statsGroup}

for _, apiGroupInfo := range groups {
klog.Info(apiGroupInfo)
if err := s.GenericAPIServer.InstallAPIGroup(apiGroupInfo); err != nil {
return err
}
Expand All @@ -132,7 +149,9 @@ func (c Config) New() (*TheiaManagerAPIServer, error) {
apiServer := &TheiaManagerAPIServer{
GenericAPIServer: s,
caCertController: c.extraConfig.caCertController,
NPRecommendationQuerier: c.extraConfig.npRecommendationQuerier}
NPRecommendationQuerier: c.extraConfig.npRecommendationQuerier,
ClickHouseStatusQuerier: c.extraConfig.clickHouseStatusQuerier,
}
if err := installAPIGroup(apiServer); err != nil {
return nil, err
}
Expand Down
Loading

0 comments on commit 4017672

Please sign in to comment.