From 8d3b3c857369baf0add5763ae456a178b2968064 Mon Sep 17 00:00:00 2001 From: David Eads Date: Tue, 16 Jan 2018 11:50:53 -0500 Subject: [PATCH] make use of the patched kubecontroller manager in openshift --- .../start/start_kube_controller_manager.go | 162 +----------------- pkg/cmd/server/start/start_master.go | 46 ++--- 2 files changed, 32 insertions(+), 176 deletions(-) diff --git a/pkg/cmd/server/start/start_kube_controller_manager.go b/pkg/cmd/server/start/start_kube_controller_manager.go index 173b60ab917b..3443cf83d043 100644 --- a/pkg/cmd/server/start/start_kube_controller_manager.go +++ b/pkg/cmd/server/start/start_kube_controller_manager.go @@ -1,27 +1,17 @@ package start import ( - "io/ioutil" - "os" "strconv" "github.com/golang/glog" "github.com/spf13/pflag" - kapiv1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" kerrors "k8s.io/apimachinery/pkg/util/errors" - kinformers "k8s.io/client-go/informers" controllerapp "k8s.io/kubernetes/cmd/kube-controller-manager/app" controlleroptions "k8s.io/kubernetes/cmd/kube-controller-manager/app/options" - "k8s.io/kubernetes/pkg/api/legacyscheme" - "k8s.io/kubernetes/pkg/volume" _ "k8s.io/kubernetes/plugin/pkg/scheduler/algorithmprovider" - "github.com/openshift/origin/pkg/cmd/server/bootstrappolicy" cmdflags "github.com/openshift/origin/pkg/cmd/util/flags" - "k8s.io/kubernetes/pkg/apis/componentconfig" ) func kubeControllerManagerAddFlags(cmserver *controlleroptions.CMServer) func(flags *pflag.FlagSet) { @@ -30,14 +20,8 @@ func kubeControllerManagerAddFlags(cmserver *controlleroptions.CMServer) func(fl } } -func newKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout, recyclerImage string, dynamicProvisioningEnabled bool, controllerArgs map[string][]string) (*controlleroptions.CMServer, []func(), error) { +func newKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout, openshiftConfigFile string, dynamicProvisioningEnabled bool) (*controlleroptions.CMServer, error) { cmdLineArgs := map[string][]string{} - // deep-copy the input args to avoid mutation conflict. - for k, v := range controllerArgs { - cmdLineArgs[k] = append([]string{}, v...) - } - cleanupFunctions := []func(){} - if _, ok := cmdLineArgs["controllers"]; !ok { cmdLineArgs["controllers"] = []string{ "*", // start everything but the exceptions} @@ -91,158 +75,24 @@ func newKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, po if _, ok := cmdLineArgs["leader-elect-resource-lock"]; !ok { cmdLineArgs["leader-elect-resource-lock"] = []string{"configmaps"} } - - _, hostPathTemplateSet := cmdLineArgs["pv-recycler-pod-template-filepath-hostpath"] - _, nfsTemplateSet := cmdLineArgs["pv-recycler-pod-template-filepath-nfs"] - if !hostPathTemplateSet || !nfsTemplateSet { - // OpenShift uses a different default volume recycler template than - // Kubernetes. This default template is hardcoded in Kubernetes and it - // isn't possible to pass it via ControllerContext. Crate a temporary - // file with OpenShift's template and let's pretend it was set by user - // as --recycler-pod-template-filepath-hostpath and - // --pv-recycler-pod-template-filepath-nfs arguments. - // This template then needs to be deleted by caller! - templateFilename, err := createRecylerTemplate(recyclerImage) - if err != nil { - return nil, nil, err - } - - cleanupFunctions = append(cleanupFunctions, func() { - // Remove the template when it's not needed. This is called aftet - // controller is initialized - glog.V(4).Infof("Removing temporary file %s", templateFilename) - err := os.Remove(templateFilename) - if err != nil { - glog.Warningf("Failed to remove %s: %v", templateFilename, err) - } - }) - - if !hostPathTemplateSet { - cmdLineArgs["pv-recycler-pod-template-filepath-hostpath"] = []string{templateFilename} - } - if !nfsTemplateSet { - cmdLineArgs["pv-recycler-pod-template-filepath-nfs"] = []string{templateFilename} - } - } + cmdLineArgs["openshift-config"] = []string{openshiftConfigFile} // resolve arguments controllerManager := controlleroptions.NewCMServer() if err := cmdflags.Resolve(cmdLineArgs, kubeControllerManagerAddFlags(controllerManager)); len(err) > 0 { - return nil, cleanupFunctions, kerrors.NewAggregate(err) + return nil, kerrors.NewAggregate(err) } - // TODO make this configurable or discoverable. This is going to prevent us from running the stock GC controller - // IF YOU ADD ANYTHING TO THIS LIST, MAKE SURE THAT YOU UPDATE THEIR STRATEGIES TO PREVENT GC FINALIZERS - controllerManager.GCIgnoredResources = append(controllerManager.GCIgnoredResources, - // explicitly disabled from GC for now - not enough value to track them - componentconfig.GroupResource{Group: "authorization.openshift.io", Resource: "rolebindingrestrictions"}, - componentconfig.GroupResource{Group: "network.openshift.io", Resource: "clusternetworks"}, - componentconfig.GroupResource{Group: "network.openshift.io", Resource: "egressnetworkpolicies"}, - componentconfig.GroupResource{Group: "network.openshift.io", Resource: "hostsubnets"}, - componentconfig.GroupResource{Group: "network.openshift.io", Resource: "netnamespaces"}, - componentconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthclientauthorizations"}, - componentconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthclients"}, - componentconfig.GroupResource{Group: "quota.openshift.io", Resource: "clusterresourcequotas"}, - componentconfig.GroupResource{Group: "user.openshift.io", Resource: "groups"}, - componentconfig.GroupResource{Group: "user.openshift.io", Resource: "identities"}, - componentconfig.GroupResource{Group: "user.openshift.io", Resource: "users"}, - componentconfig.GroupResource{Group: "image.openshift.io", Resource: "images"}, - - // virtual resource - componentconfig.GroupResource{Group: "project.openshift.io", Resource: "projects"}, - // virtual and unwatchable resource, surfaced via rbac.authorization.k8s.io objects - componentconfig.GroupResource{Group: "authorization.openshift.io", Resource: "clusterroles"}, - componentconfig.GroupResource{Group: "authorization.openshift.io", Resource: "clusterrolebindings"}, - componentconfig.GroupResource{Group: "authorization.openshift.io", Resource: "roles"}, - componentconfig.GroupResource{Group: "authorization.openshift.io", Resource: "rolebindings"}, - // these resources contain security information in their names, and we don't need to track them - componentconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthaccesstokens"}, - componentconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthauthorizetokens"}, - // exposed already as extensions v1beta1 by other controllers - componentconfig.GroupResource{Group: "apps", Resource: "deployments"}, - // exposed as autoscaling v1 - componentconfig.GroupResource{Group: "extensions", Resource: "horizontalpodautoscalers"}, - // exposed as security.openshift.io v1 - componentconfig.GroupResource{Group: "", Resource: "securitycontextconstraints"}, - ) - - return controllerManager, cleanupFunctions, nil + return controllerManager, nil } -func createRecylerTemplate(recyclerImage string) (string, error) { - uid := int64(0) - template := volume.NewPersistentVolumeRecyclerPodTemplate() - template.Namespace = "openshift-infra" - template.Spec.ServiceAccountName = bootstrappolicy.InfraPersistentVolumeRecyclerControllerServiceAccountName - template.Spec.Containers[0].Image = recyclerImage - template.Spec.Containers[0].Command = []string{"/usr/bin/openshift-recycle"} - template.Spec.Containers[0].Args = []string{"/scrub"} - template.Spec.Containers[0].SecurityContext = &kapiv1.SecurityContext{RunAsUser: &uid} - template.Spec.Containers[0].ImagePullPolicy = kapiv1.PullIfNotPresent - - templateBytes, err := runtime.Encode(legacyscheme.Codecs.LegacyCodec(kapiv1.SchemeGroupVersion), template) - if err != nil { - return "", err - } - - f, err := ioutil.TempFile("", "openshift-recycler-template-") - if err != nil { - return "", err - } - filename := f.Name() - glog.V(4).Infof("Creating file %s with recycler templates", filename) - - _, err = f.Write(templateBytes) - if err != nil { - f.Close() - os.Remove(filename) - return "", err - } - f.Close() - return filename, nil -} - -func runEmbeddedKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout string, dynamicProvisioningEnabled bool, cmdLineArgs map[string][]string, - recyclerImage string, informers *informers) { - - // Overwrite the informers, because we have our custom generic informers for quota. - // TODO update quota to create its own informer like garbage collection or if we split this out, actually add our external types to the kube generic informer - controllerapp.InformerFactoryOverride = externalKubeInformersWithExtraGenerics{ - SharedInformerFactory: informers.GetExternalKubeInformers(), - genericResourceInformer: informers.ToGenericInformer(), - } - +func runEmbeddedKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout, openshiftConfigFile string, dynamicProvisioningEnabled bool) { // TODO we need a real identity for this. Right now it's just using the loopback connection like it used to. - controllerManager, cleanupFunctions, err := newKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout, recyclerImage, dynamicProvisioningEnabled, cmdLineArgs) - defer func() { - // Clean up any temporary files and similar stuff. - // TODO: Make sure this defer is actually called - controllerapp.Run() - // below never returns -> defer is not called. - for _, f := range cleanupFunctions { - f() - } - }() - + controllerManager, err := newKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout, openshiftConfigFile, dynamicProvisioningEnabled) if err != nil { glog.Fatal(err) } - // this does a second leader election, but doing the second leader election will allow us to move out process in - // 3.8 if we so choose. if err := controllerapp.Run(controllerManager); err != nil { glog.Fatal(err) } } - -type externalKubeInformersWithExtraGenerics struct { - kinformers.SharedInformerFactory - genericResourceInformer GenericResourceInformer -} - -func (i externalKubeInformersWithExtraGenerics) ForResource(resource schema.GroupVersionResource) (kinformers.GenericInformer, error) { - return i.genericResourceInformer.ForResource(resource) -} - -func (i externalKubeInformersWithExtraGenerics) Start(stopCh <-chan struct{}) { - i.SharedInformerFactory.Start(stopCh) - i.genericResourceInformer.Start(stopCh) -} diff --git a/pkg/cmd/server/start/start_master.go b/pkg/cmd/server/start/start_master.go index 3db88a042d9e..ce7e7093c087 100644 --- a/pkg/cmd/server/start/start_master.go +++ b/pkg/cmd/server/start/start_master.go @@ -398,19 +398,12 @@ func (m *Master) Start() error { return err } - imageTemplate := variable.NewDefaultImageTemplate() - imageTemplate.Format = m.config.ImageConfig.Format - imageTemplate.Latest = m.config.ImageConfig.Latest - recyclerImage := imageTemplate.ExpandOrDie("recycler") - // you can't double run healthz, so only do this next bit if we aren't starting the API if !m.api { - glog.Infof("Starting controllers on %s (%s)", m.config.ServingInfo.BindAddress, version.Get().String()) if len(m.config.DisabledFeatures) > 0 { glog.V(4).Infof("Disabled features: %s", strings.Join(m.config.DisabledFeatures, ", ")) } - glog.Infof("Using images from %q", imageTemplate.ExpandOrDie("")) if err := origincontrollers.RunControllerServer(m.config.ServingInfo, clientGoKubeExternal); err != nil { return err @@ -465,19 +458,32 @@ func (m *Master) Start() error { go runEmbeddedScheduler(m.config.MasterClients.OpenShiftLoopbackKubeConfig, m.config.KubernetesMasterConfig.SchedulerConfigFile, m.config.KubernetesMasterConfig.SchedulerArguments) - kubeControllerInformers, err := NewInformers(*m.config) - if err != nil { - return err - } - go runEmbeddedKubeControllerManager( - m.config.MasterClients.OpenShiftLoopbackKubeConfig, - m.config.ServiceAccountConfig.PrivateKeyFile, - m.config.ServiceAccountConfig.MasterCA, - m.config.KubernetesMasterConfig.PodEvictionTimeout, - m.config.VolumeConfig.DynamicProvisioningEnabled, - m.config.KubernetesMasterConfig.ControllerArguments, - recyclerImage, - kubeControllerInformers) + go func() { + kubeControllerConfigBytes, err := configapilatest.WriteYAML(m.config) + if err != nil { + glog.Fatal(err) + } + // this creates using 0600 + kubeControllerConfigFile, err := ioutil.TempFile("", "openshift-kube-controler-manager-config.yaml") + if err != nil { + glog.Fatal(err) + } + defer func() { + os.Remove(kubeControllerConfigFile.Name()) + }() + if err := ioutil.WriteFile(kubeControllerConfigFile.Name(), kubeControllerConfigBytes, 0644); err != nil { + glog.Fatal(err) + } + + runEmbeddedKubeControllerManager( + m.config.MasterClients.OpenShiftLoopbackKubeConfig, + m.config.ServiceAccountConfig.PrivateKeyFile, + m.config.ServiceAccountConfig.MasterCA, + m.config.KubernetesMasterConfig.PodEvictionTimeout, + kubeControllerConfigFile.Name(), + m.config.VolumeConfig.DynamicProvisioningEnabled, + ) + }() go func() { controllerPlug.WaitForStart()