From 0252f6edad070d96819a061194f97d8f16270fda Mon Sep 17 00:00:00 2001 From: Michal Fojtik Date: Tue, 17 Apr 2018 11:38:59 +0200 Subject: [PATCH] up: copy configs into remote host --- pkg/oc/bootstrap/clusteradd/cmd.go | 3 +- .../components/registry/registry_install.go | 10 +- .../components/router/router_install.go | 15 +++ pkg/oc/bootstrap/docker/down.go | 1 + pkg/oc/bootstrap/docker/host/host.go | 71 +++++++++-- pkg/oc/bootstrap/docker/run_self_hosted.go | 112 ++++++++++++++---- pkg/oc/bootstrap/docker/up.go | 42 +++++-- 7 files changed, 204 insertions(+), 50 deletions(-) diff --git a/pkg/oc/bootstrap/clusteradd/cmd.go b/pkg/oc/bootstrap/clusteradd/cmd.go index d522257d068b..b4629d50ea8a 100644 --- a/pkg/oc/bootstrap/clusteradd/cmd.go +++ b/pkg/oc/bootstrap/clusteradd/cmd.go @@ -111,7 +111,8 @@ func NewCmdAdd(name, fullName string, out, errout io.Writer) *cobra.Command { // Start runs the start tasks ensuring that they are executed in sequence func (c *ClusterAddConfig) Run() error { componentsToInstall := []componentinstall.Component{} - installContext, err := componentinstall.NewComponentInstallContext(c.openshiftImage(), c.imageFormat(), c.BaseDir, c.ServerLogLevel) + installContext, err := componentinstall.NewComponentInstallContext(c.openshiftImage(), c.imageFormat(), c.BaseDir, + c.ServerLogLevel) if err != nil { return err } diff --git a/pkg/oc/bootstrap/clusteradd/components/registry/registry_install.go b/pkg/oc/bootstrap/clusteradd/components/registry/registry_install.go index 20624cecb072..de1dd5ec0747 100644 --- a/pkg/oc/bootstrap/clusteradd/components/registry/registry_install.go +++ b/pkg/oc/bootstrap/clusteradd/components/registry/registry_install.go @@ -2,11 +2,13 @@ package registry import ( "fmt" + "os" "path" "github.com/openshift/origin/pkg/oc/bootstrap/clusteradd/componentinstall" "github.com/openshift/origin/pkg/oc/bootstrap/clusterup/kubeapiserver" "github.com/openshift/origin/pkg/oc/bootstrap/docker/dockerhelper" + "github.com/openshift/origin/pkg/oc/bootstrap/docker/host" "github.com/openshift/origin/pkg/oc/bootstrap/docker/run" "github.com/openshift/origin/pkg/oc/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -48,7 +50,13 @@ func (r *RegistryComponentOptions) Install(dockerClient dockerhelper.Interface, return err } - masterConfigDir := path.Join(r.InstallContext.BaseDir(), kubeapiserver.KubeAPIServerDirName) + // If docker is on remote host, the base dir is different. + baseDir := r.InstallContext.BaseDir() + if len(os.Getenv("DOCKER_HOST")) > 0 { + baseDir = path.Join(host.RemoteHostOriginDir, r.InstallContext.BaseDir()) + } + + masterConfigDir := path.Join(baseDir, kubeapiserver.KubeAPIServerDirName) flags := []string{ "adm", "registry", diff --git a/pkg/oc/bootstrap/clusteradd/components/router/router_install.go b/pkg/oc/bootstrap/clusteradd/components/router/router_install.go index 2d6d385e5be1..431493177382 100644 --- a/pkg/oc/bootstrap/clusteradd/components/router/router_install.go +++ b/pkg/oc/bootstrap/clusteradd/components/router/router_install.go @@ -11,6 +11,7 @@ import ( goruntime "runtime" "github.com/golang/glog" + "github.com/openshift/origin/pkg/oc/bootstrap/docker/host" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -110,7 +111,21 @@ func (c *RouterComponentOptions) Install(dockerClient dockerhelper.Interface, lo return errors.NewError("cannot create aggregate router cert").WithCause(err) } + dockerHelper := dockerhelper.NewHelper(dockerClient) + + // This snowflake makes sure that when the Docker is on remote host, we copy the generated cert into + // the Docker host master config dir. + if len(os.Getenv("DOCKER_HOST")) > 0 { + hostHelper := host.NewHostHelper(dockerHelper, c.InstallContext.ClientImage()) + remoteMasterConfigDir := path.Join(host.RemoteHostOriginDir, c.InstallContext.BaseDir(), kubeapiserver.KubeAPIServerDirName) + if err := hostHelper.CopyToHost(masterConfigDir, remoteMasterConfigDir); err != nil { + return err + } + masterConfigDir = remoteMasterConfigDir + } + routerCertPath := masterConfigDir + "/router.pem" + flags := []string{ "adm", "router", "--host-ports=true", diff --git a/pkg/oc/bootstrap/docker/down.go b/pkg/oc/bootstrap/docker/down.go index 7cb5cab54a92..8b47d53e7c84 100644 --- a/pkg/oc/bootstrap/docker/down.go +++ b/pkg/oc/bootstrap/docker/down.go @@ -61,6 +61,7 @@ func (c *ClientStopConfig) Stop() error { if err = helper.StopAndRemoveContainer(openshift.ContainerName); err != nil { glog.V(2).Infof("Error stopping origin container: %v", err) } + names, err := helper.ListContainerNames() if err != nil { return err diff --git a/pkg/oc/bootstrap/docker/host/host.go b/pkg/oc/bootstrap/docker/host/host.go index 51d2715df232..956a9701f76c 100644 --- a/pkg/oc/bootstrap/docker/host/host.go +++ b/pkg/oc/bootstrap/docker/host/host.go @@ -2,7 +2,10 @@ package host import ( "fmt" + "path" + "github.com/docker/docker/api/types" + "github.com/golang/glog" "github.com/openshift/origin/pkg/oc/bootstrap/docker/dockerhelper" "github.com/openshift/origin/pkg/oc/bootstrap/docker/errors" "github.com/openshift/origin/pkg/oc/bootstrap/docker/run" @@ -17,24 +20,25 @@ nsenter --mount=/rootfs/proc/1/ns/mnt mkdir -p %[1]s grep -F %[1]s /rootfs/proc/1/mountinfo || nsenter --mount=/rootfs/proc/1/ns/mnt mount -o bind %[1]s %[1]s grep -F %[1]s /rootfs/proc/1/mountinfo | grep shared || nsenter --mount=/rootfs/proc/1/ns/mnt mount --make-shared %[1]s ` + + // RemoteHostOriginDir is a directory on the remote machine that runs Docker + RemoteHostOriginDir = "/var/lib/origin/cluster-up" ) // HostHelper contains methods to help check settings on a Docker host machine // using a privileged container type HostHelper struct { - client dockerhelper.Interface - runHelper *run.RunHelper - image string - volumesDir string + client dockerhelper.Interface + runHelper *run.RunHelper + image string } // NewHostHelper creates a new HostHelper -func NewHostHelper(dockerHelper *dockerhelper.Helper, image, volumesDir string) *HostHelper { +func NewHostHelper(dockerHelper *dockerhelper.Helper, image string) *HostHelper { return &HostHelper{ - runHelper: run.NewRunHelper(dockerHelper), - client: dockerHelper.Client(), - image: image, - volumesDir: volumesDir, + runHelper: run.NewRunHelper(dockerHelper), + client: dockerHelper.Client(), + image: image, } } @@ -50,10 +54,55 @@ func (h *HostHelper) CanUseNsenterMounter() (bool, error) { return err == nil && rc == 0, err } +func (h *HostHelper) CopyToHost(src, dst string) error { + containerID, err := h.runner(). + Image(h.image). + DiscardContainer(). + Privileged(). + HostPid(). + Bind("/:/rootfs:rw"). + Entrypoint("/bin/bash"). + Command("-c", fmt.Sprintf("mkdir -p /rootfs/%s && sleep infinity", dst)).Start() + if err != nil { + return err + } + defer func() { + h.client.ContainerStop(containerID, 1) + h.client.ContainerRemove(containerID, types.ContainerRemoveOptions{}) + }() + err = dockerhelper.UploadFileToContainer(h.client, containerID, src, path.Join("/rootfs", dst)) + if err != nil { + return err + } + glog.V(2).Infof("Succesfully copied %s to %s:%s", src, containerID, dst) + return nil +} + +func (h *HostHelper) CopyFromHost(src, dst string) error { + containerID, err := h.runner(). + Image(h.image). + DiscardContainer(). + Privileged(). + HostPid(). + Bind("/:/rootfs:rw"). + Entrypoint("/bin/bash"). + Command("-c", "sleep infinity").Start() + if err != nil { + return err + } + defer h.client.ContainerStop(containerID, 1) + err = dockerhelper.DownloadDirFromContainer(h.client, containerID, src, path.Join("/rootfs", dst)) + if err != nil { + return err + } + glog.V(2).Infof("Succesfully copied %s to %s:%s", src, containerID, dst) + return nil +} + // EnsureVolumeUseShareMount ensures that the host Docker VM has a shared directory that can be used // for OpenShift volumes. This is needed for Docker for Mac. -func (h *HostHelper) EnsureVolumeUseShareMount() error { - cmd := fmt.Sprintf(ensureVolumeShareCmd, h.volumesDir) +func (h *HostHelper) EnsureVolumeUseShareMount(volumesDir string) error { + cmd := fmt.Sprintf(ensureVolumeShareCmd, volumesDir) _, rc, err := h.runner(). Image(h.image). DiscardContainer(). diff --git a/pkg/oc/bootstrap/docker/run_self_hosted.go b/pkg/oc/bootstrap/docker/run_self_hosted.go index a167cd50bef9..a40dc6d9cda3 100644 --- a/pkg/oc/bootstrap/docker/run_self_hosted.go +++ b/pkg/oc/bootstrap/docker/run_self_hosted.go @@ -11,6 +11,7 @@ import ( "time" "github.com/golang/glog" + "github.com/openshift/origin/pkg/oc/bootstrap/docker/host" kruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/kubernetes/pkg/api/legacyscheme" @@ -34,12 +35,6 @@ import ( _ "github.com/openshift/origin/pkg/api/install" ) -// This is a special case for Docker for Mac where we need to have a directory inside the VM -// that we can re-mount with --make-shared flag. We can't use the host directories mounts as -// they are handled by the Docker for Mac directly (via fuse.osxfs). -// TODO: Figure out how to remove this snowflake -const NonLinuxHostVolumeDirPrefix = "/var/lib/origin/volumes" - var ( // staticPodLocations should only include those pods that *must* be run statically because they // bring up the services required to run the workload controllers. @@ -122,7 +117,7 @@ func (c *ClusterUpConfig) StartSelfHosted(out io.Writer) error { "LOGLEVEL": fmt.Sprintf("%d", c.ServerLogLevel), } - clientConfigBuilder, err := kclientcmd.LoadFromFile(filepath.Join(configDirs.masterConfigDir, "admin.kubeconfig")) + clientConfigBuilder, err := kclientcmd.LoadFromFile(filepath.Join(c.LocalDirFor(kubeapiserver.KubeAPIServerDirName), "admin.kubeconfig")) if err != nil { return err } @@ -133,6 +128,7 @@ func (c *ClusterUpConfig) StartSelfHosted(out io.Writer) error { return err } + clientConfig.Host = c.ServerIP + ":8443" // wait for the apiserver to be ready glog.Info("Waiting for the kube-apiserver to be ready.") if err := waitForHealthyKubeAPIServer(clientConfig); err != nil { @@ -204,11 +200,29 @@ type configDirs struct { nodeConfigDir string kubeDNSConfigDir string podManifestDir string + baseDir string err error } +// LocalDirFor returns a local directory path for the given component. +func (c *ClusterUpConfig) LocalDirFor(componentName string) string { + return filepath.Join(c.BaseDir, componentName) +} + +// RemoteDirFor returns a directory path on remote host +func (c *ClusterUpConfig) RemoteDirFor(componentName string) string { + return filepath.Join(host.RemoteHostOriginDir, c.BaseDir, componentName) +} + +func (c *ClusterUpConfig) copyToRemote(source, component string) (string, error) { + if err := c.hostHelper.CopyToHost(source, c.RemoteDirFor(component)); err != nil { + return "", err + } + return c.RemoteDirFor(component), nil +} + func (c *ClusterUpConfig) BuildConfig() (*configDirs, error) { - configLocations := &configDirs{ + configs := &configDirs{ masterConfigDir: filepath.Join(c.BaseDir, kubeapiserver.KubeAPIServerDirName), openshiftAPIServerConfigDir: filepath.Join(c.BaseDir, kubeapiserver.OpenShiftAPIServerDirName), openshiftControllerConfigDir: filepath.Join(c.BaseDir, kubeapiserver.OpenShiftControllerManagerDirName), @@ -217,46 +231,85 @@ func (c *ClusterUpConfig) BuildConfig() (*configDirs, error) { podManifestDir: filepath.Join(c.BaseDir, kubelet.PodManifestDirName), } - if _, err := os.Stat(configLocations.masterConfigDir); os.IsNotExist(err) { + originalMasterConfigDir := configs.masterConfigDir + originalNodeConfigDir := configs.nodeConfigDir + var err error + + if _, err := os.Stat(configs.masterConfigDir); os.IsNotExist(err) { _, err = c.makeMasterConfig() if err != nil { return nil, err } } - if _, err := os.Stat(configLocations.openshiftAPIServerConfigDir); os.IsNotExist(err) { - _, err = c.makeOpenShiftAPIServerConfig(configLocations.masterConfigDir) + if c.isRemoteDocker { + configs.masterConfigDir, err = c.copyToRemote(configs.masterConfigDir, kubeapiserver.KubeAPIServerDirName) + if err != nil { + return nil, err + } + } + + if _, err := os.Stat(configs.openshiftAPIServerConfigDir); os.IsNotExist(err) { + _, err = c.makeOpenShiftAPIServerConfig(originalMasterConfigDir) + if err != nil { + return nil, err + } + } + if c.isRemoteDocker { + configs.openshiftAPIServerConfigDir, err = c.copyToRemote(configs.openshiftAPIServerConfigDir, kubeapiserver.OpenShiftAPIServerDirName) + if err != nil { + return nil, err + } + } + + if _, err := os.Stat(configs.openshiftControllerConfigDir); os.IsNotExist(err) { + _, err = c.makeOpenShiftControllerConfig(originalMasterConfigDir) + if err != nil { + return nil, err + } + } + if c.isRemoteDocker { + configs.openshiftControllerConfigDir, err = c.copyToRemote(configs.openshiftControllerConfigDir, kubeapiserver.OpenShiftControllerManagerDirName) if err != nil { return nil, err } } - if _, err := os.Stat(configLocations.openshiftControllerConfigDir); os.IsNotExist(err) { - _, err = c.makeOpenShiftControllerConfig(configLocations.masterConfigDir) + + if _, err := os.Stat(configs.nodeConfigDir); os.IsNotExist(err) { + _, err = c.makeNodeConfig(configs.masterConfigDir) if err != nil { return nil, err } } - if _, err := os.Stat(configLocations.nodeConfigDir); os.IsNotExist(err) { - _, err = c.makeNodeConfig(configLocations.masterConfigDir) + if c.isRemoteDocker { + configs.nodeConfigDir, err = c.copyToRemote(configs.nodeConfigDir, kubelet.NodeConfigDirName) if err != nil { return nil, err } } - if _, err := os.Stat(configLocations.kubeDNSConfigDir); os.IsNotExist(err) { - _, err = c.makeKubeDNSConfig(configLocations.nodeConfigDir) + + if _, err := os.Stat(configs.kubeDNSConfigDir); os.IsNotExist(err) { + _, err = c.makeKubeDNSConfig(originalNodeConfigDir) if err != nil { return nil, err } + } - if _, err := os.Stat(configLocations.podManifestDir); os.IsNotExist(err) { - if err := os.MkdirAll(configLocations.podManifestDir, 0755); err != nil { + if c.isRemoteDocker { + configs.kubeDNSConfigDir, err = c.copyToRemote(configs.kubeDNSConfigDir, kubelet.KubeDNSDirName) + if err != nil { return nil, err } } - glog.V(2).Infof("configLocations = %#v", *configLocations) + if _, err := os.Stat(configs.podManifestDir); os.IsNotExist(err) { + if err := os.MkdirAll(configs.podManifestDir, 0755); err != nil { + return nil, err + } + + } substitutions := map[string]string{ - "/path/to/master/config-dir": configLocations.masterConfigDir, - "/path/to/openshift-apiserver/config-dir": configLocations.openshiftAPIServerConfigDir, + "/path/to/master/config-dir": configs.masterConfigDir, + "/path/to/openshift-apiserver/config-dir": configs.openshiftAPIServerConfigDir, "ETCD_VOLUME": "emptyDir:\n", } if len(c.HostDataDir) > 0 { @@ -264,14 +317,21 @@ func (c *ClusterUpConfig) BuildConfig() (*configDirs, error) { path: ` + c.HostDataDir + "\n" } - glog.V(2).Infof("Creating static pod definitions in %q", configLocations.podManifestDir) + glog.V(2).Infof("Creating static pod definitions in %q", configs.podManifestDir) for _, staticPodLocation := range staticPodLocations { - if err := staticpods.UpsertStaticPod(staticPodLocation, substitutions, configLocations.podManifestDir); err != nil { + if err := staticpods.UpsertStaticPod(staticPodLocation, substitutions, configs.podManifestDir); err != nil { + return nil, err + } + } + if c.isRemoteDocker { + configs.podManifestDir, err = c.copyToRemote(configs.podManifestDir, kubelet.PodManifestDirName) + if err != nil { return nil, err } } + glog.V(2).Infof("configLocations = %#v", *configs) - return configLocations, nil + return configs, nil } // makeMasterConfig returns the directory where a generated masterconfig lives @@ -282,7 +342,7 @@ func (c *ClusterUpConfig) makeMasterConfig() (string, error) { container.MasterImage = c.openshiftImage() container.Args = []string{ "--write-config=/var/lib/origin/openshift.local.config", - "--master=127.0.0.1", + fmt.Sprintf("--master=%s", c.ServerIP), fmt.Sprintf("--images=%s", c.imageFormat()), fmt.Sprintf("--dns=0.0.0.0:%d", c.DNSPort), fmt.Sprintf("--public-master=https://%s:8443", publicHost), diff --git a/pkg/oc/bootstrap/docker/up.go b/pkg/oc/bootstrap/docker/up.go index 5f3ea31b8c1e..c0c729744e11 100644 --- a/pkg/oc/bootstrap/docker/up.go +++ b/pkg/oc/bootstrap/docker/up.go @@ -165,6 +165,7 @@ type ClusterUpConfig struct { openshiftHelper *openshift.Helper command *cobra.Command defaultClientConfig clientcmdapi.Config + isRemoteDocker bool usingDefaultImages bool usingDefaultOpenShiftImage bool @@ -237,6 +238,8 @@ func (c *ClusterUpConfig) Complete(cmd *cobra.Command, out io.Writer) error { c.command = cmd + c.isRemoteDocker = len(os.Getenv("DOCKER_HOST")) > 0 + // do some defaulting if len(c.ImageTag) == 0 { c.ImageTag = strings.TrimRight("v"+version.Get().Major+"."+version.Get().Minor, "+") @@ -303,32 +306,49 @@ func (c *ClusterUpConfig) Complete(cmd *cobra.Command, out io.Writer) error { } if c.UseNsenterMount { + // This is default path when you run cluster up locally, with local docker daemon c.HostVolumesDir = path.Join(c.BaseDir, "openshift.local.volumes") - if err := os.MkdirAll(c.HostVolumesDir, 0755); err != nil { - return err + // This is a snowflake when Docker runs on remote host + if c.isRemoteDocker { + c.HostVolumesDir = c.RemoteDirFor("openshift.local.volumes") + } else { + if err := os.MkdirAll(c.HostVolumesDir, 0755); err != nil { + return err + } } } else { - c.HostVolumesDir = path.Join(NonLinuxHostVolumeDirPrefix, c.BaseDir, "openshift.local.volumes") + // Snowflake for OSX Docker for Mac + c.HostVolumesDir = c.RemoteDirFor("openshift.local.volumes") } + c.HostPersistentVolumesDir = path.Join(c.BaseDir, "openshift.local.pv") - if err := os.MkdirAll(c.HostPersistentVolumesDir, 0755); err != nil { - return err + if c.isRemoteDocker { + c.HostPersistentVolumesDir = c.RemoteDirFor("openshift.local.pv") + } else { + if err := os.MkdirAll(c.HostPersistentVolumesDir, 0755); err != nil { + return err + } } + c.HostDataDir = path.Join(c.BaseDir, "etcd") - if err := os.MkdirAll(c.HostDataDir, 0755); err != nil { - return err + if c.isRemoteDocker { + c.HostDataDir = c.RemoteDirFor("etcd") + } else { + if err := os.MkdirAll(c.HostDataDir, 0755); err != nil { + return err + } } // Ensure that host directories exist. // If not using the nsenter mounter, create a volume share on the host machine to // mount OpenShift volumes. - taskPrinter.StartTask("Creating host directories") if !c.UseNsenterMount { - if err := c.HostHelper().EnsureVolumeUseShareMount(); err != nil { + taskPrinter.StartTask("Creating shared mount directory on the remote host") + if err := c.HostHelper().EnsureVolumeUseShareMount(c.HostVolumesDir); err != nil { return taskPrinter.ToError(err) } + taskPrinter.Success() } - taskPrinter.Success() // Determine an IP to use for OpenShift. // The result is that c.ServerIP will be populated with @@ -909,7 +929,7 @@ func (c *ClusterUpConfig) OpenShiftHelper() *openshift.Helper { // HostHelper returns a helper object to check Host configuration func (c *ClusterUpConfig) HostHelper() *host.HostHelper { if c.hostHelper == nil { - c.hostHelper = host.NewHostHelper(c.DockerHelper(), c.openshiftImage(), c.HostVolumesDir) + c.hostHelper = host.NewHostHelper(c.DockerHelper(), c.openshiftImage()) } return c.hostHelper }