Skip to content

Commit

Permalink
Improve direct connections to Antrea API in antctl
Browse files Browse the repository at this point in the history 
For some commands (get featuregates, supportbundle, proxy), antctl
connects directly to the Agent / Controller API when it is run from
outside of the cluster.

We try to address some shortcomings in the implementation:

1) Antctl was giving priority to the Node's InternalIP to determine how
   to connect to the API. This doesn't work when the machine on which
   antctl runs doesn't have connectivity to the InternalIP (e.g., if I
   am running antctl on my laptop and Antrea is installed in an EKS
   cluster). To fix this issue, we instead give priority to the Node's
   ExternalIP.
2) The connections were always "insecure" (no TLS verification). To fix
   this we need to retrieve the correct CA certificate and use it in the
   client TLS config. For the Controller, the CA certificate is
   available in the kube-ssytem/antrea-ca ConfigMap, which is easy to
   retrieve. For the Agent, we have to first make sure that the
   self-signed certificate is written to disk (not just stored
   in-memory), then retrieve it using the "exec" API endpoint so that it
   can be used in the TLS config. Writing it to disk should not be a
   security issue: we already do this in the Controller, and it is
   common practice to mount certificate data in Pods.

One case that's not supported is when running antctl outside of cluster
and trying to access the Agent API for a Node where the Agent is running
as a Service. While it would be good to find a solution for this case, a
workaround is to use the `--insecure` flag for the commands mentioned
above.

Signed-off-by: Antonin Bas <[email protected]>

Secure connection to connect to Antrea API directly

Signed-off-by: Antonin Bas <[email protected]>
  • Loading branch information
@antoninbas
antoninbas committed Jun 16, 2023
1 parent eccac6b commit ff10769
Show file tree
Hide file tree
Showing 12 changed files with 558 additions and 154 deletions.
14 changes: 9 additions & 5 deletions pkg/agent/apiserver/apiserver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,10 @@ import (
antreaversion "antrea.io/antrea/pkg/version"
)

const Name = "antrea-agent-api"
const (
CertPairName = "antrea-agent-api"
CertDirectory = "/var/run/antrea/antrea-agent-tls"
)

var (
scheme = runtime.NewScheme()
Expand Down Expand Up @@ -116,7 +119,7 @@ func New(aq agentquerier.AgentQuerier,
if err != nil {
return nil, err
}
s, err := cfg.New(Name, genericapiserver.NewEmptyDelegate())
s, err := cfg.New(CertPairName, genericapiserver.NewEmptyDelegate())
if err != nil {
return nil, err
}
Expand All @@ -141,9 +144,10 @@ func newConfig(aq agentquerier.AgentQuerier,
authorization.RemoteKubeConfigFile = kubeconfig
}

// Set the PairName but leave certificate directory blank to generate in-memory by default.
secureServing.ServerCert.CertDirectory = ""
secureServing.ServerCert.PairName = Name
// Write certificate to file: if clients want to directly access the apiserver over HTTPs,
// they can retrieve the certificate first.
secureServing.ServerCert.CertDirectory = CertDirectory
secureServing.ServerCert.PairName = CertPairName

if err := secureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1"), net.IPv6loopback}); err != nil {
return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
Expand Down
24 changes: 15 additions & 9 deletions pkg/antctl/raw/featuregates/command.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ var Command *cobra.Command
var getClients = getConfigAndClients
var getRestClient = getRestClientByMode

var option = &struct {
insecure bool
}{}

func init() {
Command = &cobra.Command{
Use: "featuregates",
Expand All @@ -49,6 +53,7 @@ func init() {
Command.Long = "Print Antrea feature gates info including Controller and Agent"
} else if runtime.Mode == runtime.ModeController && !runtime.InPod {
Command.Long = "Print Antrea feature gates info including Controller and Agent"
Command.Flags().BoolVar(&option.insecure, "insecure", false, "Skip TLS verification when connecting to Antrea API.")
Command.RunE = controllerRemoteRunE
}
}
Expand All @@ -66,12 +71,13 @@ func controllerRemoteRunE(cmd *cobra.Command, _ []string) error {
}

func featureGateRequest(cmd *cobra.Command, mode string) error {
ctx := cmd.Context()
kubeconfig, k8sClientset, antreaClientset, err := getClients(cmd)
if err != nil {
return err
}

client, err := getRestClient(kubeconfig, k8sClientset, antreaClientset, mode)
client, err := getRestClient(ctx, kubeconfig, k8sClientset, antreaClientset, mode)
if err != nil {
return err
}
Expand Down Expand Up @@ -114,26 +120,26 @@ func getConfigAndClients(cmd *cobra.Command) (*rest.Config, kubernetes.Interface
return kubeconfig, k8sClientset, antreaClientset, nil
}

func getRestClientByMode(kubeconfig *rest.Config, k8sClientset kubernetes.Interface, antreaClientset antrea.Interface, mode string) (*rest.RESTClient, error) {
kubeconfig.GroupVersion = &schema.GroupVersion{Group: "", Version: ""}
restconfigTmpl := rest.CopyConfig(kubeconfig)
raw.SetupKubeconfig(restconfigTmpl)
func getRestClientByMode(ctx context.Context, kubeconfig *rest.Config, k8sClientset kubernetes.Interface, antreaClientset antrea.Interface, mode string) (*rest.RESTClient, error) {
cfg := rest.CopyConfig(kubeconfig)
cfg.GroupVersion = &schema.GroupVersion{Group: "", Version: ""}
var err error
var client *rest.RESTClient
switch mode {
case runtime.ModeAgent, runtime.ModeController:
client, err = rest.RESTClientFor(restconfigTmpl)
raw.SetupLocalKubeconfig(cfg)
client, err = rest.RESTClientFor(cfg)
case "remote":
client, err = getControllerClient(k8sClientset, antreaClientset, restconfigTmpl)
client, err = getControllerClient(ctx, k8sClientset, antreaClientset, cfg, option.insecure)
}
if err != nil {
return nil, fmt.Errorf("failed to create rest client: %w", err)
}
return client, nil
}

func getControllerClient(k8sClientset kubernetes.Interface, antreaClientset antrea.Interface, cfgTmpl *rest.Config) (*rest.RESTClient, error) {
controllerClientCfg, err := raw.CreateControllerClientCfg(k8sClientset, antreaClientset, cfgTmpl)
func getControllerClient(ctx context.Context, k8sClientset kubernetes.Interface, antreaClientset antrea.Interface, kubeconfig *rest.Config, insecure bool) (*rest.RESTClient, error) {
controllerClientCfg, err := raw.CreateControllerClientCfg(ctx, k8sClientset, antreaClientset, kubeconfig, insecure)
if err != nil {
return nil, fmt.Errorf("error when creating controller client config: %w", err)
}
Expand Down
42 changes: 3 additions & 39 deletions pkg/antctl/raw/featuregates/command_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package featuregates

import (
"bytes"
"context"
"io"
"net/http"
"os"
Expand Down Expand Up @@ -305,9 +306,9 @@ func TestGetFeatureGates(t *testing.T) {
}
}

func getFakeFunc(response []byte) func(kubeconfig *rest.Config, k8sClientset kubernetes.Interface, antreaClientset antrea.Interface, mode string) (*rest.RESTClient, error) {
func getFakeFunc(response []byte) func(ctx context.Context, kubeconfig *rest.Config, k8sClientset kubernetes.Interface, antreaClientset antrea.Interface, mode string) (*rest.RESTClient, error) {
restClient, _ := rest.RESTClientFor(clientConfig)
return func(kubeconfig *rest.Config, k8sClientset kubernetes.Interface, antreaClientset antrea.Interface, mode string) (*rest.RESTClient, error) {
return func(ctx context.Context, kubeconfig *rest.Config, k8sClientset kubernetes.Interface, antreaClientset antrea.Interface, mode string) (*rest.RESTClient, error) {
fakeHttpClient := fake.CreateHTTPClient(func(req *http.Request) (*http.Response, error) {
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewBuffer(response))}, nil
})
Expand Down Expand Up @@ -350,40 +351,3 @@ kind: Config`)
require.NoError(t, err)
assert.Equal(t, "http://192.168.1.10", newconfig.Host)
}

func TestGetRestClientByMode(t *testing.T) {
k8sClient := k8sfake.NewSimpleClientset(node1.DeepCopyObject())
tests := []struct {
name string
expectedErr string
antreaClientset antrea.Interface
mode string
}{
{
name: "get rest client by agent mode successfully",
mode: "agent",
},
{
name: "get rest client by remote mode successfully",
antreaClientset: antreafakeclient.NewSimpleClientset(controllerInfo.DeepCopyObject()),
mode: "remote",
},
{
name: "failed to get rest client by remote mode",
expectedErr: "failed to create rest client: error when creating controller client config: antreacontrollerinfos.crd.antrea.io \"antrea-controller\" not found",
antreaClientset: antreafakeclient.NewSimpleClientset(),
mode: "remote",
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := getRestClientByMode(clientConfig, k8sClient, tt.antreaClientset, tt.mode)
if tt.expectedErr == "" {
assert.NoError(t, err)
} else {
assert.EqualError(t, err, tt.expectedErr)
}
})
}
}
Loading

0 comments on commit ff10769

Please sign in to comment.