Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Do not perform explicit cluster deletion in prod e2e #3513

Merged
merged 3 commits into from
Apr 25, 2024
Merged

Do not perform explicit cluster deletion in prod e2e #3513

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
508ce32
Do not perform explicit cluster deletion in prod e2e
tsatam Apr 10, 2024
85a8435
Add more robust logging to deletion process
tsatam Apr 10, 2024
d69240c
Refactor pkg/util/cluster Delete function to declaratively define del…
tsatam Apr 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
183 changes: 99 additions & 84 deletions pkg/util/cluster/cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"context"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
"math/rand"
"net/http"
Expand Down Expand Up @@ -70,19 +71,6 @@ type Cluster struct {
vaultsClient keyvaultclient.VaultsClient
}

type errors []error

func (errs errors) Error() string {
var sb strings.Builder

for _, err := range errs {
sb.WriteString(err.Error())
sb.WriteByte('\n')
}

return sb.String()
}

func New(log *logrus.Entry, environment env.Core, ci bool) (*Cluster, error) {
if env.IsLocalDevelopmentMode() {
if err := env.ValidateVars("AZURE_FP_CLIENT_ID"); err != nil {
Expand Down Expand Up @@ -381,75 +369,32 @@ func (c *Cluster) generateSubnets() (vnetPrefix string, masterSubnet string, wor
}

func (c *Cluster) Delete(ctx context.Context, vnetResourceGroup, clusterName string) error {
var errs errors

oc, err := c.openshiftclustersv20200430.Get(ctx, vnetResourceGroup, clusterName)
if err == nil {
c.log.Print("deleting role assignments")
err = c.deleteRoleAssignments(ctx, vnetResourceGroup, *oc.OpenShiftClusterProperties.ServicePrincipalProfile.ClientID)
if err != nil {
errs = append(errs, err)
}

c.log.Print("deleting cluster")
err = c.openshiftclustersv20200430.DeleteAndWait(ctx, vnetResourceGroup, clusterName)
if err != nil {
errs = append(errs, err)
}
}

if c.ci {
_, err = c.groups.Get(ctx, vnetResourceGroup)
if err == nil {
c.log.Print("deleting resource group")
err = c.groups.DeleteAndWait(ctx, vnetResourceGroup)
if err != nil {
errs = append(errs, err)
}
}
// Only delete peering if CI=true and RP_MODE=development
if env.IsLocalDevelopmentMode() {
r, err := azure.ParseResourceID(c.ciParentVnet)
if err == nil {
err = c.ciParentVnetPeerings.DeleteAndWait(ctx, r.ResourceGroup, r.ResourceName, vnetResourceGroup+"-peer")
}
if err != nil {
errs = append(errs, err)
}
}
} else {
// Deleting the deployment does not clean up the associated resources
c.log.Info("deleting deployment")
err = c.deployments.DeleteAndWait(ctx, vnetResourceGroup, clusterName)
if err != nil {
errs = append(errs, err)
}

c.log.Info("deleting master/worker subnets")
err = c.subnets.DeleteAndWait(ctx, vnetResourceGroup, "dev-vnet", clusterName+"-master")
if err != nil {
errs = append(errs, err)
}

err = c.subnets.DeleteAndWait(ctx, vnetResourceGroup, "dev-vnet", clusterName+"-worker")
if err != nil {
errs = append(errs, err)
}

c.log.Info("deleting route table")
err = c.routetables.DeleteAndWait(ctx, vnetResourceGroup, clusterName+"-rt")
if err != nil {
errs = append(errs, err)
}
var errs []error

switch {
case c.ci && env.IsLocalDevelopmentMode(): // PR E2E
errs = append(errs,
c.deleteRoleAssignments(ctx, vnetResourceGroup, clusterName),
c.deleteCluster(ctx, vnetResourceGroup, clusterName),
c.deleteClusterResourceGroup(ctx, vnetResourceGroup),
c.deleteVnetPeerings(ctx, vnetResourceGroup),
)
case c.ci: // Prod E2E
errs = append(errs,
c.deleteRoleAssignments(ctx, vnetResourceGroup, clusterName),
c.deleteClusterResourceGroup(ctx, vnetResourceGroup),
)
default:
errs = append(errs,
c.deleteRoleAssignments(ctx, vnetResourceGroup, clusterName),
c.deleteCluster(ctx, vnetResourceGroup, clusterName),
c.deleteDeployment(ctx, vnetResourceGroup, clusterName), // Deleting the deployment does not clean up the associated resources
c.deleteVnetResources(ctx, vnetResourceGroup, "dev-vnet", clusterName),
)
}

c.log.Info("done")

if errs != nil {
return errs // https://golang.org/doc/faq#nil_error
}

return nil
return errors.Join(errs...)
}

// createCluster created new clusters, based on where it is running.
Expand Down Expand Up @@ -647,18 +592,23 @@ func (c *Cluster) fixupNSGs(ctx context.Context, vnetResourceGroup, clusterName
return nil
}

func (c *Cluster) deleteRoleAssignments(ctx context.Context, vnetResourceGroup, appID string) error {
spObjID, err := utilgraph.GetServicePrincipalIDByAppID(ctx, c.spGraphClient, appID)
func (c *Cluster) deleteRoleAssignments(ctx context.Context, vnetResourceGroup, clusterName string) error {
c.log.Print("deleting role assignments")
oc, err := c.openshiftclustersv20200430.Get(ctx, vnetResourceGroup, clusterName)
if err != nil {
return err
return fmt.Errorf("error getting cluster document: %w", err)
}
spObjID, err := utilgraph.GetServicePrincipalIDByAppID(ctx, c.spGraphClient, *oc.OpenShiftClusterProperties.ServicePrincipalProfile.ClientID)
if err != nil {
return fmt.Errorf("error getting service principal for cluster: %w", err)
}
if spObjID == nil {
return nil
}

roleAssignments, err := c.roleassignments.ListForResourceGroup(ctx, vnetResourceGroup, fmt.Sprintf("principalId eq '%s'", *spObjID))
if err != nil {
return err
return fmt.Errorf("error listing role assignments for service principal: %w", err)
}

for _, roleAssignment := range roleAssignments {
Expand All @@ -670,14 +620,79 @@ func (c *Cluster) deleteRoleAssignments(ctx context.Context, vnetResourceGroup,
c.log.Infof("deleting role assignment %s", *roleAssignment.Name)
_, err = c.roleassignments.Delete(ctx, *roleAssignment.Scope, *roleAssignment.Name)
if err != nil {
return err
return fmt.Errorf("error deleting role assignment %s: %w", *roleAssignment.Name, err)
}
}
}

return nil
}

func (c *Cluster) deleteCluster(ctx context.Context, resourceGroup, clusterName string) error {
c.log.Printf("deleting cluster %s", clusterName)
if err := c.openshiftclustersv20200430.DeleteAndWait(ctx, resourceGroup, clusterName); err != nil {
return fmt.Errorf("error deleting cluster %s: %w", clusterName, err)
}
return nil
}

func (c *Cluster) deleteClusterResourceGroup(ctx context.Context, resourceGroup string) error {
if _, err := c.groups.Get(ctx, resourceGroup); err != nil {
c.log.Printf("error getting resource group %s, skipping deletion: %v", resourceGroup, err)
return nil
}

c.log.Printf("deleting resource group %s", resourceGroup)
if err := c.groups.DeleteAndWait(ctx, resourceGroup); err != nil {
return fmt.Errorf("error deleting resource group: %w", err)
}

return nil
}

func (c *Cluster) deleteVnetPeerings(ctx context.Context, resourceGroup string) error {
r, err := azure.ParseResourceID(c.ciParentVnet)
if err == nil {
err = c.ciParentVnetPeerings.DeleteAndWait(ctx, r.ResourceGroup, r.ResourceName, resourceGroup+"-peer")
}
if err != nil {
return fmt.Errorf("error deleting vnet peerings: %w", err)
}

return nil
}

func (c *Cluster) deleteDeployment(ctx context.Context, resourceGroup, clusterName string) error {
c.log.Info("deleting deployment")
if err := c.deployments.DeleteAndWait(ctx, resourceGroup, clusterName); err != nil {
return fmt.Errorf("error deleting deployment: %w", err)
}
return nil
}

func (c *Cluster) deleteVnetResources(ctx context.Context, resourceGroup, vnetName, clusterName string) error {
var errs []error

c.log.Info("deleting master/worker subnets")
if err := c.subnets.DeleteAndWait(ctx, resourceGroup, vnetName, clusterName+"-master"); err != nil {
c.log.Errorf("error when deleting master subnet: %v", err)
errs = append(errs, err)
}

if err := c.subnets.DeleteAndWait(ctx, resourceGroup, vnetName, clusterName+"-worker"); err != nil {
c.log.Errorf("error when deleting worker subnet: %v", err)
errs = append(errs, err)
}

c.log.Info("deleting route table")
if err := c.routetables.DeleteAndWait(ctx, resourceGroup, clusterName+"-rt"); err != nil {
c.log.Errorf("error when deleting route table: %v", err)
errs = append(errs, err)
}

return errors.Join(errs...)
}

func (c *Cluster) peerSubnetsToCI(ctx context.Context, vnetResourceGroup, clusterName string) error {
cluster := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/dev-vnet", c.env.SubscriptionID(), vnetResourceGroup)

Expand Down
Loading