From eb410f66053ce1402a789822484fa511f842a0c1 Mon Sep 17 00:00:00 2001 From: Lily Chung Date: Sat, 11 Apr 2020 20:37:56 -0700 Subject: [PATCH 01/10] metallb: Build and publish containers This builds and packages the MetalLB load balancer. --- platform/WORKSPACE | 3 + platform/metallb/BUILD.bazel | 61 +++++ platform/metallb/deps.bzl | 68 ++++++ platform/metallb/kubernetes.yaml.in | 356 ++++++++++++++++++++++++++++ platform/upload/BUILD.bazel | 3 + 5 files changed, 491 insertions(+) create mode 100644 platform/metallb/BUILD.bazel create mode 100644 platform/metallb/deps.bzl create mode 100644 platform/metallb/kubernetes.yaml.in diff --git a/platform/WORKSPACE b/platform/WORKSPACE index 74ec400b5..899168ce2 100644 --- a/platform/WORKSPACE +++ b/platform/WORKSPACE @@ -88,6 +88,9 @@ load("//kube-state-metrics:deps.bzl", "kube_state_metrics_dependencies") kube_state_metrics_dependencies() +load("//metallb:deps.bzl", "metallb_dependencies") +metallb_dependencies() + load("//oci-tools:deps.bzl", "oci_tools_dependencies") oci_tools_dependencies() diff --git a/platform/metallb/BUILD.bazel b/platform/metallb/BUILD.bazel new file mode 100644 index 000000000..34c6d6ba0 --- /dev/null +++ b/platform/metallb/BUILD.bazel @@ -0,0 +1,61 @@ +load("//bazel:package.bzl", "homeworld_oci") +load("//bazel:substitute.bzl", "substitute") +load("//python:resources.bzl", "py_resources") + + +# TODO: ldflags: +# -X go.universe.tf/metallb/internal/version.gitCommit={commit} +# -X go.universe.tf/metallb/internal/version.gitBranch={branch} + +homeworld_oci( + name = "controller", + bin = { + "@tf_universe_go_metallb//controller": "/usr/bin/controller", + }, + deps = [ + "//debian:debian-micro.tgz", + ], + exec = ["/usr/bin/controller"], + visibility = ["//visibility:public"], +) + +homeworld_oci( + name = "speaker", + bin = { + "@tf_universe_go_metallb//speaker": "/usr/bin/speaker", + }, + deps = [ + "//debian:debian-micro.tgz", + ], + exec = ["/usr/bin/speaker"], + visibility = ["//visibility:public"], +) + +homeworld_oci( + name = "mirror", + bin = { + "@tf_universe_go_metallb//e2etest-mirror-server": "/usr/bin/mirror", + }, + deps = [ + "//debian:debian-micro.tgz", + ], + exec = ["/usr/bin/mirror"], + visibility = ["//visibility:public"], +) + +substitute( + name = "kubernetes.yaml", + template = ":kubernetes.yaml.in", + kfs = { + "controller_digest": ":controller.ocidigest", + "speaker_digest": ":speaker.ocidigest", + "mirror_digest": ":mirror.ocidigest", + }, + visibility = ["//visibility:public"], +) + +py_resources( + name = "kubelib", + data = [":kubernetes.yaml"], + visibility = ["//visibility:public"], +) diff --git a/platform/metallb/deps.bzl b/platform/metallb/deps.bzl new file mode 100644 index 000000000..258350afc --- /dev/null +++ b/platform/metallb/deps.bzl @@ -0,0 +1,68 @@ +load("@bazel_gazelle//:deps.bzl", "go_repository") + +def metallb_dependencies(): + go_repository( + name = "tf_universe_go_metallb", + commit = "15d9ed5cce53b457a955f894aa58e7f4856f73d5", # v0.8.3 + importpath = "go.universe.tf/metallb", + ) + + go_repository( + name = "com_github_go_kit_kit", + commit = "150a65a7ec6156b4b640c1fd55f26fd3d475d656", # v0.9.0 + importpath = "github.com/go-kit/kit", + ) + + go_repository( + name = "com_github_go_logfmt_logfmt", + commit = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5", # v0.3.0 + importpath = "github.com/go-logfmt/logfmt", + ) + + go_repository( + name = "com_github_golang_groupcache", + commit = "02826c3e79038b59d737d3b1c0a1d937f71a4433", + importpath = "github.com/golang/groupcache", + ) + + go_repository( + name = "com_github_hashicorp_golang_lru", + commit = "20f1fb78b0740ba8c3cb143a61e86ba5c8669768", # v0.5.0 + importpath = "github.com/hashicorp/golang-lru", + ) + + go_repository( + name = "com_github_mdlayher_arp", + commit = "98a83c8a27177c5179d02d41ad50b0cce8e59338", + importpath = "github.com/mdlayher/arp", + ) + + go_repository( + name = "com_github_mdlayher_ethernet", + commit = "0394541c37b7f86a10e0b49492f6d4f605c34163", + importpath = "github.com/mdlayher/ethernet", + ) + + go_repository( + name = "com_github_mdlayher_ndp", + commit = "012988d57f9ae7e329f16ec2a86b37c771cb57e8", + importpath = "github.com/mdlayher/ndp", + ) + + go_repository( + name = "com_github_mdlayher_raw", + commit = "fef19f00fc18511f735e13972bc53266d5a53f8c", + importpath = "github.com/mdlayher/raw", + ) + + go_repository( + name = "com_github_mikioh_ipaddr", + commit = "d465c8ab672111787b24b8f03326449059a4aa33", + importpath = "github.com/mikioh/ipaddr", + ) + + go_repository( + name = "com_gitlab_golang_commonmark_puny", + commit = "2cd490539afe7c6fc0eda6c59ef88fa93a00ea0d", + importpath = "gitlab.com/golang-commonmark/puny", + ) diff --git a/platform/metallb/kubernetes.yaml.in b/platform/metallb/kubernetes.yaml.in new file mode 100644 index 000000000..efde73852 --- /dev/null +++ b/platform/metallb/kubernetes.yaml.in @@ -0,0 +1,356 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + app: metallb + name: metallb-system +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + labels: + app: metallb + name: controller + namespace: metallb-system +spec: + allowPrivilegeEscalation: false + allowedCapabilities: [] + allowedHostPaths: [] + defaultAddCapabilities: [] + defaultAllowPrivilegeEscalation: false + fsGroup: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + hostIPC: false + hostNetwork: false + hostPID: false + privileged: false + readOnlyRootFilesystem: true + requiredDropCapabilities: + - ALL + runAsUser: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + seLinux: + rule: RunAsAny + supplementalGroups: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + volumes: + - configMap + - secret + - emptyDir +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + labels: + app: metallb + name: speaker + namespace: metallb-system +spec: + allowPrivilegeEscalation: false + allowedCapabilities: + - NET_ADMIN + - NET_RAW + - SYS_ADMIN + allowedHostPaths: [] + defaultAddCapabilities: [] + defaultAllowPrivilegeEscalation: false + fsGroup: + rule: RunAsAny + hostIPC: false + hostNetwork: true + hostPID: false + hostPorts: + - max: 7472 + min: 7472 + privileged: true + readOnlyRootFilesystem: true + requiredDropCapabilities: + - ALL + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - configMap + - secret + - emptyDir +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: metallb + name: controller + namespace: metallb-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: metallb + name: speaker + namespace: metallb-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: metallb + name: metallb-system:controller +rules: +- apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - update +- apiGroups: + - '' + resources: + - services/status + verbs: + - update +- apiGroups: + - '' + resources: + - events + verbs: + - create + - patch +- apiGroups: + - extensions + resourceNames: + - controller + resources: + - podsecuritypolicies + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: metallb + name: metallb-system:speaker +rules: +- apiGroups: + - '' + resources: + - services + - endpoints + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - '' + resources: + - events + verbs: + - create + - patch +- apiGroups: + - extensions + resourceNames: + - speaker + resources: + - podsecuritypolicies + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app: metallb + name: config-watcher + namespace: metallb-system +rules: +- apiGroups: + - '' + resources: + - configmaps + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: metallb + name: metallb-system:controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metallb-system:controller +subjects: +- kind: ServiceAccount + name: controller + namespace: metallb-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: metallb + name: metallb-system:speaker +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metallb-system:speaker +subjects: +- kind: ServiceAccount + name: speaker + namespace: metallb-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app: metallb + name: config-watcher + namespace: metallb-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: config-watcher +subjects: +- kind: ServiceAccount + name: controller +- kind: ServiceAccount + name: speaker +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app: metallb + component: speaker + name: speaker + namespace: metallb-system +spec: + selector: + matchLabels: + app: metallb + component: speaker + template: + metadata: + annotations: + prometheus.io/port: '7472' + prometheus.io/scrape: 'true' + labels: + app: metallb + component: speaker + spec: + containers: + - args: + - /usr/bin/speaker + - --port=7472 + - --config=config + env: + - name: METALLB_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: METALLB_HOST + valueFrom: + fieldRef: + fieldPath: status.hostIP + image: homeworld.private/metallb-speaker@{speaker_digest} + imagePullPolicy: Always + name: speaker + ports: + - containerPort: 7472 + name: monitoring + resources: + limits: + cpu: 100m + memory: 100Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_ADMIN + - NET_RAW + - SYS_ADMIN + drop: + - ALL + readOnlyRootFilesystem: true + hostNetwork: true + nodeSelector: + beta.kubernetes.io/os: linux + serviceAccountName: speaker + terminationGracePeriodSeconds: 0 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: metallb + component: controller + name: controller + namespace: metallb-system +spec: + revisionHistoryLimit: 3 + selector: + matchLabels: + app: metallb + component: controller + template: + metadata: + annotations: + prometheus.io/port: '7472' + prometheus.io/scrape: 'true' + labels: + app: metallb + component: controller + spec: + containers: + - args: + - /usr/bin/controller + - --port=7472 + - --config=config + image: homeworld.private/metallb-controller@{controller_digest} + imagePullPolicy: Always + name: controller + ports: + - containerPort: 7472 + name: monitoring + resources: + limits: + cpu: 100m + memory: 100Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + readOnlyRootFilesystem: true + nodeSelector: + beta.kubernetes.io/os: linux + securityContext: + # FIXME(#454): uncomment these lines + # runAsNonRoot: true + # runAsUser: 65534 + serviceAccountName: controller + terminationGracePeriodSeconds: 0 diff --git a/platform/upload/BUILD.bazel b/platform/upload/BUILD.bazel index 88bd3f81b..5c6eefde1 100644 --- a/platform/upload/BUILD.bazel +++ b/platform/upload/BUILD.bazel @@ -11,6 +11,9 @@ ocis = { "flannel-monitor": "//flannel-monitor:oci", "kube-dns-main": "//kube-dns:kube-dns-main", "kube-dns-sidecar": "//kube-dns:kube-dns-sidecar", + "metallb-controller": "//metallb:controller", + "metallb-speaker": "//metallb:speaker", + "metallb-mirror": "//metallb:mirror", "pause": "//cri-o/pause:oci", "user-grant": "//user-grant:oci", } From 8f619522f9afd250f09f6987dd3897e1bc0ba647 Mon Sep 17 00:00:00 2001 From: Lily Chung Date: Sat, 11 Apr 2020 20:38:18 -0700 Subject: [PATCH 02/10] metallb: deploy via spire --- platform/spire/BUILD.bazel | 3 ++- platform/spire/src/deploy.py | 7 +++++++ platform/spire/src/seq.py | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/platform/spire/BUILD.bazel b/platform/spire/BUILD.bazel index 5b733cf41..ff4bbd566 100644 --- a/platform/spire/BUILD.bazel +++ b/platform/spire/BUILD.bazel @@ -11,8 +11,9 @@ py_binary( data = [ "//dns-monitor:kubelib", "//dnsmasq:kubelib", - "//flannel:kubelib", "//flannel-monitor:kubelib", + "//flannel:kubelib", + "//metallb:kubelib", "//spire/resources:resourcelib", "//upload:uploadlib", "//user-grant:kubelib", diff --git a/platform/spire/src/deploy.py b/platform/spire/src/deploy.py index f55bc4c59..ec2a1b2c9 100644 --- a/platform/spire/src/deploy.py +++ b/platform/spire/src/deploy.py @@ -60,6 +60,12 @@ def launch_dns_monitor(export: bool=False): launch_spec("//dns-monitor:kubernetes.yaml", export=export) +@command.wrap +def launch_metallb(export: bool=False): + "deploy the specifications to run metallb" + launch_spec("metallb.yaml", export=export) + + @command.wrap def launch_user_grant(export: bool=False): "deploy the specifications to run the user grant website" @@ -94,5 +100,6 @@ def launch_user_grant(export: bool=False): "flannel-monitor": launch_flannel_monitor, "dns-addon": launch_dns_addon, "dns-monitor": launch_dns_monitor, + "metallb": launch_metallb, "user-grant": launch_user_grant, }) diff --git a/platform/spire/src/seq.py b/platform/spire/src/seq.py index 3d1f15dee..c24ca19d4 100644 --- a/platform/spire/src/seq.py +++ b/platform/spire/src/seq.py @@ -50,6 +50,7 @@ def sequence_supervisor(ops: command.Operations, skip_verify_keygateway: bool=Fa ops.add_command(deploy.launch_dns_addon) ops.add_command(deploy.launch_flannel_monitor) ops.add_command(deploy.launch_dns_monitor) + ops.add_command(deploy.launch_metallb) if config.user_grant_domain != '': ops.add_command(deploy.launch_user_grant) From 2d2f62280277d3dfe3c01f8bce73fbc5dafad174 Mon Sep 17 00:00:00 2001 From: Lily Chung Date: Sun, 2 Feb 2020 14:22:02 -0800 Subject: [PATCH 03/10] metallb: Autogenerate configmap. Add a new external-ranges key to the setup.yaml config from which external services will be assigned IP addresses. --- deploy-chroot/setup.yaml.in | 4 +++- docs/cluster-configuration.md | 6 ++++++ platform/metallb/kubernetes.yaml.in | 13 +++++++++++++ platform/spire/resources/setup-schema.yaml | 6 +++++- platform/spire/resources/setup.yaml | 1 + platform/spire/src/configuration.py | 1 + platform/spire/src/deploy.py | 5 ++++- 7 files changed, 33 insertions(+), 3 deletions(-) diff --git a/deploy-chroot/setup.yaml.in b/deploy-chroot/setup.yaml.in index e5de15eeb..c38202e13 100644 --- a/deploy-chroot/setup.yaml.in +++ b/deploy-chroot/setup.yaml.in @@ -10,9 +10,11 @@ cluster: vlan: 612 addresses: - cidr-nodes: 10.{x}.0.0/24 + cidr-nodes: 10.{x}.0.0/23 cidr-pods: 172.18.0.0/16 cidr-services: 172.28.0.0/16 + external-ranges: + - 10.{x}.1.0/24 service-api: 172.28.0.1 service-dns: 172.28.0.2 diff --git a/docs/cluster-configuration.md b/docs/cluster-configuration.md index 976f2dd12..8575c5867 100644 --- a/docs/cluster-configuration.md +++ b/docs/cluster-configuration.md @@ -97,6 +97,8 @@ Sample section: cidr-nodes: 18.4.60.0/23 cidr-pods: 172.18.0.0/16 cidr-services: 172.28.0.0/16 + external-ranges: + - 18.4.62.0/16 service-api: 172.28.0.1 service-dns: 172.28.0.2 @@ -121,6 +123,10 @@ Descriptions: This should be a subnet like `cidr-pods`, but can be smaller, because you will generally have many fewer services than pods. These are allocated to services across the cluster as needed. + * `external-ranges`: the list of publicly-addressible subnets that IP addresses for LoadBalancer services should be allocated out of. + + These may overlap with `cidr-nodes`. + * `service-api`: the service IP address for the Kubernetes apiserver within the cluster. This must always be within the service subnet, and is usually the first address, but can be anything within the range diff --git a/platform/metallb/kubernetes.yaml.in b/platform/metallb/kubernetes.yaml.in index efde73852..2ac977377 100644 --- a/platform/metallb/kubernetes.yaml.in +++ b/platform/metallb/kubernetes.yaml.in @@ -354,3 +354,16 @@ spec: # runAsUser: 65534 serviceAccountName: controller terminationGracePeriodSeconds: 0 +# Above this line is adapted from metallb's manifest +--- +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: metallb-system + name: config +data: + config: | + address-pools: + - name: default + protocol: layer2 + addresses: {{EXTERNAL_RANGES}} diff --git a/platform/spire/resources/setup-schema.yaml b/platform/spire/resources/setup-schema.yaml index 8610708ea..1edbfc448 100644 --- a/platform/spire/resources/setup-schema.yaml +++ b/platform/spire/resources/setup-schema.yaml @@ -34,11 +34,15 @@ properties: type: string cidr-services: type: string + external-ranges: + type: array + items: + type: string service-api: type: string service-dns: type: string - required: ["cidr-nodes", "cidr-pods", "cidr-services", "service-api", "service-dns"] + required: ["cidr-nodes", "cidr-pods", "cidr-services", "external-ranges", "service-api", "service-dns"] additionalProperties: false dns-upstreams: type: array diff --git a/platform/spire/resources/setup.yaml b/platform/spire/resources/setup.yaml index 7546d92d7..b6c363974 100644 --- a/platform/spire/resources/setup.yaml +++ b/platform/spire/resources/setup.yaml @@ -13,6 +13,7 @@ addresses: cidr-nodes: 18.4.60.0/23 cidr-pods: 172.18.0.0/16 cidr-services: 172.28.0.0/16 + external_ranges: [] service-api: 172.28.0.1 service-dns: 172.28.0.2 diff --git a/platform/spire/src/configuration.py b/platform/spire/src/configuration.py index 889eb9dab..408e29453 100644 --- a/platform/spire/src/configuration.py +++ b/platform/spire/src/configuration.py @@ -67,6 +67,7 @@ def __init__(self, kv: dict): self.cidr_nodes = IPv4Network(kv["addresses"]["cidr-nodes"]) self.cidr_pods = IPv4Network(kv["addresses"]["cidr-pods"]) self.cidr_services = IPv4Network(kv["addresses"]["cidr-services"]) + self.external_ranges = [IPv4Network(x) for x in kv["addresses"]["external-ranges"]] self.service_api = IPv4Address(kv["addresses"]["service-api"]) self.service_dns = IPv4Address(kv["addresses"]["service-dns"]) diff --git a/platform/spire/src/deploy.py b/platform/spire/src/deploy.py index ec2a1b2c9..b83cdee37 100644 --- a/platform/spire/src/deploy.py +++ b/platform/spire/src/deploy.py @@ -63,7 +63,10 @@ def launch_dns_monitor(export: bool=False): @command.wrap def launch_metallb(export: bool=False): "deploy the specifications to run metallb" - launch_spec("metallb.yaml", export=export) + config = configuration.get_config() + launch_spec("//metallb:kubernetes.yaml", { + "EXTERNAL_RANGES": [str(x) for x in config.external_ranges], + }, export=export) @command.wrap From 30790d969673d7e24704dd7eb2d7f47ef607a1e6 Mon Sep 17 00:00:00 2001 From: Lily Chung Date: Sat, 11 Apr 2020 20:46:14 -0700 Subject: [PATCH 04/10] metallb: scrape prometheus metrics --- platform/metallb/kubernetes.yaml.in | 32 ++++++++++++++++++++++++ platform/spire/resources/prometheus.yaml | 20 +++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/platform/metallb/kubernetes.yaml.in b/platform/metallb/kubernetes.yaml.in index 2ac977377..716988743 100644 --- a/platform/metallb/kubernetes.yaml.in +++ b/platform/metallb/kubernetes.yaml.in @@ -367,3 +367,35 @@ data: - name: default protocol: layer2 addresses: {{EXTERNAL_RANGES}} +--- +apiVersion: v1 +kind: Service +metadata: + name: metallb-controller + namespace: metallb-system + labels: + app: metallb +spec: + selector: + app: metallb + component: controller + ports: + - name: metrics + port: 7472 + protocol: TCP +--- +apiVersion: v1 +kind: Service +metadata: + name: metallb-speaker + namespace: metallb-system + labels: + app: metallb +spec: + selector: + app: metallb + component: speaker + ports: + - name: metrics + port: 7472 + protocol: TCP diff --git a/platform/spire/resources/prometheus.yaml b/platform/spire/resources/prometheus.yaml index 5de9af554..ad0ebc997 100644 --- a/platform/spire/resources/prometheus.yaml +++ b/platform/spire/resources/prometheus.yaml @@ -135,3 +135,23 @@ scrape_configs: regex: (.+) target_label: __metrics_path__ replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor + + - job_name: 'metallb-controller' + scheme: https + tls_config: + ca_file: /etc/homeworld/authorities/kubernetes.pem + cert_file: /etc/homeworld/keys/kubernetes-supervisor.pem + key_file: /etc/homeworld/keys/kubernetes-supervisor.key + static_configs: + - targets: ['{{APISERVER}}:443'] + metrics_path: '/api/v1/namespaces/metallb-system/services/metallb-controller:metrics/proxy/metrics' + + - job_name: 'metallb-speaker' + scheme: https + tls_config: + ca_file: /etc/homeworld/authorities/kubernetes.pem + cert_file: /etc/homeworld/keys/kubernetes-supervisor.pem + key_file: /etc/homeworld/keys/kubernetes-supervisor.key + static_configs: + - targets: ['{{APISERVER}}:443'] + metrics_path: '/api/v1/namespaces/metallb-system/services/metallb-speaker:metrics/proxy/metrics' From 2f93fade62df51eab8fb77ea5425e9d093970704 Mon Sep 17 00:00:00 2001 From: Lily Chung Date: Sat, 11 Apr 2020 18:54:27 -0700 Subject: [PATCH 05/10] metallb: verify prometheus job is successful during cluster launch --- platform/spire/src/seq.py | 2 ++ platform/spire/src/verify.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/platform/spire/src/seq.py b/platform/spire/src/seq.py index c24ca19d4..1b46e5f52 100644 --- a/platform/spire/src/seq.py +++ b/platform/spire/src/seq.py @@ -125,6 +125,8 @@ def sequence_cluster(ops: command.Operations) -> None: else: ops.add_operation("verify that user-grant is working properly", iterative_verifier(verify.check_user_grant, 120.0)) + ops.add_command(iterative_verifier(verify.check_metallb, 120.0)) + main_command = command.SeqMux("commands about running large sequences of cluster bring-up automatically", { "keysystem": sequence_keysystem, diff --git a/platform/spire/src/verify.py b/platform/spire/src/verify.py index 905334821..f9ea8d621 100644 --- a/platform/spire/src/verify.py +++ b/platform/spire/src/verify.py @@ -353,6 +353,13 @@ def check_user_grant(): print("autogenerated rolebinding for user", repr(authority.UPSTREAM_USER_NAME), "passed basic check!") +@command.wrap +def check_metallb(): + "verify that metallb is running" + expect_prometheus_query_exact('sum(up{job="metallb-controller"})', 1, "metallb controller is online") + expect_prometheus_query_exact('sum(up{job="metallb-speaker"})', 1, "metallb speaker is online") + print("metallb is running") + main_command = command.Mux("commands about verifying the state of a cluster", { "keystatics": check_keystatics, @@ -369,4 +376,5 @@ def check_user_grant(): "flannel": check_flannel, "dns-addon": check_dns, "user-grant": check_user_grant, + "metallb": check_metallb, }) From b9e6c7057cdaecabc1176d7f4c23651a39bab650 Mon Sep 17 00:00:00 2001 From: Lily Chung Date: Sat, 11 Apr 2020 23:51:22 -0700 Subject: [PATCH 06/10] fixup! metallb: deploy via spire --- platform/spire/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/spire/BUILD.bazel b/platform/spire/BUILD.bazel index ff4bbd566..365a1c3c0 100644 --- a/platform/spire/BUILD.bazel +++ b/platform/spire/BUILD.bazel @@ -11,8 +11,8 @@ py_binary( data = [ "//dns-monitor:kubelib", "//dnsmasq:kubelib", - "//flannel-monitor:kubelib", "//flannel:kubelib", + "//flannel-monitor:kubelib", "//metallb:kubelib", "//spire/resources:resourcelib", "//upload:uploadlib", From b1fad8a8f280b27c0c83b20fe09566f31202fff0 Mon Sep 17 00:00:00 2001 From: Lily Chung Date: Sat, 11 Apr 2020 23:52:11 -0700 Subject: [PATCH 07/10] fixup! metallb: Build and publish containers --- platform/WORKSPACE | 1 + platform/metallb/BUILD.bazel | 15 +++++++-------- platform/metallb/deps.bzl | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/platform/WORKSPACE b/platform/WORKSPACE index 899168ce2..64de16b4b 100644 --- a/platform/WORKSPACE +++ b/platform/WORKSPACE @@ -89,6 +89,7 @@ load("//kube-state-metrics:deps.bzl", "kube_state_metrics_dependencies") kube_state_metrics_dependencies() load("//metallb:deps.bzl", "metallb_dependencies") + metallb_dependencies() load("//oci-tools:deps.bzl", "oci_tools_dependencies") diff --git a/platform/metallb/BUILD.bazel b/platform/metallb/BUILD.bazel index 34c6d6ba0..7761ea5dd 100644 --- a/platform/metallb/BUILD.bazel +++ b/platform/metallb/BUILD.bazel @@ -2,7 +2,6 @@ load("//bazel:package.bzl", "homeworld_oci") load("//bazel:substitute.bzl", "substitute") load("//python:resources.bzl", "py_resources") - # TODO: ldflags: # -X go.universe.tf/metallb/internal/version.gitCommit={commit} # -X go.universe.tf/metallb/internal/version.gitBranch={branch} @@ -12,11 +11,11 @@ homeworld_oci( bin = { "@tf_universe_go_metallb//controller": "/usr/bin/controller", }, + exec = ["/usr/bin/controller"], + visibility = ["//visibility:public"], deps = [ "//debian:debian-micro.tgz", ], - exec = ["/usr/bin/controller"], - visibility = ["//visibility:public"], ) homeworld_oci( @@ -24,11 +23,11 @@ homeworld_oci( bin = { "@tf_universe_go_metallb//speaker": "/usr/bin/speaker", }, + exec = ["/usr/bin/speaker"], + visibility = ["//visibility:public"], deps = [ "//debian:debian-micro.tgz", ], - exec = ["/usr/bin/speaker"], - visibility = ["//visibility:public"], ) homeworld_oci( @@ -36,21 +35,21 @@ homeworld_oci( bin = { "@tf_universe_go_metallb//e2etest-mirror-server": "/usr/bin/mirror", }, + exec = ["/usr/bin/mirror"], + visibility = ["//visibility:public"], deps = [ "//debian:debian-micro.tgz", ], - exec = ["/usr/bin/mirror"], - visibility = ["//visibility:public"], ) substitute( name = "kubernetes.yaml", - template = ":kubernetes.yaml.in", kfs = { "controller_digest": ":controller.ocidigest", "speaker_digest": ":speaker.ocidigest", "mirror_digest": ":mirror.ocidigest", }, + template = ":kubernetes.yaml.in", visibility = ["//visibility:public"], ) diff --git a/platform/metallb/deps.bzl b/platform/metallb/deps.bzl index 258350afc..04622632a 100644 --- a/platform/metallb/deps.bzl +++ b/platform/metallb/deps.bzl @@ -3,19 +3,19 @@ load("@bazel_gazelle//:deps.bzl", "go_repository") def metallb_dependencies(): go_repository( name = "tf_universe_go_metallb", - commit = "15d9ed5cce53b457a955f894aa58e7f4856f73d5", # v0.8.3 + commit = "15d9ed5cce53b457a955f894aa58e7f4856f73d5", # v0.8.3 importpath = "go.universe.tf/metallb", ) go_repository( name = "com_github_go_kit_kit", - commit = "150a65a7ec6156b4b640c1fd55f26fd3d475d656", # v0.9.0 + commit = "150a65a7ec6156b4b640c1fd55f26fd3d475d656", # v0.9.0 importpath = "github.com/go-kit/kit", ) go_repository( name = "com_github_go_logfmt_logfmt", - commit = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5", # v0.3.0 + commit = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5", # v0.3.0 importpath = "github.com/go-logfmt/logfmt", ) @@ -27,7 +27,7 @@ def metallb_dependencies(): go_repository( name = "com_github_hashicorp_golang_lru", - commit = "20f1fb78b0740ba8c3cb143a61e86ba5c8669768", # v0.5.0 + commit = "20f1fb78b0740ba8c3cb143a61e86ba5c8669768", # v0.5.0 importpath = "github.com/hashicorp/golang-lru", ) From b74a4639f86915cb5793e9ab31e586157e66f8e0 Mon Sep 17 00:00:00 2001 From: Lily Chung Date: Sat, 11 Apr 2020 23:58:28 -0700 Subject: [PATCH 08/10] fixup! metallb: Autogenerate configmap. --- .jenkins/deploy-setup.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.jenkins/deploy-setup.yaml b/.jenkins/deploy-setup.yaml index 458625f55..7e2d46fb8 100644 --- a/.jenkins/deploy-setup.yaml +++ b/.jenkins/deploy-setup.yaml @@ -10,9 +10,11 @@ cluster: vlan: 612 addresses: - cidr-nodes: 10.1.0.0/24 + cidr-nodes: 10.1.0.0/23 cidr-pods: 172.18.0.0/16 cidr-services: 172.28.0.0/16 + external-ranges: + - 10.1.1.0/24 service-api: 172.28.0.1 service-dns: 172.28.0.2 From c624b86384227e6e275210021036860f76fee4c2 Mon Sep 17 00:00:00 2001 From: Lily Chung Date: Thu, 16 Apr 2020 20:10:18 -0700 Subject: [PATCH 09/10] fixup! metallb: Build and publish containers --- platform/upload/BUILD.bazel | 1 - 1 file changed, 1 deletion(-) diff --git a/platform/upload/BUILD.bazel b/platform/upload/BUILD.bazel index 5c6eefde1..c569f0225 100644 --- a/platform/upload/BUILD.bazel +++ b/platform/upload/BUILD.bazel @@ -13,7 +13,6 @@ ocis = { "kube-dns-sidecar": "//kube-dns:kube-dns-sidecar", "metallb-controller": "//metallb:controller", "metallb-speaker": "//metallb:speaker", - "metallb-mirror": "//metallb:mirror", "pause": "//cri-o/pause:oci", "user-grant": "//user-grant:oci", } From fc8761fbaf68ff3ff432f787ae1d414a68b17a43 Mon Sep 17 00:00:00 2001 From: Lily Chung Date: Thu, 16 Apr 2020 20:17:44 -0700 Subject: [PATCH 10/10] fixup! metallb: Build and publish containers --- platform/metallb/BUILD.bazel | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/platform/metallb/BUILD.bazel b/platform/metallb/BUILD.bazel index 7761ea5dd..b281f6962 100644 --- a/platform/metallb/BUILD.bazel +++ b/platform/metallb/BUILD.bazel @@ -30,24 +30,11 @@ homeworld_oci( ], ) -homeworld_oci( - name = "mirror", - bin = { - "@tf_universe_go_metallb//e2etest-mirror-server": "/usr/bin/mirror", - }, - exec = ["/usr/bin/mirror"], - visibility = ["//visibility:public"], - deps = [ - "//debian:debian-micro.tgz", - ], -) - substitute( name = "kubernetes.yaml", kfs = { "controller_digest": ":controller.ocidigest", "speaker_digest": ":speaker.ocidigest", - "mirror_digest": ":mirror.ocidigest", }, template = ":kubernetes.yaml.in", visibility = ["//visibility:public"],