Skip to content

Commit

Permalink
Refactor k8s auth job (apache#126)
Browse files Browse the repository at this point in the history
  • Loading branch information
Zhuli authored Apr 7, 2018
1 parent 8ad4207 commit c9183e9
Show file tree
Hide file tree
Showing 3 changed files with 69 additions and 103 deletions.
165 changes: 66 additions & 99 deletions ...s/cloud-provider-openstack-acceptance-test-keystone-authentication-authorization/run.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,46 +1,39 @@
- name: Set up Kubernetes local cluster
hosts: all
roles:
- install-k8s-jobs-dependences
- export-vexxhost-openrc
become: yes
tasks:
- name: Set up Kubernetes local cluster
shell:
cmd: |
set -x
set -e
set -o pipefail
apt-get install python-pip -y
pip install -U python-openstackclient
export OS_DOMAIN_NAME=$(echo '{{ vexxhost_credentials.user_domain_name }}')
export OS_AUTH_TYPE=$(echo '{{ vexxhost_credentials.auth_type }}')
export OS_IDENTITY_API_VERSION=$(echo '{{ vexxhost_credentials.identity_api_version }}')
export OS_VOLUME_API_VERSION=$(echo '{{ vexxhost_credentials.volume_api_version }}')
export OS_INTERFACE=$(echo '{{ vexxhost_credentials.interface }}')
export OS_AUTH_URL=$(echo '{{ vexxhost_credentials.auth_url }}')
export OS_PROJECT_ID=$(echo '{{ vexxhost_credentials.project_id }}')
export OS_PROJECT_NAME=$(echo '{{ vexxhost_credentials.project_name }}')
export OS_USER_DOMAIN_NAME=$(echo '{{ vexxhost_credentials.user_domain_name }}')
export OS_PROJECT_DOMAIN_ID=$(echo '{{ vexxhost_credentials.project_domain_id }}')
export OS_USERNAME=$(echo '{{ vexxhost_credentials.username }}')
export OS_PASSWORD=$(echo '{{ vexxhost_credentials.password }}')
export OS_REGION_NAME=$(echo '{{ vexxhost_credentials.region_name }}')
mkdir -p /etc/kubernetes/
cp ./examples/webhook/policy.json /etc/kubernetes/
sed -i "s/c1f7910086964990847dc6c8b128f63c/$OS_PROJECT_ID/g" /etc/kubernetes/policy.json
sed -i -e "s/k8s-admin/creator/g" /etc/kubernetes/policy.json
if [[ ! -d "/etc/kubernetes/" ]]; then
sudo mkdir -p /etc/kubernetes/
fi
chown zuul /etc/kubernetes/
# Create cloud-config
cat << EOF >> /etc/kubernetes/cloud-config
[Global]
domain-name = ${OS_PROJECT_DOMAIN_NAME-$OS_PROJECT_DOMAIN_ID}
domain-name = $OS_USER_DOMAIN_NAME
tenant-id = $OS_PROJECT_ID
auth-url = $OS_AUTH_URL
password = $OS_PASSWORD
username = $OS_USERNAME
region = $OS_REGION_NAME
[BlockStorage]
bs-version = v2
ignore-volume-az = yes
EOF
# Create webhook.kubeconfig
cat << EOF >> /etc/kubernetes/webhook.kubeconfig
apiVersion: v1
clusters:
Expand All @@ -60,60 +53,34 @@
- name: webhook
EOF
set -x
make depend
make build
mkdir -p "{{ ansible_user_dir }}/.kube"
export API_HOST_IP="172.17.0.1"
# Go where we cloned kubernetes repository
cd $GOPATH/src/k8s.io/kubernetes/
export API_HOST_IP=$(ifconfig | awk '/^docker0/ {getline; print $2}' | awk -F ':' '{print $2}')
export KUBELET_HOST="0.0.0.0"
echo "Stopping firewall and allow all traffic..."
iptables -F
iptables -X
iptables -t nat -F
iptables -t nat -X
iptables -t mangle -F
iptables -t mangle -X
iptables -P INPUT ACCEPT
iptables -P FORWARD ACCEPT
iptables -P OUTPUT ACCEPT
export ALLOW_SECURITY_CONTEXT=true
export ENABLE_CRI=false
export ENABLE_HOSTPATH_PROVISIONER=true
export ENABLE_SINGLE_CA_SIGNER=true
# export KUBE_ENABLE_CLUSTER_DASHBOARD=true
export KUBE_ENABLE_CLUSTER_DNS=false
export LOG_LEVEL=10
# we want to use the openstack cloud provider
export LOG_LEVEL=4
# We want to use the openstack cloud provider
export CLOUD_PROVIDER=openstack
# we want to run a separate cloud-controller-manager for openstack
# We want to run a separate cloud-controller-manager for openstack
export EXTERNAL_CLOUD_PROVIDER=true
# DO NOT change the location of the cloud-config file. It is important for the old cinder provider to work
export CLOUD_CONFIG=/etc/kubernetes/cloud-config
# specify the OCCM binary
export EXTERNAL_CLOUD_PROVIDER_BINARY="{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/openstack-cloud-controller-manager"
# Cleanup some directories just in case
sudo rm -rf /var/lib/kubelet/*
# Specify the OCCM binary
export EXTERNAL_CLOUD_PROVIDER_BINARY='{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/openstack-cloud-controller-manager'
# location of where the kubernetes processes log their output
mkdir -p "{{ ansible_user_dir }}/workspace/logs/kubernetes"
export LOG_DIR="{{ ansible_user_dir }}/workspace/logs/kubernetes"
mkdir -p '{{ ansible_user_dir }}/workspace/logs/kubernetes'
export LOG_DIR='{{ ansible_user_dir }}/workspace/logs/kubernetes'
# We need this for one of the conformance tests
export ALLOW_PRIVILEGED=true
# Just kick off all the processes and drop down to the command line
export ENABLE_DAEMON=true
# We need the hostname to match the name of the vm started by openstack
export HOSTNAME_OVERRIDE=$(curl http://169.254.169.254/openstack/latest/meta_data.json | python -c "import sys, json; print json.load(sys.stdin)['name']")
# copy the same policy json and fix up the hard coded project id
cp ./examples/webhook/policy.json /etc/kubernetes/
sed -i -e "s|c1f7910086964990847dc6c8b128f63c|$OS_PROJECT_ID|g" /etc/kubernetes/policy.json
# pick an existing role of the user and replace the k8s-admin role in policy.json
sed -i -e "s|k8s-admin|creator|g" /etc/kubernetes/policy.json
# print the modified file
cat /etc/kubernetes/policy.json
pushd ${GOPATH}/src/k8s.io/kubernetes
export MAX_TIME_FOR_URL_API_SERVER=5
export AUTHORIZATION_MODE="Node,Webhook,RBAC"
sed -i -e "/kube::util::wait_for_url.*$/,+1d" hack/local-up-cluster.sh
Expand All @@ -122,18 +89,13 @@
# -E preserves the current env vars, but we need to special case PATH
sudo -E PATH=$PATH SHELLOPTS=$SHELLOPTS ./hack/local-up-cluster.sh -O
nohup "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/k8s-keystone-auth" \
--tls-cert-file /var/run/kubernetes/serving-kube-apiserver.crt \
--tls-private-key-file /var/run/kubernetes/serving-kube-apiserver.key \
--keystone-policy-file /etc/kubernetes/policy.json \
--log-dir=${LOG_DIR} \
--v=10 \
--keystone-url ${OS_AUTH_URL} >"${LOG_DIR}/keystone-auth.log" 2>&1 &
# sudo of local-up-cluster mucks with permissions
sudo chmod -R 777 "{{ ansible_user_dir }}/.kube"
sudo chmod 777 /var/run/kubernetes/client-admin.key
--keystone-url ${OS_AUTH_URL} >"${LOG_DIR}/keystone-auth.log" 2>&1 &
# set up the config we need for kubectl to work
cluster/kubectl.sh config set-cluster local --server=https://localhost:6443 --certificate-authority=/var/run/kubernetes/server-ca.crt
Expand All @@ -150,41 +112,47 @@
cluster/kubectl.sh create clusterrolebinding --user system:kube-controller-manager kube-system-cluster-admin-6 --clusterrole cluster-admin
{
TOKEN=$(openstack token issue -f value -c id)
authenticated_info=`cat << EOF | curl -kvs -XPOST -d @- https://localhost:8443/webhook | python -c "import sys, json; print json.load(sys.stdin)"
{
"apiVersion": "authentication.k8s.io/v1beta1",
"kind": "TokenReview",
"metadata": {
"creationTimestamp": null
},
"spec": {
"token": "$TOKEN"
}
}
EOF`
base_body=`cat << EOF | python -c "import sys, json; print json.load(sys.stdin)"
{
"apiVersion": "authorization.k8s.io/v1beta1",
"kind": "SubjectAccessReview",
"spec": {
"resourceAttributes": {
"namespace": "default",
"verb": "get",
"group": "",
"resource": "pods"
}
}
}
EOF`
authorization_body=$(python -c "import json; s1=${authenticated_info}; s2=${base_body}; \
s2['spec']['user']=s1['status']['user']['username']; \
s2['spec']['group']=s1['status']['user']['groups']; \
s2['spec']['extra']=s1['status']['user']['extra'];print json.dumps(s2)")
allowed=$(echo $authorization_body | curl -kvs -XPOST -d @- https://localhost:8443/webhook | python -mjson.tool)
} 1> /dev/null 2>&1
echo ${allowed}
[[ "${allowed}" =~ '"allowed": true' ]] && echo "Testing k8s-keystone-auth sucessfully!"
authenticated_info=$(cat <<< '
{
"apiVersion": "authentication.k8s.io/v1beta1",
"kind": "TokenReview",
"metadata": {
"creationTimestamp": null
},
"spec": {
"token": "'$(openstack token issue -f value -c id)'"
}
}' | curl -kvs -XPOST -d @- https://localhost:8443/webhook | python -c "import sys, json; print json.load(sys.stdin)"
)
base_body=$(cat <<< '
{
"apiVersion": "authorization.k8s.io/v1beta1",
"kind": "SubjectAccessReview",
"spec": {
"resourceAttributes": {
"namespace": "default",
"verb": "get",
"group": "",
"resource": "pods"
}
}
}' | python -c "import sys, json; print json.load(sys.stdin)"
)
update_auth_info=$(echo "
import json;
s1=$authenticated_info;
s2=$base_body;
s2['spec']['user']=s1['status']['user']['username'];
s2['spec']['group']=s1['status']['user']['groups'];
s2['spec']['extra']=s1['status']['user']['extra'];
print json.dumps(s2)" | sed 's/^ \+//'
)
authorization_body=$(python -c "$update_auth_info")
allowed=$(echo "$authorization_body" | curl -kvs -XPOST -d @- https://localhost:8443/webhook | python -mjson.tool)
} > /dev/null 2>&1
echo "$allowed"
[[ "$allowed" =~ '"allowed": true' ]] && echo "Testing k8s-keystone-auth sucessfully!"
cluster/kubectl.sh config set-credentials openstackuser --auth-provider=openstack
cluster/kubectl.sh config set-context --cluster=local --user=openstackuser openstackuser@local
Expand All @@ -193,7 +161,6 @@
echo "Testing kubernetes+keystone authentication and authorizatio failed!"
exit 1
fi
popd
executable: /bin/bash
chdir: '{{ zuul.project.src_dir }}'
environment: '{{ golang_env }}'
environment: '{{ golang_env | combine(vexxhost_openrc) }}'
1 change: 1 addition & 0 deletions roles/export-vexxhost-openrc/tasks/main.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,4 @@
OS_USERNAME: '{{ vexxhost_credentials.username }}'
OS_PASSWORD: '{{ vexxhost_credentials.password }}'
OS_REGION_NAME: '{{ vexxhost_credentials.region_name }}'
OS_DOMAIN_NAME: '{{ vexxhost_credentials.user_domain_name }}'
6 changes: 2 additions & 4 deletions zuul.d/jobs.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,11 +300,9 @@

- job:
name: cloud-provider-openstack-acceptance-test-keystone-authentication-authorization
parent: golang-test
parent: cloud-provider-openstack-acceptance-test
description: |
Run acceptance tests of cloud-provider-openstack repo of K8S+keystone authentication and
authorization scenario against vexxhost cloud
Run keystone auth acceptance tests of cloud-provider-openstack
run: playbooks/cloud-provider-openstack-acceptance-test-keystone-authentication-authorization/run.yaml
secrets:
- vexxhost_credentials
nodeset: ubuntu-xenial-vexxhost

0 comments on commit c9183e9

Please sign in to comment.