Updates from airqo staging

airqo-platform · Sep 9, 2024 · 85ce968 · 85ce968
2 parents ef13daf + 15db7d9
commit 85ce968
Show file tree

Hide file tree

Showing 108 changed files with 1,221 additions and 3,494 deletions.
diff --git a/.github/workflows/deploy-apis-to-production.yml b/.github/workflows/deploy-apis-to-production.yml
@@ -49,6 +49,7 @@ jobs:
       run_meta_data: ${{ steps.check_files.outputs.run_meta_data }} # meta data
       run_view: ${{ steps.check_files.outputs.run_view }} # view
       run_calibrate: ${{ steps.check_files.outputs.run_calibrate }} # calibrate
+      run_insights: ${{ steps.check_files.outputs.run_insights}} # analytics
       run_kafka_cluster_operator: ${{ steps.check_files.outputs.run_kafka_cluster_operator }} # kafka cluster operator
       run_kafka_cluster: ${{ steps.check_files.outputs.run_kafka_cluster }} # kafka cluster
       run_kafka_topics: ${{ steps.check_files.outputs.run_kafka_topics }} # kafka topics
@@ -95,6 +96,7 @@ jobs:
           echo "run_kafka_topics=false" >>$GITHUB_OUTPUT
           echo "run_workflows=false" >>$GITHUB_OUTPUT
           echo "run_incentives=false" >>$GITHUB_OUTPUT
+          echo "run_insights=false" >>$GITHUB_OUTPUT
           echo "run_spatial=false" >>$GITHUB_OUTPUT
           echo "run_kafka_connectors=false" >>$GITHUB_OUTPUT
           echo "run_nginx=false" >>$GITHUB_OUTPUT
@@ -254,7 +256,13 @@ jobs:
             if [[ $file == k8s/incentives/* ]]; then
               echo "run_incentives=true" >>$GITHUB_OUTPUT              
             fi
+            if [[ $file == src/insights/* ]]; then
+              echo "run_insights=true" >>$GITHUB_OUTPUT
+            fi 
 
+            if [[ $file == k8s/insights/* ]]; then
+              echo "run_insights=true" >>$GITHUB_OUTPUT              
+            fi
             if [[ $file == src/spatial/* ]]; then
               echo "run_spatial=true" >>$GITHUB_OUTPUT
             fi 
@@ -283,6 +291,7 @@ jobs:
               echo "run_auth_service=true" >>$GITHUB_OUTPUT
               echo "run_workflows=true" >>$GITHUB_OUTPUT
               echo "run_incentives=true" >>$GITHUB_OUTPUT
+              echo "run_insights=true" >>$GITHUB_OUTPUT
               echo "run_spatial=true" >>$GITHUB_OUTPUT
               echo "run_view=true" >>$GITHUB_OUTPUT
               echo "run_kafka_connectors=true" >>$GITHUB_OUTPUT
@@ -841,6 +850,74 @@ jobs:
           kubectl create configmap --dry-run=client -o yaml --from-env-file=reports.env env-analytics-report-production | kubectl replace -f - -n production
           kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json prod-analytics-config-files | kubectl replace -f - -n production
 
+  insights:
+    name: build-push-deploy-insights
+    needs: [check, image-tag]
+    if: needs.check.outputs.run_insights == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+
+      ### run unit tests ###
+
+      - name: Login to GCR
+        uses: docker/[email protected]
+        with:
+          registry: ${{ env.REGISTRY_URL }}
+          username: _json_key
+          password: ${{ secrets.GCR_CONFIG }}
+
+      - name: Login to K8S
+        uses: azure/[email protected]
+        with:
+          method: kubeconfig
+          kubeconfig: ${{ secrets.K8S_CONFIG_PROD }}
+
+      - name: Build and push API Docker Image
+        uses: docker/[email protected]
+        with:
+          push: true
+          context: src/insights/
+          target: production
+          tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-insights-api:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-insights-api:latest
+
+      - name: Build and push Celery worker Image
+        uses: docker/[email protected]
+        with:
+          push: true
+          context: src/insights/
+          target: celery
+          tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-insights-celery:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-insights-celery:latest
+
+      - name: Update corresponding helm values file(with retry)
+        uses: Wandalen/[email protected] # Retries action on fail
+        with:
+          action: fjogeleit/yaml-update-action@main # Action to retry
+          with: |
+            valueFile: "k8s/insights/values-prod.yaml"
+            propertyPath: "images.tag"
+            value: ${{ needs.image-tag.outputs.build_id }}
+            branch: ${{ env.DEPLOY_BRANCH }}
+            token: ${{ secrets.YAML_UPDATER_TOKEN }}
+            message: "Update insights production image tag to ${{ needs.image-tag.outputs.build_id }}"
+
+      - name: Login to GCP
+        uses: google-github-actions/[email protected]
+        with:
+          credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
+
+      - name: Setup Cloud SDK
+        uses: google-github-actions/[email protected]
+
+      - name: Update the corresponding k8s configmap(s)
+        run: |
+          cd src/insights/
+          gcloud secrets versions access latest --secret="prod-env-analytics" > .env
+          gcloud secrets versions access latest --secret="prod-key-analytics-service-account" > google_application_credentials.json
+          kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-insights-production | kubectl replace -f - -n production
+          kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json prod-insights-config-files | kubectl replace -f - -n production
+
   ### device uptime ###
   device-uptime:
     name: build-push-deploy-device-uptime

diff --git a/.github/workflows/deploy-apis-to-staging.yml b/.github/workflows/deploy-apis-to-staging.yml
@@ -54,6 +54,7 @@ jobs:
       run_kafka_topics: ${{ steps.check_files.outputs.run_kafka_topics }} # kafka topics
       run_workflows: ${{ steps.check_files.outputs.run_workflows }} # workflows
       run_incentives: ${{ steps.check_files.outputs.run_incentives }} # incentives
+      run_insights: ${{ steps.check_files.outputs.run_insights }} # incentives
       run_spatial: ${{ steps.check_files.outputs.run_spatial }} # spatial
       run_kafka_connectors: ${{ steps.check_files.outputs.run_kafka_connectors }} # kafka connectors
       run_nginx: ${{ steps.check_files.outputs.run_nginx }} # nginx ingress
@@ -95,6 +96,7 @@ jobs:
           echo "run_kafka_topics=false" >>$GITHUB_OUTPUT
           echo "run_workflows=false" >>$GITHUB_OUTPUT
           echo "run_incentives=false" >>$GITHUB_OUTPUT
+          echo "run_insights=false" >>$GITHUB_OUTPUT
           echo "run_spatial=false" >>$GITHUB_OUTPUT
           echo "run_kafka_connectors=false" >>$GITHUB_OUTPUT
           echo "run_nginx=false" >>$GITHUB_OUTPUT
@@ -255,6 +257,14 @@ jobs:
               echo "run_incentives=true" >>$GITHUB_OUTPUT              
             fi
 
+            if [[ $file == src/insights/* ]]; then
+              echo "run_insights=true" >>$GITHUB_OUTPUT
+            fi
+            
+            if [[ $file == k8s/insights/* ]]; then
+              echo "run_insights=true" >>$GITHUB_OUTPUT              
+            fi
+
             if [[ $file == src/spatial/* ]]; then
               echo "run_spatial=true" >>$GITHUB_OUTPUT
             fi
@@ -284,6 +294,7 @@ jobs:
               echo "run_auth_service=true" >>$GITHUB_OUTPUT
               echo "run_workflows=true" >>$GITHUB_OUTPUT
               echo "run_incentives=true" >>$GITHUB_OUTPUT
+              echo "run_insights=true" >>$GITHUB_OUTPUT
               echo "run_spatial=true" >>$GITHUB_OUTPUT
               echo "run_view=true" >>$GITHUB_OUTPUT
               echo "run_kafka_connectors=true" >>$GITHUB_OUTPUT
@@ -990,6 +1001,73 @@ jobs:
           kubectl create configmap --dry-run=client -o yaml --from-env-file=reports.env env-analytics-report-staging | kubectl replace -f - -n staging
           kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json stage-analytics-config-files | kubectl replace -f - -n staging
 
+  insights:
+    name: build-push-deploy-insights
+    needs: [check, image-tag]
+    if: needs.check.outputs.run_insights == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+
+      ### run unit tests ###
+
+      - name: Login to GCR
+        uses: docker/[email protected]
+        with:
+          registry: ${{ env.REGISTRY_URL }}
+          username: _json_key
+          password: ${{ secrets.GCR_CONFIG }}
+
+      - name: Login to K8S
+        uses: azure/[email protected]
+        with:
+          method: kubeconfig
+          kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }}
+
+      - name: Build and push API Docker Image
+        uses: docker/[email protected]
+        with:
+          push: true
+          context: src/insights/
+          target: staging
+          tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-insights-api:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-insights-api:latest
+
+      - name: Build and push Celery Docker Image
+        uses: docker/[email protected]
+        with:
+          push: true
+          context: src/insights/
+          target: celery
+          tags: ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-insights-celery:${{ needs.image-tag.outputs.build_id }},${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-insights-celery:latest
+
+      - name: Update corresponding helm values file(with retry)
+        uses: Wandalen/[email protected] # Retries action on fail
+        with:
+          action: fjogeleit/yaml-update-action@main # Action to retry
+          with: |
+            valueFile: "k8s/insights/values-stage.yaml"
+            propertyPath: "images.tag"
+            value: ${{ needs.image-tag.outputs.build_id }}
+            branch: ${{ env.DEPLOY_BRANCH }}
+            token: ${{ secrets.YAML_UPDATER_TOKEN }}
+            message: "Update insights staging images tag to ${{ needs.image-tag.outputs.build_id }}"
+
+      - name: Login to GCP
+        uses: google-github-actions/[email protected]
+        with:
+          credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
+
+      - name: Setup Cloud SDK
+        uses: google-github-actions/[email protected]
+
+      - name: Update the corresponding k8s configmap(s)
+        run: |
+          cd src/insights/
+          gcloud secrets versions access latest --secret="sta-env-analytics" > .env
+          gcloud secrets versions access latest --secret="sta-key-analytics-service-account" > google_application_credentials.json
+          kubectl create configmap --dry-run=client -o yaml --from-env-file=.env env-insights-staging | kubectl replace -f - -n staging
+          kubectl create configmap --dry-run=client -o yaml --from-file=google_application_credentials.json stage-insights-config-files | kubectl replace -f - -n staging
   ### device uptime ###
   device-uptime:
     name: build-push-deploy-device-uptime

diff --git a/k8s/analytics/values-prod.yaml b/k8s/analytics/values-prod.yaml
@@ -8,7 +8,7 @@ images:
     celeryWorker: eu.gcr.io/airqo-250220/airqo-analytics-celery-worker
     reportJob: eu.gcr.io/airqo-250220/airqo-analytics-report-job
     devicesSummaryJob: eu.gcr.io/airqo-250220/airqo-analytics-devices-summary-job
-  tag: prod-fb910b65-1725810667
+  tag: prod-153deed2-1725870728
 api:
   name: airqo-analytics-api
   label: analytics-api

diff --git a/k8s/analytics/values-stage.yaml b/k8s/analytics/values-stage.yaml
@@ -8,7 +8,7 @@ images:
     celeryWorker: eu.gcr.io/airqo-250220/airqo-stage-analytics-celery-worker
     reportJob: eu.gcr.io/airqo-250220/airqo-stage-analytics-report-job
     devicesSummaryJob: eu.gcr.io/airqo-250220/airqo-stage-analytics-devices-summary-job
-  tag: stage-5c6990cd-1719227902
+  tag: stage-3c86d834-1725869877
 api:
   name: airqo-stage-analytics-api
   label: sta-alytics-api

diff --git a/k8s/auth-service/values-prod.yaml b/k8s/auth-service/values-prod.yaml
@@ -6,7 +6,7 @@ app:
 replicaCount: 3
 image:
   repository: eu.gcr.io/airqo-250220/airqo-auth-api
-  tag: prod-fb910b65-1725810667
+  tag: prod-153deed2-1725870728
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

diff --git a/k8s/device-registry/values-prod.yaml b/k8s/device-registry/values-prod.yaml
@@ -6,7 +6,7 @@ app:
 replicaCount: 3
 image:
   repository: eu.gcr.io/airqo-250220/airqo-device-registry-api
-  tag: prod-fb910b65-1725810667
+  tag: prod-153deed2-1725870728
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

diff --git a/k8s/device-uptime/values-prod.yaml b/k8s/device-uptime/values-prod.yaml
@@ -4,7 +4,7 @@ app:
   configmap: env-device-uptime-production
 image:
   repository: eu.gcr.io/airqo-250220/airqo-device-uptime-job
-  tag: prod-d8be0487-1725363294
+  tag: prod-153deed2-1725870728
 nameOverride: ''
 fullnameOverride: ''
 resources:

diff --git a/k8s/device-uptime/values-stage.yaml b/k8s/device-uptime/values-stage.yaml
@@ -4,7 +4,7 @@ app:
   configmap: env-device-uptime-staging
 image:
   repository: eu.gcr.io/airqo-250220/airqo-stage-device-uptime-job
-  tag: stage-96d94ce4-1715349509
+  tag: stage-3c86d834-1725869877
 nameOverride: ''
 fullnameOverride: ''
 resources:

diff --git a/k8s/exceedance/values-prod-airqo.yaml b/k8s/exceedance/values-prod-airqo.yaml
@@ -4,6 +4,6 @@ app:
   configmap: env-exceedance-production
 image:
   repository: eu.gcr.io/airqo-250220/airqo-exceedance-job
-  tag: prod-fb910b65-1725810667
+  tag: prod-153deed2-1725870728
 nameOverride: ''
 fullnameOverride: ''
diff --git a/k8s/exceedance/values-prod-kcca.yaml b/k8s/exceedance/values-prod-kcca.yaml
@@ -4,6 +4,6 @@ app:
   configmap: env-exceedance-production
 image:
   repository: eu.gcr.io/airqo-250220/kcca-exceedance-job
-  tag: prod-fb910b65-1725810667
+  tag: prod-153deed2-1725870728
 nameOverride: ''
 fullnameOverride: ''
diff --git a/k8s/insights/.helmignore b/k8s/insights/.helmignore
@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/k8s/insights/Chart.yaml b/k8s/insights/Chart.yaml
@@ -0,0 +1,10 @@
+apiVersion: v2
+appVersion: "2.0.0"
+description: AirQo Insights Helm Chart
+name: airqo-insights
+home: https://airqo.net
+version: 1.0.0
+maintainers:
+  - name: AirQo
+    email: [email protected]
+    url: https://airqo.net
diff --git a/src/insights/api/__init__.py → k8s/insights/charts/.gitkeep b/src/insights/api/__init__.py → k8s/insights/charts/.gitkeep
diff --git a/k8s/insights/templates/NOTES.txt b/k8s/insights/templates/NOTES.txt
@@ -0,0 +1,22 @@
+1. Get the application URL by running these commands:
+{{- if .Values.ingress.enabled }}
+{{- range $host := .Values.ingress.hosts }}
+  {{- range .paths }}
+  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
+  {{- end }}
+{{- end }}
+{{- else if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "airqo-analytics.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "airqo-analytics.fullname" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "airqo-analytics.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
+  echo http://$SERVICE_IP:{{ .Values.service.port }}
+{{- else if contains "ClusterIP" .Values.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "airqo-analytics.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
+{{- end }}