chore: add dashboard automation (#86)

* chore: allow generating of reports

logged to https://docs.google.com/spreadsheets/d/1xq36kjThObEaRKzb3VRtXEs9RgM-bfgfjGbi1vbPUiE/edit\?usp\=sharing

* feat: improve reports

* provide instructions for generating reports
* get 90 days of data
* if calculating=true, call active_users again

* feat: automate updating of google sheet data

* chore: update data

* fix: monthly and weekly counts

* feat: run via github actions

* chore: attempt to run workflow from feat/reports branch

* chore(tmp): run on push to feat/reports

* chore: fix npm install for reports GH action

* chore: install ts-node directly instead of npx

* chore: cache root npm install

* chore: removing data files from repo

* chore: cleanup github action

* docs: update reports readme

* chore: adress self-review fixes

* Update reports/doCountlyFetch.ts

Co-authored-by: Nishant Arora <[email protected]>

* Update reports/constants.ts

Co-authored-by: Nishant Arora <[email protected]>

* chore: address PR comments

* chore: don't write files

---------

Co-authored-by: Nishant Arora <[email protected]>

.eslintignore

@@ -1 +1,2 @@
 dist
+reports

.github/workflows/metrics-update.yml

@@ -0,0 +1,25 @@
+name: Update metrics data in google spreadsheet
+
+on:
+  workflow_dispatch:
+  schedule:
+  # Every 2 hours: "At minute 0 past every 2nd hour."
+  - cron: '0 */2 * * *'
+
+jobs:
+  update-metrics:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./reports
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-node@v3
+      with:
+        node-version: lts/*
+    - uses: ipfs/aegir/actions/cache-node-modules@master # npm install at the root
+    - run: npm install && npm run update-dashboards # npm install & run in reports directory
+      env:
+        GOOGLE_CREDENTIALS: '${{ secrets.GOOGLE_CREDENTIALS }}'
+        COUNTLY_USERNAME: '${{ secrets.COUNTLY_USERNAME }}'
+        COUNTLY_PASSWORD: '${{ secrets.COUNTLY_PASSWORD }}'

.gitignore

@@ -30,3 +30,8 @@ typings/
 storybook-static
 .envrc
 .tool-versions
+reports/data
+reports/node_modules
+ignite-metrics-dashboard-d99dde383c4c.json
+.env
+output

reports/README.md

@@ -0,0 +1,55 @@
+# Explanation
+
+The code in this folder (`reports`) is used to generate CSV files for getting daily/weekly/monthly active users for all Ignite team (ipfs-gui) projects.
+
+The CSV content is then loaded into it's respective sheet at https://docs.google.com/spreadsheets/d/1xq36kjThObEaRKzb3VRtXEs9RgM-bfgfjGbi1vbPUiE/edit#gid=755468744
+
+The charts in the "Charts" sheet are loaded in our Notion page at https://www.notion.so/pl-strflt/Ignite-IPFS-GUI-Tools-3bc1c1bf54d74f928bf11ef59c876b74#b6970aa92e914114848fbddd84eab2ba
+
+NOTE: The below instructions do not need to be followed once our GitHub Action is merged. The data will update according to the scheduled GitHub Action at [`../.github/workflows/metrics-update.yml`](../.github/workflows/metrics-update.yml)
+
+## With google sheets authentication
+
+You need the following ENV vars set properly:
+
+* `GOOGLE_CREDENTIALS` - Your 'JSON service account key' file stringified into a single line
+
+Just run `npm run update-dashboards`. This will download all data from countly and then automatically update the google sheets.
+
+## How to get the data from countly
+
+You need the following ENV vars set properly:
+
+* `COUNTLY_USERNAME` - The username you use to login to the countly server
+* `COUNTLY_PASSWORD` - The password you use to login to the countly server
+
+Inside the `./reports` folder, run
+
+```bash
+npm install
+npm run get-csv
+```
+
+## How to copy the data to google spreadsheets (manually)
+
+If you have a valid keyfile for google sheets authentication
+
+1. Open up the relevant `./reports/output/*.csv` daily/weekly/monthly file and copy its contents.
+1. Paste that content into the relevant google sheet, cell A1, at https://docs.google.com/spreadsheets/d/1xq36kjThObEaRKzb3VRtXEs9RgM-bfgfjGbi1vbPUiE/edit#gid=755468744
+1. Select "Data->Split Text to columns"
+
+The charts and everything should automatically update.
+
+## How to embed into Notion
+
+This is already done and should automatically update, but if it needs redone, it's somewhat like follows:
+
+***NOTE:*** DO NOT CHANGE THE Published Content & Settings unless you know what you're doing. You will break existing embeds if you change this.
+
+1. click the three dots in the top right of the chart.
+1. select "publish chart"
+1. Select the chart you wish to get the link for (in the first dropdown). Leave "Interactive" selected (in the second dropdown).
+1. Copy the link
+1. Go to Notion where you want to embed. Type `/embed` and select the generic embed "for PDFs, google maps, and more"
+1. Paste the link you copied from google sheets.
+

reports/constants.ts

@@ -0,0 +1,76 @@
+export const hostname = 'countly.ipfs.tech'
+const { env: { COUNTLY_USERNAME, COUNTLY_PASSWORD } } = process
+const authorizationHeader = `Basic ${
+  Buffer
+    .from(`${COUNTLY_USERNAME}:${COUNTLY_PASSWORD}`)
+    .toString('base64')
+}`
+export const baseOptions = {
+  method: 'GET',
+  headers: {
+    accept: 'application/json',
+    authorization: authorizationHeader
+  }
+}
+
+async function getApiKey (): Promise<string> {
+  const response = await fetch(`https://${hostname}/api-key`, {
+    ...baseOptions,
+    headers: {
+      ...baseOptions.headers,
+      accept: 'text/plain'
+    }
+  })
+
+  try {
+    return await response.text()
+  } catch (e) {
+    console.error('Could not get API key from Countly', e)
+    throw e
+  }
+}
+
+export const apiKey = await getApiKey()
+
+/**
+ * 90 days of data
+ */
+export const daysOfDataInMs = 1000 * 60 * 60 * 24 * 90
+
+export const appIds = {
+  // Webui.ipfs.io
+  'ipfs-webui': '5c6e72803fd4432348b8119c',
+
+  // webui-kubo
+  'ipfs-webui-kubo': '63c596762a7760344a6b2cfd',
+
+  // ipfs-desktop
+  'ipfs-desktop': '5c6ec2b13fd4432348b811a0',
+
+  // ipfs-companion
+  'ipfs-companion': '639cbbcf8e6f3439c3796738',
+
+  // public gateway checker
+  'public-gateway-checker': '6345a52a31fdc11369a2f2db',
+
+  // starmap.site
+  'starmap.site': '639915ff21fd4330c469a191',
+
+  // cid-utils-website
+  'cid-utils-website': '63cf2d6ed09125d219d3d86c',
+
+  // explore.ipld.io
+  'explore.ipld.io': '63cef029d09125d219d3d69a',
+
+  // ipfs-check
+  'ipfs-check': '63d039e622fb279599709b09',
+
+  // ipfs-dag-builder-vis
+  'ipfs-dag-builder-vis': '63cee76ad09125d219d3d640',
+
+  // pinning-service-compliance
+  'pinning-service-compliance': '63cf08ccd09125d219d3d776',
+
+  // pl-diagnose
+  'pl-diagnose': '63cef095d09125d219d3d6a6'
+}

reports/doCountlyFetch.ts

@@ -0,0 +1,21 @@
+import { apiKey, baseOptions, hostname } from './constants.js';
+
+export async function doCountlyFetch({
+  appId,
+  extraParams,
+  fetchOptions = {},
+  path = '/o'
+}: {
+  appId: string,
+  extraParams: string,
+  fetchOptions?: RequestInit,
+  path: string
+}) {
+  const response = await fetch(`https://${hostname}${path}?api_key=${apiKey}&app_id=${appId}&${extraParams}`, {...baseOptions, ...fetchOptions});
+  try {
+    return await response.json();
+  } catch (e) {
+    console.error(`Could not fetch data from https://${hostname}${path}`, e);
+    throw e;
+  }
+}

reports/downloadDashboardData.ts

@@ -0,0 +1,104 @@
+import { writeFile } from 'node:fs/promises'
+
+import { appIds, daysOfDataInMs } from './constants.js'
+import { doCountlyFetch } from './doCountlyFetch.js'
+import type { googleSheetData } from './types.js'
+
+interface ActiveUsersResponse {
+  calculating: boolean
+  data: Record<string, {
+    d: number
+    w: number
+    m: number
+  }>
+}
+
+interface DailyData {
+  _id: string
+  date: Date
+  d: number
+  w: number
+  m: number
+}
+
+export interface DashboardData {
+  daily: googleSheetData
+  weekly: googleSheetData
+  monthly: googleSheetData
+}
+
+/**
+ * Print out a CSV of the number of unique users per day for each app
+ */
+export async function downloadDashboardData ({ writeFiles = false }: { writeFiles?: boolean } = {}): Promise<DashboardData> {
+  const todayEpoch = Date.now()
+  const dailyArray: googleSheetData = []
+  const weeklyArray: googleSheetData = []
+  const monthlyArray: googleSheetData = []
+  let headers: googleSheetData[0]
+  const results = await Promise.all(Object.entries(appIds).map(async ([appName, appId]) => {
+    let response: ActiveUsersResponse = { calculating: true, data: {} }
+    while (response.calculating) {
+      /**
+       * @see https://api.count.ly/reference/oanalyticssessions
+       */
+      response = await doCountlyFetch({ path: '/o/active_users', appId, extraParams: `period=[${todayEpoch - daysOfDataInMs}, ${todayEpoch}]` })
+      // eslint-disable-next-line no-console
+      console.log(`${appName} calculating? `, response.calculating)
+      if (response.calculating) {
+        await new Promise((resolve) => setTimeout(resolve, 4000))
+      }
+    }
+    const activeUserData: DailyData[] = []
+    for (const [key, value] of Object.entries(response.data)) {
+      activeUserData.push({
+        _id: key,
+        date: new Date(key),
+        ...value
+      })
+    }
+
+    activeUserData.sort((a, b) => a.date.getTime() - a.date.getTime())
+
+    // output the name of the app as row headers and the date labels as column headers
+    if (headers == null) {
+      headers = ['App Name', ...activeUserData.map((day) => day.date.toISOString().split('T')[0])]
+      dailyArray.push(headers)
+      weeklyArray.push(headers)
+      monthlyArray.push(headers)
+    }
+    return {
+      appName,
+      daily: [appName, ...activeUserData.map((day) => day.d)],
+      weekly: [appName, ...activeUserData.map((day) => day.w)],
+      monthly: [appName, ...activeUserData.map((day) => day.m)]
+    }
+
+  }))
+
+  // now ensure that the arrays are in the same order as appIds
+  for (const [appName,] of Object.entries(appIds)) {
+    const result = results.find((result) => result.appName === appName)
+    if (result) {
+      dailyArray.push(result.daily)
+      weeklyArray.push(result.weekly)
+      monthlyArray.push(result.monthly)
+    }
+  }
+
+  if (writeFiles) {
+    // Write the outputs to their appropriate Csv files
+    await writeFile('./output/activeUsers-daily.json', JSON.stringify(dailyArray, null, 2))
+    await writeFile('./output/activeUsers-daily.csv', dailyArray.join('\n').toString())
+    await writeFile('./output/activeUsers-weekly.json', JSON.stringify(weeklyArray, null, 2))
+    await writeFile('./output/activeUsers-weekly.csv', weeklyArray.join('\n').toString())
+    await writeFile('./output/activeUsers-monthly.json', JSON.stringify(monthlyArray, null, 2))
+    await writeFile('./output/activeUsers-monthly.csv', monthlyArray.join('\n').toString())
+  }
+
+  return {
+    daily: dailyArray,
+    weekly: weeklyArray,
+    monthly: monthlyArray
+  }
+}

reports/getApiKey.ts

@@ -0,0 +1,13 @@
+import { baseOptions, hostname } from './constants.js'
+
+export async function getApiKey (): Promise<string> {
+  const response = await fetch(`https://${hostname}/api-key`, {
+    ...baseOptions,
+    headers: {
+      ...baseOptions.headers,
+      accept: 'text/plain'
+    }
+  })
+
+  return await response.text()
+}

reports/index.ts

@@ -0,0 +1,10 @@
+import { downloadDashboardData } from './downloadDashboardData.js'
+import { updateSheet } from './updateGoogleSheets.js'
+
+const {daily, weekly, monthly} = await downloadDashboardData()
+
+await Promise.all([
+  updateSheet('Daily Active Users', daily),
+  updateSheet('Weekly Active Users', weekly),
+  updateSheet('Monthly Active Users', monthly)
+])