Dev

.github/workflows/ci.yml

@@ -13,12 +13,12 @@ concurrency:
 jobs:
   main:
     name: Nx Cloud - Main Job
-    uses: nrwl/ci/.github/workflows/nx-cloud-main.yml@v0.14.0
+    uses: nrwl/ci/.github/workflows/nx-cloud-main.yml@v0.15.0
     secrets:
       NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
       NX_CLOUD_AUTH_TOKEN: ${{ secrets.NX_CLOUD_AUTH_TOKEN }}
     with:
-      node-version: 20.11.1
+      node-version: 22.8.0
       number-of-agents: 3
       init-commands: |
         npx nx-cloud start-ci-run --stop-agents-after="build" --agent-count=3
@@ -28,10 +28,10 @@ jobs:
 
   agents:
     name: Nx Cloud - Agents
-    uses: nrwl/ci/.github/workflows/nx-cloud-agents.yml@v0.11.3
+    uses: nrwl/ci/.github/workflows/nx-cloud-agents.yml@v0.15.0
     secrets:
       NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
       NX_CLOUD_AUTH_TOKEN: ${{ secrets.NX_CLOUD_AUTH_TOKEN }}
     with:
-      node-version: 20.11.1
+      node-version: 22.8.0
       number-of-agents: 3

.github/workflows/docker-publish.yml

@@ -24,14 +24,14 @@ jobs:
 
       - uses: actions/setup-node@v4
         with:
-          node-version: 20.11.1
+          node-version: 22.8.0
 
       - name: Use the node_modules cache if available [npm]
         id: use-npm-cache
         uses: actions/cache@v4
         with:
           path: node_modules
-          key: node-20.11.1-modules-${{ hashFiles('**/package-lock.json') }}
+          key: node-22.8.0-modules-${{ hashFiles('**/package-lock.json') }}
 
       - name: Install dependencies
         if: steps.use-npm-cache.outputs.cache-hit != 'true'
@@ -60,7 +60,7 @@ jobs:
         uses: docker/setup-buildx-action@v3
 
       - name: Build and push Docker image (web)
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: docker/performance_base/Dockerfile
@@ -71,7 +71,7 @@ jobs:
           cache-to: type=gha,mode=max
 
       - name: Build and push Docker image (nginx)
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: docker/performance_base/Dockerfile
@@ -82,7 +82,7 @@ jobs:
           cache-to: type=gha,mode=max
 
       - name: Build and push Docker image (db-updater)
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: docker/performance_base/Dockerfile

docker/performance_base/Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.3
-FROM node:18-alpine3.17 as builder
-WORKDIR builder
+FROM node:22.8-alpine3.20 as builder
+WORKDIR /builder
 VOLUME ["/builder"]
 COPY ["package.json", "package-lock.json", "./"]
 RUN --mount=type=cache,target=/home/node/.npm npm ci
@@ -12,11 +12,11 @@ RUN npx nx run-many --target=build --projects=api,upd,db-updater,db-cli --prod -
 RUN npx nx daemon --stop
 RUN rm -r /builder/node_modules && rm -r /builder/tmp
 
-FROM alpine:latest as apps
-WORKDIR apps
+FROM alpine:3.20 as apps
+WORKDIR /apps
 COPY --from=builder /builder/dist/apps/ .
 
-FROM node:18-alpine3.17 as node_base
+FROM node:22.8-alpine3.20 as node_base
 RUN apk update && apk add tzdata
 ENV TZ=America/Toronto
 
@@ -57,10 +57,6 @@ FROM node_base as db-updater-dev
 RUN echo -e '#!/bin/sh\nnode db-cli/main.js "$@"' >> /bin/db-cli && chmod +x /bin/db-cli
 COPY --from=db-updater-dev-base . .
 COPY dist/apps/db-updater/ .
-#RUN apk update && \
-#    apk upgrade && \
-#    apk add bash && \
-#    apk add nano
 CMD ["node", "main.js"]
 
 FROM nginx:alpine as nginx

libs/db-update/src/lib/db-update.service.ts

@@ -474,6 +474,19 @@ export class DbUpdateService {
     }) as DateRange<Date>[];
 
     try {
+      this.logger.info('Clearing unneeded data');
+
+      const deletedPages = await this.db.views.pages.clearUnusedDateRanges(
+        dateRangesWithComparison,
+      );
+      const deletedTasks = await this.db.views.tasks.clearUnusedDateRanges(
+        dateRangesWithComparison,
+      );
+
+      this.logger.info(
+        `Deleted ${deletedPages} PageView documents and ${deletedTasks} TasksView documents`,
+      );
+
       this.logger.info('Loading data for tasks metrics store');
 
       await this.db.views.tasks.loadStoreData();

libs/db/src/lib/db.views.new.ts

@@ -318,4 +318,44 @@ export abstract class DbViewNew<
    * @returns mongo.DeleteResult
    */
   abstract clearNonExisting(): Promise<mongo.DeleteResult | null>;
+
+  /**
+   * Method to delete all documents where the dateRange is not in the provided list
+   * @param dateRanges The list of dateRanges to keep
+   * @returns The number of documents deleted
+   */
+  async clearUnusedDateRanges(dateRanges: DateRange<Date>[]) {
+    const dbDateRanges = await this._model
+      .distinct<DateRange<Date>>('dateRange')
+      .exec();
+
+    const dateRangeStrings = dateRanges.map((dateRange) =>
+      JSON.stringify(dateRange),
+    );
+
+    const dateRangesToDelete = dbDateRanges
+      .map((dateRange) =>
+        dateRangeStrings.includes(JSON.stringify(dateRange)) ? null : dateRange,
+      )
+      .filter((dateRange) => dateRange !== null);
+
+    if (!dateRangesToDelete.length) {
+      return 0;
+    }
+
+    const deletionDateRangeStrings = dateRangesToDelete.map(
+      ({ start, end }) =>
+        `${start.toISOString().slice(0, 10)}/${end.toISOString().slice(0, 10)}`,
+    );
+
+    console.log(`Deleting date ranges: ${deletionDateRangeStrings}`);
+
+    return await Promise.all(
+      dateRangesToDelete.map((dateRange) =>
+        this._model.deleteMany({ dateRange }),
+      ),
+    ).then((results) =>
+      results.reduce((acc, { deletedCount }) => acc + deletedCount, 0),
+    );
+  }
 }