feat: introduce organic keyword audit post processor (SITES-19317)

src/backlinks/handler.js

@@ -13,17 +13,7 @@
 import { badRequest, noContent } from '@adobe/spacecat-shared-http-utils';
 import { hasText, isObject } from '@adobe/spacecat-shared-utils';
 import { uploadSlackFile } from '../support/slack.js';
-
-function convertToCSV(array) {
-  const headers = Object.keys(array[0]).join(',');
-  const rows = array.map((item) => Object.values(item).map((value) => {
-    if (typeof value === 'object' && value !== null) {
-      return `"${JSON.stringify(value)}"`;
-    }
-    return `"${value}"`;
-  }).join(',')).join('\r\n');
-  return `${headers}\r\n${rows}\r\n`;
-}
+import { convertToCSV } from '../support/utils.js';
 
 function isValidMessage(message) {
   return hasText(message.url)

src/index.js

@@ -18,12 +18,14 @@ import apex from './apex/handler.js';
 import cwv from './cwv/handler.js';
 import notFoundHandler from './notfound/handler.js';
 import backlinks from './backlinks/handler.js';
+import keywords from './organic-keywords/handler.js';
 
 export const HANDLERS = {
   apex,
   cwv,
   404: notFoundHandler,
   'broken-backlinks': backlinks,
+  'organic-keywords': keywords,
 };
 
 function guardEnvironmentVariables(fn) {

src/organic-keywords/handler.js

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { badRequest, noContent } from '@adobe/spacecat-shared-http-utils';
+import { hasText, isObject } from '@adobe/spacecat-shared-utils';
+import { convertToCSV } from '../support/utils.js';
+import { uploadSlackFile } from '../support/slack.js';
+
+function isValidMessage(message) {
+  return hasText(message.url)
+    && isObject(message.auditContext?.slackContext)
+    && isObject(message.auditResult);
+}
+export default async function organicKeywordsHandler(message, context) {
+  const { log } = context;
+  const { auditResult, auditContext, url } = message;
+  const { env: { SLACK_BOT_TOKEN: token } } = context;
+
+  log.info(`Received organic keywords audit request for ${url}`);
+  if (!isValidMessage(message)) {
+    return badRequest('Required parameters missing in the message.');
+  }
+
+  const { keywords } = auditResult;
+  if (!keywords || keywords.length === 0) {
+    log.info(`${url} does not have any organic keywords.`);
+    return noContent();
+  }
+
+  log.info(`Found ${keywords.length} organic keywords for ${url}`);
+
+  const {
+    increasedPositions,
+    decreasedPositions,
+    keywordsReadable,
+  } = keywords.reduce((acc, keyword) => {
+    const item = { ...keyword };
+    item.best_position_diff = item.best_position_diff || 'New';
+    if (item.best_position_diff < 0) acc.increasedPositions += 1;
+    else if (item.best_position_diff > 0) acc.decreasedPositions += 1;
+    acc.keywordsReadable.push(item);
+    return acc;
+  }, { increasedPositions: 0, decreasedPositions: 0, keywordsReadable: [] });
+
+  const csvData = convertToCSV(keywordsReadable);
+  const file = new Blob([csvData], { type: 'text/csv' });
+
+  try {
+    const { channel, ts } = auditContext.slackContext;
+    const fileName = `organic-keywords-${url.split('://')[1]?.replace(/\./g, '-')}-${new Date().toISOString().split('T')[0]}.csv`;
+    const text = `Organic Keywords report for *${url.split('://')[1]}*. There are a total of ${increasedPositions} keywords that rank higher and ${decreasedPositions} keywords thar rank lower than the previous month.`;
+    await uploadSlackFile(token, {
+      file, fileName, channel, ts, text,
+    });
+  } catch (e) {
+    log.error(`Failed to send slack message to report organic keywords for ${url}: ${e.message}`);
+  }
+
+  return noContent();
+}

src/support/slack.js

@@ -70,7 +70,7 @@ export async function uploadSlackFile(token, opts) {
     const formData = new FormData();
     formData.append('token', token);
     formData.append('channels', channel);
-    formData.append('thread_ts', ts);
+    if (ts) formData.append('thread_ts', ts);
     formData.append('file', file, fileName);
     formData.append('initial_comment', text);
 

src/support/utils.js

@@ -15,3 +15,25 @@ import { context as h2, h1 } from '@adobe/fetch';
 export const { fetch } = process.env.HELIX_FETCH_FORCE_HTTP1
   ? h1()
   : h2();
+
+/**
+ * Converts an array of objects to a CSV string.
+ *
+ * Each object in the array represents a row in the CSV.
+ * The keys of the first object in the array are used as column headers.
+ * The function handles nested objects, converting them to JSON strings.
+ * All values, including nested objects, are enclosed in double quotes.
+ *
+ * @param {Object[]} array - An array of objects to be converted into CSV format.
+ * @returns {string} A string in CSV format, where the first row contains the headers.
+ */
+export function convertToCSV(array) {
+  const headers = Object.keys(array[0]).join(',');
+  const rows = array.map((item) => Object.values(item).map((value) => {
+    if (typeof value === 'object' && value !== null) {
+      return `"${JSON.stringify(value)}"`;
+    }
+    return `"${value}"`;
+  }).join(',')).join('\r\n');
+  return `${headers}\r\n${rows}\r\n`;
+}

test/organic-keywords/handler.test.js

@@ -0,0 +1,158 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+/* eslint-env mocha */
+
+import chai from 'chai';
+import sinonChai from 'sinon-chai';
+import chaiAsPromised from 'chai-as-promised';
+import sinon from 'sinon';
+import nock from 'nock';
+import organicKeywordsHandler from '../../src/organic-keywords/handler.js';
+
+chai.use(sinonChai);
+chai.use(chaiAsPromised);
+const { expect } = chai;
+
+const sandbox = sinon.createSandbox();
+
+describe('Organic Keywords Tests', () => {
+  let message;
+  let context;
+
+  beforeEach('setup', () => {
+    message = {
+      url: 'https://foobar.com',
+      auditResult: {
+        keywords: [
+          {
+            keyword: 'foo',
+            sum_traffic: 123,
+            best_position: 1,
+            best_position_prev: 1,
+            best_position_diff: 0,
+          },
+          {
+            keyword: 'bar',
+            sum_traffic: 23,
+            best_position: 2,
+            best_position_prev: 7,
+            best_position_diff: -5,
+          },
+          {
+            keyword: 'bax',
+            sum_traffic: 3,
+            best_position: 8,
+            best_position_prev: 5,
+            best_position_diff: 3,
+          },
+        ],
+      },
+      auditContext: {
+        slackContext: {
+          channel: 'channel-id',
+          ts: 'thread-id',
+        },
+      },
+    };
+
+    context = {
+      log: {
+        info: sandbox.spy(),
+        warn: sandbox.spy(),
+        error: sandbox.spy(),
+      },
+      env: {
+        SLACK_BOT_TOKEN: 'token',
+      },
+    };
+  });
+
+  afterEach('clean', () => {
+    sandbox.restore();
+    nock.cleanAll();
+  });
+
+  it('should return 400 if message is missing url', async () => {
+    delete message.url;
+    const result = await organicKeywordsHandler(message, context);
+    expect(result.status).to.equal(400);
+  });
+
+  it('should return 400 if message is missing auditResult', async () => {
+    delete message.auditResult;
+    const result = await organicKeywordsHandler(message, context);
+    expect(result.status).to.equal(400);
+  });
+
+  it('should return 400 if message is missing auditContext', async () => {
+    delete message.auditContext;
+    const result = await organicKeywordsHandler(message, context);
+    expect(result.status).to.equal(400);
+  });
+
+  it('should return 400 if message is missing slackContext', async () => {
+    delete message.auditContext.slackContext;
+    const result = await organicKeywordsHandler(message, context);
+    expect(result.status).to.equal(400);
+  });
+
+  it('should return 204 if audit result does not have a keyword property', async () => {
+    delete message.auditResult.keywords;
+    const result = await organicKeywordsHandler(message, context);
+    expect(result.status).to.equal(204);
+    expect(context.log.info).to.have.been.calledWith('https://foobar.com does not have any organic keywords.');
+  });
+
+  it('should return 204 if there are no keywords in the audit result', async () => {
+    message.auditResult.keywords = [];
+    const result = await organicKeywordsHandler(message, context);
+    expect(result.status).to.equal(204);
+    expect(context.log.info).to.have.been.calledWith('https://foobar.com does not have any organic keywords.');
+  });
+
+  it('logs error when slack api fails to upload file', async () => {
+    nock('https://slack.com', {
+      reqheaders: {
+        authorization: `Bearer ${context.env.SLACK_BOT_TOKEN}`,
+      },
+    })
+      .post('/api/files.upload')
+      .times(1)
+      .reply(500);
+
+    const result = await organicKeywordsHandler(message, context);
+    expect(result.status).to.equal(204);
+    expect(context.log.error).to.have.been.calledWith('Failed to send slack message to report organic keywords for https://foobar.com: '
+      + 'Failed to upload file to slack. Reason: Slack upload file API request failed. Status: 500');
+  });
+
+  it('should send slack message with keywords', async () => {
+    nock('https://slack.com', {
+      reqheaders: {
+        authorization: `Bearer ${context.env.SLACK_BOT_TOKEN}`,
+      },
+    })
+      .post('/api/files.upload')
+      .times(1)
+      .reply(200, {
+        ok: true,
+        file: {
+          url_private: 'slack-file-url',
+        },
+      });
+
+    const result = await organicKeywordsHandler(message, context);
+    expect(result.status).to.equal(204);
+    expect(context.log.error).to.not.have.been.called;
+  });
+});