Merge pull request #215 from savetheclocktower/add-owner-column

Add `owner` field to `packages` table…

.eslintrc.js

@@ -15,20 +15,11 @@ module.exports = {
       {
         allowModules: [
           "supertest",
-          "../node_modules/@databases/pg-test/jest/globalSetup",
-          "../node_modules/@databases/pg-test/jest/globalTeardown",
         ],
       },
     ],
     "no-process-exit": "off",
     // Custom Rules as Determined by the maintainers of package-backend
-    camelcase: [
-      "error",
-      {
-        ignoreGlobals: true,
-        ignoreImports: true,
-      },
-    ],
     complexity: ["error"],
     eqeqeq: ["error", "smart"],
     "max-depth": ["error", 4],

docs/build.md

@@ -98,9 +98,9 @@ There are some additional scripts that you likely won't encounter or need during
 
 #### Limiting what's mocked in the dev server
 
-No additionally, when working locally if you need to test a feature where you have no option but to query the remote service (This is not recommended) you do have the option to enable some remote services as needed.
+When working locally if you need to test a feature where you have no option but to query the remote service (this is not recommended) you do have the option to enable some remote services as needed.
 
-Again setting any of the below values will contact remote services, meaning you must have the proper Configuration Values set to do so, and you become at risk of causing damage to external data, any deletions that occur here will cause permanent data loss to these services. Or may get you blocked by remote services if there is abuse.
+Again, setting any of the below values will contact remote services, meaning you must have the proper Configuration Values set to do so, and you become at risk of causing damage to external data, any deletions that occur here will cause permanent data loss to these services. Or may get you blocked by remote services if there is abuse.
 
 The following values can only ever be started when using `npm run start:dev` and will be ignored in all other instances.
 

package.json

@@ -47,6 +47,7 @@
     "express": "^4.18.1",
     "express-rate-limit": "^6.7.0",
     "js-yaml": "^4.1.0",
+    "parse-github-url": "^1.0.2",
     "postgres": "^3.3.4",
     "superagent": "^8.0.9"
   },

scripts/database/create_packages_table.sql

@@ -60,3 +60,12 @@ CREATE TRIGGER trigger_now_on_updated
     BEFORE UPDATE ON packages
     FOR EACH ROW
 EXECUTE PROCEDURE now_on_updated_package();
+
+
+-- Add an `owner` field to the table
+
+-- GitHub username length limit is apparently 39. For future-proofing, we'll
+-- envision a prefix that specifies an alternative VCS, so let's build in some
+-- wiggle room.
+ALTER TABLE packages
+ADD COLUMN owner VARCHAR(60);

scripts/migrations/0001-initial-migration.sql

@@ -14,6 +14,7 @@ CREATE TABLE packages (
     downloads BIGINT NOT NULL DEFAULT 0,
     stargazers_count BIGINT NOT NULL DEFAULT 0,
     original_stargazers BIGINT NOT NULL DEFAULT 0,
+    owner VARCHAR(60),
     data JSONB,
     -- constraints
     CONSTRAINT lowercase_names CHECK (name = LOWER(name))

scripts/tools/add-owner-field.js

@@ -0,0 +1,162 @@
+const fs = require("fs");
+const postgres = require("postgres");
+const parseGithubUrl = require("parse-github-url");
+const { DB_HOST, DB_USER, DB_PASS, DB_DB, DB_PORT, DB_SSL_CERT } =
+  require("../../src/config.js").getConfig();
+
+let sqlStorage;
+
+const LIMIT = parseInt(process.env.LIMIT ?? '-1', 10);
+const OFFSET = parseInt(process.env.OFFSET ?? '0', 10);
+const VERBOSE = (process.env.VERBOSE ?? '0') !== '0';
+
+function log (...args) {
+  if (!VERBOSE) return;
+  return console.log(...args);
+}
+
+function debug (...args) {
+  if (!VERBOSE) return;
+  return console.debug(...args);
+}
+
+function warn (...args) {
+  if (!VERBOSE) return;
+  return console.warn(...args);
+}
+
+
+async function init () {
+  let allPointers = await getPointers();
+  let totalPointers = allPointers.length;
+  log('Package count:', totalPointers);
+
+  for (let { name, pointer } of allPointers) {
+    log(`Checking: ${name}::${pointer}`);
+
+    if (typeof name !== "string") {
+      console.error(
+        `The package ${name}::${pointer} is invalid without its name!`
+      );
+      continue;
+    }
+    if (typeof pointer !== "string") {
+      console.error(
+        `The package ${name}::${pointer} likely has been deleted.`
+      );
+      continue;
+    }
+
+    let { ok, content: pack } = await getPackageData(pointer);
+    if (!ok) {
+      warn(`Error getting package ${name}!`);
+      continue;
+    }
+
+    let meta = pack?.data;
+    let repositoryUrl = meta?.repository.url;
+    if (!repositoryUrl) {
+      console.error(`No repository URL found for package ${name}!`);
+      continue;
+    }
+
+    let parsed = parseGithubUrl(repositoryUrl);
+    if (parsed === null) {
+      console.error(`Could not parse repo URL for package: ${name}: ${repositoryUrl}`);
+      continue;
+    }
+
+    let { owner } = parsed;
+    if (pack.owner === owner) {
+      debug(`Package ${name} already has the right owner!`);
+      continue;
+    }
+
+    log('Updating owner field of package:', name);
+    await sqlStorage`
+      UPDATE packages SET owner = ${owner} WHERE pointer = ${pointer};
+    `;
+  }
+
+  await sqlEnd();
+  process.exit(0);
+}
+
+async function getPointers() {
+  sqlStorage ??= setupSQL();
+
+  let command;
+
+  if (LIMIT > 0) {
+    command = await sqlStorage`
+      SELECT * FROM names LIMIT ${LIMIT} OFFSET ${OFFSET};
+    `;
+  } else {
+    command = await sqlStorage`
+      SELECT * FROM names;
+    `;
+  }
+
+  if (command.count === 0) {
+    console.log("Failed to get all package pointers.");
+    await sqlEnd();
+    process.exit(1);
+  }
+
+  return command;
+}
+
+async function getPackageData(pointer) {
+  console.log('getting package data:', pointer);
+
+  try {
+    sqlStorage ??= setupSQL();
+    const command = await sqlStorage`
+      SELECT * from packages
+      WHERE pointer = ${pointer}
+    `;
+
+    if (command.count === 0) {
+      return { ok: false, content: `Failed to get package data of ${pointer}` };
+    }
+
+    return { ok: true, content: command[0] };
+  } catch (err) {
+    console.error(`ERROR!`);
+    console.error(err);
+  }
+}
+
+function setupSQL() {
+  try {
+    let options = {
+      host: DB_HOST,
+      username: DB_USER,
+      password: DB_PASS,
+      database: DB_DB,
+      port: DB_PORT,
+    };
+    if (DB_SSL_CERT) {
+      options.ssl = {
+        rejectUnauthorized: true,
+        ca: fs.readFileSync(DB_SSL_CERT).toString(),
+      };
+    }
+    sqlStorage = postgres(options);
+
+    return sqlStorage;
+  } catch (err) {
+    console.error(err);
+    process.exit(100);
+  }
+}
+
+async function sqlEnd() {
+  if (sqlStorage !== undefined) {
+    await sqlStorage.end();
+    console.log("Task done!");
+  }
+  return;
+}
+
+module.exports = init;

src/PackageObject.js

@@ -1,3 +1,5 @@
+const parseGithubUrl = require('parse-github-url');
+
 /**
  * @module PackageObject
  * @desc This Module is used to aide in building Package Objects.
@@ -26,6 +28,7 @@ class PackageObject {
     this.stargazers_count = undefined;
     this.readme = undefined;
     this.creationMethod = undefined;
+    this.owner = undefined;
   }
 
   /**
@@ -167,6 +170,10 @@ class PackageObject {
    */
   setRepositoryURL(repoURL) {
     this.repository.url = repoURL;
+    let parsed = parseGithubUrl(repoURL);
+    if (parsed) {
+      this.owner = parsed.owner;
+    }
     return this;
   }
 

src/controllers/getPackages.js

@@ -23,7 +23,10 @@ module.exports = {
     serviceType: (context, req) => { return context.query.serviceType(req); },
     service: (context, req) => { return context.query.service(req); },
     serviceVersion: (context, req) => { return context.query.serviceVersion(req); },
-    fileExtension: (context, req) => { return context.query.fileExtension(req); }
+    fileExtension: (context, req) => { return context.query.fileExtension(req); },
+    owner: (context, req) => {
+      return context.query.owner(req);
+    }
   },
 
   /**
@@ -51,7 +54,8 @@ module.exports = {
 
     const ssoP = new context.ssoPaginate();
 
-    ssoP.total = packages.pagination.total;
+    ssoP.resultCount = packages.pagination.count;
+    ssoP.totalPages = packages.pagination.total;
     ssoP.limit = packages.pagination.limit;
     ssoP.buildLink(`${context.config.server_url}/api/packages`, packages.pagination.page, params);
 

src/controllers/getPackagesSearch.js

@@ -84,7 +84,8 @@ module.exports = {
 
     const ssoP = new context.ssoPaginate();
 
-    ssoP.total = packs.pagination.total;
+    ssoP.resultCount = packs.pagination.count;
+    ssoP.totalPages = packs.pagination.total;
     ssoP.limit = packs.pagination.limit;
     ssoP.buildLink(`${context.config.server_url}/api/packages/search`, packs.pagination.page, params);
 

src/controllers/getThemes.js

@@ -56,7 +56,8 @@ module.exports = {
 
     const ssoP = new context.ssoPaginate();
 
-    ssoP.total = packages.pagination.total;
+    ssoP.resultCount = packages.pagination.count;
+    ssoP.totalPages = packages.pagination.total;
     ssoP.limit = packages.pagination.limit;
     ssoP.buildLink(`${context.config.server_url}/api/themes`, packages.pagination.page, params);