Skip to content

Commit

Permalink
fix: address gh search bug with the old (slower) scan implementation
Browse files Browse the repository at this point in the history
  • Loading branch information
smellai committed Oct 27, 2022
1 parent 592bbd9 commit 8304b67
Show file tree
Hide file tree
Showing 4 changed files with 116 additions and 4 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ Archived repositories are filtered out.
> :warning: GitHub API are rate limited, and search API in particular has the additional [secondary rate limit](https://docs.github.com/en/rest/overview/resources-in-the-rest-api#secondary-rate-limits). _package-adoption_ implements the [best practices guidelines](https://docs.github.com/en/rest/guides/best-practices-for-integrators#dealing-with-secondary-rate-limits) to deal with it, but you should know that limitations could happen in any case.
> :warning: GitHub search API are not 100% reliable and sometimes return deleted / outdated files or multiple versions of the same file. The library version in the output could be inaccurate for this reason.
> There is a [known issue](https://github.com/community/community/discussions/20633#discussioncomment-3735796) with package names with a scope containing hyphen character, e.g. `@typescript-eslint/parser`.
> There is a [known issue](https://github.com/community/community/discussions/20633#discussioncomment-3735796) with package names with a scope containing hyphen character, e.g. `@typescript-eslint/parser`. A slower version of the main scan function will be automatically used to handle this case.
## Install

Expand Down
4 changes: 2 additions & 2 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 8 additions & 1 deletion src/getFilteredReposWithPackageForOrg.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { readPackageJson } from './readPackageJson';
import { init, octokit } from './octokitInit';
import { isStale } from './isRepoStale';
import { validateConfig } from './validateConfig';
import { getFilteredReposWithPackageForOrgSlower } from './getFilteredReposWithPackageForOrgSlower';

/**
* It takes an organization name and a package name, and returns a list of all the repositories in that
Expand Down Expand Up @@ -47,7 +48,13 @@ export const getFilteredReposWithPackageForOrg = async (

for await (const { data: items } of foundPackageJsonFiles) {
if (items.length === 0) {
console.log(`[package-adoption]: No results for ${pkgName}`);
// workaround to address github search issue https://github.com/community/community/discussions/20633#discussioncomment-3735796
const repositoriesWithPackage =
await getFilteredReposWithPackageForOrgSlower({ ...config }, octokit);
if (repositoriesWithPackage?.length === 0) {
console.log(`[package-adoption]: No results for ${pkgName}`);
}
return repositoriesWithPackage;
}

for (let i = 0; i < items.length; i++) {
Expand Down
105 changes: 105 additions & 0 deletions src/getFilteredReposWithPackageForOrgSlower.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
import path from 'path';
import { GetResponseDataTypeFromEndpointMethod } from '@octokit/types';
import { ErrorWithResponse, InputParameters, RelevantRepo } from './types';
import { readPackageJson } from './readPackageJson';
import { isStale } from './isRepoStale';
import { Octokit } from '@octokit/rest';

/**
* It takes an organization name and a package name, and returns a list of all the repositories in that
* organization that have that package in their package.json file
* @param {InputParameters} config - InputParameters
* @returns An array of objects with the installation info for the library
*/
export const getFilteredReposWithPackageForOrgSlower = async (
config: InputParameters,
octokit: Octokit
): Promise<RelevantRepo[] | undefined> => {
const { org, daysUntilStale = 365, pkgName } = config;
const repositoriesWithPackage: RelevantRepo[] = [];

type IteratorResponseItemDataType = GetResponseDataTypeFromEndpointMethod<
typeof octokit.repos.get
>;

try {
/* The plain listForOrg API just returns 30 items (a page), we need paginate iterator to get the whole list */
const allRepos = octokit.paginate.iterator<IteratorResponseItemDataType>(
'GET /orgs/:org/repos',
{
org,
type: 'all',
}
);

for await (const { data: repos } of allRepos) {
for (let i = 0; i < repos.length; i++) {
const repo = repos[i];

if (
repo.archived === false &&
(repo.language === 'TypeScript' || repo.language === 'JavaScript') &&
!isStale(repo.pushed_at, daysUntilStale)
) {
try {
const packageJsonResponse = await octokit.search.code({
q: `repo:${org}/${repo.name}+filename:package.json`,
});
const foundFiles = packageJsonResponse.data.items;

for (let i = 0; i < foundFiles.length; i++) {
const packageJsonFile = foundFiles[i];

const pathDirParts = packageJsonFile.path.split('/').slice(0, -1);

const installationPath =
pathDirParts?.length > 0 ? path.join(...pathDirParts) : 'root';

if (
// The search matches package-lock too
packageJsonFile.name === 'package.json' &&
// sometimes GitHub search returns multiple versions of the same file
!repositoriesWithPackage.find(
(relevantRepo) =>
relevantRepo.name === repo.name &&
relevantRepo.installationPath === installationPath
)
) {
const packageJsonData = await readPackageJson({
org,
repoName: repo.name,
pkgName,
packageJsonPath: packageJsonFile.path,
installationPath,
});
if (packageJsonData) {
repositoriesWithPackage.push(packageJsonData);
}
}
}
} catch (error) {
const typeSafeError = error as ErrorWithResponse;

if (typeSafeError.response === null) {
console.error(error);
} else if (typeSafeError.response.status === 404) {
console.error(
`[package-adoption]: No package.json found.\n${typeSafeError?.response?.url}\n\n`
);
} else if (typeSafeError.response.status === 403) {
// API rate limit exceeded for user ID
console.error(
`[package-adoption]: ${typeSafeError?.response?.data.message}\n${typeSafeError?.response?.url}\n\n`
);
} else {
console.error(typeSafeError.response);
}
}
}
}
}
return repositoriesWithPackage;
} catch (error) {
console.error('[package-adoption]:', error);
}
};

0 comments on commit 8304b67

Please sign in to comment.