Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(app/web): bypass metrics.api.github.overuse with OAuth #1171

Merged
merged 3 commits into from
Aug 6, 2022
Merged

feat(app/web): bypass metrics.api.github.overuse with OAuth #1171

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ad65607
draft oauth
lowlighter Aug 2, 2022
1d4f444
codql
lowlighter Aug 2, 2022
6f065ef
continue
lowlighter Aug 5, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 10 additions & 7 deletions source/app/metrics/metadata.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,9 @@ metadata.plugin = async function({__plugins, __templates, name, logger}) {
if ((meta.category === "github")&&(!meta.disclaimer))
meta.disclaimer = "This plugin is not affiliated, associated, authorized, endorsed by, or in any way officially connected with [GitHub](https://github.com).\nAll product and company names are trademarks™ or registered® trademarks of their respective holders."

//Deprecation
meta.deprecated = !!meta?.deprecation

//Inputs parser
{
meta.inputs = function({data: {user = null} = {}, q, account}, defaults = {}) {
Expand Down Expand Up @@ -345,30 +348,30 @@ metadata.plugin = async function({__plugins, __templates, name, logger}) {
//Web metadata
{
meta.web = Object.fromEntries(
Object.entries(inputs).map(([key, {type, description: text, example, default: defaulted, min = 0, max = 9999, values}]) => [
Object.entries(inputs).map(([key, {type, description: text, example, default: defaulted, min = 0, max = 9999, values, extras}]) => [
//Format key
metadata.to.query(key),
//Value descriptor
(() => {
switch (type) {
case "boolean":
return {text, type: "boolean", defaulted: /^(?:[Tt]rue|[Oo]n|[Yy]es|1)$/.test(defaulted) ? true : /^(?:[Ff]alse|[Oo]ff|[Nn]o|0)$/.test(defaulted) ? false : defaulted}
return {text, type: "boolean", defaulted: /^(?:[Tt]rue|[Oo]n|[Yy]es|1)$/.test(defaulted) ? true : /^(?:[Ff]alse|[Oo]ff|[Nn]o|0)$/.test(defaulted) ? false : defaulted, extras}
case "number":
return {text, type: "number", min, max, defaulted}
return {text, type: "number", min, max, defaulted, extras}
case "array":
return {text, type: "text", placeholder: example ?? defaulted, defaulted}
return {text, type: "text", placeholder: example ?? defaulted, defaulted, extras}
case "string": {
if (Array.isArray(values))
return {text, type: "select", values, defaulted}
return {text, type: "text", placeholder: example ?? defaulted, defaulted}
return {text, type: "text", placeholder: example ?? defaulted, defaulted, extras}
}
case "json":
return {text, type: "text", placeholder: example ?? defaulted, defaulted}
return {text, type: "text", placeholder: example ?? defaulted, defaulted, extras}
default:
return null
}
})(),
]).filter(([key, value]) => (value) && (key !== name)),
]).filter(([key, value]) => (value)&&(!((name === "base")&&(key === "repositories")))),
)
}

Expand Down
163 changes: 143 additions & 20 deletions source/app/web/instance.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,13 @@ import express from "express"
import ratelimit from "express-rate-limit"
import cache from "memory-cache"
import util from "util"
import url from "url"
import axios from "axios"
import mocks from "../../../tests/mocks/index.mjs"
import metrics from "../metrics/index.mjs"
import presets from "../metrics/presets.mjs"
import setup from "../metrics/setup.mjs"
import crypto from "crypto"

/**App */
export default async function({sandbox = false} = {}) {
Expand Down Expand Up @@ -55,7 +58,20 @@ export default async function({sandbox = false} = {}) {
//Apply mocking if needed
if (mock)
Object.assign(api, await mocks(api))
const {graphql, rest} = api
//Custom user octokits sessions
const authenticated = new Map()
const uapi = session => {
if (!/^[a-f0-9]+$/i.test(`${session}`))
return null
if (authenticated.has(session)) {
const {login, token} = authenticated.get(session)
console.debug(`metrics/app/session/${login} > authenticated with session ${session.substring(0, 6)}, using custom octokit`)
return {login, graphql: octokit.graphql.defaults({headers: {authorization: `token ${token}`}}), rest: new OctokitRest.Octokit({auth: token})}
}
else if (session)
console.debug(`metrics/app/session > unknown session ${session.substring(0, 6)}, using default octokit`)

Check failure

Code scanning / CodeQL

Log injection

[User-provided value](1) flows to log entry. [User-provided value](2) flows to log entry. [User-provided value](3) flows to log entry.
return null
}

//Setup server
const app = express()
Expand Down Expand Up @@ -87,22 +103,19 @@ export default async function({sandbox = false} = {}) {
const limiter = ratelimit({max: debug ? Number.MAX_SAFE_INTEGER : 60, windowMs: 60 * 1000, headers: false})
const metadata = Object.fromEntries(
Object.entries(conf.metadata.plugins)
.map(([key, value]) => [key, Object.fromEntries(Object.entries(value).filter(([key]) => ["name", "icon", "category", "web", "supports", "scopes"].includes(key)))])
.map(([key, value]) => [key, Object.fromEntries(Object.entries(value).filter(([key]) => ["name", "icon", "category", "web", "supports", "scopes", "deprecated"].includes(key)))])
.map(([key, value]) => [key, key === "core" ? {...value, web: Object.fromEntries(Object.entries(value.web).filter(([key]) => /^config[.]/.test(key)).map(([key, value]) => [key.replace(/^config[.]/, ""), value]))} : value]),
)
const enabled = Object.entries(metadata).filter(([_name, {category}]) => category !== "core").map(([name]) => ({name, category: metadata[name]?.category ?? "community", enabled: plugins[name]?.enabled ?? false}))
const enabled = Object.entries(metadata).filter(([_name, {category}]) => category !== "core").map(([name]) => ({name, category: metadata[name]?.category ?? "community", deprecated: metadata[name]?.deprecated ?? false, enabled: plugins[name]?.enabled ?? false}))
const templates = Object.entries(Templates).map(([name]) => ({name, enabled: (conf.settings.templates.enabled.length ? conf.settings.templates.enabled.includes(name) : true) ?? false}))
const actions = {flush: new Map()}
const requests = {rest: {limit: 0, used: 0, remaining: 0, reset: NaN}, graphql: {limit: 0, used: 0, remaining: 0, reset: NaN}}
const requests = {rest: {limit: 0, used: 0, remaining: 0, reset: NaN}, graphql: {limit: 0, used: 0, remaining: 0, reset: NaN}, search: {limit: 0, used: 0, remaining: 0, reset: NaN}}
let _requests_refresh = false
if (!conf.settings.notoken) {
const refresh = async () => {
try {
const {limit} = await graphql("{ limit:rateLimit {limit remaining reset:resetAt used} }")
Object.assign(requests, {
rest: (await rest.rateLimit.get()).data.rate,
graphql: {...limit, reset: new Date(limit.reset).getTime()},
})
const {resources} = (await api.rest.rateLimit.get()).data
Object.assign(requests, {rest: resources.core, graphql: resources.graphql, search: resources.search})
}
catch {
console.debug("metrics/app > failed to update remaining requests")
Expand Down Expand Up @@ -130,8 +143,16 @@ export default async function({sandbox = false} = {}) {
app.get("/.templates/:template", limiter, (req, res) => req.params.template in conf.templates ? res.status(200).json(conf.templates[req.params.template]) : res.sendStatus(404))
for (const template in conf.templates)
app.use(`/.templates/${template}/partials`, express.static(`${conf.paths.templates}/${template}/partials`))
//Modes
//Modes and extras
app.get("/.modes", limiter, (req, res) => res.status(200).json(conf.settings.modes))
app.get("/.extras", limiter, async (req, res) => {
if ((authenticated.has(req.headers["x-metrics-session"]))&&(conf.settings.extras?.logged)) {
if (conf.settings.extras?.features !== true)
return res.status(200).json([...conf.settings.extras.features, ...conf.settings.extras.logged])
}
res.status(200).json(conf.settings.extras?.features ?? conf.settings?.extras?.default ?? false)
})
app.get("/.extras.logged", limiter, async (req, res) => res.status(200).json(conf.settings.extras?.logged ?? []))
//Styles
app.get("/.css/style.css", limiter, (req, res) => res.sendFile(`${conf.paths.statics}/style.css`))
app.get("/.css/style.vars.css", limiter, (req, res) => res.sendFile(`${conf.paths.statics}/style.vars.css`))
Expand All @@ -152,7 +173,18 @@ export default async function({sandbox = false} = {}) {
app.get("/.js/clipboard.min.js", limiter, (req, res) => res.sendFile(`${conf.paths.node_modules}/clipboard/dist/clipboard.min.js`))
//Meta
app.get("/.version", limiter, (req, res) => res.status(200).send(conf.package.version))
app.get("/.requests", limiter, (req, res) => res.status(200).json(requests))
app.get("/.requests", limiter, async (req, res) => {
try {
const custom = uapi(req.headers["x-metrics-session"])
if (custom) {
const {data:{resources}} = await custom.rest.rateLimit.get()
if (resources)
return res.status(200).json({rest:resources.core, graphql:resources.graphql, search:resources.search, login:custom.login})
}
}
catch {} //eslint-disable-line no-empty
return res.status(200).json(requests)
})
app.get("/.hosted", limiter, (req, res) => res.status(200).json(conf.settings.hosted || null))
//Cache
app.get("/.uncache", limiter, (req, res) => {
Expand All @@ -172,6 +204,84 @@ export default async function({sandbox = false} = {}) {
}
})

//OAuth
if (conf.settings.oauth) {
console.debug("metrics/app/oauth > enabled")
const states = new Map()
app.get("/.oauth/", limiter, (req, res) => res.sendFile(`${conf.paths.statics}/oauth/index.html`))
app.get("/.oauth/index.html", limiter, (req, res) => res.sendFile(`${conf.paths.statics}/oauth/index.html`))
app.get("/.oauth/script.js", limiter, (req, res) => res.sendFile(`${conf.paths.statics}/oauth/script.js`))
app.get("/.oauth/authenticate", (req, res) => {
//Create a state to protect against cross-site request forgery attacks
const state = crypto.randomBytes(64).toString("hex")
const scopes = new url.URLSearchParams(req.query).get("scopes")
const from = new url.URLSearchParams(req.query).get("scopes")
states.set(state, {from, scopes})
console.debug(`metrics/app/oauth > request ${state}`)
//OAuth through GitHub
return res.redirect(`https://github.com/login/oauth/authorize?${new url.URLSearchParams({
client_id:conf.settings.oauth.id,
state,
redirect_uri:`${conf.settings.oauth.url}/.oauth/authorize`,
allow_signup:false,
scope:scopes,
})}`)
})
app.get("/.oauth/authorize", async (req, res) => {
//Check state
const {code, state} = req.query
if ((!state)||(!states.has(state))) {
console.debug("metrics/app/oauth > 400 (invalid state)")
return res.status(400).send("Bad request: invalid state")
}
//OAuth
try {
//Authorize user
console.debug("metrics/app/oauth > authorization")
const {data} = await axios.post("https://github.com/login/oauth/access_token", `${new url.URLSearchParams({
client_id:conf.settings.oauth.id,
client_secret:conf.settings.oauth.secret,
code,
})}`)
const token = new url.URLSearchParams(data).get("access_token")
//Validate user
const {data:{login}} = await axios.get("https://api.github.com/user", {headers:{Authorization:`token ${token}`}})
console.debug(`metrics/app/oauth > authorization success for ${login}`)
const session = crypto.randomBytes(128).toString("hex")
authenticated.set(session, {login, token})
console.debug(`metrics/app/oauth > created session ${session.substring(0, 6)}`)
//Redirect user back
const {from} = states.get(state)
return res.redirect(`/.oauth/redirect?${new url.URLSearchParams({to:from, session})}`)
}
catch {
console.debug("metrics/app/oauth > authorization failed")
return res.status(401).send("Unauthorized: oauth failed")
}
finally {
states.delete(state)
}
})
app.get("/.oauth/revoke/:session", limiter, async (req, res) => {
const session = req.params.session?.replace(/[\n\r]/g, "")
if (authenticated.has(session)) {
const {token} = authenticated.get(session)
try {
console.log(await axios.delete(`https://api.github.com/applications/${conf.settings.oauth.id}/grant`, {auth:{username:conf.settings.oauth.id, password:conf.settings.oauth.secret}, headers:{Accept:"application/vnd.github+json"}, data:{access_token:token}}))
authenticated.delete(session)
console.debug(`metrics/app/oauth > deleted session ${session.substring(0, 6)}`)
return res.redirect("/.oauth")
}
catch {} //eslint-disable-line no-empty
}
return res.status(400).send("Bad request: invalid session")
})
app.get("/.oauth/redirect", limiter, (req, res) => res.sendFile(`${conf.paths.statics}/oauth/redirect.html`))
app.get("/.oauth/enabled", limiter, (req, res) => res.json(true))
}
else
app.get("/.oauth/enabled", limiter, (req, res) => res.json(false))

//Pending requests
const pending = new Map()

Expand Down Expand Up @@ -236,7 +346,7 @@ export default async function({sandbox = false} = {}) {
}
;(async () => {
try {
const json = await metrics.insights({login}, {graphql, rest, conf, callbacks}, {Plugins, Templates})
const json = await metrics.insights({login}, {...api, ...uapi(req.headers["x-metrics-session"]), conf, callbacks}, {Plugins, Templates})
//Cache
cache.put(`insights.${login}`, json)
if ((!debug) && (cached)) {
Expand Down Expand Up @@ -289,12 +399,14 @@ export default async function({sandbox = false} = {}) {
app.get("/.js/embed/app.js", limiter, (req, res) => res.sendFile(`${conf.paths.statics}/embed/app.js`))
app.get("/.js/embed/app.placeholder.js", limiter, (req, res) => res.sendFile(`${conf.paths.statics}/embed/app.placeholder.js`))
//App routes
app.get("/:login/:repository?", ...middlewares, async (req, res) => {
app.get("/:login/:repository?", ...middlewares, async (req, res, next) => {
//Request params
const login = req.params.login?.replace(/[\n\r]/g, "")
const repository = req.params.repository?.replace(/[\n\r]/g, "")
let solve = null
//Check username
if ((login.startsWith("."))||(login.includes("/")))
return next()
if (!/^[-\w]+$/i.test(login)) {
console.debug(`metrics/app/${login} > 400 (invalid username)`)
return res.status(400).send("Bad request: username seems invalid")
Expand Down Expand Up @@ -335,19 +447,28 @@ export default async function({sandbox = false} = {}) {

//Compute rendering
try {
//Render
//Prepare settings
const q = req.query
console.debug(`metrics/app/${login} > ${util.inspect(q, {depth: Infinity, maxStringLength: 256})}`)
if ((q["config.presets"]) && ((conf.settings.extras?.features?.includes("metrics.setup.community.presets")) || (conf.settings.extras?.features === true) || (conf.settings.extras?.default))) {
const octokit = {...api, ...uapi(req.headers["x-metrics-session"])}
let uconf = conf
if ((octokit.login)&&(conf.settings.extras?.logged)&&(uconf.settings.extras?.features !== true)) {
console.debug(`metrics/app/${login} > session is authenticated, adding additional permissions ${conf.settings.extras.logged}`)
uconf = {...conf, settings:{...conf.settings, extras:{...conf.settings.extras}}}
uconf.settings.extras.features = uconf.settings.extras.features ?? []
uconf.settings.extras.features.push(...conf.settings.extras.logged)
}
//Preset
if ((q["config.presets"]) && ((uconf.settings.extras?.features?.includes("metrics.setup.community.presets")) || (uconf.settings.extras?.features === true) || (uconf.settings.extras?.default))) {
console.debug(`metrics/app/${login} > presets have been specified, loading them`)
Object.assign(q, await presets(q["config.presets"]))
}
const convert = conf.settings.outputs.includes(q["config.output"]) ? q["config.output"] : conf.settings.outputs[0]
//Render
const convert = uconf.settings.outputs.includes(q["config.output"]) ? q["config.output"] : uconf.settings.outputs[0]
const {rendered, mime} = await metrics({login, q}, {
graphql,
rest,
...octokit,
plugins,
conf,
conf:uconf,
die: q["plugins.errors.fatal"] ?? false,
verify: q.verify ?? false,
convert: convert !== "auto" ? convert : null,
Expand Down Expand Up @@ -441,9 +562,11 @@ export default async function({sandbox = false} = {}) {
"── Content ────────────────────────────────────────────────────────",
`Plugins enabled │ ${enabled.map(({name}) => name).join(", ")}`,
`Templates enabled │ ${templates.filter(({enabled}) => enabled).map(({name}) => name).join(", ")}`,
"── OAuth ──────────────────────────────────────────────────────────",
`Client id │ ${conf.settings.oauth?.id ?? "(none)"}`,
"── Extras ─────────────────────────────────────────────────────────",
`Default │ ${conf.settings.extras?.default ?? false}`,
`Features │ ${conf.settings.extras?.features ?? "(none)"}`,
`Features │ ${Array.isArray(conf.settings.extras?.features) ? conf.settings.extras.features?.length ? conf.settings.extras?.features : "(none)" : "(default)"}`,
"───────────────────────────────────────────────────────────────────",
"Server ready !",
].join("\n")))
Expand Down
Loading