diff --git a/packages/integration/controllers/documentScore.js b/packages/integration/controllers/documentScore.js index 3b564769..7534e27d 100644 --- a/packages/integration/controllers/documentScore.js +++ b/packages/integration/controllers/documentScore.js @@ -1,6 +1,6 @@ import { Octokit } from '@octokit/core'; import { ViewProjects, CncfDocumentScore, logger } from '@orginjs/oss-evaluation-data-model'; -import { getProjectByUrl, sleep } from '../util/util.js'; +import { getProjectByUrl, sleep, getValidToken } from '../util/util.js'; import { addMonitoringToTask } from '../scheduler/schdulerMonitor.js'; import { platformTypes } from '@orginjs/oss-evaluation-util'; import { fetchWithRetries } from '../util/fetchWithRetries.js'; @@ -133,24 +133,6 @@ export async function syncAllProjectCncfDocumentScore(options) { } } -async function getValidGithubToken() { - const tokenArray = JSON.parse(process.env.GITHUB_TOKEN); - for (const token of tokenArray) { - const octokit = new Octokit({ - auth: token, - }); - const result = await octokit.request('GET /rate_limit', { - headers: { - 'X-GitHub-Api-Version': '2022-11-28', - }, - }); - if (result.data.rate.remaining > 50) { - return token; - } - } - return null; -} - function runDocumentChecks(readme, filename, website, release) { // Check if there is a website cncfDocumentChecksSet.website.checked = @@ -202,7 +184,7 @@ function checkItemInReadme(item, readme) { } async function getProjectMetadata(project) { - const githubToken = await getValidGithubToken(); + const githubToken = await getValidToken(platformTypes.GITHUB); const octokit = new Octokit({ auth: githubToken }); const [owner, repo] = project.fullName.split('/'); let release; @@ -331,23 +313,17 @@ async function getGithubRepoContent(octokit, owner, repo) { }, }); if (content.headers['x-ratelimit-remaining'] <= 0) { - octokit.auth = getValidGithubToken(); + octokit.auth = await getValidToken(platformTypes.GITHUB); } return content.data; } async function getRepoContent(project) { - const tokenMap = { - [platformTypes.GITEE]: JSON.parse(process.env.GITEE_TOKEN), - [platformTypes.GITCODE]: JSON.parse(process.env.GITCODE_TOKEN), - }; - const token = tokenMap[project.platformType][0]; + const token = await getValidToken(project.platformType); const header = { 'Content-Type': 'application/json', + ...(token && { Authorization: `Bearer ${token}` }), }; - if (token) { - header['Authorization'] = `Bearer ${token}`; - } const urlMap = { [platformTypes.GITEE]: `https://gitee.com/api/v5/repos/${project.fullName}/contents`, [platformTypes.GITCODE]: `https://api.gitcode.com/api/v5/repos/${project.fullName}/contents`, @@ -382,7 +358,7 @@ async function getGithubRepoPathContent(octokit, path, owner, repo) { }, }); if (content.headers['x-ratelimit-remaining'] <= 0) { - octokit.auth = getValidGithubToken(); + octokit.auth = await getValidToken(platformTypes.GITHUB); } return content.data; } @@ -391,17 +367,11 @@ async function getGithubRepoPathContent(octokit, path, owner, repo) { Get the project root/path metadata, or 404 error if it doesn't exist. */ async function getRepoPathContent(project, path) { - const tokenMap = { - [platformTypes.GITEE]: JSON.parse(process.env.GITEE_TOKEN), - [platformTypes.GITCODE]: JSON.parse(process.env.GITCODE_TOKEN), - }; - const token = tokenMap[project.platformType][0]; + const token = await getValidToken(project.platformType); const header = { 'Content-Type': 'application/json', + ...(token && { Authorization: `Bearer ${token}` }), }; - if (token) { - header['Authorization'] = `Bearer ${token}`; - } const urlMap = { [platformTypes.GITEE]: `https://gitee.com/api/v5/repos/${project.fullName}/contents/${path}`, [platformTypes.GITCODE]: `https://api.gitcode.com/api/v5/repos/${project.fullName}/contents/${path}`, @@ -434,7 +404,7 @@ async function getGithubProjectRelease(octokit, owner, repo) { }, }); if (release.headers['x-ratelimit-remaining'] <= 0) { - octokit.auth = getValidGithubToken(); + octokit.auth = await getValidToken(platformTypes.GITHUB); } if (release.data.length === 0) { return ''; @@ -446,17 +416,11 @@ async function getGithubProjectRelease(octokit, owner, repo) { Get the latest release of the project */ async function getProjectRelease(project) { - const tokenMap = { - [platformTypes.GITEE]: JSON.parse(process.env.GITEE_TOKEN), - [platformTypes.GITCODE]: JSON.parse(process.env.GITCODE_TOKEN), - }; - const token = tokenMap[project.platformType][0]; + const token = await getValidToken(project.platformType); const header = { 'Content-Type': 'application/json', + ...(token && { Authorization: `Bearer ${token}` }), }; - if (token) { - header['Authorization'] = `Bearer ${token}`; - } const urlMap = { [platformTypes.GITEE]: `https://gitee.com/api/v5/repos/${project.fullName}/releases`, [platformTypes.GITCODE]: `https://api.gitcode.com/api/v5/repos/${project.fullName}/releases`, diff --git a/packages/integration/controllers/project.js b/packages/integration/controllers/project.js index c4d6cb2d..11b3f336 100644 --- a/packages/integration/controllers/project.js +++ b/packages/integration/controllers/project.js @@ -181,13 +181,15 @@ function saveCSVFile(projects, fileName) { } async function pagingQuery(url) { - const headers = { 'User-Agent': 'nodejs/18.19.0' }; - if (process.env.GITHUB_TOKEN) { - const tokens = JSON.parse(process.env.GITHUB_TOKEN); - [headers.Authorization] = tokens; - headers['X-GitHub-Api-Version'] = '2022-11-28'; - headers.Accept = 'application/vnd.github+json'; - } + const token = await getValidToken(platformTypes.GITHUB); + const headers = { + 'User-Agent': 'nodejs/18.19.0', + ...(token && { + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + Accept: 'application/vnd.github+json', + }), + }; return new Promise(resolve => { https @@ -200,6 +202,7 @@ async function pagingQuery(url) { res.on('end', () => { if (res.statusCode === 403) { logger.error(`Integrate github project records error 403: ${url}`); + refreshValidToken(platformTypes.GITHUB); resolve({ hasNext: false, nextPageUrl: '', data: [] }); } if (res.statusCode !== 200) { @@ -385,7 +388,7 @@ async function queryProjectByRepUrl(url) { config: { headers: { 'User-Agent': 'nodejs/18.19.0', - Authorization: `Bearer ${token}`, + ...(token && { Authorization: `Bearer ${token}` }), 'X-GitHub-Api-Version': '2022-11-28', Accept: 'application/vnd.github+json', }, @@ -395,7 +398,7 @@ async function queryProjectByRepUrl(url) { baseUrl: 'https://gitee.com/api/v5/repos', config: { headers: { - Authorization: `Bearer ${token}`, + ...(token && { Authorization: `Bearer ${token}` }), }, }, }, @@ -403,7 +406,7 @@ async function queryProjectByRepUrl(url) { baseUrl: 'https://api.gitcode.com/api/v5/repos', config: { headers: { - Authorization: `Bearer ${token}`, + ...(token && { Authorization: `Bearer ${token}` }), }, }, }, diff --git a/packages/integration/controllers/projectHistory.js b/packages/integration/controllers/projectHistory.js index c20bec06..dff8cde5 100644 --- a/packages/integration/controllers/projectHistory.js +++ b/packages/integration/controllers/projectHistory.js @@ -4,13 +4,13 @@ import { GithubProjectsTable, logger, } from '@orginjs/oss-evaluation-data-model'; -import { getProjectByUrl, sleep } from '../util/util.js'; +import { getProjectByUrl, sleep, getValidToken } from '../util/util.js'; import { fetchWithRetries } from '../util/fetchWithRetries.js'; import { getAllContributors } from './projectContributors.js'; import * as cheerio from 'cheerio'; import { storeGithubHistory } from './trendHistory.js'; import { Op } from 'sequelize'; -import { isFirstDayOfMonth, isFirstDayOfWeek } from '@orginjs/oss-evaluation-util'; +import { isFirstDayOfMonth, isFirstDayOfWeek, platformTypes } from '@orginjs/oss-evaluation-util'; import dayjs from 'dayjs'; import utc from 'dayjs/plugin/utc.js'; @@ -183,15 +183,11 @@ async function getProjectInformation(url) { } async function getStars(repoName) { - const tokens = JSON.parse(process.env.GITHUB_TOKEN); - const header = tokens - ? { - 'Content-Type': 'application/json', - Authorization: `Bearer ${tokens[0]}`, - } - : { - 'Content-Type': 'application/json', - }; + const token = await getValidToken(platformTypes.GITHUB); + const header = { + 'Content-Type': 'application/json', + ...(token && { Authorization: `Bearer ${token}` }), + }; const request = await fetchWithRetries(`https://api.github.com/repos/${repoName}`, { method: 'GET', diff --git a/packages/integration/util/util.js b/packages/integration/util/util.js index 5a7562db..3764ebc8 100644 --- a/packages/integration/util/util.js +++ b/packages/integration/util/util.js @@ -68,52 +68,94 @@ export function getCurrentDate() { return formattedDate; } -const validToken = { - [platformTypes.GITHUB]: JSON.parse(process.env.GITHUB_TOKEN)[0], - [platformTypes.GITEE]: JSON.parse(process.env.GITEE_TOKEN)[0], - [platformTypes.GITCODE]: JSON.parse(process.env.GITCODE_TOKEN)[0], +const tokenCache = { + [platformTypes.GITHUB]: { + token: null, + isValid: false, + refreshing: null, + }, + [platformTypes.GITEE]: { + token: null, + isValid: false, + refreshing: null, + }, + [platformTypes.GITCODE]: { + token: null, + isValid: false, + refreshing: null, + }, }; -export const getValidToken = platformType => { - return validToken[platformType]; -}; - -export const refreshValidToken = async platformType => { - const tokenMap = { - [platformTypes.GITHUB]: JSON.parse(process.env.GITHUB_TOKEN), - [platformTypes.GITEE]: JSON.parse(process.env.GITEE_TOKEN), - [platformTypes.GITCODE]: JSON.parse(process.env.GITCODE_TOKEN), - }; - const project = await ViewProjects.findOne({ - where: { - platformType, - }, - }); - const urlMap = { - [platformTypes.GITHUB]: `https://api.github.com/repos/${project.repoName}/contributors?per_page=100&page=1&anon=true`, - [platformTypes.GITEE]: `https://gitee.com/api/v5/repos/${project.repoName}/contributors?type=authors`, - [platformTypes.GITCODE]: `https://api.gitcode.com/api/v5/repos/${project.repoName}/contributors/statistic`, - }; - const tokens = tokenMap[platformType]; - const url = urlMap[platformType]; - for (const token of tokens) { +async function refreshAndValidateToken(tokenArray, validationUrl, platformType) { + logger.info(`Validating ${platformType} token`); + for (const token of tokenArray) { try { - const response = await fetch(url, { - method: 'GET', + const response = await fetch(validationUrl, { headers: { - 'Content-Type': 'application/json', Authorization: `Bearer ${token}`, }, }); if (response.ok && response.status === 200) { - validToken[platformType] = token; + logger.info(`${platformType} token validated successfully`); return token; } } catch (error) { - logger.error(`Error fetching repo data with token: ${token}`, error); + logger.error(`${platformType} token validation error: ${error}`); } } - logger.error('No valid token found'); - validToken[platformType] = null; + logger.error( + `All ${platformType} tokens are invalid. Please add new tokens to environment variables.`, + ); return null; +} + +export const getValidToken = async platformType => { + const cache = tokenCache[platformType]; + if (cache.isValid && cache.token) { + logger.info(`Using cached ${platformType} token`); + return cache.token; + } + if (cache.refreshing) { + logger.info(`${platformType} token is being refreshed, waiting...`); + return cache.refreshing; + } + const tokenMap = { + [platformTypes.GITHUB]: JSON.parse(process.env.GITHUB_TOKEN), + [platformTypes.GITEE]: JSON.parse(process.env.GITEE_TOKEN), + [platformTypes.GITCODE]: JSON.parse(process.env.GITCODE_TOKEN), + }; + const validationUrlMap = { + [platformTypes.GITHUB]: 'https://api.github.com/user', + [platformTypes.GITEE]: 'https://gitee.com/api/v5/user', + [platformTypes.GITCODE]: 'https://api.gitcode.com/api/v5/user', + }; + cache.refreshing = (async () => { + const validToken = await refreshAndValidateToken( + tokenMap[platformType], + validationUrlMap[platformType], + platformType, + ); + if (validToken) { + cache.token = validToken; + cache.isValid = true; + } else { + cache.token = null; + cache.isValid = false; + } + cache.refreshing = null; + return validToken; + })(); + return cache.refreshing; +}; + +export const invalidateToken = platformType => { + const cache = tokenCache[platformType]; + cache.token = null; + cache.isValid = false; + logger.info(`${platformType} token invalidated`); +}; + +export const refreshValidToken = async platformType => { + invalidateToken(platformType); + return await getValidToken(platformType); };