Skip to content

fix: optimize get valid token #877

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Jiannan-dev merged 1 commit into from
Apr 16, 2026
Merged

fix: optimize get valid token #877

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 11 additions & 47 deletions packages/integration/controllers/documentScore.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { Octokit } from '@octokit/core';
import { ViewProjects, CncfDocumentScore, logger } from '@orginjs/oss-evaluation-data-model';
import { getProjectByUrl, sleep } from '../util/util.js';
import { getProjectByUrl, sleep, getValidToken } from '../util/util.js';
import { addMonitoringToTask } from '../scheduler/schdulerMonitor.js';
import { platformTypes } from '@orginjs/oss-evaluation-util';
import { fetchWithRetries } from '../util/fetchWithRetries.js';
Expand Down Expand Up @@ -133,24 +133,6 @@ export async function syncAllProjectCncfDocumentScore(options) {
}
}

async function getValidGithubToken() {
const tokenArray = JSON.parse(process.env.GITHUB_TOKEN);
for (const token of tokenArray) {
const octokit = new Octokit({
auth: token,
});
const result = await octokit.request('GET /rate_limit', {
headers: {
'X-GitHub-Api-Version': '2022-11-28',
},
});
if (result.data.rate.remaining > 50) {
return token;
}
}
return null;
}

function runDocumentChecks(readme, filename, website, release) {
// Check if there is a website
cncfDocumentChecksSet.website.checked =
Expand Down Expand Up @@ -202,7 +184,7 @@ function checkItemInReadme(item, readme) {
}

async function getProjectMetadata(project) {
const githubToken = await getValidGithubToken();
const githubToken = await getValidToken(platformTypes.GITHUB);
const octokit = new Octokit({ auth: githubToken });
const [owner, repo] = project.fullName.split('/');
let release;
Expand Down Expand Up @@ -331,23 +313,17 @@ async function getGithubRepoContent(octokit, owner, repo) {
},
});
if (content.headers['x-ratelimit-remaining'] <= 0) {
octokit.auth = getValidGithubToken();
octokit.auth = await getValidToken(platformTypes.GITHUB);
}
return content.data;
}

async function getRepoContent(project) {
const tokenMap = {
[platformTypes.GITEE]: JSON.parse(process.env.GITEE_TOKEN),
[platformTypes.GITCODE]: JSON.parse(process.env.GITCODE_TOKEN),
};
const token = tokenMap[project.platformType][0];
const token = await getValidToken(project.platformType);
const header = {
'Content-Type': 'application/json',
...(token && { Authorization: `Bearer ${token}` }),
};
if (token) {
header['Authorization'] = `Bearer ${token}`;
}
const urlMap = {
[platformTypes.GITEE]: `https://gitee.com/api/v5/repos/${project.fullName}/contents`,
[platformTypes.GITCODE]: `https://api.gitcode.com/api/v5/repos/${project.fullName}/contents`,
Expand Down Expand Up @@ -382,7 +358,7 @@ async function getGithubRepoPathContent(octokit, path, owner, repo) {
},
});
if (content.headers['x-ratelimit-remaining'] <= 0) {
octokit.auth = getValidGithubToken();
octokit.auth = await getValidToken(platformTypes.GITHUB);
}
return content.data;
}
Expand All @@ -391,17 +367,11 @@ async function getGithubRepoPathContent(octokit, path, owner, repo) {
Get the project root/path metadata, or 404 error if it doesn't exist.
*/
async function getRepoPathContent(project, path) {
const tokenMap = {
[platformTypes.GITEE]: JSON.parse(process.env.GITEE_TOKEN),
[platformTypes.GITCODE]: JSON.parse(process.env.GITCODE_TOKEN),
};
const token = tokenMap[project.platformType][0];
const token = await getValidToken(project.platformType);
const header = {
'Content-Type': 'application/json',
...(token && { Authorization: `Bearer ${token}` }),
};
if (token) {
header['Authorization'] = `Bearer ${token}`;
}
const urlMap = {
[platformTypes.GITEE]: `https://gitee.com/api/v5/repos/${project.fullName}/contents/${path}`,
[platformTypes.GITCODE]: `https://api.gitcode.com/api/v5/repos/${project.fullName}/contents/${path}`,
Expand Down Expand Up @@ -434,7 +404,7 @@ async function getGithubProjectRelease(octokit, owner, repo) {
},
});
if (release.headers['x-ratelimit-remaining'] <= 0) {
octokit.auth = getValidGithubToken();
octokit.auth = await getValidToken(platformTypes.GITHUB);
}
if (release.data.length === 0) {
return '';
Expand All @@ -446,17 +416,11 @@ async function getGithubProjectRelease(octokit, owner, repo) {
Get the latest release of the project
*/
async function getProjectRelease(project) {
const tokenMap = {
[platformTypes.GITEE]: JSON.parse(process.env.GITEE_TOKEN),
[platformTypes.GITCODE]: JSON.parse(process.env.GITCODE_TOKEN),
};
const token = tokenMap[project.platformType][0];
const token = await getValidToken(project.platformType);
const header = {
'Content-Type': 'application/json',
...(token && { Authorization: `Bearer ${token}` }),
};
if (token) {
header['Authorization'] = `Bearer ${token}`;
}
const urlMap = {
[platformTypes.GITEE]: `https://gitee.com/api/v5/repos/${project.fullName}/releases`,
[platformTypes.GITCODE]: `https://api.gitcode.com/api/v5/repos/${project.fullName}/releases`,
Expand Down
23 changes: 13 additions & 10 deletions packages/integration/controllers/project.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,13 +181,15 @@ function saveCSVFile(projects, fileName) {
}

async function pagingQuery(url) {
const headers = { 'User-Agent': 'nodejs/18.19.0' };
if (process.env.GITHUB_TOKEN) {
const tokens = JSON.parse(process.env.GITHUB_TOKEN);
[headers.Authorization] = tokens;
headers['X-GitHub-Api-Version'] = '2022-11-28';
headers.Accept = 'application/vnd.github+json';
}
const token = await getValidToken(platformTypes.GITHUB);
const headers = {
'User-Agent': 'nodejs/18.19.0',
...(token && {
Authorization: `Bearer ${token}`,
'X-GitHub-Api-Version': '2022-11-28',
Accept: 'application/vnd.github+json',
}),
};

return new Promise(resolve => {
https
Expand All @@ -200,6 +202,7 @@ async function pagingQuery(url) {
res.on('end', () => {
if (res.statusCode === 403) {
logger.error(`Integrate github project records error 403: ${url}`);
refreshValidToken(platformTypes.GITHUB);
resolve({ hasNext: false, nextPageUrl: '', data: [] });
}
if (res.statusCode !== 200) {
Expand Down Expand Up @@ -385,7 +388,7 @@ async function queryProjectByRepUrl(url) {
config: {
headers: {
'User-Agent': 'nodejs/18.19.0',
Authorization: `Bearer ${token}`,
...(token && { Authorization: `Bearer ${token}` }),
'X-GitHub-Api-Version': '2022-11-28',
Accept: 'application/vnd.github+json',
},
Expand All @@ -395,15 +398,15 @@ async function queryProjectByRepUrl(url) {
baseUrl: 'https://gitee.com/api/v5/repos',
config: {
headers: {
Authorization: `Bearer ${token}`,
...(token && { Authorization: `Bearer ${token}` }),
},
},
},
[platformTypes.GITCODE]: {
baseUrl: 'https://api.gitcode.com/api/v5/repos',
config: {
headers: {
Authorization: `Bearer ${token}`,
...(token && { Authorization: `Bearer ${token}` }),
},
},
},
Expand Down
18 changes: 7 additions & 11 deletions packages/integration/controllers/projectHistory.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@ import {
GithubProjectsTable,
logger,
} from '@orginjs/oss-evaluation-data-model';
import { getProjectByUrl, sleep } from '../util/util.js';
import { getProjectByUrl, sleep, getValidToken } from '../util/util.js';
import { fetchWithRetries } from '../util/fetchWithRetries.js';
import { getAllContributors } from './projectContributors.js';
import * as cheerio from 'cheerio';
import { storeGithubHistory } from './trendHistory.js';
import { Op } from 'sequelize';
import { isFirstDayOfMonth, isFirstDayOfWeek } from '@orginjs/oss-evaluation-util';
import { isFirstDayOfMonth, isFirstDayOfWeek, platformTypes } from '@orginjs/oss-evaluation-util';
import dayjs from 'dayjs';
import utc from 'dayjs/plugin/utc.js';

Expand Down Expand Up @@ -183,15 +183,11 @@ async function getProjectInformation(url) {
}

async function getStars(repoName) {
const tokens = JSON.parse(process.env.GITHUB_TOKEN);
const header = tokens
? {
'Content-Type': 'application/json',
Authorization: `Bearer ${tokens[0]}`,
}
: {
'Content-Type': 'application/json',
};
const token = await getValidToken(platformTypes.GITHUB);
const header = {
'Content-Type': 'application/json',
...(token && { Authorization: `Bearer ${token}` }),
};

const request = await fetchWithRetries(`https://api.github.com/repos/${repoName}`, {
method: 'GET',
Expand Down
110 changes: 76 additions & 34 deletions packages/integration/util/util.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,52 +68,94 @@ export function getCurrentDate() {
return formattedDate;
}

const validToken = {
[platformTypes.GITHUB]: JSON.parse(process.env.GITHUB_TOKEN)[0],
[platformTypes.GITEE]: JSON.parse(process.env.GITEE_TOKEN)[0],
[platformTypes.GITCODE]: JSON.parse(process.env.GITCODE_TOKEN)[0],
const tokenCache = {
[platformTypes.GITHUB]: {
token: null,
isValid: false,
refreshing: null,
},
[platformTypes.GITEE]: {
token: null,
isValid: false,
refreshing: null,
},
[platformTypes.GITCODE]: {
token: null,
isValid: false,
refreshing: null,
},
};

export const getValidToken = platformType => {
return validToken[platformType];
};

export const refreshValidToken = async platformType => {
const tokenMap = {
[platformTypes.GITHUB]: JSON.parse(process.env.GITHUB_TOKEN),
[platformTypes.GITEE]: JSON.parse(process.env.GITEE_TOKEN),
[platformTypes.GITCODE]: JSON.parse(process.env.GITCODE_TOKEN),
};
const project = await ViewProjects.findOne({
where: {
platformType,
},
});
const urlMap = {
[platformTypes.GITHUB]: `https://api.github.com/repos/${project.repoName}/contributors?per_page=100&page=1&anon=true`,
[platformTypes.GITEE]: `https://gitee.com/api/v5/repos/${project.repoName}/contributors?type=authors`,
[platformTypes.GITCODE]: `https://api.gitcode.com/api/v5/repos/${project.repoName}/contributors/statistic`,
};
const tokens = tokenMap[platformType];
const url = urlMap[platformType];
for (const token of tokens) {
async function refreshAndValidateToken(tokenArray, validationUrl, platformType) {
logger.info(`Validating ${platformType} token`);
for (const token of tokenArray) {
try {
const response = await fetch(url, {
method: 'GET',
const response = await fetch(validationUrl, {
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${token}`,
},
});
if (response.ok && response.status === 200) {
validToken[platformType] = token;
logger.info(`${platformType} token validated successfully`);
return token;
}
} catch (error) {
logger.error(`Error fetching repo data with token: ${token}`, error);
logger.error(`${platformType} token validation error: ${error}`);
}
}
logger.error('No valid token found');
validToken[platformType] = null;
logger.error(
`All ${platformType} tokens are invalid. Please add new tokens to environment variables.`,
);
return null;
}

export const getValidToken = async platformType => {
const cache = tokenCache[platformType];
if (cache.isValid && cache.token) {
logger.info(`Using cached ${platformType} token`);
return cache.token;
}
if (cache.refreshing) {
logger.info(`${platformType} token is being refreshed, waiting...`);
return cache.refreshing;
}
const tokenMap = {
[platformTypes.GITHUB]: JSON.parse(process.env.GITHUB_TOKEN),
[platformTypes.GITEE]: JSON.parse(process.env.GITEE_TOKEN),
[platformTypes.GITCODE]: JSON.parse(process.env.GITCODE_TOKEN),
};
const validationUrlMap = {
[platformTypes.GITHUB]: 'https://api.github.com/user',
[platformTypes.GITEE]: 'https://gitee.com/api/v5/user',
[platformTypes.GITCODE]: 'https://api.gitcode.com/api/v5/user',
};
cache.refreshing = (async () => {
const validToken = await refreshAndValidateToken(
tokenMap[platformType],
validationUrlMap[platformType],
platformType,
);
if (validToken) {
cache.token = validToken;
cache.isValid = true;
} else {
cache.token = null;
cache.isValid = false;
}
cache.refreshing = null;
return validToken;
})();
return cache.refreshing;
};

export const invalidateToken = platformType => {
const cache = tokenCache[platformType];
cache.token = null;
cache.isValid = false;
logger.info(`${platformType} token invalidated`);
};

export const refreshValidToken = async platformType => {
invalidateToken(platformType);
return await getValidToken(platformType);
};
Loading