diff --git a/.github/ISSUE_TEMPLATE/new_initiative.yml b/.github/ISSUE_TEMPLATE/new_initiative.yml index 6c0c13df..c7775d48 100644 --- a/.github/ISSUE_TEMPLATE/new_initiative.yml +++ b/.github/ISSUE_TEMPLATE/new_initiative.yml @@ -48,28 +48,33 @@ body: attributes: label: Initiative Category options: - - Agriculture - - Children + - Agriculture/Main + - Children/Main - Defense/Evacuation - Defense/Shelters - Defense/Weapons - - Donations - - FamilyHosting + - Donations/Agriculture + - Donations/Main + - FamilyHosting/Main - GeneralAssistance - - Initiatives - - Medical - - MemorialRemembrance - - MentalSupport + - Initiatives/Main + - Medical/Main + - MemorialRemembrance/Main + - MentalSupport/Main - MissingPersons/Main - - Reserves - - SocialNetworks - - StrategicCommunication - - Supplies + - Reserves/Main + - StrategicCommunication/Funding + - StrategicCommunication/Info + - StrategicCommunication/Main + - StrategicCommunication/SocialNetworks + - StrategicCommunication/Tools + - Supplies/Donation + - Supplies/Main - TechInitiatives/Main - TechInitiatives/Support - TechInitiatives/Volunteers - - Transportation - - Volunteering + - Transportation/Main + - Volunteering/Main default: 0 validations: required: true @@ -151,7 +156,7 @@ body: placeholder: https://twitter.com/iron-widgets - type: input - id: logo + id: initiativeImage attributes: label: Initiative logo description: Link to an image of the initiative's logo (suitable for inclusion in an HTML `IMG` tag) diff --git a/.github/workflows/initiative-auto-comment.yml b/.github/workflows/initiative-auto-pr.yml similarity index 70% rename from .github/workflows/initiative-auto-comment.yml rename to .github/workflows/initiative-auto-pr.yml index 8925bde0..6d3312a2 100644 --- a/.github/workflows/initiative-auto-comment.yml +++ b/.github/workflows/initiative-auto-pr.yml @@ -1,16 +1,17 @@ -name: Initiative Request - Auto Comment +name: Initiative Request - Automatic PR on: issues: types: [opened, edited] jobs: - add-comment: + create-pr: if: contains(github.event.issue.labels.*.name, 'new-initiative-request') runs-on: ubuntu-latest permissions: - contents: read + contents: write issues: write + pull-requests: write steps: - name: Checkout code @@ -27,14 +28,18 @@ jobs: pip install -r "$GITHUB_WORKSPACE/scripts/requirements.txt" - name: Run GPT script + uses: nick-fields/retry@v2 env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ISSUE_BODY: ${{ github.event.issue.body }} - run: python $GITHUB_WORKSPACE/scripts/gptGenerateJsonFromIssue.py - - - name: Post comment + with: + timeout_minutes: 5 + max_attempts: 3 + command: python $GITHUB_WORKSPACE/scripts/gptGenerateJsonFromIssue.py + + - name: Generate PR uses: actions/github-script@v6 with: script: | - const script = require(process.env.GITHUB_WORKSPACE + '/scripts/commentUsingGptResponse.js') + const script = require(process.env.GITHUB_WORKSPACE + '/scripts/generatePullRequestForNewInitiative.js') script({github, context}) diff --git a/_data/links/Children/Main/links.json b/_data/links/Children/Main/links.json index 4d0e2bf2..23475ba1 100644 --- a/_data/links/Children/Main/links.json +++ b/_data/links/Children/Main/links.json @@ -55,6 +55,31 @@ "description": "קוד: בריינפופ23", "url": "https://il.brainpop.com/", "website": "https://il.brainpop.com/" + }, + { + "displayName": "Bar21", + "name": "Bar21", + "shortDescription": "FooBaz", + "description": "hello world", + "initiativeValidationDetails": "Source: Are you the developer? If not, how did you hear about it?\nLink to reputable source(s) promoting the initiative: [e.g. https://www.israelhayom.co.il/tech/tech-news/iron-swords]\nContact details for members of the initiative's team (so we can contact them directly for validation): [e.g. info@iron-widgets.com]", + "url": "N/A", + "website": "N/A", + "whatsapp": "N/A", + "telegram": "N/A", + "drive": "N/A", + "form": "N/A", + "docs": "N/A", + "discord": "N/A", + "facebook": "N/A", + "instagram": "N/A", + "tiktok": "N/A", + "twitter": "N/A", + "initiativeImage": "N/A", + "tags": [ + "Bar21", + "FooBaz", + "hello world" + ] } ] } \ No newline at end of file diff --git a/scripts/commentUsingGptResponse.js b/scripts/commentUsingGptResponse.js deleted file mode 100644 index e706770b..00000000 --- a/scripts/commentUsingGptResponse.js +++ /dev/null @@ -1,64 +0,0 @@ -module.exports = ({github, context}) => { - function tryExtractJson(text, jsonStartMarker, jsonEndMarker) { - console.log(`Attempting to extract JSON with start marker "${jsonStartMarker}" and end marker "${jsonEndMarker}"` ); - - const indexOfJsonStart = text.indexOf(jsonStartMarker); - if (indexOfJsonStart === -1) { - console.log("Could not find JSON start marter"); - return null; - } - - const indexOfJsonEnd = text.indexOf(jsonEndMarker, indexOfJsonStart + 1); - if (indexOfJsonEnd === -1) { - console.log("Could not find JSON end marter"); - return null; - } - - return text.substring(indexOfJsonStart + jsonStartMarker.length, indexOfJsonEnd); - } - - function createComment(gptResponseJson) { - console.log("Extracted JSON: " + gptResponseJson); - - try { - humanReadableJson = JSON.stringify(JSON.parse(gptResponseJson), null, 2); - } - catch (e) { - console.warn("Could not process JSON: " + e); - humanReadableJson = "// WARNING: THIS IS MOST LIKELY INVALID JSON, PLEASE REVIEW MANUALLY \n" + gptResponseJson; - } - - console.log("Human readable JSON: " + humanReadableJson); - github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body: "```json\n" + humanReadableJson + "\n```" - }); - } - - const fs = require('fs') - const gptResponse = fs.readFileSync(process.env.GITHUB_WORKSPACE + "/gpt-auto-comment.output", 'utf8') - - // https://stackoverflow.com/a/51602415/67824 - var sanitizedGptResponse = gptResponse.replace(/[\u0000-\u001F\u007F-\u009F]/g, "") - const jsonStartMarker = "```json" - const jsonEndMarker = "```" - - console.log("Sanitized GPT response: " + sanitizedGptResponse) - - json = tryExtractJson(sanitizedGptResponse, jsonStartMarker, jsonEndMarker); - if (json !== null) { - createComment(json); - return; - } - - json = tryExtractJson(sanitizedGptResponse, jsonEndMarker, jsonEndMarker) // sometimes GPT will return the JSON in the form of ``` {} ``` - if (json !== null) { - createComment(json); - return; - } - - console.log("Could not find JSON markers in GPT output, assuming raw JSON"); - createComment(sanitizedGptResponse); -} \ No newline at end of file diff --git a/scripts/generatePullRequestForNewInitiative.js b/scripts/generatePullRequestForNewInitiative.js new file mode 100644 index 00000000..a507c0b0 --- /dev/null +++ b/scripts/generatePullRequestForNewInitiative.js @@ -0,0 +1,152 @@ +module.exports = ({github, context}) => { + + const fs = require('fs'); + const cp = require('child_process'); + + function tryExtractJson(text, jsonStartMarker, jsonEndMarker) { + console.log(`Attempting to extract JSON with start marker "${jsonStartMarker}" and end marker "${jsonEndMarker}"` ); + + const indexOfJsonStart = text.indexOf(jsonStartMarker); + if (indexOfJsonStart === -1) { + console.log("Could not find JSON start marker"); + return null; + } + + const indexOfJsonEnd = text.indexOf(jsonEndMarker, indexOfJsonStart + 1); + if (indexOfJsonEnd === -1) { + console.log("Could not find JSON end marker"); + return null; + } + + return text.substring(indexOfJsonStart + jsonStartMarker.length, indexOfJsonEnd); + } + + function createOrUpdatePullRequest(branch, name) { + + const existingResponse = github.rest.pulls.list({ + owner: context.repo.owner, + repo: context.repo.repo, + head: branch, + base: 'main', + }) + console.log("existing PRs: " + JSON.stringify(existingResponse)); + + existingPrs = existingResponse.data + if (existingPrs && existingPrs.length > 0) { + console.log("At least one PR exists for this branch"); + existingPrs[0].existing = true; + return existingPrs[0]; + } + + const newResponse = github.rest.pulls.create({ + title: 'New Initiative: ' + name, + owner: context.repo.owner, + repo: context.repo.repo, + head: branch, + base: 'main', + body: "*IMPORTANT: Only merge after validating the initiative and double checking the generated JSON*" + }); + console.log("Created new PR: " + JSON.stringify(newResponse)); + + return newResponse.data + } + + function createComment(body) { + github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: body + }); + } + + function warnAndComment(warning, exception, json) { + console.warn(`${warning}: ${exception}`); + createComment(`WARNING: ${warning} (see GitHub Action logs for more details)\n` + "Automatic PR will NOT be generated:\n" + json); + } + + function executeGitCommand(args) { + console.log(`Executing: git ${args}`); + cp.execFileSync("git", args); + } + + const tempFolder = process.env.TEMP || "/tmp"; + const gptResponse = fs.readFileSync(tempFolder + "/gpt-auto-comment.output", "utf8"); + + // https://stackoverflow.com/a/51602415/67824 + const sanitizedGptResponse = gptResponse.replace(/[\u0000-\u001F\u007F-\u009F]/g, ""); + console.log("Sanitized GPT response: " + sanitizedGptResponse); + + const jsonStartMarker = "```json"; + const jsonEndMarker = "```"; + jsonString = tryExtractJson(sanitizedGptResponse, jsonStartMarker, jsonEndMarker) || tryExtractJson(sanitizedGptResponse, jsonEndMarker, jsonEndMarker); + if (jsonString == null) { + console.log("Could not find JSON markers in GPT output, assuming raw JSON"); + jsonString = sanitizedGptResponse; + } + console.log("Extracted JSON: " + jsonString); + + try { + json = JSON.parse(jsonString); + } + catch (e) { + return warnAndComment("Could not process GPT response as JSON", e, jsonString); + } + + humanReadableJson = "```json\n" + JSON.stringify(json, null, 2) + "\n```"; + + let categoryLinksJsonFile, categoryJsonString; + try { + categoryLinksJsonFile = `${process.env.GITHUB_WORKSPACE}/_data/links/${json.category}/links.json`; + console.log("resolved category links file: " + categoryLinksJsonFile); + + categoryJsonString = fs.readFileSync(categoryLinksJsonFile, "utf8"); + categoryJson = JSON.parse(categoryJsonString); + } + catch (e) { + return warnAndComment("Could not process category links JSON", e, humanReadableJson); + } + + delete json.category //not in our schema, and worse - will interfere with the existing initiative detection below + console.log("Attempting to detect already existing initiative under this category"); + + const upperCategoryJsonString = categoryJsonString.toLocaleUpperCase("en-us"); + for (const prop in json) { + + const value = json[prop]; + if (typeof value !== "string") { + continue; + } + + const PropValueUpper = value.toLocaleUpperCase("en-us"); + if (upperCategoryJsonString.indexOf(PropValueUpper) !== -1) { + return warnAndComment(`Initiative might already exist under this category, the value of property ${prop} is already present in the JSON: ${value}`, "suspected existing initiative", humanReadableJson); + } + } + + categoryJson.links.push(json); + fs.writeFileSync(categoryLinksJsonFile, JSON.stringify(categoryJson, null, 2), "utf8"); + + const branch = `auto-pr-${context.issue.number}`; + try { + executeGitCommand(["config", "user.name", "github-actions"]) + executeGitCommand(["config", "user.email", "41898282+github-actions[bot]@users.noreply.github.com"]) //https://github.com/orgs/community/discussions/26560 + executeGitCommand(["checkout", "-b", branch]) + executeGitCommand(["add", categoryLinksJsonFile]) + executeGitCommand(["commit", "-m", json.name || "new initiative"]) + executeGitCommand(["push", "origin", branch, "--force"]) + } + catch (e) { + return warnAndComment("encountered error during git execution", e, humanReadableJson); + } + + let pr; + try { + pr = createOrUpdatePullRequest(branch, json.name || "???"); + } + catch (e) { + return warnAndComment("Could not create pull request", e, humanReadableJson); + } + + createComment((pr.existing ? "Updated" : "Created") + " PR: " + pr.url); +} \ No newline at end of file diff --git a/scripts/gptGenerateJsonFromIssue.py b/scripts/gptGenerateJsonFromIssue.py index a64674cd..743bb3af 100644 --- a/scripts/gptGenerateJsonFromIssue.py +++ b/scripts/gptGenerateJsonFromIssue.py @@ -110,7 +110,7 @@ def ask_gpt_for_initiative_json(self, issue_body: str) -> str: if __name__ == "__main__": gpt_response = GPTAssistant().ask_gpt_for_initiative_json(os.environ["ISSUE_BODY"]) - respose_output_file = os.environ["GITHUB_WORKSPACE"] + "/gpt-auto-comment.output" + respose_output_file = os.getenv("TEMP", "/tmp") + "/gpt-auto-comment.output" logging.info(f"Writing GPT Response to {respose_output_file}:\n{gpt_response}") with open(respose_output_file, "w", encoding="utf-8") as output: diff --git a/scripts/test_generatePullRequestForNewInitiative.js b/scripts/test_generatePullRequestForNewInitiative.js new file mode 100644 index 00000000..72597375 --- /dev/null +++ b/scripts/test_generatePullRequestForNewInitiative.js @@ -0,0 +1,33 @@ +const github = { + rest: { + issues: { + createComment: (cmt) => { + console.log("posting comment:") + console.log(cmt) + } + }, + pulls: { + create: (pr) => { + console.log("creating PR:") + console.log(pr) + return { + url: "https://example.com" + } + } + } + } +} + +const context = { + repo: { + owner: "ohadschn", + repo: "ohadschn/ConnectPortal" + }, + issue: { + number: 42 + } +} + +const script = require('./generatePullRequestForNewInitiative.js') + +script({github, context}) \ No newline at end of file