From 45ea8340be214cd78f525ee001e586147860598b Mon Sep 17 00:00:00 2001 From: liushuyu Date: Thu, 30 Dec 2021 03:31:18 -0700 Subject: [PATCH 1/5] CI: do canary merge and nightly publishing ... ... on the GitHub Actions. This will remove the reliance on the backend server merging script and more transparent to the other contributors --- .github/workflows/ci-merge.js | 182 ++++++++++++++++++++++++++++++++++ .github/workflows/publish.yml | 99 ++++++++++++++++++ 2 files changed, 281 insertions(+) create mode 100644 .github/workflows/ci-merge.js create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/ci-merge.js b/.github/workflows/ci-merge.js new file mode 100644 index 000000000..d80091b3e --- /dev/null +++ b/.github/workflows/ci-merge.js @@ -0,0 +1,182 @@ +// Note: This is a GitHub Actions script +// It is not meant to be executed directly on your machine without modifications + +const fs = require("fs"); + +async function checkCanaryChanges(github, context) { + const delta = new Date() - new Date(context.payload.repository.pushed_at); + if (delta <= 86400000) return true; + const query = `query($owner:String!, $name:String!, $label:String!) { + repository(name:$name, owner:$owner) { + pullRequests(labels: [$label], states: OPEN, first: 100) { + nodes { number headRepository { pushedAt } } + } + } + }`; + const variables = { + owner: context.repo.owner, + name: context.repo.repo, + label: "canary-merge", + }; + const result = await github.graphql(query, variables); + const pulls = result.repository.pullRequests.nodes; + for (let i = 0; i < pulls.length; i++) { + let pull = pulls[i]; + if (new Date() - new Date(pull.headRepository.pushedAt) <= 86400000) { + console.info(`${pull.number} updated at ${pull.headRepository.pushedAt}`); + return true; + } + } + console.info("No changes detected in any tagged pull requests."); + return false; +} + +async function tagAndPush(github, owner, repo, execa, commit=false) { + let altToken = process.env.ALT_GITHUB_TOKEN; + if (!altToken) { + throw `Please set ALT_GITHUB_TOKEN environment variable. This token should have write access to ${owner}/${repo}.`; + } + const query = `query ($owner:String!, $name:String!) { + repository(name:$name, owner:$owner) { + refs(refPrefix: "refs/tags/", orderBy: {field: TAG_COMMIT_DATE, direction: DESC}, first: 10) { + nodes { name } + } + } + }`; + const variables = { + owner: owner, + name: repo, + }; + const tags = await github.graphql(query, variables); + let lastTag = tags.repository.refs.nodes[0].name; + let tagNumber = /\w+-(\d+)/.exec(lastTag)[1] | 0; + let channel = repo.split('-')[1]; + let newTag = `${channel}-${tagNumber + 1}`; + console.log(`New tag: ${newTag}`); + if (commit) { + let channelName = channel[0].toUpperCase() + channel.slice(1); + console.info(`Committing pending commit as ${channelName} #${tagNumber + 1}`); + await execa("git", ['commit', '-m', `${channelName} #${tagNumber + 1}`]); + } + console.info('Pushing tags to GitHub ...'); + await execa("git", ['tag', newTag]); + await execa("git", ['remote', 'add', 'target', `https://${altToken}@github.com/${owner}/${repo}.git`]); + await execa("git", ['push', 'target', 'master', '-f']); + await execa("git", ['push', 'target', 'master', '-f', '--tags']); + console.info('Successfully pushed new changes.'); +} + +async function generateReadme(pulls, context, mergeResults, execa) { + let baseUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/`; + let output = + "| Pull Request | Commit | Title | Author | Merged? |\n|----|----|----|----|----|\n"; + for (let pull of pulls) { + let pr = pull.number; + let result = mergeResults[pr]; + output += `| [${pr}](${baseUrl}/pull/${pr}) | [\`${result.rev || "N/A"}\`](${baseUrl}/pull/${pr}/files) | ${pull.title} | [${pull.author.login}](https://github.com/${pull.author.login}/) | ${result.success ? "Yes" : "No"} |\n`; + } + output += + "\n\nEnd of merge log. You can find the original README.md below the break.\n\n-----\n\n"; + output += fs.readFileSync("./README.md"); + fs.writeFileSync("./README.md", output); + await execa("git", ["add", "README.md"]); +} + +async function fetchPullRequests(pulls, repoUrl, execa) { + console.log("::group::Fetch pull requests"); + for (let pull of pulls) { + let pr = pull.number; + console.info(`Fetching PR ${pr} ...`); + await execa("git", [ + "fetch", + "-f", + "--no-recurse-submodules", + repoUrl, + `pull/${pr}/head:pr-${pr}`, + ]); + } + console.log("::endgroup::"); +} + +async function mergePullRequests(pulls, execa) { + let mergeResults = {}; + console.log("::group::Merge pull requests"); + await execa("git", ["config", "--global", "user.name", "citrabot"]); + await execa("git", [ + "config", + "--global", + "user.email", + "citra\x40citra-emu\x2eorg", // prevent email harvesters from scraping the address + ]); + let hasFailed = false; + for (let pull of pulls) { + let pr = pull.number; + console.info(`Merging PR ${pr} ...`); + try { + const process1 = execa("git", [ + "merge", + "--squash", + "--no-edit", + `pr-${pr}`, + ]); + process1.stdout.pipe(process.stdout); + await process1; + + const process2 = execa("git", ["commit", "-m", `Merge PR ${pr}`]); + process2.stdout.pipe(process.stdout); + await process2; + + const process3 = await execa("git", ["rev-parse", "--short", `pr-${pr}`]); + mergeResults[pr] = { + success: true, + rev: process3.stdout, + }; + } catch (err) { + console.log( + `::error title=#${pr} not merged::Failed to merge pull request: ${pr}: ${err}` + ); + mergeResults[pr] = { success: false }; + hasFailed = true; + await execa("git", ["reset", "--hard"]); + } + } + console.log("::endgroup::"); + if (hasFailed) { + throw 'There are merge failures. Aborting!'; + } + return mergeResults; +} + +async function mergebot(github, context, execa) { + const query = `query ($owner:String!, $name:String!, $label:String!) { + repository(name:$name, owner:$owner) { + pullRequests(labels: [$label], states: OPEN, first: 100) { + nodes { + number title author { login } + } + } + } + }`; + const variables = { + owner: context.repo.owner, + name: context.repo.repo, + label: "canary-merge", + }; + const result = await github.graphql(query, variables); + const pulls = result.repository.pullRequests.nodes; + let displayList = []; + for (let i = 0; i < pulls.length; i++) { + let pull = pulls[i]; + displayList.push({ PR: pull.number, Title: pull.title }); + } + console.info("The following pull requests will be merged:"); + console.table(displayList); + await fetchPullRequests(pulls, "https://github.com/citra-emu/citra", execa); + const mergeResults = await mergePullRequests(pulls, execa); + await generateReadme(pulls, context, mergeResults, execa); + await tagAndPush(github, context.repo.owner, `${context.repo.repo}-canary`, execa, true); +} + +module.exports.mergebot = mergebot; +module.exports.checkCanaryChanges = checkCanaryChanges; +module.exports.tagAndPush = tagAndPush; diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 000000000..c603603ed --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,99 @@ +name: citra-publish + +on: + schedule: + - cron: '7 0 * * *' + workflow_dispatch: + inputs: + nightly: + description: 'Whether trigger a nightly build (true/false/auto)' + required: false + default: 'true' + canary: + description: 'Whether trigger a canary build (true/false/auto)' + required: false + default: 'true' + +jobs: + nightly: + runs-on: ubuntu-latest + if: ${{ github.event.inputs.nightly != 'false' }} + steps: + - uses: actions/github-script@v5 + id: check-changes + name: 'Check for new changes' + with: + result-encoding: string + script: | + if (context.payload.inputs && context.payload.inputs.nightly === 'true') return true; + const delta = new Date() - new Date(context.payload.repository.pushed_at); + if (delta <= 86400000) { + return true; + } + console.log('No new changes detected.'); + return false; + # this checkout is required to make sure the GitHub Actions scripts are available + - uses: actions/checkout@v2 + if: ${{ steps.check-changes.outputs.result == 'true' }} + name: Pre-checkout + with: + submodules: false + - run: npm install execa@5 + if: ${{ steps.check-changes.outputs.result == 'true' }} + - uses: actions/checkout@v2 + name: Checkout + if: ${{ steps.check-changes.outputs.result == 'true' }} + with: + path: 'citra-merge' + fetch-depth: 0 + submodules: true + token: ${{ secrets.ALT_GITHUB_TOKEN }} + - uses: actions/github-script@v5 + name: 'Update and tag new commits' + if: ${{ steps.check-changes.outputs.result == 'true' }} + env: + ALT_GITHUB_TOKEN: ${{ secrets.ALT_GITHUB_TOKEN }} + with: + script: | + const execa = require("execa"); + const tagAndPush = require('./.github/workflows/ci-merge.js').tagAndPush; + process.chdir('${{ github.workspace }}/citra-merge'); + tagAndPush(github, context.repo.owner, `${context.repo.repo}-nightly`, execa); + canary: + runs-on: ubuntu-latest + if: ${{ github.event.inputs.canary != 'false' }} + steps: + # this checkout is required to make sure the GitHub Actions scripts are available + - uses: actions/checkout@v2 + name: Pre-checkout + with: + submodules: false + - uses: actions/github-script@v5 + id: check-changes + name: 'Check for new changes' + with: + script: | + if (context.payload.inputs && context.payload.inputs.canary === 'true') return true; + const checkCanaryChanges = require('./.github/workflows/ci-merge.js').checkCanaryChanges; + return checkCanaryChanges(github, context); + - run: npm install execa@5 + if: ${{ steps.check-changes.outputs.result == 'true' }} + - uses: actions/checkout@v2 + name: Checkout + if: ${{ steps.check-changes.outputs.result == 'true' }} + with: + path: 'citra-merge' + fetch-depth: 0 + submodules: true + token: ${{ secrets.ALT_GITHUB_TOKEN }} + - uses: actions/github-script@v5 + name: 'Check and merge canary changes' + if: ${{ steps.check-changes.outputs.result == 'true' }} + env: + ALT_GITHUB_TOKEN: ${{ secrets.ALT_GITHUB_TOKEN }} + with: + script: | + const execa = require("execa"); + const mergebot = require('./.github/workflows/ci-merge.js').mergebot; + process.chdir('${{ github.workspace }}/citra-merge'); + mergebot(github, context, execa); From ad1f0eed222818cd757127da4541168a7912acec Mon Sep 17 00:00:00 2001 From: liushuyu Date: Fri, 31 Dec 2021 19:14:23 -0700 Subject: [PATCH 2/5] CI: make auto-publish workflow more robust and flexible ... * workaround an issue where sometimes GHA does not pass repository object into the context variable * make detection interval adjustable --- .github/workflows/ci-merge.js | 25 ++++++++++++++++++++++--- .github/workflows/publish.yml | 25 +++++++++++++------------ 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci-merge.js b/.github/workflows/ci-merge.js index d80091b3e..44e385d16 100644 --- a/.github/workflows/ci-merge.js +++ b/.github/workflows/ci-merge.js @@ -2,10 +2,28 @@ // It is not meant to be executed directly on your machine without modifications const fs = require("fs"); +// how far back in time should we consider the changes are "recent"? (default: 24 hours) +const DETECTION_TIME_FRAME = (parseInt(process.env.DETECTION_TIME_FRAME)) || (24 * 3600 * 1000); + +async function checkBaseChanges(github, context) { + // a special robustness handling for when GHA did not pass the repository info + if (!context.payload.repository) { + const result = await github.rest.repos.get({ + owner: context.repo.owner, + repo: context.repo.repo, + }); + context.payload.repository = result.data; + } + const delta = new Date() - new Date(context.payload.repository.pushed_at); + if (delta <= DETECTION_TIME_FRAME) { + console.info('New changes detected, triggering a new build.'); + return true; + } + return false; +} async function checkCanaryChanges(github, context) { - const delta = new Date() - new Date(context.payload.repository.pushed_at); - if (delta <= 86400000) return true; + if (checkBaseChanges(github, context)) return true; const query = `query($owner:String!, $name:String!, $label:String!) { repository(name:$name, owner:$owner) { pullRequests(labels: [$label], states: OPEN, first: 100) { @@ -22,7 +40,7 @@ async function checkCanaryChanges(github, context) { const pulls = result.repository.pullRequests.nodes; for (let i = 0; i < pulls.length; i++) { let pull = pulls[i]; - if (new Date() - new Date(pull.headRepository.pushedAt) <= 86400000) { + if (new Date() - new Date(pull.headRepository.pushedAt) <= DETECTION_TIME_FRAME) { console.info(`${pull.number} updated at ${pull.headRepository.pushedAt}`); return true; } @@ -180,3 +198,4 @@ async function mergebot(github, context, execa) { module.exports.mergebot = mergebot; module.exports.checkCanaryChanges = checkCanaryChanges; module.exports.tagAndPush = tagAndPush; +module.exports.checkBaseChanges = checkBaseChanges; diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c603603ed..add387c8e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -19,25 +19,23 @@ jobs: runs-on: ubuntu-latest if: ${{ github.event.inputs.nightly != 'false' }} steps: + # this checkout is required to make sure the GitHub Actions scripts are available + - uses: actions/checkout@v2 + name: Pre-checkout + with: + submodules: false - uses: actions/github-script@v5 id: check-changes name: 'Check for new changes' + env: + # 24 hours + DETECTION_TIME_FRAME: 86400000 with: result-encoding: string script: | if (context.payload.inputs && context.payload.inputs.nightly === 'true') return true; - const delta = new Date() - new Date(context.payload.repository.pushed_at); - if (delta <= 86400000) { - return true; - } - console.log('No new changes detected.'); - return false; - # this checkout is required to make sure the GitHub Actions scripts are available - - uses: actions/checkout@v2 - if: ${{ steps.check-changes.outputs.result == 'true' }} - name: Pre-checkout - with: - submodules: false + const checkBaseChanges = require('./.github/workflows/ci-merge.js').checkBaseChanges; + return checkBaseChanges(github, context); - run: npm install execa@5 if: ${{ steps.check-changes.outputs.result == 'true' }} - uses: actions/checkout@v2 @@ -71,6 +69,9 @@ jobs: - uses: actions/github-script@v5 id: check-changes name: 'Check for new changes' + env: + # 24 hours + DETECTION_TIME_FRAME: 86400000 with: script: | if (context.payload.inputs && context.payload.inputs.canary === 'true') return true; From de0c9f3071f84885b34f41a01fbe305746f34fd4 Mon Sep 17 00:00:00 2001 From: liushuyu Date: Fri, 31 Dec 2021 19:34:38 -0700 Subject: [PATCH 3/5] gitignore: ignore GitHub Actions generated files --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 37591363a..b33205ef5 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ src/common/scm_rev.cpp .idea/ .vs/ .vscode/ +.cache/ CMakeLists.txt.user* # *nix related @@ -37,3 +38,6 @@ Thumbs.db # Flatpak generated files .flatpak-builder/ repo/ + +# GitHub Actions generated files +node_modules/ From cbe1a4f50dd503ec68a163f754e8b82595ec721a Mon Sep 17 00:00:00 2001 From: liushuyu Date: Fri, 31 Dec 2021 21:57:50 -0700 Subject: [PATCH 4/5] CI: fix input dialog wording --- .github/workflows/publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index add387c8e..6ac14f8e6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -6,11 +6,11 @@ on: workflow_dispatch: inputs: nightly: - description: 'Whether trigger a nightly build (true/false/auto)' + description: 'Whether to trigger a nightly build (true/false/auto)' required: false default: 'true' canary: - description: 'Whether trigger a canary build (true/false/auto)' + description: 'Whether to trigger a canary build (true/false/auto)' required: false default: 'true' From d49c946134f49f5e2f2fd3114a250ab60dd7c23a Mon Sep 17 00:00:00 2001 From: liushuyu Date: Tue, 4 Jan 2022 19:00:19 -0700 Subject: [PATCH 5/5] CI: limit CI runs to citra-emu/citra --- .github/workflows/publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6ac14f8e6..3bde327c2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -17,7 +17,7 @@ on: jobs: nightly: runs-on: ubuntu-latest - if: ${{ github.event.inputs.nightly != 'false' }} + if: ${{ github.event.inputs.nightly != 'false' && github.repository == 'citra-emu/citra' }} steps: # this checkout is required to make sure the GitHub Actions scripts are available - uses: actions/checkout@v2 @@ -59,7 +59,7 @@ jobs: tagAndPush(github, context.repo.owner, `${context.repo.repo}-nightly`, execa); canary: runs-on: ubuntu-latest - if: ${{ github.event.inputs.canary != 'false' }} + if: ${{ github.event.inputs.canary != 'false' && github.repository == 'citra-emu/citra' }} steps: # this checkout is required to make sure the GitHub Actions scripts are available - uses: actions/checkout@v2