diff --git a/.github/update-release-branch.py b/.github/update-release-branch.py deleted file mode 100644 index 263512682..000000000 --- a/.github/update-release-branch.py +++ /dev/null @@ -1,474 +0,0 @@ -import argparse -import datetime -import fileinput -import re -from github import Github -import json -import os -import subprocess - -EMPTY_CHANGELOG = """# CodeQL Action Changelog - -## [UNRELEASED] - -No user facing changes. - -""" - -# NB: This exact commit message is used to find commits for reverting during backports. -# Changing it requires a transition period where both old and new versions are supported. -BACKPORT_COMMIT_MESSAGE = 'Update version and changelog for v' - -# Commit message used for rebuild commits, both those produced by this script and those produced -# by the `Rebuild Action` workflow (`.github/workflows/rebuild.yml`). -REBUILD_COMMIT_MESSAGE = 'Rebuild' - -# Name of the remote -ORIGIN = 'origin' - -# Environment variables to check for a GitHub API token. -TOKEN_ENVIRONMENT_VARIABLES = ('GH_TOKEN', 'GITHUB_TOKEN') - -# Gets a GitHub API token from one of the supported environment variables. -def get_github_token(): - for variable_name in TOKEN_ENVIRONMENT_VARIABLES: - token = os.environ.get(variable_name, '').strip() - if token: - return token - raise Exception('Missing GitHub token. Set GITHUB_TOKEN or GH_TOKEN.') - -# Runs git with the given args and returns the stdout. -# Raises an error if git does not exit successfully (unless passed -# allow_non_zero_exit_code=True). -def run_git(*args, allow_non_zero_exit_code=False): - cmd = ['git', *args] - p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - if not allow_non_zero_exit_code and p.returncode != 0: - raise Exception(f'Call to {" ".join(cmd)} exited with code {p.returncode} stderr: {p.stderr.decode("ascii")}.') - return p.stdout.decode('ascii') - -# Runs the given command, streaming output to the console. -# Raises an error if the command does not exit successfully. -def run_command(*args): - cmd = list(args) - print(f'Running `{" ".join(cmd)}`.') - subprocess.run(cmd, check=True) - -# Rebuilds the action and commits any changes. -def rebuild_action(): - # For backports, the only source-level change vs the source branch is the new version number, - # so we just need to refresh the version embedded in `lib/`. - run_command('npm', 'ci') - run_command('npm', 'run', 'build') - - run_git('add', '--all') - # `git diff --cached --quiet` exits 0 if there are no staged changes, 1 if there are. - if subprocess.run(['git', 'diff', '--cached', '--quiet']).returncode == 0: - print('Rebuild produced no changes; skipping Rebuild commit.') - else: - run_git('commit', '-m', REBUILD_COMMIT_MESSAGE) - print('Created Rebuild commit.') - -# Returns true if the given branch exists on the origin remote -def branch_exists_on_remote(branch_name): - return run_git('ls-remote', '--heads', ORIGIN, branch_name).strip() != '' - -# Opens a PR from the given branch to the target branch -def open_pr( - repo, all_commits, source_branch_short_sha, new_branch_name, source_branch, target_branch, - conductor, is_primary_release, conflicted_files): - # Sort the commits into the pull requests that introduced them, - # and any commits that don't have a pull request - pull_requests = [] - commits_without_pull_requests = [] - for commit in all_commits: - pr = get_pr_for_commit(commit) - - if pr is None: - commits_without_pull_requests.append(commit) - elif not any(p for p in pull_requests if p.number == pr.number): - pull_requests.append(pr) - - print(f'Found {len(pull_requests)} pull requests.') - print(f'Found {len(commits_without_pull_requests)} commits not in a pull request.') - - # Sort PRs and commits by age - pull_requests = sorted(pull_requests, key=lambda pr: pr.number) - commits_without_pull_requests = sorted(commits_without_pull_requests, key=lambda c: c.commit.author.date) - - # Start constructing the body text - body = [] - body.append(f'Merging {source_branch_short_sha} into `{target_branch}`.') - - body.append('') - body.append(f'Conductor for this PR is @{conductor}.') - - # List all PRs merged - if len(pull_requests) > 0: - body.append('') - body.append('Contains the following pull requests:') - for pr in pull_requests: - # Use PR author if they are GitHub staff, otherwise use the merger - display_user = get_pr_author_if_staff(pr) or get_merger_of_pr(repo, pr) - body.append(f'- #{pr.number} (@{display_user})') - - # List all commits not part of a PR - if len(commits_without_pull_requests) > 0: - body.append('') - body.append('Contains the following commits not from a pull request:') - for commit in commits_without_pull_requests: - author_description = f' (@{commit.author.login})' if commit.author is not None else '' - body.append(f'- {commit.sha} - {get_truncated_commit_message(commit)}{author_description}') - - body.append('') - body.append('Please do the following:') - if len(conflicted_files) > 0: - body.append(' - [ ] Ensure `package.json` file contains the correct version.') - body.append(' - [ ] Add a commit to this branch to resolve the merge conflicts ' + - 'in the following files:') - body.extend([f' - `{file}`' for file in conflicted_files]) - body.append(' - [ ] Rebuild the Action locally (`npm run build`) and push any changes to the ' + - f'built output in `lib` as a separate commit named exactly `{REBUILD_COMMIT_MESSAGE}`.') - body.append(' - [ ] Ensure another maintainer has reviewed the additional commits you added to this ' + - 'branch to resolve the merge conflicts.') - body.append(' - [ ] Ensure the CHANGELOG displays the correct version and date.') - body.append(' - [ ] Ensure the CHANGELOG includes all relevant, user-facing changes since the last release.') - body.append(f' - [ ] Check that there are not any unexpected commits being merged into the `{target_branch}` branch.') - body.append(' - [ ] Ensure the docs team is aware of any documentation changes that need to be released.') - - body.append(' - [ ] Approve running the full set of PR checks if you have not pushed any changes.') - body.append(' - [ ] Approve and merge this PR. Make sure `Create a merge commit` is selected rather than `Squash and merge` or `Rebase and merge`.') - - if is_primary_release: - body.append(' - [ ] Merge the mergeback PR that will automatically be created once this PR is merged.') - body.append(' - [ ] Merge all backport PRs to older release branches, that will automatically be created once this PR is merged.') - - title = f'Merge {source_branch} into {target_branch}' - - # Create the pull request - pr = repo.create_pull(title=title, body='\n'.join(body), head=new_branch_name, base=target_branch) - print(f'Created PR #{str(pr.number)}') - - # Assign the conductor - pr.add_to_assignees(conductor) - print(f'Assigned PR to {conductor}') - -# Gets a list of the SHAs of all commits that have happened on the source branch -# since the last release to the target branch. -# This will not include any commits that exist on the target branch -# that aren't on the source branch. -def get_commit_difference(repo, source_branch, target_branch): - # Passing split nothing means that the empty string splits to nothing: compare `''.split() == []` - # to `''.split('\n') == ['']`. - commits = run_git('log', '--pretty=format:%H', f'{ORIGIN}/{target_branch}..{ORIGIN}/{source_branch}').strip().split() - - # Convert to full-fledged commit objects - commits = [repo.get_commit(c) for c in commits] - - # Filter out merge commits for PRs - return list(filter(lambda c: not is_pr_merge_commit(c), commits)) - -# Is the given commit the automatic merge commit from when merging a PR -def is_pr_merge_commit(commit): - return commit.committer is not None and commit.committer.login == 'web-flow' and len(commit.parents) > 1 - -# Gets a copy of the commit message that should display nicely -def get_truncated_commit_message(commit): - message = commit.commit.message.split('\n')[0] - if len(message) > 60: - return f'{message[:57]}...' - else: - return message - -# Converts a commit into the PR that introduced it to the source branch. -# Returns the PR object, or None if no PR could be found. -def get_pr_for_commit(commit): - prs = commit.get_pulls() - - if prs.totalCount > 0: - # In the case that there are multiple PRs, return the earliest one - prs = list(prs) - sorted_prs = sorted(prs, key=lambda pr: int(pr.number)) - return sorted_prs[0] - else: - return None - -# Get the person who merged the pull request. -# For most cases this will be the same as the author, but for PRs opened -# by external contributors getting the merger will get us the GitHub -# employee who reviewed and merged the PR. -def get_merger_of_pr(repo, pr): - return repo.get_commit(pr.merge_commit_sha).author.login - -# Get the PR author if they are GitHub staff, otherwise None. -def get_pr_author_if_staff(pr): - if pr.user is None: - return None - if getattr(pr.user, 'site_admin', False): - return pr.user.login - return None - -def get_current_version(): - with open('package.json', 'r') as f: - return json.load(f)['version'] - -# `npm version` doesn't always work because of merge conflicts, so we -# replace the version in package.json textually. -def replace_version_package_json(prev_version, new_version): - prev_line_is_codeql = False - for line in fileinput.input('package.json', inplace = True, encoding='utf-8'): - if prev_line_is_codeql and f'\"version\": \"{prev_version}\"' in line: - print(line.replace(prev_version, new_version), end='') - else: - prev_line_is_codeql = False - print(line, end='') - if '\"name\": \"codeql\",' in line: - prev_line_is_codeql = True - -def get_today_string(): - today = datetime.datetime.today() - return '{:%d %b %Y}'.format(today) - -def process_changelog_for_backports(source_branch_major_version, target_branch_major_version): - - # changelog entries can use the following format to indicate - # that they only apply to newer versions - some_versions_only_regex = re.compile(r'\[v(\d+)\+ only\]') - - output = '' - - with open('CHANGELOG.md', 'r') as f: - - # until we find the first section, just duplicate all lines - found_first_section = False - while not found_first_section: - line = f.readline() - if not line: - raise Exception('Could not find any change sections in CHANGELOG.md') # EOF - - if line.startswith('## '): - line = line.replace(f'## {source_branch_major_version}', f'## {target_branch_major_version}') - found_first_section = True - - output += line - - # found_content tracks whether we hit two headings in a row - found_content = False - output += '\n' - while True: - line = f.readline() - if not line: - break # EOF - line = line.rstrip('\n') - - # filter out changenote entries that apply only to newer versions - match = some_versions_only_regex.search(line) - if match: - if int(target_branch_major_version) < int(match.group(1)): - continue - - if line.startswith('## '): - line = line.replace(f'## {source_branch_major_version}', f'## {target_branch_major_version}') - if found_content == False: - # we have found two headings in a row, so we need to add the placeholder message. - output += 'No user facing changes.\n' - found_content = False - output += f'\n{line}\n\n' - else: - if line.strip() != '': - found_content = True - # we use the original line here, rather than the stripped version - # so that we preserve indentation - output += line + '\n' - - with open('CHANGELOG.md', 'w') as f: - f.write(output) - -def update_changelog(version): - if (os.path.exists('CHANGELOG.md')): - content = '' - with open('CHANGELOG.md', 'r') as f: - content = f.read() - else: - content = EMPTY_CHANGELOG - - newContent = content.replace('[UNRELEASED]', f'{version} - {get_today_string()}', 1) - - with open('CHANGELOG.md', 'w') as f: - f.write(newContent) - - -def main(): - parser = argparse.ArgumentParser('update-release-branch.py') - - parser.add_argument( - '--repository-nwo', - type=str, - required=True, - help='The nwo of the repository, for example github/codeql-action.' - ) - parser.add_argument( - '--source-branch', - type=str, - required=True, - help='Source branch for release branch update.' - ) - parser.add_argument( - '--target-branch', - type=str, - required=True, - help='Target branch for release branch update.' - ) - parser.add_argument( - '--is-primary-release', - action='store_true', - default=False, - help='Whether this update is the primary release for the current major version.' - ) - parser.add_argument( - '--conductor', - type=str, - required=True, - help='The GitHub handle of the person who is conducting the release process.' - ) - - args = parser.parse_args() - - source_branch = args.source_branch - target_branch = args.target_branch - is_primary_release = args.is_primary_release - - repo = Github(get_github_token()).get_repo(args.repository_nwo) - - # the target branch will be of the form releases/vN, where N is the major version number - target_branch_major_version = target_branch.strip('releases/v') - - # split version into major, minor, patch - _, v_minor, v_patch = get_current_version().split('.') - - version = f"{target_branch_major_version}.{v_minor}.{v_patch}" - - # Print what we intend to go - print(f'Considering difference between {source_branch} and {target_branch}...') - source_branch_short_sha = run_git('rev-parse', '--short', f'{ORIGIN}/{source_branch}').strip() - print(f'Current head of {source_branch} is {source_branch_short_sha}.') - - # See if there are any commits to merge in - commits = get_commit_difference(repo=repo, source_branch=source_branch, target_branch=target_branch) - if len(commits) == 0: - print(f'No commits to merge from {source_branch} to {target_branch}.') - return - - # define distinct prefix in order to support specific pr checks on backports - branch_prefix = 'update' if is_primary_release else 'backport' - - # The branch name is based off of the name of branch being merged into - # and the SHA of the branch being merged from. Thus if the branch already - # exists we can assume we don't need to recreate it. - new_branch_name = f'{branch_prefix}-v{version}-{source_branch_short_sha}' - print(f'Branch name is {new_branch_name}.') - - # Check if the branch already exists. If so we can abort as this script - # has already run on this combination of branches. - if branch_exists_on_remote(new_branch_name): - print(f'Branch {new_branch_name} already exists. Nothing to do.') - return - - # Create the new branch and push it to the remote - print(f'Creating branch {new_branch_name}.') - - # The process of creating the v{Older} release can run into merge conflicts. We commit the unresolved - # conflicts so a maintainer can easily resolve them (vs erroring and requiring maintainers to - # reconstruct the release manually) - conflicted_files = [] - - if not is_primary_release: - - # the source branch will be of the form releases/vN, where N is the major version number - source_branch_major_version = source_branch.strip('releases/v') - - # If we're performing a backport, start from the target branch - print(f'Creating {new_branch_name} from the {ORIGIN}/{target_branch} branch') - run_git('checkout', '-b', new_branch_name, f'{ORIGIN}/{target_branch}') - - # Revert the commit that we made as part of the last release that updated the version number and - # changelog to refer to {older}.x.x variants. This avoids merge conflicts in the changelog and - # package.json files when we merge in the v{latest} branch. - # This commit will not exist the first time we release the v{N-1} branch from the v{N} branch, so we - # use `git log --grep` to conditionally revert the commit. - print('Reverting the version number and changelog updates from the last release to avoid conflicts') - vOlder_update_commits = run_git('log', '--grep', f'^{BACKPORT_COMMIT_MESSAGE}', '--format=%H').split() - - if len(vOlder_update_commits) > 0: - print(f' Reverting {vOlder_update_commits[0]}') - # Only revert the newest commit as older ones will already have been reverted in previous - # releases. - run_git('revert', vOlder_update_commits[0], '--no-edit') - - # Also revert the "Rebuild" commit, whether created by this script or by the - # `Rebuild Action` workflow. - rebuild_commit = run_git('log', '--grep', f'^{REBUILD_COMMIT_MESSAGE}$', '--format=%H').split()[0] - print(f' Reverting {rebuild_commit}') - run_git('revert', rebuild_commit, '--no-edit') - - else: - print(' Nothing to revert.') - - print(f'Merging {ORIGIN}/{source_branch} into the release prep branch') - # Commit any conflicts (see the comment for `conflicted_files`) - run_git('merge', f'{ORIGIN}/{source_branch}', allow_non_zero_exit_code=True) - conflicted_files = run_git('diff', '--name-only', '--diff-filter', 'U').splitlines() - if len(conflicted_files) > 0: - run_git('add', '.') - run_git('commit', '--no-edit') - - # Migrate the package version number from a vLatest version number to a vOlder version number. - # `package-lock.json` is updated as part of the subsequent rebuild step (see `rebuild_action`). - print(f'Setting version number to {version} in package.json') - replace_version_package_json(get_current_version(), version) - run_git('add', 'package.json') - - # Migrate the changelog notes from vLatest version numbers to vOlder version numbers - print(f'Migrating changelog notes from v{source_branch_major_version} to v{target_branch_major_version}') - process_changelog_for_backports(source_branch_major_version, target_branch_major_version) - - # Amend the commit generated by `npm version` to update the CHANGELOG - run_git('add', 'CHANGELOG.md') - run_git('commit', '-m', f'{BACKPORT_COMMIT_MESSAGE}{version}') - else: - # If we're performing a standard release, there won't be any new commits on the target branch, - # as these will have already been merged back into the source branch. Therefore we can just - # start from the source branch. - run_git('checkout', '-b', new_branch_name, f'{ORIGIN}/{source_branch}') - - print('Updating changelog') - update_changelog(version) - - # Create a commit that updates the CHANGELOG - run_git('add', 'CHANGELOG.md') - run_git('commit', '-m', f'Update changelog for v{version}') - - if not is_primary_release: - if len(conflicted_files) == 0: - print('Rebuilding the Action.') - rebuild_action() - else: - print(f'Skipping automatic rebuild because the merge produced conflicts in {conflicted_files}.') - - run_git('push', ORIGIN, new_branch_name) - - # Open a PR to update the branch - open_pr( - repo, - commits, - source_branch_short_sha, - new_branch_name, - source_branch=source_branch, - target_branch=target_branch, - conductor=args.conductor, - is_primary_release=is_primary_release, - conflicted_files=conflicted_files - ) - -if __name__ == '__main__': - main()