diff --git a/cherry_picker/cherry_picker.py b/cherry_picker/cherry_picker.py old mode 100644 new mode 100755 index 580c939..2172306 --- a/cherry_picker/cherry_picker.py +++ b/cherry_picker/cherry_picker.py @@ -1,116 +1,304 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + import click import os import subprocess import webbrowser +import sys +class CherryPicker: -@click.command() -@click.option('--dry-run', is_flag=True) -@click.option('--push', 'pr_remote', metavar='REMOTE', - help='git remote to use for PR branches', default='origin') -@click.argument('commit_sha1', 'The commit sha1 to be cherry-picked') -@click.argument('branches', 'The branches to backport to', nargs=-1) -def cherry_pick(dry_run, pr_remote, commit_sha1, branches): - if not os.path.exists('./pyconfig.h.in'): - os.chdir('./cpython/') - upstream = get_git_fetch_remote() - username = get_forked_repo_name(pr_remote) + def __init__(self, pr_remote, commit_sha1, branches, + *, dry_run=False): + self.pr_remote = pr_remote + self.commit_sha1 = commit_sha1 + self.branches = branches + self.dry_run = dry_run - if dry_run: - click.echo("Dry run requested, listing expected command sequence") + @property + def upstream(self): + """Get the remote name to use for upstream branches + Uses "upstream" if it exists, "origin" otherwise + """ + cmd = "git remote get-url upstream" + try: + subprocess.check_output(cmd.split(), stderr=subprocess.DEVNULL) + except subprocess.CalledProcessError: + return "origin" + return "upstream" + @property + def sorted_branches(self): + return sorted( + self.branches, + reverse=True, + key=lambda v: tuple(map(int, v.split('.')))) - click.echo("fetching upstream ...") - run_cmd(f"git fetch {upstream}", dry_run=dry_run) + @property + def username(self): + cmd = f"git config --get remote.{self.pr_remote}.url" + raw_result = subprocess.check_output(cmd.split(), + stderr=subprocess.STDOUT) + result = raw_result.decode('utf-8') + username_end = result.index('/cpython.git') + if result.startswith("https"): + username = result[len("https://github.com/"):username_end] + else: + username = result[len("git@github.com:"):username_end] + return username - if not branches: - raise ValueError("at least one branch is required") + def get_cherry_pick_branch(self, maint_branch): + return f"backport-{self.commit_sha1[:7]}-{maint_branch}" - for branch in get_sorted_branch(branches): - click.echo(f"Now backporting '{commit_sha1}' into '{branch}'") + def get_pr_url(self, base_branch, head_branch): + return f"https://github.com/python/cpython/compare/{base_branch}...{self.username}:{head_branch}?expand=1" - # git checkout -b 61e2bc7-3.5 upstream/3.5 - cherry_pick_branch = f"backport-{commit_sha1[:7]}-{branch}" - cmd = f"git checkout -b {cherry_pick_branch} {upstream}/{branch}" - run_cmd(cmd, dry_run=dry_run) + def fetch_upstream(self): + """ git fetch """ + self.run_cmd(f"git fetch {self.upstream}") - cmd = f"git cherry-pick -x {commit_sha1}" - if run_cmd(cmd, dry_run=dry_run): - cmd = f"git push {pr_remote} {cherry_pick_branch}" - if not run_cmd(cmd, dry_run=dry_run): - click.echo(f"Failed to push to {pr_remote} :(") - else: - open_pr(username, branch, cherry_pick_branch, dry_run=dry_run) + def run_cmd(self, cmd, shell=False): + if self.dry_run: + click.echo(f" dry-run: {cmd}") + return + if not shell: + output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) else: - click.echo(f"Failed to cherry-pick {commit_sha1} into {branch} :(") + output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + click.echo(output.decode('utf-8')) + + def checkout_branch(self, branch_name): + """ git checkout -b """ + cmd = f"git checkout -b {self.get_cherry_pick_branch(branch_name)} {self.upstream}/{branch_name}" + try: + self.run_cmd(cmd) + except subprocess.CalledProcessError as err: + click.echo("error checking out branch") + click.echo(err.output) + def get_commit_message(self, commit_sha): + """ + Return the commit message for the current commit hash, + replace # with GH- + """ + cmd = f"git show -s --format=%B {commit_sha}" + output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + updated_commit_message = output.strip().decode('utf-8').replace('#', 'GH-') + return updated_commit_message + + def checkout_master(self): + """ git checkout master """ cmd = "git checkout master" - run_cmd(cmd, dry_run=dry_run) + self.run_cmd(cmd) + + def status(self): + """ + git status + :return: + """ + cmd = "git status" + self.run_cmd(cmd) + + def cherry_pick(self): + """ git cherry-pick -x """ + cmd = f"git cherry-pick -x {self.commit_sha1}" + self.run_cmd(cmd) + + def get_exit_message(self, branch): + return \ +f""" +Failed to cherry-pick {self.commit_sha1} into {branch} \u2639 +... Stopping here. + +To continue and resolve the conflict: + $ python -m cherry_picker --status # to find out which files need attention + $ cd cpython + # Fix the conflict + $ cd .. + $ python -m cherry_picker --status # should now say 'all conflict fixed' + $ python -m cherry_picker --continue - cmd = f"git branch -D {cherry_pick_branch}" - if run_cmd(cmd, dry_run=dry_run): - if not dry_run: - click.echo(f"branch {cherry_pick_branch} has been deleted.") +To abort the cherry-pick and cleanup: + $ python -m cherry_picker --abort +""" + + def amend_commit_message(self, cherry_pick_branch): + """ prefix the commit message with (X.Y) """ + base_branch = get_base_branch(cherry_pick_branch) + + updated_commit_message = f"[{base_branch}] {self.get_commit_message(self.commit_sha1)}{os.linesep}(cherry picked from commit {self.commit_sha1})" + updated_commit_message = updated_commit_message.replace('#', 'GH-') + if self.dry_run: + click.echo(f" dry-run: git commit --amend -m '{updated_commit_message}'") else: - click.echo(f"branch {cherry_pick_branch} NOT deleted.") + try: + subprocess.check_output(["git", "commit", "--amend", "-m", + updated_commit_message], + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as cpe: + click.echo("Failed to amend the commit message \u2639") + click.echo(cpe.output) -def get_git_fetch_remote(): - """Get the remote name to use for upstream branches - Uses "upstream" if it exists, "origin" otherwise - """ - cmd = "git remote get-url upstream" - try: - subprocess.check_output(cmd.split(), stderr=subprocess.DEVNULL) - except subprocess.CalledProcessError: - return "origin" - return "upstream" + def push_to_remote(self, base_branch, head_branch): + """ git push """ + cmd = f"git push {self.pr_remote} {head_branch}" + try: + self.run_cmd(cmd) + except subprocess.CalledProcessError: + click.echo(f"Failed to push to {self.pr_remote} \u2639") + else: + self.open_pr(self.get_pr_url(base_branch, head_branch)) -def get_forked_repo_name(pr_remote): - """ - Return 'myusername' out of https://github.com/myusername/cpython - :return: - """ - cmd = f"git config --get remote.{pr_remote}.url" - raw_result = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) - result = raw_result.decode('utf-8') - username_end = result.index('/cpython.git') - if result.startswith("https"): - username = result[len("https://github.com/"):username_end] - else: - username = result[len("git@github.com:"):username_end] - return username + def open_pr(self, url): + """ + open url in the web browser + """ + if self.dry_run: + click.echo(f" dry-run: Create new PR: {url}") + else: + webbrowser.open_new_tab(url) + + def delete_branch(self, branch): + cmd = f"git branch -D {branch}" + self.run_cmd(cmd) + + def cleanup_branch(self, branch): + self.checkout_master() + try: + self.delete_branch(branch) + except subprocess.CalledProcessError: + click.echo(f"branch {branch} NOT deleted.") + else: + click.echo(f"branch {branch} has been deleted.") + def backport(self): + if not self.branches: + raise ValueError("At least one branch must be specified.") + self.fetch_upstream() + + for maint_branch in self.sorted_branches: + click.echo(f"Now backporting '{self.commit_sha1}' into '{maint_branch}'") + + cherry_pick_branch = self.get_cherry_pick_branch(maint_branch) + self.checkout_branch(maint_branch) + try: + self.cherry_pick() + self.amend_commit_message(cherry_pick_branch) + except subprocess.CalledProcessError as cpe: + click.echo(cpe.output) + click.echo(self.get_exit_message(maint_branch)) + sys.exit(-1) + else: + self.push_to_remote(maint_branch, cherry_pick_branch) + self.cleanup_branch(cherry_pick_branch) + + def abort_cherry_pick(self): + """ + run `git cherry-pick --abort` and then clean up the branch + """ + cmd = "git cherry-pick --abort" + try: + self.run_cmd(cmd) + except subprocess.CalledProcessError as cpe: + click.echo(cpe.output) + else: + self.cleanup_branch(get_current_branch()) + + def continue_cherry_pick(self): + """ + git push origin + open the PR + clean up branch + """ + cherry_pick_branch = get_current_branch() + + if cherry_pick_branch != 'master': + # amend the commit message, prefix with [X.Y] + base = get_base_branch(cherry_pick_branch) + short_sha = cherry_pick_branch[cherry_pick_branch.index('-')+1:cherry_pick_branch.index(base)-1] + full_sha = get_full_sha_from_short(short_sha) + commit_message = self.get_commit_message(short_sha) + updated_commit_message = f'[{base}] {commit_message}. \n(cherry picked from commit {full_sha})' + if self.dry_run: + click.echo(f" dry-run: git commit -am '{updated_commit_message}' --allow-empty") + else: + subprocess.check_output(["git", "commit", "-am", updated_commit_message, "--allow-empty"], + stderr=subprocess.STDOUT) + + self.push_to_remote(base, cherry_pick_branch) + + self.cleanup_branch(cherry_pick_branch) + + else: + click.echo(u"Currently in `master` branch. Will not continue. \U0001F61B") + + +@click.command() +@click.option('--dry-run', is_flag=True, + help="Prints out the commands, but not executed.") +@click.option('--push', 'pr_remote', metavar='REMOTE', + help='git remote to use for PR branches', default='origin') +@click.option('--abort', 'abort', flag_value=True, default=None, + help="Abort current cherry-pick and clean up branch") +@click.option('--continue', 'abort', flag_value=False, default=None, + help="Continue cherry-pick, push, and clean up branch") +@click.option('--status', 'status', flag_value = True, default=None, + help="Get the status of cherry-pick") +@click.argument('commit_sha1', 'The commit sha1 to be cherry-picked', nargs=1, + default = "") +@click.argument('branches', 'The branches to backport to', nargs=-1) +def cherry_pick_cli(dry_run, pr_remote, abort, status, commit_sha1, branches): + + click.echo("\U0001F40D \U0001F352 \u26CF") + current_dir = os.path.dirname(os.path.abspath(__file__)) + pyconfig_path = os.path.join(current_dir, '/pyconfig.h.in') + + if not os.path.exists(pyconfig_path): + os.chdir(os.path.join(current_dir, 'cpython')) -def run_cmd(cmd, *, dry_run=False): if dry_run: - click.echo(f" dry-run: {cmd}") - return True - try: - subprocess.check_output(cmd.split()) - except subprocess.CalledProcessError: - return False - return True + click.echo("Dry run requested, listing expected command sequence") + + cherry_picker = CherryPicker(pr_remote, commit_sha1, branches, dry_run=dry_run) + + if abort is not None: + if abort: + cherry_picker.abort_cherry_pick() + else: + cherry_picker.continue_cherry_pick() + + elif status: + click.echo(cherry_picker.status()) + else: + cherry_picker.backport() -def open_pr(forked_repo, base_branch, cherry_pick_branch, *, dry_run=False): +def get_base_branch(cherry_pick_branch): """ - construct the url for pull request and open it in the web browser + return '2.7' from 'backport-sha-2.7' """ - url = f"https://github.com/python/cpython/compare/{base_branch}...{forked_repo}:{cherry_pick_branch}?expand=1" - if dry_run: - click.echo(f" dry-run: Create new PR: {url}") - return - webbrowser.open_new_tab(url) + prefix, sep, base_branch = cherry_pick_branch.rpartition('-') + return base_branch + + +def get_current_branch(): + """ + Return the current branch + """ + cmd = "git symbolic-ref HEAD | sed 's!refs\/heads\/!!'" + output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + return output.strip().decode('utf-8') -def get_sorted_branch(branches): - return sorted( - branches, - reverse=True, - key=lambda v: tuple(map(int, v.split('.')))) +def get_full_sha_from_short(short_sha): + cmd = f"git show --format=raw {short_sha}" + output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + full_sha = output.strip().decode('utf-8').split('\n')[0].split()[1] + return full_sha if __name__ == '__main__': - cherry_pick() + cherry_pick_cli() diff --git a/cherry_picker/readme.rst b/cherry_picker/readme.rst index f002733..4ec7cab 100644 --- a/cherry_picker/readme.rst +++ b/cherry_picker/readme.rst @@ -1,6 +1,6 @@ Usage:: - python -m cherry_picker [--push REMOTE] [--dry-run] + python -m cherry_picker [--push REMOTE] [--dry-run] [--status] [--abort/--continue] @@ -9,8 +9,11 @@ Usage:: About ===== -Use this to backport cpython changes from ``master`` into one or more of the maintenance -branches (``3.6``, ``3.5``, ``2.7``). +Use this to backport cpython changes from ``master`` into one or more of the +maintenance branches (``3.6``, ``3.5``, ``2.7``). + +It will prefix the commit message with the branch, e.g. ``[3.6]``, and then +opens up the pull request page. This script will become obsolete once the cherry-picking bot is implemented. @@ -43,6 +46,7 @@ repository are pushed to `origin`. If this is incorrect, then the correct remote will need be specified using the ``--push`` option (e.g. ``--push pr`` to use a remote named ``pr``). + Cherry-picking :snake: :cherries: :pick: ============== @@ -50,9 +54,35 @@ Cherry-picking :snake: :cherries: :pick: :: - (venv) $ python -m cherry_picker + (venv) $ python -m cherry_picker [--dry-run] [--abort/--continue] [--status] + +The commit sha1 is obtained from the merged pull request on ``master``. + + +Options +------- + +:: + + -- dry-run Dry Run Mode. Prints out the commands, but not executed. + -- push REMOTE Specify the git remote to push into. Default is 'origin'. + -- status Do `git status` in cpython directory. + + +Additional options:: + + -- abort Abort current cherry-pick and clean up branch + -- continue Continue cherry-pick, push, and clean up branch + + +Demo +---- + +https://asciinema.org/a/dtayqmjvd5hy5389oubkdk323 -The commit sha1 is obtained from the merged pull request on ``master``. + +Example +------- For example, to cherry-pick ``6de2b7817f-some-commit-sha1-d064`` into ``3.5`` and ``3.6``: @@ -82,7 +112,22 @@ What this will do: (venv) $ git checkout master (venv) $ git branch -D backport-6de2b78-3.6 -In case of merge conflicts or errors, then... the script will fail :stuck_out_tongue: +In case of merge conflicts or errors, the following message will be displayed:: + + Failed to cherry-pick 554626ada769abf82a5dabe6966afa4265acb6a6 into 2.7 :frowning_face: + ... Stopping here. + + To continue and resolve the conflict: + $ python -m cherry_picker --status # to find out which files need attention + $ cd cpython + # Fix the conflict + $ cd .. + $ python -m cherry_picker --status # should now say 'all conflict fixed' + $ python -m cherry_picker --continue + + To abort the cherry-pick and cleanup: + $ python -m cherry_picker --abort + Passing the `--dry-run` option will cause the script to print out all the steps it would execute without actually executing any of them. For example:: @@ -106,6 +151,23 @@ steps it would execute without actually executing any of them. For example:: dry_run: git checkout master dry_run: git branch -D backport-1e32a1b-3.5 + +`--status` option +----------------- + +This will do `git status` for the CPython directory. + +`--abort` option +---------------- + +Cancels the current cherry-pick and cleans up the cherry-pick branch. + +`--continue` option +------------------- + +Continues the current cherry-pick, commits, pushes the current branch to origin, +opens the PR page, and cleans up the branch. + Creating Pull Requests ====================== @@ -117,15 +179,11 @@ The url of the pull request page looks similar to the following:: https://github.com/python/cpython/compare/3.5...:backport-6de2b78-3.5?expand=1 -1. Prefix the pull request description with the branch ``[X.Y]``, e.g.:: - - [3.6] bpo-xxxxx: Fix this and that - -2. Apply the appropriate ``cherry-pick for ...`` label +1. Apply the appropriate ``cherry-pick for ...`` label -3. Press the ``Create Pull Request`` button. +2. Press the ``Create Pull Request`` button. -4. Remove the ``needs backport to ...`` label from the original pull request +3. Remove the ``needs backport to ...`` label from the original pull request against ``master``. diff --git a/cherry_picker/requirements.txt b/cherry_picker/requirements.txt index c3caddc..989ad50 100644 --- a/cherry_picker/requirements.txt +++ b/cherry_picker/requirements.txt @@ -4,5 +4,8 @@ packaging==16.8 py==1.4.33 pyparsing==2.1.10 pytest==3.0.7 +pytest-cov==2.4.0 +pytest-mock==1.6.0 requests==2.13.0 six==1.10.0 +coverage==4.3.4 \ No newline at end of file diff --git a/cherry_picker/test.py b/cherry_picker/test.py index 1e3f833..9487234 100644 --- a/cherry_picker/test.py +++ b/cherry_picker/test.py @@ -1,7 +1,103 @@ -from . import cherry_picker +from unittest import mock +from .cherry_picker import get_base_branch, get_current_branch, \ + get_full_sha_from_short, CherryPicker -def test_sorted_branch(): + +def test_get_base_branch(): + cherry_pick_branch = 'backport-afc23f4-2.7' + result = get_base_branch(cherry_pick_branch) + assert result == '2.7' + + +def test_get_base_branch_without_dash(): + cherry_pick_branch ='master' + result = get_base_branch(cherry_pick_branch) + assert result == 'master' + + +@mock.patch('subprocess.check_output') +def test_get_current_branch(subprocess_check_output): + subprocess_check_output.return_value = b'master' + assert get_current_branch() == 'master' + + +@mock.patch('subprocess.check_output') +def test_get_full_sha_from_short(subprocess_check_output): + mock_output = b"""commit 22a594a0047d7706537ff2ac676cdc0f1dcb329c +tree 14ab2ea85e7a28adb9d40f185006308d87a67f47 +parent 5908300e4b0891fc5ab8bd24fba8fac72012eaa7 +author Armin Rigo 1492106895 +0200 +committer Mariatta 1492106895 -0700 + + bpo-29694: race condition in pathlib mkdir with flags parents=True (GH-1089) + +diff --git a/Lib/pathlib.py b/Lib/pathlib.py +index fc7ce5e..1914229 100644 +--- a/Lib/pathlib.py ++++ b/Lib/pathlib.py +""" + subprocess_check_output.return_value = mock_output + assert get_full_sha_from_short('22a594a') == '22a594a0047d7706537ff2ac676cdc0f1dcb329c' + + +@mock.patch('os.path.exists') +def test_sorted_branch(os_path_exists): + os_path_exists.return_value = True branches = ["3.1", "2.7", "3.10", "3.6"] - result = cherry_picker.get_sorted_branch(branches) - assert result == ["3.10", "3.6", "3.1", "2.7"] + cp = CherryPicker('origin', '22a594a0047d7706537ff2ac676cdc0f1dcb329c', branches) + assert cp.sorted_branches == ["3.10", "3.6", "3.1", "2.7"] + + +@mock.patch('os.path.exists') +def test_get_cherry_pick_branch(os_path_exists): + os_path_exists.return_value = True + branches = ["3.6"] + cp = CherryPicker('origin', '22a594a0047d7706537ff2ac676cdc0f1dcb329c', branches) + assert cp.get_cherry_pick_branch("3.6") == "backport-22a594a-3.6" + + +@mock.patch('os.path.exists') +@mock.patch('subprocess.check_output') +def test_get_pr_url(subprocess_check_output, os_path_exists): + os_path_exists.return_value = True + subprocess_check_output.return_value = b'https://github.com/mock_user/cpython.git' + branches = ["3.6"] + cp = CherryPicker('origin', '22a594a0047d7706537ff2ac676cdc0f1dcb329c', + branches) + assert cp.get_pr_url("3.6", cp.get_cherry_pick_branch("3.6")) \ + == "https://github.com/python/cpython/compare/3.6...mock_user:backport-22a594a-3.6?expand=1" + + +@mock.patch('os.path.exists') +@mock.patch('subprocess.check_output') +def test_username_ssh(subprocess_check_output, os_path_exists): + os_path_exists.return_value = True + subprocess_check_output.return_value = b'git@github.com:mock_user/cpython.git' + branches = ["3.6"] + cp = CherryPicker('origin', '22a594a0047d7706537ff2ac676cdc0f1dcb329c', + branches) + assert cp.username == 'mock_user' + + +@mock.patch('os.path.exists') +@mock.patch('subprocess.check_output') +def test_username_https(subprocess_check_output, os_path_exists): + os_path_exists.return_value = True + subprocess_check_output.return_value = b'https://github.com/mock_user/cpython.git' + branches = ["3.6"] + cp = CherryPicker('origin', '22a594a0047d7706537ff2ac676cdc0f1dcb329c', + branches) + assert cp.username == 'mock_user' + + +@mock.patch('os.path.exists') +@mock.patch('subprocess.check_output') +def test_get_updated_commit_message(subprocess_check_output, os_path_exists): + os_path_exists.return_value = True + subprocess_check_output.return_value = b'bpo-123: Fix Spam Module (#113)' + branches = ["3.6"] + cp = CherryPicker('origin', '22a594a0047d7706537ff2ac676cdc0f1dcb329c', + branches) + assert cp.get_commit_message('22a594a0047d7706537ff2ac676cdc0f1dcb329c') \ + == 'bpo-123: Fix Spam Module (GH-113)' \ No newline at end of file