Skip to content

Modify update command to directly edit the mbed-os.lib files for each example #3528

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jan 12, 2017
Merged

Modify update command to directly edit the mbed-os.lib files for each example #3528

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
01c3962
Modify update command to directly edit the mbed-os.lib files for each
adbridge Jan 5, 2017
fb27b64
Review comments: Use os.rename rather that system mv command.
adbridge Jan 9, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
292 changes: 182 additions & 110 deletions tools/test/examples/update.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
import subprocess
import shutil
import stat
import re
from github import Github, GithubException

ROOT = abspath(dirname(dirname(dirname(dirname(__file__)))))
sys.path.insert(0, ROOT)
Expand All @@ -33,6 +35,26 @@ def run_cmd(command, print_warning_on_fail=True):

return return_code

def run_cmd_with_output(command, print_warning_on_fail=True):
""" Takes the command specified and runs it in a sub-process, obtaining the return code
and the returned output.

Args:
command - command to run, provided as a list of individual fields which are combined into a
single command before passing to the sub-process call.
return_code - result of the command.
output - the output of the command

"""
print('[Exec] %s' % ' '.join(command))
returncode = 0
output = None
try:
output = subprocess.check_output(command)
except subprocess.CalledProcessError as e:
print("The command '%s' failed with return code: %s" % (' '.join(command), e.returncode))
returncode = e.returncode
return returncode, output

def rmtree_readonly(directory):
""" Deletes a readonly directory tree.
Expand Down Expand Up @@ -63,7 +85,7 @@ def find_all_examples(path):

return examples

def upgrade_single_example(example, tag, directory):
def upgrade_single_example(example, tag, directory, ref):
""" Updates the mbed-os.lib file in the example specified to correspond to the
version specified by the GitHub tag supplied. Also deals with
multiple sub-examples in the GitHub repo, updating them in the same way.
Expand All @@ -72,113 +94,157 @@ def upgrade_single_example(example, tag, directory):
example - json example object containing the GitHub repo to update.
tag - GitHub tag corresponding to a version of mbed-os to upgrade to.
directory - directory path for the example.
ref - SHA corresponding to the supplied tag
returns - True if the upgrade was successful, False otherwise.

"""
print("Upgrading single example at path '%s'" % directory)
cwd = os.getcwd()
os.chdir(directory)

return_code = None

# Change directories to the mbed-os library
if not os.path.exists('mbed-os'):
print("'mbed-os' directory not found in the root of '%s'" % directory)
print("Ignoring and moving on to the next example")
os.chdir(cwd)
return False
return_code = False

os.chdir('mbed-os')

# Setup and run the update command
update_cmd = ['mbed', 'update', tag]
return_code = run_cmd(update_cmd)

if return_code:
os.chdir(cwd)
if os.path.isfile("mbed-os.lib"):
# Rename command will fail on some OS's if the target file already exist,
# so ensure if it does, it is deleted first.
if os.path.isfile("mbed-os.lib_bak"):
os.remove("mbed-os.lib_bak")

os.rename("mbed-os.lib", "mbed-os.lib_bak")
else:
print("!! Error trying to backup mbed-os.lib prior to updating.")
return False

os.chdir('../')

# Setup and run the add command
add_cmd = ['git', 'add', 'mbed-os.lib']
return_code = run_cmd(add_cmd)
# mbed-os.lib file contains one line with the following format
# e.g. https://github.com/ARMmbed/mbed-os/#0789928ee7f2db08a419fa4a032fffd9bd477aa7
lib_re = re.compile('https://github.com/ARMmbed/mbed-os/#[A-Za-z0-9]+')
updated = False

# Scan through mbed-os.lib line by line
with open('mbed-os.lib_bak', 'r') as ip, open('mbed-os.lib', 'w') as op:
for line in ip:

opline = line

regexp = lib_re.match(line)
if regexp:
opline = 'https://github.com/ARMmbed/mbed-os/#' + ref
updated = True

op.write(opline)

if updated:
# Setup and run the git add command
cmd = ['git', 'add', 'mbed-os.lib']
return_code = run_cmd(cmd)

os.chdir(cwd)
return not return_code

def upgrade_example(example, tag):
""" Clones the example specified from GitHub and updates the associated mbed-os.lib file
to correspond to the version specified by the GitHub tag supplied. Also deals with
multiple sub-examples in the GitHub repo, updating them in the same way.
def prepare_fork(arm_example):
""" Synchronises a cloned fork to ensure it is up to date with the original.

Args:
arm_example - Full GitHub repo path for original example
ret - True if the fork was synchronised successfully, False otherwise

"""

print "In " + os.getcwd()

for cmd in [['git', 'remote', 'add', 'armmbed', arm_example],
['git', 'fetch', 'armmbed'],
['git', 'reset', '--hard', 'armmbed/master'],
['git', 'push', '-f', 'origin']]:
if run_cmd(cmd):
print("preparation of the fork failed!")
return False
return True


def upgrade_example(github, example, tag, user, ref):
""" Clone a fork of the example specified.
Ensures the fork is up to date with the original and then and updates the associated
mbed-os.lib file on that fork to correspond to the version specified by the GitHub tag supplied.
Also deals with multiple sub-examples in the GitHub repo, updating them in the same way.
The updates are pushed to the forked repo.
Finally a PR is raised against the original example repo for the changes.

Args:
github - GitHub instance to allow internal git commands to be run
example - json example object containing the GitHub repo to update.
tag - GitHub tag corresponding to a version of mbed-os to upgrade to.
user - GitHub user name
ref - SHA corresponding to the tag

"""
print("Updating example '%s'" % example['name'])
ret = False
print("\nUpdating example '%s'" % example['name'])
cwd = os.getcwd()

# Setup and run the import command
clone_cmd = ['git', 'clone', example['github']]
return_code = run_cmd(clone_cmd)

if return_code:

full_repo_name = 'ARMmbed/'+ example['name']
fork = "https://github.com/" + user + '/' + example['name']

# Check access to mbed-os repo
try:
repo = github.get_repo(full_repo_name, False)

except:
print("\t\t!! Repo does not exist - skipping\n")
return False


# Clone the forked example repo
clone_cmd = ['git', 'clone', fork]
return_code = run_cmd(clone_cmd)

# Find all examples
example_directories = find_all_examples(example['name'])

os.chdir(example['name'])

# Setup and run the update command
import_cmd = ['mbed', 'update']
return_code = run_cmd(import_cmd)
if return_code:
os.chdir(cwd)
return False
if not return_code:

for example_directory in example_directories:
if not upgrade_single_example(example, tag, os.path.relpath(example_directory, example['name'])):
os.chdir(cwd)
return False
# Find all examples
example_directories = find_all_examples(example['name'])

# Setup the default commit message
commit_message = 'Updating mbed-os to {{' + tag +'}}'
os.chdir(example['name'])

# checkout and synchronise the release-candidate branch
prepare_fork(example['github'])

for example_directory in example_directories:
if not upgrade_single_example(example, tag, os.path.relpath(example_directory, example['name']), ref):
os.chdir(cwd)
return False

# Setup the default commit message
commit_message = 'Updating mbed-os to ' + tag

# Setup and run the commit command
commit_cmd = ['git', 'commit', '-m', commit_message]
return_code = run_cmd(commit_cmd)
if not return_code:

# Setup and run the push command
push_cmd = ['git', 'push', 'origin']
return_code = run_cmd(push_cmd)

# Setup and run the commit command
commit_cmd = ['git', 'commit', '-m', commit_message]
return_code = run_cmd(commit_cmd)
if return_code:
if return_code == 1:
print("[WARNING] 'git commit' exited with a return code of 1. " + \
"This usually inidicates that no update was made. Still " + \
"attempting to create a tag.")
if not return_code:
body = "Please test/merge this PR and then tag Master with " + tag
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yay for bots! 😄 🤖

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Until all the example owners have automatic testing of their examples for each PR, then they need the prompt!

# Raise a PR from release-candidate to master
user_fork = user + ':master'
try:
pr = repo.create_pull(title='Updating mbed-os to ' + tag, head=user_fork, base='master', body=body)
ret = True
except GithubException as e:
# Default to False
print("Creation of Pull Request from release-candidate to master failed with the following error!")
print e
else:
print("!!! Git push command failed.")
else:
os.chdir(cwd)
return False

# Setup and run the tag command
tag_cmd = ['git', 'tag', '-a', tag, '-m', tag]
return_code = run_cmd(tag_cmd)
if return_code:
os.chdir(cwd)
return False

# Setup and run the push command
push_cmd = ['git', 'push', 'origin', 'master']
return_code = run_cmd(push_cmd)

if return_code:
os.chdir(cwd)
return False

push_cmd = ['git', 'push', 'origin', tag]
return_code = run_cmd(push_cmd)

print("!!! Git commit command failed.")
else:
print("!!! Could not clone user fork %s\n" % fork)


os.chdir(cwd)
return not return_code
return ret

def create_work_directory(path):
""" Create a new directory specified in 'path', overwrite if the directory already
Expand Down Expand Up @@ -220,11 +286,27 @@ def test_compile(config, tag):


def main(arguments):
""" Will update any mbed-os.lib files found in the example list specified by the config file.
If no config file is specified the default 'examples.json' is used.
The update is done by cloning a fork of each example (the fork must be present in the
github account specified by the github user parameter). The fork is searched for any
mbed-os.lib files and each one found is updated with the SHA corresponding to the supplied
github tag. A pull request is then made from the fork to the original example repo.

Args:
tag - tag to update the mbed-os.lib to. E.g. mbed-os-5.3.1
github_token - Pre-authorised token to allow github access
github_user - github username whose account contains the example forks
config_file - optional parameter to specify a list of examples

"""

parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('tag', help="mbed-os tag to which all examples will be updated")
parser.add_argument('-c', '--config_file', help="Path to the configuration file (default is 'examples.json')", default='examples.json')
parser.add_argument('-T', '--github_token', help="GitHub token for secure access")
parser.add_argument('-U', '--github_user', help="GitHub user for forked repos")

args = parser.parse_args(arguments)

Expand All @@ -238,45 +320,41 @@ def main(arguments):
print("Failed to load config file '%s'" % args.config_file)
sys.exit(1)

# Create work directories
# Create working directory
create_work_directory('examples')


github = Github(args.github_token)

# Get the github sha corresponding to the specified mbed-os tag
cmd = ['git', 'rev-list', '-1', args.tag]
return_code, ref = run_cmd_with_output(cmd)

if return_code:
print("Could not obtain SHA for tag: %s\n" % args.tag)
sys.exit(1)

# Loop through the examples
failures = []
successes = []
not_compiled = []
results = {}
os.chdir('examples')

results = test_compile(config, args.tag)
lib.print_compilation_summary(results)

for example in config['examples']:
# Determine if this example should be updated

# Attempt to update if:
# group of examples passed compilation and
# auto update is set to True
# Note: results fields are [compiled flag, pass flag, successes list, failures list]
if not results[example['name']][0]:
# Example was not compiled
not_compiled += [example['name']]
# Determine if this example should be updated and if so update any found
# mbed-os.lib files.

if upgrade_example(github, example, args.tag, args.github_user, ref):
successes += [example['name']]
else:
if results[example['name']][1] and example['auto-update']:
if upgrade_example(example, args.tag):
successes += [example['name']]
else:
failures += [example['name']]
else:
failures += [example['name']]
failures += [example['name']]

os.chdir('../')

# Finish the script and report the results
print(os.linesep + os.linesep +'Finished updating examples!' + os.linesep)

if successes:
print('The following examples updated successfully:')
print('\nThe following examples updated successfully:')
for success in successes:
print(' - %s' % success)

Expand All @@ -285,11 +363,5 @@ def main(arguments):
for fail in failures:
print(' - %s' % fail)

if not_compiled:
print('The following examples were skipped:')
for example in not_compiled:
print(' - %s' % example)


if __name__ == '__main__':
sys.exit(main(sys.argv[1:]))