Browse Source
Release management using CircleCI (#3498)
* Release management using CircleCI * Changelog updatedpull/3438/head
Greg Szabo
6 years ago
committed by
Ismail Khoffi
Ismail Khoffi
13 changed files with 578 additions and 9 deletions
Unified View
Diff Options
-
+126 -2.circleci/config.yml
-
+2 -0CHANGELOG_PENDING.md
-
+2 -2DOCKER/Dockerfile
-
+6 -5Makefile
-
+65 -0scripts/release_management/README.md
-
+37 -0scripts/release_management/bump-semver.py
-
+60 -0scripts/release_management/github-draft.py
-
+52 -0scripts/release_management/github-openpr.py
-
+28 -0scripts/release_management/github-public-newbranch.bash
-
+53 -0scripts/release_management/github-publish.py
-
+68 -0scripts/release_management/github-upload.py
-
+35 -0scripts/release_management/sha-files.py
-
+44 -0scripts/release_management/zip-file.py
+ 126
- 2
.circleci/config.yml
View File
+ 2
- 0
CHANGELOG_PENDING.md
View File
+ 2
- 2
DOCKER/Dockerfile
View File
+ 6
- 5
Makefile
View File
+ 65
- 0
scripts/release_management/README.md
View File
| @ -0,0 +1,65 @@ | |||||
| # Release management scripts | |||||
| ## Overview | |||||
| The scripts in this folder are used for release management in CircleCI. Although the scripts are fully configurable using input parameters, | |||||
| the default settings were modified to accommodate CircleCI execution. | |||||
| # Build scripts | |||||
| These scripts help during the build process. They prepare the release files. | |||||
| ## bump-semver.py | |||||
| Bumps the semantic version of the input `--version`. Versions are expected in vMAJOR.MINOR.PATCH format or vMAJOR.MINOR format. | |||||
| In vMAJOR.MINOR format, the result will be patch version 0 of that version, for example `v1.2 -> v1.2.0`. | |||||
| In vMAJOR.MINOR.PATCH format, the result will be a bumped PATCH version, for example `v1.2.3 -> v1.2.4`. | |||||
| If the PATCH number contains letters, it is considered a development version, in which case, the result is the non-development version of that number. | |||||
| The patch number will not be bumped, only the "-dev" or similar additional text will be removed. For example: `v1.2.6-rc1 -> v1.2.6`. | |||||
| ## zip-file.py | |||||
| Specialized ZIP command for release management. Special features: | |||||
| 1. Uses Python ZIP libaries, so the `zip` command does not need to be installed. | |||||
| 1. Can only zip one file. | |||||
| 1. Optionally gets file version, Go OS and architecture. | |||||
| 1. By default all inputs and output is formatted exactly how CircleCI needs it. | |||||
| By default, the command will try to ZIP the file at `build/tendermint_${GOOS}_${GOARCH}`. | |||||
| This can be changed with the `--file` input parameter. | |||||
| By default, the command will output the ZIP file to `build/tendermint_${CIRCLE_TAG}_${GOOS}_${GOARCH}.zip`. | |||||
| This can be changed with the `--destination` (folder), `--version`, `--goos` and `--goarch` input parameters respectively. | |||||
| ## sha-files.py | |||||
| Specialized `shasum` command for release management. Special features: | |||||
| 1. Reads all ZIP files in the given folder. | |||||
| 1. By default all inputs and output is formatted exactly how CircleCI needs it. | |||||
| By default, the command will look up all ZIP files in the `build/` folder. | |||||
| By default, the command will output results into the `build/SHA256SUMS` file. | |||||
| # GitHub management | |||||
| Uploading build results to GitHub requires at least these steps: | |||||
| 1. Create a new release on GitHub with content | |||||
| 2. Upload all binaries to the release | |||||
| 3. Publish the release | |||||
| The below scripts help with these steps. | |||||
| ## github-draft.py | |||||
| Creates a GitHub release and fills the content with the CHANGELOG.md link. The version number can be changed by the `--version` parameter. | |||||
| By default, the command will use the tendermint/tendermint organization/repo, which can be changed using the `--org` and `--repo` parameters. | |||||
| By default, the command will get the version number from the `${CIRCLE_TAG}` variable. | |||||
| Returns the GitHub release ID. | |||||
| ## github-upload.py | |||||
| Upload a file to a GitHub release. The release is defined by the mandatory `--id` (release ID) input parameter. | |||||
| By default, the command will upload the file `/tmp/workspace/tendermint_${CIRCLE_TAG}_${GOOS}_${GOARCH}.zip`. This can be changed by the `--file` input parameter. | |||||
| ## github-publish.py | |||||
| Publish a GitHub release. The release is defined by the mandatory `--id` (release ID) input parameter. | |||||
+ 37
- 0
scripts/release_management/bump-semver.py
View File
| @ -0,0 +1,37 @@ | |||||
| #!/usr/bin/env python | |||||
| # Bump the release number of a semantic version number and print it. --version is required. | |||||
| # Version is | |||||
| # - vA.B.C, in which case vA.B.C+1 will be returned | |||||
| # - vA.B.C-devorwhatnot in which case vA.B.C will be returned | |||||
| # - vA.B in which case vA.B.0 will be returned | |||||
| import re | |||||
| import argparse | |||||
| def semver(ver): | |||||
| if re.match('v[0-9]+\.[0-9]+',ver) is None: | |||||
| ver="v0.0" | |||||
| #raise argparse.ArgumentTypeError('--version must be a semantic version number with major, minor and patch numbers') | |||||
| return ver | |||||
| if __name__ == "__main__": | |||||
| parser = argparse.ArgumentParser() | |||||
| parser.add_argument("--version", help="Version number to bump, e.g.: v1.0.0", required=True, type=semver) | |||||
| args = parser.parse_args() | |||||
| found = re.match('(v[0-9]+\.[0-9]+)(\.(.+))?', args.version) | |||||
| majorminorprefix = found.group(1) | |||||
| patch = found.group(3) | |||||
| if patch is None: | |||||
| patch = "0-new" | |||||
| if re.match('[0-9]+$',patch) is None: | |||||
| patchfound = re.match('([0-9]+)',patch) | |||||
| patch = int(patchfound.group(1)) | |||||
| else: | |||||
| patch = int(patch) + 1 | |||||
| print("{0}.{1}".format(majorminorprefix, patch)) | |||||
+ 60
- 0
scripts/release_management/github-draft.py
View File
| @ -0,0 +1,60 @@ | |||||
| #!/usr/bin/env python | |||||
| # Create a draft release on GitHub. By default in the tendermint/tendermint repo. | |||||
| # Optimized for CircleCI | |||||
| import argparse | |||||
| import httplib | |||||
| import json | |||||
| import os | |||||
| from base64 import b64encode | |||||
| def request(org, repo, data): | |||||
| user_and_pass = b64encode(b"{0}:{1}".format(os.environ['GITHUB_USERNAME'], os.environ['GITHUB_TOKEN'])).decode("ascii") | |||||
| headers = { | |||||
| 'User-Agent': 'tenderbot', | |||||
| 'Accept': 'application/vnd.github.v3+json', | |||||
| 'Authorization': 'Basic %s' % user_and_pass | |||||
| } | |||||
| conn = httplib.HTTPSConnection('api.github.com', timeout=5) | |||||
| conn.request('POST', '/repos/{0}/{1}/releases'.format(org,repo), data, headers) | |||||
| response = conn.getresponse() | |||||
| if response.status < 200 or response.status > 299: | |||||
| print("{0}: {1}".format(response.status, response.reason)) | |||||
| conn.close() | |||||
| raise IOError(response.reason) | |||||
| responsedata = response.read() | |||||
| conn.close() | |||||
| return json.loads(responsedata) | |||||
| def create_draft(org,repo,version): | |||||
| draft = { | |||||
| 'tag_name': version, | |||||
| 'target_commitish': 'master', | |||||
| 'name': '{0} (WARNING: ALPHA SOFTWARE)'.format(version), | |||||
| 'body': '<a href=https://github.com/{0}/{1}/blob/master/CHANGELOG.md#{2}>https://github.com/{0}/{1}/blob/master/CHANGELOG.md#{2}</a>'.format(org,repo,version.replace('v','').replace('.','')), | |||||
| 'draft': True, | |||||
| 'prerelease': False | |||||
| } | |||||
| data=json.dumps(draft) | |||||
| return request(org, repo, data) | |||||
| if __name__ == "__main__": | |||||
| parser = argparse.ArgumentParser() | |||||
| parser.add_argument("--org", default="tendermint", help="GitHub organization") | |||||
| parser.add_argument("--repo", default="tendermint", help="GitHub repository") | |||||
| parser.add_argument("--version", default=os.environ.get('CIRCLE_TAG'), help="Version number for binary, e.g.: v1.0.0") | |||||
| args = parser.parse_args() | |||||
| if not os.environ.has_key('GITHUB_USERNAME'): | |||||
| raise parser.error('environment variable GITHUB_USERNAME is required') | |||||
| if not os.environ.has_key('GITHUB_TOKEN'): | |||||
| raise parser.error('environment variable GITHUB_TOKEN is required') | |||||
| release = create_draft(args.org,args.repo,args.version) | |||||
| print(release["id"]) | |||||
+ 52
- 0
scripts/release_management/github-openpr.py
View File
| @ -0,0 +1,52 @@ | |||||
| #!/usr/bin/env python | |||||
| # Open a PR against the develop branch. --branch required. | |||||
| # Optimized for CircleCI | |||||
| import json | |||||
| import os | |||||
| import argparse | |||||
| import httplib | |||||
| from base64 import b64encode | |||||
| def request(org, repo, data): | |||||
| user_and_pass = b64encode(b"{0}:{1}".format(os.environ['GITHUB_USERNAME'], os.environ['GITHUB_TOKEN'])).decode("ascii") | |||||
| headers = { | |||||
| 'User-Agent': 'tenderbot', | |||||
| 'Accept': 'application/vnd.github.v3+json', | |||||
| 'Authorization': 'Basic %s' % user_and_pass | |||||
| } | |||||
| conn = httplib.HTTPSConnection('api.github.com', timeout=5) | |||||
| conn.request('POST', '/repos/{0}/{1}/pulls'.format(org,repo), data, headers) | |||||
| response = conn.getresponse() | |||||
| if response.status < 200 or response.status > 299: | |||||
| print(response) | |||||
| conn.close() | |||||
| raise IOError(response.reason) | |||||
| responsedata = response.read() | |||||
| conn.close() | |||||
| return json.loads(responsedata) | |||||
| if __name__ == "__main__": | |||||
| parser = argparse.ArgumentParser() | |||||
| parser.add_argument("--org", default="tendermint", help="GitHub organization. Defaults to tendermint.") | |||||
| parser.add_argument("--repo", default="tendermint", help="GitHub repository. Defaults to tendermint.") | |||||
| parser.add_argument("--head", help="The name of the branch where your changes are implemented.", required=True) | |||||
| parser.add_argument("--base", help="The name of the branch you want the changes pulled into.", required=True) | |||||
| parser.add_argument("--title", default="Security release {0}".format(os.environ.get('CIRCLE_TAG')), help="The title of the pull request.") | |||||
| args = parser.parse_args() | |||||
| if not os.environ.has_key('GITHUB_USERNAME'): | |||||
| raise parser.error('GITHUB_USERNAME not set.') | |||||
| if not os.environ.has_key('GITHUB_TOKEN'): | |||||
| raise parser.error('GITHUB_TOKEN not set.') | |||||
| if os.environ.get('CIRCLE_TAG') is None: | |||||
| raise parser.error('CIRCLE_TAG not set.') | |||||
| result = request(args.org, args.repo, data=json.dumps({'title':"{0}".format(args.title),'head':"{0}".format(args.head),'base':"{0}".format(args.base),'body':"<Please fill in details.>"})) | |||||
| print(result['html_url']) | |||||
+ 28
- 0
scripts/release_management/github-public-newbranch.bash
View File
| @ -0,0 +1,28 @@ | |||||
| #!/bin/sh | |||||
| # github-public-newbranch.bash - create public branch from the security repository | |||||
| set -euo pipefail | |||||
| # Create new branch | |||||
| BRANCH="${CIRCLE_TAG:-v0.0.0}-security-`date -u +%Y%m%d%H%M%S`" | |||||
| # Check if the patch release exist already as a branch | |||||
| if [ -n "`git branch | grep '${BRANCH}'`" ]; then | |||||
| echo "WARNING: Branch ${BRANCH} already exists." | |||||
| else | |||||
| echo "Creating branch ${BRANCH}." | |||||
| git branch "${BRANCH}" | |||||
| fi | |||||
| # ... and check it out | |||||
| git checkout "${BRANCH}" | |||||
| # Add entry to public repository | |||||
| git remote add tendermint-origin git@github.com:tendermint/tendermint.git | |||||
| # Push branch and tag to public repository | |||||
| git push tendermint-origin | |||||
| git push tendermint-origin --tags | |||||
| # Create a PR from the public branch to the assumed release branch in public (release branch has to exist) | |||||
| python -u scripts/release_management/github-openpr.py --head "${BRANCH}" --base "${BRANCH:%.*}" | |||||
+ 53
- 0
scripts/release_management/github-publish.py
View File
| @ -0,0 +1,53 @@ | |||||
| #!/usr/bin/env python | |||||
| # Publish an existing GitHub draft release. --id required. | |||||
| # Optimized for CircleCI | |||||
| import json | |||||
| import os | |||||
| import argparse | |||||
| import httplib | |||||
| from base64 import b64encode | |||||
| def request(org, repo, id, data): | |||||
| user_and_pass = b64encode(b"{0}:{1}".format(os.environ['GITHUB_USERNAME'], os.environ['GITHUB_TOKEN'])).decode("ascii") | |||||
| headers = { | |||||
| 'User-Agent': 'tenderbot', | |||||
| 'Accept': 'application/vnd.github.v3+json', | |||||
| 'Authorization': 'Basic %s' % user_and_pass | |||||
| } | |||||
| conn = httplib.HTTPSConnection('api.github.com', timeout=5) | |||||
| conn.request('POST', '/repos/{0}/{1}/releases/{2}'.format(org,repo,id), data, headers) | |||||
| response = conn.getresponse() | |||||
| if response.status < 200 or response.status > 299: | |||||
| print(response) | |||||
| conn.close() | |||||
| raise IOError(response.reason) | |||||
| responsedata = response.read() | |||||
| conn.close() | |||||
| return json.loads(responsedata) | |||||
| if __name__ == "__main__": | |||||
| parser = argparse.ArgumentParser() | |||||
| parser.add_argument("--org", default="tendermint", help="GitHub organization") | |||||
| parser.add_argument("--repo", default="tendermint", help="GitHub repository") | |||||
| parser.add_argument("--id", help="GitHub release ID", required=True, type=int) | |||||
| parser.add_argument("--version", default=os.environ.get('CIRCLE_TAG'), help="Version number for the release, e.g.: v1.0.0") | |||||
| args = parser.parse_args() | |||||
| if not os.environ.has_key('GITHUB_USERNAME'): | |||||
| raise parser.error('GITHUB_USERNAME not set.') | |||||
| if not os.environ.has_key('GITHUB_TOKEN'): | |||||
| raise parser.error('GITHUB_TOKEN not set.') | |||||
| try: | |||||
| result = request(args.org, args.repo, args.id, data=json.dumps({'draft':False,'tag_name':"{0}".format(args.version)})) | |||||
| except IOError as e: | |||||
| print(e) | |||||
| result = request(args.org, args.repo, args.id, data=json.dumps({'draft':False,'tag_name':"{0}-autorelease".format(args.version)})) | |||||
| print(result['name']) | |||||
+ 68
- 0
scripts/release_management/github-upload.py
View File
| @ -0,0 +1,68 @@ | |||||
| #!/usr/bin/env python | |||||
| # Upload a file to a GitHub draft release. --id and --file are required. | |||||
| # Optimized for CircleCI | |||||
| import json | |||||
| import os | |||||
| import re | |||||
| import argparse | |||||
| import mimetypes | |||||
| import httplib | |||||
| from base64 import b64encode | |||||
| def request(baseurl, path, mimetype, mimeencoding, data): | |||||
| user_and_pass = b64encode(b"{0}:{1}".format(os.environ['GITHUB_USERNAME'], os.environ['GITHUB_TOKEN'])).decode("ascii") | |||||
| headers = { | |||||
| 'User-Agent': 'tenderbot', | |||||
| 'Accept': 'application/vnd.github.v3.raw+json', | |||||
| 'Authorization': 'Basic %s' % user_and_pass, | |||||
| 'Content-Type': mimetype, | |||||
| 'Content-Encoding': mimeencoding | |||||
| } | |||||
| conn = httplib.HTTPSConnection(baseurl, timeout=5) | |||||
| conn.request('POST', path, data, headers) | |||||
| response = conn.getresponse() | |||||
| if response.status < 200 or response.status > 299: | |||||
| print(response) | |||||
| conn.close() | |||||
| raise IOError(response.reason) | |||||
| responsedata = response.read() | |||||
| conn.close() | |||||
| return json.loads(responsedata) | |||||
| if __name__ == "__main__": | |||||
| parser = argparse.ArgumentParser() | |||||
| parser.add_argument("--id", help="GitHub release ID", required=True, type=int) | |||||
| parser.add_argument("--file", default="/tmp/workspace/tendermint_{0}_{1}_{2}.zip".format(os.environ.get('CIRCLE_TAG'),os.environ.get('GOOS'),os.environ.get('GOARCH')), help="File to upload") | |||||
| parser.add_argument("--return-id-only", help="Return only the release ID after upload to GitHub.", action='store_true') | |||||
| args = parser.parse_args() | |||||
| if not os.environ.has_key('GITHUB_USERNAME'): | |||||
| raise parser.error('GITHUB_USERNAME not set.') | |||||
| if not os.environ.has_key('GITHUB_TOKEN'): | |||||
| raise parser.error('GITHUB_TOKEN not set.') | |||||
| mimetypes.init() | |||||
| filename = os.path.basename(args.file) | |||||
| mimetype,mimeencoding = mimetypes.guess_type(filename, strict=False) | |||||
| if mimetype is None: | |||||
| mimetype = 'application/zip' | |||||
| if mimeencoding is None: | |||||
| mimeencoding = 'utf8' | |||||
| with open(args.file,'rb') as f: | |||||
| asset = f.read() | |||||
| result = request('uploads.github.com', '/repos/tendermint/tendermint/releases/{0}/assets?name={1}'.format(args.id, filename), mimetype, mimeencoding, asset) | |||||
| if args.return_id_only: | |||||
| print(result['id']) | |||||
| else: | |||||
| print(result['browser_download_url']) | |||||
+ 35
- 0
scripts/release_management/sha-files.py
View File
| @ -0,0 +1,35 @@ | |||||
| #!/usr/bin/env python | |||||
| # Create SHA256 summaries from all ZIP files in a folder | |||||
| # Optimized for CircleCI | |||||
| import re | |||||
| import os | |||||
| import argparse | |||||
| import zipfile | |||||
| import hashlib | |||||
| BLOCKSIZE = 65536 | |||||
| if __name__ == "__main__": | |||||
| parser = argparse.ArgumentParser() | |||||
| parser.add_argument("--folder", default="/tmp/workspace", help="Folder to look for, for ZIP files") | |||||
| parser.add_argument("--shafile", default="/tmp/workspace/SHA256SUMS", help="SHA256 summaries File") | |||||
| args = parser.parse_args() | |||||
| for filename in os.listdir(args.folder): | |||||
| if re.search('\.zip$',filename) is None: | |||||
| continue | |||||
| if not os.path.isfile(os.path.join(args.folder, filename)): | |||||
| continue | |||||
| with open(args.shafile,'a+') as shafile: | |||||
| hasher = hashlib.sha256() | |||||
| with open(os.path.join(args.folder, filename),'r') as f: | |||||
| buf = f.read(BLOCKSIZE) | |||||
| while len(buf) > 0: | |||||
| hasher.update(buf) | |||||
| buf = f.read(BLOCKSIZE) | |||||
| shafile.write("{0} {1}\n".format(hasher.hexdigest(),filename)) | |||||
+ 44
- 0
scripts/release_management/zip-file.py
View File
| @ -0,0 +1,44 @@ | |||||
| #!/usr/bin/env python | |||||
| # ZIP one file as "tendermint" into a ZIP like tendermint_VERSION_OS_ARCH.zip | |||||
| # Use environment variables CIRCLE_TAG, GOOS and GOARCH for easy input parameters. | |||||
| # Optimized for CircleCI | |||||
| import os | |||||
| import argparse | |||||
| import zipfile | |||||
| import hashlib | |||||
| BLOCKSIZE = 65536 | |||||
| def zip_asset(file,destination,arcname,version,goos,goarch): | |||||
| filename = os.path.basename(file) | |||||
| output = "{0}/{1}_{2}_{3}_{4}.zip".format(destination,arcname,version,goos,goarch) | |||||
| with zipfile.ZipFile(output,'w') as f: | |||||
| f.write(filename=file,arcname=arcname) | |||||
| f.comment=filename | |||||
| return output | |||||
| if __name__ == "__main__": | |||||
| parser = argparse.ArgumentParser() | |||||
| parser.add_argument("--file", default="build/tendermint_{0}_{1}".format(os.environ.get('GOOS'),os.environ.get('GOARCH')), help="File to zip") | |||||
| parser.add_argument("--destination", default="build", help="Destination folder for files") | |||||
| parser.add_argument("--version", default=os.environ.get('CIRCLE_TAG'), help="Version number for binary, e.g.: v1.0.0") | |||||
| parser.add_argument("--goos", default=os.environ.get('GOOS'), help="GOOS parameter") | |||||
| parser.add_argument("--goarch", default=os.environ.get('GOARCH'), help="GOARCH parameter") | |||||
| args = parser.parse_args() | |||||
| if args.version is None: | |||||
| raise parser.error("argument --version is required") | |||||
| if args.goos is None: | |||||
| raise parser.error("argument --goos is required") | |||||
| if args.goarch is None: | |||||
| raise parser.error("argument --goarch is required") | |||||
| file = zip_asset(args.file,args.destination,"tendermint",args.version,args.goos,args.goarch) | |||||
| print(file) | |||||