Browse Source
Release management using CircleCI (#3498)
* Release management using CircleCI * Changelog updatedpull/3438/head
Greg Szabo
5 years ago
committed by
Ismail Khoffi
Ismail Khoffi
13 changed files with 578 additions and 9 deletions
Split View
Diff Options
-
+126 -2.circleci/config.yml
-
+2 -0CHANGELOG_PENDING.md
-
+2 -2DOCKER/Dockerfile
-
+6 -5Makefile
-
+65 -0scripts/release_management/README.md
-
+37 -0scripts/release_management/bump-semver.py
-
+60 -0scripts/release_management/github-draft.py
-
+52 -0scripts/release_management/github-openpr.py
-
+28 -0scripts/release_management/github-public-newbranch.bash
-
+53 -0scripts/release_management/github-publish.py
-
+68 -0scripts/release_management/github-upload.py
-
+35 -0scripts/release_management/sha-files.py
-
+44 -0scripts/release_management/zip-file.py
+ 126
- 2
.circleci/config.yml
View File
+ 2
- 0
CHANGELOG_PENDING.md
View File
+ 2
- 2
DOCKER/Dockerfile
View File
+ 6
- 5
Makefile
View File
+ 65
- 0
scripts/release_management/README.md
View File
| @ -0,0 +1,65 @@ | |||
| # Release management scripts | |||
| ## Overview | |||
| The scripts in this folder are used for release management in CircleCI. Although the scripts are fully configurable using input parameters, | |||
| the default settings were modified to accommodate CircleCI execution. | |||
| # Build scripts | |||
| These scripts help during the build process. They prepare the release files. | |||
| ## bump-semver.py | |||
| Bumps the semantic version of the input `--version`. Versions are expected in vMAJOR.MINOR.PATCH format or vMAJOR.MINOR format. | |||
| In vMAJOR.MINOR format, the result will be patch version 0 of that version, for example `v1.2 -> v1.2.0`. | |||
| In vMAJOR.MINOR.PATCH format, the result will be a bumped PATCH version, for example `v1.2.3 -> v1.2.4`. | |||
| If the PATCH number contains letters, it is considered a development version, in which case, the result is the non-development version of that number. | |||
| The patch number will not be bumped, only the "-dev" or similar additional text will be removed. For example: `v1.2.6-rc1 -> v1.2.6`. | |||
| ## zip-file.py | |||
| Specialized ZIP command for release management. Special features: | |||
| 1. Uses Python ZIP libaries, so the `zip` command does not need to be installed. | |||
| 1. Can only zip one file. | |||
| 1. Optionally gets file version, Go OS and architecture. | |||
| 1. By default all inputs and output is formatted exactly how CircleCI needs it. | |||
| By default, the command will try to ZIP the file at `build/tendermint_${GOOS}_${GOARCH}`. | |||
| This can be changed with the `--file` input parameter. | |||
| By default, the command will output the ZIP file to `build/tendermint_${CIRCLE_TAG}_${GOOS}_${GOARCH}.zip`. | |||
| This can be changed with the `--destination` (folder), `--version`, `--goos` and `--goarch` input parameters respectively. | |||
| ## sha-files.py | |||
| Specialized `shasum` command for release management. Special features: | |||
| 1. Reads all ZIP files in the given folder. | |||
| 1. By default all inputs and output is formatted exactly how CircleCI needs it. | |||
| By default, the command will look up all ZIP files in the `build/` folder. | |||
| By default, the command will output results into the `build/SHA256SUMS` file. | |||
| # GitHub management | |||
| Uploading build results to GitHub requires at least these steps: | |||
| 1. Create a new release on GitHub with content | |||
| 2. Upload all binaries to the release | |||
| 3. Publish the release | |||
| The below scripts help with these steps. | |||
| ## github-draft.py | |||
| Creates a GitHub release and fills the content with the CHANGELOG.md link. The version number can be changed by the `--version` parameter. | |||
| By default, the command will use the tendermint/tendermint organization/repo, which can be changed using the `--org` and `--repo` parameters. | |||
| By default, the command will get the version number from the `${CIRCLE_TAG}` variable. | |||
| Returns the GitHub release ID. | |||
| ## github-upload.py | |||
| Upload a file to a GitHub release. The release is defined by the mandatory `--id` (release ID) input parameter. | |||
| By default, the command will upload the file `/tmp/workspace/tendermint_${CIRCLE_TAG}_${GOOS}_${GOARCH}.zip`. This can be changed by the `--file` input parameter. | |||
| ## github-publish.py | |||
| Publish a GitHub release. The release is defined by the mandatory `--id` (release ID) input parameter. | |||
+ 37
- 0
scripts/release_management/bump-semver.py
View File
| @ -0,0 +1,37 @@ | |||
| #!/usr/bin/env python | |||
| # Bump the release number of a semantic version number and print it. --version is required. | |||
| # Version is | |||
| # - vA.B.C, in which case vA.B.C+1 will be returned | |||
| # - vA.B.C-devorwhatnot in which case vA.B.C will be returned | |||
| # - vA.B in which case vA.B.0 will be returned | |||
| import re | |||
| import argparse | |||
| def semver(ver): | |||
| if re.match('v[0-9]+\.[0-9]+',ver) is None: | |||
| ver="v0.0" | |||
| #raise argparse.ArgumentTypeError('--version must be a semantic version number with major, minor and patch numbers') | |||
| return ver | |||
| if __name__ == "__main__": | |||
| parser = argparse.ArgumentParser() | |||
| parser.add_argument("--version", help="Version number to bump, e.g.: v1.0.0", required=True, type=semver) | |||
| args = parser.parse_args() | |||
| found = re.match('(v[0-9]+\.[0-9]+)(\.(.+))?', args.version) | |||
| majorminorprefix = found.group(1) | |||
| patch = found.group(3) | |||
| if patch is None: | |||
| patch = "0-new" | |||
| if re.match('[0-9]+$',patch) is None: | |||
| patchfound = re.match('([0-9]+)',patch) | |||
| patch = int(patchfound.group(1)) | |||
| else: | |||
| patch = int(patch) + 1 | |||
| print("{0}.{1}".format(majorminorprefix, patch)) | |||
+ 60
- 0
scripts/release_management/github-draft.py
View File
| @ -0,0 +1,60 @@ | |||
| #!/usr/bin/env python | |||
| # Create a draft release on GitHub. By default in the tendermint/tendermint repo. | |||
| # Optimized for CircleCI | |||
| import argparse | |||
| import httplib | |||
| import json | |||
| import os | |||
| from base64 import b64encode | |||
| def request(org, repo, data): | |||
| user_and_pass = b64encode(b"{0}:{1}".format(os.environ['GITHUB_USERNAME'], os.environ['GITHUB_TOKEN'])).decode("ascii") | |||
| headers = { | |||
| 'User-Agent': 'tenderbot', | |||
| 'Accept': 'application/vnd.github.v3+json', | |||
| 'Authorization': 'Basic %s' % user_and_pass | |||
| } | |||
| conn = httplib.HTTPSConnection('api.github.com', timeout=5) | |||
| conn.request('POST', '/repos/{0}/{1}/releases'.format(org,repo), data, headers) | |||
| response = conn.getresponse() | |||
| if response.status < 200 or response.status > 299: | |||
| print("{0}: {1}".format(response.status, response.reason)) | |||
| conn.close() | |||
| raise IOError(response.reason) | |||
| responsedata = response.read() | |||
| conn.close() | |||
| return json.loads(responsedata) | |||
| def create_draft(org,repo,version): | |||
| draft = { | |||
| 'tag_name': version, | |||
| 'target_commitish': 'master', | |||
| 'name': '{0} (WARNING: ALPHA SOFTWARE)'.format(version), | |||
| 'body': '<a href=https://github.com/{0}/{1}/blob/master/CHANGELOG.md#{2}>https://github.com/{0}/{1}/blob/master/CHANGELOG.md#{2}</a>'.format(org,repo,version.replace('v','').replace('.','')), | |||
| 'draft': True, | |||
| 'prerelease': False | |||
| } | |||
| data=json.dumps(draft) | |||
| return request(org, repo, data) | |||
| if __name__ == "__main__": | |||
| parser = argparse.ArgumentParser() | |||
| parser.add_argument("--org", default="tendermint", help="GitHub organization") | |||
| parser.add_argument("--repo", default="tendermint", help="GitHub repository") | |||
| parser.add_argument("--version", default=os.environ.get('CIRCLE_TAG'), help="Version number for binary, e.g.: v1.0.0") | |||
| args = parser.parse_args() | |||
| if not os.environ.has_key('GITHUB_USERNAME'): | |||
| raise parser.error('environment variable GITHUB_USERNAME is required') | |||
| if not os.environ.has_key('GITHUB_TOKEN'): | |||
| raise parser.error('environment variable GITHUB_TOKEN is required') | |||
| release = create_draft(args.org,args.repo,args.version) | |||
| print(release["id"]) | |||
+ 52
- 0
scripts/release_management/github-openpr.py
View File
| @ -0,0 +1,52 @@ | |||
| #!/usr/bin/env python | |||
| # Open a PR against the develop branch. --branch required. | |||
| # Optimized for CircleCI | |||
| import json | |||
| import os | |||
| import argparse | |||
| import httplib | |||
| from base64 import b64encode | |||
| def request(org, repo, data): | |||
| user_and_pass = b64encode(b"{0}:{1}".format(os.environ['GITHUB_USERNAME'], os.environ['GITHUB_TOKEN'])).decode("ascii") | |||
| headers = { | |||
| 'User-Agent': 'tenderbot', | |||
| 'Accept': 'application/vnd.github.v3+json', | |||
| 'Authorization': 'Basic %s' % user_and_pass | |||
| } | |||
| conn = httplib.HTTPSConnection('api.github.com', timeout=5) | |||
| conn.request('POST', '/repos/{0}/{1}/pulls'.format(org,repo), data, headers) | |||
| response = conn.getresponse() | |||
| if response.status < 200 or response.status > 299: | |||
| print(response) | |||
| conn.close() | |||
| raise IOError(response.reason) | |||
| responsedata = response.read() | |||
| conn.close() | |||
| return json.loads(responsedata) | |||
| if __name__ == "__main__": | |||
| parser = argparse.ArgumentParser() | |||
| parser.add_argument("--org", default="tendermint", help="GitHub organization. Defaults to tendermint.") | |||
| parser.add_argument("--repo", default="tendermint", help="GitHub repository. Defaults to tendermint.") | |||
| parser.add_argument("--head", help="The name of the branch where your changes are implemented.", required=True) | |||
| parser.add_argument("--base", help="The name of the branch you want the changes pulled into.", required=True) | |||
| parser.add_argument("--title", default="Security release {0}".format(os.environ.get('CIRCLE_TAG')), help="The title of the pull request.") | |||
| args = parser.parse_args() | |||
| if not os.environ.has_key('GITHUB_USERNAME'): | |||
| raise parser.error('GITHUB_USERNAME not set.') | |||
| if not os.environ.has_key('GITHUB_TOKEN'): | |||
| raise parser.error('GITHUB_TOKEN not set.') | |||
| if os.environ.get('CIRCLE_TAG') is None: | |||
| raise parser.error('CIRCLE_TAG not set.') | |||
| result = request(args.org, args.repo, data=json.dumps({'title':"{0}".format(args.title),'head':"{0}".format(args.head),'base':"{0}".format(args.base),'body':"<Please fill in details.>"})) | |||
| print(result['html_url']) | |||
+ 28
- 0
scripts/release_management/github-public-newbranch.bash
View File
| @ -0,0 +1,28 @@ | |||
| #!/bin/sh | |||
| # github-public-newbranch.bash - create public branch from the security repository | |||
| set -euo pipefail | |||
| # Create new branch | |||
| BRANCH="${CIRCLE_TAG:-v0.0.0}-security-`date -u +%Y%m%d%H%M%S`" | |||
| # Check if the patch release exist already as a branch | |||
| if [ -n "`git branch | grep '${BRANCH}'`" ]; then | |||
| echo "WARNING: Branch ${BRANCH} already exists." | |||
| else | |||
| echo "Creating branch ${BRANCH}." | |||
| git branch "${BRANCH}" | |||
| fi | |||
| # ... and check it out | |||
| git checkout "${BRANCH}" | |||
| # Add entry to public repository | |||
| git remote add tendermint-origin git@github.com:tendermint/tendermint.git | |||
| # Push branch and tag to public repository | |||
| git push tendermint-origin | |||
| git push tendermint-origin --tags | |||
| # Create a PR from the public branch to the assumed release branch in public (release branch has to exist) | |||
| python -u scripts/release_management/github-openpr.py --head "${BRANCH}" --base "${BRANCH:%.*}" | |||
+ 53
- 0
scripts/release_management/github-publish.py
View File
| @ -0,0 +1,53 @@ | |||
| #!/usr/bin/env python | |||
| # Publish an existing GitHub draft release. --id required. | |||
| # Optimized for CircleCI | |||
| import json | |||
| import os | |||
| import argparse | |||
| import httplib | |||
| from base64 import b64encode | |||
| def request(org, repo, id, data): | |||
| user_and_pass = b64encode(b"{0}:{1}".format(os.environ['GITHUB_USERNAME'], os.environ['GITHUB_TOKEN'])).decode("ascii") | |||
| headers = { | |||
| 'User-Agent': 'tenderbot', | |||
| 'Accept': 'application/vnd.github.v3+json', | |||
| 'Authorization': 'Basic %s' % user_and_pass | |||
| } | |||
| conn = httplib.HTTPSConnection('api.github.com', timeout=5) | |||
| conn.request('POST', '/repos/{0}/{1}/releases/{2}'.format(org,repo,id), data, headers) | |||
| response = conn.getresponse() | |||
| if response.status < 200 or response.status > 299: | |||
| print(response) | |||
| conn.close() | |||
| raise IOError(response.reason) | |||
| responsedata = response.read() | |||
| conn.close() | |||
| return json.loads(responsedata) | |||
| if __name__ == "__main__": | |||
| parser = argparse.ArgumentParser() | |||
| parser.add_argument("--org", default="tendermint", help="GitHub organization") | |||
| parser.add_argument("--repo", default="tendermint", help="GitHub repository") | |||
| parser.add_argument("--id", help="GitHub release ID", required=True, type=int) | |||
| parser.add_argument("--version", default=os.environ.get('CIRCLE_TAG'), help="Version number for the release, e.g.: v1.0.0") | |||
| args = parser.parse_args() | |||
| if not os.environ.has_key('GITHUB_USERNAME'): | |||
| raise parser.error('GITHUB_USERNAME not set.') | |||
| if not os.environ.has_key('GITHUB_TOKEN'): | |||
| raise parser.error('GITHUB_TOKEN not set.') | |||
| try: | |||
| result = request(args.org, args.repo, args.id, data=json.dumps({'draft':False,'tag_name':"{0}".format(args.version)})) | |||
| except IOError as e: | |||
| print(e) | |||
| result = request(args.org, args.repo, args.id, data=json.dumps({'draft':False,'tag_name':"{0}-autorelease".format(args.version)})) | |||
| print(result['name']) | |||
+ 68
- 0
scripts/release_management/github-upload.py
View File
| @ -0,0 +1,68 @@ | |||
| #!/usr/bin/env python | |||
| # Upload a file to a GitHub draft release. --id and --file are required. | |||
| # Optimized for CircleCI | |||
| import json | |||
| import os | |||
| import re | |||
| import argparse | |||
| import mimetypes | |||
| import httplib | |||
| from base64 import b64encode | |||
| def request(baseurl, path, mimetype, mimeencoding, data): | |||
| user_and_pass = b64encode(b"{0}:{1}".format(os.environ['GITHUB_USERNAME'], os.environ['GITHUB_TOKEN'])).decode("ascii") | |||
| headers = { | |||
| 'User-Agent': 'tenderbot', | |||
| 'Accept': 'application/vnd.github.v3.raw+json', | |||
| 'Authorization': 'Basic %s' % user_and_pass, | |||
| 'Content-Type': mimetype, | |||
| 'Content-Encoding': mimeencoding | |||
| } | |||
| conn = httplib.HTTPSConnection(baseurl, timeout=5) | |||
| conn.request('POST', path, data, headers) | |||
| response = conn.getresponse() | |||
| if response.status < 200 or response.status > 299: | |||
| print(response) | |||
| conn.close() | |||
| raise IOError(response.reason) | |||
| responsedata = response.read() | |||
| conn.close() | |||
| return json.loads(responsedata) | |||
| if __name__ == "__main__": | |||
| parser = argparse.ArgumentParser() | |||
| parser.add_argument("--id", help="GitHub release ID", required=True, type=int) | |||
| parser.add_argument("--file", default="/tmp/workspace/tendermint_{0}_{1}_{2}.zip".format(os.environ.get('CIRCLE_TAG'),os.environ.get('GOOS'),os.environ.get('GOARCH')), help="File to upload") | |||
| parser.add_argument("--return-id-only", help="Return only the release ID after upload to GitHub.", action='store_true') | |||
| args = parser.parse_args() | |||
| if not os.environ.has_key('GITHUB_USERNAME'): | |||
| raise parser.error('GITHUB_USERNAME not set.') | |||
| if not os.environ.has_key('GITHUB_TOKEN'): | |||
| raise parser.error('GITHUB_TOKEN not set.') | |||
| mimetypes.init() | |||
| filename = os.path.basename(args.file) | |||
| mimetype,mimeencoding = mimetypes.guess_type(filename, strict=False) | |||
| if mimetype is None: | |||
| mimetype = 'application/zip' | |||
| if mimeencoding is None: | |||
| mimeencoding = 'utf8' | |||
| with open(args.file,'rb') as f: | |||
| asset = f.read() | |||
| result = request('uploads.github.com', '/repos/tendermint/tendermint/releases/{0}/assets?name={1}'.format(args.id, filename), mimetype, mimeencoding, asset) | |||
| if args.return_id_only: | |||
| print(result['id']) | |||
| else: | |||
| print(result['browser_download_url']) | |||
+ 35
- 0
scripts/release_management/sha-files.py
View File
| @ -0,0 +1,35 @@ | |||
| #!/usr/bin/env python | |||
| # Create SHA256 summaries from all ZIP files in a folder | |||
| # Optimized for CircleCI | |||
| import re | |||
| import os | |||
| import argparse | |||
| import zipfile | |||
| import hashlib | |||
| BLOCKSIZE = 65536 | |||
| if __name__ == "__main__": | |||
| parser = argparse.ArgumentParser() | |||
| parser.add_argument("--folder", default="/tmp/workspace", help="Folder to look for, for ZIP files") | |||
| parser.add_argument("--shafile", default="/tmp/workspace/SHA256SUMS", help="SHA256 summaries File") | |||
| args = parser.parse_args() | |||
| for filename in os.listdir(args.folder): | |||
| if re.search('\.zip$',filename) is None: | |||
| continue | |||
| if not os.path.isfile(os.path.join(args.folder, filename)): | |||
| continue | |||
| with open(args.shafile,'a+') as shafile: | |||
| hasher = hashlib.sha256() | |||
| with open(os.path.join(args.folder, filename),'r') as f: | |||
| buf = f.read(BLOCKSIZE) | |||
| while len(buf) > 0: | |||
| hasher.update(buf) | |||
| buf = f.read(BLOCKSIZE) | |||
| shafile.write("{0} {1}\n".format(hasher.hexdigest(),filename)) | |||
+ 44
- 0
scripts/release_management/zip-file.py
View File
| @ -0,0 +1,44 @@ | |||
| #!/usr/bin/env python | |||
| # ZIP one file as "tendermint" into a ZIP like tendermint_VERSION_OS_ARCH.zip | |||
| # Use environment variables CIRCLE_TAG, GOOS and GOARCH for easy input parameters. | |||
| # Optimized for CircleCI | |||
| import os | |||
| import argparse | |||
| import zipfile | |||
| import hashlib | |||
| BLOCKSIZE = 65536 | |||
| def zip_asset(file,destination,arcname,version,goos,goarch): | |||
| filename = os.path.basename(file) | |||
| output = "{0}/{1}_{2}_{3}_{4}.zip".format(destination,arcname,version,goos,goarch) | |||
| with zipfile.ZipFile(output,'w') as f: | |||
| f.write(filename=file,arcname=arcname) | |||
| f.comment=filename | |||
| return output | |||
| if __name__ == "__main__": | |||
| parser = argparse.ArgumentParser() | |||
| parser.add_argument("--file", default="build/tendermint_{0}_{1}".format(os.environ.get('GOOS'),os.environ.get('GOARCH')), help="File to zip") | |||
| parser.add_argument("--destination", default="build", help="Destination folder for files") | |||
| parser.add_argument("--version", default=os.environ.get('CIRCLE_TAG'), help="Version number for binary, e.g.: v1.0.0") | |||
| parser.add_argument("--goos", default=os.environ.get('GOOS'), help="GOOS parameter") | |||
| parser.add_argument("--goarch", default=os.environ.get('GOARCH'), help="GOARCH parameter") | |||
| args = parser.parse_args() | |||
| if args.version is None: | |||
| raise parser.error("argument --version is required") | |||
| if args.goos is None: | |||
| raise parser.error("argument --goos is required") | |||
| if args.goarch is None: | |||
| raise parser.error("argument --goarch is required") | |||
| file = zip_asset(args.file,args.destination,"tendermint",args.version,args.goos,args.goarch) | |||
| print(file) | |||