Compare commits

...
Sign in to create a new pull request.

27 commits

Author SHA1 Message Date
Michael Telatynski
b7052a5c5f
Merge branch 'develop' of github.com:vector-im/element-web into t3chguy/ci3
 Conflicts:
	.editorconfig
	.github/workflows/preview_changelog.yaml
	.github/workflows/static_analysis.yaml
	.github/workflows/triage-labelled.yml
	.github/workflows/triage-priority-bugs.yml
	.gitignore
	scripts/layered.sh
2022-06-06 11:47:18 +01:00
Michael Telatynski
b18c944b42 strings 2022-05-03 09:22:18 +01:00
Michael Telatynski
0bbad38929 Must it really be single quotes only? 2022-05-03 09:07:47 +01:00
Michael Telatynski
a431363768 Add CI to prevent i18n non-EN changes 2022-05-03 09:04:30 +01:00
Michael Telatynski
2d7bada5c4 Tweak 2022-04-27 23:09:35 +01:00
Michael Telatynski
0e22503860 Reusable workflows don't help us here 2022-04-27 23:07:06 +01:00
Michael Telatynski
a340358551 🤦 2022-04-27 23:01:00 +01:00
Michael Telatynski
e53f2695a4 Fix reusable workflow call 2022-04-27 22:56:46 +01:00
Michael Telatynski
0a239a30fa Update fetchdep to also take into account forks 2022-04-27 22:50:28 +01:00
Michael Telatynski
dab1488ffe Fix PR_NUMBER env variable 2022-04-27 22:43:51 +01:00
Michael Telatynski
61f8ec5e10 Test 2022-04-27 22:41:12 +01:00
Michael Telatynski
7a01c6c61b Update deploy_develop.yaml 2022-04-27 22:31:03 +01:00
Michael Telatynski
0ab7cd05c7 Consolidate the two secrets environments 2022-04-27 22:27:00 +01:00
Michael Telatynski
fa28d2400b Fix layered.sh 2022-04-27 22:11:23 +01:00
Michael Telatynski
7cc52e68d1 Consolidate workflows by means of reuse 2022-04-27 17:23:06 +01:00
Michael Telatynski
8bdd965122 Use fetchdep everywhere 2022-04-27 17:12:34 +01:00
Michael Telatynski
418de7998a Switch up develop redeploy script to work off Github Actions 2022-04-27 16:48:02 +01:00
Michael Telatynski
008889d2a8 Split back out into two workflows for artifact access 2022-04-27 13:29:14 +01:00
Michael Telatynski
b577d0f2f2 Fix usage of wrong action 2022-04-27 13:15:16 +01:00
Michael Telatynski
98733057a7 Attempt to use deployments more properly 2022-04-27 13:10:39 +01:00
Michael Telatynski
0cd6e02c99 Iterate 2022-04-27 12:41:46 +01:00
Michael Telatynski
449a0e64d9 Test 2022-04-27 12:38:09 +01:00
Michael Telatynski
89b3e4aaab Lets try this 2022-04-27 08:38:12 +01:00
Michael Telatynski
fe8c583e09 Attempt all of the CI 2022-04-27 07:53:30 +01:00
Michael Telatynski
5adf38c87f temporarily increase trigger 2022-04-26 18:19:19 +01:00
Michael Telatynski
59d7265e69 Test deployment hook 2022-04-26 18:17:47 +01:00
Michael Telatynski
446b510b82 Delint workflow scripts 2022-04-26 18:17:31 +01:00
9 changed files with 342 additions and 297 deletions

33
.github/workflows/build.yaml vendored Normal file
View file

@ -0,0 +1,33 @@
name: Build
on:
pull_request: { }
push:
branches: [ master ]
# develop pushes and repository_dispatch handled in build_develop.yaml
env:
# These must be set for fetchdep.sh to get the right branch
REPOSITORY: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
jobs:
build:
name: "Build"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-node@v3
with:
cache: 'yarn'
- name: Install Dependencies
run: "./scripts/layered.sh"
- name: Build & Package
run: "./scripts/ci_package.sh"
- name: Upload Artifact
uses: actions/upload-artifact@v2
with:
name: previewbuild
path: dist/*.tar.gz
retention-days: 28

38
.github/workflows/build_develop.yaml vendored Normal file
View file

@ -0,0 +1,38 @@
# Separate to the main build workflow for access to develop
# environment secrets, largely similar to build.yaml.
name: Build develop
on:
push:
branches: [ develop ]
repository_dispatch:
types: [ element-web-notify ]
jobs:
build:
name: "Build & Upload source maps to Sentry"
runs-on: ubuntu-latest
environment: develop
steps:
- uses: actions/checkout@v2
- uses: actions/setup-node@v3
with:
cache: 'yarn'
- name: Install Dependencies
run: "./scripts/layered.sh"
- name: Build, Package & Upload sourcemaps
run: "./scripts/ci_package.sh"
env:
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
SENTRY_URL: ${{ secrets.SENTRY_URL }}
SENTRY_ORG: sentry
SENTRY_PROJECT: riot-web
- name: Upload Artifact
uses: actions/upload-artifact@v2
with:
name: previewbuild
path: dist/*.tar.gz
retention-days: 1

64
.github/workflows/deploy_develop.yaml vendored Normal file
View file

@ -0,0 +1,64 @@
# Triggers after the Build has finished,
# because artifacts are not externally available
# until the end of their workflow.
name: Deploy develop.element.io
concurrency: deploy_develop
on:
workflow_run:
workflows: [ "Build develop" ]
types:
- completed
jobs:
deploy:
runs-on: ubuntu-latest
environment: develop
if: github.event.workflow_run.conclusion == 'success'
steps:
- name: Find Artifact ID
uses: actions/github-script@v3.1.0
id: find_artifact
with:
result-encoding: string
script: |
const artifacts = await github.actions.listWorkflowRunArtifacts({
owner: context.repo.owner,
repo: context.repo.repo,
run_id: ${{ github.event.workflow_run.id }},
});
const matchArtifact = artifacts.data.artifacts.filter((artifact) => {
return artifact.name == "previewbuild"
})[0];
const download = await github.actions.downloadArtifact({
owner: context.repo.owner,
repo: context.repo.repo,
artifact_id: matchArtifact.id,
archive_format: 'zip',
});
return download.url;
- name: Create Deployment
uses: bobheadxi/deployments@v1
id: deployment
with:
step: start
token: ${{ secrets.GITHUB_TOKEN }}
env: Develop
ref: ${{ github.head_ref }}
- name: Notify the redeploy script
uses: distributhor/workflow-webhook@v2
env:
webhook_url: ${{ secrets.DEVELOP_DEPLOY_WEBHOOK_URL }}
webhook_secret: ${{ secrets.DEVELOP_DEPLOY_WEBHOOK_SECRET }}
data: '{"url": "${{ steps.find_artifact.outputs.result }}"}'
- name: Update deployment status
uses: bobheadxi/deployments@v1
if: always()
with:
step: finish
token: ${{ secrets.GITHUB_TOKEN }}
status: ${{ job.status }}
env: ${{ steps.deployment.outputs.env }}
deployment_id: ${{ steps.deployment.outputs.deployment_id }}
env_url: https://develop.element.io

27
.github/workflows/test.yaml vendored Normal file
View file

@ -0,0 +1,27 @@
name: Test
on:
pull_request: { }
push:
branches: [ master, develop ]
repository_dispatch:
types: [ element-web-notify ]
env:
# These must be set for fetchdep.sh to get the right branch
REPOSITORY: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
jobs:
test:
name: "Test"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-node@v3
with:
cache: 'yarn'
- name: Install Dependencies
run: "./scripts/layered.sh"
- name: Run Tests
run: "yarn test"

2
.gitignore vendored
View file

@ -24,3 +24,5 @@ electron/pub
.vscode/
.env
/coverage
/scripts/extracted/
/scripts/latest

View file

@ -1,7 +1,7 @@
#!/usr/bin/env python
#
# download and unpack a element-web tarball.
#
# Allows `bundles` to be extracted to a common directory, and a link to
# config.json to be added.
@ -23,9 +23,11 @@ except ImportError:
# python2
from urllib import urlretrieve
class DeployException(Exception):
pass
def create_relative_symlink(linkname, target):
relpath = os.path.relpath(target, os.path.dirname(linkname))
print ("Symlink %s -> %s" % (linkname, relpath))
@ -57,10 +59,11 @@ def move_bundles(source, dest):
else:
renames[os.path.join(source, f)] = dst
for (src, dst) in renames.iteritems():
for (src, dst) in renames.items():
print ("Move %s -> %s" % (src, dst))
os.rename(src, dst)
class Deployer:
def __init__(self):
self.packages_path = "."
@ -100,7 +103,7 @@ class Deployer:
print ("Extracted into: %s" % extracted_dir)
if self.symlink_paths:
for link_path, file_path in self.symlink_paths.iteritems():
for link_path, file_path in self.symlink_paths.items():
create_relative_symlink(
target=file_path,
linkname=os.path.join(extracted_dir, link_path)
@ -139,6 +142,7 @@ class Deployer:
print ("Done")
return local_filename
if __name__ == "__main__":
parser = argparse.ArgumentParser("Deploy a Riot build on a web server.")
parser.add_argument(

View file

@ -1,99 +0,0 @@
#!/bin/bash
# Fetches the js-sdk and matrix-react-sdk dependencies for development
# or testing purposes
# If there exists a branch of that dependency with the same name as
# the branch the current checkout is on, use that branch. Otherwise,
# use develop.
set -ex
GIT_CLONE_ARGS=("$@")
[ -z "$defbranch" ] && defbranch="develop"
# clone a specific branch of a github repo
function clone() {
org=$1
repo=$2
branch=$3
# Chop 'origin' off the start as jenkins ends up using
# branches on the origin, but this doesn't work if we
# specify the branch when cloning.
branch=${branch#origin/}
if [ -n "$branch" ]
then
echo "Trying to use $org/$repo#$branch"
# Disable auth prompts: https://serverfault.com/a/665959
GIT_TERMINAL_PROMPT=0 git clone https://github.com/$org/$repo.git $repo --branch $branch \
"${GIT_CLONE_ARGS[@]}"
return $?
fi
return 1
}
function dodep() {
deforg=$1
defrepo=$2
rm -rf $defrepo
# Try the PR author's branch in case it exists on the deps as well.
# Try the target branch of the push or PR.
# Use the default branch as the last resort.
if [[ "$BUILDKITE" == true ]]; then
# If BUILDKITE_BRANCH is set, it will contain either:
# * "branch" when the author's branch and target branch are in the same repo
# * "author:branch" when the author's branch is in their fork
# We can split on `:` into an array to check.
BUILDKITE_BRANCH_ARRAY=(${BUILDKITE_BRANCH//:/ })
if [[ "${#BUILDKITE_BRANCH_ARRAY[@]}" == "2" ]]; then
prAuthor=${BUILDKITE_BRANCH_ARRAY[0]}
prBranch=${BUILDKITE_BRANCH_ARRAY[1]}
else
prAuthor=$deforg
prBranch=$BUILDKITE_BRANCH
fi
clone $prAuthor $defrepo $prBranch ||
clone $deforg $defrepo $BUILDKITE_PULL_REQUEST_BASE_BRANCH ||
clone $deforg $defrepo $defbranch ||
return $?
else
clone $deforg $defrepo $ghprbSourceBranch ||
clone $deforg $defrepo $GIT_BRANCH ||
clone $deforg $defrepo `git rev-parse --abbrev-ref HEAD` ||
clone $deforg $defrepo $defbranch ||
return $?
fi
echo "$defrepo set to branch "`git -C "$defrepo" rev-parse --abbrev-ref HEAD`
}
##############################
echo 'Setting up matrix-js-sdk'
dodep matrix-org matrix-js-sdk
pushd matrix-js-sdk
yarn link
yarn install --pure-lockfile
popd
yarn link matrix-js-sdk
##############################
echo 'Setting up matrix-react-sdk'
dodep matrix-org matrix-react-sdk
pushd matrix-react-sdk
yarn link
yarn link matrix-js-sdk
yarn install --pure-lockfile
popd
yarn link matrix-react-sdk
##############################

78
scripts/fetchdep.sh Executable file
View file

@ -0,0 +1,78 @@
#!/bin/bash
set -x
deforg="$1"
defrepo="$2"
defbranch="$3"
[ -z "$defbranch" ] && defbranch="develop"
rm -r "$defrepo" || true
# A function that clones a branch of a repo based on the org, repo and branch
clone() {
org=$1
repo=$2
branch=$3
if [ -n "$branch" ]
then
echo "Trying to use $org/$repo#$branch"
# Disable auth prompts: https://serverfault.com/a/665959
GIT_TERMINAL_PROMPT=0 git clone https://github.com/$org/$repo.git $repo --branch "$branch" --depth 1 && exit 0
fi
}
# A function that gets info about a PR from the GitHub API based on its number
getPRInfo() {
number=$1
if [ -n "$number" ]; then
echo "Getting info about a PR with number $number"
apiEndpoint="https://api.github.com/repos/${REPOSITORY:-"vector-im/element-web"}/pulls/"
apiEndpoint+=$number
head=$(curl $apiEndpoint | jq -r '.head.label')
fi
}
# Some CIs don't give us enough info, so we just get the PR number and ask the
# GH API for more info - "fork:branch". Some give us this directly.
if [ -n "$BUILDKITE_BRANCH" ]; then
# BuildKite
head=$BUILDKITE_BRANCH
elif [ -n "$PR_NUMBER" ]; then
# GitHub
getPRInfo $PR_NUMBER
elif [ -n "$REVIEW_ID" ]; then
# Netlify
getPRInfo $REVIEW_ID
fi
# for forks, $head will be in the format "fork:branch", so we split it by ":"
# into an array. On non-forks, this has the effect of splitting into a single
# element array given ":" shouldn't appear in the head - it'll just be the
# branch name. Based on the results, we clone.
BRANCH_ARRAY=(${head//:/ })
TRY_ORG=$deforg
TRY_BRANCH=${BRANCH_ARRAY[0]}
if [[ "$head" == *":"* ]]; then
# ... but only match that fork if it's a real fork
if [ "${BRANCH_ARRAY[0]}" != "matrix-org" ]; then
TRY_ORG=${BRANCH_ARRAY[0]}
fi
TRY_BRANCH=${BRANCH_ARRAY[1]}
fi
clone ${TRY_ORG} $defrepo ${TRY_BRANCH}
# Try the target branch of the push or PR.
if [ -n "$GITHUB_BASE_REF" ]; then
clone $deforg $defrepo $GITHUB_BASE_REF
elif [ -n "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" ]; then
clone $deforg $defrepo $BUILDKITE_PULL_REQUEST_BASE_BRANCH
fi
# Try HEAD which is the branch name in Netlify (not BRANCH which is pull/xxxx/head for PR builds)
clone $deforg $defrepo $HEAD
# Use the default branch as the last resort.
clone $deforg $defrepo $defbranch

View file

@ -1,45 +1,40 @@
#!/usr/bin/env python
#
# auto-deploy script for https://develop.element.io
#
# Listens for buildkite webhook pokes (https://buildkite.com/docs/apis/webhooks)
# When it gets one, downloads the artifact from buildkite
# and deploys it as the new version.
#
# Listens for Github Action webhook pokes (https://github.com/marketplace/actions/workflow-webhook-action)
# When it gets one: downloads the artifact from github actions and deploys it as the new version.
# Requires the following python packages:
#
# - requests
# - flask
#
# - python-github-webhook
from __future__ import print_function
import json, requests, tarfile, argparse, os, errno
import argparse
import os
import errno
import time
import traceback
from urlparse import urljoin
import glob
import re
import shutil
import threading
from Queue import Queue
from flask import Flask, jsonify, request, abort
import glob
from io import BytesIO
from urllib.request import urlopen
from zipfile import ZipFile
from github_webhook import Webhook
from flask import Flask, abort
from deploy import Deployer, DeployException
app = Flask(__name__)
webhook = Webhook(app, endpoint="/")
deployer = None
arg_extract_path = None
arg_symlink = None
arg_webhook_token = None
arg_api_token = None
workQueue = Queue()
def create_symlink(source, linkname):
def create_symlink(source: str, linkname: str):
try:
os.symlink(source, linkname)
except OSError, e:
except OSError as e:
if e.errno == errno.EEXIST:
# atomic modification
os.symlink(source, linkname + ".tmp")
@ -47,118 +42,43 @@ def create_symlink(source, linkname):
else:
raise e
def req_headers():
return {
"Authorization": "Bearer %s" % (arg_api_token,),
}
# Buildkite considers a poke to have failed if it has to wait more than 10s for
# data (any data, not just the initial response) and it normally takes longer than
# that to download an artifact from buildkite. Apparently there is no way in flask
# to finish the response and then keep doing stuff, so instead this has to involve
# threading. Sigh.
def worker_thread():
while True:
toDeploy = workQueue.get()
deploy_buildkite_artifact(*toDeploy)
@app.route("/", methods=["POST"])
def on_receive_buildkite_poke():
got_webhook_token = request.headers.get('X-Buildkite-Token')
if got_webhook_token != arg_webbook_token:
print("Denying request with incorrect webhook token: %s" % (got_webhook_token,))
abort(400, "Incorrect webhook token")
@webhook.hook(event_type="workflow_run")
def on_deployment(payload: dict):
repository = payload.get("repository")
if repository is None:
abort(400, "No 'repository' specified")
return
required_api_prefix = None
if arg_buildkite_org is not None:
required_api_prefix = 'https://api.buildkite.com/v2/organizations/%s' % (arg_buildkite_org,)
incoming_json = request.get_json()
if not incoming_json:
abort(400, "No JSON provided!")
return
print("Incoming JSON: %s" % (incoming_json,))
event = incoming_json.get("event")
if event is None:
abort(400, "No 'event' specified")
workflow = payload.get("workflow")
if repository is None:
abort(400, "No 'workflow' specified")
return
if event == 'ping':
print("Got ping request - responding")
return jsonify({'response': 'pong!'})
if event != 'build.finished':
print("Rejecting '%s' event")
abort(400, "Unrecognised event")
request_id = payload.get("requestID")
if request_id is None:
abort(400, "No 'request_id' specified")
return
build_obj = incoming_json.get("build")
if build_obj is None:
abort(400, "No 'build' object")
if arg_github_org is not None and not repository.startswith(arg_github_org):
print("Denying poke for repository with incorrect prefix: %s" % (repository,))
abort(400, "Invalid repository")
return
build_url = build_obj.get('url')
if build_url is None:
abort(400, "build has no url")
if arg_github_workflow is not None and workflow != arg_github_workflow:
print("Denying poke for incorrect workflow: %s" % (workflow,))
abort(400, "Incorrect workflow")
return
if required_api_prefix is not None and not build_url.startswith(required_api_prefix):
print("Denying poke for build url with incorrect prefix: %s" % (build_url,))
abort(400, "Invalid build url")
artifact_url = payload.get("data", {}).get("url")
if artifact_url is None:
abort(400, "No 'data.url' specified")
return
build_num = build_obj.get('number')
if build_num is None:
abort(400, "build has no number")
return
deploy_artifact(artifact_url, request_id)
pipeline_obj = incoming_json.get("pipeline")
if pipeline_obj is None:
abort(400, "No 'pipeline' object")
return
pipeline_name = pipeline_obj.get('name')
if pipeline_name is None:
abort(400, "pipeline has no name")
return
artifacts_url = build_url + "/artifacts"
artifacts_resp = requests.get(artifacts_url, headers=req_headers())
artifacts_resp.raise_for_status()
artifacts_array = artifacts_resp.json()
artifact_to_deploy = None
for artifact in artifacts_array:
if re.match(r"dist/.*.tar.gz", artifact['path']):
artifact_to_deploy = artifact
if artifact_to_deploy is None:
print("No suitable artifacts found")
return jsonify({})
# double paranoia check: make sure the artifact is on the right org too
if required_api_prefix is not None and not artifact_to_deploy['url'].startswith(required_api_prefix):
print("Denying poke for build url with incorrect prefix: %s" % (artifact_to_deploy['url'],))
abort(400, "Refusing to deploy artifact from URL %s", artifact_to_deploy['url'])
return
# there's no point building up a queue of things to deploy, so if there are any pending jobs,
# remove them
while not workQueue.empty():
try:
workQueue.get(False)
except:
pass
workQueue.put([artifact_to_deploy, pipeline_name, build_num])
return jsonify({})
def deploy_buildkite_artifact(artifact, pipeline_name, build_num):
artifact_response = requests.get(artifact['url'], headers=req_headers())
artifact_response.raise_for_status()
artifact_obj = artifact_response.json()
def deploy_artifact(artifact_url: str, request_id: str):
# we extract into a directory based on the build number. This avoids the
# problem of multiple builds building the same git version and thus having
# the same tarball name. That would lead to two potential problems:
@ -166,58 +86,42 @@ def deploy_buildkite_artifact(artifact, pipeline_name, build_num):
# a good deploy with a bad one
# (b) we'll be overwriting the live deployment, which means people might
# see half-written files.
build_dir = os.path.join(arg_extract_path, "%s-#%s" % (pipeline_name, build_num))
try:
extracted_dir = deploy_tarball(artifact_obj, build_dir)
except DeployException as e:
traceback.print_exc()
abort(400, e.message)
build_dir = os.path.join(arg_extract_path, "gha-%s" % (request_id,))
create_symlink(source=extracted_dir, linkname=arg_symlink)
def deploy_tarball(artifact, build_dir):
"""Download a tarball from jenkins and unpack it
Returns:
(str) the path to the unpacked deployment
"""
if os.path.exists(build_dir):
raise DeployException(
"Not deploying. We have previously deployed this build."
)
# We have already deployed this, nop
return
os.mkdir(build_dir)
print("Fetching artifact %s -> %s..." % (artifact['download_url'], artifact['filename']))
# Download the tarball here as buildkite needs auth to do this
# we don't pgp-sign buildkite artifacts, relying on HTTPS and buildkite
# not being evil. If that's not good enough for you, don't use develop.element.io.
resp = requests.get(artifact['download_url'], stream=True, headers=req_headers())
resp.raise_for_status()
with open(artifact['filename'], 'wb') as ofp:
shutil.copyfileobj(resp.raw, ofp)
print("...download complete. Deploying...")
# we rely on the fact that flask only serves one request at a time to
# ensure that we do not overwrite a tarball from a concurrent request.
return deployer.deploy(artifact['filename'], build_dir)
try:
with urlopen(artifact_url) as f:
with ZipFile(BytesIO(f.read()), "r") as z:
name = next((x for x in z.namelist() if x.endswith(".tar.gz")))
z.extract(name, build_dir)
extracted_dir = deployer.deploy(os.path.join(build_dir, name), build_dir)
create_symlink(source=extracted_dir, linkname=arg_symlink)
except DeployException as e:
traceback.print_exc()
abort(400, str(e))
finally:
if deployer.should_clean:
os.remove(os.path.join(build_dir, name))
if __name__ == "__main__":
parser = argparse.ArgumentParser("Runs a Vector redeployment server.")
parser = argparse.ArgumentParser("Runs an Element redeployment server.")
parser.add_argument(
"-p", "--port", dest="port", default=4000, type=int, help=(
"The port to listen on for requests from Jenkins."
"The port to listen on for redeployment requests."
)
)
parser.add_argument(
"-e", "--extract", dest="extract", default="./extracted", help=(
"-e", "--extract", dest="extract", default="./extracted", type=str, help=(
"The location to extract .tar.gz files to."
)
)
parser.add_argument(
"-b", "--bundles-dir", dest="bundles_dir", help=(
"-b", "--bundles-dir", dest="bundles_dir", type=str, help=(
"A directory to move the contents of the 'bundles' directory to. A \
symlink to the bundles directory will also be written inside the \
extracted tarball. Example: './bundles'."
@ -229,7 +133,7 @@ if __name__ == "__main__":
)
)
parser.add_argument(
"-s", "--symlink", dest="symlink", default="./latest", help=(
"-s", "--symlink", dest="symlink", default="./latest", type=str, help=(
"Write a symlink to this location pointing to the extracted tarball. \
New builds will keep overwriting this symlink. The symlink will point \
to the /vector directory INSIDE the tarball."
@ -238,71 +142,65 @@ if __name__ == "__main__":
# --include ../../config.json ./localhost.json homepages/*
parser.add_argument(
"--include", nargs='*', default='./config*.json', help=(
"--include", nargs='*', default='./config*.json', type=str, help=(
"Symlink these files into the root of the deployed tarball. \
Useful for config files and home pages. Supports glob syntax. \
(Default: '%(default)s')"
)
)
parser.add_argument(
"--test", dest="tarball_uri", help=(
"Don't start an HTTP listener. Instead download a build from Jenkins \
immediately."
"--test", dest="tarball_uri", type=str, help=(
"Don't start an HTTP listener. Instead download a build from this URL immediately."
),
)
parser.add_argument(
"--webhook-token", dest="webhook_token", help=(
"Only accept pokes with this buildkite token."
), required=True,
)
parser.add_argument(
"--api-token", dest="api_token", help=(
"API access token for buildkite. Require read_artifacts scope."
"--webhook-token", dest="webhook_token", type=str, help=(
"Only accept pokes signed with this github token."
), required=True,
)
# We require a matching webhook token, but because we take everything else
# about what to deploy from the poke body, we can be a little more paranoid
# and only accept builds / artifacts from a specific buildkite org
# and only accept builds / artifacts from a specific github org
parser.add_argument(
"--org", dest="buildkite_org", help=(
"Lock down to this buildkite org"
"--org", dest="github_org", type=str, help=(
"Lock down to this github org"
)
)
# Optional matching workflow name
parser.add_argument(
"--workflow", dest="github_workflow", type=str, help=(
"Lock down to this github workflow"
)
)
args = parser.parse_args()
arg_extract_path = args.extract
arg_symlink = args.symlink
arg_webbook_token = args.webhook_token
arg_api_token = args.api_token
arg_buildkite_org = args.buildkite_org
arg_github_org = args.github_org
arg_github_workflow = args.github_workflow
if not os.path.isdir(arg_extract_path):
os.mkdir(arg_extract_path)
webhook.secret = args.webhook_token
deployer = Deployer()
deployer.bundles_path = args.bundles_dir
deployer.should_clean = args.clean
for include in args.include:
for include in args.include.split(" "):
deployer.symlink_paths.update({ os.path.basename(pth): pth for pth in glob.iglob(include) })
if args.tarball_uri is not None:
build_dir = os.path.join(arg_extract_path, "test-%i" % (time.time()))
deploy_tarball(args.tarball_uri, build_dir)
else:
print(
"Listening on port %s. Extracting to %s%s. Symlinking to %s. Include files: %s" %
(args.port,
arg_extract_path,
" (clean after)" if deployer.should_clean else "",
arg_symlink,
deployer.symlink_paths,
)
print(
"Listening on port %s. Extracting to %s%s. Symlinking to %s. Include files: %s" %
(args.port,
arg_extract_path,
" (clean after)" if deployer.should_clean else "",
arg_symlink,
deployer.symlink_paths,
)
fred = threading.Thread(target=worker_thread)
fred.daemon = True
fred.start()
app.run(port=args.port, debug=False)
)
app.run(port=args.port, debug=False)