Switch up develop redeploy script to work off Github Actions

.github/workflows/build.yaml

@@ -28,12 +28,6 @@ jobs:
       - name: Build & Package
         run: "./scripts/ci_package.sh"
 
-      - name: Upload webpack-stats.json
-        uses: actions/upload-artifact@v2
-        with:
-          path: webpack-stats.json
-          retention-days: 28
-
       - name: Upload Artifact
         uses: actions/upload-artifact@v2
         with:

.github/workflows/deploy_develop.yaml

@@ -1,7 +1,8 @@
 # Triggers after the Build has finished,
 # because artifacts are not externally available
 # until the end of their workflow.
-name: Build
+name: Deploy develop.element.io
+concurrency: deploy_develop
 on:
   workflow_run:
     workflows: [ "Build" ]
@@ -9,8 +10,8 @@ on:
       - completed
 jobs:
   deploy:
-    needs: build
     runs-on: ubuntu-latest
+    environment: deploy_develop
     if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request'
     steps:
       - name: Find Artifact ID
@@ -19,47 +20,45 @@ jobs:
         with:
           result-encoding: string
           script: |
-            var artifacts = await github.actions.listWorkflowRunArtifacts({
+            const artifacts = await github.actions.listWorkflowRunArtifacts({
-               owner: context.repo.owner,
+                owner: context.repo.owner,
-               repo: context.repo.repo,
+                repo: context.repo.repo,
-               run_id: ${{github.run_id }},
+                run_id: ${{ github.event.workflow_run.id }},
             });
-            var matchArtifact = artifacts.data.artifacts.filter((artifact) => {
+            const matchArtifact = artifacts.data.artifacts.filter((artifact) => {
-              return artifact.name == "previewbuild"
+                return artifact.name == "previewbuild"
             })[0];
-            return matchArtifact.id;
+            const download = await github.actions.downloadArtifact({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                artifact_id: matchArtifact.id,
+                archive_format: 'zip',
+            });
+            return download.url;
 
       - name: Create Deployment
-        uses: avakar/create-deployment@v1.0.2
+        uses: bobheadxi/deployments@v1
         id: deployment
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          environment: Develop
+          step: start
-          initial_status: in_progress
+          token: ${{ secrets.GITHUB_TOKEN }}
+          env: Develop
           ref: ${{ github.head_ref }}
-          transient_environment: false
-          auto_merge: false
-          task: deploy
-          payload: '{"artifact_id": "${{ steps.find_artifact.outputs.result }}"}'
 
-      - name: Update deployment status (success)
+      - name: Notify the redeploy script
-        if: success()
+        uses: distributhor/workflow-webhook@v2
-        uses: avakar/set-deployment-status@v1.1.0
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          webhook_url: ${{ secrets.WEBHOOK_URL }}
+          webhook_secret: ${{ secrets.WEBHOOK_SECRET }}
+          data: '{"url": "${{ steps.find_artifact.outputs.result }}"}'
+
+      - name: Update deployment status
+        uses: bobheadxi/deployments@v1
+        if: always()
         with:
-          environment: Develop
+          step: finish
-          environment_url: https://develop.element.io
+          token: ${{ secrets.GITHUB_TOKEN }}
-          state: "success"
+          status: ${{ job.status }}
+          env: ${{ steps.deployment.outputs.env }}
           deployment_id: ${{ steps.deployment.outputs.deployment_id }}
-          auto_inactive: true
+          env_url: https://develop.element.io
-      - name: Update deployment status (failure)
-        if: failure()
-        uses: avakar/set-deployment-status@v1.1.0
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          state: "failure"
-          deployment_id: ${{ steps.deployment.outputs.deployment_id }}
-          environment: Develop

.gitignore

@@ -23,3 +23,5 @@ electron/pub
 .vscode
 .vscode/
 .env
+/scripts/extracted/
+/scripts/latest

scripts/deploy.py

@@ -23,9 +23,11 @@ except ImportError:
     # python2
     from urllib import urlretrieve
 
+
 class DeployException(Exception):
     pass
 
+
 def create_relative_symlink(linkname, target):
     relpath = os.path.relpath(target, os.path.dirname(linkname))
     print ("Symlink %s -> %s" % (linkname, relpath))
@@ -57,10 +59,11 @@ def move_bundles(source, dest):
         else:
             renames[os.path.join(source, f)] = dst
 
-    for (src, dst) in renames.iteritems():
+    for (src, dst) in renames.items():
         print ("Move %s -> %s" % (src, dst))
         os.rename(src, dst)
 
+
 class Deployer:
     def __init__(self):
         self.packages_path = "."
@@ -100,7 +103,7 @@ class Deployer:
         print ("Extracted into: %s" % extracted_dir)
 
         if self.symlink_paths:
-            for link_path, file_path in self.symlink_paths.iteritems():
+            for link_path, file_path in self.symlink_paths.items():
                 create_relative_symlink(
                     target=file_path,
                     linkname=os.path.join(extracted_dir, link_path)
@@ -139,6 +142,7 @@ class Deployer:
         print ("Done")
         return local_filename
 
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser("Deploy a Riot build on a web server.")
     parser.add_argument(

scripts/package.sh

@@ -9,7 +9,7 @@ else
 fi
 
 yarn clean
-VERSION=$version yarn build-stats
+VERSION=$version yarn build
 
 # include the sample config in the tarball. Arguably this should be done by
 # `yarn build`, but it's just too painful.

scripts/redeploy.py

@@ -1,45 +1,40 @@
 #!/usr/bin/env python
-#
+
 # auto-deploy script for https://develop.element.io
-#
+
-# Listens for buildkite webhook pokes (https://buildkite.com/docs/apis/webhooks)
+# Listens for Github Action webhook pokes (https://github.com/marketplace/actions/workflow-webhook-action)
-# When it gets one, downloads the artifact from buildkite
+# When it gets one: downloads the artifact from github actions and deploys it as the new version.
-# and deploys it as the new version.
+
-#
 # Requires the following python packages:
 #
-#   - requests
 #   - flask
-#
+#   - python-github-webhook
+
 from __future__ import print_function
-import json, requests, tarfile, argparse, os, errno
+import argparse
+import os
+import errno
 import time
 import traceback
-from urlparse import urljoin
-import glob
-import re
-import shutil
-import threading
-from Queue import Queue
 
-from flask import Flask, jsonify, request, abort
+import glob
+from io import BytesIO
+from urllib.request import urlopen
+from zipfile import ZipFile
+
+from github_webhook import Webhook
+from flask import Flask, abort
 
 from deploy import Deployer, DeployException
 
 app = Flask(__name__)
+webhook = Webhook(app, endpoint="/")
 
-deployer = None
-arg_extract_path = None
-arg_symlink = None
-arg_webhook_token = None
-arg_api_token = None
 
-workQueue = Queue()
+def create_symlink(source: str, linkname: str):
-
-def create_symlink(source, linkname):
     try:
         os.symlink(source, linkname)
-    except OSError, e:
+    except OSError as e:
         if e.errno == errno.EEXIST:
             # atomic modification
             os.symlink(source, linkname + ".tmp")
@@ -47,118 +42,43 @@ def create_symlink(source, linkname):
         else:
             raise e
 
-def req_headers():
-    return {
-        "Authorization": "Bearer %s" % (arg_api_token,),
-    }
 
-# Buildkite considers a poke to have failed if it has to wait more than 10s for
+@webhook.hook(event_type="workflow_run")
-# data (any data, not just the initial response) and it normally takes longer than
+def on_deployment(payload: dict):
-# that to download an artifact from buildkite. Apparently there is no way in flask
+    repository = payload.get("repository")
-# to finish the response and then keep doing stuff, so instead this has to involve
+    if repository is None:
-# threading. Sigh.
+        abort(400, "No 'repository' specified")
-def worker_thread():
-    while True:
-        toDeploy = workQueue.get()
-        deploy_buildkite_artifact(*toDeploy)
-
-@app.route("/", methods=["POST"])
-def on_receive_buildkite_poke():
-    got_webhook_token = request.headers.get('X-Buildkite-Token')
-    if got_webhook_token != arg_webbook_token:
-        print("Denying request with incorrect webhook token: %s" % (got_webhook_token,))
-        abort(400, "Incorrect webhook token")
         return
 
-    required_api_prefix = None
+    workflow = payload.get("workflow")
-    if arg_buildkite_org is not None:
+    if repository is None:
-        required_api_prefix = 'https://api.buildkite.com/v2/organizations/%s' % (arg_buildkite_org,)
+        abort(400, "No 'workflow' specified")
-
-    incoming_json = request.get_json()
-    if not incoming_json:
-        abort(400, "No JSON provided!")
-        return
-    print("Incoming JSON: %s" % (incoming_json,))
-
-    event = incoming_json.get("event")
-    if event is None:
-        abort(400, "No 'event' specified")
         return
 
-    if event == 'ping':
+    request_id = payload.get("requestID")
-        print("Got ping request - responding")
+    if request_id is None:
-        return jsonify({'response': 'pong!'})
+        abort(400, "No 'request_id' specified")
-
-    if event != 'build.finished':
-        print("Rejecting '%s' event")
-        abort(400, "Unrecognised event")
         return
 
-    build_obj = incoming_json.get("build")
+    if arg_github_org is not None and not repository.startswith(arg_github_org):
-    if build_obj is None:
+        print("Denying poke for repository with incorrect prefix: %s" % (repository,))
-        abort(400, "No 'build' object")
+        abort(400, "Invalid repository")
         return
 
-    build_url = build_obj.get('url')
+    if arg_github_workflow is not None and workflow != arg_github_workflow:
-    if build_url is None:
+        print("Denying poke for incorrect workflow: %s" % (workflow,))
-        abort(400, "build has no url")
+        abort(400, "Incorrect workflow")
         return
 
-    if required_api_prefix is not None and not build_url.startswith(required_api_prefix):
+    artifact_url = payload.get("data", {}).get("url")
-        print("Denying poke for build url with incorrect prefix: %s" % (build_url,))
+    if artifact_url is None:
-        abort(400, "Invalid build url")
+        abort(400, "No 'data.url' specified")
         return
 
-    build_num = build_obj.get('number')
+    deploy_artifact(artifact_url, request_id)
-    if build_num is None:
-        abort(400, "build has no number")
-        return
 
-    pipeline_obj = incoming_json.get("pipeline")
-    if pipeline_obj is None:
-        abort(400, "No 'pipeline' object")
-        return
-
-    pipeline_name = pipeline_obj.get('name')
-    if pipeline_name is None:
-        abort(400, "pipeline has no name")
-        return
-
-    artifacts_url = build_url + "/artifacts"
-    artifacts_resp = requests.get(artifacts_url, headers=req_headers())
-    artifacts_resp.raise_for_status()
-    artifacts_array = artifacts_resp.json()
-    
-    artifact_to_deploy = None
-    for artifact in artifacts_array:
-        if re.match(r"dist/.*.tar.gz", artifact['path']):
-            artifact_to_deploy = artifact
-    if artifact_to_deploy is None:
-        print("No suitable artifacts found")
-        return jsonify({})
-
-    # double paranoia check: make sure the artifact is on the right org too
-    if required_api_prefix is not None and not artifact_to_deploy['url'].startswith(required_api_prefix):
-        print("Denying poke for build url with incorrect prefix: %s" % (artifact_to_deploy['url'],))
-        abort(400, "Refusing to deploy artifact from URL %s", artifact_to_deploy['url'])
-        return
-
-    # there's no point building up a queue of things to deploy, so if there are any pending jobs,
-    # remove them
-    while not workQueue.empty():
-        try:
-            workQueue.get(False)
-        except:
-            pass
-    workQueue.put([artifact_to_deploy, pipeline_name, build_num])
-
-    return jsonify({})
-
-def deploy_buildkite_artifact(artifact, pipeline_name, build_num):
-    artifact_response = requests.get(artifact['url'], headers=req_headers())
-    artifact_response.raise_for_status()
-    artifact_obj = artifact_response.json()
 
+def deploy_artifact(artifact_url: str, request_id: str):
     # we extract into a directory based on the build number. This avoids the
     # problem of multiple builds building the same git version and thus having
     # the same tarball name. That would lead to two potential problems:
@@ -166,58 +86,42 @@ def deploy_buildkite_artifact(artifact, pipeline_name, build_num):
     #       a good deploy with a bad one
     #   (b) we'll be overwriting the live deployment, which means people might
     #       see half-written files.
-    build_dir = os.path.join(arg_extract_path, "%s-#%s" % (pipeline_name, build_num))
+    build_dir = os.path.join(arg_extract_path, "gha-%s" % (request_id,))
-    try:
-        extracted_dir = deploy_tarball(artifact_obj, build_dir)
-    except DeployException as e:
-        traceback.print_exc()
-        abort(400, e.message)
 
-    create_symlink(source=extracted_dir, linkname=arg_symlink)
-
-def deploy_tarball(artifact, build_dir):
-    """Download a tarball from jenkins and unpack it
-
-    Returns:
-        (str) the path to the unpacked deployment
-    """
     if os.path.exists(build_dir):
-        raise DeployException(
+        # We have already deployed this, nop
-            "Not deploying. We have previously deployed this build."
+        return
-        )
     os.mkdir(build_dir)
 
-    print("Fetching artifact %s -> %s..." % (artifact['download_url'], artifact['filename']))
+    try:
-
+        with urlopen(artifact_url) as f:
-    # Download the tarball here as buildkite needs auth to do this
+            with ZipFile(BytesIO(f.read()), "r") as z:
-    # we don't pgp-sign buildkite artifacts, relying on HTTPS and buildkite
+                name = next((x for x in z.namelist() if x.endswith(".tar.gz")))
-    # not being evil. If that's not good enough for you, don't use develop.element.io.
+                z.extract(name, build_dir)
-    resp = requests.get(artifact['download_url'], stream=True, headers=req_headers())
+        extracted_dir = deployer.deploy(os.path.join(build_dir, name), build_dir)
-    resp.raise_for_status()
+        create_symlink(source=extracted_dir, linkname=arg_symlink)
-    with open(artifact['filename'], 'wb') as ofp:
+    except DeployException as e:
-        shutil.copyfileobj(resp.raw, ofp)
+        traceback.print_exc()
-    print("...download complete. Deploying...")
+        abort(400, str(e))
-
+    finally:
-    # we rely on the fact that flask only serves one request at a time to
+        if deployer.should_clean:
-    # ensure that we do not overwrite a tarball from a concurrent request.
+            os.remove(os.path.join(build_dir, name))
-
-    return deployer.deploy(artifact['filename'], build_dir)
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser("Runs a Vector redeployment server.")
+    parser = argparse.ArgumentParser("Runs an Element redeployment server.")
     parser.add_argument(
         "-p", "--port", dest="port", default=4000, type=int, help=(
-            "The port to listen on for requests from Jenkins."
+            "The port to listen on for redeployment requests."
         )
     )
     parser.add_argument(
-        "-e", "--extract", dest="extract", default="./extracted", help=(
+        "-e", "--extract", dest="extract", default="./extracted", type=str, help=(
             "The location to extract .tar.gz files to."
         )
     )
     parser.add_argument(
-        "-b", "--bundles-dir", dest="bundles_dir", help=(
+        "-b", "--bundles-dir", dest="bundles_dir", type=str, help=(
             "A directory to move the contents of the 'bundles' directory to. A \
             symlink to the bundles directory will also be written inside the \
             extracted tarball. Example: './bundles'."
@@ -229,7 +133,7 @@ if __name__ == "__main__":
         )
     )
     parser.add_argument(
-        "-s", "--symlink", dest="symlink", default="./latest", help=(
+        "-s", "--symlink", dest="symlink", default="./latest", type=str, help=(
             "Write a symlink to this location pointing to the extracted tarball. \
             New builds will keep overwriting this symlink. The symlink will point \
             to the /vector directory INSIDE the tarball."
@@ -238,71 +142,65 @@ if __name__ == "__main__":
 
     # --include ../../config.json ./localhost.json homepages/*
     parser.add_argument(
-        "--include", nargs='*', default='./config*.json', help=(
+        "--include", nargs='*', default='./config*.json', type=str, help=(
             "Symlink these files into the root of the deployed tarball. \
              Useful for config files and home pages. Supports glob syntax. \
              (Default: '%(default)s')"
         )
     )
     parser.add_argument(
-        "--test", dest="tarball_uri", help=(
+        "--test", dest="tarball_uri", type=str, help=(
-            "Don't start an HTTP listener. Instead download a build from Jenkins \
+            "Don't start an HTTP listener. Instead download a build from this URL immediately."
-            immediately."
         ),
     )
 
     parser.add_argument(
-        "--webhook-token", dest="webhook_token", help=(
+        "--webhook-token", dest="webhook_token", type=str, help=(
-            "Only accept pokes with this buildkite token."
+            "Only accept pokes signed with this github token."
-        ), required=True,
-    )
-
-    parser.add_argument(
-        "--api-token", dest="api_token", help=(
-            "API access token for buildkite. Require read_artifacts scope."
         ), required=True,
     )
 
     # We require a matching webhook token, but because we take everything else
     # about what to deploy from the poke body, we can be a little more paranoid
-    # and only accept builds / artifacts from a specific buildkite org
+    # and only accept builds / artifacts from a specific github org
     parser.add_argument(
-        "--org", dest="buildkite_org", help=(
+        "--org", dest="github_org", type=str, help=(
-            "Lock down to this buildkite org"
+            "Lock down to this github org"
+        )
+    )
+    # Optional matching workflow name
+    parser.add_argument(
+        "--workflow", dest="github_workflow", type=str, help=(
+            "Lock down to this github workflow"
         )
     )
 
     args = parser.parse_args()
     arg_extract_path = args.extract
     arg_symlink = args.symlink
-    arg_webbook_token = args.webhook_token
+    arg_github_org = args.github_org
-    arg_api_token = args.api_token
+    arg_github_workflow = args.github_workflow
-    arg_buildkite_org = args.buildkite_org
 
     if not os.path.isdir(arg_extract_path):
         os.mkdir(arg_extract_path)
 
+    webhook.secret = args.webhook_token
+
     deployer = Deployer()
     deployer.bundles_path = args.bundles_dir
     deployer.should_clean = args.clean
 
-    for include in args.include:
+    for include in args.include.split(" "):
         deployer.symlink_paths.update({ os.path.basename(pth): pth for pth in glob.iglob(include) })
 
-    if args.tarball_uri is not None:
+    print(
-        build_dir = os.path.join(arg_extract_path, "test-%i" % (time.time()))
+        "Listening on port %s. Extracting to %s%s. Symlinking to %s. Include files: %s" %
-        deploy_tarball(args.tarball_uri, build_dir)
+        (args.port,
-    else:
+         arg_extract_path,
-        print(
+         " (clean after)" if deployer.should_clean else "",
-            "Listening on port %s. Extracting to %s%s. Symlinking to %s. Include files: %s" %
+         arg_symlink,
-            (args.port,
+         deployer.symlink_paths,
-             arg_extract_path,
-             " (clean after)" if deployer.should_clean else "",
-             arg_symlink,
-             deployer.symlink_paths,
-            )
         )
-        fred = threading.Thread(target=worker_thread)
+    )
-        fred.daemon = True
+
-        fred.start()
+    app.run(port=args.port, debug=False)
-        app.run(port=args.port, debug=False)