Use fetchdep everywhere

.github/workflows/build.yaml

@@ -19,11 +19,8 @@ jobs:
         with:
           cache: 'yarn'
 
-      - name: Install SDKs
-        run: "./scripts/fetch-develop.deps.sh --depth 1"
-
       - name: Install Dependencies
-        run: "yarn install"
+        run: "./scripts/layered.sh"
 
       - name: Build & Package
         run: "./scripts/ci_package.sh"

.github/workflows/sentry-sourcemaps.yaml

@@ -1,4 +1,7 @@
-name: Upload Sentry Sourcemaps
+# We cannot reuse the build from the main Builder workflow
+# due to needing to inject SENTRY_ env vars into ci_package.sh
+# For security we harbor the secrets in a develop-only environment.
+name: Sentry Sourcemaps
 on:
   push:
     branches:
@@ -6,18 +9,23 @@ on:
   repository_dispatch:
     types: [ element-web-notify ]
 jobs:
-  upload-sentry-sourcemaps:
+  upload:
+    name: Upload
     runs-on: ubuntu-latest
     environment: develop
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2
+
+      - uses: actions/setup-node@v3
         with:
           node-version: '14'
           cache: 'yarn'
-      - run: ./scripts/fetch-develop.deps.sh --depth 1
+
-      - run: yarn install
+      - name: Install Dependencies
-      - run: ./scripts/ci_package.sh
+        run: "./scripts/layered.sh"
+
+      - name: Build & Package
+        run: "./scripts/ci_package.sh"
         env:
           SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
           SENTRY_DSN: ${{ secrets.SENTRY_DSN }}

.github/workflows/static_analysis.yaml

@@ -19,11 +19,8 @@ jobs:
         with:
           cache: 'yarn'
 
-      - name: Install SDKs
-        run: "./scripts/fetch-develop.deps.sh --depth 1"
-
       - name: Install Dependencies
-        run: "yarn install"
+        run: "./scripts/layered.sh"
 
       - name: Typecheck
         run: "yarn run lint:types"

.github/workflows/test.yaml

@@ -19,11 +19,8 @@ jobs:
         with:
           cache: 'yarn'
 
-      - name: Install SDKs
-        run: "./scripts/fetch-develop.deps.sh --depth 1"
-
       - name: Install Dependencies
-        run: "yarn install"
+        run: "./scripts/layered.sh"
 
       - name: Run Tests
         run: "yarn test"

scripts/fetch-develop.deps.sh

@@ -1,99 +0,0 @@
-#!/bin/bash
-
-# Fetches the js-sdk and matrix-react-sdk dependencies for development
-# or testing purposes
-# If there exists a branch of that dependency with the same name as
-# the branch the current checkout is on, use that branch. Otherwise,
-# use develop.
-
-set -ex
-
-GIT_CLONE_ARGS=("$@")
-[ -z "$defbranch" ] && defbranch="develop"
-
-# clone a specific branch of a github repo
-function clone() {
-    org=$1
-    repo=$2
-    branch=$3
-
-    # Chop 'origin' off the start as jenkins ends up using
-    # branches on the origin, but this doesn't work if we
-    # specify the branch when cloning.
-    branch=${branch#origin/}
-
-    if [ -n "$branch" ]
-    then
-        echo "Trying to use $org/$repo#$branch"
-        # Disable auth prompts: https://serverfault.com/a/665959
-        GIT_TERMINAL_PROMPT=0 git clone https://github.com/$org/$repo.git $repo --branch $branch \
-            "${GIT_CLONE_ARGS[@]}"
-        return $?
-    fi
-    return 1
-}
-
-function dodep() {
-    deforg=$1
-    defrepo=$2
-    rm -rf $defrepo
-
-    # Try the PR author's branch in case it exists on the deps as well.
-    # Try the target branch of the push or PR.
-    # Use the default branch as the last resort.
-    if [[ "$BUILDKITE" == true ]]; then
-        # If BUILDKITE_BRANCH is set, it will contain either:
-        #   * "branch" when the author's branch and target branch are in the same repo
-        #   * "author:branch" when the author's branch is in their fork
-        # We can split on `:` into an array to check.
-        BUILDKITE_BRANCH_ARRAY=(${BUILDKITE_BRANCH//:/ })
-        if [[ "${#BUILDKITE_BRANCH_ARRAY[@]}" == "2" ]]; then
-            prAuthor=${BUILDKITE_BRANCH_ARRAY[0]}
-            prBranch=${BUILDKITE_BRANCH_ARRAY[1]}
-        else
-            prAuthor=$deforg
-            prBranch=$BUILDKITE_BRANCH
-        fi
-        clone $prAuthor $defrepo $prBranch ||
-        clone $deforg $defrepo $BUILDKITE_PULL_REQUEST_BASE_BRANCH ||
-        clone $deforg $defrepo $defbranch ||
-        return $?
-    else
-        clone $deforg $defrepo $ghprbSourceBranch ||
-        clone $deforg $defrepo $GIT_BRANCH ||
-        clone $deforg $defrepo `git rev-parse --abbrev-ref HEAD` ||
-        clone $deforg $defrepo $defbranch ||
-        return $?
-    fi
-
-    echo "$defrepo set to branch "`git -C "$defrepo" rev-parse --abbrev-ref HEAD`
-}
-
-##############################
-
-echo 'Setting up matrix-js-sdk'
-
-dodep matrix-org matrix-js-sdk
-
-pushd matrix-js-sdk
-yarn link
-yarn install --pure-lockfile
-popd
-
-yarn link matrix-js-sdk
-
-##############################
-
-echo 'Setting up matrix-react-sdk'
-
-dodep matrix-org matrix-react-sdk
-
-pushd matrix-react-sdk
-yarn link
-yarn link matrix-js-sdk
-yarn install --pure-lockfile
-popd
-
-yarn link matrix-react-sdk
-
-##############################

scripts/fetchdep.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+set -x
+
+deforg="$1"
+defrepo="$2"
+defbranch="$3"
+
+[ -z "$defbranch" ] && defbranch="develop"
+
+rm -r "$defrepo" || true
+
+# A function that clones a branch of a repo based on the org, repo and branch
+clone() {
+    org=$1
+    repo=$2
+    branch=$3
+    if [ -n "$branch" ]
+    then
+        echo "Trying to use $org/$repo#$branch"
+        # Disable auth prompts: https://serverfault.com/a/665959
+        GIT_TERMINAL_PROMPT=0 git clone https://github.com/$org/$repo.git $repo --branch "$branch" --depth 1 && exit 0
+    fi
+}
+
+# A function that gets info about a PR from the GitHub API based on its number
+getPRInfo() {
+    number=$1
+    if [ -n "$number" ]; then
+        echo "Getting info about a PR with number $number"
+
+        apiEndpoint="https://api.github.com/repos/matrix-org/matrix-react-sdk/pulls/"
+        apiEndpoint+=$number
+
+        head=$(curl $apiEndpoint | jq -r '.head.label')
+    fi
+}
+
+# Some CIs don't give us enough info, so we just get the PR number and ask the
+# GH API for more info - "fork:branch". Some give us this directly.
+if [ -n "$BUILDKITE_BRANCH" ]; then
+    # BuildKite
+    head=$BUILDKITE_BRANCH
+elif [ -n "$PR_NUMBER" ]; then
+    # GitHub
+    getPRInfo $PR_NUMBER
+elif [ -n "$REVIEW_ID" ]; then
+    # Netlify
+    getPRInfo $REVIEW_ID
+fi
+
+# for forks, $head will be in the format "fork:branch", so we split it by ":"
+# into an array. On non-forks, this has the effect of splitting into a single
+# element array given ":" shouldn't appear in the head - it'll just be the
+# branch name. Based on the results, we clone.
+BRANCH_ARRAY=(${head//:/ })
+TRY_ORG=$deforg
+TRY_BRANCH=${BRANCH_ARRAY[0]}
+if [[ "$head" == *":"* ]]; then
+    # ... but only match that fork if it's a real fork
+    if [ "${BRANCH_ARRAY[0]}" != "matrix-org" ]; then
+        TRY_ORG=${BRANCH_ARRAY[0]}
+    fi
+    TRY_BRANCH=${BRANCH_ARRAY[1]}
+fi
+clone ${TRY_ORG} $defrepo ${TRY_BRANCH}
+
+# Try the target branch of the push or PR.
+if [ -n "$GITHUB_BASE_REF" ]; then
+    clone $deforg $defrepo $GITHUB_BASE_REF
+elif [ -n "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" ]; then
+    clone $deforg $defrepo $BUILDKITE_PULL_REQUEST_BASE_BRANCH
+fi
+
+# Try HEAD which is the branch name in Netlify (not BRANCH which is pull/xxxx/head for PR builds)
+clone $deforg $defrepo $HEAD
+# Use the default branch as the last resort.
+clone $deforg $defrepo $defbranch

scripts/layered.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Creates a layered environment with the full repo for the app and SDKs cloned
+# and linked. This gives an element-web dev environment ready to build with
+# matching branches of react-sdk's dependencies so that changes can be tested
+# in element-web.
+
+# Note that this style is different from the recommended developer setup: this
+# file nests js-sdk and matrix-react-sdk inside element-web, while the local
+# development setup places them all at the same level. We are nesting them here
+# because some CI systems do not allow moving to a directory above the checkout
+# for the primary repo (element-web in this case).
+
+# Set up the js-sdk first
+scripts/fetchdep.sh matrix-org matrix-js-sdk
+pushd matrix-js-sdk
+yarn link
+yarn install --pure-lockfile
+popd
+
+# Now set up the react-sdk
+scripts/fetchdep.sh matrix-org matrix-react-sdk
+pushd matrix-react-sdk
+yarn link
+yarn install --pure-lockfile
+popd
+
+# Also set up matrix-analytics-events so we get the latest from
+# the main branch or a branch with matching name
+scripts/fetchdep.sh matrix-org matrix-analytics-events main
+pushd matrix-analytics-events
+yarn link
+yarn install --pure-lockfile
+popd
+
+# Finally, set up element-web
+yarn link matrix-js-sdk
+yarn link matrix-react-sdk
+yarn link matrix-analytics-events
+yarn install --pure-lockfile