diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 0d8381bc10..e40b90fd7d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -19,11 +19,8 @@ jobs: with: cache: 'yarn' - - name: Install SDKs - run: "./scripts/fetch-develop.deps.sh --depth 1" - - name: Install Dependencies - run: "yarn install" + run: "./scripts/layered.sh" - name: Build & Package run: "./scripts/ci_package.sh" diff --git a/.github/workflows/sentry-sourcemaps.yaml b/.github/workflows/sentry-sourcemaps.yaml index 71ad7ed759..e430135dd3 100644 --- a/.github/workflows/sentry-sourcemaps.yaml +++ b/.github/workflows/sentry-sourcemaps.yaml @@ -1,4 +1,7 @@ -name: Upload Sentry Sourcemaps +# We cannot reuse the build from the main Builder workflow +# due to needing to inject SENTRY_ env vars into ci_package.sh +# For security we harbor the secrets in a develop-only environment. +name: Sentry Sourcemaps on: push: branches: @@ -6,18 +9,23 @@ on: repository_dispatch: types: [ element-web-notify ] jobs: - upload-sentry-sourcemaps: + upload: + name: Upload runs-on: ubuntu-latest environment: develop steps: - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 + + - uses: actions/setup-node@v3 with: node-version: '14' cache: 'yarn' - - run: ./scripts/fetch-develop.deps.sh --depth 1 - - run: yarn install - - run: ./scripts/ci_package.sh + + - name: Install Dependencies + run: "./scripts/layered.sh" + + - name: Build & Package + run: "./scripts/ci_package.sh" env: SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} SENTRY_DSN: ${{ secrets.SENTRY_DSN }} diff --git a/.github/workflows/static_analysis.yaml b/.github/workflows/static_analysis.yaml index be8ee9eaa3..7ea12f7707 100644 --- a/.github/workflows/static_analysis.yaml +++ b/.github/workflows/static_analysis.yaml @@ -19,11 +19,8 @@ jobs: with: cache: 'yarn' - - name: Install SDKs - run: "./scripts/fetch-develop.deps.sh --depth 1" - - name: Install Dependencies - run: "yarn install" + run: "./scripts/layered.sh" - name: Typecheck run: "yarn run lint:types" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2be1ef5e1f..df35852817 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -19,11 +19,8 @@ jobs: with: cache: 'yarn' - - name: Install SDKs - run: "./scripts/fetch-develop.deps.sh --depth 1" - - name: Install Dependencies - run: "yarn install" + run: "./scripts/layered.sh" - name: Run Tests run: "yarn test" diff --git a/scripts/fetch-develop.deps.sh b/scripts/fetch-develop.deps.sh deleted file mode 100755 index 71850188a6..0000000000 --- a/scripts/fetch-develop.deps.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/bin/bash - -# Fetches the js-sdk and matrix-react-sdk dependencies for development -# or testing purposes -# If there exists a branch of that dependency with the same name as -# the branch the current checkout is on, use that branch. Otherwise, -# use develop. - -set -ex - -GIT_CLONE_ARGS=("$@") -[ -z "$defbranch" ] && defbranch="develop" - -# clone a specific branch of a github repo -function clone() { - org=$1 - repo=$2 - branch=$3 - - # Chop 'origin' off the start as jenkins ends up using - # branches on the origin, but this doesn't work if we - # specify the branch when cloning. - branch=${branch#origin/} - - if [ -n "$branch" ] - then - echo "Trying to use $org/$repo#$branch" - # Disable auth prompts: https://serverfault.com/a/665959 - GIT_TERMINAL_PROMPT=0 git clone https://github.com/$org/$repo.git $repo --branch $branch \ - "${GIT_CLONE_ARGS[@]}" - return $? - fi - return 1 -} - -function dodep() { - deforg=$1 - defrepo=$2 - rm -rf $defrepo - - # Try the PR author's branch in case it exists on the deps as well. - # Try the target branch of the push or PR. - # Use the default branch as the last resort. - if [[ "$BUILDKITE" == true ]]; then - # If BUILDKITE_BRANCH is set, it will contain either: - # * "branch" when the author's branch and target branch are in the same repo - # * "author:branch" when the author's branch is in their fork - # We can split on `:` into an array to check. - BUILDKITE_BRANCH_ARRAY=(${BUILDKITE_BRANCH//:/ }) - if [[ "${#BUILDKITE_BRANCH_ARRAY[@]}" == "2" ]]; then - prAuthor=${BUILDKITE_BRANCH_ARRAY[0]} - prBranch=${BUILDKITE_BRANCH_ARRAY[1]} - else - prAuthor=$deforg - prBranch=$BUILDKITE_BRANCH - fi - clone $prAuthor $defrepo $prBranch || - clone $deforg $defrepo $BUILDKITE_PULL_REQUEST_BASE_BRANCH || - clone $deforg $defrepo $defbranch || - return $? - else - clone $deforg $defrepo $ghprbSourceBranch || - clone $deforg $defrepo $GIT_BRANCH || - clone $deforg $defrepo `git rev-parse --abbrev-ref HEAD` || - clone $deforg $defrepo $defbranch || - return $? - fi - - echo "$defrepo set to branch "`git -C "$defrepo" rev-parse --abbrev-ref HEAD` -} - -############################## - -echo 'Setting up matrix-js-sdk' - -dodep matrix-org matrix-js-sdk - -pushd matrix-js-sdk -yarn link -yarn install --pure-lockfile -popd - -yarn link matrix-js-sdk - -############################## - -echo 'Setting up matrix-react-sdk' - -dodep matrix-org matrix-react-sdk - -pushd matrix-react-sdk -yarn link -yarn link matrix-js-sdk -yarn install --pure-lockfile -popd - -yarn link matrix-react-sdk - -############################## diff --git a/scripts/fetchdep.sh b/scripts/fetchdep.sh new file mode 100755 index 0000000000..9091a2bacd --- /dev/null +++ b/scripts/fetchdep.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +set -x + +deforg="$1" +defrepo="$2" +defbranch="$3" + +[ -z "$defbranch" ] && defbranch="develop" + +rm -r "$defrepo" || true + +# A function that clones a branch of a repo based on the org, repo and branch +clone() { + org=$1 + repo=$2 + branch=$3 + if [ -n "$branch" ] + then + echo "Trying to use $org/$repo#$branch" + # Disable auth prompts: https://serverfault.com/a/665959 + GIT_TERMINAL_PROMPT=0 git clone https://github.com/$org/$repo.git $repo --branch "$branch" --depth 1 && exit 0 + fi +} + +# A function that gets info about a PR from the GitHub API based on its number +getPRInfo() { + number=$1 + if [ -n "$number" ]; then + echo "Getting info about a PR with number $number" + + apiEndpoint="https://api.github.com/repos/matrix-org/matrix-react-sdk/pulls/" + apiEndpoint+=$number + + head=$(curl $apiEndpoint | jq -r '.head.label') + fi +} + +# Some CIs don't give us enough info, so we just get the PR number and ask the +# GH API for more info - "fork:branch". Some give us this directly. +if [ -n "$BUILDKITE_BRANCH" ]; then + # BuildKite + head=$BUILDKITE_BRANCH +elif [ -n "$PR_NUMBER" ]; then + # GitHub + getPRInfo $PR_NUMBER +elif [ -n "$REVIEW_ID" ]; then + # Netlify + getPRInfo $REVIEW_ID +fi + +# for forks, $head will be in the format "fork:branch", so we split it by ":" +# into an array. On non-forks, this has the effect of splitting into a single +# element array given ":" shouldn't appear in the head - it'll just be the +# branch name. Based on the results, we clone. +BRANCH_ARRAY=(${head//:/ }) +TRY_ORG=$deforg +TRY_BRANCH=${BRANCH_ARRAY[0]} +if [[ "$head" == *":"* ]]; then + # ... but only match that fork if it's a real fork + if [ "${BRANCH_ARRAY[0]}" != "matrix-org" ]; then + TRY_ORG=${BRANCH_ARRAY[0]} + fi + TRY_BRANCH=${BRANCH_ARRAY[1]} +fi +clone ${TRY_ORG} $defrepo ${TRY_BRANCH} + +# Try the target branch of the push or PR. +if [ -n "$GITHUB_BASE_REF" ]; then + clone $deforg $defrepo $GITHUB_BASE_REF +elif [ -n "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" ]; then + clone $deforg $defrepo $BUILDKITE_PULL_REQUEST_BASE_BRANCH +fi + +# Try HEAD which is the branch name in Netlify (not BRANCH which is pull/xxxx/head for PR builds) +clone $deforg $defrepo $HEAD +# Use the default branch as the last resort. +clone $deforg $defrepo $defbranch diff --git a/scripts/layered.sh b/scripts/layered.sh new file mode 100755 index 0000000000..ada578497d --- /dev/null +++ b/scripts/layered.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Creates a layered environment with the full repo for the app and SDKs cloned +# and linked. This gives an element-web dev environment ready to build with +# matching branches of react-sdk's dependencies so that changes can be tested +# in element-web. + +# Note that this style is different from the recommended developer setup: this +# file nests js-sdk and matrix-react-sdk inside element-web, while the local +# development setup places them all at the same level. We are nesting them here +# because some CI systems do not allow moving to a directory above the checkout +# for the primary repo (element-web in this case). + +# Set up the js-sdk first +scripts/fetchdep.sh matrix-org matrix-js-sdk +pushd matrix-js-sdk +yarn link +yarn install --pure-lockfile +popd + +# Now set up the react-sdk +scripts/fetchdep.sh matrix-org matrix-react-sdk +pushd matrix-react-sdk +yarn link +yarn install --pure-lockfile +popd + +# Also set up matrix-analytics-events so we get the latest from +# the main branch or a branch with matching name +scripts/fetchdep.sh matrix-org matrix-analytics-events main +pushd matrix-analytics-events +yarn link +yarn install --pure-lockfile +popd + +# Finally, set up element-web +yarn link matrix-js-sdk +yarn link matrix-react-sdk +yarn link matrix-analytics-events +yarn install --pure-lockfile