commit 369cfa02da5b3e2b3c99ab33e09553c63d00ef71 · pyrox.dev/nixpkgs

ci/default.nix

···

       24
       24
        
       in

     

       25
       25
        
       {

     

       26
       26
        
         inherit pkgs;

     

       27
       27
       +
         requestReviews = pkgs.callPackage ./request-reviews { };

     

       27
       28
        
       }

+43

ci/request-reviews/default.nix

···

       1
       1
       +
       {

     

       2
       2
       +
         lib,

     

       3
       3
       +
         stdenvNoCC,

     

       4
       4
       +
         makeWrapper,

     

       5
       5
       +
         coreutils,

     

       6
       6
       +
         codeowners,

     

       7
       7
       +
         jq,

     

       8
       8
       +
         curl,

     

       9
       9
       +
         github-cli,

     

       10
       10
       +
         gitMinimal,

     

       11
       11
       +
       }:

     

       12
       12
       +
       stdenvNoCC.mkDerivation {

     

       13
       13
       +
         name = "request-reviews";

     

       14
       14
       +
         src = lib.fileset.toSource {

     

       15
       15
       +
           root = ./.;

     

       16
       16
       +
           fileset = lib.fileset.unions [

     

       17
       17
       +
             ./get-reviewers.sh

     

       18
       18
       +
             ./request-reviews.sh

     

       19
       19
       +
             ./verify-base-branch.sh

     

       20
       20
       +
             ./dev-branches.txt

     

       21
       21
       +
           ];

     

       22
       22
       +
         };

     

       23
       23
       +
         nativeBuildInputs = [ makeWrapper ];

     

       24
       24
       +
         dontBuild = true;

     

       25
       25
       +
         installPhase = ''

     

       26
       26
       +
           mkdir -p $out/bin

     

       27
       27
       +
           mv dev-branches.txt $out/bin

     

       28
       28
       +
           for bin in *.sh; do

     

       29
       29
       +
             mv "$bin" "$out/bin"

     

       30
       30
       +
             wrapProgram "$out/bin/$bin" \

     

       31
       31
       +
               --set PATH ${

     

       32
       32
       +
                 lib.makeBinPath [

     

       33
       33
       +
                   coreutils

     

       34
       34
       +
                   codeowners

     

       35
       35
       +
                   jq

     

       36
       36
       +
                   curl

     

       37
       37
       +
                   github-cli

     

       38
       38
       +
                   gitMinimal

     

       39
       39
       +
                 ]

     

       40
       40
       +
               }

     

       41
       41
       +
           done

     

       42
       42
       +
         '';

     

       43
       43
       +
       }

ci/request-reviews/dev-branches.txt

···

       1
       1
       +
       # Trusted development branches:

     

       2
       2
       +
       # These generally require PRs to update and are built by Hydra.

     

       3
       3
       +
       master

     

       4
       4
       +
       staging

     

       5
       5
       +
       release-*

     

       6
       6
       +
       staging-*

     

       7
       7
       +
       haskell-updates

+87

ci/request-reviews/get-reviewers.sh

···

       1
       1
       +
       #!/usr/bin/env bash

     

       2
       2
       +
       

     

       3
       3
       +
       # Get the code owners of the files changed by a PR,

     

       4
       4
       +
       # suitable to be consumed by the API endpoint to request reviews:

     

       5
       5
       +
       # https://docs.github.com/en/rest/pulls/review-requests?apiVersion=2022-11-28#request-reviewers-for-a-pull-request

     

       6
       6
       +
       

     

       7
       7
       +
       set -euo pipefail

     

       8
       8
       +
       

     

       9
       9
       +
       log() {

     

       10
       10
       +
           echo "$@" >&2

     

       11
       11
       +
       }

     

       12
       12
       +
       

     

       13
       13
       +
       if (( "$#" < 5 )); then

     

       14
       14
       +
           log "Usage: $0 GIT_REPO BASE_REF HEAD_REF OWNERS_FILE PR_AUTHOR"

     

       15
       15
       +
           exit 1

     

       16
       16
       +
       fi

     

       17
       17
       +
       

     

       18
       18
       +
       gitRepo=$1

     

       19
       19
       +
       baseRef=$2

     

       20
       20
       +
       headRef=$3

     

       21
       21
       +
       ownersFile=$4

     

       22
       22
       +
       prAuthor=$5

     

       23
       23
       +
       

     

       24
       24
       +
       tmp=$(mktemp -d)

     

       25
       25
       +
       trap 'rm -rf "$tmp"' exit

     

       26
       26
       +
       

     

       27
       27
       +
       git -C "$gitRepo" diff --name-only --merge-base "$baseRef" "$headRef" > "$tmp/touched-files"

     

       28
       28
       +
       readarray -t touchedFiles < "$tmp/touched-files"

     

       29
       29
       +
       log "This PR touches ${#touchedFiles[@]} files"

     

       30
       30
       +
       

     

       31
       31
       +
       # Get the owners file from the base, because we don't want to allow PRs to

     

       32
       32
       +
       # remove code owners to avoid pinging them

     

       33
       33
       +
       git -C "$gitRepo" show "$baseRef":"$ownersFile" > "$tmp"/codeowners

     

       34
       34
       +
       

     

       35
       35
       +
       # Associative arrays with the team/user as the key for easy deduplication

     

       36
       36
       +
       declare -A teams users

     

       37
       37
       +
       

     

       38
       38
       +
       for file in "${touchedFiles[@]}"; do

     

       39
       39
       +
           result=$(codeowners --file "$tmp"/codeowners "$file")

     

       40
       40
       +
       

     

       41
       41
       +
           read -r file owners <<< "$result"

     

       42
       42
       +
           if [[ "$owners" == "(unowned)" ]]; then

     

       43
       43
       +
               log "File $file is unowned"

     

       44
       44
       +
               continue

     

       45
       45
       +
           fi

     

       46
       46
       +
           log "File $file is owned by $owners"

     

       47
       47
       +
       

     

       48
       48
       +
           # Split up multiple owners, separated by arbitrary amounts of spaces

     

       49
       49
       +
           IFS=" " read -r -a entries <<< "$owners"

     

       50
       50
       +
       

     

       51
       51
       +
           for entry in "${entries[@]}"; do

     

       52
       52
       +
               # GitHub technically also supports Emails as code owners,

     

       53
       53
       +
               # but we can't easily support that, so let's not

     

       54
       54
       +
               if [[ ! "$entry" =~ @(.*) ]]; then

     

       55
       55
       +
                   warn -e "\e[33mCodeowner \"$entry\" for file $file is not valid: Must start with \"@\"\e[0m" >&2

     

       56
       56
       +
                   # Don't fail, because the PR for which this script runs can't fix it,

     

       57
       57
       +
                   # it has to be fixed in the base branch

     

       58
       58
       +
                   continue

     

       59
       59
       +
               fi

     

       60
       60
       +
               # The first regex match is everything after the @

     

       61
       61
       +
               entry=${BASH_REMATCH[1]}

     

       62
       62
       +
               if [[ "$entry" =~ .*/(.*) ]]; then

     

       63
       63
       +
                   # Teams look like $org/$team, where we only need $team for the API

     

       64
       64
       +
                   # call to request reviews from teams

     

       65
       65
       +
                   teams[${BASH_REMATCH[1]}]=

     

       66
       66
       +
               else

     

       67
       67
       +
                   # Everything else is a user

     

       68
       68
       +
                   users[$entry]=

     

       69
       69
       +
               fi

     

       70
       70
       +
           done

     

       71
       71
       +
       

     

       72
       72
       +
       done

     

       73
       73
       +
       

     

       74
       74
       +
       # Cannot request a review from the author

     

       75
       75
       +
       if [[ -v users[$prAuthor] ]]; then

     

       76
       76
       +
           log "One or more files are owned by the PR author, ignoring"

     

       77
       77
       +
           unset 'users[$prAuthor]'

     

       78
       78
       +
       fi

     

       79
       79
       +
       

     

       80
       80
       +
       # Turn it into a JSON for the GitHub API call to request PR reviewers

     

       81
       81
       +
       jq -n \

     

       82
       82
       +
           --arg users "${!users[*]}" \

     

       83
       83
       +
           --arg teams "${!teams[*]}" \

     

       84
       84
       +
           '{

     

       85
       85
       +
             reviewers: $users | split(" "),

     

       86
       86
       +
             team_reviewers: $teams | split(" ")

     

       87
       87
       +
           }'

+97

ci/request-reviews/request-reviews.sh

···

       1
       1
       +
       #!/usr/bin/env bash

     

       2
       2
       +
       

     

       3
       3
       +
       # Requests reviews for a PR after verifying that the base branch is correct

     

       4
       4
       +
       

     

       5
       5
       +
       set -euo pipefail

     

       6
       6
       +
       tmp=$(mktemp -d)

     

       7
       7
       +
       trap 'rm -rf "$tmp"' exit

     

       8
       8
       +
       SCRIPT_DIR=$(dirname "$0")

     

       9
       9
       +
       

     

       10
       10
       +
       log() {

     

       11
       11
       +
           echo "$@" >&2

     

       12
       12
       +
       }

     

       13
       13
       +
       

     

       14
       14
       +
       effect() {

     

       15
       15
       +
           if [[ -n "${DRY_MODE:-}" ]]; then

     

       16
       16
       +
               log "Skipping in dry mode:" "${@@Q}"

     

       17
       17
       +
           else

     

       18
       18
       +
               "$@"

     

       19
       19
       +
           fi

     

       20
       20
       +
       }

     

       21
       21
       +
       

     

       22
       22
       +
       if (( $# < 3 )); then

     

       23
       23
       +
           log "Usage: $0 GITHUB_REPO PR_NUMBER OWNERS_FILE"

     

       24
       24
       +
           exit 1

     

       25
       25
       +
       fi

     

       26
       26
       +
       baseRepo=$1

     

       27
       27
       +
       prNumber=$2

     

       28
       28
       +
       ownersFile=$3

     

       29
       29
       +
       

     

       30
       30
       +
       log "Fetching PR info"

     

       31
       31
       +
       prInfo=$(gh api \

     

       32
       32
       +
         -H "Accept: application/vnd.github+json" \

     

       33
       33
       +
         -H "X-GitHub-Api-Version: 2022-11-28" \

     

       34
       34
       +
         "/repos/$baseRepo/pulls/$prNumber")

     

       35
       35
       +
       

     

       36
       36
       +
       baseBranch=$(jq -r .base.ref <<< "$prInfo")

     

       37
       37
       +
       log "Base branch: $baseBranch"

     

       38
       38
       +
       prRepo=$(jq -r .head.repo.full_name <<< "$prInfo")

     

       39
       39
       +
       log "PR repo: $prRepo"

     

       40
       40
       +
       prBranch=$(jq -r .head.ref <<< "$prInfo")

     

       41
       41
       +
       log "PR branch: $prBranch"

     

       42
       42
       +
       prAuthor=$(jq -r .user.login <<< "$prInfo")

     

       43
       43
       +
       log "PR author: $prAuthor"

     

       44
       44
       +
       

     

       45
       45
       +
       extraArgs=()

     

       46
       46
       +
       if pwdRepo=$(git rev-parse --show-toplevel 2>/dev/null); then

     

       47
       47
       +
           # Speedup for local runs

     

       48
       48
       +
           extraArgs+=(--reference-if-able "$pwdRepo")

     

       49
       49
       +
       fi

     

       50
       50
       +
       

     

       51
       51
       +
       log "Fetching Nixpkgs commit history"

     

       52
       52
       +
       # We only need the commit history, not the contents, so we can do a tree-less clone using tree:0

     

       53
       53
       +
       # https://github.blog/open-source/git/get-up-to-speed-with-partial-clone-and-shallow-clone/#user-content-quick-summary

     

       54
       54
       +
       git clone --bare --filter=tree:0 --no-tags --origin upstream "${extraArgs[@]}" https://github.com/"$baseRepo".git "$tmp"/nixpkgs.git

     

       55
       55
       +
       

     

       56
       56
       +
       log "Fetching the PR commit history"

     

       57
       57
       +
       # Fetch the PR

     

       58
       58
       +
       git -C "$tmp/nixpkgs.git" remote add fork https://github.com/"$prRepo".git

     

       59
       59
       +
       # This remote config is the same as --filter=tree:0 when cloning

     

       60
       60
       +
       git -C "$tmp/nixpkgs.git" config remote.fork.partialclonefilter tree:0

     

       61
       61
       +
       git -C "$tmp/nixpkgs.git" config remote.fork.promisor true

     

       62
       62
       +
       

     

       63
       63
       +
       # This should not conflict with any refs in Nixpkgs

     

       64
       64
       +
       headRef=refs/remotes/fork/pr

     

       65
       65
       +
       # Only fetch into a remote ref, because the local ref namespace is used by Nixpkgs, don't want any conflicts

     

       66
       66
       +
       git -C "$tmp/nixpkgs.git" fetch --no-tags fork "$prBranch":"$headRef"

     

       67
       67
       +
       

     

       68
       68
       +
       log "Checking correctness of the base branch"

     

       69
       69
       +
       if ! "$SCRIPT_DIR"/verify-base-branch.sh "$tmp/nixpkgs.git" "$headRef" "$baseRepo" "$baseBranch" "$prRepo" "$prBranch" | tee "$tmp/invalid-base-error" >&2; then

     

       70
       70
       +
           log "Posting error as comment"

     

       71
       71
       +
           if ! response=$(effect gh api \

     

       72
       72
       +
               --method POST \

     

       73
       73
       +
               -H "Accept: application/vnd.github+json" \

     

       74
       74
       +
               -H "X-GitHub-Api-Version: 2022-11-28" \

     

       75
       75
       +
               "/repos/$baseRepo/issues/$prNumber/comments" \

     

       76
       76
       +
               -F "body=@$tmp/invalid-base-error"); then

     

       77
       77
       +
               log "Failed to post the comment: $response"

     

       78
       78
       +
           fi

     

       79
       79
       +
           exit 1

     

       80
       80
       +
       fi

     

       81
       81
       +
       

     

       82
       82
       +
       log "Getting code owners to request reviews from"

     

       83
       83
       +
       "$SCRIPT_DIR"/get-reviewers.sh "$tmp/nixpkgs.git" "$baseBranch" "$headRef" "$ownersFile" "$prAuthor" > "$tmp/reviewers.json"

     

       84
       84
       +
       

     

       85
       85
       +
       log "Requesting reviews from: $(<"$tmp/reviewers.json")"

     

       86
       86
       +
       

     

       87
       87
       +
       if ! response=$(effect gh api \

     

       88
       88
       +
           --method POST \

     

       89
       89
       +
           -H "Accept: application/vnd.github+json" \

     

       90
       90
       +
           -H "X-GitHub-Api-Version: 2022-11-28" \

     

       91
       91
       +
           "/repos/$baseRepo/pulls/$prNumber/requested_reviewers" \

     

       92
       92
       +
           --input "$tmp/reviewers.json"); then

     

       93
       93
       +
           log "Failed to request reviews: $response"

     

       94
       94
       +
           exit 1

     

       95
       95
       +
       fi

     

       96
       96
       +
       

     

       97
       97
       +
       log "Successfully requested reviews"

+103

ci/request-reviews/verify-base-branch.sh

···

       1
       1
       +
       #!/usr/bin/env bash

     

       2
       2
       +
       

     

       3
       3
       +
       # Check that a PR doesn't include commits from other development branches.

     

       4
       4
       +
       # Fails with next steps if it does

     

       5
       5
       +
       

     

       6
       6
       +
       set -euo pipefail

     

       7
       7
       +
       tmp=$(mktemp -d)

     

       8
       8
       +
       trap 'rm -rf "$tmp"' exit

     

       9
       9
       +
       SCRIPT_DIR=$(dirname "$0")

     

       10
       10
       +
       

     

       11
       11
       +
       log() {

     

       12
       12
       +
           echo "$@" >&2

     

       13
       13
       +
       }

     

       14
       14
       +
       

     

       15
       15
       +
       # Small helper to check whether an element is in a list

     

       16
       16
       +
       # Usage: `elementIn foo "${list[@]}"`

     

       17
       17
       +
       elementIn() {

     

       18
       18
       +
           local e match=$1

     

       19
       19
       +
           shift

     

       20
       20
       +
           for e; do

     

       21
       21
       +
               if [[ "$e" == "$match" ]]; then

     

       22
       22
       +
                   return 0

     

       23
       23
       +
               fi

     

       24
       24
       +
           done

     

       25
       25
       +
           return 1

     

       26
       26
       +
       }

     

       27
       27
       +
       

     

       28
       28
       +
       if (( $# < 6 )); then

     

       29
       29
       +
           log "Usage: $0 LOCAL_REPO HEAD_REF BASE_REPO BASE_BRANCH PR_REPO PR_BRANCH"

     

       30
       30
       +
           exit 1

     

       31
       31
       +
       fi

     

       32
       32
       +
       localRepo=$1

     

       33
       33
       +
       headRef=$2

     

       34
       34
       +
       baseRepo=$3

     

       35
       35
       +
       baseBranch=$4

     

       36
       36
       +
       prRepo=$5

     

       37
       37
       +
       prBranch=$6

     

       38
       38
       +
       

     

       39
       39
       +
       # All development branches

     

       40
       40
       +
       devBranchPatterns=()

     

       41
       41
       +
       while read -r pattern; do

     

       42
       42
       +
           if [[ "$pattern" != '#'* ]]; then

     

       43
       43
       +
               devBranchPatterns+=("$pattern")

     

       44
       44
       +
           fi

     

       45
       45
       +
       done < "$SCRIPT_DIR/dev-branches.txt"

     

       46
       46
       +
       

     

       47
       47
       +
       git -C "$localRepo" branch --list --format "%(refname:short)" "${devBranchPatterns[@]}" > "$tmp/dev-branches"

     

       48
       48
       +
       readarray -t devBranches < "$tmp/dev-branches"

     

       49
       49
       +
       

     

       50
       50
       +
       if [[ "$baseRepo" == "$prRepo" ]] && elementIn "$prBranch" "${devBranches[@]}"; then

     

       51
       51
       +
           log "This PR merges $prBranch into $baseBranch, no commit check necessary"

     

       52
       52
       +
           exit 0

     

       53
       53
       +
       fi

     

       54
       54
       +
       

     

       55
       55
       +
       # The current merge base of the PR

     

       56
       56
       +
       prMergeBase=$(git -C "$localRepo" merge-base "$baseBranch" "$headRef")

     

       57
       57
       +
       log "The PR's merge base with the base branch $baseBranch is $prMergeBase"

     

       58
       58
       +
       

     

       59
       59
       +
       # This is purely for debugging

     

       60
       60
       +
       git -C "$localRepo" rev-list --reverse "$baseBranch".."$headRef" > "$tmp/pr-commits"

     

       61
       61
       +
       log "The PR includes these $(wc -l < "$tmp/pr-commits") commits:"

     

       62
       62
       +
       cat <"$tmp/pr-commits" >&2

     

       63
       63
       +
       

     

       64
       64
       +
       for testBranch in "${devBranches[@]}"; do

     

       65
       65
       +
       

     

       66
       66
       +
           if [[ -z "$(git -C "$localRepo" rev-list -1 --since="1 month ago" "$testBranch")" ]]; then

     

       67
       67
       +
               log "Not checking $testBranch, was inactive for the last month"

     

       68
       68
       +
               continue

     

       69
       69
       +
           fi

     

       70
       70
       +
           log "Checking if commits from $testBranch are included in the PR"

     

       71
       71
       +
       

     

       72
       72
       +
           # We need to check for any commits that are in the PR which are also in the test branch.

     

       73
       73
       +
           # We could check each commit from the PR individually, but that's unnecessarily slow.

     

       74
       74
       +
           #

     

       75
       75
       +
           # This does _almost_ what we want: `git rev-list --count headRef testBranch ^baseBranch`,

     

       76
       76
       +
           # except that it includes commits that are reachable from _either_ headRef or testBranch,

     

       77
       77
       +
           # instead of restricting it to ones reachable by both

     

       78
       78
       +
       

     

       79
       79
       +
           # Easily fixable though, because we can use `git merge-base testBranch headRef`

     

       80
       80
       +
           # to get the least common ancestor (aka merge base) commit reachable by both.

     

       81
       81
       +
           # If the branch being tested is indeed the right base branch,

     

       82
       82
       +
           # this is then also the commit from that branch that the PR is based on top of.

     

       83
       83
       +
           testMergeBase=$(git -C "$localRepo" merge-base "$testBranch" "$headRef")

     

       84
       84
       +
       

     

       85
       85
       +
           # And then use the `git rev-list --count`, but replacing the non-working

     

       86
       86
       +
           # `headRef testBranch` with the merge base of the two.

     

       87
       87
       +
           extraCommits=$(git -C "$localRepo" rev-list --count "$testMergeBase" ^"$baseBranch")

     

       88
       88
       +
       

     

       89
       89
       +
           if (( extraCommits != 0 )); then

     

       90
       90
       +
               log -e "\e[33m"

     

       91
       91
       +
               echo "The PR's base branch is set to $baseBranch, but $extraCommits commits from the $testBranch branch are included. Make sure you know the [right base branch for your changes](https://github.com/NixOS/nixpkgs/blob/master/CONTRIBUTING.md#branch-conventions), then:"

     

       92
       92
       +
               echo "- If the changes should go to the $testBranch branch, [change the base branch](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-base-branch-of-a-pull-request) to $testBranch"

     

       93
       93
       +
               echo "- If the changes should go to the $baseBranch branch, rebase your PR onto the merge base with the $testBranch branch:"

     

       94
       94
       +
               echo "  \`\`\`"

     

       95
       95
       +
               echo "  git rebase --onto $prMergeBase $testMergeBase"

     

       96
       96
       +
               echo "  git push --force-with-lease"

     

       97
       97
       +
               echo "  \`\`\`"

     

       98
       98
       +
               log -e "\e[m"

     

       99
       99
       +
               exit 1

     

       100
       100
       +
           fi

     

       101
       101
       +
       done

     

       102
       102
       +
       

     

       103
       103
       +
       log "Base branch is correct, no commits from development branches are included"