From b9fcb61c0bafb11f8030496d7a638eeb37d19c79 Mon Sep 17 00:00:00 2001 From: Desi McAdam Date: Mon, 3 Mar 2025 13:13:16 -0700 Subject: [PATCH 1/4] feat(ci): Add changed lines check --- .github/workflows/pr-line-check.yml | 54 +++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .github/workflows/pr-line-check.yml diff --git a/.github/workflows/pr-line-check.yml b/.github/workflows/pr-line-check.yml new file mode 100644 index 00000000..ee6cac5b --- /dev/null +++ b/.github/workflows/pr-line-check.yml @@ -0,0 +1,54 @@ +name: Check PR Lines Changed + +on: + workflow_call: + inputs: + max_lines: + description: 'Maximum allowed total lines changed' + required: false + type: number + default: 1000 + base_ref: + description: 'Default base branch to compare against (if not running on a PR)' + required: false + type: string + default: 'main' + +jobs: + check-lines: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Determine base branch + id: get-base-branch + run: | + # Use the PR base branch if available; otherwise use the default input. + if [ -n "${{ github.event.pull_request.base.ref }}" ]; then + echo "Using PR base branch: ${{ github.event.pull_request.base.ref }}" + echo "base_branch=${{ github.event.pull_request.base.ref }}" >> $GITHUB_OUTPUT + else + echo "Using default base branch: ${{ inputs.base_ref }}" + echo "base_branch=${{ inputs.base_ref }}" >> $GITHUB_OUTPUT + fi + + - name: Calculate changed lines + id: line_count + run: | + BASE_BRANCH="${{ steps.get-base-branch.outputs.base_branch }}" + # Fetch the base branch for comparison + git fetch origin "$BASE_BRANCH" + + # Calculate additions and deletions across all changes between the base and HEAD + additions=$(git diff origin/"$BASE_BRANCH"...HEAD --numstat | awk '{add += $1} END {print add}') + deletions=$(git diff origin/"$BASE_BRANCH"...HEAD --numstat | awk '{del += $2} END {print del}') + total=$((additions + deletions)) + + echo "Additions: $additions, Deletions: $deletions, Total: $total" + echo "lines_changed=$total" >> $GITHUB_OUTPUT + + if [ "$total" -gt "${{ inputs.max_lines }}" ]; then + echo "Error: Total changed lines ($total) exceed the limit of ${{ inputs.max_lines }}." + exit 1 + fi From 3fdda9148f4342a78ac80641126b593110b0559b Mon Sep 17 00:00:00 2001 From: Desi McAdam Date: Mon, 3 Mar 2025 15:07:28 -0700 Subject: [PATCH 2/4] Fix linting issues --- .github/workflows/pr-line-check.yml | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pr-line-check.yml b/.github/workflows/pr-line-check.yml index ee6cac5b..e9146045 100644 --- a/.github/workflows/pr-line-check.yml +++ b/.github/workflows/pr-line-check.yml @@ -27,10 +27,10 @@ jobs: # Use the PR base branch if available; otherwise use the default input. if [ -n "${{ github.event.pull_request.base.ref }}" ]; then echo "Using PR base branch: ${{ github.event.pull_request.base.ref }}" - echo "base_branch=${{ github.event.pull_request.base.ref }}" >> $GITHUB_OUTPUT + echo "base_branch=${{ github.event.pull_request.base.ref }}" >> "$GITHUB_OUTPUT" else echo "Using default base branch: ${{ inputs.base_ref }}" - echo "base_branch=${{ inputs.base_ref }}" >> $GITHUB_OUTPUT + echo "base_branch=${{ inputs.base_ref }}" >> "$GITHUB_OUTPUT" fi - name: Calculate changed lines @@ -39,16 +39,18 @@ jobs: BASE_BRANCH="${{ steps.get-base-branch.outputs.base_branch }}" # Fetch the base branch for comparison git fetch origin "$BASE_BRANCH" - + # Calculate additions and deletions across all changes between the base and HEAD - additions=$(git diff origin/"$BASE_BRANCH"...HEAD --numstat | awk '{add += $1} END {print add}') - deletions=$(git diff origin/"$BASE_BRANCH"...HEAD --numstat | awk '{del += $2} END {print del}') + additions=$(git diff "origin/$BASE_BRANCH"...HEAD --numstat | awk '{add += $1} END {print add}') + deletions=$(git diff "origin/$BASE_BRANCH"...HEAD --numstat | awk '{del += $2} END {print del}') total=$((additions + deletions)) - + echo "Additions: $additions, Deletions: $deletions, Total: $total" - echo "lines_changed=$total" >> $GITHUB_OUTPUT - - if [ "$total" -gt "${{ inputs.max_lines }}" ]; then - echo "Error: Total changed lines ($total) exceed the limit of ${{ inputs.max_lines }}." + echo "lines_changed=$total" >> "$GITHUB_OUTPUT" + + # Assign the input value to a shell variable to ensure numeric comparison + max_lines="${{ inputs.max_lines }}" + if [ "$total" -gt "$max_lines" ]; then + echo "Error: Total changed lines ($total) exceed the limit of $max_lines." exit 1 fi From 666a843b9be8a62ca09a3a008c7b7ee082e0eb4c Mon Sep 17 00:00:00 2001 From: Desi McAdam Date: Wed, 5 Mar 2025 11:05:21 -0700 Subject: [PATCH 3/4] Add ability to pass in files to ignore in this check --- .github/workflows/pr-line-check.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr-line-check.yml b/.github/workflows/pr-line-check.yml index e9146045..f0f31453 100644 --- a/.github/workflows/pr-line-check.yml +++ b/.github/workflows/pr-line-check.yml @@ -13,6 +13,11 @@ on: required: false type: string default: 'main' + ignore_patterns: + description: 'Regex pattern for files to ignore when calculating changes' + required: false + type: string + default: '(\.lock$)' jobs: check-lines: @@ -40,15 +45,18 @@ jobs: # Fetch the base branch for comparison git fetch origin "$BASE_BRANCH" - # Calculate additions and deletions across all changes between the base and HEAD - additions=$(git diff "origin/$BASE_BRANCH"...HEAD --numstat | awk '{add += $1} END {print add}') - deletions=$(git diff "origin/$BASE_BRANCH"...HEAD --numstat | awk '{del += $2} END {print del}') + # Set the ignore pattern from input + ignore_pattern="${{ inputs.ignore_patterns }}" + + # Calculate additions and deletions across all changes between the base and HEAD, + # filtering out files matching the ignore pattern. + additions=$(git diff "origin/$BASE_BRANCH"...HEAD --numstat | grep -Ev "$ignore_pattern" | awk '{add += $1} END {print add}') + deletions=$(git diff "origin/$BASE_BRANCH"...HEAD --numstat | grep -Ev "$ignore_pattern" | awk '{del += $2} END {print del}') total=$((additions + deletions)) echo "Additions: $additions, Deletions: $deletions, Total: $total" echo "lines_changed=$total" >> "$GITHUB_OUTPUT" - # Assign the input value to a shell variable to ensure numeric comparison max_lines="${{ inputs.max_lines }}" if [ "$total" -gt "$max_lines" ]; then echo "Error: Total changed lines ($total) exceed the limit of $max_lines." From 03752d93d125cebdff3cc63c6426bc8ca8a8a868 Mon Sep 17 00:00:00 2001 From: Desi McAdam Date: Thu, 10 Apr 2025 07:56:38 -0600 Subject: [PATCH 4/4] Optimize shallow fetch for PR diff calculation - Added incremental shallow fetches with depths 1, 10, and 100 to avoid a full repo fetch. - Implemented logic to detect when the merge-base is available using the fetched commits. - Fall back to fetching the full history (unshallow) if necessary to ensure proper diff calculation. --- .github/workflows/pr-line-check.yml | 33 +++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-line-check.yml b/.github/workflows/pr-line-check.yml index f0f31453..8789d974 100644 --- a/.github/workflows/pr-line-check.yml +++ b/.github/workflows/pr-line-check.yml @@ -41,9 +41,38 @@ jobs: - name: Calculate changed lines id: line_count run: | + set -e + BASE_BRANCH="${{ steps.get-base-branch.outputs.base_branch }}" - # Fetch the base branch for comparison - git fetch origin "$BASE_BRANCH" + echo "Using base branch: $BASE_BRANCH" + + # Instead of a full fetch, perform incremental fetches at increasing depth + # until the merge-base between origin/ and HEAD is present. + fetch_with_depth() { + local depth=$1 + echo "Attempting to fetch with depth $depth..." + git fetch --depth="$depth" origin "$BASE_BRANCH" + } + + depths=(1 10 100) + merge_base_found=false + + for d in "${depths[@]}"; do + fetch_with_depth "$d" + if git merge-base "origin/$BASE_BRANCH" HEAD > /dev/null 2>&1; then + echo "Merge base found with depth $d." + merge_base_found=true + break + else + echo "Merge base not found with depth $d, increasing depth..." + fi + done + + # If we haven't found the merge base with shallow fetches, unshallow the repo. + if [ "$merge_base_found" = false ]; then + echo "Could not find merge base with shallow fetches, fetching full history..." + git fetch --unshallow origin "$BASE_BRANCH" || git fetch origin "$BASE_BRANCH" + fi # Set the ignore pattern from input ignore_pattern="${{ inputs.ignore_patterns }}"