ci: Add jscpd duplication-scan workflow (no-changelog) (#40331)

Co-authored-by: Claude Sonnet 5 <[email protected]>
This commit is contained in:
Emilia
2026-10-07 07:37:01 +00:00
committed by GitHub
co-authored by Claude Sonnet 5
parent e500386a54
commit 81cdc278c2
3 changed files with 488 additions and 0 deletions
+68
View File
@@ -0,0 +1,68 @@
#!/usr/bin/env node
// Splits a jscpd JSON report's `duplicates` array into N bucket files so a
// caller can download and sample one small file instead of the whole report
// (which can run into tens of MB on a large monorepo). Round-robin, not
// contiguous chunks, so a run of similarly-sized clusters in scan order
// doesn't pile up in one bucket.
import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
import { join } from 'node:path';
export function splitDuplicates(duplicates, bucketCount) {
const buckets = Array.from({ length: bucketCount }, () => []);
duplicates.forEach((dup, i) => buckets[i % bucketCount].push(dup));
return buckets;
}
export function writeBuckets(buckets, outputDir, deps = {}) {
const mkdirImpl = deps.mkdir ?? mkdirSync;
const writeImpl = deps.write ?? writeFileSync;
mkdirImpl(outputDir, { recursive: true });
return buckets.map((duplicates, i) => {
const path = join(outputDir, `bucket-${i}.json`);
writeImpl(path, JSON.stringify({ duplicates }));
return path;
});
}
export function parseBucketCount(raw) {
const count = Number(raw ?? 20);
if (!Number.isInteger(count) || count <= 0) {
throw new Error(`BUCKET_COUNT must be a positive integer, got '${raw}'`);
}
return count;
}
export function loadDuplicates(reportPath, deps = {}) {
const readImpl = deps.read ?? readFileSync;
const report = JSON.parse(readImpl(reportPath, 'utf8'));
if (!Array.isArray(report.duplicates)) {
throw new Error('report.duplicates must be an array');
}
return report.duplicates;
}
// Exits non-zero on any failure. The workflow step already runs with
// continue-on-error, so this can't fail the job — it only stops a broken
// split (zero buckets written) from reporting success.
function main() {
const reportPath = process.env.REPORT_PATH;
const outputDir = process.env.OUTPUT_DIR;
if (!reportPath || !outputDir) {
process.stderr.write('split-jscpd-report: REPORT_PATH and OUTPUT_DIR are required\n');
process.exit(1);
}
try {
const bucketCount = parseBucketCount(process.env.BUCKET_COUNT);
const duplicates = loadDuplicates(reportPath);
const buckets = splitDuplicates(duplicates, bucketCount);
writeBuckets(buckets, outputDir);
} catch (err) {
process.stderr.write(`split-jscpd-report: ${err.message}\n`);
process.exit(1);
}
}
if (import.meta.url === `file://${process.argv[1]}`) {
main();
}
+111
View File
@@ -0,0 +1,111 @@
/**
* Run these tests by running
*
* node --test .github/scripts/split-jscpd-report.test.mjs
* */
import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { splitDuplicates, writeBuckets, parseBucketCount, loadDuplicates } from './split-jscpd-report.mjs';
describe('splitDuplicates', () => {
it('distributes entries round-robin across buckets', () => {
const duplicates = [1, 2, 3, 4, 5, 6];
const buckets = splitDuplicates(duplicates, 3);
assert.deepEqual(buckets, [
[1, 4],
[2, 5],
[3, 6],
]);
});
it('leaves later buckets empty when there are fewer entries than buckets', () => {
const buckets = splitDuplicates([1, 2], 5);
assert.equal(buckets.length, 5);
assert.deepEqual(buckets[0], [1]);
assert.deepEqual(buckets[1], [2]);
assert.deepEqual(buckets[2], []);
});
it('returns bucketCount empty arrays for an empty report', () => {
const buckets = splitDuplicates([], 4);
assert.deepEqual(buckets, [[], [], [], []]);
});
});
describe('writeBuckets', () => {
it('writes one JSON file per bucket, named by index', () => {
const written = [];
const paths = writeBuckets([[{ a: 1 }], [{ b: 2 }]], '/tmp/buckets', {
mkdir: () => {},
write: (path, contents) => written.push({ path, contents }),
});
assert.deepEqual(paths, ['/tmp/buckets/bucket-0.json', '/tmp/buckets/bucket-1.json']);
assert.deepEqual(JSON.parse(written[0].contents), { duplicates: [{ a: 1 }] });
assert.deepEqual(JSON.parse(written[1].contents), { duplicates: [{ b: 2 }] });
});
it('creates the output directory before writing', () => {
let mkdirCalledWith = null;
writeBuckets([[]], '/tmp/buckets', {
mkdir: (dir, opts) => {
mkdirCalledWith = { dir, opts };
},
write: () => {},
});
assert.deepEqual(mkdirCalledWith, { dir: '/tmp/buckets', opts: { recursive: true } });
});
});
describe('parseBucketCount', () => {
it('parses a numeric string', () => {
assert.equal(parseBucketCount('20'), 20);
});
it('defaults to 20 when unset', () => {
assert.equal(parseBucketCount(undefined), 20);
});
it('rejects zero', () => {
assert.throws(() => parseBucketCount('0'), /positive integer/);
});
it('rejects a negative count', () => {
assert.throws(() => parseBucketCount('-1'), /positive integer/);
});
it('rejects a non-integer', () => {
assert.throws(() => parseBucketCount('1.5'), /positive integer/);
});
it('rejects a non-numeric value', () => {
assert.throws(() => parseBucketCount('abc'), /positive integer/);
});
});
describe('loadDuplicates', () => {
it('returns the duplicates array from a valid report', () => {
const duplicates = loadDuplicates('/tmp/report.json', {
read: () => JSON.stringify({ duplicates: [{ a: 1 }] }),
});
assert.deepEqual(duplicates, [{ a: 1 }]);
});
it('rejects a report missing the duplicates array', () => {
assert.throws(
() => loadDuplicates('/tmp/report.json', { read: () => JSON.stringify({}) }),
/duplicates must be an array/,
);
});
it('rejects a report whose duplicates field is not an array', () => {
assert.throws(
() => loadDuplicates('/tmp/report.json', { read: () => JSON.stringify({ duplicates: 'x' }) }),
/duplicates must be an array/,
);
});
it('propagates a JSON parse failure', () => {
assert.throws(() => loadDuplicates('/tmp/report.json', { read: () => 'not json' }), SyntaxError);
});
});
+309
View File
@@ -0,0 +1,309 @@
# 🍝 jscpd — deterministic copy-paste/duplication scan.
#
# Runs jscpd against this repo and uploads the full JSON+HTML report, plus
# the report split into BUCKET_COUNT small JSON files, so a caller can
# download one small bucket at random per invocation instead of the full
# report (which can run into tens of MB on a monorepo this size).
#
# Reporters/ignores/threshold come from this repo's own .jscpd.json; this
# workflow passes none of those flags itself.
name: 🍝 jscpd
on:
workflow_dispatch:
permissions:
contents: read
env:
# Number of small sample files the report is split into (split-jscpd-report.mjs).
# Fixed, not an input: changing it means adding/removing the matching
# per-bucket upload steps below, so it's a code change, not a dispatch option.
BUCKET_COUNT: '20'
jobs:
jscpd:
runs-on: ${{ vars.RUNNER_PROVIDER == 'github' && 'ubuntu-latest' || 'blacksmith-4vcpu-ubuntu-2204' }}
timeout-minutes: 30
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 1
persist-credentials: false
- name: Setup Node.js and install dependencies
uses: ./.github/actions/setup-nodejs
with:
build-command: ''
- name: Run jscpd
id: jscpd
continue-on-error: true
run: |
mkdir -p "$RUNNER_TEMP/jscpd-report"
pnpm jscpd -- . --output "$RUNNER_TEMP/jscpd-report" --silent
- name: Upload jscpd report
id: report-artifact
if: always()
continue-on-error: true
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: jscpd-report-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/jscpd-report
retention-days: 31
if-no-files-found: warn
# jscpd exits non-zero once duplication crosses .jscpd.json's threshold,
# but still writes the report first. Gating the split on the report
# file existing (not on steps.jscpd.outcome) means an over-threshold
# run still gets its buckets instead of silently dropping all of them.
- name: Check jscpd report exists
id: report-exists
if: always()
run: |
if [ -f "$RUNNER_TEMP/jscpd-report/jscpd-report.json" ]; then
echo "exists=true" >> "$GITHUB_OUTPUT"
else
echo "exists=false" >> "$GITHUB_OUTPUT"
fi
# Splits `duplicates` into BUCKET_COUNT files so a caller can download
# one small sample instead of the full report (tens of MB on a large
# monorepo). Logic + tests live in split-jscpd-report.mjs.
- name: Split report into buckets
id: split
if: always() && steps.report-exists.outputs.exists == 'true'
continue-on-error: true
env:
REPORT_PATH: ${{ runner.temp }}/jscpd-report/jscpd-report.json
OUTPUT_DIR: ${{ runner.temp }}/jscpd-buckets
run: node .github/scripts/split-jscpd-report.mjs
# BUCKET_COUNT static upload steps, one per bucket file: GitHub Actions
# has no per-job step loop, and each bucket needs its own small
# artifact (not one artifact holding all of them) so a caller can
# fetch a single sample without pulling the rest. Keep this block's
# step count in sync with BUCKET_COUNT above.
- name: Upload jscpd bucket 0
id: bucket-artifact-0
if: always() && steps.split.outcome == 'success'
continue-on-error: true
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: jscpd-bucket-${{ github.run_id }}-${{ github.run_attempt }}-0
path: ${{ runner.temp }}/jscpd-buckets/bucket-0.json
retention-days: 31
if-no-files-found: error
- name: Upload jscpd bucket 1
id: bucket-artifact-1
if: always() && steps.split.outcome == 'success'
continue-on-error: true
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: jscpd-bucket-${{ github.run_id }}-${{ github.run_attempt }}-1
path: ${{ runner.temp }}/jscpd-buckets/bucket-1.json
retention-days: 31
if-no-files-found: error
- name: Upload jscpd bucket 2
id: bucket-artifact-2
if: always() && steps.split.outcome == 'success'
continue-on-error: true
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: jscpd-bucket-${{ github.run_id }}-${{ github.run_attempt }}-2
path: ${{ runner.temp }}/jscpd-buckets/bucket-2.json
retention-days: 31
if-no-files-found: error
- name: Upload jscpd bucket 3
id: bucket-artifact-3
if: always() && steps.split.outcome == 'success'
continue-on-error: true
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: jscpd-bucket-${{ github.run_id }}-${{ github.run_attempt }}-3
path: ${{ runner.temp }}/jscpd-buckets/bucket-3.json
retention-days: 31
if-no-files-found: error
- name: Upload jscpd bucket 4
id: bucket-artifact-4
if: always() && steps.split.outcome == 'success'
continue-on-error: true
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: jscpd-bucket-${{ github.run_id }}-${{ github.run_attempt }}-4
path: ${{ runner.temp }}/jscpd-buckets/bucket-4.json
retention-days: 31
if-no-files-found: error
- name: Upload jscpd bucket 5
id: bucket-artifact-5
if: always() && steps.split.outcome == 'success'
continue-on-error: true
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: jscpd-bucket-${{ github.run_id }}-${{ github.run_attempt }}-5
path: ${{ runner.temp }}/jscpd-buckets/bucket-5.json
retention-days: 31
if-no-files-found: error
- name: Upload jscpd bucket 6
id: bucket-artifact-6
if: always() && steps.split.outcome == 'success'
continue-on-error: true
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: jscpd-bucket-${{ github.run_id }}-${{ github.run_attempt }}-6
path: ${{ runner.temp }}/jscpd-buckets/bucket-6.json
retention-days: 31
if-no-files-found: error
- name: Upload jscpd bucket 7
id: bucket-artifact-7
if: always() && steps.split.outcome == 'success'
continue-on-error: true
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: jscpd-bucket-${{ github.run_id }}-${{ github.run_attempt }}-7
path: ${{ runner.temp }}/jscpd-buckets/bucket-7.json
retention-days: 31
if-no-files-found: error
- name: Upload jscpd bucket 8
id: bucket-artifact-8
if: always() && steps.split.outcome == 'success'
continue-on-error: true
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: jscpd-bucket-${{ github.run_id }}-${{ github.run_attempt }}-8
path: ${{ runner.temp }}/jscpd-buckets/bucket-8.json
retention-days: 31
if-no-files-found: error
- name: Upload jscpd bucket 9
id: bucket-artifact-9
if: always() && steps.split.outcome == 'success'
continue-on-error: true
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: jscpd-bucket-${{ github.run_id }}-${{ github.run_attempt }}-9
path: ${{ runner.temp }}/jscpd-buckets/bucket-9.json
retention-days: 31
if-no-files-found: error
- name: Upload jscpd bucket 10
id: bucket-artifact-10
if: always() && steps.split.outcome == 'success'
continue-on-error: true
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: jscpd-bucket-${{ github.run_id }}-${{ github.run_attempt }}-10
path: ${{ runner.temp }}/jscpd-buckets/bucket-10.json
retention-days: 31
if-no-files-found: error
- name: Upload jscpd bucket 11
id: bucket-artifact-11
if: always() && steps.split.outcome == 'success'
continue-on-error: true
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: jscpd-bucket-${{ github.run_id }}-${{ github.run_attempt }}-11
path: ${{ runner.temp }}/jscpd-buckets/bucket-11.json
retention-days: 31
if-no-files-found: error
- name: Upload jscpd bucket 12
id: bucket-artifact-12
if: always() && steps.split.outcome == 'success'
continue-on-error: true
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: jscpd-bucket-${{ github.run_id }}-${{ github.run_attempt }}-12
path: ${{ runner.temp }}/jscpd-buckets/bucket-12.json
retention-days: 31
if-no-files-found: error
- name: Upload jscpd bucket 13
id: bucket-artifact-13
if: always() && steps.split.outcome == 'success'
continue-on-error: true
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: jscpd-bucket-${{ github.run_id }}-${{ github.run_attempt }}-13
path: ${{ runner.temp }}/jscpd-buckets/bucket-13.json
retention-days: 31
if-no-files-found: error
- name: Upload jscpd bucket 14
id: bucket-artifact-14
if: always() && steps.split.outcome == 'success'
continue-on-error: true
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: jscpd-bucket-${{ github.run_id }}-${{ github.run_attempt }}-14
path: ${{ runner.temp }}/jscpd-buckets/bucket-14.json
retention-days: 31
if-no-files-found: error
- name: Upload jscpd bucket 15
id: bucket-artifact-15
if: always() && steps.split.outcome == 'success'
continue-on-error: true
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: jscpd-bucket-${{ github.run_id }}-${{ github.run_attempt }}-15
path: ${{ runner.temp }}/jscpd-buckets/bucket-15.json
retention-days: 31
if-no-files-found: error
- name: Upload jscpd bucket 16
id: bucket-artifact-16
if: always() && steps.split.outcome == 'success'
continue-on-error: true
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: jscpd-bucket-${{ github.run_id }}-${{ github.run_attempt }}-16
path: ${{ runner.temp }}/jscpd-buckets/bucket-16.json
retention-days: 31
if-no-files-found: error
- name: Upload jscpd bucket 17
id: bucket-artifact-17
if: always() && steps.split.outcome == 'success'
continue-on-error: true
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: jscpd-bucket-${{ github.run_id }}-${{ github.run_attempt }}-17
path: ${{ runner.temp }}/jscpd-buckets/bucket-17.json
retention-days: 31
if-no-files-found: error
- name: Upload jscpd bucket 18
id: bucket-artifact-18
if: always() && steps.split.outcome == 'success'
continue-on-error: true
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: jscpd-bucket-${{ github.run_id }}-${{ github.run_attempt }}-18
path: ${{ runner.temp }}/jscpd-buckets/bucket-18.json
retention-days: 31
if-no-files-found: error
- name: Upload jscpd bucket 19
id: bucket-artifact-19
if: always() && steps.split.outcome == 'success'
continue-on-error: true
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: jscpd-bucket-${{ github.run_id }}-${{ github.run_attempt }}-19
path: ${{ runner.temp }}/jscpd-buckets/bucket-19.json
retention-days: 31
if-no-files-found: error