| 6dd74de | | | 1 | /** |
| 6dd74de | | | 2 | * Compares new E2E test timings with previous timings and determines whether to keep the new timings. |
| 6dd74de | | | 3 | * |
| 6dd74de | | | 4 | * The script will: |
| 6dd74de | | | 5 | * 1. Read old timings from git HEAD |
| 6dd74de | | | 6 | * 2. Read new timings from the current file |
| 6dd74de | | | 7 | * 3. Compare the timings and specs |
| 6dd74de | | | 8 | * 4. Keep new timings if: |
| 6dd74de | | | 9 | * - Specs were added/removed |
| 6dd74de | | | 10 | * - Any timing changed by 20% or more |
| 6dd74de | | | 11 | * 5. Revert to old timings if: |
| 6dd74de | | | 12 | * - No significant timing changes |
| 6dd74de | | | 13 | * |
| 6dd74de | | | 14 | * This helps prevent unnecessary timing updates when test performance hasn't changed significantly. |
| 6dd74de | | | 15 | */ |
| 6dd74de | | | 16 | |
| 6dd74de | | | 17 | import fs from 'node:fs'; |
| 6dd74de | | | 18 | import path from 'node:path'; |
| 6dd74de | | | 19 | import { execSync } from 'node:child_process'; |
| 6dd74de | | | 20 | |
| 6dd74de | | | 21 | interface Timing { |
| 6dd74de | | | 22 | spec: string; |
| 6dd74de | | | 23 | duration: number; |
| 6dd74de | | | 24 | } |
| 6dd74de | | | 25 | |
| 6dd74de | | | 26 | interface TimingsFile { |
| 6dd74de | | | 27 | durations: Timing[]; |
| 6dd74de | | | 28 | } |
| 6dd74de | | | 29 | |
| 6dd74de | | | 30 | interface CleanupOptions { |
| 6dd74de | | | 31 | keepNew: boolean; |
| 6dd74de | | | 32 | reason: string; |
| 6dd74de | | | 33 | } |
| 6dd74de | | | 34 | |
| 6dd74de | | | 35 | const TIMINGS_FILE = 'cypress/timings.json'; |
| 6dd74de | | | 36 | const TIMINGS_PATH = path.join(process.cwd(), TIMINGS_FILE); |
| 6dd74de | | | 37 | |
| 6dd74de | | | 38 | function log(message: string): void { |
| 6dd74de | | | 39 | // eslint-disable-next-line no-console |
| 6dd74de | | | 40 | console.log(message); |
| 6dd74de | | | 41 | } |
| 6dd74de | | | 42 | |
| 6dd74de | | | 43 | function readOldTimings(): TimingsFile { |
| 6dd74de | | | 44 | try { |
| 6dd74de | | | 45 | const oldContent = execSync(`git show HEAD:${TIMINGS_FILE}`, { encoding: 'utf8' }); |
| 6dd74de | | | 46 | return JSON.parse(oldContent); |
| 6dd74de | | | 47 | } catch { |
| 6dd74de | | | 48 | log('Error getting old timings, using empty file'); |
| 6dd74de | | | 49 | return { durations: [] }; |
| 6dd74de | | | 50 | } |
| 6dd74de | | | 51 | } |
| 6dd74de | | | 52 | |
| 6dd74de | | | 53 | function readNewTimings(): TimingsFile { |
| 6dd74de | | | 54 | return JSON.parse(fs.readFileSync(TIMINGS_PATH, 'utf8')); |
| 6dd74de | | | 55 | } |
| 6dd74de | | | 56 | |
| 6dd74de | | | 57 | function cleanupFiles({ keepNew, reason }: CleanupOptions): void { |
| 6dd74de | | | 58 | if (keepNew) { |
| 6dd74de | | | 59 | log(`Keeping new timings: ${reason}`); |
| 6dd74de | | | 60 | } else { |
| 6dd74de | | | 61 | log(`Reverting to old timings: ${reason}`); |
| 6dd74de | | | 62 | execSync(`git checkout HEAD -- ${TIMINGS_FILE}`); |
| 6dd74de | | | 63 | } |
| 6dd74de | | | 64 | } |
| 6dd74de | | | 65 | |
| 6dd74de | | | 66 | function compareTimings(): void { |
| 6dd74de | | | 67 | const oldTimings = readOldTimings(); |
| 6dd74de | | | 68 | const newTimings = readNewTimings(); |
| 6dd74de | | | 69 | |
| 6dd74de | | | 70 | const oldSpecs = new Set(oldTimings.durations.map((d) => d.spec)); |
| 6dd74de | | | 71 | const newSpecs = new Set(newTimings.durations.map((d) => d.spec)); |
| 6dd74de | | | 72 | |
| 6dd74de | | | 73 | // Check if specs were added or removed |
| 6dd74de | | | 74 | const addedSpecs = [...newSpecs].filter((spec) => !oldSpecs.has(spec)); |
| 6dd74de | | | 75 | const removedSpecs = [...oldSpecs].filter((spec) => !newSpecs.has(spec)); |
| 6dd74de | | | 76 | |
| 6dd74de | | | 77 | if (addedSpecs.length > 0 || removedSpecs.length > 0) { |
| 6dd74de | | | 78 | log('Specs changed:'); |
| 6dd74de | | | 79 | if (addedSpecs.length > 0) { |
| 6dd74de | | | 80 | log(`Added: ${addedSpecs.join(', ')}`); |
| 6dd74de | | | 81 | } |
| 6dd74de | | | 82 | if (removedSpecs.length > 0) { |
| 6dd74de | | | 83 | log(`Removed: ${removedSpecs.join(', ')}`); |
| 6dd74de | | | 84 | } |
| 6dd74de | | | 85 | return cleanupFiles({ keepNew: true, reason: 'Specs were added or removed' }); |
| 6dd74de | | | 86 | } |
| 6dd74de | | | 87 | |
| 6dd74de | | | 88 | // Check timing variations |
| 6dd74de | | | 89 | const timingChanges = newTimings.durations.map((newTiming) => { |
| 6dd74de | | | 90 | const oldTiming = oldTimings.durations.find((d) => d.spec === newTiming.spec); |
| 6dd74de | | | 91 | if (!oldTiming) { |
| 6dd74de | | | 92 | throw new Error(`Could not find old timing for spec: ${newTiming.spec}`); |
| 6dd74de | | | 93 | } |
| 6dd74de | | | 94 | const change = Math.abs(newTiming.duration - oldTiming.duration); |
| 6dd74de | | | 95 | const changePercent = change / oldTiming.duration; |
| 6dd74de | | | 96 | return { spec: newTiming.spec, change, changePercent }; |
| 6dd74de | | | 97 | }); |
| 6dd74de | | | 98 | |
| 6dd74de | | | 99 | // Filter changes that's more than 5 seconds and 20% different |
| 6dd74de | | | 100 | const significantChanges = timingChanges.filter((t) => t.change > 5000 && t.changePercent >= 0.2); |
| 6dd74de | | | 101 | |
| 6dd74de | | | 102 | if (significantChanges.length === 0) { |
| 6dd74de | | | 103 | log('No significant timing changes detected (threshold: 5s and 20%)'); |
| 6dd74de | | | 104 | return cleanupFiles({ keepNew: false, reason: 'No significant timing changes' }); |
| 6dd74de | | | 105 | } |
| 6dd74de | | | 106 | |
| 6dd74de | | | 107 | log('Significant timing changes:'); |
| 6dd74de | | | 108 | significantChanges.forEach((t) => { |
| 6dd74de | | | 109 | log(`${t.spec}: ${t.change.toFixed(1)}ms (${(t.changePercent * 100).toFixed(1)}%)`); |
| 6dd74de | | | 110 | }); |
| 6dd74de | | | 111 | |
| 6dd74de | | | 112 | cleanupFiles({ keepNew: true, reason: 'Significant timing changes detected' }); |
| 6dd74de | | | 113 | } |
| 6dd74de | | | 114 | |
| 6dd74de | | | 115 | compareTimings(); |