3.5 KB116 lines
Blame
1/**
2 * Compares new E2E test timings with previous timings and determines whether to keep the new timings.
3 *
4 * The script will:
5 * 1. Read old timings from git HEAD
6 * 2. Read new timings from the current file
7 * 3. Compare the timings and specs
8 * 4. Keep new timings if:
9 * - Specs were added/removed
10 * - Any timing changed by 20% or more
11 * 5. Revert to old timings if:
12 * - No significant timing changes
13 *
14 * This helps prevent unnecessary timing updates when test performance hasn't changed significantly.
15 */
16
17import fs from 'node:fs';
18import path from 'node:path';
19import { execSync } from 'node:child_process';
20
21interface Timing {
22 spec: string;
23 duration: number;
24}
25
26interface TimingsFile {
27 durations: Timing[];
28}
29
30interface CleanupOptions {
31 keepNew: boolean;
32 reason: string;
33}
34
35const TIMINGS_FILE = 'cypress/timings.json';
36const TIMINGS_PATH = path.join(process.cwd(), TIMINGS_FILE);
37
38function log(message: string): void {
39 // eslint-disable-next-line no-console
40 console.log(message);
41}
42
43function readOldTimings(): TimingsFile {
44 try {
45 const oldContent = execSync(`git show HEAD:${TIMINGS_FILE}`, { encoding: 'utf8' });
46 return JSON.parse(oldContent);
47 } catch {
48 log('Error getting old timings, using empty file');
49 return { durations: [] };
50 }
51}
52
53function readNewTimings(): TimingsFile {
54 return JSON.parse(fs.readFileSync(TIMINGS_PATH, 'utf8'));
55}
56
57function cleanupFiles({ keepNew, reason }: CleanupOptions): void {
58 if (keepNew) {
59 log(`Keeping new timings: ${reason}`);
60 } else {
61 log(`Reverting to old timings: ${reason}`);
62 execSync(`git checkout HEAD -- ${TIMINGS_FILE}`);
63 }
64}
65
66function compareTimings(): void {
67 const oldTimings = readOldTimings();
68 const newTimings = readNewTimings();
69
70 const oldSpecs = new Set(oldTimings.durations.map((d) => d.spec));
71 const newSpecs = new Set(newTimings.durations.map((d) => d.spec));
72
73 // Check if specs were added or removed
74 const addedSpecs = [...newSpecs].filter((spec) => !oldSpecs.has(spec));
75 const removedSpecs = [...oldSpecs].filter((spec) => !newSpecs.has(spec));
76
77 if (addedSpecs.length > 0 || removedSpecs.length > 0) {
78 log('Specs changed:');
79 if (addedSpecs.length > 0) {
80 log(`Added: ${addedSpecs.join(', ')}`);
81 }
82 if (removedSpecs.length > 0) {
83 log(`Removed: ${removedSpecs.join(', ')}`);
84 }
85 return cleanupFiles({ keepNew: true, reason: 'Specs were added or removed' });
86 }
87
88 // Check timing variations
89 const timingChanges = newTimings.durations.map((newTiming) => {
90 const oldTiming = oldTimings.durations.find((d) => d.spec === newTiming.spec);
91 if (!oldTiming) {
92 throw new Error(`Could not find old timing for spec: ${newTiming.spec}`);
93 }
94 const change = Math.abs(newTiming.duration - oldTiming.duration);
95 const changePercent = change / oldTiming.duration;
96 return { spec: newTiming.spec, change, changePercent };
97 });
98
99 // Filter changes that's more than 5 seconds and 20% different
100 const significantChanges = timingChanges.filter((t) => t.change > 5000 && t.changePercent >= 0.2);
101
102 if (significantChanges.length === 0) {
103 log('No significant timing changes detected (threshold: 5s and 20%)');
104 return cleanupFiles({ keepNew: false, reason: 'No significant timing changes' });
105 }
106
107 log('Significant timing changes:');
108 significantChanges.forEach((t) => {
109 log(`${t.spec}: ${t.change.toFixed(1)}ms (${(t.changePercent * 100).toFixed(1)}%)`);
110 });
111
112 cleanupFiles({ keepNew: true, reason: 'Significant timing changes detected' });
113}
114
115compareTimings();
116