22.0 KB597 lines
Blame
1/**
2 * Copyright (c) Meta Platforms, Inc. and affiliates.
3 *
4 * This source code is licensed under the MIT license found in the
5 * LICENSE file in the root directory of this source tree.
6 */
7
8import type {PageVisibility, ValidatedRepoInfo} from 'isl/src/types';
9import type {PageFocusTracker} from './PageFocusTracker';
10import type {Logger} from './logger';
11
12import fs from 'node:fs/promises';
13import path from 'node:path';
14import {debounce} from 'shared/debounce';
15import {Internal} from './Internal';
16import {stagedThrottler} from './StagedThrottler';
17import type {SubscriptionCallback} from './__generated__/node-edenfs-notifications-client';
18import {EdenFSUtils} from './__generated__/node-edenfs-notifications-client';
19import {type ServerSideTracker} from './analytics/serverSideTracker';
20import {ONE_MINUTE_MS} from './constants';
21import {EdenFSNotifications} from './edenFsNotifications';
22import type {RepositoryContext} from './serverTypes';
23import {Watchman} from './watchman';
24
25const DEFAULT_POLL_INTERVAL = 15 * ONE_MINUTE_MS;
26// When the page is hidden, aggressively reduce polling.
27const HIDDEN_POLL_INTERVAL = 3 * 60 * ONE_MINUTE_MS;
28// When visible or focused, poll frequently
29const VISIBLE_POLL_INTERVAL = 2 * ONE_MINUTE_MS;
30const FOCUSED_POLL_INTERVAL = 0.5 * ONE_MINUTE_MS;
31const ON_FOCUS_REFETCH_THROTTLE = 15_000;
32const ON_VISIBLE_REFETCH_THROTTLE = 30_000;
33
34export type KindOfChange = 'uncommitted changes' | 'commits' | 'merge conflicts' | 'everything';
35export type PollKind = PageVisibility | 'force';
36
37/**
38 * Handles watching for changes to files on disk which should trigger refetching data,
39 * and polling for changes when watching is not reliable.
40 */
41export class WatchForChanges {
42 static WATCHMAN_DEFER = `hg.update`; // TODO: update to sl
43 static WATCHMAN_DEFER_TRANSACTION = `hg.transaction`; // TODO: update to sl
44 public watchman: Watchman;
45 public edenfs: EdenFSNotifications;
46
47 private dirstateDisposables: Array<() => unknown> = [];
48 private watchmanDisposables: Array<() => unknown> = [];
49 private edenfsDisposables: Array<() => unknown> = [];
50 private logger: Logger;
51 private tracker: ServerSideTracker;
52 private dirstateSubscriptionPromise: Promise<void>;
53
54 public onWatchmanStatusChange: ((status: Watchman['status']) => void) | undefined;
55
56 constructor(
57 private repoInfo: ValidatedRepoInfo,
58 private pageFocusTracker: PageFocusTracker,
59 private changeCallback: (kind: KindOfChange, pollKind?: PollKind) => unknown,
60 ctx: RepositoryContext,
61 watchman?: Watchman | undefined,
62 edenfs?: EdenFSNotifications | undefined,
63 ) {
64 this.logger = ctx.logger;
65 this.tracker = ctx.tracker;
66 this.watchman = watchman ?? new Watchman(ctx.logger, ctx.tracker);
67 this.watchman.onStatusChange = status => this.onWatchmanStatusChange?.(status);
68
69 const {repoRoot} = this.repoInfo;
70 this.edenfs = edenfs ?? new EdenFSNotifications(ctx.logger, repoRoot);
71
72 // Watch dirstate right away for commit changes
73 this.dirstateSubscriptionPromise = this.setupDirstateSubscriptions(ctx);
74 this.setupPolling();
75 this.pageFocusTracker.onChange(this.poll.bind(this));
76 // poll right away so we get data immediately, without waiting for timeout on startup
77 this.poll('force');
78 }
79
80 private timeout: NodeJS.Timeout | undefined;
81 private lastFetch = new Date().valueOf();
82
83 /**
84 * Waits for the dirstate subscription to be set up
85 * since we can't await in the constructor
86 * Resolves when dirstateSubscriptionPromise is fulfilled.
87 */
88 public async waitForDirstateSubscriptionReady(): Promise<void> {
89 await this.dirstateSubscriptionPromise;
90 }
91
92 /**
93 * Combine different signals to determine what interval to poll for information
94 */
95 private setupPolling() {
96 this.timeout = setTimeout(this.poll, DEFAULT_POLL_INTERVAL);
97 }
98
99 /**
100 * Re-trigger fetching data from the repository,
101 * depending on how recently that data was last fetched,
102 * and whether any ISL windows are focused or visible.
103 *
104 * This function calls itself on an interval to check whether we should fetch changes,
105 * but it can also be called in response to events like focus being gained.
106 */
107 public poll = (kind?: PollKind) => {
108 // calculate how long we'd like to be waiting from what we know of the windows.
109 let desiredNextTickTime = DEFAULT_POLL_INTERVAL;
110
111 if (this.repoInfo.isEdenFs !== true) {
112 if (this.watchman.status !== 'healthy') {
113 if (this.pageFocusTracker.hasPageWithFocus()) {
114 desiredNextTickTime = FOCUSED_POLL_INTERVAL;
115 } else if (this.pageFocusTracker.hasVisiblePage()) {
116 desiredNextTickTime = VISIBLE_POLL_INTERVAL;
117 }
118 } else {
119 // if watchman is working normally, and we're not visible, don't poll nearly as often
120 if (!this.pageFocusTracker.hasPageWithFocus() && !this.pageFocusTracker.hasVisiblePage()) {
121 desiredNextTickTime = HIDDEN_POLL_INTERVAL;
122 }
123 }
124 } else {
125 // if using eden and we're not visible, don't poll nearly as often
126 if (!this.pageFocusTracker.hasPageWithFocus() && !this.pageFocusTracker.hasVisiblePage()) {
127 desiredNextTickTime = HIDDEN_POLL_INTERVAL;
128 }
129 }
130
131 const now = Date.now();
132 const elapsedTickTime = now - this.lastFetch;
133
134 if (
135 kind === 'force' ||
136 // we've been waiting longer than desired
137 elapsedTickTime >= desiredNextTickTime ||
138 // the moment a window gains focus or visibility, consider polling immediately
139 (kind === 'focused' && elapsedTickTime >= ON_FOCUS_REFETCH_THROTTLE) ||
140 (kind === 'visible' && elapsedTickTime >= ON_VISIBLE_REFETCH_THROTTLE)
141 ) {
142 // it's time to fetch
143 this.changeCallback('everything', kind);
144 this.lastFetch = Date.now();
145
146 clearTimeout(this.timeout);
147 this.timeout = setTimeout(this.poll, desiredNextTickTime);
148 } else {
149 // we have some time left before we we would expect to need to poll, schedule next poll
150 clearTimeout(this.timeout);
151 this.timeout = setTimeout(this.poll, desiredNextTickTime - elapsedTickTime);
152 }
153 };
154
155 private async setupDirstateSubscriptions(ctx: RepositoryContext) {
156 const enabled = await Internal.fetchFeatureFlag?.(ctx, 'isl_use_edenfs_notifications');
157 this.logger.info('dirstate edenfs notifications flag state: ', enabled);
158 if (enabled) {
159 if (this.repoInfo.isEdenFs === true) {
160 this.logger.info('Valid eden repo'); // For testing, remove when implemented
161 await this.setupEdenDirstateSubscriptions();
162 return;
163 } else {
164 this.logger.info('Non-eden repo');
165 await this.setupWatchmanDirstateSubscriptions();
166 }
167 } else {
168 await this.setupWatchmanDirstateSubscriptions();
169 }
170 }
171
172 private async setupWatchmanDirstateSubscriptions() {
173 const {repoRoot, dotdir} = this.repoInfo;
174
175 if (repoRoot == null || dotdir == null) {
176 this.logger.error(`skipping dirstate subscription since ${repoRoot} is not a repository`);
177 return;
178 }
179
180 // Resolve the repo dot dir in case it is a symlink. Watchman doesn't follow symlinks,
181 // so we must follow it and watch the target.
182 const realDotdir = await fs.realpath(dotdir);
183
184 if (realDotdir != dotdir) {
185 this.logger.info(`resolved dotdir ${dotdir} to ${realDotdir}`);
186
187 // Write out ".watchmanconfig" so realDotdir passes muster as a watchman "root dir"
188 // (otherwise watchman will refuse to watch it).
189 await fs.writeFile(path.join(realDotdir, '.watchmanconfig'), '{}');
190 }
191
192 const DIRSTATE_WATCHMAN_SUBSCRIPTION = 'sapling-smartlog-dirstate-change';
193 try {
194 const handleRepositoryStateChange = debounce(() => {
195 // if the repo changes, also recheck files. E.g. if you commit, your uncommitted changes will also change.
196 this.changeCallback('everything');
197
198 // reset timer for polling
199 this.lastFetch = new Date().valueOf();
200 }, 100); // debounce so that multiple quick changes don't trigger multiple fetches for no reason
201
202 this.logger.info('setting up dirstate subscription', realDotdir);
203
204 const dirstateSubscription = await this.watchman.watchDirectoryRecursive(
205 realDotdir,
206 DIRSTATE_WATCHMAN_SUBSCRIPTION,
207 {
208 fields: ['name'],
209 expression: [
210 'name',
211 ['bookmarks.current', 'bookmarks', 'dirstate', 'merge'],
212 'wholename',
213 ],
214 defer: [WatchForChanges.WATCHMAN_DEFER],
215 empty_on_fresh_instance: true,
216 },
217 );
218 dirstateSubscription.emitter.on('change', changes => {
219 if (changes.includes('merge')) {
220 this.changeCallback('merge conflicts');
221 }
222 if (changes.includes('dirstate')) {
223 handleRepositoryStateChange();
224 }
225 });
226 dirstateSubscription.emitter.on('fresh-instance', handleRepositoryStateChange);
227
228 this.dirstateDisposables.push(() => {
229 this.logger.info('unsubscribe dirstate watcher');
230 this.watchman.unwatch(realDotdir, DIRSTATE_WATCHMAN_SUBSCRIPTION);
231 });
232 } catch (err) {
233 this.logger.error('failed to setup dirstate subscriptions', err);
234 this.tracker.error(
235 'WatchmanEvent',
236 'WatchmanError',
237 `failed to setup watchman dirstate subscriptions ${err}`,
238 );
239 }
240 }
241
242 private async setupEdenDirstateSubscriptions() {
243 const {repoRoot, dotdir} = this.repoInfo;
244
245 if (repoRoot == null || dotdir == null) {
246 this.logger.error(`skipping dirstate subscription since ${repoRoot} is not a repository`);
247 return;
248 }
249
250 const relativeRoot = path.relative(repoRoot, dotdir);
251
252 const DIRSTATE_EDENFS_SUBSCRIPTION = 'sapling-smartlog-dirstate-change-edenfs';
253 try {
254 const handleRepositoryStateChange = debounce(() => {
255 // if the repo changes, also recheck files. E.g. if you commit, your uncommitted changes will also change.
256 this.changeCallback('everything');
257
258 // reset timer for polling
259 this.lastFetch = new Date().valueOf();
260 }, 100); // debounce so that multiple quick changes don't trigger multiple fetches for no reason
261
262 this.logger.info(
263 'setting up dirstate edenfs subscription in root',
264 repoRoot,
265 'at',
266 relativeRoot,
267 );
268
269 const subscriptionCallback: SubscriptionCallback = (error, resp) => {
270 if (error) {
271 this.logger.error('EdenFS dirstate subscription error:', error.message);
272 this.tracker.error('EdenWatcherEvent', 'EdenWatcherError', error);
273 return;
274 } else if (resp === null) {
275 // EdenFS subscription closed
276 return;
277 } else {
278 if (resp.changes && resp.changes.length > 0) {
279 resp.changes.forEach(change => {
280 if (change.SmallChange) {
281 const paths = EdenFSUtils.extractPaths([change]);
282 if (paths.includes('merge')) {
283 this.changeCallback('merge conflicts');
284 return;
285 }
286 if (paths.includes('dirstate')) {
287 handleRepositoryStateChange();
288 return;
289 }
290 } else if (change.LargeChange) {
291 handleRepositoryStateChange();
292 return;
293 }
294 });
295 }
296 return;
297 }
298 };
299
300 await this.edenfs.watchDirectoryRecursive(
301 repoRoot,
302 DIRSTATE_EDENFS_SUBSCRIPTION,
303 {
304 useCase: 'isl-server-node',
305 mountPoint: repoRoot,
306 throttle: 100,
307 relativeRoot,
308 deferredStates: [
309 WatchForChanges.WATCHMAN_DEFER,
310 WatchForChanges.WATCHMAN_DEFER_TRANSACTION,
311 ],
312 includeVcsRoots: true,
313 },
314 subscriptionCallback,
315 );
316 this.dirstateDisposables.push(() => {
317 this.logger.info('unsubscribe dirstate edenfs watcher');
318 this.edenfs.unwatch(repoRoot, DIRSTATE_EDENFS_SUBSCRIPTION);
319 });
320 } catch (err) {
321 this.logger.error('failed to setup dirstate edenfs subscriptions', err);
322 this.tracker.error(
323 'EdenWatcherEvent',
324 'EdenWatcherError',
325 `failed to setup dirstate edenfs subscriptions ${err}`,
326 );
327 }
328 }
329
330 public async setupSubscriptions(ctx: RepositoryContext) {
331 await this.waitForDirstateSubscriptionReady();
332 const enabled = await Internal.fetchFeatureFlag?.(ctx, 'isl_use_edenfs_notifications');
333 this.logger.info('subscription edenfs notifications flag state: ', enabled);
334 if (enabled) {
335 if (this.repoInfo.isEdenFs === true) {
336 await this.setupEdenSubscriptions();
337 return;
338 }
339 } else {
340 // TODO: move watchman here after implementing eden
341 }
342 await this.setupWatchmanSubscriptions();
343 }
344
345 /**
346 * Some Watchmans subscriptions should only activate when ISL is actually opened.
347 * On platforms like vscode, it's possible to create a Repository without actually opening ISL.
348 * In those cases, we only want the minimum set of subscriptions to be active.
349 * We care about the dirstate watcher, but not the watchman subscriptions in that case.
350 */
351 public async setupWatchmanSubscriptions() {
352 const {repoRoot, dotdir} = this.repoInfo;
353
354 if (repoRoot == null || dotdir == null) {
355 this.logger.error(`skipping watchman subscription since ${repoRoot} is not a repository`);
356 return;
357 }
358 const relativeDotdir = path.relative(repoRoot, dotdir);
359 // if working from a git clone, the dotdir lives in .git/sl,
360 // but we need to ignore changes in .git in our watchman subscriptions
361 const outerDotDir =
362 relativeDotdir.indexOf(path.sep) >= 0 ? path.dirname(relativeDotdir) : relativeDotdir;
363
364 await this.maybeModifyGitignore(repoRoot, outerDotDir);
365
366 const FILE_CHANGE_WATCHMAN_SUBSCRIPTION = 'sapling-smartlog-file-change';
367 try {
368 // In some bad cases, a file may not be getting ignored by watchman properly,
369 // and ends up constantly triggering the watchman subscription.
370 // Incrementally increase the throttling of events to avoid spamming `status`.
371 // This does mean "legit" changes will start being missed.
372 // TODO: can we scan the list of changes and build a list of files that are overfiring, then send those to the UI as a warning?
373 // This would allow a user to know it's happening and possibly fix it for their repo by adding it to a .watchmanconfig.
374 const handleUncommittedChanges = stagedThrottler(
375 [
376 {
377 throttleMs: 0,
378 numToNextStage: 5,
379 resetAfterMs: 5_000,
380 onEnter: () => {
381 this.logger.info('no longer throttling uncommitted changes');
382 },
383 },
384 {
385 throttleMs: 5_000,
386 numToNextStage: 10,
387 resetAfterMs: 20_000,
388 onEnter: () => {
389 this.logger.info('slightly throttling uncommitted changes');
390 },
391 },
392 {
393 throttleMs: 30_000,
394 resetAfterMs: 30_000,
395 onEnter: () => {
396 this.logger.info('aggressively throttling uncommitted changes');
397 },
398 },
399 ],
400 () => {
401 this.changeCallback('uncommitted changes');
402
403 // reset timer for polling
404 this.lastFetch = new Date().valueOf();
405 },
406 );
407 const uncommittedChangesSubscription = await this.watchman.watchDirectoryRecursive(
408 repoRoot,
409 FILE_CHANGE_WATCHMAN_SUBSCRIPTION,
410 {
411 // We only need to know that a change happened (not the list of files) so that we can trigger `status`
412 fields: ['name'],
413 expression: [
414 'allof',
415 // This watchman subscription is used to determine when and which
416 // files to fetch new statuses for. There is no reason to include
417 // directories in these updates, and in fact they may make us overfetch
418 // statuses.
419 // This line restricts this subscription to only return files.
420 ['type', 'f'],
421 ['not', ['dirname', outerDotDir]],
422 // Even though we tell it not to match .sl, modifying a file inside .sl
423 // will emit an event for the folder itself, which we want to ignore.
424 ['not', ['match', outerDotDir, 'basename']],
425 // Exclude edenfs notifications directory - EdenFS notifications can modify hidden files
426 // in this directory which would otherwise trigger watchman events.
427 ['not', ['dirname', '.edenfs-notifications-state']],
428 ['not', ['match', '.edenfs-notifications-state', 'basename']],
429 ],
430 defer: [WatchForChanges.WATCHMAN_DEFER],
431 empty_on_fresh_instance: true,
432 },
433 );
434 uncommittedChangesSubscription.emitter.on('change', handleUncommittedChanges);
435 uncommittedChangesSubscription.emitter.on('fresh-instance', handleUncommittedChanges);
436
437 this.watchmanDisposables.push(() => {
438 this.logger.info('unsubscribe watchman');
439 this.watchman.unwatch(repoRoot, FILE_CHANGE_WATCHMAN_SUBSCRIPTION);
440 });
441 } catch (err) {
442 this.logger.error('failed to setup watchman subscriptions', err);
443 this.tracker.error(
444 'WatchmanEvent',
445 'WatchmanError',
446 `failed to setup watchman subscriptions ${err}`,
447 );
448 }
449 }
450
451 public async setupEdenSubscriptions() {
452 const {repoRoot, dotdir} = this.repoInfo;
453
454 if (repoRoot == null || dotdir == null) {
455 this.logger.error(`skipping edenfs subscription since ${repoRoot} is not a repository`);
456 return;
457 }
458 const relativeDotdir = path.relative(repoRoot, dotdir);
459 // if working from a git clone, the dotdir lives in .git/sl,
460 // but we need to ignore changes in .git in our watchman subscriptions
461 const outerDotDir =
462 relativeDotdir.indexOf(path.sep) >= 0 ? path.dirname(relativeDotdir) : relativeDotdir;
463
464 this.logger.info(
465 'setting up edenfs subscription in',
466 repoRoot,
467 'at',
468 outerDotDir,
469 'relativeDotdir',
470 relativeDotdir,
471 );
472
473 const FILE_CHANGE_EDENFS_SUBSCRIPTION = 'sapling-smartlog-file-change-edenfs';
474 try {
475 // In some bad cases, a file that has a lot of activity can constantly trigger the subscription.
476 // Incrementally increase the throttling of events to avoid spamming `status`.
477 // This does mean "legit" changes will start being missed.
478 // TODO: can we scan the list of changes and build a list of files that are overfiring, then send those to the UI as a warning?
479 // This would allow a user to know it's happening and possibly fix it for their repo.
480 const handleUncommittedChanges = stagedThrottler(
481 [
482 {
483 throttleMs: 0,
484 numToNextStage: 5,
485 resetAfterMs: 5_000,
486 onEnter: () => {
487 this.logger.info('no longer throttling uncommitted changes');
488 },
489 },
490 {
491 throttleMs: 5_000,
492 numToNextStage: 10,
493 resetAfterMs: 20_000,
494 onEnter: () => {
495 this.logger.info('slightly throttling uncommitted changes');
496 },
497 },
498 {
499 throttleMs: 30_000,
500 resetAfterMs: 30_000,
501 onEnter: () => {
502 this.logger.info('aggressively throttling uncommitted changes');
503 },
504 },
505 ],
506 () => {
507 this.changeCallback('uncommitted changes');
508
509 // reset timer for polling
510 this.lastFetch = new Date().valueOf();
511 },
512 );
513 const subscriptionCallback: SubscriptionCallback = (error, resp) => {
514 if (error) {
515 this.logger.error('EdenFS subscription error:', error.message);
516 this.tracker.error('EdenWatcherEvent', 'EdenWatcherError', error);
517 return;
518 } else if (resp === null) {
519 // EdenFS subscription closed
520 return;
521 } else {
522 if (resp.changes && resp.changes.length > 0) {
523 handleUncommittedChanges();
524 }
525 }
526 };
527 await this.edenfs.watchDirectoryRecursive(
528 repoRoot,
529 FILE_CHANGE_EDENFS_SUBSCRIPTION,
530 {
531 useCase: 'isl-server-node',
532 mountPoint: repoRoot,
533 throttle: 100,
534 deferredStates: [
535 WatchForChanges.WATCHMAN_DEFER,
536 WatchForChanges.WATCHMAN_DEFER_TRANSACTION,
537 ],
538 excludedRoots: [outerDotDir, relativeDotdir],
539 },
540 subscriptionCallback,
541 );
542
543 this.edenfsDisposables.push(() => {
544 this.logger.info('unsubscribe edenfs');
545 this.edenfs.unwatch(repoRoot, FILE_CHANGE_EDENFS_SUBSCRIPTION);
546 });
547 } catch (err) {
548 this.logger.error('failed to setup edenfs subscriptions', err);
549 this.tracker.error(
550 'EdenWatcherEvent',
551 'EdenWatcherError',
552 `failed to setup edenfs subscriptions ${err}`,
553 );
554 }
555 }
556
557 /**
558 * Modify gitignore to ignore watchman cookie files. This is needed when using ISL
559 * with git repos. `git status` does not exclude watchman cookie files by default.
560 * `sl` does not use watchman in dotgit mode.
561 */
562 private async maybeModifyGitignore(repoRoot: string, outerDotDir: string) {
563 if (outerDotDir !== '.git') {
564 return;
565 }
566 const gitIgnorePath = path.join(repoRoot, outerDotDir, 'info', 'exclude');
567 // https://github.com/facebook/watchman/blob/76bd924b1169dae9cb9f5371845ab44ea1f836bf/watchman/Cookie.h#L15
568 const rule = '/.watchman-cookie-*';
569 try {
570 const gitIgnoreContent = await fs.readFile(gitIgnorePath, 'utf8');
571 if (!gitIgnoreContent.includes(rule)) {
572 await fs.appendFile(gitIgnorePath, `\n${rule}\n`, 'utf8');
573 }
574 } catch (err) {
575 this.logger.error(`failed to read or write ${gitIgnorePath}`, err);
576 }
577 }
578
579 public disposeWatchmanSubscriptions() {
580 this.watchmanDisposables.forEach(dispose => dispose());
581 }
582
583 public disposeEdenFSSubscriptions() {
584 this.edenfsDisposables.forEach(dispose => dispose());
585 }
586
587 public dispose() {
588 this.dirstateDisposables.forEach(dispose => dispose());
589 this.disposeWatchmanSubscriptions();
590 this.disposeEdenFSSubscriptions();
591 if (this.timeout) {
592 clearTimeout(this.timeout);
593 this.timeout = undefined;
594 }
595 }
596}
597