| b69ab31 | | | 1 | /** |
| b69ab31 | | | 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. |
| b69ab31 | | | 3 | * |
| b69ab31 | | | 4 | * This source code is licensed under the MIT license found in the |
| b69ab31 | | | 5 | * LICENSE file in the root directory of this source tree. |
| b69ab31 | | | 6 | */ |
| b69ab31 | | | 7 | |
| b69ab31 | | | 8 | import type {IGrammar} from 'vscode-textmate'; |
| b69ab31 | | | 9 | import type {ThemeColor} from '../../theme'; |
| b69ab31 | | | 10 | import type { |
| b69ab31 | | | 11 | SyntaxWorkerRequest, |
| b69ab31 | | | 12 | SyntaxWorkerResponse, |
| b69ab31 | | | 13 | TokenizedDiffHunks, |
| b69ab31 | | | 14 | TokenizedHunk, |
| b69ab31 | | | 15 | } from './syntaxHighlightingTypes'; |
| b69ab31 | | | 16 | |
| b69ab31 | | | 17 | import {CancellationToken} from 'shared/CancellationToken'; |
| b69ab31 | | | 18 | import FilepathClassifier from 'shared/textmate-lib/FilepathClassifier'; |
| b69ab31 | | | 19 | import {tokenizeLines} from 'shared/textmate-lib/tokenize'; |
| b69ab31 | | | 20 | import {loadWASM} from 'vscode-oniguruma'; |
| b69ab31 | | | 21 | import {grammars, languages} from '../../generated/textmate/TextMateGrammarManifest'; |
| b69ab31 | | | 22 | import {getGrammar, getGrammarStore} from './grammar'; |
| b69ab31 | | | 23 | |
| b69ab31 | | | 24 | const URL_TO_ONIG_WASM = './generated/textmate/onig.wasm'; |
| b69ab31 | | | 25 | |
| b69ab31 | | | 26 | /* This file is intended to be executed in a WebWorker, without access to the DOM. */ |
| b69ab31 | | | 27 | |
| b69ab31 | | | 28 | /** |
| b69ab31 | | | 29 | * Fetch requests inside the webworker must be made relative to the base URI. |
| b69ab31 | | | 30 | * By executing the web worker via a blob: URL, we can't depend on the base being inherited. |
| b69ab31 | | | 31 | */ |
| b69ab31 | | | 32 | let globalBaseUri: string; |
| b69ab31 | | | 33 | |
| b69ab31 | | | 34 | async function loadGrammar( |
| b69ab31 | | | 35 | theme: ThemeColor, |
| b69ab31 | | | 36 | path: string, |
| b69ab31 | | | 37 | postMessage: (msg: SyntaxWorkerResponse) => void, |
| b69ab31 | | | 38 | ): Promise<IGrammar | undefined> { |
| b69ab31 | | | 39 | await ensureOnigurumaIsLoaded(globalBaseUri); |
| b69ab31 | | | 40 | |
| b69ab31 | | | 41 | const scopeName = getFilepathClassifier().findScopeNameForPath(path); |
| b69ab31 | | | 42 | if (!scopeName) { |
| b69ab31 | | | 43 | return undefined; |
| b69ab31 | | | 44 | } |
| b69ab31 | | | 45 | |
| b69ab31 | | | 46 | const store = getGrammarStore(theme, globalBaseUri, colorMap => { |
| b69ab31 | | | 47 | // tell client the newest colorMap |
| b69ab31 | | | 48 | postMessage({type: 'cssColorMap', colorMap} as SyntaxWorkerResponse); |
| b69ab31 | | | 49 | }); |
| b69ab31 | | | 50 | |
| b69ab31 | | | 51 | const grammar = await getGrammar(store, scopeName); |
| b69ab31 | | | 52 | return grammar ?? undefined; |
| b69ab31 | | | 53 | } |
| b69ab31 | | | 54 | |
| b69ab31 | | | 55 | const cancellationTokenForId = new Map<number, CancellationToken>(); |
| b69ab31 | | | 56 | |
| b69ab31 | | | 57 | class WorkQueue { |
| b69ab31 | | | 58 | private queue: Array<() => Promise<void>> = []; |
| b69ab31 | | | 59 | private isProcessing = false; |
| b69ab31 | | | 60 | |
| b69ab31 | | | 61 | public push(work: () => Promise<void>) { |
| b69ab31 | | | 62 | this.queue.push(work); |
| b69ab31 | | | 63 | |
| b69ab31 | | | 64 | if (!this.isProcessing) { |
| b69ab31 | | | 65 | this.processNext(); |
| b69ab31 | | | 66 | } |
| b69ab31 | | | 67 | } |
| b69ab31 | | | 68 | |
| b69ab31 | | | 69 | private async processNext() { |
| b69ab31 | | | 70 | if (this.queue.length > 0) { |
| b69ab31 | | | 71 | const work = this.queue.shift(); |
| b69ab31 | | | 72 | this.isProcessing = true; |
| b69ab31 | | | 73 | // Allow the task queue to be emptied before continuing, |
| b69ab31 | | | 74 | // so we can process cancel messages |
| b69ab31 | | | 75 | await new Promise(res => setTimeout(res, 0)); |
| b69ab31 | | | 76 | await work?.().catch(err => { |
| b69ab31 | | | 77 | // eslint-disable-next-line no-console |
| b69ab31 | | | 78 | console.error(err); |
| b69ab31 | | | 79 | return null; |
| b69ab31 | | | 80 | }); |
| b69ab31 | | | 81 | this.isProcessing = false; |
| b69ab31 | | | 82 | this.processNext(); |
| b69ab31 | | | 83 | } |
| b69ab31 | | | 84 | } |
| b69ab31 | | | 85 | } |
| b69ab31 | | | 86 | |
| b69ab31 | | | 87 | const workQueue = new WorkQueue(); |
| b69ab31 | | | 88 | |
| b69ab31 | | | 89 | export function handleMessage( |
| b69ab31 | | | 90 | postMessage: (msg: SyntaxWorkerResponse & {id?: number}) => unknown, |
| b69ab31 | | | 91 | event: MessageEvent, |
| b69ab31 | | | 92 | ) { |
| b69ab31 | | | 93 | const data = event.data as SyntaxWorkerRequest & {id: number}; |
| b69ab31 | | | 94 | |
| b69ab31 | | | 95 | const token = new CancellationToken(); |
| b69ab31 | | | 96 | if (data.id != null) { |
| b69ab31 | | | 97 | cancellationTokenForId.set(data.id, token); |
| b69ab31 | | | 98 | } |
| b69ab31 | | | 99 | switch (data.type) { |
| b69ab31 | | | 100 | case 'setBaseUri': { |
| b69ab31 | | | 101 | globalBaseUri = data.base; |
| b69ab31 | | | 102 | break; |
| b69ab31 | | | 103 | } |
| b69ab31 | | | 104 | case 'tokenizeContents': { |
| b69ab31 | | | 105 | workQueue.push(async () => { |
| b69ab31 | | | 106 | const grammar = await loadGrammar(data.theme, data.path, postMessage); |
| b69ab31 | | | 107 | const result = tokenizeContent(grammar, data.content, token); |
| b69ab31 | | | 108 | postMessage({type: data.type, id: data.id, result}); |
| b69ab31 | | | 109 | cancellationTokenForId.delete(data.id); |
| b69ab31 | | | 110 | }); |
| b69ab31 | | | 111 | break; |
| b69ab31 | | | 112 | } |
| b69ab31 | | | 113 | case 'tokenizeHunks': { |
| b69ab31 | | | 114 | workQueue.push(async () => { |
| b69ab31 | | | 115 | const grammar = await loadGrammar(data.theme, data.path, postMessage); |
| b69ab31 | | | 116 | const result = tokenizeHunks(grammar, data.hunks, token); |
| b69ab31 | | | 117 | postMessage({type: data.type, id: data.id, result}); |
| b69ab31 | | | 118 | cancellationTokenForId.delete(data.id); |
| b69ab31 | | | 119 | }); |
| b69ab31 | | | 120 | break; |
| b69ab31 | | | 121 | } |
| b69ab31 | | | 122 | case 'cancel': { |
| b69ab31 | | | 123 | const token = cancellationTokenForId.get(data.idToCancel); |
| b69ab31 | | | 124 | token?.cancel(); |
| b69ab31 | | | 125 | } |
| b69ab31 | | | 126 | } |
| b69ab31 | | | 127 | } |
| b69ab31 | | | 128 | |
| b69ab31 | | | 129 | if (typeof self.document === 'undefined') { |
| b69ab31 | | | 130 | // inside WebWorker, use global onmessage and postMessage |
| b69ab31 | | | 131 | onmessage = handleMessage.bind(undefined, postMessage); |
| b69ab31 | | | 132 | // outside of a WebWorker, the exported `handleMessage` function should be used instead. |
| b69ab31 | | | 133 | } |
| b69ab31 | | | 134 | |
| b69ab31 | | | 135 | function tokenizeHunks( |
| b69ab31 | | | 136 | grammar: IGrammar | undefined, |
| b69ab31 | | | 137 | hunks: Array<{lines: Array<string>}>, |
| b69ab31 | | | 138 | cancellationToken: CancellationToken, |
| b69ab31 | | | 139 | ): TokenizedDiffHunks | undefined { |
| b69ab31 | | | 140 | if (grammar == null) { |
| b69ab31 | | | 141 | return undefined; |
| b69ab31 | | | 142 | } |
| b69ab31 | | | 143 | |
| b69ab31 | | | 144 | if (cancellationToken.isCancelled) { |
| b69ab31 | | | 145 | // check for cancellation before doing expensive highlighting |
| b69ab31 | | | 146 | return undefined; |
| b69ab31 | | | 147 | } |
| b69ab31 | | | 148 | |
| b69ab31 | | | 149 | const tokenizedPatches: TokenizedDiffHunks = hunks |
| b69ab31 | | | 150 | .map(hunk => recoverFileContentsFromPatchLines(hunk.lines)) |
| b69ab31 | | | 151 | .map(([before, after]) => [tokenizeLines(before, grammar), tokenizeLines(after, grammar)]); |
| b69ab31 | | | 152 | |
| b69ab31 | | | 153 | return tokenizedPatches; |
| b69ab31 | | | 154 | } |
| b69ab31 | | | 155 | |
| b69ab31 | | | 156 | function tokenizeContent( |
| b69ab31 | | | 157 | grammar: IGrammar | undefined, |
| b69ab31 | | | 158 | content: Array<string>, |
| b69ab31 | | | 159 | cancellationToken: CancellationToken, |
| b69ab31 | | | 160 | ): TokenizedHunk | undefined { |
| b69ab31 | | | 161 | if (grammar == null) { |
| b69ab31 | | | 162 | return undefined; |
| b69ab31 | | | 163 | } |
| b69ab31 | | | 164 | |
| b69ab31 | | | 165 | if (cancellationToken.isCancelled) { |
| b69ab31 | | | 166 | // check for cancellation before doing expensive highlighting |
| b69ab31 | | | 167 | return undefined; |
| b69ab31 | | | 168 | } |
| b69ab31 | | | 169 | |
| b69ab31 | | | 170 | return tokenizeLines(content, grammar); |
| b69ab31 | | | 171 | } |
| b69ab31 | | | 172 | |
| b69ab31 | | | 173 | /** |
| b69ab31 | | | 174 | * Patch lines start with ' ', '+', or '-'. From this we can reconstruct before & after file contents as strings, |
| b69ab31 | | | 175 | * which we can actually use in the syntax highlighting. |
| b69ab31 | | | 176 | */ |
| b69ab31 | | | 177 | function recoverFileContentsFromPatchLines( |
| b69ab31 | | | 178 | lines: Array<string>, |
| b69ab31 | | | 179 | ): [before: Array<string>, after: Array<string>] { |
| b69ab31 | | | 180 | const linesBefore = []; |
| b69ab31 | | | 181 | const linesAfter = []; |
| b69ab31 | | | 182 | for (const line of lines) { |
| b69ab31 | | | 183 | if (line[0] === ' ') { |
| b69ab31 | | | 184 | linesBefore.push(line.slice(1)); |
| b69ab31 | | | 185 | linesAfter.push(line.slice(1)); |
| b69ab31 | | | 186 | } else if (line[0] === '+') { |
| b69ab31 | | | 187 | linesAfter.push(line.slice(1)); |
| b69ab31 | | | 188 | } else if (line[0] === '-') { |
| b69ab31 | | | 189 | linesBefore.push(line.slice(1)); |
| b69ab31 | | | 190 | } |
| b69ab31 | | | 191 | } |
| b69ab31 | | | 192 | |
| b69ab31 | | | 193 | return [linesBefore, linesAfter]; |
| b69ab31 | | | 194 | } |
| b69ab31 | | | 195 | |
| b69ab31 | | | 196 | let onigurumaLoadingJob: Promise<void> | null = null; |
| b69ab31 | | | 197 | function ensureOnigurumaIsLoaded(base: string): Promise<void> { |
| b69ab31 | | | 198 | if (onigurumaLoadingJob === null) { |
| b69ab31 | | | 199 | onigurumaLoadingJob = loadOniguruma(base); |
| b69ab31 | | | 200 | } |
| b69ab31 | | | 201 | return onigurumaLoadingJob; |
| b69ab31 | | | 202 | } |
| b69ab31 | | | 203 | |
| b69ab31 | | | 204 | async function loadOniguruma(base: string): Promise<void> { |
| b69ab31 | | | 205 | const url = new URL(URL_TO_ONIG_WASM, base); |
| b69ab31 | | | 206 | const onigurumaWASMRequest = fetch(url); |
| b69ab31 | | | 207 | const response = await onigurumaWASMRequest; |
| b69ab31 | | | 208 | |
| b69ab31 | | | 209 | const contentType = response.headers.get('content-type'); |
| b69ab31 | | | 210 | const useStreamingParser = contentType === 'application/wasm'; |
| b69ab31 | | | 211 | |
| b69ab31 | | | 212 | if (useStreamingParser) { |
| b69ab31 | | | 213 | await loadWASM(response); |
| b69ab31 | | | 214 | } else { |
| b69ab31 | | | 215 | const dataOrOptions = { |
| b69ab31 | | | 216 | data: await response.arrayBuffer(), |
| b69ab31 | | | 217 | }; |
| b69ab31 | | | 218 | await loadWASM(dataOrOptions); |
| b69ab31 | | | 219 | } |
| b69ab31 | | | 220 | } |
| b69ab31 | | | 221 | |
| b69ab31 | | | 222 | let _classifier: FilepathClassifier | null = null; |
| b69ab31 | | | 223 | |
| b69ab31 | | | 224 | function getFilepathClassifier(): FilepathClassifier { |
| b69ab31 | | | 225 | if (_classifier == null) { |
| b69ab31 | | | 226 | _classifier = new FilepathClassifier(grammars, languages); |
| b69ab31 | | | 227 | } |
| b69ab31 | | | 228 | return _classifier; |
| b69ab31 | | | 229 | } |