pax_global_header00006660000000000000000000000064145571512210014515gustar00rootroot0000000000000052 comment=3c9ad8ffee32e158512d252fee9457a0e6469e2e language-6.10.1/000077500000000000000000000000001455715122100133655ustar00rootroot00000000000000language-6.10.1/.github/000077500000000000000000000000001455715122100147255ustar00rootroot00000000000000language-6.10.1/.github/workflows/000077500000000000000000000000001455715122100167625ustar00rootroot00000000000000language-6.10.1/.github/workflows/dispatch.yml000066400000000000000000000006371455715122100213120ustar00rootroot00000000000000name: Trigger CI on: push jobs: build: name: Dispatch to main repo runs-on: ubuntu-latest steps: - name: Emit repository_dispatch uses: mvasigh/dispatch-action@main with: # You should create a personal access token and store it in your repository token: ${{ secrets.DISPATCH_AUTH }} repo: dev owner: codemirror event_type: push language-6.10.1/.gitignore000066400000000000000000000001271455715122100153550ustar00rootroot00000000000000/node_modules package-lock.json /dist /test/*.js /test/*.d.ts /test/*.d.ts.map .tern-* language-6.10.1/.npmignore000066400000000000000000000001001455715122100153530ustar00rootroot00000000000000/src /test /node_modules .tern-* rollup.config.js tsconfig.json language-6.10.1/CHANGELOG.md000066400000000000000000000223151455715122100152010ustar00rootroot00000000000000## 6.10.1 (2024-02-02) ### Bug fixes Fix an issue where, when a lot of code is visible in the initial editor, the bottom bit of code is shown without highlighting for one frame. ## 6.10.0 (2023-12-28) ### New features The new `bidiIsolates` extension can be used to wrap syntactic elements where this is appropriate in an element that isolates their text direction, avoiding weird ordering of neutral characters on direction boundaries. ## 6.9.3 (2023-11-27) ### Bug fixes Fix an issue in `StreamLanguage` where it ran out of node type ids if you repeatedly redefined a language with the same token table. ## 6.9.2 (2023-10-24) ### Bug fixes Allow `StreamParser` tokens get multiple highlighting tags. ## 6.9.1 (2023-09-20) ### Bug fixes Indentation now works a lot better in mixed-language documents that interleave the languages in a complex way. Code folding is now able to pick the right foldable syntax node when the line end falls in a mixed-parsing language that doesn't match the target node. ## 6.9.0 (2023-08-16) ### Bug fixes Make `getIndentation` return null, rather than 0, when there is no syntax tree available. ### New features The new `preparePlaceholder` option to `codeFolding` makes it possible to display contextual information in a folded range placeholder widget. ## 6.8.0 (2023-06-12) ### New features The new `baseIndentFor` method in `TreeIndentContext` can be used to find the base indentation for an arbitrary node. ## 6.7.0 (2023-05-19) ### New features Export `DocInput` class for feeding editor documents to a Lezer parser. ## 6.6.0 (2023-02-13) ### New features Syntax-driven language data queries now support sublanguages, which make it possible to return different data for specific parts of the tree produced by a single language. ## 6.5.0 (2023-02-07) ### Bug fixes Make indentation for stream languages more reliable by having `StringStream.indentation` return overridden indentations from the indent context. ### New features The `toggleFold` command folds or unfolds depending on whether there's an existing folded range on the current line. `indentUnit` now accepts any (repeated) whitespace character, not just spaces and tabs. ## 6.4.0 (2023-01-12) ### New features The `bracketMatchingHandle` node prop can now be used to limit bracket matching behavior for larger nodes to a single subnode (for example the tag name of an HTML tag). ## 6.3.2 (2022-12-16) ### Bug fixes Fix a bug that caused `ensureSyntaxTree` to return incomplete trees when using a viewport-aware parser like `StreamLanguage`. ## 6.3.1 (2022-11-14) ### Bug fixes Make syntax-based folding include syntax nodes that start right at the end of a line as potential fold targets. Fix the `indentService` protocol to allow a distinction between declining to handle the indentation and returning null to indicate the line has no definite indentation. ## 6.3.0 (2022-10-24) ### New features `HighlightStyle` objects now have a `specs` property holding the tag styles that were used to define them. `Language` objects now have a `name` field holding the language name. ## 6.2.1 (2022-07-21) ### Bug fixes Fix a bug where `bracketMatching` would incorrectly match nested brackets in syntax trees that put multiple pairs of brackets in the same parent node. Fix a bug that could cause `indentRange` to loop infinitely. ## 6.2.0 (2022-06-30) ### Bug fixes Fix a bug that prevented bracket matching to recognize plain brackets inside a language parsed as an overlay. ### New features The `indentRange` function provides an easy way to programatically auto-indent a range of the document. ## 6.1.0 (2022-06-20) ### New features The `foldState` field is now public, and can be used to serialize and deserialize the fold state. ## 6.0.0 (2022-06-08) ### New features The `foldingChanged` option to `foldGutter` can now be used to trigger a recomputation of the fold markers. ## 0.20.2 (2022-05-20) ### Bug fixes List style-mod as a dependency. ## 0.20.1 (2022-05-18) ### Bug fixes Make sure `all` styles in the CSS generated for a `HighlightStyle` have a lower precedence than the other rules defined for the style. Use a shorthand property ## 0.20.0 (2022-04-20) ### Breaking changes `HighlightStyle.get` is now called `highlightingFor`. `HighlightStyles` no longer function as extensions (to improve tree shaking), and must be wrapped with `syntaxHighlighting` to add to an editor configuration. `Language` objects no longer have a `topNode` property. ### New features `HighlightStyle` and `defaultHighlightStyle` from the now-removed @codemirror/highlight package now live in this package. The new `forceParsing` function can be used to run the parser forward on an editor view. The exports that used to live in @codemirror/matchbrackets are now exported from this package. The @codemirror/fold package has been merged into this one. The exports from the old @codemirror/stream-parser package now live in this package. ## 0.19.10 (2022-03-31) ### Bug fixes Autocompletion may now also trigger automatic indentation on input. ## 0.19.9 (2022-03-30) ### Bug fixes Make sure nodes that end at the end of a partial parse aren't treated as valid fold targets. Fix an issue where the parser sometimes wouldn't reuse parsing work done in the background on transactions. ## 0.19.8 (2022-03-03) ### Bug fixes Fix an issue that could cause indentation logic to use the wrong line content when indenting multiple lines at once. ## 0.19.7 (2021-12-02) ### Bug fixes Fix an issue where the parse worker could incorrectly stop working when the parse tree has skipped gaps in it. ## 0.19.6 (2021-11-26) ### Bug fixes Fixes an issue where the background parse work would be scheduled too aggressively, degrading responsiveness on a newly-created editor with a large document. Improve initial highlight for mixed-language editors and limit the amount of parsing done on state creation for faster startup. ## 0.19.5 (2021-11-17) ### New features The new function `syntaxTreeAvailable` can be used to check if a fully-parsed syntax tree is available up to a given document position. The module now exports `syntaxParserRunning`, which tells you whether the background parser is still planning to do more work for a given editor view. ## 0.19.4 (2021-11-13) ### New features `LanguageDescription.of` now takes an optional already-loaded extension. ## 0.19.3 (2021-09-13) ### Bug fixes Fix an issue where a parse that skipped content with `skipUntilInView` would in some cases not be restarted when the range came into view. ## 0.19.2 (2021-08-11) ### Bug fixes Fix a bug that caused `indentOnInput` to fire for the wrong kinds of transactions. Fix a bug that could cause `indentOnInput` to apply its changes incorrectly. ## 0.19.1 (2021-08-11) ### Bug fixes Fix incorrect versions for @lezer dependencies. ## 0.19.0 (2021-08-11) ### Breaking changes CodeMirror now uses lezer 0.15, which means different package names (scoped with @lezer) and some breaking changes in the library. `EditorParseContext` is now called `ParseContext`. It is no longer passed to parsers, but must be retrieved with `ParseContext.get`. `IndentContext.lineIndent` now takes a position, not a `Line` object, as argument. `LezerLanguage` was renamed to `LRLanguage` (because all languages must emit Lezer-style trees, the name was misleading). `Language.parseString` no longer exists. You can just call `.parser.parse(...)` instead. ### New features New `IndentContext.lineAt` method to access lines in a way that is aware of simulated line breaks. `IndentContext` now provides a `simulatedBreak` property through which client code can query whether the context has a simulated line break. ## 0.18.2 (2021-06-01) ### Bug fixes Fix an issue where asynchronous re-parsing (with dynamically loaded languages) sometimes failed to fully happen. ## 0.18.1 (2021-03-31) ### Breaking changes `EditorParseContext.getSkippingParser` now replaces `EditorParseContext.skippingParser` and allows you to provide a promise that'll cause parsing to start again. (The old property remains available until the next major release.) ### Bug fixes Fix an issue where nested parsers could see past the end of the nested region. ## 0.18.0 (2021-03-03) ### Breaking changes Update dependencies to 0.18. ### Breaking changes The `Language` constructor takes an additional argument that provides the top node type. ### New features `Language` instances now have a `topNode` property giving their top node type. `TreeIndentContext` now has a `continue` method that allows an indenter to defer to the indentation of the parent nodes. ## 0.17.5 (2021-02-19) ### New features This package now exports a `foldInside` helper function, a fold function that should work for most delimited node types. ## 0.17.4 (2021-01-15) ## 0.17.3 (2021-01-15) ### Bug fixes Parse scheduling has been improved to reduce the likelyhood of the user looking at unparsed code in big documents. Prevent parser from running too far past the current viewport in huge documents. ## 0.17.2 (2021-01-06) ### New features The package now also exports a CommonJS module. ## 0.17.1 (2020-12-30) ### Bug fixes Fix a bug where changing the editor configuration wouldn't update the language parser used. ## 0.17.0 (2020-12-29) ### Breaking changes First numbered release. language-6.10.1/LICENSE000066400000000000000000000021361455715122100143740ustar00rootroot00000000000000MIT License Copyright (C) 2018-2021 by Marijn Haverbeke and others Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. language-6.10.1/README.md000066400000000000000000000020211455715122100146370ustar00rootroot00000000000000# @codemirror/language [![NPM version](https://img.shields.io/npm/v/@codemirror/language.svg)](https://www.npmjs.org/package/@codemirror/language) [ [**WEBSITE**](https://codemirror.net/) | [**DOCS**](https://codemirror.net/docs/ref/#language) | [**ISSUES**](https://github.com/codemirror/dev/issues) | [**FORUM**](https://discuss.codemirror.net/c/next/) | [**CHANGELOG**](https://github.com/codemirror/language/blob/main/CHANGELOG.md) ] This package implements the language support infrastructure for the [CodeMirror](https://codemirror.net/) code editor. The [project page](https://codemirror.net/) has more information, a number of [examples](https://codemirror.net/examples/) and the [documentation](https://codemirror.net/docs/). This code is released under an [MIT license](https://github.com/codemirror/language/tree/main/LICENSE). We aim to be an inclusive, welcoming community. To make that explicit, we have a [code of conduct](http://contributor-covenant.org/version/1/1/0/) that applies to communication around the project. language-6.10.1/package.json000066400000000000000000000020471455715122100156560ustar00rootroot00000000000000{ "name": "@codemirror/language", "version": "6.10.1", "description": "Language support infrastructure for the CodeMirror code editor", "scripts": { "test": "cm-runtests", "prepare": "cm-buildhelper src/index.ts" }, "keywords": [ "editor", "code" ], "author": { "name": "Marijn Haverbeke", "email": "marijn@haverbeke.berlin", "url": "http://marijnhaverbeke.nl" }, "type": "module", "main": "dist/index.cjs", "exports": { "import": "./dist/index.js", "require": "./dist/index.cjs" }, "types": "dist/index.d.ts", "module": "dist/index.js", "sideEffects": false, "license": "MIT", "dependencies": { "@codemirror/state": "^6.0.0", "@codemirror/view": "^6.23.0", "@lezer/common": "^1.1.0", "@lezer/highlight": "^1.0.0", "@lezer/lr": "^1.0.0", "style-mod": "^4.0.0" }, "devDependencies": { "@codemirror/buildhelper": "^1.0.0", "@lezer/javascript": "^1.0.0" }, "repository": { "type": "git", "url": "https://github.com/codemirror/language.git" } } language-6.10.1/src/000077500000000000000000000000001455715122100141545ustar00rootroot00000000000000language-6.10.1/src/README.md000066400000000000000000000025671455715122100154450ustar00rootroot00000000000000@languageDataProp @Language @defineLanguageFacet @Sublanguage @sublanguageProp @language @LRLanguage @ParseContext @syntaxTree @ensureSyntaxTree @syntaxTreeAvailable @forceParsing @syntaxParserRunning @LanguageSupport @LanguageDescription @DocInput ### Highlighting @HighlightStyle @syntaxHighlighting @TagStyle @defaultHighlightStyle @highlightingFor @bidiIsolates ### Folding These exports provide commands and other functionality related to code folding (temporarily hiding pieces of code). @foldService @foldNodeProp @foldInside @foldable @foldCode @unfoldCode @toggleFold @foldAll @unfoldAll @foldKeymap @codeFolding @foldGutter The following functions provide more direct, low-level control over the fold state. @foldedRanges @foldState @foldEffect @unfoldEffect ### Indentation @indentService @indentNodeProp @getIndentation @indentRange @indentUnit @getIndentUnit @indentString @IndentContext @TreeIndentContext @delimitedIndent @continuedIndent @flatIndent @indentOnInput ### Bracket Matching @bracketMatching @Config @matchBrackets @MatchResult @bracketMatchingHandle ### Stream Parser Stream parsers provide a way to adapt language modes written in the CodeMirror 5 style (see [@codemirror/legacy-modes](https://github.com/codemirror/legacy-modes)) to the `Language` interface. @StreamLanguage @StreamParser @StringStream language-6.10.1/src/fold.ts000066400000000000000000000433231455715122100154550ustar00rootroot00000000000000import {NodeProp, SyntaxNode, NodeIterator} from "@lezer/common" import {combineConfig, EditorState, StateEffect, ChangeDesc, Facet, StateField, Extension, RangeSet, RangeSetBuilder} from "@codemirror/state" import {EditorView, BlockInfo, Command, Decoration, DecorationSet, WidgetType, KeyBinding, ViewPlugin, ViewUpdate, gutter, GutterMarker} from "@codemirror/view" import {language, syntaxTree} from "./language" /// A facet that registers a code folding service. When called with /// the extent of a line, such a function should return a foldable /// range that starts on that line (but continues beyond it), if one /// can be found. export const foldService = Facet.define< (state: EditorState, lineStart: number, lineEnd: number) => ({from: number, to: number} | null) >() /// This node prop is used to associate folding information with /// syntax node types. Given a syntax node, it should check whether /// that tree is foldable and return the range that can be collapsed /// when it is. export const foldNodeProp = new NodeProp<(node: SyntaxNode, state: EditorState) => ({from: number, to: number} | null)>() /// [Fold](#language.foldNodeProp) function that folds everything but /// the first and the last child of a syntax node. Useful for nodes /// that start and end with delimiters. export function foldInside(node: SyntaxNode): {from: number, to: number} | null { let first = node.firstChild, last = node.lastChild return first && first.to < last!.from ? {from: first.to, to: last!.type.isError ? node.to : last!.from} : null } function syntaxFolding(state: EditorState, start: number, end: number) { let tree = syntaxTree(state) if (tree.length < end) return null let stack = tree.resolveStack(end, 1) let found: null | {from: number, to: number} = null for (let iter: NodeIterator | null = stack; iter; iter = iter.next) { let cur = iter.node if (cur.to <= end || cur.from > end) continue if (found && cur.from < start) break let prop = cur.type.prop(foldNodeProp) if (prop && (cur.to < tree.length - 50 || tree.length == state.doc.length || !isUnfinished(cur))) { let value = prop(cur, state) if (value && value.from <= end && value.from >= start && value.to > end) found = value } } return found } function isUnfinished(node: SyntaxNode) { let ch = node.lastChild return ch && ch.to == node.to && ch.type.isError } /// Check whether the given line is foldable. First asks any fold /// services registered through /// [`foldService`](#language.foldService), and if none of them return /// a result, tries to query the [fold node /// prop](#language.foldNodeProp) of syntax nodes that cover the end /// of the line. export function foldable(state: EditorState, lineStart: number, lineEnd: number) { for (let service of state.facet(foldService)) { let result = service(state, lineStart, lineEnd) if (result) return result } return syntaxFolding(state, lineStart, lineEnd) } type DocRange = {from: number, to: number} function mapRange(range: DocRange, mapping: ChangeDesc) { let from = mapping.mapPos(range.from, 1), to = mapping.mapPos(range.to, -1) return from >= to ? undefined : {from, to} } /// State effect that can be attached to a transaction to fold the /// given range. (You probably only need this in exceptional /// circumstances—usually you'll just want to let /// [`foldCode`](#language.foldCode) and the [fold /// gutter](#language.foldGutter) create the transactions.) export const foldEffect = StateEffect.define({map: mapRange}) /// State effect that unfolds the given range (if it was folded). export const unfoldEffect = StateEffect.define({map: mapRange}) function selectedLines(view: EditorView) { let lines: BlockInfo[] = [] for (let {head} of view.state.selection.ranges) { if (lines.some(l => l.from <= head && l.to >= head)) continue lines.push(view.lineBlockAt(head)) } return lines } /// The state field that stores the folded ranges (as a [decoration /// set](#view.DecorationSet)). Can be passed to /// [`EditorState.toJSON`](#state.EditorState.toJSON) and /// [`fromJSON`](#state.EditorState^fromJSON) to serialize the fold /// state. export const foldState = StateField.define({ create() { return Decoration.none }, update(folded, tr) { folded = folded.map(tr.changes) for (let e of tr.effects) { if (e.is(foldEffect) && !foldExists(folded, e.value.from, e.value.to)) { let {preparePlaceholder} = tr.state.facet(foldConfig) let widget = !preparePlaceholder ? foldWidget : Decoration.replace({widget: new PreparedFoldWidget(preparePlaceholder(tr.state, e.value))}) folded = folded.update({add: [widget.range(e.value.from, e.value.to)]}) } else if (e.is(unfoldEffect)) { folded = folded.update({filter: (from, to) => e.value.from != from || e.value.to != to, filterFrom: e.value.from, filterTo: e.value.to}) } } // Clear folded ranges that cover the selection head if (tr.selection) { let onSelection = false, {head} = tr.selection.main folded.between(head, head, (a, b) => { if (a < head && b > head) onSelection = true }) if (onSelection) folded = folded.update({ filterFrom: head, filterTo: head, filter: (a, b) => b <= head || a >= head }) } return folded }, provide: f => EditorView.decorations.from(f), toJSON(folded, state) { let ranges: number[] = [] folded.between(0, state.doc.length, (from, to) => {ranges.push(from, to)}) return ranges }, fromJSON(value) { if (!Array.isArray(value) || value.length % 2) throw new RangeError("Invalid JSON for fold state") let ranges = [] for (let i = 0; i < value.length;) { let from = value[i++], to = value[i++] if (typeof from != "number" || typeof to != "number") throw new RangeError("Invalid JSON for fold state") ranges.push(foldWidget.range(from, to)) } return Decoration.set(ranges, true) } }) /// Get a [range set](#state.RangeSet) containing the folded ranges /// in the given state. export function foldedRanges(state: EditorState): DecorationSet { return state.field(foldState, false) || RangeSet.empty } function findFold(state: EditorState, from: number, to: number) { let found: {from: number, to: number} | null = null state.field(foldState, false)?.between(from, to, (from, to) => { if (!found || found.from > from) found = {from, to} }) return found } function foldExists(folded: DecorationSet, from: number, to: number) { let found = false folded.between(from, from, (a, b) => { if (a == from && b == to) found = true }) return found } function maybeEnable(state: EditorState, other: readonly StateEffect[]) { return state.field(foldState, false) ? other : other.concat(StateEffect.appendConfig.of(codeFolding())) } /// Fold the lines that are selected, if possible. export const foldCode: Command = view => { for (let line of selectedLines(view)) { let range = foldable(view.state, line.from, line.to) if (range) { view.dispatch({effects: maybeEnable(view.state, [foldEffect.of(range), announceFold(view, range)])}) return true } } return false } /// Unfold folded ranges on selected lines. export const unfoldCode: Command = view => { if (!view.state.field(foldState, false)) return false let effects = [] for (let line of selectedLines(view)) { let folded = findFold(view.state, line.from, line.to) if (folded) effects.push(unfoldEffect.of(folded), announceFold(view, folded, false)) } if (effects.length) view.dispatch({effects}) return effects.length > 0 } function announceFold(view: EditorView, range: {from: number, to: number}, fold = true) { let lineFrom = view.state.doc.lineAt(range.from).number, lineTo = view.state.doc.lineAt(range.to).number return EditorView.announce.of(`${view.state.phrase(fold ? "Folded lines" : "Unfolded lines")} ${lineFrom} ${ view.state.phrase("to")} ${lineTo}.`) } /// Fold all top-level foldable ranges. Note that, in most cases, /// folding information will depend on the [syntax /// tree](#language.syntaxTree), and folding everything may not work /// reliably when the document hasn't been fully parsed (either /// because the editor state was only just initialized, or because the /// document is so big that the parser decided not to parse it /// entirely). export const foldAll: Command = view => { let {state} = view, effects = [] for (let pos = 0; pos < state.doc.length;) { let line = view.lineBlockAt(pos), range = foldable(state, line.from, line.to) if (range) effects.push(foldEffect.of(range)) pos = (range ? view.lineBlockAt(range.to) : line).to + 1 } if (effects.length) view.dispatch({effects: maybeEnable(view.state, effects)}) return !!effects.length } /// Unfold all folded code. export const unfoldAll: Command = view => { let field = view.state.field(foldState, false) if (!field || !field.size) return false let effects: StateEffect[] = [] field.between(0, view.state.doc.length, (from, to) => { effects.push(unfoldEffect.of({from, to})) }) view.dispatch({effects}) return true } // Find the foldable region containing the given line, if one exists function foldableContainer(view: EditorView, lineBlock: BlockInfo) { // Look backwards through line blocks until we find a foldable region that // intersects with the line for (let line = lineBlock;;) { let foldableRegion = foldable(view.state, line.from, line.to) if (foldableRegion && foldableRegion.to > lineBlock.from) return foldableRegion if (!line.from) return null line = view.lineBlockAt(line.from - 1) } } /// Toggle folding at cursors. Unfolds if there is an existing fold /// starting in that line, tries to find a foldable range around it /// otherwise. export const toggleFold: Command = (view) => { let effects: StateEffect[] = [] for (let line of selectedLines(view)) { let folded = findFold(view.state, line.from, line.to) if (folded) { effects.push(unfoldEffect.of(folded), announceFold(view, folded, false)) } else { let foldRange = foldableContainer(view, line) if (foldRange) effects.push(foldEffect.of(foldRange), announceFold(view, foldRange)) } } if (effects.length > 0) view.dispatch({effects: maybeEnable(view.state, effects)}) return !!effects.length } /// Default fold-related key bindings. /// /// - Ctrl-Shift-[ (Cmd-Alt-[ on macOS): [`foldCode`](#language.foldCode). /// - Ctrl-Shift-] (Cmd-Alt-] on macOS): [`unfoldCode`](#language.unfoldCode). /// - Ctrl-Alt-[: [`foldAll`](#language.foldAll). /// - Ctrl-Alt-]: [`unfoldAll`](#language.unfoldAll). export const foldKeymap: readonly KeyBinding[] = [ {key: "Ctrl-Shift-[", mac: "Cmd-Alt-[", run: foldCode}, {key: "Ctrl-Shift-]", mac: "Cmd-Alt-]", run: unfoldCode}, {key: "Ctrl-Alt-[", run: foldAll}, {key: "Ctrl-Alt-]", run: unfoldAll} ] interface FoldConfig { /// A function that creates the DOM element used to indicate the /// position of folded code. The `onclick` argument is the default /// click event handler, which toggles folding on the line that /// holds the element, and should probably be added as an event /// handler to the returned element. If /// [`preparePlaceholder`](#language.FoldConfig.preparePlaceholder) /// is given, its result will be passed as 3rd argument. Otherwise, /// this will be null. /// /// When this option isn't given, the `placeholderText` option will /// be used to create the placeholder element. placeholderDOM?: ((view: EditorView, onclick: (event: Event) => void, prepared: any) => HTMLElement) | null, /// Text to use as placeholder for folded text. Defaults to `"…"`. /// Will be styled with the `"cm-foldPlaceholder"` class. placeholderText?: string /// Given a range that is being folded, create a value that /// describes it, to be used by `placeholderDOM` to render a custom /// widget that, for example, indicates something about the folded /// range's size or type. preparePlaceholder?: (state: EditorState, range: {from: number, to: number}) => any } const defaultConfig: Required = { placeholderDOM: null, preparePlaceholder: null as any, placeholderText: "…" } const foldConfig = Facet.define>({ combine(values) { return combineConfig(values, defaultConfig) } }) /// Create an extension that configures code folding. export function codeFolding(config?: FoldConfig): Extension { let result = [foldState, baseTheme] if (config) result.push(foldConfig.of(config)) return result } function widgetToDOM(view: EditorView, prepared: any) { let {state} = view, conf = state.facet(foldConfig) let onclick = (event: Event) => { let line = view.lineBlockAt(view.posAtDOM(event.target as HTMLElement)) let folded = findFold(view.state, line.from, line.to) if (folded) view.dispatch({effects: unfoldEffect.of(folded)}) event.preventDefault() } if (conf.placeholderDOM) return conf.placeholderDOM(view, onclick, prepared) let element = document.createElement("span") element.textContent = conf.placeholderText element.setAttribute("aria-label", state.phrase("folded code")) element.title = state.phrase("unfold") element.className = "cm-foldPlaceholder" element.onclick = onclick return element } const foldWidget = Decoration.replace({widget: new class extends WidgetType { toDOM(view: EditorView) { return widgetToDOM(view, null) } }}) class PreparedFoldWidget extends WidgetType { constructor(readonly value: any) { super() } eq(other: PreparedFoldWidget) { return this.value == other.value } toDOM(view: EditorView) { return widgetToDOM(view, this.value) } } type Handlers = {[event: string]: (view: EditorView, line: BlockInfo, event: Event) => boolean} interface FoldGutterConfig { /// A function that creates the DOM element used to indicate a /// given line is folded or can be folded. /// When not given, the `openText`/`closeText` option will be used instead. markerDOM?: ((open: boolean) => HTMLElement) | null /// Text used to indicate that a given line can be folded. /// Defaults to `"⌄"`. openText?: string /// Text used to indicate that a given line is folded. /// Defaults to `"›"`. closedText?: string /// Supply event handlers for DOM events on this gutter. domEventHandlers?: Handlers /// When given, if this returns true for a given view update, /// recompute the fold markers. foldingChanged?: (update: ViewUpdate) => boolean } const foldGutterDefaults: Required = { openText: "⌄", closedText: "›", markerDOM: null, domEventHandlers: {}, foldingChanged: () => false } class FoldMarker extends GutterMarker { constructor(readonly config: Required, readonly open: boolean) { super() } eq(other: FoldMarker) { return this.config == other.config && this.open == other.open } toDOM(view: EditorView) { if (this.config.markerDOM) return this.config.markerDOM(this.open) let span = document.createElement("span") span.textContent = this.open ? this.config.openText : this.config.closedText span.title = view.state.phrase(this.open ? "Fold line" : "Unfold line") return span } } /// Create an extension that registers a fold gutter, which shows a /// fold status indicator before foldable lines (which can be clicked /// to fold or unfold the line). export function foldGutter(config: FoldGutterConfig = {}): Extension { let fullConfig = {...foldGutterDefaults, ...config} let canFold = new FoldMarker(fullConfig, true), canUnfold = new FoldMarker(fullConfig, false) let markers = ViewPlugin.fromClass(class { markers: RangeSet from: number constructor(view: EditorView) { this.from = view.viewport.from this.markers = this.buildMarkers(view) } update(update: ViewUpdate) { if (update.docChanged || update.viewportChanged || update.startState.facet(language) != update.state.facet(language) || update.startState.field(foldState, false) != update.state.field(foldState, false) || syntaxTree(update.startState) != syntaxTree(update.state) || fullConfig.foldingChanged(update)) this.markers = this.buildMarkers(update.view) } buildMarkers(view: EditorView) { let builder = new RangeSetBuilder() for (let line of view.viewportLineBlocks) { let mark = findFold(view.state, line.from, line.to) ? canUnfold : foldable(view.state, line.from, line.to) ? canFold : null if (mark) builder.add(line.from, line.from, mark) } return builder.finish() } }) let { domEventHandlers } = fullConfig; return [ markers, gutter({ class: "cm-foldGutter", markers(view) { return view.plugin(markers)?.markers || RangeSet.empty }, initialSpacer() { return new FoldMarker(fullConfig, false) }, domEventHandlers: { ...domEventHandlers, click: (view, line, event) => { if (domEventHandlers.click && domEventHandlers.click(view, line, event)) return true let folded = findFold(view.state, line.from, line.to) if (folded) { view.dispatch({effects: unfoldEffect.of(folded)}) return true } let range = foldable(view.state, line.from, line.to) if (range) { view.dispatch({effects: foldEffect.of(range)}) return true } return false } } }), codeFolding() ] } const baseTheme = EditorView.baseTheme({ ".cm-foldPlaceholder": { backgroundColor: "#eee", border: "1px solid #ddd", color: "#888", borderRadius: ".2em", margin: "0 1px", padding: "0 1px", cursor: "pointer" }, ".cm-foldGutter span": { padding: "0 1px", cursor: "pointer" } }) language-6.10.1/src/highlight.ts000066400000000000000000000224441455715122100165010ustar00rootroot00000000000000import {Tree, NodeType} from "@lezer/common" import {Tag, tags, tagHighlighter, Highlighter, highlightTree} from "@lezer/highlight" import {StyleSpec, StyleModule} from "style-mod" import {EditorView, ViewPlugin, ViewUpdate, Decoration, DecorationSet} from "@codemirror/view" import {EditorState, Prec, Facet, Extension, RangeSetBuilder} from "@codemirror/state" import {syntaxTree, Language, languageDataProp} from "./language" /// A highlight style associates CSS styles with higlighting /// [tags](https://lezer.codemirror.net/docs/ref#highlight.Tag). export class HighlightStyle implements Highlighter { /// A style module holding the CSS rules for this highlight style. /// When using /// [`highlightTree`](https://lezer.codemirror.net/docs/ref#highlight.highlightTree) /// outside of the editor, you may want to manually mount this /// module to show the highlighting. readonly module: StyleModule | null /// @internal readonly themeType: "dark" | "light" | undefined readonly style: (tags: readonly Tag[]) => string | null readonly scope: ((type: NodeType) => boolean) | undefined private constructor( /// The tag styles used to create this highlight style. readonly specs: readonly TagStyle[], options: {scope?: NodeType | Language, all?: string | StyleSpec, themeType?: "dark" | "light"} ) { let modSpec: {[name: string]: StyleSpec} | undefined function def(spec: StyleSpec) { let cls = StyleModule.newName() ;(modSpec || (modSpec = Object.create(null)))["." + cls] = spec return cls } const all = typeof options.all == "string" ? options.all : options.all ? def(options.all) : undefined const scopeOpt = options.scope this.scope = scopeOpt instanceof Language ? (type: NodeType) => type.prop(languageDataProp) == scopeOpt.data : scopeOpt ? (type: NodeType) => type == scopeOpt : undefined this.style = tagHighlighter(specs.map(style => ({ tag: style.tag, class: style.class as string || def(Object.assign({}, style, {tag: null})) })), { all, }).style this.module = modSpec ? new StyleModule(modSpec) : null this.themeType = options.themeType } /// Create a highlighter style that associates the given styles to /// the given tags. The specs must be objects that hold a style tag /// or array of tags in their `tag` property, and either a single /// `class` property providing a static CSS class (for highlighter /// that rely on external styling), or a /// [`style-mod`](https://github.com/marijnh/style-mod#documentation)-style /// set of CSS properties (which define the styling for those tags). /// /// The CSS rules created for a highlighter will be emitted in the /// order of the spec's properties. That means that for elements that /// have multiple tags associated with them, styles defined further /// down in the list will have a higher CSS precedence than styles /// defined earlier. static define(specs: readonly TagStyle[], options?: { /// By default, highlighters apply to the entire document. You can /// scope them to a single language by providing the language /// object or a language's top node type here. scope?: Language | NodeType, /// Add a style to _all_ content. Probably only useful in /// combination with `scope`. all?: string | StyleSpec, /// Specify that this highlight style should only be active then /// the theme is dark or light. By default, it is active /// regardless of theme. themeType?: "dark" | "light" }) { return new HighlightStyle(specs, options || {}) } } const highlighterFacet = Facet.define() const fallbackHighlighter = Facet.define({ combine(values) { return values.length ? [values[0]] : null } }) function getHighlighters(state: EditorState): readonly Highlighter[] | null { let main = state.facet(highlighterFacet) return main.length ? main : state.facet(fallbackHighlighter) } /// Wrap a highlighter in an editor extension that uses it to apply /// syntax highlighting to the editor content. /// /// When multiple (non-fallback) styles are provided, the styling /// applied is the union of the classes they emit. export function syntaxHighlighting(highlighter: Highlighter, options?: { /// When enabled, this marks the highlighter as a fallback, which /// only takes effect if no other highlighters are registered. fallback: boolean }): Extension { let ext: Extension[] = [treeHighlighter], themeType: string | undefined if (highlighter instanceof HighlightStyle) { if (highlighter.module) ext.push(EditorView.styleModule.of(highlighter.module)) themeType = highlighter.themeType } if (options?.fallback) ext.push(fallbackHighlighter.of(highlighter)) else if (themeType) ext.push(highlighterFacet.computeN([EditorView.darkTheme], state => { return state.facet(EditorView.darkTheme) == (themeType == "dark") ? [highlighter] : [] })) else ext.push(highlighterFacet.of(highlighter)) return ext } /// Returns the CSS classes (if any) that the highlighters active in /// the state would assign to the given style /// [tags](https://lezer.codemirror.net/docs/ref#highlight.Tag) and /// (optional) language /// [scope](#language.HighlightStyle^define^options.scope). export function highlightingFor(state: EditorState, tags: readonly Tag[], scope?: NodeType): string | null { let highlighters = getHighlighters(state) let result = null if (highlighters) for (let highlighter of highlighters) { if (!highlighter.scope || scope && highlighter.scope(scope)) { let cls = highlighter.style(tags) if (cls) result = result ? result + " " + cls : cls } } return result } /// The type of object used in /// [`HighlightStyle.define`](#language.HighlightStyle^define). /// Assigns a style to one or more highlighting /// [tags](https://lezer.codemirror.net/docs/ref#highlight.Tag), which can either be a fixed class name /// (which must be defined elsewhere), or a set of CSS properties, for /// which the library will define an anonymous class. export interface TagStyle { /// The tag or tags to target. tag: Tag | readonly Tag[], /// If given, this maps the tags to a fixed class name. class?: string, /// Any further properties (if `class` isn't given) will be /// interpreted as in style objects given to /// [style-mod](https://github.com/marijnh/style-mod#documentation). /// (The type here is `any` because of TypeScript limitations.) [styleProperty: string]: any } class TreeHighlighter { decorations: DecorationSet decoratedTo: number tree: Tree markCache: {[cls: string]: Decoration} = Object.create(null) constructor(view: EditorView) { this.tree = syntaxTree(view.state) this.decorations = this.buildDeco(view, getHighlighters(view.state)) this.decoratedTo = view.viewport.to } update(update: ViewUpdate) { let tree = syntaxTree(update.state), highlighters = getHighlighters(update.state) let styleChange = highlighters != getHighlighters(update.startState) let {viewport} = update.view, decoratedToMapped = update.changes.mapPos(this.decoratedTo, 1) if (tree.length < viewport.to && !styleChange && tree.type == this.tree.type && decoratedToMapped >= viewport.to) { this.decorations = this.decorations.map(update.changes) this.decoratedTo = decoratedToMapped } else if (tree != this.tree || update.viewportChanged || styleChange) { this.tree = tree this.decorations = this.buildDeco(update.view, highlighters) this.decoratedTo = viewport.to } } buildDeco(view: EditorView, highlighters: readonly Highlighter[] | null) { if (!highlighters || !this.tree.length) return Decoration.none let builder = new RangeSetBuilder() for (let {from, to} of view.visibleRanges) { highlightTree(this.tree, highlighters, (from, to, style) => { builder.add(from, to, this.markCache[style] || (this.markCache[style] = Decoration.mark({class: style}))) }, from, to) } return builder.finish() } } const treeHighlighter = Prec.high(ViewPlugin.fromClass(TreeHighlighter, { decorations: v => v.decorations })) /// A default highlight style (works well with light themes). export const defaultHighlightStyle = HighlightStyle.define([ {tag: tags.meta, color: "#404740"}, {tag: tags.link, textDecoration: "underline"}, {tag: tags.heading, textDecoration: "underline", fontWeight: "bold"}, {tag: tags.emphasis, fontStyle: "italic"}, {tag: tags.strong, fontWeight: "bold"}, {tag: tags.strikethrough, textDecoration: "line-through"}, {tag: tags.keyword, color: "#708"}, {tag: [tags.atom, tags.bool, tags.url, tags.contentSeparator, tags.labelName], color: "#219"}, {tag: [tags.literal, tags.inserted], color: "#164"}, {tag: [tags.string, tags.deleted], color: "#a11"}, {tag: [tags.regexp, tags.escape, tags.special(tags.string)], color: "#e40"}, {tag: tags.definition(tags.variableName), color: "#00f"}, {tag: tags.local(tags.variableName), color: "#30a"}, {tag: [tags.typeName, tags.namespace], color: "#085"}, {tag: tags.className, color: "#167"}, {tag: [tags.special(tags.variableName), tags.macroName], color: "#256"}, {tag: tags.definition(tags.propertyName), color: "#00c"}, {tag: tags.comment, color: "#940"}, {tag: tags.invalid, color: "#f00"} ]) language-6.10.1/src/indent.ts000066400000000000000000000402031455715122100160040ustar00rootroot00000000000000import {NodeProp, SyntaxNode, NodeIterator, Tree} from "@lezer/common" import {EditorState, Extension, Facet, countColumn, ChangeSpec} from "@codemirror/state" import {syntaxTree} from "./language" /// Facet that defines a way to provide a function that computes the /// appropriate indentation depth, as a column number (see /// [`indentString`](#language.indentString)), at the start of a given /// line. A return value of `null` indicates no indentation can be /// determined, and the line should inherit the indentation of the one /// above it. A return value of `undefined` defers to the next indent /// service. export const indentService = Facet.define<(context: IndentContext, pos: number) => number | null | undefined>() /// Facet for overriding the unit by which indentation happens. Should /// be a string consisting either entirely of the same whitespace /// character. When not set, this defaults to 2 spaces. export const indentUnit = Facet.define({ combine: values => { if (!values.length) return " " let unit = values[0] if (!unit || /\S/.test(unit) || Array.from(unit).some(e => e != unit[0])) throw new Error("Invalid indent unit: " + JSON.stringify(values[0])) return unit } }) /// Return the _column width_ of an indent unit in the state. /// Determined by the [`indentUnit`](#language.indentUnit) /// facet, and [`tabSize`](#state.EditorState^tabSize) when that /// contains tabs. export function getIndentUnit(state: EditorState) { let unit = state.facet(indentUnit) return unit.charCodeAt(0) == 9 ? state.tabSize * unit.length : unit.length } /// Create an indentation string that covers columns 0 to `cols`. /// Will use tabs for as much of the columns as possible when the /// [`indentUnit`](#language.indentUnit) facet contains /// tabs. export function indentString(state: EditorState, cols: number) { let result = "", ts = state.tabSize, ch = state.facet(indentUnit)[0] if (ch == "\t") { while (cols >= ts) { result += "\t" cols -= ts } ch = " " } for (let i = 0; i < cols; i++) result += ch return result } /// Get the indentation, as a column number, at the given position. /// Will first consult any [indent services](#language.indentService) /// that are registered, and if none of those return an indentation, /// this will check the syntax tree for the [indent node /// prop](#language.indentNodeProp) and use that if found. Returns a /// number when an indentation could be determined, and null /// otherwise. export function getIndentation(context: IndentContext | EditorState, pos: number): number | null { if (context instanceof EditorState) context = new IndentContext(context) for (let service of context.state.facet(indentService)) { let result = service(context, pos) if (result !== undefined) return result } let tree = syntaxTree(context.state) return tree.length >= pos ? syntaxIndentation(context, tree, pos) : null } /// Create a change set that auto-indents all lines touched by the /// given document range. export function indentRange(state: EditorState, from: number, to: number) { let updated: {[lineStart: number]: number} = Object.create(null) let context = new IndentContext(state, {overrideIndentation: start => updated[start] ?? -1}) let changes: ChangeSpec[] = [] for (let pos = from; pos <= to;) { let line = state.doc.lineAt(pos) pos = line.to + 1 let indent = getIndentation(context, line.from) if (indent == null) continue if (!/\S/.test(line.text)) indent = 0 let cur = /^\s*/.exec(line.text)![0] let norm = indentString(state, indent) if (cur != norm) { updated[line.from] = indent changes.push({from: line.from, to: line.from + cur.length, insert: norm}) } } return state.changes(changes) } /// Indentation contexts are used when calling [indentation /// services](#language.indentService). They provide helper utilities /// useful in indentation logic, and can selectively override the /// indentation reported for some lines. export class IndentContext { /// The indent unit (number of columns per indentation level). unit: number /// Create an indent context. constructor( /// The editor state. readonly state: EditorState, /// @internal readonly options: { /// Override line indentations provided to the indentation /// helper function, which is useful when implementing region /// indentation, where indentation for later lines needs to refer /// to previous lines, which may have been reindented compared to /// the original start state. If given, this function should /// return -1 for lines (given by start position) that didn't /// change, and an updated indentation otherwise. overrideIndentation?: (pos: number) => number, /// Make it look, to the indent logic, like a line break was /// added at the given position (which is mostly just useful for /// implementing something like /// [`insertNewlineAndIndent`](#commands.insertNewlineAndIndent)). simulateBreak?: number, /// When `simulateBreak` is given, this can be used to make the /// simulated break behave like a double line break. simulateDoubleBreak?: boolean } = {} ) { this.unit = getIndentUnit(state) } /// Get a description of the line at the given position, taking /// [simulated line /// breaks](#language.IndentContext.constructor^options.simulateBreak) /// into account. If there is such a break at `pos`, the `bias` /// argument determines whether the part of the line line before or /// after the break is used. lineAt(pos: number, bias: -1 | 1 = 1): {text: string, from: number} { let line = this.state.doc.lineAt(pos) let {simulateBreak, simulateDoubleBreak} = this.options if (simulateBreak != null && simulateBreak >= line.from && simulateBreak <= line.to) { if (simulateDoubleBreak && simulateBreak == pos) return {text: "", from: pos} else if (bias < 0 ? simulateBreak < pos : simulateBreak <= pos) return {text: line.text.slice(simulateBreak - line.from), from: simulateBreak} else return {text: line.text.slice(0, simulateBreak - line.from), from: line.from} } return line } /// Get the text directly after `pos`, either the entire line /// or the next 100 characters, whichever is shorter. textAfterPos(pos: number, bias: -1 | 1 = 1) { if (this.options.simulateDoubleBreak && pos == this.options.simulateBreak) return "" let {text, from} = this.lineAt(pos, bias) return text.slice(pos - from, Math.min(text.length, pos + 100 - from)) } /// Find the column for the given position. column(pos: number, bias: -1 | 1 = 1) { let {text, from} = this.lineAt(pos, bias) let result = this.countColumn(text, pos - from) let override = this.options.overrideIndentation ? this.options.overrideIndentation(from) : -1 if (override > -1) result += override - this.countColumn(text, text.search(/\S|$/)) return result } /// Find the column position (taking tabs into account) of the given /// position in the given string. countColumn(line: string, pos: number = line.length) { return countColumn(line, this.state.tabSize, pos) } /// Find the indentation column of the line at the given point. lineIndent(pos: number, bias: -1 | 1 = 1) { let {text, from} = this.lineAt(pos, bias) let override = this.options.overrideIndentation if (override) { let overriden = override(from) if (overriden > -1) return overriden } return this.countColumn(text, text.search(/\S|$/)) } /// Returns the [simulated line /// break](#language.IndentContext.constructor^options.simulateBreak) /// for this context, if any. get simulatedBreak(): number | null { return this.options.simulateBreak || null } } /// A syntax tree node prop used to associate indentation strategies /// with node types. Such a strategy is a function from an indentation /// context to a column number (see also /// [`indentString`](#language.indentString)) or null, where null /// indicates that no definitive indentation can be determined. export const indentNodeProp = new NodeProp<(context: TreeIndentContext) => number | null>() // Compute the indentation for a given position from the syntax tree. function syntaxIndentation(cx: IndentContext, ast: Tree, pos: number) { let stack = ast.resolveStack(pos) let inner = stack.node.enterUnfinishedNodesBefore(pos) if (inner != stack.node) { let add = [] for (let cur = inner; cur != stack.node; cur = cur.parent!) add.push(cur) for (let i = add.length - 1; i >= 0; i--) stack = {node: add[i], next: stack} } return indentFor(stack, cx, pos) } function indentFor(stack: NodeIterator | null, cx: IndentContext, pos: number): number | null { for (let cur: NodeIterator | null = stack; cur; cur = cur.next) { let strategy = indentStrategy(cur.node) if (strategy) return strategy(TreeIndentContext.create(cx, pos, cur)) } return 0 } function ignoreClosed(cx: TreeIndentContext) { return cx.pos == cx.options.simulateBreak && cx.options.simulateDoubleBreak } function indentStrategy(tree: SyntaxNode): ((context: TreeIndentContext) => number | null) | null { let strategy = tree.type.prop(indentNodeProp) if (strategy) return strategy let first = tree.firstChild, close: readonly string[] | undefined if (first && (close = first.type.prop(NodeProp.closedBy))) { let last = tree.lastChild, closed = last && close.indexOf(last.name) > -1 return cx => delimitedStrategy(cx, true, 1, undefined, closed && !ignoreClosed(cx) ? last!.from : undefined) } return tree.parent == null ? topIndent : null } function topIndent() { return 0 } /// Objects of this type provide context information and helper /// methods to indentation functions registered on syntax nodes. export class TreeIndentContext extends IndentContext { private constructor( private base: IndentContext, /// The position at which indentation is being computed. readonly pos: number, /// @internal readonly context: NodeIterator ) { super(base.state, base.options) } /// The syntax tree node to which the indentation strategy /// applies. get node(): SyntaxNode { return this.context.node } /// @internal static create(base: IndentContext, pos: number, context: NodeIterator) { return new TreeIndentContext(base, pos, context) } /// Get the text directly after `this.pos`, either the entire line /// or the next 100 characters, whichever is shorter. get textAfter() { return this.textAfterPos(this.pos) } /// Get the indentation at the reference line for `this.node`, which /// is the line on which it starts, unless there is a node that is /// _not_ a parent of this node covering the start of that line. If /// so, the line at the start of that node is tried, again skipping /// on if it is covered by another such node. get baseIndent() { return this.baseIndentFor(this.node) } /// Get the indentation for the reference line of the given node /// (see [`baseIndent`](#language.TreeIndentContext.baseIndent)). baseIndentFor(node: SyntaxNode) { let line = this.state.doc.lineAt(node.from) // Skip line starts that are covered by a sibling (or cousin, etc) for (;;) { let atBreak = node.resolve(line.from) while (atBreak.parent && atBreak.parent.from == atBreak.from) atBreak = atBreak.parent if (isParent(atBreak, node)) break line = this.state.doc.lineAt(atBreak.from) } return this.lineIndent(line.from) } /// Continue looking for indentations in the node's parent nodes, /// and return the result of that. continue() { return indentFor(this.context.next, this.base, this.pos) } } function isParent(parent: SyntaxNode, of: SyntaxNode) { for (let cur: SyntaxNode | null = of; cur; cur = cur.parent) if (parent == cur) return true return false } // Check whether a delimited node is aligned (meaning there are // non-skipped nodes on the same line as the opening delimiter). And // if so, return the opening token. function bracketedAligned(context: TreeIndentContext) { let tree = context.node let openToken = tree.childAfter(tree.from), last = tree.lastChild if (!openToken) return null let sim = context.options.simulateBreak let openLine = context.state.doc.lineAt(openToken.from) let lineEnd = sim == null || sim <= openLine.from ? openLine.to : Math.min(openLine.to, sim) for (let pos = openToken.to;;) { let next = tree.childAfter(pos) if (!next || next == last) return null if (!next.type.isSkipped) return next.from < lineEnd ? openToken : null pos = next.to } } /// An indentation strategy for delimited (usually bracketed) nodes. /// Will, by default, indent one unit more than the parent's base /// indent unless the line starts with a closing token. When `align` /// is true and there are non-skipped nodes on the node's opening /// line, the content of the node will be aligned with the end of the /// opening node, like this: /// /// foo(bar, /// baz) export function delimitedIndent({closing, align = true, units = 1}: {closing: string, align?: boolean, units?: number}) { return (context: TreeIndentContext) => delimitedStrategy(context, align, units, closing) } function delimitedStrategy(context: TreeIndentContext, align: boolean, units: number, closing?: string, closedAt?: number) { let after = context.textAfter, space = after.match(/^\s*/)![0].length let closed = closing && after.slice(space, space + closing.length) == closing || closedAt == context.pos + space let aligned = align ? bracketedAligned(context) : null if (aligned) return closed ? context.column(aligned.from) : context.column(aligned.to) return context.baseIndent + (closed ? 0 : context.unit * units) } /// An indentation strategy that aligns a node's content to its base /// indentation. export const flatIndent = (context: TreeIndentContext) => context.baseIndent /// Creates an indentation strategy that, by default, indents /// continued lines one unit more than the node's base indentation. /// You can provide `except` to prevent indentation of lines that /// match a pattern (for example `/^else\b/` in `if`/`else` /// constructs), and you can change the amount of units used with the /// `units` option. export function continuedIndent({except, units = 1}: {except?: RegExp, units?: number} = {}) { return (context: TreeIndentContext) => { let matchExcept = except && except.test(context.textAfter) return context.baseIndent + (matchExcept ? 0 : units * context.unit) } } const DontIndentBeyond = 200 /// Enables reindentation on input. When a language defines an /// `indentOnInput` field in its [language /// data](#state.EditorState.languageDataAt), which must hold a regular /// expression, the line at the cursor will be reindented whenever new /// text is typed and the input from the start of the line up to the /// cursor matches that regexp. /// /// To avoid unneccesary reindents, it is recommended to start the /// regexp with `^` (usually followed by `\s*`), and end it with `$`. /// For example, `/^\s*\}$/` will reindent when a closing brace is /// added at the start of a line. export function indentOnInput(): Extension { return EditorState.transactionFilter.of(tr => { if (!tr.docChanged || !tr.isUserEvent("input.type") && !tr.isUserEvent("input.complete")) return tr let rules = tr.startState.languageDataAt("indentOnInput", tr.startState.selection.main.head) if (!rules.length) return tr let doc = tr.newDoc, {head} = tr.newSelection.main, line = doc.lineAt(head) if (head > line.from + DontIndentBeyond) return tr let lineStart = doc.sliceString(line.from, head) if (!rules.some(r => r.test(lineStart))) return tr let {state} = tr, last = -1, changes = [] for (let {head} of state.selection.ranges) { let line = state.doc.lineAt(head) if (line.from == last) continue last = line.from let indent = getIndentation(state, line.from) if (indent == null) continue let cur = /^\s*/.exec(line.text)![0] let norm = indentString(state, indent) if (cur != norm) changes.push({from: line.from, to: line.from + cur.length, insert: norm}) } return changes.length ? [tr, {changes, sequential: true}] : tr }) } language-6.10.1/src/index.ts000066400000000000000000000021371455715122100156360ustar00rootroot00000000000000export {language, Language, LRLanguage, Sublanguage, sublanguageProp, defineLanguageFacet, syntaxTree, ensureSyntaxTree, languageDataProp, ParseContext, LanguageSupport, LanguageDescription, syntaxTreeAvailable, syntaxParserRunning, forceParsing, DocInput} from "./language" export {IndentContext, getIndentUnit, indentString, indentOnInput, indentService, getIndentation, indentRange, indentUnit, TreeIndentContext, indentNodeProp, delimitedIndent, continuedIndent, flatIndent} from "./indent" export {foldService, foldNodeProp, foldInside, foldable, foldCode, unfoldCode, toggleFold, foldAll, unfoldAll, foldKeymap, codeFolding, foldGutter, foldedRanges, foldEffect, unfoldEffect, foldState} from "./fold" export {HighlightStyle, syntaxHighlighting, highlightingFor, TagStyle, defaultHighlightStyle} from "./highlight" export {bracketMatching, Config, matchBrackets, MatchResult, bracketMatchingHandle} from "./matchbrackets" export {StreamLanguage, StreamParser} from "./stream-parser" export {StringStream} from "./stringstream" export {bidiIsolates} from "./isolate" language-6.10.1/src/isolate.ts000066400000000000000000000104301455715122100161620ustar00rootroot00000000000000import {EditorView, ViewUpdate, ViewPlugin, DecorationSet, Decoration, Direction} from "@codemirror/view" import {syntaxTree} from "./language" import {NodeProp, Tree} from "@lezer/common" import {RangeSetBuilder, Prec, Text, Extension, ChangeSet, Facet} from "@codemirror/state" function buildForLine(line: string) { return line.length <= 4096 && /[\u0590-\u05f4\u0600-\u06ff\u0700-\u08ac\ufb50-\ufdff]/.test(line) } function textHasRTL(text: Text) { for (let i = text.iter(); !i.next().done;) if (buildForLine(i.value)) return true return false } function changeAddsRTL(change: ChangeSet) { let added = false change.iterChanges((fA, tA, fB, tB, ins) => { if (!added && textHasRTL(ins)) added = true }) return added } const alwaysIsolate = Facet.define({combine: values => values.some(x => x)}) /// Make sure nodes /// [marked](https://lezer.codemirror.net/docs/ref/#common.NodeProp^isolate) /// as isolating for bidirectional text are rendered in a way that /// isolates them from the surrounding text. export function bidiIsolates(options: { /// By default, isolating elements are only added when the editor /// direction isn't uniformly left-to-right, or if it is, on lines /// that contain right-to-left character. When true, disable this /// optimization and add them everywhere. alwaysIsolate?: boolean } = {}): Extension { let extensions: Extension[] = [isolateMarks] if (options.alwaysIsolate) extensions.push(alwaysIsolate.of(true)) return extensions } const isolateMarks = ViewPlugin.fromClass(class { decorations: DecorationSet tree: Tree hasRTL: boolean always: boolean constructor(view: EditorView) { this.always = view.state.facet(alwaysIsolate) || view.textDirection != Direction.LTR || view.state.facet(EditorView.perLineTextDirection) this.hasRTL = !this.always && textHasRTL(view.state.doc) this.tree = syntaxTree(view.state) this.decorations = this.always || this.hasRTL ? buildDeco(view, this.tree, this.always) : Decoration.none } update(update: ViewUpdate) { let always = update.state.facet(alwaysIsolate) || update.view.textDirection != Direction.LTR || update.state.facet(EditorView.perLineTextDirection) if (!always && !this.hasRTL && changeAddsRTL(update.changes)) this.hasRTL = true if (!always && !this.hasRTL) return let tree = syntaxTree(update.state) if (always != this.always || tree != this.tree || update.docChanged || update.viewportChanged) { this.tree = tree this.always = always this.decorations = buildDeco(update.view, tree, always) } } }, { provide: plugin => { function access(view: EditorView) { return view.plugin(plugin)?.decorations ?? Decoration.none } return [EditorView.outerDecorations.of(access), Prec.lowest(EditorView.bidiIsolatedRanges.of(access))] } }) function buildDeco(view: EditorView, tree: Tree, always: boolean) { let deco = new RangeSetBuilder() let ranges = view.visibleRanges if (!always) ranges = clipRTLLines(ranges, view.state.doc) for (let {from, to} of ranges) { tree.iterate({ enter: node => { let iso = node.type.prop(NodeProp.isolate) if (iso) deco.add(node.from, node.to, marks[iso]) }, from, to }) } return deco.finish() } function clipRTLLines(ranges: readonly {from: number, to: number}[], doc: Text) { let cur = doc.iter(), pos = 0, result: {from: number, to: number}[] = [], last = null for (let {from, to} of ranges) { if (from != pos) { if (pos < from) cur.next(from - pos) pos = from } for (;;) { let start = pos, end = pos + cur.value.length if (!cur.lineBreak && buildForLine(cur.value)) { if (last && last.to > start - 10) last.to = Math.min(to, end) else result.push(last = {from: start, to: Math.min(to, end)}) } if (pos >= to) break pos = end cur.next() } } return result } const marks = { rtl: Decoration.mark({class: "cm-iso", inclusive: true, attributes: {dir: "rtl"}, bidiIsolate: Direction.RTL}), ltr: Decoration.mark({class: "cm-iso", inclusive: true, attributes: {dir: "ltr"}, bidiIsolate: Direction.LTR}), auto: Decoration.mark({class: "cm-iso", inclusive: true, attributes: {dir: "auto"}, bidiIsolate: null}) } language-6.10.1/src/language.ts000066400000000000000000000732311455715122100163150ustar00rootroot00000000000000import {Tree, SyntaxNode, ChangedRange, TreeFragment, NodeProp, NodeType, Input, PartialParse, Parser, IterMode} from "@lezer/common" import type {LRParser, ParserConfig} from "@lezer/lr" import {EditorState, StateField, Transaction, Extension, StateEffect, Facet, ChangeDesc, Text, TextIterator} from "@codemirror/state" import {ViewPlugin, ViewUpdate, EditorView, logException} from "@codemirror/view" /// Node prop stored in a parser's top syntax node to provide the /// facet that stores language-specific data for that language. export const languageDataProp = new NodeProp>() /// Helper function to define a facet (to be added to the top syntax /// node(s) for a language via /// [`languageDataProp`](#language.languageDataProp)), that will be /// used to associate language data with the language. You /// probably only need this when subclassing /// [`Language`](#language.Language). export function defineLanguageFacet(baseData?: {[name: string]: any}) { return Facet.define<{[name: string]: any}>({ combine: baseData ? values => values.concat(baseData!) : undefined }) } /// Some languages need to return different [language /// data](#state.EditorState.languageDataAt) for some parts of their /// tree. Sublanguages, registered by adding a [node /// prop](#language.sublanguageProp) to the language's top syntax /// node, provide a mechanism to do this. /// /// (Note that when using nested parsing, where nested syntax is /// parsed by a different parser and has its own top node type, you /// don't need a sublanguage.) export interface Sublanguage { /// Determines whether the data provided by this sublanguage should /// completely replace the regular data or be added to it (with /// higher-precedence). The default is `"extend"`. type?: "replace" | "extend", /// A predicate that returns whether the node at the queried /// position is part of the sublanguage. test: (node: SyntaxNode, state: EditorState) => boolean, /// The language data facet that holds the sublanguage's data. /// You'll want to use /// [`defineLanguageFacet`](#language.defineLanguageFacet) to create /// this. facet: Facet<{[name: string]: any}> } /// Syntax node prop used to register sublanguages. Should be added to /// the top level node type for the language. export const sublanguageProp = new NodeProp() /// A language object manages parsing and per-language /// [metadata](#state.EditorState.languageDataAt). Parse data is /// managed as a [Lezer](https://lezer.codemirror.net) tree. The class /// can be used directly, via the [`LRLanguage`](#language.LRLanguage) /// subclass for [Lezer](https://lezer.codemirror.net/) LR parsers, or /// via the [`StreamLanguage`](#language.StreamLanguage) subclass /// for stream parsers. export class Language { /// The extension value to install this as the document language. readonly extension: Extension /// The parser object. Can be useful when using this as a [nested /// parser](https://lezer.codemirror.net/docs/ref#common.Parser). parser: Parser /// Construct a language object. If you need to invoke this /// directly, first define a data facet with /// [`defineLanguageFacet`](#language.defineLanguageFacet), and then /// configure your parser to [attach](#language.languageDataProp) it /// to the language's outer syntax node. constructor( /// The [language data](#state.EditorState.languageDataAt) facet /// used for this language. readonly data: Facet<{[name: string]: any}>, parser: Parser, extraExtensions: Extension[] = [], /// A language name. readonly name: string = "" ) { // Kludge to define EditorState.tree as a debugging helper, // without the EditorState package actually knowing about // languages and lezer trees. if (!EditorState.prototype.hasOwnProperty("tree")) Object.defineProperty(EditorState.prototype, "tree", {get() { return syntaxTree(this) }}) this.parser = parser this.extension = [ language.of(this), EditorState.languageData.of((state, pos, side) => { let top = topNodeAt(state, pos, side), data = top.type.prop(languageDataProp) if (!data) return [] let base = state.facet(data), sub = top.type.prop(sublanguageProp) if (sub) { let innerNode = top.resolve(pos - top.from, side) for (let sublang of sub) if (sublang.test(innerNode, state)) { let data = state.facet(sublang.facet) return sublang.type == "replace" ? data : data.concat(base) } } return base }) ].concat(extraExtensions) } /// Query whether this language is active at the given position. isActiveAt(state: EditorState, pos: number, side: -1 | 0 | 1 = -1) { return topNodeAt(state, pos, side).type.prop(languageDataProp) == this.data } /// Find the document regions that were parsed using this language. /// The returned regions will _include_ any nested languages rooted /// in this language, when those exist. findRegions(state: EditorState) { let lang = state.facet(language) if (lang?.data == this.data) return [{from: 0, to: state.doc.length}] if (!lang || !lang.allowsNesting) return [] let result: {from: number, to: number}[] = [] let explore = (tree: Tree, from: number) => { if (tree.prop(languageDataProp) == this.data) { result.push({from, to: from + tree.length}) return } let mount = tree.prop(NodeProp.mounted) if (mount) { if (mount.tree.prop(languageDataProp) == this.data) { if (mount.overlay) for (let r of mount.overlay) result.push({from: r.from + from, to: r.to + from}) else result.push({from: from, to: from + tree.length}) return } else if (mount.overlay) { let size = result.length explore(mount.tree, mount.overlay[0].from + from) if (result.length > size) return } } for (let i = 0; i < tree.children.length; i++) { let ch = tree.children[i] if (ch instanceof Tree) explore(ch, tree.positions[i] + from) } } explore(syntaxTree(state), 0) return result } /// Indicates whether this language allows nested languages. The /// default implementation returns true. get allowsNesting() { return true } /// @internal static state: StateField /// @internal static setState = StateEffect.define() } function topNodeAt(state: EditorState, pos: number, side: -1 | 0 | 1) { let topLang = state.facet(language), tree = syntaxTree(state).topNode if (!topLang || topLang.allowsNesting) { for (let node: SyntaxNode | null = tree; node; node = node.enter(pos, side, IterMode.ExcludeBuffers)) if (node.type.isTop) tree = node } return tree } /// A subclass of [`Language`](#language.Language) for use with Lezer /// [LR parsers](https://lezer.codemirror.net/docs/ref#lr.LRParser) /// parsers. export class LRLanguage extends Language { private constructor(data: Facet<{[name: string]: any}>, readonly parser: LRParser, name?: string) { super(data, parser, [], name) } /// Define a language from a parser. static define(spec: { /// The [name](#Language.name) of the language. name?: string, /// The parser to use. Should already have added editor-relevant /// node props (and optionally things like dialect and top rule) /// configured. parser: LRParser, /// [Language data](#state.EditorState.languageDataAt) /// to register for this language. languageData?: {[name: string]: any} }) { let data = defineLanguageFacet(spec.languageData) return new LRLanguage(data, spec.parser.configure({ props: [languageDataProp.add(type => type.isTop ? data : undefined)] }), spec.name) } /// Create a new instance of this language with a reconfigured /// version of its parser and optionally a new name. configure(options: ParserConfig, name?: string): LRLanguage { return new LRLanguage(this.data, this.parser.configure(options), name || this.name) } get allowsNesting() { return this.parser.hasWrappers() } } /// Get the syntax tree for a state, which is the current (possibly /// incomplete) parse tree of the active /// [language](#language.Language), or the empty tree if there is no /// language available. export function syntaxTree(state: EditorState): Tree { let field = state.field(Language.state, false) return field ? field.tree : Tree.empty } /// Try to get a parse tree that spans at least up to `upto`. The /// method will do at most `timeout` milliseconds of work to parse /// up to that point if the tree isn't already available. export function ensureSyntaxTree(state: EditorState, upto: number, timeout = 50): Tree | null { let parse = state.field(Language.state, false)?.context if (!parse) return null let oldVieport = parse.viewport parse.updateViewport({from: 0, to: upto}) let result = parse.isDone(upto) || parse.work(timeout, upto) ? parse.tree : null parse.updateViewport(oldVieport) return result } /// Queries whether there is a full syntax tree available up to the /// given document position. If there isn't, the background parse /// process _might_ still be working and update the tree further, but /// there is no guarantee of that—the parser will [stop /// working](#language.syntaxParserRunning) when it has spent a /// certain amount of time or has moved beyond the visible viewport. /// Always returns false if no language has been enabled. export function syntaxTreeAvailable(state: EditorState, upto = state.doc.length) { return state.field(Language.state, false)?.context.isDone(upto) || false } /// Move parsing forward, and update the editor state afterwards to /// reflect the new tree. Will work for at most `timeout` /// milliseconds. Returns true if the parser managed get to the given /// position in that time. export function forceParsing(view: EditorView, upto = view.viewport.to, timeout = 100): boolean { let success = ensureSyntaxTree(view.state, upto, timeout) if (success != syntaxTree(view.state)) view.dispatch({}) return !!success } /// Tells you whether the language parser is planning to do more /// parsing work (in a `requestIdleCallback` pseudo-thread) or has /// stopped running, either because it parsed the entire document, /// because it spent too much time and was cut off, or because there /// is no language parser enabled. export function syntaxParserRunning(view: EditorView) { return view.plugin(parseWorker)?.isWorking() || false } /// Lezer-style /// [`Input`](https://lezer.codemirror.net/docs/ref#common.Input) /// object for a [`Text`](#state.Text) object. export class DocInput implements Input { private cursor: TextIterator private cursorPos = 0 private string = "" /// Create an input object for the given document. constructor(readonly doc: Text) { this.cursor = doc.iter() } get length() { return this.doc.length } private syncTo(pos: number) { this.string = this.cursor.next(pos - this.cursorPos).value this.cursorPos = pos + this.string.length return this.cursorPos - this.string.length } chunk(pos: number) { this.syncTo(pos) return this.string } get lineChunks() { return true } read(from: number, to: number) { let stringStart = this.cursorPos - this.string.length if (from < stringStart || to >= this.cursorPos) return this.doc.sliceString(from, to) else return this.string.slice(from - stringStart, to - stringStart) } } const enum Work { // Milliseconds of work time to perform immediately for a state doc change Apply = 20, // Minimum amount of work time to perform in an idle callback MinSlice = 25, // Amount of work time to perform in pseudo-thread when idle callbacks aren't supported Slice = 100, // Minimum pause between pseudo-thread slices MinPause = 100, // Maximum pause (timeout) for the pseudo-thread MaxPause = 500, // Parse time budgets are assigned per chunk—the parser can run for // ChunkBudget milliseconds at most during ChunkTime milliseconds. // After that, no further background parsing is scheduled until the // next chunk in which the editor is active. ChunkBudget = 3000, ChunkTime = 30000, // For every change the editor receives while focused, it gets a // small bonus to its parsing budget (as a way to allow active // editors to continue doing work). ChangeBonus = 50, // Don't eagerly parse this far beyond the end of the viewport MaxParseAhead = 1e5, // When initializing the state field (before viewport info is // available), pretend the viewport goes from 0 to here. InitViewport = 3000, } let currentContext: ParseContext | null = null /// A parse context provided to parsers working on the editor content. export class ParseContext { private parse: PartialParse | null = null /// @internal tempSkipped: {from: number, to: number}[] = [] private constructor( private parser: Parser, /// The current editor state. readonly state: EditorState, /// Tree fragments that can be reused by incremental re-parses. public fragments: readonly TreeFragment[] = [], /// @internal public tree: Tree, /// @internal public treeLen: number, /// The current editor viewport (or some overapproximation /// thereof). Intended to be used for opportunistically avoiding /// work (in which case /// [`skipUntilInView`](#language.ParseContext.skipUntilInView) /// should be called to make sure the parser is restarted when the /// skipped region becomes visible). public viewport: {from: number, to: number}, /// @internal public skipped: {from: number, to: number}[], /// This is where skipping parsers can register a promise that, /// when resolved, will schedule a new parse. It is cleared when /// the parse worker picks up the promise. @internal public scheduleOn: Promise | null ) {} /// @internal static create(parser: Parser, state: EditorState, viewport: {from: number, to: number}) { return new ParseContext(parser, state, [], Tree.empty, 0, viewport, [], null) } private startParse() { return this.parser.startParse(new DocInput(this.state.doc), this.fragments) } /// @internal work(until: number | (() => boolean), upto?: number) { if (upto != null && upto >= this.state.doc.length) upto = undefined if (this.tree != Tree.empty && this.isDone(upto ?? this.state.doc.length)) { this.takeTree() return true } return this.withContext(() => { if (typeof until == "number") { let endTime = Date.now() + until until = () => Date.now() > endTime } if (!this.parse) this.parse = this.startParse() if (upto != null && (this.parse.stoppedAt == null || this.parse.stoppedAt > upto) && upto < this.state.doc.length) this.parse.stopAt(upto) for (;;) { let done = this.parse.advance() if (done) { this.fragments = this.withoutTempSkipped(TreeFragment.addTree(done, this.fragments, this.parse.stoppedAt != null)) this.treeLen = this.parse.stoppedAt ?? this.state.doc.length this.tree = done this.parse = null if (this.treeLen < (upto ?? this.state.doc.length)) this.parse = this.startParse() else return true } if (until()) return false } }) } /// @internal takeTree() { let pos, tree: Tree | undefined | null if (this.parse && (pos = this.parse.parsedPos) >= this.treeLen) { if (this.parse.stoppedAt == null || this.parse.stoppedAt > pos) this.parse.stopAt(pos) this.withContext(() => { while (!(tree = this.parse!.advance())) {} }) this.treeLen = pos this.tree = tree! this.fragments = this.withoutTempSkipped(TreeFragment.addTree(this.tree, this.fragments, true)) this.parse = null } } private withContext(f: () => T): T { let prev = currentContext currentContext = this try { return f() } finally { currentContext = prev } } private withoutTempSkipped(fragments: readonly TreeFragment[]) { for (let r; r = this.tempSkipped.pop();) fragments = cutFragments(fragments, r.from, r.to) return fragments } /// @internal changes(changes: ChangeDesc, newState: EditorState) { let {fragments, tree, treeLen, viewport, skipped} = this this.takeTree() if (!changes.empty) { let ranges: ChangedRange[] = [] changes.iterChangedRanges((fromA, toA, fromB, toB) => ranges.push({fromA, toA, fromB, toB})) fragments = TreeFragment.applyChanges(fragments, ranges) tree = Tree.empty treeLen = 0 viewport = {from: changes.mapPos(viewport.from, -1), to: changes.mapPos(viewport.to, 1)} if (this.skipped.length) { skipped = [] for (let r of this.skipped) { let from = changes.mapPos(r.from, 1), to = changes.mapPos(r.to, -1) if (from < to) skipped.push({from, to}) } } } return new ParseContext(this.parser, newState, fragments, tree, treeLen, viewport, skipped, this.scheduleOn) } /// @internal updateViewport(viewport: {from: number, to: number}) { if (this.viewport.from == viewport.from && this.viewport.to == viewport.to) return false this.viewport = viewport let startLen = this.skipped.length for (let i = 0; i < this.skipped.length; i++) { let {from, to} = this.skipped[i] if (from < viewport.to && to > viewport.from) { this.fragments = cutFragments(this.fragments, from, to) this.skipped.splice(i--, 1) } } if (this.skipped.length >= startLen) return false this.reset() return true } /// @internal reset() { if (this.parse) { this.takeTree() this.parse = null } } /// Notify the parse scheduler that the given region was skipped /// because it wasn't in view, and the parse should be restarted /// when it comes into view. skipUntilInView(from: number, to: number) { this.skipped.push({from, to}) } /// Returns a parser intended to be used as placeholder when /// asynchronously loading a nested parser. It'll skip its input and /// mark it as not-really-parsed, so that the next update will parse /// it again. /// /// When `until` is given, a reparse will be scheduled when that /// promise resolves. static getSkippingParser(until?: Promise): Parser { return new class extends Parser { createParse( input: Input, fragments: readonly TreeFragment[], ranges: readonly {from: number, to: number}[] ): PartialParse { let from = ranges[0].from, to = ranges[ranges.length - 1].to let parser = { parsedPos: from, advance() { let cx = currentContext if (cx) { for (let r of ranges) cx.tempSkipped.push(r) if (until) cx.scheduleOn = cx.scheduleOn ? Promise.all([cx.scheduleOn, until]) : until } this.parsedPos = to return new Tree(NodeType.none, [], [], to - from) }, stoppedAt: null, stopAt() {} } return parser } } } /// @internal isDone(upto: number) { upto = Math.min(upto, this.state.doc.length) let frags = this.fragments return this.treeLen >= upto && frags.length && frags[0].from == 0 && frags[0].to >= upto } /// Get the context for the current parse, or `null` if no editor /// parse is in progress. static get() { return currentContext } } function cutFragments(fragments: readonly TreeFragment[], from: number, to: number) { return TreeFragment.applyChanges(fragments, [{fromA: from, toA: to, fromB: from, toB: to}]) } class LanguageState { // The current tree. Immutable, because directly accessible from // the editor state. readonly tree: Tree constructor( // A mutable parse state that is used to preserve work done during // the lifetime of a state when moving to the next state. readonly context: ParseContext ) { this.tree = context.tree } apply(tr: Transaction) { if (!tr.docChanged && this.tree == this.context.tree) return this let newCx = this.context.changes(tr.changes, tr.state) // If the previous parse wasn't done, go forward only up to its // end position or the end of the viewport, to avoid slowing down // state updates with parse work beyond the viewport. let upto = this.context.treeLen == tr.startState.doc.length ? undefined : Math.max(tr.changes.mapPos(this.context.treeLen), newCx.viewport.to) if (!newCx.work(Work.Apply, upto)) newCx.takeTree() return new LanguageState(newCx) } static init(state: EditorState) { let vpTo = Math.min(Work.InitViewport, state.doc.length) let parseState = ParseContext.create(state.facet(language)!.parser, state, {from: 0, to: vpTo}) if (!parseState.work(Work.Apply, vpTo)) parseState.takeTree() return new LanguageState(parseState) } } Language.state = StateField.define({ create: LanguageState.init, update(value, tr) { for (let e of tr.effects) if (e.is(Language.setState)) return e.value if (tr.startState.facet(language) != tr.state.facet(language)) return LanguageState.init(tr.state) return value.apply(tr) } }) let requestIdle = (callback: (deadline?: IdleDeadline) => void) => { let timeout = setTimeout(() => callback(), Work.MaxPause) return () => clearTimeout(timeout) } if (typeof requestIdleCallback != "undefined") requestIdle = (callback: (deadline?: IdleDeadline) => void) => { let idle = -1, timeout = setTimeout(() => { idle = requestIdleCallback(callback, {timeout: Work.MaxPause - Work.MinPause}) }, Work.MinPause) return () => idle < 0 ? clearTimeout(timeout) : cancelIdleCallback(idle) } const isInputPending = typeof navigator != "undefined" && (navigator as any).scheduling?.isInputPending ? () => (navigator as any).scheduling.isInputPending() : null const parseWorker = ViewPlugin.fromClass(class ParseWorker { working: (() => void) | null = null workScheduled = 0 // End of the current time chunk chunkEnd = -1 // Milliseconds of budget left for this chunk chunkBudget = -1 constructor(readonly view: EditorView) { this.work = this.work.bind(this) this.scheduleWork() } update(update: ViewUpdate) { let cx = this.view.state.field(Language.state).context if (cx.updateViewport(update.view.viewport) || this.view.viewport.to > cx.treeLen) this.scheduleWork() if (update.docChanged || update.selectionSet) { if (this.view.hasFocus) this.chunkBudget += Work.ChangeBonus this.scheduleWork() } this.checkAsyncSchedule(cx) } scheduleWork() { if (this.working) return let {state} = this.view, field = state.field(Language.state) if (field.tree != field.context.tree || !field.context.isDone(state.doc.length)) this.working = requestIdle(this.work) } work(deadline?: IdleDeadline) { this.working = null let now = Date.now() if (this.chunkEnd < now && (this.chunkEnd < 0 || this.view.hasFocus)) { // Start a new chunk this.chunkEnd = now + Work.ChunkTime this.chunkBudget = Work.ChunkBudget } if (this.chunkBudget <= 0) return // No more budget let {state, viewport: {to: vpTo}} = this.view, field = state.field(Language.state) if (field.tree == field.context.tree && field.context.isDone(vpTo + Work.MaxParseAhead)) return let endTime = Date.now() + Math.min( this.chunkBudget, Work.Slice, deadline && !isInputPending ? Math.max(Work.MinSlice, deadline.timeRemaining() - 5) : 1e9) let viewportFirst = field.context.treeLen < vpTo && state.doc.length > vpTo + 1000 let done = field.context.work(() => { return isInputPending && isInputPending() || Date.now() > endTime } , vpTo + (viewportFirst ? 0 : Work.MaxParseAhead)) this.chunkBudget -= Date.now() - now if (done || this.chunkBudget <= 0) { field.context.takeTree() this.view.dispatch({effects: Language.setState.of(new LanguageState(field.context))}) } if (this.chunkBudget > 0 && !(done && !viewportFirst)) this.scheduleWork() this.checkAsyncSchedule(field.context) } checkAsyncSchedule(cx: ParseContext) { if (cx.scheduleOn) { this.workScheduled++ cx.scheduleOn .then(() => this.scheduleWork()) .catch(err => logException(this.view.state, err)) .then(() => this.workScheduled--) cx.scheduleOn = null } } destroy() { if (this.working) this.working() } isWorking() { return !!(this.working || this.workScheduled > 0) } }, { eventHandlers: {focus() { this.scheduleWork() }} }) /// The facet used to associate a language with an editor state. Used /// by `Language` object's `extension` property (so you don't need to /// manually wrap your languages in this). Can be used to access the /// current language on a state. export const language = Facet.define({ combine(languages) { return languages.length ? languages[0] : null }, enables: language => [ Language.state, parseWorker, EditorView.contentAttributes.compute([language], state => { let lang = state.facet(language) return lang && lang.name ? {"data-language": lang.name} : {} as {} }) ] }) /// This class bundles a [language](#language.Language) with an /// optional set of supporting extensions. Language packages are /// encouraged to export a function that optionally takes a /// configuration object and returns a `LanguageSupport` instance, as /// the main way for client code to use the package. export class LanguageSupport { /// An extension including both the language and its support /// extensions. (Allowing the object to be used as an extension /// value itself.) extension: Extension /// Create a language support object. constructor( /// The language object. readonly language: Language, /// An optional set of supporting extensions. When nesting a /// language in another language, the outer language is encouraged /// to include the supporting extensions for its inner languages /// in its own set of support extensions. readonly support: Extension = [] ) { this.extension = [language, support] } } /// Language descriptions are used to store metadata about languages /// and to dynamically load them. Their main role is finding the /// appropriate language for a filename or dynamically loading nested /// parsers. export class LanguageDescription { private loading: Promise | null = null private constructor( /// The name of this language. readonly name: string, /// Alternative names for the mode (lowercased, includes `this.name`). readonly alias: readonly string[], /// File extensions associated with this language. readonly extensions: readonly string[], /// Optional filename pattern that should be associated with this /// language. readonly filename: RegExp | undefined, private loadFunc: () => Promise, /// If the language has been loaded, this will hold its value. public support: LanguageSupport | undefined = undefined ) {} /// Start loading the the language. Will return a promise that /// resolves to a [`LanguageSupport`](#language.LanguageSupport) /// object when the language successfully loads. load(): Promise { return this.loading || (this.loading = this.loadFunc().then( support => this.support = support, err => { this.loading = null; throw err } )) } /// Create a language description. static of(spec: { /// The language's name. name: string, /// An optional array of alternative names. alias?: readonly string[], /// An optional array of filename extensions associated with this /// language. extensions?: readonly string[], /// An optional filename pattern associated with this language. filename?: RegExp, /// A function that will asynchronously load the language. load?: () => Promise, /// Alternatively to `load`, you can provide an already loaded /// support object. Either this or `load` should be provided. support?: LanguageSupport }) { let {load, support} = spec if (!load) { if (!support) throw new RangeError("Must pass either 'load' or 'support' to LanguageDescription.of") load = () => Promise.resolve(support!) } return new LanguageDescription(spec.name, (spec.alias || []).concat(spec.name).map(s => s.toLowerCase()), spec.extensions || [], spec.filename, load, support) } /// Look for a language in the given array of descriptions that /// matches the filename. Will first match /// [`filename`](#language.LanguageDescription.filename) patterns, /// and then [extensions](#language.LanguageDescription.extensions), /// and return the first language that matches. static matchFilename(descs: readonly LanguageDescription[], filename: string) { for (let d of descs) if (d.filename && d.filename.test(filename)) return d let ext = /\.([^.]+)$/.exec(filename) if (ext) for (let d of descs) if (d.extensions.indexOf(ext[1]) > -1) return d return null } /// Look for a language whose name or alias matches the the given /// name (case-insensitively). If `fuzzy` is true, and no direct /// matchs is found, this'll also search for a language whose name /// or alias occurs in the string (for names shorter than three /// characters, only when surrounded by non-word characters). static matchLanguageName(descs: readonly LanguageDescription[], name: string, fuzzy = true) { name = name.toLowerCase() for (let d of descs) if (d.alias.some(a => a == name)) return d if (fuzzy) for (let d of descs) for (let a of d.alias) { let found = name.indexOf(a) if (found > -1 && (a.length > 2 || !/\w/.test(name[found - 1]) && !/\w/.test(name[found + a.length]))) return d } return null } } language-6.10.1/src/matchbrackets.ts000066400000000000000000000211101455715122100173320ustar00rootroot00000000000000import {combineConfig, EditorState, Facet, StateField, Extension, Range} from "@codemirror/state" import {syntaxTree} from "./language" import {EditorView, Decoration, DecorationSet} from "@codemirror/view" import {Tree, SyntaxNode, SyntaxNodeRef, NodeType, NodeProp} from "@lezer/common" export interface Config { /// Whether the bracket matching should look at the character after /// the cursor when matching (if the one before isn't a bracket). /// Defaults to true. afterCursor?: boolean /// The bracket characters to match, as a string of pairs. Defaults /// to `"()[]{}"`. Note that these are only used as fallback when /// there is no [matching /// information](https://lezer.codemirror.net/docs/ref/#common.NodeProp^closedBy) /// in the syntax tree. brackets?: string /// The maximum distance to scan for matching brackets. This is only /// relevant for brackets not encoded in the syntax tree. Defaults /// to 10 000. maxScanDistance?: number /// Can be used to configure the way in which brackets are /// decorated. The default behavior is to add the /// `cm-matchingBracket` class for matching pairs, and /// `cm-nonmatchingBracket` for mismatched pairs or single brackets. renderMatch?: (match: MatchResult, state: EditorState) => readonly Range[] } const baseTheme = EditorView.baseTheme({ "&.cm-focused .cm-matchingBracket": {backgroundColor: "#328c8252"}, "&.cm-focused .cm-nonmatchingBracket": {backgroundColor: "#bb555544"} }) const DefaultScanDist = 10000, DefaultBrackets = "()[]{}" const bracketMatchingConfig = Facet.define>({ combine(configs) { return combineConfig(configs, { afterCursor: true, brackets: DefaultBrackets, maxScanDistance: DefaultScanDist, renderMatch: defaultRenderMatch }) } }) const matchingMark = Decoration.mark({class: "cm-matchingBracket"}), nonmatchingMark = Decoration.mark({class: "cm-nonmatchingBracket"}) function defaultRenderMatch(match: MatchResult) { let decorations = [] let mark = match.matched ? matchingMark : nonmatchingMark decorations.push(mark.range(match.start.from, match.start.to)) if (match.end) decorations.push(mark.range(match.end.from, match.end.to)) return decorations } const bracketMatchingState = StateField.define({ create() { return Decoration.none }, update(deco, tr) { if (!tr.docChanged && !tr.selection) return deco let decorations: Range[] = [] let config = tr.state.facet(bracketMatchingConfig) for (let range of tr.state.selection.ranges) { if (!range.empty) continue let match = matchBrackets(tr.state, range.head, -1, config) || (range.head > 0 && matchBrackets(tr.state, range.head - 1, 1, config)) || (config.afterCursor && (matchBrackets(tr.state, range.head, 1, config) || (range.head < tr.state.doc.length && matchBrackets(tr.state, range.head + 1, -1, config)))) if (match) decorations = decorations.concat(config.renderMatch(match, tr.state)) } return Decoration.set(decorations, true) }, provide: f => EditorView.decorations.from(f) }) const bracketMatchingUnique = [ bracketMatchingState, baseTheme ] /// Create an extension that enables bracket matching. Whenever the /// cursor is next to a bracket, that bracket and the one it matches /// are highlighted. Or, when no matching bracket is found, another /// highlighting style is used to indicate this. export function bracketMatching(config: Config = {}): Extension { return [bracketMatchingConfig.of(config), bracketMatchingUnique] } /// When larger syntax nodes, such as HTML tags, are marked as /// opening/closing, it can be a bit messy to treat the whole node as /// a matchable bracket. This node prop allows you to define, for such /// a node, a ‘handle’—the part of the node that is highlighted, and /// that the cursor must be on to activate highlighting in the first /// place. export const bracketMatchingHandle = new NodeProp<(node: SyntaxNode) => SyntaxNode | null>() function matchingNodes(node: NodeType, dir: -1 | 1, brackets: string): null | readonly string[] { let byProp = node.prop(dir < 0 ? NodeProp.openedBy : NodeProp.closedBy) if (byProp) return byProp if (node.name.length == 1) { let index = brackets.indexOf(node.name) if (index > -1 && index % 2 == (dir < 0 ? 1 : 0)) return [brackets[index + dir]] } return null } /// The result returned from `matchBrackets`. export interface MatchResult { /// The extent of the bracket token found. start: {from: number, to: number}, /// The extent of the matched token, if any was found. end?: {from: number, to: number}, /// Whether the tokens match. This can be false even when `end` has /// a value, if that token doesn't match the opening token. matched: boolean } function findHandle(node: SyntaxNodeRef) { let hasHandle = node.type.prop(bracketMatchingHandle) return hasHandle ? hasHandle(node.node) : node } /// Find the matching bracket for the token at `pos`, scanning /// direction `dir`. Only the `brackets` and `maxScanDistance` /// properties are used from `config`, if given. Returns null if no /// bracket was found at `pos`, or a match result otherwise. export function matchBrackets(state: EditorState, pos: number, dir: -1 | 1, config: Config = {}): MatchResult | null { let maxScanDistance = config.maxScanDistance || DefaultScanDist, brackets = config.brackets || DefaultBrackets let tree = syntaxTree(state), node = tree.resolveInner(pos, dir) for (let cur: SyntaxNode | null = node; cur; cur = cur.parent) { let matches = matchingNodes(cur.type, dir, brackets) if (matches && cur.from < cur.to) { let handle = findHandle(cur) if (handle && (dir > 0 ? pos >= handle.from && pos < handle.to : pos > handle.from && pos <= handle.to)) return matchMarkedBrackets(state, pos, dir, cur, handle, matches, brackets) } } return matchPlainBrackets(state, pos, dir, tree, node.type, maxScanDistance, brackets) } function matchMarkedBrackets(_state: EditorState, _pos: number, dir: -1 | 1, token: SyntaxNode, handle: SyntaxNodeRef, matching: readonly string[], brackets: string) { let parent = token.parent, firstToken = {from: handle.from, to: handle.to} let depth = 0, cursor = parent?.cursor() if (cursor && (dir < 0 ? cursor.childBefore(token.from) : cursor.childAfter(token.to))) do { if (dir < 0 ? cursor.to <= token.from : cursor.from >= token.to) { if (depth == 0 && matching.indexOf(cursor.type.name) > -1 && cursor.from < cursor.to) { let endHandle = findHandle(cursor) return {start: firstToken, end: endHandle ? {from: endHandle.from, to: endHandle.to} : undefined, matched: true} } else if (matchingNodes(cursor.type, dir, brackets)) { depth++ } else if (matchingNodes(cursor.type, -dir as -1 | 1, brackets)) { if (depth == 0) { let endHandle = findHandle(cursor) return { start: firstToken, end: endHandle && endHandle.from < endHandle.to ? {from: endHandle.from, to: endHandle.to} : undefined, matched: false } } depth-- } } } while (dir < 0 ? cursor.prevSibling() : cursor.nextSibling()) return {start: firstToken, matched: false} } function matchPlainBrackets(state: EditorState, pos: number, dir: number, tree: Tree, tokenType: NodeType, maxScanDistance: number, brackets: string) { let startCh = dir < 0 ? state.sliceDoc(pos - 1, pos) : state.sliceDoc(pos, pos + 1) let bracket = brackets.indexOf(startCh) if (bracket < 0 || (bracket % 2 == 0) != (dir > 0)) return null let startToken = {from: dir < 0 ? pos - 1 : pos, to: dir > 0 ? pos + 1 : pos} let iter = state.doc.iterRange(pos, dir > 0 ? state.doc.length : 0), depth = 0 for (let distance = 0; !(iter.next()).done && distance <= maxScanDistance;) { let text = iter.value if (dir < 0) distance += text.length let basePos = pos + distance * dir for (let pos = dir > 0 ? 0 : text.length - 1, end = dir > 0 ? text.length : -1; pos != end; pos += dir) { let found = brackets.indexOf(text[pos]) if (found < 0 || tree.resolveInner(basePos + pos, 1).type != tokenType) continue if ((found % 2 == 0) == (dir > 0)) { depth++ } else if (depth == 1) { // Closing return {start: startToken, end: {from: basePos + pos, to: basePos + pos + 1}, matched: (found >> 1) == (bracket >> 1)} } else { depth-- } } if (dir > 0) distance += text.length } return iter.done ? {start: startToken, matched: false} : null } language-6.10.1/src/stream-parser.ts000066400000000000000000000376021455715122100173210ustar00rootroot00000000000000import {Tree, Input, TreeFragment, NodeType, NodeSet, SyntaxNode, PartialParse, Parser, NodeProp} from "@lezer/common" import {Tag, tags as highlightTags, styleTags} from "@lezer/highlight" import {EditorState, Facet} from "@codemirror/state" import {Language, defineLanguageFacet, languageDataProp, syntaxTree, ParseContext} from "./language" import {IndentContext, indentService, getIndentUnit} from "./indent" import {StringStream} from "./stringstream" export {StringStream} /// A stream parser parses or tokenizes content from start to end, /// emitting tokens as it goes over it. It keeps a mutable (but /// copyable) object with state, in which it can store information /// about the current context. export interface StreamParser { /// A name for this language. name?: string /// Produce a start state for the parser. startState?(indentUnit: number): State /// Read one token, advancing the stream past it, and returning a /// string indicating the token's style tag—either the name of one /// of the tags in /// [`tags`](https://lezer.codemirror.net/docs/ref#highlight.tags) /// or [`tokenTable`](#language.StreamParser.tokenTable), or such a /// name suffixed by one or more tag /// [modifier](https://lezer.codemirror.net/docs/ref#highlight.Tag^defineModifier) /// names, separated by periods. For example `"keyword"` or /// "`variableName.constant"`, or a space-separated set of such /// token types. /// /// It is okay to return a zero-length token, but only if that /// updates the state so that the next call will return a non-empty /// token again. token(stream: StringStream, state: State): string | null /// This notifies the parser of a blank line in the input. It can /// update its state here if it needs to. blankLine?(state: State, indentUnit: number): void /// Copy a given state. By default, a shallow object copy is done /// which also copies arrays held at the top level of the object. copyState?(state: State): State /// Compute automatic indentation for the line that starts with the /// given state and text. indent?(state: State, textAfter: string, context: IndentContext): number | null /// Default [language data](#state.EditorState.languageDataAt) to /// attach to this language. languageData?: {[name: string]: any} /// Extra tokens to use in this parser. When the tokenizer returns a /// token name that exists as a property in this object, the /// corresponding tags will be assigned to the token. tokenTable?: {[name: string]: Tag | readonly Tag[]} } function fullParser(spec: StreamParser): Required> { return { name: spec.name || "", token: spec.token, blankLine: spec.blankLine || (() => {}), startState: spec.startState || (() => (true as any)), copyState: spec.copyState || defaultCopyState, indent: spec.indent || (() => null), languageData: spec.languageData || {}, tokenTable: spec.tokenTable || noTokens } } function defaultCopyState(state: State) { if (typeof state != "object") return state let newState = {} as State for (let prop in state) { let val = state[prop] newState[prop] = (val instanceof Array ? val.slice() : val) as any } return newState } const IndentedFrom = new WeakMap() /// A [language](#language.Language) class based on a CodeMirror /// 5-style [streaming parser](#language.StreamParser). export class StreamLanguage extends Language { /// @internal streamParser: Required> /// @internal stateAfter: NodeProp /// @internal tokenTable: TokenTable /// @internal topNode: NodeType private constructor(parser: StreamParser) { let data = defineLanguageFacet(parser.languageData) let p = fullParser(parser), self: StreamLanguage let impl = new class extends Parser { createParse(input: Input, fragments: readonly TreeFragment[], ranges: readonly {from: number, to: number}[]) { return new Parse(self, input, fragments, ranges) } } super(data, impl, [indentService.of((cx, pos) => this.getIndent(cx, pos))], parser.name) this.topNode = docID(data) self = this this.streamParser = p this.stateAfter = new NodeProp({perNode: true}) this.tokenTable = parser.tokenTable ? new TokenTable(p.tokenTable) : defaultTokenTable } /// Define a stream language. static define(spec: StreamParser) { return new StreamLanguage(spec) } private getIndent(cx: IndentContext, pos: number) { let tree = syntaxTree(cx.state), at: SyntaxNode | null = tree.resolve(pos) while (at && at.type != this.topNode) at = at.parent if (!at) return null let from = undefined let {overrideIndentation} = cx.options if (overrideIndentation) { from = IndentedFrom.get(cx.state) if (from != null && from < pos - 1e4) from = undefined } let start = findState(this, tree, 0, at.from, from ?? pos), statePos, state if (start) { state = start.state; statePos = start.pos + 1 } else { state = this.streamParser.startState(cx.unit) ; statePos = 0 } if (pos - statePos > C.MaxIndentScanDist) return null while (statePos < pos) { let line = cx.state.doc.lineAt(statePos), end = Math.min(pos, line.to) if (line.length) { let indentation = overrideIndentation ? overrideIndentation(line.from) : -1 let stream = new StringStream(line.text, cx.state.tabSize, cx.unit, indentation < 0 ? undefined : indentation) while (stream.pos < end - line.from) readToken(this.streamParser.token, stream, state) } else { this.streamParser.blankLine(state, cx.unit) } if (end == pos) break statePos = line.to + 1 } let line = cx.lineAt(pos) if (overrideIndentation && from == null) IndentedFrom.set(cx.state, line.from) return this.streamParser.indent(state, /^\s*(.*)/.exec(line.text)![1], cx) } get allowsNesting() { return false } } function findState( lang: StreamLanguage, tree: Tree, off: number, startPos: number, before: number ): {state: State, pos: number} | null { let state = off >= startPos && off + tree.length <= before && tree.prop(lang.stateAfter) if (state) return {state: lang.streamParser.copyState(state), pos: off + tree.length} for (let i = tree.children.length - 1; i >= 0; i--) { let child = tree.children[i], pos = off + tree.positions[i] let found = child instanceof Tree && pos < before && findState(lang, child, pos, startPos, before) if (found) return found } return null } function cutTree(lang: StreamLanguage, tree: Tree, from: number, to: number, inside: boolean): Tree | null { if (inside && from <= 0 && to >= tree.length) return tree if (!inside && tree.type == lang.topNode) inside = true for (let i = tree.children.length - 1; i >= 0; i--) { let pos = tree.positions[i], child = tree.children[i], inner if (pos < to && child instanceof Tree) { if (!(inner = cutTree(lang, child, from - pos, to - pos, inside))) break return !inside ? inner : new Tree(tree.type, tree.children.slice(0, i).concat(inner), tree.positions.slice(0, i + 1), pos + inner.length) } } return null } function findStartInFragments(lang: StreamLanguage, fragments: readonly TreeFragment[], startPos: number, editorState?: EditorState) { for (let f of fragments) { let from = f.from + (f.openStart ? 25 : 0), to = f.to - (f.openEnd ? 25 : 0) let found = from <= startPos && to > startPos && findState(lang, f.tree, 0 - f.offset, startPos, to), tree if (found && (tree = cutTree(lang, f.tree, startPos + f.offset, found.pos + f.offset, false))) return {state: found.state, tree} } return {state: lang.streamParser.startState(editorState ? getIndentUnit(editorState) : 4), tree: Tree.empty} } const enum C { ChunkSize = 2048, MaxDistanceBeforeViewport = 1e5, MaxIndentScanDist = 1e4, MaxLineLength = 1e4 } class Parse implements PartialParse { state: State parsedPos: number stoppedAt: number | null = null chunks: Tree[] = [] chunkPos: number[] = [] chunkStart: number chunk: number[] = [] chunkReused: undefined | Tree[] = undefined rangeIndex = 0 to: number constructor(readonly lang: StreamLanguage, readonly input: Input, readonly fragments: readonly TreeFragment[], readonly ranges: readonly {from: number, to: number}[]) { this.to = ranges[ranges.length - 1].to let context = ParseContext.get(), from = ranges[0].from let {state, tree} = findStartInFragments(lang, fragments, from, context?.state) this.state = state this.parsedPos = this.chunkStart = from + tree.length for (let i = 0; i < tree.children.length; i++) { this.chunks.push(tree.children[i] as Tree) this.chunkPos.push(tree.positions[i]) } if (context && this.parsedPos < context.viewport.from - C.MaxDistanceBeforeViewport) { this.state = this.lang.streamParser.startState(getIndentUnit(context.state)) context.skipUntilInView(this.parsedPos, context.viewport.from) this.parsedPos = context.viewport.from } this.moveRangeIndex() } advance() { let context = ParseContext.get() let parseEnd = this.stoppedAt == null ? this.to : Math.min(this.to, this.stoppedAt) let end = Math.min(parseEnd, this.chunkStart + C.ChunkSize) if (context) end = Math.min(end, context.viewport.to) while (this.parsedPos < end) this.parseLine(context) if (this.chunkStart < this.parsedPos) this.finishChunk() if (this.parsedPos >= parseEnd) return this.finish() if (context && this.parsedPos >= context.viewport.to) { context.skipUntilInView(this.parsedPos, parseEnd) return this.finish() } return null } stopAt(pos: number) { this.stoppedAt = pos } lineAfter(pos: number) { let chunk = this.input.chunk(pos) if (!this.input.lineChunks) { let eol = chunk.indexOf("\n") if (eol > -1) chunk = chunk.slice(0, eol) } else if (chunk == "\n") { chunk = "" } return pos + chunk.length <= this.to ? chunk : chunk.slice(0, this.to - pos) } nextLine() { let from = this.parsedPos, line = this.lineAfter(from), end = from + line.length for (let index = this.rangeIndex;;) { let rangeEnd = this.ranges[index].to if (rangeEnd >= end) break line = line.slice(0, rangeEnd - (end - line.length)) index++ if (index == this.ranges.length) break let rangeStart = this.ranges[index].from let after = this.lineAfter(rangeStart) line += after end = rangeStart + after.length } return {line, end} } skipGapsTo(pos: number, offset: number, side: -1 | 1) { for (;;) { let end = this.ranges[this.rangeIndex].to, offPos = pos + offset if (side > 0 ? end > offPos : end >= offPos) break let start = this.ranges[++this.rangeIndex].from offset += start - end } return offset } moveRangeIndex() { while (this.ranges[this.rangeIndex].to < this.parsedPos) this.rangeIndex++ } emitToken(id: number, from: number, to: number, size: number, offset: number) { if (this.ranges.length > 1) { offset = this.skipGapsTo(from, offset, 1) from += offset let len0 = this.chunk.length offset = this.skipGapsTo(to, offset, -1) to += offset size += this.chunk.length - len0 } this.chunk.push(id, from, to, size) return offset } parseLine(context: ParseContext | null) { let {line, end} = this.nextLine(), offset = 0, {streamParser} = this.lang let stream = new StringStream(line, context ? context.state.tabSize : 4, context ? getIndentUnit(context.state) : 2) if (stream.eol()) { streamParser.blankLine(this.state, stream.indentUnit) } else { while (!stream.eol()) { let token = readToken(streamParser.token, stream, this.state) if (token) offset = this.emitToken(this.lang.tokenTable.resolve(token), this.parsedPos + stream.start, this.parsedPos + stream.pos, 4, offset) if (stream.start > C.MaxLineLength) break } } this.parsedPos = end this.moveRangeIndex() if (this.parsedPos < this.to) this.parsedPos++ } finishChunk() { let tree = Tree.build({ buffer: this.chunk, start: this.chunkStart, length: this.parsedPos - this.chunkStart, nodeSet, topID: 0, maxBufferLength: C.ChunkSize, reused: this.chunkReused }) tree = new Tree(tree.type, tree.children, tree.positions, tree.length, [[this.lang.stateAfter, this.lang.streamParser.copyState(this.state)]]) this.chunks.push(tree) this.chunkPos.push(this.chunkStart - this.ranges[0].from) this.chunk = [] this.chunkReused = undefined this.chunkStart = this.parsedPos } finish() { return new Tree(this.lang.topNode, this.chunks, this.chunkPos, this.parsedPos - this.ranges[0].from).balance() } } function readToken(token: (stream: StringStream, state: State) => string | null, stream: StringStream, state: State) { stream.start = stream.pos for (let i = 0; i < 10; i++) { let result = token(stream, state) if (stream.pos > stream.start) return result } throw new Error("Stream parser failed to advance stream.") } const noTokens: {[name: string]: Tag} = Object.create(null) const typeArray: NodeType[] = [NodeType.none] const nodeSet = new NodeSet(typeArray) const warned: string[] = [] // Cache of node types by name and tags const byTag: {[key: string]: NodeType} = Object.create(null) const defaultTable: {[name: string]: number} = Object.create(null) for (let [legacyName, name] of [ ["variable", "variableName"], ["variable-2", "variableName.special"], ["string-2", "string.special"], ["def", "variableName.definition"], ["tag", "tagName"], ["attribute", "attributeName"], ["type", "typeName"], ["builtin", "variableName.standard"], ["qualifier", "modifier"], ["error", "invalid"], ["header", "heading"], ["property", "propertyName"] ]) defaultTable[legacyName] = createTokenType(noTokens, name) class TokenTable { table: {[name: string]: number} = Object.assign(Object.create(null), defaultTable) constructor(readonly extra: {[name: string]: Tag | readonly Tag[]}) {} resolve(tag: string) { return !tag ? 0 : this.table[tag] || (this.table[tag] = createTokenType(this.extra, tag)) } } const defaultTokenTable = new TokenTable(noTokens) function warnForPart(part: string, msg: string) { if (warned.indexOf(part) > -1) return warned.push(part) console.warn(msg) } function createTokenType(extra: {[name: string]: Tag | readonly Tag[]}, tagStr: string) { let tags = [] for (let name of tagStr.split(" ")) { let found: readonly Tag[] = [] for (let part of name.split(".")) { let value = (extra[part] || (highlightTags as any)[part]) as Tag | readonly Tag[] | ((t: Tag) => Tag) | undefined if (!value) { warnForPart(part, `Unknown highlighting tag ${part}`) } else if (typeof value == "function") { if (!found.length) warnForPart(part, `Modifier ${part} used at start of tag`) else found = found.map(value) as Tag[] } else { if (found.length) warnForPart(part, `Tag ${part} used as modifier`) else found = Array.isArray(value) ? value : [value] } } for (let tag of found) tags.push(tag) } if (!tags.length) return 0 let name = tagStr.replace(/ /g, "_"), key = name + " " + tags.map(t => (t as any).id) let known = byTag[key] if (known) return known.id let type = byTag[key] = NodeType.define({ id: typeArray.length, name, props: [styleTags({[name]: tags})] }) typeArray.push(type) return type.id } function docID(data: Facet<{[name: string]: any}>) { let type = NodeType.define({id: typeArray.length, name: "Document", props: [languageDataProp.add(() => data)], top: true}) typeArray.push(type) return type } language-6.10.1/src/stringstream.ts000066400000000000000000000106541455715122100172540ustar00rootroot00000000000000// Counts the column offset in a string, taking tabs into account. // Used mostly to find indentation. function countCol(string: string, end: number | null, tabSize: number, startIndex = 0, startValue = 0): number { if (end == null) { end = string.search(/[^\s\u00a0]/) if (end == -1) end = string.length } let n = startValue for (let i = startIndex; i < end; i++) { if (string.charCodeAt(i) == 9) n += tabSize - (n % tabSize) else n++ } return n } /// Encapsulates a single line of input. Given to stream syntax code, /// which uses it to tokenize the content. export class StringStream { /// The current position on the line. pos: number = 0 /// The start position of the current token. start: number = 0 private lastColumnPos: number = 0 private lastColumnValue: number = 0 /// Create a stream. constructor( /// The line. public string: string, private tabSize: number, /// The current indent unit size. public indentUnit: number, private overrideIndent?: number ) {} /// True if we are at the end of the line. eol(): boolean {return this.pos >= this.string.length} /// True if we are at the start of the line. sol(): boolean {return this.pos == 0} /// Get the next code unit after the current position, or undefined /// if we're at the end of the line. peek() {return this.string.charAt(this.pos) || undefined} /// Read the next code unit and advance `this.pos`. next(): string | void { if (this.pos < this.string.length) return this.string.charAt(this.pos++) } /// Match the next character against the given string, regular /// expression, or predicate. Consume and return it if it matches. eat(match: string | RegExp | ((ch: string) => boolean)): string | void { let ch = this.string.charAt(this.pos) let ok if (typeof match == "string") ok = ch == match else ok = ch && (match instanceof RegExp ? match.test(ch) : match(ch)) if (ok) {++this.pos; return ch} } /// Continue matching characters that match the given string, /// regular expression, or predicate function. Return true if any /// characters were consumed. eatWhile(match: string | RegExp | ((ch: string) => boolean)): boolean { let start = this.pos while (this.eat(match)){} return this.pos > start } /// Consume whitespace ahead of `this.pos`. Return true if any was /// found. eatSpace() { let start = this.pos while (/[\s\u00a0]/.test(this.string.charAt(this.pos))) ++this.pos return this.pos > start } /// Move to the end of the line. skipToEnd() {this.pos = this.string.length} /// Move to directly before the given character, if found on the /// current line. skipTo(ch: string): boolean | void { let found = this.string.indexOf(ch, this.pos) if (found > -1) {this.pos = found; return true} } /// Move back `n` characters. backUp(n: number) {this.pos -= n} /// Get the column position at `this.pos`. column() { if (this.lastColumnPos < this.start) { this.lastColumnValue = countCol(this.string, this.start, this.tabSize, this.lastColumnPos, this.lastColumnValue) this.lastColumnPos = this.start } return this.lastColumnValue } /// Get the indentation column of the current line. indentation() { return this.overrideIndent ?? countCol(this.string, null, this.tabSize) } /// Match the input against the given string or regular expression /// (which should start with a `^`). Return true or the regexp match /// if it matches. /// /// Unless `consume` is set to `false`, this will move `this.pos` /// past the matched text. /// /// When matching a string `caseInsensitive` can be set to true to /// make the match case-insensitive. match(pattern: string | RegExp, consume?: boolean, caseInsensitive?: boolean): boolean | RegExpMatchArray | null { if (typeof pattern == "string") { let cased = (str: string) => caseInsensitive ? str.toLowerCase() : str let substr = this.string.substr(this.pos, pattern.length) if (cased(substr) == cased(pattern)) { if (consume !== false) this.pos += pattern.length return true } else return null } else { let match = this.string.slice(this.pos).match(pattern) if (match && match.index! > 0) return null if (match && consume !== false) this.pos += match[0].length return match } } /// Get the current token. current(){return this.string.slice(this.start, this.pos)} } language-6.10.1/test/000077500000000000000000000000001455715122100143445ustar00rootroot00000000000000language-6.10.1/test/test-fold.ts000066400000000000000000000022371455715122100166210ustar00rootroot00000000000000import ist from "ist" import {foldEffect, unfoldEffect, foldState} from "@codemirror/language" import {EditorState} from "@codemirror/state" import {DecorationSet} from "@codemirror/view" let doc = "1\n2\n3\n4\n5\n6\n7\n8\n" function ranges(set: DecorationSet) { let result: string[] = [] set.between(0, 1e8, (f, t) => {result.push(`${f}-${t}`)}) return result.join(" ") } describe("Folding", () => { it("stores fold state", () => { let state = EditorState.create({doc, extensions: foldState}).update({ effects: [foldEffect.of({from: 0, to: 3}), foldEffect.of({from: 4, to: 7})] }).state ist(ranges(state.field(foldState)), "0-3 4-7") state = state.update({ effects: unfoldEffect.of({from: 4, to: 7}) }).state ist(ranges(state.field(foldState)), "0-3") }) it("can store fold state as JSON", () => { let state = EditorState.create({doc, extensions: foldState}).update({ effects: [foldEffect.of({from: 4, to: 7}), foldEffect.of({from: 8, to: 11})] }).state let fields = {fold: foldState} state = EditorState.fromJSON(state.toJSON(fields), {}, fields) ist(ranges(state.field(foldState)), "4-7 8-11") }) }) language-6.10.1/test/test-stream-parser.ts000066400000000000000000000107321455715122100204610ustar00rootroot00000000000000import ist from "ist" import {EditorState} from "@codemirror/state" import {Tag} from "@lezer/highlight" import {StreamLanguage, syntaxTree, getIndentation, Language} from "@codemirror/language" import {SyntaxNode} from "@lezer/common" let startStates = 0, keywords = ["if", "else", "return"] const language = StreamLanguage.define<{count: number}>({ startState() { startStates++ return {count: 0} }, token(stream, state) { if (stream.eatSpace()) return null state.count++ if (stream.match(/^\/\/.*/)) return "lineComment" if (stream.match(/^"[^"]*"/)) return "string" if (stream.match(/^\d+/)) return "number" if (stream.match(/^\w+/)) return keywords.indexOf(stream.current()) >= 0 ? "keyword" : "variableName" if (stream.match(/^[();{}]/)) return "punctuation" stream.next() return "invalid" }, indent(state) { return state.count } }) describe("StreamLanguage", () => { it("can parse content", () => { ist(language.parser.parse("if (x) return 500").toString(), "Document(keyword,punctuation,variableName,punctuation,keyword,number)") }) it("can reuse state on updates", () => { let state = EditorState.create({ doc: "// filler content\nif (a) foo()\nelse if (b) bar()\nelse quux()\n\n".repeat(100), extensions: language }) startStates = 0 state = state.update({changes: {from: 5000, to: 5001}}).state ist(startStates, 0) }) it("can find the correct parse state for indentation", () => { let state = EditorState.create({ doc: '"abcdefg"\n'.repeat(200), extensions: language }) ist(getIndentation(state, 0), 0) ist(getIndentation(state, 10), 1) ist(getIndentation(state, 100), 10) ist(getIndentation(state, 1000), 100) }) // Fragile kludge to set the parser context viewport without // actually having access to the relevant field function setViewport(state: EditorState, from: number, to: number) { let field = (Language as any).state ;(state.field(field) as any).context.updateViewport({from, to}) } it("will make up a state when the viewport is far away from the frontier", () => { let line = "1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0\n" let state = EditorState.create({doc: line.repeat(100), extensions: language}) setViewport(state, 4000, 8000) state = state.update({changes: {from: 3000, insert: line.repeat(10000)}}).state let tree = syntaxTree(state) // No nodes in the skipped range ist(tree.resolve(10000, 1).name, "Document") // But the viewport is populated ist(tree.resolve(805000, 1).name, "number") let treeSize = 0 tree.iterate({enter() { treeSize++ }}) ist(treeSize, 2000, ">") ist(treeSize, 4000, "<") setViewport(state, 4000, 8000) state = state.update({changes: {from: 100000, insert: "?"}}).state tree = syntaxTree(state) ist(tree.resolve(5000, 1).name, "number") ist(tree.resolve(50000, 1).name, "Document") }) it("doesn't parse beyond the viewport", () => { let line = "1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0\n" let state = EditorState.create({doc: line.repeat(100), extensions: language}) setViewport(state, 0, 4000) state = state.update({changes: {from: 5000, insert: line.repeat(100)}}).state ist(syntaxTree(state).resolve(2000, 1).name, "number") ist(syntaxTree(state).resolve(6000, 1).name, "Document") }) function isNode(node: SyntaxNode | null, name: string, from: number, to: number) { ist(node) ist(node!.type.name, name) ist(node!.from, from) ist(node!.to, to) } it("supports gaps", () => { let text = "1 50 xxx\nxxx\nxxx 60\n70 xxx80xxx 9xxx0" let ranges = [{from: 0, to: 5}, {from: 16, to: 23}, {from: 26, to: 28}, {from: 31, to: 33}, {from: 36, to: 37}] let tree = language.parser.parse(text, [], ranges) ist(tree.toString(), "Document(number,number,number,number,number,number)") isNode(tree.resolve(17, 1), "number", 17, 19) isNode(tree.resolve(20, 1), "number", 20, 22) isNode(tree.resolve(26, 1), "number", 26, 28) isNode(tree.resolve(32, 1), "number", 32, 37) }) it("accepts custom token types", () => { let tag = Tag.define() let lang = StreamLanguage.define({ token(stream) { if (stream.match(/^\w+/)) return "foo" stream.next() return null }, tokenTable: {foo: tag} }) ist(lang.parser.parse("hello").toString(), "Document(foo)") }) }) language-6.10.1/test/test-syntax.ts000066400000000000000000000045401455715122100172220ustar00rootroot00000000000000import ist from "ist" import {getIndentUnit, indentString, indentUnit, ParseContext} from "@codemirror/language" import {EditorState, ChangeSet, Text} from "@codemirror/state" import {parser} from "@lezer/javascript" let lines = `const {readFile} = require("fs"); readFile("package.json", "utf8", (err, data) => { console.log(data); }); `.split("\n") for (let l0 = lines.length, i = l0; i < 5000; i++) lines[i] = lines[i % l0] let doc = Text.of(lines) function pContext(doc: Text) { return ParseContext.create(parser, EditorState.create({doc}), {from: 0, to: doc.length}) } describe("ParseContext", () => { it("can parse a document", () => { let cx = pContext(Text.of(["let x = 10"])) cx.work(1e8) ist(cx.tree.toString(), "Script(VariableDeclaration(let,VariableDefinition,Equals,Number))") }) it("can parse incrementally", () => { let cx = pContext(doc), t0 = Date.now() if (cx.work(10)) { console.warn("Machine too fast for the incremental parsing test, skipping") return } ist(Date.now() - t0, 25, "<") ist(cx.work(1e8)) ist(cx.tree.length, doc.length) let change = ChangeSet.of({from: 0, to: 5, insert: "let"}, doc.length) let newDoc = change.apply(doc) cx = cx.changes(change, EditorState.create({doc: newDoc})) ist(cx.work(50)) ist(cx.tree.length, newDoc.length) ist(cx.tree.toString().slice(0, 31), "Script(VariableDeclaration(let,") }) }) describe("Indentation", () => { it("tracks indent units", () => { let s0 = EditorState.create({}) ist(getIndentUnit(s0), 2) ist(indentString(s0, 4), " ") let s1 = EditorState.create({extensions: indentUnit.of(" ")}) ist(getIndentUnit(s1), 3) ist(indentString(s1, 4), " ") let s2 = EditorState.create({extensions: [indentUnit.of("\t"), EditorState.tabSize.of(8)]}) ist(getIndentUnit(s2), 8) ist(indentString(s2, 16), "\t\t") let s3 = EditorState.create({extensions: indentUnit.of("　")}) ist(getIndentUnit(s3), 1) ist(indentString(s3, 2), "　　") }) it("errors for bad indent units", () => { ist.throws(() => EditorState.create({extensions: indentUnit.of("")}), /Invalid indent unit/) ist.throws(() => EditorState.create({extensions: indentUnit.of("\t ")}), /Invalid indent unit/) ist.throws(() => EditorState.create({extensions: indentUnit.of("hello")}), /Invalid indent unit/) }) })