|
| 1 | +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. |
| 2 | +// See LICENSE in the project root for license information. |
| 3 | + |
| 4 | +import { |
| 5 | + ExcerptTokenKind, |
| 6 | + type IExcerptToken, |
| 7 | + type IExcerptTokenRange |
| 8 | +} from '@microsoft/api-extractor-model'; |
| 9 | + |
| 10 | +/** |
| 11 | + * Condenses the provided excerpt tokens by merging tokens where possible. Updates the provided token ranges to |
| 12 | + * remain accurate after token merging. |
| 13 | + * |
| 14 | + * @remarks |
| 15 | + * For example, suppose we have excerpt tokens ["A", "B", "C"] and a token range [0, 2]. If the excerpt tokens |
| 16 | + * are condensed to ["AB", "C"], then the token range would be updated to [0, 1]. Note that merges are only |
| 17 | + * performed if they are compatible with the provided token ranges. In the example above, if our token range was |
| 18 | + * originally [0, 1], we would not be able to merge tokens "A" and "B". |
| 19 | + */ |
| 20 | +export function condenseTokens(excerptTokens: IExcerptToken[], tokenRanges: IExcerptTokenRange[]): void { |
| 21 | + const originalTokenCount: number = excerptTokens.length; |
| 22 | + |
| 23 | + // A token that sits at the start or end index of any token range must be preserved (never merged |
| 24 | + // away), so that every range can be accurately remapped after condensing. These indices refer to |
| 25 | + // the original positions in `excerptTokens`, and since preserved tokens are never removed, this |
| 26 | + // set never needs to be rebuilt. |
| 27 | + const preservedIndices: Set<number> = new Set(); |
| 28 | + for (const tokenRange of tokenRanges) { |
| 29 | + preservedIndices.add(tokenRange.startIndex); |
| 30 | + preservedIndices.add(tokenRange.endIndex); |
| 31 | + } |
| 32 | + |
| 33 | + // Build the condensed token list in a single forward pass, treating it as a stack so that a |
| 34 | + // token merged into its predecessor can itself be merged into a further predecessor (e.g. a |
| 35 | + // chain of reference tokens such as "A" "." "B" "." "C"). |
| 36 | + // |
| 37 | + // `newIndexByOriginalIndex` maps each kept token's original index to its index in the condensed |
| 38 | + // list, which is then used to remap the token ranges. Every range boundary refers to a preserved |
| 39 | + // token (or, for an exclusive `endIndex`, the token count), and preserved tokens are never merged |
| 40 | + // away, so a direct lookup always resolves. It is sized `originalTokenCount + 1` to hold the |
| 41 | + // mapping for an `endIndex` equal to the token count. |
| 42 | + const condensedTokens: IExcerptToken[] = []; |
| 43 | + const newIndexByOriginalIndex: Int32Array = new Int32Array(originalTokenCount + 1); |
| 44 | + |
| 45 | + // Whether the token currently on top of the stack is preserved. Only consulted when that token is |
| 46 | + // the "." of a reference merge; because a "." only ever becomes the top via a push (content tokens |
| 47 | + // are always separated by references, so a "." is never uncovered by a pop), this scalar is always |
| 48 | + // up to date at the point it is read, avoiding a parallel array of original indices. |
| 49 | + let prevTokenIsPreserved: boolean = false; |
| 50 | + |
| 51 | + for (let currentIndex: number = 0; currentIndex < originalTokenCount; ++currentIndex) { |
| 52 | + const currentToken: IExcerptToken = excerptTokens[currentIndex]; |
| 53 | + const currentIsPreserved: boolean = preservedIndices.has(currentIndex); |
| 54 | + const condensedCount: number = condensedTokens.length; |
| 55 | + |
| 56 | + // A preserved token must never be merged away, so merges are only attempted when the current |
| 57 | + // token is not preserved. There are two types of merges that can occur, and both consume the |
| 58 | + // current token. Reads of the top two stack entries are guarded so they never index out of |
| 59 | + // bounds. |
| 60 | + let merged: boolean = false; |
| 61 | + if (!currentIsPreserved && condensedCount >= 1) { |
| 62 | + const prevToken: IExcerptToken = condensedTokens[condensedCount - 1]; |
| 63 | + |
| 64 | + if ( |
| 65 | + condensedCount >= 2 && |
| 66 | + currentToken.kind === ExcerptTokenKind.Reference && |
| 67 | + prevToken.kind === ExcerptTokenKind.Content && |
| 68 | + prevToken.text.trim() === '.' && |
| 69 | + !prevTokenIsPreserved && |
| 70 | + condensedTokens[condensedCount - 2].kind === ExcerptTokenKind.Reference |
| 71 | + ) { |
| 72 | + // If the current token is a reference token, the previous token is a ".", and the previous- |
| 73 | + // previous token is a reference token, then merge all three tokens into a reference token. |
| 74 | + // |
| 75 | + // For example: Given ["MyNamespace" (R), ".", "MyClass" (R)], tokens "." and "MyClass" might |
| 76 | + // be merged into "MyNamespace". The condensed token would be ["MyNamespace.MyClass" (R)]. |
| 77 | + const prevPrevToken: IExcerptToken = condensedTokens[condensedCount - 2]; |
| 78 | + prevPrevToken.text += prevToken.text + currentToken.text; |
| 79 | + prevPrevToken.canonicalReference = currentToken.canonicalReference; |
| 80 | + |
| 81 | + // The "." token (already kept) and the current token are both merged into prevPrevToken. |
| 82 | + condensedTokens.pop(); |
| 83 | + merged = true; |
| 84 | + } else if ( |
| 85 | + // If the current and previous tokens are both content tokens, then merge the tokens into a |
| 86 | + // single content token. For example: Given ["export ", "declare class"], these tokens |
| 87 | + // might be merged into "export declare class". |
| 88 | + prevToken.kind === ExcerptTokenKind.Content && |
| 89 | + currentToken.kind === ExcerptTokenKind.Content |
| 90 | + ) { |
| 91 | + prevToken.text += currentToken.text; |
| 92 | + merged = true; |
| 93 | + } |
| 94 | + } |
| 95 | + |
| 96 | + if (!merged) { |
| 97 | + // No merging occurred, so keep the current token, record its new index, and update the |
| 98 | + // preservation flag to reflect the new top of the stack. |
| 99 | + newIndexByOriginalIndex[currentIndex] = condensedTokens.length; |
| 100 | + condensedTokens.push(currentToken); |
| 101 | + prevTokenIsPreserved = currentIsPreserved; |
| 102 | + } |
| 103 | + } |
| 104 | + |
| 105 | + // Remap the token ranges directly. Each boundary is a preserved token's original index (or the |
| 106 | + // token count, for an exclusive `endIndex`), which maps straight to its position in the condensed |
| 107 | + // list. `endIndex` is clamped because it may equal the token count. |
| 108 | + newIndexByOriginalIndex[originalTokenCount] = condensedTokens.length; |
| 109 | + for (const tokenRange of tokenRanges) { |
| 110 | + tokenRange.startIndex = newIndexByOriginalIndex[Math.min(tokenRange.startIndex, originalTokenCount)]; |
| 111 | + tokenRange.endIndex = newIndexByOriginalIndex[Math.min(tokenRange.endIndex, originalTokenCount)]; |
| 112 | + } |
| 113 | + |
| 114 | + // Replace the excerpt tokens in place with the condensed list. |
| 115 | + excerptTokens.length = 0; |
| 116 | + for (const token of condensedTokens) { |
| 117 | + excerptTokens.push(token); |
| 118 | + } |
| 119 | +} |
0 commit comments