diff --git a/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md b/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md index 9690a6c3c3d..067389a012e 100644 --- a/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md +++ b/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md @@ -170,6 +170,7 @@ * IL: use empty tables for members when possible ([PR #20249](https://github.com/dotnet/fsharp/pull/20249)) ### Improved +* `FSharpLineTokenizer.ScanToken` no longer allocates an `option` box per token; it returns a `struct (FSharpTokenInfo voption * FSharpTokenizerLexState)` instead, removing per-token heap allocations on the hot tokenization path used by classification, brace matching, and the deprecated `FSharp.LanguageService` colorizer. ([PR #20113](https://github.com/dotnet/fsharp/pull/20113)) * Nullness warning FS3261 on dotted method or property access (e.g. `x.Member`) now underlines the receiver expression and includes the member name and (when known) the binding name in the message. ([Issue #19658](https://github.com/dotnet/fsharp/issues/19658), [PR #19814](https://github.com/dotnet/fsharp/pull/19814)) * Direct delegate construction ([PR ##19993](https://github.com/dotnet/fsharp/pull/19993)) @@ -189,5 +190,6 @@ ### Breaking Changes * `FSharp.Compiler.Syntax.SynInterpolatedStringPart.FillExpr` now carries a `SynInterpolationFormatting` value (separating .NET alignment/format from printf specifiers) instead of an `Ident option`. ([PR #19971](https://github.com/dotnet/fsharp/pull/19971)) +* `FSharpLineTokenizer.ScanToken: lexState -> FSharpTokenInfo option * FSharpTokenizerLexState` now returns `struct (FSharpTokenInfo voption * FSharpTokenizerLexState)`. All in-tree callers (`FSharpChecker.TokenizeLine`, the editor `Tokenizer`, and the deprecated `FSharp.LanguageService` colorizer) have been updated accordingly. ([PR #20113](https://github.com/dotnet/fsharp/pull/20113)) * Optimizer: don't inline named functions in debug builds ([PR #19548](https://github.com/dotnet/fsharp/pull/19548) * LexFilter: drop non-strict mode ([PR #20106](https://github.com/dotnet/fsharp/pull/20106)) diff --git a/docs/release-notes/.VisualStudio/18.vNext.md b/docs/release-notes/.VisualStudio/18.vNext.md index cffa42edc9c..a09f38096da 100644 --- a/docs/release-notes/.VisualStudio/18.vNext.md +++ b/docs/release-notes/.VisualStudio/18.vNext.md @@ -11,6 +11,7 @@ * Improve static compilation of state machines. ([PR #19297](https://github.com/dotnet/fsharp/pull/19297)) * Make Alt+F1 (momentary toggle) work for inlay hints. ([PR #19421](https://github.com/dotnet/fsharp/pull/19421)) * Fix doubled F# diagnostics in tooltips. ([Issue #16360](https://github.com/dotnet/fsharp/issues/16360)) +* Fix a race condition in the editor's per-document token cache (`SourceTextData`) that could corrupt classification/tagging and symbol-lookup state under concurrent access, by backing it with a `ConcurrentDictionary`. Also removed redundant allocations on the tokenizer hot path (reused already-materialized line text in cache validation, dropped mutable indirection in token scanning). ([PR #20113](https://github.com/dotnet/fsharp/pull/20113)) ### Changed diff --git a/src/Compiler/Service/ServiceLexing.fs b/src/Compiler/Service/ServiceLexing.fs index ce501ac7755..2ea04beb6f2 100644 --- a/src/Compiler/Service/ServiceLexing.fs +++ b/src/Compiler/Service/ServiceLexing.fs @@ -1011,7 +1011,7 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, fi false, (EOF LexerStateEncoding.revertToDefaultLexCont, 0, 0) // Scan a token starting with the given lexer state - member x.ScanToken(lexState: FSharpTokenizerLexState) : FSharpTokenInfo option * FSharpTokenizerLexState = + member x.ScanToken(lexState: FSharpTokenizerLexState) : struct (FSharpTokenInfo voption * FSharpTokenizerLexState) = use _ = UseBuildPhase BuildPhase.Parse use _ = UseDiagnosticsLogger DiscardErrorsLogger @@ -1022,12 +1022,12 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, fi let isCached, (token, leftc, rightc) = getTokenWithPosition lexcont // Check for end-of-string and failure - let tokenDataOption, lexcontFinal, tokenTag = + let struct (tokenDataOption, lexcontFinal, tokenTag) = match token with | EOF lexcont -> // End of text! No more tokens. - None, lexcont, 0 - | LEX_FAILURE _ -> None, LexerStateEncoding.revertToDefaultLexCont, 0 + struct (ValueNone, lexcont, 0) + | LEX_FAILURE _ -> struct (ValueNone, LexerStateEncoding.revertToDefaultLexCont, 0) | _ -> // Get the information about the token let colorClass, charClass, triggerClass = TokenClassifications.tokenInfo token @@ -1058,14 +1058,14 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, fi FullMatchedLength = fullMatchedLength } - Some tokenData, lexcontFinal, tokenTag + struct (ValueSome tokenData, lexcontFinal, tokenTag) // Check for patterns like #-IDENT and see if they look like meta commands for .fsx files. If they do then merge them into a single token. - let tokenDataOption, lexintFinal = + let struct (tokenDataOption, lexintFinal) = let lexintFinal = LexerStateEncoding.encodeLexInt lexcontFinal match tokenDataOption, singleLineTokenState, tokenTagToTokenId tokenTag with - | Some tokenData, SingleLineTokenState.BeforeHash, TOKEN_HASH -> + | ValueSome tokenData, SingleLineTokenState.BeforeHash, TOKEN_HASH -> // Don't allow further matches. singleLineTokenState <- SingleLineTokenState.NoFurtherMatchPossible // Peek at the next token @@ -1110,17 +1110,17 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, fi let lexintFinal = LexerStateEncoding.encodeLexInt lexcontFinal - Some tokenData, lexintFinal - | _ -> tokenDataOption, lexintFinal - | _ -> tokenDataOption, lexintFinal + struct (ValueSome tokenData, lexintFinal) + | _ -> struct (tokenDataOption, lexintFinal) + | _ -> struct (tokenDataOption, lexintFinal) | _, SingleLineTokenState.BeforeHash, TOKEN_WHITESPACE -> // Allow leading whitespace. - tokenDataOption, lexintFinal + struct (tokenDataOption, lexintFinal) | _ -> singleLineTokenState <- SingleLineTokenState.NoFurtherMatchPossible - tokenDataOption, lexintFinal + struct (tokenDataOption, lexintFinal) - tokenDataOption, lexintFinal + struct (tokenDataOption, lexintFinal) static member ColorStateOfLexState(lexState: FSharpTokenizerLexState) = LexerStateEncoding.colorStateOfLexState lexState diff --git a/src/Compiler/Service/ServiceLexing.fsi b/src/Compiler/Service/ServiceLexing.fsi index ea7d05b60fe..bcfc56c545a 100755 --- a/src/Compiler/Service/ServiceLexing.fsi +++ b/src/Compiler/Service/ServiceLexing.fsi @@ -310,7 +310,7 @@ type FSharpTokenInfo = [] type FSharpLineTokenizer = /// Scan one token from the line - member ScanToken: lexState: FSharpTokenizerLexState -> FSharpTokenInfo option * FSharpTokenizerLexState + member ScanToken: lexState: FSharpTokenizerLexState -> struct (FSharpTokenInfo voption * FSharpTokenizerLexState) /// Get the color state from the lexer state static member ColorStateOfLexState: FSharpTokenizerLexState -> FSharpTokenizerColorState diff --git a/src/Compiler/Service/service.fs b/src/Compiler/Service/service.fs index 1006def6da1..e58845a836f 100644 --- a/src/Compiler/Service/service.fs +++ b/src/Compiler/Service/service.fs @@ -629,16 +629,22 @@ type FSharpChecker member _.TokenizeLine(line: string, state: FSharpTokenizerLexState) = let tokenizer = FSharpSourceTokenizer([], None, None) let lineTokenizer = tokenizer.CreateLineTokenizer line - let mutable state = (None, state) + let mutable lexState = state + let mutable token = ValueNone + + let scanNext () = + let struct (t, s) = lineTokenizer.ScanToken(lexState) + token <- t + lexState <- s + token.IsSome let tokens = [| - while (state <- lineTokenizer.ScanToken(snd state) - (fst state).IsSome) do - yield (fst state).Value + while scanNext () do + yield token.Value |] - tokens, snd state + tokens, lexState /// Tokenize an entire file, line by line member x.TokenizeFile(source: string) : FSharpTokenInfo[][] = diff --git a/tests/FSharp.Compiler.Service.Tests/TokenizerTests.fs b/tests/FSharp.Compiler.Service.Tests/TokenizerTests.fs index 566dc150ce7..8bdbde460b3 100644 --- a/tests/FSharp.Compiler.Service.Tests/TokenizerTests.fs +++ b/tests/FSharp.Compiler.Service.Tests/TokenizerTests.fs @@ -7,12 +7,12 @@ open Xunit let rec parseLine(line: string, state: FSharpTokenizerLexState ref, tokenizer: FSharpLineTokenizer) = seq { match tokenizer.ScanToken(state.Value) with - | Some(tok), nstate -> + | ValueSome(tok), nstate -> let str = line.Substring(tok.LeftColumn, tok.RightColumn - tok.LeftColumn + 1) yield str, tok state.Value <- nstate yield! parseLine(line, state, tokenizer) - | None, nstate -> + | ValueNone, nstate -> state.Value <- nstate } let tokenizeLines (lines:string[]) = @@ -30,21 +30,21 @@ let scanTokens (defines: string list) (source: string) = let tokenizer = sourceTok.CreateLineTokenizer(source) let rec loop (state: FSharpTokenizerLexState) acc = match tokenizer.ScanToken(state) with - | Some tok, nstate -> loop nstate (tok :: acc) - | None, _ -> List.rev acc + | ValueSome tok, nstate -> loop nstate (tok :: acc) + | ValueNone, _ -> List.rev acc loop FSharpTokenizerLexState.Initial [] [] let ``Tokenizer test - simple let with string``() = - let tokenizedLines = + let tokenizedLines = tokenizeLines [| "// Sets the hello world variable" "let hello = \"Hello world\" " |] - let actual = + let actual = [ for lineNo, lineToks in tokenizedLines do yield lineNo, [ for str, info in lineToks do yield info.TokenName, str ] ] - let expected = + let expected = [(0, [("LINE_COMMENT", "//"); ("LINE_COMMENT", " "); ("LINE_COMMENT", "Sets"); ("LINE_COMMENT", " "); ("LINE_COMMENT", "the"); ("LINE_COMMENT", " "); @@ -57,14 +57,14 @@ let ``Tokenizer test - simple let with string``() = ("STRING_TEXT", "\""); ("STRING_TEXT", "Hello"); ("STRING_TEXT", " "); ("STRING_TEXT", "world"); ("STRING", "\""); ("WHITESPACE", " ")])] - if actual <> expected then + if actual <> expected then printfn "actual = %A" actual printfn "expected = %A" expected actual |> Assert.shouldBeEqualWith expected (sprintf "actual and expected did not match,actual =\n%A\nexpected=\n%A\n" actual expected) [] let ``Tokenizer test 2 - single line non-nested string interpolation``() = - let tokenizedLines = + let tokenizedLines = tokenizeLines [| "// Tests tokenizing string interpolation" "let hello0 = $\"\"" @@ -74,10 +74,10 @@ let ``Tokenizer test 2 - single line non-nested string interpolation``() = "let hello1v = @$\"Hello world\" " "let hello2v = @$\"Hello world {1+1} = {2}\" " |] - let actual = + let actual = [ for lineNo, lineToks in tokenizedLines do yield lineNo, [ for str, info in lineToks do yield info.TokenName, str ] ] - let expected = + let expected = [(0, [("LINE_COMMENT", "//"); ("LINE_COMMENT", " "); ("LINE_COMMENT", "Tests"); ("LINE_COMMENT", " "); ("LINE_COMMENT", "tokenizing"); ("LINE_COMMENT", " "); @@ -121,23 +121,23 @@ let ``Tokenizer test 2 - single line non-nested string interpolation``() = ("STRING_TEXT", "="); ("STRING_TEXT", " "); ("INTERP_STRING_PART", "{"); ("INT32", "2"); ("STRING_TEXT", "}"); ("INTERP_STRING_END", "\""); ("WHITESPACE", " ")]);] - - if actual <> expected then + + if actual <> expected then printfn "actual = %A" actual printfn "expected = %A" expected actual |> Assert.shouldBeEqualWith expected (sprintf "actual and expected did not match,actual =\n%A\nexpected=\n%A\n" actual expected) [] let ``Tokenizer test - multiline non-nested string interpolation``() = - let tokenizedLines = + let tokenizedLines = tokenizeLines [| "let hello1t = $\"\"\"abc {1+" " 1} def\"\"\"" |] - let actual = + let actual = [ for lineNo, lineToks in tokenizedLines do yield lineNo, [ for str, info in lineToks do yield info.TokenName, str ] ] - let expected = + let expected = [(0, [("LET", "let"); ("WHITESPACE", " "); ("IDENT", "hello1t"); ("WHITESPACE", " "); ("EQUALS", "="); ("WHITESPACE", " "); @@ -146,8 +146,8 @@ let ``Tokenizer test - multiline non-nested string interpolation``() = (1, [("WHITESPACE", " "); ("INT32", "1"); ("STRING_TEXT", "}"); ("STRING_TEXT", " "); ("STRING_TEXT", "def"); ("INTERP_STRING_END", "\"\"\"")])] - - if actual <> expected then + + if actual <> expected then printfn "actual = %A" actual printfn "expected = %A" expected actual |> Assert.shouldBeEqualWith expected (sprintf "actual and expected did not match,actual =\n%A\nexpected=\n%A\n" actual expected) @@ -155,7 +155,7 @@ let ``Tokenizer test - multiline non-nested string interpolation``() = [] // checks nested '{' and nested single-quote strings let ``Tokenizer test - multi-line nested string interpolation``() = - let tokenizedLines = + let tokenizedLines = tokenizeLines [| "let hello1t = $\"\"\"abc {\"a\" + " " { " @@ -163,10 +163,10 @@ let ``Tokenizer test - multi-line nested string interpolation``() = " }.contents " " } def\"\"\"" |] - let actual = + let actual = [ for lineNo, lineToks in tokenizedLines do yield lineNo, [ for str, info in lineToks do yield info.TokenName, str ] ] - let expected = + let expected = [(0, [("LET", "let"); ("WHITESPACE", " "); ("IDENT", "hello1t"); ("WHITESPACE", " "); ("EQUALS", "="); ("WHITESPACE", " "); @@ -188,22 +188,22 @@ let ``Tokenizer test - multi-line nested string interpolation``() = (4, [("WHITESPACE", " "); ("STRING_TEXT", "}"); ("STRING_TEXT", " "); ("STRING_TEXT", "def"); ("INTERP_STRING_END", "\"\"\"")])] - - if actual <> expected then + + if actual <> expected then printfn "actual = %A" actual printfn "expected = %A" expected actual |> Assert.shouldBeEqualWith expected (sprintf "actual and expected did not match,actual =\n%A\nexpected=\n%A\n" actual expected) [] let ``Tokenizer test - single-line nested string interpolation``() = - let tokenizedLines = + let tokenizedLines = tokenizeLines [| " $\"abc { { contents = 1 } }\" " |] - let actual = + let actual = [ for lineNo, lineToks in tokenizedLines do yield lineNo, [ for str, info in lineToks do yield info.TokenName, str ] ] - let expected = + let expected = [(0, [("WHITESPACE", " "); ("STRING_TEXT", "$\""); ("STRING_TEXT", "abc"); ("STRING_TEXT", " "); ("INTERP_STRING_BEGIN_PART", "{"); ("WHITESPACE", " "); @@ -211,8 +211,8 @@ let ``Tokenizer test - single-line nested string interpolation``() = ("WHITESPACE", " "); ("EQUALS", "="); ("WHITESPACE", " "); ("INT32", "1"); ("WHITESPACE", " "); ("RBRACE", "}"); ("WHITESPACE", " "); ("STRING_TEXT", "}"); ("INTERP_STRING_END", "\""); ("WHITESPACE", " ")])] - - if actual <> expected then + + if actual <> expected then printfn "actual = %A" actual printfn "expected = %A" expected actual |> Assert.shouldBeEqualWith expected (sprintf "actual and expected did not match,actual =\n%A\nexpected=\n%A\n" actual expected) @@ -291,7 +291,7 @@ let ``Tokenizer test - optional parameters with question mark``() = let actual = [ for lineNo, lineToks in tokenizedLines do yield lineNo, [ for str, info in lineToks do yield info.TokenName, str ] ] - + let expected = [(0, [("MEMBER", "member"); ("WHITESPACE", " "); ("UNDERSCORE", "_"); ("DOT", "."); @@ -299,7 +299,7 @@ let ``Tokenizer test - optional parameters with question mark``() = ("IDENT", "optional"); ("COLON", ":"); ("IDENT", "string"); ("RPAREN", ")"); ("WHITESPACE", " "); ("EQUALS", "="); ("WHITESPACE", " "); ("IDENT", "optional")])] - + if actual <> expected then printfn "actual = %A" actual printfn "expected = %A" expected diff --git a/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs b/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs index 6901ceb97b1..9c80bf12a63 100644 --- a/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs +++ b/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs @@ -454,35 +454,39 @@ module internal Tokenizer = member val ClassifiedSpans = classifiedSpans member val SavedTokens = savedTokens + member data.IsValid(textLine: TextLine, lineContents: string) = + data.LineStart = textLine.Start && data.HashCode = lineContents.GetHashCode() + member data.IsValid(textLine: TextLine) = data.LineStart = textLine.Start && let lineContents = textLine.Text.ToString(textLine.Span) in data.HashCode = lineContents.GetHashCode() + // Shared by concurrent editor operations (classification and symbol lookup), so each entry access + // must be thread-safe. This only guarantees per-index atomicity, not a coherent snapshot across a + // range of lines: concurrent scans over overlapping line ranges can still interleave, but the cache + // self-heals on the next read via the IsValid/LexStateAtStartOfLine checks. type private SourceTextData(approxLines: int) = - let data = ResizeArray(approxLines) - - let extendTo i = - if i >= data.Count then - data.Capacity <- i + 1 - - for j in data.Count .. i do - data.Add(None) + let data = + ConcurrentDictionary(Environment.ProcessorCount, approxLines) member x.Item with get (i: int) = - extendTo i - data.[i] + match data.TryGetValue(i) with + | true, v -> ValueSome v + | _ -> ValueNone and set (i: int) v = - extendTo i - data.[i] <- v + match v with + | ValueSome v -> data.[i] <- v + | ValueNone -> data.TryRemove(i) |> ignore member x.ClearFrom(n) = let mutable i = n + let mutable cont = true - while i < data.Count && data.[i].IsSome do - data.[i] <- None - i <- i + 1 + while cont do + let removed, _ = data.TryRemove(i) + if removed then i <- i + 1 else cont <- false /// This saves the tokenization data for a file for as long as the DocumentId object is alive. /// This seems risky - if one single thing leaks a DocumentId (e.g. stores it in some global table of documents @@ -513,66 +517,54 @@ module internal Tokenizer = let colorMap = Array.create textLine.Span.Length ClassificationTypeNames.Text let lineTokenizer = sourceTokenizer.CreateLineTokenizer(lineContents) let tokens = ResizeArray() - let mutable tokenInfoOption = None let mutable previousLexState = lexState - let processToken () = - let classificationType = - compilerTokenToRoslynToken (tokenInfoOption.Value.ColorClass) + let processToken token = + let classificationType = compilerTokenToRoslynToken token.ColorClass - for i = tokenInfoOption.Value.LeftColumn to tokenInfoOption.Value.RightColumn do + for i = token.LeftColumn to token.RightColumn do Array.set colorMap i classificationType - let token = tokenInfoOption.Value - let savedToken = SavedTokenInfo.Create token - - tokens.Add savedToken + tokens.Add(SavedTokenInfo.Create token) let scanAndColorNextToken () = - let info, nextLexState = lineTokenizer.ScanToken(previousLexState) - tokenInfoOption <- info + let struct (info, nextLexState) = lineTokenizer.ScanToken(previousLexState) previousLexState <- nextLexState // Apply some hacks to clean up the token stream (we apply more later) match info with - | Some info when info.Tag = FSharpTokenTag.INT32_DOT_DOT -> - tokenInfoOption <- - Some - { - LeftColumn = info.LeftColumn - RightColumn = info.RightColumn - 2 - ColorClass = FSharpTokenColorKind.Number - CharClass = FSharpTokenCharKind.Literal - FSharpTokenTriggerClass = info.FSharpTokenTriggerClass - Tag = info.Tag - TokenName = "INT32" - FullMatchedLength = info.FullMatchedLength - 2 - } - - processToken () - - tokenInfoOption <- - Some - { - LeftColumn = info.RightColumn - 1 - RightColumn = info.RightColumn - ColorClass = FSharpTokenColorKind.Operator - CharClass = FSharpTokenCharKind.Operator - FSharpTokenTriggerClass = info.FSharpTokenTriggerClass - Tag = FSharpTokenTag.DOT_DOT - TokenName = "DOT_DOT" - FullMatchedLength = 2 - } - - processToken () - - | Some _ -> processToken () + | ValueSome info when info.Tag = FSharpTokenTag.INT32_DOT_DOT -> + processToken + { + LeftColumn = info.LeftColumn + RightColumn = info.RightColumn - 2 + ColorClass = FSharpTokenColorKind.Number + CharClass = FSharpTokenCharKind.Literal + FSharpTokenTriggerClass = info.FSharpTokenTriggerClass + Tag = info.Tag + TokenName = "INT32" + FullMatchedLength = info.FullMatchedLength - 2 + } + + processToken + { + LeftColumn = info.RightColumn - 1 + RightColumn = info.RightColumn + ColorClass = FSharpTokenColorKind.Operator + CharClass = FSharpTokenCharKind.Operator + FSharpTokenTriggerClass = info.FSharpTokenTriggerClass + Tag = FSharpTokenTag.DOT_DOT + TokenName = "DOT_DOT" + FullMatchedLength = 2 + } + + | ValueSome info -> processToken info | _ -> () - scanAndColorNextToken () + info.IsSome - while tokenInfoOption.IsSome do - scanAndColorNextToken () + while scanAndColorNextToken () do + () let mutable startPosition = 0 let mutable endPosition = startPosition @@ -635,8 +627,8 @@ module internal Tokenizer = while i > 0 && (match sourceTextDataCache.[i] with - | Some data -> not (data.IsValid(lines.[i])) - | None -> true) do + | ValueSome data -> not (data.IsValid(lines.[i])) + | ValueNone -> true) do i <- i - 1 i @@ -658,11 +650,15 @@ module internal Tokenizer = // 2. the hash codes match // 3. the start-of-line lex states are the same match sourceTextDataCache.[i] with - | Some data when data.IsValid(textLine) && data.LexStateAtStartOfLine.Equals(lexState) -> data + | ValueSome data when + data.IsValid(textLine, lineContents) + && data.LexStateAtStartOfLine.Equals(lexState) + -> + data | _ -> // Otherwise, we recompute let newData = scanSourceLine (sourceTokenizer, textLine, lineContents, lexState) - sourceTextDataCache.[i] <- Some newData + sourceTextDataCache.[i] <- ValueSome newData newData lexState <- lineData.LexStateAtEndOfLine @@ -673,10 +669,10 @@ module internal Tokenizer = // If necessary, invalidate all subsequent lines after endLine if endLine < lines.Count - 1 then match sourceTextDataCache.[endLine + 1] with - | Some data -> + | ValueSome data -> if not (data.LexStateAtStartOfLine.Equals(lexState)) then sourceTextDataCache.ClearFrom(endLine + 1) - | None -> () + | ValueNone -> () ] /// Generates a list of Classified Spans for tokens which undergo syntactic classification (i.e., are not typechecked). @@ -844,14 +840,10 @@ module internal Tokenizer = | SymbolLookupKind.Precise -> 0 | SymbolLookupKind.Greedy -> 1 - [ - for x in draftTokens do - if - x.LeftColumn <= linePos.Character - && (x.RightColumn + rightColumnCorrection) >= linePos.Character - then - yield x - ] + draftTokens + |> List.filter (fun x -> + x.LeftColumn <= linePos.Character + && (x.RightColumn + rightColumnCorrection) >= linePos.Character) // Select IDENT token. If failed, select OPERATOR token. let symbol = @@ -1024,6 +1016,9 @@ module internal Tokenizer = else false + let private forbiddenSymbolNameChars = + [| '.'; '+'; '$'; '&'; '['; ']'; '/'; '\\'; '*'; '"' |] + let isValidNameForSymbol (lexerSymbolKind: LexerSymbolKind, symbol: FSharpSymbol, name: string) : bool = let inline isIdentifier (ident: string) = @@ -1042,11 +1037,9 @@ module internal Tokenizer = not (String.IsNullOrEmpty s) && FSharpKeywords.NormalizeIdentifierBackticks s |> isIdentifier - let forbiddenChars = [| '.'; '+'; '$'; '&'; '['; ']'; '/'; '\\'; '*'; '\"' |] - let inline isTypeNameIdent (s: string) = not (String.IsNullOrEmpty s) - && s.IndexOfAny forbiddenChars = -1 + && s.IndexOfAny forbiddenSymbolNameChars = -1 && isFixableIdentifier s let inline isUnionCaseIdent (s: string) = diff --git a/vsintegration/src/FSharp.LanguageService/Colorize.fs b/vsintegration/src/FSharp.LanguageService/Colorize.fs index 922305d7a11..53ba15e2e18 100644 --- a/vsintegration/src/FSharp.LanguageService/Colorize.fs +++ b/vsintegration/src/FSharp.LanguageService/Colorize.fs @@ -24,11 +24,11 @@ open FSharp.Compiler.Tokenization /// Maintain a two-way lookup of lexstate to colorstate /// In practice this table will be quite small. All of F# only uses 38 distinct LexStates. // -// Note: DEPRECATED CODE ONLY ACTIVE IN UNIT TESTING VIA "UNROSLYNIZED" UNIT TESTS +// Note: DEPRECATED CODE ONLY ACTIVE IN UNIT TESTING VIA "UNROSLYNIZED" UNIT TESTS // // Note: Tests using this code should either be adjusted to test the corresponding feature in -// FSharp.Editor, or deleted. However, the tests may be exercising underlying F# Compiler -// functionality and thus have considerable value, they should ony be deleted if we are sure this +// FSharp.Editor, or deleted. However, the tests may be exercising underlying F# Compiler +// functionality and thus have considerable value, they should ony be deleted if we are sure this // is not the case. // module internal ColorStateLookup_DEPRECATED = @@ -78,11 +78,11 @@ module internal ColorStateLookup_DEPRECATED = // - SetLineText() is called one line at a time. // - An instance of FSharpScanner_DEPRECATED is associated with exactly one buffer (IVsTextLines). // -// Note: DEPRECATED CODE ONLY ACTIVE IN UNIT TESTING VIA "UNROSLYNIZED" UNIT TESTS. +// Note: DEPRECATED CODE ONLY ACTIVE IN UNIT TESTING VIA "UNROSLYNIZED" UNIT TESTS. // // Note: Tests using this code should either be adjusted to test the corresponding feature in -// FSharp.Editor, or deleted. However, the tests may be exercising underlying F# Compiler -// functionality and thus have considerable value, they should ony be deleted if we are sure this +// FSharp.Editor, or deleted. However, the tests may be exercising underlying F# Compiler +// functionality and thus have considerable value, they should ony be deleted if we are sure this // is not the case. // type internal FSharpScanner_DEPRECATED(makeLineTokenizer : string -> FSharpLineTokenizer) = @@ -124,17 +124,17 @@ type internal FSharpScanner_DEPRECATED(makeLineTokenizer : string -> FSharpLineT /// Scan a token from a line. This should only be used in cases where color information is irrelevant. /// Used by GetFullLineInfo (and only thus in a small workaround in GetDeclarations) and GetTokenInformationAt (thus GetF1KeywordString). member ws.ScanTokenWithDetails (lexState: _ ref) = - let colorInfoOption, newLexState = lineTokenizer.ScanToken(lexState.Value) + let struct (colorInfoOption, newLexState) = lineTokenizer.ScanToken(lexState.Value) lexState.Value <- newLexState colorInfoOption /// Scan a token from a line and write information about it into the tokeninfo object. member ws.ScanTokenAndProvideInfoAboutIt(_line, tokenInfo:TokenInfo, lexState: _ ref) = - let colorInfoOption, newLexState = lineTokenizer.ScanToken(!lexState) + let struct (colorInfoOption, newLexState) = lineTokenizer.ScanToken(!lexState) lexState.Value <- newLexState match colorInfoOption with - | None -> false - | Some colorInfo -> + | ValueNone -> false + | ValueSome colorInfo -> let color = colorInfo.ColorClass tokenInfo.Trigger <- enum (int32 colorInfo.FSharpTokenTriggerClass) // cast one enum to another tokenInfo.StartIndex <- colorInfo.LeftColumn @@ -174,11 +174,11 @@ type internal FSharpScanner_DEPRECATED(makeLineTokenizer : string -> FSharpLineT /// Implement the MPF Colorizer functionality. /// onClose is a method to call when shutting down the colorizer. // -// Note: DEPRECATED CODE ONLY ACTIVE IN UNIT TESTING VIA "UNROSLYNIZED" UNIT TESTS. +// Note: DEPRECATED CODE ONLY ACTIVE IN UNIT TESTING VIA "UNROSLYNIZED" UNIT TESTS. // // Note: Tests using this code should either be adjusted to test the corresponding feature in -// FSharp.Editor, or deleted. However, the tests may be exercising underlying F# Compiler -// functionality and thus have considerable value, they should ony be deleted if we are sure this +// FSharp.Editor, or deleted. However, the tests may be exercising underlying F# Compiler +// functionality and thus have considerable value, they should ony be deleted if we are sure this // is not the case. // type internal FSharpColorizer_DEPRECATED @@ -259,10 +259,10 @@ type internal FSharpColorizer_DEPRECATED scanner.SetLineText lineText let rec tokens() = seq { match scanner.ScanTokenWithDetails(refState) with - | Some tok -> + | ValueSome tok -> yield tok yield! tokens() - | None -> () } + | ValueNone -> () } tokens() |> Array.ofSeq member private c.GetColorInfo(line,lineText,length,lastColorState) = @@ -342,8 +342,8 @@ type internal FSharpColorizer_DEPRECATED let rec searchForToken () = match scanner.ScanTokenWithDetails lexState with - | None -> None - | Some ti as result -> + | ValueNone -> ValueNone + | ValueSome ti as result -> if col >= ti.LeftColumn && col <= ti.RightColumn then result else @@ -357,11 +357,11 @@ type internal FSharpColorizer_DEPRECATED /// Implements IVsColorableItem and IVsMergeableUIItem, for colored text items // -// Note: DEPRECATED CODE ONLY ACTIVE IN UNIT TESTING VIA "UNROSLYNIZED" UNIT TESTS. +// Note: DEPRECATED CODE ONLY ACTIVE IN UNIT TESTING VIA "UNROSLYNIZED" UNIT TESTS. // // Note: Tests using this code should either be adjusted to test the corresponding feature in -// FSharp.Editor, or deleted. However, the tests may be exercising underlying F# Compiler -// functionality and thus have considerable value, they should ony be deleted if we are sure this +// FSharp.Editor, or deleted. However, the tests may be exercising underlying F# Compiler +// functionality and thus have considerable value, they should ony be deleted if we are sure this // is not the case. // type internal FSharpColorableItem_DEPRECATED(canonicalName: string, displayName : Lazy, foreground, background) = diff --git a/vsintegration/src/FSharp.LanguageService/Intellisense.fs b/vsintegration/src/FSharp.LanguageService/Intellisense.fs index 6f11b4a4314..6bedf5bf273 100644 --- a/vsintegration/src/FSharp.LanguageService/Intellisense.fs +++ b/vsintegration/src/FSharp.LanguageService/Intellisense.fs @@ -10,8 +10,8 @@ open System open System.Collections.Generic open System.Collections.Immutable open Microsoft.VisualStudio -open Microsoft.VisualStudio.Shell.Interop -open Microsoft.VisualStudio.TextManager.Interop +open Microsoft.VisualStudio.Shell.Interop +open Microsoft.VisualStudio.TextManager.Interop open Microsoft.VisualStudio.Text open Microsoft.VisualStudio.OLE.Interop open FSharp.Compiler @@ -23,20 +23,20 @@ open FSharp.Compiler.Text open FSharp.Compiler.Tokenization module internal TaggedText = - let appendTo (sb: System.Text.StringBuilder) (t: TaggedText) = sb.Append t.Text |> ignore - -// Note: DEPRECATED CODE ONLY ACTIVE IN UNIT TESTING VIA "UNROSLYNIZED" UNIT TESTS. + let appendTo (sb: System.Text.StringBuilder) (t: TaggedText) = sb.Append t.Text |> ignore + +// Note: DEPRECATED CODE ONLY ACTIVE IN UNIT TESTING VIA "UNROSLYNIZED" UNIT TESTS. // // Note: Tests using this code should either be adjusted to test the corresponding feature in -// FSharp.Editor, or deleted. However, the tests may be exercising underlying F# Compiler -// functionality and thus have considerable value, they should ony be deleted if we are sure this +// FSharp.Editor, or deleted. However, the tests may be exercising underlying F# Compiler +// functionality and thus have considerable value, they should ony be deleted if we are sure this // is not the case. // type internal FSharpMethodListForAMethodTip_DEPRECATED(documentationBuilder: IDocumentationBuilder_DEPRECATED, methodsName, methods: MethodGroupItem[], nwpl: ParameterLocations, snapshot: ITextSnapshot, isThisAStaticArgumentsTip: bool) = - inherit MethodListForAMethodTip_DEPRECATED() + inherit MethodListForAMethodTip_DEPRECATED() // Compute the tuple end points - let tupleEnds = + let tupleEnds = let oneColAfter ((l,c): Position01) = (l,c+1) let oneColBefore ((l,c): Position01) = (l,c-1) [| yield Position.toZ nwpl.LongIdStartLocation @@ -72,19 +72,19 @@ type internal FSharpMethodListForAMethodTip_DEPRECATED(documentationBuilder: IDo override x.GetCount() = methods.Length - override x.GetDescription(methodIndex) = safe methodIndex "" (fun m -> + override x.GetDescription(methodIndex) = safe methodIndex "" (fun m -> let buf = Text.StringBuilder() XmlDocumentation.BuildMethodOverloadTipText_DEPRECATED(documentationBuilder, TaggedText.appendTo buf, TaggedText.appendTo buf, m.Description, true) buf.ToString() ) - + override x.GetReturnTypeText(methodIndex) = safe methodIndex "" (fun m -> m.ReturnTypeText.Text) override x.GetParameterCount(methodIndex) = safe methodIndex 0 (fun m -> getParameters(m).Length) - + override x.GetParameterInfo(methodIndex, parameterIndex, nameOut, displayOut, descriptionOut) = let name,display = safe methodIndex ("","") (fun m -> let p = getParameters(m).[parameterIndex] in p.ParameterName, p.Display.Text ) - + nameOut <- name displayOut <- display descriptionOut <- "" @@ -113,16 +113,16 @@ type internal ObsoleteGlyph = | Record = 126 | DiscriminatedUnion = 132 -// Note: DEPRECATED CODE ONLY ACTIVE IN UNIT TESTING VIA "UNROSLYNIZED" UNIT TESTS. +// Note: DEPRECATED CODE ONLY ACTIVE IN UNIT TESTING VIA "UNROSLYNIZED" UNIT TESTS. // // Note: Tests using this code should either be adjusted to test the corresponding feature in -// FSharp.Editor, or deleted. However, the tests may be exercising underlying F# Compiler -// functionality and thus have considerable value, they should ony be deleted if we are sure this +// FSharp.Editor, or deleted. However, the tests may be exercising underlying F# Compiler +// functionality and thus have considerable value, they should ony be deleted if we are sure this // is not the case. // -type internal FSharpDeclarations_DEPRECATED(documentationBuilder, declarations: DeclarationListItem[], reason: BackgroundRequestReason) = - - inherit Declarations_DEPRECATED() +type internal FSharpDeclarations_DEPRECATED(documentationBuilder, declarations: DeclarationListItem[], reason: BackgroundRequestReason) = + + inherit Declarations_DEPRECATED() // Sort the declarations, NOTE: we used ORDINAL comparison here, this is "by design" from F# 2.0, partly because it puts lowercase last. let declarations = declarations |> Array.sortWith (fun d1 d2 -> compare d1.NameInList d2.NameInList) @@ -133,25 +133,25 @@ type internal FSharpDeclarations_DEPRECATED(documentationBuilder, declarations: // Given a prefix, narrow the items to the include the ones containing that prefix, and store in a lookaside table // attached to this declaration set. - let trimmedDeclarations filterText = - if reason = BackgroundRequestReason.DisplayMemberList then declarations - elif tab.ContainsKey filterText then tab.[filterText] - else + let trimmedDeclarations filterText = + if reason = BackgroundRequestReason.DisplayMemberList then declarations + elif tab.ContainsKey filterText then tab.[filterText] + else let matcher = AbstractPatternMatcher.Singleton - let decls = + let decls = // Find the first prefix giving a non-empty declaration set after filtering - seq { for i in filterText.Length-1 .. -1 .. 0 do + seq { for i in filterText.Length-1 .. -1 .. 0 do let filterTextPrefix = filterText.[0..i] match tab.TryGetValue filterTextPrefix with | true, decls -> yield decls - | false, _ -> yield declarations |> Array.filter (fun s -> matcher.MatchSingleWordPattern(s.NameInList, filterTextPrefix)<>null) + | false, _ -> yield declarations |> Array.filter (fun s -> matcher.MatchSingleWordPattern(s.NameInList, filterTextPrefix)<>null) yield declarations } |> Seq.tryFind (fun arr -> arr.Length > 0) |> (function None -> declarations | Some s -> s) tab.[filterText] <- decls decls - override decl.GetCount(filterText) = + override decl.GetCount(filterText) = let decls = trimmedDeclarations filterText decls.Length @@ -169,17 +169,17 @@ type internal FSharpDeclarations_DEPRECATED(documentationBuilder, declarations: let item = decls.[index] if (item.Glyph = FSharpGlyph.Error) then "" - else + else item.NameInList else String.Empty - + override decl.GetNameInCode(filterText, index) = let decls = trimmedDeclarations filterText if (index >= 0 && index < decls.Length) then let item = decls.[index] if (item.Glyph = FSharpGlyph.Error) then "" - else + else item.NameInCode else String.Empty @@ -187,7 +187,7 @@ type internal FSharpDeclarations_DEPRECATED(documentationBuilder, declarations: let decls = trimmedDeclarations filterText if (index >= 0 && index < decls.Length) then let buf = Text.StringBuilder() - XmlDocumentation.BuildDataTipText_DEPRECATED(documentationBuilder, TaggedText.appendTo buf, TaggedText.appendTo buf, decls.[index].Description) + XmlDocumentation.BuildDataTipText_DEPRECATED(documentationBuilder, TaggedText.appendTo buf, TaggedText.appendTo buf, decls.[index].Description) buf.ToString() else "" @@ -234,35 +234,35 @@ type internal FSharpDeclarations_DEPRECATED(documentationBuilder, declarations: override decl.IsCommitChar(commitCharacter) = // Usual language identifier rules... not (Char.IsLetterOrDigit(commitCharacter) || commitCharacter = '_') - + // A helper to aid in determining how much text is relevant to the items chosen in the completion list. override decl.Reason = reason - + // Note, there is no real reason for this code to use byrefs, except that we're calling it from C#. override decl.GetBestMatch(filterText, textSoFar, index : int byref, uniqueMatch : bool byref, shouldSelectItem : bool byref) = let decls = trimmedDeclarations filterText let compareStrings(s,t,l,b : bool) = System.String.Compare(s,0,t,0,l,b) - let tryFindDeclIndex text length ignoreCase = - decls + let tryFindDeclIndex text length ignoreCase = + decls |> Array.tryFindIndex (fun d -> compareStrings(d.NameInList, text, length, ignoreCase) = 0) - // The best match is the first item that begins with the longest prefix of the - // given word (value). - let rec findMatchOfLength len ignoreCase = + // The best match is the first item that begins with the longest prefix of the + // given word (value). + let rec findMatchOfLength len ignoreCase = if len = 0 then let indexLastBestMatch = tryFindDeclIndex lastBestMatch lastBestMatch.Length ignoreCase match indexLastBestMatch with | Some index -> (index, false, false) | None -> (0,false, false) - else + else let firstMatchingLenChars = tryFindDeclIndex textSoFar len ignoreCase match firstMatchingLenChars with - | Some index -> + | Some index -> lastBestMatch <- decls.[index].NameInList let select = len = textSoFar.Length - if (index <> decls.Length- 1) && (compareStrings(decls.[index+1].NameInList , textSoFar, len, ignoreCase) = 0) + if (index <> decls.Length- 1) && (compareStrings(decls.[index+1].NameInList , textSoFar, len, ignoreCase) = 0) then (index, false, select) else (index, select, select) - | None -> + | None -> match ignoreCase with | false -> findMatchOfLength len true | true -> findMatchOfLength (len-1) false @@ -288,15 +288,15 @@ type internal FSharpDeclarations_DEPRECATED(documentationBuilder, declarations: '\000' - -// Note: DEPRECATED CODE ONLY ACTIVE IN UNIT TESTING VIA "UNROSLYNIZED" UNIT TESTS. + +// Note: DEPRECATED CODE ONLY ACTIVE IN UNIT TESTING VIA "UNROSLYNIZED" UNIT TESTS. // // Note: Tests using this code should either be adjusted to test the corresponding feature in -// FSharp.Editor, or deleted. However, the tests may be exercising underlying F# Compiler -// functionality and thus have considerable value, they should ony be deleted if we are sure this +// FSharp.Editor, or deleted. However, the tests may be exercising underlying F# Compiler +// functionality and thus have considerable value, they should ony be deleted if we are sure this // is not the case. // -type internal FSharpIntellisenseInfo_DEPRECATED +type internal FSharpIntellisenseInfo_DEPRECATED (// The recent result of parsing untypedResults: FSharpParseFileResults, // Line/column/snapshot of BackgroundRequest that initiated creation of this scope @@ -312,21 +312,21 @@ type internal FSharpIntellisenseInfo_DEPRECATED // A service that will provide Xml Content documentationBuilder : IDocumentationBuilder_DEPRECATED, provideMethodList : bool - ) = - inherit IntellisenseInfo_DEPRECATED() + ) = + inherit IntellisenseInfo_DEPRECATED() - let methodList = - if provideMethodList then + let methodList = + if provideMethodList then try // go ahead and compute this now, on this background thread, so will have info ready when UI thread asks let noteworthyParamInfoLocations = untypedResults.FindParameterLocations(Position.fromZ brLine brCol) // we need some typecheck info, even if stale, in order to look up e.g. method overload types/xmldocs - if typedResults.HasFullTypeCheckInfo then + if typedResults.HasFullTypeCheckInfo then // we need recent parse info to e.g. know how many commas and thus how many args there are match noteworthyParamInfoLocations with - | Some nwpl -> + | Some nwpl -> // Note: this may alternatively workaround some parts of 90778 - the real fix for that is to have before-overload-resolution name-sink work correctly. // However it also deals with stale typecheck info that may not have recorded name resolutions for a recently-typed long-id. let names = nwpl.LongId @@ -335,26 +335,26 @@ type internal FSharpIntellisenseInfo_DEPRECATED // the name you just typed, but fresh enough that you do have the right name-resolution-environment to look up the name. let lidEnd = nwpl.LongIdEndLocation let methods = typedResults.GetMethods(lidEnd.Line, lidEnd.Column, "", Some names) - - // If the name is an operator ending with ">" then it is a mistake - // we can't tell whether " >(" is a generic method call or an operator use + + // If the name is an operator ending with ">" then it is a mistake + // we can't tell whether " >(" is a generic method call or an operator use // (it depends on the previous line), so we filter it // - // Note: this test isn't particularly elegant - encoded operator name would be something like "( ...> )" + // Note: this test isn't particularly elegant - encoded operator name would be something like "( ...> )" if (methods.Methods.Length = 0 || methods.MethodName.EndsWith("> )")) then None - else + else // "methods" contains both real methods for this longId, as well as static-parameters in the case of type providers. // They "conflict" for cases of TP(...) (calling a constructor, no static args provided) versus TP<...> (static args), since // both point to the same longId. However we can look at the character at the 'OpenParen' location and see if it is a '(' or a '<' and then // filter the "methods" list accordingly. let isThisAStaticArgumentsTip = - let parenLine, parenCol = Position.toZ nwpl.OpenParenLocation + let parenLine, parenCol = Position.toZ nwpl.OpenParenLocation let textAtOpenParenLocation = if brSnapshot=null then // we are unit testing, use the view let _hr, buf = view.GetBuffer() - let _hr, s = buf.GetLineText(parenLine, parenCol, parenLine, parenCol+1) + let _hr, s = buf.GetLineText(parenLine, parenCol, parenLine, parenCol+1) s else // we are in the product, use the ITextSnapshot @@ -364,7 +364,7 @@ type internal FSharpIntellisenseInfo_DEPRECATED else false // note: textAtOpenParenLocation is not necessarily otherwise "(", for example in "sin 42.0" it is "4" let filteredMethods = - [| for m in methods.Methods do + [| for m in methods.Methods do if (isThisAStaticArgumentsTip && m.StaticParameters.Length > 0) || (not isThisAStaticArgumentsTip && m.HasParameters) then // need to distinguish TP<...>(...) angle brackets tip from parens tip yield m |] @@ -372,12 +372,12 @@ type internal FSharpIntellisenseInfo_DEPRECATED Some (FSharpMethodListForAMethodTip_DEPRECATED(documentationBuilder, methods.MethodName, filteredMethods, nwpl, brSnapshot, isThisAStaticArgumentsTip) :> MethodListForAMethodTip_DEPRECATED) else None - | _ -> + | _ -> None else // GetMethodListForAMethodTip found no TypeCheckInfo in ParseResult. None - with e-> + with e-> Assert.Exception(e) reraise() else None @@ -388,46 +388,46 @@ type internal FSharpIntellisenseInfo_DEPRECATED // '<' can be treated both as operator and as part of identifier // in this case we'll do 2 passes: // 1. treatTokenAsIdentifier=false - we'll pick raw token under the cursor and try find it among resolved names, is attempt was successful - great we are done, otherwise - // 2. treatTokenAsIdentifier=true - even if raw token was recognized as operator we'll use different branch + // 2. treatTokenAsIdentifier=true - even if raw token was recognized as operator we'll use different branch // that calls QuickParse.GetCompleteIdentifierIsland and then tries previous column... let rec getDataTip alwaysTreatTokenAsIdentifier = let token = colorizer.Value.GetTokenInfoAt(VsTextLines.TextColorState (VsTextView.Buffer view),line,col) try let lineText = VsTextLines.LineText (VsTextView.Buffer view) line - + // Try the actual column first... let tokenTag, col, possibleIdentifier, makeSecondAttempt = - if token.Type = TokenType.Operator && not alwaysTreatTokenAsIdentifier then - let tag, startCol, endCol = OperatorToken.asIdentifier_DEPRECATED token + if token.Type = TokenType.Operator && not alwaysTreatTokenAsIdentifier then + let tag, startCol, endCol = OperatorToken.asIdentifier_DEPRECATED token let op = lineText.Substring(startCol, endCol - startCol) tag, startCol, Some(op, endCol, false), true else match (QuickParse.GetCompleteIdentifierIsland false lineText col) with - | None when col > 0 -> + | None when col > 0 -> // Try the previous column & get the token info for it - let tokenTag = + let tokenTag = let token = colorizer.Value.GetTokenInfoAt(VsTextLines.TextColorState (VsTextView.Buffer view),line,col - 1) - token.Token + token.Token let possibleIdentifier = QuickParse.GetCompleteIdentifierIsland false lineText (col - 1) tokenTag, col - 1, possibleIdentifier, false | _ as poss -> token.Token, col, poss, false let diagnosticTipSpan = TextSpan(iStartLine=line, iEndLine=line, iStartIndex=col, iEndIndex=col+1) - match possibleIdentifier with + match possibleIdentifier with | None -> "",diagnosticTipSpan - | Some (s,colAtEndOfNames, isQuotedIdentifier) -> + | Some (s,colAtEndOfNames, isQuotedIdentifier) -> - if typedResults.HasFullTypeCheckInfo then + if typedResults.HasFullTypeCheckInfo then let qualId = PrettyNaming.GetLongNameFromString s - + // Correct the identifier (e.g. to correctly handle active pattern names that end with "BAR" token) let tokenTag = QuickParse.CorrectIdentifierToken s tokenTag let dataTip = typedResults.GetToolTip(Line.fromZ line, colAtEndOfNames, lineText, qualId, tokenTag) match dataTip with | ToolTipText.ToolTipText [] when makeSecondAttempt -> getDataTip true - | _ -> + | _ -> let buf = Text.StringBuilder() XmlDocumentation.BuildDataTipText_DEPRECATED(documentationBuilder, TaggedText.appendTo buf, TaggedText.appendTo buf, dataTip) @@ -438,17 +438,17 @@ type internal FSharpIntellisenseInfo_DEPRECATED // This is the span of text over which the data tip is active. If the mouse moves away from it then the // data tip goes away let dataTipSpan = TextSpan(iStartLine=line, iEndLine=line, iStartIndex=max 0 (colAtEndOfNames-lastStringLength), iEndIndex=colAtEndOfNames) - (buf.ToString(), dataTipSpan) + (buf.ToString(), dataTipSpan) else "Bug: TypeCheckInfo option was None", diagnosticTipSpan - with e -> + with e -> Assert.Exception(e) reraise() getDataTip false - - /// Determine whether to force the use a synchronous parse + + /// Determine whether to force the use a synchronous parse static member IsReasonRequiringSyncParse(reason) = match reason with | BackgroundRequestReason.MethodTip // param info... @@ -468,14 +468,14 @@ type internal FSharpIntellisenseInfo_DEPRECATED let prevTokenInfo = colorizer.Value.GetTokenInfoAt(VsTextLines.TextColorState (VsTextView.Buffer view),line,prevCol) // denotes if we got token that matches exact specified position or it was just last token before EOF let exactMatch = col >= tokenInfo.StartIndex && col <= tokenInfo.EndIndex - exactMatch && ((tokenInfo.Color = TokenColor.Comment && prevTokenInfo.Color = TokenColor.Comment) || + exactMatch && ((tokenInfo.Color = TokenColor.Comment && prevTokenInfo.Color = TokenColor.Comment) || (tokenInfo.Color = TokenColor.String && prevTokenInfo.Color = TokenColor.String)) if isInCommentOrString then // We don't want to show info in comments & strings (in case of exact match) // (but we want to show it if the thing before or after isn't comment/string) - return null - - elif typedResults.HasFullTypeCheckInfo then + return null + + elif typedResults.HasFullTypeCheckInfo then let lineText = VsTextLines.LineText (VsTextView.Buffer view) line let colorState = VsTextLines.TextColorState (VsTextView.Buffer view) let state = VsTextColorState.GetColorStateAtStartOfLine colorState line @@ -489,7 +489,7 @@ type internal FSharpIntellisenseInfo_DEPRECATED // here ^ return null // An ugly check to suppress declaration lists at 'member' declarations - elif QuickParse.TestMemberOrOverrideDeclaration tokens then + elif QuickParse.TestMemberOrOverrideDeclaration tokens then return null else let untypedParseInfoOpt = @@ -498,15 +498,15 @@ type internal FSharpIntellisenseInfo_DEPRECATED else None // TODO don't use QuickParse below, we have parse info available - let pname = QuickParse.GetPartialLongNameEx(lineText, col-1) + let pname = QuickParse.GetPartialLongNameEx(lineText, col-1) let _x = 1 // for breakpoint - let decls = typedResults.GetDeclarationListInfo(untypedParseInfoOpt, Line.fromZ line, lineText, pname, (fun() -> [])) - return (new FSharpDeclarations_DEPRECATED(documentationBuilder, decls.Items, reason) :> Declarations_DEPRECATED) + let decls = typedResults.GetDeclarationListInfo(untypedParseInfoOpt, Line.fromZ line, lineText, pname, (fun() -> [])) + return (new FSharpDeclarations_DEPRECATED(documentationBuilder, decls.Items, reason) :> Declarations_DEPRECATED) else // no TypeCheckInfo in ParseResult. - return null - with e-> + return null + with e-> Assert.Exception(e) raise e return null @@ -524,9 +524,9 @@ type internal FSharpIntellisenseInfo_DEPRECATED let keyword = let line = span.iStartLine - let lineText = VsTextLines.LineText (VsTextView.Buffer view) line + let lineText = VsTextLines.LineText (VsTextView.Buffer view) line let tokenInformation, col = - let col = + let col = if span.iStartIndex = lineText.Length && span.iStartIndex > 0 then // if we are at the end of the line, we always step back one character span.iStartIndex - 1 @@ -534,37 +534,37 @@ type internal FSharpIntellisenseInfo_DEPRECATED span.iStartIndex let textColorState = VsTextLines.TextColorState (VsTextView.Buffer view) match colorizer.Value.GetTokenInformationAt(textColorState,line,col) with - | Some token as original when col > 0 && shouldTryToFindIdentToTheLeft token -> + | ValueSome token as original when col > 0 && shouldTryToFindIdentToTheLeft token -> // try to step back one char match colorizer.Value.GetTokenInformationAt(textColorState,line,col-1) with - | Some token as newInfo when token.CharClass <> FSharpTokenCharKind.WhiteSpace -> newInfo, col - 1 + | ValueSome token as newInfo when token.CharClass <> FSharpTokenCharKind.WhiteSpace -> newInfo, col - 1 | _ -> original, col | otherwise -> otherwise, col match tokenInformation with - | None -> None - | Some token -> + | ValueNone -> None + | ValueSome token -> match token.CharClass, token.ColorClass with | FSharpTokenCharKind.Keyword, _ - | FSharpTokenCharKind.Operator, _ + | FSharpTokenCharKind.Operator, _ | _, FSharpTokenColorKind.PreprocessorKeyword -> lineText.Substring(token.LeftColumn, token.RightColumn - token.LeftColumn + 1) + "_FS" |> Some - + | (FSharpTokenCharKind.Comment|FSharpTokenCharKind.LineComment), _ -> Some "comment_FS" - - | FSharpTokenCharKind.Identifier, _ -> + + | FSharpTokenCharKind.Identifier, _ -> try let lineText = VsTextLines.LineText (VsTextView.Buffer view) line let possibleIdentifier = QuickParse.GetCompleteIdentifierIsland false lineText col match possibleIdentifier with | None -> None // no help keyword | Some(s,colAtEndOfNames, _) -> - if typedResults.HasFullTypeCheckInfo then + if typedResults.HasFullTypeCheckInfo then let qualId = PrettyNaming.GetLongNameFromString s match typedResults.GetF1Keyword(Line.fromZ line,colAtEndOfNames, lineText, qualId) with | Some s -> Some s - | None -> None - else None + | None -> None + else None with e -> Assert.Exception (e) reraise() @@ -579,7 +579,7 @@ type internal FSharpIntellisenseInfo_DEPRECATED () | None -> () - + // for tests member this.GotoDefinition (textView, line, column) = GotoDefinition.GotoDefinition_DEPRECATED (colorizer.Value, typedResults, textView, line, column)