Skip to content

Commit

Permalink
Add a version of TryGetRecentCheckResultsForFile with project snapshot (
Browse files Browse the repository at this point in the history
dotnet#16720)

* First stab at adding a version of TryGetRecentCheckResultsForFile that takes a snapshot instead of project options

* use version type without defaultof<_> = null

* take sourcetext out of project snapshot, kind of forces us to an async return but let's see if this is acceptable

* format

* - reuse ParseAndCheckFileInProject cache for TryGetRecentCheckResultsForFile
- extend version of ParseAndCheckFileInProject cache with the check sum of the source code
- add test

* format

* cleanup

* use a new LruCache member GetAll to get rid of the dummy version

* cleanup

* Update src/Compiler/Service/TransparentCompiler.fs

Co-authored-by: Petr Pokorny <[email protected]>

* unify key creation

* just use ProjectSnapShot.FileKey

* to have it on record, push a version with "f.Version |> Md5Hasher.toString" as the second part of lastFileKey.Version

* use FileKeyWithExtraFileSnapshotVersion for the ParseAndCheckFileInProject cache

* replace FileSnapShot after edit in Test

* add CustomOperation tryGetRecentCheckResults for tests

* - Make API non-async and don't return hash
- let tests run for background compiler, too

* better fix for commandLineOptions order

* Update src/Compiler/Service/FSharpProjectSnapshot.fs

Co-authored-by: Petr Pokorny <[email protected]>

* Update tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs

Co-authored-by: Petr Pokorny <[email protected]>

* Update tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs

Co-authored-by: Petr Pokorny <[email protected]>

* fix version predicate

* let LruCache.GetAll(key: 'TKey) return a seq instead of a list

* compare signatures in CustomOperation tryGetRecentCheckResults to tighten the tests

---------

Co-authored-by: Petr Pokorny <[email protected]>
  • Loading branch information
dawedawe and 0101 authored Feb 23, 2024
1 parent d071ddc commit db6b8cd
Show file tree
Hide file tree
Showing 16 changed files with 237 additions and 11 deletions.
9 changes: 9 additions & 0 deletions src/Compiler/Facilities/AsyncMemoize.fs
Original file line number Diff line number Diff line change
Expand Up @@ -534,6 +534,15 @@ type internal AsyncMemoize<'TKey, 'TVersion, 'TValue when 'TKey: equality and 'T

}

member _.TryGet(key: 'TKey, predicate: 'TVersion -> bool) : 'TValue option =
let versionsAndJobs = cache.GetAll(key)

versionsAndJobs
|> Seq.tryPick (fun (version, job) ->
match predicate version, job with
| true, Completed(completed, _) -> Some completed
| _ -> None)

member _.Clear() = cache.Clear()

member _.Clear predicate = cache.Clear predicate
Expand Down
2 changes: 2 additions & 0 deletions src/Compiler/Facilities/AsyncMemoize.fsi
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ type internal AsyncMemoize<'TKey, 'TVersion, 'TValue when 'TKey: equality and 'T

member Get': key: 'TKey * computation: NodeCode<'TValue> -> NodeCode<'TValue>

member TryGet: key: 'TKey * predicate: ('TVersion -> bool) -> 'TValue option

member Event: IEvent<JobEvent * (string * 'TKey * 'TVersion)>

member OnEvent: ((JobEvent * (string * 'TKey * 'TVersion) -> unit) -> unit)
Expand Down
22 changes: 22 additions & 0 deletions src/Compiler/Service/BackgroundCompiler.fs
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,10 @@ type internal IBackgroundCompiler =
fileName: string * options: FSharpProjectOptions * sourceText: ISourceText option * userOpName: string ->
(FSharpParseFileResults * FSharpCheckFileResults * SourceTextHash) option

abstract member TryGetRecentCheckResultsForFile:
fileName: string * projectSnapshot: FSharpProjectSnapshot * userOpName: string ->
(FSharpParseFileResults * FSharpCheckFileResults) option

abstract member BeforeBackgroundFileCheck: IEvent<string * FSharpProjectOptions>

abstract member FileChecked: IEvent<string * FSharpProjectOptions>
Expand Down Expand Up @@ -1174,6 +1178,16 @@ type internal BackgroundCompiler
| None -> None
| None -> None

member _.TryGetRecentCheckResultsForFile(fileName: string, projectSnapshot: FSharpProjectSnapshot, userOpName: string) =
projectSnapshot.ProjectSnapshot.SourceFiles
|> List.tryFind (fun f -> f.FileName = fileName)
|> Option.bind (fun (f: FSharpFileSnapshot) ->
let options = projectSnapshot.ToOptions()
let sourceText = f.GetSource() |> Async.AwaitTask |> Async.RunSynchronously

self.TryGetRecentCheckResultsForFile(fileName, options, Some sourceText, userOpName)
|> Option.map (fun (parseFileResults, checkFileResults, _hash) -> (parseFileResults, checkFileResults)))

/// Parse and typecheck the whole project (the implementation, called recursively as project graph is evaluated)
member private _.ParseAndCheckProjectImpl(options, userOpName) =
node {
Expand Down Expand Up @@ -1732,3 +1746,11 @@ type internal BackgroundCompiler
userOpName: string
) : (FSharpParseFileResults * FSharpCheckFileResults * SourceTextHash) option =
self.TryGetRecentCheckResultsForFile(fileName, options, sourceText, userOpName)

member _.TryGetRecentCheckResultsForFile
(
fileName: string,
projectSnapshot: FSharpProjectSnapshot,
userOpName: string
) : (FSharpParseFileResults * FSharpCheckFileResults) option =
self.TryGetRecentCheckResultsForFile(fileName, projectSnapshot, userOpName)
4 changes: 4 additions & 0 deletions src/Compiler/Service/BackgroundCompiler.fsi
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,10 @@ type internal IBackgroundCompiler =
fileName: string * options: FSharpProjectOptions * sourceText: ISourceText option * userOpName: string ->
(FSharpParseFileResults * FSharpCheckFileResults * SourceTextHash) option

abstract TryGetRecentCheckResultsForFile:
fileName: string * projectSnapshot: FSharpProjectSnapshot * userOpName: string ->
(FSharpParseFileResults * FSharpCheckFileResults) option

abstract BeforeBackgroundFileCheck: IEvent<string * FSharpProjectOptions>

abstract FileChecked: IEvent<string * FSharpProjectOptions>
Expand Down
10 changes: 8 additions & 2 deletions src/Compiler/Service/FSharpProjectSnapshot.fs
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,12 @@ type internal ProjectSnapshotBase<'T when 'T :> IFileSnapshot>(projectCore: Proj
member this.FileKey(fileName: string) = this.UpTo(fileName).LastFileKey
member this.FileKey(index: FileIndex) = this.UpTo(index).LastFileKey

member this.FileKeyWithExtraFileSnapshotVersion(fileName: string) =
let fileKey = this.FileKey fileName
let fileSnapshot = this.SourceFiles |> Seq.find (fun f -> f.FileName = fileName)

fileKey.WithExtraVersion(fileSnapshot.Version |> Md5Hasher.toString)

/// Project snapshot with filenames and versions given as initial input
and internal ProjectSnapshot = ProjectSnapshotBase<FSharpFileSnapshot>

Expand Down Expand Up @@ -394,10 +400,10 @@ and internal ProjectCore
let commandLineOptions =
lazy
(seq {
yield! OtherOptions

for r in ReferencesOnDisk do
$"-r:{r.Path}"

yield! OtherOptions
}
|> Seq.toList)

Expand Down
36 changes: 33 additions & 3 deletions src/Compiler/Service/TransparentCompiler.fs
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,7 @@ type internal CompilerCaches(sizeFactor: int) =
this.AssemblyData.Clear(shouldClear)
this.SemanticClassification.Clear(snd >> shouldClear)
this.ItemKeyStore.Clear(snd >> shouldClear)
this.ScriptClosure.Clear(snd >> shouldClear) // Todo check if correct predicate
this.ScriptClosure.Clear(snd >> shouldClear)

type internal TransparentCompiler
(
Expand Down Expand Up @@ -1472,9 +1472,8 @@ type internal TransparentCompiler

let ComputeParseAndCheckFileInProject (fileName: string) (projectSnapshot: ProjectSnapshot) =
caches.ParseAndCheckFileInProject.Get(
projectSnapshot.FileKey fileName,
projectSnapshot.FileKeyWithExtraFileSnapshotVersion fileName,
node {

use _ =
Activity.start "ComputeParseAndCheckFileInProject" [| Activity.Tags.fileName, fileName |> Path.GetFileName |]

Expand Down Expand Up @@ -1605,6 +1604,29 @@ type internal TransparentCompiler
}
)

let TryGetRecentCheckResultsForFile
(
fileName: string,
projectSnapshot: FSharpProjectSnapshot,
userOpName: string
) : (FSharpParseFileResults * FSharpCheckFileResults) option =
ignore userOpName

let cacheKey =
projectSnapshot.ProjectSnapshot.FileKeyWithExtraFileSnapshotVersion fileName

let version = cacheKey.GetVersion()

let parseFileResultsAndcheckFileAnswer =
caches.ParseAndCheckFileInProject.TryGet(
cacheKey.GetKey(),
(fun (_fullVersion, fileContentVersion) -> fileContentVersion = (snd version))
)

match parseFileResultsAndcheckFileAnswer with
| Some(parseFileResults, FSharpCheckFileAnswer.Succeeded checkFileResults) -> Some(parseFileResults, checkFileResults)
| _ -> None

let ComputeProjectExtras (bootstrapInfo: BootstrapInfo) (projectSnapshot: ProjectSnapshotWithSources) =
caches.ProjectExtras.Get(
projectSnapshot.SignatureKey,
Expand Down Expand Up @@ -2356,3 +2378,11 @@ type internal TransparentCompiler
userOpName: string
) : (FSharpParseFileResults * FSharpCheckFileResults * SourceTextHash) option =
backgroundCompiler.TryGetRecentCheckResultsForFile(fileName, options, sourceText, userOpName)

member this.TryGetRecentCheckResultsForFile
(
fileName: string,
projectSnapshot: FSharpProjectSnapshot,
userOpName: string
) : (FSharpParseFileResults * FSharpCheckFileResults) option =
TryGetRecentCheckResultsForFile(fileName, projectSnapshot, userOpName)
2 changes: 1 addition & 1 deletion src/Compiler/Service/TransparentCompiler.fsi
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ type internal CompilerCaches =
member ParseAndCheckAllFilesInProject: AsyncMemoizeDisabled<obj, obj, obj>

member ParseAndCheckFileInProject:
AsyncMemoize<(string * (string * string)), string, (FSharpParseFileResults * FSharpCheckFileAnswer)>
AsyncMemoize<(string * (string * string)), string * string, (FSharpParseFileResults * FSharpCheckFileAnswer)>

member ParseAndCheckProject: AsyncMemoize<(string * string), string, FSharpCheckProjectResults>

Expand Down
4 changes: 4 additions & 0 deletions src/Compiler/Service/service.fs
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,10 @@ type FSharpChecker
let userOpName = defaultArg userOpName "Unknown"
backgroundCompiler.TryGetRecentCheckResultsForFile(fileName, options, sourceText, userOpName)

member _.TryGetRecentCheckResultsForFile(fileName: string, projectSnapshot: FSharpProjectSnapshot, ?userOpName: string) =
let userOpName = defaultArg userOpName "Unknown"
backgroundCompiler.TryGetRecentCheckResultsForFile(fileName, projectSnapshot, userOpName)

member _.Compile(argv: string[], ?userOpName: string) =
let _userOpName = defaultArg userOpName "Unknown"
use _ = Activity.start "FSharpChecker.Compile" [| Activity.Tags.userOpName, _userOpName |]
Expand Down
5 changes: 5 additions & 0 deletions src/Compiler/Service/service.fsi
Original file line number Diff line number Diff line change
Expand Up @@ -422,6 +422,11 @@ type public FSharpChecker =
fileName: string * options: FSharpProjectOptions * ?sourceText: ISourceText * ?userOpName: string ->
(FSharpParseFileResults * FSharpCheckFileResults (* hash *) * int64) option

[<Experimental("This FCS API is experimental and subject to change.")>]
member TryGetRecentCheckResultsForFile:
fileName: string * projectSnapshot: FSharpProjectSnapshot * ?userOpName: string ->
(FSharpParseFileResults * FSharpCheckFileResults) option

/// This function is called when the entire environment is known to have changed for reasons not encoded in the ProjectOptions of any project/compilation.
member InvalidateAll: unit -> unit

Expand Down
14 changes: 10 additions & 4 deletions src/Compiler/Utilities/LruCache.fs
Original file line number Diff line number Diff line change
Expand Up @@ -198,21 +198,27 @@ type internal LruCache<'TKey, 'TVersion, 'TValue when 'TKey: equality and 'TVers

/// Returns an option of a value for given key and version, and also a list of all other versions for given key
member this.GetAll(key, version) =
this.TryGet(key, version),
let others =
this.GetAll(key) |> Seq.filter (fun (ver, _val) -> ver <> version) |> Seq.toList

this.TryGet(key, version), others

/// Returns a list of version * value pairs for a given key. The strongly held value is first in the list.
member _.GetAll(key: 'TKey) : ('TVersion * 'TValue) seq =
match dictionary.TryGetValue key with
| false, _ -> []
| true, versionDict ->
versionDict.Values
|> Seq.map (fun node -> node.Value)
|> Seq.filter (p24 >> ((<>) version))
|> Seq.map (_.Value)
|> Seq.sortBy (function
| _, _, _, Strong _ -> 0
| _ -> 1)
|> Seq.choose (function
| _, ver, _, Strong v -> Some(ver, v)
| _, ver, _, Weak r ->
match r.TryGetTarget() with
| true, x -> Some(ver, x)
| _ -> None)
|> Seq.toList

member _.Remove(key, version) =
match dictionary.TryGetValue key with
Expand Down
3 changes: 3 additions & 0 deletions src/Compiler/Utilities/LruCache.fsi
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@ type internal LruCache<'TKey, 'TVersion, 'TValue when 'TKey: equality and 'TVers
/// Returns an option of a value for given key and version, and also a list of all other versions for given key
member GetAll: key: 'TKey * version: 'TVersion -> 'TValue option * ('TVersion * 'TValue) list

/// Returns a list of version * value pairs for a given key. The strongly held value is first in the list.
member GetAll: key: 'TKey -> ('TVersion * 'TValue) seq

member GetValues: unit -> (string * 'TVersion * 'TValue) seq

member Remove: key: 'TKey -> unit
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -896,6 +896,52 @@ let ``LoadClosure for script is recomputed after changes`` () =
|> Map

Assert.Equal<JobEvent list>([Weakened; Requested; Started; Finished; Weakened; Requested; Started; Finished], closureComputations["FileFirst.fs"])

[<Fact>]
let ``TryGetRecentCheckResultsForFile returns None before first call to ParseAndCheckFileInProject`` () =
let project = SyntheticProject.Create(
sourceFile "First" [])

ProjectWorkflowBuilder(project) {
clearCache
tryGetRecentCheckResults "First" expectNone
} |> ignore

[<Fact>]
let ``TryGetRecentCheckResultsForFile returns result after first call to ParseAndCheckFileInProject`` () =
let project = SyntheticProject.Create(
sourceFile "First" [] )

ProjectWorkflowBuilder(project) {
tryGetRecentCheckResults "First" expectSome
} |> ignore

[<Fact>]
let ``TryGetRecentCheckResultsForFile returns no result after edit`` () =
let project = SyntheticProject.Create(
sourceFile "First" [])

ProjectWorkflowBuilder(project) {
tryGetRecentCheckResults "First" expectSome
updateFile "First" updatePublicSurface
tryGetRecentCheckResults "First" expectNone
checkFile "First" expectOk
tryGetRecentCheckResults "First" expectSome
} |> ignore

[<Fact>]
let ``TryGetRecentCheckResultsForFile returns result after edit of other file`` () =
let project = SyntheticProject.Create(
sourceFile "First" [],
sourceFile "Second" ["First"])

ProjectWorkflowBuilder(project) {
tryGetRecentCheckResults "First" expectSome
tryGetRecentCheckResults "Second" expectSome
updateFile "First" updatePublicSurface
tryGetRecentCheckResults "First" expectNone
tryGetRecentCheckResults "Second" expectSome // file didn't change so we still want to get the recent result
} |> ignore

[<Fact>]
let ``Background compiler and Transparent compiler return the same options`` () =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2079,6 +2079,7 @@ FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.IEvent`2[Mi
FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.IEvent`2[Microsoft.FSharp.Control.FSharpHandler`1[System.Tuple`2[System.String,FSharp.Compiler.CodeAnalysis.FSharpProjectOptions]],System.Tuple`2[System.String,FSharp.Compiler.CodeAnalysis.FSharpProjectOptions]] get_BeforeBackgroundFileCheck()
FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.IEvent`2[Microsoft.FSharp.Control.FSharpHandler`1[System.Tuple`2[System.String,FSharp.Compiler.CodeAnalysis.FSharpProjectOptions]],System.Tuple`2[System.String,FSharp.Compiler.CodeAnalysis.FSharpProjectOptions]] get_FileChecked()
FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.IEvent`2[Microsoft.FSharp.Control.FSharpHandler`1[System.Tuple`2[System.String,FSharp.Compiler.CodeAnalysis.FSharpProjectOptions]],System.Tuple`2[System.String,FSharp.Compiler.CodeAnalysis.FSharpProjectOptions]] get_FileParsed()
FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Core.FSharpOption`1[System.Tuple`2[FSharp.Compiler.CodeAnalysis.FSharpParseFileResults,FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults]] TryGetRecentCheckResultsForFile(System.String, FSharpProjectSnapshot, Microsoft.FSharp.Core.FSharpOption`1[System.String])
FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Core.FSharpOption`1[System.Tuple`3[FSharp.Compiler.CodeAnalysis.FSharpParseFileResults,FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults,System.Int64]] TryGetRecentCheckResultsForFile(System.String, FSharp.Compiler.CodeAnalysis.FSharpProjectOptions, Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.Text.ISourceText], Microsoft.FSharp.Core.FSharpOption`1[System.String])
FSharp.Compiler.CodeAnalysis.FSharpChecker: System.Tuple`2[FSharp.Compiler.CodeAnalysis.FSharpParsingOptions,Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.Diagnostics.FSharpDiagnostic]] GetParsingOptionsFromCommandLineArgs(Microsoft.FSharp.Collections.FSharpList`1[System.String], Microsoft.FSharp.Collections.FSharpList`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean])
FSharp.Compiler.CodeAnalysis.FSharpChecker: System.Tuple`2[FSharp.Compiler.CodeAnalysis.FSharpParsingOptions,Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.Diagnostics.FSharpDiagnostic]] GetParsingOptionsFromCommandLineArgs(Microsoft.FSharp.Collections.FSharpList`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean])
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2079,6 +2079,7 @@ FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.IEvent`2[Mi
FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.IEvent`2[Microsoft.FSharp.Control.FSharpHandler`1[System.Tuple`2[System.String,FSharp.Compiler.CodeAnalysis.FSharpProjectOptions]],System.Tuple`2[System.String,FSharp.Compiler.CodeAnalysis.FSharpProjectOptions]] get_BeforeBackgroundFileCheck()
FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.IEvent`2[Microsoft.FSharp.Control.FSharpHandler`1[System.Tuple`2[System.String,FSharp.Compiler.CodeAnalysis.FSharpProjectOptions]],System.Tuple`2[System.String,FSharp.Compiler.CodeAnalysis.FSharpProjectOptions]] get_FileChecked()
FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.IEvent`2[Microsoft.FSharp.Control.FSharpHandler`1[System.Tuple`2[System.String,FSharp.Compiler.CodeAnalysis.FSharpProjectOptions]],System.Tuple`2[System.String,FSharp.Compiler.CodeAnalysis.FSharpProjectOptions]] get_FileParsed()
FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Core.FSharpOption`1[System.Tuple`2[FSharp.Compiler.CodeAnalysis.FSharpParseFileResults,FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults]] TryGetRecentCheckResultsForFile(System.String, FSharpProjectSnapshot, Microsoft.FSharp.Core.FSharpOption`1[System.String])
FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Core.FSharpOption`1[System.Tuple`3[FSharp.Compiler.CodeAnalysis.FSharpParseFileResults,FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults,System.Int64]] TryGetRecentCheckResultsForFile(System.String, FSharp.Compiler.CodeAnalysis.FSharpProjectOptions, Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.Text.ISourceText], Microsoft.FSharp.Core.FSharpOption`1[System.String])
FSharp.Compiler.CodeAnalysis.FSharpChecker: System.Tuple`2[FSharp.Compiler.CodeAnalysis.FSharpParsingOptions,Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.Diagnostics.FSharpDiagnostic]] GetParsingOptionsFromCommandLineArgs(Microsoft.FSharp.Collections.FSharpList`1[System.String], Microsoft.FSharp.Collections.FSharpList`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean])
FSharp.Compiler.CodeAnalysis.FSharpChecker: System.Tuple`2[FSharp.Compiler.CodeAnalysis.FSharpParsingOptions,Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.Diagnostics.FSharpDiagnostic]] GetParsingOptionsFromCommandLineArgs(Microsoft.FSharp.Collections.FSharpList`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean])
Expand Down
Loading

0 comments on commit db6b8cd

Please sign in to comment.