Implement LSP-based code city creation #686

This implements an algorithm in LSPImporter that
creates the graph for a code city based on information retrieved using
the Language Server Protocol. The details of how this works are out of
scope for this commit message, but the code is documented, and the
general approach will be described in detail in my master's thesis.

Note that there are still some TODOs and FIXMEs that need to be
addressed before this is ready for production use. Most notably, the
call hierarchy is not retrieved (and hence no "Call" edges are being
created), as there seems to be a bug in the OmniSharp library that I
have not been able to fix yet.
Additionally, I still need to test various LSP servers—right now, only
pyright, Omnisharp, dart-lsp, and rust-analyzer have been tested.

Assets/SEE/DataModel/DG/Edge.cs

@@ -134,7 +134,7 @@ public override string ID
             {
                 if (string.IsNullOrEmpty(id))
                 {
-                    id = Type + "#" + Source.ID + "#" + Target.ID;
+                    id = GetGeneratedID(Source, Target, Type);
                 }
                 return id;
             }
@@ -149,6 +149,18 @@ public override string ID
             }
         }
 
+        /// <summary>
+        /// Returns the auto-generated ID of an edge with the given source, target, and type.
+        /// </summary>
+        /// <param name="source">The source node of the edge.</param>
+        /// <param name="target">The target node of the edge.</param>
+        /// <param name="type">The type of the edge.</param>
+        /// <returns>The auto-generated ID of an edge with the given source, target, and type.</returns>
+        public static string GetGeneratedID(Node source, Node target, string type)
+        {
+            return type + "#" + source.ID + "#" + target.ID;
+        }
+
         /// <summary>
         /// Returns true if <paramref name="edge"/> is not null.
         /// </summary>

Assets/SEE/DataModel/DG/Graph.cs

@@ -581,6 +581,16 @@ public bool ContainsNodeID(string id)
             return nodes.ContainsKey(id);
         }
 
+        /// <summary>
+        /// Returns true if an edge with the given <paramref name="id"/> is part of the graph.
+        /// </summary>
+        /// <param name="id">unique ID of the edge searched</param>
+        /// <returns>true if an edge with the given <paramref name="id"/> is part of the graph</returns>
+        public bool ContainsEdgeID(string id)
+        {
+            return edges.ContainsKey(id);
+        }
+
         /// <summary>
         /// Returns the node with the given unique <paramref name="id"/> in <paramref name="node"/>.
         /// If there is no such node, <paramref name="node"/> will be null and false will be returned;

Assets/SEE/DataModel/DG/GraphElement.cs

@@ -282,7 +282,7 @@ public int? SourceLine
 
             set
             {
-                Debug.Assert(value == null || value > 0);
+                Debug.Assert(value is null or > 0, $"expected positive line number, but got {value}");
                 SetInt(sourceLineAttribute, value);
             }
         }

Assets/SEE/DataModel/DG/IO/LSPImporter.cs.meta

@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: 0a05a26601c74048a7d77da61fab3493
+timeCreated: 1709579471

Assets/SEE/DataModel/DG/Node.cs

@@ -416,10 +416,10 @@ public int NumberOfChildren()
         }
 
         /// <summary>
-        /// The descendants of the node.
+        /// The immediate descendants of the node.
         /// Note: This is not a copy. The result can't be modified.
         /// </summary>
-        /// <returns>descendants of the node</returns>
+        /// <returns>immediate descendants of the node</returns>
         public IList<Node> Children()
         {
             return children.AsReadOnly();

Assets/SEE/DataModel/DG/Range.cs

@@ -105,7 +105,12 @@ public override string ToString()
         /// <returns>The converted range.</returns>
         public static Range FromLspRange(OmniSharp.Extensions.LanguageServer.Protocol.Models.Range lspRange)
         {
-            return new Range(lspRange.Start.Line, lspRange.End.Line, lspRange.Start.Character, lspRange.End.Character);
+            if (lspRange == null)
+            {
+                return null;
+            }
+            return new Range(lspRange.Start.Line+1, lspRange.End.Line+1,
+                             lspRange.Start.Character+1, lspRange.End.Character+1);
         }
 
         /// <summary>

Assets/SEE/Game/HolisticMetrics/Metrics/LinesOfCode.cs

@@ -24,12 +24,12 @@ internal class LinesOfCode : Metric
         /// metric for <paramref name="city"/></returns>
         internal override MetricValue Refresh(AbstractSEECity city)
         {
-            MetricValueCollection collection = new MetricValueCollection();
+            MetricValueCollection collection = new();
             foreach (Node node in city.LoadedGraph.Nodes())
             {
                 if (node.TryGetNumeric(attributeName, out float lines))
                 {
-                    MetricValueRange metricValue = new MetricValueRange
+                    MetricValueRange metricValue = new()
                     {
                         DecimalPlaces = 2,
                         Higher = 300, // FIXME: There can be more than 300 LOC.

Assets/SEE/GraphProviders/LSPGraphProvider.cs

@@ -3,13 +3,13 @@
 using System.IO;
 using System.Linq;
 using Cysharp.Threading.Tasks;
-using OmniSharp.Extensions.LanguageServer.Protocol.Document;
-using OmniSharp.Extensions.LanguageServer.Protocol.Models;
 using SEE.DataModel.DG;
+using SEE.DataModel.DG.IO;
 using SEE.Game.City;
-using SEE.GO;
 using SEE.Tools.LSP;
+using SEE.UI;
 using SEE.UI.RuntimeConfigMenu;
+using SEE.Utils;
 using SEE.Utils.Config;
 using SEE.Utils.Paths;
 using Sirenix.OdinInspector;
@@ -26,32 +26,102 @@ public class LSPGraphProvider : GraphProvider
         /// <summary>
         /// The path to the software project for which the graph shall be generated.
         /// </summary>
-        [Tooltip("Path to the project to be analyzed."), RuntimeTab(GraphProviderFoldoutGroup), HideReferenceObjectPicker]
+        [Tooltip("Root path of the project to be analyzed."), RuntimeTab(GraphProviderFoldoutGroup), HideReferenceObjectPicker]
         public DirectoryPath ProjectPath = new();
 
         /// <summary>
-        /// The language server to be used for the analysis.
+        /// The name of the language server to be used for the analysis.
         /// </summary>
         [Tooltip("The language server to be used for the analysis."),
+         LabelText("Language Server"),
          RuntimeTab(GraphProviderFoldoutGroup),
-         HideReferenceObjectPicker, ValueDropdown(nameof(ServerDropdown), ExpandAllMenuItems = true)]
-        public LSPServer Server = LSPServer.Pyright;
+         ValueDropdown(nameof(ServerDropdown), ExpandAllMenuItems = false)]
+        public string ServerName = $"{LSPLanguage.Python}/{LSPServer.Pyright}";
+
+        /// <summary>
+        /// The paths within the project (recursively) containing the source code to be analyzed.
+        /// </summary>
+        [Tooltip("The paths within the project (recursively) containing the source code to be analyzed."),
+         FolderPath(AbsolutePath = true, ParentFolder = "@" + nameof(ProjectPath) + ".Path", RequireExistingPath = true),
+         InfoBox("If no source paths are specified, all directories within the project path "
+                 + "containing source code will be considered.",
+                 InfoMessageType.Info, "@" + nameof(SourcePaths) + ".Length == 0"),
+         ValidateInput(nameof(ValidSourcePaths), "The source paths must be within the project path."),
+         RuntimeTab(GraphProviderFoldoutGroup)]
+        public string[] SourcePaths = Array.Empty<string>();
+
+        /// <summary>
+        /// The paths within the project whose contents should be excluded from the analysis.
+        /// </summary>
+        [Tooltip("The paths within the project whose contents should be excluded from the analysis."),
+         FolderPath(AbsolutePath = true, ParentFolder = "@" + nameof(ProjectPath) + ".Path", RequireExistingPath = true),
+         ValidateInput(nameof(ValidSourcePaths), "The source paths must be within the project path."),
+         RuntimeTab(GraphProviderFoldoutGroup)]
+        public string[] ExcludedSourcePaths = Array.Empty<string>();
+
+        // By default, all edge types are included, except for <see cref="EdgeKind.Definition"/>
+        // and <see cref="EdgeKind.Declaration"/>, since nodes would otherwise often get a self-reference.
+        [Title("Edge types"), Tooltip("The edge types to be included in the graph."), HideLabel]
+        [EnumToggleButtons, FoldoutGroup("Import Settings")]
+        public EdgeKind IncludedEdgeTypes = EdgeKind.All & ~(EdgeKind.Definition | EdgeKind.Declaration);
+
+        /// <summary>
+        /// If true, self-references will be avoided in the graph.
+        /// </summary>
+        [Tooltip("If true, self-references will be avoided in the graph."), FoldoutGroup("Import Settings")]
+        [LabelWidth(200)]
+        public bool AvoidSelfReferences = true;
+
+        /// <summary>
+        /// If true, references from a node to its direct parent will be avoided in the graph.
+        /// </summary>
+        [Tooltip("If true, references from a node to its direct parent will be avoided in the graph.")]
+        [FoldoutGroup("Import Settings"), LabelWidth(200)]
+        public bool AvoidParentReferences = true;
+
+        /// <summary>
+        /// The node types to be included in the graph.
+        /// </summary>
+        [Title("Node types"), Tooltip("The node types to be included in the graph."), HideLabel]
+        [EnumToggleButtons, FoldoutGroup("Import Settings")]
+        public NodeKind IncludedNodeTypes = NodeKind.All;
 
         /// <summary>
         /// If true, the communication between the language server and SEE will be logged.
         /// </summary>
         [Tooltip("If true, the communication between the language server and SEE will be logged."), RuntimeTab(GraphProviderFoldoutGroup)]
-        [InfoBox("@\"Logfiles can be found in \" + System.IO.Path.GetTempPath()", InfoMessageType.Info, nameof(LogLSP))]
+        [InfoBox("@\"Logfiles can be found in \" + System.IO.Path.GetTempPath() + "
+                 + "\" under inputLogLsp.txt and outputLogLsp.txt\"", InfoMessageType.Info, nameof(LogLSP))]
         public bool LogLSP;
 
+        /// <summary>
+        /// The maximum time to wait for the language server to respond.
+        /// </summary>
+        [LabelText("Timeout (seconds)")]
+        [Tooltip("The maximum time to wait for the language server to respond."
+             + " Responses after the timeout will not be considered."), RuntimeTab(GraphProviderFoldoutGroup)]
+        [InfoBox("No timeout will be applied.", InfoMessageType.Info, "@" + nameof(Timeout) + " == 0")]
+        [Range(0, 10), Unit(Units.Second)]
+        public double Timeout = 2;
+
+        /// <summary>
+        /// The language server to be used for the analysis.
+        /// </summary>
+        private LSPServer Server => LSPServer.GetByName(ServerName.Split('/')[1]);
+
+        /// <summary>
+        /// Returns whether all source paths are within the project path.
+        /// </summary>
+        private bool ValidSourcePaths => SourcePaths.All(path => path.StartsWith(ProjectPath.Path));
+
         /// <summary>
         /// Returns the available language servers as a dropdown list, grouped by language.
         /// </summary>
         /// <returns>The available language servers as a dropdown list.</returns>
-        private IEnumerable<ValueDropdownItem<LSPServer>> ServerDropdown()
+        private IEnumerable<string> ServerDropdown()
         {
             return LSPLanguage.All.Select(language => (language, LSPServer.All.Where(server => server.Languages.Contains(language))))
-                              .SelectMany(pair => pair.Item2.Select(server => new ValueDropdownItem<LSPServer>($"{pair.language.Name}/{server.Name}", server)));
+                              .SelectMany(pair => pair.Item2.Select(server => $"{pair.language}/{server}"));
         }
 
         public override GraphProviderKind GetKind()
@@ -74,37 +144,65 @@ public override async UniTask<Graph> ProvideAsync(Graph graph, AbstractSEECity c
                 throw new ArgumentException("The given city is null.\n");
             }
 
-            LSPHandler handler = city.gameObject.AddOrGetComponent<LSPHandler>();
+            if (city.gameObject.TryGetComponent(out LSPHandler oldHandler))
+            {
+                // We need to shut down the old handler before we can create a new one.
+                if (!Application.isPlaying)
+                {
+                    using (LoadingSpinner.ShowIndeterminate("Shutting down old LSP handler..."))
+                    {
+                        await oldHandler.ShutdownAsync(token);
+                    }
+                }
+                Destroyer.Destroy(oldHandler);
+            }
+            // Start with a small value to indicate that the process has started.
+            changePercentage?.Invoke(float.Epsilon);
+
+            LSPHandler handler = city.gameObject.AddComponent<LSPHandler>();
             handler.enabled = true;
             handler.Server = Server;
             handler.ProjectPath = ProjectPath.Path;
             handler.LogLSP = LogLSP;
-            if (Application.isPlaying)
+            handler.TimeoutSpan = TimeSpan.FromSeconds(Timeout);
+            await handler.InitializeAsync(executablePath: ServerPath ?? Server.ServerExecutable, token);
+            if (token.IsCancellationRequested)
             {
-                await handler.WaitUntilReadyAsync();
+                throw new OperationCanceledException();
             }
-            else
+            changePercentage?.Invoke(0.0001f);
+
+            if (SourcePaths.Length == 0)
             {
-                // Since OnEnable is not called in the editor, we have to initialize the handler manually.
-                await handler.InitializeAsync();
+                SourcePaths = new[] { ProjectPath.Path };
             }
 
-            SymbolInformationOrDocumentSymbolContainer result = await handler.Client.RequestDocumentSymbol(new DocumentSymbolParams
+            IDisposable spinner = LoadingSpinner.Show("Creating graph from language server...");
+            try
             {
-                // TODO: Use root path to query all relevant filetypes.
-                TextDocument = new TextDocumentIdentifier(Path.Combine(ProjectPath.Path, "src/token/mod.rs"))
-            });
-            foreach (SymbolInformationOrDocumentSymbol symbol in result)
+                await UniTask.SwitchToThreadPool();
+                // TODO: Use cancellation token to cancel the task if requested.
+                LSPImporter importer = new(handler, SourcePaths, ExcludedSourcePaths, IncludedNodeTypes,
+                                           IncludedEdgeTypes, AvoidSelfReferences, AvoidParentReferences);
+                await importer.LoadAsync(graph, changePercentage);
+            }
+            catch (TimeoutException)
             {
-                if (symbol.IsDocumentSymbolInformation)
+                string message = "The language server did not respond in time.";
+                if (LogLSP)
                 {
-                    Debug.LogError("This language server emits SymbolInformation, which is deprecated and not "
-                                   + "supported by SEE. Please choose a language server that is capable of returning "
-                                   + "hierarchic DocumentSymbols.\n");
-                    break;
+                    message += $" Check the output log at {Path.GetTempPath()}outputLogLsp.txt";
                 }
-
-                // TODO: Use algorithm 1 from master's thesis.
+                else
+                {
+                    message += " Enable logging in the graph provider to see what went wrong.";
+                }
+                Debug.LogError(message + "\n");
+            }
+            finally
+            {
+                await UniTask.SwitchToMainThread();
+                spinner.Dispose();
             }
 
             // We shut down the LSP server for now. If it is needed again, it can still be restarted.
@@ -114,9 +212,9 @@ public override async UniTask<Graph> ProvideAsync(Graph graph, AbstractSEECity c
             }
             else
             {
-                await handler.ShutdownAsync();
+                handler.ShutdownAsync().Forget();
             }
-            return null;
+            return graph;
         }
 
 
@@ -140,14 +238,14 @@ public override async UniTask<Graph> ProvideAsync(Graph graph, AbstractSEECity c
         protected override void SaveAttributes(ConfigWriter writer)
         {
             ProjectPath.Save(writer, pathLabel);
-            writer.Save(Server.Name, serverLabel);
+            writer.Save(ServerName, serverLabel);
             writer.Save(LogLSP, logLSPLabel);
         }
 
         protected override void RestoreAttributes(Dictionary<string, object> attributes)
         {
             ProjectPath.Restore(attributes, pathLabel);
-            Server = LSPServer.GetByName((string)attributes[serverLabel]);
+            ServerName = (string)attributes[serverLabel];
             LogLSP = (bool)attributes[logLSPLabel];
         }
 

Assets/SEE/SEE.asmdef

@@ -62,7 +62,8 @@
         "OmniSharp.Extensions.LanguageServer.Shared.dll",
         "OmniSharp.Extensions.JsonRpc.dll",
         "System.IO.Pipelines.dll",
-        "Supercluster.KDTree.Standard.dll"
+        "Supercluster.KDTree.Standard.dll",
+        "Markdig.dll"
     ],
     "autoReferenced": false,
     "defineConstraints": [],