Semantic tokens WIP (#4)

* WIP

* More tokens

* Use published version of langoustine

Author: keynmol

.gitignore

@@ -1 +1,5 @@
 bin/
+.bsp
+.metals
+.scala-build
+.vscode

LSP.scala

@@ -15,6 +15,9 @@ import langoustine.lsp.runtime.uinteger
 
 import cats.syntax.all.*
 import langoustine.lsp.app.LangoustineApp
+import langoustine.lsp.tools.SemanticTokensEncoder
+import langoustine.lsp.tools.SemanticToken
+import cats.effect.std.Semaphore
 
 object LSP extends LangoustineApp:
   import QuickmaffsLSP.{server, State}
@@ -85,10 +88,9 @@ object QuickmaffsLSP:
       processFile(Path(path))
 
     def set(u: DocumentUri)(st: State) =
-      state.update(_.updated(u, st)) <*
-        cats.effect.std
-          .Console[IO]
-          .errorln(s"Setting $st for $u")
+      state.update(_.updated(u, st)) <* IO.consoleForIO.errorln(
+        s"State update: $u is set to ${st.getClass}"
+      )
 
     def get(u: DocumentUri) =
       state.get.map(_.get(u))
@@ -150,6 +152,15 @@ object QuickmaffsLSP:
         case _ => None
       }
 
+    val encoder = SemanticTokensEncoder(
+      tokenTypes = Vector(
+        SemanticTokenTypes.variable,
+        SemanticTokenTypes.number,
+        SemanticTokenTypes.operator
+      ),
+      modifiers = Vector.empty
+    )
+
     LSPBuilder
       .create[IO]
       .handleRequest(initialize) { (in, back) =>
@@ -167,6 +178,12 @@ object QuickmaffsLSP:
                 definitionProvider = Opt(true),
                 documentSymbolProvider = Opt(true),
                 renameProvider = Opt(true),
+                semanticTokensProvider = Opt(
+                  SemanticTokensOptions(
+                    legend = encoder.legend,
+                    full = Opt(true)
+                  )
+                ),
                 textDocumentSync = Opt(
                   TextDocumentSyncOptions(
                     openClose = Opt(true),
@@ -187,6 +204,72 @@ object QuickmaffsLSP:
       .handleNotification(textDocument.didSave) { (in, back) =>
         recompile(in.textDocument.uri, back)
       }
+      .handleRequest(textDocument.semanticTokens.full) { (in, back) =>
+        get(in.textDocument.uri).flatMap {
+          case Some(State.Ok(idx, values, program)) =>
+            val tokens = Vector.newBuilder[SemanticToken]
+            program.statements.map(_.value).foreach { st =>
+              st match
+                case Statement.Ass(name, e) =>
+                  inline def nameToken(tok: Expr.Name[WithSpan]) =
+                    tokenFromSpan(tok.value.span, SemanticTokenTypes.variable)
+
+                  inline def tokenFromSpan(
+                      span: Span,
+                      tpe: SemanticTokenTypes
+                  ) =
+                    SemanticToken.fromRange(
+                      span.toRange,
+                      tokenType = tpe
+                    )
+
+                  tokens += nameToken(name)
+
+                  def go(expr: Expr[WithSpan]): Unit =
+                    expr match
+                      case Expr.Add(l, r, operator) =>
+                        go(l)
+                        go(r)
+                        tokens += tokenFromSpan(
+                          operator.span,
+                          SemanticTokenTypes.operator
+                        )
+
+                      case Expr.Mul(l, r, operator) =>
+                        go(l)
+                        go(r)
+                        tokens += tokenFromSpan(
+                          operator.span,
+                          SemanticTokenTypes.operator
+                        )
+
+                      case n @ Expr.Name(_) =>
+                        tokens += nameToken(n)
+
+                      case Expr.Lit(value) =>
+                        tokens += tokenFromSpan(
+                          value.span,
+                          SemanticTokenTypes.number
+                        )
+
+                  go(e)
+            }
+
+            IO.consoleForIO
+              .errorln(
+                s"Sending over the following tokens: ${tokens.result().map(_.toString)}"
+              ) *>
+              IO.fromEither(encoder.encode(tokens.result())).map(Opt(_))
+
+          case other =>
+            IO.consoleForIO
+              .errorln(
+                s"Got weird state for ${in.textDocument.uri}: $other"
+              )
+              .as(Opt.empty)
+
+        }
+      }
       .handleRequest(textDocument.definition) { (in, back) =>
         variableUnderCursor(in.textDocument.uri, in.position).map {
           foundMaybe =>

Makefile

@@ -4,13 +4,17 @@ clean:
 	rm -rf bin/*
 	scala-cli clean .
 
+setup:
+	scala-cli setup-ide *.scala
+
+
 bin/repl:
 	mkdir -p bin
 	scala-cli package . --main-class REPL --force --output bin/repl
 
 bin/lsp:
 	mkdir -p bin
-	scala-cli package . --main-class LSP --force --output bin/lsp
+	scala-cli package . --main-class LSP --force --output bin/lsp 
 
 bin/quickmaffs:
 	mkdir -p bin

build.dependencies.scala

@@ -19,7 +19,7 @@ object QuickmaffsCompiler:
   def compile(p: Program[WithSpan]): Result =
     val errors = Vector.newBuilder[CompileError]
 
-    def indexReferences(e: Expr[WithSpan]) =
+    def indexReferences(e: Expr[WithSpan]): Map[String, Vector[Span]] =
       inline def pair(e1: Expr[WithSpan], e2: Expr[WithSpan]) =
         val m1      = go(e1)
         val m2      = go(e2)
@@ -33,8 +33,8 @@ object QuickmaffsCompiler:
         expr match
           case _: Lit[?]                 => Map.empty
           case Name(WithSpan(pos, name)) => Map(name -> Vector(pos))
-          case Add(e1, e2)               => pair(e1, e2)
-          case Mul(e1, e2)               => pair(e1, e2)
+          case Add(e1, e2, _)            => pair(e1, e2)
+          case Mul(e1, e2, _)            => pair(e1, e2)
 
       go(e)
     end indexReferences

compiler.scala

@@ -37,10 +37,10 @@ object QuickmaffsEvaluator:
       case Expr.Name(v) =>
         state.get(v).toRight(EvaluationError.NameNotFound(v))
 
-      case Expr.Add(e1, e2) =>
+      case Expr.Add(e1, e2, _) =>
         (evaluate(e1, state), evaluate(e2, state)).mapN(_ + _)
 
-      case Expr.Mul(e1, e2) =>
+      case Expr.Mul(e1, e2, _) =>
         (evaluate(e1, state), evaluate(e2, state)).mapN(_ * _)
 
   @targetName("evaluate_WithSpan")

evaluator.scala

@@ -1,9 +1,10 @@
 worsts = (5 + 1)
 x = (worsts + 11)
 z = 17
-t = (x * 1)
+test = (x * 1)
 h = (x * (11 + worsts))
 hs = (x * (z + worsts))
+ts = 25
 a = 11
 r = (x + a)
 

examples/test.qmf

@@ -1,6 +1,8 @@
 b = (5 + 1)
 x = (b + 11)
 s = 17
-t = (asd * 1)
 h = (x * (11 + b))
 hs = (x * (11 + s))
+
+
+

examples/test2.qmf

@@ -3,18 +3,22 @@ import cats.parse.Parser0 as P0
 import cats.parse.Caret
 import cats.parse.Rfc5234.{sp, alpha, digit, crlf, lf}
 
+import cats.syntax.all.*
+
+case class Operator(raw: Char)
+
 enum Expr[F[_]]:
-  case Add(l: Expr[F], r: Expr[F])
-  case Mul(l: Expr[F], r: Expr[F])
+  case Add(l: Expr[F], r: Expr[F], operator: F[Operator])
+  case Mul(l: Expr[F], r: Expr[F], operator: F[Operator])
   case Name(value: F[String])
   case Lit(value: F[Int])
 
   def map[G[_]](f: [A] => F[A] => G[A]): Expr[G] =
     this match
-      case Lit(num)    => Lit(f(num))
-      case Name(num)   => Name(f(num))
-      case Add(e1, e2) => Add(e1.map(f), e2.map(f))
-      case Mul(e1, e2) => Mul(e1.map(f), e2.map(f))
+      case Lit(num)       => Lit(f(num))
+      case Name(num)      => Name(f(num))
+      case Add(e1, e2, o) => Add(e1.map(f), e2.map(f), f(o))
+      case Mul(e1, e2, o) => Mul(e1.map(f), e2.map(f), f(o))
 end Expr
 
 enum Statement[F[_]]:
@@ -65,17 +69,23 @@ object QuickmaffsParser:
   val litNum =
     withSpan(P.charsWhile(_.isDigit).map(_.toInt)).map(Expr.Lit.apply)
 
-  val LB        = P.char('(').surroundedBy(sp.rep0)
-  val RB        = P.char(')').surroundedBy(sp.rep0)
-  val PLUS      = P.char('+').surroundedBy(sp.rep0)
-  val MUL       = P.char('*').surroundedBy(sp.rep0)
-  val ASS       = P.char('=').surroundedBy(sp.rep0)
-  val SEMICOLON = P.char(';')
+  inline def operator(ch: Char): P[WithSpan[Operator]] =
+    withSpan(P.char(ch).as(Operator(ch))).surroundedBy(sp.rep0)
+
+  val LB        = operator('(')
+  val RB        = operator(')')
+  val PLUS      = operator('+')
+  val MUL       = operator('*')
+  val ASS       = operator('=')
+  val SEMICOLON = operator(';')
 
   private val expr = P.recursive[Expr[WithSpan]] { recurse =>
 
-    inline def pair[F[_]](p: P[Expr[F]], sym: P0[Unit]) =
-      (p <* sym) ~ p
+    inline def pair[F[_]](
+        p: P[Expr[F]],
+        sym: P[F[Operator]]
+    ): P[(Expr[F], Expr[F], F[Operator])] =
+      (p, sym, p).tupled.map { case (l, o, r) => (l, r, o) }
 
     val e = recurse.surroundedBy(sp.rep0)
 

parser.scala

@@ -0,0 +1,4 @@
+//> using scala "3.2.1"
+//> using repository "sonatype-s01:snapshots"
+//> using lib "tech.neander::langoustine-app::0.0.18+1-97948520-SNAPSHOT"
+//> using lib "org.typelevel::cats-parse::0.3.8"