Building a Neural-Symbolic Knowledge Graph Engine in Scala

1 $ make repl

1 (ask (?a) [?a :wrote :nsk] [?a :codes-in :lisp])

 1 project.scala      scala-cli build config (Scala version, main class)
 2 src/Term.scala     terms and triples, the data model
 3 src/Unify.scala    Norvig-style unification over Option
 4 src/Graph.scala    the triplestore: indices and the append-only log
 5 src/Reader.scala   the NSK reader: text to an abstract syntax tree
 6 src/Query.scala    pattern matching, conjunction, and the neural fallback
 7 src/Neural.scala   the Ollama client and text-to-triples
 8 src/Json.scala     a self-contained JSON model, parser, and writer
 9 src/Dsl.scala      the nsk"..." interpolator and the native Scala DSL
10 src/Repl.scala     the interactive read-eval-print loop
11 src/Server.scala   the REST server on the JDK HTTP server
12 src/Main.scala     the command-line entry point
13 Tests.scala        the test suite

 1 //> using scala 3.8.4
 2 //> using mainClass nsk.main
 3 
 4 // NSK: Neural-Symbolic Knowledge Graph engine, ported from Common Lisp.
 5 //
 6 // The core has no external dependencies. JSON is hand-rolled, the Ollama
 7 // client uses the JDK HttpClient, and the REST server uses the JDK's
 8 // built-in com.sun.net.httpserver. Build and run with scala-cli:
 9 //
10 //   scala-cli run .                         start the REPL
11 //   scala-cli run . -- --serve --port 8800  start the REST server
12 //   scala-cli run . --main-class nsk.test   run the test suite

1 ?who          a logic variable
2 :mark         a keyword atom (stored upper case as MARK)
3 "hello"       a string value
4 42            an integer
5 ~:codes-in    a neural predicate wrapping the atom :codes-in

 1 package nsk
 2 
 3 /** A term in a triple or a query pattern.
 4   *
 5   * Keyword names are stored upper-case, the way the Common Lisp reader
 6   * upcases `:mark` to `:MARK`. The three `toString`/`display`/`jsonText`
 7   * views mirror Common Lisp's `~S`, `~A`, and the JSON rendering.
 8   */
 9 enum Term:
10   case Var(name: String)     // logic variable:   ?who
11   case Kw(name: String)      // symbolic atom:     :mark   (name is UPPER-CASE)
12   case Str(value: String)    // string literal:    "hello"
13   case Num(value: Long)      // integer literal:   42
14   case Neural(inner: Term)   // neural predicate:  ~:codes-in
15 
16   /** Readable form (Common Lisp `~S`). Round-trips back through the reader. */
17   override def toString: String = this match
18     case Var(n)    => s"?$n"
19     case Kw(n)     => s":$n"
20     case Str(s)    => "\"" + s + "\""
21     case Num(n)    => n.toString
22     case Neural(t) => s"~$t"
23 
24   /** Plain form (Common Lisp `~A`): a keyword without its colon. */
25   def display: String = this match
26     case Kw(n)     => n
27     case Str(s)    => s
28     case Num(n)    => n.toString
29     case Var(n)    => s"?$n"
30     case Neural(t) => s"~${t.display}"
31 
32   /** Lower-case text used for JSON output. */
33   def jsonText: String = this match
34     case Kw(n)     => n.toLowerCase
35     case Str(s)    => s
36     case Num(n)    => n.toString
37     case Var(n)    => s"?$n"
38     case Neural(t) => t.jsonText
39 
40 object Term:
41   /** True when TERM is a concrete value usable as an index key. */
42   def concrete(t: Term): Boolean = t match
43     case _: Var | _: Neural => false
44     case _                  => true
45 
46 /** A subject-predicate-object triple. */
47 case class Triple(s: Term, p: Term, o: Term):
48   def toList: List[Term]      = List(s, p, o)
49   override def toString: String = s"[$s $p $o]"

 1 package nsk
 2 
 3 /** Unification in the Norvig/PAIP style.
 4   *
 5   * The environment is a map from logic variable to value. Common Lisp uses a
 6   * `+FAIL+` sentinel so that `NIL` can stand for the empty successful
 7   * environment; Scala models the same idea more directly with `Option`, where
 8   * `None` is failure and `Some(env)` is success.
 9   */
10 object Unify:
11   type Env = Map[Term.Var, Term]
12 
13   val empty: Env = Map.empty
14 
15   def unify(x: Term, y: Term, env: Option[Env] = Some(empty)): Option[Env] =
16     env.flatMap: e =>
17       (x, y) match
18         case _ if x == y      => Some(e)
19         case (v: Term.Var, _) => unifyVar(v, y, e)
20         case (_, v: Term.Var) => unifyVar(v, x, e)
21         case _                => None
22 
23   private def unifyVar(v: Term.Var, x: Term, e: Env): Option[Env] =
24     e.get(v) match
25       case Some(bound) => unify(bound, x, Some(e))
26       case None        => Some(e + (v -> x))
27 
28   /** Unify a pattern triple against a stored triple, position by position. */
29   def unifyTriple(pat: Triple, tr: Triple, env: Env): Option[Env] =
30     unify(pat.s, tr.s, Some(env))
31       .flatMap(e => unify(pat.p, tr.p, Some(e)))
32       .flatMap(e => unify(pat.o, tr.o, Some(e)))
33 
34   /** Replace bound variables in X with their values, recursively. */
35   def resolve(x: Term, e: Env): Term = x match
36     case v: Term.Var => e.get(v).map(resolve(_, e)).getOrElse(v)
37     case _           => x

1 (:add :mark :wrote :nsk)
2 (:add :jane :wrote :book)
3 (:add :mark :codes-in :lisp)
4 (:del :jane :wrote :book)

  1 package nsk
  2 
  3 import java.io.{BufferedReader, File, FileReader, FileWriter, PrintWriter}
  4 import scala.collection.mutable
  5 
  6 /** The triplestore: two hash indices plus an append-only log.
  7   *
  8   * Triples index by subject and by object for fast pattern narrowing. State
  9   * survives restarts through a log of forms that is replayed on startup. The
 10   * log uses the NSK surface syntax, so [[Reader]] replays it and no `eval` is
 11   * involved, which makes replay safe by construction.
 12   */
 13 final class Graph private (val logPath: Option[String]):
 14   private val spo  = mutable.Map.empty[Term, List[Triple]] // subject -> triples
 15   private val osp  = mutable.Map.empty[Term, List[Triple]] // object  -> triples
 16   private var live = List.empty[Triple]                    // every live triple, newest first
 17   private var n    = 0
 18   private var log  = Option.empty[PrintWriter]
 19 
 20   def count: Int                    = n
 21   def all: List[Triple]             = live.reverse         // insertion order
 22   def spoIndex(k: Term): List[Triple] = spo.getOrElse(k, Nil)
 23   def ospIndex(k: Term): List[Triple] = osp.getOrElse(k, Nil)
 24 
 25   private def index(tr: Triple): Unit =
 26     spo(tr.s) = tr :: spoIndex(tr.s)
 27     osp(tr.o) = tr :: ospIndex(tr.o)
 28     live = tr :: live
 29     n += 1
 30 
 31   private def unindex(tr: Triple): Unit =
 32     spo(tr.s) = spoIndex(tr.s).filterNot(_ == tr)
 33     osp(tr.o) = ospIndex(tr.o).filterNot(_ == tr)
 34     live = live.filterNot(_ == tr)
 35     n -= 1
 36 
 37   def present(tr: Triple): Boolean = spoIndex(tr.s).contains(tr)
 38 
 39   private def writeLog(op: String, tr: Triple): Unit =
 40     log.foreach { w => w.println(Graph.entry(op, tr)); w.flush() }
 41 
 42   /** Add a triple and append it to the log. Duplicates are ignored. */
 43   def add(s: Term, p: Term, o: Term): Triple =
 44     val tr = Triple(s, p, o)
 45     if !present(tr) then { index(tr); writeLog("add", tr) }
 46     tr
 47 
 48   /** Remove a triple and record the deletion in the log. */
 49   def remove(s: Term, p: Term, o: Term): Triple =
 50     val tr = Triple(s, p, o)
 51     if present(tr) then { unindex(tr); writeLog("del", tr) }
 52     tr
 53 
 54   /** Triples that could match PATTERN, narrowed by the indices. */
 55   def candidates(pat: Triple): List[Triple] =
 56     if Term.concrete(pat.s) then spoIndex(pat.s)
 57     else if Term.concrete(pat.o) then ospIndex(pat.o)
 58     else live
 59 
 60   private def applyEntry(form: Form): Unit = form match
 61     case Form.Cmd("add", List(s, p, o)) =>
 62       val tr = Triple(s, p, o); if !present(tr) then index(tr)
 63     case Form.Cmd("del", List(s, p, o)) =>
 64       val tr = Triple(s, p, o); if present(tr) then unindex(tr)
 65     case _ => ()
 66 
 67   private def replay(path: String): Unit =
 68     val f = File(path)
 69     if f.exists then
 70       val in = BufferedReader(FileReader(f))
 71       try
 72         var line = in.readLine()
 73         while line != null do
 74           val t = line.trim
 75           if t.nonEmpty then applyEntry(Reader.readForm(t))
 76           line = in.readLine()
 77       finally in.close()
 78 
 79   private def openLog(path: String): Unit =
 80     log = Some(PrintWriter(FileWriter(path, /* append = */ true)))
 81 
 82   def flush(): Unit = log.foreach(_.flush())
 83 
 84   def close(): Unit =
 85     log.foreach { w => w.flush(); w.close() }
 86     log = None
 87 
 88 object Graph:
 89   /** A fresh in-memory graph with no backing log. */
 90   def inMemory: Graph = new Graph(None)
 91 
 92   /** Open (or create) the store at PATH, replay its log, keep it open. */
 93   def open(path: String): Graph =
 94     val g = new Graph(Some(path))
 95     g.replay(path)
 96     g.openLog(path)
 97     g
 98 
 99   private def entry(op: String, tr: Triple): String =
100     s"(:$op ${source(tr.s)} ${source(tr.p)} ${source(tr.o)})"
101 
102   private def source(t: Term): String = t match
103     case Term.Kw(name) => ":" + name.toLowerCase
104     case Term.Str(s)   => "\"" + s.replace("\\", "\\\\").replace("\"", "\\\"") + "\""
105     case Term.Num(x)   => x.toString
106     case other         => other.toString

1 (:add :mark :wrote :nsk)                            a command
2 [?who :wrote :nsk]                                  a single pattern
3 (ask (?a) [?a :wrote :nsk] [?a :codes-in :lisp])    a conjunctive query

  1 package nsk
  2 
  3 import scala.collection.mutable.ListBuffer
  4 
  5 /** A parsed top-level form. */
  6 enum Form:
  7   case Cmd(name: String, args: List[Term])          // (:add s p o) or a bare :count
  8   case Pat(triple: Triple)                          // [s p o]
  9   case Ask(vars: List[Term.Var], clauses: List[Triple]) // (ask (?a) [..] [..])
 10 
 11 /** Thrown when the input runs out in the middle of a form. The REPL treats
 12   * this as "read another line", the way a Lisp reader waits for more input. */
 13 final class IncompleteInput extends RuntimeException("incomplete input")
 14 
 15 /** Thrown on malformed input. */
 16 final class ReadError(msg: String) extends RuntimeException(msg)
 17 
 18 /** The NSK reader.
 19   *
 20   * This is the Scala answer to the Common Lisp reader macros. It turns the
 21   * same surface syntax into an abstract syntax tree:
 22   *
 23   *   ?name    -> Term.Var(name)
 24   *   :name    -> Term.Kw(NAME)          (upper-cased, as in Common Lisp)
 25   *   ~term    -> Term.Neural(term)
 26   *   [s p o]  -> Form.Pat(Triple(...))
 27   *   (:op ..) -> Form.Cmd(op, args)
 28   *   (ask ..) -> Form.Ask(vars, clauses)
 29   */
 30 object Reader:
 31   /** Read exactly one form from S (trailing text is ignored). */
 32   def readForm(s: String): Form =
 33     val c = Cursor(s)
 34     c.form()
 35 
 36   /** Read as many complete forms as possible; return them and the index of
 37     * the first unconsumed character (an incomplete trailing form is left). */
 38   def readForms(s: String): (List[Form], Int) =
 39     val c    = Cursor(s)
 40     val out  = ListBuffer.empty[Form]
 41     var more = true
 42     while more do
 43       c.skipWs()
 44       if c.atEnd then more = false
 45       else
 46         val mark = c.pos
 47         try out += c.form()
 48         catch
 49           case _: IncompleteInput =>
 50             c.pos = mark
 51             more = false
 52     (out.toList, c.pos)
 53 
 54   private final class Cursor(s: String):
 55     var pos = 0
 56 
 57     def atEnd: Boolean = pos >= s.length
 58     def peek: Char     = s(pos)
 59 
 60     def skipWs(): Unit = while pos < s.length && s(pos).isWhitespace do pos += 1
 61 
 62     private def expect(ch: Char): Unit =
 63       skipWs()
 64       if atEnd then throw IncompleteInput()
 65       if s(pos) == ch then pos += 1 else throw ReadError(s"expected '$ch' at $pos")
 66 
 67     private def tokenChars(): String =
 68       val start = pos
 69       while pos < s.length && !s(pos).isWhitespace && !"()[]".contains(s(pos)) do pos += 1
 70       if pos == start then throw ReadError(s"empty token at $pos")
 71       s.substring(start, pos)
 72 
 73     /** A single term. */
 74     def term(): Term =
 75       skipWs()
 76       if atEnd then throw IncompleteInput()
 77       peek match
 78         case '?'                        => pos += 1; Term.Var(tokenChars())
 79         case ':'                        => pos += 1; Term.Kw(tokenChars().toUpperCase)
 80         case '~'                        => pos += 1; Term.Neural(term())
 81         case '"'                        => Term.Str(stringLiteral())
 82         case c if c == '-' || c.isDigit => number()
 83         case c                          => throw ReadError(s"unexpected term char '$c' at $pos")
 84 
 85     private def number(): Term =
 86       val start = pos
 87       if peek == '-' then pos += 1
 88       while pos < s.length && s(pos).isDigit do pos += 1
 89       Term.Num(s.substring(start, pos).toLong)
 90 
 91     private def stringLiteral(): String =
 92       expect('"')
 93       val sb   = StringBuilder()
 94       var done = false
 95       while !done do
 96         if atEnd then throw IncompleteInput()
 97         val ch = s(pos); pos += 1
 98         ch match
 99           case '"'  => done = true
100           case '\\' =>
101             if atEnd then throw IncompleteInput()
102             val e = s(pos); pos += 1
103             sb.append(e match
104               case 'n'   => '\n'
105               case 't'   => '\t'
106               case 'r'   => '\r'
107               case other => other
108             )
109           case c => sb.append(c)
110       sb.toString
111 
112     /** A [s p o] triple pattern. */
113     def triple(): Triple =
114       expect('[')
115       val terms = termsUntil(']')
116       if terms.length != 3 then
117         throw ReadError(s"triple pattern needs exactly three terms, got ${terms.length}")
118       Triple(terms(0), terms(1), terms(2))
119 
120     private def termsUntil(close: Char): List[Term] =
121       val out  = ListBuffer.empty[Term]
122       var done = false
123       while !done do
124         skipWs()
125         if atEnd then throw IncompleteInput()
126         if peek == close then { pos += 1; done = true }
127         else out += term()
128       out.toList
129 
130     private def clausesUntil(close: Char): List[Triple] =
131       val out  = ListBuffer.empty[Triple]
132       var done = false
133       while !done do
134         skipWs()
135         if atEnd then throw IncompleteInput()
136         peek match
137           case c if c == close => pos += 1; done = true
138           case '['             => out += triple()
139           case _               => throw ReadError(s"ask clause is not a [triple] at $pos")
140       out.toList
141 
142     private def bareWord(): String =
143       skipWs()
144       val start = pos
145       while pos < s.length && !s(pos).isWhitespace && !"()[]".contains(s(pos)) do pos += 1
146       s.substring(start, pos)
147 
148     /** A top-level form. */
149     def form(): Form =
150       skipWs()
151       if atEnd then throw IncompleteInput()
152       peek match
153         case '(' => listForm()
154         case '[' => Form.Pat(triple())
155         case ':' => pos += 1; Form.Cmd(tokenChars().toLowerCase, Nil)
156         case c   => throw ReadError(s"unexpected char '$c' at $pos")
157 
158     private def listForm(): Form =
159       expect('(')
160       skipWs()
161       if atEnd then throw IncompleteInput()
162       if peek == ':' then
163         pos += 1
164         val name = tokenChars().toLowerCase
165         Form.Cmd(name, termsUntil(')'))
166       else
167         bareWord() match
168           case "ask" =>
169             skipWs(); expect('(')
170             val vars    = termsUntil(')').collect { case v: Term.Var => v }
171             val clauses = clausesUntil(')')
172             Form.Ask(vars, clauses)
173           case head => throw ReadError(s"unknown list head '$head'")

 1 package nsk
 2 
 3 import Unify.{Env, empty, unify, unifyTriple, resolve}
 4 
 5 /** The result of a query: one row of `(name, value)` bindings per solution.
 6   *
 7   * The `toString` view mirrors the Common Lisp `query-result` printer so the
 8   * REPL shows a readable table.
 9   */
10 case class QueryResult(solutions: List[List[(String, Term)]]):
11   override def toString: String =
12     if solutions.isEmpty then "#<no solutions>"
13     else if solutions == List(Nil) then "yes"
14     else
15       solutions
16         .map(sol => sol.map((k, v) => s"$k=$v").mkString(", "))
17         .mkString("\n")
18 
19 /** Pattern matching, conjunction, and the neural fallback.
20   *
21   * The active graph and the neural configuration arrive as `using` parameters.
22   * They are the Scala stand-in for the Common Lisp special variables `*graph*`
23   * and `*ollama-url*`: a caller supplies a local `given` to rebind them, just
24   * as Lisp code rebinds a special with `let`.
25   */
26 object Query:
27 
28   /** Replace bound variables in a pattern with their values, for indexing. */
29   private def ground(pat: Triple, env: Env): Triple =
30     def g(t: Term): Term = t match
31       case v: Term.Var => resolve(v, env)
32       case _           => t
33     Triple(g(pat.s), g(pat.p), g(pat.o))
34 
35   /** Every environment that satisfies PAT under ENV. A neural predicate first
36     * tries a plain symbolic match on its bare name, then asks the model. */
37   def matchPattern(pat: Triple, env: Env)(using graph: Graph, cfg: NeuralConfig): List[Env] =
38     val neuralp = pat.p.isInstanceOf[Term.Neural]
39     val spat = pat.p match
40       case Term.Neural(inner) => pat.copy(p = inner)
41       case _                  => pat
42     val gpat    = ground(spat, env)
43     val results = graph.candidates(gpat).flatMap(tr => unifyTriple(spat, tr, env))
44     if results.isEmpty && neuralp then neuralMatch(gpat, spat, env).toList
45     else results
46 
47   private def neuralMatch(grounded: Triple, spat: Triple, env: Env)(using
48       cfg: NeuralConfig
49   ): Option[Env] =
50     (grounded.s, spat.o) match
51       case (subject, target: Term.Var) if Term.concrete(subject) =>
52         Neural
53           .queryFallback(subject, grounded.p)
54           .flatMap(answer => unify(target, Neural.sanitizeToKeyword(answer), Some(env)))
55       case _ => None
56 
57   /** Prove PATTERNS as a conjunction, threading environments forward. */
58   def prove(patterns: List[Triple], env: Env)(using Graph, NeuralConfig): List[Env] =
59     patterns match
60       case Nil          => List(env)
61       case pat :: rest  => matchPattern(pat, env).flatMap(e => prove(rest, e))
62 
63   def runQuery(patterns: List[Triple], vars: List[Term.Var])(using
64       Graph,
65       NeuralConfig
66   ): QueryResult =
67     val sols = prove(patterns, empty).map(env => vars.map(v => v.name -> resolve(v, env)))
68     QueryResult(sols.distinct)
69 
70   def matchTriple(pat: Triple)(using Graph, NeuralConfig): QueryResult =
71     runQuery(List(pat), collectVars(pat))
72 
73   def matchTriple(s: Term, p: Term, o: Term)(using Graph, NeuralConfig): QueryResult =
74     matchTriple(Triple(s, p, o))
75 
76   /** Run a query form produced by the reader or the `nsk` interpolator. */
77   def run(form: Form)(using Graph, NeuralConfig): QueryResult = form match
78     case Form.Pat(t)      => matchTriple(t)
79     case Form.Ask(vs, cs) => runQuery(cs, vs)
80     case Form.Cmd(n, _)   => throw IllegalArgumentException(s"not a query: $n")
81 
82   private def collectVars(pat: Triple): List[Term.Var] =
83     pat.toList.flatMap {
84       case v: Term.Var        => List(v)
85       case Term.Neural(inner) => inner match { case v: Term.Var => List(v); case _ => Nil }
86       case _                  => Nil
87     }.distinct

1 {
2   "model": "qwen3.5:4b",
3   "system": "You are a graph database inference node. Given a Subject and a Predicate, infer the single most likely Object. Reply ONLY as JSON: {\"result\": \"value\"}.",
4   "prompt": "Subject: mark. Predicate: favorite-language. What is the Object?",
5   "format": "json",
6   "stream": false
7 }

1 {
2   "model": "qwen3.5:4b",
3   "created_at": "2026-07-28T12:00:00Z",
4   "response": "{\"result\": \"Common Lisp\"}",
5   "done": true
6 }

  1 package nsk
  2 
  3 import java.net.URI
  4 import java.net.http.{HttpClient, HttpRequest, HttpResponse}
  5 import java.time.Duration
  6 
  7 /** Configuration for the neural layer.
  8   *
  9   * Passed as a `given`, this replaces the Common Lisp specials `*ollama-url*`,
 10   * `*ollama-model*`, and `*ollama-timeout*`. A test rebinds it with a local
 11   * `given`, the way Lisp code rebinds a special with `let`.
 12   */
 13 case class NeuralConfig(
 14     url: String = "http://localhost:11434",
 15     model: String = "qwen3.5:4b",
 16     timeoutSeconds: Int = 60
 17 )
 18 
 19 object NeuralConfig:
 20   given default: NeuralConfig = NeuralConfig()
 21 
 22 /** The neural integration layer (a local Ollama daemon).
 23   *
 24   * When a symbolic query fails on a `~` predicate, NSK asks the model to infer
 25   * the missing object. The same layer turns free text into triples. HTTP goes
 26   * through the JDK's own `HttpClient`, so the core keeps no dependency on an
 27   * external HTTP library.
 28   */
 29 object Neural:
 30   private val systemInference =
 31     "You are a graph database inference node. Given a Subject and a Predicate, " +
 32       "infer the single most likely Object. Reply ONLY as JSON: {\"result\": \"value\"}."
 33 
 34   private val systemExtraction =
 35     "You extract knowledge-graph triples from text. Reply ONLY as JSON of the form " +
 36       "{\"triples\": [{\"subject\": \"..\", \"predicate\": \"..\", \"object\": \"..\"}]}. " +
 37       "Use short lower-case tokens."
 38 
 39   /** Readable label for a subject or predicate term. */
 40   def termLabel(t: Term): String = t match
 41     case Term.Neural(inner) => termLabel(inner)
 42     case Term.Kw(name)      => name.toLowerCase
 43     case Term.Str(s)        => s
 44     case other              => other.display
 45 
 46   /** Convert an LLM string such as "Common Lisp" into the keyword `:COMMON-LISP`. */
 47   def sanitizeToKeyword(s: String): Term.Kw =
 48     val trimSet = " \t\n\r.,".toSet
 49     val trimmed = s.dropWhile(trimSet).reverse.dropWhile(trimSet).reverse
 50     Term.Kw(trimmed.toUpperCase.replace(' ', '-'))
 51 
 52   private def post(url: String, body: String, cfg: NeuralConfig): String =
 53     val client = HttpClient
 54       .newBuilder()
 55       .connectTimeout(Duration.ofSeconds(cfg.timeoutSeconds))
 56       .build()
 57     val req = HttpRequest
 58       .newBuilder(URI.create(url))
 59       .timeout(Duration.ofSeconds(cfg.timeoutSeconds))
 60       .header("Content-Type", "application/json")
 61       .POST(HttpRequest.BodyPublishers.ofString(body))
 62       .build()
 63     client.send(req, HttpResponse.BodyHandlers.ofString()).body()
 64 
 65   /** Send a /api/generate request and return the model's raw response string. */
 66   private def generate(prompt: String, system: String)(using cfg: NeuralConfig): String =
 67     val payload = Json
 68       .obj(
 69         "model"  -> Json.Str(cfg.model),
 70         "system" -> Json.Str(system),
 71         "prompt" -> Json.Str(prompt),
 72         "format" -> Json.Str("json"),
 73         "stream" -> Json.Bool(false)
 74       )
 75       .render
 76     val raw = post(s"${cfg.url}/api/generate", payload, cfg)
 77     Json.parse(raw)("response").flatMap(_.str).getOrElse("")
 78 
 79   /** Ask the model to infer the object for (SUBJECT PREDICATE). Returns the
 80     * answer string, or None when the daemon is unreachable or gives nothing. */
 81   def queryFallback(subject: Term, predicate: Term)(using NeuralConfig): Option[String] =
 82     try
 83       val prompt =
 84         s"Subject: ${termLabel(subject)}. Predicate: ${termLabel(predicate)}. What is the Object?"
 85       val response = generate(prompt, systemInference)
 86       val inner    = try Json.parse(response) catch case _: Throwable => Json.Null
 87       inner("result").flatMap(_.str).filter(_.nonEmpty).orElse(Some(response).filter(_.nonEmpty))
 88     catch
 89       case e: Throwable =>
 90         Console.err.println(s"; neural fallback unavailable: ${e.getMessage}")
 91         None
 92 
 93   /** Use the model to parse TEXT into a list of keyword triples. */
 94   def textToTriples(text: String)(using NeuralConfig): List[Triple] =
 95     try
 96       val response = generate(text, systemExtraction)
 97       val rows = Json.parse(response)("triples").collect { case Json.Arr(items) => items }
 98         .getOrElse(Nil)
 99       rows.flatMap { row =>
100         for
101           s <- row("subject").flatMap(_.str)
102           p <- row("predicate").flatMap(_.str)
103           o <- row("object").flatMap(_.str)
104         yield Triple(sanitizeToKeyword(s), sanitizeToKeyword(p), sanitizeToKeyword(o))
105       }
106     catch
107       case e: Throwable =>
108         Console.err.println(s"; extraction unavailable: ${e.getMessage}")
109         Nil
110 
111   /** Extract triples from TEXT and add them to the graph. */
112   def ingestText(text: String)(using g: Graph, cfg: NeuralConfig): List[Triple] =
113     val triples = textToTriples(text)
114     triples.foreach(tr => g.add(tr.s, tr.p, tr.o))
115     triples

  1 package nsk
  2 
  3 /** A small, self-contained JSON model, parser, and writer.
  4   *
  5   * NSK keeps its own JSON code so the core has no external dependency, the
  6   * same choice the Common Lisp version makes. The Lisp writer takes a tagged
  7   * list and the reader returns alists; here JSON is a proper algebraic data
  8   * type, which is the idiomatic Scala model.
  9   */
 10 enum Json:
 11   case Str(value: String)
 12   case Num(value: Double)
 13   case Bool(value: Boolean)
 14   case Null
 15   case Arr(items: List[Json])
 16   case Obj(fields: List[(String, Json)])
 17 
 18   def render: String =
 19     val sb = StringBuilder()
 20     Json.write(this, sb)
 21     sb.toString
 22 
 23   /** Look up KEY in an object, the analog of Common Lisp `json-get`. */
 24   def apply(key: String): Option[Json] = this match
 25     case Obj(fs) => fs.collectFirst { case (k, v) if k == key => v }
 26     case _       => None
 27 
 28   def str: Option[String]    = this match { case Str(s) => Some(s); case _ => None }
 29   def double: Option[Double] = this match { case Num(n) => Some(n); case _ => None }
 30 
 31 object Json:
 32   def obj(fields: (String, Json)*): Json = Obj(fields.toList)
 33   def arr(items: Json*): Json            = Arr(items.toList)
 34 
 35   final class JsonError(msg: String) extends RuntimeException(msg)
 36 
 37   def parse(input: String): Json =
 38     val p = Parser(input)
 39     val v = p.value()
 40     p.skipWs()
 41     v
 42 
 43   // ----- writer -----------------------------------------------------------
 44 
 45   private def write(v: Json, sb: StringBuilder): Unit = v match
 46     case Str(s)  => writeString(s, sb)
 47     case Num(n)  => if n == n.toLong.toDouble then sb.append(n.toLong) else sb.append(n)
 48     case Bool(b) => sb.append(if b then "true" else "false")
 49     case Null    => sb.append("null")
 50     case Arr(items) =>
 51       sb.append('[')
 52       items.zipWithIndex.foreach { (it, i) => if i > 0 then sb.append(','); write(it, sb) }
 53       sb.append(']')
 54     case Obj(fields) =>
 55       sb.append('{')
 56       fields.zipWithIndex.foreach { case ((k, vv), i) =>
 57         if i > 0 then sb.append(',')
 58         writeString(k, sb); sb.append(':'); write(vv, sb)
 59       }
 60       sb.append('}')
 61 
 62   private def writeString(s: String, sb: StringBuilder): Unit =
 63     sb.append('"')
 64     s.foreach {
 65       case '"'          => sb.append("\\\"")
 66       case '\\'         => sb.append("\\\\")
 67       case '\n'         => sb.append("\\n")
 68       case '\r'         => sb.append("\\r")
 69       case '\t'         => sb.append("\\t")
 70       case '\b'         => sb.append("\\b")
 71       case '\f'         => sb.append("\\f")
 72       case c if c < ' ' => sb.append("\\u%04x".format(c.toInt))
 73       case c            => sb.append(c)
 74     }
 75     sb.append('"')
 76 
 77   // ----- reader -----------------------------------------------------------
 78 
 79   private final class Parser(s: String):
 80     private var pos = 0
 81 
 82     private def fail(msg: String): Nothing = throw JsonError(s"$msg at $pos")
 83     private def atEnd: Boolean             = pos >= s.length
 84     private def peek: Char                 = s(pos)
 85     private def next(): Char               = { val c = s(pos); pos += 1; c }
 86 
 87     def skipWs(): Unit =
 88       while pos < s.length && " \t\n\r".contains(s(pos)) do pos += 1
 89 
 90     def value(): Json =
 91       skipWs()
 92       if atEnd then fail("unexpected end of input")
 93       peek match
 94         case '{'                       => obj()
 95         case '['                       => arr()
 96         case '"'                       => Str(string())
 97         case 't'                       => literal("true", Bool(true))
 98         case 'f'                       => literal("false", Bool(false))
 99         case 'n'                       => literal("null", Null)
100         case c if c == '-' || c.isDigit => number()
101         case c                         => fail(s"unexpected char $c")
102 
103     private def literal(text: String, v: Json): Json =
104       text.foreach(e => if atEnd || next() != e then fail(s"bad literal, expected $text"))
105       v
106 
107     private def obj(): Json =
108       next() // {
109       skipWs()
110       if !atEnd && peek == '}' then { next(); return Obj(Nil) }
111       val fields = List.newBuilder[(String, Json)]
112       var more   = true
113       while more do
114         skipWs()
115         val key = string()
116         skipWs()
117         if atEnd || next() != ':' then fail("expected : after object key")
118         fields += (key -> value())
119         skipWs()
120         next() match
121           case ',' => ()
122           case '}' => more = false
123           case _   => fail("expected , or } in object")
124       Obj(fields.result())
125 
126     private def arr(): Json =
127       next() // [
128       skipWs()
129       if !atEnd && peek == ']' then { next(); return Arr(Nil) }
130       val items = List.newBuilder[Json]
131       var more  = true
132       while more do
133         items += value()
134         skipWs()
135         next() match
136           case ',' => ()
137           case ']' => more = false
138           case _   => fail("expected , or ] in array")
139       Arr(items.result())
140 
141     private def string(): String =
142       if atEnd || next() != '"' then fail("expected a string")
143       val sb   = StringBuilder()
144       var done = false
145       while !done do
146         if atEnd then fail("unterminated string")
147         next() match
148           case '"'  => done = true
149           case '\\' =>
150             next() match
151               case '"'  => sb.append('"')
152               case '\\' => sb.append('\\')
153               case '/'  => sb.append('/')
154               case 'b'  => sb.append('\b')
155               case 'f'  => sb.append('\f')
156               case 'n'  => sb.append('\n')
157               case 'r'  => sb.append('\r')
158               case 't'  => sb.append('\t')
159               case 'u'  => sb.append(hex(4).toChar)
160               case _    => fail("bad string escape")
161           case c => sb.append(c)
162       sb.toString
163 
164     private def hex(n: Int): Int =
165       var v = 0
166       for _ <- 0 until n do
167         val d = Character.digit(next(), 16)
168         if d < 0 then fail("bad \\u escape")
169         v = v * 16 + d
170       v
171 
172     private def number(): Json =
173       val start = pos
174       if peek == '-' then next()
175       while !atEnd && (peek.isDigit || "+-.eE".contains(peek)) do next()
176       Num(s.substring(start, pos).toDouble)

 1 package nsk
 2 
 3 import java.io.{BufferedReader, ByteArrayOutputStream, PrintStream}
 4 import java.nio.charset.StandardCharsets.UTF_8
 5 
 6 /** The interactive NSK read-eval-print loop.
 7   *
 8   * Common Lisp reads a form and then `eval`s it. Scala has no `eval`, so the
 9   * loop instead reads a [[Form]] with [[Reader]] and interprets it directly.
10   * That is a little more code but keeps the language surface small and safe.
11   */
12 object Repl:
13   val banner: String =
14     "NSK: Neural-Symbolic Knowledge Graph Engine\nType :help for commands, :quit to exit."
15 
16   val help: String =
17     """Commands:
18       |  :help                 show this help
19       |  :facts                list every triple
20       |  :count                show the triple count
21       |  :add s p o            add a triple, e.g. (:add :mark :wrote :nsk)
22       |  :del s p o            remove a triple
23       |  :ingest "text"        extract triples from text with the model
24       |  :save                 flush the log to disk
25       |  :quit                 leave the REPL
26       |
27       |Queries use the NSK syntax:
28       |  [?who :wrote :nsk]                       one pattern
29       |  (ask (?a) [?a :wrote :nsk] [?a :codes-in :lisp])
30       |  [:mark ~:codes-in ?lang]                 ~ falls back to the model""".stripMargin
31 
32   /** Evaluate one form. Returns true to keep looping, false to quit. */
33   def eval(form: Form, out: PrintStream)(using Graph, NeuralConfig): Boolean =
34     form match
35       case Form.Cmd(name, args) => command(name, args, out)
36       case Form.Pat(_) | Form.Ask(_, _) =>
37         out.println(Query.run(form)); true
38 
39   private def command(name: String, args: List[Term], out: PrintStream)(using
40       g: Graph,
41       cfg: NeuralConfig
42   ): Boolean =
43     name match
44       case "quit" | "exit" => false
45       case "help"          => out.println(help); true
46       case "count"         => out.println(s"${g.count} triples"); true
47       case "save"          => g.flush(); out.println("saved."); true
48       case "facts" =>
49         g.all.foreach(tr => out.println(s"  ${tr.s.display}  ${tr.p.display}  ${tr.o.display}"))
50         out.println(s"(${g.count} triples)"); true
51       case "add" =>
52         args match
53           case List(s, p, o) =>
54             g.add(s, p, o); out.println(s"added ${s.display} ${p.display} ${o.display}")
55           case _ => out.println("usage: (:add s p o)")
56         true
57       case "del" =>
58         args match
59           case List(s, p, o) =>
60             g.remove(s, p, o); out.println(s"removed ${s.display} ${p.display} ${o.display}")
61           case _ => out.println("usage: (:del s p o)")
62         true
63       case "ingest" =>
64         args match
65           case List(Term.Str(text)) =>
66             val added = Neural.ingestText(text)
67             out.println(s"ingested ${added.length} triple${if added.length == 1 then "" else "s"}")
68           case _ => out.println("usage: (:ingest \"text\")")
69         true
70       case other => out.println(s"unknown command: $other"); true
71 
72   /** Process every complete form in INPUT, returning captured output. Used by
73     * the tests and by anyone who wants to script the REPL. */
74   def feed(input: String)(using Graph, NeuralConfig): String =
75     val buf = ByteArrayOutputStream()
76     val out = PrintStream(buf, true, UTF_8)
77     val (forms, _) = Reader.readForms(input + "\n")
78     forms.foreach(f => eval(f, out))
79     buf.toString(UTF_8)
80 
81   /** The interactive loop, reading line by line and buffering partial forms. */
82   def run(in: BufferedReader, out: PrintStream)(using Graph, NeuralConfig): Unit =
83     out.println(banner)
84     var buffer  = ""
85     var running = true
86     while running do
87       out.print("nsk> "); out.flush()
88       val line = in.readLine()
89       if line == null then running = false
90       else
91         buffer += line + "\n"
92         val (forms, consumed) = Reader.readForms(buffer)
93         buffer = buffer.substring(consumed)
94         val it = forms.iterator
95         while running && it.hasNext do
96           try if !eval(it.next(), out) then running = false
97           catch case e: Throwable => out.println(s"; error: ${e.getMessage}")
98     out.println("Bye.")

1 {"subject": null, "predicate": "wrote", "object": "nsk"}

1 {"count":1,"results":[{"subject":"mark"}]}

 1 package nsk
 2 
 3 import com.sun.net.httpserver.{HttpExchange, HttpServer}
 4 import java.net.InetSocketAddress
 5 import java.nio.charset.StandardCharsets.UTF_8
 6 
 7 /** An optional REST server for the `--serve` flag.
 8   *
 9   * Common Lisp loads Hunchentoot on demand; here the server uses the JDK's own
10   * `com.sun.net.httpserver`, so it needs no external dependency at all.
11   */
12 object Server:
13   @volatile private var server: Option[HttpServer] = None
14 
15   /** A request field maps to a graph term: null or missing means "any", a
16     * value starting with `?` is a variable, any other string is a keyword. */
17   def fieldToTerm(value: Option[Json], role: String): Term = value match
18     case None | Some(Json.Null)                        => Term.Var(role)
19     case Some(Json.Str(s)) if s.startsWith("?")        => Term.Var(s.drop(1))
20     case Some(Json.Str(s))                             => Neural.sanitizeToKeyword(s)
21     case Some(other)                                   => Term.Str(other.render)
22 
23   def termJson(t: Term): String = t match
24     case Term.Kw(name)  => name.toLowerCase
25     case Term.Str(s)    => s
26     case Term.Var(n)    => s"?$n"
27     case Term.Num(x)    => x.toString
28     case Term.Neural(i) => termJson(i)
29 
30   def solutionToJson(solution: List[(String, Term)]): Json =
31     Json.Obj(solution.map((k, v) => k -> Json.Str(termJson(v))))
32 
33   private def body(ex: HttpExchange): String =
34     new String(ex.getRequestBody.readAllBytes(), UTF_8)
35 
36   private def respond(ex: HttpExchange, json: String): Unit =
37     val bytes = json.getBytes(UTF_8)
38     ex.getResponseHeaders.set("Content-Type", "application/json")
39     ex.sendResponseHeaders(200, bytes.length.toLong)
40     val os = ex.getResponseBody
41     os.write(bytes)
42     os.close()
43 
44   private def handleQuery(ex: HttpExchange)(using Graph, NeuralConfig): Unit =
45     val req = try Json.parse(body(ex)) catch case _: Throwable => Json.Null
46     val s   = fieldToTerm(req("subject"), "subject")
47     val p   = fieldToTerm(req("predicate"), "predicate")
48     val o   = fieldToTerm(req("object"), "object")
49     val sols = Query.matchTriple(s, p, o).solutions
50     respond(
51       ex,
52       Json
53         .obj(
54           "count"   -> Json.Num(sols.length),
55           "results" -> Json.Arr(sols.map(solutionToJson))
56         )
57         .render
58     )
59 
60   private def handleHealth(ex: HttpExchange)(using g: Graph, cfg: NeuralConfig): Unit =
61     respond(
62       ex,
63       Json
64         .obj(
65           "status"  -> Json.Str("ok"),
66           "triples" -> Json.Num(g.count),
67           "model"   -> Json.Str(cfg.model)
68         )
69         .render
70     )
71 
72   /** Start serving /query and /health on PORT. */
73   def start(port: Int)(using Graph, NeuralConfig): HttpServer =
74     stop()
75     val s = HttpServer.create(InetSocketAddress(port), 0)
76     s.createContext("/query", (ex: HttpExchange) => handleQuery(ex))
77     s.createContext("/health", (ex: HttpExchange) => handleHealth(ex))
78     s.setExecutor(null)
79     s.start()
80     server = Some(s)
81     s
82 
83   /** Stop the running server, if any. */
84   def stop(): Unit =
85     server.foreach(_.stop(0))
86     server = None

 1 package nsk
 2 
 3 import java.io.{BufferedReader, InputStreamReader}
 4 
 5 /** Command-line entry point. Parse flags, then serve or drop into the REPL.
 6   * The object is named `App` (not `Main`) so its class file does not collide
 7   * with the `main.class` from `@main` on a case-insensitive filesystem.
 8   */
 9 @main def main(args: String*): Unit = App.run(args.toArray)
10 
11 object App:
12   val version = "NSK 1.0.0"
13 
14   def run(args: Array[String]): Unit =
15     if args.contains("--help") then
16       println(usage)
17     else
18       val dbPath = flagValue(args, "--db").getOrElse("nsk-graph.log")
19       given Graph        = Graph.open(dbPath)
20       given NeuralConfig = NeuralConfig()
21       try
22         if args.contains("--serve") then
23           val port = flagValue(args, "--port").map(_.toInt).getOrElse(8800)
24           Server.start(port)
25           println(s"$version serving on http://localhost:$port  (Ctrl-C to stop)")
26           while true do Thread.sleep(3600000)
27         else
28           Repl.run(BufferedReader(InputStreamReader(System.in)), System.out)
29       catch case e: Throwable => Console.err.println(s"fatal: ${e.getMessage}")
30       finally summon[Graph].close()
31 
32   private def flagValue(args: Array[String], flag: String): Option[String] =
33     val i = args.indexOf(flag)
34     Option.when(i >= 0 && i + 1 < args.length)(args(i + 1))
35 
36   def usage: String =
37     s"""$version
38        |
39        |Usage: nsk [options]
40        |
41        |  (no options)     start the interactive REPL
42        |  --serve          start the REST server instead of the REPL
43        |  --port N         server port (default 8800)
44        |  --db PATH        transaction log path (default nsk-graph.log)
45        |  --help           show this message
46        |
47        |Wrap the REPL with rlwrap for history and line editing:
48        |  rlwrap scala-cli run .""".stripMargin