Merge remote-tracking branch 'origin/main' into dz-csv-export-table-2

Author: davidwzhao

.github/workflows/ci.yml

@@ -69,16 +69,9 @@ jobs:
       with:
         version: ${{ matrix.julia-version }}
 
-    - name: Sync test data
-      run: |
-        mkdir -p sdks/julia/LogicalQueryProtocol/test/lqp sdks/julia/LogicalQueryProtocol/test/pretty sdks/julia/LogicalQueryProtocol/test/bin
-        cp tests/lqp/*.lqp sdks/julia/LogicalQueryProtocol/test/lqp/
-        cp tests/pretty/*.lqp sdks/julia/LogicalQueryProtocol/test/pretty/
-        cp tests/bin/*.bin sdks/julia/LogicalQueryProtocol/test/bin/
-
     - name: Run tests
       run: |
-        cd sdks/julia/LogicalQueryProtocol
+        cd sdks/julia/LogicalQueryProtocol.jl
         julia --project=. -e 'using Pkg; Pkg.instantiate(); Pkg.test()'
 
   go-test:

.gitignore

@@ -16,15 +16,6 @@ sdks/python/out/
 sdks/python/.lsp/.cache
 sdks/python/.clj-kondo
 
-# Julia SDK
-sdks/julia/LogicalQueryProtocol/Manifest.toml
-sdks/julia/LogicalQueryProtocol/test/lqp/**
-sdks/julia/LogicalQueryProtocol/test/bin/**
-sdks/julia/LogicalQueryProtocol/test/pretty/**
-sdks/julia/LogicalQueryProtocol/test/lqp_output/**
-sdks/julia/LogicalQueryProtocol/test/lqp_output/**
-sdks/julia/LogicalQueryProtocol/test/lqp_pretty_output/**
-
 # Go SDK
 sdks/go/test/lqp_pretty_output
 

Makefile

@@ -30,18 +30,18 @@ GRAMMAR := meta/src/meta/grammar.y
 
 # Generated protobuf outputs
 PY_PROTO_DIR := sdks/python/src/lqp/proto/v1
-JL_PROTO_DIR := sdks/julia/LogicalQueryProtocol/src/gen/relationalai/lqp/v1
+JL_PROTO_DIR := sdks/julia/LogicalQueryProtocol.jl/src/gen/relationalai/lqp/v1
 GO_PROTO_DIR := sdks/go/src/lqp/v1
 
 # Generated parser outputs
 PY_PARSER := sdks/python/src/lqp/gen/parser.py
-JL_PARSER := sdks/julia/LogicalQueryProtocol/src/parser.jl
+JL_PARSER := sdks/julia/LogicalQueryProtocol.jl/src/parser.jl
 GO_PARSER := sdks/go/src/parser.go
 
 # Generated printer outputs
 PY_PRINTER := sdks/python/src/lqp/gen/pretty.py
 GO_PRINTER := sdks/go/src/pretty.go
-JL_PRINTER := sdks/julia/LogicalQueryProtocol/src/pretty.jl
+JL_PRINTER := sdks/julia/LogicalQueryProtocol.jl/src/pretty.jl
 
 # Parser templates
 PY_TEMPLATE := meta/src/meta/templates/parser.py.template
@@ -80,8 +80,7 @@ JL_PROTO_GENERATED := \
 	force-printers force-printer-python force-printer-julia force-printer-go \
 	test test-python update-snapshots test-julia test-go \
 	test-meta check-python check-meta lint-meta format-meta \
-	lint-python format-python clean \
-	sync-julia-test-data
+	lint-python format-python clean
 
 all: protobuf parsers printers
 
@@ -112,7 +111,7 @@ $(PY_PROTO_GENERATED) $(GO_PROTO_GENERATED): $(PROTO_FILES)
 $(JL_PROTO_GENERATED): $(PROTO_FILES)
 	buf lint
 	buf breaking --against ".git#branch=main,subdir=proto"
-	cd sdks/julia && julia --project=LogicalQueryProtocol generate_proto.jl
+	cd sdks/julia && julia --project=LogicalQueryProtocol.jl generate_proto.jl
 
 # ---------- parser generation ----------
 
@@ -124,7 +123,7 @@ $(PY_PARSER): $(PROTO_FILES) $(GRAMMAR) $(PY_TEMPLATE)
 
 parser-julia: $(JL_PARSER)
 $(JL_PARSER): $(PROTO_FILES) $(GRAMMAR) $(JL_TEMPLATE)
-	$(META_CLI) $(META_PROTO_ARGS) --parser julia -o ../sdks/julia/LogicalQueryProtocol/src/parser.jl
+	$(META_CLI) $(META_PROTO_ARGS) --parser julia -o ../sdks/julia/LogicalQueryProtocol.jl/src/parser.jl
 
 parser-go: $(GO_PARSER)
 $(GO_PARSER): $(PROTO_FILES) $(GRAMMAR) $(GO_TEMPLATE)
@@ -136,7 +135,7 @@ force-parser-python:
 	$(META_CLI) $(META_PROTO_ARGS) --parser python -o ../sdks/python/src/lqp/gen/parser.py
 
 force-parser-julia:
-	$(META_CLI) $(META_PROTO_ARGS) --parser julia -o ../sdks/julia/LogicalQueryProtocol/src/parser.jl
+	$(META_CLI) $(META_PROTO_ARGS) --parser julia -o ../sdks/julia/LogicalQueryProtocol.jl/src/parser.jl
 
 force-parser-go:
 	$(META_CLI) $(META_PROTO_ARGS) --parser go -o ../sdks/go/src/parser.go
@@ -151,7 +150,7 @@ $(PY_PRINTER): $(PROTO_FILES) $(GRAMMAR) $(PY_PRINTER_TEMPLATE)
 
 printer-julia: $(JL_PRINTER)
 $(JL_PRINTER): $(PROTO_FILES) $(GRAMMAR) $(JL_PRINTER_TEMPLATE)
-	$(META_CLI) $(META_PROTO_ARGS) --printer julia -o ../sdks/julia/LogicalQueryProtocol/src/pretty.jl
+	$(META_CLI) $(META_PROTO_ARGS) --printer julia -o ../sdks/julia/LogicalQueryProtocol.jl/src/pretty.jl
 
 printer-go: $(GO_PRINTER)
 $(GO_PRINTER): $(PROTO_FILES) $(GRAMMAR) $(GO_PRINTER_TEMPLATE)
@@ -163,7 +162,7 @@ force-printer-python:
 	$(META_CLI) $(META_PROTO_ARGS) --printer python -o ../sdks/python/src/lqp/gen/pretty.py
 
 force-printer-julia:
-	$(META_CLI) $(META_PROTO_ARGS) --printer julia -o ../sdks/julia/LogicalQueryProtocol/src/pretty.jl
+	$(META_CLI) $(META_PROTO_ARGS) --printer julia -o ../sdks/julia/LogicalQueryProtocol.jl/src/pretty.jl
 
 force-printer-go:
 	$(META_CLI) $(META_PROTO_ARGS) --printer go -o ../sdks/go/src/pretty.go
@@ -178,16 +177,8 @@ test-python: $(PY_PARSER) $(PY_PROTO_GENERATED) check-python
 update-snapshots: $(PY_PARSER) $(PY_PROTO_GENERATED)
 	cd sdks/python && uv run python -m pytest --snapshot-update
 
-JL_TEST_DIR := sdks/julia/LogicalQueryProtocol/test
-
-sync-julia-test-data:
-	@mkdir -p $(JL_TEST_DIR)/lqp $(JL_TEST_DIR)/pretty $(JL_TEST_DIR)/bin
-	cp tests/lqp/*.lqp $(JL_TEST_DIR)/lqp/
-	cp tests/pretty/*.lqp $(JL_TEST_DIR)/pretty/
-	cp tests/bin/*.bin $(JL_TEST_DIR)/bin/
-
-test-julia: $(JL_PARSER) $(JL_PROTO_GENERATED) sync-julia-test-data
-	cd sdks/julia && julia --project=LogicalQueryProtocol -e 'using Pkg; Pkg.test()'
+test-julia: $(JL_PARSER) $(JL_PROTO_GENERATED)
+	cd sdks/julia && julia --project=LogicalQueryProtocol.jl -e 'using Pkg; Pkg.test()'
 
 test-go: $(GO_PARSER) $(GO_PROTO_GENERATED)
 	cd sdks/go && go test ./test/...

README.md

@@ -58,11 +58,11 @@ grammar in `meta/src/meta/grammar.y`. From the grammar we can automatically deri
 and pretty printers for the S-expression representation of LQP, which is used for testing
 and debugging.
 
-When you have updated the grammar, you can regenerate the parsers and verify that they match
-by running
+When you have updated the grammar, you can regenerate the SDKs (protobuf bindings, parser,
+and pretty printer for each language) and run the tests to verify them.
 
 ```
-make parsers
+make
 make test
 ```
 
@@ -76,8 +76,14 @@ The Python SDK is [deployed to PyPI](https://pypi.org/project/lqp/). Publishing
 via GitHub Actions: creating a GitHub release triggers a workflow that builds and uploads the
 package to PyPI. To release a new version:
 
-1. Update the version in `sdks/python/pyproject.toml`
-2. Create a new GitHub release
+1. Make sure that everything is up-to-date by running `make`.
+2. File a PR that updates the version in `sdks/python/pyproject.toml` and
+   `sdks/julia/LogicalQueryProtocol.jl/Project.toml`. Note that the Go SDK version is
+   automatically determined by the release tag in GitHub.
+3. Get approval and merge into `main`.
+4. Create a new GitHub release from `main` with a tag matching the version (e.g.
+   `gh release create v0.3.0 --title "v0.3.0" --generate-notes`). The release must be
+   created after merging the version bump, since the workflow checks out the default branch.
 
 To publish manually instead:
 

meta/src/meta/cli.py

@@ -296,7 +296,9 @@ def run(args) -> int:
                 + ["--printer", args.printer]
             )
             gen_fn = _PRINTER_GENERATORS[args.printer]
-            output_text = gen_fn(grammar, command_line, proto_messages)
+            output_text = gen_fn(
+                grammar, command_line, proto_messages, proto_enums=proto_parser.enums
+            )
             write_output(
                 output_text,
                 args.output,

meta/src/meta/codegen_base.py

@@ -65,7 +65,7 @@ class CodegenConfig:
 
 
 PARSER_CONFIG = CodegenConfig(
-    receiver_type="Parser", receiver_var="p", first_param="parser::Parser"
+    receiver_type="Parser", receiver_var="p", first_param="parser::ParserState"
 )
 PRINTER_CONFIG = CodegenConfig(
     receiver_type="PrettyPrinter", receiver_var="p", first_param="pp::PrettyPrinter"
@@ -1145,5 +1145,25 @@ def format_command_line_comment(self, command_line: str) -> str:
         """Format a command line comment for the generated file header."""
         pass
 
+    def gen_pprint_dispatch_line(self, type_str: str, func_ref: str) -> str | None:
+        """Generate a pprint dispatch line for the given type and function.
+
+        Returns None if the language does not support pprint dispatch generation.
+        """
+        return None
+
+    def gen_dispatch_function(
+        self,
+        entries: list[tuple[str, str]],
+        enum_entries: list[tuple[str, str]],
+    ) -> str:
+        """Generate a dispatch function for all pretty printer entries.
+
+        Returns empty string by default. Override in language-specific
+        generators that need a centralized dispatch function (Go, Python).
+        Julia uses multiple dispatch instead.
+        """
+        return ""
+
     # Parser generation indent for parse method definitions
     parse_def_indent: str = ""

meta/src/meta/codegen_go.py

@@ -947,6 +947,25 @@ def format_named_token_spec(self, token_name: str, token_pattern: str) -> str:
     def format_command_line_comment(self, command_line: str) -> str:
         return f"Command: {command_line}"
 
+    def gen_dispatch_function(
+        self, entries: list[tuple[str, str]], enum_entries: list[tuple[str, str]]
+    ) -> str:
+        """Generate the Go type-switch pprintDispatch method."""
+        lines: list[str] = []
+        lines.append("func (p *PrettyPrinter) pprintDispatch(msg interface{}) {")
+        lines.append("\tswitch m := msg.(type) {")
+        for type_str, func_ref in entries:
+            lines.append(f"\tcase {type_str}:")
+            lines.append(f"\t\t{func_ref}(m)")
+        for enum_type, func_ref in enum_entries:
+            lines.append(f"\tcase {enum_type}:")
+            lines.append(f"\t\t{func_ref}(m)")
+        lines.append("\tdefault:")
+        lines.append('\t\tpanic(fmt.Sprintf("no pretty printer for %T", msg))')
+        lines.append("\t}")
+        lines.append("}")
+        return "\n".join(lines)
+
 
 def escape_identifier(name: str) -> str:
     """Escape a Go identifier if it's a keyword."""

meta/src/meta/codegen_julia.py

@@ -210,6 +210,9 @@ def gen_parse_nonterminal_ref(self, name: str) -> str:
     def gen_pretty_nonterminal_ref(self, name: str) -> str:
         return f"pretty_{name}"
 
+    def gen_pprint_dispatch_line(self, type_str: str, func_ref: str) -> str | None:
+        return f"_pprint_dispatch(pp::PrettyPrinter, x::{type_str}) = {func_ref}(pp, x)"
+
     # --- Type generation ---
 
     def gen_message_type(self, module: str, name: str) -> str:
@@ -218,9 +221,10 @@ def gen_message_type(self, module: str, name: str) -> str:
         return f"Proto.{name}"
 
     def gen_enum_type(self, module: str, name: str) -> str:
+        # EnumX enums: the dispatch type is Proto.EnumName.T
         if name in self.keywords:
-            return f'Proto.var"#{name}"'
-        return f"Proto.{name}"
+            return f'Proto.var"#{name}".T'
+        return f"Proto.{name}.T"
 
     def gen_enum_value(self, module: str, enum_name: str, value_name: str) -> str:
         if enum_name in self.keywords:

meta/src/meta/codegen_python.py

@@ -155,8 +155,9 @@ def gen_pretty_nonterminal_ref(self, name: str) -> str:
     def gen_message_type(self, module: str, name: str) -> str:
         return f"{module}_pb2.{name}"
 
-    def gen_enum_type(self, module: str, name: str) -> str:
-        return f"{module}_pb2.{name}"
+    def gen_enum_type(self, module: str, name: str) -> str:  # noqa: ARG002
+        # Proto3 enums are int-valued in Python; use int for type annotations.
+        return "int"
 
     def gen_enum_value(self, module: str, enum_name: str, value_name: str) -> str:
         return f"{module}_pb2.{enum_name}.{value_name}"
@@ -392,6 +393,38 @@ def format_named_token_spec(self, token_name: str, token_pattern: str) -> str:
     def format_command_line_comment(self, command_line: str) -> str:
         return f"\nCommand: {command_line}\n"
 
+    def gen_dispatch_function(
+        self, entries: list[tuple[str, str]], enum_entries: list[tuple[str, str]]
+    ) -> str:
+        """Generate the Python isinstance-chain pprint_dispatch method."""
+        indent = self.parse_def_indent
+        bi = indent + self.indent_str
+        lines: list[str] = []
+        lines.append(f"{indent}def pprint_dispatch(self, msg):")
+        first = True
+        for type_str, func_ref in entries:
+            # Skip parameterized types (Sequence[T], tuple[...]) — these are
+            # list-typed grammar nonterminals, not individual proto messages.
+            if "[" in type_str:
+                continue
+            cond = "if" if first else "elif"
+            first = False
+            lines.append(f"{bi}{cond} isinstance(msg, {type_str}):")
+            lines.append(f"{bi}    {func_ref}(msg)")
+        for enum_type, func_ref in enum_entries:
+            # Enums in Python protobuf are ints, so we need type() check
+            cond = "elif" if not first else "if"
+            first = False
+            lines.append(f"{bi}# enum: {enum_type}")
+            lines.append(f"{bi}{cond} isinstance(msg, int):")
+            lines.append(f"{bi}    {func_ref}(msg)")
+        if not first:
+            lines.append(f"{bi}else:")
+            lines.append(
+                f'{bi}    raise ParseError(f"no pretty printer for {{type(msg)}}")'
+            )
+        return "\n".join(lines)
+
 
 def escape_identifier(name: str) -> str:
     """Escape a Python identifier if it's a keyword."""

meta/src/meta/codegen_templates.py

@@ -103,6 +103,9 @@ class BuiltinTemplate:
     "start_pretty_fragment": BuiltinTemplate(
         "{0}", ["self.start_pretty_fragment({0})"]
     ),
+    "format_bytes": BuiltinTemplate('"0x" + {0}.hex()'),
+    "pp_dispatch": BuiltinTemplate("None", ["self.pprint_dispatch({0})"]),
+    "get_at": BuiltinTemplate("{0}[{1}]"),
 }
 
 
@@ -166,12 +169,12 @@ class BuiltinTemplate:
     "indent_sexp_io": BuiltinTemplate("nothing", ["indent_sexp!(pp)"]),
     "dedent_io": BuiltinTemplate("nothing", ["dedent!(pp)"]),
     "try_flat_io": BuiltinTemplate("try_flat(pp, {0}, {1})"),
-    "format_int64": BuiltinTemplate("string({0})"),
-    "format_int32": BuiltinTemplate("string({0})"),
-    "format_float64": BuiltinTemplate("format_float64({0})"),
-    "format_string": BuiltinTemplate("format_string_value({0})"),
+    "format_int64": BuiltinTemplate("format_int(pp, {0})"),
+    "format_int32": BuiltinTemplate("format_int(pp, Int64({0}))"),
+    "format_float64": BuiltinTemplate("format_float(pp, {0})"),
+    "format_string": BuiltinTemplate("format_string(pp, {0})"),
     "format_symbol": BuiltinTemplate("{0}"),
-    "format_bool": BuiltinTemplate('({0} ? "true" : "false")'),
+    "format_bool": BuiltinTemplate("format_bool(pp, {0})"),
     "format_decimal": BuiltinTemplate("format_decimal(pp, {0})"),
     "format_int128": BuiltinTemplate("format_int128(pp, {0})"),
     "format_uint128": BuiltinTemplate("format_uint128(pp, {0})"),
@@ -185,6 +188,9 @@ class BuiltinTemplate:
     "relation_id_to_string": BuiltinTemplate("relation_id_to_string(pp, {0})"),
     "relation_id_to_uint128": BuiltinTemplate("relation_id_to_uint128(pp, {0})"),
     "start_pretty_fragment": BuiltinTemplate("{0}", ["start_pretty_fragment(pp, {0})"]),
+    "format_bytes": BuiltinTemplate('"0x" * bytes2hex({0})'),
+    "pp_dispatch": BuiltinTemplate("nothing", ["_pprint_dispatch(pp, {0})"]),
+    "get_at": BuiltinTemplate("{0}[{1} + 1]"),
 }
 
 
@@ -271,6 +277,9 @@ class BuiltinTemplate:
     "relation_id_to_string": BuiltinTemplate("p.relationIdToString({0})"),
     "relation_id_to_uint128": BuiltinTemplate("p.relationIdToUint128({0})"),
     "start_pretty_fragment": BuiltinTemplate("{0}", ["p.startPrettyFragment({0})"]),
+    "format_bytes": BuiltinTemplate('fmt.Sprintf("0x%x", {0})'),
+    "pp_dispatch": BuiltinTemplate("nil", ["p.pprintDispatch({0})"]),
+    "get_at": BuiltinTemplate("{0}[{1}]"),
 }
 
 __all__ = [