Merge remote-tracking branch 'origin/main' into nn-meta-provenance

Author: nystrom

Makefile

@@ -14,6 +14,7 @@
 #   make force-printer-X    Force-regenerate a single printer.
 #   make test         Run tests for all languages.
 #   make test-X       Run tests for one language (X = python, julia, go).
+#   make update-bins       Regenerate binary test files from .lqp sources.
 #   make update-snapshots  Regenerate Python snapshot test outputs.
 #   make lint-python  Run ruff lint and format checks.
 #   make format-python  Auto-format Python code with ruff.
@@ -79,7 +80,7 @@ JL_PROTO_GENERATED := \
 	force-parsers force-parser-python force-parser-julia force-parser-go \
 	printers printer-python printer-julia printer-go \
 	force-printers force-printer-python force-printer-julia force-printer-go \
-	test test-python update-snapshots test-julia test-go \
+	test test-python update-bins update-snapshots test-julia test-go \
 	test-meta check-python check-meta lint-meta format-meta \
 	lint-python format-python clean
 
@@ -192,6 +193,12 @@ test: test-meta test-python test-julia test-go
 test-python: $(PY_PARSER) $(PY_PROTO_GENERATED) check-python
 	cd sdks/python && uv run python -m pytest
 
+update-bins: $(PY_PARSER) $(PY_PROTO_GENERATED)
+	@for lqp in tests/lqp/*.lqp; do \
+		name=$$(basename "$$lqp" .lqp); \
+		cd sdks/python && uv run lqp "../../$$lqp" --bin --out > "../../tests/bin/$${name}.bin" && cd ../..; \
+	done
+
 update-snapshots: $(PY_PARSER) $(PY_PROTO_GENERATED)
 	cd sdks/python && uv run python -m pytest --snapshot-update
 

meta/src/meta/templates/parser.go.template

@@ -10,6 +10,7 @@ package lqp
 
 import (
 	"crypto/sha256"
+	"encoding/binary"
 	"fmt"
 	"math"
 	"math/big"
@@ -432,16 +433,10 @@ func (p *Parser) startFragment(fragmentID *pb.FragmentId) *pb.FragmentId {{
 }}
 
 func (p *Parser) relationIdFromString(name string) *pb.RelationId {{
-	// Create RelationId from string hash (matching Python implementation)
-	// Python uses: int(hashlib.sha256(name.encode()).hexdigest()[:16], 16)
-	// This takes only first 8 bytes (16 hex chars) as id_low, id_high is always 0
-	// Python interprets the hex as big-endian, so we read bytes in big-endian order
 	hash := sha256.Sum256([]byte(name))
-	var low uint64
-	for i := 0; i < 8; i++ {{
-		low = (low << 8) | uint64(hash[i])
-	}}
-	high := uint64(0)
+	// Use big-endian and the lower 128 bits of the hash, consistent with pyrel.
+	high := binary.BigEndian.Uint64(hash[16:24])
+	low := binary.BigEndian.Uint64(hash[24:32])
 	relationId := &pb.RelationId{{IdLow: low, IdHigh: high}}
 
 	// Store the mapping for the current fragment if we're inside one

meta/src/meta/templates/parser.jl.template

@@ -278,8 +278,9 @@ end
 function relation_id_from_string(parser::ParserState, name::String)
     # Create RelationId from string and track mapping for debug info
     hash_bytes = sha256(name)
-    id_low = Base.parse(UInt64, bytes2hex(hash_bytes[1:8]), base=16)
-    id_high = UInt64(0)
+    # Use big-endian and the lower 128 bits of the hash, consistent with pyrel.
+    id_high = ntoh(reinterpret(UInt64, hash_bytes[17:24])[1])
+    id_low = ntoh(reinterpret(UInt64, hash_bytes[25:32])[1])
     relation_id = Proto.RelationId(id_low, id_high)
 
     # Store the mapping for the current fragment if we're inside one

meta/src/meta/templates/parser.py.template

@@ -298,8 +298,10 @@ class Parser:
 
     def relation_id_from_string(self, name: str) -> Any:
         """Create RelationId from string and track mapping for debug info."""
-        id_low = int(hashlib.sha256(name.encode()).hexdigest()[:16], 16)
-        id_high = 0
+        hash_bytes = hashlib.sha256(name.encode()).digest()
+        # Use big-endian and the lower 128 bits of the hash, consistent with pyrel.
+        id_high = int.from_bytes(hash_bytes[16:24], byteorder='big')
+        id_low = int.from_bytes(hash_bytes[24:32], byteorder='big')
         relation_id = logic_pb2.RelationId(id_low=id_low, id_high=id_high)
 
         # Store the mapping for the current fragment if we're inside one

sdks/go/src/hash_test.go

@@ -0,0 +1,16 @@
+package lqp
+
+import "testing"
+
+// TestRelationIdFromString verifies that relation_id_from_string produces
+// the same id across all SDKs (Julia, Python, Go).
+func TestRelationIdFromString(t *testing.T) {
+	p := NewParser([]Token{})
+	rid := p.relationIdFromString("my_relation")
+	if rid.IdLow != 0xf2fc83ec57cf8fbc {
+		t.Errorf("id_low: got 0x%016x, want 0xf2fc83ec57cf8fbc", rid.IdLow)
+	}
+	if rid.IdHigh != 0x503f7dc862f367b7 {
+		t.Errorf("id_high: got 0x%016x, want 0x503f7dc862f367b7", rid.IdHigh)
+	}
+}

sdks/go/src/parser.go

@@ -10,6 +10,7 @@ package lqp
 
 import (
 	"crypto/sha256"
+	"encoding/binary"
 	"fmt"
 	"math"
 	"math/big"
@@ -458,16 +459,10 @@ func (p *Parser) startFragment(fragmentID *pb.FragmentId) *pb.FragmentId {
 }
 
 func (p *Parser) relationIdFromString(name string) *pb.RelationId {
-	// Create RelationId from string hash (matching Python implementation)
-	// Python uses: int(hashlib.sha256(name.encode()).hexdigest()[:16], 16)
-	// This takes only first 8 bytes (16 hex chars) as id_low, id_high is always 0
-	// Python interprets the hex as big-endian, so we read bytes in big-endian order
 	hash := sha256.Sum256([]byte(name))
-	var low uint64
-	for i := 0; i < 8; i++ {
-		low = (low << 8) | uint64(hash[i])
-	}
-	high := uint64(0)
+	// Use big-endian and the lower 128 bits of the hash, consistent with pyrel.
+	high := binary.BigEndian.Uint64(hash[16:24])
+	low := binary.BigEndian.Uint64(hash[24:32])
 	relationId := &pb.RelationId{IdLow: low, IdHigh: high}
 
 	// Store the mapping for the current fragment if we're inside one

sdks/julia/LogicalQueryProtocol.jl/src/parser.jl

@@ -304,8 +304,9 @@ end
 function relation_id_from_string(parser::ParserState, name::String)
     # Create RelationId from string and track mapping for debug info
     hash_bytes = sha256(name)
-    id_low = Base.parse(UInt64, bytes2hex(hash_bytes[1:8]), base=16)
-    id_high = UInt64(0)
+    # Use big-endian and the lower 128 bits of the hash, consistent with pyrel.
+    id_high = ntoh(reinterpret(UInt64, hash_bytes[17:24])[1])
+    id_low = ntoh(reinterpret(UInt64, hash_bytes[25:32])[1])
     relation_id = Proto.RelationId(id_low, id_high)
 
     # Store the mapping for the current fragment if we're inside one

sdks/julia/LogicalQueryProtocol.jl/test/parser_tests.jl

@@ -112,6 +112,16 @@ end
     @test r.scale == 1
 end
 
+@testitem "Parser - relation_id_from_string" setup=[ParserSetup] begin
+    using LogicalQueryProtocol.Parser: ParserState, Token, relation_id_from_string
+
+    # All SDKs must produce the same id for the same string.
+    parser = ParserState(Token[])
+    rid = relation_id_from_string(parser, "my_relation")
+    @test rid.id_low == 0xf2fc83ec57cf8fbc
+    @test rid.id_high == 0x503f7dc862f367b7
+end
+
 @testitem "parse_transaction entry point" setup=[ParserSetup] begin
     input = "(transaction (epoch (writes) (reads)))"
     result = Parser.parse_transaction(input)

sdks/python/src/lqp/gen/parser.py

@@ -326,8 +326,10 @@ def start_fragment(
 
     def relation_id_from_string(self, name: str) -> Any:
         """Create RelationId from string and track mapping for debug info."""
-        id_low = int(hashlib.sha256(name.encode()).hexdigest()[:16], 16)
-        id_high = 0
+        hash_bytes = hashlib.sha256(name.encode()).digest()
+        # Use big-endian and the lower 128 bits of the hash, consistent with pyrel.
+        id_high = int.from_bytes(hash_bytes[16:24], byteorder='big')
+        id_low = int.from_bytes(hash_bytes[24:32], byteorder='big')
         relation_id = logic_pb2.RelationId(id_low=id_low, id_high=id_high)
 
         # Store the mapping for the current fragment if we're inside one

sdks/python/tests/test_parser.py

@@ -3,12 +3,20 @@
 import pytest
 from pytest_snapshot.plugin import Snapshot
 
-from lqp.gen.parser import ParseError, parse, parse_fragment, parse_transaction
+from lqp.gen.parser import ParseError, Parser, parse, parse_fragment, parse_transaction
 from lqp.proto.v1 import fragments_pb2, transactions_pb2
 
 from .utils import BIN_SNAPSHOTS_DIR, get_lqp_input_files
 
 
+def test_relation_id_from_string():
+    """All SDKs must produce the same id for the same string."""
+    parser = Parser([])
+    rid = parser.relation_id_from_string("my_relation")
+    assert rid.id_low == 0xF2FC83EC57CF8FBC
+    assert rid.id_high == 0x503F7DC862F367B7
+
+
 @pytest.mark.parametrize("input_file", get_lqp_input_files())
 def test_parse_lqp(snapshot: Snapshot, input_file):
     """Test that each input file can be parsed and matches its binary snapshot."""