[LQP Meta] Make relation_id_from_string consistent with BackIR2LQP

Makefile

@@ -13,6 +13,7 @@
 #   make force-printer-X    Force-regenerate a single printer.
 #   make test         Run tests for all languages.
 #   make test-X       Run tests for one language (X = python, julia, go).
+#   make update-bins       Regenerate binary test files from .lqp sources.
 #   make update-snapshots  Regenerate Python snapshot test outputs.
 #   make lint-python  Run ruff lint and format checks.
 #   make format-python  Auto-format Python code with ruff.
@@ -78,7 +79,7 @@ JL_PROTO_GENERATED := \
 	force-parsers force-parser-python force-parser-julia force-parser-go \
 	printers printer-python printer-julia printer-go \
 	force-printers force-printer-python force-printer-julia force-printer-go \
-	test test-python update-snapshots test-julia test-go \
+	test test-python update-bins update-snapshots test-julia test-go \
 	test-meta check-python check-meta lint-meta format-meta \
 	lint-python format-python clean
 
@@ -174,6 +175,12 @@ test: test-meta test-python test-julia test-go
 test-python: $(PY_PARSER) $(PY_PROTO_GENERATED) check-python
 	cd sdks/python && uv run python -m pytest
 
+update-bins: $(PY_PARSER) $(PY_PROTO_GENERATED)
+	@for lqp in tests/lqp/*.lqp; do \
+		name=$$(basename "$$lqp" .lqp); \
+		cd sdks/python && uv run lqp "../../$$lqp" --bin --out > "../../tests/bin/$${name}.bin" && cd ../..; \
+	done
+
 update-snapshots: $(PY_PARSER) $(PY_PROTO_GENERATED)
 	cd sdks/python && uv run python -m pytest --snapshot-update
 

meta/src/meta/templates/parser.go.template

@@ -10,6 +10,7 @@ package lqp
 
 import (
 	"crypto/sha256"
+	"encoding/binary"
 	"fmt"
 	"math"
 	"math/big"
@@ -362,16 +363,10 @@ func (p *Parser) startFragment(fragmentID *pb.FragmentId) *pb.FragmentId {{
 }}
 
 func (p *Parser) relationIdFromString(name string) *pb.RelationId {{
-	// Create RelationId from string hash (matching Python implementation)
-	// Python uses: int(hashlib.sha256(name.encode()).hexdigest()[:16], 16)
-	// This takes only first 8 bytes (16 hex chars) as id_low, id_high is always 0
-	// Python interprets the hex as big-endian, so we read bytes in big-endian order
 	hash := sha256.Sum256([]byte(name))
-	var low uint64
-	for i := 0; i < 8; i++ {{
-		low = (low << 8) | uint64(hash[i])
-	}}
-	high := uint64(0)
+	// Use big-endian and the lower 128 bits of the hash, consistent with pyrel.
+	high := binary.BigEndian.Uint64(hash[16:24])
+	low := binary.BigEndian.Uint64(hash[24:32])
 	relationId := &pb.RelationId{{IdLow: low, IdHigh: high}}
 
 	// Store the mapping for the current fragment if we're inside one

meta/src/meta/templates/parser.jl.template

@@ -228,8 +228,9 @@ end
 function relation_id_from_string(parser::ParserState, name::String)
     # Create RelationId from string and track mapping for debug info
     hash_bytes = sha256(name)
-    id_low = Base.parse(UInt64, bytes2hex(hash_bytes[1:8]), base=16)
-    id_high = UInt64(0)
+    # Use big-endian and the lower 128 bits of the hash, consistent with pyrel.
+    id_high = ntoh(reinterpret(UInt64, hash_bytes[17:24])[1])
+    id_low = ntoh(reinterpret(UInt64, hash_bytes[25:32])[1])
     relation_id = Proto.RelationId(id_low, id_high)
 
     # Store the mapping for the current fragment if we're inside one

meta/src/meta/templates/parser.py.template

@@ -213,8 +213,10 @@ class Parser:
 
     def relation_id_from_string(self, name: str) -> Any:
         """Create RelationId from string and track mapping for debug info."""
-        id_low = int(hashlib.sha256(name.encode()).hexdigest()[:16], 16)
-        id_high = 0
+        hash_bytes = hashlib.sha256(name.encode()).digest()
+        # Use big-endian and the lower 128 bits of the hash, consistent with pyrel.
+        id_high = int.from_bytes(hash_bytes[16:24], byteorder='big')
+        id_low = int.from_bytes(hash_bytes[24:32], byteorder='big')
         relation_id = logic_pb2.RelationId(id_low=id_low, id_high=id_high)
 
         # Store the mapping for the current fragment if we're inside one

sdks/go/src/hash_test.go

@@ -0,0 +1,16 @@
+package lqp
+
+import "testing"
+
+// TestRelationIdFromString verifies that relation_id_from_string produces
+// the same id across all SDKs (Julia, Python, Go).
+func TestRelationIdFromString(t *testing.T) {
+	p := NewParser([]Token{})
+	rid := p.relationIdFromString("my_relation")
+	if rid.IdLow != 0xf2fc83ec57cf8fbc {
+		t.Errorf("id_low: got 0x%016x, want 0xf2fc83ec57cf8fbc", rid.IdLow)
+	}
+	if rid.IdHigh != 0x503f7dc862f367b7 {
+		t.Errorf("id_high: got 0x%016x, want 0x503f7dc862f367b7", rid.IdHigh)
+	}
+}

sdks/go/src/parser.go

@@ -10,6 +10,7 @@ package lqp
 
 import (
 	"crypto/sha256"
+	"encoding/binary"
 	"fmt"
 	"math"
 	"math/big"
@@ -388,16 +389,10 @@ func (p *Parser) startFragment(fragmentID *pb.FragmentId) *pb.FragmentId {
 }
 
 func (p *Parser) relationIdFromString(name string) *pb.RelationId {
-	// Create RelationId from string hash (matching Python implementation)
-	// Python uses: int(hashlib.sha256(name.encode()).hexdigest()[:16], 16)
-	// This takes only first 8 bytes (16 hex chars) as id_low, id_high is always 0
-	// Python interprets the hex as big-endian, so we read bytes in big-endian order
 	hash := sha256.Sum256([]byte(name))
-	var low uint64
-	for i := 0; i < 8; i++ {
-		low = (low << 8) | uint64(hash[i])
-	}
-	high := uint64(0)
+	// Use big-endian and the lower 128 bits of the hash, consistent with pyrel.
+	high := binary.BigEndian.Uint64(hash[16:24])
+	low := binary.BigEndian.Uint64(hash[24:32])
 	relationId := &pb.RelationId{IdLow: low, IdHigh: high}
 
 	// Store the mapping for the current fragment if we're inside one

sdks/julia/LogicalQueryProtocol.jl/src/parser.jl

@@ -254,8 +254,9 @@ end
 function relation_id_from_string(parser::ParserState, name::String)
     # Create RelationId from string and track mapping for debug info
     hash_bytes = sha256(name)
-    id_low = Base.parse(UInt64, bytes2hex(hash_bytes[1:8]), base=16)
-    id_high = UInt64(0)
+    # Use big-endian and the lower 128 bits of the hash, consistent with pyrel.
+    id_high = ntoh(reinterpret(UInt64, hash_bytes[17:24])[1])
+    id_low = ntoh(reinterpret(UInt64, hash_bytes[25:32])[1])
     relation_id = Proto.RelationId(id_low, id_high)
 
     # Store the mapping for the current fragment if we're inside one

sdks/julia/LogicalQueryProtocol.jl/test/parser_tests.jl

@@ -112,6 +112,16 @@ end
     @test r.scale == 1
 end
 
+@testitem "Parser - relation_id_from_string" setup=[ParserSetup] begin
+    using LogicalQueryProtocol.Parser: ParserState, Token, relation_id_from_string
+
+    # All SDKs must produce the same id for the same string.
+    parser = ParserState(Token[])
+    rid = relation_id_from_string(parser, "my_relation")
+    @test rid.id_low == 0xf2fc83ec57cf8fbc
+    @test rid.id_high == 0x503f7dc862f367b7
+end
+
 @testitem "parse_transaction entry point" setup=[ParserSetup] begin
     input = "(transaction (epoch (writes) (reads)))"
     result = Parser.parse_transaction(input)

sdks/python/src/lqp/gen/parser.py

@@ -241,8 +241,10 @@ def start_fragment(
 
     def relation_id_from_string(self, name: str) -> Any:
         """Create RelationId from string and track mapping for debug info."""
-        id_low = int(hashlib.sha256(name.encode()).hexdigest()[:16], 16)
-        id_high = 0
+        hash_bytes = hashlib.sha256(name.encode()).digest()
+        # Use big-endian and the lower 128 bits of the hash, consistent with pyrel.
+        id_high = int.from_bytes(hash_bytes[16:24], byteorder='big')
+        id_low = int.from_bytes(hash_bytes[24:32], byteorder='big')
         relation_id = logic_pb2.RelationId(id_low=id_low, id_high=id_high)
 
         # Store the mapping for the current fragment if we're inside one

sdks/python/tests/test_parser.py

@@ -3,12 +3,20 @@
 import pytest
 from pytest_snapshot.plugin import Snapshot
 
-from lqp.gen.parser import ParseError, parse, parse_fragment, parse_transaction
+from lqp.gen.parser import ParseError, Parser, parse, parse_fragment, parse_transaction
 from lqp.proto.v1 import fragments_pb2, transactions_pb2
 
 from .utils import BIN_SNAPSHOTS_DIR, get_lqp_input_files
 
 
+def test_relation_id_from_string():
+    """All SDKs must produce the same id for the same string."""
+    parser = Parser([])
+    rid = parser.relation_id_from_string("my_relation")
+    assert rid.id_low == 0xF2FC83EC57CF8FBC
+    assert rid.id_high == 0x503F7DC862F367B7
+
+
 @pytest.mark.parametrize("input_file", get_lqp_input_files())
 def test_parse_lqp(snapshot: Snapshot, input_file):
     """Test that each input file can be parsed and matches its binary snapshot."""

tests/bin/arithmetic.bin

@@ -1,12 +1,12 @@
 
-é
-·
-´
-±
+Ÿ
+Û
+Ø
+Õ
 
-f1¹
-¶
-		V×ijN&⼨
+f1Â
+¿
+	èž5z$¥=ö‘§©y.Û¨
 
 
 plusR
@@ -65,9 +65,9 @@ AJ?
 
 
b	
 
-div»
-¸
-		VŃñÚª
+divÄ
+Á
+	ìQÒ°¬uÝqéd°	tÖ]ª
 
 
 plusR&
@@ -132,9 +132,9 @@ AJ?
 
 
b	
 
-div/
-		V×ijN&â¼
-		VŃñÚ
-decimal_64decimal_128
-dec_64		V×ijN&â¼
-dec_128		VŃñÚ

+divA
+	èž5z$¥=ö‘§©y.Û
+	ìQÒ°¬uÝqéd°	tÖ]
+decimal_64decimal_128
+dec_64	èž5z$¥=ö‘§©y.Û
+dec_128	ìQÒ°¬uÝqéd°	tÖ]

tests/bin/configure_levels.bin

@@ -1,12 +1,12 @@
 
-g
-R
-P
-N
+‚

+d
+b
+`
 
-f15
-3
-		kܕÆéó‚&$J"
-rel_primitive_eq


-		kܕÆéó‚r1
-r1		kܕÆéó‚


+f1>
+<
+	(xåy_H³>|Ôèm(&$J"
+rel_primitive_eq


+	(xåy_H³>|Ôèm(r1
+r1	(xåy_H³>|Ôèm(


tests/bin/configure_levels_all.bin

@@ -1,12 +1,12 @@
 
-g
-R
-P
-N
+‚

+d
+b
+`
 
-f25
-3
-		!r•¯
ýwÛ&$J"
-rel_primitive_eq
-		!r•¯
ýwÛr2
-r2		!r•¯
ýwÛ
+f2>
+<
+	äýW®þ”Ÿ[@h`Å£&$J"
+rel_primitive_eq
+	äýW®þ”Ÿ[@h`Å£r2
+r2	äýW®þ”Ÿ[@h`Å£

tests/bin/simple_ic.bin

@@ -1,21 +1,24 @@
 
-ñ

-º

-·

-´

+Ë
+‚
+ÿ

+ü

 
-f15
-3
-		Æÿhk´&,&$J"
-rel_primitive_eq


-
-		îŠ,ŠÛS::
-		Æÿhk´&,
-
-		_à…µ‹îà:
-		Æÿhk´&,5
-		Æÿhk´&,
-		îŠ,ŠÛS:
-		_à…µ‹îàfooabortoutput
-result		_à…µ‹îà"
-abort_1_eq_1		îŠ,ŠÛS:

+f1>
+<
+	®çfbˆ^Šù ¿ƒdp-B&$J"
+rel_primitive_eq

0
+.
+	/¼×Îcå&ÔF
+gîmDÔ:
+	®çfbˆ^Šù ¿ƒdp-B0
+.
+	:ŒO%eê’(½õUÐß?:
+	®çfbˆ^Šù ¿ƒdp-BP
+	®çfbˆ^Šù ¿ƒdp-B
+	/¼×Îcå&ÔF
+gîmDÔ
+	:ŒO%eê’(½õUÐß?fooabortoutput
+result	:ŒO%eê’(½õUÐß?$""
+abort_1_eq_1	/¼×Îcå&ÔF
+gîmDÔ

tests/pretty/context_write.lqp

@@ -3,7 +3,7 @@
   (epoch
     (writes
       (define (fragment :f1 (def :data ([x::INT] (= x 1)))))
-      (context 0xd0d332779f54ce74 0x3ade11b0fceb3e01))
+      (context 0xef2d3cf12a6745c5485c7aeaf26d7e24 0xefbc92f5bf93034978c02ace451fc15f))
     (reads (output :data :data)))
   (epoch (writes (define (fragment :f2 (def :data2 ([x::INT] (= x 2)))))))
   (epoch (reads (output :data :data))))

tests/pretty/undefined_relation.lqp

@@ -2,7 +2,9 @@
   (configure { :ivm.maintenance_level "off" :semantics_version 0})
   (epoch
     (writes (define (fragment :f1 (def :output ([v::INT] (atom :hello v))))))
-    (reads (output :epoch0 :output) (output :nonexistent 0x7945bc2d6e4fd0a0)))
+    (reads
+      (output :epoch0 :output)
+      (output :nonexistent 0x83a80b6af0acbcdf06866f5c473b9367)))
   (epoch
     (writes (define (fragment :f2 (def :hello ([v::INT] (= v 101))))))
     (reads (output :epoch1 :output))))

tests/pretty_debug/arithmetic.lqp

@@ -6,7 +6,7 @@
         (fragment
           :f1
           (def
-            0xbce2264eb3c4d756
+            0xdb082e79a9a791f63da5247a16359ee8
             ([plus::(DECIMAL 18 6)
               minus::(DECIMAL 18 6)
               mult::(DECIMAL 18 6)
@@ -21,7 +21,7 @@
                  (* a b mult)
                  (/ a b div)))))
           (def
-            0x4da0ef183c55619
+            0x5dd6740906b064e971dd75acb0d251ec
             ([plus::(DECIMAL 38 10)
               minus::(DECIMAL 38 10)
               mult::(DECIMAL 38 10)
@@ -35,10 +35,12 @@
                  (- a b minus)
                  (* a b mult)
                  (/ a b div))))))))
-    (reads (output :dec_64 0xbce2264eb3c4d756) (output :dec_128 0x4da0ef183c55619))))
+    (reads
+      (output :dec_64 0xdb082e79a9a791f63da5247a16359ee8)
+      (output :dec_128 0x5dd6740906b064e971dd75acb0d251ec))))
 
 ;; Debug information
 ;; -----------------------
 ;; Original names
-;; 	 ID `0x4da0ef183c55619` -> `decimal_128`
-;; 	 ID `0xbce2264eb3c4d756` -> `decimal_64`
+;; 	 ID `0x5dd6740906b064e971dd75acb0d251ec` -> `decimal_128`
+;; 	 ID `0xdb082e79a9a791f63da5247a16359ee8` -> `decimal_64`