Merge pull request #140 from ethscriptions-protocol/fix_collections

Refactor Protocol Parsing and Introduce Word Domains Support

Author: RogerPodacter

app/models/erc20_fixed_denomination_parser.rb

@@ -1,83 +1,61 @@
 # Extracts fixed-denomination ERC-20 parameters from strict JSON inscriptions.
 class Erc20FixedDenominationParser
   # Constants
-  DEFAULT_PARAMS = [''.b, ''.b, ''.b, 0, 0, 0].freeze
+  DEFAULT_PARAMS = [''.b, ''.b, ''.b].freeze
   UINT256_MAX = 2**256 - 1
-  
+  PROTOCOL = 'erc-20-fixed-denomination'.b
+
   # Exact regex patterns for valid formats
   # Protocol must be "erc-20" (legacy inscription) or the canonical identifier
   # Tick must be lowercase letters/numbers, max 28 chars
   # Numbers must be positive decimals without leading zeros
   PROTOCOL_PATTERN = '(?:erc-20|erc-20-fixed-denomination)'
-  DEPLOY_REGEX = /\Adata:,\{"p":"#{PROTOCOL_PATTERN}","op":"deploy","tick":"([a-z0-9]{1,28})","max":"(0|[1-9][0-9]*)","lim":"(0|[1-9][0-9]*)"\}\z/
-  MINT_REGEX = /\Adata:,\{"p":"#{PROTOCOL_PATTERN}","op":"mint","tick":"([a-z0-9]{1,28})","id":"(0|[1-9][0-9]*)","amt":"(0|[1-9][0-9]*)"\}\z/
+  DEPLOY_REGEX = /\A\{"p":"#{PROTOCOL_PATTERN}","op":"deploy","tick":"([a-z0-9]{1,28})","max":"(0|[1-9][0-9]*)","lim":"(0|[1-9][0-9]*)"\}\z/
+  MINT_REGEX = /\A\{"p":"#{PROTOCOL_PATTERN}","op":"mint","tick":"([a-z0-9]{1,28})","id":"(0|[1-9][0-9]*)","amt":"(0|[1-9][0-9]*)"\}\z/
+
+  # Validate and encode protocol params
+  # Unified interface - accepts all possible parameters, uses what it needs
+  def self.validate_and_encode(decoded_content:, operation:, params:, source:, ethscription_id: nil, **_extras)
+    new.validate_and_encode(
+      decoded_content: decoded_content,
+      operation: operation,
+      params: params,
+      source: source
+    )
+  end
 
-  def self.extract(content_uri)
-    return DEFAULT_PARAMS unless content_uri.is_a?(String)
+  def validate_and_encode(decoded_content:, operation:, params:, source:)
+    # Only support JSON source - no header parameters for ERC-20
+    return DEFAULT_PARAMS unless source == :json
+    return DEFAULT_PARAMS unless decoded_content.is_a?(String)
 
     # Try deploy format first
-    if match = DEPLOY_REGEX.match(content_uri)
+    if match = DEPLOY_REGEX.match(decoded_content)
       tick = match[1]  # Group 1: tick
       max = match[2].to_i  # Group 2: max
       lim = match[3].to_i  # Group 3: lim
 
       # Validate uint256 bounds
       return DEFAULT_PARAMS if max > UINT256_MAX || lim > UINT256_MAX
 
-      return ['deploy'.b, 'erc-20-fixed-denomination'.b, tick.b, max, lim, 0]
+      encoded = Eth::Abi.encode(['(string,uint256,uint256)'], [[tick.b, max, lim]])
+      return [PROTOCOL, 'deploy'.b, encoded.b]
     end
 
     # Try mint format
-    if match = MINT_REGEX.match(content_uri)
+    if match = MINT_REGEX.match(decoded_content)
       tick = match[1]  # Group 1: tick
       id = match[2].to_i   # Group 2: id
       amt = match[3].to_i  # Group 3: amt
 
       # Validate uint256 bounds
       return DEFAULT_PARAMS if id > UINT256_MAX || amt > UINT256_MAX
 
-      return ['mint'.b, 'erc-20-fixed-denomination'.b, tick.b, id, 0, amt]
+      encoded = Eth::Abi.encode(['(string,uint256,uint256)'], [[tick.b, id, amt]])
+      return [PROTOCOL, 'mint'.b, encoded.b]
     end
 
     # No match - return default
     DEFAULT_PARAMS
   end
-
-  # Returns a hash representation of params that downstream services expect.
-  def self.structured_params(params)
-    op, _protocol, tick, val1, val2, val3 = params
-
-    case op
-    when 'deploy'.b
-      {
-        op: op,
-        tick: tick,
-        max: val1,
-        lim: val2,
-        amt: 0
-      }
-    when 'mint'.b
-      {
-        op: op,
-        tick: tick,
-        id: val1,
-        amt: val3
-      }
-    else
-      nil
-    end
-  end
-
-  # Encodes params into the ABI tuple required by the manager contract.
-  def self.encode_calldata(params)
-    op, _protocol, tick, val1, val2, val3 = params
-
-    if op == 'deploy'.b
-      Eth::Abi.encode(['(string,uint256,uint256)'], [[tick.b, val1, val2]])
-    elsif op == 'mint'.b
-      Eth::Abi.encode(['(string,uint256,uint256)'], [[tick.b, val1, val3]])
-    else
-      ''.b
-    end
-  end
-end
+end

app/models/erc721_ethscriptions_collection_parser.rb

@@ -28,8 +28,8 @@ class Erc721EthscriptionsCollectionParser
     # New combined create op name used by the contract; keep legacy alias below
     'create_collection_and_add_self' => {
       keys: %w[metadata item],
-      # ((CollectionParams),(ItemData)) - ItemData mirrors ItemData struct (no ethscriptionId field)
-      abi_type: '((string,string,uint256,string,string,string,string,string,string,string,bytes32),(uint256,string,string,string,(string,string)[],bytes32[]))',
+      # ((CollectionParams),(ItemData)) - ItemData now includes contentHash as first field
+      abi_type: '((string,string,uint256,string,string,string,string,string,string,string,bytes32),(bytes32,uint256,string,string,string,(string,string)[],bytes32[]))',
       validators: {
         'metadata' => :collection_metadata,
         'item' => :single_item
@@ -38,7 +38,7 @@ class Erc721EthscriptionsCollectionParser
     # Legacy alias retained for backwards compatibility
     'create_and_add_self' => {
       keys: %w[metadata item],
-      abi_type: '((string,string,uint256,string,string,string,string,string,string,string,bytes32),(uint256,string,string,string,(string,string)[],bytes32[]))',
+      abi_type: '((string,string,uint256,string,string,string,string,string,string,string,bytes32),(bytes32,uint256,string,string,string,(string,string)[],bytes32[]))',
       validators: {
         'metadata' => :collection_metadata,
         'item' => :single_item
@@ -47,7 +47,7 @@ class Erc721EthscriptionsCollectionParser
     # New single-item add op; keep legacy batch below for compatibility
     'add_self_to_collection' => {
       keys: %w[collection_id item],
-      abi_type: '(bytes32,(uint256,string,string,string,(string,string)[],bytes32[]))',
+      abi_type: '(bytes32,(bytes32,uint256,string,string,string,(string,string)[],bytes32[]))',
       validators: {
         'collection_id' => :bytes32,
         'item' => :single_item
@@ -101,7 +101,7 @@ class Erc721EthscriptionsCollectionParser
   ZERO_HEX_BYTES32 = '0x' + '0' * 64
 
   # Item keys for validation (merkle_proof always present, can be empty array)
-  ITEM_KEYS = %w[item_index name background_color description attributes merkle_proof].freeze
+  ITEM_KEYS = %w[content_hash item_index name background_color description attributes merkle_proof].freeze
 
   # Attribute keys for NFT metadata
   ATTRIBUTE_KEYS = %w[trait_type value].freeze
@@ -111,44 +111,79 @@ class ValidationError < StandardError; end
   DEFAULT_ITEMS_PATH = ENV['COLLECTIONS_ITEMS_PATH'] || Rails.root.join('items_by_ethscription.json')
   DEFAULT_COLLECTIONS_PATH = ENV['COLLECTIONS_META_PATH'] || Rails.root.join('collections_by_name.json')
 
-  def self.extract(content_uri, ethscription_id: nil)
-    new.extract(content_uri, ethscription_id: ethscription_id)
+  # New API: validate and encode protocol params
+  # Unified interface - accepts all possible parameters, uses what it needs
+  def self.validate_and_encode(decoded_content:, operation:, params:, source:, ethscription_id: nil, **_extras)
+    new.validate_and_encode(
+      decoded_content: decoded_content,
+      operation: operation,
+      params: params,
+      source: source,
+      ethscription_id: ethscription_id
+    )
   end
 
-  def extract(content_uri, ethscription_id: nil)
+  def validate_and_encode(decoded_content:, operation:, params:, source:, ethscription_id: nil)
+    # Check import fallback first (if ethscription_id provided)
     if ethscription_id
       normalized_id = normalize_id(ethscription_id)
-      if normalized_id && (preplanned = build_import_encoded_params(normalized_id))
+      if normalized_id && (preplanned = build_import_encoded_params(normalized_id, decoded_content))
         return preplanned
       end
     end
 
-    return DEFAULT_PARAMS unless valid_data_uri?(content_uri)
-
-    begin
-      json_str = DataUri.new(content_uri).decoded_data
+    return DEFAULT_PARAMS unless OPERATION_SCHEMAS.key?(operation)
 
-      # TODO: make sure this is safe
-      data = JSON.parse(json_str)
-      return DEFAULT_PARAMS unless data.is_a?(Hash)
-      return DEFAULT_PARAMS unless data['p'] == 'erc-721-ethscriptions-collection'
+    schema = OPERATION_SCHEMAS[operation]
 
-      operation = data['op']
-      return DEFAULT_PARAMS unless OPERATION_SCHEMAS.key?(operation)
+    begin
+      if source == :json
+        # Strict JSON validation - enforce exact key order
+        validate_json_structure(params, operation, schema)
+      end
 
-      schema = OPERATION_SCHEMAS[operation]
-      expected_keys = ['p', 'op'] + schema[:keys]
-      return DEFAULT_PARAMS unless data.keys == expected_keys
+      # Extract encoding data (skip 'p' and 'op' for JSON source)
+      encoding_data = if source == :json
+                        params.reject { |k, _| k == 'p' || k == 'op' }
+                      else
+                        params
+                      end
+
+      # Compute content hash ONLY for operations that need it
+      content_hash = nil
+      if ['create_collection_and_add_self', 'create_and_add_self', 'add_self_to_collection'].include?(operation)
+        # Calculate keccak256 of decoded content for item verification
+        hash = Eth::Util.keccak256(decoded_content).unpack1('H*')
+        content_hash = '0x' + hash
+
+        # Inject content_hash into item data
+        item_data = encoding_data['item']
+        if item_data.is_a?(Hash)
+          # Add content_hash as first key (OrderedHash maintains insertion order)
+          encoding_data['item'] = self.class::OrderedHash['content_hash', content_hash].merge(item_data)
+        end
+      end
 
-      encoding_data = data.reject { |k, _| k == 'p' || k == 'op' }
-      encoded_data = encode_operation(operation, encoding_data, schema)
+      encoded_data = encode_operation(operation, encoding_data, schema, content_hash: content_hash)
       ['erc-721-ethscriptions-collection'.b, operation.b, encoded_data.b]
     rescue JSON::ParserError, ValidationError => e
-      Rails.logger.debug "Collections extraction failed: #{e.message}" if defined?(Rails)
+      Rails.logger.debug "Collections validation failed: #{e.message}" if defined?(Rails)
       DEFAULT_PARAMS
     end
   end
 
+  def validate_json_structure(params, operation, schema)
+    # For JSON source, enforce strict key ordering
+    expected_keys = ['p', 'op'] + schema[:keys]
+    unless params.keys == expected_keys
+      raise ValidationError, "Invalid key order for #{operation}"
+    end
+  end
+
+  # Removed extract() method - use ProtocolParser.for_calldata() instead
+  # This avoids circular dependencies and keeps the architecture cleaner
+  # The import fallback logic is now handled in validate_and_encode()
+
   def normalize_id(value)
     case value
     when ByteString
@@ -163,7 +198,7 @@ def normalize_id(value)
   # -------------------- Import fallback --------------------
 
   # Returns [protocol, operation, encoded_data] or nil
-  def build_import_encoded_params(id)
+  def build_import_encoded_params(id, decoded_content)
     data = self.class.load_import_data(
       items_path: DEFAULT_ITEMS_PATH,
       collections_path: DEFAULT_COLLECTIONS_PATH
@@ -180,6 +215,10 @@ def build_import_encoded_params(id)
 
     item_index = data[:zero_index_by_id][id] || 0
 
+    # Always compute content hash from the actual decoded content
+    hash = Eth::Util.keccak256(decoded_content || ''.b).unpack1('H*')
+    content_hash = '0x' + hash
+
     if id == leader_id
       raw_metadata = data[:collections_by_name][coll_name]
       return nil unless raw_metadata
@@ -190,18 +229,18 @@ def build_import_encoded_params(id)
       schema = OPERATION_SCHEMAS[operation]
       encoding_data = {
         'metadata' => build_metadata_object(metadata),
-        'item' => build_item_object(item: item, item_index: item_index)
+        'item' => build_item_object(item: item, item_index: item_index, content_hash: content_hash)
       }
-      encoded_data = encode_operation(operation, encoding_data, schema)
+      encoded_data = encode_operation(operation, encoding_data, schema, content_hash: content_hash)
       ['erc-721-ethscriptions-collection'.b, operation.b, encoded_data.b]
     else
       operation = 'add_self_to_collection'
       schema = OPERATION_SCHEMAS[operation]
       encoding_data = {
         'collection_id' => to_bytes32_hex(leader_id),
-        'item' => build_item_object(item: item, item_index: item_index)
+        'item' => build_item_object(item: item, item_index: item_index, content_hash: content_hash)
       }
-      encoded_data = encode_operation(operation, encoding_data, schema)
+      encoded_data = encode_operation(operation, encoding_data, schema, content_hash: content_hash)
       ['erc-721-ethscriptions-collection'.b, operation.b, encoded_data.b]
     end
   end
@@ -285,14 +324,15 @@ def build_metadata_object(meta)
     result
   end
 
-  def build_item_object(item:, item_index:)
+  def build_item_object(item:, item_index:, content_hash:)
     attrs = Array(item['attributes']).map do |a|
       OrderedHash['trait_type', safe_string(a['trait_type']), 'value', safe_string(a['value'])]
     end
 
     proofs = item.key?('merkle_proof') ? Array(item['merkle_proof']) : []
 
     OrderedHash[
+      'content_hash', content_hash,
       'item_index', safe_uint_string(item_index),
       'name', safe_string(item['name']),
       'background_color', safe_string(item['background_color']),
@@ -347,7 +387,7 @@ def valid_data_uri?(uri)
     DataUri.valid?(uri)
   end
 
-  def encode_operation(operation, data, schema)
+  def encode_operation(operation, data, schema, content_hash: nil)
     # Validate and transform fields according to schema
     validated_data = validate_fields(data, schema[:validators])
 
@@ -356,9 +396,9 @@ def encode_operation(operation, data, schema)
     when 'create_collection'
       build_create_collection_values(validated_data)
     when 'create_collection_and_add_self', 'create_and_add_self'
-      build_create_and_add_self_values(validated_data)
+      build_create_and_add_self_values(validated_data, content_hash: content_hash)
     when 'add_self_to_collection'
-      build_add_self_to_collection_values(validated_data)
+      build_add_self_to_collection_values(validated_data, content_hash: content_hash)
     when 'remove_items'
       build_remove_items_values(validated_data)
     when 'edit_collection'
@@ -514,6 +554,7 @@ def validate_item(item)
     end
 
     {
+      contentHash: validate_bytes32(item['content_hash'], 'content_hash'),
       itemIndex: validate_uint256(item['item_index'], 'item_index'),
       name: validate_string(item['name'], 'name'),
       backgroundColor: validate_string(item['background_color'], 'background_color'),
@@ -568,7 +609,7 @@ def build_create_collection_values(data)
     ]
   end
 
-  def build_create_and_add_self_values(data)
+  def build_create_and_add_self_values(data, content_hash:)
     meta = data['metadata']
     item = data['item']
 
@@ -588,7 +629,17 @@ def build_create_and_add_self_values(data)
       merkle_root
     ]
 
+    # Item tuple - contentHash comes first (keccak256 of ethscription content)
+    # Always use the computed content_hash if provided, otherwise use validated item contentHash
+    content_hash_bytes = if content_hash
+      [content_hash[2..]].pack('H*')
+    elsif item[:contentHash]
+      item[:contentHash]  # Already packed bytes from validate_item
+    else
+      raise ValidationError, "Content hash missing"
+    end
     item_tuple = [
+      content_hash_bytes,  # Already packed bytes, don't call to_bytes32_hex
       item[:itemIndex],
       item[:name],
       item[:backgroundColor],
@@ -600,9 +651,20 @@ def build_create_and_add_self_values(data)
     [metadata_tuple, item_tuple]
   end
 
-  def build_add_self_to_collection_values(data)
+  def build_add_self_to_collection_values(data, content_hash:)
     item = data['item']
+
+    # Item tuple - contentHash comes first (keccak256 of ethscription content)
+    # Always use the computed content_hash if provided, otherwise use validated item contentHash
+    content_hash_bytes = if content_hash
+      [content_hash[2..]].pack('H*')
+    elsif item[:contentHash]
+      item[:contentHash]  # Already packed bytes from validate_item
+    else
+      raise ValidationError, "Content hash missing"
+    end
     item_tuple = [
+      content_hash_bytes,  # Already packed bytes, don't call to_bytes32_hex
       item[:itemIndex],
       item[:name],
       item[:backgroundColor],