Merge pull request #268 from KitHat/master

 ST-600 & ST-601 -- remove utxo from state and limit the number of utxo's returned

Author: skhoroshavin

sovtoken/sovtoken/constants.py

@@ -24,3 +24,6 @@
                           TokenTransactions.XFER_PUBLIC.value}
 ACCEPTABLE_QUERY_TYPES = {TokenTransactions.GET_UTXO.value, }
 ACCEPTABLE_ACTION_TYPES = {}
+
+UTXO_LIMIT = 1000
+NEXT_SEQNO = "next"

sovtoken/sovtoken/request_handlers/read_req_handler/get_utxo_handler.py

@@ -1,5 +1,11 @@
+try:
+    import ujson as json
+except ImportError:
+    import json
+from copy import deepcopy
+
 from sovtoken import TokenTransactions
-from sovtoken.constants import ADDRESS, OUTPUTS, TOKEN_LEDGER_ID
+from sovtoken.constants import ADDRESS, OUTPUTS, TOKEN_LEDGER_ID, NEXT_SEQNO, UTXO_LIMIT
 from sovtoken.messages.txn_validator import txt_get_utxo_validate
 from sovtoken.request_handlers.token_utils import parse_state_key
 from sovtoken.types import Output
@@ -13,6 +19,7 @@
 from plenum.server.database_manager import DatabaseManager
 from plenum.server.request_handlers.handler_interfaces.read_request_handler import ReadRequestHandler
 from state.trie.pruning_trie import rlp_decode
+from stp_core.config import MSG_LEN_LIMIT
 
 
 class GetUtxoHandler(ReadRequestHandler):
@@ -56,10 +63,21 @@ def get_result(self, request: Request):
                 continue
             outputs.add(Output(addr, int(seq_no), int(amount)))
 
+        utxos = outputs.sorted_list
+        next_seqno = None
+        if len(utxos) > UTXO_LIMIT:
+            next_seqno = utxos[UTXO_LIMIT].seqNo
+            utxos = utxos[:UTXO_LIMIT]
+
         result = {f.IDENTIFIER.nm: request.identifier,
-                  f.REQ_ID.nm: request.reqId, OUTPUTS: outputs.sorted_list}
-        if proof:
-            result[STATE_PROOF] = proof
+                  f.REQ_ID.nm: request.reqId, OUTPUTS: utxos}
 
         result.update(request.operation)
+        if next_seqno:
+            result[NEXT_SEQNO] = next_seqno
+        if proof:
+            res_sub = deepcopy(result)
+            res_sub[STATE_PROOF] = proof
+            if len(json.dumps(res_sub)) <= MSG_LEN_LIMIT:
+                result = res_sub
         return result

sovtoken/sovtoken/request_handlers/token_utils.py

@@ -13,7 +13,7 @@
 
 def spend_input(state, utxo_cache: UTXOCache, address, seq_no, is_committed=False):
     state_key = create_state_key(address, seq_no)
-    state.set(state_key, b'')
+    state.remove(state_key)
     utxo_cache.spend_output(Output(address, seq_no, None),
                             is_committed=is_committed)
 

sovtoken/sovtoken/test/helper.py

@@ -13,6 +13,10 @@
 from sovtoken.test.wallet import TokenWallet
 
 
+def libsovtoken_address_to_address(addr):
+    return addr[8:]
+
+
 def xfer_request(inputs, outputs, extra_data=None):
     payload = {
         TXN_TYPE: XFER_PUBLIC,

sovtoken/sovtoken/test/req_handlers/get_utxo_req_handler/conftest.py

@@ -4,6 +4,8 @@
 
 from sovtoken import TOKEN_LEDGER_ID, TokenTransactions
 from sovtoken.request_handlers.read_req_handler.get_utxo_handler import GetUtxoHandler
+from sovtoken.request_handlers.token_utils import create_state_key
+from sovtoken.test.helper import libsovtoken_address_to_address
 
 from plenum.test.helper import sdk_json_to_request_object
 
@@ -22,3 +24,10 @@ def get_utxo_request(looper, payment_address, wallet):
     get_utxo_request = json.loads(get_utxo_request)
     get_utxo_request = sdk_json_to_request_object(get_utxo_request)
     return get_utxo_request
+
+
+@pytest.fixture(scope="module")
+def insert_over_thousand_utxos(db_manager, payment_address):
+    token_state = db_manager.get_state(TOKEN_LEDGER_ID)
+    for i in range(1200):
+        token_state.set(create_state_key(libsovtoken_address_to_address(payment_address), i), str(i).encode())

sovtoken/sovtoken/test/req_handlers/get_utxo_req_handler/test_get_utxo_get_result.py

@@ -1,27 +1,28 @@
 import pytest
 from sovtoken.constants import OUTPUTS, ADDRESS
 from sovtoken.request_handlers.token_utils import create_state_key
+from sovtoken.test.helper import libsovtoken_address_to_address
 
 from plenum.common.constants import STATE_PROOF
 
 
 @pytest.fixture(scope="module", autouse=True)
 def add_utxo(payment_address, get_utxo_handler):
-    get_utxo_handler.state.set(create_state_key(payment_address.replace("pay:sov:", ""), 1), "3".encode())
+    get_utxo_handler.state.set(create_state_key(libsovtoken_address_to_address(payment_address), 1), "3".encode())
 
 
 def test_get_utxo_request_has_utxos(get_utxo_request, get_utxo_handler, payment_address, add_utxo):
     result = get_utxo_handler.get_result(get_utxo_request)
     assert result[STATE_PROOF]
     assert result[OUTPUTS]
     assert len(result[OUTPUTS]) == 1
-    assert result[OUTPUTS][0].address == payment_address.replace("pay:sov:", "")
+    assert result[OUTPUTS][0].address == libsovtoken_address_to_address(payment_address)
     assert result[OUTPUTS][0].amount == 3
     assert result[OUTPUTS][0].seqNo == 1
 
 
 def test_get_utxo_request_no_utxos(get_utxo_request, get_utxo_handler, payment_address_2, add_utxo):
-    get_utxo_request.operation[ADDRESS] = payment_address_2.replace("pay:sov:", "")
+    get_utxo_request.operation[ADDRESS] = libsovtoken_address_to_address(payment_address_2)
     result = get_utxo_handler.get_result(get_utxo_request)
     assert result[STATE_PROOF]
     assert len(result[OUTPUTS]) == 0

sovtoken/sovtoken/test/req_handlers/get_utxo_req_handler/test_get_utxo_get_result_big_number_of_utxos.py

@@ -0,0 +1,15 @@
+from sovtoken.constants import OUTPUTS
+from sovtoken.test.helper import libsovtoken_address_to_address
+
+from plenum.common.constants import STATE_PROOF
+
+
+def test_get_utxo_request_has_utxos(get_utxo_request, get_utxo_handler, payment_address, insert_over_thousand_utxos):
+    result = get_utxo_handler.get_result(get_utxo_request)
+    assert result[STATE_PROOF]
+    assert result[OUTPUTS]
+    assert len(result[OUTPUTS]) == 1000
+    for i in range(1000):
+        assert result[OUTPUTS][i].address == libsovtoken_address_to_address(payment_address)
+        assert result[OUTPUTS][i].seqNo == i
+        assert result[OUTPUTS][i].amount == i

sovtoken/sovtoken/test/req_handlers/mint_req_handler/test_mint_req_handler_update_state.py

@@ -1,11 +1,13 @@
+from sovtoken.test.helper import libsovtoken_address_to_address
+
 
 def test_mint_handler_update_state_valid_txn(mint_txn, mint_handler, payment_address):
     mint_handler.update_state(mint_txn, None, None, is_committed=True)
 
     token_state = mint_handler.state
     utxo_cache = mint_handler.database_manager.get_store("utxo_cache")
 
-    assert int(token_state.get((payment_address[8:] + ":1").encode(), isCommitted=False)) == 10
-    assert utxo_cache.get(payment_address[8:].encode()).decode() == '1:10'
+    assert int(token_state.get((libsovtoken_address_to_address(payment_address) + ":1").encode(), isCommitted=False)) == 10
+    assert utxo_cache.get(libsovtoken_address_to_address(payment_address).encode()).decode() == '1:10'
 
 

sovtoken/sovtoken/test/req_handlers/xfer_req_handler/conftest.py

@@ -6,6 +6,8 @@
 from sovtoken.sovtoken_auth_map import sovtoken_auth_map
 from base58 import b58encode_check
 from indy.payment import build_payment_req
+from sovtoken.test.helper import libsovtoken_address_to_address
+
 from indy_common.test.auth.conftest import write_auth_req_validator, constraint_serializer, config_state, idr_cache
 from plenum.common.txn_util import append_txn_metadata
 from plenum.test.helper import sdk_json_to_request_object
@@ -20,7 +22,7 @@ def xfer_handler(utxo_cache, db_manager, write_auth_req_validator, mint_tokens):
 
 @pytest.fixture(scope="module")
 def mint_tokens(payment_address, utxo_cache, db_manager):
-    addr = payment_address[8:]
+    addr = libsovtoken_address_to_address(payment_address)
     utxo_cache.set(addr, "1:10".encode())
     db_manager.get_state(TOKEN_LEDGER_ID).set((addr + ":1").encode(), "10".encode())
 

sovtoken/sovtoken/test/req_handlers/xfer_req_handler/test_xfer_handler_dynamic_validation.py

@@ -1,5 +1,6 @@
 import pytest
 from sovtoken.exceptions import InsufficientFundsError, ExtraFundsError, InvalidFundsError
+from sovtoken.test.helper import libsovtoken_address_to_address
 
 
 def test_xfer_handler_dynamic_validation_valid(xfer_handler, xfer_request):
@@ -22,5 +23,5 @@ def test_xfer_handler_dynamic_validation_utxo_not_exists(xfer_handler, invalid_a
                                                          payment_address):
     with pytest.raises(InvalidFundsError,
                        message="InvalidFundsError(\"seq_nos {{2}} are not found in list of seq_nos_amounts for "
-                               "address {} -- current list: ['1', '10']\",)".format(payment_address[8:])):
+                               "address {} -- current list: ['1', '10']\",)".format(libsovtoken_address_to_address(payment_address))):
         xfer_handler.dynamic_validation(invalid_amount_xfer_request_utxo_does_not_exist)