66
77_INITIAL_MMAP_SIZE = 1 << 16
88_pack_integer_func = struct .Struct (b'i' ).pack
9- _pack_double_func = struct .Struct (b'd ' ).pack
9+ _pack_two_doubles_func = struct .Struct (b'dd ' ).pack
1010_unpack_integer = struct .Struct (b'i' ).unpack_from
11- _unpack_double = struct .Struct (b'd ' ).unpack_from
11+ _unpack_two_doubles = struct .Struct (b'dd ' ).unpack_from
1212
1313
1414# struct.pack_into has atomicity issues because it will temporarily write 0 into
1515# the mmap, resulting in false reads to 0 when experiencing a lot of writes.
1616# Using direct assignment solves this issue.
1717
18- def _pack_double (data , pos , value ):
19- data [pos :pos + 8 ] = _pack_double_func (value )
18+
19+ def _pack_two_doubles (data , pos , value , timestamp ):
20+ data [pos :pos + 16 ] = _pack_two_doubles_func (value , timestamp )
2021
2122
2223def _pack_integer (data , pos , value ):
2324 data [pos :pos + 4 ] = _pack_integer_func (value )
2425
2526
2627def _read_all_values (data , used = 0 ):
27- """Yield (key, value, pos). No locking is performed."""
28+ """Yield (key, value, timestamp, pos). No locking is performed."""
2829
2930 if used <= 0 :
3031 # If not valid `used` value is passed in, read it from the file.
@@ -41,9 +42,9 @@ def _read_all_values(data, used=0):
4142 encoded_key = data [pos :pos + encoded_len ]
4243 padded_len = encoded_len + (8 - (encoded_len + 4 ) % 8 )
4344 pos += padded_len
44- value = _unpack_double (data , pos )[ 0 ]
45- yield encoded_key .decode ('utf-8' ), value , pos
46- pos += 8
45+ value , timestamp = _unpack_two_doubles (data , pos )
46+ yield encoded_key .decode ('utf-8' ), value , timestamp , pos
47+ pos += 16
4748
4849
4950class MmapedDict :
@@ -53,7 +54,8 @@ class MmapedDict:
5354 Then 4 bytes of padding.
5455 There's then a number of entries, consisting of a 4 byte int which is the
5556 size of the next field, a utf-8 encoded string key, padding to a 8 byte
56- alignment, and then a 8 byte float which is the value.
57+ alignment, and then a 8 byte float which is the value and a 8 byte float
58+ which is a UNIX timestamp in seconds.
5759
5860 Not thread safe.
5961 """
@@ -76,7 +78,7 @@ def __init__(self, filename, read_mode=False):
7678 _pack_integer (self ._m , 0 , self ._used )
7779 else :
7880 if not read_mode :
79- for key , _ , pos in self ._read_all_values ():
81+ for key , _ , _ , pos in self ._read_all_values ():
8082 self ._positions [key ] = pos
8183
8284 @staticmethod
@@ -95,7 +97,7 @@ def _init_value(self, key):
9597 encoded = key .encode ('utf-8' )
9698 # Pad to be 8-byte aligned.
9799 padded = encoded + (b' ' * (8 - (len (encoded ) + 4 ) % 8 ))
98- value = struct .pack (f'i{ len (padded )} sd ' .encode (), len (encoded ), padded , 0.0 )
100+ value = struct .pack (f'i{ len (padded )} sdd ' .encode (), len (encoded ), padded , 0.0 , 0.0 )
99101 while self ._used + len (value ) > self ._capacity :
100102 self ._capacity *= 2
101103 self ._f .truncate (self ._capacity )
@@ -105,30 +107,28 @@ def _init_value(self, key):
105107 # Update how much space we've used.
106108 self ._used += len (value )
107109 _pack_integer (self ._m , 0 , self ._used )
108- self ._positions [key ] = self ._used - 8
110+ self ._positions [key ] = self ._used - 16
109111
110112 def _read_all_values (self ):
111113 """Yield (key, value, pos). No locking is performed."""
112114 return _read_all_values (data = self ._m , used = self ._used )
113115
114116 def read_all_values (self ):
115- """Yield (key, value). No locking is performed."""
116- for k , v , _ in self ._read_all_values ():
117- yield k , v
117+ """Yield (key, value, timestamp ). No locking is performed."""
118+ for k , v , ts , _ in self ._read_all_values ():
119+ yield k , v , ts
118120
119121 def read_value (self , key ):
120122 if key not in self ._positions :
121123 self ._init_value (key )
122124 pos = self ._positions [key ]
123- # We assume that reading from an 8 byte aligned value is atomic
124- return _unpack_double (self ._m , pos )[0 ]
125+ return _unpack_two_doubles (self ._m , pos )
125126
126- def write_value (self , key , value ):
127+ def write_value (self , key , value , timestamp ):
127128 if key not in self ._positions :
128129 self ._init_value (key )
129130 pos = self ._positions [key ]
130- # We assume that writing to an 8 byte aligned value is atomic
131- _pack_double (self ._m , pos , value )
131+ _pack_two_doubles (self ._m , pos , value , timestamp )
132132
133133 def close (self ):
134134 if self ._f :
0 commit comments