chg: Use best source from ipasnhistory if possible.

Author: Rafiot

bin/dbinsert.py

@@ -57,7 +57,7 @@ def _to_run_forever(self):
                 if not data:
                     self.logger.warning(f'No data for UUID {uuid}. This should not happen, but lets move on.')
                     continue
-                for_query.append({'ip': data['ip'], 'address_family': data['address_family'], 'source': 'caida',
+                for_query.append({'ip': data['ip'], 'address_family': data['address_family'],
                                   'date': data['datetime'], 'precision_delta': {'days': 3}})
             try:
                 responses = self.ipasn.mass_query(for_query)
@@ -89,11 +89,11 @@ def _to_run_forever(self):
                     # routing info is missing, need to try again later.
                     retry.append(uuid)
                     continue
-                if 'asn' in entry and entry['asn'] is None:
-                    self.logger.warning(f"Unable to find the AS number associated to {data['ip']} - {data['datetime']} (got None). This should not happen...")
+                if 'asn' in entry and entry['asn'] in [None, '0']:
+                    self.logger.warning(f"Unable to find the AS number associated to {data['ip']} - {data['datetime']} (got {entry['asn']}).")
                     continue
-                if 'prefix' in entry and entry['prefix'] is None:
-                    self.logger.warning(f"Unable to find the prefix associated to {data['ip']} - {data['datetime']} (got None). This should not happen...")
+                if 'prefix' in entry and entry['prefix'] in [None, '0.0.0.0/0', '::/0']:
+                    self.logger.warning(f"Unable to find the prefix associated to {data['ip']} - {data['datetime']} (got {entry['prefix']}).")
                     continue
 
                 # Format: <YYYY-MM-DD>|sources -> set([<source>, ...])

bin/sanitizer.py

@@ -57,7 +57,7 @@ def _sanitize_ip(self, pipeline: Redis, uuid: str, data: Dict) -> Optional[Dict]
                               'date': datetime.date().isoformat(), 'datetime': datetime.isoformat()})
         pipeline.sadd('to_insert', uuid)
 
-        return {'ip': str(ip), 'address_family': address_family, 'source': 'caida',
+        return {'ip': str(ip), 'address_family': address_family,
                 'date': datetime.isoformat(), 'precision_delta': {'days': 3}}
 
     def _sanitize_network(self, pipeline: Redis, uuid: str, data: Dict) -> List[Dict]:
@@ -90,7 +90,7 @@ def _sanitize_network(self, pipeline: Redis, uuid: str, data: Dict) -> List[Dict
                                   'date': datetime.date().isoformat(), 'datetime': datetime.isoformat()})
             pipeline.sadd('to_insert', uuid)
 
-            for_cache.append({'ip': str(ip), 'address_family': address_family, 'source': 'caida',
+            for_cache.append({'ip': str(ip), 'address_family': address_family,
                               'date': datetime.isoformat(), 'precision_delta': {'days': 3}})
         return for_cache