Merge pull request #6 from LeakIX/dannywillems/implement-bulk-export

Add bulk export

Author: dannywillems

example/example_client.py

@@ -99,6 +99,13 @@ def example_get_plugins():
         print(p.description)
 
 
+def example_bulk_export():
+    raw_query = '+plugin:"SmbPlugin" +country:"Belgium"'
+    query = RawQuery(raw_query)
+    response = CLIENT.bulk_export([query])
+    print(response.json())
+
+
 if __name__ == "__main__":
     example_get_host_filter_plugin()
     example_get_service_filter_plugin()
@@ -108,3 +115,4 @@ def example_get_plugins():
     example_get_leak_plugins_with_time()
     example_get_leak_raw_query()
     example_get_plugins()
+    example_bulk_export()

leakix/client.py

@@ -1,5 +1,4 @@
-from abc import ABCMeta, abstractmethod
-import os
+import json
 from typing import Optional, List
 import requests
 from l9format import l9format
@@ -52,7 +51,7 @@ def __get(self, url, params):
             headers=self.headers,
         )
         if r.status_code == 200:
-            response_json = r.json() or []
+            response_json = r.json() if r.content else []
             return SuccessResponse(response=r, response_json=response_json)
         elif r.status_code == 429:
             return RateLimitResponse(response=r)
@@ -153,3 +152,32 @@ def get_plugins(self):
         if r.is_success():
             r.response_json = [APIResult.from_dict(d) for d in r.json()]
         return r
+
+    def bulk_export(self, queries: Optional[List[Query]] = None):
+        url = "%s/bulk/search" % (self.base_url)
+        if queries is None or len(queries) == 0:
+            serialized_query = EmptyQuery().serialize()
+        else:
+            serialized_query = [q.serialize() for q in queries]
+            serialized_query = " ".join(serialized_query)
+            serialized_query = "%s" % serialized_query
+        params = {"q": serialized_query}
+        r = requests.get(
+            url,
+            params=params,
+            headers=self.headers,
+            stream=True
+        )
+        if r.status_code == 200:
+            response_json = []
+            for line in r.iter_lines():
+                json_event = json.loads(line)
+                response_json.append(l9format.L9Aggregation.from_dict(json_event))
+            return SuccessResponse(response=r, response_json=response_json)
+        elif r.status_code == 429:
+            return RateLimitResponse(response=r)
+        elif r.status_code == 204:
+            return ErrorResponse(response=r, response_json=[], status_code=200)
+        else:
+            return ErrorResponse(response=r, response_json=r.json())
+        return r

pyproject.toml

@@ -1,13 +1,13 @@
 [tool.poetry]
 name = "leakix"
-version = "0.1.6"
+version = "0.1.7"
 description = "Official python client for LeakIX (https://leakix.net)"
 authors = ["Danny Willems <danny@leakix.net>"]
 
 [tool.poetry.dependencies]
 python = "^3.7"
 requests = "*"
-l9format = "=1.3.1a1"
+l9format = "=1.3.1a2"
 
 [tool.poetry.dev-dependencies]
 python-decouple = "*"