Merge pull request #11 from EVEprosper/plex_split

Plex split functionality

Author: lockefox

.travis.yml

@@ -9,11 +9,12 @@ install:
   - "travis_wait 15 pip install ."
 script: 
   - "python scripts/manage_api.py --testkey"
+  - "python scripts/create_splitcache.py --regions=10000002 --type=34,35 --range=30 --db=publicAPI/cache/travis_splitcache.json --source=esi"
   - "python setup.py test"
 branches:
   only:
   - master
   - travis_test
 after_success:
   - "pip install python-coveralls"
-  - "coveralls"
+  - "coveralls"

debian/changelog

@@ -1,3 +1,10 @@
+prosper-api (1.2.0.dev0) stable; urgency=low
+
+  * Adding split coverage ahead of PLEX split
+  * Converting CCP source to ESI by default
+
+ -- John Purcell <jpurcell.ee@gmail.com>  Mon, 08 May 2017 20:00:00 -0700
+
 prosper-api (1.1.4) stable; urgency=low
 
   * changing to tinymongo for API key checking (Kane Larrete)

debian/rules

@@ -4,7 +4,7 @@
 
 ifneq ($(wildcard /opt/intel/intelpython3/.*),)	#use intelpython bindings if available
 override_dh_virtualenv:
-	dh_virtualenv --python /opt/intel/intelpython3/bin/python3.5 --preinstall cython==0.24 --preinstall numpy==1.11.1 --preinstall matplotlib --extra-index-url https://repo.fury.io/lockefox/
+	dh_virtualenv --python /opt/intel/intelpython3/bin/python3.5 --preinstall cython==0.24 --preinstall numpy==1.11.1 --preinstall matplotlib --extra-index-url https://repo.fury.io/lockefox/ --use-system-packages
 else
 override_dh_virtualenv:
 	dh_virtualenv --python /usr/bin/python3.5 --preinstall cython --preinstall numpy --preinstall matplotlib --extra-index-url https://repo.fury.io/lockefox/

docs/release.md

@@ -32,7 +32,11 @@ _Testing Should Be Easy_
 
 What is code without coverage?  [pytest](http://doc.pytest.org/en/latest/) hooks have been built-in to make testing easy.  Testing is just one command away:
 
->`python setup.py test`
+> `python scripts/manage_api.py -t` (first time only)
+
+> `python scripts/create_splitcache.py --regions=10000002 --type=34,35 --range=30 --db=publicAPI/cache/travis_splitcache.json --source=esi` (first time only)
+
+> `python setup.py test`
 
 Included in the test pass:
 * unit testing

docs/split.md

@@ -0,0 +1,47 @@
+# Stock Splits
+_Because Lockefox is an idiot and made the project no-db_
+
+So, they announced [PLEX was going to be split](https://community.eveonline.com/news/dev-blogs/plex-changes-on-the-way/).  NBD, right?  Because ProsperAPI has no backing database, this is actually more difficult than it should be.
+
+# Declaring A Split
+
+```javascript
+publicAPI/split_info.json
+[
+	
+	{
+		"type_id": ,		#which item
+		"type_name": ,		#type name for debug
+		"original_id": , 	#ID before the split
+		"new_id": , 		#ID after the split
+		"split_date": ,		#%Y-%m-%d of split
+		"bool_mult_div": , 	#price multiplied by split?
+		"split_rate": ,		#new quantity
+	}
+]
+```
+
+Split info is designed to work in pairs.  Define both old and new items in paired splits for backwards-compatability.
+
+Also can be used with a `"split_rate": 1` for a `type_id` remapping.
+
+# Backloading cache data
+Because [Prophet](https://github.com/EVEprosper/ProsperAPI/blob/master/docs/crest_endpoint.md#prophet) requires 720 days of back history, we need to keep an archive in-project to keep expected quality of service.  Also, though CCP has said ESI will allow some back-compat, [EVE-Marketdata](https://eve-marketdata.com/) makes no such promises.
+
+To backload:
+> `python scripts/create_splitcache.py`
+
+See `-h` for settings for switching source, type, filename, regions, etc
+
+Don't forget to commit updates ;)
+
+# Testing
+Test suite uses a hacky sideload that uses Tritanium (34) and Pyerite (35) to validate behavior.  
+
+Also, `.travis.yml` creates a local split_cache.json file just for automated testing.  See `.travis.yml` for preparing [test environment](https://github.com/EVEprosper/ProsperAPI/blob/master/docs/release.md#2-test-your-shit).
+
+# Database Note:
+Splitcache has purposefully been made to run in a flat-file, rather than use [tinymongo](https://github.com/schapman1974/tinymongo) for remote database connection compatability.  This was done to continue the spirit of "anyone can launch ProsperAPI anywhere".
+
+PLEX split was prepared with the following:
+> `python scripts\create_splitcache.py -t 29668 -r 720 -s eve-marketdata`

publicAPI/__init__.py

@@ -5,6 +5,7 @@
 
 import publicAPI.crest_endpoint as crest_endpoint
 import publicAPI.config as config
+import publicAPI.split_utils as split_utils
 
 import prosper.common.prosper_logging as p_logging
 
@@ -34,7 +35,7 @@ def create_app(
 
     log_builder = p_logging.ProsperLogger(
         'publicAPI',
-        HERE,
+        '/var/log/prosper/',
         local_configs
     )
     if not app.debug:
@@ -51,4 +52,6 @@ def create_app(
     config.CONFIG = local_configs
     config.load_globals(local_configs)
     crest_endpoint.LOGGER = app.logger
+
+    config.SPLIT_INFO = split_utils.read_split_info(logger=crest_endpoint.LOGGER)
     return app

publicAPI/api_utils.py

@@ -31,8 +31,8 @@ def check_key(
         (bool) access allowed or not
 
     """
-    
-    # Connect to TinyMongoDB and use prosperAPI DB 
+
+    # Connect to TinyMongoDB and use prosperAPI DB
     connection = TinyMongoClient(CACHE_PATH)
     userdb = connection.prosperAPI.users
     api_value = userdb.find_one({'api_key': api_key})
@@ -49,10 +49,8 @@ def check_key(
         currenttime = datetime.now().isoformat()
         userdb.update(
             {'api_key': api_key},
-            {
-                '$set': {'last_accessed': currenttime}
-            }
-            )
+            {'$set': {'last_accessed': currenttime}}
+        )
         connection.close()
         access_allowed = True
     else:

publicAPI/cache/splitcache.json

@@ -1,5 +1,6 @@
 """config.py: a place to hold config/globals"""
 from os import path
+from enum import Enum
 
 import prosper.common.prosper_logging as p_logging
 import prosper.common.prosper_config as p_config
@@ -16,6 +17,15 @@
 MAX_RANGE = 180
 DEFAULT_HISTORY_RANGE = 700
 EXPECTED_CREST_RANGE = 400
+
+SPLIT_CACHE_FILE = path.join(HERE, 'cache', 'splitcache.json')
+
+class SwitchCCPSource(Enum):
+    """enum for switching between crest/esi"""
+    ESI = 'ESI'
+    CREST = 'CREST'
+    EMD = 'EMD'
+
 def load_globals(config=CONFIG):
     """loads global vars from config object"""
     global USER_AGENT, USER_AGENT_SHORT, DEFAULT_RANGE, MAX_RANGE
@@ -25,3 +35,5 @@ def load_globals(config=CONFIG):
 
     DEFAULT_RANGE = int(config.get('CREST', 'prophet_range'))
     MAX_RANGE = int(config.get('CREST', 'prophet_max'))
+
+SPLIT_INFO = {}