Merge pull request #26 from RamanjaneyuluIdavalapati/master

wordvecspace Service

Author: Ram Idavalapati

.travis.yml

@@ -1,15 +1,15 @@
 language: python
 python:
-- '2.7'
+- '3.5'
 before_install:
 - wget 'https://s3.amazonaws.com/deepcompute-public/data/wordvecspace/small_test_data.tgz'
   && tar xvzf small_test_data.tgz
 - export WORDVECSPACE_DATADIR='./small_test_data'
 install: 
-- pip install .
+- pip install .[service]
 - sudo apt install libopenblas-base
 script:
-- python setup.py test
+- python3 setup.py test
 deploy:
 - provider: releases
   skip_cleanup: true
@@ -24,8 +24,8 @@ deploy:
     - wordvecspace/command.py
     - wordvecspace/convert.py
     - wordvecspace/cuda.py
-  name: wordvecspace-0.4.2
-  tag_name: 0.4.2
+  name: wordvecspace-0.5
+  tag_name: 0.5
   on:
     repo: deep-compute/wordvecspace
   # pypitest

README.md

@@ -3,11 +3,11 @@
 A high performance pure python module that helps in loading and performing operations on word vector spaces created using Google's Word2vec tool.
 
 ## Installation
-> Prerequisites: Python2.7
+> Prerequisites: Python3.5
 
 ```bash
 $ sudo apt install libopenblas-base
-$ sudo pip install wordvecspace
+$ sudo pip3 install wordvecspace
 ```
 
 ## Usage
@@ -62,25 +62,6 @@ $ wordvecspace convert <input_dir> <output_dir>
 
 # You can also generate shards by specifying number of vectors per each shard
 $ wordvecspace convert <input_dir> <output_dir> -n 5000
-```
-### Interactive console
-```bash
-$ wordvecspace interact <input_dir>
-
-# <input_dir> is the directory which has vocab.txt and vectors.npy
-```
-Example:
-```bash
-$ wordvecspace interact /home/user/data
-
-Total number of vectors and dimensions in .npy file (71291, 5)
-
->>> help
-['DEFAULT_K', 'VECTOR_FNAME', 'VOCAB_FNAME', '__class__', '__delattr__', '__dict__', '__doc__', '__format__', '__getattribute__', '__hash__', '__init__', '__module__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', '_load_vocab', '_make_array', '_perform_dot', '_perform_sgemm', '_perform_sgemv', 'data_dir', 'does_word_exist', 'get_distance', 'get_distances', 'get_nearest_neighbors', 'get_vector_magnitudes', 'get_word_at_index', 'get_word_index', 'get_word_occurrences', 'get_word_vector', 'get_word_vectors', 'load', 'magnitudes', 'num_dimensions', 'num_vectors', 'vectors', 'word_indices', 'word_occurrences', 'words']
-
-WordVecSpace console
->>> wv = WordVecSpace
-
 ```
 ### Importing
 ```python
@@ -256,6 +237,73 @@ Int64Index([  509,   486, 14208, 20639,  8573,  3389,  5226, 20919, 10172,
            dtype='int64')
 ```
 
+### Service
+
+```bash
+# Run wordvecspace as a service (which continuously listens on some port for API requests)
+$ wordvecspace runserver <input_dir> -p <port_no>
+
+# -p is for giving port. If it is not mentioned, by default wordvecspace will run on 8900 port.
+# <port_no> is the port number of wordvecspace
+# <input_dir> is the directory which has vocab.txt and vectors.npy.
+```
+
+Example:
+
+```bash
+$ wordvecspace runserver /home/user/data -p 8000
+
+# Make API request
+$ curl "http://localhost:8000/api/v1/does_word_exist?word=india"
+{"result": true, "success": true}
+```
+
+#### Making call to all API methods
+
+```bash
+$ http://localhost:8000/api/v1/does_word_exist?word=india
+
+$ http://localhost:8000/api/v1/get_word_index?word=india
+
+$ http://localhost:8000/api/v1/get_word_at_index?index=509
+
+$ http://localhost:8000/api/v1/get_word_vector?word_or_index=509
+
+$ http://localhost:8000/api/v1/get_vector_magnitudes?words_or_indices=[88, "india"]
+
+$ http://localhost:8000/api/v1/get_word_occurrences?word_or_index=india
+
+$ http://localhost:8000/api/v1/get_word_vectors?words_or_indices=[1, 'india']
+
+$ http://localhost:8000/api/v1/get_distance?word1=ap&word2=india
+
+$ http://localhost:8000/api/v1/get_distances?row_words=india
+
+$ http://localhost:8000/api/v1/get_nearest_neighbors?word=india&k=100
+```
+
+> To see all API methods of wordvecspace please run http://localhost:8000/api/v1/apidoc
+
+### Interactive console
+```bash
+$ wordvecspace interact <input_dir>
+
+# <input_dir> is the directory which has vocab.txt and vectors.npy
+```
+
+Example:
+```bash
+$ wordvecspace interact /home/user/data
+
+Total number of vectors and dimensions in .npy file (71291, 5)
+
+>>> help
+['DEFAULT_K', 'VECTOR_FNAME', 'VOCAB_FNAME', '__class__', '__delattr__', '__dict__', '__doc__', '__format__', '__getattribute__', '__hash__', '__init__', '__module__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', '_load_vocab', '_make_array', '_perform_dot', '_perform_sgemm', '_perform_sgemv', 'data_dir', 'does_word_exist', 'get_distance', 'get_distances', 'get_nearest_neighbors', 'get_vector_magnitudes', 'get_word_at_index', 'get_word_index', 'get_word_occurrences', 'get_word_vector', 'get_word_vectors', 'load', 'magnitudes', 'num_dimensions', 'num_vectors', 'vectors', 'word_indices', 'word_occurrences', 'words']
+
+WordVecSpace console
+>>> wv = WordVecSpace
+
+```
 ## Running tests
 
 ```bash

setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages
 
-version = '0.4.2'
+version = '0.5'
 setup(
     name="wordvecspace",
     version=version,
@@ -19,21 +19,22 @@
         'numpy==1.13.1',
         'pandas==0.20.3',
         'numba==0.36.2',
-        'basescript'
+        'basescript==0.2.0',
     ],
     extras_require={
         'cuda': ['pycuda==2017.1.1', 'scikit-cuda==0.5.1'],
+        'service': ['kwikapi[tornado]==0.2']
     },
     package_dir={'wordvecspace': 'wordvecspace'},
     packages=find_packages('.'),
     include_package_data=True,
     classifiers=[
-        "Programming Language :: Python :: 2.7",
+        "Programming Language :: Python :: 3.5",
         "Environment :: Console",
         "Intended Audience :: Developers",
         "License :: OSI Approved :: MIT License",
     ],
-    test_suite='test.suite',
+    test_suite='test.suitefn',
     entry_points={
         "console_scripts": [
             "wordvecspace = wordvecspace:main",

test.py

@@ -1,10 +1,18 @@
 #!/usr/bin/env python
 
 import doctest
+import unittest
 
-from wordvecspace import wordvecspace
+from wordvecspace import wordvecspace, server
 
-suite = doctest.DocTestSuite(wordvecspace)
+def suitefn():
+    suite = unittest.TestSuite()
+
+    suite.addTests(doctest.DocTestSuite(wordvecspace))
+    suite.addTests(doctest.DocTestSuite(server))
+
+    return suite
 
 if __name__ == "__main__":
     doctest.testmod(wordvecspace)
+    doctest.testmod(server)

wordvecspace/__init__.py

@@ -1,2 +1,2 @@
-from wordvecspace import WordVecSpace
-from command import main
+from .wordvecspace import WordVecSpace
+from .command import main

wordvecspace/command.py

@@ -1,35 +1,45 @@
 import code
 
 from basescript import BaseScript
-from convert import GWVec2WordVecSpace
-from wordvecspace import WordVecSpace
+
+from .convert import GWVec2WordVecSpace
+from .wordvecspace import WordVecSpace
+from .server import WordVecSpaceServer
 
 class WordVecSpaceCommand(BaseScript):
-    DESC = 'Word Vector Space command-line tool'
+    DESC = 'Word Vector Space command-line tool and Service'
+
+    DEFAULT_PORT = 8900
 
     def convert(self):
+        #FIXME: track issue to send logger
         convertor = GWVec2WordVecSpace(
                         self.args.input_dir,
                         self.args.output_dir,
                         self.args.num_vecs_per_shard)
         convertor.start()
 
+    # FIXME: track issue to remove sharding or not
     DEFAULT_NUM_VECS_PER_SHARD = 0
 
     def interact(self):
-	interact = WordVecSpace(self.args.input_dir)
+        interact = WordVecSpace(self.args.input_dir)
         interact.load()
 
-	vectors = interact.num_vectors
-	dimensions = interact.num_dimensions
+        vectors = interact.num_vectors
+        dimensions = interact.num_dimensions
+
+        print("Total number of vectors and dimensions in .npy file (%s, %s)" %(vectors, dimensions))
+        print("")
+        print("help")
+        print("%s" %(dir(interact)))
 
-	print "Total number of vectors and dimensions in .npy file (%s, %s)" %(vectors, dimensions)
-	print ""
-	print "help"
-	print "%s" %(dir(interact))
+        namespace=dict(WordVecSpace=interact)
+        code.interact("WordVecSpace Console", local=namespace)
 
-	namespace=dict(WordVecSpace=interact)
-    	code.interact("WordVecSpace Console", local=namespace)
+    def runserver(self):
+        server = WordVecSpaceServer(self.args.input_dir, self.args.port, log=self.log)
+        server.start()
 
     def define_subcommands(self, subcommands):
         super(WordVecSpaceCommand, self).define_subcommands(subcommands)
@@ -53,6 +63,16 @@ def define_subcommands(self, subcommands):
                 help='Input directory containing WordVecSpace format files'
                 ' (vocab.txt, vectors.npy)')
 
+        runserver_cmd = subcommands.add_parser('runserver',
+                help='WordVecSpace Service')
+        runserver_cmd.set_defaults(func=self.runserver)
+        runserver_cmd.add_argument('input_dir',
+                help='Input directory containing WordVecSpace format files'
+                ' (vocab.txt, vectors.npy)')
+        runserver_cmd.add_argument('-p', '--port',
+                default=self.DEFAULT_PORT, type=int,
+                help='Port for wordvecspace service to run.')
+
 def main():
     WordVecSpaceCommand().start()