Merge pull request #23 from juanjux/add-libuast

Add libuast, upgrade libuast API, bugfixing

Author: juanjux

.gitignore

@@ -101,3 +101,5 @@ ENV/
 
 # mypy
 .mypy_cache/
+
+libuast

.travis.yml

@@ -10,10 +10,12 @@ python:
   - "3.5"
   - "3.6"
 install:
-  - pip install --upgrade pip
-  - pip install grpcio
-  - CC=gcc-5 CXX=g++-5 pip install -e .
+  - pip3 install --upgrade pip
+  - pip3 install grpcio
+  - make bblfsh/libuast
+  - pip3 install . --upgrade
 script:
+  - python3 setup.py build_ext -i
   - python3 -m unittest discover .
 notifications:
   email: false

Makefile

@@ -2,18 +2,35 @@ PYTHON ?= python3
 
 makefile_dir := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 
-all: bblfsh/github/com/gogo/protobuf/gogoproto/gogo_pb2.py \
-     bblfsh/github/com/bblfsh/sdk/uast/generated_pb2.py \
-     bblfsh/github/com/bblfsh/sdk/protocol/generated_pb2_*.py \
-     bblfsh/github/__init__.py \
-     bblfsh/github/com/__init__.py \
-     bblfsh/github/com/gogo/__init__.py \
-     bblfsh/github/com/gogo/protobuf/__init__.py \
-     bblfsh/github/com/gogo/protobuf/gogoproto/__init__.py \
-     bblfsh/github/com/bblfsh/__init__.py \
-     bblfsh/github/com/bblfsh/sdk/__init__.py \
-     bblfsh/github/com/bblfsh/sdk/uast/__init__.py \
-     bblfsh/github/com/bblfsh/sdk/protocol/__init__.py
+LIBUAST_VERSION = v0.2.0
+
+.PHONY : all clean deps
+
+all: deps \
+	bblfsh/github/com/gogo/protobuf/gogoproto/gogo_pb2.py \
+	bblfsh/github/com/bblfsh/sdk/uast/generated_pb2.py \
+	bblfsh/github/com/bblfsh/sdk/protocol/generated_pb2_*.py \
+	bblfsh/github/__init__.py \
+	bblfsh/github/com/__init__.py \
+	bblfsh/github/com/gogo/__init__.py \
+	bblfsh/github/com/gogo/protobuf/__init__.py \
+	bblfsh/github/com/gogo/protobuf/gogoproto/__init__.py \
+	bblfsh/github/com/bblfsh/__init__.py \
+	bblfsh/github/com/bblfsh/sdk/__init__.py \
+	bblfsh/github/com/bblfsh/sdk/uast/__init__.py \
+	bblfsh/github/com/bblfsh/sdk/protocol/__init__.py
+
+clean:
+	rm -rf bblfsh/libuast
+	rm -rf bblfsh/github
+
+deps: bblfsh/libuast
+
+bblfsh/libuast:
+	curl -SL https://github.com/bblfsh/libuast/releases/download/$(LIBUAST_VERSION)/libuast-$(LIBUAST_VERSION).tar.gz | tar xz
+	mv libuast-$(LIBUAST_VERSION) libuast
+	cp -a libuast/src bblfsh/libuast
+	rm -rf libuast
 
 bblfsh/github/com/gogo/protobuf/gogoproto/gogo_pb2.py: github.com/gogo/protobuf/gogoproto/gogo.proto
 	protoc --python_out bblfsh github.com/gogo/protobuf/gogoproto/gogo.proto
@@ -25,7 +42,7 @@ bblfsh/github/com/bblfsh/sdk/protocol:
 	@mkdir -p $@
 
 bblfsh/github/com/bblfsh/sdk/protocol/generated_pb2_*.py: \
-        bblfsh/github/com/bblfsh/sdk/protocol github.com/bblfsh/sdk/protocol/generated.proto
+	bblfsh/github/com/bblfsh/sdk/protocol github.com/bblfsh/sdk/protocol/generated.proto
 	$(PYTHON) -m grpc.tools.protoc --python_out=bblfsh/github/com/bblfsh/sdk/protocol \
 	    --grpc_python_out=bblfsh/github/com/bblfsh/sdk/protocol \
 	    -I github.com/bblfsh/sdk/protocol -I $(makefile_dir) \

README.md

@@ -19,7 +19,17 @@ python3 -m bblfsh -f file.py
 
 ### Installation
 
+#### From the source code
+
+```bash
+git clone https://github.com/bblfsh/client-python.git
+cd client-python
+make install
 ```
+
+#### Using pip3
+
+```bash
 pip3 install bblfsh
 ```
 

bblfsh/__init__.py

@@ -1 +1,2 @@
 from bblfsh.client import BblfshClient
+from bblfsh.pyuast import filter

bblfsh/__main__.py

@@ -1,6 +1,8 @@
 import argparse
 import sys
 
+from bblfsh.pyuast import filter
+
 from bblfsh.client import BblfshClient
 from bblfsh.launcher import ensure_bblfsh_is_running
 
@@ -18,17 +20,46 @@ def setup():
     parser.add_argument("--disable-bblfsh-autorun", action="store_true",
                         help="Do not automatically launch Babelfish server "
                              "if it is not running.")
+
+    parser.add_argument("-q", "--query", default="", help="xpath query")
+    parser.add_argument("-m", "--mapn", default="", help="transform function of the results (n)")
+    parser.add_argument("-a", "--array", help='print results as an array', action='store_true')
+
     args = parser.parse_args()
     return args
 
+def run_query(root, query, mapn, as_array):
+    result = filter(root, query)
+
+    if not result:
+        print("Nothing found")
+
+    else:
+        if mapn:
+            result = [eval(mapn) for n in result]
+
+        if as_array:
+            print("results[{}] = {}".format(len(result), result))
+        else:
+            print("Running xpath query: {}".format(query))
+            print("FOUND {} roots".format(len(result)))
+
+            for i, node in enumerate(result):
+                print("== {} ==================================".format(i+1))
+                print(node)
 
 def main():
     args = setup()
     if not args.disable_bblfsh_autorun:
         ensure_bblfsh_is_running()
-    client = BblfshClient(args.endpoint)
-    print(client.parse(args.file, args.language))
 
+    client = BblfshClient(args.endpoint)
+    root = client.parse(args.file, args.language).uast
+    query = args.query
+    if query:
+        run_query(root, query, args.mapn, args.array)
+    else:
+        print(root)
 
 if __name__ == "__main__":
     sys.exit(main())

bblfsh/client.py

@@ -8,7 +8,6 @@
                                 "github/com/bblfsh/sdk/protocol"))
 sys.path.insert(0, os.path.dirname(__file__))
 
-
 class BblfshClient(object):
     """
     Babelfish gRPC client. Currently it is only capable of fetching UASTs.
@@ -28,13 +27,13 @@ def __init__(self, endpoint):
         self._stub = ProtocolServiceStub(self._channel)
 
     def parse(self, filename, language=None, contents=None, timeout=None,
-            unicode_errors="ignore"):
+              unicode_errors="ignore"):
         """
-        Queries the Babelfish server and receives the UAST for the specified
+        Queries the Babelfish server and receives the UAST response for the specified
         file.
 
         :param filename: The path to the file. Can be arbitrary if contents \
-                          is not None.
+                         is not None.
         :param language: The programming language of the file. Refer to \
                          https://doc.bblf.sh/languages.html for the list of \
                          currently supported languages. None means autodetect.
@@ -57,8 +56,8 @@ def parse(self, filename, language=None, contents=None, timeout=None,
         request = ParseRequest(filename=os.path.basename(filename),
                                content=contents,
                                language=self._scramble_language(language))
-        response = self._stub.Parse(request, timeout=timeout)
-        return response
+        return self._stub.Parse(request, timeout=timeout)
+
 
     @staticmethod
     def _scramble_language(lang):

bblfsh/github/com/bblfsh/sdk/protocol/generated_pb2.py

@@ -5,6 +5,8 @@
 
 
 class ProtocolServiceStub(object):
+  # missing associated documentation comment in .proto file
+  pass
 
   def __init__(self, channel):
     """Constructor.
@@ -20,6 +22,8 @@ def __init__(self, channel):
 
 
 class ProtocolServiceServicer(object):
+  # missing associated documentation comment in .proto file
+  pass
 
   def Parse(self, request, context):
     """Parse uses DefaultParser to process the given parsing request to get the UAST.