Skip to content
This repository was archived by the owner on Mar 8, 2020. It is now read-only.

Commit 0a914d6

Browse files
juanjuxjuanjux
authored
Merge pull request #66 from juanjux/feature/iterator
Add libuast iterators support
2 parents 147288f + 867e8ac commit 0a914d6

5 files changed

Lines changed: 184 additions & 16 deletions

File tree

‎README.md‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,13 +44,18 @@ docker exec -it bblfshd bblfshctl driver install python bblfsh/python-driver:lat
4444
Please, read the [getting started](https://doc.bblf.sh/user/getting-started.html) guide to learn more about how to use and deploy a bblfshd.
4545

4646
```python
47-
from bblfsh import BblfshClient, filter
47+
import bblfsh
4848

49-
client = BblfshClient("0.0.0.0:9432")
49+
client = bblfsh.BblfshClient("0.0.0.0:9432")
5050
uast = client.parse("/path/to/file.py").uast
5151
print(uast)
5252
# "filter' allows you to use XPath queries to filter on result nodes:
53-
print(filter(uast, "//Import[@roleImport and @roleDeclaration]//alias")
53+
print(bblfsh.filter(uast, "//Import[@roleImport and @roleDeclaration]//alias")
54+
55+
# You can also iterate on several tree iteration orders:
56+
it = bblfsh.iterator(uast, bblfsh.TreeOrder.pre_order)
57+
for node in it:
58+
print(node.internal_type)
5459
```
5560

5661
Please read the [Babelfish clients](https://doc.bblf.sh/user/language-clients.html)

‎bblfsh/__init__.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
11
from bblfsh.client import BblfshClient
2-
from bblfsh.pyuast import filter
2+
from bblfsh.pyuast import filter, iterator
33
from bblfsh.aliases import *
44

5+
class TreeOrder:
6+
PRE_ORDER = 0
7+
POST_ORDER = 1
8+
LEVEL_ORDER = 2
9+
510
# "in" is a reserved keyword in Python thus can't be used as package name, so
611
# we import by string
712

8-
913
class RoleSearchException(Exception):
1014
pass
1115

‎bblfsh/pyuast.c‎

Lines changed: 115 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ static const char *String(const void *node, const char *prop) {
2020
return o ? PyUnicode_AsUTF8(o) : NULL;
2121
}
2222

23-
static int Size(const void *node, const char *prop) {
23+
static size_t Size(const void *node, const char *prop) {
2424
PyObject *o = Attribute(node, prop);
2525
return o ? PySequence_Size(o) : 0;
2626
}
@@ -39,7 +39,7 @@ static const char *Token(const void *node) {
3939
return String(node, "token");
4040
}
4141

42-
static int ChildrenSize(const void *node) {
42+
static size_t ChildrenSize(const void *node) {
4343
return Size(node, "children");
4444
}
4545

@@ -48,7 +48,7 @@ static void *ChildAt(const void *node, int index) {
4848
return children ? ItemAt(children, index) : NULL;
4949
}
5050

51-
static int RolesSize(const void *node) {
51+
static size_t RolesSize(const void *node) {
5252
return Size(node, "roles");
5353
}
5454

@@ -57,7 +57,7 @@ static uint16_t RoleAt(const void *node, int index) {
5757
return roles ? (uint16_t)PyLong_AsUnsignedLong(ItemAt(roles, index)) : 0;
5858
}
5959

60-
static int PropertiesSize(const void *node) {
60+
static size_t PropertiesSize(const void *node) {
6161
PyObject *properties = AttributeValue(node, "properties");
6262
return properties ? PyMapping_Size(properties) : 0;
6363
}
@@ -146,10 +146,116 @@ static Uast *ctx;
146146
/////////// PYTHON API //////////////
147147
/////////////////////////////////////
148148

149+
typedef struct {
150+
PyObject_HEAD
151+
UastIterator *iter;
152+
} PyUastIter;
153+
154+
// iterator.__iter__()
155+
static PyObject *PyUastIter_iter(PyObject *self)
156+
{
157+
Py_INCREF(self);
158+
return self;
159+
}
160+
161+
// iterator.__next__()
162+
static PyObject *PyUastIter_next(PyObject *self)
163+
{
164+
165+
PyUastIter *it = (PyUastIter *)self;
166+
167+
void *node = UastIteratorNext(it->iter);
168+
if (!node) {
169+
PyErr_SetNone(PyExc_StopIteration);
170+
return NULL;
171+
}
172+
173+
Py_INCREF(node);
174+
return (PyObject *)node;
175+
}
176+
177+
// Forward declaration for the Type ref
178+
static PyObject *PyUastIter_new(PyObject *self, PyObject *args);
179+
static void PyUastIter_dealloc(PyObject *self);
180+
181+
static PyTypeObject PyUastIterType = {
182+
PyVarObject_HEAD_INIT(NULL, 0)
183+
"pyuast.UastIterator", // tp_name
184+
sizeof(PyUastIter), // tp_basicsize
185+
0, // tp_itemsize
186+
PyUastIter_dealloc, // tp_dealloc
187+
0, // tp_print
188+
0, // tp_getattr
189+
0, // tp_setattr
190+
0, // tp_reserved
191+
0, // tp_repr
192+
0, // tp_as_number
193+
0, // tp_as_sequence
194+
0, // tp_as_mapping
195+
0, // tp_hash
196+
0, // tp_call
197+
0, // tp_str
198+
0, // tp_getattro
199+
0, // tp_setattro
200+
0, // tp_as_buffer
201+
Py_TPFLAGS_DEFAULT, // tp_flags
202+
"Internal UastIterator object", // tp_doc
203+
0, // tp_traverse
204+
0, // tp_clear
205+
0, // tp_richcompare
206+
0, // tp_weaklistoffset
207+
PyUastIter_iter, // tp_iter: __iter()__ method
208+
(iternextfunc)PyUastIter_next, // tp_iternext: next() method
209+
0, // tp_methods
210+
0, // tp_members
211+
0, // tp_getset
212+
0, // tp_base
213+
0, // tp_dict
214+
0, // tp_descr_get
215+
0, // tp_descr_set
216+
0, // tp_dictoffset
217+
0, // tp_init
218+
PyType_GenericAlloc, // tp_alloc
219+
0, // tp_new
220+
};
221+
222+
static PyObject *PyUastIter_new(PyObject *self, PyObject *args)
223+
{
224+
void *node = NULL;
225+
uint8_t order;
226+
227+
if (!PyArg_ParseTuple(args, "OB", &node, &order))
228+
return NULL;
229+
230+
PyUastIter *pyIt = PyObject_New(PyUastIter, &PyUastIterType);
231+
if (!pyIt)
232+
return NULL;
233+
234+
if (!PyObject_Init((PyObject *)pyIt, &PyUastIterType)) {
235+
Py_DECREF(pyIt);
236+
return NULL;
237+
}
238+
239+
pyIt->iter = UastIteratorNew(ctx, node, (TreeOrder)order);
240+
if (!pyIt->iter) {
241+
Py_DECREF(pyIt);
242+
return NULL;
243+
}
244+
245+
return (PyObject*)pyIt;
246+
}
247+
248+
249+
static void PyUastIter_dealloc(PyObject *self)
250+
{
251+
UastIteratorFree(((PyUastIter *)self)->iter);
252+
}
253+
149254
static PyObject *PyFilter(PyObject *self, PyObject *args)
150255
{
151256
PyObject *obj = NULL;
152257
const char *query = NULL;
258+
153259
if (!PyArg_ParseTuple(args, "Os", &obj, &query))
154260
return NULL;
155261

@@ -160,10 +266,10 @@ static PyObject *PyFilter(PyObject *self, PyObject *args)
160266
free(error);
161267
return NULL;
162268
}
163-
int len = NodesSize(nodes);
269+
size_t len = NodesSize(nodes);
164270
PyObject *list = PyList_New(len);
165271

166-
for (int i = 0; i < len; i++) {
272+
for (size_t i = 0; i < len; i++) {
167273
PyObject *node = (PyObject *)NodeAt(nodes, i);
168274
Py_INCREF(node);
169275
PyList_SET_ITEM(list, i, node);
@@ -174,6 +280,7 @@ static PyObject *PyFilter(PyObject *self, PyObject *args)
174280

175281
static PyMethodDef extension_methods[] = {
176282
{"filter", PyFilter, METH_VARARGS, "Filter nodes in the UAST using the given query"},
283+
{"iterator", PyUastIter_new, METH_VARARGS, "Get an iterator over a node"},
177284
{NULL, NULL, 0, NULL}
178285
};
179286

@@ -189,7 +296,8 @@ static struct PyModuleDef module_def = {
189296
NULL
190297
};
191298

192-
PyMODINIT_FUNC PyInit_pyuast(void)
299+
PyMODINIT_FUNC
300+
PyInit_pyuast(void)
193301
{
194302
NodeIface iface = {
195303
.InternalType = InternalType,

‎bblfsh/test.py‎

Lines changed: 53 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@
33

44
import docker
55

6-
from bblfsh import BblfshClient, filter, role_id, role_name, Node, ParseResponse
6+
from bblfsh import (BblfshClient, filter, iterator, role_id,
7+
role_name, Node, ParseResponse, TreeOrder)
78
from bblfsh.launcher import ensure_bblfsh_is_running
89
from bblfsh.client import NonUTF8ContentException
910

@@ -122,10 +123,60 @@ def testFilterBadQuery(self):
122123
node = Node()
123124
self.assertRaises(RuntimeError, filter, node, "//*roleModule")
124125

125-
def testRoleIdName(sedlf):
126+
def testRoleIdName(self):
126127
assert(role_id(role_name(1)) == 1)
127128
assert(role_name(role_id("IDENTIFIER")) == "IDENTIFIER")
128129

130+
def _itTestTree(self):
131+
root = Node()
132+
root.internal_type = 'root'
133+
son1 = Node()
134+
son1.internal_type = 'son1'
135+
136+
son1_1 = Node()
137+
son1_1.internal_type = 'son1_1'
138+
139+
son1_2 = Node()
140+
son1_2.internal_type = 'son1_2'
141+
son1.children.extend([son1_1, son1_2])
142+
143+
son2 = Node()
144+
son2.internal_type = 'son2'
145+
son2_1 = Node()
146+
son2_1.internal_type = 'son2_1'
147+
148+
son2_2 = Node()
149+
son2_2.internal_type = 'son2_2'
150+
son2.children.extend([son2_1, son2_2])
151+
152+
root.children.extend([son1, son2])
153+
154+
return root
155+
156+
def testIteratorPreOrder(self):
157+
root = self._itTestTree()
158+
it = iterator(root, TreeOrder.PRE_ORDER)
159+
self.assertIsNotNone(it)
160+
expanded = [node.internal_type for node in it]
161+
self.assertListEqual(expanded, ['root', 'son1', 'son1_1', 'son1_2',
162+
'son2', 'son2_1', 'son2_2'])
163+
164+
def testIteratorPostOrder(self):
165+
root = self._itTestTree()
166+
it = iterator(root, TreeOrder.POST_ORDER)
167+
self.assertIsNotNone(it)
168+
expanded = [node.internal_type for node in it]
169+
self.assertListEqual(expanded, ['son1_1', 'son1_2', 'son1', 'son2_1',
170+
'son2_2', 'son2', 'root'])
171+
172+
def testIteratorLevelOrder(self):
173+
root = self._itTestTree()
174+
it = iterator(root, TreeOrder.LEVEL_ORDER)
175+
self.assertIsNotNone(it)
176+
expanded = [node.internal_type for node in it]
177+
self.assertListEqual(expanded, ['root', 'son1', 'son2', 'son1_1',
178+
'son1_2', 'son2_1', 'son2_2'])
179+
129180
def _validate_resp(self, resp):
130181
self.assertIsNotNone(resp)
131182
self.assertEqual(type(resp).DESCRIPTOR.full_name,

‎setup.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
from setuptools import setup, find_packages, Extension
66
from setuptools.command.build_ext import build_ext
77

8-
LIBUAST_VERSION = "v1.5.1"
8+
LIBUAST_VERSION = "v1.6.0"
99
SDK_VERSION = "v1.8.0"
1010
SDK_MAJOR = SDK_VERSION.split('.')[0]
1111
PYTHON = "python3"
@@ -134,7 +134,7 @@ def main():
134134
},
135135
name="bblfsh",
136136
description="Fetches Universal Abstract Syntax Trees from Babelfish.",
137-
version="2.6.1",
137+
version="2.7.0",
138138
license="Apache 2.0",
139139
author="source{d}",
140140
author_email="language-analysis@sourced.tech",

0 commit comments

Comments
 (0)