First version of pipline wiht contexts addition.

Author: gsvgit

cfpq_add_context/add_contexts.py

@@ -1,24 +1,108 @@
-from load_graph import load_graph
-from intersection import intersection
-from gen_automata import generate
+import graphblas
+
+from cfpq_add_context.load_graph import load_graph
+from cfpq_add_context.intersection import intersection
+from cfpq_add_context.gen_automata import generate
+from graphblas.core.dtypes import BOOL, UINT64
+from graphblas.core.matrix import Matrix
+
+from cfpq_matrix.block.block_matrix_space_impl import BlockMatrixSpaceImpl
+from cfpq_model.cnf_grammar_template import Symbol
+from cfpq_model.label_decomposed_graph import LabelDecomposedGraph
+
 import time
 
+def indexed_to_boolean_decomposition(graph, block_count):
+    #print("graph = ", graph)
+    #print("block count = ", block_count )
+    vertex_count = graph.nrows
+    edges = graph.to_edgelist()
+    edges = zip(edges[0], edges[1])
+    new_edges = [(_edg[0] + vertex_count * _lbl, _edg[1]) for (_edg, _lbl) in edges]
+    result = Matrix.from_edgelist(new_edges, values=True, dtype=BOOL, nrows = block_count * vertex_count,
+                                             ncols=vertex_count, name = "boolean_decomposition_of_indexed")
+    return result
+
+def to_label_decomposed_graph(graph):
+    vertex_count = graph.nrows
+    alloc = Matrix(BOOL, graph.ncols, graph.nrows, name = "alloc_after_intersection")
+    alloc << graph.select(graphblas.select.select_alloc)
+    
+    alloc_r = Matrix(BOOL, graph.ncols, graph.nrows, name = "alloc_r_after_intersection")
+    alloc_r << graph.select(graphblas.select.select_alloc_r)
+
+    assign = Matrix(BOOL, graph.ncols, graph.nrows, name = "assign_after_intersection")
+    assign << graph.select(graphblas.select.select_assign)
+    
+    assign_r = Matrix(BOOL, graph.ncols, graph.nrows, name = "assign_r_after_intersection")
+    assign_r << graph.select(graphblas.select.select_assign_r)
+
+    load_i = Matrix(UINT64, graph.ncols, graph.nrows, name = "load_i_after_intersection")
+    load_i << graph.select(graphblas.select.select_load).apply(graphblas.unary.decode_load)
+
+    load_r_i = Matrix(UINT64, graph.ncols, graph.nrows, name = "load_r_i_after_intersection")
+    load_r_i << graph.select(graphblas.select.select_load_r).apply(graphblas.unary.decode_load_r)
+
+    store_i = Matrix(UINT64, graph.ncols, graph.nrows, name = "store_i_after_intersection")
+    store_i << graph.select(graphblas.select.select_store).apply(graphblas.unary.decode_store)
+
+    store_r_i = Matrix(UINT64, graph.ncols, graph.nrows, name = "store_r_i_after_intersection")
+    store_r_i << graph.select(graphblas.select.select_store_r).apply(graphblas.unary.decode_store_r)
+
+    store_block_count = store_i.reduce_scalar("max").get(0) + 1
+    load_block_count = load_i.reduce_scalar("max").get(0) + 1
+    block_count = max(store_block_count, load_block_count)
+
+    boolean_decompose_load = indexed_to_boolean_decomposition(load_i, block_count)
+    boolean_decompose_load_r = indexed_to_boolean_decomposition(load_r_i, block_count)
+    boolean_decompose_store = indexed_to_boolean_decomposition(store_i, block_count)
+    boolean_decompose_store_r = indexed_to_boolean_decomposition(store_r_i, block_count)
+
+    matrices: Dict[Symbol, Matrix] = {}
+
+    matrices[Symbol('alloc')] = alloc
+    matrices[Symbol('alloc_r')] = alloc_r
+
+    matrices[Symbol('assign')] = assign
+    matrices[Symbol('assign_r')] = assign_r
+
+    matrices[Symbol('store_i')] = boolean_decompose_store
+    matrices[Symbol('load_i')] = boolean_decompose_load
+    matrices[Symbol('store_r_i')] = boolean_decompose_store_r
+    matrices[Symbol('load_r_i')] = boolean_decompose_load_r
+
+    return LabelDecomposedGraph(
+                vertex_count=vertex_count,
+                block_matrix_space=BlockMatrixSpaceImpl(n=vertex_count, block_count=block_count),
+                dtype=BOOL,
+                matrices=matrices
+            )
+
 def add_context(file_path):
     load_graph_start = time.perf_counter()
 
     graph,number_of_contexts = load_graph(file_path)
 
     load_graph_end = time.perf_counter()
     print("Graph loaded in ", load_graph_end - load_graph_start)
+    print("Vertices in graph: ", graph.ncols)
+    print("Edges in graph: ", graph.nvals)
+    print("Numer of contexts: ", number_of_contexts)
 
     automata = generate(number_of_contexts)
 
     automata_generation_end = time.perf_counter()
     print("Automata generated in ", automata_generation_end - load_graph_end)
+    print("Vertices in automata: ", automata.ncols)
+    print("Edges in automata: ", automata.nvals)
 
     result = intersection(graph, automata)
 
     intersection_end = time.perf_counter()
     print("Graph and automata intersection competed in ", intersection_end - automata_generation_end)
+    print("Vertices in intersection: ", result.ncols)
+    print("Edges in intersection: ", result.nvals)
 
-    return result
+    return to_label_decomposed_graph(result)
+
+

cfpq_add_context/gen_automata.py

@@ -3,8 +3,8 @@
 from graphblas.core.matrix import Matrix
 from graphblas.core.dtypes import UINT64
 
-from labels import SIGMA, SIGMA_WITHOUT_CONTEXTS, SIGMA_WITHOUT_OPEN_CONTEXTS, ALL_OPEN_CONTEXTS, mk_close_context, mk_open_context
-from utils import print_matrix_to_dot
+from cfpq_add_context.labels import SIGMA, SIGMA_WITHOUT_CONTEXTS, SIGMA_WITHOUT_OPEN_CONTEXTS, ALL_OPEN_CONTEXTS, mk_close_context, mk_open_context
+from cfpq_add_context.utils import print_matrix_to_dot
 
 def generate (number_of_contexts):
     number_of_contexts = number_of_contexts * 2 # with reversed edges

cfpq_add_context/intersection.py

@@ -5,10 +5,11 @@
 from graphblas.core.operator import Semiring, Monoid, SelectOp
 from graphblas.core.dtypes import UINT64
 from graphblas import op, semiring
-import gen_automata
+import cfpq_add_context.gen_automata
+import time
 
-import labels
-from utils import print_matrix_to_dot
+import cfpq_add_context.labels
+from cfpq_add_context.utils import print_matrix_to_dot
 
 def kronecker (graph, automata):
     result = Matrix(graph.dtype, graph.nrows * automata.nrows, graph.ncols * automata.ncols, name="intersection")
@@ -17,7 +18,6 @@ def kronecker (graph, automata):
 
 def bfs (matrix, sources):
     n = matrix.nrows
-    print ("n = ", n)
 
     reachable = Vector.from_coo(sources, True, size=n, name="reachable")
     frontier = Vector.from_coo(sources, True, size=n, name="frontier")
@@ -29,7 +29,7 @@ def bfs (matrix, sources):
         new_frontier = Vector(bool, size=n)
         new_frontier(~reachable.S) << semiring.any_first(frontier @ matrix)
 
-        print("new_frontier nvals = ",  new_frontier.nvals)
+        #print("new_frontier nvals = ",  new_frontier.nvals)
 
         if new_frontier.nvals == 0:
             break
@@ -46,19 +46,37 @@ def filter_not_zero_op(x,i,j,k):
 SelectOp.register_new("filter_not_zero", filter_not_zero_op)
 
 def intersection (graph, automata) :
+    intersection_start = time.perf_counter()
+    
     intersection = kronecker(graph, automata)
+    
+    intersection_end = time.perf_counter()
+    print("Intersection done in ", intersection_end - intersection_start)
+    print("Vertices in intersection before zeroes removing: ", intersection.ncols)
+    print("Edges in intersection before zeroes removing: ", intersection.nvals)
+    
     #TODO remove when moved to patched kronecker
     intersection << intersection.select(graphblas.select.filter_not_zero)
 
-    print_matrix_to_dot(intersection, "kron.dot")
+    zeroes_removing_from_intersection_end = time.perf_counter()
+    print("Removing of zeroes from intersection done in ", zeroes_removing_from_intersection_end - intersection_end)
+    print("Vertices in intersection after zeroes removing: ", intersection.ncols)
+    print("Edges in intersection after zeroes removing: ", intersection.nvals)
+
+
+    #print_matrix_to_dot(intersection, "kron.dot")
 
     sources = [i * automata.nrows for i in range(0,graph.nrows)]
 
     reachable_vertices_mask = bfs(intersection, sources).diag(name="reachable_vertices_mask")
 
     result = Matrix(intersection.dtype, intersection.nrows, intersection.ncols, name = "filtered intersection")
     result << reachable_vertices_mask @ intersection
-    print_matrix_to_dot(result, "kron_filtered.dot")
+    
+    unreachable_vertices_removing_end = time.perf_counter()
+    print("Removing of unreachable vertices done in ", unreachable_vertices_removing_end - zeroes_removing_from_intersection_end)
+
+    #print_matrix_to_dot(result, "kron_filtered.dot")
     return result
 
 

cfpq_add_context/labels.py

@@ -1,6 +1,6 @@
 import graphblas
 
-from graphblas.core.operator import Monoid, BinaryOp
+from graphblas.core.operator import Monoid, BinaryOp, SelectOp, UnaryOp
 from graphblas import binary
 
 '''
@@ -13,6 +13,7 @@
 SIGMA =                       0b1111111111111111111111111111111111111111111111111111111111111111
 SIGMA_WITHOUT_CONTEXTS =      0b0000000000000000000000000000000000000000001111111111111111111111
 ALL_OPEN_CONTEXTS =           0b1111111111111111111110000000000000000000000000000000000000000000
+ALL_CLOSE_CONTEXTS =          0b0000000000000000000001111111111111111111110000000000000000000000
 SIGMA_WITHOUT_OPEN_CONTEXTS = 0b0000000000000000000001111111111111111111111111111111111111111111
 NOTHING =                     0b0000000000000000000000000000000000000000000000000000000000000000
 
@@ -35,6 +36,62 @@ def mk_close_context(x:int) -> int :
 def mk_other(x:int) -> int : 
     return NOTHING | x
 def mk_load(i):
-    return mk_other(2 * i + OFFSET)
+    return mk_other(4 * i + OFFSET)
+def mk_load_r(i):
+    return mk_other(4 * i + 1+ OFFSET)
 def mk_store (i):
-    return mk_other(2 * i + 1 + OFFSET)
+    return mk_other(4 * i + 2 + OFFSET)
+def mk_store_r (i):
+    return mk_other(4 * i + 3 + OFFSET)
+
+
+def select_alloc_op(x,i,j,k):
+    return x & SIGMA_WITHOUT_CONTEXTS == ALLOC
+
+def select_alloc_r_op(x,i,j,k):
+    return x & SIGMA_WITHOUT_CONTEXTS == ALLOC_R
+
+def select_assign_op(x,i,j,k):
+    return x & SIGMA_WITHOUT_CONTEXTS == ASSIGN or (x & ALL_OPEN_CONTEXTS > 0 and x & ALL_OPEN_CONTEXTS % 2 == 1) or (x & ALL_OPEN_CONTEXTS > 0 and x & ALL_CLOSE_CONTEXTS % 2 == 1)
+
+def select_assign_r_op(x,i,j,k):
+    return x & SIGMA_WITHOUT_CONTEXTS == ASSIGN_R or (x & ALL_OPEN_CONTEXTS > 0 and x & ALL_OPEN_CONTEXTS % 2 == 0) or (x & ALL_OPEN_CONTEXTS > 0 and x & ALL_CLOSE_CONTEXTS % 2 == 0)
+
+def select_load_op(x,i,j,k):
+    return x & SIGMA_WITHOUT_CONTEXTS >= OFFSET and ((x & SIGMA_WITHOUT_CONTEXTS) - OFFSET) % 4 == 0
+
+def select_load_r_op(x,i,j,k):
+    return x & SIGMA_WITHOUT_CONTEXTS >= OFFSET and ((x & SIGMA_WITHOUT_CONTEXTS) - OFFSET) % 4 == 1
+
+def select_store_op(x,i,j,k):
+    return x & SIGMA_WITHOUT_CONTEXTS >= OFFSET and ((x & SIGMA_WITHOUT_CONTEXTS) - OFFSET) % 4 == 2
+
+def select_store_r_op(x,i,j,k):
+    return x & SIGMA_WITHOUT_CONTEXTS >= OFFSET and ((x & SIGMA_WITHOUT_CONTEXTS) - OFFSET) % 4 == 3
+
+SelectOp.register_new("select_alloc", select_alloc_op)
+SelectOp.register_new("select_alloc_r", select_alloc_r_op)
+SelectOp.register_new("select_assign", select_assign_op)
+SelectOp.register_new("select_assign_r", select_assign_r_op)
+SelectOp.register_new("select_load", select_load_op)
+SelectOp.register_new("select_store", select_store_op)
+SelectOp.register_new("select_load_r", select_load_r_op)
+SelectOp.register_new("select_store_r", select_store_r_op)
+
+def decode_load_op(x):
+    return ((x & SIGMA_WITHOUT_CONTEXTS) - OFFSET) // 4
+
+def decode_load_r_op(x):
+    return ((x & SIGMA_WITHOUT_CONTEXTS) - OFFSET - 1) // 4
+
+def decode_store_op(x):
+    return ((x & SIGMA_WITHOUT_CONTEXTS) - OFFSET - 2) // 4
+
+def decode_store_r_op(x):
+    return ((x & SIGMA_WITHOUT_CONTEXTS) - OFFSET - 3) // 4
+
+
+UnaryOp.register_new("decode_load", decode_load_op)
+UnaryOp.register_new("decode_load_r", decode_load_r_op)
+UnaryOp.register_new("decode_store", decode_store_op)
+UnaryOp.register_new("decode_store_r", decode_store_r_op)

cfpq_add_context/load_graph.py

@@ -3,8 +3,8 @@
 from graphblas.core.matrix import Matrix
 from graphblas.core.dtypes import UINT64
 
-import labels
-from utils import print_matrix_to_dot
+import cfpq_add_context.labels as labels
+from cfpq_add_context.utils import print_matrix_to_dot
 
 
 
@@ -15,8 +15,12 @@ def get_edge_lbl(data_arr):
         _context = 0
         nonlocal number_of_contexts
         if len(data_arr) == 4:
-            if "load" in data_arr[2]:
+            if "load_r" in data_arr[2]:
+                return labels.mk_load_r(int(data_arr[3]))
+            elif "load" in data_arr[2]:
                 return labels.mk_load(int(data_arr[3]))
+            elif "store_r" in data_arr[2]:
+                return labels.mk_store_r(int(data_arr[3]))
             else:
                 return labels.mk_store(int(data_arr[3]))
         elif data_arr[2] == "assign":
@@ -29,9 +33,9 @@ def get_edge_lbl(data_arr):
             return labels.mk_other(labels.ALLOC_R)
         elif "open" in data_arr[2]:
             if "_r_" in data_arr[2]:
-                _context = 2 * int(data_arr[2].split('_')[2])
+                _context = 2 * (1 + int(data_arr[2].split('_')[2]))
             else:
-                _context =  2 * int(data_arr[2].split('_')[1]) + 1
+                _context =  2 * (1 + int(data_arr[2].split('_')[1])) + 1
             number_of_contexts = max(number_of_contexts, _context)
             return labels.mk_open_context(_context)
         else:
@@ -57,6 +61,6 @@ def handle_line(line):
     with open (file_path,'r') as file:
         edges = [handle_line(line) for line in file if len(line.strip()) > 0]
         result = Matrix.from_edgelist(edges, dtype=UINT64, nrows=nvertices + 1, ncols=nvertices + 1, name="graph")
-        print_matrix_to_dot(result, "graph.dot")
+        #print_matrix_to_dot(result, "graph.dot")
         return (result, (number_of_contexts // 2) + 1)
 

cfpq_cli/run_all_pairs_cflr.py

@@ -13,18 +13,23 @@
 from cfpq_cli.time_limit import time_limit, TimeoutException
 from cfpq_model.cnf_grammar_template import CnfGrammarTemplate
 from cfpq_model.label_decomposed_graph import LabelDecomposedGraph
+from cfpq_add_context.add_contexts import add_context
 
 
 def run_all_pairs_cflr(
         algo_name: str,
         graph_path: str,
         grammar_path: str,
+        add_contexts: bool,
         time_limit_sec: Optional[int],
         out_path: Optional[str],
         settings: List[AlgoSetting]
 ):
     algo = get_all_pairs_cfl_reachability_algo(algo_name)
-    graph = LabelDecomposedGraph.read_from_pocr_graph_file(graph_path)
+    if add_contexts:
+        graph = add_context(graph_path)
+    else:
+        graph = LabelDecomposedGraph.read_from_pocr_graph_file(graph_path)
     grammar = CnfGrammarTemplate.read_from_pocr_cnf_file(grammar_path)
     graph, grammar = preprocess_graph_and_grammar(graph, grammar, settings)
     try:
@@ -78,13 +83,17 @@ def main(raw_args: List[str]):
                              'to [END_VERTEX], labels along which spell a word from '
                              'the specified Context-Free Language (CFL).'
                         )
+    parser.add_argument('--add_contexts', dest='add_contexts', default=False,
+                        help='Specifies whether approximation of context sensitivity should be added.'
+                        )
     settings_manager = AlgoSettingsManager()
     settings_manager.add_args(parser)
     args = parser.parse_args(raw_args)
     run_all_pairs_cflr(
         algo_name=args.algo,
         graph_path=args.graph,
         grammar_path=args.grammar,
+        add_contexts=args.add_contexts,
         time_limit_sec=args.time_limit,
         out_path=args.out,
         settings=settings_manager.read_args(args)