diff --git a/ZDD/jl_zdd/node_auxillaries.jl b/ZDD/jl_zdd/node_auxillaries.jl index 3e9ee23..a8394f8 100644 --- a/ZDD/jl_zdd/node_auxillaries.jl +++ b/ZDD/jl_zdd/node_auxillaries.jl @@ -38,6 +38,10 @@ function readable(arr::Vector{UInt8})::Array{Int64, 1} Array{Int, 1}([Int64(x) for x in arr]) end +function readable(arr::Vector{UInt32})::Array{Int64, 1} + Array{Int, 1}([Int64(x) for x in arr]) +end + function readable(cc::UInt8)::Int64 Int64(cc) end diff --git a/ZDD/jl_zdd/old_two_way_zdd.jl b/ZDD/jl_zdd/old_two_way_zdd.jl new file mode 100644 index 0000000..9ea96c9 --- /dev/null +++ b/ZDD/jl_zdd/old_two_way_zdd.jl @@ -0,0 +1,93 @@ +include("zdd.jl") + +function all_vertices_connected_to(v, node) + representative_vertex = node.comp_assign[v] + return Set(findall(==(representative_vertex), node.comp_assign)) +end + +function setup(fnode, bnode, frontier) + fcomp, bcomp, intcomp = Dict(), Dict(), Dict() # add type annotations + ffrontier, bfrontier, frontier_span = Dict(), Dict(), Dict() # add type annotations + isolated_frontier_vtxs = Set() + + for v ∈ frontier + fcomp[v] = all_vertices_connected_to(v, fnode) + bcomp[v] = all_vertices_connected_to(v, bnode) + intcomp[v] = intersect(fcomp[v],bcomp[v]) + + ffrontier[v] = intersect(fcomp[v], frontier) + bfrontier[v] = intersect(bcomp[v], frontier) + frontier_span[v] = union(ffrontier[v], bfrontier[v]) + + push!(isolated_frontier_vtxs, maximum(frontier_span[v])) + end + return fcomp, bcomp, intcomp, isolated_frontier_vtxs +end + +function check_weights(fcomp, bcomp, intcomp, frontier, acceptable, w) + for v ∈ frontier + if !(w(fcomp[v]) + w(bcomp[v]) - w(intcomp[v]) ∈ acceptable) + return false + end + end + return true +end + +function check_fps(fnode, bnode, fcomp, bcomp, intcomp, frontier) + for v ∈ frontier + for x ∈ fcomp[v] + for y ∈ bcomp[v] + if (x,y) ∈ union(fnode.fps, bnode.fps) + return false + end + end + end + end + return true +end + +function check_cc(fnode, bnode, k, isolated_frontier_vtxs) + if fnode.cc + bnode.cc + length(isolated_frontier_vtxs) == k + return true + end + return false +end + +function is_compatible(fnode, bnode, frontier, acceptable, w, k, checking) + fcomp, bcomp, intcomp, isolated_frontier_vtxs = setup(fnode, bnode, frontier) + weights = check_weights(fcomp, bcomp, intcomp, frontier, acceptable, w) + fps = check_fps(fnode, bnode, fcomp, bcomp, intcomp, frontier) + cc = check_cc(fnode, bnode, k, isolated_frontier_vtxs) + + checklist = [] + if "weights" ∈ checking + push!(checklist, weights) + end + if "fps" ∈ checking + push!(checklist, fps) + end + if "cc" ∈ checking + push!(checklist, cc) + end + if all(checklist) + return true + end + return false +end + +function w(v) + return 1 +end + +function number_compatible(fnodes, bnodes, frontier, acceptable, w, k, checking) + println("Total # of node pairs: $(length(fnodes)) * $(length(bnodes)) = $(length(fnodes) * length(bnodes))") + num_partitions = 0 + for fnode ∈ fnodes + for bnode ∈ bnodes + if is_compatible(fnode, bnode, frontier, acceptable, w, k, checking) + num_partitions += 1 + end + end + end + return num_partitions +end \ No newline at end of file diff --git a/ZDD/jl_zdd/two_way_algo_notes.txt b/ZDD/jl_zdd/two_way_algo_notes.txt new file mode 100644 index 0000000..bf2eb4f --- /dev/null +++ b/ZDD/jl_zdd/two_way_algo_notes.txt @@ -0,0 +1,55 @@ +1-2-3 + +4-5 6 +| +7 8 9 + +1 2 3 + | +4 5 6 + +7-8-9 + +f: {3}, {5,7} +b: {3}, {5}, {7} +--> {3}, {5,7} + +---- +{3,5,7} + + +f: {1,2} {3} {4,5,6} +b: {1,4} {2,3}, {5,6} + +d = {1:2, 2:2, 3:3, 4:6, 5:6, 6:6} # start with the forwards dictionary +weights = {} +for each bset in b: + find: find the 1 group, find the 4 group. + union those groups into U + add weights of 1group to 4group + weight of bset - intersection (which is the sum of weights of vertices in bset) + and find the max: max(d[u] for u in U) + relabel: d[u] = max for u in U + d = {1:6, 2:6, 3:3, 4:6, 5:6, 6:6} # after one iteration + d = {1:6, 2:6, 3:6, 4:6, 5:6, 6:6} # after second iteration + +{{1,2,3,4,5,6}} + +Union-Find: +{1,2} {1,4} -> 4 +{3} -> 3 +{2,3} -> 4 which changes 3 +{4,5,6} -> 6 which changes 1-4 +{5,6} = 6 + + + +--- (answer is {1,2,3,4,5,6}) +{1,2,4,3} +{3,2} +{4,5,6,1} +--> {1,2,3,4,5,6} + + + + + diff --git a/ZDD/jl_zdd/two_way_zdd.jl b/ZDD/jl_zdd/two_way_zdd.jl new file mode 100644 index 0000000..86bbedf --- /dev/null +++ b/ZDD/jl_zdd/two_way_zdd.jl @@ -0,0 +1,324 @@ +include("zdd.jl") + +function construct_half_zdd(g::SimpleGraph, + k::Int64, # can we change this to Int8? should we? + d::Int64, + g_edges::Array{NodeEdge,1}, + weights::Vector{Int64}=Vector{Int64}([1 for i in 1:nv(g)]), + viz::Bool=false, + save_fp::String="zdd_tree.txt")::Set{Node} + + # delete file if it already exists + if isfile(save_fp) + rm(save_fp) + end + + weights = Vector{UInt32}([convert(UInt32,i) for i in weights]) + root = Node(g_edges[1], g, weights) + + lower_bound = Int32(floor(sum(weights)/k - d)) + upper_bound = Int32(floor(sum(weights)/k + d)) + + println("Constructing a half-ZDD...") + + zdd = ZDD(g, root, viz=viz) + halfway = Int(ne(g)/2) + N = Vector{Set{Node}}([Set{Node}([]) for a in 1:halfway+1]) + N[1] = Set([root]) # why not Set(root) + frontiers = compute_all_frontiers(g, g_edges) # only need to do half... + xs = Vector{Int8}([0,1]) + zero_terminal = Node(0) + one_terminal = Node(1) + fp_container = Vector{ForbiddenPair}([]) + rm_container = Vector{ForbiddenPair}([]) + reusable_set = Set{ForbiddenPair}([]) + recycler = Stack{Node}() # what is this? + lower_vs = Vector{UInt8}([]) + + for i = 1:halfway + for n in N[i] + n_idx = zdd.nodes[n.hash] + for x in xs + n′ = make_new_node(g, g_edges, k, n, i, x, d, frontiers, + lower_bound, upper_bound, + zero_terminal, one_terminal, + fp_container, rm_container, lower_vs, recycler) + + if n′ === one_terminal + zdd.paths += n.paths + end + + if !(n′.label == NodeEdge(0,0) || n′.label == NodeEdge(1,1)) # if not a Terminal Node + n′.label = g_edges[i+1] # update the label of n′ + reusable_unique!(n′.fps, reusable_set) + sort!(n′.fps, alg=QuickSort) + n′.hash = hash(n′) + + if n′ in N[i+1] + index = Base.ht_keyindex2!(N[i+1].dict, n′) + N[i+1].dict.keys[index].paths += n.paths + else + add_zdd_node_and_edge!(zdd, n′, n, n_idx, x) + push!(N[i+1], n′) + continue + end + end + add_zdd_edge!(zdd, n, n′, n_idx, x) # the order of n and n′ are switched, but probably ok + end + end + if i == halfway + return N[i+1] + end + zdd.deleted_nodes += length(N[i]) + save_tree_so_far!(zdd, save_fp, length(N[i])) + erase_upper_levels!(zdd, N[i+1], zero_terminal, one_terminal, length(N[i])) # release memory + N[i] = Set{Node}([]) # release memory + # println(i, ": ", Base.summarysize(zdd)) + end + # return zdd +end + +function frontier_sets(node, frontier) + """ + Returns Set(C₁, C₂, ..., Cₙ) where every v ∈ Cᵢ is in the same connected component, + and Cᵢ ⊆ F where F is the set of all vertices in the frontier. + """ + frontier_sets = Set() + seen_vertices = Set() + for v ∈ collect(frontier) + if v ∈ seen_vertices + continue + end + Cᵥ = Set(findall(n -> n == v, node.comp_assign)) + if length(Cᵥ) > 0 + push!(frontier_sets, Cᵥ) + for seen_vertex ∈ Cᵥ + push!(seen_vertices, seen_vertex) + end + end + end + return frontier_sets +end + +function compute_frontier_sets(fnodes, bnodes, frontier) + frontier_sets_dictionary = Dict() + for fnode ∈ fnodes + frontier_sets_dictionary[fnode] = frontier_sets(fnode, frontier) + end + for bnode ∈ bnodes + frontier_sets_dictionary[bnode] = frontier_sets(bnode, frontier) + end + return frontier_sets_dictionary +end + +function generate_dictionaries(fnodes, bnodes, frontier) + """ Pre-compute necessary lookup tables """ + allnodes = union(fnodes, bnodes) + frontier_projection_dict = Dict(node => frontier_sets(node, frontier) for node ∈ allnodes) + fps_dict = Dict(node => node.fps for node ∈ allnodes) + ffps_dict = Dict(fnode => fnode.fps for fnode ∈ fnodes) + + merged_fps_dict = Dict() + for ffps ∈ values(ffps_dict) + for bfps ∈ values(ffps_dict) + merged_fps_dict[(ffps, bfps)] = union(ffps, bfps) + end + end + + merged_frontier_projection_dict = Dict() + fsets = Set([frontier_projection_dict[fnode] for fnode ∈ fnodes]) + for fset ∈ fsets + for bset ∈ fsets + merged_frontier_projection_dict[(fset, bset)] = union_find(fset, bset, frontier) + end + end + + intersection_dict = Dict() + for connected_components ∈ Set(values(merged_frontier_projection_dict)) + for comp ∈ connected_components + for sets_list ∈ Set(values(frontier_projection_dict)) + for set ∈ sets_list + intersection_dict[(set, comp)] = intersect(set, comp) + end + end + end + end + return frontier_projection_dict, merged_frontier_projection_dict, fps_dict, merged_fps_dict, intersection_dict +end + +function union_find(fsets, bsets, frontier) + """ Given the projections for each node, compute the merged projection """ + labels = Dict() + for fset ∈ fsets + for v ∈ fset + labels[v] = maximum(fset) + end + end + for bset ∈ bsets + M = Set() + for v ∈ bset + fvcomp = findall(x -> (labels[x] == labels[v]), collect(frontier)) + for x ∈ fvcomp + push!(M, collect(frontier)[x]) + end + end + max_label = maximum(labels[v] for v ∈ M) + for v ∈ M + labels[v] = max_label + end + end + connected_components = Set() + for v ∈ frontier + push!(connected_components, Set(findall(x -> (labels[x] == labels[v]), labels))) + end + return connected_components +end + + +function initialize(fnode, frontier, frontier_sets_dictionary) + """ DEPRECATED """ + labels = Dict() + weights = Dict(v => -1 for v ∈ frontier) + for set ∈ frontier_sets_dictionary[fnode] + for v ∈ set + labels[v] = maximum(set) + end + end + return labels, weights +end + +function merge_nodes(fnode, bnode, frontier, frontier_sets_dictionary) #, labels, weights, local_fnode_comp_weights) + """ + DEPRECATED + Merges two nodes and returns the following: + - labels :: Dict({v:l}) where v ∈ F and l ∈ F is v's connected component number + - weights :: Dict({v:n}) where v ∈ F and n ∈ N is the weight of v's component + - connected_components :: Set(C₁, C₂, ..., Cₙ) frontier sets of the merged node + - merged_fps :: Array(ForbiddenPair) union of the FPS from each node + + Outline of algorithm: + 0. Make a copy of fnode.comp_weights since we'll be modifying it later + 1. Initialize `weights` dictionary to -1 for all v ∈ F + 2. Initialize `labels` dictionary from fnode's perspective + 3. For each connected component `bcomp` in bnode: + # We want to figure out the merged component `M`, which is all the vertices in F + # that are connected by the merge of bcomp and fnode. + a. Initialize `M` = Set() and `w` = 0 (the weight of all vertices in `M`) + b. For each vertex v ∈ `bcomp`: + i. Find the set `fvcomp`: {x ∈ F | x touches v from fnode's perspective}, + and add each x ∈ `fvcomp` to M + ii. Once per fcomp, add the weight of fcomp to `w` + c. The weight added to `w` by `bcomp` is the weight of bcomp minus the weight + of the intersection of bcomp and any components it touches in fnode. + d. Now that `M` and `w` are finalized, we store the information by: + i. Relabel `labels` such that every v ∈ M has is connected + ii. In `weights`, assign `w` to every v ∈ M + iii. Update our copy of fnode.comp_weights to `w` for future bcomps + 4. Make `merged_fps` — just the union of fnode.fps and bnode.fps + 5. Make `connected_components` — just the Set(values(labels)) + """ + + labels, weights = initialize(fnode, frontier, frontier_sets_dictionary) + + local_fnode_comp_weights = deepcopy(fnode.comp_weights) + for bcomp ∈ frontier_sets_dictionary[bnode] + M = Set() + w = 0 + seen_fcomps = Set() + for v ∈ bcomp + fvcomp = findall(x -> (labels[x] == labels[v]), collect(frontier)) + for x ∈ fvcomp + push!(M,collect(frontier)[x]) + end + if !(labels[v] ∈ seen_fcomps) + w += local_fnode_comp_weights[labels[v]] + push!(seen_fcomps, labels[v]) + end + end + + w_bcomp = maximum(bnode.comp_weights[v] for v ∈ bcomp) # if you just pick one, you might hit a weird 0 + w_intersection = length(bcomp) # TODO: generalize past unit weights + w += (w_bcomp - w_intersection) + + max_label = maximum(labels[v] for v ∈ M) + for v ∈ M + labels[v] = max_label + weights[v] = w + local_fnode_comp_weights[labels[v]] = w + end + end + + return labels, weights +end + +function check_cc(fnode, bnode, connected_components, k) + if fnode.cc + bnode.cc + length(connected_components) == k + return true + else + return false + end +end + +function check_weights(fnode, bnode, frontier_projection_dict, intersection_dict, connected_components, acceptable) + for comp ∈ connected_components + w = 0 + if length(comp) > ceil(maximum(acceptable)/2) # needs rook contiguity and unit pop. + return false + end + for node ∈ [fnode, bnode] + for set ∈ frontier_projection_dict[node] + if length(intersection_dict[(set, comp)]) == length(set) # there should be no partial intersections + wset = maximum(node.comp_weights[v] for v ∈ set) + w += wset - length(set) # assumes unit pop + end + end + end + w += length(comp) + if w ∉ acceptable + return false + end + end + return true +end + +function check_fps(fnode, bnode, fps_dict, merged_fps_dict, connected_components) + merged_fps = merged_fps_dict[(fps_dict[fnode], fps_dict[bnode])] + for comp ∈ connected_components + for v₁ ∈ comp + for v₂ ∈ comp + if v₁ != v₂ + if ForbiddenPair(v₁, v₂) ∈ merged_fps + return false + end + end + end + end + end + return true +end + +function count_paths_from_halfway(fnodes, bnodes, + frontier_projection_dict, + merged_frontier_projection_dict, + fps_dict, merged_fps_dict, + intersection_dict, + acceptable, k, verbose=false) + num_paths = 0 + for (fi, fnode) ∈ enumerate(fnodes) + fset = frontier_projection_dict[fnode] + for (bi, bnode) ∈ enumerate(bnodes) + bset = frontier_projection_dict[bnode] + connected_components = merged_frontier_projection_dict[(fset, bset)] + cc = check_cc(fnode, bnode, connected_components, k) + w = check_weights(fnode, bnode, frontier_projection_dict, intersection_dict, connected_components, acceptable) + fps = check_fps(fnode, bnode, fps_dict, merged_fps_dict, connected_components) + if cc & w & fps + num_paths += fnode.paths * bnode.paths + end + if verbose + println("($fi,$bi) contributes $(fnode.paths*bnode.paths) solns") + end + end + end + return num_paths +end \ No newline at end of file diff --git a/ZDD/jl_zdd/weighted.jl b/ZDD/jl_zdd/weighted.jl index 045a27c..ac27409 100644 --- a/ZDD/jl_zdd/weighted.jl +++ b/ZDD/jl_zdd/weighted.jl @@ -11,6 +11,7 @@ include("edge_ordering.jl") include("zdd.jl") include("count_enumerate.jl") include("visualization.jl") +include("two_way_zdd.jl") function make_new_node(g::SimpleGraph, diff --git a/ZDD/jl_zdd/weighted_node.jl b/ZDD/jl_zdd/weighted_node.jl index 35b35cf..02f1547 100644 --- a/ZDD/jl_zdd/weighted_node.jl +++ b/ZDD/jl_zdd/weighted_node.jl @@ -43,7 +43,7 @@ function custom_deepcopy(n::Node, recycler::Stack{Node}, x::Int8)::Node return n end if isempty(recycler) - comp_weights = Vector{UInt32}(undef, length(n.comp_weights)) + comp_weights = zeros(UInt32, length(n.comp_weights)) comp_assign = zeros(UInt8, length(n.comp_assign)) fps = Vector{ForbiddenPair}(undef, length(n.fps)) diff --git a/ZDD/jl_zdd/weightless.jl b/ZDD/jl_zdd/weightless.jl index 70f7d32..7566f9f 100644 --- a/ZDD/jl_zdd/weightless.jl +++ b/ZDD/jl_zdd/weightless.jl @@ -10,6 +10,7 @@ include("edge_ordering.jl") include("zdd.jl") include("count_enumerate.jl") include("visualization.jl") +include("two_way_zdd.jl") function make_new_node(g::SimpleGraph, diff --git a/ZDD/jl_zdd/zdd_jl.ipynb b/ZDD/jl_zdd/zdd_jl.ipynb index 2fd0ca8..77788dd 100644 --- a/ZDD/jl_zdd/zdd_jl.ipynb +++ b/ZDD/jl_zdd/zdd_jl.ipynb @@ -9,11 +9,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[32m\u001b[1m Activating\u001b[22m\u001b[39m environment at `~/.julia/environments/zdd/Project.toml`\n", - "┌ Info: Precompiling GraphPlot [a2cc645c-3eea-5389-862e-a155d0052231]\n", - "└ @ Base loading.jl:1278\n", - "┌ Info: Precompiling BenchmarkTools [6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf]\n", - "└ @ Base loading.jl:1278\n" + "\u001b[32m\u001b[1m Activating\u001b[22m\u001b[39m environment at `~/.julia/environments/zdd/Project.toml`\n" ] } ], @@ -24,12 +20,13 @@ "using Compose\n", "using Random\n", "using Traceur\n", - "using BenchmarkTools" + "using BenchmarkTools\n", + "using StatProfilerHTML" ] }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -38,7 +35,7 @@ "adjust_node! (generic function with 1 method)" ] }, - "execution_count": 40, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -50,77 +47,132 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 14, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Accepting districts with population in [3, 3]\n", - " 0.002601 seconds (1.17 k allocations: 105.688 KiB)\n" + "Constructing a half-ZDD...\n", + " 0.017077 seconds (24.74 k allocations: 2.123 MiB)\n", + "Constructing a half-ZDD...\n", + " 0.014161 seconds (24.76 k allocations: 2.159 MiB)\n" ] } ], "source": [ - "# grid graph\n", - "m = 3\n", - "dims = [m, m]\n", + "m = 5\n", + "dims = [m,m]\n", "k = m\n", "d = 0\n", - "contiguity = \"rook\"\n", + "acceptable = m-d:m+d\n", + "g = grid(dims)\n", + "g_edges = optimal_grid_edge_order_diags(g, dims[1], dims[2])\n", + "forwards = convert_lightgraphs_edges_to_node_edges(g_edges)\n", + "backwards = reverse(forwards)\n", "\n", - "if contiguity == \"queen\"\n", - " g = queen_grid(dims)\n", - " g_edges = optimal_queen_grid_edge_order(g, dims[1], dims[2])\n", - "else\n", - " g = grid(dims)\n", - " g_edges = optimal_grid_edge_order_diags(g, dims[1], dims[2])\n", - "end\n", + "frontiers = compute_all_frontiers(g, forwards)\n", + "middle_frontier = frontiers[Int(ceil(length(frontiers)/2))]\n", "\n", - "g_edges = convert_lightgraphs_edges_to_node_edges(g_edges)\n", - "@time zdd = construct_zdd(g, k, d, g_edges)\n", - "nothing" + "# @time zdd = construct_zdd(g, k, d, forwards)\n", + "@time fnodes = construct_half_zdd(g, k, d, forwards); nothing\n", + "@time bnodes = construct_half_zdd(g, k, d, backwards); nothing\n", + "# count_paths(zdd)" ] }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 15, "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 1.507757 seconds (16.31 M allocations: 790.128 MiB, 9.94% gc time)\n" + ] + } + ], + "source": [ + "@time frontier_projection_dict, merged_frontier_projection_dict, fps_dict, merged_fps_dict, intersection_dict = generate_dictionaries(fnodes, bnodes, middle_frontier); nothing" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 10.856397 seconds (110.11 M allocations: 2.480 GiB, 3.63% gc time)\n" + ] + }, { "data": { "text/plain": [ - "10" + "4006" ] }, - "execution_count": 48, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "count_paths(zdd)" + "@time count_paths_from_halfway(fnodes, bnodes, \n", + " frontier_projection_dict, \n", + " merged_frontier_projection_dict,\n", + " fps_dict, merged_fps_dict,\n", + " intersection_dict,\n", + " acceptable, k, false)" ] }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 18, "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accepting districts with population in [5, 5]\n", + " 0.068781 seconds (131.75 k allocations: 9.191 MiB)\n" + ] + } + ], + "source": [ + "@time zdd = construct_zdd(g, k, d, forwards); nothing" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 0.004653 seconds (1.34 k allocations: 83.282 KiB)\n" + ] + }, { "data": { "text/plain": [ - "79" + "4006" ] }, - "execution_count": 49, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "num_nodes(zdd)" + "@time count_paths(zdd)" ] }, { @@ -133,7 +185,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Julia 1.5.3", + "display_name": "Julia 1.5.1", "language": "julia", "name": "julia-1.5" }, @@ -141,7 +193,7 @@ "file_extension": ".jl", "mimetype": "application/julia", "name": "julia", - "version": "1.5.3" + "version": "1.5.1" } }, "nbformat": 4,