diff --git a/minitests/graph_folding/Makefile b/minitests/graph_folding/Makefile index f888f6056..2f7ac2479 100644 --- a/minitests/graph_folding/Makefile +++ b/minitests/graph_folding/Makefile @@ -8,7 +8,7 @@ all: build_${PART}/wire_patterns.bin build_${PART}: mkdir -p build_${PART} -${DATABASE}: build_node_lookup.py node_lookup.py build_${PART} +${DATABASE}: build_node_lookup.py node_lookup.py | build_${PART} python3 build_node_lookup.py \ --db-root ${DB_ROOT} \ --part ${PART} \ diff --git a/minitests/graph_folding/distributed_bsc.py b/minitests/graph_folding/distributed_bsc.py new file mode 100644 index 000000000..fe4fa15ad --- /dev/null +++ b/minitests/graph_folding/distributed_bsc.py @@ -0,0 +1,499 @@ +import random +import bitarray +import multiprocessing +import sys + + +class BipartiteAdjacencyMatrix(): + def __init__(self): + self.u = set() + self.v = set() + + self.edges = set() + self.frozen_edges = None + + def add_u(self, u): + self.u.add(u) + + def add_v(self, v): + self.v.add(v) + + def add_edge(self, ui, vj): + assert ui in self.u + assert vj in self.v + self.edges.add((ui, vj)) + + def build(self): + self.u = sorted(self.u) + self.v = sorted(self.v) + + edges = self.edges + self.edges = None + self.frozen_edges = edges + + self.ui_to_idx = {} + self.vj_to_idx = {} + + for idx, ui in enumerate(self.u): + self.ui_to_idx[ui] = idx + + for idx, vj in enumerate(self.v): + self.vj_to_idx[vj] = idx + + self.u_to_v = [ + bitarray.bitarray(len(self.v)) for _ in range(len(self.u)) + ] + self.v_to_u = [ + bitarray.bitarray(len(self.u)) for _ in range(len(self.v)) + ] + + for arr in self.u_to_v: + arr.setall(False) + + for arr in self.v_to_u: + arr.setall(False) + + def add_edge(ui, vj): + ui_idx = self.ui_to_idx[ui] + vj_idx = self.vj_to_idx[vj] + + self.u_to_v[ui_idx][vj_idx] = True + self.v_to_u[vj_idx][ui_idx] = True + + for ui, vj in edges: + add_edge(ui, vj) + assert self.is_edge(ui, vj) + assert self.is_edge_reverse(ui, vj) + + for ui_idx in range(len(self.u)): + for vj_idx in range(len(self.v)): + ui = self.u[ui_idx] + vj = self.v[vj_idx] + + if self.u_to_v[ui_idx][vj_idx]: + assert self.v_to_u[vj_idx][ui_idx] + assert (ui, vj) in self.frozen_edges + else: + assert not self.v_to_u[vj_idx][ui_idx] + assert (ui, vj) not in self.frozen_edges + + def get_row(self, ui): + ui_idx = self.ui_to_idx[ui] + return self.u_to_v[ui_idx] + + def get_col(self, vj): + vj_idx = self.vj_to_idx[vj] + return self.v_to_u[vj_idx] + + def is_edge(self, ui, vj): + ui_idx = self.ui_to_idx[ui] + vj_idx = self.vj_to_idx[vj] + + return self.u_to_v[ui_idx][vj_idx] + + def is_edge_reverse(self, ui, vj): + ui_idx = self.ui_to_idx[ui] + vj_idx = self.vj_to_idx[vj] + + return self.v_to_u[vj_idx][ui_idx] + + def density(self): + return len(self.frozen_edges) / (len(self.u) * len(self.v)) + + +def test_selected_rows(graph, selected_rows, test_col, test_row): + rows = graph.u + columns = graph.v + test_col.setall(False) + test_row.setall(False) + + for row in selected_rows: + test_col[graph.ui_to_idx[row]] = True + + I = set() + J = set() + + for j in columns: + if test_col & graph.get_col(j) == test_col: + J.add(j) + test_row[graph.vj_to_idx[j]] = True + + if len(J) > 0: + for i in rows: + if test_row & graph.get_row(i) == test_row: + I.add(i) + + if len(I) > 0: + for i in I: + for j in J: + assert graph.is_edge(i, j), (i, j, selected_rows) + + return frozenset(I), frozenset(J) + + +def one_iteration_bsc_row(graph, P, test_col, test_row): + selected_rows = random.choices(graph.u, k=P) + return test_selected_rows(graph, selected_rows, test_col, test_row) + + +def test_selected_cols(graph, selected_cols, test_col, test_row): + rows = graph.u + columns = graph.v + + test_col.setall(False) + test_row.setall(False) + + for col in selected_cols: + test_row[graph.vj_to_idx[col]] = True + + I = set() + J = set() + + for i in rows: + if test_row & graph.get_row(i) == test_row: + I.add(i) + test_col[graph.ui_to_idx[i]] = True + + if len(I) > 0: + for j in columns: + if test_col & graph.get_col(j) == test_col: + J.add(j) + + if len(J) > 0: + for i in I: + for j in J: + assert graph.is_edge(i, j), (i, j, selected_cols) + + return frozenset(I), frozenset(J) + + +def one_iteration_bsc_col(graph, P, test_col, test_row): + selected_cols = random.choices(graph.v, k=P) + return test_selected_cols(graph, selected_cols, test_col, test_row) + + +def worker(graph, P, work_queue, cancel_queue, result_queue): + test_col = bitarray.bitarray(len(graph.u)) + test_row = bitarray.bitarray(len(graph.v)) + + while True: + if not cancel_queue.empty(): + _ = cancel_queue.get() + break + + itrs = work_queue.get() + + if itrs is None: + break + + iter_type, start_iter, stop_iter = itrs + + if iter_type == 'random': + for itr in range(start_iter, stop_iter): + result = one_iteration_bsc_row(graph, P, test_col, test_row) + if result is not None: + result_queue.put((itr, result)) + + result = one_iteration_bsc_col(graph, P, test_col, test_row) + if result is not None: + result_queue.put((itr, result)) + elif iter_type == 'row': + for row in range(start_iter, stop_iter): + selected_rows = [graph.u[row]] + result = test_selected_rows( + graph, selected_rows, test_col, test_row) + if result is not None: + result_queue.put((row, result)) + elif iter_type == 'col': + for col in range(start_iter, stop_iter): + selected_cols = [graph.v[col]] + result = test_selected_cols( + graph, selected_cols, test_col, test_row) + if result is not None: + result_queue.put((len(graph.u) + col, result)) + else: + print( + 'Failed to understand iter_type = {}'.format(iter_type), + file=sys.stderr) + break + + # Mark that this worker has terminated. + result_queue.put(None) + + +VERBOSE = False + + +def find_bsc_par(num_workers, batch_size, graph, N, P): + remaining_edges = set(graph.frozen_edges) + num_edges = len(remaining_edges) + graph.frozen_edges = None + + work_queue = multiprocessing.Queue() + cancel_queue = multiprocessing.Queue() + result_queue = multiprocessing.Queue() + + #if P == 1 and len(graph.u)+len(graph.v) < 4*N: + # all_row_and_col = True + # P = 2 + #else: + # all_row_and_col = False + all_row_and_col = True + + processes = [] + for _ in range(num_workers): + p = multiprocessing.Process( + target=worker, + args=( + graph, + P, + work_queue, + cancel_queue, + result_queue, + )) + processes.append(p) + p.start() + + #for start_iter in range(0, N, batch_size): + # work_queue.put(('random', start_iter, start_iter+batch_size)) + + if all_row_and_col: + # Just test every row and col rather than doing it randomly. + for start_iter in range(0, len(graph.u), batch_size): + work_queue.put( + ( + 'row', start_iter, + min(start_iter + batch_size, len(graph.u)))) + + for start_iter in range(0, len(graph.v), batch_size): + work_queue.put( + ( + 'col', start_iter, + min(start_iter + batch_size, len(graph.v)))) + + for _ in range(num_workers): + work_queue.put(None) + + remaining_workers = num_workers + found_solutions = set() + while remaining_workers > 0 or not result_queue.empty(): + result = result_queue.get() + + if result is None: + # A worker finished and has terminated + remaining_workers -= 1 + else: + itr, (I, J) = result + + if (I, J) not in found_solutions: + found_solutions.add((I, J)) + for i in I: + for j in J: + remaining_edges.discard((i, j)) + + if VERBOSE: + print( + '{: 10d}/{: 10d} ({: 10.3g} %) ({: 10d}, {: 10d}) = {: 10d}, remaining {: 10d} / {: 10d} ({: 10.3g} %)' + .format( + itr, N, 100 * itr / N, len(I), len(J), + len(I) * len(J), len(remaining_edges), num_edges, + 100 - 100 * len(remaining_edges) / num_edges)) + + #if len(remaining_edges) == 0: + # # Stop once we have an initial set. + # # We still need to wait for all workers to finish, so don't + # # break out of the loop. + # for _ in range(num_workers): + # cancel_queue.put(None) + + for p in processes: + p.join() + + while not work_queue.empty(): + _ = work_queue.get() + + while not cancel_queue.empty(): + _ = cancel_queue.get() + + return found_solutions, remaining_edges + + +def find_bsc(graph, N, P): + rows = graph.u + columns = graph.v + + remaining_edges = graph.frozen_edges + + test_col = bitarray.bitarray(len(rows)) + test_row = bitarray.bitarray(len(columns)) + for itr in range(N): + result = one_iteration_bsc_row(graph, N, P, test_col, test_row) + + if result is not None: + I, J = result + print( + '{}/{} ({} %) ({}, {}) = {}, remaining {}'.format( + itr, N, itr / N, len(I), len(J), + len(I) * len(J), len(remaining_edges))) + else: + print('No I or no J!') + + +def do_subgraph_intersect(I, J, edge_to_idx): + subgraph_intersect = bitarray.bitarray(len(edge_to_idx)) + subgraph_intersect.setall(False) + + for i in I: + for j in J: + edge_idx = edge_to_idx.get((i, j), None) + if edge_idx is not None: + subgraph_intersect[edge_idx] = True + + return subgraph_intersect + + +def greedy_set_cover_with_complete_bipartite_subgraphs( + edges, complete_bipartite_subgraphs): + # Order by largest to smallest graph. At a minimum should speed up the + # first loop because the largest complete_bipartite_subgraphs always will + # be the first selected in a greedy algo. + complete_bipartite_subgraphs = sorted( + complete_bipartite_subgraphs, + key=lambda k: len(k[0]) * len(k[1]), + reverse=True) + + edges = sorted(edges) + edge_to_idx = {} + for idx, edge in enumerate(edges): + assert edge not in edge_to_idx + edge_to_idx[edge] = idx + + remaining_edges = bitarray.bitarray(len(edges)) + remaining_edges.setall(True) + + intersect = [None for _ in range(len(complete_bipartite_subgraphs))] + + unused_set = set() + for idx, (I, J) in enumerate(complete_bipartite_subgraphs): + subgraph_intersect = do_subgraph_intersect(I, J, edge_to_idx) + if subgraph_intersect.count() != 0: + intersect[idx] = subgraph_intersect + else: + unused_set.add(idx) + + output_idx = set() + + right_nodes = [0, set()] + + def add_index_to_output(idx): + nonlocal remaining_edges + output_idx.add(idx) + I, J = complete_bipartite_subgraphs[idx] + + mask = ~intersect[idx] + remaining_edges &= mask + + for inter_idx, s in enumerate(intersect): + if inter_idx in unused_set: + continue + + if inter_idx in output_idx: + continue + + intersect[inter_idx] = s & mask + + right_nodes[0] += len(J) + right_nodes[1] |= J + + def get_cost_for_subset(idx): + I, J = complete_bipartite_subgraphs[idx] + + count = intersect[idx].count() + #cost = len(I)+len(J) + cost = 1 + + return cost, count + + while remaining_edges.count() > 0: + if VERBOSE: + print( + len(output_idx), remaining_edges.count(), right_nodes[0], + len(right_nodes[1])) + + best_idx = None + lowest_cost = None + for idx, (I, J) in enumerate(complete_bipartite_subgraphs): + if idx in output_idx: + continue + + if idx in unused_set: + continue + + cost, count = get_cost_for_subset(idx) + + if count == 0: + unused_set.add(idx) + else: + cost = cost / count + + if lowest_cost is None or cost < lowest_cost: + lowest_cost = cost + best_idx = idx + + assert best_idx is not None, len(output_idx) + + add_index_to_output(best_idx) + + return [complete_bipartite_subgraphs[idx] for idx in output_idx] + + +def greedy_set_cover_worker(required_solutions, work_queue, result_queue): + while True: + work = work_queue.get() + if work is None: + break + + key, edges = work + subgraphs = greedy_set_cover_with_complete_bipartite_subgraphs( + edges, required_solutions) + result_queue.put((key, subgraphs)) + + result_queue.put(None) + + +def greed_set_cover_par(num_workers, required_solutions, edges_iter): + work_queue = multiprocessing.Queue() + result_queue = multiprocessing.Queue() + + processes = [] + for _ in range(num_workers): + p = multiprocessing.Process( + target=greedy_set_cover_worker, + args=( + required_solutions, + work_queue, + result_queue, + )) + processes.append(p) + p.start() + + for key, edges in edges_iter: + work_queue.put((key, edges)) + + for _ in range(num_workers): + work_queue.put(None) + + remaining_workers = num_workers + while remaining_workers > 0 or not result_queue.empty(): + result = result_queue.get() + + if result is None: + remaining_workers -= 1 + else: + yield result + + for p in processes: + p.join() + + assert work_queue.empty() + assert result_queue.empty() diff --git a/minitests/graph_folding/estimate_sizes.py b/minitests/graph_folding/estimate_sizes.py new file mode 100644 index 000000000..90add95de --- /dev/null +++ b/minitests/graph_folding/estimate_sizes.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (C) 2017-2020 The Project X-Ray Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + +import argparse + +from prjxray import util +from prjxray.db import Database +import struct + + +def main(): + parser = argparse.ArgumentParser() + util.db_root_arg(parser) + util.part_arg(parser) + + args = parser.parse_args() + + db = Database(args.db_root, args.part) + grid = db.grid() + + sizeof_delta = struct.calcsize('i') + sizeof_wire_in_tile_idx = struct.calcsize('i') + cost_per_wire = 2 * sizeof_delta + sizeof_wire_in_tile_idx + _ = cost_per_wire + + all_wires = 0 + tile_type_to_count = {} + for tile in grid.tiles(): + gridinfo = grid.gridinfo_at_tilename(tile) + tile_type = gridinfo.tile_type + + if tile_type not in tile_type_to_count: + tile_type_to_count[tile_type] = 0 + + tile_type_to_count[tile_type] += 1 + + tile_type_to_wires = {} + for tile_type in tile_type_to_count: + tile_type_info = db.get_tile_type(tile_type) + tile_type_to_wires[tile_type] = len(tile_type_info.get_wires()) + all_wires += len(tile_type_info.get_wires()) + + for tile_type in sorted( + tile_type_to_count, key= + lambda tile_type: tile_type_to_count[tile_type] * tile_type_to_wires[tile_type] + ): + print( + tile_type, tile_type_to_count[tile_type], + tile_type_to_wires[tile_type]) + print(all_wires) + + +if __name__ == "__main__": + main() diff --git a/minitests/graph_folding/reduce_graph_for_type.py b/minitests/graph_folding/reduce_graph_for_type.py new file mode 100644 index 000000000..a3cf832c0 --- /dev/null +++ b/minitests/graph_folding/reduce_graph_for_type.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (C) 2017-2020 The Project X-Ray Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + +import argparse +from collections import namedtuple +import progressbar +import capnp +import capnp.lib.capnp +capnp.remove_import_hook() +import math +from distributed_bsc import BipartiteAdjacencyMatrix, find_bsc_par, \ + greedy_set_cover_with_complete_bipartite_subgraphs, \ + greed_set_cover_par +import gc +import multiprocessing + +from prjxray.node_lookup import NodeLookup + +Tile = namedtuple('Tile', 'tile_pkey') +WireToNode = namedtuple( + 'WireToNode', 'wire_in_tile_pkey delta_x delta_y node_wire_in_tile_pkey') +NodeToWire = namedtuple('NodeToWire', 'wire_in_tile_pkey delta_x delta_y') + + +def get_graph(database, tile): + lookup = NodeLookup(database=database) + cur = lookup.conn.cursor() + cur2 = lookup.conn.cursor() + cur3 = lookup.conn.cursor() + + all_tiles = set() + all_wire_to_nodes = set() + + graph = BipartiteAdjacencyMatrix() + + cur.execute("SELECT pkey FROM tile_type WHERE name = ?;", (tile, )) + tile_type_pkey = cur.fetchone()[0] + + for tile_pkey, tile_type_pkey, tile_name, tile_x, tile_y in progressbar.progressbar( + cur.execute( + "SELECT pkey, tile_type_pkey, name, x, y FROM tile WHERE tile_type_pkey = ?;", + (tile_type_pkey, ))): + tile = Tile(tile_pkey=tile_pkey) + graph.add_u(tile) + all_tiles.add(tile) + + for wire_in_tile_pkey, wire_pkey, node_pkey in cur2.execute(""" +SELECT wire_in_tile_pkey, wire.pkey, wire.node_pkey +FROM wire +WHERE tile_pkey = ?; + """, (tile_pkey, )): + cur3.execute( + """ +SELECT tile.x, tile.y, node.wire_in_tile_pkey +FROM node +INNER JOIN tile ON node.tile_pkey = tile.pkey +WHERE node.pkey = ?; + """, (node_pkey, )) + node_tile_x, node_tile_y, node_wire_in_tile_pkey = cur3.fetchone() + + pattern = WireToNode( + wire_in_tile_pkey=wire_in_tile_pkey, + delta_x=node_tile_x - tile_x, + delta_y=node_tile_y - tile_y, + node_wire_in_tile_pkey=node_wire_in_tile_pkey) + + if pattern not in all_wire_to_nodes: + all_wire_to_nodes.add(pattern) + graph.add_v(pattern) + + graph.add_edge(tile, pattern) + + graph.build() + + return graph + + +def main(): + multiprocessing.set_start_method('spawn') + + parser = argparse.ArgumentParser() + parser.add_argument('--database', required=True) + parser.add_argument('--tile', required=True) + + args = parser.parse_args() + + graph = get_graph(args.database, args.tile) + all_edges = set(graph.frozen_edges) + gc.collect() + + density = graph.density() + beta = .5 + P = (0.6 - 0.8 * beta) * math.exp((4 + 3 * beta) * density) + N = 0.01 * len(graph.u) * len(graph.v) + + tile_wire_ids = set() + dxdys = set() + max_dxdy = 0 + for pattern in graph.v: + tile_wire_ids.add(pattern.node_wire_in_tile_pkey) + dxdys.add((pattern.delta_x, pattern.delta_y)) + max_dxdy = max(max_dxdy, abs(pattern.delta_x)) + max_dxdy = max(max_dxdy, abs(pattern.delta_y)) + + print('Unique node wire in tile pkey {}'.format(len(tile_wire_ids))) + print('Unique pattern {}'.format(len(graph.v))) + print('Unique dx dy {}'.format(len(dxdys))) + print('Unique dx dy dist {}'.format(max_dxdy)) + print( + 'density = {}, beta = {}, P = {}, N = {}'.format(density, beta, P, N)) + + P = math.ceil(P) + N = math.ceil(N) + + found_solutions, remaining_edges = find_bsc_par( + num_workers=40, batch_size=100, graph=graph, N=N, P=P) + assert len(remaining_edges) == 0 + print('Found {} possible complete subgraphs'.format(len(found_solutions))) + + required_solutions = greedy_set_cover_with_complete_bipartite_subgraphs( + all_edges, found_solutions) + print( + '{} complete subgraphs required for solution'.format( + len(required_solutions))) + + required_solutions.sort() + + solution_to_idx = {} + for idx, solution in enumerate(required_solutions): + solution_to_idx[solution] = idx + + def get_tile_edges(): + for tile in graph.u: + edges = set() + for vj_idx, is_set in enumerate(graph.get_row(tile)): + if is_set: + pattern = graph.v[vj_idx] + edges.add((tile, pattern)) + + yield tile, edges + + tile_patterns = set() + tile_to_tile_patterns = {} + + for tile, solutions_for_tile in progressbar.progressbar( + greed_set_cover_par(num_workers=40, + required_solutions=required_solutions, + edges_iter=get_tile_edges())): + tile_pattern = set() + for solution in solutions_for_tile: + tile_pattern.add(solution_to_idx[solution]) + + tile_pattern = frozenset(tile_pattern) + tile_to_tile_patterns[tile] = tile_pattern + tile_patterns.add(tile_pattern) + + print('Have {} tile patterns'.format(len(tile_patterns))) + print( + 'Max {} patterns'.format( + max(len(patterns) for patterns in tile_to_tile_patterns.values()))) + + #for tile, pattern in tile_to_tile_patterns.items(): + # print(tile, pattern) + + +if __name__ == "__main__": + main() diff --git a/minitests/graph_folding/test_reduction.py b/minitests/graph_folding/test_reduction.py index 7b639a181..cdde78183 100644 --- a/minitests/graph_folding/test_reduction.py +++ b/minitests/graph_folding/test_reduction.py @@ -40,6 +40,9 @@ def main(): wire_to_node_patterns = {} tile_nodes_to_wire_patterns = {} + all_wire_to_nodes = set() + all_node_to_wires = set() + for tile_pkey, tile_type_pkey, tile_name, tile_x, tile_y in progressbar.progressbar( cur.execute("SELECT pkey, tile_type_pkey, name, x, y FROM tile;")): @@ -59,12 +62,13 @@ def main(): """, (node_pkey, )) node_tile_x, node_tile_y, node_wire_in_tile_pkey = cur3.fetchone() - wire_to_nodes.append( - WireToNode( - wire_in_tile_pkey=wire_in_tile_pkey, - delta_x=node_tile_x - tile_x, - delta_y=node_tile_y - tile_y, - node_wire_in_tile_pkey=node_wire_in_tile_pkey)) + pattern = WireToNode( + wire_in_tile_pkey=wire_in_tile_pkey, + delta_x=node_tile_x - tile_x, + delta_y=node_tile_y - tile_y, + node_wire_in_tile_pkey=node_wire_in_tile_pkey) + wire_to_nodes.append(pattern) + all_wire_to_nodes.add(pattern) key = frozenset(wire_to_nodes) @@ -94,6 +98,7 @@ def main(): delta_y=wire_tile_y - tile_y, wire_in_tile_pkey=wire_in_tile_pkey)) + all_node_to_wires.add(frozenset(node_to_wires)) tile_nodes_to_wires.append( (node_wire_in_tile_pkey, frozenset(node_to_wires))) @@ -104,6 +109,10 @@ def main(): tile_nodes_to_wire_patterns[key].add(tile_pkey) + cur.execute("SELECT COUNT(pkey) FROM tile;") + num_tiles = cur.fetchone()[0] + print(len(all_wire_to_nodes), len(all_node_to_wires), num_tiles) + #wire_names = {} #for wire_in_tile_pkey, name in cur.execute("SELECT pkey, name FROM wire_in_tile;"): # wire_names[wire_in_tile_pkey] = name