From 51884914fa607d392110adc56ff9def47bf722d9 Mon Sep 17 00:00:00 2001 From: Keith Rothman <537074+litghost@users.noreply.github.com> Date: Wed, 4 Nov 2020 16:44:06 -0800 Subject: [PATCH] Add output of reduced graph and add checking out output. Signed-off-by: Keith Rothman <537074+litghost@users.noreply.github.com> --- minitests/graph_folding/Makefile | 38 ++- minitests/graph_folding/check_patterns.py | 139 ++++++++ minitests/graph_folding/distributed_bsc.py | 11 + minitests/graph_folding/estimate_sizes.py | 3 +- minitests/graph_folding/graph_lookup.py | 213 ++++++++++++ minitests/graph_folding/graph_storage.capnp | 52 +++ minitests/graph_folding/node_lookup.py | 6 +- minitests/graph_folding/print_tile_types.py | 32 ++ .../graph_folding/reduce_graph_for_type.py | 312 ++++++++++++++++-- minitests/graph_folding/reference_model.py | 212 ++++++++++++ minitests/graph_folding/sum_all_patterns.py | 44 +++ 11 files changed, 1030 insertions(+), 32 deletions(-) create mode 100644 minitests/graph_folding/check_patterns.py create mode 100644 minitests/graph_folding/graph_lookup.py create mode 100644 minitests/graph_folding/graph_storage.capnp create mode 100644 minitests/graph_folding/print_tile_types.py create mode 100644 minitests/graph_folding/reference_model.py create mode 100644 minitests/graph_folding/sum_all_patterns.py diff --git a/minitests/graph_folding/Makefile b/minitests/graph_folding/Makefile index adaf9bdf6..17f3666b2 100644 --- a/minitests/graph_folding/Makefile +++ b/minitests/graph_folding/Makefile @@ -1,3 +1,10 @@ +# Copyright (C) 2017-2020 The Project X-Ray Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC DB_ROOT ?= ${XRAY_DATABASE_DIR}/${XRAY_DATABASE} PART ?= ${XRAY_PART} DATABASE ?= build_${PART}/${PART}.db @@ -14,16 +21,33 @@ ${DATABASE}: build_node_lookup.py node_lookup.py | build_${PART} --part ${PART} \ ${DATABASE} -${WIRE_PATTERNS}: test_reduction.py ${DATABASE} - python3 test_reduction.py \ - --database ${DATABASE} \ - --wire_patterns ${WIRE_PATTERNS} - -.PHONY: clean database wire_patterns +.PHONY: clean database build_patterns check_patterns print_size database: ${DATABASE} -wire_patterns: ${WIRE_PATTERNS} +build_patterns: ${DATABASE} reduce_graph_for_type.py distributed_bsc.py reference_model.py + python3 print_tile_types.py | \ + /usr/bin/time -v xargs -n1 \ + /usr/bin/time -v python3 reduce_graph_for_type.py \ + --database ${DATABASE} \ + --output_dir build_${PART} \ + --node_to_wires \ + --tile &> build_${PART}/node_to_wires.log + python3 print_tile_types.py | \ + /usr/bin/time -v xargs -n1 \ + /usr/bin/time -v python3 reduce_graph_for_type.py \ + --database ${DATABASE} \ + --output_dir build_${PART} \ + --wire_to_node \ + --tile &> build_${PART}/wire_to_node.log + +check_patterns: + /usr/bin/time -v python3 check_patterns.py \ + --database ${DATABASE} \ + --output_dir build_${PART}/ + +print_size: + python3 sum_all_patterns.py --output_dir build_${PART} clean: rm -rf build_${PART} diff --git a/minitests/graph_folding/check_patterns.py b/minitests/graph_folding/check_patterns.py new file mode 100644 index 000000000..54a834b40 --- /dev/null +++ b/minitests/graph_folding/check_patterns.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (C) 2017-2020 The Project X-Ray Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + +import argparse +import capnp +import capnp.lib.capnp +capnp.remove_import_hook() +import progressbar +import os.path + +from node_lookup import NodeLookup + +from graph_lookup import WireToNodeLookup, NodeToWiresLookup + + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument('--database', required=True) + parser.add_argument('--output_dir') + + args = parser.parse_args() + + graph_storage_schema = capnp.load('graph_storage.capnp') + + def read_tile_type_w2n(tile_type): + return WireToNodeLookup( + graph_storage_schema, + os.path.join( + args.output_dir, '{}_wire_to_nodes.bin'.format(tile_type))) + + def read_tile_type_n2w(tile_type): + return NodeToWiresLookup( + graph_storage_schema, + os.path.join( + args.output_dir, '{}_node_to_wires.bin'.format(tile_type))) + + lookup = NodeLookup(database=args.database) + cur = lookup.conn.cursor() + + tile_xy_to_tile_pkey = {} + tile_pkey_to_xy = {} + tile_pkey_to_tile_type_name = {} + tile_type_to_wire_to_node_lookup = {} + tile_type_to_node_to_wires_lookup = {} + + for tile_pkey, tile_x, tile_y, tile_type_name in cur.execute(""" +SELECT tile.pkey, tile.x, tile.y, tile_type.name +FROM tile +INNER JOIN tile_type ON tile.tile_type_pkey = tile_type.pkey;"""): + tile_xy_to_tile_pkey[tile_x, tile_y] = tile_pkey + tile_pkey_to_xy[tile_pkey] = (tile_x, tile_y) + tile_pkey_to_tile_type_name[tile_pkey] = tile_type_name + + def check_node(node_pkey, current_node_elements): + cur = lookup.conn.cursor() + cur.execute( + "SELECT tile_pkey, wire_in_tile_pkey FROM node WHERE pkey = ?", + (node_pkey, )) + node_tile_pkey, node_wire_in_tile_pkey = cur.fetchone() + + node_x, node_y = tile_pkey_to_xy[node_tile_pkey] + + any_errors = False + for tile_pkey, wire_in_tile_pkey in current_node_elements: + wire_x, wire_y = tile_pkey_to_xy[tile_pkey] + tile_type = tile_pkey_to_tile_type_name[tile_pkey] + + if tile_type not in tile_type_to_wire_to_node_lookup: + tile_type_to_wire_to_node_lookup[ + tile_type] = read_tile_type_w2n(tile_type) + + lookup_node_x, lookup_node_y, lookup_node_wire_in_tile_pkey = tile_type_to_wire_to_node_lookup[ + tile_type].get_node( + tile_pkey, wire_x, wire_y, wire_in_tile_pkey) + + if (node_x, node_y, + node_wire_in_tile_pkey) != (lookup_node_x, lookup_node_y, + lookup_node_wire_in_tile_pkey): + print( + 'ERROR: For ({}, {}), db ({}, {}, {}) != lookup ({}, {}, {})' + .format( + tile_pkey, wire_in_tile_pkey, node_x, node_y, + node_wire_in_tile_pkey, lookup_node_x, lookup_node_y, + lookup_node_wire_in_tile_pkey)) + any_errors = True + + current_node_elements_set = set(current_node_elements) + + node_tile_type = tile_pkey_to_tile_type_name[node_tile_pkey] + if node_tile_type not in tile_type_to_node_to_wires_lookup: + tile_type_to_node_to_wires_lookup[ + node_tile_type] = read_tile_type_n2w(node_tile_type) + + for wire_x, wire_y, wire_in_tile_pkey in tile_type_to_node_to_wires_lookup[ + node_tile_type].get_wires_for_node( + node_tile_pkey, node_x, node_y, node_wire_in_tile_pkey): + tile_pkey = tile_xy_to_tile_pkey[wire_x, wire_y] + assert (tile_pkey, wire_in_tile_pkey) in current_node_elements_set + + # FIXME: This only confirms that all wires from the lookup are + # part of the node. This does not confirm that all wires that + # should be part of this node are. + + return any_errors + + any_errors = False + current_node_pkey = None + current_node_elements = [] + for tile_pkey, wire_in_tile_pkey, node_pkey in progressbar.progressbar( + cur.execute(""" +SELECT tile_pkey, wire_in_tile_pkey, node_pkey FROM wire ORDER BY node_pkey;""" + )): + if current_node_pkey is None: + current_node_pkey = node_pkey + elif current_node_pkey != node_pkey: + any_errors = any_errors or check_node( + current_node_pkey, current_node_elements) + current_node_elements = [] + + current_node_pkey = node_pkey + current_node_elements.append((tile_pkey, wire_in_tile_pkey)) + + any_errors = any_errors or check_node( + current_node_pkey, current_node_elements) + + assert any_errors == False + + +if __name__ == "__main__": + main() diff --git a/minitests/graph_folding/distributed_bsc.py b/minitests/graph_folding/distributed_bsc.py index fe4fa15ad..3e831028c 100644 --- a/minitests/graph_folding/distributed_bsc.py +++ b/minitests/graph_folding/distributed_bsc.py @@ -1,3 +1,14 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (C) 2017-2020 The Project X-Ray Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + import random import bitarray import multiprocessing diff --git a/minitests/graph_folding/estimate_sizes.py b/minitests/graph_folding/estimate_sizes.py index 907ae5be7..623559c3d 100644 --- a/minitests/graph_folding/estimate_sizes.py +++ b/minitests/graph_folding/estimate_sizes.py @@ -49,7 +49,8 @@ def main(): tile_type_info = db.get_tile_type(tile_type) tile_type_to_wires[tile_type] = len(tile_type_info.get_wires()) all_wires += len(tile_type_info.get_wires()) - max_wires_per_tile = max(max_wires_per_tile, len(tile_type_info.get_wires())) + max_wires_per_tile = max( + max_wires_per_tile, len(tile_type_info.get_wires())) for tile_type in sorted( tile_type_to_count, key= diff --git a/minitests/graph_folding/graph_lookup.py b/minitests/graph_folding/graph_lookup.py new file mode 100644 index 000000000..36b6eef31 --- /dev/null +++ b/minitests/graph_folding/graph_lookup.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (C) 2017-2020 The Project X-Ray Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + +from reference_model import CompactArray + + +class WireToNodeLookup(): + def __init__(self, schema, wire_to_node_patterns_fname): + with open(wire_to_node_patterns_fname, 'rb') as f: + self.wire_to_node_capnp = schema.WireToNodeStorage.read(f) + + self.wire_in_tile_pkeys = CompactArray() + self.wire_in_tile_pkeys.read_from_capnp( + self.wire_to_node_capnp.wireInTilePkeys) + + self.node_pattern_dx = CompactArray() + self.node_pattern_dx.read_from_capnp( + self.wire_to_node_capnp.nodePatternDx) + + self.node_pattern_dy = CompactArray() + self.node_pattern_dy.read_from_capnp( + self.wire_to_node_capnp.nodePatternDy) + + self.node_pattern_to_node_wire = CompactArray() + self.node_pattern_to_node_wire.read_from_capnp( + self.wire_to_node_capnp.nodePatternToNodeWire) + + assert len(self.node_pattern_dx.items) == len( + self.node_pattern_dy.items), wire_to_node_patterns_fname + assert len(self.node_pattern_dx.items) == len( + self.node_pattern_to_node_wire.items), wire_to_node_patterns_fname + + self.subgraphs = [] + for subgraph_capnp in self.wire_to_node_capnp.subgraphs: + subgraph = CompactArray() + subgraph.read_from_capnp(subgraph_capnp) + + for pattern_idx in subgraph.items: + if pattern_idx is not None: + assert pattern_idx < len(self.node_pattern_dx.items) + + assert len(subgraph.items) == len(self.wire_in_tile_pkeys.items) + + self.subgraphs.append(subgraph) + + self.tile_patterns = [] + for tile_pattern_capnp in self.wire_to_node_capnp.tilePatterns: + tile_pattern = CompactArray() + tile_pattern.read_from_capnp(tile_pattern_capnp) + + for subgraph_idx in tile_pattern.items: + assert subgraph_idx < len(self.subgraphs) + + self.tile_patterns.append(tile_pattern) + + tile_pkeys = CompactArray() + tile_pkeys.read_from_capnp(self.wire_to_node_capnp.tilePkeys) + + tile_to_tile_patterns = CompactArray() + tile_to_tile_patterns.read_from_capnp( + self.wire_to_node_capnp.tileToTilePatterns) + + self.tile_to_tile_patterns = {} + + for tile_pkey, tile_pattern_idx in zip(tile_pkeys.items, + tile_to_tile_patterns.items): + assert tile_pattern_idx < len( + self.tile_patterns), wire_to_node_patterns_fname + self.tile_to_tile_patterns[tile_pkey] = tile_pattern_idx + + def get_node(self, tile_pkey, x, y, wire_in_tile_pkey): + wire_idx = self.wire_in_tile_pkeys.index_get(wire_in_tile_pkey, None) + if wire_idx is None: + # This is already the node! + return x, y, wire_in_tile_pkey + + tile_pattern_idx = self.tile_to_tile_patterns[tile_pkey] + tile_pattern = self.tile_patterns[tile_pattern_idx] + + node_pattern_idx = None + for subgraph_idx in tile_pattern.items: + subgraph = self.subgraphs[subgraph_idx] + + if subgraph.items[wire_idx] is not None: + node_pattern_idx = subgraph.items[wire_idx] + break + + if node_pattern_idx is None: + # This is already the node! + return x, y, wire_in_tile_pkey + + node_x = x + self.node_pattern_dx.items[node_pattern_idx] + node_y = y + self.node_pattern_dy.items[node_pattern_idx] + node_wire_in_tile_pkey = self.node_pattern_to_node_wire.items[ + node_pattern_idx] + + return node_x, node_y, node_wire_in_tile_pkey + + +class NodeToWiresLookup(): + def __init__(self, schema, node_to_wires_fname): + with open(node_to_wires_fname, 'rb') as f: + self.node_to_wires_capnp = schema.NodeToWiresStorage.read(f) + + self.node_wire_in_tile_pkeys = CompactArray() + self.node_wire_in_tile_pkeys.read_from_capnp( + self.node_to_wires_capnp.nodeWireInTilePkeys) + + self.wire_pattern_dx = CompactArray() + self.wire_pattern_dx.read_from_capnp( + self.node_to_wires_capnp.wirePatternDx) + + self.wire_pattern_dy = CompactArray() + self.wire_pattern_dy.read_from_capnp( + self.node_to_wires_capnp.wirePatternDy) + + self.wire_pattern_to_wire = CompactArray() + self.wire_pattern_to_wire.read_from_capnp( + self.node_to_wires_capnp.wirePatternToWire) + + assert len(self.wire_pattern_dx.items) == len( + self.wire_pattern_dy.items), node_to_wires_fname + assert len(self.wire_pattern_dx.items) == len( + self.wire_pattern_to_wire.items), node_to_wires_fname + + self.node_patterns = [] + for node_pattern_capnp in self.node_to_wires_capnp.nodePatterns: + node_pattern = CompactArray() + node_pattern.read_from_capnp(node_pattern_capnp) + + for pattern_idx in node_pattern.items: + assert pattern_idx < len(self.wire_pattern_dx.items) + + self.node_patterns.append(node_pattern) + + self.subgraphs = [] + for subgraph_capnp in self.node_to_wires_capnp.subgraphs: + subgraph = CompactArray() + subgraph.read_from_capnp(subgraph_capnp) + + for node_pattern_idx in subgraph.items: + if node_pattern_idx is not None: + assert node_pattern_idx < len(self.node_patterns) + + assert len(subgraph.items) == len( + self.node_wire_in_tile_pkeys.items) + + self.subgraphs.append(subgraph) + + self.tile_patterns = [] + for tile_pattern_capnp in self.node_to_wires_capnp.tilePatterns: + tile_pattern = CompactArray() + tile_pattern.read_from_capnp(tile_pattern_capnp) + + for subgraph_idx in tile_pattern.items: + assert subgraph_idx < len(self.subgraphs) + + self.tile_patterns.append(tile_pattern) + + tile_pkeys = CompactArray() + tile_pkeys.read_from_capnp(self.node_to_wires_capnp.tilePkeys) + + tile_to_tile_patterns = CompactArray() + tile_to_tile_patterns.read_from_capnp( + self.node_to_wires_capnp.tileToTilePatterns) + + self.tile_to_tile_patterns = {} + + for tile_pkey, tile_pattern_idx in zip(tile_pkeys.items, + tile_to_tile_patterns.items): + assert tile_pattern_idx < len( + self.tile_patterns), node_to_wires_fname + self.tile_to_tile_patterns[tile_pkey] = tile_pattern_idx + + def get_wires_for_node(self, tile_pkey, x, y, node_wire_in_tile_pkey): + yield (x, y, node_wire_in_tile_pkey) + + node_wire_idx = self.node_wire_in_tile_pkeys.index_get( + node_wire_in_tile_pkey, None) + if node_wire_idx is None: + # This node is only itself! + return + + tile_pattern_idx = self.tile_to_tile_patterns[tile_pkey] + tile_pattern = self.tile_patterns[tile_pattern_idx] + + node_pattern_idx = None + for subgraph_idx in tile_pattern.items: + subgraph = self.subgraphs[subgraph_idx] + + if subgraph.items[node_wire_idx] is not None: + node_pattern_idx = subgraph.items[node_wire_idx] + break + + if node_pattern_idx is None: + # This node is only itself! + return + + for wire_pattern_idx in self.node_patterns[node_pattern_idx].items: + wire_x = x + self.wire_pattern_dx.items[wire_pattern_idx] + wire_y = y + self.wire_pattern_dy.items[wire_pattern_idx] + wire_in_tile_pkey = self.wire_pattern_to_wire.items[ + wire_pattern_idx] + + yield wire_x, wire_y, wire_in_tile_pkey diff --git a/minitests/graph_folding/graph_storage.capnp b/minitests/graph_folding/graph_storage.capnp new file mode 100644 index 000000000..7a89c337e --- /dev/null +++ b/minitests/graph_folding/graph_storage.capnp @@ -0,0 +1,52 @@ +@0xa4b49111bb3d0fc5; + +struct CompactArray { + storage :union { + u8 @0 : List(UInt8); + u16 @1 : List(UInt16); + u32 @2 : List(UInt32); + i8 @3 : List(Int8); + i16 @4 : List(Int16); + i32 @5 : List(Int32); + } +} + +struct WireToNodeStorage { + wireInTilePkeys @0 : CompactArray; + + # Storage of wire -> node patterns + nodePatternDx @1 : CompactArray; + nodePatternDy @2 : CompactArray; + nodePatternToNodeWire @3 : CompactArray; + + # Storage of subgraph to wire + subgraphs @4 : List(CompactArray); + + # Tile patterns + tilePatterns @5 : List(CompactArray); + + tilePkeys @6 : CompactArray; + tileToTilePatterns @7 : CompactArray; +} + +struct NodeToWiresStorage { + # Id of node wire in tile pkey + nodeWireInTilePkeys @0 : CompactArray; + + # Storage of node -> wire patterns + wirePatternDx @1 : CompactArray; + wirePatternDy @2 : CompactArray; + wirePatternToWire @3 : CompactArray; + + # Storage of node to wire pattern lists. + nodePatterns @4 : List(CompactArray); + + # Storage of subgraph to node + subgraphs @5 : List(CompactArray); + + # Tile patterns + tilePatterns @6 : List(CompactArray); + + tilePkeys @7 : CompactArray; + tileToTilePatterns @8 : CompactArray; +} diff --git a/minitests/graph_folding/node_lookup.py b/minitests/graph_folding/node_lookup.py index d0ac3d69f..2161a4270 100644 --- a/minitests/graph_folding/node_lookup.py +++ b/minitests/graph_folding/node_lookup.py @@ -107,7 +107,11 @@ def build_database(self, db, progressbar=lambda x: x): for pip in tile_type.get_pips(): c.execute( "INSERT INTO pip_in_tile(tile_type_pkey, wire0_in_tile_pkey, wire1_in_tile_pkey, is_directional) VALUES (?, ?, ?, ?);", - (tile_type_pkey, wire_in_tile_pkeys[tile_type_name, pip.net_from], wire_in_tile_pkeys[tile_type_name, pip.net_to], pip.is_directional)) + ( + tile_type_pkey, + wire_in_tile_pkeys[tile_type_name, pip.net_from], + wire_in_tile_pkeys[tile_type_name, pip.net_to], + pip.is_directional)) tile_pkeys = {} for tile_name in progressbar(grid.tiles()): diff --git a/minitests/graph_folding/print_tile_types.py b/minitests/graph_folding/print_tile_types.py new file mode 100644 index 000000000..8c4468759 --- /dev/null +++ b/minitests/graph_folding/print_tile_types.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (C) 2017-2020 The Project X-Ray Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + +import argparse + +from prjxray import util +from prjxray.db import Database + + +def main(): + parser = argparse.ArgumentParser() + util.db_root_arg(parser) + util.part_arg(parser) + + args = parser.parse_args() + + db = Database(args.db_root, args.part) + + for tile_type in sorted(db.get_tile_types()): + print(tile_type) + + +if __name__ == "__main__": + main() diff --git a/minitests/graph_folding/reduce_graph_for_type.py b/minitests/graph_folding/reduce_graph_for_type.py index b6c1ed348..387ec1cb4 100644 --- a/minitests/graph_folding/reduce_graph_for_type.py +++ b/minitests/graph_folding/reduce_graph_for_type.py @@ -21,8 +21,10 @@ greed_set_cover_par import gc import multiprocessing +from reference_model import CompactArray, StructOfArray +import os.path -from prjxray.node_lookup import NodeLookup +from node_lookup import NodeLookup Tile = namedtuple('Tile', 'tile_pkey') WireToNode = namedtuple( @@ -154,32 +156,243 @@ def get_node_to_wires_graph(database, tile_type): return graph -def main(): - multiprocessing.set_start_method('spawn') +def write_wire_to_node( + graph, required_solutions, tile_patterns, tile_to_tile_patterns, + output_dir, tile_type): + wire_in_tile_pkeys = set() + all_node_patterns = set() - parser = argparse.ArgumentParser() - parser.add_argument('--database', required=True) - parser.add_argument('--tile', required=True) - parser.add_argument('--wire_to_node', action='store_true') - parser.add_argument('--node_to_wires', action='store_true') + for pattern in graph.v: + wire_in_tile_pkeys.add(pattern.wire_in_tile_pkey) + all_node_patterns.add( + (pattern.delta_x, pattern.delta_y, pattern.node_wire_in_tile_pkey)) - args = parser.parse_args() + wire_in_tile_pkeys_data = CompactArray() + wire_in_tile_pkeys_data.set_items(sorted(wire_in_tile_pkeys)) - if args.wire_to_node and args.node_to_wires: - parser.error('Cannot supply both --wire_to_node and --node_to_wires') - elif not args.wire_to_node and not args.node_to_wires: - parser.error('Must supply --wire_to_node or --node_to_wires') + node_patterns = StructOfArray( + 'WireToNodePattern', ('delta_x', 'delta_y', 'node_wire_in_tile_pkey')) + node_patterns.set_items(sorted(all_node_patterns)) - if args.wire_to_node: - graph = get_wire_to_node_graph(args.database, args.tile) - elif args.node_to_wires: - graph = get_node_to_wires_graph(args.database, args.tile) - else: - assert False + subgraphs = [] + subgraphs_null_count = [] + subgraph_idx_to_tiles = {} - all_edges = set(graph.frozen_edges) - gc.collect() + for subgraph_idx, (tiles, patterns) in enumerate(required_solutions): + subgraph_idx_to_tiles[subgraph_idx] = tiles + + subgraph = CompactArray() + subgraph.init_items(len(wire_in_tile_pkeys)) + + for pattern in patterns: + idx = wire_in_tile_pkeys_data.index(pattern.wire_in_tile_pkey) + subgraph.items[idx] = node_patterns.index( + ( + pattern.delta_x, pattern.delta_y, + pattern.node_wire_in_tile_pkey)) + + null_count = 0 + for item in subgraph.items: + if item is None: + null_count += 1 + + subgraphs.append(subgraph) + subgraphs_null_count.append(null_count) + + tile_patterns_data = [] + tile_pattern_to_index = {} + tile_patterns = sorted(tile_patterns) + for idx, tile_pattern in enumerate(tile_patterns): + tile_pattern_to_index[tile_pattern] = idx + tile_pattern_data = CompactArray() + + # Have tile patterns put more complete subgraphs earlier than later. + tile_pattern_data.set_items( + sorted(tile_pattern, key=lambda x: subgraphs_null_count[x])) + tile_patterns_data.append(tile_pattern_data) + tile_to_tile_patterns_data = [] + for tile, tile_pattern in tile_to_tile_patterns.items(): + tile_pattern_index = tile_pattern_to_index[tile_pattern] + tile_to_tile_patterns_data.append((tile.tile_pkey, tile_pattern_index)) + + for subgraph_idx in tile_patterns[tile_pattern_index]: + assert tile in subgraph_idx_to_tiles[subgraph_idx] + + subgraph = subgraphs[subgraph_idx] + + for wire_idx, node_pattern_idx in enumerate(subgraph.items): + if node_pattern_idx is None: + continue + + wire_in_tile_pkey = wire_in_tile_pkeys_data.items[wire_idx] + pattern = node_patterns.get(node_pattern_idx) + + pattern_tup = WireToNode( + wire_in_tile_pkey=wire_in_tile_pkey, + delta_x=pattern['delta_x'], + delta_y=pattern['delta_y'], + node_wire_in_tile_pkey=pattern['node_wire_in_tile_pkey'], + ) + + assert graph.is_edge(tile, pattern_tup) + + tile_to_tile_patterns = StructOfArray( + 'TileToTilePatterns', ('tile_pkey', 'tile_pattern_index')) + tile_to_tile_patterns.set_items(sorted(tile_to_tile_patterns_data)) + + graph_storage_schema = capnp.load('graph_storage.capnp') + wire_to_nodes = graph_storage_schema.WireToNodeStorage.new_message() + + wire_in_tile_pkeys_data.write_to_capnp(wire_to_nodes.wireInTilePkeys) + node_patterns.write_to_capnp( + ( + wire_to_nodes.nodePatternDx, + wire_to_nodes.nodePatternDy, + wire_to_nodes.nodePatternToNodeWire, + )) + + subgraphs_capnp = wire_to_nodes.init('subgraphs', len(subgraphs)) + for subgraph_capnp, subgraph in zip(subgraphs_capnp, subgraphs): + subgraph.write_to_capnp(subgraph_capnp) + + tile_patterns_capnp = wire_to_nodes.init( + 'tilePatterns', len(tile_patterns_data)) + for tile_pattern_capnp, tile_pattern in zip(tile_patterns_capnp, + tile_patterns_data): + tile_pattern.write_to_capnp(tile_pattern_capnp) + + tile_to_tile_patterns.write_to_capnp( + (wire_to_nodes.tilePkeys, wire_to_nodes.tileToTilePatterns)) + + serialized = wire_to_nodes.to_bytes() + print('Size on disk: ', len(serialized)) + if output_dir: + with open(os.path.join(output_dir, + '{}_wire_to_nodes.bin'.format(tile_type)), + 'wb') as f: + f.write(serialized) + + +def write_node_to_wires( + graph, required_solutions, tile_patterns, tile_to_tile_patterns, + output_dir, tile_type): + node_wire_in_tile_pkeys = set() + all_wire_patterns = set() + + for node_wire_in_tile_pkey, node_to_wires in graph.v: + node_wire_in_tile_pkeys.add(node_wire_in_tile_pkey) + + for pattern in node_to_wires: + all_wire_patterns.add( + (pattern.delta_x, pattern.delta_y, pattern.wire_in_tile_pkey)) + + node_wire_in_tile_pkeys_array = CompactArray() + node_wire_in_tile_pkeys_array.set_items(sorted(node_wire_in_tile_pkeys)) + + wire_patterns = StructOfArray( + "NodeToWirePattern", ('delta_x', 'delta_y', 'wire_in_tile_pkey')) + wire_patterns.set_items(sorted(all_wire_patterns)) + + node_patterns_to_idx = {} + node_patterns_data = [] + for _, node_patterns in required_solutions: + for _, patterns in node_patterns: + if patterns in node_patterns_to_idx: + continue + + node_patterns_to_idx[patterns] = len(node_patterns_data) + node_patterns_data.append(CompactArray()) + + node_patterns = [] + for pattern in patterns: + key = ( + pattern.delta_x, pattern.delta_y, + pattern.wire_in_tile_pkey) + assert key in all_wire_patterns + node_patterns.append(wire_patterns.index(key)) + node_patterns_data[-1].set_items(sorted(node_patterns)) + + subgraphs = [] + subgraphs_null_count = [] + + for tile_pkeys, node_patterns in required_solutions: + subgraph = CompactArray() + subgraph.init_items(len(node_wire_in_tile_pkeys)) + + for node_wire_in_tile_pkey, node_patterns in node_patterns: + idx = node_wire_in_tile_pkeys_array.index(node_wire_in_tile_pkey) + subgraph.items[idx] = node_patterns_to_idx[node_patterns] + + null_count = 0 + for item in subgraph.items: + if item is None: + null_count += 1 + + subgraphs.append(subgraph) + subgraphs_null_count.append(null_count) + + tile_patterns_data = [] + tile_pattern_to_index = {} + for idx, tile_pattern in enumerate(tile_patterns): + tile_pattern_to_index[tile_pattern] = idx + tile_pattern_data = CompactArray() + + # Have tile patterns put more complete subgraphs earlier than later. + tile_pattern_data.set_items( + sorted(tile_pattern, key=lambda x: -subgraphs_null_count[x])) + tile_patterns_data.append(tile_pattern_data) + + tile_to_tile_patterns_data = [] + for tile, tile_pattern in tile_to_tile_patterns.items(): + tile_to_tile_patterns_data.append( + (tile.tile_pkey, tile_pattern_to_index[tile_pattern])) + + tile_to_tile_patterns = StructOfArray( + 'TileToTilePatterns', ('tile_pkey', 'tile_pattern_index')) + tile_to_tile_patterns.set_items(sorted(tile_to_tile_patterns_data)) + + graph_storage_schema = capnp.load('graph_storage.capnp') + node_to_wires = graph_storage_schema.NodeToWiresStorage.new_message() + + node_wire_in_tile_pkeys_array.write_to_capnp( + node_to_wires.nodeWireInTilePkeys) + wire_patterns.write_to_capnp( + ( + node_to_wires.wirePatternDx, + node_to_wires.wirePatternDy, + node_to_wires.wirePatternToWire, + )) + + node_patterns_capnp = node_to_wires.init( + 'nodePatterns', len(node_patterns_data)) + for node_pattern_capnp, node_pattern in zip(node_patterns_capnp, + node_patterns_data): + node_pattern.write_to_capnp(node_pattern_capnp) + + subgraphs_capnp = node_to_wires.init('subgraphs', len(subgraphs)) + for subgraph_capnp, subgraph in zip(subgraphs_capnp, subgraphs): + subgraph.write_to_capnp(subgraph_capnp) + + tile_patterns_capnp = node_to_wires.init( + 'tilePatterns', len(tile_patterns_data)) + for tile_pattern_capnp, tile_pattern in zip(tile_patterns_capnp, + tile_patterns_data): + tile_pattern.write_to_capnp(tile_pattern_capnp) + + tile_to_tile_patterns.write_to_capnp( + (node_to_wires.tilePkeys, node_to_wires.tileToTilePatterns)) + + serialized = node_to_wires.to_bytes() + print('Size on disk: ', len(serialized)) + if output_dir: + with open(os.path.join(output_dir, + '{}_node_to_wires.bin'.format(tile_type)), + 'wb') as f: + f.write(serialized) + + +def reduce_graph(args, all_edges, graph): density = graph.density() beta = .5 P = (0.6 - 0.8 * beta) * math.exp((4 + 3 * beta) * density) @@ -218,7 +431,8 @@ def main(): if node_to_wires not in node_to_wires_to_count: node_to_wires_to_count[node_to_wires] = len(node_to_wires) - max_patterns_to_node = max(max_patterns_to_node, len(node_to_wires)) + max_patterns_to_node = max( + max_patterns_to_node, len(node_to_wires)) for pattern in node_to_wires: patterns.add(pattern) tile_wire_ids.add(pattern.wire_in_tile_pkey) @@ -299,7 +513,59 @@ def get_tile_edges(): print( 'Max {} patterns'.format( max(len(patterns) for patterns in tile_to_tile_patterns.values()))) - print('Number of tile pattern elements: {}'.format(number_of_tile_pattern_elements)) + print( + 'Number of tile pattern elements: {}'.format( + number_of_tile_pattern_elements)) + + return required_solutions, tile_patterns, tile_to_tile_patterns + + +def main(): + multiprocessing.set_start_method('spawn') + + parser = argparse.ArgumentParser() + parser.add_argument('--database', required=True) + parser.add_argument('--tile', required=True) + parser.add_argument('--wire_to_node', action='store_true') + parser.add_argument('--node_to_wires', action='store_true') + parser.add_argument('--output_dir') + + args = parser.parse_args() + + if args.wire_to_node and args.node_to_wires: + parser.error('Cannot supply both --wire_to_node and --node_to_wires') + elif not args.wire_to_node and not args.node_to_wires: + parser.error('Must supply --wire_to_node or --node_to_wires') + + if args.wire_to_node: + graph = get_wire_to_node_graph(args.database, args.tile) + elif args.node_to_wires: + graph = get_node_to_wires_graph(args.database, args.tile) + else: + assert False + + print('Processing {} : {}'.format(args.database, args.tile)) + + all_edges = set(graph.frozen_edges) + gc.collect() + + if len(all_edges) != 0: + required_solutions, tile_patterns, tile_to_tile_patterns = reduce_graph( + args, all_edges, graph) + else: + required_solutions = set() + tile_patterns = set() + tile_to_tile_patterns = {} + + if args.wire_to_node: + write_wire_to_node( + graph, required_solutions, tile_patterns, tile_to_tile_patterns, + args.output_dir, args.tile) + + if args.node_to_wires: + write_node_to_wires( + graph, required_solutions, tile_patterns, tile_to_tile_patterns, + args.output_dir, args.tile) if __name__ == "__main__": diff --git a/minitests/graph_folding/reference_model.py b/minitests/graph_folding/reference_model.py new file mode 100644 index 000000000..f9ee998b9 --- /dev/null +++ b/minitests/graph_folding/reference_model.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (C) 2017-2020 The Project X-Ray Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + +from collections import namedtuple + + +def check_signed_range(min_value, max_value, bits): + """ + + Check if fits in signed range, saving 1 value for sentinel. + + >>> check_signed_range(-127, 127, 8) + True + >>> check_signed_range(-128, 128, 8) + False + >>> check_signed_range(-32767, 32767, 16) + True + >>> check_signed_range(-32768, 32768, 16) + False + + """ + assert bits / 2 == bits // 2 + return min_value >= -(2**(bits // 2) - 1) and max_value <= ( + (2**(bits // 2)) - 1) + + +def get_sentinel(attr): + if attr == 'u8': + return 2**8 - 1 + elif attr == 'u16': + return 2**16 - 1 + elif attr == 'u32': + return 2**32 - 1 + elif attr == 'i8': + return -(2**8) + elif attr == 'i16': + return -(2**15) + elif attr == 'i32': + return -(2**31) + else: + assert False, attr + + +class CompactArray(): + def __init__(self): + self.items = None + self.item_to_idx = None + + def init_items(self, number_of_items): + self.items = [None for _ in range(number_of_items)] + + def set_items(self, items): + self.items = tuple(items) + + def get(self, idx): + return self.items[idx] + + def build_index(self): + if self.item_to_idx is None: + self.item_to_idx = {} + for idx, item in enumerate(self.items): + assert item not in self.item_to_idx + self.item_to_idx[item] = idx + + def index(self, key): + self.build_index() + return self.item_to_idx[key] + + def index_get(self, key, default=None): + self.build_index() + return self.item_to_idx.get(key, default) + + def read_from_capnp(self, compact_array): + attr = compact_array.storage.which() + sentinel = get_sentinel(attr) + self.items = [ + (item if item != sentinel else None) + for item in getattr(compact_array.storage, attr) + ] + self.item_to_idx = None + + def store_data(self, attr, arr): + sentinel = get_sentinel(attr) + for idx, item in enumerate(self.items): + if item is None: + arr[idx] = sentinel + else: + arr[idx] = item + + def write_to_capnp(self, compact_array): + if len(self.items) > 0: + min_value = min(item for item in self.items if item is not None) + max_value = max(item for item in self.items if item is not None) + else: + min_value = 0 + max_value = 0 + + if min_value >= 0: + # Unsigned, saving max room for the sentinel value. + if max_value < 2**8 - 1: + attr = 'u8' + elif max_value < 2**16 - 1: + attr = 'u16' + elif max_value < 2**32 - 1: + attr = 'u32' + else: + assert False, max_value + else: + # Signed. + if check_signed_range(min_value, max_value, 8): + attr = 'i8' + elif check_signed_range(min_value, max_value, 16): + attr = 'i16' + elif check_signed_range(min_value, max_value, 32): + attr = 'i32' + else: + assert False, (min_value, max_value) + + compact_array.storage.init(attr, len(self.items)) + self.store_data(attr, getattr(compact_array.storage, attr)) + + +class StructOfArrayProxy(): + def __init__(self, struct_of_array, index): + self.struct_of_array = struct_of_array + self.index = index + + def __getitem__(self, attr): + sub_index = self.struct_of_array.field_to_index[attr] + + return self.struct_of_array.items[sub_index].items[self.index] + + def to_tuple(self): + return self.struct_of_array.namedtuple( + *( + self.struct_of_array.items[sub_index].items[self.index] + for sub_index in range(self.struct_of_array.number_of_fields))) + + def __hash__(self): + return hash(self.to_tuple()) + + def __eq__(self, other): + return self._asdict() == other._asdict() + + def _asdict(self): + return self.to_tuple()._asdict() + + +class StructOfArray(): + def __init__(self, element_name, fields): + self.namedtuple = namedtuple(element_name, ' '.join(fields)) + self.number_of_fields = len(fields) + self.field_to_index = {} + for idx, field in enumerate(fields): + self.field_to_index[field] = idx + self.items = None + self.item_to_idx = None + + def set_items(self, items): + self.items = [] + + for _ in range(self.number_of_fields): + self.items.append(CompactArray()) + self.items[-1].init_items(len(items)) + + for idx, item in enumerate(items): + assert len(item) == self.number_of_fields + + for field in range(self.number_of_fields): + self.items[field].items[idx] = item[field] + + def get(self, index): + return StructOfArrayProxy(self, index) + + def index(self, key): + if self.item_to_idx is None: + self.item_to_idx = {} + for idx in range(len(self.items[0].items)): + item = self.get(idx) + assert item not in self.item_to_idx + self.item_to_idx[item] = idx + + return self.item_to_idx[self.namedtuple(*key)] + + def read_from_capnp(self, compact_arrays): + assert len(compact_arrays) == self.number_of_fields + + item_counts = set() + self.items = [] + for compact_array in compact_arrays: + self.items.append(CompactArray()) + self.items[-1].read_from_capnp(compact_array) + + item_counts.add(len(self.items[-1].items)) + + assert len(item_counts) == 1 + + self.item_to_idx = None + + def write_to_capnp(self, compact_arrays): + assert len(compact_arrays) == self.number_of_fields + + for idx, compact_array in enumerate(compact_arrays): + self.items[idx].write_to_capnp(compact_array) diff --git a/minitests/graph_folding/sum_all_patterns.py b/minitests/graph_folding/sum_all_patterns.py new file mode 100644 index 000000000..c796faacd --- /dev/null +++ b/minitests/graph_folding/sum_all_patterns.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (C) 2017-2020 The Project X-Ray Authors. +# +# Use of this source code is governed by a ISC-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/ISC +# +# SPDX-License-Identifier: ISC + +import argparse +import os +from stat import S_ISDIR + + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument('--output_dir', required=True) + + args = parser.parse_args() + + sum_node_to_wires = 0 + sum_wire_to_nodes = 0 + + for fname in os.listdir(args.output_dir): + file_stats = os.stat(os.path.join(args.output_dir, fname)) + + if S_ISDIR(file_stats.st_mode): + continue + + if fname.endswith('_node_to_wires.bin'): + sum_node_to_wires += file_stats.st_size + + if fname.endswith('_wire_to_nodes.bin'): + sum_wire_to_nodes += file_stats.st_size + + print('Node to wires (bytes): ', sum_node_to_wires) + print('Wire to nodes (bytes): ', sum_wire_to_nodes) + + +if __name__ == '__main__': + main()