Add output of reduced graph and add checking out output.

Signed-off-by: Keith Rothman <[email protected]>

minitests/graph_folding/Makefile

@@ -1,3 +1,10 @@
+# Copyright (C) 2017-2020  The Project X-Ray Authors.
+#
+# Use of this source code is governed by a ISC-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/ISC
+#
+# SPDX-License-Identifier: ISC
 DB_ROOT ?= ${XRAY_DATABASE_DIR}/${XRAY_DATABASE}
 PART ?= ${XRAY_PART}
 DATABASE ?= build_${PART}/${PART}.db
@@ -14,16 +21,33 @@ ${DATABASE}: build_node_lookup.py node_lookup.py | build_${PART}
 		--part ${PART} \
 		${DATABASE}
 
-${WIRE_PATTERNS}: test_reduction.py ${DATABASE}
-	python3 test_reduction.py \
-		--database ${DATABASE} \
-		--wire_patterns ${WIRE_PATTERNS}
-
-.PHONY: clean database wire_patterns
+.PHONY: clean database build_patterns check_patterns print_size
 
 database: ${DATABASE}
 
-wire_patterns: ${WIRE_PATTERNS}
+build_patterns: ${DATABASE} reduce_graph_for_type.py distributed_bsc.py reference_model.py
+	python3 print_tile_types.py | \
+		/usr/bin/time -v xargs -n1 \
+			/usr/bin/time -v python3 reduce_graph_for_type.py \
+				--database ${DATABASE} \
+				--output_dir build_${PART} \
+				--node_to_wires \
+				--tile &> build_${PART}/node_to_wires.log
+	python3 print_tile_types.py | \
+		/usr/bin/time -v xargs -n1 \
+			/usr/bin/time -v python3 reduce_graph_for_type.py \
+				--database ${DATABASE} \
+				--output_dir build_${PART} \
+				--wire_to_node \
+				--tile &> build_${PART}/wire_to_node.log
+
+check_patterns:
+	/usr/bin/time -v python3 check_patterns.py \
+		--database ${DATABASE} \
+		--output_dir build_${PART}/
+
+print_size:
+	python3 sum_all_patterns.py --output_dir build_${PART}
 
 clean:
 	rm -rf build_${PART}

minitests/graph_folding/check_patterns.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2017-2020  The Project X-Ray Authors.
+#
+# Use of this source code is governed by a ISC-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/ISC
+#
+# SPDX-License-Identifier: ISC
+
+import argparse
+import capnp
+import capnp.lib.capnp
+capnp.remove_import_hook()
+import progressbar
+import os.path
+
+from node_lookup import NodeLookup
+
+from graph_lookup import WireToNodeLookup, NodeToWiresLookup
+
+
+def main():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('--database', required=True)
+    parser.add_argument('--output_dir')
+
+    args = parser.parse_args()
+
+    graph_storage_schema = capnp.load('graph_storage.capnp')
+
+    def read_tile_type_w2n(tile_type):
+        return WireToNodeLookup(
+            graph_storage_schema,
+            os.path.join(
+                args.output_dir, '{}_wire_to_nodes.bin'.format(tile_type)))
+
+    def read_tile_type_n2w(tile_type):
+        return NodeToWiresLookup(
+            graph_storage_schema,
+            os.path.join(
+                args.output_dir, '{}_node_to_wires.bin'.format(tile_type)))
+
+    lookup = NodeLookup(database=args.database)
+    cur = lookup.conn.cursor()
+
+    tile_xy_to_tile_pkey = {}
+    tile_pkey_to_xy = {}
+    tile_pkey_to_tile_type_name = {}
+    tile_type_to_wire_to_node_lookup = {}
+    tile_type_to_node_to_wires_lookup = {}
+
+    for tile_pkey, tile_x, tile_y, tile_type_name in cur.execute("""
+SELECT tile.pkey, tile.x, tile.y, tile_type.name
+FROM tile
+INNER JOIN tile_type ON tile.tile_type_pkey = tile_type.pkey;"""):
+        tile_xy_to_tile_pkey[tile_x, tile_y] = tile_pkey
+        tile_pkey_to_xy[tile_pkey] = (tile_x, tile_y)
+        tile_pkey_to_tile_type_name[tile_pkey] = tile_type_name
+
+    def check_node(node_pkey, current_node_elements):
+        cur = lookup.conn.cursor()
+        cur.execute(
+            "SELECT tile_pkey, wire_in_tile_pkey FROM node WHERE pkey = ?",
+            (node_pkey, ))
+        node_tile_pkey, node_wire_in_tile_pkey = cur.fetchone()
+
+        node_x, node_y = tile_pkey_to_xy[node_tile_pkey]
+
+        any_errors = False
+        for tile_pkey, wire_in_tile_pkey in current_node_elements:
+            wire_x, wire_y = tile_pkey_to_xy[tile_pkey]
+            tile_type = tile_pkey_to_tile_type_name[tile_pkey]
+
+            if tile_type not in tile_type_to_wire_to_node_lookup:
+                tile_type_to_wire_to_node_lookup[
+                    tile_type] = read_tile_type_w2n(tile_type)
+
+            lookup_node_x, lookup_node_y, lookup_node_wire_in_tile_pkey = tile_type_to_wire_to_node_lookup[
+                tile_type].get_node(
+                    tile_pkey, wire_x, wire_y, wire_in_tile_pkey)
+
+            if (node_x, node_y,
+                    node_wire_in_tile_pkey) != (lookup_node_x, lookup_node_y,
+                                                lookup_node_wire_in_tile_pkey):
+                print(
+                    'ERROR: For ({}, {}), db ({}, {}, {}) != lookup ({}, {}, {})'
+                    .format(
+                        tile_pkey, wire_in_tile_pkey, node_x, node_y,
+                        node_wire_in_tile_pkey, lookup_node_x, lookup_node_y,
+                        lookup_node_wire_in_tile_pkey))
+                any_errors = True
+
+        current_node_elements_set = set(current_node_elements)
+
+        node_tile_type = tile_pkey_to_tile_type_name[node_tile_pkey]
+        if node_tile_type not in tile_type_to_node_to_wires_lookup:
+            tile_type_to_node_to_wires_lookup[
+                node_tile_type] = read_tile_type_n2w(node_tile_type)
+
+        for wire_x, wire_y, wire_in_tile_pkey in tile_type_to_node_to_wires_lookup[
+                node_tile_type].get_wires_for_node(
+                    node_tile_pkey, node_x, node_y, node_wire_in_tile_pkey):
+            tile_pkey = tile_xy_to_tile_pkey[wire_x, wire_y]
+            assert (tile_pkey, wire_in_tile_pkey) in current_node_elements_set
+
+            # FIXME: This only confirms that all wires from the lookup are
+            # part of the node.  This does not confirm that all wires that
+            # should be part of this node are.
+
+        return any_errors
+
+    any_errors = False
+    current_node_pkey = None
+    current_node_elements = []
+    for tile_pkey, wire_in_tile_pkey, node_pkey in progressbar.progressbar(
+            cur.execute("""
+SELECT tile_pkey, wire_in_tile_pkey, node_pkey FROM wire ORDER BY node_pkey;"""
+                        )):
+        if current_node_pkey is None:
+            current_node_pkey = node_pkey
+        elif current_node_pkey != node_pkey:
+            any_errors = any_errors or check_node(
+                current_node_pkey, current_node_elements)
+            current_node_elements = []
+
+        current_node_pkey = node_pkey
+        current_node_elements.append((tile_pkey, wire_in_tile_pkey))
+
+    any_errors = any_errors or check_node(
+        current_node_pkey, current_node_elements)
+
+    assert any_errors == False
+
+
+if __name__ == "__main__":
+    main()

minitests/graph_folding/distributed_bsc.py

@@ -1,3 +1,14 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2017-2020  The Project X-Ray Authors.
+#
+# Use of this source code is governed by a ISC-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/ISC
+#
+# SPDX-License-Identifier: ISC
+
 import random
 import bitarray
 import multiprocessing

minitests/graph_folding/estimate_sizes.py

@@ -49,7 +49,8 @@ def main():
         tile_type_info = db.get_tile_type(tile_type)
         tile_type_to_wires[tile_type] = len(tile_type_info.get_wires())
         all_wires += len(tile_type_info.get_wires())
-        max_wires_per_tile = max(max_wires_per_tile, len(tile_type_info.get_wires()))
+        max_wires_per_tile = max(
+            max_wires_per_tile, len(tile_type_info.get_wires()))
 
     for tile_type in sorted(
             tile_type_to_count, key=