Includes all deferred conversion costs in benchmarks (#34)

This refactors the helper that returns the callable to benchmark to fully expand Mapping return types. This is needed in case those return types lazily convert values from device to host, so the complete cost can be included in the measured/benchmarked run.  This is similar to exhausting returned generators that deferred computations in NetworkX, and doing this keeps us consistent with benchmarking the full compute and/or conversion costs for both libraries.

I changed the option name from `exhaust_returned_iterator` to `force_unlazy_eval` since this is no longer being used only for iterators, but I'm taking suggestions for a better name.

Authors:
  - Rick Ratzel (https://github.com/rlratzel)
  - Erik Welch (https://github.com/eriknw)

Approvers:
  - Erik Welch (https://github.com/eriknw)
  - Ralph Liu (https://github.com/nv-rliu)

URL: #34

benchmarks/nx-cugraph/pytest-based/bench_algos.py

@@ -130,12 +130,21 @@ def get_backend_wrapper(backend_name):
     "backend" kwarg on it.
     """
 
-    def wrap_callable_for_dispatch(func, exhaust_returned_iterator=False):
+    # FIXME: consider a pytest param to run force_unlazy_eval=False for
+    # benchmarks that set it to True in order to measure the time spent for any
+    # deferred computation/conversions.
+    def wrap_callable_for_dispatch(func, force_unlazy_eval=False):
+        # force_unlazy_eval=True forces iterators and other containers to
+        # generate a complete set of results in order to include any deferred
+        # compute or conversion in the benchmark
         def wrapper(*args, **kwargs):
             kwargs["backend"] = backend_name
             retval = func(*args, **kwargs)
-            if exhaust_returned_iterator:
-                retval = list(retval)
+            if force_unlazy_eval:
+                if isinstance(retval, Mapping):
+                    retval = dict(retval)
+                else:
+                    retval = list(retval)
             return retval
 
         return wrapper
@@ -440,7 +449,7 @@ def bench_shortest_path(benchmark, graph_obj, backend_wrapper):
     node = get_highest_degree_node(graph_obj)
 
     result = benchmark.pedantic(
-        target=backend_wrapper(nx.shortest_path),
+        target=backend_wrapper(nx.shortest_path, force_unlazy_eval=True),
         args=(G,),
         kwargs=dict(
             source=node,
@@ -474,7 +483,7 @@ def bench_single_target_shortest_path_length(benchmark, graph_obj, backend_wrapp
     node = get_highest_degree_node(graph_obj)
     result = benchmark.pedantic(
         target=backend_wrapper(
-            nx.single_target_shortest_path_length, exhaust_returned_iterator=True
+            nx.single_target_shortest_path_length, force_unlazy_eval=True
         ),
         args=(G,),
         kwargs=dict(
@@ -484,9 +493,11 @@ def bench_single_target_shortest_path_length(benchmark, graph_obj, backend_wrapp
         iterations=iterations,
         warmup_rounds=warmup_rounds,
     )
-    # exhaust_returned_iterator=True forces the result to a list, but is not
-    # needed for this algo in NX 3.3+ which returns a dict instead of an
-    # iterator. Forcing to a list does not change the benchmark timing.
+    # force_unlazy_eval=True forces iterators and other containers to generate
+    # a complete set of results (in order to include any deferred compute or
+    # conversion in the benchmark), but is not needed for this algo in NX 3.3+
+    # since it returns a dict instead of an iterator. Forcing eval does not
+    # change the benchmark timing.
     assert type(result) is list
 
 
@@ -525,7 +536,7 @@ def bench_generic_bfs_edges(benchmark, graph_obj, backend_wrapper):
     G = get_graph_obj_for_benchmark(graph_obj, backend_wrapper)
     node = get_highest_degree_node(graph_obj)
     result = benchmark.pedantic(
-        target=backend_wrapper(nx.generic_bfs_edges, exhaust_returned_iterator=True),
+        target=backend_wrapper(nx.generic_bfs_edges, force_unlazy_eval=True),
         args=(G,),
         kwargs=dict(
             source=node,
@@ -541,7 +552,7 @@ def bench_bfs_edges(benchmark, graph_obj, backend_wrapper):
     G = get_graph_obj_for_benchmark(graph_obj, backend_wrapper)
     node = get_highest_degree_node(graph_obj)
     result = benchmark.pedantic(
-        target=backend_wrapper(nx.bfs_edges, exhaust_returned_iterator=True),
+        target=backend_wrapper(nx.bfs_edges, force_unlazy_eval=True),
         args=(G,),
         kwargs=dict(
             source=node,
@@ -557,7 +568,7 @@ def bench_bfs_layers(benchmark, graph_obj, backend_wrapper):
     G = get_graph_obj_for_benchmark(graph_obj, backend_wrapper)
     node = get_highest_degree_node(graph_obj)
     result = benchmark.pedantic(
-        target=backend_wrapper(nx.bfs_layers, exhaust_returned_iterator=True),
+        target=backend_wrapper(nx.bfs_layers, force_unlazy_eval=True),
         args=(G,),
         kwargs=dict(
             sources=node,
@@ -573,7 +584,7 @@ def bench_bfs_predecessors(benchmark, graph_obj, backend_wrapper):
     G = get_graph_obj_for_benchmark(graph_obj, backend_wrapper)
     node = get_highest_degree_node(graph_obj)
     result = benchmark.pedantic(
-        target=backend_wrapper(nx.bfs_predecessors, exhaust_returned_iterator=True),
+        target=backend_wrapper(nx.bfs_predecessors, force_unlazy_eval=True),
         args=(G,),
         kwargs=dict(
             source=node,
@@ -589,7 +600,7 @@ def bench_bfs_successors(benchmark, graph_obj, backend_wrapper):
     G = get_graph_obj_for_benchmark(graph_obj, backend_wrapper)
     node = get_highest_degree_node(graph_obj)
     result = benchmark.pedantic(
-        target=backend_wrapper(nx.bfs_successors, exhaust_returned_iterator=True),
+        target=backend_wrapper(nx.bfs_successors, force_unlazy_eval=True),
         args=(G,),
         kwargs=dict(
             source=node,
@@ -773,7 +784,7 @@ def bench_strongly_connected_components(benchmark, graph_obj, backend_wrapper):
     G = get_graph_obj_for_benchmark(graph_obj, backend_wrapper)
     result = benchmark.pedantic(
         target=backend_wrapper(
-            nx.strongly_connected_components, exhaust_returned_iterator=True
+            nx.strongly_connected_components, force_unlazy_eval=True
         ),
         args=(G,),
         rounds=rounds,
@@ -816,9 +827,7 @@ def bench_triangles(benchmark, graph_obj, backend_wrapper):
 def bench_weakly_connected_components(benchmark, graph_obj, backend_wrapper):
     G = get_graph_obj_for_benchmark(graph_obj, backend_wrapper)
     result = benchmark.pedantic(
-        target=backend_wrapper(
-            nx.weakly_connected_components, exhaust_returned_iterator=True
-        ),
+        target=backend_wrapper(nx.weakly_connected_components, force_unlazy_eval=True),
         args=(G,),
         rounds=rounds,
         iterations=iterations,