Skip to content

Commit

Permalink
2024-12-03 nightly release (4f47cc9)
Browse files Browse the repository at this point in the history
  • Loading branch information
pytorchbot committed Dec 3, 2024
1 parent e2a771d commit 727fb8e
Show file tree
Hide file tree
Showing 44 changed files with 1,878 additions and 291 deletions.
153 changes: 104 additions & 49 deletions .github/scripts/extract_benchmark_results.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,7 @@ def transform(
workflow_run_attempt: int,
job_name: str,
job_id: int,
schema_version: str,
) -> List:
"""
Transform the benchmark results into the format writable into the benchmark database
Expand All @@ -319,45 +320,91 @@ def transform(
for r in benchmark_results:
r["deviceInfo"]["device"] = job_name

# TODO (huydhn): This is the current schema of the database oss_ci_benchmark_v2,
# and I'm trying to fit ET benchmark results into it, which is kind of awkward.
# However, the schema is going to be updated soon
return [
{
# GH-info to identify where the benchmark is run
"repo": repo,
"head_branch": head_branch,
"workflow_id": workflow_run_id,
"run_attempt": workflow_run_attempt,
"job_id": job_id,
# The model
"name": f"{r['benchmarkModel']['name']} {r['benchmarkModel'].get('backend', '')}".strip(),
"dtype": (
r["benchmarkModel"]["quantization"]
if r["benchmarkModel"]["quantization"]
else "unknown"
),
# The metric value
"metric": r["metric"],
"actual": r["actualValue"],
"target": r["targetValue"],
# The device
"device": r["deviceInfo"]["device"],
"arch": r["deviceInfo"].get("os", ""),
# Not used here, just set it to something unique here
"filename": workflow_name,
"test_name": app_type,
"runner": job_name,
}
for r in benchmark_results
]
if schema_version == "v2":
# TODO (huydhn): Clean up this branch after ExecuTorch dashboard migrates to v3
return [
{
# GH-info to identify where the benchmark is run
"repo": repo,
"head_branch": head_branch,
"workflow_id": workflow_run_id,
"run_attempt": workflow_run_attempt,
"job_id": job_id,
# The model
"name": f"{r['benchmarkModel']['name']} {r['benchmarkModel'].get('backend', '')}".strip(),
"dtype": (
r["benchmarkModel"]["quantization"]
if r["benchmarkModel"]["quantization"]
else "unknown"
),
# The metric value
"metric": r["metric"],
"actual": r["actualValue"],
"target": r["targetValue"],
# The device
"device": r["deviceInfo"]["device"],
"arch": r["deviceInfo"].get("os", ""),
# Not used here, just set it to something unique here
"filename": workflow_name,
"test_name": app_type,
"runner": job_name,
}
for r in benchmark_results
]
elif schema_version == "v3":
quantization = (
r["benchmarkModel"]["quantization"]
if r["benchmarkModel"]["quantization"]
else "unknown"
)
# From https://github.com/pytorch/pytorch/wiki/How-to-integrate-with-PyTorch-OSS-benchmark-database
return [
{
"benchmark": {
"name": "ExecuTorch",
"mode": "inference",
"dtype": quantization,
"extra_info": {
"app_type": app_type,
},
},
"model": {
"name": r["benchmarkModel"]["name"],
"type": "OSS model",
"backend": r["benchmarkModel"].get("backend", ""),
"extra_info": {
"quantization": quantization,
},
},
"metric": {
"name": r["metric"],
"benchmark_values": [r["actualValue"]],
"target_value": r["targetValue"],
"extra_info": {
"method": r.get("method", ""),
},
},
"runners": [
{
"name": r["deviceInfo"]["device"],
"type": r["deviceInfo"]["os"],
"avail_mem_in_gb": r["deviceInfo"].get("availMem", ""),
"total_mem_in_gb": r["deviceInfo"].get("totalMem", ""),
}
],
}
for r in benchmark_results
]


def main() -> None:
args = parse_args()

# Across all devices
all_benchmark_results = []
# Across all devices, keeping both schemas for now until ExecuTorch dashboard migrates to v3
all_benchmark_results = {
"v2": [],
"v3": [],
}

with open(args.artifacts) as f:
for artifact in json.load(f):
Expand All @@ -384,23 +431,31 @@ def main() -> None:
)

if benchmark_results:
benchmark_results = transform(
app_type,
benchmark_results,
args.repo,
args.head_branch,
args.workflow_name,
args.workflow_run_id,
args.workflow_run_attempt,
job_name,
extract_job_id(args.artifacts),
)
all_benchmark_results.extend(benchmark_results)
for schema in all_benchmark_results.keys():
results = transform(
app_type,
benchmark_results,
args.repo,
args.head_branch,
args.workflow_name,
args.workflow_run_id,
args.workflow_run_attempt,
job_name,
extract_job_id(args.artifacts),
schema,
)
all_benchmark_results[schema].extend(results)

for schema in all_benchmark_results.keys():
if not all_benchmark_results.get(schema):
continue

output_dir = os.path.join(args.output_dir, schema)
os.mkdir(output_dir)

if all_benchmark_results:
output_file = os.path.basename(args.artifacts)
with open(f"{args.output_dir}/{output_file}", "w") as f:
json.dump(all_benchmark_results, f)
with open(f"{output_dir}/{output_file}", "w") as f:
json.dump(all_benchmark_results[schema], f)


if __name__ == "__main__":
Expand Down
24 changes: 17 additions & 7 deletions .github/workflows/android-perf.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -298,15 +298,25 @@ jobs:
--workflow-run-attempt ${{ github.run_attempt }}
done
ls -lah benchmark-results
for BENCHMARK_RESULTS in benchmark-results/*.json; do
cat "${BENCHMARK_RESULTS}"
echo
for SCHEMA in v2 v3; do
for BENCHMARK_RESULTS in benchmark-results/"${SCHEMA}"/*.json; do
cat "${BENCHMARK_RESULTS}"
echo
done
done
- name: Upload the benchmark results
# TODO (huydhn): Remove v2 schema once the benchmark dashboard finishes the migration
- name: Upload the benchmark results (v2)
uses: pytorch/test-infra/.github/actions/upload-benchmark-results@main
with:
benchmark-results-dir: benchmark-results/v2
dry-run: false
schema-version: v2

- name: Upload the benchmark results (v3)
uses: pytorch/test-infra/.github/actions/upload-benchmark-results@main
with:
benchmark-results-dir: 'benchmark-results'
benchmark-results-dir: benchmark-results/v3
dry-run: false
schema-version: v3
github-token: ${{ secrets.GITHUB_TOKEN }}
24 changes: 17 additions & 7 deletions .github/workflows/apple-perf.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -372,15 +372,25 @@ jobs:
--workflow-run-attempt ${{ github.run_attempt }}
done
ls -lah benchmark-results
for BENCHMARK_RESULTS in benchmark-results/*.json; do
cat "${BENCHMARK_RESULTS}"
echo
for SCHEMA in v2 v3; do
for BENCHMARK_RESULTS in benchmark-results/"${SCHEMA}"/*.json; do
cat "${BENCHMARK_RESULTS}"
echo
done
done
- name: Upload the benchmark results
# TODO (huydhn): Remove v2 schema once the benchmark dashboard finishes the migration
- name: Upload the benchmark results (v2)
uses: pytorch/test-infra/.github/actions/upload-benchmark-results@main
with:
benchmark-results-dir: benchmark-results/v2
dry-run: false
schema-version: v2

- name: Upload the benchmark results (v3)
uses: pytorch/test-infra/.github/actions/upload-benchmark-results@main
with:
benchmark-results-dir: 'benchmark-results'
benchmark-results-dir: benchmark-results/v3
dry-run: false
schema-version: v3
github-token: ${{ secrets.GITHUB_TOKEN }}
31 changes: 15 additions & 16 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,21 @@ if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE Debug)
endif()

# Setup RPATH.
# See https://gitlab.kitware.com/cmake/community/-/wikis/doc/cmake/RPATH-handling
# Use separate rpaths during build and install phases
set(CMAKE_SKIP_BUILD_RPATH OFF)
# Don't use the install-rpath during the build phase
set(CMAKE_BUILD_WITH_INSTALL_RPATH ON)
# Automatically add all linked folders that are NOT in the build directory to
# the rpath (per library?)
# TODO: Doesn't work for us right now because we are not installing .so's into the
# correct locations. For example we have libcustom_ops_aot_lib.so depending on
# _portable_lib.so, which was eventually put under <site-packages>/executorch/extension/pybindings/
# but this rpath is not automatically added because at build time it seems `portable_lib`
# is being built under the same directory, so no extra rpath is being added. To
# properly fix this we need to install `portable_lib` into the correct path.
set(CMAKE_INSTALL_RPATH_USE_LINK_PATH ON)
# ------------------------------ OPTIONS -------------------------------------
# WARNING: Please don't add example specific options in this CMakeLists.txt.
# Instead please use `find_package(executorch REQUIRED)` in the example
Expand Down Expand Up @@ -682,22 +697,6 @@ if(EXECUTORCH_BUILD_PTHREADPOOL
endif()

if(EXECUTORCH_BUILD_PYBIND)
# Setup RPATH.
# See https://gitlab.kitware.com/cmake/community/-/wikis/doc/cmake/RPATH-handling
if(APPLE)
set(CMAKE_MACOSX_RPATH ON)
set(_rpath_portable_origin "@loader_path")
else()
set(_rpath_portable_origin $ORIGIN)
endif(APPLE)
# Use separate rpaths during build and install phases
set(CMAKE_SKIP_BUILD_RPATH FALSE)
# Don't use the install-rpath during the build phase
set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)
set(CMAKE_INSTALL_RPATH "${_rpath_portable_origin}")
# Automatically add all linked folders that are NOT in the build directory to
# the rpath (per library?)
set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/third-party/pybind11)

if(NOT EXECUTORCH_BUILD_EXTENSION_DATA_LOADER)
Expand Down
4 changes: 4 additions & 0 deletions backends/arm/_passes/arm_pass_manager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@
ScalarsToAttributePass,
)
from executorch.backends.arm._passes.size_adjust_conv2d_pass import SizeAdjustConv2DPass
from executorch.backends.arm._passes.unsqueeze_before_repeat_pass import (
UnsqueezeBeforeRepeatPass,
)
from executorch.backends.arm._passes.unsqueeze_scalar_placeholders_pass import (
UnsqueezeScalarPlaceholdersPass,
)
Expand All @@ -66,6 +69,7 @@ def transform_to_backend_pipeline(
self.add_pass(RemoveClonePass())
self.add_pass(ConvertExpandCopyToRepeatPass())
self.add_pass(DecomposeLayerNormPass())
self.add_pass(UnsqueezeBeforeRepeatPass())
self.add_pass(DecomposeVarPass())
self.add_pass(ConvertMeanDimToAveragePool())
self.add_pass(DecomposeMeanDimPass())
Expand Down
62 changes: 62 additions & 0 deletions backends/arm/_passes/unsqueeze_before_repeat_pass.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
# Copyright 2024 Arm Limited and/or its affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
# pyre-unsafe
import torch
import torch.fx
from executorch.backends.arm._passes.arm_pass_utils import (
create_node,
get_first_fake_tensor,
)
from executorch.exir.dialects._ops import ops as exir_ops
from executorch.exir.pass_base import ExportPass, PassResult


class UnsqueezeBeforeRepeatPass(ExportPass):
"""
A TOSA TILE op only supports rank(in) == rank(out).
To support Pytorch's repeat which can also add dimensions,
we add an explicit view op before which adds the new dimensions.
New dimensions are appendend at the front, see
https://pytorch.org/docs/stable/generated/torch.Tensor.expand.html
Original:
repeat(multiples)
After pass:
view(shape = [1]*num_new_dims + old_shape)
repeat(multiples)
"""

def call(self, graph_module: torch.fx.GraphModule):
modified_graph = False
for node in graph_module.graph.nodes:
if node.op != "call_function":
continue
if node.target != exir_ops.edge.aten.repeat.default:
continue

old_shape = list(get_first_fake_tensor(node.all_input_nodes[0]).shape)
old_rank = len(old_shape)
multiples = node.args[1]
new_rank = len(multiples)
if old_rank == new_rank:
continue

num_new_dims = new_rank - old_rank
new_shape = [1] * num_new_dims + old_shape

with graph_module.graph.inserting_before(node):
view_node = create_node(
graph_module.graph,
exir_ops.edge.aten.view_copy.default,
(node.all_input_nodes[0], new_shape),
)
node.replace_input_with(node.all_input_nodes[0], view_node)
modified_graph = True

if modified_graph:
graph_module.recompile()
graph_module = super().call(graph_module).graph_module
return PassResult(graph_module, modified_graph)
Loading

0 comments on commit 727fb8e

Please sign in to comment.