[Backend Tester] Add backend operator test suite skeleton

Author: GregoryComer

backends/test/operators/__init__.py renamed to backends/test/facto/__init__.py

@@ -366,11 +366,11 @@ def _assert_outputs_equal(model_output, ref_output, atol=1e-03, rtol=1e-03):
                     f"Output {i} does not match reference output.\n"
                     f"\tGiven atol: {atol}, rtol: {rtol}.\n"
                     f"\tOutput tensor shape: {model.shape}, dtype: {model.dtype}\n"
-                    f"\tDifference: max: {torch.max(model-ref)}, abs: {torch.max(torch.abs(model-ref))}, mean abs error: {torch.mean(torch.abs(model-ref))}.\n"
+                    f"\tDifference: max: {torch.max(model-ref)}, abs: {torch.max(torch.abs(model-ref))}, mean abs error: {torch.mean(torch.abs(model-ref).to(torch.double))}.\n"
                     f"\t-- Model vs. Reference --\n"
                     f"\t Numel: {model.numel()}, {ref.numel()}\n"
                     f"\tMedian: {model.median()}, {ref.median()}\n"
-                    f"\t  Mean: {model.mean()}, {ref.mean()}\n"
+                    f"\t  Mean: {model.to(torch.double).mean()}, {ref.to(torch.double).mean()}\n"
                     f"\t   Max: {model.max()}, {ref.max()}\n"
                     f"\t   Min: {model.min()}, {ref.min()}\n"
                 )

backends/test/operators/facto_specs.py renamed to backends/test/facto/facto_specs.py

@@ -0,0 +1,16 @@
+add_executable(executorch-test-runner
+    test_runner.cpp
+    # TODO
+    ../../../runtime/platform/runtime.cpp
+)
+
+target_link_libraries(
+    executorch-test-runner
+    PRIVATE executorch
+    gflags
+    extension_flat_tensor
+    extension_flat_tensor_serialize
+    extension_module
+    extension_tensor
+    optimized_native_cpu_ops_lib
+    xnnpack_backend)

backends/test/operators/test_facto.py renamed to backends/test/facto/test_facto.py

@@ -0,0 +1,260 @@
+#include <executorch/extension/data_loader/file_data_loader.h>
+#include <executorch/extension/flat_tensor/flat_tensor_data_map.h>
+#include <executorch/extension/flat_tensor/serialize/serialize.h>
+#include <executorch/extension/module/module.h>
+#include <executorch/extension/tensor/tensor.h>
+#include <executorch/runtime/platform/runtime.h>
+
+#include <iostream>
+#include <map>
+#include <optional>
+#include <tuple>
+#include <vector>
+
+#include <gflags/gflags.h>
+
+/*
+ * This runner is intended to built and run as part of the backend test flow. It takes a
+ * set of inputs from a flat_tensor-format file, runs each case, and then serializes the
+ * outputs to a file, also in flat_tensor format.
+ */
+ 
+DEFINE_string(
+    model_path,
+    "model.pte",
+    "Model serialized in flatbuffer format.");
+
+DEFINE_string(
+    input_path,
+    "inputs.ptd",
+    "Input tensors in flat tensor (ptd) format.");
+
+DEFINE_string(
+    output_path,
+    "outputs.ptd",
+    "Path to write output tensor in flat tensor (ptd) format.");
+
+DEFINE_string(
+    method,
+    "forward",
+    "The model method to run.");
+
+using executorch::aten::Tensor;
+using executorch::runtime::Error;
+using executorch::runtime::EValue;
+using executorch::runtime::Result;
+using executorch::extension::FileDataLoader;
+using executorch::extension::FlatTensorDataMap;
+using executorch::extension::Module;
+using executorch::extension::TensorPtr;
+using executorch::ET_RUNTIME_NAMESPACE::TensorLayout;
+
+// Contains method inputs for a single run.
+struct TestCase {
+  std::map<int, TensorPtr> inputs;
+};
+
+std::map<std::string, TestCase> collect_test_cases(FlatTensorDataMap& input_map);
+TensorPtr create_tensor(TensorLayout& layout, std::unique_ptr<char[], decltype(&free)> buffer);
+Result<FlatTensorDataMap> load_input_data(FileDataLoader& loader);
+std::optional<std::tuple<std::string, int>> parse_key(const std::string& key);
+Result<std::vector<EValue>> run_test_case(Module& module, TestCase& test_case);
+void store_outputs(std::map<std::string, TensorPtr>& output_map, const std::string& case_name, const std::vector<EValue>& outputs);
+
+const int TensorAlignment = 16;
+
+int main(int argc, char** argv){
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  executorch::runtime::runtime_init();
+
+  // Load the model.
+  Module model(FLAGS_model_path.c_str());
+  auto load_method_error = model.load_method(FLAGS_method.c_str());
+  if (load_method_error != Error::Ok) {
+      std::cerr << "Failed to load method \"" << FLAGS_method << "\": " << static_cast<int>(load_method_error) << std::endl;
+      return -1;
+  }
+  
+  // Load the input tensor data. Note that the data loader has to live as long as the flat
+  // tensor data map does.
+  auto input_loader_result = FileDataLoader::from(FLAGS_input_path.c_str());
+  if (!input_loader_result.ok()) {
+    std::cerr << "Failed to open input file: error " << static_cast<int>(input_loader_result.error()) << std::endl;
+  }
+
+  auto load_result = load_input_data(*input_loader_result);
+  if (!load_result.ok()) {
+    return -1;   
+  }
+  auto input_map = std::move(load_result.get());
+
+  auto cases = collect_test_cases(input_map);
+  std::map<std::string, TensorPtr> output_map;
+
+  // Run each case and store the outputs.
+  for (auto& [name, test_case] : cases) {
+    auto result = run_test_case(model, test_case);
+    if (!result.ok()) {
+      std::cerr << "Failed to run test case \"" << name << "\": " << static_cast<int>(result.error()) << std::endl;
+      return -1;
+    }
+
+    store_outputs(output_map, name, result.get());
+  }
+
+  // Create a map of Tensor (unowned), rather than TensorPtr (owned).
+  std::map<std::string, Tensor> output_map_tensors;
+  for (auto& [key, value] : output_map) {
+    output_map_tensors.emplace(key, *value);
+  }
+
+  // Write the output data in .ptd format.
+  auto save_result = executorch::extension::flat_tensor::save_ptd(
+    FLAGS_output_path.c_str(),
+    output_map_tensors,
+    TensorAlignment
+  );
+
+  if (save_result != Error::Ok) {
+    std::cerr << "Failed to save outputs: " << static_cast<int>(save_result) << std::endl;
+    return -1;
+  }
+
+  std::cout << "Successfully wrote output tensors to " << FLAGS_output_path << "." << std::endl; 
+}
+
+// Group inputs by test case and build tensors.
+std::map<std::string, TestCase> collect_test_cases(FlatTensorDataMap& input_map) {
+  std::map<std::string, TestCase> cases;
+
+  for (auto i = 0u; i < input_map.get_num_keys().get(); i++) {
+    auto key = input_map.get_key(i).get();
+
+    // Split key into test_case : input index
+    auto [test_case_name, input_index] = *parse_key(key);
+
+    // Get or create the test case instance.
+    auto& test_case = cases[test_case_name];
+    
+    // Create a tensor from the layout and data.
+    auto tensor_layout = input_map.get_tensor_layout(key).get();
+    auto tensor_data = std::unique_ptr<char[], decltype(&free)>((char*) malloc(tensor_layout.nbytes()), free);
+    auto load_result = input_map.load_data_into(key, tensor_data.get(), tensor_layout.nbytes());
+    if (load_result != Error::Ok) {
+      std::cerr << "Load failed: " << static_cast<int>(load_result) << std::endl;
+      exit(-1);
+    }
+
+    auto input_tensor = create_tensor(tensor_layout, std::move(tensor_data));
+    test_case.inputs[input_index] = std::move(input_tensor);
+  }
+
+  return cases;
+}
+
+// Create a tensor from a layout and data blob.
+TensorPtr create_tensor(TensorLayout& layout, std::unique_ptr<char[], decltype(&free)> buffer) {
+  // Sizes and dim order are have different types in TensorLayout vs Tensor.
+  std::vector<executorch::aten::SizesType> sizes;
+  for (auto x : layout.sizes()) {
+    sizes.push_back(x);
+  }
+  std::vector<executorch::aten::DimOrderType> dim_order;
+  for (auto x : layout.dim_order()) {
+    dim_order.push_back(x);
+  }
+
+  auto raw_data = buffer.release();
+
+  return executorch::extension::make_tensor_ptr(
+    sizes,
+    raw_data,
+    dim_order,
+    {}, // Strides - infer from sizes + dim order.
+    layout.scalar_type(),
+    exec_aten::TensorShapeDynamism::STATIC,
+    [](void* ptr) {
+      free(ptr);
+    }
+  );
+}
+
+// Load the input data (in .ptd file format) from the given path.
+Result<FlatTensorDataMap> load_input_data(FileDataLoader& loader) {
+  auto input_data_map_load_result = FlatTensorDataMap::load(&loader);
+  if (!input_data_map_load_result.ok()) {
+    std::cerr << "Failed to open load input data map: error " << static_cast<int>(input_data_map_load_result.error()) << std::endl;
+  }
+
+  return input_data_map_load_result;
+}
+
+// Parse a string key of the form "test_case:input index". Returns a tuple of the test case name
+// and input index.
+std::optional<std::tuple<std::string, int>> parse_key(const std::string& key) {
+  auto delimiter = key.find(":");
+  if (delimiter == std::string::npos) { return std::nullopt; }
+
+  auto test_case = key.substr(0, delimiter);
+  auto index_str = key.substr(delimiter + 1);
+  auto index = std::stoi(index_str);
+
+  return {{ test_case, index }};
+}
+
+// Run a given test case and return the resulting output values.
+Result<std::vector<EValue>> run_test_case(Module& module, TestCase& test_case) {
+  for (auto& [index, value] : test_case.inputs) {
+    auto set_input_error = module.set_input(FLAGS_method, value, index);
+    if (set_input_error != Error::Ok) {
+      std::cerr << "Failed to set input " << index << ": " << static_cast<int>(set_input_error) << "." << std::endl;
+    }
+  }
+
+  return module.execute(FLAGS_method.c_str());
+}
+
+// Store output tensors into the named data map.
+void store_outputs(
+    std::map<std::string, TensorPtr>& output_map, 
+    const std::string& case_name, 
+    const std::vector<EValue>& outputs) {
+  // Because the outputs are likely memory planned, we need to clone the tensor
+  // here to avoid having the data clobbered by the next run.
+  
+  for (auto i = 0u; i < outputs.size(); i++) {
+    if (!outputs[i].isTensor()) {
+      continue;
+    }
+
+    auto key_name = case_name + ":" + std::to_string(i);
+    auto& tensor = outputs[i].toTensor();
+    
+    // Copy tensor storage.
+    auto tensor_memory = malloc(tensor.nbytes());
+    memcpy(tensor_memory, tensor.const_data_ptr(), tensor.nbytes());
+
+    // Copy tensor metadata.
+    std::vector<executorch::aten::SizesType> sizes(
+      tensor.sizes().begin(),
+      tensor.sizes().end()
+    );
+
+    std::vector<executorch::aten::DimOrderType> dim_order(
+      tensor.dim_order().begin(),
+      tensor.dim_order().end()
+    );
+
+    output_map.emplace(key_name, executorch::extension::make_tensor_ptr(
+      sizes,
+      tensor_memory,
+      dim_order,
+      {}, // Strides - implicit
+      tensor.scalar_type(),
+      exec_aten::TensorShapeDynamism::STATIC,
+      [](void* ptr) {
+        free(ptr);
+      }
+    ));
+  }
+}

backends/test/harness/tester.py

@@ -0,0 +1,15 @@
+# Backend Test Suite
+
+This directory contains tests that validate correctness and coverage of backends. These tests are written such that the backend is treated as a black box. The test logic verifies that the backend is able to handle a given pattern without erroring out (not partitioning is fine) and is able to run the graphs and yield reasonable outputs. As backends may differ significantly in implementation, numerical bounds are intentionally left loose.
+
+## Backend Registration
+
+To plug into the test framework, each backend should provide an implementation of the Tester class, defined in backends/test/harness/tester.py. Backends can provide implementations of each stage, or use the default implementation, as appropriate.
+
+At a minimum, the backend will likely need to provide a custom implementation of the Partition and ToEdgeTransformAndLower stages using the appropriate backend partitioner. See backends/xnnpack/test/tester/tester.py for an example implementation.
+
+Once a tester is available, the backend flow(s) can be added in __init__.py in this directory by adding an entry to `ALL_TESTER_FLOWS`. Each flow entry consists of a name (used in the test case naming) and a function to instantiate a tester for a given model and input tuple.
+
+## Test Cases
+
+Operator test cases are defined under the operators/ directory. Tests are written in a backend-independent manner, and each test is programmatically expanded to generate a variant for each registered backend flow. The `@operator_test` decorator is applied to each test class to trigger this behavior. Tests can also be tagged with an appropriate type specifier, such as `@dtype_test`, to generate variants for each dtype. The decorators and "magic" live in __init__.py in this directory.

backends/test/runner/CMakeLists.txt

@@ -0,0 +1,3 @@
+load(":targets.bzl", "define_common_targets")
+
+define_common_targets(is_fbcode = True)