Skip to content

Commit 3295e8b

Browse files
JakeStevensJakeStevens
authored
Split neutron backend test based on executor dependency (#11934)
Summary: The current test_neutron_backend has a hard requirement on tflite due to the Executor comparing the output. Not all systems have tflite, but we would like to have a sanity check on the backend still. To achieve this, this PR splits the existing test into two files test_neutron_backend contains the tests that do not use any Executor logic (ie, no end to end comparison of outputs) test_neutron_backend_executor contains the rest of the tests. Differential Revision: D77248751
1 parent 4321307 commit 3295e8b

File tree

2 files changed

+111
-108
lines changed

2 files changed

+111
-108
lines changed

‎backends/nxp/tests/test_neutron_backend.py

Lines changed: 1 addition & 108 deletions
Original file line numberDiff line numberDiff line change
@@ -3,27 +3,8 @@
33
# This source code is licensed under the BSD-style license found in the
44
# LICENSE file in the root directory of this source tree.
55

6-
import numpy as np
7-
import torch
8-
9-
from executorch.backends.nxp.backend.edge_program_converter import (
10-
EdgeProgramToIRConverter,
11-
)
12-
from executorch.backends.nxp.backend.ir.lib.tflite.BuiltinOptions import BuiltinOptions
13-
from executorch.backends.nxp.backend.ir.lib.tflite.Model import Model
146
from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program
15-
from executorch.backends.nxp.tests.executors import (
16-
convert_run_compare,
17-
EdgeProgramExecutor,
18-
TFLiteExecutor,
19-
ToNHWCPreprocess,
20-
)
21-
from executorch.backends.nxp.tests.models import (
22-
Conv2dModule,
23-
ConvFCSoftmaxModule,
24-
LinearSoftmaxModule,
25-
)
26-
from torch.export import ExportedProgram
7+
from executorch.backends.nxp.tests.models import Conv2dModule, LinearSoftmaxModule
278

289

2910
def test_neutron_backend__single_conv_model():
@@ -66,91 +47,3 @@ def test_neutron_backend__linear_softmax_model__payload_header_formatless():
6647
assert payload[3] == 0x0 # Formatless
6748
assert all(byte == 0x0 for byte in payload[4:16]) # Aligned to 16 bytes
6849
assert payload[17] != 0x0 # Followed by non-zero content
69-
70-
71-
def test_lowered_program_and_tflite_output_match__conv2d__no_bias(mocker):
72-
converter_spy = mocker.spy(EdgeProgramToIRConverter, "convert_program")
73-
74-
model = Conv2dModule(bias=False)
75-
input_shape = (1, 4, 32, 32)
76-
77-
# Run conversion
78-
to_quantized_edge_program(model, input_shape)
79-
80-
# Capture generated model
81-
tflite_flatbuffers_model, io_formats = converter_spy.spy_return
82-
83-
tflite_model = Model.GetRootAs(tflite_flatbuffers_model)
84-
sub_graph = tflite_model.Subgraphs(0)
85-
86-
assert sub_graph.OperatorsLength() == 1
87-
assert sub_graph.Operators(0).BuiltinOptionsType() == BuiltinOptions.Conv2DOptions
88-
89-
# Capture converted program
90-
exported_program: ExportedProgram = converter_spy.call_args.args[1]
91-
92-
input_data = (
93-
(torch.randn(input_shape, dtype=torch.float32) * 50)
94-
.type(torch.int8)
95-
.detach()
96-
.numpy()
97-
)
98-
input_data_tflite = np.transpose(input_data, [0, 2, 3, 1])
99-
100-
# Execute program and TFLite model
101-
program_executor = EdgeProgramExecutor(exported_program)
102-
tflite_executor = TFLiteExecutor(model_content=tflite_flatbuffers_model)
103-
104-
output_edge = program_executor.inference(input_data)
105-
output_tflite = tflite_executor.inference(input_data_tflite)
106-
107-
output_tflite = np.transpose(output_tflite, [0, 3, 1, 2])
108-
109-
# Outputs difference is smaller than 1 (rounding error in quantization)
110-
assert np.max(np.abs(output_edge - output_tflite)) <= 1
111-
112-
113-
def test_conv_fc__lowered_program_and_tflite_output_match(mocker):
114-
converter_spy = mocker.spy(EdgeProgramToIRConverter, "convert_program")
115-
116-
model = ConvFCSoftmaxModule()
117-
input_shape = (1, 4, 5, 5)
118-
119-
# Run conversion
120-
_ = to_quantized_edge_program(model, input_shape)
121-
122-
# Capture converted program
123-
exported_program: ExportedProgram = converter_spy.call_args.args[1]
124-
125-
# Capture generated model
126-
tflite_flatbuffers_model, _ = converter_spy.spy_return
127-
128-
# No Transpose ops in produced TFLite model
129-
tflite_subgraph = Model.GetRootAs(tflite_flatbuffers_model).Subgraphs(0)
130-
131-
assert tflite_subgraph.OperatorsLength() == 3
132-
assert (
133-
tflite_subgraph.Operators(0).BuiltinOptionsType()
134-
== BuiltinOptions.Conv2DOptions
135-
)
136-
assert (
137-
tflite_subgraph.Operators(1).BuiltinOptionsType()
138-
== BuiltinOptions.ReshapeOptions
139-
)
140-
assert (
141-
tflite_subgraph.Operators(2).BuiltinOptionsType()
142-
== BuiltinOptions.FullyConnectedOptions
143-
)
144-
145-
# Verify outputs of program and TFLite model
146-
input_data = (
147-
(torch.randn(input_shape, dtype=torch.float32))
148-
.type(torch.int8)
149-
.detach()
150-
.numpy()
151-
)
152-
convert_run_compare(
153-
exported_program,
154-
input_data=input_data,
155-
tflite_input_preprocess=ToNHWCPreprocess(),
156-
)

‎backends/nxp/tests/test_neutron_backend_executor.py

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
# Copyright 2024 NXP
2+
#
3+
# This source code is licensed under the BSD-style license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
import numpy as np
7+
import torch
8+
9+
from executorch.backends.nxp.backend.edge_program_converter import (
10+
EdgeProgramToIRConverter,
11+
)
12+
from executorch.backends.nxp.backend.ir.lib.tflite.BuiltinOptions import BuiltinOptions
13+
from executorch.backends.nxp.backend.ir.lib.tflite.Model import Model
14+
from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program
15+
from executorch.backends.nxp.tests.executors import (
16+
convert_run_compare,
17+
EdgeProgramExecutor,
18+
TFLiteExecutor,
19+
ToNHWCPreprocess,
20+
)
21+
from executorch.backends.nxp.tests.models import Conv2dModule, ConvFCSoftmaxModule
22+
from torch.export import ExportedProgram
23+
24+
25+
def test_lowered_program_and_tflite_output_match__conv2d__no_bias(mocker):
26+
converter_spy = mocker.spy(EdgeProgramToIRConverter, "convert_program")
27+
28+
model = Conv2dModule(bias=False)
29+
input_shape = (1, 4, 32, 32)
30+
31+
# Run conversion
32+
to_quantized_edge_program(model, input_shape)
33+
34+
# Capture generated model
35+
tflite_flatbuffers_model, io_formats = converter_spy.spy_return
36+
37+
tflite_model = Model.GetRootAs(tflite_flatbuffers_model)
38+
sub_graph = tflite_model.Subgraphs(0)
39+
40+
assert sub_graph.OperatorsLength() == 1
41+
assert sub_graph.Operators(0).BuiltinOptionsType() == BuiltinOptions.Conv2DOptions
42+
43+
# Capture converted program
44+
exported_program: ExportedProgram = converter_spy.call_args.args[1]
45+
46+
input_data = (
47+
(torch.randn(input_shape, dtype=torch.float32) * 50)
48+
.type(torch.int8)
49+
.detach()
50+
.numpy()
51+
)
52+
input_data_tflite = np.transpose(input_data, [0, 2, 3, 1])
53+
54+
# Execute program and TFLite model
55+
program_executor = EdgeProgramExecutor(exported_program)
56+
tflite_executor = TFLiteExecutor(model_content=tflite_flatbuffers_model)
57+
58+
output_edge = program_executor.inference(input_data)
59+
output_tflite = tflite_executor.inference(input_data_tflite)
60+
61+
output_tflite = np.transpose(output_tflite, [0, 3, 1, 2])
62+
63+
# Outputs difference is smaller than 1 (rounding error in quantization)
64+
assert np.max(np.abs(output_edge - output_tflite)) <= 1
65+
66+
67+
def test_conv_fc__lowered_program_and_tflite_output_match(mocker):
68+
converter_spy = mocker.spy(EdgeProgramToIRConverter, "convert_program")
69+
70+
model = ConvFCSoftmaxModule()
71+
input_shape = (1, 4, 5, 5)
72+
73+
# Run conversion
74+
_ = to_quantized_edge_program(model, input_shape)
75+
76+
# Capture converted program
77+
exported_program: ExportedProgram = converter_spy.call_args.args[1]
78+
79+
# Capture generated model
80+
tflite_flatbuffers_model, _ = converter_spy.spy_return
81+
82+
# No Transpose ops in produced TFLite model
83+
tflite_subgraph = Model.GetRootAs(tflite_flatbuffers_model).Subgraphs(0)
84+
85+
assert tflite_subgraph.OperatorsLength() == 3
86+
assert (
87+
tflite_subgraph.Operators(0).BuiltinOptionsType()
88+
== BuiltinOptions.Conv2DOptions
89+
)
90+
assert (
91+
tflite_subgraph.Operators(1).BuiltinOptionsType()
92+
== BuiltinOptions.ReshapeOptions
93+
)
94+
assert (
95+
tflite_subgraph.Operators(2).BuiltinOptionsType()
96+
== BuiltinOptions.FullyConnectedOptions
97+
)
98+
99+
# Verify outputs of program and TFLite model
100+
input_data = (
101+
(torch.randn(input_shape, dtype=torch.float32))
102+
.type(torch.int8)
103+
.detach()
104+
.numpy()
105+
)
106+
convert_run_compare(
107+
exported_program,
108+
input_data=input_data,
109+
tflite_input_preprocess=ToNHWCPreprocess(),
110+
)

0 commit comments

Comments
 (0)