Skip to content

Commit d245519

Browse files
JakeStevensfacebook-github-bot
JakeStevens
authored and
facebook-github-bot
committed
Split neutron backend test based on executor dependency
Summary: The current test_neutron_backend has a hard requirement on tflite due to the Executor comparing the output. Not all systems have tflite, but we would like to have a sanity check on the backend still. To achieve this, this PR splits the existing test into two files test_neutron_backend contains the tests that do not use any Executor logic (ie, no end to end comparison of outputs) test_neutron_backend_executor contains the rest of the tests. Differential Revision: D77248751
1 parent 91c9ffa commit d245519

File tree

2 files changed

+114
-108
lines changed

2 files changed

+114
-108
lines changed

‎backends/nxp/tests/test_neutron_backend.py

Lines changed: 1 addition & 108 deletions
Original file line numberDiff line numberDiff line change
@@ -3,27 +3,8 @@
33
# This source code is licensed under the BSD-style license found in the
44
# LICENSE file in the root directory of this source tree.
55

6-
import numpy as np
7-
import torch
8-
9-
from executorch.backends.nxp.backend.edge_program_converter import (
10-
EdgeProgramToIRConverter,
11-
)
12-
from executorch.backends.nxp.backend.ir.lib.tflite.BuiltinOptions import BuiltinOptions
13-
from executorch.backends.nxp.backend.ir.lib.tflite.Model import Model
146
from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program
15-
from executorch.backends.nxp.tests.executors import (
16-
convert_run_compare,
17-
EdgeProgramExecutor,
18-
TFLiteExecutor,
19-
ToNHWCPreprocess,
20-
)
21-
from executorch.backends.nxp.tests.models import (
22-
Conv2dModule,
23-
ConvFCSoftmaxModule,
24-
LinearSoftmaxModule,
25-
)
26-
from torch.export import ExportedProgram
7+
from executorch.backends.nxp.tests.models import Conv2dModule, LinearSoftmaxModule
278

289

2910
def test_neutron_backend__single_conv_model():
@@ -66,91 +47,3 @@ def test_neutron_backend__linear_softmax_model__payload_header_formatless():
6647
assert payload[3] == 0x0 # Formatless
6748
assert all(byte == 0x0 for byte in payload[4:16]) # Aligned to 16 bytes
6849
assert payload[17] != 0x0 # Followed by non-zero content
69-
70-
71-
def test_lowered_program_and_tflite_output_match__conv2d__no_bias(mocker):
72-
converter_spy = mocker.spy(EdgeProgramToIRConverter, "convert_program")
73-
74-
model = Conv2dModule(bias=False)
75-
input_shape = (1, 4, 32, 32)
76-
77-
# Run conversion
78-
to_quantized_edge_program(model, input_shape)
79-
80-
# Capture generated model
81-
tflite_flatbuffers_model, io_formats = converter_spy.spy_return
82-
83-
tflite_model = Model.GetRootAs(tflite_flatbuffers_model)
84-
sub_graph = tflite_model.Subgraphs(0)
85-
86-
assert sub_graph.OperatorsLength() == 1
87-
assert sub_graph.Operators(0).BuiltinOptionsType() == BuiltinOptions.Conv2DOptions
88-
89-
# Capture converted program
90-
exported_program: ExportedProgram = converter_spy.call_args.args[1]
91-
92-
input_data = (
93-
(torch.randn(input_shape, dtype=torch.float32) * 50)
94-
.type(torch.int8)
95-
.detach()
96-
.numpy()
97-
)
98-
input_data_tflite = np.transpose(input_data, [0, 2, 3, 1])
99-
100-
# Execute program and TFLite model
101-
program_executor = EdgeProgramExecutor(exported_program)
102-
tflite_executor = TFLiteExecutor(model_content=tflite_flatbuffers_model)
103-
104-
output_edge = program_executor.inference(input_data)
105-
output_tflite = tflite_executor.inference(input_data_tflite)
106-
107-
output_tflite = np.transpose(output_tflite, [0, 3, 1, 2])
108-
109-
# Outputs difference is smaller than 1 (rounding error in quantization)
110-
assert np.max(np.abs(output_edge - output_tflite)) <= 1
111-
112-
113-
def test_conv_fc__lowered_program_and_tflite_output_match(mocker):
114-
converter_spy = mocker.spy(EdgeProgramToIRConverter, "convert_program")
115-
116-
model = ConvFCSoftmaxModule()
117-
input_shape = (1, 4, 5, 5)
118-
119-
# Run conversion
120-
_ = to_quantized_edge_program(model, input_shape)
121-
122-
# Capture converted program
123-
exported_program: ExportedProgram = converter_spy.call_args.args[1]
124-
125-
# Capture generated model
126-
tflite_flatbuffers_model, _ = converter_spy.spy_return
127-
128-
# No Transpose ops in produced TFLite model
129-
tflite_subgraph = Model.GetRootAs(tflite_flatbuffers_model).Subgraphs(0)
130-
131-
assert tflite_subgraph.OperatorsLength() == 3
132-
assert (
133-
tflite_subgraph.Operators(0).BuiltinOptionsType()
134-
== BuiltinOptions.Conv2DOptions
135-
)
136-
assert (
137-
tflite_subgraph.Operators(1).BuiltinOptionsType()
138-
== BuiltinOptions.ReshapeOptions
139-
)
140-
assert (
141-
tflite_subgraph.Operators(2).BuiltinOptionsType()
142-
== BuiltinOptions.FullyConnectedOptions
143-
)
144-
145-
# Verify outputs of program and TFLite model
146-
input_data = (
147-
(torch.randn(input_shape, dtype=torch.float32))
148-
.type(torch.int8)
149-
.detach()
150-
.numpy()
151-
)
152-
convert_run_compare(
153-
exported_program,
154-
input_data=input_data,
155-
tflite_input_preprocess=ToNHWCPreprocess(),
156-
)

‎backends/nxp/tests/test_neutron_backend_executor.py

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
# Copyright 2024 NXP
2+
#
3+
# This source code is licensed under the BSD-style license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
import numpy as np
7+
import torch
8+
9+
from executorch.backends.nxp.backend.edge_program_converter import (
10+
EdgeProgramToIRConverter,
11+
)
12+
from executorch.backends.nxp.backend.ir.lib.tflite.BuiltinOptions import BuiltinOptions
13+
from executorch.backends.nxp.backend.ir.lib.tflite.Model import Model
14+
from executorch.backends.nxp.tests.executorch_pipeline import to_quantized_edge_program
15+
from executorch.backends.nxp.tests.executors import (
16+
convert_run_compare,
17+
EdgeProgramExecutor,
18+
TFLiteExecutor,
19+
ToNHWCPreprocess,
20+
)
21+
from executorch.backends.nxp.tests.models import (
22+
Conv2dModule,
23+
ConvFCSoftmaxModule,
24+
)
25+
from torch.export import ExportedProgram
26+
27+
28+
def test_lowered_program_and_tflite_output_match__conv2d__no_bias(mocker):
29+
converter_spy = mocker.spy(EdgeProgramToIRConverter, "convert_program")
30+
31+
model = Conv2dModule(bias=False)
32+
input_shape = (1, 4, 32, 32)
33+
34+
# Run conversion
35+
to_quantized_edge_program(model, input_shape)
36+
37+
# Capture generated model
38+
tflite_flatbuffers_model, io_formats = converter_spy.spy_return
39+
40+
tflite_model = Model.GetRootAs(tflite_flatbuffers_model)
41+
sub_graph = tflite_model.Subgraphs(0)
42+
43+
assert sub_graph.OperatorsLength() == 1
44+
assert sub_graph.Operators(0).BuiltinOptionsType() == BuiltinOptions.Conv2DOptions
45+
46+
# Capture converted program
47+
exported_program: ExportedProgram = converter_spy.call_args.args[1]
48+
49+
input_data = (
50+
(torch.randn(input_shape, dtype=torch.float32) * 50)
51+
.type(torch.int8)
52+
.detach()
53+
.numpy()
54+
)
55+
input_data_tflite = np.transpose(input_data, [0, 2, 3, 1])
56+
57+
# Execute program and TFLite model
58+
program_executor = EdgeProgramExecutor(exported_program)
59+
tflite_executor = TFLiteExecutor(model_content=tflite_flatbuffers_model)
60+
61+
output_edge = program_executor.inference(input_data)
62+
output_tflite = tflite_executor.inference(input_data_tflite)
63+
64+
output_tflite = np.transpose(output_tflite, [0, 3, 1, 2])
65+
66+
# Outputs difference is smaller than 1 (rounding error in quantization)
67+
assert np.max(np.abs(output_edge - output_tflite)) <= 1
68+
69+
70+
def test_conv_fc__lowered_program_and_tflite_output_match(mocker):
71+
converter_spy = mocker.spy(EdgeProgramToIRConverter, "convert_program")
72+
73+
model = ConvFCSoftmaxModule()
74+
input_shape = (1, 4, 5, 5)
75+
76+
# Run conversion
77+
_ = to_quantized_edge_program(model, input_shape)
78+
79+
# Capture converted program
80+
exported_program: ExportedProgram = converter_spy.call_args.args[1]
81+
82+
# Capture generated model
83+
tflite_flatbuffers_model, _ = converter_spy.spy_return
84+
85+
# No Transpose ops in produced TFLite model
86+
tflite_subgraph = Model.GetRootAs(tflite_flatbuffers_model).Subgraphs(0)
87+
88+
assert tflite_subgraph.OperatorsLength() == 3
89+
assert (
90+
tflite_subgraph.Operators(0).BuiltinOptionsType()
91+
== BuiltinOptions.Conv2DOptions
92+
)
93+
assert (
94+
tflite_subgraph.Operators(1).BuiltinOptionsType()
95+
== BuiltinOptions.ReshapeOptions
96+
)
97+
assert (
98+
tflite_subgraph.Operators(2).BuiltinOptionsType()
99+
== BuiltinOptions.FullyConnectedOptions
100+
)
101+
102+
# Verify outputs of program and TFLite model
103+
input_data = (
104+
(torch.randn(input_shape, dtype=torch.float32))
105+
.type(torch.int8)
106+
.detach()
107+
.numpy()
108+
)
109+
convert_run_compare(
110+
exported_program,
111+
input_data=input_data,
112+
tflite_input_preprocess=ToNHWCPreprocess(),
113+
)

0 commit comments

Comments
 (0)