diff --git a/sycl/include/CL/sycl/handler.hpp b/sycl/include/CL/sycl/handler.hpp
index 982250a52879b..a675f8aa66a0f 100644
--- a/sycl/include/CL/sycl/handler.hpp
+++ b/sycl/include/CL/sycl/handler.hpp
@@ -805,9 +805,9 @@ class __SYCL_EXPORT handler {
 
   template <typename T> struct ShouldEnableSetArg {
     static constexpr bool value =
-        std::is_trivially_copyable<T>::value
+        std::is_trivially_copyable<detail::remove_reference_t<T>>::value
 #if CL_SYCL_LANGUAGE_VERSION && CL_SYCL_LANGUAGE_VERSION <= 121
-            && std::is_standard_layout<T>::value
+            && std::is_standard_layout<detail::remove_reference_t<T>>::value
 #endif
         || is_same_type<sampler, T>::value // Sampler
         || (!is_same_type<cl_mem, T>::value &&
diff --git a/sycl/test/basic_tests/set_arg_error.cpp b/sycl/test/basic_tests/set_arg_error.cpp
index 4472b50bcd669..52df555902b44 100644
--- a/sycl/test/basic_tests/set_arg_error.cpp
+++ b/sycl/test/basic_tests/set_arg_error.cpp
@@ -33,15 +33,23 @@ int main() {
     cl::sycl::accessor<int, 1, cl::sycl::access::mode::read_write,
                        cl::sycl::access::target::local>
         local_acc({size}, h);
+    TriviallyCopyable tc{1, 2};
+    NonTriviallyCopyable ntc;
     h.set_arg(0, local_acc);
     h.set_arg(1, global_acc);
     h.set_arg(2, samp);
-    h.set_arg(3, TriviallyCopyable{});
+    h.set_arg(3, tc);
+    h.set_arg(4, TriviallyCopyable{});
+    h.set_arg( // expected-error {{no matching member function for call to 'set_arg'}}
+        5, ntc);
     h.set_arg( // expected-error {{no matching member function for call to 'set_arg'}}
         4, NonTriviallyCopyable{});
 #if CL_SYCL_LANGUAGE_VERSION && CL_SYCL_LANGUAGE_VERSION <= 121
+    NonStdLayout nstd;
+    h.set_arg( // expected-error {{no matching member function for call to 'set_arg'}}
+        6, nstd);
     h.set_arg( // expected-error {{no matching member function for call to 'set_arg'}}
-        5, NonStdLayout{});
+        7, NonStdLayout{});
 #endif
   });
 }