From d2458eb2fadd5ec52633624b1152521dceaf345d Mon Sep 17 00:00:00 2001 From: "Todd A. Anderson" Date: Wed, 25 Nov 2020 09:08:41 -0600 Subject: [PATCH 01/23] Initial separation of dparray into 2 parts. --- MANIFEST.in | 1 + dpctl/__init__.pxd | 2 +- dpctl/dparray.py | 222 +++++++++++++++++++++++++++++++++++++++++++++ setup.py | 1 + 4 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 dpctl/dparray.py diff --git a/MANIFEST.in b/MANIFEST.in index 43d8bddba4..fe7b0bebb4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,5 @@ include versioneer.py +include dparray.py recursive-include dpctl/include *.h *.hpp include dpctl/*.pxd include dpctl/*DPPL*Interface.* diff --git a/dpctl/__init__.pxd b/dpctl/__init__.pxd index 89f3dbe551..264400b0ad 100644 --- a/dpctl/__init__.pxd +++ b/dpctl/__init__.pxd @@ -29,4 +29,4 @@ from dpctl._sycl_core cimport * from dpctl._memory import * - +from dparray import * diff --git a/dpctl/dparray.py b/dpctl/dparray.py new file mode 100644 index 0000000000..88775e8d06 --- /dev/null +++ b/dpctl/dparray.py @@ -0,0 +1,222 @@ +import numpy as np +from inspect import getmembers, isfunction, isclass, isbuiltin +from numbers import Number +from types import FunctionType as ftype, BuiltinFunctionType as bftype +import sys +#import importlib +#import functools +import inspect + +debug = False + +def dprint(*args): + if debug: + print(*args) + sys.stdout.flush() + +import dpctl +from dpctl._memory import MemoryUSMShared + +functions_list = [o[0] for o in getmembers(np) if isfunction(o[1]) or isbuiltin(o[1])] +class_list = [o for o in getmembers(np) if isclass(o[1])] + +array_interface_property = "__array_interface__" +def has_array_interface(x): + return hasattr(x, array_interface_property) + +class ndarray(np.ndarray): + """ + numpy.ndarray subclass whose underlying memory buffer is allocated + with a foreign allocator. + """ + def __new__(subtype, shape, + dtype=float, buffer=None, offset=0, + strides=None, order=None): + # Create a new array. + if buffer is None: + dprint("dparray::ndarray __new__ buffer None") + nelems = np.prod(shape) + dt = np.dtype(dtype) + isz = dt.itemsize + buf = MemoryUSMShared(nbytes=isz*max(1,nelems)) + new_obj = np.ndarray.__new__( + subtype, shape, dtype=dt, + buffer=buf, offset=0, + strides=strides, order=order) + if hasattr(new_obj, array_interface_property): + dprint("buffer None new_obj already has sycl_usm") + else: + dprint("buffer None new_obj will add sycl_usm") + setattr(new_obj, array_interface_property, {}) + return new_obj + # zero copy if buffer is a usm backed array-like thing + elif hasattr(buffer, array_interface_property): + dprint("dparray::ndarray __new__ buffer", array_interface_property) + # also check for array interface + new_obj = np.ndarray.__new__( + subtype, shape, dtype=dtype, + buffer=buffer, offset=offset, + strides=strides, order=order) + if hasattr(new_obj, array_interface_property): + dprint("buffer None new_obj already has sycl_usm") + else: + dprint("buffer None new_obj will add sycl_usm") + setattr(new_obj, array_interface_property, {}) + return new_obj + else: + dprint("dparray::ndarray __new__ buffer not None and not sycl_usm") + nelems = np.prod(shape) + # must copy + ar = np.ndarray(shape, + dtype=dtype, buffer=buffer, + offset=offset, strides=strides, + order=order) + buf = MemoryUSMShared(nbytes=ar.nbytes) + new_obj = np.ndarray.__new__( + subtype, shape, dtype=dtype, + buffer=buf, offset=0, + strides=strides, order=order) + np.copyto(new_obj, ar, casting='no') + if hasattr(new_obj, array_interface_property): + dprint("buffer None new_obj already has sycl_usm") + else: + dprint("buffer None new_obj will add sycl_usm") + setattr(new_obj, array_interface_property, {}) + return new_obj + + def __array_finalize__(self, obj): + dprint("__array_finalize__:", obj, hex(id(obj)), type(obj)) + # When called from the explicit constructor, obj is None + if obj is None: return + # When called in new-from-template, `obj` is another instance of our own + # subclass, that we might use to update the new `self` instance. + # However, when called from view casting, `obj` can be an instance of any + # subclass of ndarray, including our own. + if hasattr(obj, array_interface_property): + return + if isinstance(obj, numba.core.runtime._nrt_python._MemInfo): + mobj = obj + while isinstance(mobj, numba.core.runtime._nrt_python._MemInfo): + dprint("array_finalize got Numba MemInfo") + ea = mobj.external_allocator + d = mobj.data + dprint("external_allocator:", hex(ea), type(ea)) + dprint("data:", hex(d), type(d)) + dppl_rt_allocator = numba.dppl._dppl_rt.get_external_allocator() + dprint("dppl external_allocator:", hex(dppl_rt_allocator), type(dppl_rt_allocator)) + dprint(dir(mobj)) + if ea == dppl_rt_allocator: + return + mobj = mobj.parent + if isinstance(mobj, ndarray): + mobj = mobj.base + if isinstance(obj, np.ndarray): + ob = self + while isinstance(ob, np.ndarray): + if hasattr(obj, array_interface_property): + return + ob = ob.base + + # Just raise an exception since __array_ufunc__ makes all reasonable cases not + # need the code below. + raise ValueError("Non-USM allocated ndarray can not viewed as a USM-allocated one without a copy") + + # Tell Numba to not treat this type just like a NumPy ndarray but to propagate its type. + # This way it will use the custom dparray allocator. + __numba_no_subtype_ndarray__ = True + + # Convert to a NumPy ndarray. + def as_ndarray(self): + return np.copy(self) + + def __array__(self): + return self + + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + if method == '__call__': + N = None + scalars = [] + typing = [] + for inp in inputs: + if isinstance(inp, Number): + scalars.append(inp) + typing.append(inp) + elif isinstance(inp, (self.__class__, np.ndarray)): + if isinstance(inp, self.__class__): + scalars.append(np.ndarray(inp.shape, inp.dtype, inp)) + typing.append(np.ndarray(inp.shape, inp.dtype)) + else: + scalars.append(inp) + typing.append(inp) + if N is not None: + if N != inp.shape: + raise TypeError("inconsistent sizes") + else: + N = inp.shape + else: + return NotImplemented + # Have to avoid recursive calls to array_ufunc here. + # If no out kwarg then we create a dparray out so that we get + # USM memory. However, if kwarg has dparray-typed out then + # array_ufunc is called recursively so we cast out as regular + # NumPy ndarray (having a USM data pointer). + if kwargs.get('out', None) is None: + # maybe copy? + # deal with multiple returned arrays, so kwargs['out'] can be tuple + res_type = np.result_type(*typing) + out = empty(inputs[0].shape, dtype=res_type) + out_as_np = np.ndarray(out.shape, out.dtype, out) + kwargs['out'] = out_as_np + else: + # If they manually gave dparray as out kwarg then we have to also + # cast as regular NumPy ndarray to avoid recursion. + if isinstance(kwargs['out'], ndarray): + out = kwargs['out'] + kwargs['out'] = np.ndarray(out.shape, out.dtype, out) + else: + out = kwargs['out'] + ret = ufunc(*scalars, **kwargs) + return out + else: + return NotImplemented + +def isdef(x): + try: + eval(x) + return True + except NameError: + return False + +for c in class_list: + cname = c[0] + if isdef(cname): + continue + # For now we do the simple thing and copy the types from NumPy module into dparray module. + new_func = "%s = np.%s" % (cname, cname) + try: + the_code = compile(new_func, '__init__', 'exec') + exec(the_code) + except: + print("Failed to exec type propagation", cname) + pass + +# Redefine all Numpy functions in this module and if they +# return a Numpy array, transform that to a USM-backed array +# instead. This is a stop-gap. We should eventually find a +# way to do the allocation correct to start with. +for fname in functions_list: + if isdef(fname): + continue + new_func = "def %s(*args, **kwargs):\n" % fname + new_func += " ret = np.%s(*args, **kwargs)\n" % fname + new_func += " if type(ret) == np.ndarray:\n" + new_func += " ret = ndarray(ret.shape, ret.dtype, ret)\n" + new_func += " return ret\n" + the_code = compile(new_func, '__init__', 'exec') + exec(the_code) + +def from_ndarray(x): + return copy(x) + +def as_ndarray(x): + return np.copy(x) diff --git a/setup.py b/setup.py index f358b4ed43..98ab999c32 100644 --- a/setup.py +++ b/setup.py @@ -201,6 +201,7 @@ def _get_cmdclass(): cmdclass["develop"] = develop return cmdclass +print("packages:", find_packages(include=["*"])) setup( name="dpctl", From 7311b94025d7a88f5f194e59f5b0167864e1037f Mon Sep 17 00:00:00 2001 From: vlad-perevezentsev Date: Tue, 1 Dec 2020 05:36:04 -0600 Subject: [PATCH 02/23] fix comments --- dpctl/dparray.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dpctl/dparray.py b/dpctl/dparray.py index 88775e8d06..533da241ac 100644 --- a/dpctl/dparray.py +++ b/dpctl/dparray.py @@ -15,7 +15,7 @@ def dprint(*args): sys.stdout.flush() import dpctl -from dpctl._memory import MemoryUSMShared +from dpctl.memory import MemoryUSMShared functions_list = [o[0] for o in getmembers(np) if isfunction(o[1]) or isbuiltin(o[1])] class_list = [o for o in getmembers(np) if isclass(o[1])] @@ -38,7 +38,8 @@ def __new__(subtype, shape, nelems = np.prod(shape) dt = np.dtype(dtype) isz = dt.itemsize - buf = MemoryUSMShared(nbytes=isz*max(1,nelems)) + nbytes = int(isz*max(1, nelems)) + buf = MemoryUSMShared(nbytes) new_obj = np.ndarray.__new__( subtype, shape, dtype=dt, buffer=buf, offset=0, @@ -71,7 +72,8 @@ def __new__(subtype, shape, dtype=dtype, buffer=buffer, offset=offset, strides=strides, order=order) - buf = MemoryUSMShared(nbytes=ar.nbytes) + nbytes = int(ar.nbytes) + buf = MemoryUSMShared(nbytes) new_obj = np.ndarray.__new__( subtype, shape, dtype=dtype, buffer=buf, offset=0, From fb163ecebed84f7137d68135d8bb869f30a66d3d Mon Sep 17 00:00:00 2001 From: vlad-perevezentsev Date: Tue, 1 Dec 2020 09:50:03 -0600 Subject: [PATCH 03/23] add tests for dparray --- dpctl/tests/test_dparray.py | 60 +++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 dpctl/tests/test_dparray.py diff --git a/dpctl/tests/test_dparray.py b/dpctl/tests/test_dparray.py new file mode 100644 index 0000000000..3c3f3624f2 --- /dev/null +++ b/dpctl/tests/test_dparray.py @@ -0,0 +1,60 @@ +import unittest +from dpctl import dparray +import numpy + + +def func_operation_with_const(dpctl_array): + return dpctl_array * 2.0 + 13 + + +def multiply_func(np_array, dpcrtl_array): + return np_array * dpcrtl_array + + +class TestOverloadList(unittest.TestCase): + maxDiff = None + + X = dparray.ndarray((256, 4), dtype='d') + X.fill(1.0) + + def test_dparray_type(self): + self.assertIsInstance(self.X, dparray.ndarray) + + def test_dparray_as_ndarray_self(self): + Y = self.X.as_ndarray() + self.assertEqual(type(Y), numpy.ndarray) + + def test_dparray_as_ndarray(self): + Y = dparray.as_ndarray(self.X) + self.assertEqual(type(Y), numpy.ndarray) + + def test_dparray_from_ndarray(self): + Y = dparray.as_ndarray(self.X) + dp1 = dparray.from_ndarray(Y) + self.assertIsInstance(dp1, dparray.ndarray) + + def test_multiplication_dparray(self): + C = self.X * 5 + self.assertIsInstance(C, dparray.ndarray) + + def test_dparray_through_python_func(self): + C = self.X * 5 + dp_func = func_operation_with_const(C) + self.assertIsInstance(dp_func, dparray.ndarray) + + def test_dparray_mixing_dpctl_and_numpy(self): + dp_numpy = numpy.ones((256, 4), dtype='d') + res = multiply_func(dp_numpy, self.X) + self.assertIsInstance(res, dparray.ndarray) + + def test_dparray_shape(self): + res = self.X.shape + self.assertEqual(res, (256, 4)) + + def test_dparray_T(self): + res = self.X.T + self.assertEqual(res.shape, (4, 256)) + + +if __name__ == '__main__': + unittest.main() From f0c3c2ba8a0219d25ea30cdf66245af9f8e7f3c6 Mon Sep 17 00:00:00 2001 From: vlad-perevezentsev Date: Wed, 2 Dec 2020 08:50:18 -0600 Subject: [PATCH 04/23] fix comments --- dpctl/__init__.pxd | 2 +- dpctl/dparray.py | 111 +++++++++++++++++++++++------------- dpctl/tests/test_dparray.py | 33 ++++++----- setup.py | 13 +---- 4 files changed, 94 insertions(+), 65 deletions(-) diff --git a/dpctl/__init__.pxd b/dpctl/__init__.pxd index 264400b0ad..ce55613b23 100644 --- a/dpctl/__init__.pxd +++ b/dpctl/__init__.pxd @@ -29,4 +29,4 @@ from dpctl._sycl_core cimport * from dpctl._memory import * -from dparray import * +from .dparray import * diff --git a/dpctl/dparray.py b/dpctl/dparray.py index 533da241ac..545696120c 100644 --- a/dpctl/dparray.py +++ b/dpctl/dparray.py @@ -3,47 +3,55 @@ from numbers import Number from types import FunctionType as ftype, BuiltinFunctionType as bftype import sys -#import importlib -#import functools import inspect +import dpctl +from dpctl.memory import MemoryUSMShared debug = False + def dprint(*args): if debug: print(*args) sys.stdout.flush() -import dpctl -from dpctl.memory import MemoryUSMShared functions_list = [o[0] for o in getmembers(np) if isfunction(o[1]) or isbuiltin(o[1])] class_list = [o for o in getmembers(np) if isclass(o[1])] array_interface_property = "__array_interface__" + + def has_array_interface(x): return hasattr(x, array_interface_property) + class ndarray(np.ndarray): """ numpy.ndarray subclass whose underlying memory buffer is allocated with a foreign allocator. """ - def __new__(subtype, shape, - dtype=float, buffer=None, offset=0, - strides=None, order=None): + + def __new__( + subtype, shape, dtype=float, buffer=None, offset=0, strides=None, order=None + ): # Create a new array. if buffer is None: dprint("dparray::ndarray __new__ buffer None") nelems = np.prod(shape) dt = np.dtype(dtype) isz = dt.itemsize - nbytes = int(isz*max(1, nelems)) + nbytes = int(isz * max(1, nelems)) buf = MemoryUSMShared(nbytes) new_obj = np.ndarray.__new__( - subtype, shape, dtype=dt, - buffer=buf, offset=0, - strides=strides, order=order) + subtype, + shape, + dtype=dt, + buffer=buf, + offset=0, + strides=strides, + order=order, + ) if hasattr(new_obj, array_interface_property): dprint("buffer None new_obj already has sycl_usm") else: @@ -55,9 +63,14 @@ def __new__(subtype, shape, dprint("dparray::ndarray __new__ buffer", array_interface_property) # also check for array interface new_obj = np.ndarray.__new__( - subtype, shape, dtype=dtype, - buffer=buffer, offset=offset, - strides=strides, order=order) + subtype, + shape, + dtype=dtype, + buffer=buffer, + offset=offset, + strides=strides, + order=order, + ) if hasattr(new_obj, array_interface_property): dprint("buffer None new_obj already has sycl_usm") else: @@ -68,17 +81,26 @@ def __new__(subtype, shape, dprint("dparray::ndarray __new__ buffer not None and not sycl_usm") nelems = np.prod(shape) # must copy - ar = np.ndarray(shape, - dtype=dtype, buffer=buffer, - offset=offset, strides=strides, - order=order) + ar = np.ndarray( + shape, + dtype=dtype, + buffer=buffer, + offset=offset, + strides=strides, + order=order, + ) nbytes = int(ar.nbytes) buf = MemoryUSMShared(nbytes) new_obj = np.ndarray.__new__( - subtype, shape, dtype=dtype, - buffer=buf, offset=0, - strides=strides, order=order) - np.copyto(new_obj, ar, casting='no') + subtype, + shape, + dtype=dtype, + buffer=buf, + offset=0, + strides=strides, + order=order, + ) + np.copyto(new_obj, ar, casting="no") if hasattr(new_obj, array_interface_property): dprint("buffer None new_obj already has sycl_usm") else: @@ -89,7 +111,8 @@ def __new__(subtype, shape, def __array_finalize__(self, obj): dprint("__array_finalize__:", obj, hex(id(obj)), type(obj)) # When called from the explicit constructor, obj is None - if obj is None: return + if obj is None: + return # When called in new-from-template, `obj` is another instance of our own # subclass, that we might use to update the new `self` instance. # However, when called from view casting, `obj` can be an instance of any @@ -105,7 +128,11 @@ def __array_finalize__(self, obj): dprint("external_allocator:", hex(ea), type(ea)) dprint("data:", hex(d), type(d)) dppl_rt_allocator = numba.dppl._dppl_rt.get_external_allocator() - dprint("dppl external_allocator:", hex(dppl_rt_allocator), type(dppl_rt_allocator)) + dprint( + "dppl external_allocator:", + hex(dppl_rt_allocator), + type(dppl_rt_allocator), + ) dprint(dir(mobj)) if ea == dppl_rt_allocator: return @@ -118,24 +145,26 @@ def __array_finalize__(self, obj): if hasattr(obj, array_interface_property): return ob = ob.base - + # Just raise an exception since __array_ufunc__ makes all reasonable cases not # need the code below. - raise ValueError("Non-USM allocated ndarray can not viewed as a USM-allocated one without a copy") - + raise ValueError( + "Non-USM allocated ndarray can not viewed as a USM-allocated one without a copy" + ) + # Tell Numba to not treat this type just like a NumPy ndarray but to propagate its type. # This way it will use the custom dparray allocator. __numba_no_subtype_ndarray__ = True # Convert to a NumPy ndarray. def as_ndarray(self): - return np.copy(self) + return np.copy(self) def __array__(self): return self def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): - if method == '__call__': + if method == "__call__": N = None scalars = [] typing = [] @@ -162,26 +191,27 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): # USM memory. However, if kwarg has dparray-typed out then # array_ufunc is called recursively so we cast out as regular # NumPy ndarray (having a USM data pointer). - if kwargs.get('out', None) is None: + if kwargs.get("out", None) is None: # maybe copy? # deal with multiple returned arrays, so kwargs['out'] can be tuple res_type = np.result_type(*typing) out = empty(inputs[0].shape, dtype=res_type) out_as_np = np.ndarray(out.shape, out.dtype, out) - kwargs['out'] = out_as_np + kwargs["out"] = out_as_np else: # If they manually gave dparray as out kwarg then we have to also # cast as regular NumPy ndarray to avoid recursion. - if isinstance(kwargs['out'], ndarray): - out = kwargs['out'] - kwargs['out'] = np.ndarray(out.shape, out.dtype, out) + if isinstance(kwargs["out"], ndarray): + out = kwargs["out"] + kwargs["out"] = np.ndarray(out.shape, out.dtype, out) else: - out = kwargs['out'] + out = kwargs["out"] ret = ufunc(*scalars, **kwargs) return out else: return NotImplemented + def isdef(x): try: eval(x) @@ -189,6 +219,7 @@ def isdef(x): except NameError: return False + for c in class_list: cname = c[0] if isdef(cname): @@ -196,7 +227,7 @@ def isdef(x): # For now we do the simple thing and copy the types from NumPy module into dparray module. new_func = "%s = np.%s" % (cname, cname) try: - the_code = compile(new_func, '__init__', 'exec') + the_code = compile(new_func, "__init__", "exec") exec(the_code) except: print("Failed to exec type propagation", cname) @@ -209,16 +240,18 @@ def isdef(x): for fname in functions_list: if isdef(fname): continue - new_func = "def %s(*args, **kwargs):\n" % fname + new_func = "def %s(*args, **kwargs):\n" % fname new_func += " ret = np.%s(*args, **kwargs)\n" % fname new_func += " if type(ret) == np.ndarray:\n" new_func += " ret = ndarray(ret.shape, ret.dtype, ret)\n" new_func += " return ret\n" - the_code = compile(new_func, '__init__', 'exec') + the_code = compile(new_func, "__init__", "exec") exec(the_code) + def from_ndarray(x): return copy(x) + def as_ndarray(x): - return np.copy(x) + return np.copy(x) diff --git a/dpctl/tests/test_dparray.py b/dpctl/tests/test_dparray.py index 3c3f3624f2..744699c664 100644 --- a/dpctl/tests/test_dparray.py +++ b/dpctl/tests/test_dparray.py @@ -3,19 +3,10 @@ import numpy -def func_operation_with_const(dpctl_array): - return dpctl_array * 2.0 + 13 - - -def multiply_func(np_array, dpcrtl_array): - return np_array * dpcrtl_array - - class TestOverloadList(unittest.TestCase): - maxDiff = None - - X = dparray.ndarray((256, 4), dtype='d') - X.fill(1.0) + def setUp(self): + self.X = dparray.ndarray((256, 4), dtype="d") + self.X.fill(1.0) def test_dparray_type(self): self.assertIsInstance(self.X, dparray.ndarray) @@ -38,13 +29,16 @@ def test_multiplication_dparray(self): self.assertIsInstance(C, dparray.ndarray) def test_dparray_through_python_func(self): + def func_operation_with_const(dpctl_array): + return dpctl_array * 2.0 + 13 + C = self.X * 5 dp_func = func_operation_with_const(C) self.assertIsInstance(dp_func, dparray.ndarray) def test_dparray_mixing_dpctl_and_numpy(self): - dp_numpy = numpy.ones((256, 4), dtype='d') - res = multiply_func(dp_numpy, self.X) + dp_numpy = numpy.ones((256, 4), dtype="d") + res = dp_numpy * self.X self.assertIsInstance(res, dparray.ndarray) def test_dparray_shape(self): @@ -55,6 +49,15 @@ def test_dparray_T(self): res = self.X.T self.assertEqual(res.shape, (4, 256)) + def test_numpy_ravel_with_dparray(self): + res = numpy.ravel(self.X) + self.assertEqual(res.shape, (1024,)) + + @unittest.expectedFailure + def test_numpy_sum_with_dparray(self): + res = numpy.sum(self.X) + self.assertEqual(res, 1024.0) + -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/setup.py b/setup.py index 98ab999c32..463c2c8209 100644 --- a/setup.py +++ b/setup.py @@ -148,9 +148,7 @@ def extensions(): runtime_library_dirs = [] extension_args = { - "depends": [ - dppl_sycl_interface_include, - ], + "depends": [dppl_sycl_interface_include,], "include_dirs": [np.get_include(), dppl_sycl_interface_include], "extra_compile_args": eca + get_other_cxxflags() @@ -165,16 +163,12 @@ def extensions(): extensions = [ Extension( "dpctl._sycl_core", - [ - os.path.join("dpctl", "_sycl_core.pyx"), - ], + [os.path.join("dpctl", "_sycl_core.pyx"),], **extension_args ), Extension( "dpctl.memory._memory", - [ - os.path.join("dpctl", "memory", "_memory.pyx"), - ], + [os.path.join("dpctl", "memory", "_memory.pyx"),], **extension_args ), ] @@ -201,7 +195,6 @@ def _get_cmdclass(): cmdclass["develop"] = develop return cmdclass -print("packages:", find_packages(include=["*"])) setup( name="dpctl", From 935a2a10623bf4beddae83d42c42e7bd4aa3ea35 Mon Sep 17 00:00:00 2001 From: vlad-perevezentsev Date: Wed, 2 Dec 2020 09:02:08 -0600 Subject: [PATCH 05/23] use black --- setup.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/setup.py b/setup.py index 463c2c8209..298e75bde9 100644 --- a/setup.py +++ b/setup.py @@ -1,25 +1,25 @@ ##===---------- setup.py - dpctl.ocldrv interface -----*- Python -*-----===## ## -## Data Parallel Control Library (dpCtl) +# Data Parallel Control Library (dpCtl) ## -## Copyright 2020 Intel Corporation +# Copyright 2020 Intel Corporation ## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at ## -## http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 ## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. ## ##===----------------------------------------------------------------------===## ### -### \file -### This file builds the dpctl and dpctl.ocldrv extension modules. +# \file +# This file builds the dpctl and dpctl.ocldrv extension modules. ##===----------------------------------------------------------------------===## import os import os.path From efcfcd93a017fb9e3213366deb5192faa99cb959 Mon Sep 17 00:00:00 2001 From: vlad-perevezentsev Date: Wed, 2 Dec 2020 10:43:01 -0600 Subject: [PATCH 06/23] fix the error from black and remove one test --- dpctl/tests/test_dparray.py | 8 -------- setup.py | 12 +++++++++--- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/dpctl/tests/test_dparray.py b/dpctl/tests/test_dparray.py index 744699c664..b0192cd987 100644 --- a/dpctl/tests/test_dparray.py +++ b/dpctl/tests/test_dparray.py @@ -28,14 +28,6 @@ def test_multiplication_dparray(self): C = self.X * 5 self.assertIsInstance(C, dparray.ndarray) - def test_dparray_through_python_func(self): - def func_operation_with_const(dpctl_array): - return dpctl_array * 2.0 + 13 - - C = self.X * 5 - dp_func = func_operation_with_const(C) - self.assertIsInstance(dp_func, dparray.ndarray) - def test_dparray_mixing_dpctl_and_numpy(self): dp_numpy = numpy.ones((256, 4), dtype="d") res = dp_numpy * self.X diff --git a/setup.py b/setup.py index 298e75bde9..eaf0376b7b 100644 --- a/setup.py +++ b/setup.py @@ -148,7 +148,9 @@ def extensions(): runtime_library_dirs = [] extension_args = { - "depends": [dppl_sycl_interface_include,], + "depends": [ + dppl_sycl_interface_include, + ], "include_dirs": [np.get_include(), dppl_sycl_interface_include], "extra_compile_args": eca + get_other_cxxflags() @@ -163,12 +165,16 @@ def extensions(): extensions = [ Extension( "dpctl._sycl_core", - [os.path.join("dpctl", "_sycl_core.pyx"),], + [ + os.path.join("dpctl", "_sycl_core.pyx"), + ], **extension_args ), Extension( "dpctl.memory._memory", - [os.path.join("dpctl", "memory", "_memory.pyx"),], + [ + os.path.join("dpctl", "memory", "_memory.pyx"), + ], **extension_args ), ] From 7ebd06cfe279adfe7f5d0dd21ea99f160dc9b2fe Mon Sep 17 00:00:00 2001 From: "Todd A. Anderson" Date: Wed, 2 Dec 2020 18:21:11 -0600 Subject: [PATCH 07/23] Resolve some of the code review issues. Not explicitly checking Numba MemInfo. We'll have to add __sycl_usm_array_interface__ property to Numba objects to get this equivalent behavior without this dependency in the wrong direction. --- dpctl/dparray.py | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/dpctl/dparray.py b/dpctl/dparray.py index 545696120c..35c292ae22 100644 --- a/dpctl/dparray.py +++ b/dpctl/dparray.py @@ -19,7 +19,7 @@ def dprint(*args): functions_list = [o[0] for o in getmembers(np) if isfunction(o[1]) or isbuiltin(o[1])] class_list = [o for o in getmembers(np) if isclass(o[1])] -array_interface_property = "__array_interface__" +array_interface_property = "__sycl_usm_array_interface__" def has_array_interface(x): @@ -119,30 +119,10 @@ def __array_finalize__(self, obj): # subclass of ndarray, including our own. if hasattr(obj, array_interface_property): return - if isinstance(obj, numba.core.runtime._nrt_python._MemInfo): - mobj = obj - while isinstance(mobj, numba.core.runtime._nrt_python._MemInfo): - dprint("array_finalize got Numba MemInfo") - ea = mobj.external_allocator - d = mobj.data - dprint("external_allocator:", hex(ea), type(ea)) - dprint("data:", hex(d), type(d)) - dppl_rt_allocator = numba.dppl._dppl_rt.get_external_allocator() - dprint( - "dppl external_allocator:", - hex(dppl_rt_allocator), - type(dppl_rt_allocator), - ) - dprint(dir(mobj)) - if ea == dppl_rt_allocator: - return - mobj = mobj.parent - if isinstance(mobj, ndarray): - mobj = mobj.base if isinstance(obj, np.ndarray): ob = self while isinstance(ob, np.ndarray): - if hasattr(obj, array_interface_property): + if hasattr(ob, array_interface_property): return ob = ob.base From c3cd7359a004ff28bec1fed6dcad5640f71e4a9f Mon Sep 17 00:00:00 2001 From: vlad-perevezentsev Date: Fri, 4 Dec 2020 05:42:08 -0600 Subject: [PATCH 08/23] add the new dparray class inder dpctl.dptensor.dparray --- MANIFEST.in | 1 - dpctl/__init__.pxd | 2 +- dpctl/dptensor/__init__.py | 1 + dpctl/{ => dptensor}/dparray.py | 0 dpctl/tests/__init__.py | 1 + dpctl/tests/test_dparray.py | 2 +- 6 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 dpctl/dptensor/__init__.py rename dpctl/{ => dptensor}/dparray.py (100%) diff --git a/MANIFEST.in b/MANIFEST.in index fe7b0bebb4..43d8bddba4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,4 @@ include versioneer.py -include dparray.py recursive-include dpctl/include *.h *.hpp include dpctl/*.pxd include dpctl/*DPPL*Interface.* diff --git a/dpctl/__init__.pxd b/dpctl/__init__.pxd index ce55613b23..86b66be9d6 100644 --- a/dpctl/__init__.pxd +++ b/dpctl/__init__.pxd @@ -29,4 +29,4 @@ from dpctl._sycl_core cimport * from dpctl._memory import * -from .dparray import * +from dpctl.dptensor.dparray import * diff --git a/dpctl/dptensor/__init__.py b/dpctl/dptensor/__init__.py new file mode 100644 index 0000000000..5fe6f3ac9b --- /dev/null +++ b/dpctl/dptensor/__init__.py @@ -0,0 +1 @@ +from .dparray import * diff --git a/dpctl/dparray.py b/dpctl/dptensor/dparray.py similarity index 100% rename from dpctl/dparray.py rename to dpctl/dptensor/dparray.py diff --git a/dpctl/tests/__init__.py b/dpctl/tests/__init__.py index a53980d17a..25c4d2f42b 100644 --- a/dpctl/tests/__init__.py +++ b/dpctl/tests/__init__.py @@ -30,3 +30,4 @@ from .test_sycl_queue_manager import * from .test_sycl_queue_memcpy import * from .test_sycl_usm import * +from .test_dparray import * diff --git a/dpctl/tests/test_dparray.py b/dpctl/tests/test_dparray.py index b0192cd987..1dd597b167 100644 --- a/dpctl/tests/test_dparray.py +++ b/dpctl/tests/test_dparray.py @@ -1,5 +1,5 @@ import unittest -from dpctl import dparray +from dpctl.dptensor import dparray import numpy From 359b5dcf22356eb3f6d4ae4e01b981ae232b1f7d Mon Sep 17 00:00:00 2001 From: vlad-perevezentsev Date: Fri, 4 Dec 2020 05:42:08 -0600 Subject: [PATCH 09/23] add the new dparray class inder dpctl.dptensor.dparray --- MANIFEST.in | 1 - dpctl/__init__.pxd | 2 +- dpctl/dptensor/__init__.py | 1 + dpctl/{ => dptensor}/dparray.py | 0 dpctl/tests/__init__.py | 1 + dpctl/tests/test_dparray.py | 2 +- 6 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 dpctl/dptensor/__init__.py rename dpctl/{ => dptensor}/dparray.py (100%) diff --git a/MANIFEST.in b/MANIFEST.in index fe7b0bebb4..43d8bddba4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,4 @@ include versioneer.py -include dparray.py recursive-include dpctl/include *.h *.hpp include dpctl/*.pxd include dpctl/*DPPL*Interface.* diff --git a/dpctl/__init__.pxd b/dpctl/__init__.pxd index ce55613b23..86b66be9d6 100644 --- a/dpctl/__init__.pxd +++ b/dpctl/__init__.pxd @@ -29,4 +29,4 @@ from dpctl._sycl_core cimport * from dpctl._memory import * -from .dparray import * +from dpctl.dptensor.dparray import * diff --git a/dpctl/dptensor/__init__.py b/dpctl/dptensor/__init__.py new file mode 100644 index 0000000000..5fe6f3ac9b --- /dev/null +++ b/dpctl/dptensor/__init__.py @@ -0,0 +1 @@ +from .dparray import * diff --git a/dpctl/dparray.py b/dpctl/dptensor/dparray.py similarity index 100% rename from dpctl/dparray.py rename to dpctl/dptensor/dparray.py diff --git a/dpctl/tests/__init__.py b/dpctl/tests/__init__.py index a53980d17a..25c4d2f42b 100644 --- a/dpctl/tests/__init__.py +++ b/dpctl/tests/__init__.py @@ -30,3 +30,4 @@ from .test_sycl_queue_manager import * from .test_sycl_queue_memcpy import * from .test_sycl_usm import * +from .test_dparray import * diff --git a/dpctl/tests/test_dparray.py b/dpctl/tests/test_dparray.py index b0192cd987..1dd597b167 100644 --- a/dpctl/tests/test_dparray.py +++ b/dpctl/tests/test_dparray.py @@ -1,5 +1,5 @@ import unittest -from dpctl import dparray +from dpctl.dptensor import dparray import numpy From 65925b6ddbd5dd717087f6874ebe78bdc57176be Mon Sep 17 00:00:00 2001 From: Sergey Pokhodenko Date: Fri, 4 Dec 2020 08:16:49 -0600 Subject: [PATCH 10/23] Add test_dparray.py into test system --- dpctl/tests/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dpctl/tests/__init__.py b/dpctl/tests/__init__.py index a53980d17a..2ce3a0cdbd 100644 --- a/dpctl/tests/__init__.py +++ b/dpctl/tests/__init__.py @@ -22,6 +22,7 @@ ## Top-level module of all dpctl Python unit test cases. ##===----------------------------------------------------------------------===## +from .test_dparray import * from .test_dump_functions import * from .test_sycl_device import * from .test_sycl_kernel_submit import * From ef2902f7d766a1ebf7d9c21a40813cdec7d28908 Mon Sep 17 00:00:00 2001 From: Sergey Pokhodenko Date: Fri, 4 Dec 2020 08:17:22 -0600 Subject: [PATCH 11/23] Rename test class for dparray --- dpctl/tests/test_dparray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpctl/tests/test_dparray.py b/dpctl/tests/test_dparray.py index b0192cd987..3a5ad12738 100644 --- a/dpctl/tests/test_dparray.py +++ b/dpctl/tests/test_dparray.py @@ -3,7 +3,7 @@ import numpy -class TestOverloadList(unittest.TestCase): +class Test_dparray(unittest.TestCase): def setUp(self): self.X = dparray.ndarray((256, 4), dtype="d") self.X.fill(1.0) From 3c69ed7e11f66574cdd94813e77e1a45ca45fafb Mon Sep 17 00:00:00 2001 From: Sergey Pokhodenko Date: Fri, 4 Dec 2020 08:20:23 -0600 Subject: [PATCH 12/23] Remove formatting of setup.py --- setup.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/setup.py b/setup.py index eaf0376b7b..f358b4ed43 100644 --- a/setup.py +++ b/setup.py @@ -1,25 +1,25 @@ ##===---------- setup.py - dpctl.ocldrv interface -----*- Python -*-----===## ## -# Data Parallel Control Library (dpCtl) +## Data Parallel Control Library (dpCtl) ## -# Copyright 2020 Intel Corporation +## Copyright 2020 Intel Corporation ## -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at ## -# http://www.apache.org/licenses/LICENSE-2.0 +## http://www.apache.org/licenses/LICENSE-2.0 ## -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. ## ##===----------------------------------------------------------------------===## ### -# \file -# This file builds the dpctl and dpctl.ocldrv extension modules. +### \file +### This file builds the dpctl and dpctl.ocldrv extension modules. ##===----------------------------------------------------------------------===## import os import os.path From ea5e319216fb50fb88faa0c991219fc08885d7f6 Mon Sep 17 00:00:00 2001 From: Sergey Pokhodenko Date: Fri, 4 Dec 2020 08:26:54 -0600 Subject: [PATCH 13/23] Add license in new files. --- dpctl/dparray.py | 24 ++++++++++++++++++++++++ dpctl/tests/test_dparray.py | 24 ++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/dpctl/dparray.py b/dpctl/dparray.py index 35c292ae22..636876e6d8 100644 --- a/dpctl/dparray.py +++ b/dpctl/dparray.py @@ -1,3 +1,27 @@ +##===---------- dparray.py - dpctl -------*- Python -*----===## +## +## Data Parallel Control (dpCtl) +## +## Copyright 2020 Intel Corporation +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +##===----------------------------------------------------------------------===## +### +### \file +### This file implements a dparray - USM aware implementation of ndarray. +##===----------------------------------------------------------------------===## + import numpy as np from inspect import getmembers, isfunction, isclass, isbuiltin from numbers import Number diff --git a/dpctl/tests/test_dparray.py b/dpctl/tests/test_dparray.py index 3a5ad12738..86300fc7fc 100644 --- a/dpctl/tests/test_dparray.py +++ b/dpctl/tests/test_dparray.py @@ -1,3 +1,27 @@ +##===---------- test_dparray.py - dpctl -------*- Python -*----===## +## +## Data Parallel Control (dpCtl) +## +## Copyright 2020 Intel Corporation +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +##===----------------------------------------------------------------------===## +### +### \file +### A basic unit test for dpctl.dparray. +##===----------------------------------------------------------------------===## + import unittest from dpctl import dparray import numpy From 5fbef1c6d76fdd1929f6aa538560f9ac119fc07b Mon Sep 17 00:00:00 2001 From: Sergey Pokhodenko Date: Fri, 4 Dec 2020 08:57:51 -0600 Subject: [PATCH 14/23] Remove import dparray from __inti__.pxd --- dpctl/__init__.pxd | 1 - 1 file changed, 1 deletion(-) diff --git a/dpctl/__init__.pxd b/dpctl/__init__.pxd index ce55613b23..492de01f68 100644 --- a/dpctl/__init__.pxd +++ b/dpctl/__init__.pxd @@ -29,4 +29,3 @@ from dpctl._sycl_core cimport * from dpctl._memory import * -from .dparray import * From 897afc91057cfb59d057f2b9a748d680f573cd93 Mon Sep 17 00:00:00 2001 From: Sergey Pokhodenko Date: Mon, 7 Dec 2020 17:50:09 +0300 Subject: [PATCH 15/23] Update MANIFEST.in Remove `include dparray.py` --- MANIFEST.in | 1 - 1 file changed, 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index fe7b0bebb4..43d8bddba4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,4 @@ include versioneer.py -include dparray.py recursive-include dpctl/include *.h *.hpp include dpctl/*.pxd include dpctl/*DPPL*Interface.* From 4a7b94ea478d0f29a055b16653510d280d3398a2 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 7 Dec 2020 10:49:38 -0600 Subject: [PATCH 16/23] dpctl.dptensor.ndarray provides proper __sycl_usm_array_interface__ --- dpctl/dptensor/dparray.py | 43 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/dpctl/dptensor/dparray.py b/dpctl/dptensor/dparray.py index 636876e6d8..ef4d45fd14 100644 --- a/dpctl/dptensor/dparray.py +++ b/dpctl/dptensor/dparray.py @@ -49,6 +49,19 @@ def dprint(*args): def has_array_interface(x): return hasattr(x, array_interface_property) +def _get_usm_base(ary): + ob = ary + while True: + if ob is None: + return None + elif hasattr(ob, '__sycl_usm_array_interface__'): + return ob + elif isinstance(ob, np.ndarray): + ob = ob.base + elif isinstance(ob, memoryview): + ob = ob.obj + else: + return None class ndarray(np.ndarray): """ @@ -80,7 +93,9 @@ def __new__( dprint("buffer None new_obj already has sycl_usm") else: dprint("buffer None new_obj will add sycl_usm") - setattr(new_obj, array_interface_property, {}) + setattr(new_obj, + array_interface_property, + new_obj._getter_sycl_usm_array_interface_()) return new_obj # zero copy if buffer is a usm backed array-like thing elif hasattr(buffer, array_interface_property): @@ -99,7 +114,8 @@ def __new__( dprint("buffer None new_obj already has sycl_usm") else: dprint("buffer None new_obj will add sycl_usm") - setattr(new_obj, array_interface_property, {}) + setattr(new_obj, array_interface_property, + new_obj._getter_sycl_usm_array_interface_()) return new_obj else: dprint("dparray::ndarray __new__ buffer not None and not sycl_usm") @@ -129,9 +145,29 @@ def __new__( dprint("buffer None new_obj already has sycl_usm") else: dprint("buffer None new_obj will add sycl_usm") - setattr(new_obj, array_interface_property, {}) + setattr(new_obj, array_interface_property, + new_obj._getter_sycl_usm_array_interface_()) return new_obj + + def _getter_sycl_usm_array_interface_(self): + ary_iface = self.__array_interface__ + _base = _get_usm_base(self) + if _base is None: + raise TypeError + + usm_iface = getattr(_base, '__sycl_usm_array_interface__', None) + if usm_iface is None: + raise TypeError + + if (ary_iface['data'][0] == usm_iface['data'][0]): + ary_iface['version'] = usm_iface['version'] + ary_iface['syclobj'] = usm_iface['syclobj'] + else: + raise TypeError + return ary_iface + + def __array_finalize__(self, obj): dprint("__array_finalize__:", obj, hex(id(obj)), type(obj)) # When called from the explicit constructor, obj is None @@ -156,6 +192,7 @@ def __array_finalize__(self, obj): "Non-USM allocated ndarray can not viewed as a USM-allocated one without a copy" ) + # Tell Numba to not treat this type just like a NumPy ndarray but to propagate its type. # This way it will use the custom dparray allocator. __numba_no_subtype_ndarray__ = True From a0da813a7cd4000f74d47d79ab25a57f484e4446 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 7 Dec 2020 10:51:16 -0600 Subject: [PATCH 17/23] package types is not used --- dpctl/dptensor/dparray.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dpctl/dptensor/dparray.py b/dpctl/dptensor/dparray.py index ef4d45fd14..5f66efc636 100644 --- a/dpctl/dptensor/dparray.py +++ b/dpctl/dptensor/dparray.py @@ -25,7 +25,6 @@ import numpy as np from inspect import getmembers, isfunction, isclass, isbuiltin from numbers import Number -from types import FunctionType as ftype, BuiltinFunctionType as bftype import sys import inspect import dpctl From 0b2931c5812b3d587d6f5cf61ca5d5f34f7374b4 Mon Sep 17 00:00:00 2001 From: vlad-perevezentsev Date: Mon, 7 Dec 2020 14:39:44 -0600 Subject: [PATCH 18/23] Add a remote test and fix black --- dpctl/dptensor/dparray.py | 36 +++++++++++++++++++++--------------- dpctl/tests/test_dparray.py | 8 ++++++++ 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/dpctl/dptensor/dparray.py b/dpctl/dptensor/dparray.py index 5f66efc636..33f5d44690 100644 --- a/dpctl/dptensor/dparray.py +++ b/dpctl/dptensor/dparray.py @@ -48,12 +48,13 @@ def dprint(*args): def has_array_interface(x): return hasattr(x, array_interface_property) + def _get_usm_base(ary): ob = ary while True: if ob is None: return None - elif hasattr(ob, '__sycl_usm_array_interface__'): + elif hasattr(ob, "__sycl_usm_array_interface__"): return ob elif isinstance(ob, np.ndarray): ob = ob.base @@ -92,9 +93,11 @@ def __new__( dprint("buffer None new_obj already has sycl_usm") else: dprint("buffer None new_obj will add sycl_usm") - setattr(new_obj, - array_interface_property, - new_obj._getter_sycl_usm_array_interface_()) + setattr( + new_obj, + array_interface_property, + new_obj._getter_sycl_usm_array_interface_(), + ) return new_obj # zero copy if buffer is a usm backed array-like thing elif hasattr(buffer, array_interface_property): @@ -113,8 +116,11 @@ def __new__( dprint("buffer None new_obj already has sycl_usm") else: dprint("buffer None new_obj will add sycl_usm") - setattr(new_obj, array_interface_property, - new_obj._getter_sycl_usm_array_interface_()) + setattr( + new_obj, + array_interface_property, + new_obj._getter_sycl_usm_array_interface_(), + ) return new_obj else: dprint("dparray::ndarray __new__ buffer not None and not sycl_usm") @@ -144,29 +150,30 @@ def __new__( dprint("buffer None new_obj already has sycl_usm") else: dprint("buffer None new_obj will add sycl_usm") - setattr(new_obj, array_interface_property, - new_obj._getter_sycl_usm_array_interface_()) + setattr( + new_obj, + array_interface_property, + new_obj._getter_sycl_usm_array_interface_(), + ) return new_obj - def _getter_sycl_usm_array_interface_(self): ary_iface = self.__array_interface__ _base = _get_usm_base(self) if _base is None: raise TypeError - usm_iface = getattr(_base, '__sycl_usm_array_interface__', None) + usm_iface = getattr(_base, "__sycl_usm_array_interface__", None) if usm_iface is None: raise TypeError - if (ary_iface['data'][0] == usm_iface['data'][0]): - ary_iface['version'] = usm_iface['version'] - ary_iface['syclobj'] = usm_iface['syclobj'] + if ary_iface["data"][0] == usm_iface["data"][0]: + ary_iface["version"] = usm_iface["version"] + ary_iface["syclobj"] = usm_iface["syclobj"] else: raise TypeError return ary_iface - def __array_finalize__(self, obj): dprint("__array_finalize__:", obj, hex(id(obj)), type(obj)) # When called from the explicit constructor, obj is None @@ -191,7 +198,6 @@ def __array_finalize__(self, obj): "Non-USM allocated ndarray can not viewed as a USM-allocated one without a copy" ) - # Tell Numba to not treat this type just like a NumPy ndarray but to propagate its type. # This way it will use the custom dparray allocator. __numba_no_subtype_ndarray__ = True diff --git a/dpctl/tests/test_dparray.py b/dpctl/tests/test_dparray.py index 9b4ee88f0f..dc35139689 100644 --- a/dpctl/tests/test_dparray.py +++ b/dpctl/tests/test_dparray.py @@ -52,6 +52,14 @@ def test_multiplication_dparray(self): C = self.X * 5 self.assertIsInstance(C, dparray.ndarray) + def test_dparray_through_python_func(self): + def func_operation_with_const(dpctl_array): + return dpctl_array * 2.0 + 13 + + C = self.X * 5 + dp_func = func_operation_with_const(C) + self.assertIsInstance(dp_func, dparray.ndarray) + def test_dparray_mixing_dpctl_and_numpy(self): dp_numpy = numpy.ones((256, 4), dtype="d") res = dp_numpy * self.X From 25e864b826b3fa03ed2cd23ff020b06e37c766df Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 7 Dec 2020 15:29:18 -0600 Subject: [PATCH 19/23] dparray.py -> numpy_usm_shared.py --- dpctl/dptensor/{dparray.py => numpy_usm_shared.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename dpctl/dptensor/{dparray.py => numpy_usm_shared.py} (100%) diff --git a/dpctl/dptensor/dparray.py b/dpctl/dptensor/numpy_usm_shared.py similarity index 100% rename from dpctl/dptensor/dparray.py rename to dpctl/dptensor/numpy_usm_shared.py From 15d0364716a8253eaef553c0bac9c24ce18bb857 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 7 Dec 2020 15:31:23 -0600 Subject: [PATCH 20/23] changes after renaming dparray.py->numpy_usm_shared.py --- dpctl/dptensor/__init__.py | 2 +- dpctl/tests/test_dparray.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dpctl/dptensor/__init__.py b/dpctl/dptensor/__init__.py index 5fe6f3ac9b..c7695fcd4f 100644 --- a/dpctl/dptensor/__init__.py +++ b/dpctl/dptensor/__init__.py @@ -1 +1 @@ -from .dparray import * +import dpctl.dptensor.numpy_usm_shared diff --git a/dpctl/tests/test_dparray.py b/dpctl/tests/test_dparray.py index dc35139689..5768c1cadf 100644 --- a/dpctl/tests/test_dparray.py +++ b/dpctl/tests/test_dparray.py @@ -23,7 +23,7 @@ ##===----------------------------------------------------------------------===## import unittest -from dpctl.dptensor import dparray +from dpctl.dptensor import numpy_usm_shared as dparray import numpy From 2aab77b733bae189f9ec50a4d5b16258c5a49ac7 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 7 Dec 2020 15:53:29 -0600 Subject: [PATCH 21/23] black voodoo applied --- dpctl/dptensor/numpy_usm_shared.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dpctl/dptensor/numpy_usm_shared.py b/dpctl/dptensor/numpy_usm_shared.py index 33f5d44690..a2bd452350 100644 --- a/dpctl/dptensor/numpy_usm_shared.py +++ b/dpctl/dptensor/numpy_usm_shared.py @@ -63,6 +63,7 @@ def _get_usm_base(ary): else: return None + class ndarray(np.ndarray): """ numpy.ndarray subclass whose underlying memory buffer is allocated @@ -167,7 +168,7 @@ def _getter_sycl_usm_array_interface_(self): if usm_iface is None: raise TypeError - if ary_iface["data"][0] == usm_iface["data"][0]: + if ary_iface["data"][0] == usm_iface["data"][0]: ary_iface["version"] = usm_iface["version"] ary_iface["syclobj"] = usm_iface["syclobj"] else: From d53d8b87823427b2b52a483f72b6568108f5b8d4 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 7 Dec 2020 15:57:46 -0600 Subject: [PATCH 22/23] more black voodoo --- dpctl/tests/test_dparray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpctl/tests/test_dparray.py b/dpctl/tests/test_dparray.py index 5768c1cadf..fc974eb4a4 100644 --- a/dpctl/tests/test_dparray.py +++ b/dpctl/tests/test_dparray.py @@ -59,7 +59,7 @@ def func_operation_with_const(dpctl_array): C = self.X * 5 dp_func = func_operation_with_const(C) self.assertIsInstance(dp_func, dparray.ndarray) - + def test_dparray_mixing_dpctl_and_numpy(self): dp_numpy = numpy.ones((256, 4), dtype="d") res = dp_numpy * self.X From cc2bd6cec7262d2767e56914fc6886a1d64879a0 Mon Sep 17 00:00:00 2001 From: Diptorup Deb <3046810+diptorupd@users.noreply.github.com> Date: Mon, 7 Dec 2020 16:38:07 -0600 Subject: [PATCH 23/23] Update __init__.pxd Removed `from dpctl._memory import *` as the module is no longer present. Removed `from dpctl.dptensor.dparray import *` as there is no point in having the import here. --- dpctl/__init__.pxd | 3 --- 1 file changed, 3 deletions(-) diff --git a/dpctl/__init__.pxd b/dpctl/__init__.pxd index fa0458cea7..e4bb2eaeb6 100644 --- a/dpctl/__init__.pxd +++ b/dpctl/__init__.pxd @@ -28,6 +28,3 @@ # cython: language_level=3 from dpctl._sycl_core cimport * -from dpctl._memory import * -from dpctl.dptensor.dparray import * -