From 6e810500b57742c1bf7f440972e269162f1ac5e9 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 19 Aug 2024 11:57:48 -0500 Subject: [PATCH 01/11] Correct test_full_gh_1230 for NumPy 2.1.0 NumPy 2.1.0 now raises exception when fill_value is not in bounds for the array data type. Rework the logic of the test to accomodate that. The test continue to work with NumPy 2.0.1 --- dpctl/tests/test_usm_ndarray_ctor.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/dpctl/tests/test_usm_ndarray_ctor.py b/dpctl/tests/test_usm_ndarray_ctor.py index 9221e62ad4..739f863b27 100644 --- a/dpctl/tests/test_usm_ndarray_ctor.py +++ b/dpctl/tests/test_usm_ndarray_ctor.py @@ -1791,6 +1791,15 @@ def test_full_strides(): assert np.array_equal(dpt.asnumpy(X), Xnp) +def _reduce_to_idtype_range(o, dtype): + assert isinstance(dtype, np.dtype) + assert dtype.kind in "ui" + _iinfo = np.iinfo(dtype) + _max = _iinfo.max + _min = _iinfo.min + return ((o - _min) % (1 + _max - _min)) + _min + + def test_full_gh_1230(): q = get_queue_or_skip() dtype = "i4" @@ -1798,7 +1807,10 @@ def test_full_gh_1230(): X = dpt.full(1, dt_maxint + 1, dtype=dtype, sycl_queue=q) X_np = dpt.asnumpy(X) assert X.dtype == dpt.dtype(dtype) - assert np.array_equal(X_np, np.full_like(X_np, dt_maxint + 1)) + expected = np.full_like( + X_np, _reduce_to_idtype_range(dt_maxint + 1, X.dtype) + ) + assert np.array_equal(X_np, expected) with pytest.raises(OverflowError): dpt.full(1, dpt.iinfo(dpt.uint64).max + 1, sycl_queue=q) From 845382deada05e9ebe7095249ac3e4bea42983b1 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 19 Aug 2024 11:59:14 -0500 Subject: [PATCH 02/11] Adapt from_dlpack to NumPy 2.1.0 behavior NumPy 2.1.0 supports DLPack 1.0 now, and raises ValueError if __dlpack__ method gets unsupport dl_device value. Function from_dlpack now handles ValueError, the same way as it handles BufferError --- dpctl/tensor/_dlpack.pyx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dpctl/tensor/_dlpack.pyx b/dpctl/tensor/_dlpack.pyx index 098003ead2..326a1f7aa7 100644 --- a/dpctl/tensor/_dlpack.pyx +++ b/dpctl/tensor/_dlpack.pyx @@ -1083,9 +1083,10 @@ def from_dlpack(x, /, *, device=None, copy=None): except TypeError: # exporter does not support max_version keyword got_type_error = True - except (BufferError, NotImplementedError): - # Either dl_device, or copy can be satisfied + except (BufferError, NotImplementedError, ValueError) as e: + # Either dl_device, or copy cannot be satisfied got_buffer_error = True + saved_exception = e except Exception as e: got_other_error = True saved_exception = e @@ -1144,6 +1145,8 @@ def from_dlpack(x, /, *, device=None, copy=None): raise BufferError( "Importing data via DLPack requires copying, but copy=False was provided" ) + if dl_device is None: + raise saved_exception # must copy via host if dl_device[0] != device_OneAPI: raise BufferError(f"Can not import to requested device {dl_device}") From 1c17c345240d6e6c75ea0ae49558f0dcbec64129 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 19 Aug 2024 17:04:54 -0500 Subject: [PATCH 03/11] Convert object for fill_value using asarray for requested type This relies on NumPy to raise OverflowError if the object can not be represented by requested data type. --- dpctl/tensor/_ctors.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dpctl/tensor/_ctors.py b/dpctl/tensor/_ctors.py index 6c5f3888a6..90a6208c87 100644 --- a/dpctl/tensor/_ctors.py +++ b/dpctl/tensor/_ctors.py @@ -765,8 +765,11 @@ def _get_arange_length(start, stop, step): def _to_scalar(obj, sc_ty): - "A way to convert object to NumPy scalar type" - zd_arr = np.asarray(obj).astype(sc_ty, casting="unsafe") + """A way to convert object to NumPy scalar type. + Raises OverflowError if obj can not be represented + using the requested scalar type. + """ + zd_arr = np.asarray(obj, dtype=sc_ty) return zd_arr[tuple()] From abfee14bb3a5943bcedd67cb7213627476163993 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 19 Aug 2024 17:10:38 -0500 Subject: [PATCH 04/11] Check that full(shape, fill_val, dtype=dt) handles out of bound integral fill values It should raise OverflowError, like NumPy 2 does. --- dpctl/tests/test_usm_ndarray_ctor.py | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/dpctl/tests/test_usm_ndarray_ctor.py b/dpctl/tests/test_usm_ndarray_ctor.py index 739f863b27..1ba1db1bff 100644 --- a/dpctl/tests/test_usm_ndarray_ctor.py +++ b/dpctl/tests/test_usm_ndarray_ctor.py @@ -1791,29 +1791,14 @@ def test_full_strides(): assert np.array_equal(dpt.asnumpy(X), Xnp) -def _reduce_to_idtype_range(o, dtype): - assert isinstance(dtype, np.dtype) - assert dtype.kind in "ui" - _iinfo = np.iinfo(dtype) - _max = _iinfo.max - _min = _iinfo.min - return ((o - _min) % (1 + _max - _min)) + _min - - -def test_full_gh_1230(): +@pytest.mark.parametrize("dt", ["i1", "u1", "i2", "u2", "i4", "u4", "i8", "u8"]) +def test_full_gh_1230(dt): q = get_queue_or_skip() dtype = "i4" dt_maxint = dpt.iinfo(dtype).max - X = dpt.full(1, dt_maxint + 1, dtype=dtype, sycl_queue=q) - X_np = dpt.asnumpy(X) - assert X.dtype == dpt.dtype(dtype) - expected = np.full_like( - X_np, _reduce_to_idtype_range(dt_maxint + 1, X.dtype) - ) - assert np.array_equal(X_np, expected) with pytest.raises(OverflowError): - dpt.full(1, dpt.iinfo(dpt.uint64).max + 1, sycl_queue=q) + dpt.full(1, dt_maxint + 1, dtype=dtype, sycl_queue=q) @pytest.mark.parametrize( From a53ca0625da705b10959049aa7f4bceb9259b7ce Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 20 Aug 2024 08:09:15 -0500 Subject: [PATCH 05/11] test_full_gh_1230 to be have differently dependending on NumPy version For 2.0.0 and later, it is to raise OverflowError, otherwise it is expected to wrap up the out-of-bounds integral fill-in values given as Python scalars. --- dpctl/tests/test_usm_ndarray_ctor.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/dpctl/tests/test_usm_ndarray_ctor.py b/dpctl/tests/test_usm_ndarray_ctor.py index 1ba1db1bff..4d137a4c05 100644 --- a/dpctl/tests/test_usm_ndarray_ctor.py +++ b/dpctl/tests/test_usm_ndarray_ctor.py @@ -1793,12 +1793,17 @@ def test_full_strides(): @pytest.mark.parametrize("dt", ["i1", "u1", "i2", "u2", "i4", "u4", "i8", "u8"]) def test_full_gh_1230(dt): - q = get_queue_or_skip() - dtype = "i4" + get_queue_or_skip() + dtype = dpt.dtype(dt) dt_maxint = dpt.iinfo(dtype).max - with pytest.raises(OverflowError): - dpt.full(1, dt_maxint + 1, dtype=dtype, sycl_queue=q) + if (dtype.itemsize < 8) and (np.lib.NumpyVersion(np.__version__) < "2.0.0"): + X = dpt.full(1, fill_value=(dt_maxint + 1), dtype=dt) + Y = dpt.full_like(X, fill_value=dpt.iinfo(dt).min) + assert dpt.all(X == Y) + else: + with pytest.raises(OverflowError): + dpt.full(1, dt_maxint + 1, dtype=dt) @pytest.mark.parametrize( From f441f015b49de00a2a6ffd57da01718b3e9b68b9 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 20 Aug 2024 06:39:52 -0500 Subject: [PATCH 06/11] Fix test_create_program_from_source to not use out-of-bounds fill-value Restrict the input size to allow math the test relies on to work without using out-of-bounds Python scalar inputs to tensor.arange --- dpctl/tests/test_sycl_kernel_submit.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dpctl/tests/test_sycl_kernel_submit.py b/dpctl/tests/test_sycl_kernel_submit.py index 31ae7713ff..839d84bbe8 100644 --- a/dpctl/tests/test_sycl_kernel_submit.py +++ b/dpctl/tests/test_sycl_kernel_submit.py @@ -66,6 +66,9 @@ def test_create_program_from_source(ctype_str, dtype, ctypes_ctor): n_elems = 1024 * 512 lws = 128 + if dtype.kind in "ui": + n_elems = min(n_elems, dpt.iinfo(dtype).max) + n_elems = (n_elems // lws) * lws a = dpt.arange(n_elems, dtype=dtype, sycl_queue=q) b = dpt.arange(n_elems, stop=0, step=-1, dtype=dtype, sycl_queue=q) c = dpt.zeros(n_elems, dtype=dtype, sycl_queue=q) From 032ceca49605ebe63ddb51c22652a21949497d6c Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 20 Aug 2024 06:43:22 -0500 Subject: [PATCH 07/11] Fix test_matmul_simple to avoid use of out-of-bounds Python scalars Change the test so that input matrices that get multiplied only have blocks of ones no larger than the max integer for the type, rest is populated with zeros. This change applies to integral types only. --- dpctl/tests/test_usm_ndarray_linalg.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/dpctl/tests/test_usm_ndarray_linalg.py b/dpctl/tests/test_usm_ndarray_linalg.py index c36c195769..e2aa23873f 100644 --- a/dpctl/tests/test_usm_ndarray_linalg.py +++ b/dpctl/tests/test_usm_ndarray_linalg.py @@ -89,12 +89,20 @@ def test_matmul_simple(dtype): skip_if_dtype_not_supported(dtype, q) n, m = 235, 17 - m1 = dpt.ones((m, n), dtype=dtype) - m2 = dpt.ones((n, m), dtype=dtype) + m1 = dpt.zeros((m, n), dtype=dtype) + m2 = dpt.zeros((n, m), dtype=dtype) + + dt = m1.dtype + if dt.kind in "ui": + n1 = min(n, dpt.iinfo(dt).max) + else: + n1 = n + m1[:, :n1] = dpt.ones((m, n1), dtype=dt) + m2[:n1, :] = dpt.ones((n1, m), dtype=dt) for k in [1, 2, 3, 4, 7, 8, 9, 15, 16, 17]: r = dpt.matmul(m1[:k, :], m2[:, :k]) - assert dpt.all(r == dpt.full((k, k), n, dtype=dtype)) + assert dpt.all(r == dpt.full((k, k), fill_value=n1, dtype=dt)) @pytest.mark.parametrize("dtype", _numeric_types) From 2be49897fd4420e906a53722800fd7918ef31346 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 20 Aug 2024 11:59:59 -0500 Subject: [PATCH 08/11] Pin setuptools to <72.2 in hope to fix test_examples_linux workflow --- .github/workflows/conda-package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml index 02e43558aa..ebd8fb136a 100644 --- a/.github/workflows/conda-package.yml +++ b/.github/workflows/conda-package.yml @@ -490,8 +490,8 @@ jobs: run: | CHANNELS="${{ env.CHANNELS }}" . $CONDA/etc/profile.d/conda.sh - conda create -n ${{ env.EXAMPLES_ENV_NAME }} -y pytest python=${{ matrix.python }} $CHANNELS - conda install -n ${{ env.EXAMPLES_ENV_NAME }} -y cmake">=3.22" $CHANNELS || exit 1 + conda create -n ${{ env.EXAMPLES_ENV_NAME }} -y pytest python=${{ matrix.python }} setuptools"<72.2.0" $CHANNELS + conda install -n ${{ env.EXAMPLES_ENV_NAME }} -y cmake $CHANNELS || exit 1 conda install -n ${{ env.EXAMPLES_ENV_NAME }} -y ninja $CHANNELS || exit 1 conda install -n ${{ env.EXAMPLES_ENV_NAME }} -y pybind11 cython scikit-build $CHANNELS || exit 1 conda install -n ${{ env.EXAMPLES_ENV_NAME }} -y mkl-dpcpp mkl-devel-dpcpp dpcpp_cpp_rt $CHANNELS || exit 1 From 7c461f11ba3145a83d42114398fb9c9bd5a8b35b Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 20 Aug 2024 08:04:18 -0500 Subject: [PATCH 09/11] Use Python 3.12 to run OS LLVM workflow --- .github/workflows/os-llvm-sycl-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/os-llvm-sycl-build.yml b/.github/workflows/os-llvm-sycl-build.yml index b8870bddef..02c8c7249a 100644 --- a/.github/workflows/os-llvm-sycl-build.yml +++ b/.github/workflows/os-llvm-sycl-build.yml @@ -100,7 +100,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '3.12' architecture: x64 - name: Install dpctl dependencies From 04f8a4cf7232a655b7f6ecc62b21e28572026d85 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 20 Aug 2024 08:06:47 -0500 Subject: [PATCH 10/11] Update components in collect coverage Use Python 3.12, install recent NumPy (instead of 1.25), update gtest from 1.13 to 1.15.2 --- .github/workflows/generate-coverage.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/generate-coverage.yaml b/.github/workflows/generate-coverage.yaml index 3d82bdd4fc..6e2de07baa 100644 --- a/.github/workflows/generate-coverage.yaml +++ b/.github/workflows/generate-coverage.yaml @@ -15,7 +15,7 @@ jobs: env: ONEAPI_ROOT: /opt/intel/oneapi - GTEST_ROOT: /home/runner/work/googletest-1.13.0/install + GTEST_ROOT: /home/runner/work/googletest-1.15.2/install # Use oneAPI compiler 2023 to work around an issue USE_2023: 0 @@ -52,7 +52,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '3.12' architecture: x64 - name: Cache Gtest @@ -60,8 +60,8 @@ jobs: uses: actions/cache@v4 with: path: | - /home/runner/work/googletest-1.13.0/install - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('/home/runner/work/googletest-1.13.0/install/include/gtest/*') }} + /home/runner/work/googletest-1.15.2/install + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('/home/runner/work/googletest-1.15.2/install/include/gtest/*') }} restore-keys: | ${{ runner.os }}-build-${{ env.cache-name }}- ${{ runner.os }}-build- @@ -72,12 +72,12 @@ jobs: shell: bash -l {0} run: | cd /home/runner/work - wget https://github.com/google/googletest/archive/refs/tags/v1.13.0.tar.gz - tar xf v1.13.0.tar.gz - cd googletest-1.13.0 + wget https://github.com/google/googletest/archive/refs/tags/v1.15.2.tar.gz + tar xf v1.15.2.tar.gz + cd googletest-1.15.2 mkdir build cd build - cmake .. -DCMAKE_INSTALL_PREFIX=/home/runner/work/googletest-1.13.0/install + cmake .. -DCMAKE_INSTALL_PREFIX=/home/runner/work/googletest-1.15.2/install make && make install - name: Checkout repo @@ -92,7 +92,7 @@ jobs: - name: Install dpctl dependencies shell: bash -l {0} run: | - pip install numpy"<1.26.0" cython setuptools pytest pytest-cov scikit-build cmake coverage[toml] versioneer[toml]==0.29 + pip install numpy cython setuptools pytest pytest-cov scikit-build cmake coverage[toml] versioneer[toml]==0.29 - name: Build dpctl with coverage shell: bash -l {0} From 5e6271d6b3a9889338aacfcf001d8447b26910d6 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Tue, 20 Aug 2024 16:47:10 -0500 Subject: [PATCH 11/11] Be forgiving for test_fill_gh_1230 with older numpy versions --- dpctl/tests/test_usm_ndarray_ctor.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dpctl/tests/test_usm_ndarray_ctor.py b/dpctl/tests/test_usm_ndarray_ctor.py index 4d137a4c05..f43f931e47 100644 --- a/dpctl/tests/test_usm_ndarray_ctor.py +++ b/dpctl/tests/test_usm_ndarray_ctor.py @@ -1798,7 +1798,10 @@ def test_full_gh_1230(dt): dt_maxint = dpt.iinfo(dtype).max if (dtype.itemsize < 8) and (np.lib.NumpyVersion(np.__version__) < "2.0.0"): - X = dpt.full(1, fill_value=(dt_maxint + 1), dtype=dt) + try: + X = dpt.full(1, fill_value=(dt_maxint + 1), dtype=dt) + except OverflowError: + pytest.skip("Expected OverflowError raised") Y = dpt.full_like(X, fill_value=dpt.iinfo(dt).min) assert dpt.all(X == Y) else: