From c4939837a2a92fe0e126634c11f41e09b416c30b Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 14 Sep 2020 10:28:43 -0500 Subject: [PATCH 1/6] Implemented suggestion #10 ``` In [1]: import mkl_random In [2]: rs = mkl_random.RandomState(1234, brng='NONDETERM') In [3]: rs Out[3]: In [4]: rs.get_state() Out[4]: ('NON_DETERMINISTIC', b'\x02RNG\x14\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\xe0\x00\x00\x00\x00\x00\n\x00\x00\x00') In [6]: rs.seed(126) # brng has not been reset In [7]: rs.get_state() Out[7]: ('NON_DETERMINISTIC', b'\x02RNG\x14\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\xe0\x00\x00\x00\x00\x00\n\x00\x00\x00') In [8]: rs.seed(1234, brng='MT2203') In [9]: rs.get_state() Out[9]: ('MT2203', b"\x02RNG\x14\x00\x00\x00(\x01\x00\x00\x00\x00\x00\x00\x00\x00\x90\x00\x00\x00\x00\x80i\xc0w\xa7\x0eF\xd8L\xfbi\rk\xd6[\t\x80F\xa0\xae\xa2as3\x9eQ\xd5f\x9b\x95>\xb9\x0f\xf5\xbe\xb0p\xe7o\xf e\xe22\xc3\xad\x80D?\xe3X\x14\x86(D\xed(\xb1\xeb\xc6\xc5\xae\x00\xf8m,@\xd8\xd2\xd2\x98\x8b\xe74\x16\xd6\x07\xe3\xa9\x94q\xe5\xf1\xe1\xd8\x94\xf6\xf3\x8e8z\xddYrMym\xd9\x9a\xf1\x1e\xa8v\x97\ x1a\xcdvY\x82\xe9\x086!\xc8\x8db\xe5<\x1aH\xac&\x1c\xf8\x87\xc3\xefm\xc6\x17q\xb9\xda\xdcw\xcd.\xffI!3\xe8\x8e\xd5\x89\x19=\xca\x94\x88\xb2e|\xe5\xa0\xf0\xe3\xe5\xea\x0f\xd7K\xd4\xf4\xe2\x17 _\x1e\x89\x8f\xe7\x9dp\xa8}B\xa22l\x1c\xed\xedvtu6\xf3up\xc2W\xa8\xd9\xcf\xb59\r\xa6@\xce\xc7\x7f\xc2\x05[\xed\xd4y:h&\x8b\xe4\xc9\xcc\xa6\xf9\x07'\xa8g4\xd7\t\x07\xcf#\x94\xe8%\xa6\xa1\xcd\ xc1nd\xcc\x92\x80n\xe1i\x8a\xdc\x0b\x0e|\xc1[\xe2QWX\x98\xc6\x9f\x08\xe3\x1e\x00 W\xd6\xf7\x9c+\x11\xfa\xee\xbf\nH\xe6nmE\x00\x00\x00_\x05\x81\xaf\x80\xef~\xdd\x00\x80\xd7\xef\x08\x01\x0c\x0 2") ``` --- mkl_random/_version.py | 2 +- mkl_random/mklrand.pyx | 32 ++++++++++++++++++-------------- mkl_random/src/numpy.pxd | 4 ++-- mkl_random/src/randomkit.c | 21 +++++++++++++++++++++ mkl_random/src/randomkit.h | 1 + 5 files changed, 43 insertions(+), 17 deletions(-) diff --git a/mkl_random/_version.py b/mkl_random/_version.py index a842075..3c3f0a3 100644 --- a/mkl_random/_version.py +++ b/mkl_random/_version.py @@ -1,2 +1,2 @@ -__version__ = '1.1.1' +__version__ = '1.2.0' diff --git a/mkl_random/mklrand.pyx b/mkl_random/mklrand.pyx index 56458c6..e241e47 100644 --- a/mkl_random/mklrand.pyx +++ b/mkl_random/mklrand.pyx @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2017-2019, Intel Corporation +# Copyright (c) 2017-2020, Intel Corporation # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -24,9 +24,6 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import -include "numpy.pxd" - cdef extern from "Python.h": void* PyMem_Malloc(size_t n) void PyMem_Free(void* buf) @@ -38,6 +35,7 @@ cdef extern from "Python.h": void PyErr_Clear() +include "numpy.pxd" from libc.string cimport memset, memcpy cdef extern from "math.h": @@ -83,6 +81,7 @@ cdef extern from "randomkit.h": void irk_get_state_mkl(irk_state * state, char * buf) int irk_set_state_mkl(irk_state * state, char * buf) int irk_get_brng_mkl(irk_state *state) nogil + int irk_get_brng_and_stream_mkl(irk_state *state, unsigned int * stream_id) nogil int irk_leapfrog_stream_mkl(irk_state *state, int k, int nstreams) nogil int irk_skipahead_stream_mkl(irk_state *state, long long int nskips) nogil @@ -181,8 +180,9 @@ ctypedef void (* irk_discd_long_vec)(irk_state *state, npy_intp len, long *res, ctypedef void (* irk_discdptr_vec)(irk_state *state, npy_intp len, int *res, double *a) nogil -# Initialize numpy -import_array() +cdef int r = _import_array() +if (r<0): + raise ImportError("Failed to import NumPy") cimport cython import numpy as np @@ -888,7 +888,7 @@ _brng_dict_stream_max = { NONDETERM: 1, } -def _default_fallback_brng_token_(brng): +cdef irk_brng_t _default_fallback_brng_token_(brng): cdef irk_brng_t brng_token warnings.warn(("The basic random generator specification {given} is not recognized. " "\"MT19937\" will be used instead").format(given=brng), @@ -896,7 +896,7 @@ def _default_fallback_brng_token_(brng): brng_token = MT19937 return brng_token -def _parse_brng_token_(brng): +cdef irk_brng_t _parse_brng_token_(brng): cdef irk_brng_t brng_token if isinstance(brng, str): @@ -946,7 +946,7 @@ cdef class RandomState: """ RandomState(seed=None, brng='MT19937') - Container for the Mersenne Twister pseudo-random number generator. + Container for the Intel(R) MKL-powered (pseudo-)random number generators. `RandomState` exposes a number of methods for generating random numbers drawn from a variety of probability distributions. In addition to the @@ -1008,9 +1008,9 @@ cdef class RandomState: PyMem_Free(self.internal_state) self.internal_state = NULL - def seed(self, seed=None, brng='MT19937'): + def seed(self, seed=None, brng=None): """ - seed(seed=None, brng='MT19937') + seed(seed=None, brng=None) Seed the generator. @@ -1023,9 +1023,10 @@ cdef class RandomState: Seed for `RandomState`. Must be convertible to 32 bit unsigned integers. brng : {'MT19937', 'SFMT19937', 'MT2203', 'R250', 'WH', 'MCG31', - 'MCG59', 'MRG32K3A', 'PHILOX4X32X10', 'NONDETERM'}, optional + 'MCG59', 'MRG32K3A', 'PHILOX4X32X10', 'NONDETERM', None}, optional Basic pseudo-random number generation algorithms, provided by - Intel MKL. The default choice is 'MT19937' - the Mersenne Twister. + Intel MKL. Use `brng==None` to keep the `brng` specified to construct + the class instance. See Also -------- @@ -1041,7 +1042,10 @@ cdef class RandomState: cdef unsigned int stream_id cdef ndarray obj "arrayObject_obj" - brng_token, stream_id = _parse_brng_argument(brng); + if (brng): + brng_token, stream_id = _parse_brng_argument(brng); + else: + brng_token = irk_get_brng_and_stream_mkl(self.internal_state, &stream_id) try: if seed is None: with self.lock: diff --git a/mkl_random/src/numpy.pxd b/mkl_random/src/numpy.pxd index d5b0d74..fa8af74 100644 --- a/mkl_random/src/numpy.pxd +++ b/mkl_random/src/numpy.pxd @@ -133,8 +133,6 @@ cdef extern from "numpy/arrayobject.h": dtype PyArray_DescrFromType(int) - void import_array() - # include functions that were once macros in the new api int PyArray_NDIM(ndarray arr) @@ -150,3 +148,5 @@ cdef extern from "numpy/arrayobject.h": int PyArray_TYPE(ndarray arr) int PyArray_CHKFLAGS(ndarray arr, int flags) object PyArray_GETITEM(ndarray arr, char *itemptr) + + int _import_array() diff --git a/mkl_random/src/randomkit.c b/mkl_random/src/randomkit.c index 322efdb..b931291 100644 --- a/mkl_random/src/randomkit.c +++ b/mkl_random/src/randomkit.c @@ -145,6 +145,27 @@ int irk_get_brng_mkl(irk_state *state) return -1; } +int irk_get_brng_and_stream_mkl(irk_state *state, unsigned int* stream_id) +{ + int i, mkl_brng_id = vslGetStreamStateBrng(state->stream); + + if ((VSL_BRNG_MT2203 <= mkl_brng_id) && (mkl_brng_id < VSL_BRNG_MT2203 + SIZE_OF_MT2203_FAMILY)) { + *stream_id = (unsigned int)(mkl_brng_id - VSL_BRNG_MT2203); + mkl_brng_id = VSL_BRNG_MT2203; + } else if ((VSL_BRNG_WH <= mkl_brng_id ) && (mkl_brng_id < VSL_BRNG_WH + SIZE_OF_WH_FAMILY)) { + *stream_id = (unsigned int)(mkl_brng_id - VSL_BRNG_WH); + mkl_brng_id = VSL_BRNG_WH; + } + + for(i = 0; i < BRNG_KINDS; i++) + if(mkl_brng_id == brng_list[i]) { + *stream_id = (unsigned int)(0); + return i; + } + + return -1; +} + void irk_seed_mkl(irk_state *state, const unsigned int seed, const irk_brng_t brng, const unsigned int stream_id) { VSLStreamStatePtr stream_loc; diff --git a/mkl_random/src/randomkit.h b/mkl_random/src/randomkit.h index 0ae60bd..fee28db 100644 --- a/mkl_random/src/randomkit.h +++ b/mkl_random/src/randomkit.h @@ -97,6 +97,7 @@ extern int irk_get_stream_size(irk_state *state); extern void irk_get_state_mkl(irk_state *state, char * buf); extern int irk_set_state_mkl(irk_state *state, char * buf); extern int irk_get_brng_mkl(irk_state *state); +extern int irk_get_brng_and_stream_mkl(irk_state *state, unsigned int* stream_id); extern int irk_leapfrog_stream_mkl(irk_state *state, const int k, const int nstreams); extern int irk_skipahead_stream_mkl(irk_state *state, const long long int nskip); From 5354d575dce93d5489c542ee1e773c517b732423 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 14 Sep 2020 10:57:29 -0500 Subject: [PATCH 2/6] no need to build for Python 2 any more --- .travis.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 874f8c3..6bad04e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,23 +1,10 @@ matrix: include: - - name: "Linux-Py2" - os: linux - env: - - MINICONDA=Miniconda2-latest-Linux-x86_64.sh - - PYVER="--python=27" - name: "Linux-Py3" os: linux env: - MINICONDA=Miniconda3-latest-Linux-x86_64.sh - PYVER="" - - name: "OsX-Py2" - os: osx - osx_image: xcode8 - env: - - MATRIX_EVAL="brew install gcc && CC=gcc-7 && CXX=g++-7" - - MINICONDA=Miniconda3-latest-MacOSX-x86_64.sh - - MACOSX_DEPLOYMENT_TARGET="10.9" - - PYVER="--python=27" - name: "OsX-Py3" os: osx osx_image: xcode8 From 239a8af096e60a261106e36b7b9e36558893d0cc Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 14 Sep 2020 11:22:27 -0500 Subject: [PATCH 3/6] updating build scripts to set MKLROOT variable --- conda-recipe/bld.bat | 1 + conda-recipe/build.sh | 2 +- conda-recipe/meta.yaml | 10 +--------- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/conda-recipe/bld.bat b/conda-recipe/bld.bat index d6dc348..c639b91 100644 --- a/conda-recipe/bld.bat +++ b/conda-recipe/bld.bat @@ -1,4 +1,5 @@ @rem Remember to source the compiler +set MKLROOT=%CONDA_PREFIX% %PYTHON% setup.py install --old-and-unmanageable if errorlevel 1 exit 1 diff --git a/conda-recipe/build.sh b/conda-recipe/build.sh index d3ba124..0429491 100644 --- a/conda-recipe/build.sh +++ b/conda-recipe/build.sh @@ -1,4 +1,4 @@ #!/bin/bash -x export CFLAGS="-I$PREFIX/include $CFLAGS" -$PYTHON setup.py install --old-and-unmanageable +MKLROOT=$CONDA_PREFIX $PYTHON setup.py install --old-and-unmanageable diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 3c3ebde..3203798 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -14,15 +14,7 @@ source: build: number: {{buildnumber}} - skip: True # [win and py27] - always_include_files: - - {{ SP_DIR.replace('\\', '/') if win else SP_DIR }}/mkl_random/__init__.py - - {{ SP_DIR.replace('\\', '/') if win else SP_DIR }}/mkl_random/mklrand.* - - {{ SP_DIR.replace('\\', '/') if win else SP_DIR }}/mkl_random/setup.py - - {{ SP_DIR.replace('\\', '/') if win else SP_DIR }}/mkl_random/tests/test_random.py - - {{ SP_DIR.replace('\\', '/') if win else SP_DIR }}/mkl_random/tests/test_regressions.py - - {{ SP_DIR.replace('\\', '/') if win else SP_DIR }}/mkl_random/__pycache__/* - - {{ SP_DIR.replace('\\', '/') if win else SP_DIR }}/mkl_random/tests/__pycache__/* + requirements: build: From 4e0d22711d53be8987d342d7fe49936825291a0d Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 14 Sep 2020 13:07:21 -0500 Subject: [PATCH 4/6] skipping test known to fail --- conda-recipe/meta.yaml | 5 ++++- mkl_random/tests/test_regression.py | 7 ++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 3203798..f44b9f5 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -1,4 +1,4 @@ -{% set version = "1.1.0" %} +{% set version = "1.2.0" %} {% set buildnumber = 0 %} ### If you change the iccver here, you must also set the path correctly in build.sh / bld.bat!!! @@ -14,6 +14,8 @@ source: build: number: {{buildnumber}} + ignore_run_exports: + - blas requirements: @@ -36,6 +38,7 @@ test: - nosetests -v mkl_random requires: - nose + - mkl-service - numpy >=1.14 imports: - mkl_random diff --git a/mkl_random/tests/test_regression.py b/mkl_random/tests/test_regression.py index 46fc225..8e39cc9 100644 --- a/mkl_random/tests/test_regression.py +++ b/mkl_random/tests/test_regression.py @@ -24,11 +24,10 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from __future__ import division, absolute_import, print_function - import sys from numpy.testing import (TestCase, run_module_suite, assert_, - assert_array_equal, assert_raises) + assert_array_equal, assert_raises, dec) +import mkl import mkl_random as rnd from numpy.compat import long import numpy as np @@ -118,6 +117,8 @@ def test_multivariate_normal_size_types(self): rnd.multivariate_normal([0], [[0]], size=np.int_(1)) rnd.multivariate_normal([0], [[0]], size=np.int64(1)) + @dec.skipif(tuple(map(mkl.get_version().get, ['MajorVersion', 'UpdateVersion'])) == (2020,3), + msg="Intel(R) MKL 2020.3 produces NaN for these parameters") def test_beta_small_parameters(self): # Test that beta with small a and b parameters does not produce # NaNs due to roundoff errors causing 0 / 0, gh-5851 From 1ec0e71f0e30e5f03218f8b847166e3c323e0e84 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 14 Sep 2020 13:44:33 -0500 Subject: [PATCH 5/6] work-around issue with pkgs/main::conda-build --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6bad04e..0deec9d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ install: - hash -r - conda config --set always_yes yes --set changeps1 no - conda update -q conda - - conda install conda-build + - conda install -c conda-forge conda-forge::conda-build # Useful for debugging any issues with conda - conda info -a - gcc -v From 34de589a02234b4cb2f0465a04ce9357a7c432ec Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 14 Sep 2020 14:05:08 -0500 Subject: [PATCH 6/6] Use numpy 1.16 for building --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0deec9d..7551481 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,4 +28,4 @@ install: - g++ -v script: - - conda build -c intel -c conda-forge $PYVER conda-recipe + - conda build -c intel -c conda-forge --numpy=1.16 $PYVER conda-recipe