diff --git a/appveyor.yml b/appveyor.yml index 7557d85203..9925436c69 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -28,7 +28,7 @@ install: - cmd: conda info -a # install depenencies - - cmd: conda create -n test_env --yes --quiet python=%PYTHON_VERSION% pip numpy scipy pandas nose pytest pytz ephem numba siphon -c conda-forge + - cmd: conda create -n test_env --yes --quiet python=%PYTHON_VERSION% pip numpy scipy pytables pandas nose pytest pytz ephem numba siphon -c conda-forge - cmd: activate test_env - cmd: python --version - cmd: conda list diff --git a/ci/requirements-py27.yml b/ci/requirements-py27.yml index dd93786152..e6189d073b 100644 --- a/ci/requirements-py27.yml +++ b/ci/requirements-py27.yml @@ -6,6 +6,7 @@ dependencies: - python=2.7 - numpy - scipy + - pytables - pandas - pytz - ephem diff --git a/ci/requirements-py34.yml b/ci/requirements-py34.yml index 91ba94f11e..3ad4dd7e65 100644 --- a/ci/requirements-py34.yml +++ b/ci/requirements-py34.yml @@ -6,6 +6,7 @@ dependencies: - python=3.4 - numpy - scipy + - pytables - pandas - pytz - ephem diff --git a/ci/requirements-py35.yml b/ci/requirements-py35.yml index 8f8c648517..8e7678fab0 100644 --- a/ci/requirements-py35.yml +++ b/ci/requirements-py35.yml @@ -6,6 +6,7 @@ dependencies: - python=3.5 - numpy - scipy + - pytables - pandas - pytz - ephem diff --git a/ci/requirements-py36.yml b/ci/requirements-py36.yml index a89b2a9c35..ba8e9cdda5 100644 --- a/ci/requirements-py36.yml +++ b/ci/requirements-py36.yml @@ -6,6 +6,7 @@ dependencies: - python=3.6 - numpy - scipy + - pytables - pandas - pytz #- ephem diff --git a/docs/environment.yml b/docs/environment.yml index 998f699a81..4c446addbd 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -7,6 +7,7 @@ dependencies: - mock # needed for local python 2.7 builds - numpy=1.11.2 - scipy + - tables - pandas=0.19.1 - pytz - ephem diff --git a/docs/sphinx/source/clearsky.rst b/docs/sphinx/source/clearsky.rst index 79afbcb238..c84cc857b6 100644 --- a/docs/sphinx/source/clearsky.rst +++ b/docs/sphinx/source/clearsky.rst @@ -136,24 +136,24 @@ the year. You could run it in a loop to create plots for all months. In [1]: import os - In [1]: import scipy.io + In [1]: import tables In [1]: pvlib_path = os.path.dirname(os.path.abspath(pvlib.clearsky.__file__)) - In [1]: filepath = os.path.join(pvlib_path, 'data', 'LinkeTurbidities.mat') - - In [1]: mat = scipy.io.loadmat(filepath) + In [1]: filepath = os.path.join(pvlib_path, 'data', 'LinkeTurbidities.h5') # data is in units of 20 x turbidity - In [1]: linke_turbidity_table = mat['LinkeTurbidity'] # / 20. # crashes on rtd + In [1]: lt_h5_file = tables.open_file(filepath) In [1]: def plot_turbidity_map(month, vmin=1, vmax=100): ...: plt.figure(); - ...: plt.imshow(linke_turbidity_table[:, :, month-1], vmin=vmin, vmax=vmax); + ...: plt.imshow(lt_h5_file.root.LinkeTurbidity[:, :, month-1], vmin=vmin, vmax=vmax); ...: plt.title('Linke turbidity x 20, ' + calendar.month_name[month]); ...: plt.colorbar(shrink=0.5); ...: plt.tight_layout(); + In [1]: lt_h5_file.close() + @savefig turbidity-1.png width=10in In [1]: plot_turbidity_map(1) diff --git a/docs/sphinx/source/whatsnew/v0.5.2.rst b/docs/sphinx/source/whatsnew/v0.5.2.rst index 3cdabf5942..9fb4667a57 100644 --- a/docs/sphinx/source/whatsnew/v0.5.2.rst +++ b/docs/sphinx/source/whatsnew/v0.5.2.rst @@ -9,7 +9,8 @@ API Changes Enhancements ~~~~~~~~~~~~ -* +* Improve clearsky.lookup_linke_turbidity speed, changing .mat source + file to .h5 (:issue:`437`) Bug fixes ~~~~~~~~~ @@ -42,5 +43,4 @@ Contributors * Will Holmgren * KonstantinTr * Anton Driesse - - +* Cedric Leroy diff --git a/pvlib/clearsky.py b/pvlib/clearsky.py index b43849e311..75f598baeb 100644 --- a/pvlib/clearsky.py +++ b/pvlib/clearsky.py @@ -153,7 +153,7 @@ def ineichen(apparent_zenith, airmass_absolute, linke_turbidity, def lookup_linke_turbidity(time, latitude, longitude, filepath=None, interp_turbidity=True): """ - Look up the Linke Turibidity from the ``LinkeTurbidities.mat`` + Look up the Linke Turibidity from the ``LinkeTurbidities.h5`` data file supplied with pvlib. Parameters @@ -165,18 +165,18 @@ def lookup_linke_turbidity(time, latitude, longitude, filepath=None, longitude : float filepath : None or string, default None - The path to the ``.mat`` file. + The path to the ``.h5`` file. interp_turbidity : bool, default True If ``True``, interpolates the monthly Linke turbidity values - found in ``LinkeTurbidities.mat`` to daily values. + found in ``LinkeTurbidities.h5`` to daily values. Returns ------- turbidity : Series """ - # The .mat file 'LinkeTurbidities.mat' contains a single 2160 x 4320 x 12 + # The .h5 file 'LinkeTurbidities.h5' contains a single 2160 x 4320 x 12 # matrix of type uint8 called 'LinkeTurbidity'. The rows represent global # latitudes from 90 to -90 degrees; the columns represent global longitudes # from -180 to 180; and the depth (third dimension) represents months of @@ -194,18 +194,15 @@ def lookup_linke_turbidity(time, latitude, longitude, filepath=None, # 1st column: 179.9583 W, 2nd column: 179.875 W try: - import scipy.io + import tables except ImportError: - raise ImportError('The Linke turbidity lookup table requires scipy. ' + raise ImportError('The Linke turbidity lookup table requires tables. ' 'You can still use clearsky.ineichen if you ' 'supply your own turbidities.') if filepath is None: pvlib_path = os.path.dirname(os.path.abspath(__file__)) - filepath = os.path.join(pvlib_path, 'data', 'LinkeTurbidities.mat') - - mat = scipy.io.loadmat(filepath) - linke_turbidity_table = mat['LinkeTurbidity'] + filepath = os.path.join(pvlib_path, 'data', 'LinkeTurbidities.h5') latitude_index = ( np.around(_linearly_scale(latitude, 90, -90, 0, 2160)) @@ -214,7 +211,14 @@ def lookup_linke_turbidity(time, latitude, longitude, filepath=None, np.around(_linearly_scale(longitude, -180, 180, 0, 4320)) .astype(np.int64)) - lts = linke_turbidity_table[latitude_index][longitude_index] + lt_h5_file = tables.open_file(filepath) + try: + lts = lt_h5_file.root.LinkeTurbidity[latitude_index, longitude_index, :] + except IndexError: + raise IndexError('Latitude should be between 90 and -90, ' + 'longitude between -180 and 180.') + finally: + lt_h5_file.close() if interp_turbidity: linke_turbidity = _interpolate_turbidity(lts, time) diff --git a/pvlib/data/LinkeTurbidities.mat b/pvlib/data/LinkeTurbidities.h5 similarity index 57% rename from pvlib/data/LinkeTurbidities.mat rename to pvlib/data/LinkeTurbidities.h5 index cfe2069089..6f73de334f 100644 Binary files a/pvlib/data/LinkeTurbidities.mat and b/pvlib/data/LinkeTurbidities.h5 differ diff --git a/pvlib/test/conftest.py b/pvlib/test/conftest.py index a706b9ab8c..acfa06e489 100644 --- a/pvlib/test/conftest.py +++ b/pvlib/test/conftest.py @@ -19,6 +19,15 @@ requires_scipy = pytest.mark.skipif(not has_scipy, reason='requires scipy') +try: + import tables + has_tables = True +except ImportError: + has_tables = False + +requires_tables = pytest.mark.skipif(not has_tables, reason='requires tables') + + try: import ephem has_ephem = True diff --git a/pvlib/test/test_clearsky.py b/pvlib/test/test_clearsky.py index 8900153041..e858cdf0d3 100644 --- a/pvlib/test/test_clearsky.py +++ b/pvlib/test/test_clearsky.py @@ -16,7 +16,7 @@ from pvlib import atmosphere from pvlib import irradiance -from conftest import requires_scipy +from conftest import requires_scipy, requires_tables def test_ineichen_series(): @@ -158,7 +158,7 @@ def test_ineichen_altitude(): assert_frame_equal(expected, out) -@requires_scipy +@requires_tables def test_lookup_linke_turbidity(): times = pd.date_range(start='2014-06-24', end='2014-06-25', freq='12h', tz='America/Phoenix') @@ -171,7 +171,7 @@ def test_lookup_linke_turbidity(): assert_series_equal(expected, out) -@requires_scipy +@requires_tables def test_lookup_linke_turbidity_leapyear(): times = pd.date_range(start='2016-06-24', end='2016-06-25', freq='12h', tz='America/Phoenix') @@ -184,7 +184,7 @@ def test_lookup_linke_turbidity_leapyear(): assert_series_equal(expected, out) -@requires_scipy +@requires_tables def test_lookup_linke_turbidity_nointerp(): times = pd.date_range(start='2014-06-24', end='2014-06-25', freq='12h', tz='America/Phoenix') @@ -195,7 +195,7 @@ def test_lookup_linke_turbidity_nointerp(): assert_series_equal(expected, out) -@requires_scipy +@requires_tables def test_lookup_linke_turbidity_months(): times = pd.date_range(start='2014-04-01', end='2014-07-01', freq='1M', tz='America/Phoenix') @@ -206,7 +206,7 @@ def test_lookup_linke_turbidity_months(): assert_series_equal(expected, out) -@requires_scipy +@requires_tables def test_lookup_linke_turbidity_months_leapyear(): times = pd.date_range(start='2016-04-01', end='2016-07-01', freq='1M', tz='America/Phoenix') @@ -217,7 +217,7 @@ def test_lookup_linke_turbidity_months_leapyear(): assert_series_equal(expected, out) -@requires_scipy +@requires_tables def test_lookup_linke_turbidity_nointerp_months(): times = pd.date_range(start='2014-04-10', end='2014-07-10', freq='1M', tz='America/Phoenix') @@ -448,7 +448,7 @@ def test_simplified_solis_nans_series(): assert_frame_equal(expected, out) -@requires_scipy +@requires_tables def test_linke_turbidity_corners(): """Test Linke turbidity corners out of bounds.""" months = pd.DatetimeIndex('%d/1/2016' % (m + 1) for m in range(12))