From 46389bd89b5c07bad29245c19c5a1cb8ddb768d3 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Tue, 14 Jul 2020 20:52:34 -0600 Subject: [PATCH 1/7] remove networking code from tmy.read_tmy3 --- pvlib/iotools/tmy.py | 19 +++---------------- pvlib/tests/iotools/test_tmy.py | 13 ++----------- 2 files changed, 5 insertions(+), 27 deletions(-) diff --git a/pvlib/iotools/tmy.py b/pvlib/iotools/tmy.py index 3b9a501fa8..878c5986f9 100644 --- a/pvlib/iotools/tmy.py +++ b/pvlib/iotools/tmy.py @@ -3,9 +3,7 @@ """ import datetime -import io import re -from urllib.request import urlopen, Request import pandas as pd @@ -15,7 +13,7 @@ def read_tmy3(filename=None, coerce_year=None, recolumn=True): Note that values contained in the metadata dictionary are unchanged from the TMY3 file (i.e. units are retained). In the case of any - discrepencies between this documentation and the TMY3 User's Manual + discrepancies between this documentation and the TMY3 User's Manual [1]_, the TMY3 User's Manual takes precedence. The TMY3 files were updated in Jan. 2015. This function requires the @@ -25,7 +23,7 @@ def read_tmy3(filename=None, coerce_year=None, recolumn=True): ---------- filename : None or string, default None If None, attempts to use a Tkinter file browser. A string can be - a relative file path, absolute file path, or url. + a relative file path or absolute file path. coerce_year : None or int, default None If supplied, the year of the index will be set to `coerce_year`, except @@ -171,21 +169,10 @@ def read_tmy3(filename=None, coerce_year=None, recolumn=True): head = ['USAF', 'Name', 'State', 'TZ', 'latitude', 'longitude', 'altitude'] - if str(filename).startswith('http'): - request = Request(filename, headers={'User-Agent': ( - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) ' - 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 ' - 'Safari/537.36')}) - response = urlopen(request) - csvdata = io.StringIO(response.read().decode(errors='ignore')) - else: - # assume it's accessible via the file system - csvdata = open(str(filename), 'r') + csvdata = open(str(filename), 'r') # read in file metadata, advance buffer to second line firstline = csvdata.readline() - if 'Request Rejected' in firstline: - raise IOError('Remote server rejected TMY file request') meta = dict(zip(head, firstline.rstrip('\n').split(","))) diff --git a/pvlib/tests/iotools/test_tmy.py b/pvlib/tests/iotools/test_tmy.py index bf10e3b830..566744c702 100644 --- a/pvlib/tests/iotools/test_tmy.py +++ b/pvlib/tests/iotools/test_tmy.py @@ -1,9 +1,7 @@ -from pandas.util.testing import network import numpy as np import pandas as pd -import pytest from pvlib.iotools import tmy -from conftest import DATA_DIR, RERUNS, RERUNS_DELAY +from conftest import DATA_DIR # test the API works from pvlib.iotools import read_tmy3 @@ -17,14 +15,6 @@ def test_read_tmy3(): tmy.read_tmy3(TMY3_TESTFILE) -@network -@pytest.mark.remote_data -@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) -def test_read_tmy3_remote(): - url = 'http://rredc.nrel.gov/solar/old_data/nsrdb/1991-2005/data/tmy3/703165TYA.CSV' - tmy.read_tmy3(url) - - def test_read_tmy3_recolumn(): data, meta = tmy.read_tmy3(TMY3_TESTFILE) assert 'GHISource' in data.columns @@ -49,6 +39,7 @@ def test_read_tmy3_no_coerce_year(): assert data.index[-2] == pd.Timestamp('1998-12-31 23:00:00-09:00') assert data.index[-1] == pd.Timestamp('1999-01-01 00:00:00-09:00') + def test_read_tmy2(): tmy.read_tmy2(TMY2_TESTFILE) From 338ba9936c888f99c895ca552e1c3708af4cd5af Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Tue, 14 Jul 2020 20:56:13 -0600 Subject: [PATCH 2/7] whatsnew --- docs/sphinx/source/whatsnew/v0.8.0.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/sphinx/source/whatsnew/v0.8.0.rst b/docs/sphinx/source/whatsnew/v0.8.0.rst index f901add383..1ce55a417f 100644 --- a/docs/sphinx/source/whatsnew/v0.8.0.rst +++ b/docs/sphinx/source/whatsnew/v0.8.0.rst @@ -17,6 +17,8 @@ API Changes ~~~~~~~~~~~ * Removed ``run_parallel_calculations`` and ``n_workers_for_parallel_calcs`` from :py:func:`pvlib.bifacial.pvfactors_timeseries` inputs (:issue:`902`)(:pull:`934`) +* :py:func:`pvlib.iotools.read_tmy3` can now only read local data files + (:issue:`996`)(:pull:`1003`) Enhancements ~~~~~~~~~~~~ From 7bc942f622426dcbf062f1df4b6bf47da3dd6834 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Tue, 14 Jul 2020 21:00:00 -0600 Subject: [PATCH 3/7] whats even newer --- docs/sphinx/source/whatsnew/v0.8.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sphinx/source/whatsnew/v0.8.0.rst b/docs/sphinx/source/whatsnew/v0.8.0.rst index 1ce55a417f..41e0d33b3b 100644 --- a/docs/sphinx/source/whatsnew/v0.8.0.rst +++ b/docs/sphinx/source/whatsnew/v0.8.0.rst @@ -18,7 +18,7 @@ API Changes * Removed ``run_parallel_calculations`` and ``n_workers_for_parallel_calcs`` from :py:func:`pvlib.bifacial.pvfactors_timeseries` inputs (:issue:`902`)(:pull:`934`) * :py:func:`pvlib.iotools.read_tmy3` can now only read local data files - (:issue:`996`)(:pull:`1003`) + (:issue:`996`)(:pull:`1004`) Enhancements ~~~~~~~~~~~~ From 1942200e6cbfc1153e4ba759e67ef95147c58d04 Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Wed, 15 Jul 2020 21:02:49 -0600 Subject: [PATCH 4/7] improvements from review --- docs/sphinx/source/whatsnew/v0.8.0.rst | 9 +++-- pvlib/iotools/tmy.py | 55 +++++++------------------- 2 files changed, 20 insertions(+), 44 deletions(-) diff --git a/docs/sphinx/source/whatsnew/v0.8.0.rst b/docs/sphinx/source/whatsnew/v0.8.0.rst index 41e0d33b3b..dfa59f4cf3 100644 --- a/docs/sphinx/source/whatsnew/v0.8.0.rst +++ b/docs/sphinx/source/whatsnew/v0.8.0.rst @@ -16,9 +16,12 @@ API Changes with Deprecations API Changes ~~~~~~~~~~~ * Removed ``run_parallel_calculations`` and ``n_workers_for_parallel_calcs`` - from :py:func:`pvlib.bifacial.pvfactors_timeseries` inputs (:issue:`902`)(:pull:`934`) -* :py:func:`pvlib.iotools.read_tmy3` can now only read local data files - (:issue:`996`)(:pull:`1004`) + from :py:func:`pvlib.bifacial.pvfactors_timeseries` inputs (:issue:`902`) (:pull:`934`) +* :py:func:`pvlib.iotools.read_tmy3` can now only read local data files because + the NREL RREDC server hosting the TMY3 dataset has been retired. For + fetching TMY data from NREL servers, :py:func:`pvlib.iotools.get_psm3` is + now recommended to retrieve newer PSM3 data over the older TMY3 data. + (:issue:`996`) (:pull:`1004`) Enhancements ~~~~~~~~~~~~ diff --git a/pvlib/iotools/tmy.py b/pvlib/iotools/tmy.py index 878c5986f9..5cbb067f46 100644 --- a/pvlib/iotools/tmy.py +++ b/pvlib/iotools/tmy.py @@ -7,7 +7,7 @@ import pandas as pd -def read_tmy3(filename=None, coerce_year=None, recolumn=True): +def read_tmy3(filename, coerce_year=None, recolumn=True): ''' Read a TMY3 file in to a pandas dataframe. @@ -21,9 +21,8 @@ def read_tmy3(filename=None, coerce_year=None, recolumn=True): Parameters ---------- - filename : None or string, default None - If None, attempts to use a Tkinter file browser. A string can be - a relative file path or absolute file path. + filename : str + A relative file path or absolute file path. coerce_year : None or int, default None If supplied, the year of the index will be set to `coerce_year`, except @@ -159,22 +158,17 @@ def read_tmy3(filename=None, coerce_year=None, recolumn=True): Update: Users Manual. 472 pp.; NREL Report No. TP-581-41364. ''' - if filename is None: - try: - filename = _interactive_load() - except ImportError: - raise ImportError('Interactive load failed. tkinter not supported ' - 'on this system. Try installing X-Quartz and ' - 'reloading') - head = ['USAF', 'Name', 'State', 'TZ', 'latitude', 'longitude', 'altitude'] - csvdata = open(str(filename), 'r') - - # read in file metadata, advance buffer to second line - firstline = csvdata.readline() - - meta = dict(zip(head, firstline.rstrip('\n').split(","))) + with open(filename, 'r') as csvdata: + # read in file metadata, advance buffer to second line + firstline = csvdata.readline() + meta = dict(zip(head, firstline.rstrip('\n').split(","))) + # use pandas to read the csv file buffer + # header is actually the second line, but tell pandas to look for + # header information on the 1st line (0 indexing) because we've already + # advanced past the true first line with the readline call above. + data = pd.read_csv(csvdata, header=0) # convert metadata strings to numeric types meta['altitude'] = float(meta['altitude']) @@ -183,11 +177,6 @@ def read_tmy3(filename=None, coerce_year=None, recolumn=True): meta['TZ'] = float(meta['TZ']) meta['USAF'] = int(meta['USAF']) - # use pandas to read the csv file/stringio buffer - # header is actually the second line in file, but tell pandas to look for - # header information on the 1st line (0 indexing) because we've already - # advanced past the true first line with the readline call above. - data = pd.read_csv(csvdata, header=0) # get the date column as a pd.Series of numpy datetime64 data_ymd = pd.to_datetime(data['Date (MM/DD/YYYY)'], format='%m/%d/%Y') # shift the time column so that midnite is 00:00 instead of 24:00 @@ -218,13 +207,6 @@ def read_tmy3(filename=None, coerce_year=None, recolumn=True): return data, meta -def _interactive_load(): - import tkinter - from tkinter.filedialog import askopenfilename - tkinter.Tk().withdraw() # Start interactive file input - return askopenfilename() - - def _recolumn(tmy3_dataframe): """ Rename the columns of the TMY3 DataFrame. @@ -282,9 +264,8 @@ def read_tmy2(filename): Parameters ---------- - filename : None or string - If None, attempts to use a Tkinter file browser. A string can be - a relative file path, absolute file path, or url. + filename : str + A relative or absolute file path. Returns ------- @@ -399,14 +380,6 @@ def read_tmy2(filename): for TMY2s". NREL 1995. ''' - if filename is None: - try: - filename = _interactive_load() - except ImportError: - raise ImportError('Interactive load failed. tkinter not supported ' - 'on this system. Try installing X-Quartz and ' - 'reloading') - # paste in the column info as one long line string = '%2d%2d%2d%2d%4d%4d%4d%1s%1d%4d%1s%1d%4d%1s%1d%4d%1s%1d%4d%1s%1d%4d%1s%1d%4d%1s%1d%2d%1s%1d%2d%1s%1d%4d%1s%1d%4d%1s%1d%3d%1s%1d%4d%1s%1d%3d%1s%1d%3d%1s%1d%4d%1s%1d%5d%1s%1d%10d%3d%1s%1d%3d%1s%1d%3d%1s%1d%2d%1s%1d' # noqa: E501 columns = 'year,month,day,hour,ETR,ETRN,GHI,GHISource,GHIUncertainty,DNI,DNISource,DNIUncertainty,DHI,DHISource,DHIUncertainty,GHillum,GHillumSource,GHillumUncertainty,DNillum,DNillumSource,DNillumUncertainty,DHillum,DHillumSource,DHillumUncertainty,Zenithlum,ZenithlumSource,ZenithlumUncertainty,TotCld,TotCldSource,TotCldUncertainty,OpqCld,OpqCldSource,OpqCldUncertainty,DryBulb,DryBulbSource,DryBulbUncertainty,DewPoint,DewPointSource,DewPointUncertainty,RHum,RHumSource,RHumUncertainty,Pressure,PressureSource,PressureUncertainty,Wdir,WdirSource,WdirUncertainty,Wspd,WspdSource,WspdUncertainty,Hvis,HvisSource,HvisUncertainty,CeilHgt,CeilHgtSource,CeilHgtUncertainty,PresentWeather,Pwat,PwatSource,PwatUncertainty,AOD,AODSource,AODUncertainty,SnowDepth,SnowDepthSource,SnowDepthUncertainty,LastSnowfall,LastSnowfallSource,LastSnowfallUncertaint' # noqa: E501 From fa40be926269a1dccd5e0a4885cab34e47737faa Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Wed, 15 Jul 2020 21:14:59 -0600 Subject: [PATCH 5/7] forgot open(pathlib.Path) doesn't work on python 3.5 --- pvlib/iotools/tmy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/iotools/tmy.py b/pvlib/iotools/tmy.py index 5cbb067f46..558dd372af 100644 --- a/pvlib/iotools/tmy.py +++ b/pvlib/iotools/tmy.py @@ -160,7 +160,7 @@ def read_tmy3(filename, coerce_year=None, recolumn=True): head = ['USAF', 'Name', 'State', 'TZ', 'latitude', 'longitude', 'altitude'] - with open(filename, 'r') as csvdata: + with open(str(filename), 'r') as csvdata: # read in file metadata, advance buffer to second line firstline = csvdata.readline() meta = dict(zip(head, firstline.rstrip('\n').split(","))) From 96a60e46d64d2e605000e9e96e76f93820f5962b Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Sat, 18 Jul 2020 11:01:26 -0600 Subject: [PATCH 6/7] move line outside with block --- pvlib/iotools/tmy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/iotools/tmy.py b/pvlib/iotools/tmy.py index 558dd372af..0f43aa83bb 100644 --- a/pvlib/iotools/tmy.py +++ b/pvlib/iotools/tmy.py @@ -163,13 +163,13 @@ def read_tmy3(filename, coerce_year=None, recolumn=True): with open(str(filename), 'r') as csvdata: # read in file metadata, advance buffer to second line firstline = csvdata.readline() - meta = dict(zip(head, firstline.rstrip('\n').split(","))) # use pandas to read the csv file buffer # header is actually the second line, but tell pandas to look for # header information on the 1st line (0 indexing) because we've already # advanced past the true first line with the readline call above. data = pd.read_csv(csvdata, header=0) + meta = dict(zip(head, firstline.rstrip('\n').split(","))) # convert metadata strings to numeric types meta['altitude'] = float(meta['altitude']) meta['latitude'] = float(meta['latitude']) From 6a34f8f8cd96420a9c1ddf4fcd61e48291866e2a Mon Sep 17 00:00:00 2001 From: Kevin Anderson Date: Sat, 18 Jul 2020 11:37:26 -0600 Subject: [PATCH 7/7] whatsnew entry for tkinter removal --- docs/sphinx/source/whatsnew/v0.8.0.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/sphinx/source/whatsnew/v0.8.0.rst b/docs/sphinx/source/whatsnew/v0.8.0.rst index dfa59f4cf3..495cd219a7 100644 --- a/docs/sphinx/source/whatsnew/v0.8.0.rst +++ b/docs/sphinx/source/whatsnew/v0.8.0.rst @@ -22,6 +22,9 @@ API Changes fetching TMY data from NREL servers, :py:func:`pvlib.iotools.get_psm3` is now recommended to retrieve newer PSM3 data over the older TMY3 data. (:issue:`996`) (:pull:`1004`) +* The tkinter-based file selection dialog has been removed from + :py:func:`pvlib.iotools.read_tmy2` and :py:func:`pvlib.iotools.read_tmy3`; + the filepath is now a required parameter. (:pull:`1004`) Enhancements ~~~~~~~~~~~~