reorganize tests into subfolders and use pathlib for conftest DATA_DIR (#859)

* reorganize tests into subfolders

* closes #848
* create pvlib/tests/iotools and move all iotools tests into that
subfolder
* use fixtures in test_ecmwf_macc.py for expected_test_data
* use conftest.data_dir instead of redundant boilerplate process of
retrieving test folder for each tests, mostly for iotools
* fix test_ivtools.py was using full path to conftest but tests is not a
python package, so no
* change "pvlib/test" -> "pvlib/tests" to conform to popular convention

Signed-off-by: Mark Mikofski <[email protected]>

* in azure pipelines, pytest pvlib only, not pvlib/test

* Update v0.7.1.rst

* Update .codecov.yml

* Update docs/sphinx/source/whatsnew/v0.7.1.rst

with suggestion from Will

Co-Authored-By: Will Holmgren <[email protected]>

* use pathlib, remove os, inspect

* some iotools expect string to check for url using startswith() - so
  stringify path objects first
* in midc, use requests.get params arg to build querystring instead of
  doing it manually
* a couple of places chnage constants to all CAPS, some other places
  make a fixture, sometimes nothing, not consistent
* also in test_midc, comment out unused URL, was it part of a test once?

* path objects only work seemlessly on Python-3.6 or later

* stringify a few more path objects

* in psm3.read_psm3 still expects a string filename
* tmy3 and tmy2 also could be strings

* last two: stringify epw and surfrad

* fingers crossed!

* fix typo, mention pathlib in what's new

Co-authored-by: Will Holmgren <[email protected]>

Author: mikofski

.codecov.yml

@@ -22,6 +22,6 @@ coverage:
       tests:
         target: 95%
         paths:
-          - "pvlib/test/.*"
+          - "pvlib/tests/.*"
 
 comment: off

azure-pipelines.yml

@@ -155,7 +155,7 @@ jobs:
   - script: |
       pip install pytest pytest-cov pytest-mock pytest-timeout pytest-azurepipelines
       pip install -e .
-      pytest pvlib/test --junitxml=junit/test-results.xml --cov=pvlib --cov-report=xml --cov-report=html
+      pytest pvlib --junitxml=junit/test-results.xml --cov=pvlib --cov-report=xml --cov-report=html
     displayName: 'Test with pytest'
 
   - task: PublishTestResults@2

docs/sphinx/source/whatsnew/v0.7.1.rst

@@ -26,6 +26,9 @@ Testing
 ~~~~~~~
 * Added single-year PSM3 API test for `iotools.get_psm3`.
 * Added tests for `iotools.parse_psm3` and `iotools.read_psm3`.
+* Change `pvlib/test` folder to `pvlib/tests` and reorganize tests into
+  subfolders, *e.g.*: created `pvlib/tests/iotools` (:pull:`859`)
+* replace `os.path` with `pathlib` and stringify path objects for Python<=3.5
 
 Documentation
 ~~~~~~~~~~~~~

pvlib/iotools/crn.py

@@ -46,7 +46,7 @@ def read_crn(filename):
 
     Parameters
     ----------
-    filename: str
+    filename: str, path object, or file-like
         filepath or url to read for the fixed-width file.
 
     Returns

pvlib/iotools/epw.py

@@ -217,7 +217,7 @@ def read_epw(filename, coerce_year=None):
        <https://energyplus.net/documentation>`_
     '''
 
-    if filename.startswith('http'):
+    if str(filename).startswith('http'):
         # Attempts to download online EPW file
         # See comments above for possible online sources
         request = Request(filename, headers={'User-Agent': (
@@ -228,7 +228,7 @@ def read_epw(filename, coerce_year=None):
         csvdata = io.StringIO(response.read().decode(errors='ignore'))
     else:
         # Assume it's accessible via the file system
-        csvdata = open(filename, 'r')
+        csvdata = open(str(filename), 'r')
     try:
         data, meta = parse_epw(csvdata, coerce_year)
     finally:

pvlib/iotools/midc.py

@@ -249,12 +249,12 @@ def read_midc_raw_data_from_nrel(site, start, end, variable_map={},
     args = {'site': site,
             'begin': start.strftime('%Y%m%d'),
             'end': end.strftime('%Y%m%d')}
-    endpoint = 'https://midcdmz.nrel.gov/apps/data_api.pl?'
-    url = endpoint + '&'.join(['{}={}'.format(k, v) for k, v in args.items()])
+    url = 'https://midcdmz.nrel.gov/apps/data_api.pl'
+    # NOTE: just use requests.get(url, params=args) to build querystring
     # number of header columns and data columns do not always match,
     # so first parse the header to determine the number of data columns
     # to parse
-    csv_request = requests.get(url, timeout=timeout)
+    csv_request = requests.get(url, timeout=timeout, params=args)
     csv_request.raise_for_status()
     raw_csv = io.StringIO(csv_request.text)
     first_row = pd.read_csv(raw_csv, nrows=0)

pvlib/iotools/psm3.py

@@ -295,6 +295,6 @@ def read_psm3(filename):
     .. [2] `Standard Time Series Data File Format
        <https://rredc.nrel.gov/solar/old_data/nsrdb/2005-2012/wfcsv.pdf>`_
     """
-    with open(filename, 'r') as fbuf:
+    with open(str(filename), 'r') as fbuf:
         content = parse_psm3(fbuf)
     return content

pvlib/iotools/solrad.py

@@ -81,7 +81,7 @@ def read_solrad(filename):
        Program. Bull. Amer. Meteor. Soc., 77, 2857-2864.
        :doi:`10.1175/1520-0477(1996)077<2857:TNISIS>2.0.CO;2`
     """
-    if 'msn' in filename:
+    if 'msn' in str(filename):
         names = MADISON_HEADERS
         widths = MADISON_WIDTHS
         dtypes = MADISON_DTYPES

pvlib/iotools/surfrad.py

@@ -122,12 +122,12 @@ def read_surfrad(filename, map_variables=True):
     .. [2] NOAA SURFRAD Data Archive
        `SURFRAD Archive <ftp://aftp.cmdl.noaa.gov/data/radiation/surfrad/>`_
     """
-    if filename.startswith('ftp'):
+    if str(filename).startswith('ftp'):
         req = Request(filename)
         response = urlopen(req)
         file_buffer = io.StringIO(response.read().decode(errors='ignore'))
     else:
-        file_buffer = open(filename, 'r')
+        file_buffer = open(str(filename), 'r')
 
     # Read and parse the first two lines to build the metadata dict.
     station = file_buffer.readline()

pvlib/iotools/tmy.py

@@ -160,7 +160,7 @@ def read_tmy3(filename=None, coerce_year=None, recolumn=True):
 
     head = ['USAF', 'Name', 'State', 'TZ', 'latitude', 'longitude', 'altitude']
 
-    if filename.startswith('http'):
+    if str(filename).startswith('http'):
         request = Request(filename, headers={'User-Agent': (
             'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) '
             'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 '
@@ -169,7 +169,7 @@ def read_tmy3(filename=None, coerce_year=None, recolumn=True):
         csvdata = io.StringIO(response.read().decode(errors='ignore'))
     else:
         # assume it's accessible via the file system
-        csvdata = open(filename, 'r')
+        csvdata = open(str(filename), 'r')
 
     # read in file metadata, advance buffer to second line
     firstline = csvdata.readline()
@@ -409,7 +409,7 @@ def read_tmy2(filename):
     columns = 'year,month,day,hour,ETR,ETRN,GHI,GHISource,GHIUncertainty,DNI,DNISource,DNIUncertainty,DHI,DHISource,DHIUncertainty,GHillum,GHillumSource,GHillumUncertainty,DNillum,DNillumSource,DNillumUncertainty,DHillum,DHillumSource,DHillumUncertainty,Zenithlum,ZenithlumSource,ZenithlumUncertainty,TotCld,TotCldSource,TotCldUncertainty,OpqCld,OpqCldSource,OpqCldUncertainty,DryBulb,DryBulbSource,DryBulbUncertainty,DewPoint,DewPointSource,DewPointUncertainty,RHum,RHumSource,RHumUncertainty,Pressure,PressureSource,PressureUncertainty,Wdir,WdirSource,WdirUncertainty,Wspd,WspdSource,WspdUncertainty,Hvis,HvisSource,HvisUncertainty,CeilHgt,CeilHgtSource,CeilHgtUncertainty,PresentWeather,Pwat,PwatSource,PwatUncertainty,AOD,AODSource,AODUncertainty,SnowDepth,SnowDepthSource,SnowDepthUncertainty,LastSnowfall,LastSnowfallSource,LastSnowfallUncertaint'  # noqa: E501
     hdr_columns = 'WBAN,City,State,TZ,latitude,longitude,altitude'
 
-    tmy2, tmy2_meta = _read_tmy2(string, columns, hdr_columns, filename)
+    tmy2, tmy2_meta = _read_tmy2(string, columns, hdr_columns, str(filename))
 
     return tmy2, tmy2_meta