fix first and last hour issue are 1AM and midnite

* elaborate on warning regarding definition of TMY and possibly
  unexpected behavior when coercing the year
* add more comments
* use mod instead of subtracting hour to change midnite from 24 to zero
* shift all midnite rows one day forward
* check for leapday (2/29) and shift to March 1st
* add copious comments and a note that if pandas is updated to 0.24
  then we can use the `array` property to detect leapday
* add test to confirm that TMY3 1st hour is 1AM and last hour is midnite

Author: mikofski

pvlib/iotools/tmy.py

@@ -6,6 +6,7 @@
 import io
 import re
 from urllib.request import urlopen, Request
+import numpy as np
 import pandas as pd
 
 
@@ -139,8 +140,15 @@ def read_tmy3(filename=None, coerce_year=None, recolumn=True):
     TMYData.PresWthUncertainty          Present weather code uncertainty, see [2]_.
     =============================       ======================================================================================================================================================
 
-    .. warning:: when coercing the year, the last index in the dataframe will
-        actually be the first hour of the year
+    .. warning:: TMY3 irradiance data corresponds to the previous hour, so the
+        first hour is 1AM, corresponding to the net irradiance from midnite to
+        1AM, and the last hour is midnite of the *next* year, unless the year
+        has been coerced. EG: if TMY3 was 1988-12-31 24:00:00 this becomes
+        1989-01-01 00:00:00
+
+    .. warning:: When coercing the year, the last index in the dataframe will
+        be the first hour of the same year, EG: if TMY3 was 1988-12-31 24:00:00
+        and year is coerced to 1990 this becomes 1990-01-01
 
     References
     ----------
@@ -191,9 +199,22 @@ def read_tmy3(filename=None, coerce_year=None, recolumn=True):
     # header is actually the second line in file, but tell pandas to look for
     # header information on the 1st line (0 indexing) because we've already
     # advanced past the true first line with the readline call above.
-    data = pd.read_csv(csvdata, header=0, index_col='Date (MM/DD/YYYY)')
-    data.index = pd.to_datetime(data.index, format='%m/%d/%Y')
-    shifted_hour = data['Time (HH:MM)'].str[:2].astype(int) - 1
+    data = pd.read_csv(csvdata, header=0)
+    # get the date column as a pd.Series of numpy datetime64
+    data_ymd = pd.to_datetime(data['Date (MM/DD/YYYY)'], format='%m/%d/%Y')
+    # shift the time column so that midnite is 00:00 instead of 24:00
+    shifted_hour = data['Time (HH:MM)'].str[:2].astype(int) % 24
+    # shift the dates at midnite so they correspond to the next day
+    data_ymd[shifted_hour == 0] += datetime.timedelta(days=1)
+    # NOTE: as of pandas>=0.24 the pd.Series.array has a month attribute, but
+    # in pandas-0.18.1, only DatetimeIndex has month, but indices are immutable
+    # so we need to continue to work with the panda series of dates `data_ymd`
+    data_index = pd.DatetimeIndex(data_ymd)
+    # use indices to check for a leap day and advance it to March 1st
+    leapday = (data_index.month == 2) & (data_index.day == 29)
+    data_ymd[leapday] += datetime.timedelta(days=1)
+    # is this necessary? convert the series to DatetimeIndex
+    data.index = pd.DatetimeIndex(data_ymd)
     # shifted_hour is a pd.Series, so use pd.to_timedelta to get a pd.Series of
     # timedeltas
     # NOTE: as of pvlib-0.6.3, min req is pandas-0.18.1, so pd.to_timedelta

pvlib/tests/iotools/test_tmy.py

@@ -69,3 +69,7 @@ def test_gh865_read_tmy3_feb_leapyear_hr24():
     assert all(data.index.year == 1990)
     # let's do a quick sanity check, are the indices monotonically increasing?
     assert all(np.diff(data.index[:-1].astype(int)) == 3600000000000)
+    # according to the TMY3 manual, each record corresponds to the previous
+    # hour so check that the 1st hour is 1AM and the last hour is midnite
+    assert data.index[0].hour == 1
+    assert data.index[-1].hour == 0