From f8e6d6fddd71cf1822b7f26bba9c8fa76be54e83 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Tue, 23 Feb 2021 16:21:33 +0100 Subject: [PATCH 01/51] Add pvgis.get_pvgis_hourly function --- pvlib/iotools/pvgis.py | 162 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 90c54ae839..96d726aca1 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -24,6 +24,168 @@ URL = 'https://re.jrc.ec.europa.eu/api/' +def get_pvgis_hourly(lat, lon, angle=0, aspect=0, + outputformat='json', usehorizon=True, + userhorizon=None, raddatabase=None, + startyear=None, endyear=None, + pvcalculation=False, peakpower=None, + pvtechchoice='crystSi', mountingplace='free', loss=None, + trackingtype=0, + optimal_inclination=False, optimalangles=False, + components=True, url=URL, timeout=30): + """ + Get hourly solar radiation data and optimal modeled PV power output from + PVGIS. For more information see the PVGIS [1]_ TMY tool + documentation [2]_. + Parameters + ---------- + lat : float + Latitude in degrees north + lon : float + Longitude in degrees east + angle: float, default: 0 + Tilt angle from horizontal plane. Not relevant for 2-axis tracking. + aspect: float, default: 0 + Orientation (azimuth angle) of the (fixed) plane. 0=south, 90=west, + -90: east. Not relevant for tracking systems. + outputformat : str, default: 'json' + Must be in ``['csv', 'basic', 'json']``. See PVGIS TMY tool + documentation [2]_ for more info. + usehorizon : bool, default: True + include effects of horizon + userhorizon : list of float, default: None + optional user specified elevation of horizon in degrees, at equally + spaced azimuth clockwise from north, only valid if `usehorizon` is + true, if `usehorizon` is true but `userhorizon` is `None` then PVGIS + will calculate the horizon [4]_ + raddatabase : str, default: None + Name of radiation database. Options depend on location, see [3]_. + startyear : int, default: None + first year of the radiation time series. Defaults to first year avaiable. + endyear : int, default: None + last year of the radiation time series. Defaults to last year avaiable. + pvcalculation : bool, default: False + Also return estimate of hourly production. + peakpower : float, default: None + Nominal power of PV system in kW. Required if pvcalculation=True. + pvtechchoice : {'crystSi', 'CIS', 'CdTe', 'Unknown'}, default: 'crystSi' + PV technology. + mountingplace : {'free', 'building'}, default: free + Type of mounting for PV system. Options of 'free' for free-standing + and 'building' for building-integrated. + loss : float, default: None + Sum of PV system losses in percent. Required if pvcalculation=True + trackingtype: {0, 1, 2, 3, 4, 5}, default: 0 + Type of suntracking. 0=fixed, 1=single horizontal axis aligned + north-south, 2=two-axis tracking, 3=vertical axis tracking, 4=single + horizontal axis aligned east-west, 5=single inclined axis aligned + north-south. + optimalinclination : bool, default: False + Calculate the optimum tilt angle. Not relevant for 2-axis tracking + optimalangles : bool, default: False + Calculate the optimum tilt and azimuth angles. Not relevant for 2-axis + tracking. + components : bool, default: True + Output solar radiation components (beam, diffuse, and reflected). + Otherwise only global irradiance is returned. + url : str, default :const:`pvlib.iotools.pvgis.URL` + base url of PVGIS API, append ``tmy`` to get TMY endpoint + timeout : int, default: 30 + time in seconds to wait for server response before timeout + Returns + ------- + data : pandas.DataFrame + the weather data + months_selected : list + TMY year for each month, ``None`` for basic and EPW + inputs : dict + the inputs, ``None`` for basic and EPW + meta : list or dict + meta data, ``None`` for basic + Raises + ------ + requests.HTTPError + if the request response status is ``HTTP/1.1 400 BAD REQUEST``, then + the error message in the response will be raised as an exception, + otherwise raise whatever ``HTTP/1.1`` error occurred + See also + -------- + read_pvgis_tmy + References + ---------- + .. [1] `PVGIS `_ + .. [2] `PVGIS hourly radiation data + `_ + .. [3] `PVGIS Non-interactive service + ` + .. [3] `PVGIS horizon profile tool + `_ + """ + # use requests to format the query string by passing params dictionary + params = {'lat': lat, 'lon': lon, 'outputformat': outputformat, + 'angle': angle, 'aspect': aspect, + 'pvtechchoice': pvtechchoice, 'mountingplace': mountingplace, + 'trackingtype': trackingtype, 'components': int(components)} + # pvgis only likes 0 for False, and 1 for True, not strings, also the + # default for usehorizon is already 1 (ie: True), so only set if False + # default for pvcalculation, optimalangles, optimalinclination, + # is already 0 i.e. False, so only set if True + if not usehorizon: + params['usehorizon'] = 0 + if userhorizon is not None: + params['userhorizon'] = ','.join(str(x) for x in userhorizon) + if raddatabase is not None: + params['raddatabase'] = raddatabase + if startyear is not None: + params['startyear'] = startyear + if endyear is not None: + params['endyear'] = endyear + if pvcalculation: + params['pvcalculation'] = 1 + if peakpower is not None: + params['peakpower'] = peakpower + if loss is not None: + params['loss'] = loss + if optimal_inclination: + params['optimal_inclination'] = 1 + if optimalangles: + params['optimalangles'] = 1 + + # The url endpoint for hourly radiation is 'seriescalc' + res = requests.get(url + 'seriescalc', params=params, timeout=timeout) + + # PVGIS returns really well formatted error messages in JSON for HTTP/1.1 + # 400 BAD REQUEST so try to return that if possible, otherwise raise the + # HTTP/1.1 error caught by requests + if not res.ok: + try: + err_msg = res.json() + except Exception: + res.raise_for_status() + else: + raise requests.HTTPError(err_msg['message']) + + # initialize data to None in case API fails to respond to bad outputformat + data = None, None, None, None + if outputformat == 'json': + src = res.json() + return _parse_pvgis_hourly_json(src) + else: + # this line is never reached because if outputformat is not valid then + # the response is HTTP/1.1 400 BAD REQUEST which is handled earlier + pass + return data + + +def _parse_pvgis_hourly_json(src): + inputs = src['inputs'] + meta = src['meta'] + data = pd.DataFrame(src['outputs']['hourly']) + data.index = pd.to_datetime(data['time'], format='%Y%m%d:%H%M', utc=True) + data = data.drop('time', axis=1) + return data, inputs, meta + + def get_pvgis_tmy(lat, lon, outputformat='json', usehorizon=True, userhorizon=None, startyear=None, endyear=None, url=URL, timeout=30): From 23b65f7b3101287b386932ea5008a8fb2eaa66ad Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sun, 28 Feb 2021 00:42:31 +0100 Subject: [PATCH 02/51] Update documentation --- pvlib/iotools/pvgis.py | 86 ++++++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 40 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 96d726aca1..609f8ec1c1 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -24,9 +24,9 @@ URL = 'https://re.jrc.ec.europa.eu/api/' -def get_pvgis_hourly(lat, lon, angle=0, aspect=0, +def get_pvgis_hourly(lat, lon, angle=0, aspect=0, outputformat='json', usehorizon=True, - userhorizon=None, raddatabase=None, + userhorizon=None, raddatabase=None, startyear=None, endyear=None, pvcalculation=False, peakpower=None, pvtechchoice='crystSi', mountingplace='free', loss=None, @@ -34,88 +34,92 @@ def get_pvgis_hourly(lat, lon, angle=0, aspect=0, optimal_inclination=False, optimalangles=False, components=True, url=URL, timeout=30): """ - Get hourly solar radiation data and optimal modeled PV power output from - PVGIS. For more information see the PVGIS [1]_ TMY tool - documentation [2]_. + Get hourly solar radiation data and modeled PV power output from PVGIS + [1]_. + Parameters ---------- - lat : float + lat: float Latitude in degrees north - lon : float + lon: float Longitude in degrees east angle: float, default: 0 Tilt angle from horizontal plane. Not relevant for 2-axis tracking. aspect: float, default: 0 Orientation (azimuth angle) of the (fixed) plane. 0=south, 90=west, -90: east. Not relevant for tracking systems. - outputformat : str, default: 'json' - Must be in ``['csv', 'basic', 'json']``. See PVGIS TMY tool - documentation [2]_ for more info. - usehorizon : bool, default: True - include effects of horizon - userhorizon : list of float, default: None - optional user specified elevation of horizon in degrees, at equally + outputformat: str, default: 'json' + Must be in ``['csv', 'json']``. See PVGIS hourly data documentation + [2]_ for more info. + usehorizon: bool, default: True + Include effects of horizon + userhorizon: list of float, default: None + Optional user specified elevation of horizon in degrees, at equally spaced azimuth clockwise from north, only valid if `usehorizon` is true, if `usehorizon` is true but `userhorizon` is `None` then PVGIS will calculate the horizon [4]_ - raddatabase : str, default: None + raddatabase: str, default: None Name of radiation database. Options depend on location, see [3]_. - startyear : int, default: None - first year of the radiation time series. Defaults to first year avaiable. - endyear : int, default: None - last year of the radiation time series. Defaults to last year avaiable. - pvcalculation : bool, default: False + startyear: int, default: None + First year of the radiation time series. Defaults to first year + avaiable. + endyear: int, default: None + Last year of the radiation time series. Defaults to last year avaiable. + pvcalculation: bool, default: False Also return estimate of hourly production. - peakpower : float, default: None + peakpower: float, default: None Nominal power of PV system in kW. Required if pvcalculation=True. - pvtechchoice : {'crystSi', 'CIS', 'CdTe', 'Unknown'}, default: 'crystSi' + pvtechchoice: {'crystSi', 'CIS', 'CdTe', 'Unknown'}, default: 'crystSi' PV technology. - mountingplace : {'free', 'building'}, default: free + mountingplace: {'free', 'building'}, default: free Type of mounting for PV system. Options of 'free' for free-standing and 'building' for building-integrated. - loss : float, default: None - Sum of PV system losses in percent. Required if pvcalculation=True + loss: float, default: None + Sum of PV system losses in percent. Required if pvcalculation=True trackingtype: {0, 1, 2, 3, 4, 5}, default: 0 Type of suntracking. 0=fixed, 1=single horizontal axis aligned north-south, 2=two-axis tracking, 3=vertical axis tracking, 4=single horizontal axis aligned east-west, 5=single inclined axis aligned north-south. - optimalinclination : bool, default: False + optimalinclination: bool, default: False Calculate the optimum tilt angle. Not relevant for 2-axis tracking - optimalangles : bool, default: False + optimalangles: bool, default: False Calculate the optimum tilt and azimuth angles. Not relevant for 2-axis tracking. - components : bool, default: True + components: bool, default: True Output solar radiation components (beam, diffuse, and reflected). Otherwise only global irradiance is returned. - url : str, default :const:`pvlib.iotools.pvgis.URL` - base url of PVGIS API, append ``tmy`` to get TMY endpoint - timeout : int, default: 30 - time in seconds to wait for server response before timeout + url: str, default:const:`pvlib.iotools.pvgis.URL` + Base url of PVGIS API, append ``seriescalc`` to get hourly data + endpoint + timeout: int, default: 30 + Time in seconds to wait for server response before timeout + Returns ------- data : pandas.DataFrame - the weather data - months_selected : list - TMY year for each month, ``None`` for basic and EPW + Time-series of hourly data inputs : dict - the inputs, ``None`` for basic and EPW + Dictionary of the request input parameters meta : list or dict - meta data, ``None`` for basic + meta data + Raises ------ requests.HTTPError if the request response status is ``HTTP/1.1 400 BAD REQUEST``, then the error message in the response will be raised as an exception, otherwise raise whatever ``HTTP/1.1`` error occurred + See also -------- - read_pvgis_tmy + pvlib.iotools.read_pvgis_hourly + References ---------- .. [1] `PVGIS `_ - .. [2] `PVGIS hourly radiation data - `_ + .. [2] `PVGIS Hourly Radiation + `_ .. [3] `PVGIS Non-interactive service ` .. [3] `PVGIS horizon profile tool @@ -170,6 +174,8 @@ def get_pvgis_hourly(lat, lon, angle=0, aspect=0, if outputformat == 'json': src = res.json() return _parse_pvgis_hourly_json(src) + elif outputformat == 'csv': + return _parse_pvgis_hourly_csv(src) else: # this line is never reached because if outputformat is not valid then # the response is HTTP/1.1 400 BAD REQUEST which is handled earlier From 53de6feb71579b9d079b6ccdc605337e5999a154 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Mon, 1 Mar 2021 00:17:35 +0100 Subject: [PATCH 03/51] Add parser for pvgis_hourly csv and basic --- pvlib/iotools/pvgis.py | 66 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 609f8ec1c1..419bf2c23a 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -175,7 +175,11 @@ def get_pvgis_hourly(lat, lon, angle=0, aspect=0, src = res.json() return _parse_pvgis_hourly_json(src) elif outputformat == 'csv': - return _parse_pvgis_hourly_csv(src) + with io.StringIO(res.content.decode('utf-8')) as src: + return _parse_pvgis_hourly_csv(src) + elif outputformat == 'basic': + with io.BytesIO(res.content) as src: + return _parse_pvgis_hourly_basic(src) else: # this line is never reached because if outputformat is not valid then # the response is HTTP/1.1 400 BAD REQUEST which is handled earlier @@ -189,9 +193,69 @@ def _parse_pvgis_hourly_json(src): data = pd.DataFrame(src['outputs']['hourly']) data.index = pd.to_datetime(data['time'], format='%Y%m%d:%H%M', utc=True) data = data.drop('time', axis=1) + data = data.astype(dtype={'Int': 'int'}) # The 'Int' column to be integer return data, inputs, meta +def _parse_pvgis_hourly_basic(src): + # Hourly data with outputformat='basic' does not include header or metadata + data = pd.read_csv(src, header=None, skiprows=2) + return data, None, None + + +def _parse_pvgis_hourly_csv(src): + # The first 4 rows are latitude, longitude, elevation, radiation database + inputs = {} + # 'Latitude (decimal degrees): 45.000\r\n' + inputs['latitude'] = float(src.readline().split(':')[1]) + # 'Longitude (decimal degrees): 8.000\r\n' + inputs['longitude'] = float(src.readline().split(':')[1]) + # Elevation (m): 1389.0\r\n + inputs['elevation'] = float(src.readline().split(':')[1]) + # 'Radiation database: \tPVGIS-SARAH\r\n' + inputs['radiation_database'] = str(src.readline().split(':')[1] + .replace('\t', '').replace('\n', '')) + + # Parse through the remaining metadata section (the number of lines for + # this section depends on the requested parameters) + while True: + line = src.readline() + if line.startswith('time,'): # The data header starts with 'time,' + # The last line of the meta-data section contains the column names + names = line.replace('\n', '').replace('\r', '').split(',') + break + # Only retrieve metadata from non-empty lines + elif (line != '\n') & (line != '\r\n'): + inputs[line.split(':')[0]] = str(line.split(':')[1] + .replace('\n', '') + .replace('\r', '').strip()) + + # The data section covers a variable number of lines (depends on requested + # years) and ends with a blank line + data_lines = [] + while True: + line = src.readline() + if (line == '\n') | (line == '\r\n'): + break + else: + data_lines.append(line.replace('\n', '').replace('\r', '') + .split(',')) + + data = pd.DataFrame(data_lines, columns=names) + data.index = pd.to_datetime(data['time'], format='%Y%m%d:%H%M', utc=True) + data = data.drop('time', axis=1) + # All columns should have the dtype=float, except 'Int' which should be + # integer. It is necessary to convert to float, before converting to int + data = data.astype(float).astype(dtype={'Int': 'int'}) + + # Generate metadata dictionary containing description of parameters + meta = {} + for line in src.readlines(): + if ':' in line: + meta[line.split(':')[0]] = line.split(':')[1].strip() + + return data, inputs, meta + def get_pvgis_tmy(lat, lon, outputformat='json', usehorizon=True, userhorizon=None, startyear=None, endyear=None, url=URL, timeout=30): From 9e50c6a9be53a62b7adad91ae8eee38a4f7f425d Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sun, 7 Mar 2021 19:57:57 +0100 Subject: [PATCH 04/51] Update variable map and documentation --- pvlib/iotools/pvgis.py | 98 ++++++++++++++++++++++++++++++------------ 1 file changed, 70 insertions(+), 28 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 419bf2c23a..166454022a 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -23,19 +23,33 @@ URL = 'https://re.jrc.ec.europa.eu/api/' - -def get_pvgis_hourly(lat, lon, angle=0, aspect=0, - outputformat='json', usehorizon=True, - userhorizon=None, raddatabase=None, - startyear=None, endyear=None, - pvcalculation=False, peakpower=None, - pvtechchoice='crystSi', mountingplace='free', loss=None, - trackingtype=0, +# Dictionary mapping PVGIS names to pvlib names +VARIABLE_MAP = { + 'G(h)': 'ghi', + 'Gb(n)': 'dni', + 'Gd(h)': 'dhi', + 'G(i)': 'poa_global', + 'Gb(i)': 'poa_direct', + 'Gd(i)': 'poa_diffuse', + 'Gr(i)': 'poa_ground_diffuse', + 'H_sun': 'solar_elevation', + 'T2m': 'temp_air', + 'RH': 'relative_humidity', + 'SP': 'pressure', + 'WS10m': 'wind_speed', + 'WD10m': 'wind_direction', +} + + +def get_pvgis_hourly(lat, lon, angle=0, aspect=0, outputformat='json', + usehorizon=True, userhorizon=None, raddatabase=None, + startyear=None, endyear=None, pvcalculation=False, + peakpower=None, pvtechchoice='crystSi', + mountingplace='free', loss=None, trackingtype=0, optimal_inclination=False, optimalangles=False, - components=True, url=URL, timeout=30): + components=True, url=URL, map_variables=True, timeout=30): """ - Get hourly solar radiation data and modeled PV power output from PVGIS - [1]_. + Get hourly solar irradiation and modeled PV power output from PVGIS [1]_. Parameters ---------- @@ -49,8 +63,8 @@ def get_pvgis_hourly(lat, lon, angle=0, aspect=0, Orientation (azimuth angle) of the (fixed) plane. 0=south, 90=west, -90: east. Not relevant for tracking systems. outputformat: str, default: 'json' - Must be in ``['csv', 'json']``. See PVGIS hourly data documentation - [2]_ for more info. + Must be in ``['json', 'csv', 'basic']``. See PVGIS hourly data + documentation [2]_ for more info. Note basic does not include a header. usehorizon: bool, default: True Include effects of horizon userhorizon: list of float, default: None @@ -92,17 +106,45 @@ def get_pvgis_hourly(lat, lon, angle=0, aspect=0, url: str, default:const:`pvlib.iotools.pvgis.URL` Base url of PVGIS API, append ``seriescalc`` to get hourly data endpoint + map_variables: bool, default True + When true, renames columns of the Dataframe to pvlib variable names + where applicable. See variable VARIABLE_MAP. timeout: int, default: 30 Time in seconds to wait for server response before timeout Returns ------- data : pandas.DataFrame - Time-series of hourly data + Time-series of hourly data, see Notes for fields inputs : dict - Dictionary of the request input parameters + Dictionary of the request input parameters, ``None`` for basic meta : list or dict - meta data + meta data, ``None`` for basic + + Notes + ----- + data includes the following fields: + + ======================= ====== ========================================== + raw, mapped Format Description + ======================= ====== ========================================== + **Mapped field names are returned when the map_variables argument is True** + --------------------------------------------------------------------------- + P* float PV system power (W) + G(i)** float Global irradiance on inclined plane (W/m^2) + Gb(i)** float Beam (direct) irradiance on inclined plane (W/m^2) + Gd(i)** float Diffuse irradiance on inclined plane (W/m^2) + Gr(i)** float Reflected irradiance on inclined plane (W/m^2) + H_sun, solar_elevation float Sun height/elevation (degrees) + T2m, temp_air float Air temperature at 2 (degrees Celsius) + WS10m, wind_speed float Wind speed at 10 m (m/s) + Int int Solar radiation reconstructed (1/0) + ======================= ====== ========================================== + + *P (PV system power) is only returned when pvcalculation=True. + + **Gb(i), Gd(i), and Gr(i) are returned when components=True, whereas + otherwise the sum of the three components, G(i), is returned. Raises ------ @@ -113,7 +155,7 @@ def get_pvgis_hourly(lat, lon, angle=0, aspect=0, See also -------- - pvlib.iotools.read_pvgis_hourly + pvlib.iotools.read_pvgis_hourly, pvlib.iotools.get_pvgis_tmy References ---------- @@ -173,10 +215,10 @@ def get_pvgis_hourly(lat, lon, angle=0, aspect=0, data = None, None, None, None if outputformat == 'json': src = res.json() - return _parse_pvgis_hourly_json(src) + return _parse_pvgis_hourly_json(src, map_variables=map_variables) elif outputformat == 'csv': with io.StringIO(res.content.decode('utf-8')) as src: - return _parse_pvgis_hourly_csv(src) + return _parse_pvgis_hourly_csv(src, map_variables=map_variables) elif outputformat == 'basic': with io.BytesIO(res.content) as src: return _parse_pvgis_hourly_basic(src) @@ -187,13 +229,15 @@ def get_pvgis_hourly(lat, lon, angle=0, aspect=0, return data -def _parse_pvgis_hourly_json(src): +def _parse_pvgis_hourly_json(src, map_variables=True): inputs = src['inputs'] meta = src['meta'] data = pd.DataFrame(src['outputs']['hourly']) data.index = pd.to_datetime(data['time'], format='%Y%m%d:%H%M', utc=True) data = data.drop('time', axis=1) data = data.astype(dtype={'Int': 'int'}) # The 'Int' column to be integer + if map_variables: + data.rename(columns=VARIABLE_MAP, inplace=True) return data, inputs, meta @@ -203,7 +247,7 @@ def _parse_pvgis_hourly_basic(src): return data, None, None -def _parse_pvgis_hourly_csv(src): +def _parse_pvgis_hourly_csv(src, map_variables=True): # The first 4 rows are latitude, longitude, elevation, radiation database inputs = {} # 'Latitude (decimal degrees): 45.000\r\n' @@ -215,7 +259,6 @@ def _parse_pvgis_hourly_csv(src): # 'Radiation database: \tPVGIS-SARAH\r\n' inputs['radiation_database'] = str(src.readline().split(':')[1] .replace('\t', '').replace('\n', '')) - # Parse through the remaining metadata section (the number of lines for # this section depends on the requested parameters) while True: @@ -229,9 +272,8 @@ def _parse_pvgis_hourly_csv(src): inputs[line.split(':')[0]] = str(line.split(':')[1] .replace('\n', '') .replace('\r', '').strip()) - - # The data section covers a variable number of lines (depends on requested - # years) and ends with a blank line + # Save the entries from the data section to a list, until an empty line is + # reached an empty line. The length of the section depends on the request data_lines = [] while True: line = src.readline() @@ -240,22 +282,22 @@ def _parse_pvgis_hourly_csv(src): else: data_lines.append(line.replace('\n', '').replace('\r', '') .split(',')) - data = pd.DataFrame(data_lines, columns=names) data.index = pd.to_datetime(data['time'], format='%Y%m%d:%H%M', utc=True) data = data.drop('time', axis=1) + if map_variables: + data.rename(columns=VARIABLE_MAP, inplace=True) # All columns should have the dtype=float, except 'Int' which should be # integer. It is necessary to convert to float, before converting to int data = data.astype(float).astype(dtype={'Int': 'int'}) - # Generate metadata dictionary containing description of parameters meta = {} for line in src.readlines(): if ':' in line: meta[line.split(':')[0]] = line.split(':')[1].strip() - return data, inputs, meta + def get_pvgis_tmy(lat, lon, outputformat='json', usehorizon=True, userhorizon=None, startyear=None, endyear=None, url=URL, timeout=30): From a0ad2bbfa2413a466d75482e299ab0da2151a552 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sun, 7 Mar 2021 20:06:23 +0100 Subject: [PATCH 05/51] Update whatsnew, api.rst, and __init__ --- docs/sphinx/source/api.rst | 2 ++ docs/sphinx/source/whatsnew/v0.9.0.rst | 4 ++++ pvlib/iotools/__init__.py | 2 ++ pvlib/iotools/pvgis.py | 8 ++++---- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/sphinx/source/api.rst b/docs/sphinx/source/api.rst index 62ac0a8586..80550d98be 100644 --- a/docs/sphinx/source/api.rst +++ b/docs/sphinx/source/api.rst @@ -487,6 +487,8 @@ of sources and file formats relevant to solar energy modeling. iotools.parse_psm3 iotools.get_pvgis_tmy iotools.read_pvgis_tmy + iotools.get_pvgis_hourly + iotools.read_pvgis_hourly iotools.read_bsrn A :py:class:`~pvlib.location.Location` object may be created from metadata diff --git a/docs/sphinx/source/whatsnew/v0.9.0.rst b/docs/sphinx/source/whatsnew/v0.9.0.rst index 7fcb18e83d..9d9235a4f9 100644 --- a/docs/sphinx/source/whatsnew/v0.9.0.rst +++ b/docs/sphinx/source/whatsnew/v0.9.0.rst @@ -103,6 +103,10 @@ Deprecations Enhancements ~~~~~~~~~~~~ +* Add :func:`~pvlib.iotools.read_pvgis_hourly` and + :func:`~pvlib.iotools.get_pvgis_hourly` for reading and retrieving PVGIS + hourly solar radiation data and modelled PV power output. + files. (:pull:`1186`, :issue:`849`) * Add :func:`~pvlib.iotools.read_bsrn` for reading BSRN solar radiation data files. (:pull:`1145`, :issue:`1015`) * In :py:class:`~pvlib.modelchain.ModelChain`, attributes which contain diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index ba5d5e8807..5b588b72a9 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -13,4 +13,6 @@ from pvlib.iotools.psm3 import read_psm3 # noqa: F401 from pvlib.iotools.psm3 import parse_psm3 # noqa: F401 from pvlib.iotools.pvgis import get_pvgis_tmy, read_pvgis_tmy # noqa: F401 +from pvlib.iotools.pvgis import read_pvgis_hourly # noqa: F401 +from pvlib.iotools.pvgis import get_pvgis_hourly # noqa: F401 from pvlib.iotools.bsrn import read_bsrn # noqa: F401 diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 166454022a..61c27e7997 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -131,10 +131,10 @@ def get_pvgis_hourly(lat, lon, angle=0, aspect=0, outputformat='json', **Mapped field names are returned when the map_variables argument is True** --------------------------------------------------------------------------- P* float PV system power (W) - G(i)** float Global irradiance on inclined plane (W/m^2) - Gb(i)** float Beam (direct) irradiance on inclined plane (W/m^2) - Gd(i)** float Diffuse irradiance on inclined plane (W/m^2) - Gr(i)** float Reflected irradiance on inclined plane (W/m^2) + G(i)** float Global irradiance on inclined plane (W/m^2) # noqa + Gb(i)** float Beam (direct) irradiance on inclined plane (W/m^2) # noqa + Gd(i)** float Diffuse irradiance on inclined plane (W/m^2) # noqa + Gr(i)** float Reflected irradiance on inclined plane (W/m^2) # noqa H_sun, solar_elevation float Sun height/elevation (degrees) T2m, temp_air float Air temperature at 2 (degrees Celsius) WS10m, wind_speed float Wind speed at 10 m (m/s) From 3b4112073f20d2ca98ca62a501a3c02623b09b58 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sun, 7 Mar 2021 21:22:07 +0100 Subject: [PATCH 06/51] Add read_pvgis_hourly --- pvlib/iotools/pvgis.py | 86 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 61c27e7997..97f5010f32 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -298,6 +298,92 @@ def _parse_pvgis_hourly_csv(src, map_variables=True): return data, inputs, meta +def read_pvgis_hourly(filename, map_variables=True, pvgis_format=None): + """ + Read a file downloaded from PVGIS. + + Parameters + ---------- + filename : str, pathlib.Path, or file-like buffer + Name, path, or buffer of file downloaded from PVGIS. + pvgis_format : str, default None + Format of PVGIS file or buffer. Equivalent to the ``outputformat`` + parameter in the PVGIS TMY API. If `filename` is a file and + `pvgis_format` is ``None`` then the file extension will be used to + determine the PVGIS format to parse. For PVGIS files from the API with + ``outputformat='basic'``, please set `pvgis_format` to ``'basic'``. If + `filename` is a buffer, then `pvgis_format` is required and must be in + ``['csv', 'json', 'basic']``. + + Returns + ------- + data : pandas.DataFrame + the weather data + inputs : dict + the inputs, ``None`` for basic + meta : list or dict + meta data, ``None`` for basic + + Raises + ------ + ValueError + if `pvgis_format` is ``None`` and the file extension is neither + ``.csv`` nor ``.json`` or if `pvgis_format` is provided as + input but isn't in ``['csv', 'json', 'basic']`` + TypeError + if `pvgis_format` is ``None`` and `filename` is a buffer + + See also + -------- + get_pvgis_hourly + """ + # get the PVGIS outputformat + if pvgis_format is None: + # get the file extension from suffix, but remove the dot and make sure + # it's lower case to compare with csv, or json + # NOTE: raises TypeError if filename is a buffer + outputformat = Path(filename).suffix[1:].lower() + else: + outputformat = pvgis_format + + # parse the pvgis file based on the output format, either 'json', 'csv', + # or 'basic' + + # NOTE: json, csv, and basic output formats have parsers defined as private + # functions in this module + + # JSON: use Python built-in json module to convert file contents to a + # Python dictionary, and pass the dictionary to the + # _parse_pvgis_hourly_json() function from this module + if outputformat == 'json': + try: + src = json.load(filename) + except AttributeError: # str/path has no .read() attribute + with open(str(filename), 'r') as fbuf: + src = json.load(fbuf) + return _parse_pvgis_hourly_json(src) + + # CSV or basic: use the correct parser from this module + # eg: _parse_pvgis_tmy_csv() or _parse_pvgist_tmy_basic() + if outputformat in ['csv', 'basic']: + # get the correct parser function for this output format from globals() + pvgis_parser = globals()['_parse_pvgis_hourly_{:s}'.format(outputformat)] # noqa + # NOTE: pvgis_parse() is a pvgis parser function from this module, + # either _parse_pvgis_hourly_csv() or _parse_pvgist_hourly_basic() + try: + pvgis_data = pvgis_parser(filename) + except AttributeError: # str/path has no .read() attribute + with open(str(filename), 'r') as fbuf: + pvgis_data = pvgis_parser(fbuf) + return pvgis_data + + # raise exception if pvgis format isn't in ['csv', 'basic', 'json'] + err_msg = ( + "pvgis format '{:s}' was unknown, must be either 'json', 'csv', or" + "'basic'").format(outputformat) + raise ValueError(err_msg) + + def get_pvgis_tmy(lat, lon, outputformat='json', usehorizon=True, userhorizon=None, startyear=None, endyear=None, url=URL, timeout=30): From dea2a17b6afec7808aca01dd3cdb2f3029dd62fe Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sun, 13 Jun 2021 12:06:10 -0500 Subject: [PATCH 07/51] Update input and output names Change lat/lon to latitude/longitude and changed startyear/endyear to start/end. Also, changed meta to metadata --- pvlib/iotools/pvgis.py | 49 +++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 97f5010f32..f1ab01a001 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -41,9 +41,10 @@ } -def get_pvgis_hourly(lat, lon, angle=0, aspect=0, outputformat='json', +def get_pvgis_hourly(latitude, longitude, angle=0, aspect=0, + outputformat='json', usehorizon=True, userhorizon=None, raddatabase=None, - startyear=None, endyear=None, pvcalculation=False, + start=None, end=None, pvcalculation=False, peakpower=None, pvtechchoice='crystSi', mountingplace='free', loss=None, trackingtype=0, optimal_inclination=False, optimalangles=False, @@ -53,9 +54,9 @@ def get_pvgis_hourly(lat, lon, angle=0, aspect=0, outputformat='json', Parameters ---------- - lat: float + latitude: float Latitude in degrees north - lon: float + longitude: float Longitude in degrees east angle: float, default: 0 Tilt angle from horizontal plane. Not relevant for 2-axis tracking. @@ -74,10 +75,10 @@ def get_pvgis_hourly(lat, lon, angle=0, aspect=0, outputformat='json', will calculate the horizon [4]_ raddatabase: str, default: None Name of radiation database. Options depend on location, see [3]_. - startyear: int, default: None + start: int, default: None First year of the radiation time series. Defaults to first year avaiable. - endyear: int, default: None + end: int, default: None Last year of the radiation time series. Defaults to last year avaiable. pvcalculation: bool, default: False Also return estimate of hourly production. @@ -118,8 +119,8 @@ def get_pvgis_hourly(lat, lon, angle=0, aspect=0, outputformat='json', Time-series of hourly data, see Notes for fields inputs : dict Dictionary of the request input parameters, ``None`` for basic - meta : list or dict - meta data, ``None`` for basic + metadata : list or dict + metadata, ``None`` for basic Notes ----- @@ -168,7 +169,7 @@ def get_pvgis_hourly(lat, lon, angle=0, aspect=0, outputformat='json', `_ """ # use requests to format the query string by passing params dictionary - params = {'lat': lat, 'lon': lon, 'outputformat': outputformat, + params = {'lat': latitude, 'lon': longitude, 'outputformat': outputformat, 'angle': angle, 'aspect': aspect, 'pvtechchoice': pvtechchoice, 'mountingplace': mountingplace, 'trackingtype': trackingtype, 'components': int(components)} @@ -182,10 +183,10 @@ def get_pvgis_hourly(lat, lon, angle=0, aspect=0, outputformat='json', params['userhorizon'] = ','.join(str(x) for x in userhorizon) if raddatabase is not None: params['raddatabase'] = raddatabase - if startyear is not None: - params['startyear'] = startyear - if endyear is not None: - params['endyear'] = endyear + if start is not None: + params['startyear'] = start + if end is not None: + params['endyear'] = end if pvcalculation: params['pvcalculation'] = 1 if peakpower is not None: @@ -229,16 +230,16 @@ def get_pvgis_hourly(lat, lon, angle=0, aspect=0, outputformat='json', return data -def _parse_pvgis_hourly_json(src, map_variables=True): +def _parse_pvgis_hourly_json(src, map_variables): inputs = src['inputs'] - meta = src['meta'] + metadata = src['meta'] data = pd.DataFrame(src['outputs']['hourly']) data.index = pd.to_datetime(data['time'], format='%Y%m%d:%H%M', utc=True) data = data.drop('time', axis=1) data = data.astype(dtype={'Int': 'int'}) # The 'Int' column to be integer if map_variables: data.rename(columns=VARIABLE_MAP, inplace=True) - return data, inputs, meta + return data, inputs, metadata def _parse_pvgis_hourly_basic(src): @@ -264,7 +265,7 @@ def _parse_pvgis_hourly_csv(src, map_variables=True): while True: line = src.readline() if line.startswith('time,'): # The data header starts with 'time,' - # The last line of the meta-data section contains the column names + # The last line of the metadata section contains the column names names = line.replace('\n', '').replace('\r', '').split(',') break # Only retrieve metadata from non-empty lines @@ -291,11 +292,11 @@ def _parse_pvgis_hourly_csv(src, map_variables=True): # integer. It is necessary to convert to float, before converting to int data = data.astype(float).astype(dtype={'Int': 'int'}) # Generate metadata dictionary containing description of parameters - meta = {} + metadata = {} for line in src.readlines(): if ':' in line: - meta[line.split(':')[0]] = line.split(':')[1].strip() - return data, inputs, meta + metadata[line.split(':')[0]] = line.split(':')[1].strip() + return data, inputs, metadata def read_pvgis_hourly(filename, map_variables=True, pvgis_format=None): @@ -308,7 +309,7 @@ def read_pvgis_hourly(filename, map_variables=True, pvgis_format=None): Name, path, or buffer of file downloaded from PVGIS. pvgis_format : str, default None Format of PVGIS file or buffer. Equivalent to the ``outputformat`` - parameter in the PVGIS TMY API. If `filename` is a file and + parameter in the PVGIS API. If `filename` is a file and `pvgis_format` is ``None`` then the file extension will be used to determine the PVGIS format to parse. For PVGIS files from the API with ``outputformat='basic'``, please set `pvgis_format` to ``'basic'``. If @@ -321,8 +322,8 @@ def read_pvgis_hourly(filename, map_variables=True, pvgis_format=None): the weather data inputs : dict the inputs, ``None`` for basic - meta : list or dict - meta data, ``None`` for basic + metadata : list or dict + metadata, ``None`` for basic Raises ------ @@ -364,7 +365,7 @@ def read_pvgis_hourly(filename, map_variables=True, pvgis_format=None): return _parse_pvgis_hourly_json(src) # CSV or basic: use the correct parser from this module - # eg: _parse_pvgis_tmy_csv() or _parse_pvgist_tmy_basic() + # eg: _parse_pvgis_hourly_csv() or _parse_pvgis_hourly_basic() if outputformat in ['csv', 'basic']: # get the correct parser function for this output format from globals() pvgis_parser = globals()['_parse_pvgis_hourly_{:s}'.format(outputformat)] # noqa From 1ce1697a5794dc243663a067d3e5bf34bb8cab28 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Mon, 22 Feb 2021 22:14:15 +0100 Subject: [PATCH 08/51] Add cams.get_cams_radiation function --- docs/sphinx/source/api.rst | 1 + docs/sphinx/source/whatsnew/v0.9.0.rst | 3 + pvlib/iotools/__init__.py | 1 + pvlib/iotools/cams.py | 207 +++++++++++++++++++++++++ 4 files changed, 212 insertions(+) create mode 100644 pvlib/iotools/cams.py diff --git a/docs/sphinx/source/api.rst b/docs/sphinx/source/api.rst index 80550d98be..8a5b69bead 100644 --- a/docs/sphinx/source/api.rst +++ b/docs/sphinx/source/api.rst @@ -490,6 +490,7 @@ of sources and file formats relevant to solar energy modeling. iotools.get_pvgis_hourly iotools.read_pvgis_hourly iotools.read_bsrn + iotools.get_cams_mcclear A :py:class:`~pvlib.location.Location` object may be created from metadata in some files. diff --git a/docs/sphinx/source/whatsnew/v0.9.0.rst b/docs/sphinx/source/whatsnew/v0.9.0.rst index 9d9235a4f9..22c2e2761c 100644 --- a/docs/sphinx/source/whatsnew/v0.9.0.rst +++ b/docs/sphinx/source/whatsnew/v0.9.0.rst @@ -109,6 +109,9 @@ Enhancements files. (:pull:`1186`, :issue:`849`) * Add :func:`~pvlib.iotools.read_bsrn` for reading BSRN solar radiation data files. (:pull:`1145`, :issue:`1015`) +* Add :func:`~pvlib.iotools.get_cams_radiation` for retrieving CAMS McClear + clear-sky radiation time series. + files. (:pull:`1145`, :issue:`1015`) * In :py:class:`~pvlib.modelchain.ModelChain`, attributes which contain output of models are now collected into ``ModelChain.results``. (:pull:`1076`, :issue:`1067`) diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index 5b588b72a9..5c0e3a48ba 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -16,3 +16,4 @@ from pvlib.iotools.pvgis import read_pvgis_hourly # noqa: F401 from pvlib.iotools.pvgis import get_pvgis_hourly # noqa: F401 from pvlib.iotools.bsrn import read_bsrn # noqa: F401 +from pvlib.iotools.cams import get_cams_radiation # noqa: F401 diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/cams.py new file mode 100644 index 0000000000..c802420623 --- /dev/null +++ b/pvlib/iotools/cams.py @@ -0,0 +1,207 @@ +"""Functions to access data from Copernicus Atmosphere Monitoring Service + (CAMS) radiation service. +.. codeauthor:: Adam R. Jensen +""" + +import pandas as pd +import requests +import io + + +MCCLEAR_COLUMNS = ['Observation period', 'TOA', 'Clear sky GHI', + 'Clear sky BHI', 'Clear sky DHI', 'Clear sky BNI'] + +MCCLEAR_VERBOSE_COLUMNS = ['sza', 'summer/winter split', 'tco3', 'tcwv', + 'AOD BC', 'AOD DU', 'AOD SS', 'AOD OR', 'AOD SU', + 'AOD NI', 'AOD AM', 'alpha', 'Aerosol type', + 'fiso', 'fvol', 'fgeo', 'albedo'] + +# Dictionary mapping CAMS MCCLEAR variables to pvlib names +MCCLEAR_VARIABLE_MAP = { + 'TOA': 'ghi_extra', + 'Clear sky GHI': 'ghi_clear', + 'Clear sky BHI': 'bhi_clear', + 'Clear sky DHI': 'dhi_clear', + 'Clear sky BNI': 'dni_clear', + 'sza': 'solar_zenith', +} + + +# Dictionary mapping Python time steps to CAMS time step format +TIME_STEPS = {'1min': 'PT01M', '15min': 'PT15M', '1h': 'PT01H', '1d': 'P01D', + '1M': 'P01M'} + +TIME_STEPS_HOURS = {'1min': 1/60, '15min': 15/60, '1h': 1, '1d': 24} + + +def get_cams_mcclear(start_date, end_date, latitude, longitude, email, + altitude=None, time_step='1h', time_ref='UT', + integrated=False, label=None, verbose=False, + map_variables=True, server='www.soda-is.com'): + """ + Retrieve time-series of clear-sky global, beam, and diffuse radiation + anywhere in the world from CAMS McClear [1]_ using the WGET service [2]_. + + + Geographical coverage: wordwide + Time coverage: 2004-01-01 to two days ago + Access: free, but requires registration, see [1]_ + Requests: max. 100 per day + + + Parameters + ---------- + start_date: datetime like + First day of the requested period + end_date: datetime like + Last day of the requested period + latitude: float + in decimal degrees, between -90 and 90, north is positive (ISO 19115) + longitude : float + in decimal degrees, between -180 and 180, east is positive (ISO 19115) + altitude: float, default: None + Altitude in meters. If None, then the altitude is determined from the + NASA SRTM database + email: str + Email address linked to a SoDa account + time_step: str, {'1min', '15min', '1h', '1d', '1M'}, default: '1h' + Time step of the time series, either 1 minute, 15 minute, hourly, + daily, or monthly. + time_reference: str, {'UT', 'TST'}, default: 'UT' + 'UT' (universal time) or 'TST' (True Solar Time) + integrated: boolean, default False + Whether to return integrated irradiation values (Wh/m^2) from CAMS or + average irradiance values (W/m^2) as is more commonly used + label: {‘right’, ‘left’}, default: None + Which bin edge label to label bucket with. The default is ‘left’ for + all frequency offsets except for ‘M’ which has a default of ‘right’. + verbose: boolean, default: False + Verbose mode outputs additional parameters (aerosols). Only avaiable + for 1 minute and universal time. See [1] for parameter description. + map_variables: bool, default: True + When true, renames columns of the Dataframe to pvlib variable names + where applicable. See variable MCCLEAR_VARIABLE_MAP. + server: str, default: 'www.soda-is.com' + Main server (www.soda-is.com) or backup mirror server (pro.soda-is.com) + + + Notes + ---------- + The returned data Dataframe includes the following fields: + + ======================= ====== ========================================== + Key, mapped key Format Description + ======================= ====== ========================================== + **Mapped field names are returned when the map_variables argument is True** + -------------------------------------------------------------------------- + Observation period str Beginning/end of time period + TOA, ghi_extra float Horizontal radiation at top of atmosphere + Clear sky GHI, ghi_clear float Clear sky global radiation on horizontal + Clear sky BHI, bhi_clear float Clear sky beam radiation on horizontal + Clear sky DHI, dhi_clear float Clear sky diffuse radiation on horizontal + Clear sky BNI, dni_clear float Clear sky beam radiation normal to sun + ======================= ====== ========================================== + + For the returned units see the integrated argument. For description of + additional output parameters in verbose mode, see [1]. + + Note that it is recommended to specify the latitude and longitude to at + least the fourth decimal place. + + Variables corresponding to standard pvlib variables are renamed, + e.g. `sza` becomes `solar_zenith`. See the + `pvlib.iotools.cams.MCCLEAR_VARIABLE_MAP` dict for the complete mapping. + + + References + ---------- + .. [1] `CAMS McClear Service Info + `_ + .. [2] `CAMS McClear Automatic Access + `_ + """ + + if time_step in TIME_STEPS.keys(): + time_step_str = TIME_STEPS[time_step] + else: + print('WARNING: time step not recognized, 1 hour time step used!') + time_step_str = 'PT01H' + + names = MCCLEAR_COLUMNS + if verbose: + if (time_step == '1min') & (time_ref == 'UT'): + names += MCCLEAR_VERBOSE_COLUMNS + else: + verbose = False + print("Verbose mode only supports 1 min. UT time series!") + + if altitude is None: # Let SoDa get elevation from the NASA SRTM database + altitude = -999 + + # Start and end date should be in the format: yyyy-mm-dd + start_date = start_date.strftime('%Y-%m-%d') + end_date = end_date.strftime('%Y-%m-%d') + + email = email.replace('@', '%2540') # Format email address + + # Format verbose variable to the required format: {'true', 'false'} + verbose = str(verbose).lower() + + # Manual format the request url, due to uncommon usage of & and ; in url + url = ("http://{}/service/wps?Service=WPS&Request=Execute&" + "Identifier=get_mcclear&version=1.0.0&RawDataOutput=irradiation&" + "DataInputs=latitude={};longitude={};altitude={};" + "date_begin={};date_end={};time_ref={};summarization={};" + "username={};verbose={}" + ).format(server, latitude, longitude, altitude, start_date, + end_date, time_ref, time_step_str, email, verbose) + + res = requests.get(url) + + # Invalid requests returns helpful XML error message + if res.headers['Content-Type'] == 'application/xml': + print('REQUEST ERROR MESSAGE:') + print(res.text.split('ows:ExceptionText')[1][1:-2]) + + # Check if returned file is a csv data file + elif res.headers['Content-Type'] == 'application/csv': + data = pd.read_csv(io.StringIO(res.content.decode('utf-8')), sep=';', + comment='#', header=None, names=names) + + obs_period = data['Observation period'].str.split('/') + + # Set index as the start observation time (left) and localize to UTC + if (label == 'left') | ((label is None) & (time_step != '1M')): + data.index = pd.to_datetime(obs_period.str[0], utc=True) + # Set index as the stop observation time (right) and localize to UTC + elif (label == 'right') | ((label is None) & (time_step == '1M')): + data.index = pd.to_datetime(obs_period.str[1], utc=True) + + data.index.name = None # Set index name to None + + # Change index for '1d' and '1M' to be date and not datetime + if time_step == '1d': + data.index = data.index.date + elif (time_step == '1M') & (label is not None): + data.index = data.index.date + # For monthly data with 'right' label, the index should be the last + # date of the month and not the first date of the following month + elif (time_step == '1M') & (time_step != 'left'): + data.index = data.index.date - pd.Timestamp(days=1) + + if not integrated: # Convert from Wh/m2 to W/m2 + integrated_cols = MCCLEAR_COLUMNS[1:6] + + if time_step == '1M': + time_delta = (pd.to_datetime(obs_period.str[1]) + - pd.to_datetime(obs_period.str[0])) + hours = time_delta.dt.total_seconds()/60/60 + data[integrated_cols] = data[integrated_cols] / hours + else: + data[integrated_cols] = (data[integrated_cols] / + TIME_STEPS_HOURS[time_step]) + + if map_variables: + data = data.rename(columns=MCCLEAR_VARIABLE_MAP) + + return data From 334a99769efa0e71b0e565aab85dbc03cd96ac08 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Mon, 22 Feb 2021 22:14:29 +0100 Subject: [PATCH 09/51] Revert "Add cams.get_cams_radiation function" This reverts commit d7deb80cdc5d1b63de5b2865a0c5cf24d4655fc1. --- docs/sphinx/source/api.rst | 1 - docs/sphinx/source/whatsnew/v0.9.0.rst | 3 - pvlib/iotools/__init__.py | 1 - pvlib/iotools/cams.py | 207 ------------------------- 4 files changed, 212 deletions(-) delete mode 100644 pvlib/iotools/cams.py diff --git a/docs/sphinx/source/api.rst b/docs/sphinx/source/api.rst index 8a5b69bead..80550d98be 100644 --- a/docs/sphinx/source/api.rst +++ b/docs/sphinx/source/api.rst @@ -490,7 +490,6 @@ of sources and file formats relevant to solar energy modeling. iotools.get_pvgis_hourly iotools.read_pvgis_hourly iotools.read_bsrn - iotools.get_cams_mcclear A :py:class:`~pvlib.location.Location` object may be created from metadata in some files. diff --git a/docs/sphinx/source/whatsnew/v0.9.0.rst b/docs/sphinx/source/whatsnew/v0.9.0.rst index 22c2e2761c..9d9235a4f9 100644 --- a/docs/sphinx/source/whatsnew/v0.9.0.rst +++ b/docs/sphinx/source/whatsnew/v0.9.0.rst @@ -109,9 +109,6 @@ Enhancements files. (:pull:`1186`, :issue:`849`) * Add :func:`~pvlib.iotools.read_bsrn` for reading BSRN solar radiation data files. (:pull:`1145`, :issue:`1015`) -* Add :func:`~pvlib.iotools.get_cams_radiation` for retrieving CAMS McClear - clear-sky radiation time series. - files. (:pull:`1145`, :issue:`1015`) * In :py:class:`~pvlib.modelchain.ModelChain`, attributes which contain output of models are now collected into ``ModelChain.results``. (:pull:`1076`, :issue:`1067`) diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index 5c0e3a48ba..5b588b72a9 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -16,4 +16,3 @@ from pvlib.iotools.pvgis import read_pvgis_hourly # noqa: F401 from pvlib.iotools.pvgis import get_pvgis_hourly # noqa: F401 from pvlib.iotools.bsrn import read_bsrn # noqa: F401 -from pvlib.iotools.cams import get_cams_radiation # noqa: F401 diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/cams.py deleted file mode 100644 index c802420623..0000000000 --- a/pvlib/iotools/cams.py +++ /dev/null @@ -1,207 +0,0 @@ -"""Functions to access data from Copernicus Atmosphere Monitoring Service - (CAMS) radiation service. -.. codeauthor:: Adam R. Jensen -""" - -import pandas as pd -import requests -import io - - -MCCLEAR_COLUMNS = ['Observation period', 'TOA', 'Clear sky GHI', - 'Clear sky BHI', 'Clear sky DHI', 'Clear sky BNI'] - -MCCLEAR_VERBOSE_COLUMNS = ['sza', 'summer/winter split', 'tco3', 'tcwv', - 'AOD BC', 'AOD DU', 'AOD SS', 'AOD OR', 'AOD SU', - 'AOD NI', 'AOD AM', 'alpha', 'Aerosol type', - 'fiso', 'fvol', 'fgeo', 'albedo'] - -# Dictionary mapping CAMS MCCLEAR variables to pvlib names -MCCLEAR_VARIABLE_MAP = { - 'TOA': 'ghi_extra', - 'Clear sky GHI': 'ghi_clear', - 'Clear sky BHI': 'bhi_clear', - 'Clear sky DHI': 'dhi_clear', - 'Clear sky BNI': 'dni_clear', - 'sza': 'solar_zenith', -} - - -# Dictionary mapping Python time steps to CAMS time step format -TIME_STEPS = {'1min': 'PT01M', '15min': 'PT15M', '1h': 'PT01H', '1d': 'P01D', - '1M': 'P01M'} - -TIME_STEPS_HOURS = {'1min': 1/60, '15min': 15/60, '1h': 1, '1d': 24} - - -def get_cams_mcclear(start_date, end_date, latitude, longitude, email, - altitude=None, time_step='1h', time_ref='UT', - integrated=False, label=None, verbose=False, - map_variables=True, server='www.soda-is.com'): - """ - Retrieve time-series of clear-sky global, beam, and diffuse radiation - anywhere in the world from CAMS McClear [1]_ using the WGET service [2]_. - - - Geographical coverage: wordwide - Time coverage: 2004-01-01 to two days ago - Access: free, but requires registration, see [1]_ - Requests: max. 100 per day - - - Parameters - ---------- - start_date: datetime like - First day of the requested period - end_date: datetime like - Last day of the requested period - latitude: float - in decimal degrees, between -90 and 90, north is positive (ISO 19115) - longitude : float - in decimal degrees, between -180 and 180, east is positive (ISO 19115) - altitude: float, default: None - Altitude in meters. If None, then the altitude is determined from the - NASA SRTM database - email: str - Email address linked to a SoDa account - time_step: str, {'1min', '15min', '1h', '1d', '1M'}, default: '1h' - Time step of the time series, either 1 minute, 15 minute, hourly, - daily, or monthly. - time_reference: str, {'UT', 'TST'}, default: 'UT' - 'UT' (universal time) or 'TST' (True Solar Time) - integrated: boolean, default False - Whether to return integrated irradiation values (Wh/m^2) from CAMS or - average irradiance values (W/m^2) as is more commonly used - label: {‘right’, ‘left’}, default: None - Which bin edge label to label bucket with. The default is ‘left’ for - all frequency offsets except for ‘M’ which has a default of ‘right’. - verbose: boolean, default: False - Verbose mode outputs additional parameters (aerosols). Only avaiable - for 1 minute and universal time. See [1] for parameter description. - map_variables: bool, default: True - When true, renames columns of the Dataframe to pvlib variable names - where applicable. See variable MCCLEAR_VARIABLE_MAP. - server: str, default: 'www.soda-is.com' - Main server (www.soda-is.com) or backup mirror server (pro.soda-is.com) - - - Notes - ---------- - The returned data Dataframe includes the following fields: - - ======================= ====== ========================================== - Key, mapped key Format Description - ======================= ====== ========================================== - **Mapped field names are returned when the map_variables argument is True** - -------------------------------------------------------------------------- - Observation period str Beginning/end of time period - TOA, ghi_extra float Horizontal radiation at top of atmosphere - Clear sky GHI, ghi_clear float Clear sky global radiation on horizontal - Clear sky BHI, bhi_clear float Clear sky beam radiation on horizontal - Clear sky DHI, dhi_clear float Clear sky diffuse radiation on horizontal - Clear sky BNI, dni_clear float Clear sky beam radiation normal to sun - ======================= ====== ========================================== - - For the returned units see the integrated argument. For description of - additional output parameters in verbose mode, see [1]. - - Note that it is recommended to specify the latitude and longitude to at - least the fourth decimal place. - - Variables corresponding to standard pvlib variables are renamed, - e.g. `sza` becomes `solar_zenith`. See the - `pvlib.iotools.cams.MCCLEAR_VARIABLE_MAP` dict for the complete mapping. - - - References - ---------- - .. [1] `CAMS McClear Service Info - `_ - .. [2] `CAMS McClear Automatic Access - `_ - """ - - if time_step in TIME_STEPS.keys(): - time_step_str = TIME_STEPS[time_step] - else: - print('WARNING: time step not recognized, 1 hour time step used!') - time_step_str = 'PT01H' - - names = MCCLEAR_COLUMNS - if verbose: - if (time_step == '1min') & (time_ref == 'UT'): - names += MCCLEAR_VERBOSE_COLUMNS - else: - verbose = False - print("Verbose mode only supports 1 min. UT time series!") - - if altitude is None: # Let SoDa get elevation from the NASA SRTM database - altitude = -999 - - # Start and end date should be in the format: yyyy-mm-dd - start_date = start_date.strftime('%Y-%m-%d') - end_date = end_date.strftime('%Y-%m-%d') - - email = email.replace('@', '%2540') # Format email address - - # Format verbose variable to the required format: {'true', 'false'} - verbose = str(verbose).lower() - - # Manual format the request url, due to uncommon usage of & and ; in url - url = ("http://{}/service/wps?Service=WPS&Request=Execute&" - "Identifier=get_mcclear&version=1.0.0&RawDataOutput=irradiation&" - "DataInputs=latitude={};longitude={};altitude={};" - "date_begin={};date_end={};time_ref={};summarization={};" - "username={};verbose={}" - ).format(server, latitude, longitude, altitude, start_date, - end_date, time_ref, time_step_str, email, verbose) - - res = requests.get(url) - - # Invalid requests returns helpful XML error message - if res.headers['Content-Type'] == 'application/xml': - print('REQUEST ERROR MESSAGE:') - print(res.text.split('ows:ExceptionText')[1][1:-2]) - - # Check if returned file is a csv data file - elif res.headers['Content-Type'] == 'application/csv': - data = pd.read_csv(io.StringIO(res.content.decode('utf-8')), sep=';', - comment='#', header=None, names=names) - - obs_period = data['Observation period'].str.split('/') - - # Set index as the start observation time (left) and localize to UTC - if (label == 'left') | ((label is None) & (time_step != '1M')): - data.index = pd.to_datetime(obs_period.str[0], utc=True) - # Set index as the stop observation time (right) and localize to UTC - elif (label == 'right') | ((label is None) & (time_step == '1M')): - data.index = pd.to_datetime(obs_period.str[1], utc=True) - - data.index.name = None # Set index name to None - - # Change index for '1d' and '1M' to be date and not datetime - if time_step == '1d': - data.index = data.index.date - elif (time_step == '1M') & (label is not None): - data.index = data.index.date - # For monthly data with 'right' label, the index should be the last - # date of the month and not the first date of the following month - elif (time_step == '1M') & (time_step != 'left'): - data.index = data.index.date - pd.Timestamp(days=1) - - if not integrated: # Convert from Wh/m2 to W/m2 - integrated_cols = MCCLEAR_COLUMNS[1:6] - - if time_step == '1M': - time_delta = (pd.to_datetime(obs_period.str[1]) - - pd.to_datetime(obs_period.str[0])) - hours = time_delta.dt.total_seconds()/60/60 - data[integrated_cols] = data[integrated_cols] / hours - else: - data[integrated_cols] = (data[integrated_cols] / - TIME_STEPS_HOURS[time_step]) - - if map_variables: - data = data.rename(columns=MCCLEAR_VARIABLE_MAP) - - return data From b416f6243954ae399326ccc33dfdacfc6c847209 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sun, 13 Jun 2021 13:09:54 -0500 Subject: [PATCH 10/51] Add requests-mock to CI files --- ci/azure/posix.yml | 2 +- ci/requirements-py36-min.yml | 1 + ci/requirements-py36.yml | 1 + ci/requirements-py37.yml | 1 + ci/requirements-py38.yml | 1 + ci/requirements-py39.yml | 1 + setup.py | 3 ++- 7 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index cbcdf5694d..086f03dd69 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -23,7 +23,7 @@ jobs: versionSpec: '$(python.version)' - script: | - pip install pytest pytest-cov pytest-mock pytest-timeout pytest-azurepipelines pytest-rerunfailures pytest-remotedata + pip install pytest pytest-cov pytest-mock requests-mock pytest-timeout pytest-azurepipelines pytest-rerunfailures pytest-remotedata pip install -e . pytest pvlib --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html displayName: 'Test with pytest' diff --git a/ci/requirements-py36-min.yml b/ci/requirements-py36-min.yml index 29f63c1be1..84adcb360d 100644 --- a/ci/requirements-py36-min.yml +++ b/ci/requirements-py36-min.yml @@ -19,3 +19,4 @@ dependencies: - scipy==1.2.0 - pytest-rerunfailures # conda version is >3.6 - pytest-remotedata # conda package is 0.3.0, needs > 0.3.1 + - requests-mock diff --git a/ci/requirements-py36.yml b/ci/requirements-py36.yml index fb37fe8404..c49455119f 100644 --- a/ci/requirements-py36.yml +++ b/ci/requirements-py36.yml @@ -16,6 +16,7 @@ dependencies: - pytest - pytest-cov - pytest-mock + - requests-mock - pytest-rerunfailures - pytest-remotedata - pytest-timeout diff --git a/ci/requirements-py37.yml b/ci/requirements-py37.yml index 1542bb35d9..3203b004d1 100644 --- a/ci/requirements-py37.yml +++ b/ci/requirements-py37.yml @@ -16,6 +16,7 @@ dependencies: - pytest - pytest-cov - pytest-mock + - requests-mock - pytest-timeout - pytest-rerunfailures - pytest-remotedata diff --git a/ci/requirements-py38.yml b/ci/requirements-py38.yml index 6db508fd53..ca3a968335 100644 --- a/ci/requirements-py38.yml +++ b/ci/requirements-py38.yml @@ -16,6 +16,7 @@ dependencies: - pytest - pytest-cov - pytest-mock + - requests-mock - pytest-timeout - pytest-rerunfailures - pytest-remotedata diff --git a/ci/requirements-py39.yml b/ci/requirements-py39.yml index 35a3fa1952..16c6449158 100644 --- a/ci/requirements-py39.yml +++ b/ci/requirements-py39.yml @@ -16,6 +16,7 @@ dependencies: - pytest - pytest-cov - pytest-mock + - requests-mock - pytest-timeout - pytest-rerunfailures - pytest-remotedata diff --git a/setup.py b/setup.py index a876f0e995..216dc34a28 100755 --- a/setup.py +++ b/setup.py @@ -49,7 +49,8 @@ INSTALL_REQUIRES.append('dataclasses') TESTS_REQUIRE = ['nose', 'pytest', 'pytest-cov', 'pytest-mock', - 'pytest-timeout', 'pytest-rerunfailures', 'pytest-remotedata'] + 'requests-mock', 'pytest-timeout', 'pytest-rerunfailures', + 'pytest-remotedata'] EXTRAS_REQUIRE = { 'optional': ['cython', 'ephem', 'netcdf4', 'nrel-pysam', 'numba', 'pvfactors', 'siphon', 'statsmodels', 'tables', From fb8e1dc3bb537c2d175f3b169a69443a3809818f Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Wed, 16 Jun 2021 13:47:27 -0500 Subject: [PATCH 11/51] Revert "Add requests-mock to CI files" This reverts commit b416f6243954ae399326ccc33dfdacfc6c847209. --- ci/azure/posix.yml | 2 +- ci/requirements-py36-min.yml | 1 - ci/requirements-py36.yml | 1 - ci/requirements-py37.yml | 1 - ci/requirements-py38.yml | 1 - ci/requirements-py39.yml | 1 - setup.py | 3 +-- 7 files changed, 2 insertions(+), 8 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 086f03dd69..cbcdf5694d 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -23,7 +23,7 @@ jobs: versionSpec: '$(python.version)' - script: | - pip install pytest pytest-cov pytest-mock requests-mock pytest-timeout pytest-azurepipelines pytest-rerunfailures pytest-remotedata + pip install pytest pytest-cov pytest-mock pytest-timeout pytest-azurepipelines pytest-rerunfailures pytest-remotedata pip install -e . pytest pvlib --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html displayName: 'Test with pytest' diff --git a/ci/requirements-py36-min.yml b/ci/requirements-py36-min.yml index 84adcb360d..29f63c1be1 100644 --- a/ci/requirements-py36-min.yml +++ b/ci/requirements-py36-min.yml @@ -19,4 +19,3 @@ dependencies: - scipy==1.2.0 - pytest-rerunfailures # conda version is >3.6 - pytest-remotedata # conda package is 0.3.0, needs > 0.3.1 - - requests-mock diff --git a/ci/requirements-py36.yml b/ci/requirements-py36.yml index c49455119f..fb37fe8404 100644 --- a/ci/requirements-py36.yml +++ b/ci/requirements-py36.yml @@ -16,7 +16,6 @@ dependencies: - pytest - pytest-cov - pytest-mock - - requests-mock - pytest-rerunfailures - pytest-remotedata - pytest-timeout diff --git a/ci/requirements-py37.yml b/ci/requirements-py37.yml index 3203b004d1..1542bb35d9 100644 --- a/ci/requirements-py37.yml +++ b/ci/requirements-py37.yml @@ -16,7 +16,6 @@ dependencies: - pytest - pytest-cov - pytest-mock - - requests-mock - pytest-timeout - pytest-rerunfailures - pytest-remotedata diff --git a/ci/requirements-py38.yml b/ci/requirements-py38.yml index ca3a968335..6db508fd53 100644 --- a/ci/requirements-py38.yml +++ b/ci/requirements-py38.yml @@ -16,7 +16,6 @@ dependencies: - pytest - pytest-cov - pytest-mock - - requests-mock - pytest-timeout - pytest-rerunfailures - pytest-remotedata diff --git a/ci/requirements-py39.yml b/ci/requirements-py39.yml index 16c6449158..35a3fa1952 100644 --- a/ci/requirements-py39.yml +++ b/ci/requirements-py39.yml @@ -16,7 +16,6 @@ dependencies: - pytest - pytest-cov - pytest-mock - - requests-mock - pytest-timeout - pytest-rerunfailures - pytest-remotedata diff --git a/setup.py b/setup.py index 216dc34a28..a876f0e995 100755 --- a/setup.py +++ b/setup.py @@ -49,8 +49,7 @@ INSTALL_REQUIRES.append('dataclasses') TESTS_REQUIRE = ['nose', 'pytest', 'pytest-cov', 'pytest-mock', - 'requests-mock', 'pytest-timeout', 'pytest-rerunfailures', - 'pytest-remotedata'] + 'pytest-timeout', 'pytest-rerunfailures', 'pytest-remotedata'] EXTRAS_REQUIRE = { 'optional': ['cython', 'ephem', 'netcdf4', 'nrel-pysam', 'numba', 'pvfactors', 'siphon', 'statsmodels', 'tables', From 7612640379bfb07f3df1f7039eda101eec3738d2 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Wed, 16 Jun 2021 22:13:59 -0500 Subject: [PATCH 12/51] Add tests and test files --- ...000_8.000_CM_10kWp_CIS_5_2a_2013_2014.json | 1 + ...s_45.000_8.000_SA_30deg_0deg_2016_2016.csv | 35 ++++++ pvlib/iotools/pvgis.py | 24 ++--- pvlib/tests/iotools/test_pvgis.py | 100 +++++++++++++++++- 4 files changed, 147 insertions(+), 13 deletions(-) create mode 100644 pvlib/data/pvgis_hourly_Timeseries_45.000_8.000_CM_10kWp_CIS_5_2a_2013_2014.json create mode 100644 pvlib/data/pvgis_hourly_Timeseries_45.000_8.000_SA_30deg_0deg_2016_2016.csv diff --git a/pvlib/data/pvgis_hourly_Timeseries_45.000_8.000_CM_10kWp_CIS_5_2a_2013_2014.json b/pvlib/data/pvgis_hourly_Timeseries_45.000_8.000_CM_10kWp_CIS_5_2a_2013_2014.json new file mode 100644 index 0000000000..3a27f4f368 --- /dev/null +++ b/pvlib/data/pvgis_hourly_Timeseries_45.000_8.000_CM_10kWp_CIS_5_2a_2013_2014.json @@ -0,0 +1 @@ +{"inputs": {"location": {"latitude": 45.0, "longitude": 8.0, "elevation": 250.0}, "meteo_data": {"radiation_db": "PVGIS-CMSAF", "meteo_db": "ERA-Interim", "year_min": 2013, "year_max": 2014, "use_horizon": true, "horizon_db": null, "horizon_data": "DEM-calculated"}, "mounting_system": {"two_axis": {"slope": {"value": "-", "optimal": "-"}, "azimuth": {"value": "-", "optimal": "-"}}}, "pv_module": {"technology": "CIS", "peak_power": 10.0, "system_loss": 5.0}}, "outputs": {"hourly": [{"time": "20130101:0055", "P": 0.0, "Gb(i)": 0.0, "Gd(i)": 0.0, "Gr(i)": 0.0, "H_sun": 0.0, "T2m": 3.01, "WS10m": 1.23, "Int": 0.0}, {"time": "20130101:0155", "P": 0.0, "Gb(i)": 0.0, "Gd(i)": 0.0, "Gr(i)": 0.0, "H_sun": 0.0, "T2m": 2.22, "WS10m": 1.46, "Int": 0.0}, {"time": "20130101:0255", "P": 0.0, "Gb(i)": 0.0, "Gd(i)": 0.0, "Gr(i)": 0.0, "H_sun": 0.0, "T2m": 1.43, "WS10m": 1.7, "Int": 0.0}, {"time": "20130101:0355", "P": 0.0, "Gb(i)": 0.0, "Gd(i)": 0.0, "Gr(i)": 0.0, "H_sun": 0.0, "T2m": 0.64, "WS10m": 1.93, "Int": 0.0}, {"time": "20130101:0455", "P": 0.0, "Gb(i)": 0.0, "Gd(i)": 0.0, "Gr(i)": 0.0, "H_sun": 0.0, "T2m": 0.77, "WS10m": 1.8, "Int": 0.0}, {"time": "20130101:0555", "P": 0.0, "Gb(i)": 0.0, "Gd(i)": 0.0, "Gr(i)": 0.0, "H_sun": 0.0, "T2m": 0.91, "WS10m": 1.66, "Int": 0.0}, {"time": "20130101:0655", "P": 0.0, "Gb(i)": 0.0, "Gd(i)": 0.0, "Gr(i)": 0.0, "H_sun": 0.0, "T2m": 1.05, "WS10m": 1.53, "Int": 0.0}, {"time": "20130101:0755", "P": 3464.5, "Gb(i)": 270.35, "Gd(i)": 91.27, "Gr(i)": 6.09, "H_sun": 6.12, "T2m": 1.92, "WS10m": 1.44, "Int": 0.0}, {"time": "20130101:0855", "P": 1586.9, "Gb(i)": 80.76, "Gd(i)": 83.95, "Gr(i)": 9.04, "H_sun": 13.28, "T2m": 2.79, "WS10m": 1.36, "Int": 0.0}, {"time": "20130101:0955", "P": 713.3, "Gb(i)": 5.18, "Gd(i)": 70.57, "Gr(i)": 7.31, "H_sun": 18.56, "T2m": 3.66, "WS10m": 1.27, "Int": 0.0}]}, "meta": {"inputs": {"location": {"description": "Selected location", "variables": {"latitude": {"description": "Latitude", "units": "decimal degree"}, "longitude": {"description": "Longitude", "units": "decimal degree"}, "elevation": {"description": "Elevation", "units": "m"}}}, "meteo_data": {"description": "Sources of meteorological data", "variables": {"radiation_db": {"description": "Solar radiation database"}, "meteo_db": {"description": "Database used for meteorological variables other than solar radiation"}, "year_min": {"description": "First year of the calculations"}, "year_max": {"description": "Last year of the calculations"}, "use_horizon": {"description": "Include horizon shadows"}, "horizon_db": {"description": "Source of horizon data"}}}, "mounting_system": {"description": "Mounting system", "choices": "fixed, vertical_axis, inclined_axis, two_axis", "fields": {"slope": {"description": "Inclination angle from the horizontal plane", "units": "degree"}, "azimuth": {"description": "Orientation (azimuth) angle of the (fixed) PV system (0 = S, 90 = W, -90 = E)", "units": "degree"}}}, "pv_module": {"description": "PV module parameters", "variables": {"technology": {"description": "PV technology"}, "peak_power": {"description": "Nominal (peak) power of the PV module", "units": "kW"}, "system_loss": {"description": "Sum of system losses", "units": "%"}}}}, "outputs": {"hourly": {"type": "time series", "timestamp": "hourly averages", "variables": {"P": {"description": "PV system power", "units": "W"}, "Gb(i)": {"description": "Beam (direct) irradiance on the inclined plane (plane of the array)", "units": "W/m2"}, "Gd(i)": {"description": "Diffuse irradiance on the inclined plane (plane of the array)", "units": "W/m2"}, "Gr(i)": {"description": "Reflected irradiance on the inclined plane (plane of the array)", "units": "W/m2"}, "H_sun": {"description": "Sun height", "units": "degree"}, "T2m": {"description": "2-m air temperature", "units": "degree Celsius"}, "WS10m": {"description": "10-m total wind speed", "units": "m/s"}, "Int": {"description": "1 means solar radiation values are reconstructed"}}}}}} \ No newline at end of file diff --git a/pvlib/data/pvgis_hourly_Timeseries_45.000_8.000_SA_30deg_0deg_2016_2016.csv b/pvlib/data/pvgis_hourly_Timeseries_45.000_8.000_SA_30deg_0deg_2016_2016.csv new file mode 100644 index 0000000000..a71a213a80 --- /dev/null +++ b/pvlib/data/pvgis_hourly_Timeseries_45.000_8.000_SA_30deg_0deg_2016_2016.csv @@ -0,0 +1,35 @@ +Latitude (decimal degrees): 45.000 +Longitude (decimal degrees): 8.000 +Elevation (m): 250 +Radiation database: PVGIS-SARAH + + +Slope: 30 deg. +Azimuth: 0 deg. +time,Gb(i),Gd(i),Gr(i),H_sun,T2m,WS10m,Int +20160101:0010,0.0,0.0,0.0,0.0,3.44,1.43,0.0 +20160101:0110,0.0,0.0,0.0,0.0,2.94,1.47,0.0 +20160101:0210,0.0,0.0,0.0,0.0,2.43,1.51,0.0 +20160101:0310,0.0,0.0,0.0,0.0,1.93,1.54,0.0 +20160101:0410,0.0,0.0,0.0,0.0,2.03,1.62,0.0 +20160101:0510,0.0,0.0,0.0,0.0,2.14,1.69,0.0 +20160101:0610,0.0,0.0,0.0,0.0,2.25,1.77,0.0 +20160101:0710,0.0,0.0,0.0,0.0,3.06,1.49,0.0 +20160101:0810,26.71,8.28,0.21,8.06,3.87,1.22,1.0 +20160101:0910,14.69,5.76,0.16,14.8,4.67,0.95,1.0 +20160101:1010,2.19,0.94,0.03,19.54,5.73,0.77,1.0 +20160101:1110,2.11,0.94,0.03,21.82,6.79,0.58,1.0 +20160101:1210,4.25,1.88,0.05,21.41,7.84,0.4,1.0 +20160101:1310,0.0,0.0,0.0,0.0,7.43,0.72,0.0 + +Gb(i): Beam (direct) irradiance on the inclined plane (plane of the array) (W/m2) +Gd(i): Diffuse irradiance on the inclined plane (plane of the array) (W/m2) +Gr(i): Reflected irradiance on the inclined plane (plane of the array) (W/m2) +H_sun: Sun height (degree) +T2m: 2-m air temperature (degree Celsius) +WS10m: 10-m total wind speed (m/s) +Int: 1 means solar radiation values are reconstructed + + + +PVGIS (c) European Union, 2001-2021 \ No newline at end of file diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index f1ab01a001..4852c0b622 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -41,13 +41,13 @@ } -def get_pvgis_hourly(latitude, longitude, angle=0, aspect=0, +def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, outputformat='json', usehorizon=True, userhorizon=None, raddatabase=None, start=None, end=None, pvcalculation=False, peakpower=None, pvtechchoice='crystSi', mountingplace='free', loss=None, trackingtype=0, - optimal_inclination=False, optimalangles=False, + optimalinclination=False, optimalangles=False, components=True, url=URL, map_variables=True, timeout=30): """ Get hourly solar irradiation and modeled PV power output from PVGIS [1]_. @@ -58,9 +58,9 @@ def get_pvgis_hourly(latitude, longitude, angle=0, aspect=0, Latitude in degrees north longitude: float Longitude in degrees east - angle: float, default: 0 + surface_tilt: float, default: 0 Tilt angle from horizontal plane. Not relevant for 2-axis tracking. - aspect: float, default: 0 + surface_azimuth: float, default: 0 Orientation (azimuth angle) of the (fixed) plane. 0=south, 90=west, -90: east. Not relevant for tracking systems. outputformat: str, default: 'json' @@ -170,7 +170,7 @@ def get_pvgis_hourly(latitude, longitude, angle=0, aspect=0, """ # use requests to format the query string by passing params dictionary params = {'lat': latitude, 'lon': longitude, 'outputformat': outputformat, - 'angle': angle, 'aspect': aspect, + 'angle': surface_tilt, 'aspect': surface_azimuth, 'pvtechchoice': pvtechchoice, 'mountingplace': mountingplace, 'trackingtype': trackingtype, 'components': int(components)} # pvgis only likes 0 for False, and 1 for True, not strings, also the @@ -193,8 +193,8 @@ def get_pvgis_hourly(latitude, longitude, angle=0, aspect=0, params['peakpower'] = peakpower if loss is not None: params['loss'] = loss - if optimal_inclination: - params['optimal_inclination'] = 1 + if optimalinclination: + params['optimalinclination'] = 1 if optimalangles: params['optimalangles'] = 1 @@ -242,13 +242,13 @@ def _parse_pvgis_hourly_json(src, map_variables): return data, inputs, metadata -def _parse_pvgis_hourly_basic(src): +def _parse_pvgis_hourly_basic(src, map_variables): # Hourly data with outputformat='basic' does not include header or metadata data = pd.read_csv(src, header=None, skiprows=2) return data, None, None -def _parse_pvgis_hourly_csv(src, map_variables=True): +def _parse_pvgis_hourly_csv(src, map_variables): # The first 4 rows are latitude, longitude, elevation, radiation database inputs = {} # 'Latitude (decimal degrees): 45.000\r\n' @@ -362,7 +362,7 @@ def read_pvgis_hourly(filename, map_variables=True, pvgis_format=None): except AttributeError: # str/path has no .read() attribute with open(str(filename), 'r') as fbuf: src = json.load(fbuf) - return _parse_pvgis_hourly_json(src) + return _parse_pvgis_hourly_json(src, map_variables=map_variables) # CSV or basic: use the correct parser from this module # eg: _parse_pvgis_hourly_csv() or _parse_pvgis_hourly_basic() @@ -372,10 +372,10 @@ def read_pvgis_hourly(filename, map_variables=True, pvgis_format=None): # NOTE: pvgis_parse() is a pvgis parser function from this module, # either _parse_pvgis_hourly_csv() or _parse_pvgist_hourly_basic() try: - pvgis_data = pvgis_parser(filename) + pvgis_data = pvgis_parser(filename, map_variables=map_variables) except AttributeError: # str/path has no .read() attribute with open(str(filename), 'r') as fbuf: - pvgis_data = pvgis_parser(fbuf) + pvgis_data = pvgis_parser(fbuf, map_variables=map_variables) return pvgis_data # raise exception if pvgis format isn't in ['csv', 'basic', 'json'] diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index a6c3c4510b..dcf41a5d8b 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -7,9 +7,107 @@ import pytest import requests from pvlib.iotools import get_pvgis_tmy, read_pvgis_tmy -from ..conftest import DATA_DIR, RERUNS, RERUNS_DELAY +from pvlib.iotools import get_pvgis_hourly, read_pvgis_hourly +from ..conftest import DATA_DIR, RERUNS, RERUNS_DELAY, assert_frame_equal + + +testfile_radiation_csv = DATA_DIR / 'pvgis_hourly_Timeseries_45.000_8.000_SA_30deg_0deg_2016_2016.csv' +testfile_pv_json = DATA_DIR / 'pvgis_hourly_Timeseries_45.000_8.000_CM_10kWp_CIS_5_2a_2013_2014.json' + +index_radiation_csv = \ + pd.date_range('20160101 00:10', freq='1h', periods=14, tz='UTC') + +index_pv_json = \ + pd.date_range('2013-01-01 00:55', freq='1h', periods=10, tz='UTC') + +columns_radiation_csv = [ + 'Gb(i)', 'Gd(i)', 'Gr(i)', 'H_sun', 'T2m', 'WS10m','Int'] + +columns_pv_json = [ + 'P', 'Gb(i)', 'Gd(i)', 'Gr(i)', 'H_sun', 'T2m', 'WS10m', 'Int'] + +columns_pv_json_mapped = [ + 'P', 'poa_direct', 'poa_diffuse', 'poa_ground_diffuse', 'solar_elevation', + 'temp_air', 'wind_speed', 'Int'] + +data_radiation_csv = [ + [0.0, 0.0, 0.0, 0.0, 3.44, 1.43, 0.0], + [0.0, 0.0, 0.0, 0.0, 2.94, 1.47, 0.0], + [0.0, 0.0, 0.0, 0.0, 2.43, 1.51, 0.0], + [0.0, 0.0, 0.0, 0.0, 1.93, 1.54, 0.0], + [0.0, 0.0, 0.0, 0.0, 2.03, 1.62, 0.0], + [0.0, 0.0, 0.0, 0.0, 2.14, 1.69, 0.0], + [0.0, 0.0, 0.0, 0.0, 2.25, 1.77, 0.0], + [0.0, 0.0, 0.0, 0.0, 3.06, 1.49, 0.0], + [26.71, 8.28, 0.21, 8.06, 3.87, 1.22, 1.0], + [14.69, 5.76, 0.16, 14.8, 4.67, 0.95, 1.0], + [2.19, 0.94, 0.03, 19.54, 5.73, 0.77, 1.0], + [2.11, 0.94, 0.03, 21.82, 6.79, 0.58, 1.0], + [4.25, 1.88, 0.05, 21.41, 7.84, 0.4, 1.0], + [0.0, 0.0, 0.0, 0.0, 7.43, 0.72, 0.0]] + +data_pv_json = [ + [0.0, 0.0, 0.0, 0.0, 0.0, 3.01, 1.23, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 2.22, 1.46, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 1.43, 1.7, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.64, 1.93, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.77, 1.8, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.91, 1.66, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 1.05, 1.53, 0.0], + [3464.5, 270.35, 91.27, 6.09, 6.12, 1.92, 1.44, 0.0], + [1586.9, 80.76, 83.95, 9.04, 13.28, 2.79, 1.36, 0.0], + [713.3, 5.18, 70.57, 7.31, 18.56, 3.66, 1.27, 0.0]] + +@pytest.fixture +def expected_radiation_csv(): + expected = pd.DataFrame(index=index_radiation_csv, data=data_radiation_csv, + columns=columns_radiation_csv) + expected['Int'] = expected['Int'].astype(int) + expected.index.name = 'time' + expected.index.freq = None + return expected + +def test_read_pvgis_hourly_csv(expected_radiation_csv): + out, inputs, metadata = read_pvgis_hourly(testfile_radiation_csv, + map_variables=False) + assert_frame_equal(out, expected_radiation_csv) + + +@pytest.fixture +def expected_pv_json(): + expected = pd.DataFrame(index=index_pv_json, data=data_pv_json, + columns=columns_pv_json) + expected['Int'] = expected['Int'].astype(int) + expected.index.name = 'time' + expected.index.freq = None + return expected + + + + +def test_read_pvgis_hourly_json(expected_pv_json): + out, inputs, metadata = read_pvgis_hourly(testfile_pv_json, + map_variables=False) + assert_frame_equal(out, expected_pv_json) + +@pytest.fixture +def expected_pv_json_mapped(): + expected = pd.DataFrame(index=index_pv_json, data=data_pv_json, + columns=columns_pv_json_mapped) + expected['Int'] = expected['Int'].astype(int) + expected.index.name = 'time' + expected.index.freq = None + return expected + + +def test_read_pvgis_hourly_json_mapped(expected_pv_json_mapped): + out, inputs, metadata = read_pvgis_hourly(testfile_pv_json, + map_variables=True, + pvgis_format='json') + assert_frame_equal(out, expected_pv_json_mapped) +# PVGIS TMY tests @pytest.fixture def expected(): return pd.read_csv(DATA_DIR / 'pvgis_tmy_test.dat', index_col='time(UTC)') From 902ed2b4847aa0c759569a2491e7697b2c21118f Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Wed, 16 Jun 2021 22:19:37 -0500 Subject: [PATCH 13/51] Fix stickler --- pvlib/tests/iotools/test_pvgis.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index dcf41a5d8b..b10e887e65 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -7,12 +7,14 @@ import pytest import requests from pvlib.iotools import get_pvgis_tmy, read_pvgis_tmy -from pvlib.iotools import get_pvgis_hourly, read_pvgis_hourly +from pvlib.iotools import read_pvgis_hourly # get_pvgis_hourly, from ..conftest import DATA_DIR, RERUNS, RERUNS_DELAY, assert_frame_equal -testfile_radiation_csv = DATA_DIR / 'pvgis_hourly_Timeseries_45.000_8.000_SA_30deg_0deg_2016_2016.csv' -testfile_pv_json = DATA_DIR / 'pvgis_hourly_Timeseries_45.000_8.000_CM_10kWp_CIS_5_2a_2013_2014.json' +testfile_radiation_csv = DATA_DIR / \ + 'pvgis_hourly_Timeseries_45.000_8.000_SA_30deg_0deg_2016_2016.csv' +testfile_pv_json = DATA_DIR / \ + 'pvgis_hourly_Timeseries_45.000_8.000_CM_10kWp_CIS_5_2a_2013_2014.json' index_radiation_csv = \ pd.date_range('20160101 00:10', freq='1h', periods=14, tz='UTC') @@ -21,7 +23,7 @@ pd.date_range('2013-01-01 00:55', freq='1h', periods=10, tz='UTC') columns_radiation_csv = [ - 'Gb(i)', 'Gd(i)', 'Gr(i)', 'H_sun', 'T2m', 'WS10m','Int'] + 'Gb(i)', 'Gd(i)', 'Gr(i)', 'H_sun', 'T2m', 'WS10m', 'Int'] columns_pv_json = [ 'P', 'Gb(i)', 'Gd(i)', 'Gr(i)', 'H_sun', 'T2m', 'WS10m', 'Int'] @@ -58,15 +60,17 @@ [1586.9, 80.76, 83.95, 9.04, 13.28, 2.79, 1.36, 0.0], [713.3, 5.18, 70.57, 7.31, 18.56, 3.66, 1.27, 0.0]] + @pytest.fixture def expected_radiation_csv(): expected = pd.DataFrame(index=index_radiation_csv, data=data_radiation_csv, - columns=columns_radiation_csv) + columns=columns_radiation_csv) expected['Int'] = expected['Int'].astype(int) expected.index.name = 'time' expected.index.freq = None return expected + def test_read_pvgis_hourly_csv(expected_radiation_csv): out, inputs, metadata = read_pvgis_hourly(testfile_radiation_csv, map_variables=False) @@ -76,24 +80,23 @@ def test_read_pvgis_hourly_csv(expected_radiation_csv): @pytest.fixture def expected_pv_json(): expected = pd.DataFrame(index=index_pv_json, data=data_pv_json, - columns=columns_pv_json) + columns=columns_pv_json) expected['Int'] = expected['Int'].astype(int) expected.index.name = 'time' expected.index.freq = None return expected - - def test_read_pvgis_hourly_json(expected_pv_json): out, inputs, metadata = read_pvgis_hourly(testfile_pv_json, map_variables=False) assert_frame_equal(out, expected_pv_json) + @pytest.fixture def expected_pv_json_mapped(): expected = pd.DataFrame(index=index_pv_json, data=data_pv_json, - columns=columns_pv_json_mapped) + columns=columns_pv_json_mapped) expected['Int'] = expected['Int'].astype(int) expected.index.name = 'time' expected.index.freq = None From 8a3d2367411170ddcf650252a2fa08d7f860a6a2 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Wed, 16 Jun 2021 22:55:08 -0500 Subject: [PATCH 14/51] Parametrize read_pvgis_hourly tests --- pvlib/tests/iotools/test_pvgis.py | 69 +++++++++---------------------- 1 file changed, 20 insertions(+), 49 deletions(-) diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index b10e887e65..7c38b64381 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -7,7 +7,7 @@ import pytest import requests from pvlib.iotools import get_pvgis_tmy, read_pvgis_tmy -from pvlib.iotools import read_pvgis_hourly # get_pvgis_hourly, +from pvlib.iotools import read_pvgis_hourly # get_pvgis_hourly, from ..conftest import DATA_DIR, RERUNS, RERUNS_DELAY, assert_frame_equal @@ -18,16 +18,16 @@ index_radiation_csv = \ pd.date_range('20160101 00:10', freq='1h', periods=14, tz='UTC') - index_pv_json = \ pd.date_range('2013-01-01 00:55', freq='1h', periods=10, tz='UTC') columns_radiation_csv = [ 'Gb(i)', 'Gd(i)', 'Gr(i)', 'H_sun', 'T2m', 'WS10m', 'Int'] - +columns_radiation_csv_mapped = [ + 'poa_direct', 'poa_diffuse', 'poa_ground_diffuse', 'solar_elevation', + 'temp_air', 'wind_speed', 'Int'] columns_pv_json = [ 'P', 'Gb(i)', 'Gd(i)', 'Gr(i)', 'H_sun', 'T2m', 'WS10m', 'Int'] - columns_pv_json_mapped = [ 'P', 'poa_direct', 'poa_diffuse', 'poa_ground_diffuse', 'solar_elevation', 'temp_air', 'wind_speed', 'Int'] @@ -47,7 +47,6 @@ [2.11, 0.94, 0.03, 21.82, 6.79, 0.58, 1.0], [4.25, 1.88, 0.05, 21.41, 7.84, 0.4, 1.0], [0.0, 0.0, 0.0, 0.0, 7.43, 0.72, 0.0]] - data_pv_json = [ [0.0, 0.0, 0.0, 0.0, 0.0, 3.01, 1.23, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 2.22, 1.46, 0.0], @@ -61,53 +60,25 @@ [713.3, 5.18, 70.57, 7.31, 18.56, 3.66, 1.27, 0.0]] -@pytest.fixture -def expected_radiation_csv(): - expected = pd.DataFrame(index=index_radiation_csv, data=data_radiation_csv, - columns=columns_radiation_csv) - expected['Int'] = expected['Int'].astype(int) - expected.index.name = 'time' - expected.index.freq = None - return expected - - -def test_read_pvgis_hourly_csv(expected_radiation_csv): - out, inputs, metadata = read_pvgis_hourly(testfile_radiation_csv, - map_variables=False) - assert_frame_equal(out, expected_radiation_csv) - - -@pytest.fixture -def expected_pv_json(): - expected = pd.DataFrame(index=index_pv_json, data=data_pv_json, - columns=columns_pv_json) - expected['Int'] = expected['Int'].astype(int) - expected.index.name = 'time' - expected.index.freq = None - return expected - - -def test_read_pvgis_hourly_json(expected_pv_json): - out, inputs, metadata = read_pvgis_hourly(testfile_pv_json, - map_variables=False) - assert_frame_equal(out, expected_pv_json) - - -@pytest.fixture -def expected_pv_json_mapped(): - expected = pd.DataFrame(index=index_pv_json, data=data_pv_json, - columns=columns_pv_json_mapped) +@pytest.mark.parametrize('testfile,index,columns,values,map_variables,' + 'pvgis_format', [ + (testfile_radiation_csv, index_radiation_csv, columns_radiation_csv, + data_radiation_csv, False, None), + (testfile_radiation_csv, index_radiation_csv, columns_radiation_csv_mapped, + data_radiation_csv, True, 'csv'), + (testfile_pv_json, index_pv_json, columns_pv_json, + data_pv_json, False, None), + (testfile_pv_json, index_pv_json, columns_pv_json_mapped, + data_pv_json, True, 'json')]) +def test_read_pvgis_hourly(testfile, index, columns, values, map_variables, + pvgis_format): + expected = pd.DataFrame(index=index, data=values, columns=columns) expected['Int'] = expected['Int'].astype(int) expected.index.name = 'time' expected.index.freq = None - return expected - - -def test_read_pvgis_hourly_json_mapped(expected_pv_json_mapped): - out, inputs, metadata = read_pvgis_hourly(testfile_pv_json, - map_variables=True, - pvgis_format='json') - assert_frame_equal(out, expected_pv_json_mapped) + out, inputs, metadata = read_pvgis_hourly( + testfile, map_variables=map_variables, pvgis_format=pvgis_format) + assert_frame_equal(out, expected) # PVGIS TMY tests From b2e7ee39265f4c2527dc1a87b49c0c5e491cee15 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Wed, 16 Jun 2021 23:06:54 -0500 Subject: [PATCH 15/51] Add test for metadata assertion --- pvlib/tests/iotools/test_pvgis.py | 80 +++++++++++++++++++++++++++---- 1 file changed, 71 insertions(+), 9 deletions(-) diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index 7c38b64381..a1d97c5a25 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -59,19 +59,79 @@ [1586.9, 80.76, 83.95, 9.04, 13.28, 2.79, 1.36, 0.0], [713.3, 5.18, 70.57, 7.31, 18.56, 3.66, 1.27, 0.0]] - -@pytest.mark.parametrize('testfile,index,columns,values,map_variables,' - 'pvgis_format', [ +inputs_radiation_csv = {'latitude': 45.0, 'longitude': 8.0, 'elevation': 250.0, + 'radiation_database': 'PVGIS-SARAH', + 'Slope': '30 deg.', 'Azimuth': '0 deg.'} + +metadata_radiation_csv = { + 'Gb(i)': 'Beam (direct) irradiance on the inclined plane (plane of the array) (W/m2)', # noqa: F401 + 'Gd(i)': 'Diffuse irradiance on the inclined plane (plane of the array) (W/m2)', # noqa: F401 + 'Gr(i)': 'Reflected irradiance on the inclined plane (plane of the array) (W/m2)', # noqa: F401 + 'H_sun': 'Sun height (degree)', + 'T2m': '2-m air temperature (degree Celsius)', + 'WS10m': '10-m total wind speed (m/s)', + 'Int': '1 means solar radiation values are reconstructed'} + +inputs_pv_json = { + 'location': {'latitude': 45.0, 'longitude': 8.0, 'elevation': 250.0}, + 'meteo_data': {'radiation_db': 'PVGIS-CMSAF', 'meteo_db': 'ERA-Interim', + 'year_min': 2013, 'year_max': 2014, 'use_horizon': True, + 'horizon_db': None, 'horizon_data': 'DEM-calculated'}, + 'mounting_system': {'two_axis': { + 'slope': {'value': '-', 'optimal': '-'}, + 'azimuth': {'value': '-', 'optimal': '-'}}}, + 'pv_module': {'technology': 'CIS', 'peak_power': 10.0, 'system_loss': 5.0}} + +metadata_pv_json = {'inputs': {'location': {'description': 'Selected location', + 'variables': {'latitude': {'description': 'Latitude', + 'units': 'decimal degree'}, + 'longitude': {'description': 'Longitude', 'units': 'decimal degree'}, + 'elevation': {'description': 'Elevation', 'units': 'm'}}}, + 'meteo_data': {'description': 'Sources of meteorological data', + 'variables': {'radiation_db': {'description': 'Solar radiation database'}, + 'meteo_db': {'description': 'Database used for meteorological variables other than solar radiation'}, + 'year_min': {'description': 'First year of the calculations'}, + 'year_max': {'description': 'Last year of the calculations'}, + 'use_horizon': {'description': 'Include horizon shadows'}, + 'horizon_db': {'description': 'Source of horizon data'}}}, + 'mounting_system': {'description': 'Mounting system', + 'choices': 'fixed, vertical_axis, inclined_axis, two_axis', + 'fields': {'slope': {'description': 'Inclination angle from the horizontal plane', + 'units': 'degree'}, + 'azimuth': {'description': 'Orientation (azimuth) angle of the (fixed) PV system (0 = S, 90 = W, -90 = E)', + 'units': 'degree'}}}, + 'pv_module': {'description': 'PV module parameters', + 'variables': {'technology': {'description': 'PV technology'}, + 'peak_power': {'description': 'Nominal (peak) power of the PV module', + 'units': 'kW'}, + 'system_loss': {'description': 'Sum of system losses', 'units': '%'}}}}, + 'outputs': {'hourly': {'type': 'time series', + 'timestamp': 'hourly averages', + 'variables': {'P': {'description': 'PV system power', 'units': 'W'}, + 'Gb(i)': {'description': 'Beam (direct) irradiance on the inclined plane (plane of the array)', + 'units': 'W/m2'}, + 'Gd(i)': {'description': 'Diffuse irradiance on the inclined plane (plane of the array)', + 'units': 'W/m2'}, + 'Gr(i)': {'description': 'Reflected irradiance on the inclined plane (plane of the array)', + 'units': 'W/m2'}, + 'H_sun': {'description': 'Sun height', 'units': 'degree'}, + 'T2m': {'description': '2-m air temperature', 'units': 'degree Celsius'}, + 'WS10m': {'description': '10-m total wind speed', 'units': 'm/s'}, + 'Int': {'description': '1 means solar radiation values are reconstructed'}}}}} + + +@pytest.mark.parametrize('testfile,index,columns,values,metadata_exp,' + 'inputs_exp,map_variables,pvgis_format', [ (testfile_radiation_csv, index_radiation_csv, columns_radiation_csv, - data_radiation_csv, False, None), + data_radiation_csv, metadata_radiation_csv, inputs_radiation_csv, False, None), (testfile_radiation_csv, index_radiation_csv, columns_radiation_csv_mapped, - data_radiation_csv, True, 'csv'), + data_radiation_csv, metadata_radiation_csv, inputs_radiation_csv, True, 'csv'), (testfile_pv_json, index_pv_json, columns_pv_json, - data_pv_json, False, None), + data_pv_json, metadata_pv_json, inputs_pv_json, False, None), (testfile_pv_json, index_pv_json, columns_pv_json_mapped, - data_pv_json, True, 'json')]) -def test_read_pvgis_hourly(testfile, index, columns, values, map_variables, - pvgis_format): + data_pv_json, metadata_pv_json, inputs_pv_json, True, 'json')]) +def test_read_pvgis_hourly(testfile, index, columns, values, metadata_exp, + inputs_exp, map_variables, pvgis_format): expected = pd.DataFrame(index=index, data=values, columns=columns) expected['Int'] = expected['Int'].astype(int) expected.index.name = 'time' @@ -79,6 +139,8 @@ def test_read_pvgis_hourly(testfile, index, columns, values, map_variables, out, inputs, metadata = read_pvgis_hourly( testfile, map_variables=map_variables, pvgis_format=pvgis_format) assert_frame_equal(out, expected) + assert inputs == inputs_exp + # PVGIS TMY tests From a3f54cbe3dd44db0ab89fec329bfaf66c45bb297 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Wed, 16 Jun 2021 23:15:12 -0500 Subject: [PATCH 16/51] Add test documentation --- pvlib/tests/iotools/test_pvgis.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index a1d97c5a25..208627ae37 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -11,6 +11,7 @@ from ..conftest import DATA_DIR, RERUNS, RERUNS_DELAY, assert_frame_equal +# PVGIS Hourly tests testfile_radiation_csv = DATA_DIR / \ 'pvgis_hourly_Timeseries_45.000_8.000_SA_30deg_0deg_2016_2016.csv' testfile_pv_json = DATA_DIR / \ @@ -120,27 +121,32 @@ 'Int': {'description': '1 means solar radiation values are reconstructed'}}}}} +# Test read_pvgis_hourly function using two different files with different +# input arguments (to test variable mapping and pvgis_format) @pytest.mark.parametrize('testfile,index,columns,values,metadata_exp,' 'inputs_exp,map_variables,pvgis_format', [ (testfile_radiation_csv, index_radiation_csv, columns_radiation_csv, - data_radiation_csv, metadata_radiation_csv, inputs_radiation_csv, False, None), + data_radiation_csv, metadata_radiation_csv, inputs_radiation_csv, False, None), # noqa: F401 (testfile_radiation_csv, index_radiation_csv, columns_radiation_csv_mapped, - data_radiation_csv, metadata_radiation_csv, inputs_radiation_csv, True, 'csv'), + data_radiation_csv, metadata_radiation_csv, inputs_radiation_csv, True, 'csv'), # noqa: F401 (testfile_pv_json, index_pv_json, columns_pv_json, data_pv_json, metadata_pv_json, inputs_pv_json, False, None), (testfile_pv_json, index_pv_json, columns_pv_json_mapped, data_pv_json, metadata_pv_json, inputs_pv_json, True, 'json')]) def test_read_pvgis_hourly(testfile, index, columns, values, metadata_exp, inputs_exp, map_variables, pvgis_format): + # Create expected dataframe expected = pd.DataFrame(index=index, data=values, columns=columns) expected['Int'] = expected['Int'].astype(int) expected.index.name = 'time' expected.index.freq = None + # Read data from file out, inputs, metadata = read_pvgis_hourly( testfile, map_variables=map_variables, pvgis_format=pvgis_format) + # Assert whether dataframe, metadata, and inputs are as expected assert_frame_equal(out, expected) assert inputs == inputs_exp - + assert metadata == metadata_exp # PVGIS TMY tests From e65bd4cbfba44be3df6e2c7dd7d2bf50100737b5 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Wed, 16 Jun 2021 23:22:00 -0500 Subject: [PATCH 17/51] Fixed minor documentation issues --- pvlib/tests/iotools/test_pvgis.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index 208627ae37..8d30c73b3e 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -65,9 +65,9 @@ 'Slope': '30 deg.', 'Azimuth': '0 deg.'} metadata_radiation_csv = { - 'Gb(i)': 'Beam (direct) irradiance on the inclined plane (plane of the array) (W/m2)', # noqa: F401 - 'Gd(i)': 'Diffuse irradiance on the inclined plane (plane of the array) (W/m2)', # noqa: F401 - 'Gr(i)': 'Reflected irradiance on the inclined plane (plane of the array) (W/m2)', # noqa: F401 + 'Gb(i)': 'Beam (direct) irradiance on the inclined plane (plane of the array) (W/m2)', # noqa: F501 + 'Gd(i)': 'Diffuse irradiance on the inclined plane (plane of the array) (W/m2)', # noqa: F501 + 'Gr(i)': 'Reflected irradiance on the inclined plane (plane of the array) (W/m2)', # noqa: F501 'H_sun': 'Sun height (degree)', 'T2m': '2-m air temperature (degree Celsius)', 'WS10m': '10-m total wind speed (m/s)', @@ -126,9 +126,9 @@ @pytest.mark.parametrize('testfile,index,columns,values,metadata_exp,' 'inputs_exp,map_variables,pvgis_format', [ (testfile_radiation_csv, index_radiation_csv, columns_radiation_csv, - data_radiation_csv, metadata_radiation_csv, inputs_radiation_csv, False, None), # noqa: F401 + data_radiation_csv, metadata_radiation_csv, inputs_radiation_csv, False, None), # noqa: F501 (testfile_radiation_csv, index_radiation_csv, columns_radiation_csv_mapped, - data_radiation_csv, metadata_radiation_csv, inputs_radiation_csv, True, 'csv'), # noqa: F401 + data_radiation_csv, metadata_radiation_csv, inputs_radiation_csv, True, 'csv'), # noqa: E501 (testfile_pv_json, index_pv_json, columns_pv_json, data_pv_json, metadata_pv_json, inputs_pv_json, False, None), (testfile_pv_json, index_pv_json, columns_pv_json_mapped, From 1c03e8505d168f013dd0c68add2c1dded456fd4f Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Thu, 17 Jun 2021 12:34:08 -0500 Subject: [PATCH 18/51] Remove basic parser and add bad extension test --- pvlib/iotools/pvgis.py | 66 ++++++++++++------------------- pvlib/tests/iotools/test_pvgis.py | 11 ++++++ 2 files changed, 37 insertions(+), 40 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 4852c0b622..cbb9a206c6 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -47,7 +47,7 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, start=None, end=None, pvcalculation=False, peakpower=None, pvtechchoice='crystSi', mountingplace='free', loss=None, trackingtype=0, - optimalinclination=False, optimalangles=False, + optimal_surface_tilt=False, optimalangles=False, components=True, url=URL, map_variables=True, timeout=30): """ Get hourly solar irradiation and modeled PV power output from PVGIS [1]_. @@ -64,8 +64,8 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, Orientation (azimuth angle) of the (fixed) plane. 0=south, 90=west, -90: east. Not relevant for tracking systems. outputformat: str, default: 'json' - Must be in ``['json', 'csv', 'basic']``. See PVGIS hourly data - documentation [2]_ for more info. Note basic does not include a header. + Must be in ``['json', 'csv']``. See PVGIS hourly data + documentation [2]_ for more info. usehorizon: bool, default: True Include effects of horizon userhorizon: list of float, default: None @@ -96,7 +96,7 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, north-south, 2=two-axis tracking, 3=vertical axis tracking, 4=single horizontal axis aligned east-west, 5=single inclined axis aligned north-south. - optimalinclination: bool, default: False + optimal_surface_tilt: bool, default: False Calculate the optimum tilt angle. Not relevant for 2-axis tracking optimalangles: bool, default: False Calculate the optimum tilt and azimuth angles. Not relevant for 2-axis @@ -118,9 +118,9 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, data : pandas.DataFrame Time-series of hourly data, see Notes for fields inputs : dict - Dictionary of the request input parameters, ``None`` for basic + Dictionary of the request input parameters metadata : list or dict - metadata, ``None`` for basic + metadata Notes ----- @@ -193,7 +193,7 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, params['peakpower'] = peakpower if loss is not None: params['loss'] = loss - if optimalinclination: + if optimal_surface_tilt: params['optimalinclination'] = 1 if optimalangles: params['optimalangles'] = 1 @@ -220,9 +220,6 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, elif outputformat == 'csv': with io.StringIO(res.content.decode('utf-8')) as src: return _parse_pvgis_hourly_csv(src, map_variables=map_variables) - elif outputformat == 'basic': - with io.BytesIO(res.content) as src: - return _parse_pvgis_hourly_basic(src) else: # this line is never reached because if outputformat is not valid then # the response is HTTP/1.1 400 BAD REQUEST which is handled earlier @@ -242,12 +239,6 @@ def _parse_pvgis_hourly_json(src, map_variables): return data, inputs, metadata -def _parse_pvgis_hourly_basic(src, map_variables): - # Hourly data with outputformat='basic' does not include header or metadata - data = pd.read_csv(src, header=None, skiprows=2) - return data, None, None - - def _parse_pvgis_hourly_csv(src, map_variables): # The first 4 rows are latitude, longitude, elevation, radiation database inputs = {} @@ -311,46 +302,44 @@ def read_pvgis_hourly(filename, map_variables=True, pvgis_format=None): Format of PVGIS file or buffer. Equivalent to the ``outputformat`` parameter in the PVGIS API. If `filename` is a file and `pvgis_format` is ``None`` then the file extension will be used to - determine the PVGIS format to parse. For PVGIS files from the API with - ``outputformat='basic'``, please set `pvgis_format` to ``'basic'``. If - `filename` is a buffer, then `pvgis_format` is required and must be in - ``['csv', 'json', 'basic']``. + determine the PVGIS format to parse. If `filename` is a buffer, then + `pvgis_format` is required and must be in ``['csv', 'json']``. Returns ------- data : pandas.DataFrame the weather data inputs : dict - the inputs, ``None`` for basic + the inputs metadata : list or dict - metadata, ``None`` for basic + metadata Raises ------ ValueError if `pvgis_format` is ``None`` and the file extension is neither ``.csv`` nor ``.json`` or if `pvgis_format` is provided as - input but isn't in ``['csv', 'json', 'basic']`` + input but isn't in ``['csv', 'json']`` TypeError if `pvgis_format` is ``None`` and `filename` is a buffer See also -------- - get_pvgis_hourly + get_pvgis_hourly, get_pvgis_tmy """ # get the PVGIS outputformat if pvgis_format is None: # get the file extension from suffix, but remove the dot and make sure # it's lower case to compare with csv, or json + # NOTE: basic format is not supported for PVGIS Hourly as the data + # format does not include a header # NOTE: raises TypeError if filename is a buffer outputformat = Path(filename).suffix[1:].lower() else: outputformat = pvgis_format - # parse the pvgis file based on the output format, either 'json', 'csv', - # or 'basic' - - # NOTE: json, csv, and basic output formats have parsers defined as private + # parse the pvgis file based on the output format, either 'json' or 'csv' + # NOTE: json and csv output formats have parsers defined as private # functions in this module # JSON: use Python built-in json module to convert file contents to a @@ -364,24 +353,21 @@ def read_pvgis_hourly(filename, map_variables=True, pvgis_format=None): src = json.load(fbuf) return _parse_pvgis_hourly_json(src, map_variables=map_variables) - # CSV or basic: use the correct parser from this module - # eg: _parse_pvgis_hourly_csv() or _parse_pvgis_hourly_basic() - if outputformat in ['csv', 'basic']: - # get the correct parser function for this output format from globals() - pvgis_parser = globals()['_parse_pvgis_hourly_{:s}'.format(outputformat)] # noqa - # NOTE: pvgis_parse() is a pvgis parser function from this module, - # either _parse_pvgis_hourly_csv() or _parse_pvgist_hourly_basic() + # CSV: use _parse_pvgis_hourly_csv() + if outputformat == 'csv': try: - pvgis_data = pvgis_parser(filename, map_variables=map_variables) + pvgis_data = _parse_pvgis_hourly_csv( + filename,map_variables=map_variables) except AttributeError: # str/path has no .read() attribute with open(str(filename), 'r') as fbuf: - pvgis_data = pvgis_parser(fbuf, map_variables=map_variables) + pvgis_data = _parse_pvgis_hourly_csv( + fbuf, map_variables=map_variables) return pvgis_data - # raise exception if pvgis format isn't in ['csv', 'basic', 'json'] + # raise exception if pvgis format isn't in ['csv', 'json'] err_msg = ( - "pvgis format '{:s}' was unknown, must be either 'json', 'csv', or" - "'basic'").format(outputformat) + "pvgis format '{:s}' was unknown, must be either 'json' or 'csv'")\ + .format(outputformat) raise ValueError(err_msg) diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index 8d30c73b3e..7b7a0f6b37 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -149,6 +149,17 @@ def test_read_pvgis_hourly(testfile, index, columns, values, metadata_exp, assert metadata == metadata_exp +def test_read_pvgis_hourly_bad_extension(): + # Test if ValueError is raised if file extension cannot be recognized and + # pvgis_format is not specified + with pytest.raises(ValueError, match="pvgis format 'txt' was unknown"): + read_pvgis_hourly('testfile.txt') + # Test if ValueError is raised if an unkonwn pvgis_format is specified + with pytest.raises(ValueError, match="pvgis format 'txt' was unknown"): + read_pvgis_hourly(testfile_pv_json, pvgis_format='txt') + + + # PVGIS TMY tests @pytest.fixture def expected(): From 7563d285c3240cde05c713418cb3eecfefe9af1e Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Thu, 17 Jun 2021 13:09:32 -0500 Subject: [PATCH 19/51] Add TypeError test and minor doc fixes --- pvlib/iotools/pvgis.py | 24 ++++++++--------- pvlib/tests/iotools/test_pvgis.py | 43 +++++++++++++++++++------------ 2 files changed, 38 insertions(+), 29 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index cbb9a206c6..53fa445544 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -81,7 +81,7 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, end: int, default: None Last year of the radiation time series. Defaults to last year avaiable. pvcalculation: bool, default: False - Also return estimate of hourly production. + Return estimate of hourly production. peakpower: float, default: None Nominal power of PV system in kW. Required if pvcalculation=True. pvtechchoice: {'crystSi', 'CIS', 'CdTe', 'Unknown'}, default: 'crystSi' @@ -120,7 +120,7 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, inputs : dict Dictionary of the request input parameters metadata : list or dict - metadata + Dictionary containing metadata Notes ----- @@ -131,20 +131,20 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, ======================= ====== ========================================== **Mapped field names are returned when the map_variables argument is True** --------------------------------------------------------------------------- - P* float PV system power (W) - G(i)** float Global irradiance on inclined plane (W/m^2) # noqa - Gb(i)** float Beam (direct) irradiance on inclined plane (W/m^2) # noqa - Gd(i)** float Diffuse irradiance on inclined plane (W/m^2) # noqa - Gr(i)** float Reflected irradiance on inclined plane (W/m^2) # noqa + P\* float PV system power (W) + G(i)\** float Global irradiance on inclined plane (W/m^2) # noqa + Gb(i)\* float Beam (direct) irradiance on inclined plane (W/m^2) # noqa + Gd(i)\** float Diffuse irradiance on inclined plane (W/m^2) # noqa + Gr(i)\** float Reflected irradiance on inclined plane (W/m^2) # noqa H_sun, solar_elevation float Sun height/elevation (degrees) - T2m, temp_air float Air temperature at 2 (degrees Celsius) + T2m, temp_air float Air temperature at 2 m (degrees Celsius) WS10m, wind_speed float Wind speed at 10 m (m/s) Int int Solar radiation reconstructed (1/0) ======================= ====== ========================================== - *P (PV system power) is only returned when pvcalculation=True. + \*P (PV system power) is only returned when pvcalculation=True. - **Gb(i), Gd(i), and Gr(i) are returned when components=True, whereas + v**Gb(i), Gd(i), and Gr(i) are returned when components=True, whereas otherwise the sum of the three components, G(i), is returned. Raises @@ -165,7 +165,7 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, `_ .. [3] `PVGIS Non-interactive service ` - .. [3] `PVGIS horizon profile tool + .. [4] `PVGIS horizon profile tool `_ """ # use requests to format the query string by passing params dictionary @@ -357,7 +357,7 @@ def read_pvgis_hourly(filename, map_variables=True, pvgis_format=None): if outputformat == 'csv': try: pvgis_data = _parse_pvgis_hourly_csv( - filename,map_variables=map_variables) + filename, map_variables=map_variables) except AttributeError: # str/path has no .read() attribute with open(str(filename), 'r') as fbuf: pvgis_data = _parse_pvgis_hourly_csv( diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index 7b7a0f6b37..b9141be92e 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -4,6 +4,7 @@ import json import numpy as np import pandas as pd +import io import pytest import requests from pvlib.iotools import get_pvgis_tmy, read_pvgis_tmy @@ -90,16 +91,16 @@ 'elevation': {'description': 'Elevation', 'units': 'm'}}}, 'meteo_data': {'description': 'Sources of meteorological data', 'variables': {'radiation_db': {'description': 'Solar radiation database'}, - 'meteo_db': {'description': 'Database used for meteorological variables other than solar radiation'}, + 'meteo_db': {'description': 'Database used for meteorological variables other than solar radiation'}, # noqa: F501 'year_min': {'description': 'First year of the calculations'}, 'year_max': {'description': 'Last year of the calculations'}, 'use_horizon': {'description': 'Include horizon shadows'}, 'horizon_db': {'description': 'Source of horizon data'}}}, 'mounting_system': {'description': 'Mounting system', 'choices': 'fixed, vertical_axis, inclined_axis, two_axis', - 'fields': {'slope': {'description': 'Inclination angle from the horizontal plane', + 'fields': {'slope': {'description': 'Inclination angle from the horizontal plane', # noqa: F501 'units': 'degree'}, - 'azimuth': {'description': 'Orientation (azimuth) angle of the (fixed) PV system (0 = S, 90 = W, -90 = E)', + 'azimuth': {'description': 'Orientation (azimuth) angle of the (fixed) PV system (0 = S, 90 = W, -90 = E)', # noqa: F501 'units': 'degree'}}}, 'pv_module': {'description': 'PV module parameters', 'variables': {'technology': {'description': 'PV technology'}, @@ -109,30 +110,36 @@ 'outputs': {'hourly': {'type': 'time series', 'timestamp': 'hourly averages', 'variables': {'P': {'description': 'PV system power', 'units': 'W'}, - 'Gb(i)': {'description': 'Beam (direct) irradiance on the inclined plane (plane of the array)', + 'Gb(i)': {'description': 'Beam (direct) irradiance on the inclined plane (plane of the array)', # noqa: F501 'units': 'W/m2'}, - 'Gd(i)': {'description': 'Diffuse irradiance on the inclined plane (plane of the array)', + 'Gd(i)': {'description': 'Diffuse irradiance on the inclined plane (plane of the array)', # noqa: F501 'units': 'W/m2'}, - 'Gr(i)': {'description': 'Reflected irradiance on the inclined plane (plane of the array)', + 'Gr(i)': {'description': 'Reflected irradiance on the inclined plane (plane of the array)', # noqa: F501 'units': 'W/m2'}, 'H_sun': {'description': 'Sun height', 'units': 'degree'}, 'T2m': {'description': '2-m air temperature', 'units': 'degree Celsius'}, 'WS10m': {'description': '10-m total wind speed', 'units': 'm/s'}, - 'Int': {'description': '1 means solar radiation values are reconstructed'}}}}} + 'Int': {'description': '1 means solar radiation values are reconstructed'}}}}} # noqa: F501 # Test read_pvgis_hourly function using two different files with different # input arguments (to test variable mapping and pvgis_format) @pytest.mark.parametrize('testfile,index,columns,values,metadata_exp,' 'inputs_exp,map_variables,pvgis_format', [ - (testfile_radiation_csv, index_radiation_csv, columns_radiation_csv, - data_radiation_csv, metadata_radiation_csv, inputs_radiation_csv, False, None), # noqa: F501 - (testfile_radiation_csv, index_radiation_csv, columns_radiation_csv_mapped, - data_radiation_csv, metadata_radiation_csv, inputs_radiation_csv, True, 'csv'), # noqa: E501 - (testfile_pv_json, index_pv_json, columns_pv_json, - data_pv_json, metadata_pv_json, inputs_pv_json, False, None), - (testfile_pv_json, index_pv_json, columns_pv_json_mapped, - data_pv_json, metadata_pv_json, inputs_pv_json, True, 'json')]) + (testfile_radiation_csv, index_radiation_csv, + columns_radiation_csv, data_radiation_csv, + metadata_radiation_csv, inputs_radiation_csv, + False, None), + (testfile_radiation_csv, index_radiation_csv, + columns_radiation_csv_mapped, data_radiation_csv, + metadata_radiation_csv, inputs_radiation_csv, + True, 'csv'), + (testfile_pv_json, index_pv_json, columns_pv_json, + data_pv_json, metadata_pv_json, inputs_pv_json, + False, None), + (testfile_pv_json, index_pv_json, + columns_pv_json_mapped, data_pv_json, + metadata_pv_json, inputs_pv_json, True, 'json')]) def test_read_pvgis_hourly(testfile, index, columns, values, metadata_exp, inputs_exp, map_variables, pvgis_format): # Create expected dataframe @@ -153,11 +160,13 @@ def test_read_pvgis_hourly_bad_extension(): # Test if ValueError is raised if file extension cannot be recognized and # pvgis_format is not specified with pytest.raises(ValueError, match="pvgis format 'txt' was unknown"): - read_pvgis_hourly('testfile.txt') + read_pvgis_hourly('filename.txt') # Test if ValueError is raised if an unkonwn pvgis_format is specified with pytest.raises(ValueError, match="pvgis format 'txt' was unknown"): read_pvgis_hourly(testfile_pv_json, pvgis_format='txt') - + # Test if TypeError is raised if input is a buffer and pvgis_format=None + with pytest.raises(TypeError, match="expected str, bytes or os.PathLike"): + read_pvgis_hourly(io.StringIO()) # PVGIS TMY tests From de6ebc917058e8afd2dddee4e67175225cb3e780 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Thu, 17 Jun 2021 13:47:43 -0500 Subject: [PATCH 20/51] Fix stickler --- pvlib/tests/iotools/test_pvgis.py | 81 ++++++++++++++++--------------- 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index b9141be92e..33b593eaa4 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -66,9 +66,9 @@ 'Slope': '30 deg.', 'Azimuth': '0 deg.'} metadata_radiation_csv = { - 'Gb(i)': 'Beam (direct) irradiance on the inclined plane (plane of the array) (W/m2)', # noqa: F501 - 'Gd(i)': 'Diffuse irradiance on the inclined plane (plane of the array) (W/m2)', # noqa: F501 - 'Gr(i)': 'Reflected irradiance on the inclined plane (plane of the array) (W/m2)', # noqa: F501 + 'Gb(i)': 'Beam (direct) irradiance on the inclined plane (plane of the array) (W/m2)', # noqa: E501 + 'Gd(i)': 'Diffuse irradiance on the inclined plane (plane of the array) (W/m2)', # noqa: E501 + 'Gr(i)': 'Reflected irradiance on the inclined plane (plane of the array) (W/m2)', # noqa: E501 'H_sun': 'Sun height (degree)', 'T2m': '2-m air temperature (degree Celsius)', 'WS10m': '10-m total wind speed (m/s)', @@ -84,42 +84,45 @@ 'azimuth': {'value': '-', 'optimal': '-'}}}, 'pv_module': {'technology': 'CIS', 'peak_power': 10.0, 'system_loss': 5.0}} -metadata_pv_json = {'inputs': {'location': {'description': 'Selected location', - 'variables': {'latitude': {'description': 'Latitude', - 'units': 'decimal degree'}, - 'longitude': {'description': 'Longitude', 'units': 'decimal degree'}, - 'elevation': {'description': 'Elevation', 'units': 'm'}}}, - 'meteo_data': {'description': 'Sources of meteorological data', - 'variables': {'radiation_db': {'description': 'Solar radiation database'}, - 'meteo_db': {'description': 'Database used for meteorological variables other than solar radiation'}, # noqa: F501 - 'year_min': {'description': 'First year of the calculations'}, - 'year_max': {'description': 'Last year of the calculations'}, - 'use_horizon': {'description': 'Include horizon shadows'}, - 'horizon_db': {'description': 'Source of horizon data'}}}, - 'mounting_system': {'description': 'Mounting system', - 'choices': 'fixed, vertical_axis, inclined_axis, two_axis', - 'fields': {'slope': {'description': 'Inclination angle from the horizontal plane', # noqa: F501 - 'units': 'degree'}, - 'azimuth': {'description': 'Orientation (azimuth) angle of the (fixed) PV system (0 = S, 90 = W, -90 = E)', # noqa: F501 - 'units': 'degree'}}}, - 'pv_module': {'description': 'PV module parameters', - 'variables': {'technology': {'description': 'PV technology'}, - 'peak_power': {'description': 'Nominal (peak) power of the PV module', - 'units': 'kW'}, - 'system_loss': {'description': 'Sum of system losses', 'units': '%'}}}}, - 'outputs': {'hourly': {'type': 'time series', - 'timestamp': 'hourly averages', - 'variables': {'P': {'description': 'PV system power', 'units': 'W'}, - 'Gb(i)': {'description': 'Beam (direct) irradiance on the inclined plane (plane of the array)', # noqa: F501 - 'units': 'W/m2'}, - 'Gd(i)': {'description': 'Diffuse irradiance on the inclined plane (plane of the array)', # noqa: F501 - 'units': 'W/m2'}, - 'Gr(i)': {'description': 'Reflected irradiance on the inclined plane (plane of the array)', # noqa: F501 - 'units': 'W/m2'}, - 'H_sun': {'description': 'Sun height', 'units': 'degree'}, - 'T2m': {'description': '2-m air temperature', 'units': 'degree Celsius'}, - 'WS10m': {'description': '10-m total wind speed', 'units': 'm/s'}, - 'Int': {'description': '1 means solar radiation values are reconstructed'}}}}} # noqa: F501 +metadata_pv_json = { + 'inputs': { + 'location': {'description': 'Selected location', 'variables': { + 'latitude': {'description': 'Latitude', 'units': 'decimal degree'}, + 'longitude': {'description': 'Longitude', 'units': 'decimal degree'}, # noqa: E501 + 'elevation': {'description': 'Elevation', 'units': 'm'}}}, + 'meteo_data': { + 'description': 'Sources of meteorological data', + 'variables': { + 'radiation_db': {'description': 'Solar radiation database'}, # noqa: E501 + 'meteo_db': {'description': 'Database used for meteorological variables other than solar radiation'}, # noqa: E501 + 'year_min': {'description': 'First year of the calculations'}, # noqa: E501 + 'year_max': {'description': 'Last year of the calculations'}, # noqa: E501 + 'use_horizon': {'description': 'Include horizon shadows'}, + 'horizon_db': {'description': 'Source of horizon data'}}}, + 'mounting_system': { + 'description': 'Mounting system', + 'choices': 'fixed, vertical_axis, inclined_axis, two_axis', + 'fields': { + 'slope': {'description': 'Inclination angle from the horizontal plane', 'units': 'degree'}, # noqa: E501 + 'azimuth': {'description': 'Orientation (azimuth) angle of the (fixed) PV system (0 = S, 90 = W, -90 = E)', 'units': 'degree'}}}, # noqa: E5011 + 'pv_module': { + 'description': 'PV module parameters', + 'variables': { + 'technology': {'description': 'PV technology'}, + 'peak_power': {'description': 'Nominal (peak) power of the PV module', 'units': 'kW'}, # noqa: E501 + 'system_loss': {'description': 'Sum of system losses', 'units': '%'}}}}, # noqa: E501 + 'outputs': { + 'hourly': { + 'type': 'time series', 'timestamp': 'hourly averages', + 'variables': { + 'P': {'description': 'PV system power', 'units': 'W'}, + 'Gb(i)': {'description': 'Beam (direct) irradiance on the inclined plane (plane of the array)', 'units': 'W/m2'}, # noqa: E501 + 'Gd(i)': {'description': 'Diffuse irradiance on the inclined plane (plane of the array)', 'units': 'W/m2'}, # noqa: E501 + 'Gr(i)': {'description': 'Reflected irradiance on the inclined plane (plane of the array)', 'units': 'W/m2'}, # noqa: E501 + 'H_sun': {'description': 'Sun height', 'units': 'degree'}, + 'T2m': {'description': '2-m air temperature', 'units': 'degree Celsius'}, # noqa: E501 + 'WS10m': {'description': '10-m total wind speed', 'units': 'm/s'}, # noqa: E501 + 'Int': {'description': '1 means solar radiation values are reconstructed'}}}}} # noqa: E501 # Test read_pvgis_hourly function using two different files with different From 8095d6165a0b5b93e64e8a030a63ea426115dae4 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Thu, 17 Jun 2021 13:52:51 -0500 Subject: [PATCH 21/51] Fix stickler again --- pvlib/tests/iotools/test_pvgis.py | 42 +++++++++++++++---------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index 33b593eaa4..5b1e14a646 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -90,27 +90,27 @@ 'latitude': {'description': 'Latitude', 'units': 'decimal degree'}, 'longitude': {'description': 'Longitude', 'units': 'decimal degree'}, # noqa: E501 'elevation': {'description': 'Elevation', 'units': 'm'}}}, - 'meteo_data': { - 'description': 'Sources of meteorological data', - 'variables': { - 'radiation_db': {'description': 'Solar radiation database'}, # noqa: E501 - 'meteo_db': {'description': 'Database used for meteorological variables other than solar radiation'}, # noqa: E501 - 'year_min': {'description': 'First year of the calculations'}, # noqa: E501 - 'year_max': {'description': 'Last year of the calculations'}, # noqa: E501 - 'use_horizon': {'description': 'Include horizon shadows'}, - 'horizon_db': {'description': 'Source of horizon data'}}}, - 'mounting_system': { - 'description': 'Mounting system', - 'choices': 'fixed, vertical_axis, inclined_axis, two_axis', - 'fields': { - 'slope': {'description': 'Inclination angle from the horizontal plane', 'units': 'degree'}, # noqa: E501 - 'azimuth': {'description': 'Orientation (azimuth) angle of the (fixed) PV system (0 = S, 90 = W, -90 = E)', 'units': 'degree'}}}, # noqa: E5011 - 'pv_module': { - 'description': 'PV module parameters', - 'variables': { - 'technology': {'description': 'PV technology'}, - 'peak_power': {'description': 'Nominal (peak) power of the PV module', 'units': 'kW'}, # noqa: E501 - 'system_loss': {'description': 'Sum of system losses', 'units': '%'}}}}, # noqa: E501 + 'meteo_data': { + 'description': 'Sources of meteorological data', + 'variables': { + 'radiation_db': {'description': 'Solar radiation database'}, + 'meteo_db': {'description': 'Database used for meteorological variables other than solar radiation'}, # noqa: E501 + 'year_min': {'description': 'First year of the calculations'}, + 'year_max': {'description': 'Last year of the calculations'}, + 'use_horizon': {'description': 'Include horizon shadows'}, + 'horizon_db': {'description': 'Source of horizon data'}}}, + 'mounting_system': { + 'description': 'Mounting system', + 'choices': 'fixed, vertical_axis, inclined_axis, two_axis', + 'fields': { + 'slope': {'description': 'Inclination angle from the horizontal plane', 'units': 'degree'}, # noqa: E501 + 'azimuth': {'description': 'Orientation (azimuth) angle of the (fixed) PV system (0 = S, 90 = W, -90 = E)', 'units': 'degree'}}}, # noqa: E5011 + 'pv_module': { + 'description': 'PV module parameters', + 'variables': { + 'technology': {'description': 'PV technology'}, + 'peak_power': {'description': 'Nominal (peak) power of the PV module', 'units': 'kW'}, # noqa: E501 + 'system_loss': {'description': 'Sum of system losses', 'units': '%'}}}}, # noqa: E501 'outputs': { 'hourly': { 'type': 'time series', 'timestamp': 'hourly averages', From 9b1ae98edaf19e0c29f21d004d3129e71a420837 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Thu, 17 Jun 2021 16:50:03 -0500 Subject: [PATCH 22/51] Add tests for get_pvgis_hourly --- pvlib/iotools/pvgis.py | 5 ++- pvlib/tests/iotools/test_pvgis.py | 69 ++++++++++++++++++++++++++++++- 2 files changed, 70 insertions(+), 4 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 53fa445544..448c8ecfba 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -49,8 +49,9 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, mountingplace='free', loss=None, trackingtype=0, optimal_surface_tilt=False, optimalangles=False, components=True, url=URL, map_variables=True, timeout=30): - """ - Get hourly solar irradiation and modeled PV power output from PVGIS [1]_. + """Get hourly solar irradiation and modeled PV power output from PVGIS. + + PVGIS is avaiable at [1]_. Parameters ---------- diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index 5b1e14a646..582df91a84 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -8,11 +8,13 @@ import pytest import requests from pvlib.iotools import get_pvgis_tmy, read_pvgis_tmy -from pvlib.iotools import read_pvgis_hourly # get_pvgis_hourly, +from pvlib.iotools import get_pvgis_hourly, read_pvgis_hourly from ..conftest import DATA_DIR, RERUNS, RERUNS_DELAY, assert_frame_equal # PVGIS Hourly tests +# The test files are actual files from PVGIS where the data section have been +# reduced to only a few lines testfile_radiation_csv = DATA_DIR / \ 'pvgis_hourly_Timeseries_45.000_8.000_SA_30deg_0deg_2016_2016.csv' testfile_pv_json = DATA_DIR / \ @@ -104,7 +106,7 @@ 'choices': 'fixed, vertical_axis, inclined_axis, two_axis', 'fields': { 'slope': {'description': 'Inclination angle from the horizontal plane', 'units': 'degree'}, # noqa: E501 - 'azimuth': {'description': 'Orientation (azimuth) angle of the (fixed) PV system (0 = S, 90 = W, -90 = E)', 'units': 'degree'}}}, # noqa: E5011 + 'azimuth': {'description': 'Orientation (azimuth) angle of the (fixed) PV system (0 = S, 90 = W, -90 = E)', 'units': 'degree'}}}, # noqa: E501 'pv_module': { 'description': 'PV module parameters', 'variables': { @@ -172,6 +174,69 @@ def test_read_pvgis_hourly_bad_extension(): read_pvgis_hourly(io.StringIO()) +args_radiation_csv = { + 'surface_tilt': 30, 'surface_azimuth': 0, 'outputformat': 'csv', + 'usehorizon': True, 'userhorizon': None, 'raddatabase': 'PVGIS-SARAH', + 'start': 2016, 'end': 2016, 'pvcalculation': False, 'components': True} + +url_hourly_radiation_csv = 'https://re.jrc.ec.europa.eu/api/seriescalc?lat=45&lon=8&outputformat=csv&angle=30&aspect=0&pvtechchoice=crystSi&mountingplace=free&trackingtype=0&components=1&raddatabase=PVGIS-SARAH&startyear=2016&endyear=2016' # noqa: E501 + +args_pv_json = { + 'surface_tilt': 30, 'surface_azimuth': 0, 'outputformat': 'json', + 'usehorizon': True, 'userhorizon': None, 'raddatabase': 'PVGIS-CMSAF', + 'start': 2013, 'end': 2014, 'pvcalculation': True, 'peakpower': 10, + 'pvtechchoice': 'CIS', 'loss': 5, 'trackingtype': 2, 'optimalangles': True, + 'components': True} + +url_pv_json = 'https://re.jrc.ec.europa.eu/api/seriescalc?lat=45&lon=8&outputformat=json&angle=30&aspect=0&pvtechchoice=CIS&mountingplace=free&trackingtype=2&components=1&raddatabase=PVGIS-CMSAF&startyear=2013&endyear=2014&pvcalculation=1&peakpower=10&loss=5&optimalangles=1' # noqa: E501 + + +@pytest.mark.parametrize('testfile,index,columns,values,args,map_variables,' + 'url_test', [ + (testfile_radiation_csv, index_radiation_csv, + columns_radiation_csv, data_radiation_csv, + args_radiation_csv, False, + url_hourly_radiation_csv), + (testfile_radiation_csv, index_radiation_csv, + columns_radiation_csv_mapped, data_radiation_csv, + args_radiation_csv, True, + url_hourly_radiation_csv), + (testfile_pv_json, index_pv_json, columns_pv_json, + data_pv_json, args_pv_json, False, url_pv_json), + (testfile_pv_json, index_pv_json, + columns_pv_json_mapped, data_pv_json, + args_pv_json, True, url_pv_json)]) +def test_get_pvgis_hourly(requests_mock, testfile, index, columns, values, + args, map_variables, url_test): + """Test that get_pvgis_hourly generates the correct URI request and that + _parse_pvgis_hourly_json and _parse_pvgis_hourly_csv is called correctly""" + # Open local test file containing McClear mothly data + with open(testfile, 'r') as test_file: + mock_response = test_file.read() + # Specify the full URI of a specific example, this ensures that all of the + # inputs are passing on correctly + + requests_mock.get(url_test, text=mock_response) + + # Make API call - an error is raised if requested URI does not match + out, inputs, metadata = get_pvgis_hourly( + latitude=45, longitude=8, map_variables=map_variables, **args) + # Create expected dataframe + expected = pd.DataFrame(index=index, data=values, columns=columns) + expected['Int'] = expected['Int'].astype(int) + expected.index.name = 'time' + expected.index.freq = None + # Compare out and expected dataframes + assert_frame_equal(out, expected) + + +def test_get_pvgis_hourly_bad_status_code(requests_mock): + # Test if a HTTPError is raised if a bad request is returned + requests_mock.get(url_pv_json, status_code=400) # text=mock_response) + with pytest.raises(requests.HTTPError): + get_pvgis_hourly(latitude=45, longitude=8, **args_pv_json) + + # PVGIS TMY tests @pytest.fixture def expected(): From 52fa8d223ef9050d02e52ce4f34f1bee0a9fb016 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Thu, 17 Jun 2021 17:20:44 -0500 Subject: [PATCH 23/51] Add tests for HTTPError message with json --- pvlib/iotools/pvgis.py | 6 +++--- pvlib/tests/iotools/test_pvgis.py | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 448c8ecfba..641985258a 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -56,9 +56,9 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, Parameters ---------- latitude: float - Latitude in degrees north + in decimal degrees, between -90 and 90, north is positive (ISO 19115) longitude: float - Longitude in degrees east + in decimal degrees, between -180 and 180, east is positive (ISO 19115) surface_tilt: float, default: 0 Tilt angle from horizontal plane. Not relevant for 2-axis tracking. surface_azimuth: float, default: 0 @@ -82,7 +82,7 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, end: int, default: None Last year of the radiation time series. Defaults to last year avaiable. pvcalculation: bool, default: False - Return estimate of hourly production. + Return estimate of hourly PV production. peakpower: float, default: None Nominal power of PV system in kW. Required if pvcalculation=True. pvtechchoice: {'crystSi', 'CIS', 'CdTe', 'Unknown'}, default: 'crystSi' diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index 582df91a84..ea86bc291a 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -232,7 +232,12 @@ def test_get_pvgis_hourly(requests_mock, testfile, index, columns, values, def test_get_pvgis_hourly_bad_status_code(requests_mock): # Test if a HTTPError is raised if a bad request is returned - requests_mock.get(url_pv_json, status_code=400) # text=mock_response) + requests_mock.get(url_pv_json, status_code=400) + with pytest.raises(requests.HTTPError): + get_pvgis_hourly(latitude=45, longitude=8, **args_pv_json) + # Test if HTTPError is raised and error message is returned if avaiable + requests_mock.get(url_pv_json, status_code=400, + json={'message': 'peakpower Mandatory'}) with pytest.raises(requests.HTTPError): get_pvgis_hourly(latitude=45, longitude=8, **args_pv_json) From ed709cfd9c2f1ff0a0ababafd850768dbdfd08f2 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Thu, 17 Jun 2021 19:01:10 -0500 Subject: [PATCH 24/51] Fix documentation issues --- pvlib/iotools/pvgis.py | 43 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 641985258a..66a9bab1f2 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -127,27 +127,27 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, ----- data includes the following fields: - ======================= ====== ========================================== - raw, mapped Format Description - ======================= ====== ========================================== + ========================== ====== ======================================= + raw, mapped Format Description + ========================== ====== ======================================= **Mapped field names are returned when the map_variables argument is True** --------------------------------------------------------------------------- - P\* float PV system power (W) - G(i)\** float Global irradiance on inclined plane (W/m^2) # noqa - Gb(i)\* float Beam (direct) irradiance on inclined plane (W/m^2) # noqa - Gd(i)\** float Diffuse irradiance on inclined plane (W/m^2) # noqa - Gr(i)\** float Reflected irradiance on inclined plane (W/m^2) # noqa - H_sun, solar_elevation float Sun height/elevation (degrees) - T2m, temp_air float Air temperature at 2 m (degrees Celsius) - WS10m, wind_speed float Wind speed at 10 m (m/s) - Int int Solar radiation reconstructed (1/0) - ======================= ====== ========================================== - - \*P (PV system power) is only returned when pvcalculation=True. - - v**Gb(i), Gd(i), and Gr(i) are returned when components=True, whereas + P\** float PV system power (W) + G(i), poa_global\* float Global irradiance on inclined plane (W/m^2) + Gb(i), poa_direct\* float Beam (direct) irradiance on inclined plane (W/m^2) + Gd(i), poa_diffuse\* float Diffuse irradiance on inclined plane (W/m^2) + Gr(i), poa_ground_diffuse\* float Reflected irradiance on inclined plane (W/m^2) + H_sun, solar_elevation float Sun height/elevation (degrees) + T2m, temp_air float Air temperature at 2 m (degrees Celsius) + WS10m, wind_speed float Wind speed at 10 m (m/s) + Int int Solar radiation reconstructed (1/0) + ========================== ====== ======================================= + + \*Gb(i), Gd(i), and Gr(i) are returned when components=True, whereas otherwise the sum of the three components, G(i), is returned. + \**P (PV system power) is only returned when pvcalculation=True. + Raises ------ requests.HTTPError @@ -155,7 +155,7 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, the error message in the response will be raised as an exception, otherwise raise whatever ``HTTP/1.1`` error occurred - See also + See Also -------- pvlib.iotools.read_pvgis_hourly, pvlib.iotools.get_pvgis_tmy @@ -165,7 +165,7 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, .. [2] `PVGIS Hourly Radiation `_ .. [3] `PVGIS Non-interactive service - ` + `_ .. [4] `PVGIS horizon profile tool `_ """ @@ -292,8 +292,7 @@ def _parse_pvgis_hourly_csv(src, map_variables): def read_pvgis_hourly(filename, map_variables=True, pvgis_format=None): - """ - Read a file downloaded from PVGIS. + """Read a PVGIS hourly file. Parameters ---------- @@ -324,7 +323,7 @@ def read_pvgis_hourly(filename, map_variables=True, pvgis_format=None): TypeError if `pvgis_format` is ``None`` and `filename` is a buffer - See also + See Also -------- get_pvgis_hourly, get_pvgis_tmy """ From 5b89dba5b96f80b0b6398c286bc755220ad4c714 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Thu, 17 Jun 2021 19:42:36 -0500 Subject: [PATCH 25/51] Update data columns data --- pvlib/iotools/pvgis.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 66a9bab1f2..66656ee796 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -127,21 +127,21 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, ----- data includes the following fields: - ========================== ====== ======================================= - raw, mapped Format Description - ========================== ====== ======================================= + =========================== ====== ====================================== + raw, mapped Format Description + =========================== ====== ====================================== **Mapped field names are returned when the map_variables argument is True** --------------------------------------------------------------------------- - P\** float PV system power (W) - G(i), poa_global\* float Global irradiance on inclined plane (W/m^2) - Gb(i), poa_direct\* float Beam (direct) irradiance on inclined plane (W/m^2) - Gd(i), poa_diffuse\* float Diffuse irradiance on inclined plane (W/m^2) - Gr(i), poa_ground_diffuse\* float Reflected irradiance on inclined plane (W/m^2) - H_sun, solar_elevation float Sun height/elevation (degrees) - T2m, temp_air float Air temperature at 2 m (degrees Celsius) - WS10m, wind_speed float Wind speed at 10 m (m/s) - Int int Solar radiation reconstructed (1/0) - ========================== ====== ======================================= + P\*\* float PV system power (W) + G(i), poa_global \* float Global irradiance on inclined plane (W/m^2) + Gb(i), poa_direct \* float Beam (direct) irradiance on inclined plane (W/m^2) + Gd(i), poa_diffuse\* float Diffuse irradiance on inclined plane (W/m^2) + Gr(i), poa_ground_diffuse\* float Reflected irradiance on inclined plane (W/m^2) + H_sun, solar_elevation float Sun height/elevation (degrees) + T2m, temp_air float Air temperature at 2 m (degrees Celsius) + WS10m, wind_speed float Wind speed at 10 m (m/s) + Int int Solar radiation reconstructed (1/0) + =========================== ====== ====================================== \*Gb(i), Gd(i), and Gr(i) are returned when components=True, whereas otherwise the sum of the three components, G(i), is returned. From 70e58b32690450f7573308ff143f1715ef15ce3a Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Thu, 17 Jun 2021 19:56:46 -0500 Subject: [PATCH 26/51] Change asterisk to dagger symbol in data table --- pvlib/iotools/pvgis.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 66656ee796..6201b181db 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -130,23 +130,23 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, =========================== ====== ====================================== raw, mapped Format Description =========================== ====== ====================================== - **Mapped field names are returned when the map_variables argument is True** + *Mapped field names are returned when the map_variables argument is True* --------------------------------------------------------------------------- - P\*\* float PV system power (W) - G(i), poa_global \* float Global irradiance on inclined plane (W/m^2) - Gb(i), poa_direct \* float Beam (direct) irradiance on inclined plane (W/m^2) - Gd(i), poa_diffuse\* float Diffuse irradiance on inclined plane (W/m^2) - Gr(i), poa_ground_diffuse\* float Reflected irradiance on inclined plane (W/m^2) + P† float PV system power (W) + G(i), poa_global‡ float Global irradiance on inclined plane (W/m^2) # noqa: E501 + Gb(i), poa_direct‡ float Beam (direct) irradiance on inclined plane (W/m^2) # noqa: E501 + Gd(i), poa_diffuse‡ float Diffuse irradiance on inclined plane (W/m^2) # noqa: E501 + Gr(i), poa_ground_diffuse‡ float Reflected irradiance on inclined plane (W/m^2) # noqa: E501 H_sun, solar_elevation float Sun height/elevation (degrees) - T2m, temp_air float Air temperature at 2 m (degrees Celsius) + T2m, temp_air float Air temperature at 2 m (degrees Celsius) # noqa: E501 WS10m, wind_speed float Wind speed at 10 m (m/s) Int int Solar radiation reconstructed (1/0) =========================== ====== ====================================== - \*Gb(i), Gd(i), and Gr(i) are returned when components=True, whereas - otherwise the sum of the three components, G(i), is returned. + †P (PV system power) is only returned when pvcalculation=True. - \**P (PV system power) is only returned when pvcalculation=True. + ‡Gb(i), Gd(i), and Gr(i) are returned when components=True, otherwise the + sum of the three components, G(i), is returned. Raises ------ From 76f584977a30174c6282652de2a3d4ac06b1b9e6 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Thu, 17 Jun 2021 22:59:38 -0500 Subject: [PATCH 27/51] Update documentation --- pvlib/iotools/pvgis.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 6201b181db..1cb9010c93 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -133,12 +133,12 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, *Mapped field names are returned when the map_variables argument is True* --------------------------------------------------------------------------- P† float PV system power (W) - G(i), poa_global‡ float Global irradiance on inclined plane (W/m^2) # noqa: E501 - Gb(i), poa_direct‡ float Beam (direct) irradiance on inclined plane (W/m^2) # noqa: E501 - Gd(i), poa_diffuse‡ float Diffuse irradiance on inclined plane (W/m^2) # noqa: E501 - Gr(i), poa_ground_diffuse‡ float Reflected irradiance on inclined plane (W/m^2) # noqa: E501 + G(i), poa_global‡ float Global irradiance on inclined plane (W/m^2) + Gb(i), poa_direct‡ float Beam (direct) irradiance on inclined plane (W/m^2) + Gd(i), poa_diffuse‡ float Diffuse irradiance on inclined plane (W/m^2) + Gr(i), poa_ground_diffuse‡ float Reflected irradiance on inclined plane (W/m^2) H_sun, solar_elevation float Sun height/elevation (degrees) - T2m, temp_air float Air temperature at 2 m (degrees Celsius) # noqa: E501 + T2m, temp_air float Air temperature at 2 m (degrees Celsius) WS10m, wind_speed float Wind speed at 10 m (m/s) Int int Solar radiation reconstructed (1/0) =========================== ====== ====================================== From a6e7f3666ed1a25d0fd3d2cd78c4bcec066e8aa9 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Mon, 21 Jun 2021 22:31:58 -0400 Subject: [PATCH 28/51] Rename poa_sky_diffuse and minor doc fixes --- pvlib/iotools/pvgis.py | 16 +++++++++------- pvlib/tests/iotools/test_pvgis.py | 4 ++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 1cb9010c93..312b76e8a1 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -30,7 +30,7 @@ 'Gd(h)': 'dhi', 'G(i)': 'poa_global', 'Gb(i)': 'poa_direct', - 'Gd(i)': 'poa_diffuse', + 'Gd(i)': 'poa_sky_diffuse', 'Gr(i)': 'poa_ground_diffuse', 'H_sun': 'solar_elevation', 'T2m': 'temp_air', @@ -120,7 +120,7 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, Time-series of hourly data, see Notes for fields inputs : dict Dictionary of the request input parameters - metadata : list or dict + metadata : dict Dictionary containing metadata Notes @@ -258,7 +258,7 @@ def _parse_pvgis_hourly_csv(src, map_variables): line = src.readline() if line.startswith('time,'): # The data header starts with 'time,' # The last line of the metadata section contains the column names - names = line.replace('\n', '').replace('\r', '').split(',') + names = line.strip().split(',') break # Only retrieve metadata from non-empty lines elif (line != '\n') & (line != '\r\n'): @@ -273,8 +273,7 @@ def _parse_pvgis_hourly_csv(src, map_variables): if (line == '\n') | (line == '\r\n'): break else: - data_lines.append(line.replace('\n', '').replace('\r', '') - .split(',')) + data_lines.append(line.strip().split(',')) data = pd.DataFrame(data_lines, columns=names) data.index = pd.to_datetime(data['time'], format='%Y%m%d:%H%M', utc=True) data = data.drop('time', axis=1) @@ -291,7 +290,7 @@ def _parse_pvgis_hourly_csv(src, map_variables): return data, inputs, metadata -def read_pvgis_hourly(filename, map_variables=True, pvgis_format=None): +def read_pvgis_hourly(filename, pvgis_format=None, map_variables=True): """Read a PVGIS hourly file. Parameters @@ -304,6 +303,9 @@ def read_pvgis_hourly(filename, map_variables=True, pvgis_format=None): `pvgis_format` is ``None`` then the file extension will be used to determine the PVGIS format to parse. If `filename` is a buffer, then `pvgis_format` is required and must be in ``['csv', 'json']``. + map_variables: bool, default True + When true, renames columns of the Dataframe to pvlib variable names + where applicable. See variable VARIABLE_MAP. Returns ------- @@ -311,7 +313,7 @@ def read_pvgis_hourly(filename, map_variables=True, pvgis_format=None): the weather data inputs : dict the inputs - metadata : list or dict + metadata : dict metadata Raises diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index ea86bc291a..cf6e0a8b42 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -28,12 +28,12 @@ columns_radiation_csv = [ 'Gb(i)', 'Gd(i)', 'Gr(i)', 'H_sun', 'T2m', 'WS10m', 'Int'] columns_radiation_csv_mapped = [ - 'poa_direct', 'poa_diffuse', 'poa_ground_diffuse', 'solar_elevation', + 'poa_direct', 'poa_sky_diffuse', 'poa_ground_diffuse', 'solar_elevation', 'temp_air', 'wind_speed', 'Int'] columns_pv_json = [ 'P', 'Gb(i)', 'Gd(i)', 'Gr(i)', 'H_sun', 'T2m', 'WS10m', 'Int'] columns_pv_json_mapped = [ - 'P', 'poa_direct', 'poa_diffuse', 'poa_ground_diffuse', 'solar_elevation', + 'P', 'poa_direct', 'poa_sky_diffuse', 'poa_ground_diffuse', 'solar_elevation', 'temp_air', 'wind_speed', 'Int'] data_radiation_csv = [ From 7fd98f071f318595282f2e462d2a934bb57d0e7d Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Mon, 28 Jun 2021 23:19:11 -0400 Subject: [PATCH 29/51] Update whatsnew description --- docs/sphinx/source/whatsnew/v0.9.0.rst | 8 ++++---- pvlib/iotools/pvgis.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/sphinx/source/whatsnew/v0.9.0.rst b/docs/sphinx/source/whatsnew/v0.9.0.rst index 4d7afeb031..0a6a2b30e0 100644 --- a/docs/sphinx/source/whatsnew/v0.9.0.rst +++ b/docs/sphinx/source/whatsnew/v0.9.0.rst @@ -103,10 +103,10 @@ Deprecations Enhancements ~~~~~~~~~~~~ -* Add :func:`~pvlib.iotools.read_pvgis_hourly` and - :func:`~pvlib.iotools.get_pvgis_hourly` for reading and retrieving PVGIS - hourly solar radiation data and modelled PV power output. - files. (:pull:`1186`, :issue:`849`) +* Added :func:`~pvlib.iotools.read_pvgis_hourly` and + :func:`~pvlib.iotools.get_pvgis_hourly` for reading and retrieving hourly + solar radiation data and PV power output from PVGIS. (:pull:`1186`, + :issue:`849`) * Add :func:`~pvlib.iotools.read_bsrn` for reading BSRN solar radiation data files. (:pull:`1145`, :issue:`1015`) * Add :func:`~pvlib.iotools.get_cams`, diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 312b76e8a1..3a50d4c17e 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -51,7 +51,7 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, components=True, url=URL, map_variables=True, timeout=30): """Get hourly solar irradiation and modeled PV power output from PVGIS. - PVGIS is avaiable at [1]_. + PVGIS is available at [1]_. Parameters ---------- @@ -78,9 +78,9 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, Name of radiation database. Options depend on location, see [3]_. start: int, default: None First year of the radiation time series. Defaults to first year - avaiable. + available. end: int, default: None - Last year of the radiation time series. Defaults to last year avaiable. + Last year of the radiation time series. Defaults to last year available. pvcalculation: bool, default: False Return estimate of hourly PV production. peakpower: float, default: None From 1223be624295c2a117afd91b3ecd286d64462616 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Tue, 29 Jun 2021 12:52:39 -0400 Subject: [PATCH 30/51] Refactor test_read_pvgis_hourly Also add datetime like input for start and end --- pvlib/iotools/pvgis.py | 17 +++++--- pvlib/tests/iotools/test_pvgis.py | 65 ++++++++++++++++++++++--------- 2 files changed, 58 insertions(+), 24 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 3a50d4c17e..3ff7c67c20 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -76,11 +76,12 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, will calculate the horizon [4]_ raddatabase: str, default: None Name of radiation database. Options depend on location, see [3]_. - start: int, default: None + start: int or datetime like, default: None First year of the radiation time series. Defaults to first year available. - end: int, default: None - Last year of the radiation time series. Defaults to last year available. + end: int or datetime like, default: None + Last year of the radiation time series. Defaults to last year + available. pvcalculation: bool, default: False Return estimate of hourly PV production. peakpower: float, default: None @@ -168,7 +169,7 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, `_ .. [4] `PVGIS horizon profile tool `_ - """ + """ # noqa: E501 # use requests to format the query string by passing params dictionary params = {'lat': latitude, 'lon': longitude, 'outputformat': outputformat, 'angle': surface_tilt, 'aspect': surface_azimuth, @@ -184,10 +185,14 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, params['userhorizon'] = ','.join(str(x) for x in userhorizon) if raddatabase is not None: params['raddatabase'] = raddatabase - if start is not None: + if (start is not None) & (type(start) is int): params['startyear'] = start - if end is not None: + elif (start is not None) & (type(start) is not int): + params['startyear'] = start.year + if (end is not None) & (type(end) is int): params['endyear'] = end + elif (end is not None) & (type(end) is not int): + params['endyear'] = end.year if pvcalculation: params['pvcalculation'] = 1 if peakpower is not None: diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index cf6e0a8b42..c5d957c663 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -126,32 +126,61 @@ 'WS10m': {'description': '10-m total wind speed', 'units': 'm/s'}, # noqa: E501 'Int': {'description': '1 means solar radiation values are reconstructed'}}}}} # noqa: E501 +def generate_expected_dataframe(values, columns, index): + """Create dataframe from arrays of values, columns and index, in order to + use this dataframe to compare to. + """ + expected = pd.DataFrame(index=index, data=values, columns=columns) + expected['Int'] = expected['Int'].astype(int) + expected.index.name = 'time' + expected.index.freq = None + return expected + + +@pytest.fixture +def expected_radiation_csv(): + expected = generate_expected_dataframe( + data_radiation_csv, columns_radiation_csv, index_radiation_csv) + return expected + +@pytest.fixture +def expected_radiation_csv_mapped(): + expected = generate_expected_dataframe( + data_radiation_csv, columns_radiation_csv_mapped, index_radiation_csv) + return expected + +@pytest.fixture +def expected_pv_json(): + expected = generate_expected_dataframe( + data_pv_json, columns_pv_json, index_pv_json) + return expected + +@pytest.fixture +def expected_pv_json_mapped(): + expected = generate_expected_dataframe( + data_pv_json, columns_pv_json_mapped, index_pv_json) + return expected + # Test read_pvgis_hourly function using two different files with different # input arguments (to test variable mapping and pvgis_format) -@pytest.mark.parametrize('testfile,index,columns,values,metadata_exp,' - 'inputs_exp,map_variables,pvgis_format', [ - (testfile_radiation_csv, index_radiation_csv, - columns_radiation_csv, data_radiation_csv, +@pytest.mark.parametrize('testfile,expected_name,metadata_exp,inputs_exp,' + 'map_variables,pvgis_format', [ + (testfile_radiation_csv, 'expected_radiation_csv', metadata_radiation_csv, inputs_radiation_csv, False, None), - (testfile_radiation_csv, index_radiation_csv, - columns_radiation_csv_mapped, data_radiation_csv, + (testfile_radiation_csv, + 'expected_radiation_csv_mapped', metadata_radiation_csv, inputs_radiation_csv, True, 'csv'), - (testfile_pv_json, index_pv_json, columns_pv_json, - data_pv_json, metadata_pv_json, inputs_pv_json, - False, None), - (testfile_pv_json, index_pv_json, - columns_pv_json_mapped, data_pv_json, + (testfile_pv_json, 'expected_pv_json', + metadata_pv_json, inputs_pv_json, False, None), + (testfile_pv_json, 'expected_pv_json_mapped', metadata_pv_json, inputs_pv_json, True, 'json')]) -def test_read_pvgis_hourly(testfile, index, columns, values, metadata_exp, - inputs_exp, map_variables, pvgis_format): - # Create expected dataframe - expected = pd.DataFrame(index=index, data=values, columns=columns) - expected['Int'] = expected['Int'].astype(int) - expected.index.name = 'time' - expected.index.freq = None +def test_read_pvgis_hourly(testfile, expected_name, metadata_exp, + inputs_exp, map_variables, pvgis_format, request): + # Get expected dataframe from fixture + expected = request.getfixturevalue(expected_name) # Read data from file out, inputs, metadata = read_pvgis_hourly( testfile, map_variables=map_variables, pvgis_format=pvgis_format) From 434b7516ae261c1e8a4c99490ff150d398327e65 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Tue, 29 Jun 2021 15:44:04 -0400 Subject: [PATCH 31/51] Coverage for userhorizon, usehorizon, & optimal_surface_tilt --- pvlib/iotools/pvgis.py | 33 +++++----- pvlib/tests/iotools/test_pvgis.py | 100 +++++++++++++++++------------- 2 files changed, 70 insertions(+), 63 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 3ff7c67c20..07b14e4fc8 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -24,7 +24,7 @@ URL = 'https://re.jrc.ec.europa.eu/api/' # Dictionary mapping PVGIS names to pvlib names -VARIABLE_MAP = { +PVGIS_VARIABLE_MAP = { 'G(h)': 'ghi', 'Gb(n)': 'dni', 'Gd(h)': 'dhi', @@ -46,12 +46,12 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, usehorizon=True, userhorizon=None, raddatabase=None, start=None, end=None, pvcalculation=False, peakpower=None, pvtechchoice='crystSi', - mountingplace='free', loss=None, trackingtype=0, + mountingplace='free', loss=0, trackingtype=0, optimal_surface_tilt=False, optimalangles=False, components=True, url=URL, map_variables=True, timeout=30): """Get hourly solar irradiation and modeled PV power output from PVGIS. - PVGIS is available at [1]_. + PVGIS data is freely available at [1]_. Parameters ---------- @@ -91,7 +91,7 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, mountingplace: {'free', 'building'}, default: free Type of mounting for PV system. Options of 'free' for free-standing and 'building' for building-integrated. - loss: float, default: None + loss: float, default: 0 Sum of PV system losses in percent. Required if pvcalculation=True trackingtype: {0, 1, 2, 3, 4, 5}, default: 0 Type of suntracking. 0=fixed, 1=single horizontal axis aligned @@ -111,7 +111,7 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, endpoint map_variables: bool, default True When true, renames columns of the Dataframe to pvlib variable names - where applicable. See variable VARIABLE_MAP. + where applicable. See variable PVGIS_VARIABLE_MAP. timeout: int, default: 30 Time in seconds to wait for server response before timeout @@ -179,8 +179,7 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, # default for usehorizon is already 1 (ie: True), so only set if False # default for pvcalculation, optimalangles, optimalinclination, # is already 0 i.e. False, so only set if True - if not usehorizon: - params['usehorizon'] = 0 + params['usehorizon'] = int(usehorizon) if userhorizon is not None: params['userhorizon'] = ','.join(str(x) for x in userhorizon) if raddatabase is not None: @@ -193,20 +192,16 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, params['endyear'] = end elif (end is not None) & (type(end) is not int): params['endyear'] = end.year - if pvcalculation: - params['pvcalculation'] = 1 + params['pvcalculation'] = int(pvcalculation) if peakpower is not None: params['peakpower'] = peakpower - if loss is not None: - params['loss'] = loss - if optimal_surface_tilt: - params['optimalinclination'] = 1 - if optimalangles: - params['optimalangles'] = 1 + params['loss'] = loss + params['optimalinclination'] = int(optimal_surface_tilt) + params['optimalangles'] = int(optimalangles) # The url endpoint for hourly radiation is 'seriescalc' res = requests.get(url + 'seriescalc', params=params, timeout=timeout) - + print(res.url) # PVGIS returns really well formatted error messages in JSON for HTTP/1.1 # 400 BAD REQUEST so try to return that if possible, otherwise raise the # HTTP/1.1 error caught by requests @@ -241,7 +236,7 @@ def _parse_pvgis_hourly_json(src, map_variables): data = data.drop('time', axis=1) data = data.astype(dtype={'Int': 'int'}) # The 'Int' column to be integer if map_variables: - data.rename(columns=VARIABLE_MAP, inplace=True) + data.rename(columns=PVGIS_VARIABLE_MAP, inplace=True) return data, inputs, metadata @@ -283,7 +278,7 @@ def _parse_pvgis_hourly_csv(src, map_variables): data.index = pd.to_datetime(data['time'], format='%Y%m%d:%H%M', utc=True) data = data.drop('time', axis=1) if map_variables: - data.rename(columns=VARIABLE_MAP, inplace=True) + data.rename(columns=PVGIS_VARIABLE_MAP, inplace=True) # All columns should have the dtype=float, except 'Int' which should be # integer. It is necessary to convert to float, before converting to int data = data.astype(float).astype(dtype={'Int': 'int'}) @@ -310,7 +305,7 @@ def read_pvgis_hourly(filename, pvgis_format=None, map_variables=True): `pvgis_format` is required and must be in ``['csv', 'json']``. map_variables: bool, default True When true, renames columns of the Dataframe to pvlib variable names - where applicable. See variable VARIABLE_MAP. + where applicable. See variable PVGIS_VARIABLE_MAP. Returns ------- diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index c5d957c663..f0ab9b7879 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -33,8 +33,8 @@ columns_pv_json = [ 'P', 'Gb(i)', 'Gd(i)', 'Gr(i)', 'H_sun', 'T2m', 'WS10m', 'Int'] columns_pv_json_mapped = [ - 'P', 'poa_direct', 'poa_sky_diffuse', 'poa_ground_diffuse', 'solar_elevation', - 'temp_air', 'wind_speed', 'Int'] + 'P', 'poa_direct', 'poa_sky_diffuse', 'poa_ground_diffuse', + 'solar_elevation', 'temp_air', 'wind_speed', 'Int'] data_radiation_csv = [ [0.0, 0.0, 0.0, 0.0, 3.44, 1.43, 0.0], @@ -126,6 +126,7 @@ 'WS10m': {'description': '10-m total wind speed', 'units': 'm/s'}, # noqa: E501 'Int': {'description': '1 means solar radiation values are reconstructed'}}}}} # noqa: E501 + def generate_expected_dataframe(values, columns, index): """Create dataframe from arrays of values, columns and index, in order to use this dataframe to compare to. @@ -143,18 +144,21 @@ def expected_radiation_csv(): data_radiation_csv, columns_radiation_csv, index_radiation_csv) return expected + @pytest.fixture def expected_radiation_csv_mapped(): expected = generate_expected_dataframe( data_radiation_csv, columns_radiation_csv_mapped, index_radiation_csv) return expected + @pytest.fixture def expected_pv_json(): expected = generate_expected_dataframe( data_pv_json, columns_pv_json, index_pv_json) return expected + @pytest.fixture def expected_pv_json_mapped(): expected = generate_expected_dataframe( @@ -164,19 +168,16 @@ def expected_pv_json_mapped(): # Test read_pvgis_hourly function using two different files with different # input arguments (to test variable mapping and pvgis_format) -@pytest.mark.parametrize('testfile,expected_name,metadata_exp,inputs_exp,' - 'map_variables,pvgis_format', [ - (testfile_radiation_csv, 'expected_radiation_csv', - metadata_radiation_csv, inputs_radiation_csv, - False, None), - (testfile_radiation_csv, - 'expected_radiation_csv_mapped', - metadata_radiation_csv, inputs_radiation_csv, - True, 'csv'), - (testfile_pv_json, 'expected_pv_json', - metadata_pv_json, inputs_pv_json, False, None), - (testfile_pv_json, 'expected_pv_json_mapped', - metadata_pv_json, inputs_pv_json, True, 'json')]) +# pytest request.getfixturevalue is used to simplify the input arguments +@pytest.mark.parametrize('testfile,expected_name,metadata_exp,inputs_exp,map_variables,pvgis_format', [ # noqa: E501 + (testfile_radiation_csv, 'expected_radiation_csv', metadata_radiation_csv, + inputs_radiation_csv, False, None), + (testfile_radiation_csv, 'expected_radiation_csv_mapped', + metadata_radiation_csv, inputs_radiation_csv, True, 'csv'), + (testfile_pv_json, 'expected_pv_json', metadata_pv_json, inputs_pv_json, + False, None), + (testfile_pv_json, 'expected_pv_json_mapped', metadata_pv_json, + inputs_pv_json, True, 'json')]) def test_read_pvgis_hourly(testfile, expected_name, metadata_exp, inputs_exp, map_variables, pvgis_format, request): # Get expected dataframe from fixture @@ -205,10 +206,10 @@ def test_read_pvgis_hourly_bad_extension(): args_radiation_csv = { 'surface_tilt': 30, 'surface_azimuth': 0, 'outputformat': 'csv', - 'usehorizon': True, 'userhorizon': None, 'raddatabase': 'PVGIS-SARAH', + 'usehorizon': False, 'userhorizon': None, 'raddatabase': 'PVGIS-SARAH', 'start': 2016, 'end': 2016, 'pvcalculation': False, 'components': True} -url_hourly_radiation_csv = 'https://re.jrc.ec.europa.eu/api/seriescalc?lat=45&lon=8&outputformat=csv&angle=30&aspect=0&pvtechchoice=crystSi&mountingplace=free&trackingtype=0&components=1&raddatabase=PVGIS-SARAH&startyear=2016&endyear=2016' # noqa: E501 +url_hourly_radiation_csv = 'https://re.jrc.ec.europa.eu/api/seriescalc?lat=45&lon=8&outputformat=csv&angle=30&aspect=0&usehorizon=0&pvtechchoice=crystSi&mountingplace=free&trackingtype=0&components=1&raddatabase=PVGIS-SARAH&startyear=2016&endyear=2016' # noqa: E501 args_pv_json = { 'surface_tilt': 30, 'surface_azimuth': 0, 'outputformat': 'json', @@ -217,26 +218,18 @@ def test_read_pvgis_hourly_bad_extension(): 'pvtechchoice': 'CIS', 'loss': 5, 'trackingtype': 2, 'optimalangles': True, 'components': True} -url_pv_json = 'https://re.jrc.ec.europa.eu/api/seriescalc?lat=45&lon=8&outputformat=json&angle=30&aspect=0&pvtechchoice=CIS&mountingplace=free&trackingtype=2&components=1&raddatabase=PVGIS-CMSAF&startyear=2013&endyear=2014&pvcalculation=1&peakpower=10&loss=5&optimalangles=1' # noqa: E501 - - -@pytest.mark.parametrize('testfile,index,columns,values,args,map_variables,' - 'url_test', [ - (testfile_radiation_csv, index_radiation_csv, - columns_radiation_csv, data_radiation_csv, - args_radiation_csv, False, - url_hourly_radiation_csv), - (testfile_radiation_csv, index_radiation_csv, - columns_radiation_csv_mapped, data_radiation_csv, - args_radiation_csv, True, - url_hourly_radiation_csv), - (testfile_pv_json, index_pv_json, columns_pv_json, - data_pv_json, args_pv_json, False, url_pv_json), - (testfile_pv_json, index_pv_json, - columns_pv_json_mapped, data_pv_json, - args_pv_json, True, url_pv_json)]) -def test_get_pvgis_hourly(requests_mock, testfile, index, columns, values, - args, map_variables, url_test): +url_pv_json = 'https://re.jrc.ec.europa.eu/api/seriescalc?lat=45&lon=8&outputformat=json&angle=30&aspect=0&pvtechchoice=CIS&mountingplace=free&trackingtype=2&components=1&usehorizon=1&raddatabase=PVGIS-CMSAF&startyear=2013&endyear=2014&pvcalculation=1&peakpower=10&loss=5&optimalangles=1' # noqa: E501 + +@pytest.mark.parametrize('testfile,expected_name,args,map_variables,url_test', [ # noqa: E501 + (testfile_radiation_csv, 'expected_radiation_csv', + args_radiation_csv, False, url_hourly_radiation_csv), + (testfile_radiation_csv, 'expected_radiation_csv_mapped', + args_radiation_csv, True, url_hourly_radiation_csv), + (testfile_pv_json, 'expected_pv_json', args_pv_json, False, url_pv_json), + (testfile_pv_json, 'expected_pv_json_mapped', args_pv_json, True, + url_pv_json)]) +def test_get_pvgis_hourly(requests_mock, testfile, expected_name, args, + map_variables, url_test, request): """Test that get_pvgis_hourly generates the correct URI request and that _parse_pvgis_hourly_json and _parse_pvgis_hourly_csv is called correctly""" # Open local test file containing McClear mothly data @@ -244,17 +237,12 @@ def test_get_pvgis_hourly(requests_mock, testfile, index, columns, values, mock_response = test_file.read() # Specify the full URI of a specific example, this ensures that all of the # inputs are passing on correctly - requests_mock.get(url_test, text=mock_response) - # Make API call - an error is raised if requested URI does not match out, inputs, metadata = get_pvgis_hourly( latitude=45, longitude=8, map_variables=map_variables, **args) - # Create expected dataframe - expected = pd.DataFrame(index=index, data=values, columns=columns) - expected['Int'] = expected['Int'].astype(int) - expected.index.name = 'time' - expected.index.freq = None + # Get expected dataframe from fixture + expected = request.getfixturevalue(expected_name) # Compare out and expected dataframes assert_frame_equal(out, expected) @@ -271,6 +259,30 @@ def test_get_pvgis_hourly_bad_status_code(requests_mock): get_pvgis_hourly(latitude=45, longitude=8, **args_pv_json) +url_additional_inputs = 'https://re.jrc.ec.europa.eu/api/seriescalc?lat=55.6814&lon=12.5758&outputformat=csv&angle=0&aspect=0&pvtechchoice=crystSi&mountingplace=free&trackingtype=0&components=1&usehorizon=1&userhorizon=10%2C15%2C20%2C10&pvcalculation=1&peakpower=5&loss=2&optimalinclination=0&optimalangles=1' # noqa: E501 + + +def test_get_pvgis_hourly_additional_inputs(requests_mock): + # Test additional inputs, including userhorizons + # Necessary to pass a test file in order for the parser not to fail + with open(testfile_radiation_csv, 'r') as test_file: + mock_response = test_file.read() + requests_mock.get(url_additional_inputs, text=mock_response) + # Make request with userhorizon specified + get_pvgis_hourly( + latitude=55.6814, + longitude=12.5758, + outputformat='csv', + usehorizon=True, + userhorizon=[10, 15, 20, 10], + pvcalculation=True, + peakpower=5, + loss=2, + trackingtype=0, + components=True, + optimalangles=True) + + # PVGIS TMY tests @pytest.fixture def expected(): From d37f3cc5e28a11e553aca49035dc06c87dd86124 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Tue, 29 Jun 2021 16:08:08 -0400 Subject: [PATCH 32/51] Add inputs to params dict instead of if statements It is more robust to always pass in the input arguments instead of relying on the PVGIS defaults (these could change and then the pvlib documentation would be wrong). --- pvlib/iotools/pvgis.py | 17 ++++++----------- pvlib/tests/iotools/test_pvgis.py | 23 ++++++++--------------- 2 files changed, 14 insertions(+), 26 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 07b14e4fc8..e8cf24b265 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -173,13 +173,12 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, # use requests to format the query string by passing params dictionary params = {'lat': latitude, 'lon': longitude, 'outputformat': outputformat, 'angle': surface_tilt, 'aspect': surface_azimuth, - 'pvtechchoice': pvtechchoice, 'mountingplace': mountingplace, - 'trackingtype': trackingtype, 'components': int(components)} - # pvgis only likes 0 for False, and 1 for True, not strings, also the - # default for usehorizon is already 1 (ie: True), so only set if False - # default for pvcalculation, optimalangles, optimalinclination, - # is already 0 i.e. False, so only set if True - params['usehorizon'] = int(usehorizon) + 'pvcalculation': int(pvcalculation), 'pvtechchoice': pvtechchoice, + 'mountingplace': mountingplace, 'trackingtype': trackingtype, + 'components': int(components), 'usehorizon': int(usehorizon), + 'optimalangles': int(optimalangles), + 'optimalinclination': int(optimalangles), 'loss': loss} + # pvgis only takes 0 for False, and 1 for True, not strings, also the if userhorizon is not None: params['userhorizon'] = ','.join(str(x) for x in userhorizon) if raddatabase is not None: @@ -192,12 +191,8 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, params['endyear'] = end elif (end is not None) & (type(end) is not int): params['endyear'] = end.year - params['pvcalculation'] = int(pvcalculation) if peakpower is not None: params['peakpower'] = peakpower - params['loss'] = loss - params['optimalinclination'] = int(optimal_surface_tilt) - params['optimalangles'] = int(optimalangles) # The url endpoint for hourly radiation is 'seriescalc' res = requests.get(url + 'seriescalc', params=params, timeout=timeout) diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index f0ab9b7879..be50b8272a 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -214,12 +214,13 @@ def test_read_pvgis_hourly_bad_extension(): args_pv_json = { 'surface_tilt': 30, 'surface_azimuth': 0, 'outputformat': 'json', 'usehorizon': True, 'userhorizon': None, 'raddatabase': 'PVGIS-CMSAF', - 'start': 2013, 'end': 2014, 'pvcalculation': True, 'peakpower': 10, - 'pvtechchoice': 'CIS', 'loss': 5, 'trackingtype': 2, 'optimalangles': True, - 'components': True} + 'start': pd.Timestamp(2013,1,1), 'end': pd.Timestamp(2014,5,1), + 'pvcalculation': True, 'peakpower': 10, 'pvtechchoice': 'CIS', 'loss': 5, + 'trackingtype': 2, 'optimalangles': True, 'components': True} url_pv_json = 'https://re.jrc.ec.europa.eu/api/seriescalc?lat=45&lon=8&outputformat=json&angle=30&aspect=0&pvtechchoice=CIS&mountingplace=free&trackingtype=2&components=1&usehorizon=1&raddatabase=PVGIS-CMSAF&startyear=2013&endyear=2014&pvcalculation=1&peakpower=10&loss=5&optimalangles=1' # noqa: E501 + @pytest.mark.parametrize('testfile,expected_name,args,map_variables,url_test', [ # noqa: E501 (testfile_radiation_csv, 'expected_radiation_csv', args_radiation_csv, False, url_hourly_radiation_csv), @@ -259,8 +260,7 @@ def test_get_pvgis_hourly_bad_status_code(requests_mock): get_pvgis_hourly(latitude=45, longitude=8, **args_pv_json) -url_additional_inputs = 'https://re.jrc.ec.europa.eu/api/seriescalc?lat=55.6814&lon=12.5758&outputformat=csv&angle=0&aspect=0&pvtechchoice=crystSi&mountingplace=free&trackingtype=0&components=1&usehorizon=1&userhorizon=10%2C15%2C20%2C10&pvcalculation=1&peakpower=5&loss=2&optimalinclination=0&optimalangles=1' # noqa: E501 - +url_additional_inputs = 'https://re.jrc.ec.europa.eu/api/seriescalc?lat=55.6814&lon=12.5758&outputformat=csv&angle=0&aspect=0&pvcalculation=1&pvtechchoice=crystSi&mountingplace=free&trackingtype=0&components=1&usehorizon=1&optimalangles=1&optimalinclination=1&loss=2&userhorizon=10%2C15%2C20%2C10&peakpower=5' # noqa: E501 def test_get_pvgis_hourly_additional_inputs(requests_mock): # Test additional inputs, including userhorizons @@ -270,16 +270,9 @@ def test_get_pvgis_hourly_additional_inputs(requests_mock): requests_mock.get(url_additional_inputs, text=mock_response) # Make request with userhorizon specified get_pvgis_hourly( - latitude=55.6814, - longitude=12.5758, - outputformat='csv', - usehorizon=True, - userhorizon=[10, 15, 20, 10], - pvcalculation=True, - peakpower=5, - loss=2, - trackingtype=0, - components=True, + latitude=55.6814, longitude=12.5758, outputformat='csv', + usehorizon=True, userhorizon=[10, 15, 20, 10], pvcalculation=True, + peakpower=5, loss=2, trackingtype=0, components=True, optimalangles=True) From 98fb5f14516a0bc55a9e32c61da4364c4f62e00c Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Tue, 29 Jun 2021 16:29:27 -0400 Subject: [PATCH 33/51] Fix stickler --- pvlib/iotools/pvgis.py | 17 +++++++++-------- pvlib/tests/iotools/test_pvgis.py | 3 ++- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index e8cf24b265..54111b6beb 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -107,8 +107,8 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, Output solar radiation components (beam, diffuse, and reflected). Otherwise only global irradiance is returned. url: str, default:const:`pvlib.iotools.pvgis.URL` - Base url of PVGIS API, append ``seriescalc`` to get hourly data - endpoint + Base url of PVGIS API. ``seriescalc`` is appended to get hourly data + endpoint. map_variables: bool, default True When true, renames columns of the Dataframe to pvlib variable names where applicable. See variable PVGIS_VARIABLE_MAP. @@ -173,9 +173,10 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, # use requests to format the query string by passing params dictionary params = {'lat': latitude, 'lon': longitude, 'outputformat': outputformat, 'angle': surface_tilt, 'aspect': surface_azimuth, - 'pvcalculation': int(pvcalculation), 'pvtechchoice': pvtechchoice, - 'mountingplace': mountingplace, 'trackingtype': trackingtype, - 'components': int(components), 'usehorizon': int(usehorizon), + 'pvcalculation': int(pvcalculation), + 'pvtechchoice': pvtechchoice, 'mountingplace': mountingplace, + 'trackingtype': trackingtype, 'components': int(components), + 'usehorizon': int(usehorizon), 'optimalangles': int(optimalangles), 'optimalinclination': int(optimalangles), 'loss': loss} # pvgis only takes 0 for False, and 1 for True, not strings, also the @@ -291,7 +292,7 @@ def read_pvgis_hourly(filename, pvgis_format=None, map_variables=True): Parameters ---------- filename : str, pathlib.Path, or file-like buffer - Name, path, or buffer of file downloaded from PVGIS. + Name, path, or buffer of hourly data file downloaded from PVGIS. pvgis_format : str, default None Format of PVGIS file or buffer. Equivalent to the ``outputformat`` parameter in the PVGIS API. If `filename` is a file and @@ -299,13 +300,13 @@ def read_pvgis_hourly(filename, pvgis_format=None, map_variables=True): determine the PVGIS format to parse. If `filename` is a buffer, then `pvgis_format` is required and must be in ``['csv', 'json']``. map_variables: bool, default True - When true, renames columns of the Dataframe to pvlib variable names + When true, renames columns of the DataFrame to pvlib variable names where applicable. See variable PVGIS_VARIABLE_MAP. Returns ------- data : pandas.DataFrame - the weather data + the time series data inputs : dict the inputs metadata : dict diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index be50b8272a..8a96d11211 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -214,7 +214,7 @@ def test_read_pvgis_hourly_bad_extension(): args_pv_json = { 'surface_tilt': 30, 'surface_azimuth': 0, 'outputformat': 'json', 'usehorizon': True, 'userhorizon': None, 'raddatabase': 'PVGIS-CMSAF', - 'start': pd.Timestamp(2013,1,1), 'end': pd.Timestamp(2014,5,1), + 'start': pd.Timestamp(2013, 1, 1), 'end': pd.Timestamp(2014, 5, 1), 'pvcalculation': True, 'peakpower': 10, 'pvtechchoice': 'CIS', 'loss': 5, 'trackingtype': 2, 'optimalangles': True, 'components': True} @@ -262,6 +262,7 @@ def test_get_pvgis_hourly_bad_status_code(requests_mock): url_additional_inputs = 'https://re.jrc.ec.europa.eu/api/seriescalc?lat=55.6814&lon=12.5758&outputformat=csv&angle=0&aspect=0&pvcalculation=1&pvtechchoice=crystSi&mountingplace=free&trackingtype=0&components=1&usehorizon=1&optimalangles=1&optimalinclination=1&loss=2&userhorizon=10%2C15%2C20%2C10&peakpower=5' # noqa: E501 + def test_get_pvgis_hourly_additional_inputs(requests_mock): # Test additional inputs, including userhorizons # Necessary to pass a test file in order for the parser not to fail From cd13cda20c62205fe5e25ffae4bed2ab0f00794f Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Thu, 1 Jul 2021 00:08:04 -0400 Subject: [PATCH 34/51] Doc. update & refactoring of start/end --- pvlib/iotools/pvgis.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 54111b6beb..30bec662e9 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -136,7 +136,7 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, P† float PV system power (W) G(i), poa_global‡ float Global irradiance on inclined plane (W/m^2) Gb(i), poa_direct‡ float Beam (direct) irradiance on inclined plane (W/m^2) - Gd(i), poa_diffuse‡ float Diffuse irradiance on inclined plane (W/m^2) + Gd(i), poa_sky_diffuse‡ float Diffuse irradiance on inclined plane (W/m^2) Gr(i), poa_ground_diffuse‡ float Reflected irradiance on inclined plane (W/m^2) H_sun, solar_elevation float Sun height/elevation (degrees) T2m, temp_air float Air temperature at 2 m (degrees Celsius) @@ -186,18 +186,15 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, params['raddatabase'] = raddatabase if (start is not None) & (type(start) is int): params['startyear'] = start - elif (start is not None) & (type(start) is not int): - params['startyear'] = start.year - if (end is not None) & (type(end) is int): - params['endyear'] = end - elif (end is not None) & (type(end) is not int): - params['endyear'] = end.year + elif start is not None: + params['startyear'] = start if isinstance(start, int) else start.year + if end is not None: + params['endyear'] = end if isinstance(end, int) else end.year if peakpower is not None: params['peakpower'] = peakpower # The url endpoint for hourly radiation is 'seriescalc' res = requests.get(url + 'seriescalc', params=params, timeout=timeout) - print(res.url) # PVGIS returns really well formatted error messages in JSON for HTTP/1.1 # 400 BAD REQUEST so try to return that if possible, otherwise raise the # HTTP/1.1 error caught by requests @@ -210,7 +207,7 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, raise requests.HTTPError(err_msg['message']) # initialize data to None in case API fails to respond to bad outputformat - data = None, None, None, None + data = None, None, None if outputformat == 'json': src = res.json() return _parse_pvgis_hourly_json(src, map_variables=map_variables) From aad4f4da45a100f35949d36e0ef6a3f9db37ab97 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Tue, 6 Jul 2021 17:26:29 -0400 Subject: [PATCH 35/51] Reorder inputs & capitalize input descriptions --- pvlib/iotools/pvgis.py | 44 ++++++++++++++++--------------- pvlib/tests/iotools/test_pvgis.py | 2 ++ 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 30bec662e9..5d522c4524 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -41,14 +41,16 @@ } -def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, +def get_pvgis_hourly(latitude, longitude, start=None, end=None, + raddatabase=None, components=True, + surface_tilt=0, surface_azimuth=0, outputformat='json', - usehorizon=True, userhorizon=None, raddatabase=None, - start=None, end=None, pvcalculation=False, + usehorizon=True, userhorizon=None, + pvcalculation=False, peakpower=None, pvtechchoice='crystSi', mountingplace='free', loss=0, trackingtype=0, optimal_surface_tilt=False, optimalangles=False, - components=True, url=URL, map_variables=True, timeout=30): + url=URL, map_variables=True, timeout=30): """Get hourly solar irradiation and modeled PV power output from PVGIS. PVGIS data is freely available at [1]_. @@ -56,17 +58,25 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, Parameters ---------- latitude: float - in decimal degrees, between -90 and 90, north is positive (ISO 19115) + In decimal degrees, between -90 and 90, north is positive (ISO 19115) longitude: float - in decimal degrees, between -180 and 180, east is positive (ISO 19115) + In decimal degrees, between -180 and 180, east is positive (ISO 19115) + start: int or datetime like, default: None + First year of the radiation time series. Defaults to first year + available. + end: int or datetime like, default: None + Last year of the radiation time series. Defaults to last year + available. + raddatabase: str, default: None + Name of radiation database. Options depend on location, see [3]_. + components: bool, default: True + Output solar radiation components (beam, diffuse, and reflected). + Otherwise only global irradiance is returned. surface_tilt: float, default: 0 Tilt angle from horizontal plane. Not relevant for 2-axis tracking. surface_azimuth: float, default: 0 Orientation (azimuth angle) of the (fixed) plane. 0=south, 90=west, -90: east. Not relevant for tracking systems. - outputformat: str, default: 'json' - Must be in ``['json', 'csv']``. See PVGIS hourly data - documentation [2]_ for more info. usehorizon: bool, default: True Include effects of horizon userhorizon: list of float, default: None @@ -74,14 +84,6 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, spaced azimuth clockwise from north, only valid if `usehorizon` is true, if `usehorizon` is true but `userhorizon` is `None` then PVGIS will calculate the horizon [4]_ - raddatabase: str, default: None - Name of radiation database. Options depend on location, see [3]_. - start: int or datetime like, default: None - First year of the radiation time series. Defaults to first year - available. - end: int or datetime like, default: None - Last year of the radiation time series. Defaults to last year - available. pvcalculation: bool, default: False Return estimate of hourly PV production. peakpower: float, default: None @@ -103,9 +105,9 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, optimalangles: bool, default: False Calculate the optimum tilt and azimuth angles. Not relevant for 2-axis tracking. - components: bool, default: True - Output solar radiation components (beam, diffuse, and reflected). - Otherwise only global irradiance is returned. + outputformat: str, default: 'json' + Must be in ``['json', 'csv']``. See PVGIS hourly data + documentation [2]_ for more info. url: str, default:const:`pvlib.iotools.pvgis.URL` Base url of PVGIS API. ``seriescalc`` is appended to get hourly data endpoint. @@ -152,7 +154,7 @@ def get_pvgis_hourly(latitude, longitude, surface_tilt=0, surface_azimuth=0, Raises ------ requests.HTTPError - if the request response status is ``HTTP/1.1 400 BAD REQUEST``, then + If the request response status is ``HTTP/1.1 400 BAD REQUEST``, then the error message in the response will be raised as an exception, otherwise raise whatever ``HTTP/1.1`` error occurred diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index 8a96d11211..f2364afa8f 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -270,6 +270,8 @@ def test_get_pvgis_hourly_additional_inputs(requests_mock): mock_response = test_file.read() requests_mock.get(url_additional_inputs, text=mock_response) # Make request with userhorizon specified + # Test passes if the request made by get_pvgis_hourly matches exactly the + # url passed to the mock request (url_additional_inputs) get_pvgis_hourly( latitude=55.6814, longitude=12.5758, outputformat='csv', usehorizon=True, userhorizon=[10, 15, 20, 10], pvcalculation=True, From 116c4278b0a11be14744f91b167d544f9f1774d7 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Tue, 6 Jul 2021 19:01:36 -0400 Subject: [PATCH 36/51] Use .strip() instead of replace('\r\n','') Also remove instances of using inplace --- pvlib/iotools/pvgis.py | 23 ++++++++++------------- pvlib/tests/iotools/test_pvgis.py | 2 +- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 5d522c4524..798f404bba 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -231,7 +231,7 @@ def _parse_pvgis_hourly_json(src, map_variables): data = data.drop('time', axis=1) data = data.astype(dtype={'Int': 'int'}) # The 'Int' column to be integer if map_variables: - data.rename(columns=PVGIS_VARIABLE_MAP, inplace=True) + data = data.rename(columns=PVGIS_VARIABLE_MAP) return data, inputs, metadata @@ -245,35 +245,32 @@ def _parse_pvgis_hourly_csv(src, map_variables): # Elevation (m): 1389.0\r\n inputs['elevation'] = float(src.readline().split(':')[1]) # 'Radiation database: \tPVGIS-SARAH\r\n' - inputs['radiation_database'] = str(src.readline().split(':')[1] - .replace('\t', '').replace('\n', '')) + inputs['radiation_database'] = src.readline().split(':')[1].strip() # Parse through the remaining metadata section (the number of lines for # this section depends on the requested parameters) while True: - line = src.readline() + line = src.readline().strip() if line.startswith('time,'): # The data header starts with 'time,' # The last line of the metadata section contains the column names - names = line.strip().split(',') + names = line.split(',') break # Only retrieve metadata from non-empty lines - elif (line != '\n') & (line != '\r\n'): - inputs[line.split(':')[0]] = str(line.split(':')[1] - .replace('\n', '') - .replace('\r', '').strip()) + elif line != '': + inputs[line.split(':')[0]] = line.split(':')[1] # Save the entries from the data section to a list, until an empty line is # reached an empty line. The length of the section depends on the request data_lines = [] while True: - line = src.readline() - if (line == '\n') | (line == '\r\n'): + line = src.readline().strip() + if line == '': break else: - data_lines.append(line.strip().split(',')) + data_lines.append(line.split(',')) data = pd.DataFrame(data_lines, columns=names) data.index = pd.to_datetime(data['time'], format='%Y%m%d:%H%M', utc=True) data = data.drop('time', axis=1) if map_variables: - data.rename(columns=PVGIS_VARIABLE_MAP, inplace=True) + data = data.rename(columns=PVGIS_VARIABLE_MAP) # All columns should have the dtype=float, except 'Int' which should be # integer. It is necessary to convert to float, before converting to int data = data.astype(float).astype(dtype={'Int': 'int'}) diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index f2364afa8f..c1e307de0b 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -233,7 +233,7 @@ def test_get_pvgis_hourly(requests_mock, testfile, expected_name, args, map_variables, url_test, request): """Test that get_pvgis_hourly generates the correct URI request and that _parse_pvgis_hourly_json and _parse_pvgis_hourly_csv is called correctly""" - # Open local test file containing McClear mothly data + # Open local test file containing McClear monthly data with open(testfile, 'r') as test_file: mock_response = test_file.read() # Specify the full URI of a specific example, this ensures that all of the From 1a0f4280d039fc9874a5bd58b012d1da70a9f2aa Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Tue, 6 Jul 2021 19:28:40 -0400 Subject: [PATCH 37/51] Update pvgis.py --- pvlib/iotools/pvgis.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 798f404bba..5cac49f355 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -249,23 +249,25 @@ def _parse_pvgis_hourly_csv(src, map_variables): # Parse through the remaining metadata section (the number of lines for # this section depends on the requested parameters) while True: - line = src.readline().strip() + line = src.readline() if line.startswith('time,'): # The data header starts with 'time,' # The last line of the metadata section contains the column names - names = line.split(',') + names = line.strip().split(',') break # Only retrieve metadata from non-empty lines - elif line != '': - inputs[line.split(':')[0]] = line.split(':')[1] + elif line.strip() != '': + inputs[line.split(':')[0]] = line.split(':')[1].strip() + elif line == '': # If end of file is reached + break # Save the entries from the data section to a list, until an empty line is # reached an empty line. The length of the section depends on the request data_lines = [] while True: - line = src.readline().strip() - if line == '': + line = src.readline() + if line.strip() == '': break else: - data_lines.append(line.split(',')) + data_lines.append(line.strip().split(',')) data = pd.DataFrame(data_lines, columns=names) data.index = pd.to_datetime(data['time'], format='%Y%m%d:%H%M', utc=True) data = data.drop('time', axis=1) From 5ffceda546ca4c76f69cb9bd05281fa061ea2f4b Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Tue, 6 Jul 2021 20:06:59 -0400 Subject: [PATCH 38/51] Raise ValueError for invalid outputformat --- pvlib/iotools/pvgis.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 5cac49f355..e91b0efc34 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -208,8 +208,6 @@ def get_pvgis_hourly(latitude, longitude, start=None, end=None, else: raise requests.HTTPError(err_msg['message']) - # initialize data to None in case API fails to respond to bad outputformat - data = None, None, None if outputformat == 'json': src = res.json() return _parse_pvgis_hourly_json(src, map_variables=map_variables) @@ -219,8 +217,7 @@ def get_pvgis_hourly(latitude, longitude, start=None, end=None, else: # this line is never reached because if outputformat is not valid then # the response is HTTP/1.1 400 BAD REQUEST which is handled earlier - pass - return data + raise ValueError('Invalid outputformat.') def _parse_pvgis_hourly_json(src, map_variables): From ff904112c9ec636e01e1a029df0d82f65e937e5b Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Tue, 6 Jul 2021 20:07:11 -0400 Subject: [PATCH 39/51] Coverage for bad outputformat --- pvlib/tests/iotools/test_pvgis.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index c1e307de0b..fcb63355d3 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -260,6 +260,17 @@ def test_get_pvgis_hourly_bad_status_code(requests_mock): get_pvgis_hourly(latitude=45, longitude=8, **args_pv_json) +url_bad_outputformat = 'https://re.jrc.ec.europa.eu/api/seriescalc?lat=45&lon=8&outputformat=basic&angle=0&aspect=0&pvcalculation=0&pvtechchoice=crystSi&mountingplace=free&trackingtype=0&components=1&usehorizon=1&optimalangles=0&optimalinclination=0&loss=0' # noqa: E501 + + +def test_get_pvgis_hourly_bad_outputformat(requests_mock): + # Test if a ValueError is raised if an unsupported outputformat is used + # E.g. 'basic' is a valid PVGIS format, but is not supported by pvlib + requests_mock.get(url_bad_outputformat) + with pytest.raises(ValueError): + get_pvgis_hourly(latitude=45, longitude=8, outputformat='basic') + + url_additional_inputs = 'https://re.jrc.ec.europa.eu/api/seriescalc?lat=55.6814&lon=12.5758&outputformat=csv&angle=0&aspect=0&pvcalculation=1&pvtechchoice=crystSi&mountingplace=free&trackingtype=0&components=1&usehorizon=1&optimalangles=1&optimalinclination=1&loss=2&userhorizon=10%2C15%2C20%2C10&peakpower=5' # noqa: E501 From 0faeeb847835ab57e745ee554ab9f9d1d1d8fc6e Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Tue, 6 Jul 2021 20:21:23 -0400 Subject: [PATCH 40/51] Minor doc fixes --- pvlib/iotools/pvgis.py | 6 +++--- pvlib/tests/iotools/test_pvgis.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index e91b0efc34..3a820c27d3 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -108,10 +108,10 @@ def get_pvgis_hourly(latitude, longitude, start=None, end=None, outputformat: str, default: 'json' Must be in ``['json', 'csv']``. See PVGIS hourly data documentation [2]_ for more info. - url: str, default:const:`pvlib.iotools.pvgis.URL` + url: str, default: const:`pvlib.iotools.pvgis.URL` Base url of PVGIS API. ``seriescalc`` is appended to get hourly data endpoint. - map_variables: bool, default True + map_variables: bool, default: True When true, renames columns of the Dataframe to pvlib variable names where applicable. See variable PVGIS_VARIABLE_MAP. timeout: int, default: 30 @@ -318,7 +318,7 @@ def read_pvgis_hourly(filename, pvgis_format=None, map_variables=True): See Also -------- - get_pvgis_hourly, get_pvgis_tmy + get_pvgis_hourly, read_pvgis_tmy """ # get the PVGIS outputformat if pvgis_format is None: diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index fcb63355d3..e3a9207049 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -269,7 +269,7 @@ def test_get_pvgis_hourly_bad_outputformat(requests_mock): requests_mock.get(url_bad_outputformat) with pytest.raises(ValueError): get_pvgis_hourly(latitude=45, longitude=8, outputformat='basic') - + url_additional_inputs = 'https://re.jrc.ec.europa.eu/api/seriescalc?lat=55.6814&lon=12.5758&outputformat=csv&angle=0&aspect=0&pvcalculation=1&pvtechchoice=crystSi&mountingplace=free&trackingtype=0&components=1&usehorizon=1&optimalangles=1&optimalinclination=1&loss=2&userhorizon=10%2C15%2C20%2C10&peakpower=5' # noqa: E501 From 919fb8d7a8f803ba4ffd9dbff16aa51ff6f13b10 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Tue, 6 Jul 2021 20:49:18 -0400 Subject: [PATCH 41/51] Change 'not relevant' to 'ignored' --- pvlib/iotools/pvgis.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 3a820c27d3..4776c312dc 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -73,10 +73,10 @@ def get_pvgis_hourly(latitude, longitude, start=None, end=None, Output solar radiation components (beam, diffuse, and reflected). Otherwise only global irradiance is returned. surface_tilt: float, default: 0 - Tilt angle from horizontal plane. Not relevant for 2-axis tracking. + Tilt angle from horizontal plane. Ignored for two-axis tracking. surface_azimuth: float, default: 0 Orientation (azimuth angle) of the (fixed) plane. 0=south, 90=west, - -90: east. Not relevant for tracking systems. + -90: east. Ignored for tracking systems. usehorizon: bool, default: True Include effects of horizon userhorizon: list of float, default: None @@ -101,9 +101,9 @@ def get_pvgis_hourly(latitude, longitude, start=None, end=None, horizontal axis aligned east-west, 5=single inclined axis aligned north-south. optimal_surface_tilt: bool, default: False - Calculate the optimum tilt angle. Not relevant for 2-axis tracking + Calculate the optimum tilt angle. Ignored for 2-axis tracking optimalangles: bool, default: False - Calculate the optimum tilt and azimuth angles. Not relevant for 2-axis + Calculate the optimum tilt and azimuth angles. Ignored for two-axis tracking. outputformat: str, default: 'json' Must be in ``['json', 'csv']``. See PVGIS hourly data From b52a8d925ccf0f3e42c08346f2a206bb2c827c9f Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Wed, 7 Jul 2021 10:23:09 -0400 Subject: [PATCH 42/51] Have get function call read function, instead of individual parse functions --- pvlib/iotools/pvgis.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 4776c312dc..c5d10908e6 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -101,7 +101,7 @@ def get_pvgis_hourly(latitude, longitude, start=None, end=None, horizontal axis aligned east-west, 5=single inclined axis aligned north-south. optimal_surface_tilt: bool, default: False - Calculate the optimum tilt angle. Ignored for 2-axis tracking + Calculate the optimum tilt angle. Ignored for two-axis tracking optimalangles: bool, default: False Calculate the optimum tilt and azimuth angles. Ignored for two-axis tracking. @@ -208,16 +208,8 @@ def get_pvgis_hourly(latitude, longitude, start=None, end=None, else: raise requests.HTTPError(err_msg['message']) - if outputformat == 'json': - src = res.json() - return _parse_pvgis_hourly_json(src, map_variables=map_variables) - elif outputformat == 'csv': - with io.StringIO(res.content.decode('utf-8')) as src: - return _parse_pvgis_hourly_csv(src, map_variables=map_variables) - else: - # this line is never reached because if outputformat is not valid then - # the response is HTTP/1.1 400 BAD REQUEST which is handled earlier - raise ValueError('Invalid outputformat.') + return read_pvgis_hourly(io.StringIO(res.text), pvgis_format=outputformat, + map_variables=map_variables) def _parse_pvgis_hourly_json(src, map_variables): From 9db85913d759c0c10198a606097c7ae4983059c7 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Wed, 7 Jul 2021 11:57:41 -0400 Subject: [PATCH 43/51] Raise ValueError if no data section is detected --- pvlib/iotools/pvgis.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index c5d10908e6..9a1a675189 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -126,6 +126,13 @@ def get_pvgis_hourly(latitude, longitude, start=None, end=None, metadata : dict Dictionary containing metadata + Raises + ------ + requests.HTTPError + If the request response status is ``HTTP/1.1 400 BAD REQUEST``, then + the error message in the response will be raised as an exception, + otherwise raise whatever ``HTTP/1.1`` error occurred + Notes ----- data includes the following fields: @@ -151,13 +158,6 @@ def get_pvgis_hourly(latitude, longitude, start=None, end=None, ‡Gb(i), Gd(i), and Gr(i) are returned when components=True, otherwise the sum of the three components, G(i), is returned. - Raises - ------ - requests.HTTPError - If the request response status is ``HTTP/1.1 400 BAD REQUEST``, then - the error message in the response will be raised as an exception, - otherwise raise whatever ``HTTP/1.1`` error occurred - See Also -------- pvlib.iotools.read_pvgis_hourly, pvlib.iotools.get_pvgis_tmy @@ -247,7 +247,8 @@ def _parse_pvgis_hourly_csv(src, map_variables): elif line.strip() != '': inputs[line.split(':')[0]] = line.split(':')[1].strip() elif line == '': # If end of file is reached - break + raise ValueError('No data section was detected. File has probably ' + 'been modified since being downloaded from PVGIS') # Save the entries from the data section to a list, until an empty line is # reached an empty line. The length of the section depends on the request data_lines = [] From cc67fc13f3528712aaece8ee45edf56e9deaa0ec Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Wed, 7 Jul 2021 11:58:04 -0400 Subject: [PATCH 44/51] Coverage for empty file passed to read_pvgis_hourly --- pvlib/tests/iotools/test_pvgis.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index e3a9207049..fc0638ed74 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -290,6 +290,14 @@ def test_get_pvgis_hourly_additional_inputs(requests_mock): optimalangles=True) +def test_read_pvgis_hourly_empty_file(): + # Check if a IOError is raised if file does not contain a data section + with pytest.raises(ValueError, match='No data section'): + read_pvgis_hourly( + io.StringIO('1:1\n2:2\n3:3\n4:4\n5:5\n'), + pvgis_format='csv') + + # PVGIS TMY tests @pytest.fixture def expected(): From 16d9cba592bd9413f7040b17647a7602933db56f Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Mon, 12 Jul 2021 22:33:55 -0400 Subject: [PATCH 45/51] Add info box concerning databases and timestamps --- pvlib/iotools/pvgis.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 9a1a675189..426cdc83bb 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -133,6 +133,14 @@ def get_pvgis_hourly(latitude, longitude, start=None, end=None, the error message in the response will be raised as an exception, otherwise raise whatever ``HTTP/1.1`` error occurred + Info + ---- + PVGIS provides access to a number of different solar radiation datasets, + both satellite-based (SARAH, CMSAF, and NSRDB PSM3) and re-analysis products + (ERA5 and COSMO). Each data source has a different geographical coverage + and time stamp convention, e.g., SARAH and CMSAF are instantaneous values, + whereas ERA5 is the average for the hour. + Notes ----- data includes the following fields: From bf6712fd2f330eb4a1e15377ad7e2fbe84d8a46e Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Mon, 12 Jul 2021 22:46:51 -0400 Subject: [PATCH 46/51] Update admonition --- pvlib/iotools/pvgis.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 426cdc83bb..c33af2b1c6 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -133,13 +133,13 @@ def get_pvgis_hourly(latitude, longitude, start=None, end=None, the error message in the response will be raised as an exception, otherwise raise whatever ``HTTP/1.1`` error occurred - Info +!!! info "Radiation databases and timestamp convention" ---- PVGIS provides access to a number of different solar radiation datasets, both satellite-based (SARAH, CMSAF, and NSRDB PSM3) and re-analysis products (ERA5 and COSMO). Each data source has a different geographical coverage - and time stamp convention, e.g., SARAH and CMSAF are instantaneous values, - whereas ERA5 is the average for the hour. + and time stamp convention, e.g., SARAH and CMSAF provide instantaneous + values, whereas values from ERA5 are the average for the hour. Notes ----- From 00139f23c2a86dd6d4a954fd406ade544f1a423f Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Tue, 13 Jul 2021 11:22:08 -0400 Subject: [PATCH 47/51] Change admonition style to 'attention' --- pvlib/iotools/pvgis.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index c33af2b1c6..d2d21dbfe9 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -133,13 +133,13 @@ def get_pvgis_hourly(latitude, longitude, start=None, end=None, the error message in the response will be raised as an exception, otherwise raise whatever ``HTTP/1.1`` error occurred -!!! info "Radiation databases and timestamp convention" + .. Attention:: ---- PVGIS provides access to a number of different solar radiation datasets, - both satellite-based (SARAH, CMSAF, and NSRDB PSM3) and re-analysis products - (ERA5 and COSMO). Each data source has a different geographical coverage - and time stamp convention, e.g., SARAH and CMSAF provide instantaneous - values, whereas values from ERA5 are the average for the hour. + including satellite-based (SARAH, CMSAF, and NSRDB PSM3) and re-analysis + products (ERA5 and COSMO). Each data source has a different geographical + coverage and time stamp convention, e.g., SARAH and CMSAF provide + instantaneous values, whereas values from ERA5 are averages for the hour. Notes ----- From 88734c7eb97290af2399a508311b26a14d90905d Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Tue, 13 Jul 2021 14:19:10 -0400 Subject: [PATCH 48/51] Update admonition --- pvlib/iotools/pvgis.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index d2d21dbfe9..5aa0362f54 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -133,13 +133,13 @@ def get_pvgis_hourly(latitude, longitude, start=None, end=None, the error message in the response will be raised as an exception, otherwise raise whatever ``HTTP/1.1`` error occurred - .. Attention:: - ---- - PVGIS provides access to a number of different solar radiation datasets, - including satellite-based (SARAH, CMSAF, and NSRDB PSM3) and re-analysis - products (ERA5 and COSMO). Each data source has a different geographical - coverage and time stamp convention, e.g., SARAH and CMSAF provide - instantaneous values, whereas values from ERA5 are averages for the hour. + .. Hint:: + PVGIS provides access to a number of different solar radiation + datasets, including satellite-based (SARAH, CMSAF, and NSRDB PSM3) and + re-analysis products (ERA5 and COSMO). Each data source has a + different geographical coverage and time stamp convention, e.g., SARAH + and CMSAF provide instantaneous values, whereas values from ERA5 are + averages for the hour. Notes ----- From 73d046a1aeef901a3cbbec314f6f89dd3828a954 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Tue, 13 Jul 2021 14:30:56 -0400 Subject: [PATCH 49/51] Admonition update --- pvlib/iotools/pvgis.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 5aa0362f54..e256645c04 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -134,12 +134,12 @@ def get_pvgis_hourly(latitude, longitude, start=None, end=None, otherwise raise whatever ``HTTP/1.1`` error occurred .. Hint:: - PVGIS provides access to a number of different solar radiation - datasets, including satellite-based (SARAH, CMSAF, and NSRDB PSM3) and - re-analysis products (ERA5 and COSMO). Each data source has a - different geographical coverage and time stamp convention, e.g., SARAH - and CMSAF provide instantaneous values, whereas values from ERA5 are - averages for the hour. + PVGIS provides access to a number of different solar radiation + datasets, including satellite-based (SARAH, CMSAF, and NSRDB PSM3) and + re-analysis products (ERA5 and COSMO). Each data source has a + different geographical coverage and time stamp convention, e.g., SARAH + and CMSAF provide instantaneous values, whereas values from ERA5 are + averages for the hour. Notes ----- From 84449acf5c4ee030dabed2faed3b0d1b08820fd0 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Tue, 13 Jul 2021 15:08:11 -0400 Subject: [PATCH 50/51] Fix admonition --- pvlib/iotools/pvgis.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index e256645c04..00181b2d5d 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -133,13 +133,13 @@ def get_pvgis_hourly(latitude, longitude, start=None, end=None, the error message in the response will be raised as an exception, otherwise raise whatever ``HTTP/1.1`` error occurred - .. Hint:: - PVGIS provides access to a number of different solar radiation - datasets, including satellite-based (SARAH, CMSAF, and NSRDB PSM3) and - re-analysis products (ERA5 and COSMO). Each data source has a - different geographical coverage and time stamp convention, e.g., SARAH - and CMSAF provide instantaneous values, whereas values from ERA5 are - averages for the hour. + Hint + ---- + PVGIS provides access to a number of different solar radiation datasets, + including satellite-based (SARAH, CMSAF, and NSRDB PSM3) and re-analysis + products (ERA5 and COSMO). Each data source has a different geographical + coverage and time stamp convention, e.g., SARAH and CMSAF provide + instantaneous values, whereas values from ERA5 are averages for the hour. Notes ----- From ac19b9d0865b5078488a4a5f951988755587cbf8 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Wed, 14 Jul 2021 12:32:02 -0400 Subject: [PATCH 51/51] Refactor start input --- pvlib/iotools/pvgis.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 00181b2d5d..d43d4db87e 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -189,14 +189,12 @@ def get_pvgis_hourly(latitude, longitude, start=None, end=None, 'usehorizon': int(usehorizon), 'optimalangles': int(optimalangles), 'optimalinclination': int(optimalangles), 'loss': loss} - # pvgis only takes 0 for False, and 1 for True, not strings, also the + # pvgis only takes 0 for False, and 1 for True, not strings if userhorizon is not None: params['userhorizon'] = ','.join(str(x) for x in userhorizon) if raddatabase is not None: params['raddatabase'] = raddatabase - if (start is not None) & (type(start) is int): - params['startyear'] = start - elif start is not None: + if start is not None: params['startyear'] = start if isinstance(start, int) else start.year if end is not None: params['endyear'] = end if isinstance(end, int) else end.year