WIP gcsfs

bnaul · bnaul · commit 1240f1ea31e8 · 2018-04-17T18:42:30.000-07:00
diff --git a/ci/check_imports.py b/ci/check_imports.py
@@ -5,6 +5,7 @@
 
 blacklist = {
     'bs4',
+    'gcsfs',
     'html5lib',
     'ipython',
     'jinja2'
diff --git a/ci/requirements-2.7.run b/ci/requirements-2.7.run
@@ -18,3 +18,4 @@ patsy
 pymysql=0.6.3
 jinja2=2.8
 xarray=0.8.0
+gcsfs
diff --git a/ci/requirements-2.7_WIN.run b/ci/requirements-2.7_WIN.run
@@ -16,3 +16,4 @@ bottleneck
 html5lib
 beautifulsoup4
 jinja2=2.8
+gcsfs
diff --git a/ci/requirements-3.5.run b/ci/requirements-3.5.run
@@ -18,3 +18,4 @@ psycopg2
 s3fs
 beautifulsoup4
 ipython
+gcsfs
diff --git a/ci/requirements-3.5_OSX.run b/ci/requirements-3.5_OSX.run
@@ -14,3 +14,4 @@ bottleneck
 xarray
 s3fs
 beautifulsoup4
+gcsfs
diff --git a/ci/requirements-3.6.run b/ci/requirements-3.6.run
@@ -23,3 +23,4 @@ beautifulsoup4
 s3fs
 xarray
 ipython
+gcsfs
diff --git a/ci/requirements-3.6_LOCALE.run b/ci/requirements-3.6_LOCALE.run
@@ -20,3 +20,4 @@ beautifulsoup4
 s3fs
 xarray
 ipython
+gcsfs
diff --git a/ci/requirements-3.6_LOCALE_SLOW.run b/ci/requirements-3.6_LOCALE_SLOW.run
@@ -20,3 +20,4 @@ beautifulsoup4
 s3fs
 xarray
 ipython
+gcsfs
diff --git a/ci/requirements-3.6_SLOW.run b/ci/requirements-3.6_SLOW.run
@@ -17,3 +17,4 @@ psycopg2
 pymysql
 html5lib
 beautifulsoup4
+gcsfs
diff --git a/ci/requirements-optional-conda.txt b/ci/requirements-optional-conda.txt
@@ -3,6 +3,7 @@ blosc
 bottleneck
 fastparquet
 feather-format
+gcsfs
 html5lib
 ipython
 ipykernel
diff --git a/ci/requirements-optional-pip.txt b/ci/requirements-optional-pip.txt
@@ -3,6 +3,7 @@
 beautifulsoup4>=4.2.1
 blosc
 bottleneck
+gcsfs
 fastparquet
 feather-format
 html5lib
@@ -26,4 +27,4 @@ sqlalchemy
 xarray
 xlrd
 xlsxwriter
-xlwt
+xlwt
diff --git a/doc/source/install.rst b/doc/source/install.rst
@@ -275,6 +275,7 @@ Optional Dependencies
 
 * `Jinja2 <http://jinja.pocoo.org/>`__: Template engine for conditional HTML formatting.
 * `s3fs <http://s3fs.readthedocs.io/>`__: necessary for Amazon S3 access (s3fs >= 0.0.7).
+* `gcsfs <http://gcsfs.readthedocs.io/>`__: necessary for Google Cloud Storage access (gcsfs >= 0.6.0).
 * `blosc <https://pypi.python.org/pypi/blosc>`__: for msgpack compression using ``blosc``
 * One of
   `qtpy  <https://github.com/spyder-ide/qtpy>`__ (requires PyQt or PySide),
diff --git a/doc/source/whatsnew/v0.23.0.txt b/doc/source/whatsnew/v0.23.0.txt
@@ -441,6 +441,7 @@ Other Enhancements
 - Updated ``to_gbq`` and ``read_gbq`` signature and documentation to reflect changes from
   the Pandas-GBQ library version 0.4.0. Adds intersphinx mapping to Pandas-GBQ
   library. (:issue:`20564`)
+- Added support for reading from Google Cloud Storage via the ``gcsfs`` library (:issue:`19454`)
 
 .. _whatsnew_0230.api_breaking:
 
diff --git a/pandas/io/common.py b/pandas/io/common.py
@@ -169,6 +169,14 @@ def is_s3_url(url):
         return False
 
 
+def is_gcs_url(url):
+    """Check for a gcs url"""
+    try:
+        return parse_url(url).scheme in ['gcs', 'gs']
+    except:  # noqa
+        return False
+
+
 def get_filepath_or_buffer(filepath_or_buffer, encoding=None,
                            compression=None, mode=None):
     """
@@ -208,6 +216,13 @@ def get_filepath_or_buffer(filepath_or_buffer, encoding=None,
                                          compression=compression,
                                          mode=mode)
 
+    if is_gcs_url(filepath_or_buffer):
+        from pandas.io import gcs
+        return gcs.get_filepath_or_buffer(filepath_or_buffer,
+                                          encoding=encoding,
+                                          compression=compression,
+                                          mode=mode)
+
     if isinstance(filepath_or_buffer, (compat.string_types,
                                        compat.binary_type,
                                        mmap.mmap)):
diff --git a/pandas/io/excel.py b/pandas/io/excel.py
@@ -46,7 +46,7 @@
 io : string, path object (pathlib.Path or py._path.local.LocalPath),
     file-like object, pandas ExcelFile, or xlrd workbook.
     The string could be a URL. Valid URL schemes include http, ftp, s3,
-    and file. For file URLs, a host is expected. For instance, a local
+    gcs, and file. For file URLs, a host is expected. For instance, a local
     file could be file://localhost/path/to/workbook.xlsx
 sheet_name : string, int, mixed list of strings/ints, or None, default 0
 
diff --git a/pandas/io/gcs.py b/pandas/io/gcs.py
@@ -0,0 +1,26 @@
+""" GCS support for remote file interactivity """
+from pandas import compat
+try:
+    from google.auth.exceptions import GoogleAuthError
+    import gcsfs
+except ImportError:
+    raise ImportError("The gcsfs library is required to handle GCS files")
+
+
+def get_filepath_or_buffer(filepath_or_buffer, encoding=None,
+                           compression=None, mode=None):
+
+    if mode is None:
+        mode = 'rb'
+
+    gcsfs_logger_disabled = gcsfs.core.logger.disabled
+    gcsfs.core.logger.disabled = True
+    fs = gcsfs.GCSFileSystem()
+    try:
+        filepath_or_buffer = fs.open(filepath_or_buffer, mode)
+    except (compat.FileNotFoundError, GoogleAuthError, gcsfs.utils.HtmlError):
+        fs = gcsfs.GCSFileSystem(token='anon')
+        filepath_or_buffer = fs.open(filepath_or_buffer, mode)
+    finally:
+        gcsfs.core.logger.disabled = gcsfs_logger_disabled
+    return filepath_or_buffer, None, compression, True
diff --git a/pandas/io/json/json.py b/pandas/io/json/json.py
@@ -231,9 +231,9 @@ def read_json(path_or_buf=None, orient=None, typ='frame', dtype=True,
     Parameters
     ----------
     path_or_buf : a valid JSON string or file-like, default: None
-        The string could be a URL. Valid URL schemes include http, ftp, s3, and
-        file. For file URLs, a host is expected. For instance, a local file
-        could be ``file://localhost/path/to/table.json``
+        The string could be a URL. Valid URL schemes include http, ftp, s3,
+        gcs, and file. For file URLs, a host is expected. For instance, a local
+        file could be ``file://localhost/path/to/table.json``
 
     orient : string,
         Indication of expected JSON string format.
diff --git a/pandas/util/_print_versions.py b/pandas/util/_print_versions.py
@@ -96,6 +96,7 @@ def show_versions(as_json=False):
         ("fastparquet", lambda mod: mod.__version__),
         ("pandas_gbq", lambda mod: mod.__version__),
         ("pandas_datareader", lambda mod: mod.__version__),
+        ("gcsfs", lambda mod: mod.__version__),
     ]
 
     deps_blob = list()

-Original file line number
+Diff line change
 s3fs
 beautifulsoup4
 ipython
 +gcsfs