From d1d90fe9e18fb06fa42a775b6764e300a495d9b3 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 11 Aug 2021 13:59:47 -0500 Subject: [PATCH 01/11] TST: refactor iris table creation --- pandas/tests/io/test_sql.py | 118 +++++++++++++++++++++++------------- 1 file changed, 75 insertions(+), 43 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 1ce1bac3b2b7b..a5bb46acaa813 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -24,6 +24,7 @@ time, ) from io import StringIO +from pathlib import Path import sqlite3 import warnings @@ -72,34 +73,6 @@ SQLALCHEMY_INSTALLED = False SQL_STRINGS = { - "create_iris": { - "sqlite": """CREATE TABLE iris ( - "SepalLength" REAL, - "SepalWidth" REAL, - "PetalLength" REAL, - "PetalWidth" REAL, - "Name" TEXT - )""", - "mysql": """CREATE TABLE iris ( - `SepalLength` DOUBLE, - `SepalWidth` DOUBLE, - `PetalLength` DOUBLE, - `PetalWidth` DOUBLE, - `Name` VARCHAR(200) - )""", - "postgresql": """CREATE TABLE iris ( - "SepalLength" DOUBLE PRECISION, - "SepalWidth" DOUBLE PRECISION, - "PetalLength" DOUBLE PRECISION, - "PetalWidth" DOUBLE PRECISION, - "Name" VARCHAR(200) - )""", - }, - "insert_iris": { - "sqlite": """INSERT INTO iris VALUES(?, ?, ?, ?, ?)""", - "mysql": """INSERT INTO iris VALUES(%s, %s, %s, %s, "%s");""", - "postgresql": """INSERT INTO iris VALUES(%s, %s, %s, %s, %s);""", - }, "create_test_types": { "sqlite": """CREATE TABLE types_test_data ( "TextCol" TEXT, @@ -222,6 +195,74 @@ } +def iris_table_metadata(): + from sqlalchemy import ( + FLOAT, + Column, + MetaData, + String, + Table, + ) + + metadata = MetaData() + iris = Table( + "iris", + metadata, + Column("SepalLength", FLOAT), + Column("SepalWidth", FLOAT), + Column("PetalLength", FLOAT), + Column("PetalWidth", FLOAT), + Column("Name", String), + ) + return iris + + +def create_and_load_iris_sqlite3(conn: sqlite3.Connection, iris_file: Path): + cur = conn.cursor() + stmt = """CREATE TABLE iris ( + "SepalLength" REAL, + "SepalWidth" REAL, + "PetalLength" REAL, + "PetalWidth" REAL, + "Name" TEXT + )""" + cur.execute(stmt) + with iris_file.open(newline=None) as csvfile: + reader = csv.reader(csvfile) + next(reader) + cur = conn.cursor() + stmt = "INSERT INTO iris VALUES(?, ?, ?, ?, ?)" + for row in reader: + cur.execute(stmt, row) + + +def create_and_load_iris(conn, iris_file: Path): + from sqlalchemy import insert + from sqlalchemy.engine import Engine + + iris = iris_table_metadata() + iris.create(conn) + + with iris_file.open(newline=None) as csvfile: + reader = csv.reader(csvfile) + header = next(reader) + if isinstance(conn, Engine): + with conn.connect() as conn: + for row in reader: + params = {key: value for key, value in zip(header, row)} + conn.execute(insert(iris), params) + else: + for row in reader: + params = {key: value for key, value in zip(header, row)} + conn.execute(insert(iris), params) + + +@pytest.fixture +def iris_path(datapath): + iris_path = datapath("io", "data", "csv", "iris.csv") + return Path(iris_path) + + @pytest.fixture def test_frame1(): columns = ["index", "A", "B", "C", "D"] @@ -341,24 +382,15 @@ def _get_exec(self): else: return self.conn.cursor() - @pytest.fixture(params=[("io", "data", "csv", "iris.csv")]) - def load_iris_data(self, datapath, request): - - iris_csv_file = datapath(*request.param) - + @pytest.fixture + def load_iris_data(self, iris_path): if not hasattr(self, "conn"): self.setup_connect() - self.drop_table("iris") - self._get_exec().execute(SQL_STRINGS["create_iris"][self.flavor]) - - with open(iris_csv_file, newline=None) as iris_csv: - r = csv.reader(iris_csv) - next(r) # skip header row - ins = SQL_STRINGS["insert_iris"][self.flavor] - - for row in r: - self._get_exec().execute(ins, row) + if isinstance(self.conn, sqlite3.Connection): + create_and_load_iris_sqlite3(self.conn, iris_path) + else: + create_and_load_iris(self.conn, iris_path) def _load_iris_view(self): self.drop_table("iris_view") From 783d7fdbf79b5082794643ef90c7c6d8d38b0e0f Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 11 Aug 2021 14:27:20 -0500 Subject: [PATCH 02/11] debug --- pandas/tests/io/test_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index a5bb46acaa813..d981a4a4661b3 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -212,7 +212,7 @@ def iris_table_metadata(): Column("SepalWidth", FLOAT), Column("PetalLength", FLOAT), Column("PetalWidth", FLOAT), - Column("Name", String), + Column("Name", String(200)), ) return iris From 903c43a0483d1710e51a5ab7cfe24d3a1c40e6b8 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 11 Aug 2021 15:56:33 -0500 Subject: [PATCH 03/11] debug --- pandas/tests/io/test_sql.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index d981a4a4661b3..279f7cbc3b4b5 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -241,7 +241,7 @@ def create_and_load_iris(conn, iris_file: Path): from sqlalchemy.engine import Engine iris = iris_table_metadata() - iris.create(conn) + iris.create(bind=conn) with iris_file.open(newline=None) as csvfile: reader = csv.reader(csvfile) @@ -505,12 +505,14 @@ def _read_sql_iris_parameter(self): query = SQL_STRINGS["read_parameters"][self.flavor] params = ["Iris-setosa", 5.1] iris_frame = self.pandasSQL.read_query(query, params=params) + assert len(iris_frame) != 0 self._check_iris_loaded_frame(iris_frame) def _read_sql_iris_named_parameter(self): query = SQL_STRINGS["read_named_parameters"][self.flavor] params = {"name": "Iris-setosa", "length": 5.1} iris_frame = self.pandasSQL.read_query(query, params=params) + assert len(iris_frame) != 0 self._check_iris_loaded_frame(iris_frame) def _read_sql_iris_no_parameter_with_percent(self): From 2217c134db42b2ad31ddc295ed2c4a0c31a77adb Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Thu, 12 Aug 2021 09:34:20 -0500 Subject: [PATCH 04/11] debug --- pandas/tests/io/test_sql.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 279f7cbc3b4b5..50b9e41c31f4f 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -230,10 +230,8 @@ def create_and_load_iris_sqlite3(conn: sqlite3.Connection, iris_file: Path): with iris_file.open(newline=None) as csvfile: reader = csv.reader(csvfile) next(reader) - cur = conn.cursor() stmt = "INSERT INTO iris VALUES(?, ?, ?, ?, ?)" - for row in reader: - cur.execute(stmt, row) + cur.executemany(stmt, reader) def create_and_load_iris(conn, iris_file: Path): @@ -250,11 +248,11 @@ def create_and_load_iris(conn, iris_file: Path): with conn.connect() as conn: for row in reader: params = {key: value for key, value in zip(header, row)} - conn.execute(insert(iris), params) + conn.execute(insert(iris, values=params)) else: for row in reader: params = {key: value for key, value in zip(header, row)} - conn.execute(insert(iris), params) + conn.execute(insert(iris, values=params)) @pytest.fixture From a7dfd629b7c891b84051d23579c957c2b1d78a8d Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Thu, 12 Aug 2021 10:04:16 -0500 Subject: [PATCH 05/11] debug --- pandas/tests/io/test_sql.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 50b9e41c31f4f..19fbbefcd00bd 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -244,15 +244,13 @@ def create_and_load_iris(conn, iris_file: Path): with iris_file.open(newline=None) as csvfile: reader = csv.reader(csvfile) header = next(reader) + params = [{key: value for key, value in zip(header, row)} for row in reader] + stmt = insert(iris).values(params) if isinstance(conn, Engine): with conn.connect() as conn: - for row in reader: - params = {key: value for key, value in zip(header, row)} - conn.execute(insert(iris, values=params)) + conn.execute(stmt) else: - for row in reader: - params = {key: value for key, value in zip(header, row)} - conn.execute(insert(iris, values=params)) + conn.execute(stmt) @pytest.fixture From af8356a99b526dfa6572894e4dc1bbdf3c5c39f3 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Thu, 12 Aug 2021 20:22:33 -0500 Subject: [PATCH 06/11] debug --- pandas/tests/io/test_sql.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 19fbbefcd00bd..05c2a678ec3b0 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -165,7 +165,7 @@ }, "read_parameters": { "sqlite": "SELECT * FROM iris WHERE Name=? AND SepalLength=?", - "mysql": 'SELECT * FROM iris WHERE `Name`="%s" AND `SepalLength`=%s', + "mysql": "SELECT * FROM iris WHERE `Name`=%s AND `SepalLength`=%s", "postgresql": 'SELECT * FROM iris WHERE "Name"=%s AND "SepalLength"=%s', }, "read_named_parameters": { @@ -174,7 +174,7 @@ """, "mysql": """ SELECT * FROM iris WHERE - `Name`="%(name)s" AND `SepalLength`=%(length)s + `Name`=%(name)s AND `SepalLength`=%(length)s """, "postgresql": """ SELECT * FROM iris WHERE @@ -197,7 +197,7 @@ def iris_table_metadata(): from sqlalchemy import ( - FLOAT, + REAL, Column, MetaData, String, @@ -208,10 +208,10 @@ def iris_table_metadata(): iris = Table( "iris", metadata, - Column("SepalLength", FLOAT), - Column("SepalWidth", FLOAT), - Column("PetalLength", FLOAT), - Column("PetalWidth", FLOAT), + Column("SepalLength", REAL), + Column("SepalWidth", REAL), + Column("PetalLength", REAL), + Column("PetalWidth", REAL), Column("Name", String(200)), ) return iris @@ -239,6 +239,7 @@ def create_and_load_iris(conn, iris_file: Path): from sqlalchemy.engine import Engine iris = iris_table_metadata() + iris.drop(conn, checkfirst=True) iris.create(bind=conn) with iris_file.open(newline=None) as csvfile: From e872e18be1172c6447580beddc5f60d2ac17db29 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Thu, 12 Aug 2021 20:46:32 -0500 Subject: [PATCH 07/11] debug --- pandas/tests/io/test_sql.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 05c2a678ec3b0..d5235161f1788 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -197,8 +197,8 @@ def iris_table_metadata(): from sqlalchemy import ( - REAL, Column, + Float, MetaData, String, Table, @@ -208,10 +208,10 @@ def iris_table_metadata(): iris = Table( "iris", metadata, - Column("SepalLength", REAL), - Column("SepalWidth", REAL), - Column("PetalLength", REAL), - Column("PetalWidth", REAL), + Column("SepalLength", Float), + Column("SepalWidth", Float), + Column("PetalLength", Float), + Column("PetalWidth", Float), Column("Name", String(200)), ) return iris From b7e7a5247102205f09bbfa202baeb70361fb0d8e Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Thu, 12 Aug 2021 21:30:05 -0500 Subject: [PATCH 08/11] debug dtype --- pandas/tests/io/test_sql.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index d5235161f1788..a354e22424e60 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -30,6 +30,7 @@ import numpy as np import pytest +from sqlalchemy.sql.sqltypes import FLOAT from pandas.core.dtypes.common import ( is_datetime64_dtype, @@ -195,8 +196,9 @@ } -def iris_table_metadata(): +def iris_table_metadata(dialect: str): from sqlalchemy import ( + REAL, Column, Float, MetaData, @@ -204,14 +206,15 @@ def iris_table_metadata(): Table, ) + dtype = REAL if dialect == "mysql" else Float metadata = MetaData() iris = Table( "iris", metadata, - Column("SepalLength", Float), - Column("SepalWidth", Float), - Column("PetalLength", Float), - Column("PetalWidth", Float), + Column("SepalLength", dtype), + Column("SepalWidth", dtype), + Column("PetalLength", dtype), + Column("PetalWidth", dtype), Column("Name", String(200)), ) return iris @@ -234,11 +237,11 @@ def create_and_load_iris_sqlite3(conn: sqlite3.Connection, iris_file: Path): cur.executemany(stmt, reader) -def create_and_load_iris(conn, iris_file: Path): +def create_and_load_iris(conn, iris_file: Path, dialect: str): from sqlalchemy import insert from sqlalchemy.engine import Engine - iris = iris_table_metadata() + iris = iris_table_metadata(dialect) iris.drop(conn, checkfirst=True) iris.create(bind=conn) @@ -387,7 +390,7 @@ def load_iris_data(self, iris_path): if isinstance(self.conn, sqlite3.Connection): create_and_load_iris_sqlite3(self.conn, iris_path) else: - create_and_load_iris(self.conn, iris_path) + create_and_load_iris(self.conn, iris_path, self.flavor) def _load_iris_view(self): self.drop_table("iris_view") From 6b1e76db82e4ce5a8900c4194280c70c5fcaa1af Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Thu, 12 Aug 2021 21:31:26 -0500 Subject: [PATCH 09/11] debug dtype --- pandas/tests/io/test_sql.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index a354e22424e60..c275ff26e9737 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -30,7 +30,6 @@ import numpy as np import pytest -from sqlalchemy.sql.sqltypes import FLOAT from pandas.core.dtypes.common import ( is_datetime64_dtype, From ab6d1f7e450b8a1e5a39cb62cc8cdf6b987f91de Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Thu, 12 Aug 2021 22:08:08 -0500 Subject: [PATCH 10/11] remove debug info --- pandas/tests/io/test_sql.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index c275ff26e9737..9ee2dfed0fc34 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -504,14 +504,12 @@ def _read_sql_iris_parameter(self): query = SQL_STRINGS["read_parameters"][self.flavor] params = ["Iris-setosa", 5.1] iris_frame = self.pandasSQL.read_query(query, params=params) - assert len(iris_frame) != 0 self._check_iris_loaded_frame(iris_frame) def _read_sql_iris_named_parameter(self): query = SQL_STRINGS["read_named_parameters"][self.flavor] params = {"name": "Iris-setosa", "length": 5.1} iris_frame = self.pandasSQL.read_query(query, params=params) - assert len(iris_frame) != 0 self._check_iris_loaded_frame(iris_frame) def _read_sql_iris_no_parameter_with_percent(self): From 85c6934122612466727212a4e22e896b8f064bc2 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Thu, 12 Aug 2021 23:08:49 -0500 Subject: [PATCH 11/11] move table creation --- pandas/tests/io/test_sql.py | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 9ee2dfed0fc34..d9747c525771b 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -205,7 +205,7 @@ def iris_table_metadata(dialect: str): Table, ) - dtype = REAL if dialect == "mysql" else Float + dtype = Float if dialect == "postgresql" else REAL metadata = MetaData() iris = Table( "iris", @@ -1279,21 +1279,6 @@ def test_database_uri_string(self, test_frame1): with pytest.raises(ImportError, match="pg8000"): sql.read_sql("select * from table", db_uri) - def _make_iris_table_metadata(self): - sa = sqlalchemy - metadata = sa.MetaData() - iris = sa.Table( - "iris", - metadata, - sa.Column("SepalLength", sa.REAL), - sa.Column("SepalWidth", sa.REAL), - sa.Column("PetalLength", sa.REAL), - sa.Column("PetalWidth", sa.REAL), - sa.Column("Name", sa.TEXT), - ) - - return iris - def test_query_by_text_obj(self): # WIP : GH10846 name_text = sqlalchemy.text("select * from iris where name=:name") @@ -1303,7 +1288,7 @@ def test_query_by_text_obj(self): def test_query_by_select_obj(self): # WIP : GH10846 - iris = self._make_iris_table_metadata() + iris = iris_table_metadata(self.flavor) name_select = sqlalchemy.select([iris]).where( iris.c.Name == sqlalchemy.bindparam("name")