From c92e6047bde01bcb846ad1fcf4126d3001e30f4b Mon Sep 17 00:00:00 2001 From: palaviv Date: Tue, 28 Feb 2017 22:36:42 +0200 Subject: [PATCH 1/8] Add optional use of sqlite3_open_v2, sqlite3_close_v2 and sqlite3_prepare_v2 --- Modules/_sqlite/connection.c | 15 ++++++++++----- Modules/_sqlite/cursor.c | 2 +- Modules/_sqlite/statement.c | 4 ++-- Modules/_sqlite/util.h | 12 ++++++++++++ configure.ac | 8 ++++++++ pyconfig.h.in | 9 +++++++++ 6 files changed, 42 insertions(+), 8 deletions(-) diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 37b45f330b3493..5129c72729c39c 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -118,7 +118,12 @@ int pysqlite_connection_init(pysqlite_Connection* self, PyObject* args, PyObject return -1; } Py_BEGIN_ALLOW_THREADS +#ifdef HAVE_SQLITE3_OPEN_V2 + rc = sqlite3_open_v2(database, &self->db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL); +#else rc = sqlite3_open(database, &self->db); +#endif #endif Py_END_ALLOW_THREADS @@ -241,7 +246,7 @@ void pysqlite_connection_dealloc(pysqlite_Connection* self) /* Clean up if user has not called .close() explicitly. */ if (self->db) { Py_BEGIN_ALLOW_THREADS - sqlite3_close(self->db); + SQLITE3_CLOSE(self->db); Py_END_ALLOW_THREADS } @@ -334,7 +339,7 @@ PyObject* pysqlite_connection_close(pysqlite_Connection* self, PyObject* args) if (self->db) { Py_BEGIN_ALLOW_THREADS - rc = sqlite3_close(self->db); + rc = SQLITE3_CLOSE(self->db); Py_END_ALLOW_THREADS if (rc != SQLITE_OK) { @@ -375,7 +380,7 @@ PyObject* _pysqlite_connection_begin(pysqlite_Connection* self) sqlite3_stmt* statement; Py_BEGIN_ALLOW_THREADS - rc = sqlite3_prepare(self->db, self->begin_statement, -1, &statement, &tail); + rc = SQLITE3_PREPARE(self->db, self->begin_statement, -1, &statement, &tail); Py_END_ALLOW_THREADS if (rc != SQLITE_OK) { @@ -417,7 +422,7 @@ PyObject* pysqlite_connection_commit(pysqlite_Connection* self, PyObject* args) if (!sqlite3_get_autocommit(self->db)) { Py_BEGIN_ALLOW_THREADS - rc = sqlite3_prepare(self->db, "COMMIT", -1, &statement, &tail); + rc = SQLITE3_PREPARE(self->db, "COMMIT", -1, &statement, &tail); Py_END_ALLOW_THREADS if (rc != SQLITE_OK) { _pysqlite_seterror(self->db, NULL); @@ -460,7 +465,7 @@ PyObject* pysqlite_connection_rollback(pysqlite_Connection* self, PyObject* args pysqlite_do_all_statements(self, ACTION_RESET, 1); Py_BEGIN_ALLOW_THREADS - rc = sqlite3_prepare(self->db, "ROLLBACK", -1, &statement, &tail); + rc = SQLITE3_PREPARE(self->db, "ROLLBACK", -1, &statement, &tail); Py_END_ALLOW_THREADS if (rc != SQLITE_OK) { _pysqlite_seterror(self->db, NULL); diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index 8341fb8480172c..212c5a7f8b1670 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -706,7 +706,7 @@ PyObject* pysqlite_cursor_executescript(pysqlite_Cursor* self, PyObject* args) while (1) { Py_BEGIN_ALLOW_THREADS - rc = sqlite3_prepare(self->connection->db, + rc = SQLITE3_PREPARE(self->connection->db, script_cstr, -1, &statement, diff --git a/Modules/_sqlite/statement.c b/Modules/_sqlite/statement.c index 27b2654b103419..bc0d9401d0fe34 100644 --- a/Modules/_sqlite/statement.c +++ b/Modules/_sqlite/statement.c @@ -93,7 +93,7 @@ int pysqlite_statement_create(pysqlite_Statement* self, pysqlite_Connection* con } Py_BEGIN_ALLOW_THREADS - rc = sqlite3_prepare(connection->db, + rc = SQLITE3_PREPARE(connection->db, sql_cstr, -1, &self->st, @@ -334,7 +334,7 @@ int pysqlite_statement_recompile(pysqlite_Statement* self, PyObject* params) } Py_BEGIN_ALLOW_THREADS - rc = sqlite3_prepare(self->db, + rc = SQLITE3_PREPARE(self->db, sql_cstr, -1, &new_st, diff --git a/Modules/_sqlite/util.h b/Modules/_sqlite/util.h index 88ea90689d518c..35d82cb5d6c7f8 100644 --- a/Modules/_sqlite/util.h +++ b/Modules/_sqlite/util.h @@ -39,4 +39,16 @@ int _pysqlite_seterror(sqlite3* db, sqlite3_stmt* st); PyObject * _pysqlite_long_from_int64(sqlite_int64 value); sqlite_int64 _pysqlite_long_as_int64(PyObject * value); +#ifdef HAVE_SQLITE3_PREPARE_V2 +#define SQLITE3_PREPARE sqlite3_prepare_v2 +#else +#define SQLITE3_PREPARE sqlite3_prepare +#endif + +#ifdef HAVE_SQLITE3_CLOSE_V2 +#define SQLITE3_CLOSE sqlite3_close_v2 +#else +#define SQLITE3_CLOSE sqlite3_close +#endif + #endif diff --git a/configure.ac b/configure.ac index 5610c5759d601a..853e6eea22903a 100644 --- a/configure.ac +++ b/configure.ac @@ -2854,6 +2854,14 @@ AC_ARG_ENABLE(loadable-sqlite-extensions, AC_MSG_RESULT($enable_loadable_sqlite_extensions) +# Check for support of sqlite3 _v2 interface +AC_CHECK_LIB(sqlite3, sqlite3_prepare_v2, + AC_DEFINE(HAVE_SQLITE3_PREPARE_V2, 1, [Define if the sqlite3 library has sqlite3_prepare_v2])) +AC_CHECK_LIB(sqlite3, sqlite3_open_v2, + AC_DEFINE(HAVE_SQLITE3_OPEN_V2, 1, [Define if the sqlite3 library has sqlite3_open_v2])) +AC_CHECK_LIB(sqlite3, sqlite3_close_v2, + AC_DEFINE(HAVE_SQLITE3_CLOSE_V2, 1, [Define if the sqlite3 library has sqlite3_close_v2])) + # Check for --with-tcltk-includes=path and --with-tcltk-libs=path AC_SUBST(TCLTK_INCLUDES) AC_SUBST(TCLTK_LIBS) diff --git a/pyconfig.h.in b/pyconfig.h.in index 21354a5cb84fe5..c7bf19ee49e6e3 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -898,6 +898,15 @@ /* Define to 1 if you have the header file. */ #undef HAVE_SPAWN_H +/* Define if the sqlite3 library has sqlite3_close_v2 */ +#undef HAVE_SQLITE3_CLOSE_V2 + +/* Define if the sqlite3 library has sqlite3_open_v2 */ +#undef HAVE_SQLITE3_OPEN_V2 + +/* Define if the sqlite3 library has sqlite3_prepare_v2 */ +#undef HAVE_SQLITE3_PREPARE_V2 + /* Define if your compiler provides ssize_t */ #undef HAVE_SSIZE_T From 8eccb8460532f097cdf1d4ae2ba58307165f1621 Mon Sep 17 00:00:00 2001 From: palaviv Date: Wed, 1 Mar 2017 21:09:01 +0200 Subject: [PATCH 2/8] Use SQLITE_VERSION_NUMBER --- Modules/_sqlite/connection.c | 19 +++++++++---------- Modules/_sqlite/util.h | 4 ++-- configure.ac | 8 -------- pyconfig.h.in | 9 --------- 4 files changed, 11 insertions(+), 29 deletions(-) diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 5129c72729c39c..22e006c43c6f7a 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -84,6 +84,7 @@ int pysqlite_connection_init(pysqlite_Connection* self, PyObject* args, PyObject int uri = 0; double timeout = 5.0; int rc; + int flags = 0; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s|diOiOip", kwlist, &database, &timeout, &detect_types, @@ -107,23 +108,21 @@ int pysqlite_connection_init(pysqlite_Connection* self, PyObject* args, PyObject Py_INCREF(&PyUnicode_Type); self->text_factory = (PyObject*)&PyUnicode_Type; -#ifdef SQLITE_OPEN_URI - Py_BEGIN_ALLOW_THREADS - rc = sqlite3_open_v2(database, &self->db, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | - (uri ? SQLITE_OPEN_URI : 0), NULL); -#else +#if SQLITE_VERSION_NUMBER >= 3005000 +#ifndef SQLITE_OPEN_URI if (uri) { PyErr_SetString(pysqlite_NotSupportedError, "URIs not supported"); return -1; } +#else + flags |= (uri ? SQLITE_OPEN_URI : 0); +#endif + flags |= SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; Py_BEGIN_ALLOW_THREADS -#ifdef HAVE_SQLITE3_OPEN_V2 - rc = sqlite3_open_v2(database, &self->db, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL); + rc = sqlite3_open_v2(database, &self->db, flags, NULL); #else + Py_BEGIN_ALLOW_THREADS rc = sqlite3_open(database, &self->db); -#endif #endif Py_END_ALLOW_THREADS diff --git a/Modules/_sqlite/util.h b/Modules/_sqlite/util.h index 35d82cb5d6c7f8..9106fcaf54c6b3 100644 --- a/Modules/_sqlite/util.h +++ b/Modules/_sqlite/util.h @@ -39,13 +39,13 @@ int _pysqlite_seterror(sqlite3* db, sqlite3_stmt* st); PyObject * _pysqlite_long_from_int64(sqlite_int64 value); sqlite_int64 _pysqlite_long_as_int64(PyObject * value); -#ifdef HAVE_SQLITE3_PREPARE_V2 +#if SQLITE_VERSION_NUMBER >= 3003009 #define SQLITE3_PREPARE sqlite3_prepare_v2 #else #define SQLITE3_PREPARE sqlite3_prepare #endif -#ifdef HAVE_SQLITE3_CLOSE_V2 +#if SQLITE_VERSION_NUMBER >= 3007014 #define SQLITE3_CLOSE sqlite3_close_v2 #else #define SQLITE3_CLOSE sqlite3_close diff --git a/configure.ac b/configure.ac index 853e6eea22903a..5610c5759d601a 100644 --- a/configure.ac +++ b/configure.ac @@ -2854,14 +2854,6 @@ AC_ARG_ENABLE(loadable-sqlite-extensions, AC_MSG_RESULT($enable_loadable_sqlite_extensions) -# Check for support of sqlite3 _v2 interface -AC_CHECK_LIB(sqlite3, sqlite3_prepare_v2, - AC_DEFINE(HAVE_SQLITE3_PREPARE_V2, 1, [Define if the sqlite3 library has sqlite3_prepare_v2])) -AC_CHECK_LIB(sqlite3, sqlite3_open_v2, - AC_DEFINE(HAVE_SQLITE3_OPEN_V2, 1, [Define if the sqlite3 library has sqlite3_open_v2])) -AC_CHECK_LIB(sqlite3, sqlite3_close_v2, - AC_DEFINE(HAVE_SQLITE3_CLOSE_V2, 1, [Define if the sqlite3 library has sqlite3_close_v2])) - # Check for --with-tcltk-includes=path and --with-tcltk-libs=path AC_SUBST(TCLTK_INCLUDES) AC_SUBST(TCLTK_LIBS) diff --git a/pyconfig.h.in b/pyconfig.h.in index c7bf19ee49e6e3..21354a5cb84fe5 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -898,15 +898,6 @@ /* Define to 1 if you have the header file. */ #undef HAVE_SPAWN_H -/* Define if the sqlite3 library has sqlite3_close_v2 */ -#undef HAVE_SQLITE3_CLOSE_V2 - -/* Define if the sqlite3 library has sqlite3_open_v2 */ -#undef HAVE_SQLITE3_OPEN_V2 - -/* Define if the sqlite3 library has sqlite3_prepare_v2 */ -#undef HAVE_SQLITE3_PREPARE_V2 - /* Define if your compiler provides ssize_t */ #undef HAVE_SSIZE_T From 53607957972adb504efb80abda699e4ef8314814 Mon Sep 17 00:00:00 2001 From: palaviv Date: Wed, 1 Mar 2017 21:25:20 +0200 Subject: [PATCH 3/8] Don't call sqlite3_reset in _pysqlite_seterror in using sqlite3_prepare_v2 --- Modules/_sqlite/util.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Modules/_sqlite/util.c b/Modules/_sqlite/util.c index 351b1b47a44d43..20791430b6e55a 100644 --- a/Modules/_sqlite/util.c +++ b/Modules/_sqlite/util.c @@ -49,10 +49,14 @@ int _pysqlite_seterror(sqlite3* db, sqlite3_stmt* st) { int errorcode; - /* SQLite often doesn't report anything useful, unless you reset the statement first */ +#if SQLITE_VERSION_NUMBER >= 3003009 + /* SQLite often doesn't report anything useful, unless you reset the statement first. + When using sqlite3_prepare_v2 this is not needed. */ if (st != NULL) { (void)sqlite3_reset(st); } +#endif + errorcode = sqlite3_errcode(db); From 9185cb9f100ee2bb3487b68371c7124a222f4375 Mon Sep 17 00:00:00 2001 From: palaviv Date: Wed, 1 Mar 2017 21:38:21 +0200 Subject: [PATCH 4/8] Revert sqlite3_open change --- Modules/_sqlite/connection.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 22e006c43c6f7a..1ff1688b697300 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -84,7 +84,6 @@ int pysqlite_connection_init(pysqlite_Connection* self, PyObject* args, PyObject int uri = 0; double timeout = 5.0; int rc; - int flags = 0; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s|diOiOip", kwlist, &database, &timeout, &detect_types, @@ -108,20 +107,19 @@ int pysqlite_connection_init(pysqlite_Connection* self, PyObject* args, PyObject Py_INCREF(&PyUnicode_Type); self->text_factory = (PyObject*)&PyUnicode_Type; -#if SQLITE_VERSION_NUMBER >= 3005000 -#ifndef SQLITE_OPEN_URI +#ifdef SQLITE_OPEN_URI + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_open_v2(database, &self->db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | + (uri ? SQLITE_OPEN_URI : 0), NULL); +#else if (uri) { PyErr_SetString(pysqlite_NotSupportedError, "URIs not supported"); return -1; } -#else - flags |= (uri ? SQLITE_OPEN_URI : 0); -#endif - flags |= SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; - Py_BEGIN_ALLOW_THREADS - rc = sqlite3_open_v2(database, &self->db, flags, NULL); -#else Py_BEGIN_ALLOW_THREADS + /* No need to use sqlite3_open_v2 as sqlite3_open(filename, db) is the + same as sqlite3_open_v2(filename, db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL)*/ rc = sqlite3_open(database, &self->db); #endif Py_END_ALLOW_THREADS From f7dcd02dd48f16a15e1f8c093650e69a1aef1d26 Mon Sep 17 00:00:00 2001 From: palaviv Date: Thu, 2 Mar 2017 20:44:52 +0200 Subject: [PATCH 5/8] Don't reset the statment if using sqlite3_prepare_v2 --- Modules/_sqlite/cursor.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index 212c5a7f8b1670..ba6e52db568367 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -548,8 +548,10 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* /* If it worked, let's get out of the loop */ break; } +#if SQLITE_VERSION_NUMBER < 3003009 /* Something went wrong. Re-set the statement and try again. */ rc = pysqlite_statement_reset(self->statement); +#endif if (rc == SQLITE_SCHEMA) { /* If this was a result of the schema changing, let's try again. */ From d4d9c4e76a96f7cd59708382af5d73b763ca2cd8 Mon Sep 17 00:00:00 2001 From: palaviv Date: Thu, 2 Mar 2017 20:45:45 +0200 Subject: [PATCH 6/8] CR fixes --- Modules/_sqlite/connection.c | 2 +- Modules/_sqlite/util.c | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 1ff1688b697300..774399933ee73a 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -119,7 +119,7 @@ int pysqlite_connection_init(pysqlite_Connection* self, PyObject* args, PyObject } Py_BEGIN_ALLOW_THREADS /* No need to use sqlite3_open_v2 as sqlite3_open(filename, db) is the - same as sqlite3_open_v2(filename, db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL)*/ + same as sqlite3_open_v2(filename, db, SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, NULL). */ rc = sqlite3_open(database, &self->db); #endif Py_END_ALLOW_THREADS diff --git a/Modules/_sqlite/util.c b/Modules/_sqlite/util.c index 20791430b6e55a..79fa7f9dc0bde0 100644 --- a/Modules/_sqlite/util.c +++ b/Modules/_sqlite/util.c @@ -57,7 +57,6 @@ int _pysqlite_seterror(sqlite3* db, sqlite3_stmt* st) } #endif - errorcode = sqlite3_errcode(db); switch (errorcode) From 64c3bb744716bed0e16a771ff6a64edda45bead2 Mon Sep 17 00:00:00 2001 From: palaviv Date: Thu, 2 Mar 2017 20:50:36 +0200 Subject: [PATCH 7/8] Add NEWS entry --- Misc/NEWS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 4f19e75aeaa8fc..020dec0f4a4b0a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -249,6 +249,8 @@ Extension Modules Library ------- +- bpo-9303: Migrate sqlite3 module to _v2 API. Patch by Aviv Palivoda. + - bpo-7769: Method register_function() of xmlrpc.server.SimpleXMLRPCDispatcher and its subclasses can now be used as a decorator. From f1ddabac25d978b1d082b413d1602a821501698e Mon Sep 17 00:00:00 2001 From: palaviv Date: Fri, 3 Mar 2017 10:11:43 +0200 Subject: [PATCH 8/8] Fix ifdef in _pysqlite_seterror --- Modules/_sqlite/util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_sqlite/util.c b/Modules/_sqlite/util.c index 79fa7f9dc0bde0..b371aed99dbe4d 100644 --- a/Modules/_sqlite/util.c +++ b/Modules/_sqlite/util.c @@ -49,7 +49,7 @@ int _pysqlite_seterror(sqlite3* db, sqlite3_stmt* st) { int errorcode; -#if SQLITE_VERSION_NUMBER >= 3003009 +#if SQLITE_VERSION_NUMBER < 3003009 /* SQLite often doesn't report anything useful, unless you reset the statement first. When using sqlite3_prepare_v2 this is not needed. */ if (st != NULL) {