Skip to content

Added tableTypes argument to readAllSqlTables #1270

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Added tableTypes argument to readAllSqlTables #1270

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
24ba113
Removed types = ["TABLE"] argument from readAllSqlTables
Jolanrensen Jun 23, 2025
99ebe3d
added tableTypes parameter to readAllSqlTables instead, with a helpfu…
Jolanrensen Jun 23, 2025
97db01f
added BASE TABLE as default tableType too
Jolanrensen Jun 23, 2025
dc029b5
adjusted getSchemaForAllSqlTables too
Jolanrensen Jun 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions dataframe-jdbc/api/dataframe-jdbc.api
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,10 @@ public final class org/jetbrains/kotlinx/dataframe/io/ReadJdbcKt {
public static final fun getDataFrameSchema (Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema;
public static synthetic fun getDataFrameSchema$default (Ljava/sql/Connection;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema;
public static synthetic fun getDataFrameSchema$default (Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema;
public static final fun getSchemaForAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Ljava/util/Map;
public static final fun getSchemaForAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Ljava/util/Map;
public static synthetic fun getSchemaForAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Ljava/util/Map;
public static synthetic fun getSchemaForAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Ljava/util/Map;
public static final fun getSchemaForAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;Ljava/util/List;)Ljava/util/Map;
public static final fun getSchemaForAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;Ljava/util/List;)Ljava/util/Map;
public static synthetic fun getSchemaForAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;Ljava/util/List;ILjava/lang/Object;)Ljava/util/Map;
public static synthetic fun getSchemaForAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;Ljava/util/List;ILjava/lang/Object;)Ljava/util/Map;
public static final fun getSchemaForResultSet (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/ResultSet;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema;
public static final fun getSchemaForSqlQuery (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema;
public static final fun getSchemaForSqlQuery (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema;
Expand All @@ -50,10 +50,10 @@ public final class org/jetbrains/kotlinx/dataframe/io/ReadJdbcKt {
public static final fun getSchemaForSqlTable (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema;
public static synthetic fun getSchemaForSqlTable$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema;
public static synthetic fun getSchemaForSqlTable$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema;
public static final fun readAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Ljava/util/Map;
public static final fun readAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Ljava/util/Map;
public static synthetic fun readAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Ljava/util/Map;
public static synthetic fun readAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;ILjava/lang/Object;)Ljava/util/Map;
public static final fun readAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;Ljava/util/List;)Ljava/util/Map;
public static final fun readAllSqlTables (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;Ljava/util/List;)Ljava/util/Map;
public static synthetic fun readAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/sql/Connection;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;Ljava/util/List;ILjava/lang/Object;)Ljava/util/Map;
public static synthetic fun readAllSqlTables$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;Ljava/util/List;ILjava/lang/Object;)Ljava/util/Map;
public static final fun readDataFrame (Ljava/sql/Connection;Ljava/lang/String;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;Z)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public static final fun readDataFrame (Ljava/sql/ResultSet;Ljava/sql/Connection;IZLorg/jetbrains/kotlinx/dataframe/io/db/DbType;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public static final fun readDataFrame (Ljava/sql/ResultSet;Lorg/jetbrains/kotlinx/dataframe/io/db/DbType;IZ)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
Expand Down
67 changes: 60 additions & 7 deletions dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -604,6 +604,8 @@ public fun ResultSet.readDataFrame(
* @param [inferNullability] indicates how the column nullability should be inferred.
* @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`,
* in that case the [dbType] will be recognized from the [dbConfig].
* @param [tableTypes] an optional list of table types, which must be from the list of table types
* returned from [DatabaseMetaData.getTableTypes]; `null` returns all types. By default, it's `["TABLE", "BASE TABLE"]`.
* @return a map of [String] to [AnyFrame] objects representing the non-system tables from the database.
*/
public fun DataFrame.Companion.readAllSqlTables(
Expand All @@ -612,9 +614,10 @@ public fun DataFrame.Companion.readAllSqlTables(
limit: Int = DEFAULT_LIMIT,
inferNullability: Boolean = true,
dbType: DbType? = null,
tableTypes: List<String>? = listOf("TABLE", "BASE TABLE"),
): Map<String, AnyFrame> {
DriverManager.getConnection(dbConfig.url, dbConfig.user, dbConfig.password).use { connection ->
return readAllSqlTables(connection, catalogue, limit, inferNullability, dbType)
return readAllSqlTables(connection, catalogue, limit, inferNullability, dbType, tableTypes)
}
}

Expand All @@ -628,6 +631,8 @@ public fun DataFrame.Companion.readAllSqlTables(
* @param [inferNullability] indicates how the column nullability should be inferred.
* @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`,
* in that case the [dbType] will be recognized from the [connection].
* @param [tableTypes] an optional list of table types, which must be from the list of table types
* returned from [DatabaseMetaData.getTableTypes]; `null` returns all types. By default, it's `["TABLE", "BASE TABLE"]`.
* @return a map of [String] to [AnyFrame] objects representing the non-system tables from the database.
*
* @see DriverManager.getConnection
Expand All @@ -638,12 +643,13 @@ public fun DataFrame.Companion.readAllSqlTables(
limit: Int = DEFAULT_LIMIT,
inferNullability: Boolean = true,
dbType: DbType? = null,
tableTypes: List<String>? = listOf("TABLE", "BASE TABLE"),
): Map<String, AnyFrame> {
val metaData = connection.metaData
val determinedDbType = dbType ?: extractDBTypeFromConnection(connection)

// exclude a system and other tables without data, but it looks like it is supported badly for many databases
val tables = metaData.getTables(catalogue, null, null, arrayOf("TABLE"))
// exclude system- and other tables without data, but it looks like it is supported badly for many databases
val tables = metaData.getTables(catalogue, null, null, tableTypes?.toTypedArray())

val dataFrames = mutableMapOf<String, AnyFrame>()

Expand All @@ -668,6 +674,27 @@ public fun DataFrame.Companion.readAllSqlTables(
}
}

// We may have no tables or filtered for the wrong table type, let's give a helpful error message
if (dataFrames.isEmpty()) {
val supportedTableTypes = metaData.tableTypes.let {
buildList { while (it.next()) add(it.getString("TABLE_TYPE")) }
}
if (tableTypes?.any { it !in supportedTableTypes } == true) {
val unsupportedTypes = tableTypes.filter { it !in supportedTableTypes }
throw IllegalArgumentException(
buildString {
appendLine("Found no tables with type(s) $tableTypes.")
if (unsupportedTypes.isNotEmpty()) {
appendLine(
"Table type(s) $unsupportedTypes are unsupported for ${determinedDbType::class.simpleName}.",
)
}
appendLine("If this is unexpected, try adjusting `tableTypes=` to any of $supportedTableTypes.")
},
)
}
}

return dataFrames
}

Expand Down Expand Up @@ -835,14 +862,17 @@ public fun ResultSet.getDataFrameSchema(dbType: DbType): DataFrameSchema = DataF
* @param [dbConfig] the database configuration to connect to the database, including URL, user, and password.
* @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`,
* in that case the [dbType] will be recognized from the [dbConfig].
* @param [tableTypes] an optional list of table types, which must be from the list of table types
* returned from [DatabaseMetaData.getTableTypes]; `null` returns all types. By default, it's `["TABLE", "BASE TABLE"]`.
* @return a map of [String, DataFrameSchema] objects representing the table name and its schema for each non-system table.
*/
public fun DataFrame.Companion.getSchemaForAllSqlTables(
dbConfig: DbConnectionConfig,
dbType: DbType? = null,
tableTypes: List<String>? = listOf("TABLE", "BASE TABLE"),
): Map<String, DataFrameSchema> {
DriverManager.getConnection(dbConfig.url, dbConfig.user, dbConfig.password).use { connection ->
return getSchemaForAllSqlTables(connection, dbType)
return getSchemaForAllSqlTables(connection, dbType, tableTypes)
}
}

Expand All @@ -852,18 +882,20 @@ public fun DataFrame.Companion.getSchemaForAllSqlTables(
* @param [connection] the database connection.
* @param [dbType] the type of database, could be a custom object, provided by user, optional, default is `null`,
* in that case the [dbType] will be recognized from the [connection].
* @param [tableTypes] an optional list of table types, which must be from the list of table types
* returned from [DatabaseMetaData.getTableTypes]; `null` returns all types. By default, it's `["TABLE", "BASE TABLE"]`.
* @return a map of [String, DataFrameSchema] objects representing the table name and its schema for each non-system table.
*/
public fun DataFrame.Companion.getSchemaForAllSqlTables(
connection: Connection,
dbType: DbType? = null,
tableTypes: List<String>? = listOf("TABLE", "BASE TABLE"),
): Map<String, DataFrameSchema> {
val metaData = connection.metaData
val determinedDbType = dbType ?: extractDBTypeFromConnection(connection)

val tableTypes = arrayOf("TABLE")
// exclude a system and other tables without data
val tables = metaData.getTables(null, null, null, tableTypes)
// exclude system- and other tables without data
val tables = metaData.getTables(null, null, null, tableTypes?.toTypedArray())

val dataFrameSchemas = mutableMapOf<String, DataFrameSchema>()

Expand All @@ -877,6 +909,27 @@ public fun DataFrame.Companion.getSchemaForAllSqlTables(
}
}

// We may have no tables or filtered for the wrong table type, let's give a helpful error message
if (dataFrameSchemas.isEmpty()) {
val supportedTableTypes = metaData.tableTypes.let {
buildList { while (it.next()) add(it.getString("TABLE_TYPE")) }
}
if (tableTypes?.any { it !in supportedTableTypes } == true) {
val unsupportedTypes = tableTypes.filter { it !in supportedTableTypes }
throw IllegalArgumentException(
buildString {
appendLine("Found no tables with type(s) $tableTypes.")
if (unsupportedTypes.isNotEmpty()) {
appendLine(
"Table type(s) $unsupportedTypes are unsupported for ${determinedDbType::class.simpleName}.",
)
}
appendLine("If this is unexpected, try adjusting `tableTypes=` to any of $supportedTableTypes.")
},
)
}
}

return dataFrameSchemas
}

Expand Down