Optimized generation so table filters are applied before processing c…

…olumns.
JordanMarr · Aug 22, 2023 · a57c476 · a57c476
1 parent 0b51c9f
commit a57c476
Show file tree

Hide file tree

Showing 6 changed files with 81 additions and 72 deletions.
diff --git a/src/SqlHydra.Cli/Npgsql/NpgsqlSchemaProvider.fs b/src/SqlHydra.Cli/Npgsql/NpgsqlSchemaProvider.fs
@@ -104,10 +104,10 @@ let getSchema (cfg: Config) : Schema =
         |> Seq.cast<DataRow>
         |> Seq.map (fun tbl -> 
             {| 
-                TableCatalog = tbl["TABLE_CATALOG"] :?> string
-                TableSchema = tbl["TABLE_SCHEMA"] :?> string
-                TableName  = tbl["TABLE_NAME"] :?> string
-                TableType = "view"
+                Catalog = tbl["TABLE_CATALOG"] :?> string
+                Schema = tbl["TABLE_SCHEMA"] :?> string
+                Name  = tbl["TABLE_NAME"] :?> string
+                Type = "view"
             |}
         )
 
@@ -116,21 +116,22 @@ let getSchema (cfg: Config) : Schema =
         |> Seq.cast<DataRow>
         |> Seq.map (fun tbl -> 
             {| 
-                TableCatalog = tbl["TABLE_CATALOG"] :?> string
-                TableSchema = tbl["TABLE_SCHEMA"] :?> string
-                TableName  = tbl["TABLE_NAME"] :?> string
-                TableType = tbl["TABLE_TYPE"] :?> string 
+                Catalog = tbl["TABLE_CATALOG"] :?> string
+                Schema = tbl["TABLE_SCHEMA"] :?> string
+                Name  = tbl["TABLE_NAME"] :?> string
+                Type = tbl["TABLE_TYPE"] :?> string 
             |}
         )
-        |> Seq.filter (fun tbl -> tbl.TableType <> "SYSTEM_TABLE")
+        |> Seq.filter (fun tbl -> tbl.Type <> "SYSTEM_TABLE")
         |> Seq.append views
+        |> SchemaFilters.filterTables cfg.Filters
         |> Seq.map (fun tbl -> 
             let tableColumns = 
                 allColumns
                 |> Seq.filter (fun col -> 
-                    col.TableCatalog = tbl.TableCatalog && 
-                    col.TableSchema = tbl.TableSchema &&
-                    col.TableName = tbl.TableName
+                    col.TableCatalog = tbl.Catalog && 
+                    col.TableSchema = tbl.Schema &&
+                    col.TableName = tbl.Name
                 )
 
             let mappedColumns = 
@@ -185,13 +186,14 @@ let getSchema (cfg: Config) : Schema =
 
             let filteredColumns = 
                 supportedColumns
-                |> SchemaFilters.filterColumns cfg.Filters tbl.TableSchema tbl.TableName
+                |> SchemaFilters.filterColumns cfg.Filters tbl.Schema tbl.Name
+                |> Seq.toList
 
             { 
-                Table.Catalog = tbl.TableCatalog
-                Table.Schema = tbl.TableSchema
-                Table.Name =  tbl.TableName
-                Table.Type = if tbl.TableType = "table" then TableType.Table else TableType.View
+                Table.Catalog = tbl.Catalog
+                Table.Schema = tbl.Schema
+                Table.Name =  tbl.Name
+                Table.Type = if tbl.Type = "table" then TableType.Table else TableType.View
                 Table.Columns = filteredColumns
                 Table.TotalColumns = tableColumns |> Seq.length
             }

diff --git a/src/SqlHydra.Cli/Oracle/OracleSchemaProvider.fs b/src/SqlHydra.Cli/Oracle/OracleSchemaProvider.fs
@@ -71,10 +71,10 @@ let getSchema (cfg: Config) : Schema =
         |> Seq.filter (fun view -> not (systemOwners.Contains(view.["OWNER"] :?> string)))
         |> Seq.map (fun view -> 
             {| 
-                TableCatalog = view.["OWNER"] :?> string
-                TableSchema = view.["OWNER"] :?> string
-                TableName  = view.["VIEW_NAME"] :?> string
-                TableType = "view"
+                Catalog = view.["OWNER"] :?> string
+                Schema = view.["OWNER"] :?> string
+                Name  = view.["VIEW_NAME"] :?> string
+                Type = "view"
             |}
         )
         |> Seq.toList
@@ -84,25 +84,26 @@ let getSchema (cfg: Config) : Schema =
         |> Seq.cast<DataRow>
         |> Seq.map (fun tbl -> 
             {| 
-                TableCatalog = tbl.["OWNER"] :?> string
-                TableSchema = tbl.["OWNER"] :?> string
-                TableName  = tbl.["TABLE_NAME"] :?> string
-                TableType = tbl.["TYPE"] :?> string // [ "view"; "User"; "System" ]
+                Catalog = tbl.["OWNER"] :?> string
+                Schema = tbl.["OWNER"] :?> string
+                Name  = tbl.["TABLE_NAME"] :?> string
+                Type = tbl.["TYPE"] :?> string // [ "view"; "User"; "System" ]
             |}
         )
-        |> Seq.filter (fun tbl -> System.String.Compare(tbl.TableType, "System", true) <> 0) // Exclude system
+        |> Seq.filter (fun tbl -> System.String.Compare(tbl.Type, "System", true) <> 0) // Exclude system
         |> Seq.append views
+        |> SchemaFilters.filterTables cfg.Filters
         |> Seq.map (fun tbl -> 
             let tableColumns = 
                 columns
                 |> Seq.filter (fun col -> 
-                    col.TableCatalog = tbl.TableCatalog && 
-                    col.TableSchema = tbl.TableSchema &&
-                    col.TableName = tbl.TableName
-                )
-
+                    col.TableCatalog = tbl.Catalog && 
+                    col.TableSchema = tbl.Schema &&
+                    col.TableName = tbl.Name
+                )                
+                
             let supportedColumns = 
-                tableColumns
+                tableColumns                
                 |> Seq.choose (fun col -> 
                     OracleDataTypes.tryFindTypeMapping (col.ProviderTypeName, col.Precision, col.Scale)
                     |> Option.map (fun typeMapping ->
@@ -114,17 +115,17 @@ let getSchema (cfg: Config) : Schema =
                         }
                     )
                 )
-                |> Seq.toList
 
-            let filteredColumns = 
+            let filteredColumns =
                 supportedColumns
-                |> SchemaFilters.filterColumns cfg.Filters tbl.TableSchema tbl.TableName
+                |> SchemaFilters.filterColumns cfg.Filters tbl.Schema tbl.Name
+                |> Seq.toList
 
             { 
-                Table.Catalog = tbl.TableCatalog
-                Table.Schema = tbl.TableSchema
-                Table.Name =  tbl.TableName
-                Table.Type = if System.String.Compare(tbl.TableType, "view", true) = 0 then TableType.View else TableType.Table
+                Table.Catalog = tbl.Catalog
+                Table.Schema = tbl.Schema
+                Table.Name =  tbl.Name
+                Table.Type = if System.String.Compare(tbl.Type, "view", true) = 0 then TableType.View else TableType.Table
                 Table.Columns = filteredColumns
                 Table.TotalColumns = tableColumns |> Seq.length
             }

diff --git a/src/SqlHydra.Cli/SchemaFilters.fs b/src/SqlHydra.Cli/SchemaFilters.fs
@@ -3,7 +3,7 @@ module SqlHydra.SchemaFilters
 open GlobExpressions
 open SqlHydra.Domain
 
-let filterTables (filters: FilterPatterns) (tables: Table list) = 
+let inline filterTables (filters: FilterPatterns) (tables: 'Table seq when 'Table : (member Schema: string) and 'Table : (member Name: string)) = 
     let isTableFilter (filter: string) = not (filter.Contains ".")
     let includeFilters = filters.Includes |> List.filter isTableFilter
     let excludeFilters = filters.Excludes |> List.filter isTableFilter
@@ -12,8 +12,8 @@ let filterTables (filters: FilterPatterns) (tables: Table list) =
     | [], [] -> 
         tables
     | _ -> 
-        let getPath (tbl: Table) = $"{tbl.Schema}/{tbl.Name}"
-        let tablesByPath = tables |> List.map (fun t -> getPath t, t) |> Map.ofList
+        let getPath (tbl: 'Table) = $"{tbl.Schema}/{tbl.Name}"
+        let tablesByPath = tables |> Seq.map (fun t -> getPath t, t) |> Map.ofSeq
         let tablePaths = tablesByPath |> Map.toList |> List.map fst
 
         let getMatchingTablePaths = 
@@ -29,7 +29,7 @@ let filterTables (filters: FilterPatterns) (tables: Table list) =
         let filteredTables = filteredPaths |> Seq.map (fun path -> tablesByPath.[path]) |> Seq.toList
         filteredTables
 
-let filterColumns (filters: FilterPatterns) (schema: string) (table: string) (columns: Column list) = 
+let inline filterColumns (filters: FilterPatterns) (schema: string) (table: string) (columns: Column seq when 'Column : (member Name: string)) = 
     let isColumnFilter (filter: string) = filter.Contains "."
     let includeFilters = filters.Includes |> List.filter isColumnFilter
     let excludeFilters = filters.Excludes |> List.filter isColumnFilter
@@ -38,8 +38,8 @@ let filterColumns (filters: FilterPatterns) (schema: string) (table: string) (co
     | [], [] -> 
         columns
     | _ -> 
-        let getPath (col: Column) = $"{schema}/{table}.{col.Name}"
-        let columnsByPath = columns |> List.map (fun c -> getPath c, c) |> Map.ofList
+        let getPath (col: 'Column) = $"{schema}/{table}.{col.Name}"
+        let columnsByPath = columns |> Seq.map (fun c -> getPath c, c) |> Map.ofSeq
         let columnPaths = columnsByPath |> Map.toList |> List.map fst
 
         let getMatchingColumnPaths = 

diff --git a/src/SqlHydra.Cli/SchemaGenerator.fs b/src/SqlHydra.Cli/SchemaGenerator.fs
@@ -667,7 +667,6 @@ let createHydraReaderClass (db: Schema) (rdrCfg: ReadersConfig) (app: AppInfo) (
 let generateModule (cfg: Config) (app: AppInfo) (db: Schema) = 
     let filteredTables = 
         db.Tables 
-        |> filterTables cfg.Filters
         |> List.sortBy (fun tbl -> tbl.Schema, tbl.Name)
 
     let schemas = 

diff --git a/src/SqlHydra.Cli/SqlServer/SqlServerSchemaProvider.fs b/src/SqlHydra.Cli/SqlServer/SqlServerSchemaProvider.fs
@@ -58,17 +58,21 @@ let getSchema (cfg: Config) : Schema =
         sTables.Rows
         |> Seq.cast<DataRow>
         |> Seq.map (fun tbl -> 
-            let tableCatalog = tbl.["TABLE_CATALOG"] :?> string
-            let tableSchema = tbl.["TABLE_SCHEMA"] :?> string
-            let tableName  = tbl.["TABLE_NAME"] :?> string
-            let tableType = tbl.["TABLE_TYPE"] :?> string
-
+            {| 
+                Catalog = tbl.["TABLE_CATALOG"] :?> string
+                Schema = tbl.["TABLE_SCHEMA"] :?> string
+                Name  = tbl.["TABLE_NAME"] :?> string
+                Type = tbl.["TABLE_TYPE"] :?> string 
+            |}
+        )
+        |> SchemaFilters.filterTables cfg.Filters
+        |> Seq.map (fun tbl ->             
             let tableColumns = 
                 allColumns
                 |> Seq.filter (fun col -> 
-                    col.TableCatalog = tableCatalog && 
-                    col.TableSchema = tableSchema &&
-                    col.TableName = tableName
+                    col.TableCatalog = tbl.Catalog && 
+                    col.TableSchema = tbl.Schema &&
+                    col.TableName = tbl.Name
                 )
 
             let supportedColumns = 
@@ -88,13 +92,14 @@ let getSchema (cfg: Config) : Schema =
 
             let filteredColumns = 
                 supportedColumns
-                |> SchemaFilters.filterColumns cfg.Filters tableSchema tableName
+                |> SchemaFilters.filterColumns cfg.Filters tbl.Schema tbl.Name
+                |> Seq.toList
 
             { 
-                Table.Catalog = tableCatalog
-                Table.Schema = tableSchema
-                Table.Name =  tableName
-                Table.Type = if tableType = "BASE TABLE" then TableType.Table else TableType.View
+                Table.Catalog = tbl.Catalog
+                Table.Schema = tbl.Schema
+                Table.Name =  tbl.Name
+                Table.Type = if tbl.Type = "BASE TABLE" then TableType.Table else TableType.View
                 Table.Columns = filteredColumns
                 Table.TotalColumns = tableColumns |> Seq.length
             }

diff --git a/src/SqlHydra.Cli/Sqlite/SqliteSchemaProvider.fs b/src/SqlHydra.Cli/Sqlite/SqliteSchemaProvider.fs
@@ -42,20 +42,21 @@ let getSchema (cfg: Config) : Schema =
         |> Seq.cast<DataRow>
         |> Seq.map (fun tbl -> 
             {| 
-                TableCatalog = tbl.["TABLE_CATALOG"] :?> string
-                TableSchema = tbl.["TABLE_SCHEMA"] |> dbNullOpt<string> |> Option.defaultValue defaultSchema
-                TableName  = tbl.["TABLE_NAME"] :?> string
-                TableType = tbl.["TABLE_TYPE"] :?> string 
+                Catalog = tbl.["TABLE_CATALOG"] :?> string
+                Schema = tbl.["TABLE_SCHEMA"] |> dbNullOpt<string> |> Option.defaultValue defaultSchema
+                Name  = tbl.["TABLE_NAME"] :?> string
+                Type = tbl.["TABLE_TYPE"] :?> string 
             |}
         )
-        |> Seq.filter (fun tbl -> tbl.TableType <> "SYSTEM_TABLE")
+        |> Seq.filter (fun tbl -> tbl.Type <> "SYSTEM_TABLE")
+        |> SchemaFilters.filterTables cfg.Filters
         |> Seq.map (fun tbl -> 
             let tableColumns = 
                 allColumns
                 |> Seq.filter (fun col -> 
-                    col.TableCatalog = tbl.TableCatalog && 
-                    col.TableSchema = tbl.TableSchema &&
-                    col.TableName = tbl.TableName
+                    col.TableCatalog = tbl.Catalog && 
+                    col.TableSchema = tbl.Schema &&
+                    col.TableName = tbl.Name
                 )
 
             let supportedColumns = 
@@ -75,13 +76,14 @@ let getSchema (cfg: Config) : Schema =
 
             let filteredColumns = 
                 supportedColumns
-                |> SchemaFilters.filterColumns cfg.Filters tbl.TableSchema tbl.TableName
+                |> SchemaFilters.filterColumns cfg.Filters tbl.Schema tbl.Name
+                |> Seq.toList
 
             { 
-                Table.Catalog = tbl.TableCatalog
-                Table.Schema = tbl.TableSchema
-                Table.Name =  tbl.TableName
-                Table.Type = if tbl.TableType = "table" then TableType.Table else TableType.View
+                Table.Catalog = tbl.Catalog
+                Table.Schema = tbl.Schema
+                Table.Name =  tbl.Name
+                Table.Type = if tbl.Type = "table" then TableType.Table else TableType.View
                 Table.Columns = filteredColumns
                 Table.TotalColumns = tableColumns |> Seq.length
             }