diff --git a/latest/examples/generated/UserGuide/getting_started/index.html b/latest/examples/generated/UserGuide/getting_started/index.html index 51c29fd..6961633 100644 --- a/latest/examples/generated/UserGuide/getting_started/index.html +++ b/latest/examples/generated/UserGuide/getting_started/index.html @@ -708,6 +708,8 @@

Minimizing Compute Costsend
+

Tip: Setting t(table) = from_query(table) will save some keystrokes. This means after saving the results of db_table you can start all chains/refer to the data with `t(table)`` –-

+

This page was generated using Literate.jl.

diff --git a/latest/examples/generated/UserGuide/ibis_comp/index.html b/latest/examples/generated/UserGuide/ibis_comp/index.html index 316d669..d652664 100644 --- a/latest/examples/generated/UserGuide/ibis_comp/index.html +++ b/latest/examples/generated/UserGuide/ibis_comp/index.html @@ -778,7 +778,7 @@

Loading Data

Previewing the data¤

-

TidierDB and Ibis use head/@head to preview the first rows of a dataset.

+

TidierDB and Ibis use head/@head to preview the first rows of a dataset. Ibis

mtcars.head(6)
 
┏━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┳━━━━━━━┳━━━━━━━┳━━━━━━━┳━━━━━━━┓
@@ -794,6 +794,7 @@ 
Previewing the data
@chain t(mtcars) @head(6) @collect
 
6×12 DataFrame
@@ -810,7 +811,7 @@ 
Previewing the dataFiltering¤

-
The example below demonstrates how to filter using multiple criteria in both Ibis and TidierData

+
The example below demonstrates how to filter using multiple criteria in both Ibis and TidierData Ibis

 
mtcars.filter(((_.mpg > 22) & (_.drat > 4) | (_.hp == 113)))
 

 
┏━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┳━━━━━━━┳━━━━━━━┳━━━━━━━┳━━━━━━━┓
@@ -846,7 +847,7 @@ 
FilteringCreating new columns¤

-
Both TidierDB and Ibis use mutate/@mutate to add new columns

+
Both TidierDB and Ibis use mutate/@mutate to add new columns Ibis

 
(
    mtcars
         .mutate(kpg = _.mpg * 1.61)
@@ -871,6 +872,7 @@ 
Creating new columns
@chain t(mtcars) begin
        @mutate(kpg = mpg * 1.61)
        @select(model, kpg)
@@ -899,7 +901,7 @@ 
Creating new columnsSorting columns¤

-
Ibis uses order_by similar to SQLs ORDER BY

+
Ibis uses order_by similar to SQLs ORDER BY Ibis

 
mtcars.order_by(_.mpg)
 

 
┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┳━━━━━━━┳━━━━━━━┳━━━━━━━┳━━━━━━━┓
@@ -920,7 +922,7 @@ 
Sorting columns
@chain t(mtcars) @arrange(mpg) @collect
 

 
32×12 DataFrame
@@ -945,7 +947,7 @@ 
Sorting columnsSelecting columns¤

-
In Ibis, columns must be prefixed with the table name, or in this case _, or they can be given as a string. Finally to using helper functions like startswith requires importing selectors as above.

+
In Ibis, columns must be prefixed with the table name, or in this case _, or they can be given as a string. Finally to using helper functions like startswith requires importing selectors as above. Ibis

 
mtcars.select(s.startswith("m"), "drat", _.wt)
 

 
┏━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓
@@ -966,11 +968,10 @@ 
Selecting columns
@chain t(mtcars) @select(starts_with("m"), "drat", wt) @collect
 

-
32×2 DataFrame
-32×4 DataFrame
+
32×4 DataFrame
  Row │ model              mpg       drat      wt
      │ String?            Float64?  Float64?  Float64?
 ─────┼─────────────────────────────────────────────────
@@ -992,7 +993,7 @@ 
Selecting columnsMulti step queries and summarizing¤

-
Aggregating data is done with aggregate in ibis and @summarize in TidierDB. There is a slight difference in grouping data. Ibis uses by = within the aggregate call vs TidierDB adheres to @group_by convention

+
Aggregating data is done with aggregate in Ibis and @summarize in TidierDB. To group data, Ibis uses by = within the aggregate call vs TidierDB adheres to @group_by convention Ibis

 
mtcars.aggregate(
     total_hp=_.hp.sum(),
     avg_hp=_.hp.mean(),
@@ -1009,7 +1010,7 @@ 
Multi step queries and summarizing

-
In TidierDB, @filter will automatically determine whether the criteria belong in a WHERE or HAVING in SQL clause.

+
In TidierDB, @filter will automatically determine whether the criteria belong in a WHERE or HAVING SQL clause. TidierDB

 
@chain t(mtcars) begin
     @group_by(cyl)
     @summarize(total_hp = sum(hp),
@@ -1028,7 +1029,7 @@ 
Multi step queries and summarizing

 

 
Renaming columns¤

-
Both tools use rename/@rename to rename columns

+
Both tools use rename/@rename to rename columns Ibis

 
mtcars.rename(make_model = "model").select(_.make_model)
 

 
┏━━━━━━━━━━━━━━━━━━━┓
@@ -1049,6 +1050,7 @@ 
Renaming columns
@chain t(mtcars) @rename(model_make = model) @select(model_make) @collect
 

 
32×1 DataFrame
diff --git a/latest/reference/index.html b/latest/reference/index.html
index 050c192..16704c9 100644
--- a/latest/reference/index.html
+++ b/latest/reference/index.html
@@ -721,7 +721,7 @@ 
Reference - Exported functionsjulia> db = connect(duckdb())
 DuckDB.Connection(":memory:")
 

-
source

+
source

 
#
 TidierDB.copy_to — Method.

 
   copy_to(conn, df_or_path, "name")
@@ -739,7 +739,7 @@ 
Reference - Exported functionsjulia> copy_to(db, df, "test");
 

-
source

+
source

 
#
 TidierDB.db_table — Function.

 
db_table(database, table_name, athena_params, delta = false, iceberg = false)
@@ -779,7 +779,7 @@ 
Reference - Exported functions   3 │ value    BIGINT               1  df_mem
    4 │ percent  DOUBLE               1  df_mem, false, DuckDB.Connection(":memory:"), TidierDB.CTE[], 0, nothing)
 

-
source

+
source

 
#
 TidierDB.@anti_join — Macro.

 
@anti_join(sql_query, join_table, new_table_col, orignal_table_col)
@@ -822,7 +822,7 @@ 
Reference - Exported functions   4 │ AH       aa            3       0.8
    5 │ AJ       aa            5       1.0
 

-
source

+
source

 
#
 TidierDB.@arrange — Macro.

 
@arrange(sql_query, columns...)
@@ -862,7 +862,7 @@ 
Reference - Exported functions   9 │ AJ       aa            5       1.0
   10 │ AE       bb            5       0.5
 

-
source

+
source

 
#
 TidierDB.@collect — Macro.

 
@collect(sql_query, stream = false)
@@ -899,7 +899,7 @@ 
Reference - Exported functions   9 │ AI       bb            4       0.9
   10 │ AJ       aa            5       1.0
 

-
source

+
source

 
#
 TidierDB.@count — Macro.

 
@count(sql_query, columns...)
@@ -932,7 +932,7 @@ 
Reference - Exported functions   1 │ aa            5
    2 │ bb            5
 

-
source

+
source

 
#
 TidierDB.@distinct — Macro.

 
@distinct(sql_query, columns...)
@@ -985,7 +985,7 @@ 
Reference - Exported functions   9 │ AI       bb            4       0.9
   10 │ AJ       aa            5       1.0
 

-
source

+
source

 
#
 TidierDB.@filter — Macro.

 
@filter(sql_query, conditions...)
@@ -1041,7 +1041,7 @@ 
Reference - Exported functions   1 │ aa            0.6
    2 │ bb            0.5
 

-
source

+
source

 
#
 TidierDB.@full_join — Macro.

 
@inner_join(sql_query, join_table, new_table_col, orignal_table_col)
@@ -1091,7 +1091,7 @@ 
Reference - Exported functions  11 │ missing  missing  missing  missing    AK       Y              68
   12 │ missing  missing  missing  missing    AM       X              74
 

-
source

+
source

 
#
 TidierDB.@group_by — Macro.

 
@group_by(sql_query, columns...)
@@ -1124,7 +1124,7 @@ 
Reference - Exported functions   1 │ aa
    2 │ bb
 

-
source

+
source

 
#
 TidierDB.@head — Macro.

 
@head(sql_query, value)
@@ -1155,7 +1155,7 @@ 
Reference - Exported functions─────┼────────────────────────────────────
    1 │ AA       bb            1       0.1
 

-
source

+
source

 
#
 TidierDB.@inner_join — Macro.

 
@inner_join(sql_query, join_table, new_table_col, orignal_table_col)
@@ -1198,7 +1198,7 @@ 
Reference - Exported functions   4 │ AG       bb            2       0.7  AG       Y             83
    5 │ AI       bb            4       0.9  AI       X             95
 

-
source

+
source

 
#
 TidierDB.@interpolate — Macro.

 
@interpolate(args...)
@@ -1246,7 +1246,7 @@ 
Reference - Exported functions─────┼───────────────────────────
    1 │ AA            1       0.1
 

-
source

+
source

 
#
 TidierDB.@left_join — Macro.

 
@left_join(sql_query, join_table, new_table_col, orignal_table_col)
@@ -1294,7 +1294,7 @@ 
Reference - Exported functions   9 │ AH       aa            3       0.8  missing  missing   missing 
   10 │ AJ       aa            5       1.0  missing  missing   missing 
 

-
source

+
source

 
#
 TidierDB.@mutate — Macro.

 
@mutate(sql_query, exprs...)
@@ -1334,7 +1334,7 @@ 
Reference - Exported functions   9 │ AI       bb           16       0.9      0.81
   10 │ AJ       aa           20       1.0      1.0
 

-
source

+
source

 
#
 TidierDB.@rename — Macro.

 
@rename(sql_query, renamings...)
@@ -1371,7 +1371,7 @@ 
Reference - Exported functions   9 │ AI       bb            4       0.9
   10 │ AJ       aa            5       1.0
 

-
source

+
source

 
#
 TidierDB.@right_join — Macro.

 
@right_join(sql_query, join_table, new_table_col, orignal_table_col)
@@ -1416,7 +1416,7 @@ 
Reference - Exported functions   6 │ missing  missing  missing  missing    AK       Y             68
    7 │ missing  missing  missing  missing    AM       X             74
 

-
source

+
source

 
#
 TidierDB.@select — Macro.

 
@select(sql_query, columns)
@@ -1475,7 +1475,7 @@ 
Reference - Exported functions   9 │      4       0.9
   10 │      5       1.0
 

-
source

+
source

 
#
 TidierDB.@semi_join — Macro.

 
@semi_join(sql_query, join_table, new_table_col, orignal_table_col)
@@ -1518,7 +1518,7 @@ 
Reference - Exported functions   4 │ AG       bb            2       0.7
    5 │ AI       bb            4       0.9
 

-
source

+
source

 
#
 TidierDB.@slice_max — Macro.

 
@slice_max(sql_query, column, n = 1)
@@ -1557,7 +1557,7 @@ 
Reference - Exported functions   1 │ AE       bb            5       0.5         1
    2 │ AJ       aa            5       1.0         1
 

-
source

+
source

 
#
 TidierDB.@slice_min — Macro.

 
@slice_min(sql_query, column, n = 1)
@@ -1596,7 +1596,7 @@ 
Reference - Exported functions   1 │ AA       bb            1       0.1         1
    2 │ AF       aa            1       0.6         1
 

-
source

+
source

 
#
 TidierDB.@slice_sample — Macro.

 
@slice_sample(sql_query, n)
@@ -1628,7 +1628,7 @@ 
Reference - Exported functions       @collect
        end;
 

-
source

+
source

 
#
 TidierDB.@summarise — Macro.

 
   @summarise(sql_query, exprs...)
@@ -1675,7 +1675,7 @@ 
Reference - Exported functions   1 │ aa            3.0       5
    2 │ bb            2.5       5
 

-
source

+
source

 
#
 TidierDB.@summarize — Macro.

 
   @summarize(sql_query, exprs...)
@@ -1722,7 +1722,7 @@ 
Reference - Exported functions   1 │ aa            3.0       5
    2 │ bb            2.5       5
 

-
source

+
source

 
#
 TidierDB.@window_frame — Macro.

 
@window_frame(sql_query, frame_start::Int, frame_end::Int)
@@ -1744,7 +1744,7 @@ 
Reference - Exported functionsjulia> copy_to(db, df, "df_mem");
 

-
source

+
source

 
#
 TidierDB.@window_order — Macro.

 
   @window_order(sql_query, columns...)
@@ -1765,7 +1765,7 @@ 
Reference - Exported functionsjulia> copy_to(db, df, "df_mem");
 

-
source

+
source

 

 

 
Reference - Internal functions¤

diff --git a/latest/search/search_index.json b/latest/search/search_index.json
index 839c22b..91a9ec7 100644
--- a/latest/search/search_index.json
+++ b/latest/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Home","text":""},{"location":"#what-is-tidierdbjl","title":"What is TidierDB.jl?","text":"
TiderDB.jl is a 100% Julia implementation of the dbplyr R package, and similar to Python's ibis package.
 
The main goal of TidierDB.jl is to bring the syntax of Tidier.jl to multiple SQL backends, making it possible to analyze data directly on databases without needing to copy the entire database into memory.
 
 
"},{"location":"#currently-supported-backends-include","title":"Currently supported backends include:","text":"
     
  • DuckDB (the default) duckdb()
    •  
  • ClickHouse clickhouse()
    •  
  • SQLite sqlite()
    •  
  • MySQL and MariaDB mysql()
    •  
  • MSSQL mssql()
    •  
  • Postgres postgres()
    •  
  • Athena athena()
    •  
  • Snowflake snowflake()
    •  
  • Google Big Query gbq()
    •  
  • Oracle oracle()
    •  
  • Databricks databricks()
    •  
 
Change the backend using set_sql_mode() - for example  - set_sql_mode(databricks())
 
 
"},{"location":"#installation","title":"Installation","text":"
For the stable version:
 
] add TidierDB\n
 
TidierDB.jl currently supports the following top-level macros:
 
     
  • @arrange
    •  
  • @group_by
    •  
  • @filter
    •  
  • @select
    •  
  • @mutate, which supports across()
    •  
  • @summarize and @summarise, which supports across()
    •  
  • @distinct
    •  
  • @left_join, @right_join, @inner_join, @anti_join, @full_join, and @semi_join (slight syntax differences from TidierData.jl)
    •  
  • @count
    •  
  • @slice_min, @slice_max, @slice_sample
    •  
  • @window_order and window_frame
    •  
  • @show_query
    •  
  • @collect
    •  
 
Supported helper functions for most backends include:
 
     
  • across()
    •  
  • desc()
    •  
  • if_else() and case_when()
    •  
  • n()
    •  
  • starts_with(), ends_with(), and contains()
    •  
  • as_float(), as_integer(), and as_string()
    •  
  • is_missing()
    •  
  • missing_if() and replace_missing()
    •  
 
From TidierStrings.jl:
 
     
  • str_detect, str_replace, str_replace_all, str_remove_all, str_remove
    •  
 
From TidierDates.jl:
 
     
  • year, month, day, hour, min, second, floor_date, difftime
    •  
 
Supported aggregate functions (as supported by the backend) with more to come
 
     
  • mean, minimium, maximum, std, sum, cumsum, cor, cov, var
    •  
  • @summarize supports any SQL aggregate function in addition to the list above. Simply write the function as written in SQL syntax and it will work
    •  
 
When using the DuckDB backend, if db_table recieves a file path ( .parquet, .json, .csv, iceberg or delta), it does not copy it into memory. This allows for queries on files too big for memory. db_table also supports S3 bucket locations via DuckDB.
 
 
"},{"location":"#what-is-the-recommended-way-to-use-tidierdb","title":"What is the recommended way to use TidierDB?","text":"
Typically, you will want to use TidierDB alongside TidierData because there are certain functionality (such as pivoting) which are only supported in TidierData and can only be performed on data frames.
 
Our recommended path for using TidierDB is to import the package so that there are no namespace conflicts with TidierData. Once TidierDB is integrated with Tidier, then Tidier will automatically load the packages in this fashion.
 
First, let's develop and execute a query using TidierDB. Notice that all top-level macros and functions originating from TidierDB start with a DB prefix. Any functions defined within macros do not need to be prefixed within DB because they are actually pseudofunctions that are in actuality converted into SQL code.
 
Even though the code reads similarly to TidierData, note that no computational work actually occurs until you run DB.@collect(), which runs the SQL query and instantiates the result as a DataFrame.
 
using TidierData\nimport TidierDB as DB\n\ndb = DB.connect(DB.duckdb());\npath_or_name = \"https://gist.githubusercontent.com/seankross/a412dfbd88b3db70b74b/raw/5f23f993cd87c283ce766e7ac6b329ee7cc2e1d1/mtcars.csv\"\n\n@chain DB.db_table(db, path_or_name) begin\n    DB.@filter(!starts_with(model, \"M\"))\n    DB.@group_by(cyl)\n    DB.@summarize(mpg = mean(mpg))\n    DB.@mutate(mpg_squared = mpg^2, \n               mpg_rounded = round(mpg), \n               mpg_efficiency = case_when(\n                                 mpg >= cyl^2 , \"efficient\",\n                                 mpg < 15.2 , \"inefficient\",\n                                 \"moderate\"))            \n    DB.@filter(mpg_efficiency in (\"moderate\", \"efficient\"))\n    DB.@arrange(desc(mpg_rounded))\n    DB.@collect\nend\n
 
2\u00d75 DataFrame\n Row \u2502 cyl     mpg       mpg_squared  mpg_rounded  mpg_efficiency \n     \u2502 Int64?  Float64?  Float64?     Float64?     String?        \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502      4   27.3444      747.719         27.0  efficient\n   2 \u2502      6   19.7333      389.404         20.0  moderate\n
 
 
"},{"location":"#what-if-we-wanted-to-pivot-the-result","title":"What if we wanted to pivot the result?","text":"
We cannot do this using TidierDB. However, we can call @pivot_longer() from TidierData after the result of the query has been instantiated as a DataFrame, like this: 
 
@chain DB.db_table(db, path_or_name) begin\n    DB.@filter(!starts_with(model, \"M\"))\n    DB.@group_by(cyl)\n    DB.@summarize(mpg = mean(mpg))\n    DB.@mutate(mpg_squared = mpg^2, \n               mpg_rounded = round(mpg), \n               mpg_efficiency = case_when(\n                                 mpg >= cyl^2 , \"efficient\",\n                                 mpg < 15.2 , \"inefficient\",\n                                 \"moderate\"))            \n    DB.@filter(mpg_efficiency in (\"moderate\", \"efficient\"))\n    DB.@arrange(desc(mpg_rounded))\n    DB.@collect\n    @pivot_longer(everything(), names_to = \"variable\", values_to = \"value\")\nend\n
 
10\u00d72 DataFrame\n Row \u2502 variable        value     \n     \u2502 String          Any       \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 cyl             4\n   2 \u2502 cyl             6\n   3 \u2502 mpg             27.3444\n   4 \u2502 mpg             19.7333\n   5 \u2502 mpg_squared     747.719\n   6 \u2502 mpg_squared     389.404\n   7 \u2502 mpg_rounded     27.0\n   8 \u2502 mpg_rounded     20.0\n   9 \u2502 mpg_efficiency  efficient\n  10 \u2502 mpg_efficiency  moderate\n
 
 
"},{"location":"#what-sql-query-does-tidierdb-generate-for-a-given-piece-of-julia-code","title":"What SQL query does TidierDB generate for a given piece of Julia code?","text":"
We can replace DB.collect() with DB.@show_query to reveal the underlying SQL query being generated by TidierDB. To handle complex queries, TidierDB makes heavy use of Common Table Expressions (CTE), which are a useful tool to organize long queries.
 
@chain DB.db_table(db, path_or_name) begin\n    DB.@filter(!starts_with(model, \"M\"))\n    DB.@group_by(cyl)\n    DB.@summarize(mpg = mean(mpg))\n    DB.@mutate(mpg_squared = mpg^2, \n               mpg_rounded = round(mpg), \n               mpg_efficiency = case_when(\n                                 mpg >= cyl^2 , \"efficient\",\n                                 mpg < 15.2 , \"inefficient\",\n                                 \"moderate\"))            \n    DB.@filter(mpg_efficiency in (\"moderate\", \"efficient\"))\n    DB.@arrange(desc(mpg_rounded))\n    DB.@show_query\nend\n
 
WITH cte_1 AS (\nSELECT *\n        FROM mtcars\n        WHERE NOT (starts_with(model, 'M'))),\ncte_2 AS (\nSELECT cyl, AVG(mpg) AS mpg\n        FROM cte_1\n        GROUP BY cyl),\ncte_3 AS (\nSELECT  cyl, mpg, POWER(mpg, 2) AS mpg_squared, ROUND(mpg) AS mpg_rounded, CASE WHEN mpg >= POWER(cyl, 2) THEN 'efficient' WHEN mpg < 15.2 THEN 'inefficient' ELSE 'moderate' END AS mpg_efficiency\n        FROM cte_2 ),\ncte_4 AS (\nSELECT *\n        FROM cte_3\n        WHERE mpg_efficiency in ('moderate', 'efficient'))  \nSELECT *\n        FROM cte_4  \n        ORDER BY mpg_rounded DESC\n
 
 
"},{"location":"#tidierdb-is-already-quite-fully-featured-supporting-advanced-tidierdata-functions-like-across-for-multi-column-selection","title":"TidierDB is already quite fully-featured, supporting advanced TidierData functions like across() for multi-column selection.","text":"
@chain DB.db_table(db, path_or_name) begin\n    DB.@group_by(cyl)\n    DB.@summarize(across((starts_with(\"a\"), ends_with(\"s\")), (mean, sum)))\n    DB.@collect\nend\n
 
3\u00d75 DataFrame\n Row \u2502 cyl     mean_am   mean_vs   sum_am   sum_vs  \n     \u2502 Int64?  Float64?  Float64?  Int128?  Int128? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502      4  0.727273  0.909091        8       10\n   2 \u2502      6  0.428571  0.571429        3        4\n   3 \u2502      8  0.142857  0.0             2        0\n
 
Bang bang !! interpolation for columns and values is also supported.
 
There are a few subtle but important differences from Tidier.jl outlined here.
 
 
"},{"location":"#missing-a-function-or-backend","title":"Missing a function or backend?","text":"
You can use any existing SQL function within @mutate with the correct SQL syntax and it should just work.
 
But if you run into problems please open an issue, and we will be happy to take a look!
"},{"location":"reference/","title":"Reference","text":""},{"location":"reference/#index","title":"Index","text":"
     
  • TidierDB.connect
    •  
  • TidierDB.copy_to
    •  
  • TidierDB.db_table
    •  
  • TidierDB.@anti_join
    •  
  • TidierDB.@arrange
    •  
  • TidierDB.@collect
    •  
  • TidierDB.@count
    •  
  • TidierDB.@distinct
    •  
  • TidierDB.@filter
    •  
  • TidierDB.@full_join
    •  
  • TidierDB.@group_by
    •  
  • TidierDB.@head
    •  
  • TidierDB.@inner_join
    •  
  • TidierDB.@interpolate
    •  
  • TidierDB.@left_join
    •  
  • TidierDB.@mutate
    •  
  • TidierDB.@rename
    •  
  • TidierDB.@right_join
    •  
  • TidierDB.@select
    •  
  • TidierDB.@semi_join
    •  
  • TidierDB.@slice_max
    •  
  • TidierDB.@slice_min
    •  
  • TidierDB.@slice_sample
    •  
  • TidierDB.@summarise
    •  
  • TidierDB.@summarize
    •  
  • TidierDB.@window_frame
    •  
  • TidierDB.@window_order
    •  
"},{"location":"reference/#reference-exported-functions","title":"Reference - Exported functions","text":"
# TidierDB.connect \u2014 Method.
 
connect(backend; kwargs...)\n
 
This function establishes a database connection based on the specified backend and connection parameters and sets the SQL mode
 
Arguments
 
     
  •  
    backend: type specifying the database backend to connect to. Supported backends are:
     
       
    • duckdb(), sqlite()(SQLite), mssql(), mysql()(for MariaDB and MySQL), clickhouse(), postgres()
      •  
    •  
      kwargs: Keyword arguments specifying the connection parameters for the selected backend. The required parameters vary depending on the backend:
       
      •  
    •  
      MySQL:
       
         
      • host: The host name or IP address of the MySQL server. Default is \"localhost\".
        •  
      • user: The username for authentication. Default is an empty string.
        •  
      • password: The password for authentication.
        •  
      • db: The name of the database to connect to (optional).
        •  
      • port: The port number of the MySQL server (optional).
        •  
       
      •  
     
    •  
 
Returns
 
     
  • A database connection object based on the selected backend.
    •  
 
Examples
 
# Connect to MySQL\n# conn = connect(mysql(); host=\"localhost\", user=\"root\", password=\"password\", db=\"mydb\")\n# Connect to PostgreSQL using LibPQ\n# conn = connect(postgres(); host=\"localhost\", dbname=\"mydb\", user=\"postgres\", password=\"password\")\n# Connect to ClickHouse\n# conn = connect(clickhouse(); host=\"localhost\", port=9000, database=\"mydb\", user=\"default\", password=\"\")\n# Connect to SQLite\n# conn = connect(sqlite())\n# Connect to Google Big Query\n# conn = connect(gbq(), \"json_user_key_path\", \"project_id\")\n# Connect to Snowflake\n# conn = connect(snowflake(), \"ac_id\", \"token\", \"Database_name\", \"Schema_name\", \"warehouse_name\")\n# Connect to DuckDB\n# connect to Google Cloud via DuckDB\n# google_db = connect(duckdb(), :gbq, access_key=\"string\", secret_key=\"string\")\n# Connect to AWS via DuckDB\n# aws_db = connect2(duckdb(), :aws, aws_access_key_id=get(ENV, \"AWS_ACCESS_KEY_ID\", \"access_key\"), aws_secret_access_key=get(ENV, \"AWS_SECRET_ACCESS_KEY\", \"secret_access key\"), aws_region=get(ENV, \"AWS_DEFAULT_REGION\", \"us-east-1\"))\n# Connect to MotherDuck\n# connect(duckdb(), \"token\") for first connection, vs connect(duckdb(), \"md:\") for reconnection\njulia> db = connect(duckdb())\nDuckDB.Connection(\":memory:\")\n
 
source
 
# TidierDB.copy_to \u2014 Method.
 
   copy_to(conn, df_or_path, \"name\")\n
 
Allows user to copy a df to the database connection. Currently supports DuckDB, SQLite, MySql
 
Arguments
 
-conn: the database connection -df: dataframe to be copied or path to serve as source. With DuckDB, path supports .csv, .json, .parquet to be used without copying intermediary df. -name: name as string for the database to be used
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"test\");\n
 
source
 
# TidierDB.db_table \u2014 Function.
 
db_table(database, table_name, athena_params, delta = false, iceberg = false)\n
 
db_table starts the underlying SQL query struct, adding the metadata and table. If paths are passed directly to db_table instead of a  name it will not copy it to memory, but rather ready directly from the file.
 
Arguments
 
     
  • database: The Database or connection object
    •  
  •  
    table_name: tablename as a string (name, local path, or URL).     - CSV/TSV       - Parquet     - Json      - Iceberg     - Delta     - S3 tables from AWS or Google Cloud 
     
       
    • DuckDB and ClickHouse support vectors of paths and URLs.
      •  
    • DuckDB and ClickHouse also support use of * wildcards to read all files of a type in a location such as:
      •  
    • db_table(db, \"Path/to/testing_files/*.parquet\")
      •  
    • delta: must be true to read delta files
      •  
    • iceberg: must be true to read iceberg finalize_ctes
      •  
     
    •  
 
Example
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> db_table(db, \"df_mem\")\nTidierDB.SQLQuery(\"\", \"df_mem\", \"\", \"\", \"\", \"\", \"\", \"\", false, false, 4\u00d74 DataFrame\n Row \u2502 name     type     current_selxn  table_name \n     \u2502 String?  String?  Int64          String     \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 id       VARCHAR              1  df_mem\n   2 \u2502 groups   VARCHAR              1  df_mem\n   3 \u2502 value    BIGINT               1  df_mem\n   4 \u2502 percent  DOUBLE               1  df_mem, false, DuckDB.Connection(\":memory:\"), TidierDB.CTE[], 0, nothing)\n
 
source
 
# TidierDB.@anti_join \u2014 Macro.
 
@anti_join(sql_query, join_table, new_table_col, orignal_table_col)\n
 
Perform an anti join between two SQL queries based on a specified condition.  This syntax here is slightly different than TidierData.jl, however, because  SQL does not drop the joining column, for the metadata storage, it is  preferrable for the names to be different 
 
Arguments
 
     
  • sql_query: The primary SQL query to operate on.
    •  
  • join_table: The secondary SQL table to join with the primary query table.
    •  
  • new_table_col: Column from the new table that matches for join.
    •  
  • orignal_table_col: Column from the original table that matches for join.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> df2 = DataFrame(id2 = [\"AA\", \"AC\", \"AE\", \"AG\", \"AI\", \"AK\", \"AM\"],\n                category = [\"X\", \"Y\", \"X\", \"Y\", \"X\", \"Y\", \"X\"],\n                score = [88, 92, 77, 83, 95, 68, 74]);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> copy_to(db, df2, \"df_join\");\n\njulia> @chain db_table(db, :df_mem) begin\n        @anti_join(df_join, id2, id)\n        @collect\n       end\n5\u00d74 DataFrame\n Row \u2502 id       groups   value   percent  \n     \u2502 String?  String?  Int64?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AB       aa            2       0.2\n   2 \u2502 AD       aa            4       0.4\n   3 \u2502 AF       aa            1       0.6\n   4 \u2502 AH       aa            3       0.8\n   5 \u2502 AJ       aa            5       1.0\n
 
source
 
# TidierDB.@arrange \u2014 Macro.
 
@arrange(sql_query, columns...)\n
 
Order SQL table rows based on specified column(s).
 
Arguments
 
     
  • sql_query: The SQL query to operate on.
    •  
  • columns: Columns to order the rows by. Can include multiple columns for nested sorting. Wrap column name with desc() for descending order.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @arrange(value, desc(percent))\n         @collect\n       end\n10\u00d74 DataFrame\n Row \u2502 id       groups   value   percent  \n     \u2502 String?  String?  Int64?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AF       aa            1       0.6\n   2 \u2502 AA       bb            1       0.1\n   3 \u2502 AG       bb            2       0.7\n   4 \u2502 AB       aa            2       0.2\n   5 \u2502 AH       aa            3       0.8\n   6 \u2502 AC       bb            3       0.3\n   7 \u2502 AI       bb            4       0.9\n   8 \u2502 AD       aa            4       0.4\n   9 \u2502 AJ       aa            5       1.0\n  10 \u2502 AE       bb            5       0.5\n
 
source
 
# TidierDB.@collect \u2014 Macro.
 
@collect(sql_query, stream = false)\n
 
db_table starts the underlying SQL query struct, adding the metadata and table. 
 
Arguments
 
     
  • sql_query: The SQL query to operate on.
    •  
  • stream: optional streaming for query/execution of results when using duck db. Defaults to false
    •  
 
Example
 
julia> db = connect(duckdb());\n\njulia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @collect db_table(db, \"df_mem\")\n10\u00d74 DataFrame\n Row \u2502 id       groups   value   percent  \n     \u2502 String?  String?  Int64?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA       bb            1       0.1\n   2 \u2502 AB       aa            2       0.2\n   3 \u2502 AC       bb            3       0.3\n   4 \u2502 AD       aa            4       0.4\n   5 \u2502 AE       bb            5       0.5\n   6 \u2502 AF       aa            1       0.6\n   7 \u2502 AG       bb            2       0.7\n   8 \u2502 AH       aa            3       0.8\n   9 \u2502 AI       bb            4       0.9\n  10 \u2502 AJ       aa            5       1.0\n
 
source
 
# TidierDB.@count \u2014 Macro.
 
@count(sql_query, columns...)\n
 
Count the number of rows grouped by specified column(s).
 
Arguments
 
     
  • sql_query: The SQL query to operate on.
    •  
  • columns: Columns to group by before counting. If no columns are specified, counts all rows in the query.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @count(groups)\n         @arrange(groups)\n         @collect\n       end\n2\u00d72 DataFrame\n Row \u2502 groups   count  \n     \u2502 String?  Int64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 aa            5\n   2 \u2502 bb            5\n
 
source
 
# TidierDB.@distinct \u2014 Macro.
 
@distinct(sql_query, columns...)\n
 
Select distinct rows based on specified column(s). Distinct works differently in TidierData vs SQL and therefore TidierDB. Distinct will also select only the only columns it is given (or all if given none)
 
Arguments
 
sql_query: The SQL query to operate on. columns: Columns to determine uniqueness. If no columns are specified, all columns are used to identify distinct rows.
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @distinct(value)\n         @arrange(value)\n         @collect\n       end\n5\u00d71 DataFrame\n Row \u2502 value  \n     \u2502 Int64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502      1\n   2 \u2502      2\n   3 \u2502      3\n   4 \u2502      4\n   5 \u2502      5\n\njulia> @chain db_table(db, :df_mem) begin\n         @distinct\n         @arrange(id)\n         @collect\n       end\n10\u00d74 DataFrame\n Row \u2502 id       groups   value   percent  \n     \u2502 String?  String?  Int64?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA       bb            1       0.1\n   2 \u2502 AB       aa            2       0.2\n   3 \u2502 AC       bb            3       0.3\n   4 \u2502 AD       aa            4       0.4\n   5 \u2502 AE       bb            5       0.5\n   6 \u2502 AF       aa            1       0.6\n   7 \u2502 AG       bb            2       0.7\n   8 \u2502 AH       aa            3       0.8\n   9 \u2502 AI       bb            4       0.9\n  10 \u2502 AJ       aa            5       1.0\n
 
source
 
# TidierDB.@filter \u2014 Macro.
 
@filter(sql_query, conditions...)\n
 
Filter rows in a SQL table based on specified conditions.
 
Arguments
 
     
  • sql_query: The SQL query to filter rows from.
    •  
  •  
    conditions: Expressions specifying the conditions that rows must satisfy to be included in the output.                   Rows for which the expression evaluates to true will be included in the result.                   Multiple conditions can be combined using logical operators (&&, ||). It will automatically                   detect whether the conditions belong in WHERE vs HAVING. 
     
                 Temporarily, it is best to use begin and end when filtering multiple conditions. (ex 2 below)\n
     
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @filter(percent > .5)\n         @collect\n       end\n5\u00d74 DataFrame\n Row \u2502 id       groups   value   percent  \n     \u2502 String?  String?  Int64?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AF       aa            1       0.6\n   2 \u2502 AG       bb            2       0.7\n   3 \u2502 AH       aa            3       0.8\n   4 \u2502 AI       bb            4       0.9\n   5 \u2502 AJ       aa            5       1.0\n\njulia> @chain db_table(db, :df_mem) begin\n         @group_by(groups)\n         @summarise(mean = mean(percent))\n         @filter begin \n           groups == \"bb\" || # logical operators can still be used like this\n           mean > .5\n         end\n         @arrange(groups)\n         @collect\n       end\n2\u00d72 DataFrame\n Row \u2502 groups   mean     \n     \u2502 String?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 aa            0.6\n   2 \u2502 bb            0.5\n
 
source
 
# TidierDB.@full_join \u2014 Macro.
 
@inner_join(sql_query, join_table, new_table_col, orignal_table_col)\n
 
Perform an full join between two SQL queries based on a specified condition.  This syntax here is slightly different than TidierData.jl, however, because  SQL does not drop the joining column, for the metadata storage, it is  preferrable for the names to be different 
 
Arguments
 
     
  • sql_query: The primary SQL query to operate on.
    •  
  • join_table: The secondary SQL table to join with the primary query table.
    •  
  • new_table_col: Column from the new table that matches for join.
    •  
  • orignal_table_col: Column from the original table that matches for join.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> df2 = DataFrame(id2 = [\"AA\", \"AC\", \"AE\", \"AG\", \"AI\", \"AK\", \"AM\"],\n                category = [\"X\", \"Y\", \"X\", \"Y\", \"X\", \"Y\", \"X\"],\n                score = [88, 92, 77, 83, 95, 68, 74]);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> copy_to(db, df2, \"df_join\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @full_join(df_join, id2, id)\n         @collect\n       end\n12\u00d77 DataFrame\n Row \u2502 id       groups   value    percent    id2      category  score   \n     \u2502 String?  String?  Int64?   Float64?   String?  String?   Int64?  \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA       bb             1        0.1  AA       X              88\n   2 \u2502 AC       bb             3        0.3  AC       Y              92\n   3 \u2502 AE       bb             5        0.5  AE       X              77\n   4 \u2502 AG       bb             2        0.7  AG       Y              83\n   5 \u2502 AI       bb             4        0.9  AI       X              95\n   6 \u2502 AB       aa             2        0.2  missing  missing   missing \n   7 \u2502 AD       aa             4        0.4  missing  missing   missing \n   8 \u2502 AF       aa             1        0.6  missing  missing   missing \n   9 \u2502 AH       aa             3        0.8  missing  missing   missing \n  10 \u2502 AJ       aa             5        1.0  missing  missing   missing \n  11 \u2502 missing  missing  missing  missing    AK       Y              68\n  12 \u2502 missing  missing  missing  missing    AM       X              74\n
 
source
 
# TidierDB.@group_by \u2014 Macro.
 
@group_by(sql_query, columns...)\n
 
Group SQL table rows by specified column(s). If grouping is performed as a terminal operation without a subsequent mutatation or summarization (as in the example below), then the resulting data frame will be ungrouped when @collect is applied.
 
Arguments
 
     
  • sql_query: The SQL query to operate on.
    •  
  • exprs: Expressions specifying the columns to group by. Columns can be specified by name.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @group_by(groups)\n         @arrange(groups)\n         @collect\n       end\n2\u00d71 DataFrame\n Row \u2502 groups  \n     \u2502 String? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 aa\n   2 \u2502 bb\n
 
source
 
# TidierDB.@head \u2014 Macro.
 
@head(sql_query, value)\n
 
Limit SQL table number of rows returned based on specified value.  LIMIT in SQL
 
Arguments
 
     
  • sql_query: The SQL query to operate on.
    •  
  • value: Number to limit how many rows are returned.
    •  
 
Examples
 
julia> db = connect(duckdb());\n\njulia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> copy_to(db, df, \"df_mem\");                     \n\njulia> @chain db_table(db, :df_mem) begin\n        @head(1) ## supports expressions ie `3-2` would return the same df below\n        @collect\n       end\n1\u00d74 DataFrame\n Row \u2502 id       groups   value   percent  \n     \u2502 String?  String?  Int64?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA       bb            1       0.1\n
 
source
 
# TidierDB.@inner_join \u2014 Macro.
 
@inner_join(sql_query, join_table, new_table_col, orignal_table_col)\n
 
Perform an inner join between two SQL queries based on a specified condition.  This syntax here is slightly different than TidierData.jl, however, because  SQL does not drop the joining column, for the metadata storage, it is  preferrable for the names to be different 
 
Arguments
 
     
  • sql_query: The primary SQL query to operate on.
    •  
  • join_table: The secondary SQL table to join with the primary query table.
    •  
  • new_table_col: Column from the new table that matches for join.
    •  
  • orignal_table_col: Column from the original table that matches for join.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> df2 = DataFrame(id2 = [\"AA\", \"AC\", \"AE\", \"AG\", \"AI\", \"AK\", \"AM\"],\n                category = [\"X\", \"Y\", \"X\", \"Y\", \"X\", \"Y\", \"X\"],\n                score = [88, 92, 77, 83, 95, 68, 74]);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> copy_to(db, df2, \"df_join\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @inner_join(df_join, id2, id)\n         @collect\n       end\n5\u00d77 DataFrame\n Row \u2502 id       groups   value   percent   id2      category  score  \n     \u2502 String?  String?  Int64?  Float64?  String?  String?   Int64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA       bb            1       0.1  AA       X             88\n   2 \u2502 AC       bb            3       0.3  AC       Y             92\n   3 \u2502 AE       bb            5       0.5  AE       X             77\n   4 \u2502 AG       bb            2       0.7  AG       Y             83\n   5 \u2502 AI       bb            4       0.9  AI       X             95\n
 
source
 
# TidierDB.@interpolate \u2014 Macro.
 
@interpolate(args...)\n
 
Interpolate parameters into expressions for database queries.
 
Arguments
 
     
  •  
    args...: A variable number of tuples. Each tuple should contain:
     
       
    • name: The name of the parameter to interpolate.
      •  
    • value: (Any): The value/vector to interpolate for the corresponding parameter name.
      •  
     
    •  
 
Example
 
julia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> col_names = [:id, :value, :percent];\n\njulia> cond1 = .2;\n\njulia> cond2 = 5;\n\njulia> @interpolate((condition1, cond1), (columns, col_names), (condition2, cond2));\n\njulia> @chain db_table(db, \"df_mem\") begin \n          @select(!!columns)\n          @filter begin \n              percent < !!condition1\n              value < !!condition2\n          end\n          @collect\n          end\n1\u00d73 DataFrame\n Row \u2502 id       value   percent  \n     \u2502 String?  Int64?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA            1       0.1\n
 
source
 
# TidierDB.@left_join \u2014 Macro.
 
@left_join(sql_query, join_table, new_table_col, orignal_table_col)\n
 
Perform a left join between two SQL queries based on a specified condition.  This syntax here is slightly different than TidierData.jl, however, because  SQL does not drop the joining column, for the metadata storage, it is  preferrable for the names to be different 
 
Arguments
 
     
  • sql_query: The primary SQL query to operate on.
    •  
  • join_table: The secondary SQL table to join with the primary query table.
    •  
  • new_table_col: Column from the new table that matches for join.
    •  
  • orignal_table_col: Column from the original table that matches for join.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> df2 = DataFrame(id2 = [\"AA\", \"AC\", \"AE\", \"AG\", \"AI\", \"AK\", \"AM\"],\n                category = [\"X\", \"Y\", \"X\", \"Y\", \"X\", \"Y\", \"X\"],\n                score = [88, 92, 77, 83, 95, 68, 74]);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> copy_to(db, df2, \"df_join\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @left_join(df_join, id2, id)\n         @collect\n       end\n10\u00d77 DataFrame\n Row \u2502 id       groups   value   percent   id2      category  score   \n     \u2502 String?  String?  Int64?  Float64?  String?  String?   Int64?  \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA       bb            1       0.1  AA       X              88\n   2 \u2502 AC       bb            3       0.3  AC       Y              92\n   3 \u2502 AE       bb            5       0.5  AE       X              77\n   4 \u2502 AG       bb            2       0.7  AG       Y              83\n   5 \u2502 AI       bb            4       0.9  AI       X              95\n   6 \u2502 AB       aa            2       0.2  missing  missing   missing \n   7 \u2502 AD       aa            4       0.4  missing  missing   missing \n   8 \u2502 AF       aa            1       0.6  missing  missing   missing \n   9 \u2502 AH       aa            3       0.8  missing  missing   missing \n  10 \u2502 AJ       aa            5       1.0  missing  missing   missing \n
 
source
 
# TidierDB.@mutate \u2014 Macro.
 
@mutate(sql_query, exprs...)\n
 
Mutate SQL table rows by adding new columns or modifying existing ones.
 
Arguments
 
     
  • sql_query: The SQL query to operate on.
    •  
  • exprs: Expressions for mutating the table. New columns can be added or existing columns modified using column_name = expression syntax, where expression can involve existing columns.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @mutate(value = value * 4, new_col = percent^2)\n         @collect\n       end\n10\u00d75 DataFrame\n Row \u2502 id       groups   value   percent   new_col  \n     \u2502 String?  String?  Int64?  Float64?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA       bb            4       0.1      0.01\n   2 \u2502 AB       aa            8       0.2      0.04\n   3 \u2502 AC       bb           12       0.3      0.09\n   4 \u2502 AD       aa           16       0.4      0.16\n   5 \u2502 AE       bb           20       0.5      0.25\n   6 \u2502 AF       aa            4       0.6      0.36\n   7 \u2502 AG       bb            8       0.7      0.49\n   8 \u2502 AH       aa           12       0.8      0.64\n   9 \u2502 AI       bb           16       0.9      0.81\n  10 \u2502 AJ       aa           20       1.0      1.0\n
 
source
 
# TidierDB.@rename \u2014 Macro.
 
@rename(sql_query, renamings...)\n
 
Rename one or more columns in a SQL query.
 
Arguments
 
-sql_query: The SQL query to operate on. -renamings: One or more pairs of old and new column names, specified as new name = old name 
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n       @rename(new_name = percent)\n       @collect\n       end\n10\u00d74 DataFrame\n Row \u2502 id       groups   value   new_name \n     \u2502 String?  String?  Int64?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA       bb            1       0.1\n   2 \u2502 AB       aa            2       0.2\n   3 \u2502 AC       bb            3       0.3\n   4 \u2502 AD       aa            4       0.4\n   5 \u2502 AE       bb            5       0.5\n   6 \u2502 AF       aa            1       0.6\n   7 \u2502 AG       bb            2       0.7\n   8 \u2502 AH       aa            3       0.8\n   9 \u2502 AI       bb            4       0.9\n  10 \u2502 AJ       aa            5       1.0\n
 
source
 
# TidierDB.@right_join \u2014 Macro.
 
@right_join(sql_query, join_table, new_table_col, orignal_table_col)\n
 
Perform a right join between two SQL queries based on a specified condition.  This syntax here is slightly different than TidierData.jl, however, because  SQL does not drop the joining column, for the metadata storage, it is  preferrable for the names to be different 
 
Arguments
 
     
  • sql_query: The primary SQL query to operate on.
    •  
  • join_table: The secondary SQL table to join with the primary query table.
    •  
  • new_table_col: Column from the new table that matches for join.
    •  
  • orignal_table_col: Column from the original table that matches for join.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> df2 = DataFrame(id2 = [\"AA\", \"AC\", \"AE\", \"AG\", \"AI\", \"AK\", \"AM\"],\n                category = [\"X\", \"Y\", \"X\", \"Y\", \"X\", \"Y\", \"X\"],\n                score = [88, 92, 77, 83, 95, 68, 74]);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> copy_to(db, df2, \"df_join\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @right_join(df_join, id2, id)\n         @collect\n       end\n7\u00d77 DataFrame\n Row \u2502 id       groups   value    percent    id2      category  score  \n     \u2502 String?  String?  Int64?   Float64?   String?  String?   Int64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA       bb             1        0.1  AA       X             88\n   2 \u2502 AC       bb             3        0.3  AC       Y             92\n   3 \u2502 AE       bb             5        0.5  AE       X             77\n   4 \u2502 AG       bb             2        0.7  AG       Y             83\n   5 \u2502 AI       bb             4        0.9  AI       X             95\n   6 \u2502 missing  missing  missing  missing    AK       Y             68\n   7 \u2502 missing  missing  missing  missing    AM       X             74\n
 
source
 
# TidierDB.@select \u2014 Macro.
 
@select(sql_query, columns)\n
 
Select specified columns from a SQL table.
 
Arguments
 
     
  • sql_query: The SQL query to select columns from.
    •  
  • columns: Expressions specifying the columns to select. Columns can be specified by name,                and new columns can be created with expressions using existing column values.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @select(groups:percent)\n         @collect\n       end\n10\u00d73 DataFrame\n Row \u2502 groups   value   percent  \n     \u2502 String?  Int64?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 bb            1       0.1\n   2 \u2502 aa            2       0.2\n   3 \u2502 bb            3       0.3\n   4 \u2502 aa            4       0.4\n   5 \u2502 bb            5       0.5\n   6 \u2502 aa            1       0.6\n   7 \u2502 bb            2       0.7\n   8 \u2502 aa            3       0.8\n   9 \u2502 bb            4       0.9\n  10 \u2502 aa            5       1.0\n\njulia> @chain db_table(db, :df_mem) begin\n         @select(contains(\"e\"))\n         @collect\n       end\n10\u00d72 DataFrame\n Row \u2502 value   percent  \n     \u2502 Int64?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502      1       0.1\n   2 \u2502      2       0.2\n   3 \u2502      3       0.3\n   4 \u2502      4       0.4\n   5 \u2502      5       0.5\n   6 \u2502      1       0.6\n   7 \u2502      2       0.7\n   8 \u2502      3       0.8\n   9 \u2502      4       0.9\n  10 \u2502      5       1.0\n
 
source
 
# TidierDB.@semi_join \u2014 Macro.
 
@semi_join(sql_query, join_table, new_table_col, orignal_table_col)\n
 
Perform an semi join between two SQL queries based on a specified condition.  This syntax here is slightly different than TidierData.jl, however, because  SQL does not drop the joining column, for the metadata storage, it is  preferrable for the names to be different 
 
Arguments
 
     
  • sql_query: The primary SQL query to operate on.
    •  
  • join_table: The secondary SQL table to join with the primary query table.
    •  
  • new_table_col: Column from the new table that matches for join.
    •  
  • orignal_table_col: Column from the original table that matches for join.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> df2 = DataFrame(id2 = [\"AA\", \"AC\", \"AE\", \"AG\", \"AI\", \"AK\", \"AM\"],\n                category = [\"X\", \"Y\", \"X\", \"Y\", \"X\", \"Y\", \"X\"],\n                score = [88, 92, 77, 83, 95, 68, 74]);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> copy_to(db, df2, \"df_join\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @semi_join(df_join, id2, id)\n         @collect\n       end\n5\u00d74 DataFrame\n Row \u2502 id       groups   value   percent  \n     \u2502 String?  String?  Int64?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA       bb            1       0.1\n   2 \u2502 AC       bb            3       0.3\n   3 \u2502 AE       bb            5       0.5\n   4 \u2502 AG       bb            2       0.7\n   5 \u2502 AI       bb            4       0.9\n
 
source
 
# TidierDB.@slice_max \u2014 Macro.
 
@slice_max(sql_query, column, n = 1)\n
 
Select rows with the largest values in specified column. This will always return ties. 
 
Arguments
 
     
  • sql_query: The SQL query to operate on.
    •  
  • column: Column to identify the smallest values.
    •  
  • n: The number of rows to select with the largest values for each specified column. Default is 1, which selects the row with the smallest value.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @group_by(groups)\n         @slice_max(value, n = 2)\n         @collect\n       end;\n\njulia> @chain db_table(db, :df_mem) begin\n         @slice_max(value)\n         @collect\n       end\n2\u00d75 DataFrame\n Row \u2502 id       groups   value   percent   rank_col \n     \u2502 String?  String?  Int64?  Float64?  Int64?   \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AE       bb            5       0.5         1\n   2 \u2502 AJ       aa            5       1.0         1\n
 
source
 
# TidierDB.@slice_min \u2014 Macro.
 
@slice_min(sql_query, column, n = 1)\n
 
Select rows with the smallest values in specified column. This will always return ties. 
 
Arguments
 
     
  • sql_query: The SQL query to operate on.
    •  
  • column: Column to identify the smallest values.
    •  
  • n: The number of rows to select with the smallest values for each specified column. Default is 1, which selects the row with the smallest value.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @group_by(groups)\n         @slice_min(value, n = 2)\n         @collect\n       end;\n\njulia> @chain db_table(db, :df_mem) begin\n         @slice_min(value)\n         @collect\n       end\n2\u00d75 DataFrame\n Row \u2502 id       groups   value   percent   rank_col \n     \u2502 String?  String?  Int64?  Float64?  Int64?   \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA       bb            1       0.1         1\n   2 \u2502 AF       aa            1       0.6         1\n
 
source
 
# TidierDB.@slice_sample \u2014 Macro.
 
@slice_sample(sql_query, n)\n
 
Randomly select a specified number of rows from a SQL table.
 
Arguments
 
     
  • sql_query: The SQL query to operate on.
    •  
  • n: The number of rows to randomly select.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @group_by(groups)\n         @slice_sample(n = 2)\n         @collect\n       end;\n\njulia> @chain db_table(db, :df_mem) begin\n       @slice_sample()\n       @collect\n       end;\n
 
source
 
# TidierDB.@summarise \u2014 Macro.
 
   @summarise(sql_query, exprs...)\n
 
Aggregate and summarize specified columns of a SQL table.
 
Arguments
 
     
  • sql_query: The SQL query to operate on.
    •  
  • exprs: Expressions defining the aggregation and summarization operations. These can specify simple aggregations like mean, sum, and count, or more complex expressions involving existing column values.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @group_by(groups)\n         @summarise(across((value:percent), (mean, sum)))\n         @arrange(groups)\n         @collect\n       end\n2\u00d75 DataFrame\n Row \u2502 groups   mean_value  mean_percent  sum_value  sum_percent \n     \u2502 String?  Float64?    Float64?      Int128?    Float64?    \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 aa              3.0           0.6         15          3.0\n   2 \u2502 bb              3.0           0.5         15          2.5\n\njulia> @chain db_table(db, :df_mem) begin\n         @group_by(groups)\n         @summarise(test = sum(percent), n = n())\n         @arrange(groups)\n         @collect\n       end\n2\u00d73 DataFrame\n Row \u2502 groups   test      n      \n     \u2502 String?  Float64?  Int64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 aa            3.0       5\n   2 \u2502 bb            2.5       5\n
 
source
 
# TidierDB.@summarize \u2014 Macro.
 
   @summarize(sql_query, exprs...)\n
 
Aggregate and summarize specified columns of a SQL table.
 
Arguments
 
     
  • sql_query: The SQL query to operate on.
    •  
  • exprs: Expressions defining the aggregation and summarization operations. These can specify simple aggregations like mean, sum, and count, or more complex expressions involving existing column values.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @group_by(groups)\n         @summarise(across((ends_with(\"e\"), starts_with(\"p\")), (mean, sum)))\n         @arrange(groups)\n         @collect\n       end\n2\u00d75 DataFrame\n Row \u2502 groups   mean_value  mean_percent  sum_value  sum_percent \n     \u2502 String?  Float64?    Float64?      Int128?    Float64?    \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 aa              3.0           0.6         15          3.0\n   2 \u2502 bb              3.0           0.5         15          2.5\n\njulia> @chain db_table(db, :df_mem) begin\n         @group_by(groups)\n         @summarise(test = sum(percent), n = n())\n         @arrange(groups)\n         @collect\n       end\n2\u00d73 DataFrame\n Row \u2502 groups   test      n      \n     \u2502 String?  Float64?  Int64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 aa            3.0       5\n   2 \u2502 bb            2.5       5\n
 
source
 
# TidierDB.@window_frame \u2014 Macro.
 
@window_frame(sql_query, frame_start::Int, frame_end::Int)\n
 
Define the window frame for window functions in a SQL query, specifying the range of rows to include in the calculation relative to the current row.
 
Arguments
 
sql_query: The SQL query to operate on, expected to be an instance of SQLQuery.
 
     
  • frame_start: The starting point of the window frame. A positive value indicates the start after the current row (FOLLOWING), a negative value indicates before the current row (PRECEDING), and 0 indicates the current row.
    •  
  • frame_end: The ending point of the window frame. A positive value indicates the end after the current row (FOLLOWING), a negative value indicates before the current row (PRECEDING), and 0 indicates the current row.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n
 
source
 
# TidierDB.@window_order \u2014 Macro.
 
   @window_order(sql_query, columns...)\n
 
Specify the order of rows for window functions within a SQL query.
 
Arguments
 
     
  • sql_query: The SQL query to operate on.
    •  
  • columns: Columns to order the rows by for the window function. Can include multiple columns for nested sorting. Prepend a column name with - for descending order.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n
 
source
 
 
"},{"location":"reference/#reference-internal-functions","title":"Reference - Internal functions","text":""},{"location":"examples/generated/UserGuide/Snowflake/","title":"Using Snowflake","text":"
Establishing a connection with the Snowflake SQL Rest API requires a OAuth token specific to the Role the user will use to query tables with.
 
 
"},{"location":"examples/generated/UserGuide/Snowflake/#connecting","title":"Connecting","text":"
Connection is established with the connect function as shown below. Connection requires 5 items as strings
 
     
  • Account Identifier
    •  
  • OAuth token
    •  
  • Database Name
    •  
  • Schema Name
    •  
  • Compute Warehouse name
    •  
 
Two things to note:
 
     
  • Your OAuth Token may frequently expire, which may require you to rerun your connection line.
    •  
  •  
    Since each time db_table runs, it runs a query to pull the metadata, you may choose to use run db_table and save the results, and use these results withfrom_query()
     
       
    • This will reduce the number of queries to your database
      •  
    • Allow you to build a a SQL query and @show_query even if the OAuthtoken has expired. To @collect you will have to reconnect and rerun dbtable if your OAuth token has expired
      •  
     
    •  
 
set_sql_mode(snowflake())\nac_id = \"string_id\"\ntoken = \"OAuth_token_string\"\ncon = connect(:snowflake, ac_id, token, \"DEMODB\", \"PUBLIC\", \"COMPUTE_WH\")\n# After connection is established, a you may begin querying.\nstable_table_metadata = db_table(con, \"MTCARS\")\n@chain from_query(stable_table_metadata) begin\n   @select(WT)\n   @mutate(TEST = WT *2)\n   #@aside @show_query _\n   @collect\nend\n
 
32\u00d72 DataFrame\n Row \u2502 WT       TEST\n     \u2502 Float64  Float64\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502   2.62     5.24\n   2 \u2502   2.875    5.75\n   3 \u2502   2.32     4.64\n   4 \u2502   3.215    6.43\n  \u22ee  \u2502    \u22ee        \u22ee\n  29 \u2502   3.17     6.34\n  30 \u2502   2.77     5.54\n  31 \u2502   3.57     7.14\n  32 \u2502   2.78     5.56\n         24 rows omitted\n
 
This page was generated using Literate.jl.
"},{"location":"examples/generated/UserGuide/athena/","title":"Using Athena","text":"
To use the Athena AWS backend with TidierDB, set up and a small syntax difference are covered here.
 
 
"},{"location":"examples/generated/UserGuide/athena/#connecting","title":"Connecting","text":"
Connection is established through AWS.jl as shwon below.
 
using TidierDB, AWS\nset_sql_mode(athena())\n# Replace your credentials as needed below\naws_access_key_id = get(ENV,\"AWS_ACCESS_KEY_ID\",\"key\")\naws_secret_access_key = get(ENV, \"AWS_SECRET_ACCESS_KEY\",\"secret_key\")\naws_region = get(ENV,\"AWS_DEFAULT_REGION\",\"region\")\n\nconst AWS_GLOBAL_CONFIG = Ref{AWS.AWSConfig}()\ncreds = AWSCredentials(aws_access_key_id, aws_secret_access_key)\n\nAWS_GLOBAL_CONFIG[] = AWS.global_aws_config(region=aws_region, creds=creds)\n\ncatalog = \"AwsDataCatalog\"\nworkgroup = \"primary\"\ndb = \"demodb\"\nall_results = true\nresults_per_increment = 10\nout_loc = \"s3://location/\"\n\nathena_params = Dict(\n    \"ResultConfiguration\" => Dict(\n        \"OutputLocation\" => out_loc\n    ),\n    \"QueryExecutionContext\" => Dict(\n        \"Database\" => db,\n        \"Catalog\" => catalog\n    ),\n    \"Workgroup\" => workgroup\n)\n
 
 
"},{"location":"examples/generated/UserGuide/athena/#db_table-differences","title":"db_table differences","text":"
There are two differences for db_table which are seen in the query below
 
     
  1. The table needs to be passed as a string in the format database.table, ie \"demodb.table_name
    2.  
  2. db_table requires a third argument: the athena_params from above.
    3.  
 
 
"},{"location":"examples/generated/UserGuide/athena/#leveraging-from_query-with-athena-to-reduce-number-of-queries","title":"Leveraging from_query with Athena to reduce number of queries","text":"
Throughout TidierDB, each time db_table is called, it queries the databases to get the metadata. Consider how AWS Athena logs queries, a user may want to reduce the number of queries. This can be done saving the results of db_table, and then using from_query with those results for furthe queries as shown below.
 
mtcars = db_table(AWS_GLOBAL_CONFIG[], \"demodb.mtcars\", athena_params)\n@chain from_query(mtcars) begin\n    @filter(cyl > 4)\n    @group_by(cyl)\n    @summarize(mpg = mean(mpg))\n   #@show_query\n    @collect\nend\n
 
2\u00d72 DataFrame\n Row \u2502 cyl    mpg\n     \u2502 Int64  Float64\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502     6  19.7429\n   2 \u2502     8  15.1\n
 
I would like to acknowledge the work of Manu Francis and this blog post, which helped guide this process
 
This page was generated using Literate.jl.
"},{"location":"examples/generated/UserGuide/databricks/","title":"Using Databricks","text":"
Establishing a connection with the Databricks SQL Rest API requires a token.
 
 
"},{"location":"examples/generated/UserGuide/databricks/#connecting","title":"Connecting","text":"
Connection is established with the connect function as shown below. Connection requires 5 items as strings
 
     
  • Account Instance : how to find your instance
    •  
  • OAuth token : how to generate your token
    •  
  • Database Name
    •  
  • Schema Name
    •  
  • warehouse_id
    •  
 
One thing to note, Since each time db_table runs, it runs a query to pull the metadata, you may choose to use run db_table and save the results, and use these results with from_query(). This will reduce the number of queries to your database and is illustrated below.
 
set_sql_mode(databricks())\ninstance_id = \"string_id\"\ntoken \"string_token\"\nwarehouse_id = \"e673cd4f387f964a\"\ncon = connect(:databricks, instance_id, token, \"DEMODB\", \"PUBLIC\", warehouse_id)\n# After connection is established, a you may begin querying.\nstable_table_metadata = db_table(con, \"mtcars\")\n@chain from_query(stable_table_metadata) begin\n   @select(wt)\n   @mutate(test = wt *2)\n   #@aside @show_query _\n   @collect\nend\n
 
32\u00d72 DataFrame\n Row \u2502 wt       test\n     \u2502 Float64  Float64\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502   2.62     5.24\n   2 \u2502   2.875    5.75\n   3 \u2502   2.32     4.64\n   4 \u2502   3.215    6.43\n  \u22ee  \u2502    \u22ee        \u22ee\n  29 \u2502   3.17     6.34\n  30 \u2502   2.77     5.54\n  31 \u2502   3.57     7.14\n  32 \u2502   2.78     5.56\n         24 rows omitted\n
 
This page was generated using Literate.jl.
"},{"location":"examples/generated/UserGuide/from_queryex/","title":"Reusing Part of a Query","text":"
While using TidierDB, you may need to generate part of a query and reuse it multiple times. from_query() enables a query portion to be reused multiple times as shown below.
 
import TidierDB as DB\ncon = DB.connect(duckdb())\nmtcars_path = \"https://gist.githubusercontent.com/seankross/a412dfbd88b3db70b74b/raw/5f23f993cd87c283ce766e7ac6b329ee7cc2e1d1/mtcars.csv\"\n
 
Start a query to analyze fuel efficiency by number of cylinders. However, to further build on this query later, end the chain without using @show_query or @collect
 
query = DB.@chain DB.db_table(con, mtcars_path) begin\n    DB.@group_by cyl\n    DB.@summarize begin\n        across(mpg, (mean, minimum, maximum))\n        num_cars = n()\n        end\n    DB.@mutate begin\n        efficiency = case_when(\n            mean_mpg >= 25, \"High\",\n            mean_mpg >= 15, \"Moderate\",\n            \"Low\" )\n       end\nend;\n
 
Now, from_query will allow you to reuse the query to calculate the average horsepower for each efficiency category
 
DB.@chain DB.from_query(query) begin\n   DB.@left_join(mtcars2, cyl, cyl)\n   DB.@group_by(efficiency)\n   DB.@summarize(avg_hp = mean(hp))\n   DB.@collect\nend\n
 
2\u00d72 DataFrame\n Row \u2502 efficiency  avg_hp\n     \u2502 String?     Float64?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 Moderate    180.238\n   2 \u2502 High         82.6364\n
 
Reuse the query again to find the car with the highest MPG for each cylinder category
 
DB.@chain DB.from_query(query) begin\n   DB.@left_join(mtcars2, cyl, cyl)\n   DB.@group_by cyl\n   DB.@slice_max(mpg)\n   DB.@select model cyl mpg\n   DB.@collect\nend\n
 
3\u00d73 DataFrame\n Row \u2502 model             cyl     mpg\n     \u2502 String?           Int64?  Float64?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 Pontiac Firebird       8      19.2\n   2 \u2502 Toyota Corolla         4      33.9\n   3 \u2502 Hornet 4 Drive         6      21.4\n
 
 
"},{"location":"examples/generated/UserGuide/from_queryex/#preview-or-save-an-intermediate-table","title":"Preview or save an intermediate table","text":"
While querying a dataset, you may wish to see an intermediate table, or even save it. You can use @aside and from_query(_), illustrated below, to do just that. While we opted to print the results in this simple example below, we could have saved them by using name = DB.@chain...
 
import ClickHouse;\nconn = conn = DB.connect(DB.clickhouse(); host=\"localhost\", port=19000, database=\"default\", user=\"default\", password=\"\")\npath = \"https://huggingface.co/datasets/maharshipandya/spotify-tracks-dataset/resolve/refs%2Fconvert%2Fparquet/default/train/0000.parquet\"\nDB.@chain DB.db_table(conn, path) begin\n   DB.@count(cyl)\n   @aside println(DB.@chain DB.from_query(_) DB.@head(5) DB.@collect)\n   DB.@arrange(desc(count))\n   DB.@collect\nend\n
 
5\u00d72 DataFrame\n Row \u2502 artists  count\n     \u2502 String?  UInt64\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 missing       1\n   2 \u2502 Wizo          3\n   3 \u2502 MAGIC!        3\n   4 \u2502 Macaco        1\n   5 \u2502 SOYOU         1\n31438\u00d72 DataFrame\n   Row \u2502 artists          count\n       \u2502 String?          UInt64\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n     1 \u2502 The Beatles         279\n     2 \u2502 George Jones        271\n     3 \u2502 Stevie Wonder       236\n     4 \u2502 Linkin Park         224\n     5 \u2502 Ella Fitzgerald     222\n     6 \u2502 Prateek Kuhad       217\n     7 \u2502 Feid                202\n   \u22ee   \u2502        \u22ee           \u22ee\n 31432 \u2502 Leonard               1\n 31433 \u2502 marcos g              1\n 31434 \u2502 BLVKSHP               1\n 31435 \u2502 Memtrix               1\n 31436 \u2502 SOYOU                 1\n 31437 \u2502 Macaco                1\n 31438 \u2502 missing               1\n               31424 rows omitted\n
 
This page was generated using Literate.jl.
"},{"location":"examples/generated/UserGuide/functions_pass_to_DB/","title":"Writing Functions/Macros with TidierDB Chains","text":"
How can functions pass arguments to a TidierDB chain?
 
In short, you have to use a macro instead in conjuction with @interpolate
 
 
"},{"location":"examples/generated/UserGuide/functions_pass_to_DB/#setting-up-the-macro","title":"Setting up the macro","text":"
To write a macro that will take arguments and pass them to a TidierDB chain, there are 3 steps:
 
     
  1. Write macro with the desired argument(s), and, after the quote, add the chain. Arguments to be changed/interpolated must be prefixed with !!
    2.  
  2. Use @interpolate to make these arguemnts accessible to the chain. @interpolate takes touples as argument (one for the !!name, and one for the actual content you want the chain to use)
    3.  
  3. Run @interpolate and then the chain macro sequentially
    4.  
 
using TidierDB\ndb = connect(duckdb())\npath = \"https://gist.githubusercontent.com/seankross/a412dfbd88b3db70b74b/raw/5f23f993cd87c283ce766e7ac6b329ee7cc2e1d1/mtcars.csv\"\ncopy_to(db, path, \"mtcars\");\n\n# STEP 1\nmacro f1(conditions, columns) # The arguemnt names will be names of the `!!` values\n    return quote\n    # add chain here\n      @chain db_table(db, :mtcars) begin\n           @filter(!!conditions > 3)\n           @select(!!columns)\n           @aside @show_query _\n           @collect\n         end # ends the chain\n    end # ends the quote.\nend # ends the macro\n
 
# STEP 2\nvariable = :gear;\ncols = [:model, :mpg, :gear, :wt];\n@interpolate((conditions, variable), (columns, cols));\n@f1(variable, cols)\n
 
17\u00d74 DataFrame\n Row \u2502 model           mpg       gear    wt\n     \u2502 String?         Float64?  Int32?  Float64?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 Mazda RX4           21.0       4     2.62\n   2 \u2502 Mazda RX4 Wag       21.0       4     2.875\n   3 \u2502 Datsun 710          22.8       4     2.32\n  \u22ee  \u2502       \u22ee            \u22ee        \u22ee        \u22ee\n  15 \u2502 Ferrari Dino        19.7       5     2.77\n  16 \u2502 Maserati Bora       15.0       5     3.57\n  17 \u2502 Volvo 142E          21.4       4     2.78\n                                   11 rows omitted\n
 
Lets say you wanted to filter on new variable with a different name and select new columns,
 
new_condition = :wt;\nnew_cols = [:model, :drat]\n@interpolate((conditions, new_condition), (columns, new_cols));\n@f1(new_condition, new_cols)\n
 
20\u00d72 DataFrame\n Row \u2502 model              drat\n     \u2502 String?            Float64?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 Hornet 4 Drive         3.08\n   2 \u2502 Hornet Sportabout      3.15\n   3 \u2502 Valiant                2.76\n  \u22ee  \u2502         \u22ee             \u22ee\n  18 \u2502 Pontiac Firebird       3.08\n  19 \u2502 Ford Pantera L         4.22\n  20 \u2502 Maserati Bora          3.54\n                    14 rows omitted\n
 
You can also interpolate vectors of strings into a @filter(col in (values)) as well by using the following syntax @filter(col in [!!values])
 
In short, the first argument in @interpolate must be the name of the macro argument it refers to, and the second argument is what you would like to replace it.
 
We recognize this adds friction and that it is not ideal, but given the TidierDB macro expressions/string interplay, this is currently the most graceful and functional option available and hopefully a temporary solution to better interpolation that mirrors TidierData.jl.
 
This page was generated using Literate.jl.
"},{"location":"examples/generated/UserGuide/getting_started/","title":"Getting Started","text":"
To use TidierDB.jl, you will have to set up a connection. TidierDB.jl gives you access to duckdb via duckdb_open and duckdb_connect. However, to use MySql, ClickHouse, MSSQL, Postgres, or SQLite, you will have to load those packages in first.
 
If you plan to use TidierDB.jl with TidierData.jl or Tidier.jl, it is most convenenient to load the packages as follows:
 
using TidierData\nimport TidierDB as DB\n
 
Alternatively, using Tidier will import TidierDB in the above manner for you, where TidierDB functions and macros will be available as DB.@mutate() and so on, and the TidierData equivalent would be @mutate().
 
 
"},{"location":"examples/generated/UserGuide/getting_started/#connecting","title":"Connecting","text":"
To connect to a database, you can uset the connect function  as shown below, or establish your own connection through the respecitve libraries.
 
For example Connecting to MySQL
 
conn = DB.connect(DB.mysql(); host=\"localhost\", user=\"root\", password=\"password\", db=\"mydb\")\n
 
versus connecting to DuckDB
 
conn = DB.connect(DB.duckdb())\n
 
You can also use establish a connection through an alternate method that you preferred, and use that as your connection as well.
 
 
"},{"location":"examples/generated/UserGuide/getting_started/#package-extensions","title":"Package Extensions","text":"
The following backends utilize package extensions. To use one of backends listed below, you will need to write using Library
 
     
  • ClickHouse: import ClickHouse
    •  
  • MySQL and MariaDB: using MySQL
    •  
  • MSSQL: using ODBC
    •  
  • Postgres: using LibPQ
    •  
  • SQLite: using SQLite
    •  
  • Athena: using AWS
    •  
  • Oracle: using ODBC
    •  
  • Google BigQuery: using GoogleCloud
    •  
 
 
"},{"location":"examples/generated/UserGuide/getting_started/#db_table","title":"db_table","text":"
What does db_table do?
 
db_table starts the underlying SQL query struct, in addition to pulling the table metadata and storing it there. Storing metadata is what enables a lazy interface that also supports tidy selection.
 
     
  • db_table has two required arguments: connection and table
    •  
  • table can be a table name on a database or a path/url to file to read.  When passing db_table a path or url, the table is not copied into memory.
    •  
  • With DuckDB and ClickHouse, if you have a folder of multiple files to read, you can use * read in all files matching the pattern.
    •  
  • For example, the below would read all files that end in .csv in the given folder.
    •  
 
db_table(db, \"folder/path/*.csv\")\n
 
db_table also supports iceberg, delta, and S3 file paths via DuckDB.
 
 
"},{"location":"examples/generated/UserGuide/getting_started/#minimizing-compute-costs","title":"Minimizing Compute Costs","text":"
If you are working with a backend where compute cost is important, it will be important to minimize using db_table as this will requery for metadata each time. Compute costs are relevant to backends such as AWS, databricks and Snowflake.
 
To do this, save the results of db_table and use them with from_query. Using from_query pulls the relevant information (metadata, con, etc) from the mutable SQLquery struct, allowing you to repeatedly query and collect the table without requerying for the metadata each time
 
table = DB.db_table(con, \"path\")\n@chain DB.from_query(table) begin\n    ## data wrangling here\nend\n
 
This page was generated using Literate.jl.
"},{"location":"examples/generated/UserGuide/ibis_comp/","title":"TidierDB.jl vs Ibis","text":""},{"location":"examples/generated/UserGuide/ibis_comp/#comparing-tidierdb-vs-ibis","title":"Comparing TidierDB vs Ibis","text":"
TidierDB is a reimplementation of dbplyr from R, so the syntax is remarkably similar. But how does TidierDB compare to Python's Ibis? This page will perform a similar comparison to the Ibis Documentation comparing Ibis and dplyr
 
 
"},{"location":"examples/generated/UserGuide/ibis_comp/#set-up","title":"Set up","text":"
Ibis
 
import ibis\nimport ibis.selectors as s # allows for different styles of column selection\nfrom ibis import _ # eliminates need to type table name before each column vs typing cols as strings\nibis.options.interactive = True # automatically collects first 10 rows of table\n
 
TidierDB
 
using TidierDB\ndb = connect(duckdb())\n# This next line is optional, but it will let us avoid writing `db_table` or `from_query` for each query\nt(table) = from_query(table)\n
 
Of note, TidierDB does not yet have an \"interactive mode\" so each example result will be collected.
 
 
"},{"location":"examples/generated/UserGuide/ibis_comp/#loading-data","title":"Loading Data","text":"
With Ibis, there are specific functions to read in different file types
 
mtcars = ibis.read_csv(\"https://gist.githubusercontent.com/seankross/a412dfbd88b3db70b74b/raw/5f23f993cd87c283ce766e7ac6b329ee7cc2e1d1/mtcars.csv\")\n
 
In TidierDB, there is only db_table, which determines the file type and generates the syntax appropriate for the backend in use.
 
mtcars = db_table(db, \"https://gist.githubusercontent.com/seankross/a412dfbd88b3db70b74b/raw/5f23f993cd87c283ce766e7ac6b329ee7cc2e1d1/mtcars.csv\");\n
 
 
"},{"location":"examples/generated/UserGuide/ibis_comp/#previewing-the-data","title":"Previewing the data","text":"
TidierDB and Ibis use head/@head to preview the first rows of a dataset.
 
mtcars.head(6)\n
 
\u250f\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2513\n\u2503 model             \u2503 mpg     \u2503 cyl   \u2503 disp    \u2503 hp    \u2503 drat    \u2503 wt      \u2503 qsec    \u2503 vs    \u2503 am    \u2503 gear  \u2503 carb  \u2503\n\u2521\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2529\n\u2502 string            \u2502 float64 \u2502 int64 \u2502 float64 \u2502 int64 \u2502 float64 \u2502 float64 \u2502 float64 \u2502 int64 \u2502 int64 \u2502 int64 \u2502 int64 \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 Mazda RX4         \u2502    21.0 \u2502     6 \u2502   160.0 \u2502   110 \u2502    3.90 \u2502   2.620 \u2502   16.46 \u2502     0 \u2502     1 \u2502     4 \u2502     4 \u2502\n\u2502 Mazda RX4 Wag     \u2502    21.0 \u2502     6 \u2502   160.0 \u2502   110 \u2502    3.90 \u2502   2.875 \u2502   17.02 \u2502     0 \u2502     1 \u2502     4 \u2502     4 \u2502\n\u2502 Datsun 710        \u2502    22.8 \u2502     4 \u2502   108.0 \u2502    93 \u2502    3.85 \u2502   2.320 \u2502   18.61 \u2502     1 \u2502     1 \u2502     4 \u2502     1 \u2502\n\u2502 Hornet 4 Drive    \u2502    21.4 \u2502     6 \u2502   258.0 \u2502   110 \u2502    3.08 \u2502   3.215 \u2502   19.44 \u2502     1 \u2502     0 \u2502     3 \u2502     1 \u2502\n\u2502 Hornet Sportabout \u2502    18.7 \u2502     8 \u2502   360.0 \u2502   175 \u2502    3.15 \u2502   3.440 \u2502   17.02 \u2502     0 \u2502     0 \u2502     3 \u2502     2 \u2502\n\u2502 Valiant           \u2502    18.1 \u2502     6 \u2502   225.0 \u2502   105 \u2502    2.76 \u2502   3.460 \u2502   20.22 \u2502     1 \u2502     0 \u2502     3 \u2502     1 \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n
 
@chain t(mtcars) @head(6) @collect\n
 
6\u00d712 DataFrame\n Row \u2502 model              mpg       cyl     disp      hp      drat      wt        qsec      vs      am      gear    carb\n     \u2502 String?            Float64?  Int64?  Float64?  Int64?  Float64?  Float64?  Float64?  Int64?  Int64?  Int64?  Int64?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 Mazda RX4              21.0       6     160.0     110      3.9      2.62      16.46       0       1       4       4\n   2 \u2502 Mazda RX4 Wag          21.0       6     160.0     110      3.9      2.875     17.02       0       1       4       4\n   3 \u2502 Datsun 710             22.8       4     108.0      93      3.85     2.32      18.61       1       1       4       1\n   4 \u2502 Hornet 4 Drive         21.4       6     258.0     110      3.08     3.215     19.44       1       0       3       1\n   5 \u2502 Hornet Sportabout      18.7       8     360.0     175      3.15     3.44      17.02       0       0       3       2\n   6 \u2502 Valiant                18.1       6     225.0     105      2.76     3.46      20.22       1       0       3       1\n
 
 
"},{"location":"examples/generated/UserGuide/ibis_comp/#filtering","title":"Filtering","text":"
The example below demonstrates how to filter using multiple criteria in both Ibis and TidierData
 
mtcars.filter(((_.mpg > 22) & (_.drat > 4) | (_.hp == 113)))\n
 
\u250f\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2513\n\u2503 model          \u2503 mpg     \u2503 cyl   \u2503 disp    \u2503 hp    \u2503 drat    \u2503 wt      \u2503 qsec    \u2503 vs    \u2503 am    \u2503 gear  \u2503 carb  \u2503\n\u2521\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2529\n\u2502 string         \u2502 float64 \u2502 int64 \u2502 float64 \u2502 int64 \u2502 float64 \u2502 float64 \u2502 float64 \u2502 int64 \u2502 int64 \u2502 int64 \u2502 int64 \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 Lotus Europa   \u2502    30.4 \u2502     4 \u2502    95.1 \u2502   113 \u2502    3.77 \u2502   1.513 \u2502   16.90 \u2502     1 \u2502     1 \u2502     5 \u2502     2 \u2502\n\u2502 Fiat 128       \u2502    32.4 \u2502     4 \u2502    78.7 \u2502    66 \u2502    4.08 \u2502   2.200 \u2502   19.47 \u2502     1 \u2502     1 \u2502     4 \u2502     1 \u2502\n\u2502 Honda Civic    \u2502    30.4 \u2502     4 \u2502    75.7 \u2502    52 \u2502    4.93 \u2502   1.615 \u2502   18.52 \u2502     1 \u2502     1 \u2502     4 \u2502     2 \u2502\n\u2502 Toyota Corolla \u2502    33.9 \u2502     4 \u2502    71.1 \u2502    65 \u2502    4.22 \u2502   1.835 \u2502   19.90 \u2502     1 \u2502     1 \u2502     4 \u2502     1 \u2502\n\u2502 Fiat X1-9      \u2502    27.3 \u2502     4 \u2502    79.0 \u2502    66 \u2502    4.08 \u2502   1.935 \u2502   18.90 \u2502     1 \u2502     1 \u2502     4 \u2502     1 \u2502\n\u2502 Porsche 914-2  \u2502    26.0 \u2502     4 \u2502   120.3 \u2502    91 \u2502    4.43 \u2502   2.140 \u2502   16.70 \u2502     0 \u2502     1 \u2502     5 \u2502     2 \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n
 
TidierDB
 
@chain t(mtcars) begin\n       @filter((mpg > 22 && drat > 4) || hp == 113)\n       @collect\nend\n
 
6\u00d712 DataFrame\n Row \u2502 model           mpg       cyl     disp      hp      drat      wt        qsec      vs      am      gear    carb\n     \u2502 String?         Float64?  Int64?  Float64?  Int64?  Float64?  Float64?  Float64?  Int64?  Int64?  Int64?  Int64?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 Lotus Europa        30.4       4      95.1     113      3.77     1.513     16.9        1       1       5       2\n   2 \u2502 Fiat 128            32.4       4      78.7      66      4.08     2.2       19.47       1       1       4       1\n   3 \u2502 Honda Civic         30.4       4      75.7      52      4.93     1.615     18.52       1       1       4       2\n   4 \u2502 Toyota Corolla      33.9       4      71.1      65      4.22     1.835     19.9        1       1       4       1\n   5 \u2502 Fiat X1-9           27.3       4      79.0      66      4.08     1.935     18.9        1       1       4       1\n   6 \u2502 Porsche 914-2       26.0       4     120.3      91      4.43     2.14      16.7        0       1       5       2\n
 
 
"},{"location":"examples/generated/UserGuide/ibis_comp/#creating-new-columns","title":"Creating new columns","text":"
Both TidierDB and Ibis use mutate/@mutate to add new columns
 
(\n   mtcars\n        .mutate(kpg = _.mpg * 1.61)\n        .select(\"model\", \"kpg\")\n)\n
 
\u250f\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2513\n\u2503 model             \u2503 kpg     \u2503\n\u2521\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2529\n\u2502 string            \u2502 float64 \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 Mazda RX4         \u2502  33.810 \u2502\n\u2502 Mazda RX4 Wag     \u2502  33.810 \u2502\n\u2502 Datsun 710        \u2502  36.708 \u2502\n\u2502 Hornet 4 Drive    \u2502  34.454 \u2502\n\u2502 Hornet Sportabout \u2502  30.107 \u2502\n\u2502 Valiant           \u2502  29.141 \u2502\n\u2502 Duster 360        \u2502  23.023 \u2502\n\u2502 Merc 240D         \u2502  39.284 \u2502\n\u2502 Merc 230          \u2502  36.708 \u2502\n\u2502 Merc 280          \u2502  30.912 \u2502\n\u2502 \u2026                 \u2502       \u2026 \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n
 
@chain t(mtcars) begin\n       @mutate(kpg = mpg * 1.61)\n       @select(model, kpg)\n       @collect\nend\n
 
32\u00d72 DataFrame\n Row \u2502 model              kpg\n     \u2502 String?            Float64?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 Mazda RX4            33.81\n   2 \u2502 Mazda RX4 Wag        33.81\n   3 \u2502 Datsun 710           36.708\n   4 \u2502 Hornet 4 Drive       34.454\n   5 \u2502 Hornet Sportabout    30.107\n   6 \u2502 Valiant              29.141\n  \u22ee  \u2502         \u22ee             \u22ee\n  27 \u2502 Porsche 914-2        41.86\n  28 \u2502 Lotus Europa         48.944\n  29 \u2502 Ford Pantera L       25.438\n  30 \u2502 Ferrari Dino         31.717\n  31 \u2502 Maserati Bora        24.15\n  32 \u2502 Volvo 142E           34.454\n                    20 rows omitted\n
 
 
"},{"location":"examples/generated/UserGuide/ibis_comp/#sorting-columns","title":"Sorting columns","text":"
Ibis uses order_by similar to SQLs ORDER BY
 
mtcars.order_by(_.mpg)\n
 
\u250f\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2513\n\u2503 model               \u2503 mpg     \u2503 cyl   \u2503 disp    \u2503 hp    \u2503 drat    \u2503 wt      \u2503 qsec    \u2503 vs    \u2503 am    \u2503 gear  \u2503 carb  \u2503\n\u2521\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2529\n\u2502 string              \u2502 float64 \u2502 int64 \u2502 float64 \u2502 int64 \u2502 float64 \u2502 float64 \u2502 float64 \u2502 int64 \u2502 int64 \u2502 int64 \u2502 int64 \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 Cadillac Fleetwood  \u2502    10.4 \u2502     8 \u2502   472.0 \u2502   205 \u2502    2.93 \u2502   5.250 \u2502   17.98 \u2502     0 \u2502     0 \u2502     3 \u2502     4 \u2502\n\u2502 Lincoln Continental \u2502    10.4 \u2502     8 \u2502   460.0 \u2502   215 \u2502    3.00 \u2502   5.424 \u2502   17.82 \u2502     0 \u2502     0 \u2502     3 \u2502     4 \u2502\n\u2502 Camaro Z28          \u2502    13.3 \u2502     8 \u2502   350.0 \u2502   245 \u2502    3.73 \u2502   3.840 \u2502   15.41 \u2502     0 \u2502     0 \u2502     3 \u2502     4 \u2502\n\u2502 Duster 360          \u2502    14.3 \u2502     8 \u2502   360.0 \u2502   245 \u2502    3.21 \u2502   3.570 \u2502   15.84 \u2502     0 \u2502     0 \u2502     3 \u2502     4 \u2502\n\u2502 Chrysler Imperial   \u2502    14.7 \u2502     8 \u2502   440.0 \u2502   230 \u2502    3.23 \u2502   5.345 \u2502   17.42 \u2502     0 \u2502     0 \u2502     3 \u2502     4 \u2502\n\u2502 Maserati Bora       \u2502    15.0 \u2502     8 \u2502   301.0 \u2502   335 \u2502    3.54 \u2502   3.570 \u2502   14.60 \u2502     0 \u2502     1 \u2502     5 \u2502     8 \u2502\n\u2502 Merc 450SLC         \u2502    15.2 \u2502     8 \u2502   275.8 \u2502   180 \u2502    3.07 \u2502   3.780 \u2502   18.00 \u2502     0 \u2502     0 \u2502     3 \u2502     3 \u2502\n\u2502 AMC Javelin         \u2502    15.2 \u2502     8 \u2502   304.0 \u2502   150 \u2502    3.15 \u2502   3.435 \u2502   17.30 \u2502     0 \u2502     0 \u2502     3 \u2502     2 \u2502\n\u2502 Dodge Challenger    \u2502    15.5 \u2502     8 \u2502   318.0 \u2502   150 \u2502    2.76 \u2502   3.520 \u2502   16.87 \u2502     0 \u2502     0 \u2502     3 \u2502     2 \u2502\n\u2502 Ford Pantera L      \u2502    15.8 \u2502     8 \u2502   351.0 \u2502   264 \u2502    4.22 \u2502   3.170 \u2502   14.50 \u2502     0 \u2502     1 \u2502     5 \u2502     4 \u2502\n\u2502 \u2026                   \u2502       \u2026 \u2502     \u2026 \u2502       \u2026 \u2502     \u2026 \u2502       \u2026 \u2502       \u2026 \u2502       \u2026 \u2502     \u2026 \u2502     \u2026 \u2502     \u2026 \u2502     \u2026 \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n
 
While TidierDB uses @arrange like TidierData.jl
 
@chain t(mtcars) @arrange(mpg) @collect\n
 
32\u00d712 DataFrame\n Row \u2502 model                mpg       cyl     disp      hp      drat      wt        qsec      vs      am      gear    carb\n     \u2502 String?              Float64?  Int64?  Float64?  Int64?  Float64?  Float64?  Float64?  Int64?  Int64?  Int64?  Int64?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 Cadillac Fleetwood       10.4       8     472.0     205      2.93     5.25      17.98       0       0       3       4\n   2 \u2502 Lincoln Continental      10.4       8     460.0     215      3.0      5.424     17.82       0       0       3       4\n   3 \u2502 Camaro Z28               13.3       8     350.0     245      3.73     3.84      15.41       0       0       3       4\n   4 \u2502 Duster 360               14.3       8     360.0     245      3.21     3.57      15.84       0       0       3       4\n   5 \u2502 Chrysler Imperial        14.7       8     440.0     230      3.23     5.345     17.42       0       0       3       4\n   6 \u2502 Maserati Bora            15.0       8     301.0     335      3.54     3.57      14.6        0       1       5       8\n  \u22ee  \u2502          \u22ee              \u22ee        \u22ee        \u22ee        \u22ee        \u22ee         \u22ee         \u22ee        \u22ee       \u22ee       \u22ee       \u22ee\n  27 \u2502 Porsche 914-2            26.0       4     120.3      91      4.43     2.14      16.7        0       1       5       2\n  28 \u2502 Fiat X1-9                27.3       4      79.0      66      4.08     1.935     18.9        1       1       4       1\n  29 \u2502 Honda Civic              30.4       4      75.7      52      4.93     1.615     18.52       1       1       4       2\n  30 \u2502 Lotus Europa             30.4       4      95.1     113      3.77     1.513     16.9        1       1       5       2\n  31 \u2502 Fiat 128                 32.4       4      78.7      66      4.08     2.2       19.47       1       1       4       1\n  32 \u2502 Toyota Corolla           33.9       4      71.1      65      4.22     1.835     19.9        1       1       4       1\n                                                                                                              20 rows omitted\n
 
 
"},{"location":"examples/generated/UserGuide/ibis_comp/#selecting-columns","title":"Selecting columns","text":"
In Ibis, columns must be prefixed with the table name, or in this case _, or they can be given as a string. Finally to using helper functions like startswith requires importing selectors as above.
 
mtcars.select(s.startswith(\"m\"), \"drat\", _.wt)\n
 
\u250f\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2513\n\u2503 model             \u2503 mpg     \u2503 drat    \u2503 wt      \u2503\n\u2521\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2529\n\u2502 string            \u2502 float64 \u2502 float64 \u2502 float64 \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 Mazda RX4         \u2502    21.0 \u2502    3.90 \u2502   2.620 \u2502\n\u2502 Mazda RX4 Wag     \u2502    21.0 \u2502    3.90 \u2502   2.875 \u2502\n\u2502 Datsun 710        \u2502    22.8 \u2502    3.85 \u2502   2.320 \u2502\n\u2502 Hornet 4 Drive    \u2502    21.4 \u2502    3.08 \u2502   3.215 \u2502\n\u2502 Hornet Sportabout \u2502    18.7 \u2502    3.15 \u2502   3.440 \u2502\n\u2502 Valiant           \u2502    18.1 \u2502    2.76 \u2502   3.460 \u2502\n\u2502 Duster 360        \u2502    14.3 \u2502    3.21 \u2502   3.570 \u2502\n\u2502 Merc 240D         \u2502    24.4 \u2502    3.69 \u2502   3.190 \u2502\n\u2502 Merc 230          \u2502    22.8 \u2502    3.92 \u2502   3.150 \u2502\n\u2502 Merc 280          \u2502    19.2 \u2502    3.92 \u2502   3.440 \u2502\n\u2502 \u2026                 \u2502       \u2026 \u2502       \u2026 \u2502       \u2026 \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n
 
TidierDB does not require names to be prefixed and, like TidierData, tidy column selection with starts_with, ends_with, and contains is supported at base. TidierDB also supports providing column names as strings, although this would only be needed in the setting of renaming a column with a space in it.
 
@chain t(mtcars) @select(starts_with(\"m\"), \"drat\", wt) @collect\n
 
32\u00d72 DataFrame\n32\u00d74 DataFrame\n Row \u2502 model              mpg       drat      wt\n     \u2502 String?            Float64?  Float64?  Float64?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 Mazda RX4              21.0      3.9      2.62\n   2 \u2502 Mazda RX4 Wag          21.0      3.9      2.875\n   3 \u2502 Datsun 710             22.8      3.85     2.32\n   4 \u2502 Hornet 4 Drive         21.4      3.08     3.215\n   5 \u2502 Hornet Sportabout      18.7      3.15     3.44\n   6 \u2502 Valiant                18.1      2.76     3.46\n  \u22ee  \u2502         \u22ee             \u22ee         \u22ee         \u22ee\n  27 \u2502 Porsche 914-2          26.0      4.43     2.14\n  28 \u2502 Lotus Europa           30.4      3.77     1.513\n  29 \u2502 Ford Pantera L         15.8      4.22     3.17\n  30 \u2502 Ferrari Dino           19.7      3.62     2.77\n  31 \u2502 Maserati Bora          15.0      3.54     3.57\n  32 \u2502 Volvo 142E             21.4      4.11     2.78\n                                        20 rows omitted\n
 
 
"},{"location":"examples/generated/UserGuide/ibis_comp/#multi-step-queries-and-summarizing","title":"Multi step queries and summarizing","text":"
Aggregating data is done with aggregate in ibis and @summarize in TidierDB. There is a slight difference in grouping data. Ibis uses by = within the aggregate call vs TidierDB adheres to @group_by convention
 
mtcars.aggregate(\n    total_hp=_.hp.sum(),\n    avg_hp=_.hp.mean(),\n    having=_.hp.sum() < 1000,\n    by=['cyl']\n)\n
 
\u250f\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2513\n\u2503 cyl   \u2503 total_hp \u2503 avg_hp     \u2503\n\u2521\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2529\n\u2502 int64 \u2502 int64    \u2502 float64    \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502     6 \u2502      856 \u2502 122.285714 \u2502\n\u2502     4 \u2502      909 \u2502  82.636364 \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n
 
In TidierDB, @filter will automatically determine whether the criteria belong in a WHERE or HAVING in SQL clause.
 
@chain t(mtcars) begin\n    @group_by(cyl)\n    @summarize(total_hp = sum(hp),\n               avg_hp = avg(hp))\n    @filter(total_hp < 1000)\n    @collect\nend\n
 
2\u00d73 DataFrame\n Row \u2502 cyl     total_hp  avg_hp\n     \u2502 Int64?  Int128?   Float64?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502      6       856  122.286\n   2 \u2502      4       909   82.6364\n
 
 
"},{"location":"examples/generated/UserGuide/ibis_comp/#renaming-columns","title":"Renaming columns","text":"
Both tools use rename/@rename to rename columns
 
mtcars.rename(make_model = \"model\").select(_.make_model)\n
 
\u250f\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2513\n\u2503 make_model        \u2503\n\u2521\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2529\n\u2502 string            \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 Mazda RX4         \u2502\n\u2502 Mazda RX4 Wag     \u2502\n\u2502 Datsun 710        \u2502\n\u2502 Hornet 4 Drive    \u2502\n\u2502 Hornet Sportabout \u2502\n\u2502 Valiant           \u2502\n\u2502 Duster 360        \u2502\n\u2502 Merc 240D         \u2502\n\u2502 Merc 230          \u2502\n\u2502 Merc 280          \u2502\n\u2502 \u2026                 \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n
 
@chain t(mtcars) @rename(model_make = model) @select(model_make) @collect\n
 
32\u00d71 DataFrame\n Row \u2502 model_make\n     \u2502 String?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 Mazda RX4\n   2 \u2502 Mazda RX4 Wag\n   3 \u2502 Datsun 710\n   4 \u2502 Hornet 4 Drive\n   5 \u2502 Hornet Sportabout\n   6 \u2502 Valiant\n  \u22ee  \u2502         \u22ee\n  27 \u2502 Porsche 914-2\n  28 \u2502 Lotus Europa\n  29 \u2502 Ford Pantera L\n  30 \u2502 Ferrari Dino\n  31 \u2502 Maserati Bora\n  32 \u2502 Volvo 142E\n          20 rows omitted\n
 
This page was generated using Literate.jl.
"},{"location":"examples/generated/UserGuide/key_differences/","title":"Key Differences from TidierData.jl","text":"
There are a few important syntax and behavior differences between TidierDB.jl and TidierData.jl outlined below.
 
 
"},{"location":"examples/generated/UserGuide/key_differences/#creating-a-database","title":"Creating a database","text":"
For these examples we will use DuckDB, the default backend, although SQLite, Postgres, MySQL, MariaDB, MSSQL, and ClickHouse are possible. If you have an existing DuckDB connection, then this step is not required. For these examples, we will create a data frame and copy it to an in-memory DuckDB database.
 
using DataFrames, TidierDB\n\ndf = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9],\n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10],\n                        value = repeat(1:5, 2),\n                        percent = 0.1:0.1:1.0);\n\ndb = connect(duckdb());\n\ncopy_to(db, df, \"df_mem\"); # copying over the data frame to an in-memory database\n
 
 
"},{"location":"examples/generated/UserGuide/key_differences/#row-ordering","title":"Row ordering","text":"
DuckDB benefits from aggressive parallelization of pipelines. This means that if you have multiple threads enabled in Julia, which you can check or set using Threads.nthreads(), DuckDB will use multiple threads. However, because many operations are multi-threaded, the resulting row order is inconsistent. If row order needs to be deterministic for your use case, make sure to apply an @arrange(column_name_1, column_name_2, etc...) prior to collecting the results.
 
 
"},{"location":"examples/generated/UserGuide/key_differences/#starting-a-chain","title":"Starting a chain","text":"
When using TidierDB, db_table(connection, :table_name) is used to start a chain.
 
 
"},{"location":"examples/generated/UserGuide/key_differences/#grouped-mutation","title":"Grouped mutation","text":"
In TidierDB, when performing @group_by then @mutate, the table will be ungrouped after applying all of the mutations in the clause to the grouped data. To perform subsequent grouped operations, the user would have to regroup the data. This is demonstrated below.
 
@chain db_table(db, :df_mem) begin\n    @group_by(groups)\n    @summarize(mean_percent = mean(percent))\n    @collect\n end\n
 2\u00d72 DataFrame Rowgroupsmean_percentString?Float64?1bb0.52aa0.6 
Regrouping following @mutate
 
@chain db_table(db, :df_mem) begin\n    @group_by(groups)\n    @mutate(max = maximum(percent), min = minimum(percent))\n    @group_by(groups)\n    @summarise(mean_percent = mean(percent))\n    @collect\nend\n
 2\u00d72 DataFrame Rowgroupsmean_percentString?Float64?1bb0.52aa0.6 
 
"},{"location":"examples/generated/UserGuide/key_differences/#joining","title":"Joining","text":"
There is one key difference for joining:
 
The column on both the new and old table must be specified. They do not need to be the same, and given SQL behavior where both columns are kept when joining two tables, it is preferable if they have different names. This avoids \"ambiguous reference\" errors that would otherwise come up and complicate the use of tidy selection for columns. Athena has an additional slight difference given the need for parameters, which is covered in the Athena documentation page.
 
df2 = DataFrame(id2 = [\"AA\", \"AC\", \"AE\", \"AG\", \"AI\", \"AK\", \"AM\"],\n                category = [\"X\", \"Y\", \"X\", \"Y\", \"X\", \"Y\", \"X\"],\n                score = [88, 92, 77, 83, 95, 68, 74]);\n\ncopy_to(db, df2, \"df_join\");\n\n@chain db_table(db, :df_mem) begin\n    @left_join(df_join, id2, id)\n    @collect\nend\n
 10\u00d77 DataFrame Rowidgroupsvaluepercentid2categoryscoreString?String?Int64?Float64?String?String?Int64?1AAbb10.1AAX882ACbb30.3ACY923AEbb50.5AEX774AGbb20.7AGY835AIbb40.9AIX956ABaa20.2missingmissingmissing7ADaa40.4missingmissingmissing8AFaa10.6missingmissingmissing9AHaa30.8missingmissingmissing10AJaa51.0missingmissingmissing 
 
"},{"location":"examples/generated/UserGuide/key_differences/#differences-in-case_when","title":"Differences in case_when()","text":"
In TidierDB, after the clause is completed, the result for the new column should is separated by a comma , in contrast to TidierData.jl, where the result for the new column is separated by a => .
 
@chain db_table(db, :df_mem) begin\n    @mutate(new_col = case_when(percent > .5, \"Pass\",  # in TidierData, percent > .5 => \"Pass\",\n                                percent <= .5, \"Try Again\", # percent <= .5 => \"Try Again\"\n                                true, \"middle\"))\n    @collect\n end\n
 10\u00d75 DataFrame Rowidgroupsvaluepercentnew_colString?String?Int64?Float64?String?1AAbb10.1Try Again2ABaa20.2Try Again3ACbb30.3Try Again4ADaa40.4Try Again5AEbb50.5Try Again6AFaa10.6Pass7AGbb20.7Pass8AHaa30.8Pass9AIbb40.9Pass10AJaa51.0Pass 
 
"},{"location":"examples/generated/UserGuide/key_differences/#interpolation","title":"Interpolation","text":"
To use !! Interpolation, instead of being able to define the alternate names/value in the global context, the user has to use @interpolate. This will hopefully be fixed in future versions. Otherwise, the behavior is generally the same, although this creates friction around calling functions.
 
Also, when using interpolation with exponenents, the interpolated value must go inside of parenthesis.
 
@interpolate((test, :percent)); # this still supports strings, vectors of names, and values\n\n@chain db_table(db, :df_mem) begin\n    @mutate(new_col = case_when((!!test)^2 > .5, \"Pass\",\n                                (!!test)^2 < .5, \"Try Again\",\n                                \"middle\"))\n    @collect\nend\n
 
10\u00d75 DataFrame\n Row \u2502 id       groups   value   percent   new_col\n     \u2502 String?  String?  Int64?  Float64?  String?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA       bb            1       0.1  Try Again\n   2 \u2502 AB       aa            2       0.2  Try Again\n   3 \u2502 AC       bb            3       0.3  Try Again\n  \u22ee  \u2502    \u22ee        \u22ee       \u22ee        \u22ee          \u22ee\n   8 \u2502 AH       aa            3       0.8  Pass\n   9 \u2502 AI       bb            4       0.9  Pass\n  10 \u2502 AJ       aa            5       1.0  Pass\n                                       4 rows omitted\n
 
 
"},{"location":"examples/generated/UserGuide/key_differences/#slicing-ties","title":"Slicing ties","text":"
slice_min() and @slice_max() will always return ties due to SQL behavior.
 
This page was generated using Literate.jl.
"},{"location":"examples/generated/UserGuide/outofmemex/","title":"Working With Larger than RAM Datasets","text":"
While using the DuckDB backend, TidierDB's lazy intferace enables querying datasets larger than your available RAM.
 
To illustrate this, we will recreate the Hugging Face x Polars example. The final table results are shown below and in this Hugging Face x DuckDB example
 
First we will load TidierDB, set up a local database and then set the URLs for the 2 training datasets from huggingface.co
 
using TidierDB\ndb = connect(duckdb())\n\nurls = [\"https://huggingface.co/datasets/blog_authorship_corpus/resolve/refs%2Fconvert%2Fparquet/blog_authorship_corpus/train/0000.parquet\",\n \"https://huggingface.co/datasets/blog_authorship_corpus/resolve/refs%2Fconvert%2Fparquet/blog_authorship_corpus/train/0001.parquet\"];\n
 
Here, we pass the vector of URLs to db_table, which will not copy them into memory. Since these datasets are so large, we will also set stream = true in @collect to stream the results. If we wanted to read all the files in the folder we could have replace the 0000 with * (wildcard) db_table(db, \"Path/to/folder/*.parquet\") Of note, reading these files from URLs is not as rapid as reading them from local files.
 
@chain db_table(db, urls) begin\n    @group_by(horoscope)\n    @summarise(count = n(), avg_blog_length = mean(length(text)))\n    @arrange(desc(count))\n    @aside @show_query _\n    @collect(stream = true)\nend\n
 
Placing @aside @show_query _ before @collect above lets us see the SQL query and collect it to a local DataFrame at the same time.
 
SELECT horoscope, COUNT(*) AS count, AVG(length(text)) AS avg_blog_length\n        FROM read_parquet(['https://huggingface.co/datasets/blog_authorship_corpus/resolve/refs%2Fconvert%2Fparquet/blog_authorship_corpus/train/0000.parquet', 'https://huggingface.co/datasets/blog_authorship_corpus/resolve/refs%2Fconvert%2Fparquet/blog_authorship_corpus/train/0001.parquet'])\n        GROUP BY horoscope\n        ORDER BY avg_blog_length DESC\n12\u00d73 DataFrame\n Row \u2502 horoscope    count   avg_blog_length\n     \u2502 String?      Int64?  Float64?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 Aquarius      49568         1125.83\n   2 \u2502 Cancer        63512         1097.96\n   3 \u2502 Libra         60304         1060.61\n   4 \u2502 Capricorn     49402         1059.56\n   5 \u2502 Sagittarius   50431         1057.46\n   6 \u2502 Leo           58010         1049.6\n   7 \u2502 Taurus        61571         1022.69\n   8 \u2502 Gemini        52925         1020.26\n   9 \u2502 Scorpio       56495         1014.03\n  10 \u2502 Pisces        53812         1011.75\n  11 \u2502 Virgo         64629          996.684\n  12 \u2502 Aries         69134          918.081\n
 
To learn more about memory efficient queries on larger than RAM files, this blog from DuckDB will help maximize your local db
 
This page was generated using Literate.jl.
"},{"location":"examples/generated/UserGuide/s3viaduckdb/","title":"S3 + DuckDB + TidierDB","text":"
TidierDB allows you leverage DuckDB's seamless database integration.
 
Using DuckDB, you can connect to an AWS or GoogleCloud Database to query directly without making any local copies.
 
You can also use DBInterface.execute to set up any DuckDB database connection you need and then use that db to query with TidierDB
 
using TidierDB\n\n#Connect to Google Cloud via DuckDB\n#google_db = connect(duckdb(), :gbq, access_key=\"string\", secret_key=\"string\")\n\n#Connect to AWS via DuckDB\naws_db = connect(duckdb(), :aws, aws_access_key_id= \"string\",\n                                aws_secret_access_key= \"string\",\n                                aws_region=\"us-east-1\")\ns3_csv_path = \"s3://path/to_data.csv\"\n\n@chain db_table(aws_db, s3_csv_path) begin\n    @filter(!starts_with(column1, \"M\"))\n    @group_by(cyl)\n    @summarize(mpg = mean(mpg))\n    @mutate(mpg_squared = mpg^2,\n               mpg_rounded = round(mpg),\n               mpg_efficiency = case_when(\n                                 mpg >= cyl^2 , \"efficient\",\n                                 mpg < 15.2 , \"inefficient\",\n                                 \"moderate\"))\n    @filter(mpg_efficiency in (\"moderate\", \"efficient\"))\n    @arrange(desc(mpg_rounded))\n    @collect\nend\n
 
2\u00d75 DataFrame\n Row \u2502 cyl     mpg       mpg_squared  mpg_rounded  mpg_efficiency\n     \u2502 Int64?  Float64?  Float64?     Float64?     String?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502      4   27.3444      747.719         27.0  efficient\n   2 \u2502      6   19.7333      389.404         20.0  moderate\n
 
This page was generated using Literate.jl.
"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Home","text":""},{"location":"#what-is-tidierdbjl","title":"What is TidierDB.jl?","text":"
TiderDB.jl is a 100% Julia implementation of the dbplyr R package, and similar to Python's ibis package.
 
The main goal of TidierDB.jl is to bring the syntax of Tidier.jl to multiple SQL backends, making it possible to analyze data directly on databases without needing to copy the entire database into memory.
 
 
"},{"location":"#currently-supported-backends-include","title":"Currently supported backends include:","text":"
     
  • DuckDB (the default) duckdb()
    •  
  • ClickHouse clickhouse()
    •  
  • SQLite sqlite()
    •  
  • MySQL and MariaDB mysql()
    •  
  • MSSQL mssql()
    •  
  • Postgres postgres()
    •  
  • Athena athena()
    •  
  • Snowflake snowflake()
    •  
  • Google Big Query gbq()
    •  
  • Oracle oracle()
    •  
  • Databricks databricks()
    •  
 
Change the backend using set_sql_mode() - for example  - set_sql_mode(databricks())
 
 
"},{"location":"#installation","title":"Installation","text":"
For the stable version:
 
] add TidierDB\n
 
TidierDB.jl currently supports the following top-level macros:
 
     
  • @arrange
    •  
  • @group_by
    •  
  • @filter
    •  
  • @select
    •  
  • @mutate, which supports across()
    •  
  • @summarize and @summarise, which supports across()
    •  
  • @distinct
    •  
  • @left_join, @right_join, @inner_join, @anti_join, @full_join, and @semi_join (slight syntax differences from TidierData.jl)
    •  
  • @count
    •  
  • @slice_min, @slice_max, @slice_sample
    •  
  • @window_order and window_frame
    •  
  • @show_query
    •  
  • @collect
    •  
 
Supported helper functions for most backends include:
 
     
  • across()
    •  
  • desc()
    •  
  • if_else() and case_when()
    •  
  • n()
    •  
  • starts_with(), ends_with(), and contains()
    •  
  • as_float(), as_integer(), and as_string()
    •  
  • is_missing()
    •  
  • missing_if() and replace_missing()
    •  
 
From TidierStrings.jl:
 
     
  • str_detect, str_replace, str_replace_all, str_remove_all, str_remove
    •  
 
From TidierDates.jl:
 
     
  • year, month, day, hour, min, second, floor_date, difftime
    •  
 
Supported aggregate functions (as supported by the backend) with more to come
 
     
  • mean, minimium, maximum, std, sum, cumsum, cor, cov, var
    •  
  • @summarize supports any SQL aggregate function in addition to the list above. Simply write the function as written in SQL syntax and it will work
    •  
 
When using the DuckDB backend, if db_table recieves a file path ( .parquet, .json, .csv, iceberg or delta), it does not copy it into memory. This allows for queries on files too big for memory. db_table also supports S3 bucket locations via DuckDB.
 
 
"},{"location":"#what-is-the-recommended-way-to-use-tidierdb","title":"What is the recommended way to use TidierDB?","text":"
Typically, you will want to use TidierDB alongside TidierData because there are certain functionality (such as pivoting) which are only supported in TidierData and can only be performed on data frames.
 
Our recommended path for using TidierDB is to import the package so that there are no namespace conflicts with TidierData. Once TidierDB is integrated with Tidier, then Tidier will automatically load the packages in this fashion.
 
First, let's develop and execute a query using TidierDB. Notice that all top-level macros and functions originating from TidierDB start with a DB prefix. Any functions defined within macros do not need to be prefixed within DB because they are actually pseudofunctions that are in actuality converted into SQL code.
 
Even though the code reads similarly to TidierData, note that no computational work actually occurs until you run DB.@collect(), which runs the SQL query and instantiates the result as a DataFrame.
 
using TidierData\nimport TidierDB as DB\n\ndb = DB.connect(DB.duckdb());\npath_or_name = \"https://gist.githubusercontent.com/seankross/a412dfbd88b3db70b74b/raw/5f23f993cd87c283ce766e7ac6b329ee7cc2e1d1/mtcars.csv\"\n\n@chain DB.db_table(db, path_or_name) begin\n    DB.@filter(!starts_with(model, \"M\"))\n    DB.@group_by(cyl)\n    DB.@summarize(mpg = mean(mpg))\n    DB.@mutate(mpg_squared = mpg^2, \n               mpg_rounded = round(mpg), \n               mpg_efficiency = case_when(\n                                 mpg >= cyl^2 , \"efficient\",\n                                 mpg < 15.2 , \"inefficient\",\n                                 \"moderate\"))            \n    DB.@filter(mpg_efficiency in (\"moderate\", \"efficient\"))\n    DB.@arrange(desc(mpg_rounded))\n    DB.@collect\nend\n
 
2\u00d75 DataFrame\n Row \u2502 cyl     mpg       mpg_squared  mpg_rounded  mpg_efficiency \n     \u2502 Int64?  Float64?  Float64?     Float64?     String?        \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502      4   27.3444      747.719         27.0  efficient\n   2 \u2502      6   19.7333      389.404         20.0  moderate\n
 
 
"},{"location":"#what-if-we-wanted-to-pivot-the-result","title":"What if we wanted to pivot the result?","text":"
We cannot do this using TidierDB. However, we can call @pivot_longer() from TidierData after the result of the query has been instantiated as a DataFrame, like this: 
 
@chain DB.db_table(db, path_or_name) begin\n    DB.@filter(!starts_with(model, \"M\"))\n    DB.@group_by(cyl)\n    DB.@summarize(mpg = mean(mpg))\n    DB.@mutate(mpg_squared = mpg^2, \n               mpg_rounded = round(mpg), \n               mpg_efficiency = case_when(\n                                 mpg >= cyl^2 , \"efficient\",\n                                 mpg < 15.2 , \"inefficient\",\n                                 \"moderate\"))            \n    DB.@filter(mpg_efficiency in (\"moderate\", \"efficient\"))\n    DB.@arrange(desc(mpg_rounded))\n    DB.@collect\n    @pivot_longer(everything(), names_to = \"variable\", values_to = \"value\")\nend\n
 
10\u00d72 DataFrame\n Row \u2502 variable        value     \n     \u2502 String          Any       \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 cyl             4\n   2 \u2502 cyl             6\n   3 \u2502 mpg             27.3444\n   4 \u2502 mpg             19.7333\n   5 \u2502 mpg_squared     747.719\n   6 \u2502 mpg_squared     389.404\n   7 \u2502 mpg_rounded     27.0\n   8 \u2502 mpg_rounded     20.0\n   9 \u2502 mpg_efficiency  efficient\n  10 \u2502 mpg_efficiency  moderate\n
 
 
"},{"location":"#what-sql-query-does-tidierdb-generate-for-a-given-piece-of-julia-code","title":"What SQL query does TidierDB generate for a given piece of Julia code?","text":"
We can replace DB.collect() with DB.@show_query to reveal the underlying SQL query being generated by TidierDB. To handle complex queries, TidierDB makes heavy use of Common Table Expressions (CTE), which are a useful tool to organize long queries.
 
@chain DB.db_table(db, path_or_name) begin\n    DB.@filter(!starts_with(model, \"M\"))\n    DB.@group_by(cyl)\n    DB.@summarize(mpg = mean(mpg))\n    DB.@mutate(mpg_squared = mpg^2, \n               mpg_rounded = round(mpg), \n               mpg_efficiency = case_when(\n                                 mpg >= cyl^2 , \"efficient\",\n                                 mpg < 15.2 , \"inefficient\",\n                                 \"moderate\"))            \n    DB.@filter(mpg_efficiency in (\"moderate\", \"efficient\"))\n    DB.@arrange(desc(mpg_rounded))\n    DB.@show_query\nend\n
 
WITH cte_1 AS (\nSELECT *\n        FROM mtcars\n        WHERE NOT (starts_with(model, 'M'))),\ncte_2 AS (\nSELECT cyl, AVG(mpg) AS mpg\n        FROM cte_1\n        GROUP BY cyl),\ncte_3 AS (\nSELECT  cyl, mpg, POWER(mpg, 2) AS mpg_squared, ROUND(mpg) AS mpg_rounded, CASE WHEN mpg >= POWER(cyl, 2) THEN 'efficient' WHEN mpg < 15.2 THEN 'inefficient' ELSE 'moderate' END AS mpg_efficiency\n        FROM cte_2 ),\ncte_4 AS (\nSELECT *\n        FROM cte_3\n        WHERE mpg_efficiency in ('moderate', 'efficient'))  \nSELECT *\n        FROM cte_4  \n        ORDER BY mpg_rounded DESC\n
 
 
"},{"location":"#tidierdb-is-already-quite-fully-featured-supporting-advanced-tidierdata-functions-like-across-for-multi-column-selection","title":"TidierDB is already quite fully-featured, supporting advanced TidierData functions like across() for multi-column selection.","text":"
@chain DB.db_table(db, path_or_name) begin\n    DB.@group_by(cyl)\n    DB.@summarize(across((starts_with(\"a\"), ends_with(\"s\")), (mean, sum)))\n    DB.@collect\nend\n
 
3\u00d75 DataFrame\n Row \u2502 cyl     mean_am   mean_vs   sum_am   sum_vs  \n     \u2502 Int64?  Float64?  Float64?  Int128?  Int128? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502      4  0.727273  0.909091        8       10\n   2 \u2502      6  0.428571  0.571429        3        4\n   3 \u2502      8  0.142857  0.0             2        0\n
 
Bang bang !! interpolation for columns and values is also supported.
 
There are a few subtle but important differences from Tidier.jl outlined here.
 
 
"},{"location":"#missing-a-function-or-backend","title":"Missing a function or backend?","text":"
You can use any existing SQL function within @mutate with the correct SQL syntax and it should just work.
 
But if you run into problems please open an issue, and we will be happy to take a look!
"},{"location":"reference/","title":"Reference","text":""},{"location":"reference/#index","title":"Index","text":"
     
  • TidierDB.connect
    •  
  • TidierDB.copy_to
    •  
  • TidierDB.db_table
    •  
  • TidierDB.@anti_join
    •  
  • TidierDB.@arrange
    •  
  • TidierDB.@collect
    •  
  • TidierDB.@count
    •  
  • TidierDB.@distinct
    •  
  • TidierDB.@filter
    •  
  • TidierDB.@full_join
    •  
  • TidierDB.@group_by
    •  
  • TidierDB.@head
    •  
  • TidierDB.@inner_join
    •  
  • TidierDB.@interpolate
    •  
  • TidierDB.@left_join
    •  
  • TidierDB.@mutate
    •  
  • TidierDB.@rename
    •  
  • TidierDB.@right_join
    •  
  • TidierDB.@select
    •  
  • TidierDB.@semi_join
    •  
  • TidierDB.@slice_max
    •  
  • TidierDB.@slice_min
    •  
  • TidierDB.@slice_sample
    •  
  • TidierDB.@summarise
    •  
  • TidierDB.@summarize
    •  
  • TidierDB.@window_frame
    •  
  • TidierDB.@window_order
    •  
"},{"location":"reference/#reference-exported-functions","title":"Reference - Exported functions","text":"
# TidierDB.connect \u2014 Method.
 
connect(backend; kwargs...)\n
 
This function establishes a database connection based on the specified backend and connection parameters and sets the SQL mode
 
Arguments
 
     
  •  
    backend: type specifying the database backend to connect to. Supported backends are:
     
       
    • duckdb(), sqlite()(SQLite), mssql(), mysql()(for MariaDB and MySQL), clickhouse(), postgres()
      •  
    •  
      kwargs: Keyword arguments specifying the connection parameters for the selected backend. The required parameters vary depending on the backend:
       
      •  
    •  
      MySQL:
       
         
      • host: The host name or IP address of the MySQL server. Default is \"localhost\".
        •  
      • user: The username for authentication. Default is an empty string.
        •  
      • password: The password for authentication.
        •  
      • db: The name of the database to connect to (optional).
        •  
      • port: The port number of the MySQL server (optional).
        •  
       
      •  
     
    •  
 
Returns
 
     
  • A database connection object based on the selected backend.
    •  
 
Examples
 
# Connect to MySQL\n# conn = connect(mysql(); host=\"localhost\", user=\"root\", password=\"password\", db=\"mydb\")\n# Connect to PostgreSQL using LibPQ\n# conn = connect(postgres(); host=\"localhost\", dbname=\"mydb\", user=\"postgres\", password=\"password\")\n# Connect to ClickHouse\n# conn = connect(clickhouse(); host=\"localhost\", port=9000, database=\"mydb\", user=\"default\", password=\"\")\n# Connect to SQLite\n# conn = connect(sqlite())\n# Connect to Google Big Query\n# conn = connect(gbq(), \"json_user_key_path\", \"project_id\")\n# Connect to Snowflake\n# conn = connect(snowflake(), \"ac_id\", \"token\", \"Database_name\", \"Schema_name\", \"warehouse_name\")\n# Connect to DuckDB\n# connect to Google Cloud via DuckDB\n# google_db = connect(duckdb(), :gbq, access_key=\"string\", secret_key=\"string\")\n# Connect to AWS via DuckDB\n# aws_db = connect2(duckdb(), :aws, aws_access_key_id=get(ENV, \"AWS_ACCESS_KEY_ID\", \"access_key\"), aws_secret_access_key=get(ENV, \"AWS_SECRET_ACCESS_KEY\", \"secret_access key\"), aws_region=get(ENV, \"AWS_DEFAULT_REGION\", \"us-east-1\"))\n# Connect to MotherDuck\n# connect(duckdb(), \"token\") for first connection, vs connect(duckdb(), \"md:\") for reconnection\njulia> db = connect(duckdb())\nDuckDB.Connection(\":memory:\")\n
 
source
 
# TidierDB.copy_to \u2014 Method.
 
   copy_to(conn, df_or_path, \"name\")\n
 
Allows user to copy a df to the database connection. Currently supports DuckDB, SQLite, MySql
 
Arguments
 
-conn: the database connection -df: dataframe to be copied or path to serve as source. With DuckDB, path supports .csv, .json, .parquet to be used without copying intermediary df. -name: name as string for the database to be used
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"test\");\n
 
source
 
# TidierDB.db_table \u2014 Function.
 
db_table(database, table_name, athena_params, delta = false, iceberg = false)\n
 
db_table starts the underlying SQL query struct, adding the metadata and table. If paths are passed directly to db_table instead of a  name it will not copy it to memory, but rather ready directly from the file.
 
Arguments
 
     
  • database: The Database or connection object
    •  
  •  
    table_name: tablename as a string (name, local path, or URL).     - CSV/TSV       - Parquet     - Json      - Iceberg     - Delta     - S3 tables from AWS or Google Cloud 
     
       
    • DuckDB and ClickHouse support vectors of paths and URLs.
      •  
    • DuckDB and ClickHouse also support use of * wildcards to read all files of a type in a location such as:
      •  
    • db_table(db, \"Path/to/testing_files/*.parquet\")
      •  
    • delta: must be true to read delta files
      •  
    • iceberg: must be true to read iceberg finalize_ctes
      •  
     
    •  
 
Example
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> db_table(db, \"df_mem\")\nTidierDB.SQLQuery(\"\", \"df_mem\", \"\", \"\", \"\", \"\", \"\", \"\", false, false, 4\u00d74 DataFrame\n Row \u2502 name     type     current_selxn  table_name \n     \u2502 String?  String?  Int64          String     \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 id       VARCHAR              1  df_mem\n   2 \u2502 groups   VARCHAR              1  df_mem\n   3 \u2502 value    BIGINT               1  df_mem\n   4 \u2502 percent  DOUBLE               1  df_mem, false, DuckDB.Connection(\":memory:\"), TidierDB.CTE[], 0, nothing)\n
 
source
 
# TidierDB.@anti_join \u2014 Macro.
 
@anti_join(sql_query, join_table, new_table_col, orignal_table_col)\n
 
Perform an anti join between two SQL queries based on a specified condition.  This syntax here is slightly different than TidierData.jl, however, because  SQL does not drop the joining column, for the metadata storage, it is  preferrable for the names to be different 
 
Arguments
 
     
  • sql_query: The primary SQL query to operate on.
    •  
  • join_table: The secondary SQL table to join with the primary query table.
    •  
  • new_table_col: Column from the new table that matches for join.
    •  
  • orignal_table_col: Column from the original table that matches for join.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> df2 = DataFrame(id2 = [\"AA\", \"AC\", \"AE\", \"AG\", \"AI\", \"AK\", \"AM\"],\n                category = [\"X\", \"Y\", \"X\", \"Y\", \"X\", \"Y\", \"X\"],\n                score = [88, 92, 77, 83, 95, 68, 74]);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> copy_to(db, df2, \"df_join\");\n\njulia> @chain db_table(db, :df_mem) begin\n        @anti_join(df_join, id2, id)\n        @collect\n       end\n5\u00d74 DataFrame\n Row \u2502 id       groups   value   percent  \n     \u2502 String?  String?  Int64?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AB       aa            2       0.2\n   2 \u2502 AD       aa            4       0.4\n   3 \u2502 AF       aa            1       0.6\n   4 \u2502 AH       aa            3       0.8\n   5 \u2502 AJ       aa            5       1.0\n
 
source
 
# TidierDB.@arrange \u2014 Macro.
 
@arrange(sql_query, columns...)\n
 
Order SQL table rows based on specified column(s).
 
Arguments
 
     
  • sql_query: The SQL query to operate on.
    •  
  • columns: Columns to order the rows by. Can include multiple columns for nested sorting. Wrap column name with desc() for descending order.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @arrange(value, desc(percent))\n         @collect\n       end\n10\u00d74 DataFrame\n Row \u2502 id       groups   value   percent  \n     \u2502 String?  String?  Int64?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AF       aa            1       0.6\n   2 \u2502 AA       bb            1       0.1\n   3 \u2502 AG       bb            2       0.7\n   4 \u2502 AB       aa            2       0.2\n   5 \u2502 AH       aa            3       0.8\n   6 \u2502 AC       bb            3       0.3\n   7 \u2502 AI       bb            4       0.9\n   8 \u2502 AD       aa            4       0.4\n   9 \u2502 AJ       aa            5       1.0\n  10 \u2502 AE       bb            5       0.5\n
 
source
 
# TidierDB.@collect \u2014 Macro.
 
@collect(sql_query, stream = false)\n
 
db_table starts the underlying SQL query struct, adding the metadata and table. 
 
Arguments
 
     
  • sql_query: The SQL query to operate on.
    •  
  • stream: optional streaming for query/execution of results when using duck db. Defaults to false
    •  
 
Example
 
julia> db = connect(duckdb());\n\njulia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @collect db_table(db, \"df_mem\")\n10\u00d74 DataFrame\n Row \u2502 id       groups   value   percent  \n     \u2502 String?  String?  Int64?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA       bb            1       0.1\n   2 \u2502 AB       aa            2       0.2\n   3 \u2502 AC       bb            3       0.3\n   4 \u2502 AD       aa            4       0.4\n   5 \u2502 AE       bb            5       0.5\n   6 \u2502 AF       aa            1       0.6\n   7 \u2502 AG       bb            2       0.7\n   8 \u2502 AH       aa            3       0.8\n   9 \u2502 AI       bb            4       0.9\n  10 \u2502 AJ       aa            5       1.0\n
 
source
 
# TidierDB.@count \u2014 Macro.
 
@count(sql_query, columns...)\n
 
Count the number of rows grouped by specified column(s).
 
Arguments
 
     
  • sql_query: The SQL query to operate on.
    •  
  • columns: Columns to group by before counting. If no columns are specified, counts all rows in the query.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @count(groups)\n         @arrange(groups)\n         @collect\n       end\n2\u00d72 DataFrame\n Row \u2502 groups   count  \n     \u2502 String?  Int64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 aa            5\n   2 \u2502 bb            5\n
 
source
 
# TidierDB.@distinct \u2014 Macro.
 
@distinct(sql_query, columns...)\n
 
Select distinct rows based on specified column(s). Distinct works differently in TidierData vs SQL and therefore TidierDB. Distinct will also select only the only columns it is given (or all if given none)
 
Arguments
 
sql_query: The SQL query to operate on. columns: Columns to determine uniqueness. If no columns are specified, all columns are used to identify distinct rows.
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @distinct(value)\n         @arrange(value)\n         @collect\n       end\n5\u00d71 DataFrame\n Row \u2502 value  \n     \u2502 Int64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502      1\n   2 \u2502      2\n   3 \u2502      3\n   4 \u2502      4\n   5 \u2502      5\n\njulia> @chain db_table(db, :df_mem) begin\n         @distinct\n         @arrange(id)\n         @collect\n       end\n10\u00d74 DataFrame\n Row \u2502 id       groups   value   percent  \n     \u2502 String?  String?  Int64?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA       bb            1       0.1\n   2 \u2502 AB       aa            2       0.2\n   3 \u2502 AC       bb            3       0.3\n   4 \u2502 AD       aa            4       0.4\n   5 \u2502 AE       bb            5       0.5\n   6 \u2502 AF       aa            1       0.6\n   7 \u2502 AG       bb            2       0.7\n   8 \u2502 AH       aa            3       0.8\n   9 \u2502 AI       bb            4       0.9\n  10 \u2502 AJ       aa            5       1.0\n
 
source
 
# TidierDB.@filter \u2014 Macro.
 
@filter(sql_query, conditions...)\n
 
Filter rows in a SQL table based on specified conditions.
 
Arguments
 
     
  • sql_query: The SQL query to filter rows from.
    •  
  •  
    conditions: Expressions specifying the conditions that rows must satisfy to be included in the output.                   Rows for which the expression evaluates to true will be included in the result.                   Multiple conditions can be combined using logical operators (&&, ||). It will automatically                   detect whether the conditions belong in WHERE vs HAVING. 
     
                 Temporarily, it is best to use begin and end when filtering multiple conditions. (ex 2 below)\n
     
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @filter(percent > .5)\n         @collect\n       end\n5\u00d74 DataFrame\n Row \u2502 id       groups   value   percent  \n     \u2502 String?  String?  Int64?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AF       aa            1       0.6\n   2 \u2502 AG       bb            2       0.7\n   3 \u2502 AH       aa            3       0.8\n   4 \u2502 AI       bb            4       0.9\n   5 \u2502 AJ       aa            5       1.0\n\njulia> @chain db_table(db, :df_mem) begin\n         @group_by(groups)\n         @summarise(mean = mean(percent))\n         @filter begin \n           groups == \"bb\" || # logical operators can still be used like this\n           mean > .5\n         end\n         @arrange(groups)\n         @collect\n       end\n2\u00d72 DataFrame\n Row \u2502 groups   mean     \n     \u2502 String?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 aa            0.6\n   2 \u2502 bb            0.5\n
 
source
 
# TidierDB.@full_join \u2014 Macro.
 
@inner_join(sql_query, join_table, new_table_col, orignal_table_col)\n
 
Perform an full join between two SQL queries based on a specified condition.  This syntax here is slightly different than TidierData.jl, however, because  SQL does not drop the joining column, for the metadata storage, it is  preferrable for the names to be different 
 
Arguments
 
     
  • sql_query: The primary SQL query to operate on.
    •  
  • join_table: The secondary SQL table to join with the primary query table.
    •  
  • new_table_col: Column from the new table that matches for join.
    •  
  • orignal_table_col: Column from the original table that matches for join.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> df2 = DataFrame(id2 = [\"AA\", \"AC\", \"AE\", \"AG\", \"AI\", \"AK\", \"AM\"],\n                category = [\"X\", \"Y\", \"X\", \"Y\", \"X\", \"Y\", \"X\"],\n                score = [88, 92, 77, 83, 95, 68, 74]);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> copy_to(db, df2, \"df_join\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @full_join(df_join, id2, id)\n         @collect\n       end\n12\u00d77 DataFrame\n Row \u2502 id       groups   value    percent    id2      category  score   \n     \u2502 String?  String?  Int64?   Float64?   String?  String?   Int64?  \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA       bb             1        0.1  AA       X              88\n   2 \u2502 AC       bb             3        0.3  AC       Y              92\n   3 \u2502 AE       bb             5        0.5  AE       X              77\n   4 \u2502 AG       bb             2        0.7  AG       Y              83\n   5 \u2502 AI       bb             4        0.9  AI       X              95\n   6 \u2502 AB       aa             2        0.2  missing  missing   missing \n   7 \u2502 AD       aa             4        0.4  missing  missing   missing \n   8 \u2502 AF       aa             1        0.6  missing  missing   missing \n   9 \u2502 AH       aa             3        0.8  missing  missing   missing \n  10 \u2502 AJ       aa             5        1.0  missing  missing   missing \n  11 \u2502 missing  missing  missing  missing    AK       Y              68\n  12 \u2502 missing  missing  missing  missing    AM       X              74\n
 
source
 
# TidierDB.@group_by \u2014 Macro.
 
@group_by(sql_query, columns...)\n
 
Group SQL table rows by specified column(s). If grouping is performed as a terminal operation without a subsequent mutatation or summarization (as in the example below), then the resulting data frame will be ungrouped when @collect is applied.
 
Arguments
 
     
  • sql_query: The SQL query to operate on.
    •  
  • exprs: Expressions specifying the columns to group by. Columns can be specified by name.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @group_by(groups)\n         @arrange(groups)\n         @collect\n       end\n2\u00d71 DataFrame\n Row \u2502 groups  \n     \u2502 String? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 aa\n   2 \u2502 bb\n
 
source
 
# TidierDB.@head \u2014 Macro.
 
@head(sql_query, value)\n
 
Limit SQL table number of rows returned based on specified value.  LIMIT in SQL
 
Arguments
 
     
  • sql_query: The SQL query to operate on.
    •  
  • value: Number to limit how many rows are returned.
    •  
 
Examples
 
julia> db = connect(duckdb());\n\njulia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> copy_to(db, df, \"df_mem\");                     \n\njulia> @chain db_table(db, :df_mem) begin\n        @head(1) ## supports expressions ie `3-2` would return the same df below\n        @collect\n       end\n1\u00d74 DataFrame\n Row \u2502 id       groups   value   percent  \n     \u2502 String?  String?  Int64?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA       bb            1       0.1\n
 
source
 
# TidierDB.@inner_join \u2014 Macro.
 
@inner_join(sql_query, join_table, new_table_col, orignal_table_col)\n
 
Perform an inner join between two SQL queries based on a specified condition.  This syntax here is slightly different than TidierData.jl, however, because  SQL does not drop the joining column, for the metadata storage, it is  preferrable for the names to be different 
 
Arguments
 
     
  • sql_query: The primary SQL query to operate on.
    •  
  • join_table: The secondary SQL table to join with the primary query table.
    •  
  • new_table_col: Column from the new table that matches for join.
    •  
  • orignal_table_col: Column from the original table that matches for join.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> df2 = DataFrame(id2 = [\"AA\", \"AC\", \"AE\", \"AG\", \"AI\", \"AK\", \"AM\"],\n                category = [\"X\", \"Y\", \"X\", \"Y\", \"X\", \"Y\", \"X\"],\n                score = [88, 92, 77, 83, 95, 68, 74]);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> copy_to(db, df2, \"df_join\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @inner_join(df_join, id2, id)\n         @collect\n       end\n5\u00d77 DataFrame\n Row \u2502 id       groups   value   percent   id2      category  score  \n     \u2502 String?  String?  Int64?  Float64?  String?  String?   Int64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA       bb            1       0.1  AA       X             88\n   2 \u2502 AC       bb            3       0.3  AC       Y             92\n   3 \u2502 AE       bb            5       0.5  AE       X             77\n   4 \u2502 AG       bb            2       0.7  AG       Y             83\n   5 \u2502 AI       bb            4       0.9  AI       X             95\n
 
source
 
# TidierDB.@interpolate \u2014 Macro.
 
@interpolate(args...)\n
 
Interpolate parameters into expressions for database queries.
 
Arguments
 
     
  •  
    args...: A variable number of tuples. Each tuple should contain:
     
       
    • name: The name of the parameter to interpolate.
      •  
    • value: (Any): The value/vector to interpolate for the corresponding parameter name.
      •  
     
    •  
 
Example
 
julia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> col_names = [:id, :value, :percent];\n\njulia> cond1 = .2;\n\njulia> cond2 = 5;\n\njulia> @interpolate((condition1, cond1), (columns, col_names), (condition2, cond2));\n\njulia> @chain db_table(db, \"df_mem\") begin \n          @select(!!columns)\n          @filter begin \n              percent < !!condition1\n              value < !!condition2\n          end\n          @collect\n          end\n1\u00d73 DataFrame\n Row \u2502 id       value   percent  \n     \u2502 String?  Int64?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA            1       0.1\n
 
source
 
# TidierDB.@left_join \u2014 Macro.
 
@left_join(sql_query, join_table, new_table_col, orignal_table_col)\n
 
Perform a left join between two SQL queries based on a specified condition.  This syntax here is slightly different than TidierData.jl, however, because  SQL does not drop the joining column, for the metadata storage, it is  preferrable for the names to be different 
 
Arguments
 
     
  • sql_query: The primary SQL query to operate on.
    •  
  • join_table: The secondary SQL table to join with the primary query table.
    •  
  • new_table_col: Column from the new table that matches for join.
    •  
  • orignal_table_col: Column from the original table that matches for join.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> df2 = DataFrame(id2 = [\"AA\", \"AC\", \"AE\", \"AG\", \"AI\", \"AK\", \"AM\"],\n                category = [\"X\", \"Y\", \"X\", \"Y\", \"X\", \"Y\", \"X\"],\n                score = [88, 92, 77, 83, 95, 68, 74]);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> copy_to(db, df2, \"df_join\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @left_join(df_join, id2, id)\n         @collect\n       end\n10\u00d77 DataFrame\n Row \u2502 id       groups   value   percent   id2      category  score   \n     \u2502 String?  String?  Int64?  Float64?  String?  String?   Int64?  \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA       bb            1       0.1  AA       X              88\n   2 \u2502 AC       bb            3       0.3  AC       Y              92\n   3 \u2502 AE       bb            5       0.5  AE       X              77\n   4 \u2502 AG       bb            2       0.7  AG       Y              83\n   5 \u2502 AI       bb            4       0.9  AI       X              95\n   6 \u2502 AB       aa            2       0.2  missing  missing   missing \n   7 \u2502 AD       aa            4       0.4  missing  missing   missing \n   8 \u2502 AF       aa            1       0.6  missing  missing   missing \n   9 \u2502 AH       aa            3       0.8  missing  missing   missing \n  10 \u2502 AJ       aa            5       1.0  missing  missing   missing \n
 
source
 
# TidierDB.@mutate \u2014 Macro.
 
@mutate(sql_query, exprs...)\n
 
Mutate SQL table rows by adding new columns or modifying existing ones.
 
Arguments
 
     
  • sql_query: The SQL query to operate on.
    •  
  • exprs: Expressions for mutating the table. New columns can be added or existing columns modified using column_name = expression syntax, where expression can involve existing columns.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @mutate(value = value * 4, new_col = percent^2)\n         @collect\n       end\n10\u00d75 DataFrame\n Row \u2502 id       groups   value   percent   new_col  \n     \u2502 String?  String?  Int64?  Float64?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA       bb            4       0.1      0.01\n   2 \u2502 AB       aa            8       0.2      0.04\n   3 \u2502 AC       bb           12       0.3      0.09\n   4 \u2502 AD       aa           16       0.4      0.16\n   5 \u2502 AE       bb           20       0.5      0.25\n   6 \u2502 AF       aa            4       0.6      0.36\n   7 \u2502 AG       bb            8       0.7      0.49\n   8 \u2502 AH       aa           12       0.8      0.64\n   9 \u2502 AI       bb           16       0.9      0.81\n  10 \u2502 AJ       aa           20       1.0      1.0\n
 
source
 
# TidierDB.@rename \u2014 Macro.
 
@rename(sql_query, renamings...)\n
 
Rename one or more columns in a SQL query.
 
Arguments
 
-sql_query: The SQL query to operate on. -renamings: One or more pairs of old and new column names, specified as new name = old name 
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n       @rename(new_name = percent)\n       @collect\n       end\n10\u00d74 DataFrame\n Row \u2502 id       groups   value   new_name \n     \u2502 String?  String?  Int64?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA       bb            1       0.1\n   2 \u2502 AB       aa            2       0.2\n   3 \u2502 AC       bb            3       0.3\n   4 \u2502 AD       aa            4       0.4\n   5 \u2502 AE       bb            5       0.5\n   6 \u2502 AF       aa            1       0.6\n   7 \u2502 AG       bb            2       0.7\n   8 \u2502 AH       aa            3       0.8\n   9 \u2502 AI       bb            4       0.9\n  10 \u2502 AJ       aa            5       1.0\n
 
source
 
# TidierDB.@right_join \u2014 Macro.
 
@right_join(sql_query, join_table, new_table_col, orignal_table_col)\n
 
Perform a right join between two SQL queries based on a specified condition.  This syntax here is slightly different than TidierData.jl, however, because  SQL does not drop the joining column, for the metadata storage, it is  preferrable for the names to be different 
 
Arguments
 
     
  • sql_query: The primary SQL query to operate on.
    •  
  • join_table: The secondary SQL table to join with the primary query table.
    •  
  • new_table_col: Column from the new table that matches for join.
    •  
  • orignal_table_col: Column from the original table that matches for join.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> df2 = DataFrame(id2 = [\"AA\", \"AC\", \"AE\", \"AG\", \"AI\", \"AK\", \"AM\"],\n                category = [\"X\", \"Y\", \"X\", \"Y\", \"X\", \"Y\", \"X\"],\n                score = [88, 92, 77, 83, 95, 68, 74]);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> copy_to(db, df2, \"df_join\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @right_join(df_join, id2, id)\n         @collect\n       end\n7\u00d77 DataFrame\n Row \u2502 id       groups   value    percent    id2      category  score  \n     \u2502 String?  String?  Int64?   Float64?   String?  String?   Int64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA       bb             1        0.1  AA       X             88\n   2 \u2502 AC       bb             3        0.3  AC       Y             92\n   3 \u2502 AE       bb             5        0.5  AE       X             77\n   4 \u2502 AG       bb             2        0.7  AG       Y             83\n   5 \u2502 AI       bb             4        0.9  AI       X             95\n   6 \u2502 missing  missing  missing  missing    AK       Y             68\n   7 \u2502 missing  missing  missing  missing    AM       X             74\n
 
source
 
# TidierDB.@select \u2014 Macro.
 
@select(sql_query, columns)\n
 
Select specified columns from a SQL table.
 
Arguments
 
     
  • sql_query: The SQL query to select columns from.
    •  
  • columns: Expressions specifying the columns to select. Columns can be specified by name,                and new columns can be created with expressions using existing column values.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @select(groups:percent)\n         @collect\n       end\n10\u00d73 DataFrame\n Row \u2502 groups   value   percent  \n     \u2502 String?  Int64?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 bb            1       0.1\n   2 \u2502 aa            2       0.2\n   3 \u2502 bb            3       0.3\n   4 \u2502 aa            4       0.4\n   5 \u2502 bb            5       0.5\n   6 \u2502 aa            1       0.6\n   7 \u2502 bb            2       0.7\n   8 \u2502 aa            3       0.8\n   9 \u2502 bb            4       0.9\n  10 \u2502 aa            5       1.0\n\njulia> @chain db_table(db, :df_mem) begin\n         @select(contains(\"e\"))\n         @collect\n       end\n10\u00d72 DataFrame\n Row \u2502 value   percent  \n     \u2502 Int64?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502      1       0.1\n   2 \u2502      2       0.2\n   3 \u2502      3       0.3\n   4 \u2502      4       0.4\n   5 \u2502      5       0.5\n   6 \u2502      1       0.6\n   7 \u2502      2       0.7\n   8 \u2502      3       0.8\n   9 \u2502      4       0.9\n  10 \u2502      5       1.0\n
 
source
 
# TidierDB.@semi_join \u2014 Macro.
 
@semi_join(sql_query, join_table, new_table_col, orignal_table_col)\n
 
Perform an semi join between two SQL queries based on a specified condition.  This syntax here is slightly different than TidierData.jl, however, because  SQL does not drop the joining column, for the metadata storage, it is  preferrable for the names to be different 
 
Arguments
 
     
  • sql_query: The primary SQL query to operate on.
    •  
  • join_table: The secondary SQL table to join with the primary query table.
    •  
  • new_table_col: Column from the new table that matches for join.
    •  
  • orignal_table_col: Column from the original table that matches for join.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> df2 = DataFrame(id2 = [\"AA\", \"AC\", \"AE\", \"AG\", \"AI\", \"AK\", \"AM\"],\n                category = [\"X\", \"Y\", \"X\", \"Y\", \"X\", \"Y\", \"X\"],\n                score = [88, 92, 77, 83, 95, 68, 74]);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> copy_to(db, df2, \"df_join\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @semi_join(df_join, id2, id)\n         @collect\n       end\n5\u00d74 DataFrame\n Row \u2502 id       groups   value   percent  \n     \u2502 String?  String?  Int64?  Float64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA       bb            1       0.1\n   2 \u2502 AC       bb            3       0.3\n   3 \u2502 AE       bb            5       0.5\n   4 \u2502 AG       bb            2       0.7\n   5 \u2502 AI       bb            4       0.9\n
 
source
 
# TidierDB.@slice_max \u2014 Macro.
 
@slice_max(sql_query, column, n = 1)\n
 
Select rows with the largest values in specified column. This will always return ties. 
 
Arguments
 
     
  • sql_query: The SQL query to operate on.
    •  
  • column: Column to identify the smallest values.
    •  
  • n: The number of rows to select with the largest values for each specified column. Default is 1, which selects the row with the smallest value.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @group_by(groups)\n         @slice_max(value, n = 2)\n         @collect\n       end;\n\njulia> @chain db_table(db, :df_mem) begin\n         @slice_max(value)\n         @collect\n       end\n2\u00d75 DataFrame\n Row \u2502 id       groups   value   percent   rank_col \n     \u2502 String?  String?  Int64?  Float64?  Int64?   \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AE       bb            5       0.5         1\n   2 \u2502 AJ       aa            5       1.0         1\n
 
source
 
# TidierDB.@slice_min \u2014 Macro.
 
@slice_min(sql_query, column, n = 1)\n
 
Select rows with the smallest values in specified column. This will always return ties. 
 
Arguments
 
     
  • sql_query: The SQL query to operate on.
    •  
  • column: Column to identify the smallest values.
    •  
  • n: The number of rows to select with the smallest values for each specified column. Default is 1, which selects the row with the smallest value.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @group_by(groups)\n         @slice_min(value, n = 2)\n         @collect\n       end;\n\njulia> @chain db_table(db, :df_mem) begin\n         @slice_min(value)\n         @collect\n       end\n2\u00d75 DataFrame\n Row \u2502 id       groups   value   percent   rank_col \n     \u2502 String?  String?  Int64?  Float64?  Int64?   \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA       bb            1       0.1         1\n   2 \u2502 AF       aa            1       0.6         1\n
 
source
 
# TidierDB.@slice_sample \u2014 Macro.
 
@slice_sample(sql_query, n)\n
 
Randomly select a specified number of rows from a SQL table.
 
Arguments
 
     
  • sql_query: The SQL query to operate on.
    •  
  • n: The number of rows to randomly select.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @group_by(groups)\n         @slice_sample(n = 2)\n         @collect\n       end;\n\njulia> @chain db_table(db, :df_mem) begin\n       @slice_sample()\n       @collect\n       end;\n
 
source
 
# TidierDB.@summarise \u2014 Macro.
 
   @summarise(sql_query, exprs...)\n
 
Aggregate and summarize specified columns of a SQL table.
 
Arguments
 
     
  • sql_query: The SQL query to operate on.
    •  
  • exprs: Expressions defining the aggregation and summarization operations. These can specify simple aggregations like mean, sum, and count, or more complex expressions involving existing column values.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @group_by(groups)\n         @summarise(across((value:percent), (mean, sum)))\n         @arrange(groups)\n         @collect\n       end\n2\u00d75 DataFrame\n Row \u2502 groups   mean_value  mean_percent  sum_value  sum_percent \n     \u2502 String?  Float64?    Float64?      Int128?    Float64?    \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 aa              3.0           0.6         15          3.0\n   2 \u2502 bb              3.0           0.5         15          2.5\n\njulia> @chain db_table(db, :df_mem) begin\n         @group_by(groups)\n         @summarise(test = sum(percent), n = n())\n         @arrange(groups)\n         @collect\n       end\n2\u00d73 DataFrame\n Row \u2502 groups   test      n      \n     \u2502 String?  Float64?  Int64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 aa            3.0       5\n   2 \u2502 bb            2.5       5\n
 
source
 
# TidierDB.@summarize \u2014 Macro.
 
   @summarize(sql_query, exprs...)\n
 
Aggregate and summarize specified columns of a SQL table.
 
Arguments
 
     
  • sql_query: The SQL query to operate on.
    •  
  • exprs: Expressions defining the aggregation and summarization operations. These can specify simple aggregations like mean, sum, and count, or more complex expressions involving existing column values.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n\njulia> @chain db_table(db, :df_mem) begin\n         @group_by(groups)\n         @summarise(across((ends_with(\"e\"), starts_with(\"p\")), (mean, sum)))\n         @arrange(groups)\n         @collect\n       end\n2\u00d75 DataFrame\n Row \u2502 groups   mean_value  mean_percent  sum_value  sum_percent \n     \u2502 String?  Float64?    Float64?      Int128?    Float64?    \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 aa              3.0           0.6         15          3.0\n   2 \u2502 bb              3.0           0.5         15          2.5\n\njulia> @chain db_table(db, :df_mem) begin\n         @group_by(groups)\n         @summarise(test = sum(percent), n = n())\n         @arrange(groups)\n         @collect\n       end\n2\u00d73 DataFrame\n Row \u2502 groups   test      n      \n     \u2502 String?  Float64?  Int64? \n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 aa            3.0       5\n   2 \u2502 bb            2.5       5\n
 
source
 
# TidierDB.@window_frame \u2014 Macro.
 
@window_frame(sql_query, frame_start::Int, frame_end::Int)\n
 
Define the window frame for window functions in a SQL query, specifying the range of rows to include in the calculation relative to the current row.
 
Arguments
 
sql_query: The SQL query to operate on, expected to be an instance of SQLQuery.
 
     
  • frame_start: The starting point of the window frame. A positive value indicates the start after the current row (FOLLOWING), a negative value indicates before the current row (PRECEDING), and 0 indicates the current row.
    •  
  • frame_end: The ending point of the window frame. A positive value indicates the end after the current row (FOLLOWING), a negative value indicates before the current row (PRECEDING), and 0 indicates the current row.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n
 
source
 
# TidierDB.@window_order \u2014 Macro.
 
   @window_order(sql_query, columns...)\n
 
Specify the order of rows for window functions within a SQL query.
 
Arguments
 
     
  • sql_query: The SQL query to operate on.
    •  
  • columns: Columns to order the rows by for the window function. Can include multiple columns for nested sorting. Prepend a column name with - for descending order.
    •  
 
Examples
 
julia> df = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9], \n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10], \n                        value = repeat(1:5, 2), \n                        percent = 0.1:0.1:1.0);\n\njulia> db = connect(duckdb());\n\njulia> copy_to(db, df, \"df_mem\");\n
 
source
 
 
"},{"location":"reference/#reference-internal-functions","title":"Reference - Internal functions","text":""},{"location":"examples/generated/UserGuide/Snowflake/","title":"Using Snowflake","text":"
Establishing a connection with the Snowflake SQL Rest API requires a OAuth token specific to the Role the user will use to query tables with.
 
 
"},{"location":"examples/generated/UserGuide/Snowflake/#connecting","title":"Connecting","text":"
Connection is established with the connect function as shown below. Connection requires 5 items as strings
 
     
  • Account Identifier
    •  
  • OAuth token
    •  
  • Database Name
    •  
  • Schema Name
    •  
  • Compute Warehouse name
    •  
 
Two things to note:
 
     
  • Your OAuth Token may frequently expire, which may require you to rerun your connection line.
    •  
  •  
    Since each time db_table runs, it runs a query to pull the metadata, you may choose to use run db_table and save the results, and use these results withfrom_query()
     
       
    • This will reduce the number of queries to your database
      •  
    • Allow you to build a a SQL query and @show_query even if the OAuthtoken has expired. To @collect you will have to reconnect and rerun dbtable if your OAuth token has expired
      •  
     
    •  
 
set_sql_mode(snowflake())\nac_id = \"string_id\"\ntoken = \"OAuth_token_string\"\ncon = connect(:snowflake, ac_id, token, \"DEMODB\", \"PUBLIC\", \"COMPUTE_WH\")\n# After connection is established, a you may begin querying.\nstable_table_metadata = db_table(con, \"MTCARS\")\n@chain from_query(stable_table_metadata) begin\n   @select(WT)\n   @mutate(TEST = WT *2)\n   #@aside @show_query _\n   @collect\nend\n
 
32\u00d72 DataFrame\n Row \u2502 WT       TEST\n     \u2502 Float64  Float64\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502   2.62     5.24\n   2 \u2502   2.875    5.75\n   3 \u2502   2.32     4.64\n   4 \u2502   3.215    6.43\n  \u22ee  \u2502    \u22ee        \u22ee\n  29 \u2502   3.17     6.34\n  30 \u2502   2.77     5.54\n  31 \u2502   3.57     7.14\n  32 \u2502   2.78     5.56\n         24 rows omitted\n
 
This page was generated using Literate.jl.
"},{"location":"examples/generated/UserGuide/athena/","title":"Using Athena","text":"
To use the Athena AWS backend with TidierDB, set up and a small syntax difference are covered here.
 
 
"},{"location":"examples/generated/UserGuide/athena/#connecting","title":"Connecting","text":"
Connection is established through AWS.jl as shwon below.
 
using TidierDB, AWS\nset_sql_mode(athena())\n# Replace your credentials as needed below\naws_access_key_id = get(ENV,\"AWS_ACCESS_KEY_ID\",\"key\")\naws_secret_access_key = get(ENV, \"AWS_SECRET_ACCESS_KEY\",\"secret_key\")\naws_region = get(ENV,\"AWS_DEFAULT_REGION\",\"region\")\n\nconst AWS_GLOBAL_CONFIG = Ref{AWS.AWSConfig}()\ncreds = AWSCredentials(aws_access_key_id, aws_secret_access_key)\n\nAWS_GLOBAL_CONFIG[] = AWS.global_aws_config(region=aws_region, creds=creds)\n\ncatalog = \"AwsDataCatalog\"\nworkgroup = \"primary\"\ndb = \"demodb\"\nall_results = true\nresults_per_increment = 10\nout_loc = \"s3://location/\"\n\nathena_params = Dict(\n    \"ResultConfiguration\" => Dict(\n        \"OutputLocation\" => out_loc\n    ),\n    \"QueryExecutionContext\" => Dict(\n        \"Database\" => db,\n        \"Catalog\" => catalog\n    ),\n    \"Workgroup\" => workgroup\n)\n
 
 
"},{"location":"examples/generated/UserGuide/athena/#db_table-differences","title":"db_table differences","text":"
There are two differences for db_table which are seen in the query below
 
     
  1. The table needs to be passed as a string in the format database.table, ie \"demodb.table_name
    2.  
  2. db_table requires a third argument: the athena_params from above.
    3.  
 
 
"},{"location":"examples/generated/UserGuide/athena/#leveraging-from_query-with-athena-to-reduce-number-of-queries","title":"Leveraging from_query with Athena to reduce number of queries","text":"
Throughout TidierDB, each time db_table is called, it queries the databases to get the metadata. Consider how AWS Athena logs queries, a user may want to reduce the number of queries. This can be done saving the results of db_table, and then using from_query with those results for furthe queries as shown below.
 
mtcars = db_table(AWS_GLOBAL_CONFIG[], \"demodb.mtcars\", athena_params)\n@chain from_query(mtcars) begin\n    @filter(cyl > 4)\n    @group_by(cyl)\n    @summarize(mpg = mean(mpg))\n   #@show_query\n    @collect\nend\n
 
2\u00d72 DataFrame\n Row \u2502 cyl    mpg\n     \u2502 Int64  Float64\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502     6  19.7429\n   2 \u2502     8  15.1\n
 
I would like to acknowledge the work of Manu Francis and this blog post, which helped guide this process
 
This page was generated using Literate.jl.
"},{"location":"examples/generated/UserGuide/databricks/","title":"Using Databricks","text":"
Establishing a connection with the Databricks SQL Rest API requires a token.
 
 
"},{"location":"examples/generated/UserGuide/databricks/#connecting","title":"Connecting","text":"
Connection is established with the connect function as shown below. Connection requires 5 items as strings
 
     
  • Account Instance : how to find your instance
    •  
  • OAuth token : how to generate your token
    •  
  • Database Name
    •  
  • Schema Name
    •  
  • warehouse_id
    •  
 
One thing to note, Since each time db_table runs, it runs a query to pull the metadata, you may choose to use run db_table and save the results, and use these results with from_query(). This will reduce the number of queries to your database and is illustrated below.
 
set_sql_mode(databricks())\ninstance_id = \"string_id\"\ntoken \"string_token\"\nwarehouse_id = \"e673cd4f387f964a\"\ncon = connect(:databricks, instance_id, token, \"DEMODB\", \"PUBLIC\", warehouse_id)\n# After connection is established, a you may begin querying.\nstable_table_metadata = db_table(con, \"mtcars\")\n@chain from_query(stable_table_metadata) begin\n   @select(wt)\n   @mutate(test = wt *2)\n   #@aside @show_query _\n   @collect\nend\n
 
32\u00d72 DataFrame\n Row \u2502 wt       test\n     \u2502 Float64  Float64\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502   2.62     5.24\n   2 \u2502   2.875    5.75\n   3 \u2502   2.32     4.64\n   4 \u2502   3.215    6.43\n  \u22ee  \u2502    \u22ee        \u22ee\n  29 \u2502   3.17     6.34\n  30 \u2502   2.77     5.54\n  31 \u2502   3.57     7.14\n  32 \u2502   2.78     5.56\n         24 rows omitted\n
 
This page was generated using Literate.jl.
"},{"location":"examples/generated/UserGuide/from_queryex/","title":"Reusing Part of a Query","text":"
While using TidierDB, you may need to generate part of a query and reuse it multiple times. from_query() enables a query portion to be reused multiple times as shown below.
 
import TidierDB as DB\ncon = DB.connect(duckdb())\nmtcars_path = \"https://gist.githubusercontent.com/seankross/a412dfbd88b3db70b74b/raw/5f23f993cd87c283ce766e7ac6b329ee7cc2e1d1/mtcars.csv\"\n
 
Start a query to analyze fuel efficiency by number of cylinders. However, to further build on this query later, end the chain without using @show_query or @collect
 
query = DB.@chain DB.db_table(con, mtcars_path) begin\n    DB.@group_by cyl\n    DB.@summarize begin\n        across(mpg, (mean, minimum, maximum))\n        num_cars = n()\n        end\n    DB.@mutate begin\n        efficiency = case_when(\n            mean_mpg >= 25, \"High\",\n            mean_mpg >= 15, \"Moderate\",\n            \"Low\" )\n       end\nend;\n
 
Now, from_query will allow you to reuse the query to calculate the average horsepower for each efficiency category
 
DB.@chain DB.from_query(query) begin\n   DB.@left_join(mtcars2, cyl, cyl)\n   DB.@group_by(efficiency)\n   DB.@summarize(avg_hp = mean(hp))\n   DB.@collect\nend\n
 
2\u00d72 DataFrame\n Row \u2502 efficiency  avg_hp\n     \u2502 String?     Float64?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 Moderate    180.238\n   2 \u2502 High         82.6364\n
 
Reuse the query again to find the car with the highest MPG for each cylinder category
 
DB.@chain DB.from_query(query) begin\n   DB.@left_join(mtcars2, cyl, cyl)\n   DB.@group_by cyl\n   DB.@slice_max(mpg)\n   DB.@select model cyl mpg\n   DB.@collect\nend\n
 
3\u00d73 DataFrame\n Row \u2502 model             cyl     mpg\n     \u2502 String?           Int64?  Float64?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 Pontiac Firebird       8      19.2\n   2 \u2502 Toyota Corolla         4      33.9\n   3 \u2502 Hornet 4 Drive         6      21.4\n
 
 
"},{"location":"examples/generated/UserGuide/from_queryex/#preview-or-save-an-intermediate-table","title":"Preview or save an intermediate table","text":"
While querying a dataset, you may wish to see an intermediate table, or even save it. You can use @aside and from_query(_), illustrated below, to do just that. While we opted to print the results in this simple example below, we could have saved them by using name = DB.@chain...
 
import ClickHouse;\nconn = conn = DB.connect(DB.clickhouse(); host=\"localhost\", port=19000, database=\"default\", user=\"default\", password=\"\")\npath = \"https://huggingface.co/datasets/maharshipandya/spotify-tracks-dataset/resolve/refs%2Fconvert%2Fparquet/default/train/0000.parquet\"\nDB.@chain DB.db_table(conn, path) begin\n   DB.@count(cyl)\n   @aside println(DB.@chain DB.from_query(_) DB.@head(5) DB.@collect)\n   DB.@arrange(desc(count))\n   DB.@collect\nend\n
 
5\u00d72 DataFrame\n Row \u2502 artists  count\n     \u2502 String?  UInt64\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 missing       1\n   2 \u2502 Wizo          3\n   3 \u2502 MAGIC!        3\n   4 \u2502 Macaco        1\n   5 \u2502 SOYOU         1\n31438\u00d72 DataFrame\n   Row \u2502 artists          count\n       \u2502 String?          UInt64\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n     1 \u2502 The Beatles         279\n     2 \u2502 George Jones        271\n     3 \u2502 Stevie Wonder       236\n     4 \u2502 Linkin Park         224\n     5 \u2502 Ella Fitzgerald     222\n     6 \u2502 Prateek Kuhad       217\n     7 \u2502 Feid                202\n   \u22ee   \u2502        \u22ee           \u22ee\n 31432 \u2502 Leonard               1\n 31433 \u2502 marcos g              1\n 31434 \u2502 BLVKSHP               1\n 31435 \u2502 Memtrix               1\n 31436 \u2502 SOYOU                 1\n 31437 \u2502 Macaco                1\n 31438 \u2502 missing               1\n               31424 rows omitted\n
 
This page was generated using Literate.jl.
"},{"location":"examples/generated/UserGuide/functions_pass_to_DB/","title":"Writing Functions/Macros with TidierDB Chains","text":"
How can functions pass arguments to a TidierDB chain?
 
In short, you have to use a macro instead in conjuction with @interpolate
 
 
"},{"location":"examples/generated/UserGuide/functions_pass_to_DB/#setting-up-the-macro","title":"Setting up the macro","text":"
To write a macro that will take arguments and pass them to a TidierDB chain, there are 3 steps:
 
     
  1. Write macro with the desired argument(s), and, after the quote, add the chain. Arguments to be changed/interpolated must be prefixed with !!
    2.  
  2. Use @interpolate to make these arguemnts accessible to the chain. @interpolate takes touples as argument (one for the !!name, and one for the actual content you want the chain to use)
    3.  
  3. Run @interpolate and then the chain macro sequentially
    4.  
 
using TidierDB\ndb = connect(duckdb())\npath = \"https://gist.githubusercontent.com/seankross/a412dfbd88b3db70b74b/raw/5f23f993cd87c283ce766e7ac6b329ee7cc2e1d1/mtcars.csv\"\ncopy_to(db, path, \"mtcars\");\n\n# STEP 1\nmacro f1(conditions, columns) # The arguemnt names will be names of the `!!` values\n    return quote\n    # add chain here\n      @chain db_table(db, :mtcars) begin\n           @filter(!!conditions > 3)\n           @select(!!columns)\n           @aside @show_query _\n           @collect\n         end # ends the chain\n    end # ends the quote.\nend # ends the macro\n
 
# STEP 2\nvariable = :gear;\ncols = [:model, :mpg, :gear, :wt];\n@interpolate((conditions, variable), (columns, cols));\n@f1(variable, cols)\n
 
17\u00d74 DataFrame\n Row \u2502 model           mpg       gear    wt\n     \u2502 String?         Float64?  Int32?  Float64?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 Mazda RX4           21.0       4     2.62\n   2 \u2502 Mazda RX4 Wag       21.0       4     2.875\n   3 \u2502 Datsun 710          22.8       4     2.32\n  \u22ee  \u2502       \u22ee            \u22ee        \u22ee        \u22ee\n  15 \u2502 Ferrari Dino        19.7       5     2.77\n  16 \u2502 Maserati Bora       15.0       5     3.57\n  17 \u2502 Volvo 142E          21.4       4     2.78\n                                   11 rows omitted\n
 
Lets say you wanted to filter on new variable with a different name and select new columns,
 
new_condition = :wt;\nnew_cols = [:model, :drat]\n@interpolate((conditions, new_condition), (columns, new_cols));\n@f1(new_condition, new_cols)\n
 
20\u00d72 DataFrame\n Row \u2502 model              drat\n     \u2502 String?            Float64?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 Hornet 4 Drive         3.08\n   2 \u2502 Hornet Sportabout      3.15\n   3 \u2502 Valiant                2.76\n  \u22ee  \u2502         \u22ee             \u22ee\n  18 \u2502 Pontiac Firebird       3.08\n  19 \u2502 Ford Pantera L         4.22\n  20 \u2502 Maserati Bora          3.54\n                    14 rows omitted\n
 
You can also interpolate vectors of strings into a @filter(col in (values)) as well by using the following syntax @filter(col in [!!values])
 
In short, the first argument in @interpolate must be the name of the macro argument it refers to, and the second argument is what you would like to replace it.
 
We recognize this adds friction and that it is not ideal, but given the TidierDB macro expressions/string interplay, this is currently the most graceful and functional option available and hopefully a temporary solution to better interpolation that mirrors TidierData.jl.
 
This page was generated using Literate.jl.
"},{"location":"examples/generated/UserGuide/getting_started/","title":"Getting Started","text":"
To use TidierDB.jl, you will have to set up a connection. TidierDB.jl gives you access to duckdb via duckdb_open and duckdb_connect. However, to use MySql, ClickHouse, MSSQL, Postgres, or SQLite, you will have to load those packages in first.
 
If you plan to use TidierDB.jl with TidierData.jl or Tidier.jl, it is most convenenient to load the packages as follows:
 
using TidierData\nimport TidierDB as DB\n
 
Alternatively, using Tidier will import TidierDB in the above manner for you, where TidierDB functions and macros will be available as DB.@mutate() and so on, and the TidierData equivalent would be @mutate().
 
 
"},{"location":"examples/generated/UserGuide/getting_started/#connecting","title":"Connecting","text":"
To connect to a database, you can uset the connect function  as shown below, or establish your own connection through the respecitve libraries.
 
For example Connecting to MySQL
 
conn = DB.connect(DB.mysql(); host=\"localhost\", user=\"root\", password=\"password\", db=\"mydb\")\n
 
versus connecting to DuckDB
 
conn = DB.connect(DB.duckdb())\n
 
You can also use establish a connection through an alternate method that you preferred, and use that as your connection as well.
 
 
"},{"location":"examples/generated/UserGuide/getting_started/#package-extensions","title":"Package Extensions","text":"
The following backends utilize package extensions. To use one of backends listed below, you will need to write using Library
 
     
  • ClickHouse: import ClickHouse
    •  
  • MySQL and MariaDB: using MySQL
    •  
  • MSSQL: using ODBC
    •  
  • Postgres: using LibPQ
    •  
  • SQLite: using SQLite
    •  
  • Athena: using AWS
    •  
  • Oracle: using ODBC
    •  
  • Google BigQuery: using GoogleCloud
    •  
 
 
"},{"location":"examples/generated/UserGuide/getting_started/#db_table","title":"db_table","text":"
What does db_table do?
 
db_table starts the underlying SQL query struct, in addition to pulling the table metadata and storing it there. Storing metadata is what enables a lazy interface that also supports tidy selection.
 
     
  • db_table has two required arguments: connection and table
    •  
  • table can be a table name on a database or a path/url to file to read.  When passing db_table a path or url, the table is not copied into memory.
    •  
  • With DuckDB and ClickHouse, if you have a folder of multiple files to read, you can use * read in all files matching the pattern.
    •  
  • For example, the below would read all files that end in .csv in the given folder.
    •  
 
db_table(db, \"folder/path/*.csv\")\n
 
db_table also supports iceberg, delta, and S3 file paths via DuckDB.
 
 
"},{"location":"examples/generated/UserGuide/getting_started/#minimizing-compute-costs","title":"Minimizing Compute Costs","text":"
If you are working with a backend where compute cost is important, it will be important to minimize using db_table as this will requery for metadata each time. Compute costs are relevant to backends such as AWS, databricks and Snowflake.
 
To do this, save the results of db_table and use them with from_query. Using from_query pulls the relevant information (metadata, con, etc) from the mutable SQLquery struct, allowing you to repeatedly query and collect the table without requerying for the metadata each time
 
table = DB.db_table(con, \"path\")\n@chain DB.from_query(table) begin\n    ## data wrangling here\nend\n
 
Tip: Setting t(table) = from_query(table) will save some keystrokes. This means after saving the results of db_table you can start all chains/refer to the data with `t(table)`` \u2013-
 
This page was generated using Literate.jl.
"},{"location":"examples/generated/UserGuide/ibis_comp/","title":"TidierDB.jl vs Ibis","text":""},{"location":"examples/generated/UserGuide/ibis_comp/#comparing-tidierdb-vs-ibis","title":"Comparing TidierDB vs Ibis","text":"
TidierDB is a reimplementation of dbplyr from R, so the syntax is remarkably similar. But how does TidierDB compare to Python's Ibis? This page will perform a similar comparison to the Ibis Documentation comparing Ibis and dplyr
 
 
"},{"location":"examples/generated/UserGuide/ibis_comp/#set-up","title":"Set up","text":"
Ibis
 
import ibis\nimport ibis.selectors as s # allows for different styles of column selection\nfrom ibis import _ # eliminates need to type table name before each column vs typing cols as strings\nibis.options.interactive = True # automatically collects first 10 rows of table\n
 
TidierDB
 
using TidierDB\ndb = connect(duckdb())\n# This next line is optional, but it will let us avoid writing `db_table` or `from_query` for each query\nt(table) = from_query(table)\n
 
Of note, TidierDB does not yet have an \"interactive mode\" so each example result will be collected.
 
 
"},{"location":"examples/generated/UserGuide/ibis_comp/#loading-data","title":"Loading Data","text":"
With Ibis, there are specific functions to read in different file types
 
mtcars = ibis.read_csv(\"https://gist.githubusercontent.com/seankross/a412dfbd88b3db70b74b/raw/5f23f993cd87c283ce766e7ac6b329ee7cc2e1d1/mtcars.csv\")\n
 
In TidierDB, there is only db_table, which determines the file type and generates the syntax appropriate for the backend in use.
 
mtcars = db_table(db, \"https://gist.githubusercontent.com/seankross/a412dfbd88b3db70b74b/raw/5f23f993cd87c283ce766e7ac6b329ee7cc2e1d1/mtcars.csv\");\n
 
 
"},{"location":"examples/generated/UserGuide/ibis_comp/#previewing-the-data","title":"Previewing the data","text":"
TidierDB and Ibis use head/@head to preview the first rows of a dataset. Ibis
 
mtcars.head(6)\n
 
\u250f\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2513\n\u2503 model             \u2503 mpg     \u2503 cyl   \u2503 disp    \u2503 hp    \u2503 drat    \u2503 wt      \u2503 qsec    \u2503 vs    \u2503 am    \u2503 gear  \u2503 carb  \u2503\n\u2521\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2529\n\u2502 string            \u2502 float64 \u2502 int64 \u2502 float64 \u2502 int64 \u2502 float64 \u2502 float64 \u2502 float64 \u2502 int64 \u2502 int64 \u2502 int64 \u2502 int64 \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 Mazda RX4         \u2502    21.0 \u2502     6 \u2502   160.0 \u2502   110 \u2502    3.90 \u2502   2.620 \u2502   16.46 \u2502     0 \u2502     1 \u2502     4 \u2502     4 \u2502\n\u2502 Mazda RX4 Wag     \u2502    21.0 \u2502     6 \u2502   160.0 \u2502   110 \u2502    3.90 \u2502   2.875 \u2502   17.02 \u2502     0 \u2502     1 \u2502     4 \u2502     4 \u2502\n\u2502 Datsun 710        \u2502    22.8 \u2502     4 \u2502   108.0 \u2502    93 \u2502    3.85 \u2502   2.320 \u2502   18.61 \u2502     1 \u2502     1 \u2502     4 \u2502     1 \u2502\n\u2502 Hornet 4 Drive    \u2502    21.4 \u2502     6 \u2502   258.0 \u2502   110 \u2502    3.08 \u2502   3.215 \u2502   19.44 \u2502     1 \u2502     0 \u2502     3 \u2502     1 \u2502\n\u2502 Hornet Sportabout \u2502    18.7 \u2502     8 \u2502   360.0 \u2502   175 \u2502    3.15 \u2502   3.440 \u2502   17.02 \u2502     0 \u2502     0 \u2502     3 \u2502     2 \u2502\n\u2502 Valiant           \u2502    18.1 \u2502     6 \u2502   225.0 \u2502   105 \u2502    2.76 \u2502   3.460 \u2502   20.22 \u2502     1 \u2502     0 \u2502     3 \u2502     1 \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n
 
TidierDB
 
@chain t(mtcars) @head(6) @collect\n
 
6\u00d712 DataFrame\n Row \u2502 model              mpg       cyl     disp      hp      drat      wt        qsec      vs      am      gear    carb\n     \u2502 String?            Float64?  Int64?  Float64?  Int64?  Float64?  Float64?  Float64?  Int64?  Int64?  Int64?  Int64?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 Mazda RX4              21.0       6     160.0     110      3.9      2.62      16.46       0       1       4       4\n   2 \u2502 Mazda RX4 Wag          21.0       6     160.0     110      3.9      2.875     17.02       0       1       4       4\n   3 \u2502 Datsun 710             22.8       4     108.0      93      3.85     2.32      18.61       1       1       4       1\n   4 \u2502 Hornet 4 Drive         21.4       6     258.0     110      3.08     3.215     19.44       1       0       3       1\n   5 \u2502 Hornet Sportabout      18.7       8     360.0     175      3.15     3.44      17.02       0       0       3       2\n   6 \u2502 Valiant                18.1       6     225.0     105      2.76     3.46      20.22       1       0       3       1\n
 
 
"},{"location":"examples/generated/UserGuide/ibis_comp/#filtering","title":"Filtering","text":"
The example below demonstrates how to filter using multiple criteria in both Ibis and TidierData Ibis
 
mtcars.filter(((_.mpg > 22) & (_.drat > 4) | (_.hp == 113)))\n
 
\u250f\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2513\n\u2503 model          \u2503 mpg     \u2503 cyl   \u2503 disp    \u2503 hp    \u2503 drat    \u2503 wt      \u2503 qsec    \u2503 vs    \u2503 am    \u2503 gear  \u2503 carb  \u2503\n\u2521\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2529\n\u2502 string         \u2502 float64 \u2502 int64 \u2502 float64 \u2502 int64 \u2502 float64 \u2502 float64 \u2502 float64 \u2502 int64 \u2502 int64 \u2502 int64 \u2502 int64 \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 Lotus Europa   \u2502    30.4 \u2502     4 \u2502    95.1 \u2502   113 \u2502    3.77 \u2502   1.513 \u2502   16.90 \u2502     1 \u2502     1 \u2502     5 \u2502     2 \u2502\n\u2502 Fiat 128       \u2502    32.4 \u2502     4 \u2502    78.7 \u2502    66 \u2502    4.08 \u2502   2.200 \u2502   19.47 \u2502     1 \u2502     1 \u2502     4 \u2502     1 \u2502\n\u2502 Honda Civic    \u2502    30.4 \u2502     4 \u2502    75.7 \u2502    52 \u2502    4.93 \u2502   1.615 \u2502   18.52 \u2502     1 \u2502     1 \u2502     4 \u2502     2 \u2502\n\u2502 Toyota Corolla \u2502    33.9 \u2502     4 \u2502    71.1 \u2502    65 \u2502    4.22 \u2502   1.835 \u2502   19.90 \u2502     1 \u2502     1 \u2502     4 \u2502     1 \u2502\n\u2502 Fiat X1-9      \u2502    27.3 \u2502     4 \u2502    79.0 \u2502    66 \u2502    4.08 \u2502   1.935 \u2502   18.90 \u2502     1 \u2502     1 \u2502     4 \u2502     1 \u2502\n\u2502 Porsche 914-2  \u2502    26.0 \u2502     4 \u2502   120.3 \u2502    91 \u2502    4.43 \u2502   2.140 \u2502   16.70 \u2502     0 \u2502     1 \u2502     5 \u2502     2 \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n
 
TidierDB
 
@chain t(mtcars) begin\n       @filter((mpg > 22 && drat > 4) || hp == 113)\n       @collect\nend\n
 
6\u00d712 DataFrame\n Row \u2502 model           mpg       cyl     disp      hp      drat      wt        qsec      vs      am      gear    carb\n     \u2502 String?         Float64?  Int64?  Float64?  Int64?  Float64?  Float64?  Float64?  Int64?  Int64?  Int64?  Int64?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 Lotus Europa        30.4       4      95.1     113      3.77     1.513     16.9        1       1       5       2\n   2 \u2502 Fiat 128            32.4       4      78.7      66      4.08     2.2       19.47       1       1       4       1\n   3 \u2502 Honda Civic         30.4       4      75.7      52      4.93     1.615     18.52       1       1       4       2\n   4 \u2502 Toyota Corolla      33.9       4      71.1      65      4.22     1.835     19.9        1       1       4       1\n   5 \u2502 Fiat X1-9           27.3       4      79.0      66      4.08     1.935     18.9        1       1       4       1\n   6 \u2502 Porsche 914-2       26.0       4     120.3      91      4.43     2.14      16.7        0       1       5       2\n
 
 
"},{"location":"examples/generated/UserGuide/ibis_comp/#creating-new-columns","title":"Creating new columns","text":"
Both TidierDB and Ibis use mutate/@mutate to add new columns Ibis
 
(\n   mtcars\n        .mutate(kpg = _.mpg * 1.61)\n        .select(\"model\", \"kpg\")\n)\n
 
\u250f\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2513\n\u2503 model             \u2503 kpg     \u2503\n\u2521\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2529\n\u2502 string            \u2502 float64 \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 Mazda RX4         \u2502  33.810 \u2502\n\u2502 Mazda RX4 Wag     \u2502  33.810 \u2502\n\u2502 Datsun 710        \u2502  36.708 \u2502\n\u2502 Hornet 4 Drive    \u2502  34.454 \u2502\n\u2502 Hornet Sportabout \u2502  30.107 \u2502\n\u2502 Valiant           \u2502  29.141 \u2502\n\u2502 Duster 360        \u2502  23.023 \u2502\n\u2502 Merc 240D         \u2502  39.284 \u2502\n\u2502 Merc 230          \u2502  36.708 \u2502\n\u2502 Merc 280          \u2502  30.912 \u2502\n\u2502 \u2026                 \u2502       \u2026 \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n
 
TidierDB
 
@chain t(mtcars) begin\n       @mutate(kpg = mpg * 1.61)\n       @select(model, kpg)\n       @collect\nend\n
 
32\u00d72 DataFrame\n Row \u2502 model              kpg\n     \u2502 String?            Float64?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 Mazda RX4            33.81\n   2 \u2502 Mazda RX4 Wag        33.81\n   3 \u2502 Datsun 710           36.708\n   4 \u2502 Hornet 4 Drive       34.454\n   5 \u2502 Hornet Sportabout    30.107\n   6 \u2502 Valiant              29.141\n  \u22ee  \u2502         \u22ee             \u22ee\n  27 \u2502 Porsche 914-2        41.86\n  28 \u2502 Lotus Europa         48.944\n  29 \u2502 Ford Pantera L       25.438\n  30 \u2502 Ferrari Dino         31.717\n  31 \u2502 Maserati Bora        24.15\n  32 \u2502 Volvo 142E           34.454\n                    20 rows omitted\n
 
 
"},{"location":"examples/generated/UserGuide/ibis_comp/#sorting-columns","title":"Sorting columns","text":"
Ibis uses order_by similar to SQLs ORDER BY Ibis
 
mtcars.order_by(_.mpg)\n
 
\u250f\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2513\n\u2503 model               \u2503 mpg     \u2503 cyl   \u2503 disp    \u2503 hp    \u2503 drat    \u2503 wt      \u2503 qsec    \u2503 vs    \u2503 am    \u2503 gear  \u2503 carb  \u2503\n\u2521\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2529\n\u2502 string              \u2502 float64 \u2502 int64 \u2502 float64 \u2502 int64 \u2502 float64 \u2502 float64 \u2502 float64 \u2502 int64 \u2502 int64 \u2502 int64 \u2502 int64 \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 Cadillac Fleetwood  \u2502    10.4 \u2502     8 \u2502   472.0 \u2502   205 \u2502    2.93 \u2502   5.250 \u2502   17.98 \u2502     0 \u2502     0 \u2502     3 \u2502     4 \u2502\n\u2502 Lincoln Continental \u2502    10.4 \u2502     8 \u2502   460.0 \u2502   215 \u2502    3.00 \u2502   5.424 \u2502   17.82 \u2502     0 \u2502     0 \u2502     3 \u2502     4 \u2502\n\u2502 Camaro Z28          \u2502    13.3 \u2502     8 \u2502   350.0 \u2502   245 \u2502    3.73 \u2502   3.840 \u2502   15.41 \u2502     0 \u2502     0 \u2502     3 \u2502     4 \u2502\n\u2502 Duster 360          \u2502    14.3 \u2502     8 \u2502   360.0 \u2502   245 \u2502    3.21 \u2502   3.570 \u2502   15.84 \u2502     0 \u2502     0 \u2502     3 \u2502     4 \u2502\n\u2502 Chrysler Imperial   \u2502    14.7 \u2502     8 \u2502   440.0 \u2502   230 \u2502    3.23 \u2502   5.345 \u2502   17.42 \u2502     0 \u2502     0 \u2502     3 \u2502     4 \u2502\n\u2502 Maserati Bora       \u2502    15.0 \u2502     8 \u2502   301.0 \u2502   335 \u2502    3.54 \u2502   3.570 \u2502   14.60 \u2502     0 \u2502     1 \u2502     5 \u2502     8 \u2502\n\u2502 Merc 450SLC         \u2502    15.2 \u2502     8 \u2502   275.8 \u2502   180 \u2502    3.07 \u2502   3.780 \u2502   18.00 \u2502     0 \u2502     0 \u2502     3 \u2502     3 \u2502\n\u2502 AMC Javelin         \u2502    15.2 \u2502     8 \u2502   304.0 \u2502   150 \u2502    3.15 \u2502   3.435 \u2502   17.30 \u2502     0 \u2502     0 \u2502     3 \u2502     2 \u2502\n\u2502 Dodge Challenger    \u2502    15.5 \u2502     8 \u2502   318.0 \u2502   150 \u2502    2.76 \u2502   3.520 \u2502   16.87 \u2502     0 \u2502     0 \u2502     3 \u2502     2 \u2502\n\u2502 Ford Pantera L      \u2502    15.8 \u2502     8 \u2502   351.0 \u2502   264 \u2502    4.22 \u2502   3.170 \u2502   14.50 \u2502     0 \u2502     1 \u2502     5 \u2502     4 \u2502\n\u2502 \u2026                   \u2502       \u2026 \u2502     \u2026 \u2502       \u2026 \u2502     \u2026 \u2502       \u2026 \u2502       \u2026 \u2502       \u2026 \u2502     \u2026 \u2502     \u2026 \u2502     \u2026 \u2502     \u2026 \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n
 
While TidierDB uses @arrange like TidierData.jl TidierDB
 
@chain t(mtcars) @arrange(mpg) @collect\n
 
32\u00d712 DataFrame\n Row \u2502 model                mpg       cyl     disp      hp      drat      wt        qsec      vs      am      gear    carb\n     \u2502 String?              Float64?  Int64?  Float64?  Int64?  Float64?  Float64?  Float64?  Int64?  Int64?  Int64?  Int64?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 Cadillac Fleetwood       10.4       8     472.0     205      2.93     5.25      17.98       0       0       3       4\n   2 \u2502 Lincoln Continental      10.4       8     460.0     215      3.0      5.424     17.82       0       0       3       4\n   3 \u2502 Camaro Z28               13.3       8     350.0     245      3.73     3.84      15.41       0       0       3       4\n   4 \u2502 Duster 360               14.3       8     360.0     245      3.21     3.57      15.84       0       0       3       4\n   5 \u2502 Chrysler Imperial        14.7       8     440.0     230      3.23     5.345     17.42       0       0       3       4\n   6 \u2502 Maserati Bora            15.0       8     301.0     335      3.54     3.57      14.6        0       1       5       8\n  \u22ee  \u2502          \u22ee              \u22ee        \u22ee        \u22ee        \u22ee        \u22ee         \u22ee         \u22ee        \u22ee       \u22ee       \u22ee       \u22ee\n  27 \u2502 Porsche 914-2            26.0       4     120.3      91      4.43     2.14      16.7        0       1       5       2\n  28 \u2502 Fiat X1-9                27.3       4      79.0      66      4.08     1.935     18.9        1       1       4       1\n  29 \u2502 Honda Civic              30.4       4      75.7      52      4.93     1.615     18.52       1       1       4       2\n  30 \u2502 Lotus Europa             30.4       4      95.1     113      3.77     1.513     16.9        1       1       5       2\n  31 \u2502 Fiat 128                 32.4       4      78.7      66      4.08     2.2       19.47       1       1       4       1\n  32 \u2502 Toyota Corolla           33.9       4      71.1      65      4.22     1.835     19.9        1       1       4       1\n                                                                                                              20 rows omitted\n
 
 
"},{"location":"examples/generated/UserGuide/ibis_comp/#selecting-columns","title":"Selecting columns","text":"
In Ibis, columns must be prefixed with the table name, or in this case _, or they can be given as a string. Finally to using helper functions like startswith requires importing selectors as above. Ibis
 
mtcars.select(s.startswith(\"m\"), \"drat\", _.wt)\n
 
\u250f\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2513\n\u2503 model             \u2503 mpg     \u2503 drat    \u2503 wt      \u2503\n\u2521\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2529\n\u2502 string            \u2502 float64 \u2502 float64 \u2502 float64 \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 Mazda RX4         \u2502    21.0 \u2502    3.90 \u2502   2.620 \u2502\n\u2502 Mazda RX4 Wag     \u2502    21.0 \u2502    3.90 \u2502   2.875 \u2502\n\u2502 Datsun 710        \u2502    22.8 \u2502    3.85 \u2502   2.320 \u2502\n\u2502 Hornet 4 Drive    \u2502    21.4 \u2502    3.08 \u2502   3.215 \u2502\n\u2502 Hornet Sportabout \u2502    18.7 \u2502    3.15 \u2502   3.440 \u2502\n\u2502 Valiant           \u2502    18.1 \u2502    2.76 \u2502   3.460 \u2502\n\u2502 Duster 360        \u2502    14.3 \u2502    3.21 \u2502   3.570 \u2502\n\u2502 Merc 240D         \u2502    24.4 \u2502    3.69 \u2502   3.190 \u2502\n\u2502 Merc 230          \u2502    22.8 \u2502    3.92 \u2502   3.150 \u2502\n\u2502 Merc 280          \u2502    19.2 \u2502    3.92 \u2502   3.440 \u2502\n\u2502 \u2026                 \u2502       \u2026 \u2502       \u2026 \u2502       \u2026 \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n
 
TidierDB does not require names to be prefixed and, like TidierData, tidy column selection with starts_with, ends_with, and contains is supported at base. TidierDB also supports providing column names as strings, although this would only be needed in the setting of renaming a column with a space in it. TidierDB
 
@chain t(mtcars) @select(starts_with(\"m\"), \"drat\", wt) @collect\n
 
32\u00d74 DataFrame\n Row \u2502 model              mpg       drat      wt\n     \u2502 String?            Float64?  Float64?  Float64?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 Mazda RX4              21.0      3.9      2.62\n   2 \u2502 Mazda RX4 Wag          21.0      3.9      2.875\n   3 \u2502 Datsun 710             22.8      3.85     2.32\n   4 \u2502 Hornet 4 Drive         21.4      3.08     3.215\n   5 \u2502 Hornet Sportabout      18.7      3.15     3.44\n   6 \u2502 Valiant                18.1      2.76     3.46\n  \u22ee  \u2502         \u22ee             \u22ee         \u22ee         \u22ee\n  27 \u2502 Porsche 914-2          26.0      4.43     2.14\n  28 \u2502 Lotus Europa           30.4      3.77     1.513\n  29 \u2502 Ford Pantera L         15.8      4.22     3.17\n  30 \u2502 Ferrari Dino           19.7      3.62     2.77\n  31 \u2502 Maserati Bora          15.0      3.54     3.57\n  32 \u2502 Volvo 142E             21.4      4.11     2.78\n                                        20 rows omitted\n
 
 
"},{"location":"examples/generated/UserGuide/ibis_comp/#multi-step-queries-and-summarizing","title":"Multi step queries and summarizing","text":"
Aggregating data is done with aggregate in Ibis and @summarize in TidierDB. To group data, Ibis uses by = within the aggregate call vs TidierDB adheres to @group_by convention Ibis
 
mtcars.aggregate(\n    total_hp=_.hp.sum(),\n    avg_hp=_.hp.mean(),\n    having=_.hp.sum() < 1000,\n    by=['cyl']\n)\n
 
\u250f\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2513\n\u2503 cyl   \u2503 total_hp \u2503 avg_hp     \u2503\n\u2521\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2529\n\u2502 int64 \u2502 int64    \u2502 float64    \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502     6 \u2502      856 \u2502 122.285714 \u2502\n\u2502     4 \u2502      909 \u2502  82.636364 \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n
 
In TidierDB, @filter will automatically determine whether the criteria belong in a WHERE or HAVING SQL clause. TidierDB
 
@chain t(mtcars) begin\n    @group_by(cyl)\n    @summarize(total_hp = sum(hp),\n               avg_hp = avg(hp))\n    @filter(total_hp < 1000)\n    @collect\nend\n
 
2\u00d73 DataFrame\n Row \u2502 cyl     total_hp  avg_hp\n     \u2502 Int64?  Int128?   Float64?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502      6       856  122.286\n   2 \u2502      4       909   82.6364\n
 
 
"},{"location":"examples/generated/UserGuide/ibis_comp/#renaming-columns","title":"Renaming columns","text":"
Both tools use rename/@rename to rename columns Ibis
 
mtcars.rename(make_model = \"model\").select(_.make_model)\n
 
\u250f\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2513\n\u2503 make_model        \u2503\n\u2521\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2529\n\u2502 string            \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502 Mazda RX4         \u2502\n\u2502 Mazda RX4 Wag     \u2502\n\u2502 Datsun 710        \u2502\n\u2502 Hornet 4 Drive    \u2502\n\u2502 Hornet Sportabout \u2502\n\u2502 Valiant           \u2502\n\u2502 Duster 360        \u2502\n\u2502 Merc 240D         \u2502\n\u2502 Merc 230          \u2502\n\u2502 Merc 280          \u2502\n\u2502 \u2026                 \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n
 
TidierDB
 
@chain t(mtcars) @rename(model_make = model) @select(model_make) @collect\n
 
32\u00d71 DataFrame\n Row \u2502 model_make\n     \u2502 String?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 Mazda RX4\n   2 \u2502 Mazda RX4 Wag\n   3 \u2502 Datsun 710\n   4 \u2502 Hornet 4 Drive\n   5 \u2502 Hornet Sportabout\n   6 \u2502 Valiant\n  \u22ee  \u2502         \u22ee\n  27 \u2502 Porsche 914-2\n  28 \u2502 Lotus Europa\n  29 \u2502 Ford Pantera L\n  30 \u2502 Ferrari Dino\n  31 \u2502 Maserati Bora\n  32 \u2502 Volvo 142E\n          20 rows omitted\n
 
This page was generated using Literate.jl.
"},{"location":"examples/generated/UserGuide/key_differences/","title":"Key Differences from TidierData.jl","text":"
There are a few important syntax and behavior differences between TidierDB.jl and TidierData.jl outlined below.
 
 
"},{"location":"examples/generated/UserGuide/key_differences/#creating-a-database","title":"Creating a database","text":"
For these examples we will use DuckDB, the default backend, although SQLite, Postgres, MySQL, MariaDB, MSSQL, and ClickHouse are possible. If you have an existing DuckDB connection, then this step is not required. For these examples, we will create a data frame and copy it to an in-memory DuckDB database.
 
using DataFrames, TidierDB\n\ndf = DataFrame(id = [string('A' + i \u00f7 26, 'A' + i % 26) for i in 0:9],\n                        groups = [i % 2 == 0 ? \"aa\" : \"bb\" for i in 1:10],\n                        value = repeat(1:5, 2),\n                        percent = 0.1:0.1:1.0);\n\ndb = connect(duckdb());\n\ncopy_to(db, df, \"df_mem\"); # copying over the data frame to an in-memory database\n
 
 
"},{"location":"examples/generated/UserGuide/key_differences/#row-ordering","title":"Row ordering","text":"
DuckDB benefits from aggressive parallelization of pipelines. This means that if you have multiple threads enabled in Julia, which you can check or set using Threads.nthreads(), DuckDB will use multiple threads. However, because many operations are multi-threaded, the resulting row order is inconsistent. If row order needs to be deterministic for your use case, make sure to apply an @arrange(column_name_1, column_name_2, etc...) prior to collecting the results.
 
 
"},{"location":"examples/generated/UserGuide/key_differences/#starting-a-chain","title":"Starting a chain","text":"
When using TidierDB, db_table(connection, :table_name) is used to start a chain.
 
 
"},{"location":"examples/generated/UserGuide/key_differences/#grouped-mutation","title":"Grouped mutation","text":"
In TidierDB, when performing @group_by then @mutate, the table will be ungrouped after applying all of the mutations in the clause to the grouped data. To perform subsequent grouped operations, the user would have to regroup the data. This is demonstrated below.
 
@chain db_table(db, :df_mem) begin\n    @group_by(groups)\n    @summarize(mean_percent = mean(percent))\n    @collect\n end\n
 2\u00d72 DataFrame Rowgroupsmean_percentString?Float64?1bb0.52aa0.6 
Regrouping following @mutate
 
@chain db_table(db, :df_mem) begin\n    @group_by(groups)\n    @mutate(max = maximum(percent), min = minimum(percent))\n    @group_by(groups)\n    @summarise(mean_percent = mean(percent))\n    @collect\nend\n
 2\u00d72 DataFrame Rowgroupsmean_percentString?Float64?1bb0.52aa0.6 
 
"},{"location":"examples/generated/UserGuide/key_differences/#joining","title":"Joining","text":"
There is one key difference for joining:
 
The column on both the new and old table must be specified. They do not need to be the same, and given SQL behavior where both columns are kept when joining two tables, it is preferable if they have different names. This avoids \"ambiguous reference\" errors that would otherwise come up and complicate the use of tidy selection for columns. Athena has an additional slight difference given the need for parameters, which is covered in the Athena documentation page.
 
df2 = DataFrame(id2 = [\"AA\", \"AC\", \"AE\", \"AG\", \"AI\", \"AK\", \"AM\"],\n                category = [\"X\", \"Y\", \"X\", \"Y\", \"X\", \"Y\", \"X\"],\n                score = [88, 92, 77, 83, 95, 68, 74]);\n\ncopy_to(db, df2, \"df_join\");\n\n@chain db_table(db, :df_mem) begin\n    @left_join(df_join, id2, id)\n    @collect\nend\n
 10\u00d77 DataFrame Rowidgroupsvaluepercentid2categoryscoreString?String?Int64?Float64?String?String?Int64?1AAbb10.1AAX882ACbb30.3ACY923AEbb50.5AEX774AGbb20.7AGY835AIbb40.9AIX956ABaa20.2missingmissingmissing7ADaa40.4missingmissingmissing8AFaa10.6missingmissingmissing9AHaa30.8missingmissingmissing10AJaa51.0missingmissingmissing 
 
"},{"location":"examples/generated/UserGuide/key_differences/#differences-in-case_when","title":"Differences in case_when()","text":"
In TidierDB, after the clause is completed, the result for the new column should is separated by a comma , in contrast to TidierData.jl, where the result for the new column is separated by a => .
 
@chain db_table(db, :df_mem) begin\n    @mutate(new_col = case_when(percent > .5, \"Pass\",  # in TidierData, percent > .5 => \"Pass\",\n                                percent <= .5, \"Try Again\", # percent <= .5 => \"Try Again\"\n                                true, \"middle\"))\n    @collect\n end\n
 10\u00d75 DataFrame Rowidgroupsvaluepercentnew_colString?String?Int64?Float64?String?1AAbb10.1Try Again2ABaa20.2Try Again3ACbb30.3Try Again4ADaa40.4Try Again5AEbb50.5Try Again6AFaa10.6Pass7AGbb20.7Pass8AHaa30.8Pass9AIbb40.9Pass10AJaa51.0Pass 
 
"},{"location":"examples/generated/UserGuide/key_differences/#interpolation","title":"Interpolation","text":"
To use !! Interpolation, instead of being able to define the alternate names/value in the global context, the user has to use @interpolate. This will hopefully be fixed in future versions. Otherwise, the behavior is generally the same, although this creates friction around calling functions.
 
Also, when using interpolation with exponenents, the interpolated value must go inside of parenthesis.
 
@interpolate((test, :percent)); # this still supports strings, vectors of names, and values\n\n@chain db_table(db, :df_mem) begin\n    @mutate(new_col = case_when((!!test)^2 > .5, \"Pass\",\n                                (!!test)^2 < .5, \"Try Again\",\n                                \"middle\"))\n    @collect\nend\n
 
10\u00d75 DataFrame\n Row \u2502 id       groups   value   percent   new_col\n     \u2502 String?  String?  Int64?  Float64?  String?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 AA       bb            1       0.1  Try Again\n   2 \u2502 AB       aa            2       0.2  Try Again\n   3 \u2502 AC       bb            3       0.3  Try Again\n  \u22ee  \u2502    \u22ee        \u22ee       \u22ee        \u22ee          \u22ee\n   8 \u2502 AH       aa            3       0.8  Pass\n   9 \u2502 AI       bb            4       0.9  Pass\n  10 \u2502 AJ       aa            5       1.0  Pass\n                                       4 rows omitted\n
 
 
"},{"location":"examples/generated/UserGuide/key_differences/#slicing-ties","title":"Slicing ties","text":"
slice_min() and @slice_max() will always return ties due to SQL behavior.
 
This page was generated using Literate.jl.
"},{"location":"examples/generated/UserGuide/outofmemex/","title":"Working With Larger than RAM Datasets","text":"
While using the DuckDB backend, TidierDB's lazy intferace enables querying datasets larger than your available RAM.
 
To illustrate this, we will recreate the Hugging Face x Polars example. The final table results are shown below and in this Hugging Face x DuckDB example
 
First we will load TidierDB, set up a local database and then set the URLs for the 2 training datasets from huggingface.co
 
using TidierDB\ndb = connect(duckdb())\n\nurls = [\"https://huggingface.co/datasets/blog_authorship_corpus/resolve/refs%2Fconvert%2Fparquet/blog_authorship_corpus/train/0000.parquet\",\n \"https://huggingface.co/datasets/blog_authorship_corpus/resolve/refs%2Fconvert%2Fparquet/blog_authorship_corpus/train/0001.parquet\"];\n
 
Here, we pass the vector of URLs to db_table, which will not copy them into memory. Since these datasets are so large, we will also set stream = true in @collect to stream the results. If we wanted to read all the files in the folder we could have replace the 0000 with * (wildcard) db_table(db, \"Path/to/folder/*.parquet\") Of note, reading these files from URLs is not as rapid as reading them from local files.
 
@chain db_table(db, urls) begin\n    @group_by(horoscope)\n    @summarise(count = n(), avg_blog_length = mean(length(text)))\n    @arrange(desc(count))\n    @aside @show_query _\n    @collect(stream = true)\nend\n
 
Placing @aside @show_query _ before @collect above lets us see the SQL query and collect it to a local DataFrame at the same time.
 
SELECT horoscope, COUNT(*) AS count, AVG(length(text)) AS avg_blog_length\n        FROM read_parquet(['https://huggingface.co/datasets/blog_authorship_corpus/resolve/refs%2Fconvert%2Fparquet/blog_authorship_corpus/train/0000.parquet', 'https://huggingface.co/datasets/blog_authorship_corpus/resolve/refs%2Fconvert%2Fparquet/blog_authorship_corpus/train/0001.parquet'])\n        GROUP BY horoscope\n        ORDER BY avg_blog_length DESC\n12\u00d73 DataFrame\n Row \u2502 horoscope    count   avg_blog_length\n     \u2502 String?      Int64?  Float64?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502 Aquarius      49568         1125.83\n   2 \u2502 Cancer        63512         1097.96\n   3 \u2502 Libra         60304         1060.61\n   4 \u2502 Capricorn     49402         1059.56\n   5 \u2502 Sagittarius   50431         1057.46\n   6 \u2502 Leo           58010         1049.6\n   7 \u2502 Taurus        61571         1022.69\n   8 \u2502 Gemini        52925         1020.26\n   9 \u2502 Scorpio       56495         1014.03\n  10 \u2502 Pisces        53812         1011.75\n  11 \u2502 Virgo         64629          996.684\n  12 \u2502 Aries         69134          918.081\n
 
To learn more about memory efficient queries on larger than RAM files, this blog from DuckDB will help maximize your local db
 
This page was generated using Literate.jl.
"},{"location":"examples/generated/UserGuide/s3viaduckdb/","title":"S3 + DuckDB + TidierDB","text":"
TidierDB allows you leverage DuckDB's seamless database integration.
 
Using DuckDB, you can connect to an AWS or GoogleCloud Database to query directly without making any local copies.
 
You can also use DBInterface.execute to set up any DuckDB database connection you need and then use that db to query with TidierDB
 
using TidierDB\n\n#Connect to Google Cloud via DuckDB\n#google_db = connect(duckdb(), :gbq, access_key=\"string\", secret_key=\"string\")\n\n#Connect to AWS via DuckDB\naws_db = connect(duckdb(), :aws, aws_access_key_id= \"string\",\n                                aws_secret_access_key= \"string\",\n                                aws_region=\"us-east-1\")\ns3_csv_path = \"s3://path/to_data.csv\"\n\n@chain db_table(aws_db, s3_csv_path) begin\n    @filter(!starts_with(column1, \"M\"))\n    @group_by(cyl)\n    @summarize(mpg = mean(mpg))\n    @mutate(mpg_squared = mpg^2,\n               mpg_rounded = round(mpg),\n               mpg_efficiency = case_when(\n                                 mpg >= cyl^2 , \"efficient\",\n                                 mpg < 15.2 , \"inefficient\",\n                                 \"moderate\"))\n    @filter(mpg_efficiency in (\"moderate\", \"efficient\"))\n    @arrange(desc(mpg_rounded))\n    @collect\nend\n
 
2\u00d75 DataFrame\n Row \u2502 cyl     mpg       mpg_squared  mpg_rounded  mpg_efficiency\n     \u2502 Int64?  Float64?  Float64?     Float64?     String?\n\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n   1 \u2502      4   27.3444      747.719         27.0  efficient\n   2 \u2502      6   19.7333      389.404         20.0  moderate\n
 
This page was generated using Literate.jl.
"}]}
\ No newline at end of file