diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 1cea25f910..fb6e8fb3ea 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -39,39 +39,15 @@ jobs: fail-fast: false matrix: language: [ 'javascript', 'python' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] - # Learn more: - # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed - + # Learn more: https://docs.github.com/en/code-security/code-scanning steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v1 - - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v3 diff --git a/Dockerfile b/Dockerfile index a18759a501..2afcc24032 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ #=========== STAGE: BASE =====================================================# -ARG PYTHON_VERSION=3.9-bookworm +ARG PYTHON_VERSION=3.13-bookworm FROM python:$PYTHON_VERSION AS base ENV PYTHONUNBUFFERED=1 ENV DOCKERIZE_VERSION v0.6.1 -ARG BUILD_PG_MAJOR=15 +ARG BUILD_PG_MAJOR=17 ENV PG_MAJOR=$BUILD_PG_MAJOR RUN set -eux; @@ -42,14 +42,6 @@ ENV PGDATA /var/lib/postgresql/mathesar VOLUME /etc/postgresql/ VOLUME /var/lib/postgresql/ -# We set the default STOPSIGNAL to SIGINT, which corresponds to what PostgreSQL -# calls "Fast Shutdown mode" wherein new connections are disallowed and any -# in-progress transactions are aborted, allowing PostgreSQL to stop cleanly and -# flush tables to disk, which is the best compromise available to avoid data -# corruption. - -STOPSIGNAL SIGINT - EXPOSE 5432 # Mathesar source @@ -59,7 +51,7 @@ COPY . . #=========== STAGE: TESTING ==================================================# -ARG PYTHON_VERSION=3.9-bookworm +ARG PYTHON_VERSION=3.13-bookworm FROM python:$PYTHON_VERSION AS testing # Mathesar source diff --git a/Dockerfile.caddy b/Dockerfile.caddy index e2e42a891d..464f9cc5c1 100644 --- a/Dockerfile.caddy +++ b/Dockerfile.caddy @@ -1,5 +1,5 @@ # Useful when we need to build a single dockerfile to run a complete webservice -ARG BASE_IMAGE=python:3.9-buster +ARG BASE_IMAGE=python:3.13-bookworm FROM $BASE_IMAGE as base_image ENV PORT=8000 @@ -21,4 +21,4 @@ EXPOSE 2019 COPY Caddyfile /etc/caddy/Caddyfile -CMD ["caddy", "run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"] \ No newline at end of file +CMD ["caddy", "run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"] diff --git a/LICENSES/python-dockerfile b/LICENSES/python-dockerfile deleted file mode 100644 index 4907aa2814..0000000000 --- a/LICENSES/python-dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -Component: python3.9 docker image source file -Copyright: 2014 Docker, Inc -License: MIT -License Text: https://github.com/docker-library/python/blob/33751272d8171cece37c59180c049ab77cf9c837/LICENSE -Derivative files: - /Dockerfile.integ-tests - -================================= -BEGIN of license -================================= - -The MIT License (MIT) - -Copyright (c) 2014 Docker, Inc. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be included -in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -================================= -END of license -================================= diff --git a/api_tests/Dockerfile b/api_tests/Dockerfile index 38187455aa..63e5e01963 100644 --- a/api_tests/Dockerfile +++ b/api_tests/Dockerfile @@ -1,4 +1,4 @@ -ARG PYTHON_VERSION=3.9-bookworm +ARG PYTHON_VERSION=3.13-bookworm FROM python:$PYTHON_VERSION WORKDIR /code/ COPY . . diff --git a/db/sql/05_msar.sql b/db/sql/05_msar.sql index a8f2ad2e3a..fdb201767d 100644 --- a/db/sql/05_msar.sql +++ b/db/sql/05_msar.sql @@ -4653,63 +4653,63 @@ $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION msar.build_results_jsonb_array_expr( cte_name text, - columns text[], order_by_expr text ) RETURNS TEXT AS $$/* Build an SQL expresson string that, when added to the record listing query, produces a JSON array with the records resulting from the request. */ SELECT format( - 'coalesce(jsonb_agg(jsonb_build_object(' - || string_agg(format('%1$L, %2$I.%1$I', column_, cte_name), ', ') - || ') %1$s), jsonb_build_array())', - order_by_expr -) -FROM unnest(columns) AS column_ + $j$ + COALESCE( + jsonb_agg( + to_jsonb(%2$I) - %3$L - %4$L %1$s + ), jsonb_build_array() + ) + $j$, + /* %1 */ order_by_expr, + /* %2 */ cte_name, + /* %3 */ '__mathesar_gid', + /* %4 */ '__mathesar_gcount' +); $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION msar.build_results_setof_jsonb_expr( - cte_name text, - columns text[] + cte_name text ) RETURNS TEXT AS $$/* Build an SQL expresson string that, when added to the record listing query, produces a setof jsonb results with the records resulting from the request. */ SELECT format( - 'jsonb_build_object(' - || string_agg(format('%1$L, %2$I.%1$I', column_, cte_name), ', ') - || ')' -) -FROM unnest(columns) AS column_ + 'to_jsonb(%1$I) - %2$L - %3$L', + /* %1 */ cte_name, + /* %2 */ '__mathesar_gid', + /* %3 */ '__mathesar_gcount' +); $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION -msar.build_groups_cte_expr(tab_id oid, cte_name text, group_ jsonb) RETURNS TEXT AS $$/* -*/ -SELECT format( - $gj$ - __mathesar_gid AS id, - __mathesar_gcount AS count, - jsonb_build_object(%1$s) AS results_eq, - jsonb_agg(__mathesar_result_idx) AS result_indices - FROM %2$I - GROUP BY id, count, results_eq - $gj$, - string_agg( - format( - '%1$L, %2$s', - col_id, - COALESCE( - format(expr_template, quote_ident(cte_name) || '.' || quote_ident(col_id)), - quote_ident(cte_name) || '.' || quote_ident(col_id) - ) +msar.build_results_eq_cte_expr(tab_id oid, cte_name text, group_ jsonb) RETURNS TEXT AS $$ +SELECT string_agg( + format( + '%1$s AS %2$I', + COALESCE( + format(expr_template, quote_ident(cte_name) || '.' || quote_ident(col_id)), + quote_ident(cte_name) || '.' || quote_ident(col_id) ), - ', ' ORDER BY ordinality + col_id ), - cte_name + ', ' ORDER BY ordinality +) || ', __mathesar_gid FROM ' || quote_ident(cte_name) +|| ' GROUP BY __mathesar_gid, ' +|| string_agg( + format( + '%1$I', + col_id + ), + ', ' ORDER BY ordinality ) FROM msar.expr_templates RIGHT JOIN ROWS FROM( jsonb_array_elements_text(group_ -> 'columns'), @@ -4719,6 +4719,24 @@ WHERE has_column_privilege(tab_id, col_id::smallint, 'SELECT'); $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; +CREATE OR REPLACE FUNCTION +msar.build_groups_cte_expr(tab_id oid, eq_cte_name text, ranked_cte_name text, group_ jsonb) RETURNS TEXT AS $$/* +*/ +SELECT format( + $gj$ + %1$I.__mathesar_gid AS id, + __mathesar_gcount AS count, + to_jsonb(%1$I) - '__mathesar_gid' AS results_eq, + jsonb_agg( DISTINCT __mathesar_result_idx) AS result_indices + FROM %1$I LEFT JOIN %2$I AS rcn ON %1$I.__mathesar_gid = rcn.__mathesar_gid + GROUP BY id, count, results_eq + $gj$, + eq_cte_name, + ranked_cte_name +); +$$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; + + CREATE OR REPLACE FUNCTION msar.build_grouping_results_jsonb_expr(tab_id oid, cte_name text, group_ jsonb) RETURNS TEXT AS $$/* Build an SQL expresson string that, when added to the record listing query, produces a JSON array @@ -5125,18 +5143,18 @@ Args: tab_oid: The OID of the table for which we're getting linked record summaries. */ WITH fkey_map_cte AS (SELECT * FROM msar.get_fkey_map_table(tab_id)) -SELECT 'jsonb_build_object(' || string_agg( +SELECT string_agg( format( $j$ - %1$L, COALESCE( - jsonb_object_agg( - summary_cte_%1$s.key, summary_cte_%1$s.summary - ) FILTER (WHERE summary_cte_%1$s.key IS NOT NULL), '{}'::jsonb - ) + COALESCE( + jsonb_object_agg( + summary_cte_%1$s.key, summary_cte_%1$s.summary + ) FILTER (WHERE summary_cte_%1$s.key IS NOT NULL), '{}'::jsonb + ) AS %1$I $j$, conkey ), ', ' -) || ')' +) FROM fkey_map_cte; $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; @@ -5150,7 +5168,7 @@ SELECT CASE WHEN quote_ident(msar.get_selectable_pkey_attnum(tab_id)::text) IS N jsonb_object_agg( summary_cte_self.key, summary_cte_self.summary ) FILTER (WHERE summary_cte_self.key IS NOT NULL), '{}'::jsonb - ) + ) AS summary_self $j$ END; $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; @@ -5276,19 +5294,41 @@ BEGIN results_ranked_cte AS ( SELECT *, row_number() OVER (%3$s) - 1 AS __mathesar_result_idx FROM enriched_results_cte ), + results_eq_cte AS ( + SELECT %11$s + ), groups_cte AS ( SELECT %6$s ), summary_cte_self AS (%7$s) - %8$s - SELECT jsonb_build_object( + %8$s, + summary_cte AS ( SELECT %10$s FROM enriched_results_cte %9$s ), + summaries_json_cte AS ( + SELECT + jsonb_build_object( + 'linked_record_summaries', + NULLIF( + to_jsonb(summary_cte) - 'summary_self' - 'count_hack', + '{}'::jsonb + ), + 'record_summaries', + NULLIF( + to_jsonb(summary_cte) - 'count_hack' -> 'summary_self', + '{}'::jsonb + ) + ) + AS sj + FROM summary_cte + ), + records_json_cte AS ( SELECT jsonb_build_object( 'results', %4$s, 'count', coalesce(max(count_cte.count), 0), - 'grouping', %5$s, - 'linked_record_summaries', %10$s, - 'record_summaries', %11$s - ) + 'grouping', %5$s + ) AS rj FROM enriched_results_cte - LEFT JOIN groups_cte ON enriched_results_cte.__mathesar_gid = groups_cte.id %9$s + LEFT JOIN groups_cte ON enriched_results_cte.__mathesar_gid = groups_cte.id CROSS JOIN count_cte + ) + SELECT records_json_cte.rj || summaries_json_cte.sj + FROM records_json_cte, summaries_json_cte; $q$, /* %1 */ expr_and_ctes ->> 'count_cte_query', /* %2 */ expr_and_ctes ->> 'results_cte_query', @@ -5296,7 +5336,6 @@ BEGIN /* %4 */ COALESCE( msar.build_results_jsonb_array_expr( 'enriched_results_cte', - msar.jsonb_keys_to_array(expr_and_ctes -> 'selectable_columns'), expr_and_ctes ->> 'order_by_expr' ), 'NULL' @@ -5306,7 +5345,7 @@ BEGIN 'NULL' ), /* %6 */ COALESCE( - msar.build_groups_cte_expr(tab_id, 'results_ranked_cte', group_), + msar.build_groups_cte_expr(tab_id, 'results_eq_cte', 'results_ranked_cte', group_), 'NULL AS id' ), /* %7 */ msar.build_record_summary_query_for_table( @@ -5319,11 +5358,19 @@ BEGIN table_record_summary_templates ), /* %9 */ msar.build_summary_join_expr_for_table(tab_id, 'enriched_results_cte'), - /* %10 */ COALESCE(msar.build_summary_json_expr_for_table(tab_id), 'NULL'), - /* %11 */ COALESCE( - CASE WHEN return_record_summaries THEN msar.build_self_summary_json_expr(tab_id) END, - 'NULL' - ) + /* %10 */ COALESCE( + NULLIF( + concat_ws(', ', + msar.build_summary_json_expr_for_table(tab_id), + CASE WHEN return_record_summaries + THEN msar.build_self_summary_json_expr(tab_id) + END + ), '' + ), 'COUNT(1) AS count_hack' + -- count_hack ensures that summary_cte is not empty, + -- which in turn helps to generate summaries_json_cte + ), + /* %11 */ msar.build_results_eq_cte_expr(tab_id, 'results_ranked_cte', group_) ) INTO records; RETURN records; END; @@ -5358,10 +5405,7 @@ BEGIN $q$, expr_and_ctes ->> 'results_cte_query', COALESCE( - msar.build_results_setof_jsonb_expr( - 'results_cte', - msar.jsonb_keys_to_array(expr_and_ctes -> 'selectable_columns') - ), + msar.build_results_setof_jsonb_expr('results_cte'), 'NULL' ) ); @@ -5430,15 +5474,35 @@ BEGIN results_cte AS ( SELECT %1$s FROM %2$I.%3$I %4$s %6$s LIMIT %5$L ), - summary_cte_self AS (%7$s) %8$s - SELECT jsonb_build_object( - 'results', coalesce(jsonb_agg(row_to_json(results_cte.*)), jsonb_build_array()), - 'count', coalesce(max(count_cte.count), 0), - 'linked_record_summaries', %10$s, - 'record_summaries', %11$s + summary_cte_self AS (%7$s) + %8$s, + summary_cte AS ( SELECT %10$s FROM results_cte %9$s ), + summaries_json_cte AS ( + SELECT + jsonb_build_object( + 'linked_record_summaries', + NULLIF( + to_jsonb(summary_cte) - 'summary_self' - 'count_hack', + '{}'::jsonb + ), + 'record_summaries', + NULLIF( + to_jsonb(summary_cte) - 'count_hack' -> 'summary_self', + '{}'::jsonb + ) + ) + AS sj + FROM summary_cte + ), + results_json_cte AS ( + SELECT jsonb_build_object( + 'results', coalesce(jsonb_agg(row_to_json(results_cte.*)), jsonb_build_array()), + 'count', coalesce(max(count_cte.count), 0) + ) AS rj + FROM results_cte CROSS JOIN count_cte ) - FROM results_cte %9$s - CROSS JOIN count_cte + SELECT results_json_cte.rj || summaries_json_cte.sj + FROM results_json_cte, summaries_json_cte; $q$, /* %1 */ COALESCE(msar.build_selectable_column_expr(tab_id), 'NULL'), /* %2 */ msar.get_relation_schema_name(tab_id), @@ -5459,10 +5523,17 @@ BEGIN ), /* %8 */ msar.build_linked_record_summaries_ctes(tab_id), /* %9 */ msar.build_summary_join_expr_for_table(tab_id, 'results_cte'), - /* %10 */ COALESCE(msar.build_summary_json_expr_for_table(tab_id), 'NULL'), - /* %11 */ COALESCE( - CASE WHEN return_record_summaries THEN msar.build_self_summary_json_expr(tab_id) END, - 'NULL' + /* %10 */ COALESCE( + NULLIF( + concat_ws(', ', + msar.build_summary_json_expr_for_table(tab_id), + CASE WHEN return_record_summaries + THEN msar.build_self_summary_json_expr(tab_id) + END + ), '' + ), 'COUNT(1) AS count_hack' + -- count_hack ensures that summary_cte is not empty, + -- which in turn helps to generate summaries_json_cte ) ) INTO records; RETURN records; diff --git a/db/sql/test_sql_functions.sql b/db/sql/test_sql_functions.sql index 0c20e0544f..78f93ccaef 100644 --- a/db/sql/test_sql_functions.sql +++ b/db/sql/test_sql_functions.sql @@ -3116,6 +3116,190 @@ END; $$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION __setup_wide_table() RETURNS SETOF TEXT AS $$ +BEGIN + CREATE TABLE wide_table ( + id integer PRIMARY KEY GENERATED ALWAYS AS IDENTITY, + col_1 VARCHAR(50), + col_2 INT, + col_3 DATE, + col_4 BOOLEAN, + col_5 VARCHAR(50), + col_6 INT, + col_7 VARCHAR(50), + col_8 INT, + col_9 DATE, + col_10 BOOLEAN, + col_11 VARCHAR(50), + col_12 INT, + col_13 VARCHAR(50), + col_14 INT, + col_15 DATE, + col_16 BOOLEAN, + col_17 VARCHAR(50), + col_18 INT, + col_19 VARCHAR(50), + col_20 INT, + col_21 DATE, + col_22 BOOLEAN, + col_23 VARCHAR(50), + col_24 INT, + col_25 VARCHAR(50), + col_26 INT, + col_27 DATE, + col_28 BOOLEAN, + col_29 VARCHAR(50), + col_30 INT, + col_31 VARCHAR(50), + col_32 INT, + col_33 DATE, + col_34 BOOLEAN, + col_35 VARCHAR(50), + col_36 INT, + col_37 VARCHAR(50), + col_38 INT, + col_39 DATE, + col_40 BOOLEAN, + col_41 VARCHAR(50), + col_42 INT, + col_43 VARCHAR(50), + col_44 INT, + col_45 DATE, + col_46 BOOLEAN, + col_47 VARCHAR(50), + col_48 INT, + col_49 VARCHAR(50), + col_50 INT, + col_51 DATE, + col_52 BOOLEAN, + col_53 VARCHAR(50), + col_54 INT, + col_55 VARCHAR(50), + col_56 INT, + col_57 DATE, + col_58 BOOLEAN, + col_59 VARCHAR(50), + col_60 INT, + col_61 VARCHAR(50), + col_62 INT, + col_63 DATE, + col_64 BOOLEAN, + col_65 VARCHAR(50), + col_66 INT, + col_67 VARCHAR(50), + col_68 INT, + col_69 DATE, + col_70 BOOLEAN, + col_71 VARCHAR(50), + col_72 INT, + col_73 VARCHAR(50), + col_74 INT, + col_75 DATE, + col_76 BOOLEAN, + col_77 VARCHAR(50), + col_78 INT, + col_79 VARCHAR(50), + col_80 INT, + col_81 DATE, + col_82 BOOLEAN, + col_83 VARCHAR(50), + col_84 INT, + col_85 VARCHAR(50), + col_86 INT, + col_87 DATE, + col_88 BOOLEAN, + col_89 VARCHAR(50), + col_90 INT, + col_91 VARCHAR(50), + col_92 INT, + col_93 DATE, + col_94 BOOLEAN, + col_95 VARCHAR(50), + col_96 INT, + col_97 VARCHAR(50), + col_98 INT, + col_99 DATE, + col_100 BOOLEAN, + col_101 VARCHAR(50), + coltodrop integer + ); + ALTER TABLE wide_table DROP COLUMN coltodrop; + INSERT INTO wide_table ( + col_1, col_2, col_3, col_4, col_5, col_6, col_7, col_8, col_9, col_10, + col_11, col_12, col_13, col_14, col_15, col_16, col_17, col_18, col_19, col_20, + col_21, col_22, col_23, col_24, col_25, col_26, col_27, col_28, col_29, col_30, + col_31, col_32, col_33, col_34, col_35, col_36, col_37, col_38, col_39, col_40, + col_41, col_42, col_43, col_44, col_45, col_46, col_47, col_48, col_49, col_50, + col_51, col_52, col_53, col_54, col_55, col_56, col_57, col_58, col_59, col_60, + col_61, col_62, col_63, col_64, col_65, col_66, col_67, col_68, col_69, col_70, + col_71, col_72, col_73, col_74, col_75, col_76, col_77, col_78, col_79, col_80, + col_81, col_82, col_83, col_84, col_85, col_86, col_87, col_88, col_89, col_90, + col_91, col_92, col_93, col_94, col_95, col_96, col_97, col_98, col_99, col_100, + col_101 + ) VALUES ( + 'John', 42, '1980-05-12', TRUE, 'Developer', 50000, 'New York', 10, '2025-01-01', TRUE, + 'Alice', 35, 'Bob', 28, '1995-12-14', TRUE, 'Analyst', 40000, 'Chicago', 15, + '2025-01-03', TRUE, 'Eve', 31, 'Dave', 29, '1994-06-10', TRUE, 'Engineer', 48000, + 'Austin', 18, '2025-01-05', TRUE, 'Grace', 33, 'Frank', 40, '1983-03-15', TRUE, + 'Consultant', 60000, 'Denver', 30, '2025-01-07', TRUE, 'Carol', 27, 'Henry', 36, + '1987-09-20', TRUE, 'Technician', 42000, 'Phoenix', 17, '2025-01-09', TRUE, 'Judy', 45, + 'Zara', 32, '1991-01-25', TRUE, 'Data Scientist', 70000, 'Houston', 35, '2025-01-11', TRUE, + 'Product Manager', 65000, 'San Diego', 40, '2025-01-12', FALSE, 'Grace', 33, 'Frank', 40, + '1983-03-15', TRUE, 'Technician', 42000, 'Phoenix', 17, '2025-01-09', TRUE, 'Engineer', 48000, + 'Austin', 18, '2025-01-05', TRUE, 'Grace', 33, 'Frank', 40, '1983-03-15', FALSE, + 'California' + ); +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION test_list_records_from_wide_table() RETURNS SETOF TEXT AS $$ +DECLARE + rel_id oid; +BEGIN + PERFORM __setup_wide_table(); + rel_id := 'wide_table'::regclass::oid; + RETURN NEXT is( + msar.list_records_from_table( + tab_id => rel_id, + limit_ => null, + offset_ => null, + order_ => null, + filter_ => null, + group_ => null + ), + $j${ + "count": 1, + "results": [ + { + "1": 1, "2": "John", "3": 42, "4": "1980-05-12 AD", "5": true, + "6": "Developer", "7": 50000, "8": "New York", "9": 10, "10": "2025-01-01 AD", "11": true, + "12": "Alice", "13": 35, "14": "Bob", "15": 28, "16": "1995-12-14 AD", "17": true, + "18": "Analyst", "19": 40000, "20": "Chicago", "21": 15, "22": "2025-01-03 AD", "23": true, + "24": "Eve", "25": 31, "26": "Dave", "27": 29, "28": "1994-06-10 AD", "29": true, + "30": "Engineer", "31": 48000, "32": "Austin", "33": 18, "34": "2025-01-05 AD", "35": true, + "36": "Grace", "37": 33, "38": "Frank", "39": 40, "40": "1983-03-15 AD", "41": true, + "42": "Consultant", "43": 60000, "44": "Denver", "45": 30, "46": "2025-01-07 AD", "47": true, + "48": "Carol", "49": 27, "50": "Henry", "51": 36, "52": "1987-09-20 AD", "53": true, + "54": "Technician", "55": 42000, "56": "Phoenix", "57": 17, "58": "2025-01-09 AD", "59": true, + "60": "Judy", "61": 45, "62": "Zara", "63": 32, "64": "1991-01-25 AD", "65": true, + "66": "Data Scientist", "67": 70000, "68": "Houston", "69": 35, "70": "2025-01-11 AD", "71": true, + "72": "Product Manager", "73": 65000, "74": "San Diego", "75": 40, "76": "2025-01-12 AD", "77": false, + "78": "Grace", "79": 33, "80": "Frank", "81": 40, "82": "1983-03-15 AD", "83": true, + "84": "Technician", "85": 42000, "86": "Phoenix", "87": 17, "88": "2025-01-09 AD", "89": true, + "90": "Engineer", "91": 48000, "92": "Austin", "93": 18, "94": "2025-01-05 AD", "95": true, + "96": "Grace", "97": 33, "98": "Frank", "99": 40, "100": "1983-03-15 AD", "101": false, + "102": "California" + } + ], + "grouping": null, + "linked_record_summaries": null, + "record_summaries": null + }$j$ + ); +END; +$$ LANGUAGE plpgsql; + CREATE OR REPLACE FUNCTION test_list_records_from_table() RETURNS SETOF TEXT AS $$ DECLARE rel_id oid; diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 3a29c920a9..ea736bdfee 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,5 +1,4 @@ -version: "3.9" - +--- services: # Mathesar App built with the same configurations as the production image # but with additional testing dependencies. @@ -42,7 +41,7 @@ services: target: development dockerfile: Dockerfile args: - PYTHON_VERSION: ${PYTHON_VERSION-3.9-bookworm} + PYTHON_VERSION: ${PYTHON_VERSION-3.13-bookworm} environment: - MODE=${MODE-DEVELOPMENT} - DEBUG=${DEBUG-True} @@ -84,7 +83,7 @@ services: target: ${TARGET-testing} dockerfile: Dockerfile args: - PYTHON_VERSION: ${PYTHON_VERSION-3.9-bookworm} + PYTHON_VERSION: ${PYTHON_VERSION-3.13-bookworm} depends_on: - dev-db ports: diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 083d84cd1e..63639a0e38 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -25,7 +25,7 @@ services: target: testing dockerfile: Dockerfile args: - PYTHON_VERSION: ${PYTHON_VERSION-3.9-bookworm} + PYTHON_VERSION: ${PYTHON_VERSION-3.13-bookworm} environment: - ALLOWED_HOSTS=* - DJANGO_SUPERUSER_PASSWORD=password @@ -52,7 +52,7 @@ services: build: context: api_tests args: - PYTHON_VERSION: ${PYTHON_VERSION-3.9-bookworm} + PYTHON_VERSION: ${PYTHON_VERSION-3.13-bookworm} depends_on: api-test-service: condition: service_healthy diff --git a/docker-compose.yml b/docker-compose.yml index d1560a986c..845f361256 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,4 @@ -version: "3.9" - +--- # This file defines a viable production setup for Mathesar. # # It can be used in production directly, or used as an example to help define diff --git a/docs/docs/administration/install-from-scratch.md b/docs/docs/administration/install-from-scratch.md index 98a0e8f726..eabaa632ff 100644 --- a/docs/docs/administration/install-from-scratch.md +++ b/docs/docs/administration/install-from-scratch.md @@ -23,22 +23,14 @@ You should have **root access** to the machine you're installing Mathesar on. You'll need to install the following system packages before you install Mathesar: -- [Python](https://www.python.org/downloads/) 3.9, 3.10, or 3.11 (along with appropriate [`venv`](https://docs.python.org/3/library/venv.html) module) +- [Python](https://www.python.org/downloads/), along with the appropriate [`venv`](https://docs.python.org/3/library/venv.html) module. See [version support](version-support.md). - !!! note "Python version" - - Python _older_ than 3.9 will not run Mathesar. - - Python 3.12 will run Mathesar, but you'll have to take extra steps to get some dependencies to build. Installing a package for your OS that provides the `libpq-fe.h` header file should be enough in most cases. On Debian 12, this header is provided by the `libpq-dev` package. - -- [PostgreSQL](https://www.postgresql.org/download/linux/) 13 or newer (Verify by logging in, and running the query: `SELECT version();`) +- [PostgreSQL](https://www.postgresql.org/download/linux/) 13 or newer (Verify by logging in, and running the query: `SELECT version();`). See [version support](version-support.md). - [Caddy](https://caddyserver.com/docs/install) (Verify with `caddy version`) - [git](https://git-scm.com/downloads) (Verify with `git --version`) -- [GNU gettext](https://www.gnu.org/software/gettext/) (Verify with `gettext --version`) - - [unzip](https://packages.debian.org/search?keywords=unzip) A utility tool to de-archive .zip files (Verify with `unzip -v`) ### Domain (optional) @@ -144,7 +136,7 @@ Then press Enter to customize this guide with your domain name. ``` -m venv ./mathesar-venv - # /usr/bin/python3.9 -m venv ./mathesar-venv + # /usr/bin/python3.13 -m venv ./mathesar-venv ``` 1. Next we will activate our virtual environment: diff --git a/docs/docs/administration/version-support.md b/docs/docs/administration/version-support.md new file mode 100644 index 0000000000..7e6d49f3dd --- /dev/null +++ b/docs/docs/administration/version-support.md @@ -0,0 +1,22 @@ +# PostgreSQL and Python Version Support + +The general strategy of Mathesar is to support whichever versions of Python and PostgreSQL are supported upstream when each release is made. We will only remove support on minor version increases. The following table will be updated as future versions of Mathesar are released. + +| Mathesar version | Release Date | Python Versions | PostgreSQL versions | +|------------------|--------------|-----------------|---------------------| +| 0.2.x | 2025-01 | 3.9-3.13 | 13-17 | + +## Upstream EOL dates to note + +- Python 3.9 is supported upstream until October 2025 +- PostgreSQL 13 is supported upstream until November 2025 + +## Default Python and PostgreSQL versions for Mathesar 0.2.0 + +- Mathesar's Docker image uses PostgreSQL 17. +- Mathesar's Docker image uses Python 3.13. +- The default PostgreSQL version provided in our example `docker-compose.yml` is 13. + +## Regarding Python Support + +Python support is mostly only relevant for installations which followed the [Install From Scratch](install-from-scratch.md) instructions. diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 8c58c058d0..a6d091b9de 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -14,6 +14,7 @@ nav: - Upgrade Mathesar: administration/upgrade.md - Uninstall Mathesar: administration/uninstall.md - Debug Mathesar: administration/debug.md + - Version Support: administration/version-support.md - User Guide: - Introduction: user-guide/index.md - Your data in PostgreSQL: diff --git a/mathesar/rpc/exceptions/error_codes.py b/mathesar/rpc/exceptions/error_codes.py index af25368002..231ce16c61 100644 --- a/mathesar/rpc/exceptions/error_codes.py +++ b/mathesar/rpc/exceptions/error_codes.py @@ -535,13 +535,15 @@ def get_error_code(err: Exception) -> int: "MultipleDataFileAPIException": -28026, "MultipleObjectsReturned": -28027, "NetworkException": -28028, - "NotFoundAPIException": -28029, - "ProgrammingAPIException": -28030, - "TypeErrorAPIException": -28031, - "UnknownDatabaseTypeIdentifier": -28032, - "UnsupportedConstraintAPIException": -28033, - "UnsupportedInstallationDatabase": -28034, - "ValueAPIException": -28035, + "NoAdminConnectionAvailable": -28029, + "NoConnectionAvailable": -28030, + "NotFoundAPIException": -28031, + "ProgrammingAPIException": -28032, + "TypeErrorAPIException": -28033, + "UnknownDatabaseTypeIdentifier": -28034, + "UnsupportedConstraintAPIException": -28035, + "UnsupportedInstallationDatabase": -28036, + "ValueAPIException": -28037, }) dblib_error_map = frozendict({ diff --git a/mathesar_ui/src/components/Errors.svelte b/mathesar_ui/src/components/Errors.svelte deleted file mode 100644 index 3db46b9cdd..0000000000 --- a/mathesar_ui/src/components/Errors.svelte +++ /dev/null @@ -1,30 +0,0 @@ - - -{#if errors.length} - - {#if errors.length === 1} - {errors[0]} - {:else} - - {/if} - -{/if} - - diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/formatted-input/FormattedInputCell.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/formatted-input/FormattedInputCell.svelte index 303400a8a6..ac8ebeac42 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/formatted-input/FormattedInputCell.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/formatted-input/FormattedInputCell.svelte @@ -14,11 +14,6 @@ export let formatter: $$Props['formatter']; export let formatForDisplay: $$Props['formatForDisplay']; export let useTabularNumbers: $$Props['useTabularNumbers'] = undefined; - - $: cssVariables = { - '--input-element-text-align': 'right', - ...($$restProps.cssVariables || {}), - }; diff --git a/mathesar_ui/src/components/cell-fabric/data-types/components/money/MoneyCellInput.svelte b/mathesar_ui/src/components/cell-fabric/data-types/components/money/MoneyCellInput.svelte index b8437f6af4..d1a8816452 100644 --- a/mathesar_ui/src/components/cell-fabric/data-types/components/money/MoneyCellInput.svelte +++ b/mathesar_ui/src/components/cell-fabric/data-types/components/money/MoneyCellInput.svelte @@ -26,16 +26,11 @@ childValue = getNewChildValue(newParentValue); } $: handleParentValueChange(parentValue); - $: cssVariables = { - '--input-element-text-align': 'right', - ...($$restProps.cssVariables || {}), - }; + export let errorStrings: string[]; + + $: uniqueErrors = [...new Set(errorStrings)]; + + +{#if uniqueErrors.length === 1} +

{uniqueErrors[0]}

+{:else} +
    + {#each uniqueErrors as error (error)} +
  • {error}
  • + {/each} +
+{/if} + + diff --git a/mathesar_ui/src/components/errors/Errors.svelte b/mathesar_ui/src/components/errors/Errors.svelte new file mode 100644 index 0000000000..bd953dfeb4 --- /dev/null +++ b/mathesar_ui/src/components/errors/Errors.svelte @@ -0,0 +1,37 @@ + + +
+ {#each richErrors as richError} + + + + {/each} + + {#if stringErrors.length} + + + + {/if} +
+ + diff --git a/mathesar_ui/src/components/errors/customized/NoConnection.svelte b/mathesar_ui/src/components/errors/customized/NoConnection.svelte new file mode 100644 index 0000000000..46fee60a5a --- /dev/null +++ b/mathesar_ui/src/components/errors/customized/NoConnection.svelte @@ -0,0 +1,8 @@ + + +

{$_('not_a_collaborator_help')}

+

diff --git a/mathesar_ui/src/components/errors/customized/UnableToConnect.svelte b/mathesar_ui/src/components/errors/customized/UnableToConnect.svelte new file mode 100644 index 0000000000..e25ea0fa51 --- /dev/null +++ b/mathesar_ui/src/components/errors/customized/UnableToConnect.svelte @@ -0,0 +1,8 @@ + + +

{$_('unable_to_connect_to_database')}

+

{message}

diff --git a/mathesar_ui/src/components/errors/errorUtils.ts b/mathesar_ui/src/components/errors/errorUtils.ts new file mode 100644 index 0000000000..6415316be3 --- /dev/null +++ b/mathesar_ui/src/components/errors/errorUtils.ts @@ -0,0 +1,74 @@ +import { distinct } from 'iter-tools'; +import type { ComponentProps, ComponentType, SvelteComponent } from 'svelte'; + +import type { ComponentWithProps } from '@mathesar/component-library/types'; +import type { RpcError } from '@mathesar/packages/json-rpc-client-builder'; + +import NoConnection from './customized/NoConnection.svelte'; +import UnableToConnect from './customized/UnableToConnect.svelte'; + +const NO_CONNECTION_AVAILABLE = -28030; +const PSYCOPG_OPERATIONAL_ERROR = -30193; + +export type GeneralizedError = string | RpcError; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type ReturnableComponent = ComponentWithProps; + +function component( + c: ComponentType, + p: ComponentProps, +): ComponentWithProps { + return { component: c, props: p }; +} + +function getCustomizedRpcError({ + code, + message, +}: RpcError): string | ReturnableComponent { + switch (code) { + case NO_CONNECTION_AVAILABLE: + return component(NoConnection, {}); + case PSYCOPG_OPERATIONAL_ERROR: + return component(UnableToConnect, { message }); + default: + return message; + } +} + +function getCustomizedError( + error: GeneralizedError, +): string | ReturnableComponent { + return typeof error === 'string' ? error : getCustomizedRpcError(error); +} + +function getErrorHash(error: GeneralizedError): string { + if (typeof error === 'string') { + return JSON.stringify({ type: 'string', error }); + } + const { code, message } = error; + return JSON.stringify({ type: 'RpcError', code, message }); +} + +export function getDistinctErrors( + errors: Iterable, +): Iterable { + return distinct(getErrorHash, errors); +} + +export function groupErrors(errors: Iterable): { + stringErrors: string[]; + richErrors: ReturnableComponent[]; +} { + const stringErrors: string[] = []; + const richErrors: ReturnableComponent[] = []; + for (const error of errors) { + const customizedError = getCustomizedError(error); + if (typeof customizedError === 'string') { + stringErrors.push(customizedError); + } else { + richErrors.push(customizedError); + } + } + return { stringErrors, richErrors }; +} diff --git a/mathesar_ui/src/components/form/FieldErrors.svelte b/mathesar_ui/src/components/form/FieldErrors.svelte index 4c6df052e1..5397b9f6ef 100644 --- a/mathesar_ui/src/components/form/FieldErrors.svelte +++ b/mathesar_ui/src/components/form/FieldErrors.svelte @@ -1,5 +1,5 @@