Upgrade to Postgres 17

CHANGELOG.md

@@ -2,6 +2,21 @@
 
 All versions are tagged by the major Postgres version, plus an individual semver for this library itself.
 
+## 17-6.0.0   2024-09-27
+
+* Upgrade to Postgres 17
+* Deparser:
+  - Add support for deparsing `JSON_TABLE`, `JSON_QUERY`, `JSON_EXISTS`, `JSON_VALUE`
+  - Add support for deparsing `JSON`, `JSON_SCALAR`, `JSON_SERIALIZE`
+  - Add support for deparsing `COPY ... FORCE_NULL(*)`
+  - Add support for deparsing `ALTER COLUMN ... SET EXPRESSION AS`
+  - Add support for deparsing `SET STATISTICS DEFAULT`
+  - Add support for deparsing `SET ACCESS METHOD DEFAULT`
+  - Add support for deparsing `... AT LOCAL`
+  - Add support for deparsing `merge_action()`
+  - Add support for deparsing `MERGE ... RETURNING`
+  - Add support for deparsing `NOT MATCHED [ BY TARGET ]`
+
 ## 16-5.1.0   2024-01-08
 
 * Add support for compiling on Windows

Makefile

@@ -6,9 +6,9 @@ ARLIB = lib$(TARGET).a
 PGDIR = $(root_dir)/tmp/postgres
 PGDIRBZ2 = $(root_dir)/tmp/postgres.tar.bz2
 
-PG_VERSION = 16.1
+PG_VERSION = 17.0
 PG_VERSION_MAJOR = $(call word-dot,$(PG_VERSION),1)
-PG_VERSION_NUM = 160001
+PG_VERSION_NUM = 170000
 PROTOC_VERSION = 25.1
 
 VERSION = 5.1.0
@@ -125,12 +125,13 @@ $(PGDIR):
 	cd $(PGDIR); patch -p1 < $(root_dir)/patches/07_plpgsql_start_finish_datums.patch
 	cd $(PGDIR); patch -p1 < $(root_dir)/patches/08_avoid_zero_length_delimiter_in_regression_tests.patch
 	cd $(PGDIR); patch -p1 < $(root_dir)/patches/09_allow_param_junk.patch
+	cd $(PGDIR); patch -p1 < $(root_dir)/patches/10_avoid_namespace_hashtab_impl_gen.patch
 	cd $(PGDIR); ./configure $(PG_CONFIGURE_FLAGS)
-	cd $(PGDIR); rm src/pl/plpgsql/src/pl_gram.h
-	cd $(PGDIR); make -C src/pl/plpgsql/src pl_gram.h
+	cd $(PGDIR); make -C src/pl/plpgsql/src pl_gram.h plerrcodes.h pl_reserved_kwlist_d.h pl_unreserved_kwlist_d.h
 	cd $(PGDIR); make -C src/port pg_config_paths.h
 	cd $(PGDIR); make -C src/backend generated-headers
 	cd $(PGDIR); make -C src/backend parser-recursive # Triggers copying of includes to where they belong, as well as generating gram.c/scan.c
+	cd $(PGDIR); make -C src/common kwlist_d.h
 	# Avoid problems with static asserts
 	echo "#undef StaticAssertDecl" >> $(PGDIR)/src/include/c.h
 	echo "#define StaticAssertDecl(condition, errmessage)" >> $(PGDIR)/src/include/c.h
@@ -170,7 +171,9 @@ extract_source: $(PGDIR)
 	sed -i "" '$(shell echo 's/\#define PG_VERSION_NUM .*/#define PG_VERSION_NUM $(PG_VERSION_NUM)/')' pg_query.h
 	# Copy regress SQL files so we can use them in tests
 	rm -f ./test/sql/postgres_regress/*.sql
+	rm -f ./test/sql/plpgsql_regress/*.sql
 	cp $(PGDIR)/src/test/regress/sql/*.sql ./test/sql/postgres_regress/
+	cp $(PGDIR)/src/pl/plpgsql/src/sql/*.sql ./test/sql/plpgsql_regress/
 
 .c.o:
 	@$(ECHO) compiling $(<)

README.md

@@ -12,7 +12,7 @@ You can find further background to why a query's parse tree is useful here: http
 ## Installation
 
 ```sh
-git clone -b 16-latest git://github.com/pganalyze/libpg_query
+git clone -b 17-latest git://github.com/pganalyze/libpg_query
 cd libpg_query
 make
 ```
@@ -57,7 +57,7 @@ This will output the parse tree (whitespace adjusted here for better readability
 
 ```json
 {
-    "version": 160001,
+    "version": 170000,
     "stmts": [
         {
             "stmt": {
@@ -130,7 +130,7 @@ int main() {
 This will output the following:
 
 ```
-  version: 160001, tokens: 7, size: 77
+  version: 170000, tokens: 7, size: 77
   "SELECT" = [ 0, 6, SELECT, RESERVED_KEYWORD ]
   "update" = [ 7, 13, UPDATE, UNRESERVED_KEYWORD ]
   "AS" = [ 14, 16, AS, RESERVED_KEYWORD ]
@@ -237,7 +237,8 @@ Each major version is maintained in a dedicated git branch. Only the latest Post
 
 | PostgreSQL Major Version | Branch     | Status              |
 |--------------------------|------------|---------------------|
-| 16                       | 16-latest  | Active development  |
+| 17                       | 17-latest  | Active development  |
+| 16                       | 16-latest  | Critical fixes only |
 | 15                       | 15-latest  | Critical fixes only |
 | 14                       | 14-latest  | Critical fixes only |
 | 13                       | 13-latest  | Critical fixes only |
@@ -291,4 +292,4 @@ Portions Copyright (c) 1994, The Regents of the University of California
 
 All other parts are licensed under the 3-clause BSD license, see LICENSE file for details.<br>
 Copyright (c) 2015, Lukas Fittl <lukas@fittl.com>
-Copyright (c) 2016-2023, Duboce Labs, Inc. (pganalyze) <team@pganalyze.com>
+Copyright (c) 2016-2024, Duboce Labs, Inc. (pganalyze) <team@pganalyze.com>

patches/01_parser_additional_param_ref_support.patch

@@ -23,10 +23,10 @@ Date:   Sun Jan 3 15:06:25 2021 -0800
     SELECT INTERVAL (6) $1;
 
 diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
-index c31b373358..164b9586c1 100644
+index 215d47e5a2..452e17edf2 100644
 --- a/src/backend/parser/gram.y
 +++ b/src/backend/parser/gram.y
-@@ -178,6 +178,8 @@ static Node *makeBoolAConst(bool state, int location);
+@@ -177,6 +177,8 @@ static Node *makeBoolAConst(bool state, int location);
  static Node *makeBitStringConst(char *str, int location);
  static Node *makeNullAConst(int location);
  static Node *makeAConst(Node *v, int location);
@@ -35,7 +35,7 @@ index c31b373358..164b9586c1 100644
  static RoleSpec *makeRoleSpec(RoleSpecType type, int location);
  static void check_qualified_name(List *names, core_yyscan_t yyscanner);
  static List *check_func_name(List *names, core_yyscan_t yyscanner);
-@@ -1176,6 +1178,11 @@ AlterOptRoleElem:
+@@ -1210,6 +1212,11 @@ AlterOptRoleElem:
  					$$ = makeDefElem("password",
  									 (Node *) makeString($2), @1);
  				}
@@ -47,7 +47,7 @@ index c31b373358..164b9586c1 100644
  			| PASSWORD NULL_P
  				{
  					$$ = makeDefElem("password", NULL, @1);
-@@ -1190,6 +1197,16 @@ AlterOptRoleElem:
+@@ -1224,6 +1231,16 @@ AlterOptRoleElem:
  					$$ = makeDefElem("password",
  									 (Node *) makeString($3), @1);
  				}
@@ -64,7 +64,7 @@ index c31b373358..164b9586c1 100644
  			| UNENCRYPTED PASSWORD Sconst
  				{
  					ereport(ERROR,
-@@ -1703,6 +1720,14 @@ set_rest_more:	/* Generic SET syntaxes: */
+@@ -1737,6 +1754,14 @@ set_rest_more:	/* Generic SET syntaxes: */
  					n->args = list_make1(makeStringConst($2, @2));
  					$$ = n;
  				}
@@ -79,7 +79,7 @@ index c31b373358..164b9586c1 100644
  			| NAMES opt_encoding
  				{
  					VariableSetStmt *n = makeNode(VariableSetStmt);
-@@ -1724,6 +1749,14 @@ set_rest_more:	/* Generic SET syntaxes: */
+@@ -1758,6 +1783,14 @@ set_rest_more:	/* Generic SET syntaxes: */
  					n->args = list_make1(makeStringConst($2, @2));
  					$$ = n;
  				}
@@ -94,7 +94,7 @@ index c31b373358..164b9586c1 100644
  			| SESSION AUTHORIZATION NonReservedWord_or_Sconst
  				{
  					VariableSetStmt *n = makeNode(VariableSetStmt);
-@@ -1733,6 +1766,14 @@ set_rest_more:	/* Generic SET syntaxes: */
+@@ -1767,6 +1800,14 @@ set_rest_more:	/* Generic SET syntaxes: */
  					n->args = list_make1(makeStringConst($3, @3));
  					$$ = n;
  				}
@@ -109,7 +109,7 @@ index c31b373358..164b9586c1 100644
  			| SESSION AUTHORIZATION DEFAULT
  				{
  					VariableSetStmt *n = makeNode(VariableSetStmt);
-@@ -1775,6 +1816,8 @@ var_value:	opt_boolean_or_string
+@@ -1809,6 +1850,8 @@ var_value:	opt_boolean_or_string
  				{ $$ = makeStringConst($1, @1); }
  			| NumericOnly
  				{ $$ = makeAConst($1, @1); }
@@ -118,7 +118,7 @@ index c31b373358..164b9586c1 100644
  		;
 
  iso_level:	READ UNCOMMITTED						{ $$ = "read uncommitted"; }
-@@ -1808,6 +1851,10 @@ zone_value:
+@@ -1842,6 +1885,10 @@ zone_value:
  				{
  					$$ = makeStringConst($1, @1);
  				}
@@ -129,7 +129,7 @@ index c31b373358..164b9586c1 100644
  			| IDENT
  				{
  					$$ = makeStringConst($1, @1);
-@@ -16125,6 +16172,10 @@ extract_list:
+@@ -16590,6 +16637,10 @@ extract_list:
  				{
  					$$ = list_make2(makeStringConst($1, @1), $3);
  				}
@@ -140,7 +140,7 @@ index c31b373358..164b9586c1 100644
  		;
 
  /* Allow delimited string Sconst in extract_arg as an SQL extension.
-@@ -16669,6 +16720,45 @@ AexprConst: Iconst
+@@ -17246,6 +17297,45 @@ AexprConst: Iconst
  					t->location = @1;
  					$$ = makeStringConstCast($6, @6, t);
  				}
@@ -186,7 +186,7 @@ index c31b373358..164b9586c1 100644
  			| ConstTypename Sconst
  				{
  					$$ = makeStringConstCast($2, @2, $1);
-@@ -16688,6 +16778,23 @@ AexprConst: Iconst
+@@ -17265,6 +17355,23 @@ AexprConst: Iconst
  											makeIntConst($3, @3));
  					$$ = makeStringConstCast($5, @5, t);
  				}
@@ -210,7 +210,7 @@ index c31b373358..164b9586c1 100644
  			| TRUE_P
  				{
  					$$ = makeBoolAConst(true, @1);
-@@ -18093,6 +18200,24 @@ makeAConst(Node *v, int location)
+@@ -18698,6 +18805,24 @@ makeAConst(Node *v, int location)
  	return n;
  }
 

patches/03_lexer_track_yyllocend.patch

@@ -8,18 +8,18 @@ Date:   Sun Jan 3 15:59:40 2021 -0800
     as this is made available by pg_query for uses such as syntax highlighting.
 
 diff --git a/src/backend/parser/scan.l b/src/backend/parser/scan.l
-index b2216a9eac..a62798cf98 100644
+index 3248fb5108..28fdd262b1 100644
 --- a/src/backend/parser/scan.l
 +++ b/src/backend/parser/scan.l
-@@ -537,6 +537,7 @@ other			.
+@@ -538,6 +538,7 @@ other			.
  					{
  						/* If NCHAR isn't a keyword, just return "n" */
  						yylval->str = pstrdup("n");
 +						yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
  						return IDENT;
  					}
  				}
-@@ -605,9 +606,11 @@ other			.
+@@ -606,9 +607,11 @@ other			.
  					{
  						case xb:
  							yylval->str = litbufdup(yyscanner);
@@ -31,7 +31,7 @@ index b2216a9eac..a62798cf98 100644
  							return XCONST;
  						case xq:
  						case xe:
-@@ -620,9 +623,11 @@ other			.
+@@ -621,9 +624,11 @@ other			.
  											   yyextra->literallen,
  											   false);
  							yylval->str = litbufdup(yyscanner);
@@ -43,39 +43,39 @@ index b2216a9eac..a62798cf98 100644
  							return USCONST;
  						default:
  							yyerror("unhandled previous state in xqs");
-@@ -760,6 +765,7 @@ other			.
+@@ -761,6 +766,7 @@ other			.
  						yyextra->dolqstart = NULL;
  						BEGIN(INITIAL);
  						yylval->str = litbufdup(yyscanner);
 +						yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
  						return SCONST;
  					}
  					else
-@@ -805,6 +811,7 @@ other			.
+@@ -806,6 +812,7 @@ other			.
  					if (yyextra->literallen >= NAMEDATALEN)
  						truncate_identifier(ident, yyextra->literallen, true);
  					yylval->str = ident;
 +					yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
  					return IDENT;
  				}
  <xui>{dquote}	{
-@@ -813,6 +820,7 @@ other			.
+@@ -814,6 +821,7 @@ other			.
  						yyerror("zero-length delimited identifier");
  					/* can't truncate till after we de-escape the ident */
  					yylval->str = litbufdup(yyscanner);
 +					yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
  					return UIDENT;
  				}
  <xd,xui>{xddouble}	{
-@@ -832,6 +840,7 @@ other			.
+@@ -833,6 +841,7 @@ other			.
  					/* and treat it as {identifier} */
  					ident = downcase_truncate_identifier(yytext, yyleng, true);
  					yylval->str = ident;
 +					yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
  					return IDENT;
  				}
 
-@@ -1096,6 +1105,7 @@ other			.
+@@ -1097,6 +1106,7 @@ other			.
  					 */
  					ident = downcase_truncate_identifier(yytext, yyleng, true);
  					yylval->str = ident;
@@ -84,7 +84,7 @@ index b2216a9eac..a62798cf98 100644
  				}
 
 diff --git a/src/include/parser/scanner.h b/src/include/parser/scanner.h
-index da013837cd..e2a73201c4 100644
+index d6293b1e87..59f492c6de 100644
 --- a/src/include/parser/scanner.h
 +++ b/src/include/parser/scanner.h
 @@ -113,6 +113,8 @@ typedef struct core_yy_extra_type

patches/04_lexer_comments_as_tokens.patch

@@ -12,10 +12,10 @@ Date:   Sun Jan 3 16:01:44 2021 -0800
     this change, the lexer returns them as SQL_COMMENT/C_COMMENT tokens.
 
 diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
-index 164b9586c1..7dd5505cd1 100644
+index 452e17edf2..87e6e1c858 100644
 --- a/src/backend/parser/gram.y
 +++ b/src/backend/parser/gram.y
-@@ -678,6 +678,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
+@@ -695,6 +695,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
  %token <ival>	ICONST PARAM
  %token			TYPECAST DOT_DOT COLON_EQUALS EQUALS_GREATER
  %token			LESS_EQUALS GREATER_EQUALS NOT_EQUALS
@@ -24,7 +24,7 @@ index 164b9586c1..7dd5505cd1 100644
  /*
   * If you want to make any keyword changes, update the keyword table in
 diff --git a/src/backend/parser/parser.c b/src/backend/parser/parser.c
-index e17c310cc1..9b2926878a 100644
+index 118488c3f3..551b37a09a 100644
 --- a/src/backend/parser/parser.c
 +++ b/src/backend/parser/parser.c
 @@ -156,6 +156,9 @@ base_yylex(YYSTYPE *lvalp, YYLTYPE *llocp, core_yyscan_t yyscanner)
@@ -38,10 +38,10 @@ index e17c310cc1..9b2926878a 100644
  			return cur_token;
  	}
 diff --git a/src/backend/parser/scan.l b/src/backend/parser/scan.l
-index a62798cf98..154defc6b8 100644
+index 28fdd262b1..5c930dfda2 100644
 --- a/src/backend/parser/scan.l
 +++ b/src/backend/parser/scan.l
-@@ -226,7 +226,7 @@ non_newline		[^\n\r]
+@@ -226,7 +226,7 @@ non_newline			[^\n\r]
 
  comment			("--"{non_newline}*)
 
@@ -55,13 +55,13 @@ index a62798cf98..154defc6b8 100644
   */
 
 -special_whitespace		({space}+|{comment}{newline})
--horiz_whitespace		({horiz_space}|{comment})
+-non_newline_whitespace	({non_newline_space}|{comment})
 +special_whitespace		({space}+)
-+horiz_whitespace		({horiz_space})
- whitespace_with_newline	({horiz_whitespace}*{newline}{special_whitespace}*)
++non_newline_whitespace	({non_newline_space})
+ whitespace_with_newline	({non_newline_whitespace}*{newline}{special_whitespace}*)
 
  quote			'
-@@ -443,6 +443,11 @@ other			.
+@@ -444,6 +444,11 @@ other			.
  					/* ignore */
  				}
 
@@ -73,7 +73,7 @@ index a62798cf98..154defc6b8 100644
  {xcstart}		{
  					/* Set location in case of syntax error in comment */
  					SET_YYLLOC();
-@@ -461,7 +466,11 @@ other			.
+@@ -462,7 +467,11 @@ other			.
 
  {xcstop}		{
  					if (yyextra->xcdepth <= 0)
@@ -86,7 +86,7 @@ index a62798cf98..154defc6b8 100644
  						(yyextra->xcdepth)--;
  				}
 diff --git a/src/interfaces/ecpg/preproc/parser.c b/src/interfaces/ecpg/preproc/parser.c
-index 38e7acb680..4814158aaf 100644
+index 9daeee3303..1c32d3ccce 100644
 --- a/src/interfaces/ecpg/preproc/parser.c
 +++ b/src/interfaces/ecpg/preproc/parser.c
 @@ -86,6 +86,9 @@ filtered_base_yylex(void)
@@ -100,7 +100,7 @@ index 38e7acb680..4814158aaf 100644
  			return cur_token;
  	}
 diff --git a/src/pl/plpgsql/src/pl_gram.y b/src/pl/plpgsql/src/pl_gram.y
-index 6a09bfdd67..fb1b519b27 100644
+index 97be9239e3..165b9b0a8c 100644
 --- a/src/pl/plpgsql/src/pl_gram.y
 +++ b/src/pl/plpgsql/src/pl_gram.y
 @@ -232,6 +232,7 @@ static	void			check_raise_parameters(PLpgSQL_stmt_raise *stmt);
@@ -112,10 +112,10 @@ index 6a09bfdd67..fb1b519b27 100644
  /*
   * Other tokens recognized by plpgsql's lexer interface layer (pl_scanner.c).
 diff --git a/src/pl/plpgsql/src/pl_scanner.c b/src/pl/plpgsql/src/pl_scanner.c
-index 4e98af2395..40a17caaf9 100644
+index 9407da51ef..51919febe5 100644
 --- a/src/pl/plpgsql/src/pl_scanner.c
 +++ b/src/pl/plpgsql/src/pl_scanner.c
-@@ -342,6 +342,11 @@ internal_yylex(TokenAuxData *auxdata)
+@@ -359,6 +359,11 @@ internal_yylex(TokenAuxData *auxdata)
  		{
  			auxdata->lval.str = pstrdup(yytext);
  		}