diff --git a/CHANGELOG b/CHANGELOG index 5ac3e6329..9004d6da0 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,10 @@ # All notable changes to this project will be documented in this file. # This project adheres to [Semantic Versioning](http://semver.org/). +## [0.41.1] 2022-08-30 +### Added +- Add 4 new knobs to align assignment operators and dictionary colons. They are align_assignment, align_argument_assignment, align_dict_colon and new_alignment_after_commentline. + ## [0.40.0] UNRELEASED ### Added - Add a new Python parser to generate logical lines. diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 054ef2652..1852a9133 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -15,3 +15,4 @@ Sam Clegg Łukasz Langa Oleg Butuzov Mauricio Herrera Cuadra +Xiao Wang \ No newline at end of file diff --git a/README.rst b/README.rst index 5734a5d76..c54801650 100644 --- a/README.rst +++ b/README.rst @@ -390,6 +390,61 @@ Options:: Knobs ===== +``ALIGN_ASSIGNMENT`` + Align assignment or augmented assignment operators. + If there is a blank line or a newline comment or a multiline object + (e.g. a dictionary, a list, a function call) in between, + it will start new block alignment. Lines in the same block have the same + indentation level. + + .. code-block:: python + + a = 1 + abc = 2 + if condition == None: + var += '' + var_long -= 4 + b = 3 + bc = 4 + +``ALIGN_ARGUMENT_ASSIGNMENT`` + Align assignment operators in the argument list if they are all split on newlines. + Arguments without assignment in between will initiate new block alignment calulation; + for example, a comment line. + Multiline objects in between will also initiate a new alignment block. + + .. code-block:: python + + rglist = test( + var_first = 0, + var_second = '', + var_dict = { + "key_1" : '', + "key_2" : 2, + "key_3" : True, + }, + var_third = 1, + var_very_long = None ) + +``ALIGN_DICT_COLON`` + Align the colons in the dictionary if all entries in dictionay are split on newlines + or 'EACH_DICT_ENTRY_ON_SEPERATE_LINE' is set True. + A commentline or multi-line object in between will start new alignment block. + + .. code-block:: python + + fields = + { + "field" : "ediid", + "type" : "text", + # key: value + "required" : True, + } + +``NEW_ALIGNMENT_AFTER_COMMENTLINE`` + Make it optional to start a new alignmetn block for assignment + alignment and colon alignment after a comment line. + ``ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT`` Align closing bracket with visual indentation. diff --git a/yapf/pytree/subtype_assigner.py b/yapf/pytree/subtype_assigner.py index dd3ea3d1e..0ee247a82 100644 --- a/yapf/pytree/subtype_assigner.py +++ b/yapf/pytree/subtype_assigner.py @@ -240,6 +240,7 @@ def Visit_argument(self, node): # pylint: disable=invalid-name # argument ::= # test [comp_for] | test '=' test self._ProcessArgLists(node) + #TODO add a subtype to each argument? def Visit_arglist(self, node): # pylint: disable=invalid-name # arglist ::= @@ -309,6 +310,10 @@ def Visit_typedargslist(self, node): # pylint: disable=invalid-name tname = True _SetArgListSubtype(child, subtypes.TYPED_NAME, subtypes.TYPED_NAME_ARG_LIST) + # NOTE Every element of the tynamme argument + # should have this list type + _AppendSubtypeRec(child, subtypes.TYPED_NAME_ARG_LIST) + elif child.type == grammar_token.COMMA: tname = False elif child.type == grammar_token.EQUAL and tname: diff --git a/yapf/yapflib/format_decision_state.py b/yapf/yapflib/format_decision_state.py index c299d1c85..efcef0ba4 100644 --- a/yapf/yapflib/format_decision_state.py +++ b/yapf/yapflib/format_decision_state.py @@ -978,6 +978,7 @@ def _GetNewlineColumn(self): not self.param_list_stack[-1].SplitBeforeClosingBracket( top_of_stack.indent) and top_of_stack.indent == ((self.line.depth + 1) * style.Get('INDENT_WIDTH'))): + # NOTE: comment inside argument list is not excluded in subtype assigner if (subtypes.PARAMETER_START in current.subtypes or (previous.is_comment and subtypes.PARAMETER_START in previous.subtypes)): diff --git a/yapf/yapflib/format_token.py b/yapf/yapflib/format_token.py index 6eea05473..070987851 100644 --- a/yapf/yapflib/format_token.py +++ b/yapf/yapflib/format_token.py @@ -322,3 +322,82 @@ def is_pytype_comment(self): def is_copybara_comment(self): return self.is_comment and re.match( r'#.*\bcopybara:\s*(strip|insert|replace)', self.value) + + @property + def is_assign(self): + return subtypes.ASSIGN_OPERATOR in self.subtypes + + @property + def is_dict_colon(self): + # if the token is dictionary colon and + # the dictionary has no comp_for + return self.value == ':' and self.previous_token.is_dict_key + + @property + def is_dict_key(self): + # if the token is dictionary key which is not preceded by doubel stars and + # the dictionary has no comp_for + return subtypes.DICTIONARY_KEY_PART in self.subtypes + + @property + def is_dict_key_start(self): + # if the token is dictionary key start + return subtypes.DICTIONARY_KEY in self.subtypes + + @property + def is_dict_value(self): + return subtypes.DICTIONARY_VALUE in self.subtypes + + @property + def is_augassign(self): + augassigns = {'+=', '-=' , '*=' , '@=' , '/=' , '%=' , '&=' , '|=' , '^=' , + '<<=' , '>>=' , '**=' , '//='} + return self.value in augassigns + + @property + def is_argassign(self): + return (subtypes.DEFAULT_OR_NAMED_ASSIGN in self.subtypes + or subtypes.VARARGS_LIST in self.subtypes) + + @property + def is_argname(self): + # it's the argument part before argument assignment operator, + # including tnames and data type + # not the assign operator, + # not the value after the assign operator + + # argument without assignment is also included + # the token is arg part before '=' but not after '=' + if self.is_argname_start: + return True + + # exclude comment inside argument list + if not self.is_comment: + # the token is any element in typed arglist + if subtypes.TYPED_NAME_ARG_LIST in self.subtypes: + return True + + return False + + @property + def is_argname_start(self): + # return true if it's the start of every argument entry + previous_subtypes = {0} + if self.previous_token: + previous_subtypes = self.previous_token.subtypes + + return ( + (not self.is_comment + and subtypes.DEFAULT_OR_NAMED_ASSIGN not in self.subtypes + and subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST in self.subtypes + and subtypes.DEFAULT_OR_NAMED_ASSIGN not in previous_subtypes + and (not subtypes.PARAMETER_STOP in self.subtypes + or subtypes.PARAMETER_START in self.subtypes) + ) + or # if there is comment, the arg after it is the argname start + (not self.is_comment and self.previous_token and self.previous_token.is_comment + and + (subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST in previous_subtypes + or subtypes.TYPED_NAME_ARG_LIST in self.subtypes + or subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST in self.subtypes)) + ) diff --git a/yapf/yapflib/reformatter.py b/yapf/yapflib/reformatter.py index 14e0bde70..8f8a103f8 100644 --- a/yapf/yapflib/reformatter.py +++ b/yapf/yapflib/reformatter.py @@ -22,6 +22,7 @@ from __future__ import unicode_literals import collections +from distutils.errors import LinkError import heapq import re @@ -102,6 +103,14 @@ def Reformat(llines, verify=False, lines=None): final_lines.append(lline) prev_line = lline + if style.Get('ALIGN_ASSIGNMENT'): + _AlignAssignment(final_lines) + if (style.Get('EACH_DICT_ENTRY_ON_SEPARATE_LINE') + and style.Get('ALIGN_DICT_COLON')): + _AlignDictColon(final_lines) + if style.Get('ALIGN_ARGUMENT_ASSIGNMENT'): + _AlignArgAssign(final_lines) + _AlignTrailingComments(final_lines) return _FormatFinalLines(final_lines, verify) @@ -394,6 +403,504 @@ def _AlignTrailingComments(final_lines): final_lines_index += 1 +def _AlignAssignment(final_lines): + """Align assignment operators and augmented assignment operators to the same column""" + + final_lines_index = 0 + while final_lines_index < len(final_lines): + line = final_lines[final_lines_index] + + assert line.tokens + process_content = False + + for tok in line.tokens: + if tok.is_assign or tok.is_augassign: + # all pre assignment variable lengths in one block of lines + all_pa_variables_lengths = [] + max_variables_length = 0 + + while True: + # EOF + if final_lines_index + len(all_pa_variables_lengths) == len(final_lines): + break + + this_line_index = final_lines_index + len(all_pa_variables_lengths) + this_line = final_lines[this_line_index] + + next_line = None + if this_line_index < len(final_lines) - 1: + next_line = final_lines[final_lines_index + len(all_pa_variables_lengths) + 1 ] + + assert this_line.tokens, next_line.tokens + + # align them differently when there is a blank line in between + if (all_pa_variables_lengths and + this_line.tokens[0].formatted_whitespace_prefix.startswith('\n\n') + ): + break + + # if there is a standalone comment or keyword statement line + # or other lines without assignment in between, break + elif (all_pa_variables_lengths and + True not in [tok.is_assign or tok.is_augassign for tok in this_line.tokens]): + if this_line.tokens[0].is_comment: + if style.Get('NEW_ALIGNMENT_AFTER_COMMENTLINE'): + break + else: break + + if this_line.disable: + all_pa_variables_lengths.append([]) + continue + + variables_content = '' + pa_variables_lengths = [] + contain_object = False + line_tokens = this_line.tokens + # only one assignment expression is on each line + for index in range(len(line_tokens)): + line_tok = line_tokens[index] + + prefix = line_tok.formatted_whitespace_prefix + newline_index = prefix.rfind('\n') + if newline_index != -1: + variables_content = '' + prefix = prefix[newline_index + 1:] + + if line_tok.is_assign or line_tok.is_augassign: + next_toks = [line_tokens[i] for i in range(index+1, len(line_tokens))] + # if there is object(list/tuple/dict) with newline entries, break, + # update the alignment so far and start to calulate new alignment + for tok in next_toks: + if tok.value in ['(', '[', '{'] and tok.next_token: + if (tok.next_token.formatted_whitespace_prefix.startswith('\n') + or (tok.next_token.is_comment and tok.next_token.next_token.formatted_whitespace_prefix.startswith('\n'))): + pa_variables_lengths.append(len(variables_content)) + contain_object = True + break + if not contain_object: + if line_tok.is_assign: + pa_variables_lengths.append(len(variables_content)) + # if augassign, add the extra augmented part to the max length caculation + elif line_tok.is_augassign: + pa_variables_lengths.append(len(variables_content) + len(line_tok.value) - 1 ) + # don't add the tokens + # after the assignment operator + break + else: + variables_content += '{}{}'.format(prefix, line_tok.value) + + if pa_variables_lengths: + max_variables_length = max(max_variables_length, max(pa_variables_lengths)) + + all_pa_variables_lengths.append(pa_variables_lengths) + + # after saving this line's max variable length, + # we check if next line has the same depth as this line, + # if not, we don't want to calculate their max variable length together + # so we break the while loop, update alignment so far, and + # then go to next line that has '=' + if next_line: + if this_line.depth != next_line.depth: + break + # if this line contains objects with newline entries, + # start new block alignment + if contain_object: + break + + # if no update of max_length, just go to the next block + if max_variables_length == 0: continue + + max_variables_length += 2 + + # Update the assignment token values based on the max variable length + for all_pa_variables_lengths_index, pa_variables_lengths in enumerate( + all_pa_variables_lengths): + if not pa_variables_lengths: + continue + this_line = final_lines[final_lines_index + all_pa_variables_lengths_index] + + # only the first assignment operator on each line + pa_variables_lengths_index = 0 + for line_tok in this_line.tokens: + if line_tok.is_assign or line_tok.is_augassign: + assert pa_variables_lengths[0] < max_variables_length + + if pa_variables_lengths_index < len(pa_variables_lengths): + whitespace = ' ' * ( + max_variables_length - pa_variables_lengths[0] - 1) + + assign_content = '{}{}'.format(whitespace, line_tok.value.strip()) + + existing_whitespace_prefix = \ + line_tok.formatted_whitespace_prefix.lstrip('\n') + + # in case the existing spaces are larger than padded spaces + if (len(whitespace) == 1 or len(whitespace) > 1 and + len(existing_whitespace_prefix)>len(whitespace)): + line_tok.whitespace_prefix = '' + elif assign_content.startswith(existing_whitespace_prefix): + assign_content = assign_content[len(existing_whitespace_prefix):] + + # update the assignment operator value + line_tok.value = assign_content + + pa_variables_lengths_index += 1 + + final_lines_index += len(all_pa_variables_lengths) + + process_content = True + break + + if not process_content: + final_lines_index += 1 + + +def _AlignArgAssign(final_lines): + """Align the assign operators in a argument list to the same column""" + """NOTE One argument list of one function is on one logical line! + But funtion calls/argument lists can be in argument list. + """ + final_lines_index = 0 + while final_lines_index < len(final_lines): + line = final_lines[final_lines_index] + if line.disable: + final_lines_index += 1 + continue + + assert line.tokens + process_content = False + + for tok in line.tokens: + if tok.is_argassign: + + this_line = line + line_tokens = this_line.tokens + + for open_index in range(len(line_tokens)): + line_tok = line_tokens[open_index] + + if (line_tok.value == '(' and not line_tok.is_pseudo + and line_tok.next_token.formatted_whitespace_prefix.startswith('\n')): + index = open_index + # skip the comments in the beginning + index += 1 + line_tok = line_tokens[index] + while not line_tok.is_argname_start and index < len(line_tokens)-1: + index += 1 + line_tok = line_tokens[index] + + # check if the argstart is on newline + if line_tok.is_argname_start and line_tok.formatted_whitespace_prefix.startswith('\n'): + first_arg_index = index + first_arg_column = len(line_tok.formatted_whitespace_prefix.lstrip('\n')) + + closing = False + all_arg_name_lengths = [] + arg_name_lengths = [] + name_content = '' + arg_column = first_arg_column + + # start with the first argument + # that has nextline prefix + while not closing: + # if there is a comment in between, save, reset and continue to calulate new alignment + if (style.Get('NEW_ALIGNMENT_AFTER_COMMENTLINE') + and arg_name_lengths and line_tok.is_comment + and line_tok.formatted_whitespace_prefix.startswith('\n')): + all_arg_name_lengths.append(arg_name_lengths) + arg_name_lengths = [] + index += 1 + line_tok = line_tokens[index] + continue + + prefix = line_tok.formatted_whitespace_prefix + newline_index = prefix.rfind('\n') + + if newline_index != -1: + if line_tok.is_argname_start: + name_content = '' + prefix = prefix[newline_index + 1:] + arg_column = len(prefix) + # if a typed arg name is so long + # that there are newlines inside + # only calulate the last line arg_name that has the assignment + elif line_tok.is_argname: + name_content = '' + prefix = prefix[newline_index + 1:] + # if any argument not on newline + elif line_tok.is_argname_start: + name_content = '' + arg_column = line_tok.column + # in case they are formatted into one line in final_line + # but are put in separated lines in original codes + if arg_column == first_arg_column: + arg_column = line_tok.formatted_whitespace_prefix + # on the same argument level + if (line_tok.is_argname_start and arg_name_lengths + and arg_column==first_arg_column): + argname_end = line_tok + while argname_end.is_argname: + argname_end = argname_end.next_token + # argument without assignment in between + if not argname_end.is_argassign: + all_arg_name_lengths.append(arg_name_lengths) + arg_name_lengths = [] + index += 1 + line_tok = line_tokens[index] + continue + + if line_tok.is_argassign and arg_column == first_arg_column: + arg_name_lengths.append(len(name_content)) + elif line_tok.is_argname and arg_column == first_arg_column: + name_content += '{}{}'.format(prefix, line_tok.value) + # add up all token values before the arg assign operator + + index += 1 + if index < len(line_tokens): + line_tok = line_tokens[index] + # when the matching closing bracket is never found + # due to edge cases where the closing bracket + # is not indented or dedented + else: + all_arg_name_lengths.append(arg_name_lengths) + break + + # if there is a new object(list/tuple/dict) with its entries on newlines, + # save, reset and continue to calulate new alignment + if (line_tok.value in ['(', '[','{'] and line_tok.next_token + and line_tok.next_token.formatted_whitespace_prefix.startswith('\n')): + if arg_name_lengths: + all_arg_name_lengths.append(arg_name_lengths) + arg_name_lengths = [] + index += 1 + line_tok = line_tokens[index] + continue + + if line_tok.value == ')'and not line_tok.is_pseudo: + if line_tok.formatted_whitespace_prefix.startswith('\n'): + close_column = len(line_tok.formatted_whitespace_prefix.lstrip('\n')) + else: close_column = line_tok.column + if close_column < first_arg_column: + if arg_name_lengths: + all_arg_name_lengths.append(arg_name_lengths) + closing = True + + # update the alignment once one full arg list is processed + if all_arg_name_lengths: + # if argument list with only the first argument on newline + if len(all_arg_name_lengths) == 1 and len(all_arg_name_lengths[0]) == 1: + continue + max_name_length = 0 + all_arg_name_lengths_index = 0 + arg_name_lengths = all_arg_name_lengths[all_arg_name_lengths_index] + max_name_length = max(arg_name_lengths or [0]) + 2 + arg_lengths_index = 0 + for token in line_tokens[first_arg_index:index]: + if token.is_argassign: + name_token = token.previous_token + while name_token.is_argname and not name_token.is_argname_start: + name_token = name_token.previous_token + name_column = len(name_token.formatted_whitespace_prefix.lstrip('\n')) + if name_column == first_arg_column: + if all_arg_name_lengths_index < len(all_arg_name_lengths): + if arg_lengths_index == len(arg_name_lengths): + all_arg_name_lengths_index += 1 + arg_name_lengths = all_arg_name_lengths[all_arg_name_lengths_index] + max_name_length = max(arg_name_lengths or [0]) + 2 + arg_lengths_index = 0 + + if arg_lengths_index < len(arg_name_lengths): + + assert arg_name_lengths[arg_lengths_index] < max_name_length + + padded_spaces = ' ' * ( + max_name_length - arg_name_lengths[arg_lengths_index] - 1) + arg_lengths_index += 1 + + assign_content = '{}{}'.format(padded_spaces, token.value.strip()) + existing_whitespace_prefix = \ + token.formatted_whitespace_prefix.lstrip('\n') + + # in case the existing spaces are larger than padded spaces + if (len(padded_spaces)==1 or len(padded_spaces)>1 and + len(existing_whitespace_prefix)>len(padded_spaces)): + token.whitespace_prefix = '' + elif assign_content.startswith(existing_whitespace_prefix): + assign_content = assign_content[len(existing_whitespace_prefix):] + + token.value = assign_content + + final_lines_index += 1 + process_content = True + break + + if not process_content: + final_lines_index += 1 + + +def _AlignDictColon(final_lines): + """Align colons in a dict to the same column""" + """NOTE One (nested) dict/list is one logical line!""" + final_lines_index = 0 + while final_lines_index < len(final_lines): + line = final_lines[final_lines_index] + if line.disable: + final_lines_index += 1 + continue + + assert line.tokens + process_content = False + + for tok in line.tokens: + # make sure each dict entry on separate lines and + # the dict has more than one entry + if (tok.is_dict_key and tok.formatted_whitespace_prefix.startswith('\n') and + not tok.is_comment): + + this_line = line + + line_tokens = this_line.tokens + for open_index in range(len(line_tokens)): + line_tok = line_tokens[open_index] + + # check each time if the detected dict is the dict we aim for + if line_tok.value == '{' and line_tok.next_token.formatted_whitespace_prefix.startswith('\n'): + index = open_index + # skip the comments in the beginning + index += 1 + line_tok = line_tokens[index] + while not line_tok.is_dict_key and index < len(line_tokens)-1: + index += 1 + line_tok = line_tokens[index] + # in case empty dict, check if dict key again + if line_tok.is_dict_key and line_tok.formatted_whitespace_prefix.startswith('\n'): + closing = False # the closing bracket in dict '}'. + keys_content = '' + all_dict_keys_lengths = [] + dict_keys_lengths = [] + + # record the column number of the first key + first_key_column = len(line_tok.formatted_whitespace_prefix.lstrip('\n')) + key_column = first_key_column + + # while not closing: + while not closing: + prefix = line_tok.formatted_whitespace_prefix + newline = prefix.startswith('\n') + if newline: + # if comments inbetween, save, reset and continue to caluclate new alignment + if (style.Get('NEW_ALIGNMENT_AFTER_COMMENTLINE') + and dict_keys_lengths and line_tok.is_comment): + all_dict_keys_lengths.append(dict_keys_lengths) + dict_keys_lengths =[] + index += 1 + line_tok = line_tokens[index] + continue + if line_tok.is_dict_key_start: + keys_content = '' + prefix = prefix.lstrip('\n') + key_column = len(prefix) + # if the dict key is so long that it has multi-lines + # only caculate the last line that has the colon + elif line_tok.is_dict_key: + keys_content = '' + prefix = prefix.lstrip('\n') + elif line_tok.is_dict_key_start: + key_column = line_tok.column + + if line_tok.is_dict_colon and key_column == first_key_column: + dict_keys_lengths.append(len(keys_content)) + elif line_tok.is_dict_key and key_column == first_key_column: + keys_content += '{}{}'.format(prefix, line_tok.value) + + index += 1 + if index < len(line_tokens): + line_tok = line_tokens[index] + # when the matching closing bracket is never found + # due to edge cases where the closing bracket + # is not indented or dedented, e.g. ']}', with another bracket before + else: + all_dict_keys_lengths.append(dict_keys_lengths) + break + + # if there is new objects(list/tuple/dict) with its entries on newlines, + # or a function call with any of its arguments on newlines, + # save, reset and continue to calulate new alignment + if (line_tok.value in ['(', '[', '{'] and not line_tok.is_pseudo and line_tok.next_token + and line_tok.next_token.formatted_whitespace_prefix.startswith('\n')): + if dict_keys_lengths: + all_dict_keys_lengths.append(dict_keys_lengths) + dict_keys_lengths = [] + index += 1 + line_tok = line_tokens[index] + continue + # the matching closing bracket is either same indented or dedented + # accordingly to previous level's indentation + # the first found, immediately break the while loop + if line_tok.value == '}': + if line_tok.formatted_whitespace_prefix.startswith('\n'): + close_column = len(line_tok.formatted_whitespace_prefix.lstrip('\n')) + else: close_column = line_tok.column + if close_column < first_key_column: + if dict_keys_lengths: + all_dict_keys_lengths.append(dict_keys_lengths) + closing = True + + # update the alignment once one dict is processed + if all_dict_keys_lengths: + max_keys_length = 0 + all_dict_keys_lengths_index = 0 + dict_keys_lengths = all_dict_keys_lengths[all_dict_keys_lengths_index] + max_keys_length = max(dict_keys_lengths or [0]) + 2 + keys_lengths_index = 0 + for token in line_tokens[open_index+1:index]: + if token.is_dict_colon: + # check if the key has multiple tokens and + # get the first key token in this key + key_token = token.previous_token + while key_token.is_dict_key and not key_token.is_dict_key_start: + key_token = key_token.previous_token + key_column = len(key_token.formatted_whitespace_prefix.lstrip('\n')) + + if key_column == first_key_column: + + if keys_lengths_index == len(dict_keys_lengths): + all_dict_keys_lengths_index += 1 + dict_keys_lengths = all_dict_keys_lengths[all_dict_keys_lengths_index] + max_keys_length = max(dict_keys_lengths or [0]) + 2 + keys_lengths_index = 0 + + if keys_lengths_index < len(dict_keys_lengths): + assert dict_keys_lengths[keys_lengths_index] < max_keys_length + + padded_spaces = ' ' * ( + max_keys_length - dict_keys_lengths[keys_lengths_index] - 1) + keys_lengths_index += 1 + #NOTE if the existing whitespaces are larger than padded spaces + existing_whitespace_prefix = \ + token.formatted_whitespace_prefix.lstrip('\n') + colon_content = '{}{}'.format(padded_spaces, token.value.strip()) + + # in case the existing spaces are larger than the paddes spaces + if (len(padded_spaces) == 1 or len(padded_spaces) > 1 + and len(existing_whitespace_prefix) >= len(padded_spaces)): + # remove the existing spaces + token.whitespace_prefix = '' + elif colon_content.startswith(existing_whitespace_prefix): + colon_content = colon_content[len(existing_whitespace_prefix):] + + token.value = colon_content + + final_lines_index += 1 + + process_content = True + break + + if not process_content: + final_lines_index += 1 + + + def _FormatFinalLines(final_lines, verify): """Compose the final output from the finalized lines.""" formatted_code = [] diff --git a/yapf/yapflib/style.py b/yapf/yapflib/style.py index 233a64e6b..d9e9e5e9e 100644 --- a/yapf/yapflib/style.py +++ b/yapf/yapflib/style.py @@ -54,6 +54,22 @@ def SetGlobalStyle(style): _STYLE_HELP = dict( ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT=textwrap.dedent("""\ Align closing bracket with visual indentation."""), + ALIGN_ASSIGNMENT=textwrap.dedent("""\ + Align assignment or augmented assignment operators. + If there is a blank line or newline comment or objects with newline entries in between, + it will start new block alignment."""), + ALIGN_ARGUMENT_ASSIGNMENT=textwrap.dedent("""\ + Align assignment operators in the argument list if they are all split on newlines. + Arguments without assignment are ignored. + Arguments without assignment in between will initiate new block alignment calulation. + Newline comments or objects with newline entries will also start new block alignment."""), + ALIGN_DICT_COLON=textwrap.dedent("""\ + Align the colons in the dictionary + if all entries in dictionay are split on newlines. + or 'EACH_DICT_ENTRY_ON_SEPERATE_LINE' is set True. + """), + NEW_ALIGNMENT_AFTER_COMMENTLINE=textwrap.dedent("""\ + Start new assignment or colon alignment when there is a newline comment in between."""), ALLOW_MULTILINE_LAMBDAS=textwrap.dedent("""\ Allow lambdas to be formatted on more than one line."""), ALLOW_MULTILINE_DICTIONARY_KEYS=textwrap.dedent("""\ @@ -419,6 +435,10 @@ def CreatePEP8Style(): """Create the PEP8 formatting style.""" return dict( ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT=True, + ALIGN_ASSIGNMENT=False, + ALIGN_ARGUMENT_ASSIGNMENT=False, + ALIGN_DICT_COLON=False, + NEW_ALIGNMENT_AFTER_COMMENTLINE=False, ALLOW_MULTILINE_LAMBDAS=False, ALLOW_MULTILINE_DICTIONARY_KEYS=False, ALLOW_SPLIT_BEFORE_DEFAULT_OR_NAMED_ASSIGNS=True, @@ -607,6 +627,10 @@ def _IntOrIntListConverter(s): # Note: this dict has to map all the supported style options. _STYLE_OPTION_VALUE_CONVERTER = dict( ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT=_BoolConverter, + ALIGN_ASSIGNMENT=_BoolConverter, + ALIGN_DICT_COLON=_BoolConverter, + NEW_ALIGNMENT_AFTER_COMMENTLINE=_BoolConverter, + ALIGN_ARGUMENT_ASSIGNMENT=_BoolConverter, ALLOW_MULTILINE_LAMBDAS=_BoolConverter, ALLOW_MULTILINE_DICTIONARY_KEYS=_BoolConverter, ALLOW_SPLIT_BEFORE_DEFAULT_OR_NAMED_ASSIGNS=_BoolConverter, diff --git a/yapftests/format_token_test.py b/yapftests/format_token_test.py index 6ea24af63..3bb1ce9f5 100644 --- a/yapftests/format_token_test.py +++ b/yapftests/format_token_test.py @@ -15,10 +15,11 @@ import unittest -from lib2to3 import pytree +from lib2to3 import pytree, pygram from lib2to3.pgen2 import token from yapf.yapflib import format_token +from yapf.pytree import subtype_assigner class TabbedContinuationAlignPaddingTest(unittest.TestCase): @@ -89,6 +90,37 @@ def testIsMultilineString(self): pytree.Leaf(token.STRING, 'r"""hello"""'), 'STRING') self.assertTrue(tok.is_multiline_string) + #------------test argument names------------ + # fun( + # a='hello world', + # # comment, + # b='') + child1 = pytree.Leaf(token.NAME, 'a') + child2 = pytree.Leaf(token.EQUAL, '=') + child3 = pytree.Leaf(token.STRING, "'hello world'") + child4 = pytree.Leaf(token.COMMA, ',') + child5 = pytree.Leaf(token.COMMENT,'# comment') + child6 = pytree.Leaf(token.COMMA, ',') + child7 = pytree.Leaf(token.NAME, 'b') + child8 = pytree.Leaf(token.EQUAL, '=') + child9 = pytree.Leaf(token.STRING, "''") + node_type = pygram.python_grammar.symbol2number['arglist'] + node = pytree.Node(node_type, [child1, child2, child3, child4, child5, + child6, child7, child8,child9]) + subtype_assigner.AssignSubtypes(node) + + def testIsArgName(self, node=node): + tok = format_token.FormatToken(node.children[0],'NAME') + self.assertTrue(tok.is_argname) + + def testIsArgAssign(self, node=node): + tok = format_token.FormatToken(node.children[1], 'EQUAL') + self.assertTrue(tok.is_argassign) + + # test if comment inside is not argname + def testCommentNotIsArgName(self, node=node): + tok = format_token.FormatToken(node.children[4], 'COMMENT') + self.assertFalse(tok.is_argname) if __name__ == '__main__': unittest.main() diff --git a/yapftests/reformatter_basic_test.py b/yapftests/reformatter_basic_test.py index 657d1e246..0eeeefdce 100644 --- a/yapftests/reformatter_basic_test.py +++ b/yapftests/reformatter_basic_test.py @@ -3165,6 +3165,364 @@ def testWalrus(self): llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) self.assertCodeEqual(expected, reformatter.Reformat(llines)) + #------tests for alignment functions-------- + def testAlignAssignBlankLineInbetween(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{align_assignment: true}')) + unformatted_code = textwrap.dedent("""\ + val_first = 1 + val_second += 2 + + val_third = 3 + """) + expected_formatted_code = textwrap.dedent("""\ + val_first = 1 + val_second += 2 + + val_third = 3 + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testAlignAssignCommentLineInbetween(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{align_assignment: true,' + 'new_alignment_after_commentline = true}')) + unformatted_code = textwrap.dedent("""\ + val_first = 1 + val_second += 2 + # comment + val_third = 3 + """) + expected_formatted_code = textwrap.dedent("""\ + val_first = 1 + val_second += 2 + # comment + val_third = 3 + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testAlignAssignDefLineInbetween(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{align_assignment: true}')) + unformatted_code = textwrap.dedent("""\ + val_first = 1 + val_second += 2 + def fun(): + a = 'example' + abc = '' + val_third = 3 + """) + expected_formatted_code = textwrap.dedent("""\ + val_first = 1 + val_second += 2 + + + def fun(): + a = 'example' + abc = '' + + + val_third = 3 + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testAlignAssignObjectWithNewLineInbetween(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{align_assignment: true}')) + unformatted_code = textwrap.dedent("""\ + val_first = 1 + val_second += 2 + object = { + entry1:1, + entry2:2, + entry3:3, + } + val_third = 3 + """) + expected_formatted_code = textwrap.dedent("""\ + val_first = 1 + val_second += 2 + object = { + entry1: 1, + entry2: 2, + entry3: 3, + } + val_third = 3 + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testAlignAssignWithOnlyOneAssignmentLine(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{align_assignment: true}')) + unformatted_code = textwrap.dedent("""\ + val_first = 1 + """) + expected_formatted_code = unformatted_code + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + ########## for Align_ArgAssign()########### + def testAlignArgAssignTypedName(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{align_argument_assignment: true,' + 'split_before_first_argument: true}')) + unformatted_code = textwrap.dedent("""\ +def f1( + self, + *, + app_name:str="", + server=None, + main_app=None, + db: Optional[NemDB]=None, + root: Optional[str]="", + conf: Optional[dict]={1, 2}, + ini_section: str="" +): pass +""") + expected_formatted_code = textwrap.dedent("""\ +def f1( + self, + *, + app_name: str = "", + server =None, + main_app =None, + db: Optional[NemDB] = None, + root: Optional[str] = "", + conf: Optional[dict] = {1, 2}, + ini_section: str = ""): + pass +""") + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + # test both object/nested argument list with newlines and + # argument without assignment in between + def testAlignArgAssignNestedArglistInBetween(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{align_argument_assignment: true}')) + unformatted_code = textwrap.dedent("""\ +arglist = test( + first_argument='', + second_argument=fun( + self, role=None, client_name='', client_id=1, very_long_long_long_long_long='' + ), + third_argument=3, + fourth_argument=4 +) +""") + expected_formatted_code = textwrap.dedent("""\ +arglist = test( + first_argument ='', + second_argument =fun( + self, + role =None, + client_name ='', + client_id =1, + very_long_long_long_long_long =''), + third_argument =3, + fourth_argument =4) +""") + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + # start new alignment after comment line in between + def testAlignArgAssignCommentLineInBetween(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{align_argument_assignment: true,' + 'new_alignment_after_commentline:true}')) + unformatted_code = textwrap.dedent("""\ +arglist = test( + client_id=0, + username_id=1, + # comment + user_name='xxxxxxxxxxxxxxxxxxxxx' +) +""") + expected_formatted_code = textwrap.dedent("""\ +arglist = test( + client_id =0, + username_id =1, + # comment + user_name ='xxxxxxxxxxxxxxxxxxxxx') +""") + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testAlignArgAssignWithOnlyFirstArgOnNewline(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{align_argument_assignment: true}')) + unformatted_code = textwrap.dedent("""\ +arglist = self.get_data_from_excelsheet( + client_id=0, username_id=1, user_name='xxxxxxxxxxxxxxxxxxxx') +""") + expected_formatted_code = unformatted_code + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testAlignArgAssignArgumentsCanFitInOneLine(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{align_argument_assignment: true}')) + unformatted_code = textwrap.dedent("""\ +def function( + first_argument_xxxxxx =(0,), + second_argument =None +) -> None: + pass +""") + expected_formatted_code = textwrap.dedent("""\ +def function(first_argument_xxxxxx=(0,), second_argument=None) -> None: + pass +""") + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + ########for align dictionary colons######### + def testAlignDictColonNestedDictInBetween(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{align_dict_colon: true}')) + unformatted_code = textwrap.dedent("""\ +fields = [{"type": "text","required": True,"html": {"attr": 'style="width: 250px;" maxlength="30"',"page": 0,}, + "list" : [1, 2, 3, 4]}] +""") + expected_formatted_code = textwrap.dedent("""\ +fields = [{ + "type" : "text", + "required" : True, + "html" : { + "attr" : 'style="width: 250px;" maxlength="30"', + "page" : 0, + }, + "list" : [1, 2, 3, 4] +}] +""") + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testAlignDictColonCommentLineInBetween(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{align_dict_colon: true,' + 'new_alignment_after_commentline: true}')) + unformatted_code = textwrap.dedent("""\ +fields = [{ + "type": "text", + "required": True, + # comment + "list": [1, 2, 3, 4]}] +""") + expected_formatted_code = textwrap.dedent("""\ +fields = [{ + "type" : "text", + "required" : True, + # comment + "list" : [1, 2, 3, 4] +}] +""") + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testAlignDictColonLargerExistingSpacesBefore(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{align_dict_colon: true}')) + unformatted_code = textwrap.dedent("""\ +fields = [{ + "type" : "text", + "required" : True, + "list" : [1, 2, 3, 4], +}] +""") + expected_formatted_code = textwrap.dedent("""\ +fields = [{ + "type" : "text", + "required" : True, + "list" : [1, 2, 3, 4], +}] +""") + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + def testAlignDictColonCommentAfterOpenBracket(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{align_dict_colon: true}')) + unformatted_code = textwrap.dedent("""\ +fields = [{ + # comment + "type": "text", + "required": True, + "list": [1, 2, 3, 4]}] +""") + expected_formatted_code = textwrap.dedent("""\ +fields = [{ + # comment + "type" : "text", + "required" : True, + "list" : [1, 2, 3, 4] +}] +""") + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(style.CreateYapfStyle()) + + + if __name__ == '__main__': unittest.main() diff --git a/yapftests/subtype_assigner_test.py b/yapftests/subtype_assigner_test.py index 8616169c9..97f9cd3ac 100644 --- a/yapftests/subtype_assigner_test.py +++ b/yapftests/subtype_assigner_test.py @@ -129,6 +129,87 @@ def testFuncCallWithDefaultAssign(self): ], ]) + #----test comment subtype inside the argument list---- + def testCommentSubtypesInsideArglist(self): + code = textwrap.dedent("""\ + foo( + # comment + x, + a='hello world') + """) + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckFormatTokenSubtypes(llines, [ + [ + ('foo', {subtypes.NONE}), + ('(', {subtypes.NONE}), + ('# comment', {subtypes.NONE, + subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST}), + ('x', { + subtypes.NONE, + subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST}), + (',', {subtypes.NONE}), + ('a', { + subtypes.NONE, + subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST}), + ('=', {subtypes.DEFAULT_OR_NAMED_ASSIGN}), + ("'hello world'", {subtypes.NONE}), + (')', {subtypes.NONE}), + ], + ]) + + # ----test typed arguments subtypes------ + def testTypedArgumentsInsideArglist(self): + code = textwrap.dedent("""\ +def foo( + self, + preprocess: Callable[[str], str] = identity + ): pass +""") + llines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckFormatTokenSubtypes(llines, [ + [ + ('def', {subtypes.NONE}), + ('foo', {subtypes.FUNC_DEF}), + ('(', {subtypes.NONE}), + ('self', {subtypes.NONE, + subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST, + subtypes.PARAMETER_START, + subtypes.PARAMETER_STOP}), + (',', {subtypes.NONE}), + ('preprocess', { + subtypes.NONE, + subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST, + subtypes.PARAMETER_START, + subtypes.TYPED_NAME_ARG_LIST}), + (':', { + subtypes.TYPED_NAME, + subtypes.TYPED_NAME_ARG_LIST}), + ('Callable', {subtypes.TYPED_NAME_ARG_LIST}), + ('[', { + subtypes.SUBSCRIPT_BRACKET, + subtypes.TYPED_NAME_ARG_LIST}), + ('[', {subtypes.TYPED_NAME_ARG_LIST}), + ('str', {subtypes.TYPED_NAME_ARG_LIST}), + (']', {subtypes.TYPED_NAME_ARG_LIST}), + (',', {subtypes.TYPED_NAME_ARG_LIST}), + ('str', {subtypes.TYPED_NAME_ARG_LIST}), + (']', { + subtypes.SUBSCRIPT_BRACKET, + subtypes.TYPED_NAME_ARG_LIST}), + ('=', { + subtypes.DEFAULT_OR_NAMED_ASSIGN, + subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST, + subtypes.TYPED_NAME}), + ('identity', { + subtypes.NONE, + subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST, + subtypes.PARAMETER_STOP}), + (')', {subtypes.NONE}), + (':', {subtypes.NONE})], + [('pass', {subtypes.NONE}), + ], + ]) + def testSetComprehension(self): code = textwrap.dedent("""\ def foo(strs):