diff --git a/process_pr.py b/process_pr.py index 88a5fe046e6a..84cba66af5b4 100644 --- a/process_pr.py +++ b/process_pr.py @@ -8,7 +8,11 @@ CMSDIST_REPOS, ) from categories import CMSSW_CATEGORIES -from releases import RELEASE_BRANCH_MILESTONE, RELEASE_BRANCH_PRODUCTION, CMSSW_DEVEL_BRANCH +from releases import ( + RELEASE_BRANCH_MILESTONE, + RELEASE_BRANCH_PRODUCTION, + CMSSW_DEVEL_BRANCH, +) from cms_static import ( VALID_CMSDIST_BRANCHES, NEW_ISSUE_PREFIX, @@ -30,7 +34,12 @@ from os.path import join, exists, dirname from os import environ from github_utils import edit_pr, api_rate_limits -from github_utils import set_comment_emoji, get_comment_emojis, delete_comment_emoji, set_gh_user +from github_utils import ( + set_comment_emoji, + get_comment_emojis, + delete_comment_emoji, + set_gh_user, +) from socket import setdefaulttimeout from _py2with3compatibility import run_cmd from json import dumps, load, loads @@ -100,11 +109,17 @@ def format(s, **kwds): CMSSW_PACKAGE_PATTERN = "[A-Z][a-zA-Z0-9]+(/[a-zA-Z0-9]+|)" ARCH_PATTERN = "[a-z0-9]+_[a-z0-9]+_[a-z0-9]+" CMSSW_RELEASE_QUEUE_PATTERN = format( - "(%(cmssw)s|%(arch)s|%(cmssw)s/%(arch)s)", cmssw=CMSSW_QUEUE_PATTERN, arch=ARCH_PATTERN + "(%(cmssw)s|%(arch)s|%(cmssw)s/%(arch)s)", + cmssw=CMSSW_QUEUE_PATTERN, + arch=ARCH_PATTERN, ) RELVAL_OPTS = "[-][a-zA-Z0-9_.,\s/'-]+" -CLOSE_REQUEST = re.compile("^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)close\s*$", re.I) -REOPEN_REQUEST = re.compile("^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)(re|)open\s*$", re.I) +CLOSE_REQUEST = re.compile( + "^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)close\s*$", re.I +) +REOPEN_REQUEST = re.compile( + "^\s*((@|)cmsbuild\s*[,]*\s+|)(please\s*[,]*\s+|)(re|)open\s*$", re.I +) CMS_PR_PATTERN = format( "(#[1-9][0-9]*|(%(cmsorgs)s)/+[a-zA-Z0-9_-]+#[1-9][0-9]*|https://+github.com/+(%(cmsorgs)s)/+[a-zA-Z0-9_-]+/+pull/+[1-9][0-9]*)", cmsorgs="|".join(EXTERNAL_REPOS), @@ -139,7 +154,10 @@ def format(s, **kwds): MULTILINE_COMMENTS_MAP = { "(workflow|relval)(s|)(" + EXTRA_RELVALS_TESTS_OPTS - + "|)": [format("%(workflow)s(\s*,\s*%(workflow)s|)*", workflow=WF_PATTERN), "MATRIX_EXTRAS"], + + "|)": [ + format("%(workflow)s(\s*,\s*%(workflow)s|)*", workflow=WF_PATTERN), + "MATRIX_EXTRAS", + ], "(workflow|relval)(s|)_profiling": [ format("%(workflow)s(\s*,\s*%(workflow)s|)*", workflow=WF_PATTERN), "PROFILING_WORKFLOWS", @@ -152,10 +170,16 @@ def format(s, **kwds): "disable_poison": ["true|false", "DISABLE_POISON"], "use_ib_tag": ["true|false", "USE_IB_TAG"], "baseline": ["self|default", "USE_BASELINE"], - "skip_test(s|)": [format("(%(tests)s)(\s*,\s*(%(tests)s))*", tests=SKIP_TESTS), "SKIP_TESTS"], + "skip_test(s|)": [ + format("(%(tests)s)(\s*,\s*(%(tests)s))*", tests=SKIP_TESTS), + "SKIP_TESTS", + ], "dry_run": ["true|false", "DRY_RUN"], "jenkins_(slave|node)": [JENKINS_NODES, "RUN_ON_SLAVE"], - "(arch(itecture(s|))|release|release/arch)": [CMSSW_RELEASE_QUEUE_PATTERN, "RELEASE_FORMAT"], + "(arch(itecture(s|))|release|release/arch)": [ + CMSSW_RELEASE_QUEUE_PATTERN, + "RELEASE_FORMAT", + ], ENABLE_TEST_PTRN: [ format("(%(tests)s)(\s*,\s*(%(tests)s))*", tests=EXTRA_TESTS), "ENABLE_BOT_TESTS", @@ -271,7 +295,13 @@ def read_repo_file(repo_config, repo_file, default=None): def create_properties_file_tests( - repository, pr_number, parameters, dryRun, abort=False, req_type="tests", repo_config=None + repository, + pr_number, + parameters, + dryRun, + abort=False, + req_type="tests", + repo_config=None, ): if abort: req_type = "abort" @@ -335,16 +365,24 @@ def find_last_comment(issue, user, match): if (user != comment.user.login) or (not comment.body): continue if not re.match( - match, comment.body.encode("ascii", "ignore").decode().strip("\n\t\r "), re.MULTILINE + match, + comment.body.encode("ascii", "ignore").decode().strip("\n\t\r "), + re.MULTILINE, ): continue last_comment = comment - print("Matched comment from ", comment.user.login + " with comment id ", comment.id) + print( + "Matched comment from ", + comment.user.login + " with comment id ", + comment.id, + ) return last_comment def modify_comment(comment, match, replace, dryRun): - comment_msg = comment.body.encode("ascii", "ignore").decode() if comment.body else "" + comment_msg = ( + comment.body.encode("ascii", "ignore").decode() if comment.body else "" + ) if match: new_comment_msg = re.sub(match, replace, comment_msg) else: @@ -434,7 +472,9 @@ def check_extra_labels(first_line, extra_labels): def check_type_labels(first_line, extra_labels): ex_labels = {} rem_labels = {} - for type_cmd in [x.strip() for x in first_line.split(" ", 1)[-1].split(",") if x.strip()]: + for type_cmd in [ + x.strip() for x in first_line.split(" ", 1)[-1].split(",") if x.strip() + ]: valid_lab = False rem_lab = type_cmd[0] == "-" if type_cmd[0] in ["-", "+"]: @@ -663,7 +703,9 @@ def cmssw_file2Package(repo_config, filename): def get_jenkins_job(issue): test_line = "" - for line in [l.strip() for l in issue.body.encode("ascii", "ignore").decode().split("\n")]: + for line in [ + l.strip() for l in issue.body.encode("ascii", "ignore").decode().split("\n") + ]: if line.startswith("Build logs are available at:"): test_line = line if test_line: @@ -712,7 +754,14 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F pass if not cmsbuild_user: cmsbuild_user = repo_config.CMSBUILD_USER - print("Working on ", repo.full_name, " for PR/Issue ", prId, "with admin user", cmsbuild_user) + print( + "Working on ", + repo.full_name, + " for PR/Issue ", + prId, + "with admin user", + cmsbuild_user, + ) print("Notify User: ", gh_user_char) set_gh_user(cmsbuild_user) cmssw_repo = repo_name == GH_CMSSW_REPO @@ -723,7 +772,6 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F create_test_property = False repo_cache = {repository: repo} packages = set([]) - chg_files = [] package_categories = {} extra_labels = {"mtype": []} add_external_category = False @@ -736,16 +784,21 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F watchers = [] # Process Pull Request pkg_categories = set([]) - REGEX_TYPE_CMDS = "^type\s+(([-+]|)[a-z][a-z0-9-]+)(\s*,\s*([-+]|)[a-z][a-z0-9-]+)*$" - REGEX_EX_CMDS = "^urgent$|^backport\s+(of\s+|)(#|http(s|):/+github\.com/+%s/+pull/+)\d+$" % ( - repo.full_name + REGEX_TYPE_CMDS = ( + "^type\s+(([-+]|)[a-z][a-z0-9-]+)(\s*,\s*([-+]|)[a-z][a-z0-9-]+)*$" + ) + REGEX_EX_CMDS = ( + "^urgent$|^backport\s+(of\s+|)(#|http(s|):/+github\.com/+%s/+pull/+)\d+$" + % (repo.full_name) ) known_ignore_tests = "%s" % MULTILINE_COMMENTS_MAP["ignore_test(s|)"][0] REGEX_EX_IGNORE_CHKS = "^ignore\s+((%s)(\s*,\s*(%s))*|none)$" % ( known_ignore_tests, known_ignore_tests, ) - REGEX_EX_ENABLE_TESTS = "^enable\s+(%s)$" % MULTILINE_COMMENTS_MAP[ENABLE_TEST_PTRN][0] + REGEX_EX_ENABLE_TESTS = ( + "^enable\s+(%s)$" % MULTILINE_COMMENTS_MAP[ENABLE_TEST_PTRN][0] + ) L2_DATA = init_l2_data(cms_repo) last_commit_date = None last_commit_obj = None @@ -769,7 +822,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F events = {} commit_cache = {} all_commits = [] - override_too_many_commits = False + ok_too_many_commits = False warned_too_many_commits = False l2s = ", ".join([gh_user_char + name for name in CMSSW_ISSUES_TRACKERS]) @@ -796,114 +849,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # A pull request is by default closed if the branch is a closed one. if is_closed_branch(pr.base.ref): mustClose = True - # Process the changes for the given pull request so that we can determine the - # signatures it requires. - if cmssw_repo or not external_repo: - if cmssw_repo: - if pr.base.ref == "master": - signing_categories.add("code-checks") - updateMilestone(repo, issue, pr, dryRun) - chg_files = get_changed_files(repo, pr) - packages = sorted( - [x for x in set([cmssw_file2Package(repo_config, f) for f in chg_files])] - ) - for pkg_file in chg_files: - for ex_lab, pkgs_regexp in list(CMSSW_LABELS.items()): - for regex in pkgs_regexp: - if regex.match(pkg_file): - extra_labels["mtype"].append(ex_lab) - print( - "Non-Blocking label:%s:%s:%s" % (ex_lab, regex.pattern, pkg_file) - ) - break - if not extra_labels["mtype"]: - del extra_labels["mtype"] - print("Extra non-blocking labels:", extra_labels) - print("First Package: ", packages[0]) - create_test_property = True - else: - add_external_category = True - packages = set(["externals/" + repository]) - ex_pkg = external_to_package(repository) - if ex_pkg: - packages.add(ex_pkg) - if (repo_org != GH_CMSSW_ORGANIZATION) or (repo_name in VALID_CMS_SW_REPOS_FOR_TESTS): - create_test_property = True - if (repo_name == GH_CMSDIST_REPO) and ( - not re.match(VALID_CMSDIST_BRANCHES, pr.base.ref) - ): - print("Skipping PR as it does not belong to valid CMSDIST branch") - return - - print("Following packages affected:") - print("\n".join(packages)) - for package in packages: - package_categories[package] = set([]) - for category in get_package_categories(package): - package_categories[package].add(category) - pkg_categories.add(category) - signing_categories.update(pkg_categories) - - # For PR, we always require tests. - signing_categories.add("tests") - if add_external_category: - signing_categories.add("externals") - if cms_repo: - print("This pull request requires ORP approval") - signing_categories.add("orp") - print("Following categories affected:") - print("\n".join(signing_categories)) - - if cmssw_repo: - # If there is a new package, add also a dummy "new" category. - all_packages = [ - package - for category_packages in list(CMSSW_CATEGORIES.values()) - for package in category_packages - ] - has_category = all([package in all_packages for package in packages]) - if not has_category: - new_package_message = "\nThe following packages do not have a category, yet:\n\n" - new_package_message += ( - "\n".join([package for package in packages if not package in all_packages]) - + "\n" - ) - new_package_message += "Please create a PR for https://github.com/cms-sw/cms-bot/blob/master/categories_map.py to assign category\n" - print(new_package_message) - signing_categories.add("new-package") - - # Add watchers.yaml information to the WATCHERS dict. - WATCHERS = read_repo_file(repo_config, "watchers.yaml", {}) - # Given the files modified by the PR, check if there are additional developers watching one or more. - author = pr.user.login - watchers = set( - [ - user - for chg_file in chg_files - for user, watched_regexp in list(WATCHERS.items()) - for regexp in watched_regexp - if re.match("^" + regexp + ".*", chg_file) and user != author - ] - ) - # Handle category watchers - catWatchers = read_repo_file(repo_config, "category-watchers.yaml", {}) - non_block_cats = [] if not "mtype" in extra_labels else extra_labels["mtype"] - for user, cats in list(catWatchers.items()): - for cat in cats: - if (cat in signing_categories) or (cat in non_block_cats): - print("Added ", user, " to watch due to cat", cat) - watchers.add(user) - - # Handle watchers - watchingGroups = read_repo_file(repo_config, "groups.yaml", {}) - for watcher in [x for x in watchers]: - if not watcher in watchingGroups: - continue - watchers.remove(watcher) - watchers.update(set(watchingGroups[watcher])) - watchers = set([gh_user_char + u for u in watchers]) - print("Watchers " + ", ".join(watchers)) all_commits = get_pr_commits_reversed(pr) if all_commits: @@ -921,7 +867,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F cms_status_prefix = "cms/%s" % prId bot_status = get_status(bot_status_name, commit_statuses) code_checks_status = [ - s for s in commit_statuses if s.context == "%s/code-checks" % cms_status_prefix + s + for s in commit_statuses + if s.context == "%s/code-checks" % cms_status_prefix ] print("PR Statuses:", commit_statuses) print(len(commit_statuses)) @@ -932,7 +880,10 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F " at ", last_commit_date, ) - print("Latest commit message: ", last_commit.message.encode("ascii", "ignore").decode()) + print( + "Latest commit message: ", + last_commit.message.encode("ascii", "ignore").decode(), + ) print("Latest commit sha: ", last_commit.sha) print("PR update time", pr.updated_at) print("Time UTC:", datetime.utcnow()) @@ -944,7 +895,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F except: pass if (not dryRun) and add_labels: - labels = [x.name.encode("ascii", "ignore").decode() for x in issue.labels] + labels = [ + x.name.encode("ascii", "ignore").decode() for x in issue.labels + ] if not "future-commit" in labels: labels.append("future-commit") issue.edit(labels=labels) @@ -967,7 +920,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F except: pass if repository == CMSSW_REPO_NAME and re.match(CREATE_REPO, issue.title): - with open("query-new-data-repo-issues-" + str(issue.number) + ".properties", "w") as f: + with open( + "query-new-data-repo-issues-" + str(issue.number) + ".properties", "w" + ) as f: f.write("ISSUE_NUMBER=" + str(issue.number) + "\n") # Process the issue comments @@ -1020,12 +975,14 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F commenter_categories = get_commenter_categories( commenter, int(comment.created_at.strftime("%s")) ) - valid_commenter = (commenter in TRIGGER_PR_TESTS + releaseManagers + [repo_org]) or ( - len(commenter_categories) > 0 - ) + valid_commenter = ( + commenter in TRIGGER_PR_TESTS + releaseManagers + [repo_org] + ) or (len(commenter_categories) > 0) if (not valid_commenter) and (requestor != commenter): continue - comment_msg = comment.body.encode("ascii", "ignore").decode() if comment.body else "" + comment_msg = ( + comment.body.encode("ascii", "ignore").decode() if comment.body else "" + ) # The first line is an invariant. comment_lines = [l.strip() for l in comment_msg.split("\n") if l.strip()] first_line = comment_lines[0:1] @@ -1049,15 +1006,21 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ): technical_comment = comment - if (commenter == cmsbuild_user) and "This PR contains too many commits" in first_line: + if ( + commenter == cmsbuild_user + ) and "This PR contains too many commits" in first_line: warned_too_many_commits = True - if commenter in l2s and re.match("^\s*" + REGEX_IGNORE_COMMIT_COUNT + "\s*$", first_line): - override_too_many_commits = True + if commenter in l2s and re.match( + "^\s*" + REGEX_IGNORE_COMMIT_COUNT + "\s*$", first_line + ): + ok_too_many_commits = True assign_type, new_cats = get_assign_categories(first_line) if new_cats: - if (assign_type == "new categories assigned:") and (commenter == cmsbuild_user): + if (assign_type == "new categories assigned:") and ( + commenter == cmsbuild_user + ): for ex_cat in new_cats: if ex_cat in assign_cats: assign_cats[ex_cat] = 1 @@ -1079,7 +1042,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # Some of the special users can say "hold" prevent automatic merging of # fully signed PRs. if re.match("^hold$", first_line, re.I): - if commenter_categories or (commenter in releaseManagers + PR_HOLD_MANAGERS): + if commenter_categories or ( + commenter in releaseManagers + PR_HOLD_MANAGERS + ): hold[commenter] = 1 continue if re.match(REGEX_EX_CMDS, first_line, re.I): @@ -1100,7 +1065,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F continue if re.match(REGEX_EX_ENABLE_TESTS, first_line, re.I): if valid_commenter: - enable_tests, ignore = check_enable_bot_tests(first_line.split(" ", 1)[-1]) + enable_tests, ignore = check_enable_bot_tests( + first_line.split(" ", 1)[-1] + ) if not dryRun: set_comment_emoji(comment.id, repository, emoji="+1") continue @@ -1115,11 +1082,15 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if re.match("^unhold$", first_line, re.I): if "orp" in commenter_categories: hold = {} - elif commenter_categories or (commenter in releaseManagers + PR_HOLD_MANAGERS): + elif commenter_categories or ( + commenter in releaseManagers + PR_HOLD_MANAGERS + ): if commenter in hold: del hold[commenter] continue - if (commenter == cmsbuild_user) and (re.match("^" + HOLD_MSG + ".+", first_line)): + if (commenter == cmsbuild_user) and ( + re.match("^" + HOLD_MSG + ".+", first_line) + ): for u in first_line.split(HOLD_MSG, 2)[1].split(","): u = u.strip().lstrip("@") if u in hold: @@ -1138,14 +1109,18 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F (not issue.pull_request) and (commenter in CMSSW_ISSUES_TRACKERS) ): mustClose = False - if (issue.state == "closed") and (comment.created_at >= issue.closed_at): + if (issue.state == "closed") and ( + comment.created_at >= issue.closed_at + ): reOpen = True print("==>Reopen request received from %s" % commenter) continue if valid_commenter: - valid_multiline_comment, test_params, test_params_m = multiline_check_function( - first_line, comment_lines, repository - ) + ( + valid_multiline_comment, + test_params, + test_params_m, + ) = multiline_check_function(first_line, comment_lines, repository) if test_params_m: test_params_msg = str(comment.id) + ":" + test_params_m test_params_comment = comment @@ -1154,7 +1129,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F global_test_params = dict(test_params) if "ENABLE_BOT_TESTS" in global_test_params: enable_tests = global_test_params["ENABLE_BOT_TESTS"] - test_params_msg = str(comment.id) + ":" + dumps(global_test_params, sort_keys=True) + test_params_msg = ( + str(comment.id) + ":" + dumps(global_test_params, sort_keys=True) + ) continue if cmssw_repo: @@ -1209,7 +1186,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F signatures[first_line] = "pending" if first_line not in pre_checks + extra_pre_checks: extra_pre_checks.append(first_line) - if code_checks_status and (code_checks_status[0].updated_at >= comment.created_at): + if code_checks_status and ( + code_checks_status[0].updated_at >= comment.created_at + ): continue if first_line in pre_checks: if pre_checks_state["code-checks"] in ["pending", ""]: @@ -1245,7 +1224,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F IGNORING_TESTS_MSG, first_line ): signatures["tests"] = "pending" - elif re.match("Pull request ([^ #]+|)[#][0-9]+ was updated[.].*", first_line): + elif re.match( + "Pull request ([^ #]+|)[#][0-9]+ was updated[.].*", first_line + ): pull_request_updated = False elif re.match(TRIGERING_TESTS_MSG, first_line) or re.match( TRIGERING_TESTS_MSG1, first_line @@ -1276,7 +1257,14 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if "+1" in first_line: signatures["tests"] = "approved" comp_warnings = ( - len([1 for l in comment_lines if "Compilation Warnings: Yes" in l]) > 0 + len( + [ + 1 + for l in comment_lines + if "Compilation Warnings: Yes" in l + ] + ) + > 0 ) pre_checks_url["tests"] = comment.html_url elif "-1" in first_line: @@ -1291,9 +1279,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if issue.pull_request or push_test_issue: # Check if the release manager asked for merging this. - if ((commenter in releaseManagers) or ("orp" in commenter_categories)) and re.match( - "^\s*(merge)\s*$", first_line, re.I - ): + if ( + (commenter in releaseManagers) or ("orp" in commenter_categories) + ) and re.match("^\s*(merge)\s*$", first_line, re.I): mustMerge = True mustClose = False if ("orp" in commenter_categories) and ("orp" in signatures): @@ -1302,7 +1290,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # Check if the someone asked to trigger the tests if valid_commenter: - ok, v2, v3, v4 = check_test_cmd(first_line, repository, global_test_params) + ok, v2, v3, v4 = check_test_cmd( + first_line, repository, global_test_params + ) if ok: test_comment = comment abort_test = None @@ -1324,7 +1314,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F print("Comment message:", first_line) signatures["tests"] = "pending" continue - elif REGEX_TEST_ABORT.match(first_line) and (signatures["tests"] == "pending"): + elif REGEX_TEST_ABORT.match(first_line) and ( + signatures["tests"] == "pending" + ): abort_test = comment test_comment = None signatures["tests"] = "pending" @@ -1342,9 +1334,10 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F set_comment_emoji(comment.id, repository) # end of parsing comments section + if ( (not warned_too_many_commits) - and not override_too_many_commits + and not ok_too_many_commits and len(all_commits) >= MAX_INITIAL_COMMITS_IN_PR ): issue.create_comment( @@ -1353,10 +1346,6 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F "\n{l2s}, you can override this check with `+commit-count`." ) - if len(all_commits) < MAX_INITIAL_COMMITS_IN_PR or override_too_many_commits: - for commit in all_commits: - events[commit.commit.committer.date] = get_changed_files_in_commit(repo, commit) - # Get the commit cache from `already_seen` commit or technical commit print("Recalculating signatures") cache_comment = None @@ -1378,18 +1367,23 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F "", old_body ) - for commit in all_commits: - if commit.sha not in commit_cache: - commit_cache[commit.sha] = { - "time": int(commit.commit.committer.date.timestamp()), - "files": get_changed_files_in_commit(repo, commit), + chg_files = set() + + if len(all_commits) < MAX_INITIAL_COMMITS_IN_PR or ok_too_many_commits: + for commit in all_commits: + if commit.sha not in commit_cache: + commit_cache[commit.sha] = { + "time": int(commit.commit.committer.date.timestamp()), + "files": get_changed_files_in_commit(repo, commit), + } + + cache_entry = commit_cache[commit.sha] + events[datetime.fromtimestamp(cache_entry["time"])] = { + "type": "commit", + "value": cache_entry["files"], } - cache_entry = commit_cache[commit.sha] - events[datetime.fromtimestamp(cache_entry["time"])] = { - "type": "commit", - "value": cache_entry["files"], - } + chg_files.update(cache_entry["files"]) if not dryRun: if cache_comment: @@ -1407,13 +1401,133 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F events = dict(sorted(events.items())) + if issue.pull_request: + pr = repo.get_pull(prId) + + # Process the changes for the given pull request so that we can determine the + # signatures it requires. + if cmssw_repo or not external_repo: + if cmssw_repo: + if pr.base.ref == "master": + signing_categories.add("code-checks") + updateMilestone(repo, issue, pr, dryRun) + + packages = sorted( + list(set(cmssw_file2Package(repo_config, f) for f in chg_files)) + ) + for pkg_file in chg_files: + for ex_lab, pkgs_regexp in list(CMSSW_LABELS.items()): + for regex in pkgs_regexp: + if regex.match(pkg_file): + extra_labels["mtype"].append(ex_lab) + print( + "Non-Blocking label:%s:%s:%s" + % (ex_lab, regex.pattern, pkg_file) + ) + break + if not extra_labels["mtype"]: + del extra_labels["mtype"] + print("Extra non-blocking labels:", extra_labels) + print("First Package: ", packages[0]) + create_test_property = True + else: + add_external_category = True + packages = {"externals/" + repository} + ex_pkg = external_to_package(repository) + if ex_pkg: + packages.add(ex_pkg) + if (repo_org != GH_CMSSW_ORGANIZATION) or ( + repo_name in VALID_CMS_SW_REPOS_FOR_TESTS + ): + create_test_property = True + if (repo_name == GH_CMSDIST_REPO) and ( + not re.match(VALID_CMSDIST_BRANCHES, pr.base.ref) + ): + print("Skipping PR as it does not belong to valid CMSDIST branch") + return + + print("Following packages affected:") + print("\n".join(packages)) + for package in packages: + package_categories[package] = set([]) + for category in get_package_categories(package): + package_categories[package].add(category) + pkg_categories.add(category) + signing_categories.update(pkg_categories) + + # For PR, we always require tests. + signing_categories.add("tests") + if add_external_category: + signing_categories.add("externals") + if cms_repo: + print("This pull request requires ORP approval") + signing_categories.add("orp") + + print("Following categories affected:") + print("\n".join(signing_categories)) + + if cmssw_repo: + # If there is a new package, add also a dummy "new" category. + all_packages = [ + package + for category_packages in list(CMSSW_CATEGORIES.values()) + for package in category_packages + ] + has_category = all([package in all_packages for package in packages]) + if not has_category: + new_package_message = ( + "\nThe following packages do not have a category, yet:\n\n" + ) + new_package_message += ( + "\n".join( + [package for package in packages if not package in all_packages] + ) + + "\n" + ) + new_package_message += "Please create a PR for https://github.com/cms-sw/cms-bot/blob/master/categories_map.py to assign category\n" + print(new_package_message) + signing_categories.add("new-package") + + # Add watchers.yaml information to the WATCHERS dict. + WATCHERS = read_repo_file(repo_config, "watchers.yaml", {}) + # Given the files modified by the PR, check if there are additional developers watching one or more. + author = pr.user.login + watchers = set( + [ + user + for chg_file in chg_files + for user, watched_regexp in list(WATCHERS.items()) + for regexp in watched_regexp + if re.match("^" + regexp + ".*", chg_file) and user != author + ] + ) + # Handle category watchers + catWatchers = read_repo_file(repo_config, "category-watchers.yaml", {}) + non_block_cats = [] if not "mtype" in extra_labels else extra_labels["mtype"] + for user, cats in list(catWatchers.items()): + for cat in cats: + if (cat in signing_categories) or (cat in non_block_cats): + print("Added ", user, " to watch due to cat", cat) + watchers.add(user) + + # Handle watchers + watchingGroups = read_repo_file(repo_config, "groups.yaml", {}) + for watcher in [x for x in watchers]: + if not watcher in watchingGroups: + continue + watchers.remove(watcher) + watchers.update(set(watchingGroups[watcher])) + watchers = set([gh_user_char + u for u in watchers]) + print("Watchers " + ", ".join(watchers)) + for event in events.values(): if event["type"] == "sign": for cat, sign in event["value"].items(): signatures[cat] = sign elif event["type"] == "commit": chg_categories = [ - x for x in set(cmssw_file2Package(repo_config, f) for f in event["value"]) + x + for x in set(cmssw_file2Package(repo_config, f) for f in event["value"]) ] signatures["orp"] = "pending" for cat in chg_categories: @@ -1516,11 +1630,19 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F print(desc) if not dryRun: last_commit_obj.create_status( - "success", description=desc, target_url=turl, context=bot_status_name + "success", + description=desc, + target_url=turl, + context=bot_status_name, ) set_comment_emoji(test_comment.id, repository) if bot_status: - print(bot_status.target_url, turl, signatures["tests"], bot_status.description) + print( + bot_status.target_url, + turl, + signatures["tests"], + bot_status.description, + ) if ( bot_status and bot_status.target_url == turl @@ -1529,7 +1651,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ): signatures["tests"] = "started" if ( - get_status_state("%s/unknown/release" % cms_status_prefix, commit_statuses) + get_status_state( + "%s/unknown/release" % cms_status_prefix, commit_statuses + ) == "error" ): signatures["tests"] = "pending" @@ -1552,7 +1676,10 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F for s in [ i for i in commit_statuses - if ((i.context == scontext) or (i.context.startswith(scontext + "/"))) + if ( + (i.context == scontext) + or (i.context.startswith(scontext + "/")) + ) ]: if (not result_url) and ("/jenkins-artifacts/" in s.target_url): xdata = s.target_url.split("/") @@ -1673,10 +1800,15 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if already_seen: if dryRun: - print("Update PR seen message to include backport PR number", backport_pr_num) + print( + "Update PR seen message to include backport PR number", + backport_pr_num, + ) else: new_msg = "" - for l in already_seen.body.encode("ascii", "ignore").decode().split("\n"): + for l in ( + already_seen.body.encode("ascii", "ignore").decode().split("\n") + ): if BACKPORT_STR in l: continue new_msg += l + "\n" @@ -1825,7 +1957,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F # get release managers SUPER_USERS = read_repo_file(repo_config, "super-users.yaml", []) - releaseManagersList = ", ".join([gh_user_char + x for x in set(releaseManagers + SUPER_USERS)]) + releaseManagersList = ", ".join( + [gh_user_char + x for x in set(releaseManagers + SUPER_USERS)] + ) if cmssw_prs: global_test_params["PULL_REQUESTS"] = cmssw_prs @@ -1861,11 +1995,20 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F global_test_params["CONTEXT_PREFIX"] = cms_status_prefix if trigger_test: create_properties_file_tests( - repository, prId, global_test_params, dryRun, abort=False, repo_config=repo_config + repository, + prId, + global_test_params, + dryRun, + abort=False, + repo_config=repo_config, ) if not dryRun: set_comment_emoji(test_comment.id, repository) - elif abort_test and bot_status and (not bot_status.description.startswith("Aborted")): + elif ( + abort_test + and bot_status + and (not bot_status.description.startswith("Aborted")) + ): if not has_user_emoji(abort_test, repository, "+1", cmsbuild_user): create_properties_file_tests( repository, prId, global_test_params, dryRun, abort=True @@ -1947,7 +2090,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F gh_user_char + name for name, l2_categories in list(CMSSW_L2.items()) for signature in signing_categories - if signature in l2_categories and signature in unsigned and signature not in ["orp"] + if signature in l2_categories + and signature in unsigned + and signature not in ["orp"] ] missing_notifications = set(missing_notifications) @@ -1987,7 +2132,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F pkg_msg = [] for pkg in packages: if pkg in package_categories: - pkg_msg.append("- %s (**%s**)" % (pkg, ", ".join(package_categories[pkg]))) + pkg_msg.append( + "- %s (**%s**)" % (pkg, ", ".join(package_categories[pkg])) + ) else: pkg_msg.append("- %s (**new**)" % pkg) messageNewPR = format( @@ -2075,7 +2222,8 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if pre_check not in signatures: signatures[pre_check] = "pending" print( - "PRE CHECK: %s,%s,%s" % (pre_check, signatures[pre_check], pre_checks_state[pre_check]) + "PRE CHECK: %s,%s,%s" + % (pre_check, signatures[pre_check], pre_checks_state[pre_check]) ) if signatures[pre_check] != "pending": if pre_checks_state[pre_check] in ["pending", ""]: @@ -2091,7 +2239,10 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ) continue if (not dryRunOrig) and (pre_checks_state[pre_check] == ""): - params = {"PULL_REQUEST": "%s" % (prId), "CONTEXT_PREFIX": cms_status_prefix} + params = { + "PULL_REQUEST": "%s" % (prId), + "CONTEXT_PREFIX": cms_status_prefix, + } if pre_check == "code-checks": params["CMSSW_TOOL_CONF"] = code_checks_tools params["APPLY_PATCH"] = str(code_check_apply_patch).lower() @@ -2123,7 +2274,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F mustMerge = True else: print("This pull request will not be automatically merged.") - if mustMerge == True: + if mustMerge: print("This pull request must be merged.") if not dryRun and (pr.state == "open"): pr.merge() @@ -2144,7 +2295,9 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F if not dryRun: emoji = "-1" if "ERRORS: " in test_params_msg else "+1" if e and (e["content"] != emoji): - delete_comment_emoji(str(e["id"]), test_params_comment.id, repository) + delete_comment_emoji( + str(e["id"]), test_params_comment.id, repository + ) state = "success" if emoji == "+1" else "error" last_commit_obj.create_status( state,