SERVER-111676 Port the create_todo_tickets.py script to server. (#42953)

GitOrigin-RevId: 66610fb2bf6a746ac7061fc87f320d7948642ecd

buildscripts/create_todo_tickets.py

@@ -0,0 +1,161 @@
+"""
+Walk all files in a directory, checking whether any TODOs are linked to a
+resolved JIRA ticket, and labeling those JIRA tickets.
+"""
+
+#!/usr/bin/env python3
+
+import argparse
+import os
+import re
+import sys
+
+import structlog
+from jira import JIRAError
+from structlog.stdlib import LoggerFactory
+
+# Get relative imports to work when the package is not installed on the PYTHONPATH.
+if __name__ == "__main__" and __package__ is None:
+    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from buildscripts.client.jiraclient import JiraAuth, JiraClient
+
+structlog.configure(logger_factory=LoggerFactory())
+LOG = structlog.getLogger(__name__)
+JIRA_SERVER = "https://jira.mongodb.org"
+
+
+def find_todos(search_file, jira, file_name):
+    """Iterate through a file, finding TODOs with resolved tickets and creating new tickets."""
+    for i, line in enumerate(search_file):
+        if "todo" in line.lower():
+            issue_key = get_issue_key_from_line(line)
+            if issue_key:
+                LOG.info("\n")
+                LOG.info("=== Found Issue Key ===")
+                LOG.info(line.lstrip().rstrip())
+                LOG.info(f"{JIRA_SERVER}/browse/{issue_key}")
+                LOG.info(f"Found in {file_name} on line {i+1}")
+
+                # It is possible the referenced ticket in the code was deleted, so we need to
+                # handle the possibility that searching for it will return nothing.
+                try:
+                    issue = jira.issue(issue_key)
+                except JIRAError:
+                    LOG.warn(f"{issue_key} not found in Jira.  Skipping.")
+                    continue
+
+                status = str(issue.fields.status)
+                if status not in ["Resolved", "Closed"]:
+                    LOG.info(f"{issue_key} is not resolved.  Skipping.")
+                    continue
+
+                if todo_ticket_exists(jira, issue):
+                    LOG.info(f"Autogenerated ticket linked to {issue_key} already exists.")
+                    continue
+
+                create_todo_ticket(jira, issue)
+
+
+def todo_ticket_exists(jira, resolved_issue):
+    """Check if we have already created a ticket for this resolved issue"""
+    jql = (
+        "labels = autogen-todo AND resolution is empty AND issueFunction in"
+        f" linkedIssuesOf('key={resolved_issue.key}')"
+    )
+    results = jira.search_issues(jql, maxResults=1000)
+    if not results or results.total == 0:
+        return False
+    return True
+
+
+def create_todo_ticket(jira, resolved_issue):
+    """Given a resolved ticket, create a new ticket for that work and link to the resolved ticket."""
+    key = resolved_issue.key
+    assignee = get_assignable_user(resolved_issue)
+    assigned_team = resolved_issue.fields.customfield_12751
+    # Derive Team from Resolved ticket
+    team = assigned_team[0].value if assigned_team else "Server Triage"
+
+    issue_dict = {
+        "project": {"key": "SERVER"},
+        "issuetype": {"name": "Task"},
+        "summary": "Complete TODO listed in " + key,
+        "assignee": {"name": assignee},
+        "description": construct_description(key),
+        "labels": ["autogen-todo"],
+        "customfield_12751": [{"value": team}],
+    }
+
+    if "REP" in key:
+        issue_dict["project"] = {"key": "REP"}
+
+    # It's possible for us to try and create a ticket with an illegal assignee (most commonly
+    # a former employee) so for now we default to hard assigning these to Joe to reassign.
+    # This situation should be very infrequent.
+    try:
+        new_issue = jira.create_issue(fields=issue_dict)
+    except JIRAError:
+        issue_dict["assignee"]["name"] = "joseph.kanaan@mongodb.com"
+        new_issue = jira.create_issue(fields=issue_dict)
+    jira.create_issue_link(
+        type="Related", inwardIssue=resolved_issue.key, outwardIssue=new_issue.key
+    )
+
+
+def construct_description(key):
+    repo = "10gen/mongosync" if "REP" in key else "10gen/mongo"
+    return (
+        "There is a TODO in the codebase referencing a resolved ticket which is"
+        " assigned to you.\n\nPlease follow this link to see the lines of code"
+        " referencing this resolved"
+        f" ticket:\nhttps://github.com/{repo}/search?q={key}&type=Code\n\nThe next"
+        " steps for this ticket are to either remove the outdated TODO or follow the"
+        " steps in the TODO if it is correct. If the latter, please update the summary"
+        " and description of this ticket to represent the work you're actually doing."
+    )
+
+
+def get_assignable_user(ticket):
+    """If the original ticket is assigned to an existing employee, return the assignee"""
+    assignee = None
+    if ticket.fields.assignee:
+        assignee = ticket.fields.assignee.name
+    return assignee
+
+
+def get_issue_key_from_line(line):
+    """Given a string of text, find and return any issue keys from relevent projects."""
+    match = re.search(
+        "(BUILD|SERVER|WT|SPM|TOOLS|TIG|PERF|BF|REP|BACKPORT|WRITING|STAR)-[0-9]+",
+        line,
+        re.IGNORECASE,
+    )
+    if match:
+        return match.group(0)
+
+
+def main():
+    """Execute main function."""
+    argparser = argparse.ArgumentParser(description="")
+    argparser.add_argument("--env", "-e", help="Jira environment {stg, prod}", required=False)
+    argparser.add_argument("--path", "-p", help="File path to walk", required=True)
+    args = vars(argparser.parse_args())
+
+    jira = JiraClient(JIRA_SERVER, JiraAuth(), dry_run=False)
+
+    for root, _, files in os.walk(args["path"]):
+        for file_name in files:
+            # ignore .git/
+            if ".git" in root:
+                continue
+            try:
+                with open(os.path.join(root, file_name), "r") as search_file:
+                    find_todos(search_file, jira._jira, file_name)
+                    search_file.close()
+            except UnicodeDecodeError:
+                continue
+
+
+if __name__ == "__main__":
+    main()

etc/evergreen_yml_components/tasks/misc_tasks.yml

@@ -2025,3 +2025,29 @@ tasks:
       - func: "set up venv"
       - func: "configure evergreen api credentials"
       - func: "do multiversion selection"
+
+  - name: create_todo_tickets
+    patchable: false
+    tags: []
+    exec_timeout_secs: 600 # 10 minute timeout
+    commands:
+      - command: manifest.load
+      - func: "git get shallow project"
+      - func: "f_expansions_write"
+      - func: "kill processes"
+      - func: "cleanup environment"
+      - func: "set up venv"
+      - func: "upload pip requirements"
+      - command: subprocess.exec
+        display_name: "create todo tickets"
+        params:
+          binary: bash
+          args:
+            - "./src/evergreen/run_python_script.sh"
+            - "buildscripts/create_todo_tickets.py"
+            - "--path=."
+          env:
+            JIRA_AUTH_ACCESS_TOKEN: ${jira_auth_access_token}
+            JIRA_AUTH_ACCESS_TOKEN_SECRET: ${jira_auth_access_token_secret}
+            JIRA_AUTH_CONSUMER_KEY: ${jira_auth_consumer_key}
+            JIRA_AUTH_KEY_CERT: ${jira_auth_key_cert}

etc/evergreen_yml_components/variants/misc/misc_master_branch_only.yml

@@ -31,6 +31,7 @@ buildvariants:
       - name: test_copybara_sync
         distros:
           - ubuntu2204-small
+      - name: create_todo_tickets
 
   # Experimental variant running bazel targets for integration tests. To be removed with SERVER-103537.
   - name: bazel-integration-tests