diff --git a/.github/workflows/ok-check.yml b/.github/workflows/ok-check.yml index 0e2267c775..320c20362b 100644 --- a/.github/workflows/ok-check.yml +++ b/.github/workflows/ok-check.yml @@ -8,6 +8,11 @@ on: branches: - master +env: + GITHUB_ORG: "zeldaret" + GITHUB_REPO: "tp" + STATE_FILE: "tools/pjstate.yml" + jobs: build: runs-on: ubuntu-latest @@ -39,7 +44,7 @@ jobs: with: files: | **/*.{c,cpp,inc} - - name: Update Status + - name: Update Issue(s) if: github.event_name != 'pull_request' && steps.changed-files-specific.outputs.any_changed == 'true' run: | # Install libclang-16-dev for FunctionChecker @@ -49,7 +54,7 @@ jobs: sudo apt install -y libclang-16-dev FILENAMES="${{ steps.changed-files-specific.outputs.all_changed_files }}" - CMD="./tp github-check-update-status --personal-access-token ${{ secrets.PAT_TOKEN }} --debug" + CMD="./tp github-update-issues --personal-access-token ${{ secrets.PAT_TOKEN }} --debug --owner ${{ env.GITHUB_ORG }} --repo ${{ env.GITHUB_REPO }} --state-file ${{ env.STATE_FILE }}" IFS=' ' read -ra FILE_ARRAY <<< "$FILENAMES" INC_FOUND=false @@ -70,7 +75,9 @@ jobs: fi for FILE in "${FILE_ARRAY[@]}"; do - CMD="$CMD --filename $FILE" + AUTHOR=$(git log -1 --pretty=format:'%an' -- $FILE) + CMD="$CMD --filename $FILE --author $AUTHOR" done - $CMD + # Update the status and assignees for every issue identified + $CMD \ No newline at end of file diff --git a/tools/libgithub/__init__.py b/tools/libgithub/__init__.py index 0cd8412198..30a00074be 100644 --- a/tools/libgithub/__init__.py +++ b/tools/libgithub/__init__.py @@ -1,4 +1,5 @@ from .issue import * from .project import * from .label import * -from .repository import * \ No newline at end of file +from .repository import * +from .user import * \ No newline at end of file diff --git a/tools/libgithub/graphql.py b/tools/libgithub/graphql.py index 4b1c1e67d9..27dafb6b39 100644 --- a/tools/libgithub/graphql.py +++ b/tools/libgithub/graphql.py @@ -56,5 +56,14 @@ class GraphQLClient: else: LOG.error(f"Fail. Error: {error_message}") return None + + if data.get('extensions', ''): + warning_message = data['extensions']['warnings'][0]['message'] + LOG.warning(warning_message) + + if 'Bad credentials' in data.get('message', ''): + LOG.error(data['message']) + LOG.error('Invalid personal access token. Please provide a valid one with the --personal-access-token flag.') + sys.exit(1) return data \ No newline at end of file diff --git a/tools/libgithub/issue.py b/tools/libgithub/issue.py index 37bd3fee18..f0e8238bc9 100644 --- a/tools/libgithub/issue.py +++ b/tools/libgithub/issue.py @@ -1,6 +1,7 @@ import sys from .label import * +from .user import * from typing import Optional @dataclass @@ -90,11 +91,14 @@ class Issue: return Issue.get_by_state("OPEN") @staticmethod - def get_all_from_yaml(data): + def get_all_from_yaml(data, project_name): ret_issues = [] labels_dict = {label['name']: label['id'] for label in StateFile.data["labels"]} for d in data: + if d.get('project', {}).get('title', 'MISSING_TITLE') != project_name and project_name is not None: + LOG.debug("Project name was passed in but doesn't match the current project, skipping.") + continue # Get tu, labels, filepath for current project tu_info = get_translation_units(d) @@ -128,6 +132,42 @@ class Issue: return ret_issues + def get_all_assignees(self) -> list[User]: + LOG.debug(f'Getting all assignees on for Issue {self.title}') + + query = ''' + query ($id: ID!) { + node(id: $id) { + ... on Issue { + assignees(first: 100) { + nodes { + id + login + } + } + } + } + } + ''' + + variables = { + "id": self.id + } + + data = GraphQLClient.get_instance().make_request(query, variables) + if data: + assignees = data["data"]["node"]["assignees"]["nodes"] + LOG.debug(f'Got assignees: {assignees}') + + ret_users = [] + for assignee in assignees: + ret_users.append(User(id=assignee["id"],name=assignee["login"])) + + return ret_users + else: + LOG.error(f'Failed to get assignees for issue {self.title}') + sys.exit(1) + @staticmethod def get_by_unique_id(unique_id: str) -> 'Issue': LOG.debug(f'Getting issue with unique ID {unique_id} on {RepoInfo.owner.name}/{RepoInfo.name}') @@ -313,51 +353,34 @@ class Issue: LOG.info(f"Issue {self.title} from TU {self.file_path} already setup!") self.id = issue_dict[self.file_path]["id"] else: - LOG.info(f'Creating missing issue {self.title}.') + LOG.debug(f'Creating missing issue {self.title}.') self.create() - # def check_and_attach_labels(self) -> None: - # LOG.debug(f'Checking labels for issue {self.title} on {RepoInfo.owner.name}/{RepoInfo.name}') + def add_assignees(self, assignees: list[User]) -> None: + LOG.debug(f'Adding assignees to issue {self.id} on {RepoInfo.owner.name}/{RepoInfo.name}') - # issues = StateFile.data.get('issues') + mutation = """ + mutation UpdateIssue($input: UpdateIssueInput!) { + updateIssue(input: $input) { + clientMutationId + } + } + """ - # if issues is None: - # issue_dict = {} - # else: - # issue_dict = {issue['file_path']: issue for issue in issues} + input_dict = { + "assigneeIds": [assignee.id for assignee in assignees], + "id": self.id + } - # if self.file_path in issue_dict: - # state_labels = StateFile.data.get('labels') - # label_ids = issue_dict[self.file_path]["label_ids"] + variables = { + "input": input_dict + } - # if label_ids is not None: - # state_label_ids = [label['id'] for label in state_labels] - # for label_id in label_ids: - # if label_id in state_label_ids: - # LOG.debug(f'Label {label_id} exists in state, continuing') - # continue - # else: - # LOG.error(f'Label {label_id} does not exist in state, please run sync-labels first') - # sys.exit(1) - - # LOG.info(f'All labels already attached to issue {self.title} on {RepoInfo.owner.name}/{RepoInfo.name}') - # else: - # LOG.info(f'Attaching labels to issue {self.title} on {RepoInfo.owner.name}/{RepoInfo.name}') - - # # use yaml data to fetch label names for this issue - # # lookup id from state and attach to issue - # = - # for label in : - # self.attach_label_by_id() # finish - - # LOG.info(f'Labels attached to issue {self.title} on {RepoInfo.owner.name}/{RepoInfo.name}') - - - # print(label_ids) - # sys.exit(0) - # else: - # LOG.error(f"Issue {self.title} from TU {self.file_path} is missing") - # sys.exit(1) + data = GraphQLClient.get_instance().make_request(mutation, variables) + if data: + LOG.debug(f'Assignees added to issue {self.id} on {RepoInfo.owner.name}/{RepoInfo.name}') + else: + LOG.error(f'Assignees could not be added to issue {self.id} on {RepoInfo.owner.name}/{RepoInfo.name}') def create(self): repo_id = RepoInfo.id @@ -414,6 +437,33 @@ class Issue: else: LOG.error(f'Failed to delete issue {self.title}') + def get_id(self,number) -> None: + LOG.debug(f'Getting ID for issue {self.title} on {RepoInfo.owner.name}/{RepoInfo.name}') + + query = ''' + query ($number: Int!, $owner: String!, $repo: String!) { + repository(owner: $owner, name: $repo) { + issue(number: $number) { + id + } + } + } + ''' + + variables = { + "owner": RepoInfo.owner.name, + "repo": RepoInfo.name, + "number": number + } + + data = GraphQLClient.get_instance().make_request(query, variables) + if data: + self.id = data['data']['repository']['issue']['id'] + LOG.debug(f'ID retrieved: {self.id} for issue {self.title}') + else: + LOG.error(f'No ID found for issue {self.title}') + sys.exit(1) + def write_state_to_file(self, delete: bool = False): state = { "id": self.id, @@ -439,5 +489,5 @@ class Issue: StateFile.data['issues'] = [state] - with open("tools/pjstate.yml", 'w') as f: + with open(StateFile.file_name, 'w') as f: yaml.safe_dump(StateFile.data, f) \ No newline at end of file diff --git a/tools/libgithub/label.py b/tools/libgithub/label.py index c4c64072e1..5c994825f5 100644 --- a/tools/libgithub/label.py +++ b/tools/libgithub/label.py @@ -11,11 +11,15 @@ import yaml, sys @dataclass class Label: @staticmethod - def get_all_from_yaml(data): + def get_all_from_yaml(data, project_name: str) -> list['Label']: ret_labels = [] sub_labels = [] + for d in data: + if d.get('project', {}).get('title', 'MISSING_TITLE') != project_name and project_name is not None: + LOG.debug("Project name was passed in but doesn't match the current project, skipping.") + continue sub_labels = get_sub_labels(d) for label in sub_labels: @@ -24,7 +28,6 @@ class Label: title_label = Label(data=d) ret_labels.append(title_label) - return ret_labels @@ -224,5 +227,5 @@ class Label: StateFile.data['labels'] = [state] - with open("tools/pjstate.yml", 'w') as f: + with open(StateFile.file_name, 'w') as f: yaml.safe_dump(StateFile.data, f) diff --git a/tools/libgithub/project.py b/tools/libgithub/project.py index 52795ecfae..92ba075837 100644 --- a/tools/libgithub/project.py +++ b/tools/libgithub/project.py @@ -30,11 +30,14 @@ class Project: self.status_field = status_field @staticmethod - def get_all_from_yaml(data) -> list['Project']: + def get_all_from_yaml(data, project_name) -> list['Project']: ret_projects = [] issues_dict = {issue['file_path']: issue['id'] for issue in StateFile.data["issues"]} for d in data: + if d.get('project', {}).get('title', 'MISSING_TITLE') != project_name and project_name is not None: + LOG.debug("Project name was passed in but doesn't match the current project, skipping.") + continue items = [] for tu, _, file_path in get_translation_units(d): @@ -511,7 +514,7 @@ class Project: StateFile.data['projects'] = [state] - with open("tools/pjstate.yml", 'w') as f: + with open(StateFile.file_name, 'w') as f: yaml.safe_dump(StateFile.data, f) # Custom representer for Option diff --git a/tools/libgithub/state.py b/tools/libgithub/state.py index 87e0459a91..9d501e3edd 100644 --- a/tools/libgithub/state.py +++ b/tools/libgithub/state.py @@ -1,9 +1,22 @@ -import yaml, pathlib +import yaml, pathlib, os class StateFile: data = None + file_name = None @classmethod def load(self, file_name: pathlib.Path): + if not os.path.exists(file_name): + default_payload = { + "issues": [], + "labels": [], + "projects": [] + } + + with open(file_name, 'w') as f: + yaml.dump(default_payload, f) + + self.file_name = file_name + with open(file_name, 'r') as f: self.data = yaml.safe_load(f) \ No newline at end of file diff --git a/tools/libgithub/user.py b/tools/libgithub/user.py new file mode 100644 index 0000000000..a07e9b91d5 --- /dev/null +++ b/tools/libgithub/user.py @@ -0,0 +1,31 @@ +from .graphql import GraphQLClient +from logger import LOG + +import sys + +class User: + def __init__(self, id = None, name = None): + self.id = id + self.name = name + + def get_id(self): + LOG.debug(f'Fetch ID for user {self.name}') + + query = ''' + query ($name: String!) { + user(login: $name) { + id + } + } + ''' + variables = { + "name": self.name, + } + + data = GraphQLClient.get_instance().make_request(query, variables) + if data: + self.id = data['data']['user']['id'] + LOG.info(f"Found user {self.name} with ID {self.id}!") + else: + LOG.error(f'Failed to find user {self.name}!') + sys.exit(1) \ No newline at end of file diff --git a/tools/tp.py b/tools/tp.py index c51427ab15..4e0a290bae 100644 --- a/tools/tp.py +++ b/tools/tp.py @@ -71,6 +71,10 @@ loggers = [logging.getLogger(name) for name in logging.root.manager.loggerDict] for logger in loggers: logger.setLevel(logging.INFO) +if sys.version_info < (3, 10): + LOG.error("This script requires Python 3.10 or newer!") + sys.exit(1) + DEFAULT_GAME_PATH = "game" DEFAULT_TOOLS_PATH = "tools" DEFAULT_BUILD_PATH = "build/dolzel2" @@ -1223,12 +1227,18 @@ def common_github_options(func): required=False, default="tp" ) + @click.option( + "--state-file", + help="File to store the state of the issues in. Defaults to tools/projects.yml", + required=False, + default="tools/pjstate.yml" + ) @functools.wraps(func) def wrapper(*args, **kwargs): return func(*args, **kwargs) return wrapper -def prereqs(owner: str, repo: str, personal_access_token: str): +def prereqs(owner: str, repo: str, personal_access_token: str, state_file: str): # Setup GraphQL client singleton libgithub.GraphQLClient.setup(personal_access_token) @@ -1239,9 +1249,9 @@ def prereqs(owner: str, repo: str, personal_access_token: str): libgithub.RepoInfo.set_ids() # Load in the project state - libgithub.StateFile.load("tools/pjstate.yml") + libgithub.StateFile.load(state_file) -def load_from_yaml(type: str) -> any: +def load_from_yaml(type: str, project_name: str) -> any: with open("./tools/projects.yml", 'r') as stream: try: import yaml @@ -1251,11 +1261,11 @@ def load_from_yaml(type: str) -> any: match type: case "labels": - ret_data = libgithub.Label.get_all_from_yaml(projects_data) + ret_data = libgithub.Label.get_all_from_yaml(projects_data, project_name) case "issues": - ret_data = libgithub.Issue.get_all_from_yaml(projects_data) + ret_data = libgithub.Issue.get_all_from_yaml(projects_data, project_name) case "projects": - ret_data = libgithub.Project.get_all_from_yaml(projects_data) + ret_data = libgithub.Project.get_all_from_yaml(projects_data, project_name) case _: LOG.error(f"Invalid type: {type}") sys.exit(1) @@ -1274,12 +1284,18 @@ def load_from_yaml(type: str) -> any: @tp.command(name="github-sync-labels", help="Creates all labels based on tools/projects.yml") @common_github_options -def github_sync_labels(debug: bool, personal_access_token: str, owner: str, repo: str): +@click.option( + "--project", + help="Only sync labels for a specific project", + required=False, + default=None +) +def github_sync_labels(debug: bool, personal_access_token: str, owner: str, repo: str, project: str, state_file: str): if debug: LOG.setLevel(logging.DEBUG) - prereqs(owner, repo, personal_access_token) - yaml_labels = load_from_yaml("labels") + prereqs(owner, repo, personal_access_token, state_file) + yaml_labels = load_from_yaml("labels", project) LOG.info("Syncing up labels") for label in yaml_labels: @@ -1287,12 +1303,18 @@ def github_sync_labels(debug: bool, personal_access_token: str, owner: str, repo @tp.command(name="github-sync-issues", help="Creates all issues and labels based on tools/projects.yml") @common_github_options -def github_sync_issues(debug: bool, personal_access_token: str, owner: str, repo: str): +@click.option( + "--project", + help="Only sync labels for a specific project", + required=False, + default=None +) +def github_sync_issues(debug: bool, personal_access_token: str, owner: str, repo: str, project: str, state_file: str): if debug: LOG.setLevel(logging.DEBUG) - prereqs(owner,repo,personal_access_token) - yaml_issues = load_from_yaml("issues") + prereqs(owner,repo,personal_access_token, state_file) + yaml_issues = load_from_yaml("issues", project) LOG.info("Syncing up issues") for issue in yaml_issues: @@ -1300,24 +1322,37 @@ def github_sync_issues(debug: bool, personal_access_token: str, owner: str, repo @tp.command(name="github-sync-projects", help="Creates all projects, issues and labels based on tools/projects.yml") @common_github_options -def github_sync_projects(debug: bool, personal_access_token: str, owner: str, repo: str): +@click.option( + "--project", + help="Only sync labels for a specific project", + required=False, + default=None +) +def github_sync_projects(debug: bool, personal_access_token: str, owner: str, repo: str, project: str, state_file: str): if debug: LOG.setLevel(logging.DEBUG) - prereqs(owner, repo, personal_access_token) - yaml_projects = load_from_yaml("projects") + prereqs(owner, repo, personal_access_token, state_file) + yaml_projects = load_from_yaml("projects", project) LOG.info("Syncing up projects") for project in yaml_projects: project.check_and_create() -@tp.command(name="github-check-update-status", help="Checks all issues and updates their status based on their local file path.") +@tp.command(name="github-update-issues", help="Checks all issues and updates their status and assigness.") @common_github_options @click.option( '--filename','filenames', + help="Filename(s) used to look for and update issues.", multiple=True, type=click.Path(exists=True) ) +@click.option( + '--author', + multiple=True, + help="Author(s) to assign issues to.", + default=None +) @click.option( '--all', help="Check all items in every project and update their status.", @@ -1329,28 +1364,36 @@ def github_sync_projects(debug: bool, personal_access_token: str, owner: str, re help="Path to libclang.so", default="/usr/lib/x86_64-linux-gnu/libclang-16.so" ) -def github_check_update_status(debug: bool, personal_access_token: str, owner: str, repo: str, filenames: Tuple[click.Path], all: bool, clang_lib_path: str): +def github_update_issues(debug: bool, personal_access_token: str, owner: str, repo: str, filenames: Tuple[click.Path], all: bool, author: str, clang_lib_path: str, state_file: str): if debug: LOG.setLevel("DEBUG") - prereqs(owner, repo, personal_access_token) + if author == () and all == False: + LOG.error("Author is required when --all is not set. Please set it using the --author argument.") + sys.exit(1) + + prereqs(owner, repo, personal_access_token, state_file) issues = libgithub.StateFile.data.get('issues') projects = libgithub.StateFile.data.get('projects') filenames_list = list(filenames) + author_list = list(author) + + if len(author_list) == 0: + author_list = [""] * len(filenames_list) # If all flag is set, check all issue file paths in state file if all: for issue in issues: filenames_list.append(issue["file_path"]) - import classify_tu, clang + import classify_tu, clang, itertools # Set the clang library file clang.cindex.Config.set_library_file(clang_lib_path) - for filename in filenames_list: + for filename,author in itertools.zip_longest(filenames_list,author_list): LOG.info(f"Classifying TU {filename}") status = classify_tu.run(filename) @@ -1364,6 +1407,7 @@ def github_check_update_status(debug: bool, personal_access_token: str, owner: s for issue in issues: if issue["file_path"] == filename: issue_id = issue["id"] + issue_title = issue["title"] break if issue_id is None: @@ -1385,8 +1429,22 @@ def github_check_update_status(debug: bool, personal_access_token: str, owner: s sys.exit(1) libgithub.Project(id=project_id,status_field=status_field).set_status_for_item(item_id, status) + github_issue = libgithub.Issue(id=issue_id,title=issue_title) + + # Add the author as an assignee if it was passed in + if author is not None: + # Find the matching author + author_user = libgithub.User(name=author) + author_user.get_id() + + # Add the author as an assignee + assignees = github_issue.get_all_assignees() + assignees.append(author_user) + github_issue.add_assignees(assignees) + + # Close the issue if status is done if status == "done": - libgithub.Issue(id=issue_id).set_closed() + github_issue.set_closed() # # Github Clean Commands @@ -1394,7 +1452,7 @@ def github_check_update_status(debug: bool, personal_access_token: str, owner: s @tp.command(name="github-clean-labels", help="Delete all labels for a given owner/repository.") @common_github_options -def github_clean_labels(debug: bool, personal_access_token: str, owner: str, repo: str) -> None: +def github_clean_labels(debug: bool, personal_access_token: str, owner: str, repo: str, state_file: str) -> None: if debug: LOG.setLevel("DEBUG") @@ -1402,14 +1460,14 @@ def github_clean_labels(debug: bool, personal_access_token: str, owner: str, rep confirmation = input().lower() if confirmation == 'y': - prereqs(owner,repo,personal_access_token) + prereqs(owner,repo,personal_access_token, state_file) libgithub.Label.delete_all() else: sys.exit(0) @tp.command(name="github-clean-issues", help="Delete all issues for a given owner/repository.") @common_github_options -def github_clean_issues(debug: bool, personal_access_token: str, owner: str, repo: str): +def github_clean_issues(debug: bool, personal_access_token: str, owner: str, repo: str, state_file: str) -> None: if debug: LOG.setLevel("DEBUG") @@ -1417,14 +1475,14 @@ def github_clean_issues(debug: bool, personal_access_token: str, owner: str, rep confirmation = input().lower() if confirmation == 'y': - prereqs(owner,repo,personal_access_token) + prereqs(owner,repo,personal_access_token, state_file) libgithub.Issue.delete_all() else: sys.exit(0) @tp.command(name="github-clean-projects", help="Delete all projects for a given owner/repository.") @common_github_options -def github_clean_projects(debug: bool, personal_access_token: str, owner: str, repo: str): +def github_clean_projects(debug: bool, personal_access_token: str, owner: str, repo: str, state_file: str) -> None: if debug: LOG.setLevel("DEBUG") @@ -1432,7 +1490,7 @@ def github_clean_projects(debug: bool, personal_access_token: str, owner: str, r confirmation = input().lower() if confirmation == 'y': - prereqs(owner,repo,personal_access_token) + prereqs(owner,repo,personal_access_token, state_file) libgithub.Project.delete_all() else: sys.exit(0)