From 8067751638b039c23b9e4e10206b7254b828a09a Mon Sep 17 00:00:00 2001 From: Hiroshi Nishio <4620828+hiroshinishio@users.noreply.github.com> Date: Wed, 3 Jul 2024 20:33:08 +0900 Subject: [PATCH] Add a loop process to scheduler --- cloudformation.yml | 2 +- main.py | 2 +- scheduler.py | 76 +++++++++++++++++----------- services/github/github_manager.py | 33 ++++++++++-- services/supabase/gitauto_manager.py | 13 +++++ 5 files changed, 90 insertions(+), 36 deletions(-) diff --git a/cloudformation.yml b/cloudformation.yml index 6176ffe8..1ce2acbe 100644 --- a/cloudformation.yml +++ b/cloudformation.yml @@ -20,7 +20,7 @@ Resources: Properties: Name: SchedulerEventRule Description: "Schedule Lambda function to run every weekday at 0 AM UTC" - ScheduleExpression: cron(45 7 ? * MON-FRI *) # min hour day month day-of-week year + ScheduleExpression: cron(45 11 ? * MON-FRI *) # min hour day month day-of-week year State: ENABLED Targets: - Arn: !Ref LambdaFunctionArn diff --git a/main.py b/main.py index 3e8dc71d..7e7c1ae8 100644 --- a/main.py +++ b/main.py @@ -31,7 +31,7 @@ # Here is an entry point for the AWS Lambda function. Mangum is a library that allows you to use FastAPI with AWS Lambda. def handler(event, context): if "source" in event and event["source"] == "aws.events": - schedule_handler(event=event, context=context) + schedule_handler(_event=event, _context=context) return {"statusCode": 200} return mangum_handler(event=event, context=context) diff --git a/scheduler.py b/scheduler.py index bc221c92..3d96466c 100644 --- a/scheduler.py +++ b/scheduler.py @@ -1,9 +1,11 @@ """This is scheduled to run by AWS Lambda""" +import logging from config import PRODUCT_ID, SUPABASE_SERVICE_ROLE_KEY, SUPABASE_URL from services.github.github_manager import ( add_label_to_issue, get_installation_access_token, + get_installed_owners_and_repos, get_oldest_unassigned_open_issue, ) from services.github.github_types import IssueInfo @@ -12,33 +14,47 @@ supabase_manager = SupabaseManager(url=SUPABASE_URL, key=SUPABASE_SERVICE_ROLE_KEY) -def schedule_handler(event, context) -> dict[str, int]: - print(f"Event: {event}") - print(f"Context: {context}") - - # List installed and paid repositories. - - # Check available remaining counts for each repository and exclude those with zero or less. - - # Identify the oldest open and unassigned issue for each repository. - owner_id = 159883862 # gitautoai - owner = "gitautoai" - repo = "gitauto" - installation_id = supabase_manager.get_installation_id(owner_id=owner_id) - token: str = get_installation_access_token(installation_id=installation_id) - issue: IssueInfo | None = get_oldest_unassigned_open_issue( - owner=owner, repo=repo, token=token - ) - - # Return early if there are no open issues. - if issue is None: - return {"statusCode": 200} - - # Extract the issue number if there is an open issue. - issue_number = issue["number"] - - # Label the issue with the product ID to trigger GitAuto. - add_label_to_issue( - owner=owner, repo=repo, issue_number=issue_number, label=PRODUCT_ID, token=token - ) - return {"statusCode": 200} +def schedule_handler(_event, _context) -> dict[str, int]: + print("\n" * 3 + "-" * 70) + + # Get all active installation IDs from Supabase including free customers. + installation_ids: list[int] = supabase_manager.get_installation_ids() + + # Get all owners and repositories from GitHub. + for installation_id in installation_ids: + token: str = get_installation_access_token(installation_id=installation_id) + owners_repos: list[dict[str, str]] = get_installed_owners_and_repos( + installation_id=installation_id, token=token + ) + + # Process each owner and repository. + for owner_repo in owners_repos: + owner: str = owner_repo["owner"] + repo: str = owner_repo["repo"] + logging.info("Processing %s/%s", owner, repo) + + # Identify an oldest, open, unassigned, and not gitauto labeled issue for each repository. + issue: IssueInfo | None = get_oldest_unassigned_open_issue( + owner=owner, repo=repo, token=token + ) + logging.info("Issue: %s", issue) + + # This is testing purpose. + if owner != "gitautoai": + continue + + # Continue to the next set of owners and repositories if there is no open issue. + if issue is None: + continue + + # Extract the issue number if there is an open issue. + issue_number = issue["number"] + + # Label the issue with the product ID to trigger GitAuto. + add_label_to_issue( + owner=owner, + repo=repo, + issue_number=issue_number, + label=PRODUCT_ID, + token=token, + ) diff --git a/services/github/github_manager.py b/services/github/github_manager.py index e1352795..9a37c031 100644 --- a/services/github/github_manager.py +++ b/services/github/github_manager.py @@ -305,7 +305,7 @@ def create_remote_branch( timeout=TIMEOUT_IN_SECONDS, ) response.raise_for_status() - except Exception as e: + except Exception as e: # pylint: disable=broad-except update_comment_for_raised_errors( error=e, comment_url=comment_url, @@ -341,6 +341,21 @@ def get_installation_access_token(installation_id: int) -> str: return response.json()["token"] +@handle_exceptions(default_return_value=[], raise_on_error=False) +def get_installed_owners_and_repos( + installation_id: str, token: str +) -> list[dict[str, str]]: + """https://docs.github.com/en/rest/apps/installations?apiVersion=2022-11-28#list-repositories-accessible-to-the-app-installation""" + response: requests.Response = requests.get( + url=f"{GITHUB_API_URL}/user/installations/{installation_id}/repositories", + headers=create_headers(token=token), + timeout=TIMEOUT_IN_SECONDS, + ) + response.raise_for_status() + repos = response.json().get("repositories", []) + return [{"owner": repo["owner"]["login"], "repo": repo["name"]} for repo in repos] + + @handle_exceptions(default_return_value=[], raise_on_error=False) def get_issue_comments( owner: str, repo: str, issue_number: int, token: str @@ -445,13 +460,23 @@ def get_oldest_unassigned_open_issue( # Find the first issue without the PRODUCT_ID label for issue in issues: - if all(label['name'] != PRODUCT_ID for label in issue['labels']): + if all(label["name"] != PRODUCT_ID for label in issue["labels"]): return issue # If there are open issues, but all of them have the PRODUCT_ID label, continue to the next page page += 1 +@handle_exceptions(default_return_value=None, raise_on_error=False) +def get_owner_name(owner_id: int, token: str) -> str | None: + """https://docs.github.com/en/rest/users/users?apiVersion=2022-11-28#get-a-user-using-their-id""" + response: requests.Response = requests.get( + url=f"{GITHUB_API_URL}/user/{owner_id}", + headers=create_headers(token=token), + timeout=TIMEOUT_IN_SECONDS, + ) + response.raise_for_status() + return response.json()["login"] @handle_exceptions(default_return_value="", raise_on_error=False) @@ -497,7 +522,7 @@ def get_remote_file_tree( msg=f"get_remote_file_tree HTTP Error: {http_err.response.status_code} - {http_err.response.text}" ) return [] - except Exception as e: + except Exception as e: # pylint: disable=broad-except update_comment_for_raised_errors( error=e, comment_url=comment_url, @@ -581,7 +606,7 @@ def update_comment_for_raised_errors( ) else: logging.error("%s Error: %s", which_function, error) - except Exception as e: + except Exception as e: # pylint: disable=broad-except logging.error("%s Error: %s", which_function, e) update_comment(comment_url=comment_url, token=token, body=body) diff --git a/services/supabase/gitauto_manager.py b/services/supabase/gitauto_manager.py index ebe47c30..738f3d72 100644 --- a/services/supabase/gitauto_manager.py +++ b/services/supabase/gitauto_manager.py @@ -1,6 +1,7 @@ """Class to manage all GitAuto related operations""" from datetime import datetime, timezone +import json from supabase import Client from services.stripe.customer import create_stripe_customer, subscribe_to_free_plan from utils.handle_exceptions import handle_exceptions @@ -162,6 +163,18 @@ def get_installation_id(self, owner_id: int) -> int: # Return the first installation id even if there are multiple installations return data[1][0]["installation_id"] + @handle_exceptions(default_return_value=None, raise_on_error=False) + def get_installation_ids(self) -> list[int]: + """https://supabase.com/docs/reference/python/is""" + data, _ = ( + self.client.table(table_name="installations") + .select("installation_id") + .is_(column="uninstalled_at", value="null") # Not uninstalled + .execute() + ) + print(f"Installation ids: {json.dumps(data)}") + return data[1] + @handle_exceptions(default_return_value=False, raise_on_error=False) def is_users_first_issue(self, user_id: int, installation_id: int) -> bool: """Checks if it's the users first issue"""