From 7a4af9b91308a55c25ef78c639423950dab98d70 Mon Sep 17 00:00:00 2001 From: Hiroshi Nishio Date: Sat, 14 Dec 2024 14:36:12 +0900 Subject: [PATCH 1/2] Stop skipping check run review in dev env --- services/check_run_handler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/check_run_handler.py b/services/check_run_handler.py index aea323a6..7272da93 100644 --- a/services/check_run_handler.py +++ b/services/check_run_handler.py @@ -3,6 +3,7 @@ from config import ( EMAIL_LINK, GITHUB_APP_USER_NAME, + IS_PRD, STRIPE_PRODUCT_ID_FREE, SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY, @@ -112,7 +113,7 @@ def handle_check_run(payload: CheckRunCompletedPayload) -> None: # Return here if product_id is not found or is in free tier product_id: str | None = get_stripe_product_id(customer_id=stripe_customer_id) - if product_id is None or product_id == STRIPE_PRODUCT_ID_FREE: + if product_id is None or product_id == STRIPE_PRODUCT_ID_FREE and IS_PRD: msg = f"Skipping because product_id is not found or is in free tier. product_id: '{product_id}'" print(colorize(text=msg, color="yellow")) return From 3a00ec450f1047282f4fd7077b13ac19d9caea86 Mon Sep 17 00:00:00 2001 From: Hiroshi Nishio Date: Sat, 14 Dec 2024 15:52:03 +0900 Subject: [PATCH 2/2] Enable GitAuto to analyze images in GitHub issue body and comments and add the feedback to the issue --- services/gitauto_handler.py | 19 +++++++++++++ services/github/asset_manager.py | 28 +++++++++++++++++++ services/github/github_utils.py | 5 +--- .../openai/instructions/describe_image.py | 13 ++++++++- services/openai/vision.py | 12 +++++++- utils/extract_urls.py | 12 ++++---- 6 files changed, 77 insertions(+), 12 deletions(-) create mode 100644 services/github/asset_manager.py diff --git a/services/gitauto_handler.py b/services/gitauto_handler.py index a45f1c75..86fc3313 100644 --- a/services/gitauto_handler.py +++ b/services/gitauto_handler.py @@ -13,6 +13,7 @@ SUPABASE_SERVICE_ROLE_KEY, PR_BODY_STARTS_WITH, ) +from services.github.asset_manager import get_base64, render_text from services.github.comment_manager import delete_my_comments from services.github.github_manager import ( create_pull_request, @@ -31,7 +32,9 @@ from services.openai.commit_changes import chat_with_agent from services.openai.instructions.write_pr_body import WRITE_PR_BODY from services.openai.chat import chat_with_ai +from services.openai.vision import describe_image from services.supabase import SupabaseManager +from utils.extract_urls import extract_image_urls from utils.progress_bar import create_progress_bar from utils.text_copy import ( UPDATE_COMMENT_FOR_422, @@ -71,12 +74,14 @@ async def handle_gitauto( issue_number = base_args["issue_number"] issue_title = base_args["issue_title"] issue_body = base_args["issue_body"] + issue_body_rendered = render_text(base_args=base_args, text=issue_body) issuer_name = base_args["issuer_name"] new_branch_name = base_args["new_branch"] sender_id = base_args["sender_id"] sender_name = base_args["sender_name"] sender_email = base_args["sender_email"] github_urls = base_args["github_urls"] + # other_urls = base_args["other_urls"] token = base_args["token"] is_automation = base_args["is_automation"] # Check if the user has reached the request limit @@ -122,6 +127,7 @@ async def handle_gitauto( comment_body = "Checking the issue title, body, comments, and root files list..." update_comment(body=comment_body, base_args=base_args, p=10) root_files_and_dirs: list[str] = get_remote_file_tree(base_args=base_args) + issue_comments: list[str] = [] if input_from == "github": issue_comments = get_issue_comments( issue_number=issue_number, base_args=base_args @@ -129,6 +135,19 @@ async def handle_gitauto( elif input_from == "jira": issue_comments = base_args["issue_comments"] + # Check out the image URLs in the issue body and comments + image_urls = extract_image_urls(text=issue_body_rendered) + for issue_comment in issue_comments: + issue_comment_rendered = render_text(base_args=base_args, text=issue_comment) + image_urls.extend(extract_image_urls(text=issue_comment_rendered)) + for url in image_urls: + base64_image = get_base64(url=url["url"]) + context = f"## Issue:\n{issue_title}\n\n## Issue Body:\n{issue_body}\n\n## Issue Comments:\n{'\n'.join(issue_comments)}" + description = describe_image(base64_image=base64_image, context=context) + description = f"## {url['alt']}\n\n{description}" + issue_comments.append(description) + create_comment(body=description, base_args=base_args) + # Check out the URLs in the issue body reference_contents: list[str] = [] for url in github_urls: diff --git a/services/github/asset_manager.py b/services/github/asset_manager.py new file mode 100644 index 00000000..31d89995 --- /dev/null +++ b/services/github/asset_manager.py @@ -0,0 +1,28 @@ +from base64 import b64encode +from requests import get, post +from config import GITHUB_API_URL, TIMEOUT, UTF8 +from services.github.create_headers import create_headers +from services.github.github_types import BaseArgs +from utils.handle_exceptions import handle_exceptions + + +@handle_exceptions(default_return_value="", raise_on_error=False) +def get_base64(url: str) -> str: + response = get(url=url, timeout=TIMEOUT) + response.raise_for_status() + base64_image: str = b64encode(response.content).decode(encoding=UTF8) + return base64_image + + +@handle_exceptions(default_return_value="", raise_on_error=False) +def render_text(base_args: BaseArgs, text: str) -> str: + """https://docs.github.com/en/rest/markdown/markdown?apiVersion=2022-11-28#render-a-markdown-document""" + owner = base_args["owner"] + repo = base_args["repo"] + token = base_args["token"] + url = f"{GITHUB_API_URL}/markdown" + headers = create_headers(token=token) + body = {"text": text, "mode": "gfm", "context": f"{owner}/{repo}"} + response = post(url=url, headers=headers, json=body, timeout=TIMEOUT) + response.raise_for_status() + return response.text diff --git a/services/github/github_utils.py b/services/github/github_utils.py index 25c088ce..6be24d20 100644 --- a/services/github/github_utils.py +++ b/services/github/github_utils.py @@ -14,7 +14,7 @@ get_installation_access_token, get_user_public_email, ) -from utils.extract_urls import extract_image_urls, extract_urls +from utils.extract_urls import extract_urls from utils.handle_exceptions import handle_exceptions @@ -67,8 +67,6 @@ def deconstruct_github_payload(payload: GitHubLabeledPayload): github_urls, other_urls = extract_urls(text=issue_body) # print(f"github_urls: {github_urls}") # print(f"other_urls: {other_urls}") - image_urls = extract_image_urls(text=issue_body) - # print(f"image_urls: {image_urls}") installation_id: int = payload["installation"]["id"] token: str = get_installation_access_token(installation_id=installation_id) sender_email: str = get_user_public_email(username=sender_name, token=token) @@ -95,7 +93,6 @@ def deconstruct_github_payload(payload: GitHubLabeledPayload): "is_automation": is_automation, "reviewers": reviewers, "github_urls": github_urls, - "image_urls": image_urls, "other_urls": other_urls, } diff --git a/services/openai/instructions/describe_image.py b/services/openai/instructions/describe_image.py index 706e214c..b0e0e0e9 100644 --- a/services/openai/instructions/describe_image.py +++ b/services/openai/instructions/describe_image.py @@ -1 +1,12 @@ -DESCRIBE_IMAGE = "Describe images found in my GitHub repositories. These images often include elements like text, shapes, arrows, red lines, and boxed areas, and may also contain screenshots of customer business services or SaaS interfaces. Extract and describe these elements, noting their positions and relationships, such as connections indicated by arrows or emphasis through red lines and boxes. Provide a comprehensive understanding of the visual and textual content." +DESCRIBE_IMAGE = """Analyze technical images from GitHub repositories with the depth and precision of a senior software engineer, focusing particularly on diagnostic content in issues and tickets. + +Rather than providing broad surface-level observations, focus deeply on the most critical aspects relevant to the context - just as an experienced engineer would prioritize the key technical signals while debugging. + +Key analysis points: +- For network traces/waterfalls: Identify specific bottlenecks, long-running requests, failed calls, and timing anomalies +- For error screenshots: Parse exact error messages, stack traces, and surrounding context that could indicate root causes +- For UI/UX issues: Note specific components affected, state inconsistencies, and visual regressions +- For console outputs: Highlight critical errors, warnings, or unexpected patterns in logs +- For architectural diagrams: Focus on system interactions, potential failure points, and data flow issues + +Provide detailed technical insights that would help debug the issue, not just describe what's visible. Include specific metrics, timings, error codes, and other quantitative data when present.""" diff --git a/services/openai/vision.py b/services/openai/vision.py index bd27781d..4b1a72f8 100644 --- a/services/openai/vision.py +++ b/services/openai/vision.py @@ -10,7 +10,7 @@ @handle_exceptions(default_return_value="", raise_on_error=None) -def describe_image(base64_image: str) -> str: +def describe_image(base64_image: str, context: str | None = None) -> str: """ 1. API doc: https://platform.openai.com/docs/api-reference/chat/create 2. 20MB per image is allowed: https://platform.openai.com/docs/guides/vision/is-there-a-limit-to-the-size-of-the-image-i-can-upload @@ -26,6 +26,16 @@ def describe_image(base64_image: str) -> str: "type": "text", "text": DESCRIBE_IMAGE, }, + *( + [ + { + "type": "text", + "text": context, + } + ] + if context is not None + else [] + ), { "type": "image_url", "image_url": { diff --git a/utils/extract_urls.py b/utils/extract_urls.py index 2f8298d4..8584e05f 100644 --- a/utils/extract_urls.py +++ b/utils/extract_urls.py @@ -3,14 +3,14 @@ @handle_exceptions(default_return_value=[], raise_on_error=False) -def extract_image_urls(text: str) -> list[str]: - """Extract URLs from img tags in the given text. +def extract_image_urls(text: str) -> list[dict[str, str]]: + """Extract alt text and URLs from img tags in the given text. - Example 1: ['https://github.com/user-attachments/assets/123'] + Example: Screenshot 2024-12-12 at 6 25 41 PM """ - pattern = r']*src="([^"]*)"[^>]*>' - urls: list[str] = findall(pattern, text) - return urls + pattern = r']*alt="([^"]*)"[^>]*src="([^"]*)"[^>]*>' + matches = findall(pattern, text) + return [{"alt": alt, "url": url} for alt, url in matches] def extract_urls(text: str) -> tuple[list[str], list[str]]: