yuyr/AgentOccam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
Yao Liu
2025-01-22 11:32:35 -08:00
parent dac181a456
commit 4b8dbfd9c5
298 changed files with 56437 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
Agent_E/ae/core/skills/__init__.py Normal file
View File

@@ -0,0 +1,18 @@
from Agent_E.ae.core.skills.click_using_selector import click
from Agent_E.ae.core.skills.click_using_selector import do_click
from Agent_E.ae.core.skills.click_using_selector import is_element_present
from Agent_E.ae.core.skills.click_using_selector import perform_javascript_click
from Agent_E.ae.core.skills.click_using_selector import perform_playwright_click
from Agent_E.ae.core.skills.enter_text_and_click import enter_text_and_click
from Agent_E.ae.core.skills.enter_text_using_selector import bulk_enter_text
from Agent_E.ae.core.skills.enter_text_using_selector import custom_fill_element
from Agent_E.ae.core.skills.enter_text_using_selector import do_entertext
from Agent_E.ae.core.skills.get_dom_with_content_type import get_dom_with_content_type
from Agent_E.ae.core.skills.get_url import geturl
from Agent_E.ae.core.skills.get_user_input import get_user_input
from Agent_E.ae.core.skills.open_url import openurl
from Agent_E.ae.core.skills.press_key_combination import press_key_combination

217
Agent_E/ae/core/skills/click_using_selector.py Normal file
View File

@@ -0,0 +1,217 @@
import asyncio
import inspect
import traceback
from typing import Annotated
from playwright.async_api import ElementHandle
from playwright.async_api import Page
from Agent_E.ae.core.playwright_manager import PlaywrightManager
from Agent_E.ae.utils.dom_helper import get_element_outer_html
from Agent_E.ae.utils.dom_mutation_observer import subscribe # type: ignore
from Agent_E.ae.utils.dom_mutation_observer import unsubscribe # type: ignore
from Agent_E.ae.utils.logger import logger
from Agent_E.ae.utils.ui_messagetype import MessageType
async def click(selector: Annotated[str, "The properly formed query selector string to identify the element for the click action (e.g. [mmid='114']). When \"mmid\" attribute is present, use it for the query selector."],
wait_before_execution: Annotated[float, "Optional wait time in seconds before executing the click event logic.", float] = 0.0) -> Annotated[str, "A message indicating success or failure of the click."]:
"""
Executes a click action on the element matching the given query selector string within the currently open web page.
If there is no page open, it will raise a ValueError. An optional wait time can be specified before executing the click logic. Use this to wait for the page to load especially when the last action caused the DOM/Page to load.
Parameters:
- selector: The query selector string to identify the element for the click action.
- wait_before_execution: Optional wait time in seconds before executing the click event logic. Defaults to 0.0 seconds.
Returns:
- Success if the click was successful, Appropropriate error message otherwise.
"""
logger.info(f"Executing ClickElement with \"{selector}\" as the selector")
# Initialize PlaywrightManager and get the active browser page
browser_manager = PlaywrightManager(browser_type='chromium', headless=False)
page = await browser_manager.get_current_page()
if page is None: # type: ignore
raise ValueError('No active page found. OpenURL command opens a new page.')
function_name = inspect.currentframe().f_code.co_name # type: ignore
await browser_manager.take_screenshots(f"{function_name}_start", page)
await browser_manager.highlight_element(selector, True)
dom_changes_detected=None
def detect_dom_changes(changes:str): # type: ignore
nonlocal dom_changes_detected
dom_changes_detected = changes # type: ignore
subscribe(detect_dom_changes)
result = await do_click(page, selector, wait_before_execution)
await asyncio.sleep(0.1) # sleep for 100ms to allow the mutation observer to detect changes
unsubscribe(detect_dom_changes)
await browser_manager.take_screenshots(f"{function_name}_end", page)
await browser_manager.notify_user(result["summary_message"], message_type=MessageType.ACTION)
if dom_changes_detected:
return f"Success: {result['summary_message']}.\n As a consequence of this action, new elements have appeared in view: {dom_changes_detected}. This means that the action to click {selector} is not yet executed and needs further interaction. Get all_fields DOM to complete the interaction."
return result["detailed_message"]
async def do_click(page: Page, selector: str, wait_before_execution: float) -> dict[str, str]:
"""
Executes the click action on the element with the given selector within the provided page.
Parameters:
- page: The Playwright page instance.
- selector: The query selector string to identify the element for the click action.
- wait_before_execution: Optional wait time in seconds before executing the click event logic.
Returns:
dict[str,str] - Explanation of the outcome of this operation represented as a dictionary with 'summary_message' and 'detailed_message'.
"""
logger.info(f"Executing ClickElement with \"{selector}\" as the selector. Wait time before execution: {wait_before_execution} seconds.")
# Wait before execution if specified
if wait_before_execution > 0:
await asyncio.sleep(wait_before_execution)
# Wait for the selector to be present and ensure it's attached and visible. If timeout, try javascript click
try:
logger.info(f"Executing ClickElement with \"{selector}\" as the selector. Waiting for the element to be attached and visible.")
element = await asyncio.wait_for(
page.wait_for_selector(selector, state="attached", timeout=2000),
timeout=2000
)
if element is None:
raise ValueError(f"Element with selector: \"{selector}\" not found")
logger.info(f"Element with selector: \"{selector}\" is attached. scrolling it into view if needed.")
try:
await element.scroll_into_view_if_needed(timeout=200)
logger.info(f"Element with selector: \"{selector}\" is attached and scrolled into view. Waiting for the element to be visible.")
except Exception:
# If scrollIntoView fails, just move on, not a big deal
pass
try:
await element.wait_for_element_state("visible", timeout=200)
logger.info(f"Executing ClickElement with \"{selector}\" as the selector. Element is attached and visibe. Clicking the element.")
except Exception:
# If the element is not visible, try to click it anyway
pass
element_tag_name = await element.evaluate("element => element.tagName.toLowerCase()")
element_outer_html = await get_element_outer_html(element, page, element_tag_name)
if element_tag_name == "option":
element_value = await element.get_attribute("value") # get the text that is in the value of the option
parent_element = await element.evaluate_handle("element => element.parentNode")
# await parent_element.evaluate(f"element => element.select_option(value=\"{element_value}\")")
await parent_element.select_option(value=element_value) # type: ignore
logger.info(f'Select menu option "{element_value}" selected')
return {"summary_message": f'Select menu option "{element_value}" selected',
"detailed_message": f'Select menu option "{element_value}" selected. The select element\'s outer HTML is: {element_outer_html}.'}
#Playwright click seems to fail more often than not, disabling it for now and just going with JS click
#await perform_playwright_click(element, selector)
msg = await perform_javascript_click(page, selector)
return {"summary_message": msg, "detailed_message": f"{msg} The clicked element's outer HTML is: {element_outer_html}."} # type: ignore
except Exception as e:
logger.error(f"Unable to click element with selector: \"{selector}\". Error: {e}")
traceback.print_exc()
msg = f"Unable to click element with selector: \"{selector}\" since the selector is invalid. Proceed by retrieving DOM again."
return {"summary_message": msg, "detailed_message": f"{msg}. Error: {e}"}
async def is_element_present(page: Page, selector: str) -> bool:
"""
Checks if an element is present on the page.
Parameters:
- page: The Playwright page instance.
- selector: The query selector string to identify the element.
Returns:
- True if the element is present, False otherwise.
"""
element = await page.query_selector(selector)
return element is not None
async def perform_playwright_click(element: ElementHandle, selector: str):
"""
Performs a click action on the element using Playwright's click method.
Parameters:
- element: The Playwright ElementHandle instance representing the element to be clicked.
- selector: The query selector string of the element.
Returns:
- None
"""
logger.info(f"Performing first Step: Playwright Click on element with selector: {selector}")
await element.click(force=False, timeout=200)
async def perform_javascript_click(page: Page, selector: str):
"""
Performs a click action on the element using JavaScript.
Parameters:
- page: The Playwright page instance.
- selector: The query selector string of the element.
Returns:
- None
"""
js_code = """(selector) => {
let element = document.querySelector(selector);
if (!element) {
console.log(`perform_javascript_click: Element with selector ${selector} not found`);
return `perform_javascript_click: Element with selector ${selector} not found`;
}
if (element.tagName.toLowerCase() === "option") {
let value = element.text;
let parent = element.parentElement;
parent.value = element.value; // Directly set the value if possible
// Trigger change event if necessary
let event = new Event('change', { bubbles: true });
parent.dispatchEvent(event);
console.log("Select menu option", value, "selected");
return "Select menu option: "+ value+ " selected";
}
else {
console.log("About to click selector", selector);
// If the element is a link, make it open in the same tab
if (element.tagName.toLowerCase() === "a") {
element.target = "_self";
}
let ariaExpandedBeforeClick = element.getAttribute('aria-expanded');
element.click();
let ariaExpandedAfterClick = element.getAttribute('aria-expanded');
if (ariaExpandedBeforeClick === 'false' && ariaExpandedAfterClick === 'true') {
return "Executed JavaScript Click on element with selector: "+selector +". Very important: As a consequence a menu has appeared where you may need to make further selction. Very important: Get all_fields DOM to complete the action.";
}
return "Executed JavaScript Click on element with selector: "+selector;
}
}"""
try:
logger.info(f"Executing JavaScript click on element with selector: {selector}")
result:str = await page.evaluate(js_code, selector)
logger.debug(f"Executed JavaScript Click on element with selector: {selector}")
return result
except Exception as e:
logger.error(f"Error executing JavaScript click on element with selector: {selector}. Error: {e}")
traceback.print_exc()

82
Agent_E/ae/core/skills/enter_text_and_click.py Normal file
View File

@@ -0,0 +1,82 @@
import asyncio
import inspect
from typing import Annotated
from Agent_E.ae.core.playwright_manager import PlaywrightManager
from Agent_E.ae.core.skills.click_using_selector import do_click
from Agent_E.ae.core.skills.enter_text_using_selector import do_entertext
from Agent_E.ae.core.skills.press_key_combination import do_press_key_combination
from Agent_E.ae.utils.logger import logger
from Agent_E.ae.utils.ui_messagetype import MessageType
async def enter_text_and_click(
text_selector: Annotated[str, "The properly formatted DOM selector query, for example [mmid='1234'], where the text will be entered. Use mmid attribute."],
text_to_enter: Annotated[str, "The text that will be entered into the element specified by text_selector."],
click_selector: Annotated[str, "The properly formatted DOM selector query, for example [mmid='1234'], for the element that will be clicked after text entry."],
wait_before_click_execution: Annotated[float, "Optional wait time in seconds before executing the click.", float] = 0.0
) -> Annotated[str, "A message indicating success or failure of the text entry and click."]:
"""
Enters text into an element and then clicks on another element.
Parameters:
- text_selector: The selector for the element to enter text into. It should be a properly formatted DOM selector query, for example [mmid='1234'], where the text will be entered. Use the mmid attribute.
- text_to_enter: The text to enter into the element specified by text_selector.
- click_selector: The selector for the element to click. It should be a properly formatted DOM selector query, for example [mmid='1234'].
- wait_before_click_execution: Optional wait time in seconds before executing the click action. Default is 0.0.
Returns:
- A message indicating the success or failure of the text entry and click.
Raises:
- ValueError: If no active page is found. The OpenURL command opens a new page.
Example usage:
```
await enter_text_and_click("[mmid='1234']", "Hello, World!", "[mmid='5678']", wait_before_click_execution=1.5)
```
"""
logger.info(f"Entering text '{text_to_enter}' into element with selector '{text_selector}' and then clicking element with selector '{click_selector}'.")
# Initialize PlaywrightManager and get the active browser page
browser_manager = PlaywrightManager(browser_type='chromium', headless=False)
page = await browser_manager.get_current_page()
if page is None: # type: ignore
logger.error("No active page found")
raise ValueError('No active page found. OpenURL command opens a new page.')
await browser_manager.highlight_element(text_selector, True)
function_name = inspect.currentframe().f_code.co_name # type: ignore
await browser_manager.take_screenshots(f"{function_name}_start", page)
text_entry_result = await do_entertext(page, text_selector, text_to_enter, use_keyboard_fill=True)
#await browser_manager.notify_user(text_entry_result["summary_message"])
if not text_entry_result["summary_message"].startswith("Success"):
await browser_manager.take_screenshots(f"{function_name}_end", page)
return(f"Failed to enter text '{text_to_enter}' into element with selector '{text_selector}'. Check that the selctor is valid.")
result = text_entry_result
#if the text_selector is the same as the click_selector, press the Enter key instead of clicking
if text_selector == click_selector:
do_press_key_combination_result = await do_press_key_combination(browser_manager, page, "Enter")
if do_press_key_combination_result:
result["detailed_message"] += f" Instead of click, pressed the Enter key successfully on element: \"{click_selector}\"."
await browser_manager.notify_user(f"Pressed the Enter key successfully on element: \"{click_selector}\".", message_type=MessageType.ACTION)
else:
result["detailed_message"] += f" Clicking the same element after entering text in it, is of no value. Tried pressing the Enter key on element \"{click_selector}\" instead of click and failed."
await browser_manager.notify_user("Failed to press the Enter key on element \"{click_selector}\".", message_type=MessageType.ACTION)
else:
await browser_manager.highlight_element(click_selector, True)
do_click_result = await do_click(page, click_selector, wait_before_click_execution)
result["detailed_message"] += f' {do_click_result["detailed_message"]}'
#await browser_manager.notify_user(do_click_result["summary_message"])
await asyncio.sleep(0.1) # sleep for 100ms to allow the mutation observer to detect changes
await browser_manager.take_screenshots(f"{function_name}_end", page)
return result["detailed_message"]

263
Agent_E/ae/core/skills/enter_text_using_selector.py Normal file
View File

@@ -0,0 +1,263 @@
import asyncio
import inspect
import traceback
from dataclasses import dataclass
from typing import Annotated
from typing import List # noqa: UP035
from playwright.async_api import Page
from Agent_E.ae.core.playwright_manager import PlaywrightManager
from Agent_E.ae.core.skills.press_key_combination import press_key_combination
from Agent_E.ae.utils.dom_helper import get_element_outer_html
from Agent_E.ae.utils.dom_mutation_observer import subscribe
from Agent_E.ae.utils.dom_mutation_observer import unsubscribe
from Agent_E.ae.utils.logger import logger
from Agent_E.ae.utils.ui_messagetype import MessageType
@dataclass
class EnterTextEntry:
"""
Represents an entry for text input.
Attributes:
query_selector (str): A valid DOM selector query. Use the mmid attribute.
text (str): The text to enter in the element identified by the query_selector.
"""
query_selector: str
text: str
def __getitem__(self, key: str) -> str:
if key == "query_selector":
return self.query_selector
elif key == "text":
return self.text
else:
raise KeyError(f"{key} is not a valid key")
async def custom_fill_element(page: Page, selector: str, text_to_enter: str):
"""
Sets the value of a DOM element to a specified text without triggering keyboard input events.
This function directly sets the 'value' property of a DOM element identified by the given CSS selector,
effectively changing its current value to the specified text. This approach bypasses the need for
simulating keyboard typing, providing a more efficient and reliable way to fill in text fields,
especially in automated testing scenarios where speed and accuracy are paramount.
Args:
page (Page): The Playwright Page object representing the browser tab in which the operation will be performed.
selector (str): The CSS selector string used to locate the target DOM element. The function will apply the
text change to the first element that matches this selector.
text_to_enter (str): The text value to be set in the target element. Existing content will be overwritten.
Example:
await custom_fill_element(page, '#username', 'test_user')
Note:
This function does not trigger input-related events (like 'input' or 'change'). If application logic
relies on these events being fired, additional steps may be needed to simulate them.
"""
selector = f"{selector}" # Ensures the selector is treated as a string
try:
result = await page.evaluate(
"""(inputParams) => {
const selector = inputParams.selector;
let text_to_enter = inputParams.text_to_enter;
text_to_enter = text_to_enter.trim();
const element = document.querySelector(selector);
if (!element) {
throw new Error(`Element not found: ${selector}`);
}
element.value = text_to_enter;
return `Value set for ${selector}`;
}""",
{"selector": selector, "text_to_enter": text_to_enter},
)
logger.debug(f"custom_fill_element result: {result}")
except Exception as e:
logger.error(f"Error in custom_fill_element, Selector: {selector}, Text: {text_to_enter}. Error: {str(e)}")
raise
async def entertext(entry: Annotated[EnterTextEntry, "An object containing 'query_selector' (DOM selector query using mmid attribute e.g. [mmid='114']) and 'text' (text to enter on the element)."]) -> Annotated[str, "Explanation of the outcome of this operation."]:
"""
Enters text into a DOM element identified by a CSS selector.
This function enters the specified text into a DOM element identified by the given CSS selector.
It uses the Playwright library to interact with the browser and perform the text entry operation.
The function supports both direct setting of the 'value' property and simulating keyboard typing.
Args:
entry (EnterTextEntry): An object containing 'query_selector' (DOM selector query using mmid attribute)
and 'text' (text to enter on the element).
Returns:
str: Explanation of the outcome of this operation.
Example:
entry = EnterTextEntry(query_selector='#username', text='test_user')
result = await entertext(entry)
Note:
- The 'query_selector' should be a valid CSS selector that uniquely identifies the target element.
- The 'text' parameter specifies the text to be entered into the element.
- The function uses the PlaywrightManager to manage the browser instance.
- If no active page is found, an error message is returned.
- The function internally calls the 'do_entertext' function to perform the text entry operation.
- The 'do_entertext' function applies a pulsating border effect to the target element during the operation.
- The 'use_keyboard_fill' parameter in 'do_entertext' determines whether to simulate keyboard typing or not.
- If 'use_keyboard_fill' is set to True, the function uses the 'page.keyboard.type' method to enter the text.
- If 'use_keyboard_fill' is set to False, the function uses the 'custom_fill_element' method to enter the text.
"""
logger.info(f"Entering text: {entry}")
query_selector: str = entry['query_selector']
text_to_enter: str = entry['text']
# Create and use the PlaywrightManager
browser_manager = PlaywrightManager(browser_type='chromium', headless=False)
page = await browser_manager.get_current_page()
if page is None: # type: ignore
return "Error: No active page found. OpenURL command opens a new page."
function_name = inspect.currentframe().f_code.co_name # type: ignore
await browser_manager.take_screenshots(f"{function_name}_start", page)
await browser_manager.highlight_element(query_selector, True)
dom_changes_detected=None
def detect_dom_changes(changes:str): # type: ignore
nonlocal dom_changes_detected
dom_changes_detected = changes # type: ignore
subscribe(detect_dom_changes)
await page.evaluate(
"""
(selector) => {
const element = document.querySelector(selector);
if (element) {
element.value = '';
} else {
console.error('Element not found:', selector);
}
}
""",
query_selector,
)
result = await do_entertext(page, query_selector, text_to_enter)
await asyncio.sleep(0.1) # sleep for 100ms to allow the mutation observer to detect changes
unsubscribe(detect_dom_changes)
await browser_manager.take_screenshots(f"{function_name}_end", page)
await browser_manager.notify_user(result["summary_message"], message_type=MessageType.ACTION)
if dom_changes_detected:
return f"{result['detailed_message']}.\n As a consequence of this action, new elements have appeared in view: {dom_changes_detected}. This means that the action of entering text {text_to_enter} is not yet executed and needs further interaction. Get all_fields DOM to complete the interaction."
return result["detailed_message"]
async def do_entertext(page: Page, selector: str, text_to_enter: str, use_keyboard_fill: bool=True):
"""
Performs the text entry operation on a DOM element.
This function performs the text entry operation on a DOM element identified by the given CSS selector.
It applies a pulsating border effect to the element during the operation for visual feedback.
The function supports both direct setting of the 'value' property and simulating keyboard typing.
Args:
page (Page): The Playwright Page object representing the browser tab in which the operation will be performed.
selector (str): The CSS selector string used to locate the target DOM element.
text_to_enter (str): The text value to be set in the target element. Existing content will be overwritten.
use_keyboard_fill (bool, optional): Determines whether to simulate keyboard typing or not.
Defaults to False.
Returns:
dict[str, str]: Explanation of the outcome of this operation represented as a dictionary with 'summary_message' and 'detailed_message'.
Example:
result = await do_entertext(page, '#username', 'test_user')
Note:
- The 'use_keyboard_fill' parameter determines whether to simulate keyboard typing or not.
- If 'use_keyboard_fill' is set to True, the function uses the 'page.keyboard.type' method to enter the text.
- If 'use_keyboard_fill' is set to False, the function uses the 'custom_fill_element' method to enter the text.
"""
try:
logger.debug(f"Looking for selector {selector} to enter text: {text_to_enter}")
elem = await page.query_selector(selector)
if elem is None:
error = f"Error: Selector {selector} not found. Unable to continue."
return {"summary_message": error, "detailed_message": error}
logger.info(f"Found selector {selector} to enter text")
element_outer_html = await get_element_outer_html(elem, page)
if use_keyboard_fill:
await elem.focus()
await asyncio.sleep(0.1)
await press_key_combination("Control+A")
await asyncio.sleep(0.1)
await press_key_combination("Backspace")
await asyncio.sleep(0.1)
logger.debug(f"Focused element with selector {selector} to enter text")
#add a 100ms delay
await page.keyboard.type(text_to_enter, delay=1)
else:
await custom_fill_element(page, selector, text_to_enter)
await elem.focus()
logger.info(f"Success. Text \"{text_to_enter}\" set successfully in the element with selector {selector}")
success_msg = f"Success. Text \"{text_to_enter}\" set successfully in the element with selector {selector}"
return {"summary_message": success_msg, "detailed_message": f"{success_msg} and outer HTML: {element_outer_html}."}
except Exception as e:
traceback.print_exc()
error = f"Error entering text in selector {selector}."
return {"summary_message": error, "detailed_message": f"{error} Error: {e}"}
async def bulk_enter_text(
entries: Annotated[List[dict[str, str]], "List of objects, each containing 'query_selector' and 'text'."] # noqa: UP006
) -> Annotated[List[dict[str, str]], "List of dictionaries, each containing 'query_selector' and the result of the operation."]: # noqa: UP006
"""
Enters text into multiple DOM elements using a bulk operation.
This function enters text into multiple DOM elements using a bulk operation.
It takes a list of dictionaries, where each dictionary contains a 'query_selector' and 'text' pair.
The function internally calls the 'entertext' function to perform the text entry operation for each entry.
Args:
entries: List of objects, each containing 'query_selector' and 'text'.
Returns:
List of dictionaries, each containing 'query_selector' and the result of the operation.
Example:
entries = [
{"query_selector": "#username", "text": "test_user"},
{"query_selector": "#password", "text": "test_password"}
]
results = await bulk_enter_text(entries)
Note:
- Each entry in the 'entries' list should be a dictionary with 'query_selector' and 'text' keys.
- The result is a list of dictionaries, where each dictionary contains the 'query_selector' and the result of the operation.
"""
results: List[dict[str, str]] = [] # noqa: UP006
logger.info("Executing bulk Enter Text Command")
for entry in entries:
query_selector = entry['query_selector']
text_to_enter = entry['text']
logger.info(f"Entering text: {text_to_enter} in element with selector: {query_selector}")
result = await entertext(EnterTextEntry(query_selector=query_selector, text=text_to_enter))
results.append({"query_selector": query_selector, "result": result})
return results

115
Agent_E/ae/core/skills/get_dom_with_content_type.py Normal file
View File

@@ -0,0 +1,115 @@
import os
import time
from typing import Annotated
from typing import Any
from playwright.async_api import Page
from Agent_E.ae.config import SOURCE_LOG_FOLDER_PATH
from Agent_E.ae.core.playwright_manager import PlaywrightManager
from Agent_E.ae.utils.dom_helper import wait_for_non_loading_dom_state
from Agent_E.ae.utils.get_detailed_accessibility_tree import do_get_accessibility_info
from Agent_E.ae.utils.logger import logger
from Agent_E.ae.utils.ui_messagetype import MessageType
async def get_dom_with_content_type(
content_type: Annotated[str, "The type of content to extract: 'text_only': Extracts the innerText of the highest element in the document and responds with text, or 'input_fields': Extracts the text input and button elements in the dom."]
) -> Annotated[dict[str, Any] | str | None, "The output based on the specified content type."]:
"""
Retrieves and processes the DOM of the active page in a browser instance based on the specified content type.
Parameters
----------
content_type : str
The type of content to extract. Possible values are:
- 'text_only': Extracts the innerText of the highest element in the document and responds with text.
- 'input_fields': Extracts the text input and button elements in the DOM and responds with a JSON object.
- 'all_fields': Extracts all the fields in the DOM and responds with a JSON object.
Returns
-------
dict[str, Any] | str | None
The processed content based on the specified content type. This could be:
- A JSON object for 'input_fields' with just inputs.
- Plain text for 'text_only'.
- A minified DOM represented as a JSON object for 'all_fields'.
Raises
------
ValueError
If an unsupported content_type is provided.
"""
logger.info(f"Executing Get DOM Command based on content_type: {content_type}")
start_time = time.time()
# Create and use the PlaywrightManager
browser_manager = PlaywrightManager(browser_type='chromium', headless=False)
page = await browser_manager.get_current_page()
if page is None: # type: ignore
raise ValueError('No active page found. OpenURL command opens a new page.')
extracted_data = None
await wait_for_non_loading_dom_state(page, 2000) # wait for the DOM to be ready, non loading means external resources do not need to be loaded
user_success_message = ""
if content_type == 'all_fields':
user_success_message = "Fetched all the fields in the DOM"
extracted_data = await do_get_accessibility_info(page, only_input_fields=False)
elif content_type == 'input_fields':
logger.debug('Fetching DOM for input_fields')
extracted_data = await do_get_accessibility_info(page, only_input_fields=True)
if extracted_data is None:
return "Could not fetch input fields. Please consider trying with content_type all_fields."
user_success_message = "Fetched only input fields in the DOM"
elif content_type == 'text_only':
# Extract text from the body or the highest-level element
logger.debug('Fetching DOM for text_only')
text_content = await get_filtered_text_content(page)
with open(os.path.join(SOURCE_LOG_FOLDER_PATH, 'text_only_dom.txt'), 'w', encoding='utf-8') as f:
f.write(text_content)
extracted_data = text_content
user_success_message = "Fetched the text content of the DOM"
else:
raise ValueError(f"Unsupported content_type: {content_type}")
elapsed_time = time.time() - start_time
logger.info(f"Get DOM Command executed in {elapsed_time} seconds")
await browser_manager.notify_user(user_success_message, message_type=MessageType.ACTION)
return extracted_data # type: ignore
async def get_filtered_text_content(page: Page) -> str:
text_content = await page.evaluate("""
() => {
// Array of query selectors to filter out
const selectorsToFilter = ['#agente-overlay'];
// Store the original visibility values to revert later
const originalStyles = [];
// Hide the elements matching the query selectors
selectorsToFilter.forEach(selector => {
const elements = document.querySelectorAll(selector);
elements.forEach(element => {
originalStyles.push({ element: element, originalStyle: element.style.visibility });
element.style.visibility = 'hidden';
});
});
// Get the text content of the page
let textContent = document?.body?.innerText || document?.documentElement?.innerText || "";
// Get all the alt text from images on the page
let altTexts = Array.from(document.querySelectorAll('img')).map(img => img.alt);
altTexts="Other Alt Texts in the page: " + altTexts.join(' ');
// Revert the visibility changes
originalStyles.forEach(entry => {
entry.element.style.visibility = entry.originalStyle;
});
textContent=textContent+" "+altTexts;
return textContent;
}
""")
return text_content

40
Agent_E/ae/core/skills/get_url.py Normal file
View File

@@ -0,0 +1,40 @@
from typing import Annotated
from Agent_E.ae.core.playwright_manager import PlaywrightManager
async def geturl() -> Annotated[str, "Returns the full URL of the current active web site/page."]:
"""
Returns the full URL of the current page
Parameters:
Returns:
- Full URL the browser's active page.
"""
try:
# Create and use the PlaywrightManager
browser_manager = PlaywrightManager(browser_type='chromium', headless=False)
page = await browser_manager.get_current_page()
if not page:
raise ValueError('No active page found. OpenURL command opens a new page.')
await page.wait_for_load_state("domcontentloaded")
# Get the URL of the current page
try:
title = await page.title()
current_url = page.url
if len(current_url) >250:
current_url = current_url[:250] + "..."
return f"Current Page: {current_url}, Title: {title}" # type: ignore
except: # noqa: E722
current_url = page.url
return f"Current Page: {current_url}"
except Exception as e:
raise ValueError('No active page found. OpenURL command opens a new page.') from e

26
Agent_E/ae/core/skills/get_user_input.py Normal file
View File

@@ -0,0 +1,26 @@
from typing import Annotated
from typing import List # noqa: UP035
from Agent_E.ae.core.playwright_manager import PlaywrightManager
from Agent_E.ae.utils.cli_helper import answer_questions_over_cli
async def get_user_input(questions: Annotated[List[str], "List of questions to ask the user each one represented as a string"] ) -> dict[str, str]: # noqa: UP006
"""
Asks the user a list of questions and returns the answers in a dictionary.
Parameters:
- questions: A list of questions to ask the user ["What is Username?", "What is your password?"].
Returns:
- Newline separated list of questions to ask the user
"""
answers: dict[str, str] = {}
browser_manager = PlaywrightManager(browser_type='chromium', headless=False)
if browser_manager.ui_manager:
for question in questions:
answers[question] = await browser_manager.prompt_user(f"Question: {question}")
else:
answers = await answer_questions_over_cli(questions)
return answers

70
Agent_E/ae/core/skills/open_url.py Normal file
View File

@@ -0,0 +1,70 @@
import inspect
from typing import Annotated
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
from Agent_E.ae.core.playwright_manager import PlaywrightManager
from Agent_E.ae.utils.logger import logger
from Agent_E.ae.utils.ui_messagetype import MessageType
async def openurl(url: Annotated[str, "The URL to navigate to. Value must include the protocol (http:// or https://)."],
timeout: Annotated[int, "Additional wait time in seconds after initial load."] = 3) -> Annotated[str, "Returns the result of this request in text form"]:
"""
Opens a specified URL in the active browser instance. Waits for an initial load event, then waits for either
the 'domcontentloaded' event or a configurable timeout, whichever comes first.
Parameters:
- url: The URL to navigate to.
- timeout: Additional time in seconds to wait after the initial load before considering the navigation successful.
Returns:
- URL of the new page.
"""
logger.info(f"Opening URL: {url}")
browser_manager = PlaywrightManager(browser_type='chromium', headless=False)
await browser_manager.get_browser_context()
page = await browser_manager.get_current_page()
try:
url = ensure_protocol(url)
if page.url == url:
logger.info(f"Current page URL is the same as the new URL: {url}. No need to refresh.")
title = await page.title()
return f"Page already loaded: {url}, Title: {title}" # type: ignore
# Navigate to the URL with a short timeout to ensure the initial load starts
function_name = inspect.currentframe().f_code.co_name # type: ignore
await browser_manager.take_screenshots(f"{function_name}_start", page)
await page.goto(url, timeout=timeout*1000) # type: ignore
except PlaywrightTimeoutError as pte:
logger.warn(f"Initial navigation to {url} failed: {pte}. Will try to continue anyway.") # happens more often than not, but does not seem to be a problem
except Exception as e:
logger.error(f"An error occurred while opening the URL: {url}. Error: {e}")
import traceback
traceback.print_exc()
await browser_manager.take_screenshots(f"{function_name}_end", page)
await browser_manager.notify_user(f"Opened URL: {url}", message_type=MessageType.ACTION)
# Get the page title
title = await page.title()
url=page.url
return f"Page loaded: {url}, Title: {title}" # type: ignore
def ensure_protocol(url: str) -> str:
"""
Ensures that a URL has a protocol (http:// or https://). If it doesn't have one,
https:// is added by default.
Parameters:
- url: The URL to check and modify if necessary.
Returns:
- A URL string with a protocol.
"""
if not url.startswith(('http://', 'https://')):
url = 'https://' + url # Default to http if no protocol is specified
logger.info(f"Added 'https://' protocol to URL because it was missing. New URL is: {url}")
return url

88
Agent_E/ae/core/skills/pdf_text_extractor.py Normal file
View File

@@ -0,0 +1,88 @@
import os
from typing import Annotated
import httpx
import pdfplumber
from Agent_E.ae.config import PROJECT_TEMP_PATH
from Agent_E.ae.core.playwright_manager import PlaywrightManager
from Agent_E.ae.utils.logger import logger
from Agent_E.ae.utils.ui_messagetype import MessageType
async def extract_text_from_pdf(pdf_url: Annotated[str, "The URL of the PDF file to extract text from."]) -> Annotated[str, "All the text found in the PDF file."]:
"""
Extract text from a PDF file.
pdf_url: str - The URL of the PDF file to extract text from.
returns: str - All the text found in the PDF.
"""
file_path = os.path.join(PROJECT_TEMP_PATH, "downloaded_file.pdf") # fixed file path for downloading the PDF
try:
# Create and use the PlaywrightManager
browser_manager = PlaywrightManager(browser_type='chromium', headless=False)
# Download the PDF
download_result = await download_pdf(pdf_url, file_path)
if not os.path.exists(download_result):
return download_result # Return error message if download failed
# Open the PDF using pdfplumber and extract text
text = ""
with pdfplumber.open(download_result) as pdf:
for page in pdf.pages:
page_text = page.extract_text()
if page_text:
text += page_text + "\n"
extracted_text = text.strip()
word_count = len(extracted_text.split())
await browser_manager.notify_user(f"Extracted text from the PDF successfully. Found {word_count} words.", message_type=MessageType.ACTION)
return "Text found in the PDF:\n" + extracted_text
except httpx.HTTPStatusError as e:
logger.error(f"An error occurred while downloading the PDF from {pdf_url}: {str(e)}")
return f"An error occurred while downloading the PDF: {str(e)}"
except Exception as e:
logger.error(f"An error occurred while extracting text from the PDF that was downloaded from {pdf_url}: {str(e)}")
return f"An error occurred while extracting text: {str(e)}"
finally:
# Cleanup: Ensure the downloaded file is removed
cleanup_temp_files(file_path)
def cleanup_temp_files(*file_paths: str) -> None:
"""
Remove the specified temporary files.
*file_paths: str - One or more file paths to be removed.
"""
for file_path in file_paths:
if os.path.exists(file_path):
try:
os.remove(file_path)
logger.debug(f"Cleaned file from the filesystem: {file_path}")
except Exception as e:
logger.error(f"Failed to remove {file_path}: {str(e)}")
else:
logger.debug(f"File not found. Unable to clean it from the filesystem: {file_path}")
async def download_pdf(pdf_url: str, file_path: str) -> str:
"""
Download the PDF file from the given URL and save it to the specified path.
pdf_url: str - The URL of the PDF file to download.
file_path: str - The local path to save the downloaded PDF.
returns: str - The file path of the downloaded PDF if successful, otherwise an error message.
raises: Exception - If an error occurs during the download process.
"""
try:
logger.info(f"Downloading PDF from: {pdf_url} to: {file_path}")
async with httpx.AsyncClient() as client:
response = await client.get(pdf_url)
response.raise_for_status() # Ensure the request was successful
with open(file_path, 'wb') as pdf_file:
pdf_file.write(response.content)
return file_path
# except httpx.HTTPStatusError as e:
# raise e
except Exception as e:
raise e

111
Agent_E/ae/core/skills/press_key_combination.py Normal file
View File

@@ -0,0 +1,111 @@
import asyncio
import inspect
from typing import Annotated
from playwright.async_api import Page # type: ignore
from Agent_E.ae.core.playwright_manager import PlaywrightManager
from Agent_E.ae.utils.dom_mutation_observer import subscribe # type: ignore
from Agent_E.ae.utils.dom_mutation_observer import unsubscribe # type: ignore
from Agent_E.ae.utils.logger import logger
from Agent_E.ae.utils.ui_messagetype import MessageType
async def press_key_combination(key_combination: Annotated[str, "The key to press, e.g., Enter, PageDown etc"]) -> str:
"""
Presses a key combination on the current active page managed by PlaywrightManager.
This function simulates the pressing of a key or a combination of keys on the current active web page.
The `key_combination` should be a string that represents the keys to be pressed, separated by '+' if it's a combination.
For example, 'Control+C' to copy or 'Alt+F4' to close a window on Windows.
Parameters:
- key_combination (Annotated[str, "The key combination to press, e.g., 'Control+C'."]): The key combination to press, represented as a string. For combinations, use '+' as a separator.
Raises:
- ValueError: If no active page is found.
Returns:
str: status of the operation expressed as a string
"""
logger.info(f"Executing press_key_combination with key combo: {key_combination}")
# Create and use the PlaywrightManager
browser_manager = PlaywrightManager()
page = await browser_manager.get_current_page()
if page is None: # type: ignore
raise ValueError('No active page found. OpenURL command opens a new page.')
# Split the key combination if it's a combination of keys
keys = key_combination.split('+')
dom_changes_detected=None
def detect_dom_changes(changes:str): # type: ignore
nonlocal dom_changes_detected
dom_changes_detected = changes # type: ignore
subscribe(detect_dom_changes)
# If it's a combination, hold down the modifier keys
for key in keys[:-1]: # All keys except the last one are considered modifier keys
await page.keyboard.down(key)
# Press the last key in the combination
await page.keyboard.press(keys[-1])
# Release the modifier keys
for key in keys[:-1]:
await page.keyboard.up(key)
await asyncio.sleep(0.1) # sleep for 100ms to allow the mutation observer to detect changes
unsubscribe(detect_dom_changes)
if dom_changes_detected:
return f"Key {key_combination} executed successfully.\n As a consequence of this action, new elements have appeared in view:{dom_changes_detected}. This means that the action is not yet executed and needs further interaction. Get all_fields DOM to complete the interaction."
await browser_manager.notify_user(f"Key {key_combination} executed successfully", message_type=MessageType.ACTION)
return f"Key {key_combination} executed successfully"
async def do_press_key_combination(browser_manager: PlaywrightManager, page: Page, key_combination: str) -> bool:
"""
Presses a key combination on the provided page.
This function simulates the pressing of a key or a combination of keys on a web page.
The `key_combination` should be a string that represents the keys to be pressed, separated by '+' if it's a combination.
For example, 'Control+C' to copy or 'Alt+F4' to close a window on Windows.
Parameters:
- browser_manager (PlaywrightManager): The PlaywrightManager instance.
- page (Page): The Playwright page instance.
- key_combination (str): The key combination to press, represented as a string. For combinations, use '+' as a separator.
Returns:
bool: True if success and False if failed
"""
logger.info(f"Executing press_key_combination with key combo: {key_combination}")
try:
function_name = inspect.currentframe().f_code.co_name # type: ignore
await browser_manager.take_screenshots(f"{function_name}_start", page)
# Split the key combination if it's a combination of keys
keys = key_combination.split('+')
# If it's a combination, hold down the modifier keys
for key in keys[:-1]: # All keys except the last one are considered modifier keys
await page.keyboard.down(key)
# Press the last key in the combination
await page.keyboard.press(keys[-1])
# Release the modifier keys
for key in keys[:-1]:
await page.keyboard.up(key)
except Exception as e:
logger.error(f"Error executing press_key_combination \"{key_combination}\": {e}")
return False
await browser_manager.take_screenshots(f"{function_name}_end", page)
return True

29
Agent_E/ae/core/skills/skill_registry.py Normal file
View File

@@ -0,0 +1,29 @@
# skill_registry.py
from collections.abc import Callable
from typing import Any
# Define the type of the functions that will be registered as skills
SkillType = Callable[..., Any]
# Global registry to store private skill functions and their metadata
skill_registry: list[dict[str, Any]] = []
def skill(description: str, name: str|None = None) -> Callable[[SkillType], SkillType]:
"""
Decorator for registering private skills.
Parameters:
- description: A string describing the skill's function.
- name: Optional name to register the skill with. If not provided, the function's name will be used.
Returns:
- A decorator function that registers the skill in the global registry.
"""
def decorator(func: SkillType) -> SkillType:
skill_registry.append({
"name": name if name else func.__name__, # Use provided name or fallback to function name
"func": func,
"description": description
})
return func
return decorator