yuyr/AgentOccam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
Yao Liu
2025-01-22 11:32:35 -08:00
parent dac181a456
commit 4b8dbfd9c5
298 changed files with 56437 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
webarena/agent/prompts/__init__.py Normal file
View File

@@ -0,0 +1 @@
from .prompt_constructor import *

21
webarena/agent/prompts/jsons/new_action_prompt.json Normal file
View File

@@ -0,0 +1,21 @@
{
"intro": "",
"examples": [],
"template": "<html> {html} </html>\n\nYou are a helpful assistant that can assist with web navigation tasks.\nYou are given a simplified html webpage and a task description.\nYour goal is to complete the task. You can use the provided functions below to interact with the current webpage.\n\n#Provided functions:\ndef click(element_id: str) -> None:\n \"\"\"\n Click on the element with the specified id.\n\n Args:\n element_id: The id of the element.\n \"\"\"\n\ndef hover(element_id: str) -> None:\n \"\"\"\n Hover on the element with the specified id.\n\n Args:\n element_id: The id of the element.\n \"\"\"\n\ndef select(element_id: str, option: str) -> None:\n \"\"\"\n Select an option from a dropdown.\n\n Args:\n element_id: The id of the element.\n option: Value of the option to select.\n \"\"\"\n\ndef type_string(element_id: str, content: str, press_enter: bool) -> None:\n \"\"\"\n Type a string into the element with the specified id.\n\n Args:\n element_id: The id of the element.\n content: The string to type.\n press_enter: Whether to press enter after typing the string.\n \"\"\"\n\ndef scroll_page(direction: Literal['up', 'down']) -> None:\n \"\"\"\n Scroll down/up one page.\n\n Args:\n direction: The direction to scroll.\n \"\"\"\n\ndef go(direction: Literal['forward', 'backward']) -> None:\n \"\"\"\n Go forward/backward\n\n Args:\n direction: The direction to go to.\n \"\"\"\n\ndef jump_to(url: str, new_tab: bool) -> None:\n \"\"\"\n Jump to the specified url.\n\n Args:\n url: The url to jump to.\n new_tab: Whether to open the url in a new tab.\n \"\"\"\n\ndef switch_tab(tab_index: int) -> None:\n \"\"\"\n Switch to the specified tab.\n\n Args:\n tab_index: The index of the tab to switch to.\n \"\"\"\n\ndef user_input(message: str) -> str:\n \"\"\"\n Wait for user input.\n\n Args:\n message: The message to display to the user.\n\n Returns: The user input.\n \"\"\"\n\ndef finish(answer: Optional[str]) -> None:\n \"\"\"\n Finish the task (optionally with an answer).\n\n Args:\n answer: The answer to the task.\n \"\"\"\n\n#Previous commands: {previous_action}\n\n#Window tabs: {tabs}\n\n#Current viewport (pages): {position}\n\n#Task: {objective}\n\nYou should output one command to interact to the currrent webpage.\nYou should add a brief comment to your command to explain your reasoning and thinking process.\n",
"finale": "",
"meta_data": {
"observation": "html",
"action_type": "id_html_nasc_tree",
"keywords": [
"url",
"html",
"objective",
"position",
"previous_action",
"tabs"
],
"prompt_constructor": "NewASPromptConstructor",
"answer_phrase": "",
"action_splitter": "#"
}
}

27
webarena/agent/prompts/jsons/p_cot_id_actree_2s.json Normal file
View File

@@ -0,0 +1,27 @@
{
"intro": "You are an autonomous intelligent agent tasked with navigating a web browser. You will be given web-based tasks. These tasks will be accomplished through the use of specific actions you can issue.\n\nHere's the information you'll have:\nThe user's objective: This is the task you're trying to complete.\nThe current web page's accessibility tree: This is a simplified representation of the webpage, providing key information.\nThe current web page's URL: This is the page you're currently navigating.\nThe open tabs: These are the tabs you have open.\nThe previous action: This is the action you just performed. It may be helpful to track your progress.\n\nThe actions you can perform fall into several categories:\n\nPage Operation Actions:\n`click [id]`: This action clicks on an element with a specific id on the webpage.\n`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the \"Enter\" key is pressed after typing unless press_enter_after is set to 0.\n`hover [id]`: Hover over an element with id.\n`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).\n`scroll [direction=down|up]`: Scroll the page up or down.\n\nTab Management Actions:\n`new_tab`: Open a new, empty browser tab.\n`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.\n`close_tab`: Close the currently active tab.\n\nURL Navigation Actions:\n`goto [url]`: Navigate to a specific URL.\n`go_back`: Navigate to the previously viewed page.\n`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).\n\nCompletion Action:\n`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as \"N/A\" in the bracket.\n\nHomepage:\nIf you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.\nhttp://homepage.com/password.html lists all the account name and password for the websites. You can use them to log in to the websites.\n\nTo be successful, it is very important to follow the following rules:\n1. You should only issue an action that is valid given the current observation\n2. You should only issue one action at a time.\n3. You should follow the examples to reason step by step and then issue the next action.\n4. Generate the action in the correct format. Start with a \"In summary, the next action I will perform is\" phrase, followed by action inside ``````. For example, \"In summary, the next action I will perform is ```click [1234]```\".\n5. Issue stop action when you think you have achieved the objective. Don't generate anything after stop.",
"examples": [
[
"OBSERVATION:\n[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'\n\t\t[1749] StaticText '$279.49'\n\t\t[1757] button 'Add to Cart'\n\t\t[1760] button 'Add to Wish List'\n\t\t[1761] button 'Add to Compare'\nURL: http://onestopmarket.com/office-products/office-electronics.html\nOBJECTIVE: What is the price of HP Inkjet Fax Machine\nPREVIOUS ACTION: None",
"Let's think step-by-step. This page list the information of HP Inkjet Fax Machine, which is the product identified in the objective. Its price is $279.49. I think I have achieved the objective. I will issue the stop action with the answer. In summary, the next action I will perform is ```stop [$279.49]```"
],
[
"OBSERVATION:\n[164] textbox 'Search' focused: True required: False\n[171] button 'Go'\n[174] link 'Find directions between two points'\n[212] heading 'Search Results'\n[216] button 'Close'\nURL: http://openstreetmap.org\nOBJECTIVE: Show me the restaurants near CMU\nPREVIOUS ACTION: None",
"Let's think step-by-step. This page has a search box whose ID is [164]. According to the nominatim rule of openstreetmap, I can search for the restaurants near a location by \"restaurants near\". I can submit my typing by pressing the Enter afterwards. In summary, the next action I will perform is ```type [164] [restaurants near CMU] [1]```"
]
],
"template": "OBSERVATION:\n{observation}\nURL: {url}\nOBJECTIVE: {objective}\nPREVIOUS ACTION: {previous_action}",
"meta_data": {
"observation": "accessibility_tree",
"action_type": "id_accessibility_tree",
"keywords": [
"url",
"objective",
"observation",
"previous_action"
],
"prompt_constructor": "CoTPromptConstructor",
"answer_phrase": "In summary, the next action I will perform is",
"action_splitter": "```"
}
}

27
webarena/agent/prompts/jsons/p_cot_id_actree_2s_no_na.json Normal file
View File

@@ -0,0 +1,27 @@
{
"intro": "You are an autonomous intelligent agent tasked with navigating a web browser. You will be given web-based tasks. These tasks will be accomplished through the use of specific actions you can issue.\n\nHere's the information you'll have:\nThe user's objective: This is the task you're trying to complete.\nThe current web page's accessibility tree: This is a simplified representation of the webpage, providing key information.\nThe current web page's URL: This is the page you're currently navigating.\nThe open tabs: These are the tabs you have open.\nThe previous action: This is the action you just performed. It may be helpful to track your progress.\n\nThe actions you can perform fall into several categories:\n\nPage Operation Actions:\n`click [id]`: This action clicks on an element with a specific id on the webpage.\n`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the \"Enter\" key is pressed after typing unless press_enter_after is set to 0.\n`hover [id]`: Hover over an element with id.\n`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).\n`scroll [direction=down|up]`: Scroll the page up or down.\n\nTab Management Actions:\n`new_tab`: Open a new, empty browser tab.\n`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.\n`close_tab`: Close the currently active tab.\n\nURL Navigation Actions:\n`goto [url]`: Navigate to a specific URL.\n`go_back`: Navigate to the previously viewed page.\n`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).\n\nCompletion Action:\n`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket.\n\nHomepage:\nIf you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.\nhttp://homepage.com/password.html lists all the account name and password for the websites. You can use them to log in to the websites.\n\nTo be successful, it is very important to follow the following rules:\n1. You should only issue an action that is valid given the current observation\n2. You should only issue one action at a time.\n3. You should follow the examples to reason step by step and then issue the next action.\n4. Generate the action in the correct format. Start with a \"In summary, the next action I will perform is\" phrase, followed by action inside ``````. For example, \"In summary, the next action I will perform is ```click [1234]```\".\n5. Issue stop action when you think you have achieved the objective. Don't generate anything after stop.",
"examples": [
[
"OBSERVATION:\n[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'\n\t\t[1749] StaticText '$279.49'\n\t\t[1757] button 'Add to Cart'\n\t\t[1760] button 'Add to Wish List'\n\t\t[1761] button 'Add to Compare'\nURL: http://onestopmarket.com/office-products/office-electronics.html\nOBJECTIVE: What is the price of HP Inkjet Fax Machine\nPREVIOUS ACTION: None",
"Let's think step-by-step. This page list the information of HP Inkjet Fax Machine, which is the product identified in the objective. Its price is $279.49. I think I have achieved the objective. I will issue the stop action with the answer. In summary, the next action I will perform is ```stop [$279.49]```"
],
[
"OBSERVATION:\n[164] textbox 'Search' focused: True required: False\n[171] button 'Go'\n[174] link 'Find directions between two points'\n[212] heading 'Search Results'\n[216] button 'Close'\nURL: http://openstreetmap.org\nOBJECTIVE: Show me the restaurants near CMU\nPREVIOUS ACTION: None",
"Let's think step-by-step. This page has a search box whose ID is [164]. According to the nominatim rule of openstreetmap, I can search for the restaurants near a location by \"restaurants near\". I can submit my typing by pressing the Enter afterwards. In summary, the next action I will perform is ```type [164] [restaurants near CMU] [1]```"
]
],
"template": "OBSERVATION:\n{observation}\nURL: {url}\nOBJECTIVE: {objective}\nPREVIOUS ACTION: {previous_action}",
"meta_data": {
"observation": "accessibility_tree",
"action_type": "id_accessibility_tree",
"keywords": [
"url",
"objective",
"observation",
"previous_action"
],
"prompt_constructor": "CoTPromptConstructor",
"answer_phrase": "In summary, the next action I will perform is",
"action_splitter": "```"
}
}

26
webarena/agent/prompts/jsons/p_direct_id_actree_2s.json Normal file
View File

@@ -0,0 +1,26 @@
{
"intro": "You are an autonomous intelligent agent tasked with navigating a web browser. You will be given web-based tasks. These tasks will be accomplished through the use of specific actions you can issue.\n\nHere's the information you'll have:\nThe user's objective: This is the task you're trying to complete.\nThe current web page's accessibility tree: This is a simplified representation of the webpage, providing key information.\nThe current web page's URL: This is the page you're currently navigating.\nThe open tabs: These are the tabs you have open.\nThe previous action: This is the action you just performed. It may be helpful to track your progress.\n\nThe actions you can perform fall into several categories:\n\nPage Operation Actions:\n`click [id]`: This action clicks on an element with a specific id on the webpage.\n`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the \"Enter\" key is pressed after typing unless press_enter_after is set to 0.\n`hover [id]`: Hover over an element with id.\n`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).\n`scroll [direction=down|up]`: Scroll the page up or down.\n\nTab Management Actions:\n`new_tab`: Open a new, empty browser tab.\n`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.\n`close_tab`: Close the currently active tab.\n\nURL Navigation Actions:\n`goto [url]`: Navigate to a specific URL.\n`go_back`: Navigate to the previously viewed page.\n`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).\n\nCompletion Action:\n`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as \"N/A\" in the bracket.\n\nHomepage:\nIf you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.\nhttp://homepage.com/password.html lists all the account name and password for the websites. You can use them to log in to the websites.\n\nTo be successful, it is very important to follow the following rules:\n1. You should only issue an action that is valid given the current observation\n2. You should only issue one action at a time.\n3. Generate the action in the correct format. Always put the action inside a pair of ```. For example, ```click [1234]```.\n5. Issue stop action when you think you have achieved the objective. Don't generate anything after stop.",
"examples": [
[
"OBSERVATION:\n[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'\n\t\t[1749] StaticText '$279.49'\n\t\t[1757] button 'Add to Cart'\n\t\t[1760] button 'Add to Wish List'\n\t\t[1761] button 'Add to Compare'\nURL: http://onestopmarket.com/office-products/office-electronics.html\nOBJECTIVE: What is the price of HP Inkjet Fax Machine\nPREVIOUS ACTION: None",
"```stop [$279.49]```"
],
[
"OBSERVATION:\n[164] textbox 'Search' focused: True required: False\n[171] button 'Go'\n[174] link 'Find directions between two points'\n[212] heading 'Search Results'\n[216] button 'Close'\nURL: http://openstreetmap.org\nOBJECTIVE: Show me the restaurants near CMU\nPREVIOUS ACTION: None",
"```type [164] [restaurants near CMU] [1]```"
]
],
"template": "OBSERVATION:\n{observation}\nURL: {url}\nOBJECTIVE: {objective}\nPREVIOUS ACTION: {previous_action}",
"meta_data": {
"observation": "accessibility_tree",
"action_type": "id_accessibility_tree",
"keywords": [
"url",
"objective",
"observation",
"previous_action"
],
"prompt_constructor": "DirectPromptConstructor",
"action_splitter": "```"
}
}

27
webarena/agent/prompts/jsons/p_direct_id_actree_2s_no_na.json Normal file
View File

@@ -0,0 +1,27 @@
{
"intro": "You are an autonomous intelligent agent tasked with navigating a web browser. You will be given web-based tasks. These tasks will be accomplished through the use of specific actions you can issue.\n\nHere's the information you'll have:\nThe user's objective: This is the task you're trying to complete.\nThe current web page's accessibility tree: This is a simplified representation of the webpage, providing key information.\nThe current web page's URL: This is the page you're currently navigating.\nThe open tabs: These are the tabs you have open.\nThe previous action: This is the action you just performed. It may be helpful to track your progress.\n\nThe actions you can perform fall into several categories:\n\nPage Operation Actions:\n`click [id]`: This action clicks on an element with a specific id on the webpage.\n`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the \"Enter\" key is pressed after typing unless press_enter_after is set to 0.\n`hover [id]`: Hover over an element with id.\n`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).\n`scroll [direction=down|up]`: Scroll the page up or down.\n\nTab Management Actions:\n`new_tab`: Open a new, empty browser tab.\n`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.\n`close_tab`: Close the currently active tab.\n\nURL Navigation Actions:\n`goto [url]`: Navigate to a specific URL.\n`go_back`: Navigate to the previously viewed page.\n`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).\n\nCompletion Action:\n`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket.\n\nHomepage:\nIf you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.\nhttp://homepage.com/password.html lists all the account name and password for the websites. You can use them to log in to the websites.\n\nTo be successful, it is very important to follow the following rules:\n1. You should only issue an action that is valid given the current observation\n2. You should only issue one action at a time.\n4. Generate the action in the correct format, wrap the action inside ``````. For example, ```click [1234]```\".\n5. Issue stop action when you think you have achieved the objective.",
"examples": [
[
"OBSERVATION:\n[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'\n\t\t[1749] StaticText '$279.49'\n\t\t[1757] button 'Add to Cart'\n\t\t[1760] button 'Add to Wish List'\n\t\t[1761] button 'Add to Compare'\nURL: http://onestopmarket.com/office-products/office-electronics.html\nOBJECTIVE: What is the price of HP Inkjet Fax Machine\nPREVIOUS ACTION: None",
"```stop [$279.49]```"
],
[
"OBSERVATION:\n[164] textbox 'Search' focused: True required: False\n[171] button 'Go'\n[174] link 'Find directions between two points'\n[212] heading 'Search Results'\n[216] button 'Close'\nURL: http://openstreetmap.org\nOBJECTIVE: Show me the restaurants near CMU\nPREVIOUS ACTION: None",
"```type [164] [restaurants near CMU] [1]```"
]
],
"template": "OBSERVATION:\n{observation}\nURL: {url}\nOBJECTIVE: {objective}\nPREVIOUS ACTION: {previous_action}",
"meta_data": {
"observation": "accessibility_tree",
"action_type": "id_accessibility_tree",
"keywords": [
"url",
"objective",
"observation",
"previous_action"
],
"prompt_constructor": "CoTPromptConstructor",
"answer_phrase": "In summary, the next action I will perform is",
"action_splitter": "```"
}
}

32
webarena/agent/prompts/jsons/p_direct_id_actree_3s_llama.json Normal file
View File

@@ -0,0 +1,32 @@
{
"intro": "You are an autonomous intelligent agent tasked with navigating a web browser. The actions you can perform fall into several categories:\n\nPage Operation Actions:\n`click [id]`: This action clicks on an element with a specific id on the webpage.\n`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the \"Enter\" key is pressed after typing unless press_enter_after is set to 0.\n`hover [id]`: Hover over an element with id.\n`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).\n`scroll [direction=down|up]`: Scroll the page up or down.\n\nTab Management Actions:\n`new_tab`: Open a new, empty browser tab.\n`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.\n`close_tab`: Close the currently active tab.\n\nURL Navigation Actions:\n`goto [url]`: Navigate to a specific URL.\n`go_back`: Navigate to the previously viewed page.\n`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).\n\nCompletion Action:\n`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket.\n\nHomepage:\nIf you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.\n\nYou can only issue one action at a time",
"examples": [
[
"Observation:\n[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'\n\t[1749] StaticText '$279.49'\n\t[1757] button 'Add to Cart'\n\t[1760] button 'Add to Wish List'\n\t[1761] button 'Add to Compare'\nURL: http://onestopmarket.com/office-products/office-electronics.html\nObjective: What is the price of HP Inkjet Fax Machine\nPrevious action: None",
"```stop [$279.49]```"
],
[
"Observation:\n[164] textbox 'Search' focused: True required: False\n[171] button 'Go'\n[174] link 'Find directions between two points'\n[212] heading 'Search Results'\n[216] button 'Close'\nURL: http://openstreetmap.org\nObjective: Show me the restaurants near CMU\nPrevious action: None",
"```type [164] [restaurants near CMU] [1]```"
],
[
"Observation:\n[2036] button 'Sort by: New' hasPopup: menu expanded: False\n\t[587] link 'US Marine\u2019s adoption of Afghan war orphan voided'\n\t\t[989] time 'March 30, 2023 at 15:03:48 AM UTC'\n\t[602] link 'York student uses AI chatbot to get parking fine revoked'\n\t\t[1025] time 'March 15, 2023 at 7:48:34 AM UTC'\n\t[617] link 'Loveland parents furious after teachers leave, communication lagged during school threat investigation'\n\t\t[1025] time 'March 2, 2023 at 3:46:01 AM UTC'\nURL: http://reddit.com/f/news/new\nObjective: Open the most recent post that was published prior to March 1st.\nPrevious action: None",
"```scroll [down]```"
]
],
"template": "Observation:\n{observation}\nURL: {url}\nObjective: {objective}\nPrevious action: {previous_action}",
"meta_data": {
"observation": "accessibility_tree",
"action_type": "id_accessibility_tree",
"keywords": [
"url",
"objective",
"observation",
"previous_action"
],
"prompt_constructor": "DirectPromptConstructor",
"answer_phrase": "In summary, the next action I will perform is",
"action_splitter": "```",
"force_prefix": "```"
}
}

20
webarena/agent/prompts/jsons/test_prompt.json Normal file
View File

@@ -0,0 +1,20 @@
{
"intro": "",
"examples": [],
"template": "<html> {html} </html>\n\nYou are a helpful assistant that can assist with web navigation tasks.\nYou are given a simplified html webpage and a task description. \nYour goal is to complete the task. You can perform the specified operations below to interact with the webpage.\n\n#Valid operations: - #Click# id: Click on the element with the specified id\n- #Scroll_up#: Scroll up 1 page.\n- #Scroll_down#: Scroll down 1 page.\n- #Go_backward#: Go back to the previous page.\n- #Go_forward#: Go forward to the next page.\n- #Hover# id: Hover over the element with the specified id.\n- #Type# id \"text\": Type in the text at the element with the specified id.\n- #Select# id \"option\": Select the option at the element with the specified id.\n- #Record# \"content\": Mark content that is useful in answering the question.\n- #Answer# \"text\": output the text as the answer to the user.\n- #Exit#: Complete the task and exit the program.\n\n#Current viewport position: {position}\n\n#Previous Operation: {previous_action}\n\n#Task: {objective}\n",
"finale": "\nYour output SHOULD be in the following format:\n#Operation: {Next operation to perform}\n",
"meta_data": {
"observation": "html",
"action_type": "id_html_tree",
"keywords": [
"url",
"html",
"objective",
"position",
"previous_action"
],
"prompt_constructor": "MyPromptConstructor",
"answer_phrase": "",
"action_splitter": "#"
}
}

683
webarena/agent/prompts/prompt_constructor.py Normal file
View File

@@ -0,0 +1,683 @@
import json
import re
from pathlib import Path
from typing import Any, TypedDict
from browser_env import Action, ActionParsingError, Trajectory
from browser_env.env_config import URL_MAPPINGS
from browser_env.utils import StateInfo
from llms import lm_config
from llms.tokenizers import Tokenizer
from llms.utils import APIInput
class Instruction(TypedDict):
"""Instruction for constructing prompt"""
intro: str
examples: list[tuple[str, str]]
template: str
meta_data: dict[str, Any]
class PromptConstructor(object):
def __init__(
self,
instruction_path: str | Path,
lm_config: lm_config.LMConfig,
tokenizer: Tokenizer,
):
self.instruction_path = Path(instruction_path)
self.obs_modality = "text"
self.lm_config = lm_config
instruction = json.load(open(self.instruction_path))
instruction["examples"] = [tuple(e) for e in instruction["examples"]]
self.instruction: Instruction = instruction
self.tokenizer = tokenizer
def get_lm_api_input(
self, intro: str, examples: list[tuple[str, str]], current: str
) -> APIInput:
"""Return the require format for an API"""
message: list[dict[str, str]] | str
if "openai" in self.lm_config.provider:
if self.lm_config.mode == "chat":
message = [{"role": "system", "content": intro}]
for (x, y) in examples:
message.append(
{
"role": "system",
"name": "example_user",
"content": x,
}
)
message.append(
{
"role": "system",
"name": "example_assistant",
"content": y,
}
)
message.append({"role": "user", "content": current})
return message
elif self.lm_config.mode == "completion":
message = f"{intro}\n\n"
message += "Here are a few examples:\n"
for example in examples:
message += f"Observation\n:{example[0]}\n\n"
message += f"Action: {example[1]}\n\n"
message += "Now make prediction given the observation\n\n"
message += f"Observation\n:{current}\n\n"
message += "Action:"
return message
else:
raise ValueError(
f"OpenAI models do not support mode {self.lm_config.mode}"
)
elif "huggingface" in self.lm_config.provider:
# https://huggingface.co/blog/llama2#how-to-prompt-llama-2
# https://github.com/facebookresearch/llama/blob/main/llama/generation.py#L320
if "Llama-2" in self.lm_config.model:
if self.lm_config.mode == "chat":
B_INST, E_INST = "[INST]", "[/INST]"
B_SYS, E_SYS = "<<SYS>>\n", "\n<</SYS>>\n\n"
BOS, EOS = "<s>", "</s>"
# adding the system message to be the starting of the first example
examples = [
(
B_SYS + intro + E_SYS + examples[0][0],
examples[0][1],
)
] + examples[1:]
message = "".join(
[
f"{BOS}{B_INST} {x.strip()} {E_INST} {y.strip()} {EOS}"
for (x, y) in examples
]
)
# add the current observation
message += f"{BOS}{B_INST} {current.strip()} {E_INST} {self.instruction['meta_data'].get('force_prefix', '')}"
return message
else:
raise ValueError("Only chat mode is supported for Llama-2")
else:
raise ValueError(
f"Huggingface models do not support model_tag {self.lm_config.gen_config['model_tag']}"
)
elif "ours" in self.lm_config.provider:
message = f"{intro}\n\n"
message += "Now make prediction given the observation\n\n"
message += f"Observation\n:{current}\n\n"
message += "Action:"
return message
else:
raise NotImplementedError(
f"Provider {self.lm_config.provider} not implemented"
)
def construct(
self,
trajectory: Trajectory,
intent: str,
meta_data: dict[str, Any] = {},
) -> APIInput:
raise NotImplementedError
def map_url_to_real(self, url: str) -> str:
"""Map the urls to their real world counterparts"""
for i, j in URL_MAPPINGS.items():
if i in url:
url = url.replace(i, j)
return url
def map_url_to_local(self, url: str) -> str:
"""Map the urls to their local counterparts"""
for i, j in URL_MAPPINGS.items():
if j in url:
url = url.replace(j, i)
# https
if j.replace("http", "https") in url:
url = url.replace(j.replace("http", "https"), i)
return url
def _extract_action(self, response: str) -> str:
raise NotImplementedError
def extract_action(self, response: str) -> str:
response = self._extract_action(response)
response = self.map_url_to_local(response)
return response
class DirectPromptConstructor(PromptConstructor):
"""The agent will direct predict the action"""
def __init__(
self,
instruction_path: str | Path,
lm_config: lm_config.LMConfig,
tokenizer: Tokenizer,
):
super().__init__(instruction_path, lm_config, tokenizer)
def construct(
self,
trajectory: Trajectory,
intent: str,
meta_data: dict[str, Any] = {},
) -> APIInput:
"""Construct prompt given the trajectory"""
intro = self.instruction["intro"]
examples = self.instruction["examples"]
template = self.instruction["template"]
keywords = self.instruction["meta_data"]["keywords"]
state_info: StateInfo = trajectory[-1] # type: ignore[assignment]
obs = state_info["observation"][self.obs_modality]
max_obs_length = self.lm_config.gen_config["max_obs_length"]
if max_obs_length:
obs = self.tokenizer.decode(self.tokenizer.encode(obs)[:max_obs_length]) # type: ignore[arg-type]
page = state_info["info"]["page"]
url = page.url
previous_action_str = meta_data["action_history"][-1]
# input x
current = template.format(
objective=intent,
url=self.map_url_to_real(url),
observation=obs,
previous_action=previous_action_str,
)
# make sure all keywords are replaced
assert all([f"{{k}}" not in current for k in keywords])
prompt = self.get_lm_api_input(intro, examples, current)
return prompt
def _extract_action(self, response: str) -> str:
action_splitter = self.instruction["meta_data"]["action_splitter"]
pattern = rf"{action_splitter}((.|\n)*?){action_splitter}"
match = re.search(pattern, response)
if match:
return match.group(1).strip()
else:
raise ActionParsingError(
f"Cannot parse action from response {response}"
)
class CoTPromptConstructor(PromptConstructor):
"""The agent will perform step-by-step reasoning before the answer"""
def __init__(
self,
instruction_path: str | Path,
lm_config: lm_config.LMConfig,
tokenizer: Tokenizer,
):
super().__init__(instruction_path, lm_config, tokenizer)
self.answer_phrase = self.instruction["meta_data"]["answer_phrase"]
def construct(
self,
trajectory: Trajectory,
intent: str,
meta_data: dict[str, Any] = {},
) -> APIInput:
intro = self.instruction["intro"]
examples = self.instruction["examples"]
template = self.instruction["template"]
keywords = self.instruction["meta_data"]["keywords"]
state_info: StateInfo = trajectory[-1] # type: ignore[assignment]
obs = state_info["observation"][self.obs_modality]
max_obs_length = self.lm_config.gen_config["max_obs_length"]
if max_obs_length:
obs = self.tokenizer.decode(self.tokenizer.encode(obs)[:max_obs_length]) # type: ignore[arg-type]
page = state_info["info"]["page"]
url = page.url
previous_action_str = meta_data["action_history"][-1]
current = template.format(
objective=intent,
url=self.map_url_to_real(url),
observation=obs,
previous_action=previous_action_str,
)
assert all([f"{{k}}" not in current for k in keywords])
prompt = self.get_lm_api_input(intro, examples, current)
return prompt
def _extract_action(self, response: str) -> str:
# find the first occurence of action
action_splitter = self.instruction["meta_data"]["action_splitter"]
pattern = rf"{action_splitter}((.|\n)*?){action_splitter}"
match = re.search(pattern, response)
if match:
return match.group(1).strip()
else:
raise ActionParsingError(
f'Cannot find the answer phrase "{self.answer_phrase}" in "{response}"'
)
class MyPromptConstructor(PromptConstructor):
"""The agent will perform step-by-step reasoning before the answer"""
operation = [
r"#?(Click)#?\s*([0-9]+)",
r"#?(Type)#?\s*([0-9]+)\s+[\'\"]{0,1}([\s\S]+)[\'\"]{0,1}",
r"#?(Select)#?\s*([0-9]+)\s+[\'\"]{0,1}(.+)[\'\"]{0,1}",
r"#?(Scroll_up)#?",
r"#?(Scroll_down)#?",
r"#?(Goto)#?\s*(https?:\/\/[-a-z0-9]+(?:\.[-a-z0-9]+)*\.(?:com|cn|edu|uk)(?:\/[-a-z0-9_:@&?=+,.!/~*'%$]*)?)",
r"#?(Go_backward)#?",
r"#?(Go_forward)#?",
r"#?(Hover)#?\s*([0-9]+)",
r"#?(Answer)#?\s+(.+)",
r"#?(Login)#?",
r"#?(Verify)#?",
r"#?(Exit)#?",
r"#?(Record)#?\s+[\'\"]{0,1}(.+)[\'\"]{0,1}",
]
translate = [
"click",
"type",
"select",
"scroll [up]",
"scroll [down]",
"goto",
"go_back",
"go_forward",
"hover",
"stop",
"stop",
"stop",
"stop",
"record",
]
def __init__(
self,
instruction_path: str | Path,
lm_config: lm_config.LMConfig,
tokenizer: Tokenizer,
):
super().__init__(instruction_path, lm_config, tokenizer)
self.answer_phrase = self.instruction["meta_data"]["answer_phrase"]
self.state = {}
def construct(
self,
trajectory: Trajectory,
intent: str,
meta_data: dict[str, Any] = {},
) -> APIInput:
intro = self.instruction["intro"]
examples = self.instruction["examples"]
template = self.instruction["template"]
keywords = self.instruction["meta_data"]["keywords"]
finale = self.instruction["finale"]
state_info: StateInfo = trajectory[-1] # type: ignore[assignment]
obs = state_info["observation"][self.obs_modality]
max_obs_length = self.lm_config.gen_config["max_obs_length"]
if max_obs_length:
obs = self.tokenizer.decode(self.tokenizer.encode(obs)[:max_obs_length]) # type: ignore[arg-type]
info = state_info["info"]
obs_metadata = info["observation_metadata"]["text"]
nodes = obs_metadata["obs_nodes_info"]
position_info = obs_metadata["position_info"]
html_parser = obs_metadata["html_parser"]
self.nodes = nodes
page = info["page"]
url = self.map_url_to_real(page.url)
position_bar = self._get_position_bar(position_info)
history = [f"{ix}. {his}" for his in meta_data["action_history"]]
if len(history) == 1:
previous_action_str = "None"
else:
previous_action_str = '\n'.join(history[1:])
self.state.update({
"url": url,
"html": obs,
"html_parser": html_parser,
"segment": "None",
"operation": "None",
})
current = template.format(
objective=intent,
url=url,
html=obs,
position=position_bar,
previous_action=previous_action_str,
)
assert all([f"{{k}}" not in current for k in keywords])
# prompt = self.get_lm_api_input(intro, examples, current)
prompt = current + finale
return prompt
def _extract_action(self, response: str) -> str:
# find the first occurence of action
self.state["intention"] = self._extract_intention(response)
for regex, act in zip(self.operation, self.translate):
match = re.search(regex, response)
if match:
m = match.groups()
if isinstance(m, tuple):
exact_act = m[0]
param = m[1:]
else:
exact_act = m
param = []
param = list(param)
if act in ['click', 'hover', 'type', 'select']:
if len(param) == 0:
continue
for node_id, node in self.nodes.items():
if node['label'] == param[0]:
label = param[0]
hp = self.state["html_parser"]
bid = hp.id_label_converter(label)
segment = hp.get_segment(bid)
print('[Label]', label, bid, segment)
self.state["segment"] = segment
#self._extract_segment(self.state["html"], label)
if act not in ['select']:
param[0] = node_id
break
if act in ['stop', 'select', 'record']:
if len(param) > 0:
param[-1] = param[-1].strip("\'\"")
if act in ['type']:
print('In prompt constructer', param[-1])
if len(param) > 0:
param[-1] = param[-1].strip("\'\"")
print(param[-1])
if param[-1].endswith('\n'):
param[-1] = param[-1][:-1]
param.append('1')
else:
param.append('0')
command = act
for p in param:
command += f" [{p}]"
print(command)
return command
raise ActionParsingError(
f'Cannot find the answer phrase in "{response}"'
)
@staticmethod
def _get_position_bar(data):
position = data.get("position", 0.0)
page_height = data.get("page_height", 1.0)
left_bar = '-' * int(position)
right_bar = '-' * int(max(1, page_height - position))
return f'[0{left_bar}|{round(position, 1)}{right_bar}{round(page_height, 1)}]'
@staticmethod
def _extract_intention(response, lang='en'):
if lang == 'en':
matches = re.findall(r"#Thinking Process:\s*(.+)\s*#Operation:", response)
print('[Try to match]', matches)
else:
matches = re.findall(r"#思考过程: (.+)", response)
if matches:
return matches[-1]
else:
return None
@staticmethod
def _extract_segment(html: str, tag: str):
tag = f'[{tag}]'
has_content = False
def _left(html, start):
nonlocal has_content
left_cnt, right_cnt = 0, 0
for i in range(start, -1, -1):
if html[i] == '<':
left_cnt += 1
elif html[i] == '>':
if html[i - 2] != '|' and html[i - 2] != '>':
has_content = True
right_cnt += 1
elif html[i] == '|':
if html[i + 2] != '<' and html[i + 2] != '>':
has_content = True
if left_cnt == right_cnt + 1:
return i
return -1
def _right(html, start):
nonlocal has_content
left_cnt, right_cnt = 0, 0
for i in range(start, len(html), 1):
if html[i] == '<':
left_cnt += 1
elif html[i] == '>':
if html[i - 2] != '|' and html[i - 2] != '>':
has_content = True
right_cnt += 1
elif html[i] == '|':
if html[i + 2] != '<' and html[i + 2] != '>':
has_content = True
if left_cnt + 1 == right_cnt:
return i + 1
return -1
tag_start = html.find(tag)
if tag_start == -1:
return None
left_bound, right_bound = _left(html, tag_start), _right(html, tag_start)
while True:
if left_bound == -1 or right_bound == -1:
return None
if has_content:
break
else:
lb, rb = _left(html, left_bound - 1), _right(html, right_bound + 1)
if lb == -1 or rb == -1:
break
if rb - lb > 150:
break
else:
left_bound, right_bound = lb, rb
segment = html[left_bound:right_bound]
if len(segment) > 150:
return segment[:150] + '...>'
return segment
class NewASPromptConstructor(PromptConstructor):
"""The agent will perform step-by-step reasoning before the answer"""
operation = [
r"(click)\(\s*[\'\"]([A-Z]{1,3})[\'\"]\s*\)",
r"(type_string)\(\s*[\'\"]([A-Z]{1,3})[\'\"]\s*,\s*[\'\"]([\s\S]+)[\'\"]\s*,\s*(True|False)\s*\)",
r"(select)\(\s*[\'\"]([A-Z]{1,3})[\'\"]\s*,\s*[\'\"]([\s\S]+)[\'\"]\s*\)",
r"(scroll_page)\(\s*[\'\"]up[\'\"]\s*\)",
r"(scroll_page)\(\s*[\'\"]down[\'\"]\s*\)",
r"(jump_to)\(\s*[\'\"](.+)[\'\"]\s*,\s*(True|False)\s*\)",
r"(go)\(\s*[\'\"]backward[\'\"]\s*\)",
r"(go)\(\s*[\'\"]forward[\'\"]\s*\)",
r"(hover)\(\s*[\'\"]([A-Z]{1,3})[\'\"]\s*\)",
r"(finish)\(\s*\)",
r"(finish)\(\s*(.+)\s*\)",
r"(record)\(\s*[\'\"](.+)[\'\"]\s*\)",
r"(switch_tab)\([\d]+\)"
]
translate = [
"click",
"type",
"select",
"scroll [up]",
"scroll [down]",
"goto",
"go_back",
"go_forward",
"hover",
"stop",
"stop",
"record",
"page_focus",
]
def __init__(
self,
instruction_path: str | Path,
lm_config: lm_config.LMConfig,
tokenizer: Tokenizer,
):
super().__init__(instruction_path, lm_config, tokenizer)
self.answer_phrase = self.instruction["meta_data"]["answer_phrase"]
self.state = {}
def construct(
self,
trajectory: Trajectory,
intent: str,
meta_data: dict[str, Any] = {},
) -> APIInput:
intro = self.instruction["intro"]
examples = self.instruction["examples"]
template = self.instruction["template"]
keywords = self.instruction["meta_data"]["keywords"]
finale = self.instruction["finale"]
state_info: StateInfo = trajectory[-1] # type: ignore[assignment]
obs = state_info["observation"][self.obs_modality]
max_obs_length = self.lm_config.gen_config["max_obs_length"]
if max_obs_length:
obs = self.tokenizer.decode(self.tokenizer.encode(obs)[:max_obs_length]) # type: ignore[arg-type]
info = state_info["info"]
obs_metadata = info["observation_metadata"]["text"]
nodes = obs_metadata["obs_nodes_info"]
position_info = obs_metadata["position_info"]
html_parser = obs_metadata["html_parser"]
tabs_str = obs_metadata["tab_title"]
self.nodes = nodes
page = info["page"]
url = self.map_url_to_real(page.url)
position_bar = self._get_position_bar(position_info)
history = meta_data["action_history"]
if len(history) == 1:
previous_action_str = "None"
else:
previous_action_str = '\n'.join(history[1:])
self.state.update({
"url": url,
"html": obs,
"html_parser": html_parser,
"segment": "None",
"operation": "None",
})
current = template.format(
objective=intent,
url=url,
html=obs,
position=position_bar,
previous_action=previous_action_str,
tabs=tabs_str,
)
assert all([f"{{k}}" not in current for k in keywords])
# prompt = self.get_lm_api_input(intro, examples, current)
prompt = current + finale
return prompt
def _extract_action(self, response: str) -> str:
# find the first occurence of action
# self.state["intention"] = self._extract_intention(response)
for regex, act in zip(self.operation, self.translate):
match = re.search(regex, response)
if match:
m = match.groups()
if isinstance(m, tuple):
exact_act = m[0]
param = m[1:]
else:
exact_act = m
param = []
print(exact_act, param)
param = list(param)
if act in ['click', 'hover', 'type', 'select']:
if len(param) == 0:
continue
for node_id, node in self.nodes.items():
if node['label'] == param[0]:
label = param[0]
hp = self.state["html_parser"]
bid = hp.id_label_converter(label)
segment = hp.get_segment(bid)
print('[Label]', label, bid, segment)
self.state["segment"] = segment
#self._extract_segment(self.state["html"], label)
if act not in ['select']:
param[0] = node_id
break
if len(param) > 0:
if act in ['stop', 'select', 'record']:
param[-1] = param[-1].strip("\'\"")
if act in ['type', 'goto']:
param[-1] = '1' if param[-1] == 'True' else '0'
command = act
for p in param:
command += f" [{p}]"
print(command)
return command
raise ActionParsingError(
f'Cannot find the answer phrase in "{response}"'
)
@staticmethod
def _get_position_bar(data):
position = data.get("position", 0.0)
page_height = data.get("page_height", 1.0)
return f"{round(position, 1)} / {round(page_height, 1)}"

118
webarena/agent/prompts/raw/new_action_prompt.py Normal file
View File

@@ -0,0 +1,118 @@
prompt = {
"intro": "",
"examples": [],
"template": """<html> {html} </html>
You are a helpful assistant that can assist with web navigation tasks.
You are given a simplified html webpage and a task description.
Your goal is to complete the task. You can use the provided functions below to interact with the current webpage.
#Provided functions:
def click(element_id: str) -> None:
\"\"\"
Click on the element with the specified id.
Args:
element_id: The id of the element.
\"\"\"
def hover(element_id: str) -> None:
\"\"\"
Hover on the element with the specified id.
Args:
element_id: The id of the element.
\"\"\"
def select(element_id: str, option: str) -> None:
\"\"\"
Select an option from a dropdown.
Args:
element_id: The id of the element.
option: Value of the option to select.
\"\"\"
def type_string(element_id: str, content: str, press_enter: bool) -> None:
\"\"\"
Type a string into the element with the specified id.
Args:
element_id: The id of the element.
content: The string to type.
press_enter: Whether to press enter after typing the string.
\"\"\"
def scroll_page(direction: Literal['up', 'down']) -> None:
\"\"\"
Scroll down/up one page.
Args:
direction: The direction to scroll.
\"\"\"
def go(direction: Literal['forward', 'backward']) -> None:
\"\"\"
Go forward/backward
Args:
direction: The direction to go to.
\"\"\"
def jump_to(url: str, new_tab: bool) -> None:
\"\"\"
Jump to the specified url.
Args:
url: The url to jump to.
new_tab: Whether to open the url in a new tab.
\"\"\"
def switch_tab(tab_index: int) -> None:
\"\"\"
Switch to the specified tab.
Args:
tab_index: The index of the tab to switch to.
\"\"\"
def user_input(message: str) -> str:
\"\"\"
Wait for user input.
Args:
message: The message to display to the user.
Returns: The user input.
\"\"\"
def finish(answer: Optional[str]) -> None:
\"\"\"
Finish the task (optionally with an answer).
Args:
answer: The answer to the task.
\"\"\"
#Previous commands: {previous_action}
#Window tabs: {tabs}
#Current viewport (pages): {position}
#Task: {objective}
You should output one command to interact to the currrent webpage.
You should add a brief comment to your command to explain your reasoning and thinking process.
""",
"finale": "",
"meta_data": {
"observation": "html",
"action_type": "id_html_nasc_tree",
"keywords": ["url", "html", "objective", "position", "previous_action", "tabs"],
"prompt_constructor": "NewASPromptConstructor",
"answer_phrase": "",
"action_splitter": "#"
},
}

82
webarena/agent/prompts/raw/p_cot_id_actree_2s.py Normal file
View File

@@ -0,0 +1,82 @@
prompt = {
"intro": """You are an autonomous intelligent agent tasked with navigating a web browser. You will be given web-based tasks. These tasks will be accomplished through the use of specific actions you can issue.
Here's the information you'll have:
The user's objective: This is the task you're trying to complete.
The current web page's accessibility tree: This is a simplified representation of the webpage, providing key information.
The current web page's URL: This is the page you're currently navigating.
The open tabs: These are the tabs you have open.
The previous action: This is the action you just performed. It may be helpful to track your progress.
The actions you can perform fall into several categories:
Page Operation Actions:
`click [id]`: This action clicks on an element with a specific id on the webpage.
`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
`hover [id]`: Hover over an element with id.
`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).
`scroll [direction=down|up]`: Scroll the page up or down.
Tab Management Actions:
`new_tab`: Open a new, empty browser tab.
`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.
`close_tab`: Close the currently active tab.
URL Navigation Actions:
`goto [url]`: Navigate to a specific URL.
`go_back`: Navigate to the previously viewed page.
`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).
Completion Action:
`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
Homepage:
If you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.
http://homepage.com/password.html lists all the account name and password for the websites. You can use them to log in to the websites.
To be successful, it is very important to follow the following rules:
1. You should only issue an action that is valid given the current observation
2. You should only issue one action at a time.
3. You should follow the examples to reason step by step and then issue the next action.
4. Generate the action in the correct format. Start with a "In summary, the next action I will perform is" phrase, followed by action inside ``````. For example, "In summary, the next action I will perform is ```click [1234]```".
5. Issue stop action when you think you have achieved the objective. Don't generate anything after stop.""",
"examples": [
(
"""OBSERVATION:
[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'
[1749] StaticText '$279.49'
[1757] button 'Add to Cart'
[1760] button 'Add to Wish List'
[1761] button 'Add to Compare'
URL: http://onestopmarket.com/office-products/office-electronics.html
OBJECTIVE: What is the price of HP Inkjet Fax Machine
PREVIOUS ACTION: None""",
"Let's think step-by-step. This page list the information of HP Inkjet Fax Machine, which is the product identified in the objective. Its price is $279.49. I think I have achieved the objective. I will issue the stop action with the answer. In summary, the next action I will perform is ```stop [$279.49]```",
),
(
"""OBSERVATION:
[164] textbox 'Search' focused: True required: False
[171] button 'Go'
[174] link 'Find directions between two points'
[212] heading 'Search Results'
[216] button 'Close'
URL: http://openstreetmap.org
OBJECTIVE: Show me the restaurants near CMU
PREVIOUS ACTION: None""",
"Let's think step-by-step. This page has a search box whose ID is [164]. According to the nominatim rule of openstreetmap, I can search for the restaurants near a location by \"restaurants near\". I can submit my typing by pressing the Enter afterwards. In summary, the next action I will perform is ```type [164] [restaurants near CMU] [1]```",
),
],
"template": """OBSERVATION:
{observation}
URL: {url}
OBJECTIVE: {objective}
PREVIOUS ACTION: {previous_action}""",
"meta_data": {
"observation": "accessibility_tree",
"action_type": "id_accessibility_tree",
"keywords": ["url", "objective", "observation", "previous_action"],
"prompt_constructor": "CoTPromptConstructor",
"answer_phrase": "In summary, the next action I will perform is",
"action_splitter": "```"
},
}

82
webarena/agent/prompts/raw/p_cot_id_actree_2s_no_na.py Normal file
View File

@@ -0,0 +1,82 @@
prompt = {
"intro": """You are an autonomous intelligent agent tasked with navigating a web browser. You will be given web-based tasks. These tasks will be accomplished through the use of specific actions you can issue.
Here's the information you'll have:
The user's objective: This is the task you're trying to complete.
The current web page's accessibility tree: This is a simplified representation of the webpage, providing key information.
The current web page's URL: This is the page you're currently navigating.
The open tabs: These are the tabs you have open.
The previous action: This is the action you just performed. It may be helpful to track your progress.
The actions you can perform fall into several categories:
Page Operation Actions:
`click [id]`: This action clicks on an element with a specific id on the webpage.
`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
`hover [id]`: Hover over an element with id.
`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).
`scroll [direction=down|up]`: Scroll the page up or down.
Tab Management Actions:
`new_tab`: Open a new, empty browser tab.
`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.
`close_tab`: Close the currently active tab.
URL Navigation Actions:
`goto [url]`: Navigate to a specific URL.
`go_back`: Navigate to the previously viewed page.
`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).
Completion Action:
`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket.
Homepage:
If you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.
http://homepage.com/password.html lists all the account name and password for the websites. You can use them to log in to the websites.
To be successful, it is very important to follow the following rules:
1. You should only issue an action that is valid given the current observation
2. You should only issue one action at a time.
3. You should follow the examples to reason step by step and then issue the next action.
4. Generate the action in the correct format. Start with a "In summary, the next action I will perform is" phrase, followed by action inside ``````. For example, "In summary, the next action I will perform is ```click [1234]```".
5. Issue stop action when you think you have achieved the objective. Don't generate anything after stop.""",
"examples": [
(
"""OBSERVATION:
[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'
[1749] StaticText '$279.49'
[1757] button 'Add to Cart'
[1760] button 'Add to Wish List'
[1761] button 'Add to Compare'
URL: http://onestopmarket.com/office-products/office-electronics.html
OBJECTIVE: What is the price of HP Inkjet Fax Machine
PREVIOUS ACTION: None""",
"Let's think step-by-step. This page list the information of HP Inkjet Fax Machine, which is the product identified in the objective. Its price is $279.49. I think I have achieved the objective. I will issue the stop action with the answer. In summary, the next action I will perform is ```stop [$279.49]```",
),
(
"""OBSERVATION:
[164] textbox 'Search' focused: True required: False
[171] button 'Go'
[174] link 'Find directions between two points'
[212] heading 'Search Results'
[216] button 'Close'
URL: http://openstreetmap.org
OBJECTIVE: Show me the restaurants near CMU
PREVIOUS ACTION: None""",
"Let's think step-by-step. This page has a search box whose ID is [164]. According to the nominatim rule of openstreetmap, I can search for the restaurants near a location by \"restaurants near\". I can submit my typing by pressing the Enter afterwards. In summary, the next action I will perform is ```type [164] [restaurants near CMU] [1]```",
),
],
"template": """OBSERVATION:
{observation}
URL: {url}
OBJECTIVE: {objective}
PREVIOUS ACTION: {previous_action}""",
"meta_data": {
"observation": "accessibility_tree",
"action_type": "id_accessibility_tree",
"keywords": ["url", "objective", "observation", "previous_action"],
"prompt_constructor": "CoTPromptConstructor",
"answer_phrase": "In summary, the next action I will perform is",
"action_splitter": "```"
},
}

80
webarena/agent/prompts/raw/p_direct_id_actree_2s.py Normal file
View File

@@ -0,0 +1,80 @@
prompt = {
"intro": """You are an autonomous intelligent agent tasked with navigating a web browser. You will be given web-based tasks. These tasks will be accomplished through the use of specific actions you can issue.
Here's the information you'll have:
The user's objective: This is the task you're trying to complete.
The current web page's accessibility tree: This is a simplified representation of the webpage, providing key information.
The current web page's URL: This is the page you're currently navigating.
The open tabs: These are the tabs you have open.
The previous action: This is the action you just performed. It may be helpful to track your progress.
The actions you can perform fall into several categories:
Page Operation Actions:
`click [id]`: This action clicks on an element with a specific id on the webpage.
`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
`hover [id]`: Hover over an element with id.
`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).
`scroll [direction=down|up]`: Scroll the page up or down.
Tab Management Actions:
`new_tab`: Open a new, empty browser tab.
`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.
`close_tab`: Close the currently active tab.
URL Navigation Actions:
`goto [url]`: Navigate to a specific URL.
`go_back`: Navigate to the previously viewed page.
`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).
Completion Action:
`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket. If you believe the task is impossible to complete, provide the answer as "N/A" in the bracket.
Homepage:
If you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.
http://homepage.com/password.html lists all the account name and password for the websites. You can use them to log in to the websites.
To be successful, it is very important to follow the following rules:
1. You should only issue an action that is valid given the current observation
2. You should only issue one action at a time.
3. Generate the action in the correct format. Always put the action inside a pair of ```. For example, ```click [1234]```.
5. Issue stop action when you think you have achieved the objective. Don't generate anything after stop.""",
"examples": [
(
"""OBSERVATION:
[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'
[1749] StaticText '$279.49'
[1757] button 'Add to Cart'
[1760] button 'Add to Wish List'
[1761] button 'Add to Compare'
URL: http://onestopmarket.com/office-products/office-electronics.html
OBJECTIVE: What is the price of HP Inkjet Fax Machine
PREVIOUS ACTION: None""",
"```stop [$279.49]```",
),
(
"""OBSERVATION:
[164] textbox 'Search' focused: True required: False
[171] button 'Go'
[174] link 'Find directions between two points'
[212] heading 'Search Results'
[216] button 'Close'
URL: http://openstreetmap.org
OBJECTIVE: Show me the restaurants near CMU
PREVIOUS ACTION: None""",
"```type [164] [restaurants near CMU] [1]```",
),
],
"template": """OBSERVATION:
{observation}
URL: {url}
OBJECTIVE: {objective}
PREVIOUS ACTION: {previous_action}""",
"meta_data": {
"observation": "accessibility_tree",
"action_type": "id_accessibility_tree",
"keywords": ["url", "objective", "observation", "previous_action"],
"prompt_constructor": "DirectPromptConstructor",
"action_splitter": "```"
},
}

81
webarena/agent/prompts/raw/p_direct_id_actree_2s_no_na.py Normal file
View File

@@ -0,0 +1,81 @@
prompt = {
"intro": """You are an autonomous intelligent agent tasked with navigating a web browser. You will be given web-based tasks. These tasks will be accomplished through the use of specific actions you can issue.
Here's the information you'll have:
The user's objective: This is the task you're trying to complete.
The current web page's accessibility tree: This is a simplified representation of the webpage, providing key information.
The current web page's URL: This is the page you're currently navigating.
The open tabs: These are the tabs you have open.
The previous action: This is the action you just performed. It may be helpful to track your progress.
The actions you can perform fall into several categories:
Page Operation Actions:
`click [id]`: This action clicks on an element with a specific id on the webpage.
`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
`hover [id]`: Hover over an element with id.
`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).
`scroll [direction=down|up]`: Scroll the page up or down.
Tab Management Actions:
`new_tab`: Open a new, empty browser tab.
`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.
`close_tab`: Close the currently active tab.
URL Navigation Actions:
`goto [url]`: Navigate to a specific URL.
`go_back`: Navigate to the previously viewed page.
`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).
Completion Action:
`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket.
Homepage:
If you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.
http://homepage.com/password.html lists all the account name and password for the websites. You can use them to log in to the websites.
To be successful, it is very important to follow the following rules:
1. You should only issue an action that is valid given the current observation
2. You should only issue one action at a time.
4. Generate the action in the correct format, wrap the action inside ``````. For example, ```click [1234]```".
5. Issue stop action when you think you have achieved the objective.""",
"examples": [
(
"""OBSERVATION:
[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'
[1749] StaticText '$279.49'
[1757] button 'Add to Cart'
[1760] button 'Add to Wish List'
[1761] button 'Add to Compare'
URL: http://onestopmarket.com/office-products/office-electronics.html
OBJECTIVE: What is the price of HP Inkjet Fax Machine
PREVIOUS ACTION: None""",
"```stop [$279.49]```",
),
(
"""OBSERVATION:
[164] textbox 'Search' focused: True required: False
[171] button 'Go'
[174] link 'Find directions between two points'
[212] heading 'Search Results'
[216] button 'Close'
URL: http://openstreetmap.org
OBJECTIVE: Show me the restaurants near CMU
PREVIOUS ACTION: None""",
"```type [164] [restaurants near CMU] [1]```",
),
],
"template": """OBSERVATION:
{observation}
URL: {url}
OBJECTIVE: {objective}
PREVIOUS ACTION: {previous_action}""",
"meta_data": {
"observation": "accessibility_tree",
"action_type": "id_accessibility_tree",
"keywords": ["url", "objective", "observation", "previous_action"],
"prompt_constructor": "CoTPromptConstructor",
"answer_phrase": "In summary, the next action I will perform is",
"action_splitter": "```"
},
}

83
webarena/agent/prompts/raw/p_direct_id_actree_3s_llama.py Normal file
View File

@@ -0,0 +1,83 @@
prompt = {
"intro": """You are an autonomous intelligent agent tasked with navigating a web browser. The actions you can perform fall into several categories:
Page Operation Actions:
`click [id]`: This action clicks on an element with a specific id on the webpage.
`type [id] [content] [press_enter_after=0|1]`: Use this to type the content into the field with id. By default, the "Enter" key is pressed after typing unless press_enter_after is set to 0.
`hover [id]`: Hover over an element with id.
`press [key_comb]`: Simulates the pressing of a key combination on the keyboard (e.g., Ctrl+v).
`scroll [direction=down|up]`: Scroll the page up or down.
Tab Management Actions:
`new_tab`: Open a new, empty browser tab.
`tab_focus [tab_index]`: Switch the browser's focus to a specific tab using its index.
`close_tab`: Close the currently active tab.
URL Navigation Actions:
`goto [url]`: Navigate to a specific URL.
`go_back`: Navigate to the previously viewed page.
`go_forward`: Navigate to the next page (if a previous 'go_back' action was performed).
Completion Action:
`stop [answer]`: Issue this action when you believe the task is complete. If the objective is to find a text-based answer, provide the answer in the bracket.
Homepage:
If you want to visit other websites, check out the homepage at http://homepage.com. It has a list of websites you can visit.
You can only issue one action at a time""",
"examples": [
(
"""Observation:
[1744] link 'HP CB782A#ABA 640 Inkjet Fax Machine (Renewed)'
[1749] StaticText '$279.49'
[1757] button 'Add to Cart'
[1760] button 'Add to Wish List'
[1761] button 'Add to Compare'
URL: http://onestopmarket.com/office-products/office-electronics.html
Objective: What is the price of HP Inkjet Fax Machine
Previous action: None""",
"```stop [$279.49]```",
),
(
"""Observation:
[164] textbox 'Search' focused: True required: False
[171] button 'Go'
[174] link 'Find directions between two points'
[212] heading 'Search Results'
[216] button 'Close'
URL: http://openstreetmap.org
Objective: Show me the restaurants near CMU
Previous action: None""",
"```type [164] [restaurants near CMU] [1]```",
),
(
"""Observation:
[2036] button 'Sort by: New' hasPopup: menu expanded: False
[587] link 'US Marine’s adoption of Afghan war orphan voided'
[989] time 'March 30, 2023 at 15:03:48 AM UTC'
[602] link 'York student uses AI chatbot to get parking fine revoked'
[1025] time 'March 15, 2023 at 7:48:34 AM UTC'
[617] link 'Loveland parents furious after teachers leave, communication lagged during school threat investigation'
[1025] time 'March 2, 2023 at 3:46:01 AM UTC'
URL: http://reddit.com/f/news/new
Objective: Open the most recent post that was published prior to March 1st.
Previous action: None""",
"```scroll [down]```",
)
],
"template": """Observation:
{observation}
URL: {url}
Objective: {objective}
Previous action: {previous_action}""",
"meta_data": {
"observation": "accessibility_tree",
"action_type": "id_accessibility_tree",
"keywords": ["url", "objective", "observation", "previous_action"],
"prompt_constructor": "DirectPromptConstructor",
"answer_phrase": "In summary, the next action I will perform is",
"action_splitter": "```",
"force_prefix": "```"
},
}

41
webarena/agent/prompts/raw/test_prompt.py Normal file
View File

@@ -0,0 +1,41 @@
prompt = {
"intro": "",
"examples": [],
"template": """<html> {html} </html>
You are a helpful assistant that can assist with web navigation tasks.
You are given a simplified html webpage and a task description.
Your goal is to complete the task. You can perform the specified operations below to interact with the webpage.
#Valid operations: - #Click# id: Click on the element with the specified id
- #Scroll_up#: Scroll up 1 page.
- #Scroll_down#: Scroll down 1 page.
- #Go_backward#: Go back to the previous page.
- #Go_forward#: Go forward to the next page.
- #Hover# id: Hover over the element with the specified id.
- #Type# id "text": Type in the text at the element with the specified id.
- #Select# id "option": Select the option at the element with the specified id.
- #Record# "content": Mark content that is useful in answering the question.
- #Answer# "text": output the text as the answer to the user.
- #Exit#: Complete the task and exit the program.
#Current viewport position: {position}
#Previous Operation: {previous_action}
#Task: {objective}
""",
"finale": """
Your output SHOULD be in the following format:
#Operation: {Next operation to perform}
""",
"meta_data": {
"observation": "html",
"action_type": "id_html_tree",
"keywords": ["url", "html", "objective", "position", "previous_action"],
"prompt_constructor": "MyPromptConstructor",
"answer_phrase": "",
"action_splitter": "#"
},
}

26
webarena/agent/prompts/to_json.py Normal file
View File

@@ -0,0 +1,26 @@
import glob
import importlib
import json
import os
# use the current directory as the root
def run() -> None:
"""Convert all python files in agent/prompts to json files in agent/prompts/jsons
Python files are easiser to edit
"""
for p_file in glob.glob(f"agent/prompts/raw/*.py"):
# import the file as a module
base_name = os.path.basename(p_file).replace(".py", "")
module = importlib.import_module(f"agent.prompts.raw.{base_name}")
prompt = module.prompt
# save the prompt as a json file
os.makedirs("agent/prompts/jsons", exist_ok=True)
with open(f"agent/prompts/jsons/{base_name}.json", "w+") as f:
json.dump(prompt, f, indent=2)
print(f"Done convert python files to json")
if __name__ == "__main__":
run()