init

webagents_step/agents/agent.py

@@ -0,0 +1,106 @@
+from typing import List
+
+
+class Agent:
+    def __init__(
+        self,
+        max_actions,
+        verbose=0,
+        logging=False,
+        previous_actions: List = None,
+        previous_reasons: List = None,
+        previous_responses: List = None,
+    ):
+        self.previous_actions = [] if previous_actions is None else previous_actions 
+        self.previous_reasons = [] if previous_reasons is None else previous_reasons
+        self.previous_responses = [] if previous_responses is None else previous_responses
+        self.max_actions = max_actions
+        self.verbose = verbose
+        self.logging = logging
+        self.trajectory = []
+        self.data_to_log = {}
+
+    def reset(self):
+        self.previous_actions = []
+        self.previous_reasons = []
+        self.previous_responses = []
+        self.trajectory = []
+        self.data_to_log = {}
+
+    def get_trajectory(self):
+        return self.trajectory
+    
+    def update_history(self, action, reason):
+        if action:
+            self.previous_actions += [action]
+        if reason:
+            self.previous_reasons += [reason]    
+
+    def predict_action(self, objective, observation, url=None):
+        pass
+
+    def receive_response(self, response):
+        self.previous_responses += [response]
+
+    def act(self, objective, env):
+        while not env.done():
+            observation = env.observation()
+            action, reason = self.predict_action(
+                objective=objective, observation=observation, url=env.get_url()
+            )
+            status = env.step(action)
+
+            if self.logging:
+                self.log_step(
+                    objective=objective,
+                    url=env.get_url(),
+                    observation=observation,
+                    action=action,
+                    reason=reason,
+                    status=status,
+                )
+
+            if len(self.previous_actions) >= self.max_actions:
+                print(f"Agent exceeded max actions: {self.max_actions}")
+                break
+
+        return status
+
+    async def async_act(self, objective, env):
+        while not env.done():
+            observation = await env.observation()
+            action, reason = self.predict_action(
+                objective=objective, observation=observation, url=env.get_url()
+            )
+            status = await env.step(action)
+
+            if self.logging:
+                self.log_step(
+                    objective=objective,
+                    url=env.get_url(),
+                    observation=observation,
+                    action=action,
+                    reason=reason,
+                    status=status,
+                )
+
+            if len(self.previous_actions) >= self.max_actions:
+                print(f"Agent exceeded max actions: {self.max_actions}")
+                break
+
+        return status
+
+    def log_step(self, objective, url, observation, action, reason, status):
+        self.data_to_log['objective'] = objective
+        self.data_to_log['url'] = url
+        self.data_to_log['observation'] = observation if isinstance(observation, str) else observation["text"]
+        self.data_to_log['previous_actions'] = self.previous_actions[:-1]
+        self.data_to_log['previous_responses'] = self.previous_responses[:-1]
+        self.data_to_log['previous_reasons'] = self.previous_reasons[:-1]
+        self.data_to_log['action'] = action
+        self.data_to_log['reason'] = reason
+        if status:
+            for (k, v) in status.items():
+                self.data_to_log[k] = v
+        self.trajectory.append(self.data_to_log)
+        self.data_to_log = {}

webagents_step/agents/prompt_agent.py

@@ -0,0 +1,59 @@
+from webagents_step.agents.agent import Agent
+from typing import List
+from webagents_step.utils.llm import fill_prompt_template, construct_llm_message_openai, call_openai_llm, parse_action_reason, calculate_cost_openai, call_anthropic_llm
+
+class PromptAgent(Agent):
+    def __init__(self, max_actions: int = 10, verbose: bool = False, logging: bool = False, 
+                 debug: bool = False, prompt_template: str = None, model: str = "gpt-3.5-turbo", 
+                 prompt_mode: str = "chat", previous_actions: List = None, previous_reasons: List = None, previous_responses: List = None):
+        super().__init__(max_actions=max_actions, verbose=verbose, logging=logging, previous_actions=previous_actions, previous_reasons=previous_reasons, previous_responses=previous_responses)        
+        self.debug = debug 
+        self.prompt_template = prompt_template
+        self.model = model
+        self.prompt_mode = prompt_mode
+
+    def previous_history(self):
+        previous_history = []
+        
+        if len(self.previous_actions) == len(self.previous_responses):
+            for action, response in zip(self.previous_actions, self.previous_responses):
+                if response:
+                    previous_history.append(f"{response} = {action}")
+                else:
+                    previous_history.append(action)
+            previous_history="\n".join(previous_history)
+        else:
+            previous_history = "\n".join(action for action in self.previous_actions if action is not None) if self.previous_actions is not None else ""
+
+            
+        return previous_history
+
+    def predict_action(self, objective, observation, url=None):
+        prompt = fill_prompt_template(prompt_template=self.prompt_template, objective=objective, 
+                                      observation=observation, url=url, 
+                                      previous_history=self.previous_history())
+        messages = construct_llm_message_openai(prompt=prompt, prompt_mode=self.prompt_mode)
+        model_response = call_openai_llm(messages=messages, model=self.model)
+        action, reason = parse_action_reason(model_response)
+                
+        if self.logging:
+            self.data_to_log['prompt'] = messages
+        
+        if self.verbose > 0:
+            print(f"\n OBJECTIVE: {objective}")
+            print(f"\n URL: {url}")
+            print(f"\n PREVIOUS HISTORY: {self.previous_history()}")
+            print(f"\n REASON: {reason}")
+            print(f"\n ACTION: {action}")
+            if self.verbose > 1:
+                print(f"\n OBSERVATION: {observation}")
+                print(f"\n RESPONSE: {model_response}")
+        
+        if self.debug:
+            human_input = input()
+            if human_input != "c":
+                action = human_input
+                reason = "None"
+
+        self.update_history(action=action, reason=reason)
+        return action, reason

webagents_step/agents/step_agent.py

@@ -0,0 +1,105 @@
+from webagents_step.agents.agent import Agent
+from webagents_step.utils.stack import Stack
+from webagents_step.agents.prompt_agent import PromptAgent
+
+from typing import List, Dict
+import re
+import time
+
+class StepAgent(Agent):
+    def __init__(self, max_actions: int = 10, verbose: bool = False, logging: bool = False, 
+                 debug: bool = False, 
+                 root_action: str = None,
+                 action_to_prompt_dict: Dict = None,
+                 low_level_action_list: List = None,
+                 model: str = "gpt-3.5-turbo", 
+                 prompt_mode: str = "chat", previous_actions: List = None):
+        super().__init__(max_actions=max_actions, verbose=verbose, logging=logging, previous_actions=previous_actions)
+        self.debug = debug
+        self.root_action = root_action
+        self.action_to_prompt_dict = {} if action_to_prompt_dict is None else action_to_prompt_dict
+        self.low_level_action_list = [] if low_level_action_list is None else low_level_action_list
+        self.model = model
+        self.prompt_mode = prompt_mode
+        self.stack = Stack()
+    
+    def is_done(self, action):
+        if action and "stop" in action:
+            return True
+        return False
+    
+    def is_low_level_action(self, action):
+        if not action:
+            return False
+        action_type = action.split()[0]
+        return (action_type in self.low_level_action_list)
+    
+    def is_high_level_action(self, action):
+        if not action:
+            return False
+        action_type = action.split()[0]
+        return (action_type in self.action_to_prompt_dict) 
+    
+    def init_root_agent(self, objective):
+        root_prompt_template = self.action_to_prompt_dict[self.root_action]
+        agent = PromptAgent(
+            prompt_template=root_prompt_template,
+            model=self.model,
+            prompt_mode=self.prompt_mode,
+            max_actions=self.max_actions,
+            verbose=self.verbose,
+            logging=self.logging,
+            debug=self.debug,
+            previous_actions=[],
+            previous_reasons=[], 
+            previous_responses=[]
+        )
+        return {'agent': agent, 'objective': objective}
+    
+    def init_agent(self, action):
+        pattern = r'(\w+)\s+\[(.*?)\]'
+        matches = re.findall(pattern, action)
+        action_type, _ = matches[0]
+        objective = action
+        prompt_template = self.action_to_prompt_dict[action_type]
+        agent = PromptAgent(
+            prompt_template=prompt_template,
+            model=self.model,
+            prompt_mode=self.prompt_mode,
+            max_actions=self.max_actions,
+            verbose=self.verbose,
+            logging=self.logging,
+            debug=self.debug,
+            previous_actions=[],
+            previous_reasons=[], 
+            previous_responses=[]
+        )
+        return {'agent': agent, 'objective': objective}
+        
+    def predict_action(self, objective, observation, url=None):
+        observation = observation["text"] if isinstance(observation, dict) else observation
+        if self.stack.is_empty():
+            new_element = self.init_root_agent(objective=objective)
+            self.stack.push(new_element)
+            
+        action, reason = None, None
+        while not self.stack.is_empty():
+            element = self.stack.peek()
+            action, reason = element['agent'].predict_action(objective=element['objective'], observation=observation, url=url)
+            if (not self.is_done(action)) and self.is_low_level_action(action):
+                element['agent'].receive_response("")
+                return action, reason
+            if (not self.is_done(action)) and self.is_high_level_action(action):
+                new_element = self.init_agent(action)
+                self.stack.push(new_element)
+                if self.logging:
+                    self.log_step(objective=element['objective'], url=url, observation=observation, action=action, reason=reason, status={})
+                continue
+            if self.is_done(action):
+                self.stack.pop()
+                if not self.stack.is_empty():
+                    self.stack.peek()['agent'].receive_response(re.search(r"\[(.*?)\]", action).group(1))
+                if self.logging:
+                    self.log_step(objective=element['objective'], url=url, observation=observation, action=action, reason=reason, status={})
+                continue
+        return action, reason

webagents_step/environment/env.py

@@ -0,0 +1,18 @@
+class WebEnvironment():
+    def __init__(self):
+        pass
+
+    def reset(self):
+        pass
+
+    def observation(self):
+        pass
+
+    def get_url(self):
+        pass
+
+    def step(self, action):
+        pass
+
+    def done(self):
+        pass

webagents_step/environment/liveweb.py

@@ -0,0 +1,175 @@
+from time import sleep
+import pandas as pd
+import re
+
+from webagents_step.parser import (
+    heihei_web_parser,
+    playwright_parser_nat,
+    playwright_parser_webarena,
+)
+from webagents_step.environment.env import WebEnvironment
+
+
+class LiveWebEnvironmentWrapper(WebEnvironment):
+    def __init__(
+        self,
+        url=None,
+        objective=None,
+        parser_type="heihei",
+        observation_type="text",
+        text_observation_type="accesibility_tree",
+        max_browser_rows=1000,
+        max_steps=50,
+        step_delay=2,
+        headless=False,
+    ):
+        self.url = url
+        self.objective = objective
+        self.headless = headless
+        self.parser_type = parser_type
+        self.observation_type = observation_type
+        self.text_observation_type = text_observation_type
+        self.max_browser_rows = max_browser_rows
+        self.max_steps = max_steps
+
+        self.steps = 0
+        self.is_done = False
+        self.parse_timeout = 5
+        self.step_delay = step_delay
+        self.response = ""
+
+    async def init_parser(self):
+        if self.parser_type == "heihei":
+            self.parser = heihei_web_parser.HeiHeiWebParser()
+            await self.parser.init()
+        elif self.parser_type == "playwright_webarena":
+            self.parser = playwright_parser_webarena.PlaywrightParserWebArena(
+                headless=self.headless,
+                observation_type=self.observation_type,
+                text_observation_type=self.text_observation_type,
+            )
+            self.parser.init()
+        elif self.parser_type == "playwright_nat":
+            self.parser = playwright_parser_nat.PlaywrightParserNat(
+                headless=self.headless
+            )
+            await self.parser.init()
+        else:
+            raise NotImplementedError(f"{self.parser_type} not implemented.")
+
+        if self.url is not None:
+            await self.parser.go_to_page(self.url)
+            self.clear_page_presets()
+            await self.parser.parse_page()
+
+    def clear_page_presets(self):
+        pass
+
+    async def reset(self):
+        await self.close()
+        await self.init_parser()
+
+    async def close(self):
+        await self.parser.close()
+
+    async def observation(self, tab_id=None, format=None):
+        format = self.text_observation_type if format is None else format
+        if self.parser_type == "heihei":
+            try:
+                browser_content = await self.parser.parse_page(
+                    format=format, tab_id=tab_id
+                )
+            except:
+                sleep(self.parse_timeout)
+                browser_content = await self.parser.parse_page(
+                    format=format, tab_id=tab_id
+                )
+        else:
+            browser_content = await self.parser.parse_page()
+
+        if format not in ["htree", "html", "json"]:
+            browser_content = [str(w) for w in browser_content]
+            browser_content = browser_content[: self.max_browser_rows]
+            browser_content = "\n".join(browser_content)
+
+        return browser_content
+
+    def get_log(self):
+        return self.df_log
+
+    def get_response(self):
+        return self.response
+
+    def get_url(self):
+        return self.parser.get_url()
+
+    async def execute_action(self, action):
+        """
+        Execute a given action based on the action type,
+        - click [id]: Clicks an element based on the provided id.
+        - type [id] [content]: Types the provided content into the element with the specified id.
+        - goto [url]: Navigates to an existing tab at that URL
+        - open [url]: Opens a new tab with provided URL
+        - copy [content]: Copies content, but no-op action
+        - stop [response]: Stops execution and optionally provides a response.
+        """
+        click_match = re.match(r"click \[(\S+)\]", action, re.IGNORECASE)
+        type_match = re.match(r"type \[(\S+)\] \[(.+)\]", action, re.IGNORECASE)
+        goto_match = re.match(r"goto \[(\S+)\]", action, re.IGNORECASE)
+        open_match = re.match(r"open \[(\S+)\]", action, re.IGNORECASE)
+        copy_match = re.match(r"copy \[(\S+)\]", action, re.IGNORECASE)
+        stop_match = re.match(r"stop \[([^\]]*)\]", action, re.IGNORECASE)
+
+        if click_match:
+            id = click_match.group(1)
+            if not id.isdigit():
+                raise Exception("Id not a valid integer")
+            await self.parser.click(int(id))
+
+        elif type_match:
+            id = type_match.group(1)
+            content = type_match.group(2)
+            if not id.isdigit():
+                raise Exception("Id not a valid integer")
+            await self.parser.type(int(id), content)
+
+        elif goto_match:
+            url = goto_match.group(1)
+            tab_id, tab_url = await self.parser.get_tab_from_url(url)
+            await self.parser.go_to_page(url)
+
+        elif open_match:
+            url = open_match.group(1)
+            await self.parser.go_to_page(url)
+
+        elif copy_match:
+            pass
+
+        elif stop_match:
+            self.response = stop_match.group(1)
+            self.is_done = True
+
+        else:
+            print(f"[execute_action] Error {action} not defined")
+
+    async def step(self, action, delay=None):
+        delay = self.step_delay if delay is None else delay
+
+        if self.steps > self.max_steps:
+            print(f"Steps {self.steps} exceeded maximum {self.max_steps}")
+            self.is_done = True
+            return
+
+        print(f"[Step {self.steps+1}] {action}")
+        try:
+            await self.execute_action(action)
+        except Exception as e:
+            print(f"Error while executing action '{action}'. Details: {e}")
+
+        sleep(delay)
+        self.steps = self.steps + 1
+
+        return {"done": self.is_done, "response": self.response}
+
+    def done(self):
+        return self.is_done

webagents_step/parser/playwright_parser_webarena.py

@@ -0,0 +1,99 @@
+from sys import platform
+from playwright.sync_api import sync_playwright
+from browser_env.processors import TextObervationProcessor, ImageObservationProcessor
+
+
+class PlaywrightParserWebArena:
+    def __init__(
+        self,
+        headless=True,
+        observation_type="text",
+        text_observation_type="accessibility_tree",
+        viewport_size={"width": 1280, "height": 1080},
+        current_viewport_only=True,
+    ):
+        self.headless = headless
+        self.viewport_size = viewport_size
+        self.current_viewport_only = current_viewport_only
+        self.observation_type = observation_type
+        self.text_observation_type = text_observation_type
+
+        self.playwright = sync_playwright().start()
+        self.browser = self.playwright.chromium.launch(headless=self.headless)
+        self.context = self.browser.new_context(
+            viewport=self.viewport_size,
+            device_scale_factor=1,
+        )
+        
+        self.page = self.context.new_page()
+        client = self.page.context.new_cdp_session(self.page)
+        if (self.observation_type == "text") and (
+            self.text_observation_type == "accessibility_tree"
+        ):
+            client.send("Accessibility.enable")
+        self.page.client = client
+        
+        ## scratch ##
+        # initialize with html string
+        # self.page.goto(url if "://" in url else "http://" + url)
+        # potentially later
+        # self.page.goto("https://www.google.com", wait_until='networkidle')
+        # print(self.page.accessibility.snapshot())
+        # self.page = self.page.accessibility.snapshot()
+
+        self.text_processor = TextObervationProcessor(
+            observation_type=self.text_observation_type,
+            current_viewport_only=self.current_viewport_only,
+            viewport_size=self.viewport_size,
+        )
+        self.image_processor = ImageObservationProcessor(observation_type="image")
+    
+    def clear_page_presets():
+        pass
+    
+    def observation_processor(self):
+        if self.observation_type == "text":
+            return self.text_processor
+        elif self.observation_type == "image":
+            return self.image_processor
+        else:
+            raise ValueError("Invalid observation type")
+
+    def get_url(self):
+        return self.page.url
+
+    def go_to_page(self, url: str):
+        self.page.goto(url if "://" in url else "http://" + url)
+
+    def close(self):
+        self.browser.close()
+        self.playwright_context.stop()
+
+    def click_xy(self, x: float, y: float) -> None:
+        viewport_size = self.page.viewport_size
+        self.page.mouse.click(x * viewport_size["width"], y * viewport_size["height"])
+
+    def click(self, id: int) -> None:
+        element_center = self.observation_processor().get_element_center(id)
+        self.click_xy(element_center[0], element_center[1])
+
+    def type(self, id: int, text: str, clear: bool = True):
+        if clear:
+            self.clear(id)
+        self.click(id)
+        self.page.keyboard.type(text)
+
+    def clear(self, id: int) -> None:
+        self.click(id)
+        select_key = "Meta" if platform.startswith("darwin") else "Control"
+        self.page.keyboard.down(select_key)
+        self.page.keyboard.press("a")
+        self.page.keyboard.up(select_key)
+        self.page.keyboard.press("Backspace")
+
+    def parse_page(self):
+        observation = self.observation_processor().process(
+            page=self.page, client=self.page.client
+        )
+
+        return observation

webagents_step/utils/data_prep.py

@@ -0,0 +1,137 @@
+import os
+import json
+import pandas as pd
+import random
+import yaml
+import copy
+import numpy as np
+
+class yamlConfig:
+    def __init__(self, dictionary):
+        for key, value in dictionary.items():
+            if isinstance(value, dict):
+                value = yamlConfig(value)
+            setattr(self, key, value)
+
+class DotDict:
+    """access dictionary attributes with dot notation"""
+    def __init__(self, dictionary):
+        for key, value in dictionary.items():
+            if isinstance(value, dict):
+                value = DotDict(value)
+            setattr(self, key, value)
+    
+    def to_dict(self):
+        regular_dict = {}
+        for key, value in self.__dict__.items():
+            if isinstance(value, DotDict):
+                regular_dict[key] = value.to_dict()
+            else:
+                regular_dict[key] = value
+        return regular_dict
+    
+    def has_key(self, key):
+        """Check if the DotDict has a specific key"""
+        return hasattr(self, key)
+    
+#####
+# Dataset processing functions
+#####
+
+def split_list_by_proportion(proportion, original_list):
+    list_size = len(original_list)
+    first_list_size = int(proportion * list_size)
+    shuffled_list = copy.copy(original_list)
+    random.shuffle(shuffled_list)
+    first_list = shuffled_list[:first_list_size]
+    second_list = shuffled_list[first_list_size:]
+    return first_list, second_list
+
+
+def examples_to_prompts(data_config, json_df, prompt_template, inference):
+    all_prompt_strings = []
+    for idx, row in json_df.iterrows():
+        try:
+            # read samples
+            json_filepath = os.path.join(data_config.basedir, str(row.logfile))
+            with open(json_filepath,'r') as json_file:
+                example_json = json.load(json_file)
+            all_prompt_strings.extend(convert_example_to_prompts(example_json, data_config, prompt_template, inference))
+        except FileNotFoundError:
+            print(f"File {data_config.basedir}/{row.logfile} not found. Skipping ...")
+            continue
+            
+    return all_prompt_strings
+
+def convert_example_to_prompts(example_json, data_config, prompt_template, inference):
+    filled_prompt_strings = []
+    previous_actions = []
+    for step_ind, step in enumerate(example_json["trajectory"]):
+        # Fill in prompt components
+        prompt_components = copy.deepcopy(prompt_template)        
+        prompt_components["input"] = prompt_components["input"].format(
+            objective = step["objective"], 
+            observation = step["observation"], 
+            url = step["url"], 
+            previous_actions = "\n".join([str(step) for step in step["previous_actions"][-data_config.action_lookback:] if step is not None]) if "previous_actions" in step else "\n".join([str(action) for action in previous_actions[-data_config.action_lookback:] if action is not None])
+        )
+        previous_actions.append(step["action"])
+
+        prompt_components["response"] = convert_actions_and_reasons_to_response(step["reason"], step["action"])
+        filled_prompt_strings.append(create_prompt(prompt_components, inference=inference))
+    return filled_prompt_strings
+
+def convert_actions_and_reasons_to_response(reasons, actions):
+    response = f"""
+REASON:
+{reasons}
+ACTION:
+{actions}
+"""
+    return response
+
+def create_prompt(prompt_components, inference=False):
+    #If inference mode, do not fill in the response
+    if inference:
+        prompt_template = """
+<s>[INST] <<SYS>>
+{instruction}
+<</SYS>>
+
+{input} [/INST]"""
+        return prompt_template.format(instruction = prompt_components["instruction"],
+                                  input = prompt_components["input"])
+    else:
+        prompt_template = """
+<s>[INST] <<SYS>>
+{instruction}
+<</SYS>>
+
+{input} [/INST] {response} </s>
+"""
+        return prompt_template.format(instruction = prompt_components["instruction"],
+                                  input = prompt_components["input"],
+                                  response = prompt_components["response"])
+
+def log_run(log_file, log_data, summary_file=None, summary_data=None, json_indent=4, verbose=1):
+    """
+    Logs demo data to a JSON file and optionally updates a summary CSV file.
+    """
+    # Write log data to JSON file
+    with open(log_file, 'w') as json_file:
+        json.dump(log_data, json_file, indent=json_indent)
+    if verbose:
+        print(f"Saved log to {log_file}")
+
+    # If summary data and file path are provided, update the summary
+    if summary_data and summary_file:
+        if os.path.exists(summary_file):
+            df_summary = pd.read_csv(summary_file)
+        else:
+            df_summary = pd.DataFrame()
+        df_summary = pd.concat([df_summary, pd.DataFrame([summary_data])], ignore_index=True)
+        df_summary.to_csv(summary_file, index=False)
+        if verbose:
+            print(f"Updated summary: {df_summary}")
+        
+        return df_summary

webagents_step/utils/llm.py

@@ -0,0 +1,422 @@
+import openai
+import re
+import copy
+import torch
+from transformers import (
+    AutoModelForCausalLM,
+    AutoTokenizer,
+    BitsAndBytesConfig,
+    AutoModelForSeq2SeqLM,
+)
+import ctranslate2
+from time import sleep
+import tiktoken
+from openai import OpenAI
+
+import warnings
+warnings.simplefilter("ignore")
+
+input_token_cost_usd_by_model = {
+    "gpt-4-1106-preview": 0.01 / 1000,
+    "gpt-4": 0.03 / 1000,
+    "gpt-4-32k": 0.06 / 1000,
+    "gpt-3.5-turbo": 0.001 / 1000,
+    "gpt-3.5-turbo-instruct": 0.0015 / 1000,
+    "gpt-3.5-turbo-16k": 0.003 / 1000,
+    "babbage-002": 0.0016 / 1000,
+    "davinci-002": 0.012 / 1000,
+    "ada-v2": 0.0001 / 1000,
+}
+
+output_token_cost_usd_by_model = {
+    "gpt-4-1106-preview": 0.03 / 1000,
+    "gpt-4": 0.06 / 1000,
+    "gpt-4-32k": 0.12 / 1000,
+    "gpt-3.5-turbo": 0.002 / 1000,
+    "gpt-3.5-turbo-instruct": 0.002 / 1000,
+    "gpt-3.5-turbo-16k": 0.004 / 1000,
+    "babbage-002": 0.0016 / 1000,
+    "davinci-002": 0.012 / 1000,
+    "ada-v2": 0.0001 / 1000,
+}
+
+def fill_prompt_template(prompt_template, objective, observation, url, previous_history):
+    prompt = copy.deepcopy(prompt_template)
+    prompt["input"] = prompt["input"].replace("{objective}", objective)
+    if isinstance(observation, dict):
+        prompt["input"] = prompt["input"].replace("{observation}", observation["text"])
+    else:
+        prompt["input"] = prompt["input"].replace("{observation}", observation)
+    prompt["input"] = prompt["input"].replace("{url}", url)   
+    prompt["input"] = prompt["input"].replace("{previous_actions}", previous_history)   
+    return prompt
+
+def filter_quotes_if_matches_template(action):
+    if action is None:
+        return None
+
+    # Regex pattern to match the entire 'type [X] ["Y"]' template, allowing for Y to be digits as well
+    pattern = r'^type \[\d+\] \["([^"\[\]]+)"\]$'
+    # Check if the action matches the specific template
+    match = re.match(pattern, action)
+    if match:
+        # Extract the matched part that needs to be unquoted
+        y_part = match.group(1)
+        # Reconstruct the action string without quotes around Y
+        filtered_action = f'type [{match.group(0).split("[")[1].split("]")[0]}] [{y_part}]'
+        return filtered_action.strip() # filtered_action.split("\n")[0].strip()
+    else:
+        # Return the original action if it doesn't match the template
+        return action.strip() # action.split("\n")[0].strip()
+
+def parse_action_reason(model_response):
+    reason_match = re.search(r'REASON:\s*(.*?)\s*(?=\n[A-Z]|$)', model_response, re.DOTALL) 
+    reason = reason_match.group(1) if reason_match else None
+
+    # action_match = re.search(r'ACTION:\s*(.*?)\s*(?=\n[A-Z]|$)', model_response, re.DOTALL) 
+    action_match = re.search(r'(?:ACTION|ACTIONS):\s*(.*?)\s*(?=\n[A-Z]|$)', model_response, re.DOTALL) 
+    action = action_match.group(1) if action_match else None
+    
+    action = filter_quotes_if_matches_template(action)
+    
+    return action, reason
+
+def construct_llm_message_hf(prompt, prompt_mode, model_type="llama2"):
+    if model_type == "llama2":
+        instruction = "<s>[INST] " + prompt["instruction"]
+    else:
+        instruction = prompt["instruction"]
+    
+    messages = [{"role": "system", "content": instruction}]
+    
+    if prompt["examples"]:
+        messages.append({"role": "system", "content": "Here are a few examples:"})
+        for example in prompt["examples"]:
+            messages.append({"role": "system", "content": f"\n### Input:\n{example['input']}\n\n### Response:\n{example['response']}"})
+    
+    if model_type == "llama2":
+        query = f"\nHere is the current Input. Please respond with REASON and ACTION.\n### Input:\n{prompt['input']}\n[/INST]\n"
+    else:
+        query = f"\nHere is the current Input. Please respond with REASON and ACTION.\n### Input:\n{prompt['input']}\n\n### Response:"
+    
+    messages.append({"role": "user", "content": query})
+    if prompt_mode == "chat":
+        return messages
+    elif prompt_mode == "completion":
+        all_content = ''.join(message['content'] for message in messages)
+        messages_completion = [{"role": "user", "content": all_content}]
+        return messages_completion
+
+def construct_llm_message_anthropic(prompt, plan_list=None, action_list=None):
+    if plan_list and action_list:
+        import os
+        from global_utils import CURRENT_DIR
+
+        assert len(plan_list) > 0 and len(action_list) > 0
+        plan_instructions = "\n".join(["".join(open(os.path.join(CURRENT_DIR, "prompts", "plan_instructions", f"{p}.txt"), "r").readlines()) for p in plan_list])
+        action_instructions = "\n".join(["".join(open(os.path.join(CURRENT_DIR, "prompts", "action_instructions", f"{a}.txt"), "r").readlines()) for a in action_list])
+        prompt["instruction"] = prompt["instruction"].replace("{plan_instructions}", plan_instructions)
+        prompt["instruction"] = prompt["instruction"].replace("{action_instructions}", action_instructions)
+
+    system_message = prompt["instruction"]
+        
+    if prompt["examples"]:
+        system_message += f"\n\n## Here are a few examples:"
+        for i, example in enumerate(prompt["examples"]):
+            example_input = example["input"]
+            example_response = example["response"]
+            if "example_format" in prompt.keys():
+                system_message += "\n\n"
+                system_message += prompt["example_format"].replace("{i}", i).replace("{example_input}", example_input).replace("{example_response}", example_response)
+            else:
+                system_message += f"\n\n| Example {i}\n\n### Input:\n{example_input}\n\n### Response: Let's think step by step.\n{example_response}"
+
+    if "input_format" in prompt.keys():
+        if "{visual_observation}" in prompt.keys():
+            from claude import arrange_message_for_claude
+            text = prompt["input_format"].replace("{input}", prompt['input'])
+            text_prior, text_subsequent = text.split("{visual_observation}")
+            messages = arrange_message_for_claude([("text", text_prior), ("image", prompt["{visual_observation}"]), ("text", text_subsequent)])
+        else:
+            messages = [{"role": "user", "content": [{"type": "text", "text": prompt["input_format"].replace("{input}", prompt['input'])}]}]
+    else:
+        if "{visual_observation}" in prompt.keys():
+            pass
+        else:
+            messages = [{"role": "user", "content": [{"type": "text", "text": f"## Here is the current Input. Please respond with REASON and ACTION.\n### Input:\n{prompt['input']}\n\n### Response:"}]}]
+
+    return system_message, messages
+
+def construct_llm_message_openai(prompt, prompt_mode, plan_list=None, action_list=None):
+    if not (plan_list and action_list):
+        messages = [{"role": "system", "content": prompt["instruction"]}]
+            
+        if prompt["examples"]:
+            messages.append({"role": "system", "content": "Here are a few examples:"})
+            for example in prompt["examples"]:
+                messages.append({"role": "system", "content": f"\n### Input:\n{example['input']}\n\n### Response:\n{example['response']}"})
+        
+        messages.append({"role": "user", "content": f"Here is the current Input. Please respond with REASON and ACTION.\n### Input:\n{prompt['input']}\n\n### Response:"})
+        if prompt_mode == "chat":
+            return messages
+        elif prompt_mode == "completion":
+            all_content = ''.join(message['content'] for message in messages)
+            messages_completion = [{"role": "user", "content": all_content}]
+            return messages_completion
+    import os
+    from global_utils import CURRENT_DIR
+
+    assert len(plan_list) > 0 and len(action_list) > 0
+    plan_instructions = "\n".join(["".join(open(os.path.join(CURRENT_DIR, "prompts", "plan_instructions", f"{p}.txt"), "r").readlines()) for p in plan_list])
+    action_instructions = "\n".join(["".join(open(os.path.join(CURRENT_DIR, "prompts", "action_instructions", f"{a}.txt"), "r").readlines()) for a in action_list])
+    prompt["instruction"] = prompt["instruction"].replace("{plan_instructions}", plan_instructions)
+    prompt["instruction"] = prompt["instruction"].replace("{action_instructions}", action_instructions)
+
+    messages = [{"role": "system", "content": prompt["instruction"]}]
+
+    if prompt["examples"]:
+        messages.append({"role": "system", "content": "## Here are a few examples:"})
+        for i, example in enumerate(prompt["examples"]):
+            example_input = example["input"]
+            example_response = example["response"]
+            messages.append({"role": "system", "content": f"| Example {i}\n\n### Input:\n{example_input}\n\n### Response: Let's think step by step.\n{example_response}"})
+
+    if "input_format" in prompt.keys():
+        messages.append({"role": "user", "content": prompt["input_format"].replace("{input}", prompt['input'])})
+    else:
+        messages.append({"role": "user", "content": f"## Here is the current Input. Please respond with PLAN, REASON and ACTION.\n### Input:\n{prompt['input']}\n\n### Response:"})
+    if prompt_mode == "chat":
+        return messages
+    elif prompt_mode == "completion":
+        all_content = ''.join(message['content'] for message in messages)
+        messages_completion = [{"role": "user", "content": all_content}]
+        return messages_completion
+
+def call_anthropic_llm(system_message, messages, model="anthropic.claude-3-haiku-20240307-v1:0", **model_kwargs):
+    # Use the native inference API to send a text message to Anthropic Claude.
+
+    import boto3
+    import json
+
+    # Create a Bedrock Runtime client in the AWS Region of your choice.
+    client = boto3.client("bedrock-runtime", region_name="us-east-1")
+    print(system_message, file=open("trash.txt", "a"))
+    print("\n".join(item["content"][0]["text"] for item in messages), end="\n"+"#"*100+"\n", file=open("trash.txt", "a"))
+    native_request = {
+        "anthropic_version": "bedrock-2023-05-31",
+        "max_tokens": 512,
+        "temperature": 0.5,
+        "system": system_message,
+        "messages": messages,
+    }
+
+    # Convert the native request to JSON.
+    request = json.dumps(native_request)
+
+    try:
+        # Invoke the model with the request.
+        response = client.invoke_model(modelId=model, body=request)
+
+    except Exception as e:
+        raise KeyError(f"ERROR: Can't invoke '{model}'. Reason: {e}")
+
+    # Decode the response body.
+    model_response = json.loads(response["body"].read())
+
+    # Extract and print the response text.
+    response_text = model_response["content"][0]["text"]
+    return response_text
+
+def call_openai_llm(messages, model="gpt-3.5-turbo", **model_kwargs):
+    """
+    Sends a request with a chat conversation to OpenAI's chat API and returns a response.
+
+    Parameters:
+        messages (list)
+            A list of dictionaries containing the messages to send to the chatbot.
+        model (str)
+            The model to use for the chatbot. Default is "gpt-3.5-turbo".
+        temperature (float)
+            The temperature to use for the chatbot. Defaults to 0. Note that a temperature
+            of 0 does not guarantee the same response (https://platform.openai.com/docs/models/gpt-3-5).
+    
+    Returns:
+        response (Optional[dict])
+            The response from OpenAI's chat API, if any.
+    """
+    # client = OpenAI()
+    temperature = model_kwargs.get('temperature', 0.7)
+    top_p = model_kwargs.get('top_p', 1.0)
+    n = model_kwargs.get('n', 1)
+
+    for m in messages:
+        print(m["content"], file=open("trash.txt", "a"))
+    print("*"*100, file=open("trash.txt", "a"))
+    
+    num_attempts = 0
+    while True:
+        if num_attempts >= 10:
+            raise ValueError("OpenAI request failed.")
+        try:
+            if model=="text-davinci-003":
+                response = openai.Completion.create(
+                model=model,
+                prompt=messages[0]["content"],
+                temperature=temperature,
+                top_p=top_p,
+                n=n,
+                max_tokens=128)
+                return response.choices[0].text.strip()
+                        
+            response = OpenAI().chat.completions.create(
+                model=model,
+                messages=messages,
+                temperature=temperature,
+                top_p=top_p,
+                n=n
+            )
+            return response.choices[0].message.content.strip()
+        except openai.AuthenticationError as e:
+            print(e)
+            return None
+        except openai.RateLimitError as e:
+            print(e)
+            print("Sleeping for 10 seconds...")
+            sleep(10)
+            num_attempts += 1
+        except Exception as e:
+            print(e)
+            print("Sleeping for 10 seconds...")
+            sleep(10)
+            num_attempts += 1
+
+def get_num_tokens(text: str, model_name: str) -> int:
+    tokenizer = tiktoken.encoding_for_model(model_name=model_name)
+    return len(tokenizer.encode_ordinary(text))
+
+def calculate_cost_openai(messages: str, response: str, model_name: str) -> int:
+    input_text = " ".join([msg["content"] for msg in messages]) 
+    num_input_tokens = get_num_tokens(input_text, model_name)
+    num_output_tokens = get_num_tokens(response, model_name)
+    
+    input_token_cost = input_token_cost_usd_by_model.get(model_name, None)
+    output_token_cost = output_token_cost_usd_by_model.get(model_name, None)
+    if input_token_cost is None or output_token_cost is None:
+        print(f"[calculate_cost_openai] unknown model {model_name}")
+        return 0
+    return num_input_tokens * input_token_cost + num_output_tokens * output_token_cost
+
+def load_tokenizer(mpath, context_size):
+    tokenizer = AutoTokenizer.from_pretrained(mpath, return_token_type_ids=False)
+    # tokenizer.pad_token = tokenizer.eos_token
+    # tokenizer.pad_token_id = tokenizer.eos_token_id
+    # tokenizer.model_max_length = context_size
+    # tokenizer.padding_side = "right"
+    # tokenizer.truncation_side = "left"
+    # tokenizer.add_eos_token = True
+    return tokenizer
+
+def load_model(mpath, dtype, device="cuda", context_len=4096, is_seq2seq=False, ct2_mpath=None):
+    if is_seq2seq:
+        model_loader = AutoModelForSeq2SeqLM
+    else:
+        model_loader = AutoModelForCausalLM
+
+    if dtype == "bf16":
+        model = model_loader.from_pretrained(
+            mpath,
+            max_position_embeddings=context_len,
+            low_cpu_mem_usage=True,
+            torch_dtype=torch.bfloat16,
+            device_map="balanced_low_0",
+        )
+    elif dtype == "4bit":
+        model = model_loader.from_pretrained(
+            mpath,
+            max_position_embeddings=context_len,
+            low_cpu_mem_usage=True,
+            load_in_4bit=True,
+            device_map="auto",
+        )
+    elif dtype == "4bit-optimized":
+        bnb_config = BitsAndBytesConfig(
+            load_in_4bit=True,
+            bnb_4bit_use_double_quant=True,
+            bnb_4bit_quant_type="nf4",
+            bnb_4bit_compute_dtype=torch.bfloat16,
+        )
+        model = model_loader.from_pretrained(
+            mpath,
+            use_cache=True,
+            device_map="auto",
+            quantization_config=bnb_config,
+            max_position_embeddings=context_len,
+        )
+    elif dtype == "8bit":
+        model = model_loader.from_pretrained(
+            mpath,
+            max_position_embeddings=context_len,
+            low_cpu_mem_usage=True,
+            load_in_8bit=True,
+            device_map="auto",
+        )
+    elif dtype == "ct2":
+        assert ct2_mpath is not None
+        model = ctranslate2.Generator(ct2_mpath, device=device)
+
+    return model
+
+# @torch.no_grad()
+# def generate_prediction(
+#     inputs,
+#     model,
+#     tokenizer,
+#     max_new_tokens,
+#     is_seq2seq=False,
+#     **kwargs,
+#     # num_beams,
+#     # do_sample,
+#     # no_repeat_ngram_size,
+#     # temperature,
+#     # top_k,
+#     # top_p,
+# ):
+#     input_ids = tokenizer(inputs, return_tensors="pt", truncation=True, max_length=tokenizer.model_max_length - max_new_tokens).input_ids
+    
+#     outputs = model.generate(
+#         input_ids=input_ids.cuda(),
+#         max_new_tokens=max_new_tokens,
+#         **kwargs,
+#     ).cpu()
+
+#     torch.cuda.empty_cache()
+#     if not is_seq2seq:
+#         outputs = outputs[:, input_ids.shape[1] :]
+
+#     prediction = [
+#         p.split(tokenizer.pad_token, 1)[0]
+#         for p in tokenizer.batch_decode(outputs, skip_special_tokens=True)
+#     ][0].strip()
+        
+#     return prediction
+
+def generate_prediction(
+    inputs,
+    model,
+    tokenizer,
+    **kwargs,
+):
+    inputs = tokenizer([inputs], return_tensors='pt', truncation=True, add_special_tokens=False).to(model.device)
+    
+    # if torch.cuda.is_available():
+    #     inputs = inputs.to('cuda')
+    outputs = model.generate(
+        input_ids=inputs['input_ids'],
+        attention_mask=inputs['attention_mask'],
+        **kwargs,
+    )
+    
+    outputs = outputs[:, inputs.input_ids.shape[1] :]
+    prediction = tokenizer.decode(outputs[0], skip_special_tokens=True)
+        
+    return prediction

webagents_step/utils/stack.py

@@ -0,0 +1,22 @@
+class Stack:
+    def __init__(self):
+        self.items = []
+
+    def is_empty(self):
+        return len(self.items) == 0
+
+    def push(self, item):
+        self.items.append(item)
+
+    def pop(self):
+        if self.is_empty():
+            raise IndexError("pop from empty stack")
+        return self.items.pop()
+
+    def peek(self):
+        if self.is_empty():
+            raise IndexError("peek from empty stack")
+        return self.items[-1]
+
+    def size(self):
+        return len(self.items)