init

Agent_E/ae/utils/anthropic_llm_helper.py

@@ -0,0 +1,52 @@
+import os
+
+import anthropic
+from anthropic import AsyncAnthropic
+from dotenv import load_dotenv
+
+
+class AnthropicLLMHelper:
+    def __init__(self):
+        load_dotenv()
+        self.client = AsyncAnthropic(api_key=os.environ.get("ANTHROPIC_API_KEY"))
+
+    async def get_chat_completion_response_async(self, system_msg:str, user_msgs:list[str], model_name:str="claude-3-opus-20240229", temperature:float=0.1, max_tokens:int=256, top_p:int=1, top_k: int=1) -> str:
+        formatted_user_msgs: list[dict[str, str]] = []
+        for user_msg in user_msgs:
+            formatted_user_msgs.append({"type": "text", "text": user_msg})
+
+        try:
+            message = await self.client.messages.create(
+                model=model_name,
+                max_tokens=max_tokens,
+                temperature=temperature,
+                system=system_msg,
+                messages=[
+                    {
+                        "role": "user",
+                        "content": formatted_user_msgs # type: ignore
+
+                    }
+                ]
+            )
+            print(message)
+            return message.content[0].text
+        except anthropic.APIConnectionError as e:
+            print("The server could not be reached")
+            print(e.__cause__)  # an underlying Exception, likely raised within httpx.
+            raise Exception(f"Calling {model_name} LLM failed. The server could not be reached. Error: {e}")  # noqa: B904
+        except anthropic.RateLimitError as e:
+            print("A 429 status code was received; we should back off a bit.")
+            raise Exception(f"Calling {model_name} LLM failed. Rate limit error. Error: {e}")  # noqa: B904
+        except anthropic.APIStatusError as e:
+            print(e.status_code)
+            print(e.response)
+            raise Exception(f"Calling {model_name} LLM failed. Error: {e}")  # noqa: B904
+
+# async def main():
+#     from ae.core.prompts import LLM_PROMPTS
+#     helper = AnthropicLLMHelper()
+#     response = await helper.get_chat_completion_response_async(LLM_PROMPTS["SKILLS_HARVESTING_PROMPT"], ["What is the weather like today?"], temperature=0, max_tokens=4000)
+#     print("*******\nResponse: ", response, "\n*******\n")
+
+# asyncio.run(main())

Agent_E/ae/utils/autogen_sequential_function_call.py

@@ -0,0 +1,85 @@
+
+import asyncio
+import inspect
+from typing import Any
+
+from autogen import Agent  # type: ignore
+from autogen import UserProxyAgent  # type: ignore
+
+
+class UserProxyAgent_SequentialFunctionExecution(UserProxyAgent):
+    def __init__(self, *args, **kwargs): # type: ignore
+        super().__init__(*args, **kwargs) # type: ignore
+        #position = 2 allows termination check to be called earlier, this helps detect loops.
+        self.register_reply(Agent, UserProxyAgent_SequentialFunctionExecution.sequential_generate_tool_calls_reply, position=2) # type: ignore
+
+
+    def sequential_generate_tool_calls_reply( # type: ignore
+        self,
+        messages: list[dict] | None = None, # type: ignore
+        sender: Agent | None = None,
+        config: Any | None = None,
+    ) -> tuple[bool, dict[str, Any] | None]:
+        """Generate a reply using tool call."""
+        if config is None:
+            config = self
+        if messages is None:
+            messages = self._oai_messages[sender] # type: ignore
+        message = messages[-1] # type: ignore
+        tool_returns = []
+        skip_flag:bool = False
+        for tool_call in message.get("tool_calls", []): # type: ignore
+            function_call = tool_call.get("function", {}) # type: ignore
+            func = self._function_map.get(function_call.get("name", None), None) # type: ignore
+            func_return = None
+            if inspect.iscoroutinefunction(func): # type: ignore
+                try:
+                    # get the running loop if it was already created
+                    loop = asyncio.get_running_loop()
+                    close_loop = False
+                except RuntimeError:
+                    # create a loop if there is no running loop
+                    loop = asyncio.new_event_loop()
+                    close_loop = True
+                if (not skip_flag):
+                    _, func_return = loop.run_until_complete(self.a_execute_function(function_call)) # type: ignore
+                    if close_loop:
+                        loop.close()
+            else:
+                if (not skip_flag):
+                    _, func_return = self.execute_function(function_call) # type: ignore
+            if func_return is None: # type: ignore
+                if skip_flag:
+                    content = "VERY IMPORTANT: This function could not be executed since previous function resulted in a Webpage change. You must get all_fields DOM and repeat the function if needed."
+                else:
+                    content = ""
+            else:
+                content = func_return.get("content", "") # type: ignore
+
+            if content is None:
+                content = ""
+
+            if ("as a consequence of this action" in content.lower()): # type: ignore
+                skip_flag = True
+
+            tool_call_id = tool_call.get("id", None) # type: ignore
+            if tool_call_id is not None:
+                tool_call_response = { # type: ignore
+                    "tool_call_id": tool_call_id,
+                    "role": "tool",
+                    "content": content,
+                }
+            else:
+                tool_call_response = { # type: ignore
+                    "role": "tool",
+                    "content": content,
+                }
+            tool_returns.append(tool_call_response) # type: ignore
+
+        if tool_returns:
+            return True, {
+                "role": "tool",
+                "tool_responses": tool_returns,
+                "content": "\n\n".join([self._str_for_tool_response(tool_return) for tool_return in tool_returns]), # type: ignore
+            }
+        return False, None

Agent_E/ae/utils/cli_helper.py

@@ -0,0 +1,34 @@
+import asyncio
+from asyncio import Future
+
+
+def async_input(prompt: str) -> Future: # type: ignore
+    """
+    Display a prompt to the user and wait for input in an asynchronous manner.
+
+    Parameters:
+    - prompt: The message to display to the user.
+
+    Returns:
+    - A Future object that will be fulfilled with the user's input.
+    """
+    loop = asyncio.get_event_loop()
+    return loop.run_in_executor(None, input, prompt)
+
+
+async def answer_questions_over_cli(questions: list[str]) -> dict[str, str]:
+    """
+    Asks a question over the command line and awaits the user's response.
+
+    Parameters:
+    - questions: A list of questions to ask the user, e.g., ["What is your favorite site?", "What do you want to search for?"].
+
+    Returns:
+    - A dictionary where each key is a question and each value is the user's response.
+    """
+    answers: dict[str, str] = {}
+    print("*********************************")
+    for question in questions:
+        answers[question] = await async_input("Question: "+str(question)+" : ")
+    print("*********************************")
+    return answers

Agent_E/ae/utils/detect_llm_loops.py

@@ -0,0 +1,46 @@
+from typing import Any
+
+from Agent_E.ae.utils.logger import logger
+
+
+def is_agent_stuck_in_loop(messages: list[dict[str, Any]]) -> bool:
+    """
+    Detects loops in the agent's responses by iterating over the last N responses.
+
+    Parameters
+    ----------
+    messages : list[dict[str, Any]]
+        A list of dictionaries representing the agent's messages.
+
+    Returns
+    -------
+    bool
+        True if a loop is detected, False otherwise.
+    """
+    number_of_turns_to_check_for_loops: int = 6
+    number_of_rounds_to_check_for_loops: int = number_of_turns_to_check_for_loops // 2 #integer division since we are checking for pairs of messages and can't have fractions
+    # Detect any loops by checking the last number_of_rounds_to_check_for_loops tool responses and their corresponding tool calls
+    if len(messages) > number_of_turns_to_check_for_loops:
+        last_six_items = messages[-number_of_turns_to_check_for_loops:]
+        logger.debug(f"More than {number_of_turns_to_check_for_loops} messages in the conversation. Checking for loops..")
+        # Filter items by role
+        tool_calls = [item for item in last_six_items if item.get("role") == "assistant"]
+
+        # Check if function attributes are the same for tool items
+        if tool_calls:
+            tool_functions = [item.get("tool_calls", [{}])[0].get("function") for item in tool_calls]
+            logger.debug(f"Last {number_of_rounds_to_check_for_loops} tool calls: {tool_functions}")
+            if all(func == tool_functions[0] for func in tool_functions):
+                logger.debug(f"Last {number_of_rounds_to_check_for_loops} tool calls are identical. Checking Tool responses..")
+                # Check if content attributes are the same for assistant items
+                tool_responses = [item for item in last_six_items if item.get("role") == "tool"]
+
+                if tool_responses:
+                    assistant_contents = [item.get("content") for item in tool_responses]
+                    logger.debug(f"Last N tool responses: {assistant_contents}")
+                    if all(content == assistant_contents[0] for content in assistant_contents):
+                        logger.debug(f"Last {number_of_rounds_to_check_for_loops} tool responses are identical. Terminating")
+                        logger.info("Terminating browser executor since a loop was detected...")
+                        return True
+
+    return False

Agent_E/ae/utils/dom_helper.py

@@ -0,0 +1,45 @@
+import asyncio
+
+from playwright.async_api import ElementHandle
+from playwright.async_api import Page
+
+from Agent_E.ae.utils.logger import logger
+
+
+async def wait_for_non_loading_dom_state(page: Page, max_wait_millis: int):
+    max_wait_seconds = max_wait_millis / 1000
+    end_time = asyncio.get_event_loop().time() + max_wait_seconds
+    while asyncio.get_event_loop().time() < end_time:
+        dom_state = await page.evaluate("document.readyState")
+        if dom_state != "loading":
+            logger.debug(f"DOM state is not 'loading': {dom_state}")
+            break  # Exit the loop if the DOM state is not 'loading'
+
+        await asyncio.sleep(0.05)
+
+
+async def get_element_outer_html(element: ElementHandle, page: Page, element_tag_name: str|None = None) -> str:
+    """
+    Constructs the opening tag of an HTML element along with its attributes.
+
+    Args:
+        element (ElementHandle): The element to retrieve the opening tag for.
+        page (Page): The page object associated with the element.
+        element_tag_name (str, optional): The tag name of the element. Defaults to None. If not passed, it will be retrieved from the element.
+
+    Returns:
+        str: The opening tag of the HTML element, including a select set of attributes.
+    """
+    tag_name: str = element_tag_name if element_tag_name else await page.evaluate("element => element.tagName.toLowerCase()", element)
+
+    attributes_of_interest: list[str] = ['id', 'name', 'aria-label', 'placeholder', 'href', 'src', 'aria-autocomplete', 'role', 'type',
+                                         'data-testid', 'value', 'selected', 'aria-labelledby', 'aria-describedby', 'aria-haspopup']
+    opening_tag: str = f'<{tag_name}'
+
+    for attr in attributes_of_interest:
+        value: str = await element.get_attribute(attr) # type: ignore
+        if value:
+            opening_tag += f' {attr}="{value}"'
+    opening_tag += '>'
+
+    return opening_tag

Agent_E/ae/utils/dom_mutation_observer.py

@@ -0,0 +1,88 @@
+
+import asyncio
+import json
+from typing import Callable  # noqa: UP035
+
+from playwright.async_api import Page
+
+# Create an event loop
+loop = asyncio.get_event_loop()
+
+DOM_change_callback: list[Callable[[str], None]] = []
+
+def subscribe(callback: Callable[[str], None]) -> None:
+    DOM_change_callback.append(callback)
+
+def unsubscribe(callback: Callable[[str], None]) -> None:
+    DOM_change_callback.remove(callback)
+
+
+async def add_mutation_observer(page:Page):
+    """
+    Adds a mutation observer to the page to detect changes in the DOM.
+    When changes are detected, the observer calls the dom_mutation_change_detected function in the browser context.
+    This changes can be detected by subscribing to the dom_mutation_change_detected function by individual skills.
+
+    Current implementation only detects when a new node is added to the DOM.
+    However, in many cases, the change could be a change in the style or class of an existing node (e.g. toggle visibility of a hidden node).
+    """
+
+    await page.evaluate("""
+        console.log('Adding a mutation observer for DOM changes');
+        new MutationObserver((mutationsList, observer) => {
+            let changes_detected = [];
+            for(let mutation of mutationsList) {
+                if (mutation.type === 'childList') {
+                    let allAddedNodes=mutation.addedNodes;
+                    for(let node of allAddedNodes) {
+                        if(node.tagName && !['SCRIPT', 'NOSCRIPT', 'STYLE'].includes(node.tagName) && !node.closest('#agentDriveAutoOverlay')) {
+                            let visibility=true;
+                            let content = node.innerText.trim();
+                            if(visibility && node.innerText.trim()){
+                                if(content) {
+                                    changes_detected.push({tag: node.tagName, content: content});
+                                }
+                            }
+                        }
+                    }
+                } else if (mutation.type === 'characterData') {
+                    let node = mutation.target;
+                    if(node.parentNode && !['SCRIPT', 'NOSCRIPT', 'STYLE'].includes(node.parentNode.tagName) && !node.parentNode.closest('#agentDriveAutoOverlay')) {
+                        let visibility=true;
+                        let content = node.data.trim();
+                        if(visibility && content && window.getComputedStyle(node.parentNode).display !== 'none'){
+                            if(content && !changes_detected.some(change => change.content.includes(content))) {
+                                changes_detected.push({tag: node.parentNode.tagName, content: content});
+                            }
+                        }
+                    }
+                }
+            }
+            if(changes_detected.length > 0) {
+                window.dom_mutation_change_detected(JSON.stringify(changes_detected));
+            }
+        }).observe(document, {subtree: true, childList: true, characterData: true});
+        """)
+
+
+async def handle_navigation_for_mutation_observer(page:Page):
+    await add_mutation_observer(page)
+
+async def dom_mutation_change_detected(changes_detected: str):
+    """
+    Detects changes in the DOM (new nodes added) and emits the event to all subscribed callbacks.
+    The changes_detected is a string in JSON formatt containing the tag and content of the new nodes added to the DOM.
+
+    e.g.  The following will be detected when autocomplete recommendations show up when one types Nelson Mandela on google search
+    [{'tag': 'SPAN', 'content': 'nelson mandela wikipedia'}, {'tag': 'SPAN', 'content': 'nelson mandela movies'}]
+    """
+    changes_detected = json.loads(changes_detected.replace('\t', '').replace('\n', ''))
+    if len(changes_detected) > 0:
+        # Emit the event to all subscribed callbacks
+        for callback in DOM_change_callback:
+            # If the callback is a coroutine function
+            if asyncio.iscoroutinefunction(callback):
+                await callback(changes_detected)
+            # If the callback is a regular function
+            else:
+                callback(changes_detected)

Agent_E/ae/utils/formatting_helper.py

@@ -0,0 +1,56 @@
+
+import json
+import re
+from typing import Any
+
+
+def str_to_bool(s: str | bool) -> bool:
+    """
+    Convert a string representation of truth to True or False.
+
+    Parameters:
+    s (str | bool): The string to convert, or a boolean.
+
+    Returns:
+    bool: True if the string represents a truth value, False otherwise.
+    """
+    if isinstance(s, bool):
+        return s
+    return s.lower() in ['true', '1', 't', 'y', 'yes']
+
+def str_to_json(s: str) -> dict[str, Any] | None:
+    """
+    Convert a string representation of a JSON object to a dictionary.
+
+    Parameters:
+    s (str): The string to convert.
+
+    Returns:
+    dict[str, Any] | None: The dictionary representation of the JSON object. If the parsing fails, returns None.
+    """
+    s_fixed = re.sub(r'(?<!\\)\n', '\\n', s) #escape newline characters as long as they are not already escaped
+
+# Now you can safely load it using json.loads
+    try:
+        obj = json.loads(s_fixed)
+        return obj
+    except json.JSONDecodeError as e:
+        return None
+
+def is_terminating_message(message: str) -> bool:
+    """
+    Check if a message is a terminating message.
+
+    Parameters:
+    message (str): The message to check.
+
+    Returns:
+    bool: True if the message is a terminating message, False otherwise.
+    """
+    message_as_json = str_to_json(message)
+    if message_as_json is None:
+        if message.find('"terminate": "yes"') != -1:
+            return True
+        return False
+    else:
+        return message_as_json.get("terminate") == "yes"

Agent_E/ae/utils/gemini_llm_helper.py

@@ -0,0 +1,77 @@
+import os
+import re
+from typing import Any
+
+import google.generativeai as genai  # type: ignore
+from dotenv import load_dotenv
+
+from Agent_E.ae.utils.logger import logger
+
+GCP_BLOCK_NONE_SAFETY_SETTINGS: list[dict[str, str]] = [
+    {
+        "category": "HARM_CATEGORY_HARASSMENT",
+        "threshold": "BLOCK_NONE"
+    },
+    {
+        "category": "HARM_CATEGORY_HATE_SPEECH",
+        "threshold": "BLOCK_NONE"
+    },
+    {
+        "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
+        "threshold": "BLOCK_NONE"
+    },
+    {
+        "category": "HARM_CATEGORY_DANGEROUS_CONTENT",
+        "threshold": "BLOCK_NONE"
+    },
+]
+
+# Pre-compile the regular expression pattern for removing json markers from LLM response
+llm_json_or_python_begin_response_pattern = re.compile(r"^```(python|json)?\n?")
+llm_end_response_pattern = re.compile(r"```$")
+
+class GeminiLLMHelper:
+    def __init__(self):
+        load_dotenv()
+        genai.configure(api_key=os.environ.get("GEMINI_API_KEY")) # type: ignore
+
+    def process_llm_response(self, response: str):
+        if response:
+            # Use the compiled regex to replace the patterns with an empty string
+            response = llm_json_or_python_begin_response_pattern.sub("", response)
+            response = llm_end_response_pattern.sub("", response)
+        return response
+
+    async def get_chat_completion_response_async(self, system_msg:str, user_msgs:list[str], model_name:str="gemini-1.5-pro-latest", temperature:float=0.1,
+                                                 max_tokens:int=256, top_p:int=1, top_k: int=1, safety_settings:list[dict[str, str]]=GCP_BLOCK_NONE_SAFETY_SETTINGS) -> str|None:
+        formatted_msgs: list[dict[str, Any]] = [{"role": "user", "parts": [system_msg]}]
+        user_msgs_parts: list[str] = []
+        for user_msg in user_msgs:
+            user_msgs_parts.append(user_msg)
+
+        formatted_msgs.append({"role": "user", "parts": user_msgs_parts})
+        response = None
+        try:
+            model = genai.GenerativeModel(model_name)
+            response = model.generate_content(formatted_msgs, stream=False,  # type: ignore
+                                              generation_config=genai.types.GenerationConfig(
+                                                  max_output_tokens=max_tokens,
+                                                  temperature=temperature,
+                                                  top_p=top_p,
+                                                  top_k=top_k),
+                                            safety_settings=safety_settings)
+            return self.process_llm_response(response.text)
+        except ValueError:
+            if response:
+                logger.error(f"Response from GCP Gen AI did not contain text. prompt: {system_msg} and user messages: {user_msgs}. Candidates: {response.candidates}")
+            else:
+                logger.error(f"There was no response from GCP Gen AI for prompt: {system_msg} and user messages: {user_msgs}")
+            return None
+
+# async def main():
+#     from Agent_E.ae.core.prompts import LLM_PROMPTS
+#     helper = GeminiLLMHelper()
+#     response = await helper.get_chat_completion_response_async(LLM_PROMPTS["SKILLS_HARVESTING_PROMPT"], ["What is the weather like today?", "And How are you?"], temperature=0, max_tokens=4000)
+#     print("*******\nResponse: ", response, "\n*******\n")
+
+# asyncio.run(main())

Agent_E/ae/utils/get_detailed_accessibility_tree.py

@@ -0,0 +1,529 @@
+import json
+import os
+import re
+import traceback
+from typing import Annotated
+from typing import Any
+
+from playwright.async_api import Page
+
+from Agent_E.ae.config import SOURCE_LOG_FOLDER_PATH
+from Agent_E.ae.core.playwright_manager import PlaywrightManager
+from Agent_E.ae.utils.logger import logger
+
+space_delimited_mmid = re.compile(r'^[\d ]+$')
+
+def is_space_delimited_mmid(s: str) -> bool:
+    """
+    Check if the given string matches the the mmid pattern of number space repeated.
+
+    Parameters:
+    - s (str): The string to check against the pattern.
+
+    Returns:
+    - bool: True if the string matches the pattern, False otherwise.
+    """
+    # Use fullmatch() to ensure the entire string matches the pattern
+    return bool(space_delimited_mmid.fullmatch(s))
+
+
+async def __inject_attributes(page: Page):
+    """
+    Injects 'mmid' and 'aria-keyshortcuts' into all DOM elements. If an element already has an 'aria-keyshortcuts',
+    it renames it to 'orig-aria-keyshortcuts' before injecting the new 'aria-keyshortcuts'
+    This will be captured in the accessibility tree and thus make it easier to reconcile the tree with the DOM.
+    'aria-keyshortcuts' is choosen because it is not widely used aria attribute.
+    """
+
+    last_mmid = await page.evaluate("""() => {
+        const allElements = document.querySelectorAll('*');
+        let id = 0;
+        allElements.forEach(element => {
+            const origAriaAttribute = element.getAttribute('aria-keyshortcuts');
+            const mmid = `${++id}`;
+            element.setAttribute('mmid', mmid);
+            element.setAttribute('aria-keyshortcuts', mmid);
+            //console.log(`Injected 'mmid'into element with tag: ${element.tagName} and mmid: ${mmid}`);
+            if (origAriaAttribute) {
+                element.setAttribute('orig-aria-keyshortcuts', origAriaAttribute);
+            }
+        });
+        return id;
+    }""")
+    logger.debug(f"Added MMID into {last_mmid} elements")
+
+
+async def __fetch_dom_info(page: Page, accessibility_tree: dict[str, Any], only_input_fields: bool):
+    """
+    Iterates over the accessibility tree, fetching additional information from the DOM based on 'mmid',
+    and constructs a new JSON structure with detailed information.
+
+    Args:
+        page (Page): The page object representing the web page.
+        accessibility_tree (dict[str, Any]): The accessibility tree JSON structure.
+        only_input_fields (bool): Flag indicating whether to include only input fields in the new JSON structure.
+
+    Returns:
+        dict[str, Any]: The pruned tree with detailed information from the DOM.
+    """
+
+    logger.debug("Reconciling the Accessibility Tree with the DOM")
+    # Define the attributes to fetch for each element
+    attributes = ['name', 'aria-label', 'placeholder', 'mmid', "id", "for", "data-testid"]
+    backup_attributes = [] #if the attributes are not found, then try to get these attributes
+    tags_to_ignore = ['head','style', 'script', 'link', 'meta', 'noscript', 'template', 'iframe', 'g', 'main', 'c-wiz','svg', 'path']
+    attributes_to_delete = ["level", "multiline", "haspopup", "id", "for"]
+    ids_to_ignore = ['agentDriveAutoOverlay']
+
+    # Recursive function to process each node in the accessibility tree
+    async def process_node(node: dict[str, Any]):
+        if 'children' in node:
+            for child in node['children']:
+                await process_node(child)
+
+        # Use 'name' attribute from the accessibility node as 'mmid'
+        mmid_temp: str = node.get('keyshortcuts') # type: ignore
+
+        # If the name has multiple mmids, take the last one
+        if(mmid_temp and is_space_delimited_mmid(mmid_temp)):
+            #TODO: consider if we should grab each of the mmids and process them separately as seperate nodes copying this node's attributes
+            mmid_temp = mmid_temp.split(' ')[-1]
+
+        #focusing on nodes with mmid, which is the attribute we inject
+        try:
+            mmid = int(mmid_temp)
+        except (ValueError, TypeError):
+            #logger.error(f"'name attribute contains \"{node.get('name')}\", which is not a valid numeric mmid. Adding node as is: {node}")
+            return node.get('name')
+
+        if node['role'] == 'menuitem':
+            return node.get('name')
+
+        if node.get('role') == 'dialog' and node.get('modal') == True:  # noqa: E712
+            node["important information"] = "This is a modal dialog. Please interact with this dialog and close it to be able to interact with the full page (e.g. by pressing the close button or selecting an option)."
+
+        if mmid:
+            # Determine if we need to fetch 'innerText' based on the absence of 'children' in the accessibility node
+            should_fetch_inner_text = 'children' not in node
+
+            js_code = """
+            (input_params) => {
+                const should_fetch_inner_text = input_params.should_fetch_inner_text;
+                const mmid = input_params.mmid;
+                const attributes = input_params.attributes;
+                const tags_to_ignore = input_params.tags_to_ignore;
+                const ids_to_ignore = input_params.ids_to_ignore;
+
+                const element = document.querySelector(`[mmid="${mmid}"]`);
+
+                if (!element) {
+                    console.log(`No element found with mmid: ${mmid}`);
+                    return null;
+                }
+
+                if (ids_to_ignore.includes(element.id)) {
+                    console.log(`Ignoring element with id: ${element.id}`, element);
+                    return null;
+                }
+                //Ignore "option" because it would have been processed with the select element
+                if (tags_to_ignore.includes(element.tagName.toLowerCase()) || element.tagName.toLowerCase() === "option") return null;
+
+                let attributes_to_values = {
+                    'tag': element.tagName.toLowerCase() // Always include the tag name
+                };
+
+                // If the element is an input, include its type as well
+                if (element.tagName.toLowerCase() === 'input') {
+                    attributes_to_values['tag_type'] = element.type; // This will capture 'checkbox', 'radio', etc.
+                }
+                else if (element.tagName.toLowerCase() === 'select') {
+                    attributes_to_values["mmid"] = element.getAttribute('mmid');
+                    attributes_to_values["role"] = "combobox";
+                    attributes_to_values["options"] = [];
+
+                    for (const option of element.options) {
+                        let option_attributes_to_values = {
+                            "mmid": option.getAttribute('mmid'),
+                            "text": option.text,
+                            "value": option.value,
+                            "selected": option.selected
+                        };
+                        attributes_to_values["options"].push(option_attributes_to_values);
+                    }
+                    return attributes_to_values;
+                }
+
+                for (const attribute of attributes) {
+                    let value = element.getAttribute(attribute);
+
+                    if(value){
+                        /*
+                        if(attribute === 'href'){
+                            value = value.split('?')[0]
+                        }
+                        */
+                        attributes_to_values[attribute] = value;
+                    }
+                }
+
+                if (should_fetch_inner_text && element.innerText) {
+                    attributes_to_values['description'] = element.innerText;
+                }
+
+                let role = element.getAttribute('role');
+                if(role==='listbox' || element.tagName.toLowerCase()=== 'ul'){
+                    let children=element.children;
+                    let filtered_children = Array.from(children).filter(child => child.getAttribute('role') === 'option');
+                    console.log("Listbox or ul found: ", filtered_children);
+                    let attributes_to_include = ['mmid', 'role', 'aria-label','value'];
+                    attributes_to_values["additional_info"]=[]
+                    for (const child of children) {
+                        let children_attributes_to_values = {};
+
+                        for (let attr of child.attributes) {
+                            // If the attribute is not in the predefined list, add it to children_attributes_to_values
+                            if (attributes_to_include.includes(attr.name)) {
+                                children_attributes_to_values[attr.name] = attr.value;
+                            }
+                        }
+
+                        attributes_to_values["additional_info"].push(children_attributes_to_values);
+                    }
+                }
+                // Check if attributes_to_values contains more than just 'name', 'role', and 'mmid'
+                const keys = Object.keys(attributes_to_values);
+                const minimalKeys = ['tag', 'mmid'];
+                const hasMoreThanMinimalKeys = keys.length > minimalKeys.length || keys.some(key => !minimalKeys.includes(key));
+
+                if (!hasMoreThanMinimalKeys) {
+                    //If there were no attributes found, then try to get the backup attributes
+                    for (const backupAttribute of input_params.backup_attributes) {
+                        let value = element.getAttribute(backupAttribute);
+                        if(value){
+                            attributes_to_values[backupAttribute] = value;
+                        }
+                    }
+
+                    //if even the backup attributes are not found, then return null, which will cause this element to be skipped
+                    if(Object.keys(attributes_to_values).length <= minimalKeys.length) {
+                        if (element.tagName.toLowerCase() === 'button') {
+                                attributes_to_values["mmid"] = element.getAttribute('mmid');
+                                attributes_to_values["role"] = "button";
+                                attributes_to_values["additional_info"] = [];
+                                let children=element.children;
+                                let attributes_to_exclude = ['width', 'height', 'path', 'class', 'viewBox', 'mmid']
+
+                                // Check if the button has no text and no attributes
+                                if (element.innerText.trim() === '') {
+
+                                    for (const child of children) {
+                                        let children_attributes_to_values = {};
+
+                                        for (let attr of child.attributes) {
+                                            // If the attribute is not in the predefined list, add it to children_attributes_to_values
+                                            if (!attributes_to_exclude.includes(attr.name)) {
+                                                children_attributes_to_values[attr.name] = attr.value;
+                                            }
+                                        }
+
+                                        attributes_to_values["additional_info"].push(children_attributes_to_values);
+                                    }
+                                    console.log("Button with no text and no attributes: ", attributes_to_values);
+                                    return attributes_to_values;
+                                }
+                        }
+
+                        return null; // Return null if only minimal keys are present
+                    }
+                }
+                return attributes_to_values;
+            }
+            """
+
+            # Fetch attributes and possibly 'innerText' from the DOM element by 'mmid'
+            element_attributes = await page.evaluate(js_code,
+                                                     {"mmid": mmid, "attributes": attributes, "backup_attributes": backup_attributes,
+                                                      "should_fetch_inner_text": should_fetch_inner_text,
+                                                      "tags_to_ignore": tags_to_ignore,
+                                                      "ids_to_ignore": ids_to_ignore})
+
+            if 'keyshortcuts' in node:
+                    del node['keyshortcuts'] #remove keyshortcuts since it is not needed
+
+            node["mmid"]=mmid
+
+            # Update the node with fetched information
+            if element_attributes:
+                node.update(element_attributes)
+
+                # check if 'name' and 'mmid' are the same
+                if node.get('name') == node.get('mmid') and node.get('role') != "textbox":
+                    del node['name']  # Remove 'name' from the node
+
+                if 'name' in node and 'description' in node and (node['name'] == node['description'] or node['name'] == node['description'].replace('\n', ' ') or node['description'].replace('\n', '') in node['name']):
+                    del node['description'] #if the name is same as description, then remove the description to avoid duplication
+
+                if 'name' in node and 'aria-label' in node and  node['aria-label'] in node['name']:
+                    del node['aria-label'] #if the name is same as the aria-label, then remove the aria-label to avoid duplication
+
+                if 'name' in node and 'text' in node and node['name'] == node['text']:
+                    del node['text'] #if the name is same as the text, then remove the text to avoid duplication
+
+                if node.get('tag') == "select": #children are not needed for select menus since "options" attriburte is already added
+                    node.pop("children", None)
+                    node.pop("role", None)
+                    node.pop("description", None)
+
+                #role and tag can have the same info. Get rid of role if it is the same as tag
+                if node.get('role') == node.get('tag'):
+                    del node['role']
+
+                # avoid duplicate aria-label
+                if node.get("aria-label") and node.get("placeholder") and node.get("aria-label") == node.get("placeholder"):
+                    del node["aria-label"]
+
+                if node.get("role") == "link":
+                    del node["role"]
+                    if node.get("description"):
+                        node["text"] = node["description"]
+                        del node["description"]
+
+                #textbox just means a text input and that is expressed well enough with the rest of the attributes returned
+                #if node.get('role') == "textbox":
+                #    del node['role']
+
+                if node.get('role') == "textbox":
+                    #get the id attribute of this field from the DOM
+                    if "id" in element_attributes and element_attributes["id"]:
+                        #find if there is an element in the DOM that has this id in aria-labelledby.
+                        js_code = """
+                        (inputParams) => {
+                            let referencingElements = [];
+                            const referencedElement = document.querySelector(`[aria-labelledby="${inputParams.aria_labelled_by_query_value}"]`);
+                            if(referencedElement) {
+                                const mmid = referencedElement.getAttribute('mmid');
+                                if (mmid) {
+                                    return {"mmid": mmid, "tag": referencedElement.tagName.toLowerCase()};
+                                }
+                            }
+                            return null;
+                        }
+                        """
+                    #textbox just means a text input and that is expressed well enough with the rest of the attributes returned
+                    #del node['role']
+
+            #remove attributes that are not needed once processing of a node is complete
+            for attribute_to_delete in attributes_to_delete:
+                if attribute_to_delete in node:
+                    node.pop(attribute_to_delete, None)
+        else:
+            logger.debug(f"No element found with mmid: {mmid}, deleting node: {node}")
+            node["marked_for_deletion_by_mm"] = True
+
+
+    # Process each node in the tree starting from the root
+    await process_node(accessibility_tree)
+
+    pruned_tree = __prune_tree(accessibility_tree, only_input_fields)
+
+    logger.debug("Reconciliation complete")
+    return pruned_tree
+
+
+async def __cleanup_dom(page: Page):
+    """
+    Cleans up the DOM by removing injected 'aria-description' attributes and restoring any original 'aria-keyshortcuts'
+    from 'orig-aria-keyshortcuts'.
+    """
+    logger.debug("Cleaning up the DOM's previous injections")
+    await page.evaluate("""() => {
+        const allElements = document.querySelectorAll('*[mmid]');
+        allElements.forEach(element => {
+            element.removeAttribute('aria-keyshortcuts');
+            const origAriaLabel = element.getAttribute('orig-aria-keyshortcuts');
+            if (origAriaLabel) {
+                element.setAttribute('aria-keyshortcuts', origAriaLabel);
+                element.removeAttribute('orig-aria-keyshortcuts');
+            }
+        });
+    }""")
+    logger.debug("DOM cleanup complete")
+
+
+def __prune_tree(node: dict[str, Any], only_input_fields: bool) -> dict[str, Any] | None:
+    """
+    Recursively prunes a tree starting from `node`, based on pruning conditions and handling of 'unraveling'.
+
+    The function has two main jobs:
+    1. Pruning: Remove nodes that don't meet certain conditions, like being marked for deletion.
+    2. Unraveling: For nodes marked with 'marked_for_unravel_children', we replace them with their children,
+       effectively removing the node and lifting its children up a level in the tree.
+
+    This happens in place, meaning we modify the tree as we go, which is efficient but means you should
+    be cautious about modifying the tree outside this function during a prune operation.
+
+    Args:
+    - node (Dict[str, Any]): The node we're currently looking at. We'll check this node, its children,
+      and so on, recursively down the tree.
+    - only_input_fields (bool): If True, we're only interested in pruning input-related nodes (like form fields).
+      This lets you narrow the focus if, for example, you're only interested in cleaning up form-related parts
+      of a larger tree.
+
+    Returns:
+    - dict[str, Any] | None: The pruned version of `node`, or None if `node` was pruned away. When we 'unravel'
+      a node, we directly replace it with its children in the parent's list of children, so the return value
+      will be the parent, updated in place.
+
+    Notes:
+    - 'marked_for_deletion_by_mm' is our flag for nodes that should definitely be removed.
+    - Unraveling is neat for flattening the tree when a node is just a wrapper without semantic meaning.
+    - We use a while loop with manual index management to safely modify the list of children as we iterate over it.
+    """
+    if "marked_for_deletion_by_mm" in node:
+        return None
+
+    if 'children' in node:
+        i = 0
+        while i < len(node['children']):
+            child = node['children'][i]
+            if 'marked_for_unravel_children' in child:
+                # Replace the current child with its children
+                if 'children' in child:
+                    node['children'] = node['children'][:i] + child['children'] + node['children'][i+1:]
+                    i += len(child['children']) - 1  # Adjust the index for the new children
+                else:
+                    # If the node marked for unraveling has no children, remove it
+                    node['children'].pop(i)
+                    i -= 1  # Adjust the index since we removed an element
+            else:
+                # Recursively prune the child if it's not marked for unraveling
+                pruned_child = __prune_tree(child, only_input_fields)
+                if pruned_child is None:
+                    # If the child is pruned, remove it from the children list
+                    node['children'].pop(i)
+                    i -= 1  # Adjust the index since we removed an element
+                else:
+                    # Update the child with the pruned version
+                    node['children'][i] = pruned_child
+            i += 1  # Move to the next child
+
+        # After processing all children, if the children array is empty, remove it
+        if not node['children']:
+            del node['children']
+
+    # Apply existing conditions to decide if the current node should be pruned
+    return None if __should_prune_node(node, only_input_fields) else node
+
+
+def __should_prune_node(node: dict[str, Any], only_input_fields: bool):
+    """
+    Determines if a node should be pruned based on its 'role' and 'element_attributes'.
+
+    Args:
+        node (dict[str, Any]): The node to be evaluated.
+        only_input_fields (bool): Flag indicating whether only input fields should be considered.
+
+    Returns:
+        bool: True if the node should be pruned, False otherwise.
+    """
+    #If the request is for only input fields and this is not an input field, then mark the node for prunning
+    if node.get("role") != "WebArea" and only_input_fields and not (node.get("tag") in ("input", "button", "textarea") or node.get("role") == "button"):
+        return True
+
+    if node.get('role') == 'generic' and 'children' not in node and not ('name' in node and node.get('name')):  # The presence of 'children' is checked after potentially deleting it above
+        return True
+
+    if node.get('role') in ['separator', 'LineBreak']:
+        return True
+    processed_name = ""
+    if 'name' in node:
+        processed_name:str =node.get('name') # type: ignore
+        processed_name = processed_name.replace(',', '')
+        processed_name = processed_name.replace(':', '')
+        processed_name = processed_name.replace('\n', '')
+        processed_name = processed_name.strip()
+        if len(processed_name) <3:
+            processed_name = ""
+
+    #check if the node only have name and role, then delete that node
+    if len(node) == 2 and 'name' in node and 'role' in node and not (node.get('role') == "text" and processed_name != ""):
+        return True
+    return False
+
+async def get_node_dom_element(page: Page, mmid: str):
+    return await page.evaluate("""
+        (mmid) => {
+            return document.querySelector(`[mmid="${mmid}"]`);
+        }
+    """, mmid)
+
+
+async def get_element_attributes(page: Page, mmid: str, attributes: list[str]):
+    return await page.evaluate("""
+        (inputParams) => {
+            const mmid = inputParams.mmid;
+            const attributes = inputParams.attributes;
+            const element = document.querySelector(`[mmid="${mmid}"]`);
+            if (!element) return null;  // Return null if element is not found
+
+            let attrs = {};
+            for (let attr of attributes) {
+                attrs[attr] = element.getAttribute(attr);
+            }
+            return attrs;
+        }
+    """, {"mmid": mmid, "attributes": attributes})
+
+
+async def get_dom_with_accessibility_info() -> Annotated[dict[str, Any] | None, "A minified representation of the HTML DOM for the current webpage"]:
+    """
+    Retrieves, processes, and minifies the Accessibility tree of the active page in a browser instance.
+    Strictly follow the name and role tag for any interaction with the nodes.
+
+    Returns:
+    - The minified JSON content of the browser's active page.
+    """
+    logger.debug("Executing Get Accessibility Tree Command")
+    # Create and use the PlaywrightManager
+    browser_manager = PlaywrightManager(browser_type='chromium', headless=False)
+    page = await browser_manager.get_current_page()
+    if page is None: # type: ignore
+        raise ValueError('No active page found')
+
+    return await do_get_accessibility_info(page)
+
+
+async def do_get_accessibility_info(page: Page, only_input_fields: bool = False):
+    """
+    Retrieves the accessibility information of a web page and saves it as JSON files.
+
+    Args:
+        page (Page): The page object representing the web page.
+        only_input_fields (bool, optional): If True, only retrieves accessibility information for input fields.
+            Defaults to False.
+
+    Returns:
+        dict[str, Any] or None: The enhanced accessibility tree as a dictionary, or None if an error occurred.
+    """
+    await __inject_attributes(page)
+    accessibility_tree: dict[str, Any] = await page.accessibility.snapshot(interesting_only=True)  # type: ignore
+
+    with open(os.path.join(SOURCE_LOG_FOLDER_PATH, 'json_accessibility_dom.json'), 'w',  encoding='utf-8') as f:
+        f.write(json.dumps(accessibility_tree, indent=2))
+        logger.debug("json_accessibility_dom.json saved")
+
+    await __cleanup_dom(page)
+    try:
+        enhanced_tree = await __fetch_dom_info(page, accessibility_tree, only_input_fields)
+
+        logger.debug("Enhanced Accessibility Tree ready")
+
+        with open(os.path.join(SOURCE_LOG_FOLDER_PATH, 'json_accessibility_dom_enriched.json'), 'w',  encoding='utf-8') as f:
+            f.write(json.dumps(enhanced_tree, indent=2))
+            logger.debug("json_accessibility_dom_enriched.json saved")
+
+        return enhanced_tree
+    except Exception as e:
+        logger.error(f"Error while fetching DOM info: {e}")
+        traceback.print_exc()
+        return None

Agent_E/ae/utils/http_helper.py

@@ -0,0 +1,43 @@
+from typing import Any
+
+import requests
+
+
+def make_post_request(url: str, data: dict[str, Any], api_key: str, api_key_header_name: str = "apikey") -> dict[str, Any]|None:
+    """
+    Makes a POST request to the specified URL with a JSON body and an API key header.
+
+    Args:
+        url (str): The URL to send the POST request to.
+        data (Dict[str, Any]): The JSON data to include in the POST request body.
+        api_key (str): The API key to include in the request headers.
+        api_key_header_name (str): The name of the header to include the API key in. Defaults to "apikey".
+
+    Returns:
+        Optional[Dict[str, Any]]: The JSON response from the server if the request was successful and the response is in JSON format.
+        None: If the request failed or the response is not in JSON format.
+
+    Raises:
+        requests.exceptions.RequestException: If an error occurs during the HTTP request.
+    """
+    # Define the headers for the request
+    headers = {
+        'Content-Type': 'application/json',
+        api_key_header_name: api_key
+    }
+
+    try:
+        # Make the POST request with the given URL, data, and headers
+        response = requests.post(url, json=data, headers=headers)
+
+        # Check if the request was successful
+        response.raise_for_status()
+
+        # Attempt to return the JSON response
+        return response.json()
+    except requests.exceptions.RequestException as e:
+        print(f"Error: {e}")
+        return None
+    except ValueError:
+        print("Error: Response is not in JSON format")
+        return None

Agent_E/ae/utils/js_helper.py

@@ -0,0 +1,34 @@
+import json
+import re
+
+from Agent_E.ae.utils.logger import logger
+
+
+def escape_js_message(message: str) -> str:
+    """
+    Escape a message for use in JavaScript code.
+
+    Args:
+        message (str): The message to escape.
+
+    Returns:
+        str: The escaped message.
+    """
+    return json.dumps(message)
+
+
+def beautify_plan_message(message:str) -> str:
+    """
+    Add a newline between each numbered step in the plan message if it does not already exist.
+
+    Args:
+        message (str): The plan message.
+
+    Returns:
+        str: The plan message with newlines added between each numbered step.
+    """
+    logger.debug(f"beautify_plan_message original:\n{message}")
+    # Add a newline before each numbered step that is not already preceded by a newline
+    plan_with_newlines = re.sub(r'(?<!\n)( \d+\.)', r'\n\1', message)
+    logger.debug(f"beautify_plan_message modified:\n{plan_with_newlines}")
+    return plan_with_newlines

Agent_E/ae/utils/logger.py

@@ -0,0 +1,71 @@
+import logging
+import os
+
+from dotenv import load_dotenv
+from pythonjsonlogger import jsonlogger
+
+# Load environment variables from a .env file
+load_dotenv()
+
+logger = logging.getLogger(__name__)
+
+# Custom function to configure the logger
+def configure_logger(level: str = "INFO") -> None:
+    log_format = os.getenv("LOG_MESSAGES_FORMAT", "text").lower()
+
+    # Set log level for the main logger
+    logger.setLevel(level.upper())
+
+    # Create a handler for logging
+    handler = logging.StreamHandler()
+
+    if log_format == "json":
+        # JSON format
+        formatter = jsonlogger.JsonFormatter(
+            fmt='%(asctime)s %(name)s %(levelname)s %(message)s %(filename)s %(lineno)d',
+            datefmt='%Y-%m-%d %H:%M:%S'
+        )
+    else:
+        # Text format
+        formatter = logging.Formatter(
+            fmt='[%(asctime)s] %(levelname)s {%(filename)s:%(lineno)d} - %(message)s',
+            datefmt='%Y-%m-%d %H:%M:%S'
+        )
+
+    handler.setFormatter(formatter)
+    logger.handlers = []  # Clear existing handlers
+    logger.addHandler(handler)
+
+    # Ensure other loggers have the same handler
+    http_loggers = ["openai", "autogen"]
+    for http_logger in http_loggers:
+        lib_logger = logging.getLogger(http_logger)
+        lib_logger.setLevel(logging.DEBUG)
+        lib_logger.handlers = []  # Clear any existing handlers
+        lib_logger.addHandler(handler)  # Add the same handler
+
+
+# Call the configure logger function to set up the logger initially
+configure_logger(level="INFO")
+
+# Function to set log level
+def set_log_level(level: str) -> None:
+    """
+    Set the log level for the logger.
+
+    Parameters:
+    - level (str): A logging level such as 'debug', 'info', 'warning', 'error', or 'critical'.
+    """
+    configure_logger(level)
+
+# Set default log levels for other libraries
+# logging.getLogger("httpcore").setLevel(logging.DEBUG)
+# logging.getLogger("httpx").setLevel(logging.DEBUG)
+# logging.getLogger("openai").setLevel(logging.DEBUG)
+# logging.getLogger("autogen").setLevel(logging.DEBUG)
+logging.getLogger("matplotlib.pyplot").setLevel(logging.WARNING)
+logging.getLogger("PIL.PngImagePlugin").setLevel(logging.WARNING)
+logging.getLogger("PIL.Image").setLevel(logging.WARNING)
+
+# Re-export the logger for ease of use
+__all__ = ["logger", "set_log_level"]

Agent_E/ae/utils/openai_llm_helper.py

@@ -0,0 +1,51 @@
+import os
+from typing import Any
+
+import openai
+from dotenv import load_dotenv
+from openai import AsyncOpenAI
+
+
+class OpenAILLMHelper:
+    def __init__(self):
+        load_dotenv()
+        self.client = AsyncOpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
+
+    async def get_chat_completion_response_async(self, system_msg:str, user_msgs:list[str], model_name:str="gpt-4-turbo-preview", temperature:float=0.1, max_tokens:int=256, frequency_penalty:float=0.0, top_p: float=1.0, top_k: int=1, presence_penalty: float=0.0):
+        formatted_msgs: list[dict[str, Any]] = [{"role": "system", "content": system_msg}]
+
+        for user_msg in user_msgs:
+            formatted_msgs.append({"role": "user", "content": user_msg})
+
+        try:
+            response = await self.client.chat.completions.create(
+                model=model_name,
+                max_tokens=max_tokens,
+                temperature=temperature,
+                frequency_penalty=frequency_penalty,
+                top_p=top_p,
+                presence_penalty=presence_penalty,
+                messages=formatted_msgs # type: ignore
+            )
+            print(">>> openai response:", response)
+            if response.choices and len(response.choices) > 0 and response.choices[0].message and response.choices[0].message.content:
+                return response.choices[0].message.content
+            return None
+        except openai.APIConnectionError as e:
+            print("The server could not be reached")
+            print(e.__cause__)  # an underlying Exception, likely raised within httpx.
+            raise Exception(f"Calling {model_name} LLM failed. The server could not be reached.") from e
+        except openai.RateLimitError as e:
+            print("A 429 status code was received; we should back off a bit.")
+            raise Exception(f"Calling {model_name} LLM failed. Rate limit error.") from e
+        except openai.APIStatusError as e:
+            print(e.status_code)
+            print(e.response)
+            raise Exception(f"Calling {model_name} LLM failed. Error: {e}") from e
+
+# async def main():
+#     helper = OpenAILLMHelper()
+#     response = await helper.get_chat_completion_response_async(LLM_PROMPTS["SKILLS_HARVESTING_PROMPT"], ["What is the weather like today?"], temperature=0, max_tokens=4000)
+#     print("*******\nResponse: ", response, "\n*******\n")
+
+# asyncio.run(main())

Agent_E/ae/utils/response_parser.py

@@ -0,0 +1,60 @@
+import json
+from typing import Any
+
+from Agent_E.ae.utils.logger import logger
+
+
+def parse_response(message: str) -> dict[str, Any]:
+    """
+    Parse the response from the browser agent and return the response as a dictionary.
+    """
+    # Parse the response content
+    json_response = {}
+    #if message starts with ``` and ends with ``` then remove them
+    if message.startswith("```"):
+        message = message[3:]
+    if message.endswith("```"):
+        message = message[:-3]
+    if message.startswith("json"):
+        message = message[4:]
+
+    message = message.strip()
+    try:
+        json_response: dict[str, Any] = json.loads(message)
+    except Exception as e:
+        # If the response is not a valid JSON, try pass it using string matching.
+        #This should seldom be triggered
+        logger.warn(f"LLM response was not properly formed JSON. Will try to use it as is. LLM response: \"{message}\". Error: {e}")
+        message = message.replace("\\n", "\n")
+        message = message.replace("\n", " ") # type: ignore
+        if ("plan" in message and "next_step" in message):
+            start = message.index("plan") + len("plan")
+            end = message.index("next_step")
+            json_response["plan"] = message[start:end].replace('"', '').strip()
+        if ("next_step" in message and "terminate" in message):
+            start = message.index("next_step") + len("next_step")
+            end = message.index("terminate")
+            json_response["next_step"] = message[start:end].replace('"', '').strip()
+        if ("terminate" in message and "final_response" in message):
+            start = message.index("terminate") + len("terminate")
+            end = message.index("final_response")
+            matched_string=message[start:end].replace('"', '').strip()
+            if ("yes" in matched_string):
+                json_response["terminate"] = "yes"
+            else:
+                json_response["terminate"] = "no"
+
+            start=message.index("final_response") + len("final_response")
+            end=len(message)-1
+            json_response["final_response"] = message[start:end].replace('"', '').strip()
+
+        elif ("terminate" in message):
+            start = message.index("terminate") + len("terminate")
+            end = len(message)-1
+            matched_string=message[start:end].replace('"', '').strip()
+            if ("yes" in matched_string):
+                json_response["terminate"] = "yes"
+            else:
+                json_response["terminate"] = "no"
+
+    return json_response

Agent_E/ae/utils/ui_messagetype.py

@@ -0,0 +1,14 @@
+from enum import Enum
+
+
+class MessageType(Enum):
+    PLAN = "plan"
+    STEP = "step"
+    ACTION ="action"
+    ANSWER = "answer"
+    QUESTION = "question"
+    INFO = "info"
+    FINAL = "final"
+    DONE = "transaction_done"
+    ERROR = "error"
+    MAX_TURNS_REACHED = "max_turns_reached"