init

Agent_E/test/evaluators.py

@@ -0,0 +1,437 @@
+"""base class for evaluation"""
+import collections
+import html
+import time
+import urllib
+import urllib.parse
+from Agent_E.test.test_utils import clean_answer
+from Agent_E.test.test_utils import evaluate_exact_match
+from Agent_E.test.test_utils import evaluate_fuzzy_match
+from Agent_E.test.test_utils import evaluate_must_include
+from Agent_E.test.test_utils import evaluate_ua_match
+from typing import Any
+
+from Agent_E.ae.utils.logger import logger
+from playwright.sync_api import CDPSession
+from playwright.sync_api import Page
+from termcolor import colored
+
+
+class Evaluator:
+    """Base class for evaluation strategies.
+
+    Attributes:
+        eval_tag (str): A tag to identify or categorize the evaluator.
+    """
+
+    def __init__(self, eval_tag: str = "") -> None:
+        """Initialize the evaluator with an optional evaluation tag."""
+        self.eval_tag = eval_tag
+
+    async def __call__(self, task_config: dict[str, Any], page: Page, client: CDPSession, answer: str) -> dict[str, float|str]:
+        """Abstract method to be implemented by subclasses for evaluation.
+
+        Raises:
+            NotImplementedError: This method should be overridden by subclasses.
+        """
+        raise NotImplementedError("This method should be overridden by subclasses.")
+
+
+class StringEvaluator(Evaluator):
+    """Evaluates string-based answers using various matching criteria.
+
+    Supports exact matches, some matches, fuzzy matching using LLM, and unachievable task matching.
+    """
+
+    async def __call__(
+        self,
+        task_config: dict[str, Any],
+        page: Page | None = None,
+        client: CDPSession | None = None,
+        answer: str | None = None,
+
+    ) -> dict[str, float|str]:
+        last_action = answer or ""
+        pred = clean_answer(last_action)
+
+        score = 1.0
+        for approach, value in task_config["eval"]["reference_answers"].items():
+
+            match approach:
+                case "exact_match":
+                    logger.info(f"Evaluating exact_match for answer: Predicted: {pred} , Reference: {value}")
+                    score *= evaluate_exact_match(ref=value, pred=pred)
+
+                case "must_include":
+                    logger.info(f"Evaluating must_include for answer: \"{answer}\" to see if it includes the expeced values: \"{value}\"\n")
+                    assert isinstance(value, list)
+                    for must_value in value: # type: ignore
+                        score *= evaluate_must_include(
+                            ref=must_value, # type: ignore
+                            pred=pred,
+                            tokenize=(len(value) == 1), # type: ignore
+                        )
+                case "some_matches":
+                    min_required_matches = value.get("min_required", 1)
+                    matches = sum(evaluate_must_include(ref=phrase, pred=pred, tokenize=False) for phrase in value["phrases"])
+                    score *= float(matches >= min_required_matches)
+                case "fuzzy_match":
+                    logger.info(f"Evaluating fuzzy_match for answer: {answer}")
+                    intent = task_config["intent"]
+                    if value == "N/A":
+                        # if the instruction only asks the model to generate N/A when encountering an unachievable task
+                        # without more concrete reasons
+                        score *= evaluate_exact_match(ref=value, pred=pred)
+                        # if the instruction also asks the model to generate the reason why the task is unachievable
+                        # this should be the default as it will prevent false positive N/A`
+                        if score != 1:
+                            score = 1.0 * evaluate_ua_match(
+                                intent=task_config["intent"],
+                                ref=task_config["eval"]["string_note"],
+                                pred=pred,
+                            )
+                    else:
+                        logger.info(f"Evaluating generic for answer: {answer}")
+                        assert isinstance(value, list)
+                        for reference in value: # type: ignore
+                            score *= evaluate_fuzzy_match(
+                                ref=reference, pred=pred, intent=intent # type: ignore
+                            )
+                case _:
+                    logger.info(f"Unknown approach value received: {approach}")
+        return {"score": score}
+
+
+class URLEvaluator(Evaluator):
+    """Evaluates if the given URL matches the expected URL criteria defined in the configuration.
+
+    This includes checking if the base path of the URL and its query parameters match those specified in the reference URLs.
+    """
+
+    async def __call__(
+        self,
+        task_config: dict[str, Any],
+        page: Page,
+        client: CDPSession | None = None,
+        answer: str | None = None
+    ) -> dict[str, float|str]:
+        """Evaluates the current page URL against reference URLs specified in the config file.
+
+        Parameters:
+            task_config (dict[str, Any]): The task configuration containing evaluation criteria.
+            page (Page): The Playwright page object for the current webpage.
+            client (CDPSession | None, optional): The Chrome DevTools Protocol session object. Not used in this evaluator.
+            answer (str | None, optional): Not used in this evaluator.
+
+        Returns:
+            dict[str, float|str]: "score" 1.0 if the page URL matches any of the reference URLs, considering the matching rule; otherwise 0.0.
+
+        Raises:
+            ValueError: If an unknown matching rule is specified in the config file.
+        """
+
+        def clean_url(url: str) -> str:
+            url = str(url)
+            url = url.rstrip("/")
+            url = url.lower()
+            return url
+
+        def parse_url(url: str) -> tuple[str, dict[str, list[str]]]:
+            """Parse a URL into its base, path, and query components."""
+            parsed_url = urllib.parse.urlparse(url)
+            base_path = parsed_url.netloc + parsed_url.path
+            query = urllib.parse.parse_qs(parsed_url.query)
+            return base_path, query
+
+        def parse_urls(
+            urls: list[str],
+        ) -> tuple[list[str], dict[str, set[str]]]:
+            """Parse a list of URLs."""
+            base_paths: list[str] = []
+            queries: dict[str, set[str]] = collections.defaultdict(set)
+            for url in urls:
+                base_path, query = parse_url(url)
+                base_paths.append(base_path)
+                for k, v in query.items():
+                    queries[k].update(v)
+            return base_paths, queries
+
+        pred = clean_url(page.url)
+        ref_urls = task_config["eval"]["reference_url"].split(" |OR| ")
+        ref_urls = [clean_url(url) for url in ref_urls]
+        matching_rule = task_config["eval"].get("url_note", "GOLD in PRED")
+        if matching_rule == "GOLD in PRED":
+            for ref_url in ref_urls:
+                ref_base_path, ref_query = parse_url(ref_url)
+                pred_base_paths, pred_query = parse_url(pred)
+                # Calculate base score for each ref_url
+                base_score = float(ref_base_path in pred_base_paths)
+                # Calculate query score for each ref_url
+                query_score = 1.0
+                for k, possible_values in ref_query.items(): # type: ignore
+                    if k in pred_query:
+                        query_score *= float(
+                            any(
+                                possible_ref_value in pred_query.get(k, []) # type: ignore
+                                for possible_ref_value in possible_values # type: ignore
+                            )
+                        )
+                    else:
+                        # If the key is not in pred_query, check if the reference URL has no query parameters
+                        if not possible_values:
+                            query_score *= 1.0  # No query parameters to match, so consider it a match
+                        else:
+                            query_score *= 0.0  # Reference URL has query parameters but predicted URL does not
+                # Calculate final score for each ref_url
+                score = base_score * query_score
+                # Return immediately if any score is 1
+                if score == 1.0:
+                    return {"score": score}
+
+        else:
+            raise ValueError(f"Unknown matching rule: {matching_rule}")
+
+        return {"score": 0.0}
+
+
+class HTMLContentEvaluator(Evaluator):
+    """Evaluates if specified HTML content or elements appear on the webpage.
+
+    This involves navigating to URLs specified in the configuration and checking for the presence of HTML elements or content using various strategies.
+    """
+
+    async def __call__(
+        self,
+        task_config: dict[str, Any],
+        page: Page,
+        client: CDPSession | None = None,
+        answer: str | None = None
+    ) -> dict[str, float|str]:
+        """Evaluates the presence of specified HTML content on the webpage.
+
+        Parameters:
+            task_config (dict[str, Any]): The task configuration containing evaluation criteria.
+            page (Page): The Playwright page object for the current webpage.
+            client (CDPSession | None, optional): The Chrome DevTools Protocol session object. Not used in this evaluator.
+            answer (str | None, optional): Not used in this evaluator.
+
+        Returns:
+            dict[str, float|str]: "score" A score between 0.0 and 1.0 representing the presence of required HTML content on the webpage.
+
+        Raises:
+            ValueError: If an unknown locator strategy is specified in the config file.
+        """
+        targets = task_config["eval"]["program_html"]
+
+        score = 1.0
+        for target in targets:
+            target_url: str = target["url"]  # which url to check
+            if target_url.startswith("func"):
+                func = target_url.split("func:")[1]
+                func = func.replace("__last_url__", page.url)
+                target_url = eval(func)
+
+            locator: str = target["locator"]  # js element locator
+
+            # navigate to that url
+            if target_url != "last":
+                page.goto(target_url)
+                time.sleep(3)
+
+            # empty, use the full page
+            if not locator.strip():
+                selected_element = page.content()
+            # use JS to select the element
+            elif locator.startswith("document.") or locator.startswith("[...document.") or locator.startswith("jsblock:"):
+                if "prep_actions" in target:
+                    try:
+                        for prep_action in target["prep_actions"]:
+                            page.evaluate(f"() => {prep_action}")
+                    except Exception:
+                        pass
+                try:
+                    if locator.startswith("jsblock:"):
+                        locator = locator.split("jsblock:")[1]
+
+                    selected_element = str(await page.evaluate(f"() => {locator}"))
+                    if not selected_element:
+                        selected_element = ""
+                except Exception:
+                    # the page is wrong, return empty
+                    selected_element = ""
+            # run program to call API
+            elif locator.startswith("func:"):  # a helper function
+                func = locator.split("func:")[1]
+                func = func.replace("__page__", "page")
+                selected_element = eval(func)
+            else:
+                raise ValueError(f"Unknown locator: {locator}")
+
+            selected_element = html.unescape(selected_element)
+
+            if "exact_match" in target["required_contents"]:
+                required_contents = target["required_contents"]["exact_match"]
+                cur_score = evaluate_exact_match(
+                    ref=required_contents, pred=selected_element
+                )
+                score *= float(cur_score)
+                # logger.info(f"[exact match] {cur_score}, selected element: {selected_element}, required contents: {required_contents}")
+            elif "must_include" in target["required_contents"]:
+                required_contents = target["required_contents"]["must_include"]
+                assert isinstance(required_contents, list)
+                for content in required_contents: # type: ignore
+                    content_or = content.split(" |OR| ") # type: ignore
+                    cur_score = any(
+                        [
+                            evaluate_must_include(
+                                ref=content, # type: ignore
+                                pred=selected_element,
+                                tokenize=False,
+                            )
+                            for content in content_or # type: ignore
+                        ]
+                    )
+                    score *= float(cur_score)
+                    # logger.info(f"[must include] {cur_score}, selected element: {selected_element}, required contents: {content_or}")
+            else:
+                raise ValueError(
+                    f"Unknown required_contents: {target['required_contents'].keys()}"
+                )
+        return {"score": score}
+
+class ManualContentEvaluator(Evaluator):
+    """Evaluation Route for Manual Evaluation."""
+    async def __call__(
+        self,
+        task_config: dict[str, Any],
+        page: Page,
+        client: CDPSession | None = None,
+        answer: str | None = None
+    ) -> dict[str, float|str]:
+        """Pauses Execution to get manual evaluation score from user.
+
+        Parameters:
+            task_config (dict[str, Any]): The task configuration containing evaluation criteria.
+            page (Page): The Playwright page object for the current webpage.
+            client (CDPSession | None, optional): The Chrome DevTools Protocol session object. Not used in this evaluator.
+            answer (str | None, optional): Not used in this evaluator.
+
+        Returns:
+            dict[str, float|str]: A score representig the status 1 = pass, 0 = fail and -0.1 is a skip. Additionaly, a reason can be provided for the score (mainly for fail/skip).
+        """
+        task = task_config["intent"]
+        reference_answer = task_config["eval"]["reference_answers"]["manual_check"]["answer"]
+        answer_type = task_config["eval"]["reference_answers"]["manual_check"]["type"]
+        id = str(task_config["task_id"])
+        index = str(task_config["task_index"])
+
+        print(colored("\n\n***************************\n", "green", attrs=["bold"]))
+        print(colored("Task ID: ", "blue", attrs=["bold"]) + id + "\n")
+        print(colored("Task Index: ", "blue", attrs=["bold"]) + index + "\n")
+        print(colored("Task: ", "blue", attrs=["bold"]) + task + "\n")
+        print(colored("Agent answer: ", "blue", attrs=["bold"]) + str(answer or "") + "\n")
+
+        if answer_type.strip().lower() == "possible":
+            print(colored("Possible answer (reference): ", "yellow") + f"~~~{reference_answer}~~~")
+        elif answer_type.strip().lower() == "golden":
+            print(colored("Golden answer (reference): ", "yellow") + reference_answer)
+
+        user_response = input(colored("Annotate the task as Pass, Fail or Skip (please use Skip sparingly)? ", "magenta", attrs=["bold"]))
+        eval_response: dict[str, float|str] = {}
+        if(user_response.lower()=="pass"):
+            eval_response["score"] = 1.0
+        elif user_response.lower()=="fail":
+            eval_response["score"] = 0.0
+        elif user_response.lower()=="skip":
+            eval_response["score"] = -0.1
+        else:
+            print(colored(f"Received response: {user_response}", "red"))
+            raise ValueError("Invalid user response. Please enter 'Pass', 'Fail' or 'Skip'.")
+        reason: str|None = None
+
+        if eval_response["score"] <= 0:
+            reason = input("Reason for rating: ")
+            eval_response["reason"] = reason
+
+        return eval_response
+
+class EvaluatorComb(Evaluator):
+    """Combines multiple evaluators to perform a comprehensive evaluation based on different criteria.
+
+    Attributes:
+        evaluators (list[Evaluator]): A list of evaluator instances to be used for evaluation.
+    """
+
+    def __init__(self, evaluators: list[Evaluator]) -> None:
+        """Initializes the composite evaluator with a list of individual evaluators.
+
+        Parameters:
+            evaluators (list[Evaluator]): The list of evaluators to include in the composite evaluation.
+        """
+        self.evaluators = evaluators
+
+
+    async def __call__(
+        self,
+        task_config: dict[str, Any],
+        page: Page,
+        client: CDPSession,
+        answer: str,
+    ) -> dict[str, float|str]:
+        """Performs the evaluation using all included evaluators and aggregates their scores.
+
+        Parameters:
+            task_config (dict[str, Any]): The task configuration containing evaluation criteria.
+            page (Page): The Playwright page object for the current webpage.
+            client (CDPSession): The Chrome DevTools Protocol session object.
+            answer (str): The answer or content to be evaluated.
+
+        Returns:
+            dict[str, float|str]: "score" - The aggregated score from all evaluators, representing the overall evaluation result. "reason" - The reason for the evaluation score, if applicable.
+        """
+        score: float = 1.0
+        reason: str | None = None
+        for evaluator in self.evaluators:
+            eval_result = await evaluator(task_config, page, client, answer)
+            score: float = score * eval_result["score"] # type: ignore
+            if "reason" in eval_result:
+                if reason is None:
+                    reason = eval_result["reason"] # type: ignore
+                else:
+                    reason += f"\n{eval_result['reason']}"
+        return {"score": score, "reason": reason} # type: ignore
+
+
+def evaluator_router(task_config: dict[str, Any]) -> EvaluatorComb:
+    """Creates and configures a composite evaluator based on the evaluation types specified in the configuration file.
+
+    Parameters:
+        task_config dict[str, Any]: configuration specifying the evaluation types to use.
+
+    Returns:
+        EvaluatorComb: A composite evaluator configured with the specified types of individual evaluators.
+
+    Raises:
+        ValueError: If an unsupported evaluation type is specified in the configuration file.
+    """
+
+    eval_types = task_config["eval"]["eval_types"]
+    evaluators: list[Evaluator] = []
+    for eval_type in eval_types:
+        match eval_type:
+            case "string_match":
+                logger.info("Adding string evaluator")
+                evaluators.append(StringEvaluator())
+            case "url_match":
+                logger.info("Adding URL evaluator")
+                evaluators.append(URLEvaluator())
+            case "program_html":
+                logger.info("Adding HTML evaluator")
+                evaluators.append(HTMLContentEvaluator())
+            case "manual":
+                logger.info("Adding manual evaluator")
+                evaluators.append(ManualContentEvaluator())
+            case _:
+                raise ValueError(f"eval_type {eval_type} is not supported")
+
+    return EvaluatorComb(evaluators)

Agent_E/test/test_utils.py

@@ -0,0 +1,263 @@
+"""Implements helper functions to assist evaluation cases where other evaluators are not suitable."""
+import json
+import os
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from dotenv import load_dotenv
+from nltk.tokenize import word_tokenize  # type: ignore
+from openai import OpenAI
+
+load_dotenv()
+client = OpenAI()
+
+
+def llm_fuzzy_match(pred: str, reference: str, question: str) -> float:
+    """
+    Evaluates if a predicted answer matches a reference answer semantically, considering the context of a question.
+
+    This function simulates a grading scenario, understanding that a student's answer may use different wording or phrasing from the reference answer. It uses GPT-4-turbo model to assess semantic equivalence.
+
+    Parameters:
+        pred (str): The student's predicted answer.
+        reference (str): The reference answer to compare against.
+        question (str): The question related to the answers.
+
+    Returns:
+        float: Returns 1.0 if the predicted answer is semantically equivalent to the reference, otherwise 0.0.
+    """
+    messages: list[dict[str, Any]] = []
+    # construct the question to ask
+    message = "Help a teacher to grade the answer of a student given a question. Keep in mind that the student may use different phrasing or wording to answer the question. The goal is to evaluate whether the answer is semantically equivalent to the reference answer.\n"
+    message += f"question: {question}\n"
+    message += f"reference answer: {reference}\n"
+    message += "all the string 'N/A' that you see is a special sequence that means 'not achievable'\n"
+    message += f"student answer: {pred}\n"
+    message += "Conclude the judgement by correct/incorrect/partially correct."
+    messages = [
+        {"role": "system", "content": "You are a helpful assistant"},
+        {"role": "user", "content": message},
+    ]
+
+    response = generate_from_openai_chat_completion(
+        model="gpt-4-turbo-preview",
+        messages=messages,
+        temperature=0,
+        max_tokens=768,
+        top_p=1.0,
+        context_length=0,
+    ).lower()
+    if "partially correct" in response or "incorrect" in response:
+        return 0.0
+    else:
+        assert "correct" in response
+        return 1.0
+
+
+def llm_ua_match(pred: str, reference: str, question: str) -> float:
+    """
+    Evaluates the alignment between a reported reason for a task being unachievable and the actual reason.
+
+    This function reviews both the actual and reported reasons for a task's unachievability within the context of the task.
+    It assesses if the reported reason is implicitly or explicitly in line with the actual reason, using GPT-turbo model.
+
+    Parameters:
+        pred (str): The reported unachievable reason by an individual.
+        reference (str): The actual reason why the task is unachievable.
+        question (str): The task in question.
+
+    Returns:
+        float: Returns 1.0 if the reported reason aligns with the actual reason, otherwise 0.0.
+    """
+    messages: list[dict[str, Any]] = []
+    # construct the question to ask
+    message = ""
+    message += f"task: {question}\n"
+    message += f"actual unachievable reason: {reference}\n"
+    message += f"reported unachievable reason: {pred}\n"
+    message += (
+        "The task described above is inherently unachievable due to the reason specified under 'actual unachievable reason'. "
+        "An individual previously attempted this task and was unable to complete it. They provided a reason for their failure, "
+        "which is listed under 'reported unachievable reason'. Your role is to review both the actual and reported reasons. "
+        "Determine if the reported reason aligns with the actual reason, even if implicitly. "
+        "If the stated reason is in line with the actual reason, respond with 'same'. Otherwise, respond with 'different'."
+    )
+    messages = [
+        {"role": "system", "content": "You are a helpful assistant"},
+        {"role": "user", "content": message},
+    ]
+
+    response = generate_from_openai_chat_completion(
+        model="gpt-4-turbo-preview",
+        messages=messages,
+        temperature=0,
+        max_tokens=768,
+        top_p=1.0,
+        context_length=0,
+    ).lower()
+    if "different" in response:
+        return 0.0
+    else:
+        assert "same" in response
+        return 1.0
+
+
+
+def generate_from_openai_chat_completion(
+    messages: list[dict[str, str]],
+    model: str,
+    temperature: float,
+    max_tokens: int,
+    top_p: float,
+    context_length: int,
+    stop_token: str | None = None,
+) -> str:
+    """
+    Generates a response from OpenAI's chat completions based on a conversation constructed from a list of messages.
+
+    This function makes a call to the OpenAI API using specified parameters to control the generation.
+    It requires an API key to be set in the environment variables.
+
+    Parameters:
+        messages (list[dict[str, str]]): A list of messages to construct the conversation context.
+        model (str): The model name to use for generating the completion.
+        temperature (float): Sampling temperature for generation.
+        max_tokens (int): Maximum number of tokens to generate.
+        top_p (float): Nucleus sampling parameter controlling the size of the probability mass to sample from.
+        context_length (int): The maximum number of tokens from `messages` to use for context.
+        stop_token (str, optional): A token at which to stop generating further tokens.
+
+    Returns:
+        str: The generated response as a string.
+
+    Raises:
+        ValueError: If the 'OPENAI_API_KEY' environment variable is not set.
+    """
+    if "OPENAI_API_KEY" not in os.environ:
+        raise ValueError(
+            "OPENAI_API_KEY environment variable must be set when using OpenAI API."
+        )
+    client.api_key = os.environ["OPENAI_API_KEY"]
+    client.organization = os.environ.get("OPENAI_ORGANIZATION", "")
+
+    response = client.chat.completions.create(
+        model=model,
+        messages=messages, # type: ignore
+        temperature=temperature,
+        max_tokens=max_tokens,
+        top_p=top_p,
+        n=1,
+        stop=[stop_token] if stop_token else None,
+    )
+    answer: str = response.choices[0].message.content # type: ignore
+    return answer
+
+def clean_answer(answer: str) -> str:
+    """Cleans and preprocesses the answer string for evaluation.
+
+    Parameters:
+        answer (str): The answer string to clean.
+
+    Returns:
+        str: The cleaned and lowercased answer string.
+    """
+    answer = answer.strip().strip('"').strip("'").lower()
+    return answer
+
+def evaluate_exact_match(ref: str, pred: str) -> float:
+    """Evaluates if the predicted answer exactly matches the reference answer.
+
+    Parameters:
+        ref (str): The reference answer.
+        pred (str): The predicted answer.
+
+    Returns:
+        float: 1.0 if the answers match exactly, otherwise 0.0.
+    """
+    return float(clean_answer(pred) == clean_answer(ref))
+
+def evaluate_must_include(ref: str, pred: str, tokenize: bool = False) -> float:
+    """Checks if the predicted answer includes all phrases from the reference answer.
+
+    Parameters:
+        ref (str): The reference answer containing phrases that must be included.
+        pred (str): The predicted answer to be evaluated.
+        tokenize (bool, optional): Tokenizes the answers before evaluation if True. Default is False.
+
+    Returns:
+        float: 1.0 if all phrases are included, otherwise 0.0.
+    """
+    clean_ref = clean_answer(ref)
+    clean_pred = clean_answer(pred)
+    if tokenize and len(clean_ref) == 1:
+        return float(clean_ref in word_tokenize(clean_pred))
+    else:
+        return float(clean_ref in clean_pred)
+
+def evaluate_fuzzy_match(ref: str, pred: str, intent: str) -> float:
+    """Evaluates if the predicted answer is semantically similar to the reference answer.
+
+    Uses a large language model to assess similarity based on the intent of the question.
+
+    Parameters:
+        ref (str): The reference answer.
+        pred (str): The predicted answer.
+        intent (str): The intent or context of the question.
+
+    Returns:
+        float: 1.0 if the answers are considered semantically similar, otherwise 0.0.
+    """
+    return llm_fuzzy_match(pred, ref, intent)
+
+def evaluate_ua_match(ref: str, pred: str, intent: str) -> float:
+    """Evaluates if the predicted reason for a task being unachievable matches the reference reason.
+
+    Parameters:
+        ref (str): The reference reason why the task is unachievable.
+        pred (str): The predicted reason reported by the model.
+        intent (str): The intent or context of the task.
+
+    Returns:
+        float: 1.0 if the reasons match, otherwise 0.0.
+    """
+    return llm_ua_match(pred, ref, intent)
+
+
+def load_config(config_file: Path | str) -> list[dict[str, Any]]:
+    """Load the confiufiguration for the test cases
+
+    Args:
+        config_file (Path | str): Path to the config file
+
+    Returns:
+        list[dict[str, Any]]: All the test cases in the config file
+    """
+    with open(config_file, "r") as f:  # noqa: UP015
+        configs = json.load(f)
+    return configs
+
+def task_config_validator(task_config: dict[str, Any]) -> bool:
+    # Access the attributes
+    command = task_config.get('intent')
+
+    if not command:
+        raise ValueError("Intent is missing in the task config file. Without it the task cannot be run.")
+
+    return True
+
+def get_formatted_current_timestamp(format: str = "%Y-%m-%d %H:%M:%S") -> str:
+    """Get the current timestamp in the specified format.
+
+    Args:
+        format (str, optional): The format of the timestamp. Defaults to "%Y-%m-%d %H:%M:%S".
+
+    Returns:
+        str: The current timestamp in the specified format.
+    """
+    # Get the current time
+    current_time = datetime.now()
+
+    # Format the timestamp as a human-readable string
+    timestamp_str = current_time.strftime(format)
+    return timestamp_str

Agent_E/test/tests_processor.py

@@ -0,0 +1,409 @@
+import asyncio
+import json
+import os
+import time
+from Agent_E.ae.core.agents_llm_config import AgentsLLMConfig
+from Agent_E.test.test_utils import get_formatted_current_timestamp
+from Agent_E.test.test_utils import load_config
+from Agent_E.test.test_utils import task_config_validator
+from typing import Any
+
+import Agent_E.ae.core.playwright_manager as browserManager
+import nltk  # type: ignore
+from Agent_E.ae.config import PROJECT_TEST_ROOT
+from Agent_E.ae.core.autogen_wrapper import AutogenWrapper
+from Agent_E.ae.core.playwright_manager import PlaywrightManager
+from Agent_E.ae.utils.logger import logger
+from Agent_E.ae.utils.response_parser import parse_response
+from autogen.agentchat.chat import ChatResult  # type: ignore
+from playwright.async_api import Page
+from tabulate import tabulate
+from termcolor import colored
+
+from evaluation_harness.evaluators import evaluator_router
+
+nltk.download('punkt') # type: ignore
+
+last_agent_response = ""
+
+def check_top_level_test_folders(test_log_dir, test_result_dir):
+    if not os.path.exists(test_log_dir):
+        os.makedirs(test_log_dir)
+        logger.info(f"Created log folder at: {test_log_dir}")
+
+    if not os.path.exists(test_result_dir):
+        os.makedirs(test_result_dir)
+        logger.info(f"Created scores folder at: {test_result_dir}")
+
+def create_task_log_folders(test_log_dir, task_id):
+    task_log_dir = os.path.join(test_log_dir, task_id)
+    task_screenshots_dir = os.path.join(task_log_dir, 'snapshots')
+    if not os.path.exists(task_log_dir):
+        os.makedirs(task_log_dir)
+        logger.info(f"Created log dir for task {task_id} at: {task_log_dir}")
+    if not os.path.exists(task_screenshots_dir):
+        os.makedirs(task_screenshots_dir)
+        logger.info(f"Created screenshots dir for task {task_id} at: {task_screenshots_dir}")
+
+    return {"task_log_folder": task_log_dir, "task_screenshots_folder": task_screenshots_dir}
+
+
+def create_results_dir(test_file: str, test_results_id: str|None) -> str:
+    results_dir = ""
+    if test_results_id:
+        results_dir = os.path.join(TEST_RESULTS, f"results_for_{test_results_id}")
+    else:
+        test_file_base = os.path.basename(test_file)
+        test_file_name = os.path.splitext(test_file_base)[0]
+        results_dir = os.path.join(TEST_RESULTS, f"results_for_test_file_{test_file_name}")
+
+    if not os.path.exists(results_dir):
+        os.makedirs(results_dir)
+        logger.info(f"Created results directory: {results_dir}")
+
+    return results_dir
+
+
+def dump_log(task_id: str, messages_str_keys: dict[str, str], logs_dir: str):
+    file_name = os.path.join(logs_dir, f'execution_logs_{task_id}.json')
+    with open(file_name, 'w',  encoding='utf-8') as f:
+            json.dump(messages_str_keys, f, ensure_ascii=False, indent=4)
+
+
+def save_test_results(test_results: list[dict[str, str | int | float | None]], test_results_id: str):
+    file_name = os.path.join(TEST_RESULTS, f'test_results_{test_results_id}.json')
+    with open(file_name, 'w',  encoding='utf-8') as f:
+        json.dump(test_results, f, ensure_ascii=False, indent=4)
+    logger.info(f"Test results dumped to: {file_name}")
+
+
+def save_individual_test_result(test_result: dict[str, str | int | float | None], results_dir: str):
+    task_id = test_result["task_id"]
+    file_name = os.path.join(results_dir, f'{task_id}.json')
+    with open(file_name, 'w',  encoding='utf-8') as f:
+        json.dump(test_result, f, ensure_ascii=False, indent=4)
+    logger.info(f"Test result for task {task_id} dumped to: {file_name}")
+
+
+def extract_last_response(messages: list[dict[str, Any]]) -> str:
+    """Extract the last response message from chat history."""
+    try:
+        # Iterate over the messages in reverse order
+        for message in reversed(messages):
+            if message and 'content' in message:
+                content=message.get('content', "")
+                content_json = parse_response(content)
+                final_answer = content_json.get('final_response', None)
+                if final_answer:
+                    return final_answer
+        return ""
+    except:
+        logger.error("Error extracting last response from chat history.")
+        return ""
+
+
+def print_progress_bar(current: int, total: int, bar_length: int = 50) -> None:
+    """
+    Prints a progress bar to the console.
+
+    Parameters:
+    - current (int): The current progress of the task.
+    - total (int): The total number of tasks to complete.
+    - bar_length (int): The character length of the progress bar (default is 50).
+
+    This function dynamically updates a single line in the console to reflect current progress.
+
+    """
+    percent = float(current) * 100 / total
+    arrow = '-' * int(percent/100 * bar_length - 1) + '>'
+    spaces = ' ' * (bar_length - len(arrow))
+
+    print(f'\rProgress: [{arrow}{spaces}] {current}/{total} ({percent:.2f}%)', end='')
+
+def determine_status_and_color(score: float) -> tuple[str, str]:
+    """
+    Determines the status and color for a test result based on the score.
+
+    Parameters:
+    - score (float): The score of the test task, indicating success (1), failure (0), or skip (-0.1).
+
+    Returns:
+    - tuple[str, str]: A tuple containing the status ('Pass', 'Fail', or 'Skip') and the corresponding color ('green', 'red', or 'yellow').
+
+    """
+    if score == 1:
+        return 'Pass', 'green'
+    elif score < 0:
+        return 'Skip', 'yellow'
+    else:
+        return 'Fail', 'red'
+
+
+def print_test_result(task_result: dict[str, str | int | float | None], index: int, total: int) -> None:
+    """
+    Prints the result of a single test task in a tabulated format.
+
+    Parameters:
+    - task_result (dict): A dictionary containing the task's evaluation results, including task ID, intent, score, and total command time.
+    - index (int): The current index of the test in the sequence of all tests being run.
+    - total (int): The total number of tests to be run.
+
+    The function determines the test status (Pass/Fail) based on the 'score' key in task_result and prints the result with colored status.
+
+    """
+    status, color = determine_status_and_color(task_result['score']) # type: ignore
+
+    cost = task_result.get("compute_cost", None)
+    total_cost = None if cost is None else round(cost.get("cost", -1), 4)  # type: ignore
+    total_tokens = None if cost is None else cost.get("total_tokens", -1)  # type: ignore
+    result_table = [  # type: ignore
+        ['Test Index', 'Task ID', 'Intent', 'Status', 'Time Taken (s)', 'Total Tokens', 'Total Cost ($)'],
+        [index, task_result['task_id'], task_result['intent'], colored(status, color), round(task_result['tct'], 2), total_tokens, total_cost]  # type: ignore
+    ]
+    print('\n' + tabulate(result_table, headers='firstrow', tablefmt='grid')) # type: ignore
+
+def get_command_exec_cost(command_exec_result: ChatResult):
+    output: dict[str, Any] = {}
+    try:
+        cost = command_exec_result.cost # type: ignore
+        usage: dict[str, Any] = None
+        if "usage_including_cached_inference" in cost:
+            usage: dict[str, Any] = cost["usage_including_cached_inference"]
+        elif "usage_excluding_cached_inference" in cost:
+            usage: dict[str, Any] = cost["usage_excluding_cached_inference"]
+        else:
+            raise ValueError("Cost not found in the command execution result.")
+        print("Usage: ", usage)
+
+        for key in usage.keys():
+            if isinstance(usage[key], dict) and "prompt_tokens" in usage[key]:
+                output["cost"] = usage[key]["cost"]
+                output["prompt_tokens"] = usage[key]["prompt_tokens"]
+                output["completion_tokens"] = usage[key]["completion_tokens"]
+                output["total_tokens"] = usage[key]["total_tokens"]
+    except Exception as e:
+        logger.debug(f"Error getting command execution cost: {e}")
+    return output
+
+
+async def execute_single_task(task_config_file: str, browser_manager: PlaywrightManager, ag: AutogenWrapper, page: Page, logs_dir: str) -> dict[str, Any]:
+    """
+    Executes a single test task based on a specified task configuration and evaluates its performance.
+
+    Parameters:
+    - task_config (dict): The task configuration dictionary containing all necessary parameters for the task.
+    - browser_manager (PlaywrightManager): The manager handling browser interactions, responsible for page navigation and control.
+    - ag (AutogenWrapper): The automation generator wrapper that processes commands and interacts with the web page.
+    - page (Page): The Playwright page object representing the browser tab where the task is executed.
+
+    Returns:
+    - dict: A dictionary containing the task's evaluation results, including task ID, intent, score, total command time (tct),
+            the last statement from the chat agent, and the last URL accessed during the task.
+    """
+    command = ""
+    start_url = None
+    task_id = None
+
+    start_ts = get_formatted_current_timestamp()
+
+    task_config = json.load(open(task_config_file, "r"))
+
+    task_config_validator(task_config)
+
+    command: str = task_config.get('intent', "")
+    task_id = task_config.get('task_id')
+    task_index = task_config.get('task_index')
+    start_url = task_config.get('start_url')
+    logger.info(f"Intent: {command}, Task ID: {task_id}")
+
+    if start_url:
+        await page.goto(start_url, wait_until='load', timeout=30000)
+
+    start_time = time.time()
+    current_url = await browser_manager.get_current_url()
+    command_exec_result = await ag.process_command(command, current_url)
+    end_time = time.time()
+
+    evaluator_result: dict[str, float | str] = {}
+    last_agent_response: str = ""
+    command_cost: dict[str, Any] = {}
+    single_task_result: dict[str, Any] = {}
+    try:
+        single_task_result = {
+            "task_id": task_id,
+            "task_index": task_index,
+            "start_url": start_url,
+            "intent": str(command),
+            "last_url": page.url,
+            "tct": end_time - start_time,
+            "start_ts": start_ts,
+            "completion_ts": get_formatted_current_timestamp()
+        }
+
+        agent_name: str = "planner_agent" if ag.agents_map is not None and "planner_agent" in ag.agents_map else "browser_nav_agent"
+
+        command_cost = get_command_exec_cost(command_exec_result) # type: ignore
+        print(f"Command cost: {command_cost}")
+        single_task_result["compute_cost"] = command_cost
+
+        logger.info(f"Command \"{command}\" took: {round(end_time - start_time, 2)} seconds.")
+        logger.info(f"Task {task_id} completed.")
+
+        messages = ag.agents_map[agent_name].chat_messages # type: ignore
+        messages_str_keys = {str(key): value for key, value in messages.items()} # type: ignore
+        agent_key = list(messages.keys())[0] # type: ignore
+        last_agent_response = extract_last_response(messages[agent_key]) # type: ignore
+
+        dump_log(str(task_id), messages_str_keys, logs_dir)
+
+        single_task_result["last_statement"] = last_agent_response
+
+
+        evaluator = evaluator_router(task_config_file)
+        # cdp_session = await page.context.new_cdp_session(page)
+        evaluator_result = evaluator(
+            config_file=task_config_file,
+            page=None,
+            client=None,
+            trajectory=[{"answer":last_agent_response}]
+        )
+
+        single_task_result["score"] = evaluator_result
+    except Exception as e:
+        logger.error(f"Error getting command cost: {e}")
+        command_cost = {"cost": -1, "total_tokens": -1}
+        single_task_result["compute_cost"] = command_cost
+        single_task_result["error"] = str(e)
+
+    return single_task_result
+
+
+async def run_tests(ag: AutogenWrapper, browser_manager: PlaywrightManager, task_ids,
+               logdir: str="", logname: str="", relative_task_dir: str="", test_results_id: str = "", wait_time_non_headless: int=5, take_screenshots: bool = False) -> list[dict[str, Any]]:
+    """
+    Runs a specified range of test tasks using Playwright for browser interactions and AutogenWrapper for task automation.
+    It initializes necessary components, processes each task, handles exceptions, and compiles test results into a structured list.
+
+    Parameters:
+    - ag (AutogenWrapper): The AutoGen wrapper that processes commands.
+    - browser_manager (PlaywrightManager): The manager handling browser interactions, responsible for page navigation and control.
+    - logdir (str)
+    - logname (str)
+    - task_ids (List[str])
+    - relative_task_dir (str)
+    - wait_time_non_headless (int): Time to wait between tasks when running in non-headless mode, useful for live monitoring or debugging.
+    - take_screenshots (bool): Whether to take screenshots during test execution. Defaults to False.
+
+    Returns:
+    - list[dict[str, Any]]: A list of dictionaries, each containing the results from executing a test task. Results include task ID, intent, score, total command time, etc.
+
+    This function also manages logging and saving of test results, updates the progress bar to reflect test execution status, and prints a detailed summary report at the end of the testing session.
+    """
+    test_log_dir = os.path.join(logdir, logname)
+    test_result_dir = os.path.join(logdir, logname, "results")
+    check_top_level_test_folders(test_log_dir, test_result_dir)
+
+    config_file_list = []
+    if not relative_task_dir or relative_task_dir == "":
+        relative_task_dir = "tasks"
+    if task_ids == "all" or task_ids == ["all"]:
+        task_ids = [filename[:-len(".json")] for filename in os.listdir(f"config_files/{relative_task_dir}") if filename.endswith(".json")]
+    for task_id in task_ids:
+        config_file_list.append(f"config_files/{relative_task_dir}/{task_id}.json")
+
+    test_results: list[dict[str, str | int | float | None]] = []
+
+    llm_config = AgentsLLMConfig()
+    if not ag:
+        ag = await AutogenWrapper.create(llm_config.get_planner_agent_config(), llm_config.get_browser_nav_agent_config())
+
+    if not browser_manager:
+        browser_manager = browserManager.PlaywrightManager(headless=True)
+        await browser_manager.async_initialize()
+
+    page=await browser_manager.get_current_page()
+    test_results = []
+    total_tests = len(config_file_list)
+
+    for index, task_config_file in enumerate(config_file_list):
+        task_config = json.load(open(task_config_file, "r"))
+        task_id = str(task_config.get('task_id'))
+        if os.path.exists(os.path.join(test_result_dir, f'{task_id}.json')):
+            continue
+
+        log_folders = create_task_log_folders(test_log_dir, task_id)
+
+        ag.set_chat_logs_dir(log_folders["task_log_folder"])
+
+        browser_manager.set_take_screenshots(take_screenshots)
+        if take_screenshots:
+            browser_manager.set_screenshots_dir(log_folders["task_screenshots_folder"])
+
+        print_progress_bar(index, total_tests)
+        task_result = await execute_single_task(task_config_file, browser_manager, ag, page, log_folders["task_log_folder"])
+        test_results.append(task_result)
+        save_individual_test_result(task_result, test_result_dir)
+        print_test_result(task_result, index + 1, total_tests)
+
+        if not browser_manager.isheadless: # no need to wait if we are running headless
+            await asyncio.sleep(wait_time_non_headless)  # give time for switching between tasks in case there is a human observer
+
+        await browser_manager.take_screenshots("final", None)
+
+        await browser_manager.close_except_specified_tab(page) # cleanup pages that are not the one we opened here
+
+    print_progress_bar(total_tests, total_tests)  # Complete the progress bar
+    print('\n\nAll tests completed.')
+
+    # Aggregate and print individual test results
+    print("\nDetailed Test Results:")
+    detailed_results_table = [['Test Index', 'Task ID', 'Intent', 'Status', 'Time Taken (s)', 'Total Tokens', 'Total Cost ($)']]
+    for idx, result in enumerate(test_results, 1):
+        status, color = determine_status_and_color(result['score']) # type: ignore
+
+        cost: str | int | float | None = result.get("compute_cost", None)
+        total_cost = None if cost is None else round(cost.get("cost", -1), 4)  # type: ignore
+        total_tokens = None if cost is None else cost.get("total_tokens", -1)  # type: ignore
+
+        detailed_results_table.append([
+            idx, result['task_id'], result['intent'], colored(status, color), round(result['tct'], 2), # type: ignore
+            total_tokens, total_cost
+        ])
+
+    print(tabulate(detailed_results_table, headers='firstrow', tablefmt='grid'))
+
+    # Summary report
+
+    # Calculate aggregated cost and token totals for all tests that have compute cost
+    total_cost = 0
+    total_tokens = 0
+
+    for result in test_results:
+        compute_cost = result.get("compute_cost",0) # type: ignore
+        if compute_cost is not None and isinstance(compute_cost, dict):
+            total_cost += compute_cost.get("cost", 0) # type: ignore
+            total_tokens += compute_cost.get("total_tokens", 0) # type: ignore
+
+    passed_tests = []
+    skipped_tests = []
+    failed_tests = []
+    for result in test_results:
+        if result["score"] == 1:
+            passed_tests.append(result) # type: ignore
+        elif result["score"] < 0: # type: ignore
+            skipped_tests.append(result) # type: ignore
+        else:
+            failed_tests.append(result) # type: ignore
+
+    summary_table = [ # type: ignore
+        ['Total Tests', 'Passed', 'Failed', 'Skipped', 'Average Time Taken (s)', 'Total Time Taken (s)', 'Total Tokens', 'Total Cost ($)'],
+        [total_tests, len(passed_tests), len(failed_tests), len(skipped_tests),
+        round(sum(test['tct'] for test in test_results) / total_tests, 2), # type: ignore
+        round(sum(test['tct'] for test in test_results), 2),  # type: ignore
+        total_tokens, total_cost]
+    ]
+
+    print('\nSummary Report:')
+    print(tabulate(summary_table, headers='firstrow', tablefmt='grid')) # type: ignore
+
+    return test_results