# run_qwen_mysql_agent.py
import os
import json
import re
from qwen_agent.agents import Assistant
from qwen_agent.utils.output_beautify import typewriter_print
import subprocess


# DIFFICULTY = "Easy"
# DIFFICULTY = "Medium"
DIFFICULTY = "Hard"

PREVIOUS_GENERATED_TASKS = ""


# 1. 选定 LLM（示例用 DashScope 云端，替换为你自己的即可）
llm_cfg = {
    # 'model': 'qwen3-8b',  # 不能tool call
    'model': 'qwen3-235b-a22b',  # ! 这个可以
    # 'model': 'qwen3-30b-a3b',  # 不能tool call
    # 'model': 'qwen-plus-latest', # 没有thinking
    # 'model': 'qwen-turbo-latest',  # 没有thinking

    # Use the endpoint provided by Alibaba Model Studio:
    # 'model_type': 'qwen_dashscope',
    # 'api_key': os.getenv('DASHSCOPE_API_KEY'),

    # Use a custom endpoint compatible with OpenAI API:
    'model_server': 'https://dashscope.aliyuncs.com/compatible-mode/v1',  # api_base
    # 'api_key': 'sk-5d90a63c1e784e8f801dee65add68867',
    'api_key': 'sk-a6a00cb727ee4913ba22530ec3c2b30d',

    # Other parameters:
    # 'generate_cfg': {
    #         # Add: When the response content is `<think>this is the thought</think>this is the answer;
    #         # Do not add: When the response has been separated by reasoning_content and content.
    #         'thought_in_content': True,
    #     },
}

# 2. 描述可用工具 —— 这里挂载刚启动的 MySQL‑MCP Server
# https://www.modelscope.cn/mcp/servers/@designcomputer/mysql_mcp_server
tools = [{
  "mcpServers": {
    "mysql": {
      "command": "uv",
      "args": [
        "--directory", 
        "/home/ubuntu/.mcp",
        "run",
        "mysql_mcp_server"
      ],
      "env": {
        "MYSQL_HOST": "localhost",
        "MYSQL_PORT": "23306",
        "MYSQL_USER": "mcpuser",
        "MYSQL_PASSWORD": "StrongPass123!",
        "MYSQL_DATABASE": "magentodb"
      }
    }
  }
}]

MAGENTO_SCHEMA_CONTENT = ""
script_dir = ""
try:
    script_dir = os.path.dirname(os.path.abspath(__file__))
    # schema_file_path = os.path.join(script_dir, "schema_nonempty.txt")
    print(f"curated_schema")
    schema_file_path = os.path.join(script_dir, "curated_schema.txt")
    with open(schema_file_path, "r", encoding="utf-8") as f:
        MAGENTO_SCHEMA_CONTENT = f.read()
    print(f"Schema loaded successfully from {schema_file_path}")
except FileNotFoundError:
    print(f"curated_schema.txt not found at {schema_file_path}. Exiting.")
except Exception as e:
    print(f"Error loading schema file: {e}. Exiting.")

TABLE_SAMPLES_CONTENT = ""
script_dir = ""
try:
    script_dir = os.path.dirname(os.path.abspath(__file__))
    print(f"table_samples_cache")
    schema_file_path = os.path.join(script_dir, "table_samples_cache.txt")
    with open(schema_file_path, "r", encoding="utf-8") as f:
        TABLE_SAMPLES_CONTENT = f.read()
    print(f"Schema loaded successfully from {schema_file_path}")
except FileNotFoundError:
    print(f"table_samples_cache.txt not found at {schema_file_path}. Exiting.")
except Exception as e:
    print(f"Error loading table samples cache file: {e}. Exiting.")

# 3. 创建智能体
bot = Assistant(
    llm=llm_cfg,
    function_list=tools,
)

prompt = f"""
You are an expert Magento 2 database analyst, Python programmer, and an autonomous agent. You are equipped with a tool to interact directly with a MySQL database. Your current task is to generate ONE comprehensive Question-Answer (QA) item related to the Magento 2 database, focusing on "query" type tasks.

This process involves:
1.  Formulating an initial question.
2.  Iteratively using your database tool to execute SELECT SQL queries to gather information.
3.  Analyzing SQL results to refine the question, ensure its answerability, and collect all necessary data.
4.  Deriving a precise answer based on the verified data from the database.
5.  Designing specific validation rules (`exact_match`, `must_include`, `fuzzy_match`) for the derived answer.
6.  Identifying a single, representative SQL query for reference.

**1. Contextual Information:**

*   **Database Schema:**
    --- START OF DATABASE SCHEMA ---
    {MAGENTO_SCHEMA_CONTENT}
    --- END OF DATABASE SCHEMA ---

*   **Sample Data from Tables (Illustrative - your queries will hit the live DB):**
    --- START OF SAMPLE DATA ---
    {TABLE_SAMPLES_CONTENT}
    --- END OF SAMPLE DATA ---

*   **Key Magento Schema Characteristics & EAV Model (Essential for query design):**
    *   **EAV (Entity-Attribute-Value):** Many entities (products, categories, customers) use EAV.
        *   Core entity table: e.g., `catalog_product_entity`.
        *   Attribute definition: `eav_attribute` (use `attribute_code` to find `attribute_id`).
        *   Value tables by data type: e.g., `catalog_product_entity_varchar`, `_int`, `_decimal`, `_datetime`, `_text`. Joined on `entity_id` and `attribute_id`.
        *   `eav_entity_type`: Use `entity_type_code` (e.g., 'catalog_product', 'customer') to find `entity_type_id` for joining with `eav_attribute`.
    *   **Store Scopes:** Data can be global (store_id=0 or admin), website-specific, or store-view-specific. Queries for attributes often need to specify `store_id`. `store_id = 0` is the admin/default scope for many attributes. If not specified, assume default/admin scope.
    *   **Product Types:** `catalog_product_entity.type_id` can be 'simple', 'configurable', 'virtual', 'bundle', 'downloadable', 'grouped'. This affects related tables.
    *   **Inventory (MSI - Multi-Source Inventory if applicable, else `cataloginventory_stock_item`):**
        *   MSI: `inventory_source_item` manages stock per source (e.g., 'default'). `status = 1` (In Stock), `status = 0` (Out of Stock).
        *   Legacy: `cataloginventory_stock_item` for stock details, `is_in_stock` field.
    *   **Order Workflow:** `quote` (cart) -> `sales_order` -> `sales_invoice`, `sales_shipment`, `sales_creditmemo`. `sales_order_item` for items in an order.
    *   **Flat/Grid Tables:** Tables like `sales_order_grid`, `customer_grid_flat` are denormalized for admin panel performance. For precise, current data, querying base tables is often preferred.
    *   **Date/Time:** Timestamps (e.g., `created_at`, `updated_at`) are common. Use standard MySQL date/time functions.
    *   **Foreign Keys:** Pay close attention to foreign key relationships for accurate JOINs (e.g., `sales_order_item.order_id` -> `sales_order.entity_id`).

**2. Your Task: Generate ONE Query-Based QA Item**

Follow these phases meticulously:

**Phase A: Question Formulation & Iterative Data Collection**
1.  **Formulate an `initial_question` (string):**
    *   This should be a clear, natural language question seeking specific information from the database.
    *   Aim for questions that are interesting, non-trivial (may require joins or simple aggregations), and definitively answerable using SELECT queries with the provided schema.
2.  **Iterative SQL Execution and Refinement (using your tool call for the mysql mcp server):**
    *   **Plan & Execute:** Based on the `initial_question` and your understanding of the schema, formulate a precise SELECT SQL query. Execute this query using your database tool.
    *   **Analyze Results:** Carefully examine the data returned by your query.
    *   **Self-Correct & Iterate:**
        *   Is the question fully answerable with the current results?
        *   Do the results suggest the question is ambiguous, too broad, too narrow, or unanswerable with the available data?
        *   If necessary, **refine your question** into a `final_question` that is perfectly aligned with the data you can retrieve.
        *   If current results are insufficient or lead to new questions to pinpoint the answer, **formulate and execute additional SELECT SQL queries**.
        *   This is an iterative loop. Continue executing SQL queries and refining your `final_question` until you are confident that:
            a.  The `final_question` is clear, well-posed, and reasonable for this database.
            b.  You have successfully retrieved all the necessary data from the database to form a definitive answer to the `final_question`.
        *   To increase diversity, avoid using the elements from the `table_samples_cache` in your question formulation. You can check the database table for other elements.
    *   **Record Keeping (Internal):** Mentally (or if you have a scratchpad, use it) keep track of the sequence of SQL queries you ran and the key pieces of information extracted from their results. This trail is vital for the next phases.

**Phase B: Answer Derivation**
1.  Based on the **verified and complete data** collected from the database in Phase A:
    *   Formulate an **`llm_derived_answer`** (string): This is the concise, factual, natural language answer to your `final_question`. It must be *exclusively and directly supported* by the data you retrieved from the database. Avoid inference beyond the retrieved facts.

**Phase C: Validation Rule Design**
1.  Based on your `final_question` and the specific `llm_derived_answer` (which is grounded in actual database data):
    *   Design one or more **`validation_rules`**. These rules specify how the `llm_derived_answer` (or a future, independently generated answer to the same `final_question`) should be programmatically checked for correctness.
    *   The `validation_rules` field in the output will be a **list of rule objects**.
    *   Each rule object in the list must have two keys:
        *   `type` (string): Must be one of:
            *   `"exact_match"`: The candidate answer string must exactly match the `expected_value`.
            *   `"must_include"`: The candidate answer string must contain the `expected_value` as a substring.
            *   `"fuzzy_match"`: The candidate answer string should be close to the `expected_value`. For the purpose of this task, if you choose `fuzzy_match`, the `expected_value` should still be the precise string or numerical part extracted from the database that you are allowing some fuzziness around (e.g., for "Approximately 2h 58min", the `expected_value` could be "2h 58min"). The fuzzy matching logic itself will be handled by an external evaluator.
        *   `expected_value` (string): The specific string, number (as a string), or piece of information that the candidate answer will be compared against. This value MUST be derived directly from your `llm_derived_answer` and therefore from the data retrieved from the database.
    *   **Multiple Rules:** If multiple rules are provided in the list, they are treated with an AND relationship (i.e., a candidate answer must satisfy all rules to be considered valid).
    *   **Examples:**
        *   `final_question`: "What is the name of the customer with email 'test@example.com'?"
        *   `llm_derived_answer`: "The customer's name is Samantha Jones."
        *   `validation_rules`: `[{{ "type": "must_include", "expected_value": "Samantha Jones" }}]`
        *   `final_question`: "List the product name and SKU for product ID 123."
        *   `llm_derived_answer`: "Product: Cool T-Shirt, SKU: CT001."
        *   `validation_rules`: `[{{ "type": "must_include", "expected_value": "Cool T-Shirt" }}, {{ "type": "must_include", "expected_value": "CT001" }}]`

**Phase D: Reference SQL Formulation**
1.  From the sequence of SQL queries you executed in Phase A, select or compose a **single, concise `reference_sql` (string)**.
    *   This SQL query should be the most direct and representative query that could be used to obtain or verify the core information presented in the `llm_derived_answer`.
    *   It doesn't have to be the *only* query you ran, but it should be the most illustrative one for someone trying to independently verify the answer.

**3. Final Output Format (for this ONE QA item):**

Provide your response strictly as a single JSON object with the following structure. Do not include any explanatory text outside of this JSON structure.

```json
{{
  "final_question": "Your refined and validated question string.",
  "llm_derived_answer": "The natural language answer derived exclusively from actual database data.",
  "validation_rules": [
    {{
      "type": "exact_match" /* or "must_include" or "fuzzy_match */,
      "expected_value": "Specific value derived from the database data"
    }}
    /* , {{ ... more rules if needed, each as an object in this list }} */
  ],
  "reference_sql": "A single, representative SELECT SQL query."
}}
"""

prompt2 = f"""
You are an expert Magento 2 database analyst, Python programmer, and an autonomous agent. **You ARE equipped with a tool (e.g., 'mysql_mcp_server_tool_call') to interact directly with a MySQL database. You MUST use this tool for all database queries. Your answers and derived values will be based SOLELY on the data retrieved via this tool from the live database.** Your current task is to generate ONE comprehensive Question-Answer (QA) item related to the Magento 2 database, focusing on "query" type tasks.

This process involves:
1.  Formulating an initial question.
2.  Iteratively **using your database tool** to execute SELECT SQL queries against the **live database** to gather information.
3.  Analyzing the **actual SQL results returned by your tool** to refine the question, ensure its answerability, and collect all necessary data.
4.  Deriving a precise answer based on the **verified data obtained from your tool calls to the live database**.
5.  Designing specific validation rules (`exact_match`, `must_include`, `fuzzy_match`) for the derived answer, based on **actual data from your tool**.
6.  Identifying a single, representative SQL query for reference.

**1. Contextual Information:**

*   **Database Schema:**
    --- START OF DATABASE SCHEMA ---
    {MAGENTO_SCHEMA_CONTENT}
    --- END OF DATABASE SCHEMA ---

*   **Sample Data from Tables (Illustrative ONLY - for initial understanding and question diversity ideas. DO NOT use this sample data to derive your final answer or any values within it. Your queries will hit the live DB via your tool, and your answer MUST be based on those live results.):**
    --- START OF SAMPLE DATA ---
    {TABLE_SAMPLES_CONTENT}
    --- END OF SAMPLE DATA ---

*   **Key Magento Schema Characteristics & EAV Model (Essential for query design):**
    *   EAV (Entity-Attribute-Value): Many entities (products, categories, customers) use EAV.
        *   Core entity table: e.g., `catalog_product_entity`.
        *   Attribute definition: `eav_attribute` (use `attribute_code` to find `attribute_id`).
        *   Value tables by data type: e.g., `catalog_product_entity_varchar`, `_int`, `_decimal`, `_datetime`, `_text`. Joined on `entity_id` and `attribute_id`.
        *   `eav_entity_type`: Use `entity_type_code` (e.g., 'catalog_product', 'customer') to find `entity_type_id` for joining with `eav_attribute`.
    *   Store Scopes: Data can be global (store_id=0 or admin), website-specific, or store-view-specific. Queries for attributes often need to specify `store_id`. `store_id = 0` is the admin/default scope for many attributes. If not specified, assume default/admin scope.
    *   Product Types: `catalog_product_entity.type_id` can be 'simple', 'configurable', 'virtual', 'bundle', 'downloadable', 'grouped'. This affects related tables.
    *   Inventory (MSI - Multi-Source Inventory if applicable, else `cataloginventory_stock_item`):
        *   MSI: `inventory_source_item` manages stock per source (e.g., 'default'). `status = 1` (In Stock), `status = 0` (Out of Stock).
        *   Legacy: `cataloginventory_stock_item` for stock details, `is_in_stock` field.
    *   Order Workflow: `quote` (cart) -> `sales_order` -> `sales_invoice`, `sales_shipment`, `sales_creditmemo`. `sales_order_item` for items in an order.
    *   Flat/Grid Tables: Tables like `sales_order_grid`, `customer_grid_flat` are denormalized for admin panel performance. For precise, current data, querying base tables is often preferred.
    *   Date/Time: Timestamps (e.g., `created_at`, `updated_at`) are common. Use standard MySQL date/time functions.
    *   Foreign Keys: Pay close attention to foreign key relationships for accurate JOINs (e.g., `sales_order_item.order_id` -> `sales_order.entity_id`).

**2. Your Task: Generate ONE Query-Based QA Item**

Follow these phases meticulously:

**Phase A: Question Formulation & Iterative Data Collection (USING YOUR DATABASE TOOL)**
1.  **Formulate an `initial_question` (string):**
    *   This should be a clear, natural language question seeking specific information from the database.
    *   Aim for questions that are interesting, non-trivial, and definitively answerable using SELECT queries with the provided schema.
    *   To increase diversity, you may look at the `TABLE_SAMPLES_CONTENT` for *ideas* on what kind of data exists, but **do not use the specific values from `TABLE_SAMPLES_CONTENT` in your question if you intend to use them in your answer later.** Your goal is to query the live database for fresh, potentially different data.
2.  **Iterative SQL Execution and Refinement (using your tool call for the mysql mcp server):**
    *   **Plan & Execute SQL with Tool:** Based on the `initial_question` and your understanding of the schema, formulate a precise SELECT SQL query. **State: "I will now execute the following SQL query using my database tool: [Your SQL Query]". Then, (as an agent) you would make the tool call.**
    *   **Process Tool Results:** After the conceptual tool call, **state: "My database tool returned the following results: [Describe or list the key data returned by the tool, e.g., 'a count of 50', or 'rows with product names and prices', or 'customer email xxxx@example.com']".**
    *   **Analyze ACTUAL Tool Results:** Carefully examine the **actual data returned by your tool**.
    *   **Self-Correct & Iterate based on ACTUAL Tool Results:**
        *   Is the question fully answerable with the current **actual tool results**?
        *   Do the **actual tool results** suggest the question is ambiguous, too broad, too narrow, or unanswerable?
        *   If necessary, **refine your question** into a `final_question` that is perfectly aligned with the **actual data your tool can retrieve**.
        *   If current **actual tool results** are insufficient, **formulate and execute additional SELECT SQL queries using your tool**, again stating your intent and the (conceptual) results.
        *   This is an iterative loop. Continue **using your tool to execute SQL queries** and refining your `final_question` until you are confident that:
            a.  The `final_question` is clear, well-posed, and reasonable.
            b.  You have successfully retrieved all necessary data **from the live database via your tool** to form a definitive answer.
    *   **Record Keeping (Internal):** Keep track of the sequence of SQL queries you **executed with your tool** and the key information extracted from their **actual results**.

**Phase B: Answer Derivation (from ACTUAL Tool Results)**
1.  Based on the **verified and complete data collected from the live database via your tool** in Phase A:
    *   Formulate an **`llm_derived_answer`** (string): This is the concise, factual, natural language answer to your `final_question`. It must be *exclusively and directly supported* by the **actual data you retrieved using your database tool**. Avoid inference beyond these facts.

**Phase C: Validation Rule Design (from ACTUAL Tool Results)**
1.  Based on your `final_question` and the specific `llm_derived_answer` (which is grounded in **actual data retrieved via your tool**):
    *   Design one or more **`validation_rules`**.
    *   The `validation_rules` field in the output will be a **list of rule objects**.
    *   Each rule object: `type` ("exact_match", "must_include", "fuzzy_match"), `expected_value` (string).
    *   The `expected_value` MUST be derived directly from your `llm_derived_answer` and therefore from the **actual data retrieved from the live database via your tool**.
    *   Multiple rules imply an AND relationship.

**Phase D: Reference SQL Formulation**
1.  From the sequence of SQL queries you **executed with your tool** in Phase A, select or compose a **single, concise `reference_sql` (string)**.
    *   This SQL query should be the most direct and representative query that could be used to obtain or verify the core information presented in the `llm_derived_answer` (which came from **actual tool results**).

**3. Final Output Format (for this ONE QA item):**

Provide your response strictly as a single JSON object. Do not include any explanatory text outside of this JSON structure.

```json
{{
  "final_question": "Your refined and validated question string.",
  "llm_derived_answer": "The natural language answer derived exclusively from actual database data obtained via your tool.",
  "validation_rules": [
    {{
      "type": "exact_match" /* or "must_include" or "fuzzy_match */,
      "expected_value": "Specific value derived from the actual database data obtained via your tool"
    }}
    /* , {{ ... more rules if needed, each as an object in this list }} */
  ],
  "reference_sql": "A single, representative SELECT SQL query that reflects a tool execution."
}}
```

**Agent Workflow and Self-Correction Reminder:**
Your thought process should explicitly narrate steps like:
1.  "I will formulate an initial question..."
2.  "To answer this, I will now execute the following SQL query using my database tool: `SELECT ...`"
3.  "My database tool returned: `[{{'column_name': 'actual_value_from_db'}}]` (or a summary if large)."
4.  "Based on these *actual results from my tool*, I will now..." (derive answer, refine question, design rules, etc.).
**CRITICALLY: Do NOT use the `TABLE_SAMPLES_CONTENT` to determine the `llm_derived_answer` or `expected_value` in validation rules. These MUST come from the (simulated if you are an LLM, but conceptually *actual*) results of your tool calls.** If you find yourself deriving an answer value from the sample data text, you are violating the instructions. You must act as if you received real data from your tool.

Please proceed with generating one QA item according to these **strict** instructions.

"""

prompt3 = f"""
You are an expert Magento 2 database analyst, Python programmer, and an autonomous agent. **You ARE equipped with a tool (e.g., 'mysql_mcp_server_tool_call') to interact directly with a MySQL database. You MUST use this tool for all database queries. Your answers and derived values will be based SOLELY on the data retrieved via this tool from the live database.** Your current task is to generate ONE comprehensive Question-Answer (QA) item related to the Magento 2 database, focusing on "query" type tasks.

This process involves:
1.  Formulating an initial question, considering ranking and tie-handling.
2.  Iteratively **using your database tool** to execute SELECT SQL queries against the **live database** to gather information.
3.  Analyzing the **actual SQL results returned by your tool** to refine the question, ensure its answerability, and collect all necessary data, especially for ranking scenarios.
4.  Deriving a precise answer based on the **verified data obtained from your tool calls to the live database**.
5.  Designing specific, more flexible validation rules (`must_include`, `fuzzy_match`) for the derived answer, based on **actual data from your tool**.
6.  Identifying a single, representative SQL query for reference.

**1. Contextual Information:**

*   **Database Schema:**
    --- START OF DATABASE SCHEMA ---
    {MAGENTO_SCHEMA_CONTENT}
    --- END OF DATABASE SCHEMA ---

*   **Sample Data from Tables (Illustrative ONLY - for initial understanding and question diversity ideas. DO NOT use this sample data to derive your final answer or any values within it. Your queries will hit the live DB via your tool, and your answer MUST be based on those live results.):**
    --- START OF SAMPLE DATA ---
    {TABLE_SAMPLES_CONTENT}
    --- END OF SAMPLE DATA ---

*   **Key Magento Schema Characteristics & EAV Model (Essential for query design):**
    *   EAV (Entity-Attribute-Value): (Details as before)
    *   Store Scopes: (Details as before)
    *   Product Types: (Details as before)
    *   Inventory: (Details as before)
    *   Order Workflow: (Details as before)
    *   Flat/Grid Tables: (Details as before)
    *   Date/Time: (Details as before)
    *   Foreign Keys: (Details as before)

**2. Your Task: Generate ONE Query-Based QA Item**

Follow these phases meticulously:

**Phase A: Question Formulation & Iterative Data Collection (USING YOUR DATABASE TOOL)**
1.  **Formulate an `initial_question` (string):**
    *   This should be a clear, natural language question seeking specific information.
    *   **Ranking Questions & Tie Handling Constraint:**
        *   If your question involves ranking (e.g., "most X", "top N", "second most Y"), you **MUST** design your SQL and question to handle ties using a "dense rank" or "skip rank" (natural ranking) approach. For example, if there are two items tied for 1st place, both should be listed as 1st, and the next rank would be 3rd (not 2nd).
        *   If your initial idea is to ask for a specific rank (e.g., "the second most expensive"), and your database query (or a preliminary check) reveals that this rank might be skipped due to ties at a higher rank, you **MUST adjust your `final_question`** to ask for a rank that will definitely exist (e.g., "the most expensive products" if there are ties, or "the third most expensive product" if the 2nd rank is skipped).
    *   Aim for questions that are interesting, non-trivial, and definitively answerable.
    *   To increase diversity, look at `TABLE_SAMPLES_CONTENT` for *ideas* on data types, but **do not use its specific values in your question if they are part of the intended answer from the live DB.**
2.  **Iterative SQL Execution and Refinement (using your tool call for the mysql mcp server):**
    *   **Plan & Execute SQL with Tool:** Based on the `initial_question` (and ranking constraints), formulate a precise SELECT SQL query. **State: "I will now execute the following SQL query using my database tool: [Your SQL Query]". Then, (as an agent) you would make the tool call.** Remember to use window functions like `DENSE_RANK()` or `RANK()` if needed for ranking questions to handle ties correctly.
    *   **Process Tool Results:** After the tool call, **state: "My database tool returned the following results: [Describe or list the key data returned by the tool, e.g., 'product A price $50 rank 1, product B price $50 rank 1, product C price $40 rank 3']".**
    *   **Analyze ACTUAL Tool Results:** Carefully examine the **actual data returned by your tool**, paying close attention to how ties affect rankings if applicable.
    *   **Self-Correct & Iterate based on ACTUAL Tool Results:**
        *   Is the question fully answerable? Does it align with the ranking constraint if ties exist?
        *   If necessary, **refine your question** into a `final_question` perfectly aligned with the **actual data your tool can retrieve and the ranking rules**.
        *   If current **actual tool results** are insufficient, **formulate and execute additional SELECT SQL queries using your tool**.
        *   This is an iterative loop. Continue **using your tool to execute SQL queries** and refining your `final_question` until you are confident that:
            a.  The `final_question` is clear, well-posed, reasonable, and respects ranking/tie rules.
            b.  You have successfully retrieved all necessary data **from the live database via your tool**.
    *   **Record Keeping (Internal):** Track SQL queries **executed with your tool** and key information from their **actual results**.

**Phase B: Answer Derivation (from ACTUAL Tool Results)**
1.  Based on the **verified and complete data collected from the live database via your tool** in Phase A:
    *   Formulate an **`llm_derived_answer`** (string): This is the concise, factual, natural language answer to your `final_question`. It must be *exclusively and directly supported* by the **actual data you retrieved using your database tool**. Avoid inference beyond these facts.

**Phase C: Validation Rule Design (from ACTUAL Tool Results)**
1.  Based on your `final_question` and the specific `llm_derived_answer` (grounded in **actual data retrieved via your tool**):
    *   Design one or more **`validation_rules`**. These rules specify how key pieces of information within the `llm_derived_answer` should be programmatically checked.
    *   The `validation_rules` field in the output will be a **list of rule objects**.
    *   Each rule object:
        *   `type` (string): **Primarily use `"must_include"` or `"fuzzy_match"`.** Use `"exact_match"` very sparingly, only if an entire short, specific, and non-rephraseable string is the sole answer.
        *   `expected_value` (string): The specific string, number (as a string), or piece of information *extracted* from your `llm_derived_answer` that the candidate answer will be compared against. This value MUST be derived directly from the **actual data retrieved from the live database via your tool**.
    *   **Focus on validating key entities or values within the answer, not the entire rephrased sentence.**
        *   Example: If `llm_derived_answer` is "The most expensive products are P1 (ID: 10) and P2 (ID: 20), both costing $100.00.", good rules might be:
            `[{{ "type": "must_include", "expected_value": "P1" }}, {{ "type": "must_include", "expected_value": "10" }}, {{ "type": "must_include", "expected_value": "P2" }}, {{ "type": "must_include", "expected_value": "20" }}, {{ "type": "must_include", "expected_value": "100.00" }}]`
        *   For "fuzzy_match", the `expected_value` should be the precise string/number you're allowing fuzziness around (e.g., "100.00" for price, "Product Name Example" for a name).
    *   Multiple rules imply an AND relationship.

**Phase D: Reference SQL Formulation**
1.  From the sequence of SQL queries you **executed with your tool** in Phase A, select or compose a **single, concise `reference_sql` (string)**.
    *   If it was a ranking question, this SQL should ideally include the logic (e.g., window function) used to handle ties and determine the rank.

**3. Final Output Format (for this ONE QA item):**

Provide your response strictly as a single JSON object.

```json
{{
  "final_question": "Your refined and validated question string, respecting ranking/tie rules and boundary exploration.",
  "llm_derived_answer": "The natural language answer derived exclusively from actual database data obtained via your tool. If multiple items due to ties, list them.",
  "validation_rules": [
    {{
      "type": "must_include" /* or "fuzzy_match", rarely "exact_match" for specific short values */,
      "expected_value": "Specific key value/entity extracted from the answer, based on actual database data obtained via your tool"
    }}
    /* , {{ ... more rules if needed, each as an object in this list }} */
  ],
  "reference_sql": "A single, representative SELECT SQL query, including ranking logic if applicable."
}}
```

**Agent Workflow and Self-Correction Reminder:**
Your thought process should explicitly narrate steps like:
1.  "I will formulate an initial question..."
2.  "To answer this, I will now execute the following SQL query using my database tool: `SELECT ...`"
3.  "My database tool returned: `[{{'column_name': 'actual_value_from_db'}}]` (or a summary if large, including results of probing/ranking)."
4.  "Based on these *actual results from my tool*, I will now..." (derive answer, refine question, design rules, etc.).
**CRITICALLY: Do NOT use the `TABLE_SAMPLES_CONTENT` to determine the `llm_derived_answer` or `expected_value` in validation rules. These MUST come from the (simulated if you are an LLM, but conceptually *actual*) results of your tool calls.** Adhere strictly to the ranking/tie handling, top rank boundary exploration (preferring window functions over iterative LIMIT adjustments where possible for the final reference SQL), and validation rule type guidelines.

Please proceed with generating one QA item according to these **strict** instructions.

"""

prompt4 = f"""
You are an expert Magento 2 database analyst, Python programmer, and an autonomous agent. **You ARE equipped with a tool (e.g., 'mysql_mcp_server_tool_call') to interact directly with a MySQL database. You MUST use this tool for all database queries. Your answers and derived values will be based SOLELY on the data retrieved via this tool from the live database.** Your current task is to generate ONE comprehensive Question-Answer (QA) item related to the Magento 2 database, focusing on "query" type tasks.

This process involves:
1.  Formulating an initial question, considering ranking and tie-handling, including boundary exploration for top ranks.
2.  Iteratively **using your database tool** to execute SELECT SQL queries against the **live database** to gather information.
3.  Analyzing the **actual SQL results returned by your tool** to refine the question, ensure its answerability, and collect all necessary data, especially for ranking scenarios with ties.
4.  Deriving a precise answer based on the **verified data obtained from your tool calls to the live database**.
5.  Designing specific, more flexible validation rules (`must_include`, `fuzzy_match`) for the derived answer, based on **actual data from your tool**.
6.  Identifying a single, representative SQL query for reference.

**1. Contextual Information:**

*   **Database Schema:**
    --- START OF DATABASE SCHEMA ---
    {MAGENTO_SCHEMA_CONTENT}
    --- END OF DATABASE SCHEMA ---

*   **Sample Data from Tables (Illustrative ONLY - for initial understanding and question diversity ideas. DO NOT use this sample data to derive your final answer or any values within it. Your queries will hit the live DB via your tool, and your answer MUST be based on those live results.):**
    --- START OF SAMPLE DATA ---
    {TABLE_SAMPLES_CONTENT} 
    --- END OF SAMPLE DATA ---

*   **Key Magento Schema Characteristics & EAV Model (Essential for query design):**
    *   **EAV (Entity-Attribute-Value):** Many entities (products, categories, customers) use EAV.
        *   Core entity table: e.g., `catalog_product_entity`.
        *   Attribute definition: `eav_attribute` (use `attribute_code` to find `attribute_id`).
        *   Value tables by data type: e.g., `catalog_product_entity_varchar`, `_int`, `_decimal`, `_datetime`, `_text`. Joined on `entity_id` and `attribute_id`.
        *   `eav_entity_type`: Use `entity_type_code` (e.g., 'catalog_product', 'customer') to find `entity_type_id` for joining with `eav_attribute`.
    *   **Store Scopes:** Data can be global (store_id=0 or admin), website-specific, or store-view-specific. Queries for attributes often need to specify `store_id`. `store_id = 0` is the admin/default scope for many attributes. If not specified, assume default/admin scope.
    *   **Product Types:** `catalog_product_entity.type_id` can be 'simple', 'configurable', 'virtual', 'bundle', 'downloadable', 'grouped'. This affects related tables.
    *   **Inventory (MSI - Multi-Source Inventory if applicable, else `cataloginventory_stock_item`):**
        *   MSI: `inventory_source_item` manages stock per source (e.g., 'default'). `status = 1` (In Stock), `status = 0` (Out of Stock).
        *   Legacy: `cataloginventory_stock_item` for stock details, `is_in_stock` field.
    *   **Order Workflow:** `quote` (cart) -> `sales_order` -> `sales_invoice`, `sales_shipment`, `sales_creditmemo`. `sales_order_item` for items in an order.
    *   **Flat/Grid Tables:** Tables like `sales_order_grid`, `customer_grid_flat` are denormalized for admin panel performance. For precise, current data, querying base tables is often preferred.
    *   **Date/Time:** Timestamps (e.g., `created_at`, `updated_at`) are common. Use standard MySQL date/time functions.
    *   **Foreign Keys:** Pay close attention to foreign key relationships for accurate JOINs (e.g., `sales_order_item.order_id` -> `sales_order.entity_id`).

**2. Your Task: Generate ONE Query-Based QA Item**

Follow these phases meticulously:

**Phase A: Question Formulation & Iterative Data Collection (USING YOUR DATABASE TOOL)**
1.  **Formulate an `initial_question` (string):**
    *   This should be a clear, natural language question seeking specific information from the database.
    *   Aim for questions that are interesting, non-trivial (may require joins or simple aggregations), and definitively answerable using SELECT queries with the provided schema.
    *   **Ranking Questions & Tie Handling Constraint (Including Top Rank Boundary Exploration):**
        *   If your question involves ranking (e.g., "most X", "top N", "Nth ranked Y"), you **MUST** design your SQL and question to handle ties using a "dense rank" or "skip rank" (natural ranking) approach. For example, if there are two items tied for 1st place, both should be listed as 1st, and the next rank would be 3rd (not 2nd). Use window functions like `DENSE_RANK()` or `RANK()` in your SQL for this.
        *   **Boundary Exploration for Top Ranks (e.g., "the most X", "the highest Y"):**
            1.  When querying for the *absolute top* ranked items (e.g., "most expensive," "highest quantity"), first execute your query (e.g., `ORDER BY value DESC`) with a moderate `LIMIT` (e.g., `LIMIT 10` or `LIMIT 20`).
            2.  **Analyze the results:** If all items returned by this initial limited query share the *exact same value* for the ranking criterion (e.g., all 10 products have the same price of $99.00), you **MUST assume** the `LIMIT` might be cutting off other items that also share this top rank.
            3.  **Probe Further:** In this scenario, you **MUST** re-execute the query with a significantly larger `LIMIT` (e.g., `LIMIT 50` or `LIMIT 100`). The goal is to retrieve enough rows to see where the top value ends and the next distinct value begins. *Alternatively, and preferably for a robust solution, reformulate your query to use a window function like `DENSE_RANK()` and then select all items where the rank is 1, without relying on `LIMIT` to find all ties at the top.* For example: `SELECT ... FROM (SELECT ..., DENSE_RANK() OVER (ORDER BY ranking_column DESC) as dr FROM ...) ranked_items WHERE dr = 1;`
            4.  Only after this probing (or using a robust window function approach) can you confidently identify *all* items that genuinely share the absolute top rank. Your `final_question` and `llm_derived_answer` must reflect all such tied items.
        *   If your initial idea is to ask for a specific rank (e.g., "the second most expensive"), and your database query (or a preliminary check including the boundary exploration above) reveals that this rank might be skipped due to ties at a higher rank, you **MUST adjust your `final_question`** to ask for a rank that will definitely exist (e.g., "the most expensive products" or "the products ranked third most expensive").
    *   To increase diversity, look at `TABLE_SAMPLES_CONTENT` for *ideas* on data types, but **do not use its specific values in your question if they are part of the intended answer from the live DB.**
2.  **Iterative SQL Execution and Refinement (using your tool call for the mysql mcp server):**
    *   **Plan & Execute SQL with Tool:** Based on the `initial_question` (and ranking/boundary exploration constraints), formulate a precise SELECT SQL query. **State: "I will now execute the following SQL query using my database tool: [Your SQL Query]". Then, (as an agent) you would make the tool call.** For top rank exploration, you might narrate: "Initial query for top items: `... ORDER BY ... DESC LIMIT 10`". Then after results: "All 10 items have the same value. I will now use a DENSE_RANK query to find all items with rank 1."
    *   **Process Tool Results:** After each tool call, **state: "My database tool returned the following results: [Describe or list the key data returned by the tool, e.g., 'The DENSE_RANK query returned 16 items with rank 1, all having price $99.00, followed by items at $92.00 with rank 2. So there are 16 items at the top price.']".**
    *   **Analyze ACTUAL Tool Results:** Carefully examine the **actual data returned by your tool**, paying close attention to tie handling and the outcome of any boundary exploration.
    *   **Self-Correct & Iterate based on ACTUAL Tool Results:**
        *   Is the question fully answerable? Does it align with the ranking constraints, including accurately identifying all items at the top rank if boundary exploration was needed?
        *   If necessary, **refine your question** into a `final_question` that is perfectly aligned with the **actual data your tool can retrieve and the ranking rules**.
        *   If current **actual tool results** are insufficient, **formulate and execute additional SELECT SQL queries using your tool**, again stating your intent and the (conceptual) results.
        *   This is an iterative loop. Continue **using your tool to execute SQL queries** and refining your `final_question` until you are confident that:
            a.  The `final_question` is clear, well-posed, reasonable, and respects ranking/tie rules.
            b.  You have successfully retrieved all necessary data **from the live database via your tool** to form a definitive answer.
    *   **Record Keeping (Internal):** Keep track of the sequence of SQL queries you **executed with your tool** (including probing queries or window function based queries) and the key information extracted from their **actual results**.

**Phase B: Answer Derivation (from ACTUAL Tool Results)**
1.  Based on the **verified and complete data collected from the live database via your tool** (including results from any boundary exploration for top ranks):
    *   Formulate an **`llm_derived_answer`** (string): This is the concise, factual, natural language answer to your `final_question`. It must be *exclusively and directly supported* by the **actual data you retrieved using your database tool**. If listing multiple items due to ties (especially at the top rank after exploration), present them clearly and accurately reflect the total count of such tied items if appropriate for the question.

**Phase C: Validation Rule Design (from ACTUAL Tool Results)**
1.  Based on your `final_question` and the specific `llm_derived_answer` (which is grounded in **actual data retrieved via your tool**):
    *   Design one or more **`validation_rules`**. These rules specify how key pieces of information within the `llm_derived_answer` should be programmatically checked for correctness.
    *   The `validation_rules` field in the output will be a **list of rule objects**.
    *   Each rule object in the list must have two keys:
        *   `type` (string): **Primarily use `"must_include"` or `"fuzzy_match"`.** Use `"exact_match"` very sparingly, only if an entire short, specific, and non-rephraseable string (like a status code or a very fixed short identifier) is the sole answer or a key part of it.
        *   `expected_value` (string): The specific string, number (as a string), or piece of information *extracted* from your `llm_derived_answer` that the candidate answer will be compared against. This value MUST be derived directly from the **actual data retrieved from the live database via your tool**.
    *   **Focus on validating key entities or values within the answer, not the entire rephrased sentence.**
        *   Example: If `final_question` was "What are the names and prices of the most expensive products?" and `llm_derived_answer` is "The most expensive products are 'Awesome T-Shirt' ($99.00) and 'Super Gadget' ($99.00).", rules might be:
            `[{{ "type": "must_include", "expected_value": "Awesome T-Shirt" }}, {{ "type": "must_include", "expected_value": "Super Gadget" }}, {{ "type": "must_include", "expected_value": "99.00" }}]`
        *   For "fuzzy_match", the `expected_value` should be the precise string/number you're allowing fuzziness around (e.g., "99.00" for price, "Product Name Example" for a name).
    *   Multiple rules imply an AND relationship (i.e., a candidate answer must satisfy all rules to be considered valid).

**Phase D: Reference SQL Formulation**
1.  From the sequence of SQL queries you **executed with your tool** in Phase A, select or compose a **single, concise `reference_sql` (string)**.
    *   If it was a ranking question involving top-rank boundary exploration, this SQL should ideally be the query that definitively identifies *all* items belonging to that top rank. **Using a window function like `DENSE_RANK()` in conjunction with filtering on rank = 1 (e.g., `... FROM (SELECT ..., DENSE_RANK() OVER (ORDER BY ranking_column DESC) as dr FROM ...) ranked_items WHERE dr = 1;`) is strongly preferred as it's robust against an unknown number of ties.**

**3. Final Output Format (for this ONE QA item):**

Provide your response strictly as a single JSON object with the following structure. Do not include any explanatory text outside of this JSON structure.

```json
{{
  "final_question": "Your refined and validated question string, respecting ranking/tie rules and boundary exploration.",
  "llm_derived_answer": "The natural language answer derived exclusively from actual database data obtained via your tool. If multiple items due to ties, list them clearly and accurately.",
  "validation_rules": [
    {{
      "type": "must_include" /* or "fuzzy_match", rarely "exact_match" for specific short values */,
      "expected_value": "Specific key value/entity extracted from the answer, based on actual database data obtained via your tool"
    }}
    /* , {{ ... more rules if needed, each as an object in this list }} */
  ],
  "reference_sql": "A single, representative SELECT SQL query, ideally using window functions for robust ranking and tie handling."
}}
```

**Agent Workflow and Self-Correction Reminder:**
Your thought process should explicitly narrate steps like:
1.  "I will formulate an initial question..."
2.  "To answer this, I will now execute the following SQL query using my database tool: `SELECT ...`"
3.  "My database tool returned: `[{{'column_name': 'actual_value_from_db'}}]` (or a summary if large, including results of probing/ranking and fetched product attributes like names)."
4.  "Based on these *actual results from my tool*, I will now..." (derive answer, refine question, design rules, etc.).
**CRITICALLY: Do NOT use the `TABLE_SAMPLES_CONTENT` to determine the `llm_derived_answer` or `expected_value` in validation rules. These MUST come from the (simulated if you are an LLM, but conceptually *actual*) results of your tool calls.** Adhere strictly to all guidelines: tool usage, ranking/tie handling, top rank boundary exploration (preferring window functions), product attribute clarity and congruence, and validation rule types.

Please proceed with generating one QA item according to these **strict** instructions.
"""

prompt5 = f"""
You are an expert Magento 2 database analyst, Python programmer, and an autonomous agent. **You ARE equipped with a tool (e.g., 'mysql_mcp_server_tool_call') to interact directly with a MySQL database. You MUST use this tool for all database queries. Your answers and derived values will be based SOLELY on the data retrieved via this tool from the live database.** Your current task is to generate ONE comprehensive Question-Answer (QA) item related to the Magento 2 database, focusing on "query" type tasks.

This process involves:
1.  Formulating an initial question, ensuring clarity about product attributes (especially name) if "products" are requested.
2.  Iteratively **using your database tool** to execute SELECT SQL queries against the **live database** to gather information, including product names where appropriate.
3.  Analyzing the **actual SQL results returned by your tool** to refine the question and ensure the answer provides the expected product attributes.
4.  Deriving a precise answer based on the **verified data obtained from your tool calls to the live database**, ensuring it aligns with the question's implied or explicit request for product attributes.
5.  Designing specific, more flexible validation rules (`must_include`, `fuzzy_match`) for the derived answer.
6.  Identifying a single, representative SQL query for reference.

**1. Contextual Information:**

*   **Database Schema:**
    --- START OF DATABASE SCHEMA ---
    {MAGENTO_SCHEMA_CONTENT}
    --- END OF DATABASE SCHEMA ---

*   **Sample Data from Tables (Illustrative ONLY - for initial understanding and question diversity ideas. DO NOT use this sample data to derive your final answer or any values within it. Your queries will hit the live DB via your tool, and your answer MUST be based on those live results.):**
    --- START OF SAMPLE DATA ---
    {TABLE_SAMPLES_CONTENT}
    --- END OF SAMPLE DATA ---

*   **Key Magento Schema Characteristics & EAV Model (Essential for query design):**
    *   **EAV (Entity-Attribute-Value):** Many entities (products, categories, customers) use EAV.
        *   Core entity table: e.g., `catalog_product_entity`.
        *   Attribute definition: `eav_attribute` (use `attribute_code` to find `attribute_id`).
        *   Value tables by data type: e.g., `catalog_product_entity_varchar`, `_int`, `_decimal`, `_datetime`, `_text`. Joined on `entity_id` and `attribute_id`.
        *   `eav_entity_type`: Use `entity_type_code` (e.g., 'catalog_product', 'customer') to find `entity_type_id` for joining with `eav_attribute`.
    *   **Product Name Attribute:** The product name is typically stored in `catalog_product_entity_varchar` where `eav_attribute.attribute_code` = 'name'. You will need to find the `attribute_id` for 'name' (usually from `eav_attribute` table, for entity_type_id corresponding to 'catalog_product') and join `catalog_product_entity` with `catalog_product_entity_varchar` on this `attribute_id` and `entity_id` (product ID). Remember store scope (`store_id`) for product names; `store_id = 0` (admin) is a common default.
    *   **Store Scopes:** Data can be global (store_id=0 or admin), website-specific, or store-view-specific. Queries for attributes often need to specify `store_id`. `store_id = 0` is the admin/default scope for many attributes. If not specified, assume default/admin scope.
    *   **Product Types:** `catalog_product_entity.type_id` can be 'simple', 'configurable', 'virtual', 'bundle', 'downloadable', 'grouped'. This affects related tables.
    *   **Inventory (MSI - Multi-Source Inventory if applicable, else `cataloginventory_stock_item`):**
        *   MSI: `inventory_source_item` manages stock per source (e.g., 'default'). `status = 1` (In Stock), `status = 0` (Out of Stock).
        *   Legacy: `cataloginventory_stock_item` for stock details, `is_in_stock` field.
    *   **Order Workflow:** `quote` (cart) -> `sales_order` -> `sales_invoice`, `sales_shipment`, `sales_creditmemo`. `sales_order_item` for items in an order.
    *   **Flat/Grid Tables:** Tables like `sales_order_grid`, `customer_grid_flat` are denormalized for admin panel performance. For precise, current data, querying base tables is often preferred.
    *   **Date/Time:** Timestamps (e.g., `created_at`, `updated_at`) are common. Use standard MySQL date/time functions.
    *   **Foreign Keys:** Pay close attention to foreign key relationships for accurate JOINs (e.g., `sales_order_item.order_id` -> `sales_order.entity_id`).

**2. Your Task: Generate ONE Query-Based QA Item**

Follow these phases meticulously:

**Phase A: Question Formulation & Iterative Data Collection (USING YOUR DATABASE TOOL)**
1.  **Formulate an `initial_question` (string):**
    *   This should be a clear, natural language question seeking specific information from the database.
    *   Aim for questions that are interesting, non-trivial (may require joins or simple aggregations), and definitively answerable using SELECT queries with the provided schema.
    *   **Clarity on "Product" Attributes:**
        *   If your question refers to "products" generally (e.g., "What are the products...?", "List products..."), your primary goal should be to retrieve and provide the **product name** as part of the answer.
        *   Alongside the name, you might also include other key identifiers like SKU or ID if they are relevant or easily retrievable.
        *   If retrieving the product name significantly complicates an already complex query or if the focus is purely on other attributes (e.g., "How many products...?"), you should consider refining your `final_question` to be explicit about what product attributes will be returned (e.g., "What are the SKUs of the products...?", "List the product IDs and their prices...").
    *   **Ranking Questions & Tie Handling Constraint (Including Top Rank Boundary Exploration):**
        *   If your question involves ranking (e.g., "most X", "top N", "Nth ranked Y"), you **MUST** design your SQL and question to handle ties using a "dense rank" or "skip rank" (natural ranking) approach. For example, if there are two items tied for 1st place, both should be listed as 1st, and the next rank would be 3rd (not 2nd). Use window functions like `DENSE_RANK()` or `RANK()` in your SQL for this.
        *   **Boundary Exploration for Top Ranks (e.g., "the most X", "the highest Y"):**
            1.  When querying for the *absolute top* ranked items (e.g., "most expensive," "highest quantity"), first execute your query (e.g., `ORDER BY value DESC`) with a moderate `LIMIT` (e.g., `LIMIT 10` or `LIMIT 20`).
            2.  **Analyze the results:** If all items returned by this initial limited query share the *exact same value* for the ranking criterion (e.g., all 10 products have the same price of $99.00), you **MUST assume** the `LIMIT` might be cutting off other items that also share this top rank.
            3.  **Probe Further:** In this scenario, you **MUST** re-execute the query with a significantly larger `LIMIT` (e.g., `LIMIT 50` or `LIMIT 100`). The goal is to retrieve enough rows to see where the top value ends and the next distinct value begins. *Alternatively, and preferably for a robust solution, reformulate your query to use a window function like `DENSE_RANK()` and then select all items where the rank is 1, without relying on `LIMIT` to find all ties at the top.* For example: `SELECT ... FROM (SELECT ..., DENSE_RANK() OVER (ORDER BY ranking_column DESC) as dr FROM ...) ranked_items WHERE dr = 1;`
            4.  Only after this probing (or using a robust window function approach) can you confidently identify *all* items that genuinely share the absolute top rank. Your `final_question` and `llm_derived_answer` must reflect all such tied items.
        *   If your initial idea is to ask for a specific rank (e.g., "the second most expensive"), and your database query (or a preliminary check including the boundary exploration above) reveals that this rank might be skipped due to ties at a higher rank, you **MUST adjust your `final_question`** to ask for a rank that will definitely exist (e.g., "the most expensive products" or "the products ranked third most expensive").
    *   To increase diversity, look at `TABLE_SAMPLES_CONTENT` for *ideas* on data types, but **do not use its specific values in your question if they are part of the intended answer from the live DB.**
2.  **Iterative SQL Execution and Refinement (using your tool call for the mysql mcp server):**
    *   **Plan & Execute SQL with Tool:** Based on the `initial_question` (and its implications for product attributes, and ranking/boundary exploration constraints), formulate a precise SELECT SQL query. **If product names are expected, ensure your query includes the necessary JOINs to `catalog_product_entity_varchar` for the 'name' attribute.**
        **State: "I will now execute the following SQL query using my database tool: [Your SQL Query]". Then, (as an agent) you would make the tool call.** For top rank exploration, you might narrate: "Initial query for top items: `... ORDER BY ... DESC LIMIT 10`". Then after results: "All 10 items have the same value. I will now use a DENSE_RANK query to find all items with rank 1."
    *   **Process Tool Results:** After each tool call, **state: "My database tool returned the following results: [Describe or list the key data returned by the tool, e.g., 'The DENSE_RANK query returned 16 items with rank 1, all having price $99.00 and including their names like 'Product A', 'Product B', etc., followed by items at $92.00 with rank 2. So there are 16 items at the top price.']".**
    *   **Analyze ACTUAL Tool Results & Ensure Question-Answer Congruence:**
        *   Carefully examine the **actual data returned by your tool**.
        *   Does the `initial_question` (or current refined question) imply product names, but your query didn't fetch them or couldn't fetch them easily?
        *   **If there's a mismatch:**
            *   **Option 1 (Preferred if feasible):** Modify your SQL query to include the product names and re-execute.
            *   **Option 2 (If fetching names is too complex for *this specific task* or not the core intent):** Refine the `final_question` to be more specific about the product attributes you *did* retrieve (e.g., change "What are the products..." to "What are the SKUs and prices of the products...").
        *   Is the question fully answerable? Does it align with the ranking constraints, including accurately identifying all items at the top rank if boundary exploration was needed?
        *   If necessary, **refine your question** into a `final_question` that is perfectly aligned with the **actual data your tool can retrieve and the ranking rules**.
        *   If current **actual tool results** are insufficient, **formulate and execute additional SELECT SQL queries using your tool**, again stating your intent and the (conceptual) results.
        *   This is an iterative loop. Continue **using your tool to execute SQL queries** and refining your `final_question` until you are confident that:
            a.  The `final_question` is clear, well-posed, reasonable, and respects ranking/tie rules and product attribute expectations.
            b.  You have successfully retrieved all necessary data **from the live database via your tool** to form a definitive answer.
    *   **Record Keeping (Internal):** Keep track of the sequence of SQL queries you **executed with your tool** (including probing queries or window function based queries) and the key information extracted from their **actual results**.

**Phase B: Answer Derivation (from ACTUAL Tool Results)**
1.  Based on the **verified and complete data collected from the live database via your tool** (including results from any boundary exploration for top ranks):
    *   Formulate an **`llm_derived_answer`** (string): This answer must be concise, factual, and **directly align with what the `final_question` asks for regarding product attributes.** If product names were requested or implied and successfully retrieved, they MUST be in the answer. If the question was refined to ask for SKUs/IDs, then those should be primary in the answer. If listing multiple items due to ties (especially at the top rank after exploration), present them clearly and accurately reflect the total count of such tied items if appropriate for the question.

**Phase C: Validation Rule Design (from ACTUAL Tool Results)**
1.  Based on your `final_question` and the specific `llm_derived_answer` (which is grounded in **actual data retrieved via your tool**):
    *   Design one or more **`validation_rules`**. These rules specify how key pieces of information within the `llm_derived_answer` should be programmatically checked for correctness.
    *   The `validation_rules` field in the output will be a **list of rule objects**.
    *   Each rule object in the list must have two keys:
        *   `type` (string): **Primarily use `"must_include"` or `"fuzzy_match"`.** Use `"exact_match"` very sparingly, only if an entire short, specific, and non-rephraseable string (like a status code or a very fixed short identifier) is the sole answer or a key part of it.
        *   `expected_value` (string): The specific string, number (as a string), or piece of information *extracted* from your `llm_derived_answer` that the candidate answer will be compared against. This value MUST be derived directly from the **actual data retrieved from the live database via your tool**.
    *   **Focus on validating key entities or values within the answer, not the entire rephrased sentence.**
        *   Example: If `final_question` was "What are the names and prices of the most expensive products?" and `llm_derived_answer` is "The most expensive products are 'Awesome T-Shirt' ($99.00) and 'Super Gadget' ($99.00).", rules might be:
            `[{{ "type": "must_include", "expected_value": "Awesome T-Shirt" }}, {{ "type": "must_include", "expected_value": "Super Gadget" }}, {{ "type": "must_include", "expected_value": "99.00" }}]`
        *   For "fuzzy_match", the `expected_value` should be the precise string/number you're allowing fuzziness around (e.g., "99.00" for price, "Product Name Example" for a name).
    *   Multiple rules imply an AND relationship (i.e., a candidate answer must satisfy all rules to be considered valid).

**Phase D: Reference SQL Formulation**
1.  From the sequence of SQL queries you **executed with your tool** in Phase A, select or compose a **single, concise `reference_sql` (string)**.
    *   If it was a ranking question involving top-rank boundary exploration, this SQL should ideally be the query that definitively identifies *all* items belonging to that top rank. **Using a window function like `DENSE_RANK()` in conjunction with filtering on rank = 1 (e.g., `... FROM (SELECT ..., DENSE_RANK() OVER (ORDER BY ranking_column DESC) as dr FROM ...) ranked_items WHERE dr = 1;`) is strongly preferred as it's robust against an unknown number of ties.** If product names were included, the reference SQL should show the join for the 'name' attribute.

**3. Final Output Format (for this ONE QA item):**

Provide your response strictly as a single JSON object with the following structure. Do not include any explanatory text outside of this JSON structure.

```json
{{
  "final_question": "Your refined and validated question string, respecting ranking/tie rules, boundary exploration, and product attribute clarity.",
  "llm_derived_answer": "The natural language answer derived exclusively from actual database data obtained via your tool. If product names were requested/implied, they are included. If multiple items due to ties, list them clearly and accurately.",
  "validation_rules": [
    {{
      "type": "must_include" /* or "fuzzy_match", rarely "exact_match" for specific short values */,
      "expected_value": "Specific key value/entity extracted from the answer, based on actual database data obtained via your tool"
    }}
    /* , {{ ... more rules if needed, each as an object in this list }} */
  ],
  "reference_sql": "A single, representative SELECT SQL query, ideally using window functions for robust ranking and tie handling, and showing joins for product names if applicable."
}}
```

**Agent Workflow and Self-Correction Reminder:**
Your thought process should explicitly narrate steps like:
1.  "I will formulate an initial question..."
2.  "To answer this, I will now execute the following SQL query using my database tool: `SELECT ...`"
3.  "My database tool returned: `[{{'column_name': 'actual_value_from_db'}}]` (or a summary if large, including results of probing/ranking and fetched product attributes like names)."
4.  "Based on these *actual results from my tool*, I will now..." (derive answer, refine question, design rules, etc.).
**CRITICALLY: Do NOT use the `TABLE_SAMPLES_CONTENT` to determine the `llm_derived_answer` or `expected_value` in validation rules. These MUST come from the (simulated if you are an LLM, but conceptually *actual*) results of your tool calls.** Adhere strictly to all guidelines: tool usage, ranking/tie handling, top rank boundary exploration (preferring window functions), product attribute clarity and congruence, and validation rule types.

Please proceed with generating one QA item according to these **strict** instructions.
"""

prompt6 = f"""
You are an expert Magento 2 database analyst, Python programmer, and an autonomous agent. **You ARE equipped with a tool (e.g., 'mysql_mcp_server_tool_call') to interact directly with a MySQL database. You MUST use this tool for all database queries. Your answers and derived values will be based SOLELY on the data retrieved via this tool from the live database.** Your current task is to generate ONE comprehensive Question-Answer (QA) item related to the Magento 2 database, focusing on "query" type tasks.

This process involves:
1.  **Drawing inspiration from a variety of question types to ensure diversity.**
2.  Formulating an initial question.
3.  Iteratively **using your database tool** to execute SELECT SQL queries against the **live database** to gather information.
4.  Analyzing the **actual SQL results returned by your tool** to refine the question and ensure its answerability.
5.  Deriving a precise answer based on the **verified data obtained from your tool calls**.
6.  Designing specific validation rules (`must_include`, `fuzzy_match`) based on the **actual data**.
7.  Identifying a single, representative SQL query for reference.

**1. Contextual Information:**

*   **Database Schema:**
    --- START OF DATABASE SCHEMA ---
    {MAGENTO_SCHEMA_CONTENT}
    --- END OF DATABASE SCHEMA ---

*   **Sample Data from Tables (Illustrative ONLY - for initial understanding and question diversity ideas. DO NOT use this sample data to derive your final answer or any values within it. Your queries will hit the live DB via your tool, and your answer MUST be based on those live results.):**
    --- START OF SAMPLE DATA ---
    {TABLE_SAMPLES_CONTENT}
    --- END OF SAMPLE DATA ---

*   **Key Magento Schema Characteristics & EAV Model (Essential for query design):**
    *   **EAV (Entity-Attribute-Value):** Many entities (products, categories, customers) use EAV.
        *   Core entity table: e.g., `catalog_product_entity`.
        *   Attribute definition: `eav_attribute` (use `attribute_code` to find `attribute_id`).
        *   Value tables by data type: e.g., `catalog_product_entity_varchar`, `_int`, `_decimal`, `_datetime`, `_text`. Joined on `entity_id` and `attribute_id`.
        *   `eav_entity_type`: Use `entity_type_code` (e.g., 'catalog_product', 'customer') to find `entity_type_id` for joining with `eav_attribute`.
    *   **Store Scopes:** Data can be global (store_id=0 or admin), website-specific, or store-view-specific. Queries for attributes often need to specify `store_id`. `store_id = 0` is the admin/default scope for many attributes. If not specified, assume default/admin scope.
    *   **Product Types:** `catalog_product_entity.type_id` can be 'simple', 'configurable', 'virtual', 'bundle', 'downloadable', 'grouped'. This affects related tables.
    *   **Inventory (MSI - Multi-Source Inventory if applicable, else `cataloginventory_stock_item`):**
        *   MSI: `inventory_source_item` manages stock per source (e.g., 'default'). `status = 1` (In Stock), `status = 0` (Out of Stock).
        *   Legacy: `cataloginventory_stock_item` for stock details, `is_in_stock` field.
    *   **Order Workflow:** `quote` (cart) -> `sales_order` -> `sales_invoice`, `sales_shipment`, `sales_creditmemo`. `sales_order_item` for items in an order.
    *   **Flat/Grid Tables:** Tables like `sales_order_grid`, `customer_grid_flat` are denormalized for admin panel performance. For precise, current data, querying base tables is often preferred.
    *   **Date/Time:** Timestamps (e.g., `created_at`, `updated_at`) are common. Use standard MySQL date/time functions.
    *   **Foreign Keys:** Pay close attention to foreign key relationships for accurate JOINs (e.g., `sales_order_item.order_id` -> `sales_order.entity_id`).

**--- [NEW SECTION] ---**

**2. Question Diversity Inspiration**

**To ensure a wide variety of tasks, you MUST generate a question inspired by one of the following categories. For each new task, randomly select a category and theme. Do not simply copy the examples; use them as a blueprint for creating novel questions based on the provided database schema and the live data you observe.**

*   **A. Ranking & Top/Bottom N:** (Questions about extremes or ordered lists)
    *   *Example 1:* "What are the top 5 best-selling products (by quantity sold)?"
    *   *Example 2:* "Which customer has the highest lifetime total spending?"
    *   *Example 3:* "List the 3 least expensive 'configurable' products."

*   **B. Aggregation & Summarization:** (Questions requiring `COUNT`, `SUM`, `AVG`, `GROUP BY`)
    *   *Example 1:* "What is the total number of orders for each status (e.g., 'processing', 'complete')?"
    *   *Example 2:* "Calculate the average number of items per order."
    *   *Example 3:* "What is the total sales amount generated from guest checkouts versus registered customer checkouts?"

*   **C. Temporal / Date-Based Analysis:** (Questions focusing on a specific time frame)
    *   *Example 1:* "How many new customer accounts were created in the last 7 days?"
    *   *Example 2:* "What was the total revenue generated in the month of May 2023?"
    *   *Example 3:* "List all products that were updated between two specific dates."

*   **D. Conditional Filtering & Property Checks:** (Questions that filter based on specific, non-rankable attributes)
    *   *Example 1:* "Find all 'simple' products that are currently out of stock."
    *   *Example 2:* "List all customers from a specific state (e.g., 'California') who have 'pending' orders."
    *   *Example 3:* "What are the names of all products with a weight greater than 10?"

*   **E. Existence & Non-Existence:** (Questions about what is present or missing)
    *   *Example 1:* "Are there any categories that do not contain any active products?"
    *   *Example 2:* "List all products that do not have a 'description' attribute set."
    *   *Example 3:* "Find all customers who have never placed an order."

*   **F. Complex Joins & EAV Lookups:** (Questions that require navigating complex relationships, especially the EAV model)
    *   *Example 1:* "Find the email addresses of customers who have purchased a product with a specific SKU."
    *   *Example 2:* "What is the value of a specific custom product attribute (e.g., 'color') for a given product name?"
    *   *Example 3:* "List all 'simple' products that are children of a specific 'configurable' product."

**--- [END OF NEW SECTION] ---**

**3. Your Task: Generate ONE Query-Based QA Item**

Follow these phases meticulously:

**Phase A: Question Formulation & Iterative Data Collection (USING YOUR DATABASE TOOL)**
1.  **Formulate an `initial_question` (string):**
    *   **CRITICAL: First, choose a category from the "Question Diversity Inspiration" section to guide your question.** This ensures task variety.
    *   This should be a clear, natural language question seeking specific information from the database.
    *   If your chosen category is "Ranking & Top/Bottom N", you **MUST** follow the ranking, tie-handling, and boundary exploration rules outlined below. For other categories, these rules may not apply.
    *   **Ranking Questions & Tie Handling Constraint (Applicable for Category A):**
        *   If your question involves ranking (e.g., "most X", "top N"), you **MUST** design your SQL and question to handle ties using a "dense rank" or "skip rank" approach. Use window functions like `DENSE_RANK()` or `RANK()`.
        *   **Boundary Exploration for Top Ranks:** When querying for the *absolute top* ranked items, you must probe to find *all* items tied at the top rank, preferably using a robust window function approach (e.g., `DENSE_RANK() ... WHERE dr = 1`).
        *   If your initial idea is to ask for a specific rank (e.g., "the second most expensive"), and your database query reveals that this rank might be skipped due to ties, you **MUST adjust your `final_question`** to ask for a rank that will definitely exist.
2.  **Iterative SQL Execution and Refinement (using your tool):**
    *   **Plan & Execute SQL with Tool:** Based on the `initial_question`, formulate a precise SELECT SQL query. **State: "I will now execute the following SQL query using my database tool: [Your SQL Query]".**
    *   **Process Tool Results:** After each tool call, **state: "My database tool returned the following results: [Describe or list the key data returned by the tool]".**
    *   **Analyze ACTUAL Tool Results:** Carefully examine the **actual data returned by your tool**.
    *   **Self-Correct & Iterate:** Refine your question into a `final_question` that is perfectly aligned with the **actual data your tool can retrieve**. If current results are insufficient, **formulate and execute additional SQL queries using your tool**.

**Phase B: Answer Derivation (from ACTUAL Tool Results)**
1.  Based on the **verified and complete data collected from the live database via your tool**, formulate an **`llm_derived_answer`** (string). This is the concise, factual answer to your `final_question`, exclusively supported by the **actual data you retrieved**.

**Phase C: Validation Rule Design (from ACTUAL Tool Results)**
1.  Based on your `final_question` and `llm_derived_answer`, design one or more **`validation_rules`** as a list of objects.
    *   Each rule must have `type` (e.g., `"must_include"`) and `expected_value` (the specific value/entity extracted from your answer, based on **actual tool data**).
    *   Focus on validating key entities or values.

**Phase D: Reference SQL Formulation**
1.  Select or compose a **single, concise `reference_sql` (string)** from the queries you executed. For ranking questions, this should be the robust query that correctly handles ties (e.g., using `DENSE_RANK`).

**4. Final Output Format (for this ONE QA item):**

Provide your response strictly as a single JSON object with the following structure. Do not include any explanatory text outside of this JSON structure.

```json
{{
  "final_question": "Your refined and validated question string, inspired by a chosen diversity category.",
  "llm_derived_answer": "The natural language answer derived exclusively from actual database data obtained via your tool.",
  "validation_rules": [
    {{
      "type": "must_include" /* or "fuzzy_match" */,
      "expected_value": "Specific key value/entity extracted from the answer, based on actual database data obtained via your tool"
    }}
    /* , {{ ... more rules if needed }} */
  ],
  "reference_sql": "A single, representative SELECT SQL query."
}}
```

**5. Agent Workflow and Self-Correction Reminder:**
Your thought process should explicitly narrate steps like:
1.  "I will choose a category for diversity. I choose **Category [Letter]: [Category Name]**."
2.  "Based on this category, I will formulate an initial question..."
3.  "To answer this, I will now execute the following SQL query using my database tool: `SELECT ...`"
4.  "My database tool returned: `[{{'column_name': 'actual_value_from_db'}}]`."
5.  "Based on these *actual results from my tool*, I will now..." (derive answer, refine question, design rules, etc.).
**CRITICALLY: Do NOT use the `TABLE_SAMPLES_CONTENT` to determine the `llm_derived_answer` or `expected_value` in validation rules. These MUST come from the (simulated if you are an LLM, but conceptually *actual*) results of your tool calls.** Adhere strictly to all guidelines.

Please proceed with generating one QA item according to these **strict** instructions.
"""


prompt7 = f"""
You are an expert Magento 2 database analyst, Python programmer, and an autonomous agent. **You ARE equipped with a tool (e.g., 'mysql_mcp_server_tool_call') to interact directly with a MySQL database. You MUST use this tool for all database queries to simulate data gathering. Your answers and derived values will be based SOLELY on the data retrieved via this tool from the live database.** Your current task is to generate ONE comprehensive Question-Answer (QA) item related to the Magento 2 database.

**1. Core Principle: Web Agent Feasibility**

**Crucially, every question you generate MUST be answerable by a web agent interacting with a standard Magento admin panel.** The agent does not have direct database access. Its capabilities are limited to what a human user can do through a web browser:

*   **Searching:** Using search bars on product, order, or customer pages.
*   **Filtering:** Applying filters to grids (e.g., filter orders by status 'Processing', filter customers by State).
*   **Sorting:** Sorting columns in a grid (e.g., sort customers by 'Lifetime Sales' to find the top one).
*   **Navigation & Reading:** Clicking into a specific item's detail/edit page (e.g., a product page) and reading a value from a field.
*   **Counting from Grids:** Reading the total count of items after applying a filter (e.g., "Showing 1-20 of **45** items").

**Avoid questions that require complex, database-only operations.**
*   **BAD (Not Web-Feasible):** `"What is the average number of items per order?"` - There is no single page in the Magento admin that displays this calculated average across all orders.
*   **GOOD (Web-Feasible):** `"How many orders currently have the status 'pending'?"` - A web agent can navigate to the Sales > Orders grid, apply a filter for "Status: Pending", and read the total number of records found.

**2. Contextual Information:**

*   **Database Schema:**
    --- START OF DATABASE SCHEMA ---
    {MAGENTO_SCHEMA_CONTENT}
    --- END OF DATABASE SCHEMA ---

*   **Sample Data from Tables (Illustrative ONLY - for initial understanding and question diversity ideas. DO NOT use this sample data to derive your final answer or any values within it. Your queries will hit the live DB via your tool, and your answer MUST be based on those live results.):**
    --- START OF SAMPLE DATA ---
    {TABLE_SAMPLES_CONTENT}
    --- END OF DATABASE DATA ---

*   **Key Magento Schema Characteristics & EAV Model (Essential for query design):**
    *   **EAV (Entity-Attribute-Value):** Many entities (products, categories, customers) use EAV. Your SQL will need to join these tables correctly.
    *   **Store Scopes:** Data can be global (store_id=0) or store-specific. Assume default/admin scope unless specified.
    *   **Order Workflow:** `quote` -> `sales_order` -> `sales_invoice`, etc.
    *   **Flat/Grid Tables:** Tables like `sales_order_grid`, `customer_grid_flat` are good indicators of what data might be available in an admin grid for a web agent to see.

**3. Question Diversity Inspiration (Web Agent Focused)**

**To ensure a wide variety of tasks, you MUST generate a question inspired by one of the following categories. For each new task, randomly select a category and theme. Use the examples as a blueprint for creating novel, web-feasible questions.**

*   **A. Ranking & Sorting:** (Questions answered by sorting a grid column)
    *   *Web-Feasible Example 1:* "Which customer has the highest Lifetime Sales value?" (Action: Go to Customer Grid, sort by 'Lifetime Sales' column descending).
    *   *Web-Feasible Example 2:* "What are the 5 most recently created products?" (Action: Go to Product Grid, sort by 'ID' or 'Created Date' descending).

*   **B. Aggregation via Filtered Count:** (Questions requiring a `COUNT` on a filtered grid)
    *   *Web-Feasible Example 1:* "How many products are assigned to the 'Men's T-Shirts' category?" (Action: Navigate to that category page, read the product count).
    *   *Web-Feasible Example 2:* "What is the total number of orders with the status 'complete'?" (Action: Go to Order Grid, filter by status, read the total record count).

*   **C. Temporal / Date-Based Filtering:** (Questions filtering a grid by a date range)
    *   *Web-Feasible Example 1:* "How many new customer accounts were created in the last 7 days?" (Action: Go to Customer Grid, apply a date filter to the 'Created' column).
    *   *Web-Feasible Example 2:* "List the order numbers for all orders placed yesterday." (Action: Go to Order Grid, set the 'Order Date' filter).

*   **D. Conditional Filtering & Property Checks:** (Questions filtering by a specific attribute)
    *   *Web-Feasible Example 1:* "Find all 'simple' products that are currently out of stock." (Action: Go to Product Grid, filter by 'Type: Simple' and 'Stock Status: Out of Stock').
    *   *Web-Feasible Example 2:* "What is the shipping address for order number '000000123'?" (Action: Search for the order, open its detail page, read the shipping address block).

*   **E. Existence & Specific Lookups:** (Questions checking for presence or finding a specific value on a page)
    *   *Web-Feasible Example 1:* "Is the product with SKU '24-MB01' currently enabled?" (Action: Search for the SKU, go to the product edit page, check the 'Enable Product' toggle).
    *   *Web-Feasible Example 2:* "Find all customers whose email address contains '@example.com'." (Action: Go to Customer Grid, use the email filter).

*   **F. EAV Attribute Lookups:** (Questions that require viewing a detail page to find a specific attribute value)
    *   *Web-Feasible Example 1:* "What is the 'Color' attribute set for the product named 'Strive Shoulder Pack'?" (Action: Find the product, go to its edit page, read the 'Color' field).
    *   *Web-Feasible Example 2:* "What is the customer's Group for the user with email 'test@user.com'?" (Action: Search for the customer, open their edit page, read the 'Group' field).

**4. Your Task: Generate ONE Query-Based QA Item**

Follow these phases meticulously:

**Phase A: Question Formulation & Iterative Data Collection (USING YOUR DATABASE TOOL)**
1.  **Formulate an `initial_question` (string):**
    *   **CRITICAL:** First, choose a category from the "Question Diversity Inspiration" section.
    *   **CRITICAL:** Formulate a question that is **strictly feasible for a web agent**, as per the core principle.
2.  **Iterative SQL Execution and Refinement (using your tool):**
    *   **Plan & Execute SQL with Tool:** Based on the `initial_question`, formulate a precise SELECT SQL query to simulate finding the answer.
    *   **Tool Response Length Constraint:** To manage context size, your exploratory `SELECT` queries **MUST include a `LIMIT` clause** (e.g., `LIMIT 50`) to prevent excessively large outputs.
    *   **State your plan:** "I will now execute the following SQL query using my database tool: [Your SQL Query with LIMIT]".
    *   **Process Tool Results:** After each tool call, **state: "My database tool returned the following results: [Describe or list the key data returned by the tool]".**
    *   **Analyze & Self-Correct:** Examine the actual tool data. Refine your question into a `final_question` that is perfectly aligned with the data and the web agent feasibility principle. If needed, execute more queries (always with `LIMIT`).

**Phase B: Answer Derivation (from ACTUAL Tool Results)**
1.  Based on the **verified and complete data collected from the live database via your tool**, formulate an **`llm_derived_answer`** (string). This is the concise, factual answer to your `final_question`.

**Phase C: Validation Rule Design (from ACTUAL Tool Results)**
1.  Based on your `final_question` and `llm_derived_answer`, design one or more **`validation_rules`** as a list of objects (`type`: "must_include", `expected_value`: "key value from tool data").

**Phase D: Reference SQL Formulation**
1.  Select or compose a **single, concise `reference_sql` (string)**. This SQL represents the ground truth for the question. It may or may not include the `LIMIT` clause; its main purpose is to be correct and representative of the logic.

**5. Final Output Format (for this ONE QA item):**

Provide your response strictly as a single JSON object. Do not include any explanatory text outside of this JSON structure.

```json
{{
  "final_question": "Your refined, web-agent-feasible question string.",
  "llm_derived_answer": "The natural language answer derived exclusively from actual database data obtained via your tool.",
  "validation_rules": [
    {{
      "type": "must_include",
      "expected_value": "Specific key value/entity extracted from the answer, based on actual database data."
    }}
  ],
  "reference_sql": "A single, representative SELECT SQL query that finds the ground truth for the question."
}}
```

**6. Agent Workflow and Self-Correction Reminder:**
Your thought process should explicitly narrate steps like:
1.  "I will choose a category for diversity. I choose **Category [Letter]: [Category Name]**."
2.  "I will now formulate a web-agent-feasible question based on this category..."
3.  "To answer this, I will execute a query with `LIMIT 50` using my tool: `SELECT ...`"
4.  "My database tool returned: `[...]`."
5.  "Based on these actual results, I will derive the answer and finalize the question..."
**CRITICALLY: Your entire process must be guided by the Web Agent Feasibility principle. The `llm_derived_answer` and `expected_value` must come from the results of your tool calls.**

Please proceed with generating one QA item according to these **strict** instructions.
"""


prompt8 = f"""
You are an expert Magento 2 database analyst, Python programmer, and an autonomous agent. **You ARE equipped with a tool (e.g., 'mysql_mcp_server_tool_call') to interact directly with a MySQL database. You MUST use this tool for all database queries to simulate data gathering. Your answers and derived values will be based SOLELY on the data retrieved via this tool from the live database.** Your current task is to generate ONE comprehensive Question-Answer (QA) item related to the Magento 2 database.

**1. Core Principles: Web Agent Feasibility & Difficulty**

**A. Principle 1: Web Agent Feasibility**

**Crucially, every question you generate MUST be answerable by a web agent interacting with a standard Magento admin panel.** The agent does not have direct database access. Its capabilities are limited to what a human user can do through a web browser:

*   **Searching:** Using search bars on product, order, or customer pages.
*   **Filtering:** Applying filters to grids (e.g., filter orders by status 'Processing').
*   **Sorting:** Sorting columns in a grid (e.g., sort customers by 'Lifetime Sales' to find the top one).
*   **Navigation & Reading:** Clicking into a specific item's detail/edit page and reading a value.
*   **Counting from Grids:** Reading the total count of items after applying a filter (e.g., "Showing 1-20 of **45** items").

**Avoid questions that require complex, database-only operations.**
*   **BAD (Not Web-Feasible):** `"What is the average number of items per order?"` - No single page in the admin panel calculates and displays this value.
*   **GOOD (Web-Feasible):** `"How many orders currently have the status 'pending'?"` - An agent can navigate to the Sales > Orders grid, apply a filter, and read the total count.

**B. Principle 2: Task Difficulty Levels**

You must generate a task that aligns with the specified `{DIFFICULTY}` level. The difficulty is determined by the complexity of the workflow the web agent must execute.

*   **### Easy**
    *   **Definition:** Tasks that can be completed in a **single step or on a single page** with a simple action.
    *   **Typical Actions:** Applying a single filter to a grid, performing a direct search for a known item, or reading a clearly visible value on a main page.
    *   **Web-Feasible Example:** "How many orders currently have the status 'pending'?"
    *   **Agent Workflow:** Navigate to the Sales > Orders grid, apply one filter ("Status: Pending"), and read the total record count displayed on the page.

*   **### Medium**
    *   **Definition:** Tasks that require a **sequence of 2-4 distinct, linear steps**. This often involves navigating from a list/grid view to a detail/edit view.
    *   **Typical Actions:** Searching for an item then opening it, applying multiple filters, or finding an item in one grid and using that info to look something up on its detail page.
    *   **Web-Feasible Example:** "What is the shipping address for the order with the increment ID '000000123'?"
    *   **Agent Workflow:** 1. Navigate to the Sales > Orders grid. 2. Search for '000000123'. 3. Click on the order to open its detail page. 4. Locate and read the shipping address block.

*   **### Hard**
    *   **Definition:** Tasks that require **complex logic, data comparison/synthesis across different pages, or looping through items**. The agent cannot rely on a simple, linear sequence of clicks.
    *   **Typical Actions:** Finding information on one page to use as a filter on another, comparing values across multiple items manually, or tasks where the UI doesn't natively support the required sorting/filtering combination.
    *   **Web-Feasible Example:** "What is the name of the most expensive product within the 'Tops' category?"
    *   **Agent Workflow (is complex):** 1. Navigate to the Products grid. 2. Filter by "Category: Tops". 3. The grid likely cannot be sorted by final price directly in this view (especially for configurable products). The agent would need to iterate through the filtered product list, potentially clicking into *each product's page* to find its price, store it, and compare it against the others to find the maximum. This looping and comparison makes it hard.

**2. Contextual & Inspirational Information**

*   **Database Schema:**
    --- START OF DATABASE SCHEMA ---
    {MAGENTO_SCHEMA_CONTENT}
    --- END OF DATABASE SCHEMA ---

*   **Sample Data from Tables (Illustrative ONLY - for initial understanding. DO NOT use this sample data to derive your final answer or any values within it.):**
    --- START OF SAMPLE DATA ---
    {TABLE_SAMPLES_CONTENT}
    --- END OF SAMPLE DATA ---

*   **Key Magento Schema Characteristics & EAV Model (Essential for query design):**
    *   **EAV (Entity-Attribute-Value):** Many entities (products, categories, customers) use EAV. Your SQL will need to join `eav_attribute` (to find `attribute_id` from `attribute_code`) with the correct value table (e.g., `catalog_product_entity_varchar`, `_int`).
    *   **Store Scopes:** Data can be global (store_id=0 or admin), website-specific, or store-view-specific. Queries for attributes often need to specify `store_id = 0` for admin/default values.
    *   **Order Workflow:** `quote` (cart) -> `sales_order` -> `sales_invoice`, `sales_shipment`, `sales_creditmemo`.
    *   **Flat/Grid Tables:** Tables like `sales_order_grid` and `customer_grid_flat` are excellent indicators of what data is likely available in an admin grid for a web agent to see, filter, and sort.

*   **Question Diversity Inspiration (Themes for Web-Feasible Tasks)**
    *   **A. Ranking & Sorting:** (Questions answered by sorting a grid column)
        *   *Example 1:* "Which customer has the highest Lifetime Sales value?" (Action: Go to Customer Grid, sort by 'Lifetime Sales' column descending).
        *   *Example 2:* "What are the 5 most recently created products?" (Action: Go to Product Grid, sort by 'ID' or 'Created Date' descending).
    *   **B. Aggregation via Filtered Count:** (Questions requiring a `COUNT` on a filtered grid)
        *   *Example 1:* "How many products are assigned to the 'Men's T-Shirts' category?" (Action: Navigate to that category page, read the product count).
        *   *Example 2:* "What is the total number of orders with the status 'complete'?" (Action: Go to Order Grid, filter by status, read the total record count).
    *   **C. Temporal / Date-Based Filtering:** (Questions filtering a grid by a date range)
        *   *Example 1:* "How many new customer accounts were created in the last 7 days?" (Action: Go to Customer Grid, apply a date filter to the 'Created' column).
        *   *Example 2:* "List the order numbers for all orders placed yesterday." (Action: Go to Order Grid, set the 'Order Date' filter).
    *   **D. Conditional Filtering & Property Checks:** (Questions filtering by a specific attribute)
        *   *Example 1:* "Find all 'simple' products that are currently out of stock." (Action: Go to Product Grid, filter by 'Type: Simple' and 'Stock Status: Out of Stock').
        *   *Example 2:* "What is the shipping address for order number '000000123'?" (Action: Search for the order, open its detail page, read the shipping address block).
    *   **E. Existence & Specific Lookups:** (Questions checking for presence or finding a specific value on a page)
        *   *Example 1:* "Is the product with SKU '24-MB01' currently enabled?" (Action: Search for the SKU, go to the product edit page, check the 'Enable Product' toggle).
        *   *Example 2:* "Find all customers whose email address contains '@example.com'." (Action: Go to Customer Grid, use the email filter).
    *   **F. EAV Attribute Lookups:** (Questions that require viewing a detail page to find a specific attribute value)
        *   *Example 1:* "What is the 'Color' attribute set for the product named 'Strive Shoulder Pack'?" (Action: Find the product, go to its edit page, read the 'Color' field).
        *   *Example 2:* "What is the customer's Group for the user with email 'test@user.com'?" (Action: Search for the customer, open their edit page, read the 'Group' field).

**3. Your Task: Generate ONE QA Item of `{DIFFICULTY}` difficulty**

Follow these phases meticulously:

**Phase A: Question Formulation & Iterative Data Collection (USING YOUR DATABASE TOOL)**
1.  **Formulate an `initial_question` (string):**
    *   **CRITICAL:** Your question's complexity MUST match the specified `{DIFFICULTY}` level. Use the definitions and examples above for guidance.
    *   The question must be **strictly feasible for a web agent**.
    *   Choose a theme from the "Question Diversity Inspiration" section.
2.  **Iterative SQL Execution and Refinement (using your tool):**
    *   **Plan & Execute SQL with Tool:** Formulate a precise SELECT SQL query to simulate finding the answer.
    *   **Tool Response Length Constraint:** To manage context size, your exploratory `SELECT` queries **MUST include a `LIMIT` clause** (e.g., `LIMIT 50`) to prevent excessively large outputs.
    *   **State your plan:** "I will now execute the following SQL query with `LIMIT 50` using my database tool: [Your SQL Query]".
    *   **Process Tool Results:** After each tool call, state: "My database tool returned the following results: [Describe or list the key data returned by the tool]".
    *   **Analyze & Refine:** Examine the actual tool data. Refine your question into a `final_question` that is perfectly aligned with the data, web agent feasibility, and the required difficulty.

**Phase B: Answer Derivation (from ACTUAL Tool Results)**
1.  Based on the **verified and complete data collected from the live database via your tool**, formulate an **`llm_derived_answer`** (string). This is the concise, factual answer to your `final_question`.

**Phase C: Validation Rule Design & Difficulty Rationale**
1.  **Design `validation_rules`:** Based on your `final_question` and `llm_derived_answer` (which is grounded in **actual data retrieved via your tool**):
    *   The `validation_rules` field will be a **list of rule objects**.
    *   Each rule object must have two keys: `type` (string) and `expected_value` (string).
    *   **`type`:** Primarily use `"must_include"` or `"fuzzy_match"`. Use `"exact_match"` very sparingly.
    *   **`expected_value`:** This is the specific string or number *extracted* from your `llm_derived_answer` that must be validated. It MUST be derived directly from the **actual data retrieved from the live database via your tool**.
    *   **Focus on validating key entities or values**, not the entire sentence structure.
        *   Example: For answer "The product 'Strive Shoulder Pack' is in stock.", a good rule is `{{ "type": "must_include", "expected_value": "Strive Shoulder Pack" }}`.
    *   Multiple rules imply an AND relationship (all must be satisfied).
2.  **Formulate a `difficulty_reason` (string):** This MUST be a concise explanation of why the task matches the specified difficulty, referencing the agent's required actions (e.g., "This is Medium because it requires the agent to first search for a specific order, then navigate to its detail page to read the shipping information.").

**Phase D: Reference SQL Formulation**
1.  Select or compose a **single, concise `reference_sql` (string)** that represents the ground truth for the question. This SQL is for verification and does not need a `LIMIT`.

**4. Final Output Format (for this ONE QA item):**

Provide your response strictly as a single JSON object. Do not include any explanatory text outside of this JSON structure.

```json
{{
  "final_question": "Your refined, web-agent-feasible question string, matching the required difficulty.",
  "llm_derived_answer": "The natural language answer derived exclusively from actual database data obtained via your tool.",
  "validation_rules": [
    {{
      "type": "must_include",
      "expected_value": "Specific key value/entity extracted from the answer, based on actual database data."
    }}
  ],
  "reference_sql": "A single, representative SELECT SQL query that finds the ground truth for the question.",
  "difficulty_reason": "A concise explanation of why the task's complexity matches the specified difficulty level, based on the web agent's required workflow (e.g., single-step, multi-step linear, or complex looping/comparison)."
}}
```

**5. Agent Workflow and Self-Correction Reminder:**
Your thought process should explicitly narrate steps like:
1.  "I am tasked with generating a `{DIFFICULTY}` question."
2.  "I will formulate a web-agent-feasible question that matches this difficulty, using the theme 'Conditional Filtering'."
3.  "To find the answer, I will execute a query with `LIMIT 50` using my tool: `SELECT ...`"
4.  "My database tool returned: `[...]`."
5.  "Based on these results, I will derive the answer. The reason this is `{DIFFICULTY}` is because the agent must [describe the specific workflow steps, e.g., 'apply two filters to the product grid and then read the count']."
**CRITICALLY: Your entire process must be guided by the Web Agent Feasibility principle. The `llm_derived_answer` and `expected_value` must come from the results of your tool calls.**

Please proceed with generating one QA item according to these **strict and detailed** instructions.
"""

prompt9 = f"""
You are an expert Magento 2 database analyst, Python programmer, and an autonomous agent. **You ARE equipped with a tool (e.g., 'mysql_mcp_server_tool_call') to interact directly with a MySQL database. You MUST use this tool for all database queries to simulate data gathering. Your answers and derived values will be based SOLELY on the data retrieved via this tool from the live database.** Your current task is to generate ONE comprehensive Question-Answer (QA) item related to the Magento 2 database.

**1. Core Principles: Feasibility, Difficulty, and Diversity**

**A. Principle 1: Web Agent Feasibility**

**Crucially, every question you generate MUST be answerable by a web agent interacting with a standard Magento admin panel.** The agent does not have direct database access. Its capabilities are limited to what a human user can do through a web browser:

*   **Searching:** Using search bars on product, order, or customer pages.
*   **Filtering:** Applying filters to grids (e.g., filter orders by status 'Processing').
*   **Sorting:** Sorting columns in a grid (e.g., sort customers by 'Lifetime Sales' to find the top one).
*   **Navigation & Reading:** Clicking into a specific item's detail/edit page and reading a value.
*   **Counting from Grids:** Reading the total count of items after applying a filter (e.g., "Showing 1-20 of **45** items").

**Avoid questions that require complex, database-only operations.**
*   **BAD (Not Web-Feasible):** `"What is the average number of items per order?"` - No single page in the admin panel calculates and displays this value.
*   **GOOD (Web-Feasible):** `"How many orders currently have the status 'pending'?"` - An agent can navigate to the Sales > Orders grid, apply a filter, and read the total count.

**B. Principle 2: Task Difficulty Levels**

You must generate a task that aligns with the specified `{DIFFICULTY}` level. The difficulty is determined by the complexity of the workflow the web agent must execute.

*   **### Easy**
    *   **Definition:** Tasks that can be completed in a **single step or on a single page** with a simple action.
    *   **Typical Actions:** Applying a single filter to a grid, performing a direct search for a known item, or reading a clearly visible value on a main page.
    *   **Web-Feasible Example:** "How many orders currently have the status 'pending'?"
    *   **Agent Workflow:** Navigate to the Sales > Orders grid, apply one filter ("Status: Pending"), and read the total record count displayed on the page.

*   **### Medium**
    *   **Definition:** Tasks that require a **sequence of 2-4 distinct, linear steps**. This often involves navigating from a list/grid view to a detail/edit view.
    *   **Typical Actions:** Searching for an item then opening it, applying multiple filters, or finding an item in one grid and using that info to look something up on its detail page.
    *   **Web-Feasible Example:** "What is the shipping address for the order with the increment ID '000000123'?"
    *   **Agent Workflow:** 1. Navigate to the Sales > Orders grid. 2. Search for '000000123'. 3. Click on the order to open its detail page. 4. Locate and read the shipping address block.

*   **### Hard**
    *   **Definition:** Tasks that require **complex logic, data comparison/synthesis across different pages, or looping through items**. The agent cannot rely on a simple, linear sequence of clicks.
    *   **Typical Actions:** Finding information on one page to use as a filter on another, comparing values across multiple items manually, or tasks where the UI doesn't natively support the required sorting/filtering combination.
    *   **Web-Feasible Example:** "What is the name of the most expensive product within the 'Tops' category?"
    *   **Agent Workflow (is complex):** 1. Navigate to the Products grid. 2. Filter by "Category: Tops". 3. The grid likely cannot be sorted by final price directly in this view. The agent would need to iterate through the filtered product list, potentially clicking into *each product's page* to find its price, store it, and compare it against the others to find the maximum. This looping and comparison makes it hard.

**IMPORTANT: You MUST NOT generate questions that require the agent to answer YES or NO, because the web agent may randomly guess an answer.**

**C. Principle 3: Dynamic Entity Selection for True Diversity**

**To prevent generating repetitive questions using the same few examples, you MUST dynamically source the key entity for your question from the live database.** This is a mandatory first step in your process.

*   **Problem to Avoid:** Using SKUs, order numbers, or customer emails directly from the illustrative `TABLE_SAMPLES_CONTENT`.
*   **Required Action:** Before formulating your question, you **MUST** perform an initial, exploratory query to fetch a list of random, valid identifiers from the database.
*   **Example Exploratory Queries (using `ORDER BY RAND()`):**
    *   To get random product SKUs: `SELECT sku FROM catalog_product_entity ORDER BY RAND() LIMIT 10;`
    *   To get random order increment IDs: `SELECT increment_id FROM sales_order ORDER BY RAND() LIMIT 10;`
    *   To get random customer emails: `SELECT email FROM customer_entity ORDER BY RAND() LIMIT 10;`
*   **Rule:** After fetching this list via your tool, you **MUST** select ONE entity which you NEVER MET before from the returned results to use as the subject of your `final_question`.

**2. Contextual & Inspirational Information**

*   **Database Schema:**
    --- START OF DATABASE SCHEMA ---
    {MAGENTO_SCHEMA_CONTENT}
    --- END OF DATABASE SCHEMA ---

*   **Sample Data from Tables (Illustrative ONLY - for initial understanding. DO NOT use this sample data to derive your final answer or to select entities for your question.):**
    --- START OF SAMPLE DATA ---
    {TABLE_SAMPLES_CONTENT}
    --- END OF SAMPLE DATA ---

*   **Key Magento Schema Characteristics & EAV Model (Essential for query design):**
    *   **EAV (Entity-Attribute-Value):** Many entities (products, categories, customers) use EAV. Your SQL will need to join `eav_attribute` (to find `attribute_id` from `attribute_code`) with the correct value table (e.g., `catalog_product_entity_varchar`, `_int`).
    *   **Store Scopes:** Data can be global (store_id=0 or admin), website-specific, or store-view-specific. Queries for attributes often need to specify `store_id = 0` for admin/default values.
    *   **Order Workflow:** `quote` (cart) -> `sales_order` -> `sales_invoice`, `sales_shipment`, `sales_creditmemo`.
    *   **Flat/Grid Tables:** Tables like `sales_order_grid` and `customer_grid_flat` are excellent indicators of what data is likely available in an admin grid for a web agent to see, filter, and sort.

*   **Question Diversity Inspiration (Themes for Web-Feasible Tasks)**
    *   **A. Ranking & Sorting:** "Which customer has the highest Lifetime Sales value?"
    *   **B. Aggregation via Filtered Count:** "What is the total number of orders with the status 'complete'?"
    *   **C. Temporal / Date-Based Filtering:** "How many new customer accounts were created in the last 7 days?"
    *   **D. Conditional Filtering & Property Checks:** "Find all 'simple' products that are currently out of stock."
    *   **E. Existence & Specific Lookups:** "Is the product with SKU '[Dynamically Selected SKU]' currently enabled?"
    *   **F. EAV Attribute Lookups:** "What is the customer's Group for the user with email '[Dynamically Selected Email]'?"

**3. Your Task: Generate ONE QA Item of `{DIFFICULTY}` difficulty**

Follow these phases meticulously:

**Phase A: Question Formulation & Iterative Data Collection (USING YOUR DATABASE TOOL)**

1.  **Dynamic Entity Selection (MANDATORY FIRST STEP):**
    *   First, based on the type of question you intend to ask (e.g., about a product, an order), perform an exploratory query to fetch a list of random, valid identifiers. You **MUST** use a method like `ORDER BY RAND() LIMIT 10` for this.
    *   **State your plan:** "To ensure diversity, I will first fetch a list of random entities. I will execute using my tool: `[Your exploratory SQL query]`".
    *   **Process Tool Results:** After the tool returns the data, state: "My tool returned the following entities: `[...]`. I will select '[Chosen Entity]' for my question."

2.  **Formulate an `initial_question` (string):**
    *   **CRITICAL:** Now, using the entity you selected in the previous step, formulate a question.
    *   The question's complexity MUST match the specified `{DIFFICULTY}` level. Use the definitions and examples in Section 1B for guidance.
    *   The question must be **strictly feasible for a web agent**.
    *   Choose a theme from the "Question Diversity Inspiration" section.

3.  **Iterative SQL Execution and Refinement (to find the answer):**
    *   **Plan & Execute SQL with Tool:** Formulate the specific SELECT SQL query needed to find the answer to your `initial_question`.
    *   **Tool Response Length Constraint:** To manage context size, your data-gathering `SELECT` queries **MUST include a `LIMIT` clause** (e.g., `LIMIT 50`).
    *   **State your plan:** "To answer my question, I will now execute the following SQL query with `LIMIT 50` using my database tool: `[Your answer-finding SQL query]`".
    *   **Process Tool Results:** After each tool call, state: "My database tool returned the following results: `[Describe or list the key data returned by the tool]`".
    *   **Analyze & Refine:** Examine the actual tool data. Refine your question into a `final_question` that is perfectly aligned with the data, web agent feasibility, and the required difficulty.

**Phase B: Answer Derivation (from ACTUAL Tool Results)**
1.  Based on the **verified and complete data collected from the live database via your tool**, formulate an **`llm_derived_answer`** (string). This is the concise, factual answer to your `final_question`.

**Phase C: Validation Rule Design & Difficulty Rationale**
1.  **Design `validation_rules`:** Based on your `final_question` and `llm_derived_answer` (which is grounded in **actual data retrieved via your tool**):
    *   The `validation_rules` field will be a **list of rule objects**.
    *   Each rule object must have two keys: `type` (string) and `expected_value` (string).
    *   **`type`:** Primarily use `"must_include"` or `"fuzzy_match"`. Use `"exact_match"` very sparingly.
    *   **`expected_value`:** This is the specific string or number *extracted* from your `llm_derived_answer` that must be validated. It MUST be derived directly from the **actual data retrieved from the live database via your tool**.
    *   **Focus on validating key entities or values**, not the entire sentence structure.
        *   Example: For answer "The product 'Strive Shoulder Pack' is in stock.", a good rule is `{{ "type": "must_include", "expected_value": "Strive Shoulder Pack" }}`.
    *   Multiple rules imply an AND relationship (all must be satisfied).
2.  **Formulate a `difficulty_reason` (string):** This MUST be a concise explanation of why the task matches the specified difficulty, referencing the agent's required actions (e.g., "This is Medium because it requires the agent to first search for a specific order, then navigate to its detail page to read the shipping information.").

**Phase D: Reference SQL Formulation**
1.  Select or compose a **single, concise `reference_sql` (string)** that represents the ground truth for the question. This SQL is for verification and does not need a `LIMIT`.

**4. Final Output Format (for this ONE QA item):**

Provide your response strictly as a single JSON object. Do not include any explanatory text outside of this JSON structure.

```json
{{
  "final_question": "Your refined, web-agent-feasible question string, using a dynamically selected entity and matching the required difficulty.",
  "llm_derived_answer": "The natural language answer derived exclusively from actual database data obtained via your tool.",
  "validation_rules": [
    {{
      "type": "must_include",
      "expected_value": "Specific key value/entity extracted from the answer, based on actual database data."
    }}
  ],
  "reference_sql": "A single, representative SELECT SQL query that finds the ground truth for the question.",
  "difficulty_reason": "A concise explanation of why the task's complexity matches the specified difficulty level, based on the web agent's required workflow (e.g., single-step, multi-step linear, or complex looping/comparison)."
}}
```

**5. Agent Workflow and Self-Correction Reminder:**
Your thought process should explicitly narrate steps like:
1.  **"Step 1: Dynamic Entity Selection.** I am tasked with generating a `{DIFFICULTY}` question. To make it diverse, I first need a random, real entity. I'll pick a product SKU. I will execute using my tool: `SELECT sku FROM catalog_product_entity WHERE type_id = 'simple' ORDER BY RAND() LIMIT 10;`"
2.  "My tool returned SKUs: `['WB04', 'MH07', '24-MB01', 'VT10', '...']`. I will select **'MH07'** for my question."
3.  **"Step 2: Question Formulation.** Using the SKU 'MH07', I will formulate a `{DIFFICULTY}` question, for instance: 'What is the current stock quantity for the product with SKU MH07?'"
4.  **"Step 3: Answering the Question.** To find the answer, I will now execute a query with `LIMIT 50` using my tool: `SELECT qty FROM cataloginventory_stock_item csi JOIN catalog_product_entity cpe ON csi.product_id = cpe.entity_id WHERE cpe.sku = 'MH07' LIMIT 50;`"
5.  **"Step 4: Deriving the Final Output.** My database tool returned `qty: 88`. Based on this, I will derive the answer and other fields. The reason this is `Easy` is because it only requires the agent to search for a SKU in the product grid and read the 'Quantity' value from the corresponding row, a single-page action."

Please proceed with generating one QA item according to these **strict, complete, and highly detailed** instructions.
"""

prompt10 = f"""
You are an expert Magento 2 database analyst, Python programmer, and an autonomous agent. **You ARE equipped with a tool (e.g., 'mysql_mcp_server_tool_call') to interact directly with a MySQL database. You MUST use this tool for all database queries to simulate data gathering. Your answers and derived values will be based SOLELY on the data retrieved via this tool from the live database.** Your current task is to generate ONE comprehensive Question-Answer (QA) item related to the Magento 2 database.

**1. Core Principles: Feasibility, Difficulty, and Diversity**

**A. Principle 1: Web Agent Feasibility**

**Crucially, every question you generate MUST be answerable by a web agent interacting with a standard Magento admin panel.** The agent does not have direct database access. Its capabilities are limited to what a human user can do through a web browser:

*   **Searching:** Using search bars on product, order, or customer pages.
*   **Filtering:** Applying filters to grids (e.g., filter orders by status 'Processing').
*   **Sorting:** Sorting columns in a grid (e.g., sort customers by 'Lifetime Sales' to find the top one).
*   **Navigation & Reading:** Clicking into a specific item's detail/edit page and reading a value.
*   **Counting from Grids:** Reading the total count of items after applying a filter (e.g., "Showing 1-20 of **45** items").

**Avoid questions that require complex, database-only operations.**
*   **BAD (Not Web-Feasible):** `"What is the average number of items per order?"` - No single page in the admin panel calculates and displays this value.
*   **GOOD (Web-Feasible):** `"How many orders currently have the status 'pending'?"` - An agent can navigate to the Sales > Orders grid, apply a filter, and read the total count.

**B. Principle 2: Task Difficulty Levels**

You must generate a task that aligns with the specified `{DIFFICULTY}` level. The difficulty is determined by the complexity of the workflow the web agent must execute.

*   **### Easy**
    *   **Definition:** Tasks that can be completed in a **single step or on a single page** with a simple action.
    *   **Typical Actions:** Applying a single filter to a grid, performing a direct search for a known item, or reading a clearly visible value on a main page.
    *   **Web-Feasible Example:** "How many orders currently have the status 'pending'?"
    *   **Agent Workflow:** Navigate to the Sales > Orders grid, apply one filter ("Status: Pending"), and read the total record count displayed on the page.

*   **### Medium**
    *   **Definition:** Tasks that require a **sequence of 2-4 distinct, linear steps**. This often involves navigating from a list/grid view to a detail/edit view.
    *   **Typical Actions:** Searching for an item then opening it, applying multiple filters, or finding an item in one grid and using that info to look something up on its detail page.
    *   **Web-Feasible Example:** "What is the shipping address for the order with the increment ID '000000123'?"
    *   **Agent Workflow:** 1. Navigate to the Sales > Orders grid. 2. Search for '000000123'. 3. Click on the order to open its detail page. 4. Locate and read the shipping address block.

*   **### Hard**
    *   **Definition:** Tasks that require **complex logic, data comparison/synthesis across different pages, or looping through items**. The agent cannot rely on a simple, linear sequence of clicks.
    *   **Typical Actions:** Finding information on one page to use as a filter on another, comparing values across multiple items manually, or tasks where the UI doesn't natively support the required sorting/filtering combination.
    *   **Web-Feasible Example:** "What is the name of the most expensive product within the 'Tops' category?"
    *   **Agent Workflow (is complex):** 1. Navigate to the Products grid. 2. Filter by "Category: Tops". 3. The grid likely cannot be sorted by final price directly in this view. The agent would need to iterate through the filtered product list, potentially clicking into *each product's page* to find its price, store it, and compare it against the others to find the maximum. This looping and comparison makes it hard.

**IMPORTANT: You MUST NOT generate questions that require the agent to answer YES or NO, because the web agent may randomly guess an answer.**

**C. Principle 3: Dynamic Entity Selection for True Diversity**

**To prevent generating repetitive questions using the same few examples, you MUST dynamically source the key entity for your question from the live database.** This is a mandatory first step in your process.

*   **Problem to Avoid:** Using SKUs, order numbers, or customer emails directly from the illustrative `TABLE_SAMPLES_CONTENT`.
*   **Required Action:** Before formulating your question, you **MUST** perform an initial, exploratory query to fetch a list of random, valid identifiers from the database.
*   **Example Exploratory Queries (using `ORDER BY RAND()`):**
    *   To get random product SKUs: `SELECT sku FROM catalog_product_entity ORDER BY RAND() LIMIT 10;`
    *   To get random order increment IDs: `SELECT increment_id FROM sales_order ORDER BY RAND() LIMIT 10;`
    *   To get random customer emails: `SELECT email FROM customer_entity ORDER BY RAND() LIMIT 10;`
*   **Rule:** After fetching this list via your tool, you **MUST** select ONE entity which you NEVER MET before from the returned results to use as the subject of your `final_question`.

**2. Contextual & Inspirational Information**

*   **Database Schema:**
    --- START OF DATABASE SCHEMA ---
    {MAGENTO_SCHEMA_CONTENT}
    --- END OF DATABASE SCHEMA ---

*   **Sample Data from Tables (Illustrative ONLY - for initial understanding. DO NOT use this sample data to derive your final answer or to select entities for your question.):**
    --- START OF SAMPLE DATA ---
    {TABLE_SAMPLES_CONTENT}
    --- END OF SAMPLE DATA ---

*   **Key Magento Schema Characteristics & EAV Model (Essential for query design):**
    *   **EAV (Entity-Attribute-Value):** Many entities (products, categories, customers) use EAV. Your SQL will need to join `eav_attribute` (to find `attribute_id` from `attribute_code`) with the correct value table (e.g., `catalog_product_entity_varchar`, `_int`).
    *   **Store Scopes:** Data can be global (store_id=0 or admin), website-specific, or store-view-specific. Queries for attributes often need to specify `store_id = 0` for admin/default values.
    *   **Order Workflow:** `quote` (cart) -> `sales_order` -> `sales_invoice`, `sales_shipment`, `sales_creditmemo`.
    *   **Flat/Grid Tables:** Tables like `sales_order_grid` and `customer_grid_flat` are excellent indicators of what data is likely available in an admin grid for a web agent to see, filter, and sort.

*   **Question Diversity Inspiration (Themes for Web-Feasible Tasks)**
    *   **A. Ranking & Sorting:** "Which customer has the highest Lifetime Sales value?"
    *   **B. Aggregation via Filtered Count:** "What is the total number of orders with the status 'complete'?"
    *   **C. Temporal / Date-Based Filtering:** "How many new customer accounts were created in the last 7 days?"
    *   **D. Conditional Filtering & Property Checks:** "Find all 'simple' products that are currently out of stock."
    *   **E. Existence & Specific Lookups:** "Is the product with SKU '[Dynamically Selected SKU]' currently enabled?"
    *   **F. EAV Attribute Lookups:** "What is the customer's Group for the user with email '[Dynamically Selected Email]'?"

**3. Your Task: Generate ONE QA Item of `{DIFFICULTY}` difficulty**

Follow these phases meticulously:

**Phase A: Question Formulation & Iterative Data Collection (USING YOUR DATABASE TOOL)**

1.  **Dynamic Entity Selection (MANDATORY FIRST STEP):**
    *   First, based on the type of question you intend to ask (e.g., about a product, an order), perform an exploratory query to fetch a list of random, valid identifiers. You **MUST** use a method like `ORDER BY RAND() LIMIT 10` for this.
    *   **State your plan:** "To ensure diversity, I will first fetch a list of random entities. I will execute using my tool: `[Your exploratory SQL query]`".
    *   **Process Tool Results:** After the tool returns the data, state: "My tool returned the following entities: `[...]`. I will select '[Chosen Entity]' for my question."

2.  **Formulate an `initial_question` (string):**
    *   **CRITICAL:** Now, using the entity you selected in the previous step, formulate a question.
    *   The question's complexity MUST match the specified `{DIFFICULTY}` level. Use the definitions and examples in Section 1B for guidance.
    *   The question must be **strictly feasible for a web agent**.
    *   Choose a theme from the "Question Diversity Inspiration" section.
    *   **Special Instructions for Ranking Questions:** If you choose the "Ranking & Sorting" theme, you **MUST** follow these additional rules during your query and refinement process:
        *   **Tie Handling:** If your question involves ranking (e.g., "most X", "top N"), you **MUST** design your SQL and question to handle ties correctly. Your answer must include *all* items that share the same rank. Use window functions like `DENSE_RANK()` or `RANK()` in your verification queries to ensure accuracy.
        *   **Boundary Exploration for Top Ranks:** When querying for the *absolute top* ranked items (e.g., "the most expensive," "highest quantity"), you must probe to find *all* items tied at that top rank. A simple `LIMIT 1` is not sufficient. The preferred method is to use a robust window function query like `SELECT ... FROM (SELECT ..., DENSE_RANK() OVER (ORDER BY ranking_column DESC) as dr FROM ...) ranked_items WHERE dr = 1;`. This guarantees you find all ties.
        *   **Rank Existence:** If your initial idea is to ask for a specific rank (e.g., "the second most expensive"), and your database query reveals that this rank might be skipped due to ties at a higher rank (e.g., three items tied for 1st place means the next rank is 4th), you **MUST adjust your `final_question`** to ask for a rank that will definitely exist (e.g., "the most expensive products" or "the products ranked fourth most expensive").

3.  **Iterative SQL Execution and Refinement (to find the answer):**
    *   **Plan & Execute SQL with Tool:** Formulate the specific SELECT SQL query needed to find the answer to your `initial_question`, applying any special ranking rules if necessary.
    *   **Tool Response Length Constraint:** To manage context size, your data-gathering `SELECT` queries **MUST include a `LIMIT` clause** (e.g., `LIMIT 50`).
    *   **State your plan:** "To answer my question, I will now execute the following SQL query with `LIMIT 50` using my database tool: `[Your answer-finding SQL query]`".
    *   **Process Tool Results:** After each tool call, state: "My database tool returned the following results: `[Describe or list the key data returned by the tool]`".
    *   **Analyze & Refine:** Examine the actual tool data. Refine your question into a `final_question` that is perfectly aligned with the data, web agent feasibility, the required difficulty, and the ranking/tie rules.

**Phase B: Answer Derivation (from ACTUAL Tool Results)**
1.  Based on the **verified and complete data collected from the live database via your tool**, formulate an **`llm_derived_answer`** (string). This is the concise, factual answer to your `final_question`.

**Phase C: Validation Rule Design & Difficulty Rationale**
1.  **Design `validation_rules`:** Based on your `final_question` and `llm_derived_answer` (which is grounded in **actual data retrieved via your tool**):
    *   The `validation_rules` field will be a **list of rule objects**.
    *   Each rule object must have two keys: `type` (string) and `expected_value` (string).
    *   **`type`:** Primarily use `"must_include"` or `"fuzzy_match"`. Use `"exact_match"` very sparingly.
    *   **`expected_value`:** This is the specific string or number *extracted* from your `llm_derived_answer` that must be validated. It MUST be derived directly from the **actual data retrieved from the live database via your tool**.
    *   **Focus on validating key entities or values**, not the entire sentence structure.
        *   Example: For answer "The product 'Strive Shoulder Pack' is in stock.", a good rule is `{{ "type": "must_include", "expected_value": "Strive Shoulder Pack" }}`.
    *   Multiple rules imply an AND relationship (all must be satisfied).
2.  **Formulate a `difficulty_reason` (string):** This MUST be a concise explanation of why the task matches the specified difficulty, referencing the agent's required actions (e.g., "This is Medium because it requires the agent to first search for a specific order, then navigate to its detail page to read the shipping information.").

**Phase D: Reference SQL Formulation**
1.  Select or compose a **single, concise `reference_sql` (string)** that represents the ground truth for the question. This SQL is for verification and does not need a `LIMIT`. For ranking questions, this should be the robust query that correctly handles all ties (e.g., using a window function like `DENSE_RANK`).

**4. Final Output Format (for this ONE QA item):**

Provide your response strictly as a single JSON object. Do not include any explanatory text outside of this JSON structure.

```json
{{
  "final_question": "Your refined, web-agent-feasible question string, using a dynamically selected entity and matching the required difficulty.",
  "llm_derived_answer": "The natural language answer derived exclusively from actual database data obtained via your tool.",
  "validation_rules": [
    {{
      "type": "must_include",
      "expected_value": "Specific key value/entity extracted from the answer, based on actual database data."
    }}
  ],
  "reference_sql": "A single, representative SELECT SQL query that finds the ground truth for the question.",
  "difficulty_reason": "A concise explanation of why the task's complexity matches the specified difficulty level, based on the web agent's required workflow (e.g., single-step, multi-step linear, or complex looping/comparison)."
}}
```

**5. Agent Workflow and Self-Correction Reminder:**
Your thought process should explicitly narrate steps like:
1.  **"Step 1: Dynamic Entity Selection.** I am tasked with generating a `{DIFFICULTY}` question... I will execute using my tool: `SELECT sku FROM ... ORDER BY RAND() LIMIT 10;`"
2.  "My tool returned SKUs: `[...]`. I will select **'[Chosen SKU]'** for my question."
3.  **"Step 2: Question Formulation.** Using the chosen SKU, I will formulate a `{DIFFICULTY}` question..."
4.  **"Step 3: Answering the Question.** To find the answer, I will execute a query... (If it's a ranking question, my narration would be: 'To find the top-ranked item, I must check for ties. I will execute `... DENSE_RANK() ... WHERE dr = 1;` to ensure I get all tied results.')"
5.  **"Step 4: Deriving the Final Output.** My database tool returned `[...]`. Based on this, I will derive the answer... The reason this is `{DIFFICULTY}` is because..."

Please proceed with generating one QA item according to these **strict, complete, and highly detailed** instructions.
"""

prompt11 = f"""
You are an expert Magento 2 database analyst, Python programmer, and an autonomous agent. **You ARE equipped with a tool (e.g., 'mysql_mcp_server_tool_call') to interact directly with a MySQL database. You MUST use this tool for all database queries to simulate data gathering. Your answers and derived values will be based SOLELY on the data retrieved via this tool from the live database.** Your current task is to generate ONE comprehensive Question-Answer (QA) item related to the Magento 2 database.

**1. Core Principles: Feasibility, Difficulty, and Diversity**

**A. Principle 1: Web Agent Feasibility**

**Crucially, every question you generate MUST be answerable by a web agent interacting with a standard Magento admin panel.** The agent does not have direct database access. Its capabilities are limited to what a human user can do through a web browser:

*   **Searching:** Using search bars on product, order, or customer pages.
*   **Filtering:** Applying filters to grids (e.g., filter orders by status 'Processing').
*   **Sorting:** Sorting columns in a grid (e.g., sort customers by 'Lifetime Sales' to find the top one).
*   **Navigation & Reading:** Clicking into a specific item's detail/edit page and reading a value.
*   **Counting from Grids:** Reading the total count of items after applying a filter (e.g., "Showing 1-20 of **45** items").

**Avoid questions that require complex, database-only operations.**
*   **BAD (Not Web-Feasible):** `"What is the average number of items per order?"` - No single page in the admin panel calculates and displays this value.
*   **GOOD (Web-Feasible):** `"How many orders currently have the status 'pending'?"` - An agent can navigate to the Sales > Orders grid, apply a filter, and read the total count.

**B. Principle 2: Task Difficulty Levels**

You must generate a task that aligns with the specified `{DIFFICULTY}` level. The difficulty is determined by the complexity of the workflow the web agent must execute.

*   **### Easy**
    *   **Definition:** Tasks that can be completed in a **single step or on a single page** with a simple action.
    *   **Typical Actions:** Applying a single filter to a grid, performing a direct search for a known item, or reading a clearly visible value on a main page.
    *   **Web-Feasible Example:** "How many orders currently have the status 'pending'?"
    *   **Agent Workflow:** Navigate to the Sales > Orders grid, apply one filter ("Status: Pending"), and read the total record count displayed on the page.

*   **### Medium**
    *   **Definition:** Tasks that require a **sequence of 2-4 distinct, linear steps**. This often involves navigating from a list/grid view to a detail/edit view.
    *   **Typical Actions:** Searching for an item then opening it, applying multiple filters, or finding an item in one grid and using that info to look something up on its detail page.
    *   **Web-Feasible Example:** "What is the shipping address for the order with the increment ID '000000123'?"
    *   **Agent Workflow:** 1. Navigate to the Sales > Orders grid. 2. Search for '000000123'. 3. Click on the order to open its detail page. 4. Locate and read the shipping address block.

*   **### Hard**
    *   **Definition:** Tasks that require **complex logic, data comparison/synthesis across different pages, or looping through items**. The agent cannot rely on a simple, linear sequence of clicks.
    *   **Typical Actions:** Finding information on one page to use as a filter on another, comparing values across multiple items manually, or tasks where the UI doesn't natively support the required sorting/filtering combination.
    *   **Web-Feasible Example:** "What is the name of the most expensive product within the 'Tops' category?"
    *   **Agent Workflow (is complex):** 1. Navigate to the Products grid. 2. Filter by "Category: Tops". 3. The grid likely cannot be sorted by final price directly in this view. The agent would need to iterate through the filtered product list, potentially clicking into *each product's page* to find its price, store it, and compare it against the others to find the maximum. This looping and comparison makes it hard.

**IMPORTANT: You MUST NOT generate questions that require the agent to answer YES or NO, because the web agent may randomly guess an answer.**

**C. Principle 3: Dynamic Entity Selection for True Diversity**

**To prevent generating repetitive questions using the same few examples, you MUST dynamically source the key entity for your question from the live database.** This is a mandatory first step in your process.

*   **Problem to Avoid:** Using SKUs, order numbers, or customer emails directly from the illustrative `TABLE_SAMPLES_CONTENT`.
*   **Required Action:** Before formulating your question, you **MUST** perform an initial, exploratory query to fetch a list of random, valid identifiers from the database.
*   **Example Exploratory Queries (using `ORDER BY RAND()`):**
    *   To get random product SKUs: `SELECT sku FROM catalog_product_entity ORDER BY RAND() LIMIT 10;`
    *   To get random order increment IDs: `SELECT increment_id FROM sales_order ORDER BY RAND() LIMIT 10;`
    *   To get random customer emails: `SELECT email FROM customer_entity ORDER BY RAND() LIMIT 10;`
*   **Rule:** After fetching this list via your tool, you **MUST** select ONE entity which you NEVER MET before from the returned results to use as the subject of your `final_question`.

**D. Principle 4: Avoiding Repetition of Previous Tasks**

**You will be provided with a list of previously generated questions. Your primary goal is to create a new task that is fundamentally different in its core logic and agent workflow.**

*   **List of Previous Tasks for Reference:**
    --- START OF PREVIOUSLY GENERATED TASKS ---
    {PREVIOUS_GENERATED_TASKS}
    --- END OF PREVIOUSLY GENERATED TASKS ---
*   **Definition of Repetition (to be AVOIDED):**
    *   Simply changing the entity (e.g., asking for the stock of SKU 'B' instead of SKU 'A').
    *   Asking for the same information about a different entity type (e.g., asking for a customer's creation date instead of an order's creation date).
    *   Minor variations in filtering (e.g., asking for 'processing' orders instead of 'pending' orders).
*   **Required Action:**
    1.  **Analyze the PREVIOUS_GENERATED_TASKS list.** Identify the core agent workflows and question patterns that have already been used (e.g., "find item by X and read property Y", "filter grid by Z and count results").
    2.  **Innovate a new task.** Your new question must introduce a new combination of actions, a different sequence of steps, or query a different aspect of the system that has not been explored in the previous tasks.
    3.  **Self-Correction:** If your initial idea feels too similar to a previous task, you MUST discard it and formulate a new, more distinct one. Narrate this in your thought process: "My initial idea to ask for a product's price is too similar to the previous task about a product's stock. I will instead create a task about finding all products within a certain price range."

**2. Contextual & Inspirational Information**

*   **Database Schema:**
    --- START OF DATABASE SCHEMA ---
    {MAGENTO_SCHEMA_CONTENT}
    --- END OF DATABASE SCHEMA ---

*   **Sample Data from Tables (Illustrative ONLY - for initial understanding. DO NOT use this sample data to derive your final answer or to select entities for your question.):**
    --- START OF SAMPLE DATA ---
    {TABLE_SAMPLES_CONTENT}
    --- END OF SAMPLE DATA ---

*   **Key Magento Schema Characteristics & EAV Model (Essential for query design):**
    *   **EAV (Entity-Attribute-Value):** Many entities (products, categories, customers) use EAV. Your SQL will need to join `eav_attribute` (to find `attribute_id` from `attribute_code`) with the correct value table (e.g., `catalog_product_entity_varchar`, `_int`).
    *   **Store Scopes:** Data can be global (store_id=0 or admin), website-specific, or store-view-specific. Queries for attributes often need to specify `store_id = 0` for admin/default values.
    *   **Order Workflow:** `quote` (cart) -> `sales_order` -> `sales_invoice`, `sales_shipment`, `sales_creditmemo`.
    *   **Flat/Grid Tables:** Tables like `sales_order_grid` and `customer_grid_flat` are excellent indicators of what data is likely available in an admin grid for a web agent to see, filter, and sort.

*   **Question Diversity Inspiration (Themes for Web-Feasible Tasks)**
    *   **A. Ranking & Sorting:** "Which customer has the highest Lifetime Sales value?"
    *   **B. Aggregation via Filtered Count:** "What is the total number of orders with the status 'complete'?"
    *   **C. Temporal / Date-Based Filtering:** "How many new customer accounts were created in the last 7 days?"
    *   **D. Conditional Filtering & Property Checks:** "Find all 'simple' products that are currently out of stock."
    *   **E. Existence & Specific Lookups:** "Is the product with SKU '[Dynamically Selected SKU]' currently enabled?"
    *   **F. EAV Attribute Lookups:** "What is the customer's Group for the user with email '[Dynamically Selected Email]'?"

**3. Your Task: Generate ONE QA Item of `{DIFFICULTY}` difficulty**

Follow these phases meticulously:

**Phase A: Question Formulation & Iterative Data Collection (USING YOUR DATABASE TOOL)**

1.  **Analyze Previous Tasks & Innovate (MANDATORY FIRST STEP):**
    *   Review the `{PREVIOUS_GENERATED_TASKS}` list to understand existing task patterns.
    *   **State your analysis:** "I have reviewed the previous tasks. I see patterns like [describe a pattern]. To avoid repetition, I will create a new task that involves [describe the novel workflow/logic]."

2.  **Dynamic Entity Selection (MANDATORY SECOND STEP):**
    *   If your novel question idea requires a specific entity, perform an exploratory query to fetch a list of random, valid identifiers. You **MUST** use a method like `ORDER BY RAND() LIMIT 10` for this.
    *   **State your plan:** "For my novel question, I need a random entity. I will execute using my tool: `[Your exploratory SQL query]`".
    *   **Process Tool Results:** "My tool returned: `[...]`. I will select '[Chosen Entity]'."

3.  **Formulate an `initial_question` (string):**
    *   **CRITICAL:** Now, using your novel idea and any selected entity, formulate a question.
    *   The question's complexity MUST match the specified `{DIFFICULTY}` level.
    *   The question must be **strictly feasible for a web agent**.
    *   **Special Instructions for Ranking Questions:** If your novel idea involves ranking, you **MUST** follow these rules:
        *   **Tie Handling:** Your answer must include *all* items that share the same rank. Use window functions like `DENSE_RANK()` in your verification queries.
        *   **Boundary Exploration for Top Ranks:** You must probe to find *all* items tied at the top rank. Use a robust window function query (`... WHERE dr = 1;`) to guarantee you find all ties.
        *   **Rank Existence:** If a rank might be skipped due to ties, you **MUST adjust your question** to ask for a rank that will definitely exist.

4.  **Iterative SQL Execution and Refinement (to find the answer):**
    *   **Plan & Execute SQL with Tool:** Formulate the query needed to find the answer.
    *   **Tool Response Length Constraint:** Data-gathering queries **MUST include a `LIMIT` clause** (e.g., `LIMIT 50`).
    *   **State your plan:** "To answer, I will now execute...".
    *   **Process Tool Results:** "My tool returned...".
    *   **Analyze & Refine:** Refine your question into a `final_question` that is perfectly aligned with the data, all principles, and rules.

**Phase B: Answer Derivation (from ACTUAL Tool Results)**
1.  Based on the **verified and complete data collected via your tool**, formulate an **`llm_derived_answer`**.

**Phase C: Validation Rule Design & Difficulty Rationale**
1.  **Design `validation_rules`:**
    *   A **list of rule objects**, each with `type` and `expected_value`.
    *   **`type`:** Primarily `"must_include"` or `"fuzzy_match"`.
    *   **`expected_value`:** The specific value *extracted* from your answer, derived **directly from your tool's results**.
    *   **Focus on key entities/values**. Multiple rules imply AND.
2.  **Formulate a `difficulty_reason`:** A concise explanation of why the task matches the difficulty, referencing the agent's workflow.

**Phase D: Reference SQL Formulation**
1.  Compose a **single, concise `reference_sql`**. For ranking, this must be the robust query that handles ties.

**4. Final Output Format (for this ONE QA item):**

Provide your response strictly as a single JSON object. Do not include any explanatory text outside of this JSON structure.

```json
{{
  "final_question": "Your refined, novel, web-agent-feasible question string.",
  "llm_derived_answer": "The natural language answer derived exclusively from actual database data obtained via your tool.",
  "validation_rules": [
    {{
      "type": "must_include",
      "expected_value": "Specific key value/entity extracted from the answer, based on actual database data."
    }}
  ],
  "reference_sql": "A single, representative SELECT SQL query that finds the ground truth for the question.",
  "difficulty_reason": "A concise explanation of why the task's complexity matches the specified difficulty level, based on the web agent's required workflow."
}}
```

**5. Agent Workflow and Self-Correction Reminder:**
Your thought process should explicitly narrate steps like:
1.  **"Step 1: Analyze Previous Tasks.** I have reviewed the `{PREVIOUS_GENERATED_TASKS}`. The pattern of 'find a single item and read one of its direct properties' is common. To innovate, I will create a task that requires filtering a grid and then performing an action on the *entire set* of results, like finding a common attribute among them."
2.  **"Step 2: Dynamic Entity Selection (if needed).** My new idea needs a category name. I'll execute `SELECT ... FROM catalog_category_entity_varchar ...` to get one."
3.  **"Step 3: Question Formulation.** I will now formulate my novel, `{DIFFICULTY}` question..."
4.  **"Step 4: Answering the Question.** To find the answer, I will execute a query..."
5.  **"Step 5: Deriving the Final Output.** My database tool returned... The reason this is `{DIFFICULTY}` is because..."

Please proceed with generating one QA item according to these **strict, complete, and highly detailed** instructions.
"""

# 4. 运行示例 —— 用自然语言问数据库
messages = [{
    "role": "user",
    # "content": "数据库里 catalog_product_entity_varchar 表有多少条记录？同时请展示前 5 行。"
    # "content": f"数据库里 catalog_product_entity_varchar 表有多少条记录？同时请展示前 5 行。\n\n{prompt2}"
    # "content": prompt
    # "content": prompt2
    # "content": prompt3
    # "content": prompt4
    # "content": prompt5
    # "content": prompt6
    # "content": prompt7
    # "content": prompt8
    # "content": prompt9
    "content": prompt10
}]

response_plain_text = ''
for responses in bot.run(messages=messages, stream=True):
    # stream=True 将逐步打印 LLM 思考与结果
    response_plain_text = typewriter_print(responses, response_plain_text)

# The final QA item json is in the last response
qa_item_str = responses[-1]["content"]
print("\n--- Generated QA Item ---")
print(qa_item_str)


# 新增：统计 [TOOL_RESPONSE] 子串出现的次数并打印
# 如果次数为0，意味着实际没有调用工具，得到的答案肯定是错的
tool_response_count = response_plain_text.count("[TOOL_RESPONSE]")
print(f"\n[INFO] [TOOL_RESPONSE] was observed {tool_response_count} time(s) in the generation phase.")

# --- Start of Verification Logic ---

if tool_response_count == 0:
    print("\n[VERIFICATION] SKIPPED: No tool calls were made during generation, the result is likely invalid.")
else:
    print("\n[VERIFICATION] STARTING: Tool calls were observed, proceeding with verification.")

    # 1. Parse the generated QA item
    qa_item = None
    try:
        # Clean up the string: find the JSON block, which might be wrapped in markdown
        match = re.search(r'\{.*\}', qa_item_str, re.DOTALL)
        if match:
            json_str = match.group(0)
            qa_item = json.loads(json_str)
        else:
            # Fallback for when the string is just the JSON without wrappers
            qa_item = json.loads(qa_item_str)

        final_question = qa_item.get("final_question")
        llm_derived_answer = qa_item.get("llm_derived_answer")
        reference_sql = qa_item.get("reference_sql")

        if not all([final_question, llm_derived_answer, reference_sql]):
            print(
                "[VERIFICATION] FAILED: The generated JSON is missing one or more required keys (final_question, llm_derived_answer, reference_sql).")
            qa_item = None  # Invalidate qa_item to skip next step

    except (json.JSONDecodeError, AttributeError) as e:
        print(f"[VERIFICATION] FAILED: Could not parse the JSON response from the generator bot. Error: {e}")
        qa_item = None  # Invalidate qa_item to skip next step

    if qa_item:
        # 2. Create the verifier prompt
        verifier_prompt_template = """
You are a meticulous database query verifier. Your task is to verify the consistency between a user's question, a generated answer, and a reference SQL query. You are given a tool to execute SQL queries against the database.

**Your Goal:**
Assess whether the `llm_derived_answer` is a correct and faithful response to the `final_question`, based *exclusively* on the real-time results of executing the `reference_sql`.

**Input for Verification:**
1.  **`final_question`**: The natural language question that was asked.
    ```
    {final_question}
    ```
2.  **`llm_derived_answer`**: The natural language answer that was generated.
    ```
    {llm_derived_answer}
    ```
3.  **`reference_sql`**: The SQL query intended to produce the data for the answer.
    ```sql
    {reference_sql}
    ```

**Verification Steps:**
1.  **Execute the SQL:** Use your database tool to execute the `reference_sql` exactly as provided.
2.  **Analyze SQL Results:** Carefully examine the data returned by the query. Note the number of rows, the values in each column, and whether the result is empty.
3.  **Compare and Contrast:** Critically compare the `SQL Results`, the `final_question`, and the `llm_derived_answer`.
    *   **Data Consistency:** Does the data in `llm_derived_answer` *exactly* match the data from your `SQL Results`? For example, if the answer mentions a count of "65", did your query actually return "65"? If the answer lists specific names or SKUs, are those the exact names/SKUs your query returned?
    *   **Question-Answer Alignment:** Does the `llm_derived_answer` truly answer the `final_question`?
        *   *Example of Mismatch:* The question asks for "product names," but the answer provides only "SKUs." Even if the SKUs are correct according to the SQL, this is an alignment failure.
    *   **Hallucination Check:** Does the `llm_derived_answer` contain information that is NOT supported by your `SQL Results`?
        *   *Example of Hallucination:* The answer lists several products, but your `SQL Results` are empty. This is a critical failure.

**Final Output Format:**
Provide your response strictly as a single JSON object with two keys: `verification_result` and `verification_reason`.

*   `verification_result` (string): Must be one of `CONSISTENT`, `INCONSISTENT`, or `ERROR_IN_SQL`.
    *   `CONSISTENT`: The answer is fully supported by the SQL results and correctly addresses the question.
    *   `INCONSISTENT`: There is a mismatch. This could be due to hallucinated data, incorrect values, or a failure to align with the question's intent (e.g., providing SKU instead of name).
    *   `ERROR_IN_SQL`: The `reference_sql` failed to execute due to a syntax error or other database error.
*   `verification_reason` (string): A clear, concise explanation for your conclusion. If inconsistent, explain exactly what the mismatch was. If the SQL failed, include the error message.

**Example 1 (Consistent):**
*   `llm_derived_answer`: "There are 65 orders..."
*   `SQL Result`: `[{{'order_count': 65}}]`
*   Your Output:
    ```json
    {{
        "verification_result": "CONSISTENT",
        "verification_reason": "The reference_sql executed successfully and returned a count of 65, which matches the llm_derived_answer."
    }}
    ```

**Example 2 (Inconsistent - Hallucination):**
*   `llm_derived_answer`: "The product is 'Super Widget'."
*   `SQL Result`: `[]` (empty)
*   Your Output:
    ```json
    {{
        "verification_result": "INCONSISTENT",
        "verification_reason": "The llm_derived_answer states the product is 'Super Widget', but the reference_sql returned no results. The answer appears to be a hallucination."
    }}
    ```

**Example 3 (Inconsistent - Alignment):**
*   `final_question`: "What are the names of the top products?"
*   `llm_derived_answer`: "The top product SKUs are 'WIDGET-001' and 'GADGET-002'."
*   `SQL Result`: `[{{'sku': 'WIDGET-001'}}, {{'sku': 'GADGET-002'}}]`
*   Your Output:
    ```json
    {{
        "verification_result": "INCONSISTENT",
        "verification_reason": "The final_question asks for product names, but the llm_derived_answer and reference_sql only provide SKUs. The answer does not align with the question's requirement."
    }}
    ```
    
Now, perform the verification for the provided inputs.
"""
        verifier_prompt = verifier_prompt_template.format(
            final_question=final_question,
            llm_derived_answer=llm_derived_answer,
            reference_sql=reference_sql
        )

        # 3. Create and run the verifier bot
        verifier = Assistant(
            llm=llm_cfg,
            function_list=tools,
        )

        verifier_messages = [{"role": "user", "content": verifier_prompt}]

        print("\n--- Verifier Bot ---")
        verifier_response_text = ''
        for verifier_responses in verifier.run(messages=verifier_messages, stream=True):
            verifier_response_text = typewriter_print(verifier_responses, verifier_response_text)

        print("\n--- Verification Result ---")
        print(verifier_responses[-1]["content"])

