yuyr/webrlvr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add verbose mode

Browse Source
This commit is contained in:
yuyr 2025-07-01 10:58:17 +00:00
parent 17f1b1f8c7
commit 33a016a89c
3 changed files with 310544 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
random_sample/generate_tasks.py
View File

@ -217,7 +217,7 @@ def get_random_tables_and_samples(cursor, tables, num_tables=5, num_samples=5):
return sampled_data
def generate_questions(client, schema_context, sampled_data):
def generate_questions(client, schema_context, sampled_data, verbose=False):
"""Generates questions by calling the OpenAI API."""
if not client:
raise ValueError("OpenAI client not provided.")
@ -229,6 +229,11 @@ def generate_questions(client, schema_context, sampled_data):
sampled_data_str=sampled_data_str
)
if verbose:
print("\n--- Generation Prompt ---")
print(prompt)
print("-------------------------\n")
try:
response = client.chat.completions.create(
model=OPENAI_CONFIG["model"],
@ -240,6 +245,10 @@ def generate_questions(client, schema_context, sampled_data):
response_format={"type": "json_object"},
)
content = response.choices[0].message.content
if verbose:
print("\n--- GPT-4o Raw Generation Response ---")
print(content)
print("--------------------------------------\n")
data = json.loads(content)
# The prompt asks for {"questions": [...]}, so we extract the list.
@ -271,7 +280,7 @@ def load_existing_tasks(filepath):
print(f"Warning: Could not read or parse {filepath}. Starting with an empty list.")
return []
def evaluate_and_refine_tasks(tasks, client):
def evaluate_and_refine_tasks(tasks, client, verbose=False):
"""
Uses an LLM to evaluate if a SQL result answers the question and refines the answer.
"""
@ -293,6 +302,11 @@ def evaluate_and_refine_tasks(tasks, client):
prompt = SEMANTIC_EVALUATION_PROMPT_TEMPLATE.format(task_data_json=task_data_json)
if verbose:
print("\n--- Evaluation Prompt ---")
print(prompt)
print("-------------------------\n")
try:
print(f" - Evaluating question: \"{task['question'][:80]}...\"")
response = client.chat.completions.create(
@ -305,6 +319,10 @@ def evaluate_and_refine_tasks(tasks, client):
response_format={"type": "json_object"},
)
content = response.choices[0].message.content
if verbose:
print("\n--- GPT-4o Raw Evaluation Response ---")
print(content)
print("----------------------------------------\n")
evaluation_result = json.loads(content)
if evaluation_result.get("can_answer") is True and "new_answer" in evaluation_result:
@ -351,6 +369,11 @@ def main():
default="generated_tasks.json",
help="The file to save the generated tasks to (in JSON format)."
)
parser.add_argument(
"-v", "--verbose",
action="store_true",
help="Enable verbose output, including prompts and raw LLM responses."
)
args = parser.parse_args()
# Load existing tasks from the output file
@ -389,10 +412,14 @@ def main():
# Get random samples for this round
print("Sampling data from 5 random tables...")
sampled_data = get_random_tables_and_samples(cursor, core_tables, num_tables=5, num_samples=5)
if args.verbose:
print("\n--- Sampled Data ---")
print(json.dumps(sampled_data, indent=2, default=str))
print("--------------------\n")
# Generate questions
print("Generating questions with GPT-4o...")
generated_tasks = generate_questions(client, schema_context, sampled_data)
generated_tasks = generate_questions(client, schema_context, sampled_data, verbose=args.verbose)
# Execute SQL for generated tasks
tasks_for_evaluation = []
@ -416,7 +443,7 @@ def main():
print(f"Filtering task due to SQL error: {err} on SQL: {task['sql']}")
# Perform semantic evaluation and get validated tasks
validated_tasks = evaluate_and_refine_tasks(tasks_for_evaluation, client)
validated_tasks = evaluate_and_refine_tasks(tasks_for_evaluation, client, verbose=args.verbose)
# Append new tasks and save to file
if validated_tasks:

310131
random_sample/generated_tasks_5k.json Normal file
View File

File diff suppressed because it is too large Load Diff

382
random_sample/sample.json Normal file
View File

@ -0,0 +1,382 @@
[
{
"question": "What is the email address for the customer named Jane Doe?",
"sql": "SELECT email FROM customer_entity WHERE firstname = 'Jane' AND lastname = 'Doe';",
"answer": [
"jane.doe@hotmail.com",
"jane.doe@gmail.com"
],
"sql_execute_result": [
[
"jane.doe@hotmail.com"
],
[
"jane.doe@gmail.com"
]
]
},
{
"question": "Find the total base grand total for orders with the status 'complete'.",
"sql": "SELECT SUM(base_grand_total) FROM sales_order_grid WHERE status = 'complete';",
"answer": [
"20625.5000"
],
"sql_execute_result": [
[
"20625.5000"
]
]
},
{
"question": "What is the shipping address for order with increment_id '000000100'?",
"sql": "SELECT shipping_address FROM sales_order_grid WHERE increment_id = '000000100';",
"answer": [
"789 W Olympic Blvd, Los Angeles, California, 90015"
],
"sql_execute_result": [
[
"789 W Olympic Blvd,Los Angeles,California,90015"
]
]
},
{
"question": "List all customer group codes available in the system.",
"sql": "SELECT customer_group_code FROM customer_group;",
"answer": [
"NOT LOGGED IN",
"General",
"Wholesale",
"Retailer"
],
"sql_execute_result": [
[
"NOT LOGGED IN"
],
[
"General"
],
[
"Wholesale"
],
[
"Retailer"
]
]
},
{
"question": "How many orders have been placed by the customer with email 'michael.nguyen@yahoo.com'?",
"sql": "SELECT COUNT(entity_id) FROM sales_order_grid WHERE customer_email = 'michael.nguyen@yahoo.com';",
"answer": [
"12"
],
"sql_execute_result": [
[
12
]
]
},
{
"question": "What is the product description for the product with entity_id 1636?",
"sql": "SELECT value FROM catalog_product_entity_text WHERE entity_id = 1636;",
"answer": [
"<p>Whatever your goals for the day's workout, the Celeste Sports Bra lets you do it in comfort and coolness, plus enhanced support and shaping. A power mesh back zone and moisture-wicking fabric ensure you stay dry.</p>\n<p>&bull; Mint bra top. <br />&bull; Seam-free interior molded cups<br />&bull; Odor control. <br />&bull; UV protection. <br />&bull; Machine wash/dry.</p>"
],
"sql_execute_result": [
[
"<p>Whatever your goals for the day's workout, the Celeste Sports Bra lets you do it in comfort and coolness, plus enhanced support and shaping. A power mesh back zone and moisture-wicking fabric ensure you stay dry.</p>\n<p>&bull; Mint bra top. <br />&bull; Seam-free interior molded cups<br />&bull; Odor control. <br />&bull; UV protection. <br />&bull; Machine wash/dry.</p>"
],
[
"142,153"
],
[
"130"
],
[
"196"
],
[
"204,208"
]
]
},
{
"question": "What is the billing name for the order with entity_id 229?",
"sql": "SELECT billing_name FROM sales_order_grid WHERE entity_id = 229;",
"answer": [
"Sarah Miller"
],
"sql_execute_result": [
[
"Sarah Miller"
]
]
},
{
"question": "Find the store name where the order with increment_id '000000258' was placed.",
"sql": "SELECT store_name FROM sales_order_grid WHERE increment_id = '000000258';",
"answer": [
"Main Website",
"Main Website Store",
"Default Store View"
],
"sql_execute_result": [
[
"Main Website\nMain Website Store\nDefault Store View"
]
]
},
{
"question": "What is the base currency code for the order placed by 'Bob Jones'?",
"sql": "SELECT base_currency_code FROM sales_order_grid WHERE shipping_name = 'Bob Jones';",
"answer": [
"USD"
],
"sql_execute_result": [
[
"USD"
],
[
"USD"
],
[
"USD"
],
[
"USD"
],
[
"USD"
],
[
"USD"
],
[
"USD"
],
[
"USD"
],
[
"USD"
],
[
"USD"
]
]
},
{
"question": "List the attribute values for the category with entity ID 3.",
"sql": "SELECT value FROM catalog_category_entity_varchar WHERE entity_id = 3;",
"answer": [
"Gear",
"PAGE",
"gear"
],
"sql_execute_result": [
[
"Gear"
],
[
"PAGE"
],
[
"gear"
],
[
"gear"
]
]
},
{
"question": "What is the tax amount for the product 'Iris Workout Top'?",
"sql": "SELECT tax_amount FROM sales_invoice_item WHERE name = 'Iris Workout Top';",
"answer": [
"2.3900"
],
"sql_execute_result": [
[
"2.3900"
]
]
},
{
"question": "How many option values were found for attribute ID 144?",
"sql": "SELECT value FROM eav_attribute_option_value WHERE option_id IN (SELECT option_id FROM eav_attribute_option WHERE attribute_id = 144);",
"answer": [
"20"
],
"sql_execute_result": [
[
"55 cm"
],
[
"65 cm"
],
[
"75 cm"
],
[
"6 foot"
],
[
"8 foot"
],
[
"10 foot"
],
[
"XS"
],
[
"S"
],
[
"M"
],
[
"L"
],
[
"XL"
],
[
"28"
],
[
"29"
],
[
"30"
],
[
"31"
],
[
"32"
],
[
"33"
],
[
"34"
],
[
"36"
],
[
"38"
]
]
},
{
"question": "What is the base price for the product with SKU 'WS03-XS-Red'?",
"sql": "SELECT base_price FROM sales_invoice_item WHERE sku = 'WS03-XS-Red';",
"answer": [
"29.0000"
],
"sql_execute_result": [
[
"29.0000"
]
]
},
{
"question": "How many sort orders are there for the attribute option with ID 152?",
"sql": "SELECT sort_order FROM eav_attribute_option WHERE attribute_id = 152;",
"answer": [
"26"
],
"sql_execute_result": [
[
0
],
[
1
],
[
2
],
[
3
],
[
4
],
[
5
],
[
6
],
[
7
],
[
8
],
[
9
],
[
10
],
[
11
],
[
12
],
[
13
],
[
14
],
[
15
],
[
16
],
[
17
],
[
18
],
[
19
],
[
20
],
[
21
],
[
22
],
[
23
],
[
24
],
[
25
]
]
},
{
"question": "Retrieve the product ID for the category 'collections/performance-new'.",
"sql": "SELECT entity_id FROM catalog_category_entity_varchar WHERE value = 'collections/performance-new';",
"answer": [
"39"
],
"sql_execute_result": [
[
39
]
]
}
]