diff --git a/.note/interfaces.md b/.note/interfaces.md
index 54cbac2..811605a 100644
--- a/.note/interfaces.md
+++ b/.note/interfaces.md
@@ -576,17 +576,19 @@ reranker = get_jina_reranker()
```python
from ranking.jina_reranker import JinaReranker
+# Initialize with specific model
reranker = JinaReranker()
-query = "What is quantum computing?"
-documents = [
- "Quantum computing is a computation system that uses quantum mechanics.",
- "Classical computers use bits while quantum computers use qubits.",
- "Artificial intelligence is transforming various industries."
-]
-reranked = reranker.rerank(query, documents)
-for doc in reranked:
- print(f"Score: {doc['score']}, Document: {doc['document']}")
+# Rerank documents
+results = reranker.rerank(
+ query="What is quantum computing?",
+ documents=["Document about quantum physics", "Document about quantum computing", "Document about classical computing"],
+ top_n=2
+)
+
+# Process results
+for result in results:
+ print(f"Score: {result['score']}, Document: {result['document']}")
```
#### Integration with ResultCollector
@@ -785,115 +787,141 @@ results = reranker.rerank(
# Process results
for result in results:
print(f"Score: {result['score']}, Document: {result['document']}")
-
-## Query Processor Testing
-
-The query processor module has been tested with the Groq LLM provider to ensure it functions correctly with the newly integrated models.
-
-### Test Scripts
-
-Two test scripts have been created to validate the query processor functionality:
-
-#### Basic Test Script (test_query_processor.py)
-
-```python
-# Get the query processor
-processor = get_query_processor()
-
-# Process a query
-result = processor.process_query("What are the latest advancements in quantum computing?")
-
-# Generate search queries
-search_result = processor.generate_search_queries(result, ["google", "bing", "scholar"])
```
-- **Purpose**: Tests the core functionality of the query processor
-- **Features**:
- - Uses monkey patching to ensure the Groq model is used
- - Provides detailed output of processing results
+## Report Generation Module
-#### Comprehensive Test Script (test_query_processor_comprehensive.py)
+### ReportDetailLevelManager Class
-```python
-# Test query enhancement
-enhanced_query = test_enhance_query("What is quantum computing?")
-
-# Test query classification
-classification = test_classify_query("What is quantum computing?")
-
-# Test the full processing pipeline
-structured_query = test_process_query("What is quantum computing?")
-
-# Test search query generation
-search_result = test_generate_search_queries(structured_query, ["google", "bing", "scholar"])
-```
-
-- **Purpose**: Tests all aspects of the query processor in detail
-- **Features**:
- - Tests individual components in isolation
- - Tests a variety of query types
- - Saves detailed test results to a JSON file
-
-## LLM Interface
-
-### LLMInterface Class
-
-The `LLMInterface` class provides a unified interface for interacting with various LLM providers through LiteLLM.
+The `ReportDetailLevelManager` class manages configurations for different report detail levels.
#### Initialization
```python
-llm = LLMInterface(model_name="gpt-4")
+detail_level_manager = get_report_detail_level_manager()
```
-- **Description**: Initializes the LLM interface with the specified model
-- **Parameters**:
- - `model_name` (Optional[str]): The name of the model to use (defaults to config value)
-- **Requirements**: Appropriate API key must be set in environment or config
+- **Description**: Gets a singleton instance of the ReportDetailLevelManager
-#### complete
+#### get_detail_level_config
```python
-response = llm.complete(prompt, system_prompt=None, temperature=None, max_tokens=None)
+config = detail_level_manager.get_detail_level_config(detail_level)
```
-- **Description**: Generates a completion for the given prompt
+- **Description**: Gets configuration parameters for a specific detail level
- **Parameters**:
- - `prompt` (str): The prompt to complete
- - `system_prompt` (Optional[str]): System prompt for context
- - `temperature` (Optional[float]): Temperature for generation
- - `max_tokens` (Optional[int]): Maximum tokens to generate
-- **Returns**: str - The generated completion
-- **Raises**: LLMError if the completion fails
+ - `detail_level` (str): Detail level as a string (brief, standard, detailed, comprehensive)
+- **Returns**: Dict[str, Any] - Configuration parameters for the specified detail level
+- **Raises**: ValueError if the detail level is not valid
-#### complete_json
+#### get_template_modifier
```python
-json_response = llm.complete_json(prompt, system_prompt=None, json_schema=None)
+template = detail_level_manager.get_template_modifier(detail_level, query_type)
```
-- **Description**: Generates a JSON response for the given prompt
+- **Description**: Gets template modifier for a specific detail level and query type
- **Parameters**:
- - `prompt` (str): The prompt to complete
- - `system_prompt` (Optional[str]): System prompt for context
- - `json_schema` (Optional[Dict]): JSON schema for validation
-- **Returns**: Dict - The generated JSON response
-- **Raises**: LLMError if the completion fails or JSON is invalid
+ - `detail_level` (str): Detail level as a string (brief, standard, detailed, comprehensive)
+ - `query_type` (str): Query type as a string (factual, exploratory, comparative)
+- **Returns**: str - Template modifier as a string
+- **Raises**: ValueError if the detail level or query type is not valid
-#### Supported Providers
-- OpenAI
-- Azure OpenAI
-- Anthropic
-- Ollama
-- Groq
-- OpenRouter
-
-#### Example Usage
+#### get_available_detail_levels
```python
-from query.llm_interface import LLMInterface
+levels = detail_level_manager.get_available_detail_levels()
+```
+- **Description**: Gets a list of available detail levels with descriptions
+- **Returns**: List[Tuple[str, str]] - List of tuples containing detail level and description
-# Initialize with specific model
-llm = LLMInterface(model_name="llama-3.1-8b-instant")
+### ReportGenerator Class
-# Generate a completion
-response = llm.complete(
- prompt="Explain quantum computing",
- system_prompt="You are a helpful assistant that explains complex topics simply.",
- temperature=0.7
+The `ReportGenerator` class generates reports from search results.
+
+#### Initialization
+```python
+report_generator = get_report_generator()
+```
+- **Description**: Gets a singleton instance of the ReportGenerator
+
+#### initialize
+```python
+await report_generator.initialize()
+```
+- **Description**: Initializes the report generator by setting up the database
+- **Returns**: None
+
+#### set_detail_level
+```python
+report_generator.set_detail_level(detail_level)
+```
+- **Description**: Sets the detail level for report generation
+- **Parameters**:
+ - `detail_level` (str): Detail level (brief, standard, detailed, comprehensive)
+- **Returns**: None
+- **Raises**: ValueError if the detail level is not valid
+
+#### get_detail_level_config
+```python
+config = report_generator.get_detail_level_config()
+```
+- **Description**: Gets the current detail level configuration
+- **Returns**: Dict[str, Any] - Configuration parameters for the current detail level
+
+#### get_available_detail_levels
+```python
+levels = report_generator.get_available_detail_levels()
+```
+- **Description**: Gets a list of available detail levels with descriptions
+- **Returns**: List[Tuple[str, str]] - List of tuples containing detail level and description
+
+#### process_search_results
+```python
+documents = await report_generator.process_search_results(search_results)
+```
+- **Description**: Processes search results by scraping the URLs and storing them in the database
+- **Parameters**:
+ - `search_results` (List[Dict[str, Any]]): List of search results, each containing at least a 'url' field
+- **Returns**: List[Dict[str, Any]] - List of processed documents
+
+#### prepare_documents_for_report
+```python
+chunks = await report_generator.prepare_documents_for_report(search_results, token_budget, chunk_size, overlap_size)
+```
+- **Description**: Prepares documents for report generation by chunking and selecting relevant content
+- **Parameters**:
+ - `search_results` (List[Dict[str, Any]]): List of search results
+ - `token_budget` (Optional[int]): Maximum number of tokens to use
+ - `chunk_size` (Optional[int]): Maximum number of tokens per chunk
+ - `overlap_size` (Optional[int]): Number of tokens to overlap between chunks
+- **Returns**: List[Dict[str, Any]] - List of selected document chunks
+
+#### generate_report
+```python
+report = await report_generator.generate_report(
+ search_results=search_results,
+ query=query,
+ token_budget=token_budget,
+ chunk_size=chunk_size,
+ overlap_size=overlap_size,
+ detail_level=detail_level
)
+```
+- **Description**: Generates a report from search results
+- **Parameters**:
+ - `search_results` (List[Dict[str, Any]]): List of search results
+ - `query` (str): Original search query
+ - `token_budget` (Optional[int]): Maximum number of tokens to use
+ - `chunk_size` (Optional[int]): Maximum number of tokens per chunk
+ - `overlap_size` (Optional[int]): Number of tokens to overlap between chunks
+ - `detail_level` (Optional[str]): Level of detail for the report (brief, standard, detailed, comprehensive)
+- **Returns**: str - Generated report as a string
-print(response)
+#### initialize_report_generator
+```python
+await initialize_report_generator()
+```
+- **Description**: Initializes the global report generator instance
+- **Returns**: None
+
+#### get_report_generator
+```python
+report_generator = get_report_generator()
+```
+- **Description**: Gets the global report generator instance
+- **Returns**: ReportGenerator - Initialized report generator instance
diff --git a/ui/gradio_interface.py b/ui/gradio_interface.py
index 60ad314..eabb231 100644
--- a/ui/gradio_interface.py
+++ b/ui/gradio_interface.py
@@ -8,7 +8,9 @@ import json
import gradio as gr
import sys
import time
+import asyncio
from pathlib import Path
+from datetime import datetime
# Add the parent directory to the path to allow importing from other modules
sys.path.append(str(Path(__file__).parent.parent))
@@ -16,6 +18,9 @@ sys.path.append(str(Path(__file__).parent.parent))
from query.query_processor import QueryProcessor
from execution.search_executor import SearchExecutor
from execution.result_collector import ResultCollector
+from report.report_generator import get_report_generator, initialize_report_generator
+from report.report_detail_levels import get_report_detail_level_manager, DetailLevel
+from config.config import Config
class GradioInterface:
@@ -28,6 +33,20 @@ class GradioInterface:
self.result_collector = ResultCollector()
self.results_dir = Path(__file__).parent.parent / "results"
self.results_dir.mkdir(exist_ok=True)
+ self.reports_dir = Path(__file__).parent.parent
+ self.reports_dir.mkdir(exist_ok=True)
+ self.detail_level_manager = get_report_detail_level_manager()
+ self.config = Config()
+
+ # The report generator will be initialized in the async init method
+ self.report_generator = None
+
+ async def async_init(self):
+ """Asynchronously initialize components that require async initialization."""
+ # Initialize the report generator
+ await initialize_report_generator()
+ self.report_generator = get_report_generator()
+ return self
def process_query(self, query, num_results=10, use_reranker=True):
"""
@@ -165,6 +184,146 @@ class GradioInterface:
return markdown
+ async def generate_report(self, query, detail_level, custom_model=None, process_thinking_tags=False, results_file=None):
+ """
+ Generate a report from a query.
+
+ Args:
+ query (str): The query to process
+ detail_level (str): Detail level for the report
+ custom_model (str): Custom model to use for report generation
+ process_thinking_tags (bool): Whether to process thinking tags in the output
+ results_file (str): Path to results file (optional)
+
+ Returns:
+ tuple: (report_markdown, report_file_path)
+ """
+ try:
+ # Create a timestamped output file
+ timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+ model_suffix = ""
+ if custom_model:
+ model_name = custom_model.split('/')[-1]
+ model_suffix = f"_{model_name}"
+
+ output_file = self.reports_dir / f"report_{timestamp}{model_suffix}.md"
+
+ # Get detail level configuration
+ config = self.detail_level_manager.get_detail_level_config(detail_level)
+
+ # If custom model is provided, use it
+ if custom_model:
+ config["model"] = custom_model
+
+ print(f"Generating report with detail level: {detail_level}")
+ print(f"Detail level configuration: {config}")
+ print(f"Using model: {config['model']}")
+ print(f"Processing thinking tags: {process_thinking_tags}")
+
+ # If results file is provided, load results from it
+ search_results = []
+ if results_file and os.path.exists(results_file):
+ with open(results_file, 'r') as f:
+ search_results = json.load(f)
+ print(f"Loaded {len(search_results)} results from {results_file}")
+ else:
+ # If no results file is provided, perform a search
+ print(f"No results file provided, performing search for: {query}")
+
+ # Process the query to create a structured query
+ structured_query = self.query_processor.process_query(query)
+
+ # Generate search queries for different engines
+ structured_query = self.query_processor.generate_search_queries(
+ structured_query,
+ self.search_executor.get_available_search_engines()
+ )
+
+ # Execute the search with the structured query
+ search_results_dict = self.search_executor.execute_search(
+ structured_query,
+ num_results=config["num_results"]
+ )
+
+ # Flatten the search results
+ search_results = []
+ for engine_results in search_results_dict.values():
+ search_results.extend(engine_results)
+
+ # Rerank results if we have a reranker
+ if hasattr(self, 'reranker') and self.reranker:
+ search_results = self.reranker.rerank_with_metadata(
+ query,
+ search_results,
+ document_key='snippet',
+ top_n=config["num_results"]
+ )
+
+ # Set the model for report generation if custom model is provided
+ if custom_model:
+ # This will update the report synthesizer to use the custom model
+ self.report_generator.set_detail_level(detail_level)
+
+ # Generate the report
+ report = await self.report_generator.generate_report(
+ search_results=search_results,
+ query=query,
+ token_budget=config["token_budget"],
+ chunk_size=config["chunk_size"],
+ overlap_size=config["overlap_size"],
+ detail_level=detail_level
+ )
+
+ # Process thinking tags if requested
+ if process_thinking_tags:
+ report = self._process_thinking_tags(report)
+
+ # Save report to file
+ with open(output_file, 'w', encoding='utf-8') as f:
+ f.write(report)
+
+ print(f"Report saved to: {output_file}")
+
+ return report, str(output_file)
+
+ except Exception as e:
+ error_message = f"Error generating report: {str(e)}"
+ print(f"ERROR: {error_message}")
+ import traceback
+ traceback.print_exc()
+ return f"## Error\n\n{error_message}", None
+
+ def _process_thinking_tags(self, text):
+ """
+ Process thinking tags in the text.
+
+ Args:
+ text (str): Text to process
+
+ Returns:
+ str: Processed text
+ """
+ # Remove content between and tags
+ import re
+ return re.sub(r'.*?', '', text, flags=re.DOTALL)
+
+ def get_available_models(self):
+ """
+ Get a list of available models for report generation.
+
+ Returns:
+ list: List of available model names
+ """
+ # Get models from config
+ models = [
+ "llama-3.1-8b-instant",
+ "llama-3.3-70b-versatile",
+ "groq/deepseek-r1-distill-llama-70b-specdec",
+ "openrouter-mixtral",
+ "openrouter-claude"
+ ]
+ return models
+
def create_interface(self):
"""
Create and return the Gradio interface.
@@ -179,62 +338,126 @@ class GradioInterface:
This system helps you research topics by searching across multiple sources
including Google (via Serper), Google Scholar, and arXiv.
- The system will return ALL results from each search engine, up to the maximum
- number specified by the "Results Per Engine" slider. Results are ranked by
- relevance across all sources.
+ You can either search for results or generate a comprehensive report.
"""
)
- with gr.Row():
- with gr.Column(scale=4):
- query_input = gr.Textbox(
- label="Research Query",
- placeholder="Enter your research question here...",
- lines=3
+ with gr.Tabs() as tabs:
+ with gr.TabItem("Search"):
+ with gr.Row():
+ with gr.Column(scale=4):
+ search_query_input = gr.Textbox(
+ label="Research Query",
+ placeholder="Enter your research question here...",
+ lines=3
+ )
+ with gr.Column(scale=1):
+ search_num_results = gr.Slider(
+ minimum=5,
+ maximum=50,
+ value=20,
+ step=5,
+ label="Results Per Engine"
+ )
+ search_use_reranker = gr.Checkbox(
+ label="Use Semantic Reranker",
+ value=True,
+ info="Uses Jina AI's reranker for more relevant results"
+ )
+ search_button = gr.Button("Search", variant="primary")
+
+ gr.Examples(
+ examples=[
+ ["What are the latest advancements in quantum computing?"],
+ ["Compare transformer and RNN architectures for NLP tasks"],
+ ["Explain the environmental impact of electric vehicles"]
+ ],
+ inputs=search_query_input
)
- with gr.Column(scale=1):
- num_results = gr.Slider(
- minimum=5,
- maximum=50,
- value=20,
- step=5,
- label="Results Per Engine"
- )
- use_reranker = gr.Checkbox(
- label="Use Semantic Reranker",
- value=True,
- info="Uses Jina AI's reranker for more relevant results"
- )
- search_button = gr.Button("Search", variant="primary")
-
- gr.Examples(
- examples=[
- ["What are the latest advancements in quantum computing?"],
- ["Compare transformer and RNN architectures for NLP tasks"],
- ["Explain the environmental impact of electric vehicles"]
- ],
- inputs=query_input
- )
-
- with gr.Row():
- with gr.Column():
- results_output = gr.Markdown(label="Results")
+
+ with gr.Row():
+ with gr.Column():
+ search_results_output = gr.Markdown(label="Results")
+
+ with gr.Row():
+ with gr.Column():
+ search_file_output = gr.Textbox(
+ label="Results saved to file",
+ interactive=False
+ )
- with gr.Row():
- with gr.Column():
- file_output = gr.Textbox(
- label="Results saved to file",
- interactive=False
+ with gr.TabItem("Generate Report"):
+ with gr.Row():
+ with gr.Column(scale=4):
+ report_query_input = gr.Textbox(
+ label="Research Query",
+ placeholder="Enter your research question here...",
+ lines=3
+ )
+ with gr.Column(scale=1):
+ report_detail_level = gr.Dropdown(
+ choices=["brief", "standard", "detailed", "comprehensive"],
+ value="standard",
+ label="Detail Level",
+ info="Controls the depth and breadth of the report"
+ )
+ report_custom_model = gr.Dropdown(
+ choices=self.get_available_models(),
+ value=None,
+ label="Custom Model (Optional)",
+ info="Select a custom model for report generation"
+ )
+ report_process_thinking = gr.Checkbox(
+ label="Process Thinking Tags",
+ value=False,
+ info="Process tags in model output"
+ )
+ report_button = gr.Button("Generate Report", variant="primary")
+
+ gr.Examples(
+ examples=[
+ ["What are the latest advancements in quantum computing?"],
+ ["Compare transformer and RNN architectures for NLP tasks"],
+ ["Explain the environmental impact of electric vehicles"],
+ ["Explain the potential relationship between creatine supplementation and muscle loss due to GLP1-ar drugs for weight loss."]
+ ],
+ inputs=report_query_input
)
+
+ with gr.Row():
+ with gr.Column():
+ report_output = gr.Markdown(label="Generated Report")
+
+ with gr.Row():
+ with gr.Column():
+ report_file_output = gr.Textbox(
+ label="Report saved to file",
+ interactive=False
+ )
+
+ # Add information about detail levels
+ detail_levels_info = ""
+ for level, description in self.detail_level_manager.get_available_detail_levels():
+ detail_levels_info += f"- **{level}**: {description}\n"
+
+ gr.Markdown(f"### Detail Levels\n{detail_levels_info}")
+ # Set up event handlers
search_button.click(
fn=self.process_query,
- inputs=[query_input, num_results, use_reranker],
- outputs=[results_output, file_output]
+ inputs=[search_query_input, search_num_results, search_use_reranker],
+ outputs=[search_results_output, search_file_output]
+ )
+
+ report_button.click(
+ fn=lambda q, d, m, p, f: asyncio.run(self.generate_report(q, d, m, p, f)),
+ inputs=[report_query_input, report_detail_level, report_custom_model,
+ report_process_thinking, search_file_output],
+ outputs=[report_output, report_file_output]
)
return interface
-
+
def launch(self, **kwargs):
"""
Launch the Gradio interface.
@@ -248,7 +471,14 @@ class GradioInterface:
def main():
"""Main function to launch the Gradio interface."""
+ # Create interface and initialize async components
interface = GradioInterface()
+
+ # Run the async initialization in the event loop
+ loop = asyncio.get_event_loop()
+ loop.run_until_complete(interface.async_init())
+
+ # Launch the interface
interface.launch(share=True)