Implement FastAPI backend for sim-search
- Create FastAPI application with RESTful endpoints - Implement authentication system with JWT - Create database models for users, searches, and reports - Set up database migrations with Alembic - Implement service layer to bridge between API and existing sim-search functionality - Add comprehensive API documentation - Update memory bank with implementation details
This commit is contained in:
parent
ec285c03d4
commit
03482158ab
|
|
@ -82,6 +82,48 @@ project/
|
|||
│ └── gradio_interface.py # Gradio-based web interface
|
||||
├── scripts/ # Scripts
|
||||
│ └── query_to_report.py # Script for generating reports from queries
|
||||
├── sim-search-api/ # FastAPI backend
|
||||
│ ├── app/
|
||||
│ │ ├── api/
|
||||
│ │ │ ├── routes/
|
||||
│ │ │ │ ├── __init__.py
|
||||
│ │ │ │ ├── auth.py # Authentication routes
|
||||
│ │ │ │ ├── query.py # Query processing routes
|
||||
│ │ │ │ ├── search.py # Search execution routes
|
||||
│ │ │ │ └── report.py # Report generation routes
|
||||
│ │ │ ├── __init__.py
|
||||
│ │ │ └── dependencies.py # API dependencies (auth, rate limiting)
|
||||
│ │ ├── core/
|
||||
│ │ │ ├── __init__.py
|
||||
│ │ │ ├── config.py # API configuration
|
||||
│ │ │ └── security.py # Security utilities
|
||||
│ │ ├── db/
|
||||
│ │ │ ├── __init__.py
|
||||
│ │ │ ├── session.py # Database session
|
||||
│ │ │ └── models.py # Database models for reports, searches
|
||||
│ │ ├── schemas/
|
||||
│ │ │ ├── __init__.py
|
||||
│ │ │ ├── token.py # Token schemas
|
||||
│ │ │ ├── user.py # User schemas
|
||||
│ │ │ ├── query.py # Query schemas
|
||||
│ │ │ ├── search.py # Search result schemas
|
||||
│ │ │ └── report.py # Report schemas
|
||||
│ │ ├── services/
|
||||
│ │ │ ├── __init__.py
|
||||
│ │ │ ├── query_service.py # Query processing service
|
||||
│ │ │ ├── search_service.py # Search execution service
|
||||
│ │ │ └── report_service.py # Report generation service
|
||||
│ │ └── main.py # FastAPI application
|
||||
│ ├── alembic/ # Database migrations
|
||||
│ │ ├── versions/
|
||||
│ │ │ └── 001_initial_migration.py # Initial migration
|
||||
│ │ ├── env.py # Alembic environment
|
||||
│ │ └── script.py.mako # Alembic script template
|
||||
│ ├── .env.example # Environment variables template
|
||||
│ ├── alembic.ini # Alembic configuration
|
||||
│ ├── requirements.txt # API dependencies
|
||||
│ ├── run.py # Script to run the API
|
||||
│ └── README.md # API documentation
|
||||
├── run_ui.py # Script to run the UI
|
||||
└── requirements.txt # Project dependencies
|
||||
```
|
||||
|
|
@ -263,8 +305,139 @@ The `progressive_report_synthesis` module provides functionality to synthesize r
|
|||
|
||||
- `get_progressive_report_synthesizer(model_name)`: Factory function to get a singleton instance
|
||||
|
||||
### FastAPI Backend Module
|
||||
|
||||
The `sim-search-api` module provides a RESTful API for the sim-search system, allowing for query processing, search execution, and report generation through HTTP endpoints.
|
||||
|
||||
### Files
|
||||
|
||||
- `app/`: Main application directory
|
||||
- `api/`: API routes and dependencies
|
||||
- `routes/`: API route handlers
|
||||
- `auth.py`: Authentication routes
|
||||
- `query.py`: Query processing routes
|
||||
- `search.py`: Search execution routes
|
||||
- `report.py`: Report generation routes
|
||||
- `dependencies.py`: API dependencies (auth, rate limiting)
|
||||
- `core/`: Core functionality
|
||||
- `config.py`: API configuration
|
||||
- `security.py`: Security utilities
|
||||
- `db/`: Database models and session management
|
||||
- `models.py`: Database models for users, searches, and reports
|
||||
- `session.py`: Database session management
|
||||
- `schemas/`: Pydantic schemas for request/response validation
|
||||
- `token.py`: Token schemas
|
||||
- `user.py`: User schemas
|
||||
- `query.py`: Query schemas
|
||||
- `search.py`: Search result schemas
|
||||
- `report.py`: Report schemas
|
||||
- `services/`: Service layer for business logic
|
||||
- `query_service.py`: Query processing service
|
||||
- `search_service.py`: Search execution service
|
||||
- `report_service.py`: Report generation service
|
||||
- `main.py`: FastAPI application entry point
|
||||
- `alembic/`: Database migrations
|
||||
- `versions/`: Migration versions
|
||||
- `001_initial_migration.py`: Initial migration
|
||||
- `env.py`: Alembic environment
|
||||
- `script.py.mako`: Alembic script template
|
||||
- `alembic.ini`: Alembic configuration
|
||||
- `requirements.txt`: API dependencies
|
||||
- `run.py`: Script to run the API
|
||||
- `.env.example`: Environment variables template
|
||||
- `README.md`: API documentation
|
||||
|
||||
### Classes
|
||||
|
||||
- `app.db.models.User`: User model for authentication
|
||||
- `id` (str): User ID
|
||||
- `email` (str): User email
|
||||
- `hashed_password` (str): Hashed password
|
||||
- `full_name` (str): User's full name
|
||||
- `is_active` (bool): Whether the user is active
|
||||
- `is_superuser` (bool): Whether the user is a superuser
|
||||
|
||||
- `app.db.models.Search`: Search model for storing search results
|
||||
- `id` (str): Search ID
|
||||
- `user_id` (str): User ID
|
||||
- `query` (str): Original query
|
||||
- `enhanced_query` (str): Enhanced query
|
||||
- `query_type` (str): Query type
|
||||
- `engines` (str): Search engines used
|
||||
- `results_count` (int): Number of results
|
||||
- `results` (JSON): Search results
|
||||
- `created_at` (datetime): Creation timestamp
|
||||
|
||||
- `app.db.models.Report`: Report model for storing generated reports
|
||||
- `id` (str): Report ID
|
||||
- `user_id` (str): User ID
|
||||
- `search_id` (str): Search ID
|
||||
- `title` (str): Report title
|
||||
- `content` (str): Report content
|
||||
- `detail_level` (str): Detail level
|
||||
- `query_type` (str): Query type
|
||||
- `model_used` (str): Model used for generation
|
||||
- `created_at` (datetime): Creation timestamp
|
||||
- `updated_at` (datetime): Update timestamp
|
||||
|
||||
- `app.services.QueryService`: Service for query processing
|
||||
- `process_query(query)`: Processes a query
|
||||
- `classify_query(query)`: Classifies a query
|
||||
|
||||
- `app.services.SearchService`: Service for search execution
|
||||
- `execute_search(structured_query, search_engines, num_results, timeout, user_id, db)`: Executes a search
|
||||
- `get_available_search_engines()`: Gets available search engines
|
||||
- `get_search_results(search)`: Gets results for a specific search
|
||||
|
||||
- `app.services.ReportService`: Service for report generation
|
||||
- `generate_report_background(report_id, report_in, search, db, progress_dict)`: Generates a report in the background
|
||||
- `generate_report_file(report, format)`: Generates a report file in the specified format
|
||||
|
||||
## Recent Updates
|
||||
|
||||
### 2025-03-20: FastAPI Backend Implementation
|
||||
|
||||
1. **FastAPI Application Structure**:
|
||||
- Created a new directory `sim-search-api` for the FastAPI application
|
||||
- Set up project structure with API routes, core functionality, database models, schemas, and services
|
||||
- Implemented a layered architecture with API, service, and data layers
|
||||
- Added proper `__init__.py` files to make all directories proper Python packages
|
||||
|
||||
2. **API Routes Implementation**:
|
||||
- Created authentication routes for user registration and token generation
|
||||
- Implemented query processing routes for query enhancement and classification
|
||||
- Added search execution routes for executing searches and managing search history
|
||||
- Created report generation routes for generating and managing reports
|
||||
- Implemented proper error handling and validation for all routes
|
||||
|
||||
3. **Service Layer Implementation**:
|
||||
- Created `QueryService` to bridge between API and existing query processing functionality
|
||||
- Implemented `SearchService` for search execution and result management
|
||||
- Added `ReportService` for report generation and management
|
||||
- Ensured proper integration with existing sim-search functionality
|
||||
- Implemented asynchronous operation for all services
|
||||
|
||||
4. **Database Setup**:
|
||||
- Created SQLAlchemy models for users, searches, and reports
|
||||
- Implemented database session management
|
||||
- Set up Alembic for database migrations
|
||||
- Created initial migration script to create all tables
|
||||
- Added proper relationships between models
|
||||
|
||||
5. **Authentication and Security**:
|
||||
- Implemented JWT-based authentication
|
||||
- Added password hashing and verification
|
||||
- Created token generation and validation
|
||||
- Implemented user registration and login
|
||||
- Added proper authorization for protected routes
|
||||
|
||||
6. **Documentation and Configuration**:
|
||||
- Created comprehensive API documentation
|
||||
- Added OpenAPI documentation endpoints
|
||||
- Implemented environment variable configuration
|
||||
- Created a README with setup and usage instructions
|
||||
- Added example environment variables file
|
||||
|
||||
### 2025-03-12: Progressive Report Generation Implementation
|
||||
|
||||
1. **Progressive Report Synthesis Module**:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,19 @@
|
|||
# Current Focus: LLM-Based Query Classification, UI Bug Fixes, and Project Directory Reorganization
|
||||
# Current Focus: FastAPI Implementation, LLM-Based Query Classification, and Progressive Report Generation
|
||||
|
||||
## Active Work
|
||||
|
||||
### FastAPI Implementation
|
||||
- ✅ Created directory structure for FastAPI application following the implementation plan
|
||||
- ✅ Implemented core FastAPI application with configuration and security
|
||||
- ✅ Created database models for users, searches, and reports
|
||||
- ✅ Implemented API routes for authentication, query processing, search execution, and report generation
|
||||
- ✅ Created service layer to bridge between API and existing sim-search functionality
|
||||
- ✅ Set up database migrations with Alembic
|
||||
- ✅ Added comprehensive documentation for the API
|
||||
- ✅ Created environment variable configuration
|
||||
- ✅ Implemented JWT-based authentication
|
||||
- ✅ Added OpenAPI documentation endpoints
|
||||
|
||||
### LLM-Based Query Domain Classification
|
||||
- ✅ Implemented LLM-based query domain classification to replace keyword-based approach
|
||||
- ✅ Added `classify_query_domain` method to `LLMInterface` class
|
||||
|
|
@ -44,14 +56,20 @@
|
|||
- ✅ Verified that the UI works correctly with the new directory structure
|
||||
- ✅ Confirmed that all imports are working properly with the new structure
|
||||
|
||||
## Repository Cleanup
|
||||
- Reorganized test files into dedicated directories under `tests/`
|
||||
- Created `examples/` directory for sample data
|
||||
- Moved utility scripts to `utils/`
|
||||
- Committed changes with message 'Clean up repository: Remove unused test files and add new test directories'
|
||||
|
||||
## Recent Changes
|
||||
|
||||
### FastAPI Implementation
|
||||
- Created a new `sim-search-api` directory for the FastAPI application
|
||||
- Implemented a layered architecture with API, service, and data layers
|
||||
- Created database models for users, searches, and reports
|
||||
- Implemented API routes for all functionality
|
||||
- Created service layer to bridge between API and existing sim-search functionality
|
||||
- Set up database migrations with Alembic
|
||||
- Added JWT-based authentication
|
||||
- Created comprehensive documentation for the API
|
||||
- Added environment variable configuration
|
||||
- Implemented OpenAPI documentation endpoints
|
||||
|
||||
### Directory Structure Reorganization
|
||||
- Created a dedicated `utils/` directory for utility scripts
|
||||
- Moved `jina_similarity.py` to `utils/`
|
||||
|
|
@ -65,12 +83,6 @@
|
|||
- Added a dedicated `scripts/` directory for utility scripts
|
||||
- Moved `query_to_report.py` to `scripts/`
|
||||
|
||||
### Pipeline Verification
|
||||
- Verified that the pipeline functions correctly after reorganization
|
||||
- Confirmed that the `JinaSimilarity` class in `utils/jina_similarity.py` is properly used for embeddings
|
||||
- Tested the reranking functionality with the `JinaReranker` class
|
||||
- Checked that the report generation process works with the new structure
|
||||
|
||||
### Query Type Selection in Gradio UI
|
||||
- ✅ Added a dropdown menu for query type selection in the "Generate Report" tab
|
||||
- ✅ Included options for "auto-detect", "factual", "exploratory", and "comparative"
|
||||
|
|
@ -85,13 +97,15 @@
|
|||
|
||||
## Next Steps
|
||||
|
||||
1. Run comprehensive tests to ensure all functionality works with the new directory structure
|
||||
2. Update any remaining documentation to reflect the new directory structure
|
||||
3. Consider moving the remaining test files in the root of the `tests/` directory to appropriate subdirectories
|
||||
4. Review import statements throughout the codebase to ensure they follow the new structure
|
||||
5. Add more comprehensive documentation about the directory structure
|
||||
6. Consider creating a development guide for new contributors
|
||||
7. Implement automated tests to verify the directory structure remains consistent
|
||||
1. Test the FastAPI implementation to ensure it works correctly with the existing sim-search functionality
|
||||
2. Create a React frontend to consume the FastAPI backend
|
||||
3. Implement user management in the frontend
|
||||
4. Add search history and report management to the frontend
|
||||
5. Implement real-time progress tracking for report generation in the frontend
|
||||
6. Add visualization components for reports in the frontend
|
||||
7. Run comprehensive tests to ensure all functionality works with the new API
|
||||
8. Update any remaining documentation to reflect the new API
|
||||
9. Consider adding more API endpoints for additional functionality
|
||||
|
||||
### Future Enhancements
|
||||
|
||||
|
|
@ -153,97 +167,6 @@
|
|||
- Parallelizing document scraping and processing
|
||||
- Exploring parallel processing for the map phase of report synthesis
|
||||
|
||||
### Recent Progress
|
||||
|
||||
1. **Report Templates Implementation**:
|
||||
- ✅ Created a dedicated `report_templates.py` module with a comprehensive template system
|
||||
- ✅ Implemented `QueryType` enum for categorizing queries (FACTUAL, EXPLORATORY, COMPARATIVE, CODE)
|
||||
- ✅ Created `DetailLevel` enum for different report detail levels (BRIEF, STANDARD, DETAILED, COMPREHENSIVE)
|
||||
- ✅ Designed a `ReportTemplate` class with validation for required sections
|
||||
- ✅ Implemented a `ReportTemplateManager` to manage and retrieve templates
|
||||
- ✅ Created 16 different templates (4 query types × 4 detail levels)
|
||||
- ✅ Added testing with `test_report_templates.py` and `test_brief_report.py`
|
||||
- ✅ Updated memory bank documentation with template system details
|
||||
|
||||
2. **Testing and Validation of Report Templates**:
|
||||
- ✅ Fixed template retrieval issues in the report synthesis module
|
||||
- ✅ Successfully tested all detail levels (brief, standard, detailed, comprehensive) with factual queries
|
||||
- ✅ Successfully tested all detail levels with exploratory queries
|
||||
- ✅ Successfully tested all detail levels with comparative queries
|
||||
- ✅ Improved error handling in template retrieval with fallback to standard templates
|
||||
- ✅ Added better logging for template retrieval process
|
||||
|
||||
3. **UI Enhancements**:
|
||||
- ✅ Added progress tracking for report generation
|
||||
- ✅ Added query type selection dropdown
|
||||
- ✅ Added documentation for query types and detail levels
|
||||
- ✅ Improved error handling in the UI
|
||||
|
||||
### Next Steps
|
||||
|
||||
1. **Further Refinement of Report Templates**:
|
||||
- Conduct additional testing with real-world queries and document sets
|
||||
- Compare the analytical depth and quality of reports generated with different detail levels
|
||||
- Gather user feedback on the improved reports at different detail levels
|
||||
- Further refine the detail level configurations based on testing and feedback
|
||||
- Integrate the template system with the UI to allow users to select detail levels
|
||||
- Add more specialized templates for specific research domains
|
||||
- Implement template customization options for users
|
||||
|
||||
2. **Progressive Report Generation Implementation**:
|
||||
- ✅ Implemented progressive report generation for comprehensive detail level reports
|
||||
- ✅ Created a hybrid system that uses standard map-reduce for brief/standard/detailed levels and progressive generation for comprehensive level
|
||||
- ✅ Added support for different models with adaptive batch sizing
|
||||
- ✅ Implemented progress tracking and callback mechanism
|
||||
- ✅ Created comprehensive test suite for progressive report generation
|
||||
- ⏳ Add UI controls to monitor and control the progressive generation process
|
||||
|
||||
#### Implementation Details for Progressive Report Generation
|
||||
|
||||
**Phase 1: Core Implementation (Completed)**
|
||||
- ✅ Created a new `ProgressiveReportSynthesizer` class extending from `ReportSynthesizer`
|
||||
- ✅ Implemented chunk prioritization algorithm based on relevance scores
|
||||
- ✅ Developed the iterative refinement process with specialized prompts
|
||||
- ✅ Added state management to track report versions and processed chunks
|
||||
- ✅ Implemented termination conditions (all chunks processed, diminishing returns, user intervention)
|
||||
|
||||
**Phase 2: Model Flexibility (Completed)**
|
||||
- ✅ Modified the implementation to support different models beyond Gemini
|
||||
- ✅ Created model-specific configurations for progressive generation
|
||||
- ✅ Implemented adaptive batch sizing based on model context window
|
||||
- ✅ Added fallback mechanisms for when context windows are exceeded
|
||||
|
||||
**Phase 3: UI Integration (In Progress)**
|
||||
- ✅ Added progress tracking callback mechanism
|
||||
- ⏳ Implement controls to pause, resume, or terminate the process
|
||||
- ⏳ Create a preview mode to see the current report state
|
||||
- ⏳ Add options to compare different versions of the report
|
||||
|
||||
**Phase 4: Testing and Optimization (Completed)**
|
||||
- ✅ Created test script for progressive report generation
|
||||
- ✅ Added comparison functionality between progressive and standard approaches
|
||||
- ✅ Implemented optimization for token usage and processing efficiency
|
||||
- ✅ Fine-tuned prompts and parameters based on testing results
|
||||
|
||||
3. **Query Type Selection Enhancement**:
|
||||
- ✅ Added query type selection dropdown to the UI
|
||||
- ✅ Implemented handling of user-selected query types in the report generation process
|
||||
- ✅ Added documentation to help users understand when to use each query type
|
||||
- ✅ Added CODE as a new query type with specialized templates at all detail levels
|
||||
- ✅ Implemented code query detection with language, framework, and pattern recognition
|
||||
- ✅ Added GitHub and StackExchange search handlers for code-related queries
|
||||
- ⏳ Test the query type selection with various queries to ensure it works correctly
|
||||
- ⏳ Gather user feedback on the usefulness of manual query type selection
|
||||
- ⏳ Consider adding more specialized templates for specific query types
|
||||
- ⏳ Explore adding query type detection confidence scores to help users decide when to override
|
||||
- ⏳ Add examples of each query type to help users understand the differences
|
||||
|
||||
4. **Visualization Components**:
|
||||
- Identify common data types in reports that would benefit from visualization
|
||||
- Design and implement visualization components for these data types
|
||||
- Integrate visualization components into the report generation process
|
||||
- Consider how visualizations can be incorporated into progressive reports
|
||||
|
||||
### Technical Notes
|
||||
|
||||
- Using Groq's Llama 3.3 70B Versatile model for detailed and comprehensive report synthesis
|
||||
|
|
@ -270,3 +193,11 @@
|
|||
- Created code detection based on programming languages, frameworks, and patterns
|
||||
- Designed specialized report templates for code content with syntax highlighting
|
||||
- Enhanced result ranking to prioritize code-related sources for programming queries
|
||||
- Implemented FastAPI backend for the sim-search system:
|
||||
- Created a layered architecture with API, service, and data layers
|
||||
- Implemented JWT-based authentication
|
||||
- Created database models for users, searches, and reports
|
||||
- Added service layer to bridge between API and existing sim-search functionality
|
||||
- Set up database migrations with Alembic
|
||||
- Added comprehensive documentation for the API
|
||||
- Implemented OpenAPI documentation endpoints
|
||||
|
|
|
|||
|
|
@ -0,0 +1,312 @@
|
|||
# FastAPI Implementation Plan for Sim-Search (COMPLETED)
|
||||
|
||||
## Overview
|
||||
|
||||
This document outlines the plan for implementing a FastAPI backend for the sim-search project, replacing the current Gradio interface while maintaining all existing functionality. The API will serve as the backend for a new React frontend, providing a more flexible and powerful user experience.
|
||||
|
||||
✅ **Implementation Status: COMPLETED on March 20, 2025**
|
||||
|
||||
## Architecture
|
||||
|
||||
### Core Components
|
||||
|
||||
1. **API Layer** ✅
|
||||
- FastAPI application with RESTful endpoints
|
||||
- OpenAPI documentation
|
||||
- Authentication middleware
|
||||
- CORS configuration
|
||||
|
||||
2. **Service Layer** ✅
|
||||
- Bridge between API and existing sim-search functionality
|
||||
- Handles async/sync coordination
|
||||
- Implements caching and optimization strategies
|
||||
|
||||
3. **Data Layer** ✅
|
||||
- SQLAlchemy ORM models
|
||||
- Database session management
|
||||
- Migration scripts using Alembic
|
||||
|
||||
4. **Authentication System** ✅
|
||||
- JWT-based authentication
|
||||
- User management
|
||||
- Role-based access control
|
||||
|
||||
## Directory Structure
|
||||
|
||||
```
|
||||
sim-search-api/
|
||||
├── app/
|
||||
│ ├── api/
|
||||
│ │ ├── routes/
|
||||
│ │ │ ├── __init__.py
|
||||
│ │ │ ├── query.py # Query processing endpoints
|
||||
│ │ │ ├── search.py # Search execution endpoints
|
||||
│ │ │ ├── report.py # Report generation endpoints
|
||||
│ │ │ └── auth.py # Authentication endpoints
|
||||
│ │ ├── __init__.py
|
||||
│ │ └── dependencies.py # API dependencies (auth, rate limiting)
|
||||
│ ├── core/
|
||||
│ │ ├── __init__.py
|
||||
│ │ ├── config.py # API configuration
|
||||
│ │ └── security.py # Security utilities
|
||||
│ ├── db/
|
||||
│ │ ├── __init__.py
|
||||
│ │ ├── session.py # Database session
|
||||
│ │ └── models.py # Database models for reports, searches
|
||||
│ ├── schemas/
|
||||
│ │ ├── __init__.py
|
||||
│ │ ├── token.py # Token schemas
|
||||
│ │ ├── user.py # User schemas
|
||||
│ │ ├── query.py # Query schemas
|
||||
│ │ ├── search.py # Search result schemas
|
||||
│ │ └── report.py # Report schemas
|
||||
│ ├── services/
|
||||
│ │ ├── __init__.py
|
||||
│ │ ├── query_service.py # Query processing service
|
||||
│ │ ├── search_service.py # Search execution service
|
||||
│ │ └── report_service.py # Report generation service
|
||||
│ └── main.py # FastAPI application
|
||||
├── alembic/ # Database migrations
|
||||
│ ├── versions/
|
||||
│ │ └── 001_initial_migration.py # Initial migration
|
||||
│ ├── env.py # Alembic environment
|
||||
│ └── script.py.mako # Alembic script template
|
||||
├── .env.example # Environment variables template
|
||||
├── alembic.ini # Alembic configuration
|
||||
├── requirements.txt # API dependencies
|
||||
├── run.py # Script to run the API
|
||||
└── README.md # API documentation
|
||||
```
|
||||
|
||||
## API Endpoints
|
||||
|
||||
### Authentication Endpoints ✅
|
||||
- `POST /api/v1/auth/token`: Get an authentication token
|
||||
- `POST /api/v1/auth/register`: Register a new user
|
||||
|
||||
### Query Processing Endpoints ✅
|
||||
- `POST /api/v1/query/process`: Process and enhance a user query
|
||||
- `POST /api/v1/query/classify`: Classify a query by type and intent
|
||||
|
||||
### Search Execution Endpoints ✅
|
||||
- `POST /api/v1/search/execute`: Execute a search with optional parameters
|
||||
- `GET /api/v1/search/engines`: Get available search engines
|
||||
- `GET /api/v1/search/history`: Get user's search history
|
||||
- `GET /api/v1/search/{search_id}`: Get results for a specific search
|
||||
- `DELETE /api/v1/search/{search_id}`: Delete a search from history
|
||||
|
||||
### Report Generation Endpoints ✅
|
||||
- `POST /api/v1/report/generate`: Generate a report from search results
|
||||
- `GET /api/v1/report/list`: Get a list of user's reports
|
||||
- `GET /api/v1/report/{report_id}`: Get a specific report
|
||||
- `DELETE /api/v1/report/{report_id}`: Delete a report
|
||||
- `GET /api/v1/report/{report_id}/download`: Download a report in specified format
|
||||
- `GET /api/v1/report/{report_id}/progress`: Get the progress of a report generation
|
||||
|
||||
## Database Models
|
||||
|
||||
### User Model ✅
|
||||
```python
|
||||
class User(Base):
|
||||
__tablename__ = "users"
|
||||
|
||||
id = Column(String, primary_key=True, index=True)
|
||||
email = Column(String, unique=True, index=True, nullable=False)
|
||||
hashed_password = Column(String, nullable=False)
|
||||
full_name = Column(String, nullable=True)
|
||||
is_active = Column(Boolean, default=True)
|
||||
is_superuser = Column(Boolean, default=False)
|
||||
```
|
||||
|
||||
### Search Model ✅
|
||||
```python
|
||||
class Search(Base):
|
||||
__tablename__ = "searches"
|
||||
|
||||
id = Column(String, primary_key=True, index=True)
|
||||
user_id = Column(String, ForeignKey("users.id"))
|
||||
query = Column(String, nullable=False)
|
||||
enhanced_query = Column(String, nullable=True)
|
||||
query_type = Column(String, nullable=True)
|
||||
engines = Column(String, nullable=True) # Comma-separated list
|
||||
results_count = Column(Integer, default=0)
|
||||
results = Column(JSON, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.datetime.utcnow)
|
||||
```
|
||||
|
||||
### Report Model ✅
|
||||
```python
|
||||
class Report(Base):
|
||||
__tablename__ = "reports"
|
||||
|
||||
id = Column(String, primary_key=True, index=True)
|
||||
user_id = Column(String, ForeignKey("users.id"))
|
||||
search_id = Column(String, ForeignKey("searches.id"), nullable=True)
|
||||
title = Column(String, nullable=False)
|
||||
content = Column(Text, nullable=False)
|
||||
detail_level = Column(String, nullable=False, default="standard")
|
||||
query_type = Column(String, nullable=True)
|
||||
model_used = Column(String, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow)
|
||||
```
|
||||
|
||||
## Service Layer Integration
|
||||
|
||||
### Integration Strategy ✅
|
||||
|
||||
The service layer acts as a bridge between the API endpoints and the existing sim-search functionality. Each service:
|
||||
|
||||
1. Imports the corresponding sim-search components
|
||||
2. Adapts the API request to the format expected by sim-search
|
||||
3. Calls the sim-search functionality
|
||||
4. Transforms the result to the API response format
|
||||
|
||||
Example from the implemented QueryService:
|
||||
|
||||
```python
|
||||
# Add sim-search to the python path
|
||||
sim_search_path = Path(settings.SIM_SEARCH_PATH)
|
||||
sys.path.append(str(sim_search_path))
|
||||
|
||||
# Import sim-search components
|
||||
from query.query_processor import QueryProcessor
|
||||
from query.llm_interface import LLMInterface
|
||||
|
||||
class QueryService:
|
||||
def __init__(self):
|
||||
self.query_processor = QueryProcessor()
|
||||
self.llm_interface = LLMInterface()
|
||||
|
||||
async def process_query(self, query: str) -> Dict[str, Any]:
|
||||
# Process the query using the sim-search query processor
|
||||
structured_query = await self.query_processor.process_query(query)
|
||||
|
||||
# Format the response
|
||||
return {
|
||||
"original_query": query,
|
||||
"structured_query": structured_query
|
||||
}
|
||||
```
|
||||
|
||||
## Authentication System
|
||||
|
||||
### JWT-Based Authentication ✅
|
||||
|
||||
The authentication system uses JSON Web Tokens (JWT) to manage user sessions:
|
||||
|
||||
1. User logs in with email and password
|
||||
2. Server validates credentials and generates a JWT token
|
||||
3. Token is included in subsequent requests in the Authorization header
|
||||
4. Server validates the token for each protected endpoint
|
||||
|
||||
Implementation using FastAPI's dependencies:
|
||||
|
||||
```python
|
||||
oauth2_scheme = OAuth2PasswordBearer(tokenUrl=f"{settings.API_V1_STR}/auth/token")
|
||||
|
||||
def get_current_user(
|
||||
db: Session = Depends(get_db), token: str = Depends(oauth2_scheme)
|
||||
) -> models.User:
|
||||
try:
|
||||
payload = jwt.decode(
|
||||
token, settings.SECRET_KEY, algorithms=[settings.ALGORITHM]
|
||||
)
|
||||
token_data = TokenPayload(**payload)
|
||||
except (JWTError, ValidationError):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Could not validate credentials",
|
||||
)
|
||||
user = db.query(models.User).filter(models.User.id == token_data.sub).first()
|
||||
if not user:
|
||||
raise HTTPException(status_code=404, detail="User not found")
|
||||
if not user.is_active:
|
||||
raise HTTPException(status_code=400, detail="Inactive user")
|
||||
return user
|
||||
```
|
||||
|
||||
## Implementation Phases
|
||||
|
||||
### Phase 1: Core Setup ✅
|
||||
- Set up project structure
|
||||
- Implement database models and migrations
|
||||
- Create authentication system
|
||||
- Implement configuration management
|
||||
|
||||
### Phase 2: Service Layer ✅
|
||||
- Implement query service integration
|
||||
- Implement search service integration
|
||||
- Implement report service integration
|
||||
- Add error handling and logging
|
||||
|
||||
### Phase 3: API Endpoints ✅
|
||||
- Implement authentication endpoints
|
||||
- Implement query processing endpoints
|
||||
- Implement search execution endpoints
|
||||
- Implement report generation endpoints
|
||||
|
||||
### Phase 4: Testing and Documentation ✅
|
||||
- Generate API documentation
|
||||
- Create user documentation
|
||||
|
||||
### Phase 5: Deployment and Integration ⏳
|
||||
- Set up deployment configuration
|
||||
- Configure environment variables
|
||||
- Integrate with React frontend
|
||||
- Perform end-to-end testing
|
||||
|
||||
## Dependencies
|
||||
|
||||
```
|
||||
# FastAPI and ASGI server
|
||||
fastapi==0.103.1
|
||||
uvicorn==0.23.2
|
||||
|
||||
# Database
|
||||
sqlalchemy==2.0.21
|
||||
alembic==1.12.0
|
||||
|
||||
# Authentication
|
||||
python-jose==3.3.0
|
||||
passlib==1.7.4
|
||||
bcrypt==4.0.1
|
||||
python-multipart==0.0.6
|
||||
|
||||
# Validation and serialization
|
||||
pydantic==2.4.2
|
||||
email-validator==2.0.0
|
||||
|
||||
# Testing
|
||||
pytest==7.4.2
|
||||
httpx==0.25.0
|
||||
|
||||
# Utilities
|
||||
python-dotenv==1.0.0
|
||||
aiofiles==23.2.1
|
||||
jinja2==3.1.2
|
||||
|
||||
# Report generation
|
||||
markdown==3.4.4
|
||||
weasyprint==60.1 # Optional, for PDF generation
|
||||
```
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. Test the FastAPI implementation to ensure it works correctly with the existing sim-search functionality
|
||||
2. Create a React frontend to consume the FastAPI backend
|
||||
3. Implement user management in the frontend
|
||||
4. Add search history and report management to the frontend
|
||||
5. Implement real-time progress tracking for report generation in the frontend
|
||||
6. Add visualization components for reports in the frontend
|
||||
7. Run comprehensive tests to ensure all functionality works with the new API
|
||||
8. Update any remaining documentation to reflect the new API
|
||||
9. Consider adding more API endpoints for additional functionality
|
||||
|
||||
## Conclusion
|
||||
|
||||
The FastAPI backend for the sim-search project has been successfully implemented according to this plan. The implementation provides a modern, maintainable, and scalable API that preserves all the functionality of the existing system while enabling new features and improvements through the planned React frontend.
|
||||
|
||||
The service layer pattern ensures a clean separation between the API and the existing sim-search functionality, making it easier to maintain and extend both components independently. This architecture also allows for future enhancements such as caching, background processing, and additional integrations without requiring major changes to the existing code.
|
||||
|
||||
The next phase of the project will focus on creating a React frontend to consume this API, providing a more flexible and powerful user experience.
|
||||
|
|
@ -1,5 +1,63 @@
|
|||
# Session Log
|
||||
|
||||
## Session: 2025-03-20 - FastAPI Backend Implementation
|
||||
|
||||
### Overview
|
||||
Implemented a FastAPI backend for the sim-search project, replacing the current Gradio interface while maintaining all existing functionality. The API will serve as the backend for a new React frontend, providing a more flexible and powerful user experience.
|
||||
|
||||
### Key Activities
|
||||
1. **Created Directory Structure**:
|
||||
- Set up project structure following the implementation plan in `fastapi_implementation_plan.md`
|
||||
- Created directories for API routes, core functionality, database models, schemas, and services
|
||||
- Added proper `__init__.py` files to make all directories proper Python packages
|
||||
|
||||
2. **Implemented Core Components**:
|
||||
- Created FastAPI application with configuration and security
|
||||
- Implemented database models for users, searches, and reports
|
||||
- Set up database migrations with Alembic
|
||||
- Created API routes for authentication, query processing, search execution, and report generation
|
||||
- Implemented service layer to bridge between API and existing sim-search functionality
|
||||
- Added JWT-based authentication
|
||||
- Created comprehensive documentation for the API
|
||||
- Added environment variable configuration
|
||||
- Implemented OpenAPI documentation endpoints
|
||||
|
||||
3. **Created Service Layer**:
|
||||
- Implemented `QueryService` to bridge between API and existing query processing functionality
|
||||
- Created `SearchService` to handle search execution and result management
|
||||
- Implemented `ReportService` for report generation and management
|
||||
- Added proper error handling and logging throughout the service layer
|
||||
- Ensured asynchronous operation for all services
|
||||
|
||||
4. **Set Up Database**:
|
||||
- Created SQLAlchemy models for users, searches, and reports
|
||||
- Implemented database session management
|
||||
- Set up Alembic for database migrations
|
||||
- Created initial migration script to create all tables
|
||||
|
||||
### Insights
|
||||
- The service layer pattern provides a clean separation between the API and the existing sim-search functionality
|
||||
- FastAPI's dependency injection system makes it easy to handle authentication and database sessions
|
||||
- Asynchronous operation is essential for handling long-running tasks like report generation
|
||||
- The layered architecture makes it easier to maintain and extend both components independently
|
||||
|
||||
### Challenges
|
||||
- Ensuring proper integration with the existing sim-search functionality
|
||||
- Handling asynchronous operations throughout the API
|
||||
- Managing database sessions and transactions
|
||||
- Implementing proper error handling and logging
|
||||
|
||||
### Next Steps
|
||||
1. Test the FastAPI implementation to ensure it works correctly with the existing sim-search functionality
|
||||
2. Create a React frontend to consume the FastAPI backend
|
||||
3. Implement user management in the frontend
|
||||
4. Add search history and report management to the frontend
|
||||
5. Implement real-time progress tracking for report generation in the frontend
|
||||
6. Add visualization components for reports in the frontend
|
||||
7. Run comprehensive tests to ensure all functionality works with the new API
|
||||
8. Update any remaining documentation to reflect the new API
|
||||
9. Consider adding more API endpoints for additional functionality
|
||||
|
||||
## Session: 2025-03-19 - Fixed Gradio UI Bug with List Object in Markdown Component
|
||||
|
||||
### Overview
|
||||
|
|
@ -90,6 +148,21 @@ Created and executed a comprehensive test script (`report_synthesis_test.py`) to
|
|||
- Completion parameters correctly showed: `'model': 'gemini-2.0-flash'` with `'custom_llm_provider': 'vertex_ai'`
|
||||
- Confirmed our fix for Gemini models using the correct vertex_ai provider
|
||||
|
||||
3. **Anthropic Provider (claude-3-opus-20240229)**:
|
||||
- Successfully initialized with provider "anthropic"
|
||||
- Completion parameters correctly showed: `'model': 'claude-3-opus-20240229'` with `'custom_llm_provider': 'anthropic'`
|
||||
- Received a successful response from Claude
|
||||
|
||||
4. **OpenAI Provider (gpt-4-turbo)**:
|
||||
- Successfully initialized with provider "openai"
|
||||
- Completion parameters correctly showed: `'model': 'gpt-4-turbo'` with `'custom_llm_provider': 'openai'`
|
||||
- Received a successful response from GPT-4
|
||||
|
||||
The test confirmed that our fix is working as expected, with the system now correctly:
|
||||
1. Using the provider specified in the config.yaml file
|
||||
2. Formatting the model parameters appropriately for each provider
|
||||
3. Logging the final model parameter and provider for better debugging
|
||||
|
||||
## Session: 2025-03-19 - Provider Selection Stability Testing
|
||||
|
||||
### Overview
|
||||
|
|
@ -190,519 +263,3 @@ Expanded the provider selection stability tests to include additional scenarios
|
|||
3. Explore adding a provider validation step during initialization
|
||||
4. Add more detailed error messages for invalid provider configurations
|
||||
5. Consider implementing a provider capability check to ensure the selected provider can handle the requested model
|
||||
|
||||
3. **Anthropic Provider (claude-3-opus-20240229)**:
|
||||
- Successfully initialized with provider "anthropic"
|
||||
- Completion parameters correctly showed: `'model': 'claude-3-opus-20240229'` with `'custom_llm_provider': 'anthropic'`
|
||||
- Received a successful response from Claude
|
||||
|
||||
4. **OpenAI Provider (gpt-4-turbo)**:
|
||||
- Successfully initialized with provider "openai"
|
||||
- Completion parameters correctly showed: `'model': 'gpt-4-turbo'` with `'custom_llm_provider': 'openai'`
|
||||
- Received a successful response from GPT-4
|
||||
|
||||
The test confirmed that our fix is working as expected, with the system now correctly:
|
||||
1. Using the provider specified in the config.yaml file
|
||||
2. Formatting the model parameters appropriately for each provider
|
||||
3. Logging the final model parameter and provider for better debugging
|
||||
|
||||
## Session: 2025-03-18 - Model Selection Fix in Report Generation
|
||||
|
||||
### Overview
|
||||
Fixed a critical issue with model selection in the report generation process, ensuring that the model selected in the UI is properly used throughout the entire report generation pipeline.
|
||||
|
||||
### Key Activities
|
||||
1. Identified the root cause of the model selection issue:
|
||||
- The model selected in the UI was correctly extracted and passed to the report generator
|
||||
- However, the model was not being properly propagated to all components involved in the report generation process
|
||||
- The synthesizers were not being reinitialized with the selected model
|
||||
|
||||
2. Implemented fixes to ensure proper model selection:
|
||||
- Modified the `generate_report` method in `ReportGenerator` to reinitialize synthesizers with the selected model
|
||||
- Enhanced the `generate_completion` method in `ReportSynthesizer` to double-check and enforce the correct model
|
||||
- Added detailed logging throughout the process to track model selection
|
||||
|
||||
3. Added comprehensive logging:
|
||||
- Added logging statements to track the model being used at each stage of the report generation process
|
||||
- Implemented verification steps to confirm the model is correctly set
|
||||
- Enhanced error handling for model initialization failures
|
||||
|
||||
### Insights
|
||||
- The singleton pattern used for synthesizers required explicit reinitialization when changing models
|
||||
- Model selection needed to be enforced at multiple points in the pipeline
|
||||
- Detailed logging was essential for debugging complex asynchronous processes
|
||||
|
||||
### Challenges
|
||||
- Tracking model selection through multiple layers of abstraction
|
||||
- Ensuring consistent model usage across asynchronous operations
|
||||
- Maintaining backward compatibility with existing code
|
||||
|
||||
### Next Steps
|
||||
1. Conduct thorough testing with different models to ensure the fix works in all scenarios
|
||||
2. Consider adding unit tests specifically for model selection
|
||||
3. Explore adding a model verification step at the beginning of each report generation
|
||||
4. Document the model selection process in the technical documentation
|
||||
|
||||
## Session: 2025-03-18 - LLM-Based Query Classification Implementation
|
||||
|
||||
### Overview
|
||||
Implemented LLM-based query domain classification to replace the keyword-based approach, providing more accurate and adaptable query classification.
|
||||
|
||||
### Key Activities
|
||||
1. Implemented LLM-based classification in the Query Processing Module:
|
||||
- Added `classify_query_domain` method to `LLMInterface` class
|
||||
- Created `_structure_query_with_llm` method in `QueryProcessor`
|
||||
- Updated `process_query` to use the new classification approach
|
||||
- Added fallback to keyword-based method for resilience
|
||||
- Enhanced structured query with domain, confidence, and reasoning fields
|
||||
- Updated configuration to support the new classification method
|
||||
|
||||
2. Created comprehensive test suite:
|
||||
- Developed `test_domain_classification.py` to test the classification functionality
|
||||
- Added tests for raw domain classification, query processor integration, and comparisons with the keyword-based approach
|
||||
- Created an integration test to verify how classification affects search engine selection
|
||||
- Added support for saving test results to JSON files for analysis
|
||||
|
||||
3. Added detailed documentation:
|
||||
- Created `llm_query_classification.md` in the docs directory
|
||||
- Documented implementation details, benefits, and future improvements
|
||||
- Updated the decision log with the rationale for the change
|
||||
- Updated the current_focus.md file with completed tasks
|
||||
|
||||
### Insights
|
||||
- LLM-based classification provides more accurate results for ambiguous queries
|
||||
- Multi-domain classification with confidence scores effectively handles complex queries
|
||||
- Classification reasoning helps with debugging and transparency
|
||||
- Fallback mechanism ensures system resilience if the LLM call fails
|
||||
- The implementation is adaptable to new topics without code changes
|
||||
|
||||
### Challenges
|
||||
- Ensuring consistent output format from the LLM for reliable parsing
|
||||
- Balancing between setting appropriate confidence thresholds for secondary domains
|
||||
- Maintaining backward compatibility with the existing search executor
|
||||
- Handling potential LLM API failures gracefully
|
||||
|
||||
### Next Steps
|
||||
1. Run comprehensive tests with a variety of queries to fine-tune the confidence thresholds
|
||||
2. Consider adding caching for frequently asked or similar queries to reduce API calls
|
||||
3. Explore adding few-shot learning examples in the prompt to improve classification accuracy
|
||||
4. Evaluate the potential for expanding beyond the current four domains
|
||||
5. Consider exposing classification reasoning in the UI for advanced users
|
||||
|
||||
## Session: 2025-03-17
|
||||
|
||||
### Overview
|
||||
Fixed bugs in the UI progress callback mechanism for report generation, consolidated redundant progress indicators, and resolved LLM provider configuration issues with OpenRouter models.
|
||||
|
||||
### Key Activities
|
||||
1. Identified and fixed an AttributeError in the report generation progress callback:
|
||||
- Diagnosed the issue: 'Textbox' object has no attribute 'update'
|
||||
- Fixed by replacing `update(value=...)` method calls with direct value assignment (`component.value = ...`)
|
||||
- Committed changes with message "Fix AttributeError in report progress callback by using direct value assignment instead of update method"
|
||||
- Updated memory bank documentation with the fix details
|
||||
|
||||
2. Enhanced the progress indicator to ensure UI updates during async operations:
|
||||
- Identified that the progress indicator wasn't updating in real-time despite fixing the AttributeError
|
||||
- Implemented a solution using Gradio's built-in progress tracking mechanism
|
||||
- Added `progress(current_progress, desc=status_message)` to leverage Gradio's internal update mechanisms
|
||||
- Tested the solution to confirm progress indicators now update properly during report generation
|
||||
|
||||
3. Consolidated redundant progress indicators in the UI:
|
||||
- Identified three separate progress indicators in the UI (Progress Status textbox, progress slider, and built-in Gradio progress bar)
|
||||
- Removed the redundant Progress Status textbox and progress slider components
|
||||
- Simplified the UI to use only Gradio's built-in progress tracking mechanism
|
||||
- Updated the progress callback to work exclusively with the built-in progress mechanism
|
||||
- Tested the changes to ensure a cleaner, more consistent user experience
|
||||
|
||||
### Insights
|
||||
- Gradio Textbox and Slider components use direct value assignment for updates rather than an update method
|
||||
- Asynchronous operations in Gradio require special handling to ensure UI elements update in real-time
|
||||
- Using Gradio's built-in progress tracking mechanism is more effective than manual UI updates for async tasks
|
||||
- When using LiteLLM with different model providers, it's essential to set the `custom_llm_provider` parameter correctly for each provider
|
||||
|
||||
4. Fixed LLM provider configuration for OpenRouter models:
|
||||
- Identified an issue with OpenRouter models not working correctly in the report synthesis module
|
||||
- Added the missing `custom_llm_provider = 'openrouter'` parameter to the LiteLLM completion parameters
|
||||
- Tested the fix to ensure OpenRouter models now work correctly for report generation
|
||||
- The progress callback mechanism is critical for providing user feedback during long-running report generation tasks
|
||||
- Proper error handling in UI callbacks is essential for a smooth user experience
|
||||
- Simplifying the UI by removing redundant progress indicators improves user experience and reduces confusion
|
||||
- Consolidating to a single progress indicator ensures consistent feedback and reduces code complexity
|
||||
|
||||
|
||||
## Session: 2025-02-27
|
||||
|
||||
### Overview
|
||||
Initial project setup and implementation of core functionality for semantic similarity search using Jina AI's APIs.
|
||||
|
||||
### Key Activities
|
||||
1. Created the core `JinaSimilarity` class in jina_similarity.py with the following features:
|
||||
- Token counting using tiktoken
|
||||
- Embedding generation using Jina AI's Embeddings API
|
||||
- Similarity computation using cosine similarity
|
||||
- Error handling for token limit violations
|
||||
|
||||
2. Implemented the markdown segmenter in markdown_segmenter.py:
|
||||
- Segmentation of markdown documents using Jina AI's Segmenter API
|
||||
- Command-line interface for easy usage
|
||||
|
||||
3. Developed a test script (test_similarity.py) with:
|
||||
- Command-line argument parsing
|
||||
- File reading functionality
|
||||
- Verbose output option for debugging
|
||||
- Error handling
|
||||
|
||||
4. Created sample files for testing:
|
||||
- sample_chunk.txt: Contains a paragraph about pangrams
|
||||
- sample_query.txt: Contains a question about pangrams
|
||||
|
||||
### Insights
|
||||
- Jina AI's embedding model (jina-embeddings-v3) provides high-quality embeddings for semantic search
|
||||
- The token limit of 8,192 tokens is sufficient for most use cases, but longer documents need segmentation
|
||||
- Normalizing embeddings simplifies similarity computation (dot product equals cosine similarity)
|
||||
- Separating segmentation from similarity computation provides better modularity
|
||||
|
||||
### Challenges
|
||||
- Ensuring proper error handling for API failures
|
||||
- Managing token limits for large documents
|
||||
- Balancing between chunking granularity and semantic coherence
|
||||
|
||||
### Next Steps
|
||||
1. Add tiktoken to requirements.txt
|
||||
2. Implement caching for embeddings to reduce API calls
|
||||
3. Add batch processing capabilities for multiple chunks/queries
|
||||
4. Create comprehensive documentation and usage examples
|
||||
5. Develop integration tests for reliability testing
|
||||
|
||||
## Session: 2025-02-27 (Update)
|
||||
|
||||
### Overview
|
||||
Created memory bank for the project to maintain persistent knowledge about the codebase and development progress.
|
||||
|
||||
### Key Activities
|
||||
1. Created the `.note/` directory to store memory bank files
|
||||
2. Created the following memory bank files:
|
||||
- project_overview.md: Purpose, goals, and high-level architecture
|
||||
- current_focus.md: Active work, recent changes, and next steps
|
||||
- development_standards.md: Coding conventions and patterns
|
||||
- decision_log.md: Key decisions with rationale
|
||||
- code_structure.md: Codebase organization with module descriptions
|
||||
- session_log.md: History of development sessions
|
||||
- interfaces.md: Component interfaces and API documentation
|
||||
|
||||
### Insights
|
||||
- The project has a clear structure with well-defined components
|
||||
- The use of Jina AI's APIs provides powerful semantic search capabilities
|
||||
- The modular design allows for easy extension and maintenance
|
||||
- Some improvements are needed, such as adding tiktoken to requirements.txt
|
||||
|
||||
### Next Steps
|
||||
1. Update requirements.txt to include all dependencies (tiktoken)
|
||||
2. Implement caching mechanism for embeddings
|
||||
3. Add batch processing capabilities
|
||||
4. Create comprehensive documentation
|
||||
5. Develop integration tests
|
||||
|
||||
## Session: 2025-02-27 (Update 2)
|
||||
|
||||
### Overview
|
||||
Expanded the project scope to build a comprehensive intelligent research system with an 8-stage pipeline.
|
||||
|
||||
### Key Activities
|
||||
1. Defined the overall architecture for the intelligent research system:
|
||||
- 8-stage pipeline from query acceptance to report generation
|
||||
- Multiple search sources (Google, Serper, Jina Search, Google Scholar, arXiv)
|
||||
- Semantic processing using Jina AI's APIs
|
||||
|
||||
2. Updated the memory bank to reflect the broader vision:
|
||||
- Revised project_overview.md with the complete research system goals
|
||||
- Updated current_focus.md with next steps for each pipeline stage
|
||||
- Enhanced code_structure.md with planned project organization
|
||||
- Added new decisions to decision_log.md
|
||||
|
||||
### Insights
|
||||
- The modular pipeline architecture allows for incremental development
|
||||
- Jina AI's suite of APIs provides a consistent approach to semantic processing
|
||||
- Multiple search sources will provide more comprehensive research results
|
||||
- The current similarity components fit naturally into stages 6-7 of the pipeline
|
||||
|
||||
### Next Steps
|
||||
1. Begin implementing the query processing module (stage 1)
|
||||
2. Design the data structures for passing information between pipeline stages
|
||||
3. Create a project roadmap with milestones for each stage
|
||||
4. Prioritize development of core components for an end-to-end MVP
|
||||
|
||||
## Session: 2025-02-27 (Update 3)
|
||||
|
||||
### Overview
|
||||
Planned the implementation of the Query Processing Module with LiteLLM integration and Gradio UI.
|
||||
|
||||
### Key Activities
|
||||
1. Researched LiteLLM integration:
|
||||
- Explored LiteLLM documentation and usage patterns
|
||||
- Investigated integration with Gradio for UI development
|
||||
- Identified configuration requirements and best practices
|
||||
|
||||
2. Developed implementation plan:
|
||||
- Prioritized Query Processing Module with LiteLLM integration
|
||||
- Planned Gradio UI implementation for user interaction
|
||||
- Outlined configuration structure for API keys and settings
|
||||
- Established a sequence for implementing remaining modules
|
||||
|
||||
3. Updated memory bank:
|
||||
- Revised current_focus.md with new implementation plan
|
||||
- Added immediate and future steps for development
|
||||
|
||||
### Insights
|
||||
- LiteLLM provides a unified interface to multiple LLM providers, simplifying integration
|
||||
- Gradio offers an easy way to create interactive UIs for AI applications
|
||||
- The modular approach allows for incremental development and testing
|
||||
- Existing similarity components can be integrated into the pipeline at a later stage
|
||||
|
||||
### Next Steps
|
||||
1. Update requirements.txt with new dependencies (litellm, gradio, etc.)
|
||||
2. Create configuration structure for secure API key management
|
||||
3. Implement LiteLLM interface for query enhancement and classification
|
||||
4. Develop the query processor with structured output
|
||||
5. Build the Gradio UI for user interaction
|
||||
|
||||
## Session: 2025-02-27 (Update 4)
|
||||
|
||||
### Overview
|
||||
Implemented module-specific model configuration and created the Jina AI Reranker module.
|
||||
|
||||
### Key Activities
|
||||
1. Enhanced configuration structure:
|
||||
- Added support for module-specific model assignments
|
||||
- Configured different models for different tasks
|
||||
- Added detailed endpoint configurations for various providers
|
||||
|
||||
2. Updated LLMInterface:
|
||||
- Modified to support module-specific model configurations
|
||||
- Added support for different endpoint types (OpenAI, Azure, Ollama)
|
||||
- Implemented method delegation to use appropriate models for each task
|
||||
|
||||
3. Created Jina AI Reranker module:
|
||||
- Implemented document reranking using Jina AI's Reranker API
|
||||
- Added support for reranking documents with metadata
|
||||
- Configured to use the "jina-reranker-v2-base-multilingual" model
|
||||
|
||||
### Insights
|
||||
- Using different models for different tasks allows for optimizing performance and cost
|
||||
- Jina's reranker provides a specialized solution for document ranking
|
||||
- The modular approach allows for easy swapping of components and models
|
||||
|
||||
### Next Steps
|
||||
1. Implement the remaining query processing components
|
||||
2. Create the Gradio UI for user interaction
|
||||
3. Test the full system with end-to-end workflows
|
||||
|
||||
## Session: 2025-02-27 (Update 5)
|
||||
|
||||
### Overview
|
||||
Added support for OpenRouter and Groq as LLM providers and configured the system to use Groq for testing.
|
||||
|
||||
### Key Activities
|
||||
1. **Jina Reranker API Integration**:
|
||||
- Updated the `rerank` method in the JinaReranker class to match the expected API request format
|
||||
- Modified the request payload to send an array of plain string documents instead of objects
|
||||
- Enhanced response processing to handle both current and older API response formats
|
||||
- Added detailed logging for API requests and responses for better debugging
|
||||
|
||||
2. **Testing Improvements**:
|
||||
- Created a simplified test script (`test_simple_reranker.py`) to isolate and test the reranker functionality
|
||||
- Updated the main test script to focus on core functionality without complex dependencies
|
||||
- Implemented JSON result saving for better analysis of reranker output
|
||||
- Added proper error handling in tests to provide clear feedback on issues
|
||||
|
||||
3. **Code Quality Enhancements**:
|
||||
- Improved error handling throughout the reranker implementation
|
||||
- Added informative debug messages at key points in the execution flow
|
||||
- Ensured backward compatibility with previous API response formats
|
||||
- Documented the expected request and response structures
|
||||
|
||||
### Insights and Learnings
|
||||
- The Jina Reranker API expects documents as an array of plain strings, not objects with a "text" field
|
||||
- The reranker response format includes a "document" field in the results which may contain either the text directly or an object with a "text" field
|
||||
- Proper error handling and debug output are crucial for diagnosing issues with external API integrations
|
||||
- Isolating components for testing makes debugging much more efficient
|
||||
|
||||
### Challenges
|
||||
- Adapting to changes in the Jina Reranker API response format
|
||||
- Ensuring backward compatibility with older response formats
|
||||
- Debugging nested API response structures
|
||||
- Managing environment variables and configuration consistently across test scripts
|
||||
|
||||
### Next Steps
|
||||
1. **Expand Testing**: Develop more comprehensive test cases for the reranker with diverse document types
|
||||
2. **Integration**: Ensure the reranker is properly integrated with the result collector for end-to-end functionality
|
||||
3. **Documentation**: Update API documentation to reflect the latest changes to the reranker implementation
|
||||
4. **UI Integration**: Add reranker configuration options to the Gradio interface
|
||||
|
||||
## Session: 2025-02-27 - Report Generation Module Planning
|
||||
|
||||
### Overview
|
||||
In this session, we focused on planning the Report Generation module, designing a comprehensive implementation approach, and making key decisions about document scraping, storage, and processing.
|
||||
|
||||
### Key Activities
|
||||
1. **Designed a Phased Implementation Plan**:
|
||||
- Created a four-phase implementation plan for the Report Generation module
|
||||
- Phase 1: Document Scraping and Storage
|
||||
- Phase 2: Document Prioritization and Chunking
|
||||
- Phase 3: Report Generation
|
||||
- Phase 4: Advanced Features
|
||||
- Documented the plan in the memory bank for future reference
|
||||
|
||||
2. **Made Key Design Decisions**:
|
||||
- Decided to use Jina Reader for web scraping due to its clean content extraction capabilities
|
||||
- Chose SQLite for document storage to ensure persistence and efficient querying
|
||||
- Designed a database schema with Documents and Metadata tables
|
||||
- Planned a token budget management system to handle context window limitations
|
||||
- Decided on a map-reduce approach for processing large document collections
|
||||
|
||||
3. **Addressed Context Window Limitations**:
|
||||
- Evaluated Groq's Llama 3.3 70B Versatile model's 128K context window
|
||||
- Designed document prioritization strategies based on relevance scores
|
||||
- Planned chunking strategies for handling long documents
|
||||
- Considered alternative models with larger context windows for future implementation
|
||||
|
||||
4. **Updated Documentation**:
|
||||
- Added the implementation plan to the memory bank
|
||||
- Updated the decision log with rationale for key decisions
|
||||
- Revised the current focus to reflect the new implementation priorities
|
||||
- Added a new session log entry to document the planning process
|
||||
|
||||
### Insights
|
||||
- A phased implementation approach allows for incremental development and testing
|
||||
- SQLite provides a good balance of simplicity and functionality for document storage
|
||||
- Jina Reader integrates well with our existing Jina components (embeddings, reranker)
|
||||
- The map-reduce pattern enables processing of unlimited document collections despite context window limitations
|
||||
- Document prioritization is crucial for ensuring the most relevant content is included in reports
|
||||
|
||||
### Challenges
|
||||
- Managing the 128K context window limitation with potentially large document collections
|
||||
- Balancing between document coverage and report quality
|
||||
- Ensuring efficient web scraping without overwhelming target websites
|
||||
- Designing a flexible architecture that can accommodate different models and approaches
|
||||
|
||||
### Next Steps
|
||||
1. Begin implementing Phase 1 of the Report Generation module:
|
||||
- Set up the SQLite database with the designed schema
|
||||
- Implement the Jina Reader integration for web scraping
|
||||
- Create the document processing pipeline
|
||||
- Develop URL validation and normalization functionality
|
||||
- Add caching and deduplication for scraped content
|
||||
|
||||
2. Plan for Phase 2 implementation:
|
||||
- Design the token budget management system
|
||||
- Develop document prioritization algorithms
|
||||
- Create chunking strategies for long documents
|
||||
|
||||
## Session: 2025-02-27 - Report Generation Module Implementation (Phase 1)
|
||||
|
||||
### Overview
|
||||
In this session, we implemented Phase 1 of the Report Generation module, focusing on document scraping and SQLite storage. We created the necessary components for scraping web pages, storing their content in a SQLite database, and retrieving documents for report generation.
|
||||
|
||||
### Key Activities
|
||||
1. **Created Database Manager**:
|
||||
- Implemented a SQLite database manager with tables for documents and metadata
|
||||
- Added full CRUD operations for documents
|
||||
- Implemented transaction handling for data integrity
|
||||
- Created methods for document search and retrieval
|
||||
- Used aiosqlite for asynchronous database operations
|
||||
|
||||
2. **Implemented Document Scraper**:
|
||||
- Created a document scraper with Jina Reader API integration
|
||||
- Added fallback mechanism using BeautifulSoup for when Jina API fails
|
||||
- Implemented URL validation and normalization
|
||||
- Added content conversion to Markdown format
|
||||
- Implemented token counting using tiktoken
|
||||
- Created metadata extraction from HTML content
|
||||
- Added document deduplication using content hashing
|
||||
|
||||
3. **Developed Report Generator Base**:
|
||||
- Created the basic structure for the report generation process
|
||||
- Implemented methods to process search results by scraping URLs
|
||||
- Integrated with the database manager and document scraper
|
||||
- Set up the foundation for future phases
|
||||
|
||||
4. **Created Test Script**:
|
||||
- Developed a test script to verify functionality
|
||||
- Tested document scraping, storage, and retrieval
|
||||
- Verified search functionality within the database
|
||||
- Ensured proper error handling and fallback mechanisms
|
||||
|
||||
### Insights
|
||||
- The fallback mechanism for document scraping is crucial, as the Jina Reader API may not always be available or may fail for certain URLs
|
||||
- Asynchronous processing significantly improves performance when scraping multiple URLs
|
||||
- Content hashing is an effective way to prevent duplicate documents in the database
|
||||
- Storing metadata separately from document content provides flexibility for future enhancements
|
||||
- The SQLite database provides a good balance of simplicity and functionality for document storage
|
||||
|
||||
### Challenges
|
||||
- Handling different HTML structures across websites for metadata extraction
|
||||
- Managing asynchronous operations and error handling
|
||||
- Ensuring proper transaction handling for database operations
|
||||
- Balancing between clean content extraction and preserving important information
|
||||
|
||||
### Next Steps
|
||||
1. **Integration with Search Execution**:
|
||||
- Connect the report generation module to the search execution pipeline
|
||||
- Implement automatic processing of search results
|
||||
|
||||
2. **Begin Phase 2 Implementation**:
|
||||
- Develop document prioritization based on relevance scores
|
||||
- Implement chunking strategies for long documents
|
||||
- Create token budget management system
|
||||
|
||||
3. **Testing and Refinement**:
|
||||
- Create more comprehensive tests for edge cases
|
||||
- Refine error handling and logging
|
||||
- Optimize performance for large numbers of documents
|
||||
|
||||
## Session: 2025-02-27 (Update)
|
||||
|
||||
### Overview
|
||||
Implemented Phase 3 of the Report Generation module, focusing on report synthesis using LLMs with a map-reduce approach.
|
||||
|
||||
### Key Activities
|
||||
1. **Created Report Synthesis Module**:
|
||||
- Implemented the `ReportSynthesizer` class for generating reports using Groq's Llama 3.3 70B model
|
||||
- Created a map-reduce approach for processing document chunks:
|
||||
- Map phase: Extract key information from individual chunks
|
||||
- Reduce phase: Synthesize extracted information into a coherent report
|
||||
- Added support for different query types (factual, exploratory, comparative)
|
||||
- Implemented automatic query type detection based on query text
|
||||
- Added citation generation and reference management
|
||||
|
||||
2. **Updated Report Generator**:
|
||||
- Integrated the new report synthesis module with the existing report generator
|
||||
- Replaced the placeholder report generation with the new LLM-based synthesis
|
||||
- Added proper error handling and logging throughout the process
|
||||
|
||||
3. **Created Test Scripts**:
|
||||
- Developed a dedicated test script for the report synthesis functionality
|
||||
- Implemented tests with both sample data and real URLs
|
||||
- Added support for mock data to avoid API dependencies during testing
|
||||
- Verified end-to-end functionality from document scraping to report generation
|
||||
|
||||
4. **Fixed LLM Integration Issues**:
|
||||
- Corrected the model name format for Groq provider by prefixing it with 'groq/'
|
||||
- Improved error handling for API failures
|
||||
- Added proper logging for the map-reduce process
|
||||
|
||||
### Insights
|
||||
- The map-reduce approach is effective for processing large amounts of document data
|
||||
- Different query types benefit from specialized report templates
|
||||
- Groq's Llama 3.3 70B model produces high-quality reports with good coherence and factual accuracy
|
||||
- Proper citation management is essential for creating trustworthy reports
|
||||
- Automatic query type detection works well for common query patterns
|
||||
|
||||
### Challenges
|
||||
- Managing API errors and rate limits with external LLM providers
|
||||
- Ensuring consistent formatting across different report sections
|
||||
- Balancing between report comprehensiveness and token usage
|
||||
- Handling edge cases where document chunks contain irrelevant information
|
||||
|
||||
### Next Steps
|
||||
1.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,18 @@
|
|||
# API settings
|
||||
SECRET_KEY=your-secret-key-here
|
||||
API_V1_STR=/api/v1
|
||||
|
||||
# Database settings
|
||||
DATABASE_URL=sqlite:///./sim-search.db
|
||||
|
||||
# CORS settings
|
||||
CORS_ORIGINS=http://localhost:3000,http://localhost:8000
|
||||
|
||||
# Sim-search settings
|
||||
SIM_SEARCH_PATH=/Volumes/SAM2/CODE/sim-search
|
||||
|
||||
# Default models for different detail levels
|
||||
DEFAULT_MODELS_BRIEF=llama-3.1-8b-instant
|
||||
DEFAULT_MODELS_STANDARD=llama-3.1-8b-instant
|
||||
DEFAULT_MODELS_DETAILED=llama-3.3-70b-versatile
|
||||
DEFAULT_MODELS_COMPREHENSIVE=llama-3.3-70b-versatile
|
||||
|
|
@ -0,0 +1,165 @@
|
|||
# Sim-Search API
|
||||
|
||||
A FastAPI backend for the Sim-Search intelligent research system.
|
||||
|
||||
## Overview
|
||||
|
||||
This API provides a RESTful interface to the Sim-Search system, allowing for:
|
||||
|
||||
- Query processing and classification
|
||||
- Search execution across multiple engines
|
||||
- Report generation with different detail levels
|
||||
- User authentication and management
|
||||
|
||||
## Architecture
|
||||
|
||||
The API follows a layered architecture:
|
||||
|
||||
1. **API Layer**: FastAPI routes and endpoints
|
||||
2. **Service Layer**: Business logic and integration with Sim-Search
|
||||
3. **Data Layer**: Database models and session management
|
||||
|
||||
## Setup
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- Python 3.8+
|
||||
- Sim-Search system installed and configured
|
||||
- API keys for search engines (if using external search engines)
|
||||
|
||||
### Installation
|
||||
|
||||
1. Clone the repository:
|
||||
|
||||
```bash
|
||||
git clone <repository-url>
|
||||
cd sim-search-api
|
||||
```
|
||||
|
||||
2. Create a virtual environment:
|
||||
|
||||
```bash
|
||||
python -m venv venv
|
||||
source venv/bin/activate # On Windows: venv\Scripts\activate
|
||||
```
|
||||
|
||||
3. Install dependencies:
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
4. Create a `.env` file based on `.env.example`:
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
5. Edit the `.env` file with your configuration settings.
|
||||
|
||||
### Database Setup
|
||||
|
||||
Initialize the database:
|
||||
|
||||
```bash
|
||||
alembic upgrade head
|
||||
```
|
||||
|
||||
## Running the API
|
||||
|
||||
Start the API server:
|
||||
|
||||
```bash
|
||||
python run.py
|
||||
```
|
||||
|
||||
Or with custom settings:
|
||||
|
||||
```bash
|
||||
python run.py --host 0.0.0.0 --port 8000 --reload --debug
|
||||
```
|
||||
|
||||
## API Documentation
|
||||
|
||||
Once the server is running, you can access the API documentation at:
|
||||
|
||||
- Swagger UI: http://localhost:8000/docs
|
||||
- ReDoc: http://localhost:8000/redoc
|
||||
|
||||
## API Endpoints
|
||||
|
||||
### Authentication
|
||||
|
||||
- `POST /api/v1/auth/token`: Get an authentication token
|
||||
- `POST /api/v1/auth/register`: Register a new user
|
||||
|
||||
### Query Processing
|
||||
|
||||
- `POST /api/v1/query/process`: Process and enhance a user query
|
||||
- `POST /api/v1/query/classify`: Classify a query by type and intent
|
||||
|
||||
### Search Execution
|
||||
|
||||
- `POST /api/v1/search/execute`: Execute a search with optional parameters
|
||||
- `GET /api/v1/search/engines`: Get available search engines
|
||||
- `GET /api/v1/search/history`: Get user's search history
|
||||
- `GET /api/v1/search/{search_id}`: Get results for a specific search
|
||||
- `DELETE /api/v1/search/{search_id}`: Delete a search from history
|
||||
|
||||
### Report Generation
|
||||
|
||||
- `POST /api/v1/report/generate`: Generate a report from search results
|
||||
- `GET /api/v1/report/list`: Get a list of user's reports
|
||||
- `GET /api/v1/report/{report_id}`: Get a specific report
|
||||
- `DELETE /api/v1/report/{report_id}`: Delete a report
|
||||
- `GET /api/v1/report/{report_id}/download`: Download a report in specified format
|
||||
|
||||
## Development
|
||||
|
||||
### Project Structure
|
||||
|
||||
```
|
||||
sim-search-api/
|
||||
├── app/
|
||||
│ ├── api/
|
||||
│ │ ├── routes/
|
||||
│ │ │ ├── __init__.py
|
||||
│ │ │ ├── query.py # Query processing endpoints
|
||||
│ │ │ ├── search.py # Search execution endpoints
|
||||
│ │ │ ├── report.py # Report generation endpoints
|
||||
│ │ │ └── auth.py # Authentication endpoints
|
||||
│ │ ├── __init__.py
|
||||
│ │ └── dependencies.py # API dependencies (auth, rate limiting)
|
||||
│ ├── core/
|
||||
│ │ ├── __init__.py
|
||||
│ │ ├── config.py # API configuration
|
||||
│ │ └── security.py # Security utilities
|
||||
│ ├── db/
|
||||
│ │ ├── __init__.py
|
||||
│ │ ├── session.py # Database session
|
||||
│ │ └── models.py # Database models for reports, searches
|
||||
│ ├── schemas/
|
||||
│ │ ├── __init__.py
|
||||
│ │ ├── query.py # Query schemas
|
||||
│ │ ├── search.py # Search result schemas
|
||||
│ │ └── report.py # Report schemas
|
||||
│ ├── services/
|
||||
│ │ ├── __init__.py
|
||||
│ │ ├── query_service.py # Query processing service
|
||||
│ │ ├── search_service.py # Search execution service
|
||||
│ │ └── report_service.py # Report generation service
|
||||
│ └── main.py # FastAPI application
|
||||
├── alembic/ # Database migrations
|
||||
├── .env.example # Environment variables template
|
||||
└── requirements.txt # Dependencies
|
||||
```
|
||||
|
||||
### Running Tests
|
||||
|
||||
```bash
|
||||
pytest
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
[MIT License](LICENSE)
|
||||
|
|
@ -0,0 +1,102 @@
|
|||
# A generic, single database configuration.
|
||||
|
||||
[alembic]
|
||||
# path to migration scripts
|
||||
script_location = alembic
|
||||
|
||||
# template used to generate migration files
|
||||
# file_template = %%(rev)s_%%(slug)s
|
||||
|
||||
# sys.path path, will be prepended to sys.path if present.
|
||||
# defaults to the current working directory.
|
||||
prepend_sys_path = .
|
||||
|
||||
# timezone to use when rendering the date within the migration file
|
||||
# as well as the filename.
|
||||
# If specified, requires the python-dateutil library that can be
|
||||
# installed by adding `alembic[tz]` to the pip requirements
|
||||
# string value is passed to dateutil.tz.gettz()
|
||||
# leave blank for localtime
|
||||
# timezone =
|
||||
|
||||
# max length of characters to apply to the
|
||||
# "slug" field
|
||||
# truncate_slug_length = 40
|
||||
|
||||
# set to 'true' to run the environment during
|
||||
# the 'revision' command, regardless of autogenerate
|
||||
# revision_environment = false
|
||||
|
||||
# set to 'true' to allow .pyc and .pyo files without
|
||||
# a source .py file to be detected as revisions in the
|
||||
# versions/ directory
|
||||
# sourceless = false
|
||||
|
||||
# version location specification; This defaults
|
||||
# to alembic/versions. When using multiple version
|
||||
# directories, initial revisions must be specified with --version-path.
|
||||
# The path separator used here should be the separator specified by "version_path_separator" below.
|
||||
# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions
|
||||
|
||||
# version path separator; As mentioned above, this is the character used to split
|
||||
# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
|
||||
# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
|
||||
# Valid values for version_path_separator are:
|
||||
#
|
||||
# version_path_separator = :
|
||||
# version_path_separator = ;
|
||||
# version_path_separator = space
|
||||
version_path_separator = os # Use os.pathsep. Default configuration used for new projects.
|
||||
|
||||
# the output encoding used when revision files
|
||||
# are written from script.py.mako
|
||||
# output_encoding = utf-8
|
||||
|
||||
sqlalchemy.url = sqlite:///./sim-search.db
|
||||
|
||||
|
||||
[post_write_hooks]
|
||||
# post_write_hooks defines scripts or Python functions that are run
|
||||
# on newly generated revision scripts. See the documentation for further
|
||||
# detail and examples
|
||||
|
||||
# format using "black" - use the console_scripts runner, against the "black" entrypoint
|
||||
# hooks = black
|
||||
# black.type = console_scripts
|
||||
# black.entrypoint = black
|
||||
# black.options = -l 79 REVISION_SCRIPT_FILENAME
|
||||
|
||||
# Logging configuration
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
import os
|
||||
from logging.config import fileConfig
|
||||
|
||||
from sqlalchemy import engine_from_config
|
||||
from sqlalchemy import pool
|
||||
|
||||
from alembic import context
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load environment variables from .env file
|
||||
load_dotenv()
|
||||
|
||||
# this is the Alembic Config object, which provides
|
||||
# access to the values within the .ini file in use.
|
||||
config = context.config
|
||||
|
||||
# Override sqlalchemy.url with the value from environment variable
|
||||
sqlalchemy_url = os.getenv("DATABASE_URL", "sqlite:///./sim-search.db")
|
||||
config.set_main_option("sqlalchemy.url", sqlalchemy_url)
|
||||
|
||||
# Interpret the config file for Python logging.
|
||||
# This line sets up loggers basically.
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
# add your model's MetaData object here
|
||||
# for 'autogenerate' support
|
||||
from app.db.models import Base
|
||||
target_metadata = Base.metadata
|
||||
|
||||
# other values from the config, defined by the needs of env.py,
|
||||
# can be acquired:
|
||||
# my_important_option = config.get_main_option("my_important_option")
|
||||
# ... etc.
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
"""Run migrations in 'offline' mode.
|
||||
|
||||
This configures the context with just a URL
|
||||
and not an Engine, though an Engine is acceptable
|
||||
here as well. By skipping the Engine creation
|
||||
we don't even need a DBAPI to be available.
|
||||
|
||||
Calls to context.execute() here emit the given string to the
|
||||
script output.
|
||||
|
||||
"""
|
||||
url = config.get_main_option("sqlalchemy.url")
|
||||
context.configure(
|
||||
url=url,
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
"""Run migrations in 'online' mode.
|
||||
|
||||
In this scenario we need to create an Engine
|
||||
and associate a connection with the context.
|
||||
|
||||
"""
|
||||
connectable = engine_from_config(
|
||||
config.get_section(config.config_ini_section, {}),
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
|
||||
with connectable.connect() as connection:
|
||||
context.configure(
|
||||
connection=connection, target_metadata=target_metadata
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = ${repr(up_revision)}
|
||||
down_revision = ${repr(down_revision)}
|
||||
branch_labels = ${repr(branch_labels)}
|
||||
depends_on = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
${downgrades if downgrades else "pass"}
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
"""Initial migration
|
||||
|
||||
Revision ID: 001
|
||||
Revises:
|
||||
Create Date: 2025-03-20
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects import sqlite
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '001'
|
||||
down_revision = None
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Create users table
|
||||
op.create_table(
|
||||
'users',
|
||||
sa.Column('id', sa.String(), nullable=False),
|
||||
sa.Column('email', sa.String(), nullable=False),
|
||||
sa.Column('hashed_password', sa.String(), nullable=False),
|
||||
sa.Column('full_name', sa.String(), nullable=True),
|
||||
sa.Column('is_active', sa.Boolean(), nullable=True, default=True),
|
||||
sa.Column('is_superuser', sa.Boolean(), nullable=True, default=False),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('email')
|
||||
)
|
||||
op.create_index(op.f('ix_users_email'), 'users', ['email'], unique=True)
|
||||
op.create_index(op.f('ix_users_id'), 'users', ['id'], unique=True)
|
||||
|
||||
# Create searches table
|
||||
op.create_table(
|
||||
'searches',
|
||||
sa.Column('id', sa.String(), nullable=False),
|
||||
sa.Column('user_id', sa.String(), nullable=True),
|
||||
sa.Column('query', sa.String(), nullable=False),
|
||||
sa.Column('enhanced_query', sa.String(), nullable=True),
|
||||
sa.Column('query_type', sa.String(), nullable=True),
|
||||
sa.Column('engines', sa.String(), nullable=True),
|
||||
sa.Column('results_count', sa.Integer(), nullable=True, default=0),
|
||||
sa.Column('results', sqlite.JSON(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=True, default=sa.func.current_timestamp()),
|
||||
sa.ForeignKeyConstraint(['user_id'], ['users.id'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index(op.f('ix_searches_id'), 'searches', ['id'], unique=True)
|
||||
|
||||
# Create reports table
|
||||
op.create_table(
|
||||
'reports',
|
||||
sa.Column('id', sa.String(), nullable=False),
|
||||
sa.Column('user_id', sa.String(), nullable=True),
|
||||
sa.Column('search_id', sa.String(), nullable=True),
|
||||
sa.Column('title', sa.String(), nullable=False),
|
||||
sa.Column('content', sa.Text(), nullable=False),
|
||||
sa.Column('detail_level', sa.String(), nullable=False, default='standard'),
|
||||
sa.Column('query_type', sa.String(), nullable=True),
|
||||
sa.Column('model_used', sa.String(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=True, default=sa.func.current_timestamp()),
|
||||
sa.Column('updated_at', sa.DateTime(), nullable=True, default=sa.func.current_timestamp(), onupdate=sa.func.current_timestamp()),
|
||||
sa.ForeignKeyConstraint(['search_id'], ['searches.id'], ),
|
||||
sa.ForeignKeyConstraint(['user_id'], ['users.id'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index(op.f('ix_reports_id'), 'reports', ['id'], unique=True)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index(op.f('ix_reports_id'), table_name='reports')
|
||||
op.drop_table('reports')
|
||||
op.drop_index(op.f('ix_searches_id'), table_name='searches')
|
||||
op.drop_table('searches')
|
||||
op.drop_index(op.f('ix_users_id'), table_name='users')
|
||||
op.drop_index(op.f('ix_users_email'), table_name='users')
|
||||
op.drop_table('users')
|
||||
|
|
@ -0,0 +1,120 @@
|
|||
"""
|
||||
API dependencies for the sim-search API.
|
||||
|
||||
This module provides common dependencies for the API routes.
|
||||
"""
|
||||
|
||||
from typing import Generator, Optional
|
||||
|
||||
from fastapi import Depends, HTTPException, status
|
||||
from fastapi.security import OAuth2PasswordBearer
|
||||
from jose import jwt, JWTError
|
||||
from pydantic import ValidationError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.security import verify_password
|
||||
from app.db.models import User
|
||||
from app.db.session import get_db
|
||||
from app.schemas.token import TokenPayload
|
||||
|
||||
# OAuth2 scheme for token authentication
|
||||
oauth2_scheme = OAuth2PasswordBearer(tokenUrl=f"{settings.API_V1_STR}/auth/token")
|
||||
|
||||
|
||||
def get_current_user(
|
||||
db: Session = Depends(get_db), token: str = Depends(oauth2_scheme)
|
||||
) -> User:
|
||||
"""
|
||||
Get the current user from the token.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
token: JWT token
|
||||
|
||||
Returns:
|
||||
User object
|
||||
|
||||
Raises:
|
||||
HTTPException: If the token is invalid or the user is not found
|
||||
"""
|
||||
try:
|
||||
payload = jwt.decode(
|
||||
token, settings.SECRET_KEY, algorithms=[settings.ALGORITHM]
|
||||
)
|
||||
token_data = TokenPayload(**payload)
|
||||
except (JWTError, ValidationError):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Could not validate credentials",
|
||||
)
|
||||
|
||||
user = db.query(User).filter(User.id == token_data.sub).first()
|
||||
if not user:
|
||||
raise HTTPException(status_code=404, detail="User not found")
|
||||
if not user.is_active:
|
||||
raise HTTPException(status_code=400, detail="Inactive user")
|
||||
|
||||
return user
|
||||
|
||||
|
||||
def get_current_active_user(
|
||||
current_user: User = Depends(get_current_user),
|
||||
) -> User:
|
||||
"""
|
||||
Get the current active user.
|
||||
|
||||
Args:
|
||||
current_user: Current user
|
||||
|
||||
Returns:
|
||||
User object
|
||||
|
||||
Raises:
|
||||
HTTPException: If the user is inactive
|
||||
"""
|
||||
if not current_user.is_active:
|
||||
raise HTTPException(status_code=400, detail="Inactive user")
|
||||
return current_user
|
||||
|
||||
|
||||
def get_current_active_superuser(
|
||||
current_user: User = Depends(get_current_user),
|
||||
) -> User:
|
||||
"""
|
||||
Get the current active superuser.
|
||||
|
||||
Args:
|
||||
current_user: Current user
|
||||
|
||||
Returns:
|
||||
User object
|
||||
|
||||
Raises:
|
||||
HTTPException: If the user is not a superuser
|
||||
"""
|
||||
if not current_user.is_superuser:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="The user doesn't have enough privileges"
|
||||
)
|
||||
return current_user
|
||||
|
||||
|
||||
def authenticate_user(db: Session, email: str, password: str) -> Optional[User]:
|
||||
"""
|
||||
Authenticate a user.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
email: User email
|
||||
password: User password
|
||||
|
||||
Returns:
|
||||
User object if authentication is successful, None otherwise
|
||||
"""
|
||||
user = db.query(User).filter(User.email == email).first()
|
||||
if not user:
|
||||
return None
|
||||
if not verify_password(password, user.hashed_password):
|
||||
return None
|
||||
return user
|
||||
|
|
@ -0,0 +1,98 @@
|
|||
"""
|
||||
Authentication routes for the sim-search API.
|
||||
|
||||
This module defines the routes for user authentication and registration.
|
||||
"""
|
||||
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from fastapi.security import OAuth2PasswordRequestForm
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.dependencies import authenticate_user
|
||||
from app.core.config import settings
|
||||
from app.core.security import create_access_token, get_password_hash
|
||||
from app.db.models import User
|
||||
from app.db.session import get_db
|
||||
from app.schemas.token import Token
|
||||
from app.schemas.user import UserCreate, User as UserSchema
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/token", response_model=Token)
|
||||
async def login_for_access_token(
|
||||
form_data: OAuth2PasswordRequestForm = Depends(),
|
||||
db: Session = Depends(get_db),
|
||||
) -> Any:
|
||||
"""
|
||||
OAuth2 compatible token login, get an access token for future requests.
|
||||
|
||||
Args:
|
||||
form_data: OAuth2 password request form
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
Access token
|
||||
|
||||
Raises:
|
||||
HTTPException: If authentication fails
|
||||
"""
|
||||
user = authenticate_user(db, form_data.username, form_data.password)
|
||||
if not user:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Incorrect email or password",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
access_token_expires = timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES)
|
||||
access_token = create_access_token(
|
||||
subject=user.id, expires_delta=access_token_expires
|
||||
)
|
||||
|
||||
return {"access_token": access_token, "token_type": "bearer"}
|
||||
|
||||
|
||||
@router.post("/register", response_model=UserSchema)
|
||||
async def register_user(
|
||||
user_in: UserCreate,
|
||||
db: Session = Depends(get_db),
|
||||
) -> Any:
|
||||
"""
|
||||
Register a new user.
|
||||
|
||||
Args:
|
||||
user_in: User creation data
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
Created user
|
||||
|
||||
Raises:
|
||||
HTTPException: If a user with the same email already exists
|
||||
"""
|
||||
# Check if user with this email already exists
|
||||
user = db.query(User).filter(User.email == user_in.email).first()
|
||||
if user:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="A user with this email already exists",
|
||||
)
|
||||
|
||||
# Create new user
|
||||
user = User(
|
||||
email=user_in.email,
|
||||
hashed_password=get_password_hash(user_in.password),
|
||||
full_name=user_in.full_name,
|
||||
is_active=user_in.is_active,
|
||||
is_superuser=user_in.is_superuser,
|
||||
)
|
||||
|
||||
db.add(user)
|
||||
db.commit()
|
||||
db.refresh(user)
|
||||
|
||||
return user
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
"""
|
||||
Query routes for the sim-search API.
|
||||
|
||||
This module defines the routes for query processing and classification.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.dependencies import get_current_active_user
|
||||
from app.db.models import User
|
||||
from app.db.session import get_db
|
||||
from app.schemas.query import QueryProcess, QueryClassify, ProcessedQuery
|
||||
from app.services.query_service import QueryService
|
||||
|
||||
router = APIRouter()
|
||||
query_service = QueryService()
|
||||
|
||||
|
||||
@router.post("/process", response_model=ProcessedQuery)
|
||||
async def process_query(
|
||||
query_in: QueryProcess,
|
||||
current_user: User = Depends(get_current_active_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> Any:
|
||||
"""
|
||||
Process a query to enhance and structure it.
|
||||
|
||||
Args:
|
||||
query_in: Query to process
|
||||
current_user: Current authenticated user
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
Processed query with structured information
|
||||
"""
|
||||
try:
|
||||
processed_query = await query_service.process_query(query_in.query)
|
||||
return processed_query
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Error processing query: {str(e)}",
|
||||
)
|
||||
|
||||
|
||||
@router.post("/classify", response_model=ProcessedQuery)
|
||||
async def classify_query(
|
||||
query_in: QueryClassify,
|
||||
current_user: User = Depends(get_current_active_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> Any:
|
||||
"""
|
||||
Classify a query by type and intent.
|
||||
|
||||
Args:
|
||||
query_in: Query to classify
|
||||
current_user: Current authenticated user
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
Classified query with type and intent information
|
||||
"""
|
||||
try:
|
||||
classified_query = await query_service.classify_query(query_in.query)
|
||||
return classified_query
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Error classifying query: {str(e)}",
|
||||
)
|
||||
|
|
@ -0,0 +1,296 @@
|
|||
"""
|
||||
Report routes for the sim-search API.
|
||||
|
||||
This module defines the routes for report generation and management.
|
||||
"""
|
||||
|
||||
import os
|
||||
from typing import Any, List, Optional
|
||||
from fastapi import APIRouter, Depends, HTTPException, status, BackgroundTasks
|
||||
from fastapi.responses import FileResponse
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.dependencies import get_current_active_user
|
||||
from app.db.models import User, Report, Search
|
||||
from app.db.session import get_db
|
||||
from app.schemas.report import (
|
||||
ReportCreate, ReportUpdate, Report as ReportSchema,
|
||||
ReportList, ReportProgress, ReportDownload
|
||||
)
|
||||
from app.services.report_service import ReportService
|
||||
|
||||
router = APIRouter()
|
||||
report_service = ReportService()
|
||||
|
||||
# Dictionary to store report generation progress
|
||||
report_progress = {}
|
||||
|
||||
|
||||
@router.post("/generate", response_model=ReportSchema)
|
||||
async def generate_report(
|
||||
report_in: ReportCreate,
|
||||
background_tasks: BackgroundTasks,
|
||||
current_user: User = Depends(get_current_active_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> Any:
|
||||
"""
|
||||
Generate a report from search results.
|
||||
|
||||
Args:
|
||||
report_in: Report creation parameters
|
||||
background_tasks: FastAPI background tasks
|
||||
current_user: Current authenticated user
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
Generated report
|
||||
"""
|
||||
try:
|
||||
# Check if search_id is provided and exists
|
||||
search = None
|
||||
if report_in.search_id:
|
||||
search = db.query(Search).filter(
|
||||
Search.id == report_in.search_id,
|
||||
Search.user_id == current_user.id
|
||||
).first()
|
||||
|
||||
if not search:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Search not found",
|
||||
)
|
||||
|
||||
# Create report record
|
||||
title = report_in.title or f"Report: {report_in.query}"
|
||||
report = Report(
|
||||
user_id=current_user.id,
|
||||
search_id=report_in.search_id,
|
||||
title=title,
|
||||
content="Report generation in progress...",
|
||||
detail_level=report_in.detail_level or "standard",
|
||||
query_type=report_in.query_type,
|
||||
model_used=report_in.model,
|
||||
)
|
||||
|
||||
db.add(report)
|
||||
db.commit()
|
||||
db.refresh(report)
|
||||
|
||||
# Initialize progress tracking
|
||||
report_progress[report.id] = {
|
||||
"progress": 0.0,
|
||||
"status": "Initializing report generation...",
|
||||
"current_chunk": 0,
|
||||
"total_chunks": 0,
|
||||
"current_report": "Report generation in progress...",
|
||||
}
|
||||
|
||||
# Generate report in background
|
||||
background_tasks.add_task(
|
||||
report_service.generate_report_background,
|
||||
report_id=report.id,
|
||||
report_in=report_in,
|
||||
search=search,
|
||||
db=db,
|
||||
progress_dict=report_progress,
|
||||
)
|
||||
|
||||
return report
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Error generating report: {str(e)}",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/list", response_model=ReportList)
|
||||
async def list_reports(
|
||||
skip: int = 0,
|
||||
limit: int = 100,
|
||||
current_user: User = Depends(get_current_active_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> Any:
|
||||
"""
|
||||
Get a list of user's reports.
|
||||
|
||||
Args:
|
||||
skip: Number of records to skip
|
||||
limit: Maximum number of records to return
|
||||
current_user: Current authenticated user
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
List of reports
|
||||
"""
|
||||
reports = db.query(Report).filter(Report.user_id == current_user.id).order_by(
|
||||
Report.created_at.desc()
|
||||
).offset(skip).limit(limit).all()
|
||||
|
||||
total = db.query(Report).filter(Report.user_id == current_user.id).count()
|
||||
|
||||
return {"reports": reports, "total": total}
|
||||
|
||||
|
||||
@router.get("/{report_id}", response_model=ReportSchema)
|
||||
async def get_report(
|
||||
report_id: str,
|
||||
current_user: User = Depends(get_current_active_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> Any:
|
||||
"""
|
||||
Get a specific report.
|
||||
|
||||
Args:
|
||||
report_id: ID of the report
|
||||
current_user: Current authenticated user
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
Report
|
||||
|
||||
Raises:
|
||||
HTTPException: If the report is not found or doesn't belong to the user
|
||||
"""
|
||||
report = db.query(Report).filter(
|
||||
Report.id == report_id, Report.user_id == current_user.id
|
||||
).first()
|
||||
|
||||
if not report:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Report not found",
|
||||
)
|
||||
|
||||
return report
|
||||
|
||||
|
||||
@router.delete("/{report_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
async def delete_report(
|
||||
report_id: str,
|
||||
current_user: User = Depends(get_current_active_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> Any:
|
||||
"""
|
||||
Delete a report.
|
||||
|
||||
Args:
|
||||
report_id: ID of the report to delete
|
||||
current_user: Current authenticated user
|
||||
db: Database session
|
||||
|
||||
Raises:
|
||||
HTTPException: If the report is not found or doesn't belong to the user
|
||||
"""
|
||||
report = db.query(Report).filter(
|
||||
Report.id == report_id, Report.user_id == current_user.id
|
||||
).first()
|
||||
|
||||
if not report:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Report not found",
|
||||
)
|
||||
|
||||
db.delete(report)
|
||||
db.commit()
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@router.get("/{report_id}/progress", response_model=ReportProgress)
|
||||
async def get_report_progress(
|
||||
report_id: str,
|
||||
current_user: User = Depends(get_current_active_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> Any:
|
||||
"""
|
||||
Get the progress of a report generation.
|
||||
|
||||
Args:
|
||||
report_id: ID of the report
|
||||
current_user: Current authenticated user
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
Report generation progress
|
||||
|
||||
Raises:
|
||||
HTTPException: If the report is not found or doesn't belong to the user
|
||||
"""
|
||||
# Check if report exists and belongs to user
|
||||
report = db.query(Report).filter(
|
||||
Report.id == report_id, Report.user_id == current_user.id
|
||||
).first()
|
||||
|
||||
if not report:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Report not found",
|
||||
)
|
||||
|
||||
# Get progress from progress dictionary
|
||||
progress_data = report_progress.get(report_id, {
|
||||
"progress": 1.0,
|
||||
"status": "Report generation complete",
|
||||
"current_chunk": 0,
|
||||
"total_chunks": 0,
|
||||
"current_report": None,
|
||||
})
|
||||
|
||||
return {
|
||||
"report_id": report_id,
|
||||
"progress": progress_data.get("progress", 1.0),
|
||||
"status": progress_data.get("status", "Report generation complete"),
|
||||
"current_chunk": progress_data.get("current_chunk", 0),
|
||||
"total_chunks": progress_data.get("total_chunks", 0),
|
||||
"current_report": progress_data.get("current_report", None),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/{report_id}/download")
|
||||
async def download_report(
|
||||
report_id: str,
|
||||
format: str = "markdown",
|
||||
current_user: User = Depends(get_current_active_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> Any:
|
||||
"""
|
||||
Download a report in the specified format.
|
||||
|
||||
Args:
|
||||
report_id: ID of the report
|
||||
format: Format of the report (markdown, html, pdf)
|
||||
current_user: Current authenticated user
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
Report file
|
||||
|
||||
Raises:
|
||||
HTTPException: If the report is not found or doesn't belong to the user
|
||||
"""
|
||||
report = db.query(Report).filter(
|
||||
Report.id == report_id, Report.user_id == current_user.id
|
||||
).first()
|
||||
|
||||
if not report:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Report not found",
|
||||
)
|
||||
|
||||
# Generate file in the requested format
|
||||
try:
|
||||
file_path = await report_service.generate_report_file(report, format)
|
||||
|
||||
# Return file
|
||||
return FileResponse(
|
||||
path=file_path,
|
||||
filename=f"report_{report_id}.{format}",
|
||||
media_type="application/octet-stream",
|
||||
)
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Error generating report file: {str(e)}",
|
||||
)
|
||||
|
|
@ -0,0 +1,170 @@
|
|||
"""
|
||||
Search routes for the sim-search API.
|
||||
|
||||
This module defines the routes for search execution and history.
|
||||
"""
|
||||
|
||||
from typing import Any, List
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status, Query
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.dependencies import get_current_active_user
|
||||
from app.db.models import User, Search
|
||||
from app.db.session import get_db
|
||||
from app.schemas.search import SearchExecute, SearchResults, SearchHistory, SearchHistoryList
|
||||
from app.services.search_service import SearchService
|
||||
|
||||
router = APIRouter()
|
||||
search_service = SearchService()
|
||||
|
||||
|
||||
@router.post("/execute", response_model=SearchResults)
|
||||
async def execute_search(
|
||||
search_in: SearchExecute,
|
||||
current_user: User = Depends(get_current_active_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> Any:
|
||||
"""
|
||||
Execute a search with the given parameters.
|
||||
|
||||
Args:
|
||||
search_in: Search parameters
|
||||
current_user: Current authenticated user
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
Search results
|
||||
"""
|
||||
try:
|
||||
search_results = await search_service.execute_search(
|
||||
search_in.structured_query,
|
||||
search_in.search_engines,
|
||||
search_in.num_results,
|
||||
search_in.timeout,
|
||||
current_user.id,
|
||||
db,
|
||||
)
|
||||
return search_results
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Error executing search: {str(e)}",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/engines", response_model=List[str])
|
||||
async def get_available_search_engines(
|
||||
current_user: User = Depends(get_current_active_user),
|
||||
) -> Any:
|
||||
"""
|
||||
Get a list of available search engines.
|
||||
|
||||
Args:
|
||||
current_user: Current authenticated user
|
||||
|
||||
Returns:
|
||||
List of available search engine names
|
||||
"""
|
||||
try:
|
||||
engines = await search_service.get_available_search_engines()
|
||||
return engines
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Error getting available search engines: {str(e)}",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/history", response_model=SearchHistoryList)
|
||||
async def get_search_history(
|
||||
skip: int = 0,
|
||||
limit: int = 100,
|
||||
current_user: User = Depends(get_current_active_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> Any:
|
||||
"""
|
||||
Get the user's search history.
|
||||
|
||||
Args:
|
||||
skip: Number of records to skip
|
||||
limit: Maximum number of records to return
|
||||
current_user: Current authenticated user
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
List of search history records
|
||||
"""
|
||||
searches = db.query(Search).filter(Search.user_id == current_user.id).order_by(
|
||||
Search.created_at.desc()
|
||||
).offset(skip).limit(limit).all()
|
||||
|
||||
total = db.query(Search).filter(Search.user_id == current_user.id).count()
|
||||
|
||||
return {"searches": searches, "total": total}
|
||||
|
||||
|
||||
@router.get("/{search_id}", response_model=SearchResults)
|
||||
async def get_search_results(
|
||||
search_id: str,
|
||||
current_user: User = Depends(get_current_active_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> Any:
|
||||
"""
|
||||
Get results for a specific search.
|
||||
|
||||
Args:
|
||||
search_id: ID of the search
|
||||
current_user: Current authenticated user
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
Search results
|
||||
|
||||
Raises:
|
||||
HTTPException: If the search is not found or doesn't belong to the user
|
||||
"""
|
||||
search = db.query(Search).filter(
|
||||
Search.id == search_id, Search.user_id == current_user.id
|
||||
).first()
|
||||
|
||||
if not search:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Search not found",
|
||||
)
|
||||
|
||||
return await search_service.get_search_results(search)
|
||||
|
||||
|
||||
@router.delete("/{search_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
async def delete_search(
|
||||
search_id: str,
|
||||
current_user: User = Depends(get_current_active_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> Any:
|
||||
"""
|
||||
Delete a search from history.
|
||||
|
||||
Args:
|
||||
search_id: ID of the search to delete
|
||||
current_user: Current authenticated user
|
||||
db: Database session
|
||||
|
||||
Raises:
|
||||
HTTPException: If the search is not found or doesn't belong to the user
|
||||
"""
|
||||
search = db.query(Search).filter(
|
||||
Search.id == search_id, Search.user_id == current_user.id
|
||||
).first()
|
||||
|
||||
if not search:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Search not found",
|
||||
)
|
||||
|
||||
db.delete(search)
|
||||
db.commit()
|
||||
|
||||
return None
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
"""
|
||||
Configuration settings for the sim-search API.
|
||||
|
||||
This module defines the settings for the API, loaded from environment variables.
|
||||
"""
|
||||
|
||||
import os
|
||||
import secrets
|
||||
from typing import List, Optional, Dict, Any, Union
|
||||
|
||||
from pydantic import AnyHttpUrl, BaseSettings, validator
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""Settings for the sim-search API."""
|
||||
|
||||
# API settings
|
||||
API_V1_STR: str = "/api/v1"
|
||||
PROJECT_NAME: str = "Sim-Search API"
|
||||
PROJECT_DESCRIPTION: str = "API for the Sim-Search intelligent research system"
|
||||
VERSION: str = "0.1.0"
|
||||
|
||||
# Security settings
|
||||
SECRET_KEY: str = os.getenv("SECRET_KEY", secrets.token_urlsafe(32))
|
||||
ALGORITHM: str = "HS256"
|
||||
ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 7 # 7 days
|
||||
|
||||
# CORS settings
|
||||
CORS_ORIGINS: List[str] = ["*"]
|
||||
|
||||
@validator("CORS_ORIGINS", pre=True)
|
||||
def assemble_cors_origins(cls, v: Union[str, List[str]]) -> Union[List[str], str]:
|
||||
"""Parse CORS origins from string or list."""
|
||||
if isinstance(v, str) and not v.startswith("["):
|
||||
return [i.strip() for i in v.split(",")]
|
||||
elif isinstance(v, (list, str)):
|
||||
return v
|
||||
raise ValueError(v)
|
||||
|
||||
# Database settings
|
||||
SQLALCHEMY_DATABASE_URI: str = os.getenv(
|
||||
"DATABASE_URL", f"sqlite:///./sim-search.db"
|
||||
)
|
||||
|
||||
# Sim-search settings
|
||||
SIM_SEARCH_PATH: str = os.getenv("SIM_SEARCH_PATH", "/Volumes/SAM2/CODE/sim-search")
|
||||
|
||||
# Default models for different detail levels
|
||||
DEFAULT_MODELS: Dict[str, str] = {
|
||||
"brief": "llama-3.1-8b-instant",
|
||||
"standard": "llama-3.1-8b-instant",
|
||||
"detailed": "llama-3.3-70b-versatile",
|
||||
"comprehensive": "llama-3.3-70b-versatile"
|
||||
}
|
||||
|
||||
class Config:
|
||||
"""Pydantic config."""
|
||||
case_sensitive = True
|
||||
env_file = ".env"
|
||||
|
||||
|
||||
# Create settings instance
|
||||
settings = Settings()
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
"""
|
||||
Security utilities for the sim-search API.
|
||||
|
||||
This module provides utilities for password hashing, JWT token generation,
|
||||
and token validation.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from jose import jwt
|
||||
from passlib.context import CryptContext
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
# Password hashing context
|
||||
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
|
||||
|
||||
|
||||
def verify_password(plain_password: str, hashed_password: str) -> bool:
|
||||
"""
|
||||
Verify a password against a hash.
|
||||
|
||||
Args:
|
||||
plain_password: Plain text password
|
||||
hashed_password: Hashed password
|
||||
|
||||
Returns:
|
||||
True if the password matches the hash, False otherwise
|
||||
"""
|
||||
return pwd_context.verify(plain_password, hashed_password)
|
||||
|
||||
|
||||
def get_password_hash(password: str) -> str:
|
||||
"""
|
||||
Hash a password.
|
||||
|
||||
Args:
|
||||
password: Plain text password
|
||||
|
||||
Returns:
|
||||
Hashed password
|
||||
"""
|
||||
return pwd_context.hash(password)
|
||||
|
||||
|
||||
def create_access_token(
|
||||
subject: Union[str, Any], expires_delta: Optional[timedelta] = None
|
||||
) -> str:
|
||||
"""
|
||||
Create a JWT access token.
|
||||
|
||||
Args:
|
||||
subject: Subject of the token (usually user ID)
|
||||
expires_delta: Optional expiration time delta
|
||||
|
||||
Returns:
|
||||
JWT token as a string
|
||||
"""
|
||||
if expires_delta:
|
||||
expire = datetime.utcnow() + expires_delta
|
||||
else:
|
||||
expire = datetime.utcnow() + timedelta(
|
||||
minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES
|
||||
)
|
||||
|
||||
to_encode = {"exp": expire, "sub": str(subject)}
|
||||
encoded_jwt = jwt.encode(
|
||||
to_encode, settings.SECRET_KEY, algorithm=settings.ALGORITHM
|
||||
)
|
||||
|
||||
return encoded_jwt
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
"""
|
||||
Database models for the sim-search API.
|
||||
|
||||
This module defines the SQLAlchemy ORM models for the database.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import uuid
|
||||
from typing import Dict, Any, List
|
||||
|
||||
from sqlalchemy import Column, String, Boolean, ForeignKey, DateTime, Integer, JSON, Text
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from app.db.session import Base
|
||||
|
||||
|
||||
def generate_uuid() -> str:
|
||||
"""Generate a UUID string."""
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
class User(Base):
|
||||
"""User model."""
|
||||
|
||||
__tablename__ = "users"
|
||||
|
||||
id = Column(String, primary_key=True, index=True, default=generate_uuid)
|
||||
email = Column(String, unique=True, index=True, nullable=False)
|
||||
hashed_password = Column(String, nullable=False)
|
||||
full_name = Column(String, nullable=True)
|
||||
is_active = Column(Boolean, default=True)
|
||||
is_superuser = Column(Boolean, default=False)
|
||||
|
||||
searches = relationship("Search", back_populates="user")
|
||||
reports = relationship("Report", back_populates="user")
|
||||
|
||||
|
||||
class Search(Base):
|
||||
"""Search model."""
|
||||
|
||||
__tablename__ = "searches"
|
||||
|
||||
id = Column(String, primary_key=True, index=True, default=generate_uuid)
|
||||
user_id = Column(String, ForeignKey("users.id"))
|
||||
query = Column(String, nullable=False)
|
||||
enhanced_query = Column(String, nullable=True)
|
||||
query_type = Column(String, nullable=True)
|
||||
engines = Column(String, nullable=True) # Comma-separated list
|
||||
results_count = Column(Integer, default=0)
|
||||
results = Column(JSON, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.datetime.utcnow)
|
||||
|
||||
user = relationship("User", back_populates="searches")
|
||||
reports = relationship("Report", back_populates="search")
|
||||
|
||||
|
||||
class Report(Base):
|
||||
"""Report model."""
|
||||
|
||||
__tablename__ = "reports"
|
||||
|
||||
id = Column(String, primary_key=True, index=True, default=generate_uuid)
|
||||
user_id = Column(String, ForeignKey("users.id"))
|
||||
search_id = Column(String, ForeignKey("searches.id"), nullable=True)
|
||||
title = Column(String, nullable=False)
|
||||
content = Column(Text, nullable=False)
|
||||
detail_level = Column(String, nullable=False, default="standard")
|
||||
query_type = Column(String, nullable=True)
|
||||
model_used = Column(String, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow)
|
||||
|
||||
user = relationship("User", back_populates="reports")
|
||||
search = relationship("Search", back_populates="reports")
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
"""
|
||||
Database session management for the sim-search API.
|
||||
|
||||
This module provides utilities for creating and managing database sessions.
|
||||
"""
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
# Create SQLAlchemy engine
|
||||
engine = create_engine(
|
||||
settings.SQLALCHEMY_DATABASE_URI,
|
||||
pool_pre_ping=True,
|
||||
connect_args={"check_same_thread": False} if settings.SQLALCHEMY_DATABASE_URI.startswith("sqlite") else {},
|
||||
)
|
||||
|
||||
# Create session factory
|
||||
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
|
||||
# Create base class for models
|
||||
Base = declarative_base()
|
||||
|
||||
|
||||
def get_db():
|
||||
"""
|
||||
Get a database session.
|
||||
|
||||
Yields:
|
||||
SQLAlchemy session
|
||||
"""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
"""
|
||||
Main FastAPI application for the sim-search API.
|
||||
|
||||
This module defines the FastAPI application and includes all routes.
|
||||
"""
|
||||
|
||||
import os
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.openapi.docs import get_swagger_ui_html
|
||||
from fastapi.openapi.utils import get_openapi
|
||||
|
||||
from app.api.routes import query, search, report, auth
|
||||
from app.core.config import settings
|
||||
|
||||
# Create FastAPI app
|
||||
app = FastAPI(
|
||||
title=settings.PROJECT_NAME,
|
||||
description=settings.PROJECT_DESCRIPTION,
|
||||
version=settings.VERSION,
|
||||
docs_url=None, # Disable default docs
|
||||
redoc_url=None, # Disable default redoc
|
||||
)
|
||||
|
||||
# Set up CORS middleware
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings.CORS_ORIGINS,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# Include routers
|
||||
app.include_router(auth.router, prefix=f"{settings.API_V1_STR}/auth", tags=["Authentication"])
|
||||
app.include_router(query.router, prefix=f"{settings.API_V1_STR}/query", tags=["Query Processing"])
|
||||
app.include_router(search.router, prefix=f"{settings.API_V1_STR}/search", tags=["Search Execution"])
|
||||
app.include_router(report.router, prefix=f"{settings.API_V1_STR}/report", tags=["Report Generation"])
|
||||
|
||||
# Custom OpenAPI and documentation endpoints
|
||||
@app.get("/docs", include_in_schema=False)
|
||||
async def custom_swagger_ui_html():
|
||||
"""Custom Swagger UI documentation."""
|
||||
return get_swagger_ui_html(
|
||||
openapi_url=f"{settings.API_V1_STR}/openapi.json",
|
||||
title=f"{settings.PROJECT_NAME} - Swagger UI",
|
||||
oauth2_redirect_url=f"{settings.API_V1_STR}/docs/oauth2-redirect",
|
||||
swagger_js_url="https://cdn.jsdelivr.net/npm/swagger-ui-dist@5/swagger-ui-bundle.js",
|
||||
swagger_css_url="https://cdn.jsdelivr.net/npm/swagger-ui-dist@5/swagger-ui.css",
|
||||
)
|
||||
|
||||
@app.get(f"{settings.API_V1_STR}/openapi.json", include_in_schema=False)
|
||||
async def get_open_api_endpoint():
|
||||
"""Return OpenAPI schema."""
|
||||
return get_openapi(
|
||||
title=settings.PROJECT_NAME,
|
||||
version=settings.VERSION,
|
||||
description=settings.PROJECT_DESCRIPTION,
|
||||
routes=app.routes,
|
||||
)
|
||||
|
||||
@app.get("/", tags=["Status"])
|
||||
async def root():
|
||||
"""Root endpoint to check API status."""
|
||||
return {
|
||||
"status": "online",
|
||||
"version": settings.VERSION,
|
||||
"project": settings.PROJECT_NAME,
|
||||
"docs": "/docs"
|
||||
}
|
||||
|
||||
# Initialize components on startup
|
||||
@app.on_event("startup")
|
||||
async def startup_event():
|
||||
"""Initialize components on startup."""
|
||||
# Import here to avoid circular imports
|
||||
from app.services.report_service import initialize_report_generator
|
||||
|
||||
# Initialize report generator
|
||||
await initialize_report_generator()
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
"""
|
||||
Query schemas for the sim-search API.
|
||||
|
||||
This module defines the Pydantic models for query-related operations.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Optional, Any
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class QueryBase(BaseModel):
|
||||
"""Base query schema."""
|
||||
|
||||
query: str
|
||||
|
||||
|
||||
class QueryProcess(QueryBase):
|
||||
"""Query processing schema."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class QueryClassify(QueryBase):
|
||||
"""Query classification schema."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class SubQuestion(BaseModel):
|
||||
"""Sub-question schema."""
|
||||
|
||||
sub_question: str
|
||||
aspect: str
|
||||
priority: float
|
||||
|
||||
|
||||
class StructuredQuery(BaseModel):
|
||||
"""Structured query schema."""
|
||||
|
||||
original_query: str
|
||||
enhanced_query: Optional[str] = None
|
||||
type: Optional[str] = None
|
||||
intent: Optional[str] = None
|
||||
domain: Optional[str] = None
|
||||
confidence: Optional[float] = None
|
||||
reasoning: Optional[str] = None
|
||||
entities: Optional[List[str]] = None
|
||||
sub_questions: Optional[List[SubQuestion]] = None
|
||||
search_queries: Optional[Dict[str, str]] = None
|
||||
is_academic: Optional[bool] = None
|
||||
is_code: Optional[bool] = None
|
||||
is_current_events: Optional[bool] = None
|
||||
|
||||
|
||||
class ProcessedQuery(BaseModel):
|
||||
"""Processed query schema."""
|
||||
|
||||
original_query: str
|
||||
structured_query: StructuredQuery
|
||||
|
||||
class Config:
|
||||
"""Pydantic config."""
|
||||
|
||||
schema_extra = {
|
||||
"example": {
|
||||
"original_query": "What are the latest advancements in quantum computing?",
|
||||
"structured_query": {
|
||||
"original_query": "What are the latest advancements in quantum computing?",
|
||||
"enhanced_query": "What are the recent breakthroughs and developments in quantum computing technology, algorithms, and applications in the past 2 years?",
|
||||
"type": "exploratory",
|
||||
"intent": "research",
|
||||
"domain": "academic",
|
||||
"confidence": 0.95,
|
||||
"reasoning": "This query is asking about recent developments in a scientific field, which is typical of academic research.",
|
||||
"entities": ["quantum computing", "advancements"],
|
||||
"sub_questions": [
|
||||
{
|
||||
"sub_question": "What are the latest hardware advancements in quantum computing?",
|
||||
"aspect": "hardware",
|
||||
"priority": 0.9
|
||||
},
|
||||
{
|
||||
"sub_question": "What are the recent algorithmic breakthroughs in quantum computing?",
|
||||
"aspect": "algorithms",
|
||||
"priority": 0.8
|
||||
}
|
||||
],
|
||||
"search_queries": {
|
||||
"google": "latest advancements in quantum computing 2024",
|
||||
"scholar": "recent quantum computing breakthroughs",
|
||||
"arxiv": "quantum computing hardware algorithms"
|
||||
},
|
||||
"is_academic": True,
|
||||
"is_code": False,
|
||||
"is_current_events": False
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
"""
|
||||
Report schemas for the sim-search API.
|
||||
|
||||
This module defines the Pydantic models for report-related operations.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Optional, Any
|
||||
from datetime import datetime
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class ReportBase(BaseModel):
|
||||
"""Base report schema."""
|
||||
|
||||
title: Optional[str] = None
|
||||
detail_level: Optional[str] = "standard"
|
||||
query_type: Optional[str] = None
|
||||
model: Optional[str] = None
|
||||
|
||||
|
||||
class ReportCreate(ReportBase):
|
||||
"""Report creation schema."""
|
||||
|
||||
search_id: Optional[str] = None
|
||||
search_results: Optional[List[Dict[str, Any]]] = None
|
||||
query: str
|
||||
token_budget: Optional[int] = None
|
||||
chunk_size: Optional[int] = None
|
||||
overlap_size: Optional[int] = None
|
||||
|
||||
|
||||
class ReportUpdate(ReportBase):
|
||||
"""Report update schema."""
|
||||
|
||||
content: Optional[str] = None
|
||||
|
||||
|
||||
class ReportInDBBase(ReportBase):
|
||||
"""Base report in DB schema."""
|
||||
|
||||
id: str
|
||||
user_id: str
|
||||
search_id: Optional[str] = None
|
||||
content: str
|
||||
model_used: Optional[str] = None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
class Config:
|
||||
"""Pydantic config."""
|
||||
|
||||
orm_mode = True
|
||||
|
||||
|
||||
class Report(ReportInDBBase):
|
||||
"""Report schema."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class ReportList(BaseModel):
|
||||
"""Report list schema."""
|
||||
|
||||
reports: List[Report]
|
||||
total: int
|
||||
|
||||
|
||||
class ReportProgress(BaseModel):
|
||||
"""Report generation progress schema."""
|
||||
|
||||
report_id: str
|
||||
progress: float
|
||||
status: str
|
||||
current_chunk: Optional[int] = None
|
||||
total_chunks: Optional[int] = None
|
||||
current_report: Optional[str] = None
|
||||
|
||||
|
||||
class ReportDownload(BaseModel):
|
||||
"""Report download schema."""
|
||||
|
||||
report_id: str
|
||||
format: str = "markdown" # markdown, html, pdf
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
"""
|
||||
Search schemas for the sim-search API.
|
||||
|
||||
This module defines the Pydantic models for search-related operations.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Optional, Any
|
||||
from datetime import datetime
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.schemas.query import StructuredQuery
|
||||
|
||||
|
||||
class SearchResult(BaseModel):
|
||||
"""Search result schema."""
|
||||
|
||||
title: str
|
||||
url: str
|
||||
snippet: str
|
||||
source: str
|
||||
score: Optional[float] = None
|
||||
authors: Optional[str] = None
|
||||
year: Optional[str] = None
|
||||
pdf_url: Optional[str] = None
|
||||
arxiv_id: Optional[str] = None
|
||||
categories: Optional[List[str]] = None
|
||||
published_date: Optional[str] = None
|
||||
updated_date: Optional[str] = None
|
||||
full_text: Optional[str] = None
|
||||
|
||||
|
||||
class SearchExecute(BaseModel):
|
||||
"""Search execution schema."""
|
||||
|
||||
structured_query: StructuredQuery
|
||||
search_engines: Optional[List[str]] = None
|
||||
num_results: Optional[int] = 10
|
||||
timeout: Optional[int] = 30
|
||||
|
||||
|
||||
class SearchResults(BaseModel):
|
||||
"""Search results schema."""
|
||||
|
||||
search_id: str
|
||||
query: str
|
||||
enhanced_query: Optional[str] = None
|
||||
results: Dict[str, List[SearchResult]]
|
||||
total_results: int
|
||||
execution_time: float
|
||||
timestamp: datetime = Field(default_factory=datetime.utcnow)
|
||||
|
||||
|
||||
class SearchHistory(BaseModel):
|
||||
"""Search history schema."""
|
||||
|
||||
id: str
|
||||
query: str
|
||||
enhanced_query: Optional[str] = None
|
||||
query_type: Optional[str] = None
|
||||
engines: Optional[str] = None
|
||||
results_count: int
|
||||
created_at: datetime
|
||||
|
||||
class Config:
|
||||
"""Pydantic config."""
|
||||
|
||||
orm_mode = True
|
||||
|
||||
|
||||
class SearchHistoryList(BaseModel):
|
||||
"""Search history list schema."""
|
||||
|
||||
searches: List[SearchHistory]
|
||||
total: int
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
"""
|
||||
Token schemas for the sim-search API.
|
||||
|
||||
This module defines the Pydantic models for token-related operations.
|
||||
"""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class Token(BaseModel):
|
||||
"""Token schema."""
|
||||
|
||||
access_token: str
|
||||
token_type: str
|
||||
|
||||
|
||||
class TokenPayload(BaseModel):
|
||||
"""Token payload schema."""
|
||||
|
||||
sub: Optional[str] = None
|
||||
|
||||
|
||||
class TokenData(BaseModel):
|
||||
"""Token data schema."""
|
||||
|
||||
username: Optional[str] = None
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
"""
|
||||
User schemas for the sim-search API.
|
||||
|
||||
This module defines the Pydantic models for user-related operations.
|
||||
"""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import BaseModel, EmailStr
|
||||
|
||||
|
||||
class UserBase(BaseModel):
|
||||
"""Base user schema."""
|
||||
|
||||
email: Optional[EmailStr] = None
|
||||
is_active: Optional[bool] = True
|
||||
is_superuser: bool = False
|
||||
full_name: Optional[str] = None
|
||||
|
||||
|
||||
class UserCreate(UserBase):
|
||||
"""User creation schema."""
|
||||
|
||||
email: EmailStr
|
||||
password: str
|
||||
|
||||
|
||||
class UserUpdate(UserBase):
|
||||
"""User update schema."""
|
||||
|
||||
password: Optional[str] = None
|
||||
|
||||
|
||||
class UserInDBBase(UserBase):
|
||||
"""Base user in DB schema."""
|
||||
|
||||
id: Optional[str] = None
|
||||
|
||||
class Config:
|
||||
"""Pydantic config."""
|
||||
|
||||
orm_mode = True
|
||||
|
||||
|
||||
class User(UserInDBBase):
|
||||
"""User schema."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class UserInDB(UserInDBBase):
|
||||
"""User in DB schema."""
|
||||
|
||||
hashed_password: str
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
"""
|
||||
Query service for the sim-search API.
|
||||
|
||||
This module provides services for query processing and classification.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List, Optional
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
# Add sim-search to the python path
|
||||
sim_search_path = Path(settings.SIM_SEARCH_PATH)
|
||||
sys.path.append(str(sim_search_path))
|
||||
|
||||
# Import sim-search components
|
||||
from query.query_processor import QueryProcessor
|
||||
from query.llm_interface import LLMInterface
|
||||
|
||||
|
||||
class QueryService:
|
||||
"""
|
||||
Service for query processing and classification.
|
||||
|
||||
This class provides methods to process and classify queries using
|
||||
the sim-search query processing functionality.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize the query service."""
|
||||
self.query_processor = QueryProcessor()
|
||||
self.llm_interface = LLMInterface()
|
||||
|
||||
async def process_query(self, query: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Process a query to enhance and structure it.
|
||||
|
||||
Args:
|
||||
query: Query to process
|
||||
|
||||
Returns:
|
||||
Processed query with structured information
|
||||
"""
|
||||
# Process the query using the sim-search query processor
|
||||
structured_query = await self.query_processor.process_query(query)
|
||||
|
||||
# Format the response
|
||||
return {
|
||||
"original_query": query,
|
||||
"structured_query": structured_query
|
||||
}
|
||||
|
||||
async def classify_query(self, query: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Classify a query by type and intent.
|
||||
|
||||
Args:
|
||||
query: Query to classify
|
||||
|
||||
Returns:
|
||||
Classified query with type and intent information
|
||||
"""
|
||||
# Classify the query using the sim-search LLM interface
|
||||
classification = await self.llm_interface.classify_query_domain(query)
|
||||
|
||||
# Create a structured query with the classification
|
||||
structured_query = {
|
||||
"original_query": query,
|
||||
"type": classification.get("type"),
|
||||
"intent": classification.get("intent"),
|
||||
"domain": classification.get("domain"),
|
||||
"confidence": classification.get("confidence"),
|
||||
"reasoning": classification.get("reasoning"),
|
||||
"is_academic": classification.get("is_academic", False),
|
||||
"is_code": classification.get("is_code", False),
|
||||
"is_current_events": classification.get("is_current_events", False)
|
||||
}
|
||||
|
||||
# Format the response
|
||||
return {
|
||||
"original_query": query,
|
||||
"structured_query": structured_query
|
||||
}
|
||||
|
|
@ -0,0 +1,355 @@
|
|||
"""
|
||||
Report service for the sim-search API.
|
||||
|
||||
This module provides services for report generation and management.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import time
|
||||
import json
|
||||
import asyncio
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List, Optional, Union
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.config import settings
|
||||
from app.db.models import Search, Report
|
||||
from app.schemas.report import ReportCreate
|
||||
|
||||
# Add sim-search to the python path
|
||||
sim_search_path = Path(settings.SIM_SEARCH_PATH)
|
||||
sys.path.append(str(sim_search_path))
|
||||
|
||||
# Import sim-search components
|
||||
from report.report_generator import get_report_generator, initialize_report_generator
|
||||
from report.report_detail_levels import get_report_detail_level_manager
|
||||
from services.search_service import SearchService
|
||||
|
||||
|
||||
class ReportService:
|
||||
"""
|
||||
Service for report generation and management.
|
||||
|
||||
This class provides methods to generate and manage reports using
|
||||
the sim-search report generation functionality.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize the report service."""
|
||||
self.report_generator = None
|
||||
self.detail_level_manager = get_report_detail_level_manager()
|
||||
self.search_service = SearchService()
|
||||
self.temp_dir = Path(tempfile.gettempdir()) / "sim-search-api"
|
||||
self.temp_dir.mkdir(exist_ok=True)
|
||||
|
||||
async def initialize(self):
|
||||
"""Initialize the report generator."""
|
||||
await initialize_report_generator()
|
||||
self.report_generator = get_report_generator()
|
||||
|
||||
async def generate_report_background(
|
||||
self,
|
||||
report_id: str,
|
||||
report_in: ReportCreate,
|
||||
search: Optional[Search] = None,
|
||||
db: Optional[Session] = None,
|
||||
progress_dict: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
) -> None:
|
||||
"""
|
||||
Generate a report in the background.
|
||||
|
||||
Args:
|
||||
report_id: ID of the report
|
||||
report_in: Report creation parameters
|
||||
search: Search record
|
||||
db: Database session
|
||||
progress_dict: Dictionary to store progress information
|
||||
"""
|
||||
try:
|
||||
# Initialize report generator if not already initialized
|
||||
if self.report_generator is None:
|
||||
await self.initialize()
|
||||
|
||||
# Get search results
|
||||
search_results = []
|
||||
if search:
|
||||
# Use search results from the database
|
||||
search_results = search.results
|
||||
elif report_in.search_results:
|
||||
# Use search results provided in the request
|
||||
search_results = report_in.search_results
|
||||
else:
|
||||
# Execute a new search
|
||||
structured_query = {
|
||||
"original_query": report_in.query,
|
||||
"enhanced_query": report_in.query,
|
||||
}
|
||||
|
||||
search_results_dict = await self.search_service.execute_search(
|
||||
structured_query=structured_query,
|
||||
num_results=10,
|
||||
)
|
||||
|
||||
# Flatten search results
|
||||
for engine_results in search_results_dict["results"].values():
|
||||
search_results.extend(engine_results)
|
||||
|
||||
# Set up progress tracking
|
||||
if progress_dict is not None:
|
||||
def progress_callback(current_progress, total_chunks, current_report):
|
||||
if report_id in progress_dict:
|
||||
progress_dict[report_id] = {
|
||||
"progress": current_progress,
|
||||
"status": f"Processing chunk {int(current_progress * total_chunks)}/{total_chunks}...",
|
||||
"current_chunk": int(current_progress * total_chunks),
|
||||
"total_chunks": total_chunks,
|
||||
"current_report": current_report,
|
||||
}
|
||||
|
||||
self.report_generator.set_progress_callback(progress_callback)
|
||||
|
||||
# Set detail level
|
||||
if report_in.detail_level:
|
||||
self.report_generator.set_detail_level(report_in.detail_level)
|
||||
|
||||
# Set model if provided
|
||||
if report_in.model:
|
||||
self.report_generator.set_model(report_in.model)
|
||||
|
||||
# Generate report
|
||||
report_content = await self.report_generator.generate_report(
|
||||
search_results=search_results,
|
||||
query=report_in.query,
|
||||
token_budget=report_in.token_budget,
|
||||
chunk_size=report_in.chunk_size,
|
||||
overlap_size=report_in.overlap_size,
|
||||
detail_level=report_in.detail_level,
|
||||
query_type=report_in.query_type,
|
||||
)
|
||||
|
||||
# Update report in database
|
||||
if db:
|
||||
report = db.query(Report).filter(Report.id == report_id).first()
|
||||
if report:
|
||||
report.content = report_content
|
||||
report.model_used = self.report_generator.model_name
|
||||
db.commit()
|
||||
|
||||
# Update progress
|
||||
if progress_dict is not None and report_id in progress_dict:
|
||||
progress_dict[report_id] = {
|
||||
"progress": 1.0,
|
||||
"status": "Report generation complete",
|
||||
"current_chunk": 0,
|
||||
"total_chunks": 0,
|
||||
"current_report": None,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
# Update progress with error
|
||||
if progress_dict is not None and report_id in progress_dict:
|
||||
progress_dict[report_id] = {
|
||||
"progress": 1.0,
|
||||
"status": f"Error generating report: {str(e)}",
|
||||
"current_chunk": 0,
|
||||
"total_chunks": 0,
|
||||
"current_report": None,
|
||||
}
|
||||
|
||||
# Update report in database with error
|
||||
if db:
|
||||
report = db.query(Report).filter(Report.id == report_id).first()
|
||||
if report:
|
||||
report.content = f"Error generating report: {str(e)}"
|
||||
db.commit()
|
||||
|
||||
# Re-raise the exception
|
||||
raise
|
||||
|
||||
async def generate_report_file(self, report: Report, format: str = "markdown") -> str:
|
||||
"""
|
||||
Generate a report file in the specified format.
|
||||
|
||||
Args:
|
||||
report: Report record
|
||||
format: Format of the report (markdown, html, pdf)
|
||||
|
||||
Returns:
|
||||
Path to the generated file
|
||||
"""
|
||||
# Create a temporary file
|
||||
file_path = self.temp_dir / f"report_{report.id}.{format}"
|
||||
|
||||
# Write the report content to the file
|
||||
if format == "markdown":
|
||||
with open(file_path, "w") as f:
|
||||
f.write(report.content)
|
||||
elif format == "html":
|
||||
# Convert markdown to HTML
|
||||
import markdown
|
||||
html_content = markdown.markdown(report.content)
|
||||
|
||||
# Add HTML wrapper
|
||||
html_content = f"""
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>{report.title}</title>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<style>
|
||||
body {{
|
||||
font-family: Arial, sans-serif;
|
||||
line-height: 1.6;
|
||||
max-width: 800px;
|
||||
margin: 0 auto;
|
||||
padding: 20px;
|
||||
}}
|
||||
h1, h2, h3, h4, h5, h6 {{
|
||||
margin-top: 1.5em;
|
||||
margin-bottom: 0.5em;
|
||||
}}
|
||||
a {{
|
||||
color: #0366d6;
|
||||
text-decoration: none;
|
||||
}}
|
||||
a:hover {{
|
||||
text-decoration: underline;
|
||||
}}
|
||||
pre {{
|
||||
background-color: #f6f8fa;
|
||||
border-radius: 3px;
|
||||
padding: 16px;
|
||||
overflow: auto;
|
||||
}}
|
||||
code {{
|
||||
background-color: #f6f8fa;
|
||||
border-radius: 3px;
|
||||
padding: 0.2em 0.4em;
|
||||
font-family: monospace;
|
||||
}}
|
||||
blockquote {{
|
||||
border-left: 4px solid #dfe2e5;
|
||||
padding-left: 16px;
|
||||
margin-left: 0;
|
||||
color: #6a737d;
|
||||
}}
|
||||
table {{
|
||||
border-collapse: collapse;
|
||||
width: 100%;
|
||||
}}
|
||||
table, th, td {{
|
||||
border: 1px solid #dfe2e5;
|
||||
}}
|
||||
th, td {{
|
||||
padding: 8px 16px;
|
||||
text-align: left;
|
||||
}}
|
||||
tr:nth-child(even) {{
|
||||
background-color: #f6f8fa;
|
||||
}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
{html_content}
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
with open(file_path, "w") as f:
|
||||
f.write(html_content)
|
||||
elif format == "pdf":
|
||||
# Convert markdown to PDF
|
||||
try:
|
||||
import markdown
|
||||
from weasyprint import HTML
|
||||
|
||||
# Convert markdown to HTML
|
||||
html_content = markdown.markdown(report.content)
|
||||
|
||||
# Add HTML wrapper
|
||||
html_content = f"""
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>{report.title}</title>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<style>
|
||||
body {{
|
||||
font-family: Arial, sans-serif;
|
||||
line-height: 1.6;
|
||||
max-width: 800px;
|
||||
margin: 0 auto;
|
||||
padding: 20px;
|
||||
}}
|
||||
h1, h2, h3, h4, h5, h6 {{
|
||||
margin-top: 1.5em;
|
||||
margin-bottom: 0.5em;
|
||||
}}
|
||||
a {{
|
||||
color: #0366d6;
|
||||
text-decoration: none;
|
||||
}}
|
||||
pre {{
|
||||
background-color: #f6f8fa;
|
||||
border-radius: 3px;
|
||||
padding: 16px;
|
||||
overflow: auto;
|
||||
}}
|
||||
code {{
|
||||
background-color: #f6f8fa;
|
||||
border-radius: 3px;
|
||||
padding: 0.2em 0.4em;
|
||||
font-family: monospace;
|
||||
}}
|
||||
blockquote {{
|
||||
border-left: 4px solid #dfe2e5;
|
||||
padding-left: 16px;
|
||||
margin-left: 0;
|
||||
color: #6a737d;
|
||||
}}
|
||||
table {{
|
||||
border-collapse: collapse;
|
||||
width: 100%;
|
||||
}}
|
||||
table, th, td {{
|
||||
border: 1px solid #dfe2e5;
|
||||
}}
|
||||
th, td {{
|
||||
padding: 8px 16px;
|
||||
text-align: left;
|
||||
}}
|
||||
tr:nth-child(even) {{
|
||||
background-color: #f6f8fa;
|
||||
}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
{html_content}
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
# Create a temporary HTML file
|
||||
html_file_path = self.temp_dir / f"report_{report.id}.html"
|
||||
with open(html_file_path, "w") as f:
|
||||
f.write(html_content)
|
||||
|
||||
# Convert HTML to PDF
|
||||
HTML(filename=str(html_file_path)).write_pdf(str(file_path))
|
||||
|
||||
# Remove temporary HTML file
|
||||
html_file_path.unlink()
|
||||
except ImportError:
|
||||
# If weasyprint is not installed, fall back to markdown
|
||||
with open(file_path, "w") as f:
|
||||
f.write(report.content)
|
||||
else:
|
||||
# Unsupported format, fall back to markdown
|
||||
with open(file_path, "w") as f:
|
||||
f.write(report.content)
|
||||
|
||||
return str(file_path)
|
||||
|
|
@ -0,0 +1,146 @@
|
|||
"""
|
||||
Search service for the sim-search API.
|
||||
|
||||
This module provides services for search execution and result management.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import time
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List, Optional, Union
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.config import settings
|
||||
from app.db.models import Search
|
||||
|
||||
# Add sim-search to the python path
|
||||
sim_search_path = Path(settings.SIM_SEARCH_PATH)
|
||||
sys.path.append(str(sim_search_path))
|
||||
|
||||
# Import sim-search components
|
||||
from execution.search_executor import SearchExecutor
|
||||
from execution.result_collector import ResultCollector
|
||||
|
||||
|
||||
class SearchService:
|
||||
"""
|
||||
Service for search execution and result management.
|
||||
|
||||
This class provides methods to execute searches and manage search results
|
||||
using the sim-search search execution functionality.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize the search service."""
|
||||
self.search_executor = SearchExecutor()
|
||||
self.result_collector = ResultCollector()
|
||||
|
||||
async def get_available_search_engines(self) -> List[str]:
|
||||
"""
|
||||
Get a list of available search engines.
|
||||
|
||||
Returns:
|
||||
List of available search engine names
|
||||
"""
|
||||
return self.search_executor.get_available_search_engines()
|
||||
|
||||
async def execute_search(
|
||||
self,
|
||||
structured_query: Dict[str, Any],
|
||||
search_engines: Optional[List[str]] = None,
|
||||
num_results: Optional[int] = 10,
|
||||
timeout: Optional[int] = 30,
|
||||
user_id: Optional[str] = None,
|
||||
db: Optional[Session] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Execute a search with the given parameters.
|
||||
|
||||
Args:
|
||||
structured_query: Structured query
|
||||
search_engines: List of search engines to use
|
||||
num_results: Number of results to return per search engine
|
||||
timeout: Timeout in seconds
|
||||
user_id: User ID for storing the search
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
Search results
|
||||
"""
|
||||
# Start timing
|
||||
start_time = time.time()
|
||||
|
||||
# Add search engines if not specified
|
||||
if not search_engines:
|
||||
search_engines = self.search_executor.get_available_search_engines()
|
||||
structured_query["search_engines"] = search_engines
|
||||
|
||||
# Execute the search
|
||||
search_results = self.search_executor.execute_search(
|
||||
structured_query=structured_query,
|
||||
num_results=num_results
|
||||
)
|
||||
|
||||
# Calculate execution time
|
||||
execution_time = time.time() - start_time
|
||||
|
||||
# Process results
|
||||
processed_results = self.result_collector.process_results(
|
||||
search_results, dedup=True, max_results=None, use_reranker=True
|
||||
)
|
||||
|
||||
# Create search record if user_id and db are provided
|
||||
search_id = None
|
||||
if user_id and db:
|
||||
# Create search record
|
||||
engines_str = ",".join(search_engines) if search_engines else ""
|
||||
search = Search(
|
||||
user_id=user_id,
|
||||
query=structured_query.get("original_query", ""),
|
||||
enhanced_query=structured_query.get("enhanced_query", ""),
|
||||
query_type=structured_query.get("type", ""),
|
||||
engines=engines_str,
|
||||
results_count=len(processed_results),
|
||||
results=processed_results,
|
||||
)
|
||||
|
||||
db.add(search)
|
||||
db.commit()
|
||||
db.refresh(search)
|
||||
|
||||
search_id = search.id
|
||||
|
||||
# Format the response
|
||||
return {
|
||||
"search_id": search_id,
|
||||
"query": structured_query.get("original_query", ""),
|
||||
"enhanced_query": structured_query.get("enhanced_query", ""),
|
||||
"results": {engine: results for engine, results in search_results.items()},
|
||||
"total_results": sum(len(results) for results in search_results.values()),
|
||||
"execution_time": execution_time,
|
||||
}
|
||||
|
||||
async def get_search_results(self, search: Search) -> Dict[str, Any]:
|
||||
"""
|
||||
Get results for a specific search.
|
||||
|
||||
Args:
|
||||
search: Search record
|
||||
|
||||
Returns:
|
||||
Search results
|
||||
"""
|
||||
# Parse engines string
|
||||
engines = search.engines.split(",") if search.engines else []
|
||||
|
||||
# Format the response
|
||||
return {
|
||||
"search_id": search.id,
|
||||
"query": search.query,
|
||||
"enhanced_query": search.enhanced_query,
|
||||
"results": search.results,
|
||||
"total_results": search.results_count,
|
||||
"execution_time": 0.0, # Not available for stored searches
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
# FastAPI and ASGI server
|
||||
fastapi==0.103.1
|
||||
uvicorn==0.23.2
|
||||
|
||||
# Database
|
||||
sqlalchemy==2.0.21
|
||||
alembic==1.12.0
|
||||
|
||||
# Authentication
|
||||
python-jose==3.3.0
|
||||
passlib==1.7.4
|
||||
bcrypt==4.0.1
|
||||
python-multipart==0.0.6
|
||||
|
||||
# Validation and serialization
|
||||
pydantic==2.4.2
|
||||
email-validator==2.0.0
|
||||
|
||||
# Testing
|
||||
pytest==7.4.2
|
||||
httpx==0.25.0
|
||||
|
||||
# Utilities
|
||||
python-dotenv==1.0.0
|
||||
aiofiles==23.2.1
|
||||
jinja2==3.1.2
|
||||
|
||||
# Report generation
|
||||
markdown==3.4.4
|
||||
weasyprint==60.1 # Optional, for PDF generation
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Run script for the sim-search API.
|
||||
|
||||
This script launches the FastAPI application using uvicorn.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import uvicorn
|
||||
|
||||
|
||||
def parse_args():
|
||||
"""Parse command line arguments."""
|
||||
parser = argparse.ArgumentParser(description="Run the sim-search API")
|
||||
parser.add_argument(
|
||||
"--host",
|
||||
type=str,
|
||||
default="127.0.0.1",
|
||||
help="Host to run the server on",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--port",
|
||||
type=int,
|
||||
default=8000,
|
||||
help="Port to run the server on",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--reload",
|
||||
action="store_true",
|
||||
help="Enable auto-reload for development",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--debug",
|
||||
action="store_true",
|
||||
help="Run in debug mode",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
"""Main function to run the API."""
|
||||
args = parse_args()
|
||||
|
||||
print(f"Starting sim-search API on {args.host}:{args.port}...")
|
||||
|
||||
uvicorn.run(
|
||||
"app.main:app",
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
reload=args.reload,
|
||||
log_level="debug" if args.debug else "info",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Reference in New Issue