Jarian Cottingham
f4b84cc412
- Replace eval() with json.loads() in database.py (RCE fix) - Use json.dumps() for safe storage of list fields - Add API key authentication middleware - Remove hardcoded credentials, require env vars - Disable Flask debug mode - Restrict FTP homedir to /app/data with read-only perms - Fix threading: Lock -> RLock, add WAL mode - Fix API calls to use correct DatabaseManager methods - Fix main.py FTP method names - Fix click.click.echo typo - Implement scheduler _run_all_jobs - Add __main__.py for module execution - Pin dependency versions - Use .env vars in docker-compose, read-only DB for FTP - Implement AI text chunking with overlap windows - Add schema validation for AI responses - Skip unsupported file types instead of fallback
130 lines
4.3 KiB
Python
130 lines
4.3 KiB
Python
"""
|
|
CLI interface for FactsDB service
|
|
Provides command line interface for onboarding directories
|
|
"""
|
|
|
|
import os
|
|
import click
|
|
from typing import Optional
|
|
from .config import Config
|
|
from .database import DatabaseManager
|
|
from .file_processor import FileProcessor
|
|
from .ai_processor import AIProcessor
|
|
from .scheduler import FactExtractionScheduler, get_scheduler
|
|
from .ftp_server import FTPServerManager
|
|
|
|
@click.group()
|
|
def cli():
|
|
"""FactsDB - Service for extracting and storing facts from documents"""
|
|
pass
|
|
|
|
@cli.command()
|
|
@click.option('--directory', '-d', required=True, help='Directory to onboard')
|
|
@click.option('--table-name', '-t', required=True, help='Name of table to store facts')
|
|
@click.option('--prompt', '-p', help='Custom prompt for AI extraction')
|
|
@click.option('--model', '-m', default='gpt-oss', help='AI model to use (default: gpt-oss)')
|
|
@click.option('--db-type', '-db', default='sqlite', help='Database type (default: sqlite)')
|
|
def onboard(directory: str, table_name: str, prompt: Optional[str], model: str, db_type: str):
|
|
"""Onboard a directory for fact extraction"""
|
|
click.echo(f"Onboarding directory: {directory}")
|
|
click.echo(f"Table name: {table_name}")
|
|
click.echo(f"AI model: {model}")
|
|
|
|
# Validate directory exists
|
|
if not os.path.exists(directory):
|
|
click.echo(f"Error: Directory {directory} does not exist")
|
|
return
|
|
|
|
# Create configuration
|
|
config = Config()
|
|
|
|
# Setup components
|
|
db_manager = DatabaseManager(config.database)
|
|
file_processor = FileProcessor()
|
|
ai_processor = AIProcessor(config.ai_endpoint)
|
|
scheduler = get_scheduler(config)
|
|
|
|
# Add job to scheduler
|
|
scheduler.add_job(directory, table_name, prompt or "", model)
|
|
|
|
click.echo(f"Successfully onboarded directory {directory} for fact extraction")
|
|
click.echo(f"Job scheduled to run every 10 minutes")
|
|
|
|
@cli.command()
|
|
def status():
|
|
"""Show current status and scheduled jobs"""
|
|
config = Config()
|
|
try:
|
|
scheduler = get_scheduler(config)
|
|
|
|
click.echo("FactsDB Service Status")
|
|
click.echo("=" * 30)
|
|
|
|
jobs = scheduler.get_jobs()
|
|
if jobs:
|
|
click.echo("Scheduled Jobs:")
|
|
for i, job in enumerate(jobs, 1):
|
|
click.echo(f" {i}. Directory: {job['directory_path']}")
|
|
click.echo(f" Table: {job['table_name']}")
|
|
click.echo(f" Model: {job['model']}")
|
|
click.echo(f" Interval: {job['interval_minutes']} minutes")
|
|
click.echo()
|
|
else:
|
|
click.echo("No scheduled jobs")
|
|
except Exception as e:
|
|
click.echo(f"Error getting scheduler status: {str(e)}")
|
|
click.echo("Scheduler may not be initialized due to database configuration issues.")
|
|
|
|
@cli.command()
|
|
@click.option('--host', '-h', default='0.0.0.0', help='FTP server host')
|
|
@click.option('--port', '-p', default=2122, type=int, help='FTP server port')
|
|
def ftp(host: str, port: int):
|
|
"""Start FTP server"""
|
|
config = Config()
|
|
# Fix: Use the correct config structure - ftp_server instead of ftp
|
|
config.ftp_server.host = host
|
|
config.ftp_server.port = port
|
|
|
|
# Fix: Pass the correct config object to FTPServerManager
|
|
ftp_manager = FTPServerManager(config)
|
|
|
|
# Add current directory as allowed directory
|
|
current_dir = os.path.abspath('.')
|
|
ftp_manager.add_onboarded_directory(current_dir)
|
|
|
|
try:
|
|
ftp_manager.start()
|
|
click.echo(f"FTP Server started on {host}:{port}")
|
|
click.echo("Press Ctrl+C to stop")
|
|
|
|
# Keep running
|
|
try:
|
|
while True:
|
|
import time
|
|
time.sleep(1)
|
|
except KeyboardInterrupt:
|
|
ftp_manager.stop()
|
|
click.echo("FTP Server stopped")
|
|
|
|
except Exception as e:
|
|
click.echo(f"Error starting FTP server: {str(e)}")
|
|
|
|
@cli.command()
|
|
def tables():
|
|
"""Show all available tables and record counts"""
|
|
config = Config()
|
|
db_manager = DatabaseManager(config.database)
|
|
|
|
tables = db_manager.get_all_tables()
|
|
|
|
if not tables:
|
|
click.echo("No tables found")
|
|
return
|
|
|
|
click.echo("Available Tables:")
|
|
click.echo("=" * 40)
|
|
for table in tables:
|
|
click.echo(f"{table['table_name']}: {table['count']} records")
|
|
|
|
if __name__ == '__main__':
|
|
cli() |