news-mcp/news_mcp/config.py

import os
from pathlib import Path

from dotenv import load_dotenv

_HERE = Path(__file__).resolve().parent.parent
load_dotenv(_HERE / ".env")

DATA_DIR = Path(os.getenv("NEWS_MCP_DATA_DIR", _HERE / "data"))
DATA_DIR.mkdir(parents=True, exist_ok=True)
DB_PATH = Path(os.getenv("NEWS_MCP_DB_PATH", str(DATA_DIR / "news.sqlite")))
PROMPTS_DIR = Path(os.getenv("NEWS_PROMPTS_DIR", str(_HERE / "prompts")))
ENTITY_ALIASES_FILE = Path(os.getenv("NEWS_ENTITY_ALIASES_FILE", str(_HERE / "config" / "entity_aliases.json")))

NEWS_FEED_URL = os.getenv("NEWS_FEED_URL", os.getenv("NEWS_RSS_FEED_URL", "https://breakingthenews.net/news-feed.xml"))
NEWS_FEED_URLS = os.getenv("NEWS_FEED_URLS", os.getenv("NEWS_RSS_FEED_URLS", "")).strip()
RSS_FEED_URL = NEWS_FEED_URL
RSS_FEED_URLS = NEWS_FEED_URLS
NEWS_FEED_ITEMS_PER_POLL = int(os.getenv("NEWS_FEED_ITEMS_PER_POLL", "50"))

DEFAULT_LOOKBACK_HOURS = float(os.getenv("NEWS_DEFAULT_LOOKBACK_HOURS", os.getenv("NEWS_CLUSTERS_TTL_HOURS", "24")))
DEFAULT_TOPICS = ["crypto", "macro", "regulation", "ai", "other"]

# LLM API keys (provider-specific keys only)
GROQ_API_KEY = os.getenv("GROQ_API_KEY")
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
OPENROUTER_API_KEY = os.getenv("OPENROUTER_API_KEY")

# LLM provider/model selection (provider-agnostic)
NEWS_EXTRACT_PROVIDER = os.getenv("NEWS_EXTRACT_PROVIDER", "groq")
NEWS_EXTRACT_MODEL = os.getenv("NEWS_EXTRACT_MODEL", "llama4-16e")
NEWS_SUMMARY_PROVIDER = os.getenv("NEWS_SUMMARY_PROVIDER", "groq")
NEWS_SUMMARY_MODEL = os.getenv("NEWS_SUMMARY_MODEL", "llama4-16e")

# LLM behavior
LLM_DEBUG = os.getenv("LLM_DEBUG", "false").lower() == "true"

NEWS_ENTITY_BLACKLIST = [x.strip().lower() for x in os.getenv("ENTITY_BLACKLIST", "").split(",") if x.strip()]

# Enrichment: 0 = no limit (enrich every cluster); >0 caps per refresh cycle
ENRICHMENT_MAX_PER_REFRESH = int(os.getenv("ENRICHMENT_MAX_PER_REFRESH", "0"))
# When true, only the "other" topic gets LLM enrichment (legacy guard)
ENRICH_OTHER_TOPICS_ONLY = os.getenv("ENRICH_OTHER_TOPICS_ONLY", "false").lower() == "true"

# Optional embeddings path (Ollama first when enabled, fallback otherwise).
NEWS_EMBEDDINGS_ENABLED = os.getenv("NEWS_EMBEDDINGS_ENABLED", "false").lower() == "true"
OLLAMA_BASE_URL = os.getenv("OLLAMA_BASE_URL", os.getenv("OLLAMA_URL", "http://127.0.0.1:11434"))
OLLAMA_EMBEDDING_MODEL = os.getenv("OLLAMA_EMBEDDING_MODEL", "nomic-embed-text")
NEWS_EMBEDDING_SIMILARITY_THRESHOLD = float(os.getenv("NEWS_EMBEDDING_SIMILARITY_THRESHOLD", "0.885"))

# Cluster merge window: how far back (hours) to load existing clusters from
# the DB for cross-cycle merging.  0 = disabled (no cross-cycle merge).
NEWS_CLUSTER_MAX_AGE_HOURS = float(os.getenv("NEWS_CLUSTER_MAX_AGE_HOURS", "4"))

NEWS_REFRESH_INTERVAL_SECONDS = int(os.getenv("NEWS_REFRESH_INTERVAL_SECONDS", "900"))
NEWS_BACKGROUND_REFRESH_ENABLED = os.getenv("NEWS_BACKGROUND_REFRESH_ENABLED", "true").lower() == "true"
NEWS_BACKGROUND_REFRESH_ON_START = os.getenv("NEWS_BACKGROUND_REFRESH_ON_START", "true").lower() == "true"
NEWS_PRUNING_ENABLED = os.getenv("NEWS_PRUNING_ENABLED", "true").lower() == "true"
NEWS_RETENTION_DAYS = float(os.getenv("NEWS_RETENTION_DAYS", "180"))
NEWS_PRUNE_INTERVAL_HOURS = float(os.getenv("NEWS_PRUNE_INTERVAL_HOURS", "24"))

# ---------------------------------------------------------------------------
# Concurrency controls
# ---------------------------------------------------------------------------
# Maximum concurrent outbound LLM API calls per provider.
# Defaults are conservative for free tiers; override via env if you have
# higher rate limits or are on a paid plan.
_NEEDLE_DEFAULT_CONCURRENCY = {
    "openrouter": 2,
    "openai": 5,
    "groq": 8,
}

# Per-provider rate limits (requests per second).
# Env: NEWS_LLM_RATE_LIMIT_<PROVIDER> (float, calls/sec). 0 = disabled.
_NEEDLE_DEFAULT_RATE_LIMIT = {
    "openrouter": 2.0,
    "openai": 5.0,
    "groq": 1.0,
}

_NEEDLE_RSS_MAX_CONCURRENCY = int(os.getenv("NEWS_RSS_MAX_CONCURRENCY", "10"))
_NEEDLE_OLLAMA_MAX_CONCURRENCY = int(os.getenv("NEWS_OLLAMA_MAX_CONCURRENCY", "4"))


def llm_concurrency(provider: str) -> int:
    """Return the max concurrent LLM calls for *provider*.

    Reads from ``NEWS_LLM_CONCURRENCY_<PROVIDER>`` env var first (e.g.
    ``NEWS_LLM_CONCURRENCY_OPENROUTER``), then falls back to the built-in
    default map.
    """
    provider = provider.strip().lower()
    env_key = f"NEWS_LLM_CONCURRENCY_{provider.upper()}"
    env_val = os.getenv(env_key)
    if env_val is not None:
        try:
            return max(1, int(env_val))
        except ValueError:
            pass
    return _NEEDLE_DEFAULT_CONCURRENCY.get(provider, 3)


def llm_rate_limit(provider: str) -> float:
    """Return the max LLM calls per second for *provider*.

    Reads from ``NEWS_LLM_RATE_LIMIT_<PROVIDER>`` env var first (e.g.
    ``NEWS_LLM_RATE_LIMIT_GROQ``), then falls back to the built-in default.
    Returns 0.0 when rate limiting is disabled.
    """
    provider = provider.strip().lower()
    env_key = f"NEWS_LLM_RATE_LIMIT_{provider.upper()}"
    env_val = os.getenv(env_key)
    if env_val is not None:
        try:
            return max(0.0, float(env_val))
        except ValueError:
            pass
    return _NEEDLE_DEFAULT_RATE_LIMIT.get(provider, 0.0)