93 lines
3.2 KiB
Python
93 lines
3.2 KiB
Python
"""Abstract base class for AYN threat-intelligence feeds."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import time
|
|
from abc import ABC, abstractmethod
|
|
from datetime import datetime
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class BaseFeed(ABC):
|
|
"""Common interface for all external threat-intelligence feeds.
|
|
|
|
Provides rate-limiting, last-updated tracking, and a uniform
|
|
``fetch()`` contract so the :class:`SignatureManager` can orchestrate
|
|
updates without knowing feed internals.
|
|
|
|
Parameters
|
|
----------
|
|
rate_limit_seconds:
|
|
Minimum interval between successive HTTP requests to the same feed.
|
|
"""
|
|
|
|
def __init__(self, rate_limit_seconds: float = 2.0) -> None:
|
|
self._rate_limit = rate_limit_seconds
|
|
self._last_request_time: float = 0.0
|
|
self._last_updated: Optional[datetime] = None
|
|
|
|
# ------------------------------------------------------------------
|
|
# Identity
|
|
# ------------------------------------------------------------------
|
|
|
|
@abstractmethod
|
|
def get_name(self) -> str:
|
|
"""Return a short, human-readable feed name."""
|
|
...
|
|
|
|
# ------------------------------------------------------------------
|
|
# Fetching
|
|
# ------------------------------------------------------------------
|
|
|
|
@abstractmethod
|
|
def fetch(self) -> List[Dict[str, Any]]:
|
|
"""Download the latest entries from the feed.
|
|
|
|
Returns a list of dicts. The exact keys depend on the feed type
|
|
(hashes, IOCs, rules, etc.). The :class:`SignatureManager` is
|
|
responsible for routing each entry to the correct database.
|
|
"""
|
|
...
|
|
|
|
# ------------------------------------------------------------------
|
|
# State
|
|
# ------------------------------------------------------------------
|
|
|
|
@property
|
|
def last_updated(self) -> Optional[datetime]:
|
|
"""Timestamp of the most recent successful fetch."""
|
|
return self._last_updated
|
|
|
|
def _mark_updated(self) -> None:
|
|
"""Record the current time as the last-successful-fetch timestamp."""
|
|
self._last_updated = datetime.utcnow()
|
|
|
|
# ------------------------------------------------------------------
|
|
# Rate limiting
|
|
# ------------------------------------------------------------------
|
|
|
|
def _rate_limit_wait(self) -> None:
|
|
"""Block until the rate-limit window has elapsed."""
|
|
elapsed = time.monotonic() - self._last_request_time
|
|
remaining = self._rate_limit - elapsed
|
|
if remaining > 0:
|
|
logger.debug("[%s] Rate-limiting: sleeping %.1fs", self.get_name(), remaining)
|
|
time.sleep(remaining)
|
|
self._last_request_time = time.monotonic()
|
|
|
|
# ------------------------------------------------------------------
|
|
# Logging helpers
|
|
# ------------------------------------------------------------------
|
|
|
|
def _log(self, msg: str, *args: Any) -> None:
|
|
logger.info("[%s] " + msg, self.get_name(), *args)
|
|
|
|
def _warn(self, msg: str, *args: Any) -> None:
|
|
logger.warning("[%s] " + msg, self.get_name(), *args)
|
|
|
|
def _error(self, msg: str, *args: Any) -> None:
|
|
logger.error("[%s] " + msg, self.get_name(), *args)
|