442 lines
19 KiB
Python
442 lines
19 KiB
Python
"""Configuration loading and validation.
|
|
|
|
Every setting comes from environment variables, optionally loaded from a ``.env``
|
|
file next to this module. No secret is hard-coded anywhere in the project.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import re
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Mapping
|
|
|
|
PROJECT_DIR = Path(__file__).resolve().parent
|
|
UPLOAD_CHUNK_ALIGN = 256 * 1024 # Google requires resumable chunks in multiples of 256 KiB
|
|
|
|
_TRUE = {"1", "true", "yes", "on"}
|
|
_FALSE = {"0", "false", "no", "off"}
|
|
_SIZE_RE = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*([KMGT]?I?B?)?\s*$", re.IGNORECASE)
|
|
_SIZE_UNITS = {"": 1, "B": 1, "K": 1024, "M": 1024**2, "G": 1024**3, "T": 1024**4}
|
|
|
|
|
|
class ConfigError(Exception):
|
|
def __init__(self, problems: list[str]):
|
|
self.problems = problems
|
|
super().__init__("Configuration problems:\n" + "\n".join(f" - {p}" for p in problems))
|
|
|
|
|
|
class _Reader:
|
|
"""Typed accessors that collect every problem instead of stopping at the first."""
|
|
|
|
def __init__(self, env: Mapping[str, str]):
|
|
self.env = env
|
|
self.problems: list[str] = []
|
|
|
|
def str(self, name: str, default: str | None = None) -> str | None:
|
|
raw = self.env.get(name)
|
|
if raw is None:
|
|
return default
|
|
raw = raw.strip()
|
|
return raw if raw else default
|
|
|
|
def int(self, name: str, default: int | None = None, *, minimum: int | None = None) -> int | None:
|
|
raw = self.str(name)
|
|
if raw is None:
|
|
return default
|
|
try:
|
|
value = int(raw)
|
|
except ValueError:
|
|
self.problems.append(f"{name} must be an integer (got {raw!r})")
|
|
return default
|
|
if minimum is not None and value < minimum:
|
|
self.problems.append(f"{name} must be >= {minimum} (got {value})")
|
|
return default
|
|
return value
|
|
|
|
def float(self, name: str, default: float, *, minimum: float = 0.0) -> float:
|
|
raw = self.str(name)
|
|
if raw is None:
|
|
return default
|
|
try:
|
|
value = float(raw)
|
|
except ValueError:
|
|
self.problems.append(f"{name} must be a number (got {raw!r})")
|
|
return default
|
|
if value < minimum:
|
|
self.problems.append(f"{name} must be >= {minimum} (got {value})")
|
|
return default
|
|
return value
|
|
|
|
def bool(self, name: str, default: bool) -> bool:
|
|
raw = self.str(name)
|
|
if raw is None:
|
|
return default
|
|
low = raw.lower()
|
|
if low in _TRUE:
|
|
return True
|
|
if low in _FALSE:
|
|
return False
|
|
self.problems.append(f"{name} must be true or false (got {raw!r})")
|
|
return default
|
|
|
|
def choice(self, name: str, default: str, choices: tuple[str, ...]) -> str:
|
|
raw = self.str(name, default)
|
|
value = raw.lower()
|
|
if value not in choices:
|
|
self.problems.append(f"{name} must be one of {', '.join(choices)} (got {raw!r})")
|
|
return default
|
|
return value
|
|
|
|
def int_list(self, name: str) -> tuple[int, ...]:
|
|
raw = self.str(name)
|
|
if raw is None:
|
|
return ()
|
|
values = []
|
|
for part in raw.split(","):
|
|
part = part.strip()
|
|
if not part:
|
|
continue
|
|
try:
|
|
values.append(int(part))
|
|
except ValueError:
|
|
self.problems.append(f"{name} must be a comma-separated list of numeric IDs (bad value {part!r})")
|
|
return tuple(values)
|
|
|
|
def text_set(self, name: str) -> frozenset[str]:
|
|
raw = self.str(name)
|
|
if raw is None:
|
|
return frozenset()
|
|
return frozenset(" ".join(p.split()).casefold() for p in raw.split(",") if p.strip())
|
|
|
|
def path(self, name: str, default: str) -> Path:
|
|
raw = self.str(name, default)
|
|
path = Path(raw).expanduser()
|
|
return path if path.is_absolute() else PROJECT_DIR / path
|
|
|
|
def size(self, name: str, default: int) -> int:
|
|
raw = self.str(name)
|
|
if raw is None:
|
|
return default
|
|
match = _SIZE_RE.match(raw)
|
|
if not match:
|
|
self.problems.append(f"{name} must be a size such as 512MB or 2GB (got {raw!r})")
|
|
return default
|
|
unit = (match.group(2) or "").upper().replace("I", "").replace("B", "")
|
|
return int(float(match.group(1)) * _SIZE_UNITS[unit or ""])
|
|
|
|
def datetime(self, name: str) -> datetime | None:
|
|
raw = self.str(name)
|
|
if raw is None:
|
|
return None
|
|
try:
|
|
value = datetime.fromisoformat(raw)
|
|
except ValueError:
|
|
self.problems.append(f"{name} must be an ISO date such as 2024-01-31 (got {raw!r})")
|
|
return None
|
|
return value if value.tzinfo else value.replace(tzinfo=timezone.utc)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Config:
|
|
# Telegram
|
|
telegram_mode: str
|
|
api_id: int | None
|
|
api_hash: str | None
|
|
bot_token: str | None
|
|
session_string: str | None
|
|
admin_ids: frozenset[int]
|
|
notify_chat_ids: tuple[int, ...]
|
|
source_channel: str | None
|
|
process_mode: str
|
|
start_message_id: int
|
|
start_date: datetime | None
|
|
include_document_videos: bool
|
|
filter_topics: frozenset[str]
|
|
filter_batches: frozenset[str]
|
|
bot_scan_max_empty: int
|
|
bot_scan_end_id: int | None
|
|
flood_sleep_threshold: int
|
|
session_dir: Path
|
|
# Google Drive
|
|
google_auth_mode: str
|
|
google_client_secrets_file: Path
|
|
google_token_file: Path
|
|
google_service_account_file: Path | None
|
|
google_service_account_json: str | None
|
|
google_oauth_token_json: str | None
|
|
drive_scope: str
|
|
drive_root_folder_id: str | None
|
|
shared_drive_id: str | None
|
|
share_mode: str
|
|
upload_chunk_size: int
|
|
# Organisation and naming
|
|
folder_structure: str
|
|
fallback_topic: str
|
|
fallback_batch: str
|
|
duplicate_name_policy: str
|
|
reuse_identical_files: bool
|
|
duplicate_media_policy: str
|
|
missing_title_policy: str
|
|
extension_mismatch_policy: str
|
|
strict_filenames: bool
|
|
sequence_prefix_enabled: bool
|
|
sequence_prefix_format: str
|
|
# Storage
|
|
data_dir: Path
|
|
database_url: str | None
|
|
allow_ephemeral_database: bool
|
|
is_heroku: bool
|
|
download_dir: Path
|
|
database_path: Path
|
|
log_dir: Path
|
|
max_temp_storage: int
|
|
min_free_disk_space: int
|
|
keep_local_files: bool
|
|
keep_failed_downloads: bool
|
|
temp_cleanup_interval: float
|
|
compute_sha256: bool
|
|
# Processing
|
|
max_attempts: int
|
|
retry_base_delay: float
|
|
retry_max_delay: float
|
|
http_retries: int
|
|
processing_delay: float
|
|
strict_stop: bool
|
|
auto_start: bool
|
|
instance_lock_wait: float
|
|
upload_setup_timeout: float
|
|
upload_max_range: int
|
|
upload_restrict_to_source_channel: bool
|
|
progress_interval: float
|
|
log_level: str
|
|
|
|
@classmethod
|
|
def from_env(cls, env: Mapping[str, str] | None = None, *, load_dotenv_file: bool = True) -> "Config":
|
|
if env is None:
|
|
if load_dotenv_file:
|
|
from dotenv import load_dotenv
|
|
|
|
load_dotenv(PROJECT_DIR / ".env")
|
|
env = os.environ
|
|
r = _Reader(env)
|
|
admin_ids = frozenset(r.int_list("ADMIN_USER_IDS"))
|
|
notify = r.int_list("NOTIFY_CHAT_IDS") or tuple(sorted(admin_ids))
|
|
sa_file = r.str("GOOGLE_SERVICE_ACCOUNT_FILE")
|
|
config = cls(
|
|
telegram_mode=r.choice("TELEGRAM_MODE", "user", ("user", "bot")),
|
|
api_id=r.int("TELEGRAM_API_ID"),
|
|
api_hash=r.str("TELEGRAM_API_HASH"),
|
|
bot_token=r.str("TELEGRAM_BOT_TOKEN"),
|
|
session_string=r.str("TELEGRAM_SESSION_STRING"),
|
|
admin_ids=admin_ids,
|
|
notify_chat_ids=notify,
|
|
source_channel=r.str("SOURCE_CHANNEL_ID"),
|
|
process_mode=r.choice("PROCESS_MODE", "both", ("existing", "new", "both")),
|
|
start_message_id=r.int("START_MESSAGE_ID", 1, minimum=1),
|
|
start_date=r.datetime("START_DATE"),
|
|
include_document_videos=r.bool("INCLUDE_DOCUMENT_VIDEOS", True),
|
|
filter_topics=r.text_set("FILTER_TOPICS"),
|
|
filter_batches=r.text_set("FILTER_BATCHES"),
|
|
bot_scan_max_empty=r.int("BOT_SCAN_MAX_EMPTY", 200, minimum=1),
|
|
bot_scan_end_id=r.int("BOT_SCAN_END_ID", None, minimum=1),
|
|
flood_sleep_threshold=r.int("FLOOD_SLEEP_THRESHOLD", 60, minimum=0),
|
|
session_dir=r.path("SESSION_DIR", "data/sessions"),
|
|
google_auth_mode=r.choice("GOOGLE_AUTH_MODE", "oauth", ("oauth", "service_account")),
|
|
google_client_secrets_file=r.path("GOOGLE_CLIENT_SECRETS_FILE", "credentials/client_secret.json"),
|
|
google_token_file=r.path("GOOGLE_TOKEN_FILE", "credentials/token.json"),
|
|
google_service_account_file=r.path("GOOGLE_SERVICE_ACCOUNT_FILE", sa_file) if sa_file else None,
|
|
google_service_account_json=r.str("GOOGLE_SERVICE_ACCOUNT_JSON"),
|
|
google_oauth_token_json=r.str("GOOGLE_OAUTH_TOKEN_JSON"),
|
|
drive_scope=r.choice("GOOGLE_DRIVE_SCOPE", "drive.file", ("drive.file", "drive")),
|
|
drive_root_folder_id=r.str("GOOGLE_DRIVE_ROOT_FOLDER_ID"),
|
|
shared_drive_id=r.str("DRIVE_SHARED_DRIVE_ID"),
|
|
share_mode=r.choice("DRIVE_SHARE_MODE", "none", ("none", "anyone_with_link")),
|
|
upload_chunk_size=r.size("UPLOAD_CHUNK_SIZE", 16 * 1024 * 1024),
|
|
folder_structure="batch_topic" if r.bool("BATCH_SUBFOLDERS_ENABLED", False) else "topic",
|
|
fallback_topic=r.str("FALLBACK_TOPIC_FOLDER", "Uncategorized"),
|
|
fallback_batch=r.str("FALLBACK_BATCH_FOLDER", "Unknown Batch"),
|
|
duplicate_name_policy=r.choice(
|
|
"DUPLICATE_POLICY", "suffix_number", ("suffix_number", "suffix_message_id", "skip", "fail")
|
|
),
|
|
reuse_identical_files=r.bool("REUSE_IDENTICAL_FILES", True),
|
|
duplicate_media_policy=r.choice("DUPLICATE_MEDIA_POLICY", "skip", ("skip", "upload")),
|
|
missing_title_policy=r.choice("MISSING_TITLE_POLICY", "fallback", ("fallback", "fail")),
|
|
extension_mismatch_policy=r.choice("EXTENSION_MISMATCH_POLICY", "keep", ("keep", "append")),
|
|
strict_filenames=r.bool("STRICT_FILENAMES", False),
|
|
sequence_prefix_enabled=r.bool("SEQUENCE_PREFIX_ENABLED", False),
|
|
# not stripped: trailing spaces are meaningful here
|
|
sequence_prefix_format=env.get("SEQUENCE_PREFIX_FORMAT") or "{seq:03d} - ",
|
|
data_dir=r.path("DATA_DIR", "data"),
|
|
database_url=_normalize_database_url(r.str("DATABASE_URL")),
|
|
allow_ephemeral_database=r.bool("ALLOW_EPHEMERAL_DATABASE", False),
|
|
is_heroku=bool(env.get("DYNO")),
|
|
download_dir=r.path("DOWNLOAD_DIR", "downloads"),
|
|
database_path=r.path("DATABASE_PATH", "data/bot.db"),
|
|
log_dir=r.path("LOG_DIR", "logs"),
|
|
max_temp_storage=r.size("MAX_TEMP_STORAGE", 0),
|
|
min_free_disk_space=r.size("MIN_FREE_DISK_SPACE", 1024**3),
|
|
keep_local_files=r.bool("KEEP_LOCAL_FILES", False),
|
|
keep_failed_downloads=r.bool("KEEP_FAILED_DOWNLOADS", False),
|
|
temp_cleanup_interval=r.float("TEMP_CLEANUP_INTERVAL", 3600.0, minimum=0.0),
|
|
compute_sha256=r.bool("COMPUTE_SHA256", True),
|
|
max_attempts=r.int("MAX_RETRIES", 4, minimum=0) + 1,
|
|
retry_base_delay=r.float("RETRY_DELAY", 30.0),
|
|
retry_max_delay=r.float("RETRY_MAX_DELAY", 1800.0),
|
|
http_retries=r.int("HTTP_RETRIES", 5, minimum=0),
|
|
processing_delay=r.float("PROCESSING_DELAY", 2.0),
|
|
strict_stop=r.bool("STRICT_STOP", False),
|
|
auto_start=r.bool("AUTO_START", False),
|
|
instance_lock_wait=r.float("INSTANCE_LOCK_WAIT", 90.0 if env.get("DYNO") else 0.0),
|
|
upload_setup_timeout=r.float("UPLOAD_SETUP_TIMEOUT", 30.0, minimum=1.0),
|
|
upload_max_range=r.int("UPLOAD_MAX_RANGE", 5000, minimum=1),
|
|
upload_restrict_to_source_channel=r.bool("UPLOAD_RESTRICT_TO_SOURCE_CHANNEL", True),
|
|
progress_interval=r.float("PROGRESS_INTERVAL", 10.0, minimum=1.0),
|
|
log_level=r.choice("LOG_LEVEL", "info", ("debug", "info", "warning", "error")).upper(),
|
|
)
|
|
if config.upload_chunk_size <= 0 or config.upload_chunk_size % UPLOAD_CHUNK_ALIGN:
|
|
r.problems.append("UPLOAD_CHUNK_SIZE must be a positive multiple of 256KB (e.g. 8MB, 16MB, 32MB)")
|
|
try:
|
|
config.sequence_prefix_format.format(seq=1)
|
|
except (KeyError, IndexError, ValueError):
|
|
r.problems.append("SEQUENCE_PREFIX_FORMAT must be a format string using {seq}, e.g. '{seq:03d} - '")
|
|
import tempfile
|
|
|
|
resolved = config.download_dir.resolve()
|
|
shared = {Path(p).resolve() for p in ("/", "/tmp", "/var/tmp", tempfile.gettempdir(), Path.home(), PROJECT_DIR)}
|
|
if resolved in shared or len(resolved.parts) < 3:
|
|
r.problems.append(
|
|
f"DOWNLOAD_DIR={config.download_dir} is not a dedicated directory; use e.g. 'downloads' or '/tmp/telegram_drive_bot'"
|
|
)
|
|
if r.problems:
|
|
raise ConfigError(r.problems)
|
|
return config
|
|
|
|
# -- section validation -------------------------------------------------------
|
|
|
|
def telegram_problems(self) -> list[str]:
|
|
problems = []
|
|
if not self.api_id or not self.api_hash:
|
|
problems.append(
|
|
"TELEGRAM_API_ID and TELEGRAM_API_HASH are required (create them at https://my.telegram.org -> API development tools)"
|
|
)
|
|
if self.telegram_mode == "bot" and not self.bot_token:
|
|
problems.append("TELEGRAM_BOT_TOKEN is required when TELEGRAM_MODE=bot (get one from @BotFather)")
|
|
if self.telegram_mode == "user" and self.is_heroku and not self.session_string:
|
|
problems.append(
|
|
"TELEGRAM_SESSION_STRING is required for TELEGRAM_MODE=user on Heroku (the filesystem is ephemeral, "
|
|
"so a session file would be lost). Generate it locally with `python main.py login --session-string`."
|
|
)
|
|
if self.bot_token and not self.admin_ids:
|
|
problems.append("ADMIN_USER_IDS must list at least one Telegram user ID allowed to control the bot")
|
|
if not self.source_channel:
|
|
problems.append("SOURCE_CHANNEL_ID is required (@username, https://t.me/... link, or -100... numeric ID)")
|
|
return problems
|
|
|
|
def drive_problems(self) -> list[str]:
|
|
problems = []
|
|
if self.google_auth_mode == "oauth":
|
|
if self.google_oauth_token_json:
|
|
problems += _json_problems(self.google_oauth_token_json, "GOOGLE_OAUTH_TOKEN_JSON", ("refresh_token", "client_id", "client_secret"))
|
|
elif self.is_heroku:
|
|
problems.append(
|
|
"GOOGLE_OAUTH_TOKEN_JSON is required on Heroku when GOOGLE_AUTH_MODE=oauth "
|
|
"(create it locally with `python main.py drive-auth`, see README)"
|
|
)
|
|
elif not self.google_token_file.exists() and not self.google_client_secrets_file.exists():
|
|
problems.append(
|
|
f"Google OAuth client file not found at {self.google_client_secrets_file.name!r} "
|
|
"(set GOOGLE_CLIENT_SECRETS_FILE or GOOGLE_OAUTH_TOKEN_JSON; see README)"
|
|
)
|
|
else:
|
|
if self.google_service_account_json:
|
|
problems += _json_problems(
|
|
self.google_service_account_json, "GOOGLE_SERVICE_ACCOUNT_JSON", ("client_email", "private_key")
|
|
)
|
|
elif not self.google_service_account_file or not self.google_service_account_file.exists():
|
|
problems.append(
|
|
"GOOGLE_AUTH_MODE=service_account needs GOOGLE_SERVICE_ACCOUNT_JSON (Heroku) "
|
|
"or GOOGLE_SERVICE_ACCOUNT_FILE pointing to the JSON key"
|
|
)
|
|
return problems
|
|
|
|
def database_problems(self) -> list[str]:
|
|
if self.is_heroku and not self.database_url and not self.allow_ephemeral_database:
|
|
return [
|
|
"DATABASE_URL is not set. On Heroku the local filesystem is wiped on every restart, so SQLite would "
|
|
"lose all progress and cause re-uploads. Attach Heroku Postgres (`heroku addons:create "
|
|
"heroku-postgresql:essential-0`) or set ALLOW_EPHEMERAL_DATABASE=true to accept volatile state."
|
|
]
|
|
return []
|
|
|
|
def validate(self, *sections: str) -> None:
|
|
problems: list[str] = []
|
|
if "telegram" in sections:
|
|
problems += self.telegram_problems()
|
|
if "drive" in sections:
|
|
problems += self.drive_problems()
|
|
if "database" in sections:
|
|
problems += self.database_problems()
|
|
if problems:
|
|
raise ConfigError(problems)
|
|
|
|
def secrets(self) -> list[str]:
|
|
"""Values that must never appear in logs."""
|
|
values = [self.api_hash, self.bot_token, self.session_string, self.database_url]
|
|
for blob in (self.google_oauth_token_json, self.google_service_account_json):
|
|
try:
|
|
data = load_json_secret(blob) if blob else {}
|
|
except ValueError:
|
|
data = {}
|
|
values += [data.get(k) for k in ("refresh_token", "client_secret", "private_key", "token")]
|
|
values.append(blob)
|
|
return [v for v in values if v and len(v) >= 8]
|
|
|
|
|
|
def load_json_secret(value: str) -> dict:
|
|
"""Parse a JSON credential given either as raw JSON or base64-encoded JSON.
|
|
|
|
Base64 is accepted because some dashboards mangle multi-line values; raw JSON
|
|
(including multi-line) works fine through `heroku config:set VAR="$(cat file)"`.
|
|
"""
|
|
import base64
|
|
import binascii
|
|
import json
|
|
|
|
text = value.strip()
|
|
if not text.startswith("{"):
|
|
try:
|
|
text = base64.b64decode(text, validate=False).decode("utf-8").strip()
|
|
except (binascii.Error, UnicodeDecodeError) as exc:
|
|
raise ValueError("value is neither JSON nor base64-encoded JSON") from exc
|
|
try:
|
|
data = json.loads(text)
|
|
except json.JSONDecodeError as exc:
|
|
raise ValueError(f"invalid JSON ({exc.msg} at line {exc.lineno})") from exc
|
|
if not isinstance(data, dict):
|
|
raise ValueError("expected a JSON object")
|
|
return data
|
|
|
|
|
|
def _json_problems(value: str, name: str, required_keys: tuple[str, ...]) -> list[str]:
|
|
try:
|
|
data = load_json_secret(value)
|
|
except ValueError as exc:
|
|
return [f"{name} could not be parsed: {exc}"]
|
|
missing = [k for k in required_keys if not data.get(k)]
|
|
return [f"{name} is missing keys: {', '.join(missing)}"] if missing else []
|
|
|
|
|
|
def _normalize_database_url(url: str | None) -> str | None:
|
|
"""Heroku provides postgres://...; enforce TLS for remote hosts."""
|
|
if not url:
|
|
return None
|
|
if url.startswith("postgres://"):
|
|
url = "postgresql://" + url[len("postgres://"):]
|
|
host_part = url.split("@")[-1]
|
|
remote = not (host_part.startswith(("localhost", "127.0.0.1", "/")) or "host=/" in url)
|
|
if remote and "sslmode=" not in url:
|
|
url += ("&" if "?" in url else "?") + "sslmode=require"
|
|
return url
|