"""Configuration loading and validation. Every setting comes from environment variables, optionally loaded from a ``.env`` file next to this module. No secret is hard-coded anywhere in the project. """ from __future__ import annotations import os import re from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import Mapping PROJECT_DIR = Path(__file__).resolve().parent UPLOAD_CHUNK_ALIGN = 256 * 1024 # Google requires resumable chunks in multiples of 256 KiB _TRUE = {"1", "true", "yes", "on"} _FALSE = {"0", "false", "no", "off"} _SIZE_RE = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*([KMGT]?I?B?)?\s*$", re.IGNORECASE) _SIZE_UNITS = {"": 1, "B": 1, "K": 1024, "M": 1024**2, "G": 1024**3, "T": 1024**4} class ConfigError(Exception): def __init__(self, problems: list[str]): self.problems = problems super().__init__("Configuration problems:\n" + "\n".join(f" - {p}" for p in problems)) class _Reader: """Typed accessors that collect every problem instead of stopping at the first.""" def __init__(self, env: Mapping[str, str]): self.env = env self.problems: list[str] = [] def str(self, name: str, default: str | None = None) -> str | None: raw = self.env.get(name) if raw is None: return default raw = raw.strip() return raw if raw else default def int(self, name: str, default: int | None = None, *, minimum: int | None = None) -> int | None: raw = self.str(name) if raw is None: return default try: value = int(raw) except ValueError: self.problems.append(f"{name} must be an integer (got {raw!r})") return default if minimum is not None and value < minimum: self.problems.append(f"{name} must be >= {minimum} (got {value})") return default return value def float(self, name: str, default: float, *, minimum: float = 0.0) -> float: raw = self.str(name) if raw is None: return default try: value = float(raw) except ValueError: self.problems.append(f"{name} must be a number (got {raw!r})") return default if value < minimum: self.problems.append(f"{name} must be >= {minimum} (got {value})") return default return value def bool(self, name: str, default: bool) -> bool: raw = self.str(name) if raw is None: return default low = raw.lower() if low in _TRUE: return True if low in _FALSE: return False self.problems.append(f"{name} must be true or false (got {raw!r})") return default def choice(self, name: str, default: str, choices: tuple[str, ...]) -> str: raw = self.str(name, default) value = raw.lower() if value not in choices: self.problems.append(f"{name} must be one of {', '.join(choices)} (got {raw!r})") return default return value def int_list(self, name: str) -> tuple[int, ...]: raw = self.str(name) if raw is None: return () values = [] for part in raw.split(","): part = part.strip() if not part: continue try: values.append(int(part)) except ValueError: self.problems.append(f"{name} must be a comma-separated list of numeric IDs (bad value {part!r})") return tuple(values) def text_set(self, name: str) -> frozenset[str]: raw = self.str(name) if raw is None: return frozenset() return frozenset(" ".join(p.split()).casefold() for p in raw.split(",") if p.strip()) def path(self, name: str, default: str) -> Path: raw = self.str(name, default) path = Path(raw).expanduser() return path if path.is_absolute() else PROJECT_DIR / path def size(self, name: str, default: int) -> int: raw = self.str(name) if raw is None: return default match = _SIZE_RE.match(raw) if not match: self.problems.append(f"{name} must be a size such as 512MB or 2GB (got {raw!r})") return default unit = (match.group(2) or "").upper().replace("I", "").replace("B", "") return int(float(match.group(1)) * _SIZE_UNITS[unit or ""]) def datetime(self, name: str) -> datetime | None: raw = self.str(name) if raw is None: return None try: value = datetime.fromisoformat(raw) except ValueError: self.problems.append(f"{name} must be an ISO date such as 2024-01-31 (got {raw!r})") return None return value if value.tzinfo else value.replace(tzinfo=timezone.utc) @dataclass(frozen=True) class Config: # Telegram telegram_mode: str api_id: int | None api_hash: str | None bot_token: str | None session_string: str | None admin_ids: frozenset[int] notify_chat_ids: tuple[int, ...] source_channel: str | None process_mode: str start_message_id: int start_date: datetime | None include_document_videos: bool filter_topics: frozenset[str] filter_batches: frozenset[str] bot_scan_max_empty: int bot_scan_end_id: int | None flood_sleep_threshold: int session_dir: Path # Google Drive google_auth_mode: str google_client_secrets_file: Path google_token_file: Path google_service_account_file: Path | None google_service_account_json: str | None google_oauth_token_json: str | None drive_scope: str drive_root_folder_id: str | None shared_drive_id: str | None share_mode: str upload_chunk_size: int # Organisation and naming folder_structure: str fallback_topic: str fallback_batch: str duplicate_name_policy: str reuse_identical_files: bool duplicate_media_policy: str missing_title_policy: str extension_mismatch_policy: str strict_filenames: bool sequence_prefix_enabled: bool sequence_prefix_format: str # Storage data_dir: Path database_url: str | None allow_ephemeral_database: bool is_heroku: bool download_dir: Path database_path: Path log_dir: Path max_temp_storage: int min_free_disk_space: int keep_local_files: bool keep_failed_downloads: bool temp_cleanup_interval: float compute_sha256: bool # Processing max_attempts: int retry_base_delay: float retry_max_delay: float http_retries: int processing_delay: float strict_stop: bool auto_start: bool instance_lock_wait: float upload_setup_timeout: float upload_max_range: int upload_restrict_to_source_channel: bool progress_interval: float log_level: str @classmethod def from_env(cls, env: Mapping[str, str] | None = None, *, load_dotenv_file: bool = True) -> "Config": if env is None: if load_dotenv_file: from dotenv import load_dotenv load_dotenv(PROJECT_DIR / ".env") env = os.environ r = _Reader(env) admin_ids = frozenset(r.int_list("ADMIN_USER_IDS")) notify = r.int_list("NOTIFY_CHAT_IDS") or tuple(sorted(admin_ids)) sa_file = r.str("GOOGLE_SERVICE_ACCOUNT_FILE") config = cls( telegram_mode=r.choice("TELEGRAM_MODE", "user", ("user", "bot")), api_id=r.int("TELEGRAM_API_ID"), api_hash=r.str("TELEGRAM_API_HASH"), bot_token=r.str("TELEGRAM_BOT_TOKEN"), session_string=r.str("TELEGRAM_SESSION_STRING"), admin_ids=admin_ids, notify_chat_ids=notify, source_channel=r.str("SOURCE_CHANNEL_ID"), process_mode=r.choice("PROCESS_MODE", "both", ("existing", "new", "both")), start_message_id=r.int("START_MESSAGE_ID", 1, minimum=1), start_date=r.datetime("START_DATE"), include_document_videos=r.bool("INCLUDE_DOCUMENT_VIDEOS", True), filter_topics=r.text_set("FILTER_TOPICS"), filter_batches=r.text_set("FILTER_BATCHES"), bot_scan_max_empty=r.int("BOT_SCAN_MAX_EMPTY", 200, minimum=1), bot_scan_end_id=r.int("BOT_SCAN_END_ID", None, minimum=1), flood_sleep_threshold=r.int("FLOOD_SLEEP_THRESHOLD", 60, minimum=0), session_dir=r.path("SESSION_DIR", "data/sessions"), google_auth_mode=r.choice("GOOGLE_AUTH_MODE", "oauth", ("oauth", "service_account")), google_client_secrets_file=r.path("GOOGLE_CLIENT_SECRETS_FILE", "credentials/client_secret.json"), google_token_file=r.path("GOOGLE_TOKEN_FILE", "credentials/token.json"), google_service_account_file=r.path("GOOGLE_SERVICE_ACCOUNT_FILE", sa_file) if sa_file else None, google_service_account_json=r.str("GOOGLE_SERVICE_ACCOUNT_JSON"), google_oauth_token_json=r.str("GOOGLE_OAUTH_TOKEN_JSON"), drive_scope=r.choice("GOOGLE_DRIVE_SCOPE", "drive.file", ("drive.file", "drive")), drive_root_folder_id=r.str("GOOGLE_DRIVE_ROOT_FOLDER_ID"), shared_drive_id=r.str("DRIVE_SHARED_DRIVE_ID"), share_mode=r.choice("DRIVE_SHARE_MODE", "none", ("none", "anyone_with_link")), upload_chunk_size=r.size("UPLOAD_CHUNK_SIZE", 16 * 1024 * 1024), folder_structure="batch_topic" if r.bool("BATCH_SUBFOLDERS_ENABLED", False) else "topic", fallback_topic=r.str("FALLBACK_TOPIC_FOLDER", "Uncategorized"), fallback_batch=r.str("FALLBACK_BATCH_FOLDER", "Unknown Batch"), duplicate_name_policy=r.choice( "DUPLICATE_POLICY", "suffix_number", ("suffix_number", "suffix_message_id", "skip", "fail") ), reuse_identical_files=r.bool("REUSE_IDENTICAL_FILES", True), duplicate_media_policy=r.choice("DUPLICATE_MEDIA_POLICY", "skip", ("skip", "upload")), missing_title_policy=r.choice("MISSING_TITLE_POLICY", "fallback", ("fallback", "fail")), extension_mismatch_policy=r.choice("EXTENSION_MISMATCH_POLICY", "keep", ("keep", "append")), strict_filenames=r.bool("STRICT_FILENAMES", False), sequence_prefix_enabled=r.bool("SEQUENCE_PREFIX_ENABLED", False), # not stripped: trailing spaces are meaningful here sequence_prefix_format=env.get("SEQUENCE_PREFIX_FORMAT") or "{seq:03d} - ", data_dir=r.path("DATA_DIR", "data"), database_url=_normalize_database_url(r.str("DATABASE_URL")), allow_ephemeral_database=r.bool("ALLOW_EPHEMERAL_DATABASE", False), is_heroku=bool(env.get("DYNO")), download_dir=r.path("DOWNLOAD_DIR", "downloads"), database_path=r.path("DATABASE_PATH", "data/bot.db"), log_dir=r.path("LOG_DIR", "logs"), max_temp_storage=r.size("MAX_TEMP_STORAGE", 0), min_free_disk_space=r.size("MIN_FREE_DISK_SPACE", 1024**3), keep_local_files=r.bool("KEEP_LOCAL_FILES", False), keep_failed_downloads=r.bool("KEEP_FAILED_DOWNLOADS", False), temp_cleanup_interval=r.float("TEMP_CLEANUP_INTERVAL", 3600.0, minimum=0.0), compute_sha256=r.bool("COMPUTE_SHA256", True), max_attempts=r.int("MAX_RETRIES", 4, minimum=0) + 1, retry_base_delay=r.float("RETRY_DELAY", 30.0), retry_max_delay=r.float("RETRY_MAX_DELAY", 1800.0), http_retries=r.int("HTTP_RETRIES", 5, minimum=0), processing_delay=r.float("PROCESSING_DELAY", 2.0), strict_stop=r.bool("STRICT_STOP", False), auto_start=r.bool("AUTO_START", False), instance_lock_wait=r.float("INSTANCE_LOCK_WAIT", 90.0 if env.get("DYNO") else 0.0), upload_setup_timeout=r.float("UPLOAD_SETUP_TIMEOUT", 30.0, minimum=1.0), upload_max_range=r.int("UPLOAD_MAX_RANGE", 5000, minimum=1), upload_restrict_to_source_channel=r.bool("UPLOAD_RESTRICT_TO_SOURCE_CHANNEL", True), progress_interval=r.float("PROGRESS_INTERVAL", 10.0, minimum=1.0), log_level=r.choice("LOG_LEVEL", "info", ("debug", "info", "warning", "error")).upper(), ) if config.upload_chunk_size <= 0 or config.upload_chunk_size % UPLOAD_CHUNK_ALIGN: r.problems.append("UPLOAD_CHUNK_SIZE must be a positive multiple of 256KB (e.g. 8MB, 16MB, 32MB)") try: config.sequence_prefix_format.format(seq=1) except (KeyError, IndexError, ValueError): r.problems.append("SEQUENCE_PREFIX_FORMAT must be a format string using {seq}, e.g. '{seq:03d} - '") import tempfile resolved = config.download_dir.resolve() shared = {Path(p).resolve() for p in ("/", "/tmp", "/var/tmp", tempfile.gettempdir(), Path.home(), PROJECT_DIR)} if resolved in shared or len(resolved.parts) < 3: r.problems.append( f"DOWNLOAD_DIR={config.download_dir} is not a dedicated directory; use e.g. 'downloads' or '/tmp/telegram_drive_bot'" ) if r.problems: raise ConfigError(r.problems) return config # -- section validation ------------------------------------------------------- def telegram_problems(self) -> list[str]: problems = [] if not self.api_id or not self.api_hash: problems.append( "TELEGRAM_API_ID and TELEGRAM_API_HASH are required (create them at https://my.telegram.org -> API development tools)" ) if self.telegram_mode == "bot" and not self.bot_token: problems.append("TELEGRAM_BOT_TOKEN is required when TELEGRAM_MODE=bot (get one from @BotFather)") if self.telegram_mode == "user" and self.is_heroku and not self.session_string: problems.append( "TELEGRAM_SESSION_STRING is required for TELEGRAM_MODE=user on Heroku (the filesystem is ephemeral, " "so a session file would be lost). Generate it locally with `python main.py login --session-string`." ) if self.bot_token and not self.admin_ids: problems.append("ADMIN_USER_IDS must list at least one Telegram user ID allowed to control the bot") if not self.source_channel: problems.append("SOURCE_CHANNEL_ID is required (@username, https://t.me/... link, or -100... numeric ID)") return problems def drive_problems(self) -> list[str]: problems = [] if self.google_auth_mode == "oauth": if self.google_oauth_token_json: problems += _json_problems(self.google_oauth_token_json, "GOOGLE_OAUTH_TOKEN_JSON", ("refresh_token", "client_id", "client_secret")) elif self.is_heroku: problems.append( "GOOGLE_OAUTH_TOKEN_JSON is required on Heroku when GOOGLE_AUTH_MODE=oauth " "(create it locally with `python main.py drive-auth`, see README)" ) elif not self.google_token_file.exists() and not self.google_client_secrets_file.exists(): problems.append( f"Google OAuth client file not found at {self.google_client_secrets_file.name!r} " "(set GOOGLE_CLIENT_SECRETS_FILE or GOOGLE_OAUTH_TOKEN_JSON; see README)" ) else: if self.google_service_account_json: problems += _json_problems( self.google_service_account_json, "GOOGLE_SERVICE_ACCOUNT_JSON", ("client_email", "private_key") ) elif not self.google_service_account_file or not self.google_service_account_file.exists(): problems.append( "GOOGLE_AUTH_MODE=service_account needs GOOGLE_SERVICE_ACCOUNT_JSON (Heroku) " "or GOOGLE_SERVICE_ACCOUNT_FILE pointing to the JSON key" ) return problems def database_problems(self) -> list[str]: if self.is_heroku and not self.database_url and not self.allow_ephemeral_database: return [ "DATABASE_URL is not set. On Heroku the local filesystem is wiped on every restart, so SQLite would " "lose all progress and cause re-uploads. Attach Heroku Postgres (`heroku addons:create " "heroku-postgresql:essential-0`) or set ALLOW_EPHEMERAL_DATABASE=true to accept volatile state." ] return [] def validate(self, *sections: str) -> None: problems: list[str] = [] if "telegram" in sections: problems += self.telegram_problems() if "drive" in sections: problems += self.drive_problems() if "database" in sections: problems += self.database_problems() if problems: raise ConfigError(problems) def secrets(self) -> list[str]: """Values that must never appear in logs.""" values = [self.api_hash, self.bot_token, self.session_string, self.database_url] for blob in (self.google_oauth_token_json, self.google_service_account_json): try: data = load_json_secret(blob) if blob else {} except ValueError: data = {} values += [data.get(k) for k in ("refresh_token", "client_secret", "private_key", "token")] values.append(blob) return [v for v in values if v and len(v) >= 8] def load_json_secret(value: str) -> dict: """Parse a JSON credential given either as raw JSON or base64-encoded JSON. Base64 is accepted because some dashboards mangle multi-line values; raw JSON (including multi-line) works fine through `heroku config:set VAR="$(cat file)"`. """ import base64 import binascii import json text = value.strip() if not text.startswith("{"): try: text = base64.b64decode(text, validate=False).decode("utf-8").strip() except (binascii.Error, UnicodeDecodeError) as exc: raise ValueError("value is neither JSON nor base64-encoded JSON") from exc try: data = json.loads(text) except json.JSONDecodeError as exc: raise ValueError(f"invalid JSON ({exc.msg} at line {exc.lineno})") from exc if not isinstance(data, dict): raise ValueError("expected a JSON object") return data def _json_problems(value: str, name: str, required_keys: tuple[str, ...]) -> list[str]: try: data = load_json_secret(value) except ValueError as exc: return [f"{name} could not be parsed: {exc}"] missing = [k for k in required_keys if not data.get(k)] return [f"{name} is missing keys: {', '.join(missing)}"] if missing else [] def _normalize_database_url(url: str | None) -> str | None: """Heroku provides postgres://...; enforce TLS for remote hosts.""" if not url: return None if url.startswith("postgres://"): url = "postgresql://" + url[len("postgres://"):] host_part = url.split("@")[-1] remote = not (host_part.startswith(("localhost", "127.0.0.1", "/")) or "host=/" in url) if remote and "sslmode=" not in url: url += ("&" if "?" in url else "?") + "sslmode=require" return url