first commit
This commit is contained in:
gosangam
@@ -0,0 +1,438 @@
|
||||
"""Configuration loading and validation.
|
||||
|
||||
Every setting comes from environment variables, optionally loaded from a ``.env``
|
||||
file next to this module. No secret is hard-coded anywhere in the project.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Mapping
|
||||
|
||||
PROJECT_DIR = Path(__file__).resolve().parent
|
||||
UPLOAD_CHUNK_ALIGN = 256 * 1024 # Google requires resumable chunks in multiples of 256 KiB
|
||||
|
||||
_TRUE = {"1", "true", "yes", "on"}
|
||||
_FALSE = {"0", "false", "no", "off"}
|
||||
_SIZE_RE = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*([KMGT]?I?B?)?\s*$", re.IGNORECASE)
|
||||
_SIZE_UNITS = {"": 1, "B": 1, "K": 1024, "M": 1024**2, "G": 1024**3, "T": 1024**4}
|
||||
|
||||
|
||||
class ConfigError(Exception):
|
||||
def __init__(self, problems: list[str]):
|
||||
self.problems = problems
|
||||
super().__init__("Configuration problems:\n" + "\n".join(f" - {p}" for p in problems))
|
||||
|
||||
|
||||
class _Reader:
|
||||
"""Typed accessors that collect every problem instead of stopping at the first."""
|
||||
|
||||
def __init__(self, env: Mapping[str, str]):
|
||||
self.env = env
|
||||
self.problems: list[str] = []
|
||||
|
||||
def str(self, name: str, default: str | None = None) -> str | None:
|
||||
raw = self.env.get(name)
|
||||
if raw is None:
|
||||
return default
|
||||
raw = raw.strip()
|
||||
return raw if raw else default
|
||||
|
||||
def int(self, name: str, default: int | None = None, *, minimum: int | None = None) -> int | None:
|
||||
raw = self.str(name)
|
||||
if raw is None:
|
||||
return default
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
self.problems.append(f"{name} must be an integer (got {raw!r})")
|
||||
return default
|
||||
if minimum is not None and value < minimum:
|
||||
self.problems.append(f"{name} must be >= {minimum} (got {value})")
|
||||
return default
|
||||
return value
|
||||
|
||||
def float(self, name: str, default: float, *, minimum: float = 0.0) -> float:
|
||||
raw = self.str(name)
|
||||
if raw is None:
|
||||
return default
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
self.problems.append(f"{name} must be a number (got {raw!r})")
|
||||
return default
|
||||
if value < minimum:
|
||||
self.problems.append(f"{name} must be >= {minimum} (got {value})")
|
||||
return default
|
||||
return value
|
||||
|
||||
def bool(self, name: str, default: bool) -> bool:
|
||||
raw = self.str(name)
|
||||
if raw is None:
|
||||
return default
|
||||
low = raw.lower()
|
||||
if low in _TRUE:
|
||||
return True
|
||||
if low in _FALSE:
|
||||
return False
|
||||
self.problems.append(f"{name} must be true or false (got {raw!r})")
|
||||
return default
|
||||
|
||||
def choice(self, name: str, default: str, choices: tuple[str, ...]) -> str:
|
||||
raw = self.str(name, default)
|
||||
value = raw.lower()
|
||||
if value not in choices:
|
||||
self.problems.append(f"{name} must be one of {', '.join(choices)} (got {raw!r})")
|
||||
return default
|
||||
return value
|
||||
|
||||
def int_list(self, name: str) -> tuple[int, ...]:
|
||||
raw = self.str(name)
|
||||
if raw is None:
|
||||
return ()
|
||||
values = []
|
||||
for part in raw.split(","):
|
||||
part = part.strip()
|
||||
if not part:
|
||||
continue
|
||||
try:
|
||||
values.append(int(part))
|
||||
except ValueError:
|
||||
self.problems.append(f"{name} must be a comma-separated list of numeric IDs (bad value {part!r})")
|
||||
return tuple(values)
|
||||
|
||||
def text_set(self, name: str) -> frozenset[str]:
|
||||
raw = self.str(name)
|
||||
if raw is None:
|
||||
return frozenset()
|
||||
return frozenset(" ".join(p.split()).casefold() for p in raw.split(",") if p.strip())
|
||||
|
||||
def path(self, name: str, default: str) -> Path:
|
||||
raw = self.str(name, default)
|
||||
path = Path(raw).expanduser()
|
||||
return path if path.is_absolute() else PROJECT_DIR / path
|
||||
|
||||
def size(self, name: str, default: int) -> int:
|
||||
raw = self.str(name)
|
||||
if raw is None:
|
||||
return default
|
||||
match = _SIZE_RE.match(raw)
|
||||
if not match:
|
||||
self.problems.append(f"{name} must be a size such as 512MB or 2GB (got {raw!r})")
|
||||
return default
|
||||
unit = (match.group(2) or "").upper().replace("I", "").replace("B", "")
|
||||
return int(float(match.group(1)) * _SIZE_UNITS[unit or ""])
|
||||
|
||||
def datetime(self, name: str) -> datetime | None:
|
||||
raw = self.str(name)
|
||||
if raw is None:
|
||||
return None
|
||||
try:
|
||||
value = datetime.fromisoformat(raw)
|
||||
except ValueError:
|
||||
self.problems.append(f"{name} must be an ISO date such as 2024-01-31 (got {raw!r})")
|
||||
return None
|
||||
return value if value.tzinfo else value.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Config:
|
||||
# Telegram
|
||||
telegram_mode: str
|
||||
api_id: int | None
|
||||
api_hash: str | None
|
||||
bot_token: str | None
|
||||
session_string: str | None
|
||||
admin_ids: frozenset[int]
|
||||
notify_chat_ids: tuple[int, ...]
|
||||
source_channel: str | None
|
||||
process_mode: str
|
||||
start_message_id: int
|
||||
start_date: datetime | None
|
||||
include_document_videos: bool
|
||||
filter_topics: frozenset[str]
|
||||
filter_batches: frozenset[str]
|
||||
bot_scan_max_empty: int
|
||||
bot_scan_end_id: int | None
|
||||
flood_sleep_threshold: int
|
||||
session_dir: Path
|
||||
# Google Drive
|
||||
google_auth_mode: str
|
||||
google_client_secrets_file: Path
|
||||
google_token_file: Path
|
||||
google_service_account_file: Path | None
|
||||
google_service_account_json: str | None
|
||||
google_oauth_token_json: str | None
|
||||
drive_scope: str
|
||||
drive_root_folder_id: str | None
|
||||
shared_drive_id: str | None
|
||||
share_mode: str
|
||||
upload_chunk_size: int
|
||||
# Organisation and naming
|
||||
folder_structure: str
|
||||
fallback_topic: str
|
||||
fallback_batch: str
|
||||
duplicate_name_policy: str
|
||||
reuse_identical_files: bool
|
||||
duplicate_media_policy: str
|
||||
missing_title_policy: str
|
||||
extension_mismatch_policy: str
|
||||
strict_filenames: bool
|
||||
sequence_prefix_enabled: bool
|
||||
sequence_prefix_format: str
|
||||
# Storage
|
||||
data_dir: Path
|
||||
database_url: str | None
|
||||
allow_ephemeral_database: bool
|
||||
is_heroku: bool
|
||||
download_dir: Path
|
||||
database_path: Path
|
||||
log_dir: Path
|
||||
max_temp_storage: int
|
||||
min_free_disk_space: int
|
||||
keep_local_files: bool
|
||||
keep_failed_downloads: bool
|
||||
temp_cleanup_interval: float
|
||||
compute_sha256: bool
|
||||
# Processing
|
||||
max_attempts: int
|
||||
retry_base_delay: float
|
||||
retry_max_delay: float
|
||||
http_retries: int
|
||||
processing_delay: float
|
||||
strict_stop: bool
|
||||
auto_start: bool
|
||||
instance_lock_wait: float
|
||||
upload_setup_timeout: float
|
||||
upload_max_range: int
|
||||
upload_restrict_to_source_channel: bool
|
||||
progress_interval: float
|
||||
log_level: str
|
||||
|
||||
@classmethod
|
||||
def from_env(cls, env: Mapping[str, str] | None = None, *, load_dotenv_file: bool = True) -> "Config":
|
||||
if env is None:
|
||||
if load_dotenv_file:
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv(PROJECT_DIR / ".env")
|
||||
env = os.environ
|
||||
r = _Reader(env)
|
||||
admin_ids = frozenset(r.int_list("ADMIN_USER_IDS"))
|
||||
notify = r.int_list("NOTIFY_CHAT_IDS") or tuple(sorted(admin_ids))
|
||||
sa_file = r.str("GOOGLE_SERVICE_ACCOUNT_FILE")
|
||||
config = cls(
|
||||
telegram_mode=r.choice("TELEGRAM_MODE", "user", ("user", "bot")),
|
||||
api_id=r.int("TELEGRAM_API_ID"),
|
||||
api_hash=r.str("TELEGRAM_API_HASH"),
|
||||
bot_token=r.str("TELEGRAM_BOT_TOKEN"),
|
||||
session_string=r.str("TELEGRAM_SESSION_STRING"),
|
||||
admin_ids=admin_ids,
|
||||
notify_chat_ids=notify,
|
||||
source_channel=r.str("SOURCE_CHANNEL_ID"),
|
||||
process_mode=r.choice("PROCESS_MODE", "both", ("existing", "new", "both")),
|
||||
start_message_id=r.int("START_MESSAGE_ID", 1, minimum=1),
|
||||
start_date=r.datetime("START_DATE"),
|
||||
include_document_videos=r.bool("INCLUDE_DOCUMENT_VIDEOS", True),
|
||||
filter_topics=r.text_set("FILTER_TOPICS"),
|
||||
filter_batches=r.text_set("FILTER_BATCHES"),
|
||||
bot_scan_max_empty=r.int("BOT_SCAN_MAX_EMPTY", 200, minimum=1),
|
||||
bot_scan_end_id=r.int("BOT_SCAN_END_ID", None, minimum=1),
|
||||
flood_sleep_threshold=r.int("FLOOD_SLEEP_THRESHOLD", 60, minimum=0),
|
||||
session_dir=r.path("SESSION_DIR", "data/sessions"),
|
||||
google_auth_mode=r.choice("GOOGLE_AUTH_MODE", "oauth", ("oauth", "service_account")),
|
||||
google_client_secrets_file=r.path("GOOGLE_CLIENT_SECRETS_FILE", "credentials/client_secret.json"),
|
||||
google_token_file=r.path("GOOGLE_TOKEN_FILE", "credentials/token.json"),
|
||||
google_service_account_file=r.path("GOOGLE_SERVICE_ACCOUNT_FILE", sa_file) if sa_file else None,
|
||||
google_service_account_json=r.str("GOOGLE_SERVICE_ACCOUNT_JSON"),
|
||||
google_oauth_token_json=r.str("GOOGLE_OAUTH_TOKEN_JSON"),
|
||||
drive_scope=r.choice("GOOGLE_DRIVE_SCOPE", "drive.file", ("drive.file", "drive")),
|
||||
drive_root_folder_id=r.str("GOOGLE_DRIVE_ROOT_FOLDER_ID"),
|
||||
shared_drive_id=r.str("DRIVE_SHARED_DRIVE_ID"),
|
||||
share_mode=r.choice("DRIVE_SHARE_MODE", "none", ("none", "anyone_with_link")),
|
||||
upload_chunk_size=r.size("UPLOAD_CHUNK_SIZE", 16 * 1024 * 1024),
|
||||
folder_structure="batch_topic" if r.bool("BATCH_SUBFOLDERS_ENABLED", False) else "topic",
|
||||
fallback_topic=r.str("FALLBACK_TOPIC_FOLDER", "Uncategorized"),
|
||||
fallback_batch=r.str("FALLBACK_BATCH_FOLDER", "Unknown Batch"),
|
||||
duplicate_name_policy=r.choice(
|
||||
"DUPLICATE_POLICY", "suffix_number", ("suffix_number", "suffix_message_id", "skip", "fail")
|
||||
),
|
||||
reuse_identical_files=r.bool("REUSE_IDENTICAL_FILES", True),
|
||||
duplicate_media_policy=r.choice("DUPLICATE_MEDIA_POLICY", "skip", ("skip", "upload")),
|
||||
missing_title_policy=r.choice("MISSING_TITLE_POLICY", "fallback", ("fallback", "fail")),
|
||||
extension_mismatch_policy=r.choice("EXTENSION_MISMATCH_POLICY", "keep", ("keep", "append")),
|
||||
strict_filenames=r.bool("STRICT_FILENAMES", False),
|
||||
sequence_prefix_enabled=r.bool("SEQUENCE_PREFIX_ENABLED", False),
|
||||
# not stripped: trailing spaces are meaningful here
|
||||
sequence_prefix_format=env.get("SEQUENCE_PREFIX_FORMAT") or "{seq:03d} - ",
|
||||
data_dir=r.path("DATA_DIR", "data"),
|
||||
database_url=_normalize_database_url(r.str("DATABASE_URL")),
|
||||
allow_ephemeral_database=r.bool("ALLOW_EPHEMERAL_DATABASE", False),
|
||||
is_heroku=bool(env.get("DYNO")),
|
||||
download_dir=r.path("DOWNLOAD_DIR", "downloads"),
|
||||
database_path=r.path("DATABASE_PATH", "data/bot.db"),
|
||||
log_dir=r.path("LOG_DIR", "logs"),
|
||||
max_temp_storage=r.size("MAX_TEMP_STORAGE", 0),
|
||||
min_free_disk_space=r.size("MIN_FREE_DISK_SPACE", 1024**3),
|
||||
keep_local_files=r.bool("KEEP_LOCAL_FILES", False),
|
||||
keep_failed_downloads=r.bool("KEEP_FAILED_DOWNLOADS", False),
|
||||
temp_cleanup_interval=r.float("TEMP_CLEANUP_INTERVAL", 3600.0, minimum=0.0),
|
||||
compute_sha256=r.bool("COMPUTE_SHA256", True),
|
||||
max_attempts=r.int("MAX_RETRIES", 4, minimum=0) + 1,
|
||||
retry_base_delay=r.float("RETRY_DELAY", 30.0),
|
||||
retry_max_delay=r.float("RETRY_MAX_DELAY", 1800.0),
|
||||
http_retries=r.int("HTTP_RETRIES", 5, minimum=0),
|
||||
processing_delay=r.float("PROCESSING_DELAY", 2.0),
|
||||
strict_stop=r.bool("STRICT_STOP", False),
|
||||
auto_start=r.bool("AUTO_START", False),
|
||||
instance_lock_wait=r.float("INSTANCE_LOCK_WAIT", 90.0 if env.get("DYNO") else 0.0),
|
||||
upload_setup_timeout=r.float("UPLOAD_SETUP_TIMEOUT", 30.0, minimum=1.0),
|
||||
upload_max_range=r.int("UPLOAD_MAX_RANGE", 5000, minimum=1),
|
||||
upload_restrict_to_source_channel=r.bool("UPLOAD_RESTRICT_TO_SOURCE_CHANNEL", True),
|
||||
progress_interval=r.float("PROGRESS_INTERVAL", 10.0, minimum=1.0),
|
||||
log_level=r.choice("LOG_LEVEL", "info", ("debug", "info", "warning", "error")).upper(),
|
||||
)
|
||||
if config.upload_chunk_size <= 0 or config.upload_chunk_size % UPLOAD_CHUNK_ALIGN:
|
||||
r.problems.append("UPLOAD_CHUNK_SIZE must be a positive multiple of 256KB (e.g. 8MB, 16MB, 32MB)")
|
||||
try:
|
||||
config.sequence_prefix_format.format(seq=1)
|
||||
except (KeyError, IndexError, ValueError):
|
||||
r.problems.append("SEQUENCE_PREFIX_FORMAT must be a format string using {seq}, e.g. '{seq:03d} - '")
|
||||
resolved = config.download_dir.resolve()
|
||||
if resolved in {Path("/").resolve(), Path.home().resolve(), PROJECT_DIR.resolve()} or len(resolved.parts) < 3:
|
||||
r.problems.append(
|
||||
f"DOWNLOAD_DIR={config.download_dir} is not a dedicated directory; use e.g. 'downloads' or '/tmp/telegram_drive_bot'"
|
||||
)
|
||||
if r.problems:
|
||||
raise ConfigError(r.problems)
|
||||
return config
|
||||
|
||||
# -- section validation -------------------------------------------------------
|
||||
|
||||
def telegram_problems(self) -> list[str]:
|
||||
problems = []
|
||||
if not self.api_id or not self.api_hash:
|
||||
problems.append(
|
||||
"TELEGRAM_API_ID and TELEGRAM_API_HASH are required (create them at https://my.telegram.org -> API development tools)"
|
||||
)
|
||||
if self.telegram_mode == "bot" and not self.bot_token:
|
||||
problems.append("TELEGRAM_BOT_TOKEN is required when TELEGRAM_MODE=bot (get one from @BotFather)")
|
||||
if self.telegram_mode == "user" and self.is_heroku and not self.session_string:
|
||||
problems.append(
|
||||
"TELEGRAM_SESSION_STRING is required for TELEGRAM_MODE=user on Heroku (the filesystem is ephemeral, "
|
||||
"so a session file would be lost). Generate it locally with `python main.py login --session-string`."
|
||||
)
|
||||
if self.bot_token and not self.admin_ids:
|
||||
problems.append("ADMIN_USER_IDS must list at least one Telegram user ID allowed to control the bot")
|
||||
if not self.source_channel:
|
||||
problems.append("SOURCE_CHANNEL_ID is required (@username, https://t.me/... link, or -100... numeric ID)")
|
||||
return problems
|
||||
|
||||
def drive_problems(self) -> list[str]:
|
||||
problems = []
|
||||
if self.google_auth_mode == "oauth":
|
||||
if self.google_oauth_token_json:
|
||||
problems += _json_problems(self.google_oauth_token_json, "GOOGLE_OAUTH_TOKEN_JSON", ("refresh_token", "client_id", "client_secret"))
|
||||
elif self.is_heroku:
|
||||
problems.append(
|
||||
"GOOGLE_OAUTH_TOKEN_JSON is required on Heroku when GOOGLE_AUTH_MODE=oauth "
|
||||
"(create it locally with `python main.py drive-auth`, see README)"
|
||||
)
|
||||
elif not self.google_token_file.exists() and not self.google_client_secrets_file.exists():
|
||||
problems.append(
|
||||
f"Google OAuth client file not found at {self.google_client_secrets_file.name!r} "
|
||||
"(set GOOGLE_CLIENT_SECRETS_FILE or GOOGLE_OAUTH_TOKEN_JSON; see README)"
|
||||
)
|
||||
else:
|
||||
if self.google_service_account_json:
|
||||
problems += _json_problems(
|
||||
self.google_service_account_json, "GOOGLE_SERVICE_ACCOUNT_JSON", ("client_email", "private_key")
|
||||
)
|
||||
elif not self.google_service_account_file or not self.google_service_account_file.exists():
|
||||
problems.append(
|
||||
"GOOGLE_AUTH_MODE=service_account needs GOOGLE_SERVICE_ACCOUNT_JSON (Heroku) "
|
||||
"or GOOGLE_SERVICE_ACCOUNT_FILE pointing to the JSON key"
|
||||
)
|
||||
return problems
|
||||
|
||||
def database_problems(self) -> list[str]:
|
||||
if self.is_heroku and not self.database_url and not self.allow_ephemeral_database:
|
||||
return [
|
||||
"DATABASE_URL is not set. On Heroku the local filesystem is wiped on every restart, so SQLite would "
|
||||
"lose all progress and cause re-uploads. Attach Heroku Postgres (`heroku addons:create "
|
||||
"heroku-postgresql:essential-0`) or set ALLOW_EPHEMERAL_DATABASE=true to accept volatile state."
|
||||
]
|
||||
return []
|
||||
|
||||
def validate(self, *sections: str) -> None:
|
||||
problems: list[str] = []
|
||||
if "telegram" in sections:
|
||||
problems += self.telegram_problems()
|
||||
if "drive" in sections:
|
||||
problems += self.drive_problems()
|
||||
if "database" in sections:
|
||||
problems += self.database_problems()
|
||||
if problems:
|
||||
raise ConfigError(problems)
|
||||
|
||||
def secrets(self) -> list[str]:
|
||||
"""Values that must never appear in logs."""
|
||||
values = [self.api_hash, self.bot_token, self.session_string, self.database_url]
|
||||
for blob in (self.google_oauth_token_json, self.google_service_account_json):
|
||||
try:
|
||||
data = load_json_secret(blob) if blob else {}
|
||||
except ValueError:
|
||||
data = {}
|
||||
values += [data.get(k) for k in ("refresh_token", "client_secret", "private_key", "token")]
|
||||
values.append(blob)
|
||||
return [v for v in values if v and len(v) >= 8]
|
||||
|
||||
|
||||
def load_json_secret(value: str) -> dict:
|
||||
"""Parse a JSON credential given either as raw JSON or base64-encoded JSON.
|
||||
|
||||
Base64 is accepted because some dashboards mangle multi-line values; raw JSON
|
||||
(including multi-line) works fine through `heroku config:set VAR="$(cat file)"`.
|
||||
"""
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
|
||||
text = value.strip()
|
||||
if not text.startswith("{"):
|
||||
try:
|
||||
text = base64.b64decode(text, validate=False).decode("utf-8").strip()
|
||||
except (binascii.Error, UnicodeDecodeError) as exc:
|
||||
raise ValueError("value is neither JSON nor base64-encoded JSON") from exc
|
||||
try:
|
||||
data = json.loads(text)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"invalid JSON ({exc.msg} at line {exc.lineno})") from exc
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError("expected a JSON object")
|
||||
return data
|
||||
|
||||
|
||||
def _json_problems(value: str, name: str, required_keys: tuple[str, ...]) -> list[str]:
|
||||
try:
|
||||
data = load_json_secret(value)
|
||||
except ValueError as exc:
|
||||
return [f"{name} could not be parsed: {exc}"]
|
||||
missing = [k for k in required_keys if not data.get(k)]
|
||||
return [f"{name} is missing keys: {', '.join(missing)}"] if missing else []
|
||||
|
||||
|
||||
def _normalize_database_url(url: str | None) -> str | None:
|
||||
"""Heroku provides postgres://...; enforce TLS for remote hosts."""
|
||||
if not url:
|
||||
return None
|
||||
if url.startswith("postgres://"):
|
||||
url = "postgresql://" + url[len("postgres://"):]
|
||||
host_part = url.split("@")[-1]
|
||||
remote = not (host_part.startswith(("localhost", "127.0.0.1", "/")) or "host=/" in url)
|
||||
if remote and "sslmode=" not in url:
|
||||
url += ("&" if "?" in url else "?") + "sslmode=require"
|
||||
return url
|
||||
Reference in New Issue
Block a user