"""Acceptance tests for the interactive /upload workflow (mocked Telegram and Drive)."""
import asyncio
from datetime import timedelta
import pytest
from bot_handlers import CommandRouter
from conftest import (
CHANNEL_ID,
FakeDrive,
FakeSource,
RecordingNotifier,
SimulatedCrash,
build_queue,
make_config,
run,
video_bytes,
)
from database import Database, iso, utcnow
from errors import PermanentError
from link_parser import internal_channel_id
from models import State, TaskStatus
from progress import ProgressReporter
A, B, STRANGER = 111, 222, 999
INTERNAL = internal_channel_id(CHANNEL_ID) # 1234567890
OTHER_CHANNEL = -1000999999999 # accessible, but not the configured channel
def link(message_id, internal=INTERNAL):
return f"https://t.me/c/{internal}/{message_id}"
def caption(title, topic):
return f"Index: 001\nTitle: {title}\nTopic: {topic}\nBatch: SSC\nExtracted By: @x"
def populate(source, size=600_000):
"""Messages 13..21; 15 is text, 18 is missing; 13 and 21 lie outside the 14-20 range."""
source.add(13, caption("Outside Before.mp4", "Topic X"), video_bytes(size, 13))
source.add(14, caption("Class 01 Aman sir.mp4", "Misc• Grammar Practice"), video_bytes(size, 14))
source.add_text(15, "Announcement: new classes tomorrow")
source.add(16, caption("Class 01 Aman sir.mp4", "Error Detection"), video_bytes(size, 16))
source.add(17, caption("Class 02 Aman sir.mp4", "Misc• Grammar Practice"), video_bytes(size, 17))
source.add(19, caption("Class 03 Aman sir", "Misc• Grammar Practice"), video_bytes(size, 19))
source.add(20, "", video_bytes(size, 20))
source.add(21, caption("Outside After.mp4", "Topic X"), video_bytes(size, 21))
source.accessible[OTHER_CHANNEL] = "Other Channel"
class Env:
def __init__(self, config, db, source, drive, notifier=None):
self.config, self.db, self.source, self.drive = config, db, source, drive
self.notifier = notifier or RecordingNotifier()
reporter = ProgressReporter(self.notifier, [A], interval=60)
self.queue = build_queue(config, db, source, drive, reporter=reporter)
self.router = CommandRouter(config, db, self.queue)
async def say(self, user, *texts):
replies = []
for text in texts:
replies.append(await self.router.handle(user, text))
return replies
def setup(self, user):
return self.db.tasks_with_status(
TaskStatus.AWAITING_START, TaskStatus.AWAITING_END, TaskStatus.AWAITING_CONFIRMATION, admin_user_id=user
)
def drive_names(self):
return sorted((self.drive.files[f.parents[0]].name, f.name) for f in self.drive.uploaded())
@pytest.fixture
def env(tmp_path, db_url):
extra = {"DATABASE_URL": db_url} if db_url else {}
config = make_config(tmp_path, ADMIN_USER_IDS=f"{A},{B}", **extra)
db = Database.from_config(config)
source = FakeSource()
populate(source)
e = Env(config, db, source, FakeDrive())
yield e
db.close()
async def full_task(env, start=14, end=20, user=A):
replies = await env.say(user, "/upload", link(start), link(end), "/confirm")
await env.queue.wait()
return replies
# 1-4, 7-10: conversation, inclusive ascending range, metadata, skips ------------------------
def test_upload_conversation_and_range_processing(env):
async def go():
r = await env.say(A, "/upload")
assert "📥 Start Upload Task" in r[0] and "https://t.me/c/3909337424/14" in r[0]
assert env.setup(A)[0].status == TaskStatus.AWAITING_START
r = await env.say(A, link(14))
assert "📌 Enter the End Link" in r[0]
assert env.setup(A)[0].start_message_id == 14
r = await env.say(A, link(20))
assert "✅ Upload Task Ready" in r[0]
assert "Start Message ID: 14" in r[0] and "End Message ID: 20" in r[0]
assert "Test Channel (configured source channel)" in r[0]
assert env.source.download_offsets == [] # nothing downloaded before /confirm
r = await env.say(A, "/confirm")
assert "Upload Task #1 confirmed" in r[0]
await env.queue.wait()
run(go())
task = env.db.get_task(1)
assert task.status == TaskStatus.COMPLETED
# inclusive range 14..20, nothing outside it
assert [m for m, _ in env.db.task_items(1)] == [14, 15, 16, 17, 18, 19, 20]
downloaded = [m for kind, m in env.source.events if kind == "download"]
assert downloaded == [14, 16, 17, 19, 20] # ascending, videos only
assert not env.db.exists(CHANNEL_ID, 13) and not env.db.exists(CHANNEL_ID, 21)
# caption metadata -> folders and filenames
assert env.drive_names() == [
("Error Detection", "Class 01 Aman sir.mp4"),
("Misc• Grammar Practice", "Class 01 Aman sir.mp4"),
("Misc• Grammar Practice", "Class 02 Aman sir.mp4"),
("Misc• Grammar Practice", "Class 03 Aman sir.mp4"),
("Uncategorized", "telegram_20.mp4"),
]
# non-video and missing messages are skipped, not failed
assert dict(env.db.task_items(1))[15] == "non_video"
assert dict(env.db.task_items(1))[18] == "missing"
assert (task.uploaded_count, task.failed_count, task.non_video_count, task.missing_count) == (5, 0, 1, 1)
assert task.examined_count == 6 and task.eligible_count == 5
assert task.deleted_files_count == 5
assert list(env.config.download_dir.iterdir()) == []
summary = [t for _, t in env.notifier.sent if "Upload Task Completed" in t][0]
for fragment in ("Requested range: 14–20", "Eligible videos found: 5", "Uploaded: 5", "Duplicates skipped: 0",
"Non-video messages skipped: 1", "Failed: 0", "Local temporary files deleted: 5",
"Misc• Grammar Practice", "Error Detection"):
assert fragment in summary
def test_confirm_requires_both_validated_links(env):
async def go():
assert "no upload task waiting" in (await env.say(A, "/confirm"))[0]
await env.say(A, "/upload")
assert "Both the start and end links" in (await env.say(A, "/confirm"))[0]
await env.say(A, link(14))
assert "Both the start and end links" in (await env.say(A, "/confirm"))[0]
assert env.db.next_runnable_task() is None
run(go())
assert env.source.download_offsets == []
@pytest.mark.parametrize(
"bad,fragment",
[
("example.com/post/14", "not a Telegram"),
("https://t.me/+invite", "Invite links"),
("https://t.me/c/1234567890", "must look like"),
("https://t.me/c/555/14", "not accessible"), # channel the account cannot access
(link(14, internal_channel_id(OTHER_CHANNEL)), "not from the configured source channel"),
(link(18), "not found"), # deleted message
],
)
def test_invalid_start_links_rejected(env, bad, fragment):
replies = run(env.say(A, "/upload", bad))
assert replies[1].startswith("❌") and fragment in replies[1]
assert env.setup(A)[0].status == TaskStatus.AWAITING_START
def test_end_link_errors_keep_start_link(env):
replies = run(env.say(
A, "/upload", link(14),
link(20, internal_channel_id(OTHER_CHANNEL)), # different channel
link(13), # before start
"https://t.me/c/1234567890/abc", # malformed
))
assert "different channel" in replies[2]
assert "greater than or equal" in replies[3]
assert replies[4].startswith("❌")
setup = env.setup(A)[0]
assert setup.status == TaskStatus.AWAITING_END and setup.start_message_id == 14
assert "Upload Task Ready" in run(env.say(A, link(14)))[0] # single-message range is allowed
def test_other_channels_allowed_when_unrestricted(tmp_path):
config = make_config(tmp_path, ADMIN_USER_IDS=str(A), UPLOAD_RESTRICT_TO_SOURCE_CHANNEL="false")
db = Database.from_config(config)
source = FakeSource()
populate(source)
e = Env(config, db, source, FakeDrive())
async def other_exists(channel_id, message_id):
return True
source.message_accessible = other_exists
replies = run(e.say(A, "/upload", link(1, internal_channel_id(OTHER_CHANNEL)), link(5, internal_channel_id(OTHER_CHANNEL))))
assert "Channel: Other Channel" in replies[2]
def test_range_size_limit(tmp_path):
config = make_config(tmp_path, ADMIN_USER_IDS=str(A), UPLOAD_MAX_RANGE="3")
e = Env(config, Database.from_config(config), FakeSource(), FakeDrive())
populate(e.source)
replies = run(e.say(A, "/upload", link(14), link(20)))
assert "maximum is 3" in replies[2]
def test_non_admins_cannot_use_upload_commands(env):
for command in ("/upload", "/confirm", "/cancel", "/taskstatus", "/uploadhistory"):
assert "not authorized" in run(env.say(STRANGER, command))[0]
assert run(env.say(STRANGER, link(14)))[0] is None # plain text from non-admins is ignored
assert env.db.recent_tasks() == [] and env.setup(STRANGER) == []
# 15: administrators are isolated ------------------------------------------------------
def test_admin_setups_are_isolated(env):
async def go():
await env.say(A, "/upload", link(14))
r = await env.say(B, link(16))
assert "send /upload first" in r[0]
await env.say(B, "/upload", link(16))
assert env.setup(A)[0].start_message_id == 14
assert env.setup(B)[0].start_message_id == 16
r = await env.say(B, "/cancel")
assert "cancelled" in r[0]
assert env.setup(B) == []
assert env.setup(A)[0].status == TaskStatus.AWAITING_END # untouched
await env.say(A, link(17), "/confirm")
r = await env.say(B, "/cancel")
assert "belong to another administrator" in r[0]
await env.queue.wait()
run(go())
assert env.db.get_task(1).status == TaskStatus.COMPLETED
def test_second_upload_does_not_overwrite_setup(env):
replies = run(env.say(A, "/upload", link(14), "/upload"))
assert "already have an upload setup" in replies[2]
setup = env.setup(A)
assert len(setup) == 1 and setup[0].start_message_id == 14
# setup persistence / expiry ------------------------------------------------------------
def test_setup_survives_restart(env):
run(env.say(A, "/upload", link(14)))
# new process: fresh queue/router on the same durable database
again = Env(env.config, env.db, env.source, env.drive)
replies = run(again.say(A, link(20)))
assert "Upload Task Ready" in replies[0]
def test_stale_setup_expires(env):
run(env.say(A, "/upload", link(14)))
setup = env.setup(A)[0]
env.db.update_task(setup.id, updated_at=iso(utcnow() - timedelta(minutes=env.config.upload_setup_timeout + 1)))
reply = run(env.say(A, "/confirm"))[0]
assert "expired" in reply
assert env.db.get_task(setup.id).status == TaskStatus.CANCELLED
assert "Start Upload Task" in run(env.say(A, "/upload"))[0]
# 11, 12: cleanup ordering and failures -------------------------------------------------
def test_local_file_deleted_only_after_verified_upload_is_recorded(env):
seen = []
real_cleanup = env.queue.downloader.cleanup
def spy(rec):
current = env.db.get(rec.key)
if current.state == State.COMPLETED:
assert current.drive_file_id in env.drive.files
assert env.drive.files[current.drive_file_id].size == current.local_size
assert env.config.download_dir.joinpath(f"{abs(CHANNEL_ID)}_{rec.message_id}.video").exists()
seen.append((rec.message_id, current.state))
return real_cleanup(rec)
env.queue.downloader.cleanup = spy
run(full_task(env))
assert seen and all(state == State.COMPLETED for _, state in seen)
def test_failed_upload_is_not_completed(env):
env.drive.upload_failures = [PermanentError("insufficientFilePermissions")]
run(full_task(env))
rec = env.db.get((CHANNEL_ID, 14))
assert rec.state == State.FAILED and rec.drive_file_id is None
task = env.db.get_task(1)
assert task.status == TaskStatus.COMPLETED and task.failed_count == 1 and task.uploaded_count == 4
assert dict(env.db.task_items(1))[14] == "failed"
assert not any(p.name.startswith(f"{abs(CHANNEL_ID)}_14.") for p in env.config.download_dir.iterdir())
def test_failed_download_file_can_be_kept(tmp_path):
config = make_config(tmp_path, ADMIN_USER_IDS=str(A), KEEP_FAILED_DOWNLOADS="true")
e = Env(config, Database.from_config(config), FakeSource(), FakeDrive())
populate(e.source)
e.drive.upload_failures = [PermanentError("denied")]
run(full_task(e))
assert (config.download_dir / f"{abs(CHANNEL_ID)}_14.video").exists()
# 14: repeating a task ------------------------------------------------------------------
def test_repeating_completed_task_does_not_duplicate(env):
run(full_task(env))
calls = env.drive.upload_calls
run(full_task(env))
second = env.db.get_task(2)
assert second.status == TaskStatus.COMPLETED
assert second.uploaded_count == 0 and second.duplicate_count == 5
assert env.drive.upload_calls == calls and len(env.drive.uploaded()) == 5
# 16: cancellation ----------------------------------------------------------------------
def test_cancel_running_task_keeps_completed_work(tmp_path):
config = make_config(tmp_path, ADMIN_USER_IDS=str(A))
db = Database.from_config(config)
source = FakeSource()
populate(source, size=3_000_000)
e = Env(config, db, source, FakeDrive())
part16 = config.download_dir / f"{abs(CHANNEL_ID)}_16.part"
async def go():
await e.say(A, "/upload", link(14), link(20), "/confirm")
while not part16.exists():
await asyncio.sleep(0.001)
reply = (await e.say(A, "/cancel"))[0]
assert "cancelled at a safe checkpoint" in reply
await e.queue.wait()
run(go())
task = db.get_task(1)
assert task.status == TaskStatus.CANCELLED
assert db.get((CHANNEL_ID, 14)).state == State.COMPLETED
assert len(e.drive.uploaded()) == 1 # completed upload preserved in Drive
for message_id in (16, 17, 19, 20):
rec = db.get((CHANNEL_ID, message_id))
assert rec.state == State.SKIPPED and rec.skip_reason.startswith("cancelled:")
assert list(config.download_dir.iterdir()) == []
assert "Upload Task Cancelled" in "".join(t for _, t in e.notifier.sent)
# the same range can be requested again: cancelled items are re-queued, done ones are not re-uploaded
run(full_task(e))
second = db.get_task(2)
assert second.uploaded_count == 4 and second.duplicate_count == 1
assert len(e.drive.uploaded()) == 5
def test_cancel_queued_task_and_explicit_id(env):
async def go():
await env.say(A, "/pause") # nothing running -> nothing to pause
env.queue._persist_desired("paused")
await env.say(A, "/upload", link(14), link(20), "/confirm")
assert env.db.get_task(1).status == TaskStatus.QUEUED
reply = (await env.say(B, "/cancel 1"))[0]
assert "Task #1 cancelled" in reply
assert "already finished" in (await env.say(A, "/cancel 1"))[0]
run(go())
assert env.db.get_task(1).status == TaskStatus.CANCELLED
assert env.source.download_offsets == []
# 13, 18: restart recovery ----------------------------------------------------------------
def test_task_recovers_after_crash_and_restart(tmp_path, db_url):
extra = {"DATABASE_URL": db_url} if db_url else {}
config = make_config(tmp_path, ADMIN_USER_IDS=str(A), **extra)
source = FakeSource()
populate(source, size=1_500_000)
drive = FakeDrive()
db = Database.from_config(config)
e = Env(config, db, source, drive)
drive.crash_at_bytes = 700_000 # the process is killed half-way through the first upload
async def first_run():
await e.say(A, "/upload", link(14), link(20), "/confirm")
with pytest.raises(SimulatedCrash):
await e.queue.wait()
run(first_run())
assert db.get_task(1).status == TaskStatus.RUNNING
assert db.get((CHANNEL_ID, 14)).state == State.UPLOADING
db.close()
# Heroku restarts the worker: new process, same durable database
db2 = Database.from_config(config)
db2.recover_stale()
e2 = Env(config, db2, source, drive)
async def second_run():
assert e2.queue.boot() == "running"
await e2.queue.wait()
run(second_run())
task = db2.get_task(1)
assert task.status == TaskStatus.COMPLETED and task.uploaded_count == 5
assert len(drive.uploaded()) == 5 # no duplicates
assert drive.resumed # the interrupted upload session was resumed
def test_pause_resume_task_and_paused_boot(env):
async def go():
await env.say(A, "/upload", link(14), link(20), "/confirm")
while env.db.get((CHANNEL_ID, 14)) is None or env.db.get((CHANNEL_ID, 14)).state != State.COMPLETED:
await asyncio.sleep(0.001)
assert "Pausing" in (await env.say(A, "/pause"))[0]
await asyncio.sleep(0.05)
assert env.db.get_task(1).status == TaskStatus.PAUSED
# a restart while paused stays paused
assert Env(env.config, env.db, env.source, env.drive).queue.boot() == "paused"
assert "resumed" in (await env.say(A, "/resume"))[0]
await env.queue.wait()
run(go())
assert env.db.get_task(1).status == TaskStatus.COMPLETED
def test_status_and_history_commands(env):
async def go():
await env.say(A, "/upload", link(14))
status = (await env.say(A, "/taskstatus"))[0]
assert "waiting for the end link" in status
await env.say(A, link(20), "/confirm")
await env.queue.wait()
status = (await env.say(A, "/taskstatus 1"))[0]
assert "Task #1 — COMPLETED" in status and "Range: 14–20" in status
history = (await env.say(A, "/uploadhistory"))[0]
assert "#1 14–20 COMPLETED — uploaded 5" in history
assert "Upload tasks" not in (await env.say(A, "/status"))[0]
run(go())