fix: recover expired task leases and persist diagnostic logs

This commit is contained in:
薇星幻海
2026-10-18 02:30:56 +08:00
parent 1b497286dc
commit ef35b461f7
15 changed files with 1368 additions and 85 deletions
+88 -22
View File
@@ -8,7 +8,7 @@
'use strict';
const TERMINAL = ['completed', 'skipped', 'failed', 'expanded'];
const ACTIVE = ['loading', 'playing', 'quiz', 'answering', 'waitingSubmit', 'retrying'];
const ACTIVE = ['loading', 'playing', 'quiz', 'answering', 'waitingSubmit', 'waitingLease', 'retrying'];
const MODES = ['video', 'videoQuiz', 'exam', 'all'];
const LEASE_MS = 18000;
const SESSION_KEY = 'session.v1';
@@ -188,6 +188,7 @@
let interactionAt = -Infinity; let lastAutoPlayAt = -Infinity; let manualVideoPause = false; let watchedDocument = null;
let visibilityPlayback = null; let returnResumeRequested = false; let pageAway = false; let playbackPromptClick = null;
let sharedPaused = false;
let logSequence = 0; let leaseBanner = null;
let manualConfirmRevision = 0; let pendingManualConfirm = null;
let policySessionId = '', policyReadSerial = 0;
const authoritativePolicies = new Set(), policyAppliedReads = new Map(), policyLocalWrites = new Map();
@@ -231,9 +232,10 @@
const counted = state.tasks.filter(t => t.status !== 'expanded' && allows(t.kind));
state.total = counted.length; state.completed = counted.filter(t => t.status === 'completed').length;
state.skipped = counted.filter(t => t.status === 'skipped').length; state.waiting = counted.filter(t => t.status === 'waiting').length; state.failed = counted.filter(t => t.status === 'failed').length;
state.leaseWaiting = counted.filter(t => t.status === 'waitingLease').length;
const todo = counted.filter(t => !TERMINAL.includes(t.status));
const estimates = todo.map(t => {
if (t.status === 'waiting' || t.kind === 'course' || t.kind === 'unsupported') return null;
if (['waiting', 'waitingLease'].includes(t.status) || t.kind === 'course' || t.kind === 'unsupported') return null;
if (t.kind === 'video') {
const v = state.video && state.video.taskId === t.id ? state.video : t;
if (!(v.duration > 0)) return null;
@@ -251,13 +253,15 @@
state.etaSeconds = estimates.some(e => e == null) ? null : config.execution === 'parallel' ? Math.max(0, ...['video', 'exam'].map(kind => estimates.reduce((n, e, i) => n + (todo[i].kind === kind ? e : 0), 0))) : estimates.reduce((n, e) => n + e, 0);
if (typeof options.onState === 'function') options.onState(clone(state));
}
function log(level, message, task) {
state.logs.push({ time: now(), level: ['info', 'warn', 'error'].includes(level) ? level : 'info', message: String(message), task: task && task.title || '', taskId: task && task.id || '' });
function log(level, message, task, details, event) {
const record = { id: owner + ':' + (++logSequence), time: now(), level: ['info', 'warn', 'error'].includes(level) ? level : 'info', event: event || 'learning.message', message: String(message), task: task && task.title || '', taskId: task && task.id || '', details: details || {} };
state.logs.push(record);
if (typeof options.onLog === 'function') { try { options.onLog(clone(record)); } catch (_) {} }
state.logs = state.logs.slice(-500); notify();
}
function taskState(task, status, reason) {
const changed = task.status !== status || task.reason !== (reason || ''); task.status = status; task.reason = reason || '';
if (changed) log(status === 'waiting' || status === 'failed' ? 'warn' : 'info', reason || status, task);
if (changed) log(status === 'waiting' || status === 'failed' ? 'warn' : 'info', reason || status, task, { kind: task.kind, status }, 'task.state');
}
function allows(kind) { return kind === 'course' || kind === 'unsupported' || kind === 'video' && config.mode !== 'exam' || kind === 'exam' && (config.mode === 'exam' || config.mode === 'all'); }
function account(snap, page) { return clean(snap.accountId || page.accountId || ''); }
@@ -294,7 +298,49 @@
if (signature !== saveSignature) { await storage.set(SESSION_KEY, clone(session)); saveSignature = signature; }
}
async function release() {
for (const kind of ['video', 'exam']) { const key = 'lease:' + kind; const lease = await storage.get(key); if (lease && lease.owner === owner) await storage.set(key, null); }
for (const kind of ['video', 'exam']) {
const key = 'lease:' + kind; const lease = await storage.get(key);
if (lease && lease.owner === owner) { await storage.set(key, null); log('info', '已释放' + (kind === 'video' ? '视频/课程' : '考试') + '执行权', null, lease, 'lease.released'); }
}
}
function waitForLease(task, lease, reason) {
const previous = task.leaseWait;
task.leaseWait = { previousStatus: previous?.previousStatus || (task.status === 'waitingLease' ? 'pending' : task.status), previousReason: previous ? previous.previousReason : task.reason || '', lastLoggedAt: previous?.lastLoggedAt,
since: previous?.since == null ? now() : previous.since, kind: task.kind === 'course' ? 'video' : task.kind,
owner: lease?.owner || '', taskId: lease?.taskId || '', expires: Number(lease?.expires) || 0, pageUrl: lease?.pageUrl || '', reason };
if (!previous && activeTaskId === task.id) {
// 等待执行权期间取消本页旧动作;重新领取后仍须按当前页面重验。
if (task.kind === 'video') {
const snap = snapshot(); const video = snap.document && describePage(snap.document, snap.url).video;
if (video) video.pause(); playStarted = false; lastTime = null; attemptedAt = 0; lastAdvancedAt = now();
if (state.video) state.video.status = 'waitingLease';
}
if (task.kind === 'exam' && task.examRequested && task.leaseWait.previousStatus !== 'waiting') {
task.examNeedsRecheck = true;
if (typeof options.onExamPause === 'function') options.onExamPause('本页已失去考试执行权,停止旧自动作答并等待重新核验');
}
}
const secondsLeft = Math.max(0, Math.ceil((task.leaseWait.expires - now()) / 1000));
const holder = lease?.title || lease?.taskId || '另一页面任务';
const message = reason === 'control' ? '正在重新核对全局任务状态,暂不操作当前页面'
: '等待' + (task.leaseWait.kind === 'exam' ? '考试' : '视频/课程') + '执行权:' + holder + '由另一个页面处理,约 ' + secondsLeft + ' 秒后检查;执行权释放后自动继续';
taskState(task, 'waitingLease', '同类任务由另一个页面处理,自动等待并重试');
state.message = message; leaseBanner = { taskId: task.id, message };
if (!previous || previous.owner !== task.leaseWait.owner || previous.taskId !== task.leaseWait.taskId || previous.reason !== reason || now() - (previous.lastLoggedAt || 0) >= 15000) {
task.leaseWait.lastLoggedAt = now();
log('info', message, task, Object.assign({ ownOwner: owner }, task.leaseWait), 'lease.wait');
}
}
function acquiredLease(task) {
if (!task.leaseWait) return;
const waited = task.leaseWait;
if (task.status === 'waitingLease') { task.status = waited.previousStatus || 'pending'; task.reason = waited.previousReason || ''; }
delete task.leaseWait;
if (leaseBanner?.taskId === task.id) {
if (state.message === leaseBanner.message) state.message = '已取得' + (task.kind === 'exam' ? '考试' : '视频/课程') + '执行权,正在继续任务';
leaseBanner = null;
}
log('info', '等待结束,已取得执行权并重新核对任务', task, { waitedMs: Math.max(0, now() - waited.since), kind: waited.kind, previousOwner: waited.owner }, 'lease.resumed');
}
async function claim(task) {
const kind = task.kind === 'course' ? 'video' : task.kind;
@@ -302,13 +348,16 @@
async function acquire() {
const key = 'lease:' + kind; const lease = await storage.get(key);
const shared = await storage.get(CONTROL_KEY);
if (shared && (shared.sessionId !== session.id || shared.status !== 'running')) return false;
if (lease && lease.owner !== owner && lease.sessionId === session.id && lease.expires > now()) return false;
const wanted = { owner, taskId: task.id, sessionId: session.id, expires: now() + LEASE_MS };
if (shared && (shared.sessionId !== session.id || shared.status !== 'running')) { waitForLease(task, null, 'control'); return false; }
if (lease && lease.owner !== owner && lease.sessionId === session.id && lease.expires > now()) { waitForLease(task, lease, 'busy'); return false; }
if (lease && lease.expires <= now()) log('info', '原执行权已到期,重新领取任务', task, lease, 'lease.expired');
const wanted = { owner, taskId: task.id, title: task.title, pageUrl: canonical(snapshot().url), sessionId: session.id, expires: now() + LEASE_MS, updatedAt: now() };
await storage.set(key, wanted);
// GM存储无CAS;延后回读并在每次动作前复核。支持Web Locks时同源标签领取是互斥的。
if (!(root.navigator && root.navigator.locks)) await new Promise(resolve => root.setTimeout(resolve, 25));
const saved = await storage.get(key); return Boolean(saved && saved.owner === owner && saved.taskId === task.id);
const saved = await storage.get(key);
if (!saved || saved.owner !== owner || saved.taskId !== task.id || saved.sessionId !== session.id) { waitForLease(task, saved, 'race'); return false; }
acquiredLease(task); log('info', '已取得' + (kind === 'exam' ? '考试' : '视频/课程') + '执行权', task, wanted, 'lease.acquired'); return true;
}
return root.navigator && root.navigator.locks ? root.navigator.locks.request('yian-learning-' + kind, acquire) : acquire();
}
@@ -316,7 +365,9 @@
const key = 'lease:' + (task.kind === 'course' ? 'video' : task.kind); const lease = await storage.get(key);
if (!lease || lease.owner !== owner || lease.taskId !== task.id || lease.sessionId !== session.id) return false;
const shared = await storage.get(CONTROL_KEY); if (shared && (shared.sessionId !== session.id || shared.status !== 'running')) return false;
lease.expires = now() + LEASE_MS; await storage.set(key, lease); return true;
if (lease.expires <= now()) log('info', '本页原执行权已到期,核对所有者后恢复', task, lease, 'lease.renewed');
lease.expires = now() + LEASE_MS; lease.updatedAt = now(); lease.pageUrl = canonical(snapshot().url); lease.title = task.title;
await storage.set(key, lease); return true;
}
function mergeTasks(tasks, parent) {
tasks.forEach(task => {
@@ -375,6 +426,9 @@
const identityOK = (!saved.accountId || saved.accountId === account(snap, page)) && (!saved.planId || !snap.planId || saved.planId === snap.planId) && (!saved.bankVersion || !snap.bankVersion || saved.bankVersion === snap.bankVersion);
if (identityOK) {
session = saved; Object.assign(config, saved.config || {}); state.tasks = saved.tasks; state.logs = Array.isArray(saved.logs) ? saved.logs.slice(-500) : []; activeTaskId = saved.activeTaskId || '';
for (const task of state.tasks) if (task.status === 'waiting' && task.reason === '另一个页面正在处理同类任务,避免重复领取') {
task.status = 'waitingLease'; task.leaseWait = { previousStatus: 'pending', previousReason: '', since: now() };
}
if (marker) {
const request = await storage.get('worker:' + marker[1]);
const control = await storage.get(CONTROL_KEY);
@@ -547,7 +601,7 @@
task.preflight = preflightTopic(snap.document, snap, config.associations, task);
if (task.preflight.status !== 'matched') { taskState(task, task.preflight.status === 'missing' ? 'skipped' : 'waiting', task.preflight.reason); return false; }
}
if (!(await claim(task))) { taskState(task, 'waiting', '另一个页面正在处理同类任务,避免重复领取'); return false; }
if (!(await claim(task))) return false;
activeTaskId = task.id; playStarted = false; attemptedAt = 0; lastTime = null; lastAdvancedAt = now(); manualVideoPause = false; visibilityPlayback = null; returnResumeRequested = false;
const onPage = canonical(task.url || task.pageUrl) === page.url;
const isCurrentVideo = task.kind === 'video' && page.video && (task.current || resolveNode(task, snap) === page.activeCourseware);
@@ -618,7 +672,7 @@
if (epoch !== generation || state.status !== 'running') return;
if (!result || result.status !== 'completed' || result.verified !== true || quizOf(snap.document)) taskState(task, 'waiting', result && result.reason || '课中题尚未确认提交成功,请人工核对后继续');
else { taskState(task, 'playing', '课中题已核验且弹题已关闭,继续视频'); lastAdvancedAt = now(); }
} catch (error) { if (epoch === generation) taskState(task, 'waiting', '课中题处理失败:' + error.message); }
} catch (error) { if (epoch === generation) { log('error', '课中题处理发生异常', task, { errorName: error.name, error: error.message, stack: error.stack }, 'quiz.error'); taskState(task, 'waiting', '课中题处理失败:' + error.message); } }
finally { pendingQuiz = null; }
return;
}
@@ -644,14 +698,14 @@
playStarted = true;
lastAutoPlayAt = now();
try { const promise = video.play(); if (promise && promise.catch) await promise; }
catch (_) { taskState(task, 'waiting', '浏览器拒绝自动播放,请手动播放后继续'); state.video.status = 'waiting'; return; }
catch (error) { log('warn', '播放器启动失败', task, { errorName: error?.name, error: String(error?.message || error || '未知播放错误'), currentTime, rate: video.playbackRate }, 'video.playFailed'); taskState(task, 'waiting', '浏览器拒绝自动播放,请手动播放后继续'); state.video.status = 'waiting'; return; }
if (epoch !== generation || state.status !== 'running') { video.pause(); return; }
lastAdvancedAt = now();
}
if (lastTime === null || Math.abs(currentTime - lastTime) > 0.01) { lastTime = currentTime; lastAdvancedAt = now(); attemptedAt = 0; taskState(task, 'playing', page.hidden ? '页面在后台,按真实播放与平台记录跟踪' : '视频播放中'); return; }
if (!attemptedAt && now() - lastAdvancedAt >= config.stallSeconds * 1000) {
attemptedAt = now(); taskState(task, 'retrying', '播放进度长时间未变化,先尝试续播');
try { await video.play(); } catch (_) { taskState(task, 'waiting', '续播被浏览器拒绝,请人工操作'); }
try { await video.play(); } catch (error) { log('warn', '播放器续播失败', task, { errorName: error?.name, error: String(error?.message || error || '未知播放错误'), currentTime, rate: video.playbackRate }, 'video.resumeFailed'); taskState(task, 'waiting', '续播被浏览器拒绝,请人工操作'); }
return;
}
if (attemptedAt && now() - attemptedAt >= config.recoverySeconds * 1000) {
@@ -686,6 +740,10 @@
if (page.kind === 'paper' && task.enterClickedAt && !task.examPaperURL && page.title === task.title) task.examPaperURL = page.url;
}
const attempts = readExamAttempt(task, snap, page);
if (task.examRequested && task.examNeedsRecheck) {
await observeExamState(task, true);
if (task.status === 'completed') return;
}
if (config.skipSingleAttemptQuiz && attempts.blocked && !task.examRequested) {
state.exam = { taskId: task.id, title: task.title, stage: 'waiting', requiresManualSubmission: true, message: attempts.reason };
taskState(task, 'waiting', attempts.reason); return;
@@ -717,11 +775,12 @@
await storage.set('transition.v1', { sessionId: session.id, taskId: task.id, workerNonce: worker && worker.nonce || '', sourceURL: page.url, targetURL: '', title: task.title, accountId: account(snap, page), expires: now() + 60000 });
buttons[0].click(); return;
}
if (!task.examRequested || task.examPage !== page.kind) {
if (!task.examRequested || task.examNeedsRecheck || task.examPage !== page.kind) {
task.examPage = page.kind; taskState(task, 'answering', '考试主题已确认,交给答题与交卷控制器');
if (typeof options.onExamStart === 'function') {
const accepted = await options.onExamStart({ task: clone(task), preflight: clone(task.preflight), document: snap.document, page: page.kind, config: clone(config) });
task.examRequested = accepted !== false;
if (accepted !== false) task.examNeedsRecheck = false;
if (accepted === false) { if (!task.readyWaitAt) task.readyWaitAt = now(); if (now() - task.readyWaitAt > 15000) taskState(task, 'waiting', '试卷未完整加载,请人工核对后继续'); }
}
else taskState(task, 'waiting', '考试控制器尚未接入,请人工处理');
@@ -811,17 +870,24 @@
if (state.status !== 'running') { await persist(); notify(); return getState(); }
if (task && TERMINAL.includes(task.status)) { activeTaskId = ''; await release(); task = null; }
if (!task) {
const candidates = state.tasks.filter(t => t.status === 'pending' && allows(t.kind) && (!worker || t.id === worker.taskId));
const candidates = state.tasks.filter(t => ['pending', 'waitingLease'].includes(t.status) && allows(t.kind) && (!worker || t.id === worker.taskId));
// 当前页面的实际课件优先;顺序执行先完成可用视频,再处理关联考试。
function rank(t) { return (canonical(t.pageUrl) === page.url ? 0 : 10) + (t.kind === 'video' ? 0 : t.kind === 'exam' ? 1 : 2); }
candidates.sort((a, b) => rank(a) - rank(b));
for (const next of candidates) { if (await activate(next, snap, page)) { task = next; break; } }
const blockedKinds = new Set();
for (const next of candidates) {
const kind = next.kind === 'course' ? 'video' : next.kind; if (blockedKinds.has(kind)) continue;
if (await activate(next, snap, page)) { task = next; break; }
if (next.status === 'waitingLease') { blockedKinds.add(kind); if (config.execution === 'sequential') break; }
}
}
if (task && epoch === generation) {
const held = await heartbeat(task) || await claim(task);
if (!held) state.message = '等待同类任务租约释放,暂不操作当前页面';
else if (task.status !== 'waiting' && task.kind === 'video') await videoTick(task, snap, page, epoch);
else if (task.kind === 'exam') await examTick(task, snap, page);
if (held) {
acquiredLease(task);
if (task.status !== 'waiting' && task.kind === 'video') await videoTick(task, snap, page, epoch);
else if (task.kind === 'exam') await examTick(task, snap, page);
}
}
if (epoch === generation) await parallelTick(snap, page);
if (worker && session) { const request = await storage.get('worker:' + worker.nonce); if (request) { const own = task || state.tasks.find(t => t.id === worker.taskId); request.heartbeatAt = now(); request.status = own && own.status || 'idle'; request.video = state.video; request.exam = state.exam; await storage.set('worker:' + worker.nonce, request); } }
@@ -829,7 +895,7 @@
if (!task && !remaining.length) { state.status = state.tasks.length ? 'finished' : 'waiting'; state.message = state.tasks.length ? '本轮调度结束;请检查完成、跳过及失败清单' : '当前页面没有可识别的任务,请打开学习地图、课程或试卷'; if (session) session.status = state.status; await control(state.status); }
else if (!task && remaining.every(t => t.status === 'waiting')) { state.message = '剩余任务等待人工处理;确认后点击继续'; }
await persist(); notify(); return getState();
} catch (error) { state.message = '学习调度异常:' + error.message; state.status = 'paused'; if (session) session.status = 'paused'; await control('paused'); await release(); log('error', state.message); return getState(); }
} catch (error) { state.message = '学习调度异常:' + error.message; state.status = 'paused'; if (session) session.status = 'paused'; await control('paused'); await release(); log('error', state.message, null, { errorName: error.name, error: error.message, stack: error.stack }, 'learning.error'); return getState(); }
finally { busy = false; }
}
async function observeControl(value) {