fix: deduplicate practice snapshots and compact white panel

2026-10-29 18:49:42 +08:00
parent 83ca85cc17
commit 762408a347
14 changed files with 1996 additions and 90 deletions
@@ -45,14 +45,18 @@ const QBCore = (() => {

  function sourceInfo(value) {
    const source = value && typeof value === 'object' && !Array.isArray(value) ? value : {};
-    return { kind: clean(source.kind) || 'unknown', url: clean(source.url), observedAt: clean(source.observedAt) || new Date().toISOString() };
+    const result = { kind: clean(source.kind) || 'unknown', url: clean(source.url), observedAt: clean(source.observedAt) || new Date().toISOString() };
+    if (clean(source.batchId)) result.batchId = idText(source.batchId, '采集批次ID');
+    return result;
  }

  function recordKey(record) {
    const bank = record.bankId || record.bankName;
    if (record.questionId) return `id:${JSON.stringify([bank, record.questionId])}`;
    // Keep the complete signature, rather than a short hash whose collisions could merge answers.
-    return `fp:${JSON.stringify([bank, record.type, normalizedStem(record.stem), sorted(optionContents(record))])}`;
+    const fingerprint = [bank, record.type, normalizedStem(record.stem), sorted(optionContents(record))];
+    if (record.contentOccurrence != null) fingerprint.push(record.contentOccurrence);
+    return `fp:${JSON.stringify(fingerprint)}`;
  }

  function normalizeRecord(input) {
@@ -103,6 +107,13 @@ const QBCore = (() => {
      correctOptionIds: answerStatus === 'missing' ? [] : correctOptionIds, answerTexts, answerStatus,
      explanation: clean(input.explanation), source, updatedAt: clean(input.updatedAt) || source.observedAt, conflicts,
    };
+    for (const field of ['questionIndex', 'contentOccurrence']) {
+      if (input[field] != null) {
+        if (!Number.isSafeInteger(input[field]) || input[field] < 0) throw new Error(`${field} 必须为非负整数`);
+        record[field] = input[field];
+      }
+    }
+    if (own(input, 'questionIdAliases')) record.questionIdAliases = unique([...stringArray(input.questionIdAliases, '题目ID别名'), record.questionId].filter(Boolean));
    if (input.answerSource || answerStatus === 'confirmed') record.answerSource = sourceInfo(input.answerSource || source);
    record.key = recordKey(record);
    return record;
@@ -132,7 +143,14 @@ const QBCore = (() => {
    if (oldRecord.key !== newRecord.key) throw new Error('不能合并不同题目的记录');
    const contentMatches = sameContent(oldRecord, newRecord);
    const explanation = newRecord.explanation || oldRecord.explanation;
-    const base = { ...newRecord, explanation };
+    const metadata = {};
+    for (const field of ['questionIndex', 'contentOccurrence']) {
+      if (newRecord[field] != null || oldRecord[field] != null) metadata[field] = newRecord[field] ?? oldRecord[field];
+    }
+    if (own(oldRecord, 'questionIdAliases') || own(newRecord, 'questionIdAliases')) metadata.questionIdAliases = unique([
+      ...(oldRecord.questionIdAliases || []), ...(newRecord.questionIdAliases || []), oldRecord.questionId, newRecord.questionId
+    ].filter(Boolean));
+    const base = { ...newRecord, ...metadata, explanation };
    const conflicts = [...oldRecord.conflicts, ...newRecord.conflicts];
    const conflictAlready = oldRecord.answerStatus === 'conflict' || newRecord.answerStatus === 'conflict';
    const answersDiffer = oldRecord.answerStatus === 'confirmed' && newRecord.answerStatus === 'confirmed' && answerSignature(oldRecord) !== answerSignature(newRecord);
@@ -148,7 +166,7 @@ const QBCore = (() => {
    }
    if (newRecord.answerStatus === 'missing' && oldRecord.answerStatus === 'confirmed') {
      // The incoming DOM may use different IDs/order. Keep the original answer's own options and IDs together.
-      return normalizeRecord({ ...oldRecord, bankName: newRecord.bankName || oldRecord.bankName, explanation,
+      return normalizeRecord({ ...oldRecord, ...metadata, bankName: newRecord.bankName || oldRecord.bankName, explanation,
        updatedAt: newRecord.updatedAt, source: newRecord.source, answerSource: oldRecord.answerSource || oldRecord.source });
    }
    return normalizeRecord({ ...base, conflicts });
@@ -163,6 +181,25 @@ const QBCore = (() => {
      records.set(record.key, records.has(record.key) ? mergeRecord(records.get(record.key), record) : record);
      lastSeen.set(record.key, index);
    }
+    // Explicit aliases are produced only by content/batch repair. Ordinary
+    // records with different IDs are still kept separate until that repair.
+    const bankIdentity = bankResolver([...records.values()]);
+    for (const target of [...records.values()].filter(record => record.questionIdAliases?.length > 1)) {
+      if (!records.has(target.key)) continue;
+      for (const other of [...records.values()]) {
+        if (other.key === target.key || !records.has(other.key) || !other.questionId) continue;
+        const current = records.get(target.key);
+        const aliases = new Set(current.questionIdAliases || []);
+        if (!aliases.has(other.questionId) || bankIdentity(current) !== bankIdentity(other) || !sameContent(current, other) || current.type !== other.type) continue;
+        if (current.contentOccurrence != null && other.contentOccurrence != null && current.contentOccurrence !== other.contentOccurrence) continue;
+        records.set(target.key, foldEquivalent([current, other], { questionId: current.questionId, bankId: current.bankId, bankName: current.bankName, contentOccurrence: current.contentOccurrence }));
+        records.delete(other.key);
+      }
+    }
+    for (const record of [...records.values()]) {
+      if (record.questionId || record.contentOccurrence != null || record.answerStatus !== 'missing') continue;
+      if ([...records.values()].some(other => other.questionIdAliases?.length > 1 && bankIdentity(other) === bankIdentity(record) && other.type === record.type && sameContent(other, record))) records.delete(record.key);
+    }
    // The saved DOM has no stable question ID. Upgrade it only when the complete
    // content and bank identify one unique server record in both directions.
    const anonymous = [...records.values()].filter((record) => !record.questionId);
@@ -172,7 +209,8 @@ const QBCore = (() => {
      if (!a.bankName || !b.bankName || a.bankName === '未命名题库' || b.bankName === '未命名题库') return false;
      return normalized(a.bankName) === normalized(b.bankName);
    };
-    const possiblePair = (a, b) => a.options.length > 0 && b.options.length > 0 && a.type === b.type && sameBank(a, b) && sameContent(a, b);
+    const possiblePair = (a, b) => a.options.length > 0 && b.options.length > 0 && a.type === b.type && sameBank(a, b) && sameContent(a, b) &&
+      (a.contentOccurrence == null || b.contentOccurrence == null || a.contentOccurrence === b.contentOccurrence);
    const candidates = new Map(anonymous.map((record) => [record.key, identified.filter((other) => possiblePair(record, other))]));
    for (const record of anonymous) {
      const matches = candidates.get(record.key);
@@ -188,6 +226,125 @@ const QBCore = (() => {
    return [...records.values()];
  }

+  function contentSignature(record) {
+    return JSON.stringify([record.type, normalizedStem(record.stem), sorted(optionContents(record))]);
+  }
+
+  function bankResolver(records) {
+    const namedIds = new Map();
+    for (const record of records) if (record.bankId && record.bankName !== '未命名题库') {
+      const name = normalized(record.bankName);
+      if (!namedIds.has(name)) namedIds.set(name, new Set());
+      namedIds.get(name).add(record.bankId);
+    }
+    return record => {
+      if (record.bankId) return `id:${record.bankId}`;
+      const name = normalized(record.bankName);
+      if (!name || record.bankName === '未命名题库') return `unresolved:${record.key}`;
+      const ids = namedIds.get(name);
+      return ids?.size === 1 ? `id:${[...ids][0]}` : `name:${name}`;
+    };
+  }
+
+  function foldEquivalent(group, identity = {}) {
+    const existing = group.find(record => record.questionIdAliases?.length > 1);
+    const first = existing || group.find(record => record.questionId) || group[0];
+    const questionId = identity.questionId ?? first.questionId;
+    const bankId = identity.bankId ?? (group.find(record => record.bankId)?.bankId || first.bankId);
+    const bankName = identity.bankName || first.bankName;
+    const ids = unique(group.flatMap(record => [record.questionId, ...(record.questionIdAliases || [])]).filter(Boolean));
+    const meta = { bankId, bankName, questionId };
+    if (ids.length > 1 || group.some(record => own(record, 'questionIdAliases'))) meta.questionIdAliases = ids;
+    const occurrence = identity.contentOccurrence ?? first.contentOccurrence ?? group.find(record => record.contentOccurrence != null)?.contentOccurrence;
+    if (occurrence != null) meta.contentOccurrence = occurrence;
+    // Process missing observations first so the selected options and the answer
+    // IDs always come together from an answer-bearing record.
+    const ordered = group.filter(record => record.answerStatus === 'missing').concat(group.filter(record => record.answerStatus !== 'missing'));
+    let result;
+    for (const record of ordered) {
+      const rebased = normalizeRecord({ ...record, ...meta });
+      result = result ? mergeRecord(result, rebased) : rebased;
+    }
+    return result;
+  }
+
+  function compactRecords(input) {
+    if (!Array.isArray(input)) throw new Error('题库必须是数组');
+    const entries = input.map((value, index) => ({ record: normalizeRecord(value), index, batch: '', occurrence: null }));
+    const bankIdentity = bankResolver(entries.map(entry => entry.record));
+    const listSource = record => [record.answerSource, record.source].find(source => source?.kind === 'network' &&
+      /\/(?:upm\/(?:wrong\/)?start|pra\/[^/]+\/questions)\/?$/u.test(source.url || '') && Number.isFinite(Date.parse(source.observedAt)));
+    const legacySources = new Map();
+    for (const entry of entries) {
+      const record = entry.record;
+      if (record.contentOccurrence != null) entry.occurrence = record.contentOccurrence;
+      const explicit = record.source.batchId || record.answerSource?.batchId;
+      if (explicit) entry.batch = `${bankIdentity(record)}:${explicit}`;
+      else if (entry.occurrence == null) {
+        const source = listSource(record);
+        if (!source) continue;
+        const key = JSON.stringify([bankIdentity(record), source.url]);
+        if (!legacySources.has(key)) legacySources.set(key, []);
+        legacySources.get(key).push({ entry, time: Date.parse(source.observedAt) });
+      }
+    }
+    // Older versions timestamped individual rows inside each list response.
+    // Infer only network list-response clusters; DOM/answer observations are
+    // never assumed to be full snapshots. Their original order gives duplicate
+    // occurrence numbers and keeps legitimate repeated questions in each list.
+    for (const [key, items] of legacySources) {
+      items.sort((a, b) => a.time - b.time || a.entry.index - b.entry.index);
+      let previous = -Infinity, batch = '';
+      for (const item of items) {
+        if (item.time - previous > 1000) batch = `legacy:${key}:${item.time}`;
+        item.entry.batch = batch; previous = item.time;
+      }
+    }
+    const counters = new Map();
+    for (const entry of entries) {
+      if (!entry.batch || entry.occurrence != null) continue;
+      const key = JSON.stringify([entry.batch, contentSignature(entry.record)]);
+      entry.occurrence = counters.get(key) || 0;
+      counters.set(key, entry.occurrence + 1);
+    }
+    const groups = new Map();
+    const byContent = new Map();
+    const add = (key, entry) => {
+      if (!groups.has(key)) groups.set(key, []);
+      groups.get(key).push(entry);
+    };
+    for (const entry of entries) {
+      const content = JSON.stringify([bankIdentity(entry.record), contentSignature(entry.record)]);
+      entry.content = content;
+      if (entry.occurrence == null || entry.record.options.length < 2 || entry.record.type === 'unknown') continue;
+      const key = JSON.stringify([content, entry.occurrence]);
+      add(key, entry);
+      if (!byContent.has(content)) byContent.set(content, new Set());
+      byContent.get(content).add(key);
+    }
+    for (const entry of entries) {
+      if (entry.occurrence != null && entry.record.options.length >= 2 && entry.record.type !== 'unknown') continue;
+      const matches = [...(byContent.get(entry.content) || [])];
+      let key;
+      const matchingAlias = matches.filter(candidate => groups.get(candidate).some(item => item.record.questionId === entry.record.questionId || item.record.questionIdAliases?.includes(entry.record.questionId)));
+      if (entry.record.questionId && matchingAlias.length === 1) key = matchingAlias[0];
+      else if (matches.length === 1) key = matches[0];
+      else if (matches.length > 1 && !entry.record.questionId && entry.record.answerStatus === 'missing') continue;
+      else key = `unresolved:${entry.record.key}`;
+      add(key, entry);
+    }
+    const records = [];
+    let duplicateGroups = 0;
+    for (const group of groups.values()) {
+      if (group.length > 1) duplicateGroups++;
+      const identity = {};
+      const occurrence = group.find(entry => entry.occurrence != null)?.occurrence;
+      if (occurrence != null) identity.contentOccurrence = occurrence;
+      records.push(foldEquivalent(group.map(entry => entry.record), identity));
+    }
+    return { records, removed: input.length - records.length, groups: duplicateGroups, conflictCount: records.filter(record => record.answerStatus === 'conflict').length };
+  }
+
  function bigrams(value) {
    const chars = Array.from(value);
    if (chars.length < 2) return new Set(chars);
@@ -393,7 +550,7 @@ const QBCore = (() => {
    return records;
  }

-  return { normalizeRecord, mergeRecord, upsert, search, mapAnswers, toCSV, fromCSV };
+  return { normalizeRecord, mergeRecord, upsert, compactRecords, search, mapAnswers, toCSV, fromCSV };
 })();

 if (typeof module !== 'undefined' && module.exports) module.exports = QBCore;