fix: deduplicate practice snapshots and compact white panel
This commit is contained in:
suanyuit-cyber
@@ -0,0 +1,193 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
|
||||
// Merge exported local question-bank files; no browser credentials or network access.
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const crypto = require('node:crypto');
|
||||
const core = require('../src/core.js');
|
||||
|
||||
const TITLE = '华谊集团化工总控工（高级）';
|
||||
const OUTPUT_NAME = `${TITLE}-合并题库`;
|
||||
const EXPECTED_BANKS = ['判断题库1', '判断题库2', '单选题库1', '单选题库2', '多选题库'].map(name => `${TITLE}-${name}`);
|
||||
const nameKey = value => String(value || '').replace(/\s+/gu, '').replace(/\(/g, '（').replace(/\)/g, '）');
|
||||
const expectedNames = new Map(EXPECTED_BANKS.map(name => [nameKey(name), name]));
|
||||
const sha256 = value => crypto.createHash('sha256').update(value).digest('hex');
|
||||
|
||||
function safeFilename(value) {
|
||||
const result = Array.from(String(value).replace(/[<>:"/\\|?*\u0000-\u001f]/g, '_').trim().replace(/[. ]+$/, '')).slice(0, 150).join('');
|
||||
if (!result || /^(?:con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/i.test(result)) throw new Error('无法生成安全的题库文件名');
|
||||
return result;
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = { files: [], outDir: path.resolve(__dirname, '..', '题库'), allowPartial: false, force: false, perBank: false, manifestPath: null, help: false };
|
||||
for (let index = 0; index < argv.length; index++) {
|
||||
const value = argv[index];
|
||||
if (value === '--') { options.files.push(...argv.slice(index + 1)); break; }
|
||||
if (value === '--allow-partial') options.allowPartial = true;
|
||||
else if (value === '--force') options.force = true;
|
||||
else if (value === '--per-bank') options.perBank = true;
|
||||
else if (value === '--help' || value === '-h') options.help = true;
|
||||
else if (value === '--out-dir' || value === '--manifest') {
|
||||
const argument = argv[++index];
|
||||
if (!argument || argument.startsWith('--')) throw new Error(`${value} 需要一个路径`);
|
||||
if (value === '--out-dir') options.outDir = path.resolve(argument);
|
||||
else options.manifestPath = path.resolve(argument);
|
||||
} else if (value.startsWith('--')) throw new Error(`不支持的参数：${value}`);
|
||||
else options.files.push(value);
|
||||
}
|
||||
if (!options.help && !options.files.length) throw new Error('请传入至少一个本地 CSV 或 JSON 导出文件；使用 --help 查看用法');
|
||||
return options;
|
||||
}
|
||||
|
||||
function readExpectedCounts(manifestPath) {
|
||||
if (!manifestPath) return new Map();
|
||||
const payload = JSON.parse(fs.readFileSync(manifestPath, 'utf8').replace(/^\uFEFF/, ''));
|
||||
let rows;
|
||||
if (Array.isArray(payload.banks)) rows = payload.banks;
|
||||
else if (payload.expectedCounts && typeof payload.expectedCounts === 'object' && !Array.isArray(payload.expectedCounts)) {
|
||||
rows = Object.entries(payload.expectedCounts).map(([bankName, questionCount]) => ({ bankName, questionCount }));
|
||||
} else throw new Error('核对清单必须包含 banks 数组或 expectedCounts 对象');
|
||||
const result = new Map();
|
||||
for (const row of rows) {
|
||||
const key = nameKey(row?.bankName || row?.name);
|
||||
const count = row?.questionCount ?? row?.expectedCount;
|
||||
if (!expectedNames.has(key) || !Number.isSafeInteger(count) || count < 1) throw new Error('核对清单中存在不支持的题库名称或无效题数');
|
||||
if (result.has(key)) throw new Error(`核对清单重复指定题库：${expectedNames.get(key)}`);
|
||||
result.set(key, count);
|
||||
}
|
||||
if (result.size !== EXPECTED_BANKS.length) throw new Error('核对清单需要列出五个题库各自的页面题数');
|
||||
return result;
|
||||
}
|
||||
|
||||
function readInput(filename) {
|
||||
const absolute = path.resolve(filename);
|
||||
const stat = fs.statSync(absolute);
|
||||
if (!stat.isFile() || stat.size > 100 * 1024 * 1024) throw new Error(`输入不是文件或超过 100 MB：${path.basename(filename)}`);
|
||||
const text = fs.readFileSync(absolute, 'utf8').replace(/^\uFEFF/, '');
|
||||
const extension = path.extname(filename).toLowerCase();
|
||||
let records;
|
||||
if (extension === '.csv') records = core.fromCSV(text);
|
||||
else if (extension === '.json') {
|
||||
const payload = JSON.parse(text);
|
||||
const rows = Array.isArray(payload) ? payload : payload?.schemaVersion === 1 ? payload.records : null;
|
||||
if (!Array.isArray(rows)) throw new Error(`JSON 需要 schemaVersion: 1 和 records 数组：${path.basename(filename)}`);
|
||||
records = rows.map(core.normalizeRecord);
|
||||
} else throw new Error(`只接受 CSV 或 JSON 导出文件：${path.basename(filename)}`);
|
||||
if (!records.length) throw new Error(`导出文件没有题目：${path.basename(filename)}`);
|
||||
return { records, source: { filename: path.basename(filename), sha256: sha256(fs.readFileSync(absolute)), inputRecords: records.length } };
|
||||
}
|
||||
|
||||
function prepare(options) {
|
||||
const expectedCounts = readExpectedCounts(options.manifestPath);
|
||||
const uniquePaths = [...new Set(options.files.map(filename => path.resolve(filename)))];
|
||||
const inputs = uniquePaths.map(readInput);
|
||||
const compacted = core.compactRecords(core.upsert([], inputs.flatMap(input => input.records)));
|
||||
const groups = new Map();
|
||||
for (const record of compacted.records) {
|
||||
const key = nameKey(record.bankName);
|
||||
if (!groups.has(key)) groups.set(key, []);
|
||||
groups.get(key).push(record);
|
||||
}
|
||||
const problems = [];
|
||||
for (const [key, name] of expectedNames) if (!groups.has(key)) problems.push(`缺少题库：${name}`);
|
||||
for (const [key, rows] of groups) {
|
||||
if (!expectedNames.has(key)) problems.push(`出现五库之外的来源：${rows[0].bankName}`);
|
||||
const bankIds = [...new Set(rows.map(record => record.bankId).filter(Boolean))];
|
||||
if (bankIds.length > 1) problems.push(`同名题库包含多个不同题库 ID，未擅自合并：${rows[0].bankName}`);
|
||||
const expectedType = key.includes('判断题库') ? 'judge' : key.includes('单选题库') ? 'single' : key.includes('多选题库') ? 'multiple' : null;
|
||||
if (expectedType && rows.some(record => record.type !== expectedType)) problems.push(`题型与来源名称不一致：${rows[0].bankName}`);
|
||||
if (expectedCounts.has(key) && rows.length !== expectedCounts.get(key)) problems.push(`题数不符：${rows[0].bankName}，页面 ${expectedCounts.get(key)} 题，实际 ${rows.length} 题`);
|
||||
}
|
||||
const missing = compacted.records.filter(record => record.answerStatus === 'missing').length;
|
||||
const conflicts = compacted.records.filter(record => record.answerStatus === 'conflict').length;
|
||||
const invalidAnswers = compacted.records.filter(record => record.answerStatus === 'confirmed' && core.mapAnswers(record, record).status !== 'matched').length;
|
||||
if (missing) problems.push(`${missing} 道题没有已确认标准答案`);
|
||||
if (conflicts) problems.push(`${conflicts} 道题存在答案冲突`);
|
||||
if (invalidAnswers) problems.push(`${invalidAnswers} 道已确认答案无法唯一对应自身选项`);
|
||||
if (problems.length && !options.allowPartial) throw new Error(`合并校验未通过，尚未写入输出：\n- ${problems.join('\n- ')}\n确认需要保留未完成数据时可显式使用 --allow-partial`);
|
||||
|
||||
const orderedBanks = [...groups.keys()].sort((a, b) => {
|
||||
const left = [...expectedNames.keys()].indexOf(a), right = [...expectedNames.keys()].indexOf(b);
|
||||
return (left < 0 ? 100 : left) - (right < 0 ? 100 : right) || a.localeCompare(b, 'zh-CN');
|
||||
});
|
||||
const records = orderedBanks.flatMap(key => [...groups.get(key)].sort((a, b) => {
|
||||
if (Number.isInteger(a.questionIndex) && Number.isInteger(b.questionIndex) && a.questionIndex !== b.questionIndex) return a.questionIndex - b.questionIndex;
|
||||
return a.stem.localeCompare(b.stem, 'zh-CN') || (a.contentOccurrence || 0) - (b.contentOccurrence || 0) || a.key.localeCompare(b.key);
|
||||
}));
|
||||
const now = new Date().toISOString();
|
||||
const summary = {
|
||||
schemaVersion: 1, exportedAt: now, sourceFiles: inputs.map(input => input.source),
|
||||
complete: !problems.length && expectedCounts.size === EXPECTED_BANKS.length,
|
||||
validation: problems.length ? 'partial' : 'passed-data-checks',
|
||||
sourceCountValidation: expectedCounts.size ? 'checked-against-page-manifest' : 'not-provided',
|
||||
totalQuestions: records.length, confirmedAnswers: records.filter(record => record.answerStatus === 'confirmed').length,
|
||||
missingAnswers: missing, conflictingAnswers: conflicts, invalidAnswers,
|
||||
duplicateRowsRemoved: inputs.reduce((count, input) => count + input.records.length, 0) - records.length,
|
||||
banks: orderedBanks.map(key => ({
|
||||
bankName: expectedNames.get(key) || groups.get(key)[0].bankName,
|
||||
questionCount: groups.get(key).length, ...(expectedCounts.has(key) ? { pageQuestionCount: expectedCounts.get(key) } : {}),
|
||||
bankIds: [...new Set(groups.get(key).map(record => record.bankId).filter(Boolean))],
|
||||
confirmedAnswers: groups.get(key).filter(record => record.answerStatus === 'confirmed').length,
|
||||
missingAnswers: groups.get(key).filter(record => record.answerStatus === 'missing').length,
|
||||
conflictingAnswers: groups.get(key).filter(record => record.answerStatus === 'conflict').length
|
||||
})), problems
|
||||
};
|
||||
const files = [
|
||||
{ name: `${OUTPUT_NAME}.csv`, content: core.toCSV(records) },
|
||||
{ name: `${OUTPUT_NAME}.json`, content: JSON.stringify({ schemaVersion: 1, exportedAt: now, bankName: OUTPUT_NAME, records }, null, 2) + '\n' },
|
||||
{ name: '题库清单.json', content: JSON.stringify(summary, null, 2) + '\n' }
|
||||
];
|
||||
if (options.perBank) for (const key of orderedBanks) {
|
||||
const rows = records.filter(record => nameKey(record.bankName) === key);
|
||||
files.push({ name: `${safeFilename(expectedNames.get(key) || rows[0].bankName)}_${rows.length}题.csv`, content: core.toCSV(rows) });
|
||||
}
|
||||
if (new Set(files.map(file => file.name)).size !== files.length) throw new Error('输出文件名发生冲突，停止写入');
|
||||
return { records, summary, files };
|
||||
}
|
||||
|
||||
function writeOutput(options, prepared) {
|
||||
const outputDir = path.resolve(options.outDir);
|
||||
// Check all targets before changing any file. --force applies only to these exact outputs.
|
||||
for (const file of prepared.files) {
|
||||
const target = path.join(outputDir, file.name);
|
||||
if (fs.existsSync(target) && (!fs.statSync(target).isFile() || (!options.force && fs.readFileSync(target, 'utf8') !== file.content))) {
|
||||
throw new Error(`输出已存在且内容不同，未覆盖：${target}；需要覆盖时显式使用 --force`);
|
||||
}
|
||||
}
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
for (const file of prepared.files) {
|
||||
const target = path.join(outputDir, file.name);
|
||||
if (fs.existsSync(target) && fs.readFileSync(target, 'utf8') === file.content) continue;
|
||||
fs.writeFileSync(target, file.content, { encoding: 'utf8', flag: options.force ? 'w' : 'wx' });
|
||||
}
|
||||
return prepared.files.map(file => path.join(outputDir, file.name));
|
||||
}
|
||||
|
||||
function help() {
|
||||
return `用法：node scripts/merge-question-banks.cjs [选项] 文件1.csv 文件2.json ...\n\n` +
|
||||
`--out-dir 路径 输出目录（默认仓库的“题库”文件夹）\n` +
|
||||
`--manifest 文件.json 按页面题数核验五库；格式 {"banks":[{"bankName":"${EXPECTED_BANKS[0]}","questionCount":500},...]}\n` +
|
||||
`--per-bank 同时输出各题库独立 CSV\n` +
|
||||
`--allow-partial 明确允许输出未收齐、缺答案或有冲突的数据，清单标记为 partial\n` +
|
||||
`--force 明确允许覆盖本次指定的已有输出文件\n` +
|
||||
`--help 显示帮助\n\n` +
|
||||
`默认核对五个题库的名称、题型和标准答案。没有页面题数清单时，不声称源数据已完整收齐。\n`;
|
||||
}
|
||||
|
||||
function main(argv = process.argv.slice(2)) {
|
||||
const options = parseArgs(argv);
|
||||
if (options.help) { process.stdout.write(help()); return; }
|
||||
const prepared = prepare(options);
|
||||
const outputFiles = writeOutput(options, prepared);
|
||||
process.stdout.write(JSON.stringify({ ...prepared.summary, outputFiles }, null, 2) + '\n');
|
||||
if (!prepared.summary.complete) process.stderr.write('注意：源数据完整性尚未通过五库页面题数核验，详见题库清单.json。\n');
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
try { main(); }
|
||||
catch (error) { process.stderr.write(`合并失败：${error.message}\n`); process.exitCode = 1; }
|
||||
}
|
||||
|
||||
module.exports = { parseArgs, readExpectedCounts, readInput, prepare, writeOutput, EXPECTED_BANKS, nameKey };
|
||||
Reference in New Issue
Block a user