From 7dc85baea9f0431a1abce50a5a17dafadeb98775 Mon Sep 17 00:00:00 2001 From: Digital Drosophila Archive Date: Thu, 1 Oct 2026 03:21:48 +0800 Subject: [PATCH] Add licensed public-resource archive plan and verified LFS tooling --- .gitattributes | 2 + NOTICE.md | 8 + registry/publication-20261001/README.md | 55 ++ .../lfs-smoke-receipt.json | 21 + registry/publication-20261001/plan.json | 60 ++ .../source-guard-before.json | 123 ++++ tools/lfs-publication/capture_source_guard.py | 85 +++ tools/lfs-publication/publish_archives.py | 534 ++++++++++++++++++ .../lfs-publication/test_publish_archives.py | 245 ++++++++ 9 files changed, 1133 insertions(+) create mode 100644 registry/publication-20261001/README.md create mode 100644 registry/publication-20261001/lfs-smoke-receipt.json create mode 100644 registry/publication-20261001/plan.json create mode 100644 registry/publication-20261001/source-guard-before.json create mode 100644 tools/lfs-publication/capture_source_guard.py create mode 100644 tools/lfs-publication/publish_archives.py create mode 100644 tools/lfs-publication/test_publish_archives.py diff --git a/.gitattributes b/.gitattributes index 9d3eeb5..9a31a11 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,3 +3,5 @@ *.jsonl.gz binary artifacts/**/*.tar.zst filter=lfs diff=lfs merge=lfs -text artifacts/lfs-smoke/*.bin filter=lfs diff=lfs merge=lfs -text +artifacts/public-resources-20261001/**/payload-*.bin filter=lfs diff=lfs merge=lfs -text +artifacts/public-resources-20261001/**/*.jsonl.gz filter=lfs diff=lfs merge=lfs -text diff --git a/NOTICE.md b/NOTICE.md index 2da1a53..ca4b7bd 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -2,6 +2,14 @@ 本仓库整理既有项目，不改变原始数据和软件的权属。尚未为本项目原创代码选定统一公开许可证；不得将“可访问此 Git 仓库”等同于取得任意再分发授权。 +## 2026-10-01 公开资源发布补充 + +用户已确认保留公开仓库，仅上传明确允许公开再分发的资源。[MaleCNS 官方下载页](https://male-cns.janelia.org/download/)已核实指向 [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/)。据此，独立发布分支现在归档其已下载的数据及明确标注变更来源的既有派生数据；Cell Type Explorer、visualpathways、DoOR.data 和 Shiu 作者模型源码按各自原许可处理。作者署名、固定 commit、许可证据 SHA、排除项和恢复方法见 [本次发布说明](registry/publication-20261001/README.md)。FlyVis 不明确许可的资源、活动实验、环境和凭据不纳入。 + +下表保留初始化时的历史审查状态，不代表本次已核实许可仍然未知。新发布只在 `artifacts/` 增加只读快照，不覆盖原始数据或此前冻结索引；不为整个仓库补授统一许可证。 + +## 初始化审查记录（2026-09-29） + | 外部资源 | 冻结副本中的许可证据 | 本仓库处理 | |---|---|---| | MaleCNS v1.0 官方数据 | 来源文件、原下载脚本和校验清单；统一再分发许可本次未确证 | 只登记路径、来源、大小、SHA-256；不上传数据 | diff --git a/registry/publication-20261001/README.md b/registry/publication-20261001/README.md new file mode 100644 index 0000000..3977248 --- /dev/null +++ b/registry/publication-20261001/README.md @@ -0,0 +1,55 @@ +# 2026-10-01 公开 LFS 资源发布 + +这是现有下载资源的归档发布，不是新的科学实验，不表示 Phase 4AR 已完成或学习通过。用户明确选择保留公开仓库，仅上传确认可公开再分发的资源。活动项目工作树和 Phase 1–4AQ 冻结文件不参与写入。 + +## 范围与许可 + +| 资源 | 许可/依据 | 本次处理 | +|---|---|---| +| MaleCNS v1.0 annotations、neurotransmitters、connectome、SWC、synaptic partners | [官方数据页](https://male-cns.janelia.org/download/)明确指向 [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/) | 逐文件原字节归档、署名、来源与 SHA-256 | +| 原有 Phase 3 / 3B processed 数据 | 基于上述 CC BY 4.0 数据的本项目派生物 | 保留原有过滤/重编号/CSR 转换说明；不重新计算、不更改科学含义 | +| Cell Type Explorer | 固定 commit 的 LICENSE，CC BY 4.0 | 原字节源码/数据快照，保留 LICENSE | +| visualpathways | 固定 commit 的 LICENSE 为 GPL v3 文本 | 原字节源码及 companion resources，保留 notices；不替上游判断 only/or-later | +| DoOR.data 2.0.1 | 固定 commit 的 DESCRIPTION：CC BY-SA 4.0 | 保留作者、原数据来源、ShareAlike 条件 | +| Shiu Drosophila_brain_model | 固定 commit 的 MIT LICENSE | 已下载作者模型源码与环境描述，不称预训练权重 | + +每组精确来源、commit、许可证据 SHA 和修改说明见 `plan.json` 及资源 `catalog.json`；许可证原文随对应上游副本保留。MaleCNS 原数据署名：FlyEM/HHMI Janelia、University of Cambridge Department of Zoology、MRC Laboratory of Molecular Biology、Google Research 及原论文作者。归档封装不意味着这些机构认可本项目模型或结论。 + +DoOR 作者为 Daniel Münch、C. Giovanni Galizia、Shouwen Ma、Martin Strauch、Anja Nissler；[DoOR 2.0 论文](https://doi.org/10.1038/srep21841)及[原始 DoOR 论文](https://doi.org/10.1093/chemse/bjq042)只引用，不另外打包出版商全文。各第三方资源维持原许可，本项目没有给整个仓库强行统一换证。 + +## 明确不上传 + +- FlyVis：冻结 checkout 仅见 MIT classifier，缺独立 LICENSE；数据和预训练权重的再分发权未确认。实际盘点未发现独立下载的预训练 checkpoint/ZIP；不得把源码、几何坐标或功能通道桥接称为下载权重。 +- `datasets/male-cns-v1.0/backups/` 的既有 Phase 1/2 备份包：保留原地；本次已直接归档其 raw 数据，不再重复上传整份备份。 +- Phase 4AD 重复的 Shiu 作者源码：本次仅归档 Phase 4AC 副本；既有目录及历史索引保留。 +- 历史本项目 learned delta、当前 Phase 4AR 正在写入的结果/账本、虚拟环境、缓存、Git 内部目录、凭据、下载日志。 +- ComfyUI 及其他项目、原始 EM 图像、未下载的 syn-points、许可不明的出版商论文全文。 +- visualpathways 的 `pdfversion/` 及 `docs/column_coord.pdf`：已识别为作者图集/配套说明而非一般论文集合，但本次保守等待输出 PDF 的明确许可范围，不把生成程序的 GPL 自动视作所有输出的许可；源码和其余已明确仓库许可的资源仍保留。 + +未列入 plan 的目录默认不发布。额外路径剔除及环境/缓存规则的实际命中保存在各组 `exclusions.jsonl.gz`；文件内容未被改写或“脱敏后替代原件”。 + +## 存储及校验 + +归档位于 `artifacts/public-resources-20261001//`。每组为独立 tar+zstd，切成最多 64 MiB 的有序 LFS 分块，方便大文件传输中断后重试未完成对象。`members.jsonl.gz` 对每个原始相对路径记录大小和 SHA-256；`catalog.json` 记录每块及拼接归档 SHA、源总数、源字节数、来源和许可。分块改变的是存储容器，不是内部文件内容。 + +`pack-summary.json` 存在且 complete=true 仅表示本地打包完成，不等于上传/远程恢复成功。上传后使用独立 clone 和新的 LFS object store 实际下载，再逐块、逐归档、逐内部文件流式校验。对应 receipt 只有在真实校验完成后才会提交。 + +本次不是服务器所有内容的完整灾备。Gitea LFS 与 Git 仓库数据目录均需另有服务端备份；删除服务端 LFS 对象会破坏恢复能力。尚未核验 Gitea 主机磁盘容量和备份策略。 + +## 安全恢复（Linux 示例） + +先安装 Git LFS、Python 3 和 zstd。只在新目录恢复，不要覆盖运行中的 `/data/digital-drosophila`。 + +```sh +GIT_LFS_SKIP_SMUDGE=1 git clone --branch codex/lfs-public-resources ssh://git@192.168.10.31:2222/qishizhe/digital-drosophila.git digital-drosophila-restore +cd digital-drosophila-restore +git lfs install --local +git lfs pull --include='artifacts/public-resources-20261001/**' +python3 -B tools/lfs-publication/publish_archives.py verify --group-dir artifacts/public-resources-20261001/malecns-swc +``` + +对每组运行 verify 并全部通过后，按 catalog 顺序拼接 `payload-000000.bin` 等文件，可用 `zstd -d` 和 tar 在**新建的空恢复目录**解包。归档路径从 `datasets/` 或 `experiments/` 起，无绝对路径。不要运行归档中模型/下载脚本来“验证”，本轮只验证内容。正式使用前仍需检查对应科学阶段 manifest；旧 all-body Phase 3 不能替代已批准的 neuron-only SNN 输入。 + +## 限制 + +快照通过源文件前后 inode/size/mtime/ctime 和枚举一致性检查，不是文件系统原子快照。秘密扫描是有限的 UTF-8 小文本规则，不是对所有二进制、内嵌归档及第三方资源的安全担保。工具内容校验不独立证明 catalog 的真实性；应固定仓库 commit 和 catalog SHA。源码只读校验不能被表述为重新执行全历史科学 gate。 diff --git a/registry/publication-20261001/lfs-smoke-receipt.json b/registry/publication-20261001/lfs-smoke-receipt.json new file mode 100644 index 0000000..36ad58b --- /dev/null +++ b/registry/publication-20261001/lfs-smoke-receipt.json @@ -0,0 +1,21 @@ +{ + "schema": "lfs-smoke-roundtrip-v1", + "fixture": "artifacts/lfs-smoke/publication-probe.bin", + "bytes": 263, + "sha256": "93e0e6c4ca4fdc123c21b859ba97d8bd73670eb9c90d04ce9a431d822b5393c5", + "uploaded_commit": "be41af186c3e1831dbda0dd8d5f151bf3901552f", + "branch": "codex/lfs-public-resources", + "upload_exit_code": 0, + "fresh_clone_independent_lfs_cache": true, + "download_exit_code": 0, + "download_sha256_match": true, + "lfs_fsck_dry_run_exit_code": 0, + "lfs_client_version": "3.8.0", + "client_archive_url": "https://github.com/git-lfs/git-lfs/releases/download/v3.8.0/git-lfs-linux-amd64-v3.8.0.tar.gz", + "client_archive_sha256": "e455e00f15d9b95661b8d53498ffb0c3367962cf1ec73c31ab7369516cd6ab8d", + "client_verification": "SHA-256 matched official GitHub release asset API digest; no PGP verification claimed", + "server_version_observed": "1.26.2", + "repository_private_observed": false, + "secret_headers_saved": false, + "scope": "Synthetic 263-byte fixture only; not evidence that scientific payloads have been uploaded" +} diff --git a/registry/publication-20261001/plan.json b/registry/publication-20261001/plan.json new file mode 100644 index 0000000..01d3567 --- /dev/null +++ b/registry/publication-20261001/plan.json @@ -0,0 +1,60 @@ +{ + "schema": "lfs-publication-plan-v1", + "source_root": "/data/digital-drosophila", + "groups": [ + { + "id": "malecns-raw-core", + "paths": ["datasets/male-cns-v1.0/raw/annotations", "datasets/male-cns-v1.0/raw/connectivity", "datasets/male-cns-v1.0/raw/_manifests"], + "license": {"data": "CC-BY-4.0", "url": "https://creativecommons.org/licenses/by/4.0/", "local_manifests_scripts": "Project-authored provenance; no new blanket software license"}, + "provenance": {"version": "male-cns:v1.0", "source_page": "https://male-cns.janelia.org/download/", "source_prefix": "gs://flyem-male-cns/v1.0/connectome-data/flat-connectome/", "attribution": "Male CNS collaboration: FlyEM (HHMI Janelia), University of Cambridge Department of Zoology, MRC Laboratory of Molecular Biology, Google Research; retain original dataset author credits", "changes": "Data file bytes unchanged; archival compression and path manifest only. Download logs excluded."} + }, + { + "id": "malecns-swc", + "paths": ["datasets/male-cns-v1.0/raw/skeletons"], + "license": {"data": "CC-BY-4.0", "url": "https://creativecommons.org/licenses/by/4.0/"}, + "provenance": {"version": "male-cns:v1.0", "source_page": "https://male-cns.janelia.org/download/", "source_prefix": "gs://flyem-male-cns/v1.0/segmentation/skeletons-malecns/skeletons-swc/", "attribution": "Male CNS collaboration: FlyEM (HHMI Janelia), University of Cambridge Department of Zoology, MRC Laboratory of Molecular Biology, Google Research", "changes": "Unchanged downloaded SWC bytes; not restricted to the later neuron-only canonical set. No EM images."} + }, + { + "id": "malecns-processed", + "paths": ["datasets/male-cns-v1.0/processed"], + "license": {"derived_data": "CC-BY-4.0", "url": "https://creativecommons.org/licenses/by/4.0/", "local_scripts_reports": "Project-authored processing and provenance; no new blanket software license"}, + "provenance": {"dataset": "male-cns:v1.0", "source_page": "https://male-cns.janelia.org/download/", "attribution": "Original Male CNS collaboration; Digital Drosophila project for local processing", "changes": "Existing Phase 3 all-body and Phase 3B neuron-only tables, index mappings, CSR and audit reports, copied byte-for-byte. These were previously derived by filtering/reindexing official data, not new preprocessing. Phase 3 historical outputs are not the approved SNN input; use neuron-only for current runtime.", "canonical_neurons": 166700, "canonical_edges": 25582938, "canonical_synaptic_weight_sum": 124177617} + }, + { + "id": "malecns-synaptic-partners", + "paths": ["experiments/phase4aa-apl-pn-scope/external-resources"], + "license": {"data": "CC-BY-4.0", "url": "https://creativecommons.org/licenses/by/4.0/", "local_provenance": "Project-authored"}, + "provenance": {"source_page": "https://male-cns.janelia.org/download/", "source_url": "https://storage.googleapis.com/flyem-male-cns/v1.0/connectome-data/flat-connectome/syn-partners-male-cns-v1.0-minconf-0.5.feather?generation=1780494942562468", "registered_file_sha256": "959d8ef4173b35382a3e6acfaf5167c795b6d10b877572d146af04e1b487bc07", "attribution": "Male CNS collaboration: FlyEM (HHMI Janelia), University of Cambridge Department of Zoology, MRC Laboratory of Molecular Biology, Google Research", "changes": "Unchanged downloaded paired-synapse file and provenance; syn-points and EM were not downloaded."} + }, + { + "id": "celltype-explorer", + "paths": ["experiments/phase4f-retinotopic-vision/external-resources/celltype-explorer"], + "license": {"upstream": "CC-BY-4.0", "url": "https://creativecommons.org/licenses/by/4.0/", "evidence_file": "LICENSE", "evidence_sha256": "9ba9550ad48438d0836ddab3da480b3b69ffa0aac7b7878b5a0039e7ab429411"}, + "provenance": {"repository": "https://github.com/reiserlab/celltype-explorer-drosophila-male-cns", "commit": "789cc6c105798ce2fd70ba85dab394f90899616b", "attribution": "Reiser Lab and Male CNS Cell Type Explorer contributors; retain upstream notices", "changes": "Unchanged pinned working-tree file bytes; .git, environments, caches and credentials excluded. A source snapshot, not a full Git clone."} + }, + { + "id": "visualpathways", + "paths": ["experiments/phase4f-retinotopic-vision/external-resources/visualpathways"], + "license": {"upstream": "GPL-3.0 as supplied in upstream LICENSE; no assertion of only/or-later option", "evidence_file": "LICENSE", "evidence_sha256": "3972dc9744f6499f0f9b2dbf76696f2ae7ad8af9b23dde66d6af86c9dfb36986", "third_party_notices": "Retained; no relicensing of third-party components"}, + "provenance": {"repository": "https://github.com/reiserlab/visualpathways", "commit": "23f6ac131529b5f56894c6eeb9b88b17894fc00d", "attribution": "Reiser Lab and visualpathways contributors; original copyright notices and source retained", "changes": "Unchanged source/resource file bytes from pinned checkout; .git, credentials, environments and caches omitted; explicit exclusions recorded. No software execution or model alteration."} + }, + { + "id": "door-data", + "paths": ["experiments/phase4l-olfactory-grounding/external-resources/DoOR.data"], + "license": {"upstream": "CC-BY-SA-4.0", "url": "https://creativecommons.org/licenses/by-sa/4.0/", "evidence_file": "DESCRIPTION", "evidence_sha256": "27db2a92eb651d67969de94959990d51668a5857a33c4750a3b6ef49c5880504"}, + "provenance": {"repository": "https://github.com/ropensci/DoOR.data", "commit": "eea72d64b42a4e91dafe822cf0b5204de33f8d34", "version": "2.0.1", "attribution": "Daniel Münch, C. Giovanni Galizia, Shouwen Ma, Martin Strauch, Anja Nissler; see upstream DESCRIPTION and data-source references", "papers": ["https://doi.org/10.1038/srep21841", "https://doi.org/10.1093/chemse/bjq042"], "changes": "Unchanged pinned data package; .git omitted. ShareAlike terms remain attached to this package, not asserted over unrelated repository content."} + }, + { + "id": "shiu-author-model-source", + "paths": ["experiments/phase4ac-lif-semantics/external-resources"], + "license": {"upstream": "MIT", "evidence_file": "LICENSE", "evidence_sha256": "3621f6d6476189190e2960fa43f11b275ba4eca848fdac50a1ba2925de6223e8"}, + "provenance": {"repository": "https://github.com/philshiu/Drosophila_brain_model", "commit": "91bdd1e7dcf193f3e7ca5a8933497fcef63b7960", "attribution": "Philip Shiu and Drosophila_brain_model contributors; original MIT copyright and permission notice retained", "paper": "https://www.nature.com/articles/s41586-024-07763-9", "changes": "The previously downloaded model.py, environment_full.yml, LICENSE and local provenance only. This is model source, not pretrained weights, and not a strict original-model reproduction claim."} + } + ], + "exclude_paths": [ + "datasets/male-cns-v1.0/processed/_staging", + "experiments/phase4f-retinotopic-vision/external-resources/visualpathways/.env_sample", + "experiments/phase4f-retinotopic-vision/external-resources/visualpathways/pdfversion", + "experiments/phase4f-retinotopic-vision/external-resources/visualpathways/docs/column_coord.pdf" + ] +} diff --git a/registry/publication-20261001/source-guard-before.json b/registry/publication-20261001/source-guard-before.json new file mode 100644 index 0000000..e9833ed --- /dev/null +++ b/registry/publication-20261001/source-guard-before.json @@ -0,0 +1,123 @@ +{ + "disk_free_bytes": { + "/data/digital-drosophila": 645731733504, + "/tmp/malecns-lfs-publication-20261001-bI1u07/repo/registry/publication-20261001": 905806151680 + }, + "license_evidence": [ + { + "bytes": 18657, + "group": "celltype-explorer", + "path": "/data/digital-drosophila/experiments/phase4f-retinotopic-vision/external-resources/celltype-explorer/LICENSE", + "sha256": "9ba9550ad48438d0836ddab3da480b3b69ffa0aac7b7878b5a0039e7ab429411" + }, + { + "bytes": 35149, + "group": "visualpathways", + "path": "/data/digital-drosophila/experiments/phase4f-retinotopic-vision/external-resources/visualpathways/LICENSE", + "sha256": "3972dc9744f6499f0f9b2dbf76696f2ae7ad8af9b23dde66d6af86c9dfb36986" + }, + { + "bytes": 1042, + "group": "door-data", + "path": "/data/digital-drosophila/experiments/phase4l-olfactory-grounding/external-resources/DoOR.data/DESCRIPTION", + "sha256": "27db2a92eb651d67969de94959990d51668a5857a33c4750a3b6ef49c5880504" + }, + { + "bytes": 1085, + "group": "shiu-author-model-source", + "path": "/data/digital-drosophila/experiments/phase4ac-lif-semantics/external-resources/LICENSE", + "sha256": "3621f6d6476189190e2960fa43f11b275ba4eca848fdac50a1ba2925de6223e8" + } + ], + "observed_utc": "2026-09-30T19:12:34.846175+00:00", + "original_checkout_branch": "codex/lif-author-semantics-lab", + "original_checkout_head": "09187e4d4d67badf1b418c5f211258ca5d8fb328", + "original_checkout_status": "", + "pass_all": true, + "phase4ar_registered_sources": [ + { + "bytes": 5374, + "path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/PROTOCOL.md", + "sha256": "d61d2cb6a6bff31b5762edb78bb6406ac7d53b1e5eff30aad26eb0eea99fc313" + }, + { + "bytes": 29087, + "path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/append_ledger.py", + "sha256": "1f981f61cd26950e7d4b5393be70b82753ea7e721c0bb75d05f8fd1e92a453fb" + }, + { + "bytes": 7813, + "path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/normal_inputs.py", + "sha256": "df8f346b8fba54bb54f0b1a3f81c1511d54987684064094d0dfa46043be03e0d" + }, + { + "bytes": 3317, + "path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/normal_io.py", + "sha256": "fe6eee89ae98392a28120b69a966d7967916be94376cb7494ef3ce77e63506e0" + }, + { + "bytes": 37143, + "path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/normal_metrics.py", + "sha256": "82af63d8fadd5a6b67083f5e303ebe88674b76f9a2a8a8fff082b5c828483533" + }, + { + "bytes": 11254, + "path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/normal_plan.py", + "sha256": "d1a6b840a547cb10b00df40f4d8b287867d09f355c9777e5f7a8b9489f5555ea" + }, + { + "bytes": 15006, + "path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/normal_prepare.py", + "sha256": "9c8568a3d8c1049ada8737b71192952f72cdd27ccb3b470195e30d6a4b1cea68" + }, + { + "bytes": 24699, + "path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/normal_runner.py", + "sha256": "41a0c71508d0b39c9488bddeaa4ac2a669312c632f16d5f9b5de24dff2eb14c1" + }, + { + "bytes": 1534, + "path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/run_cpu_tests.py", + "sha256": "10ed189053731e1661284cf890ca3504a272af05b4531bdb3655b6e88cab6ff9" + }, + { + "bytes": 3043, + "path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/run_normal.py", + "sha256": "707b6be41caef7af83aa68ea1f102f10c9a4ba81a22c564e54eddf63857e81f3" + }, + { + "bytes": 15199, + "path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/test_append_ledger.py", + "sha256": "0f638c7b83117410d7fa074ec9f2ef865d9810171b73991c73a78414be581bc6" + }, + { + "bytes": 4292, + "path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/test_normal_inputs.py", + "sha256": "7aea6a3bcbbf78c3b3d671b33b83cd4d992b2a8ae9c888e29cdf13f9fa420a45" + }, + { + "bytes": 17862, + "path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/test_normal_metrics.py", + "sha256": "204586b30edbe57bb294c1d8f8ad309863e5bcb3baad32118955ac3d4adcdefb" + }, + { + "bytes": 9184, + "path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/test_normal_plan.py", + "sha256": "790bca529c58e5214c19a6dcaf07511fca11c07fb9b05ca5d71d3f06160bbef4" + }, + { + "bytes": 25798, + "path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/test_normal_runner.py", + "sha256": "ca9f69c731e56c50534d864d0ce780f286b5ab2d786a1a85fadb22dc09c0ccf5" + }, + { + "bytes": 2076, + "path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/test_run_normal.py", + "sha256": "2ee3ad3c58c0dfd83aaaa4c4c9bd01e84eb245c61048c105ed8f3031db74cb40" + } + ], + "registered_scientific_commit": "80aeb2093db655329774eff7b32fb6e417b8b40b", + "registration_sha256": "b63e6281b6505ca3b60acb467dd0b3fae274cf513dcbfd653649ba52e9ba9740", + "schema": "lfs-publication-source-guard-v1", + "scope": "16 registered Phase 4AR source files, registration, pinned upstream checkout and license evidence; not a scientific rerun or full historical checksum audit" +} diff --git a/tools/lfs-publication/capture_source_guard.py b/tools/lfs-publication/capture_source_guard.py new file mode 100644 index 0000000..1903769 --- /dev/null +++ b/tools/lfs-publication/capture_source_guard.py @@ -0,0 +1,85 @@ +"""Read-only source guard; write receipt only to a new publication path.""" +import argparse +from datetime import datetime, timezone +import hashlib +import json +from pathlib import Path +import shutil +import subprocess + +from publish_archives import no_links, sha_file, canonical + +ROOT = Path('/data/digital-drosophila') +REGISTERED = 'b63e6281b6505ca3b60acb467dd0b3fae274cf513dcbfd653649ba52e9ba9740' +SOURCE_COMMIT = '80aeb2093db655329774eff7b32fb6e417b8b40b' + + +def git(*args, cwd=ROOT): + return subprocess.check_output(['git', '-C', str(cwd), *args], text=True).strip() + + +def capture(plan_path, output): + output = no_links(output) + if output == ROOT or ROOT in output.parents or output.exists(): + raise ValueError('Receipt must be new and outside the scientific source') + plan = json.loads(Path(plan_path).read_bytes()) + path = ROOT / 'experiments/phase4ar-normal-conditioning/protocol/preregistered.json' + raw = path.read_bytes() + if hashlib.sha256(raw).hexdigest() != REGISTERED: + raise ValueError('Phase 4AR registration hash mismatch') + registration = json.loads(raw) + records = [] + for filename, expected in sorted(registration['source_sha256'].items()): + source = no_links(filename) + if ROOT not in source.parents: + raise ValueError('Registered source outside project') + actual = sha_file(source) + if actual['sha256'] != expected: + raise ValueError('Phase 4AR source mismatch: ' + filename) + records.append(dict(path=filename, **actual)) + if len(records) != 16: + raise ValueError('Registered source inventory differs from reviewed scope') + if git('cat-file', '-t', SOURCE_COMMIT) != 'commit': + raise ValueError('Registered scientific commit unavailable') + evidence = [] + for group in plan['groups']: + licence = group['license'] + if 'evidence_file' in licence: + source = ROOT / group['paths'][0] / licence['evidence_file'] + actual = sha_file(source) + if actual['sha256'] != licence['evidence_sha256']: + raise ValueError('License evidence hash mismatch: ' + group['id']) + evidence.append(dict(group=group['id'], path=str(source), **actual)) + provenance = group['provenance'] + source = ROOT / group['paths'][0] + if (source / '.git').exists(): + if git('rev-parse', 'HEAD', cwd=source) != provenance['commit']: + raise ValueError('Upstream commit mismatch: ' + group['id']) + if git('status', '--porcelain', cwd=source): + raise ValueError('Upstream checkout changed: ' + group['id']) + result = dict(schema='lfs-publication-source-guard-v1', + observed_utc=datetime.now(timezone.utc).isoformat(), + registration_sha256=REGISTERED, registered_scientific_commit=SOURCE_COMMIT, + phase4ar_registered_sources=records, license_evidence=evidence, + original_checkout_head=git('rev-parse', 'HEAD'), + original_checkout_status=git('status', '--porcelain'), + original_checkout_branch=git('branch', '--show-current'), + disk_free_bytes={str(p): shutil.disk_usage(p).free for p in (ROOT, output.parent)}, + pass_all=True, + scope='16 registered Phase 4AR source files, registration, pinned upstream checkout and license evidence; not a scientific rerun or full historical checksum audit') + if result['original_checkout_status']: + raise ValueError('Original checkout is no longer clean; preserve and investigate') + if any(x < 100 * 1024**3 for x in result['disk_free_bytes'].values()): + raise ValueError('Less than 100 GiB free reserve') + with output.open('xb') as handle: + handle.write(canonical(result)) + return dict(pass_all=True, receipt=str(output), registered_sources=len(records), + original_checkout_head=result['original_checkout_head']) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--plan', required=True) + parser.add_argument('--out', required=True) + args = parser.parse_args() + print(json.dumps(capture(args.plan, args.out), sort_keys=True)) diff --git a/tools/lfs-publication/publish_archives.py b/tools/lfs-publication/publish_archives.py new file mode 100644 index 0000000..cfe09c8 --- /dev/null +++ b/tools/lfs-publication/publish_archives.py @@ -0,0 +1,534 @@ +"""Whitelist-only LFS archives; content verification, not a redistribution grant. + +No source writes, cache-based trust, extraction, or Git operations. Failed packs +remain incomplete and cannot be retried into the same output/work directories. +File stability checks are observations, not an atomic filesystem snapshot. +""" +from __future__ import annotations + +import argparse +import gzip +import hashlib +import io +import json +import os +from pathlib import Path, PurePosixPath +import re +import stat +import subprocess +import tarfile +import tempfile +import threading + +SOURCE_ROOT = "/data/digital-drosophila" +CHUNK_BYTES = 64 * 1024 * 1024 +BUFFER = 1024 * 1024 +DENIED_DIRS = {".git", ".venv", "venv", "env", "__pycache__", ".cache", + "node_modules", ".pytest_cache", ".mypy_cache", "extension-build"} +LOG_DIRS = {"logs", "_logs"} +BINARY_SUFFIXES = {".npy", ".npz", ".feather", ".parquet", ".gz", ".zst", ".zip", + ".tar", ".png", ".jpg", ".jpeg", ".gif", ".webp", ".pdf", + ".woff", ".woff2", ".ttf", ".ico", ".so", ".o", ".a", ".bin"} +HEX = re.compile(r"^[0-9a-f]{64}$") +SECRET_RULES = ( + ("private-key", re.compile(r"-----BEGIN (?:[A-Z ]+ )?PRIVATE KEY-----")), + ("github-token", re.compile(r"\b(?:ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})\b")), + ("aws-access-key", re.compile(r"\bAKIA[0-9A-Z]{16}\b")), + ("url-user-password", re.compile(r"\b[a-zA-Z][a-zA-Z0-9+.-]*://[^\s/:@]+:[^\s/@]+@")), + ("secret-literal", re.compile(r'''(?ix)\b(?:password|passwd|api[_-]?key|secret[_-]?key)\b["']?\s*[:=]\s*(?:["']([^"'\r\n]+)["']|([A-Za-z0-9_+/@.-]{4,}))''')), +) + + +def canonical(value): + return (json.dumps(value, sort_keys=True, indent=2, ensure_ascii=False, allow_nan=False) + "\n").encode() + + +def json_bytes(data): + def unique(pairs): + value = {} + for key, item in pairs: + if key in value: + raise ValueError("Duplicate JSON key") + value[key] = item + return value + return json.loads(data, object_pairs_hook=unique) + + +def json_read(path): + return json_bytes(Path(path).read_bytes()) + + +def relpath(value): + if not isinstance(value, str) or not value or "\\" in value or "\x00" in value: + raise ValueError("Invalid relative path") + p = PurePosixPath(value) + if p.is_absolute() or any(x in ("", ".", "..") for x in value.split("/")) or ":" in value: + raise ValueError("Unsafe relative path") + return p.as_posix() + + +def forbidden(relative): + parts = PurePosixPath(relative).parts + lower = [x.lower() for x in parts] + if any(x in DENIED_DIRS or x.startswith(".venv") for x in lower): + return "git-environment-or-cache" + if any(x.startswith("phase4ar") for x in lower): + return "active-phase4ar" + if any(x in LOG_DIRS for x in lower) or lower[-1].endswith((".log", ".log.gz")): + return "log" + name = lower[-1] + if (name == ".env" or name.startswith(".env.") or name in + {"id_rsa", "id_ed25519", "id_dsa", ".git-credentials", ".netrc"} + or any(x in name for x in ("credentials", "secrets")) + or name.endswith((".pem", ".key", ".p12", ".pfx"))): + return "credential-file" + return None + + +def no_links(path): + if ".." in Path(path).parts: + raise ValueError("Parent path components refused") + p = Path(path).absolute() + for part in (p, *p.parents): + if part.exists() or part.is_symlink(): + s = part.lstat() + if (stat.S_ISLNK(s.st_mode) or + getattr(s, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)): + raise ValueError("Symlink refused: " + str(part)) + return p + + +def identity(s): + return (s.st_dev, s.st_ino, s.st_mode, s.st_size, s.st_mtime_ns, s.st_ctime_ns) + + +def opened_identity(handle, expected): + actual = identity(os.fstat(handle.fileno())) + # Some Windows Python builds expose birth time via stat().ctime but change + # time via fstat().ctime. Check each clock against itself before/after; never + # omit ctime from either endpoint's stability check. Linux uses all fields. + same = actual[:-1] == expected[:-1] if os.name == "nt" else actual == expected + if not same: + raise ValueError("File changed while opening") + return actual + + +def sha_file(path): + p = no_links(path) + before = p.stat() + if not stat.S_ISREG(before.st_mode): + raise ValueError("Not a regular file: " + str(p)) + h = hashlib.sha256() + size = 0 + with p.open("rb") as f: + opened = opened_identity(f, identity(before)) + while block := f.read(BUFFER): + h.update(block) + size += len(block) + if identity(os.fstat(f.fileno())) != opened: + raise ValueError("File changed while hashing: " + str(p)) + if identity(p.lstat()) != identity(before) or size != before.st_size: + raise ValueError("File changed while hashing: " + str(p)) + return {"sha256": h.hexdigest(), "bytes": size} + + +def write_new(path, data): + with Path(path).open("xb") as f: + f.write(data) + + +def validate_plan(plan): + if not isinstance(plan, dict) or set(plan) - {"schema", "source_root", "groups", "exclude_paths"}: + raise ValueError("Unknown plan fields") + if plan.get("schema") != "lfs-publication-plan-v1" or plan.get("source_root") != SOURCE_ROOT: + raise ValueError("Wrong plan schema/source_root") + groups = plan.get("groups") + if not isinstance(groups, list) or not groups: + raise ValueError("Nonempty groups required") + seen, all_paths = set(), [] + for g in groups: + if not isinstance(g, dict) or set(g) != {"id", "paths", "license", "provenance"}: + raise ValueError("Group requires id/paths/license/provenance only") + group_id = g["id"] + if not isinstance(group_id, str) or not re.fullmatch(r"[a-z0-9][a-z0-9._-]{0,99}", group_id) or group_id in seen: + raise ValueError("Invalid or duplicate group id") + seen.add(group_id) + if not isinstance(g["paths"], list) or not g["paths"] or not g["license"] or not g["provenance"]: + raise ValueError("Nonempty paths/license/provenance required") + for value in g["paths"]: + p = relpath(value) + if forbidden(p): + raise ValueError("Explicit forbidden path: " + p) + if any(p == old or p.startswith(old + "/") or old.startswith(p + "/") for old in all_paths): + raise ValueError("Overlapping whitelist paths") + all_paths.append(p) + excludes = plan.get("exclude_paths", []) + if not isinstance(excludes, list): + raise ValueError("exclude_paths must be a list") + for p in excludes: + relpath(p) + canonical(plan) + + +def enumerate_group(root, group, excludes): + files, omitted = [], [] + + def visit(p): + relative = p.relative_to(root).as_posix() + s = p.lstat() + if stat.S_ISLNK(s.st_mode): + raise ValueError("Symlink refused: " + relative) + reason = forbidden(relative) + if any(relative == x or relative.startswith(x + "/") for x in excludes): + reason = "plan-exclusion" + if reason: + omitted.append({"path": relative, "reason": reason, + "scope": "subtree" if stat.S_ISDIR(s.st_mode) else "file", + "bytes": s.st_size if stat.S_ISREG(s.st_mode) else None}) + elif stat.S_ISDIR(s.st_mode): + for child in sorted(p.iterdir(), key=lambda x: x.name): + visit(child) + elif stat.S_ISREG(s.st_mode): + files.append((relative, identity(s))) + else: + raise ValueError("Nonregular source refused: " + relative) + + for relative in group["paths"]: + path = no_links(root / relative) + visit(path) + files.sort(key=lambda x: x[0]) + omitted.sort(key=lambda x: x["path"]) + if not files: + raise ValueError("Empty group refused") + return files, omitted + + +def scan_secrets(root, files): + for relative, original in files: + p = root / relative + if original[3] > 2 * 1024 * 1024 or p.suffix.lower() in BINARY_SUFFIXES: + continue + data = p.read_bytes() + if identity(p.lstat()) != original: + raise ValueError("Source changed during secret scan: " + relative) + try: + text = data.decode("utf-8") + except UnicodeDecodeError: + continue + if "\x00" in text: + continue + # Per-rule ASCII necessary fragments avoid repeated scans of numeric SWC + # bodies (their header includes a public URL). Non-ASCII text still uses + # every regex because Unicode IGNORECASE has broader equivalences. + lower = text.lower() + fragments = { + "private-key": ("private key",), + "github-token": ("ghp_", "github_pat_"), + "aws-access-key": ("akia",), + "url-user-password": ("://",), + "secret-literal": ("password", "passwd", "api", "secret"), + } + for rule, pattern in SECRET_RULES: + if text.isascii() and not any(x in lower for x in fragments[rule]): + continue + for match in pattern.finditer(text): + if rule == "secret-literal": + literal = match.group(1) or match.group(2) + if literal.lower() in {"none", "null", "false", "true"}: + continue + raise ValueError("Secret scan refused: " + relative + " [" + rule + "]") + + +def gzip_rows(path, rows): + with Path(path).open("xb") as raw: + with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as out: + for row in rows: + out.write((json.dumps(row, sort_keys=True, ensure_ascii=False, allow_nan=False) + "\n").encode()) + + +class HashReader: + def __init__(self, handle): + self.handle, self.hash, self.size = handle, hashlib.sha256(), 0 + + def read(self, count=-1): + data = self.handle.read(count) + self.hash.update(data) + self.size += len(data) + return data + + +def zstd_command(decompress=False, threads=2): + return ["zstd", "-q", "-d", "-c"] if decompress else ["zstd", "-q", "-3", "-T" + str(threads), "-c"] + + +def make_archive(root, files, destination, threads): + rows = [] + with destination.open("xb") as output, tempfile.TemporaryFile() as errors: + proc = subprocess.Popen(zstd_command(threads=threads), stdin=subprocess.PIPE, + stdout=output, stderr=errors) + try: + with tarfile.open(fileobj=proc.stdin, mode="w|", format=tarfile.PAX_FORMAT) as archive: + for relative, original in files: + p = no_links(root / relative) + if identity(p.lstat()) != original: + raise ValueError("Source changed before archive: " + relative) + with p.open("rb") as source: + opened = opened_identity(source, original) + reader = HashReader(source) + info = tarfile.TarInfo(relative) + info.size, info.mode, info.mtime = original[3], stat.S_IMODE(original[2]) & 0o777, 0 + archive.addfile(info, reader) + if (reader.size != original[3] or identity(os.fstat(source.fileno())) != opened + or identity(p.lstat()) != original): + raise ValueError("Source changed during archive: " + relative) + rows.append({"path": relative, "bytes": reader.size, "sha256": reader.hash.hexdigest()}) + proc.stdin.close() + if proc.wait() != 0: + raise RuntimeError("zstd compression failed; no complete catalog published") + except BaseException: + proc.kill() + proc.wait() + raise + finally: + proc.stdin.close() + return rows + + +def split_archive(archive, group_dir): + chunks, full = [], hashlib.sha256() + with archive.open("rb") as source: + index = 0 + while first := source.read(min(BUFFER, CHUNK_BYTES)): + name = "payload-%06d.bin" % index + h, size = hashlib.sha256(), 0 + with (group_dir / name).open("xb") as out: + block = first + while block: + out.write(block) + full.update(block) + h.update(block) + size += len(block) + if size == CHUNK_BYTES: + break + block = source.read(min(BUFFER, CHUNK_BYTES - size)) + chunks.append({"path": name, "bytes": size, "sha256": h.hexdigest(), "index": index}) + index += 1 + if not chunks: + raise ValueError("Empty compressed archive") + return chunks, full.hexdigest() + + +def pack(plan_path, out, work, *, threads=2): + if type(threads) is not int or not 1 <= threads <= 64: + raise ValueError("threads must be 1..64") + plan_bytes = Path(plan_path).read_bytes() + plan = json_bytes(plan_bytes) + validate_plan(plan) + root, out, work = map(no_links, (Path(plan["source_root"]), Path(out), Path(work))) + for a, b in ((root, out), (root, work), (out, work)): + if a == b or a in b.parents or b in a.parents: + raise ValueError("source, out and work must be disjoint trees") + if out.exists() or work.exists(): + raise FileExistsError("out/work must both be new exclusive paths") + out.mkdir(parents=True, exist_ok=False) + work.mkdir(parents=True, exist_ok=False) + write_new(out / "plan.json", plan_bytes) + plan_sha = hashlib.sha256(plan_bytes).hexdigest() + results = [] + for group in plan["groups"]: + files, exclusions = enumerate_group(root, group, plan.get("exclude_paths", [])) + scan_secrets(root, files) + group_dir = out / group["id"] + group_dir.mkdir() + archive = work / (group["id"] + ".tar.zst") + members = make_archive(root, files, archive, threads) + # A second metadata enumeration catches file additions/removals during pack. + after_files, after_exclusions = enumerate_group(root, group, plan.get("exclude_paths", [])) + if after_files != files or after_exclusions != exclusions: + raise ValueError("Source inventory changed during pack") + gzip_rows(group_dir / "members.jsonl.gz", members) + gzip_rows(group_dir / "exclusions.jsonl.gz", exclusions) + chunks, combined = split_archive(archive, group_dir) + catalog = dict(schema="lfs-publication-group-v1", group_id=group["id"], source_root=str(root), + source_paths=group["paths"], source_files=len(members), + source_bytes=sum(x["bytes"] for x in members), license=group["license"], + provenance=group["provenance"], plan_sha256=plan_sha, + archive_format="tar.pax+zstd", archive_sha256=combined, + archive_bytes=sum(x["bytes"] for x in chunks), chunk_bytes=CHUNK_BYTES, + chunks=chunks, members=sha_file(group_dir / "members.jsonl.gz"), + exclusions=sha_file(group_dir / "exclusions.jsonl.gz"), + exclusion_records=len(exclusions), + verification_scope="content snapshot, not a license grant or nested-archive secret audit", + secret_scan="UTF-8 files <=2 MiB, known binary suffixes excluded; fail closed on matching rules") + write_new(group_dir / "catalog.json", canonical(catalog)) + result = {"group": group["id"], "files": len(members), "source_bytes": catalog["source_bytes"], + "archive_bytes": catalog["archive_bytes"], "chunks": len(chunks), + "catalog_sha256": sha_file(group_dir / "catalog.json")["sha256"]} + results.append(result) + print(json.dumps({"group_complete": result}), flush=True) + summary = {"schema": "lfs-publication-pack-v1", "complete": True, "plan_sha256": plan_sha, + "groups": results, "source_files": sum(x["files"] for x in results), + "source_bytes": sum(x["source_bytes"] for x in results)} + write_new(out / "pack-summary.json", canonical(summary)) + return summary + + +def check_digest_row(row): + if not isinstance(row, dict) or type(row.get("bytes")) is not int or row["bytes"] < 0 or not HEX.fullmatch(str(row.get("sha256", ""))): + raise ValueError("Invalid SHA/size metadata") + + +def iter_members(path): + previous = None + with gzip.open(path, "rt", encoding="utf-8") as inp: + for line in inp: + row = json.loads(line) + if set(row) != {"path", "bytes", "sha256"}: + raise ValueError("Invalid member row") + check_digest_row(row) + p = relpath(row["path"]) + if forbidden(p) or (previous is not None and p <= previous): + raise ValueError("Forbidden/duplicate/unsorted member: " + p) + previous = p + yield row + + +def verify(group_dir): + group_dir = no_links(group_dir) + controls_before = {name: sha_file(group_dir / name) for name in + ("catalog.json", "members.jsonl.gz", "exclusions.jsonl.gz")} + catalog = json_read(group_dir / "catalog.json") + if catalog.get("schema") != "lfs-publication-group-v1" or catalog.get("chunk_bytes") != CHUNK_BYTES or catalog.get("archive_format") != "tar.pax+zstd": + raise ValueError("Wrong catalog schema/format/chunk size") + chunks = catalog.get("chunks") + if not isinstance(chunks, list) or not chunks: + raise ValueError("Missing payload chunks") + expected = {"catalog.json", "members.jsonl.gz", "exclusions.jsonl.gz"} + for i, row in enumerate(chunks): + check_digest_row(row) + if set(row) != {"path", "bytes", "sha256", "index"} or type(row["index"]) is not int or row["index"] != i or row["path"] != "payload-%06d.bin" % i: + raise ValueError("Wrong chunk name/order") + if not 0 < row["bytes"] <= CHUNK_BYTES or (i < len(chunks) - 1 and row["bytes"] != CHUNK_BYTES): + raise ValueError("Wrong chunk size") + expected.add(row["path"]) + if {p.name for p in group_dir.iterdir()} != expected: + raise ValueError("Extra/missing downloaded files") + for p in group_dir.iterdir(): + no_links(p) + if not p.is_file(): + raise ValueError("Nonfile downloaded entry") + for name, key in (("members.jsonl.gz", "members"), ("exclusions.jsonl.gz", "exclusions")): + check_digest_row(catalog[key]) + if controls_before[name] != catalog[key]: + raise ValueError("Metadata digest mismatch: " + name) + with gzip.open(group_dir / "exclusions.jsonl.gz", "rt", encoding="utf-8") as source: + omitted = list(map(json.loads, source)) + if len(omitted) != catalog.get("exclusion_records"): + raise ValueError("Exclusion inventory mismatch") + for row in omitted: + relpath(row["path"]) + errors, combined, compressed = [], hashlib.sha256(), [0] + with tempfile.TemporaryFile() as diagnostics: + proc = subprocess.Popen(zstd_command(decompress=True), stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=diagnostics) + + def feed(): + try: + for row in chunks: + p = group_dir / row["path"] + before = identity(p.lstat()) + h, size = hashlib.sha256(), 0 + with p.open("rb") as source: + opened = opened_identity(source, before) + while data := source.read(BUFFER): + h.update(data) + combined.update(data) + size += len(data) + compressed[0] += len(data) + proc.stdin.write(data) + if identity(os.fstat(source.fileno())) != opened or identity(p.lstat()) != before: + raise ValueError("Chunk changed during read") + if size != row["bytes"] or h.hexdigest() != row["sha256"]: + raise ValueError("Payload digest mismatch: " + row["path"]) + except BaseException as exc: + errors.append(exc) + finally: + try: + proc.stdin.close() + except OSError: + pass + + producer = threading.Thread(target=feed, daemon=True) + producer.start() + count, total = 0, 0 + expected_rows = iter(iter_members(group_dir / "members.jsonl.gz")) + try: + with tarfile.open(fileobj=proc.stdout, mode="r|") as archive: + for member in archive: + row = next(expected_rows, None) + if not member.isreg() or member.linkname or member.sparse is not None: + raise ValueError("Nonregular/sparse/link tar member refused") + path = relpath(member.name) + if forbidden(path) or row is None or path != row["path"] or member.size != row["bytes"]: + raise ValueError("Tar member set/size/path mismatch: " + path) + reader = HashReader(archive.extractfile(member)) + while reader.read(BUFFER): + pass + if reader.size != row["bytes"] or reader.hash.hexdigest() != row["sha256"]: + raise ValueError("Tar member content mismatch: " + path) + count += 1 + total += reader.size + # tarfile stops at the first end marker; reject a hidden second + # archive/nonzero trailer, including bytes in its read-ahead buf. + while trailer := archive.fileobj.read(BUFFER): + if any(trailer): + raise ValueError("Nonzero data after tar end marker") + if next(expected_rows, None) is not None: + raise ValueError("Missing tar members") + # Drain to validate zstd's complete stream/checksum and finish the feeder. + while proc.stdout.read(BUFFER): + pass + producer.join() + code = proc.wait() + if errors: + raise ValueError("Payload streaming failed: " + str(errors[0])) + if code != 0: + raise ValueError("zstd decompression failed") + except BaseException: + proc.kill() + proc.wait() + producer.join(timeout=10) + raise + finally: + proc.stdout.close() + expected_rows.close() + if (count != catalog.get("source_files") or total != catalog.get("source_bytes") + or compressed[0] != catalog.get("archive_bytes") or combined.hexdigest() != catalog.get("archive_sha256")): + raise ValueError("Final inventory/archive digest mismatch") + for name, before in controls_before.items(): + if sha_file(group_dir / name) != before: + raise ValueError("Downloaded control changed during verification: " + name) + return {"schema": "lfs-publication-verify-v1", "pass_all": True, "group": catalog["group_id"], + "source_files": count, "source_bytes": total, "archive_bytes": compressed[0], + "archive_sha256": combined.hexdigest(), "catalog_sha256": controls_before["catalog.json"]["sha256"], + "scope": "downloaded payload bytes and streamed members; no extraction, no LFS-cache trust; catalog authenticity is caller responsibility"} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + command = sub.add_parser("pack") + command.add_argument("--plan", required=True) + command.add_argument("--out", required=True) + command.add_argument("--work", required=True) + command.add_argument("--threads", type=int, default=2) + command = sub.add_parser("verify") + command.add_argument("--group-dir", required=True) + args = parser.parse_args() + result = (pack(args.plan, args.out, args.work, threads=args.threads) if args.command == "pack" + else verify(args.group_dir)) + print(json.dumps(result, sort_keys=True, ensure_ascii=False, allow_nan=False), flush=True) + + +if __name__ == "__main__": + main() diff --git a/tools/lfs-publication/test_publish_archives.py b/tools/lfs-publication/test_publish_archives.py new file mode 100644 index 0000000..ff64262 --- /dev/null +++ b/tools/lfs-publication/test_publish_archives.py @@ -0,0 +1,245 @@ +"""Temporary local fixtures only. Identity codec tests do not certify zstd. + +The optional real-zstd test automatically runs when the executable is present. +""" +import contextlib +import gzip +import hashlib +import importlib.util +import io +import json +import os +from pathlib import Path +import shutil +import sys +import tarfile +import tempfile +import unittest +from unittest.mock import patch + +SPEC = importlib.util.spec_from_file_location("lfs_archive_tested", Path(__file__).with_name("publish_archives.py")) +M = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(M) + + +def codec(*args, **kwargs): + return [sys.executable, "-c", "import sys,shutil;shutil.copyfileobj(sys.stdin.buffer,sys.stdout.buffer)"] + + +class ArchiveTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.base = Path(self.temp.name) + self.root = self.base / "source" + (self.root / "data").mkdir(parents=True) + (self.root / "data" / "a.txt").write_text("public source\n", encoding="utf-8") + (self.root / "data" / "b.bin").write_bytes(bytes(range(256)) * 80) + (self.root / "data" / "zero.txt").write_bytes(b"") + self.plan = {"schema": "lfs-publication-plan-v1", "source_root": str(self.root), + "groups": [{"id": "data", "paths": ["data"], "license": "test-only", + "provenance": {"source": "synthetic"}}], "exclude_paths": []} + self.plan_path = self.base / "plan.json" + self.plan_path.write_bytes(M.canonical(self.plan)) + self.out, self.work = self.base / "out", self.base / "work" + self.addCleanup(patch.stopall) + patch.object(M, "SOURCE_ROOT", str(self.root)).start() + patch.object(M, "CHUNK_BYTES", 4096).start() + self.codec_patch = patch.object(M, "zstd_command", codec) + self.codec_patch.start() + + def pack(self): + with contextlib.redirect_stdout(io.StringIO()): + return M.pack(self.plan_path, self.out, self.work) + + def catalog(self): + return json.loads((self.out / "data" / "catalog.json").read_bytes()) + + def save_catalog(self, cat): + (self.out / "data" / "catalog.json").write_bytes(M.canonical(cat)) + + def rewrite_archive(self, data): + group = self.out / "data" + cat = self.catalog() + for row in cat["chunks"]: + (group / row["path"]).unlink() + new = self.base / "mutant.tar" + new.write_bytes(data) + cat["chunks"], cat["archive_sha256"] = M.split_archive(new, group) + cat["archive_bytes"] = len(data) + self.save_catalog(cat) + + def test_pack_verify_full_member_content_and_no_extraction(self): + before = {p.relative_to(self.root).as_posix(): p.read_bytes() for p in self.root.rglob("*") if p.is_file()} + summary = self.pack() + result = M.verify(self.out / "data") + self.assertTrue(result["pass_all"]) + self.assertEqual(result["source_files"], 3) + self.assertEqual(result["source_bytes"], sum(map(len, before.values()))) + self.assertGreater(summary["groups"][0]["chunks"], 1) + self.assertEqual(before, {p.relative_to(self.root).as_posix(): p.read_bytes() for p in self.root.rglob("*") if p.is_file()}) + self.assertFalse((self.out / "data" / "a.txt").exists()) + + def test_plan_executes_first_read_bytes_only(self): + with patch.object(M, "json_read", side_effect=AssertionError("Must not read plan a second time")): + self.pack() + self.assertEqual(self.catalog()["plan_sha256"], hashlib.sha256(self.plan_path.read_bytes()).hexdigest()) + + def test_exclusions_logged_without_descending(self): + for name in (".git", "venv", "__pycache__", ".cache", "node_modules", "logs", "phase4ar-normal-conditioning"): + d = self.root / "data" / name + d.mkdir() + (d / "hidden.txt").write_text("skip") + (self.root / "data" / "private.pem").write_text("private") + (self.root / "data" / "run.log").write_text("log") + self.pack() + with gzip.open(self.out / "data" / "exclusions.jsonl.gz", "rt") as f: + rows = [json.loads(x) for x in f] + self.assertEqual(len(rows), 9) + self.assertEqual(self.catalog()["source_files"], 3) + self.assertTrue(M.verify(self.out / "data")["pass_all"]) + + def test_unsafe_explicit_paths(self): + for name in ("../data", "/data", "data/../data", "data//x", "data\\x", "data/.git", "data/phase4ar-normal-conditioning"): + plan = json.loads(json.dumps(self.plan)) + plan["groups"][0]["paths"] = [name] + with self.subTest(name=name), self.assertRaises(ValueError): + M.validate_plan(plan) + + def test_overlapping_groups_refused(self): + self.plan["groups"].append({"id": "nested", "paths": ["data/a.txt"], "license": "test", "provenance": "test"}) + with self.assertRaisesRegex(ValueError, "Overlapping"): + M.validate_plan(self.plan) + + def test_symlink_refused(self): + try: + (self.root / "data" / "link").symlink_to(self.root / "data" / "a.txt") + except OSError: + self.skipTest("OS does not permit temporary symlinks") + with self.assertRaisesRegex(ValueError, "Symlink"): + self.pack() + + def test_exclusive_and_disjoint_outputs(self): + self.out.mkdir() + with self.assertRaises(FileExistsError): + self.pack() + with self.assertRaisesRegex(ValueError, "disjoint"): + M.pack(self.plan_path, self.root / "output", self.work) + + def test_parent_components_cannot_bypass_disjoint_gate(self): + malicious = self.base / "unused" / ".." / "source" / "new-output" + with self.assertRaisesRegex(ValueError, "Parent path components"): + M.pack(self.plan_path, malicious, self.work) + self.assertFalse((self.root / "new-output").exists()) + + def test_secret_rules_hide_values(self): + # Build synthetic fixtures at runtime; keep actual credential-like + # strings out of Git blobs so the repository secret guard stays useful. + examples = ["-----BEGIN " + "OPENSSH PRIVATE " + "KEY-----", "ghp_" + "a" * 36, + "github_pat_" + "b" * 40, "AKIA" + "A" * 16, + "https://" + "account:" + "privatevalue" + "@example.org/", + 'password' + ' = "privatevalue"', + 'api_key' + ': "privatevalue"', 'secret_key' + '=ABCDEF123456', + 'pa\u017f\u017fword' + '="privatevalue"', + 'ap\u0131_key' + '="privatevalue"', + '\u017fecret_key' + '="privatevalue"'] + path = self.root / "data" / "literal.txt" + for value in examples: + path.write_text(value) + files, _ = M.enumerate_group(self.root, self.plan["groups"][0], []) + with self.subTest(rule=value[:8]), self.assertRaisesRegex(ValueError, "Secret scan refused") as caught: + M.scan_secrets(self.root, files) + self.assertNotIn(value, str(caught.exception)) + self.assertNotIn("privatevalue", str(caught.exception)) + + def test_mutated_source_metadata_refused(self): + files, _ = M.enumerate_group(self.root, self.plan["groups"][0], []) + (self.root / "data" / "a.txt").write_text("changed bytes") + with self.assertRaisesRegex(ValueError, "Source changed"): + M.make_archive(self.root, files, self.base / "bad.tar", 1) + + def test_payload_corruption_refused(self): + self.pack() + p = self.out / "data" / self.catalog()["chunks"][0]["path"] + data = bytearray(p.read_bytes()) + data[-1] ^= 1 + p.write_bytes(data) + with self.assertRaises((ValueError, tarfile.TarError)): + M.verify(self.out / "data") + + def test_missing_extra_or_reordered_payload_refused(self): + self.pack() + extra = self.out / "data" / "extra.bin" + extra.write_bytes(b"x") + with self.assertRaisesRegex(ValueError, "Extra/missing"): + M.verify(self.out / "data") + extra.unlink() + cat = self.catalog() + cat["chunks"] = list(reversed(cat["chunks"])) + self.save_catalog(cat) + with self.assertRaisesRegex(ValueError, "chunk name/order"): + M.verify(self.out / "data") + + def test_tar_traversal_rejected_even_with_rehashed_chunks(self): + self.pack() + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w") as archive: + info = tarfile.TarInfo("../escape") + info.size = 1 + archive.addfile(info, io.BytesIO(b"x")) + self.rewrite_archive(buf.getvalue()) + with self.assertRaisesRegex(ValueError, "Unsafe relative path"): + M.verify(self.out / "data") + + def test_tar_link_rejected_even_with_rehashed_chunks(self): + self.pack() + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w") as archive: + info = tarfile.TarInfo("data/a.txt") + info.type, info.linkname = tarfile.SYMTYPE, "/tmp/escape" + archive.addfile(info) + self.rewrite_archive(buf.getvalue()) + with self.assertRaisesRegex(ValueError, "Nonregular"): + M.verify(self.out / "data") + + def test_hidden_tar_after_end_refused(self): + self.pack() + raw = (self.work / "data.tar.zst").read_bytes() + self.rewrite_archive(raw + b"nonzero hidden second archive") + with self.assertRaisesRegex(ValueError, "Nonzero data after tar"): + M.verify(self.out / "data") + + def test_second_tar_after_end_refused(self): + self.pack() + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w") as archive: + info = tarfile.TarInfo("unlisted-secret.txt") + info.size = 1 + archive.addfile(info, io.BytesIO(b"x")) + self.rewrite_archive((self.work / "data.tar.zst").read_bytes() + buf.getvalue()) + with self.assertRaisesRegex(ValueError, "Nonzero data after tar"): + M.verify(self.out / "data") + + def test_member_hash_mutation_refused(self): + self.pack() + p = self.out / "data" / "members.jsonl.gz" + with gzip.open(p, "rt") as f: + rows = [json.loads(x) for x in f] + rows[0]["sha256"] = "0" * 64 + p.unlink() + M.gzip_rows(p, rows) + cat = self.catalog() + cat["members"] = M.sha_file(p) + self.save_catalog(cat) + with self.assertRaisesRegex(ValueError, "member content mismatch"): + M.verify(self.out / "data") + + @unittest.skipUnless(shutil.which("zstd"), "Real zstd unavailable; run this integration test on Linux") + def test_real_zstd_roundtrip(self): + self.codec_patch.stop() + self.pack() + self.assertTrue(M.verify(self.out / "data")["pass_all"]) + + +if __name__ == "__main__": + unittest.main()