Publish licensed data snapshots with LAN LFS transfer and content audits

This commit is contained in:
Digital Drosophila Archive
2026-10-01 03:42:51 +08:00
parent 13cd0e0be7
commit 6e33e05ede
173 changed files with 3614 additions and 0 deletions
+10
View File
@@ -34,8 +34,18 @@ DoOR 作者为 Daniel Münch、C. Giovanni Galizia、Shouwen Ma、Martin Strauch
`pack-summary.json` 存在且 complete=true 仅表示本地打包完成,不等于上传/远程恢复成功。上传后使用独立 clone 和新的 LFS object store 实际下载,再逐块、逐归档、逐内部文件流式校验。对应 receipt 只有在真实校验完成后才会提交。
本批实际范围:8 组、226,859 个原文件、24,040,546,041 字节;tar+zstd 分块总计 9,806,419,307 字节(不含小型目录/成员清单)。本地完整内容校验已通过。226,849 个文件与 2026-09-29 历史索引逐 SHA/大小吻合,另 10 个后增参考文件不在该旧索引,明确记为部分历史覆盖;其中 synaptic-partners 登记 SHA、作者模型 LICENSE pin 另有校验,来源原记录随包保留。不得把这次检查描述为所有历史科学阶段重新通过。
本次不是服务器所有内容的完整灾备。Gitea LFS 与 Git 仓库数据目录均需另有服务端备份;删除服务端 LFS 对象会破坏恢复能力。尚未核验 Gitea 主机磁盘容量和备份策略。
### 本次内网传输
Git SSH 本来使用 `192.168.10.31:2222`,但服务器返回的 LFS HTTPS 域名在本机解析到代理地址,导致大对象传输很慢。本次使用仓库本地的 `malecns-lan-https` transfer adapter:TCP 只连接 `192.168.10.31:443`,TLS SNI/证书验证仍使用 `git.lain.asia`,不关闭证书检查、不修改 hosts、系统代理或 Gitea 服务配置。通过原 SSH key 获取短期鉴权,凭据只保留在内存并经标准输入交给 curl,不写入提交或命令行参数。
adapter 遵循 [Git LFS 官方 custom-transfer 协议](https://github.com/git-lfs/git-lfs/blob/main/docs/custom-transfers.md),只允许该仓库的 HTTPS API/object 路径;Gitea 的 `Transfer-Encoding: chunked` 只在上传 PUT 场景被精确处理。它仍上传标准 LFS 对象,其他电脑可以继续使用普通 Git LFS 下载,不依赖这个内网 adapter。并发为 2,分块可重试;科学实验不得因此重启或重试。
最初通过代理的核心数据 push 为切换路线而主动停止,未完成的远端分支更新不计为成功。Git 提交和已传对象保留,后续按内容地址续传;严格 header 校验曾在开始 PUT 前拒绝服务器 chunked header,适配后再测试,未改变任何源文件。最终状态以独立下载回执为准,不把这些准备动作记为完成。
## 安全恢复(Linux 示例)
先安装 Git LFS、Python 3 和 zstd。只在新目录恢复,不要覆盖运行中的 `/data/digital-drosophila`。
@@ -0,0 +1,40 @@
{
"adapter": {
"bytes": 16661,
"sha256": "21241d6a75d84108e720b4a051504fff00158b327f15c0ee06e5c57c6ad1c263"
},
"certificate_verification": true,
"hostname": "git.lain.asia",
"ip": "192.168.10.31",
"observed_utc": "2026-09-30T19:40:34.704953+00:00",
"pass_all": true,
"port": 443,
"proxy_used": false,
"schema": "lfs-lan-roundtrip-smoke-v1",
"transfers": [
{
"bytes": 67108864,
"complete": true,
"methods": [
"POST",
"PUT",
"POST"
],
"operation": "upload",
"seconds": 0.9692029840080068,
"sha256": "a507fb9917a63f6221d97780727831735bcf419bb6d36b17546f2be04e8f6b6b"
},
{
"bytes": 67108864,
"complete": true,
"methods": [
"POST",
"GET"
],
"new_download_file": "/tmp/malecns-lfs-publication-20261001-bI1u07/repo/.git/lfs/lan-transfer-tmp/a507fb9917a63f6221d97780727831735bcf419bb6d36b17546f2be04e8f6b6b.0vvk_cw4.partial",
"operation": "download",
"seconds": 0.6818166430020938,
"sha256": "a507fb9917a63f6221d97780727831735bcf419bb6d36b17546f2be04e8f6b6b"
}
]
}
@@ -0,0 +1,206 @@
{
"aggregate_inventory_sha256": "88814458c772fc0e5b1708bc05e6c478097fc10b55271a83d538855ccfa81c84",
"archive_bytes": 9806419307,
"fresh_download_proven": false,
"git": {
"cwd": "/tmp/malecns-lfs-publication-20261001-bI1u07/repo",
"head": "13cd0e0be7a24ca9d7c036876bc48f9b22652968",
"scope": "HEAD observed at invocation cwd; not a clean-tree or payload-to-commit proof"
},
"groups": [
{
"archive_bytes": 834266158,
"archive_sha256": "feebf3fe3f3c2c9fea3db96fab8267330de49af7ab37008b84caeeea8dc272ed",
"catalog_path": "/tmp/malecns-lfs-publication-20261001-bI1u07/repo/artifacts/public-resources-20261001/malecns-raw-core/catalog.json",
"catalog_sha256": "8a7ea9abe5f75ce31318d207a1dbb491d5e151af27bd607dc1b59fb29d4aacf8",
"group": "malecns-raw-core",
"historical_matched": 17,
"historical_missing": 0,
"license_evidence_verified": null,
"pass_all": true,
"registered_file_verified": null,
"schema": "lfs-publication-verify-v1",
"scope": "downloaded payload bytes and streamed members; no extraction, no LFS-cache trust; catalog authenticity is caller responsibility",
"source_bytes": 1207032031,
"source_files": 17
},
{
"archive_bytes": 2760276945,
"archive_sha256": "537644b322747b821580818aaa4154ab15714f0ddbcebc3183b587682760f57e",
"catalog_path": "/tmp/malecns-lfs-publication-20261001-bI1u07/repo/artifacts/public-resources-20261001/malecns-swc/catalog.json",
"catalog_sha256": "efc330317e2f74272e233634e5652982232ef9d3b6762f8e3cbd9eeaf05487cf",
"group": "malecns-swc",
"historical_matched": 211573,
"historical_missing": 0,
"license_evidence_verified": null,
"pass_all": true,
"registered_file_verified": null,
"schema": "lfs-publication-verify-v1",
"scope": "downloaded payload bytes and streamed members; no extraction, no LFS-cache trust; catalog authenticity is caller responsibility",
"source_bytes": 9299975999,
"source_files": 211573
},
{
"archive_bytes": 635332131,
"archive_sha256": "a5d8ac0e6eebd5c6235078e9a9ac5a5c224bc86185bb77dd38248833c6b946f1",
"catalog_path": "/tmp/malecns-lfs-publication-20261001-bI1u07/repo/artifacts/public-resources-20261001/malecns-processed/catalog.json",
"catalog_sha256": "e1e42f654cb005d2f073d4e84c808af50f50c3feeb9cbcb0b0c1c51746bfdb47",
"group": "malecns-processed",
"historical_matched": 64,
"historical_missing": 0,
"license_evidence_verified": null,
"pass_all": true,
"registered_file_verified": null,
"schema": "lfs-publication-verify-v1",
"scope": "downloaded payload bytes and streamed members; no extraction, no LFS-cache trust; catalog authenticity is caller responsibility",
"source_bytes": 1140025370,
"source_files": 64
},
{
"archive_bytes": 5063205053,
"archive_sha256": "cf2927389ecf3d20cbccaa269941265e8c709544c60068f28a40a0643c6be13d",
"catalog_path": "/tmp/malecns-lfs-publication-20261001-bI1u07/repo/artifacts/public-resources-20261001/malecns-synaptic-partners/catalog.json",
"catalog_sha256": "bdbc0354abdad63a9c256a2841fdf9df4c32258647b8de03d066afc89fa61053",
"group": "malecns-synaptic-partners",
"historical_matched": 0,
"historical_missing": 6,
"license_evidence_verified": null,
"pass_all": true,
"registered_file_verified": {
"bytes": 6777179098,
"path": "experiments/phase4aa-apl-pn-scope/external-resources/synaptic-partners/syn-partners-male-cns-v1.0-minconf-0.5.feather",
"sha256": "959d8ef4173b35382a3e6acfaf5167c795b6d10b877572d146af04e1b487bc07"
},
"schema": "lfs-publication-verify-v1",
"scope": "downloaded payload bytes and streamed members; no extraction, no LFS-cache trust; catalog authenticity is caller responsibility",
"source_bytes": 6777184219,
"source_files": 6
},
{
"archive_bytes": 77971754,
"archive_sha256": "2366caef95fb5ad038639a6ae2ad25139a14c42492bc19640c20af46fefa0bc7",
"catalog_path": "/tmp/malecns-lfs-publication-20261001-bI1u07/repo/artifacts/public-resources-20261001/celltype-explorer/catalog.json",
"catalog_sha256": "4dc6be6091b273312a1dd61f621654f05e4c12118bdd13c22905d588e378889c",
"group": "celltype-explorer",
"historical_matched": 8973,
"historical_missing": 0,
"license_evidence_verified": {
"bytes": 18657,
"path": "experiments/phase4f-retinotopic-vision/external-resources/celltype-explorer/LICENSE",
"sha256": "9ba9550ad48438d0836ddab3da480b3b69ffa0aac7b7878b5a0039e7ab429411"
},
"pass_all": true,
"registered_file_verified": null,
"schema": "lfs-publication-verify-v1",
"scope": "downloaded payload bytes and streamed members; no extraction, no LFS-cache trust; catalog authenticity is caller responsibility",
"source_bytes": 3957079583,
"source_files": 8973
},
{
"archive_bytes": 434458454,
"archive_sha256": "683b85394fb44e42704c275fbda93c95072f9f64c16a84e47ce2821c060cc39f",
"catalog_path": "/tmp/malecns-lfs-publication-20261001-bI1u07/repo/artifacts/public-resources-20261001/visualpathways/catalog.json",
"catalog_sha256": "bd824a656235ba08a35a67ad6d865467a946145cf985677226412280380939fc",
"group": "visualpathways",
"historical_matched": 6033,
"historical_missing": 0,
"license_evidence_verified": {
"bytes": 35149,
"path": "experiments/phase4f-retinotopic-vision/external-resources/visualpathways/LICENSE",
"sha256": "3972dc9744f6499f0f9b2dbf76696f2ae7ad8af9b23dde66d6af86c9dfb36986"
},
"pass_all": true,
"registered_file_verified": null,
"schema": "lfs-publication-verify-v1",
"scope": "downloaded payload bytes and streamed members; no extraction, no LFS-cache trust; catalog authenticity is caller responsibility",
"source_bytes": 1652696750,
"source_files": 6033
},
{
"archive_bytes": 901984,
"archive_sha256": "64d415a1ea9f5c2fe09005ce1c2bb27eb45510fad27ad3c39cf0001e6f525fd1",
"catalog_path": "/tmp/malecns-lfs-publication-20261001-bI1u07/repo/artifacts/public-resources-20261001/door-data/catalog.json",
"catalog_sha256": "6d0fcbe747dae7b38bcf335572b173e0112f6aabd10aa0040b84cfd81a8a390c",
"group": "door-data",
"historical_matched": 189,
"historical_missing": 0,
"license_evidence_verified": {
"bytes": 1042,
"path": "experiments/phase4l-olfactory-grounding/external-resources/DoOR.data/DESCRIPTION",
"sha256": "27db2a92eb651d67969de94959990d51668a5857a33c4750a3b6ef49c5880504"
},
"pass_all": true,
"registered_file_verified": null,
"schema": "lfs-publication-verify-v1",
"scope": "downloaded payload bytes and streamed members; no extraction, no LFS-cache trust; catalog authenticity is caller responsibility",
"source_bytes": 6534522,
"source_files": 189
},
{
"archive_bytes": 6828,
"archive_sha256": "4da352d80e4bf251dfec771935da93f5a9f3cef302c85e90991fafa14a072e2b",
"catalog_path": "/tmp/malecns-lfs-publication-20261001-bI1u07/repo/artifacts/public-resources-20261001/shiu-author-model-source/catalog.json",
"catalog_sha256": "95c69e6e9e07625a1f9b273db67aa9ab480c66bb3bbb273c39c430aeedb9b910",
"group": "shiu-author-model-source",
"historical_matched": 0,
"historical_missing": 4,
"license_evidence_verified": {
"bytes": 1085,
"path": "experiments/phase4ac-lif-semantics/external-resources/LICENSE",
"sha256": "3621f6d6476189190e2960fa43f11b275ba4eca848fdac50a1ba2925de6223e8"
},
"pass_all": true,
"registered_file_verified": null,
"schema": "lfs-publication-verify-v1",
"scope": "downloaded payload bytes and streamed members; no extraction, no LFS-cache trust; catalog authenticity is caller responsibility",
"source_bytes": 17567,
"source_files": 4
}
],
"historical_inventory": {
"all_current_members_historically_matched": false,
"bytes": 12285260,
"current_members_matched": 226849,
"current_members_with_nonfile_history": 0,
"current_members_without_regular_history": 10,
"historical_regular_files_outside_publication": 3390,
"missing_sample": [
"experiments/phase4aa-apl-pn-scope/external-resources/evidence-and-limitations.json",
"experiments/phase4aa-apl-pn-scope/external-resources/official-data-access.json",
"experiments/phase4aa-apl-pn-scope/external-resources/synaptic-partners/authorization-and-source.json",
"experiments/phase4aa-apl-pn-scope/external-resources/synaptic-partners/download-verification.json",
"experiments/phase4aa-apl-pn-scope/external-resources/synaptic-partners/syn-partners-male-cns-v1.0-minconf-0.5.feather",
"experiments/phase4aa-apl-pn-scope/external-resources/synaptic-partners/upstream-object.json",
"experiments/phase4ac-lif-semantics/external-resources/LICENSE",
"experiments/phase4ac-lif-semantics/external-resources/environment_full.yml",
"experiments/phase4ac-lif-semantics/external-resources/model.py",
"experiments/phase4ac-lif-semantics/external-resources/provenance.json"
],
"missing_sample_truncated": false,
"nonfile_records_skipped": 0,
"path": "/data/digital-drosophila/registry/snapshots/bootstrap-20260929/files.jsonl.gz",
"records": 230239,
"regular_file_records": 230239,
"scope": "Only publication members with regular-file history compared; not a full historical source-tree audit",
"sha256": "6f186112e6102f2f7969de7442fa7ee4432200a3678586cfc4e10c490b31bca4",
"status": "PARTIAL_CURRENT_MEMBER_COVERAGE"
},
"pack_summary": {
"bytes": 2183,
"path": "/tmp/malecns-lfs-publication-20261001-bI1u07/repo/artifacts/public-resources-20261001/pack-summary.json",
"sha256": "a097651a8f61009ec99a8bf6d1c3fcfaebd65fae16d6b0a14ebc2db841ae53cb"
},
"pass_all": true,
"payload_root": "/tmp/malecns-lfs-publication-20261001-bI1u07/repo/artifacts/public-resources-20261001",
"plan": {
"bytes": 7181,
"path": "/tmp/malecns-lfs-publication-20261001-bI1u07/repo/artifacts/public-resources-20261001/plan.json",
"sha256": "a20a61821c504e8315171a5fd14afbac011b934709af344045175bdda9fcba71"
},
"receipt": "/tmp/malecns-lfs-publication-20261001-bI1u07/repo/registry/publication-20261001/local-content-verification.json",
"schema": "lfs-publication-aggregate-verification-v1",
"scope": "Given payload bytes verified against plan/catalog/members and optional historical overlap; provenance/plan authenticity and fresh-clone transfer evidence are caller responsibilities",
"source_bytes": 24040546041,
"source_files": 226859,
"verified_utc": "2026-09-30T19:36:33.592474+00:00"
}
@@ -0,0 +1,123 @@
{
"disk_free_bytes": {
"/data/digital-drosophila": 645520629760,
"/tmp/malecns-lfs-publication-20261001-bI1u07/repo/registry/publication-20261001": 876361818112
},
"license_evidence": [
{
"bytes": 18657,
"group": "celltype-explorer",
"path": "/data/digital-drosophila/experiments/phase4f-retinotopic-vision/external-resources/celltype-explorer/LICENSE",
"sha256": "9ba9550ad48438d0836ddab3da480b3b69ffa0aac7b7878b5a0039e7ab429411"
},
{
"bytes": 35149,
"group": "visualpathways",
"path": "/data/digital-drosophila/experiments/phase4f-retinotopic-vision/external-resources/visualpathways/LICENSE",
"sha256": "3972dc9744f6499f0f9b2dbf76696f2ae7ad8af9b23dde66d6af86c9dfb36986"
},
{
"bytes": 1042,
"group": "door-data",
"path": "/data/digital-drosophila/experiments/phase4l-olfactory-grounding/external-resources/DoOR.data/DESCRIPTION",
"sha256": "27db2a92eb651d67969de94959990d51668a5857a33c4750a3b6ef49c5880504"
},
{
"bytes": 1085,
"group": "shiu-author-model-source",
"path": "/data/digital-drosophila/experiments/phase4ac-lif-semantics/external-resources/LICENSE",
"sha256": "3621f6d6476189190e2960fa43f11b275ba4eca848fdac50a1ba2925de6223e8"
}
],
"observed_utc": "2026-09-30T19:39:32.468575+00:00",
"original_checkout_branch": "codex/lif-author-semantics-lab",
"original_checkout_head": "09187e4d4d67badf1b418c5f211258ca5d8fb328",
"original_checkout_status": "",
"pass_all": true,
"phase4ar_registered_sources": [
{
"bytes": 5374,
"path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/PROTOCOL.md",
"sha256": "d61d2cb6a6bff31b5762edb78bb6406ac7d53b1e5eff30aad26eb0eea99fc313"
},
{
"bytes": 29087,
"path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/append_ledger.py",
"sha256": "1f981f61cd26950e7d4b5393be70b82753ea7e721c0bb75d05f8fd1e92a453fb"
},
{
"bytes": 7813,
"path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/normal_inputs.py",
"sha256": "df8f346b8fba54bb54f0b1a3f81c1511d54987684064094d0dfa46043be03e0d"
},
{
"bytes": 3317,
"path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/normal_io.py",
"sha256": "fe6eee89ae98392a28120b69a966d7967916be94376cb7494ef3ce77e63506e0"
},
{
"bytes": 37143,
"path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/normal_metrics.py",
"sha256": "82af63d8fadd5a6b67083f5e303ebe88674b76f9a2a8a8fff082b5c828483533"
},
{
"bytes": 11254,
"path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/normal_plan.py",
"sha256": "d1a6b840a547cb10b00df40f4d8b287867d09f355c9777e5f7a8b9489f5555ea"
},
{
"bytes": 15006,
"path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/normal_prepare.py",
"sha256": "9c8568a3d8c1049ada8737b71192952f72cdd27ccb3b470195e30d6a4b1cea68"
},
{
"bytes": 24699,
"path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/normal_runner.py",
"sha256": "41a0c71508d0b39c9488bddeaa4ac2a669312c632f16d5f9b5de24dff2eb14c1"
},
{
"bytes": 1534,
"path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/run_cpu_tests.py",
"sha256": "10ed189053731e1661284cf890ca3504a272af05b4531bdb3655b6e88cab6ff9"
},
{
"bytes": 3043,
"path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/run_normal.py",
"sha256": "707b6be41caef7af83aa68ea1f102f10c9a4ba81a22c564e54eddf63857e81f3"
},
{
"bytes": 15199,
"path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/test_append_ledger.py",
"sha256": "0f638c7b83117410d7fa074ec9f2ef865d9810171b73991c73a78414be581bc6"
},
{
"bytes": 4292,
"path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/test_normal_inputs.py",
"sha256": "7aea6a3bcbbf78c3b3d671b33b83cd4d992b2a8ae9c888e29cdf13f9fa420a45"
},
{
"bytes": 17862,
"path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/test_normal_metrics.py",
"sha256": "204586b30edbe57bb294c1d8f8ad309863e5bcb3baad32118955ac3d4adcdefb"
},
{
"bytes": 9184,
"path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/test_normal_plan.py",
"sha256": "790bca529c58e5214c19a6dcaf07511fca11c07fb9b05ca5d71d3f06160bbef4"
},
{
"bytes": 25798,
"path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/test_normal_runner.py",
"sha256": "ca9f69c731e56c50534d864d0ce780f286b5ab2d786a1a85fadb22dc09c0ccf5"
},
{
"bytes": 2076,
"path": "/data/digital-drosophila/models/male-cns-lif/normal-conditioning-v1/test_run_normal.py",
"sha256": "2ee3ad3c58c0dfd83aaaa4c4c9bd01e84eb245c61048c105ed8f3031db74cb40"
}
],
"registered_scientific_commit": "80aeb2093db655329774eff7b32fb6e417b8b40b",
"registration_sha256": "b63e6281b6505ca3b60acb467dd0b3fae274cf513dcbfd653649ba52e9ba9740",
"schema": "lfs-publication-source-guard-v1",
"scope": "16 registered Phase 4AR source files, registration, pinned upstream checkout and license evidence; not a scientific rerun or full historical checksum audit"
}
@@ -0,0 +1,15 @@
{
"schema": "lfs-publication-tool-tests-v1",
"observed_utc": "2026-09-30T19:40:34Z",
"environment": "isolated Linux publication checkout; system python3 -B, installed zstd",
"command": "python3 -B -m unittest discover -s tools/lfs-publication -p test_*.py",
"exit_code": 0,
"tests_run": 66,
"skipped": 0,
"failures": 0,
"errors": 0,
"reported_wall_seconds": 0.992,
"suites": {"archive": 18, "aggregate": 14, "download_wrapper": 12, "lan_adapter": 22},
"scope": "Archive suite includes real zstd integration. Transport/config failure tests use synthetic fixtures and mocks; actual LAN transfer is independently recorded in lan-roundtrip-smoke.json.",
"scientific_trials_executed": false
}