diff --git a/README.md b/README.md index 32c35c5..9849d44 100644 --- a/README.md +++ b/README.md @@ -203,6 +203,15 @@ In particular, the ignored Kimi raw inventory is not a quickstart dependency. Do create or download it to run the demonstration. Remote practice collectors are also unnecessary for normal offline verification and are never invoked implicitly. +## Real-world case studies + +- **Case Study 01 — Unsloth Gemma 4 E2B IT Q8_0 GGUF:** The export succeeded, and + both current artifact identities matched retained historical C1 size/SHA-256 + observations. OMIV also made + source-provenance and main/mmproj companion-binding observability gaps explicit. + Read the [case study](case-studies/unsloth-gemma4-e2b-it-q8/README.md) with its + methodology, claim registry, evidence index, results, and limitations. + ## Practice-profile limitations Practice profiles illustrate bounded evidence contracts; they are not endorsements, diff --git a/case-studies/unsloth-gemma4-e2b-it-q8/README.md b/case-studies/unsloth-gemma4-e2b-it-q8/README.md new file mode 100644 index 0000000..d855b5b --- /dev/null +++ b/case-studies/unsloth-gemma4-e2b-it-q8/README.md @@ -0,0 +1,125 @@ +# Case Study 01 — Validating an Unsloth Q8_0 GGUF Export with OMIV + +This case study records an independent structural validation and +provenance-observability analysis of an Unsloth export. It separates the +successful export result from the evidence gaps observed around the exported +artifacts. + +## Why this case + +This is OMIV's first public ecosystem case study. This case began with a [public +technical exchange](https://x.com/chen_linzh96875/status/2087210507782807856) +with Daniel Han. We are grateful to Daniel for suggesting Gemma 4 E2B IT as the +first reproducible case and for giving OMIV the opportunity to test a real Unsloth +export path. OMIV then independently executed and analyzed the export. The +relationship described here is limited to that public technical exchange. +(P1-C001; P1-AUTHORIZED-CONTEXT) + +The fixed source was `google/gemma-4-E2B-it` at revision +`3e22461f65e89153144f8adb70e3b8c2cc9845a7`: 9 files, 10,278,849,571 bytes, +with source-manifest SHA-256 +`b73a8a4dd3aa11c51f5dcc06e13dbdecbbcd1f7eaa55518febe8ffeaf10d8a75`. +The canonical source manifest remained unchanged during export. (P1-C002; +A1-LINEAGE) + +## Result in brief + +The export itself succeeded. Retained execution evidence records one successful +Q8_0 export and verified artifact custody. (P1-C003; A1-EXPORT-HISTORY) + +The positive reproducibility result is deliberately narrower: the current Q8_0 and +mmproj file sizes and SHA-256 values match retained historical C1 observations. The +historical C1 bytes are unavailable, so this is `HISTORICAL_C1_MATCH`, not a new +direct C1/C2 file comparison. (P1-C009; A1-REPRODUCIBILITY) + +| Artifact | Format | Bytes | SHA-256 | Metadata | Tensors | Type distribution | Evidence | +|---|---:|---:|---|---:|---:|---|---| +| Main Q8_0 | GGUF v3 | 4,954,594,880 | `fbf3f93d603e92dbcf00742163cc3018b9bf8fbfbcf87158cc836e5ed0bc9455` | 51 | 601 | 318 Q8_0; 283 F32 | P1-C004, P1-C005; A1-INPUT-CUSTODY, A1-MAIN-GGUF | +| BF16 mmproj | GGUF v3 | 986,833,792 | `e00089a95d2dc85c71f249bd83760e348f721757318802ddb889c65ec64eae3c` | 39 | 1,411 | 247 BF16; 1,164 F32 | P1-C006, P1-C007; A1-INPUT-CUSTODY, A1-MMPROJ-GGUF | + +All parsed tensor byte ranges were inside their files and non-overlapping. No tensor +payload values were interpreted for semantic or numerical evaluation. (P1-C008, +P1-C015; A1-MAIN-GGUF, A1-MMPROJ-GGUF, A1-LIMITATIONS) + +## Methodology + +The analysis was completely offline. It verified custody hashes and retained +evidence seals, parsed GGUF headers, metadata, and tensor descriptors, validated +descriptor ranges, compared deterministic OMIV-native reports, assembled +source/work-copy/target lineage, and evaluated the contextual association between +the main GGUF and mmproj. It did not load a model, run inference, invoke an +exporter, use a GPU, or access tensor payload values for fidelity testing. (P1-C014, +P1-C020; A1-NATIVE-REPEATABILITY, A1-CAPABILITY-COVERAGE, A1-LIMITATIONS) + +See [methodology.md](methodology.md) for the concise procedure, +[results.json](results.json) for the machine-readable result, and +[evidence-index.json](evidence-index.json) for the public evidence definitions and +commitments. + +## Source → work copy → GGUF lineage + +The retained lineage identifies the fixed source repository, revision, size, file +count, and manifest. During export, only `tokenizer_config.json` changed in the +tracked work-copy configuration set; `chat_template.jinja`, `config.json`, +`generation_config.json`, and `processor_config.json` remained unchanged. +(P1-C002, P1-C013; A1-LINEAGE) + +The semantic JSON-pointer comparison of `tokenizer_config.json` contains 195 +changes: 181 additions and 14 removals. This is a verified lineage event. It is not, +by itself, evidence of a bug. (P1-C013; A1-LINEAGE) + +Neither target GGUF metadata inventory contained the exact source revision. That is +a provenance-observability gap: it makes the exported files harder to tie back to +the fixed source without sidecar evidence, but it is not proof that the model +weights are incorrect. (P1-C010, P1-C017; A1-LINEAGE) + +## Main/mmproj association + +The main GGUF and mmproj have a moderate contextual association: retained execution +evidence places them in the same export, and their exporter and +model-family/projector-role metadata are compatible by inference. (P1-C011; +A1-ASSOCIATION) + +No cryptographic or format-native cross-reference binds the files. Their shared +name, `Work Copy`, has low identifying power. This is an artifact-association gap, +not proof that the pair is incompatible. (P1-C012, P1-C018; A1-ASSOCIATION) + +## What OMIV made visible + +- Neither target GGUF metadata inventory contained the exact source revision, + exposing a provenance-observability gap. (P1-C010, P1-C017; A1-LINEAGE) +- The main/mmproj relationship was contextual rather than cryptographically or + format-natively bound. (P1-C011, P1-C012; A1-ASSOCIATION) +- A specific work-copy mutation was measured: 195 JSON-pointer changes in + `tokenizer_config.json`, without labeling that mutation a bug. (P1-C013; + A1-LINEAGE) +- Current OMIV coverage is strong for custody, GGUF structure, scalar metadata, + tensor inventories, structural comparisons, and deterministic reporting, while + lineage assembly, large-array disclosure, companion binding, and fidelity + evaluation need additional product support. (P1-C019; A1-CAPABILITY-COVERAGE) + +These observations do not identify which component originated the behavior. In +particular, missing revision metadata and absent main/mmproj binding must not be +attributed solely to Unsloth, and the tokenizer configuration mutation must not be +attributed to a specific component without root-cause evidence. This study did not +determine whether those observations originated in Unsloth, llama.cpp tooling, +GGUF conventions, llm-compressor, or the surrounding export workflow. (P1-C021; +A1-LINEAGE, A1-ASSOCIATION, A1-LIMITATIONS) + +## What this does not prove + +- Semantic fidelity and numerical quantization fidelity were not evaluated. + (P1-C015; A1-LIMITATIONS) +- Q8_0 tensor-type classification is not a quality result. (P1-C015; + A1-MAIN-GGUF, A1-LIMITATIONS) +- Structural validation does not establish runtime compatibility. (P1-C016; + A1-LIMITATIONS) +- A successful export does not by itself establish that every runtime will load or + correctly execute the artifacts. (P1-C016; A1-EXPORT-HISTORY, A1-LIMITATIONS) +- `HISTORICAL_C1_MATCH` relies on retained C1 size/SHA-256 observations; historical + C1 bytes were unavailable and no new direct C1/C2 file comparison occurred. + (P1-C009; A1-REPRODUCIBILITY) + +Additional boundaries are in [limitations.md](limitations.md). The complete claim +registry is [claims.json](claims.json), and every evidence ID it uses is defined in +[evidence-index.json](evidence-index.json). diff --git a/case-studies/unsloth-gemma4-e2b-it-q8/SHA256SUMS b/case-studies/unsloth-gemma4-e2b-it-q8/SHA256SUMS new file mode 100644 index 0000000..c65b950 --- /dev/null +++ b/case-studies/unsloth-gemma4-e2b-it-q8/SHA256SUMS @@ -0,0 +1,6 @@ +c6ab7f2b0897bd752e58592dbbb8e76a677309dfde3c48f801c3b8d1b0622725 README.md +046c7d74e2b88d8967715a29bdb23079f0676e366707e0125580a92b241b5e42 claims.json +9c3eb687760259f3c8de793adf9eb4f4989dfcff9c4adbdb4d87ca481956c2e3 evidence-index.json +8049718bc6e5661fd9ec87bf8eab10e663d01b64307bdbf10adf93d966dc0969 limitations.md +00124b103f0eb9ea000d6133aaa3ff03dad31c09b87b72dc3b36ff910303f40e methodology.md +349a0e3381c5f667775fd250bfadc7841a33609f7b59f7970ab5fc815d426b32 results.json diff --git a/case-studies/unsloth-gemma4-e2b-it-q8/claims.json b/case-studies/unsloth-gemma4-e2b-it-q8/claims.json new file mode 100644 index 0000000..b8a6bb1 --- /dev/null +++ b/case-studies/unsloth-gemma4-e2b-it-q8/claims.json @@ -0,0 +1,133 @@ +{ + "schema": "omiv.public-evidence.claims.v1", + "version": "1.0.0", + "study_id": "OMIV-CS-01", + "claims": [ + { + "claim_id": "P1-C001", + "claim_class": "AUTHORIZED_CONTEXT", + "statement": "This is OMIV's first public ecosystem case study; it followed a public technical exchange in which Daniel Han suggested Gemma 4 E2B IT as the first reproducible case, and OMIV independently executed and analyzed it.", + "evidence_ids": ["P1-AUTHORIZED-CONTEXT"] + }, + { + "claim_id": "P1-C002", + "claim_class": "RETAINED_HISTORICAL_OBSERVATION", + "statement": "The fixed source was google/gemma-4-E2B-it at revision 3e22461f65e89153144f8adb70e3b8c2cc9845a7 with 9 files, 10278849571 bytes, and manifest b73a8a4dd3aa11c51f5dcc06e13dbdecbbcd1f7eaa55518febe8ffeaf10d8a75; the canonical source manifest remained unchanged during export.", + "evidence_ids": ["A1-LINEAGE"] + }, + { + "claim_id": "P1-C003", + "claim_class": "RETAINED_HISTORICAL_OBSERVATION", + "statement": "Retained execution evidence records one successful Q8_0 export and verified local artifact custody.", + "evidence_ids": ["A1-EXPORT-HISTORY"] + }, + { + "claim_id": "P1-C004", + "claim_class": "DIRECTLY_VERIFIED_FACT", + "statement": "The main artifact is a regular 4954594880-byte file with SHA-256 fbf3f93d603e92dbcf00742163cc3018b9bf8fbfbcf87158cc836e5ed0bc9455.", + "evidence_ids": ["A1-INPUT-CUSTODY", "A1-MAIN-GGUF"] + }, + { + "claim_id": "P1-C005", + "claim_class": "DIRECTLY_VERIFIED_FACT", + "statement": "The main artifact is GGUF v3 with 51 metadata entries and 601 tensors: 318 Q8_0 and 283 F32.", + "evidence_ids": ["A1-MAIN-GGUF"] + }, + { + "claim_id": "P1-C006", + "claim_class": "DIRECTLY_VERIFIED_FACT", + "statement": "The mmproj artifact is a regular 986833792-byte file with SHA-256 e00089a95d2dc85c71f249bd83760e348f721757318802ddb889c65ec64eae3c.", + "evidence_ids": ["A1-INPUT-CUSTODY", "A1-MMPROJ-GGUF"] + }, + { + "claim_id": "P1-C007", + "claim_class": "DIRECTLY_VERIFIED_FACT", + "statement": "The mmproj artifact is GGUF v3 with 39 metadata entries and 1411 tensors: 247 BF16 and 1164 F32.", + "evidence_ids": ["A1-MMPROJ-GGUF"] + }, + { + "claim_id": "P1-C008", + "claim_class": "DIRECTLY_VERIFIED_FACT", + "statement": "All parsed tensor byte ranges were within their files and non-overlapping.", + "evidence_ids": ["A1-MAIN-GGUF", "A1-MMPROJ-GGUF"] + }, + { + "claim_id": "P1-C009", + "claim_class": "RETAINED_HISTORICAL_OBSERVATION_WITH_LIMITATION", + "statement": "Current artifact size and SHA-256 identities match retained historical C1 observations, preserving HISTORICAL_C1_MATCH; historical C1 bytes are unavailable, and no new direct C1/C2 file comparison was performed.", + "evidence_ids": ["A1-REPRODUCIBILITY"] + }, + { + "claim_id": "P1-C010", + "claim_class": "DIRECTLY_VERIFIED_FACT", + "statement": "Neither target GGUF metadata inventory contained the exact source revision.", + "evidence_ids": ["A1-LINEAGE"] + }, + { + "claim_id": "P1-C011", + "claim_class": "INFERENCE", + "statement": "The main and mmproj artifacts have a moderate contextual association based on shared export context and compatible exporter and model-family/projector-role metadata.", + "evidence_ids": ["A1-ASSOCIATION"] + }, + { + "claim_id": "P1-C012", + "claim_class": "DIRECTLY_VERIFIED_LIMITATION", + "statement": "No cryptographic or format-native main/mmproj binding was observed, and the shared name Work Copy has low identifying power.", + "evidence_ids": ["A1-ASSOCIATION"] + }, + { + "claim_id": "P1-C013", + "claim_class": "DIRECTLY_VERIFIED_LINEAGE_EVENT", + "statement": "The tokenizer_config work-copy diff contains 195 JSON-pointer changes: 181 additions and 14 removals; this lineage event is not automatically a bug.", + "evidence_ids": ["A1-LINEAGE"] + }, + { + "claim_id": "P1-C014", + "claim_class": "DIRECTLY_VERIFIED_FACT", + "statement": "Every repeated deterministic OMIV-native output used in A1 was byte-identical across the paired runs.", + "evidence_ids": ["A1-NATIVE-REPEATABILITY"] + }, + { + "claim_id": "P1-C015", + "claim_class": "EXPLICIT_LIMITATION", + "statement": "Semantic fidelity and numerical quantization fidelity were not evaluated, and Q8_0 type classification is not a quality result.", + "evidence_ids": ["A1-CAPABILITY-COVERAGE", "A1-LIMITATIONS"] + }, + { + "claim_id": "P1-C016", + "claim_class": "EXPLICIT_LIMITATION", + "statement": "Structural validation and export success do not by themselves establish runtime compatibility.", + "evidence_ids": ["A1-CAPABILITY-COVERAGE", "A1-LIMITATIONS"] + }, + { + "claim_id": "P1-C017", + "claim_class": "EVIDENCE_BOUNDARY_INTERPRETATION", + "statement": "Missing source-revision metadata is a provenance-observability gap, not proof of incorrect model weights.", + "evidence_ids": ["A1-LINEAGE", "A1-LIMITATIONS"] + }, + { + "claim_id": "P1-C018", + "claim_class": "EVIDENCE_BOUNDARY_INTERPRETATION", + "statement": "Weak main/mmproj binding is an artifact-association gap, not proof of incompatibility.", + "evidence_ids": ["A1-ASSOCIATION", "A1-LIMITATIONS"] + }, + { + "claim_id": "P1-C019", + "claim_class": "PRODUCT_COVERAGE_ASSESSMENT", + "statement": "A1 found complete native coverage for custody, core GGUF structure, scalar metadata, tensor inventories, structural comparison, and deterministic reporting, with partial, glue-only, retained-evidence-only, or unsupported coverage for several lineage, binding, and fidelity questions.", + "evidence_ids": ["A1-CAPABILITY-COVERAGE"] + }, + { + "claim_id": "P1-C020", + "claim_class": "DIRECTLY_VERIFIED_METHOD_BOUNDARY", + "statement": "A1 was completely offline and did not load a model, run inference, invoke an exporter, use a GPU, or evaluate tensor payload values for semantic or numerical fidelity.", + "evidence_ids": ["A1-CAPABILITY-COVERAGE", "A1-LIMITATIONS"] + }, + { + "claim_id": "P1-C021", + "claim_class": "EXPLICIT_ATTRIBUTION_BOUNDARY", + "statement": "This study did not determine whether the observed provenance, association, or tokenizer-lineage behavior originated in Unsloth, llama.cpp tooling, GGUF conventions, llm-compressor, or the surrounding export workflow.", + "evidence_ids": ["A1-LINEAGE", "A1-ASSOCIATION", "A1-LIMITATIONS"] + } + ] +} diff --git a/case-studies/unsloth-gemma4-e2b-it-q8/evidence-index.json b/case-studies/unsloth-gemma4-e2b-it-q8/evidence-index.json new file mode 100644 index 0000000..40b08ff --- /dev/null +++ b/case-studies/unsloth-gemma4-e2b-it-q8/evidence-index.json @@ -0,0 +1,110 @@ +{ + "schema": "omiv.public-evidence.index.v1", + "version": "1.0.0", + "study_id": "OMIV-CS-01", + "commitment_definition": "SHA-256 of the complete canonical private A1 source record bytes; commitments disclose no source path or JSON pointer.", + "evidence": [ + { + "evidence_id": "P1-AUTHORIZED-CONTEXT", + "evidence_class": "USER_AUTHORIZED_PUBLIC_CONTEXT", + "description": "Public relationship boundary and authorization for describing the technical exchange.", + "public_disclosure_level": "PUBLIC_SUMMARY_ONLY", + "supporting_public_claim_ids": ["P1-C001"], + "public_derivative_files": ["README.md", "claims.json"], + "public_source_urls": [ + "https://x.com/chen_linzh96875/status/2087210507782807856" + ], + "commitment_sha256": "COMMITMENT_NOT_PUBLICLY_AVAILABLE" + }, + { + "evidence_id": "A1-INPUT-CUSTODY", + "evidence_class": "DIRECT_ARTIFACT_CUSTODY", + "description": "Pre- and post-analysis file identity observations for both retained GGUF artifacts.", + "public_disclosure_level": "PUBLIC_SUMMARY_AND_COMMITMENT", + "supporting_public_claim_ids": ["P1-C004", "P1-C006"], + "public_derivative_files": ["README.md", "methodology.md", "results.json", "claims.json"], + "commitment_sha256": "6f80cd55d43c8d4dad8038fb02d45d25986eca7b39d00c20c4a23aed7eaf28b2" + }, + { + "evidence_id": "A1-MAIN-GGUF", + "evidence_class": "DIRECT_STRUCTURAL_INVENTORY", + "description": "Complete private structural inventory and range validation for the main GGUF.", + "public_disclosure_level": "PUBLIC_SUMMARY_AND_COMMITMENT", + "supporting_public_claim_ids": ["P1-C004", "P1-C005", "P1-C008"], + "public_derivative_files": ["README.md", "methodology.md", "results.json", "claims.json", "limitations.md"], + "commitment_sha256": "c3d062a991190fefbdbd7fd15f40f3f126ef1e0a444aa206494ee4a400a510b2" + }, + { + "evidence_id": "A1-MMPROJ-GGUF", + "evidence_class": "DIRECT_STRUCTURAL_INVENTORY", + "description": "Complete private structural inventory and range validation for the companion mmproj GGUF.", + "public_disclosure_level": "PUBLIC_SUMMARY_AND_COMMITMENT", + "supporting_public_claim_ids": ["P1-C006", "P1-C007", "P1-C008"], + "public_derivative_files": ["README.md", "methodology.md", "results.json", "claims.json", "limitations.md"], + "commitment_sha256": "04fc4e7850abc98252843e469c790ca6008453dde0d6c420d089b72de7d2fd10" + }, + { + "evidence_id": "A1-LINEAGE", + "evidence_class": "RETAINED_AND_DIRECT_LINEAGE_ANALYSIS", + "description": "Source, work-copy, target-metadata, and tokenizer-mutation lineage analysis.", + "public_disclosure_level": "PUBLIC_SUMMARY_AND_COMMITMENT", + "supporting_public_claim_ids": ["P1-C002", "P1-C010", "P1-C013", "P1-C017", "P1-C021"], + "public_derivative_files": ["README.md", "methodology.md", "results.json", "claims.json", "limitations.md"], + "commitment_sha256": "da094ac3921bca5a316bd193ce95d0c7187b18fb88b1c7df45467cb05c0aa6cd" + }, + { + "evidence_id": "A1-REPRODUCIBILITY", + "evidence_class": "RETAINED_HISTORICAL_COMPARISON", + "description": "Comparison of current artifact identities with retained historical C1 size and SHA-256 observations.", + "public_disclosure_level": "PUBLIC_SUMMARY_AND_COMMITMENT", + "supporting_public_claim_ids": ["P1-C009"], + "public_derivative_files": ["README.md", "methodology.md", "results.json", "claims.json", "limitations.md"], + "commitment_sha256": "eede14c3359ce6417affc25d6dd3ead09226d4e5b6fc94c9284ac99298911fad" + }, + { + "evidence_id": "A1-ASSOCIATION", + "evidence_class": "DIRECT_AND_INFERRED_COMPANION_ASSOCIATION", + "description": "Contextual, metadata, cryptographic, and format-native association analysis for the main GGUF and mmproj.", + "public_disclosure_level": "PUBLIC_SUMMARY_AND_COMMITMENT", + "supporting_public_claim_ids": ["P1-C011", "P1-C012", "P1-C018", "P1-C021"], + "public_derivative_files": ["README.md", "methodology.md", "results.json", "claims.json", "limitations.md"], + "commitment_sha256": "68ddd6bd95c070933cf579e0317b21fdc0d42c9d39c830bedc6057ee9b9eebf4" + }, + { + "evidence_id": "A1-EXPORT-HISTORY", + "evidence_class": "RETAINED_EXECUTION_HISTORY", + "description": "Retained observation that artifact production succeeded and custody was verified.", + "public_disclosure_level": "PUBLIC_SUMMARY_AND_COMMITMENT", + "supporting_public_claim_ids": ["P1-C003"], + "public_derivative_files": ["README.md", "methodology.md", "results.json", "claims.json", "limitations.md"], + "commitment_sha256": "9f1f3647d90de96b5f5f43a61c0821484bf14e264b75b35307991da4f201d3af" + }, + { + "evidence_id": "A1-NATIVE-REPEATABILITY", + "evidence_class": "DIRECT_REPORT_REPEATABILITY", + "description": "Paired deterministic OMIV-native operations and their byte-identical report comparison.", + "public_disclosure_level": "PUBLIC_SUMMARY_AND_COMMITMENT", + "supporting_public_claim_ids": ["P1-C014"], + "public_derivative_files": ["README.md", "methodology.md", "claims.json"], + "commitment_sha256": "eb417040c668549bce8660a0ce58c06e48de36e8c5c83b206dce889c31c098dc" + }, + { + "evidence_id": "A1-CAPABILITY-COVERAGE", + "evidence_class": "PRODUCT_CAPABILITY_ASSESSMENT", + "description": "Coverage classification for native, partial, retained-evidence, glue, and unsupported analysis capabilities.", + "public_disclosure_level": "PUBLIC_SUMMARY_AND_COMMITMENT", + "supporting_public_claim_ids": ["P1-C015", "P1-C016", "P1-C019", "P1-C020"], + "public_derivative_files": ["README.md", "methodology.md", "results.json", "claims.json", "limitations.md"], + "commitment_sha256": "f6fa066365e381017d46cbb5bb72632d713c2ef4adcf4f5f531bd71fcba5e57d" + }, + { + "evidence_id": "A1-LIMITATIONS", + "evidence_class": "EXPLICIT_ANALYSIS_LIMITATIONS", + "description": "Canonical limitations for analysis scope, retained evidence, fidelity, runtime, lineage, and companion binding.", + "public_disclosure_level": "PUBLIC_SUMMARY_AND_COMMITMENT", + "supporting_public_claim_ids": ["P1-C015", "P1-C016", "P1-C017", "P1-C018", "P1-C020", "P1-C021"], + "public_derivative_files": ["README.md", "methodology.md", "results.json", "claims.json", "limitations.md"], + "commitment_sha256": "ecaffccc730612b65e585db358d3eea49e65fd8863f0e03750e7f1728301f38f" + } + ] +} diff --git a/case-studies/unsloth-gemma4-e2b-it-q8/limitations.md b/case-studies/unsloth-gemma4-e2b-it-q8/limitations.md new file mode 100644 index 0000000..a72909a --- /dev/null +++ b/case-studies/unsloth-gemma4-e2b-it-q8/limitations.md @@ -0,0 +1,43 @@ +# Limitations + +- Historical reproducibility is classified as `HISTORICAL_C1_MATCH` because current + file sizes and SHA-256 values match retained C1 observations. Historical C1 bytes + are unavailable, and no new direct C1/C2 file comparison was performed. + (P1-C009; A1-REPRODUCIBILITY) +- Published SHA-256 commitments bind the identity of retained private evidence + records, but commitments alone do not make undisclosed private records + independently reviewable. The public case study provides claim summaries and + commitments; it is not a standalone exporter reproduction recipe. This public + availability boundary is not an artifact-production failure. +- No model inference, model service, tokenizer execution, template rendering, or + semantic probe was performed. (P1-C020; A1-LIMITATIONS) +- Semantic fidelity and numerical quantization fidelity were not evaluated. File + hashes and structural inventories do not answer those questions. (P1-C015; + A1-CAPABILITY-COVERAGE, A1-LIMITATIONS) +- Q8_0 identifies the observed tensor representation; it is not a model-quality + result. (P1-C005, P1-C015; A1-MAIN-GGUF, A1-LIMITATIONS) +- The structural checks establish header/count consistency and tensor-range bounds + and non-overlap. They do not establish runtime compatibility. (P1-C008, P1-C016; + A1-MAIN-GGUF, A1-MMPROJ-GGUF, A1-LIMITATIONS) +- Neither target GGUF metadata inventory contained the exact source revision. This + is a provenance-observability gap, not evidence that model weights are incorrect, + and it must not be attributed solely to Unsloth. (P1-C010, P1-C017, P1-C021; + A1-LINEAGE, A1-LIMITATIONS) +- The main/mmproj association is contextual and moderate. No cryptographic or + format-native binding was observed; this is not evidence that the artifacts are + incompatible, and it must not be attributed solely to Unsloth. (P1-C011, + P1-C012, P1-C018, P1-C021; A1-ASSOCIATION, A1-LIMITATIONS) +- The 195-change tokenizer configuration result is a semantic JSON-pointer diff + linked by retained raw size/SHA-256 observations; original raw formatting is not + reconstructible from the retained parsed wrappers. The mutation is not + automatically a defect and cannot be attributed to a specific component without + root-cause evidence. (P1-C013, P1-C021; A1-LINEAGE, A1-LIMITATIONS) +- This study did not determine whether the observed provenance, association, or + tokenizer-lineage behavior originated in Unsloth, llama.cpp tooling, GGUF + conventions, llm-compressor, or the surrounding export workflow. (P1-C021; + A1-LINEAGE, A1-ASSOCIATION, A1-LIMITATIONS) +- Full source configuration and tokenizer assets were not in A1 custody. Missing + values were treated as unavailable or not observable rather than inferred from + target metadata. (P1-C019; A1-CAPABILITY-COVERAGE, A1-LIMITATIONS) +- A successful export does not by itself establish correct behavior in a particular + runtime. (P1-C003, P1-C016; A1-EXPORT-HISTORY, A1-LIMITATIONS) diff --git a/case-studies/unsloth-gemma4-e2b-it-q8/methodology.md b/case-studies/unsloth-gemma4-e2b-it-q8/methodology.md new file mode 100644 index 0000000..cc372ac --- /dev/null +++ b/case-studies/unsloth-gemma4-e2b-it-q8/methodology.md @@ -0,0 +1,77 @@ +# Methodology + +## Scope + +This study evaluated artifact custody, GGUF structure, source/work-copy/target +lineage, historical reproducibility evidence, and the association between a main +GGUF and its mmproj. The analysis was offline and did not load a model, run +inference, invoke an exporter, use a GPU, or evaluate tensor payload values for +semantic or numerical fidelity. (P1-C020; A1-CAPABILITY-COVERAGE, A1-LIMITATIONS) + +## Evidence model + +Public claims use stable claim IDs and evidence IDs. Every evidence ID is defined +in [evidence-index.json](evidence-index.json), including its disclosure class, +supporting public claims, derivative files, and a SHA-256 commitment where a safe +canonical private A1 source record was available. Public files disclose only +hashes, counts, classifications, and short derived summaries. They do not contain +model weights, tokenizer vocabulary, full configuration files, full chat templates, +internal transcripts, credentials, private infrastructure details, or local paths. + +Published SHA-256 commitments bind the identity of retained private evidence +records. Commitments alone do not make undisclosed private records independently +reviewable. The public case study provides claim summaries and commitments, but it +is not a standalone exporter reproduction recipe. This availability boundary does +not change the reported artifact-production result. + +The public result distinguishes: + +- directly verified current artifact facts; +- retained historical observations; +- contextual inferences; +- explicit limitations and attribution boundaries. + +## Procedure + +1. **Custody.** Recompute the size and SHA-256 identity of each retained GGUF and + compare it with the authorized A1 custody record. (P1-C004, P1-C006; + A1-INPUT-CUSTODY) +2. **Structure.** Parse GGUF headers, typed metadata, and tensor descriptors; + confirm declared counts; validate that tensor ranges are inside the file and do + not overlap. Tensor payload values are not interpreted. (P1-C005, P1-C007, + P1-C008; A1-MAIN-GGUF, A1-MMPROJ-GGUF) +3. **Repeatability of reporting.** Run the deterministic OMIV-native inventory, + manifest, self-comparison, and structural-contrast operations twice and compare + the resulting reports byte-for-byte. (P1-C014; A1-NATIVE-REPEATABILITY) +4. **Lineage.** Compare retained source and work-copy manifests with complete + target metadata inventories. Report the `tokenizer_config.json` JSON-pointer + change counts without disclosing its full content or vocabulary. (P1-C002, + P1-C010, P1-C013; A1-LINEAGE) +5. **Reproducibility classification.** Compare current artifact size/SHA-256 + identities with retained historical C1 observations. Because historical C1 + bytes are unavailable, do not describe this as a new direct C1/C2 file + comparison. (P1-C009; A1-REPRODUCIBILITY) +6. **Companion association.** Evaluate shared execution context, exporter + metadata, model-family/projector-role compatibility, explicit revision linkage, + cryptographic identifiers, and format-native cross-references. (P1-C011, + P1-C012; A1-ASSOCIATION) + +## Interpretation rules + +- Export success is reported separately from evidence gaps. (P1-C003, P1-C017, + P1-C018; A1-EXPORT-HISTORY, A1-LINEAGE, A1-ASSOCIATION) +- Missing revision metadata is a provenance-observability gap, not evidence of + incorrect model weights, and must not be attributed solely to Unsloth. (P1-C017, + P1-C021; A1-LINEAGE, A1-LIMITATIONS) +- Weak companion binding is an artifact-association gap, not evidence of + incompatibility, and must not be attributed solely to Unsloth. (P1-C018, + P1-C021; A1-ASSOCIATION, A1-LIMITATIONS) +- Tokenizer/config mutation is a lineage event and is not automatically a defect; + no originating component is identified without root-cause evidence. (P1-C013, + P1-C021; A1-LINEAGE, A1-LIMITATIONS) +- Structure, representation, numerical fidelity, semantic fidelity, and runtime + compatibility are separate questions. (P1-C015, P1-C016; + A1-CAPABILITY-COVERAGE, A1-LIMITATIONS) + +This is a case-study evidence workflow, not a representation of a long operational +harness as normal OMIV user experience. diff --git a/case-studies/unsloth-gemma4-e2b-it-q8/results.json b/case-studies/unsloth-gemma4-e2b-it-q8/results.json new file mode 100644 index 0000000..3929e8b --- /dev/null +++ b/case-studies/unsloth-gemma4-e2b-it-q8/results.json @@ -0,0 +1,152 @@ +{ + "schema": "omiv.public-case-study.results.v1", + "version": "1.0.0", + "study_id": "OMIV-CS-01", + "title": "Case Study 01 — Validating an Unsloth Q8_0 GGUF Export with OMIV", + "source": { + "repository": "google/gemma-4-E2B-it", + "revision": "3e22461f65e89153144f8adb70e3b8c2cc9845a7", + "file_count": 9, + "total_bytes": 10278849571, + "manifest_sha256": "b73a8a4dd3aa11c51f5dcc06e13dbdecbbcd1f7eaa55518febe8ffeaf10d8a75", + "manifest_unchanged_during_export": true, + "claim_ids": [ + "P1-C002" + ], + "evidence_ids": [ + "A1-LINEAGE" + ] + }, + "export": { + "status": "SUCCESS_RETAINED_HISTORICAL_OBSERVATION", + "claim_ids": [ + "P1-C003" + ], + "evidence_ids": [ + "A1-EXPORT-HISTORY" + ] + }, + "artifacts": [ + { + "role": "main", + "format": "GGUF", + "format_version": 3, + "size_bytes": 4954594880, + "sha256": "fbf3f93d603e92dbcf00742163cc3018b9bf8fbfbcf87158cc836e5ed0bc9455", + "metadata_entries": 51, + "tensor_count": 601, + "tensor_type_counts": { + "F32": 283, + "Q8_0": 318 + }, + "tensor_ranges_within_file": true, + "tensor_ranges_non_overlapping": true, + "claim_ids": [ + "P1-C004", + "P1-C005", + "P1-C008" + ], + "evidence_ids": [ + "A1-INPUT-CUSTODY", + "A1-MAIN-GGUF" + ] + }, + { + "role": "mmproj", + "format": "GGUF", + "format_version": 3, + "size_bytes": 986833792, + "sha256": "e00089a95d2dc85c71f249bd83760e348f721757318802ddb889c65ec64eae3c", + "metadata_entries": 39, + "tensor_count": 1411, + "tensor_type_counts": { + "BF16": 247, + "F32": 1164 + }, + "tensor_ranges_within_file": true, + "tensor_ranges_non_overlapping": true, + "claim_ids": [ + "P1-C006", + "P1-C007", + "P1-C008" + ], + "evidence_ids": [ + "A1-INPUT-CUSTODY", + "A1-MMPROJ-GGUF" + ] + } + ], + "historical_reproducibility": { + "classification": "HISTORICAL_C1_MATCH", + "basis": "RETAINED_C1_SIZE_AND_SHA256_OBSERVATIONS_VS_CURRENT_LOCAL_BYTES", + "historical_c1_bytes_retained": false, + "new_direct_c1_c2_file_comparison": false, + "claim_ids": [ + "P1-C009" + ], + "evidence_ids": [ + "A1-REPRODUCIBILITY" + ] + }, + "lineage": { + "exact_source_revision_present_in_main_metadata": false, + "exact_source_revision_present_in_mmproj_metadata": false, + "tokenizer_config_json_pointer_changes": 195, + "tokenizer_config_additions": 181, + "tokenizer_config_removals": 14, + "claim_ids": [ + "P1-C010", + "P1-C013", + "P1-C017" + ], + "evidence_ids": [ + "A1-LINEAGE" + ] + }, + "main_mmproj_association": { + "strength": "MODERATE_CONTEXTUAL_NOT_CRYPTOGRAPHIC", + "cryptographic_binding_observed": false, + "format_native_binding_observed": false, + "shared_name": "Work Copy", + "shared_name_identifying_power": "LOW", + "claim_ids": [ + "P1-C011", + "P1-C012", + "P1-C018" + ], + "evidence_ids": [ + "A1-ASSOCIATION" + ] + }, + "attribution_boundary": { + "originating_component_determined": false, + "candidate_components": [ + "Unsloth", + "llama.cpp tooling", + "GGUF conventions", + "llm-compressor", + "surrounding export workflow" + ], + "claim_ids": [ + "P1-C021" + ], + "evidence_ids": [ + "A1-LINEAGE", + "A1-ASSOCIATION", + "A1-LIMITATIONS" + ] + }, + "fidelity": { + "semantic": "NOT_EVALUATED", + "numerical": "NOT_EVALUATED", + "runtime_compatibility": "NOT_EVALUATED", + "claim_ids": [ + "P1-C015", + "P1-C016" + ], + "evidence_ids": [ + "A1-CAPABILITY-COVERAGE", + "A1-LIMITATIONS" + ] + } +}