|
21 | 21 |
|
22 | 22 | import brotli |
23 | 23 |
|
24 | | -from migrate_content_to_dictionary_brotli import migrate |
| 24 | +from migrate_content_to_dictionary_brotli import load_base_rows, migrate |
25 | 25 | from populate_db import CHUNK_SIZE, DictionaryCompressor, load_dictionary |
26 | 26 | import remint_dictionary |
27 | 27 | from remint_dictionary import remint |
@@ -242,6 +242,8 @@ def test_main_reports_how_much_smaller(self): |
242 | 242 | printed = self.run_main("--sample-seed", "999", "--seeds", "1") |
243 | 243 | stored_after = self.stored_bytes() |
244 | 244 | self.assertLess(stored_after, stored_before) |
| 245 | + self.assertEqual(list(self.db_path.parent.glob(self.db_path.name + ".backup-*")), [], |
| 246 | + "--no-backup wrote a backup") |
245 | 247 | self.assertIn(f"{stored_before:,} -> {stored_after:,} bytes " |
246 | 248 | f"({100 * (stored_before - stored_after) / stored_before:.1f}% smaller)", printed) |
247 | 249 |
|
@@ -518,6 +520,113 @@ def test_each_pass_removes_its_temporary_dictionaries_templates_included(self): |
518 | 520 | self.assertEqual(stats["templates"], len(TEMPLATE_NAMES)) |
519 | 521 | self.assertEqual(left, [], "temp dictionaries left behind") |
520 | 522 |
|
| 523 | + def test_the_newest_version_row_decides_by_changetime_then_rowid(self): |
| 524 | + """As the app's DatabaseVersionResolver reads it: the greatest changeTime wins, and rowid breaks a tie. Each |
| 525 | + case is inserted so that the other orders (rowid alone, changeTime alone, the tie broken the other way) |
| 526 | + pick the other row.""" |
| 527 | + jan, feb = "2026-01-01 00:00:00", "2026-02-01 00:00:00" |
| 528 | + cases = { |
| 529 | + "older changeTime, newer rowid: 3 then 2": ([(3, feb), (2, jan)], True), |
| 530 | + "older changeTime, newer rowid: 2 then 3": ([(2, feb), (3, jan)], False), |
| 531 | + "tie, the newer rowid declares 3": ([(2, feb), (3, feb)], True), |
| 532 | + "tie, the newer rowid declares 2": ([(3, feb), (2, feb)], False), |
| 533 | + } |
| 534 | + for label, (rows, compressed) in cases.items(): |
| 535 | + with self.subTest(label): |
| 536 | + conn = sqlite3.connect(":memory:") |
| 537 | + conn.executescript(VERSION_SQL) |
| 538 | + conn.executemany("INSERT INTO DocumentationDatabaseVersion (major, minor, patch, who, comment, " |
| 539 | + "changeTime) VALUES (?, 0, 0, 'test', 'test', ?)", rows) |
| 540 | + self.assertEqual(remint_dictionary.templates_compressed(conn), compressed) |
| 541 | + conn.close() |
| 542 | + |
| 543 | + def test_a_newer_changetime_in_an_older_row_still_gets_its_templates_reminted(self): |
| 544 | + """The whole run, for the first case above: version 3 was written first but with the later changeTime. |
| 545 | + Read by rowid it would look like 2, and the compressed templates would be left behind, undecodable.""" |
| 546 | + self.add_versions_and_templates(majors=()) |
| 547 | + self.conn.executemany("INSERT INTO DocumentationDatabaseVersion (major, minor, patch, who, comment, " |
| 548 | + "changeTime) VALUES (?, 0, 0, 'test', 'test', ?)", |
| 549 | + [(3, "2026-02-01 00:00:00"), (2, "2026-01-01 00:00:00")]) |
| 550 | + self.conn.commit() |
| 551 | + before = self.snapshot() |
| 552 | + stats = self.remint(seed=999) |
| 553 | + self.assertEqual(stats["templates"], len(TEMPLATE_NAMES)) |
| 554 | + with DictionaryCompressor(load_dictionary(self.conn)) as compressor: |
| 555 | + for name, stored in self.stored_templates().items(): |
| 556 | + self.assertEqual(compressor.decompress(stored), self.templates[name], name) |
| 557 | + self.assertEqual(verify(before, self.db_path), []) |
| 558 | + |
| 559 | + def test_the_templates_share_of_the_sample_follows_their_stored_bytes(self): |
| 560 | + """Templates are one doc set, weighted by their stored bytes like the others: heavy templates are all drawn, |
| 561 | + light ones only in proportion to their few bytes. Weighting them by anything else - nothing, or one per |
| 562 | + template - draws a different number.""" |
| 563 | + for label, size in {"heavy": 6000, "light": 8}.items(): |
| 564 | + with self.subTest(label): |
| 565 | + self.tearDown() |
| 566 | + self.setUp() |
| 567 | + self.add_versions_and_templates() |
| 568 | + with DictionaryCompressor(self.first_dictionary) as compressor: |
| 569 | + for i in range(27): |
| 570 | + text = make_template(100 + i, size) |
| 571 | + self.templates[f"extra{i}.peb"] = text |
| 572 | + self.conn.execute("INSERT INTO Templates (name, content) VALUES (?, ?)", |
| 573 | + (f"extra{i}.peb", compressor.compress(text))) |
| 574 | + for name in TEMPLATE_NAMES: # the three of add_versions_and_templates, at this size too |
| 575 | + text = make_template(TEMPLATE_NAMES.index(name), size) |
| 576 | + self.templates[name] = text |
| 577 | + self.conn.execute("UPDATE Templates SET content = ? WHERE name = ?", |
| 578 | + (compressor.compress(text), name)) |
| 579 | + self.conn.commit() |
| 580 | + |
| 581 | + # collect_training_samples' rule, at this fixture's sizes, where neither the 3x sample cap nor the |
| 582 | + # byte budget binds: a set draws up to 3 x max(1, round(sample_size x its share of stored bytes)). |
| 583 | + content_bytes = sum(row[4] for row in load_base_rows(self.conn)) |
| 584 | + template_bytes = self.conn.execute("SELECT SUM(LENGTH(content)) FROM Templates").fetchone()[0] |
| 585 | + quota = max(1, round(30 * template_bytes / (content_bytes + template_bytes))) |
| 586 | + expected = min(len(self.templates), 3 * quota) |
| 587 | + if label == "heavy": |
| 588 | + self.assertGreater(expected, 3, "fixture premise: heavy templates draw more than the floor") |
| 589 | + else: |
| 590 | + self.assertLess(expected, len(self.templates), "fixture premise: light templates draw only some") |
| 591 | + |
| 592 | + real, seen = remint_dictionary.train_dictionary, [] |
| 593 | + |
| 594 | + def train(samples, dict_size): |
| 595 | + seen.extend(samples) |
| 596 | + return real(samples, dict_size) |
| 597 | + |
| 598 | + with mock.patch.object(remint_dictionary, "train_dictionary", side_effect=train): |
| 599 | + self.remint(seed=999) |
| 600 | + drawn = sum(1 for sample in seen if sample in set(self.templates.values())) |
| 601 | + self.assertEqual(drawn, expected) |
| 602 | + |
| 603 | + def test_a_candidate_whose_content_alone_beats_the_best_total_is_not_kept(self): |
| 604 | + """The first candidate stores Content and templates smallest in all; the second stores more in all, though |
| 605 | + its Content alone is less than the first's total. Weighing a candidate's Content against the best's total |
| 606 | + would keep the second.""" |
| 607 | + self.add_versions_and_templates() |
| 608 | + before = self.snapshot() |
| 609 | + words = " ".join(WORDS).encode("utf-8") * 40 |
| 610 | + candidates = [words + b"".join(self.templates.values()), words] |
| 611 | + |
| 612 | + def stored(dictionary, texts): |
| 613 | + with DictionaryCompressor(dictionary) as compressor: |
| 614 | + return sum(len(compressor.compress(t)) for t in texts) |
| 615 | + content = [stored(d, self.originals.values()) for d in candidates] |
| 616 | + templates = [stored(d, self.templates.values()) for d in candidates] |
| 617 | + self.assertLess(content[1], content[0] + templates[0], |
| 618 | + "fixture premise: the second candidate's Content is less than the first's total") |
| 619 | + self.assertLess(content[0] + templates[0], content[1] + templates[1], |
| 620 | + "fixture premise: the first candidate stores less in all") |
| 621 | + |
| 622 | + with mock.patch.object(remint_dictionary, "train_dictionary", side_effect=list(candidates)): |
| 623 | + stats = self.remint(seed=999, seeds=2) |
| 624 | + |
| 625 | + self.assertEqual(stats["kept"], "candidate") |
| 626 | + self.assertEqual(load_dictionary(self.conn), candidates[0], "kept the candidate that stores more in all") |
| 627 | + self.assertEqual(stats["after"], content[0] + templates[0]) |
| 628 | + self.assertEqual(verify(before, self.db_path), []) |
| 629 | + |
521 | 630 | def test_templates_are_in_the_training_sample(self): |
522 | 631 | self.add_versions_and_templates() |
523 | 632 | real = remint_dictionary.train_dictionary |
@@ -658,14 +767,17 @@ def test_a_remint_that_writes_backs_up_first(self): |
658 | 767 | def test_the_verifier_reports_a_database_with_no_dictionary_on_an_error_line(self): |
659 | 768 | original = self.dir / "original.db" |
660 | 769 | shutil.copy2(self.db_path, original) |
661 | | - self.conn.execute("DELETE FROM CompressionDictionary") |
662 | | - self.conn.commit() |
663 | | - result = self.run_script("verify_remint_dictionary.py", str(original), str(self.db_path)) |
664 | | - self.assertEqual(result.returncode, 1, result.stderr) |
665 | | - self.assertNotIn("Traceback", result.stderr) |
666 | | - self.assertTrue(result.stderr.startswith("error: ") and "no CompressionDictionary row" in result.stderr, |
667 | | - result.stderr) |
668 | | - self.assertNotIn("Byte-identical", result.stdout) |
| 770 | + cases = {"no row": ("DELETE FROM CompressionDictionary", "no CompressionDictionary row"), |
| 771 | + "no table": ("DROP TABLE CompressionDictionary", "no CompressionDictionary table")} |
| 772 | + for label, (sql, says) in cases.items(): |
| 773 | + with self.subTest(label): |
| 774 | + self.conn.execute(sql) |
| 775 | + self.conn.commit() |
| 776 | + result = self.run_script("verify_remint_dictionary.py", str(original), str(self.db_path)) |
| 777 | + self.assertEqual(result.returncode, 1, result.stderr) |
| 778 | + self.assertNotIn("Traceback", result.stderr) |
| 779 | + self.assertTrue(result.stderr.startswith("error: ") and says in result.stderr, result.stderr) |
| 780 | + self.assertNotIn("Byte-identical", result.stdout) |
669 | 781 |
|
670 | 782 |
|
671 | 783 | if __name__ == "__main__": |
|
0 commit comments