Fix BOLT11 annotation loss after restart

Fixes #6978 where BOLT11 invoice annotations were lost after lightning
node restart due to broken linkage between HTLCs and payment records.

Add payment_id column to channel_htlcs table with proper foreign key
relationship to maintain HTLC-payment linkage through restarts.

Author: wqxoxo

tests/test_pay.py

@@ -7048,3 +7048,109 @@ def test_htlc_tlv_crash(node_factory):
 
     l1.rpc.waitsendpay(inv1['payment_hash'], TIMEOUT)
     l1.rpc.waitsendpay(inv2['payment_hash'], TIMEOUT)
+
+
+@pytest.mark.openchannel('v1')
+@pytest.mark.openchannel('v2')
+def test_bolt11_annotation_persistence(node_factory):
+    """Test that HTLCs maintain proper linkage to payment records with BOLT11."""
+    l1, l2 = node_factory.line_graph(2)
+
+    inv = l2.rpc.invoice(100000, 'test_annotation', 'Test Description')['bolt11']
+    l1.dev_pay(inv, dev_use_shadow=False)
+
+    payments = l1.rpc.listpays()['pays']
+    payment = [p for p in payments if p['bolt11'] == inv][0]
+    payment_hash = payment['payment_hash']
+
+    # Verify HTLCs have payment_id references
+    htlcs = l1.db_query(f"""
+        SELECT payment_id, partid, groupid
+        FROM channel_htlcs
+        WHERE payment_hash = x'{payment_hash}'
+    """)
+    assert len(htlcs) > 0
+
+    # Get payment records for verification
+    payment_records = l1.db_query(f"""
+        SELECT id, partid, groupid
+        FROM payments
+        WHERE payment_hash = x'{payment_hash}'
+    """)
+    assert len(payment_records) > 0
+
+    # Verify each HTLC links to correct payment record
+    for htlc in htlcs:
+        assert htlc['payment_id'] is not None
+
+        # Find matching payment record
+        matching_payments = [p for p in payment_records
+                           if p['id'] == htlc['payment_id']
+                           and p['partid'] == htlc['partid']
+                           and p['groupid'] == htlc['groupid']]
+        assert len(matching_payments) == 1
+
+    # Verify payment_id index exists for performance
+    indexes = l1.db_query("""
+        SELECT name FROM sqlite_master
+        WHERE type = 'index' AND name = 'channel_htlcs_payment_id_idx'
+    """)
+    assert len(indexes) == 1
+
+    l1.restart()
+
+    # After restart, verify BOLT11 annotation persistence
+    payments_after = l1.rpc.listpays()['pays']
+    payment_after = [p for p in payments_after if p['bolt11'] == inv]
+    assert len(payment_after) == 1
+
+    # Verify payment data integrity
+    assert payment_after[0]['payment_hash'] == payment_hash
+    assert payment_after[0]['status'] == 'complete'
+
+    # Verify HTLCs still have payment_id linkage after restart
+    htlcs_after = l1.db_query(f"""
+        SELECT payment_id, partid, groupid
+        FROM channel_htlcs
+        WHERE payment_hash = x'{payment_hash}'
+    """)
+    assert len(htlcs_after) == len(htlcs)
+
+    for htlc in htlcs_after:
+        assert htlc['payment_id'] is not None
+
+        # Re-verify linkage integrity after restart
+        payment_check = l1.db_query(f"""
+            SELECT COUNT(*) as count
+            FROM payments
+            WHERE id = {htlc['payment_id']}
+            AND partid = {htlc['partid']}
+            AND groupid = {htlc['groupid']}
+        """)
+        assert payment_check[0]['count'] == 1
+
+
+@pytest.mark.openchannel('v1')
+@pytest.mark.openchannel('v2')
+def test_bolt11_payment_id_migration(node_factory):
+    """Test that the migration properly adds payment_id column to channel_htlcs."""
+    l1, l2 = node_factory.line_graph(2)
+
+    # Create some payments to test migration worked
+    inv1 = l2.rpc.invoice(100000, 'pre_migration_1', 'Test Description')['bolt11']
+    inv2 = l2.rpc.invoice(200000, 'pre_migration_2', 'Test Description 2')['bolt11']
+
+    l1.dev_pay(inv1, dev_use_shadow=False)
+    l1.dev_pay(inv2, dev_use_shadow=False)
+
+    # Verify payment_id column exists in schema
+    schema = l1.db_query("SELECT sql FROM sqlite_master WHERE name = 'channel_htlcs'")
+    assert any('payment_id' in row['sql'] for row in schema)
+
+    # Verify index exists
+    indexes = l1.db_query("SELECT name FROM sqlite_master WHERE type = 'index' AND name = 'channel_htlcs_payment_id_idx'")
+    assert len(indexes) == 1
+
+    # Verify HTLCs have proper linkage
+    htlcs = l1.db_query("SELECT payment_id FROM channel_htlcs WHERE payment_id IS NOT NULL")
+    assert len(htlcs) >= 2

wallet/db.c

@@ -35,6 +35,8 @@ static void migrate_our_funding(struct lightningd *ld, struct db *db);
 
 static void migrate_last_tx_to_psbt(struct lightningd *ld, struct db *db);
 
+static void migrate_add_payment_id_to_htlcs(struct lightningd *ld, struct db *db);
+
 static void
 migrate_inflight_last_tx_to_psbt(struct lightningd *ld, struct db *db);
 
@@ -1092,6 +1094,8 @@ static struct migration dbmigrations[] = {
 	 ")"), NULL},
     /* We do a lookup before each append, to avoid duplicates */
     {SQL("CREATE INDEX chain_moves_utxo_idx ON chain_moves (utxo)"), NULL},
+    /* Add payment_id column to channel_htlcs */
+    {NULL, migrate_add_payment_id_to_htlcs},
     {NULL, migrate_from_account_db},
 };
 
@@ -2080,6 +2084,31 @@ static void migrate_initialize_alias_local(struct lightningd *ld,
 	}
 }
 
+/* Add payment_id column to channel_htlcs */
+static void migrate_add_payment_id_to_htlcs(struct lightningd *ld, struct db *db)
+{
+	struct db_stmt *stmt;
+
+	stmt = db_prepare_v2(db, SQL("ALTER TABLE channel_htlcs ADD COLUMN payment_id BIGINT"));
+	db_exec_prepared_v2(stmt);
+	tal_free(stmt);
+
+	stmt = db_prepare_v2(db, SQL("UPDATE channel_htlcs "
+				     "SET payment_id = ("
+				     "    SELECT p.id FROM payments p "
+				     "    WHERE p.payment_hash = channel_htlcs.payment_hash "
+				     "    AND p.partid = channel_htlcs.partid "
+				     "    AND p.groupid = channel_htlcs.groupid "
+				     "    LIMIT 1"
+				     ")"));
+	db_exec_prepared_v2(stmt);
+	tal_free(stmt);
+
+	stmt = db_prepare_v2(db, SQL("CREATE INDEX channel_htlcs_payment_id_idx ON channel_htlcs (payment_id)"));
+	db_exec_prepared_v2(stmt);
+	tal_free(stmt);
+}
+
 /* Insert address type as `ADDR_ALL` for issued addresses */
 static void insert_addrtype_to_addresses(struct lightningd *ld,
 					   struct db *db)

wallet/wallet.c

@@ -3316,11 +3316,29 @@ void wallet_htlc_save_out(struct wallet *wallet,
 			  struct htlc_out *out)
 {
 	struct db_stmt *stmt;
+	u64 payment_id = 0;
 
 	/* We absolutely need the incoming HTLC to be persisted before
 	 * we can persist it's dependent */
 	assert(out->in == NULL || out->in->dbid != 0);
 
+	/* Get payment_id if this is an outgoing payment */
+	if (out->am_origin) {
+		struct db_stmt *payment_stmt = db_prepare_v2(wallet->db,
+			SQL("SELECT id FROM payments "
+			    "WHERE payment_hash = ? AND partid = ? AND groupid = ? "
+			    "LIMIT 1"));
+		db_bind_sha256(payment_stmt, &out->payment_hash);
+		db_bind_u64(payment_stmt, out->partid);
+		db_bind_u64(payment_stmt, out->groupid);
+		db_query_prepared(payment_stmt);
+
+		if (db_step(payment_stmt)) {
+			payment_id = db_col_u64(payment_stmt, "id");
+		}
+		tal_free(payment_stmt);
+	}
+
 	stmt = db_prepare_v2(
 	    wallet->db,
 	    SQL("INSERT INTO channel_htlcs ("
@@ -3339,8 +3357,9 @@ void wallet_htlc_save_out(struct wallet *wallet,
 		" partid,"
 		" groupid,"
 		" fees_msat,"
-		" min_commit_num"
-		") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?);"));
+		" min_commit_num,"
+		" payment_id"
+		") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?);"));
 
 	out->dbid = htlcs_index_created(wallet->ld,
 					out->key.id,
@@ -3384,6 +3403,11 @@ void wallet_htlc_save_out(struct wallet *wallet,
 	db_bind_u64(stmt, min_u64(chan->next_index[LOCAL]-1,
 					     chan->next_index[REMOTE]-1));
 
+	if (payment_id > 0)
+		db_bind_u64(stmt, payment_id);
+	else
+		db_bind_null(stmt);
+
 	db_exec_prepared_v2(take(stmt));
 }