Hsmtool: enable dumping output descriptors of onchain wallet

common/Makefile

@@ -21,6 +21,7 @@ COMMON_SRC_NOGEN :=				\
 	common/daemon_conn.c			\
 	common/decode_array.c			\
 	common/derive_basepoints.c		\
+	common/descriptor_checksum.c		\
 	common/dev_disconnect.c			\
 	common/dijkstra.c			\
 	common/ecdh_hsmd.c			\

common/descriptor_checksum.c

@@ -0,0 +1,82 @@
+#include <ccan/short_types/short_types.h>
+#include <common/descriptor_checksum.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+
+static const char CHECKSUM_CHARSET[] = "qpzry9x8gf2tvdw0s3jn54khce6mua7l";
+
+static const char INPUT_CHARSET[] =
+	"0123456789()[],'/*abcdefgh@:$%{}"
+	"IJKLMNOPQRSTUVWXYZ&+-.;<=>?!^_|~"
+	"ijklmnopqrstuvwxyzABCDEFGH`#\"\\ ";
+
+static inline int charset_find(char ch) {
+	for (size_t i = 0; i < sizeof(INPUT_CHARSET); i++) {
+		if (INPUT_CHARSET[i] == ch)
+			return i;
+	}
+	return -1;
+}
+
+static u64 polymod(u64 c, int val)
+{
+	u8 c0 = c >> 35;
+	c = ((c & 0x7ffffffff) << 5) ^ val;
+	if (c0 & 1) c ^= 0xf5dee51989;
+	if (c0 & 2) c ^= 0xa9fdca3312;
+	if (c0 & 4) c ^= 0x1bab10e32d;
+	if (c0 & 8) c ^= 0x3706b1677a;
+	if (c0 & 16) c ^= 0x644d626ffd;
+	return c;
+}
+
+
+bool descriptor_checksum(const char *descriptor, int desc_size,
+			 struct descriptor_checksum *checksum)
+{
+	checksum->csum[0] = 0;
+
+	int j;
+	u64 c = 1;
+	int cls = 0;
+	int clscount = 0;
+
+	for (int i = 0; i < desc_size; i++) {
+		char ch = descriptor[i];
+		int pos = charset_find(ch);
+		if (pos == -1) {
+			checksum->csum[0] = 0;
+			return false;
+		}
+		/* Emit a symbol for the position inside the group, for every
+		 * character. */
+		c = polymod(c, pos & 31);
+
+		/* Accumulate the group numbers */
+		cls = cls * 3 + (pos >> 5);
+
+		if (++clscount == 3) {
+			c = polymod(c, cls);
+			cls = 0;
+			clscount = 0;
+		}
+	}
+
+	if (clscount > 0)
+		c = polymod(c, cls);
+
+	/* Shift further to determine the checksum. */
+	for (j = 0; j < DESCRIPTOR_CHECKSUM_LENGTH; ++j)
+		c = polymod(c, 0);
+
+	/* Prevent appending zeroes from not affecting the checksum. */
+	c ^= 1;
+
+	for (j = 0; j < DESCRIPTOR_CHECKSUM_LENGTH; ++j)
+		checksum->csum[j] = CHECKSUM_CHARSET[(c >> (5 * (7 - j))) & 31];
+
+	checksum->csum[DESCRIPTOR_CHECKSUM_LENGTH] = 0;
+
+	return true;
+}

common/descriptor_checksum.h

@@ -0,0 +1,16 @@
+#ifndef LIGHTNING_COMMON_DESCRIPTOR_CHECKSUM_H
+#define LIGHTNING_COMMON_DESCRIPTOR_CHECKSUM_H
+#include "config.h"
+#include <stdbool.h>
+
+/* https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md#reference */
+#define DESCRIPTOR_CHECKSUM_LENGTH 8
+
+struct descriptor_checksum {
+	char csum[DESCRIPTOR_CHECKSUM_LENGTH + 1];
+};
+
+bool descriptor_checksum(const char *descriptor, int desc_size,
+			 struct descriptor_checksum *checksum);
+
+#endif /* LIGHTNING_COMMON_DESCRIPTOR_CHECKSUM_H */

contrib/pyln-testing/pyln/testing/fixtures.py

@@ -132,6 +132,14 @@ def bitcoind(directory, teardown_checks):
         raise ValueError("bitcoind is too old. At least version 16000 (v0.16.0)"
                          " is needed, current version is {}".format(info['version']))
 
+    # Make sure we have a wallet, starting with 0.21 there is no default wallet
+    # anymore.
+    # FIXME: if we update the testsuite to use the upcoming 0.21 release we
+    # could switch to descriptor wallets and speed bitcoind operations
+    # consequently.
+    if not bitcoind.rpc.listwallets():
+        bitcoind.rpc.createwallet("lightningd-tests")
+
     info = bitcoind.rpc.getblockchaininfo()
     # Make sure we have some spendable funds
     if info['blocks'] < 101:

doc/FAQ.md

@@ -108,7 +108,6 @@ C-lightning has an internal bitcoin wallet, which you can use to make "on-chain"
 transactions, (see [withdraw](https://lightning.readthedocs.io/lightning-withdraw.7.html).
 These on-chain funds are backed up via the HD wallet seed, stored in byte-form in `hsm_secret`.
 
-and which you can backup thanks to a seed stored in the `hsm_secret`.
 `lightningd` also stores information for funds locked in Lightning Network channels, which are stored
 in a database. This database is required for on-going channel updates as well as channel closure.
 There is no single-seed backup for funds locked in channels.

doc/lightning-hsmtool.8

@@ -55,6 +55,17 @@ Specify \fIpassword\fR if the \fBhsm_secret\fR is encrypted\.
 \fBgeneratehsm\fR \fIhsm_secret_path\fR
 Generates a new hsm_secret using BIP39\.
 
+
+ \fBdumponchaindescriptors\fR \fIhsm_secret\fR [\fIpassword\fR] [\fInetwork\fR]
+Dump output descriptors for our onchain wallet\.
+The descriptors can be used by external services to be able to generate
+addresses for our onchain wallet\. (for example on \fBbitcoind\fR using the
+\fBimportmulti\fR or \fBimportdescriptors\fR RPC calls)
+We need the path to the hsm_secret containing the wallet seed, and an optional
+(skip using \fB""\fR) password if it was encrypted\.
+To generate descriptors using testnet master keys, you may specify \fItestnet\fR as
+the last parameter\. By default, mainnet-encoded keys are generated\.
+
 .SH BUGS
 
 You should report bugs on our github issues page, and maybe submit a fix
@@ -80,4 +91,4 @@ Note: the modules in the ccan/ directory have their own licenses, but
 the rest of the code is covered by the BSD-style MIT license\.
 Main web site: \fIhttps://github.com/ElementsProject/lightning\fR
 
-\" SHA256STAMP:918981692d3840344e15c539b007b473d5ea0ad481145eccff092bf61ec6ddb0
+\" SHA256STAMP:3d847c486363271e0635336caca4fd14f5007a3ff463c223fb5bdb52dbf7b98e

doc/lightning-hsmtool.8.md

@@ -51,6 +51,16 @@ Specify *password* if the `hsm_secret` is encrypted.
 **generatehsm** *hsm\_secret\_path*
 Generates a new hsm_secret using BIP39.
 
+ **dumponchaindescriptors** *hsm_secret* \[*password*\] \[*network*\]
+Dump output descriptors for our onchain wallet.
+The descriptors can be used by external services to be able to generate
+addresses for our onchain wallet. (for example on `bitcoind` using the
+`importmulti` or `importdescriptors` RPC calls)
+We need the path to the hsm_secret containing the wallet seed, and an optional
+(skip using `""`) password if it was encrypted.
+To generate descriptors using testnet master keys, you may specify *testnet* as
+the last parameter. By default, mainnet-encoded keys are generated.
+
 BUGS
 ----
 

tests/fuzz/Makefile

@@ -21,6 +21,7 @@ FUZZ_COMMON_OBJS := \
 	common/daemon.o					\
 	common/daemon_conn.o				\
 	common/derive_basepoints.o			\
+	common/descriptor_checksum.o			\
 	common/fee_states.o				\
 	common/htlc_state.o				\
 	common/permute_tx.o				\

tests/fuzz/fuzz-amount.c

@@ -21,10 +21,7 @@ void run(const uint8_t *data, size_t size)
 
 	/* We should not crash when parsing any string. */
 
-	string = tal_arr(NULL, char, size);
-	for (size_t i = 0; i < size; i++)
-		string[i] = (char) data[i] % (CHAR_MAX + 1);
-
+	string = to_string(NULL, data, size);
 	parse_amount_msat(&msat, string, tal_count(string));
 	parse_amount_sat(&sat, string, tal_count(string));
 	tal_free(string);

tests/fuzz/fuzz-descriptor_checksum.c

@@ -0,0 +1,20 @@
+#include <tests/fuzz/libfuzz.h>
+
+#include <ccan/tal/tal.h>
+#include <common/descriptor_checksum.h>
+
+void init(int *argc, char ***argv)
+{
+}
+
+void run(const uint8_t *data, size_t size)
+{
+	char *string;
+	struct descriptor_checksum checksum;
+
+	/* We should not crash nor overflow the checksum buffer. */
+
+	string = to_string(NULL, data, size);
+	descriptor_checksum(string, tal_count(string), &checksum);
+	tal_free(string);
+}

tests/fuzz/libfuzz.c

@@ -31,3 +31,13 @@ const uint8_t **get_chunks(const void *ctx, const uint8_t *data,
 
 	return chunks;
 }
+
+char *to_string(const tal_t *ctx, const u8 *data, size_t data_size)
+{
+	char *string = tal_arr(ctx, char, data_size);
+
+	for (size_t i = 0; i < data_size; i++)
+		string[i] = (char) data[i] % (CHAR_MAX + 1);
+
+	return string;
+}

tests/fuzz/libfuzz.h

@@ -1,6 +1,8 @@
 #ifndef LIGHTNING_TESTS_FUZZ_LIBFUZZ_H
 #define LIGHTNING_TESTS_FUZZ_LIBFUZZ_H
 
+#include <ccan/ccan/short_types/short_types.h>
+#include <ccan/ccan/tal/tal.h>
 #include <stddef.h>
 #include <stdint.h>
 
@@ -15,4 +17,7 @@ void run(const uint8_t *data, size_t size);
 const uint8_t **get_chunks(const void *ctx, const uint8_t *data,
 			  size_t data_size, size_t chunk_size);
 
+/* Copy the data as a string. */
+char *to_string(const tal_t *ctx, const u8 *data, size_t data_size);
+
 #endif /* LIGHTNING_TESTS_FUZZ_LIBFUZZ_H */

tests/test_wallet.py

@@ -1045,6 +1045,36 @@ def test_hsmtool_secret_decryption(node_factory):
     assert node_id == l1.rpc.getinfo()["id"]
 
 
+@unittest.skipIf(TEST_NETWORK == 'liquid-regtest', '')
+def test_hsmtool_dump_descriptors(node_factory, bitcoind):
+    l1 = node_factory.get_node()
+    l1.fundwallet(10**6)
+
+    # Get a tpub descriptor of lightningd's wallet
+    hsm_path = os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, "hsm_secret")
+    cmd_line = ["tools/hsmtool", "dumponchaindescriptors", hsm_path, "",
+                "testnet"]
+    out = subprocess.check_output(cmd_line).decode("utf8").split("\n")
+    descriptor = [l for l in out if l.startswith("wpkh(tpub")][0]
+
+    # Import the descriptor to bitcoind
+    # FIXME: if we update the testsuite to use the upcoming 0.21 we could use
+    # importdescriptors instead.
+    bitcoind.rpc.importmulti([{
+        "desc": descriptor,
+        # No need to rescan, we'll transact afterward
+        "timestamp": "now",
+        # The default
+        "range": [0, 99]
+    }])
+
+    # Funds sent to lightningd can be retrieved by bitcoind
+    addr = l1.rpc.newaddr()["bech32"]
+    txid = l1.rpc.withdraw(addr, 10**3)["txid"]
+    bitcoind.generate_block(1, txid)
+    assert len(bitcoind.rpc.listunspent(1, 1, [addr])) == 1
+
+
 # this test does a 'listtransactions' on a yet unconfirmed channel
 def test_fundchannel_listtransaction(node_factory, bitcoind):
     l1, l2 = node_factory.get_nodes(2)

tools/Makefile

@@ -17,7 +17,7 @@ tools/headerversions: FORCE tools/headerversions.o $(CCAN_OBJS)
 
 tools/check-bolt: tools/check-bolt.o $(CCAN_OBJS) $(TOOLS_COMMON_OBJS)
 
-tools/hsmtool: tools/hsmtool.o $(CCAN_OBJS) $(TOOLS_COMMON_OBJS) $(BITCOIN_OBJS) common/amount.o common/bech32.o common/bigsize.o common/derive_basepoints.o common/node_id.o common/type_to_string.o wire/fromwire.o wire/towire.o
+tools/hsmtool: tools/hsmtool.o $(CCAN_OBJS) $(TOOLS_COMMON_OBJS) $(BITCOIN_OBJS) common/amount.o common/bech32.o common/bigsize.o common/derive_basepoints.o common/descriptor_checksum.o common/node_id.o common/type_to_string.o wire/fromwire.o wire/towire.o
 
 tools/lightning-hsmtool: tools/hsmtool
 	cp $< $@

tools/hsmtool.c

@@ -6,8 +6,10 @@
 #include <ccan/read_write_all/read_write_all.h>
 #include <ccan/str/str.h>
 #include <ccan/tal/path/path.h>
+#include <ccan/tal/str/str.h>
 #include <common/bech32.h>
 #include <common/derive_basepoints.h>
+#include <common/descriptor_checksum.h>
 #include <common/node_id.h>
 #include <common/type_to_string.h>
 #include <common/utils.h>
@@ -39,6 +41,8 @@ static void show_usage(const char *progname)
 	printf("	- guesstoremote <P2WPKH address> <node id> <tries> "
 	       "<path/to/hsm_secret> [hsm_secret password]\n");
 	printf("	- generatehsm <path/to/new//hsm_secret>\n");
+	printf("	- dumponchaindescriptors <path/to/hsm_secret> [password] "
+		"[network]\n");
 	exit(0);
 }
 
@@ -513,6 +517,59 @@ static int generate_hsm(const char *hsm_secret_path)
 	return 0;
 }
 
+static int dumponchaindescriptors(const char *hsm_secret_path, const char *passwd,
+				  const bool is_testnet)
+{
+	struct secret hsm_secret;
+	u8 bip32_seed[BIP32_ENTROPY_LEN_256];
+	u32 salt = 0;
+	u32 version = is_testnet ?
+		BIP32_VER_TEST_PRIVATE : BIP32_VER_MAIN_PRIVATE;
+	struct ext_key master_extkey;
+	char *enc_xpub, *descriptor;
+	struct descriptor_checksum checksum;
+
+	if (passwd)
+		get_encrypted_hsm_secret(&hsm_secret, hsm_secret_path, passwd);
+	else
+		get_hsm_secret(&hsm_secret, hsm_secret_path);
+
+	/* We use m/0/0/k as the derivation tree for onchain funds. */
+
+	/* The root seed is derived from hsm_secret using hkdf.. */
+	do {
+		hkdf_sha256(bip32_seed, sizeof(bip32_seed),
+			    &salt, sizeof(salt),
+			    &hsm_secret, sizeof(hsm_secret),
+			    "bip32 seed", strlen("bip32 seed"));
+		salt++;
+		/* ..Which is used to derive m/ */
+	} while (bip32_key_from_seed(bip32_seed, sizeof(bip32_seed),
+				     version, 0, &master_extkey) != WALLY_OK);
+
+	if (bip32_key_to_base58(&master_extkey, BIP32_FLAG_KEY_PUBLIC, &enc_xpub) != WALLY_OK)
+		errx(ERROR_LIBWALLY, "Can't encode xpub");
+
+	/* Now we format the descriptor strings (we only ever create P2WPKH and
+	 * P2SH-P2WPKH outputs). */
+
+	descriptor = tal_fmt(NULL, "wpkh(%s/0/0/*)", enc_xpub);
+	if (!descriptor_checksum(descriptor, strlen(descriptor), &checksum))
+		errx(ERROR_LIBWALLY, "Can't derive descriptor checksum for wpkh");
+	printf("%s#%s\n", descriptor, checksum.csum);
+	tal_free(descriptor);
+
+	descriptor = tal_fmt(NULL, "sh(wpkh(%s/0/0/*))", enc_xpub);
+	if (!descriptor_checksum(descriptor, strlen(descriptor), &checksum))
+		errx(ERROR_LIBWALLY, "Can't derive descriptor checksum for sh(wpkh)");
+	printf("%s#%s\n", descriptor, checksum.csum);
+	tal_free(descriptor);
+
+	wally_free_string(enc_xpub);
+
+	return 0;
+}
+
 int main(int argc, char *argv[])
 {
 	const char *method;
@@ -573,5 +630,25 @@ int main(int argc, char *argv[])
 		return generate_hsm(hsm_secret_path);
 	}
 
+	if (streq(method, "dumponchaindescriptors")) {
+		bool is_testnet;
+		if (argc < 3)
+			show_usage(argv[0]);
+
+		if (argc > 4) {
+			is_testnet = streq(argv[4], "testnet");
+			if (!is_testnet && !streq(argv[4], "bitcoin"))
+				errx(ERROR_USAGE, "Network '%s' not supported."
+				     " Supported networks: bitcoin (default),"
+				     " testnet",
+				     argv[4]);
+		} else
+			is_testnet = false;
+
+		return dumponchaindescriptors(argv[2],
+					      argc > 3 && !streq(argv[3], "") ? argv[3] : NULL,
+					      is_testnet);
+	}
+
 	show_usage(argv[0]);
 }