Repository navigation
Funding Tx not broadcasted and unable to withdraw #2300
Description
Issue and Steps to Reproduce
Running the following sql command, finds 6 funding tx not broadcasted:
sqlite3 $HOME/.lightning/lightningd.sqlite3 "SELECT count(1) FROM transactions WHERE blockheight is null;"
When trying to broadcast them again using the hex(rawtx) I get the error "missing inputs" on all of them.
Same when trying to withdraw.
Example of a funding tx not broadcasted: 9c420a9ad6f3633aa3f72fafcd1f5f5d99b7c5f2d281c8faa57dd61d21a21c60
Unfortunately I also have a mixed situation in the dabase with both testnet and mainnet tx.
I'm now taking into account only mainnet tx.
Not sure if possible to recover from such situation... any help is appreciated.
listpeers output
{
"peers": [
{
"id": "02d28c3aac4b4f36746052a735831afbe65bc5698a7be5bd41b42fd1ddf2a1a358",
"connected": false,
"channels": [
{
"state": "AWAITING_UNILATERAL",
"scratch_txid": "1e0e5efc53b28295331adf476473dea7ab3f69469063dd766ea8ee30742e06f4",
"short_channel_id": "1261498:30:0",
"channel_id": "0d64321a1c926f51ab91834b3bb3d1a42a5ea234722b36da28c796dab6479ebd",
"funding_txid": "bd9e47b6da96c728da362b7234a25e2aa4d1b33b4b8391ab516f921c1a32640d",
"private": false,
"msatoshi_to_us": 9900000,
"msatoshi_to_us_min": 9900000,
"msatoshi_to_us_max": 9900000,
"msatoshi_total": 10000000,
"dust_limit_satoshis": 546,
"max_htlc_value_in_flight_msat": 18446744073709551615,
"their_channel_reserve_satoshis": 0,
"our_channel_reserve_satoshis": 100,
"spendable_msatoshi": 9800000,
"htlc_minimum_msat": 0,
"their_to_self_delay": 6,
"our_to_self_delay": 4,
"max_accepted_htlcs": 483,
"status": [
"AWAITING_UNILATERAL:Attempting to reconnect"
],
"in_payments_offered": 0,
"in_msatoshi_offered": 0,
"in_payments_fulfilled": 0,
"in_msatoshi_fulfilled": 0,
"out_payments_offered": 0,
"out_msatoshi_offered": 0,
"out_payments_fulfilled": 0,
"out_msatoshi_fulfilled": 0,
"htlcs": [
]
}
]
},
{
"id": "0260a9711694672b4fd55add53def7cdbd18d767f455b052e0ff63e651548c6c56",
"connected": false,
"channels": [
{
"state": "CHANNELD_NORMAL",
"scratch_txid": "4347b900ee41d301a51e544d9b41162d79e28013306b009a229c218f2874c61c",
"short_channel_id": "1261503:25:0",
"channel_id": "f5b8fa2c373aaa8b71edb60a82c2c01797441ee5fb943dd0cf89fbab70f211e1",
"funding_txid": "e111f270abfb89cfd03d94fbe51e449717c0c2820ab6ed718baa3a372cfab8f5",
"private": false,
"msatoshi_to_us": 200000000,
"msatoshi_to_us_min": 200000000,
"msatoshi_to_us_max": 200000000,
"msatoshi_total": 200000000,
"dust_limit_satoshis": 546,
"max_htlc_value_in_flight_msat": 18446744073709551615,
"their_channel_reserve_satoshis": 0,
"our_channel_reserve_satoshis": 2000,
"spendable_msatoshi": 198000000,
"htlc_minimum_msat": 0,
"their_to_self_delay": 6,
"our_to_self_delay": 144,
"max_accepted_htlcs": 483,
"status": [
"CHANNELD_NORMAL:Attempting to reconnect"
],
"in_payments_offered": 0,
"in_msatoshi_offered": 0,
"in_payments_fulfilled": 0,
"in_msatoshi_fulfilled": 0,
"out_payments_offered": 0,
"out_msatoshi_offered": 0,
"out_payments_fulfilled": 0,
"out_msatoshi_fulfilled": 0,
"htlcs": [
]
}
]
},
{
"id": "02f057867392ba8e9fe2b62b8a21ee36be00a3338d5d34cb1a2ffab3d57d8a2aea",
"connected": false,
"channels": [
{
"state": "CHANNELD_NORMAL",
"scratch_txid": "1ba68d867367d9458ed73773d0aa22e6138bc7c26f70d5359107e60622bd8a59",
"short_channel_id": "1261692:327:0",
"channel_id": "16102989b80d40586e1052802afc2bac4136ed90fad495dbb328e3499c12f873",
"funding_txid": "73f8129c49e328b3db95d4fa90ed3641ac2bfc2a8052106e58400db889291016",
"private": false,
"msatoshi_to_us": 2000000000,
"msatoshi_to_us_min": 2000000000,
"msatoshi_to_us_max": 2000000000,
"msatoshi_total": 2000000000,
"dust_limit_satoshis": 546,
"max_htlc_value_in_flight_msat": 18446744073709551615,
"their_channel_reserve_satoshis": 0,
"our_channel_reserve_satoshis": 20000,
"spendable_msatoshi": 1980000000,
"htlc_minimum_msat": 0,
"their_to_self_delay": 6,
"our_to_self_delay": 240,
"max_accepted_htlcs": 483,
"status": [
"CHANNELD_NORMAL:Attempting to reconnect"
],
"in_payments_offered": 0,
"in_msatoshi_offered": 0,
"in_payments_fulfilled": 0,
"in_msatoshi_fulfilled": 0,
"out_payments_offered": 0,
"out_msatoshi_offered": 0,
"out_payments_fulfilled": 0,
"out_msatoshi_fulfilled": 0,
"htlcs": [
]
}
]
},
{
"id": "03f113414ebdc6c1fb0f33c99cd5a1d09dd79e7fdf2468cf1fe1af6674361695d2",
"connected": false,
"channels": [
{
"state": "CHANNELD_NORMAL",
"scratch_txid": "a449bd596ac546eb381492962754ffda32b715b6138174cbdc7cf0af6878529b",
"short_channel_id": "1261692:328:0",
"channel_id": "d016c120e8e97e2e89f2a8c7a10502b647d1b0c14dbaaef6ebb6760950ad4fe3",
"funding_txid": "e34fad500976b6ebf6aeba4dc1b0d147b60205a1c7a8f2892e7ee9e820c116d0",
"private": false,
"msatoshi_to_us": 300000000,
"msatoshi_to_us_min": 300000000,
"msatoshi_to_us_max": 300000000,
"msatoshi_total": 300000000,
"dust_limit_satoshis": 546,
"max_htlc_value_in_flight_msat": 18446744073709551615,
"their_channel_reserve_satoshis": 0,
"our_channel_reserve_satoshis": 3000,
"spendable_msatoshi": 297000000,
"htlc_minimum_msat": 0,
"their_to_self_delay": 6,
"our_to_self_delay": 144,
"max_accepted_htlcs": 483,
"status": [
"CHANNELD_NORMAL:Attempting to reconnect"
],
"in_payments_offered": 0,
"in_msatoshi_offered": 0,
"in_payments_fulfilled": 0,
"in_msatoshi_fulfilled": 0,
"out_payments_offered": 0,
"out_msatoshi_offered": 0,
"out_payments_fulfilled": 0,
"out_msatoshi_fulfilled": 0,
"htlcs": [
]
}
]
},
{
"id": "02f6725f9c1c40333b67faea92fd211c183050f28df32cac3f9d69685fe9665432",
"connected": true,
"netaddr": [
"104.198.32.198:9735"
],
"global_features": "",
"local_features": "88",
"globalfeatures": "",
"localfeatures": "88",
"channels": [
{
"state": "AWAITING_UNILATERAL",
"scratch_txid": "9ecc3656db49fd4aa3df4b3edb2995e0627a798e8800994d26a8b472ed58a97d",
"channel_id": "601ca2211dd67da5fac881d2f2c5b7995d5f1fcdaf2ff7a33a63f3d69a0a429c",
"funding_txid": "9c420a9ad6f3633aa3f72fafcd1f5f5d99b7c5f2d281c8faa57dd61d21a21c60",
"private": false,
"msatoshi_to_us": 500000000,
"msatoshi_to_us_min": 500000000,
"msatoshi_to_us_max": 500000000,
"msatoshi_total": 500000000,
"dust_limit_satoshis": 546,
"max_htlc_value_in_flight_msat": 18446744073709551615,
"their_channel_reserve_satoshis": 0,
"our_channel_reserve_satoshis": 5000,
"spendable_msatoshi": 495000000,
"htlc_minimum_msat": 0,
"their_to_self_delay": 144,
"our_to_self_delay": 144,
"max_accepted_htlcs": 483,
"status": [
"AWAITING_UNILATERAL:Attempting to reconnect"
],
"in_payments_offered": 0,
"in_msatoshi_offered": 0,
"in_payments_fulfilled": 0,
"in_msatoshi_fulfilled": 0,
"out_payments_offered": 0,
"out_msatoshi_offered": 0,
"out_payments_fulfilled": 0,
"out_msatoshi_fulfilled": 0,
"htlcs": [
]
}
]
},
{
"id": "0217890e3aad8d35bc054f43acc00084b25229ecff0ab68debd82883ad65ee8266",
"connected": true,
"netaddr": [
"23.237.77.11:9735"
],
"global_features": "",
"local_features": "82",
"globalfeatures": "",
"localfeatures": "82",
"channels": [
{
"state": "CHANNELD_NORMAL",
"scratch_txid": "cc3e39dad21e1a2e611e2f3e0bcdf19ff44cc91d2096412bf2a13594af08f5e1",
"owner": "lightning_channeld",
"short_channel_id": "542582:1404:0",
"channel_id": "5e61be57fe8ce61b1e21480080b125c346a7150c48e9c0c128a07eb0ce79ed91",
"funding_txid": "91ed79ceb07ea028c1c0e9480c15a746c325b1800048211e1be68cfe57be615e",
"private": false,
"msatoshi_to_us": 50000000,
"msatoshi_to_us_min": 50000000,
"msatoshi_to_us_max": 50000000,
"msatoshi_total": 50000000,
"dust_limit_satoshis": 546,
"max_htlc_value_in_flight_msat": 18446744073709551615,
"their_channel_reserve_satoshis": 546,
"our_channel_reserve_satoshis": 546,
"spendable_msatoshi": 49454000,
"htlc_minimum_msat": 0,
"their_to_self_delay": 144,
"our_to_self_delay": 144,
"max_accepted_htlcs": 483,
"status": [
"CHANNELD_NORMAL:Reconnected, and reestablished.",
"CHANNELD_NORMAL:Funding transaction locked. Waiting for their announcement signatures."
],
"in_payments_offered": 0,
"in_msatoshi_offered": 0,
"in_payments_fulfilled": 0,
"in_msatoshi_fulfilled": 0,
"out_payments_offered": 0,
"out_msatoshi_offered": 0,
"out_payments_fulfilled": 0,
"out_msatoshi_fulfilled": 0,
"htlcs": [
]
}
]
},
{
"id": "0279c22ed7a068d10dc1a38ae66d2d6461e269226c60258c021b1ddcdfe4b00bc4",
"connected": false,
"channels": [
{
"state": "AWAITING_UNILATERAL",
"scratch_txid": "0cec0a9e9322f728253c14c69d7d1a481daac7a3176ffbc9d3df38f7099c353f",
"channel_id": "79608e0c4f9ad6b7411e1727cea8b2de9b4121db26170a6eaa40197776db8673",
"funding_txid": "7386db76771940aa6e0a1726db21419bdeb2a8ce27171e41b7d69a4f0c8e6079",
"private": false,
"msatoshi_to_us": 100000000,
"msatoshi_to_us_min": 100000000,
"msatoshi_to_us_max": 100000000,
"msatoshi_total": 100000000,
"dust_limit_satoshis": 546,
"max_htlc_value_in_flight_msat": 18446744073709551615,
"their_channel_reserve_satoshis": 1000,
"our_channel_reserve_satoshis": 1000,
"spendable_msatoshi": 99000000,
"htlc_minimum_msat": 0,
"their_to_self_delay": 144,
"our_to_self_delay": 144,
"max_accepted_htlcs": 483,
"status": [
"AWAITING_UNILATERAL:Attempting to reconnect"
],
"in_payments_offered": 0,
"in_msatoshi_offered": 0,
"in_payments_fulfilled": 0,
"in_msatoshi_fulfilled": 0,
"out_payments_offered": 0,
"out_msatoshi_offered": 0,
"out_payments_fulfilled": 0,
"out_msatoshi_fulfilled": 0,
"htlcs": [
]
}
]
},
{
"id": "0320e5b9f19a1c4dc22ebb751d757eaa5843262cd55a0f1b0d13cb5d445c5c78ae",
"connected": false,
"channels": [
{
"state": "AWAITING_UNILATERAL",
"scratch_txid": "6b39947f7931d23518026febf9c6b93af5cd2ef343173d7ee2b22c8fbe8f5c12",
"channel_id": "39a191d0ac2f84e5794a0fcd8699a1e4444475e8ceaaa43280c46fc8f03014f6",
"funding_txid": "f61430f0c86fc48032a4aacee8754444e4a19986cd0f4a79e5842facd091a139",
"private": false,
"msatoshi_to_us": 3000000000,
"msatoshi_to_us_min": 3000000000,
"msatoshi_to_us_max": 3000000000,
"msatoshi_total": 3000000000,
"dust_limit_satoshis": 546,
"max_htlc_value_in_flight_msat": 18446744073709551615,
"their_channel_reserve_satoshis": 30000,
"our_channel_reserve_satoshis": 30000,
"spendable_msatoshi": 2970000000,
"htlc_minimum_msat": 0,
"their_to_self_delay": 144,
"our_to_self_delay": 360,
"max_accepted_htlcs": 483,
"status": [
"AWAITING_UNILATERAL:Attempting to reconnect"
],
"in_payments_offered": 0,
"in_msatoshi_offered": 0,
"in_payments_fulfilled": 0,
"in_msatoshi_fulfilled": 0,
"out_payments_offered": 0,
"out_msatoshi_offered": 0,
"out_payments_fulfilled": 0,
"out_msatoshi_fulfilled": 0,
"htlcs": [
]
}
]
}, getinfo output
{
"id": "02af4ac85e83a5c6a7213090f9518e7054901e84746e10e473042ea1c230f2957e",
"num_peers": 11,
"num_pending_channels": 4,
"num_active_channels": 8,
"num_inactive_channels": 0,
"binding": [
{
"type": "ipv4",
"address": "0.0.0.0",
"port": 9735
}
],
"version": "v0.6.3",
"blockheight": 560525,
"network": "bitcoin",
}
Activity
cdecker commentedon Jan 30, 2019
This has been addressed going forward with PR #2274. In order to recover your coins I suggest updating to the latest
mastercommit, re-compile with./configure --enable-developerand then perform alightning-cli dev-rescan-outputs. That should get you your outputs back in sync withbitcoind. As for the channels I'm afraid they will never confirm (since the funding transaction will not confirm).I'd make sure that they don't confirm by withdrawing the funds to an external wallet (or even the internal
lightningdwallet could work). Once the funds have arrived in your external wallets we can check together which channels still stand a chance of being confirmed (and should be kept) and which channels can be forgotten usingdev-forget-channel.mloop1 commentedon Jan 30, 2019
Hi Christian, thanks a lot.
After re-compile and rescan I was indeed able to withdraw some funds.
I think I'm still confused with the concepts of "outputs" (whose value after withdraw is now almost zero when running listfunds) and satoshi in channels (which apparently didn't change after the withdrawal).
Any resource that explains this?
Let me know how can we progress from here in order to determine which channels should be kept and which forgotten.
cdecker commentedon Jan 31, 2019
Basically
outputsare on-chain funds, that have not been allocated to a channel (yet), and can be stocked up by transferring funds to an address created usingnewaddr, and can be spent usingwithdraw.channelsare to off-chain counterpart, meaning funds that are allocated to channels, and therefore to spend those funds you'll have to close the channels first (transitioning those funds to theoutputssection again, now being on-chain funds) and then transferring them away.It is worth noting however that your wallet is now a bit out of sync, and the channels that you see as
AWAITING_UNILATERALare likely double counting funds that are also inoutputsafter thedev-rescan-outputssince that marked them as not used in a channel, while still leaving the channel stub around.Now with a combination of the transactions in the
transactionstable in the DB we can go and see which outputs where spent by funding transactions of the stuck channels, and which we moved away, and if the outputs were successfully double-spent with the withdraw we candev-forget-channelthe stuck channels 😉mloop1 commentedon Jan 31, 2019
Thanks for your explanations, they really clarify things.
@SimonVrouwe indeed running
listfundsI still see the statusunconfimed.@cdecker indeed the channels that are in
AWAITING_UNILATERALare double counting funds that are also inoutputs.I've now run the following queries, I still need a bit of direction to understand how to determine which outputs where spent by funding transactions of the stuck channels, and which we moved away.
mloop1 commentedon Feb 11, 2019
Thanks @SimonVrouwe.
I managed to dev-forget the stuck channels.
For my curiosity was wondering about the meaning of some fields in the
outputtable: value, prev_out_index, key_index.cdecker commentedon Feb 11, 2019
Thanks for the update @mloop1, I'm glad you got everything unstuck. As for the fields you asked about:
prev_out_indexis the output index in the transaction that created the unspent transaction output. In combination with theprev_out_txit is used in transactions to reference where the funds came from (see Outpoint in the Bitcoin glossary)key_indexis part of the key derivation from the root secret. A value ofnbasically means that in order to spend this output we should use the nth keypair derived from the root secret.