Update documentation using deterministic generation

.github/workflows/ci.yaml

@@ -794,8 +794,7 @@ jobs:
           if-no-files-found: ignore
 
   update-docs-examples:
-    name: Update examples in doc schemas (disabled temporarily!)
-    if: false
+    name: Update examples in doc schemas
     runs-on: ubuntu-24.04
     timeout-minutes: 30
     strategy:
@@ -839,11 +838,57 @@ jobs:
           find . -type f -print0 | xargs -0 touch -d yesterday
           tar xaf cln-${{ matrix.CFG }}.tar.gz
 
+      - name: Debug check-doc-examples inputs
+        run: |
+          echo "== lightning-hsmtool =="
+          which lightning-hsmtool || echo "NOT ON PATH"
+          find . -name "lightning-hsmtool*" -not -path "./.git/*" 2>/dev/null
+
+          echo "== autogenerate-bitcoind-wallet.dat =="
+          ls -la tests/data/autogenerate-bitcoind-wallet.dat 2>&1
+
+          echo "== autogenerate-bitcoin-blocks.json =="
+          ls -la tests/data/autogenerate-bitcoin-blocks.json 2>&1
+          echo "-- number of blocks --"
+          python3 -c "import json; print(len(json.load(open('tests/data/autogenerate-bitcoin-blocks.json'))))" 2>&1
+          echo "-- first 300 chars --"
+          head -c 300 tests/data/autogenerate-bitcoin-blocks.json
+          echo ""
+
+          echo "== environment fingerprint =="
+          # Determinism discriminators: HAVE_USDT toggles tracing, which
+          # must not perturb the deterministic RNG stream.
+          grep -E 'HAVE_USDT|HAVE_SQLITE3' config.vars || true
+          bitcoind --version | head -1 || true
+          dpkg -l systemtap-sdt-dev 2>/dev/null | tail -1 || true
+
       - name: Test
         env:
           PYTEST_OPTS: ${{ env.PYTEST_OPTS_BASE }}
         run: |
           uv run eatmydata make -j $(($(nproc) + 1)) check-doc-examples CARGO=false CC=devtools/cc-nobuild SUPPRESS_GENERATION=1
+      - name: Save doc diff on failure
+        if: failure()
+        run: |
+          git diff -- doc > doc-examples.patch || true
+          git status --short -- doc
+      - name: Collect node logs on failure
+        if: failure()
+        run: |
+          mkdir -p ltests-debug
+          # pyln-testing leaves /tmp/ltests-* behind on failure: grab every
+          # node's daemon log and config (skip databases and sockets).
+          find /tmp/ltests-* -type f \( -name 'log' -o -name '*.log' -o -name 'config' \) \
+            -exec cp --parents -t ltests-debug {} + 2>/dev/null || true
+      - name: Upload debug artifacts
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: doc-examples-debug
+          path: |
+            doc-examples.patch
+            ltests-debug/
+          if-no-files-found: ignore
       - name: Upload test results
         if: always()
         uses: actions/upload-artifact@v7
@@ -952,6 +997,7 @@ jobs:
       - check-units
       - integration-valgrind
       - integration-sanitizers
+      - update-docs-examples
       - min-btc-support
       - check-downgrade
       - check-compiled-source

Makefile

@@ -648,19 +648,28 @@ check-amount-access:
 	@! (git grep -nE "(->|\.)(milli)?satoshis" -- "*.c" "*.h" ":(exclude)common/amount.*" ":(exclude)*/test/*" ":(exclude)tests/fuzz/*" | grep -v '/* Raw:')
 	@! git grep -nE "\\(struct amount_(m)?sat\\)" -- "*.c" "*.h" ":(exclude)common/amount.*" ":(exclude)*/test/*" ":(exclude)tests/fuzz/*" | grep -vE "sizeof.struct amount_(m)?sat."
 
+# Examples must be generated with this tree's binaries to use plugins and tools
+DOC_EXAMPLES_PATH = $(CURDIR)/lightningd:$(CURDIR)/cli:$(CURDIR)/tools:$(PATH)
+
 repeat-doc-examples:
 	@for i in $$(seq 1 $(n)); do \
+		PORT_OFFSET=$$((i * 100)); \
+		BASE_PORTNUM=$$((30000 + PORT_OFFSET)); \
 		echo "----------------------------------" >> tests/autogenerate-examples-repeat.log; \
-		echo "Iteration $$i" >> tests/autogenerate-examples-repeat.log; \
+		echo "Iteration $$i on Base Port $$BASE_PORTNUM" >> tests/autogenerate-examples-repeat.log; \
 		echo "----------------------------------" >> tests/autogenerate-examples-repeat.log; \
-		VALGRIND=0 TIMEOUT=40 TEST_DEBUG=1 GENERATE_EXAMPLES=1 pytest -vvv tests/autogenerate-rpc-examples.py; \
+		PATH="$(DOC_EXAMPLES_PATH)" TEST_DEBUG=1 VALGRIND=0 GENERATE_EXAMPLES=1 CLN_NEXT_VERSION=$(CLN_NEXT_VERSION) BASE_PORTNUM=$$BASE_PORTNUM pytest -vvv --timeout=1200 tests/autogenerate-rpc-examples.py; \
 		git diff >> tests/autogenerate-examples-repeat.log; \
 		git reset --hard; \
 		echo "----------------------------------" >> tests/autogenerate-examples-repeat.log; \
 	done
 
 update-doc-examples:
-	TEST_DEBUG=1 VALGRIND=0 GENERATE_EXAMPLES=1 $(PYTEST) $(PYTEST_OPTS) --timeout=1200 tests/autogenerate-rpc-examples.py && $(MAKE) $(MSGGEN_GEN_ALL)
+	PATH="$(DOC_EXAMPLES_PATH)" TEST_DEBUG=1 VALGRIND=0 GENERATE_EXAMPLES=1 CLN_NEXT_VERSION=$(CLN_NEXT_VERSION) $(PYTEST) $(PYTEST_OPTS) --timeout=1200 tests/autogenerate-rpc-examples.py && $(MAKE) $(MSGGEN_GEN_ALL)
+
+# If you changed tests/autogenerate-rpc-examples.py to require new blocks, you have to run this:
+update-doc-examples-newchain:
+	PATH="$(DOC_EXAMPLES_PATH)" TEST_DEBUG=1 VALGRIND=0 GENERATE_EXAMPLES=1 CLN_NEXT_VERSION=$(CLN_NEXT_VERSION) REGENERATE_BLOCKCHAIN=1 $(PYTEST) $(PYTEST_OPTS) --timeout=1200 tests/autogenerate-rpc-examples.py && $(MAKE) $(MSGGEN_GEN_ALL)
 
 check-doc-examples: update-doc-examples
 	git diff --exit-code HEAD -- doc

common/randbytes.c

@@ -9,6 +9,7 @@
 #include <common/utils.h>
 #include <sodium/randombytes.h>
 #include <stdlib.h>
+#include <string.h>
 #include <unistd.h>
 
 static bool used = false;
@@ -48,15 +49,23 @@ void randbytes_(void *bytes, size_t num_bytes, u64 *offset)
 }
 
 /* We want different seeds for each plugin (hence argv0), and for each
- * lightmingd instance, (hence seed from environment) */
+ * lightningd instance, (hence seed from environment) */
 void dev_override_randbytes(const char *argv0, long int seed)
 {
 	struct siphash_seed hashseed;
+	const char *base;
 	assert(!used);
 
+	/* Hash only the basename: binaries still get distinct seeds, but
+	* the stream no longer depends on the checkout path. Plugins and
+	* subdaemons are exec'd with absolute paths: hash only the basename,
+	* so the stream doesn't depend on where the source tree lives */
+	base = strrchr(argv0, '/');
+	base = base ? base + 1 : argv0;
+
 	hashseed.u.u64[0] = seed;
 	hashseed.u.u64[1] = 0;
 
-	dev_seed = siphash24(&hashseed, argv0, strlen(argv0));
+	dev_seed = siphash24(&hashseed, base, strlen(base));
 	assert(randbytes_overridden());
 }

common/test/Makefile

@@ -36,6 +36,7 @@ common/test/run-json:					\
 	common/lease_rates.o				\
 	common/node_id.o				\
 	common/pseudorand.o				\
+	common/randbytes.o			\
 	common/wireaddr.o				\
 	wire/fromwire.o					\
 	wire/onion_wiregen.o				\
@@ -49,6 +50,7 @@ common/test/run-route common/test/run-route-specific common/test/run-route-inflo
 	common/gossmap.o				\
 	common/node_id.o				\
 	common/pseudorand.o				\
+	common/randbytes.o				\
 	common/route.o					\
         common/gossip_store_wiregen.o			\
 	wire/fromwire.o					\
@@ -158,6 +160,7 @@ common/test/run-mkdatastorekey:				\
 	common/pseudorand.o
 
 common/test/run-htable:					\
+	common/randbytes.o				\
 	common/pseudorand.o
 
 common/test/run-shutdown_scriptpubkey: wire/towire.o wire/fromwire.o

common/test/run-bigsize.c

@@ -4,6 +4,7 @@
 #include <common/amount.h>
 #include <common/json_parse.c>
 #include <common/json_parse_simple.c>
+#include <common/randbytes.h>
 #include <common/setup.h>
 
 static const char *reason;

common/test/run-blindedpath_enctlv.c

@@ -3,6 +3,7 @@
 #include "../blinding.c"
 #include "../hmac.c"
 #include <common/channel_id.h>
+#include <common/randbytes.h>
 #include <common/setup.h>
 #include <stdio.h>
 

common/test/run-bolt11.c

@@ -10,6 +10,7 @@
 #include "../wire/fromwire.c"
 #include "../wire/towire.c"
 #include <ccan/err/err.h>
+#include <common/randbytes.h>
 #include <common/setup.h>
 #include <stdio.h>
 

common/test/run-bolt12-encode-test.c

@@ -12,6 +12,7 @@
 #include <common/bolt12.c>
 #include <common/bolt12_merkle.h>
 #include <common/features.c>
+#include <common/randbytes.h>
 #include <common/sciddir_or_pubkey.c>
 #include <common/setup.h>
 #include <inttypes.h>

common/test/run-bolt12-format-string-test.c

@@ -8,6 +8,7 @@
 #include <common/bolt12.h>
 #include <common/bolt12_merkle.h>
 #include <common/features.h>
+#include <common/randbytes.h>
 #include <common/setup.h>
 #include <common/utils.h>
 

common/test/run-bolt12-offer-decode.c

@@ -14,6 +14,7 @@
 #include <ccan/array_size/array_size.h>
 #include <ccan/tal/grab_file/grab_file.h>
 #include <ccan/tal/path/path.h>
+#include <common/randbytes.h>
 #include <common/setup.h>
 
 /* AUTOGENERATED MOCKS START */

common/test/run-bolt12_decode.c

@@ -8,6 +8,7 @@
 #include <ccan/json_escape/json_escape.h>
 #include <ccan/tal/grab_file/grab_file.h>
 #include <ccan/tal/path/path.h>
+#include <common/randbytes.h>
 #include <common/setup.h>
 
 /* AUTOGENERATED MOCKS START */

common/test/run-bolt12_merkle-json.c

@@ -12,6 +12,7 @@
 #include <ccan/tal/grab_file/grab_file.h>
 #include <ccan/tal/path/path.h>
 #include <common/channel_type.h>
+#include <common/randbytes.h>
 #include <common/setup.h>
 
 /* AUTOGENERATED MOCKS START */

common/test/run-bolt12_merkle.c

@@ -7,6 +7,7 @@
 #include <ccan/array_size/array_size.h>
 #include <common/channel_type.h>
 #include <common/features.h>
+#include <common/randbytes.h>
 #include <common/setup.h>
 #include <secp256k1_schnorrsig.h>
 

common/test/run-bolt12_period.c

@@ -5,6 +5,7 @@
 #include <ccan/array_size/array_size.h>
 #include <ccan/tal/grab_file/grab_file.h>
 #include <ccan/tal/path/path.h>
+#include <common/randbytes.h>
 #include <common/setup.h>
 
 /* AUTOGENERATED MOCKS START */

common/test/run-channel_type.c

@@ -2,6 +2,7 @@
 #include "../channel_type.c"
 #include "../features.c"
 #include <ccan/tal/str/str.h>
+#include <common/randbytes.h>
 #include <common/setup.h>
 #include <stdio.h>
 #include <lightningd/channel_state.h>

common/test/run-codex32.c

@@ -8,6 +8,7 @@
 #include <common/codex32.h>
 #include <ccan/tal/grab_file/grab_file.h>
 #include <ccan/tal/path/path.h>
+#include <common/randbytes.h>
 #include <common/setup.h>
 #include <wire/wire.h>
 

common/test/run-coin_mvt.c

@@ -1,6 +1,7 @@
 #include "config.h"
 #include "../coin_mvt.c"
 #include <ccan/tal/str/str.h>
+#include <common/randbytes.h>
 #include <common/setup.h>
 #include <stdio.h>
 

common/test/run-cryptomsg.c

@@ -1,5 +1,6 @@
 #include "config.h"
 #include <ccan/str/hex/hex.h>
+#include <common/randbytes.h>
 #include <common/setup.h>
 #include <common/status.h>
 #include <common/utils.h>

common/test/run-htable.c

@@ -81,6 +81,15 @@ u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED)
 /* Generated stub for fromwire_u8_array */
 void fromwire_u8_array(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, u8 *arr UNNEEDED, size_t num UNNEEDED)
 { fprintf(stderr, "fromwire_u8_array called!\n"); abort(); }
+/* Generated stub for memleak_add_helper_ */
+void memleak_add_helper_(const tal_t *p UNNEEDED, void (*cb)(struct htable *memtable UNNEEDED,
+						    const tal_t *)){ }
+/* Generated stub for memleak_scan_htable */
+void memleak_scan_htable(struct htable *memtable UNNEEDED, const struct htable *ht UNNEEDED)
+{ fprintf(stderr, "memleak_scan_htable called!\n"); abort(); }
+/* Generated stub for notleak_ */
+void *notleak_(void *ptr UNNEEDED, bool plus_children UNNEEDED)
+{ fprintf(stderr, "notleak_ called!\n"); abort(); }
 /* Generated stub for towire */
 void towire(u8 **pptr UNNEEDED, const void *data UNNEEDED, size_t len UNNEEDED)
 { fprintf(stderr, "towire called!\n"); abort(); }

common/test/run-json_filter.c

@@ -6,6 +6,7 @@
 #include "../json_stream.c"
 #include <ccan/json_out/json_out.h>
 #include <common/json_command.h>
+#include <common/randbytes.h>
 #include <common/setup.h>
 #include <stdio.h>
 

common/test/run-json_remove.c

@@ -2,6 +2,7 @@
 #include <common/amount.h>
 #include <common/json_param.c>
 #include <common/json_parse_simple.c>
+#include <common/randbytes.h>
 #include <common/setup.h>
 #include <stdio.h>
 #include <wire/wire.h>

common/test/run-json_scan.c

@@ -2,6 +2,7 @@
 #include "../json_parse.c"
 #include "../json_parse_simple.c"
 #include <common/amount.h>
+#include <common/randbytes.h>
 #include <common/setup.h>
 #include <stdio.h>
 #include <wire/wire.h>

common/test/run-json_stream-filter.c

@@ -4,6 +4,7 @@
 #include <assert.h>
 #include <ccan/tal/str/str.h>
 #include <common/channel_type.h>
+#include <common/randbytes.h>
 #include <common/setup.h>
 #include <inttypes.h>
 #include <stdio.h>

common/test/run-lease_rates.c

@@ -1,6 +1,7 @@
 #include "config.h"
 #include "../amount.c"
 #include "../lease_rates.c"
+#include <common/randbytes.h>
 #include <common/setup.h>
 #include <stdio.h>
 

common/test/run-psbt_diff.c

@@ -1,4 +1,5 @@
 #include "config.h"
+#include <common/randbytes.h>
 #include <common/setup.h>
 #include <stdio.h>
 #include "../amount.c"

common/test/run-route-infloop.c

@@ -32,6 +32,15 @@ bool fromwire_tlv(const u8 **cursor UNNEEDED, size_t *max UNNEEDED,
 		  void *record UNNEEDED, struct tlv_field **fields UNNEEDED,
 		  const u64 *extra_types UNNEEDED, size_t *err_off UNNEEDED, u64 *err_type UNNEEDED)
 { fprintf(stderr, "fromwire_tlv called!\n"); abort(); }
+/* Generated stub for memleak_add_helper_ */
+void memleak_add_helper_(const tal_t *p UNNEEDED, void (*cb)(struct htable *memtable UNNEEDED,
+						    const tal_t *)){ }
+/* Generated stub for memleak_scan_htable */
+void memleak_scan_htable(struct htable *memtable UNNEEDED, const struct htable *ht UNNEEDED)
+{ fprintf(stderr, "memleak_scan_htable called!\n"); abort(); }
+/* Generated stub for notleak_ */
+void *notleak_(void *ptr UNNEEDED, bool plus_children UNNEEDED)
+{ fprintf(stderr, "notleak_ called!\n"); abort(); }
 /* Generated stub for sciddir_or_pubkey_from_node_id */
 bool sciddir_or_pubkey_from_node_id(struct sciddir_or_pubkey *sciddpk UNNEEDED,
 				    const struct node_id *node_id UNNEEDED)

common/test/run-route_blinding_test.c

@@ -13,6 +13,7 @@
 #include <common/clock_time.h>
 #include <common/json_parse.h>
 #include <common/json_stream.h>
+#include <common/randbytes.h>
 #include <common/setup.h>
 #include <common/wireaddr.h>
 #include <stdio.h>

common/test/run-splice_script.c

@@ -6,6 +6,7 @@
 #include "../json_stream.c"
 #include "../json_parse_simple.c"
 #include <bitcoin/chainparams.h>
+#include <common/randbytes.h>
 #include <common/setup.h>
 #include <common/splice_script.h>
 #include <assert.h>

common/test/run-tlv_span.c

@@ -2,6 +2,7 @@
 #include "../bolt12.c"
 #include "../bigsize.c"
 #include "../../wire/fromwire.c"
+#include <common/randbytes.h>
 #include <common/setup.h>
 #include <stdio.h>
 

common/test/run-tlv_unknown.c

@@ -5,6 +5,7 @@
 #include "../../wire/towire.c"
 #include "../bigsize.c"
 #include "../bolt12.c"
+#include <common/randbytes.h>
 #include <common/setup.h>
 #include <stdio.h>