Skip to content

Commit c5b9fe4

Browse files
exd02exd02
committed
contrib: add script to generate hints periodically
1 parent 6c241ca commit c5b9fe4

2 files changed

Lines changed: 375 additions & 0 deletions

File tree

‎contrib/README.md‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
# Contrib
2+
3+
This directory contains helper scripts and examples for operating a hintsfile server.
4+
They are not required by the server itself but may be useful for automating hints generation and publication.
5+
6+
---
7+
8+
## `generate_hints.sh`
9+
10+
This script generates UTXO hint files using `bitcoin-cli generatetxohints`, it:
11+
12+
1. Queries `getblockchaininfo` to obtain the current chain, block height, and best block hash.
13+
2. Generates a `.hints` file.
14+
3. Optionally produces compressed variants (`raw`, `gzip`, `xz`, `zstd`, or `zip`).
15+
4. Publishes the files into format-specific directories.
16+
5. Updates symlinks pointing to the most recent version.
17+
18+
Versioned files are preserved so older hints remain available.
19+
20+
Example versioned file:
21+
22+
```
23+
main_860000_ab12cd34.hints
24+
```
25+
26+
Each format lives in its own directory:
27+
28+
```
29+
~/hints/
30+
├── raw/
31+
├── gzip/
32+
├── xz/
33+
├── zstd/
34+
└── zip/
35+
```
36+
37+
Inside each directory the script keeps:
38+
39+
- a **versioned file**
40+
- a **chain symlink**
41+
- (for mainnet) a **bitcoin symlink**
42+
43+
Example (`gzip`):
44+
45+
```
46+
~/hints/gzip
47+
├── main_860000_ab12cd34.hints.gzip
48+
├── main.hints.gzip -> main_860000_ab12cd34.hints.gzip
49+
└── bitcoin.hints.gzip -> main_860000_ab12cd34.hints.gzip
50+
```
51+
52+
The same layout applies to `raw`, `xz`, `zstd`, and `zip`.
53+
54+
---
55+
56+
## Requirements
57+
58+
- `bitcoind` running with RPC enabled
59+
- `bitcoin-cli`
60+
- The `generatetxohints` RPC available in your node
61+
- Compression tools depending on the formats you want to generate:
62+
- `gzip`
63+
- `xz`
64+
- `zstd`
65+
- `zip`
66+
67+
---
68+
69+
## Running the script
70+
71+
Make the script executable:
72+
73+
```
74+
chmod +x generate_hints.sh
75+
```
76+
77+
Run with helper:
78+
79+
```
80+
./generate_hints.sh --help
81+
```
82+
83+
---
84+
85+
## Automating with systemd
86+
87+
A common setup is to run the script periodically using a `systemd` service and timer.
88+
89+
### Service
90+
91+
Create `/etc/systemd/system/bitcoin-hints.service`:
92+
93+
```
94+
[Unit]
95+
Description=Generate Bitcoin Hintsfile
96+
After=bitcoind.service
97+
98+
[Service]
99+
Type=oneshot
100+
User=YOUR_LINUX_USER_HERE
101+
ExecStart=/PATH_TO_YOUR_SCRIPT/generate_hints.sh raw gzip xz
102+
TimeoutStartSec=800min
103+
104+
StandardOutput=journal
105+
StandardError=journal
106+
```
107+
108+
---
109+
110+
### Timer
111+
112+
Create `/etc/systemd/system/bitcoin-hints.timer`:
113+
114+
```
115+
[Unit]
116+
Description=Bitcoin Hintsfile Generator Timer
117+
118+
[Timer]
119+
OnCalendar=Sat 20:00
120+
Persistent=true
121+
122+
[Install]
123+
WantedBy=timers.target
124+
```
125+
126+
---
127+
128+
### Enable the timer
129+
130+
Reload systemd and enable the timer:
131+
132+
```
133+
sudo systemctl daemon-reload
134+
sudo systemctl enable --now bitcoin-hints.timer
135+
```
136+
137+
Check status:
138+
139+
```
140+
systemctl list-timers
141+
```

‎contrib/generate_hints.sh‎

Lines changed: 234 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,234 @@
1+
#!/usr/bin/env bash
2+
3+
# This script generates UTXO hint files using bitcoin-cli generatetxohints.
4+
# The output filename includes the current chain name, block height, and a
5+
# suffix of the best block hash obtained from getblockchaininfo, allowing
6+
# versioned hint files to be kept for historical reference.
7+
#
8+
# The script can optionally generate multiple output formats: raw, gzip,
9+
# xz, zstd, or zip. Each format is written to its own directory under
10+
# ~/hints/<format>/.
11+
#
12+
# For each generated file, the script updates symlinks pointing to the
13+
# latest versioned file (e.g., main.hints.gzip -> main_900000_deadbeef.hints.gzip).
14+
# For the main chain, an additional bitcoin.* symlink is also created
15+
# (e.g., bitcoin.hints.gzip -> main_900000_deadbeef.hints.gzip).
16+
#
17+
# This script can be used together with a systemd service and a periodically
18+
# systemd timer to automatically regenerate and publish updated hint files.
19+
20+
set -euo pipefail
21+
umask 022
22+
23+
BITCOINCLI="$HOME/dev/bitcoin/build/bin/bitcoin-cli"
24+
# Final hints directory. Files here are intended to be exposed for download
25+
# (e.g., via HTTP). Each compression format lives in its own subdirectory.
26+
HINT_DIR="$HOME/hints"
27+
# Temporary workspace where generatetxohints writes the raw hints file
28+
# before it is copied/compressed into HINT_DIR.
29+
NEW_HINT_DIR="$HOME/hints_new"
30+
31+
usage() {
32+
echo "Usage: $0 [raw gzip xz zstd zip]"
33+
echo
34+
echo "Examples:"
35+
echo " $0 raw"
36+
echo " -> generates raw only, e.g.:"
37+
echo " ~/hints/raw"
38+
echo " ├── main_100_deadbeef.hints"
39+
echo " ├── main.hints -> main_100_deadbeef.hints"
40+
echo " └── bitcoin.hints -> main_100_deadbeef.hints"
41+
echo
42+
echo " $0 gzip"
43+
echo " -> generates gzip only, e.g.:"
44+
echo " ~/hints/gzip"
45+
echo " ├── main_100_deadbeef.hints.gzip"
46+
echo " ├── main.hints.gzip -> main_100_deadbeef.hints.gzip"
47+
echo " └── bitcoin.hints.gzip -> main_100_deadbeef.hints.gzip"
48+
echo
49+
echo " $0 raw gzip xz"
50+
echo " -> generates raw, gzip and xz, e.g.:"
51+
echo " ~/hints/raw"
52+
echo " ├── main_100_deadbeef.hints"
53+
echo " ├── main.hints -> main_100_deadbeef.hints"
54+
echo " └── bitcoin.hints -> main_100_deadbeef.hints"
55+
echo " ~/hints/gzip"
56+
echo " ├── main_100_deadbeef.hints.gzip"
57+
echo " ├── main.hints.gzip -> main_100_deadbeef.hints.gzip"
58+
echo " └── bitcoin.hints.gzip -> main_100_deadbeef.hints.gzip"
59+
echo " ~/hints/xz"
60+
echo " ├── main_100_deadbeef.hints.xz"
61+
echo " ├── main.hints.xz -> main_100_deadbeef.hints.xz"
62+
echo " └── bitcoin.hints.xz -> main_100_deadbeef.hints.xz"
63+
echo
64+
echo "Output layout:"
65+
echo " ~/hints/raw/<chain>.hints"
66+
echo " ~/hints/gzip/<chain>.hints.gzip"
67+
echo " ~/hints/xz/<chain>.hints.xz"
68+
echo " ~/hints/zstd/<chain>.hints.zst"
69+
echo " ~/hints/zip/<chain>.hints.zip"
70+
}
71+
72+
parse_args() {
73+
if [ $# -eq 0 ]; then
74+
METHODS=("raw")
75+
return
76+
fi
77+
for arg in "$@"; do
78+
case "$arg" in
79+
raw|gzip|xz|zstd|zip)
80+
METHODS+=("$arg")
81+
;;
82+
help|-h|--help)
83+
usage
84+
exit 0
85+
;;
86+
*)
87+
echo "Unsupported option: $arg" >&2
88+
usage
89+
exit 1
90+
;;
91+
esac
92+
done
93+
}
94+
95+
get_blockchain_info() {
96+
BLOCKCHAININFO_OUTPUT=$("$BITCOINCLI" getblockchaininfo 2>&1) || {
97+
echo "ERROR: bitcoind not reachable via RPC" >&2
98+
exit 1
99+
}
100+
101+
CHAIN_NAME=$(echo "$BLOCKCHAININFO_OUTPUT" | grep '"chain"' | sed 's/.*"chain": *"\([^"]*\)".*/\1/')
102+
LATEST_BLOCK_HEIGHT=$(echo "$BLOCKCHAININFO_OUTPUT" | grep '"blocks"' | sed 's/.*"blocks": *\([0-9]*\).*/\1/')
103+
BEST_BLOCK_HASH=$(echo "$BLOCKCHAININFO_OUTPUT" | grep '"bestblockhash"' | sed 's/.*"bestblockhash": *"\([^"]*\)".*/\1/')
104+
105+
# Use latest block height minus 10 to avoid stale blocks
106+
BLOCK_HEIGHT=$((LATEST_BLOCK_HEIGHT - 10))
107+
HASH_SUFFIX="${BEST_BLOCK_HASH: -8}"
108+
109+
# Generate a unique hintfile name, e.g.: main_101_deadbeef
110+
VERSION_TAG="${CHAIN_NAME}_${BLOCK_HEIGHT}_${HASH_SUFFIX}"
111+
}
112+
113+
# Clear NEW_HINT_DIR because generatetxohints fails if an .incomplete
114+
# file from a previous run exists in the directory.
115+
prepare_workspace() {
116+
rm -rf "$NEW_HINT_DIR"
117+
mkdir -p "$NEW_HINT_DIR"
118+
mkdir -p "$HINT_DIR"
119+
}
120+
121+
generate_raw_hints() {
122+
local final_hint="$NEW_HINT_DIR/${VERSION_TAG}.hints"
123+
"$BITCOINCLI" --rpcclienttimeout=0 generatetxohints "$final_hint" "$BLOCK_HEIGHT" >/dev/null
124+
echo "$final_hint"
125+
}
126+
127+
publish_raw() {
128+
local src="$1"
129+
local dir="$HINT_DIR/raw"
130+
local versioned="${VERSION_TAG}.hints"
131+
132+
mkdir -p "$dir"
133+
cp "$src" "$dir/$versioned"
134+
ln -sf "$versioned" "$dir/${CHAIN_NAME}.hints"
135+
136+
if [ "$CHAIN_NAME" = "main" ]; then
137+
ln -sf "$versioned" "$dir/bitcoin.hints"
138+
fi
139+
}
140+
141+
publish_gzip() {
142+
local src="$1"
143+
local dir="$HINT_DIR/gzip"
144+
local versioned="${VERSION_TAG}.hints.gzip"
145+
146+
mkdir -p "$dir"
147+
gzip -c "$src" > "$dir/$versioned"
148+
ln -sf "$versioned" "$dir/${CHAIN_NAME}.hints.gzip"
149+
150+
if [ "$CHAIN_NAME" = "main" ]; then
151+
ln -sf "$versioned" "$dir/bitcoin.hints.gzip"
152+
fi
153+
}
154+
155+
publish_xz() {
156+
local src="$1"
157+
local dir="$HINT_DIR/xz"
158+
local versioned="${VERSION_TAG}.hints.xz"
159+
160+
mkdir -p "$dir"
161+
xz -c "$src" > "$dir/$versioned"
162+
ln -sf "$versioned" "$dir/${CHAIN_NAME}.hints.xz"
163+
164+
if [ "$CHAIN_NAME" = "main" ]; then
165+
ln -sf "$versioned" "$dir/bitcoin.hints.xz"
166+
fi
167+
}
168+
169+
publish_zstd() {
170+
local src="$1"
171+
local dir="$HINT_DIR/zstd"
172+
local versioned="${VERSION_TAG}.hints.zst"
173+
174+
mkdir -p "$dir"
175+
zstd -q -c "$src" > "$dir/$versioned"
176+
ln -sf "$versioned" "$dir/${CHAIN_NAME}.hints.zst"
177+
178+
if [ "$CHAIN_NAME" = "main" ]; then
179+
ln -sf "$versioned" "$dir/bitcoin.hints.zst"
180+
fi
181+
}
182+
183+
publish_zip() {
184+
local src="$1"
185+
local dir="$HINT_DIR/zip"
186+
local versioned="${VERSION_TAG}.hints.zip"
187+
188+
mkdir -p "$dir"
189+
tmp="$NEW_HINT_DIR/tmp.zip"
190+
(cd "$NEW_HINT_DIR" && zip -q "$tmp" "$(basename "$src")")
191+
mv "$tmp" "$dir/$versioned"
192+
193+
ln -sf "$versioned" "$dir/${CHAIN_NAME}.hints.zip"
194+
195+
if [ "$CHAIN_NAME" = "main" ]; then
196+
ln -sf "$versioned" "$dir/bitcoin.hints.zip"
197+
fi
198+
}
199+
200+
publish_method() {
201+
local method="$1"
202+
local src="$2"
203+
204+
case "$method" in
205+
raw) publish_raw "$src" ;;
206+
gzip) publish_gzip "$src" ;;
207+
xz) publish_xz "$src" ;;
208+
zstd) publish_zstd "$src" ;;
209+
zip) publish_zip "$src" ;;
210+
esac
211+
}
212+
213+
main() {
214+
METHODS=()
215+
parse_args "$@"
216+
217+
echo "=== $(date) starting generatetxohints ==="
218+
219+
prepare_workspace
220+
get_blockchain_info
221+
222+
echo "Chain: $CHAIN_NAME | Latest block: $LATEST_BLOCK_HEIGHT | Hint block: $BLOCK_HEIGHT | Hash suffix: $HASH_SUFFIX"
223+
echo "Selected methods: ${METHODS[*]}"
224+
225+
RAW_FILE=$(generate_raw_hints)
226+
227+
for method in "${METHODS[@]}"; do
228+
publish_method "$method" "$RAW_FILE"
229+
done
230+
231+
echo "=== $(date) finished ==="
232+
}
233+
234+
main "$@"

0 commit comments

Comments
 (0)