Backport 0.7: Widen rusqlite compatibility, use manual downloads

.github/workflows/rust.yml

@@ -7,6 +7,26 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  rusqlite-compat:
+    runs-on: self-hosted
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        # Keep these endpoints in sync with the supported range in Cargo.toml.
+        rusqlite: ["0.31.0", "0.39.0"]
+    env:
+      CARGO_RESOLVER_INCOMPATIBLE_RUST_VERSIONS: fallback
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install MSRV toolchain
+        run: rustup toolchain install 1.85.0 --profile minimal
+      - name: Test downstream rusqlite compatibility on MSRV
+        run: |
+          export CARGO_TARGET_DIR=$(mktemp -d /tmp/cargo-target-ldk-node-rusqlite-XXXXXX)
+          trap 'rm -rf "$CARGO_TARGET_DIR"' EXIT
+          cargo +1.85.0 add --manifest-path tests/rusqlite-compat/Cargo.toml rusqlite@=${{ matrix.rusqlite }}
+          cargo +1.85.0 test --manifest-path tests/rusqlite-compat/Cargo.toml
+
   build:
     defaults:
       run:
@@ -107,8 +127,8 @@ jobs:
       - name: Test on Rust ${{ matrix.toolchain }}
         if: "matrix.platform != 'windows-latest'"
         run: |
-          RUSTFLAGS="--cfg no_download" cargo test
+          cargo test
       - name: Test with UniFFI support on Rust ${{ matrix.toolchain }}
         if: "matrix.platform != 'windows-latest' && matrix.build-uniffi"
         run: |
-          RUSTFLAGS="--cfg no_download" cargo test --features uniffi
+          cargo test --features uniffi

.github/workflows/vss-integration.yml

@@ -36,6 +36,30 @@ jobs:
           repository: lightningdevkit/vss-server
           path: vss-server
 
+      - name: Enable caching for bitcoind
+        id: cache-bitcoind
+        uses: actions/cache@v4
+        with:
+          path: bin/bitcoind-${{ runner.os }}-${{ runner.arch }}
+          key: bitcoind-${{ runner.os }}-${{ runner.arch }}
+      - name: Enable caching for electrs
+        id: cache-electrs
+        uses: actions/cache@v4
+        with:
+          path: bin/electrs-${{ runner.os }}-${{ runner.arch }}
+          key: electrs-${{ runner.os }}-${{ runner.arch }}
+      - name: Download bitcoind/electrs
+        if: "steps.cache-bitcoind.outputs.cache-hit != 'true' || steps.cache-electrs.outputs.cache-hit != 'true'"
+        run: |
+          source ./ldk-node/scripts/download_bitcoind_electrs.sh
+          mkdir -p bin
+          mv "$BITCOIND_EXE" bin/bitcoind-${{ runner.os }}-${{ runner.arch }}
+          mv "$ELECTRS_EXE" bin/electrs-${{ runner.os }}-${{ runner.arch }}
+      - name: Set bitcoind/electrs environment variables
+        run: |
+          echo "BITCOIND_EXE=$( pwd )/bin/bitcoind-${{ runner.os }}-${{ runner.arch }}" >> "$GITHUB_ENV"
+          echo "ELECTRS_EXE=$( pwd )/bin/electrs-${{ runner.os }}-${{ runner.arch }}" >> "$GITHUB_ENV"
+
       - name: Build and Deploy VSS Server
         run: |
           version=$(cargo metadata --no-deps --format-version 1 --manifest-path ldk-node/Cargo.toml | jq -r '.packages[] | select(.name == "ldk-node") | .version')

Cargo.toml

@@ -48,7 +48,8 @@ bdk_wallet = { version = "2.2.0", default-features = false, features = ["std", "
 
 reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
 rustls = { version = "0.23", default-features = false }
-rusqlite = { version = "0.31.0", features = ["bundled"] }
+# rusqlite 0.40.0 and 0.40.1 do not compile on our MSRV.
+rusqlite = { version = ">=0.31.0, <0.40.0", features = ["bundled"] }
 bitcoin = "0.32.7"
 bip39 = { version = "2.0.0", features = ["rand"] }
 bip21 = { version = "0.5", features = ["std"], default-features = false }
@@ -77,10 +78,8 @@ proptest = "1.0.0"
 regex = "1.5.6"
 criterion = { version = "0.7.0", features = ["async_tokio"] }
 
-[target.'cfg(not(no_download))'.dev-dependencies]
-electrsd = { version = "0.36.1", default-features = false, features = ["legacy", "esplora_a33e97e1", "corepc-node_27_2"] }
-
-[target.'cfg(no_download)'.dev-dependencies]
+# Test binaries are provided via scripts/download_bitcoind_electrs.sh.
+# Keep automatic downloads disabled to avoid the yanked zip dependency.
 electrsd = { version = "0.36.1", default-features = false, features = ["legacy"] }
 corepc-node = { version = "0.10.0", default-features = false, features = ["27_2"] }
 

README.md

@@ -66,6 +66,20 @@ LDK Node itself is written in [Rust][rust] and may therefore be natively added a
 ## MSRV
 The Minimum Supported Rust Version (MSRV) is currently 1.85.0.
 
+## Running Tests
+
+Rust integration tests and benchmarks require Bitcoin Core and Esplora-enabled `electrs` binaries.
+On Linux and macOS, run the following from the repository root in Bash:
+
+```bash
+source ./scripts/download_bitcoind_electrs.sh
+RUSTFLAGS="--cfg tokio_unstable" cargo test
+```
+
+The script requires `curl`, `unzip`, `tar`, and `shasum`. It verifies the downloads and exports
+`BITCOIND_EXE` and `ELECTRS_EXE` for the current shell. Alternatively, set these variables to
+compatible binaries you already have. Cargo's automatic binary downloads are disabled.
+
 [api_docs]: https://docs.rs/ldk-node/*/ldk_node/
 [api_docs_node]: https://docs.rs/ldk-node/*/ldk_node/struct.Node.html
 [api_docs_builder]: https://docs.rs/ldk-node/*/ldk_node/struct.Builder.html

src/io/sqlite_store/migrations.rs

@@ -52,14 +52,13 @@ pub(super) fn migrate_schema(
 		})?;
 
 		// Update user_version
-		tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", to_version, |_| Ok(()))
-			.map_err(|e| {
-				let msg = format!(
-					"Failed to upgrade user_version from {} to {}: {}",
-					from_version, to_version, e
-				);
-				io::Error::new(io::ErrorKind::Other, msg)
-			})?;
+		tx.pragma_update(None, "user_version", to_version).map_err(|e| {
+			let msg = format!(
+				"Failed to upgrade user_version from {} to {}: {}",
+				from_version, to_version, e
+			);
+			io::Error::new(io::ErrorKind::Other, msg)
+		})?;
 
 		tx.commit().map_err(|e| {
 			let msg = format!(
@@ -104,14 +103,7 @@ mod tests {
 
 			let connection = Connection::open(db_file_path.clone()).unwrap();
 
-			connection
-				.pragma(
-					Some(rusqlite::DatabaseName::Main),
-					"user_version",
-					old_schema_version,
-					|_| Ok(()),
-				)
-				.unwrap();
+			connection.pragma_update(None, "user_version", old_schema_version).unwrap();
 
 			let sql = format!(
 				"CREATE TABLE IF NOT EXISTS {} (

src/io/sqlite_store/mod.rs

@@ -260,17 +260,10 @@ impl SqliteStoreInner {
 
 		if version_res == 0 {
 			// New database, set our SCHEMA_USER_VERSION and continue
-			connection
-				.pragma(
-					Some(rusqlite::DatabaseName::Main),
-					"user_version",
-					SCHEMA_USER_VERSION,
-					|_| Ok(()),
-				)
-				.map_err(|e| {
-					let msg = format!("Failed to set PRAGMA user_version: {}", e);
-					io::Error::new(io::ErrorKind::Other, msg)
-				})?;
+			connection.pragma_update(None, "user_version", SCHEMA_USER_VERSION).map_err(|e| {
+				let msg = format!("Failed to set PRAGMA user_version: {}", e);
+				io::Error::new(io::ErrorKind::Other, msg)
+			})?;
 		} else if version_res < SCHEMA_USER_VERSION {
 			migrations::migrate_schema(
 				&mut connection,

tests/common/mod.rs

@@ -187,19 +187,15 @@ macro_rules! expect_payment_successful_event {
 pub(crate) use expect_payment_successful_event;
 
 pub(crate) fn setup_bitcoind_and_electrsd() -> (BitcoinD, ElectrsD) {
-	let bitcoind_exe =
-		env::var("BITCOIND_EXE").ok().or_else(|| corepc_node::downloaded_exe_path().ok()).expect(
-			"you need to provide an env var BITCOIND_EXE or specify a bitcoind version feature",
-		);
+	let bitcoind_exe = env::var("BITCOIND_EXE")
+		.expect("set BITCOIND_EXE or source ./scripts/download_bitcoind_electrs.sh before testing");
 	let mut bitcoind_conf = corepc_node::Conf::default();
 	bitcoind_conf.network = "regtest";
 	bitcoind_conf.args.push("-rest");
 	let bitcoind = BitcoinD::with_conf(bitcoind_exe, &bitcoind_conf).unwrap();
 
 	let electrs_exe = env::var("ELECTRS_EXE")
-		.ok()
-		.or_else(electrsd::downloaded_exe_path)
-		.expect("you need to provide env var ELECTRS_EXE or specify an electrsd version feature");
+		.expect("set ELECTRS_EXE or source ./scripts/download_bitcoind_electrs.sh before testing");
 	let mut electrsd_conf = electrsd::Conf::default();
 	electrsd_conf.http_enabled = true;
 	electrsd_conf.network = "regtest";

tests/rusqlite-compat/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "ldk-node-rusqlite-compat"
+version = "0.0.0"
+edition = "2021"
+rust-version = "1.85"
+publish = false
+
+# Resolve as a downstream application, without LDK Node's legacy test dependencies.
+[workspace]
+
+[dependencies]
+ldk-node = { path = "../.." }
+rusqlite = "=0.39.0"
+tokio = { version = "1.39", features = ["macros", "rt"] }

tests/rusqlite-compat/README.md

@@ -0,0 +1,32 @@
+# Downstream rusqlite compatibility
+
+This separate Cargo workspace checks that an application can select a stock
+rusqlite release while using LDK Node's default features. The test writes through
+LDK Node, reopens the store, and reads the database with the application's rusqlite.
+Neither rusqlite nor libsqlite3-sys is patched.
+
+The root crate's development dependencies on LDK Node 0.6.2 and 0.7.0 require
+rusqlite 0.31 unconditionally. They constrain root builds even when individual
+tests are filtered out, but do not participate in downstream resolution. Keep
+this fixture separate so CI actually tests both ends of the supported range.
+
+From the repository root, run:
+
+```sh
+export CARGO_TARGET_DIR=$(mktemp -d /tmp/cargo-target-ldk-node-rusqlite-XXXXXX)
+export CARGO_RESOLVER_INCOMPATIBLE_RUST_VERSIONS=fallback
+cargo +1.85.0 test --manifest-path tests/rusqlite-compat/Cargo.toml
+```
+
+The fixture defaults to the maximum tested version. To test another release,
+change its direct requirement with
+`cargo +1.85.0 add --manifest-path tests/rusqlite-compat/Cargo.toml rusqlite@=0.31.0`
+and rerun the test. Restore the maximum requirement afterwards. The resolver
+setting selects MSRV-compatible versions of unrelated transitive dependencies.
+
+When changing the supported range, update both endpoints in
+`.github/workflows/rust.yml` and test each newly included release locally. The
+current upper bound excludes 0.40.0 and 0.40.1, which use `cfg_select!` and fail to
+compile on Rust 1.85. Although the bundled-feature probe for 0.40.2 passes on
+1.85, Cargo cannot express a disjoint requirement that admits it while excluding
+those earlier patches and retaining the older release lines.

tests/rusqlite-compat/src/lib.rs

@@ -0,0 +1,40 @@
+// This file is Copyright its original authors, visible in version control history.
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
+// http://opensource.org/licenses/MIT>, at your option. You may not use this file except in
+// accordance with one or both of these licenses.
+
+#[cfg(test)]
+mod tests {
+	use ldk_node::io::sqlite_store::{
+		SqliteStore, DEFAULT_KV_TABLE_NAME, DEFAULT_SQLITE_DB_FILE_NAME,
+	};
+	use ldk_node::lightning::util::persist::KVStore;
+
+	#[tokio::test]
+	async fn persists_with_the_applications_rusqlite() {
+		let data_dir =
+			std::env::temp_dir().join(format!("ldk-node-rusqlite-compat-{}", std::process::id()));
+		let store = SqliteStore::new(data_dir.clone(), None, None).unwrap();
+		KVStore::write(&store, "test", "", "key", vec![42]).await.unwrap();
+		drop(store);
+
+		let store = SqliteStore::new(data_dir.clone(), None, None).unwrap();
+		assert_eq!(KVStore::read(&store, "test", "", "key").await.unwrap(), vec![42]);
+		drop(store);
+
+		let connection =
+			rusqlite::Connection::open(data_dir.join(DEFAULT_SQLITE_DB_FILE_NAME)).unwrap();
+		let value: Vec<u8> = connection
+			.query_row(
+				&format!("SELECT value FROM {DEFAULT_KV_TABLE_NAME} WHERE key = 'key'"),
+				[],
+				|row| row.get(0),
+			)
+			.unwrap();
+		assert_eq!(value, vec![42]);
+		drop(connection);
+		std::fs::remove_dir_all(data_dir).unwrap();
+	}
+}