From c7f2eb4bb9f40906a49d9ff92d664a3aab40bb73 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 14 Jul 2025 11:55:35 +0200 Subject: [PATCH 01/19] chore(PocketIC): load time from latest state instead of PocketIC metadata --- packages/pocket-ic/tests/tests.rs | 30 +++++++++- rs/pocket_ic_server/src/pocket_ic.rs | 82 +++++++++------------------- 2 files changed, 55 insertions(+), 57 deletions(-) diff --git a/packages/pocket-ic/tests/tests.rs b/packages/pocket-ic/tests/tests.rs index 00698db6b533..19053076bddf 100644 --- a/packages/pocket-ic/tests/tests.rs +++ b/packages/pocket-ic/tests/tests.rs @@ -29,7 +29,11 @@ use sha2::{Digest, Sha256}; use std::collections::BTreeMap; #[cfg(windows)] use std::net::{IpAddr, Ipv4Addr, SocketAddr}; -use std::{io::Read, sync::OnceLock, time::SystemTime}; +use std::{ + io::Read, + sync::OnceLock, + time::{Duration, SystemTime}, +}; use tempfile::{NamedTempFile, TempDir}; #[cfg(windows)] use wslpath::windows_to_wsl; @@ -430,6 +434,30 @@ fn set_time_into_past() { pic.set_time(now.into()); } +#[test] +fn time_on_resumed_instance() { + let state = PocketIcState::new(); + + let pic = PocketIcBuilder::new() + .with_application_subnet() + .with_state(state) + .build(); + + let now = SystemTime::now(); + pic.set_certified_time(now.into()); + + let time = pic.get_time(); + let state = pic.drop_and_take_state().unwrap(); + + let pic = PocketIcBuilder::new().with_state(state).build(); + + // the time on the resumed instances increases by 2ns: + // - 1ns due to executing a checkpointed round before dropping the original instance; + // - 1ns due to bumping time when creating a new instance to ensure strict time monotonicity. + let resumed_time = pic.get_time(); + assert_eq!(resumed_time, time + Duration::from_nanos(2)); +} + #[test] fn test_get_set_cycle_balance() { let pic = PocketIc::new(); diff --git a/rs/pocket_ic_server/src/pocket_ic.rs b/rs/pocket_ic_server/src/pocket_ic.rs index 8a2b091faf15..1a89f1238413 100644 --- a/rs/pocket_ic_server/src/pocket_ic.rs +++ b/rs/pocket_ic_server/src/pocket_ic.rs @@ -98,6 +98,7 @@ use pocket_ic::{copy_dir, ErrorCode, RejectCode, RejectResponse}; use registry_canister::init::RegistryCanisterInitPayload; use serde::{Deserialize, Serialize}; use slog::Level; +use std::cmp::max; use std::hash::Hash; use std::str::FromStr; use std::{ @@ -198,18 +199,12 @@ fn compute_subnet_seed( #[derive(Clone, Deserialize, Serialize)] struct RawTopologyInternal { - pub subnet_configs: Vec, + pub subnet_configs: Vec, pub default_effective_canister_id: RawCanisterId, pub icp_features: Option, pub synced_registry_version: Option, } -#[derive(Clone, Deserialize, Serialize)] -struct RawSubnetConfigInternal { - pub subnet_config: SubnetConfigInternal, - pub time: SystemTime, -} - #[derive(Clone, Debug, Deserialize, Serialize)] struct SubnetConfigInternal { pub subnet_id: SubnetId, @@ -488,7 +483,6 @@ impl PocketIcSubnets { instruction_config: SubnetInstructionConfig, registry_data_provider: Arc, create_at_registry_version: RegistryVersion, - time: SystemTime, nonmainnet_features: bool, log_level: Option, bitcoin_adapter_uds_path: Option, @@ -534,18 +528,12 @@ impl PocketIcSubnets { .feature_flags .rate_limiting_of_debug_prints = FlagStatus::Disabled; let state_machine_config = StateMachineConfig::new(subnet_config, hypervisor_config); - let t = time - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap() - .as_nanos() as u64; - let time = Time::from_nanos_since_unix_epoch(t); StateMachineBuilder::new() .with_runtime(runtime) .with_config(Some(state_machine_config)) .with_subnet_seed(subnet_seed) .with_subnet_size(subnet_size.try_into().unwrap()) .with_subnet_type(subnet_type) - .with_time(time) .with_state_machine_state_dir(state_machine_state_dir) .with_registry_data_provider(registry_data_provider.clone()) .with_log_level(log_level) @@ -625,19 +613,8 @@ impl PocketIcSubnets { subnet_state_dir, subnet_kind, instruction_config, - mut time, } = subnet_config_info; - // All subnets must eventually have the same time and time can only advance => - // advance time of the new subnet if other subnets have higher time; - // the maximum time must be determined before adding a `StateMachine` - // for the new subnet to `self.subnets` because `self.time()` - // is only sound if all subnets in `self.subnets` have the same time. - let current_time = self.time(); - if current_time > time { - time = current_time; - } - let subnet_seed = compute_subnet_seed(ranges.clone(), alloc_range); let state_machine_state_dir: Box = @@ -669,7 +646,6 @@ impl PocketIcSubnets { instruction_config.clone(), self.registry_data_provider.clone(), create_at_registry_version, - time, self.nonmainnet_features, self.log_level, bitcoin_adapter_uds_path.clone(), @@ -784,6 +760,19 @@ impl PocketIcSubnets { subnet.state_machine.reload_registry(); } + // All subnets must have the same time and time can only advance => + // set the time to the maximum time in the latest state across all subnets. + let mut time: SystemTime = GENESIS.into(); + for subnet in self.subnets.get_all() { + let metadata = &subnet + .state_machine + .state_manager + .get_latest_state() + .take() + .metadata; + time = max(time, metadata.batch_time.into()); + } + // Make sure time is strictly monotone. time += Duration::from_nanos(1); @@ -1105,20 +1094,8 @@ impl Drop for PocketIc { for subnet in &subnets { subnet.state_machine.await_state_hash(); } - let subnet_configs = self - .subnets - .subnet_configs - .iter() - .map(|config| { - let time = self.subnets.get(config.subnet_id).unwrap().time(); - RawSubnetConfigInternal { - subnet_config: config.clone(), - time, - } - }) - .collect(); let raw_topology: RawTopologyInternal = RawTopologyInternal { - subnet_configs, + subnet_configs: self.subnets.subnet_configs.clone(), default_effective_canister_id: self.default_effective_canister_id.into(), icp_features: self.subnets.icp_features.clone(), synced_registry_version: Some(self.subnets.synced_registry_version.get()), @@ -1244,20 +1221,17 @@ impl PocketIc { .subnet_configs .into_iter() .map(|config| { - range_gen - .add_assigned(config.subnet_config.ranges.clone()) - .unwrap(); - if let Some(allocation_range) = config.subnet_config.alloc_range { + range_gen.add_assigned(config.ranges.clone()).unwrap(); + if let Some(allocation_range) = config.alloc_range { range_gen.add_assigned(vec![allocation_range]).unwrap(); } SubnetConfigInfo { - ranges: config.subnet_config.ranges, - alloc_range: config.subnet_config.alloc_range, - subnet_id: Some(config.subnet_config.subnet_id), + ranges: config.ranges, + alloc_range: config.alloc_range, + subnet_id: Some(config.subnet_id), subnet_state_dir: None, - subnet_kind: config.subnet_config.subnet_kind, - instruction_config: config.subnet_config.instruction_config, - time: config.time, + subnet_kind: config.subnet_kind, + instruction_config: config.instruction_config, } }) .collect() @@ -1305,7 +1279,7 @@ impl PocketIc { let mut subnet_config_info: Vec = vec![]; for (subnet_kind, subnet_state_dir, instruction_config) in all_subnets { - let (ranges, alloc_range, subnet_id, time) = if let Some(ref subnet_state_dir) = + let (ranges, alloc_range, subnet_id) = if let Some(ref subnet_state_dir) = subnet_state_dir { match std::fs::read_dir(subnet_state_dir) { @@ -1355,7 +1329,6 @@ impl PocketIc { }; let subnet_id = metadata.own_subnet_id; - let time = metadata.batch_time; let ranges: Vec<_> = metadata .network_topology .routing_table @@ -1390,14 +1363,14 @@ impl PocketIc { } } - (ranges, None, Some(subnet_id), time) + (ranges, None, Some(subnet_id)) } else { let RangeConfig { canister_id_ranges: ranges, canister_allocation_range: alloc_range, } = get_range_config(subnet_kind, &mut range_gen)?; - (ranges, alloc_range, None, GENESIS) + (ranges, alloc_range, None) }; subnet_config_info.push(SubnetConfigInfo { @@ -1407,7 +1380,6 @@ impl PocketIc { subnet_state_dir, subnet_kind, instruction_config, - time: time.into(), }); } @@ -1655,7 +1627,6 @@ struct SubnetConfigInfo { pub subnet_state_dir: Option, pub subnet_kind: SubnetKind, pub instruction_config: SubnetInstructionConfig, - pub time: SystemTime, } // ---------------------------------------------------------------------------------------- // @@ -3233,7 +3204,6 @@ fn route( subnet_state_dir: None, subnet_kind, instruction_config, - time: GENESIS.into(), }); Ok(pic.try_route_canister(canister_id).unwrap()) } else { From c1b814221bc573ac3bff3cad41571dcc003647c3 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 14 Jul 2025 11:59:48 +0200 Subject: [PATCH 02/19] harden test --- packages/pocket-ic/tests/tests.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/pocket-ic/tests/tests.rs b/packages/pocket-ic/tests/tests.rs index 19053076bddf..d614d3550971 100644 --- a/packages/pocket-ic/tests/tests.rs +++ b/packages/pocket-ic/tests/tests.rs @@ -447,11 +447,12 @@ fn time_on_resumed_instance() { pic.set_certified_time(now.into()); let time = pic.get_time(); + assert_eq!(time, now.into()); let state = pic.drop_and_take_state().unwrap(); let pic = PocketIcBuilder::new().with_state(state).build(); - // the time on the resumed instances increases by 2ns: + // The time on the resumed instances increases by 2ns: // - 1ns due to executing a checkpointed round before dropping the original instance; // - 1ns due to bumping time when creating a new instance to ensure strict time monotonicity. let resumed_time = pic.get_time(); From 322bc5fb0e0bde5d5f52f8fe03da91ed76a0c54a Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 14 Jul 2025 12:21:29 +0200 Subject: [PATCH 03/19] persist topology eagerly --- packages/pocket-ic/src/lib.rs | 21 ++++++++++++------ packages/pocket-ic/src/nonblocking.rs | 8 +++++-- packages/pocket-ic/tests/tests.rs | 31 ++++++++++++++++++++++++++- rs/pocket_ic_server/src/pocket_ic.rs | 31 ++++++++++++++++++--------- 4 files changed, 72 insertions(+), 19 deletions(-) diff --git a/packages/pocket-ic/src/lib.rs b/packages/pocket-ic/src/lib.rs index 9dad07ce57f3..b3f0644fca1e 100644 --- a/packages/pocket-ic/src/lib.rs +++ b/packages/pocket-ic/src/lib.rs @@ -81,7 +81,7 @@ use std::{ fs::OpenOptions, net::{IpAddr, SocketAddr}, path::PathBuf, - process::Command, + process::{Child, Command}, sync::{mpsc::channel, Arc}, thread, thread::JoinHandle, @@ -617,8 +617,13 @@ impl PocketIc { let runtime = tokio::runtime::Builder::new_current_thread() .build() .unwrap(); - let url = runtime - .block_on(async { start_or_reuse_server(None).await.join("instances").unwrap() }); + let url = runtime.block_on(async { + start_or_reuse_server(None) + .await + .1 + .join("instances") + .unwrap() + }); let instances: Vec = reqwest::blocking::Client::new() .get(url) .send() @@ -1820,7 +1825,7 @@ async fn download_pocketic_server( } /// Attempt to start a new PocketIC server if it's not already running. -pub async fn start_or_reuse_server(server_binary: Option) -> Url { +pub async fn start_or_reuse_server(server_binary: Option) -> (Child, Url) { let default_bin_dir = std::env::temp_dir().join(format!("pocket-ic-server-{}", EXPECTED_SERVER_VERSION)); let default_bin_path = default_bin_dir.join("pocket-ic"); @@ -1895,7 +1900,8 @@ pub async fn start_or_reuse_server(server_binary: Option) -> Url { // TODO: SDK-1936 #[allow(clippy::zombie_processes)] - cmd.spawn() + let child = cmd + .spawn() .unwrap_or_else(|_| panic!("Failed to start PocketIC binary ({})", bin_path.display())); loop { @@ -1905,7 +1911,10 @@ pub async fn start_or_reuse_server(server_binary: Option) -> Url { .trim_end() .parse() .expect("Failed to parse port to number"); - break Url::parse(&format!("http://{}:{}/", LOCALHOST, port)).unwrap(); + break ( + child, + Url::parse(&format!("http://{}:{}/", LOCALHOST, port)).unwrap(), + ); } } std::thread::sleep(Duration::from_millis(20)); diff --git a/packages/pocket-ic/src/nonblocking.rs b/packages/pocket-ic/src/nonblocking.rs index 9b6b5695973b..d62809baee86 100644 --- a/packages/pocket-ic/src/nonblocking.rs +++ b/packages/pocket-ic/src/nonblocking.rs @@ -144,7 +144,7 @@ impl PocketIc { let server_url = if let Some(server_url) = server_url { server_url } else { - start_or_reuse_server(server_binary).await + start_or_reuse_server(server_binary).await.1 }; let subnet_config_set = subnet_config_set @@ -314,7 +314,11 @@ impl PocketIc { /// List all instances and their status. #[instrument(ret)] pub async fn list_instances() -> Vec { - let url = start_or_reuse_server(None).await.join("instances").unwrap(); + let url = start_or_reuse_server(None) + .await + .1 + .join("instances") + .unwrap(); let instances: Vec = reqwest::Client::new() .get(url) .send() diff --git a/packages/pocket-ic/tests/tests.rs b/packages/pocket-ic/tests/tests.rs index d614d3550971..863d37997520 100644 --- a/packages/pocket-ic/tests/tests.rs +++ b/packages/pocket-ic/tests/tests.rs @@ -459,6 +459,35 @@ fn time_on_resumed_instance() { assert_eq!(resumed_time, time + Duration::from_nanos(2)); } +#[tokio::test] +async fn time_on_killed_instance() { + let (mut server, server_url) = start_or_reuse_server(None).await; + let temp_dir = TempDir::new().unwrap(); + + let state = PocketIcState::new_from_path(temp_dir.path().to_path_buf()); + let pic = PocketIcBuilder::new() + .with_application_subnet() + .with_server_url(server_url) + .with_state(state) + .build_async() + .await; + + let time = pic.get_time().await; + + // this change of time is lost after killing the server + let now = SystemTime::now(); + pic.set_certified_time(now.into()).await; + + server.kill().unwrap(); + + let state = PocketIcState::new_from_path(temp_dir.path().to_path_buf()); + let pic = PocketIcBuilder::new().with_state(state).build_async().await; + + // The time on the resumed instances increases by 1ns due to bumping time when creating a new instance to ensure strict time monotonicity. + let resumed_time = pic.get_time().await; + assert_eq!(resumed_time, time + Duration::from_nanos(1)); +} + #[test] fn test_get_set_cycle_balance() { let pic = PocketIc::new(); @@ -3181,7 +3210,7 @@ async fn with_all_icp_features_and_nns_subnet_state() { .unwrap() .into(); - let url = start_or_reuse_server(None).await; + let url = start_or_reuse_server(None).await.1; let client = reqwest::Client::new(); let instance_config = InstanceConfig { subnet_config_set: ExtendedSubnetConfigSet { diff --git a/rs/pocket_ic_server/src/pocket_ic.rs b/rs/pocket_ic_server/src/pocket_ic.rs index 1a89f1238413..d73a28188a2e 100644 --- a/rs/pocket_ic_server/src/pocket_ic.rs +++ b/rs/pocket_ic_server/src/pocket_ic.rs @@ -576,6 +576,20 @@ impl PocketIcSubnets { } } + fn persist_topology(&self, default_effective_canister_id: Principal) { + if let Some(ref state_dir) = self.state_dir { + let raw_topology: RawTopologyInternal = RawTopologyInternal { + subnet_configs: self.subnet_configs.clone(), + default_effective_canister_id: default_effective_canister_id.into(), + icp_features: self.icp_features.clone(), + synced_registry_version: Some(self.synced_registry_version.get()), + }; + let topology_json = serde_json::to_string(&raw_topology).unwrap(); + let mut topology_file = File::create(state_dir.join("topology.json")).unwrap(); + topology_file.write_all(topology_json.as_bytes()).unwrap(); + } + } + fn get_all(&self) -> Vec> { self.subnets.get_all() } @@ -1086,7 +1100,7 @@ pub struct PocketIc { impl Drop for PocketIc { fn drop(&mut self) { - if let Some(ref state_dir) = self.subnets.state_dir { + if self.subnets.state_dir.is_some() { let subnets = self.subnets.get_all(); for subnet in &subnets { subnet.state_machine.checkpointed_tick(); @@ -1094,15 +1108,8 @@ impl Drop for PocketIc { for subnet in &subnets { subnet.state_machine.await_state_hash(); } - let raw_topology: RawTopologyInternal = RawTopologyInternal { - subnet_configs: self.subnets.subnet_configs.clone(), - default_effective_canister_id: self.default_effective_canister_id.into(), - icp_features: self.subnets.icp_features.clone(), - synced_registry_version: Some(self.subnets.synced_registry_version.get()), - }; - let topology_json = serde_json::to_string(&raw_topology).unwrap(); - let mut topology_file = File::create(state_dir.join("topology.json")).unwrap(); - topology_file.write_all(topology_json.as_bytes()).unwrap(); + self.subnets + .persist_topology(self.default_effective_canister_id); } for subnet in self.subnets.get_all() { subnet.state_machine.drop_payload_builder(); @@ -1433,6 +1440,8 @@ impl PocketIc { }) .default_effective_canister_id(); + subnets.persist_topology(default_effective_canister_id); + let state_label = StateLabel::new(seed); Ok(Self { @@ -3205,6 +3214,8 @@ fn route( subnet_kind, instruction_config, }); + pic.subnets + .persist_topology(pic.default_effective_canister_id); Ok(pic.try_route_canister(canister_id).unwrap()) } else { // If the request is not an update call to create a canister using the provisional API, From 481ce8dcd06fab3b6af2b0b5b0c1c9da39ba2ba4 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 14 Jul 2025 12:29:12 +0200 Subject: [PATCH 04/19] fix tests --- packages/pocket-ic/tests/tests.rs | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/packages/pocket-ic/tests/tests.rs b/packages/pocket-ic/tests/tests.rs index 863d37997520..f1a11d72d021 100644 --- a/packages/pocket-ic/tests/tests.rs +++ b/packages/pocket-ic/tests/tests.rs @@ -472,20 +472,34 @@ async fn time_on_killed_instance() { .build_async() .await; - let time = pic.get_time().await; + let canister_id = pic.create_canister().await; + + // Execute many rounds to trigger a checkpoint. + for _ in 0..600 { + pic.tick().await; + } - // this change of time is lost after killing the server + // The following (most recent) changes will be lost after killing the instance. let now = SystemTime::now(); pic.set_certified_time(now.into()).await; + let another_canister_id = pic.create_canister().await; + + assert!(pic.canister_exists(canister_id).await); + assert!(pic.canister_exists(another_canister_id).await); + let time = pic.get_time().await; + assert!(time >= now.into()); server.kill().unwrap(); let state = PocketIcState::new_from_path(temp_dir.path().to_path_buf()); let pic = PocketIcBuilder::new().with_state(state).build_async().await; - // The time on the resumed instances increases by 1ns due to bumping time when creating a new instance to ensure strict time monotonicity. + // Only the first canister (created before the last checkpoint) is preserved, + // the other canister and time change are lost. + assert!(pic.canister_exists(canister_id).await); + assert!(!pic.canister_exists(another_canister_id).await); let resumed_time = pic.get_time().await; - assert_eq!(resumed_time, time + Duration::from_nanos(1)); + assert!(resumed_time < now.into()); } #[test] From 1c7566f6737bb594e91120ef7a51d95ecbddf7c3 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 14 Jul 2025 12:32:44 +0200 Subject: [PATCH 05/19] simplify --- packages/pocket-ic/src/lib.rs | 16 ++++++++-------- packages/pocket-ic/src/nonblocking.rs | 8 ++------ packages/pocket-ic/tests/tests.rs | 9 +++++---- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/packages/pocket-ic/src/lib.rs b/packages/pocket-ic/src/lib.rs index b3f0644fca1e..bb523f295436 100644 --- a/packages/pocket-ic/src/lib.rs +++ b/packages/pocket-ic/src/lib.rs @@ -617,13 +617,8 @@ impl PocketIc { let runtime = tokio::runtime::Builder::new_current_thread() .build() .unwrap(); - let url = runtime.block_on(async { - start_or_reuse_server(None) - .await - .1 - .join("instances") - .unwrap() - }); + let url = runtime + .block_on(async { start_or_reuse_server(None).await.join("instances").unwrap() }); let instances: Vec = reqwest::blocking::Client::new() .get(url) .send() @@ -1825,7 +1820,12 @@ async fn download_pocketic_server( } /// Attempt to start a new PocketIC server if it's not already running. -pub async fn start_or_reuse_server(server_binary: Option) -> (Child, Url) { +pub async fn start_or_reuse_server(server_binary: Option) -> Url { + start_or_reuse_server_impl(server_binary).await.1 +} + +/// Attempt to start a new PocketIC server if it's not already running. +pub async fn start_or_reuse_server_impl(server_binary: Option) -> (Child, Url) { let default_bin_dir = std::env::temp_dir().join(format!("pocket-ic-server-{}", EXPECTED_SERVER_VERSION)); let default_bin_path = default_bin_dir.join("pocket-ic"); diff --git a/packages/pocket-ic/src/nonblocking.rs b/packages/pocket-ic/src/nonblocking.rs index d62809baee86..9b6b5695973b 100644 --- a/packages/pocket-ic/src/nonblocking.rs +++ b/packages/pocket-ic/src/nonblocking.rs @@ -144,7 +144,7 @@ impl PocketIc { let server_url = if let Some(server_url) = server_url { server_url } else { - start_or_reuse_server(server_binary).await.1 + start_or_reuse_server(server_binary).await }; let subnet_config_set = subnet_config_set @@ -314,11 +314,7 @@ impl PocketIc { /// List all instances and their status. #[instrument(ret)] pub async fn list_instances() -> Vec { - let url = start_or_reuse_server(None) - .await - .1 - .join("instances") - .unwrap(); + let url = start_or_reuse_server(None).await.join("instances").unwrap(); let instances: Vec = reqwest::Client::new() .get(url) .send() diff --git a/packages/pocket-ic/tests/tests.rs b/packages/pocket-ic/tests/tests.rs index f1a11d72d021..94e6cee6b60d 100644 --- a/packages/pocket-ic/tests/tests.rs +++ b/packages/pocket-ic/tests/tests.rs @@ -18,8 +18,9 @@ use pocket_ic::{ BlobCompression, CanisterHttpReply, CanisterHttpResponse, MockCanisterHttpResponse, RawEffectivePrincipal, RawMessageId, SubnetKind, }, - query_candid, start_or_reuse_server, update_candid, DefaultEffectiveCanisterIdError, ErrorCode, - IngressStatusResult, PocketIc, PocketIcBuilder, PocketIcState, RejectCode, Time, + query_candid, start_or_reuse_server, start_or_reuse_server_impl, update_candid, + DefaultEffectiveCanisterIdError, ErrorCode, IngressStatusResult, PocketIc, PocketIcBuilder, + PocketIcState, RejectCode, Time, }; use reqwest::blocking::Client; use reqwest::header::CONTENT_LENGTH; @@ -461,7 +462,7 @@ fn time_on_resumed_instance() { #[tokio::test] async fn time_on_killed_instance() { - let (mut server, server_url) = start_or_reuse_server(None).await; + let (mut server, server_url) = start_or_reuse_server_impl(None).await; let temp_dir = TempDir::new().unwrap(); let state = PocketIcState::new_from_path(temp_dir.path().to_path_buf()); @@ -3224,7 +3225,7 @@ async fn with_all_icp_features_and_nns_subnet_state() { .unwrap() .into(); - let url = start_or_reuse_server(None).await.1; + let url = start_or_reuse_server(None).await; let client = reqwest::Client::new(); let instance_config = InstanceConfig { subnet_config_set: ExtendedSubnetConfigSet { From c0621d68542b85610ca17d00946a5ee986cd2cd5 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 14 Jul 2025 12:34:48 +0200 Subject: [PATCH 06/19] test name --- packages/pocket-ic/tests/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pocket-ic/tests/tests.rs b/packages/pocket-ic/tests/tests.rs index 94e6cee6b60d..445af3ad17e7 100644 --- a/packages/pocket-ic/tests/tests.rs +++ b/packages/pocket-ic/tests/tests.rs @@ -461,7 +461,7 @@ fn time_on_resumed_instance() { } #[tokio::test] -async fn time_on_killed_instance() { +async fn killed_instance() { let (mut server, server_url) = start_or_reuse_server_impl(None).await; let temp_dir = TempDir::new().unwrap(); From 15eed96d14c6a631fd802dc0befdc714d2c8dec4 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 14 Jul 2025 12:36:40 +0200 Subject: [PATCH 07/19] explicit drop --- packages/pocket-ic/tests/tests.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/pocket-ic/tests/tests.rs b/packages/pocket-ic/tests/tests.rs index 445af3ad17e7..5e8974cb9c96 100644 --- a/packages/pocket-ic/tests/tests.rs +++ b/packages/pocket-ic/tests/tests.rs @@ -501,6 +501,9 @@ async fn killed_instance() { assert!(!pic.canister_exists(another_canister_id).await); let resumed_time = pic.get_time().await; assert!(resumed_time < now.into()); + + // Drop instance explicitly to prevent data races in the StateManager. + pic.drop().await; } #[test] From dd0206f22f57e30be17c42732e430811206d7e8e Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 14 Jul 2025 16:34:49 +0200 Subject: [PATCH 08/19] check for corrupted state --- packages/pocket-ic/CHANGELOG.md | 2 +- packages/pocket-ic/src/common/rest.rs | 1 + packages/pocket-ic/src/lib.rs | 38 ++++++++----- packages/pocket-ic/src/nonblocking.rs | 19 +++++-- packages/pocket-ic/tests/icp_features.rs | 7 ++- packages/pocket-ic/tests/tests.rs | 62 ++++++++++++++++++--- rs/pocket_ic_server/CHANGELOG.md | 1 + rs/pocket_ic_server/src/pocket_ic.rs | 34 +++++++++-- rs/pocket_ic_server/src/state_api/routes.rs | 1 + rs/pocket_ic_server/tests/test.rs | 1 + 10 files changed, 133 insertions(+), 33 deletions(-) diff --git a/packages/pocket-ic/CHANGELOG.md b/packages/pocket-ic/CHANGELOG.md index 9e56594e5488..16e0664eed70 100644 --- a/packages/pocket-ic/CHANGELOG.md +++ b/packages/pocket-ic/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased ### Added -- The function `PocketIc::start_or_reuse_server` to manually start or reuse a PocketIC server. +- The function `start_server` and its input type `StartServerParams` to manually start a PocketIC server. - The function `PocketIcBuilder::with_all_icp_features` to specify that all ICP features (supported by PocketIC) should be enabled. diff --git a/packages/pocket-ic/src/common/rest.rs b/packages/pocket-ic/src/common/rest.rs index 30e9417664bf..7cbdc1899e3a 100644 --- a/packages/pocket-ic/src/common/rest.rs +++ b/packages/pocket-ic/src/common/rest.rs @@ -563,6 +563,7 @@ pub struct InstanceConfig { pub log_level: Option, pub bitcoind_addr: Option>, pub icp_features: Option, + pub allow_corrupted_state: Option, } #[derive(Debug, Clone, Eq, Hash, PartialEq, Serialize, Deserialize, Default, JsonSchema)] diff --git a/packages/pocket-ic/src/lib.rs b/packages/pocket-ic/src/lib.rs index bb523f295436..c599bdd3b76b 100644 --- a/packages/pocket-ic/src/lib.rs +++ b/packages/pocket-ic/src/lib.rs @@ -88,7 +88,7 @@ use std::{ time::{Duration, SystemTime, UNIX_EPOCH}, }; use strum_macros::EnumIter; -use tempfile::TempDir; +use tempfile::{NamedTempFile, TempDir}; use thiserror::Error; use tokio::runtime::Runtime; use tracing::{instrument, warn}; @@ -617,8 +617,14 @@ impl PocketIc { let runtime = tokio::runtime::Builder::new_current_thread() .build() .unwrap(); - let url = runtime - .block_on(async { start_or_reuse_server(None).await.join("instances").unwrap() }); + let url = runtime.block_on(async { + let (_, server_url) = start_server(StartServerParams { + reuse: true, + ..Default::default() + }) + .await; + server_url.join("instances").unwrap() + }); let instances: Vec = reqwest::blocking::Client::new() .get(url) .send() @@ -1819,17 +1825,19 @@ async fn download_pocketic_server( Ok(()) } -/// Attempt to start a new PocketIC server if it's not already running. -pub async fn start_or_reuse_server(server_binary: Option) -> Url { - start_or_reuse_server_impl(server_binary).await.1 +#[derive(Default)] +pub struct StartServerParams { + pub server_binary: Option, + /// Reuse an existing PocketIC server spawned by this process. + pub reuse: bool, } -/// Attempt to start a new PocketIC server if it's not already running. -pub async fn start_or_reuse_server_impl(server_binary: Option) -> (Child, Url) { +/// Attempt to start a new PocketIC server. +pub async fn start_server(params: StartServerParams) -> (Child, Url) { let default_bin_dir = std::env::temp_dir().join(format!("pocket-ic-server-{}", EXPECTED_SERVER_VERSION)); let default_bin_path = default_bin_dir.join("pocket-ic"); - let mut bin_path: PathBuf = server_binary.unwrap_or_else(|| { + let mut bin_path: PathBuf = params.server_binary.unwrap_or_else(|| { std::env::var_os("POCKET_IC_BIN") .unwrap_or_else(|| default_bin_path.clone().into()) .into() @@ -1873,10 +1881,14 @@ pub async fn start_or_reuse_server_impl(server_binary: Option) -> (Chil } } - // We use the test driver's process ID to share the PocketIC server between multiple tests - // launched by the same test driver. - let test_driver_pid = std::process::id(); - let port_file_path = std::env::temp_dir().join(format!("pocket_ic_{}.port", test_driver_pid)); + let port_file_path = if params.reuse { + // We use the test driver's process ID to share the PocketIC server between multiple tests + // launched by the same test driver. + let test_driver_pid = std::process::id(); + std::env::temp_dir().join(format!("pocket_ic_{}.port", test_driver_pid)) + } else { + NamedTempFile::new().unwrap().into_temp_path().to_path_buf() + }; let mut cmd = pocket_ic_server_cmd(&bin_path); cmd.arg("--port-file"); #[cfg(windows)] diff --git a/packages/pocket-ic/src/nonblocking.rs b/packages/pocket-ic/src/nonblocking.rs index 9b6b5695973b..164023be424d 100644 --- a/packages/pocket-ic/src/nonblocking.rs +++ b/packages/pocket-ic/src/nonblocking.rs @@ -11,8 +11,8 @@ use crate::common::rest::{ use crate::wsl_path; pub use crate::DefaultEffectiveCanisterIdError; use crate::{ - copy_dir, start_or_reuse_server, IngressStatusResult, PocketIcBuilder, PocketIcState, - RejectResponse, Time, + copy_dir, start_server, IngressStatusResult, PocketIcBuilder, PocketIcState, RejectResponse, + StartServerParams, Time, }; use backoff::backoff::Backoff; use backoff::{ExponentialBackoff, ExponentialBackoffBuilder}; @@ -144,7 +144,12 @@ impl PocketIc { let server_url = if let Some(server_url) = server_url { server_url } else { - start_or_reuse_server(server_binary).await + let (_, server_url) = start_server(StartServerParams { + server_binary, + reuse: true, + }) + .await; + server_url }; let subnet_config_set = subnet_config_set @@ -200,6 +205,7 @@ impl PocketIc { log_level: log_level.map(|l| l.to_string()), bitcoind_addr, icp_features: Some(icp_features), + allow_corrupted_state: Some(false), }; let test_driver_pid = std::process::id(); @@ -314,7 +320,12 @@ impl PocketIc { /// List all instances and their status. #[instrument(ret)] pub async fn list_instances() -> Vec { - let url = start_or_reuse_server(None).await.join("instances").unwrap(); + let (_, server_url) = start_server(StartServerParams { + reuse: true, + ..Default::default() + }) + .await; + let url = server_url.join("instances").unwrap(); let instances: Vec = reqwest::Client::new() .get(url) .send() diff --git a/packages/pocket-ic/tests/icp_features.rs b/packages/pocket-ic/tests/icp_features.rs index 8de825492703..4c704b8c6237 100644 --- a/packages/pocket-ic/tests/icp_features.rs +++ b/packages/pocket-ic/tests/icp_features.rs @@ -1,6 +1,8 @@ use candid::{CandidType, Principal}; use pocket_ic::common::rest::{ExtendedSubnetConfigSet, IcpFeatures, InstanceConfig, SubnetSpec}; -use pocket_ic::{start_or_reuse_server, update_candid, PocketIc, PocketIcBuilder, PocketIcState}; +use pocket_ic::{ + start_server, update_candid, PocketIc, PocketIcBuilder, PocketIcState, StartServerParams, +}; use reqwest::StatusCode; use serde::Deserialize; use std::collections::BTreeMap; @@ -323,7 +325,7 @@ async fn with_all_icp_features_and_nns_subnet_state() { .unwrap() .into(); - let url = start_or_reuse_server(None).await; + let (_, url) = start_server(StartServerParams::default()).await; let client = reqwest::Client::new(); let instance_config = InstanceConfig { subnet_config_set: ExtendedSubnetConfigSet { @@ -335,6 +337,7 @@ async fn with_all_icp_features_and_nns_subnet_state() { log_level: None, bitcoind_addr: None, icp_features: Some(IcpFeatures::all_icp_features()), + allow_corrupted_state: None, }; let response = client .post(url.join("instances").unwrap()) diff --git a/packages/pocket-ic/tests/tests.rs b/packages/pocket-ic/tests/tests.rs index 9d2ae7d0ce35..9c48ea711b95 100644 --- a/packages/pocket-ic/tests/tests.rs +++ b/packages/pocket-ic/tests/tests.rs @@ -12,11 +12,14 @@ use ic_transport_types::EnvelopeContent::{Call, ReadState}; use pocket_ic::common::rest::{BlockmakerConfigs, RawSubnetBlockmaker, TickConfigs}; use pocket_ic::{ common::rest::{ - BlobCompression, CanisterHttpReply, CanisterHttpResponse, MockCanisterHttpResponse, - RawEffectivePrincipal, RawMessageId, SubnetKind, + BlobCompression, CanisterHttpReply, CanisterHttpResponse, CreateInstanceResponse, + ExtendedSubnetConfigSet, InstanceConfig, MockCanisterHttpResponse, RawEffectivePrincipal, + RawMessageId, SubnetKind, }, - query_candid, start_or_reuse_server_impl, update_candid, DefaultEffectiveCanisterIdError, - ErrorCode, IngressStatusResult, PocketIc, PocketIcBuilder, PocketIcState, RejectCode, Time, + nonblocking::PocketIc as PocketIcAsync, + query_candid, start_server, update_candid, DefaultEffectiveCanisterIdError, ErrorCode, + IngressStatusResult, PocketIc, PocketIcBuilder, PocketIcState, RejectCode, StartServerParams, + Time, }; use reqwest::blocking::Client; use reqwest::header::CONTENT_LENGTH; @@ -455,9 +458,8 @@ fn time_on_resumed_instance() { assert_eq!(resumed_time, time + Duration::from_nanos(2)); } -#[tokio::test] -async fn killed_instance() { - let (mut server, server_url) = start_or_reuse_server_impl(None).await; +async fn resume_killed_instance_impl(allow_corrupted_state: Option) -> Result<(), String> { + let (mut server, server_url) = start_server(StartServerParams::default()).await; let temp_dir = TempDir::new().unwrap(); let state = PocketIcState::new_from_path(temp_dir.path().to_path_buf()); @@ -487,8 +489,31 @@ async fn killed_instance() { server.kill().unwrap(); - let state = PocketIcState::new_from_path(temp_dir.path().to_path_buf()); - let pic = PocketIcBuilder::new().with_state(state).build_async().await; + let (_, server_url) = start_server(StartServerParams::default()).await; + let client = reqwest::Client::new(); + let instance_config = InstanceConfig { + subnet_config_set: ExtendedSubnetConfigSet::default(), + state_dir: Some(temp_dir.path().to_path_buf()), + nonmainnet_features: false, + log_level: None, + bitcoind_addr: None, + icp_features: None, + allow_corrupted_state, + }; + let response = client + .post(server_url.join("instances").unwrap()) + .json(&instance_config) + .send() + .await + .unwrap(); + if !response.status().is_success() { + return Err(response.text().await.unwrap()); + } + let instance_id = match response.json::().await.unwrap() { + CreateInstanceResponse::Created { instance_id, .. } => instance_id, + CreateInstanceResponse::Error { message } => panic!("Unexpected error: {}", message), + }; + let pic = PocketIcAsync::new_from_existing_instance(server_url, instance_id, None); // Only the first canister (created before the last checkpoint) is preserved, // the other canister and time change are lost. @@ -499,6 +524,25 @@ async fn killed_instance() { // Drop instance explicitly to prevent data races in the StateManager. pic.drop().await; + + Ok(()) +} + +#[tokio::test] +async fn resume_killed_instance_default() { + let err = resume_killed_instance_impl(None).await.unwrap_err(); + assert!(err.contains("The state of subnet qxw6a-xchhl-fytzl-qfagk-kuv6t-ye6xf-box4s-fiqod-vn2ex-qkp3q-5ae is corrupted.")); +} + +#[tokio::test] +async fn resume_killed_instance_strict() { + let err = resume_killed_instance_impl(Some(false)).await.unwrap_err(); + assert!(err.contains("The state of subnet qxw6a-xchhl-fytzl-qfagk-kuv6t-ye6xf-box4s-fiqod-vn2ex-qkp3q-5ae is corrupted.")); +} + +#[tokio::test] +async fn resume_killed_instance() { + resume_killed_instance_impl(Some(true)).await.unwrap(); } #[test] diff --git a/rs/pocket_ic_server/CHANGELOG.md b/rs/pocket_ic_server/CHANGELOG.md index d5a4d264081c..42d7a9d65389 100644 --- a/rs/pocket_ic_server/CHANGELOG.md +++ b/rs/pocket_ic_server/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The endpoint `/instances//update/await_ingress_message` (execute rounds on the PocketIc instance until the message is executed): to fix a performance regression when using the two endpoints `/instances//update/tick` and `/instances//read/ingress_status` in a loop. - The argument of the endpoint `/instances/` takes an additional optional field `icp_features` specifying ICP features (implemented by system canisters) to be enabled in the newly created PocketIC instance. +- The argument of the endpoint `/instances/` takes an additional optional field `allow_corrupted_state` specifying if corrupted state (e.g., resulting from not deleting a PocketIC instance gracefully) is allowed. diff --git a/rs/pocket_ic_server/src/pocket_ic.rs b/rs/pocket_ic_server/src/pocket_ic.rs index d73a28188a2e..6d9f3fa8faee 100644 --- a/rs/pocket_ic_server/src/pocket_ic.rs +++ b/rs/pocket_ic_server/src/pocket_ic.rs @@ -203,6 +203,7 @@ struct RawTopologyInternal { pub default_effective_canister_id: RawCanisterId, pub icp_features: Option, pub synced_registry_version: Option, + pub time: SystemTime, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -583,6 +584,7 @@ impl PocketIcSubnets { default_effective_canister_id: default_effective_canister_id.into(), icp_features: self.icp_features.clone(), synced_registry_version: Some(self.synced_registry_version.get()), + time: self.time(), }; let topology_json = serde_json::to_string(&raw_topology).unwrap(); let mut topology_file = File::create(state_dir.join("topology.json")).unwrap(); @@ -619,7 +621,10 @@ impl PocketIcSubnets { .unwrap_or(GENESIS.into()) } - fn create_subnet(&mut self, subnet_config_info: SubnetConfigInfo) -> SubnetConfigInternal { + fn create_subnet( + &mut self, + subnet_config_info: SubnetConfigInfo, + ) -> Result { let SubnetConfigInfo { ranges, alloc_range, @@ -627,6 +632,7 @@ impl PocketIcSubnets { subnet_state_dir, subnet_kind, instruction_config, + time, } = subnet_config_info; let subnet_seed = compute_subnet_seed(ranges.clone(), alloc_range); @@ -709,6 +715,14 @@ impl PocketIcSubnets { // if one was provided). let subnet_id = sm.get_subnet_id(); + if let Some(expected_time) = time { + let metadata = &sm.state_manager.get_latest_state().take().metadata; + let actual_time: SystemTime = metadata.batch_time.into(); + if actual_time != expected_time { + return Err(format!("The state of subnet {} is corrupted.", subnet_id)); + } + } + // The subnet created first is marked as the NNS subnet. if self.nns_subnet.is_none() { self.nns_subnet = Some(self.subnets.get_subnet(subnet_id).unwrap()); @@ -826,7 +840,7 @@ impl PocketIcSubnets { } } - subnet_config + Ok(subnet_config) } fn get_nns(&self) -> Option> { @@ -1186,6 +1200,7 @@ impl PocketIc { log_level: Option, bitcoind_addr: Option>, icp_features: Option, + allow_corrupted_state: Option, ) -> Result { if let Some(ref icp_features) = icp_features { subnet_configs = subnet_configs.try_with_icp_features(icp_features)?; @@ -1232,6 +1247,11 @@ impl PocketIc { if let Some(allocation_range) = config.alloc_range { range_gen.add_assigned(vec![allocation_range]).unwrap(); } + let time = if let Some(true) = allow_corrupted_state { + None + } else { + Some(topology.time) + }; SubnetConfigInfo { ranges: config.ranges, alloc_range: config.alloc_range, @@ -1239,6 +1259,7 @@ impl PocketIc { subnet_state_dir: None, subnet_kind: config.subnet_kind, instruction_config: config.instruction_config, + time, } }) .collect() @@ -1387,6 +1408,7 @@ impl PocketIc { subnet_state_dir, subnet_kind, instruction_config, + time: None, }); } @@ -1412,7 +1434,7 @@ impl PocketIc { ); let mut subnet_configs = Vec::new(); for subnet_config_info in subnet_config_info.into_iter() { - let subnet_config_internal = subnets.create_subnet(subnet_config_info); + let subnet_config_internal = subnets.create_subnet(subnet_config_info)?; subnet_configs.push(subnet_config_internal); } @@ -1636,6 +1658,7 @@ struct SubnetConfigInfo { pub subnet_state_dir: Option, pub subnet_kind: SubnetKind, pub instruction_config: SubnetInstructionConfig, + pub time: Option, } // ---------------------------------------------------------------------------------------- // @@ -3213,7 +3236,8 @@ fn route( subnet_state_dir: None, subnet_kind, instruction_config, - }); + time: None, + })?; pic.subnets .persist_topology(pic.default_effective_canister_id); Ok(pic.try_route_canister(canister_id).unwrap()) @@ -3319,6 +3343,7 @@ mod tests { None, None, None, + None, ) .unwrap(); let mut pic1 = PocketIc::try_new( @@ -3333,6 +3358,7 @@ mod tests { None, None, None, + None, ) .unwrap(); assert_ne!(pic0.get_state_label(), pic1.get_state_label()); diff --git a/rs/pocket_ic_server/src/state_api/routes.rs b/rs/pocket_ic_server/src/state_api/routes.rs index da8717c16f3a..2ac8c0dfe31d 100644 --- a/rs/pocket_ic_server/src/state_api/routes.rs +++ b/rs/pocket_ic_server/src/state_api/routes.rs @@ -1213,6 +1213,7 @@ pub async fn create_instance( log_level, instance_config.bitcoind_addr, instance_config.icp_features, + instance_config.allow_corrupted_state, ) }) .await diff --git a/rs/pocket_ic_server/tests/test.rs b/rs/pocket_ic_server/tests/test.rs index b3d020559382..8ebd398a99bd 100644 --- a/rs/pocket_ic_server/tests/test.rs +++ b/rs/pocket_ic_server/tests/test.rs @@ -130,6 +130,7 @@ fn test_creation_of_instance_extended() { log_level: None, bitcoind_addr: None, icp_features: None, + allow_corrupted_state: None, }; let response = client .post(url.join("instances").unwrap()) From 3fabe9139b486ff5660c503bd6858a982cc704a9 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 14 Jul 2025 16:41:17 +0200 Subject: [PATCH 09/19] simplify --- rs/pocket_ic_server/src/pocket_ic.rs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/rs/pocket_ic_server/src/pocket_ic.rs b/rs/pocket_ic_server/src/pocket_ic.rs index 6d9f3fa8faee..46f2ef80237c 100644 --- a/rs/pocket_ic_server/src/pocket_ic.rs +++ b/rs/pocket_ic_server/src/pocket_ic.rs @@ -716,8 +716,7 @@ impl PocketIcSubnets { let subnet_id = sm.get_subnet_id(); if let Some(expected_time) = time { - let metadata = &sm.state_manager.get_latest_state().take().metadata; - let actual_time: SystemTime = metadata.batch_time.into(); + let actual_time: SystemTime = sm.get_state_time().into(); if actual_time != expected_time { return Err(format!("The state of subnet {} is corrupted.", subnet_id)); } @@ -792,13 +791,7 @@ impl PocketIcSubnets { // set the time to the maximum time in the latest state across all subnets. let mut time: SystemTime = GENESIS.into(); for subnet in self.subnets.get_all() { - let metadata = &subnet - .state_machine - .state_manager - .get_latest_state() - .take() - .metadata; - time = max(time, metadata.batch_time.into()); + time = max(time, subnet.state_machine.get_state_time().into()); } // Make sure time is strictly monotone. From 585a8e281bce83d2e635c0850b915d8c88f4bd72 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 14 Jul 2025 17:05:42 +0200 Subject: [PATCH 10/19] windows --- packages/pocket-ic/tests/tests.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/pocket-ic/tests/tests.rs b/packages/pocket-ic/tests/tests.rs index 9c48ea711b95..e99853fd8fb7 100644 --- a/packages/pocket-ic/tests/tests.rs +++ b/packages/pocket-ic/tests/tests.rs @@ -461,8 +461,14 @@ fn time_on_resumed_instance() { async fn resume_killed_instance_impl(allow_corrupted_state: Option) -> Result<(), String> { let (mut server, server_url) = start_server(StartServerParams::default()).await; let temp_dir = TempDir::new().unwrap(); + #[cfg(not(windows))] + let state_dir_path = temp_dir.path().to_path_buf(); + #[cfg(windows)] + let state_dir_path = windows_to_wsl(temp_dir.path().as_os_str().to_str().unwrap()) + .unwrap() + .into(); - let state = PocketIcState::new_from_path(temp_dir.path().to_path_buf()); + let state = PocketIcState::new_from_path(state_dir_path.clone()); let pic = PocketIcBuilder::new() .with_application_subnet() .with_server_url(server_url) @@ -493,7 +499,7 @@ async fn resume_killed_instance_impl(allow_corrupted_state: Option) -> Res let client = reqwest::Client::new(); let instance_config = InstanceConfig { subnet_config_set: ExtendedSubnetConfigSet::default(), - state_dir: Some(temp_dir.path().to_path_buf()), + state_dir: Some(state_dir_path), nonmainnet_features: false, log_level: None, bitcoind_addr: None, From be83881738e3daa39edd48400098eea64ab2897c Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 14 Jul 2025 17:13:24 +0200 Subject: [PATCH 11/19] PathBuf --- packages/pocket-ic/tests/tests.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/pocket-ic/tests/tests.rs b/packages/pocket-ic/tests/tests.rs index e99853fd8fb7..5e6f7f707fe6 100644 --- a/packages/pocket-ic/tests/tests.rs +++ b/packages/pocket-ic/tests/tests.rs @@ -28,6 +28,8 @@ use serde::Serialize; use sha2::{Digest, Sha256}; #[cfg(windows)] use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +#[cfg(windows)] +use std::path::PathBuf; use std::{ io::Read, sync::OnceLock, @@ -464,7 +466,7 @@ async fn resume_killed_instance_impl(allow_corrupted_state: Option) -> Res #[cfg(not(windows))] let state_dir_path = temp_dir.path().to_path_buf(); #[cfg(windows)] - let state_dir_path = windows_to_wsl(temp_dir.path().as_os_str().to_str().unwrap()) + let state_dir_path: PathBuf = windows_to_wsl(temp_dir.path().as_os_str().to_str().unwrap()) .unwrap() .into(); From 26f93d946a792315cf24d182d64c5bf2cca0d38b Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 14 Jul 2025 17:34:36 +0200 Subject: [PATCH 12/19] with_state_dir --- packages/pocket-ic/tests/tests.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/pocket-ic/tests/tests.rs b/packages/pocket-ic/tests/tests.rs index 5e6f7f707fe6..0d9a409d5a95 100644 --- a/packages/pocket-ic/tests/tests.rs +++ b/packages/pocket-ic/tests/tests.rs @@ -470,11 +470,10 @@ async fn resume_killed_instance_impl(allow_corrupted_state: Option) -> Res .unwrap() .into(); - let state = PocketIcState::new_from_path(state_dir_path.clone()); let pic = PocketIcBuilder::new() .with_application_subnet() .with_server_url(server_url) - .with_state(state) + .with_state_dir(state_dir_path.clone()) .build_async() .await; From 04ec09f16fc26eeedaef54c3f546f2ccd8c7b9e7 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 14 Jul 2025 22:32:33 +0200 Subject: [PATCH 13/19] subnet seed instead of subnet ID --- packages/pocket-ic/tests/tests.rs | 4 ++-- rs/pocket_ic_server/src/pocket_ic.rs | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/pocket-ic/tests/tests.rs b/packages/pocket-ic/tests/tests.rs index 0d9a409d5a95..8a013604ed98 100644 --- a/packages/pocket-ic/tests/tests.rs +++ b/packages/pocket-ic/tests/tests.rs @@ -538,13 +538,13 @@ async fn resume_killed_instance_impl(allow_corrupted_state: Option) -> Res #[tokio::test] async fn resume_killed_instance_default() { let err = resume_killed_instance_impl(None).await.unwrap_err(); - assert!(err.contains("The state of subnet qxw6a-xchhl-fytzl-qfagk-kuv6t-ye6xf-box4s-fiqod-vn2ex-qkp3q-5ae is corrupted.")); + assert!(err.contains("The state of subnet with seed 7712b2c09cb96b3aa3fbffd4034a21a39d5d13f80e043161d1d71f4c593434af is corrupted.")); } #[tokio::test] async fn resume_killed_instance_strict() { let err = resume_killed_instance_impl(Some(false)).await.unwrap_err(); - assert!(err.contains("The state of subnet qxw6a-xchhl-fytzl-qfagk-kuv6t-ye6xf-box4s-fiqod-vn2ex-qkp3q-5ae is corrupted.")); + assert!(err.contains("The state of subnet with seed 7712b2c09cb96b3aa3fbffd4034a21a39d5d13f80e043161d1d71f4c593434af is corrupted.")); } #[tokio::test] diff --git a/rs/pocket_ic_server/src/pocket_ic.rs b/rs/pocket_ic_server/src/pocket_ic.rs index 46f2ef80237c..9899b8a4bd6d 100644 --- a/rs/pocket_ic_server/src/pocket_ic.rs +++ b/rs/pocket_ic_server/src/pocket_ic.rs @@ -718,7 +718,10 @@ impl PocketIcSubnets { if let Some(expected_time) = time { let actual_time: SystemTime = sm.get_state_time().into(); if actual_time != expected_time { - return Err(format!("The state of subnet {} is corrupted.", subnet_id)); + return Err(format!( + "The state of subnet with seed {} is corrupted.", + hex::encode(subnet_seed) + )); } } From bde022d5fc7e4e9c6a6b87fc7086f6a99638b266 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 14 Jul 2025 22:40:21 +0200 Subject: [PATCH 14/19] no windows_to_wsl --- packages/pocket-ic/tests/tests.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/packages/pocket-ic/tests/tests.rs b/packages/pocket-ic/tests/tests.rs index 8a013604ed98..6d133f59d851 100644 --- a/packages/pocket-ic/tests/tests.rs +++ b/packages/pocket-ic/tests/tests.rs @@ -28,8 +28,6 @@ use serde::Serialize; use sha2::{Digest, Sha256}; #[cfg(windows)] use std::net::{IpAddr, Ipv4Addr, SocketAddr}; -#[cfg(windows)] -use std::path::PathBuf; use std::{ io::Read, sync::OnceLock, @@ -463,17 +461,13 @@ fn time_on_resumed_instance() { async fn resume_killed_instance_impl(allow_corrupted_state: Option) -> Result<(), String> { let (mut server, server_url) = start_server(StartServerParams::default()).await; let temp_dir = TempDir::new().unwrap(); - #[cfg(not(windows))] let state_dir_path = temp_dir.path().to_path_buf(); - #[cfg(windows)] - let state_dir_path: PathBuf = windows_to_wsl(temp_dir.path().as_os_str().to_str().unwrap()) - .unwrap() - .into(); + let state = PocketIcState::new_from_path(state_dir_path.clone()); let pic = PocketIcBuilder::new() .with_application_subnet() .with_server_url(server_url) - .with_state_dir(state_dir_path.clone()) + .with_state(state) .build_async() .await; From 572380a24b1622ef2b7e3631d30bf20c74016908 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 14 Jul 2025 23:19:26 +0200 Subject: [PATCH 15/19] Revert "no windows_to_wsl" This reverts commit bde022d5fc7e4e9c6a6b87fc7086f6a99638b266. --- packages/pocket-ic/tests/tests.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/pocket-ic/tests/tests.rs b/packages/pocket-ic/tests/tests.rs index 6d133f59d851..8a013604ed98 100644 --- a/packages/pocket-ic/tests/tests.rs +++ b/packages/pocket-ic/tests/tests.rs @@ -28,6 +28,8 @@ use serde::Serialize; use sha2::{Digest, Sha256}; #[cfg(windows)] use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +#[cfg(windows)] +use std::path::PathBuf; use std::{ io::Read, sync::OnceLock, @@ -461,13 +463,17 @@ fn time_on_resumed_instance() { async fn resume_killed_instance_impl(allow_corrupted_state: Option) -> Result<(), String> { let (mut server, server_url) = start_server(StartServerParams::default()).await; let temp_dir = TempDir::new().unwrap(); + #[cfg(not(windows))] let state_dir_path = temp_dir.path().to_path_buf(); + #[cfg(windows)] + let state_dir_path: PathBuf = windows_to_wsl(temp_dir.path().as_os_str().to_str().unwrap()) + .unwrap() + .into(); - let state = PocketIcState::new_from_path(state_dir_path.clone()); let pic = PocketIcBuilder::new() .with_application_subnet() .with_server_url(server_url) - .with_state(state) + .with_state_dir(state_dir_path.clone()) .build_async() .await; From a2e76e0c4bdcfab4bb483e6d1bca9222fbe809e7 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 14 Jul 2025 23:21:44 +0200 Subject: [PATCH 16/19] fix windows --- packages/pocket-ic/tests/tests.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/pocket-ic/tests/tests.rs b/packages/pocket-ic/tests/tests.rs index 8a013604ed98..38c0c6e6b272 100644 --- a/packages/pocket-ic/tests/tests.rs +++ b/packages/pocket-ic/tests/tests.rs @@ -463,17 +463,12 @@ fn time_on_resumed_instance() { async fn resume_killed_instance_impl(allow_corrupted_state: Option) -> Result<(), String> { let (mut server, server_url) = start_server(StartServerParams::default()).await; let temp_dir = TempDir::new().unwrap(); - #[cfg(not(windows))] - let state_dir_path = temp_dir.path().to_path_buf(); - #[cfg(windows)] - let state_dir_path: PathBuf = windows_to_wsl(temp_dir.path().as_os_str().to_str().unwrap()) - .unwrap() - .into(); + let state = PocketIcState::new_from_path(temp_dir.path().to_path_buf()); let pic = PocketIcBuilder::new() .with_application_subnet() .with_server_url(server_url) - .with_state_dir(state_dir_path.clone()) + .with_state(state) .build_async() .await; @@ -498,9 +493,15 @@ async fn resume_killed_instance_impl(allow_corrupted_state: Option) -> Res let (_, server_url) = start_server(StartServerParams::default()).await; let client = reqwest::Client::new(); + #[cfg(not(windows))] + let raw_state_dir_path = temp_dir.path().to_path_buf(); + #[cfg(windows)] + let raw_state_dir_path: PathBuf = windows_to_wsl(temp_dir.path().as_os_str().to_str().unwrap()) + .unwrap() + .into(); let instance_config = InstanceConfig { subnet_config_set: ExtendedSubnetConfigSet::default(), - state_dir: Some(state_dir_path), + state_dir: Some(raw_state_dir_path), nonmainnet_features: false, log_level: None, bitcoind_addr: None, From 877c208c876230cf9080fec6042c0d59115bce60 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Tue, 15 Jul 2025 09:52:41 +0200 Subject: [PATCH 17/19] do not test kill on Windows --- packages/pocket-ic/tests/tests.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/pocket-ic/tests/tests.rs b/packages/pocket-ic/tests/tests.rs index 38c0c6e6b272..55fb3d84f7d8 100644 --- a/packages/pocket-ic/tests/tests.rs +++ b/packages/pocket-ic/tests/tests.rs @@ -28,8 +28,6 @@ use serde::Serialize; use sha2::{Digest, Sha256}; #[cfg(windows)] use std::net::{IpAddr, Ipv4Addr, SocketAddr}; -#[cfg(windows)] -use std::path::PathBuf; use std::{ io::Read, sync::OnceLock, @@ -460,6 +458,8 @@ fn time_on_resumed_instance() { assert_eq!(resumed_time, time + Duration::from_nanos(2)); } +// Killing the PocketIC server inside WSL is challenging => skipping this test on Windows. +#[cfg(not(windows))] async fn resume_killed_instance_impl(allow_corrupted_state: Option) -> Result<(), String> { let (mut server, server_url) = start_server(StartServerParams::default()).await; let temp_dir = TempDir::new().unwrap(); @@ -493,15 +493,9 @@ async fn resume_killed_instance_impl(allow_corrupted_state: Option) -> Res let (_, server_url) = start_server(StartServerParams::default()).await; let client = reqwest::Client::new(); - #[cfg(not(windows))] - let raw_state_dir_path = temp_dir.path().to_path_buf(); - #[cfg(windows)] - let raw_state_dir_path: PathBuf = windows_to_wsl(temp_dir.path().as_os_str().to_str().unwrap()) - .unwrap() - .into(); let instance_config = InstanceConfig { subnet_config_set: ExtendedSubnetConfigSet::default(), - state_dir: Some(raw_state_dir_path), + state_dir: Some(temp_dir.path().to_path_buf()), nonmainnet_features: false, log_level: None, bitcoind_addr: None, @@ -536,18 +530,24 @@ async fn resume_killed_instance_impl(allow_corrupted_state: Option) -> Res Ok(()) } +// Killing the PocketIC server inside WSL is challenging => skipping this test on Windows. +#[cfg(not(windows))] #[tokio::test] async fn resume_killed_instance_default() { let err = resume_killed_instance_impl(None).await.unwrap_err(); assert!(err.contains("The state of subnet with seed 7712b2c09cb96b3aa3fbffd4034a21a39d5d13f80e043161d1d71f4c593434af is corrupted.")); } +// Killing the PocketIC server inside WSL is challenging => skipping this test on Windows. +#[cfg(not(windows))] #[tokio::test] async fn resume_killed_instance_strict() { let err = resume_killed_instance_impl(Some(false)).await.unwrap_err(); assert!(err.contains("The state of subnet with seed 7712b2c09cb96b3aa3fbffd4034a21a39d5d13f80e043161d1d71f4c593434af is corrupted.")); } +// Killing the PocketIC server inside WSL is challenging => skipping this test on Windows. +#[cfg(not(windows))] #[tokio::test] async fn resume_killed_instance() { resume_killed_instance_impl(Some(true)).await.unwrap(); From 474b6617534005a78d5c895bf1b9efb88c821723 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Tue, 15 Jul 2025 10:01:16 +0200 Subject: [PATCH 18/19] lint --- packages/pocket-ic/tests/tests.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/packages/pocket-ic/tests/tests.rs b/packages/pocket-ic/tests/tests.rs index 55fb3d84f7d8..4a0f551b78df 100644 --- a/packages/pocket-ic/tests/tests.rs +++ b/packages/pocket-ic/tests/tests.rs @@ -12,14 +12,17 @@ use ic_transport_types::EnvelopeContent::{Call, ReadState}; use pocket_ic::common::rest::{BlockmakerConfigs, RawSubnetBlockmaker, TickConfigs}; use pocket_ic::{ common::rest::{ - BlobCompression, CanisterHttpReply, CanisterHttpResponse, CreateInstanceResponse, - ExtendedSubnetConfigSet, InstanceConfig, MockCanisterHttpResponse, RawEffectivePrincipal, - RawMessageId, SubnetKind, + BlobCompression, CanisterHttpReply, CanisterHttpResponse, MockCanisterHttpResponse, + RawEffectivePrincipal, RawMessageId, SubnetKind, }, + query_candid, update_candid, DefaultEffectiveCanisterIdError, ErrorCode, IngressStatusResult, + PocketIc, PocketIcBuilder, PocketIcState, RejectCode, Time, +}; +#[cfg(not(windows))] +use pocket_ic::{ + common::rest::{CreateInstanceResponse, ExtendedSubnetConfigSet, InstanceConfig}, nonblocking::PocketIc as PocketIcAsync, - query_candid, start_server, update_candid, DefaultEffectiveCanisterIdError, ErrorCode, - IngressStatusResult, PocketIc, PocketIcBuilder, PocketIcState, RejectCode, StartServerParams, - Time, + start_server, StartServerParams, }; use reqwest::blocking::Client; use reqwest::header::CONTENT_LENGTH; From 3050f805c7c7d17d6cc45bb4ae3526ddcf1914d0 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 21 Jul 2025 08:16:43 +0200 Subject: [PATCH 19/19] corrupted -> incomplete --- packages/pocket-ic/src/common/rest.rs | 2 +- packages/pocket-ic/src/nonblocking.rs | 2 +- packages/pocket-ic/tests/icp_features.rs | 2 +- packages/pocket-ic/tests/tests.rs | 8 ++++---- rs/pocket_ic_server/CHANGELOG.md | 2 +- rs/pocket_ic_server/src/pocket_ic.rs | 6 +++--- rs/pocket_ic_server/src/state_api/routes.rs | 2 +- rs/pocket_ic_server/tests/test.rs | 2 +- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/pocket-ic/src/common/rest.rs b/packages/pocket-ic/src/common/rest.rs index 570537912905..8097f7e498e8 100644 --- a/packages/pocket-ic/src/common/rest.rs +++ b/packages/pocket-ic/src/common/rest.rs @@ -565,7 +565,7 @@ pub struct InstanceConfig { pub log_level: Option, pub bitcoind_addr: Option>, pub icp_features: Option, - pub allow_corrupted_state: Option, + pub allow_incomplete_state: Option, } #[derive(Debug, Clone, Eq, Hash, PartialEq, Serialize, Deserialize, Default, JsonSchema)] diff --git a/packages/pocket-ic/src/nonblocking.rs b/packages/pocket-ic/src/nonblocking.rs index 164023be424d..925210152cd4 100644 --- a/packages/pocket-ic/src/nonblocking.rs +++ b/packages/pocket-ic/src/nonblocking.rs @@ -205,7 +205,7 @@ impl PocketIc { log_level: log_level.map(|l| l.to_string()), bitcoind_addr, icp_features: Some(icp_features), - allow_corrupted_state: Some(false), + allow_incomplete_state: Some(false), }; let test_driver_pid = std::process::id(); diff --git a/packages/pocket-ic/tests/icp_features.rs b/packages/pocket-ic/tests/icp_features.rs index 1b6938a7a644..a7caa104741c 100644 --- a/packages/pocket-ic/tests/icp_features.rs +++ b/packages/pocket-ic/tests/icp_features.rs @@ -389,7 +389,7 @@ async fn with_all_icp_features_and_nns_subnet_state() { log_level: None, bitcoind_addr: None, icp_features: Some(IcpFeatures::all_icp_features()), - allow_corrupted_state: None, + allow_incomplete_state: None, }; let response = client .post(url.join("instances").unwrap()) diff --git a/packages/pocket-ic/tests/tests.rs b/packages/pocket-ic/tests/tests.rs index 4a0f551b78df..09591ad21bf9 100644 --- a/packages/pocket-ic/tests/tests.rs +++ b/packages/pocket-ic/tests/tests.rs @@ -463,7 +463,7 @@ fn time_on_resumed_instance() { // Killing the PocketIC server inside WSL is challenging => skipping this test on Windows. #[cfg(not(windows))] -async fn resume_killed_instance_impl(allow_corrupted_state: Option) -> Result<(), String> { +async fn resume_killed_instance_impl(allow_incomplete_state: Option) -> Result<(), String> { let (mut server, server_url) = start_server(StartServerParams::default()).await; let temp_dir = TempDir::new().unwrap(); @@ -503,7 +503,7 @@ async fn resume_killed_instance_impl(allow_corrupted_state: Option) -> Res log_level: None, bitcoind_addr: None, icp_features: None, - allow_corrupted_state, + allow_incomplete_state, }; let response = client .post(server_url.join("instances").unwrap()) @@ -538,7 +538,7 @@ async fn resume_killed_instance_impl(allow_corrupted_state: Option) -> Res #[tokio::test] async fn resume_killed_instance_default() { let err = resume_killed_instance_impl(None).await.unwrap_err(); - assert!(err.contains("The state of subnet with seed 7712b2c09cb96b3aa3fbffd4034a21a39d5d13f80e043161d1d71f4c593434af is corrupted.")); + assert!(err.contains("The state of subnet with seed 7712b2c09cb96b3aa3fbffd4034a21a39d5d13f80e043161d1d71f4c593434af is incomplete.")); } // Killing the PocketIC server inside WSL is challenging => skipping this test on Windows. @@ -546,7 +546,7 @@ async fn resume_killed_instance_default() { #[tokio::test] async fn resume_killed_instance_strict() { let err = resume_killed_instance_impl(Some(false)).await.unwrap_err(); - assert!(err.contains("The state of subnet with seed 7712b2c09cb96b3aa3fbffd4034a21a39d5d13f80e043161d1d71f4c593434af is corrupted.")); + assert!(err.contains("The state of subnet with seed 7712b2c09cb96b3aa3fbffd4034a21a39d5d13f80e043161d1d71f4c593434af is incomplete.")); } // Killing the PocketIC server inside WSL is challenging => skipping this test on Windows. diff --git a/rs/pocket_ic_server/CHANGELOG.md b/rs/pocket_ic_server/CHANGELOG.md index 42d7a9d65389..da364911198c 100644 --- a/rs/pocket_ic_server/CHANGELOG.md +++ b/rs/pocket_ic_server/CHANGELOG.md @@ -15,7 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The endpoint `/instances//update/await_ingress_message` (execute rounds on the PocketIc instance until the message is executed): to fix a performance regression when using the two endpoints `/instances//update/tick` and `/instances//read/ingress_status` in a loop. - The argument of the endpoint `/instances/` takes an additional optional field `icp_features` specifying ICP features (implemented by system canisters) to be enabled in the newly created PocketIC instance. -- The argument of the endpoint `/instances/` takes an additional optional field `allow_corrupted_state` specifying if corrupted state (e.g., resulting from not deleting a PocketIC instance gracefully) is allowed. +- The argument of the endpoint `/instances/` takes an additional optional field `allow_incomplete_state` specifying if incomplete state (e.g., resulting from not deleting a PocketIC instance gracefully) is allowed. diff --git a/rs/pocket_ic_server/src/pocket_ic.rs b/rs/pocket_ic_server/src/pocket_ic.rs index 66b16c8b6726..48401b1dfb7b 100644 --- a/rs/pocket_ic_server/src/pocket_ic.rs +++ b/rs/pocket_ic_server/src/pocket_ic.rs @@ -722,7 +722,7 @@ impl PocketIcSubnets { let actual_time: SystemTime = sm.get_state_time().into(); if actual_time != expected_time { return Err(format!( - "The state of subnet with seed {} is corrupted.", + "The state of subnet with seed {} is incomplete.", hex::encode(subnet_seed) )); } @@ -1331,7 +1331,7 @@ impl PocketIc { log_level: Option, bitcoind_addr: Option>, icp_features: Option, - allow_corrupted_state: Option, + allow_incomplete_state: Option, ) -> Result { if let Some(ref icp_features) = icp_features { subnet_configs = subnet_configs.try_with_icp_features(icp_features)?; @@ -1378,7 +1378,7 @@ impl PocketIc { if let Some(allocation_range) = config.alloc_range { range_gen.add_assigned(vec![allocation_range]).unwrap(); } - let time = if let Some(true) = allow_corrupted_state { + let time = if let Some(true) = allow_incomplete_state { None } else { Some(topology.time) diff --git a/rs/pocket_ic_server/src/state_api/routes.rs b/rs/pocket_ic_server/src/state_api/routes.rs index 2ac8c0dfe31d..4606c8aee0ab 100644 --- a/rs/pocket_ic_server/src/state_api/routes.rs +++ b/rs/pocket_ic_server/src/state_api/routes.rs @@ -1213,7 +1213,7 @@ pub async fn create_instance( log_level, instance_config.bitcoind_addr, instance_config.icp_features, - instance_config.allow_corrupted_state, + instance_config.allow_incomplete_state, ) }) .await diff --git a/rs/pocket_ic_server/tests/test.rs b/rs/pocket_ic_server/tests/test.rs index 8ebd398a99bd..371f1afff8a9 100644 --- a/rs/pocket_ic_server/tests/test.rs +++ b/rs/pocket_ic_server/tests/test.rs @@ -130,7 +130,7 @@ fn test_creation_of_instance_extended() { log_level: None, bitcoind_addr: None, icp_features: None, - allow_corrupted_state: None, + allow_incomplete_state: None, }; let response = client .post(url.join("instances").unwrap())