From 3520f4f384c55e132af3ce8f045e0e71d1b75e57 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 30 Nov 2023 16:23:20 -0800 Subject: [PATCH 1/4] Misc icx fixes --- Cargo.lock | 1 + Cargo.toml | 1 + ic-agent/Cargo.toml | 2 +- icx/Cargo.toml | 1 + icx/src/main.rs | 110 ++++++++++++++++++++++++++++++++------------ 5 files changed, 84 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eb19fc90..d81dac3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1205,6 +1205,7 @@ dependencies = [ "serde", "serde_json", "tokio", + "url", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 38758544..05e0e7d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,3 +41,4 @@ sha2 = "0.10.6" thiserror = "1.0.40" time = "0.3" tokio = "1.28.0" +url = "2.1" diff --git a/ic-agent/Cargo.toml b/ic-agent/Cargo.toml index 451a40c9..d011eeac 100644 --- a/ic-agent/Cargo.toml +++ b/ic-agent/Cargo.toml @@ -41,7 +41,7 @@ sha2 = { workspace = true } simple_asn1 = "0.6.1" thiserror = { workspace = true } time = { workspace = true } -url = "2.1.0" +url = { workspace = true } [dependencies.hyper] version = "0.14" diff --git a/icx/Cargo.toml b/icx/Cargo.toml index 61eb7f6d..7dfc1370 100644 --- a/icx/Cargo.toml +++ b/icx/Cargo.toml @@ -31,3 +31,4 @@ ring = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true, features = ["full"] } +url = { workspace = true } diff --git a/icx/src/main.rs b/icx/src/main.rs index 6e148d28..534bc489 100644 --- a/icx/src/main.rs +++ b/icx/src/main.rs @@ -5,7 +5,7 @@ use candid::{ CandidType, Decode, Deserialize, IDLArgs, TypeEnv, }; use candid_parser::{check_prog, parse_idl_args, parse_idl_value, IDLProg}; -use clap::{crate_authors, crate_version, Parser, ValueEnum}; +use clap::{builder::ArgPredicate, crate_authors, crate_version, Parser, ValueEnum}; use ic_agent::{ agent::{self, signed::SignedUpdate}, agent::{ @@ -22,11 +22,13 @@ use ic_utils::interfaces::management_canister::{ }; use ring::signature::Ed25519KeyPair; use std::{ - collections::VecDeque, convert::TryFrom, io::BufRead, path::PathBuf, str::FromStr, + convert::TryFrom, + io::Read, + path::{Path, PathBuf}, + str::FromStr, time::Duration, }; - -const DEFAULT_IC_GATEWAY: &str = "https://ic0.app"; +use url::{Host, Url}; #[derive(Parser)] #[clap( @@ -35,20 +37,27 @@ const DEFAULT_IC_GATEWAY: &str = "https://ic0.app"; propagate_version(true), )] struct Opts { - /// Some input. Because this isn't an Option it's required to be used - #[clap(default_value = "http://localhost:8000/")] - replica: String, + /// The URL of the replica to connect to. + #[clap( + default_value = "http://localhost:4943/", + default_value_if("ic", ArgPredicate::IsPresent, "https://icp0.io") + )] + replica: Url, /// An optional PEM file to read the identity from. If none is passed, /// a random identity will be created. - #[clap(long)] + #[clap(long, global = true)] pem: Option, /// An optional field to set the expiry time on requests. Can be a human /// readable time (like `100s`) or a number of seconds. - #[clap(long)] + #[clap(long, global = true)] ttl: Option, + /// Alias for `--replica https://icp0.io`. + #[clap(long, conflicts_with = "replica", global = true)] + ic: bool, + #[clap(subcommand)] subcommand: SubCommand, } @@ -64,10 +73,10 @@ enum SubCommand { /// Checks the `status` endpoints of the replica. Status, - /// Send a serialized request, taking from STDIN. - Send, + /// Send a serialized request, taking from a provided file or STDIN. + Send(SendOpts), - /// Transform Principal from hex to new text. + /// Transform a principal between text and hex. PrincipalConvert(PrincipalConvertOpts), } @@ -118,6 +127,13 @@ impl std::str::FromStr for ArgType { } } +#[derive(Parser)] +struct SendOpts { + /// The input file. Use `-` for STDIN. + #[clap(long, short, default_value = "-")] + input_file: PathBuf, +} + #[derive(Parser)] struct PrincipalConvertOpts { /// Convert from hexadecimal to the new group-based Principal text. @@ -179,7 +195,6 @@ fn blob_from_arguments( ) -> Result> { let mut buffer = Vec::new(); let arguments = if arguments == Some("-") { - use std::io::Read; std::io::stdin().read_to_end(&mut buffer).unwrap(); std::str::from_utf8(&buffer).ok() } else { @@ -251,16 +266,28 @@ fn print_idl_blob( Ok(()) } -async fn fetch_root_key_from_non_ic(agent: &Agent, replica: &str) -> Result<()> { - let normalized_replica = replica.strip_suffix('/').unwrap_or(replica); - if normalized_replica != DEFAULT_IC_GATEWAY { - agent - .fetch_root_key() - .await - .context("Failed to fetch root key from replica")?; +async fn fetch_root_key_from_non_ic(agent: &Agent, replica: &Url) -> Result<()> { + if is_mainnet(replica) { + agent.fetch_root_key().await?; } Ok(()) } + +fn is_mainnet(replica: &Url) -> bool { + if let Some(Host::Domain(domain)) = replica.host() { + let domain = domain.strip_suffix('.').unwrap_or(domain); + let subdomain_end = domain.rmatch_indices('.').nth(1); + let domain = if let Some((n, _)) = subdomain_end { + &domain[n + 1..] + } else { + domain + }; + ["ic0.app", "icp0.io", "icp-api.io"].contains(&domain) + } else { + false + } +} + pub fn get_effective_canister_id( is_management_canister: bool, method_name: &str, @@ -513,17 +540,18 @@ async fn main() -> Result<()> { eprintln!("Hexadecimal: {}", hex::encode(p.as_slice())); } } - SubCommand::Send => { - let input: VecDeque = std::io::stdin() - .lock() - .lines() - .collect::, std::io::Error>>() - .context("Failed to read from stdin")?; + SubCommand::Send(t) => { let mut buffer = String::new(); - for line in input { - buffer.push_str(&line); + if t.input_file == Path::new("-") { + std::io::stdin() + .lock() + .read_to_string(&mut buffer) + .context("failed to read from stdin")?; + } else { + buffer = std::fs::read_to_string(&t.input_file).with_context(|| { + format!("failed to read from file {}", t.input_file.display()) + })?; } - println!("{}", buffer); if let Ok(signed_update) = serde_json::from_str::(&buffer) { @@ -585,11 +613,33 @@ async fn main() -> Result<()> { #[cfg(test)] mod tests { - use crate::Opts; + use crate::{is_mainnet, Opts}; + use anyhow::Result; use clap::CommandFactory; #[test] fn valid_command() { Opts::command().debug_assert(); } + + #[test] + fn detects_mainnet() -> Result<()> { + assert!(is_mainnet(&"https://icp-api.io".parse()?)); + assert!(is_mainnet(&"https://ic0.app".parse()?)); + assert!(is_mainnet(&"https://icp0.io".parse()?)); + assert!(is_mainnet(&"https://icp-api.io:443".parse()?)); + assert!(is_mainnet(&"https://icp-api.io.".parse()?)); + assert!(is_mainnet(&"https://icp-api.io.:443".parse()?)); + assert!(is_mainnet( + &"https://ryjl3-tyaaa-aaaaa-aaaba-cai.icp0.io".parse()? + )); + + assert!(!is_mainnet(&"http://localhost".parse()?)); + assert!(!is_mainnet(&"http://[::1]".parse()?)); + assert!(!is_mainnet(&"http://127.0.0.1".parse()?)); + assert!(!is_mainnet( + &"http://ryjl3-tyaaa-aaaaa-aaaba-cai.localhost".parse()? + )); + Ok(()) + } } From 59e14ea7081be5903ef881aa006ff47b6b395714 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 30 Nov 2023 16:29:31 -0800 Subject: [PATCH 2/4] more --- icx/src/main.rs | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/icx/src/main.rs b/icx/src/main.rs index 534bc489..7731181a 100644 --- a/icx/src/main.rs +++ b/icx/src/main.rs @@ -193,10 +193,10 @@ fn blob_from_arguments( arg_type: &ArgType, method_type: &Option<(TypeEnv, Function)>, ) -> Result> { - let mut buffer = Vec::new(); + let mut buffer = String::new(); let arguments = if arguments == Some("-") { - std::io::stdin().read_to_end(&mut buffer).unwrap(); - std::str::from_utf8(&buffer).ok() + std::io::stdin().read_to_string(&mut buffer)?; + Some(&buffer[..]) } else { arguments }; @@ -343,19 +343,19 @@ pub fn get_effective_canister_id( } } -fn create_identity(maybe_pem: Option) -> impl Identity { +fn create_identity(maybe_pem: Option) -> Result { if let Some(pem_path) = maybe_pem { - BasicIdentity::from_pem_file(pem_path).expect("Could not read the key pair.") + BasicIdentity::from_pem_file(pem_path).context("Could not read the key pair.") } else { let rng = ring::rand::SystemRandom::new(); let pkcs8_bytes = ring::signature::Ed25519KeyPair::generate_pkcs8(&rng) - .expect("Could not generate a key pair.") + .context("Could not generate a key pair.")? .as_ref() .to_vec(); - BasicIdentity::from_key_pair( - Ed25519KeyPair::from_pkcs8(&pkcs8_bytes).expect("Could not generate the key pair."), - ) + Ok(BasicIdentity::from_key_pair( + Ed25519KeyPair::from_pkcs8(&pkcs8_bytes).context("Could not generate the key pair.")?, + )) } } @@ -368,7 +368,7 @@ async fn main() -> Result<()> { agent::http_transport::ReqwestTransport::create(opts.replica.clone()) .context("Failed to create Transport for Agent")?, ) - .with_boxed_identity(Box::new(create_identity(opts.pem))) + .with_boxed_identity(Box::new(create_identity(opts.pem)?)) .build() .context("Failed to build the Agent")?; @@ -493,7 +493,7 @@ async fn main() -> Result<()> { .with_effective_canister_id(effective_canister_id) .sign() .context("Failed to sign the update call")?; - let serialized = serde_json::to_string(&signed_update).unwrap(); + let serialized = serde_json::to_string(&signed_update)?; println!("{}", serialized); let signed_request_status = agent @@ -501,7 +501,7 @@ async fn main() -> Result<()> { .context( "Failed to sign the request_status call accompany with the update", )?; - let serialized = serde_json::to_string(&signed_request_status).unwrap(); + let serialized = serde_json::to_string(&signed_request_status)?; println!("{}", serialized); } &SubCommand::Query(_) => { @@ -515,7 +515,7 @@ async fn main() -> Result<()> { .with_effective_canister_id(effective_canister_id) .sign() .context("Failed to sign the query call")?; - let serialized = serde_json::to_string(&signed_query).unwrap(); + let serialized = serde_json::to_string(&signed_query)?; println!("{}", serialized); } _ => unreachable!(), @@ -531,12 +531,12 @@ async fn main() -> Result<()> { } SubCommand::PrincipalConvert(t) => { if let Some(hex) = &t.from_hex { - let p = Principal::try_from(hex::decode(hex).expect("Could not decode hex: {}")) - .expect("Could not transform into a Principal: {}"); + let p = Principal::try_from(hex::decode(hex).context("Could not decode hex")?) + .context("Could not transform into a principal")?; eprintln!("Principal: {}", p); } else if let Some(txt) = &t.to_hex { let p = Principal::from_text(txt.as_str()) - .expect("Could not transform into a Principal: {}"); + .context("Could not transform into a principal")?; eprintln!("Hexadecimal: {}", hex::encode(p.as_slice())); } } From 47febe31311bb4739cf35dc8a52f5d807e016b62 Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 30 Nov 2023 16:34:44 -0800 Subject: [PATCH 3/4] . --- Cargo.lock | 16 ++++++++-------- icx/src/main.rs | 29 ++++++++++++++++------------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d81dac3a..76294e1e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -388,9 +388,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.4.6" +version = "4.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956" +checksum = "41fffed7514f420abec6d183b1d3acfd9099c79c3a10a06ade4f8203f1411272" dependencies = [ "clap_builder", "clap_derive", @@ -398,9 +398,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.6" +version = "4.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45" +checksum = "63361bae7eef3771745f02d8d892bec2fee5f6e34af316ba556e7f97a7069ff1" dependencies = [ "anstream", "anstyle", @@ -410,9 +410,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.4.2" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ "heck", "proc-macro2", @@ -422,9 +422,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" [[package]] name = "codespan-reporting" diff --git a/icx/src/main.rs b/icx/src/main.rs index 7731181a..f24b18d2 100644 --- a/icx/src/main.rs +++ b/icx/src/main.rs @@ -5,7 +5,7 @@ use candid::{ CandidType, Decode, Deserialize, IDLArgs, TypeEnv, }; use candid_parser::{check_prog, parse_idl_args, parse_idl_value, IDLProg}; -use clap::{builder::ArgPredicate, crate_authors, crate_version, Parser, ValueEnum}; +use clap::{crate_authors, crate_version, Parser, ValueEnum}; use ic_agent::{ agent::{self, signed::SignedUpdate}, agent::{ @@ -38,10 +38,7 @@ use url::{Host, Url}; )] struct Opts { /// The URL of the replica to connect to. - #[clap( - default_value = "http://localhost:4943/", - default_value_if("ic", ArgPredicate::IsPresent, "https://icp0.io") - )] + #[clap(default_value = "http://localhost:4943/", conflicts_with = "ic")] replica: Url, /// An optional PEM file to read the identity from. If none is passed, @@ -55,7 +52,7 @@ struct Opts { ttl: Option, /// Alias for `--replica https://icp0.io`. - #[clap(long, conflicts_with = "replica", global = true)] + #[clap(long, global = true)] ic: bool, #[clap(subcommand)] @@ -363,9 +360,15 @@ fn create_identity(maybe_pem: Option) -> Result { async fn main() -> Result<()> { let opts: Opts = Opts::parse(); + let replica = if opts.ic { + "https://icp0.io".parse().unwrap() + } else { + opts.replica + }; + let agent = Agent::builder() .with_transport( - agent::http_transport::ReqwestTransport::create(opts.replica.clone()) + agent::http_transport::ReqwestTransport::create(replica.as_str()) .context("Failed to create Transport for Agent")?, ) .with_boxed_identity(Box::new(create_identity(opts.pem)?)) @@ -400,7 +403,7 @@ async fn main() -> Result<()> { let result = match &opts.subcommand { SubCommand::Update(_) => { // We need to fetch the root key for updates. - fetch_root_key_from_non_ic(&agent, &opts.replica).await?; + fetch_root_key_from_non_ic(&agent, &replica).await?; let mut builder = agent.update(&t.canister_id, &t.method_name); @@ -426,7 +429,7 @@ async fn main() -> Result<()> { result.unwrap_or(Err(AgentError::TimeoutWaitingForResponse())) } SubCommand::Query(_) => { - fetch_root_key_from_non_ic(&agent, &opts.replica).await?; + fetch_root_key_from_non_ic(&agent, &replica).await?; let mut builder = agent.query(&t.canister_id, &t.method_name); if let Some(d) = expire_after { builder = builder.expire_after(d); @@ -482,7 +485,7 @@ async fn main() -> Result<()> { // For local emulator, we need to fetch the root key for updates. // So on an air-gapped machine, we can only generate message for the IC main net // which agent hard-coded its root key - fetch_root_key_from_non_ic(&agent, &opts.replica).await?; + fetch_root_key_from_non_ic(&agent, &replica).await?; let mut builder = agent.update(&t.canister_id, &t.method_name); if let Some(d) = expire_after { @@ -505,7 +508,7 @@ async fn main() -> Result<()> { println!("{}", serialized); } &SubCommand::Query(_) => { - fetch_root_key_from_non_ic(&agent, &opts.replica).await?; + fetch_root_key_from_non_ic(&agent, &replica).await?; let mut builder = agent.query(&t.canister_id, &t.method_name); if let Some(d) = expire_after { builder = builder.expire_after(d); @@ -555,7 +558,7 @@ async fn main() -> Result<()> { println!("{}", buffer); if let Ok(signed_update) = serde_json::from_str::(&buffer) { - fetch_root_key_from_non_ic(&agent, &opts.replica).await?; + fetch_root_key_from_non_ic(&agent, &replica).await?; let request_id = agent .update_signed( signed_update.effective_canister_id, @@ -577,7 +580,7 @@ async fn main() -> Result<()> { } else if let Ok(signed_request_status) = serde_json::from_str::(&buffer) { - fetch_root_key_from_non_ic(&agent, &opts.replica).await?; + fetch_root_key_from_non_ic(&agent, &replica).await?; let response = agent .request_status_signed( &signed_request_status.request_id, From 4ce5f39316c931e564674faa8db443b371dc89fb Mon Sep 17 00:00:00 2001 From: Adam Spofford Date: Thu, 30 Nov 2023 17:19:05 -0800 Subject: [PATCH 4/4] how old is this? --- icx/src/main.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/icx/src/main.rs b/icx/src/main.rs index f24b18d2..c695bd3b 100644 --- a/icx/src/main.rs +++ b/icx/src/main.rs @@ -133,10 +133,10 @@ struct SendOpts { #[derive(Parser)] struct PrincipalConvertOpts { - /// Convert from hexadecimal to the new group-based Principal text. + /// Convert from hexadecimal to textual format. #[clap(long)] from_hex: Option, - /// Convert from the new group-based Principal text to hexadecimal. + /// Convert from textual format to hexadecimal. #[clap(long)] to_hex: Option, } @@ -305,7 +305,7 @@ pub fn get_effective_canister_id( ), MgmtMethod::InstallCode => { let install_args = Decode!(arg_value, CanisterInstall) - .context("Argument is not valid for CanisterInstall")?; + .context("Argument is not valid for install_code")?; Ok(install_args.canister_id) } MgmtMethod::StartCanister @@ -320,7 +320,7 @@ pub fn get_effective_canister_id( canister_id: Principal, } let in_args = - Decode!(arg_value, In).context("Argument is not a valid Principal")?; + Decode!(arg_value, In).context("Argument is not a valid principal")?; Ok(in_args.canister_id) } MgmtMethod::ProvisionalCreateCanisterWithCycles => Ok(Principal::management_canister()), @@ -331,7 +331,7 @@ pub fn get_effective_canister_id( settings: CanisterSettings, } let in_args = - Decode!(arg_value, In).context("Argument is not valid for UpdateSettings")?; + Decode!(arg_value, In).context("Argument is not valid for update_settings")?; Ok(in_args.canister_id) } }