From e407642a1afc0937cc984aecb6884b8b92f021f6 Mon Sep 17 00:00:00 2001 From: Charlotte Andersson Date: Mon, 3 Jun 2024 11:42:20 +0200 Subject: [PATCH] feat: paid tunnels --- Cargo.lock | 30 ++++++++ clear-unused-dns.ts | 84 +++++++++++++++----- linkup-cli/Cargo.toml | 1 + linkup-cli/src/background_booting.rs | 55 ++++++++------ linkup-cli/src/paid_tunnel.rs | 21 +++-- linkup-cli/src/services/tunnel.rs | 18 ++--- linkup-cli/src/start.rs | 110 +++++++++++++++++++++------ 7 files changed, 233 insertions(+), 86 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 083adc1..59711f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -531,6 +531,29 @@ dependencies = [ "syn", ] +[[package]] +name = "env_filter" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b35839ba51819680ba087cd351788c9a3c476841207e0b8cee0b04722343b9" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -885,6 +908,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "1.3.1" @@ -1126,6 +1155,7 @@ dependencies = [ "colored", "ctrlc", "daemonize", + "env_logger", "hickory-resolver", "linkup", "linkup-local-server", diff --git a/clear-unused-dns.ts b/clear-unused-dns.ts index 232eb16..3d4c2da 100644 --- a/clear-unused-dns.ts +++ b/clear-unused-dns.ts @@ -1,9 +1,4 @@ -async function deleteTXTRecords( - recordName: string, - zoneId: string, - apiToken: string, - email: string -) { +async function deleteTXTRecords(zoneId: string) { const baseUrl = `https://api.cloudflare.com/client/v4/zones/${zoneId}/dns_records`; // Fetch all DNS records @@ -11,24 +6,70 @@ async function deleteTXTRecords( const response = await fetch(`${baseUrl}?per_page=1000`, { headers: { "Content-Type": "application/json", - "X-Auth-Email": email, - "X-Auth-Key": apiToken, + Authorization: `Bearer ${api_token}`, }, }); const data = await response.json(); return data.result.filter( (record: any) => - record.type === "TXT" && record.name.startsWith(recordName) + record.type === "TXT" && record.name.startsWith("_acme-challenge") ); }; + const recordsToDelete = await getRecords(); + await doBatchDelete(baseUrl, recordsToDelete); +} + +async function deleteTunnelCNAMERecords() { + const baseUrl = `https://api.cloudflare.com/client/v4/zones/${mentimeter_dev_zone_id}/dns_records`; + + // Fetch all DNS records + const getRecords = async () => { + const response = await fetch(`${baseUrl}?per_page=1000`, { + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${api_token}`, + }, + }); + const data = await response.json(); + return data.result.filter( + (record: any) => + record.type === "CNAME" && record.name.startsWith("tunnel-") + ); + }; + + const recordsToDelete = await getRecords(); + await doBatchDelete(baseUrl, recordsToDelete); +} + +async function deleteTunnels() { + const baseUrl = `https://api.cloudflare.com/client/v4/accounts/${account_id}/cfd_tunnel`; + + // Fetch all tunnels + const getRecords = async () => { + const response = await fetch(`${baseUrl}?per_page=1000`, { + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${api_token}`, + }, + }); + const data = await response.json(); + return data.result.filter( + (tunnel: any) => tunnel.name.startsWith("tunnel-") && !tunnel.deleted_at + ); + }; + + const recordsToDelete = await getRecords(); + await doBatchDelete(baseUrl, recordsToDelete); +} + +async function doBatchDelete(url: string, recordsToDelete: any[]) { const deleteRecord = async (id: string) => { - const response = await fetch(`${baseUrl}/${id}`, { + const response = await fetch(`${url}/${id}`, { method: "DELETE", headers: { "Content-Type": "application/json", - "X-Auth-Email": email, - "X-Auth-Key": apiToken, + Authorization: `Bearer ${api_token}`, }, }); @@ -40,7 +81,6 @@ async function deleteTXTRecords( return response.json(); }; - const recordsToDelete = await getRecords(); console.log("Records to delete:", recordsToDelete.length); // Batch deletion, 20 records at a time @@ -56,18 +96,22 @@ async function deleteTXTRecords( await batchDelete(recordsToDelete); } -const zones = process.argv.slice(2); -const apikey = process.env.CLOUDFLARE_API_KEY; -const email = process.env.CLOUDFLARE_EMAIL; +const api_token = process.env.LINKUP_CF_API_TOKEN; +const account_id = process.env.LINKUP_CLOUDFLARE_ACCOUNT_ID; +const mentimeter_dev_zone_id = process.env.LINKUP_CLOUDFLARE_ZONE_ID; -if (!apikey || !email) { - console.error("Missing Cloudflare API key or email"); +if (!api_token || !account_id || !mentimeter_dev_zone_id) { + console.error("Missing Cloudflare API Token, Account ID or Zone ID"); console.error( - "Please set CLOUDFLARE_API_KEY and CLOUDFLARE_EMAIL environment variables" + "Run `menti localsecrets` to set the required environment variables" ); process.exit(1); } +deleteTunnelCNAMERecords(); +deleteTunnels(); + +const zones = process.argv.slice(2); if (zones.length === 0) { console.error("No zones specified"); console.error("Usage: clear-unused-dns.ts zone1 zone2 ..."); @@ -76,5 +120,5 @@ if (zones.length === 0) { for (const zone of zones) { console.log("Deleting records for zone:", zone); - deleteTXTRecords("_acme-challenge", zone, apikey, email); + deleteTXTRecords(zone); } diff --git a/linkup-cli/Cargo.toml b/linkup-cli/Cargo.toml index cb4659c..28d1775 100644 --- a/linkup-cli/Cargo.toml +++ b/linkup-cli/Cargo.toml @@ -30,6 +30,7 @@ serde_yaml = "0.9" thiserror = "1" url = { version = "2.5", features = ["serde"] } base64 = "0.22.1" +env_logger = "0.11.3" [dev-dependencies] mockall = "0.12.1" diff --git a/linkup-cli/src/background_booting.rs b/linkup-cli/src/background_booting.rs index 954343f..37bf553 100644 --- a/linkup-cli/src/background_booting.rs +++ b/linkup-cli/src/background_booting.rs @@ -12,7 +12,7 @@ use url::Url; use crate::local_config::{LocalState, ServiceTarget}; use crate::services::local_server::{is_local_server_started, start_local_server}; -use crate::services::tunnel::{is_tunnel_running, RealTunnelManager, TunnelManager}; +use crate::services::tunnel::{RealTunnelManager, TunnelManager}; use crate::status::print_session_names; use crate::worker_client::WorkerClient; use crate::{linkup_file_path, services, LINKUP_LOCALSERVER_PORT}; @@ -21,6 +21,7 @@ use crate::{CliError, LINKUP_LOCALDNS_INSTALL}; #[cfg_attr(test, mockall::automock)] pub trait BackgroundServices { fn boot_background_services(&self, state: LocalState) -> Result; + fn boot_local_dns(&self, domains: Vec, session_name: String) -> Result<(), CliError>; } pub struct RealBackgroundServices; @@ -40,19 +41,21 @@ impl BackgroundServices for RealBackgroundServices { wait_till_ok(format!("{}linkup-check", local_url))?; let should_run_free = state.linkup.is_paid.is_none() || !state.linkup.is_paid.unwrap(); - if state.should_use_tunnel() && should_run_free { - if is_tunnel_running().is_err() { - println!("Starting tunnel..."); + if should_run_free { + if state.should_use_tunnel() { let tunnel_manager = RealTunnelManager {}; - let tunnel = tunnel_manager.run_tunnel(&state)?; - state.linkup.tunnel = Some(tunnel); + if tunnel_manager.is_tunnel_running().is_err() { + println!("Starting tunnel..."); + let tunnel = tunnel_manager.run_tunnel(&state)?; + state.linkup.tunnel = Some(tunnel); + } else { + println!("Cloudflare tunnel was already running.. Try stopping linkup first if you have problems."); + } } else { - println!("Cloudflare tunnel was already running.. Try stopping linkup first if you have problems."); + println!( + "Skipping tunnel start... WARNING: not all kinds of requests will work in this mode." + ); } - } else { - println!( - "Skipping tunnel start... WARNING: not all kinds of requests will work in this mode." - ); } let server_config = ServerConfig::from(&state); @@ -72,22 +75,31 @@ impl BackgroundServices for RealBackgroundServices { state.linkup.session_name = server_session_name; state.save()?; - if linkup_file_path(LINKUP_LOCALDNS_INSTALL).exists() { - boot_local_dns(state.domain_strings(), state.linkup.session_name.clone())?; - } + if should_run_free { + if linkup_file_path(LINKUP_LOCALDNS_INSTALL).exists() { + self.boot_local_dns(state.domain_strings(), state.linkup.session_name.clone())?; + } - if let Some(tunnel) = &state.linkup.tunnel { - println!("Waiting for tunnel DNS to propagate at {}...", tunnel); + if let Some(tunnel) = &state.linkup.tunnel { + println!("Waiting for tunnel DNS to propagate at {}...", tunnel); - wait_for_dns_ok(tunnel.clone())?; + wait_for_dns_ok(tunnel.clone())?; - println!(); + println!(); + } } print_session_names(&state); Ok(state) } + + fn boot_local_dns(&self, domains: Vec, session_name: String) -> Result<(), CliError> { + services::caddy::start(domains.clone())?; + services::dnsmasq::start(domains, session_name)?; + + Ok(()) + } } pub fn load_config( @@ -110,13 +122,6 @@ pub fn load_config( Ok(content) } -pub fn boot_local_dns(domains: Vec, session_name: String) -> Result<(), CliError> { - services::caddy::start(domains.clone())?; - services::dnsmasq::start(domains, session_name)?; - - Ok(()) -} - pub struct ServerConfig { pub local: StorableSession, pub remote: StorableSession, diff --git a/linkup-cli/src/paid_tunnel.rs b/linkup-cli/src/paid_tunnel.rs index 985d4d7..7d04f31 100644 --- a/linkup-cli/src/paid_tunnel.rs +++ b/linkup-cli/src/paid_tunnel.rs @@ -18,6 +18,7 @@ struct GetTunnelApiResponse { struct TunnelResultItem { id: String, name: String, + deleted_at: Option, } #[derive(Serialize, Deserialize, Debug)] @@ -138,13 +139,21 @@ impl PaidTunnelManager for RealPaidTunnelManager { account_id ); let (client, headers) = prepare_client_and_headers(&RealSystem)?; - let query_url = format!("{}?name=tunnel-{}", url, tunnel_name); + let query_url = format!("{}?name={}", url, tunnel_name); let parsed: GetTunnelApiResponse = send_request(&client, &query_url, headers, None, "GET")?; if parsed.result.is_empty() { Ok(None) } else { - Ok(Some(parsed.result[0].id.clone())) + // Check if there exists a tunnel with this name that hasn't been deleted + match parsed + .result + .iter() + .find(|tunnel| tunnel.deleted_at.is_none()) + { + Some(tunnel) => Ok(Some(tunnel.id.clone())), + None => Ok(None), + } } } @@ -158,7 +167,7 @@ impl PaidTunnelManager for RealPaidTunnelManager { ); let (client, headers) = prepare_client_and_headers(&RealSystem)?; let body = serde_json::to_string(&CreateTunnelRequest { - name: format!("tunnel-{}", tunnel_name), + name: tunnel_name.to_string(), tunnel_secret: tunnel_secret.clone(), }) .map_err(|err| CliError::StatusErr(err.to_string()))?; @@ -182,15 +191,13 @@ impl PaidTunnelManager for RealPaidTunnelManager { ); let (client, headers) = prepare_client_and_headers(&RealSystem)?; let body = serde_json::to_string(&DNSRecord { - name: format!("tunnel-{}", tunnel_name), + name: tunnel_name.to_string(), content: format!("{}.cfargotunnel.com", tunnel_id), r#type: "CNAME".to_string(), proxied: true, }) .map_err(|err| CliError::StatusErr(err.to_string()))?; - println!("{}", body); - let _parsed: CreateDNSRecordResponse = send_request(&client, &url, headers, Some(body), "POST")?; Ok(()) @@ -251,7 +258,7 @@ fn create_config_yml(sys: &dyn System, tunnel_id: &str) -> Result<(), CliError> // Create the directory if it does not exist if !sys.file_exists(dir_path.as_path()) { - println!("Creating directory: {:?}", dir_path); + log::info!("Creating directory: {:?}", dir_path); sys.create_dir_all(&dir_path) .map_err(|err| CliError::StatusErr(err.to_string()))?; } diff --git a/linkup-cli/src/services/tunnel.rs b/linkup-cli/src/services/tunnel.rs index e9d5dc7..5a3680f 100644 --- a/linkup-cli/src/services/tunnel.rs +++ b/linkup-cli/src/services/tunnel.rs @@ -22,17 +22,10 @@ const LINKUP_CLOUDFLARED_STDERR: &str = "cloudflared-stderr"; const TUNNEL_START_WAIT: u64 = 20; -pub fn is_tunnel_running() -> Result<(), CheckErr> { - if !linkup_file_path(LINKUP_CLOUDFLARED_PID).exists() { - Err(CheckErr::TunnelNotRunning) - } else { - Ok(()) - } -} - #[cfg_attr(test, mockall::automock)] pub trait TunnelManager { fn run_tunnel(&self, state: &LocalState) -> Result; + fn is_tunnel_running(&self) -> Result<(), CheckErr>; } pub struct RealTunnelManager; @@ -61,6 +54,13 @@ impl TunnelManager for RealTunnelManager { } } } + fn is_tunnel_running(&self) -> Result<(), CheckErr> { + if !linkup_file_path(LINKUP_CLOUDFLARED_PID).exists() { + Err(CheckErr::TunnelNotRunning) + } else { + Ok(()) + } + } } fn try_run_tunnel(state: &LocalState) -> Result { @@ -179,7 +179,7 @@ fn daemonized_tunnel_child(state: &LocalState) { true => vec!["tunnel", "run", state.linkup.session_name.as_str()], false => vec!["tunnel", "--url", url.as_str()], }; - println!("Starting cloudflared tunnel with args: {:?}", cmd_args); + log::info!("Starting cloudflared tunnel with args: {:?}", cmd_args); let mut child_cmd = Command::new("cloudflared") .args(cmd_args) .stdout(Stdio::inherit()) diff --git a/linkup-cli/src/start.rs b/linkup-cli/src/start.rs index 38176cf..1459793 100644 --- a/linkup-cli/src/start.rs +++ b/linkup-cli/src/start.rs @@ -20,6 +20,7 @@ use crate::{ }; pub fn start(config_arg: &Option, no_tunnel: bool) -> Result<(), CliError> { + env_logger::init(); let is_paid = use_paid_tunnels(); let state = load_and_save_state(config_arg, no_tunnel, is_paid)?; if is_paid { @@ -51,12 +52,12 @@ fn start_paid_tunnel( ) -> Result<(), CliError> { state = boot.boot_background_services(state.clone())?; - println!( + log::info!( "Starting paid tunnel with session name: {}", state.linkup.session_name ); - let tunnel_name = state.linkup.session_name.to_string(); - let tunnel_id = match paid_manager.get_tunnel_id(&tunnel_name) { + let tunnel_name = format!("tunnel-{}", state.linkup.session_name); + let mut tunnel_id = match paid_manager.get_tunnel_id(&tunnel_name) { Ok(Some(id)) => id, Ok(None) => "".to_string(), Err(e) => return Err(e), @@ -65,29 +66,35 @@ fn start_paid_tunnel( let mut create_tunnel = false; if tunnel_id.is_empty() { - println!("Tunnel ID is empty"); + log::info!("Tunnel ID is empty"); create_tunnel = true; } else { - println!("Tunnel ID: {}", tunnel_id); + log::info!("Tunnel ID: {}", tunnel_id); let file_path = format!("{}/.cloudflared/{}.json", sys.get_env("HOME")?, tunnel_id); if sys.file_exists(Path::new(&file_path)) { - println!("Tunnel config file for {}: {}", tunnel_id, file_path); + log::info!("Tunnel config file for {}: {}", tunnel_id, file_path); } else { - println!("Tunnel config file for {} does not exist", tunnel_id); + log::info!("Tunnel config file for {} does not exist", tunnel_id); create_tunnel = true; } } if create_tunnel { - println!("Creating tunnel"); - let tunnel_id = paid_manager.create_tunnel(&tunnel_name)?; + println!("Creating tunnel..."); + tunnel_id = paid_manager.create_tunnel(&tunnel_name)?; paid_manager.create_dns_record(&tunnel_id, &tunnel_name)?; } - let tunnel = tunnel_manager.run_tunnel(&state)?; - state.linkup.tunnel = Some(tunnel); + if tunnel_manager.is_tunnel_running().is_err() { + println!("Starting paid tunnel..."); + state.linkup.tunnel = Some(tunnel_manager.run_tunnel(&state)?); + } state.save()?; + if sys.file_exists(&linkup_file_path(LINKUP_LOCALDNS_INSTALL)) { + boot.boot_local_dns(state.domain_strings(), state.linkup.session_name.clone())?; + } + Ok(()) } @@ -127,7 +134,6 @@ fn load_and_save_state( // Reuse previous session name if possible if let Ok(ps) = previous_state { - //println!("Previous session name: {}", ps.linkup.session_name); state.linkup.session_name = ps.linkup.session_name; state.linkup.session_token = ps.linkup.session_token; @@ -203,7 +209,7 @@ mod tests { use crate::{ background_booting::MockBackgroundServices, local_config::LinkupState, paid_tunnel::MockPaidTunnelManager, services::tunnel::MockTunnelManager, - system::MockSystem, + system::MockSystem, CheckErr, }; use url::Url; @@ -224,8 +230,8 @@ mod tests { session_token: "test_token".to_string(), config_path: "/tmp/home/.linkup/config".to_string(), remote: Url::parse("http://localhost:9066").unwrap(), - tunnel: None, is_paid: Some(true), + tunnel: Some(Url::parse("http://localhost:9066").unwrap()), cache_routes: None, } }, @@ -250,7 +256,7 @@ mod tests { // Check if tunnel exists -> Yes mock_paid_manager .expect_get_tunnel_id() - .with(predicate::eq("test_session")) + .with(predicate::eq("tunnel-test_session")) .returning(|_| Ok(Some("test_tunnel_id".to_string()))); // Mock HOME env var @@ -267,12 +273,27 @@ mod tests { ))) .returning(|_| true); + mock_tunnel_manager + .expect_is_tunnel_running() + .once() + .returning(|| Err(CheckErr::TunnelNotRunning)); + // Run tunnel mock_tunnel_manager .expect_run_tunnel() .once() .returning(|_| Ok(Url::parse("http://localhost:9066").unwrap())); + mock_sys + .expect_file_exists() + .with(predicate::eq(linkup_file_path(LINKUP_LOCALDNS_INSTALL))) + .returning(|_| true); + + mock_boot_bg_services + .expect_boot_local_dns() + .once() + .returning(|_, _| Ok(())); + // Don't create tunnel or DNS record mock_paid_manager.expect_create_tunnel().never(); mock_paid_manager.expect_create_dns_record().never(); @@ -306,22 +327,33 @@ mod tests { .returning(|_| Ok(None)); // Don't read config file - mock_sys.expect_file_exists().never(); + mock_sys + .expect_file_exists() + .with(predicate::eq(Path::new("/tmp/home/.cloudflared/.json"))) + .never(); // Create tunnel mock_paid_manager .expect_create_tunnel() .once() - .with(predicate::eq("test_session")) + .with(predicate::eq("tunnel-test_session")) .returning(|_| Ok("tunnel-id".to_string())); // Create DNS record mock_paid_manager .expect_create_dns_record() .once() - .with(predicate::eq("tunnel-id"), predicate::eq("test_session")) + .with( + predicate::eq("tunnel-id"), + predicate::eq("tunnel-test_session"), + ) .returning(|_, _| Ok(())); + mock_tunnel_manager + .expect_is_tunnel_running() + .once() + .returning(|| Err(CheckErr::TunnelNotRunning)); + // Run tunnel mock_tunnel_manager .expect_run_tunnel() @@ -329,6 +361,16 @@ mod tests { .with(predicate::eq(make_state("test_session"))) .returning(|_| Ok(Url::parse("http://localhost:9066").unwrap())); + mock_sys + .expect_file_exists() + .with(predicate::eq(linkup_file_path(LINKUP_LOCALDNS_INSTALL))) + .returning(|_| true); + + mock_boot_bg_services + .expect_boot_local_dns() + .once() + .returning(|_, _| Ok(())); + let result = start_paid_tunnel( &mock_sys, &mock_paid_manager, @@ -355,8 +397,8 @@ mod tests { // Check if tunnel exists -> Yes mock_paid_manager .expect_get_tunnel_id() - .with(predicate::eq("test_session")) - .returning(|_| Ok(Some("test_tunnel_id".to_string()))); + .with(predicate::eq("tunnel-test_session")) + .returning(|_| Ok(Some("tunnel_id".to_string()))); // Mock HOME env var mock_sys @@ -368,7 +410,7 @@ mod tests { mock_sys .expect_file_exists() .with(predicate::eq(Path::new( - "/tmp/home/.cloudflared/test_tunnel_id.json", + "/tmp/home/.cloudflared/tunnel_id.json", ))) .returning(|_| false); @@ -376,16 +418,24 @@ mod tests { mock_paid_manager .expect_create_tunnel() .once() - .with(predicate::eq("test_session")) - .returning(|_| Ok("tunnel-id".to_string())); + .with(predicate::eq("tunnel-test_session")) + .returning(|_| Ok("tunnel_id".to_string())); // Create DNS record mock_paid_manager .expect_create_dns_record() .once() - .with(predicate::eq("tunnel-id"), predicate::eq("test_session")) + .with( + predicate::eq("tunnel_id"), + predicate::eq("tunnel-test_session"), + ) .returning(|_, _| Ok(())); + mock_tunnel_manager + .expect_is_tunnel_running() + .once() + .returning(|| Err(CheckErr::TunnelNotRunning)); + // Run tunnel mock_tunnel_manager .expect_run_tunnel() @@ -393,6 +443,16 @@ mod tests { .with(predicate::eq(make_state("test_session"))) .returning(|_| Ok(Url::parse("http://localhost:9066").unwrap())); + mock_sys + .expect_file_exists() + .with(predicate::eq(linkup_file_path(LINKUP_LOCALDNS_INSTALL))) + .returning(|_| true); + + mock_boot_bg_services + .expect_boot_local_dns() + .once() + .returning(|_, _| Ok(())); + let result = start_paid_tunnel( &mock_sys, &mock_paid_manager, @@ -419,7 +479,7 @@ mod tests { // Check if tunnel exists -> Error mock_paid_manager .expect_get_tunnel_id() - .with(predicate::eq("test_session")) + .with(predicate::eq("tunnel-test_session")) .returning(|_| Err(CliError::StatusErr("test error".to_string()))); let result = start_paid_tunnel(