Add config options for exit routing overhaul [1/?]

This patch starts the exit routing overhal patch series with config
changes. Namely making the exit configuration expressive enough to
descirbe all of the target behaviors.

integration_tests/src/setup_utils/rita.rs

@@ -7,7 +7,6 @@ use super::namespaces::NodeType;
 use althea_kernel_interface::KernelInterfaceError;
 use althea_types::Identity;
 use clarity::Address;
-use ipnetwork::IpNetwork;
 use ipnetwork::Ipv6Network;
 use log::info;
 use nix::sched::{setns, CloneFlags};
@@ -21,6 +20,7 @@ use rita_exit::{
     operator_update::update_loop::start_operator_update_loop,
     rita_loop::{start_rita_exit_endpoints, start_rita_exit_loop},
 };
+use settings::exit::ExitIpv6RoutingSettings;
 use settings::set_flag_config;
 use settings::{client::RitaClientSettings, exit::RitaExitSettingsStruct};
 use std::{
@@ -227,9 +227,11 @@ pub fn spawn_rita_exit(
             id.try_into().unwrap(),
         )));
 
-        resettings.exit_network.subnet = Some(IpNetwork::V6(
-            Ipv6Network::new(instance.subnet, 40).unwrap(),
-        ));
+        resettings.exit_network.ipv6_routing = Some(ExitIpv6RoutingSettings {
+            subnet: Ipv6Network::new(instance.subnet, 40).unwrap(),
+            client_subnet_size: 64,
+            static_assignments: vec![],
+        });
         resettings.exit_network.registered_users_contract_addr = db_addr;
         resettings.network.wg_private_key = Some(instance.wg_priv_key);
         resettings.network.wg_public_key = Some(instance.exit_id.wg_public_key);

rita_exit/src/database/in_memory_database.rs

@@ -278,15 +278,21 @@ pub fn validate_internal_ip(
 pub fn to_exit_client(client: Identity) -> Result<ExitClient, Box<RitaExitError>> {
     let internet_ipv6 = get_client_ipv6(
         client,
-        settings::get_rita_exit().exit_network.subnet,
+        settings::get_rita_exit().exit_network.get_ipv6_subnet_alt(),
         settings::get_rita_exit()
             .get_client_subnet_size()
             .unwrap_or(DEFAULT_CLIENT_SUBNET_SIZE),
     )?;
     let internal_ip = get_client_internal_ip(
         client,
-        settings::get_rita_exit().exit_network.netmask,
-        settings::get_rita_exit().exit_network.own_internal_ip,
+        settings::get_rita_exit()
+            .exit_network
+            .internal_ipv4
+            .prefix(),
+        settings::get_rita_exit()
+            .exit_network
+            .internal_ipv4
+            .internal_ip(),
     )?;
 
     Ok(ExitClient {

rita_exit/src/database/mod.rs

@@ -71,11 +71,15 @@ pub const CLIENT_REGISTER_TIMEOUT: Duration = Duration::from_secs(5);
 pub fn get_exit_info() -> ExitDetails {
     let exit_settings = get_rita_exit();
     ExitDetails {
-        server_internal_ip: exit_settings.exit_network.own_internal_ip.into(),
+        server_internal_ip: exit_settings
+            .exit_network
+            .internal_ipv4
+            .internal_ip()
+            .into(),
         wg_exit_port: exit_settings.exit_network.wg_tunnel_port,
         exit_price: exit_settings.exit_network.exit_price,
         exit_currency: exit_settings.payment.system_chain,
-        netmask: exit_settings.exit_network.netmask,
+        netmask: exit_settings.exit_network.internal_ipv4.prefix(),
         description: exit_settings.description,
         verif_mode: ExitVerifMode::Phone,
     }
@@ -196,6 +200,8 @@ pub async fn client_status(
     trace!("Checking if record exists for {:?}", client.global.mesh_ip);
     let exit = get_rita_exit();
     let exit_network = exit.exit_network.clone();
+    let own_internal_ip = exit_network.internal_ipv4.internal_ip();
+    let internal_netmask = exit_network.internal_ipv4.prefix();
 
     match get_registered_client_using_wgkey(
         client.global.wg_public_key,
@@ -208,14 +214,11 @@ pub async fn client_status(
         Ok(their_record) => {
             trace!("record exists, updating");
 
-            let current_ip: IpAddr = get_client_internal_ip(
-                their_record,
-                exit_network.netmask,
-                exit_network.own_internal_ip,
-            )?;
+            let current_ip: IpAddr =
+                get_client_internal_ip(their_record, internal_netmask, own_internal_ip)?;
             let current_internet_ipv6 = get_client_ipv6(
                 their_record,
-                exit_network.subnet,
+                exit_network.get_ipv6_subnet_alt(),
                 exit.get_client_subnet_size()
                     .unwrap_or(DEFAULT_CLIENT_SUBNET_SIZE),
             )?;
@@ -454,7 +457,8 @@ pub fn setup_clients(
         .collect();
 
     let exit_settings = settings::get_rita_exit();
-    let internal_ip_v4 = exit_settings.exit_network.own_internal_ip;
+    let internal_ip_v4 = exit_settings.exit_network.internal_ipv4.internal_ip();
+    let internal_netmask = exit_settings.exit_network.internal_ipv4.prefix();
 
     // Get all new clients that need rule setup for wg_exit_v2 and wg_exit respectively
     let changed_clients_return = find_changed_clients(
@@ -474,11 +478,7 @@ pub fn setup_clients(
     for c_key in changed_clients_return.new_v1 {
         if let Some(c) = key_to_client_map.get(&c_key) {
             setup_individual_client_routes(
-                match get_client_internal_ip(
-                    *c,
-                    get_rita_exit().exit_network.netmask,
-                    get_rita_exit().exit_network.own_internal_ip,
-                ) {
+                match get_client_internal_ip(*c, internal_netmask, internal_ip_v4) {
                     Ok(a) => a,
                     Err(e) => {
                         error!(
@@ -496,11 +496,7 @@ pub fn setup_clients(
     for c_key in changed_clients_return.new_v2 {
         if let Some(c) = key_to_client_map.get(&c_key) {
             teardown_individual_client_routes(
-                match get_client_internal_ip(
-                    *c,
-                    get_rita_exit().exit_network.netmask,
-                    get_rita_exit().exit_network.own_internal_ip,
-                ) {
+                match get_client_internal_ip(*c, internal_netmask, internal_ip_v4) {
                     Ok(a) => a,
                     Err(e) => {
                         error!(
@@ -679,7 +675,7 @@ pub fn enforce_exit_clients(
                                 // gets the client ipv6 flow for this exit specifically
                                 let client_ipv6 = get_client_ipv6(
                                     debt_entry.identity,
-                                    settings::get_rita_exit().exit_network.subnet,
+                                    settings::get_rita_exit().exit_network.get_ipv6_subnet_alt(),
                                     settings::get_rita_exit()
                                         .get_client_subnet_size()
                                         .unwrap_or(DEFAULT_CLIENT_SUBNET_SIZE),

rita_exit/src/rita_loop/mod.rs

@@ -289,25 +289,25 @@ fn setup_exit_wg_tunnel() {
 
     let exit_settings = settings::get_rita_exit();
 
-    let local_ip = exit_settings.exit_network.own_internal_ip.into();
-    let netmask = exit_settings.exit_network.netmask;
+    let local_ip = exit_settings.exit_network.internal_ipv4.internal_ip();
+    let netmask = exit_settings.exit_network.internal_ipv4.prefix();
     let mesh_ip = exit_settings
         .network
         .mesh_ip
         .expect("Expected a mesh ip for this exit");
     let enforcement_enabled = exit_settings.exit_network.enable_enforcement;
     let external_v6 = exit_settings
         .exit_network
-        .subnet
-        .map(|ipv6_subnet| (ipv6_subnet.ip(), ipv6_subnet.prefix()));
+        .ipv6_routing
+        .map(|a| a.spit_ip_prefix());
 
     // Setup legacy wg_exit
     one_time_exit_setup(None, None, mesh_ip, LEGACY_INTERFACE, enforcement_enabled)
         .expect("Failed to setup wg_exit!");
 
     // Setup wg_exit_v2. Local address added is same as that used by wg_exit
     one_time_exit_setup(
-        Some((local_ip, netmask)),
+        Some((local_ip.into(), netmask)),
         external_v6,
         mesh_ip,
         EXIT_INTERFACE,

settings/src/exit.rs

@@ -6,13 +6,87 @@ use crate::payment::PaymentSettings;
 use crate::{json_merge, set_rita_exit, SettingsError};
 use althea_types::{regions::Regions, ExitIdentity, Identity};
 use clarity::Address;
-use ipnetwork::IpNetwork;
+use ipnetwork::{IpNetwork, Ipv4Network, Ipv6Network};
 use std::collections::HashSet;
-use std::net::Ipv4Addr;
+use std::net::{IpAddr, Ipv4Addr};
 use std::path::{Path, PathBuf};
 
 pub const APP_NAME: &str = "rita_exit";
 
+/// Represents a static ipv4 assignment for a client
+#[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq)]
+pub struct ClientIpv4StaticAssignment {
+    pub client_id: Identity,
+    pub client_external_ip: Ipv4Addr,
+}
+
+/// Represents a static ipv6 assignment for a client
+#[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq)]
+pub struct ClientIpv6StaticAssignment {
+    pub client_id: Identity,
+    pub client_subnet: Ipv6Network,
+}
+
+/// This enum describes the different ways we can route ipv4 traffic out of the exit
+/// and assign addresses to clients
+#[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq)]
+pub enum ExitIpv4RoutingSettings {
+    /// The default and simplest option, all clients are NAT'd out of the exit's own IP
+    NAT,
+    /// A provided subnet of ipv4 addresses is split between clients as evenly as possible
+    /// the exits own ip is used only for management traffic and the exit's own traffic
+    /// IP's from this range can be assigned to specific clients. In that case traffic for other
+    /// customers will be distributed as evenly as possible over the remaining addresses
+    CGNAT {
+        subnet: Ipv4Network,
+        static_assignments: Vec<ClientIpv4StaticAssignment>,
+    },
+    /// A provided subnet of ipv4 addresses is assigned one by one to clients as they connect. With an optional
+    /// list of static assignments for clients that will always be assigned the same IP. Use this option with caution
+    /// if the subnet is too small and too many clients connect there will be no more addresses to assign. Once that happens
+    /// the exit will stop accepting new connections until a client disconnects. Be mindful, in cases where a client can not
+    /// find another exit to connect to they will be unable to access the internet.
+    FLAT {
+        subnet: Ipv4Network,
+        static_assignments: Vec<ClientIpv4StaticAssignment>,
+    },
+}
+
+/// This struct describes the settings for ipv6 routing out of the exit and assignment to clients
+/// the only knob here is the subnet size, which is the size of the subnet assigned to each client
+#[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq)]
+pub struct ExitIpv6RoutingSettings {
+    pub subnet: Ipv6Network,
+    pub client_subnet_size: u8,
+    pub static_assignments: Vec<ClientIpv6StaticAssignment>,
+}
+
+impl ExitIpv6RoutingSettings {
+    pub fn spit_ip_prefix(&self) -> (IpAddr, u8) {
+        (self.subnet.ip().into(), self.subnet.prefix())
+    }
+}
+
+/// The settings for the exit's internal ipv4 network, this is the internal subnet that the exit uses to
+/// NAT traffic. If this subnet is to small for the number of active users the exit will run out of addresses
+/// to assign to clients and will stop accepting new connections. Note "active" in this context means users we
+/// have seen since the last exit restart. By default the internal ip of the exit will be the first ip in this
+/// subnet, and the exit will assign the rest of the addresses to clients
+#[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq)]
+pub struct ExitInternalIpv4Settings {
+    pub internal_subnet: Ipv4Network,
+}
+
+impl ExitInternalIpv4Settings {
+    pub fn internal_ip(&self) -> Ipv4Addr {
+        self.internal_subnet.ip()
+    }
+
+    pub fn prefix(&self) -> u8 {
+        self.internal_subnet.prefix()
+    }
+}
+
 /// This is the network settings specific to rita_exit
 #[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq)]
 pub struct ExitNetworkSettings {
@@ -24,14 +98,15 @@ pub struct ExitNetworkSettings {
     pub wg_v2_tunnel_port: u16,
     /// Price in wei per byte which is charged to traffic both coming in and out over the internet
     pub exit_price: u64,
-    /// This is the exit's own ip/gateway ip in the exit wireguard tunnel
-    pub own_internal_ip: Ipv4Addr,
-    /// The netmask, in bits to mask out, for the exit tunnel
-    pub netmask: u8,
-    /// The subnet we use to assign to client routers for ipv6
-    pub subnet: Option<IpNetwork>,
-    /// The specified client subnet, else use /56
-    pub client_subnet_size: Option<u8>,
+    /// Settings controlling the exits external ipv4 network and how traffic is routed there
+    #[serde(default = "default_ipv4_routing")]
+    pub ipv4_routing: ExitIpv4RoutingSettings,
+    /// Settings controlling the internal subnet used for NAT And CGNAT
+    #[serde(default = "default_internal_ipv4")]
+    pub internal_ipv4: ExitInternalIpv4Settings,
+    /// Settings controlled the exits external ipv6 network and how traffic is routed there
+    /// None if no ipv6 is available
+    pub ipv6_routing: Option<ExitIpv6RoutingSettings>,
     /// api credentials for Maxmind geoip
     pub geoip_api_user: Option<String>,
     pub geoip_api_key: Option<String>,
@@ -48,6 +123,16 @@ pub struct ExitNetworkSettings {
     pub allowed_countries: HashSet<Regions>,
 }
 
+impl ExitNetworkSettings {
+    pub fn get_ipv6_subnet(&self) -> Option<Ipv6Network> {
+        self.ipv6_routing.as_ref().map(|x| x.subnet)
+    }
+
+    pub fn get_ipv6_subnet_alt(&self) -> Option<IpNetwork> {
+        self.ipv6_routing.as_ref().map(|x| x.subnet.into())
+    }
+}
+
 fn default_allowed_countries() -> HashSet<Regions> {
     HashSet::new()
 }
@@ -56,6 +141,16 @@ fn enable_enforcement_default() -> bool {
     true
 }
 
+fn default_ipv4_routing() -> ExitIpv4RoutingSettings {
+    ExitIpv4RoutingSettings::NAT
+}
+
+fn default_internal_ipv4() -> ExitInternalIpv4Settings {
+    ExitInternalIpv4Settings {
+        internal_subnet: Ipv4Network::new(Ipv4Addr::new(172, 16, 255, 254), 12).unwrap(),
+    }
+}
+
 impl ExitNetworkSettings {
     /// Generates a configuration that can be used in integration tests, does not use the
     /// default trait to prevent some future code from picking up on the 'default' implementation
@@ -66,17 +161,18 @@ impl ExitNetworkSettings {
             wg_tunnel_port: 59999,
             wg_v2_tunnel_port: 59998,
             exit_price: 10,
-            own_internal_ip: "172.16.255.254".parse().unwrap(),
-            netmask: 12,
-            subnet: Some(IpNetwork::V6("ff01::0/128".parse().unwrap())),
-            client_subnet_size: None,
             geoip_api_user: None,
             geoip_api_key: None,
             enable_enforcement: true,
             registered_users_contract_addr: "0x9BAbFde52Fe18A5CD00a542b87b4D124a4879582"
                 .parse()
                 .unwrap(),
             allowed_countries: HashSet::new(),
+            ipv4_routing: ExitIpv4RoutingSettings::NAT,
+            internal_ipv4: ExitInternalIpv4Settings {
+                internal_subnet: Ipv4Network::new(Ipv4Addr::new(172, 16, 255, 254), 12).unwrap(),
+            },
+            ipv6_routing: None,
         }
     }
 }
@@ -165,7 +261,10 @@ impl RitaExitSettingsStruct {
     }
 
     pub fn get_client_subnet_size(&self) -> Option<u8> {
-        self.exit_network.client_subnet_size
+        self.exit_network
+            .ipv6_routing
+            .as_ref()
+            .map(|x| x.client_subnet_size)
     }
 
     pub fn get_all(&self) -> Result<serde_json::Value, SettingsError> {