Add 'fault_percentage' and 'client_stats' configuration settings

'client_stats' controls granularity of client request/response
tracking.

`fault_percentage` is in anticipation of adding deliberate
response corruption Roughtime ecosystem feature.

7 files changed, 142 insertions, 47 deletions

diff --git a/src/config/environment.rs b/src/config/environment.rs
index d75eee9..434ba04 100644
--- a/src/config/environment.rs
+++ b/src/config/environment.rs
@@ -33,6 +33,8 @@ use crate::Error;
 ///   status_interval   | `ROUGHENOUGH_STATUS_INTERVAL`
 ///   kms_protection    | `ROUGHENOUGH_KMS_PROTECTION`
 ///   health_check_port | `ROUGHENOUGH_HEALTH_CHECK_PORT`
+///   client_stats      | `ROUGHENOUGH_CLIENT_STATS`
+///   fault_percentage  | `ROUGHENOUGH_FAULT_PERCENTAGE`
 ///
 pub struct EnvironmentConfig {
     port: u16,
@@ -42,6 +44,8 @@ pub struct EnvironmentConfig {
     status_interval: Duration,
     kms_protection: KmsProtection,
     health_check_port: Option<u16>,
+    client_stats: bool,
+    fault_percentage: u8,
 }
 
 const ROUGHENOUGH_PORT: &str = "ROUGHENOUGH_PORT";
@@ -51,6 +55,8 @@ const ROUGHENOUGH_BATCH_SIZE: &str = "ROUGHENOUGH_BATCH_SIZE";
 const ROUGHENOUGH_STATUS_INTERVAL: &str = "ROUGHENOUGH_STATUS_INTERVAL";
 const ROUGHENOUGH_KMS_PROTECTION: &str = "ROUGHENOUGH_KMS_PROTECTION";
 const ROUGHENOUGH_HEALTH_CHECK_PORT: &str = "ROUGHENOUGH_HEALTH_CHECK_PORT";
+const ROUGHENOUGH_CLIENT_STATS: &str = "ROUGHENOUGH_CLIENT_STATS";
+const ROUGHENOUGH_FAULT_PERCENTAGE: &str = "ROUGHENOUGH_FAULT_PERCENTAGE";
 
 impl EnvironmentConfig {
     pub fn new() -> Result<Self, Error> {
@@ -62,6 +68,8 @@ impl EnvironmentConfig {
             status_interval: DEFAULT_STATUS_INTERVAL,
             kms_protection: KmsProtection::Plaintext,
             health_check_port: None,
+            client_stats: false,
+            fault_percentage: 0,
         };
 
         if let Ok(port) = env::var(ROUGHENOUGH_PORT) {
@@ -107,6 +115,18 @@ impl EnvironmentConfig {
             cfg.health_check_port = Some(val);
         };
 
+        if let Ok(mut client_stats) = env::var(ROUGHENOUGH_CLIENT_STATS) {
+            client_stats.make_ascii_lowercase();
+            
+            cfg.client_stats = client_stats == "yes" || client_stats == "on";
+        }
+
+        if let Ok(fault_percentage) = env::var(ROUGHENOUGH_FAULT_PERCENTAGE) {
+            cfg.fault_percentage = fault_percentage
+                .parse()
+                .unwrap_or_else(|_| panic!("invalid fault_percentage: {}", fault_percentage));
+        };
+
         Ok(cfg)
     }
 }
@@ -139,4 +159,12 @@ impl ServerConfig for EnvironmentConfig {
     fn health_check_port(&self) -> Option<u16> {
         self.health_check_port
     }
+
+    fn client_stats_enabled(&self) -> bool {
+        self.client_stats
+    }
+
+    fn fault_percentage(&self) -> u8 {
+        self.fault_percentage
+    }
 }
diff --git a/src/config/file.rs b/src/config/file.rs
index 34b7682..60232f3 100644
--- a/src/config/file.rs
+++ b/src/config/file.rs
@@ -42,6 +42,8 @@ pub struct FileConfig {
     status_interval: Duration,
     kms_protection: KmsProtection,
     health_check_port: Option<u16>,
+    client_stats: bool,
+    fault_percentage: u8,
 }
 
 impl FileConfig {
@@ -69,6 +71,8 @@ impl FileConfig {
             status_interval: DEFAULT_STATUS_INTERVAL,
             kms_protection: KmsProtection::Plaintext,
             health_check_port: None,
+            client_stats: false,
+            fault_percentage: 0,
         };
 
         for (key, value) in cfg[0].as_hash().unwrap() {
@@ -96,6 +100,14 @@ impl FileConfig {
                     let val = value.as_i64().unwrap() as u16;
                     config.health_check_port = Some(val);
                 }
+                "client_stats" => {
+                    let val = value.as_str().unwrap().to_ascii_lowercase();
+                    config.client_stats = val == "yes" || val == "on";
+                }
+                "fault_percentage" => {
+                    let val = value.as_i64().unwrap() as u8;
+                    config.fault_percentage = val;
+                }
                 unknown => {
                     return Err(Error::InvalidConfiguration(format!(
                         "unknown config key: {}",
@@ -137,4 +149,12 @@ impl ServerConfig for FileConfig {
     fn health_check_port(&self) -> Option<u16> {
         self.health_check_port
     }
+
+    fn client_stats_enabled(&self) -> bool {
+        self.client_stats
+    }
+
+    fn fault_percentage(&self) -> u8 {
+        self.fault_percentage
+    }
 }
diff --git a/src/config/memory.rs b/src/config/memory.rs
index bb25171..2128bb6 100644
--- a/src/config/memory.rs
+++ b/src/config/memory.rs
@@ -30,10 +30,12 @@ pub struct MemoryConfig {
     pub status_interval: Duration,
     pub kms_protection: KmsProtection,
     pub health_check_port: Option<u16>,
+    pub client_stats: bool,
+    pub fault_percentage: u8,
 }
 
 impl MemoryConfig {
-    pub fn new(port: u16) -> MemoryConfig {
+    pub fn new(port: u16) -> Self {
         MemoryConfig {
             port,
             interface: "127.0.0.1".to_string(),
@@ -43,6 +45,8 @@ impl MemoryConfig {
             status_interval: DEFAULT_STATUS_INTERVAL,
             kms_protection: KmsProtection::Plaintext,
             health_check_port: None,
+            client_stats: false,
+            fault_percentage: 0
         }
     }
 }
@@ -75,4 +79,12 @@ impl ServerConfig for MemoryConfig {
     fn health_check_port(&self) -> Option<u16> {
         self.health_check_port
     }
+
+    fn client_stats_enabled(&self) -> bool {
+        self.client_stats
+    }
+
+    fn fault_percentage(&self) -> u8 {
+        self.fault_percentage
+    }
 }
diff --git a/src/config/mod.rs b/src/config/mod.rs
index fb7854f..6dc33d6 100644
--- a/src/config/mod.rs
+++ b/src/config/mod.rs
@@ -58,13 +58,13 @@ pub const DEFAULT_STATUS_INTERVAL: Duration = Duration::from_secs(600);
 /// `status_interval` | `ROUGHENOUGH_STATUS_INTERVAL` | Optional | Number of _seconds_ between each logged status update. Default is `600` seconds (10 minutes).
 /// `health_check_port` | `ROUGHENOUGH_HEALTH_CHECK_PORT` | Optional | If present, enable an HTTP health check responder on the provided port. **Use with caution**.
 /// `kms_protection` | `ROUGHENOUGH_KMS_PROTECTION` | Optional | If compiled with KMS support, the ID of the KMS key used to protect the long-term identity.
+/// `client_stats` | `ROUGHENOUGH_CLIENT_STATS` | Optional | A value of `on` or `yes` will enable tracking of per-client request statistics that will be output each time server status is logged. Default is `off` (disabled).
+/// `fault_percentage` | `ROUGHENOUGH_FAULT_PERCENTAGE` | Optional | Likelihood (as a percentage) that the server will intentionally return an invalid client response. An integer range from `0` (disabled, all responses valid) to `50` (50% of responses will be invalid). Default is `0` (disabled).
 ///
 /// Implementations of this trait obtain a valid configuration from different back-end
 /// sources. See:
 ///   * [FileConfig](struct.FileConfig.html) - configure via a YAML file
-///   * [EnvironmentConfig](struct.EnvironmentConfig.html) - configure via environment vars
-///
-/// The health check and KMS features require
+///   * [EnvironmentConfig](struct.EnvironmentConfig.html) - configure via environment variables
 ///
 pub trait ServerConfig {
     /// [Required] IP address or interface name to listen for client requests
@@ -96,6 +96,18 @@ pub trait ServerConfig {
     /// https://cloud.google.com/load-balancing/docs/health-checks#legacy-health-checks
     fn health_check_port(&self) -> Option<u16>;
 
+    /// [Optional] A value of `on` or `yes` will enable tracking of per-client request statistics
+    /// that will be output each time server status is logged. Default is `off` (disabled).
+    fn client_stats_enabled(&self) -> bool;
+
+    /// [Optional] Likelihood (as a percentage) that the server will intentionally return an
+    /// invalid client response. An integer range from `0` (disabled, all responses valid) to `50`
+    /// (~50% of responses will be invalid). Default is `0` (disabled).
+    ///
+    /// See the [Roughtime spec](https://roughtime.googlesource.com/roughtime/+/HEAD/ECOSYSTEM.md#maintaining-a-healthy-software-ecosystem)
+    /// for background and rationale.
+    fn fault_percentage(&self) -> u8;
+
     /// Convenience function to create a `SocketAddr` from the provided `interface` and `port`
     fn udp_socket_addr(&self) -> Result<SocketAddr, Error> {
         let addr = format!("{}:{}", self.interface(), self.port());
@@ -159,6 +171,10 @@ pub fn is_valid_config(cfg: &Box<dyn ServerConfig>) -> bool {
         );
         is_valid = false;
     }
+    if cfg.fault_percentage() > 50 {
+        error!("fault_percentage {} is invalid; valid range 0-50", cfg.fault_percentage());
+        is_valid = false;
+    }
 
     if is_valid {
         match cfg.udp_socket_addr() {
diff --git a/src/stats/aggregated.rs b/src/stats/aggregated.rs
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/stats/aggregated.rs
diff --git a/src/stats.rs b/src/stats/mod.rs
index d296e40..013b7ea 100644
--- a/src/stats.rs
+++ b/src/stats/mod.rs
@@ -24,7 +24,7 @@ use std::net::IpAddr;
 ///
 /// Implementations of this trait record client activity
 ///
-pub trait ClientStats {
+pub trait ServerStats {
     fn add_valid_request(&mut self, addr: &IpAddr);
 
     fn add_invalid_request(&mut self, addr: &IpAddr);
@@ -45,9 +45,9 @@ pub trait ClientStats {
 
     fn total_unique_clients(&self) -> u64;
 
-    fn get_stats(&self, addr: &IpAddr) -> Option<&StatEntry>;
+    fn get_client_stats(&self, addr: &IpAddr) -> Option<&ClientStatEntry>;
 
-    fn iter(&self) -> Iter<IpAddr, StatEntry>;
+    fn iter(&self) -> Iter<IpAddr, ClientStatEntry>;
 
     fn clear(&mut self);
 }
@@ -56,7 +56,7 @@ pub trait ClientStats {
 /// Specific metrics tracked per each client
 ///
 #[derive(Debug, Clone, Copy)]
-pub struct StatEntry {
+pub struct ClientStatEntry {
     pub valid_requests: u64,
     pub invalid_requests: u64,
     pub health_checks: u64,
@@ -64,9 +64,9 @@ pub struct StatEntry {
     pub bytes_sent: usize,
 }
 
-impl StatEntry {
+impl ClientStatEntry {
     fn new() -> Self {
-        StatEntry {
+        ClientStatEntry {
             valid_requests: 0,
             invalid_requests: 0,
             health_checks: 0,
@@ -77,35 +77,35 @@ impl StatEntry {
 }
 
 ///
-/// Implementation of `ClientStats` backed by a hashmap.
+/// Implementation of `ServerStats` that provides granular per-client request/response counts.
 ///
 /// Maintains a maximum of `MAX_CLIENTS` unique entries to bound memory use. Excess
 /// entries beyond `MAX_CLIENTS` are ignored and `num_overflows` is incremented.
 ///
-pub struct SimpleStats {
-    clients: HashMap<IpAddr, StatEntry>,
+pub struct PerClientStats {
+    clients: HashMap<IpAddr, ClientStatEntry>,
     num_overflows: u64,
     max_clients: usize,
 }
 
-impl SimpleStats {
+impl PerClientStats {
 
     /// Maximum number of stats entries to maintain to prevent
     /// unbounded memory growth.
     pub const MAX_CLIENTS: usize = 1_000_000;
 
     pub fn new() -> Self {
-        SimpleStats {
+        PerClientStats {
             clients: HashMap::with_capacity(128),
             num_overflows: 0,
-            max_clients: SimpleStats::MAX_CLIENTS,
+            max_clients: PerClientStats::MAX_CLIENTS,
         }
     }
 
     // visible for testing
     #[cfg(test)]
-    fn with_limits(limit: usize) -> Self {
-        SimpleStats {
+    fn with_limit(limit: usize) -> Self {
+        PerClientStats {
             clients: HashMap::with_capacity(128),
             num_overflows: 0,
             max_clients: limit,
@@ -129,14 +129,14 @@ impl SimpleStats {
     }
 }
 
-impl ClientStats for SimpleStats {
+impl ServerStats for PerClientStats {
     fn add_valid_request(&mut self, addr: &IpAddr) {
         if self.too_many_entries() {
             return;
         }
         self.clients
             .entry(*addr)
-            .or_insert_with(StatEntry::new)
+            .or_insert_with(ClientStatEntry::new)
             .valid_requests += 1;
     }
 
@@ -146,7 +146,7 @@ impl ClientStats for SimpleStats {
         }
         self.clients
             .entry(*addr)
-            .or_insert_with(StatEntry::new)
+            .or_insert_with(ClientStatEntry::new)
             .invalid_requests += 1;
     }
 
@@ -156,7 +156,7 @@ impl ClientStats for SimpleStats {
         }
         self.clients
             .entry(*addr)
-            .or_insert_with(StatEntry::new)
+            .or_insert_with(ClientStatEntry::new)
             .health_checks += 1;
     }
 
@@ -166,7 +166,7 @@ impl ClientStats for SimpleStats {
         }
         let entry = self.clients
             .entry(*addr)
-            .or_insert_with(StatEntry::new);
+            .or_insert_with(ClientStatEntry::new);
 
         entry.responses_sent += 1;
         entry.bytes_sent += bytes_sent;
@@ -211,11 +211,11 @@ impl ClientStats for SimpleStats {
         self.clients.len() as u64
     }
 
-    fn get_stats(&self, addr: &IpAddr) -> Option<&StatEntry> {
+    fn get_client_stats(&self, addr: &IpAddr) -> Option<&ClientStatEntry> {
         self.clients.get(addr)
     }
 
-    fn iter(&self) -> Iter<IpAddr, StatEntry> {
+    fn iter(&self) -> Iter<IpAddr, ClientStatEntry> {
         self.clients.iter()
     }
 
@@ -226,61 +226,80 @@ impl ClientStats for SimpleStats {
 }
 
 ///
-/// A no-op implementation that does not track anything and has no runtime cost
+/// Implementation of `ServerStats` that provides high-level aggregated server statistics.
 ///
 #[allow(dead_code)]
-pub struct NoOpStats {
-    empty_map: HashMap<IpAddr, StatEntry>
+pub struct AggregatedStats {
+    valid_requests: u64,
+    invalid_requests: u64,
+    health_checks: u64,
+    responses_sent: u64,
+    bytes_sent: usize,
+    empty_map: HashMap<IpAddr, ClientStatEntry>,
 }
 
-impl NoOpStats {
+impl AggregatedStats {
 
     #[allow(dead_code)]
     pub fn new() -> Self {
-        NoOpStats {
+        AggregatedStats {
+            valid_requests: 0,
+            invalid_requests: 0,
+            health_checks: 0,
+            responses_sent: 0,
+            bytes_sent: 0,
             empty_map: HashMap::new()
         }
     }
 }
 
-impl ClientStats for NoOpStats {
-    fn add_valid_request(&mut self, _addr: &IpAddr) {}
+impl ServerStats for AggregatedStats {
+    fn add_valid_request(&mut self, _: &IpAddr) {
+        self.valid_requests += 1
+    }
 
-    fn add_invalid_request(&mut self, _addr: &IpAddr) {}
+    fn add_invalid_request(&mut self, _: &IpAddr) {
+        self.invalid_requests += 1
+    }
 
-    fn add_health_check(&mut self, _addr: &IpAddr) {}
+    fn add_health_check(&mut self, _: &IpAddr) {
+        self.health_checks += 1
+    }
 
-    fn add_response(&mut self, _addr: &IpAddr, _bytes_sent: usize) {}
+    fn add_response(&mut self, _: &IpAddr, bytes_sent: usize) {
+        self.bytes_sent += bytes_sent;
+        self.responses_sent += 1;
+    }
 
     fn total_valid_requests(&self) -> u64 {
-        0
+        self.valid_requests
     }
 
     fn total_invalid_requests(&self) -> u64 {
-        0
+        self.invalid_requests
     }
 
     fn total_health_checks(&self) -> u64 {
-        0
+        self.health_checks
     }
 
     fn total_responses_sent(&self) -> u64 {
-        0
+        self.responses_sent
     }
 
     fn total_bytes_sent(&self) -> usize {
-        0
+        self.bytes_sent
     }
 
     fn total_unique_clients(&self) -> u64 {
         0
     }
 
-    fn get_stats(&self, _addr: &IpAddr) -> Option<&StatEntry> {
+    fn get_client_stats(&self, _addr: &IpAddr) -> Option<&ClientStatEntry> {
         None
     }
 
-    fn iter(&self) -> Iter<IpAddr, StatEntry> {
+    fn iter(&self) -> Iter<IpAddr, ClientStatEntry> {
         self.empty_map.iter()
     }
 
@@ -289,12 +308,12 @@ impl ClientStats for NoOpStats {
 
 #[cfg(test)]
 mod test {
-    use crate::stats::{ClientStats, SimpleStats};
+    use crate::stats::{ServerStats, PerClientStats};
     use std::net::{IpAddr, Ipv4Addr};
 
     #[test]
     fn simple_stats_starts_empty() {
-        let stats = SimpleStats::new();
+        let stats = PerClientStats::new();
 
         assert_eq!(stats.total_valid_requests(), 0);
         assert_eq!(stats.total_invalid_requests(), 0);
@@ -307,7 +326,7 @@ mod test {
 
     #[test]
     fn client_requests_are_tracked() {
-        let mut stats = SimpleStats::new();
+        let mut stats = PerClientStats::new();
 
         let ip1 = "127.0.0.1".parse().unwrap();
         let ip2 = "127.0.0.2".parse().unwrap();
@@ -329,7 +348,7 @@ mod test {
 
     #[test]
     fn per_client_stats() {
-        let mut stats = SimpleStats::new();
+        let mut stats = PerClientStats::new();
         let ip = "127.0.0.3".parse().unwrap();
 
         stats.add_valid_request(&ip);
@@ -345,7 +364,7 @@ mod test {
 
     #[test]
     fn overflow_max_entries() {
-        let mut stats = SimpleStats::with_limits(100);
+        let mut stats = PerClientStats::with_limit(100);
 
         for i in 0..201 {
             let ipv4 = Ipv4Addr::from(i as u32);
diff --git a/src/stats/per_client.rs b/src/stats/per_client.rs
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/stats/per_client.rs