Merge pull request #17 from int08h/1.1.4

1.1.4

18 files changed, 829 insertions, 425 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 228f110..82c9a4d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,8 @@
+## Version 1.1.4
+
+* Implement Roughtime ecosystem response mangling (177372f, f851deb)
+* Doc fix from @Muncan90 (20ba144)
+
 ## Version 1.1.3
 
 * Add decrypt option to `roughenough-kms` 
diff --git a/Cargo.toml b/Cargo.toml
index f9f07fd..e358a57 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "roughenough"
-version = "1.1.3"
+version = "1.1.4"
 repository = "https://github.com/int08h/roughenough"
 authors = ["Stuart Stock <stuart@int08h.com>", "Aaron Hill <aa1ronham@gmail.com>"]
 license = "Apache-2.0"
@@ -34,6 +34,7 @@ hex = "0.3"
 base64 = "0.9"
 hashbrown = "0.1"
 humansize = "1.0"
+rand = "0.6"
 
 # Used by 'awskms'
 rusoto_core = { version = "0.36", optional = true }
diff --git a/README.md b/README.md
index 70bf5b7..4935b3b 100644
--- a/README.md
+++ b/README.md
@@ -84,6 +84,7 @@ YAML Key | Environment Variable | Necessity | Description
 `status_interval` | `ROUGHENOUGH_STATUS_INTERVAL` | Optional | Number of _seconds_ between each logged status update. Default is `600` seconds (10 minutes).
 `health_check_port` | `ROUGHENOUGH_HEALTH_CHECK_PORT` | Optional | If present, enable an HTTP health check responder on the provided port. **Use with caution**, see [Optional Features](#optional-features).
 `kms_protection` | `ROUGHENOUGH_KMS_PROTECTION` | Optional | If compiled with KMS support, the ID of the KMS key used to protect the long-term identity. See [Optional Features](#optional-features).
+`fault_percentage` | `ROUGHENOUGH_FAULT_PERCENTAGE` | Optional | Likelihood (as a percentage) that the server will intentionally return an invalid client response. An integer range from `0` (disabled, all responses valid) to `50` (50% of responses will be invalid). Default is `0` (disabled).
 
 #### YAML Configuration 
 
diff --git a/src/bin/roughenough-server.rs b/src/bin/roughenough-server.rs
index 88c8e83..b0b5819 100644
--- a/src/bin/roughenough-server.rs
+++ b/src/bin/roughenough-server.rs
@@ -45,29 +45,25 @@ macro_rules! check_ctrlc {
 
 fn polling_loop(config: Box<ServerConfig>) {
     let mut server = Server::new(config);
-
-    info!("Long-term public key    : {}", server.get_public_key());
-    info!("Online public key       : {}", server.get_online_key());
-    info!(
-        "Max response batch size : {}",
-        server.get_config().batch_size()
-    );
-    info!(
-        "Status updates every    : {} seconds",
-        server.get_config().status_interval().as_secs()
-    );
-    info!(
-        "Server listening on     : {}:{}",
-        server.get_config().interface(),
-        server.get_config().port()
+    let cfg = server.get_config(); // borrow config that was moved above
+
+    info!("Long-term public key       : {}", server.get_public_key());
+    info!("Online public key          : {}", server.get_online_key());
+    info!("Max response batch size    : {}", cfg.batch_size());
+    info!("Status updates every       : {} seconds", cfg.status_interval().as_secs());
+    info!("Server listening on        : {}:{}", cfg.interface(), cfg.port());
+    if let Some(hc_port) = cfg.health_check_port() {
+        info!("TCP health check           : {}:{}", cfg.interface(), hc_port);
+    } else {
+        info!("TCP health check           : disabled");
+    }
+    info!("Client req/resp tracking   : {}",
+          if cfg.client_stats_enabled() { "per-client" } else { "aggregated" }
     );
-
-    if let Some(hc_port) = server.get_config().health_check_port() {
-        info!(
-            "TCP health check        : {}:{}",
-            server.get_config().interface(),
-            hc_port
-        );
+    if cfg.fault_percentage() > 0 {
+        info!("Deliberate response errors : ~{}%", cfg.fault_percentage());
+    } else {
+        info!("Deliberate response errors : disabled");
     }
 
     let kr = server.get_keep_running();
diff --git a/src/config/environment.rs b/src/config/environment.rs
index d75eee9..434ba04 100644
--- a/src/config/environment.rs
+++ b/src/config/environment.rs
@@ -33,6 +33,8 @@ use crate::Error;
 ///   status_interval   | `ROUGHENOUGH_STATUS_INTERVAL`
 ///   kms_protection    | `ROUGHENOUGH_KMS_PROTECTION`
 ///   health_check_port | `ROUGHENOUGH_HEALTH_CHECK_PORT`
+///   client_stats      | `ROUGHENOUGH_CLIENT_STATS`
+///   fault_percentage  | `ROUGHENOUGH_FAULT_PERCENTAGE`
 ///
 pub struct EnvironmentConfig {
     port: u16,
@@ -42,6 +44,8 @@ pub struct EnvironmentConfig {
     status_interval: Duration,
     kms_protection: KmsProtection,
     health_check_port: Option<u16>,
+    client_stats: bool,
+    fault_percentage: u8,
 }
 
 const ROUGHENOUGH_PORT: &str = "ROUGHENOUGH_PORT";
@@ -51,6 +55,8 @@ const ROUGHENOUGH_BATCH_SIZE: &str = "ROUGHENOUGH_BATCH_SIZE";
 const ROUGHENOUGH_STATUS_INTERVAL: &str = "ROUGHENOUGH_STATUS_INTERVAL";
 const ROUGHENOUGH_KMS_PROTECTION: &str = "ROUGHENOUGH_KMS_PROTECTION";
 const ROUGHENOUGH_HEALTH_CHECK_PORT: &str = "ROUGHENOUGH_HEALTH_CHECK_PORT";
+const ROUGHENOUGH_CLIENT_STATS: &str = "ROUGHENOUGH_CLIENT_STATS";
+const ROUGHENOUGH_FAULT_PERCENTAGE: &str = "ROUGHENOUGH_FAULT_PERCENTAGE";
 
 impl EnvironmentConfig {
     pub fn new() -> Result<Self, Error> {
@@ -62,6 +68,8 @@ impl EnvironmentConfig {
             status_interval: DEFAULT_STATUS_INTERVAL,
             kms_protection: KmsProtection::Plaintext,
             health_check_port: None,
+            client_stats: false,
+            fault_percentage: 0,
         };
 
         if let Ok(port) = env::var(ROUGHENOUGH_PORT) {
@@ -107,6 +115,18 @@ impl EnvironmentConfig {
             cfg.health_check_port = Some(val);
         };
 
+        if let Ok(mut client_stats) = env::var(ROUGHENOUGH_CLIENT_STATS) {
+            client_stats.make_ascii_lowercase();
+            
+            cfg.client_stats = client_stats == "yes" || client_stats == "on";
+        }
+
+        if let Ok(fault_percentage) = env::var(ROUGHENOUGH_FAULT_PERCENTAGE) {
+            cfg.fault_percentage = fault_percentage
+                .parse()
+                .unwrap_or_else(|_| panic!("invalid fault_percentage: {}", fault_percentage));
+        };
+
         Ok(cfg)
     }
 }
@@ -139,4 +159,12 @@ impl ServerConfig for EnvironmentConfig {
     fn health_check_port(&self) -> Option<u16> {
         self.health_check_port
     }
+
+    fn client_stats_enabled(&self) -> bool {
+        self.client_stats
+    }
+
+    fn fault_percentage(&self) -> u8 {
+        self.fault_percentage
+    }
 }
diff --git a/src/config/file.rs b/src/config/file.rs
index 34b7682..b71b610 100644
--- a/src/config/file.rs
+++ b/src/config/file.rs
@@ -26,7 +26,7 @@ use crate::Error;
 /// Read a Roughenough server configuration ([ServerConfig](trait.ServerConfig.html))
 /// from a YAML file.
 ///
-/// Example config:
+/// Example minimal config:
 ///
 /// ```yaml
 /// interface: 127.0.0.1
@@ -42,6 +42,8 @@ pub struct FileConfig {
     status_interval: Duration,
     kms_protection: KmsProtection,
     health_check_port: Option<u16>,
+    client_stats: bool,
+    fault_percentage: u8,
 }
 
 impl FileConfig {
@@ -69,6 +71,8 @@ impl FileConfig {
             status_interval: DEFAULT_STATUS_INTERVAL,
             kms_protection: KmsProtection::Plaintext,
             health_check_port: None,
+            client_stats: false,
+            fault_percentage: 0,
         };
 
         for (key, value) in cfg[0].as_hash().unwrap() {
@@ -79,7 +83,7 @@ impl FileConfig {
                 "seed" => {
                     let val = value.as_str().unwrap().to_string();
                     config.seed = hex::decode(val)
-                        .expect("seed value invalid; 'seed' should be 32 byte hex value");
+                        .expect("seed value invalid; 'seed' must be a valid hex value");
                 }
                 "status_interval" => {
                     let val = value.as_i64().expect("status_interval value invalid");
@@ -96,6 +100,14 @@ impl FileConfig {
                     let val = value.as_i64().unwrap() as u16;
                     config.health_check_port = Some(val);
                 }
+                "client_stats" => {
+                    let val = value.as_str().unwrap().to_ascii_lowercase();
+                    config.client_stats = val == "yes" || val == "on";
+                }
+                "fault_percentage" => {
+                    let val = value.as_i64().unwrap() as u8;
+                    config.fault_percentage = val;
+                }
                 unknown => {
                     return Err(Error::InvalidConfiguration(format!(
                         "unknown config key: {}",
@@ -137,4 +149,12 @@ impl ServerConfig for FileConfig {
     fn health_check_port(&self) -> Option<u16> {
         self.health_check_port
     }
+
+    fn client_stats_enabled(&self) -> bool {
+        self.client_stats
+    }
+
+    fn fault_percentage(&self) -> u8 {
+        self.fault_percentage
+    }
 }
diff --git a/src/config/memory.rs b/src/config/memory.rs
index bb25171..2128bb6 100644
--- a/src/config/memory.rs
+++ b/src/config/memory.rs
@@ -30,10 +30,12 @@ pub struct MemoryConfig {
     pub status_interval: Duration,
     pub kms_protection: KmsProtection,
     pub health_check_port: Option<u16>,
+    pub client_stats: bool,
+    pub fault_percentage: u8,
 }
 
 impl MemoryConfig {
-    pub fn new(port: u16) -> MemoryConfig {
+    pub fn new(port: u16) -> Self {
         MemoryConfig {
             port,
             interface: "127.0.0.1".to_string(),
@@ -43,6 +45,8 @@ impl MemoryConfig {
             status_interval: DEFAULT_STATUS_INTERVAL,
             kms_protection: KmsProtection::Plaintext,
             health_check_port: None,
+            client_stats: false,
+            fault_percentage: 0
         }
     }
 }
@@ -75,4 +79,12 @@ impl ServerConfig for MemoryConfig {
     fn health_check_port(&self) -> Option<u16> {
         self.health_check_port
     }
+
+    fn client_stats_enabled(&self) -> bool {
+        self.client_stats
+    }
+
+    fn fault_percentage(&self) -> u8 {
+        self.fault_percentage
+    }
 }
diff --git a/src/config/mod.rs b/src/config/mod.rs
index fb7854f..1d1d149 100644
--- a/src/config/mod.rs
+++ b/src/config/mod.rs
@@ -22,20 +22,20 @@
 //! such as files or environment variables.
 //!
 
+mod environment;
+mod file;
+mod memory;
+
 use std::net::SocketAddr;
 use std::time::Duration;
 
-mod file;
-pub use self::file::FileConfig;
-
-mod environment;
 pub use self::environment::EnvironmentConfig;
-
-mod memory;
+pub use self::file::FileConfig;
 pub use self::memory::MemoryConfig;
 
 use crate::key::KmsProtection;
 use crate::Error;
+use crate::SEED_LENGTH;
 
 /// Maximum number of requests to process in one batch and include the the Merkle tree.
 pub const DEFAULT_BATCH_SIZE: u8 = 64;
@@ -47,7 +47,7 @@ pub const DEFAULT_STATUS_INTERVAL: Duration = Duration::from_secs(600);
 /// Specifies parameters needed to configure a Roughenough server.
 ///
 /// Parameters labeled "**Required**" must always be provided and have no default value
-/// while those labeled "**Optional**" provide default values that can be overridden.
+/// while those labeled "**Optional**" provide sane default values that can be overridden.
 ///
 /// YAML Key | Environment Variable | Necessity | Description
 /// --- | --- | --- | ---
@@ -58,13 +58,13 @@ pub const DEFAULT_STATUS_INTERVAL: Duration = Duration::from_secs(600);
 /// `status_interval` | `ROUGHENOUGH_STATUS_INTERVAL` | Optional | Number of _seconds_ between each logged status update. Default is `600` seconds (10 minutes).
 /// `health_check_port` | `ROUGHENOUGH_HEALTH_CHECK_PORT` | Optional | If present, enable an HTTP health check responder on the provided port. **Use with caution**.
 /// `kms_protection` | `ROUGHENOUGH_KMS_PROTECTION` | Optional | If compiled with KMS support, the ID of the KMS key used to protect the long-term identity.
+/// `client_stats` | `ROUGHENOUGH_CLIENT_STATS` | Optional | A value of `on` or `yes` will enable tracking of per-client request statistics that will be output each time server status is logged. Default is `off` (disabled).
+/// `fault_percentage` | `ROUGHENOUGH_FAULT_PERCENTAGE` | Optional | Likelihood (as a percentage) that the server will intentionally return an invalid client response. An integer range from `0` (disabled, all responses valid) to `50` (50% of responses will be invalid). Default is `0` (disabled).
 ///
 /// Implementations of this trait obtain a valid configuration from different back-end
 /// sources. See:
 ///   * [FileConfig](struct.FileConfig.html) - configure via a YAML file
-///   * [EnvironmentConfig](struct.EnvironmentConfig.html) - configure via environment vars
-///
-/// The health check and KMS features require
+///   * [EnvironmentConfig](struct.EnvironmentConfig.html) - configure via environment variables
 ///
 pub trait ServerConfig {
     /// [Required] IP address or interface name to listen for client requests
@@ -96,6 +96,18 @@ pub trait ServerConfig {
     /// https://cloud.google.com/load-balancing/docs/health-checks#legacy-health-checks
     fn health_check_port(&self) -> Option<u16>;
 
+    /// [Optional] A value of `on` or `yes` will enable tracking of per-client request statistics
+    /// that will be output each time server status is logged. Default is `off` (disabled).
+    fn client_stats_enabled(&self) -> bool;
+
+    /// [Optional] Likelihood (as a percentage) that the server will intentionally return an
+    /// invalid client response. An integer range from `0` (disabled, all responses valid) to `50`
+    /// (~50% of responses will be invalid). Default is `0` (disabled).
+    ///
+    /// See the [Roughtime spec](https://roughtime.googlesource.com/roughtime/+/HEAD/ECOSYSTEM.md#maintaining-a-healthy-software-ecosystem)
+    /// for background and rationale.
+    fn fault_percentage(&self) -> u8;
+
     /// Convenience function to create a `SocketAddr` from the provided `interface` and `port`
     fn udp_socket_addr(&self) -> Result<SocketAddr, Error> {
         let addr = format!("{}:{}", self.interface(), self.port());
@@ -133,25 +145,26 @@ pub fn is_valid_config(cfg: &Box<dyn ServerConfig>) -> bool {
     let mut is_valid = true;
 
     if cfg.port() == 0 {
-        error!("unset port: {}", cfg.port());
+        error!("server port not set: {}", cfg.port());
         is_valid = false;
     }
+
     if cfg.interface().is_empty() {
-        error!("interface is missing");
+        error!("'interface' is missing");
         is_valid = false;
     }
+
     if cfg.seed().is_empty() {
-        error!("seed value is missing");
+        error!("'seed' value is missing");
         is_valid = false;
-    }
-    if *cfg.kms_protection() == KmsProtection::Plaintext && cfg.seed().len() != 32 {
-        error!("plaintext seed value must be 32 characters long");
+    } else if *cfg.kms_protection() == KmsProtection::Plaintext && cfg.seed().len() != SEED_LENGTH as usize {
+        error!("plaintext seed value must be 32 characters long, found {}", cfg.seed().len());
         is_valid = false;
-    }
-    if *cfg.kms_protection() != KmsProtection::Plaintext && cfg.seed().len() <= 32 {
+    } else if *cfg.kms_protection() != KmsProtection::Plaintext && cfg.seed().len() <= SEED_LENGTH as usize {
         error!("KMS use enabled but seed value is too short to be an encrypted blob");
         is_valid = false;
     }
+
     if cfg.batch_size() < 1 || cfg.batch_size() > 64 {
         error!(
             "batch_size {} is invalid; valid range 1-64",
@@ -160,6 +173,11 @@ pub fn is_valid_config(cfg: &Box<dyn ServerConfig>) -> bool {
         is_valid = false;
     }
 
+    if cfg.fault_percentage() > 50 {
+        error!("fault_percentage {} is invalid; valid range 0-50", cfg.fault_percentage());
+        is_valid = false;
+    }
+
     if is_valid {
         match cfg.udp_socket_addr() {
             Err(e) => {
diff --git a/src/grease.rs b/src/grease.rs
new file mode 100644
index 0000000..b89f828
--- /dev/null
+++ b/src/grease.rs
@@ -0,0 +1,227 @@
+// Copyright 2017-2019 int08h LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//!
+//! Adds deliberate errors to client responses as part of the
+//! [Roughtime Ecosystem](https://roughtime.googlesource.com/roughtime/+/HEAD/ECOSYSTEM.md#maintaining-a-healthy-software-ecosystem).
+//!
+//! See the documentation for [ServerConfig](../config/trait.ServerConfig.html#tymethod.fault_percentage)
+//! on how to enable.
+//!
+
+use rand::{FromEntropy, Rng};
+use rand::distributions::Bernoulli;
+use rand::rngs::SmallRng;
+use rand::seq::SliceRandom;
+use rand::seq::index::sample as index_sample;
+
+use crate::RtMessage;
+use crate::tag::Tag;
+use crate::grease::Pathologies::*;
+use crate::SIGNATURE_LENGTH;
+
+///
+/// Ways that a message can be made invalid.
+///
+pub enum Pathologies {
+    /// Randomly re-order the (tag, value) pairs in the message. This violates the protocol's
+    /// requirement that tags must be in strictly increasing order.
+    RandomlyOrderTags,
+
+    /// Replace the server's signature (value of the SIG tag) with random garbage.
+    CorruptResponseSignature,
+
+    // TODO(stuart) semantic pathologies
+}
+
+static ALL_PATHOLOGIES: &[Pathologies] = &[
+    RandomlyOrderTags,
+    CorruptResponseSignature
+];
+
+///
+/// Adds deliberate errors to client responses as part of the
+/// [Roughtime Ecosystem](https://roughtime.googlesource.com/roughtime/+/HEAD/ECOSYSTEM.md#maintaining-a-healthy-software-ecosystem).
+///
+pub struct Grease {
+    enabled: bool,
+    dist: Bernoulli,
+    prng: SmallRng,
+}
+
+impl Grease {
+    ///
+    /// Creates a new instance `fault_percentage` likely to corrupt a source message.
+    ///
+    pub fn new(fault_percentage: u8) -> Self {
+        Grease {
+            enabled: fault_percentage > 0,
+            dist: Bernoulli::from_ratio(u32::from(fault_percentage), 100),
+            prng: SmallRng::from_entropy(),
+        }
+    }
+
+    ///
+    /// Returns true `fault_percentage` percent of the time.
+    ///
+    #[inline]
+    pub fn should_add_error(&mut self) -> bool {
+        if self.enabled { self.prng.sample(self.dist) } else { false }
+    }
+
+    ///
+    /// Returns a *new* `RtMessage` that has been altered to be deliberately invalid.
+    ///
+    /// The type of alteration made to `src_msg` is randomly chosen from from
+    /// [Pathologies](enum.Pathologies.html)
+    ///
+    pub fn add_errors(&mut self, src_msg: &RtMessage) -> RtMessage {
+        match ALL_PATHOLOGIES.choose(&mut self.prng) {
+            Some(CorruptResponseSignature) => self.corrupt_response_signature(src_msg),
+            Some(RandomlyOrderTags) => self.randomly_order_tags(src_msg),
+            None => unreachable!()
+        }
+    }
+
+    ///
+    /// Randomly shuffle ordering of the (tag, value) pairs in the source message.
+    ///
+    fn randomly_order_tags(&mut self, src_msg: &RtMessage) -> RtMessage {
+        let src_tags = src_msg.tags();
+        let src_values = src_msg.values();
+        let num_fields = src_msg.num_fields() as usize;
+
+        let mut new_tags: Vec<Tag> = Vec::with_capacity(num_fields);
+        let mut new_values: Vec<Vec<u8>> = Vec::with_capacity(num_fields);
+
+        // TODO(stuart) use replacement instead of copying
+        for idx in index_sample(&mut self.prng, num_fields, num_fields).iter() {
+            new_tags.push(*src_tags.get(idx).unwrap());
+            new_values.push(src_values.get(idx).unwrap().to_vec());
+        }
+
+        RtMessage::new_deliberately_invalid(new_tags, new_values)
+    }
+
+    ///
+    /// Replace valid SIG signature with random garbage
+    ///
+    fn corrupt_response_signature(&self, src_msg: &RtMessage) -> RtMessage {
+        if src_msg.get_field(Tag::SIG).is_none() {
+            return src_msg.to_owned();
+        }
+
+        let mut prng = SmallRng::from_entropy();
+        let mut random_sig: [u8; SIGNATURE_LENGTH as usize] = [0u8; SIGNATURE_LENGTH as usize];
+
+        prng.fill(&mut random_sig);
+
+        let mut new_msg = RtMessage::new(src_msg.num_fields());
+        new_msg.add_field(Tag::SIG, &random_sig).unwrap();
+        new_msg.add_field(Tag::PATH, src_msg.get_field(Tag::PATH).unwrap()).unwrap();
+        new_msg.add_field(Tag::SREP, src_msg.get_field(Tag::SREP).unwrap()).unwrap();
+        new_msg.add_field(Tag::CERT, src_msg.get_field(Tag::CERT).unwrap()).unwrap();
+        new_msg.add_field(Tag::INDX, src_msg.get_field(Tag::INDX).unwrap()).unwrap();
+
+        new_msg
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use crate::grease::Grease;
+    use crate::RtMessage;
+    use crate::tag::Tag;
+
+    #[test]
+    fn verify_error_probability() {
+        const TRIALS: u64 = 100_000;
+        const TOLERANCE: f64 = 0.75;
+
+        for target in 1..50 {
+            let mut g = Grease::new(target);
+            let (lower, upper) = (target as f64 - TOLERANCE, target as f64 + TOLERANCE);
+
+            let acc: u64 = (0..TRIALS)
+                .map(|_| if g.should_add_error() { 1 } else { 0 })
+                .sum();
+
+            let percentage = 100.0 * (acc as f64 / TRIALS as f64);
+
+            assert_eq!(
+                percentage > lower && percentage < upper,
+                true,
+                "target {}, actual {} [{}, {}]", target, percentage, lower, upper
+            );
+        }
+    }
+
+    #[test]
+    fn check_tag_reordering() {
+        let pairs = [
+            (Tag::SIG, [b'0']),
+            (Tag::NONC, [b'1']),
+            (Tag::DELE, [b'2']),
+            (Tag::PATH, [b'3']),
+            (Tag::RADI, [b'4']),
+            (Tag::PUBK, [b'5']),
+            (Tag::MIDP, [b'6']),
+            (Tag::SREP, [b'7']),
+            (Tag::MINT, [b'8']),
+            (Tag::ROOT, [b'9']),
+            (Tag::CERT, [b'a']),
+            (Tag::MAXT, [b'b']),
+            (Tag::INDX, [b'c']),
+            (Tag::PAD, [b'd'])
+        ];
+
+        let mut msg = RtMessage::new(14);
+        for pair in &pairs {
+            msg.add_field(pair.0, &pair.1).unwrap();
+        }
+
+        let mut grease = Grease::new(1);
+        let reordered = grease.randomly_order_tags(&msg);
+        println!("orig: {:?}\nnew:  {:?}", msg.tags(), reordered.tags());
+
+        // original and reordered are same length
+        assert_eq!(msg.num_fields(), reordered.num_fields());
+
+        // the shuffle took place
+        assert_ne!(msg.tags(), reordered.tags());
+        assert_ne!(msg.values(), reordered.values());
+
+        // tag still points to same value
+        for (tag, _) in pairs.iter() {
+            assert_eq!(msg.get_field(*tag).unwrap(), reordered.get_field(*tag).unwrap());
+        }
+    }
+
+    #[test]
+    fn check_signature_corruption() {
+        let mut msg = RtMessage::new(5);
+        msg.add_field(Tag::SIG, &[b'a']).unwrap();
+        msg.add_field(Tag::PATH, &[b'0']).unwrap();
+        msg.add_field(Tag::SREP, &[b'1']).unwrap();
+        msg.add_field(Tag::CERT, &[b'2']).unwrap();
+        msg.add_field(Tag::INDX, &[b'3']).unwrap();
+
+        let grease = Grease::new(1);
+        let changed = grease.corrupt_response_signature(&msg);
+
+        println!("orig: {:?}\nnew:  {:?}", msg.get_field(Tag::SIG), changed.get_field(Tag::SIG));
+
+        assert_ne!(msg.get_field(Tag::SIG).unwrap(), changed.get_field(Tag::SIG).unwrap());
+    }
+}
diff --git a/src/kms/envelope.rs b/src/kms/envelope.rs
index b82e61d..d83aa64 100644
--- a/src/kms/envelope.rs
+++ b/src/kms/envelope.rs
@@ -17,7 +17,7 @@ use std::io::{Cursor, Read, Write};
 use ring::aead::{open_in_place, seal_in_place, OpeningKey, SealingKey, AES_256_GCM};
 use ring::rand::{SecureRandom, SystemRandom};
 
-use crate::MIN_SEED_LENGTH;
+use crate::SEED_LENGTH;
 use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
 use crate::kms::{KmsError, KmsProvider, AD, DEK_SIZE_BYTES, NONCE_SIZE_BYTES, TAG_SIZE_BYTES};
 
@@ -33,7 +33,7 @@ const MIN_PAYLOAD_SIZE: usize = DEK_LEN_FIELD
     + NONCE_LEN_FIELD
     + DEK_SIZE_BYTES
     + NONCE_SIZE_BYTES
-    + MIN_SEED_LENGTH as usize
+    + SEED_LENGTH as usize
     + TAG_SIZE_BYTES;
 
 // No input prefix to skip, consume entire buffer
@@ -41,7 +41,7 @@ const IN_PREFIX_LEN: usize = 0;
 
 // Convenience function to create zero-filled Vec of given size
 fn vec_zero_filled(len: usize) -> Vec<u8> {
-    (0..len).into_iter().map(|_| 0).collect()
+    (0..len).map(|_| 0).collect()
 }
 
 /// Envelope encryption of the long-term key seed value.
diff --git a/src/lib.rs b/src/lib.rs
index 5cda7bc..4ec3955 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -63,6 +63,7 @@ mod message;
 mod tag;
 
 pub mod config;
+pub mod grease;
 pub mod key;
 pub mod kms;
 pub mod merkle;
@@ -75,7 +76,7 @@ pub use crate::message::RtMessage;
 pub use crate::tag::Tag;
 
 /// Version of Roughenough
-pub const VERSION: &str = "1.1.3";
+pub const VERSION: &str = "1.1.4";
 
 /// Roughenough version string enriched with any compile-time optional features
 pub fn roughenough_version() -> String {
@@ -95,8 +96,8 @@ pub fn roughenough_version() -> String {
 /// Minimum size (in bytes) of a client request
 pub const MIN_REQUEST_LENGTH: u32 = 1024;
 
-/// Minimum size (in bytes) of seeds used to derive private keys
-pub const MIN_SEED_LENGTH: u32 = 32;
+/// Size (in bytes) of seeds used to derive private keys
+pub const SEED_LENGTH: u32 = 32;
 
 /// Size (in bytes) of an Ed25519 public key
 pub const PUBKEY_LENGTH: u32 = 32;
diff --git a/src/message.rs b/src/message.rs
index d598dee..fea3014 100644
--- a/src/message.rs
+++ b/src/message.rs
@@ -69,6 +69,19 @@ impl RtMessage {
         }
     }
 
+    ///
+    /// Dangerous: construct a new RtMessage **without validation or error checking**.
+    ///
+    /// Intended _only_ for construction of deliberately bogus responses as part of [Roughtime's
+    /// ecosystem](https://roughtime.googlesource.com/roughtime/+/HEAD/ECOSYSTEM.md#maintaining-a-healthy-software-ecosystem).
+    ///
+    pub fn new_deliberately_invalid(tags: Vec<Tag>, values: Vec<Vec<u8>>) -> Self {
+        RtMessage {
+            tags,
+            values,
+        }
+    }
+
     /// Internal function to create a single tag message
     fn single_tag_message(bytes: &[u8], msg: &mut Cursor<&[u8]>) -> Result<Self, Error> {
         if bytes.len() < 8 {
diff --git a/src/server.rs b/src/server.rs
index 6045191..8d2d7c1 100644
--- a/src/server.rs
+++ b/src/server.rs
@@ -35,12 +35,13 @@ use mio::{Events, Poll, PollOpt, Ready, Token};
 use mio::tcp::Shutdown;
 use mio_extras::timer::Timer;
 
+use crate::{Error, RtMessage, Tag, MIN_REQUEST_LENGTH};
 use crate::config::ServerConfig;
+use crate::grease::Grease;
 use crate::key::{LongTermKey, OnlineKey};
 use crate::kms;
 use crate::merkle::MerkleTree;
-use crate::{Error, RtMessage, Tag, MIN_REQUEST_LENGTH};
-use crate::stats::{ClientStats, SimpleStats};
+use crate::stats::{AggregatedStats, PerClientStats, ServerStats};
 
 macro_rules! check_ctrlc {
     ($keep_running:expr) => {
@@ -78,6 +79,7 @@ pub struct Server {
     health_listener: Option<TcpListener>,
     keep_running: Arc<AtomicBool>,
     poll_duration: Option<Duration>,
+    grease: Grease,
     timer: Timer<()>,
     poll: Poll,
     merkle: MerkleTree,
@@ -86,11 +88,11 @@ pub struct Server {
 
     public_key: String,
 
+    stats: Box<dyn ServerStats>,
+
     // Used to send requests to ourselves in fuzzing mode
     #[cfg(fuzzing)]
     fake_client_socket: UdpSocket,
-
-    stats: SimpleStats,
 }
 
 impl Server {
@@ -98,7 +100,7 @@ impl Server {
     /// Create a new server instance from the provided
     /// [`ServerConfig`](../config/trait.ServerConfig.html) trait object instance.
     ///
-    pub fn new(config: Box<ServerConfig>) -> Server {
+    pub fn new(config: Box<dyn ServerConfig>) -> Server {
         let online_key = OnlineKey::new();
         let public_key: String;
 
@@ -148,6 +150,13 @@ impl Server {
             None
         };
 
+        let stats: Box<dyn ServerStats> = if config.client_stats_enabled() {
+            Box::new(PerClientStats::new())
+        } else {
+            Box::new(AggregatedStats::new())
+        };
+
+        let grease = Grease::new(config.fault_percentage());
         let merkle = MerkleTree::new();
         let requests = Vec::with_capacity(config.batch_size() as usize);
 
@@ -161,6 +170,7 @@ impl Server {
 
             keep_running,
             poll_duration,
+            grease,
             timer,
             poll,
             merkle,
@@ -169,10 +179,10 @@ impl Server {
 
             public_key,
 
+            stats,
+
             #[cfg(fuzzing)]
             fake_client_socket: UdpSocket::bind(&"127.0.0.1:0".parse().unwrap()).unwrap(),
-
-            stats: SimpleStats::new(),
         }
     }
 
@@ -213,7 +223,7 @@ impl Server {
     pub fn process_events(&mut self, events: &mut Events) -> bool {
         self.poll
             .poll(events, self.poll_duration)
-            .expect("poll failed");
+            .expect("server event poll failed; cannot recover");
 
         for msg in events.iter() {
             match msg.token() {
@@ -351,7 +361,7 @@ impl Server {
         }
     }
 
-    fn send_responses(&mut self) -> () {
+    fn send_responses(&mut self) {
         let merkle_root = self.merkle.compute_root();
 
         // The SREP tag is identical for each response
@@ -359,8 +369,11 @@ impl Server {
 
         for (i, &(ref nonce, ref src_addr)) in self.requests.iter().enumerate() {
             let paths = self.merkle.get_paths(i);
-            let resp = self.make_response(&srep, &self.cert_bytes, &paths, i as u32);
-            let resp_bytes = resp.encode().unwrap();
+            let resp_msg = {
+                let r = self.make_response(&srep, &self.cert_bytes, &paths, i as u32);
+                if self.grease.should_add_error() { self.grease.add_errors(&r) } else { r }
+            };
+            let resp_bytes = resp_msg.encode().unwrap();
 
             let bytes_sent = self
                 .socket
@@ -374,7 +387,7 @@ impl Server {
                 bytes_sent,
                 src_addr,
                 hex::encode(&nonce[0..4]),
-                i,
+                i + 1,
                 self.stats.total_responses_sent()
             );
         }
diff --git a/src/sign.rs b/src/sign.rs
index 11c02c0..43d3b51 100644
--- a/src/sign.rs
+++ b/src/sign.rs
@@ -116,7 +116,7 @@ impl fmt::Debug for Signer {
     }
 }
 
-#[cfg_attr(rustfmt, rustfmt_skip)] // rustfmt errors on the long signature strings
+#[rustfmt::skip] // rustfmt errors on the long signature strings
 #[cfg(test)]
 mod test {
     use super::*;
diff --git a/src/stats.rs b/src/stats.rs
deleted file mode 100644
index d296e40..0000000
--- a/src/stats.rs
+++ /dev/null
@@ -1,362 +0,0 @@
-// Copyright 2017-2019 int08h LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//!
-//! Facilities for tracking client requests to the server
-//!
-
-use hashbrown::HashMap;
-use hashbrown::hash_map::Iter;
-
-use std::net::IpAddr;
-
-///
-/// Implementations of this trait record client activity
-///
-pub trait ClientStats {
-    fn add_valid_request(&mut self, addr: &IpAddr);
-
-    fn add_invalid_request(&mut self, addr: &IpAddr);
-
-    fn add_health_check(&mut self, addr: &IpAddr);
-
-    fn add_response(&mut self, addr: &IpAddr, bytes_sent: usize);
-
-    fn total_valid_requests(&self) -> u64;
-
-    fn total_invalid_requests(&self) -> u64;
-
-    fn total_health_checks(&self) -> u64;
-
-    fn total_responses_sent(&self) -> u64;
-
-    fn total_bytes_sent(&self) -> usize;
-
-    fn total_unique_clients(&self) -> u64;
-
-    fn get_stats(&self, addr: &IpAddr) -> Option<&StatEntry>;
-
-    fn iter(&self) -> Iter<IpAddr, StatEntry>;
-
-    fn clear(&mut self);
-}
-
-///
-/// Specific metrics tracked per each client
-///
-#[derive(Debug, Clone, Copy)]
-pub struct StatEntry {
-    pub valid_requests: u64,
-    pub invalid_requests: u64,
-    pub health_checks: u64,
-    pub responses_sent: u64,
-    pub bytes_sent: usize,
-}
-
-impl StatEntry {
-    fn new() -> Self {
-        StatEntry {
-            valid_requests: 0,
-            invalid_requests: 0,
-            health_checks: 0,
-            responses_sent: 0,
-            bytes_sent: 0,
-        }
-    }
-}
-
-///
-/// Implementation of `ClientStats` backed by a hashmap.
-///
-/// Maintains a maximum of `MAX_CLIENTS` unique entries to bound memory use. Excess
-/// entries beyond `MAX_CLIENTS` are ignored and `num_overflows` is incremented.
-///
-pub struct SimpleStats {
-    clients: HashMap<IpAddr, StatEntry>,
-    num_overflows: u64,
-    max_clients: usize,
-}
-
-impl SimpleStats {
-
-    /// Maximum number of stats entries to maintain to prevent
-    /// unbounded memory growth.
-    pub const MAX_CLIENTS: usize = 1_000_000;
-
-    pub fn new() -> Self {
-        SimpleStats {
-            clients: HashMap::with_capacity(128),
-            num_overflows: 0,
-            max_clients: SimpleStats::MAX_CLIENTS,
-        }
-    }
-
-    // visible for testing
-    #[cfg(test)]
-    fn with_limits(limit: usize) -> Self {
-        SimpleStats {
-            clients: HashMap::with_capacity(128),
-            num_overflows: 0,
-            max_clients: limit,
-        }
-    }
-
-    #[inline]
-    fn too_many_entries(&mut self) -> bool {
-        let too_big = self.clients.len() >= self.max_clients;
-
-        if too_big {
-            self.num_overflows += 1;
-        }
-
-        return too_big;
-    }
-
-    #[allow(dead_code)]
-    pub fn num_overflows(&self) -> u64 {
-        self.num_overflows
-    }
-}
-
-impl ClientStats for SimpleStats {
-    fn add_valid_request(&mut self, addr: &IpAddr) {
-        if self.too_many_entries() {
-            return;
-        }
-        self.clients
-            .entry(*addr)
-            .or_insert_with(StatEntry::new)
-            .valid_requests += 1;
-    }
-
-    fn add_invalid_request(&mut self, addr: &IpAddr) {
-        if self.too_many_entries() {
-            return;
-        }
-        self.clients
-            .entry(*addr)
-            .or_insert_with(StatEntry::new)
-            .invalid_requests += 1;
-    }
-
-    fn add_health_check(&mut self, addr: &IpAddr) {
-        if self.too_many_entries() {
-            return;
-        }
-        self.clients
-            .entry(*addr)
-            .or_insert_with(StatEntry::new)
-            .health_checks += 1;
-    }
-
-    fn add_response(&mut self, addr: &IpAddr, bytes_sent: usize) {
-        if self.too_many_entries() {
-            return;
-        }
-        let entry = self.clients
-            .entry(*addr)
-            .or_insert_with(StatEntry::new);
-
-        entry.responses_sent += 1;
-        entry.bytes_sent += bytes_sent;
-    }
-
-    fn total_valid_requests(&self) -> u64 {
-        self.clients
-            .values()
-            .map(|&v| v.valid_requests)
-            .sum()
-    }
-
-    fn total_invalid_requests(&self) -> u64 {
-        self.clients
-            .values()
-            .map(|&v| v.invalid_requests)
-            .sum()
-    }
-
-    fn total_health_checks(&self) -> u64 {
-        self.clients
-            .values()
-            .map(|&v| v.health_checks)
-            .sum()
-    }
-
-    fn total_responses_sent(&self) -> u64 {
-        self.clients
-            .values()
-            .map(|&v| v.responses_sent)
-            .sum()
-    }
-
-    fn total_bytes_sent(&self) -> usize {
-        self.clients
-            .values()
-            .map(|&v| v.bytes_sent)
-            .sum()
-    }
-
-    fn total_unique_clients(&self) -> u64 {
-        self.clients.len() as u64
-    }
-
-    fn get_stats(&self, addr: &IpAddr) -> Option<&StatEntry> {
-        self.clients.get(addr)
-    }
-
-    fn iter(&self) -> Iter<IpAddr, StatEntry> {
-        self.clients.iter()
-    }
-
-    fn clear(&mut self) {
-        self.clients.clear();
-        self.num_overflows = 0;
-    }
-}
-
-///
-/// A no-op implementation that does not track anything and has no runtime cost
-///
-#[allow(dead_code)]
-pub struct NoOpStats {
-    empty_map: HashMap<IpAddr, StatEntry>
-}
-
-impl NoOpStats {
-
-    #[allow(dead_code)]
-    pub fn new() -> Self {
-        NoOpStats {
-            empty_map: HashMap::new()
-        }
-    }
-}
-
-impl ClientStats for NoOpStats {
-    fn add_valid_request(&mut self, _addr: &IpAddr) {}
-
-    fn add_invalid_request(&mut self, _addr: &IpAddr) {}
-
-    fn add_health_check(&mut self, _addr: &IpAddr) {}
-
-    fn add_response(&mut self, _addr: &IpAddr, _bytes_sent: usize) {}
-
-    fn total_valid_requests(&self) -> u64 {
-        0
-    }
-
-    fn total_invalid_requests(&self) -> u64 {
-        0
-    }
-
-    fn total_health_checks(&self) -> u64 {
-        0
-    }
-
-    fn total_responses_sent(&self) -> u64 {
-        0
-    }
-
-    fn total_bytes_sent(&self) -> usize {
-        0
-    }
-
-    fn total_unique_clients(&self) -> u64 {
-        0
-    }
-
-    fn get_stats(&self, _addr: &IpAddr) -> Option<&StatEntry> {
-        None
-    }
-
-    fn iter(&self) -> Iter<IpAddr, StatEntry> {
-        self.empty_map.iter()
-    }
-
-    fn clear(&mut self) {}
-}
-
-#[cfg(test)]
-mod test {
-    use crate::stats::{ClientStats, SimpleStats};
-    use std::net::{IpAddr, Ipv4Addr};
-
-    #[test]
-    fn simple_stats_starts_empty() {
-        let stats = SimpleStats::new();
-
-        assert_eq!(stats.total_valid_requests(), 0);
-        assert_eq!(stats.total_invalid_requests(), 0);
-        assert_eq!(stats.total_health_checks(), 0);
-        assert_eq!(stats.total_responses_sent(), 0);
-        assert_eq!(stats.total_bytes_sent(), 0);
-        assert_eq!(stats.total_unique_clients(), 0);
-        assert_eq!(stats.num_overflows(), 0);
-    }
-
-    #[test]
-    fn client_requests_are_tracked() {
-        let mut stats = SimpleStats::new();
-
-        let ip1 = "127.0.0.1".parse().unwrap();
-        let ip2 = "127.0.0.2".parse().unwrap();
-        let ip3 = "127.0.0.3".parse().unwrap();
-
-        stats.add_valid_request(&ip1);
-        stats.add_valid_request(&ip2);
-        stats.add_valid_request(&ip3);
-        assert_eq!(stats.total_valid_requests(), 3);
-
-        stats.add_invalid_request(&ip2);
-        assert_eq!(stats.total_invalid_requests(), 1);
-
-        stats.add_response(&ip2, 8192);
-        assert_eq!(stats.total_bytes_sent(), 8192);
-
-        assert_eq!(stats.total_unique_clients(), 3);
-    }
-
-    #[test]
-    fn per_client_stats() {
-        let mut stats = SimpleStats::new();
-        let ip = "127.0.0.3".parse().unwrap();
-
-        stats.add_valid_request(&ip);
-        stats.add_response(&ip, 2048);
-        stats.add_response(&ip, 1024);
-
-        let entry = stats.get_stats(&ip).unwrap();
-        assert_eq!(entry.valid_requests, 1);
-        assert_eq!(entry.invalid_requests, 0);
-        assert_eq!(entry.responses_sent, 2);
-        assert_eq!(entry.bytes_sent, 3072);
-    }
-
-    #[test]
-    fn overflow_max_entries() {
-        let mut stats = SimpleStats::with_limits(100);
-
-        for i in 0..201 {
-            let ipv4 = Ipv4Addr::from(i as u32);
-            let addr = IpAddr::from(ipv4);
-
-            stats.add_valid_request(&addr);
-        };
-
-        assert_eq!(stats.total_unique_clients(), 100);
-        assert_eq!(stats.num_overflows(), 101);
-    }
-}
-
-
diff --git a/src/stats/aggregated.rs b/src/stats/aggregated.rs
new file mode 100644
index 0000000..389a99d
--- /dev/null
+++ b/src/stats/aggregated.rs
@@ -0,0 +1,103 @@
+// Copyright 2017-2019 int08h LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use hashbrown::HashMap;
+use hashbrown::hash_map::Iter;
+
+use std::net::IpAddr;
+
+use crate::stats::ClientStatEntry;
+use crate::stats::ServerStats;
+
+///
+/// Implementation of `ServerStats` that provides high-level aggregated client statistics. No
+/// per-client statistic are maintained and runtime memory use is constant.
+///
+#[allow(dead_code)]
+pub struct AggregatedStats {
+    valid_requests: u64,
+    invalid_requests: u64,
+    health_checks: u64,
+    responses_sent: u64,
+    bytes_sent: usize,
+    empty_map: HashMap<IpAddr, ClientStatEntry>,
+}
+
+impl AggregatedStats {
+
+    #[allow(dead_code)]
+    pub fn new() -> Self {
+        AggregatedStats {
+            valid_requests: 0,
+            invalid_requests: 0,
+            health_checks: 0,
+            responses_sent: 0,
+            bytes_sent: 0,
+            empty_map: HashMap::new()
+        }
+    }
+}
+
+impl ServerStats for AggregatedStats {
+    fn add_valid_request(&mut self, _: &IpAddr) {
+        self.valid_requests += 1
+    }
+
+    fn add_invalid_request(&mut self, _: &IpAddr) {
+        self.invalid_requests += 1
+    }
+
+    fn add_health_check(&mut self, _: &IpAddr) {
+        self.health_checks += 1
+    }
+
+    fn add_response(&mut self, _: &IpAddr, bytes_sent: usize) {
+        self.bytes_sent += bytes_sent;
+        self.responses_sent += 1;
+    }
+
+    fn total_valid_requests(&self) -> u64 {
+        self.valid_requests
+    }
+
+    fn total_invalid_requests(&self) -> u64 {
+        self.invalid_requests
+    }
+
+    fn total_health_checks(&self) -> u64 {
+        self.health_checks
+    }
+
+    fn total_responses_sent(&self) -> u64 {
+        self.responses_sent
+    }
+
+    fn total_bytes_sent(&self) -> usize {
+        self.bytes_sent
+    }
+
+    fn total_unique_clients(&self) -> u64 {
+        0
+    }
+
+    fn stats_for_client(&self, _addr: &IpAddr) -> Option<&ClientStatEntry> {
+        None
+    }
+
+    fn iter(&self) -> Iter<IpAddr, ClientStatEntry> {
+        self.empty_map.iter()
+    }
+
+    fn clear(&mut self) {}
+}
diff --git a/src/stats/mod.rs b/src/stats/mod.rs
new file mode 100644
index 0000000..ad90bd2
--- /dev/null
+++ b/src/stats/mod.rs
@@ -0,0 +1,157 @@
+// Copyright 2017-2019 int08h LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//!
+//! Facilities for tracking client requests to the server
+//!
+
+mod aggregated;
+mod per_client;
+
+pub use crate::stats::aggregated::AggregatedStats;
+pub use crate::stats::per_client::PerClientStats;
+
+use hashbrown::hash_map::Iter;
+
+use std::net::IpAddr;
+
+///
+/// Specific metrics tracked per each client
+///
+#[derive(Debug, Clone, Copy)]
+pub struct ClientStatEntry {
+    pub valid_requests: u64,
+    pub invalid_requests: u64,
+    pub health_checks: u64,
+    pub responses_sent: u64,
+    pub bytes_sent: usize,
+}
+
+impl ClientStatEntry {
+    fn new() -> Self {
+        ClientStatEntry {
+            valid_requests: 0,
+            invalid_requests: 0,
+            health_checks: 0,
+            responses_sent: 0,
+            bytes_sent: 0,
+        }
+    }
+}
+
+///
+/// Implementations of this trait record client activity
+///
+pub trait ServerStats {
+    fn add_valid_request(&mut self, addr: &IpAddr);
+
+    fn add_invalid_request(&mut self, addr: &IpAddr);
+
+    fn add_health_check(&mut self, addr: &IpAddr);
+
+    fn add_response(&mut self, addr: &IpAddr, bytes_sent: usize);
+
+    fn total_valid_requests(&self) -> u64;
+
+    fn total_invalid_requests(&self) -> u64;
+
+    fn total_health_checks(&self) -> u64;
+
+    fn total_responses_sent(&self) -> u64;
+
+    fn total_bytes_sent(&self) -> usize;
+
+    fn total_unique_clients(&self) -> u64;
+
+    fn stats_for_client(&self, addr: &IpAddr) -> Option<&ClientStatEntry>;
+
+    fn iter(&self) -> Iter<IpAddr, ClientStatEntry>;
+
+    fn clear(&mut self);
+}
+
+
+#[cfg(test)]
+mod test {
+    use crate::stats::{ServerStats, PerClientStats};
+    use std::net::{IpAddr, Ipv4Addr};
+
+    #[test]
+    fn simple_stats_starts_empty() {
+        let stats = PerClientStats::new();
+
+        assert_eq!(stats.total_valid_requests(), 0);
+        assert_eq!(stats.total_invalid_requests(), 0);
+        assert_eq!(stats.total_health_checks(), 0);
+        assert_eq!(stats.total_responses_sent(), 0);
+        assert_eq!(stats.total_bytes_sent(), 0);
+        assert_eq!(stats.total_unique_clients(), 0);
+        assert_eq!(stats.num_overflows(), 0);
+    }
+
+    #[test]
+    fn client_requests_are_tracked() {
+        let mut stats = PerClientStats::new();
+
+        let ip1 = "127.0.0.1".parse().unwrap();
+        let ip2 = "127.0.0.2".parse().unwrap();
+        let ip3 = "127.0.0.3".parse().unwrap();
+
+        stats.add_valid_request(&ip1);
+        stats.add_valid_request(&ip2);
+        stats.add_valid_request(&ip3);
+        assert_eq!(stats.total_valid_requests(), 3);
+
+        stats.add_invalid_request(&ip2);
+        assert_eq!(stats.total_invalid_requests(), 1);
+
+        stats.add_response(&ip2, 8192);
+        assert_eq!(stats.total_bytes_sent(), 8192);
+
+        assert_eq!(stats.total_unique_clients(), 3);
+    }
+
+    #[test]
+    fn per_client_stats() {
+        let mut stats = PerClientStats::new();
+        let ip = "127.0.0.3".parse().unwrap();
+
+        stats.add_valid_request(&ip);
+        stats.add_response(&ip, 2048);
+        stats.add_response(&ip, 1024);
+
+        let entry = stats.stats_for_client(&ip).unwrap();
+        assert_eq!(entry.valid_requests, 1);
+        assert_eq!(entry.invalid_requests, 0);
+        assert_eq!(entry.responses_sent, 2);
+        assert_eq!(entry.bytes_sent, 3072);
+    }
+
+    #[test]
+    fn overflow_max_entries() {
+        let mut stats = PerClientStats::with_limit(100);
+
+        for i in 0..201 {
+            let ipv4 = Ipv4Addr::from(i as u32);
+            let addr = IpAddr::from(ipv4);
+
+            stats.add_valid_request(&addr);
+        };
+
+        assert_eq!(stats.total_unique_clients(), 100);
+        assert_eq!(stats.num_overflows(), 101);
+    }
+}
+
+
diff --git a/src/stats/per_client.rs b/src/stats/per_client.rs
new file mode 100644
index 0000000..8641fde
--- /dev/null
+++ b/src/stats/per_client.rs
@@ -0,0 +1,171 @@
+// Copyright 2017-2019 int08h LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use hashbrown::HashMap;
+use hashbrown::hash_map::Iter;
+
+use std::net::IpAddr;
+
+use crate::stats::ClientStatEntry;
+use crate::stats::ServerStats;
+
+///
+/// Implementation of `ServerStats` that provides granular per-client request/response counts.
+///
+/// Each unique client address is used to key a hashmap. A maximum of `MAX_CLIENTS` entries
+/// are kept in the map to bound memory use. Excess entries beyond `MAX_CLIENTS` are ignored
+/// and `num_overflows` is incremented.
+///
+pub struct PerClientStats {
+    clients: HashMap<IpAddr, ClientStatEntry>,
+    num_overflows: u64,
+    max_clients: usize,
+}
+
+impl PerClientStats {
+
+    /// Maximum number of entries to prevent unbounded memory growth.
+    pub const MAX_CLIENTS: usize = 1_000_000;
+
+    pub fn new() -> Self {
+        PerClientStats {
+            clients: HashMap::with_capacity(64),
+            num_overflows: 0,
+            max_clients: PerClientStats::MAX_CLIENTS,
+        }
+    }
+
+    // visible for testing
+    #[cfg(test)]
+    pub fn with_limit(limit: usize) -> Self {
+        PerClientStats {
+            clients: HashMap::with_capacity(64),
+            num_overflows: 0,
+            max_clients: limit,
+        }
+    }
+
+    #[inline]
+    fn too_many_entries(&mut self) -> bool {
+        let too_big = self.clients.len() >= self.max_clients;
+
+        if too_big {
+            self.num_overflows += 1;
+        }
+
+        too_big
+    }
+
+    #[allow(dead_code)]
+    pub fn num_overflows(&self) -> u64 {
+        self.num_overflows
+    }
+}
+
+impl ServerStats for PerClientStats {
+    fn add_valid_request(&mut self, addr: &IpAddr) {
+        if self.too_many_entries() {
+            return;
+        }
+        self.clients
+            .entry(*addr)
+            .or_insert_with(ClientStatEntry::new)
+            .valid_requests += 1;
+    }
+
+    fn add_invalid_request(&mut self, addr: &IpAddr) {
+        if self.too_many_entries() {
+            return;
+        }
+        self.clients
+            .entry(*addr)
+            .or_insert_with(ClientStatEntry::new)
+            .invalid_requests += 1;
+    }
+
+    fn add_health_check(&mut self, addr: &IpAddr) {
+        if self.too_many_entries() {
+            return;
+        }
+        self.clients
+            .entry(*addr)
+            .or_insert_with(ClientStatEntry::new)
+            .health_checks += 1;
+    }
+
+    fn add_response(&mut self, addr: &IpAddr, bytes_sent: usize) {
+        if self.too_many_entries() {
+            return;
+        }
+        let entry = self.clients
+            .entry(*addr)
+            .or_insert_with(ClientStatEntry::new);
+
+        entry.responses_sent += 1;
+        entry.bytes_sent += bytes_sent;
+    }
+
+    fn total_valid_requests(&self) -> u64 {
+        self.clients
+            .values()
+            .map(|&v| v.valid_requests)
+            .sum()
+    }
+
+    fn total_invalid_requests(&self) -> u64 {
+        self.clients
+            .values()
+            .map(|&v| v.invalid_requests)
+            .sum()
+    }
+
+    fn total_health_checks(&self) -> u64 {
+        self.clients
+            .values()
+            .map(|&v| v.health_checks)
+            .sum()
+    }
+
+    fn total_responses_sent(&self) -> u64 {
+        self.clients
+            .values()
+            .map(|&v| v.responses_sent)
+            .sum()
+    }
+
+    fn total_bytes_sent(&self) -> usize {
+        self.clients
+            .values()
+            .map(|&v| v.bytes_sent)
+            .sum()
+    }
+
+    fn total_unique_clients(&self) -> u64 {
+        self.clients.len() as u64
+    }
+
+    fn stats_for_client(&self, addr: &IpAddr) -> Option<&ClientStatEntry> {
+        self.clients.get(addr)
+    }
+
+    fn iter(&self) -> Iter<IpAddr, ClientStatEntry> {
+        self.clients.iter()
+    }
+
+    fn clear(&mut self) {
+        self.clients.clear();
+        self.num_overflows = 0;
+    }
+}
+