New: Store hash of imported FIT files to speed up import

Garmin devices typically keep older activity FIT files on the device.
Parsing a FIT file can be costly at scale. Postrunner now stores the
MD5 sums of recently importet files and checks against that list
before parsing the FIT file.

2 files changed, 25 insertions, 0 deletions

diff --git a/lib/postrunner/FFS_Device.rb b/lib/postrunner/FFS_Device.rb
index 06e1fc8..6ea6890 100644
--- a/lib/postrunner/FFS_Device.rb
+++ b/lib/postrunner/FFS_Device.rb
@@ -93,6 +93,13 @@ module PostRunner
       entities << entity
       entities.sort!
 
+      md5sums = @store['fit_file_md5sums']
+      md5sums << FitFileStore.calc_md5_sum(fit_file_name)
+      # We only store the 512 most recently added FIT files. This should be
+      # more than a device can store. This will allow us to skip the already
+      # imported FIT files quickly instead of having to parse them each time.
+      md5sums.shift if md5sums.length > 512
+
       # Scan the activity for any potential new personal records and register
       # them.
       if entity.is_a?(FFS_Activity)
diff --git a/lib/postrunner/FitFileStore.rb b/lib/postrunner/FitFileStore.rb
index 981da6d..6ee484f 100644
--- a/lib/postrunner/FitFileStore.rb
+++ b/lib/postrunner/FitFileStore.rb
@@ -10,6 +10,7 @@
 # published by the Free Software Foundation.
 #
 
+require 'digest'
 require 'fit4ruby'
 require 'perobs'
 
@@ -54,6 +55,9 @@ module PostRunner
       # safely to another directory.
       @store['config']['devices_dir'] = @devices_dir
       create_directory(@devices_dir, 'devices')
+      unless @store['fit_file_md5sums']
+        @store['fit_file_md5sums'] = @store.new(PEROBS::Array)
+      end
 
       # Define which View objects the HTML output will consist of. This
       # doesn't really belong in this class but for now it's the best place
@@ -76,6 +80,12 @@ module PostRunner
     # @return [FFS_Activity or FFS_Monitoring] Corresponding entry in the
     #         FitFileStore or nil if file could not be added.
     def add_fit_file(fit_file_name, fit_entity = nil, overwrite = false)
+      md5sum = FitFileStore.calc_md5_sum(fit_file_name)
+      if @store['fit_file_md5sums'].include?(md5sum)
+        # The FIT file is already stored in the DB.
+        return nil unless overwrite
+      end
+
       # If we the file hasn't been read yet, read it in as a
       # Fit4Ruby::Activity or Fit4Ruby::Monitoring entity.
       unless fit_entity
@@ -420,6 +430,14 @@ module PostRunner
       puts MonitoringStatistics.new(monitoring_files).monthly(day)
     end
 
+    def FitFileStore::calc_md5_sum(file_name)
+      begin
+        Digest::MD5.hexdigest File.read(file_name)
+      rescue IOError
+        return 0
+      end
+    end
+
     private
 
     def read_fit_file(fit_file_name)