handle redis cache upstream failure by falling back to RPC call

Author: Shaun Carlson

lib/active_remote/cached.rb

@@ -190,6 +190,9 @@ def self.#{method_name}(#{expanded_method_args}, __active_remote_cached_options
             ].compact
 
             ::ActiveRemote::Cached.cache.exist?(cache_key)
+          rescue => e
+            return false if e.message.include?("upstream failure")
+            raise
           end
         RUBY
 
@@ -217,6 +220,9 @@ def self.#{method_name}(#{expanded_method_args}, __active_remote_cached_options
             ].compact
 
             ::ActiveRemote::Cached.cache.exist?(cache_key)
+          rescue => e
+            return false if e.message.include?("upstream failure")
+            raise
           end
         RUBY
 
@@ -263,6 +269,9 @@ def self.#{method_name}(#{expanded_method_args}, __active_remote_cached_options
                 self.find(#{expanded_search_args})
               end
             end
+          rescue => e
+            return self.find(#{expanded_search_args}) if e.message.include?("upstream failure")
+            raise
           end
         RUBY
       end
@@ -311,6 +320,9 @@ def self.#{method_name}(#{expanded_method_args}, __active_remote_cached_options
                 self.search(#{expanded_search_args})
               end
             end
+          rescue => e
+            return self.search(#{expanded_search_args}) if e.message.include?("upstream failure")
+            raise
           end
         RUBY
       end
@@ -369,6 +381,13 @@ def self.#{method_name}(#{expanded_method_args}, __active_remote_cached_options
               raise ::ActiveRemote::RemoteRecordNotFound.new(self.class) if results.first.nil?
               results
             end
+          rescue => e
+            if e.message.include?("upstream failure")
+              results = self.search(#{expanded_search_args})
+              raise ::ActiveRemote::RemoteRecordNotFound.new(self.class) if results.first.nil?
+              return results
+            end
+            raise
           end
         RUBY
       end

spec/active_remote/cached_exist_methods_spec.rb

@@ -126,5 +126,19 @@ def self.search; nil; end
     it "creates 'cached_exist_search_by_client_guid_and_user_guid_and_derp?'" do
       _(ExistMethodClass).must_respond_to("cached_exist_search_by_client_guid_and_user_guid_and_derp?")
     end
+
+    describe "when cache raises upstream failure redis error" do
+      it "returns false" do
+        ::ActiveRemote::Cached.cache.expects(:exist?).raises(::RuntimeError, "upstream failure")
+        _(ExistMethodClass.cached_exist_search_by_guid?(:guid)).must_equal(false)
+      end
+    end
+
+    describe "when cache raises any other kind of error" do
+      it "allows error to pass through" do
+        ::ActiveRemote::Cached.cache.expects(:exist?).raises(::RuntimeError, "kaBOOM")
+        assert_raises(::RuntimeError, "kaBOOM") { ExistMethodClass.cached_exist_search_by_guid?(:guid) }
+      end
+    end
   end
 end

spec/active_remote/cached_find_methods_spec.rb

@@ -99,6 +99,26 @@ def self.search; nil; end
         end
       end
     end
+
+    describe "when cache raises upstream failure redis error" do
+      it "falls back to calling find" do
+        ::ActiveRemote::Cached.cache.expects(:fetch).raises(::RuntimeError, "upstream failure")
+
+        FindMethodClass.stub(:find, "foo") do
+          _(FindMethodClass.cached_find_by_guid(:guid)).must_equal("foo")
+        end
+      end
+    end
+
+    describe "when cache raises any other kind of error" do
+      it "allows error to pass through" do
+        ::ActiveRemote::Cached.cache.expects(:fetch).raises(::RuntimeError, "kaBOOM")
+
+        FindMethodClass.stub(:find, "foo") do
+          assert_raises(::RuntimeError, "kaBOOM") { FindMethodClass.cached_find_by_guid(:guid) }
+        end
+      end
+    end
   end
 
   describe "#cached_find_by_foo" do

spec/active_remote/cached_search_methods_spec.rb

@@ -166,6 +166,26 @@ def self.search; nil; end
         end
       end
     end
+
+    describe "when cache raises upstream failure redis error" do
+      it "falls back to calling search" do
+        ::ActiveRemote::Cached.cache.expects(:fetch).raises(::RuntimeError, "upstream failure")
+
+        SearchMethodClass.stub(:search, "foo") do
+          _(SearchMethodClass.cached_search_by_guid(:guid)).must_equal("foo")
+        end
+      end
+    end
+
+    describe "when cache raises any other kind of error" do
+      it "allows error to pass through" do
+        ::ActiveRemote::Cached.cache.expects(:fetch).raises(::RuntimeError, "kaBOOM")
+
+        SearchMethodClass.stub(:search, "foo") do
+          assert_raises(::RuntimeError, "kaBOOM") { SearchMethodClass.cached_search_by_guid(:guid) }
+        end
+      end
+    end
   end
 
   describe "#cached_search_by_foo" do