Skip to content

Add middleware to ensure Router only handles errors for its clients #310

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions lib/redis_client/cluster/error_identification.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# frozen_string_literal: true

class RedisClient
class Cluster
module ErrorIdentification
def self.client_owns_error?(err, client)
err.is_a?(TaggedError) && err.from?(client)
end

module TaggedError
attr_accessor :config_instance

def from?(client)
client.config.equal?(config_instance)
end
end

module Middleware
def connect(config)
super
rescue RedisClient::Error => e
identify_error(e, config)
raise
end

def call(_command, config)
super
rescue RedisClient::Error => e
identify_error(e, config)
raise
end
alias call_pipelined call

private

def identify_error(err, config)
err.singleton_class.include(TaggedError)
err.config_instance = config
end
end
end
end
end
11 changes: 9 additions & 2 deletions lib/redis_client/cluster/node.rb
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

require 'redis_client'
require 'redis_client/config'
require 'redis_client/cluster/error_identification'
require 'redis_client/cluster/errors'
require 'redis_client/cluster/node/primary_only'
require 'redis_client/cluster/node/random_replica'
Expand Down Expand Up @@ -78,9 +79,11 @@ def []=(index, element)
end

class Config < ::RedisClient::Config
def initialize(scale_read: false, **kwargs)
def initialize(scale_read: false, middlewares: nil, **kwargs)
@scale_read = scale_read
super(**kwargs)
middlewares ||= []
middlewares.unshift ErrorIdentification::Middleware
super(middlewares: middlewares, **kwargs)
end

private
Expand Down Expand Up @@ -214,6 +217,10 @@ def reload!
end
end

def owns_error?(err)
any? { |c| ErrorIdentification.client_owns_error?(err, c) }
end

private

def make_topology_class(with_replica, replica_affinity)
Expand Down
8 changes: 6 additions & 2 deletions lib/redis_client/cluster/router.rb
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ def send_command(method, command, *args, &block) # rubocop:disable Metrics/AbcSi
raise if e.errors.any?(::RedisClient::CircuitBreaker::OpenCircuitError)

update_cluster_info! if e.errors.values.any? do |err|
err.message.start_with?('CLUSTERDOWN Hash slot not served')
@node.owns_error?(err) && err.message.start_with?('CLUSTERDOWN Hash slot not served')
end

raise
Expand Down Expand Up @@ -94,6 +94,8 @@ def handle_redirection(node, retry_count:) # rubocop:disable Metrics/AbcSize, Me
rescue ::RedisClient::CircuitBreaker::OpenCircuitError
raise
rescue ::RedisClient::CommandError => e
raise unless ErrorIdentification.client_owns_error?(e, node)

if e.message.start_with?('MOVED')
node = assign_redirection_node(e.message)
retry_count -= 1
Expand All @@ -111,7 +113,9 @@ def handle_redirection(node, retry_count:) # rubocop:disable Metrics/AbcSize, Me
retry if retry_count >= 0
end
raise
rescue ::RedisClient::ConnectionError
rescue ::RedisClient::ConnectionError => e
raise unless ErrorIdentification.client_owns_error?(e, node)

update_cluster_info!

raise if retry_count <= 0
Expand Down
18 changes: 18 additions & 0 deletions test/redis_client/test_cluster.rb
Original file line number Diff line number Diff line change
Expand Up @@ -485,6 +485,24 @@ def test_circuit_breakers
cli&.close
end

def test_only_reshards_own_errors
@client.call_v(%w[SADD testkey testvalue1])
@client.call_v(%w[SADD testkey testvalue2])
slot = ::RedisClient::Cluster::KeySlotConverter.convert('testkey')
router = @client.instance_variable_get(:@router)
correct_primary_key = router.find_node_key_by_key('testkey', primary: true)
broken_primary_key = (router.node_keys - [correct_primary_key]).first
assert_raises(RedisClient::CommandError) do
@client.sscan('testkey', retry_count: 0) do
raise RedisClient::CommandError, "MOVED #{slot} #{broken_primary_key}"
end
end

# The exception should not have causes @client to update its shard mappings, because it didn't
# come from a RedisClient instance that @client knows about.
assert_equal correct_primary_key, router.find_node_key_by_key('testkey', primary: true)
end

private

def wait_for_replication
Expand Down