conf functions for interacting with Redis cluster via Hiredis. More...

#include <freeradius-devel/util/debug.h>
#include <freeradius-devel/server/cf_parse.h>
#include <freeradius-devel/util/fifo.h>
#include <freeradius-devel/util/misc.h>
#include <freeradius-devel/util/rand.h>
#include "config.h"
#include "base.h"
#include "cluster.h"
#include "crc16.h"

Include dependency graph for cluster.c:

Go to the source code of this file.

Data Structures
struct	cluster_nodes_live_t
	Live nodes data, used to perform weighted random selection of alternative nodes. More...

struct	cluster_nodes_live_t.node

struct	fr_redis_cluster
	A redis cluster. More...

struct	fr_redis_cluster_key_slot_s
	Indexes in the fr_redis_cluster_node_t array for a single key slot. More...

struct	fr_redis_cluster_node_s
	A Redis cluster node. More...

Macros
#define	CLOSED_PERIOD 10000
	How recently must the closed have occurred for us to care.

#define	CLOSED_WEIGHT 1
	What weight to give to nodes that had a connection closed recently.

#define	FAILED_PERIOD 10000
	How recently must the spawn failure occurred for us to care.

#define	FAILED_WEIGHT 1
	What weight to give to nodes that had a spawn failure recently.

#define	KEY_SLOTS 16384
	Maximum number of keyslots (should not change).

#define	MAX_SLAVES 5
	Maximum number of slaves associated with a keyslot.

#define	RELEASED_MIN_WEIGHT 1000
	Minimum weight to assign to node.

#define	RELEASED_PERIOD 10000
	Period after which we don't care about when the last connection was released.

#define	SET_ACTIVE(_node)

#define	SET_ADDR(_addr, _map)

#define	SET_INACTIVE(_node)

Functions
static int	_cluster_conn_free (fr_redis_conn_t *conn)
	Callback for freeing a Redis connection.

static int8_t	_cluster_node_cmp (void const one, void const two)
	Compare two redis nodes to check equality.

static void	_cluster_node_conf_apply (fr_pool_t pool, void opaque)
	Reconnect callback to apply new pool config.

static int	_fr_redis_cluster_free (fr_redis_cluster_t *cluster)
	Destroy mutex associated with cluster slots structure.

static uint16_t	cluster_key_hash (uint8_t const *key, size_t key_len)
	Resolve key to key slot.

static fr_redis_cluster_rcode_t	cluster_map_apply (fr_redis_cluster_t cluster, redisReply reply)
	Apply a cluster map received from a cluster node.

static fr_redis_cluster_rcode_t	cluster_map_get (redisReply *out, fr_redis_conn_t conn)
	Learn a new cluster layout by querying the node that issued the -MOVE.

static int	cluster_map_node_validate (redisReply *node, int map_idx, int node_idx)
	Validate a cluster map node entry.

static fr_redis_cluster_rcode_t	cluster_node_conf_from_redirect (uint16_t key_slot, fr_socket_t node_addr, redisReply *redirect)
	Parse a -MOVED or -ASK redirect.

static fr_redis_cluster_rcode_t	cluster_node_connect (fr_redis_cluster_t cluster, fr_redis_cluster_node_t node)
	Establish a connection to a cluster node.

static int	cluster_node_find_live (fr_redis_cluster_node_t live_node, fr_redis_conn_t live_conn, request_t request, fr_redis_cluster_t cluster, fr_redis_cluster_node_t *skip)
	Attempt to find a live pool in the cluster.

static fr_redis_cluster_rcode_t	cluster_node_ping (request_t request, fr_redis_cluster_node_t node, fr_redis_conn_t *conn)
	Issue a ping request against a cluster node.

static int	cluster_node_pool_health (fr_time_t now, fr_pool_state_t const *state)
	Try to determine the health of a cluster node passively by examining its pool state.

static fr_redis_cluster_rcode_t	cluster_redirect (fr_redis_cluster_node_t *out, fr_redis_cluster_t cluster, redisReply *reply)
	Retrieve or associate a node with the server indicated in the redirect.

fr_redis_cluster_t *	fr_redis_cluster_alloc (TALLOC_CTX ctx, CONF_SECTION module, fr_redis_conf_t conf, char const log_prefix, char const trigger_prefix, fr_pair_list_t trigger_args)
	Allocate and initialise a new cluster structure.

void *	fr_redis_cluster_conn_create (TALLOC_CTX ctx, void instance, fr_time_delta_t timeout)
	Create a new connection to a Redis node.

int	fr_redis_cluster_ipaddr (fr_ipaddr_t out, fr_redis_cluster_node_t const node)
	Return the ipaddr of a particular node.

fr_redis_cluster_node_t const *	fr_redis_cluster_master (fr_redis_cluster_t cluster, fr_redis_cluster_key_slot_t const key_slot)
	Return the master node that would be used for a particular key.

bool	fr_redis_cluster_min_version (fr_redis_cluster_t cluster, char const min_version)
	Check if members of the cluster are above a certain version.

ssize_t	fr_redis_cluster_node_addr_by_role (TALLOC_CTX ctx, fr_socket_t out[], fr_redis_cluster_t *cluster, bool is_master, bool is_slave)
	Return an array of IP addresses belonging to masters or slaves.

int	fr_redis_cluster_pool_by_node_addr (fr_pool_t *pool, fr_redis_cluster_t cluster, fr_socket_t *node_addr, bool create)
	Get the pool associated with a node in the cluster.

int	fr_redis_cluster_port (uint16_t out, fr_redis_cluster_node_t const node)
	Return the port of a particular node.

fr_redis_cluster_rcode_t	fr_redis_cluster_remap (request_t request, fr_redis_cluster_t cluster, fr_redis_conn_t *conn)
	Perform a runtime remap of the cluster.

fr_redis_cluster_node_t const *	fr_redis_cluster_slave (fr_redis_cluster_t cluster, fr_redis_cluster_key_slot_t const key_slot, uint8_t slave_num)
	Return the slave node that would be used for a particular key.

fr_redis_cluster_key_slot_t const *	fr_redis_cluster_slot_by_key (fr_redis_cluster_t cluster, request_t request, uint8_t const *key, size_t key_len)
	Implements the key slot selection scheme used by freeradius.

fr_redis_rcode_t	fr_redis_cluster_state_init (fr_redis_cluster_state_t state, fr_redis_conn_t conn, fr_redis_cluster_t cluster, request_t request, uint8_t const key, size_t key_len, bool read_only)
	Resolve a key to a pool, and reserve a connection in that pool.

fr_redis_rcode_t	fr_redis_cluster_state_next (fr_redis_cluster_state_t state, fr_redis_conn_t conn, fr_redis_cluster_t cluster, request_t request, fr_redis_rcode_t status, redisReply *reply)
	Get the next connection to attempt a command against.

Variables
fr_table_num_sorted_t const	fr_redis_cluster_rcodes_table []

size_t	fr_redis_cluster_rcodes_table_len = NUM_ELEMENTS(fr_redis_cluster_rcodes_table)

Detailed Description

conf functions for interacting with Redis cluster via Hiredis.

Id: 24ecbd15a504866791d761ef956b7b571f5eca53

Author: Arran Cudbard-Bell

Copyright: 2015 Arran Cudbard-Bell (a.cud.nosp@m.bard.nosp@m.b@fre.nosp@m.erad.nosp@m.ius.o.nosp@m.rg); 2015 Network RADIUS (legal.nosp@m.@net.nosp@m.workr.nosp@m.adiu.nosp@m.s.com); 2015 The FreeRADIUS server project

Overview

Read and understand this http://redis.io/topics/cluster-spec first, else the text below will not be useful.

Using the cluster's public API

The two functions used at runtime to issue commands are fr_redis_cluster_state_init and fr_redis_cluster_state_next.

fr_redis_cluster_state_init initialises the structure we use to track which node we're communicating with, and uses a key (string) to determine which node we should try and contact first.

fr_redis_cluster_state_next examines the result of the last command, and either gets a new connection, or errors out.

In both cases the connection should not be released by the caller, only by fr_redis_cluster_state_next.

Below is the sequence of calls required to use the cluster:

1. During initialization allocate a cluster configuration structure with
   #fr_redis_cluster_alloc.  This holds the cluster configuration and state.
2. Declare a #fr_redis_cluster_state_t in the function that needs to issue
   commands against the cluster.
3. Call #fr_redis_cluster_state_init with relevant arguments including a pointer
   to the #fr_redis_cluster_state_t struct, and a key/key_len used for initial
   lookup.  For most commands the key is the value of the second argument.
   #fr_redis_cluster_state_init will then and reserve/pass back a connection for
   the pool associated with the node associated with the key.
4. Use the connection that was passed back, to issue a Redis command against.
5. Use #fr_redis_command_status to translate the result of the command into
   a #fr_redis_rcode_t value.
6. Call #fr_redis_cluster_state_next with relevant arguments including a pointer
   to the #fr_redis_cluster_state_t struct and the #fr_redis_rcode_t value for the
   last command.
7. If #fr_redis_cluster_state_next returns 0, repeat steps 4-7.  Otherwise
   stop and analyse the return value.

See fr_redis_cluster_state_init for example code.

Structures

This code maintains a series structures for efficient lookup and lockless operations.

The important ones are:

An array of fr_redis_cluster_node_t. These are pre-allocated on startup and are never added to, or removed from.
An fr_fifo_t. This contains the queue of nodes that may be re-used.
An fr_rb_tree_t. This contains a tree of nodes which are active. The tree is built on IP address and port.

Each fr_redis_cluster_node_t contains a master ID, and an array of slave IDs. The IDs are array indexes in the fr_redis_cluster_t.node array. We use 8bit unsigned integers instead of pointers to save space. Using pointers, the node[] array would need 784K, using IDs it uses 112K. Still not light on memory, but a bit more acceptable. Currently the key_slot array is shadowed by key_slot_pending, used to stage new key_slot mappings. This doubles the memory used. We may want to consider allocating key_slot_pending only during remappings and freeing it after.

Mapping/Remapping the cluster

On startup, and during cluster operation, a remap may be performed. A remap involves the following steps:

Executing the Redis 'cluster slots' command.
Validating the result of this command. We need to do extensive validation to avoid SEGV on invalid data, due to the way libhiredis presents the result.
Determining the intersection between nodes described in the result, and those already in our fr_rb_tree_t.
Connecting to nodes that were in the result, but not in the tree. Note: If we can't connect to any of the masters, we count the map as invalid, roll back any newly connected nodes, and error out. Slave failure is OK.
Mapping keyslot ranges to nodes in the key_slot_pending array.
Verifying there are no holes in the ranges (if there are, we roll back and error out).
Applying the new keyslot ranges.
Removing nodes no longer used by the key slots, and adding them back to the free nodes queue.

cluster_map_get and cluster_map_apply, perform the operations described above. The get function, issues the 'cluster slots' command and performs validation, the apply function processes and applies the map.

Failing to apply a map is not a fatal error at runtime, and is only fatal on startup if pool.start > 0.

The cluster client can continue to operate, albeit inefficiently, with a stale cluster map by following '-ASK' and '-MOVE' redirects.

Remaps are limited to one per second. If any operation sets the remap_needed flag, or attempts a remap directly, the remap may be skipped if one occurred recently.

Processing '-ASK' and '-MOVE' redirects

The code treats '-ASK' (temporary redirect) and '-MOVE' (permanent redirect) responses similarly. If the node is known, then a connection is reserved from its pool, if the node is not known, a new pool is established, and a connection reserved.

The difference between '-ASK' and '-MOVE' is that '-MOVE' attempts a cluster remap before following the redirect.

The data from '-MOVE' responses, is not used to alter the cluster map. That is only done on successful remap.

Processing '-TRYAGAIN'

If the cluster is in a state of flux, a node may return '-TRYAGAIN' to indicated that we should attempt the operation again. The cluster spec says we should attempt the operation after some time. This time is configurable.

Definition in file cluster.c.

Data Structure Documentation

◆ cluster_nodes_live_t

struct cluster_nodes_live_t

Live nodes data, used to perform weighted random selection of alternative nodes.

Definition at line 198 of file cluster.c.

Data Fields
uint8_t	next	Next index in live.
struct cluster_nodes_live_t.node	node[UINT8_MAX - 1]	Array of live node IDs (and weights).
uint8_t	skip

◆ cluster_nodes_live_t.node

struct cluster_nodes_live_t.node

Definition at line 199 of file cluster.c.

Data Fields
unsigned int	cumulative	Cumulative weight.
uint8_t	id	Node ID.
fr_pool_state_t const *	pool_state	Connection pool stats.

◆ fr_redis_cluster

struct fr_redis_cluster

A redis cluster.

Holds all the structures and collections of nodes, to represent a Redis cluster.

Definition at line 250 of file cluster.c.

Collaboration diagram for fr_redis_cluster:

Data Fields
CONF_SECTION fr_redis_conf_t	conf	< Module configuration. Base configuration data such as the database number and passwords.
fr_fifo_t *	free_nodes	Queue of free nodes (or nodes waiting to be reused).
fr_redis_cluster_key_slot_t	key_slot[KEY_SLOTS]	Lookup table of slots to pools.
fr_redis_cluster_key_slot_t	key_slot_pending[KEY_SLOTS]	Pending key slot table.
fr_time_t	last_updated	Last time the cluster mappings were updated.
char const *	log_prefix	What to prepend to log messages.
pthread_mutex_t	mutex	Mutex to synchronise cluster operations.
fr_redis_cluster_node_t *	node	Structure containing a node id, its address and a pool of its connections.
bool	remap_needed	Set true if at least one cluster node is definitely unreachable. Set false on successful remap.
bool	remapping	True when cluster is being remapped.
fr_pair_list_t	trigger_args	Arguments to pass to triggers.
char const *	trigger_prefix	Trigger path.
bool	triggers_enabled	Whether triggers are enabled.
fr_rb_tree_t *	used_nodes	Tree of used nodes.

◆ fr_redis_cluster_key_slot_s

struct fr_redis_cluster_key_slot_s

Indexes in the fr_redis_cluster_node_t array for a single key slot.

When dealing with 16K entries, space is a concern. It's significantly more memory efficient to use 8bit indexes than 64bit pointers for each of the key slot to node mappings.

Definition at line 240 of file cluster.c.

Data Fields
uint8_t	master	R/W node (master) for this key slot.
uint8_t	slave[MAX_SLAVES]	R/O node (slave) for this key slot.
uint8_t	slave_num	Number of slaves associated with this key slot.

◆ fr_redis_cluster_node_s

struct fr_redis_cluster_node_s

A Redis cluster node.

Passed as opaque data to pools which open connection to nodes.

Definition at line 212 of file cluster.c.

Collaboration diagram for fr_redis_cluster_node_s:

Data Fields
fr_socket_t	addr	Current node address.
fr_redis_cluster_t *	cluster	Common configuration (database number, password, etc..).
uint8_t	id	Node ID (index in node array).
bool	is_active	Whether this node is in the active node set.
bool	is_master	Whether this node is a master. This is needed for commands like 'KEYS', which we need to issue to every master in the cluster.
char	name[INET6_ADDRSTRLEN]	Buffer to hold IP string. text for debug messages.
fr_socket_t	pending_addr	New node address to be applied when the pool is reconnected.
fr_pool_t *	pool	Pool associated with this node.
CONF_SECTION *	pool_cs	Pool configuration section associated with node.
fr_rb_node_t	rbnode	Entry into the tree of redis nodes.

Macro Definition Documentation

◆ CLOSED_PERIOD

#define CLOSED_PERIOD 10000

How recently must the closed have occurred for us to care.

Definition at line 179 of file cluster.c.

◆ CLOSED_WEIGHT

#define CLOSED_WEIGHT 1

What weight to give to nodes that had a connection closed recently.

Definition at line 182 of file cluster.c.

◆ FAILED_PERIOD

#define FAILED_PERIOD 10000

How recently must the spawn failure occurred for us to care.

Definition at line 185 of file cluster.c.

◆ FAILED_WEIGHT

#define FAILED_WEIGHT 1

What weight to give to nodes that had a spawn failure recently.

Definition at line 188 of file cluster.c.

◆ KEY_SLOTS

#define KEY_SLOTS 16384

Maximum number of keyslots (should not change).

Definition at line 170 of file cluster.c.

◆ MAX_SLAVES

#define MAX_SLAVES 5

Maximum number of slaves associated with a keyslot.

Definition at line 173 of file cluster.c.

◆ RELEASED_MIN_WEIGHT

#define RELEASED_MIN_WEIGHT 1000

Minimum weight to assign to node.

Definition at line 194 of file cluster.c.

◆ RELEASED_PERIOD

#define RELEASED_PERIOD 10000

Period after which we don't care about when the last connection was released.

Definition at line 192 of file cluster.c.

◆ SET_ACTIVE

#define SET_ACTIVE ( _node )

Value:

do { \
        (_node)->is_active = true; \
        fr_rb_insert(cluster->used_nodes, _node); \
        fr_fifo_pop(cluster->free_nodes); \
        active[(_node)->id] = true; \
        rollback[r++] = (_node)->id; \
} while (0)

◆ SET_ADDR

#define SET_ADDR	(	_addr,
		_map
	)

Value:

do { \
        int _ret; \
        _ret = fr_inet_pton(&_addr.inet.dst_ipaddr, _map->element[0]->str, _map->element[0]->len, AF_UNSPEC, true, true);\
        fr_assert(_ret == 0);\
        _addr.inet.dst_port = _map->element[1]->integer; \
} while (0)

◆ SET_INACTIVE

#define SET_INACTIVE ( _node )

Value:

do { \
        (_node)->is_active = false; \
        (_node)->is_master = false; \
        fr_rb_delete(cluster->used_nodes, _node); \
        fr_fifo_push(cluster->free_nodes, _node); \
} while (0)

Function Documentation

◆ _cluster_conn_free()

static int _cluster_conn_free ( fr_redis_conn_t * conn )

static

Callback for freeing a Redis connection.

Parameters

[in] conn to free.

Returns: 0.

Definition at line 1451 of file cluster.c.

Here is the caller graph for this function:

◆ _cluster_node_cmp()

static int8_t _cluster_node_cmp	(	void const *	one,
		void const *	two
	)

static

Compare two redis nodes to check equality.

Parameters

[in]	one	first node.
[in]	two	second node.

Returns: CMP(one, two)

Definition at line 322 of file cluster.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _cluster_node_conf_apply()

static void _cluster_node_conf_apply	(	fr_pool_t *	pool,
		void *	opaque
	)

static

Reconnect callback to apply new pool config.

Parameters

[in]	pool	to apply new configuration to.
[in]	opaque	data passed to the connection pool.

Definition at line 339 of file cluster.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _fr_redis_cluster_free()

static int _fr_redis_cluster_free ( fr_redis_cluster_t * cluster )

static

Destroy mutex associated with cluster slots structure.

Parameters

cluster being freed.

Returns: 0

Definition at line 2184 of file cluster.c.

Here is the caller graph for this function:

◆ cluster_key_hash()

static uint16_t cluster_key_hash	(	uint8_t const *	key,
		size_t	key_len
	)

static

Resolve key to key slot.

Identical to the example implementation, except it uses memchr which will be faster, and isn't so needlessly complex.

Parameters

[in]	key	to resolve.
[in]	key_len	length of key.

Returns: key slot index for the key.

Definition at line 298 of file cluster.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ cluster_map_apply()

static fr_redis_cluster_rcode_t cluster_map_apply	(	fr_redis_cluster_t *	cluster,
		redisReply *	reply
	)

static

Apply a cluster map received from a cluster node.

Note: Errors may be retrieved with fr_strerror().; Must be called with the cluster mutex held.

Key slot range structure

  [0] -> key slot range 0
      [0] -> key_slot_start
      [1] -> key_slot_end
      [2] -> master_node
          [0] -> master 0 ip (string)
          [1] -> master 0 port (number)
      [3..n] -> slave_node(s)
  [1] -> key slot range 1)
      [0]  -> key_slot_start
      [1] -> key_slot_end
      [2] -> master_node
          [0] -> master 1 ip (string)
          [1] -> master 1 port (number)
      [3..n] -> slave_node(s)
  [n] -> key slot range n
      [0] -> key_slot_start
      [1] -> key_slot_end
      [2] -> master_node
          [0] -> master n ip (string)
          [1] -> master n port (number)
      [3..n] -> slave_node(s)

Parameters

[in,out]	cluster	to apply map to.
[in]	reply	from cluster_map_get.

Returns

FR_REDIS_CLUSTER_RCODE_SUCCESS on success.
FR_REDIS_CLUSTER_RCODE_FAILED on failure.
FR_REDIS_CLUSTER_RCODE_NO_CONNECTION connection failure.
FR_REDIS_CLUSTER_RCODE_BAD_INPUT if the map didn't provide nodes for all keyslots.

Definition at line 537 of file cluster.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ cluster_map_get()

static fr_redis_cluster_rcode_t cluster_map_get	(	redisReply **	out,
		fr_redis_conn_t *	conn
	)

static

Learn a new cluster layout by querying the node that issued the -MOVE.

Also validates the response from the Redis cluster, so we can be sure that it's well formed, before doing more expensive operations.

Note: Errors may be retrieved with fr_strerror().

Parameters

[out]	out	Where to write cluster map.
[in]	conn	to use for learning the new cluster map.

Returns

FR_REDIS_CLUSTER_RCODE_IGNORED if 'cluster slots' returned an error (indicating clustering not supported).
FR_REDIS_CLUSTER_RCODE_SUCCESS on success.
FR_REDIS_CLUSTER_RCODE_FAILED if issuing the command resulted in an error.
FR_REDIS_CLUSTER_RCODE_NO_CONNECTION connection failure.
FR_REDIS_CLUSTER_RCODE_BAD_INPUT on validation failure (bad data returned from Redis).

Definition at line 863 of file cluster.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ cluster_map_node_validate()

static int cluster_map_node_validate	(	redisReply *	node,
		int	map_idx,
		int	node_idx
	)

static

Validate a cluster map node entry.

Note: Errors may be retrieved with fr_strerror().; In a separate function, as it's called for both master and slave nodes.

Parameters

[in]	node	we're validating.
[in]	map_idx	we're processing.
[in]	node_idx	we're processing.

Returns

FR_REDIS_CLUSTER_RCODE_SUCCESS on success.
FR_REDIS_CLUSTER_RCODE_BAD_INPUT on validation failure (bad data returned from Redis).

Definition at line 785 of file cluster.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ cluster_node_conf_from_redirect()

static fr_redis_cluster_rcode_t cluster_node_conf_from_redirect	(	uint16_t *	key_slot,
		fr_socket_t *	node_addr,
		redisReply *	redirect
	)

static

Parse a -MOVED or -ASK redirect.

Converts the body of the -MOVED or -ASK error into an IPv4/6 address and port.

Parameters

[out]	key_slot	value extracted from redirect string (may be NULL).
[out]	node_addr	Redis node ipaddr and port extracted from redirect string.
[in]	redirect	to process.

Returns

FR_REDIS_CLUSTER_RCODE_SUCCESS on success.
FR_REDIS_CLUSTER_RCODE_BAD_INPUT if the server returned an invalid redirect.

Definition at line 446 of file cluster.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ cluster_node_connect()

static fr_redis_cluster_rcode_t cluster_node_connect	(	fr_redis_cluster_t *	cluster,
		fr_redis_cluster_node_t *	node
	)

static

Establish a connection to a cluster node.

Note: Must be called with the cluster mutex locked.; Configuration to use for the connection must be set in node->pending_addr, not node->cluster->conf.

Parameters

[in]	cluster	to search in.
[in]	node	config.

Returns

FR_REDIS_CLUSTER_RCODE_SUCCESS on success.
FR_REDIS_CLUSTER_RCODE_FAILED if the operation failed.

Definition at line 371 of file cluster.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ cluster_node_find_live()

static int cluster_node_find_live	(	fr_redis_cluster_node_t **	live_node,
		fr_redis_conn_t **	live_conn,
		request_t *	request,
		fr_redis_cluster_t *	cluster,
		fr_redis_cluster_node_t *	skip
	)

static

Attempt to find a live pool in the cluster.

The intent here is to find pools/nodes where a connection was released the shortest time ago. Having a connection be released (vs closed) indicates that the pool is live.

We don't want to have all workers try and grab a connection to this node however, as it may still be dead (we don't know).

So we use an inverse transform sample, to weight the nodes, based on time between now and when the connection was released. Connections released closest to the current time are given a higher weighting.

Weight range is between 1 - 11,000.

If released > 10.0 seconds ago,information is not valid, weight 500.
If closed < 10.0 seconds ago, it's a bad pool, weight 1.
If spawn failed < 10.0 seconds ago, it's a bad pool, weight 1.
If a connection was released 0.0 seconds ago, weight 11,000.
If a connection was released 10.0 seconds ago, weight 1000.

Using the above algorithm we use the experience of other workers using the cluster to inform our alternative node selection.

Suggestions on improving live node selection appreciated.

Inverse transform sampling based roughly on the solution from this post: http://stackoverflow.com/questions/17250568/randomly-choosing-from-a-list-with-weighted-probabilities

Wikipedia page here: https://en.wikipedia.org/wiki/Inverse_transform_sampling

Note: Must be called with the cluster mutex free.

Parameters

[out]	live_node	we found.
[out]	live_conn	to that node.
[in]	request	The current request (used for logging).
[in]	cluster	to search for live pools in.
[in]	skip	this node (it's bad).

Returns: 0 (iterates over the whole tree).

Definition at line 1310 of file cluster.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ cluster_node_ping()

static fr_redis_cluster_rcode_t cluster_node_ping	(	request_t *	request,
		fr_redis_cluster_node_t *	node,
		fr_redis_conn_t *	conn
	)

static

Issue a ping request against a cluster node.

Establishes whether the connection to the node we have is live.

Parameters

request	The current request.
node	to ping.
conn	the connection to ping on.

Returns

FR_REDIS_CLUSTER_RCODE_BAD_INPUT if we got a bad response.
FR_REDIS_CLUSTER_RCODE_SUCCESS on success.
FR_REDIS_CLUSTER_RCODE_NO_CONNECTION on connection down.

Definition at line 1243 of file cluster.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ cluster_node_pool_health()

static int cluster_node_pool_health	(	fr_time_t	now,
		fr_pool_state_t const *	state
	)

static

Try to determine the health of a cluster node passively by examining its pool state.

Returns an integer value representing the likelihood that the pool is live. Range is between 1 and 11,000.

If a weight of 1 is returned, connections from the pool should be checked (by pinging) before use.

Parameters

now	The current time.
state	of the connection pool.

Returns

1 the pool is very likely to be bad.
2-11000 the pool is likely to be good, with a higher number indicating higher probability of liveness.

Definition at line 1208 of file cluster.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ cluster_redirect()

static fr_redis_cluster_rcode_t cluster_redirect	(	fr_redis_cluster_node_t **	out,
		fr_redis_cluster_t *	cluster,
		redisReply *	reply
	)

static

Retrieve or associate a node with the server indicated in the redirect.

Note: Errors may be retrieved with fr_strerror().

Parameters

[out]	out	Where to write the node representing the redirect server.
[in]	cluster	to draw node from.
[in]	reply	Redis reply containing the redirect information.

Returns

FR_REDIS_CLUSTER_RCODE_SUCCESS on success.
FR_REDIS_CLUSTER_RCODE_FAILED no more nodes available.
FR_REDIS_CLUSTER_RCODE_NO_CONNECTION connection failure.
FR_REDIS_CLUSTER_RCODE_BAD_INPUT on validation failure (bad data returned from Redis).

Definition at line 1118 of file cluster.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fr_redis_cluster_alloc()

fr_redis_cluster_t * fr_redis_cluster_alloc	(	TALLOC_CTX *	ctx,
		CONF_SECTION *	module,
		fr_redis_conf_t *	conf,
		char const *	log_prefix,
		char const *	trigger_prefix,
		fr_pair_list_t *	trigger_args
	)

Allocate and initialise a new cluster structure.

This holds all the data necessary to manage a pool of pools for a specific redis cluster.

Note: Will not error out unless cs.pool.start > 0. This is consistent with other pool based modules/code.

Parameters

ctx	to link the lifetime of the cluster structure to.
module	Configuration section to search for 'server' conf pairs in.
conf	Base redis server configuration. Cluster nodes share database number and password.
log_prefix	Custom log prefix. Defaults to rlm_<module> (<instance>) .
trigger_prefix	Custom trigger prefix. Defaults to modules.<module>.pool .
trigger_args	Argument pairs to pass to the trigger in addition to Connection-Pool-Server, and Connection-Pool-Port (which are always set by the cluster code).

Returns

New fr_redis_cluster_t on success.
NULL on error.

Definition at line 2259 of file cluster.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fr_redis_cluster_conn_create()

void * fr_redis_cluster_conn_create	(	TALLOC_CTX *	ctx,
		void *	instance,
		fr_time_delta_t	timeout
	)

Create a new connection to a Redis node.

Parameters

[in]	ctx	to allocate connection structure in. Will be freed at the same time as the pool.
[in]	instance	data of type fr_redis_cluster_node_t. Holds parameters for establishing new connection.
[in]	timeout	The maximum time allowed to complete the connection.

Returns

New fr_redis_conn_t on success.
NULL on failure.

Definition at line 1467 of file cluster.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fr_redis_cluster_ipaddr()

int fr_redis_cluster_ipaddr	(	fr_ipaddr_t *	out,
		fr_redis_cluster_node_t const *	node
	)

Return the ipaddr of a particular node.

Parameters

[out]	out	Ipaddr of the node.
[in]	node	to get ip address from.

Returns

0 on success.
-1 on failure (node is NULL).

Definition at line 1671 of file cluster.c.

Here is the caller graph for this function:

◆ fr_redis_cluster_master()

fr_redis_cluster_node_t const * fr_redis_cluster_master	(	fr_redis_cluster_t *	cluster,
		fr_redis_cluster_key_slot_t const *	key_slot
	)

Return the master node that would be used for a particular key.

Parameters

[in]	cluster	To resolve key in.
[in]	key_slot	to resolve to node.

Returns

The current master node.
NULL if no master node is currently assigned to a particular key slot.

Definition at line 1638 of file cluster.c.

Here is the caller graph for this function:

◆ fr_redis_cluster_min_version()

bool fr_redis_cluster_min_version	(	fr_redis_cluster_t *	cluster,
		char const *	min_version
	)

Check if members of the cluster are above a certain version.

Parameters

cluster	to perform check on.
min_version	that must be found on each node for the check to succeed. Must be in the format <major>.<minor>.<release> .

Returns

true if all contactable members are above min_version.
false if at least one member if not above minimum version (use fr_strerror to retrieve node information).

Definition at line 2201 of file cluster.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fr_redis_cluster_node_addr_by_role()

ssize_t fr_redis_cluster_node_addr_by_role	(	TALLOC_CTX *	ctx,
		fr_socket_t *	out[],
		fr_redis_cluster_t *	cluster,
		bool	is_master,
		bool	is_slave
	)

Return an array of IP addresses belonging to masters or slaves.

Note: We return IP addresses as they're safe to use across cluster remaps.; Result array must be freed (talloc_free()) after use.

Parameters

[in]	ctx	to allocate array of IP addresses in.
[out]	out	Where to write the addresses of the nodes.
[in]	cluster	to search for nodes in.
[in]	is_master	If true, include the addresses of all the master nodes.
[in]	is_slave	If true, include the addresses of all the slaves nodes.

Returns: the number of ip addresses written to out.

Definition at line 2137 of file cluster.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fr_redis_cluster_pool_by_node_addr()

int fr_redis_cluster_pool_by_node_addr	(	fr_pool_t **	pool,
		fr_redis_cluster_t *	cluster,
		fr_socket_t *	node_addr,
		bool	create
	)

Get the pool associated with a node in the cluster.

Note: This is used for testing only. It's not ifdef'd out because tests need to run against production builds too.

Parameters

[out]	pool	associated with the node.
[in]	cluster	to search for node in.
[in]	node_addr	to retrieve pool for. Specifies IP and port of node.
[in]	create	Establish a connection to the specified node if it was previously unknown to the cluster client.

Returns

0 on success.
-1 if no such node exists.

Definition at line 2069 of file cluster.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fr_redis_cluster_port()

int fr_redis_cluster_port	(	uint16_t *	out,
		fr_redis_cluster_node_t const *	node
	)

Return the port of a particular node.

Parameters

[out]	out	Port of the node.
[in]	node	to get ip address from.

Returns

0 on success.
-1 on failure (node is NULL).

Definition at line 1688 of file cluster.c.

Here is the caller graph for this function:

◆ fr_redis_cluster_remap()

fr_redis_cluster_rcode_t fr_redis_cluster_remap	(	request_t *	request,
		fr_redis_cluster_t *	cluster,
		fr_redis_conn_t *	conn
	)

Perform a runtime remap of the cluster.

Note: Errors may be retrieved with fr_strerror().; Must be called with the cluster mutex free.

Parameters

[in]	request	The current request.
[in,out]	cluster	to remap.
[in]	conn	to use to query the cluster.

Returns

FR_REDIS_CLUSTER_RCODE_IGNORED if 'cluster slots' returned an error (indicating clustering not supported).
FR_REDIS_CLUSTER_RCODE_SUCCESS on success.
FR_REDIS_CLUSTER_RCODE_FAILED if issuing the 'cluster slots' command resulted in a protocol error.
FR_REDIS_CLUSTER_RCODE_NO_CONNECTION connection failure.
FR_REDIS_CLUSTER_RCODE_BAD_INPUT on validation failure (bad data returned from Redis).

Definition at line 1008 of file cluster.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fr_redis_cluster_slave()

fr_redis_cluster_node_t const * fr_redis_cluster_slave	(	fr_redis_cluster_t *	cluster,
		fr_redis_cluster_key_slot_t const *	key_slot,
		uint8_t	slave_num
	)

Return the slave node that would be used for a particular key.

Parameters

[in]	cluster	To resolve key in.
[in]	key_slot	To resolve to node.
[in]	slave_num	0..n.

Returns

A slave node.
NULL if no slave node is assigned, or is at the specific key slot.

Definition at line 1654 of file cluster.c.

Here is the caller graph for this function:

◆ fr_redis_cluster_slot_by_key()

fr_redis_cluster_key_slot_t const * fr_redis_cluster_slot_by_key	(	fr_redis_cluster_t *	cluster,
		request_t *	request,
		uint8_t const *	key,
		size_t	key_len
	)

Implements the key slot selection scheme used by freeradius.

Like the scheme in the clustering specification but with some differences if the key is NULL or zero length, then a random keyslot is chosen.

If there's only a single node in the cluster, then we avoid the CRC16 and just use key slot 0.

Parameters

cluster	to determine key slot for.
request	The current request.
key	the key to resolve.
key_len	the length of the key.

Returns: pointer to key slot key resolves to.

Definition at line 1602 of file cluster.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fr_redis_cluster_state_init()

fr_redis_rcode_t fr_redis_cluster_state_init	(	fr_redis_cluster_state_t *	state,
		fr_redis_conn_t **	conn,
		fr_redis_cluster_t *	cluster,
		request_t *	request,
		uint8_t const *	key,
		size_t	key_len,
		bool	read_only
	)

Resolve a key to a pool, and reserve a connection in that pool.

This should be used with fr_redis_cluster_state_next, and fr_redis_command_status, to transparently locate the cluster node we need to perform the operation on.

Example code below shows how this function is used in conjunction with fr_redis_cluster_state_next to follow redirects, and reconnect handles.

int                 s_ret;
redis_conn_state     state;
fr_redis_conn_t      *conn;
redisReply           *reply;
fr_redis_rcode_t     status;
 
for (s_ret = fr_redis_cluster_state_init(&state, &conn, cluster, key, key_len, false);
     s_ret == REDIS_RCODE_TRY_AGAIN,
     s_ret = fr_redis_cluster_state_next(&state, &conn, cluster, request, status, &reply)) {
        reply = redisCommand(conn->handle, "SET foo bar");
        status = fr_redis_command_status(conn, reply);
}
// Reply is freed if ret == REDIS_RCODE_TRY_AGAIN, but left in all other cases to allow error
// processing, or extraction of results.
fr_redis_reply_free(&reply);
if (s_ret != REDIS_RCODE_SUCCESS) {
     // Error
}
// Success

Parameters

[out]	state	to track current pool and various counters, will be initialised.
[out]	conn	Where to write the reserved connection to.
[in]	cluster	of pools.
[in]	request	The current request.
[in]	key	to resolve to a cluster node/pool. If no key is NULL or key_len is 0 a random slot will be chosen.
[in]	key_len	Length of the key.
[in]	read_only	If true, will use random slave pool in preference to the master, falling back to the master if no slaves are available.

Returns

REDIS_RCODE_TRY_AGAIN - try your command with this connection (provided via command).
REDIS_RCODE_RECONNECT - when no additional connections available.

Definition at line 1740 of file cluster.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fr_redis_cluster_state_next()

fr_redis_rcode_t fr_redis_cluster_state_next	(	fr_redis_cluster_state_t *	state,
		fr_redis_conn_t **	conn,
		fr_redis_cluster_t *	cluster,
		request_t *	request,
		fr_redis_rcode_t	status,
		redisReply **	reply
	)

Get the next connection to attempt a command against.

Will process reconnect and redirect states performing the actions necessary.

May trigger a cluster remap on receiving a REDIS_RCODE_MOVE status.
May perform a temporary redirect on receiving a REDIS_RCODE_ASK status.
May reserve a new connection on receiving a REDIS_RCODE_RECONNECT status.

If a remap is in progress, has occurred within the last second, has recently failed, or fails, the '-MOVE' will be treated as a temporary redirect (-ASK).

This allows the server to be more responsive during remaps, as unless the worker has been redirected to a node we don't currently have a pool for, it can grab a connection for the node it was redirected to, and continue.

Note: Irrespective of return code, the connection passed via conn will be released, A new connection to attempt command on will be provided via conn.; reply will be automatically freed and set to NULL if a new connection is provided in all other cases, the caller is responsible for freeing the reply.

Parameters

[in,out]	state	containing the current pool, and various counters which control retries, and limit redirects.
[in,out]	conn	we received the '-ASK' or '-MOVE' redirect on. Will be replaced with a connection in the new pool the key points to.
[in]	request	The current request.
[in]	cluster	of pools.
[in]	status	of the last command, must be REDIS_RCODE_MOVE or REDIS_RCODE_ASK.
[in]	reply	from last command. Freed if 0 is returned, else caller must free.

Returns

REDIS_RCODE_SUCCESS - on success.
REDIS_RCODE_TRY_AGAIN - try new connection (provided via conn). Will free reply.
REDIS_RCODE_ERROR - on failure or command error.
REDIS_RCODE_RECONNECT - when no additional connections available.

Definition at line 1862 of file cluster.c.

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

◆ fr_redis_cluster_rcodes_table

fr_table_num_sorted_t const fr_redis_cluster_rcodes_table[]

Initial value:

= {
        { L("bad-input"),       FR_REDIS_CLUSTER_RCODE_BAD_INPUT        },
        { L("failed"),          FR_REDIS_CLUSTER_RCODE_FAILED           },
        { L("ignored"),         FR_REDIS_CLUSTER_RCODE_IGNORED          },
        { L("no-connection"),   FR_REDIS_CLUSTER_RCODE_NO_CONNECTION    },
        { L("success"),         FR_REDIS_CLUSTER_RCODE_SUCCESS          }
}

Definition at line 280 of file cluster.c.

◆ fr_redis_cluster_rcodes_table_len

size_t fr_redis_cluster_rcodes_table_len = NUM_ELEMENTS(fr_redis_cluster_rcodes_table)

Definition at line 287 of file cluster.c.

Data Structures

Macros

Functions

Variables

Detailed Description

Overview

Using the cluster's public API

Structures

Mapping/Remapping the cluster

Processing '-ASK' and '-MOVE' redirects

Processing '-TRYAGAIN'

Data Structure Documentation

◆ cluster_nodes_live_t

◆ cluster_nodes_live_t.node

◆ fr_redis_cluster

◆ fr_redis_cluster_key_slot_s

◆ fr_redis_cluster_node_s

Macro Definition Documentation

◆ CLOSED_PERIOD

◆ CLOSED_WEIGHT

◆ FAILED_PERIOD

◆ FAILED_WEIGHT

◆ KEY_SLOTS

◆ MAX_SLAVES

◆ RELEASED_MIN_WEIGHT

◆ RELEASED_PERIOD

◆ SET_ACTIVE

◆ SET_ADDR

◆ SET_INACTIVE

Function Documentation

◆ _cluster_conn_free()

◆ _cluster_node_cmp()

◆ _cluster_node_conf_apply()

◆ _fr_redis_cluster_free()

◆ cluster_key_hash()

◆ cluster_map_apply()

◆ cluster_map_get()

◆ cluster_map_node_validate()

◆ cluster_node_conf_from_redirect()

◆ cluster_node_connect()

◆ cluster_node_find_live()

◆ cluster_node_ping()

◆ cluster_node_pool_health()

◆ cluster_redirect()

◆ fr_redis_cluster_alloc()

◆ fr_redis_cluster_conn_create()

◆ fr_redis_cluster_ipaddr()

◆ fr_redis_cluster_master()

◆ fr_redis_cluster_min_version()

◆ fr_redis_cluster_node_addr_by_role()

◆ fr_redis_cluster_pool_by_node_addr()

◆ fr_redis_cluster_port()

◆ fr_redis_cluster_remap()

◆ fr_redis_cluster_slave()

◆ fr_redis_cluster_slot_by_key()

◆ fr_redis_cluster_state_init()

◆ fr_redis_cluster_state_next()

Variable Documentation

◆ fr_redis_cluster_rcodes_table

◆ fr_redis_cluster_rcodes_table_len