![]() |
The FreeRADIUS server $Id: f3670dba8951ca10eb4948feb3dc3db9423a334f $
|
conf functions for interacting asynchronously with Redis cluster via Hiredis. More...
#include <freeradius-devel/util/debug.h>#include "config.h"#include "attrs.h"#include "cluster_async.h"#include "crc16.h"
Include dependency graph for cluster_async.c:Go to the source code of this file.
Data Structures | |
| struct | fr_redis_async_cmd_s |
| Structure for holding the state of an async redis command set. More... | |
| struct | fr_redis_ct_key_slot_s |
| struct | fr_redis_ct_node_s |
| struct | fr_redis_ct_pend_req_t |
| Structure to record that a request is waiting for the cluster map. More... | |
| struct | fr_redis_ct_s |
| Thread local state for a cluster. More... | |
Macros | |
| #define | CONFIGURE_NODE(_node, _addr, _port) |
| #define | MAX_REPLICAS 5 |
| Maximum number of replicas associated with a keyslot. | |
| #define | SET_ACTIVE(_node) |
| #define | SET_INACTIVE(_node) |
Enumerations | |
| enum | fr_redis_ct_state_t { CLUSTER_INIT = 0 , CLUSTER_MAP_FETCHING , CLUSTER_READY , CLUSTER_FAIL } |
Functions | |
| static fr_cmp_ret_t | _cluster_thread_node_cmp (void const *one, void const *two) |
| Compare two redis nodes to check equality. | |
| static int | _fr_redis_async_cmd_free (fr_redis_async_cmd_t *cmd) |
| static int | _fr_redis_ct_pend_req_free (fr_redis_ct_pend_req_t *pend_req) |
| Ensure pending request is removed from the list on freeing. | |
| static uint16_t | cluster_key_hash (uint8_t const *key, size_t key_len) |
| Resolve key to key slot index. | |
| void | fr_redis_async_cmd_cancel (fr_redis_async_cmd_t *cmd) |
| Cancel a Redis async command. | |
| static fr_redis_async_rcode_t | fr_redis_async_cmd_enqueue (fr_redis_async_cmd_t *cmd) |
| Enqueue a command set on a node identified by the key. | |
| fr_redis_ct_node_t * | fr_redis_async_cmd_node (fr_redis_async_cmd_t *cmd) |
| Fetch the cluster node a command was last sent to. | |
| fr_redis_async_rcode_t | fr_redis_async_cmd_redirect (fr_redis_async_cmd_t *cmd) |
| Re-submit a redis async command set on a different node. | |
| fr_redis_async_rcode_t | fr_redis_async_cmd_resend (fr_redis_async_cmd_t *cmd) |
| Re-submit a redis async command set. | |
| fr_redis_async_cmd_t * | fr_redis_async_cmd_start (TALLOC_CTX *ctx, request_t *request, fr_redis_async_rcode_t *rcode, fr_redis_ct_t *rtcluster, uint8_t const *key, size_t key_len, fr_redis_command_set_t *cmds, bool read_only, fr_redis_ct_node_t *node) |
| Start running a command set on an async redis cluster. | |
| fr_redis_trunk_t * | fr_redis_async_cmd_trunk (fr_redis_async_cmd_t *cmd) |
| Fetch the redis trunk a command is associated with. | |
| fr_redis_ct_t * | fr_redis_ct_alloc (TALLOC_CTX *ctx, CONF_SECTION *tls_cs, fr_event_list_t *el, fr_redis_conf_t *conf, fr_redis_trunk_active_t active, void *active_uctx, bool active_oneshot) |
| Allocate per-thread, per-cluster instance. | |
| fr_event_list_t * | fr_redis_ct_el (fr_redis_ct_t *rtcluster) |
| char const * | fr_redis_ct_ipaddr (fr_redis_ct_node_t const *node) |
| Return the ipaddr of a particular node. | |
| int | fr_redis_ct_map_bootstrap (fr_redis_ct_t *rtcluster, fr_coord_worker_t *cw, fr_coord_pair_reg_t *coord_pair_reg) |
| Initiate bootstrapping of the cluster map. | |
| int | fr_redis_ct_map_fail (fr_redis_ct_t *rtcluster, UNUSED fr_pair_list_t const *list) |
| Process a cluster map fail message from the coordinator. | |
| fr_redis_async_rcode_t | fr_redis_ct_map_get (fr_redis_ct_t *rtcluster, fr_coord_worker_t *cw, fr_coord_pair_reg_t *coord_pair_reg, bool force) |
| Initiate updating of the cluster map. | |
| int | fr_redis_ct_map_update (fr_redis_ct_t *rtcluster, fr_pair_list_t const *list) |
| Update a Redis cluster map from a pair list returned from a coordinator. | |
| fr_redis_ct_node_t const * | fr_redis_ct_master (fr_redis_ct_t *rtcluster, fr_redis_ct_key_slot_t const *key_slot) |
| Return the master node that would be used for a particular key slot. | |
| fr_redis_async_rcode_t | fr_redis_ct_node_addr_by_role (TALLOC_CTX *ctx, fr_redis_io_conf_t *out[], uint8_t *count_out, fr_redis_ct_t *rtcluster, bool is_master, bool is_replica) |
| fr_redis_ct_node_t * | fr_redis_ct_node_by_addr (fr_redis_ct_t *rtcluster, fr_redis_io_conf_t *ioconf) |
| int | fr_redis_ct_port (uint16_t *out, fr_redis_ct_node_t const *node) |
| Return the port of a particular node. | |
| fr_redis_ct_node_t const * | fr_redis_ct_replica (fr_redis_ct_t *rtcluster, fr_redis_ct_key_slot_t const *key_slot, uint8_t replica_num) |
| Return the replica node that would be used for a particular key slot. | |
| void | fr_redis_ct_request_yield (TALLOC_CTX *ctx, fr_redis_ct_t *rtcluster, request_t *request) |
| Add a request to the list of those waiting for the cluster map. | |
| fr_redis_ct_key_slot_t const * | fr_redis_ct_slot_by_key (fr_redis_ct_t *rtcluster, request_t *request, uint8_t const *key, size_t key_len) |
| Resolve key to key slot. | |
| trunk_conf_t const * | fr_redis_ct_trunk_conf (fr_redis_ct_t *rtcluster) |
conf functions for interacting asynchronously with Redis cluster via Hiredis.
Read and understand this http://redis.io/topics/cluster-spec first, else the text below will not be useful.
The cluster requires use of a coordinator to fetch the cluster map.
Any module using a Redis cluster should register a coordinator which uses the redis virtual server.
In mod_coord_attach, fr_redis_ct_map_bootstrap can be used to initiate the loading of the cluster map. Typically this should not be called if the pool start is set to zero. In that case, the first attempt to enqueue a command set will indicate that the cluster map need to be bootstrapped.
At runtime the function fr_redis_async_cmd_start is used to enqueue a set of commands, and the statis it returns should be checked with the macro REDIS_ASYNC_START_RCODE_PROCESS to initiate the cluster bootstrap or get an updated map if needed.
With calling Redis using it's async API, the majority of results processing has to be done in a callback called by hiredis - the redisReply structure is freed after the callback is called.
This callback is associated with the individual commands in a Redis command set as they are added to the command set with the fr_redis_command_*_add functions.
This code maintains a series structures for efficient lookup and lockless operations.
The important ones are:
Each fr_redis_cluster_node_t contains a master ID, and an array of slave IDs. The IDs are array indexes in the fr_redis_cluster_t.node array. We use 8bit unsigned integers instead of pointers to save space. Using pointers, the node[] array would need 784K, using IDs it uses 112K. Still not light on memory, but a bit more acceptable. Currently the key_slot array is shadowed by key_slot_pending, used to stage new key_slot mappings. This doubles the memory used. We may want to consider allocating key_slot_pending only during re-mappings and freeing it after.
On startup, and during cluster operation, a remap may be performed. A remap involves the following steps:
cluster_state response and which nodes have the most up to date representation of the cluster, from the cluster_current_epoch response. c. Nodes reporting the cluster is OK are issued the Redis 'cluster slots' or 'cluster shards' command depending on the Redis server version. d. Validating the result of this command. We need to do extensive validation to avoid SEGV on invalid data, due to the way libhiredis presents the result. e. Return the cluster map to the workers.fr_redis_ct_map_get is used to request an updated map from the coordinator and fr_redis_ct_map_update is used to process the message received from the coordinator to update the thread local copy of the cluster map.
The cluster client can continue to operate, albeit inefficiently, with a stale cluster map by following '-ASK' and '-MOVE' redirects.
Remaps are limited to one per second. If any operation sets the remap_needed flag, or attempts a remap directly, the remap may be skipped if one occurred recently.
Resume functions which are run after an async Redis command set has completed should fetch the rcode with fr_redis_command_set_rcode. If the rcode indicates MOVE or ASK, then fr_redis_async_cmd_redirect should be used to re-enqueue the command set on the indicated node. In addition, if the response was MOVE, then fr_redis_ct_map_get should be used to initiate a refresh of the cluster map.
The data from '-MOVE' responses, is not used to alter the cluster map. That is only done on successful remap.
If the cluster is in a state of flux, a node may return '-TRYAGAIN' to indicated that we should attempt the operation again. fr_redis_async_cmd_resend can be used to re-enqueue the command set.
Definition in file cluster_async.c.
| struct fr_redis_async_cmd_s |
Structure for holding the state of an async redis command set.
Definition at line 218 of file cluster_async.c.
Collaboration diagram for fr_redis_async_cmd_s:| Data Fields | ||
|---|---|---|
| fr_redis_command_set_t * | cmds | Command set to run. |
| fr_dlist_t | entry | Entry in the list of commands waiting for a cluster remap. |
| uint8_t const * | key | Key used to identify key slot. |
| size_t | key_len | Length of key. |
| fr_redis_ct_key_slot_t const * | key_slot | Key slot identified from the command key. |
| fr_redis_ct_node_t * | node | Specific node to run command set on. |
| bool | read_only | Should this command be run read only. |
| uint8_t | replica_no | Current replica number being used. |
| request_t * | request | Request this command set relates to. |
| fr_redis_ct_t * | rtcluster | Cluster this command set is running on. |
| fr_redis_trunk_t * | rtrunk | Trunk the command set is currently running on. |
| struct fr_redis_ct_key_slot_s |
Definition at line 157 of file cluster_async.c.
| Data Fields | ||
|---|---|---|
| uint8_t | master | id of the master node. |
| uint8_t | num_replicas | Number of replica nodes. |
| uint8_t | replica[MAX_REPLICAS] | Array of ids of replica nodes. |
| struct fr_redis_ct_node_s |
Definition at line 201 of file cluster_async.c.
Collaboration diagram for fr_redis_ct_node_s:| Data Fields | ||
|---|---|---|
| uint8_t | id | Array offset in the array of available nodes. |
| fr_redis_io_conf_t | ioconf | Connection config for this node. |
| bool | is_active | Is this node currently active. |
| bool | is_master | Is this node currently a master. |
| char | name[INET6_ADDRSTRLEN] | |
| fr_rb_node_t | rbnode | Entry in the tree of used nodes. |
| fr_redis_ct_t * | rtcluster | Cluster this node belongs to. |
| fr_pair_list_t | trigger_args | Pairs to pass to trigger functions. |
| fr_redis_trunk_t * | trunk | Trunk connection to this node. |
| struct fr_redis_ct_pend_req_t |
Structure to record that a request is waiting for the cluster map.
Definition at line 235 of file cluster_async.c.
Collaboration diagram for fr_redis_ct_pend_req_t:| Data Fields | ||
|---|---|---|
| fr_dlist_t | entry | Entry in the list of pending requests. |
| request_t * | request | The request waiting for the map. |
| fr_redis_ct_t * | rtcluster | Cluster the request is waiting for. |
| struct fr_redis_ct_s |
Thread local state for a cluster.
Definition at line 173 of file cluster_async.c.
Collaboration diagram for fr_redis_ct_s:| Data Fields | ||
|---|---|---|
| fr_redis_trunk_active_t | active | Callback to run when the trunk becomes active. |
| bool | active_oneshot | Should the callback only be called once. |
| void * | active_uctx | Uctx to pass to active callback. |
| uint16_t | cluster_id | Number assigned to the cluster by coordinator. |
| fr_redis_conf_t const * | conf | Redis configuration for the cluster. |
| bool | delay_start | Prevent connections from spawning immediately. |
| fr_event_list_t * | el | |
| fr_fifo_t * | free_nodes | Nodes not currently active. |
| fr_redis_ct_key_slot_t | key_slot[KEY_SLOTS] | |
| fr_time_t | map_updated | Time the cluster last updated. |
| fr_redis_ct_node_t * | node | Array of nodes in this cluster. |
| fr_dlist_head_t | pend_cmds | Commands awaiting cluster map. |
| fr_dlist_head_t | pend_reqs | Requests awaiting cluster map. |
| fr_redis_ct_state_t | state | State of the cluster. |
| trunk_conf_t const * | tconf | Configuration for all trunks in the cluster. |
| CONF_SECTION const * | tls_cs | TLS CONF_SECTION. |
| fr_rb_tree_t * | used_nodes | Active nodes. |
| #define CONFIGURE_NODE | ( | _node, | |
| _addr, | |||
| _port | |||
| ) |
Definition at line 241 of file cluster_async.c.
| #define MAX_REPLICAS 5 |
Maximum number of replicas associated with a keyslot.
Definition at line 156 of file cluster_async.c.
| #define SET_ACTIVE | ( | _node | ) |
| #define SET_INACTIVE | ( | _node | ) |
| enum fr_redis_ct_state_t |
Definition at line 163 of file cluster_async.c.
|
static |
Compare two redis nodes to check equality.
| [in] | one | first node. |
| [in] | two | second node. |
Definition at line 590 of file cluster_async.c.
Here is the caller graph for this function:
|
static |
Definition at line 454 of file cluster_async.c.
Here is the call graph for this function:
Here is the caller graph for this function:
|
static |
Ensure pending request is removed from the list on freeing.
Definition at line 1171 of file cluster_async.c.
Here is the call graph for this function:
Here is the caller graph for this function:Resolve key to key slot index.
Identical to the example implementation, except it uses memchr which will be faster, and isn't so needlessly complex.
| [in] | key | to resolve. |
| [in] | key_len | length of key. |
Definition at line 276 of file cluster_async.c.
Here is the call graph for this function:
Here is the caller graph for this function:| void fr_redis_async_cmd_cancel | ( | fr_redis_async_cmd_t * | cmd | ) |
Cancel a Redis async command.
Definition at line 528 of file cluster_async.c.
Here is the call graph for this function:
Here is the caller graph for this function:
|
static |
Enqueue a command set on a node identified by the key.
Definition at line 396 of file cluster_async.c.
Here is the call graph for this function:
Here is the caller graph for this function:| fr_redis_ct_node_t * fr_redis_async_cmd_node | ( | fr_redis_async_cmd_t * | cmd | ) |
Fetch the cluster node a command was last sent to.
Definition at line 543 of file cluster_async.c.
Here is the caller graph for this function:| fr_redis_async_rcode_t fr_redis_async_cmd_redirect | ( | fr_redis_async_cmd_t * | cmd | ) |
Re-submit a redis async command set on a different node.
Using the node returned by a MOVED / ASK response.
| cmd | Async command set to redirect |
Definition at line 555 of file cluster_async.c.
Here is the call graph for this function:
Here is the caller graph for this function:| fr_redis_async_rcode_t fr_redis_async_cmd_resend | ( | fr_redis_async_cmd_t * | cmd | ) |
Re-submit a redis async command set.
To be used following TRYAGAIN responses
| cmd | Async command set to redirect |
Definition at line 578 of file cluster_async.c.
Here is the call graph for this function:
Here is the caller graph for this function:| fr_redis_async_cmd_t * fr_redis_async_cmd_start | ( | TALLOC_CTX * | ctx, |
| request_t * | request, | ||
| fr_redis_async_rcode_t * | rcode, | ||
| fr_redis_ct_t * | rtcluster, | ||
| uint8_t const * | key, | ||
| size_t | key_len, | ||
| fr_redis_command_set_t * | cmds, | ||
| bool | read_only, | ||
| fr_redis_ct_node_t * | node | ||
| ) |
Start running a command set on an async redis cluster.
| ctx | to allocate tracking structure. |
| request | current request. |
| rcode | Where to write the result code. |
| rtcluster | to start the command set on |
| key | to identify the cluster slot. |
| key_len | Length of key. |
| cmds | Command set to run. |
| read_only | Should the command set be run on read only nodes. |
| node | Specific node to run the command set on. |
Definition at line 474 of file cluster_async.c.
Here is the call graph for this function:
Here is the caller graph for this function:| fr_redis_trunk_t * fr_redis_async_cmd_trunk | ( | fr_redis_async_cmd_t * | cmd | ) |
Fetch the redis trunk a command is associated with.
Definition at line 536 of file cluster_async.c.
Here is the caller graph for this function:| fr_redis_ct_t * fr_redis_ct_alloc | ( | TALLOC_CTX * | ctx, |
| CONF_SECTION * | tls_cs, | ||
| fr_event_list_t * | el, | ||
| fr_redis_conf_t * | conf, | ||
| fr_redis_trunk_active_t | active, | ||
| void * | active_uctx, | ||
| bool | active_oneshot | ||
| ) |
Allocate per-thread, per-cluster instance.
This structure represents all the connections for a given thread for a given cluster. The structures holds the trunk connections to talk to each cluster member.
Definition at line 616 of file cluster_async.c.
Here is the call graph for this function:
Here is the caller graph for this function:| fr_event_list_t * fr_redis_ct_el | ( | fr_redis_ct_t * | rtcluster | ) |
| char const * fr_redis_ct_ipaddr | ( | fr_redis_ct_node_t const * | node | ) |
Return the ipaddr of a particular node.
| [in] | node | to get ip address from. |
Definition at line 369 of file cluster_async.c.
Here is the caller graph for this function:| int fr_redis_ct_map_bootstrap | ( | fr_redis_ct_t * | rtcluster, |
| fr_coord_worker_t * | cw, | ||
| fr_coord_pair_reg_t * | coord_pair_reg | ||
| ) |
Initiate bootstrapping of the cluster map.
To be used when a module first wants to fetch a cluster map
| rtcluster | Cluster to fetch map for |
| cw | Coord worker to launch request |
| coord_pair_reg | Coord pair registration |
Definition at line 1003 of file cluster_async.c.
Here is the call graph for this function:
Here is the caller graph for this function:| int fr_redis_ct_map_fail | ( | fr_redis_ct_t * | rtcluster, |
| UNUSED fr_pair_list_t const * | list | ||
| ) |
Process a cluster map fail message from the coordinator.
| rtcluster | Cluster to update |
| list | pairs sent by a coordinator |
Definition at line 985 of file cluster_async.c.
| fr_redis_async_rcode_t fr_redis_ct_map_get | ( | fr_redis_ct_t * | rtcluster, |
| fr_coord_worker_t * | cw, | ||
| fr_coord_pair_reg_t * | coord_pair_reg, | ||
| bool | force | ||
| ) |
Initiate updating of the cluster map.
To be used when a command returns MOVED
Definition at line 1064 of file cluster_async.c.
Here is the call graph for this function:
Here is the caller graph for this function:| int fr_redis_ct_map_update | ( | fr_redis_ct_t * | rtcluster, |
| fr_pair_list_t const * | list | ||
| ) |
Update a Redis cluster map from a pair list returned from a coordinator.
| rtcluster | Cluster to update |
| list | pairs sent by a coordinator |
Definition at line 762 of file cluster_async.c.
Here is the call graph for this function:| fr_redis_ct_node_t const * fr_redis_ct_master | ( | fr_redis_ct_t * | rtcluster, |
| fr_redis_ct_key_slot_t const * | key_slot | ||
| ) |
Return the master node that would be used for a particular key slot.
| [in] | rtcluster | To resolve key slot in. |
| [in] | key_slot | to resolve to node. |
Definition at line 338 of file cluster_async.c.
Here is the caller graph for this function:| fr_redis_async_rcode_t fr_redis_ct_node_addr_by_role | ( | TALLOC_CTX * | ctx, |
| fr_redis_io_conf_t * | out[], | ||
| uint8_t * | count_out, | ||
| fr_redis_ct_t * | rtcluster, | ||
| bool | is_master, | ||
| bool | is_replica | ||
| ) |
Definition at line 1121 of file cluster_async.c.
Here is the call graph for this function:
Here is the caller graph for this function:| fr_redis_ct_node_t * fr_redis_ct_node_by_addr | ( | fr_redis_ct_t * | rtcluster, |
| fr_redis_io_conf_t * | ioconf | ||
| ) |
Definition at line 1111 of file cluster_async.c.
Here is the call graph for this function:
Here is the caller graph for this function:| int fr_redis_ct_port | ( | uint16_t * | out, |
| fr_redis_ct_node_t const * | node | ||
| ) |
Return the port of a particular node.
| [out] | out | Port of the node. |
| [in] | node | to get ip address from. |
Definition at line 384 of file cluster_async.c.
Here is the caller graph for this function:| fr_redis_ct_node_t const * fr_redis_ct_replica | ( | fr_redis_ct_t * | rtcluster, |
| fr_redis_ct_key_slot_t const * | key_slot, | ||
| uint8_t | replica_num | ||
| ) |
Return the replica node that would be used for a particular key slot.
| [in] | rtcluster | To resolve key slot in. |
| [in] | key_slot | To resolve to node. |
| [in] | replica_num | 0..n. |
Definition at line 354 of file cluster_async.c.
Here is the caller graph for this function:| void fr_redis_ct_request_yield | ( | TALLOC_CTX * | ctx, |
| fr_redis_ct_t * | rtcluster, | ||
| request_t * | request | ||
| ) |
Add a request to the list of those waiting for the cluster map.
Definition at line 1181 of file cluster_async.c.
Here is the call graph for this function:
Here is the caller graph for this function:| fr_redis_ct_key_slot_t const * fr_redis_ct_slot_by_key | ( | fr_redis_ct_t * | rtcluster, |
| request_t * | request, | ||
| uint8_t const * | key, | ||
| size_t | key_len | ||
| ) |
Resolve key to key slot.
| [in] | rtcluster | to resolve the key slot in. |
| [in] | request | Current request (for debugging). |
| [in] | key | to resolve. |
| [in] | key_len | length of key. |
Definition at line 302 of file cluster_async.c.
Here is the call graph for this function:
Here is the caller graph for this function:| trunk_conf_t const * fr_redis_ct_trunk_conf | ( | fr_redis_ct_t * | rtcluster | ) |
1.9.8