导航

memcached源码分析
memcached源码分析-网络模块
memcached源码分析-指令解析模块
memcached源码分析-哈希表(hashtable)模块
memcached源码分析-slab存储机制

1.前言

前面一章节我们介绍了libevent网络事件模块，当连接有数据请求过来的时候，就会触发work线程的读写事件回调函数：event_handler，这个方法调用drive_machine函数，根据状态解析事件。我们都知道Memcached是一个高性能的开源分布式内存对象缓存系统，但memcached的服务器客户端通信并不使用复杂的XML等格式，而使用简单的基于文本行的协议，我们使用简单文本指令就可以实现对memcached的操作。
接下来主要选取set指令和get指令进行分析，从而了解memcached是如何解析客户的指令请求的。

指令简要说明：

set指令语法：
set key flags exptime bytes [noreply]
value

参数说明：

• key：键值 key-value 结构中的 key，用于查找缓存值。
• flags：可以包括键值对的整型参数，客户机使用它存储关于键值对的额外信息 。
• exptime：在缓存中保存键值对的时间长度（以秒为单位，0 表示永远）
• bytes：在缓存中存储的字节数
• noreply（可选）： 该参数告知服务器不需要返回数据
• value：存储的值（始终位于第二行）（可直接理解为key-value结构中的value）

get指令语法：
get key

参数说明如下：
key：键值 key-value 结构中的 key，用于查找缓存值。

2.指令流程图

当客户端连接发送指令给memcached，就会触发work线程的读写事件回调函数：event_handler，这个方法调用drive_machine状态机函数，下面就是set和get指令的状态机执行流程图

set指令.jpg

get指令.jpg

3.源码分析

针对上述流程图，做进一步的源码分析
在分析源码之前，这里主要介绍一下相关模块的重要数据结构

struct conn {
    int    sfd;
#ifdef TLS
    SSL    *ssl;
    char   *ssl_wbuf;
    bool ssl_enabled;
#endif
    sasl_conn_t *sasl_conn;
    bool sasl_started;
    bool authenticated;
    enum conn_states  state;
    enum bin_substates substate;
    rel_time_t last_cmd_time;
    struct event event;
    short  ev_flags;
    short  which;   /** which events were just triggered */

    //rbuf 用于存储读取命令的内存
    char   *rbuf;   /** buffer to read commands into */
    //如果我们已经解析了部分数据，rcurr游标执行已经解析的位置
    char   *rcurr;  /** but if we parsed some already, this is where we stopped */
    //为rbuf分配空间大小
    int    rsize;   /** total allocated size of rbuf */
    //未解析的数据字节数 rbytes = rszie - (rcur - rbuf)
    int    rbytes;  /** how much data, starting from rcur, do we have unparsed */

    char   *wbuf;
    char   *wcurr;
    int    wsize;
    int    wbytes;
    /** which state to go into after finishing current write */
    enum conn_states  write_and_go;
    void   *write_and_free; /** free this memory after finishing writing */

    //指向item结构，在set命令中ritem指向了item结构中内存保存关键字key之后的位置，客户端读取
    //value值之后，将value值保存于ritem所指向的内存。
    //例如客户端第一步执行set testkey 0 0 5,服务端接收到该指令，然后申请item结构保存指令内容，
    //ritem则执向item结构中保存testkey之后的那段内存空间，rlbytes为5，表示要读取5个字节的value值，
    //第二步客户端输入value值之后，服务端触发事件，服务端根据rlbytes大小从客户端连接中读取相应大小的数据
    //保存于ritem执向的内存空间
    char   *ritem;  /** when we read in an item's value, it goes here */
    //需要读取内容的大小
    int    rlbytes;

    /* data for the nread state */

    /**
     * item is used to hold an item structure created after reading the command
     * line of set/add/replace commands, but before we finished reading the actual
     * data. The data is read into ITEM_data(item) to avoid extra copying.
     */

    //为set命令申请的相关内存结构
    void   *item;     /* for commands set/add/replace  */

    /* data for the swallow state */
    int    sbytes;    /* how many bytes to swallow */

    /* data for the mwrite state */
    struct iovec *iov;
    int    iovsize;   /* number of elements allocated in iov[] */
    int    iovused;   /* number of elements used in iov[] */

    struct msghdr *msglist;
    int    msgsize;   /* number of elements allocated in msglist[] */
    int    msgused;   /* number of elements used in msglist[] */
    int    msgcurr;   /* element in msglist[] being transmitted now */
    int    msgbytes;  /* number of bytes in current msg */

    item   **ilist;   /* list of items to write out */
    int    isize;
    item   **icurr;
    int    ileft;

    char   **suffixlist;
    int    suffixsize;
    char   **suffixcurr;
    int    suffixleft;
#ifdef EXTSTORE
    int io_wrapleft;
    unsigned int recache_counter;
    io_wrap *io_wraplist; /* linked list of io_wraps */
    bool io_queued; /* FIXME: debugging flag */
#endif
    enum protocol protocol;   /* which protocol this connection speaks */
    enum network_transport transport; /* what transport is used by this connection */

    /* data for UDP clients */
    int    request_id; /* Incoming UDP request ID, if this is a UDP "connection" */
    struct sockaddr_in6 request_addr; /* udp: Who sent the most recent request */
    socklen_t request_addr_size;
    unsigned char *hdrbuf; /* udp packet headers */
    int    hdrsize;   /* number of headers' worth of space is allocated */

    bool   noreply;   /* True if the reply should not be sent. */
    /* current stats command */
    struct {
        char *buffer;
        size_t size;
        size_t offset;
    } stats;

    /* Binary protocol stuff */
    /* This is where the binary header goes */
    protocol_binary_request_header binary_header;
    uint64_t cas; /* the cas to return */
    short cmd; /* current command being processed */
    int opaque;
    int keylen;
    conn   *next;     /* Used for generating a list of conn structures */
    LIBEVENT_THREAD *thread; /* Pointer to the thread object serving this connection */
    ssize_t (*read)(conn  *c, void *buf, size_t count);
    ssize_t (*sendmsg)(conn *c, struct msghdr *msg, int flags);
    ssize_t (*write)(conn *c, void *buf, size_t count);
};

drive_machine函数就是整个事件执行流程的核心函数，内部while循环切换不同的状态，完成不同状态下的业务逻辑处理

static void drive_machine(conn *c) {
    //...
    while (!stop) {
        switch(c->state) {
            //...
            case conn_waiting:
                //...
                //设置为可读状态，等待可读事件的触发
                conn_set_state(c, conn_read);
                stop = true;
                break;
            case conn_read:
                //...
                //从conn->sfd中读取指令数据
                //例如:set testkey 0 0 5 \r\n
                res = try_read_network(c);
                switch (res) {
                    //....
                    case READ_DATA_RECEIVED:
                        conn_set_state(c, conn_parse_cmd);
                        break;
                    //...
                }
                break;
            case conn_parse_cmd:
                //命令的解析
                if (try_read_command(c) == 0) {
                    /* wee need more data! */
                    //数据不完整，需要等待完整读取数据指令
                    conn_set_state(c, conn_waiting);
                }
                break;
            case conn_new_cmd:
                --nreqs;
                if (nreqs >= 0) {
                    //设置事件的状态
                    reset_cmd_handler(c);
                }else{
                    //....
                }
                break;
            case conn_nread:
                if (c->rlbytes == 0) {
                    //数据读取完毕
                    complete_nread(c);
                    break;
                }
                //...
                if (!c->item || (((item *)c->item)->it_flags & ITEM_CHUNKED) == 0) {
                    //...
                    /* now try reading from the socket */
                    //从socket读取数据内容保存到c->ritem指向的内存空间
                    //例如：set testkey 0 0 5,指令部分try_read_network已经读取
                    //这里read就是获取key-val对应的value内容(该value内容被指定为5个字节)
                    res = c->read(c, c->ritem, c->rlbytes);
                    //...
                }
                break;
            case conn_write:
                //...
                /* fall through... */
                //注意这里，conn_write状态执行完毕，没有break，直接执行conn_mwrite状态
            case conn_mwrite:
                //...
                //给客户端写数据，例如客户端的get指令获取存储的value
                switch (transmit(c)) {
                    case TRANSMIT_COMPLETE:
                        if (c->state == conn_mwrite) {
                            conn_release_items(c);
                            /* XXX:  I don't know why this wasn't the general case */
                            if(c->protocol == binary_prot) {
                                conn_set_state(c, c->write_and_go);
                            } else {
                                //设置连接状态
                                //conn_new_cmd状态中如果rbuf中还有数据未进行处理，那么继续处理
                                //如果所有数据都处理完毕，那么会转为conn_waiting状态，等待事件触发
                                conn_set_state(c, conn_new_cmd);
                            }
                        }
                        //...
                        break;
                    //...
                }
                break;
        }
    }
}

conn_read状态下调用try_read_network函数进行连接数据的读取

/*
 * read from network as much as we can, handle buffer overflow and connection
 * close.
 * before reading, move the remaining incomplete fragment of a command
 * (if any) to the beginning of the buffer.
 *
 * To protect us from someone flooding a connection with bogus data causing
 * the connection to eat up all available memory, break out and start looking
 * at the data I've got after a number of reallocs...
 *
 * @return enum try_read_result
 */
//读取客户端传递过来的命令数据
static enum try_read_result try_read_network(conn *c) {
    //rbuf 用于存储读取命令的内存
    //rcur 如果我们已经解析了部分数据，rcurr游标执行已经解析的位置
    //rsize 为rbuf分配空间大小
    //rbytes 未解析的数据字节数 rbytes = rszie - (rcur - rbuf)

    //在读取命令数据之前，首先判断c->rcurr != c->rbuf
    if (c->rcurr != c->rbuf) {
        if (c->rbytes != 0)
            //如果发现之前还有部分命令未解析完全，那么将未解析的数据拷贝到存储命令空间rbuf的首位置
            memmove(c->rbuf, c->rcurr, c->rbytes);
        //将rcurr移动到rbuf的位置，也就是存储命令的首位置
        c->rcurr = c->rbuf;
    }
    //尽可能多的尝试读取命令数据
    while (1) {
        //如果读取的字节数大于等于rbuf的内存空间，则重新分配内存，memcached做了次数限制
        if (c->rbytes >= c->rsize) {
            if (num_allocs == 4) {
                return gotdata;
            }
            ++num_allocs;
            char *new_rbuf = realloc(c->rbuf, c->rsize * 2);
            //...
            c->rcurr = c->rbuf = new_rbuf;
            c->rsize *= 2;
        }
        //rbuf的剩余空间大小
        int avail = c->rsize - c->rbytes;
        //尽可能的读取avail字节长度内容
        res = c->read(c, c->rbuf + c->rbytes, avail);

        if (res > 0) {
            //...
            c->rbytes += res;
            //如果实际读取的字节数和我们尝试读取的字节数相等，
            //那么极有可能还有数据可读, continue继续尝试读取socket数据
            if (res == avail) {
                continue;
            }else{
                break;
            }
        }
        //...
    }
    return gotdata;
}

conn_parse_cmd状态下调用try_read_command函数进入命令解析的入口

/*
 * if we have a complete line in the buffer, process it.
 */
 //处理指令数据
static int try_read_command(conn *c) {
    //...
    if (c->protocol == binary_prot) {
        //...
    }else{
        char *el, *cont;
        //没有可解析数据
        if (c->rbytes == 0)
            return 0;
        //查找指令数据中的\n
        //例如:set testkey 0 0 5 \r\n
        //get testkey \r\n等等
        el = memchr(c->rcurr, '\n', c->rbytes);

        if (!el) {
            //...
        }
        cont = el + 1;
        if ((el - c->rcurr) > 1 && *(el - 1) == '\r') {
            el--;
        }
        //将\r替换为'\0',此时c->rcurr内容为set testkey 0 0 5 \0\n
        *el = '\0';

        //...
        //解析命令
        process_command(c, c->rcurr);
        //剩余未解析命令数据
        c->rbytes -= (cont - c->rcurr);
        c->rcurr = cont;
    }

    return 1;
}

//命令行数据分解
static void process_command(conn *c, char *command) {
    token_t tokens[MAX_TOKENS];
    size_t ntokens;
    int comm;
    //...
    //指令分解保存到tokens数组
    ntokens = tokenize_command(command, tokens, MAX_TOKENS);

    if (ntokens >= 3 &&
        ((strcmp(tokens[COMMAND_TOKEN].value, "get") == 0) ||
        (strcmp(tokens[COMMAND_TOKEN].value, "bget") == 0))) {
        //get命令处理
        process_get_command(c, tokens, ntokens, false, false);
    }else if ((ntokens == 6 || ntokens == 7) &&
        ((strcmp(tokens[COMMAND_TOKEN].value, "add") == 0 && (comm = NREAD_ADD)) ||
        (strcmp(tokens[COMMAND_TOKEN].value, "set") == 0 && (comm = NREAD_SET)) ||
        (strcmp(tokens[COMMAND_TOKEN].value, "replace") == 0 && (comm = NREAD_REPLACE)) ||
        (strcmp(tokens[COMMAND_TOKEN].value, "prepend") == 0 && (comm = NREAD_PREPEND)) ||
        (strcmp(tokens[COMMAND_TOKEN].value, "append") == 0 && (comm = NREAD_APPEND)) )) {
        //set指令的解析
        process_update_command(c, tokens, ntokens, comm, false);
    }//...

    return;
}

//指令分解
//例如:set testkey 0 0 5分解为
//tokens[0].value = 'set' ; tokens[0].length = 3
//tokens[1].value = 'testkey' ; tokens[1].length = 7
//tokens[2].value = '0' ; tokens[2].length = 1
//tokens[3].value = '0' ; tokens[3].length = 1
//tokens[4].value = '5' ; tokens[4].length = 1
//tokens[5].value = '\0' ; tokens[5].length = 1
static size_t tokenize_command(char *command, token_t *tokens, const size_t max_tokens) {
    char *s, *e;
    size_t ntokens = 0;
    size_t len = strlen(command);
    unsigned int i = 0;

    assert(command != NULL && tokens != NULL && max_tokens > 1);

    s = e = command;
    for (i = 0; i < len; i++) {
        if (*e == ' ') {
            if (s != e) {
                tokens[ntokens].value = s;
                tokens[ntokens].length = e - s;
                ntokens++;
                *e = '\0';
                if (ntokens == max_tokens - 1) {
                    e++;
                    s = e; /* so we don't add an extra token */
                    break;
                }
            }
            s = e + 1;
        }
        e++;
    }

    if (s != e) {
        tokens[ntokens].value = s;
        tokens[ntokens].length = e - s;
        ntokens++;
    }

    /*
     * If we scanned the whole string, the terminal value pointer is null,
     * otherwise it is the first unprocessed character.
     */
    tokens[ntokens].value =  *e == '\0' ? NULL : e;
    tokens[ntokens].length = 0;
    ntokens++;

    return ntokens;
}

get指令处理函数

static inline void process_get_command(conn *c, token_t *tokens, size_t ntokens, 
                                        bool return_cas, bool should_touch) {
    //...
    //key nkey
    token_t *key_token = &tokens[KEY_TOKEN];

    //...
    //主要根据key值取内存中查询对应value值
    //然后格式化回复内容

    //...
    if (key_token->value != NULL || add_iov(c, "END\r\n", 5) != 0
        || (IS_UDP(c->transport) && build_udp_headers(c) != 0)) {
        if (fail_length) {
            out_string(c, "CLIENT_ERROR bad command line format");
        } else {
            out_of_memory(c, "SERVER_ERROR out of memory writing get response");
        }
        conn_release_items(c);
    }else {
        //设置连接状态
        conn_set_state(c, conn_mwrite);
        c->msgcurr = 0;
    }
}

set指令处理函数

static void process_update_command(conn *c, token_t *tokens, const size_t ntokens,
                                        int comm, bool handle_cas){
    //...
    //key
    key = tokens[KEY_TOKEN].value;
    nkey = tokens[KEY_TOKEN].length;

    //将拆解的命令转换为整形
    if (! (safe_strtoul(tokens[2].value, (uint32_t *)&flags)
           && safe_strtol(tokens[3].value, &exptime_int)
           && safe_strtol(tokens[4].value, (int32_t *)&vlen))) {
        out_string(c, "CLIENT_ERROR bad command line format");
        return;
    }
    //...
    //为操作指令分配相关空间资源，我会在后面的学习中进行具体分析
    //这里涉及slab相关知识，暂时不做分析
    it = item_alloc(key, nkey, flags, realtime(exptime), vlen);

    //...
    //ITEM_data 用于计算命令key所在内存的偏移位置
    //c->ritem指向命令key值之后的内存，在conn_nread中会将value值读到c->ritem指向的内存位置
    //当成功读取value时,it中保存内容key value
    c->ritem = ITEM_data(it);
    //需要读取的value值长度
    c->rlbytes = it->nbytes;
    //指令类型
    c->cmd = comm;
    conn_set_state(c, conn_nread);
}

transmit将结果发送客户端

static enum transmit_result transmit(conn *c) {
    //...
    if (c->msgcurr < c->msgused) {
        //...
        //将查询的结果返回请求的客户端
        res = c->sendmsg(c, m, 0);
        //...
    }
}