token.c

Go to the documentation of this file.

/*
 *   This library is free software; you can redistribute it and/or
 *   modify it under the terms of the GNU Lesser General Public
 *   License as published by the Free Software Foundation; either
 *   version 2.1 of the License, or (at your option) any later version.
 *
 *   This library is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 *   Lesser General Public License for more details.
 *
 *   You should have received a copy of the GNU Lesser General Public
 *   License along with this library; if not, write to the Free Software
 *   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
 */
 
/** Tokenisation code and constants
 *
 * This is mostly for the attribute filter and user files.
 *
 * @file src/lib/util/token.c
 *
 * @copyright 2001,2006 The FreeRADIUS server project
 */
RCSID("$Id: 74ec6b3425c41bf7c40c7c360065d2fa1aa1f256 $")
 
#include <stdio.h>
 
#include <freeradius-devel/util/skip.h>
#include <freeradius-devel/util/token.h>
 
 
fr_table_num_ordered_t const fr_tokens_table[] = {
        { L("=~"),      T_OP_REG_EQ     }, /* order is important! */
        { L("!~"),      T_OP_REG_NE     },
        { L("{"),       T_LCBRACE       },
        { L("}"),       T_RCBRACE       },
        { L("("),       T_LBRACE        },
        { L(")"),       T_RBRACE        },
        { L(","),       T_COMMA         },
        { L("++"),      T_OP_INCRM      },
        { L("+="),      T_OP_ADD_EQ     },
        { L("-="),      T_OP_SUB_EQ     },
        { L("*="),      T_OP_MUL_EQ     },
        { L("/="),      T_OP_DIV_EQ     },
        { L(":="),      T_OP_SET        },
        { L("=*"),      T_OP_CMP_TRUE   },
        { L("!*"),      T_OP_CMP_FALSE  },
        { L("=="),      T_OP_CMP_EQ     },
        { L("==="),     T_OP_CMP_EQ_TYPE },
        { L("^="),      T_OP_PREPEND    },
        { L("|="),      T_OP_OR_EQ      },
        { L("&="),      T_OP_AND_EQ     },
        { L("="),       T_OP_EQ         },
        { L("!="),      T_OP_NE         },
        { L("!=="),     T_OP_CMP_NE_TYPE },
        { L(">>="),     T_OP_RSHIFT_EQ  },
        { L(">="),      T_OP_GE         },
        { L(">"),       T_OP_GT         },
        { L("<<="),     T_OP_LSHIFT_EQ  },
        { L("<="),      T_OP_LE         },
        { L("<"),       T_OP_LT         },
        { L("#"),       T_HASH          },
        { L(";"),       T_SEMICOLON     }
};
size_t fr_tokens_table_len = NUM_ELEMENTS(fr_tokens_table);
 
fr_table_num_sorted_t const fr_token_quotes_table[] = {
        { L(""),        T_BARE_WORD             },
        { L("'"),       T_SINGLE_QUOTED_STRING  },
        { L("/"),       T_SOLIDUS_QUOTED_STRING },
        { L("\""),      T_DOUBLE_QUOTED_STRING  },
        { L("`"),       T_BACK_QUOTED_STRING    }
};
size_t fr_token_quotes_table_len = NUM_ELEMENTS(fr_token_quotes_table);
 
/** Map each `fr_token_t` to the C identifier of its enum constant.
 *
 * Complement of `fr_tokens[]` (the operator-character / human-display
 * form like ":=" / "<BARE-WORD>") - this one returns the source-identical
 * `T_OP_SET` / `T_BARE_WORD` form so tooling (radconf2json, etc.) can
 * round-trip enum names through a JSON intermediate.
 */
static fr_table_num_indexed_t const fr_token_to_enum_str_table[] = {
        FR_TABLE_INDEXED_ENTRY(T_BARE_WORD),
        FR_TABLE_INDEXED_ENTRY(T_DOUBLE_QUOTED_STRING),
        FR_TABLE_INDEXED_ENTRY(T_SINGLE_QUOTED_STRING),
        FR_TABLE_INDEXED_ENTRY(T_BACK_QUOTED_STRING),
        FR_TABLE_INDEXED_ENTRY(T_SOLIDUS_QUOTED_STRING),
        FR_TABLE_INDEXED_ENTRY(T_OP_EQ),
        FR_TABLE_INDEXED_ENTRY(T_OP_NE),
        FR_TABLE_INDEXED_ENTRY(T_OP_GE),
        FR_TABLE_INDEXED_ENTRY(T_OP_GT),
        FR_TABLE_INDEXED_ENTRY(T_OP_LE),
        FR_TABLE_INDEXED_ENTRY(T_OP_LT),
        FR_TABLE_INDEXED_ENTRY(T_OP_CMP_EQ),
        FR_TABLE_INDEXED_ENTRY(T_OP_CMP_EQ_TYPE),
        FR_TABLE_INDEXED_ENTRY(T_OP_CMP_NE_TYPE),
        FR_TABLE_INDEXED_ENTRY(T_OP_ADD_EQ),
        FR_TABLE_INDEXED_ENTRY(T_OP_SUB_EQ),
        FR_TABLE_INDEXED_ENTRY(T_OP_MUL_EQ),
        FR_TABLE_INDEXED_ENTRY(T_OP_DIV_EQ),
        FR_TABLE_INDEXED_ENTRY(T_OP_AND_EQ),
        FR_TABLE_INDEXED_ENTRY(T_OP_OR_EQ),
        FR_TABLE_INDEXED_ENTRY(T_OP_RSHIFT_EQ),
        FR_TABLE_INDEXED_ENTRY(T_OP_LSHIFT_EQ),
        FR_TABLE_INDEXED_ENTRY(T_OP_SET),
        FR_TABLE_INDEXED_ENTRY(T_OP_PREPEND),
        FR_TABLE_INDEXED_ENTRY(T_OP_REG_EQ),
        FR_TABLE_INDEXED_ENTRY(T_OP_REG_NE),
        FR_TABLE_INDEXED_ENTRY(T_OP_CMP_TRUE),
        FR_TABLE_INDEXED_ENTRY(T_OP_CMP_FALSE),
        FR_TABLE_INDEXED_ENTRY(T_OP_INCRM),
};
static size_t fr_token_to_enum_str_table_len = NUM_ELEMENTS(fr_token_to_enum_str_table);
 
char const *fr_token_to_enum_str(fr_token_t t)
{
        return fr_table_str_by_value(fr_token_to_enum_str_table, t, "T_INVALID");
}
 
/** Quote-token names that radconf2json's JSON output uses.
 *
 * Only the five quote-typed tokens, kept sorted by name so the
 * `fr_table_value_by_str` binary search can find them.  Used by
 * radjson2conf when rebuilding a CF tree from JSON.
 */
static fr_table_num_sorted_t const fr_token_from_quote_enum_str_table[] = {
        { L("T_BACK_QUOTED_STRING"),    T_BACK_QUOTED_STRING    },
        { L("T_BARE_WORD"),             T_BARE_WORD             },
        { L("T_DOUBLE_QUOTED_STRING"),  T_DOUBLE_QUOTED_STRING  },
        { L("T_SINGLE_QUOTED_STRING"),  T_SINGLE_QUOTED_STRING  },
        { L("T_SOLIDUS_QUOTED_STRING"), T_SOLIDUS_QUOTED_STRING },
};
static size_t fr_token_from_quote_enum_str_table_len = NUM_ELEMENTS(fr_token_from_quote_enum_str_table);
 
fr_token_t fr_token_from_quote_enum_str(char const *s, fr_token_t dflt)
{
        if (!s) return dflt;
        return fr_table_value_by_str(fr_token_from_quote_enum_str_table, s, dflt);
}
 
/*
 *  String versions for all of the tokens.
 */
char const *fr_tokens[T_TOKEN_LAST] = {
        [T_INVALID] = "?",
        [T_EOL] = "EOL",
 
        [T_LCBRACE] = "{",
        [T_RCBRACE] = "}",
        [T_LBRACE] = "(",
        [T_RBRACE] = ")",
        [T_COMMA] = ",",
        [T_SEMICOLON] = ";",
 
        [T_ADD]      = "+",
        [T_SUB]      = "-",
        [T_MUL]      = "*",
        [T_DIV]      = "/",
        [T_AND]      = "&",
        [T_OR]       = "|",
        [T_NOT]      = "!",
        [T_XOR]      = "^",
        [T_COMPLEMENT]  = "~",
        [T_MOD]  = "%",
 
        [T_RSHIFT]   = ">>",
        [T_LSHIFT]   = "<<",
 
        [T_LAND]     = "&&",
        [T_LOR]      = "||",
 
        [T_OP_INCRM] = "++",
 
        [T_OP_ADD_EQ] = "+=",
        [T_OP_SUB_EQ] = "-=",
        [T_OP_SET]    = ":=",
        [T_OP_EQ]     = "=",
        [T_OP_OR_EQ]  = "|=",
        [T_OP_AND_EQ]  = "&=",
 
        [T_OP_RSHIFT_EQ]   = ">>=",
        [T_OP_LSHIFT_EQ]   = "<<=",
 
        [T_OP_NE]     = "!=",
        [T_OP_GE]     = ">=",
        [T_OP_GT]     = ">",
        [T_OP_LE]     = "<=",
        [T_OP_LT]     = "<",
        [T_OP_REG_EQ] = "=~",
        [T_OP_REG_NE] = "!~",
 
        [T_OP_CMP_TRUE] = "=*",
        [T_OP_CMP_FALSE] = "!*",
 
        [T_OP_CMP_EQ] = "==",
 
        [T_OP_CMP_EQ_TYPE] = "===",
        [T_OP_CMP_NE_TYPE] = "!==",
 
        [T_OP_PREPEND] = "^=",
 
        [T_HASH]                  = "#",
        [T_BARE_WORD]             = "<BARE-WORD>",
        [T_DOUBLE_QUOTED_STRING]  = "<\"STRING\">",
        [T_SINGLE_QUOTED_STRING]  = "<'STRING'>",
        [T_BACK_QUOTED_STRING]    = "<`STRING`>",
        [T_SOLIDUS_QUOTED_STRING] = "</STRING/>",
};
 
 
/*
 *      This is fine.  Don't complain.
 */
DIAG_OFF_OPTIONAL(gnu-designator)
 
/** Convert tokens back to a quoting character
 *
 * Non-string types convert to '?' to screw ups can be identified easily
 */
const char fr_token_quote[T_TOKEN_LAST] = {
        [ 0 ... T_HASH ] = '?', /* GCC extension for range initialization, also allowed by clang */
 
        [T_BARE_WORD] = '\0',
        [T_DOUBLE_QUOTED_STRING] = '"',
        [T_SINGLE_QUOTED_STRING] = '\'',
        [T_BACK_QUOTED_STRING] = '`',
        [T_SOLIDUS_QUOTED_STRING] = '/',
};
 
#define T(_x) [T_OP_ ## _x] = true
 
const bool fr_assignment_op[T_TOKEN_LAST] = {
        T(INCRM),               /* only used by LDAP :( */
 
        T(ADD_EQ),
        T(SUB_EQ),
        T(MUL_EQ),
        T(DIV_EQ),
        T(AND_EQ),
        T(OR_EQ),
        T(RSHIFT_EQ),
        T(LSHIFT_EQ),
 
        T(SET),
        T(EQ),
        T(PREPEND),
};
 
const bool fr_list_assignment_op[T_TOKEN_LAST] = {
        T(ADD_EQ),              /* append */
        T(SUB_EQ),              /* remove */
        T(AND_EQ),              /* intersection */
        T(OR_EQ),               /* union */
        T(LE),                  /* merge RHS */
        T(GE),                  /* merge LHS */
 
        T(SET),
        T(EQ),
        T(PREPEND),             /* prepend */
};
 
const bool fr_comparison_op[T_TOKEN_LAST] = {
        T(NE),
        T(GE),
        T(GT),
        T(LE),
        T(LT),
        T(REG_EQ),
        T(REG_NE),
        T(CMP_TRUE),
        T(CMP_FALSE),
        T(CMP_EQ),
        T(CMP_EQ_TYPE),
        T(CMP_NE_TYPE),
};
 
#undef T
#define T(_x) [T_ ## _x] = true
 
const bool fr_binary_op[T_TOKEN_LAST] = {
        T(ADD),
        T(SUB),
        T(MUL),
        T(DIV),
        T(AND),
        T(OR),
        T(MOD),
        T(RSHIFT),
        T(LSHIFT),
};
 
 
#undef T
#define T(_x) [T_## _x] = true
const bool fr_str_tok[T_TOKEN_LAST] = {
        T(BARE_WORD),
        T(DOUBLE_QUOTED_STRING),
        T(SINGLE_QUOTED_STRING),
        T(BACK_QUOTED_STRING),
};
 
/*
 *      This works only as long as special tokens
 *      are max. 2 characters, but it's fast.
 */
#define TOKEN_MATCH(bptr, tptr) \
        ( (tptr)[0] == (bptr)[0] && \
         ((tptr)[1] == (bptr)[1] || (tptr)[1] == 0))
 
/*
 *      Read a word from a buffer and advance pointer.
 *      This function knows about escapes and quotes.
 *
 *      At end-of-line, buf[0] is set to '\0'.
 *      Returns 0 or special token value.
 */
static fr_token_t getthing(char const **ptr, char *buf, int buflen, bool tok,
                         fr_table_num_ordered_t const *tokenlist, size_t tokenlist_len, bool unescape)
{
        char                    *s;
        char const              *p;
        char                    quote;
        bool                    triple = false;
        size_t                  i;
        fr_token_t              token;
 
        buf[0] = '\0';
 
        /* Skip whitespace */
        p = *ptr;
 
        fr_skip_whitespace(p);
 
        if (!*p) {
                *ptr = p;
                return T_EOL;
        }
 
        /*
         *      Might be a 1 or 2 character token.
         */
        if (tok) {
                for (i = 0; i < tokenlist_len; i++) {
                        if (TOKEN_MATCH(p, tokenlist[i].name.str)) {
                                strcpy(buf, tokenlist[i].name.str);
                                p += tokenlist[i].name.len;
 
                                /*
                                 *      Try to shut up Coverity, which claims fr_token_t can be between 0..63, not
                                 *      0..48???
                                 */
                                if ((tokenlist[i].value < 0) || (tokenlist[i].value >= T_TOKEN_LAST)) return T_INVALID;
 
                                token = tokenlist[i].value;
                                goto done;
                        }
                }
        }
 
        /* Read word. */
        quote = '\0';
        switch (*p) {
        default:
                token = T_BARE_WORD;
                break;
 
        case '\'':
                token = T_SINGLE_QUOTED_STRING;
                break;
 
        case '"':
                token = T_DOUBLE_QUOTED_STRING;
                break;
 
        case '`':
                token = T_BACK_QUOTED_STRING;
                break;
        }
 
        if (token != T_BARE_WORD) {
                quote = *p;
 
                /*
                 *      Triple-quoted strings are copied over verbatim, without escapes.
                 */
                if ((buflen >= 3) && (p[1] == quote) && (p[2] == quote)) {
                        p += 3;
                        triple = true;
                } else {
                        p++;
                }
        }
        s = buf;
 
        while (*p && buflen-- > 1) {
                /*
                 *      We're looking for strings.  Stop on spaces, or
                 *      (if given a token list), on a token, or on a
                 *      comma.
                 */
                if (!quote) {
                        if (isspace((uint8_t) *p)) break;
 
 
                        if (tok) {
                                for (i = 0; i < tokenlist_len; i++) {
                                        if (TOKEN_MATCH(p, tokenlist[i].name.str)) {
                                                *s++ = 0;
                                                goto done;
                                        }
                                }
                        }
                        if (*p == ',') break;
 
                        /*
                         *      Copy the character over.
                         */
                        *s++ = *p++;
                        continue;
                } /* else there was a quotation character */
 
                /*
                 *      Un-escaped quote character.  We're done.
                 */
                if (*p == quote) {
                        if (!triple) {
                                p++;
                                *s++ = 0;
                                goto done;
                        }
 
                        if ((buflen >= 3) && (p[1] == quote) && (p[2] == quote)) {
                                p += 3;
                                *s++ = 0;
                                goto done;
                        }
 
                        *s++ = *p++;
                        continue;
                }
 
                /*
                 *      Everything but backslash gets copied over.
                 */
                if (*p != '\\') {
                        *s++ = *p++;
                        continue;
                }
 
                /*
                 *      There's nothing after the backslash, it's an error.
                 */
                if (!p[1]) {
                        fr_strerror_const("Unterminated string");
                        *s = '\0';
                        return T_INVALID;
                }
 
                if (unescape) {
                        p++;
 
                        switch (*p) {
                                case 'r':
                                        *s++ = '\r';
                                        break;
                                case 'n':
                                        *s++ = '\n';
                                        break;
                                case 't':
                                        *s++ = '\t';
                                        break;
 
                                default:
                                        /*
                                         *      An octal escape is exactly three octal digits, and fits in one byte.
                                         */
                                        if ((p[0] >= '0') && (p[0] <= '3') &&
                                            (p[1] >= '0') && (p[1] <= '7') &&
                                            (p[2] >= '0') && (p[2] <= '7')) {
                                                *s++ = (char) (((p[0] - '0') << 6) | ((p[1] - '0') << 3) | (p[2] - '0'));
                                                p += 2;
                                        } else
                                                *s++ = *p;
                                        break;
                        }
                        p++;
 
                } else {
                        /*
                         *      Convert backslash-quote to quote, but
                         *      leave everything else alone.
                         */
                        if (p[1] == quote) { /* convert '\'' --> ' */
                                p++;
                        } else {
                                if (buflen < 2) {
                                        fr_strerror_const("Truncated input");
                                        *s = '\0';
                                        return T_INVALID;
                                }
 
                                *(s++) = *(p++);
                        }
                        *(s++) = *(p++);
                }
        }
 
        *s++ = 0;
 
        if (quote) {
                fr_strerror_const("Unterminated string");
                return T_INVALID;
        }
 
done:
        /* Skip whitespace again. */
        fr_skip_whitespace(p);
 
        *ptr = p;
 
        return token;
}
 
/*
 *      Read a "word" - this means we don't honor
 *      tokens as delimiters.
 */
int getword(char const **ptr, char *buf, int buflen, bool unescape)
{
        return getthing(ptr, buf, buflen, false, fr_tokens_table, fr_tokens_table_len, unescape) == T_EOL ? 0 : 1;
}
 
 
/*
 *      Read the next word, use tokens as delimiters.
 */
fr_token_t gettoken(char const **ptr, char *buf, int buflen, bool unescape)
{
        return getthing(ptr, buf, buflen, true, fr_tokens_table, fr_tokens_table_len, unescape);
}
 
/*
 *      Expect an operator.
 */
fr_token_t getop(char const **ptr)
{
        char op[3];
        fr_token_t token;
 
        token = getthing(ptr, op, sizeof(op), true, fr_tokens_table, fr_tokens_table_len, false);
        if (!fr_assignment_op[token] && !fr_comparison_op[token]) {
                fr_strerror_const("Expected operator");
                return T_INVALID;
        }
        return token;
}
 
/*
 *      Expect a string.
 */
fr_token_t getstring(char const **ptr, char *buf, int buflen, bool unescape)
{
        char const *p;
 
        if (!ptr || !*ptr || !buf) return T_INVALID;
 
        p = *ptr;
 
        fr_skip_whitespace(p);
 
        *ptr = p;
 
        if ((*p == '"') || (*p == '\'') || (*p == '`')) {
                return gettoken(ptr, buf, buflen, unescape);
        }
 
        return getthing(ptr, buf, buflen, false, fr_tokens_table, fr_tokens_table_len, unescape);
}
 
char const *fr_token_name(int token)
{
        return fr_table_str_by_value(fr_tokens_table, token, "<INVALID>");
}