Damien George | 04b9147 | 2014-05-03 23:27:38 +0100 | [diff] [blame^] | 1 | /* |
| 2 | * This file is part of the Micro Python project, http://micropython.org/ |
| 3 | * |
| 4 | * The MIT License (MIT) |
| 5 | * |
| 6 | * Copyright (c) 2013, 2014 Damien P. George |
| 7 | * |
| 8 | * Permission is hereby granted, free of charge, to any person obtaining a copy |
| 9 | * of this software and associated documentation files (the "Software"), to deal |
| 10 | * in the Software without restriction, including without limitation the rights |
| 11 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
| 12 | * copies of the Software, and to permit persons to whom the Software is |
| 13 | * furnished to do so, subject to the following conditions: |
| 14 | * |
| 15 | * The above copyright notice and this permission notice shall be included in |
| 16 | * all copies or substantial portions of the Software. |
| 17 | * |
| 18 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 20 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 21 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 22 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 23 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
| 24 | * THE SOFTWARE. |
| 25 | */ |
| 26 | |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 27 | /* lexer.c -- simple tokeniser for Python implementation |
| 28 | */ |
| 29 | |
xbe | efe3422 | 2014-03-16 00:14:26 -0700 | [diff] [blame] | 30 | #include <stdbool.h> |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 31 | #include <stdint.h> |
| 32 | #include <stdio.h> |
| 33 | #include <assert.h> |
| 34 | |
| 35 | #include "misc.h" |
Damien George | 55baff4 | 2014-01-21 21:40:13 +0000 | [diff] [blame] | 36 | #include "mpconfig.h" |
| 37 | #include "qstr.h" |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 38 | #include "lexer.h" |
| 39 | |
| 40 | #define TAB_SIZE (8) |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 41 | |
Damien | 92c0656 | 2013-10-22 22:32:27 +0100 | [diff] [blame] | 42 | // TODO seems that CPython allows NULL byte in the input stream |
| 43 | // don't know if that's intentional or not, but we don't allow it |
| 44 | |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 45 | struct _mp_lexer_t { |
Damien George | 0833500 | 2014-01-18 23:24:36 +0000 | [diff] [blame] | 46 | qstr source_name; // name of source |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 47 | void *stream_data; // data for stream |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 48 | mp_lexer_stream_next_char_t stream_next_char; // stream callback to get next char |
| 49 | mp_lexer_stream_close_t stream_close; // stream callback to free |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 50 | |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 51 | unichar chr0, chr1, chr2; // current cached characters from source |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 52 | |
| 53 | uint line; // source line |
| 54 | uint column; // source column |
| 55 | |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 56 | int emit_dent; // non-zero when there are INDENT/DEDENT tokens to emit |
| 57 | int nested_bracket_level; // >0 when there are nested brackets over multiple lines |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 58 | |
| 59 | uint alloc_indent_level; |
| 60 | uint num_indent_level; |
| 61 | uint16_t *indent_level; |
| 62 | |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 63 | vstr_t vstr; |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 64 | mp_token_t tok_cur; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 65 | }; |
| 66 | |
Damien George | 9528cd6 | 2014-01-15 21:23:31 +0000 | [diff] [blame] | 67 | // TODO replace with a call to a standard function |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 68 | bool str_strn_equal(const char *str, const char *strn, int len) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 69 | uint i = 0; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 70 | |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 71 | while (i < len && *str == *strn) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 72 | ++i; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 73 | ++str; |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 74 | ++strn; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 75 | } |
| 76 | |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 77 | return i == len && *str == 0; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 78 | } |
| 79 | |
Damien George | c596612 | 2014-02-15 16:10:44 +0000 | [diff] [blame] | 80 | #ifdef MICROPY_DEBUG_PRINTERS |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 81 | void mp_token_show(const mp_token_t *tok) { |
Damien George | 0833500 | 2014-01-18 23:24:36 +0000 | [diff] [blame] | 82 | printf("(%d:%d) kind:%d str:%p len:%d", tok->src_line, tok->src_column, tok->kind, tok->str, tok->len); |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 83 | if (tok->str != NULL && tok->len > 0) { |
| 84 | const char *i = tok->str; |
| 85 | const char *j = i + tok->len; |
| 86 | printf(" "); |
| 87 | while (i < j) { |
Damien George | 8cc96a3 | 2013-12-30 18:23:50 +0000 | [diff] [blame] | 88 | unichar c = utf8_get_char(i); |
| 89 | i = utf8_next_char(i); |
| 90 | if (unichar_isprint(c)) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 91 | printf("%c", c); |
| 92 | } else { |
| 93 | printf("?"); |
| 94 | } |
| 95 | } |
| 96 | } |
| 97 | printf("\n"); |
| 98 | } |
Damien George | c596612 | 2014-02-15 16:10:44 +0000 | [diff] [blame] | 99 | #endif |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 100 | |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 101 | #define CUR_CHAR(lex) ((lex)->chr0) |
| 102 | |
Paul Sokolovsky | 520e2f5 | 2014-02-12 18:31:30 +0200 | [diff] [blame] | 103 | STATIC bool is_end(mp_lexer_t *lex) { |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 104 | return lex->chr0 == MP_LEXER_CHAR_EOF; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 105 | } |
| 106 | |
Paul Sokolovsky | 520e2f5 | 2014-02-12 18:31:30 +0200 | [diff] [blame] | 107 | STATIC bool is_physical_newline(mp_lexer_t *lex) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 108 | return lex->chr0 == '\n' || lex->chr0 == '\r'; |
| 109 | } |
| 110 | |
Paul Sokolovsky | 520e2f5 | 2014-02-12 18:31:30 +0200 | [diff] [blame] | 111 | STATIC bool is_char(mp_lexer_t *lex, char c) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 112 | return lex->chr0 == c; |
| 113 | } |
| 114 | |
Paul Sokolovsky | 520e2f5 | 2014-02-12 18:31:30 +0200 | [diff] [blame] | 115 | STATIC bool is_char_or(mp_lexer_t *lex, char c1, char c2) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 116 | return lex->chr0 == c1 || lex->chr0 == c2; |
| 117 | } |
| 118 | |
Paul Sokolovsky | 520e2f5 | 2014-02-12 18:31:30 +0200 | [diff] [blame] | 119 | STATIC bool is_char_or3(mp_lexer_t *lex, char c1, char c2, char c3) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 120 | return lex->chr0 == c1 || lex->chr0 == c2 || lex->chr0 == c3; |
| 121 | } |
| 122 | |
| 123 | /* |
Paul Sokolovsky | 520e2f5 | 2014-02-12 18:31:30 +0200 | [diff] [blame] | 124 | STATIC bool is_char_following(mp_lexer_t *lex, char c) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 125 | return lex->chr1 == c; |
| 126 | } |
| 127 | */ |
| 128 | |
Paul Sokolovsky | 520e2f5 | 2014-02-12 18:31:30 +0200 | [diff] [blame] | 129 | STATIC bool is_char_following_or(mp_lexer_t *lex, char c1, char c2) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 130 | return lex->chr1 == c1 || lex->chr1 == c2; |
| 131 | } |
| 132 | |
Paul Sokolovsky | 520e2f5 | 2014-02-12 18:31:30 +0200 | [diff] [blame] | 133 | STATIC bool is_char_following_following_or(mp_lexer_t *lex, char c1, char c2) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 134 | return lex->chr2 == c1 || lex->chr2 == c2; |
| 135 | } |
| 136 | |
Paul Sokolovsky | 520e2f5 | 2014-02-12 18:31:30 +0200 | [diff] [blame] | 137 | STATIC bool is_char_and(mp_lexer_t *lex, char c1, char c2) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 138 | return lex->chr0 == c1 && lex->chr1 == c2; |
| 139 | } |
| 140 | |
Paul Sokolovsky | 520e2f5 | 2014-02-12 18:31:30 +0200 | [diff] [blame] | 141 | STATIC bool is_whitespace(mp_lexer_t *lex) { |
Damien George | 8cc96a3 | 2013-12-30 18:23:50 +0000 | [diff] [blame] | 142 | return unichar_isspace(lex->chr0); |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 143 | } |
| 144 | |
Paul Sokolovsky | 520e2f5 | 2014-02-12 18:31:30 +0200 | [diff] [blame] | 145 | STATIC bool is_letter(mp_lexer_t *lex) { |
Damien George | 8cc96a3 | 2013-12-30 18:23:50 +0000 | [diff] [blame] | 146 | return unichar_isalpha(lex->chr0); |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 147 | } |
| 148 | |
Paul Sokolovsky | 520e2f5 | 2014-02-12 18:31:30 +0200 | [diff] [blame] | 149 | STATIC bool is_digit(mp_lexer_t *lex) { |
Damien George | 8cc96a3 | 2013-12-30 18:23:50 +0000 | [diff] [blame] | 150 | return unichar_isdigit(lex->chr0); |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 151 | } |
| 152 | |
Paul Sokolovsky | 520e2f5 | 2014-02-12 18:31:30 +0200 | [diff] [blame] | 153 | STATIC bool is_following_digit(mp_lexer_t *lex) { |
Damien George | 8cc96a3 | 2013-12-30 18:23:50 +0000 | [diff] [blame] | 154 | return unichar_isdigit(lex->chr1); |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 155 | } |
| 156 | |
Paul Sokolovsky | 520e2f5 | 2014-02-12 18:31:30 +0200 | [diff] [blame] | 157 | STATIC bool is_following_odigit(mp_lexer_t *lex) { |
Paul Sokolovsky | 0b7184d | 2014-01-22 22:40:02 +0200 | [diff] [blame] | 158 | return lex->chr1 >= '0' && lex->chr1 <= '7'; |
| 159 | } |
| 160 | |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 161 | // TODO UNICODE include unicode characters in definition of identifiers |
Paul Sokolovsky | 520e2f5 | 2014-02-12 18:31:30 +0200 | [diff] [blame] | 162 | STATIC bool is_head_of_identifier(mp_lexer_t *lex) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 163 | return is_letter(lex) || lex->chr0 == '_'; |
| 164 | } |
| 165 | |
| 166 | // TODO UNICODE include unicode characters in definition of identifiers |
Paul Sokolovsky | 520e2f5 | 2014-02-12 18:31:30 +0200 | [diff] [blame] | 167 | STATIC bool is_tail_of_identifier(mp_lexer_t *lex) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 168 | return is_head_of_identifier(lex) || is_digit(lex); |
| 169 | } |
| 170 | |
Paul Sokolovsky | 520e2f5 | 2014-02-12 18:31:30 +0200 | [diff] [blame] | 171 | STATIC void next_char(mp_lexer_t *lex) { |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 172 | if (lex->chr0 == MP_LEXER_CHAR_EOF) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 173 | return; |
| 174 | } |
| 175 | |
| 176 | int advance = 1; |
| 177 | |
| 178 | if (lex->chr0 == '\n') { |
| 179 | // LF is a new line |
| 180 | ++lex->line; |
| 181 | lex->column = 1; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 182 | } else if (lex->chr0 == '\r') { |
| 183 | // CR is a new line |
| 184 | ++lex->line; |
| 185 | lex->column = 1; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 186 | if (lex->chr1 == '\n') { |
| 187 | // CR LF is a single new line |
| 188 | advance = 2; |
| 189 | } |
| 190 | } else if (lex->chr0 == '\t') { |
| 191 | // a tab |
| 192 | lex->column = (((lex->column - 1 + TAB_SIZE) / TAB_SIZE) * TAB_SIZE) + 1; |
| 193 | } else { |
| 194 | // a character worth one column |
| 195 | ++lex->column; |
| 196 | } |
| 197 | |
| 198 | for (; advance > 0; advance--) { |
| 199 | lex->chr0 = lex->chr1; |
| 200 | lex->chr1 = lex->chr2; |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 201 | lex->chr2 = lex->stream_next_char(lex->stream_data); |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 202 | if (lex->chr2 == MP_LEXER_CHAR_EOF) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 203 | // EOF |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 204 | if (lex->chr1 != MP_LEXER_CHAR_EOF && lex->chr1 != '\n' && lex->chr1 != '\r') { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 205 | lex->chr2 = '\n'; // insert newline at end of file |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 206 | } |
| 207 | } |
| 208 | } |
| 209 | } |
| 210 | |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 211 | void indent_push(mp_lexer_t *lex, uint indent) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 212 | if (lex->num_indent_level >= lex->alloc_indent_level) { |
Damien | 732407f | 2013-12-29 19:33:23 +0000 | [diff] [blame] | 213 | lex->indent_level = m_renew(uint16_t, lex->indent_level, lex->alloc_indent_level, lex->alloc_indent_level * 2); |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 214 | lex->alloc_indent_level *= 2; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 215 | } |
| 216 | lex->indent_level[lex->num_indent_level++] = indent; |
| 217 | } |
| 218 | |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 219 | uint indent_top(mp_lexer_t *lex) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 220 | return lex->indent_level[lex->num_indent_level - 1]; |
| 221 | } |
| 222 | |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 223 | void indent_pop(mp_lexer_t *lex) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 224 | lex->num_indent_level -= 1; |
| 225 | } |
| 226 | |
| 227 | // some tricky operator encoding: |
| 228 | // <op> = begin with <op>, if this opchar matches then begin here |
| 229 | // e<op> = end with <op>, if this opchar matches then end |
| 230 | // E<op> = mandatory end with <op>, this opchar must match, then end |
| 231 | // c<op> = continue with <op>, if this opchar matches then continue matching |
| 232 | // this means if the start of two ops are the same then they are equal til the last char |
| 233 | |
Paul Sokolovsky | 520e2f5 | 2014-02-12 18:31:30 +0200 | [diff] [blame] | 234 | STATIC const char *tok_enc = |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 235 | "()[]{},:;@~" // singles |
| 236 | "<e=c<e=" // < <= << <<= |
| 237 | ">e=c>e=" // > >= >> >>= |
| 238 | "*e=c*e=" // * *= ** **= |
| 239 | "+e=" // + += |
| 240 | "-e=e>" // - -= -> |
| 241 | "&e=" // & &= |
| 242 | "|e=" // | |= |
| 243 | "/e=c/e=" // / /= // //= |
| 244 | "%e=" // % %= |
| 245 | "^e=" // ^ ^= |
| 246 | "=e=" // = == |
Damien George | 2e9eb2d | 2014-04-10 12:19:33 +0100 | [diff] [blame] | 247 | "!E="; // != |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 248 | |
| 249 | // TODO static assert that number of tokens is less than 256 so we can safely make this table with byte sized entries |
Paul Sokolovsky | 520e2f5 | 2014-02-12 18:31:30 +0200 | [diff] [blame] | 250 | STATIC const uint8_t tok_enc_kind[] = { |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 251 | MP_TOKEN_DEL_PAREN_OPEN, MP_TOKEN_DEL_PAREN_CLOSE, |
| 252 | MP_TOKEN_DEL_BRACKET_OPEN, MP_TOKEN_DEL_BRACKET_CLOSE, |
| 253 | MP_TOKEN_DEL_BRACE_OPEN, MP_TOKEN_DEL_BRACE_CLOSE, |
| 254 | MP_TOKEN_DEL_COMMA, MP_TOKEN_DEL_COLON, MP_TOKEN_DEL_SEMICOLON, MP_TOKEN_DEL_AT, MP_TOKEN_OP_TILDE, |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 255 | |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 256 | MP_TOKEN_OP_LESS, MP_TOKEN_OP_LESS_EQUAL, MP_TOKEN_OP_DBL_LESS, MP_TOKEN_DEL_DBL_LESS_EQUAL, |
| 257 | MP_TOKEN_OP_MORE, MP_TOKEN_OP_MORE_EQUAL, MP_TOKEN_OP_DBL_MORE, MP_TOKEN_DEL_DBL_MORE_EQUAL, |
| 258 | MP_TOKEN_OP_STAR, MP_TOKEN_DEL_STAR_EQUAL, MP_TOKEN_OP_DBL_STAR, MP_TOKEN_DEL_DBL_STAR_EQUAL, |
| 259 | MP_TOKEN_OP_PLUS, MP_TOKEN_DEL_PLUS_EQUAL, |
| 260 | MP_TOKEN_OP_MINUS, MP_TOKEN_DEL_MINUS_EQUAL, MP_TOKEN_DEL_MINUS_MORE, |
| 261 | MP_TOKEN_OP_AMPERSAND, MP_TOKEN_DEL_AMPERSAND_EQUAL, |
| 262 | MP_TOKEN_OP_PIPE, MP_TOKEN_DEL_PIPE_EQUAL, |
| 263 | MP_TOKEN_OP_SLASH, MP_TOKEN_DEL_SLASH_EQUAL, MP_TOKEN_OP_DBL_SLASH, MP_TOKEN_DEL_DBL_SLASH_EQUAL, |
| 264 | MP_TOKEN_OP_PERCENT, MP_TOKEN_DEL_PERCENT_EQUAL, |
| 265 | MP_TOKEN_OP_CARET, MP_TOKEN_DEL_CARET_EQUAL, |
| 266 | MP_TOKEN_DEL_EQUAL, MP_TOKEN_OP_DBL_EQUAL, |
| 267 | MP_TOKEN_OP_NOT_EQUAL, |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 268 | }; |
| 269 | |
| 270 | // must have the same order as enum in lexer.h |
Paul Sokolovsky | 520e2f5 | 2014-02-12 18:31:30 +0200 | [diff] [blame] | 271 | STATIC const char *tok_kw[] = { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 272 | "False", |
| 273 | "None", |
| 274 | "True", |
| 275 | "and", |
| 276 | "as", |
| 277 | "assert", |
| 278 | "break", |
| 279 | "class", |
| 280 | "continue", |
| 281 | "def", |
| 282 | "del", |
| 283 | "elif", |
| 284 | "else", |
| 285 | "except", |
| 286 | "finally", |
| 287 | "for", |
| 288 | "from", |
| 289 | "global", |
| 290 | "if", |
| 291 | "import", |
| 292 | "in", |
| 293 | "is", |
| 294 | "lambda", |
| 295 | "nonlocal", |
| 296 | "not", |
| 297 | "or", |
| 298 | "pass", |
| 299 | "raise", |
| 300 | "return", |
| 301 | "try", |
| 302 | "while", |
| 303 | "with", |
| 304 | "yield", |
| 305 | NULL, |
| 306 | }; |
| 307 | |
Paul Sokolovsky | 520e2f5 | 2014-02-12 18:31:30 +0200 | [diff] [blame] | 308 | STATIC int hex_digit(unichar c) { |
Paul Sokolovsky | 0b7184d | 2014-01-22 22:40:02 +0200 | [diff] [blame] | 309 | // c is assumed to be hex digit |
| 310 | int n = c - '0'; |
| 311 | if (n > 9) { |
| 312 | n &= ~('a' - 'A'); |
| 313 | n -= ('A' - ('9' + 1)); |
| 314 | } |
| 315 | return n; |
| 316 | } |
| 317 | |
| 318 | // This is called with CUR_CHAR() before first hex digit, and should return with |
| 319 | // it pointing to last hex digit |
Paul Sokolovsky | 520e2f5 | 2014-02-12 18:31:30 +0200 | [diff] [blame] | 320 | STATIC bool get_hex(mp_lexer_t *lex, int num_digits, uint *result) { |
Paul Sokolovsky | 0b7184d | 2014-01-22 22:40:02 +0200 | [diff] [blame] | 321 | uint num = 0; |
| 322 | while (num_digits-- != 0) { |
| 323 | next_char(lex); |
| 324 | unichar c = CUR_CHAR(lex); |
| 325 | if (!unichar_isxdigit(c)) { |
| 326 | return false; |
| 327 | } |
| 328 | num = (num << 4) + hex_digit(c); |
| 329 | } |
| 330 | *result = num; |
| 331 | return true; |
| 332 | } |
| 333 | |
Paul Sokolovsky | 520e2f5 | 2014-02-12 18:31:30 +0200 | [diff] [blame] | 334 | STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, mp_token_t *tok, bool first_token) { |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 335 | // skip white space and comments |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 336 | bool had_physical_newline = false; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 337 | while (!is_end(lex)) { |
| 338 | if (is_physical_newline(lex)) { |
| 339 | had_physical_newline = true; |
| 340 | next_char(lex); |
| 341 | } else if (is_whitespace(lex)) { |
| 342 | next_char(lex); |
| 343 | } else if (is_char(lex, '#')) { |
| 344 | next_char(lex); |
| 345 | while (!is_end(lex) && !is_physical_newline(lex)) { |
| 346 | next_char(lex); |
| 347 | } |
| 348 | // had_physical_newline will be set on next loop |
| 349 | } else if (is_char(lex, '\\')) { |
| 350 | // backslash (outside string literals) must appear just before a physical newline |
| 351 | next_char(lex); |
| 352 | if (!is_physical_newline(lex)) { |
Damien George | 69a818d | 2014-01-12 13:55:24 +0000 | [diff] [blame] | 353 | // SyntaxError: unexpected character after line continuation character |
Damien George | 69a818d | 2014-01-12 13:55:24 +0000 | [diff] [blame] | 354 | tok->src_line = lex->line; |
| 355 | tok->src_column = lex->column; |
| 356 | tok->kind = MP_TOKEN_BAD_LINE_CONTINUATION; |
| 357 | vstr_reset(&lex->vstr); |
| 358 | tok->str = vstr_str(&lex->vstr); |
| 359 | tok->len = 0; |
| 360 | return; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 361 | } else { |
| 362 | next_char(lex); |
| 363 | } |
| 364 | } else { |
| 365 | break; |
| 366 | } |
| 367 | } |
| 368 | |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 369 | // set token source information |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 370 | tok->src_line = lex->line; |
| 371 | tok->src_column = lex->column; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 372 | |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 373 | // start new token text |
| 374 | vstr_reset(&lex->vstr); |
| 375 | |
| 376 | if (first_token && lex->line == 1 && lex->column != 1) { |
| 377 | // check that the first token is in the first column |
| 378 | // if first token is not on first line, we get a physical newline and |
| 379 | // this check is done as part of normal indent/dedent checking below |
| 380 | // (done to get equivalence with CPython) |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 381 | tok->kind = MP_TOKEN_INDENT; |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 382 | |
| 383 | } else if (lex->emit_dent < 0) { |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 384 | tok->kind = MP_TOKEN_DEDENT; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 385 | lex->emit_dent += 1; |
| 386 | |
| 387 | } else if (lex->emit_dent > 0) { |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 388 | tok->kind = MP_TOKEN_INDENT; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 389 | lex->emit_dent -= 1; |
| 390 | |
Damien | 91d387d | 2013-10-09 15:09:52 +0100 | [diff] [blame] | 391 | } else if (had_physical_newline && lex->nested_bracket_level == 0) { |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 392 | tok->kind = MP_TOKEN_NEWLINE; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 393 | |
| 394 | uint num_spaces = lex->column - 1; |
| 395 | lex->emit_dent = 0; |
| 396 | if (num_spaces == indent_top(lex)) { |
| 397 | } else if (num_spaces > indent_top(lex)) { |
| 398 | indent_push(lex, num_spaces); |
| 399 | lex->emit_dent += 1; |
| 400 | } else { |
| 401 | while (num_spaces < indent_top(lex)) { |
| 402 | indent_pop(lex); |
| 403 | lex->emit_dent -= 1; |
| 404 | } |
| 405 | if (num_spaces != indent_top(lex)) { |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 406 | tok->kind = MP_TOKEN_DEDENT_MISMATCH; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 407 | } |
| 408 | } |
| 409 | |
| 410 | } else if (is_end(lex)) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 411 | if (indent_top(lex) > 0) { |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 412 | tok->kind = MP_TOKEN_NEWLINE; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 413 | lex->emit_dent = 0; |
| 414 | while (indent_top(lex) > 0) { |
| 415 | indent_pop(lex); |
| 416 | lex->emit_dent -= 1; |
| 417 | } |
| 418 | } else { |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 419 | tok->kind = MP_TOKEN_END; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 420 | } |
| 421 | |
| 422 | } else if (is_char_or(lex, '\'', '\"') |
| 423 | || (is_char_or3(lex, 'r', 'u', 'b') && is_char_following_or(lex, '\'', '\"')) |
| 424 | || ((is_char_and(lex, 'r', 'b') || is_char_and(lex, 'b', 'r')) && is_char_following_following_or(lex, '\'', '\"'))) { |
| 425 | // a string or bytes literal |
| 426 | |
| 427 | // parse type codes |
| 428 | bool is_raw = false; |
| 429 | bool is_bytes = false; |
| 430 | if (is_char(lex, 'u')) { |
| 431 | next_char(lex); |
| 432 | } else if (is_char(lex, 'b')) { |
| 433 | is_bytes = true; |
| 434 | next_char(lex); |
| 435 | if (is_char(lex, 'r')) { |
| 436 | is_raw = true; |
| 437 | next_char(lex); |
| 438 | } |
| 439 | } else if (is_char(lex, 'r')) { |
| 440 | is_raw = true; |
| 441 | next_char(lex); |
| 442 | if (is_char(lex, 'b')) { |
| 443 | is_bytes = true; |
| 444 | next_char(lex); |
| 445 | } |
| 446 | } |
| 447 | |
| 448 | // set token kind |
| 449 | if (is_bytes) { |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 450 | tok->kind = MP_TOKEN_BYTES; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 451 | } else { |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 452 | tok->kind = MP_TOKEN_STRING; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 453 | } |
| 454 | |
| 455 | // get first quoting character |
| 456 | char quote_char = '\''; |
| 457 | if (is_char(lex, '\"')) { |
| 458 | quote_char = '\"'; |
| 459 | } |
| 460 | next_char(lex); |
| 461 | |
| 462 | // work out if it's a single or triple quoted literal |
| 463 | int num_quotes; |
| 464 | if (is_char_and(lex, quote_char, quote_char)) { |
| 465 | // triple quotes |
| 466 | next_char(lex); |
| 467 | next_char(lex); |
| 468 | num_quotes = 3; |
| 469 | } else { |
| 470 | // single quotes |
| 471 | num_quotes = 1; |
| 472 | } |
| 473 | |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 474 | // parse the literal |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 475 | int n_closing = 0; |
| 476 | while (!is_end(lex) && (num_quotes > 1 || !is_char(lex, '\n')) && n_closing < num_quotes) { |
| 477 | if (is_char(lex, quote_char)) { |
| 478 | n_closing += 1; |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 479 | vstr_add_char(&lex->vstr, CUR_CHAR(lex)); |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 480 | } else { |
| 481 | n_closing = 0; |
Damien George | a91f414 | 2014-04-10 11:30:55 +0100 | [diff] [blame] | 482 | if (is_char(lex, '\\')) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 483 | next_char(lex); |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 484 | unichar c = CUR_CHAR(lex); |
Damien George | a91f414 | 2014-04-10 11:30:55 +0100 | [diff] [blame] | 485 | if (is_raw) { |
| 486 | // raw strings allow escaping of quotes, but the backslash is also emitted |
| 487 | vstr_add_char(&lex->vstr, '\\'); |
| 488 | } else { |
| 489 | switch (c) { |
| 490 | case MP_LEXER_CHAR_EOF: break; // TODO a proper error message? |
| 491 | case '\n': c = MP_LEXER_CHAR_EOF; break; // TODO check this works correctly (we are supposed to ignore it |
| 492 | case '\\': break; |
| 493 | case '\'': break; |
| 494 | case '"': break; |
| 495 | case 'a': c = 0x07; break; |
| 496 | case 'b': c = 0x08; break; |
| 497 | case 't': c = 0x09; break; |
| 498 | case 'n': c = 0x0a; break; |
| 499 | case 'v': c = 0x0b; break; |
| 500 | case 'f': c = 0x0c; break; |
| 501 | case 'r': c = 0x0d; break; |
| 502 | case 'x': |
| 503 | { |
| 504 | uint num = 0; |
| 505 | if (!get_hex(lex, 2, &num)) { |
| 506 | // TODO error message |
| 507 | assert(0); |
Paul Sokolovsky | 0b7184d | 2014-01-22 22:40:02 +0200 | [diff] [blame] | 508 | } |
| 509 | c = num; |
Damien George | a91f414 | 2014-04-10 11:30:55 +0100 | [diff] [blame] | 510 | break; |
Paul Sokolovsky | 0b7184d | 2014-01-22 22:40:02 +0200 | [diff] [blame] | 511 | } |
Damien George | a91f414 | 2014-04-10 11:30:55 +0100 | [diff] [blame] | 512 | case 'N': break; // TODO \N{name} only in strings |
| 513 | case 'u': break; // TODO \uxxxx only in strings |
| 514 | case 'U': break; // TODO \Uxxxxxxxx only in strings |
| 515 | default: |
| 516 | if (c >= '0' && c <= '7') { |
| 517 | // Octal sequence, 1-3 chars |
| 518 | int digits = 3; |
| 519 | int num = c - '0'; |
| 520 | while (is_following_odigit(lex) && --digits != 0) { |
| 521 | next_char(lex); |
| 522 | num = num * 8 + (CUR_CHAR(lex) - '0'); |
| 523 | } |
| 524 | c = num; |
| 525 | } else { |
| 526 | // unrecognised escape character; CPython lets this through verbatim as '\' and then the character |
| 527 | vstr_add_char(&lex->vstr, '\\'); |
| 528 | } |
| 529 | break; |
| 530 | } |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 531 | } |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 532 | if (c != MP_LEXER_CHAR_EOF) { |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 533 | vstr_add_char(&lex->vstr, c); |
| 534 | } |
| 535 | } else { |
| 536 | vstr_add_char(&lex->vstr, CUR_CHAR(lex)); |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 537 | } |
| 538 | } |
| 539 | next_char(lex); |
| 540 | } |
| 541 | |
| 542 | // check we got the required end quotes |
| 543 | if (n_closing < num_quotes) { |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 544 | tok->kind = MP_TOKEN_LONELY_STRING_OPEN; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 545 | } |
| 546 | |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 547 | // cut off the end quotes from the token text |
Damien George | 280e720 | 2014-03-15 14:33:09 +0000 | [diff] [blame] | 548 | vstr_cut_tail_bytes(&lex->vstr, n_closing); |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 549 | |
| 550 | } else if (is_head_of_identifier(lex)) { |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 551 | tok->kind = MP_TOKEN_NAME; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 552 | |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 553 | // get first char |
| 554 | vstr_add_char(&lex->vstr, CUR_CHAR(lex)); |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 555 | next_char(lex); |
| 556 | |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 557 | // get tail chars |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 558 | while (!is_end(lex) && is_tail_of_identifier(lex)) { |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 559 | vstr_add_char(&lex->vstr, CUR_CHAR(lex)); |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 560 | next_char(lex); |
| 561 | } |
| 562 | |
| 563 | } else if (is_digit(lex) || (is_char(lex, '.') && is_following_digit(lex))) { |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 564 | tok->kind = MP_TOKEN_NUMBER; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 565 | |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 566 | // get first char |
| 567 | vstr_add_char(&lex->vstr, CUR_CHAR(lex)); |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 568 | next_char(lex); |
| 569 | |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 570 | // get tail chars |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 571 | while (!is_end(lex)) { |
| 572 | if (is_char_or(lex, 'e', 'E')) { |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 573 | vstr_add_char(&lex->vstr, 'e'); |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 574 | next_char(lex); |
| 575 | if (is_char(lex, '+') || is_char(lex, '-')) { |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 576 | vstr_add_char(&lex->vstr, CUR_CHAR(lex)); |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 577 | next_char(lex); |
| 578 | } |
| 579 | } else if (is_letter(lex) || is_digit(lex) || is_char_or(lex, '_', '.')) { |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 580 | vstr_add_char(&lex->vstr, CUR_CHAR(lex)); |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 581 | next_char(lex); |
| 582 | } else { |
| 583 | break; |
| 584 | } |
| 585 | } |
| 586 | |
Damien George | 2e9eb2d | 2014-04-10 12:19:33 +0100 | [diff] [blame] | 587 | } else if (is_char(lex, '.')) { |
| 588 | // special handling for . and ... operators, because .. is not a valid operator |
| 589 | |
| 590 | // get first char |
| 591 | vstr_add_char(&lex->vstr, '.'); |
| 592 | next_char(lex); |
| 593 | |
| 594 | if (is_char_and(lex, '.', '.')) { |
| 595 | vstr_add_char(&lex->vstr, '.'); |
| 596 | vstr_add_char(&lex->vstr, '.'); |
| 597 | next_char(lex); |
| 598 | next_char(lex); |
| 599 | tok->kind = MP_TOKEN_ELLIPSIS; |
| 600 | } else { |
| 601 | tok->kind = MP_TOKEN_DEL_PERIOD; |
| 602 | } |
| 603 | |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 604 | } else { |
| 605 | // search for encoded delimiter or operator |
| 606 | |
| 607 | const char *t = tok_enc; |
| 608 | uint tok_enc_index = 0; |
| 609 | for (; *t != 0 && !is_char(lex, *t); t += 1) { |
| 610 | if (*t == 'e' || *t == 'c') { |
| 611 | t += 1; |
| 612 | } else if (*t == 'E') { |
| 613 | tok_enc_index -= 1; |
| 614 | t += 1; |
| 615 | } |
| 616 | tok_enc_index += 1; |
| 617 | } |
| 618 | |
| 619 | next_char(lex); |
| 620 | |
| 621 | if (*t == 0) { |
| 622 | // didn't match any delimiter or operator characters |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 623 | tok->kind = MP_TOKEN_INVALID; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 624 | |
| 625 | } else { |
| 626 | // matched a delimiter or operator character |
| 627 | |
| 628 | // get the maximum characters for a valid token |
| 629 | t += 1; |
| 630 | uint t_index = tok_enc_index; |
| 631 | for (;;) { |
| 632 | for (; *t == 'e'; t += 1) { |
| 633 | t += 1; |
| 634 | t_index += 1; |
| 635 | if (is_char(lex, *t)) { |
| 636 | next_char(lex); |
| 637 | tok_enc_index = t_index; |
| 638 | break; |
| 639 | } |
| 640 | } |
| 641 | |
| 642 | if (*t == 'E') { |
| 643 | t += 1; |
| 644 | if (is_char(lex, *t)) { |
| 645 | next_char(lex); |
| 646 | tok_enc_index = t_index; |
| 647 | } else { |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 648 | tok->kind = MP_TOKEN_INVALID; |
Damien George | 2e9eb2d | 2014-04-10 12:19:33 +0100 | [diff] [blame] | 649 | goto tok_enc_no_match; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 650 | } |
| 651 | break; |
| 652 | } |
| 653 | |
| 654 | if (*t == 'c') { |
| 655 | t += 1; |
| 656 | t_index += 1; |
| 657 | if (is_char(lex, *t)) { |
| 658 | next_char(lex); |
| 659 | tok_enc_index = t_index; |
| 660 | t += 1; |
| 661 | } else { |
| 662 | break; |
| 663 | } |
| 664 | } else { |
| 665 | break; |
| 666 | } |
| 667 | } |
| 668 | |
| 669 | // set token kind |
| 670 | tok->kind = tok_enc_kind[tok_enc_index]; |
| 671 | |
Damien George | 2e9eb2d | 2014-04-10 12:19:33 +0100 | [diff] [blame] | 672 | tok_enc_no_match: |
| 673 | |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 674 | // compute bracket level for implicit line joining |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 675 | if (tok->kind == MP_TOKEN_DEL_PAREN_OPEN || tok->kind == MP_TOKEN_DEL_BRACKET_OPEN || tok->kind == MP_TOKEN_DEL_BRACE_OPEN) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 676 | lex->nested_bracket_level += 1; |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 677 | } else if (tok->kind == MP_TOKEN_DEL_PAREN_CLOSE || tok->kind == MP_TOKEN_DEL_BRACKET_CLOSE || tok->kind == MP_TOKEN_DEL_BRACE_CLOSE) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 678 | lex->nested_bracket_level -= 1; |
| 679 | } |
| 680 | } |
| 681 | } |
| 682 | |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 683 | // point token text to vstr buffer |
| 684 | tok->str = vstr_str(&lex->vstr); |
| 685 | tok->len = vstr_len(&lex->vstr); |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 686 | |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 687 | // check for keywords |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 688 | if (tok->kind == MP_TOKEN_NAME) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 689 | for (int i = 0; tok_kw[i] != NULL; i++) { |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 690 | if (str_strn_equal(tok_kw[i], tok->str, tok->len)) { |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 691 | tok->kind = MP_TOKEN_KW_FALSE + i; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 692 | break; |
| 693 | } |
| 694 | } |
| 695 | } |
| 696 | } |
| 697 | |
Damien George | b829b5c | 2014-01-25 13:51:19 +0000 | [diff] [blame] | 698 | mp_lexer_t *mp_lexer_new(qstr src_name, void *stream_data, mp_lexer_stream_next_char_t stream_next_char, mp_lexer_stream_close_t stream_close) { |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 699 | mp_lexer_t *lex = m_new(mp_lexer_t, 1); |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 700 | |
Damien George | b829b5c | 2014-01-25 13:51:19 +0000 | [diff] [blame] | 701 | lex->source_name = src_name; |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 702 | lex->stream_data = stream_data; |
| 703 | lex->stream_next_char = stream_next_char; |
Damien | fa2162b | 2013-10-20 17:42:00 +0100 | [diff] [blame] | 704 | lex->stream_close = stream_close; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 705 | lex->line = 1; |
| 706 | lex->column = 1; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 707 | lex->emit_dent = 0; |
| 708 | lex->nested_bracket_level = 0; |
| 709 | lex->alloc_indent_level = 16; |
| 710 | lex->num_indent_level = 1; |
| 711 | lex->indent_level = m_new(uint16_t, lex->alloc_indent_level); |
| 712 | lex->indent_level[0] = 0; |
Paul Sokolovsky | 5d2499c | 2014-01-13 23:15:23 +0200 | [diff] [blame] | 713 | vstr_init(&lex->vstr, 32); |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 714 | |
| 715 | // preload characters |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 716 | lex->chr0 = stream_next_char(stream_data); |
| 717 | lex->chr1 = stream_next_char(stream_data); |
| 718 | lex->chr2 = stream_next_char(stream_data); |
| 719 | |
| 720 | // if input stream is 0, 1 or 2 characters long and doesn't end in a newline, then insert a newline at the end |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 721 | if (lex->chr0 == MP_LEXER_CHAR_EOF) { |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 722 | lex->chr0 = '\n'; |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 723 | } else if (lex->chr1 == MP_LEXER_CHAR_EOF) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 724 | if (lex->chr0 != '\n' && lex->chr0 != '\r') { |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 725 | lex->chr1 = '\n'; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 726 | } |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 727 | } else if (lex->chr2 == MP_LEXER_CHAR_EOF) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 728 | if (lex->chr1 != '\n' && lex->chr1 != '\r') { |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 729 | lex->chr2 = '\n'; |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 730 | } |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 731 | } |
| 732 | |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 733 | // preload first token |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 734 | mp_lexer_next_token_into(lex, &lex->tok_cur, true); |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 735 | |
| 736 | return lex; |
| 737 | } |
| 738 | |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 739 | void mp_lexer_free(mp_lexer_t *lex) { |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 740 | if (lex) { |
Damien | fa2162b | 2013-10-20 17:42:00 +0100 | [diff] [blame] | 741 | if (lex->stream_close) { |
| 742 | lex->stream_close(lex->stream_data); |
Damien | a5185f4 | 2013-10-20 14:41:27 +0100 | [diff] [blame] | 743 | } |
Damien | bb5316b | 2013-10-22 21:12:29 +0100 | [diff] [blame] | 744 | vstr_clear(&lex->vstr); |
Paul Sokolovsky | 624ed5d | 2014-01-23 22:25:57 +0200 | [diff] [blame] | 745 | m_del(uint16_t, lex->indent_level, lex->alloc_indent_level); |
Damien | 732407f | 2013-12-29 19:33:23 +0000 | [diff] [blame] | 746 | m_del_obj(mp_lexer_t, lex); |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 747 | } |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 748 | } |
| 749 | |
Damien George | 0833500 | 2014-01-18 23:24:36 +0000 | [diff] [blame] | 750 | qstr mp_lexer_source_name(mp_lexer_t *lex) { |
| 751 | return lex->source_name; |
| 752 | } |
| 753 | |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 754 | void mp_lexer_to_next(mp_lexer_t *lex) { |
| 755 | mp_lexer_next_token_into(lex, &lex->tok_cur, false); |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 756 | } |
| 757 | |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 758 | const mp_token_t *mp_lexer_cur(const mp_lexer_t *lex) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 759 | return &lex->tok_cur; |
| 760 | } |
| 761 | |
Damien | d99b052 | 2013-12-21 18:17:45 +0000 | [diff] [blame] | 762 | bool mp_lexer_is_kind(mp_lexer_t *lex, mp_token_kind_t kind) { |
Damien | 429d719 | 2013-10-04 19:53:11 +0100 | [diff] [blame] | 763 | return lex->tok_cur.kind == kind; |
| 764 | } |