Usermode/Libraries/libspiderscript.so_src/lex.c

   1 /*
   2  * SpiderScript
   3  * - Script Lexer
   4  */
   5 #include "tokens.h"
   6 #include <stdlib.h>
   7 #include <stdio.h>
   8 #include <string.h>
   9
  10 #define USE_SCOPE_CHAR  0
  11
  12 #define DEBUG   0
  13
  14 #define ARRAY_SIZE(x)   ((sizeof(x))/(sizeof((x)[0])))
  15
  16 // === PROTOTYPES ===
  17  int    is_ident(char ch);
  18  int    isdigit(int ch);
  19  int    isspace(int ch);
  20  int    GetToken(tParser *File);
  21
  22 // === CONSTANTS ===
  23 const struct {
  24         const  int      Value;
  25         const char      *Name;
  26 } csaReservedWords[] = {
  27         {TOK_RWD_FUNCTION, "function"},
  28
  29         {TOK_RWD_RETURN, "return"},
  30         {TOK_RWD_NEW, "new"},
  31
  32         {TOK_RWD_IF, "if"},
  33         {TOK_RWD_ELSE, "else"},
  34         {TOK_RWD_DO, "do"},
  35         {TOK_RWD_WHILE, "while"},
  36         {TOK_RWD_FOR, "for"},
  37
  38         {TOK_RWD_VOID, "void"},
  39         {TOK_RWD_OBJECT, "Object"},
  40         {TOK_RWD_OPAQUE, "Opaque"},
  41         {TOK_RWD_INTEGER, "Integer"},
  42         {TOK_RWD_REAL, "Real"},
  43         {TOK_RWD_STRING, "String"}
  44 };
  45
  46 // === CODE ===
  47 /**
  48  * \brief Read a token from a buffer
  49  * \param File  Parser state
  50  */
  51 int GetToken(tParser *File)
  52 {
  53          int    ret;
  54
  55         if( File->NextToken != -1 ) {
  56                 // Save Last
  57                 File->LastToken = File->Token;
  58                 File->LastTokenStr = File->TokenStr;
  59                 File->LastTokenLen = File->TokenLen;
  60                 File->LastLine = File->CurLine;
  61                 // Restore Next
  62                 File->Token = File->NextToken;
  63                 File->TokenStr = File->NextTokenStr;
  64                 File->TokenLen = File->NextTokenLen;
  65                 File->CurLine = File->NextLine;
  66                 // Set State
  67                 File->CurPos = File->TokenStr + File->TokenLen;
  68                 File->NextToken = -1;
  69                 {
  70                         char    buf[ File->TokenLen + 1];
  71                         memcpy(buf, File->TokenStr, File->TokenLen);
  72                         buf[File->TokenLen] = 0;
  73                         #if DEBUG
  74                         printf(" GetToken: FAST Return %i (%i long) (%s)\n", File->Token, File->TokenLen, buf);
  75                         #endif
  76                 }
  77                 return File->Token;
  78         }
  79
  80         //printf("  GetToken: File=%p, File->CurPos = %p\n", File, File->CurPos);
  81
  82         // Clear whitespace (including comments)
  83         for( ;; )
  84         {
  85                 // Whitespace
  86                 while( isspace( *File->CurPos ) )
  87                 {
  88                         //printf("whitespace 0x%x, line = %i\n", *File->CurPos, File->CurLine);
  89                         if( *File->CurPos == '\n' )
  90                                 File->CurLine ++;
  91                         File->CurPos ++;
  92                 }
  93
  94                 // # Line Comments
  95                 if( *File->CurPos == '#' ) {
  96                         while( *File->CurPos && *File->CurPos != '\n' )
  97                                 File->CurPos ++;
  98                         continue ;
  99                 }
 100
 101                 // C-Style Line Comments
 102                 if( *File->CurPos == '/' && File->CurPos[1] == '/' ) {
 103                         while( *File->CurPos && *File->CurPos != '\n' )
 104                                 File->CurPos ++;
 105                         continue ;
 106                 }
 107
 108                 // C-Style Block Comments
 109                 if( *File->CurPos == '/' && File->CurPos[1] == '*' ) {
 110                         File->CurPos += 2;      // Eat the '/*'
 111                         while( *File->CurPos && !(File->CurPos[-1] == '*' && *File->CurPos == '/') )
 112                         {
 113                                 if( *File->CurPos == '\n' )     File->CurLine ++;
 114                                 File->CurPos ++;
 115                         }
 116                         File->CurPos ++;        // Eat the '/'
 117                         continue ;
 118                 }
 119
 120                 // No more "whitespace"
 121                 break;
 122         }
 123
 124         // Save previous tokens (speeds up PutBack and LookAhead)
 125         File->LastToken = File->Token;
 126         File->LastTokenStr = File->TokenStr;
 127         File->LastTokenLen = File->TokenLen;
 128         File->LastLine = File->CurLine;
 129
 130         // Read token
 131         File->TokenStr = File->CurPos;
 132         switch( *File->CurPos++ )
 133         {
 134         case '\0':      ret = TOK_EOF;  break;
 135
 136         // Operations
 137         case '/':       ret = TOK_DIV;  break;
 138         case '*':       ret = TOK_MUL;  break;
 139         case '+':       ret = TOK_PLUS; break;
 140         case '-':
 141                 if( *File->CurPos == '>' ) {
 142                         File->CurPos ++;
 143                         ret = TOK_ELEMENT;
 144                 }
 145                 else
 146                         ret = TOK_MINUS;
 147                 break;
 148
 149         // Strings
 150         case '"':
 151                 while( *File->CurPos && !(*File->CurPos == '"' && *File->CurPos != '\\') )
 152                         File->CurPos ++;
 153                 File->CurPos ++;
 154                 ret = TOK_STR;
 155                 break;
 156
 157         // Brackets
 158         case '(':       ret = TOK_PAREN_OPEN;   break;
 159         case ')':       ret = TOK_PAREN_CLOSE;  break;
 160         case '{':       ret = TOK_BRACE_OPEN;   break;
 161         case '}':       ret = TOK_BRACE_CLOSE;  break;
 162         case '[':       ret = TOK_SQUARE_OPEN;  break;
 163         case ']':       ret = TOK_SQUARE_CLOSE; break;
 164
 165         // Core symbols
 166         case ';':       ret = TOK_SEMICOLON;    break;
 167         case ',':       ret = TOK_COMMA;        break;
 168         #if USE_SCOPE_CHAR
 169         case '.':       ret = TOK_SCOPE;        break;
 170         #endif
 171
 172         // Equals
 173         case '=':
 174                 // Comparison Equals
 175                 if( *File->CurPos == '=' ) {
 176                         File->CurPos ++;
 177                         ret = TOK_EQUALS;
 178                         break;
 179                 }
 180                 // Assignment Equals
 181                 ret = TOK_ASSIGN;
 182                 break;
 183
 184         // Variables
 185         // \$[0-9]+ or \$[_a-zA-Z][_a-zA-Z0-9]*
 186         case '$':
 187                 // Numeric Variable
 188                 if( isdigit( *File->CurPos ) ) {
 189                         while( isdigit(*File->CurPos) )
 190                                 File->CurPos ++;
 191                 }
 192                 // Ident Variable
 193                 else {
 194                         while( is_ident(*File->CurPos) || isdigit(*File->CurPos) )
 195                                 File->CurPos ++;
 196                 }
 197                 ret = TOK_VARIABLE;
 198                 break;
 199
 200         // Default (Numbers and Identifiers)
 201         default:
 202                 File->CurPos --;
 203                 // Numbers
 204                 if( isdigit(*File->CurPos) )
 205                 {
 206                         while( isdigit(*File->CurPos) )
 207                                 File->CurPos ++;
 208                         ret = TOK_INTEGER;
 209                         break;
 210                 }
 211
 212                 // Identifier
 213                 if( is_ident(*File->CurPos) )
 214                 {
 215                         // Identifier
 216                         while( is_ident(*File->CurPos) || isdigit(*File->CurPos) )
 217                                 File->CurPos ++;
 218
 219                         // This is set later too, but we use it below
 220                         File->TokenLen = File->CurPos - File->TokenStr;
 221                         ret = TOK_IDENT;
 222
 223                         // Check if it's a reserved word
 224                         {
 225                                 char    buf[File->TokenLen + 1];
 226                                  int    i;
 227                                 memcpy(buf, File->TokenStr, File->TokenLen);
 228                                 buf[File->TokenLen] = 0;
 229                                 for( i = 0; i < ARRAY_SIZE(csaReservedWords); i ++ )
 230                                 {
 231                                         if(strcmp(csaReservedWords[i].Name, buf) == 0) {
 232                                                 ret = csaReservedWords[i].Value;
 233                                                 break ;
 234                                         }
 235                                 }
 236                         }
 237                         // If there's no match, just keep ret as TOK_IDENT
 238
 239                         break;
 240                 }
 241                 // Syntax Error
 242                 ret = TOK_INVAL;
 243
 244                 fprintf(stderr, "Syntax Error: Unknown symbol '%c'\n", *File->CurPos);
 245                 longjmp(File->JmpTarget, 1);
 246
 247                 break;
 248         }
 249         // Return
 250         File->Token = ret;
 251         File->TokenLen = File->CurPos - File->TokenStr;
 252
 253         #if DEBUG
 254         {
 255                 char    buf[ File->TokenLen + 1];
 256                 memcpy(buf, File->TokenStr, File->TokenLen);
 257                 buf[File->TokenLen] = 0;
 258                 //printf("  GetToken: File->CurPos = %p\n", File->CurPos);
 259                 printf(" GetToken: Return %i (%i long) (%s)\n", ret, File->TokenLen, buf);
 260         }
 261         #endif
 262         return ret;
 263 }
 264
 265 void PutBack(tParser *File)
 266 {
 267         if( File->LastToken == -1 ) {
 268                 // ERROR:
 269                 fprintf(stderr, "INTERNAL ERROR: Putback when LastToken==-1\n");
 270                 longjmp( File->JmpTarget, -1 );
 271                 return ;
 272         }
 273         #if DEBUG
 274         printf(" PutBack: Was on %i\n", File->Token);
 275         #endif
 276         // Save
 277         File->NextLine = File->CurLine;
 278         File->NextToken = File->Token;
 279         File->NextTokenStr = File->TokenStr;
 280         File->NextTokenLen = File->TokenLen;
 281         // Restore
 282         File->CurLine = File->LastLine;
 283         File->Token = File->LastToken;
 284         File->TokenStr = File->LastTokenStr;
 285         File->TokenLen = File->LastTokenLen;
 286         File->CurPos = File->NextTokenStr;
 287         // Invalidate
 288         File->LastToken = -1;
 289 }
 290
 291 int LookAhead(tParser *File)
 292 {
 293         // TODO: Should I save the entire state here?
 294          int    ret = GetToken(File);
 295         PutBack(File);
 296         return ret;
 297 }
 298
 299 // --- Helpers ---
 300 /**
 301  * \brief Check for ident characters
 302  * \note Matches Regex [a-zA-Z_]
 303  */
 304 int is_ident(char ch)
 305 {
 306         if('a' <= ch && ch <= 'z')      return 1;
 307         if('A' <= ch && ch <= 'Z')      return 1;
 308         if(ch == '_')   return 1;
 309         #if !USE_SCOPE_CHAR
 310         if(ch == '.')   return 1;
 311         #endif
 312         if(ch < 0)      return 1;
 313         return 0;
 314 }
 315
 316 int isdigit(int ch)
 317 {
 318         if('0' <= ch && ch <= '9')      return 1;
 319         return 0;
 320 }
 321
 322 int isspace(int ch)
 323 {
 324         if(' ' == ch)   return 1;
 325         if('\t' == ch)  return 1;
 326         if('\b' == ch)  return 1;
 327         if('\n' == ch)  return 1;
 328         if('\r' == ch)  return 1;
 329         return 0;
 330 }