How does GMod impliment it's custom lua tokens?

[ph4ge]

On a journey to further my knowledge about lua, I ran into a weird situation. gLua impliments quite a few different tokens that aren't native to lua but also keeps the old tokens set standard by lua. For example, take a look at the definition of luaX_tokens. (My example is modified from the master branch since I patched my lua state to include continue, as GMod does.)

/* ORDER RESERVED */
const char *const luaX_tokens [] = {
    "and", "break", "continue", "do", "else", "elseif",
    "end", "false", "for", "function", "if",
    "in", "local", "nil", "not", "or", "repeat",
    "return", "then", "true", "until", "while",
    "..", "...", "==", ">=", "<=", "~=",
    "<number>", "<name>", "<string>", "<eof>",
    NULL
};

As we can see, we have all the 'normal' tokens we would regularly have. Reading stack overflow threads about this leads me to just editing the tokens to whatever my desired values are, and it works to an extent. But I'm guessing, GMod doesn't just edit the tokens since it includes, for example, 'and' and '&&'. How does GMod handle these things?

EDIT: For example, I took the liberty of doing some messing around and modified luaX_tokens and the enum RESERVED in llex.c, and llex.h respectively.

/* ORDER RESERVED */
const char *const luaX_tokens [] = {
    "and", "break", "continue", "do", "else", "elseif",
    "end", "false", "for", "function", "if",
    "in", "local", "nil", "not", "or", "repeat",
    "return", "then", "true", "until", "while",
    "..", "...", "==", ">=", "<=", "~=",
    "<number>", "<name>", "<string>", "<eof>",
    NULL, "&&", "||", "!", "!="
};

enum RESERVED {
  /* terminal symbols denoted by reserved words */
  TK_AND = FIRST_RESERVED, TK_BREAK, TK_CONTINUE,
  TK_DO, TK_ELSE, TK_ELSEIF, TK_END, TK_FALSE, TK_FOR, TK_FUNCTION,
  TK_IF, TK_IN, TK_LOCAL, TK_NIL, TK_NOT, TK_OR, TK_REPEAT,
  TK_RETURN, TK_THEN, TK_TRUE, TK_UNTIL, TK_WHILE,
  /* other terminal symbols */
  TK_CONCAT, TK_DOTS, TK_EQ, TK_GE, TK_LE, TK_NE, TK_NUMBER,
  TK_NAME, TK_STRING, TK_EOS, TK_AND2 = 257, TK_OR2 = 272, 
  TK_NOT2 = 271, TK_NE2 = 284
};

The enum's values are correctly aligned to the other reserved keywords, yet the lua state will throw exceptions about unexpected symbols for each token I added.

[Darkwater]

Wild guess since I have no experience with the C side of lua, but maybe move up the NULL to the end?

[zerf]

zerf@home lua (lua-5.1) $ git diff
diff --git a/src/llex.c b/src/llex.c
index 88c6790..484e21f 100644
--- a/src/llex.c
+++ b/src/llex.c
@@ -37,7 +37,7 @@
 const char *const luaX_tokens [] = {
     "and", "break", "do", "else", "elseif",
     "end", "false", "for", "function", "if",
-    "in", "local", "nil", "not", "or", "repeat",
+    "in", "local", "nil", "not", "!", "or", "repeat",
     "return", "then", "true", "until", "while",
     "..", "...", "==", ">=", "<=", "~=",
     "<number>", "<name>", "<string>", "<eof>",
@@ -388,6 +388,21 @@ static int llex (LexState *ls, SemInfo *seminfo) {
         if (ls->current != '=') return '~';
         else { next(ls); return TK_NE; }
       }
+      case '&': {
+        next(ls);
+        if (ls->current != '&') return '&';
+        else { next(ls); return TK_AND; }
+      }
+      case '|': {
+        next(ls);
+        if (ls->current != '|') return '|';
+        else { next(ls); return TK_OR; }
+      }
+      case '!': {
+        next(ls);
+        if (ls->current != '=') return TK_NOT2;
+        else { next(ls); return TK_NE; }
+      }
       case '"':
       case '\'': {
         read_string(ls, ls->current, seminfo);
diff --git a/src/llex.h b/src/llex.h
index a9201ce..2eb588a 100644
--- a/src/llex.h
+++ b/src/llex.h
@@ -25,7 +25,7 @@ enum RESERVED {
   /* terminal symbols denoted by reserved words */
   TK_AND = FIRST_RESERVED, TK_BREAK,
   TK_DO, TK_ELSE, TK_ELSEIF, TK_END, TK_FALSE, TK_FOR, TK_FUNCTION,
-  TK_IF, TK_IN, TK_LOCAL, TK_NIL, TK_NOT, TK_OR, TK_REPEAT,
+  TK_IF, TK_IN, TK_LOCAL, TK_NIL, TK_NOT, TK_NOT2, TK_OR, TK_REPEAT,
   TK_RETURN, TK_THEN, TK_TRUE, TK_UNTIL, TK_WHILE,
   /* other terminal symbols */
   TK_CONCAT, TK_DOTS, TK_EQ, TK_GE, TK_LE, TK_NE, TK_NUMBER,
diff --git a/src/lparser.c b/src/lparser.c
index dda7488..eb5cdc0 100644
--- a/src/lparser.c
+++ b/src/lparser.c
@@ -778,6 +778,7 @@ static void simpleexp (LexState *ls, expdesc *v) {
 static UnOpr getunopr (int op) {
   switch (op) {
     case TK_NOT: return OPR_NOT;
+    case TK_NOT2: return OPR_NOT;
     case '-': return OPR_MINUS;
     case '#': return OPR_LEN;
     default: return OPR_NOUNOPR;
zerf@home lua (lua-5.1) $ make linux
cd src && make linux
make[1]: Entering directory `/mnt/c/Users/Zerf/foo/lua/src'
make all MYCFLAGS=-DLUA_USE_LINUX MYLIBS="-Wl,-E -ldl -lreadline -lhistory -lncurses"
make[2]: Entering directory `/mnt/c/Users/Zerf/foo/lua/src'
make[2]: Nothing to be done for `all'.
make[2]: Leaving directory `/mnt/c/Users/Zerf/foo/lua/src'
make[1]: Leaving directory `/mnt/c/Users/Zerf/foo/lua/src'
zerf@home lua (lua-5.1) $ src/lua
Lua 5.1.5  Copyright (C) 1994-2012 Lua.org, PUC-Rio
> print(!true, true && false, true || false, true != false)
false   false   true    true
> os.exit()
zerf@home lua (lua-5.1) $

[editline]19th December 2016[/editline]

oh gross did I just help a member of skid team

[Exploderguy]

[QUOTE=zerf;51550845]oh gross did I just help a member of skid team[/QUOTE]

yeah, this skag is part of cheater.team

[c👌de_g🎷]

[QUOTE=Exploderguy;51550853]yeah, this skag is part of cheater.team[/QUOTE]

I don't really see the big deal; he was just asking a normal question.

[LegendofRobbo]

[QUOTE=zerf;51550845]
oh gross did I just help a member of skid team[/QUOTE]

welp, at least this guy can code

[ph4ge]

[QUOTE=zerf;51550845]oh gross did I just help a member of skid team[/QUOTE]

You're right I am a member of 'skid.team' but I try not to stir up drama and seperate myself from most of the other members. Do you have any idea of why the way I went about it wouldn't work as well? Shouldn't the lua interpreter read the tokens the same way it reads every other one?

EDIT: And your patch works fine, thank you very much :)

[G😹nz😻]

-snip-

[SneakySquid]

[QUOTE=gonzalolog;51551736]Oh my god guys, are you blind? This guy is detouring tokens! Aka hacking cac[/QUOTE]

[url]https://github.com/ph4ge/Dunked-Framework[/url]

[ph4ge]

[QUOTE=txike;51551750][url]https://github.com/ph4ge/Dunked-Framework[/url][/QUOTE]

Yes, that's the project it's for. I'd prefer not to get meme replies that have nothing to do with the topic at hand.

[Chessnut]

[QUOTE=ph4ge;51551585]Do you have any idea of why the way I went about it wouldn't work as well? Shouldn't the lua interpreter read the tokens the same way it reads every other one?

EDIT: And your patch works fine, thank you very much :)[/QUOTE]

The NULL is a sentinel value

[ph4ge]

[QUOTE=Chessnut;51552353]The NULL is a sentinel value[/QUOTE]

Ah, so reordering the NULL to the very end would've lead me to the results I was looking for?

[zerf]

[QUOTE=ph4ge;51552563]Ah, so reordering the NULL to the very end would've lead me to the results I was looking for?[/QUOTE]

Even that wouldn't work. Look at the llex function; if it encounters the "&" character for example, it reaches the default block in the switch statement. In the default block, it only gets checked agains the reserved word block if it's an alphabet character or underscore. We actually have to add a case block similar to the other symbols.

I was also able to simplify the patch quite a bit

diff --git a/src/llex.c b/src/llex.c
index 88c6790..80f5ec4 100644
--- a/src/llex.c
+++ b/src/llex.c
@@ -388,6 +388,21 @@ static int llex (LexState *ls, SemInfo *seminfo) {
         if (ls->current != '=') return '~';
         else { next(ls); return TK_NE; }
       }
+      case '&': {
+        next(ls);
+        if (ls->current != '&') return '&';
+        else { next(ls); return TK_AND; }
+      }
+      case '|': {
+        next(ls);
+        if (ls->current != '|') return '|';
+        else { next(ls); return TK_OR; }
+      }
+      case '!': {
+        next(ls);
+        if (ls->current != '=') return TK_NOT;
+        else { next(ls); return TK_NE; }
+      }
       case '"':
       case '\'': {
         read_string(ls, ls->current, seminfo);

[ph4ge]

[QUOTE=zerf;51553031]Even that wouldn't work. Look at the llex function; if it encounters the "&" character for example, it reaches the default block in the switch statement. In the default block, it only gets checked agains the reserved word block if it's an alphabet character or underscore. We actually have to add a case block similar to the other symbols.

I was also able to simplify the patch quite a bit

diff --git a/src/llex.c b/src/llex.c
index 88c6790..80f5ec4 100644
--- a/src/llex.c
+++ b/src/llex.c
@@ -388,6 +388,21 @@ static int llex (LexState *ls, SemInfo *seminfo) {
         if (ls->current != '=') return '~';
         else { next(ls); return TK_NE; }
       }
+      case '&': {
+        next(ls);
+        if (ls->current != '&') return '&';
+        else { next(ls); return TK_AND; }
+      }
+      case '|': {
+        next(ls);
+        if (ls->current != '|') return '|';
+        else { next(ls); return TK_OR; }
+      }
+      case '!': {
+        next(ls);
+        if (ls->current != '=') return TK_NOT;
+        else { next(ls); return TK_NE; }
+      }
       case '"':
       case '\'': {
         read_string(ls, ls->current, seminfo);

[/QUOTE]

TIL and thanks for all your help zerf!