function lookupify(tb)
for _, v in pairs(tb) do
tb[v] = true
end
return tb
end
local WhiteChars = lookupify{' ', '\n', '\t', '\r'}
local LowerChars = lookupify{'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i',
'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r',
's', 't', 'u', 'v', 'w', 'x', 'y', 'z'}
local UpperChars = lookupify{'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I',
'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R',
'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'}
local Digits = lookupify{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9'}
local HexDigits = lookupify{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
'A', 'a', 'B', 'b', 'C', 'c', 'D', 'd', 'E', 'e', 'F', 'f'}
local Symbols = lookupify{'+', '-', '*', '/', '^', '%', ',', '{', '}', '[', ']', '(', ')', ';', '#'}
local Keywords = lookupify{
'and', 'break', 'do', 'else', 'elseif',
'end', 'false', 'for', 'function', 'goto', 'if',
'in', 'local', 'nil', 'not', 'or', 'repeat',
'return', 'then', 'true', 'until', 'while',
};
function LexLua(src)
--token dump
local tokens = {}
local st, err = pcall(function()
--line / char / pointer tracking
local p = 1
local line = 1
local char = 1
--get / peek functions
local function get()
local c = src:sub(p,p)
if c == '\n' then
char = 1
line = line + 1
else
char = char + 1
end
p = p + 1
return c
end
local function peek(n)
n = n or 0
return src:sub(p+n,p+n)
end
local function consume(chars)
local c = peek()
for i = 1, #chars do
if c == chars:sub(i,i) then return get() end
end
end
--shared stuff
local function generateError(err)
return error(">> :"..line..":"..char..": "..err, 0)
end
local function tryGetLongString()
local start = p
if peek() == '[' then
local equalsCount = 0
while peek(equalsCount+1) == '=' do
equalsCount = equalsCount + 1
end
if peek(equalsCount+1) == '[' then
--start parsing the string. Strip the starting bit
for _ = 0, equalsCount+1 do get() end
--get the contents
local contentStart = p
while true do
--check for eof
if peek() == '' then
generateError("Expected `]"..string.rep('=', equalsCount).."]` near <eof>.", 3)
end
--check for the end
local foundEnd = true
if peek() == ']' then
for i = 1, equalsCount do
if peek(i) ~= '=' then foundEnd = false end
end
if peek(equalsCount+1) ~= ']' then
foundEnd = false
end
else
foundEnd = false
end
--
if foundEnd then
break
else
get()
end
end
--get the interior string
local contentString = src:sub(contentStart, p-1)
--found the end. Get rid of the trailing bit
for i = 0, equalsCount+1 do get() end
--get the exterior string
local longString = src:sub(start, p-1)
--return the stuff
return contentString, longString
else
return nil
end
else
return nil
end
end
--main token emitting loop
while true do
--get leading whitespace. The leading whitespace will include any comments
--preceding the token. This prevents the parser needing to deal with comments
--separately.
local leadingWhite = ''
while true do
local c = peek()
if WhiteChars[c] then
--whitespace
leadingWhite = leadingWhite..get()
elseif c == '-' and peek(1) == '-' then
--comment
get();get()
leadingWhite = leadingWhite..'--'
local _, wholeText = tryGetLongString()
if wholeText then
leadingWhite = leadingWhite..wholeText
else
while peek() ~= '\n' and peek() ~= '' do
leadingWhite = leadingWhite..get()
end
end
else
break
end
end
--get the initial char
local thisLine = line
local thisChar = char
local errorAt = ":"..line..":"..char..":> "
local c = peek()
--symbol to emit
local toEmit = nil
--branch on type
if c == '' then
--eof
toEmit = {Type = 'Eof'}
elseif UpperChars[c] or LowerChars[c] or c == '_' then
--ident or keyword
local start = p
repeat
get()
c = peek()
until not (UpperChars[c] or LowerChars[c] or Digits[c] or c == '_')
local dat = src:sub(start, p-1)
if Keywords[dat] then
toEmit = {Type = 'Keyword', Data = dat}
else
toEmit = {Type = 'Ident', Data = dat}
end
elseif Digits[c] or (peek() == '.' and Digits[peek(1)]) then
--number const
local start = p
if c == '0' and peek(1) == 'x' then
get();get()
while HexDigits[peek()] do get() end
if consume('Pp') then
consume('+-')
while Digits[peek()] do get() end
end
else
while Digits[peek()] do get() end
if consume('.') then
while Digits[peek()] do get() end
end
if consume('Ee') then
consume('+-')
while Digits[peek()] do get() end
end
end
toEmit = {Type = 'Number', Data = src:sub(start, p-1)}
elseif c == '\'' or c == '\"' then
local start = p
--string const
local delim = get()
local contentStart = p
while true do
local c = get()
if c == '\\' then
get() --get the escape char
elseif c == delim then
break
elseif c == '' then
generateError("Unfinished string near <eof>")
end
end
local content = src:sub(contentStart, p-2)
local constant = src:sub(start, p-1)
toEmit = {Type = 'String', Data = constant, Constant = content}
elseif c == '[' then
local content, wholetext = tryGetLongString()
if wholetext then
toEmit = {Type = 'String', Data = wholetext, Constant = content}
else
get()
toEmit = {Type = 'Symbol', Data = '['}
end
elseif consume('>=<') then
if consume('=') then
toEmit = {Type = 'Symbol', Data = c..'='}
else
toEmit = {Type = 'Symbol', Data = c}
end
elseif consume('~') then
if consume('=') then
toEmit = {Type = 'Symbol', Data = '~='}
else
generateError("Unexpected symbol `~` in source.", 2)
end
elseif consume('.') then
if consume('.') then
if consume('.') then
toEmit = {Type = 'Symbol', Data = '...'}
else
toEmit = {Type = 'Symbol', Data = '..'}
end
else
toEmit = {Type = 'Symbol', Data = '.'}
end
elseif consume(':') then
if consume(':') then
toEmit = {Type = 'Symbol', Data = '::'}
else
toEmit = {Type = 'Symbol', Data = ':'}
end
elseif Symbols[c] then
get()
toEmit = {Type = 'Symbol', Data = c}
else
local contents, all = tryGetLongString()
if contents then
toEmit = {Type = 'String', Data = all, Constant = contents}
else
generateError("Unexpected Symbol `"..c.."` in source.", 2)
end
end
--add the emitted symbol, after adding some common data
toEmit.LeadingWhite = leadingWhite
toEmit.Line = thisLine
toEmit.Char = thisChar
toEmit.Print = function()
return "<"..(toEmit.Type..string.rep(' ', 7-#toEmit.Type)).." "..(toEmit.Data or '').." >"
end
tokens[#tokens+1] = toEmit
--halt after eof has been emitted
if toEmit.Type == 'Eof' then break end
end
end)
if not st then
return false, err
end
--public interface:
local tok = {}
local savedP = {}
local p = 1
--getters
function tok:Peek(n)
n = n or 0
return tokens[math.min(#tokens, p+n)]
end
function tok:Get()
local t = tokens[p]
p = math.min(p + 1, #tokens)
return t
end
function tok:Is(t)
return tok:Peek().Type == t
end
--save / restore points in the stream
function tok:Save()
savedP[#savedP+1] = p
end
function tok:Commit()
savedP[#savedP] = nil
end
function tok:Restore()
p = savedP[#savedP]
savedP[#savedP] = nil
end
--either return a symbol if there is one, or return true if the requested
--symbol was gotten.
function tok:ConsumeSymbol(symb)
local t = self:Peek()
if t.Type == 'Symbol' then
if symb then
if t.Data == symb then
self:Get()
return true
else
return nil
end
else
self:Get()
return t
end
else
return nil
end
end
function tok:ConsumeKeyword(kw)
local t = self:Peek()
if t.Type == 'Keyword' and t.Data == kw then
self:Get()
return true
else
return nil
end
end
function tok:IsKeyword(kw)
local t = tok:Peek()
return t.Type == 'Keyword' and t.Data == kw
end
function tok:IsSymbol(s)
local t = tok:Peek()
return t.Type == 'Symbol' and t.Data == s
end
function tok:IsEof()
return tok:Peek().Type == 'Eof'
end
return true, tok
end
function ParseLua(src)
local st, tok = LexLua(src)
if not st then
return false, tok
end
--
local function GenerateError(msg)
local err = ">> :"..tok:Peek().Line..":"..tok:Peek().Char..": "..msg.."\n"
--find the line
local lineNum = 0
for line in src:gmatch("[^\n]*\n?") do
if line:sub(-1,-1) == '\n' then line = line:sub(1,-2) end
lineNum = lineNum+1
if lineNum == tok:Peek().Line then
err = err..">> `"..line:gsub('\t',' ').."`\n"
for i = 1, tok:Peek().Char do
local c = line:sub(i,i)
if c == '\t' then
err = err..' '
else
err = err..' '
end
end
err = err.." ^---"
break
end
end
return err
end
--
local VarUid = 0
local GlobalVarGetMap = {}
local VarDigits = {'_', 'a', 'b', 'c', 'd'}
local function CreateScope(parent)
local scope = {}
scope.Parent = parent
scope.LocalList = {}
scope.LocalMap = {}
function scope:RenameVars()
for _, var in pairs(scope.LocalList) do
local id;
VarUid = 0
repeat
VarUid = VarUid + 1
local varToUse = VarUid
id = ''
while varToUse > 0 do
local d = varToUse % #VarDigits
varToUse = (varToUse - d) / #VarDigits
id = id..VarDigits[d+1]
end
until not GlobalVarGetMap[id] and not parent:GetLocal(id) and not scope.LocalMap[id]
var.Name = id
scope.LocalMap[id] = var
end
end
function scope:GetLocal(name)
--first, try to get my variable
local my = scope.LocalMap[name]
if my then return my end
--next, try parent
if scope.Parent then
local par = scope.Parent:GetLocal(name)
if par then return par end
end
return nil
end
function scope:CreateLocal(name)
--create my own var
local my = {}
my.Scope = scope
my.Name = name
my.CanRename = true
--
scope.LocalList[#scope.LocalList+1] = my
scope.LocalMap[name] = my
--
return my
end
scope.Print = function() return "<Scope>" end
return scope
end
local ParseExpr;
local ParseStatementList;
local function ParseFunctionArgsAndBody(scope)
local funcScope = CreateScope(scope)
if not tok:ConsumeSymbol('(') then
return false, GenerateError("`(` expected.")
end
--arg list
local argList = {}
local isVarArg = false
while not tok:ConsumeSymbol(')') do
if tok:Is('Ident') then
local arg = funcScope:CreateLocal(tok:Get().Data)
argList[#argList+1] = arg
if not tok:ConsumeSymbol(',') then
if tok:ConsumeSymbol(')') then
break
else
return false, GenerateError("`)` expected.")
end
end
elseif tok:ConsumeSymbol('...') then
isVarArg = true
if not tok:ConsumeSymbol(')') then
return false, GenerateError("`...` must be the last argument of a function.")
end
break
else
return false, GenerateError("Argument name or `...` expected")
end
end
--body
local st, body = ParseStatementList(funcScope)
if not st then return false, body end
--end
if not tok:ConsumeKeyword('end') then
return false, GenerateError("`end` expected after function body")
end
local nodeFunc = {}
nodeFunc.AstType = 'Function'
nodeFunc.Scope = funcScope
nodeFunc.Arguments = argList
nodeFunc.Body = body
nodeFunc.VarArg = isVarArg
--
return true, nodeFunc
end
local function ParsePrimaryExpr(scope)
if tok:ConsumeSymbol('(') then
local st, ex = ParseExpr(scope)
if not st then return false, ex end
if not tok:ConsumeSymbol(')') then
return false, GenerateError("`)` Expected.")
end
--save the information about parenthesized expressions somewhere
ex.ParenCount = (ex.ParenCount or 0) + 1
return true, ex
elseif tok:Is('Ident') then
local id = tok:Get()
local var = scope:GetLocal(id.Data)
if not var then
GlobalVarGetMap[id.Data] = true
end
--
local nodePrimExp = {}
nodePrimExp.AstType = 'VarExpr'
nodePrimExp.Name = id.Data
nodePrimExp.Local = var
--
return true, nodePrimExp
else
return false, GenerateError("primary expression expected")
end
end
local function ParseSuffixedExpr(scope, onlyDotColon)
--base primary expression
local st, prim = ParsePrimaryExpr(scope)
if not st then return false, prim end
--
while true do
if tok:IsSymbol('.') or tok:IsSymbol(':') then
local symb = tok:Get().Data
if not tok:Is('Ident') then
return false, GenerateError("<Ident> expected.")
end
local id = tok:Get()
local nodeIndex = {}
nodeIndex.AstType = 'MemberExpr'
nodeIndex.Base = prim
nodeIndex.Indexer = symb
nodeIndex.Ident = id
--
prim = nodeIndex
elseif not onlyDotColon and tok:ConsumeSymbol('[') then
local st, ex = ParseExpr(scope)
if not st then return false, ex end
if not tok:ConsumeSymbol(']') then
return false, GenerateError("`]` expected.")
end
local nodeIndex = {}
nodeIndex.AstType = 'IndexExpr'
nodeIndex.Base = prim
nodeIndex.Index = ex
--
prim = nodeIndex
elseif not onlyDotColon and tok:ConsumeSymbol('(') then
local args = {}
while not tok:ConsumeSymbol(')') do
local st, ex = ParseExpr(scope)
if not st then return false, ex end
args[#args+1] = ex
if not tok:ConsumeSymbol(',') then
if tok:ConsumeSymbol(')') then
break
else
return false, GenerateError("`)` Expected.")
end
end
end
local nodeCall = {}
nodeCall.AstType = 'CallExpr'
nodeCall.Base = prim
nodeCall.Arguments = args
--
prim = nodeCall
elseif not onlyDotColon and tok:Is('String') then
--string call
local nodeCall = {}
nodeCall.AstType = 'StringCallExpr'
nodeCall.Base = prim
nodeCall.Arguments = {tok:Get()}
--
prim = nodeCall
elseif not onlyDotColon and tok:IsSymbol('{') then
--table call
local st, ex = ParseExpr(scope)
if not st then return false, ex end
local nodeCall = {}
nodeCall.AstType = 'TableCallExpr'
nodeCall.Base = prim
nodeCall.Arguments = {ex}
--
prim = nodeCall
else
break
end
end
return true, prim
end
local function ParseSimpleExpr(scope)
if tok:Is('Number') then
local nodeNum = {}
nodeNum.AstType = 'NumberExpr'
nodeNum.Value = tok:Get()
return true, nodeNum
elseif tok:Is('String') then
local nodeStr = {}
nodeStr.AstType = 'StringExpr'
nodeStr.Value = tok:Get()
return true, nodeStr
elseif tok:ConsumeKeyword('nil') then
local nodeNil = {}
nodeNil.AstType = 'NilExpr'
return true, nodeNil
elseif tok:IsKeyword('false') or tok:IsKeyword('true') then
local nodeBoolean = {}
nodeBoolean.AstType = 'BooleanExpr'
nodeBoolean.Value = (tok:Get().Data == 'true')
return true, nodeBoolean
elseif tok:ConsumeSymbol('...') then
local nodeDots = {}
nodeDots.AstType = 'DotsExpr'
return true, nodeDots
elseif tok:ConsumeSymbol('{') then
local v = {}
v.AstType = 'ConstructorExpr'
v.EntryList = {}
--
while true do
if tok:IsSymbol('[') then
--key
tok:Get()
local st, key = ParseExpr(scope)
if not st then
return false, GenerateError("Key Expression Expected")
end
if not tok:ConsumeSymbol(']') then
return false, GenerateError("`]` Expected")
end
if not tok:ConsumeSymbol('=') then
return false, GenerateError("`=` Expected")
end
local st, value = ParseExpr(scope)
if not st then
return false, GenerateError("Value Expression Expected")
end
v.EntryList[#v.EntryList+1] = {
Type = 'Key';
Key = key;
Value = value;
}
elseif tok:Is('Ident') then
--value or key
local lookahead = tok:Peek(1)
if lookahead.Type == 'Symbol' and lookahead.Data == '=' then
--we are a key
local key = tok:Get()
if not tok:ConsumeSymbol('=') then
return false, GenerateError("`=` Expected")
end
local st, value = ParseExpr(scope)
if not st then
return false, GenerateError("Value Expression Expected")
end
v.EntryList[#v.EntryList+1] = {
Type = 'KeyString';
Key = key.Data;
Value = value;
}
else
--we are a value
local st, value = ParseExpr(scope)
if not st then
return false, GenerateError("Value Exected")
end
v.EntryList[#v.EntryList+1] = {
Type = 'Value';
Value = value;
}
end
elseif tok:ConsumeSymbol('}') then
break
else
--value
local st, value = ParseExpr(scope)
v.EntryList[#v.EntryList+1] = {
Type = 'Value';
Value = value;
}
if not st then
return false, GenerateError("Value Expected")
end
end
if tok:ConsumeSymbol(';') or tok:ConsumeSymbol(',') then
--all is good
elseif tok:ConsumeSymbol('}') then
break
else
return false, GenerateError("`}` or table entry Expected")
end
end
return true, v
elseif tok:ConsumeKeyword('function') then
local st, func = ParseFunctionArgsAndBody(scope)
if not st then return false, func end
--
func.IsLocal = true
return true, func
else
return ParseSuffixedExpr(scope)
end
end
local unops = lookupify{'-', 'not', '#'}
local unopprio = 8
local priority = {
['+'] = {6,6};
['-'] = {6,6};
['%'] = {7,7};
['/'] = {7,7};
['*'] = {7,7};
['^'] = {10,9};
['..'] = {5,4};
['=='] = {3,3};
['<'] = {3,3};
['<='] = {3,3};
['~='] = {3,3};
['>'] = {3,3};
['>='] = {3,3};
['and'] = {2,2};
['or'] = {1,1};
}
local function ParseSubExpr(scope, level)
--base item, possibly with unop prefix
local st, exp
if unops[tok:Peek().Data] then
local op = tok:Get().Data
st, exp = ParseSubExpr(scope, unopprio)
if not st then return false, exp end
local nodeEx = {}
nodeEx.AstType = 'UnopExpr'
nodeEx.Rhs = exp
nodeEx.Op = op
exp = nodeEx
else
st, exp = ParseSimpleExpr(scope)
if not st then return false, exp end
end
--next items in chain
while true do
local prio = priority[tok:Peek().Data]
if prio and prio[1] > level then
local op = tok:Get().Data
local st, rhs = ParseSubExpr(scope, prio[2])
if not st then return false, rhs end
local nodeEx = {}
nodeEx.AstType = 'BinopExpr'
nodeEx.Lhs = exp
nodeEx.Op = op
nodeEx.Rhs = rhs
--
exp = nodeEx
else
break
end
end
return true, exp
end
ParseExpr = function(scope)
return ParseSubExpr(scope, 0)
end
local function ParseStatement(scope)
local stat = nil
if tok:ConsumeKeyword('if') then
--setup
local nodeIfStat = {}
nodeIfStat.AstType = 'IfStatement'
nodeIfStat.Clauses = {}
--clauses
repeat
local st, nodeCond = ParseExpr(scope)
if not st then return false, nodeCond end
if not tok:ConsumeKeyword('then') then
return false, GenerateError("`then` expected.")
end
local st, nodeBody = ParseStatementList(scope)
if not st then return false, nodeBody end
nodeIfStat.Clauses[#nodeIfStat.Clauses+1] = {
Condition = nodeCond;
Body = nodeBody;
}
until not tok:ConsumeKeyword('elseif')
--else clause
if tok:ConsumeKeyword('else') then
local st, nodeBody = ParseStatementList(scope)
if not st then return false, nodeBody end
nodeIfStat.Clauses[#nodeIfStat.Clauses+1] = {
Body = nodeBody;
}
end
--end
if not tok:ConsumeKeyword('end') then
return false, GenerateError("`end` expected.")
end
stat = nodeIfStat
elseif tok:ConsumeKeyword('while') then
--setup
local nodeWhileStat = {}
nodeWhileStat.AstType = 'WhileStatement'
--condition
local st, nodeCond = ParseExpr(scope)
if not st then return false, nodeCond end
--do
if not tok:ConsumeKeyword('do') then
return false, GenerateError("`do` expected.")
end
--body
local st, nodeBody = ParseStatementList(scope)
if not st then return false, nodeBody end
--end
if not tok:ConsumeKeyword('end') then
return false, GenerateError("`end` expected.")
end
--return
nodeWhileStat.Condition = nodeCond
nodeWhileStat.Body = nodeBody
stat = nodeWhileStat
elseif tok:ConsumeKeyword('do') then
--do block
local st, nodeBlock = ParseStatementList(scope)
if not st then return false, nodeBlock end
if not tok:ConsumeKeyword('end') then
return false, GenerateError("`end` expected.")
end
local nodeDoStat = {}
nodeDoStat.AstType = 'DoStatement'
nodeDoStat.Body = nodeBlock
stat = nodeDoStat
elseif tok:ConsumeKeyword('for') then
--for block
if not tok:Is('Ident') then
return false, GenerateError("<ident> expected.")
end
local baseVarName = tok:Get()
if tok:ConsumeSymbol('=') then
--numeric for
local forScope = CreateScope(scope)
local forVar = forScope:CreateLocal(baseVarName.Data)
--
local st, startEx = ParseExpr(scope)
if not st then return false, startEx end
if not tok:ConsumeSymbol(',') then
return false, GenerateError("`,` Expected")
end
local st, endEx = ParseExpr(scope)
if not st then return false, endEx end
local st, stepEx;
if tok:ConsumeSymbol(',') then
st, stepEx = ParseExpr(scope)
if not st then return false, stepEx end
end
if not tok:ConsumeKeyword('do') then
return false, GenerateError("`do` expected")
end
--
local st, body = ParseStatementList(forScope)
if not st then return false, body end
if not tok:ConsumeKeyword('end') then
return false, GenerateError("`end` expected")
end
--
local nodeFor = {}
nodeFor.AstType = 'NumericForStatement'
nodeFor.Scope = forScope
nodeFor.Variable = forVar
nodeFor.Start = startEx
nodeFor.End = endEx
nodeFor.Step = stepEx
nodeFor.Body = body
stat = nodeFor
else
--generic for
local forScope = CreateScope(scope)
--
local varList = {forScope:CreateLocal(baseVarName.Data)}
while tok:ConsumeSymbol(',') do
if not tok:Is('Ident') then
return false, GenerateError("for variable expected.")
end
varList[#varList+1] = forScope:CreateLocal(tok:Get().Data)
end
if not tok:ConsumeKeyword('in') then
return false, GenerateError("`in` expected.")
end
local generators = {}
local st, firstGenerator = ParseExpr(scope)
if not st then return false, firstGenerator end
generators[#generators+1] = firstGenerator
while tok:ConsumeSymbol(',') do
local st, gen = ParseExpr(scope)
if not st then return false, gen end
generators[#generators+1] = gen
end
if not tok:ConsumeKeyword('do') then
return false, GenerateError("`do` expected.")
end
local st, body = ParseStatementList(forScope)
if not st then return false, body end
if not tok:ConsumeKeyword('end') then
return false, GenerateError("`end` expected.")
end
--
local nodeFor = {}
nodeFor.AstType = 'GenericForStatement'
nodeFor.Scope = forScope
nodeFor.VariableList = varList
nodeFor.Generators = generators
nodeFor.Body = body
stat = nodeFor
end
elseif tok:ConsumeKeyword('repeat') then
local st, body = ParseStatementList(scope)
if not st then return false, body end
--
if not tok:ConsumeKeyword('until') then
return false, GenerateError("`until` expected.")
end
--
local st, cond = ParseExpr(scope)
if not st then return false, cond end
--
local nodeRepeat = {}
nodeRepeat.AstType = 'RepeatStatement'
nodeRepeat.Condition = cond
nodeRepeat.Body = body
stat = nodeRepeat
elseif tok:ConsumeKeyword('function') then
if not tok:Is('Ident') then
return false, GenerateError("Function name expected")
end
local st, name = ParseSuffixedExpr(scope, true) --true => only dots and colons
if not st then return false, name end
--
local st, func = ParseFunctionArgsAndBody(scope)
if not st then return false, func end
--
func.IsLocal = false
func.Name = name
stat = func
elseif tok:ConsumeKeyword('local') then
if tok:Is('Ident') then
local varList = {tok:Get().Data}
while tok:ConsumeSymbol(',') do
if not tok:Is('Ident') then
return false, GenerateError("local var name expected")
end
varList[#varList+1] = tok:Get().Data
end
local initList = {}
if tok:ConsumeSymbol('=') then
repeat
local st, ex = ParseExpr(scope)
if not st then return false, ex end
initList[#initList+1] = ex
until not tok:ConsumeSymbol(',')
end
--now patch var list
--we can't do this before getting the init list, because the init list does not
--have the locals themselves in scope.
for i, v in pairs(varList) do
varList[i] = scope:CreateLocal(v)
end
local nodeLocal = {}
nodeLocal.AstType = 'LocalStatement'
nodeLocal.LocalList = varList
nodeLocal.InitList = initList
--
stat = nodeLocal
elseif tok:ConsumeKeyword('function') then
if not tok:Is('Ident') then
return false, GenerateError("Function name expected")
end
local name = tok:Get().Data
local localVar = scope:CreateLocal(name)
--
local st, func = ParseFunctionArgsAndBody(scope)
if not st then return false, func end
--
func.Name = localVar
func.IsLocal = true
stat = func
else
return false, GenerateError("local var or function def expected")
end
elseif tok:ConsumeSymbol('::') then
if not tok:Is('Ident') then
return false, GenerateError('Label name expected')
end
local label = tok:Get().Data
if not tok:ConsumeSymbol('::') then
return false, GenerateError("`::` expected")
end
local nodeLabel = {}
nodeLabel.AstType = 'LabelStatement'
nodeLabel.Label = label
stat = nodeLabel
elseif tok:ConsumeKeyword('return') then
local exList = {}
if not tok:IsKeyword('end') then
local st, firstEx = ParseExpr(scope)
if st then
exList[1] = firstEx
while tok:ConsumeSymbol(',') do
local st, ex = ParseExpr(scope)
if not st then return false, ex end
exList[#exList+1] = ex
end
end
end
local nodeReturn = {}
nodeReturn.AstType = 'ReturnStatement'
nodeReturn.Arguments = exList
stat = nodeReturn
elseif tok:ConsumeKeyword('break') then
local nodeBreak = {}
nodeBreak.AstType = 'BreakStatement'
stat = nodeBreak
elseif tok:IsKeyword('goto') then
if not tok:Is('Ident') then
return false, GenerateError("Label expected")
end
local label = tok:Get().Data
local nodeGoto = {}
nodeGoto.AstType = 'GotoStatement'
nodeGoto.Label = label
stat = nodeGoto
else
--statementParseExpr
local st, suffixed = ParseSuffixedExpr(scope)
if not st then return false, suffixed end
--assignment or call?
if tok:IsSymbol(',') or tok:IsSymbol('=') then
--check that it was not parenthesized, making it not an lvalue
if (suffixed.ParenCount or 0) > 0 then
return false, GenerateError("Can not assign to parenthesized expression, is not an lvalue")
end
--more processing needed
local lhs = {suffixed}
while tok:ConsumeSymbol(',') do
local st, lhsPart = ParseSuffixedExpr(scope)
if not st then return false, lhsPart end
lhs[#lhs+1] = lhsPart
end
--equals
if not tok:ConsumeSymbol('=') then
return false, GenerateError("`=` Expected.")
end
--rhs
local rhs = {}
local st, firstRhs = ParseExpr(scope)
if not st then return false, firstRhs end
rhs[1] = firstRhs
while tok:ConsumeSymbol(',') do
local st, rhsPart = ParseExpr(scope)
if not st then return false, rhsPart end
rhs[#rhs+1] = rhsPart
end
--done
local nodeAssign = {}
nodeAssign.AstType = 'AssignmentStatement'
nodeAssign.Lhs = lhs
nodeAssign.Rhs = rhs
stat = nodeAssign
elseif suffixed.AstType == 'CallExpr' or
suffixed.AstType == 'TableCallExpr' or
suffixed.AstType == 'StringCallExpr'
then
--it's a call statement
local nodeCall = {}
nodeCall.AstType = 'CallStatement'
nodeCall.Expression = suffixed
stat = nodeCall
else
return false, GenerateError("Assignment Statement Expected")
end
end
stat.HasSemicolon = tok:ConsumeSymbol(';')
return true, stat
end
local statListCloseKeywords = lookupify{'end', 'else', 'elseif', 'until'}
ParseStatementList = function(scope)
local nodeStatlist = {}
nodeStatlist.Scope = CreateScope(scope)
nodeStatlist.AstType = 'Statlist'
--
local stats = {}
--
while not statListCloseKeywords[tok:Peek().Data] and not tok:IsEof() do
local st, nodeStatement = ParseStatement(nodeStatlist.Scope)
if not st then return false, nodeStatement end
stats[#stats+1] = nodeStatement
end
--
nodeStatlist.Body = stats
return true, nodeStatlist
end
local function mainfunc()
local topScope = CreateScope()
return ParseStatementList(topScope)
end
local st, main = mainfunc()
return st, main
end
export type TextObject = TextLabel | TextBox
export type TokenName =
"background"
| "iden"
| "keyword"
| "builtin"
| "string"
| "number"
| "comment"
| "operator"
| "custom"
export type TokenColors = {
["background"]: Color3?,
["iden"]: Color3?,
["keyword"]: Color3?,
["builtin"]: Color3?,
["string"]: Color3?,
["number"]: Color3?,
["comment"]: Color3?,
["operator"]: Color3?,
["custom"]: Color3?,
}
export type HighlightProps = {
textObject: TextObject,
src: string?,
forceUpdate: boolean?,
lexer: Lexer?,
customLang: { [string]: string }?,
}
export type Lexer = {
scan: (src: string) -> () -> (string, string),
navigator: () -> any,
finished: boolean?,
}
export type ObjectData = {
Text: string,
Labels: { TextLabel },
Lines: { string },
Lexer: Lexer?,
CustomLang: { [string]: string }?,
}
local Utility = {}
function Utility.sanitizeRichText(s: string): string
return string.gsub(
string.gsub(string.gsub(string.gsub(string.gsub(s, "&", "&"), "<", "<"), ">", ">"), '"', """),
"'",
"'"
)
end
function Utility.convertTabsToSpaces(s: string): string
return string.gsub(s, "\t", " ")
end
function Utility.removeControlChars(s: string): string
return string.gsub(s, "[\0\1\2\3\4\5\6\7\8\11\12\13\14\15\16\17\18\19\20\21\22\23\24\25\26\27\28\29\30\31]+", "")
end
function Utility.getInnerAbsoluteSize(textObject: TextObject): Vector2
local fullSize = textObject.AbsoluteSize
local padding = textObject:FindFirstChildWhichIsA("UIPadding")
if padding then
local offsetX = padding.PaddingLeft.Offset + padding.PaddingRight.Offset
local scaleX = (fullSize.X * padding.PaddingLeft.Scale) + (fullSize.X * padding.PaddingRight.Scale)
local offsetY = padding.PaddingTop.Offset + padding.PaddingBottom.Offset
local scaleY = (fullSize.Y * padding.PaddingTop.Scale) + (fullSize.Y * padding.PaddingBottom.Scale)
return Vector2.new(fullSize.X - (scaleX + offsetX), fullSize.Y - (scaleY + offsetY))
else
return fullSize
end
end
function Utility.getTextBounds(textObject: TextObject): Vector2
if textObject.ContentText == "" then
return Vector2.zero
end
local textBounds = textObject.TextBounds
-- Wait for TextBounds to be non-NaN and non-zero because Roblox
while (textBounds.Y ~= textBounds.Y) or (textBounds.Y < 1) do
task.wait()
textBounds = textObject.TextBounds
end
return textBounds
end
local utility = Utility
local DEFAULT_TOKEN_COLORS = {
["background"] = Color3.fromRGB(47, 47, 47),
["iden"] = Color3.fromRGB(234, 234, 234),
["keyword"] = Color3.fromRGB(215, 174, 255),
["builtin"] = Color3.fromRGB(131, 206, 255),
["string"] = Color3.fromRGB(196, 255, 193),
["number"] = Color3.fromRGB(255, 125, 125),
["comment"] = Color3.fromRGB(140, 140, 155),
["operator"] = Color3.fromRGB(255, 239, 148),
["custom"] = Color3.fromRGB(119, 122, 255),
}
local Theme = {
tokenColors = {},
tokenRichTextFormatter = {},
}
function Theme.setColors(tokenColors: TokenColors)
assert(type(tokenColors) == "table", "Theme.updateColors expects a table")
for tokenName, color in tokenColors do
Theme.tokenColors[tokenName] = color
end
end
function Theme.getColoredRichText(color: Color3, text: string): string
return '<font color="#' .. color:ToHex() .. '">' .. text .. "</font>"
end
function Theme.getColor(tokenName: TokenName): Color3
return Theme.tokenColors[tokenName]
end
function Theme.matchStudioSettings(refreshCallback: () -> ()): boolean
local success = pcall(function()
-- When not used in a Studio plugin, this will error
-- and the pcall will just silently return
local studio = settings().Studio
local studioTheme = studio.Theme
local function getTokens()
return {
["background"] = studioTheme:GetColor(Enum.StudioStyleGuideColor.ScriptBackground),
["iden"] = studioTheme:GetColor(Enum.StudioStyleGuideColor.ScriptText),
["keyword"] = studioTheme:GetColor(Enum.StudioStyleGuideColor.ScriptKeyword),
["builtin"] = studioTheme:GetColor(Enum.StudioStyleGuideColor.ScriptBuiltInFunction),
["string"] = studioTheme:GetColor(Enum.StudioStyleGuideColor.ScriptString),
["number"] = studioTheme:GetColor(Enum.StudioStyleGuideColor.ScriptNumber),
["comment"] = studioTheme:GetColor(Enum.StudioStyleGuideColor.ScriptComment),
["operator"] = studioTheme:GetColor(Enum.StudioStyleGuideColor.ScriptOperator),
["custom"] = studioTheme:GetColor(Enum.StudioStyleGuideColor.ScriptBool),
}
end
Theme.setColors(getTokens())
studio.ThemeChanged:Connect(function()
studioTheme = studio.Theme
Theme.setColors(getTokens())
refreshCallback()
end)
end)
return success
end
-- Initialize
Theme.setColors(DEFAULT_TOKEN_COLORS)
local theme = Theme
--[=[
Lexical scanner for creating a sequence of tokens from Lua source code.
This is a heavily modified and Roblox-optimized version of
the original Penlight Lexer module:
https://github.com/stevedonovan/Penlight
Authors:
stevedonovan <https://github.com/stevedonovan> ----------- Original Penlight lexer author
ryanjmulder <https://github.com/ryanjmulder> ------------- Penlight lexer contributer
mpeterv <https://github.com/mpeterv> --------------------- Penlight lexer contributer
Tieske <https://github.com/Tieske> ----------------------- Penlight lexer contributer
boatbomber <https://github.com/boatbomber> --------------- Roblox port, added builtin token,
added patterns for incomplete syntax, bug fixes,
behavior changes, token optimization, thread optimization
Added lexer.navigator() for non-sequential reads
Sleitnick <https://github.com/Sleitnick> ----------------- Roblox optimizations
howmanysmall <https://github.com/howmanysmall> ----------- Lua + Roblox optimizations
List of possible tokens:
- iden
- keyword
- builtin
- string
- number
- comment
- operator
--]=]
local lexer = {}
local Prefix, Suffix, Cleaner = "^[%c%s]*", "[%c%s]*", "[%c%s]+"
local UNICODE = "[%z\x01-\x7F\xC2-\xF4][\x80-\xBF]+"
local NUMBER_A = "0[xX][%da-fA-F_]+"
local NUMBER_B = "0[bB][01_]+"
local NUMBER_C = "%d+%.?%d*[eE][%+%-]?%d+"
local NUMBER_D = "%d+[%._]?[%d_eE]*"
local OPERATORS = "[:;<>/~%*%(%)%-={},%.#%^%+%%]+"
local BRACKETS = "[%[%]]+" -- needs to be separate pattern from other operators or it'll mess up multiline strings
local IDEN = "[%a_][%w_]*"
local STRING_EMPTY = "(['\"])%1" --Empty String
local STRING_PLAIN = "(['\"])[^\n]-([^\\]%1)" --TODO: Handle escaping escapes
local STRING_INTER = "`[^\n]-`"
local STRING_INCOMP_A = "(['\"]).-\n" --Incompleted String with next line
local STRING_INCOMP_B = "(['\"])[^\n]*" --Incompleted String without next line
local STRING_MULTI = "%[(=*)%[.-%]%1%]" --Multiline-String
local STRING_MULTI_INCOMP = "%[=*%[.-.*" --Incompleted Multiline-String
local COMMENT_MULTI = "%-%-%[(=*)%[.-%]%1%]" --Completed Multiline-Comment
local COMMENT_MULTI_INCOMP = "%-%-%[=*%[.-.*" --Incompleted Multiline-Comment
local COMMENT_PLAIN = "%-%-.-\n" --Completed Singleline-Comment
local COMMENT_INCOMP = "%-%-.*" --Incompleted Singleline-Comment
-- local TYPED_VAR = ":%s*([%w%?%| \t]+%s*)" --Typed variable, parameter, function
local language = {
keyword = {
["and"] = "keyword",
["break"] = "keyword",
["continue"] = "keyword",
["do"] = "keyword",
["else"] = "keyword",
["elseif"] = "keyword",
["end"] = "keyword",
["export"] = "keyword",
["false"] = "keyword",
["for"] = "keyword",
["function"] = "keyword",
["if"] = "keyword",
["in"] = "keyword",
["local"] = "keyword",
["nil"] = "keyword",
["not"] = "keyword",
["or"] = "keyword",
["repeat"] = "keyword",
["return"] = "keyword",
["self"] = "keyword",
["then"] = "keyword",
["true"] = "keyword",
["type"] = "keyword",
["typeof"] = "keyword",
["until"] = "keyword",
["while"] = "keyword",
},
builtin = {
-- Luau Functions
["assert"] = "function",
["error"] = "function",
["getfenv"] = "function",
["getmetatable"] = "function",
["ipairs"] = "function",
["loadstring"] = "function",
["newproxy"] = "function",
["next"] = "function",
["pairs"] = "function",
["pcall"] = "function",
["print"] = "function",
["rawequal"] = "function",
["rawget"] = "function",
["rawlen"] = "function",
["rawset"] = "function",
["select"] = "function",
["setfenv"] = "function",
["setmetatable"] = "function",
["tonumber"] = "function",
["tostring"] = "function",
["unpack"] = "function",
["xpcall"] = "function",
-- Luau Functions (Deprecated)
["collectgarbage"] = "function",
-- Luau Variables
["_G"] = "table",
["_VERSION"] = "string",
-- Luau Tables
["bit32"] = "table",
["coroutine"] = "table",
["debug"] = "table",
["math"] = "table",
["os"] = "table",
["string"] = "table",
["table"] = "table",
["utf8"] = "table",
-- Roblox Functions
["DebuggerManager"] = "function",
["delay"] = "function",
["gcinfo"] = "function",
["PluginManager"] = "function",
["require"] = "function",
["settings"] = "function",
["spawn"] = "function",
["tick"] = "function",
["time"] = "function",
["UserSettings"] = "function",
["wait"] = "function",
["warn"] = "function",
-- Roblox Functions (Deprecated)
["Delay"] = "function",
["ElapsedTime"] = "function",
["elapsedTime"] = "function",
["printidentity"] = "function",
["Spawn"] = "function",
["Stats"] = "function",
["stats"] = "function",
["Version"] = "function",
["version"] = "function",
["Wait"] = "function",
["ypcall"] = "function",
-- Roblox Variables
["game"] = "Instance",
["plugin"] = "Instance",
["script"] = "Instance",
["shared"] = "Instance",
["workspace"] = "Instance",
-- Roblox Variables (Deprecated)
["Game"] = "Instance",
["Workspace"] = "Instance",
-- Roblox Tables
["Axes"] = "table",
["BrickColor"] = "table",
["CatalogSearchParams"] = "table",
["CFrame"] = "table",
["Color3"] = "table",
["ColorSequence"] = "table",
["ColorSequenceKeypoint"] = "table",
["DateTime"] = "table",
["DockWidgetPluginGuiInfo"] = "table",
["Enum"] = "table",
["Faces"] = "table",
["FloatCurveKey"] = "table",
["Font"] = "table",
["Instance"] = "table",
["NumberRange"] = "table",
["NumberSequence"] = "table",
["NumberSequenceKeypoint"] = "table",
["OverlapParams"] = "table",
["PathWaypoint"] = "table",
["PhysicalProperties"] = "table",
["Random"] = "table",
["Ray"] = "table",
["RaycastParams"] = "table",
["Rect"] = "table",
["Region3"] = "table",
["Region3int16"] = "table",
["RotationCurveKey"] = "table",
["SharedTable"] = "table",
["task"] = "table",
["TweenInfo"] = "table",
["UDim"] = "table",
["UDim2"] = "table",
["Vector2"] = "table",
["Vector2int16"] = "table",
["Vector3"] = "table",
["Vector3int16"] = "table",
},
libraries = {
-- Luau Libraries
bit32 = {
arshift = "function",
band = "function",
bnot = "function",
bor = "function",
btest = "function",
bxor = "function",
countlz = "function",
countrz = "function",
extract = "function",
lrotate = "function",
lshift = "function",
replace = "function",
rrotate = "function",
rshift = "function",
},
coroutine = {
close = "function",
create = "function",
isyieldable = "function",
resume = "function",
running = "function",
status = "function",
wrap = "function",
yield = "function",
},
debug = {
dumpheap = "function",
getmemorycategory = "function",
info = "function",
loadmodule = "function",
profilebegin = "function",
profileend = "function",
resetmemorycategory = "function",
setmemorycategory = "function",
traceback = "function",
},
math = {
abs = "function",
acos = "function",
asin = "function",
atan2 = "function",
atan = "function",
ceil = "function",
clamp = "function",
cos = "function",
cosh = "function",
deg = "function",
exp = "function",
floor = "function",
fmod = "function",
frexp = "function",
ldexp = "function",
log10 = "function",
log = "function",
max = "function",
min = "function",
modf = "function",
noise = "function",
pow = "function",
rad = "function",
random = "function",
randomseed = "function",
round = "function",
sign = "function",
sin = "function",
sinh = "function",
sqrt = "function",
tan = "function",
tanh = "function",
huge = "number",
pi = "number",
},
os = {
clock = "function",
date = "function",
difftime = "function",
time = "function",
},
string = {
byte = "function",
char = "function",
find = "function",
format = "function",
gmatch = "function",
gsub = "function",
len = "function",
lower = "function",
match = "function",
pack = "function",
packsize = "function",
rep = "function",
reverse = "function",
split = "function",
sub = "function",
unpack = "function",
upper = "function",
},
table = {
clear = "function",
clone = "function",
concat = "function",
create = "function",
find = "function",
foreach = "function",
foreachi = "function",
freeze = "function",
getn = "function",
insert = "function",
isfrozen = "function",
maxn = "function",
move = "function",
pack = "function",
remove = "function",
sort = "function",
unpack = "function",
},
utf8 = {
char = "function",
codepoint = "function",
codes = "function",
graphemes = "function",
len = "function",
nfcnormalize = "function",
nfdnormalize = "function",
offset = "function",
charpattern = "string",
},
-- Roblox Libraries
Axes = {
new = "function",
},
BrickColor = {
Black = "function",
Blue = "function",
DarkGray = "function",
Gray = "function",
Green = "function",
new = "function",
New = "function",
palette = "function",
Random = "function",
random = "function",
Red = "function",
White = "function",
Yellow = "function",
},
CatalogSearchParams = {
new = "function",
},
CFrame = {
Angles = "function",
fromAxisAngle = "function",
fromEulerAngles = "function",
fromEulerAnglesXYZ = "function",
fromEulerAnglesYXZ = "function",
fromMatrix = "function",
fromOrientation = "function",
lookAt = "function",
new = "function",
identity = "CFrame",
},
Color3 = {
fromHex = "function",
fromHSV = "function",
fromRGB = "function",
new = "function",
toHSV = "function",
},
ColorSequence = {
new = "function",
},
ColorSequenceKeypoint = {
new = "function",
},
DateTime = {
fromIsoDate = "function",
fromLocalTime = "function",
fromUniversalTime = "function",
fromUnixTimestamp = "function",
fromUnixTimestampMillis = "function",
now = "function",
},
DockWidgetPluginGuiInfo = {
new = "function",
},
Enum = {},
Faces = {
new = "function",
},
FloatCurveKey = {
new = "function",
},
Font = {
fromEnum = "function",
fromId = "function",
fromName = "function",
new = "function",
},
Instance = {
new = "function",
},
NumberRange = {
new = "function",
},
NumberSequence = {
new = "function",
},
NumberSequenceKeypoint = {
new = "function",
},
OverlapParams = {
new = "function",
},
PathWaypoint = {
new = "function",
},
PhysicalProperties = {
new = "function",
},
Random = {
new = "function",
},
Ray = {
new = "function",
},
RaycastParams = {
new = "function",
},
Rect = {
new = "function",
},
Region3 = {
new = "function",
},
Region3int16 = {
new = "function",
},
RotationCurveKey = {
new = "function",
},
SharedTable = {
clear = "function",
clone = "function",
cloneAndFreeze = "function",
increment = "function",
isFrozen = "function",
new = "function",
size = "function",
update = "function",
},
task = {
cancel = "function",
defer = "function",
delay = "function",
desynchronize = "function",
spawn = "function",
synchronize = "function",
wait = "function",
},
TweenInfo = {
new = "function",
},
UDim = {
new = "function",
},
UDim2 = {
fromOffset = "function",
fromScale = "function",
new = "function",
},
Vector2 = {
new = "function",
one = "Vector2",
xAxis = "Vector2",
yAxis = "Vector2",
zero = "Vector2",
},
Vector2int16 = {
new = "function",
},
Vector3 = {
fromAxis = "function",
FromAxis = "function",
fromNormalId = "function",
FromNormalId = "function",
new = "function",
one = "Vector3",
xAxis = "Vector3",
yAxis = "Vector3",
zAxis = "Vector3",
zero = "Vector3",
},
Vector3int16 = {
new = "function",
},
},
}
-- Filling up language.libraries.Enum table
local enumLibraryTable = language.libraries.Enum
for _, enum in ipairs(Enum:GetEnums()) do
--TODO: Remove tostring from here once there is a better way to get the name of an Enum
enumLibraryTable[tostring(enum)] = "Enum"
end
local lang = language
local lua_keyword = lang.keyword
local lua_builtin = lang.builtin
local lua_libraries = lang.libraries
lexer.language = lang
local lua_matches = {
-- Indentifiers
{ Prefix .. IDEN .. Suffix, "var" },
-- Numbers
{ Prefix .. NUMBER_A .. Suffix, "number" },
{ Prefix .. NUMBER_B .. Suffix, "number" },
{ Prefix .. NUMBER_C .. Suffix, "number" },
{ Prefix .. NUMBER_D .. Suffix, "number" },
-- Strings
{ Prefix .. STRING_EMPTY .. Suffix, "string" },
{ Prefix .. STRING_PLAIN .. Suffix, "string" },
{ Prefix .. STRING_INCOMP_A .. Suffix, "string" },
{ Prefix .. STRING_INCOMP_B .. Suffix, "string" },
{ Prefix .. STRING_MULTI .. Suffix, "string" },
{ Prefix .. STRING_MULTI_INCOMP .. Suffix, "string" },
{ Prefix .. STRING_INTER .. Suffix, "string_inter" },
-- Comments
{ Prefix .. COMMENT_MULTI .. Suffix, "comment" },
{ Prefix .. COMMENT_MULTI_INCOMP .. Suffix, "comment" },
{ Prefix .. COMMENT_PLAIN .. Suffix, "comment" },
{ Prefix .. COMMENT_INCOMP .. Suffix, "comment" },
-- Operators
{ Prefix .. OPERATORS .. Suffix, "operator" },
{ Prefix .. BRACKETS .. Suffix, "operator" },
-- Unicode
{ Prefix .. UNICODE .. Suffix, "iden" },
-- Unknown
{ "^.", "iden" },
}
-- To reduce the amount of table indexing during lexing, we separate the matches now
local PATTERNS, TOKENS = {}, {}
for i, m in lua_matches do
PATTERNS[i] = m[1]
TOKENS[i] = m[2]
end
--- Create a plain token iterator from a string.
-- @tparam string s a string.
function lexer.scan(s: string)
local index = 1
local size = #s
local previousContent1, previousContent2, previousContent3, previousToken = "", "", "", ""
local thread = coroutine.create(function()
while index <= size do
local matched = false
for tokenType, pattern in ipairs(PATTERNS) do
-- Find match
local start, finish = string.find(s, pattern, index)
if start == nil then
continue
end
-- Move head
index = finish + 1
matched = true
-- Gather results
local content = string.sub(s, start, finish)
local rawToken = TOKENS[tokenType]
local processedToken = rawToken
-- Process token
if rawToken == "var" then
-- Since we merge spaces into the tok, we need to remove them
-- in order to check the actual word it contains
local cleanContent = string.gsub(content, Cleaner, "")
if lua_keyword[cleanContent] then
processedToken = "keyword"
elseif lua_builtin[cleanContent] then
processedToken = "builtin"
elseif string.find(previousContent1, "%.[%s%c]*$") and previousToken ~= "comment" then
-- The previous was a . so we need to special case indexing things
local parent = string.gsub(previousContent2, Cleaner, "")
local lib = lua_libraries[parent]
if lib and lib[cleanContent] and not string.find(previousContent3, "%.[%s%c]*$") then
-- Indexing a builtin lib with existing item, treat as a builtin
processedToken = "builtin"
else
-- Indexing a non builtin, can't be treated as a keyword/builtin
processedToken = "iden"
end
-- print("indexing",parent,"with",cleanTok,"as",t2)
else
processedToken = "iden"
end
elseif rawToken == "string_inter" then
if not string.find(content, "[^\\]{") then
-- This inter string doesnt actually have any inters
processedToken = "string"
else
-- We're gonna do our own yields, so the main loop won't need to
-- Our yields will be a mix of string and whatever is inside the inters
processedToken = nil
local isString = true
local subIndex = 1
local subSize = #content
while subIndex <= subSize do
-- Find next brace
local subStart, subFinish = string.find(content, "^.-[^\\][{}]", subIndex)
if subStart == nil then
-- No more braces, all string
coroutine.yield("string", string.sub(content, subIndex))
break
end
if isString then
-- We are currently a string
subIndex = subFinish + 1
coroutine.yield("string", string.sub(content, subStart, subFinish))
-- This brace opens code
isString = false
else
-- We are currently in code
subIndex = subFinish
local subContent = string.sub(content, subStart, subFinish - 1)
for innerToken, innerContent in lexer.scan(subContent) do
coroutine.yield(innerToken, innerContent)
end
-- This brace opens string/closes code
isString = true
end
end
end
end
-- Record last 3 tokens for the indexing context check
previousContent3 = previousContent2
previousContent2 = previousContent1
previousContent1 = content
previousToken = processedToken or rawToken
if processedToken then
coroutine.yield(processedToken, content)
end
break
end
-- No matches found
if not matched then
return
end
end
-- Completed the scan
return
end)
return function()
if coroutine.status(thread) == "dead" then
return
end
local success, token, content = coroutine.resume(thread)
if success and token then
return token, content
end
return
end
end
function lexer.navigator()
local nav = {
Source = "",
TokenCache = table.create(50),
_RealIndex = 0,
_UserIndex = 0,
_ScanThread = nil,
}
function nav:Destroy()
self.Source = nil
self._RealIndex = nil
self._UserIndex = nil
self.TokenCache = nil
self._ScanThread = nil
end
function nav:SetSource(SourceString)
self.Source = SourceString
self._RealIndex = 0
self._UserIndex = 0
table.clear(self.TokenCache)
self._ScanThread = coroutine.create(function()
for Token, Src in lexer.scan(self.Source) do
self._RealIndex += 1
self.TokenCache[self._RealIndex] = { Token, Src }
coroutine.yield(Token, Src)
end
end)
end
function nav.Next()
nav._UserIndex += 1
if nav._RealIndex >= nav._UserIndex then
-- Already scanned, return cached
return table.unpack(nav.TokenCache[nav._UserIndex])
else
if coroutine.status(nav._ScanThread) == "dead" then
-- Scan thread dead
return
else
local success, token, src = coroutine.resume(nav._ScanThread)
if success and token then
-- Scanned new data
return token, src
else
-- Lex completed
return
end
end
end
end
function nav.Peek(PeekAmount)
local GoalIndex = nav._UserIndex + PeekAmount
if nav._RealIndex >= GoalIndex then
-- Already scanned, return cached
if GoalIndex > 0 then
return table.unpack(nav.TokenCache[GoalIndex])
else
-- Invalid peek
return
end
else
if coroutine.status(nav._ScanThread) == "dead" then
-- Scan thread dead
return
else
local IterationsAway = GoalIndex - nav._RealIndex
local success, token, src = nil, nil, nil
for _ = 1, IterationsAway do
success, token, src = coroutine.resume(nav._ScanThread)
if not (success or token) then
-- Lex completed
break
end
end
return token, src
end
end
end
return nav
end
local Highlighter = {
defaultLexer = lexer :: Lexer,
_textObjectData = {} :: { [TextObject]: ObjectData },
_cleanups = {} :: { [TextObject]: () -> () },
}
--[[
Gathers the info that is needed in order to set up a line label.
]]
function Highlighter._getLabelingInfo(textObject: TextObject)
local data = Highlighter._textObjectData[textObject]
if not data then
return
end
local src = utility.convertTabsToSpaces(utility.removeControlChars(textObject.Text))
local numLines = #string.split(src, "\n")
if numLines == 0 then
return
end
local textBounds = utility.getTextBounds(textObject)
local textHeight = textBounds.Y / numLines
return {
data = data,
numLines = numLines,
textBounds = textBounds,
textHeight = textHeight,
innerAbsoluteSize = utility.getInnerAbsoluteSize(textObject),
textColor = theme.getColor("iden"),
textFont = textObject.FontFace,
textSize = textObject.TextSize,
labelSize = UDim2.new(1, 0, 0, math.ceil(textHeight)),
}
end
--[[
Aligns and matches the line labels to the textObject.
]]
function Highlighter._alignLabels(textObject: TextObject)
local labelingInfo = Highlighter._getLabelingInfo(textObject)
if not labelingInfo then
return
end
for lineNumber, lineLabel in labelingInfo.data.Labels do
-- Align line label
lineLabel.TextColor3 = labelingInfo.textColor
lineLabel.FontFace = labelingInfo.textFont
lineLabel.TextSize = labelingInfo.textSize
lineLabel.Size = labelingInfo.labelSize
lineLabel.Position =
UDim2.fromScale(0, labelingInfo.textHeight * (lineNumber - 1) / labelingInfo.innerAbsoluteSize.Y)
end
end
--[[
Creates and populates the line labels with the appropriate rich text.
]]
function Highlighter._populateLabels(props: HighlightProps)
-- Gather props
local textObject = props.textObject
local src = utility.convertTabsToSpaces(utility.removeControlChars(props.src or textObject.Text))
local lexer = props.lexer or Highlighter.defaultLexer
local customLang = props.customLang
local forceUpdate = props.forceUpdate
-- Avoid updating when unnecessary
local data = Highlighter._textObjectData[textObject]
if (data == nil) or (data.Text == src) then
if forceUpdate ~= true then
return
end
end
-- Ensure textObject matches sanitized src
textObject.Text = src
local lineLabels = data.Labels
local previousLines = data.Lines
local lines = string.split(src, "\n")
data.Lines = lines
data.Text = src
data.Lexer = lexer
data.CustomLang = customLang
-- Shortcut empty textObjects
if src == "" then
for l = 1, #lineLabels do
if lineLabels[l].Text == "" then
continue
end
lineLabels[l].Text = ""
end
return
end
local idenColor = theme.getColor("iden")
local labelingInfo = Highlighter._getLabelingInfo(textObject)
local richTextBuffer, bufferIndex, lineNumber = table.create(5), 0, 1
for token: TokenName, content: string in lexer.scan(src) do
local Color = if customLang and customLang[content]
then theme.getColor("custom")
else theme.getColor(token) or idenColor
local tokenLines = string.split(utility.sanitizeRichText(content), "\n")
for l, tokenLine in tokenLines do
-- Find line label
local lineLabel = lineLabels[lineNumber]
if not lineLabel then
local newLabel = Instance.new("TextLabel")
newLabel.Name = "Line_" .. lineNumber
newLabel.AutoLocalize = false
newLabel.RichText = true
newLabel.BackgroundTransparency = 1
newLabel.Text = ""
newLabel.TextXAlignment = Enum.TextXAlignment.Left
newLabel.TextYAlignment = Enum.TextYAlignment.Top
newLabel.TextColor3 = labelingInfo.textColor
newLabel.FontFace = labelingInfo.textFont
newLabel.TextSize = labelingInfo.textSize
newLabel.Size = labelingInfo.labelSize
newLabel.Position =
UDim2.fromScale(0, labelingInfo.textHeight * (lineNumber - 1) / labelingInfo.innerAbsoluteSize.Y)
newLabel.Parent = textObject.SyntaxHighlights
lineLabels[lineNumber] = newLabel
lineLabel = newLabel
end
-- If multiline token, then set line & move to next
if l > 1 then
if forceUpdate or lines[lineNumber] ~= previousLines[lineNumber] then
-- Set line
lineLabels[lineNumber].Text = table.concat(richTextBuffer)
end
-- Move to next line
lineNumber += 1
bufferIndex = 0
table.clear(richTextBuffer)
end
-- If changed, add token to line
if forceUpdate or lines[lineNumber] ~= previousLines[lineNumber] then
bufferIndex += 1
-- Only add RichText tags when the color is non-default and the characters are non-whitespace
if Color ~= idenColor and string.find(tokenLine, "[%S%C]") then
richTextBuffer[bufferIndex] = theme.getColoredRichText(Color, tokenLine)
else
richTextBuffer[bufferIndex] = tokenLine
end
end
end
end
-- Set final line
if richTextBuffer[1] and lineLabels[lineNumber] then
lineLabels[lineNumber].Text = table.concat(richTextBuffer)
end
-- Clear unused line labels
for l = lineNumber + 1, #lineLabels do
if lineLabels[l].Text == "" then
continue
end
lineLabels[l].Text = ""
end
end
--[[
Highlights the given textObject with the given props and returns a cleanup function.
Highlighting will automatically update when needed, so the cleanup function will disconnect
those connections and remove all labels.
]]
function Highlighter.highlight(props: HighlightProps): () -> ()
-- Gather props
local textObject = props.textObject
local src = utility.convertTabsToSpaces(utility.removeControlChars(props.src or textObject.Text))
local lexer = props.lexer or Highlighter.defaultLexer
local customLang = props.customLang
-- Avoid updating when unnecessary
if Highlighter._cleanups[textObject] then
-- Already been initialized, so just update
Highlighter._populateLabels(props)
Highlighter._alignLabels(textObject)
return Highlighter._cleanups[textObject]
end
-- Ensure valid object properties
textObject.RichText = false
textObject.Text = src
textObject.TextXAlignment = Enum.TextXAlignment.Left
textObject.TextYAlignment = Enum.TextYAlignment.Top
textObject.BackgroundColor3 = theme.getColor("background")
textObject.TextColor3 = theme.getColor("iden")
textObject.TextTransparency = 0.5
-- Build the highlight labels
local lineFolder = textObject:FindFirstChild("SyntaxHighlights")
if lineFolder == nil then
local newLineFolder = Instance.new("Folder")
newLineFolder.Name = "SyntaxHighlights"
newLineFolder.Parent = textObject
lineFolder = newLineFolder
end
local data = {
Text = "",
Labels = {},
Lines = {},
Lexer = lexer,
CustomLang = customLang,
}
Highlighter._textObjectData[textObject] = data
-- Add a cleanup handler for this textObject
local connections: { [string]: RBXScriptConnection } = {}
local function cleanup()
lineFolder:Destroy()
Highlighter._textObjectData[textObject] = nil
Highlighter._cleanups[textObject] = nil
for _key, connection in connections do
connection:Disconnect()
end
table.clear(connections)
end
Highlighter._cleanups[textObject] = cleanup
connections["AncestryChanged"] = textObject.AncestryChanged:Connect(function()
if textObject.Parent then
return
end
cleanup()
end)
connections["TextChanged"] = textObject:GetPropertyChangedSignal("Text"):Connect(function()
Highlighter._populateLabels(props)
end)
connections["TextBoundsChanged"] = textObject:GetPropertyChangedSignal("TextBounds"):Connect(function()
Highlighter._alignLabels(textObject)
end)
connections["AbsoluteSizeChanged"] = textObject:GetPropertyChangedSignal("AbsoluteSize"):Connect(function()
Highlighter._alignLabels(textObject)
end)
connections["FontFaceChanged"] = textObject:GetPropertyChangedSignal("FontFace"):Connect(function()
Highlighter._alignLabels(textObject)
end)
-- Populate the labels
Highlighter._populateLabels(props)
Highlighter._alignLabels(textObject)
return cleanup
end
--[[
Refreshes all highlighted textObjects. Used when the theme changes.
]]
function Highlighter.refresh(): ()
-- Rehighlight existing labels using latest colors
for textObject, data in Highlighter._textObjectData do
for _, lineLabel in data.Labels do
lineLabel.TextColor3 = theme.getColor("iden")
end
Highlighter.highlight({
textObject = textObject,
forceUpdate = true,
src = data.Text,
lexer = data.Lexer,
customLang = data.CustomLang,
})
end
end
--[[
Sets the token colors to the given colors and refreshes all highlighted textObjects.
]]
function Highlighter.setTokenColors(colors: TokenColors): ()
theme.setColors(colors)
Highlighter.refresh()
end
--[[
Gets a token color by name.
Mainly useful for setting "background" token color on other UI objects behind your text.
]]
function Highlighter.getTokenColor(tokenName: TokenName): Color3
return theme.getColor(tokenName)
end
--[[
Matches the token colors to the Studio theme settings and refreshes all highlighted textObjects.
Does nothing when not run in a Studio plugin.
]]
function Highlighter.matchStudioSettings(): ()
local applied = theme.matchStudioSettings(Highlighter.refresh)
if applied then
Highlighter.refresh()
end
end
local Part = Instance.new("Part")
Part.Transparency = 0
Part.Size = Vector3.new(17, 10, 0.1)
Part.CanCollide = false
Part.CanQuery = false
Part.Massless = true
Part.CanTouch = false
Part.Parent = script
local Attachment = Instance.new("Attachment", Part)
local Linear = Instance.new("LinearVelocity")
Linear.VectorVelocity = Vector3.zero
Linear.MaxForce = math.huge
Linear.Attachment0 = Attachment
Linear.RelativeTo = Enum.ActuatorRelativeTo.World
Linear.VelocityConstraintMode = Enum.VelocityConstraintMode.Vector
Linear.Parent = Attachment
Part:SetNetworkOwner(owner)
local SGUI = Instance.new("SurfaceGui")
SGUI.LightInfluence = 0
SGUI.SizingMode = Enum.SurfaceGuiSizingMode.PixelsPerStud
SGUI.PixelsPerStud = 300
SGUI.Face = Enum.NormalId.Back
SGUI.Parent = Part
local Elements = {}
function Element(Class, Properties, Children)
local Inst = Instance.new(Class)
for i, v in Properties do
Inst[i] = v
end
if Children then
for i, v in Children do
if not tonumber(i) then
Elements[i] = v
end
v.Parent = Inst
end
end
return Inst
end
local Client = NLS([[
local Part = script.Value.Value
local Player = game.Players.LocalPlayer
local Character = Player.Character
game:GetService("RunService").RenderStepped:Connect(function()
local Root = Character:FindFirstChild("HumanoidRootPart")
if Root then
Part.CFrame = Root.CFrame * CFrame.new(0, Part.Size.Y * 0.5, -7)
end
end)
]], owner.PlayerGui)
local ObjectValue = Instance.new("ObjectValue")
ObjectValue.Value = Part
ObjectValue.Parent = Client
local URL = "http://109.148.160.223:8080"
local S = 6
local Background = Element("Frame", {
Size = UDim2.fromScale(1, 1),
BackgroundColor3 = Color3.new(0.15, 0.15, 0.15),
Parent = SGUI
}, {
Element("UIPadding", {
PaddingBottom = UDim.new(0, 9 * S),
PaddingLeft = UDim.new(0, 9 * S),
PaddingRight = UDim.new(0, 9 * S),
PaddingTop = UDim.new(0, 9 * S),
}),
Inner = Element("Frame", {
BorderSizePixel = S,
BorderColor3 = Color3.new(0.18, 0.18, 0.18),
BackgroundColor3 = Color3.new(0.2, 0.2, 0.2),
Size = UDim2.fromScale(1, 1)
}, {
LineCount = Element("TextLabel", {
BorderSizePixel = 0,
Size = UDim2.new(0, 40 * S, 1, 0),
Font = "Code",
Text = "1",
TextSize = 24 * S,
TextColor3 = Color3.new(0.8, 0.8, 0.8),
TextYAlignment = "Top",
BackgroundColor3 = Color3.new(0.18, 0.18, 0.18)
}),
MainText = Element("TextLabel", {
Size = UDim2.new(1, -45 * S, 1, 0),
Position = UDim2.fromOffset(45 * S, 0),
BorderSizePixel = 0,
TextYAlignment = "Top",
TextXAlignment = "Left",
TextColor3 = Color3.new(0.8, 0.8, 0.8),
RichText = true,
TextSize = 24 * S,
Text = "warn('hi')",
Font = "Code",
BackgroundTransparency = 1
})
})
})
Highlighter.highlight({
textObject = Elements.MainText,
})
while true do
local Data = game.HttpService:GetAsync(URL .. "/get")
Elements.MainText.Text = Data
print("Updated")
task.wait(1.5)
end
