Useful networking wrappers

[AzuiSleet]

[QUOTE=Lau;19573400]It doesn't pool smsg names for the specific reason that pooling doesn't even work right in source <-> lua interface, if you notice: umsg.PoolString does nothing, and it pools after a string has been sent 5 times.[/QUOTE]

I did a little bit of research into this a while ago:

umsg.PoolString works, it puts the string in the LuaStringTable (which the server and client use for the umsg. library as well as other parts)

code:
[server]
umsg.PoolString("Test String")

[client]
dumpstringtables_new LuaStringTable

The handling of pooled strings is transparent, when you use umsg.Start it will look up the index of the string in the table or add it otherwise.

As for pooled strings with umsg.String, it writes to the buffer a bit for pooled, and either the index or the string. Here's what I get (see this thread [url]http://www.facepunch.com/showthread.php?t=705037[/url]):

[server]
lua_run umsg.Start("a") umsg.String("a") umsg.End()
lua_run umsg.Start("a") umsg.String("Test String") umsg.End()
lua_run umsg.Start("a") umsg.String("Test Stringaaa") umsg.End()

[client]
umsg incoming: a size: 5
Warning: Unhandled usermessage 'a'
umsg incoming: a size: 5
Warning: Unhandled usermessage 'a'
umsg incoming: a size: 18
Warning: Unhandled usermessage 'a'

So yes, pooling works, unless you're expecting it to magically pool the inside of a larger string.

But you should also be aware that string table synchronization isn't instant, it still has to be networked.

[TomyLobo]

why rely on gmod's string tables?
you could just make your own string tables and at least send smsg IDs instead of string names.

and yeah, I could have read the source but
1. I couldn't be asked
2. I'm sure there's a lot of voodoo in a library like that and the interesting parts are hard to find
3. asking made AzuiSleet post his research :)

an interesting thought about pooling:
if client-side console command names are pooled as well, wouldn't that make it easy to overflow the server's string table? (provided you find a way to make it execute those commands on you)

[AzuiSleet]

I might have been wrong about the size of the lua usermessage, because I took a look at gm_transmittools

int umsgStringTableOffset table->AddString(true, "N") << 1 | 1

bf_write *bf_write = engine->UserMessageBegin(&filter, 28); // LuaUserMessage

bf_write->WriteShort(umsgStringTableOffset);
bf_write->WriteOneBit(0); // important bit!

Either way, the first part of the umsgs (before any actual data) is still 3 bytes. And I honestly don't remember why it's like that with the header, I wonder if it's supposed to be bit + short.

[Lau]

[QUOTE=AzuiSleet;19593832]I might have been wrong about the size of the lua usermessage, because I took a look at gm_transmittools

int umsgStringTableOffset table->AddString(true, "N") << 1 | 1

bf_write *bf_write = engine->UserMessageBegin(&filter, 28); // LuaUserMessage

bf_write->WriteShort(umsgStringTableOffset);
bf_write->WriteOneBit(0); // important bit!

Either way, the first part of the umsgs (before any actual data) is still 3 bytes. And I honestly don't remember why it's like that with the header, I wonder if it's supposed to be bit + short.[/QUOTE]

Yeah, I just assumed its it had a short, but the extra byte is also odd: could it be a 0 | 1 byte to signify pooling?

[Lau]

Unfortunately, i get encoding logical errors when using server to client interface.
[lua]
if SERVER then include('server/servermessage.lua') end
local NAME = "stream"
module('stream',package.seeall)
local Hooks = {}
local Streams = {
IN = SERVER and {},
OUT = {}
}
--if SERVER then Streams.IN = {} end

local hex2bin = {
["0"] = "0000",
["1"] = "0001",
["2"] = "0010",
["3"] = "0011",
["4"] = "0100",
["5"] = "0101",
["6"] = "0110",
["7"] = "0111",
["8"] = "1000",
["9"] = "1001",
["a"] = "1010",
["b"] = "1011",
["c"] = "1100",
["d"] = "1101",
["e"] = "1110",
["f"] = "1111"
}

local function NearestPower(n, pMax)
for i=0, pMax or 256 do
if 2^i >= n then
return i
end
end
return 256
end

local function string_split(str,d) --Thanks to Deco and Janorkie's datastream
local t = {}
local len = str:len()
local i = 0
while i*d < len do
t[i+1] = str:sub(i*d+1,(i+1)*d)
i=i+1
end
return t
end

local function Hex2Bin(s)
local ret = ''
local i = 0
for i in string.gmatch(s, ".") do
i = string.lower(i)
ret = ret..hex2bin[i]
end
return ret
end

_G.Dec2Bin = function (s, num)
local n
if (num == nil) then
n = 0
elseif num > 2048 then
ErrorNoHalt("Dec2Bin #2 argument is insanely high\n")
return ""
else
n = num
end
s = Hex2Bin(string.format("%x", tostring(s)))
while string.len(s) < n do
s = '0'..s
end
return s
end

local function EncodeBinaryToString(tblBits)
local str = ''
local byte
for i=1, #tblBits, 8 do
byte = 0
for bit=0, 7 do
byte = byte + ((tblBits[i+bit] and 2^bit) or 0)
end
str = str..string.char(byte)
end
return str
end

local function DecodeStringToBinary(str)
local bin = {}
local t
for i=1, #str do
t = Dec2Bin(tostring(string.byte(string.sub(str,i,i))),8)
for bit=8, 1, -1 do --normally 8 to 1
table.insert(bin,string.sub(t,bit,bit)=='1')
end
end
return bin
end

local function AppendBinary(bitsBuffer,strBin)
for i=string.len(strBin), 1, -1 do
table.insert(bitsBuffer,string.sub(strBin,i,i)=='1')
end
end

local function ReadBits(tblBits,index,numBits)
local sBits = ""
for i=index+numBits-1, index, -1 do
if type(tblBits[i])~="boolean" then error('fucking non-boolean index in bitstream :tblBits['..tostring(i)..'] dim is '..tostring(#tblBits)) end
sBits=sBits..(tblBits[i] and "1" or "0")
end
return sBits
end

--TODO: Add table recursion support and key sending too!
local tblSpecificType = {
--[[['Short'] = function (bitStream,n)
AppendBinary(bitStream,Dec2Bin(math.Clamp(tonumber(n) or 0,-(2^15),2^15 - 1)+2^15,16))
end,
['UShort'] = function (bitStream,n)
AppendBinary(bitStream,Dec2Bin(math.Clamp(tonumber(n) or 0,0,2^15),16))
end,
['ULong'] = function (bitStream,n)
AppendBinary(bitStream,Dec2Bin(math.Clamp(tonumber(n) or 0,0,2^31),32))
end,]]
['number'] = {
encode=function (bitStream,n) --signed long integer
AppendBinary(bitStream,Dec2Bin(math.Clamp(tonumber(n) or 0,-(2^31),2^31 - 1)+2^31,32))
end,
decode=function (bitStream,index)
return tonumber(ReadBits(bitStream,index,32),2)-2^31, index+32
end},

['Char'] = {
encode=function (bitStream,ch)
AppendBinary(bitStream,Dec2Bin(tostring(string.byte(ch)),8))
end,
decode=function (bitStream,index)
return string.char(tonumber(ReadBits(bitStream,index,8),2)), index+8
end},

['string'] = {
encode=function (bitStream,str)
AppendBinary(bitStream,Dec2Bin(tostring(string.len(str)),8))
for i=1, string.len(str) do
AppendBinary(bitStream,Dec2Bin(tostring(string.byte(string.sub(str,i,i))),8))
end
end,
decode=function (bitStream,index)
local s=''
for i=1, tonumber(ReadBits(bitStream,index,8),2) do
index=index+8
s=s..string.char(tonumber(ReadBits(bitStream,index,8),2))
end
return s, index+8
end},

['boolean'] = {
encode=function (bitStream,b)
table.insert(bitStream,b)
end,
decode=function (bitStream,index)
return bitStream[index], index+1
end},

['Entity'] = {
encode=function (bitStream,ent)
AppendBinary(bitStream,Dec2Bin(tostring(ent:EntIndex()),16))
end,
decode=function (bitStream,index)
return ents.GetByIndex(tonumber(ReadBits(bitStream,index,16),2)), index+16
end},
['table'] = true,
['EOS']=true,
}
local tblSpecificTypeID = {}
local numMaxID = 0
for k,v in pairs(tblSpecificType) do
numMaxID=numMaxID+1
tblSpecificTypeID[k] = numMaxID
tblSpecificTypeID[numMaxID] = k
end
local tblTypeAlias = { --Unique assignments such as player object
['Player'] = 'Entity'
}
local TBL_DIM_BITSIZE = 32
local IDENT_BITS = NearestPower(table.Count(tblSpecificType)) --derpa
print("IDENTBITS = "..IDENT_BITS)

_G.TYPTEST = function (typ,data)
local tblBin = {}
AppendBinary(tblBin,Dec2Bin(tostring(tblSpecificTypeID[typ]),IDENT_BITS))
tblSpecificType[typ].encode(tblBin,data)
PrintTable(tblBin)
print("typ converted back to : "..tostring(tblSpecificTypeID[tonumber(ReadBits(tblBin,1,IDENT_BITS),2)]).." id="..tonumber(ReadBits(tblBin,1,IDENT_BITS),2))
local dat, index = tblSpecificType[typ].decode(tblBin,IDENT_BITS+1)
print('data converted back to : '..tostring(dat)..' new index:'..tostring(index))
end

--print("DERPA IS: maxid=",numMaxID,"tablecount=",table.Count(tblSpecificTypeID),"nearestpwr=",NearestPower(table.Count(tblSpecificTypeID)))

local function table_count_nonint(t, bAllowFloat)
local n = 0
for k, v in pairs(t) do
if type(k) ~= 'number' or (bAllowFloat and math.floor(k)~=k) then
n=n+1
end
end
return n
end
local function ParseToStringX(tblBin,typ,tblX)
local nonint = table_count_nonint(tblX,false)
--AppendBinary(tblBin,Dec2Bin(tostring(#tblX),TBL_DIM_BITSIZE)) --number of integer keys
for k, v in ipairs(tblX) do
typ = type(v)
if typ == 'table' then
AppendBinary(tblBin,Dec2Bin(tostring(tblSpecificTypeID[typ]),IDENT_BITS)) --identify it as a table and then do recursion
ParseToStringX(tblBin,typ,tblX[k])
--elseif typ == 'number' then --Work on float and precision checking (try dynamic identification maybe?
elseif typ~='nil' then
typ = tblTypeAlias[typ] or typ
if tblSpecificType[typ] then
AppendBinary(tblBin,Dec2Bin(tostring(tblSpecificTypeID[typ]),IDENT_BITS))
tblSpecificType[typ].encode(tblBin,v)
end
end
end
AppendBinary(tblBin,Dec2Bin(tostring(tblSpecificTypeID['EOS']),IDENT_BITS)) --a flag to end the stream (little endian?) instead of a huge 32 bit table dimension encoded
--TODO: add non-int key sending
--[[table.insert(tblBin,nonint>0) --bit to tell if non-integer keys exist
if nonint>0 then
AppendBinary(tblBin,Dec2Bin(table_count_nonint(v,true)),2^32)
end]]
--[[ --todo non-integer indexed sending
for k, v in pairs(tbl) do
if type(k) ~= 'number' then --is NaN

elseif math.floor(k)~=k then --# that has a decimal value

end
end
]]
end

local function ParseToString(tbl, bAllowCompress)

--DATA ENCODING PROCESS
local tblBin = {}
local typ
ParseToStringX(tblBin,typ,tbl) --allows for recursion (some heavy duty shit bro)

--STRING ENCODING then COMPRESSION PROCESS
local str = EncodeBinaryToString(tblBin)
local bCompressed = false
if bAllowCompress then
str, bCompressed = LibCompress.CompressLZW(str) --decode from table of bools to an encoded string then try to compress it
end
return str, bCompressed
end

local function ParseFromStringX(tbl,bitIndex,tblBin)
local typ
local tblIndex = 1
while true do
print("Reading a value...")
typ = tblSpecificTypeID[tonumber(ReadBits(tblBin,bitIndex,IDENT_BITS),2)]
if not typ then print("ERR With ID:"..tostring(tonumber(ReadBits(tblBin,bitIndex,IDENT_BITS),2))) end
bitIndex = bitIndex + IDENT_BITS
if typ then
if typ == 'EOS' then
break --end of table parsing
else
if typ == 'table' then
tbl[i] = {}
--[[if tonumber(ReadBits(tblBin,bitIndex,TBL_DIM_BITSIZE),2) > 512 then
error("shit is so high "..tostring(tonumber(ReadBits(tblBin,bitIndex,TBL_DIM_BITSIZE),2)))
end]]
bitIndex = ParseFromStringX(tbl[i],bitIndex,tblBin)
else --other object
local dat, newBitIndex = tblSpecificType[typ].decode(tblBin,bitIndex)
tbl[i] = dat
bitIndex = newBitIndex
end
tblIndex = tblIndex + 1