This is a read-only snapshot of the ComputerCraft forums, taken in April 2020.
hbar's profile picture

Help escaping characters and parsing JSON for a Gist client.

Started by hbar, 27 June 2013 - 11:29 AM
hbar #1
Posted 27 June 2013 - 01:29 PM
Someone on the Feed The Beast subreddit made a post asking for a Github gist client (similar to the pastebin program). I took a look at the Github API and decided to take a try at making something. Currently my code can download and create gists just fine… for some files.

Certain characters (or character combinations) are a problem when trying to upload a file, namely ' and \". I can't for the life of me figure out how to properly escape those. Escaping ' with \" works fine but \' fails.

Also: how the heck are you supposed to parse the JSON response when downloading the gist if the content section contains code with {, ", : and such.

I'm a total newbie when it comes to using HTTP requests, so I ask you if there's a way to, for example, urlEncode the JSON or some kind of a solution like that (of course I could always just urlEncode the file contents but then you could not edit the gist in any reasonable way)?

My current implementation:
Spoiler

local url = "https://api.github.com/gists"
--convert a table into json notation recursively
local encodeTable
encodeTable = function(t)
  local quot = "\""
  local s = "{"
  for k,v in pairs(t) do
   k = quot..k..quot
    if type(v) == "table" then
	 s = s..k..":"..encodeTable(v)..","
    elseif #tostring(v) > 0 then
	 if v == true or v == false then
	  v = tostring(v)
	 else
	  v = quot..v..quot
	 end
	 s = s..k..":"..v..","
    end
  end
  s = s:sub(1,s:len()-1)
  return s .."}"
end
--remove unnecessary \":s and convert boolea and numeric values to appropriate types
local sanitaze = function(s)
if s:sub(1,1) == "\"" then
  s = s:sub(2,s:len())..""
end
if s:sub(s:len(),s:len()) == "\"" then
  s = s:sub(1,s:len()-1)..""
end
if s == "true" then return true end
if s == "false" then return false end
if tonumber(s) ~= nil then return tonumber(s) end
return s
end
--escape characters
local contentEncode = function(s)
s = s:gsub("\\\"","\\\\\\\"")
s = s:gsub("\\\'","\\\\\\\'")
s = s:gsub("\"","\\\"")
s = s:gsub("\n","\\n")
s = s:gsub("\t","\\t")
s = s:gsub("\'","\\\'")
return s
end
--parse the json response to a table recursively
--will fail for some content
local parseHTTPresp
parseHTTPresp = function(resp)
local t = {}
local i = 1
local j = 1
local len = resp:len()
local var
--remove { and }
resp = resp:sub(2,resp:len()-1)..""
while i ~= nil do
  j = 1
  i = resp:find(":")
  if i == nil then break end
  var = sanitaze(resp:sub(j,i-1))
  if resp:sub(i+1,i+1) == "{" then
   -- find matching }
   local n = 1
   local k = i+2
   while n > 0 do
    if resp:sub(k,resp:len()):match("[{}]") == "{" then
	 n = n+1
	 k = resp:sub(k,resp:len()):find("{")+1
    else
	 n = n-1
	 k = resp:sub(k,resp:len()):find("}")+1
    end
   end
   t[var] = parseHTTPresp(resp:sub(i+1,k-1).."")
   i = k+1
   resp = resp:sub(i,len)..""
   len = resp:len()
  else
   j = resp:sub(i,len):find(",")
   if j == nil then j = len+1 end
   t[var] = sanitaze(resp:sub(i+1,j-1))
   i = j+1
   resp = resp:sub(i,len)..""
   len = resp:len()
  end
end
return t
end
-- default parameters for POST
local putvars = {}
putvars["description"] = ""
putvars["public"] = true
putvars["files"] = {}

-- get json notation from the POST variables
putvars.data = function(self)
local q = "\""
local s = "{"
s = s..q.."description"..q..":"..q..self["description"]..q..","
s = s..q.."public"..q..":"..tostring(self["public"])..","
s = s..q.."files"..q..":"..encodeTable(self["files"])
s = s.."}"
return s
end

local printUsage = function()
print("Usage:")
print("gist get <id>")
print("gist put <filename>")
end
local args = {...}
local mode
if not http then
print( "gist requires http API" )
print( "Set enableAPI_http to 1 in mod_ComputerCraft.cfg" )
return
end
if #args ~= 2 then
printUsage()
return
end
if args[1] == "get" then
mode = "get"
elseif args[1] == "put" then
mode = "put"
else
printUsage()
return
end
if mode == "get" then

local id = args[2]
local resp = http.get(url.."/"..id)
if resp ~= nil then
  --print("Success with code "..tostring(resp.getResponseCode()).."!")
  local sresp = resp.readAll()
  resp.close()
  local data = parseHTTPresp(sresp)
  --iterate over the files (there can be several in one gist)
  for key, value in pairs(data["files"]) do
   local file = value["filename"]
   local path = shell.resolve(file)
   local confirm = true
   if fs.exists(path) then
    term.write("File "..file.." already exists. Overwrite? [y/n] ")
    local inp = io.read():lower()
    if inp ~= "y" then
	 print("Skipping file: "..file)
	 confirm = false
    else
	 print("Overwriting file: "..file)
    end
   end
   if confirm then
    --download raw url so we do not need to parse content
    --I want to get rid of this
    local raw = http.get(value["raw_url"])
    if raw == nil then print("Unable to load contents of "..file.."!") return end
    local f = fs.open(path,"w")
    f.write(raw.readAll())
    f.close()
    raw.close()
    print("File "..file.." downloaded!")
   end
  end
else
  print("Failed to download gist with id "..id.."!")
  return
end
elseif mode == "put" then
local file = args[2]
local path = shell.resolve(file)
if not fs.exists(path) then
  print("No such file!")
  return
end
print("Give a description for the gist. (Can be blank)")
putvars["description"] = io.read()
local f = fs.open(path,"r")
local files = {}
files[file] = {}
files[file]["content"] = contentEncode(f.readAll())
f.close()
putvars["files"] = files
local resp = http.post(url,putvars:data())
if resp ~= nil then
  --print("Success with code "..tostring(resp.getResponseCode()).."!")
  local data = parseHTTPresp(resp.readAll())
  resp.close()
  print("Uploaded. Gist id: "..tostring(data["id"])..".")
  print("Available for viewing at https://gist.github.com/"..tostring(data["id"]))
else
  print("Failed to upload gist.")
end
end

I understand that this might not be the correct forum for a question like this, but I also think that a gist client could be very useful for the CC community and there might be interest in figuring this out.
Lyqyd #2
Posted 27 June 2013 - 01:57 PM
Split into new topic.
airtonix #3
Posted 01 July 2013 - 07:27 PM
@hbar this package should help. http://regex.info/blog/lua/json

also put your gist client up on github. :>
hbar #4
Posted 04 July 2013 - 06:17 AM
Thanks for the suggestion, airtonix, unfortunately I didn't see it until now. However, I already managed to solve my problems by using an external json parser. And it helped to learn that per json specifications the single quotes don't need to be escaped at all.

You can find the gist client at here and use pastebin get SrFAw1qj gist to download it with a computer/turtle.

Unfortunately, as CC HTTP API doesn't support headers I can't do authentication with github, so the client only supports downloading public and uploading anonymous gists. If rumors be true, the header support might be coming in following version. If and when, I'll add the authentication support too.

I suppose I should make a post here about the client for feedback, but I'm still waiting on enough privileges to post.