Re: Simple pickler

[Luis Carvalho <carvalho@...>]

> after fiddling with your code a little, it now works like a charm...
> fast and efficient. Here are two further small remarks:
> 1) In
> 	local TREF = 9
> 	local TGUARD = 10
> the numerical values could usefully be a little bigger to allow for
> further storable lua types (I've used 254 and 255, which is probably a
> tad optimistic).

This is fine; I've updated the code.

> 2) As your functions read and write a binary format (as compared to some
> other pickle()'s I've seen which produce Lua table initialisation code),
> how about writing a small signature (2 or 4 byte) with a version id at
> the start of the file? This way future versions can read "old" dumps in
> a backwards-compatible manner.

I've settled down with <esc>LPK<version in hex> as the signature, and a few
more functions for file parsing are now in the module. Now you can do:

require "pickler"

a = {x = 1, y = 2; {3, 4, 5}}
a[2] = a -- cycle
a.z = a[1] -- shared subtable
a.f = function(n) return n+1 end

-- cross refs
b = {k = a[1]}
a[3] = b

fp = pickler.open("ab.lpk", "w")
fp:write(a)
fp:write(b)
fp:close()

and then:

$ lua -lpickler
Lua 5.1.2  Copyright (C) 1994-2007 Lua.org, PUC-Rio
> f = pickler.open"ab.lpk"
> t = {}
> for o in f:objects() do t[#t + 1] = o end
> f:close()
> print(t[1], t[2])
table: 0x309700 table: 0x309db0
> table.foreach(t[1], print)
1       table: 0x3067c0
2       table: 0x309700
3       table: 0x309db0
y       2
x       1
z       table: 0x3067c0
f       function: 0x30a0c0
> table.foreach(t[2], print)
k       table: 0x3067c0

> Thank you very much for sharing that. For one thing, I would not have
> thought of storing functions as your code does.

I'm glad it's being useful. :)

Cheers,
Luis.

-- 
A mathematician is a device for turning coffee into theorems.
        -- P. Erdos 

-- 
Luis Carvalho
Applied Math PhD Student - Brown University
PGP Key: E820854A <carvalho@dam.brown.edu>

-- pickler.lua v0.1
-- Simple pickle library based on Roberto's struct library at
--    http://www.inf.puc-rio.br/~roberto/struct
-- All Lua types but thread and userdata are supported. Function pickling is
-- based on string.dump/loadstring, and so closures are not an (easy) option.
-- Table with references to other tables are ok (including cyclic references).
-- This code is in public domain.

require "struct"
local pack = struct.pack
local unpack = struct.unpack
local size = struct.size

local type = type
local dump = string.dump
local loadstring = loadstring
local setmetatable = setmetatable
local pairs = pairs
local concat = table.concat
local error = error
local assert = assert
local fopen = io.open
local tonumber = tonumber
local wrap = coroutine.wrap
local yield = coroutine.yield

local function taddress(t) -- table address: simple hash
  return tostring(t):sub(10)
end

module(...)

_VERSION = "0.1"
_BUFSIZE = 2^13 -- 8 kB
local SIGNATURE = pack("Bc3B", -- <esc>LPK<version>
    0x1b, "LPK", tonumber(_VERSION * 10, 16))

-- from lua.h
local TNIL = 0
local TBOOLEAN = 1
local TNUMBER = 3
local TSTRING = 4
local TTABLE = 5
local TFUNCTION = 6
-- extra
local TREF = 253
local TGUARD = 254
local TBLOCK = 255

-- keep references to tables
local ref = setmetatable({}, {__mode = "v"})


-- main routines: pickle and unpickle

pickle = function(o)
  local t = type(o)
  if t == "nil" then
    return pack("B", TNIL)
  elseif t == "boolean" then
    return pack("BB", TBOOLEAN, o and 1 or 0)
  elseif t == "number" then
    return pack("Bd", TNUMBER, o)
  elseif t == "string" then
    return pack("BLc0", TSTRING, #o, o)
  elseif t == "table" then
    local a = taddress(o)
    if ref[a] == nil then -- not interned?
      ref[a] = o
      local p = {} -- hold packs
      p[1] = pack("B", TTABLE) -- type
      p[2] = pack("Bc0", #a, a) -- ref key
      for k, v in pairs(o) do
        p[#p + 1] = pickle(k)
        p[#p + 1] = pickle(v)
      end
      p[#p + 1] = pack("B", TGUARD) -- table end
      return concat(p)
    end
    return pack("BBc0", TREF, #a, a)
  elseif t == "function" then
    local f = dump(o)
    return pack("BLc0", TFUNCTION, #f, f)
  else
    error("type not supported: " .. t)
  end
end

unpickle = function(o, start)
  local p = start or 1
  local t, p = unpack("B", o, p)
  if t == TNIL then
    return nil, 2
  elseif t == TBOOLEAN then
    return unpack("B", o, p) == 1, 3
  elseif t == TNUMBER then
    return unpack("d", o, p)
  elseif t == TSTRING then
    return unpack("Lc0", o, p)
  elseif t == TTABLE then
    local a, p = unpack("Bc0", o, p)
    -- intern table
    local r = {}
    ref[taddress(r)] = r
    ref[a] = r
    local k, v
    while (unpack("B", o, p)) ~= TGUARD do
      k, p = unpickle(o, p)
      v, p = unpickle(o, p)
      r[k] = v
    end
    p = p + size"B" -- after TGUARD
    return r, p
  elseif t == TFUNCTION then
    local u, p = unpack("Lc0", o, p)
    return loadstring(u), p
  else -- TREF
    local a, p = unpack("Bc0", o, p)
    assert(ref[a] ~= nil, "table not interned")
    return ref[a], p
  end
end


-- pickler file object

local write = function(fp, o)
  local f = fp.handle
  local p = pickle(o)
  f:write(pack("BL", TBLOCK, #p))
  f:write(p)
end

local objects = function(fp)
  local f = fp.handle
  return wrap(function()
    local s, p = "", 1
    while true do
      local fs = f:read(_BUFSIZE)
      if fs == nil then break end -- EOF?
      s = s .. fs
      while p <= #s do -- any object left?
        -- read block size
        if size"BL" > #s then break end -- buffer underflow?
        local c, l, o = unpack("BL", s, p)
        assert(c == TBLOCK, "file format error")
        -- read block
        if o + l - 1 > #s then break end -- buffer underflow?
        o, p = unpickle(s, o)
        yield(o)
        s, p = s:sub(p), 1 -- reset buffer and position
      end
    end
  end)
end

local close = function(fp)
  fp.handle:close()
end

open = function(fname, opt)
  local opt = opt or "r"
  assert(opt == "r" or opt == "w", "unknown option: `" .. opt .. "'")
  local f = assert(fopen(fname, opt .. "b"),
      "cannot open file: `" .. fname .. "'")
  if opt == "r" then -- read
    local s = f:read(#SIGNATURE)
    assert(s == SIGNATURE, "wrong file format")
  else -- write
    f:write(SIGNATURE)
  end
  return setmetatable({handle=f},
      {__index = {write=write, objects=objects, close=close}})
end