Re: lzlib usage

[Ignacio Burgueño <ignaciob@...>]

On 28/01/2010 18:21, Phoenix Sol wrote:

> Right, I had no problem trying out compress() and uncompress() (although
> the options are still kinda mysterious).
>
> What I'm trying to understand is how to process in a stream, from a
> source to a sink without necessarily holding the entire message body in
> a buffer; the userdata returned by compressobj() and decompressobj()
> seem to be what I'm after but so far I don't quite grok them.
>
> Thanks for your help.

Here is an example using deflate. It does the compression using streams, 
but one needs to have the entire resulting data in memory because you 
need to send the actual (compressed) content-length before sending the 
data. (this, of course, if you're dealing with http)

Regards,
Ignacio

-----------------------------------------------------------------------------
-- Deflate filter for WSAPI
--
-- Author: Ignacio Burgueño
-- Copyright (c) 2009 inConcert
--
-----------------------------------------------------------------------------

local zlib = require "zlib"
local common = require"wsapi.common"
local coroutine = coroutine
local string = string
local table = table
local type = type
local tonumber = tonumber
local assert = assert

module ((...))

-- Content-Length must be greater than this value to apply the compression
local MINIMUM_CONTENT_SIZE = 512

local m_allowed_types = {
	["text/css"] = true,
	["text/plain"] = true,
	["text/html"] = true,
	["application/xhtml+xml"] = true,
	["application/x-javascript"] = true,
	["application/xml"] = true,
	["image/svg+xml"] = true,
	["text/sgml"] = true,
	["application/json"] = true,
	["text/javascript"] = true,
}

--
-- Allows a given mime type to be encoded
function AllowType(mimeType)
	assert(type(mimeType) == "string")
	m_allowed_types[mimeType] = true
end

--
-- Disallows a given mime type to be encoded
function DisallowType(mimeType)
	assert(type(mimeType) == "string")
	m_allowed_types[mimeType] = nil
end

--
-- Sets the minimum content size a response needs to be compressed
function SetMinimumSize(newSize)
	assert(type(newSize) == "number")
	MINIMUM_CONTENT_SIZE = newSize
end

-- helper function: tests for the presence of a header, removing it if found
local function test_and_clear(headers, header)
	local h = headers[header]
	if h then
		headers[header] = nil
	end
	return h
end

--
-- Returns a functions to be used as a WSAPI filter. It will compress the output given the following (in order):
--  The compression was not forcibly disabled
--  The browser supports 'deflate' compression
--  The output size is greater that MINIMUM_CONTENT_SIZE bytes or the compression is not forced
--  The Content-Type is one of the allowed types or the compression is not forced
--   Note: If the browser does not support compression the 'force' header is ignored.
function makeDeflater(appToFilter)
	return function(wsapi_env)
		local status, headers, iterator
		if type(appToFilter) == "table" and type(appToFilter["run"]) == "function" then
			status, headers, iterator = appToFilter.run(wsapi_env)
		else
			status, headers, iterator = appToFilter(wsapi_env)
		end
		
		local disableCompression = test_and_clear(headers, "x-inConcert-Compression-Disable")
		if disableCompression or not string.find(wsapi_env["HTTP_ACCEPT_ENCODING"], "deflate") then
			return status, headers, iterator
		end
		
		local contentLength = headers["Content-Length"]

		local forceCompress = test_and_clear(headers, "x-inConcert-Compression-Force")
		if not forceCompress then
			local contentType = headers["Content-Type"]
			if not m_allowed_types[contentType] or tonumber(contentLength) <= MINIMUM_CONTENT_SIZE then
				-- the content type is marked as not compressable
				return status, headers, iterator
			end
		end
		
		headers["x-inConcert-Old-Content-Length"] = contentLength
		
		local buffer = {}
		local acumulator = function(data)
			table.insert(buffer, data)
		end
		local stream = {
			_stream = zlib.deflate(acumulator),
			write = function(self, data)
				self._stream:write(data)
				return true
			end,
			close = function(self)
				return self._stream:close()
			end
		}
		common.send_content(stream, iterator, "write")
		stream:close()
		
		local deflatedData = table.concat(buffer)
		deflatedData = string.sub(deflatedData, 3)
		headers["Content-Encoding"] = "deflate"
		headers["Content-Length"] = #deflatedData
		headers["Vary"] = "Accept-Encoding"
		
		return status, headers, coroutine.wrap(function() 
				coroutine.yield(deflatedData)
			end)
	end
end