Skip to content

Commit

Permalink
Added support for .xz files, via the XZ Utils liblzma.
Browse files Browse the repository at this point in the history 
Eventually I'll be building 7-Zip support on top of liblzma too (the actual
 7-Zip LZMA SDK isn't really usable for our purposes, but the heavy lifting
 it would supply is also done in liblzma, I think).
  • Loading branch information
@icculus
icculus committed Nov 15, 2010
1 parent 25d5978 commit bd3d93c
Show file tree
Hide file tree
Showing 83 changed files with 14,987 additions and 11 deletions.
87 changes: 87 additions & 0 deletions CMakeLists.txt
View file
Expand Up @@ -210,6 +210,44 @@ SET(BZLIB_SRCS
${BZLIB_DIR}/bzlib.c
)

SET(LIBLZMA_DIR liblzma)
SET(LIBLZMA_SRCS
${LIBLZMA_DIR}/check/check.c
${LIBLZMA_DIR}/common/block_buffer_decoder.c
${LIBLZMA_DIR}/common/block_decoder.c
${LIBLZMA_DIR}/common/block_header_decoder.c
${LIBLZMA_DIR}/common/block_util.c
${LIBLZMA_DIR}/common/common.c
${LIBLZMA_DIR}/common/easy_decoder_memusage.c
${LIBLZMA_DIR}/common/easy_preset.c
${LIBLZMA_DIR}/common/filter_buffer_decoder.c
${LIBLZMA_DIR}/common/filter_common.c
${LIBLZMA_DIR}/common/filter_decoder.c
${LIBLZMA_DIR}/common/filter_flags_decoder.c
${LIBLZMA_DIR}/common/index.c
${LIBLZMA_DIR}/common/index_decoder.c
${LIBLZMA_DIR}/common/index_hash.c
${LIBLZMA_DIR}/common/stream_decoder.c
${LIBLZMA_DIR}/common/stream_flags_common.c
${LIBLZMA_DIR}/common/stream_flags_decoder.c
${LIBLZMA_DIR}/common/vli_decoder.c
${LIBLZMA_DIR}/common/vli_size.c
${LIBLZMA_DIR}/delta/delta_common.c
${LIBLZMA_DIR}/delta/delta_decoder.c
${LIBLZMA_DIR}/lz/lz_decoder.c
${LIBLZMA_DIR}/lzma/lzma2_decoder.c
${LIBLZMA_DIR}/lzma/lzma_decoder.c
${LIBLZMA_DIR}/lzma/lzma_encoder_presets.c
${LIBLZMA_DIR}/simple/arm.c
${LIBLZMA_DIR}/simple/armthumb.c
${LIBLZMA_DIR}/simple/ia64.c
${LIBLZMA_DIR}/simple/powerpc.c
${LIBLZMA_DIR}/simple/simple_coder.c
${LIBLZMA_DIR}/simple/simple_decoder.c
${LIBLZMA_DIR}/simple/sparc.c
${LIBLZMA_DIR}/simple/x86.c
)

SET(LIBFETCH_DIR libfetch)
SET(LIBFETCH_SRCS
${LIBFETCH_DIR}/fetch.c
Expand Down Expand Up @@ -515,6 +553,8 @@ ENDIF(MOJOSETUP_ARCHIVE_ZIP)
# BINARY SIZE += 2
OPTION(MOJOSETUP_ARCHIVE_TAR "Enable TAR support" TRUE)
IF(MOJOSETUP_ARCHIVE_TAR)
# !!! FIXME: the gzip/bzip2/xz support doesn't require .tar archives now.
# !!! FIXME: Maybe not ask here, so the questions look the same everywhere.
ADD_DEFINITIONS(-DSUPPORT_TAR=1)
OPTION(MOJOSETUP_ARCHIVE_TAR_GZ "Enable TAR.GZ support" TRUE)
IF(MOJOSETUP_ARCHIVE_TAR_GZ)
Expand All @@ -525,6 +565,11 @@ IF(MOJOSETUP_ARCHIVE_TAR)
IF(MOJOSETUP_ARCHIVE_TAR_BZ2)
SET(MOJOSETUP_INPUT_BZIP2 TRUE)
ENDIF(MOJOSETUP_ARCHIVE_TAR_BZ2)

OPTION(MOJOSETUP_ARCHIVE_TAR_XZ "Enable TAR.XZ support" TRUE)
IF(MOJOSETUP_ARCHIVE_TAR_XZ)
SET(MOJOSETUP_INPUT_XZ TRUE)
ENDIF(MOJOSETUP_ARCHIVE_TAR_XZ)
ENDIF(MOJOSETUP_ARCHIVE_TAR)

OPTION(MOJOSETUP_ARCHIVE_UZ2 "Enable UZ2 support" FALSE)
Expand Down Expand Up @@ -562,6 +607,15 @@ IF(MOJOSETUP_INPUT_BZIP2)
SET(MOJOSETUP_NEED_BZLIB TRUE)
ENDIF(MOJOSETUP_INPUT_BZIP2)

# BINARY SIZE += ???
IF(NOT MOJOSETUP_INPUT_XZ) # optional if something didn't force it.
OPTION(MOJOSETUP_INPUT_XZ "Enable XZ support" FALSE)
ENDIF(NOT MOJOSETUP_INPUT_XZ)

IF(MOJOSETUP_INPUT_XZ)
ADD_DEFINITIONS(-DSUPPORT_XZ=1)
SET(MOJOSETUP_NEED_LIBLZMA TRUE)
ENDIF(MOJOSETUP_INPUT_XZ)


# Image decoders for GUIs...
Expand Down Expand Up @@ -697,6 +751,39 @@ IF(MOJOSETUP_NEED_BZLIB)
ENDIF(MOJOSETUP_INTERNAL_BZLIB)
ENDIF(MOJOSETUP_NEED_BZLIB)

IF(MOJOSETUP_NEED_LIBLZMA)
SET(HAVE_SYSTEM_LIBLZMA FALSE)
CHECK_INCLUDE_FILE(lzma.h HAVE_LZMA_H)
IF(HAVE_LZMA_H)
CHECK_LIBRARY_EXISTS("lzma" "lzma_stream_decoder" "" HAVE_LIBLZMA)
IF(HAVE_LIBLZMA)
SET(HAVE_SYSTEM_LIBLZMA TRUE)
ENDIF(HAVE_LIBLZMA)
ENDIF(HAVE_LZMA_H)

IF(HAVE_SYSTEM_LIBLZMA)
OPTION(MOJOSETUP_INTERNAL_LIBLZMA "Link own liblzma instead of system library" FALSE)
ELSE(HAVE_SYSTEM_LIBLZMA)
SET(MOJOSETUP_INTERNAL_LIBLZMA TRUE)
ENDIF(HAVE_SYSTEM_LIBLZMA)

# BINARY SIZE += ???
IF(MOJOSETUP_INTERNAL_LIBLZMA)
ADD_DEFINITIONS(-DMOJOSETUP_INTERNAL_LIBLZMA=1)
INCLUDE_DIRECTORIES(${LIBLZMA_DIR}/api)
INCLUDE_DIRECTORIES(${LIBLZMA_DIR}/common)
INCLUDE_DIRECTORIES(${LIBLZMA_DIR}/check)
INCLUDE_DIRECTORIES(${LIBLZMA_DIR}/delta)
INCLUDE_DIRECTORIES(${LIBLZMA_DIR}/lz)
INCLUDE_DIRECTORIES(${LIBLZMA_DIR}/lzma)
INCLUDE_DIRECTORIES(${LIBLZMA_DIR}/rangecoder)
INCLUDE_DIRECTORIES(${LIBLZMA_DIR}/simple)
SET(OPTIONAL_SRCS ${OPTIONAL_SRCS} ${LIBLZMA_SRCS})
ELSE(MOJOSETUP_INTERNAL_LIBLZMA)
SET(OPTIONAL_LIBS ${OPTIONAL_LIBS} lzma)
ENDIF(MOJOSETUP_INTERNAL_LIBLZMA)
ENDIF(MOJOSETUP_NEED_LIBLZMA)

IF(UNIX)
CHECK_INCLUDE_FILE(sys/ucred.h HAVE_UCRED_H)
IF(HAVE_UCRED_H)
Expand Down
10 changes: 5 additions & 5 deletions LICENSE.txt
View file
Expand Up @@ -24,9 +24,9 @@


(Please note that other pieces of code in MojoSetup fall under BSD-like and/or
public domain licenses: lua, libfetch, zlib, bzlib, stb_image, etc. All
software statically linked to MojoSetup was explicitly chosen to be friendly
with closed-source software, in case the installer needs a proprietary
change. I am not a lawyer and this is not legal advice. Please have a lawyer
consider the licenses if you have any concerns.)
public domain licenses: lua, libfetch, zlib, bzlib, liblzma, stb_image, etc.
All software statically linked to MojoSetup was explicitly chosen to be
friendly with closed-source software, in case the installer needs a
proprietary change. I am not a lawyer and this is not legal advice. Please
have a lawyer consider the licenses if you have any concerns.)

226 changes: 220 additions & 6 deletions fileio.c
View file
Expand Up @@ -30,11 +30,13 @@ static const MojoArchiveType archives[] =
{ "zip", MojoArchive_createZIP, true },
{ "tar", MojoArchive_createTAR, true },
{ "tar.gz", MojoArchive_createTAR, true },
{ "tar.xz", MojoArchive_createTAR, true },
{ "tar.bz2", MojoArchive_createTAR, true },
{ "tgz", MojoArchive_createTAR, true },
{ "tbz2", MojoArchive_createTAR, true },
{ "tb2", MojoArchive_createTAR, true },
{ "tbz", MojoArchive_createTAR, true },
{ "txz", MojoArchive_createTAR, true },
{ "uz2", MojoArchive_createUZ2, false },
{ "pck", MojoArchive_createPCK, true },
};
Expand Down Expand Up @@ -429,24 +431,236 @@ static MojoInput *make_bzip2_input(MojoInput *origio)
#endif // SUPPORT_BZIP2


#if SUPPORT_XZ

#include "lzma.h"

#define XZ_READBUFSIZE (128 * 1024)

static MojoInput *make_xz_input(MojoInput *origio);

typedef struct XZinfo
{
MojoInput *origio;
uint64 uncompressed_position;
uint8 buffer[XZ_READBUFSIZE];
lzma_stream stream;
} XZinfo;

static void *mojoLzmaAlloc(void *opaque, size_t items, size_t size)
{
return xmalloc(items * size);
} // mojoLzmaAlloc

static void mojoLzmaFree(void *opaque, void *address)
{
free(address);
} // mojoZlibFree

static void initializeXZStream(lzma_stream *pstr)
{
static lzma_allocator lzmaAlloc = { mojoLzmaAlloc, mojoLzmaFree };
memset(pstr, '\0', sizeof (lzma_stream));
pstr->allocator = &lzmaAlloc;
} // initializeXZStream

static boolean MojoInput_xz_ready(MojoInput *io)
{
return true; // !!! FIXME: ready if there are bytes uncompressed.
} // MojoInput_xz_ready

static boolean MojoInput_xz_seek(MojoInput *io, uint64 offset)
{
// This is all really expensive.
XZinfo *info = (XZinfo *) io->opaque;

/*
* If seeking backwards, we need to redecode the file
* from the start and throw away the compressed bits until we hit
* the offset we need. If seeking forward, we still need to
* decode, but we don't rewind first.
*/
if (offset < info->uncompressed_position)
{
lzma_stream *strm = &info->stream;
if (!info->origio->seek(info->origio, 0))
return false;
lzma_end(strm);
initializeXZStream(strm);
if (lzma_stream_decoder(strm, UINT64_MAX, LZMA_CONCATENATED) != LZMA_OK)
return false;
info->uncompressed_position = 0;
} // if

while (info->uncompressed_position != offset)
{
uint8 buf[512];
uint32 maxread;
int64 br;

maxread = (uint32) (offset - info->uncompressed_position);
if (maxread > sizeof (buf))
maxread = sizeof (buf);

br = io->read(io, buf, maxread);
if (br != maxread)
return false;
} /* while */

return true;
} // MojoInput_xz_seek

static int64 MojoInput_xz_tell(MojoInput *io)
{
return (((XZinfo *) io->opaque)->uncompressed_position);
} // MojoInput_xz_tell

static int64 MojoInput_xz_length(MojoInput *io)
{
return -1;
} // MojoInput_xz_length

static int64 MojoInput_xz_read(MojoInput *io, void *buf, uint32 bufsize)
{
XZinfo *info = (XZinfo *) io->opaque;
MojoInput *origio = info->origio;
int64 retval = 0;

if (bufsize == 0)
return 0; // quick rejection.

info->stream.next_out = buf;
info->stream.avail_out = bufsize;

while (retval < ((int64) bufsize))
{
const uint32 before = info->stream.total_out;
lzma_ret rc;

lzma_action action = LZMA_FINISH;
if (info->stream.avail_in == 0)
{
int64 br = origio->length(origio) - origio->tell(origio);
if (br > 0)
{
action = LZMA_RUN;
if (br > XZ_READBUFSIZE)
br = XZ_READBUFSIZE;

br = origio->read(origio, info->buffer, (uint32) br);
if (br <= 0)
return -1;

info->stream.next_in = info->buffer;
info->stream.avail_in = (uint32) br;
} // if
} // if

rc = lzma_code(&info->stream, LZMA_RUN);
retval += (info->stream.total_out - before);

if (rc != LZMA_OK)
return -1;
} // while

assert(retval >= 0);
info->uncompressed_position += (uint32) retval;

return retval;
} // MojoInput_xz_read

static MojoInput* MojoInput_xz_duplicate(MojoInput *io)
{
XZinfo *info = (XZinfo *) io->opaque;
MojoInput *retval = NULL;
MojoInput *newio = info->origio->duplicate(info->origio);
if (newio != NULL)
{
retval = make_xz_input(newio);
if (retval != NULL)
retval->seek(retval, io->tell(io)); // slow, slow, slow...
} // if
return retval;
} // MojoInput_xz_duplicate

static void MojoInput_xz_close(MojoInput *io)
{
XZinfo *info = (XZinfo *) io->opaque;
if (info->origio != NULL)
info->origio->close(info->origio);
lzma_end(&info->stream);
free(info);
free(io);
} // MojoInput_xz_close

static MojoInput *make_xz_input(MojoInput *origio)
{
MojoInput *io = NULL;
XZinfo *info = (XZinfo *) xmalloc(sizeof (XZinfo));
lzma_stream *strm = &info->stream;

// UINT64_MAX is the memory usage limit (so basically, no artificial
// limit here). Internally, this holds its entire dictionary in
// RAM (8 megabytes by default), plus a few other structures, so we
// shouldn't eat tons of memory in the normal case. Note that the
// dictionary can max out at _four gigabytes_, but obviously no one does
// that.
initializeXZStream(strm);
if (lzma_stream_decoder(strm, UINT64_MAX, LZMA_CONCATENATED) != LZMA_OK)
{
free(info);
return NULL;
} // if

info->origio = origio;

io = (MojoInput *) xmalloc(sizeof (MojoInput));
io->ready = MojoInput_xz_ready;
io->read = MojoInput_xz_read;
io->seek = MojoInput_xz_seek;
io->tell = MojoInput_xz_tell;
io->length = MojoInput_xz_length;
io->duplicate = MojoInput_xz_duplicate;
io->close = MojoInput_xz_close;
io->opaque = info;
return io;
} // make_xz_input

#endif // SUPPORT_XZ


MojoInput *MojoInput_newCompressedStream(MojoInput *origio)
{
#if SUPPORT_GZIP || SUPPORT_BZIP2
#if SUPPORT_GZIP || SUPPORT_BZIP2 || SUPPORT_XZ
// Look at the first piece of the file to decide if it is compressed
// by a general compression algorithm, and if so, wrap the MojoInput
// in a decompressor.
uint8 magic[4];
uint8 magic[6];
const int64 br = origio->read(origio, magic, sizeof (magic));
if ((origio->seek(origio, 0)) && (br == sizeof (magic)))
{
#if SUPPORT_GZIP
if ((magic[0] == 0x1F) && (magic[1] == 0x8B) && (magic[2] == 0x08))
return make_gzip_input(origio);
{
static const uint8 gzip_sig[] = { 0x1F, 0x8B, 0x08 };
if (memcmp(magic, gzip_sig, sizeof (gzip_sig)) == 0)
return make_gzip_input(origio);
}
#endif

#if SUPPORT_BZIP2
if ((magic[0] == 0x42) && (magic[1] == 0x5A))
return make_bzip2_input(origio);
{
static const uint8 bzip2_sig[] = { 0x42, 0x5A };
if (memcmp(magic, bzip2_sig, sizeof (bzip2_sig)) == 0)
return make_bzip2_input(origio);
}
#endif

#if SUPPORT_XZ
{
static const uint8 xz_sig[] = { 0xFD, '7', 'z', 'X', 'Z', 0x00 };
if (memcmp(magic, xz_sig, sizeof (xz_sig)) == 0)
return make_xz_input(origio);
}
#endif
} // if
#endif
Expand Down

0 comments on commit bd3d93c

Please sign in to comment.