david/ipxe
david
/
ipxe
Archived
1
0
Fork 0
Code

[deflate] Add support for DEFLATE decompression 

Signed-off-by: Michael Brown <mcb30@ipxe.org>
This commit is contained in:
Michael Brown 2014-01-01 21:17:26 +01:00
parent 23f17f7972
commit 9bdfc36bcc
5 changed files with 1569 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1045
src/crypto/deflate.c Normal file
View File

File diff suppressed because it is too large Load Diff

283
src/include/ipxe/deflate.h Normal file
View File

@ -0,0 +1,283 @@
#ifndef _IPXE_DEFLATE_H
#define _IPXE_DEFLATE_H
/** @file
*
* DEFLATE decompression algorithm
*
*/
FILE_LICENCE ( GPL2_OR_LATER );
#include <stdint.h>
#include <string.h>
#include <ipxe/uaccess.h>
/** Compression formats */
enum deflate_format {
/** Raw DEFLATE data (no header or footer) */
DEFLATE_RAW,
/** ZLIB header and footer */
DEFLATE_ZLIB,
};
/** Block header length (in bits) */
#define DEFLATE_HEADER_BITS 3
/** Block header final block flags bit */
#define DEFLATE_HEADER_BFINAL_BIT 0
/** Block header type LSB */
#define DEFLATE_HEADER_BTYPE_LSB 1
/** Block header type mask */
#define DEFLATE_HEADER_BTYPE_MASK 0x03
/** Block header type: literal data */
#define DEFLATE_HEADER_BTYPE_LITERAL 0
/** Block header type: static Huffman alphabet */
#define DEFLATE_HEADER_BTYPE_STATIC 1
/** Block header type: dynamic Huffman alphabet */
#define DEFLATE_HEADER_BTYPE_DYNAMIC 2
/** Literal header LEN/NLEN field length (in bits) */
#define DEFLATE_LITERAL_LEN_BITS 16
/** Dynamic header length (in bits) */
#define DEFLATE_DYNAMIC_BITS 14
/** Dynamic header HLIT field LSB */
#define DEFLATE_DYNAMIC_HLIT_LSB 0
/** Dynamic header HLIT field mask */
#define DEFLATE_DYNAMIC_HLIT_MASK 0x1f
/** Dynamic header HDIST field LSB */
#define DEFLATE_DYNAMIC_HDIST_LSB 5
/** Dynamic header HDIST field mask */
#define DEFLATE_DYNAMIC_HDIST_MASK 0x1f
/** Dynamic header HCLEN field LSB */
#define DEFLATE_DYNAMIC_HCLEN_LSB 10
/** Dynamic header HCLEN field mask */
#define DEFLATE_DYNAMIC_HCLEN_MASK 0x0f
/** Dynamic header code length length (in bits) */
#define DEFLATE_CODELEN_BITS 3
/** Maximum length of a Huffman symbol (in bits) */
#define DEFLATE_HUFFMAN_BITS 15
/** Quick lookup length for a Huffman symbol (in bits)
*
* This is a policy decision.
*/
#define DEFLATE_HUFFMAN_QL_BITS 7
/** Quick lookup shift */
#define DEFLATE_HUFFMAN_QL_SHIFT ( 16 - DEFLATE_HUFFMAN_QL_BITS )
/** Literal/length end of block code */
#define DEFLATE_LITLEN_END 256
/** Maximum value of a literal/length code */
#define DEFLATE_LITLEN_MAX_CODE 287
/** Maximum value of a distance code */
#define DEFLATE_DISTANCE_MAX_CODE 31
/** Maximum value of a code length code */
#define DEFLATE_CODELEN_MAX_CODE 18
/** ZLIB header length (in bits) */
#define ZLIB_HEADER_BITS 16
/** ZLIB header compression method LSB */
#define ZLIB_HEADER_CM_LSB 0
/** ZLIB header compression method mask */
#define ZLIB_HEADER_CM_MASK 0x0f
/** ZLIB header compression method: DEFLATE */
#define ZLIB_HEADER_CM_DEFLATE 8
/** ZLIB header preset dictionary flag bit */
#define ZLIB_HEADER_FDICT_BIT 13
/** ZLIB ADLER32 length (in bits) */
#define ZLIB_ADLER32_BITS 32
/** A Huffman-coded set of symbols of a given length */
struct deflate_huf_symbols {
/** Length of Huffman-coded symbols */
uint8_t bits;
/** Shift to normalise symbols of this length to 16 bits */
uint8_t shift;
/** Number of Huffman-coded symbols having this length */
uint16_t freq;
/** First symbol of this length (normalised to 16 bits)
*
* Stored as a 32-bit value to allow the value 0x10000 to be
* used for empty sets of symbols longer than the maximum
* utilised length.
*/
uint32_t start;
/** Raw symbols having this length */
uint16_t *raw;
};
/** A Huffman-coded alphabet */
struct deflate_alphabet {
/** Huffman-coded symbol set for each length */
struct deflate_huf_symbols huf[DEFLATE_HUFFMAN_BITS];
/** Quick lookup table */
uint8_t lookup[ 1 << DEFLATE_HUFFMAN_QL_BITS ];
/** Raw symbols
*
* Ordered by Huffman-coded symbol length, then by symbol
* value. This field has a variable length.
*/
uint16_t raw[0];
};
/** A static Huffman alphabet length pattern */
struct deflate_static_length_pattern {
/** Length pair */
uint8_t fill;
/** Repetition count */
uint8_t count;
} __attribute__ (( packed ));
/** Decompressor */
struct deflate {
/** Resume point
*
* Used as the target of a computed goto to jump to the
* appropriate point within the state machine.
*/
void *resume;
/** Format */
enum deflate_format format;
/** Accumulator */
uint32_t accumulator;
/** Bit-reversed accumulator
*
* Don't ask.
*/
uint32_t rotalumucca;
/** Number of bits within the accumulator */
unsigned int bits;
/** Current block header */
unsigned int header;
/** Remaining length of data (e.g. within a literal block) */
size_t remaining;
/** Current length index within a set of code lengths */
unsigned int length_index;
/** Target length index within a set of code lengths */
unsigned int length_target;
/** Current length within a set of code lengths */
unsigned int length;
/** Number of extra bits required */
unsigned int extra_bits;
/** Length of a duplicated string */
size_t dup_len;
/** Distance of a duplicated string */
size_t dup_distance;
/** Literal/length Huffman alphabet */
struct deflate_alphabet litlen;
/** Literal/length raw symbols
*
* Must immediately follow the literal/length Huffman alphabet.
*/
uint16_t litlen_raw[ DEFLATE_LITLEN_MAX_CODE + 1 ];
/** Number of symbols in the literal/length Huffman alphabet */
unsigned int litlen_count;
/** Distance and code length Huffman alphabet
*
* The code length Huffman alphabet has a maximum Huffman
* symbol length of 7 and a maximum code value of 18, and is
* thus strictly smaller than the distance Huffman alphabet.
* Since we never need both alphabets simultaneously, we can
* reuse the storage space for the distance alphabet to
* temporarily hold the code length alphabet.
*/
struct deflate_alphabet distance_codelen;
/** Distance and code length raw symbols
*
* Must immediately follow the distance and code length
* Huffman alphabet.
*/
uint16_t distance_codelen_raw[ DEFLATE_DISTANCE_MAX_CODE + 1 ];
/** Number of symbols in the distance Huffman alphabet */
unsigned int distance_count;
/** Huffman code lengths
*
* The literal/length and distance code lengths are
* constructed as a single set of lengths.
*
* The code length Huffman alphabet has a maximum code value
* of 18 and the set of lengths is thus strictly smaller than
* the combined literal/length and distance set of lengths.
* Since we never need both alphabets simultaneously, we can
* reuse the storage space for the literal/length and distance
* code lengths to temporarily hold the code length code
* lengths.
*/
uint8_t lengths[ ( ( DEFLATE_LITLEN_MAX_CODE + 1 ) +
( DEFLATE_DISTANCE_MAX_CODE + 1 ) +
1 /* round up */ ) / 2 ];
};
/** A chunk of data */
struct deflate_chunk {
/** Data */
userptr_t data;
/** Current offset */
size_t offset;
/** Length of data */
size_t len;
};
/**
* Initialise chunk of data
*
* @v chunk Chunk of data to initialise
* @v data Data
* @v offset Starting offset
* @v len Length
*/
static inline __attribute__ (( always_inline )) void
deflate_chunk_init ( struct deflate_chunk *chunk, userptr_t data,
size_t offset, size_t len ) {
chunk->data = data;
chunk->offset = offset;
chunk->len = len;
}
/**
* Check if decompression has finished
*
* @v deflate Decompressor
* @ret finished Decompression has finished
*/
static inline int deflate_finished ( struct deflate *deflate ) {
return ( deflate->resume == NULL );
}
extern void deflate_init ( struct deflate *deflate,
enum deflate_format format );
extern int deflate_inflate ( struct deflate *deflate,
struct deflate_chunk *in,
struct deflate_chunk *out );
#endif /* _IPXE_DEFLATE_H */

1
src/include/ipxe/errfile.h
View File

@ -297,6 +297,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
#define ERRFILE_efi_reboot ( ERRFILE_OTHER | 0x003e0000 )
#define ERRFILE_memmap_settings ( ERRFILE_OTHER | 0x003f0000 )
#define ERRFILE_param_cmd ( ERRFILE_OTHER | 0x00400000 )
#define ERRFILE_deflate ( ERRFILE_OTHER | 0x00410000 )
/** @} */

239
src/tests/deflate_test.c Normal file
View File

@ -0,0 +1,239 @@
/*
* Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of the
* License, or any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
FILE_LICENCE ( GPL2_OR_LATER );
/** @file
*
* DEFLATE tests
*
*/
/* Forcibly enable assertions */
#undef NDEBUG
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <ipxe/deflate.h>
#include <ipxe/test.h>
/** A DEFLATE test */
struct deflate_test {
/** Compression format */
enum deflate_format format;
/** Compressed data */
const void *compressed;
/** Length of compressed data */
size_t compressed_len;
/** Expected uncompressed data */
const void *expected;
/** Length of expected uncompressed data */
size_t expected_len;
};
/** A DEFLATE fragment list */
struct deflate_test_fragments {
/** Fragment lengths */
size_t len[8];
};
/** Define inline data */
#define DATA(...) { __VA_ARGS__ }
/** Define a DEFLATE test */
#define DEFLATE( name, FORMAT, COMPRESSED, EXPECTED ) \
static const uint8_t name ## _compressed[] = COMPRESSED; \
static const uint8_t name ## _expected[] = EXPECTED; \
static struct deflate_test name = { \
.format = FORMAT, \
.compressed = name ## _compressed, \
.compressed_len = sizeof ( name ## _compressed ), \
.expected = name ## _expected, \
.expected_len = sizeof ( name ## _expected ), \
};
/* Empty file, no compression */
DEFLATE ( empty_literal, DEFLATE_RAW,
DATA ( 0x01, 0x00, 0x00, 0xff, 0xff ), DATA() );
/* "iPXE" string, no compression */
DEFLATE ( literal, DEFLATE_RAW,
DATA ( 0x01, 0x04, 0x00, 0xfb, 0xff, 0x69, 0x50, 0x58, 0x45 ),
DATA ( 0x69, 0x50, 0x58, 0x45 ) );
/* Empty file */
DEFLATE ( empty, DEFLATE_RAW, DATA ( 0x03, 0x00 ), DATA() );
/* "Hello world" */
DEFLATE ( hello_world, DEFLATE_RAW,
DATA ( 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca,
0x49, 0x01, 0x00 ),
DATA ( 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c,
0x64 ) );
/* "Hello hello world" */
DEFLATE ( hello_hello_world, DEFLATE_RAW,
DATA ( 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0xc8, 0x00, 0x93, 0xe5,
0xf9, 0x45, 0x39, 0x29, 0x00 ),
DATA ( 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x68, 0x65, 0x6c, 0x6c,
0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64 ) );
/* "This specification defines a lossless compressed data format" */
DEFLATE ( rfc_sentence, DEFLATE_RAW,
DATA ( 0x0d, 0xc6, 0xdb, 0x09, 0x00, 0x21, 0x0c, 0x04, 0xc0, 0x56,
0xb6, 0x28, 0x1b, 0x08, 0x79, 0x70, 0x01, 0x35, 0xe2, 0xa6,
0x7f, 0xce, 0xf9, 0x9a, 0xf1, 0x25, 0xc1, 0xe3, 0x9a, 0x91,
0x2a, 0x9d, 0xb5, 0x61, 0x1e, 0xb9, 0x9d, 0x10, 0xcc, 0x22,
0xa7, 0x93, 0xd0, 0x5a, 0xe7, 0xbe, 0xb8, 0xc1, 0xa4, 0x05,
0x51, 0x77, 0x49, 0xff ),
DATA ( 0x54, 0x68, 0x69, 0x73, 0x20, 0x73, 0x70, 0x65, 0x63, 0x69,
0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x64,
0x65, 0x66, 0x69, 0x6e, 0x65, 0x73, 0x20, 0x61, 0x20, 0x6c,
0x6f, 0x73, 0x73, 0x6c, 0x65, 0x73, 0x73, 0x20, 0x63, 0x6f,
0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x20, 0x64,
0x61, 0x74, 0x61, 0x20, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74 ) );
/* "ZLIB Compressed Data Format Specification" */
DEFLATE ( zlib, DEFLATE_ZLIB,
DATA ( 0x78, 0x01, 0x8b, 0xf2, 0xf1, 0x74, 0x52, 0x70, 0xce, 0xcf,
0x2d, 0x28, 0x4a, 0x2d, 0x2e, 0x4e, 0x4d, 0x51, 0x70, 0x49,
0x2c, 0x49, 0x54, 0x70, 0xcb, 0x2f, 0xca, 0x4d, 0x2c, 0x51,
0x08, 0x2e, 0x48, 0x4d, 0xce, 0x4c, 0xcb, 0x4c, 0x4e, 0x2c,
0xc9, 0xcc, 0xcf, 0x03, 0x00, 0x2c, 0x0e, 0x0e, 0xeb ),
DATA ( 0x5a, 0x4c, 0x49, 0x42, 0x20, 0x43, 0x6f, 0x6d, 0x70, 0x72,
0x65, 0x73, 0x73, 0x65, 0x64, 0x20, 0x44, 0x61, 0x74, 0x61,
0x20, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x53, 0x70,
0x65, 0x63, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f,
0x6e ) );
/* "ZLIB Compressed Data Format Specification" fragment list */
static struct deflate_test_fragments zlib_fragments[] = {
{ { -1UL, } },
{ { 0, 1, 5, -1UL, } },
{ { 0, 0, 1, 0, 0, 1, -1UL } },
{ { 10, 8, 4, 7, 11, -1UL } },
{ { 45, -1UL } },
{ { 48, -1UL } },
};
/**
* Report DEFLATE test result
*
* @v deflate Decompressor
* @v test Deflate test
* @v frags Fragment list, or NULL
* @v file Test code file
* @v line Test code line
*/
static void deflate_okx ( struct deflate *deflate,
struct deflate_test *test,
struct deflate_test_fragments *frags,
const char *file, unsigned int line ) {
uint8_t data[ test->expected_len ];
struct deflate_chunk in;
struct deflate_chunk out;
size_t frag_len = -1UL;
size_t offset = 0;
size_t remaining = test->compressed_len;
unsigned int i;
/* Initialise decompressor */
deflate_init ( deflate, test->format );
/* Initialise output chunk */
deflate_chunk_init ( &out, virt_to_user ( data ), 0, sizeof ( data ) );
/* Process input (in fragments, if applicable) */
for ( i = 0 ; i < ( sizeof ( frags->len ) /
sizeof ( frags->len[0] ) ) ; i++ ) {
/* Initialise input chunk */
if ( frags )
frag_len = frags->len[i];
if ( frag_len > remaining )
frag_len = remaining;
deflate_chunk_init ( &in, virt_to_user ( test->compressed ),
offset, ( offset + frag_len ) );
/* Decompress this fragment */
okx ( deflate_inflate ( deflate, &in, &out ) == 0, file, line );
okx ( in.len == ( offset + frag_len ), file, line );
okx ( in.offset == in.len, file, line );
/* Move to next fragment */
offset = in.offset;
remaining -= frag_len;
if ( ! remaining )
break;
/* Check that decompression has not terminated early */
okx ( ! deflate_finished ( deflate ), file, line );
}
/* Check decompression has terminated as expected */
okx ( deflate_finished ( deflate ), file, line );
okx ( offset == test->compressed_len, file, line );
okx ( out.offset == test->expected_len, file, line );
okx ( memcmp ( data, test->expected, test->expected_len ) == 0,
file, line );
}
#define deflate_ok( deflate, test, frags ) \
deflate_okx ( deflate, test, frags, __FILE__, __LINE__ )
/**
* Perform DEFLATE self-test
*
*/
static void deflate_test_exec ( void ) {
struct deflate *deflate;
unsigned int i;
/* Allocate shared structure */
deflate = malloc ( sizeof ( *deflate ) );
ok ( deflate != NULL );
/* Perform self-tests */
if ( deflate ) {
/* Test as a single pass */
deflate_ok ( deflate, &empty_literal, NULL );
deflate_ok ( deflate, &literal, NULL );
deflate_ok ( deflate, &empty, NULL );
deflate_ok ( deflate, &hello_world, NULL );
deflate_ok ( deflate, &hello_hello_world, NULL );
deflate_ok ( deflate, &rfc_sentence, NULL );
deflate_ok ( deflate, &zlib, NULL );
/* Test fragmentation */
for ( i = 0 ; i < ( sizeof ( zlib_fragments ) /
sizeof ( zlib_fragments[0] ) ) ; i++ ) {
deflate_ok ( deflate, &zlib, &zlib_fragments[i] );
}
}
/* Free shared structure */
free ( deflate );
}
/** DEFLATE self-test */
struct self_test deflate_test __self_test = {
.name = "deflate",
.exec = deflate_test_exec,
};

1
src/tests/tests.c
View File

@ -50,3 +50,4 @@ REQUIRE_OBJECT ( x509_test );
REQUIRE_OBJECT ( ocsp_test );
REQUIRE_OBJECT ( cms_test );
REQUIRE_OBJECT ( pnm_test );
REQUIRE_OBJECT ( deflate_test );