1 module lz4; 2 /** 3 * CTFEable LZ4 decompressor 4 * Copyright © 2016 Stefan Koch 5 * All rights reserved 6 */ 7 enum Endianess 8 { 9 BigEndian, 10 LittleEndian 11 } 12 /// JUST TO DEMONSTRATE THAT IT IS UNLIKELY :) 13 auto unlikely(T)(T expressionValue) 14 { 15 return expressionValue; 16 } 17 /// JUST TO DEMONSTRATE THAT IT IS LIKEY :) 18 auto likely(T)(T expressionValue) 19 { 20 return expressionValue; 21 } 22 23 void fastCopy(ubyte* dst, const (ubyte)* src, size_t length) pure nothrow @trusted 24 { 25 if (length == 0) 26 return; 27 static if (__VERSION__ <= 2068) 28 { 29 if (__ctfe) 30 { 31 foreach(i;0 .. length) 32 dst[i] = src[i]; 33 } 34 } else 35 36 dst[0 .. length] = src[0 .. length]; 37 38 return; 39 40 } 41 42 T fromBytes(T, Endianess endianess = Endianess.LittleEndian) (const ubyte[] _data) 43 pure { 44 static assert(is(T : long)); // poor man's isIntegral 45 T result; 46 static if (endianess == Endianess.LittleEndian) { 47 static if (T.sizeof == 4) { 48 result = ( 49 _data[0] | 50 (_data[1] << 8) | 51 (_data[2] << 16) | 52 (_data[3] << 24) 53 ); 54 } else static if (T.sizeof == 8) { 55 result = ( 56 _data[0] | 57 (_data[1] << 8) | 58 (_data[2] << 16) | 59 (_data[3] << 24) | 60 (cast(ulong)_data[4] << 32UL) | 61 (cast(ulong)_data[5] << 40UL) | 62 (cast(ulong)_data[6] << 48UL) | 63 (cast(ulong)_data[7] << 56UL) 64 ); 65 } else { 66 static assert(0, "only int and long are supported"); 67 } 68 } else { 69 static assert(0, "Big Endian currently not supported"); 70 } 71 72 return result; 73 74 } 75 76 struct LZ4Header 77 { 78 //TODO: finish this! ("parsing" LZ4 Frame format header) 79 uint end = 7; 80 ubyte flags; 81 82 bool hasBlockIndependence; 83 bool hasBlockChecksum; 84 bool hasContentChecksum; 85 86 ulong contentSize; 87 88 this(const ubyte[] data) pure 89 { 90 assert(((data[0] >> 6) & 0b11) == 0b01, "Format can not be read"); 91 92 hasBlockIndependence = ((data[0] >> 5) & 0b1); 93 hasBlockChecksum = ((data[0] >> 4) & 0b1); 94 95 bool hasContentSize = ((data[0] >> 3) & 0b1); 96 97 hasContentChecksum = ((data[0] >> 2) & 0b1); 98 99 if (hasContentSize) 100 { 101 contentSize = fromBytes!ulong(data[2 .. 2 + ulong.sizeof]); 102 assert(contentSize); 103 end = end + cast(uint) ulong.sizeof; 104 } 105 } 106 } 107 108 109 ubyte[] decodeLZ4File(const ubyte[] data, uint size) pure { 110 ubyte[] output; 111 output.length = size; 112 return decodeLZ4File(data, output.ptr, size); 113 } 114 ubyte[] decodeLZ4File(const ubyte[] data, ubyte* output, uint outLength) pure 115 in 116 { 117 assert(data.length > 11, "Any valid LZ4 File has to be longer then 11 bytes"); 118 } 119 body 120 { 121 ubyte[] result = output[0 .. outLength]; 122 assert(data[0 .. 4] == [0x04, 0x22, 0x4d, 0x18], "not a valid LZ4 file"); 123 auto lz4Header = LZ4Header(data[5 .. $]); 124 size_t decodedBytes = lz4Header.end; 125 uint offset; 126 while (true) 127 { 128 uint blockLength = fromBytes!uint(data[decodedBytes .. decodedBytes + uint.sizeof]); 129 if (blockLength == 0) 130 { 131 return result; 132 } 133 decodeLZ4Block(data[decodedBytes + uint.sizeof .. $], blockLength, result); 134 decodedBytes += blockLength + uint.sizeof; 135 } 136 assert(0); // "We can never get here" 137 } 138 139 //extern(C) ubyte* decodeLZ4Block(const(ubyte)* input, uint blockLength, ubyte* output, uint outLength) pure { 140 // return decodeLZ4Block(input[0 .. blockLength], blockLength, output[0 .. outLength]).ptr; 141 //} 142 143 144 ubyte[] decodeLZ4Block(const ubyte[] input, uint blockLength, ref ubyte[] output) pure 145 in 146 { 147 assert(input.length > 5, "empty or too short input passed to decodeLZ4Block"); 148 } 149 body 150 { 151 uint coffset; 152 uint dlen; 153 154 while (true) 155 { 156 immutable bitfield = input[coffset++]; 157 immutable highBits = (bitfield >> 4); 158 immutable lowBits = bitfield & 0xF; 159 160 uint literalsLength = highBits; 161 162 if (unlikely(highBits == 0xF)) 163 { 164 while (unlikely(input[coffset++] == 0xFF)) 165 { 166 literalsLength += 0xFF; 167 } 168 169 literalsLength += input[coffset - 1]; 170 } 171 172 fastCopy(output.ptr + dlen, input.ptr + coffset, literalsLength); 173 coffset += literalsLength; 174 dlen += literalsLength; 175 176 if (coffset >= blockLength) 177 return output[0 .. dlen]; 178 179 uint matchLength = lowBits + 4; 180 immutable ushort offset = (input[coffset++] | (input[coffset++] << 8)); 181 182 if (unlikely(lowBits == 0xF)) 183 { 184 while (input[coffset++] == 0xFF) 185 { 186 matchLength += 0xFF; 187 } 188 matchLength += input[coffset - 1]; 189 } 190 191 if (unlikely(offset < matchLength)) 192 { 193 uint done = matchLength; 194 195 while (likely(offset < done)) 196 { 197 //This is the point where er can speed up the copy significantly! 198 fastCopy(output.ptr + dlen, output.ptr + dlen - offset, offset); 199 200 dlen += offset; 201 done -= offset; 202 } 203 204 fastCopy(output.ptr + dlen, output.ptr + dlen - offset, done); 205 dlen += done; 206 } 207 else 208 { 209 fastCopy(output.ptr + dlen, output.ptr + dlen - offset, matchLength); 210 dlen += matchLength; 211 } 212 } 213 }