1 module lz4; 2 /** 3 * CTFEable LZ4 decompressor 4 * Copyright © 2016 Stefan Koch 5 * All rights reserved 6 */ 7 enum Endianess 8 { 9 BigEndian, 10 LittleEndian 11 } 12 13 T fromBytes(T, Endianess endianess = Endianess.LittleEndian) (const ubyte[] _data) 14 pure { 15 static assert(is(T : long)); // poor man's isIntegral 16 T result; 17 static if (endianess == Endianess.LittleEndian) { 18 static if (T.sizeof == 4) { 19 result = ( 20 _data[0] | 21 (_data[1] << 8) | 22 (_data[2] << 16) | 23 (_data[3] << 24) 24 ); 25 } else static if (T.sizeof == 8) { 26 result = ( 27 _data[0] | 28 (_data[1] << 8) | 29 (_data[2] << 16) | 30 (_data[3] << 24) | 31 (cast(ulong)_data[4] << 32UL) | 32 (cast(ulong)_data[5] << 40UL) | 33 (cast(ulong)_data[6] << 48UL) | 34 (cast(ulong)_data[7] << 56UL) 35 ); 36 } else { 37 static assert(0, "only int and long are supported"); 38 } 39 } else { 40 static assert(0, "Big Endian currently not supported"); 41 } 42 43 return result; 44 45 } 46 47 struct LZ4Header 48 { 49 //TODO: finish this! ("parsing" LZ4 Frame format header) 50 uint end = 7; 51 uint flags; 52 53 bool hasBlockIndependence; 54 bool hasBlockChecksum; 55 bool hasContentChecksum; 56 57 ulong contentSize; 58 59 this(const ubyte[] data) pure 60 { 61 immutable data0 = data[0]; 62 63 assert(((data0 >> 6) & 0b11) == 0b01, "Format can not be read"); 64 65 hasBlockIndependence = ((data0 >> 5) & 0b1); 66 hasBlockChecksum = ((data0 >> 4) & 0b1); 67 68 bool hasContentSize = ((data0 >> 3) & 0b1); 69 70 hasContentChecksum = ((data0 >> 2) & 0b1); 71 72 if (hasContentSize) 73 { 74 contentSize = fromBytes!ulong(data[2 .. 2 + ulong.sizeof]); 75 assert(contentSize > 0); 76 end = end + cast(uint) ulong.sizeof; 77 } 78 } 79 } 80 81 82 ubyte[] decodeLZ4File(const ubyte[] data, uint size) pure { 83 ubyte[] output; 84 output.length = size; 85 return decodeLZ4File(data, output, size); 86 } 87 ubyte[] decodeLZ4File(const ubyte[] data, ubyte[] output, uint outLength) pure 88 in 89 { 90 assert(data.length > 11, "Any valid LZ4 File has to be longer then 11 bytes"); 91 } 92 body 93 { 94 ubyte[] result = output[0 .. outLength]; 95 bool validFile = data[0 .. 4] == [0x04, 0x22, 0x4d, 0x18]; 96 assert(validFile, "not a valid LZ4 file"); 97 auto lz4Header = LZ4Header(data[5 .. $]); 98 uint decodedBytes = lz4Header.end; 99 uint offset; 100 while (true) 101 { 102 uint blockLength = fromBytes!uint(data[decodedBytes .. decodedBytes + cast(uint)uint.sizeof]); 103 if (blockLength == 0) 104 { 105 return result; 106 } 107 decodeLZ4Block(data[decodedBytes + uint.sizeof .. $], blockLength, result); 108 decodedBytes += blockLength + uint.sizeof; 109 } 110 assert(0); // "We can never get here" 111 } 112 113 //extern(C) ubyte* decodeLZ4Block(const(ubyte)* input, uint blockLength, ubyte* output, uint outLength) pure { 114 // return decodeLZ4Block(input[0 .. blockLength], blockLength, output[0 .. outLength]).ptr; 115 //} 116 117 118 ubyte[] decodeLZ4Block(const ubyte[] input, uint blockLength, ubyte[] output) pure 119 in 120 { 121 assert(input.length > 5, "empty or too short input passed to decodeLZ4Block"); 122 } 123 body 124 { 125 uint coffset; 126 uint dlen; 127 128 while (true) 129 { 130 immutable bitfield = input[coffset++]; 131 immutable highBits = (bitfield >> 4); 132 immutable lowBits = bitfield & 0xF; 133 134 uint literalsLength = highBits; 135 136 if (/*unlikely*/(highBits == 0xF)) 137 { 138 while (/*unlikely*/(input[coffset++] == 0xFF)) 139 { 140 literalsLength += 0xFF; 141 } 142 143 literalsLength += input[coffset - 1]; 144 } 145 uint until_d = dlen + literalsLength; 146 uint until_c = coffset + literalsLength; 147 output[dlen .. until_d] = input[coffset .. until_c]; 148 coffset = until_c; 149 dlen = until_d; 150 151 if (coffset >= blockLength) 152 return output[0 .. dlen]; 153 154 uint matchLength = lowBits + 4; 155 immutable ushort offset = (input[coffset++] | (input[coffset++] << 8)); 156 157 if (/*unlikely*/(lowBits == 0xF)) 158 { 159 while (input[coffset++] == 0xFF) 160 { 161 matchLength += 0xFF; 162 } 163 matchLength += input[coffset - 1]; 164 } 165 166 if (/*unlikely*/(offset < matchLength)) 167 { 168 uint done = matchLength; 169 170 while (/*likely*/(offset < done)) 171 { 172 //This is the point where er can speed up the copy significantly! 173 const d_until = dlen + offset; 174 const c_begin = dlen - offset; 175 output[dlen .. d_until] = output[c_begin .. dlen]; 176 177 dlen = d_until; 178 done -= offset; 179 } 180 const until = dlen + done; 181 const c_begin = dlen - offset; 182 const c_end = until - offset; 183 184 output[dlen .. until] = output[c_begin .. c_end]; 185 dlen = until; 186 } 187 else 188 { 189 const until = dlen + matchLength; 190 const c_begin = dlen - offset; 191 const c_end = until - offset; 192 193 output[dlen .. until] = output[c_begin .. c_end]; 194 dlen = until; 195 } 196 } 197 }