1 module lz4;
2 /**
3  * CTFEable LZ4 decompressor
4  * Copyright © 2016 Stefan Koch
5  * All rights reserved
6  */
7 enum Endianess
8 {
9     BigEndian,
10     LittleEndian
11 }
12 
13 T fromBytes(T, Endianess endianess = Endianess.LittleEndian) (const ubyte[] _data)
14 pure {
15     static assert(is(T : long)); // poor man's isIntegral
16     T result;
17     static if (endianess == Endianess.LittleEndian) {
18         static if (T.sizeof == 4) {
19             result = (
20                 _data[0] |
21                 (_data[1] << 8) |
22                 (_data[2] << 16) |
23                 (_data[3] << 24)
24             );
25         } else static if (T.sizeof == 8) {
26             result = (
27                 _data[0] |
28                 (_data[1] << 8) |
29                 (_data[2] << 16) |
30                 (_data[3] << 24) |
31                 (cast(ulong)_data[4] << 32UL) |
32                 (cast(ulong)_data[5] << 40UL) |
33                 (cast(ulong)_data[6] << 48UL) |
34                 (cast(ulong)_data[7] << 56UL)
35             );
36         } else {
37             static assert(0, "only int and long are supported");
38         }
39     } else {
40         static assert(0, "Big Endian currently not supported");
41     }
42 
43     return result;
44 
45 }
46 
47 struct LZ4Header
48 {
49     //TODO: finish this! ("parsing" LZ4 Frame format header)
50     uint end = 7;
51     uint flags;
52 
53     bool hasBlockIndependence;
54     bool hasBlockChecksum;
55     bool hasContentChecksum;
56 
57     ulong contentSize;
58 
59     this(const ubyte[] data) pure
60     {
61         immutable data0 = data[0];
62 
63         assert(((data0 >> 6) & 0b11) == 0b01, "Format can not be read");
64 
65         hasBlockIndependence = ((data0 >> 5) & 0b1);
66         hasBlockChecksum = ((data0 >> 4) & 0b1);
67 
68         bool hasContentSize = ((data0 >> 3) & 0b1);
69 
70         hasContentChecksum = ((data0 >> 2) & 0b1);
71 
72         if (hasContentSize)
73         {
74             contentSize = fromBytes!ulong(data[2 .. 2 + ulong.sizeof]);
75             assert(contentSize > 0);
76             end = end + cast(uint) ulong.sizeof;
77         }
78     }
79 }
80 
81 
82 ubyte[] decodeLZ4File(const ubyte[] data, uint size) pure {
83     ubyte[] output;
84     output.length = size;
85     return decodeLZ4File(data, output, size);
86 }
87 ubyte[] decodeLZ4File(const ubyte[] data, ubyte[] output, uint outLength) pure
88 in
89 {
90     assert(data.length > 11, "Any valid LZ4 File has to be longer then 11 bytes");
91 }
92 body
93 {
94     ubyte[] result = output[0 .. outLength];
95     bool validFile = data[0 .. 4] == [0x04, 0x22, 0x4d, 0x18];
96     assert(validFile, "not a valid LZ4 file");
97     auto lz4Header = LZ4Header(data[5 .. $]);
98     uint decodedBytes = lz4Header.end;
99     uint offset;
100     while (true)
101     {
102         uint blockLength = fromBytes!uint(data[decodedBytes .. decodedBytes + cast(uint)uint.sizeof]);
103         if (blockLength == 0)
104         {
105             return result;
106         }
107         decodeLZ4Block(data[decodedBytes + uint.sizeof .. $], blockLength, result);
108         decodedBytes += blockLength + uint.sizeof;
109     }
110     assert(0); // "We can never get here"
111 }
112 
113 //extern(C) ubyte* decodeLZ4Block(const(ubyte)* input, uint blockLength, ubyte* output, uint outLength) pure {
114 //    return decodeLZ4Block(input[0 .. blockLength], blockLength, output[0 .. outLength]).ptr;
115 //}
116 
117 
118 ubyte[] decodeLZ4Block(const ubyte[] input, uint blockLength, ubyte[] output) pure
119 in
120 {
121     assert(input.length > 5, "empty or too short input passed to decodeLZ4Block");
122 }
123 body
124 {
125     uint coffset;
126     uint dlen;
127 
128     while (true)
129     {
130         immutable bitfield = input[coffset++];
131         immutable highBits = (bitfield >> 4);
132         immutable lowBits = bitfield & 0xF;
133 
134         uint literalsLength = highBits;
135 
136         if (/*unlikely*/(highBits == 0xF))
137         {
138             while (/*unlikely*/(input[coffset++] == 0xFF))
139             {
140                 literalsLength += 0xFF;
141             }
142 
143             literalsLength += input[coffset - 1];
144         }
145         uint until_d = dlen + literalsLength;
146         uint until_c = coffset + literalsLength;
147         output[dlen .. until_d] = input[coffset .. until_c];
148         coffset = until_c;
149         dlen = until_d;
150 
151         if (coffset >= blockLength)
152             return output[0 .. dlen];
153 
154         uint matchLength = lowBits + 4;
155         immutable ushort offset = (input[coffset++] | (input[coffset++] << 8));
156 
157         if (/*unlikely*/(lowBits == 0xF))
158         {
159             while (input[coffset++] == 0xFF)
160             {
161                 matchLength += 0xFF;
162             }
163             matchLength += input[coffset - 1];
164         }
165 
166         if (/*unlikely*/(offset < matchLength))
167         {
168             uint done = matchLength;
169 
170             while (/*likely*/(offset < done))
171             {
172                 //This is the point where er can speed up the copy significantly!
173                 const d_until = dlen + offset;
174                 const c_begin = dlen - offset;
175                 output[dlen .. d_until] = output[c_begin .. dlen];
176 
177                 dlen = d_until;
178                 done -= offset;
179             }
180             const until = dlen + done;
181             const c_begin = dlen - offset;
182             const c_end = until - offset;
183 
184             output[dlen .. until] = output[c_begin .. c_end];
185             dlen = until;
186         }
187         else
188         {
189             const until = dlen + matchLength;
190             const c_begin = dlen - offset;
191             const c_end = until - offset;
192 
193             output[dlen .. until] = output[c_begin .. c_end];
194             dlen = until;
195         }
196     }
197 }