1 ///
2 module redisd.codec;
3 
4 import std..string;
5 import std.algorithm;
6 import std.stdio;
7 import std.conv;
8 import std.exception;
9 import std.format;
10 import std.typecons;
11 import std.traits;
12 import std.array;
13 import std.range;
14 
15 import std.experimental.logger;
16 
17 import nbuff;
18 
19 /// Type of redis value
20 enum ValueType : ubyte {
21     Incomplete = 0,
22     Null,
23     Integer = ':',
24     String = '+',
25     BulkString = '$',
26     List = '*',
27     Error = '-',
28 }
29 ///
30 class BadDataFormat : Exception {
31     this(string msg, string f = __FILE__, ulong l = __LINE__) @safe {
32         super(msg, f, l);
33     }
34 }
35 ///
36 class EncodeException : Exception {
37     this(string msg, string f = __FILE__, ulong l = __LINE__) @safe {
38         super(msg, f, l);
39     }
40 }
41 ///
42 class WrongOffset : Exception {
43     this(string msg, string f = __FILE__, ulong l = __LINE__) @safe {
44         super(msg, f, l);
45     }
46 }
47 ///
48 class WrongDataAccess : Exception {
49     this(string msg, string f = __FILE__, ulong l = __LINE__) @safe {
50         super(msg, f, l);
51     }
52 }
53 
54 struct RedisdValue {
55     private {
56         ValueType       _type;
57         string          _svar;
58         long            _ivar;
59         RedisdValue[]    _list;
60     }
61     /// get type of value
62     ValueType type() @safe const {
63         return _type;
64     }
65     /// if this is nil value
66     bool empty() const {
67         return _type == ValueType.Null;
68     }
69     /// get string value
70     string svar() @safe const {
71         switch (_type) {
72         case ValueType.String, ValueType.BulkString, ValueType.Error:
73             return _svar;
74         default:
75             throw new WrongDataAccess("You can't use svar for non-string value");
76         }
77     }
78     /// get integer value
79     auto ivar() @safe const {
80         switch (_type) {
81         case ValueType.Integer:
82             return _ivar;
83         default:
84             throw new WrongDataAccess("You can't use ivar for non-integer value");
85         }
86     }
87 
88     void opAssign(long v) @safe {
89         _type = ValueType.Integer;
90         _ivar = v;
91         _svar = string.init;
92         _list = RedisdValue[].init;
93     }
94 
95     void opAssign(string v) @safe {
96         _type = ValueType.String;
97         _svar = v;
98         _ivar = long.init;
99         _list = RedisdValue[].init;
100     }
101 
102     string toString() {
103         switch(_type) {
104         case ValueType.Incomplete:
105             return "<Incomplete>";
106         case ValueType.String:
107             return "\"%s\"".format(_svar);
108         case ValueType.Error:
109             return "<%s>".format(_svar);
110         case ValueType.Integer:
111             return "%d".format(_ivar);
112         case ValueType.BulkString:
113             return "'%s'".format(_svar);
114         case ValueType.List:
115             return "[%(%s,%)]".format(_list);
116         case ValueType.Null:
117             return "(nil)";
118         default:
119             return "unknown type " ~ to!string(_type);
120         }
121     }
122 }
123 
124 /// Build redis value from string or integer
125 RedisdValue redisdValue(T)(T v)
126 if (isSomeString!T || isIntegral!T) {
127     RedisdValue _v;
128     static if (isIntegral!T) {
129         _v._type = ValueType.Integer;
130         _v._ivar = v;
131     } else
132     static if (isSomeString!T) {
133         _v._type = ValueType.BulkString;
134         _v._svar = v;
135     } else {
136         static assert(0, T.stringof);
137     }
138     return _v;
139 }
140 
141 /// Build redis value from tuple
142 RedisdValue redisdValue(T)(T v) if (isTuple!T) {
143     RedisdValue _v;
144     _v._type = ValueType.List;
145     RedisdValue[] l;
146     foreach (element; v) {
147         l ~= redisdValue(element);
148     }
149     _v._list = l;
150     return _v;
151 }
152 
153 /// Build redis value from array
154 RedisdValue redisdValue(T:U[], U)(T v)
155 if (isArray!T && !isSomeString!T) {
156     RedisdValue _v;
157     _v._type = ValueType.List;
158     RedisdValue[] l;
159     foreach (element; v) {
160         l ~= redisdValue(element);
161     }
162     _v._list = l;
163     return _v;
164 }
165 /// serialize RedisdValue to byte array
166 immutable(ubyte)[] encode(RedisdValue v) @safe {
167     string encoded;
168     switch(v._type) {
169     case ValueType.Error:
170         encoded.reserve(1 + v._svar.length + 2);
171         encoded ~= "-";
172         encoded ~= v._svar;
173         encoded ~= "\r\n";
174         return encoded.representation;
175     case ValueType.String:
176         encoded.reserve(1+v._svar.length+2);
177         encoded ~= "+";
178         encoded ~= v._svar;
179         encoded ~= "\r\n";
180         return encoded.representation;
181     case ValueType.Integer:
182         encoded.reserve(15);
183         encoded ~= ":";
184         encoded ~= to!string(v._ivar);
185         encoded ~= "\r\n";
186         return encoded.representation;
187     case ValueType.BulkString:
188         encoded.reserve(v._svar.length+1+20+2+2);
189         encoded ~= "$";
190         encoded ~= to!string(v._svar.length);
191         encoded ~= "\r\n";
192         encoded ~= to!string(v._svar);
193         encoded ~= "\r\n";
194         return encoded.representation;
195     case ValueType.List:
196         Appender!(immutable(ubyte)[]) appender;
197         appender.put(("*" ~ to!string(v._list.length) ~ "\r\n").representation);
198         foreach(element; v._list) {
199             appender.put(element.encode);
200         }
201         return cast(immutable(ubyte)[])appender.data;
202     default:
203         throw new EncodeException("Failed to encode");
204     }
205 }
206 
207 ///
208 @safe unittest {
209     RedisdValue v;
210     v = "abc";
211     assert(encode(v) == "+abc\r\n".representation);
212     v = -1234567890;
213     assert(encode(v) == ":-1234567890\r\n".representation);
214     v = -0;
215     assert(encode(v) == ":0\r\n".representation);
216 
217     v = -1234567890;
218     assert(v.encode.decode.value == v);
219     v = redisdValue("abc");
220     assert(v.encode.decode.value == v);
221     v = redisdValue([1,2]);
222     assert(v.encode.decode.value == v);
223     v = redisdValue(tuple("abc", 1));
224     assert(v.encode.decode.value == v);
225 }
226 
227 alias DecodeResult = Tuple!(RedisdValue, "value", immutable(ubyte)[], "rest");
228 
229 /// deserialize from byte array
230 DecodeResult decode(immutable(ubyte)[] data) @safe {
231     assert(data.length >= 1);
232     RedisdValue v;
233     switch(data[0]) {
234     case '+':
235         // simple string
236         v._type = ValueType.String;
237         auto s = data[1..$].findSplit([13,10]);
238         v._svar = cast(string)s[0];
239         return DecodeResult(v, s[2]);
240     case '-':
241         // error
242         v._type = ValueType.Error;
243         auto s = data[1 .. $].findSplit([13, 10]);
244         v._svar = cast(string) s[0];
245         return DecodeResult(v, s[2]);
246     case ':':
247         // integer
248         v._type = ValueType.Integer;
249         auto s = data[1 .. $].findSplit([13, 10]);
250         v._ivar = to!long(cast(string)s[0]);
251         return DecodeResult(v, s[2]);
252     case '$':
253         // bulk string
254         // skip $, then try to split on first \r\n:
255         // s now: [length],[\r\n],[data\r\n]
256         v._type = ValueType.BulkString;
257         auto s = data[1..$].findSplit([13, 10]);
258         if (s[1].length != 2) {
259             throw new BadDataFormat("bad data: [%(%02.2#x,%)]".format(data));
260         }
261         auto len = to!long(cast(string)s[0]);
262         if ( s[2].length < len + 2 ) {
263             throw new BadDataFormat("bad data: [%(%02.2#x,%)]".format(data));
264         }
265         v._svar = cast(string)s[2][0..len];
266         return DecodeResult(v, s[2][len+2..$]);
267     case '*':
268         // list
269         v._type = ValueType.List;
270         auto s = data[1 .. $].findSplit([13, 10]);
271         if (s[1].length != 2) {
272             throw new BadDataFormat("bad data: [%(%02.2#x,%)]".format(data));
273         }
274         auto len = to!long(cast(string) s[0]);
275         if ( len == 0 ) {
276             return DecodeResult(v, s[2]);
277         }
278         if ( len == -1 ){
279             v._type = ValueType.Null;
280             return DecodeResult(v, s[2]);
281         }
282         auto rest = s[2];
283         RedisdValue[] array;
284 
285         while(len>0) {
286             auto d = decode(rest);
287             auto v0 = d.value;
288             array ~= v0;
289             rest = d.rest;
290             len--;
291         }
292         v._list = array;
293         return DecodeResult(v, rest);
294     default:
295         break;
296     }
297     assert(0);
298 }
299 
300 @safe unittest {
301     RedisdValue v;
302     v = "a";
303     v = 1;
304 }
305 ///
306 @safe unittest {
307     DecodeResult d;
308     d = decode("+OK\r\n ".representation);
309     auto v = d.value;
310     auto r = d.rest;
311 
312     assert(v._type == ValueType.String);
313     assert(v._svar == "OK");
314     assert(r == " ".representation);
315 
316     d = decode("-ERROR\r\ngarbage\r\n".representation);
317     v = d.value;
318     r = d.rest;
319     assert(v._type == ValueType.Error);
320     assert(v._svar == "ERROR");
321     assert(r == "garbage\r\n".representation);
322 
323     d = decode(":100\r\n".representation);
324     v = d.value;
325     r = d.rest;
326     assert(v._type == ValueType.Integer);
327     assert(v._ivar == 100);
328     assert(r == "".representation);
329 
330     d = decode("$8\r\nfoobar\r\n\r\n:41\r\n".representation);
331     v = d.value;
332     r = d.rest;
333     assert(v._svar == "foobar\r\n", format("<%s>", v._svar));
334     assert(r ==  ":41\r\n".representation);
335     assert(v._type == ValueType.BulkString);
336 
337     d = decode("*3\r\n:1\r\n:2\r\n$6\r\nfoobar\r\nxyz".representation);
338     v = d.value;
339     r = d.rest;
340     assert(v._type == ValueType.List);
341     assert(r == "xyz".representation);
342 }
343 
344 /// decode redis values from stream of ubyte chunks
345 class Decoder {
346     enum State {
347         Init,
348         Type,
349     }
350     private {
351         Buffer          _chunks;
352         State           _state;
353         ValueType       _frontChunkType;
354         size_t          _parsedPosition;
355         size_t          _list_len;
356         RedisdValue[]    _list;
357     }
358     /// put next data chunk to decoder
359     bool put(immutable(ubyte)[] chunk) @safe {
360         assert(chunk.length > 0, "Chunk must not be emplty");
361         if ( _chunks.length == 0 ) {
362             switch( chunk[0]) {
363             case '+','-',':','*','$':
364                 _frontChunkType = cast(ValueType)chunk[0];
365                 debug(redisd) tracef("new chunk type %s", _frontChunkType);
366                 break;
367             default:
368                 throw new BadDataFormat("on chunk" ~ to!string(chunk));
369             }
370         }
371         _chunks.put(chunk);
372         //_len += chunk.length;
373         return false;
374     }
375 
376     private RedisdValue _handleListElement(RedisdValue v) @safe {
377         // we processing list element
378         () @trusted {debug(redisd) tracef("appending %s to list", v);} ();
379         _list ~= v;
380         _list_len--;
381         if (_list_len == 0) {
382             RedisdValue result = {_type:ValueType.List, _list : _list};
383             _list.length = 0;
384             return result;
385         }
386         return RedisdValue();
387     }
388     /// try to fetch decoded value from decoder.
389     /// Return next value or value of type $(B Incomplete).
390     RedisdValue get() @safe {
391         RedisdValue v;
392     start:
393         if (_chunks.length == 0 ) {
394             return v;
395         }
396         // Flat f = Flat(_chunks);
397         // Flat[] s;
398         Buffer[] s;
399         debug(redisd) tracef("check var type %c", cast(char)_chunks[0]);
400         switch (_chunks[0]) {
401         case ':':
402             s = _chunks.findSplitOn(['\r', '\n']);
403             if ( s[1].length == 2 ) {
404                 v = to!long(cast(string)s[0].data[1..$]);
405                 _chunks = s[2];
406                 if (_list_len > 0) {
407                     v = _handleListElement(v);
408                     if (_list_len)
409                         goto start;
410                 }
411                 return v;
412             }
413             break;
414         case '+':
415             s = _chunks.findSplitOn(['\r', '\n']);
416             if (s[1].length == 2) {
417                 v = cast(string) s[0].data[1 .. $];
418                 _chunks = s[2];
419                 if (_list_len > 0) {
420                     v = _handleListElement(v);
421                     if (_list_len)
422                         goto start;
423                 }
424                 return v;
425             }
426             break;
427         case '-':
428             s = _chunks.findSplitOn(['\r', '\n']);
429             if (s[1].length == 2) {
430                 v._type = ValueType.Error;
431                 v._svar = cast(string) s[0].data[1 .. $];
432                 _chunks = s[2];
433                 if (_list_len > 0) {
434                     v = _handleListElement(v);
435                     if (_list_len)
436                         goto start;
437                 }
438                 return v;
439             }
440             break;
441         case '$':
442             s = _chunks.findSplitOn(['\r', '\n']);
443             if (s[1].length == 2) {
444                 size_t len = (cast(string)s[0].data[1..$]).to!long;
445                 if ( len == -1 ) {
446                     v._type = ValueType.Null;
447                     _chunks = s[2];
448                     if (_list_len > 0) {
449                         v = _handleListElement(v);
450                         if (_list_len)
451                             goto start;
452                     }
453                     return v;
454                 }
455                 if (s[2].length < len + 2) {
456                     return v;
457                 }
458                 auto data = s[2][0..len].data;
459                 v = cast(string) data;
460                 _chunks = s[2][data.length+2..s[2].length];
461                 if (_list_len > 0) {
462                     v = _handleListElement(v);
463                     if (_list_len) goto start;
464                 }
465             }
466             break;
467         case '*':
468             _list_len = -1;
469             s = _chunks.findSplitOn(['\r', '\n']);
470             if (s[1].length == 2) {
471                 size_t len = (cast(string) s[0].data[1 .. $]).to!long;
472                 _list_len = len;
473                 _chunks = s[2];
474                 if (len == 0) {
475                     v._type = ValueType.Null;
476                     return v;
477                 }
478                 debug (redisd)
479                     tracef("list of length %d detected", _list_len);
480                 goto start;            
481             }
482             break;
483         default:
484             break;
485         }
486         return v;
487     }
488 }
489 ///
490 @safe unittest {
491     globalLogLevel = LogLevel.info;
492     RedisdValue str = {_type:ValueType.String, _svar : "abc"};
493     RedisdValue err = {_type:ValueType.Error, _svar : "err"};
494     auto b = redisdValue(1001).encode 
495             ~ redisdValue(1002).encode
496             ~ str.encode
497             ~ err.encode
498             ~ redisdValue("\r\nBulkString\r\n").encode
499             ~ redisdValue(1002).encode
500             ~ redisdValue([1,2,3]).encode
501             ~ redisdValue(["abc", "def"]).encode
502             ~ redisdValue(1002).encode;
503 
504     foreach(chunkSize; 1..b.length) {
505         auto s = new Decoder();
506         foreach (c; b.chunks(chunkSize)) {
507             s.put(c);
508         }
509         auto v = s.get();
510         assert(v._ivar == 1001);
511         v = s.get();
512         assert(v._ivar == 1002);
513         v = s.get();
514         assert(v._svar == "abc");
515         v = s.get();
516         assert(v._svar == "err");
517         v = s.get();
518         assert(v._svar == "\r\nBulkString\r\n");
519         v = s.get();
520         assert(v._ivar == 1002);
521         int lists_to_get = 2;
522         while( lists_to_get>0 ) {
523             v = s.get();
524             debug (redisd)
525                 () @trusted { tracef("%s: %s, %s", v._type, v, lists_to_get); }();
526             if (v._type == ValueType.List) {
527                 lists_to_get--;
528             }
529         }
530         v = s.get();
531         assert(v._ivar == 1002);
532         v = s.get();
533         assert(v._type == ValueType.Incomplete);
534     }
535     info("test ok");
536 }