1 /** 2 Utility functions for array processing 3 4 Copyright: © 2012 Sönke Ludwig 5 License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. 6 Authors: Sönke Ludwig 7 */ 8 module vibe.internal.array; 9 10 import vibe.internal.allocator; 11 12 import std.algorithm; 13 import std.range : isInputRange, isOutputRange; 14 import std.traits; 15 static import std.utf; 16 17 18 void removeFromArray(T)(ref T[] array, T item) 19 { 20 foreach( i; 0 .. array.length ) 21 if( array[i] is item ){ 22 removeFromArrayIdx(array, i); 23 return; 24 } 25 } 26 27 void removeFromArrayIdx(T)(ref T[] array, size_t idx) 28 { 29 foreach( j; idx+1 .. array.length) 30 array[j-1] = array[j]; 31 array.length = array.length-1; 32 } 33 34 enum AppenderResetMode { 35 keepData, 36 freeData, 37 reuseData 38 } 39 40 struct AllocAppender(ArrayType : E[], E) { 41 alias ElemType = Unqual!E; 42 43 static assert(!hasIndirections!E && !hasElaborateDestructor!E); 44 45 private { 46 ElemType[] m_data; 47 ElemType[] m_remaining; 48 IAllocator m_alloc; 49 bool m_allocatedBuffer = false; 50 } 51 52 this(IAllocator alloc, ElemType[] initial_buffer = null) 53 @safe { 54 m_alloc = alloc; 55 m_data = initial_buffer; 56 m_remaining = initial_buffer; 57 } 58 59 @disable this(this); 60 61 @property ArrayType data() { return cast(ArrayType)m_data[0 .. m_data.length - m_remaining.length]; } 62 63 void reset(AppenderResetMode reset_mode = AppenderResetMode.keepData) 64 { 65 if (reset_mode == AppenderResetMode.keepData) m_data = null; 66 else if (reset_mode == AppenderResetMode.freeData) { if (m_allocatedBuffer) m_alloc.deallocate(m_data); m_data = null; } 67 m_remaining = m_data; 68 } 69 70 /** Grows the capacity of the internal buffer so that it can hold a minumum amount of elements. 71 72 Params: 73 amount = The minimum amount of elements that shall be appendable without 74 triggering a re-allocation. 75 76 */ 77 void reserve(size_t amount) 78 @safe { 79 size_t nelems = m_data.length - m_remaining.length; 80 if (!m_data.length) { 81 m_data = () @trusted { return cast(ElemType[])m_alloc.allocate(amount*E.sizeof); } (); 82 m_remaining = m_data; 83 m_allocatedBuffer = true; 84 } 85 if (m_remaining.length < amount) { 86 debug { 87 import std.digest.crc; 88 auto checksum = crc32Of(m_data[0 .. nelems]); 89 } 90 if (m_allocatedBuffer) () @trusted { 91 auto vdata = cast(void[])m_data; 92 m_alloc.reallocate(vdata, (nelems+amount)*E.sizeof); 93 m_data = cast(ElemType[])vdata; 94 } (); else { 95 auto newdata = () @trusted { return cast(ElemType[])m_alloc.allocate((nelems+amount)*E.sizeof); } (); 96 newdata[0 .. nelems] = m_data[0 .. nelems]; 97 m_data = newdata; 98 m_allocatedBuffer = true; 99 } 100 debug assert(crc32Of(m_data[0 .. nelems]) == checksum); 101 } 102 m_remaining = m_data[nelems .. m_data.length]; 103 } 104 105 void put(E el) 106 @safe { 107 if( m_remaining.length == 0 ) grow(1); 108 m_remaining[0] = el; 109 m_remaining = m_remaining[1 .. $]; 110 } 111 112 void put(ArrayType arr) 113 @safe { 114 if (m_remaining.length < arr.length) grow(arr.length); 115 m_remaining[0 .. arr.length] = arr[]; 116 m_remaining = m_remaining[arr.length .. $]; 117 } 118 119 static if( !hasAliasing!E ){ 120 void put(in ElemType[] arr) @trusted { 121 put(cast(ArrayType)arr); 122 } 123 } 124 125 static if( is(ElemType == char) ){ 126 void put(dchar el) 127 @trusted { 128 if( el < 128 ) put(cast(char)el); 129 else { 130 char[4] buf; 131 auto len = std.utf.encode(buf, el); 132 put(cast(ArrayType)buf[0 .. len]); 133 } 134 } 135 } 136 137 static if( is(ElemType == wchar) ){ 138 void put(dchar el) 139 @trusted { 140 if( el < 128 ) put(cast(wchar)el); 141 else { 142 wchar[3] buf; 143 auto len = std.utf.encode(buf, el); 144 put(cast(ArrayType)buf[0 .. len]); 145 } 146 } 147 } 148 149 static if (!is(E == immutable) || !hasAliasing!E) { 150 /** Appends a number of bytes in-place. 151 152 The delegate will get the memory slice of the memory that follows 153 the already written data. Use `reserve` to ensure that this slice 154 has enough room. The delegate should overwrite as much of the 155 slice as desired and then has to return the number of elements 156 that should be appended (counting from the start of the slice). 157 */ 158 void append(scope size_t delegate(scope ElemType[] dst) del) 159 { 160 auto n = del(m_remaining); 161 assert(n <= m_remaining.length); 162 m_remaining = m_remaining[n .. $]; 163 } 164 } 165 166 void grow(size_t min_free) 167 @safe { 168 if( !m_data.length && min_free < 16 ) min_free = 16; 169 170 auto min_size = m_data.length + min_free - m_remaining.length; 171 auto new_size = max(m_data.length, 16); 172 while( new_size < min_size ) 173 new_size = (new_size * 3) / 2; 174 reserve(new_size - m_data.length + m_remaining.length); 175 } 176 } 177 178 unittest { 179 auto a = AllocAppender!string(theAllocator()); 180 a.put("Hello"); 181 a.put(' '); 182 a.put("World"); 183 assert(a.data == "Hello World"); 184 a.reset(); 185 assert(a.data == ""); 186 } 187 188 unittest { 189 char[4] buf; 190 auto a = AllocAppender!string(theAllocator(), buf); 191 a.put("He"); 192 assert(a.data == "He"); 193 assert(a.data.ptr == buf.ptr); 194 a.put("ll"); 195 assert(a.data == "Hell"); 196 assert(a.data.ptr == buf.ptr); 197 a.put('o'); 198 assert(a.data == "Hello"); 199 assert(a.data.ptr != buf.ptr); 200 } 201 202 unittest { 203 char[4] buf; 204 auto a = AllocAppender!string(theAllocator(), buf); 205 a.put("Hello"); 206 assert(a.data == "Hello"); 207 assert(a.data.ptr != buf.ptr); 208 } 209 210 unittest { 211 auto app = AllocAppender!(int[])(theAllocator); 212 app.reserve(2); 213 app.append((scope mem) { 214 assert(mem.length >= 2); 215 mem[0] = 1; 216 mem[1] = 2; 217 return 2; 218 }); 219 assert(app.data == [1, 2]); 220 } 221 222 unittest { 223 auto app = AllocAppender!string(theAllocator); 224 app.reserve(3); 225 app.append((scope mem) { 226 assert(mem.length >= 3); 227 mem[0] = 'f'; 228 mem[1] = 'o'; 229 mem[2] = 'o'; 230 return 3; 231 }); 232 assert(app.data == "foo"); 233 } 234 235 236 struct FixedAppender(ArrayType : E[], size_t NELEM, E) { 237 alias ElemType = Unqual!E; 238 private { 239 ElemType[NELEM] m_data; 240 size_t m_fill; 241 } 242 243 void clear() 244 { 245 m_fill = 0; 246 } 247 248 void put(E el) 249 { 250 m_data[m_fill++] = el; 251 } 252 253 static if( is(ElemType == char) ){ 254 void put(dchar el) 255 { 256 if( el < 128 ) put(cast(char)el); 257 else { 258 char[4] buf; 259 auto len = std.utf.encode(buf, el); 260 put(cast(ArrayType)buf[0 .. len]); 261 } 262 } 263 } 264 265 static if( is(ElemType == wchar) ){ 266 void put(dchar el) 267 { 268 if( el < 128 ) put(cast(wchar)el); 269 else { 270 wchar[3] buf; 271 auto len = std.utf.encode(buf, el); 272 put(cast(ArrayType)buf[0 .. len]); 273 } 274 } 275 } 276 277 void put(ArrayType arr) 278 { 279 m_data[m_fill .. m_fill+arr.length] = (cast(ElemType[])arr)[]; 280 m_fill += arr.length; 281 } 282 283 @property ArrayType data() { return cast(ArrayType)m_data[0 .. m_fill]; } 284 285 static if (!is(E == immutable)) { 286 void reset() { m_fill = 0; } 287 } 288 } 289 290 291 /** 292 TODO: clear ring buffer fields upon removal (to run struct destructors, if T is a struct) 293 */ 294 struct FixedRingBuffer(T, size_t N = 0, bool INITIALIZE = true) { 295 private { 296 static if( N > 0 ) { 297 static if (INITIALIZE) T[N] m_buffer; 298 else T[N] m_buffer = void; 299 } else T[] m_buffer; 300 size_t m_start = 0; 301 size_t m_fill = 0; 302 } 303 304 static if( N == 0 ){ 305 bool m_freeOnDestruct; 306 this(size_t capacity) { m_buffer = new T[capacity]; } 307 ~this() { if (m_freeOnDestruct && m_buffer.length > 0) deleteCompat(m_buffer); } 308 } 309 310 @property bool empty() const { return m_fill == 0; } 311 312 @property bool full() const { return m_fill == m_buffer.length; } 313 314 @property size_t length() const { return m_fill; } 315 316 @property size_t freeSpace() const { return m_buffer.length - m_fill; } 317 318 @property size_t capacity() const { return m_buffer.length; } 319 320 static if( N == 0 ){ 321 deprecated @property void freeOnDestruct(bool b) { m_freeOnDestruct = b; } 322 323 /// Resets the capacity to zero and explicitly frees the memory for the buffer. 324 void dispose() 325 { 326 deleteCompat(m_buffer); 327 m_buffer = null; 328 m_start = m_fill = 0; 329 } 330 331 @property void capacity(size_t new_size) 332 { 333 if( m_buffer.length ){ 334 auto newbuffer = new T[new_size]; 335 auto dst = newbuffer; 336 auto newfill = min(m_fill, new_size); 337 read(dst[0 .. newfill]); 338 if (m_freeOnDestruct && m_buffer.length > 0) () @trusted { 339 deleteCompat(m_buffer); 340 } (); 341 m_buffer = newbuffer; 342 m_start = 0; 343 m_fill = newfill; 344 } else { 345 if (m_freeOnDestruct && m_buffer.length > 0) () @trusted { 346 deleteCompat(m_buffer); 347 } (); 348 m_buffer = new T[new_size]; 349 } 350 } 351 } 352 353 @property ref inout(T) front() inout { assert(!empty); return m_buffer[m_start]; } 354 355 @property ref inout(T) back() inout { assert(!empty); return m_buffer[mod(m_start+m_fill-1)]; } 356 357 void clear() 358 { 359 popFrontN(length); 360 assert(m_fill == 0); 361 m_start = 0; 362 } 363 364 void put()(T itm) { assert(m_fill < m_buffer.length); move(itm, m_buffer[mod(m_start + m_fill++)]); } 365 void put(TC : T)(scope TC[] itms) 366 { 367 if( !itms.length ) return; 368 assert(m_fill+itms.length <= m_buffer.length); 369 if( mod(m_start+m_fill) >= mod(m_start+m_fill+itms.length) ){ 370 size_t chunk1 = m_buffer.length - (m_start+m_fill); 371 size_t chunk2 = itms.length - chunk1; 372 m_buffer[m_start+m_fill .. m_buffer.length] = itms[0 .. chunk1]; 373 m_buffer[0 .. chunk2] = itms[chunk1 .. $]; 374 } else { 375 m_buffer[mod(m_start+m_fill) .. mod(m_start+m_fill)+itms.length] = itms[]; 376 } 377 m_fill += itms.length; 378 } 379 void putN(size_t n) { assert(m_fill+n <= m_buffer.length); m_fill += n; } 380 381 void popFront() { assert(!empty); m_start = mod(m_start+1); m_fill--; } 382 void popFrontN(size_t n) { assert(length >= n); m_start = mod(m_start + n); m_fill -= n; } 383 384 void popBack() { assert(!empty); m_fill--; } 385 void popBackN(size_t n) { assert(length >= n); m_fill -= n; } 386 387 void removeAt(Range r) 388 { 389 assert(r.m_buffer is m_buffer); 390 if( m_start + m_fill > m_buffer.length ){ 391 assert(r.m_start >= m_start && r.m_start < m_buffer.length || r.m_start < mod(m_start+m_fill)); 392 if( r.m_start > m_start ){ 393 foreach(i; r.m_start .. m_buffer.length-1) 394 move(m_buffer[i+1], m_buffer[i]); 395 move(m_buffer[0], m_buffer[$-1]); 396 foreach(i; 0 .. mod(m_start + m_fill - 1)) 397 move(m_buffer[i+1], m_buffer[i]); 398 } else { 399 foreach(i; r.m_start .. mod(m_start + m_fill - 1)) 400 move(m_buffer[i+1], m_buffer[i]); 401 } 402 } else { 403 assert(r.m_start >= m_start && r.m_start < m_start+m_fill); 404 foreach(i; r.m_start .. m_start+m_fill-1) 405 move(m_buffer[i+1], m_buffer[i]); 406 } 407 m_fill--; 408 destroy(m_buffer[mod(m_start+m_fill)]); // TODO: only call destroy for non-POD T 409 } 410 411 inout(T)[] peek() inout { return m_buffer[m_start .. min(m_start+m_fill, m_buffer.length)]; } 412 T[] peekDst() { 413 if (!m_buffer.length) return null; 414 if( m_start + m_fill < m_buffer.length ) return m_buffer[m_start+m_fill .. $]; 415 else return m_buffer[mod(m_start+m_fill) .. m_start]; 416 } 417 418 void read(scope T[] dst) 419 { 420 assert(dst.length <= length); 421 if( !dst.length ) return; 422 if( mod(m_start) >= mod(m_start+dst.length) ){ 423 size_t chunk1 = m_buffer.length - m_start; 424 size_t chunk2 = dst.length - chunk1; 425 static if (isCopyable!T) { 426 dst[0 .. chunk1] = m_buffer[m_start .. $]; 427 dst[chunk1 .. $] = m_buffer[0 .. chunk2]; 428 } else { 429 foreach (i; 0 .. chunk1) move(m_buffer[m_start+i], dst[i]); 430 foreach (i; chunk1 .. this.length) move(m_buffer[i-chunk1], dst[i]); 431 } 432 } else { 433 static if (isCopyable!T) { 434 dst[] = m_buffer[m_start .. m_start+dst.length]; 435 } else { 436 foreach (i; 0 .. dst.length) 437 move(m_buffer[m_start + i], dst[i]); 438 } 439 } 440 popFrontN(dst.length); 441 } 442 443 int opApply(scope int delegate(ref T itm) del) 444 { 445 if( m_start+m_fill > m_buffer.length ){ 446 foreach(i; m_start .. m_buffer.length) 447 if( auto ret = del(m_buffer[i]) ) 448 return ret; 449 foreach(i; 0 .. mod(m_start+m_fill)) 450 if( auto ret = del(m_buffer[i]) ) 451 return ret; 452 } else { 453 foreach(i; m_start .. m_start+m_fill) 454 if( auto ret = del(m_buffer[i]) ) 455 return ret; 456 } 457 return 0; 458 } 459 460 /// iterate through elements with index 461 int opApply(scope int delegate(size_t i, ref T itm) del) 462 { 463 if( m_start+m_fill > m_buffer.length ){ 464 foreach(i; m_start .. m_buffer.length) 465 if( auto ret = del(i - m_start, m_buffer[i]) ) 466 return ret; 467 foreach(i; 0 .. mod(m_start+m_fill)) 468 if( auto ret = del(i + m_buffer.length - m_start, m_buffer[i]) ) 469 return ret; 470 } else { 471 foreach(i; m_start .. m_start+m_fill) 472 if( auto ret = del(i - m_start, m_buffer[i]) ) 473 return ret; 474 } 475 return 0; 476 } 477 478 ref inout(T) opIndex(size_t idx) inout { assert(idx < length); return m_buffer[mod(m_start+idx)]; } 479 480 Range opSlice() { return Range(m_buffer, m_start, m_fill); } 481 482 Range opSlice(size_t from, size_t to) 483 { 484 assert(from <= to); 485 assert(to <= m_fill); 486 return Range(m_buffer, mod(m_start+from), to-from); 487 } 488 489 size_t opDollar(size_t dim)() const if(dim == 0) { return length; } 490 491 private size_t mod(size_t n) 492 const { 493 static if( N == 0 ){ 494 /*static if(PotOnly){ 495 return x & (m_buffer.length-1); 496 } else {*/ 497 return n % m_buffer.length; 498 //} 499 } else static if( ((N - 1) & N) == 0 ){ 500 return n & (N - 1); 501 } else return n % N; 502 } 503 504 static struct Range { 505 private { 506 T[] m_buffer; 507 size_t m_start; 508 size_t m_length; 509 } 510 511 private this(T[] buffer, size_t start, size_t length) 512 { 513 m_buffer = buffer; 514 m_start = start; 515 m_length = length; 516 } 517 518 @property bool empty() const { return m_length == 0; } 519 520 @property ref inout(T) front() inout { assert(!empty); return m_buffer[m_start]; } 521 522 void popFront() 523 { 524 assert(!empty); 525 m_start++; 526 m_length--; 527 if( m_start >= m_buffer.length ) 528 m_start = 0; 529 } 530 } 531 } 532 533 unittest { 534 static assert(isInputRange!(FixedRingBuffer!int) && isOutputRange!(FixedRingBuffer!int, int)); 535 536 FixedRingBuffer!(int, 5) buf; 537 assert(buf.length == 0 && buf.freeSpace == 5); buf.put(1); // |1 . . . . 538 assert(buf.length == 1 && buf.freeSpace == 4); buf.put(2); // |1 2 . . . 539 assert(buf.length == 2 && buf.freeSpace == 3); buf.put(3); // |1 2 3 . . 540 assert(buf.length == 3 && buf.freeSpace == 2); buf.put(4); // |1 2 3 4 . 541 assert(buf.length == 4 && buf.freeSpace == 1); buf.put(5); // |1 2 3 4 5 542 assert(buf.length == 5 && buf.freeSpace == 0); 543 assert(buf.front == 1); 544 buf.popFront(); // .|2 3 4 5 545 assert(buf.front == 2); 546 buf.popFrontN(2); // . . .|4 5 547 assert(buf.front == 4); 548 assert(buf.length == 2 && buf.freeSpace == 3); 549 buf.put([6, 7, 8]); // 6 7 8|4 5 550 assert(buf.length == 5 && buf.freeSpace == 0); 551 int[5] dst; 552 buf.read(dst); // . . .|. . 553 assert(dst == [4, 5, 6, 7, 8]); 554 assert(buf.length == 0 && buf.freeSpace == 5); 555 buf.put([1, 2]); // . . .|1 2 556 assert(buf.length == 2 && buf.freeSpace == 3); 557 buf.read(dst[0 .. 2]); //|. . . . . 558 assert(dst[0 .. 2] == [1, 2]); 559 560 buf.put([0, 0, 0, 1, 2]); //|0 0 0 1 2 561 buf.popFrontN(2); //. .|0 1 2 562 buf.put([3, 4]); // 3 4|0 1 2 563 foreach(i, item; buf) 564 { 565 assert(i == item); 566 } 567 } 568 569 570 /// Write a single batch and drain 571 struct BatchBuffer(T, size_t N = 0) { 572 private { 573 size_t m_fill; 574 size_t m_first; 575 static if (N == 0) T[] m_buffer; 576 else T[N] m_buffer; 577 } 578 579 static if (N == 0) { 580 @property void capacity(size_t n) { assert(n >= m_fill); m_buffer.length = n; } 581 } 582 583 @property bool empty() const { assert(m_first < m_fill || m_fill == 0 && m_first == 0); return m_first >= m_fill; } 584 @property size_t capacity() const { return m_buffer.length; } 585 @property size_t length() const { return m_fill - m_first; } 586 @property ref inout(T) front() inout { assert(!empty); return m_buffer[m_first]; } 587 void popFront() { popFrontN(1); } 588 void popFrontN(size_t n) { 589 assert(n <= length); 590 m_first += n; 591 if (m_first == m_fill) 592 m_first = m_fill = 0; 593 } 594 inout(T)[] peek() inout { return m_buffer[m_first .. m_fill]; } 595 T[] peekDst() { assert(empty); return m_buffer; } 596 void putN(size_t n) { assert(empty && n <= m_buffer.length); m_fill = n; } 597 void putN(T[] elems) { assert(empty && elems.length <= m_buffer.length); m_buffer[0 .. elems.length] = elems[]; m_fill = elems.length; } 598 void read(T[] dst) { 599 assert(length() >= dst.length); 600 dst[] = m_buffer[m_first .. m_first + dst.length]; 601 popFrontN(dst.length); 602 } 603 } 604 605 606 struct ArraySet(Key) 607 { 608 private { 609 Key[4] m_staticEntries; 610 Key[] m_entries; 611 } 612 613 @property ArraySet dup() 614 { 615 return ArraySet(m_staticEntries, m_entries.dup); 616 } 617 618 bool opBinaryRight(string op)(Key key) if (op == "in") { return contains(key); } 619 620 int opApply(int delegate(ref Key) del) 621 { 622 foreach (ref k; m_staticEntries) 623 if (k != Key.init) 624 if (auto ret = del(k)) 625 return ret; 626 foreach (ref k; m_entries) 627 if (k != Key.init) 628 if (auto ret = del(k)) 629 return ret; 630 return 0; 631 } 632 633 bool contains(Key key) 634 const { 635 foreach (ref k; m_staticEntries) if (k == key) return true; 636 foreach (ref k; m_entries) if (k == key) return true; 637 return false; 638 } 639 640 void insert(Key key) 641 { 642 if (contains(key)) return; 643 foreach (ref k; m_staticEntries) 644 if (k == Key.init) { 645 k = key; 646 return; 647 } 648 foreach (ref k; m_entries) 649 if (k == Key.init) { 650 k = key; 651 return; 652 } 653 m_entries ~= key; 654 } 655 656 void remove(Key key) 657 { 658 foreach (ref k; m_staticEntries) if (k == key) { k = Key.init; return; } 659 foreach (ref k; m_entries) if (k == key) { k = Key.init; return; } 660 } 661 } 662 663 private void deleteCompat(T)(ref T v) 664 { 665 static if (__VERSION__ >= 2079) { 666 import core.memory : __delete; 667 __delete(v); 668 } else mixin("delete v;"); 669 }