Improve OutBuffer's allocation strategy

Author: andralex

src/dmd/backend/outbuf.d

@@ -28,9 +28,9 @@ struct Outbuffer
 {
   @safe:
 
-    ubyte *buf;         // the buffer itself
-    ubyte *pend;        // pointer past the end of the buffer
-    private ubyte *p;   // current position in buffer
+    private ubyte *buf;   // the buffer itself
+    private ubyte *pend;  // pointer past the end of the buffer
+    private ubyte *p;     // current position in buffer
 
   nothrow:
     this(size_t initialSize)
@@ -43,26 +43,16 @@ struct Outbuffer
     @trusted
     void dtor()
     {
-        if (auto slice = this.extractSlice())
-            free(slice.ptr);
+        free(buf);
+        buf = p = pend = null;
     }
 
-    void reset()
+    private void reset()
     {
         p = buf;
     }
 
     // Returns: A slice to the data written so far
-    extern(D) inout(ubyte)[] opSlice(size_t from, size_t to) inout
-        @trusted pure nothrow @nogc
-    {
-        assert(this.buf, "Attempt to dereference a null pointer");
-        assert(from < to, "First index must be smaller than the second one");
-        assert(this.length() <= (to - from), "Out of bound access");
-        return this.buf[from .. to];
-    }
-
-    /// Ditto
     extern(D) inout(ubyte)[] opSlice() inout @trusted pure nothrow @nogc
     {
         return this.buf[0 .. this.p - this.buf];
@@ -75,13 +65,22 @@ struct Outbuffer
         return ret;
     }
 
+    private extern(D) inout(ubyte)[] opSlice(size_t from, size_t to) inout
+        @trusted pure nothrow @nogc
+    {
+        assert(this.buf, "Attempt to dereference a null pointer");
+        assert(from < to, "First index must be smaller than the second one");
+        assert(this.length() <= (to - from), "Out of bound access");
+        return this.buf[from .. to];
+    }
+
     /********************
      * Make sure we have at least `nbytes` available for writing,
      * allocate more if necessary.
      * This is the inlinable fast path. Prefer `enlarge` if allocation
      * will always happen.
      */
-    void reserve(size_t nbytes)
+    private void reserve(size_t nbytes)
     {
         // Keep small so it is inlined
         if (pend - p < nbytes)
@@ -90,34 +89,45 @@ struct Outbuffer
 
     // Reserve nbytes in buffer
     @trusted
-    void enlarge(size_t nbytes)
+    private void enlarge(size_t nbytes)
     {
         pragma(inline, false);  // do not inline slow path
-        const size_t oldlen = pend - buf;
-        const size_t used = p - buf;
-
-        size_t len = used + nbytes;
-        // No need to reallocate
-        if (nbytes < (pend - p))
-            return;
 
-        const size_t newlen = oldlen + (oldlen >> 1);   // oldlen * 1.5
-        if (len < newlen)
-            len = newlen;
-        len = (len + 15) & ~15;
+        debug assert(nbytes > pend - p, "You should call this function from reserve() only.");
 
-        buf = cast(ubyte*) realloc(buf,len);
+        if (buf is null)
+        {
+            // Special-case the overwhelmingly most frequent situation
+            if (nbytes < 64) nbytes = 64;
+            p = buf = cast(ubyte*) malloc(nbytes);
+            pend = buf + nbytes;
+        }
+        else
+        {
+            const size_t used = p - buf;
+            const size_t oldlen = pend - buf;
+            // Ensure exponential growth, oldlen * 2 for small sizes, oldlen * 1.5 for big sizes
+            const size_t minlen = oldlen + (oldlen >> (oldlen > 1024 * 64));
+
+            size_t len = used + nbytes;
+            if (len < minlen)
+                len = minlen;
+            // Round to cache line size
+            len = (len + 63) & ~63;
+
+            auto newbuf = cast(ubyte*) realloc(buf, len);
+
+            pend = newbuf + len;
+            p = newbuf + used;
+            buf = newbuf;
+        }
         if (!buf)
             err_nomem();
-
-        pend = buf + len;
-        p = buf + used;
     }
 
-
     // Write n zeros; return pointer to start of zeros
     @trusted
-    void *writezeros(size_t n)
+    private void *writezeros(size_t n)
     {
         reserve(n);
         void *pstart = memset(p,0,n);
@@ -127,7 +137,7 @@ struct Outbuffer
 
     // Position buffer to accept the specified number of bytes at offset
     @trusted
-    void position(size_t offset, size_t nbytes)
+    private void position(size_t offset, size_t nbytes)
     {
         if (offset + nbytes > pend - buf)
         {
@@ -142,7 +152,7 @@ struct Outbuffer
 
     // Write an array to the buffer, no reserve check
     @trusted
-    void writen(const void *b, size_t len)
+    private void writen(const void *b, size_t len)
     {
         memcpy(p,b,len);
         p += len;
@@ -151,15 +161,15 @@ struct Outbuffer
     // Write an array to the buffer.
     @trusted
     extern (D)
-    void write(const(void)[] b)
+    private void write(const(void)[] b)
     {
         reserve(b.length);
         memcpy(p, b.ptr, b.length);
         p += b.length;
     }
 
     @trusted
-    void write(const(void)* b, size_t len)
+    private void write(const(void)* b, size_t len)
     {
         write(b[0 .. len]);
     }
@@ -168,7 +178,7 @@ struct Outbuffer
      * Writes an 8 bit byte, no reserve check.
      */
     @trusted
-    void writeByten(int v)
+    private void writeByten(int v)
     {
         *p++ = cast(ubyte)v;
     }
@@ -177,7 +187,7 @@ struct Outbuffer
      * Writes an 8 bit byte.
      */
     @trusted
-    void writeByte(int v)
+    private void writeByte(int v)
     {
         reserve(1);
         writeByten(v);
@@ -187,7 +197,7 @@ struct Outbuffer
      * Writes a 16 bit value, no reserve check.
      */
     @trusted
-    void write16n(int v)
+    private void write16n(int v)
     {
         *(cast(ushort *) p) = cast(ushort)v;
         p += 2;
@@ -197,7 +207,7 @@ struct Outbuffer
     /**
      * Writes a 16 bit value.
      */
-    void write16(int v)
+    private void write16(int v)
     {
         reserve(2);
         write16n(v);
@@ -207,7 +217,7 @@ struct Outbuffer
      * Writes a 32 bit int, no reserve check.
      */
     @trusted
-    void write32n(int v)
+    private void write32n(int v)
     {
         *cast(int *)p = v;
         p += 4;
@@ -216,7 +226,7 @@ struct Outbuffer
     /**
      * Writes a 32 bit int.
      */
-    void write32(int v)
+    private void write32(int v)
     {
         reserve(4);
         write32n(v);
@@ -226,7 +236,7 @@ struct Outbuffer
      * Writes a 64 bit long, no reserve check
      */
     @trusted
-    void write64n(long v)
+    private void write64n(long v)
     {
         *cast(long *)p = v;
         p += 8;
@@ -235,7 +245,7 @@ struct Outbuffer
     /**
      * Writes a 64 bit long.
      */
-    void write64(long v)
+    private void write64(long v)
     {
         reserve(8);
         write64n(v);
@@ -245,7 +255,7 @@ struct Outbuffer
      * Writes a 32 bit float.
      */
     @trusted
-    void writeFloat(float v)
+    private void writeFloat(float v)
     {
         reserve(float.sizeof);
         *cast(float *)p = v;
@@ -256,7 +266,7 @@ struct Outbuffer
      * Writes a 64 bit double.
      */
     @trusted
-    void writeDouble(double v)
+    private void writeDouble(double v)
     {
         reserve(double.sizeof);
         *cast(double *)p = v;
@@ -267,7 +277,7 @@ struct Outbuffer
      * Writes a String as a sequence of bytes.
      */
     @trusted
-    void write(const(char)* s)
+    private void write(const(char)* s)
     {
         write(s[0 .. strlen(s)]);
     }
@@ -276,20 +286,20 @@ struct Outbuffer
      * Writes a 0 terminated String
      */
     @trusted
-    void writeString(const(char)* s)
+    private void writeString(const(char)* s)
     {
         write(s[0 .. strlen(s)+1]);
     }
 
     /// Ditto
-    extern(D) void writeString(const(char)[] s)
+    private extern(D) void writeString(const(char)[] s)
     {
         write(s);
         writeByte(0);
     }
 
     /// Disembiguation for `string`
-    extern(D) void writeString(string s)
+    private extern(D) void writeString(string s)
     {
         writeString(cast(const(char)[])(s));
     }
@@ -298,7 +308,7 @@ struct Outbuffer
      * Inserts string at beginning of buffer.
      */
     @trusted
-    void prependBytes(const(char)* s)
+    private void prependBytes(const(char)* s)
     {
         prepend(s, strlen(s));
     }
@@ -307,7 +317,7 @@ struct Outbuffer
      * Inserts bytes at beginning of buffer.
      */
     @trusted
-    void prepend(const(void)* b, size_t len)
+    private void prepend(const(void)* b, size_t len)
     {
         reserve(len);
         memmove(buf + len,buf,p - buf);
@@ -319,7 +329,7 @@ struct Outbuffer
      * Bracket buffer contents with c1 and c2.
      */
     @trusted
-    void bracket(char c1,char c2)
+    private void bracket(char c1,char c2)
     {
         reserve(2);
         memmove(buf + 1,buf,p - buf);
@@ -331,7 +341,7 @@ struct Outbuffer
     /**
      * Returns the number of bytes written.
      */
-    size_t length() const @safe pure nothrow @nogc
+    private size_t length() const @safe pure nothrow @nogc
     {
         return p - buf;
     }
@@ -340,14 +350,14 @@ struct Outbuffer
      * Set current size of buffer.
      */
     @trusted
-    void setsize(size_t size)
+    private void setsize(size_t size)
     {
         p = buf + size;
         //debug assert(buf <= p);
         //debug assert(p <= pend);
     }
 
-    void writesLEB128(int value)
+    private void writesLEB128(int value)
     {
         while (1)
         {
@@ -364,7 +374,7 @@ struct Outbuffer
         }
     }
 
-    void writeuLEB128(uint value)
+    private void writeuLEB128(uint value)
     {
         do
         {