[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[GNUnet-SVN] [libmicrohttpd] branch master updated (d12ed49f -> 55dc42e6
|
From: |
gnunet |
|
Subject: |
[GNUnet-SVN] [libmicrohttpd] branch master updated (d12ed49f -> 55dc42e6) |
|
Date: |
Mon, 17 Jun 2019 20:35:43 +0200 |
This is an automated email from the git hooks/post-receive script.
karlson2k pushed a change to branch master
in repository libmicrohttpd.
from d12ed49f memorypool: reallocate: more checks for value wrap, small
optimization
new c98aa3d3 memorypool: fixed MHD_pool_reset()
new 8646d3e4 memorypool: round mmap allocation for page size
new 3b42621c memorypool: re-commit memory on W32 when resetting pool
new 03df91fc memorypool: comments minor improvements
new 3cb0b6cf MHD_pool_reallocate(): never allocate additional buffer when
shrinking
new 4bc166ee test_long_header: use power of two for size to properly test
allocations, test various buffer sizes
new 20767d5d transmit_error_response(): discard read buffer to free space
for error response
new 1f9e183d process_request_body(): do not move data if nothing was
processed
new a275ae63 try_ready_chunked_body(): remove useless loop, use know free
memory size instead of guessing
new 55dc42e6 Optimised read buffer allocation
The 10 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
configure.ac | 2 +
src/microhttpd/connection.c | 113 +++++++++++++++++++-----------
src/microhttpd/daemon.c | 1 +
src/microhttpd/internal.h | 2 +-
src/microhttpd/memorypool.c | 151 +++++++++++++++++++++++++++++++---------
src/microhttpd/memorypool.h | 6 ++
src/testcurl/test_long_header.c | 20 +++---
7 files changed, 210 insertions(+), 85 deletions(-)
diff --git a/configure.ac b/configure.ac
index 92d59f1b..5c68b1be 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1202,6 +1202,8 @@ have_inet6=no
])
AC_MSG_RESULT($have_inet6)
+MHD_CHECK_FUNC([[sysconf]], [[#include <unitstd.h>]], [[long a = sysconf(0);
if (a) return 1;]])
+
HIDDEN_VISIBILITY_CFLAGS=""
AS_CASE(["$host"],
[*-*-mingw*],[
diff --git a/src/microhttpd/connection.c b/src/microhttpd/connection.c
index b3ef9441..877f514b 100644
--- a/src/microhttpd/connection.c
+++ b/src/microhttpd/connection.c
@@ -1342,9 +1342,7 @@ static int
try_ready_chunked_body (struct MHD_Connection *connection)
{
ssize_t ret;
- char *buf;
struct MHD_Response *response;
- size_t size;
char cbuf[10]; /* 10: max strlen of "%x\r\n" */
int cblen;
@@ -1353,28 +1351,26 @@ try_ready_chunked_body (struct MHD_Connection
*connection)
return MHD_YES;
if (0 == connection->write_buffer_size)
{
- size = MHD_MIN (connection->daemon->pool_size,
- 2 * (0xFFFFFF + sizeof(cbuf) + 2));
- do
+ size_t size;
+
+ size = MHD_pool_get_free (connection->pool);
+ if (size < 128)
{
- size /= 2;
- if (size < 128)
- {
#if defined(MHD_USE_POSIX_THREADS) || defined(MHD_USE_W32_THREADS)
- MHD_mutex_unlock_chk_ (&response->mutex);
+ MHD_mutex_unlock_chk_ (&response->mutex);
#endif
- /* not enough memory */
- CONNECTION_CLOSE_ERROR (connection,
- _("Closing connection (out of
memory)\n"));
- return MHD_NO;
- }
- buf = MHD_pool_allocate (connection->pool,
- size,
- false);
+ /* not enough memory */
+ CONNECTION_CLOSE_ERROR (connection,
+ _("Closing connection (out of memory)\n"));
+ return MHD_NO;
}
- while (NULL == buf);
+ if ( (2 * (0xFFFFFF + sizeof(cbuf) + 2)) < size)
+ size = 2 * (0xFFFFFF + sizeof(cbuf) + 2);
+ connection->write_buffer = MHD_pool_allocate (connection->pool,
+ size,
+ false);
+ mhd_assert (NULL != connection->write_buffer);
connection->write_buffer_size = size;
- connection->write_buffer = buf;
}
if (0 == response->total_size)
@@ -1579,28 +1575,51 @@ get_date_string (char *date,
* minimum necessary at that point.
*
* @param connection the connection
- * @return #MHD_YES on success, #MHD_NO on failure
+ * @param required set to 'true' if grow is required, i.e. connection
+ * will fail if no additional space is granted
+ * @return 'true' on success, 'false' on failure
*/
-static int
-try_grow_read_buffer (struct MHD_Connection *connection)
+static bool
+try_grow_read_buffer (struct MHD_Connection *connection,
+ bool required)
{
- void *buf;
size_t new_size;
+ size_t avail_size;
+ avail_size = MHD_pool_get_free (connection->pool);
+ if (0 == avail_size)
+ return false; /* No more space available */
if (0 == connection->read_buffer_size)
- new_size = connection->daemon->pool_size / 2;
+ new_size = avail_size / 2; /* Use half of available buffer for reading */
else
- new_size = connection->read_buffer_size + MHD_BUF_INC_SIZE;
- buf = MHD_pool_reallocate (connection->pool,
- connection->read_buffer,
- connection->read_buffer_size,
- new_size);
- if (NULL == buf)
- return MHD_NO;
+ {
+ size_t grow_size;
+
+ grow_size = avail_size / 8;
+ if (MHD_BUF_INC_SIZE > grow_size)
+ { /* Shortage of space */
+ if (!required)
+ return false; /* Grow is not mandatory, leave some space in pool */
+ else
+ {
+ /* Shortage of space, but grow is mandatory */
+ static const size_t small_inc = MHD_BUF_INC_SIZE / 8;
+ if (small_inc < avail_size)
+ grow_size = small_inc;
+ else
+ grow_size = avail_size;
+ }
+ }
+ new_size = connection->read_buffer_size + grow_size;
+ }
/* we can actually grow the buffer, do it! */
- connection->read_buffer = buf;
+ connection->read_buffer = MHD_pool_reallocate (connection->pool,
+ connection->read_buffer,
+ connection->read_buffer_size,
+ new_size);
+ mhd_assert (NULL != connection->read_buffer);
connection->read_buffer_size = new_size;
- return MHD_YES;
+ return true;
}
@@ -1995,6 +2014,16 @@ transmit_error_response (struct MHD_Connection
*connection,
}
connection->state = MHD_CONNECTION_FOOTERS_RECEIVED;
connection->read_closed = true;
+ if (0 != connection->read_buffer_size)
+ {
+ /* Read buffer is not needed anymore, discard it
+ * to free some space for error response. */
+ connection->read_buffer = MHD_pool_reallocate(connection->pool,
+ connection->read_buffer,
+
connection->read_buffer_size,
+ 0);
+ connection->read_buffer_size = 0;
+ }
#ifdef HAVE_MESSAGES
MHD_DLOG (connection->daemon,
_("Error processing request (HTTP response code is %u (`%s')).
Closing connection.\n"),
@@ -2091,7 +2120,7 @@ MHD_connection_update_event_loop_info (struct
MHD_Connection *connection)
/* while reading headers, we always grow the
read buffer if needed, no size-check required */
if ( (connection->read_buffer_offset ==
connection->read_buffer_size) &&
- (MHD_NO == try_grow_read_buffer (connection)) )
+ (!try_grow_read_buffer (connection, true)) )
{
transmit_error_response (connection,
(connection->url != NULL)
@@ -2117,9 +2146,10 @@ MHD_connection_update_event_loop_info (struct
MHD_Connection *connection)
case MHD_CONNECTION_CONTINUE_SENT:
if (connection->read_buffer_offset == connection->read_buffer_size)
{
- if ((MHD_YES != try_grow_read_buffer (connection)) &&
- (0 != (connection->daemon->options &
- MHD_USE_INTERNAL_POLLING_THREAD)))
+ const bool internal_poll = (0 != (connection->daemon->options &
+ MHD_USE_INTERNAL_POLLING_THREAD));
+ if ( (!try_grow_read_buffer (connection, true)) &&
+ internal_poll)
{
/* failed to grow the read buffer, and the
client which is supposed to handle the
@@ -2238,8 +2268,7 @@ get_next_header_line (struct MHD_Connection *connection,
{
/* not found, consider growing... */
if ( (connection->read_buffer_offset == connection->read_buffer_size) &&
- (MHD_NO ==
- try_grow_read_buffer (connection)) )
+ (!try_grow_read_buffer (connection, true)) )
{
transmit_error_response (connection,
(NULL != connection->url)
@@ -2794,7 +2823,8 @@ process_request_body (struct MHD_Connection *connection)
connection->remaining_upload_size -= processed_size;
}
while (MHD_YES == instant_retry);
- if (available > 0)
+ if ( (available > 0) &&
+ (buffer_head != connection->read_buffer) )
memmove (connection->read_buffer,
buffer_head,
available);
@@ -3150,7 +3180,8 @@ MHD_connection_handle_read (struct MHD_Connection
*connection)
in buffer to use per system call (if possible) */
if (connection->read_buffer_offset + connection->daemon->pool_increment >
connection->read_buffer_size)
- try_grow_read_buffer (connection);
+ try_grow_read_buffer (connection,
+ (connection->read_buffer_size ==
connection->read_buffer_offset));
if (connection->read_buffer_size == connection->read_buffer_offset)
return; /* No space for receiving data. */
diff --git a/src/microhttpd/daemon.c b/src/microhttpd/daemon.c
index 2729bb21..b5d151b4 100644
--- a/src/microhttpd/daemon.c
+++ b/src/microhttpd/daemon.c
@@ -7185,6 +7185,7 @@ MHD_init(void)
#ifdef HAVE_FREEBSD_SENDFILE
MHD_conn_init_static_ ();
#endif /* HAVE_FREEBSD_SENDFILE */
+ MHD_init_mem_pools_ ();
}
diff --git a/src/microhttpd/internal.h b/src/microhttpd/internal.h
index 97ac050f..8dc813a2 100644
--- a/src/microhttpd/internal.h
+++ b/src/microhttpd/internal.h
@@ -93,7 +93,7 @@
/**
- * Minimum size by which MHD tries to increment read/write buffers.
+ * Minimum reasonable size by which MHD tries to increment read/write buffers.
* We usually begin with half the available pool space for the
* IO-buffer, but if absolutely needed we additively grow by the
* number of bytes given here (up to -- theoretically -- the full pool
diff --git a/src/microhttpd/memorypool.c b/src/microhttpd/memorypool.c
index 3770b8c3..6f74db83 100644
--- a/src/microhttpd/memorypool.c
+++ b/src/microhttpd/memorypool.c
@@ -35,6 +35,14 @@
#ifdef _WIN32
#include <windows.h>
#endif
+#ifdef HAVE_SYSCONF
+#include <unistd.h>
+#if defined(_SC_PAGE_SIZE)
+#define MHD_SC_PAGESIZE _SC_PAGE_SIZE
+#elif defined(_SC_PAGESIZE)
+#define MHD_SC_PAGESIZE _SC_PAGESIZE
+#endif /* _SC_PAGESIZE */
+#endif /* HAVE_SYSCONF */
/* define MAP_ANONYMOUS for Mac OS X */
#if defined(MAP_ANON) && !defined(MAP_ANONYMOUS)
@@ -56,6 +64,40 @@
*/
#define ROUND_TO_ALIGN(n) (((n)+(ALIGN_SIZE-1)) / (ALIGN_SIZE) * (ALIGN_SIZE))
+#if defined(PAGE_SIZE)
+#define MHD_DEF_PAGE_SIZE_ PAGE_SIZE
+#elif defined(PAGESIZE)
+#define MHD_DEF_PAGE_SIZE_ PAGE_SIZE
+#else /* ! PAGESIZE */
+#define MHD_DEF_PAGE_SIZE_ (4096)
+#endif /* ! PAGESIZE */
+
+/**
+ * Size of memory page
+ */
+static size_t MHD_sys_page_size_ = MHD_DEF_PAGE_SIZE_; /* Default fallback
value */
+
+/**
+ * Initilise values for memory pools
+ */
+void
+MHD_init_mem_pools_ (void)
+{
+#ifdef MHD_SC_PAGESIZE
+ long result;
+ result = sysconf (MHD_SC_PAGESIZE);
+ if (-1 != result)
+ MHD_sys_page_size_ = (size_t) result;
+ else
+ MHD_sys_page_size_ = MHD_DEF_PAGE_SIZE_;
+#elif defined(_WIN32)
+ SYSTEM_INFO si;
+ GetSystemInfo (&si);
+ MHD_sys_page_size_ = (size_t)si.dwPageSize;
+#else
+ MHD_sys_page_size_ = MHD_DEF_PAGE_SIZE_;
+#endif /* _WIN32 */
+}
/**
* Handle for a memory pool. Pools are not reentrant and must not be
@@ -101,34 +143,41 @@ struct MemoryPool *
MHD_pool_create (size_t max)
{
struct MemoryPool *pool;
+ size_t alloc_size;
- max = ROUND_TO_ALIGN(max);
pool = malloc (sizeof (struct MemoryPool));
if (NULL == pool)
return NULL;
#if defined(MAP_ANONYMOUS) || defined(_WIN32)
- if (max <= 32 * 1024)
+ if ( (max <= 32 * 1024) ||
+ (max < MHD_sys_page_size_ * 4 / 3) )
pool->memory = MAP_FAILED;
else
+ {
+ /* Round up allocation to page granularity. */
+ alloc_size = max + MHD_sys_page_size_ - 1;
+ alloc_size -= alloc_size % MHD_sys_page_size_;
#if defined(MAP_ANONYMOUS) && !defined(_WIN32)
- pool->memory = mmap (NULL,
- max,
- PROT_READ | PROT_WRITE,
- MAP_PRIVATE | MAP_ANONYMOUS,
- -1,
- 0);
+ pool->memory = mmap (NULL,
+ alloc_size,
+ PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS,
+ -1,
+ 0);
#elif defined(_WIN32)
- pool->memory = VirtualAlloc (NULL,
- max,
- MEM_COMMIT | MEM_RESERVE,
- PAGE_READWRITE);
-#endif
-#else
+ pool->memory = VirtualAlloc (NULL,
+ alloc_size,
+ MEM_COMMIT | MEM_RESERVE,
+ PAGE_READWRITE);
+#endif /* _WIN32 */
+ }
+#else /* ! _WIN32 && ! MAP_ANONYMOUS */
pool->memory = MAP_FAILED;
-#endif
+#endif /* ! _WIN32 && ! MAP_ANONYMOUS */
if (MAP_FAILED == pool->memory)
{
- pool->memory = malloc (max);
+ alloc_size = ROUND_TO_ALIGN(max);
+ pool->memory = malloc (alloc_size);
if (NULL == pool->memory)
{
free (pool);
@@ -136,13 +185,15 @@ MHD_pool_create (size_t max)
}
pool->is_mmap = false;
}
+#if defined(MAP_ANONYMOUS) || defined(_WIN32)
else
{
pool->is_mmap = true;
}
+#endif /* _WIN32 || MAP_ANONYMOUS */
pool->pos = 0;
- pool->end = max;
- pool->size = max;
+ pool->end = alloc_size;
+ pool->size = alloc_size;
return pool;
}
@@ -270,15 +321,16 @@ MHD_pool_reallocate (struct MemoryPool *pool,
if (0 != old_size)
{ /* Need to save some data */
const size_t old_offset = (uint8_t*)old - pool->memory;
+ const bool shrinking = (old_size > new_size);
/* Try resizing in-place */
+ if (shrinking)
+ { /* Shrinking in-place, zero-out freed part */
+ memset ((uint8_t*)old + new_size, 0, old_size - new_size);
+ }
if (pool->pos == ROUND_TO_ALIGN (old_offset + old_size))
{ /* "old" block is the last allocated block */
const size_t new_apos = ROUND_TO_ALIGN (old_offset + new_size);
- if (old_size > new_size)
- { /* Shrinking in-place, zero-out freed part */
- memset ((uint8_t*)old + new_size, 0, old_size - new_size);
- }
- else
+ if (!shrinking)
{ /* Grow in-place, check for enough space. */
if ( (new_apos > pool->end) ||
(new_apos < pool->pos) ) /* Value wrap */
@@ -288,13 +340,15 @@ MHD_pool_reallocate (struct MemoryPool *pool,
pool->pos = new_apos;
return old;
}
+ if (shrinking)
+ return old; /* Resized in-place, freed part remains allocated */
}
/* Need to allocate new block */
asize = ROUND_TO_ALIGN (new_size);
if ( ( (0 == asize) &&
(0 != new_size) ) || /* Value wrap, too large new_size. */
- (asize > pool->end - pool->pos) )
- return NULL; /* No space */
+ (asize > pool->end - pool->pos) ) /* Not enough space */
+ return NULL;
new_blc = pool->memory + pool->pos;
pool->pos += asize;
@@ -303,7 +357,7 @@ MHD_pool_reallocate (struct MemoryPool *pool,
{
/* Move data to new block, old block remains allocated */
memcpy (new_blc, old, old_size);
- /* Zero-out freed old block */
+ /* Zero-out old block */
memset (old, 0, old_size);
}
return new_blc;
@@ -331,6 +385,7 @@ MHD_pool_reset (struct MemoryPool *pool,
{
mhd_assert (pool->end >= pool->pos);
mhd_assert (pool->size >= pool->end - pool->pos);
+ mhd_assert (copy_bytes < new_size);
mhd_assert (keep != NULL || copy_bytes == 0);
mhd_assert (keep == NULL || pool->memory <= (uint8_t*)keep);
mhd_assert (keep == NULL || pool->memory + pool->size >= (uint8_t*)keep +
copy_bytes);
@@ -341,17 +396,45 @@ MHD_pool_reset (struct MemoryPool *pool,
memmove (pool->memory,
keep,
copy_bytes);
- keep = pool->memory;
}
- pool->end = pool->size;
/* technically not needed, but safer to zero out */
if (pool->size > copy_bytes)
- memset (&pool->memory[copy_bytes],
- 0,
- pool->size - copy_bytes);
- if (NULL != keep)
- pool->pos = ROUND_TO_ALIGN (new_size);
- return keep;
+ {
+ size_t to_zero; /** Size of area to zero-out */
+
+ to_zero = pool->size - copy_bytes;
+#ifdef _WIN32
+ if (pool->is_mmap)
+ {
+ size_t to_recommit; /** Size of decommitted and re-committed area. */
+ uint8_t *recommit_addr;
+ /* Round down to page size */
+ to_recommit = to_zero - to_zero % MHD_sys_page_size_;
+ recommit_addr = pool->memory + pool->size - to_recommit;
+
+ /* De-committing and re-committing again clear memory and make
+ * pages free / available for other needs until accessed. */
+ if (VirtualFree (recommit_addr,
+ to_recommit,
+ MEM_DECOMMIT))
+ {
+ to_zero -= to_recommit;
+
+ if (recommit_addr != VirtualAlloc (recommit_addr,
+ to_recommit,
+ MEM_COMMIT,
+ PAGE_READWRITE))
+ abort(); /* Serious error, must never happen */
+ }
+ }
+#endif /* _WIN32 */
+ memset (&pool->memory[copy_bytes],
+ 0,
+ to_zero);
+ }
+ pool->pos = ROUND_TO_ALIGN (new_size);
+ pool->end = pool->size;
+ return pool->memory;
}
diff --git a/src/microhttpd/memorypool.h b/src/microhttpd/memorypool.h
index c5fe7ada..abd53365 100644
--- a/src/microhttpd/memorypool.h
+++ b/src/microhttpd/memorypool.h
@@ -43,6 +43,12 @@
*/
struct MemoryPool;
+/**
+ * Initilise values for memory pools
+ */
+void
+MHD_init_mem_pools_ (void);
+
/**
* Create a memory pool.
diff --git a/src/testcurl/test_long_header.c b/src/testcurl/test_long_header.c
index 80842622..dc76a136 100644
--- a/src/testcurl/test_long_header.c
+++ b/src/testcurl/test_long_header.c
@@ -42,7 +42,7 @@
* half of this value, so the actual value does not have
* to be big at all...
*/
-#define VERY_LONG (1024*10)
+#define VERY_LONG (1024*8)
static int oneone;
@@ -94,7 +94,7 @@ ahc_echo (void *cls,
static int
-testLongUrlGet ()
+testLongUrlGet (size_t buff_size)
{
struct MHD_Daemon *d;
CURL *c;
@@ -108,7 +108,7 @@ testLongUrlGet ()
port = 0;
else
{
- port = 1330;
+ port = 1330 + buff_size % 20;
if (oneone)
port += 5;
}
@@ -122,7 +122,7 @@ testLongUrlGet ()
&ahc_echo,
"GET",
MHD_OPTION_CONNECTION_MEMORY_LIMIT,
- (size_t) (VERY_LONG / 2), MHD_OPTION_END);
+ (size_t) buff_size, MHD_OPTION_END);
if (d == NULL)
return 1;
if (0 == port)
@@ -182,7 +182,7 @@ testLongUrlGet ()
static int
-testLongHeaderGet ()
+testLongHeaderGet (size_t buff_size)
{
struct MHD_Daemon *d;
CURL *c;
@@ -197,7 +197,7 @@ testLongHeaderGet ()
port = 0;
else
{
- port = 1331;
+ port = 1331 + buff_size % 20;
if (oneone)
port += 5;
}
@@ -212,7 +212,7 @@ testLongHeaderGet ()
&ahc_echo,
"GET",
MHD_OPTION_CONNECTION_MEMORY_LIMIT,
- (size_t) (VERY_LONG / 2), MHD_OPTION_END);
+ (size_t) buff_size, MHD_OPTION_END);
if (d == NULL)
return 16;
if (0 == port)
@@ -288,8 +288,10 @@ main (int argc, char *const *argv)
oneone = has_in_name (argv[0], "11");
if (0 != curl_global_init (CURL_GLOBAL_WIN32))
return 2;
- errorCount += testLongUrlGet ();
- errorCount += testLongHeaderGet ();
+ errorCount += testLongUrlGet (VERY_LONG / 2);
+ errorCount += testLongUrlGet (VERY_LONG / 2 + 978);
+ errorCount += testLongHeaderGet (VERY_LONG / 2);
+ errorCount += testLongHeaderGet (VERY_LONG / 2 + 1893);
if (errorCount != 0)
fprintf (stderr, "Error (code: %u)\n", errorCount);
curl_global_cleanup ();
--
To stop receiving notification emails like this one, please contact
address@hidden.
- [GNUnet-SVN] [libmicrohttpd] branch master updated (d12ed49f -> 55dc42e6),
gnunet <=
- [GNUnet-SVN] [libmicrohttpd] 04/10: memorypool: comments minor improvements, gnunet, 2019/06/17
- [GNUnet-SVN] [libmicrohttpd] 05/10: MHD_pool_reallocate(): never allocate additional buffer when shrinking, gnunet, 2019/06/17
- [GNUnet-SVN] [libmicrohttpd] 07/10: transmit_error_response(): discard read buffer to free space for error response, gnunet, 2019/06/17
- [GNUnet-SVN] [libmicrohttpd] 09/10: try_ready_chunked_body(): remove useless loop, use know free memory size instead of guessing, gnunet, 2019/06/17
- [GNUnet-SVN] [libmicrohttpd] 01/10: memorypool: fixed MHD_pool_reset(), gnunet, 2019/06/17
- [GNUnet-SVN] [libmicrohttpd] 03/10: memorypool: re-commit memory on W32 when resetting pool, gnunet, 2019/06/17
- [GNUnet-SVN] [libmicrohttpd] 06/10: test_long_header: use power of two for size to properly test allocations, test various buffer sizes, gnunet, 2019/06/17
- [GNUnet-SVN] [libmicrohttpd] 02/10: memorypool: round mmap allocation for page size, gnunet, 2019/06/17
- [GNUnet-SVN] [libmicrohttpd] 10/10: Optimised read buffer allocation, gnunet, 2019/06/17
- [GNUnet-SVN] [libmicrohttpd] 08/10: process_request_body(): do not move data if nothing was processed, gnunet, 2019/06/17