[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[Commit-gnuradio] r7705 - gnuradio/branches/developers/eb/gcell-multi-q/
|
From: |
eb |
|
Subject: |
[Commit-gnuradio] r7705 - gnuradio/branches/developers/eb/gcell-multi-q/src/lib |
|
Date: |
Fri, 15 Feb 2008 11:23:17 -0700 (MST) |
Author: eb
Date: 2008-02-15 11:23:16 -0700 (Fri, 15 Feb 2008)
New Revision: 7705
Added:
gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_mem_pool.cc
gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_mem_pool.h
Modified:
gnuradio/branches/developers/eb/gcell-multi-q/src/lib/Makefile.am
gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_job_manager_impl.cc
gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_job_manager_impl.h
Log:
work-in-progress
Modified: gnuradio/branches/developers/eb/gcell-multi-q/src/lib/Makefile.am
===================================================================
--- gnuradio/branches/developers/eb/gcell-multi-q/src/lib/Makefile.am
2008-02-15 07:25:49 UTC (rev 7704)
+++ gnuradio/branches/developers/eb/gcell-multi-q/src/lib/Makefile.am
2008-02-15 18:23:16 UTC (rev 7705)
@@ -31,10 +31,12 @@
gc_job_manager.cc \
gc_job_manager_impl.cc \
gc_jd_queue.c \
- gc_jd_stack.c
+ gc_jd_stack.c \
+ gc_mem_pool.c
libgcell_la_LIBADD = \
-lspe2 \
+ -lnuma \
$(GR_OMNITHREAD_LIBS)
libgcell_qa_la_SOURCES = \
Modified:
gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_job_manager_impl.cc
===================================================================
---
gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_job_manager_impl.cc
2008-02-15 07:25:49 UTC (rev 7704)
+++
gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_job_manager_impl.cc
2008-02-15 18:23:16 UTC (rev 7705)
@@ -30,7 +30,9 @@
#include <stdlib.h>
#include <atomic_dec_if_positive.h>
#include <memory_barrier.h>
+#include <numa.h>
+
static const size_t CACHE_LINE_SIZE = 128;
static const unsigned int DEFAULT_MAX_JOBS = 128;
@@ -111,7 +113,8 @@
gc_job_manager_impl::gc_job_manager_impl(const gc_jm_options *options)
- : d_debug(0), d_spu_args(0),
+ : d_numa_avail(false), d_nnodes(0), d_pinning(false),
+ d_debug(0), d_spu_args(0),
d_eh_cond(&d_eh_mutex), d_eh_thread(0), d_eh_state(EHS_INIT),
d_shutdown_requested(false),
d_client_thread(0), d_nextq(0)
@@ -135,19 +138,24 @@
if (d_options.max_client_threads == 0)
d_options.max_client_threads = DEFAULT_MAX_CLIENT_THREADS;
+ d_numa_avail = numa_available() != -1;
+ fprintf(stderr,"numa_avail = %d\n", d_numa_avail);
+
int ncpu_nodes = spe_cpu_info_get(SPE_COUNT_PHYSICAL_CPU_NODES, -1);
int nusable_spes = spe_cpu_info_get(SPE_COUNT_USABLE_SPES, -1);
if (debug()){
- printf("cpu_nodes = %d\n", ncpu_nodes);
+ fprintf(stderr,"cpu_nodes = %d\n", ncpu_nodes);
for (int i = 0; i < ncpu_nodes; i++){
- printf("node[%d].physical_spes = %2d\n", i,
+ fprintf(stderr,"node[%d].physical_spes = %2d\n", i,
spe_cpu_info_get(SPE_COUNT_PHYSICAL_SPES, i));
- printf("node[%d].usable_spes = %2d\n", i,
+ fprintf(stderr,"node[%d].usable_spes = %2d\n", i,
spe_cpu_info_get(SPE_COUNT_USABLE_SPES, i));
}
}
+ d_nnodes = std::min(ncpu_nodes, (int) MAX_NODES);
+
// clamp nspes
d_options.nspes = std::min(d_options.nspes, (unsigned int) MAX_SPES);
nusable_spes = std::min(nusable_spes, (int) MAX_SPES);
@@ -175,7 +183,7 @@
}
if (d_options.use_affinity){
- printf("gc_job_manager: warning: affinity request was ignored\n");
+ fprintf(stderr,"gc_job_manager: warning: affinity request was ignored\n");
}
if (d_options.gang_schedule){
@@ -186,13 +194,17 @@
}
}
+ // init node-specific memory pools
+ for (int i = 0; i < d_nnodes; i++)
+ d_mem_pool[i].init(POOL_SIZE, i);
+
// ----------------------------------------------------------------
// initalize the job queues
if (d_options.nqueues == 0) // work out better default
d_options.nqueues = 1;
- if ((int) d_options.nqueues > d_options.nspes)
+ if (d_options.nqueues > d_options.nspes)
d_options.nqueues = d_options.nspes;
for (unsigned int i = 0; i < d_options.nqueues; i++){
@@ -274,8 +286,8 @@
gc_jd_stack_init(d_free_list);
if (debug()){
- printf("sizeof(d_jd[0]) = %d (0x%x)\n", sizeof(d_jd[0]), sizeof(d_jd[0]));
- printf("max_jobs = %u\n", d_options.max_jobs);
+ fprintf(stderr,"sizeof(d_jd[0]) = %d (0x%x)\n", sizeof(d_jd[0]),
sizeof(d_jd[0]));
+ fprintf(stderr,"max_jobs = %u\n", d_options.max_jobs);
}
// Initialize the array of job descriptors.
@@ -505,7 +517,7 @@
// There is a race here in the modification of d_nextq.
// However, I think that fixing it will cost more than it's worth.
- // Worst case, some queue gets more or less jobs assigned to it than
+ // Worst case, some queue gets more or fewer jobs assigned to it than
// it should.
unsigned nq = d_nextq;
@@ -739,21 +751,21 @@
void
gc_job_manager_impl::print_event(spe_event_unit_t *evt)
{
- printf("evt: spe = %d events = (0x%x)", evt->data.u32, evt->events);
+ fprintf(stderr,"evt: spe = %d events = (0x%x)", evt->data.u32, evt->events);
if (evt->events & SPE_EVENT_OUT_INTR_MBOX)
- printf(" OUT_INTR_MBOX");
+ fprintf(stderr," OUT_INTR_MBOX");
if (evt->events & SPE_EVENT_IN_MBOX)
- printf(" IN_MBOX");
+ fprintf(stderr," IN_MBOX");
if (evt->events & SPE_EVENT_TAG_GROUP)
- printf(" TAG_GROUP");
+ fprintf(stderr," TAG_GROUP");
if (evt->events & SPE_EVENT_SPE_STOPPED)
- printf(" SPE_STOPPED");
+ fprintf(stderr," SPE_STOPPED");
- printf("\n");
+ fprintf(stderr,"\n");
}
struct job_client_info {
@@ -790,7 +802,7 @@
static int total_msgs;
total_msgs++;
total_jobs += ci->ncomplete;
- printf("ppe: tj = %6d tm = %6d\n", total_jobs, total_msgs);
+ fprintf(stderr,"ppe: tj = %6d tm = %6d\n", total_jobs, total_msgs);
}
job_client_info gci[GC_CI_NJOBS];
@@ -875,7 +887,7 @@
static const int NMSGS = 32;
unsigned int msg[NMSGS];
int n = spe_out_intr_mbox_read(evt->spe, msg, NMSGS,
SPE_MBOX_ANY_BLOCKING);
- // printf("spe_out_intr_mbox_read = %d\n", n);
+ // fprintf(stderr,"spe_out_intr_mbox_read = %d\n", n);
if (n < 0){
perror("spe_out_intr_mbox_read");
}
@@ -884,13 +896,13 @@
switch(MBOX_MSG_OP(msg[i])){
case OP_JOBS_DONE:
if (debug())
- printf("eh: job_done (0x%08x) from spu[%d]\n", msg[i], spe_num);
+ fprintf(stderr,"eh: job_done (0x%08x) from spu[%d]\n", msg[i],
spe_num);
notify_clients_jobs_are_done(spe_num, MBOX_MSG_ARG(msg[i]));
break;
case OP_EXIT:
default:
- printf("eh: Unexpected msg (0x%08x) from spu[%d]\n", msg[i], spe_num);
+ fprintf(stderr,"eh: Unexpected msg (0x%08x) from spu[%d]\n", msg[i],
spe_num);
break;
}
}
@@ -906,36 +918,36 @@
switch (si.stop_reason){
case SPE_EXIT:
if (debug()){
- printf("eh: spu[%d] SPE_EXIT w/ exit_code = %d\n",
+ fprintf(stderr,"eh: spu[%d] SPE_EXIT w/ exit_code = %d\n",
spe_num, si.result.spe_exit_code);
}
break;
case SPE_STOP_AND_SIGNAL:
- printf("eh: spu[%d] SPE_STOP_AND_SIGNAL w/ spe_signal_code = 0x%x\n",
+ fprintf(stderr,"eh: spu[%d] SPE_STOP_AND_SIGNAL w/ spe_signal_code =
0x%x\n",
spe_num, si.result.spe_signal_code);
break;
case SPE_RUNTIME_ERROR:
- printf("eh: spu[%d] SPE_RUNTIME_ERROR w/ spe_runtime_error = 0x%x\n",
+ fprintf(stderr,"eh: spu[%d] SPE_RUNTIME_ERROR w/ spe_runtime_error =
0x%x\n",
spe_num, si.result.spe_runtime_error);
break;
case SPE_RUNTIME_EXCEPTION:
- printf("eh: spu[%d] SPE_RUNTIME_EXCEPTION w/ spe_runtime_exception =
0x%x\n",
+ fprintf(stderr,"eh: spu[%d] SPE_RUNTIME_EXCEPTION w/
spe_runtime_exception = 0x%x\n",
spe_num, si.result.spe_runtime_exception);
break;
case SPE_RUNTIME_FATAL:
- printf("eh: spu[%d] SPE_RUNTIME_FATAL w/ spe_runtime_fatal = 0x%x\n",
+ fprintf(stderr,"eh: spu[%d] SPE_RUNTIME_FATAL w/ spe_runtime_fatal =
0x%x\n",
spe_num, si.result.spe_runtime_fatal);
break;
case SPE_CALLBACK_ERROR:
- printf("eh: spu[%d] SPE_CALLBACK_ERROR w/ spe_callback_error = 0x%x\n",
+ fprintf(stderr,"eh: spu[%d] SPE_CALLBACK_ERROR w/ spe_callback_error =
0x%x\n",
spe_num, si.result.spe_callback_error);
break;
case SPE_ISOLATION_ERROR:
- printf("eh: spu[%d] SPE_ISOLATION_ERROR w/ spe_isolation_error =
0x%x\n",
+ fprintf(stderr,"eh: spu[%d] SPE_ISOLATION_ERROR w/ spe_isolation_error
= 0x%x\n",
spe_num, si.result.spe_isolation_error);
break;
default:
- printf("eh: spu[%d] UNKNOWN STOP REASON (%d) w/ spu_status = 0x%x\n",
+ fprintf(stderr,"eh: spu[%d] UNKNOWN STOP REASON (%d) w/ spu_status =
0x%x\n",
spe_num, si.stop_reason, si.spu_status);
break;
}
@@ -967,7 +979,7 @@
spe_event_unit_t events[MAX_EVENTS];
if (d_debug)
- printf("event_handler_loop: starting\n");
+ fprintf(stderr, "event_handler_loop: starting\n");
set_eh_state(EHS_RUNNING);
@@ -997,7 +1009,7 @@
if (all_dead){
set_eh_state(EHS_DEAD);
if (d_debug)
- printf("event_handler_loop: exiting\n");
+ fprintf(stderr, "event_handler_loop: exiting\n");
return;
}
}
@@ -1005,7 +1017,7 @@
default:
set_eh_state(EHS_DEAD);
- printf("event_handler_loop(default): exiting\n");
+ fprintf(stderr, "event_handler_loop(default): exiting\n");
return;
}
@@ -1033,7 +1045,7 @@
w->state = WS_RUNNING;
if (s_worker_debug)
- printf("worker[%d]: WS_RUNNING\n", w->spe_idx);
+ fprintf(stderr, "worker[%d]: WS_RUNNING\n", w->spe_idx);
unsigned int entry = SPE_DEFAULT_ENTRY;
int r = spe_context_run(w->spe_ctx, &entry, 0, w->spu_args, 0, &si);
@@ -1046,7 +1058,7 @@
else if (r == 0){
// spe program called exit.
if (s_worker_debug)
- printf("worker[%d]: SPE_EXIT w/ exit_code = %d\n",
+ fprintf(stderr, "worker[%d]: SPE_EXIT w/ exit_code = %d\n",
w->spe_idx, si.result.spe_exit_code);
}
else {
@@ -1054,13 +1066,13 @@
//
// I'm not sure we'll ever get here. I think the event
// handler will catch this...
- printf("worker[%d]: SPE_STOP_AND_SIGNAL w/ spe_signal_code = 0x%x\n",
+ fprintf(stderr, "worker[%d]: SPE_STOP_AND_SIGNAL w/ spe_signal_code =
0x%x\n",
w->spe_idx, si.result.spe_signal_code);
}
// in any event, we're committing suicide now ;)
if (s_worker_debug)
- printf("worker[%d]: WS_DEAD\n", w->spe_idx);
+ fprintf(stderr, "worker[%d]: WS_DEAD\n", w->spe_idx);
w->state = WS_DEAD;
return 0;
Modified:
gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_job_manager_impl.h
===================================================================
--- gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_job_manager_impl.h
2008-02-15 07:25:49 UTC (rev 7704)
+++ gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_job_manager_impl.h
2008-02-15 18:23:16 UTC (rev 7705)
@@ -27,6 +27,7 @@
#include "gc_jd_stack.h"
#include "gc_jd_queue.h"
#include "gc_spu_args.h"
+#include "gc_mem_pool.h"
#include <libspe2.h>
#include <vector>
#include <boost/shared_ptr.hpp>
@@ -88,9 +89,18 @@
*/
class gc_job_manager_impl : public gc_job_manager
{
- enum { MAX_SPES = 16 };
- enum { MAX_QUEUES = MAX_SPES };
+ enum {
+ MAX_NODES = 4, // number of NUMA nodes. QS20 and QS21 have 2.
+ MAX_SPES = 16,
+ MAX_QUEUES = MAX_SPES,
+ POOL_SIZE = 4096,
+ };
+ bool d_numa_avail;
+ int d_nnodes; // number of nodes
+ bool d_pinning; // are we pinning SPEs & queues
to nodes?
+ gc_mem_pool d_mem_pool[MAX_NODES];
+
int d_debug;
gc_jm_options d_options;
spe_program_handle_sptr d_spe_image;
Added: gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_mem_pool.cc
===================================================================
--- gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_mem_pool.cc
(rev 0)
+++ gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_mem_pool.cc
2008-02-15 18:23:16 UTC (rev 7705)
@@ -0,0 +1,76 @@
+/* -*- c++ -*- */
+/*
+ * Copyright 2008 Free Software Foundation, Inc.
+ *
+ * This file is part of GNU Radio
+ *
+ * GNU Radio is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3, or (at your option)
+ * any later version.
+ *
+ * GNU Radio is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+#include <gc_mem_pool.h>
+#include <sys/mman.h>
+#include <numaif.h>
+#include <unistd.h>
+#include <new>
+#include <stdio.h>
+#include <errno.h>
+
+
+#define ROUNDUP(x, align) (((x) + (align)-1) & ~((align)-1))
+
+gc_mem_pool::~gc_mem_pool()
+{
+ if (d_base)
+ munmap(d_base, d_pool_size);
+}
+
+void
+gc_mem_pool::init(size_t pool_size, int node)
+{
+ size_t page_size = getpagesize();
+ pool_size = ROUNDUP(pool_size, page_size);
+
+ // allocate an anonymous piece of memory
+
+ void *p = mmap(NULL, pool_size,
+ PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+
+ if (p == MAP_FAILED){
+ perror("gc_mem_pool: mmap(MAP_ANONYMOUS)");
+ throw std::bad_alloc();
+ }
+
+ d_base = p;
+ d_avail = p;
+ d_pool_size = pool_size;
+
+ // now mbind it before anybody touches it
+
+ unsigned long nodemask = (1UL << node);
+ if (mbind(d_base, d_pool_size, MPOL_BIND, &nodemask, sizeof(nodemask) * 8,
0) == -1){
+ if (1 || errno != ENOSYS) // not implemented (on PS3)
+ perror("gc_mem_pool: mbind");
+ }
+}
+
+void *
+gc_mem_pool::alloc(size_t size, size_t alignment)
+{
+ return 0; // FIXME
+}
Property changes on:
gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_mem_pool.cc
___________________________________________________________________
Name: svn:eol-style
+ native
Added: gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_mem_pool.h
===================================================================
--- gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_mem_pool.h
(rev 0)
+++ gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_mem_pool.h
2008-02-15 18:23:16 UTC (rev 7705)
@@ -0,0 +1,57 @@
+/* -*- c++ -*- */
+/*
+ * Copyright 2008 Free Software Foundation, Inc.
+ *
+ * This file is part of GNU Radio
+ *
+ * GNU Radio is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3, or (at your option)
+ * any later version.
+ *
+ * GNU Radio is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+#ifndef INCLUDED_GC_MEM_POOL_H
+#define INCLUDED_GC_MEM_POOL_H
+
+#include <stddef.h>
+
+/*!
+ * \brief Very simple NUMA-aware memory pool.
+ */
+class gc_mem_pool
+{
+ void *d_base; // base of pool
+ void *d_avail; // allocation pointer
+ size_t d_pool_size; // total size of pool in bytes
+
+public:
+ gc_mem_pool() : d_base(0), d_avail(0), d_pool_size(0){}
+ ~gc_mem_pool();
+
+ /*!
+ * \param pool_size is the total size of the pool in bytes
+ * \param node specifies which NUMA node the pool should be allocated on.
+ */
+ void init(size_t pool_size, int node);
+
+ /*!
+ * \param size is the number of bytes to allocate
+ * \param alignment is the required alignment (must be a power of 2)
+ * \returns a pointer to zero-initialized memory or throws std::bad_alloc
+ *
+ * There is no "free". All the memory in the pool is reclaimed when
+ * the destructor is executed.
+ */
+ void *alloc(size_t size, size_t alignment);
+
+};
+
+#endif /* INCLUDED_GC_MEM_POOL_H */
Property changes on:
gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_mem_pool.h
___________________________________________________________________
Name: svn:eol-style
+ native
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- [Commit-gnuradio] r7705 - gnuradio/branches/developers/eb/gcell-multi-q/src/lib,
eb <=