[Commit-gnuradio] r7708 - in gnuradio/branches/developers/eb/gcell-multi-q/src: apps lib

[eb]

Author: eb
Date: 2008-02-15 14:04:57 -0700 (Fri, 15 Feb 2008)
New Revision: 7708

Modified:
   gnuradio/branches/developers/eb/gcell-multi-q/src/apps/benchmark_nop.cc
   gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_job_manager.h
   gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_job_manager_impl.cc
   gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_job_manager_impl.h
   gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_mem_pool.cc
   gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_mem_pool.h
Log:
work-in-progress

Modified: 
gnuradio/branches/developers/eb/gcell-multi-q/src/apps/benchmark_nop.cc
===================================================================
--- gnuradio/branches/developers/eb/gcell-multi-q/src/apps/benchmark_nop.cc     
2008-02-15 18:45:10 UTC (rev 7707)
+++ gnuradio/branches/developers/eb/gcell-multi-q/src/apps/benchmark_nop.cc     
2008-02-15 21:04:57 UTC (rev 7708)
@@ -51,7 +51,7 @@
   gc_jm_options opts;
   opts.nspes = nspes;
   opts.gang_schedule = true;
-  opts.nqueues = nqueues;
+  // opts.nqueues = nqueues;
   gc_job_manager *mgr = gc_make_job_manager(&opts);
 
   // allocate and init all job descriptors

Modified: gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_job_manager.h
===================================================================
--- gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_job_manager.h      
2008-02-15 18:45:10 UTC (rev 7707)
+++ gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_job_manager.h      
2008-02-15 21:04:57 UTC (rev 7708)
@@ -42,11 +42,10 @@
   unsigned int nspes;              // how many SPEs shall we use? 0 -> all of 
them
   bool gang_schedule;              // shall we gang schedule?
   bool use_affinity;               // shall we try for affinity (FIXME not 
implmented)
-  unsigned int nqueues;                    // FIXME debugging
 
   gc_jm_options() :
     max_jobs(0), max_client_threads(0), nspes(0),
-    gang_schedule(true), use_affinity(false), nqueues(0)
+    gang_schedule(true), use_affinity(false)
   {
   }
 };

Modified: 
gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_job_manager_impl.cc
===================================================================
--- 
gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_job_manager_impl.cc    
    2008-02-15 18:45:10 UTC (rev 7707)
+++ 
gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_job_manager_impl.cc    
    2008-02-15 21:04:57 UTC (rev 7708)
@@ -42,6 +42,9 @@
 static const unsigned int MAX_TOTAL_INDIRECT_LENGTH = 16 * 1024;
 
 
+static const int ANY_NODE = -1;
+
+
 static bool          s_key_initialized = false;
 static pthread_key_t s_client_key;
 
@@ -113,7 +116,7 @@
 
 
 gc_job_manager_impl::gc_job_manager_impl(const gc_jm_options *options)
-  : d_numa_avail(false), d_nnodes(0), d_pinning(false),
+  : d_numa_avail(false), d_nnodes(0), d_nqueues(0), d_pinning(false),
     d_debug(0), d_spu_args(0),
     d_eh_cond(&d_eh_mutex), d_eh_thread(0), d_eh_state(EHS_INIT),
     d_shutdown_requested(false),
@@ -195,21 +198,73 @@
   }
 
   // init node-specific memory pools
-  for (int i = 0; i < d_nnodes; i++)
+  for (unsigned int i = 0; i < d_nnodes; i++)
     d_mem_pool[i].init(POOL_SIZE, i);
 
+  // ----------------------------------------------------------------
+  // Figure out how we're going to partition everything
+  //
+  // Things that enter into the physical paritioning scheme:
+  //
+  //  d_nnodes          -- the number of physical nodes on the machine
+  //  d_options.nspes   -- the number of spes that we are to use
+  //
+  // ----------------------------------------------------------------
 
+  // initialize the maps
+
+  memset(d_spe2node, 0, sizeof(d_spe2node));
+  memset(d_spe2queue, 0, sizeof(d_spe2queue));
+  memset(d_queue2node, 0, sizeof(d_queue2node));
+
+  d_pinning = (d_numa_avail
+              && d_nnodes > 1
+              && d_options.nspes >= d_nnodes
+              && d_options.nspes > 8);
+
+  if (!d_pinning){
+
+    // use a single queue
+    d_nqueues = 1;
+    d_queue2node[0] = 0;
+
+    // spes aren't bound to a node
+    for (unsigned int i = 0; i < d_options.nspes; i++){
+      d_spe2node[i] = ANY_NODE;
+      d_spe2queue[i] = 0;      // all spes use the 1 queue
+    }
+  }
+
+  else {       // we are pinning
+
+    d_nqueues = d_nnodes;    // use 1 queue per node
+    for (unsigned int i = 0; i < d_nqueues; i++)
+      d_queue2node[i] = i;
+
+    // split spes between nodes
+    unsigned int nspes_per_node = d_options.nspes / d_nnodes;
+    for (unsigned int i = 0; i < d_options.nspes; i++){
+      d_spe2node[i]  = i / nspes_per_node;
+      d_spe2queue[i] = i / nspes_per_node;
+    }
+  }
+
+  // sanity check
+
+  for (unsigned int i = 0; i < d_options.nspes; i++)
+    assert((d_spe2node[i] == ANY_NODE && d_queue2node[d_spe2queue[i]] == 0)
+          || (d_spe2node[i] == d_queue2node[d_spe2queue[i]]));
+
+
   // ----------------------------------------------------------------
   // initalize the job queues
   
-  if (d_options.nqueues == 0)          // work out better default
-    d_options.nqueues = 1;
-  if (d_options.nqueues > d_options.nspes)
-    d_options.nqueues = d_options.nspes;
+  for (unsigned int i = 0; i < d_nqueues; i++){
+    int node = d_queue2node[i];
+    if (node == ANY_NODE)
+      node = 0;
 
-  for (unsigned int i = 0; i < d_options.nqueues; i++){
-    d_queue[i] = (gc_jd_queue_t *) aligned_alloc(sizeof(gc_jd_queue_t));
-    _d_queue_boost.push_back(void_sptr((void *) d_queue[i], free_deleter()));
+    d_queue[i] = (gc_jd_queue_t *) 
d_mem_pool[node].alloc(sizeof(gc_jd_queue_t));
     gc_jd_queue_init(d_queue[i]);
   }
 
@@ -224,6 +279,7 @@
   _d_spu_args_boost =
     boost::shared_ptr<void>((void *) d_spu_args, free_deleter());
 
+  // FIXME use node-specific allocator?
   // 2 completion info structs for each SPE (we double buffer them)
   assert(sizeof(gc_comp_info_t) % CACHE_LINE_SIZE == 0);
   d_comp_info =
@@ -258,13 +314,18 @@
       throw std::runtime_error("spe_context_create");
     }
     d_worker[i].spe_idx = i;
-    d_worker[i].node = 0;                                      // FIXME assign 
node
+    d_worker[i].node = d_spe2node[i];
     d_worker[i].spu_args = &d_spu_args[i];
-    d_worker[i].spu_args->queue = ptr_to_ea(d_queue[i % d_options.nqueues]);   
// FIXME which queue
+
+    int qidx = d_spe2queue[i];
+    printf("d_worker[%d] qidx = %d\n", i, qidx);
+    printf("d_queue[qidx] = %p\n", d_queue[qidx]);
+
+    d_worker[i].spu_args->queue = ptr_to_ea(d_queue[d_spe2queue[i]]);
     d_worker[i].spu_args->comp_info[0] = ptr_to_ea(&d_comp_info[2*i+0]);
     d_worker[i].spu_args->comp_info[1] = ptr_to_ea(&d_comp_info[2*i+1]);
     d_worker[i].spu_args->spu_idx = i;
-    d_worker[i].spu_args->nspus = d_options.nspes / d_options.nqueues;         
// scale nspes
+    d_worker[i].spu_args->nspus = d_options.nspes / d_nqueues;         // 
scale nspes
     d_worker[i].state = WS_INIT;
 
     int r = spe_program_load(d_worker[i].spe_ctx, spe_image);
@@ -1043,6 +1104,13 @@
   worker_ctx *w = (worker_ctx *) arg;
   spe_stop_info_t      si;
 
+  if (w->node != ANY_NODE){    // bind worker to node's cpu(s) and memory
+    nodemask_t mask;
+    nodemask_zero(&mask);
+    nodemask_set(&mask, w->node);
+    numa_bind(&mask);
+  }
+
   w->state = WS_RUNNING;
   if (s_worker_debug)
     fprintf(stderr, "worker[%d]: WS_RUNNING\n", w->spe_idx);

Modified: 
gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_job_manager_impl.h
===================================================================
--- gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_job_manager_impl.h 
2008-02-15 18:45:10 UTC (rev 7707)
+++ gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_job_manager_impl.h 
2008-02-15 21:04:57 UTC (rev 7708)
@@ -50,7 +50,7 @@
   volatile worker_state        state;
   unsigned int         spe_idx;        // [0, nspes-1]
   spe_context_ptr_t    spe_ctx;
-  unsigned int         node;           // which NUMA node we're bound to
+  int                  node;           // which NUMA node we're bound to
   pthread_t            thread;
   gc_spu_args_t                *spu_args;      // pointer to 16-byte aligned 
struct
 
@@ -90,17 +90,22 @@
 class gc_job_manager_impl : public gc_job_manager
 {
   enum {
-    MAX_NODES = 4,             // number of NUMA nodes.  QS20 and QS21 have 2.
+    MAX_NODES = 2,             // number of NUMA nodes.  QS20 and QS21 have 2.
     MAX_SPES =  16,
     MAX_QUEUES = MAX_SPES,
     POOL_SIZE = 4096,
   };
 
   bool                   d_numa_avail;
-  int                    d_nnodes;             // number of nodes
-  bool                   d_pinning;            // are we pinning SPEs & queues 
to nodes?
-  gc_mem_pool            d_mem_pool[MAX_NODES];
+  unsigned int           d_nnodes;                 // number of NUMA nodes
+  unsigned int           d_nqueues;                // number of queues we're 
using
+  bool                   d_pinning;                // are we pinning SPEs & 
queues to nodes?
+  gc_mem_pool            d_mem_pool[MAX_NODES];    // node-specific memory 
allocators
 
+  int                    d_spe2node[MAX_SPES];     // spe_idx -> node
+  int                    d_spe2queue[MAX_SPES];    // spe_idx -> queue_idx
+  int                    d_queue2node[MAX_QUEUES]; // queue_idx -> node
+
   int                    d_debug;
   gc_jm_options                  d_options;
   spe_program_handle_sptr d_spe_image;
@@ -166,7 +171,7 @@
   void handle_event(spe_event_unit_t *evt);
 
   unsigned int inc_nextq(unsigned int nq){
-    return nq >= d_options.nqueues - 1 ? 0 : nq + 1;
+    return nq >= d_nqueues - 1 ? 0 : nq + 1;
   }
 
   // bitvector ops

Modified: gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_mem_pool.cc
===================================================================
--- gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_mem_pool.cc        
2008-02-15 18:45:10 UTC (rev 7707)
+++ gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_mem_pool.cc        
2008-02-15 21:04:57 UTC (rev 7708)
@@ -29,6 +29,7 @@
 #include <new>
 #include <stdio.h>
 #include <errno.h>
+#include <stdint.h>
 
 
 #define ROUNDUP(x, align) (((x) + (align)-1) & ~((align)-1))
@@ -72,5 +73,14 @@
 void *
 gc_mem_pool::alloc(size_t size, size_t alignment)
 {
-  return 0;    // FIXME
+  uintptr_t t = (uintptr_t) d_avail;
+  t = ROUNDUP(t, alignment);
+  void *p = (void *) t;
+  
+  t += size;
+  if (t > (uintptr_t) d_base + d_pool_size)
+    throw std::bad_alloc();
+
+  d_avail = (void *) t;
+  return p;
 }

Modified: gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_mem_pool.h
===================================================================
--- gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_mem_pool.h 
2008-02-15 18:45:10 UTC (rev 7707)
+++ gnuradio/branches/developers/eb/gcell-multi-q/src/lib/gc_mem_pool.h 
2008-02-15 21:04:57 UTC (rev 7708)
@@ -50,7 +50,7 @@
    * There is no "free".  All the memory in the pool is reclaimed when
    * the destructor is executed.
    */
-  void *alloc(size_t size, size_t alignment);
+  void *alloc(size_t size, size_t alignment = 128);
   
 };