3 * Copyright 2007,2008 Free Software Foundation, Inc.
5 * This file is part of GNU Radio
7 * GNU Radio is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 3, or (at your option)
12 * GNU Radio is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License along
18 * with this program; if not, write to the Free Software Foundation, Inc.,
19 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
22 // #define ENABLE_GC_LOGGING // define to enable logging
24 #include <spu_intrinsics.h>
25 #include <spu_mfcio.h>
26 #include <sync_utils.h>
27 #include "gc_spu_config.h"
28 #include "spu_buffers.h"
29 #include <gcell/gc_spu_args.h>
30 #include <gcell/gc_job_desc.h>
31 #include <gcell/gc_mbox.h>
32 #include <gcell/gc_declare_proc.h>
33 #include <gcell/spu/gc_jd_queue.h>
34 #include <gcell/spu/gc_random.h>
35 #include <gcell/spu/gc_delay.h>
42 #define MIN(a,b) ((a) < (b) ? (a) : (b))
43 #define MAX(a,b) ((a) > (b) ? (a) : (b))
45 //! round x down to p2 boundary (p2 must be a power-of-2)
46 #define ROUND_DN(x, p2) ((x) & ~((p2)-1))
48 //! round x up to p2 boundary (p2 must be a power-of-2)
49 #define ROUND_UP(x, p2) (((x)+((p2)-1)) & ~((p2)-1))
52 #define USE_LLR_LOST_EVENT 0 // define to 0 or 1
54 int gc_sys_tag; // tag for misc DMA operations
55 static gc_spu_args_t spu_args;
57 static struct gc_proc_def *gc_proc_def; // procedure entry points
59 // ------------------------------------------------------------------------
61 // state for DMA'ing arguments in and out
63 static int get_tag; // 1 tag for job arg gets
64 static int put_tags; // 2 tags for job arg puts
66 static int pb_idx = 0; // current put buffer index (0 or 1)
68 // bitmask (bit per put buffer): bit is set if DMA is started but not complete
69 static int put_in_progress = 0;
70 #define PBI_MASK(_pbi_) (1 << (_pbi_))
72 // ------------------------------------------------------------------------
74 // our working copy of the completion info
75 static gc_comp_info_t comp_info = {
80 static int ci_idx = 0; // index of current comp_info
81 static int ci_tags; // two consecutive dma tags
83 // ------------------------------------------------------------------------
86 * Wait until EA copy of comp_info[idx].in_use is 0
89 wait_for_ppe_to_be_done_with_comp_info(int idx)
92 char *buf = (char *) ALIGN(_tmp, 128); // get cache-aligned buffer
93 gc_comp_info_t *p = (gc_comp_info_t *) buf;
95 assert(sizeof(gc_comp_info_t) == 128);
98 mfc_get(buf, spu_args.comp_info[idx], 128, gc_sys_tag, 0, 0);
99 mfc_write_tag_mask(1 << gc_sys_tag);
100 mfc_read_tag_status_all();
110 flush_completion_info(void)
114 static int total_complete = 0;
116 if (comp_info.ncomplete == 0)
119 // ensure that PPE is done with the buffer we're about to overwrite
120 wait_for_ppe_to_be_done_with_comp_info(ci_idx);
122 // dma the comp_info out to PPE
123 int tag = ci_tags + ci_idx;
124 mfc_put(&comp_info, spu_args.comp_info[ci_idx], sizeof(gc_comp_info_t), tag, 0, 0);
126 // we need to wait for the completion info to finish, as well as
127 // any EA argument puts.
129 int tag_mask = 1 << tag; // the comp_info tag
130 if (put_in_progress & PBI_MASK(0))
131 tag_mask |= (1 << (put_tags + 0));
132 if (put_in_progress & PBI_MASK(1))
133 tag_mask |= (1 << (put_tags + 1));
135 gc_log_write2(GCL_SS_SYS, 0x30, put_in_progress, tag_mask);
137 mfc_write_tag_mask(tag_mask); // the tags we're interested in
138 mfc_read_tag_status_all(); // wait for DMA to complete
139 put_in_progress = 0; // mark them all complete
141 total_complete += comp_info.ncomplete;
142 gc_log_write4(GCL_SS_SYS, 0x31,
143 put_in_progress, ci_idx, comp_info.ncomplete, total_complete);
145 // send PPE a message
146 spu_writech(SPU_WrOutIntrMbox, MK_MBOX_MSG(OP_JOBS_DONE, ci_idx));
148 ci_idx ^= 0x1; // switch buffers
149 comp_info.in_use = 1;
150 comp_info.ncomplete = 0;
153 // ------------------------------------------------------------------------
155 static unsigned int backoff; // current backoff value in clock cycles
156 static unsigned int _backoff_start;
157 static unsigned int _backoff_cap;
162 * 12 4095 cycles 1.3 us
163 * 13 8191 cycles 2.6 us
164 * 14 16383 cycles 5.1 us
165 * 15 32767 cycles 10.2 us
173 static unsigned char log2_backoff_start[16] = {
174 // 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
175 // -------------------------------------------------------------
176 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 16, 16
179 static unsigned char log2_backoff_cap[16] = {
180 // 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
181 // -------------------------------------------------------------
182 17, 17, 17, 18, 18, 18, 18, 19, 19, 19, 19, 20, 20, 20, 21, 21
188 _backoff_cap = (1 << (log2_backoff_cap[(spu_args.nspus - 1) & 0xf])) - 1;
189 _backoff_start = (1 << (log2_backoff_start[(spu_args.nspus - 1) & 0xf])) - 1;
191 backoff = _backoff_start;
197 backoff = _backoff_start;
207 // capped exponential backoff
208 backoff = ((backoff << 1) + 1) & _backoff_cap;
213 #define RANDOM_WEIGHT 0.2
220 backoff = ((backoff << 1) + 1);
221 if (backoff > _backoff_cap)
222 backoff = _backoff_cap;
224 float r = (RANDOM_WEIGHT * (2.0 * (gc_uniform_deviate() - 0.5)));
225 backoff = backoff * (1.0 + r);
230 // ------------------------------------------------------------------------
232 static inline unsigned int
235 return ~(~0 << nbits);
238 static unsigned int dc_work;
239 static int dc_put_tag;
240 static unsigned char *dc_ls_base;
241 static gc_eaddr_t dc_ea_base;
243 // divide and conquer
245 d_and_c(unsigned int offset, unsigned int len)
247 unsigned int mask = make_mask(len) << offset;
248 unsigned int t = mask & dc_work;
249 if (t == 0) // nothing to do
251 if (t == mask){ // got a match, generate dma
252 mfc_put(dc_ls_base + offset, dc_ea_base + offset, len, dc_put_tag, 0, 0);
256 d_and_c(offset, len);
257 d_and_c(offset + len, len);
261 // Handle the nasty case of a dma xfer that's less than 16 bytes long.
262 // len is guaranteed to be in [1, 15]
265 handle_slow_and_tedious_dma(gc_eaddr_t ea, unsigned char *ls,
266 unsigned int len, int put_tag)
268 // Set up for divide and conquer
269 unsigned int alignment = ((uintptr_t) ls) & 0x7;
270 dc_work = make_mask(len) << alignment;
271 dc_ls_base = (unsigned char *) ROUND_DN((uintptr_t) ls, 8);
272 dc_ea_base = ROUND_DN(ea, (gc_eaddr_t) 8);
273 dc_put_tag = put_tag;
282 process_job(gc_eaddr_t jd_ea, gc_job_desc_t *jd)
286 jd->status = JS_OK; // assume success
288 if (jd->proc_id >= spu_args.nproc_defs)
289 jd->status = JS_UNKNOWN_PROC;
293 if (jd->eaa.nargs == 0)
294 (*gc_proc_def[jd->proc_id].proc)(&jd->input, &jd->output, &jd->eaa);
296 else { // handle EA args that must be DMA'd in/out
298 gc_job_ea_args_t *eaa = &jd->eaa;
302 (GC_SPU_BUFSIZE + MFC_MAX_DMA_SIZE - 1) / MFC_MAX_DMA_SIZE);
304 mfc_list_element_t dma_get_list[NELMS];
305 //mfc_list_element_t dma_put_list[NELMS];
307 memset(dma_get_list, 0, sizeof(dma_get_list));
308 //memset(dma_put_list, 0, sizeof(dma_put_list));
310 int gli = 0; // get list index
311 //int pli = 0; // put list index
313 unsigned char *get_base = _gci_getbuf[0];
314 unsigned char *get_t = get_base;
315 unsigned int total_get_dma_len = 0;
317 unsigned char *put_base = _gci_putbuf[pb_idx];
318 unsigned char *put_t = put_base;
319 unsigned int total_put_alloc = 0;
320 int put_tag = put_tags + pb_idx;
322 // Do we have any "put" args? If so ensure that previous
323 // dma from this buffer is complete
325 gc_log_write2(GCL_SS_SYS, 0x24, put_in_progress, jd->sys.direction_union);
327 if ((jd->sys.direction_union & GCJD_DMA_PUT)
328 && (put_in_progress & PBI_MASK(pb_idx))){
330 gc_log_write2(GCL_SS_SYS, 0x25, put_in_progress, 1 << put_tag);
332 mfc_write_tag_mask(1 << put_tag); // the tag we're interested in
333 mfc_read_tag_status_all(); // wait for DMA to complete
334 put_in_progress &= ~(PBI_MASK(pb_idx));
336 gc_log_write1(GCL_SS_SYS, 0x26, put_in_progress);
340 // for now, all EA's must have the same high 32-bits
341 gc_eaddr_t common_ea = eaa->arg[0].ea_addr;
344 // assign LS addresses for buffers
346 for (unsigned int i = 0; i < eaa->nargs; i++){
348 gc_eaddr_t ea_base = 0;
349 unsigned char *ls_base;
351 unsigned int dma_len;
353 if (eaa->arg[i].direction == GCJD_DMA_GET){
354 ea_base = ROUND_DN(eaa->arg[i].ea_addr, (gc_eaddr_t) CACHE_LINE_SIZE);
355 offset = eaa->arg[i].ea_addr & (CACHE_LINE_SIZE-1);
356 dma_len = ROUND_UP(eaa->arg[i].get_size + offset, CACHE_LINE_SIZE);
357 total_get_dma_len += dma_len;
359 if (total_get_dma_len > GC_SPU_BUFSIZE){
360 jd->status = JS_ARGS_TOO_LONG;
366 eaa->arg[i].ls_addr = ls_base + offset;
369 assert((mfc_ea2l(eaa->arg[i].ea_addr) & 0x7f) == ((intptr_t)eaa->arg[i].ls_addr & 0x7f));
370 assert((ea_base & 0x7f) == 0);
371 assert(((intptr_t)ls_base & 0x7f) == 0);
372 assert((dma_len & 0x7f) == 0);
373 assert((eaa->arg[i].get_size <= dma_len)
374 && dma_len <= (eaa->arg[i].get_size + offset + CACHE_LINE_SIZE - 1));
377 // add to dma get list
378 // FIXME (someday) the dma list is where the JS_BAD_EAH limitation comes from
380 while (dma_len != 0){
381 int n = MIN(dma_len, MFC_MAX_DMA_SIZE);
382 dma_get_list[gli].size = n;
383 dma_get_list[gli].eal = mfc_ea2l(ea_base);
390 else if (eaa->arg[i].direction == GCJD_DMA_PUT){
392 // This case is a trickier than the PUT case since we can't
393 // write outside of the bounds of the user provided buffer.
394 // We still align the buffers to 128-bytes for good performance
395 // in the middle portion of the xfers.
397 ea_base = ROUND_DN(eaa->arg[i].ea_addr, (gc_eaddr_t) CACHE_LINE_SIZE);
398 offset = eaa->arg[i].ea_addr & (CACHE_LINE_SIZE-1);
400 uint32_t ls_alloc_len =
401 ROUND_UP(eaa->arg[i].put_size + offset, CACHE_LINE_SIZE);
403 total_put_alloc += ls_alloc_len;
405 if (total_put_alloc > GC_SPU_BUFSIZE){
406 jd->status = JS_ARGS_TOO_LONG;
411 put_t += ls_alloc_len;
412 eaa->arg[i].ls_addr = ls_base + offset;
415 assert((mfc_ea2l(eaa->arg[i].ea_addr) & 0x7f)
416 == ((intptr_t)eaa->arg[i].ls_addr & 0x7f));
417 assert((ea_base & 0x7f) == 0);
418 assert(((intptr_t)ls_base & 0x7f) == 0);
426 // fire off the dma to fetch the args and wait for it to complete
427 mfc_getl(get_base, common_ea, dma_get_list, gli*sizeof(dma_get_list[0]), get_tag, 0, 0);
428 mfc_write_tag_mask(1 << get_tag); // the tag we're interested in
429 mfc_read_tag_status_all(); // wait for DMA to complete
432 (*gc_proc_def[jd->proc_id].proc)(&jd->input, &jd->output, &jd->eaa);
435 // Do we have any "put" args? If so copy them out
436 if (jd->sys.direction_union & GCJD_DMA_PUT){
438 // Do the copy out using single DMA xfers. The LS ranges
439 // aren't generally contiguous.
441 bool started_dma = false;
443 for (unsigned int i = 0; i < eaa->nargs; i++){
444 if (eaa->arg[i].direction == GCJD_DMA_PUT && eaa->arg[i].put_size != 0){
452 ea = eaa->arg[i].ea_addr;
453 ls = (unsigned char *) eaa->arg[i].ls_addr;
454 len = eaa->arg[i].put_size;
457 handle_slow_and_tedious_dma(ea, ls, len, put_tag);
460 if ((ea & 0xf) != 0){
462 // printf("1: ea = 0x%x len = %5d\n", (int) ea, len);
464 // handle the "pre-multiple-of-16" portion
465 // do 1, 2, 4, or 8 byte xfers as required
467 if (ea & 0x1){ // do a 1-byte xfer
468 mfc_put(ls, ea, 1, put_tag, 0, 0);
473 if (ea & 0x2){ // do a 2-byte xfer
474 mfc_put(ls, ea, 2, put_tag, 0, 0);
479 if (ea & 0x4){ // do a 4-byte xfer
480 mfc_put(ls, ea, 4, put_tag, 0, 0);
485 if (ea & 0x8){ // do an 8-byte xfer
486 mfc_put(ls, ea, 8, put_tag, 0, 0);
494 // printf("2: ea = 0x%x len = %5d\n", (int) ea, len);
495 assert((ea & 0xf) == 0);
496 assert((((intptr_t) ls) & 0xf) == 0);
499 // handle the "multiple-of-16" portion
501 int aligned_len = ROUND_DN(len, 16);
502 len = len & (16 - 1);
504 while (aligned_len != 0){
505 int dma_len = MIN(aligned_len, MFC_MAX_DMA_SIZE);
506 mfc_put(ls, ea, dma_len, put_tag, 0, 0);
509 aligned_len -= dma_len;
513 // printf("3: ea = 0x%x len = %5d\n", (int)ea, len);
514 assert((ea & 0xf) == 0);
515 assert((((intptr_t) ls) & 0xf) == 0);
518 // handle "post-multiple-of-16" portion
522 if (len >= 8){ // do an 8-byte xfer
523 mfc_put(ls, ea, 8, put_tag, 0, 0);
528 if (len >= 4){ // do a 4-byte xfer
529 mfc_put(ls, ea, 4, put_tag, 0, 0);
534 if (len >= 2){ // do a 2-byte xfer
535 mfc_put(ls, ea, 2, put_tag, 0, 0);
540 if (len >= 1){ // do a 1-byte xfer
541 mfc_put(ls, ea, 1, put_tag, 0, 0);
553 put_in_progress |= PBI_MASK(pb_idx); // note it's running
554 gc_log_write2(GCL_SS_SYS, 0x27, put_in_progress, pb_idx);
555 pb_idx ^= 1; // toggle current buffer
561 wrap_up:; // semicolon creates null statement for C99 compliance
563 // Copy job descriptor back out to EA.
564 // (The dma will be waited on in flush_completion_info)
565 int tag = ci_tags + ci_idx; // use the current completion tag
566 mfc_put(jd, jd_ea, sizeof(*jd), tag, 0, 0);
568 // Tell PPE we're done with the job.
570 // We queue these up until we run out of room, or until we can send
571 // the info to the PPE w/o blocking. The blocking check is in
574 comp_info.job_id[comp_info.ncomplete++] = jd->sys.job_id;
576 if (comp_info.ncomplete == GC_CI_NJOBS){
577 gc_log_write0(GCL_SS_SYS, 0x28);
578 flush_completion_info();
587 static gc_job_desc_t jd; // static gets us proper alignment
591 #if (USE_LLR_LOST_EVENT)
593 spu_writech(SPU_WrEventMask, MFC_LLR_LOST_EVENT);
596 while (gc_jd_queue_dequeue(spu_args.queue, &jd_ea, ci_tags + ci_idx, &jd))
597 process_job(jd_ea, &jd);
598 // we're now holding a lock-line reservation
603 #if (USE_LLR_LOST_EVENT)
605 if (unlikely(spu_readchcnt(SPU_RdEventStat))){
607 // execute standard event handling prologue
609 int status = spu_readch(SPU_RdEventStat);
610 int mask = spu_readch(SPU_RdEventMask);
611 spu_writech(SPU_WrEventMask, mask & ~status); // disable active events
612 spu_writech(SPU_WrEventAck, status); // ack active events
614 // execute per-event actions
616 if (status & MFC_LLR_LOST_EVENT){
618 // We've lost a line reservation. This is most likely caused
619 // by somebody doing something to the queue. Go look and see
620 // if there's anything for us.
622 while (gc_jd_queue_dequeue(spu_args.queue, &jd_ea, ci_tags + ci_idx, &jd))
623 process_job(jd_ea, &jd);
627 // execute standard event handling epilogue
629 spu_writech(SPU_WrEventMask, mask); // restore event mask
634 // try to get a job from the job queue
635 if (gc_jd_queue_dequeue(spu_args.queue, &jd_ea, ci_tags + ci_idx, &jd)){
637 gc_log_write2(GCL_SS_SYS, 0x10, jd.sys.job_id, total_jobs);
639 process_job(jd_ea, &jd);
641 gc_log_write2(GCL_SS_SYS, 0x11, jd.sys.job_id, total_jobs);
651 if (unlikely(spu_readchcnt(SPU_RdInMbox))){
652 int msg = spu_readch(SPU_RdInMbox);
653 // printf("spu[%d] mbox_msg: 0x%08x\n", spu_args.spu_idx, msg);
654 if (MBOX_MSG_OP(msg) == OP_EXIT){
655 flush_completion_info();
658 if (MBOX_MSG_OP(msg) == OP_GET_SPU_BUFSIZE){
659 spu_writech(SPU_WrOutIntrMbox, MK_MBOX_MSG(OP_SPU_BUFSIZE, GC_SPU_BUFSIZE_BASE));
663 // If we've got job completion info for the PPE and we can send a
664 // message without blocking, do it.
666 if (comp_info.ncomplete != 0 && spu_readchcnt(SPU_WrOutIntrMbox) != 0){
667 gc_log_write0(GCL_SS_SYS, 0x12);
668 flush_completion_info();
675 main(unsigned long long spe_id __attribute__((unused)),
676 unsigned long long argp,
677 unsigned long long envp __attribute__((unused)))
679 gc_sys_tag = mfc_tag_reserve(); // allocate a tag for our misc DMA operations
680 get_tag = mfc_tag_reserve();
681 ci_tags = mfc_multi_tag_reserve(2);
682 put_tags = mfc_multi_tag_reserve(2);
685 printf("gc_sys_tag = %d\n", gc_sys_tag);
686 printf("get_tag = %d\n", get_tag);
687 printf("ci_tags = %d\n", ci_tags);
688 printf("put_tags = %d\n", put_tags);
692 mfc_get(&spu_args, argp, sizeof(spu_args), gc_sys_tag, 0, 0);
693 mfc_write_tag_mask(1 << gc_sys_tag); // the tag we're interested in
694 mfc_read_tag_status_all(); // wait for DMA to complete
696 // initialize pointer to procedure entry table
697 gc_proc_def = (gc_proc_def_t *) spu_args.proc_def_ls_addr;
699 gc_set_seed(spu_args.spu_idx);
701 // initialize logging
702 _gc_log_init(spu_args.log);
704 backoff_init(); // initialize backoff parameters