2 * Copyright (C) 2001-2003 Hewlett-Packard Co.
3 * Contributed by Stephane Eranian <eranian@hpl.hp.com>
5 * This file is part of the ELILO, the EFI Linux boot loader.
7 * ELILO is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2, or (at your option)
12 * ELILO is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with ELILO; see the file COPYING. If not, write to the Free
19 * Software Foundation, 59 Temple Place - Suite 330, Boston, MA
22 * Please check out the elilo.txt for complete documentation on how
23 * to use this program.
25 * This file is derived from the Linux/ia64 kernel source code
30 * Optimized version of the standard memcpy() function
33 * in0: destination address
35 * in2: number of bytes to copy
39 * Copyright (C) 2000 Hewlett-Packard Co
40 * Copyright (C) 2000 Stephane Eranian <eranian@hpl.hp.com>
41 * Copyright (C) 2000 David Mosberger-Tang <davidm@hpl.hp.com>
44 /* be pessimistic for now... */
45 #define CONFIG_ITANIUM_B0_SPECIFIC 1
47 #if defined(CONFIG_ITANIUM_B0_SPECIFIC) || defined(CONFIG_ITANIUM_B1_SPECIFIC)
48 # define BRP(args...) nop.b 0
50 # define BRP(args...) brp.loop.imp args
58 # define MEM_LAT 21 /* latency to memory */
75 # define N (MEM_LAT + 4)
76 # define Nrot ((N + 7) & ~7)
79 * First, check if everything (src, dst, len) is a multiple of eight. If
80 * so, we handle everything with no taken branches (other than the loop
81 * itself) and a small icache footprint. Otherwise, we jump off to
82 * the more general copy routine handling arbitrary
83 * sizes/alignment etc.
86 .save ar.pfs, saved_pfs
87 alloc saved_pfs=ar.pfs,3,Nrot,0,Nrot
99 cmp.eq p6,p0=in2,r0 // zero length?
100 mov retval=in0 // return dst
101 (p6) br.ret.spnt.many rp // zero length, return immediately
104 mov dst=in0 // copy because of rotation
105 shr.u cnt=in2,3 // number of 8-byte words to copy
109 adds cnt=-1,cnt // br.ctop is repeat/until
110 cmp.gtu p7,p0=16,in2 // copying less than 16 bytes?
119 mov src=in1 // copy because of rotation
120 (p7) br.cond.spnt.few memcpy_short
121 (p6) br.cond.spnt.few memcpy_long
134 (p[0]) ld8 val[0]=[src],8
139 (p[N-1])st8 [dst]=val[N-1],8
150 * Small (<16 bytes) unaligned copying is done via a simple byte-at-the-time
151 * copy loop. This performs relatively poorly on Itanium, but it doesn't
152 * get used very often (gcc inlines small copies) and due to atomicity
153 * issues, we want to avoid read-modify-write of entire words.
157 adds cnt=-1,in2 // br.ctop is repeat/until
173 * It is faster to put a stop bit in the loop here because it makes
174 * the pipeline shorter (and latency is what matters on short copies).
178 (p[0]) ld1 val[0]=[src],1
183 (p[MEM_LAT-1])st1 [dst]=val[MEM_LAT-1],1
193 * Large (>= 16 bytes) copying is done in a fancy way. Latency isn't
194 * an overriding concern here, but throughput is. We first do
195 * sub-word copying until the destination is aligned, then we check
196 * if the source is also aligned. If so, we do a simple load/store-loop
197 * until there are less than 8 bytes left over and then we do the tail,
198 * by storing the last few bytes using sub-word copying. If the source
199 * is not aligned, we branch off to the non-congruent loop.
207 * On Itanium, the pipeline itself runs without stalls. However, br.ctop
208 * seems to introduce an unavoidable bubble in the pipeline so the overall
209 * latency is 2 cycles/iteration. This gives us a _copy_ throughput
210 * of 4 byte/cycle. Still not bad.
214 # define N (MEM_LAT + 5) /* number of stages */
215 # define Nrot ((N+1 + 2 + 7) & ~7) /* number of rotating regs */
217 #define LOG_LOOP_SIZE 6
220 alloc t3=ar.pfs,3,Nrot,0,Nrot // resize register frame
221 and t0=-8,src // t0 = src & ~7
222 and t2=7,src // t2 = src & 7
224 ld8 t0=[t0] // t0 = 1st source word
225 adds src2=7,src // src2 = (src + 7)
226 sub t4=r0,dst // t4 = -dst
228 and src2=-8,src2 // src2 = (src + 7) & ~7
229 shl t2=t2,3 // t2 = 8*(src & 7)
230 shl t4=t4,3 // t4 = 8*(dst & 7)
232 ld8 t1=[src2] // t1 = 1st source word if src is 8-byte aligned, 2nd otherwise
233 sub t3=64,t2 // t3 = 64-8*(src & 7)
238 mov pr=t4,0x38 // (p5,p4,p3)=(dst & 7)
242 adds src_end=-1,src_end
254 and src_end=-8,src_end // src_end = last word of source buffer
257 // At this point, dst is aligned to 8 bytes and there at least 16-7=9 bytes left to copy:
259 1:{ add src=cnt,src // make src point to remainder of source buffer
260 sub cnt=in2,cnt // cnt = number of bytes left to copy
263 and src2=-8,src // align source pointer
264 adds t4=memcpy_loops-1b,t4
267 and t0=7,src // t0 = src & 7
268 shr.u t2=cnt,3 // t2 = number of 8-byte words left to copy
269 shl cnt=cnt,3 // move bits 0-2 to 3-5
275 cmp.ne p6,p0=t0,r0 // is src aligned, too?
276 shl t0=t0,LOG_LOOP_SIZE // t0 = 8*(src & 7)
277 adds t2=-1,t2 // br.ctop is repeat/until
280 mov pr=cnt,0x38 // set (p5,p4,p3) to # of bytes last-word bytes to copy
290 (p6) ld8 val[1]=[src2],8 // prime the pump...
296 // At this point, (p5,p4,p3) are set to the number of bytes left to copy (which is
297 // less than 8) and t0 contains the last few bytes of the src buffer:
310 ///////////////////////////////////////////////////////
313 #define COPY(shift,index) \
315 (p[0]) ld8 val[0]=[src2],8; \
316 (p[MEM_LAT+3]) shrp w[0]=val[MEM_LAT+3],val[MEM_LAT+4-index],shift; \
320 (p[MEM_LAT+4]) st8 [dst]=w[1],8; \
322 br.ctop.dptk.few 1b; \
325 ld8 val[N-1]=[src_end]; /* load last word (may be same as val[N]) */ \
327 shrp t0=val[N-1],val[N-index],shift; \
330 COPY(0, 1) /* no point special casing this---it doesn't go any faster without shrp */