Original Post
I'm working on an imaging library, and upon profiling, one of the performance bottle necks (more so in the application using the lib. than the lib. itself) is copying images (for many bad, bad reasons, but I digress). Anyway, I'm developing for specifically Windows in VS2008, and we need to be compatible for 64 bit compiles (so no inline assembly)... so given those contstraints here is the memcpy I currently use. Under non-optimal circumstances it defaults to Microsoft's memcpy, which we all know has been optimized to death. I hope that this can either be useful to someone or someone can give me some constructive feedback. For what it's worth, under optimal circumstances, this performs at about 62% of the time that memcpy does, and that is making a big difference for me at the moment (of course we are working with massive amounts of data though).
// If the memory source and destination addresses line up on a 16 byte boundary, or are both equally
// unaligned on the 16 bytes boundary, we can do a nice super fast memcpy that even beats
// Microsoft's heavily UV optimized memcpy... this is a highly special case though, which is
// very useful if we can force it to happen :)
__forceinline void ImageMemCopy(void *Dest, const void *Src, UInt Size) {
if (!__sse2_available || Size < 16 || ((UInt)Src & 15) != ((UInt)Dest & 15)) {
memcpy(Dest, Src, Size);
}
else {
char *pSrc = (char *)Src, *pDest = (char *)Dest;
UInt32 Count = (16 - ((UInt)pSrc & 15)) & 15;
for (UInt Index = Count; Index > 0; --Index) {
*pDest++ = *pSrc++;
}
Size -= Count;
__m128i Reg0, Reg1, Reg2, Reg3, Reg4, Reg5, Reg6, Reg7;
for (UInt Index = Size >> 7; Index > 0; --Index) {
_mm_prefetch(pSrc + 256, _MM_HINT_NTA);
_mm_prefetch(pSrc + 256 + 64, _MM_HINT_NTA);
Reg0 = _mm_load_si128((__m128i *)(pSrc));
Reg1 = _mm_load_si128((__m128i *)(pSrc + 16));
Reg2 = _mm_load_si128((__m128i *)(pSrc + 32));
Reg3 = _mm_load_si128((__m128i *)(pSrc + 48));
Reg4 = _mm_load_si128((__m128i *)(pSrc + 64));
Reg5 = _mm_load_si128((__m128i *)(pSrc + 80));
Reg6 = _mm_load_si128((__m128i *)(pSrc + 96));
Reg7 = _mm_load_si128((__m128i *)(pSrc + 112));
_mm_stream_si128((__m128i *)(pDest), Reg0);
_mm_stream_si128((__m128i *)(pDest + 16), Reg1);
_mm_stream_si128((__m128i *)(pDest + 32), Reg2);
_mm_stream_si128((__m128i *)(pDest + 48), Reg3);
_mm_stream_si128((__m128i *)(pDest + 64), Reg4);
_mm_stream_si128((__m128i *)(pDest + 80), Reg5);
_mm_stream_si128((__m128i *)(pDest + 96), Reg6);
_mm_stream_si128((__m128i *)(pDest + 112), Reg7);
pSrc += 128;
pDest += 128;
}
const UInt SizeOn16 = Size & 0x70;
switch (SizeOn16 >> 4) {
case 7: Reg7 = _mm_load_si128((__m128i *)(pSrc + 96));
case 6: Reg6 = _mm_load_si128((__m128i *)(pSrc + 80));
case 5: Reg5 = _mm_load_si128((__m128i *)(pSrc + 64));
case 4: Reg4 = _mm_load_si128((__m128i *)(pSrc + 48));
case 3: Reg3 = _mm_load_si128((__m128i *)(pSrc + 32));
case 2: Reg2 = _mm_load_si128((__m128i *)(pSrc + 16));
case 1: Reg1 = _mm_load_si128((__m128i *)(pSrc));
}
switch (SizeOn16 >> 4) {
case 7: _mm_stream_si128((__m128i *)(pDest + 96), Reg7);
case 6: _mm_stream_si128((__m128i *)(pDest + 80), Reg6);
case 5: _mm_stream_si128((__m128i *)(pDest + 64), Reg5);
case 4: _mm_stream_si128((__m128i *)(pDest + 48), Reg4);
case 3: _mm_stream_si128((__m128i *)(pDest + 32), Reg3);
case 2: _mm_stream_si128((__m128i *)(pDest + 16), Reg2);
case 1: _mm_stream_si128((__m128i *)(pDest), Reg1);
}
pSrc += SizeOn16;
pDest += SizeOn16;
for (UInt Index = Size & 15; Index > 0; --Index) {
*pDest++ = *pSrc++;
}
}
}