code.delx.au - pulseaudio/blob - src/pulsecore/svolume_sse.c

   1 /***
   2   This file is part of PulseAudio.
   3
   4   Copyright 2004-2006 Lennart Poettering
   5   Copyright 2009 Wim Taymans <wim.taymans@collabora.co.uk>
   6
   7   PulseAudio is free software; you can redistribute it and/or modify
   8   it under the terms of the GNU Lesser General Public License as published
   9   by the Free Software Foundation; either version 2.1 of the License,
  10   or (at your option) any later version.
  11
  12   PulseAudio is distributed in the hope that it will be useful, but
  13   WITHOUT ANY WARRANTY; without even the implied warranty of
  14   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  15   General Public License for more details.
  16
  17   You should have received a copy of the GNU Lesser General Public License
  18   along with PulseAudio; if not, write to the Free Software
  19   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
  20   USA.
  21 ***/
  22
  23 #ifdef HAVE_CONFIG_H
  24 #include <config.h>
  25 #endif
  26
  27 #include <pulse/timeval.h>
  28 #include <pulsecore/random.h>
  29 #include <pulsecore/macro.h>
  30 #include <pulsecore/g711.h>
  31 #include <pulsecore/core-util.h>
  32
  33 #include "cpu-x86.h"
  34
  35 #include "sample-util.h"
  36 #include "endianmacros.h"
  37
  38 #define VOLUME_32x16(s,v)                  /* .. |   vh  |   vl  | */                   \
  39       " pxor %%xmm4, %%xmm4          \n\t" /* .. |    0  |    0  | */                   \
  40       " punpcklwd %%xmm4, "#s"       \n\t" /* .. |    0  |   p0  | */                   \
  41       " pcmpgtw "#s", %%xmm4         \n\t" /* .. |    0  | s(p0) | */                   \
  42       " pand "#v", %%xmm4            \n\t" /* .. |    0  |  (vl) | */                   \
  43       " movdqa "#s", %%xmm5          \n\t"                                              \
  44       " pmulhuw "#v", "#s"           \n\t" /* .. |    0  | vl*p0 | */                   \
  45       " psubd %%xmm4, "#s"           \n\t" /* .. |    0  | vl*p0 | + sign correct */    \
  46       " psrld $16, "#v"              \n\t" /* .. |   p0  |    0  | */                   \
  47       " pmaddwd %%xmm5, "#v"         \n\t" /* .. |    p0 * vh    | */                   \
  48       " paddd "#s", "#v"             \n\t" /* .. |    p0 * v0    | */                   \
  49       " packssdw "#v", "#v"          \n\t" /* .. | p1*v1 | p0*v0 | */
  50
  51 #define MOD_ADD(a,b) \
  52       " add "#a", %3                 \n\t" /* channel += inc           */ \
  53       " mov %3, %4                   \n\t"                                \
  54       " sub "#b", %4                 \n\t" /* tmp = channel - channels */ \
  55       " cmp "#b", %3                 \n\t" /* if (channel >= channels) */ \
  56       " cmovae %4, %3                \n\t" /*   channel = tmp          */
  57
  58 /* swap 16 bits */
  59 #define SWAP_16(s) \
  60       " movdqa "#s", %%xmm4          \n\t" /* .. |  h  l |  */ \
  61       " psrlw $8, %%xmm4             \n\t" /* .. |  0  h |  */ \
  62       " psllw $8, "#s"               \n\t" /* .. |  l  0 |  */ \
  63       " por %%xmm4, "#s"             \n\t" /* .. |  l  h |  */
  64
  65 /* swap 2 registers 16 bits for better pairing */
  66 #define SWAP_16_2(s1,s2) \
  67       " movdqa "#s1", %%xmm4         \n\t" /* .. |  h  l |  */ \
  68       " movdqa "#s2", %%xmm5         \n\t"                     \
  69       " psrlw $8, %%xmm4             \n\t" /* .. |  0  h |  */ \
  70       " psrlw $8, %%xmm5             \n\t"                     \
  71       " psllw $8, "#s1"              \n\t" /* .. |  l  0 |  */ \
  72       " psllw $8, "#s2"              \n\t"                     \
  73       " por %%xmm4, "#s1"            \n\t" /* .. |  l  h |  */ \
  74       " por %%xmm5, "#s2"            \n\t"
  75
  76 static void
  77 pa_volume_s16ne_sse (int16_t *samples, int32_t *volumes, unsigned channels, unsigned length)
  78 {
  79   pa_reg_x86 channel, temp;
  80
  81   /* the max number of samples we process at a time, this is also the max amount
  82    * we overread the volume array, which should have enough padding. */
  83   channels = MAX (8, channels);
  84
  85   __asm__ __volatile__ (
  86     " xor %3, %3                    \n\t"
  87     " sar $1, %2                    \n\t" /* length /= sizeof (int16_t) */
  88
  89     " test $1, %2                   \n\t" /* check for odd samples */
  90     " je 2f                         \n\t"
  91
  92     " movd (%1, %3, 4), %%xmm0      \n\t" /* |  v0h  |  v0l  | */
  93     " movw (%0), %4                 \n\t" /*     ..  |   p0  | */
  94     " movd %4, %%xmm1               \n\t"
  95     VOLUME_32x16 (%%xmm1, %%xmm0)
  96     " movd %%xmm0, %4               \n\t" /*     ..  | p0*v0 | */
  97     " movw %4, (%0)                 \n\t"
  98     " add $2, %0                    \n\t"
  99     MOD_ADD ($1, %5)
 100
 101     "2:                             \n\t"
 102     " sar $1, %2                    \n\t" /* prepare for processing 2 samples at a time */
 103     " test $1, %2                   \n\t"
 104     " je 4f                         \n\t"
 105
 106     "3:                             \n\t" /* do samples in groups of 2 */
 107     " movq (%1, %3, 4), %%xmm0      \n\t" /* |  v1h  |  v1l  |  v0h  |  v0l  | */
 108     " movd (%0), %%xmm1             \n\t" /*              .. |   p1  |  p0   | */
 109     VOLUME_32x16 (%%xmm1, %%xmm0)
 110     " movd %%xmm0, (%0)             \n\t" /*              .. | p1*v1 | p0*v0 | */
 111     " add $4, %0                    \n\t"
 112     MOD_ADD ($2, %5)
 113
 114     "4:                             \n\t"
 115     " sar $1, %2                    \n\t" /* prepare for processing 4 samples at a time */
 116     " test $1, %2                   \n\t"
 117     " je 6f                         \n\t"
 118
 119     "5:                             \n\t" /* do samples in groups of 4 */
 120     " movdqu (%1, %3, 4), %%xmm0    \n\t" /* |  v3h  |  v3l  ..  v0h  |  v0l  | */
 121     " movq (%0), %%xmm1             \n\t" /*              .. |   p3  ..  p0   | */
 122     VOLUME_32x16 (%%xmm1, %%xmm0)
 123     " movq %%xmm0, (%0)             \n\t" /*              .. | p3*v3 .. p0*v0 | */
 124     " add $8, %0                    \n\t"
 125     MOD_ADD ($4, %5)
 126
 127     "6:                             \n\t"
 128     " sar $1, %2                    \n\t" /* prepare for processing 8 samples at a time */
 129     " cmp $0, %2                    \n\t"
 130     " je 8f                         \n\t"
 131
 132     "7:                             \n\t" /* do samples in groups of 8 */
 133     " movdqu (%1, %3, 4), %%xmm0    \n\t" /* |  v3h  |  v3l  ..  v0h  |  v0l  | */
 134     " movdqu 16(%1, %3, 4), %%xmm2  \n\t" /* |  v7h  |  v7l  ..  v4h  |  v4l  | */
 135     " movq (%0), %%xmm1             \n\t" /*              .. |   p3  ..  p0   | */
 136     " movq 8(%0), %%xmm3            \n\t" /*              .. |   p7  ..  p4   | */
 137     VOLUME_32x16 (%%xmm1, %%xmm0)
 138     VOLUME_32x16 (%%xmm3, %%xmm2)
 139     " movq %%xmm0, (%0)             \n\t" /*              .. | p3*v3 .. p0*v0 | */
 140     " movq %%xmm2, 8(%0)            \n\t" /*              .. | p7*v7 .. p4*v4 | */
 141     " add $16, %0                   \n\t"
 142     MOD_ADD ($8, %5)
 143     " dec %2                        \n\t"
 144     " jne 7b                        \n\t"
 145     "8:                             \n\t"
 146
 147     : "+r" (samples), "+r" (volumes), "+r" (length), "=D" (channel), "=&r" (temp)
 148     : "r" ((pa_reg_x86)channels)
 149     : "cc"
 150   );
 151 }
 152
 153 static void
 154 pa_volume_s16re_sse (int16_t *samples, int32_t *volumes, unsigned channels, unsigned length)
 155 {
 156   pa_reg_x86 channel, temp;
 157
 158   /* the max number of samples we process at a time, this is also the max amount
 159    * we overread the volume array, which should have enough padding. */
 160   channels = MAX (8, channels);
 161
 162   __asm__ __volatile__ (
 163     " xor %3, %3                    \n\t"
 164     " sar $1, %2                    \n\t" /* length /= sizeof (int16_t) */
 165
 166     " test $1, %2                   \n\t" /* check for odd samples */
 167     " je 2f                         \n\t"
 168
 169     " movd (%1, %3, 4), %%xmm0      \n\t" /* |  v0h  |  v0l  | */
 170     " movw (%0), %4                 \n\t" /*     ..  |   p0  | */
 171     " rorw $8, %4                   \n\t"
 172     " movd %4, %%xmm1               \n\t"
 173     VOLUME_32x16 (%%xmm1, %%xmm0)
 174     " movd %%xmm0, %4               \n\t" /*     ..  | p0*v0 | */
 175     " rorw $8, %4                   \n\t"
 176     " movw %4, (%0)                 \n\t"
 177     " add $2, %0                    \n\t"
 178     MOD_ADD ($1, %5)
 179
 180     "2:                             \n\t"
 181     " sar $1, %2                    \n\t" /* prepare for processing 2 samples at a time */
 182     " test $1, %2                   \n\t"
 183     " je 4f                         \n\t"
 184
 185     "3:                             \n\t" /* do samples in groups of 2 */
 186     " movq (%1, %3, 4), %%xmm0      \n\t" /* |  v1h  |  v1l  |  v0h  |  v0l  | */
 187     " movd (%0), %%xmm1             \n\t" /*              .. |   p1  |  p0   | */
 188     SWAP_16 (%%xmm1)
 189     VOLUME_32x16 (%%xmm1, %%xmm0)
 190     SWAP_16 (%%xmm0)
 191     " movd %%xmm0, (%0)             \n\t" /*              .. | p1*v1 | p0*v0 | */
 192     " add $4, %0                    \n\t"
 193     MOD_ADD ($2, %5)
 194
 195     "4:                             \n\t"
 196     " sar $1, %2                    \n\t" /* prepare for processing 4 samples at a time */
 197     " test $1, %2                   \n\t"
 198     " je 6f                         \n\t"
 199
 200     /* FIXME, we can do aligned access of the volume values if we can guarantee
 201      * that the array is 16 bytes aligned, we probably have to do the odd values
 202      * after this then. */
 203     "5:                             \n\t" /* do samples in groups of 4 */
 204     " movdqu (%1, %3, 4), %%xmm0    \n\t" /* |  v3h  |  v3l  ..  v0h  |  v0l  | */
 205     " movq (%0), %%xmm1             \n\t" /*              .. |   p3  ..  p0   | */
 206     SWAP_16 (%%xmm1)
 207     VOLUME_32x16 (%%xmm1, %%xmm0)
 208     SWAP_16 (%%xmm0)
 209     " movq %%xmm0, (%0)             \n\t" /*              .. | p3*v3 .. p0*v0 | */
 210     " add $8, %0                    \n\t"
 211     MOD_ADD ($4, %5)
 212
 213     "6:                             \n\t"
 214     " sar $1, %2                    \n\t" /* prepare for processing 8 samples at a time */
 215     " cmp $0, %2                    \n\t"
 216     " je 8f                         \n\t"
 217
 218     "7:                             \n\t" /* do samples in groups of 8 */
 219     " movdqu (%1, %3, 4), %%xmm0    \n\t" /* |  v3h  |  v3l  ..  v0h  |  v0l  | */
 220     " movdqu 16(%1, %3, 4), %%xmm2  \n\t" /* |  v7h  |  v7l  ..  v4h  |  v4l  | */
 221     " movq (%0), %%xmm1             \n\t" /*              .. |   p3  ..  p0   | */
 222     " movq 8(%0), %%xmm3            \n\t" /*              .. |   p7  ..  p4   | */
 223     SWAP_16_2 (%%xmm1, %%xmm3)
 224     VOLUME_32x16 (%%xmm1, %%xmm0)
 225     VOLUME_32x16 (%%xmm3, %%xmm2)
 226     SWAP_16_2 (%%xmm0, %%xmm2)
 227     " movq %%xmm0, (%0)             \n\t" /*              .. | p3*v3 .. p0*v0 | */
 228     " movq %%xmm2, 8(%0)            \n\t" /*              .. | p7*v7 .. p4*v4 | */
 229     " add $16, %0                   \n\t"
 230     MOD_ADD ($8, %5)
 231     " dec %2                        \n\t"
 232     " jne 7b                        \n\t"
 233     "8:                             \n\t"
 234
 235     : "+r" (samples), "+r" (volumes), "+r" (length), "=D" (channel), "=&r" (temp)
 236     : "r" ((pa_reg_x86)channels)
 237     : "cc"
 238   );
 239 }
 240
 241 #undef RUN_TEST
 242
 243 #ifdef RUN_TEST
 244 #define CHANNELS 2
 245 #define SAMPLES 1021
 246 #define TIMES 1000
 247 #define PADDING 16
 248
 249 static void run_test (void) {
 250   int16_t samples[SAMPLES];
 251   int16_t samples_ref[SAMPLES];
 252   int16_t samples_orig[SAMPLES];
 253   int32_t volumes[CHANNELS + PADDING];
 254   int i, j, padding;
 255   pa_do_volume_func_t func;
 256   struct timeval start, stop;
 257
 258   func = pa_get_volume_func (PA_SAMPLE_S16NE);
 259
 260   printf ("checking SSE %zd\n", sizeof (samples));
 261
 262   pa_random (samples, sizeof (samples));
 263   memcpy (samples_ref, samples, sizeof (samples));
 264   memcpy (samples_orig, samples, sizeof (samples));
 265
 266   for (i = 0; i < CHANNELS; i++)
 267     volumes[i] = rand() >> 1;
 268   for (padding = 0; padding < PADDING; padding++, i++)
 269     volumes[i] = volumes[padding];
 270
 271   func (samples_ref, volumes, CHANNELS, sizeof (samples));
 272   pa_volume_s16ne_sse (samples, volumes, CHANNELS, sizeof (samples));
 273   for (i = 0; i < SAMPLES; i++) {
 274     if (samples[i] != samples_ref[i]) {
 275       printf ("%d: %04x != %04x (%04x * %04x)\n", i, samples[i], samples_ref[i],
 276               samples_orig[i], volumes[i % CHANNELS]);
 277     }
 278   }
 279
 280   pa_gettimeofday(&start);
 281   for (j = 0; j < TIMES; j++) {
 282     memcpy (samples, samples_orig, sizeof (samples));
 283     pa_volume_s16ne_sse (samples, volumes, CHANNELS, sizeof (samples));
 284   }
 285   pa_gettimeofday(&stop);
 286   pa_log_info("SSE: %llu usec.", (long long unsigned int)pa_timeval_diff (&stop, &start));
 287
 288   pa_gettimeofday(&start);
 289   for (j = 0; j < TIMES; j++) {
 290     memcpy (samples_ref, samples_orig, sizeof (samples));
 291     func (samples_ref, volumes, CHANNELS, sizeof (samples));
 292   }
 293   pa_gettimeofday(&stop);
 294   pa_log_info("ref: %llu usec.", (long long unsigned int)pa_timeval_diff (&stop, &start));
 295 }
 296 #endif
 297
 298 void pa_volume_func_init_sse (pa_cpu_x86_flag_t flags) {
 299   pa_log_info("Initialising SSE optimized functions.");
 300
 301 #ifdef RUN_TEST
 302   run_test ();
 303 #endif
 304
 305   pa_set_volume_func (PA_SAMPLE_S16NE,     (pa_do_volume_func_t) pa_volume_s16ne_sse);
 306   pa_set_volume_func (PA_SAMPLE_S16RE,     (pa_do_volume_func_t) pa_volume_s16re_sse);
 307 }