code.delx.au - pulseaudio/blob - src/pulsecore/svolume_mmx.c

   1 /***
   2   This file is part of PulseAudio.
   3
   4   Copyright 2004-2006 Lennart Poettering
   5   Copyright 2009 Wim Taymans <wim.taymans@collabora.co.uk>
   6
   7   PulseAudio is free software; you can redistribute it and/or modify
   8   it under the terms of the GNU Lesser General Public License as published
   9   by the Free Software Foundation; either version 2.1 of the License,
  10   or (at your option) any later version.
  11
  12   PulseAudio is distributed in the hope that it will be useful, but
  13   WITHOUT ANY WARRANTY; without even the implied warranty of
  14   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  15   General Public License for more details.
  16
  17   You should have received a copy of the GNU Lesser General Public License
  18   along with PulseAudio; if not, write to the Free Software
  19   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
  20   USA.
  21 ***/
  22
  23 #ifdef HAVE_CONFIG_H
  24 #include <config.h>
  25 #endif
  26
  27 #include <pulse/timeval.h>
  28 #include <pulsecore/random.h>
  29 #include <pulsecore/macro.h>
  30 #include <pulsecore/g711.h>
  31 #include <pulsecore/core-util.h>
  32
  33 #include "cpu-x86.h"
  34
  35 #include "sample-util.h"
  36 #include "endianmacros.h"
  37
  38 #if defined (__i386__) || defined (__amd64__)
  39 /* in s: 2 int16_t samples
  40  * in v: 2 int32_t volumes, fixed point 16:16
  41  * out s: contains scaled and clamped int16_t samples.
  42  *
  43  * We calculate the high 32 bits of a 32x16 multiply which we then
  44  * clamp to 16 bits. The calulcation is:
  45  *
  46  *  vl = (v & 0xffff)
  47  *  vh = (v >> 16)
  48  *  s = ((s * vl) >> 16) + (s * vh);
  49  *
  50  * For the first multiply we have to do a sign correction as we need to
  51  * multiply a signed int with an unsigned int. Hacker's delight 8-3 gives a
  52  * simple formula to correct the sign of the high word after the signed
  53  * multiply.
  54  */
  55 #define VOLUME_32x16(s,v)                  /* .. |   vh  |   vl  | */                   \
  56       " pxor  %%mm4, %%mm4           \n\t" /* .. |    0  |    0  | */                   \
  57       " punpcklwd %%mm4, "#s"        \n\t" /* .. |    0  |   p0  | */                   \
  58       " pcmpgtw "#v", %%mm4          \n\t" /* .. |    0  | s(vl) | */                   \
  59       " pand "#s", %%mm4             \n\t" /* .. |    0  |  (p0) |  (vl >> 15) & p */   \
  60       " movq %%mm6, %%mm5            \n\t" /* .. |  ffff |   0   | */                   \
  61       " pand "#v", %%mm5             \n\t" /* .. |   vh  |   0   | */                   \
  62       " por %%mm5, %%mm4             \n\t" /* .. |   vh  |  (p0) | */                   \
  63       " pmulhw "#s", "#v"            \n\t" /* .. |    0  | vl*p0 | */                   \
  64       " paddw %%mm4, "#v"            \n\t" /* .. |   vh  | vl*p0 | vh + sign correct */ \
  65       " pslld $16, "#s"              \n\t" /* .. |   p0  |    0  | */                   \
  66       " por %%mm7, "#s"              \n\t" /* .. |   p0  |    1  | */                   \
  67       " pmaddwd "#s", "#v"           \n\t" /* .. |    p0 * v0    | */                   \
  68       " packssdw "#v", "#v"          \n\t" /* .. | p1*v1 | p0*v0 | */
  69
  70 /* approximately advances %3 = (%3 + a) % b. This function requires that
  71  * a <= b. */
  72 #define MOD_ADD(a,b) \
  73       " add "#a", %3                 \n\t" \
  74       " mov %3, %4                   \n\t" \
  75       " sub "#b", %4                 \n\t" \
  76       " cmovae %4, %3                \n\t"
  77
  78 /* swap 16 bits */
  79 #define SWAP_16(s) \
  80       " movq "#s", %%mm4             \n\t" /* .. |  h  l |  */ \
  81       " psrlw $8, %%mm4              \n\t" /* .. |  0  h |  */ \
  82       " psllw $8, "#s"               \n\t" /* .. |  l  0 |  */ \
  83       " por %%mm4, "#s"              \n\t" /* .. |  l  h |  */
  84
  85 /* swap 2 registers 16 bits for better pairing */
  86 #define SWAP_16_2(s1,s2) \
  87       " movq "#s1", %%mm4            \n\t" /* .. |  h  l |  */ \
  88       " movq "#s2", %%mm5            \n\t"                     \
  89       " psrlw $8, %%mm4              \n\t" /* .. |  0  h |  */ \
  90       " psrlw $8, %%mm5              \n\t"                     \
  91       " psllw $8, "#s1"              \n\t" /* .. |  l  0 |  */ \
  92       " psllw $8, "#s2"              \n\t"                     \
  93       " por %%mm4, "#s1"             \n\t" /* .. |  l  h |  */ \
  94       " por %%mm5, "#s2"             \n\t"
  95
  96 static void
  97 pa_volume_s16ne_mmx (int16_t *samples, int32_t *volumes, unsigned channels, unsigned length)
  98 {
  99     pa_reg_x86 channel, temp;
 100
 101     /* the max number of samples we process at a time, this is also the max amount
 102      * we overread the volume array, which should have enough padding. */
 103     channels = PA_MAX (4U, channels);
 104
 105     __asm__ __volatile__ (
 106         " xor %3, %3                    \n\t"
 107         " sar $1, %2                    \n\t" /* length /= sizeof (int16_t) */
 108         " pcmpeqw %%mm6, %%mm6          \n\t" /* .. |  ffff |  ffff | */
 109         " pcmpeqw %%mm7, %%mm7          \n\t" /* .. |  ffff |  ffff | */
 110         " pslld  $16, %%mm6             \n\t" /* .. |  ffff |     0 | */
 111         " psrld  $31, %%mm7             \n\t" /* .. |     0 |     1 | */
 112
 113         " test $1, %2                   \n\t" /* check for odd samples */
 114         " je 2f                         \n\t"
 115
 116         " movd (%1, %3, 4), %%mm0       \n\t" /* |  v0h  |  v0l  | */
 117         " movw (%0), %w4                \n\t" /*     ..  |  p0   | */
 118         " movd %4, %%mm1                \n\t"
 119         VOLUME_32x16 (%%mm1, %%mm0)
 120         " movd %%mm0, %4                \n\t" /*     ..  | p0*v0 | */
 121         " movw %w4, (%0)                \n\t"
 122         " add $2, %0                    \n\t"
 123         MOD_ADD ($1, %5)
 124
 125         "2:                             \n\t"
 126         " sar $1, %2                    \n\t" /* prepare for processing 2 samples at a time */
 127         " test $1, %2                   \n\t" /* check for odd samples */
 128         " je 4f                         \n\t"
 129
 130         "3:                             \n\t" /* do samples in groups of 2 */
 131         " movq (%1, %3, 4), %%mm0       \n\t" /* |  v1h  |  v1l  |  v0h  |  v0l  | */
 132         " movd (%0), %%mm1              \n\t" /*              .. |   p1  |  p0   | */
 133         VOLUME_32x16 (%%mm1, %%mm0)
 134         " movd %%mm0, (%0)              \n\t" /*              .. | p1*v1 | p0*v0 | */
 135         " add $4, %0                    \n\t"
 136         MOD_ADD ($2, %5)
 137
 138         "4:                             \n\t"
 139         " sar $1, %2                    \n\t" /* prepare for processing 4 samples at a time */
 140         " cmp $0, %2                    \n\t"
 141         " je 6f                         \n\t"
 142
 143         "5:                             \n\t" /* do samples in groups of 4 */
 144         " movq (%1, %3, 4), %%mm0       \n\t" /* |  v1h  |  v1l  |  v0h  |  v0l  | */
 145         " movq 8(%1, %3, 4), %%mm2      \n\t" /* |  v3h  |  v3l  |  v2h  |  v2l  | */
 146         " movd (%0), %%mm1              \n\t" /*              .. |   p1  |  p0   | */
 147         " movd 4(%0), %%mm3             \n\t" /*              .. |   p3  |  p2   | */
 148         VOLUME_32x16 (%%mm1, %%mm0)
 149         VOLUME_32x16 (%%mm3, %%mm2)
 150         " movd %%mm0, (%0)              \n\t" /*              .. | p1*v1 | p0*v0 | */
 151         " movd %%mm2, 4(%0)             \n\t" /*              .. | p3*v3 | p2*v2 | */
 152         " add $8, %0                    \n\t"
 153         MOD_ADD ($4, %5)
 154         " dec %2                        \n\t"
 155         " jne 5b                        \n\t"
 156
 157         "6:                             \n\t"
 158         " emms                          \n\t"
 159
 160         : "+r" (samples), "+r" (volumes), "+r" (length), "=D" ((pa_reg_x86)channel), "=&r" (temp)
 161         : "r" ((pa_reg_x86)channels)
 162         : "cc"
 163     );
 164 }
 165
 166 static void
 167 pa_volume_s16re_mmx (int16_t *samples, int32_t *volumes, unsigned channels, unsigned length)
 168 {
 169     pa_reg_x86 channel, temp;
 170
 171     /* the max number of samples we process at a time, this is also the max amount
 172      * we overread the volume array, which should have enough padding. */
 173     channels = PA_MAX (4U, channels);
 174
 175     __asm__ __volatile__ (
 176         " xor %3, %3                    \n\t"
 177         " sar $1, %2                    \n\t" /* length /= sizeof (int16_t) */
 178         " pcmpeqw %%mm6, %%mm6          \n\t" /* .. |  ffff |  ffff | */
 179         " pcmpeqw %%mm7, %%mm7          \n\t" /* .. |  ffff |  ffff | */
 180         " pslld  $16, %%mm6             \n\t" /* .. |  ffff |     0 | */
 181         " psrld  $31, %%mm7             \n\t" /* .. |     0 |     1 | */
 182
 183         " test $1, %2                   \n\t" /* check for odd samples */
 184         " je 2f                         \n\t"
 185
 186         " movd (%1, %3, 4), %%mm0       \n\t" /* |  v0h  |  v0l  | */
 187         " movw (%0), %w4                \n\t" /*     ..  |  p0   | */
 188         " rorw $8, %w4                  \n\t"
 189         " movd %4, %%mm1                \n\t"
 190         VOLUME_32x16 (%%mm1, %%mm0)
 191         " movd %%mm0, %4                \n\t" /*     ..  | p0*v0 | */
 192         " rorw $8, %w4                  \n\t"
 193         " movw %w4, (%0)                \n\t"
 194         " add $2, %0                    \n\t"
 195         MOD_ADD ($1, %5)
 196
 197         "2:                             \n\t"
 198         " sar $1, %2                    \n\t" /* prepare for processing 2 samples at a time */
 199         " test $1, %2                   \n\t" /* check for odd samples */
 200         " je 4f                         \n\t"
 201
 202         "3:                             \n\t" /* do samples in groups of 2 */
 203         " movq (%1, %3, 4), %%mm0       \n\t" /* |  v1h  |  v1l  |  v0h  |  v0l  | */
 204         " movd (%0), %%mm1              \n\t" /*              .. |   p1  |  p0   | */
 205         SWAP_16 (%%mm1)
 206         VOLUME_32x16 (%%mm1, %%mm0)
 207         SWAP_16 (%%mm0)
 208         " movd %%mm0, (%0)              \n\t" /*              .. | p1*v1 | p0*v0 | */
 209         " add $4, %0                    \n\t"
 210         MOD_ADD ($2, %5)
 211
 212         "4:                             \n\t"
 213         " sar $1, %2                    \n\t" /* prepare for processing 4 samples at a time */
 214         " cmp $0, %2                    \n\t"
 215         " je 6f                         \n\t"
 216
 217         "5:                             \n\t" /* do samples in groups of 4 */
 218         " movq (%1, %3, 4), %%mm0       \n\t" /* |  v1h  |  v1l  |  v0h  |  v0l  | */
 219         " movq 8(%1, %3, 4), %%mm2      \n\t" /* |  v3h  |  v3l  |  v2h  |  v2l  | */
 220         " movd (%0), %%mm1              \n\t" /*              .. |   p1  |  p0   | */
 221         " movd 4(%0), %%mm3             \n\t" /*              .. |   p3  |  p2   | */
 222         SWAP_16_2 (%%mm1, %%mm3)
 223         VOLUME_32x16 (%%mm1, %%mm0)
 224         VOLUME_32x16 (%%mm3, %%mm2)
 225         SWAP_16_2 (%%mm0, %%mm2)
 226         " movd %%mm0, (%0)              \n\t" /*              .. | p1*v1 | p0*v0 | */
 227         " movd %%mm2, 4(%0)             \n\t" /*              .. | p3*v3 | p2*v2 | */
 228         " add $8, %0                    \n\t"
 229         MOD_ADD ($4, %5)
 230         " dec %2                        \n\t"
 231         " jne 5b                        \n\t"
 232
 233         "6:                             \n\t"
 234         " emms                          \n\t"
 235
 236         : "+r" (samples), "+r" (volumes), "+r" (length), "=D" ((pa_reg_x86)channel), "=&r" (temp)
 237         : "r" ((pa_reg_x86)channels)
 238         : "cc"
 239     );
 240 }
 241
 242 #undef RUN_TEST
 243
 244 #ifdef RUN_TEST
 245 #define CHANNELS 2
 246 #define SAMPLES 1021
 247 #define TIMES 1000
 248 #define PADDING 16
 249
 250 static void run_test (void) {
 251     int16_t samples[SAMPLES];
 252     int16_t samples_ref[SAMPLES];
 253     int16_t samples_orig[SAMPLES];
 254     int32_t volumes[CHANNELS + PADDING];
 255     int i, j, padding;
 256     pa_do_volume_func_t func;
 257     pa_usec_t start, stop;
 258
 259     func = pa_get_volume_func (PA_SAMPLE_S16NE);
 260
 261     printf ("checking MMX %zd\n", sizeof (samples));
 262
 263     pa_random (samples, sizeof (samples));
 264     memcpy (samples_ref, samples, sizeof (samples));
 265     memcpy (samples_orig, samples, sizeof (samples));
 266
 267     for (i = 0; i < CHANNELS; i++)
 268         volumes[i] = rand() >> 1;
 269     for (padding = 0; padding < PADDING; padding++, i++)
 270         volumes[i] = volumes[padding];
 271
 272     func (samples_ref, volumes, CHANNELS, sizeof (samples));
 273     pa_volume_s16ne_mmx (samples, volumes, CHANNELS, sizeof (samples));
 274     for (i = 0; i < SAMPLES; i++) {
 275         if (samples[i] != samples_ref[i]) {
 276             printf ("%d: %04x != %04x (%04x * %04x)\n", i, samples[i], samples_ref[i],
 277                   samples_orig[i], volumes[i % CHANNELS]);
 278         }
 279     }
 280
 281     start = pa_rtclock_now();
 282     for (j = 0; j < TIMES; j++) {
 283         memcpy (samples, samples_orig, sizeof (samples));
 284         pa_volume_s16ne_mmx (samples, volumes, CHANNELS, sizeof (samples));
 285     }
 286     stop = pa_rtclock_now();
 287     pa_log_info("MMX: %llu usec.", (long long unsigned int)(stop - start));
 288
 289     start = pa_rtclock_now();
 290     for (j = 0; j < TIMES; j++) {
 291         memcpy (samples_ref, samples_orig, sizeof (samples));
 292         func (samples_ref, volumes, CHANNELS, sizeof (samples));
 293     }
 294     stop = pa_rtclock_now();
 295     pa_log_info("ref: %llu usec.", (long long unsigned int)(stop - start));
 296 }
 297 #endif
 298
 299 #endif /* defined (__i386__) || defined (__amd64__) */
 300
 301
 302 void pa_volume_func_init_mmx (pa_cpu_x86_flag_t flags) {
 303 #if defined (__i386__) || defined (__amd64__)
 304
 305 #ifdef RUN_TEST
 306     run_test ();
 307 #endif
 308
 309     if (flags & PA_CPU_X86_MMX) {
 310         pa_log_info("Initialising MMX optimized functions.");
 311
 312         pa_set_volume_func (PA_SAMPLE_S16NE, (pa_do_volume_func_t) pa_volume_s16ne_mmx);
 313         pa_set_volume_func (PA_SAMPLE_S16RE, (pa_do_volume_func_t) pa_volume_s16re_mmx);
 314     }
 315 #endif /* defined (__i386__) || defined (__amd64__) */
 316 }