code.delx.au - pulseaudio/blob - src/pulsecore/svolume_mmx.c

   1 /***
   2   This file is part of PulseAudio.
   3
   4   Copyright 2004-2006 Lennart Poettering
   5   Copyright 2009 Wim Taymans <wim.taymans@collabora.co.uk>
   6
   7   PulseAudio is free software; you can redistribute it and/or modify
   8   it under the terms of the GNU Lesser General Public License as published
   9   by the Free Software Foundation; either version 2.1 of the License,
  10   or (at your option) any later version.
  11
  12   PulseAudio is distributed in the hope that it will be useful, but
  13   WITHOUT ANY WARRANTY; without even the implied warranty of
  14   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  15   General Public License for more details.
  16
  17   You should have received a copy of the GNU Lesser General Public License
  18   along with PulseAudio; if not, write to the Free Software
  19   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
  20   USA.
  21 ***/
  22
  23 #ifdef HAVE_CONFIG_H
  24 #include <config.h>
  25 #endif
  26
  27 #include <pulse/timeval.h>
  28 #include <pulse/rtclock.h>
  29
  30 #include <pulsecore/random.h>
  31 #include <pulsecore/macro.h>
  32 #include <pulsecore/g711.h>
  33 #include <pulsecore/core-util.h>
  34
  35 #include "cpu-x86.h"
  36
  37 #include "sample-util.h"
  38 #include "endianmacros.h"
  39
  40 #if defined (__i386__) || defined (__amd64__)
  41 /* in s: 2 int16_t samples
  42  * in v: 2 int32_t volumes, fixed point 16:16
  43  * out s: contains scaled and clamped int16_t samples.
  44  *
  45  * We calculate the high 32 bits of a 32x16 multiply which we then
  46  * clamp to 16 bits. The calulcation is:
  47  *
  48  *  vl = (v & 0xffff)
  49  *  vh = (v >> 16)
  50  *  s = ((s * vl) >> 16) + (s * vh);
  51  *
  52  * For the first multiply we have to do a sign correction as we need to
  53  * multiply a signed int with an unsigned int. Hacker's delight 8-3 gives a
  54  * simple formula to correct the sign of the high word after the signed
  55  * multiply.
  56  */
  57 #define VOLUME_32x16(s,v)                  /* .. |   vh  |   vl  | */                   \
  58       " pxor  %%mm4, %%mm4           \n\t" /* .. |    0  |    0  | */                   \
  59       " punpcklwd %%mm4, "#s"        \n\t" /* .. |    0  |   p0  | */                   \
  60       " pcmpgtw "#v", %%mm4          \n\t" /* .. |    0  | s(vl) | */                   \
  61       " pand "#s", %%mm4             \n\t" /* .. |    0  |  (p0) |  (vl >> 15) & p */   \
  62       " movq "#s", %%mm5             \n\t"                                              \
  63       " pmulhw "#v", "#s"            \n\t" /* .. |    0  | vl*p0 | */                   \
  64       " paddw %%mm4, "#s"            \n\t" /* .. |    0  | vl*p0 | + sign correct */    \
  65       " psrld $16, "#v"              \n\t" /* .. |    0  |   vh  | */                   \
  66       " pmaddwd %%mm5, "#v"          \n\t" /* .. |    p0 * vh    | */                   \
  67       " paddd "#s", "#v"             \n\t" /* .. |    p0 * v0    | */                   \
  68       " packssdw "#v", "#v"          \n\t" /* .. | p1*v1 | p0*v0 | */
  69
  70 /* approximately advances %3 = (%3 + a) % b. This function requires that
  71  * a <= b. */
  72 #define MOD_ADD(a,b) \
  73       " add "#a", %3                 \n\t" \
  74       " mov %3, %4                   \n\t" \
  75       " sub "#b", %4                 \n\t" \
  76       " cmovae %4, %3                \n\t"
  77
  78 /* swap 16 bits */
  79 #define SWAP_16(s) \
  80       " movq "#s", %%mm4             \n\t" /* .. |  h  l |  */ \
  81       " psrlw $8, %%mm4              \n\t" /* .. |  0  h |  */ \
  82       " psllw $8, "#s"               \n\t" /* .. |  l  0 |  */ \
  83       " por %%mm4, "#s"              \n\t" /* .. |  l  h |  */
  84
  85 /* swap 2 registers 16 bits for better pairing */
  86 #define SWAP_16_2(s1,s2) \
  87       " movq "#s1", %%mm4            \n\t" /* .. |  h  l |  */ \
  88       " movq "#s2", %%mm5            \n\t"                     \
  89       " psrlw $8, %%mm4              \n\t" /* .. |  0  h |  */ \
  90       " psrlw $8, %%mm5              \n\t"                     \
  91       " psllw $8, "#s1"              \n\t" /* .. |  l  0 |  */ \
  92       " psllw $8, "#s2"              \n\t"                     \
  93       " por %%mm4, "#s1"             \n\t" /* .. |  l  h |  */ \
  94       " por %%mm5, "#s2"             \n\t"
  95
  96 static void
  97 pa_volume_s16ne_mmx (int16_t *samples, int32_t *volumes, unsigned channels, unsigned length)
  98 {
  99     pa_reg_x86 channel, temp;
 100
 101     /* the max number of samples we process at a time, this is also the max amount
 102      * we overread the volume array, which should have enough padding. */
 103     channels = PA_MAX (4U, channels);
 104
 105     __asm__ __volatile__ (
 106         " xor %3, %3                    \n\t"
 107         " sar $1, %2                    \n\t" /* length /= sizeof (int16_t) */
 108
 109         " test $1, %2                   \n\t" /* check for odd samples */
 110         " je 2f                         \n\t"
 111
 112         " movd (%1, %3, 4), %%mm0       \n\t" /* |  v0h  |  v0l  | */
 113         " movw (%0), %w4                \n\t" /*     ..  |  p0   | */
 114         " movd %4, %%mm1                \n\t"
 115         VOLUME_32x16 (%%mm1, %%mm0)
 116         " movd %%mm0, %4                \n\t" /*     ..  | p0*v0 | */
 117         " movw %w4, (%0)                \n\t"
 118         " add $2, %0                    \n\t"
 119         MOD_ADD ($1, %5)
 120
 121         "2:                             \n\t"
 122         " sar $1, %2                    \n\t" /* prepare for processing 2 samples at a time */
 123         " test $1, %2                   \n\t" /* check for odd samples */
 124         " je 4f                         \n\t"
 125
 126         "3:                             \n\t" /* do samples in groups of 2 */
 127         " movq (%1, %3, 4), %%mm0       \n\t" /* |  v1h  |  v1l  |  v0h  |  v0l  | */
 128         " movd (%0), %%mm1              \n\t" /*              .. |   p1  |  p0   | */
 129         VOLUME_32x16 (%%mm1, %%mm0)
 130         " movd %%mm0, (%0)              \n\t" /*              .. | p1*v1 | p0*v0 | */
 131         " add $4, %0                    \n\t"
 132         MOD_ADD ($2, %5)
 133
 134         "4:                             \n\t"
 135         " sar $1, %2                    \n\t" /* prepare for processing 4 samples at a time */
 136         " cmp $0, %2                    \n\t"
 137         " je 6f                         \n\t"
 138
 139         "5:                             \n\t" /* do samples in groups of 4 */
 140         " movq (%1, %3, 4), %%mm0       \n\t" /* |  v1h  |  v1l  |  v0h  |  v0l  | */
 141         " movq 8(%1, %3, 4), %%mm2      \n\t" /* |  v3h  |  v3l  |  v2h  |  v2l  | */
 142         " movd (%0), %%mm1              \n\t" /*              .. |   p1  |  p0   | */
 143         " movd 4(%0), %%mm3             \n\t" /*              .. |   p3  |  p2   | */
 144         VOLUME_32x16 (%%mm1, %%mm0)
 145         VOLUME_32x16 (%%mm3, %%mm2)
 146         " movd %%mm0, (%0)              \n\t" /*              .. | p1*v1 | p0*v0 | */
 147         " movd %%mm2, 4(%0)             \n\t" /*              .. | p3*v3 | p2*v2 | */
 148         " add $8, %0                    \n\t"
 149         MOD_ADD ($4, %5)
 150         " dec %2                        \n\t"
 151         " jne 5b                        \n\t"
 152
 153         "6:                             \n\t"
 154         " emms                          \n\t"
 155
 156         : "+r" (samples), "+r" (volumes), "+r" (length), "=D" ((pa_reg_x86)channel), "=&r" (temp)
 157         : "rm" ((pa_reg_x86)channels)
 158         : "cc"
 159     );
 160 }
 161
 162 static void
 163 pa_volume_s16re_mmx (int16_t *samples, int32_t *volumes, unsigned channels, unsigned length)
 164 {
 165     pa_reg_x86 channel, temp;
 166
 167     /* the max number of samples we process at a time, this is also the max amount
 168      * we overread the volume array, which should have enough padding. */
 169     channels = PA_MAX (4U, channels);
 170
 171     __asm__ __volatile__ (
 172         " xor %3, %3                    \n\t"
 173         " sar $1, %2                    \n\t" /* length /= sizeof (int16_t) */
 174         " pcmpeqw %%mm6, %%mm6          \n\t" /* .. |  ffff |  ffff | */
 175         " pcmpeqw %%mm7, %%mm7          \n\t" /* .. |  ffff |  ffff | */
 176         " pslld  $16, %%mm6             \n\t" /* .. |  ffff |     0 | */
 177         " psrld  $31, %%mm7             \n\t" /* .. |     0 |     1 | */
 178
 179         " test $1, %2                   \n\t" /* check for odd samples */
 180         " je 2f                         \n\t"
 181
 182         " movd (%1, %3, 4), %%mm0       \n\t" /* |  v0h  |  v0l  | */
 183         " movw (%0), %w4                \n\t" /*     ..  |  p0   | */
 184         " rorw $8, %w4                  \n\t"
 185         " movd %4, %%mm1                \n\t"
 186         VOLUME_32x16 (%%mm1, %%mm0)
 187         " movd %%mm0, %4                \n\t" /*     ..  | p0*v0 | */
 188         " rorw $8, %w4                  \n\t"
 189         " movw %w4, (%0)                \n\t"
 190         " add $2, %0                    \n\t"
 191         MOD_ADD ($1, %5)
 192
 193         "2:                             \n\t"
 194         " sar $1, %2                    \n\t" /* prepare for processing 2 samples at a time */
 195         " test $1, %2                   \n\t" /* check for odd samples */
 196         " je 4f                         \n\t"
 197
 198         "3:                             \n\t" /* do samples in groups of 2 */
 199         " movq (%1, %3, 4), %%mm0       \n\t" /* |  v1h  |  v1l  |  v0h  |  v0l  | */
 200         " movd (%0), %%mm1              \n\t" /*              .. |   p1  |  p0   | */
 201         SWAP_16 (%%mm1)
 202         VOLUME_32x16 (%%mm1, %%mm0)
 203         SWAP_16 (%%mm0)
 204         " movd %%mm0, (%0)              \n\t" /*              .. | p1*v1 | p0*v0 | */
 205         " add $4, %0                    \n\t"
 206         MOD_ADD ($2, %5)
 207
 208         "4:                             \n\t"
 209         " sar $1, %2                    \n\t" /* prepare for processing 4 samples at a time */
 210         " cmp $0, %2                    \n\t"
 211         " je 6f                         \n\t"
 212
 213         "5:                             \n\t" /* do samples in groups of 4 */
 214         " movq (%1, %3, 4), %%mm0       \n\t" /* |  v1h  |  v1l  |  v0h  |  v0l  | */
 215         " movq 8(%1, %3, 4), %%mm2      \n\t" /* |  v3h  |  v3l  |  v2h  |  v2l  | */
 216         " movd (%0), %%mm1              \n\t" /*              .. |   p1  |  p0   | */
 217         " movd 4(%0), %%mm3             \n\t" /*              .. |   p3  |  p2   | */
 218         SWAP_16_2 (%%mm1, %%mm3)
 219         VOLUME_32x16 (%%mm1, %%mm0)
 220         VOLUME_32x16 (%%mm3, %%mm2)
 221         SWAP_16_2 (%%mm0, %%mm2)
 222         " movd %%mm0, (%0)              \n\t" /*              .. | p1*v1 | p0*v0 | */
 223         " movd %%mm2, 4(%0)             \n\t" /*              .. | p3*v3 | p2*v2 | */
 224         " add $8, %0                    \n\t"
 225         MOD_ADD ($4, %5)
 226         " dec %2                        \n\t"
 227         " jne 5b                        \n\t"
 228
 229         "6:                             \n\t"
 230         " emms                          \n\t"
 231
 232         : "+r" (samples), "+r" (volumes), "+r" (length), "=D" ((pa_reg_x86)channel), "=&r" (temp)
 233         : "rm" ((pa_reg_x86)channels)
 234         : "cc"
 235     );
 236 }
 237
 238 #undef RUN_TEST
 239
 240 #ifdef RUN_TEST
 241 #define CHANNELS 2
 242 #define SAMPLES 1021
 243 #define TIMES 1000
 244 #define PADDING 16
 245
 246 static void run_test (void) {
 247     int16_t samples[SAMPLES];
 248     int16_t samples_ref[SAMPLES];
 249     int16_t samples_orig[SAMPLES];
 250     int32_t volumes[CHANNELS + PADDING];
 251     int i, j, padding;
 252     pa_do_volume_func_t func;
 253     pa_usec_t start, stop;
 254
 255     func = pa_get_volume_func (PA_SAMPLE_S16NE);
 256
 257     printf ("checking MMX %zd\n", sizeof (samples));
 258
 259     pa_random (samples, sizeof (samples));
 260     memcpy (samples_ref, samples, sizeof (samples));
 261     memcpy (samples_orig, samples, sizeof (samples));
 262
 263     for (i = 0; i < CHANNELS; i++)
 264         volumes[i] = rand() >> 1;
 265     for (padding = 0; padding < PADDING; padding++, i++)
 266         volumes[i] = volumes[padding];
 267
 268     func (samples_ref, volumes, CHANNELS, sizeof (samples));
 269     pa_volume_s16ne_mmx (samples, volumes, CHANNELS, sizeof (samples));
 270     for (i = 0; i < SAMPLES; i++) {
 271         if (samples[i] != samples_ref[i]) {
 272             printf ("%d: %04x != %04x (%04x * %04x)\n", i, samples[i], samples_ref[i],
 273                   samples_orig[i], volumes[i % CHANNELS]);
 274         }
 275     }
 276
 277     start = pa_rtclock_now();
 278     for (j = 0; j < TIMES; j++) {
 279         memcpy (samples, samples_orig, sizeof (samples));
 280         pa_volume_s16ne_mmx (samples, volumes, CHANNELS, sizeof (samples));
 281     }
 282     stop = pa_rtclock_now();
 283     pa_log_info("MMX: %llu usec.", (long long unsigned int)(stop - start));
 284
 285     start = pa_rtclock_now();
 286     for (j = 0; j < TIMES; j++) {
 287         memcpy (samples_ref, samples_orig, sizeof (samples));
 288         func (samples_ref, volumes, CHANNELS, sizeof (samples));
 289     }
 290     stop = pa_rtclock_now();
 291     pa_log_info("ref: %llu usec.", (long long unsigned int)(stop - start));
 292
 293     pa_assert_se(memcmp(samples_ref, samples, sizeof(samples)) == 0);
 294 }
 295 #endif
 296
 297 #endif /* defined (__i386__) || defined (__amd64__) */
 298
 299
 300 void pa_volume_func_init_mmx (pa_cpu_x86_flag_t flags) {
 301 #if defined (__i386__) || defined (__amd64__)
 302
 303 #ifdef RUN_TEST
 304     run_test ();
 305 #endif
 306
 307     if (flags & PA_CPU_X86_MMX) {
 308         pa_log_info("Initialising MMX optimized functions.");
 309
 310         pa_set_volume_func (PA_SAMPLE_S16NE, (pa_do_volume_func_t) pa_volume_s16ne_mmx);
 311         pa_set_volume_func (PA_SAMPLE_S16RE, (pa_do_volume_func_t) pa_volume_s16re_mmx);
 312     }
 313 #endif /* defined (__i386__) || defined (__amd64__) */
 314 }