code.delx.au - pulseaudio/blob - src/pulsecore/svolume_mmx.c

   1 /***
   2   This file is part of PulseAudio.
   3
   4   Copyright 2004-2006 Lennart Poettering
   5   Copyright 2009 Wim Taymans <wim.taymans@collabora.co.uk>
   6
   7   PulseAudio is free software; you can redistribute it and/or modify
   8   it under the terms of the GNU Lesser General Public License as published
   9   by the Free Software Foundation; either version 2.1 of the License,
  10   or (at your option) any later version.
  11
  12   PulseAudio is distributed in the hope that it will be useful, but
  13   WITHOUT ANY WARRANTY; without even the implied warranty of
  14   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  15   General Public License for more details.
  16
  17   You should have received a copy of the GNU Lesser General Public License
  18   along with PulseAudio; if not, write to the Free Software
  19   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
  20   USA.
  21 ***/
  22
  23 #ifdef HAVE_CONFIG_H
  24 #include <config.h>
  25 #endif
  26
  27 #include <pulse/timeval.h>
  28 #include <pulse/rtclock.h>
  29
  30 #include <pulsecore/random.h>
  31 #include <pulsecore/macro.h>
  32 #include <pulsecore/g711.h>
  33 #include <pulsecore/core-util.h>
  34 #include <pulsecore/endianmacros.h>
  35
  36 #include "cpu-x86.h"
  37
  38 #include "sample-util.h"
  39
  40 #if defined (__i386__) || defined (__amd64__)
  41 /* in s: 2 int16_t samples
  42  * in v: 2 int32_t volumes, fixed point 16:16
  43  * out s: contains scaled and clamped int16_t samples.
  44  *
  45  * We calculate the high 32 bits of a 32x16 multiply which we then
  46  * clamp to 16 bits. The calulcation is:
  47  *
  48  *  vl = (v & 0xffff)
  49  *  vh = (v >> 16)
  50  *  s = ((s * vl) >> 16) + (s * vh);
  51  *
  52  * For the first multiply we have to do a sign correction as we need to
  53  * multiply a signed int with an unsigned int. Hacker's delight 8-3 gives a
  54  * simple formula to correct the sign of the high word after the signed
  55  * multiply.
  56  */
  57 #define VOLUME_32x16(s,v)                  /* .. |   vh  |   vl  | */                   \
  58       " pxor  %%mm4, %%mm4           \n\t" /* .. |    0  |    0  | */                   \
  59       " punpcklwd %%mm4, "#s"        \n\t" /* .. |    0  |   p0  | */                   \
  60       " pcmpgtw "#v", %%mm4          \n\t" /* .. |    0  | s(vl) | */                   \
  61       " pand "#s", %%mm4             \n\t" /* .. |    0  |  (p0) |  (vl >> 15) & p */   \
  62       " movq "#s", %%mm5             \n\t"                                              \
  63       " pmulhw "#v", "#s"            \n\t" /* .. |    0  | vl*p0 | */                   \
  64       " paddw %%mm4, "#s"            \n\t" /* .. |    0  | vl*p0 | + sign correct */    \
  65       " pslld $16, "#s"              \n\t" /* .. | vl*p0 |   0   | */                   \
  66       " psrld $16, "#v"              \n\t" /* .. |    0  |   vh  | */                   \
  67       " psrad $16, "#s"              \n\t" /* .. |     vl*p0     | sign extend */       \
  68       " pmaddwd %%mm5, "#v"          \n\t" /* .. |    p0 * vh    | */                   \
  69       " paddd "#s", "#v"             \n\t" /* .. |    p0 * v0    | */                   \
  70       " packssdw "#v", "#v"          \n\t" /* .. | p1*v1 | p0*v0 | */
  71
  72 /* approximately advances %3 = (%3 + a) % b. This function requires that
  73  * a <= b. */
  74 #define MOD_ADD(a,b) \
  75       " add "#a", %3                 \n\t" \
  76       " mov %3, %4                   \n\t" \
  77       " sub "#b", %4                 \n\t" \
  78       " cmovae %4, %3                \n\t"
  79
  80 /* swap 16 bits */
  81 #define SWAP_16(s) \
  82       " movq "#s", %%mm4             \n\t" /* .. |  h  l |  */ \
  83       " psrlw $8, %%mm4              \n\t" /* .. |  0  h |  */ \
  84       " psllw $8, "#s"               \n\t" /* .. |  l  0 |  */ \
  85       " por %%mm4, "#s"              \n\t" /* .. |  l  h |  */
  86
  87 /* swap 2 registers 16 bits for better pairing */
  88 #define SWAP_16_2(s1,s2) \
  89       " movq "#s1", %%mm4            \n\t" /* .. |  h  l |  */ \
  90       " movq "#s2", %%mm5            \n\t"                     \
  91       " psrlw $8, %%mm4              \n\t" /* .. |  0  h |  */ \
  92       " psrlw $8, %%mm5              \n\t"                     \
  93       " psllw $8, "#s1"              \n\t" /* .. |  l  0 |  */ \
  94       " psllw $8, "#s2"              \n\t"                     \
  95       " por %%mm4, "#s1"             \n\t" /* .. |  l  h |  */ \
  96       " por %%mm5, "#s2"             \n\t"
  97
  98 static void pa_volume_s16ne_mmx(int16_t *samples, int32_t *volumes, unsigned channels, unsigned length) {
  99     pa_reg_x86 channel, temp;
 100
 101     /* Channels must be at least 4, and always a multiple of the original number.
 102      * This is also the max amount we overread the volume array, which should
 103      * have enough padding. */
 104     channels = channels == 3 ? 6 : PA_MAX (4U, channels);
 105
 106     __asm__ __volatile__ (
 107         " xor %3, %3                    \n\t"
 108         " sar $1, %2                    \n\t" /* length /= sizeof (int16_t) */
 109
 110         " test $1, %2                   \n\t" /* check for odd samples */
 111         " je 2f                         \n\t"
 112
 113         " movd (%1, %3, 4), %%mm0       \n\t" /* |  v0h  |  v0l  | */
 114         " movw (%0), %w4                \n\t" /*     ..  |  p0   | */
 115         " movd %4, %%mm1                \n\t"
 116         VOLUME_32x16 (%%mm1, %%mm0)
 117         " movd %%mm0, %4                \n\t" /*     ..  | p0*v0 | */
 118         " movw %w4, (%0)                \n\t"
 119         " add $2, %0                    \n\t"
 120         MOD_ADD ($1, %5)
 121
 122         "2:                             \n\t"
 123         " sar $1, %2                    \n\t" /* prepare for processing 2 samples at a time */
 124         " test $1, %2                   \n\t" /* check for odd samples */
 125         " je 4f                         \n\t"
 126
 127         "3:                             \n\t" /* do samples in groups of 2 */
 128         " movq (%1, %3, 4), %%mm0       \n\t" /* |  v1h  |  v1l  |  v0h  |  v0l  | */
 129         " movd (%0), %%mm1              \n\t" /*              .. |   p1  |  p0   | */
 130         VOLUME_32x16 (%%mm1, %%mm0)
 131         " movd %%mm0, (%0)              \n\t" /*              .. | p1*v1 | p0*v0 | */
 132         " add $4, %0                    \n\t"
 133         MOD_ADD ($2, %5)
 134
 135         "4:                             \n\t"
 136         " sar $1, %2                    \n\t" /* prepare for processing 4 samples at a time */
 137         " cmp $0, %2                    \n\t"
 138         " je 6f                         \n\t"
 139
 140         "5:                             \n\t" /* do samples in groups of 4 */
 141         " movq (%1, %3, 4), %%mm0       \n\t" /* |  v1h  |  v1l  |  v0h  |  v0l  | */
 142         " movq 8(%1, %3, 4), %%mm2      \n\t" /* |  v3h  |  v3l  |  v2h  |  v2l  | */
 143         " movd (%0), %%mm1              \n\t" /*              .. |   p1  |  p0   | */
 144         " movd 4(%0), %%mm3             \n\t" /*              .. |   p3  |  p2   | */
 145         VOLUME_32x16 (%%mm1, %%mm0)
 146         VOLUME_32x16 (%%mm3, %%mm2)
 147         " movd %%mm0, (%0)              \n\t" /*              .. | p1*v1 | p0*v0 | */
 148         " movd %%mm2, 4(%0)             \n\t" /*              .. | p3*v3 | p2*v2 | */
 149         " add $8, %0                    \n\t"
 150         MOD_ADD ($4, %5)
 151         " dec %2                        \n\t"
 152         " jne 5b                        \n\t"
 153
 154         "6:                             \n\t"
 155         " emms                          \n\t"
 156
 157         : "+r" (samples), "+r" (volumes), "+r" (length), "=D" (channel), "=&r" (temp)
 158 #if defined (__i386__)
 159         : "m" ((pa_reg_x86)channels)
 160 #else
 161         : "r" ((pa_reg_x86)channels)
 162 #endif
 163         : "cc"
 164     );
 165 }
 166
 167 static void pa_volume_s16re_mmx(int16_t *samples, int32_t *volumes, unsigned channels, unsigned length) {
 168     pa_reg_x86 channel, temp;
 169
 170     /* Channels must be at least 4, and always a multiple of the original number.
 171      * This is also the max amount we overread the volume array, which should
 172      * have enough padding. */
 173     channels = channels == 3 ? 6 : PA_MAX (4U, channels);
 174
 175     __asm__ __volatile__ (
 176         " xor %3, %3                    \n\t"
 177         " sar $1, %2                    \n\t" /* length /= sizeof (int16_t) */
 178         " pcmpeqw %%mm6, %%mm6          \n\t" /* .. |  ffff |  ffff | */
 179         " pcmpeqw %%mm7, %%mm7          \n\t" /* .. |  ffff |  ffff | */
 180         " pslld  $16, %%mm6             \n\t" /* .. |  ffff |     0 | */
 181         " psrld  $31, %%mm7             \n\t" /* .. |     0 |     1 | */
 182
 183         " test $1, %2                   \n\t" /* check for odd samples */
 184         " je 2f                         \n\t"
 185
 186         " movd (%1, %3, 4), %%mm0       \n\t" /* |  v0h  |  v0l  | */
 187         " movw (%0), %w4                \n\t" /*     ..  |  p0   | */
 188         " rorw $8, %w4                  \n\t"
 189         " movd %4, %%mm1                \n\t"
 190         VOLUME_32x16 (%%mm1, %%mm0)
 191         " movd %%mm0, %4                \n\t" /*     ..  | p0*v0 | */
 192         " rorw $8, %w4                  \n\t"
 193         " movw %w4, (%0)                \n\t"
 194         " add $2, %0                    \n\t"
 195         MOD_ADD ($1, %5)
 196
 197         "2:                             \n\t"
 198         " sar $1, %2                    \n\t" /* prepare for processing 2 samples at a time */
 199         " test $1, %2                   \n\t" /* check for odd samples */
 200         " je 4f                         \n\t"
 201
 202         "3:                             \n\t" /* do samples in groups of 2 */
 203         " movq (%1, %3, 4), %%mm0       \n\t" /* |  v1h  |  v1l  |  v0h  |  v0l  | */
 204         " movd (%0), %%mm1              \n\t" /*              .. |   p1  |  p0   | */
 205         SWAP_16 (%%mm1)
 206         VOLUME_32x16 (%%mm1, %%mm0)
 207         SWAP_16 (%%mm0)
 208         " movd %%mm0, (%0)              \n\t" /*              .. | p1*v1 | p0*v0 | */
 209         " add $4, %0                    \n\t"
 210         MOD_ADD ($2, %5)
 211
 212         "4:                             \n\t"
 213         " sar $1, %2                    \n\t" /* prepare for processing 4 samples at a time */
 214         " cmp $0, %2                    \n\t"
 215         " je 6f                         \n\t"
 216
 217         "5:                             \n\t" /* do samples in groups of 4 */
 218         " movq (%1, %3, 4), %%mm0       \n\t" /* |  v1h  |  v1l  |  v0h  |  v0l  | */
 219         " movq 8(%1, %3, 4), %%mm2      \n\t" /* |  v3h  |  v3l  |  v2h  |  v2l  | */
 220         " movd (%0), %%mm1              \n\t" /*              .. |   p1  |  p0   | */
 221         " movd 4(%0), %%mm3             \n\t" /*              .. |   p3  |  p2   | */
 222         SWAP_16_2 (%%mm1, %%mm3)
 223         VOLUME_32x16 (%%mm1, %%mm0)
 224         VOLUME_32x16 (%%mm3, %%mm2)
 225         SWAP_16_2 (%%mm0, %%mm2)
 226         " movd %%mm0, (%0)              \n\t" /*              .. | p1*v1 | p0*v0 | */
 227         " movd %%mm2, 4(%0)             \n\t" /*              .. | p3*v3 | p2*v2 | */
 228         " add $8, %0                    \n\t"
 229         MOD_ADD ($4, %5)
 230         " dec %2                        \n\t"
 231         " jne 5b                        \n\t"
 232
 233         "6:                             \n\t"
 234         " emms                          \n\t"
 235
 236         : "+r" (samples), "+r" (volumes), "+r" (length), "=D" (channel), "=&r" (temp)
 237 #if defined (__i386__)
 238         : "m" ((pa_reg_x86)channels)
 239 #else
 240         : "r" ((pa_reg_x86)channels)
 241 #endif
 242         : "cc"
 243     );
 244 }
 245
 246 #undef RUN_TEST
 247
 248 #ifdef RUN_TEST
 249 #define CHANNELS 2
 250 #define SAMPLES 1022
 251 #define TIMES 1000
 252 #define TIMES2 100
 253 #define PADDING 16
 254
 255 static void run_test(void) {
 256     int16_t samples[SAMPLES];
 257     int16_t samples_ref[SAMPLES];
 258     int16_t samples_orig[SAMPLES];
 259     int32_t volumes[CHANNELS + PADDING];
 260     int i, j, padding;
 261     pa_do_volume_func_t func;
 262     pa_usec_t start, stop;
 263     int k;
 264     pa_usec_t min = INT_MAX, max = 0;
 265     double s1 = 0, s2 = 0;
 266
 267     func = pa_get_volume_func(PA_SAMPLE_S16NE);
 268
 269     printf("checking MMX %zd\n", sizeof(samples));
 270
 271     pa_random(samples, sizeof(samples));
 272     /* for (i = 0; i < SAMPLES; i++)
 273        samples[i] = -1; */
 274     memcpy(samples_ref, samples, sizeof(samples));
 275     memcpy(samples_orig, samples, sizeof(samples));
 276
 277     for (i = 0; i < CHANNELS; i++)
 278         volumes[i] = PA_CLAMP_VOLUME(rand() >> 1);
 279         /* volumes[i] = 0x0000ffff; */
 280     for (padding = 0; padding < PADDING; padding++, i++)
 281         volumes[i] = volumes[padding];
 282
 283     func(samples_ref, volumes, CHANNELS, sizeof(samples));
 284     pa_volume_s16ne_mmx(samples, volumes, CHANNELS, sizeof(samples));
 285     for (i = 0; i < SAMPLES; i++) {
 286         if (samples[i] != samples_ref[i]) {
 287             printf("%d: %04x != %04x (%04x * %08x)\n", i, samples[i], samples_ref[i],
 288                   samples_orig[i], volumes[i % CHANNELS]);
 289         }
 290     }
 291
 292     for (k = 0; k < TIMES2; k++) {
 293         start = pa_rtclock_now();
 294         for (j = 0; j < TIMES; j++) {
 295             memcpy(samples, samples_orig, sizeof(samples));
 296             pa_volume_s16ne_mmx(samples, volumes, CHANNELS, sizeof(samples));
 297         }
 298         stop = pa_rtclock_now();
 299
 300         if (min > (stop - start)) min = stop - start;
 301         if (max < (stop - start)) max = stop - start;
 302         s1 += stop - start;
 303         s2 += (stop - start) * (stop - start);
 304     }
 305     pa_log_info("MMX: %llu usec (min = %llu, max = %llu, stddev = %g).", (long long unsigned int)s1,
 306             (long long unsigned int)min, (long long unsigned int)max, sqrt(TIMES2 * s2 - s1 * s1) / TIMES2);
 307
 308     min = INT_MAX; max = 0;
 309     s1 = s2 = 0;
 310     for (k = 0; k < TIMES2; k++) {
 311         start = pa_rtclock_now();
 312         for (j = 0; j < TIMES; j++) {
 313             memcpy(samples_ref, samples_orig, sizeof(samples));
 314             func(samples_ref, volumes, CHANNELS, sizeof(samples));
 315         }
 316         stop = pa_rtclock_now();
 317
 318         if (min > (stop - start)) min = stop - start;
 319         if (max < (stop - start)) max = stop - start;
 320         s1 += stop - start;
 321         s2 += (stop - start) * (stop - start);
 322     }
 323     pa_log_info("ref: %llu usec (min = %llu, max = %llu, stddev = %g).", (long long unsigned int)s1,
 324             (long long unsigned int)min, (long long unsigned int)max, sqrt(TIMES2 * s2 - s1 * s1) / TIMES2);
 325
 326     pa_assert_se(memcmp(samples_ref, samples, sizeof(samples)) == 0);
 327 }
 328 #endif
 329
 330 #endif /* defined (__i386__) || defined (__amd64__) */
 331
 332
 333 void pa_volume_func_init_mmx(pa_cpu_x86_flag_t flags) {
 334 #if defined (__i386__) || defined (__amd64__)
 335
 336 #ifdef RUN_TEST
 337     run_test();
 338 #endif
 339
 340     if ((flags & PA_CPU_X86_MMX) && (flags & PA_CPU_X86_CMOV)) {
 341         pa_log_info("Initialising MMX optimized functions.");
 342
 343         pa_set_volume_func(PA_SAMPLE_S16NE, (pa_do_volume_func_t) pa_volume_s16ne_mmx);
 344         pa_set_volume_func(PA_SAMPLE_S16RE, (pa_do_volume_func_t) pa_volume_s16re_mmx);
 345     }
 346 #endif /* defined (__i386__) || defined (__amd64__) */
 347 }