code.delx.au - pulseaudio/blob - src/pulsecore/svolume_mmx.c

   1 /***
   2   This file is part of PulseAudio.
   3
   4   Copyright 2004-2006 Lennart Poettering
   5   Copyright 2009 Wim Taymans <wim.taymans@collabora.co.uk>
   6
   7   PulseAudio is free software; you can redistribute it and/or modify
   8   it under the terms of the GNU Lesser General Public License as published
   9   by the Free Software Foundation; either version 2.1 of the License,
  10   or (at your option) any later version.
  11
  12   PulseAudio is distributed in the hope that it will be useful, but
  13   WITHOUT ANY WARRANTY; without even the implied warranty of
  14   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  15   General Public License for more details.
  16
  17   You should have received a copy of the GNU Lesser General Public License
  18   along with PulseAudio; if not, write to the Free Software
  19   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
  20   USA.
  21 ***/
  22
  23 #ifdef HAVE_CONFIG_H
  24 #include <config.h>
  25 #endif
  26
  27 #include <pulse/timeval.h>
  28 #include <pulsecore/random.h>
  29 #include <pulsecore/macro.h>
  30 #include <pulsecore/g711.h>
  31 #include <pulsecore/core-util.h>
  32
  33 #include "cpu-x86.h"
  34
  35 #include "sample-util.h"
  36 #include "endianmacros.h"
  37
  38 #if defined (__i386__) || defined (__amd64__)
  39 /* in s: 2 int16_t samples
  40  * in v: 2 int32_t volumes, fixed point 16:16
  41  * out s: contains scaled and clamped int16_t samples.
  42  *
  43  * We calculate the high 32 bits of a 32x16 multiply which we then
  44  * clamp to 16 bits. The calulcation is:
  45  *
  46  *  vl = (v & 0xffff)
  47  *  vh = (v >> 16)
  48  *  s = ((s * vl) >> 16) + (s * vh);
  49  *
  50  * For the first multiply we have to do a sign correction as we need to
  51  * multiply a signed int with an unsigned int. Hacker's delight 8-3 gives a
  52  * simple formula to correct the sign of the high word after the signed
  53  * multiply.
  54  */
  55 #define VOLUME_32x16(s,v)                  /* .. |   vh  |   vl  | */                   \
  56       " pxor  %%mm4, %%mm4           \n\t" /* .. |    0  |    0  | */                   \
  57       " punpcklwd %%mm4, "#s"        \n\t" /* .. |    0  |   p0  | */                   \
  58       " pcmpgtw "#v", %%mm4          \n\t" /* .. |    0  | s(vl) | */                   \
  59       " pand "#s", %%mm4             \n\t" /* .. |    0  |  (p0) |  (vl >> 15) & p */   \
  60       " movq "#s", %%mm5             \n\t"                                              \
  61       " pmulhw "#v", "#s"            \n\t" /* .. |    0  | vl*p0 | */                   \
  62       " paddw %%mm4, "#s"            \n\t" /* .. |    0  | vl*p0 | + sign correct */    \
  63       " psrld $16, "#v"              \n\t" /* .. |    0  |   vh  | */                   \
  64       " pmaddwd %%mm5, "#v"          \n\t" /* .. |    p0 * vh    | */                   \
  65       " paddd "#s", "#v"             \n\t" /* .. |    p0 * v0    | */                   \
  66       " packssdw "#v", "#v"          \n\t" /* .. | p1*v1 | p0*v0 | */
  67
  68 /* approximately advances %3 = (%3 + a) % b. This function requires that
  69  * a <= b. */
  70 #define MOD_ADD(a,b) \
  71       " add "#a", %3                 \n\t" \
  72       " mov %3, %4                   \n\t" \
  73       " sub "#b", %4                 \n\t" \
  74       " cmovae %4, %3                \n\t"
  75
  76 /* swap 16 bits */
  77 #define SWAP_16(s) \
  78       " movq "#s", %%mm4             \n\t" /* .. |  h  l |  */ \
  79       " psrlw $8, %%mm4              \n\t" /* .. |  0  h |  */ \
  80       " psllw $8, "#s"               \n\t" /* .. |  l  0 |  */ \
  81       " por %%mm4, "#s"              \n\t" /* .. |  l  h |  */
  82
  83 /* swap 2 registers 16 bits for better pairing */
  84 #define SWAP_16_2(s1,s2) \
  85       " movq "#s1", %%mm4            \n\t" /* .. |  h  l |  */ \
  86       " movq "#s2", %%mm5            \n\t"                     \
  87       " psrlw $8, %%mm4              \n\t" /* .. |  0  h |  */ \
  88       " psrlw $8, %%mm5              \n\t"                     \
  89       " psllw $8, "#s1"              \n\t" /* .. |  l  0 |  */ \
  90       " psllw $8, "#s2"              \n\t"                     \
  91       " por %%mm4, "#s1"             \n\t" /* .. |  l  h |  */ \
  92       " por %%mm5, "#s2"             \n\t"
  93
  94 static void
  95 pa_volume_s16ne_mmx (int16_t *samples, int32_t *volumes, unsigned channels, unsigned length)
  96 {
  97     pa_reg_x86 channel, temp;
  98
  99     /* the max number of samples we process at a time, this is also the max amount
 100      * we overread the volume array, which should have enough padding. */
 101     channels = PA_MAX (4U, channels);
 102
 103     __asm__ __volatile__ (
 104         " xor %3, %3                    \n\t"
 105         " sar $1, %2                    \n\t" /* length /= sizeof (int16_t) */
 106
 107         " test $1, %2                   \n\t" /* check for odd samples */
 108         " je 2f                         \n\t"
 109
 110         " movd (%1, %3, 4), %%mm0       \n\t" /* |  v0h  |  v0l  | */
 111         " movw (%0), %w4                \n\t" /*     ..  |  p0   | */
 112         " movd %4, %%mm1                \n\t"
 113         VOLUME_32x16 (%%mm1, %%mm0)
 114         " movd %%mm0, %4                \n\t" /*     ..  | p0*v0 | */
 115         " movw %w4, (%0)                \n\t"
 116         " add $2, %0                    \n\t"
 117         MOD_ADD ($1, %5)
 118
 119         "2:                             \n\t"
 120         " sar $1, %2                    \n\t" /* prepare for processing 2 samples at a time */
 121         " test $1, %2                   \n\t" /* check for odd samples */
 122         " je 4f                         \n\t"
 123
 124         "3:                             \n\t" /* do samples in groups of 2 */
 125         " movq (%1, %3, 4), %%mm0       \n\t" /* |  v1h  |  v1l  |  v0h  |  v0l  | */
 126         " movd (%0), %%mm1              \n\t" /*              .. |   p1  |  p0   | */
 127         VOLUME_32x16 (%%mm1, %%mm0)
 128         " movd %%mm0, (%0)              \n\t" /*              .. | p1*v1 | p0*v0 | */
 129         " add $4, %0                    \n\t"
 130         MOD_ADD ($2, %5)
 131
 132         "4:                             \n\t"
 133         " sar $1, %2                    \n\t" /* prepare for processing 4 samples at a time */
 134         " cmp $0, %2                    \n\t"
 135         " je 6f                         \n\t"
 136
 137         "5:                             \n\t" /* do samples in groups of 4 */
 138         " movq (%1, %3, 4), %%mm0       \n\t" /* |  v1h  |  v1l  |  v0h  |  v0l  | */
 139         " movq 8(%1, %3, 4), %%mm2      \n\t" /* |  v3h  |  v3l  |  v2h  |  v2l  | */
 140         " movd (%0), %%mm1              \n\t" /*              .. |   p1  |  p0   | */
 141         " movd 4(%0), %%mm3             \n\t" /*              .. |   p3  |  p2   | */
 142         VOLUME_32x16 (%%mm1, %%mm0)
 143         VOLUME_32x16 (%%mm3, %%mm2)
 144         " movd %%mm0, (%0)              \n\t" /*              .. | p1*v1 | p0*v0 | */
 145         " movd %%mm2, 4(%0)             \n\t" /*              .. | p3*v3 | p2*v2 | */
 146         " add $8, %0                    \n\t"
 147         MOD_ADD ($4, %5)
 148         " dec %2                        \n\t"
 149         " jne 5b                        \n\t"
 150
 151         "6:                             \n\t"
 152         " emms                          \n\t"
 153
 154         : "+r" (samples), "+r" (volumes), "+r" (length), "=D" ((pa_reg_x86)channel), "=&r" (temp)
 155         : "r" ((pa_reg_x86)channels)
 156         : "cc"
 157     );
 158 }
 159
 160 static void
 161 pa_volume_s16re_mmx (int16_t *samples, int32_t *volumes, unsigned channels, unsigned length)
 162 {
 163     pa_reg_x86 channel, temp;
 164
 165     /* the max number of samples we process at a time, this is also the max amount
 166      * we overread the volume array, which should have enough padding. */
 167     channels = PA_MAX (4U, channels);
 168
 169     __asm__ __volatile__ (
 170         " xor %3, %3                    \n\t"
 171         " sar $1, %2                    \n\t" /* length /= sizeof (int16_t) */
 172         " pcmpeqw %%mm6, %%mm6          \n\t" /* .. |  ffff |  ffff | */
 173         " pcmpeqw %%mm7, %%mm7          \n\t" /* .. |  ffff |  ffff | */
 174         " pslld  $16, %%mm6             \n\t" /* .. |  ffff |     0 | */
 175         " psrld  $31, %%mm7             \n\t" /* .. |     0 |     1 | */
 176
 177         " test $1, %2                   \n\t" /* check for odd samples */
 178         " je 2f                         \n\t"
 179
 180         " movd (%1, %3, 4), %%mm0       \n\t" /* |  v0h  |  v0l  | */
 181         " movw (%0), %w4                \n\t" /*     ..  |  p0   | */
 182         " rorw $8, %w4                  \n\t"
 183         " movd %4, %%mm1                \n\t"
 184         VOLUME_32x16 (%%mm1, %%mm0)
 185         " movd %%mm0, %4                \n\t" /*     ..  | p0*v0 | */
 186         " rorw $8, %w4                  \n\t"
 187         " movw %w4, (%0)                \n\t"
 188         " add $2, %0                    \n\t"
 189         MOD_ADD ($1, %5)
 190
 191         "2:                             \n\t"
 192         " sar $1, %2                    \n\t" /* prepare for processing 2 samples at a time */
 193         " test $1, %2                   \n\t" /* check for odd samples */
 194         " je 4f                         \n\t"
 195
 196         "3:                             \n\t" /* do samples in groups of 2 */
 197         " movq (%1, %3, 4), %%mm0       \n\t" /* |  v1h  |  v1l  |  v0h  |  v0l  | */
 198         " movd (%0), %%mm1              \n\t" /*              .. |   p1  |  p0   | */
 199         SWAP_16 (%%mm1)
 200         VOLUME_32x16 (%%mm1, %%mm0)
 201         SWAP_16 (%%mm0)
 202         " movd %%mm0, (%0)              \n\t" /*              .. | p1*v1 | p0*v0 | */
 203         " add $4, %0                    \n\t"
 204         MOD_ADD ($2, %5)
 205
 206         "4:                             \n\t"
 207         " sar $1, %2                    \n\t" /* prepare for processing 4 samples at a time */
 208         " cmp $0, %2                    \n\t"
 209         " je 6f                         \n\t"
 210
 211         "5:                             \n\t" /* do samples in groups of 4 */
 212         " movq (%1, %3, 4), %%mm0       \n\t" /* |  v1h  |  v1l  |  v0h  |  v0l  | */
 213         " movq 8(%1, %3, 4), %%mm2      \n\t" /* |  v3h  |  v3l  |  v2h  |  v2l  | */
 214         " movd (%0), %%mm1              \n\t" /*              .. |   p1  |  p0   | */
 215         " movd 4(%0), %%mm3             \n\t" /*              .. |   p3  |  p2   | */
 216         SWAP_16_2 (%%mm1, %%mm3)
 217         VOLUME_32x16 (%%mm1, %%mm0)
 218         VOLUME_32x16 (%%mm3, %%mm2)
 219         SWAP_16_2 (%%mm0, %%mm2)
 220         " movd %%mm0, (%0)              \n\t" /*              .. | p1*v1 | p0*v0 | */
 221         " movd %%mm2, 4(%0)             \n\t" /*              .. | p3*v3 | p2*v2 | */
 222         " add $8, %0                    \n\t"
 223         MOD_ADD ($4, %5)
 224         " dec %2                        \n\t"
 225         " jne 5b                        \n\t"
 226
 227         "6:                             \n\t"
 228         " emms                          \n\t"
 229
 230         : "+r" (samples), "+r" (volumes), "+r" (length), "=D" ((pa_reg_x86)channel), "=&r" (temp)
 231         : "r" ((pa_reg_x86)channels)
 232         : "cc"
 233     );
 234 }
 235
 236 #undef RUN_TEST
 237
 238 #ifdef RUN_TEST
 239 #define CHANNELS 2
 240 #define SAMPLES 1021
 241 #define TIMES 1000
 242 #define PADDING 16
 243
 244 static void run_test (void) {
 245     int16_t samples[SAMPLES];
 246     int16_t samples_ref[SAMPLES];
 247     int16_t samples_orig[SAMPLES];
 248     int32_t volumes[CHANNELS + PADDING];
 249     int i, j, padding;
 250     pa_do_volume_func_t func;
 251     pa_usec_t start, stop;
 252
 253     func = pa_get_volume_func (PA_SAMPLE_S16NE);
 254
 255     printf ("checking MMX %zd\n", sizeof (samples));
 256
 257     pa_random (samples, sizeof (samples));
 258     memcpy (samples_ref, samples, sizeof (samples));
 259     memcpy (samples_orig, samples, sizeof (samples));
 260
 261     for (i = 0; i < CHANNELS; i++)
 262         volumes[i] = rand() >> 1;
 263     for (padding = 0; padding < PADDING; padding++, i++)
 264         volumes[i] = volumes[padding];
 265
 266     func (samples_ref, volumes, CHANNELS, sizeof (samples));
 267     pa_volume_s16ne_mmx (samples, volumes, CHANNELS, sizeof (samples));
 268     for (i = 0; i < SAMPLES; i++) {
 269         if (samples[i] != samples_ref[i]) {
 270             printf ("%d: %04x != %04x (%04x * %04x)\n", i, samples[i], samples_ref[i],
 271                   samples_orig[i], volumes[i % CHANNELS]);
 272         }
 273     }
 274
 275     start = pa_rtclock_now();
 276     for (j = 0; j < TIMES; j++) {
 277         memcpy (samples, samples_orig, sizeof (samples));
 278         pa_volume_s16ne_mmx (samples, volumes, CHANNELS, sizeof (samples));
 279     }
 280     stop = pa_rtclock_now();
 281     pa_log_info("MMX: %llu usec.", (long long unsigned int)(stop - start));
 282
 283     start = pa_rtclock_now();
 284     for (j = 0; j < TIMES; j++) {
 285         memcpy (samples_ref, samples_orig, sizeof (samples));
 286         func (samples_ref, volumes, CHANNELS, sizeof (samples));
 287     }
 288     stop = pa_rtclock_now();
 289     pa_log_info("ref: %llu usec.", (long long unsigned int)(stop - start));
 290 }
 291 #endif
 292
 293 #endif /* defined (__i386__) || defined (__amd64__) */
 294
 295
 296 void pa_volume_func_init_mmx (pa_cpu_x86_flag_t flags) {
 297 #if defined (__i386__) || defined (__amd64__)
 298
 299 #ifdef RUN_TEST
 300     run_test ();
 301 #endif
 302
 303     if (flags & PA_CPU_X86_MMX) {
 304         pa_log_info("Initialising MMX optimized functions.");
 305
 306         pa_set_volume_func (PA_SAMPLE_S16NE, (pa_do_volume_func_t) pa_volume_s16ne_mmx);
 307         pa_set_volume_func (PA_SAMPLE_S16RE, (pa_do_volume_func_t) pa_volume_s16re_mmx);
 308     }
 309 #endif /* defined (__i386__) || defined (__amd64__) */
 310 }