code.delx.au - pulseaudio/blob - src/pulsecore/svolume_sse.c

   1 /***
   2   This file is part of PulseAudio.
   3
   4   Copyright 2004-2006 Lennart Poettering
   5   Copyright 2009 Wim Taymans <wim.taymans@collabora.co.uk>
   6
   7   PulseAudio is free software; you can redistribute it and/or modify
   8   it under the terms of the GNU Lesser General Public License as published
   9   by the Free Software Foundation; either version 2.1 of the License,
  10   or (at your option) any later version.
  11
  12   PulseAudio is distributed in the hope that it will be useful, but
  13   WITHOUT ANY WARRANTY; without even the implied warranty of
  14   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  15   General Public License for more details.
  16
  17   You should have received a copy of the GNU Lesser General Public License
  18   along with PulseAudio; if not, write to the Free Software
  19   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
  20   USA.
  21 ***/
  22
  23 #ifdef HAVE_CONFIG_H
  24 #include <config.h>
  25 #endif
  26
  27 #include <pulse/rtclock.h>
  28
  29 #include <pulsecore/random.h>
  30 #include <pulsecore/macro.h>
  31 #include <pulsecore/endianmacros.h>
  32
  33 #include "cpu-x86.h"
  34
  35 #include "sample-util.h"
  36
  37 #if defined (__i386__) || defined (__amd64__)
  38
  39 #define VOLUME_32x16(s,v)                  /* .. |   vh  |   vl  | */                   \
  40       " pxor %%xmm4, %%xmm4          \n\t" /* .. |    0  |    0  | */                   \
  41       " punpcklwd %%xmm4, "#s"       \n\t" /* .. |    0  |   p0  | */                   \
  42       " pcmpgtw "#s", %%xmm4         \n\t" /* .. |    0  | s(p0) | */                   \
  43       " pand "#v", %%xmm4            \n\t" /* .. |    0  |  (vl) | */                   \
  44       " movdqa "#s", %%xmm5          \n\t"                                              \
  45       " pmulhuw "#v", "#s"           \n\t" /* .. |    0  | vl*p0 | */                   \
  46       " psubd %%xmm4, "#s"           \n\t" /* .. |    0  | vl*p0 | + sign correct */    \
  47       " psrld $16, "#v"              \n\t" /* .. |    0  |   vh  | */                   \
  48       " pmaddwd %%xmm5, "#v"         \n\t" /* .. |    p0 * vh    | */                   \
  49       " paddd "#s", "#v"             \n\t" /* .. |    p0 * v0    | */                   \
  50       " packssdw "#v", "#v"          \n\t" /* .. | p1*v1 | p0*v0 | */
  51
  52 #define MOD_ADD(a,b) \
  53       " add "#a", %3                 \n\t" /* channel += inc           */ \
  54       " mov %3, %4                   \n\t"                                \
  55       " sub "#b", %4                 \n\t" /* tmp = channel - channels */ \
  56       " cmovae %4, %3                \n\t" /* if (tmp >= 0) channel = tmp  */
  57
  58 /* swap 16 bits */
  59 #define SWAP_16(s) \
  60       " movdqa "#s", %%xmm4          \n\t" /* .. |  h  l |  */ \
  61       " psrlw $8, %%xmm4             \n\t" /* .. |  0  h |  */ \
  62       " psllw $8, "#s"               \n\t" /* .. |  l  0 |  */ \
  63       " por %%xmm4, "#s"             \n\t" /* .. |  l  h |  */
  64
  65 /* swap 2 registers 16 bits for better pairing */
  66 #define SWAP_16_2(s1,s2) \
  67       " movdqa "#s1", %%xmm4         \n\t" /* .. |  h  l |  */ \
  68       " movdqa "#s2", %%xmm5         \n\t"                     \
  69       " psrlw $8, %%xmm4             \n\t" /* .. |  0  h |  */ \
  70       " psrlw $8, %%xmm5             \n\t"                     \
  71       " psllw $8, "#s1"              \n\t" /* .. |  l  0 |  */ \
  72       " psllw $8, "#s2"              \n\t"                     \
  73       " por %%xmm4, "#s1"            \n\t" /* .. |  l  h |  */ \
  74       " por %%xmm5, "#s2"            \n\t"
  75
  76
  77 static int channel_overread_table[8] = {8,8,8,12,8,10,12,14};
  78
  79 static void pa_volume_s16ne_sse2(int16_t *samples, const int32_t *volumes, unsigned channels, unsigned length) {
  80     pa_reg_x86 channel, temp;
  81
  82     /* Channels must be at least 8 and always a multiple of the original number.
  83      * This is also the max amount we overread the volume array, which should
  84      * have enough padding. */
  85     if (channels < 8)
  86         channels = channel_overread_table[channels];
  87
  88     __asm__ __volatile__ (
  89         " xor %3, %3                    \n\t"
  90         " sar $1, %2                    \n\t" /* length /= sizeof (int16_t) */
  91
  92         " test $1, %2                   \n\t" /* check for odd samples */
  93         " je 2f                         \n\t"
  94
  95         " movd (%q1, %3, 4), %%xmm0     \n\t" /* |  v0h  |  v0l  | */
  96         " movw (%0), %w4                \n\t" /*     ..  |   p0  | */
  97         " movd %4, %%xmm1               \n\t"
  98         VOLUME_32x16 (%%xmm1, %%xmm0)
  99         " movd %%xmm0, %4               \n\t" /*     ..  | p0*v0 | */
 100         " movw %w4, (%0)                \n\t"
 101         " add $2, %0                    \n\t"
 102         MOD_ADD ($1, %5)
 103
 104         "2:                             \n\t"
 105         " sar $1, %2                    \n\t" /* prepare for processing 2 samples at a time */
 106         " test $1, %2                   \n\t"
 107         " je 4f                         \n\t"
 108
 109         "3:                             \n\t" /* do samples in groups of 2 */
 110         " movq (%q1, %3, 4), %%xmm0     \n\t" /* |  v1h  |  v1l  |  v0h  |  v0l  | */
 111         " movd (%0), %%xmm1             \n\t" /*              .. |   p1  |  p0   | */
 112         VOLUME_32x16 (%%xmm1, %%xmm0)
 113         " movd %%xmm0, (%0)             \n\t" /*              .. | p1*v1 | p0*v0 | */
 114         " add $4, %0                    \n\t"
 115         MOD_ADD ($2, %5)
 116
 117         "4:                             \n\t"
 118         " sar $1, %2                    \n\t" /* prepare for processing 4 samples at a time */
 119         " test $1, %2                   \n\t"
 120         " je 6f                         \n\t"
 121
 122         /* FIXME, we can do aligned access of the volume values if we can guarantee
 123          * that the array is 16 bytes aligned, we probably have to do the odd values
 124          * after this then. */
 125         "5:                             \n\t" /* do samples in groups of 4 */
 126         " movdqu (%q1, %3, 4), %%xmm0   \n\t" /* |  v3h  |  v3l  ..  v0h  |  v0l  | */
 127         " movq (%0), %%xmm1             \n\t" /*              .. |   p3  ..  p0   | */
 128         VOLUME_32x16 (%%xmm1, %%xmm0)
 129         " movq %%xmm0, (%0)             \n\t" /*              .. | p3*v3 .. p0*v0 | */
 130         " add $8, %0                    \n\t"
 131         MOD_ADD ($4, %5)
 132
 133         "6:                             \n\t"
 134         " sar $1, %2                    \n\t" /* prepare for processing 8 samples at a time */
 135         " cmp $0, %2                    \n\t"
 136         " je 8f                         \n\t"
 137
 138         "7:                             \n\t" /* do samples in groups of 8 */
 139         " movdqu (%q1, %3, 4), %%xmm0   \n\t" /* |  v3h  |  v3l  ..  v0h  |  v0l  | */
 140         " movdqu 16(%q1, %3, 4), %%xmm2 \n\t" /* |  v7h  |  v7l  ..  v4h  |  v4l  | */
 141         " movq (%0), %%xmm1             \n\t" /*              .. |   p3  ..  p0   | */
 142         " movq 8(%0), %%xmm3            \n\t" /*              .. |   p7  ..  p4   | */
 143         VOLUME_32x16 (%%xmm1, %%xmm0)
 144         VOLUME_32x16 (%%xmm3, %%xmm2)
 145         " movq %%xmm0, (%0)             \n\t" /*              .. | p3*v3 .. p0*v0 | */
 146         " movq %%xmm2, 8(%0)            \n\t" /*              .. | p7*v7 .. p4*v4 | */
 147         " add $16, %0                   \n\t"
 148         MOD_ADD ($8, %5)
 149         " dec %2                        \n\t"
 150         " jne 7b                        \n\t"
 151         "8:                             \n\t"
 152
 153         : "+r" (samples), "+r" (volumes), "+r" (length), "=D" (channel), "=&r" (temp)
 154 #if defined (__i386__)
 155         : "m" (channels)
 156 #else
 157         : "r" ((pa_reg_x86)channels)
 158 #endif
 159         : "cc"
 160     );
 161 }
 162
 163 static void pa_volume_s16re_sse2(int16_t *samples, const int32_t *volumes, unsigned channels, unsigned length) {
 164     pa_reg_x86 channel, temp;
 165
 166     /* Channels must be at least 8 and always a multiple of the original number.
 167      * This is also the max amount we overread the volume array, which should
 168      * have enough padding. */
 169     if (channels < 8)
 170         channels = channel_overread_table[channels];
 171
 172     __asm__ __volatile__ (
 173         " xor %3, %3                    \n\t"
 174         " sar $1, %2                    \n\t" /* length /= sizeof (int16_t) */
 175
 176         " test $1, %2                   \n\t" /* check for odd samples */
 177         " je 2f                         \n\t"
 178
 179         " movd (%q1, %3, 4), %%xmm0     \n\t" /* |  v0h  |  v0l  | */
 180         " movw (%0), %w4                \n\t" /*     ..  |   p0  | */
 181         " rorw $8, %w4                  \n\t"
 182         " movd %4, %%xmm1               \n\t"
 183         VOLUME_32x16 (%%xmm1, %%xmm0)
 184         " movd %%xmm0, %4               \n\t" /*     ..  | p0*v0 | */
 185         " rorw $8, %w4                  \n\t"
 186         " movw %w4, (%0)                \n\t"
 187         " add $2, %0                    \n\t"
 188         MOD_ADD ($1, %5)
 189
 190         "2:                             \n\t"
 191         " sar $1, %2                    \n\t" /* prepare for processing 2 samples at a time */
 192         " test $1, %2                   \n\t"
 193         " je 4f                         \n\t"
 194
 195         "3:                             \n\t" /* do samples in groups of 2 */
 196         " movq (%q1, %3, 4), %%xmm0     \n\t" /* |  v1h  |  v1l  |  v0h  |  v0l  | */
 197         " movd (%0), %%xmm1             \n\t" /*              .. |   p1  |  p0   | */
 198         SWAP_16 (%%xmm1)
 199         VOLUME_32x16 (%%xmm1, %%xmm0)
 200         SWAP_16 (%%xmm0)
 201         " movd %%xmm0, (%0)             \n\t" /*              .. | p1*v1 | p0*v0 | */
 202         " add $4, %0                    \n\t"
 203         MOD_ADD ($2, %5)
 204
 205         "4:                             \n\t"
 206         " sar $1, %2                    \n\t" /* prepare for processing 4 samples at a time */
 207         " test $1, %2                   \n\t"
 208         " je 6f                         \n\t"
 209
 210         /* FIXME, we can do aligned access of the volume values if we can guarantee
 211          * that the array is 16 bytes aligned, we probably have to do the odd values
 212          * after this then. */
 213         "5:                             \n\t" /* do samples in groups of 4 */
 214         " movdqu (%q1, %3, 4), %%xmm0   \n\t" /* |  v3h  |  v3l  ..  v0h  |  v0l  | */
 215         " movq (%0), %%xmm1             \n\t" /*              .. |   p3  ..  p0   | */
 216         SWAP_16 (%%xmm1)
 217         VOLUME_32x16 (%%xmm1, %%xmm0)
 218         SWAP_16 (%%xmm0)
 219         " movq %%xmm0, (%0)             \n\t" /*              .. | p3*v3 .. p0*v0 | */
 220         " add $8, %0                    \n\t"
 221         MOD_ADD ($4, %5)
 222
 223         "6:                             \n\t"
 224         " sar $1, %2                    \n\t" /* prepare for processing 8 samples at a time */
 225         " cmp $0, %2                    \n\t"
 226         " je 8f                         \n\t"
 227
 228         "7:                             \n\t" /* do samples in groups of 8 */
 229         " movdqu (%q1, %3, 4), %%xmm0   \n\t" /* |  v3h  |  v3l  ..  v0h  |  v0l  | */
 230         " movdqu 16(%q1, %3, 4), %%xmm2 \n\t" /* |  v7h  |  v7l  ..  v4h  |  v4l  | */
 231         " movq (%0), %%xmm1             \n\t" /*              .. |   p3  ..  p0   | */
 232         " movq 8(%0), %%xmm3            \n\t" /*              .. |   p7  ..  p4   | */
 233         SWAP_16_2 (%%xmm1, %%xmm3)
 234         VOLUME_32x16 (%%xmm1, %%xmm0)
 235         VOLUME_32x16 (%%xmm3, %%xmm2)
 236         SWAP_16_2 (%%xmm0, %%xmm2)
 237         " movq %%xmm0, (%0)             \n\t" /*              .. | p3*v3 .. p0*v0 | */
 238         " movq %%xmm2, 8(%0)            \n\t" /*              .. | p7*v7 .. p4*v4 | */
 239         " add $16, %0                   \n\t"
 240         MOD_ADD ($8, %5)
 241         " dec %2                        \n\t"
 242         " jne 7b                        \n\t"
 243         "8:                             \n\t"
 244
 245         : "+r" (samples), "+r" (volumes), "+r" (length), "=D" (channel), "=&r" (temp)
 246 #if defined (__i386__)
 247         : "m" (channels)
 248 #else
 249         : "r" ((pa_reg_x86)channels)
 250 #endif
 251         : "cc"
 252     );
 253 }
 254
 255 #undef RUN_TEST
 256
 257 #ifdef RUN_TEST
 258 #define CHANNELS 2
 259 #define SAMPLES 1022
 260 #define TIMES 1000
 261 #define TIMES2 100
 262 #define PADDING 16
 263
 264 static void run_test(void) {
 265     int16_t samples[SAMPLES];
 266     int16_t samples_ref[SAMPLES];
 267     int16_t samples_orig[SAMPLES];
 268     int32_t volumes[CHANNELS + PADDING];
 269     int i, j, padding;
 270     pa_do_volume_func_t func;
 271     pa_usec_t start, stop;
 272     int k;
 273     pa_usec_t min = INT_MAX, max = 0;
 274     double s1 = 0, s2 = 0;
 275
 276     func = pa_get_volume_func(PA_SAMPLE_S16NE);
 277
 278     printf("checking SSE2 %zd\n", sizeof(samples));
 279
 280     pa_random(samples, sizeof(samples));
 281     memcpy(samples_ref, samples, sizeof(samples));
 282     memcpy(samples_orig, samples, sizeof(samples));
 283
 284     for (i = 0; i < CHANNELS; i++)
 285         volumes[i] = PA_CLAMP_VOLUME(rand() >> 15);
 286     for (padding = 0; padding < PADDING; padding++, i++)
 287         volumes[i] = volumes[padding];
 288
 289     func(samples_ref, volumes, CHANNELS, sizeof(samples));
 290     pa_volume_s16ne_sse2(samples, volumes, CHANNELS, sizeof(samples));
 291     for (i = 0; i < SAMPLES; i++) {
 292         if (samples[i] != samples_ref[i]) {
 293             printf ("%d: %04x != %04x (%04x * %04x)\n", i, samples[i], samples_ref[i],
 294                       samples_orig[i], volumes[i % CHANNELS]);
 295         }
 296     }
 297
 298     for (k = 0; k < TIMES2; k++) {
 299         start = pa_rtclock_now();
 300         for (j = 0; j < TIMES; j++) {
 301             memcpy(samples, samples_orig, sizeof(samples));
 302             pa_volume_s16ne_sse2(samples, volumes, CHANNELS, sizeof(samples));
 303         }
 304         stop = pa_rtclock_now();
 305
 306         if (min > (stop - start)) min = stop - start;
 307         if (max < (stop - start)) max = stop - start;
 308         s1 += stop - start;
 309         s2 += (stop - start) * (stop - start);
 310     }
 311     pa_log_info("SSE: %llu usec (min = %llu, max = %llu, stddev = %g).", (long long unsigned int)s1,
 312             (long long unsigned int)min, (long long unsigned int)max, sqrt(TIMES2 * s2 - s1 * s1) / TIMES2);
 313
 314     min = INT_MAX; max = 0;
 315     s1 = s2 = 0;
 316     for (k = 0; k < TIMES2; k++) {
 317         start = pa_rtclock_now();
 318         for (j = 0; j < TIMES; j++) {
 319             memcpy(samples_ref, samples_orig, sizeof(samples));
 320             func(samples_ref, volumes, CHANNELS, sizeof(samples));
 321         }
 322         stop = pa_rtclock_now();
 323
 324         if (min > (stop - start)) min = stop - start;
 325         if (max < (stop - start)) max = stop - start;
 326         s1 += stop - start;
 327         s2 += (stop - start) * (stop - start);
 328     }
 329     pa_log_info("ref: %llu usec (min = %llu, max = %llu, stddev = %g).", (long long unsigned int)s1,
 330             (long long unsigned int)min, (long long unsigned int)max, sqrt(TIMES2 * s2 - s1 * s1) / TIMES2);
 331
 332     pa_assert_se(memcmp(samples_ref, samples, sizeof(samples)) == 0);
 333 }
 334 #endif
 335 #endif /* defined (__i386__) || defined (__amd64__) */
 336
 337 void pa_volume_func_init_sse(pa_cpu_x86_flag_t flags) {
 338 #if defined (__i386__) || defined (__amd64__)
 339
 340 #ifdef RUN_TEST
 341     run_test();
 342 #endif
 343
 344     if (flags & PA_CPU_X86_SSE2) {
 345         pa_log_info("Initialising SSE2 optimized volume functions.");
 346
 347         pa_set_volume_func(PA_SAMPLE_S16NE, (pa_do_volume_func_t) pa_volume_s16ne_sse2);
 348         pa_set_volume_func(PA_SAMPLE_S16RE, (pa_do_volume_func_t) pa_volume_s16re_sse2);
 349     }
 350 #endif /* defined (__i386__) || defined (__amd64__) */
 351 }