mirror of https://github.com/postgres/postgres
Presently, pg_popcount() processes data in 32-bit or 64-bit chunks when possible. Newer hardware that supports AVX-512 instructions can use 512-bit chunks, which provides a nice speedup, especially for larger buffers. This commit introduces the infrastructure required to detect compiler and CPU support for the required AVX-512 intrinsic functions, and it adds a new pg_popcount() implementation that uses these functions. If CPU support for this optimized implementation is detected at runtime, a function pointer is updated so that it is used by subsequent calls to pg_popcount(). Most of the existing in-tree calls to pg_popcount() should benefit from these instructions, and calls with smaller buffers should at least not regress compared to v16. The new infrastructure introduced by this commit can also be used to optimize visibilitymap_count(), but that is left for a follow-up commit. Co-authored-by: Paul Amonson, Ants Aasma Reviewed-by: Matthias van de Meent, Tom Lane, Noah Misch, Akash Shankaran, Alvaro Herrera, Andres Freund, David Rowley Discussion: https://postgr.es/m/BL1PR11MB5304097DF7EA81D04C33F3D1DCA6A%40BL1PR11MB5304.namprd11.prod.outlook.compull/159/head
Nathan Bossart
1 year ago
parent
158f581923
commit
792752af4e
@ -0,0 +1,81 @@ |
||||
/*-------------------------------------------------------------------------
|
||||
* |
||||
* pg_popcount_avx512.c |
||||
* Holds the AVX-512 pg_popcount() implementation. |
||||
* |
||||
* Copyright (c) 2024, PostgreSQL Global Development Group |
||||
* |
||||
* IDENTIFICATION |
||||
* src/port/pg_popcount_avx512.c |
||||
* |
||||
*------------------------------------------------------------------------- |
||||
*/ |
||||
#include "c.h" |
||||
|
||||
#include <immintrin.h> |
||||
|
||||
#include "port/pg_bitutils.h" |
||||
|
||||
/*
|
||||
* It's probably unlikely that TRY_POPCNT_FAST won't be set if we are able to |
||||
* use AVX-512 intrinsics, but we check it anyway to be sure. We piggy-back on |
||||
* the function pointers that are only used when TRY_POPCNT_FAST is set. |
||||
*/ |
||||
#ifdef TRY_POPCNT_FAST |
||||
|
||||
/*
|
||||
* pg_popcount_avx512 |
||||
* Returns the number of 1-bits in buf |
||||
*/ |
||||
uint64 |
||||
pg_popcount_avx512(const char *buf, int bytes) |
||||
{ |
||||
__m512i val, |
||||
cnt; |
||||
__m512i accum = _mm512_setzero_si512(); |
||||
const char *final; |
||||
int tail_idx; |
||||
__mmask64 mask = ~UINT64CONST(0); |
||||
|
||||
/*
|
||||
* Align buffer down to avoid double load overhead from unaligned access. |
||||
* Calculate a mask to ignore preceding bytes. Find start offset of final |
||||
* iteration and ensure it is not empty. |
||||
*/ |
||||
mask <<= ((uintptr_t) buf) % sizeof(__m512i); |
||||
tail_idx = (((uintptr_t) buf + bytes - 1) % sizeof(__m512i)) + 1; |
||||
final = (const char *) TYPEALIGN_DOWN(sizeof(__m512i), buf + bytes - 1); |
||||
buf = (const char *) TYPEALIGN_DOWN(sizeof(__m512i), buf); |
||||
|
||||
/*
|
||||
* Iterate through all but the final iteration. Starting from the second |
||||
* iteration, the mask is ignored. |
||||
*/ |
||||
if (buf < final) |
||||
{ |
||||
val = _mm512_maskz_loadu_epi8(mask, (const __m512i *) buf); |
||||
cnt = _mm512_popcnt_epi64(val); |
||||
accum = _mm512_add_epi64(accum, cnt); |
||||
|
||||
buf += sizeof(__m512i); |
||||
mask = ~UINT64CONST(0); |
||||
|
||||
for (; buf < final; buf += sizeof(__m512i)) |
||||
{ |
||||
val = _mm512_load_si512((const __m512i *) buf); |
||||
cnt = _mm512_popcnt_epi64(val); |
||||
accum = _mm512_add_epi64(accum, cnt); |
||||
} |
||||
} |
||||
|
||||
/* Final iteration needs to ignore bytes that are not within the length */ |
||||
mask &= (~UINT64CONST(0) >> (sizeof(__m512i) - tail_idx)); |
||||
|
||||
val = _mm512_maskz_loadu_epi8(mask, (const __m512i *) buf); |
||||
cnt = _mm512_popcnt_epi64(val); |
||||
accum = _mm512_add_epi64(accum, cnt); |
||||
|
||||
return _mm512_reduce_add_epi64(accum); |
||||
} |
||||
|
||||
#endif /* TRY_POPCNT_FAST */ |
||||
@ -0,0 +1,88 @@ |
||||
/*-------------------------------------------------------------------------
|
||||
* |
||||
* pg_popcount_avx512_choose.c |
||||
* Test whether we can use the AVX-512 pg_popcount() implementation. |
||||
* |
||||
* Copyright (c) 2024, PostgreSQL Global Development Group |
||||
* |
||||
* IDENTIFICATION |
||||
* src/port/pg_popcount_avx512_choose.c |
||||
* |
||||
*------------------------------------------------------------------------- |
||||
*/ |
||||
#include "c.h" |
||||
|
||||
#if defined(HAVE__GET_CPUID) || defined(HAVE__GET_CPUID_COUNT) |
||||
#include <cpuid.h> |
||||
#endif |
||||
|
||||
#ifdef HAVE_XSAVE_INTRINSICS |
||||
#include <immintrin.h> |
||||
#endif |
||||
|
||||
#if defined(HAVE__CPUID) || defined(HAVE__CPUIDEX) |
||||
#include <intrin.h> |
||||
#endif |
||||
|
||||
#include "port/pg_bitutils.h" |
||||
|
||||
/*
|
||||
* It's probably unlikely that TRY_POPCNT_FAST won't be set if we are able to |
||||
* use AVX-512 intrinsics, but we check it anyway to be sure. We piggy-back on |
||||
* the function pointers that are only used when TRY_POPCNT_FAST is set. |
||||
*/ |
||||
#ifdef TRY_POPCNT_FAST |
||||
|
||||
/*
|
||||
* Returns true if the CPU supports the instructions required for the AVX-512 |
||||
* pg_popcount() implementation. |
||||
*/ |
||||
bool |
||||
pg_popcount_avx512_available(void) |
||||
{ |
||||
unsigned int exx[4] = {0, 0, 0, 0}; |
||||
|
||||
/* Does CPUID say there's support for AVX-512 popcount instructions? */ |
||||
#if defined(HAVE__GET_CPUID_COUNT) |
||||
__get_cpuid_count(7, 0, &exx[0], &exx[1], &exx[2], &exx[3]); |
||||
#elif defined(HAVE__CPUIDEX) |
||||
__cpuidex(exx, 7, 0); |
||||
#else |
||||
#error cpuid instruction not available |
||||
#endif |
||||
if ((exx[2] & (1 << 14)) == 0) /* avx512-vpopcntdq */ |
||||
return false; |
||||
|
||||
/* Does CPUID say there's support for AVX-512 byte and word instructions? */ |
||||
memset(exx, 0, sizeof(exx)); |
||||
#if defined(HAVE__GET_CPUID_COUNT) |
||||
__get_cpuid_count(7, 0, &exx[0], &exx[1], &exx[2], &exx[3]); |
||||
#elif defined(HAVE__CPUIDEX) |
||||
__cpuidex(exx, 7, 0); |
||||
#else |
||||
#error cpuid instruction not available |
||||
#endif |
||||
if ((exx[1] & (1 << 30)) == 0) /* avx512-bw */ |
||||
return false; |
||||
|
||||
/* Does CPUID say there's support for XSAVE instructions? */ |
||||
memset(exx, 0, sizeof(exx)); |
||||
#if defined(HAVE__GET_CPUID) |
||||
__get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]); |
||||
#elif defined(HAVE__CPUID) |
||||
__cpuid(exx, 1); |
||||
#else |
||||
#error cpuid instruction not available |
||||
#endif |
||||
if ((exx[2] & (1 << 26)) == 0) /* xsave */ |
||||
return false; |
||||
|
||||
/* Does XGETBV say the ZMM registers are enabled? */ |
||||
#ifdef HAVE_XSAVE_INTRINSICS |
||||
return (_xgetbv(0) & 0xe0) != 0; |
||||
#else |
||||
return false; |
||||
#endif |
||||
} |
||||
|
||||
#endif /* TRY_POPCNT_FAST */ |
||||
Loading…
Reference in new issue