From 8fae5f853e190e809b5f2bdee45cab7b22efe15a Mon Sep 17 00:00:00 2001 From: teooooozhang Date: Mon, 12 Jun 2023 09:58:46 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E3=80=90=E5=9B=9E=E5=90=88=E3=80=91shared?= =?UTF-8?q?=5Fbuffer=E6=94=AF=E6=8C=81=E5=A4=A7=E9=A1=B5=E5=86=85=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/gs_guc/cluster_guc.conf | 2 + src/bin/gs_guc/cluster_guc.cpp | 1 + src/common/backend/port/sysv_shmem.cpp | 72 ++++++++++++++++++- .../backend/utils/misc/guc/guc_storage.cpp | 29 +++++++- .../utils/misc/postgresql_single.conf.sample | 3 + .../knl/knl_guc/knl_instance_attr_storage.h | 2 + .../regress/output/recovery_2pc_tools.source | 2 + 7 files changed, 108 insertions(+), 3 deletions(-) diff --git a/src/bin/gs_guc/cluster_guc.conf b/src/bin/gs_guc/cluster_guc.conf index df97d96ccb..34ae34280a 100755 --- a/src/bin/gs_guc/cluster_guc.conf +++ b/src/bin/gs_guc/cluster_guc.conf @@ -32,6 +32,7 @@ seq_page_cost|real|0,1.79769e+308|NULL|NULL| alarm_component|string|0,0|NULL|NULL| alarm_report_interval|int|0,2147483647|NULL|NULL| allow_concurrent_tuple_update|bool|0,0|NULL|NULL| +enable_huge_pages|bool|0,0|NULL|NULL| allow_create_sysobject|bool|0,0|NULL|NULL| allow_system_table_mods|bool|0,0|NULL|NULL| application_name|string|0,0|NULL|NULL| @@ -442,6 +443,7 @@ session_replication_role|enum|origin,replica,local|NULL|When this parameter is s session_timeout|int|0,86400|s|GaussDB Kernel gsql client has an automatic reconnection mechanism, when the timeout, the gsql will be reconnection after disconnection.| idle_in_transaction_session_timeout|int|0,86400|s|Sets the maximum allowed idle time between queries, when in a transaction.| shared_buffers|int|16,1073741823|kB|NULL| +huge_page_size|int|0,1073741823|kB|NULL| shared_preload_libraries|string|0,0|NULL|NULL| show_acce_estimate_detail|bool|0,0|NULL|NULL| skew_option|enum|normal,lazy,off|NULL|NULL| diff --git a/src/bin/gs_guc/cluster_guc.cpp b/src/bin/gs_guc/cluster_guc.cpp index dbf5c96439..00f2a6f02f 100644 --- a/src/bin/gs_guc/cluster_guc.cpp +++ b/src/bin/gs_guc/cluster_guc.cpp @@ -276,6 +276,7 @@ const char* unit_eight_kB_parameter_list[] = { "temp_buffers", "wal_buffers", "wal_segment_size", + "huge_page_size", }; /* the size of page, unit is kB */ #define PAGE_SIZE 8 diff --git a/src/common/backend/port/sysv_shmem.cpp b/src/common/backend/port/sysv_shmem.cpp index 35e61dc884..5b3e424408 100644 --- a/src/common/backend/port/sysv_shmem.cpp +++ b/src/common/backend/port/sysv_shmem.cpp @@ -28,8 +28,10 @@ #endif #include "miscadmin.h" +#include "port/pg_bitutils.h" #include "postmaster/postmaster.h" #include "storage/ipc.h" +#include "storage/smgr/fd.h" #include "storage/pg_shmem.h" #include "securec.h" @@ -47,11 +49,65 @@ typedef int IpcMemoryId; /* shared memory ID returned by shmget(2) */ THR_LOCAL unsigned long UsedShmemSegID = 0; THR_LOCAL void* UsedShmemSegAddr = NULL; +static int GetSystemDefaultHugepagesSize(); +static void GetHugepageSize(Size* hugepageSize, int* flag); static void* InternalIpcMemoryCreate(IpcMemoryKey memKey, Size size); static void IpcMemoryDetach(int status, Datum shmaddr); static void IpcMemoryDelete(int status, Datum shmId); static PGShmemHeader* PGSharedMemoryAttach(IpcMemoryKey key, IpcMemoryId* shmid); +static int GetSystemDefaultHugepagesSize() +{ + int result = 0; + FILE *fp = AllocateFile("/proc/meminfo", "r"); + if (!fp) { + return result; + } + char buf[128]; + unsigned int size; + char ch; + while (fgets(buf, sizeof(buf), fp)) { + if (sscanf_s(buf, "Hugepagesize: %u %c", &size, &ch, sizeof(char)) == 2) { + if (ch == 'k') { + result = ((Size)1024) * size; + break; + } + // extend for mB, gB if needed in the future + } + } + FreeFile(fp); + return result; +} + +static void GetHugepageSize(Size* hugepageSize, int* flag) +{ + Assert(g_instance.attr.attr_storage.enable_huge_pages); +#ifdef SHM_HUGETLB + Size sizeLocal = 0; + if (g_instance.attr.attr_storage.huge_page_size != 0) { + sizeLocal = (Size)(g_instance.attr.attr_storage.huge_page_size * (Size)BLCKSZ); + } else { + sizeLocal = GetSystemDefaultHugepagesSize(); + } + if (sizeLocal == 0) { + /* if we failed to read default size, set hugepage size to 2MB */ + sizeLocal = 2 * 1024 * 1024; + } + int flagLocal = SHM_HUGETLB; + +#if defined(MAP_HUGE_MASK) && defined(MAP_HUGE_SHIFT) + int shift = pg_leftmost_one_pos64(sizeLocal - 1) + 1; + flagLocal |= (shift & MAP_HUGE_MASK) << MAP_HUGE_SHIFT; +#endif + + *hugepageSize = sizeLocal; + *flag = flagLocal; +#else + *hugepageSize = 0; + *flags = 0; +#endif /* SHM_HUGETLB */ +} + /* * InternalIpcMemoryCreate(memKey, size) * @@ -69,7 +125,21 @@ static void* InternalIpcMemoryCreate(IpcMemoryKey memKey, Size size) IpcMemoryId shmid; void* memAddress = NULL; - shmid = shmget(memKey, size, IPC_CREAT | IPC_EXCL | IPCProtection); + if (g_instance.attr.attr_storage.enable_huge_pages) { + Size hugepageSize; + Size allocSize = size; + int hugepageFlag; + GetHugepageSize(&hugepageSize, &hugepageFlag); + // make sure the allocated shared memory size is multiple of hugepage size. + if (allocSize % hugepageSize != 0) { + allocSize += hugepageSize - (allocSize % hugepageSize); + } + ereport(LOG, (errmsg("Allocate shared memory as huge pages. Huge page size: %d KB, required pages count: %d", + (int)hugepageSize / 1024, (int)(allocSize / hugepageSize)))); + shmid = shmget(memKey, allocSize, IPC_CREAT | IPC_EXCL | IPCProtection | hugepageFlag); + } else { + shmid = shmget(memKey, size, IPC_CREAT | IPC_EXCL | IPCProtection); + } if (shmid < 0) { /* * Fail quietly if error indicates a collision with existing segment. diff --git a/src/common/backend/utils/misc/guc/guc_storage.cpp b/src/common/backend/utils/misc/guc/guc_storage.cpp index 8e1c8365d6..85513dea54 100755 --- a/src/common/backend/utils/misc/guc/guc_storage.cpp +++ b/src/common/backend/utils/misc/guc/guc_storage.cpp @@ -1013,6 +1013,17 @@ static void InitStorageConfigureNamesBool() NULL}, #endif + {{"enable_huge_pages", + PGC_POSTMASTER, + NODE_SINGLENODE, + RESOURCES_MEM, + gettext_noop("whether shared memory using huge pages."), + NULL}, + &g_instance.attr.attr_storage.enable_huge_pages, + false, + NULL, + NULL, + NULL}, /* End-of-list marker */ {{NULL, (GucContext)0, @@ -1069,6 +1080,20 @@ static void InitStorageConfigureNamesInt() NULL, NULL, NULL}, + {{"huge_page_size", + PGC_POSTMASTER, + NODE_SINGLENODE, + RESOURCES_MEM, + gettext_noop("Sets the size of huge pages used by the server."), + NULL, + GUC_UNIT_BLOCKS}, + &g_instance.attr.attr_storage.huge_page_size, + 0, + 0, + INT_MAX / 2, + NULL, + NULL, + NULL}, {{"segment_buffers", PGC_POSTMASTER, NODE_ALL, @@ -4448,8 +4473,8 @@ static int IsReplConnInfoChanged(const char* replConnInfo, const char* newval) pfree_ext(newReplStr); return ADD_REPL_CONN_INFO_WITH_NEW_LOCAL_IP_PORT; } - - if (strcmp(ReplInfo_1->localhost, newReplInfo->localhost) != 0 || + + if (strcmp(ReplInfo_1->localhost, newReplInfo->localhost) != 0 || ReplInfo_1->localport != newReplInfo->localport || ReplInfo_1->localheartbeatport != newReplInfo->localheartbeatport) { pfree_ext(newReplInfo); diff --git a/src/common/backend/utils/misc/postgresql_single.conf.sample b/src/common/backend/utils/misc/postgresql_single.conf.sample index 420932a262..366035de2b 100644 --- a/src/common/backend/utils/misc/postgresql_single.conf.sample +++ b/src/common/backend/utils/misc/postgresql_single.conf.sample @@ -138,6 +138,9 @@ session_timeout = 10min # allowed duration of any unused session, 0s-86400s(1 #max_process_memory = 12GB #UDFWorkerMemHardLimit = 1GB +#enable_huge_pages = off # (change requires restart) +#huge_page_size = 0 # make sure huge_page_size is valid for os. 0 as default. + # (change requires restart) #shared_buffers = 32MB # min 128kB # (change requires restart) bulk_write_ring_size = 2GB # for bulkload, max shared_buffers diff --git a/src/include/knl/knl_guc/knl_instance_attr_storage.h b/src/include/knl/knl_guc/knl_instance_attr_storage.h index 5d5ff10bac..cf5287235b 100755 --- a/src/include/knl/knl_guc/knl_instance_attr_storage.h +++ b/src/include/knl/knl_guc/knl_instance_attr_storage.h @@ -159,6 +159,8 @@ typedef struct knl_instance_attr_storage { bool enable_save_confirmed_lsn; #endif int max_active_gtt; + bool enable_huge_pages; + int huge_page_size; } knl_instance_attr_storage; #endif /* SRC_INCLUDE_KNL_KNL_INSTANCE_ATTR_STORAGE_H_ */ diff --git a/src/test/regress/output/recovery_2pc_tools.source b/src/test/regress/output/recovery_2pc_tools.source index ca3b40c757..5935b565d9 100755 --- a/src/test/regress/output/recovery_2pc_tools.source +++ b/src/test/regress/output/recovery_2pc_tools.source @@ -255,6 +255,7 @@ select name,vartype,unit,min_val,max_val from pg_settings where name <> 'qunit_c enable_hashagg | bool | | | enable_hashjoin | bool | | | enable_hdfs_predicate_pushdown | bool | | | + enable_huge_pages | bool | | | enable_hypo_index | bool | | | enable_incremental_catchup | bool | | | enable_incremental_checkpoint | bool | | | @@ -359,6 +360,7 @@ select name,vartype,unit,min_val,max_val from pg_settings where name <> 'qunit_c hll_max_sparse | integer | | -1 | 2147483647 hot_standby | bool | | | hot_standby_feedback | bool | | | + huge_page_size | integer | 8kB | 0 | 1073741823 ident_file | string | | | idle_in_transaction_session_timeout | integer | s | 0 | 86400 ignore_checksum_failure | bool | | | -- Gitee From 60237f760a557261365dad933ec8df2fe9bba4d4 Mon Sep 17 00:00:00 2001 From: teooooozhang Date: Mon, 12 Jun 2023 15:03:34 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E7=A7=BB=E6=A4=8D=E4=BD=8D=E8=BF=90?= =?UTF-8?q?=E7=AE=97=E5=87=BD=E6=95=B0=E7=9B=B8=E5=85=B3=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/port/CMakeLists.txt | 4 + src/common/port/Makefile | 6 +- src/common/port/pg_bitutils.cpp | 308 ++++++++++++++++++++++++++++++++ src/include/port/pg_bitutils.h | 147 +++++++++++++++ 4 files changed, 462 insertions(+), 3 deletions(-) create mode 100644 src/common/port/pg_bitutils.cpp create mode 100644 src/include/port/pg_bitutils.h diff --git a/src/common/port/CMakeLists.txt b/src/common/port/CMakeLists.txt index 92f207efd3..b85ee48650 100755 --- a/src/common/port/CMakeLists.txt +++ b/src/common/port/CMakeLists.txt @@ -18,6 +18,7 @@ execute_process( COMMAND ln -fs ${CMAKE_CURRENT_SOURCE_DIR}/inet_net_ntop.cpp ${CMAKE_CURRENT_SOURCE_DIR}/port_srv/inet_net_ntop.cpp COMMAND ln -fs ${CMAKE_CURRENT_SOURCE_DIR}/noblock.cpp ${CMAKE_CURRENT_SOURCE_DIR}/port_srv/noblock.cpp COMMAND ln -fs ${CMAKE_CURRENT_SOURCE_DIR}/path.cpp ${CMAKE_CURRENT_SOURCE_DIR}/port_srv/path.cpp + COMMAND ln -fs ${CMAKE_CURRENT_SOURCE_DIR}/pg_bitutils.cpp ${CMAKE_CURRENT_SOURCE_DIR}/port_srv/pg_bitutils.cpp COMMAND ln -fs ${CMAKE_CURRENT_SOURCE_DIR}/pgcheckdir.cpp ${CMAKE_CURRENT_SOURCE_DIR}/port_srv/pgcheckdir.cpp COMMAND ln -fs ${CMAKE_CURRENT_SOURCE_DIR}/pgmkdirp.cpp ${CMAKE_CURRENT_SOURCE_DIR}/port_srv/pgmkdirp.cpp COMMAND ln -fs ${CMAKE_CURRENT_SOURCE_DIR}/pgsleep.cpp ${CMAKE_CURRENT_SOURCE_DIR}/port_srv/pgsleep.cpp @@ -54,6 +55,7 @@ list(APPEND TGT_port_SRC ${CMAKE_CURRENT_SOURCE_DIR}/inet_net_ntop.cpp ${CMAKE_CURRENT_SOURCE_DIR}/noblock.cpp ${CMAKE_CURRENT_SOURCE_DIR}/path.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/pg_bitutils.cpp ${CMAKE_CURRENT_SOURCE_DIR}/pgcheckdir.cpp ${CMAKE_CURRENT_SOURCE_DIR}/pgmkdirp.cpp ${CMAKE_CURRENT_SOURCE_DIR}/pgsleep.cpp @@ -120,6 +122,7 @@ list(APPEND TGT_pgport_srv_SRC ${CMAKE_CURRENT_SOURCE_DIR}/port_srv/inet_net_ntop.cpp ${CMAKE_CURRENT_SOURCE_DIR}/port_srv/noblock.cpp ${CMAKE_CURRENT_SOURCE_DIR}/port_srv/path.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/port_srv/pg_bitutils.cpp ${CMAKE_CURRENT_SOURCE_DIR}/port_srv/pgcheckdir.cpp ${CMAKE_CURRENT_SOURCE_DIR}/port_srv/pgmkdirp.cpp ${CMAKE_CURRENT_SOURCE_DIR}/port_srv/pgsleep.cpp @@ -181,6 +184,7 @@ set(TGT_pgport_tool_SRC ${CMAKE_CURRENT_SOURCE_DIR}/inet_net_ntop.cpp ${CMAKE_CURRENT_SOURCE_DIR}/noblock.cpp ${CMAKE_CURRENT_SOURCE_DIR}/path.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/pg_bitutils.cpp ${CMAKE_CURRENT_SOURCE_DIR}/pgcheckdir.cpp ${CMAKE_CURRENT_SOURCE_DIR}/pgmkdirp.cpp ${CMAKE_CURRENT_SOURCE_DIR}/pgsleep.cpp diff --git a/src/common/port/Makefile b/src/common/port/Makefile index 51767b7e39..ba494cad3d 100644 --- a/src/common/port/Makefile +++ b/src/common/port/Makefile @@ -44,7 +44,7 @@ ifneq "$(MAKECMDGOALS)" "clean" endif endif OBJS = $(LIBOBJS) pg_crc32c_sse42.o pg_crc32c_sb8.o pg_crc32c_choose.o chklocale.o dirmod.o erand48.o exec.o fls.o inet_net_ntop.o \ - noblock.o path.o pgcheckdir.o pgmkdirp.o pgsleep.o \ + noblock.o path.o pg_bitutils.o pgcheckdir.o pgmkdirp.o pgsleep.o \ pgstrcasecmp.o qsort.o qsort_arg.o sprompt.o thread.o flock.o pgstrcasestr.o\ gs_thread.o gs_env_r.o gs_getopt_r.o \ gs_readdir.o gs_strerror.o gs_syscall_lock.o \ @@ -53,7 +53,7 @@ OBJS = $(LIBOBJS) pg_crc32c_sse42.o pg_crc32c_sb8.o pg_crc32c_choose.o chklocale ifeq "${host_cpu}" "aarch64" OBJS = $(LIBOBJS) pg_crc32c_choose.o chklocale.o dirmod.o erand48.o exec.o fls.o inet_net_ntop.o \ - noblock.o path.o pgcheckdir.o pgmkdirp.o pgsleep.o \ + noblock.o path.o pg_bitutils.o pgcheckdir.o pgmkdirp.o pgsleep.o \ pgstrcasecmp.o qsort.o qsort_arg.o sprompt.o thread.o flock.o pgstrcasestr.o\ gs_thread.o gs_env_r.o gs_getopt_r.o \ gs_readdir.o gs_strerror.o gs_syscall_lock.o \ @@ -70,7 +70,7 @@ ifeq "${host_cpu}" "aarch64" endif OBJS_TOOL = fls.o strlcat.o strlcpy.o getpeereid.o chklocale.o dirmod.o erand48.o exec.o inet_net_ntop.o \ - noblock.o path.o pgcheckdir.o pgmkdirp.o pgsleep.o pgstrcasecmp.o qsort.o qsort_arg.o sprompt.o thread.o flock.o pgstrcasestr.o \ + noblock.o path.o pg_bitutils.o pgcheckdir.o pgmkdirp.o pgsleep.o pgstrcasecmp.o qsort.o qsort_arg.o sprompt.o thread.o flock.o pgstrcasestr.o \ gs_thread.o gs_env_r.o gs_getopt_r.o gs_readdir.o gs_strerror.o gs_syscall_lock.o gs_system.o cipher.o all: libpgport.a libpgport_srv.a libpgport_tool.so diff --git a/src/common/port/pg_bitutils.cpp b/src/common/port/pg_bitutils.cpp new file mode 100644 index 0000000000..b61d842bdf --- /dev/null +++ b/src/common/port/pg_bitutils.cpp @@ -0,0 +1,308 @@ +/*------------------------------------------------------------------------- +* +* pg_bitutils.c +* Miscellaneous functions for bit-wise operations. +* +* Copyright (c) 2019-2021, PostgreSQL Global Development Group +* +* IDENTIFICATION +* src/common/port/pg_bitutils.cpp +* +*------------------------------------------------------------------------- +*/ +#include "c.h" + +#ifdef HAVE__GET_CPUID +#include +#endif +#ifdef HAVE__CPUID +#include +#endif + +#ifdef FRONTEND +#include "postgres_fe.h" +#else +#include "postgres.h" +#endif + +#include "port/pg_bitutils.h" + +#ifndef Assert +#define Assert(x) assert(x) +#endif + +/* +* Array giving the position of the left-most set bit for each possible +* byte value. We count the right-most position as the 0th bit, and the +* left-most the 7th bit. The 0th entry of the array should not be used. +* +* Note: this is not used by the functions in pg_bitutils.h when +* HAVE__BUILTIN_CLZ is defined, but we provide it anyway, so that +* extensions possibly compiled with a different compiler can use it. +*/ +const uint8 pg_leftmost_one_pos[256] = { + 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 +}; + +/* +* Array giving the position of the right-most set bit for each possible +* byte value. We count the right-most position as the 0th bit, and the +* left-most the 7th bit. The 0th entry of the array should not be used. +* +* Note: this is not used by the functions in pg_bitutils.h when +* HAVE__BUILTIN_CTZ is defined, but we provide it anyway, so that +* extensions possibly compiled with a different compiler can use it. +*/ +const uint8 pg_rightmost_one_pos[256] = { + 0, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 7, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0 +}; + +/* +* Array giving the number of 1-bits in each possible byte value. +* +* Note: we export this for use by functions in which explicit use +* of the popcount functions seems unlikely to be a win. +*/ +const uint8 pg_number_of_ones[256] = { + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8 +}; + +/* +* On x86_64, we can use the hardware popcount instruction, but only if +* we can verify that the CPU supports it via the cpuid instruction. +* +* Otherwise, we fall back to __builtin_popcount if the compiler has that, +* or a hand-rolled implementation if not. +*/ +#ifdef HAVE_X86_64_POPCNTQ +#if defined(HAVE__GET_CPUID) || defined(HAVE__CPUID) +#define USE_POPCNT_ASM 1 +#endif +#endif + +static int pg_popcount32_slow(uint32 word); +static int pg_popcount64_slow(uint64 word); + +#ifdef USE_POPCNT_ASM +static bool pg_popcount_available(void); +static int pg_popcount32_choose(uint32 word); +static int pg_popcount64_choose(uint64 word); +static int pg_popcount32_asm(uint32 word); +static int pg_popcount64_asm(uint64 word); + +int (*pg_popcount32)(uint32 word) = pg_popcount32_choose; +int (*pg_popcount64)(uint64 word) = pg_popcount64_choose; +#else +int (*pg_popcount32)(uint32 word) = pg_popcount32_slow; +int (*pg_popcount64)(uint64 word) = pg_popcount64_slow; +#endif /* USE_POPCNT_ASM */ + +#ifdef USE_POPCNT_ASM + +/* +* Return true if CPUID indicates that the POPCNT instruction is available. +*/ +static bool pg_popcount_available(void) +{ + unsigned int exx[4] = {0, 0, 0, 0}; + +#if defined(HAVE__GET_CPUID) + __get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]); +#elif defined(HAVE__CPUID) + __cpuid(exx, 1); +#else +#error cpuid instruction not available +#endif + + return (exx[2] & (1 << 23)) != 0; /* POPCNT */ +} + +/* +* These functions get called on the first call to pg_popcount32 etc. +* They detect whether we can use the asm implementations, and replace +* the function pointers so that subsequent calls are routed directly to +* the chosen implementation. +*/ +static int pg_popcount32_choose(uint32 word) +{ + if (pg_popcount_available()) { + pg_popcount32 = pg_popcount32_asm; + pg_popcount64 = pg_popcount64_asm; + } else { + pg_popcount32 = pg_popcount32_slow; + pg_popcount64 = pg_popcount64_slow; + } + + return pg_popcount32(word); +} + +static int pg_popcount64_choose(uint64 word) +{ + if (pg_popcount_available()) { + pg_popcount32 = pg_popcount32_asm; + pg_popcount64 = pg_popcount64_asm; + } else { + pg_popcount32 = pg_popcount32_slow; + pg_popcount64 = pg_popcount64_slow; + } + + return pg_popcount64(word); +} + +/* +* pg_popcount32_asm +* Return the number of 1 bits set in word +*/ +static int pg_popcount32_asm(uint32 word) +{ + uint32 res; + + __asm__ __volatile__(" popcntl %1,%0\n" : "=q"(res) : "rm"(word) : "cc"); + return (int)res; +} + +/* +* pg_popcount64_asm +* Return the number of 1 bits set in word +*/ +static int pg_popcount64_asm(uint64 word) +{ + uint64 res; + + __asm__ __volatile__(" popcntq %1,%0\n" : "=q"(res) : "rm"(word) : "cc"); + return (int)res; +} + +#endif /* USE_POPCNT_ASM */ + +/* +* pg_popcount32_slow +* Return the number of 1 bits set in word +*/ +static int pg_popcount32_slow(uint32 word) +{ +#ifdef HAVE__BUILTIN_POPCOUNT + return __builtin_popcount(word); +#else /* !HAVE__BUILTIN_POPCOUNT */ + int result = 0; + + while (word != 0) { + result += pg_number_of_ones[word & 255]; + word >>= 8; + } + + return result; +#endif /* HAVE__BUILTIN_POPCOUNT */ +} + +/* +* pg_popcount64_slow +* Return the number of 1 bits set in word +*/ +static int pg_popcount64_slow(uint64 word) +{ +#ifdef HAVE__BUILTIN_POPCOUNT +#if defined(HAVE_LONG_INT_64) + return __builtin_popcountl(word); +#elif defined(HAVE_LONG_LONG_INT_64) + return __builtin_popcountll(word); +#else +#error must have a working 64-bit integer datatype +#endif +#else /* !HAVE__BUILTIN_POPCOUNT */ + int result = 0; + + while (word != 0) { + result += pg_number_of_ones[word & 255]; + word >>= 8; + } + + return result; +#endif /* HAVE__BUILTIN_POPCOUNT */ +} + +/* +* pg_popcount +* Returns the number of 1-bits in buf +*/ +uint64 pg_popcount(const char *buf, int bytes) +{ + uint64 popcnt = 0; + +#if SIZEOF_VOID_P >= 8 + /* Process in 64-bit chunks if the buffer is aligned. */ + if (buf == (const char *)TYPEALIGN(8, buf)) { + const uint64 *words = (const uint64 *)buf; + + while (bytes >= 8) { + popcnt += pg_popcount64(*words++); + bytes -= 8; + } + + buf = (const char *)words; + } +#else + /* Process in 32-bit chunks if the buffer is aligned. */ + if (buf == (const char *)TYPEALIGN(4, buf)) { + const uint32 *words = (const uint32 *)buf; + + while (bytes >= 4) { + popcnt += pg_popcount32(*words++); + bytes -= 4; + } + + buf = (const char *)words; + } +#endif + + /* Process any remaining bytes */ + while (bytes--) + popcnt += pg_number_of_ones[(unsigned char)*buf++]; + + return popcnt; +} diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h new file mode 100644 index 0000000000..fd4ddeada5 --- /dev/null +++ b/src/include/port/pg_bitutils.h @@ -0,0 +1,147 @@ +/*------------------------------------------------------------------------- +* +* pg_bitutils.h +* Miscellaneous functions for bit-wise operations. +* +* +* Copyright (c) 2019-2021, PostgreSQL Global Development Group +* +* src/include/port/pg_bitutils.h +* +*------------------------------------------------------------------------- +*/ +#ifndef PG_BITUTILS_H +#define PG_BITUTILS_H + +#ifndef FRONTEND +extern PGDLLIMPORT const uint8 pg_leftmost_one_pos[256]; +extern PGDLLIMPORT const uint8 pg_rightmost_one_pos[256]; +extern PGDLLIMPORT const uint8 pg_number_of_ones[256]; +#else +extern const uint8 pg_leftmost_one_pos[256]; +extern const uint8 pg_rightmost_one_pos[256]; +extern const uint8 pg_number_of_ones[256]; +#endif + +/* +* pg_leftmost_one_pos32 +* Returns the position of the most significant set bit in "word", +* measured from the least significant bit. word must not be 0. +*/ +static inline int pg_leftmost_one_pos32(uint32 word) +{ +#ifdef HAVE__BUILTIN_CLZ + Assert(word != 0); + + return 31 - __builtin_clz(word); +#else + int shift = 32 - 8; + + Assert(word != 0); + + while ((word >> shift) == 0) + shift -= 8; + + return shift + pg_leftmost_one_pos[(word >> shift) & 255]; +#endif /* HAVE__BUILTIN_CLZ */ +} + +/* +* pg_leftmost_one_pos64 +* As above, but for a 64-bit word. +*/ +static inline int pg_leftmost_one_pos64(uint64 word) +{ +#ifdef HAVE__BUILTIN_CLZ + Assert(word != 0); + +#if defined(HAVE_LONG_INT_64) + return 63 - __builtin_clzl(word); +#elif defined(HAVE_LONG_LONG_INT_64) + return 63 - __builtin_clzll(word); +#else +#error must have a working 64-bit integer datatype +#endif +#else /* !HAVE__BUILTIN_CLZ */ + int shift = 64 - 8; + + Assert(word != 0); + + while ((word >> shift) == 0) + shift -= 8; + + return shift + pg_leftmost_one_pos[(word >> shift) & 255]; +#endif /* HAVE__BUIILTIN_CLZ */ +} + +/* +* pg_rightmost_one_pos32 +* Returns the position of the least significant set bit in "word", +* measured from the least significant bit. word must not be 0. +*/ +static inline int pg_rightmost_one_pos32(uint32 word) +{ +#ifdef HAVE__BUILTIN_CTZ + Assert(word != 0); + + return __builtin_ctz(word); +#else + int result = 0; + + Assert(word != 0); + + while ((word & 255) == 0) { + word >>= 8; + result += 8; + } + result += pg_rightmost_one_pos[word & 255]; + return result; +#endif /* HAVE__BUILTIN_CTZ */ +} + +/* +* pg_rightmost_one_pos64 +* As above, but for a 64-bit word. +*/ +static inline int pg_rightmost_one_pos64(uint64 word) +{ +#ifdef HAVE__BUILTIN_CTZ + Assert(word != 0); + +#if defined(HAVE_LONG_INT_64) + return __builtin_ctzl(word); +#elif defined(HAVE_LONG_LONG_INT_64) + return __builtin_ctzll(word); +#else +#error must have a working 64-bit integer datatype +#endif +#else /* !HAVE__BUILTIN_CTZ */ + int result = 0; + + Assert(word != 0); + + while ((word & 255) == 0) { + word >>= 8; + result += 8; + } + result += pg_rightmost_one_pos[word & 255]; + return result; +#endif /* HAVE__BUILTIN_CTZ */ +} + +/* Count the number of one-bits in a uint32 or uint64 */ +extern int (*pg_popcount32)(uint32 word); +extern int (*pg_popcount64)(uint64 word); + +/* Count the number of one-bits in a byte array */ +extern uint64 pg_popcount(const char *buf, int bytes); + +/* +* Rotate the bits of "word" to the right by n bits. +*/ +static inline uint32 pg_rotate_right32(uint32 word, int n) +{ + return (word >> n) | (word << (sizeof(word) * BITS_PER_BYTE - n)); +} + +#endif /* PG_BITUTILS_H */ \ No newline at end of file -- Gitee