Revision e03a5125ec7bd1d4e8b062816a8813436876dc7c authored by Nicholas Piggin on 12 July 2019, 03:59:12 UTC, committed by Linus Torvalds on 12 July 2019, 18:05:46 UTC
CONFIG_NUMA on 64-bit CPUs currently enables hashdist unconditionally even when booting on single node machines. This causes the large system hashes to be allocated with vmalloc, and mapped with small pages. This change clears hashdist if only one node has come up with memory. This results in the important large inode and dentry hashes using memblock allocations. All others are within 4MB size up to about 128GB of RAM, which allows them to be allocated from the linear map on most non-NUMA images. Other big hashes like futex and TCP should eventually be moved over to the same style of allocation as those vfs caches that use HASH_EARLY if !hashdist, so they don't exceed MAX_ORDER on very large non-NUMA images. This brings dTLB misses for linux kernel tree `git diff` from ~45,000 to ~8,000 on a Kaby Lake KVM guest with 8MB dentry hash and mitigations=off (performance is in the noise, under 1% difference, page tables are likely to be well cached for this workload). Link: http://lkml.kernel.org/r/20190605144814.29319-2-npiggin@gmail.com Signed-off-by: Nicholas Piggin <npiggin@gmail.com> Reviewed-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
1 parent ec11408
mpi-inline.h
/* SPDX-License-Identifier: GPL-2.0-or-later */
/* mpi-inline.h - Internal to the Multi Precision Integers
* Copyright (C) 1994, 1996, 1998, 1999 Free Software Foundation, Inc.
*
* This file is part of GnuPG.
*
* Note: This code is heavily based on the GNU MP Library.
* Actually it's the same code with only minor changes in the
* way the data is stored; this is to support the abstraction
* of an optional secure memory allocation which may be used
* to avoid revealing of sensitive data due to paging etc.
* The GNU MP Library itself is published under the LGPL;
* however I decided to publish this code under the plain GPL.
*/
#ifndef G10_MPI_INLINE_H
#define G10_MPI_INLINE_H
#ifndef G10_MPI_INLINE_DECL
#define G10_MPI_INLINE_DECL static inline
#endif
G10_MPI_INLINE_DECL mpi_limb_t
mpihelp_add_1(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr,
mpi_size_t s1_size, mpi_limb_t s2_limb)
{
mpi_limb_t x;
x = *s1_ptr++;
s2_limb += x;
*res_ptr++ = s2_limb;
if (s2_limb < x) { /* sum is less than the left operand: handle carry */
while (--s1_size) {
x = *s1_ptr++ + 1; /* add carry */
*res_ptr++ = x; /* and store */
if (x) /* not 0 (no overflow): we can stop */
goto leave;
}
return 1; /* return carry (size of s1 to small) */
}
leave:
if (res_ptr != s1_ptr) { /* not the same variable */
mpi_size_t i; /* copy the rest */
for (i = 0; i < s1_size - 1; i++)
res_ptr[i] = s1_ptr[i];
}
return 0; /* no carry */
}
G10_MPI_INLINE_DECL mpi_limb_t
mpihelp_add(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr, mpi_size_t s1_size,
mpi_ptr_t s2_ptr, mpi_size_t s2_size)
{
mpi_limb_t cy = 0;
if (s2_size)
cy = mpihelp_add_n(res_ptr, s1_ptr, s2_ptr, s2_size);
if (s1_size - s2_size)
cy = mpihelp_add_1(res_ptr + s2_size, s1_ptr + s2_size,
s1_size - s2_size, cy);
return cy;
}
G10_MPI_INLINE_DECL mpi_limb_t
mpihelp_sub_1(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr,
mpi_size_t s1_size, mpi_limb_t s2_limb)
{
mpi_limb_t x;
x = *s1_ptr++;
s2_limb = x - s2_limb;
*res_ptr++ = s2_limb;
if (s2_limb > x) {
while (--s1_size) {
x = *s1_ptr++;
*res_ptr++ = x - 1;
if (x)
goto leave;
}
return 1;
}
leave:
if (res_ptr != s1_ptr) {
mpi_size_t i;
for (i = 0; i < s1_size - 1; i++)
res_ptr[i] = s1_ptr[i];
}
return 0;
}
G10_MPI_INLINE_DECL mpi_limb_t
mpihelp_sub(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr, mpi_size_t s1_size,
mpi_ptr_t s2_ptr, mpi_size_t s2_size)
{
mpi_limb_t cy = 0;
if (s2_size)
cy = mpihelp_sub_n(res_ptr, s1_ptr, s2_ptr, s2_size);
if (s1_size - s2_size)
cy = mpihelp_sub_1(res_ptr + s2_size, s1_ptr + s2_size,
s1_size - s2_size, cy);
return cy;
}
#endif /*G10_MPI_INLINE_H */
Computing file changes ...
