Revision e03a5125ec7bd1d4e8b062816a8813436876dc7c authored by Nicholas Piggin on 12 July 2019, 03:59:12 UTC, committed by Linus Torvalds on 12 July 2019, 18:05:46 UTC
CONFIG_NUMA on 64-bit CPUs currently enables hashdist unconditionally even when booting on single node machines. This causes the large system hashes to be allocated with vmalloc, and mapped with small pages. This change clears hashdist if only one node has come up with memory. This results in the important large inode and dentry hashes using memblock allocations. All others are within 4MB size up to about 128GB of RAM, which allows them to be allocated from the linear map on most non-NUMA images. Other big hashes like futex and TCP should eventually be moved over to the same style of allocation as those vfs caches that use HASH_EARLY if !hashdist, so they don't exceed MAX_ORDER on very large non-NUMA images. This brings dTLB misses for linux kernel tree `git diff` from ~45,000 to ~8,000 on a Kaby Lake KVM guest with 8MB dentry hash and mitigations=off (performance is in the noise, under 1% difference, page tables are likely to be well cached for this workload). Link: http://lkml.kernel.org/r/20190605144814.29319-2-npiggin@gmail.com Signed-off-by: Nicholas Piggin <npiggin@gmail.com> Reviewed-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
1 parent ec11408
File | Mode | Size |
---|---|---|
842 | ||
crypto | ||
dim | ||
fonts | ||
livepatch | ||
lz4 | ||
lzo | ||
math | ||
mpi | ||
raid6 | ||
reed_solomon | ||
vdso | ||
xz | ||
zlib_deflate | ||
zlib_inflate | ||
zstd | ||
.gitignore | -rw-r--r-- | 98 bytes |
Kconfig | -rw-r--r-- | 14.9 KB |
Kconfig.debug | -rw-r--r-- | 70.9 KB |
Kconfig.kasan | -rw-r--r-- | 5.2 KB |
Kconfig.kgdb | -rw-r--r-- | 4.2 KB |
Kconfig.ubsan | -rw-r--r-- | 1.7 KB |
Makefile | -rw-r--r-- | 9.4 KB |
argv_split.c | -rw-r--r-- | 2.1 KB |
ashldi3.c | -rw-r--r-- | 541 bytes |
ashrdi3.c | -rw-r--r-- | 565 bytes |
asn1_decoder.c | -rw-r--r-- | 13.2 KB |
assoc_array.c | -rw-r--r-- | 51.8 KB |
atomic64.c | -rw-r--r-- | 4.5 KB |
atomic64_test.c | -rw-r--r-- | 6.4 KB |
audit.c | -rw-r--r-- | 1.8 KB |
bcd.c | -rw-r--r-- | 297 bytes |
bch.c | -rw-r--r-- | 36.1 KB |
bitmap.c | -rw-r--r-- | 35.5 KB |
bitrev.c | -rw-r--r-- | 1.9 KB |
bsearch.c | -rw-r--r-- | 1.4 KB |
btree.c | -rw-r--r-- | 19.2 KB |
bucket_locks.c | -rw-r--r-- | 1.4 KB |
bug.c | -rw-r--r-- | 5.5 KB |
build_OID_registry | -rwxr-xr-x | 4.5 KB |
bust_spinlocks.c | -rw-r--r-- | 676 bytes |
chacha.c | -rw-r--r-- | 3.6 KB |
check_signature.c | -rw-r--r-- | 635 bytes |
checksum.c | -rw-r--r-- | 4.8 KB |
clz_ctz.c | -rw-r--r-- | 1.2 KB |
clz_tab.c | -rw-r--r-- | 891 bytes |
cmdline.c | -rw-r--r-- | 5.1 KB |
cmpdi2.c | -rw-r--r-- | 501 bytes |
compat_audit.c | -rw-r--r-- | 832 bytes |
cpu_rmap.c | -rw-r--r-- | 7.6 KB |
cpumask.c | -rw-r--r-- | 6.0 KB |
crc-ccitt.c | -rw-r--r-- | 5.6 KB |
crc-itu-t.c | -rw-r--r-- | 2.7 KB |
crc-t10dif.c | -rw-r--r-- | 2.9 KB |
crc16.c | -rw-r--r-- | 2.7 KB |
crc32.c | -rw-r--r-- | 9.3 KB |
crc32defs.h | -rw-r--r-- | 1.6 KB |
crc32test.c | -rw-r--r-- | 37.5 KB |
crc4.c | -rw-r--r-- | 1003 bytes |
crc64.c | -rw-r--r-- | 1.7 KB |
crc7.c | -rw-r--r-- | 2.5 KB |
crc8.c | -rw-r--r-- | 2.4 KB |
ctype.c | -rw-r--r-- | 1.4 KB |
debug_info.c | -rw-r--r-- | 777 bytes |
debug_locks.c | -rw-r--r-- | 1.2 KB |
debugobjects.c | -rw-r--r-- | 34.2 KB |
dec_and_lock.c | -rw-r--r-- | 1.2 KB |
decompress.c | -rw-r--r-- | 1.7 KB |
decompress_bunzip2.c | -rw-r--r-- | 23.5 KB |
decompress_inflate.c | -rw-r--r-- | 4.5 KB |
decompress_unlz4.c | -rw-r--r-- | 4.0 KB |
decompress_unlzma.c | -rw-r--r-- | 15.8 KB |
decompress_unlzo.c | -rw-r--r-- | 6.4 KB |
decompress_unxz.c | -rw-r--r-- | 10.9 KB |
devres.c | -rw-r--r-- | 11.1 KB |
digsig.c | -rw-r--r-- | 5.5 KB |
dump_stack.c | -rw-r--r-- | 3.0 KB |
dynamic_debug.c | -rw-r--r-- | 26.2 KB |
dynamic_queue_limits.c | -rw-r--r-- | 4.3 KB |
earlycpio.c | -rw-r--r-- | 3.6 KB |
error-inject.c | -rw-r--r-- | 5.4 KB |
errseq.c | -rw-r--r-- | 6.6 KB |
extable.c | -rw-r--r-- | 3.0 KB |
fault-inject.c | -rw-r--r-- | 6.2 KB |
fdt.c | -rw-r--r-- | 69 bytes |
fdt_empty_tree.c | -rw-r--r-- | 80 bytes |
fdt_ro.c | -rw-r--r-- | 72 bytes |
fdt_rw.c | -rw-r--r-- | 72 bytes |
fdt_strerror.c | -rw-r--r-- | 78 bytes |
fdt_sw.c | -rw-r--r-- | 72 bytes |
fdt_wip.c | -rw-r--r-- | 73 bytes |
find_bit.c | -rw-r--r-- | 5.1 KB |
find_bit_benchmark.c | -rw-r--r-- | 3.9 KB |
flex_proportions.c | -rw-r--r-- | 6.9 KB |
gen_crc32table.c | -rw-r--r-- | 3.3 KB |
gen_crc64table.c | -rw-r--r-- | 1.4 KB |
genalloc.c | -rw-r--r-- | 21.7 KB |
generic-radix-tree.c | -rw-r--r-- | 4.8 KB |
glob.c | -rw-r--r-- | 3.5 KB |
globtest.c | -rw-r--r-- | 4.2 KB |
hexdump.c | -rw-r--r-- | 8.2 KB |
hweight.c | -rw-r--r-- | 1.9 KB |
idr.c | -rw-r--r-- | 17.5 KB |
inflate.c | -rw-r--r-- | 38.7 KB |
interval_tree.c | -rw-r--r-- | 540 bytes |
interval_tree_test.c | -rw-r--r-- | 3.4 KB |
iomap.c | -rw-r--r-- | 9.1 KB |
iomap_copy.c | -rw-r--r-- | 2.2 KB |
iommu-helper.c | -rw-r--r-- | 755 bytes |
ioremap.c | -rw-r--r-- | 5.0 KB |
iov_iter.c | -rw-r--r-- | 40.9 KB |
irq_poll.c | -rw-r--r-- | 5.4 KB |
irq_regs.c | -rw-r--r-- | 394 bytes |
is_single_threaded.c | -rw-r--r-- | 1.2 KB |
jedec_ddr_data.c | -rw-r--r-- | 2.9 KB |
kasprintf.c | -rw-r--r-- | 1.4 KB |
kfifo.c | -rw-r--r-- | 12.1 KB |
klist.c | -rw-r--r-- | 10.4 KB |
kobject.c | -rw-r--r-- | 27.8 KB |
kobject_uevent.c | -rw-r--r-- | 18.8 KB |
kstrtox.c | -rw-r--r-- | 10.5 KB |
kstrtox.h | -rw-r--r-- | 293 bytes |
libcrc32c.c | -rw-r--r-- | 2.0 KB |
list_debug.c | -rw-r--r-- | 1.8 KB |
list_sort.c | -rw-r--r-- | 8.4 KB |
llist.c | -rw-r--r-- | 2.5 KB |
locking-selftest-hardirq.h | -rw-r--r-- | 246 bytes |
locking-selftest-mutex.h | -rw-r--r-- | 159 bytes |
locking-selftest-rlock-hardirq.h | -rw-r--r-- | 74 bytes |
locking-selftest-rlock-softirq.h | -rw-r--r-- | 74 bytes |
locking-selftest-rlock.h | -rw-r--r-- | 197 bytes |
locking-selftest-rsem.h | -rw-r--r-- | 202 bytes |
locking-selftest-rtmutex.h | -rw-r--r-- | 162 bytes |
locking-selftest-softirq.h | -rw-r--r-- | 246 bytes |
locking-selftest-spin-hardirq.h | -rw-r--r-- | 73 bytes |
locking-selftest-spin-softirq.h | -rw-r--r-- | 73 bytes |
locking-selftest-spin.h | -rw-r--r-- | 157 bytes |
locking-selftest-wlock-hardirq.h | -rw-r--r-- | 74 bytes |
locking-selftest-wlock-softirq.h | -rw-r--r-- | 74 bytes |
locking-selftest-wlock.h | -rw-r--r-- | 197 bytes |
locking-selftest-wsem.h | -rw-r--r-- | 202 bytes |
locking-selftest.c | -rw-r--r-- | 43.7 KB |
lockref.c | -rw-r--r-- | 4.5 KB |
logic_pio.c | -rw-r--r-- | 7.7 KB |
lru_cache.c | -rw-r--r-- | 18.8 KB |
lshrdi3.c | -rw-r--r-- | 559 bytes |
memcat_p.c | -rw-r--r-- | 753 bytes |
memory-notifier-error-inject.c | -rw-r--r-- | 1.1 KB |
memweight.c | -rw-r--r-- | 1.0 KB |
muldi3.c | -rw-r--r-- | 1.7 KB |
net_utils.c | -rw-r--r-- | 640 bytes |
netdev-notifier-error-inject.c | -rw-r--r-- | 1.5 KB |
nlattr.c | -rw-r--r-- | 22.6 KB |
nmi_backtrace.c | -rw-r--r-- | 3.0 KB |
nodemask.c | -rw-r--r-- | 653 bytes |
notifier-error-inject.c | -rw-r--r-- | 2.7 KB |
notifier-error-inject.h | -rw-r--r-- | 653 bytes |
objagg.c | -rw-r--r-- | 28.3 KB |
of-reconfig-notifier-error-inject.c | -rw-r--r-- | 1.3 KB |
oid_registry.c | -rw-r--r-- | 3.7 KB |
once.c | -rw-r--r-- | 1.4 KB |
packing.c | -rw-r--r-- | 6.5 KB |
parman.c | -rw-r--r-- | 10.6 KB |
parser.c | -rw-r--r-- | 8.1 KB |
pci_iomap.c | -rw-r--r-- | 4.2 KB |
percpu-refcount.c | -rw-r--r-- | 13.3 KB |
percpu_counter.c | -rw-r--r-- | 5.8 KB |
percpu_test.c | -rw-r--r-- | 3.2 KB |
plist.c | -rw-r--r-- | 5.9 KB |
pm-notifier-error-inject.c | -rw-r--r-- | 1.2 KB |
radix-tree.c | -rw-r--r-- | 43.3 KB |
random32.c | -rw-r--r-- | 12.8 KB |
ratelimit.c | -rw-r--r-- | 1.6 KB |
rbtree.c | -rw-r--r-- | 18.0 KB |
rbtree_test.c | -rw-r--r-- | 9.5 KB |
refcount.c | -rw-r--r-- | 11.4 KB |
rhashtable.c | -rw-r--r-- | 29.4 KB |
sbitmap.c | -rw-r--r-- | 16.6 KB |
scatterlist.c | -rw-r--r-- | 24.7 KB |
seq_buf.c | -rw-r--r-- | 8.0 KB |
sg_pool.c | -rw-r--r-- | 3.6 KB |
sg_split.c | -rw-r--r-- | 5.0 KB |
sha1.c | -rw-r--r-- | 6.1 KB |
sha256.c | -rw-r--r-- | 9.9 KB |
show_mem.c | -rw-r--r-- | 1.2 KB |
siphash.c | -rw-r--r-- | 12.0 KB |
smp_processor_id.c | -rw-r--r-- | 1.5 KB |
sort.c | -rw-r--r-- | 8.0 KB |
stackdepot.c | -rw-r--r-- | 8.6 KB |
stmp_device.c | -rw-r--r-- | 1.9 KB |
string.c | -rw-r--r-- | 23.9 KB |
string_helpers.c | -rw-r--r-- | 13.9 KB |
strncpy_from_user.c | -rw-r--r-- | 3.2 KB |
strnlen_user.c | -rw-r--r-- | 3.5 KB |
syscall.c | -rw-r--r-- | 2.5 KB |
test-kstrtox.c | -rw-r--r-- | 17.3 KB |
test-string_helpers.c | -rw-r--r-- | 10.3 KB |
test_bitfield.c | -rw-r--r-- | 4.3 KB |
test_bitmap.c | -rw-r--r-- | 10.6 KB |
test_blackhole_dev.c | -rw-r--r-- | 2.5 KB |
test_bpf.c | -rw-r--r-- | 158.8 KB |
test_debug_virtual.c | -rw-r--r-- | 981 bytes |
test_firmware.c | -rw-r--r-- | 21.6 KB |
test_hash.c | -rw-r--r-- | 6.3 KB |
test_hexdump.c | -rw-r--r-- | 6.3 KB |
test_ida.c | -rw-r--r-- | 4.3 KB |
test_kasan.c | -rw-r--r-- | 15.6 KB |
test_kmod.c | -rw-r--r-- | 30.0 KB |
test_list_sort.c | -rw-r--r-- | 3.3 KB |
test_memcat_p.c | -rw-r--r-- | 2.2 KB |
test_module.c | -rw-r--r-- | 794 bytes |
test_objagg.c | -rw-r--r-- | 24.6 KB |
test_overflow.c | -rw-r--r-- | 22.3 KB |
test_parman.c | -rw-r--r-- | 11.2 KB |
test_printf.c | -rw-r--r-- | 14.3 KB |
test_rhashtable.c | -rw-r--r-- | 20.0 KB |
test_siphash.c | -rw-r--r-- | 7.5 KB |
test_sort.c | -rw-r--r-- | 870 bytes |
test_stackinit.c | -rw-r--r-- | 10.9 KB |
test_static_key_base.c | -rw-r--r-- | 1.6 KB |
test_static_keys.c | -rw-r--r-- | 5.6 KB |
test_string.c | -rw-r--r-- | 2.4 KB |
test_strscpy.c | -rw-r--r-- | 4.0 KB |
test_sysctl.c | -rw-r--r-- | 3.7 KB |
test_ubsan.c | -rw-r--r-- | 2.4 KB |
test_user_copy.c | -rw-r--r-- | 5.1 KB |
test_uuid.c | -rw-r--r-- | 3.4 KB |
test_vmalloc.c | -rw-r--r-- | 10.6 KB |
test_xarray.c | -rw-r--r-- | 41.4 KB |
textsearch.c | -rw-r--r-- | 9.3 KB |
timerqueue.c | -rw-r--r-- | 2.6 KB |
ts_bm.c | -rw-r--r-- | 5.1 KB |
ts_fsm.c | -rw-r--r-- | 10.4 KB |
ts_kmp.c | -rw-r--r-- | 4.1 KB |
ubsan.c | -rw-r--r-- | 10.5 KB |
ubsan.h | -rw-r--r-- | 1.7 KB |
ucmpdi2.c | -rw-r--r-- | 568 bytes |
ucs2_string.c | -rw-r--r-- | 2.5 KB |
usercopy.c | -rw-r--r-- | 737 bytes |
uuid.c | -rw-r--r-- | 2.6 KB |
vsprintf.c | -rw-r--r-- | 79.1 KB |
win_minmax.c | -rw-r--r-- | 3.4 KB |
xarray.c | -rw-r--r-- | 52.7 KB |
xxhash.c | -rw-r--r-- | 12.7 KB |
Computing file changes ...