mirror/linux
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
3205f8063b6cc54b20d5080fb79dfcbd9c39e93d
linux/kernel
History
Steven Rostedt (Red Hat) 3205f8063b ring-buffer: Add unlikelys to make fast path the default 
I was running the trace_event benchmark and noticed that the times
to record a trace_event was all over the place. I looked at the assembly
of the ring_buffer_lock_reserver() and saw this:

 <ring_buffer_lock_reserve>:
       31 c0                   xor    %eax,%eax
       48 83 3d 76 47 bd 00    cmpq   $0x1,0xbd4776(%rip)        # ffffffff81d10d60 <ring_buffer_flags>
       01
       55                      push   %rbp
       48 89 e5                mov    %rsp,%rbp
       75 1d                   jne    ffffffff8113c60d <ring_buffer_lock_reserve+0x2d>
       65 ff 05 69 e3 ec 7e    incl   %gs:0x7eece369(%rip)        # a960 <__preempt_count>
       8b 47 08                mov    0x8(%rdi),%eax
       85 c0                   test   %eax,%eax
 +---- 74 12                   je     ffffffff8113c610 <ring_buffer_lock_reserve+0x30>
 |     65 ff 0d 5b e3 ec 7e    decl   %gs:0x7eece35b(%rip)        # a960 <__preempt_count>
 |     0f 84 85 00 00 00       je     ffffffff8113c690 <ring_buffer_lock_reserve+0xb0>
 |     31 c0                   xor    %eax,%eax
 |     5d                      pop    %rbp
 |     c3                      retq
 |     90                      nop
 +---> 65 44 8b 05 48 e3 ec    mov    %gs:0x7eece348(%rip),%r8d        # a960 <__preempt_count>
       7e
       41 81 e0 ff ff ff 7f    and    $0x7fffffff,%r8d
       b0 08                   mov    $0x8,%al
       65 8b 0d 58 36 ed 7e    mov    %gs:0x7eed3658(%rip),%ecx        # fc80 <current_context>
       41 f7 c0 00 ff 1f 00    test   $0x1fff00,%r8d
       74 1e                   je     ffffffff8113c64f <ring_buffer_lock_reserve+0x6f>
       41 f7 c0 00 00 10 00    test   $0x100000,%r8d
       b0 01                   mov    $0x1,%al
       75 13                   jne    ffffffff8113c64f <ring_buffer_lock_reserve+0x6f>
       41 81 e0 00 00 0f 00    and    $0xf0000,%r8d
       49 83 f8 01             cmp    $0x1,%r8
       19 c0                   sbb    %eax,%eax
       83 e0 02                and    $0x2,%eax
       83 c0 02                add    $0x2,%eax
       85 c8                   test   %ecx,%eax
       75 ab                   jne    ffffffff8113c5fe <ring_buffer_lock_reserve+0x1e>
       09 c8                   or     %ecx,%eax
       65 89 05 24 36 ed 7e    mov    %eax,%gs:0x7eed3624(%rip)        # fc80 <current_context>

The arrow is the fast path.

After adding the unlikely's, the fast path looks a bit better:

 <ring_buffer_lock_reserve>:
       31 c0                   xor    %eax,%eax
       48 83 3d 76 47 bd 00    cmpq   $0x1,0xbd4776(%rip)        # ffffffff81d10d60 <ring_buffer_flags>
       01
       55                      push   %rbp
       48 89 e5                mov    %rsp,%rbp
       75 7b                   jne    ffffffff8113c66b <ring_buffer_lock_reserve+0x8b>
       65 ff 05 69 e3 ec 7e    incl   %gs:0x7eece369(%rip)        # a960 <__preempt_count>
       8b 47 08                mov    0x8(%rdi),%eax
       85 c0                   test   %eax,%eax
       0f 85 9f 00 00 00       jne    ffffffff8113c6a1 <ring_buffer_lock_reserve+0xc1>
       65 8b 0d 57 e3 ec 7e    mov    %gs:0x7eece357(%rip),%ecx        # a960 <__preempt_count>
       81 e1 ff ff ff 7f       and    $0x7fffffff,%ecx
       b0 08                   mov    $0x8,%al
       65 8b 15 68 36 ed 7e    mov    %gs:0x7eed3668(%rip),%edx        # fc80 <current_context>
       f7 c1 00 ff 1f 00       test   $0x1fff00,%ecx
       75 50                   jne    ffffffff8113c670 <ring_buffer_lock_reserve+0x90>
       85 d0                   test   %edx,%eax
       75 7d                   jne    ffffffff8113c6a1 <ring_buffer_lock_reserve+0xc1>
       09 d0                   or     %edx,%eax
       65 89 05 53 36 ed 7e    mov    %eax,%gs:0x7eed3653(%rip)        # fc80 <current_context>
       65 8b 05 fc da ec 7e    mov    %gs:0x7eecdafc(%rip),%eax        # a130 <cpu_number>
       89 c2                   mov    %eax,%edx

Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2015-05-21 17:39:29 -04:00
..
bpf
bpf: fix 64-bit divide
2015-04-27 23:11:49 -04:00
configs
…
debug
debug: prevent entering debug mode on panic/exception.
2015-02-19 12:39:03 -06:00
events
tracing: Rename ftrace_event.h to trace_events.h
2015-05-13 14:05:12 -04:00
gcov
gcov: fix softlockups
2015-04-17 09:04:08 -04:00
irq
genirq: Set IRQCHIP_SKIP_SET_WAKE flag for dummy_irq_chip
2015-04-24 20:57:06 +02:00
livepatch
Merge branch 'for-4.1/core-noarch' into for-linus
2015-04-13 23:57:20 +02:00
locking
lockdep: Make print_lock() robust against concurrent release
2015-04-17 09:42:14 +02:00
power
Merge back earlier suspend/hibernate material for v4.1.
2015-04-10 12:01:59 +02:00
printk
Merge tag 'tty-4.1-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty
2015-04-21 09:33:10 -07:00
rcu
tracing: Rename ftrace_event.h to trace_events.h
2015-05-13 14:05:12 -04:00
sched
Merge tag 'pm+acpi-4.1-rc2' of git://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm
2015-04-30 14:23:31 -07:00
time
Merge branch 'timers-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
2015-05-09 14:57:49 -07:00
trace
ring-buffer: Add unlikelys to make fast path the default
2015-05-21 17:39:29 -04:00
.gitignore
…
acct.c
acct: check FMODE_CAN_WRITE
2015-04-11 22:27:55 -04:00
async.c
kernel/async.c: switch to pr_foo()
2014-10-09 22:26:04 -04:00
audit_tree.c
Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs
2015-04-26 17:22:07 -07:00
audit_watch.c
VFS: audit: d_backing_inode() annotations
2015-04-15 15:06:55 -04:00
audit.c
Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs
2015-04-26 17:22:07 -07:00
audit.h
Merge branch 'upstream' of git://git.infradead.org/users/pcmoore/audit
2015-04-22 14:49:23 -07:00
auditfilter.c
Merge branch 'upstream' of git://git.infradead.org/users/pcmoore/audit
2015-02-11 20:07:47 -08:00
auditsc.c
Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs
2015-04-26 17:22:07 -07:00
backtracetest.c
…
bounds.c
…
capability.c
kernel: conditionally support non-root users, groups and capabilities
2015-04-15 16:35:22 -07:00
cgroup_freezer.c
…
cgroup.c
cgroup: remove use of seq_printf return value
2015-04-15 16:35:25 -07:00
compat.c
all arches, signal: move restart_block to struct task_struct
2015-02-12 18:54:12 -08:00
configs.c
…
context_tracking.c
context_tracking: Export context_tracking_user_enter/exit
2015-03-09 15:43:00 +01:00
cpu_pm.c
…
cpu.c
Merge branch 'core-rcu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
2015-04-14 13:36:04 -07:00
cpuset.c
kernel, cpuset: remove exception for __GFP_THISNODE
2015-04-14 16:49:03 -07:00
crash_dump.c
…
cred.c
kernel: conditionally support non-root users, groups and capabilities
2015-04-15 16:35:22 -07:00
delayacct.c
…
dma.c
…
elfcore.c
…
exec_domain.c
Remove rest of exec domains.
2015-04-12 21:03:31 +02:00
exit.c
Remove execution domain support
2015-04-12 20:58:24 +02:00
extable.c
ftrace/x86/extable: Add is_ftrace_trampoline() function
2014-11-19 15:25:26 -05:00
fork.c
oprofile: reduce mmap_sem hold for mm->exe_file
2015-04-17 09:04:11 -04:00
freezer.c
freezer: remove obsolete comments in __thaw_task()
2014-10-21 23:44:20 +02:00
futex_compat.c
…
futex.c
Merge tag 'v4.0-rc1' into locking/core, to refresh the tree before merging new changes
2015-02-24 08:41:07 +01:00
groups.c
kernel: conditionally support non-root users, groups and capabilities
2015-04-15 16:35:22 -07:00
hung_task.c
kernel/hung_task.c: change hung_task.c to use for_each_process_thread()
2015-04-15 16:35:22 -07:00
irq_work.c
percpu: Convert remaining __get_cpu_var uses in 3.18-rcX
2014-10-29 11:18:18 -04:00
jump_label.c
…
kallsyms.c
kernel/kallsyms.c: use __seq_open_private()
2014-10-14 02:18:16 +02:00
kcmp.c
…
Kconfig.freezer
…
Kconfig.hz
…
Kconfig.locks
locking/mcs: Better differentiate between MCS variants
2015-01-14 15:07:32 +01:00
Kconfig.preempt
…
kexec.c
kexec: allocate the kexec control page with KEXEC_CONTROL_MEMORY_GFP
2015-04-23 16:52:01 +02:00
kmod.c
usermodehelper: kill the kmod_thread_locker logic
2014-12-10 17:41:17 -08:00
kprobes.c
kprobes: makes kprobes/enabled works correctly for optimized kprobes.
2015-02-13 21:21:42 -08:00
ksysfs.c
…
kthread.c
kernel/kthread.c: partial revert of 81c98869fa ("kthread: ensure locality of task_struct allocations")
2014-10-09 22:25:51 -04:00
latencytop.c
…
Makefile
modsign: change default key details
2015-04-30 09:35:41 -07:00
module_signing.c
…
module-internal.h
…
module.c
tracing: Rename ftrace_event.h to trace_events.h
2015-05-13 14:05:12 -04:00
notifier.c
rcu: Make SRCU optional by using CONFIG_SRCU
2015-01-06 11:04:29 -08:00
nsproxy.c
bury struct proc_ns in fs/proc
2014-12-04 14:34:54 -05:00
padata.c
padata: use %*pb[l] to print bitmaps including cpumasks and nodemasks
2015-02-13 21:21:38 -08:00
panic.c
livepatch: kernel: add TAINT_LIVEPATCH
2014-12-22 15:40:48 +01:00
params.c
params: handle quotes properly for values not of form foo="bar".
2015-04-15 13:31:23 +09:30
pid_namespace.c
Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs
2014-12-16 15:53:03 -08:00
pid.c
fork: report pid reservation failure properly
2015-04-17 09:04:06 -04:00
profile.c
profile: use %*pb[l] to print bitmaps including cpumasks and nodemasks
2015-02-13 21:21:38 -08:00
ptrace.c
ptrace: ptrace_detach() can no longer race with SIGKILL
2015-04-17 09:04:06 -04:00
range.c
kernel: avoid overflow in cmp_range
2015-01-17 10:02:23 +13:00
reboot.c
kernel/reboot.c: add orderly_reboot for graceful reboot
2015-04-15 16:35:23 -07:00
relay.c
VFS: kernel/: d_inode() annotations
2015-04-15 15:06:55 -04:00
resource.c
kernel/resource.c: remove deprecated __check_region() and friends
2015-04-15 16:35:22 -07:00
seccomp.c
seccomp: cap SECCOMP_RET_ERRNO data to MAX_ERRNO
2015-02-17 14:34:55 -08:00
signal.c
signal: remove warning about using SI_TKILL in rt_[tg]sigqueueinfo
2015-04-17 09:04:06 -04:00
smp.c
smp: Fix error case handling in smp_call_function_*()
2015-04-19 13:19:23 -07:00
smpboot.c
smpboot: Add common code for notification from dying CPU
2015-03-11 13:20:25 -07:00
smpboot.h
…
softirq.c
Merge branch 'locking-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
2015-02-09 15:24:03 -08:00
stacktrace.c
stacktrace: introduce snprint_stack_trace for buffer output
2014-12-13 12:42:48 -08:00
stop_machine.c
…
sys_ni.c
kernel: conditionally support non-root users, groups and capabilities
2015-04-15 16:35:22 -07:00
sys.c
prctl: avoid using mmap_sem for exe_file serialization
2015-04-17 09:04:07 -04:00
sysctl_binary.c
kernel: add panic_on_warn
2014-12-10 17:41:10 -08:00
sysctl.c
kernel/sysctl.c: detect overflows when converting to int
2015-04-17 09:04:08 -04:00
system_certificates.S
…
system_keyring.c
…
task_work.c
…
taskstats.c
netlink: make nlmsg_end() and genlmsg_end() void
2015-01-18 01:03:45 -05:00
test_kprobes.c
…
torture.c
…
tracepoint.c
…
tsacct.c
…
uid16.c
groups: Consolidate the setgroups permission checks
2014-12-05 17:19:27 -06:00
up.c
…
user_namespace.c
Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiederm/user-namespace
2014-12-17 12:31:40 -08:00
user-return-notifier.c
…
user.c
Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiederm/user-namespace
2014-12-17 12:31:40 -08:00
utsname_sysctl.c
…
utsname.c
copy address of proc_ns_ops into ns_common
2014-12-04 14:34:47 -05:00
watchdog.c
Merge branch 'akpm' (patches from Andrew)
2015-04-14 16:49:17 -07:00
workqueue_internal.h
…
workqueue.c
workqueue: Reorder sysfs code
2015-04-06 11:16:04 -04:00