Eric Dumazet
5d4cc87414
net: reorganize "struct sock" fields
Last major reorg happened in commit 9115e8cd2a ("net: reorganize
struct sock for better data locality")
Since then, many changes have been done.
Before SO_PEEK_OFF support is added to TCP, we need
to move sk_peek_off to a better location.
It is time to make another pass, and add six groups,
without explicit alignment.
- sock_write_rx (following sk_refcnt) read-write fields in rx path.
- sock_read_rx read-mostly fields in rx path.
- sock_read_rxtx read-mostly fields in both rx and tx paths.
- sock_write_rxtx read-write fields in both rx and tx paths.
- sock_write_tx read-write fields in tx paths.
- sock_read_tx read-mostly fields in tx paths.
Results on TCP_RR benchmarks seem to show a gain (4 to 5 %).
It is possible UDP needs a change, because sk_peek_off
shares a cache line with sk_receive_queue.
If this the case, we can exchange roles of sk->sk_receive
and up->reader_queue queues.
After this change, we have the following layout:
struct sock {
struct sock_common __sk_common; /* 0 0x88 */
/* --- cacheline 2 boundary (128 bytes) was 8 bytes ago --- */
__u8 __cacheline_group_begin__sock_write_rx[0]; /* 0x88 0 */
atomic_t sk_drops; /* 0x88 0x4 */
__s32 sk_peek_off; /* 0x8c 0x4 */
struct sk_buff_head sk_error_queue; /* 0x90 0x18 */
struct sk_buff_head sk_receive_queue; /* 0xa8 0x18 */
/* --- cacheline 3 boundary (192 bytes) --- */
struct {
atomic_t rmem_alloc; /* 0xc0 0x4 */
int len; /* 0xc4 0x4 */
struct sk_buff * head; /* 0xc8 0x8 */
struct sk_buff * tail; /* 0xd0 0x8 */
} sk_backlog; /* 0xc0 0x18 */
struct {
atomic_t rmem_alloc; /* 0 0x4 */
int len; /* 0x4 0x4 */
struct sk_buff * head; /* 0x8 0x8 */
struct sk_buff * tail; /* 0x10 0x8 */
/* size: 24, cachelines: 1, members: 4 */
/* last cacheline: 24 bytes */
};
__u8 __cacheline_group_end__sock_write_rx[0]; /* 0xd8 0 */
__u8 __cacheline_group_begin__sock_read_rx[0]; /* 0xd8 0 */
rcu * sk_rx_dst; /* 0xd8 0x8 */
int sk_rx_dst_ifindex; /* 0xe0 0x4 */
u32 sk_rx_dst_cookie; /* 0xe4 0x4 */
unsigned int sk_ll_usec; /* 0xe8 0x4 */
unsigned int sk_napi_id; /* 0xec 0x4 */
u16 sk_busy_poll_budget; /* 0xf0 0x2 */
u8 sk_prefer_busy_poll; /* 0xf2 0x1 */
u8 sk_userlocks; /* 0xf3 0x1 */
int sk_rcvbuf; /* 0xf4 0x4 */
rcu * sk_filter; /* 0xf8 0x8 */
/* --- cacheline 4 boundary (256 bytes) --- */
union {
rcu * sk_wq; /* 0x100 0x8 */
struct socket_wq * sk_wq_raw; /* 0x100 0x8 */
}; /* 0x100 0x8 */
union {
rcu * sk_wq; /* 0 0x8 */
struct socket_wq * sk_wq_raw; /* 0 0x8 */
};
void (*sk_data_ready)(struct sock *); /* 0x108 0x8 */
long sk_rcvtimeo; /* 0x110 0x8 */
int sk_rcvlowat; /* 0x118 0x4 */
__u8 __cacheline_group_end__sock_read_rx[0]; /* 0x11c 0 */
__u8 __cacheline_group_begin__sock_read_rxtx[0]; /* 0x11c 0 */
int sk_err; /* 0x11c 0x4 */
struct socket * sk_socket; /* 0x120 0x8 */
struct mem_cgroup * sk_memcg; /* 0x128 0x8 */
rcu * sk_policy[2]; /* 0x130 0x10 */
/* --- cacheline 5 boundary (320 bytes) --- */
__u8 __cacheline_group_end__sock_read_rxtx[0]; /* 0x140 0 */
__u8 __cacheline_group_begin__sock_write_rxtx[0]; /* 0x140 0 */
socket_lock_t sk_lock; /* 0x140 0x20 */
u32 sk_reserved_mem; /* 0x160 0x4 */
int sk_forward_alloc; /* 0x164 0x4 */
u32 sk_tsflags; /* 0x168 0x4 */
__u8 __cacheline_group_end__sock_write_rxtx[0]; /* 0x16c 0 */
__u8 __cacheline_group_begin__sock_write_tx[0]; /* 0x16c 0 */
int sk_write_pending; /* 0x16c 0x4 */
atomic_t sk_omem_alloc; /* 0x170 0x4 */
int sk_sndbuf; /* 0x174 0x4 */
int sk_wmem_queued; /* 0x178 0x4 */
refcount_t sk_wmem_alloc; /* 0x17c 0x4 */
/* --- cacheline 6 boundary (384 bytes) --- */
unsigned long sk_tsq_flags; /* 0x180 0x8 */
union {
struct sk_buff * sk_send_head; /* 0x188 0x8 */
struct rb_root tcp_rtx_queue; /* 0x188 0x8 */
}; /* 0x188 0x8 */
union {
struct sk_buff * sk_send_head; /* 0 0x8 */
struct rb_root tcp_rtx_queue; /* 0 0x8 */
};
struct sk_buff_head sk_write_queue; /* 0x190 0x18 */
u32 sk_dst_pending_confirm; /* 0x1a8 0x4 */
u32 sk_pacing_status; /* 0x1ac 0x4 */
struct page_frag sk_frag; /* 0x1b0 0x10 */
/* --- cacheline 7 boundary (448 bytes) --- */
struct timer_list sk_timer; /* 0x1c0 0x28 */
/* XXX last struct has 4 bytes of padding */
unsigned long sk_pacing_rate; /* 0x1e8 0x8 */
atomic_t sk_zckey; /* 0x1f0 0x4 */
atomic_t sk_tskey; /* 0x1f4 0x4 */
__u8 __cacheline_group_end__sock_write_tx[0]; /* 0x1f8 0 */
__u8 __cacheline_group_begin__sock_read_tx[0]; /* 0x1f8 0 */
unsigned long sk_max_pacing_rate; /* 0x1f8 0x8 */
/* --- cacheline 8 boundary (512 bytes) --- */
long sk_sndtimeo; /* 0x200 0x8 */
u32 sk_priority; /* 0x208 0x4 */
u32 sk_mark; /* 0x20c 0x4 */
rcu * sk_dst_cache; /* 0x210 0x8 */
netdev_features_t sk_route_caps; /* 0x218 0x8 */
u16 sk_gso_type; /* 0x220 0x2 */
u16 sk_gso_max_segs; /* 0x222 0x2 */
unsigned int sk_gso_max_size; /* 0x224 0x4 */
gfp_t sk_allocation; /* 0x228 0x4 */
u32 sk_txhash; /* 0x22c 0x4 */
u8 sk_pacing_shift; /* 0x230 0x1 */
bool sk_use_task_frag; /* 0x231 0x1 */
__u8 __cacheline_group_end__sock_read_tx[0]; /* 0x232 0 */
u8 sk_gso_disabled:1; /* 0x232: 0 0x1 */
u8 sk_kern_sock:1; /* 0x232:0x1 0x1 */
u8 sk_no_check_tx:1; /* 0x232:0x2 0x1 */
u8 sk_no_check_rx:1; /* 0x232:0x3 0x1 */
/* XXX 4 bits hole, try to pack */
u8 sk_shutdown; /* 0x233 0x1 */
u16 sk_type; /* 0x234 0x2 */
u16 sk_protocol; /* 0x236 0x2 */
unsigned long sk_lingertime; /* 0x238 0x8 */
/* --- cacheline 9 boundary (576 bytes) --- */
struct proto * sk_prot_creator; /* 0x240 0x8 */
rwlock_t sk_callback_lock; /* 0x248 0x8 */
int sk_err_soft; /* 0x250 0x4 */
u32 sk_ack_backlog; /* 0x254 0x4 */
u32 sk_max_ack_backlog; /* 0x258 0x4 */
kuid_t sk_uid; /* 0x25c 0x4 */
spinlock_t sk_peer_lock; /* 0x260 0x4 */
int sk_bind_phc; /* 0x264 0x4 */
struct pid * sk_peer_pid; /* 0x268 0x8 */
const struct cred * sk_peer_cred; /* 0x270 0x8 */
ktime_t sk_stamp; /* 0x278 0x8 */
/* --- cacheline 10 boundary (640 bytes) --- */
int sk_disconnects; /* 0x280 0x4 */
u8 sk_txrehash; /* 0x284 0x1 */
u8 sk_clockid; /* 0x285 0x1 */
u8 sk_txtime_deadline_mode:1; /* 0x286: 0 0x1 */
u8 sk_txtime_report_errors:1; /* 0x286:0x1 0x1 */
u8 sk_txtime_unused:6; /* 0x286:0x2 0x1 */
/* XXX 1 byte hole, try to pack */
void * sk_user_data; /* 0x288 0x8 */
void * sk_security; /* 0x290 0x8 */
struct sock_cgroup_data sk_cgrp_data; /* 0x298 0x8 */
void (*sk_state_change)(struct sock *); /* 0x2a0 0x8 */
void (*sk_write_space)(struct sock *); /* 0x2a8 0x8 */
void (*sk_error_report)(struct sock *); /* 0x2b0 0x8 */
int (*sk_backlog_rcv)(struct sock *, struct sk_buff *); /* 0x2b8 0x8 */
/* --- cacheline 11 boundary (704 bytes) --- */
void (*sk_destruct)(struct sock *); /* 0x2c0 0x8 */
rcu * sk_reuseport_cb; /* 0x2c8 0x8 */
rcu * sk_bpf_storage; /* 0x2d0 0x8 */
struct callback_head sk_rcu __attribute__((__aligned__(8))); /* 0x2d8 0x10 */
netns_tracker ns_tracker; /* 0x2e8 0x8 */
/* size: 752, cachelines: 12, members: 105 */
/* sum members: 749, holes: 1, sum holes: 1 */
/* sum bitfield members: 12 bits, bit holes: 1, sum bit holes: 4 bits */
/* paddings: 1, sum paddings: 4 */
/* forced alignments: 1 */
/* last cacheline: 48 bytes */
};
Signed-off-by: Eric Dumazet <edumazet@google.com>
Acked-by: Paolo Abeni <pabeni@redhat.com>
Link: https://lore.kernel.org/r/20240216162006.2342759-1-edumazet@google.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
2024-02-20 12:01:45 +01:00
..
2023-03-27 02:33:48 +00:00
2023-12-22 12:56:21 -05:00
2023-08-10 18:24:48 -07:00
2023-12-29 07:46:38 +00:00
2024-01-12 13:52:21 -08:00
2024-02-08 15:30:33 -08:00
2023-12-15 11:01:27 +00:00
2021-11-17 20:17:05 -08:00
2024-02-19 12:30:27 -08:00
2023-06-15 22:33:26 -07:00
2024-02-05 11:08:06 +00:00
2024-01-02 12:41:16 +00:00
2024-02-02 10:57:55 -08:00
2024-02-15 15:34:40 +01:00
2023-12-24 15:22:50 +00:00
2024-02-01 15:12:37 -08:00
2023-12-15 10:37:35 +00:00
2022-07-22 12:53:22 +01:00
2023-03-21 21:32:18 -07:00
2023-03-18 12:23:34 +00:00
2022-08-09 22:14:02 -07:00
2021-12-13 12:34:09 +00:00
2024-02-06 13:17:54 +01:00
2022-11-02 20:38:13 -07:00
2024-02-06 13:17:54 +01:00
2024-02-06 13:17:54 +01:00
2021-03-26 17:43:55 +01:00
2024-02-09 10:01:09 -08:00
2022-11-16 11:31:47 +02:00
2024-02-02 13:08:58 +01:00
2023-12-15 11:14:57 +01:00
2023-01-27 11:19:46 +00:00
2021-12-22 15:03:47 -08:00
2022-07-22 12:53:22 +01:00
2023-07-14 20:39:29 -07:00
2022-07-24 18:39:17 -06:00
2023-07-27 17:17:32 -07:00
2022-07-22 12:53:22 +01:00
2023-01-20 09:33:22 +00:00
2023-10-20 11:34:51 +01:00
2023-12-20 11:50:13 +00:00
2023-08-14 08:01:06 +01:00
2023-10-24 13:08:14 -07:00
2024-01-31 12:30:47 +00:00
2021-11-29 19:50:45 -08:00
2022-12-05 21:58:27 -08:00
2023-08-22 13:19:02 +02:00
2024-02-06 11:45:53 +01:00
2022-07-22 12:53:22 +01:00
2022-07-22 12:53:22 +01:00
2022-07-22 12:53:22 +01:00
2021-12-06 16:06:02 -08:00
2024-01-03 16:42:48 -08:00
2022-07-28 22:21:54 -07:00
2023-08-02 10:09:31 +01:00
2023-10-06 11:37:02 +01:00
2023-11-02 09:31:02 +01:00
2023-04-12 16:40:39 -07:00
2022-12-01 15:09:10 +01:00
2023-08-08 15:58:23 -07:00
2022-07-22 12:53:22 +01:00
2021-10-18 12:54:41 +01:00
2024-02-15 17:49:16 -08:00
2022-10-31 10:43:04 +00:00
2021-01-29 20:39:14 -08:00
2023-10-06 07:30:40 +02:00
2023-06-10 00:11:41 -07:00
2022-07-22 12:53:22 +01:00
2022-07-22 12:53:22 +01:00
2023-07-28 14:07:59 -07:00
2022-07-22 12:53:22 +01:00
2021-06-28 14:29:45 -07:00
2023-10-12 15:14:27 +03:00
2023-11-20 11:43:03 +01:00
2023-12-08 10:40:51 +00:00
2022-07-22 12:53:22 +01:00
2023-08-15 13:57:51 -07:00
2023-08-14 07:06:13 +01:00
2024-01-19 21:13:25 -08:00
2022-02-07 20:12:45 -08:00
2021-10-15 11:33:08 +01:00
2023-07-14 20:39:29 -07:00
2023-12-22 22:15:35 +00:00
2024-01-21 18:09:30 +00:00
2023-12-22 22:15:35 +00:00
2022-02-17 07:00:39 -06:00
2021-11-16 13:16:54 +00:00
2024-02-12 10:24:12 +00:00
2024-02-12 10:24:12 +00:00
2022-04-25 11:40:45 +01:00
2023-10-05 13:16:47 -07:00
2024-02-07 18:55:12 -08:00
2023-04-22 01:39:41 +02:00
2024-01-26 21:05:26 -08:00
2022-09-29 07:18:00 +02:00
2022-07-22 12:53:22 +01:00
2022-10-31 20:14:27 -07:00
2023-10-30 14:36:57 -07:00
2023-12-22 22:15:34 +00:00
2023-08-22 21:40:40 +02:00
2023-06-12 21:13:23 -07:00
2021-03-23 14:14:50 -07:00
2023-08-04 15:33:17 -07:00
2023-08-04 15:33:17 -07:00
2022-07-22 12:53:22 +01:00
2023-07-20 10:46:28 +02:00
2021-10-13 09:40:46 -07:00
2024-01-19 21:30:09 -08:00
2022-07-22 12:53:22 +01:00
2022-07-22 12:53:22 +01:00
2022-07-28 11:29:36 +02:00
2021-11-07 19:25:29 +00:00
2023-08-18 16:05:26 +02:00
2023-12-21 20:35:15 +01:00
2022-10-12 12:57:19 +02:00
2023-12-27 13:08:10 +00:00
2023-06-20 20:06:16 -07:00
2021-10-29 13:23:51 +01:00
2021-03-26 15:14:56 -07:00
2022-07-22 12:53:22 +01:00
2023-08-22 17:31:18 -07:00
2022-11-18 12:14:55 +00:00
2022-07-22 12:53:22 +01:00
2023-08-07 08:53:55 +01:00
2023-11-28 12:18:29 +01:00
2022-05-11 12:43:10 +01:00
2024-02-07 18:55:10 -08:00
2021-12-10 06:38:26 -08:00
2024-01-13 18:26:23 +00:00
2023-12-04 18:04:05 -08:00
2022-07-22 12:53:22 +01:00
2023-11-20 10:15:16 -08:00
2024-02-07 12:38:30 +00:00
2023-12-19 15:31:40 +01:00
2024-02-20 09:22:58 +01:00
2022-07-22 12:53:22 +01:00
2024-02-07 18:55:11 -08:00
2023-11-20 11:43:45 +01:00
2023-07-14 20:39:29 -07:00
2023-08-04 15:33:50 -07:00
2023-07-14 20:39:30 -07:00
2023-06-02 09:55:22 +01:00
2024-02-02 10:57:55 -08:00
2024-02-02 10:57:55 -08:00
2022-07-22 12:53:22 +01:00
2022-07-15 18:50:35 -07:00
2021-08-09 15:34:21 -07:00
2022-07-22 12:53:22 +01:00
2023-04-06 12:01:20 -07:00
2023-03-17 08:56:37 +00:00
2022-10-11 17:42:58 -06:00
2023-09-18 09:44:05 +02:00
2024-01-23 14:40:24 -08:00
2022-08-01 11:59:23 -07:00
2024-02-12 17:33:05 -08:00
2023-06-19 11:32:58 -07:00
2023-07-14 20:39:30 -07:00
2023-08-20 11:40:03 +01:00
2024-01-24 01:33:59 +00:00
2024-01-26 20:34:25 -08:00
2022-07-22 12:53:22 +01:00
2022-01-04 12:17:35 +00:00
2021-04-28 14:06:45 -07:00
2023-12-26 20:24:33 +00:00
2022-10-25 11:35:16 +02:00
2024-02-20 12:01:45 +01:00
2023-10-04 11:49:20 -07:00
2022-07-22 12:53:22 +01:00
2022-07-18 11:24:10 +01:00
2023-08-09 13:12:15 -07:00
2024-01-02 12:41:16 +00:00
2024-01-04 18:06:46 -08:00
2023-12-04 14:45:26 -08:00
2024-01-23 15:08:03 -08:00
2023-10-23 15:01:53 -07:00
2023-07-28 14:07:59 -07:00
2024-02-10 21:38:19 +00:00
2023-08-01 15:06:27 -07:00
2022-12-12 15:04:39 -08:00
2022-07-22 12:53:22 +01:00
2023-10-23 08:48:57 +01:00
2023-08-07 08:53:55 +01:00
2023-09-14 16:16:36 +02:00
2023-11-16 22:33:31 +00:00
2023-03-18 12:23:34 +00:00
2022-07-22 12:53:22 +01:00
2024-01-24 16:24:06 -08:00
2023-11-29 14:59:40 -08:00
2023-12-14 16:38:59 +01:00
2024-02-05 16:45:49 -08:00
2023-12-13 16:16:40 -08:00