mirror/linux
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge tag 'locking-urgent-2020-06-11' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Browse Source 
Pull atomics rework from Thomas Gleixner:
 "Peter Zijlstras rework of atomics and fallbacks. This solves two
  problems:

   1) Compilers uninline small atomic_* static inline functions which
      can expose them to instrumentation.

   2) The instrumentation of atomic primitives was done at the
      architecture level while composites or fallbacks were provided at
      the generic level. As a result there are no uninstrumented
      variants of the fallbacks.

  Both issues were in the way of fully isolating fragile entry code
  pathes and especially the text poke int3 handler which is prone to an
  endless recursion problem when anything in that code path is about to
  be instrumented. This was always a problem, but got elevated due to
  the new batch mode updates of tracing.

  The solution is to mark the functions __always_inline and to flip the
  fallback and instrumentation so the non-instrumented variants are at
  the architecture level and the instrumentation is done in generic
  code.

  The latter introduces another fallback variant which will go away once
  all architectures have been moved over to arch_atomic_*"

* tag 'locking-urgent-2020-06-11' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  locking/atomics: Flip fallbacks and instrumentation
  asm-generic/atomic: Use __always_inline for fallback wrappers
This commit is contained in:
Linus Torvalds
2020-06-11 18:27:19 -07:00
parent b1a6274994 37f8173dd8
commit 9716e57a01
28 changed files with 2594 additions and 269 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
scripts/atomic/fallbacks/acquire
View File

@@ -1,8 +1,8 @@
cat <<EOF
static inline ${ret}
${atomic}_${pfx}${name}${sfx}_acquire(${params})
static __always_inline ${ret}
${arch}${atomic}_${pfx}${name}${sfx}_acquire(${params})
{
${ret} ret = ${atomic}_${pfx}${name}${sfx}_relaxed(${args});
${ret} ret = ${arch}${atomic}_${pfx}${name}${sfx}_relaxed(${args});
__atomic_acquire_fence();
return ret;
}

8
scripts/atomic/fallbacks/add_negative
View File

@@ -1,6 +1,6 @@
cat <<EOF
/**
* ${atomic}_add_negative - add and test if negative
* ${arch}${atomic}_add_negative - add and test if negative
* @i: integer value to add
* @v: pointer of type ${atomic}_t
*
@@ -8,9 +8,9 @@ cat <<EOF
* if the result is negative, or false when
* result is greater than or equal to zero.
*/
static inline bool
${atomic}_add_negative(${int} i, ${atomic}_t *v)
static __always_inline bool
${arch}${atomic}_add_negative(${int} i, ${atomic}_t *v)
{
return ${atomic}_add_return(i, v) < 0;
return ${arch}${atomic}_add_return(i, v) < 0;
}
EOF

8
scripts/atomic/fallbacks/add_unless
View File

@@ -1,6 +1,6 @@
cat << EOF
/**
* ${atomic}_add_unless - add unless the number is already a given value
* ${arch}${atomic}_add_unless - add unless the number is already a given value
* @v: pointer of type ${atomic}_t
* @a: the amount to add to v...
* @u: ...unless v is equal to u.
@@ -8,9 +8,9 @@ cat << EOF
* Atomically adds @a to @v, if @v was not already @u.
* Returns true if the addition was done.
*/
static inline bool
${atomic}_add_unless(${atomic}_t *v, ${int} a, ${int} u)
static __always_inline bool
${arch}${atomic}_add_unless(${atomic}_t *v, ${int} a, ${int} u)
{
return ${atomic}_fetch_add_unless(v, a, u) != u;
return ${arch}${atomic}_fetch_add_unless(v, a, u) != u;
}
EOF

6
scripts/atomic/fallbacks/andnot
View File

@@ -1,7 +1,7 @@
cat <<EOF
static inline ${ret}
${atomic}_${pfx}andnot${sfx}${order}(${int} i, ${atomic}_t *v)
static __always_inline ${ret}
${arch}${atomic}_${pfx}andnot${sfx}${order}(${int} i, ${atomic}_t *v)
{
${retstmt}${atomic}_${pfx}and${sfx}${order}(~i, v);
${retstmt}${arch}${atomic}_${pfx}and${sfx}${order}(~i, v);
}
EOF

6
scripts/atomic/fallbacks/dec
View File

@@ -1,7 +1,7 @@
cat <<EOF
static inline ${ret}
${atomic}_${pfx}dec${sfx}${order}(${atomic}_t *v)
static __always_inline ${ret}
${arch}${atomic}_${pfx}dec${sfx}${order}(${atomic}_t *v)
{
${retstmt}${atomic}_${pfx}sub${sfx}${order}(1, v);
${retstmt}${arch}${atomic}_${pfx}sub${sfx}${order}(1, v);
}
EOF

8
scripts/atomic/fallbacks/dec_and_test
View File

@@ -1,15 +1,15 @@
cat <<EOF
/**
* ${atomic}_dec_and_test - decrement and test
* ${arch}${atomic}_dec_and_test - decrement and test
* @v: pointer of type ${atomic}_t
*
* Atomically decrements @v by 1 and
* returns true if the result is 0, or false for all other
* cases.
*/
static inline bool
${atomic}_dec_and_test(${atomic}_t *v)
static __always_inline bool
${arch}${atomic}_dec_and_test(${atomic}_t *v)
{
return ${atomic}_dec_return(v) == 0;
return ${arch}${atomic}_dec_return(v) == 0;
}
EOF

8
scripts/atomic/fallbacks/dec_if_positive
View File

@@ -1,14 +1,14 @@
cat <<EOF
static inline ${ret}
${atomic}_dec_if_positive(${atomic}_t *v)
static __always_inline ${ret}
${arch}${atomic}_dec_if_positive(${atomic}_t *v)
{
${int} dec, c = ${atomic}_read(v);
${int} dec, c = ${arch}${atomic}_read(v);
do {
dec = c - 1;
if (unlikely(dec < 0))
break;
} while (!${atomic}_try_cmpxchg(v, &c, dec));
} while (!${arch}${atomic}_try_cmpxchg(v, &c, dec));
return dec;
}

8
scripts/atomic/fallbacks/dec_unless_positive
View File

@@ -1,13 +1,13 @@
cat <<EOF
static inline bool
${atomic}_dec_unless_positive(${atomic}_t *v)
static __always_inline bool
${arch}${atomic}_dec_unless_positive(${atomic}_t *v)
{
${int} c = ${atomic}_read(v);
${int} c = ${arch}${atomic}_read(v);
do {
if (unlikely(c > 0))
return false;
} while (!${atomic}_try_cmpxchg(v, &c, c - 1));
} while (!${arch}${atomic}_try_cmpxchg(v, &c, c - 1));
return true;
}

6
scripts/atomic/fallbacks/fence
View File

@@ -1,10 +1,10 @@
cat <<EOF
static inline ${ret}
${atomic}_${pfx}${name}${sfx}(${params})
static __always_inline ${ret}
${arch}${atomic}_${pfx}${name}${sfx}(${params})
{
${ret} ret;
__atomic_pre_full_fence();
ret = ${atomic}_${pfx}${name}${sfx}_relaxed(${args});
ret = ${arch}${atomic}_${pfx}${name}${sfx}_relaxed(${args});
__atomic_post_full_fence();
return ret;
}

10
scripts/atomic/fallbacks/fetch_add_unless
View File

@@ -1,6 +1,6 @@
cat << EOF
/**
* ${atomic}_fetch_add_unless - add unless the number is already a given value
* ${arch}${atomic}_fetch_add_unless - add unless the number is already a given value
* @v: pointer of type ${atomic}_t
* @a: the amount to add to v...
* @u: ...unless v is equal to u.
@@ -8,15 +8,15 @@ cat << EOF
* Atomically adds @a to @v, so long as @v was not already @u.
* Returns original value of @v
*/
static inline ${int}
${atomic}_fetch_add_unless(${atomic}_t *v, ${int} a, ${int} u)
static __always_inline ${int}
${arch}${atomic}_fetch_add_unless(${atomic}_t *v, ${int} a, ${int} u)
{
${int} c = ${atomic}_read(v);
${int} c = ${arch}${atomic}_read(v);
do {
if (unlikely(c == u))
break;
} while (!${atomic}_try_cmpxchg(v, &c, c + a));
} while (!${arch}${atomic}_try_cmpxchg(v, &c, c + a));
return c;
}

6
scripts/atomic/fallbacks/inc
View File

@@ -1,7 +1,7 @@
cat <<EOF
static inline ${ret}
${atomic}_${pfx}inc${sfx}${order}(${atomic}_t *v)
static __always_inline ${ret}
${arch}${atomic}_${pfx}inc${sfx}${order}(${atomic}_t *v)
{
${retstmt}${atomic}_${pfx}add${sfx}${order}(1, v);
${retstmt}${arch}${atomic}_${pfx}add${sfx}${order}(1, v);
}
EOF

8
scripts/atomic/fallbacks/inc_and_test
View File

@@ -1,15 +1,15 @@
cat <<EOF
/**
* ${atomic}_inc_and_test - increment and test
* ${arch}${atomic}_inc_and_test - increment and test
* @v: pointer of type ${atomic}_t
*
* Atomically increments @v by 1
* and returns true if the result is zero, or false for all
* other cases.
*/
static inline bool
${atomic}_inc_and_test(${atomic}_t *v)
static __always_inline bool
${arch}${atomic}_inc_and_test(${atomic}_t *v)
{
return ${atomic}_inc_return(v) == 0;
return ${arch}${atomic}_inc_return(v) == 0;
}
EOF

8
scripts/atomic/fallbacks/inc_not_zero
View File

@@ -1,14 +1,14 @@
cat <<EOF
/**
* ${atomic}_inc_not_zero - increment unless the number is zero
* ${arch}${atomic}_inc_not_zero - increment unless the number is zero
* @v: pointer of type ${atomic}_t
*
* Atomically increments @v by 1, if @v is non-zero.
* Returns true if the increment was done.
*/
static inline bool
${atomic}_inc_not_zero(${atomic}_t *v)
static __always_inline bool
${arch}${atomic}_inc_not_zero(${atomic}_t *v)
{
return ${atomic}_add_unless(v, 1, 0);
return ${arch}${atomic}_add_unless(v, 1, 0);
}
EOF

8
scripts/atomic/fallbacks/inc_unless_negative
View File

@@ -1,13 +1,13 @@
cat <<EOF
static inline bool
${atomic}_inc_unless_negative(${atomic}_t *v)
static __always_inline bool
${arch}${atomic}_inc_unless_negative(${atomic}_t *v)
{
${int} c = ${atomic}_read(v);
${int} c = ${arch}${atomic}_read(v);
do {
if (unlikely(c < 0))
return false;
} while (!${atomic}_try_cmpxchg(v, &c, c + 1));
} while (!${arch}${atomic}_try_cmpxchg(v, &c, c + 1));
return true;
}

4
scripts/atomic/fallbacks/read_acquire
View File

@@ -1,6 +1,6 @@
cat <<EOF
static inline ${ret}
${atomic}_read_acquire(const ${atomic}_t *v)
static __always_inline ${ret}
${arch}${atomic}_read_acquire(const ${atomic}_t *v)
{
return smp_load_acquire(&(v)->counter);
}

6
scripts/atomic/fallbacks/release
View File

@@ -1,8 +1,8 @@
cat <<EOF
static inline ${ret}
${atomic}_${pfx}${name}${sfx}_release(${params})
static __always_inline ${ret}
${arch}${atomic}_${pfx}${name}${sfx}_release(${params})
{
__atomic_release_fence();
${retstmt}${atomic}_${pfx}${name}${sfx}_relaxed(${args});
${retstmt}${arch}${atomic}_${pfx}${name}${sfx}_relaxed(${args});
}
EOF

4
scripts/atomic/fallbacks/set_release
View File

@@ -1,6 +1,6 @@
cat <<EOF
static inline void
${atomic}_set_release(${atomic}_t *v, ${int} i)
static __always_inline void
${arch}${atomic}_set_release(${atomic}_t *v, ${int} i)
{
smp_store_release(&(v)->counter, i);
}

8
scripts/atomic/fallbacks/sub_and_test
View File

@@ -1,6 +1,6 @@
cat <<EOF
/**
* ${atomic}_sub_and_test - subtract value from variable and test result
* ${arch}${atomic}_sub_and_test - subtract value from variable and test result
* @i: integer value to subtract
* @v: pointer of type ${atomic}_t
*
@@ -8,9 +8,9 @@ cat <<EOF
* true if the result is zero, or false for all
* other cases.
*/
static inline bool
${atomic}_sub_and_test(${int} i, ${atomic}_t *v)
static __always_inline bool
${arch}${atomic}_sub_and_test(${int} i, ${atomic}_t *v)
{
return ${atomic}_sub_return(i, v) == 0;
return ${arch}${atomic}_sub_return(i, v) == 0;
}
EOF

6
scripts/atomic/fallbacks/try_cmpxchg
View File

@@ -1,9 +1,9 @@
cat <<EOF
static inline bool
${atomic}_try_cmpxchg${order}(${atomic}_t *v, ${int} *old, ${int} new)
static __always_inline bool
${arch}${atomic}_try_cmpxchg${order}(${atomic}_t *v, ${int} *old, ${int} new)
{
${int} r, o = *old;
r = ${atomic}_cmpxchg${order}(v, o, new);
r = ${arch}${atomic}_cmpxchg${order}(v, o, new);
if (unlikely(r != o))
*old = r;
return likely(r == o);

31
scripts/atomic/gen-atomic-fallback.sh
View File

@@ -2,10 +2,11 @@
# SPDX-License-Identifier: GPL-2.0
ATOMICDIR=$(dirname $0)
ARCH=$2
. ${ATOMICDIR}/atomic-tbl.sh
#gen_template_fallback(template, meta, pfx, name, sfx, order, atomic, int, args...)
#gen_template_fallback(template, meta, pfx, name, sfx, order, arch, atomic, int, args...)
gen_template_fallback()
{
local template="$1"; shift
@@ -14,10 +15,11 @@ gen_template_fallback()
local name="$1"; shift
local sfx="$1"; shift
local order="$1"; shift
local arch="$1"; shift
local atomic="$1"; shift
local int="$1"; shift
local atomicname="${atomic}_${pfx}${name}${sfx}${order}"
local atomicname="${arch}${atomic}_${pfx}${name}${sfx}${order}"
local ret="$(gen_ret_type "${meta}" "${int}")"
local retstmt="$(gen_ret_stmt "${meta}")"
@@ -32,7 +34,7 @@ gen_template_fallback()
fi
}
#gen_proto_fallback(meta, pfx, name, sfx, order, atomic, int, args...)
#gen_proto_fallback(meta, pfx, name, sfx, order, arch, atomic, int, args...)
gen_proto_fallback()
{
local meta="$1"; shift
@@ -56,16 +58,17 @@ cat << EOF
EOF
}
#gen_proto_order_variants(meta, pfx, name, sfx, atomic, int, args...)
#gen_proto_order_variants(meta, pfx, name, sfx, arch, atomic, int, args...)
gen_proto_order_variants()
{
local meta="$1"; shift
local pfx="$1"; shift
local name="$1"; shift
local sfx="$1"; shift
local atomic="$1"
local arch="$1"
local atomic="$2"
local basename="${atomic}_${pfx}${name}${sfx}"
local basename="${arch}${atomic}_${pfx}${name}${sfx}"
local template="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")"
@@ -94,7 +97,7 @@ gen_proto_order_variants()
gen_basic_fallbacks "${basename}"
if [ ! -z "${template}" ]; then
printf "#endif /* ${atomic}_${pfx}${name}${sfx} */\n\n"
printf "#endif /* ${arch}${atomic}_${pfx}${name}${sfx} */\n\n"
gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"
gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"
gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"
@@ -149,20 +152,19 @@ cat << EOF
#ifndef _LINUX_ATOMIC_FALLBACK_H
#define _LINUX_ATOMIC_FALLBACK_H
#include <linux/compiler.h>
EOF
for xchg in "xchg" "cmpxchg" "cmpxchg64"; do
for xchg in "${ARCH}xchg" "${ARCH}cmpxchg" "${ARCH}cmpxchg64"; do
gen_xchg_fallbacks "${xchg}"
done
grep '^[a-z]' "$1" | while read name meta args; do
gen_proto "${meta}" "${name}" "atomic" "int" ${args}
gen_proto "${meta}" "${name}" "${ARCH}" "atomic" "int" ${args}
done
cat <<EOF
#define atomic_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
#define atomic_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
#ifdef CONFIG_GENERIC_ATOMIC64
#include <asm-generic/atomic64.h>
#endif
@@ -170,12 +172,9 @@ cat <<EOF
EOF
grep '^[a-z]' "$1" | while read name meta args; do
gen_proto "${meta}" "${name}" "atomic64" "s64" ${args}
gen_proto "${meta}" "${name}" "${ARCH}" "atomic64" "s64" ${args}
done
cat <<EOF
#define atomic64_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
#define atomic64_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
#endif /* _LINUX_ATOMIC_FALLBACK_H */
EOF

5
scripts/atomic/gen-atomics.sh
View File

@@ -10,10 +10,11 @@ LINUXDIR=${ATOMICDIR}/../..
cat <<EOF |
gen-atomic-instrumented.sh asm-generic/atomic-instrumented.h
gen-atomic-long.sh asm-generic/atomic-long.h
gen-atomic-fallback.sh linux/atomic-arch-fallback.h arch_
gen-atomic-fallback.sh linux/atomic-fallback.h
EOF
while read script header; do
/bin/sh ${ATOMICDIR}/${script} ${ATOMICTBL} > ${LINUXDIR}/include/${header}
while read script header args; do
/bin/sh ${ATOMICDIR}/${script} ${ATOMICTBL} ${args} > ${LINUXDIR}/include/${header}
HASH="$(sha1sum ${LINUXDIR}/include/${header})"
HASH="${HASH%% *}"
printf "// %s\n" "${HASH}" >> ${LINUXDIR}/include/${header}