rhashtable: Remove RCU marking from rhash_lock_head
The opaque type rhash_lock_head should not be marked with __rcu because it can never be dereferenced. We should apply the RCU marking when we turn it into a pointer which can be dereferenced. This patch does exactly that. This fixes a number of sparse warnings as well as getting rid of some unnecessary RCU checking. Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au> Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
parent
c7d5ec26ea
commit
ba6306e3f6
|
@ -84,7 +84,7 @@ struct bucket_table {
|
||||||
|
|
||||||
struct lockdep_map dep_map;
|
struct lockdep_map dep_map;
|
||||||
|
|
||||||
struct rhash_lock_head __rcu *buckets[] ____cacheline_aligned_in_smp;
|
struct rhash_lock_head *buckets[] ____cacheline_aligned_in_smp;
|
||||||
};
|
};
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -261,13 +261,13 @@ void rhashtable_free_and_destroy(struct rhashtable *ht,
|
||||||
void *arg);
|
void *arg);
|
||||||
void rhashtable_destroy(struct rhashtable *ht);
|
void rhashtable_destroy(struct rhashtable *ht);
|
||||||
|
|
||||||
struct rhash_lock_head __rcu **rht_bucket_nested(const struct bucket_table *tbl,
|
struct rhash_lock_head **rht_bucket_nested(const struct bucket_table *tbl,
|
||||||
unsigned int hash);
|
unsigned int hash);
|
||||||
struct rhash_lock_head __rcu **__rht_bucket_nested(const struct bucket_table *tbl,
|
struct rhash_lock_head **__rht_bucket_nested(const struct bucket_table *tbl,
|
||||||
unsigned int hash);
|
unsigned int hash);
|
||||||
struct rhash_lock_head __rcu **rht_bucket_nested_insert(struct rhashtable *ht,
|
struct rhash_lock_head **rht_bucket_nested_insert(struct rhashtable *ht,
|
||||||
struct bucket_table *tbl,
|
struct bucket_table *tbl,
|
||||||
unsigned int hash);
|
unsigned int hash);
|
||||||
|
|
||||||
#define rht_dereference(p, ht) \
|
#define rht_dereference(p, ht) \
|
||||||
rcu_dereference_protected(p, lockdep_rht_mutex_is_held(ht))
|
rcu_dereference_protected(p, lockdep_rht_mutex_is_held(ht))
|
||||||
|
@ -284,21 +284,21 @@ struct rhash_lock_head __rcu **rht_bucket_nested_insert(struct rhashtable *ht,
|
||||||
#define rht_entry(tpos, pos, member) \
|
#define rht_entry(tpos, pos, member) \
|
||||||
({ tpos = container_of(pos, typeof(*tpos), member); 1; })
|
({ tpos = container_of(pos, typeof(*tpos), member); 1; })
|
||||||
|
|
||||||
static inline struct rhash_lock_head __rcu *const *rht_bucket(
|
static inline struct rhash_lock_head *const *rht_bucket(
|
||||||
const struct bucket_table *tbl, unsigned int hash)
|
const struct bucket_table *tbl, unsigned int hash)
|
||||||
{
|
{
|
||||||
return unlikely(tbl->nest) ? rht_bucket_nested(tbl, hash) :
|
return unlikely(tbl->nest) ? rht_bucket_nested(tbl, hash) :
|
||||||
&tbl->buckets[hash];
|
&tbl->buckets[hash];
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline struct rhash_lock_head __rcu **rht_bucket_var(
|
static inline struct rhash_lock_head **rht_bucket_var(
|
||||||
struct bucket_table *tbl, unsigned int hash)
|
struct bucket_table *tbl, unsigned int hash)
|
||||||
{
|
{
|
||||||
return unlikely(tbl->nest) ? __rht_bucket_nested(tbl, hash) :
|
return unlikely(tbl->nest) ? __rht_bucket_nested(tbl, hash) :
|
||||||
&tbl->buckets[hash];
|
&tbl->buckets[hash];
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline struct rhash_lock_head __rcu **rht_bucket_insert(
|
static inline struct rhash_lock_head **rht_bucket_insert(
|
||||||
struct rhashtable *ht, struct bucket_table *tbl, unsigned int hash)
|
struct rhashtable *ht, struct bucket_table *tbl, unsigned int hash)
|
||||||
{
|
{
|
||||||
return unlikely(tbl->nest) ? rht_bucket_nested_insert(ht, tbl, hash) :
|
return unlikely(tbl->nest) ? rht_bucket_nested_insert(ht, tbl, hash) :
|
||||||
|
@ -349,6 +349,12 @@ static inline void rht_unlock(struct bucket_table *tbl,
|
||||||
local_bh_enable();
|
local_bh_enable();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static inline struct rhash_head __rcu *__rht_ptr(
|
||||||
|
struct rhash_lock_head *const *bkt)
|
||||||
|
{
|
||||||
|
return (struct rhash_head __rcu *)((unsigned long)*bkt & ~BIT(0));
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Where 'bkt' is a bucket and might be locked:
|
* Where 'bkt' is a bucket and might be locked:
|
||||||
* rht_ptr() dereferences that pointer and clears the lock bit.
|
* rht_ptr() dereferences that pointer and clears the lock bit.
|
||||||
|
@ -356,30 +362,30 @@ static inline void rht_unlock(struct bucket_table *tbl,
|
||||||
* access is guaranteed, such as when destroying the table.
|
* access is guaranteed, such as when destroying the table.
|
||||||
*/
|
*/
|
||||||
static inline struct rhash_head *rht_ptr(
|
static inline struct rhash_head *rht_ptr(
|
||||||
struct rhash_lock_head __rcu * const *bkt,
|
struct rhash_lock_head *const *bkt,
|
||||||
struct bucket_table *tbl,
|
struct bucket_table *tbl,
|
||||||
unsigned int hash)
|
unsigned int hash)
|
||||||
{
|
{
|
||||||
const struct rhash_lock_head *p =
|
struct rhash_head __rcu *p = __rht_ptr(bkt);
|
||||||
rht_dereference_bucket_rcu(*bkt, tbl, hash);
|
|
||||||
|
|
||||||
if ((((unsigned long)p) & ~BIT(0)) == 0)
|
|
||||||
return RHT_NULLS_MARKER(bkt);
|
|
||||||
return (void *)(((unsigned long)p) & ~BIT(0));
|
|
||||||
}
|
|
||||||
|
|
||||||
static inline struct rhash_head *rht_ptr_exclusive(
|
|
||||||
struct rhash_lock_head __rcu * const *bkt)
|
|
||||||
{
|
|
||||||
const struct rhash_lock_head *p =
|
|
||||||
rcu_dereference_protected(*bkt, 1);
|
|
||||||
|
|
||||||
if (!p)
|
if (!p)
|
||||||
return RHT_NULLS_MARKER(bkt);
|
return RHT_NULLS_MARKER(bkt);
|
||||||
return (void *)(((unsigned long)p) & ~BIT(0));
|
|
||||||
|
return rht_dereference_bucket_rcu(p, tbl, hash);
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline void rht_assign_locked(struct rhash_lock_head __rcu **bkt,
|
static inline struct rhash_head *rht_ptr_exclusive(
|
||||||
|
struct rhash_lock_head *const *bkt)
|
||||||
|
{
|
||||||
|
struct rhash_head __rcu *p = __rht_ptr(bkt);
|
||||||
|
|
||||||
|
if (!p)
|
||||||
|
return RHT_NULLS_MARKER(bkt);
|
||||||
|
|
||||||
|
return rcu_dereference_protected(p, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline void rht_assign_locked(struct rhash_lock_head **bkt,
|
||||||
struct rhash_head *obj)
|
struct rhash_head *obj)
|
||||||
{
|
{
|
||||||
struct rhash_head __rcu **p = (struct rhash_head __rcu **)bkt;
|
struct rhash_head __rcu **p = (struct rhash_head __rcu **)bkt;
|
||||||
|
@ -390,7 +396,7 @@ static inline void rht_assign_locked(struct rhash_lock_head __rcu **bkt,
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline void rht_assign_unlock(struct bucket_table *tbl,
|
static inline void rht_assign_unlock(struct bucket_table *tbl,
|
||||||
struct rhash_lock_head __rcu **bkt,
|
struct rhash_lock_head **bkt,
|
||||||
struct rhash_head *obj)
|
struct rhash_head *obj)
|
||||||
{
|
{
|
||||||
struct rhash_head __rcu **p = (struct rhash_head __rcu **)bkt;
|
struct rhash_head __rcu **p = (struct rhash_head __rcu **)bkt;
|
||||||
|
@ -587,7 +593,7 @@ static inline struct rhash_head *__rhashtable_lookup(
|
||||||
.ht = ht,
|
.ht = ht,
|
||||||
.key = key,
|
.key = key,
|
||||||
};
|
};
|
||||||
struct rhash_lock_head __rcu * const *bkt;
|
struct rhash_lock_head *const *bkt;
|
||||||
struct bucket_table *tbl;
|
struct bucket_table *tbl;
|
||||||
struct rhash_head *he;
|
struct rhash_head *he;
|
||||||
unsigned int hash;
|
unsigned int hash;
|
||||||
|
@ -703,7 +709,7 @@ static inline void *__rhashtable_insert_fast(
|
||||||
.ht = ht,
|
.ht = ht,
|
||||||
.key = key,
|
.key = key,
|
||||||
};
|
};
|
||||||
struct rhash_lock_head __rcu **bkt;
|
struct rhash_lock_head **bkt;
|
||||||
struct rhash_head __rcu **pprev;
|
struct rhash_head __rcu **pprev;
|
||||||
struct bucket_table *tbl;
|
struct bucket_table *tbl;
|
||||||
struct rhash_head *head;
|
struct rhash_head *head;
|
||||||
|
@ -989,7 +995,7 @@ static inline int __rhashtable_remove_fast_one(
|
||||||
struct rhash_head *obj, const struct rhashtable_params params,
|
struct rhash_head *obj, const struct rhashtable_params params,
|
||||||
bool rhlist)
|
bool rhlist)
|
||||||
{
|
{
|
||||||
struct rhash_lock_head __rcu **bkt;
|
struct rhash_lock_head **bkt;
|
||||||
struct rhash_head __rcu **pprev;
|
struct rhash_head __rcu **pprev;
|
||||||
struct rhash_head *he;
|
struct rhash_head *he;
|
||||||
unsigned int hash;
|
unsigned int hash;
|
||||||
|
@ -1141,7 +1147,7 @@ static inline int __rhashtable_replace_fast(
|
||||||
struct rhash_head *obj_old, struct rhash_head *obj_new,
|
struct rhash_head *obj_old, struct rhash_head *obj_new,
|
||||||
const struct rhashtable_params params)
|
const struct rhashtable_params params)
|
||||||
{
|
{
|
||||||
struct rhash_lock_head __rcu **bkt;
|
struct rhash_lock_head **bkt;
|
||||||
struct rhash_head __rcu **pprev;
|
struct rhash_head __rcu **pprev;
|
||||||
struct rhash_head *he;
|
struct rhash_head *he;
|
||||||
unsigned int hash;
|
unsigned int hash;
|
||||||
|
|
|
@ -34,7 +34,7 @@
|
||||||
|
|
||||||
union nested_table {
|
union nested_table {
|
||||||
union nested_table __rcu *table;
|
union nested_table __rcu *table;
|
||||||
struct rhash_lock_head __rcu *bucket;
|
struct rhash_lock_head *bucket;
|
||||||
};
|
};
|
||||||
|
|
||||||
static u32 head_hashfn(struct rhashtable *ht,
|
static u32 head_hashfn(struct rhashtable *ht,
|
||||||
|
@ -216,7 +216,7 @@ static struct bucket_table *rhashtable_last_table(struct rhashtable *ht,
|
||||||
}
|
}
|
||||||
|
|
||||||
static int rhashtable_rehash_one(struct rhashtable *ht,
|
static int rhashtable_rehash_one(struct rhashtable *ht,
|
||||||
struct rhash_lock_head __rcu **bkt,
|
struct rhash_lock_head **bkt,
|
||||||
unsigned int old_hash)
|
unsigned int old_hash)
|
||||||
{
|
{
|
||||||
struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);
|
struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);
|
||||||
|
@ -269,7 +269,7 @@ static int rhashtable_rehash_chain(struct rhashtable *ht,
|
||||||
unsigned int old_hash)
|
unsigned int old_hash)
|
||||||
{
|
{
|
||||||
struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);
|
struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);
|
||||||
struct rhash_lock_head __rcu **bkt = rht_bucket_var(old_tbl, old_hash);
|
struct rhash_lock_head **bkt = rht_bucket_var(old_tbl, old_hash);
|
||||||
int err;
|
int err;
|
||||||
|
|
||||||
if (!bkt)
|
if (!bkt)
|
||||||
|
@ -478,7 +478,7 @@ fail:
|
||||||
}
|
}
|
||||||
|
|
||||||
static void *rhashtable_lookup_one(struct rhashtable *ht,
|
static void *rhashtable_lookup_one(struct rhashtable *ht,
|
||||||
struct rhash_lock_head __rcu **bkt,
|
struct rhash_lock_head **bkt,
|
||||||
struct bucket_table *tbl, unsigned int hash,
|
struct bucket_table *tbl, unsigned int hash,
|
||||||
const void *key, struct rhash_head *obj)
|
const void *key, struct rhash_head *obj)
|
||||||
{
|
{
|
||||||
|
@ -529,7 +529,7 @@ static void *rhashtable_lookup_one(struct rhashtable *ht,
|
||||||
}
|
}
|
||||||
|
|
||||||
static struct bucket_table *rhashtable_insert_one(struct rhashtable *ht,
|
static struct bucket_table *rhashtable_insert_one(struct rhashtable *ht,
|
||||||
struct rhash_lock_head __rcu **bkt,
|
struct rhash_lock_head **bkt,
|
||||||
struct bucket_table *tbl,
|
struct bucket_table *tbl,
|
||||||
unsigned int hash,
|
unsigned int hash,
|
||||||
struct rhash_head *obj,
|
struct rhash_head *obj,
|
||||||
|
@ -584,7 +584,7 @@ static void *rhashtable_try_insert(struct rhashtable *ht, const void *key,
|
||||||
{
|
{
|
||||||
struct bucket_table *new_tbl;
|
struct bucket_table *new_tbl;
|
||||||
struct bucket_table *tbl;
|
struct bucket_table *tbl;
|
||||||
struct rhash_lock_head __rcu **bkt;
|
struct rhash_lock_head **bkt;
|
||||||
unsigned int hash;
|
unsigned int hash;
|
||||||
void *data;
|
void *data;
|
||||||
|
|
||||||
|
@ -1166,8 +1166,8 @@ void rhashtable_destroy(struct rhashtable *ht)
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL_GPL(rhashtable_destroy);
|
EXPORT_SYMBOL_GPL(rhashtable_destroy);
|
||||||
|
|
||||||
struct rhash_lock_head __rcu **__rht_bucket_nested(const struct bucket_table *tbl,
|
struct rhash_lock_head **__rht_bucket_nested(const struct bucket_table *tbl,
|
||||||
unsigned int hash)
|
unsigned int hash)
|
||||||
{
|
{
|
||||||
const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *));
|
const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *));
|
||||||
unsigned int index = hash & ((1 << tbl->nest) - 1);
|
unsigned int index = hash & ((1 << tbl->nest) - 1);
|
||||||
|
@ -1195,10 +1195,10 @@ struct rhash_lock_head __rcu **__rht_bucket_nested(const struct bucket_table *tb
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL_GPL(__rht_bucket_nested);
|
EXPORT_SYMBOL_GPL(__rht_bucket_nested);
|
||||||
|
|
||||||
struct rhash_lock_head __rcu **rht_bucket_nested(const struct bucket_table *tbl,
|
struct rhash_lock_head **rht_bucket_nested(const struct bucket_table *tbl,
|
||||||
unsigned int hash)
|
unsigned int hash)
|
||||||
{
|
{
|
||||||
static struct rhash_lock_head __rcu *rhnull;
|
static struct rhash_lock_head *rhnull;
|
||||||
|
|
||||||
if (!rhnull)
|
if (!rhnull)
|
||||||
INIT_RHT_NULLS_HEAD(rhnull);
|
INIT_RHT_NULLS_HEAD(rhnull);
|
||||||
|
@ -1206,9 +1206,9 @@ struct rhash_lock_head __rcu **rht_bucket_nested(const struct bucket_table *tbl,
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL_GPL(rht_bucket_nested);
|
EXPORT_SYMBOL_GPL(rht_bucket_nested);
|
||||||
|
|
||||||
struct rhash_lock_head __rcu **rht_bucket_nested_insert(struct rhashtable *ht,
|
struct rhash_lock_head **rht_bucket_nested_insert(struct rhashtable *ht,
|
||||||
struct bucket_table *tbl,
|
struct bucket_table *tbl,
|
||||||
unsigned int hash)
|
unsigned int hash)
|
||||||
{
|
{
|
||||||
const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *));
|
const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *));
|
||||||
unsigned int index = hash & ((1 << tbl->nest) - 1);
|
unsigned int index = hash & ((1 << tbl->nest) - 1);
|
||||||
|
|
Loading…
Reference in New Issue