|
|
@ -19,14 +19,14 @@ |
|
|
|
#include <net/flow_dissector.h> |
|
|
|
#include <net/flow_dissector.h> |
|
|
|
#include <scsi/fc/fc_fcoe.h> |
|
|
|
#include <scsi/fc/fc_fcoe.h> |
|
|
|
|
|
|
|
|
|
|
|
static bool skb_flow_dissector_uses_key(struct flow_dissector *flow_dissector, |
|
|
|
static bool dissector_uses_key(const struct flow_dissector *flow_dissector, |
|
|
|
enum flow_dissector_key_id key_id) |
|
|
|
enum flow_dissector_key_id key_id) |
|
|
|
{ |
|
|
|
{ |
|
|
|
return flow_dissector->used_keys & (1 << key_id); |
|
|
|
return flow_dissector->used_keys & (1 << key_id); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
static void skb_flow_dissector_set_key(struct flow_dissector *flow_dissector, |
|
|
|
static void dissector_set_key(struct flow_dissector *flow_dissector, |
|
|
|
enum flow_dissector_key_id key_id) |
|
|
|
enum flow_dissector_key_id key_id) |
|
|
|
{ |
|
|
|
{ |
|
|
|
flow_dissector->used_keys |= (1 << key_id); |
|
|
|
flow_dissector->used_keys |= (1 << key_id); |
|
|
|
} |
|
|
|
} |
|
|
@ -51,20 +51,20 @@ void skb_flow_dissector_init(struct flow_dissector *flow_dissector, |
|
|
|
* boundaries of unsigned short. |
|
|
|
* boundaries of unsigned short. |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
BUG_ON(key->offset > USHRT_MAX); |
|
|
|
BUG_ON(key->offset > USHRT_MAX); |
|
|
|
BUG_ON(skb_flow_dissector_uses_key(flow_dissector, |
|
|
|
BUG_ON(dissector_uses_key(flow_dissector, |
|
|
|
key->key_id)); |
|
|
|
key->key_id)); |
|
|
|
|
|
|
|
|
|
|
|
skb_flow_dissector_set_key(flow_dissector, key->key_id); |
|
|
|
dissector_set_key(flow_dissector, key->key_id); |
|
|
|
flow_dissector->offset[key->key_id] = key->offset; |
|
|
|
flow_dissector->offset[key->key_id] = key->offset; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
/* Ensure that the dissector always includes control and basic key.
|
|
|
|
/* Ensure that the dissector always includes control and basic key.
|
|
|
|
* That way we are able to avoid handling lack of these in fast path. |
|
|
|
* That way we are able to avoid handling lack of these in fast path. |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
BUG_ON(!skb_flow_dissector_uses_key(flow_dissector, |
|
|
|
BUG_ON(!dissector_uses_key(flow_dissector, |
|
|
|
FLOW_DISSECTOR_KEY_CONTROL)); |
|
|
|
FLOW_DISSECTOR_KEY_CONTROL)); |
|
|
|
BUG_ON(!skb_flow_dissector_uses_key(flow_dissector, |
|
|
|
BUG_ON(!dissector_uses_key(flow_dissector, |
|
|
|
FLOW_DISSECTOR_KEY_BASIC)); |
|
|
|
FLOW_DISSECTOR_KEY_BASIC)); |
|
|
|
} |
|
|
|
} |
|
|
|
EXPORT_SYMBOL(skb_flow_dissector_init); |
|
|
|
EXPORT_SYMBOL(skb_flow_dissector_init); |
|
|
|
|
|
|
|
|
|
|
@ -154,8 +154,8 @@ bool __skb_flow_dissect(const struct sk_buff *skb, |
|
|
|
FLOW_DISSECTOR_KEY_BASIC, |
|
|
|
FLOW_DISSECTOR_KEY_BASIC, |
|
|
|
target_container); |
|
|
|
target_container); |
|
|
|
|
|
|
|
|
|
|
|
if (skb_flow_dissector_uses_key(flow_dissector, |
|
|
|
if (dissector_uses_key(flow_dissector, |
|
|
|
FLOW_DISSECTOR_KEY_ETH_ADDRS)) { |
|
|
|
FLOW_DISSECTOR_KEY_ETH_ADDRS)) { |
|
|
|
struct ethhdr *eth = eth_hdr(skb); |
|
|
|
struct ethhdr *eth = eth_hdr(skb); |
|
|
|
struct flow_dissector_key_eth_addrs *key_eth_addrs; |
|
|
|
struct flow_dissector_key_eth_addrs *key_eth_addrs; |
|
|
|
|
|
|
|
|
|
|
@ -178,8 +178,8 @@ ip: |
|
|
|
|
|
|
|
|
|
|
|
ip_proto = iph->protocol; |
|
|
|
ip_proto = iph->protocol; |
|
|
|
|
|
|
|
|
|
|
|
if (!skb_flow_dissector_uses_key(flow_dissector, |
|
|
|
if (!dissector_uses_key(flow_dissector, |
|
|
|
FLOW_DISSECTOR_KEY_IPV4_ADDRS)) |
|
|
|
FLOW_DISSECTOR_KEY_IPV4_ADDRS)) |
|
|
|
break; |
|
|
|
break; |
|
|
|
|
|
|
|
|
|
|
|
key_addrs = skb_flow_dissector_target(flow_dissector, |
|
|
|
key_addrs = skb_flow_dissector_target(flow_dissector, |
|
|
@ -218,8 +218,8 @@ ipv6: |
|
|
|
ip_proto = iph->nexthdr; |
|
|
|
ip_proto = iph->nexthdr; |
|
|
|
nhoff += sizeof(struct ipv6hdr); |
|
|
|
nhoff += sizeof(struct ipv6hdr); |
|
|
|
|
|
|
|
|
|
|
|
if (skb_flow_dissector_uses_key(flow_dissector, |
|
|
|
if (dissector_uses_key(flow_dissector, |
|
|
|
FLOW_DISSECTOR_KEY_IPV6_ADDRS)) { |
|
|
|
FLOW_DISSECTOR_KEY_IPV6_ADDRS)) { |
|
|
|
struct flow_dissector_key_ipv6_addrs *key_ipv6_addrs; |
|
|
|
struct flow_dissector_key_ipv6_addrs *key_ipv6_addrs; |
|
|
|
|
|
|
|
|
|
|
|
key_ipv6_addrs = skb_flow_dissector_target(flow_dissector, |
|
|
|
key_ipv6_addrs = skb_flow_dissector_target(flow_dissector, |
|
|
@ -232,8 +232,8 @@ ipv6: |
|
|
|
|
|
|
|
|
|
|
|
flow_label = ip6_flowlabel(iph); |
|
|
|
flow_label = ip6_flowlabel(iph); |
|
|
|
if (flow_label) { |
|
|
|
if (flow_label) { |
|
|
|
if (skb_flow_dissector_uses_key(flow_dissector, |
|
|
|
if (dissector_uses_key(flow_dissector, |
|
|
|
FLOW_DISSECTOR_KEY_FLOW_LABEL)) { |
|
|
|
FLOW_DISSECTOR_KEY_FLOW_LABEL)) { |
|
|
|
key_tags = skb_flow_dissector_target(flow_dissector, |
|
|
|
key_tags = skb_flow_dissector_target(flow_dissector, |
|
|
|
FLOW_DISSECTOR_KEY_FLOW_LABEL, |
|
|
|
FLOW_DISSECTOR_KEY_FLOW_LABEL, |
|
|
|
target_container); |
|
|
|
target_container); |
|
|
@ -257,8 +257,8 @@ ipv6: |
|
|
|
if (!vlan) |
|
|
|
if (!vlan) |
|
|
|
goto out_bad; |
|
|
|
goto out_bad; |
|
|
|
|
|
|
|
|
|
|
|
if (skb_flow_dissector_uses_key(flow_dissector, |
|
|
|
if (dissector_uses_key(flow_dissector, |
|
|
|
FLOW_DISSECTOR_KEY_VLANID)) { |
|
|
|
FLOW_DISSECTOR_KEY_VLANID)) { |
|
|
|
key_tags = skb_flow_dissector_target(flow_dissector, |
|
|
|
key_tags = skb_flow_dissector_target(flow_dissector, |
|
|
|
FLOW_DISSECTOR_KEY_VLANID, |
|
|
|
FLOW_DISSECTOR_KEY_VLANID, |
|
|
|
target_container); |
|
|
|
target_container); |
|
|
@ -298,8 +298,8 @@ ipv6: |
|
|
|
if (!hdr) |
|
|
|
if (!hdr) |
|
|
|
goto out_bad; |
|
|
|
goto out_bad; |
|
|
|
|
|
|
|
|
|
|
|
if (skb_flow_dissector_uses_key(flow_dissector, |
|
|
|
if (dissector_uses_key(flow_dissector, |
|
|
|
FLOW_DISSECTOR_KEY_TIPC_ADDRS)) { |
|
|
|
FLOW_DISSECTOR_KEY_TIPC_ADDRS)) { |
|
|
|
key_addrs = skb_flow_dissector_target(flow_dissector, |
|
|
|
key_addrs = skb_flow_dissector_target(flow_dissector, |
|
|
|
FLOW_DISSECTOR_KEY_TIPC_ADDRS, |
|
|
|
FLOW_DISSECTOR_KEY_TIPC_ADDRS, |
|
|
|
target_container); |
|
|
|
target_container); |
|
|
@ -320,8 +320,8 @@ mpls: |
|
|
|
|
|
|
|
|
|
|
|
if ((ntohl(hdr[0].entry) & MPLS_LS_LABEL_MASK) >> |
|
|
|
if ((ntohl(hdr[0].entry) & MPLS_LS_LABEL_MASK) >> |
|
|
|
MPLS_LS_LABEL_SHIFT == MPLS_LABEL_ENTROPY) { |
|
|
|
MPLS_LS_LABEL_SHIFT == MPLS_LABEL_ENTROPY) { |
|
|
|
if (skb_flow_dissector_uses_key(flow_dissector, |
|
|
|
if (dissector_uses_key(flow_dissector, |
|
|
|
FLOW_DISSECTOR_KEY_MPLS_ENTROPY)) { |
|
|
|
FLOW_DISSECTOR_KEY_MPLS_ENTROPY)) { |
|
|
|
key_keyid = skb_flow_dissector_target(flow_dissector, |
|
|
|
key_keyid = skb_flow_dissector_target(flow_dissector, |
|
|
|
FLOW_DISSECTOR_KEY_MPLS_ENTROPY, |
|
|
|
FLOW_DISSECTOR_KEY_MPLS_ENTROPY, |
|
|
|
target_container); |
|
|
|
target_container); |
|
|
@ -374,8 +374,8 @@ ip_proto_again: |
|
|
|
if (!keyid) |
|
|
|
if (!keyid) |
|
|
|
goto out_bad; |
|
|
|
goto out_bad; |
|
|
|
|
|
|
|
|
|
|
|
if (skb_flow_dissector_uses_key(flow_dissector, |
|
|
|
if (dissector_uses_key(flow_dissector, |
|
|
|
FLOW_DISSECTOR_KEY_GRE_KEYID)) { |
|
|
|
FLOW_DISSECTOR_KEY_GRE_KEYID)) { |
|
|
|
key_keyid = skb_flow_dissector_target(flow_dissector, |
|
|
|
key_keyid = skb_flow_dissector_target(flow_dissector, |
|
|
|
FLOW_DISSECTOR_KEY_GRE_KEYID, |
|
|
|
FLOW_DISSECTOR_KEY_GRE_KEYID, |
|
|
|
target_container); |
|
|
|
target_container); |
|
|
@ -470,8 +470,8 @@ ip_proto_again: |
|
|
|
break; |
|
|
|
break; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
if (skb_flow_dissector_uses_key(flow_dissector, |
|
|
|
if (dissector_uses_key(flow_dissector, |
|
|
|
FLOW_DISSECTOR_KEY_PORTS)) { |
|
|
|
FLOW_DISSECTOR_KEY_PORTS)) { |
|
|
|
key_ports = skb_flow_dissector_target(flow_dissector, |
|
|
|
key_ports = skb_flow_dissector_target(flow_dissector, |
|
|
|
FLOW_DISSECTOR_KEY_PORTS, |
|
|
|
FLOW_DISSECTOR_KEY_PORTS, |
|
|
|
target_container); |
|
|
|
target_container); |
|
|
@ -497,18 +497,21 @@ static __always_inline void __flow_hash_secret_init(void) |
|
|
|
net_get_random_once(&hashrnd, sizeof(hashrnd)); |
|
|
|
net_get_random_once(&hashrnd, sizeof(hashrnd)); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
static __always_inline u32 __flow_hash_words(u32 *words, u32 length, u32 keyval) |
|
|
|
static __always_inline u32 __flow_hash_words(const u32 *words, u32 length, |
|
|
|
|
|
|
|
u32 keyval) |
|
|
|
{ |
|
|
|
{ |
|
|
|
return jhash2(words, length, keyval); |
|
|
|
return jhash2(words, length, keyval); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
static inline void *flow_keys_hash_start(struct flow_keys *flow) |
|
|
|
static inline const u32 *flow_keys_hash_start(const struct flow_keys *flow) |
|
|
|
{ |
|
|
|
{ |
|
|
|
|
|
|
|
const void *p = flow; |
|
|
|
|
|
|
|
|
|
|
|
BUILD_BUG_ON(FLOW_KEYS_HASH_OFFSET % sizeof(u32)); |
|
|
|
BUILD_BUG_ON(FLOW_KEYS_HASH_OFFSET % sizeof(u32)); |
|
|
|
return (void *)flow + FLOW_KEYS_HASH_OFFSET; |
|
|
|
return (const u32 *)(p + FLOW_KEYS_HASH_OFFSET); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
static inline size_t flow_keys_hash_length(struct flow_keys *flow) |
|
|
|
static inline size_t flow_keys_hash_length(const struct flow_keys *flow) |
|
|
|
{ |
|
|
|
{ |
|
|
|
size_t diff = FLOW_KEYS_HASH_OFFSET + sizeof(flow->addrs); |
|
|
|
size_t diff = FLOW_KEYS_HASH_OFFSET + sizeof(flow->addrs); |
|
|
|
BUILD_BUG_ON((sizeof(*flow) - FLOW_KEYS_HASH_OFFSET) % sizeof(u32)); |
|
|
|
BUILD_BUG_ON((sizeof(*flow) - FLOW_KEYS_HASH_OFFSET) % sizeof(u32)); |
|
|
@ -598,7 +601,7 @@ static inline u32 __flow_hash_from_keys(struct flow_keys *keys, u32 keyval) |
|
|
|
|
|
|
|
|
|
|
|
__flow_hash_consistentify(keys); |
|
|
|
__flow_hash_consistentify(keys); |
|
|
|
|
|
|
|
|
|
|
|
hash = __flow_hash_words((u32 *)flow_keys_hash_start(keys), |
|
|
|
hash = __flow_hash_words(flow_keys_hash_start(keys), |
|
|
|
flow_keys_hash_length(keys), keyval); |
|
|
|
flow_keys_hash_length(keys), keyval); |
|
|
|
if (!hash) |
|
|
|
if (!hash) |
|
|
|
hash = 1; |
|
|
|
hash = 1; |
|
|
@ -677,7 +680,7 @@ __u32 skb_get_hash_perturb(const struct sk_buff *skb, u32 perturb) |
|
|
|
} |
|
|
|
} |
|
|
|
EXPORT_SYMBOL(skb_get_hash_perturb); |
|
|
|
EXPORT_SYMBOL(skb_get_hash_perturb); |
|
|
|
|
|
|
|
|
|
|
|
__u32 __skb_get_hash_flowi6(struct sk_buff *skb, struct flowi6 *fl6) |
|
|
|
__u32 __skb_get_hash_flowi6(struct sk_buff *skb, const struct flowi6 *fl6) |
|
|
|
{ |
|
|
|
{ |
|
|
|
struct flow_keys keys; |
|
|
|
struct flow_keys keys; |
|
|
|
|
|
|
|
|
|
|
@ -701,7 +704,7 @@ __u32 __skb_get_hash_flowi6(struct sk_buff *skb, struct flowi6 *fl6) |
|
|
|
} |
|
|
|
} |
|
|
|
EXPORT_SYMBOL(__skb_get_hash_flowi6); |
|
|
|
EXPORT_SYMBOL(__skb_get_hash_flowi6); |
|
|
|
|
|
|
|
|
|
|
|
__u32 __skb_get_hash_flowi4(struct sk_buff *skb, struct flowi4 *fl4) |
|
|
|
__u32 __skb_get_hash_flowi4(struct sk_buff *skb, const struct flowi4 *fl4) |
|
|
|
{ |
|
|
|
{ |
|
|
|
struct flow_keys keys; |
|
|
|
struct flow_keys keys; |
|
|
|
|
|
|
|
|
|
|
@ -787,7 +790,7 @@ u32 skb_get_poff(const struct sk_buff *skb) |
|
|
|
return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb)); |
|
|
|
return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb)); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
__u32 __get_hash_from_flowi6(struct flowi6 *fl6, struct flow_keys *keys) |
|
|
|
__u32 __get_hash_from_flowi6(const struct flowi6 *fl6, struct flow_keys *keys) |
|
|
|
{ |
|
|
|
{ |
|
|
|
memset(keys, 0, sizeof(*keys)); |
|
|
|
memset(keys, 0, sizeof(*keys)); |
|
|
|
|
|
|
|
|
|
|
@ -806,7 +809,7 @@ __u32 __get_hash_from_flowi6(struct flowi6 *fl6, struct flow_keys *keys) |
|
|
|
} |
|
|
|
} |
|
|
|
EXPORT_SYMBOL(__get_hash_from_flowi6); |
|
|
|
EXPORT_SYMBOL(__get_hash_from_flowi6); |
|
|
|
|
|
|
|
|
|
|
|
__u32 __get_hash_from_flowi4(struct flowi4 *fl4, struct flow_keys *keys) |
|
|
|
__u32 __get_hash_from_flowi4(const struct flowi4 *fl4, struct flow_keys *keys) |
|
|
|
{ |
|
|
|
{ |
|
|
|
memset(keys, 0, sizeof(*keys)); |
|
|
|
memset(keys, 0, sizeof(*keys)); |
|
|
|
|
|
|
|
|
|
|
|