mptcp/openmptcprouter-feeds
1
0
Fork 0
mirror of https://github.com/Ysurac/openmptcprouter-feeds.git synced 2025-03-09 15:40:03 +00:00
Code Issues Releases Wiki Activity

fix short

Browse source
This commit is contained in:
suyuan 2022-09-13 21:43:54 +08:00
parent e9f4104f88
commit 848c1b9cc2
64 changed files with 11944 additions and 16640 deletions
Download patch file Download diff file Expand all files Collapse all files

5
shortcut-fe/.gitignore vendored
View file

@ -1,5 +0,0 @@
# Ouptut files
*.o
*.s

42
shortcut-fe/Makefile
View file

@ -1,42 +0,0 @@
#
# Makefile for Shortcut FE.
#
KERNELVERSION := $(word 1, $(subst ., ,$(KERNELVERSION))).$(word 2, $(subst ., ,$(KERNELVERSION)))
SFE_BASE_OBJS := sfe.o sfe_init.o
SFE_IPV4_OBJS := sfe_ipv4.o sfe_ipv4_udp.o sfe_ipv4_tcp.o sfe_ipv4_icmp.o sfe_ipv4_tun6rd.o sfe_ipv4_pppoe_br.o sfe_ipv4_esp.o
SFE_IPV6_OBJS := sfe_ipv6.o sfe_ipv6_udp.o sfe_ipv6_tcp.o sfe_ipv6_icmp.o sfe_ipv6_tunipip6.o sfe_ipv6_pppoe_br.o sfe_ipv6_esp.o
SFE_PPPOE_OBJS := sfe_pppoe.o sfe_pppoe_mgr.o
ifeq ($(findstring 4.4, $(KERNELVERSION)),)
SFE_IPV4_OBJS += sfe_ipv4_gre.o
SFE_IPV6_OBJS += sfe_ipv6_gre.o
ccflags-y += -DSFE_GRE_TUN_ENABLE
endif
obj-m += qca-nss-sfe.o
#
# Base files
#
qca-nss-sfe-objs := $(SFE_BASE_OBJS)
#
# IPv4 files
#
qca-nss-sfe-objs += $(SFE_IPV4_OBJS)
qca-nss-sfe-objs += $(SFE_PPPOE_OBJS)
ifdef SFE_SUPPORT_IPV6
qca-nss-sfe-objs += $(SFE_IPV6_OBJS)
ccflags-y += -DSFE_SUPPORT_IPV6
endif
ifdef SFE_PROCESS_LOCAL_OUT
ccflags-y += -DSFE_PROCESS_LOCAL_OUT
endif
ccflags-y += -Werror -Wall -Iexports/

122
shortcut-fe/README
View file

@ -1,122 +0,0 @@
Shortcut Forwarding Engine
--------------------------
Welcome to "Shortcut" :-)
Here's a quick FAQ:
Q) What is Shortcut?
A) Shortcut is an in-Linux-kernel IP packet forwarding engine. It's designed
to offer very high speed IP packet forwarding based on IP connection tracking.
It's dramatically faster than the standard netfilter-based NAT forwarding path
but is designed to synchronise state back to netfilter/conntrack so that it
doesn't need to deal with all of the complexities of special cases.
Q) What versions of IP does it support?
A) The current version only supports IPv4 but will be extended to support IPv6 in
the future.
Q) What transport protocols does it support?
A) TCP and UDP. It also knows enough about ICMP to spot ICMP error messages
related to TCP and UDP and handle things accordingly.
Q) Is there a design spec for this software?
A) Not at the moment. I'll write one when I get more time. The code is
intended to be a good tutorial though - it's very heavily commented. If you
find yourself reading something and not understanding it then I take that to
mean I've probably not done a sufficently good job of explaining what it's
doing in the comments. Let me know - I will try to fix it :-)
Q) Why was it written?
A) It was written as a demonstration of what can be done to provide high
performance forwarding inside the kernel. There were two initial motivations:
1) To provide a platform to enable research into how QoS analysis systems can
offload work and avoid huge Linux overheads.
2) To provide a tool to investigate the behaviour of various processors, SoCs
and software sets so that we can characterize and design new network processor
SoCs.
Q) How much faster is it than the Linux kernel forwarding path?
A) At the time of pushing this to github it's been tested on a QCA AP135.
This has a Scorpion (QCA Scopion, not the QMC one :-)) SoC, QCA9550. The
SoC's processor is a MIPS74K running at 720 MHz and with a DDR2 memory
subsystem that offers a peak of 600 MT/s (16-bit transfers).
Running IPv4 NAT forwarding of UDP between the board's 2 GMAC ports and
using a SmartBits 200 as a traffic generator Linux is able to forward 70k PPS.
Once the SFE code is invoked this will increase to 350k PPS!
There's also a slightly hacky mode which causes SFE to bypass the Linux
bridge layer, but this isn't really ready for use because it doesn't have
sufficient MAC address checks or integration of statistics back to the
Ethernet bridge, but that runs at 436k PPS.
Q) Are there any diagnostics?
A) Yes, this is a research tool after all! There's a complex way to do this
that's more general purpose and a simple one - here's the simple one:
mknod /dev/sfe c 253 0
The file /dev/sfe is an XML-ish output and provides details of all the
network connections currently being offloaded. It also reports the numbers
of packets that took various "exception" paths within the code. In addition
it provides a summary of the number of connections, attempts to accelerate
connections, cancel accelerations, etc. It also reports the numbers of
packets that were forwarded and not forwarded by the engine and has some
stats on the effectiveness of the hashing algorithm it uses.
Q) How does the code interact with Linux?
A) There are four minor patches required to make this software run with
Linux. These are currently against a 3.3.8 or 3.4.0 kernel:
* (net/core/dev.c) adds a hook to allow packets to be extracted out.
* (net/netfilter/nf_conntrack_proto_tcp.c) exposes a state variable inside
netfilter that's necessary to enable TCP sequence and ACK checking within
the offload path. Note that this specific patch is against the QCA QSDK
patched version of 3.3.8 - there's a slightly braindead "performance"
patch in that kernel, courtesy of the OpenWrt community that makes the
Linux forwarding path slightly faster at the expense of losing
functionality :-(
* (net/Kconfig) adds the shortcut-fe option.
* (net/Makefile) adds the shortcut-fe build support.
Once these are applied and the module is loaded then everything else
is automatic :-) The patches are in this git repo.
Q) Are any of the pieces reused from other projects?
A) Yes! Some of the forwarding concepts are reused from the Ubicom Network
Accelerator that morphed into part of the Akronite NSS. This code has all
been substantially changed though to accomodate Linux's needs.
There are also some pieces that I borrowed from the QCA "FastNAT" software
written by Xiaoping Fan <xfan@qca.qualcomm.com>. Xiaoping's code was the
first actual demonstration within QCA that this in-kernel concept could yield
signficant performance gains.
Enjoy!
Dave Hudson <dhudson@qti.qualcomm.com>

832
shortcut-fe/exports/sfe_api.h
View file

@ -1,832 +0,0 @@
/*
* sfe_api.h
* SFE exported function headers for SFE engine.
*
* Copyright (c) 2015,2016, The Linux Foundation. All rights reserved.
* Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
/**
* @file sfe_api.h
* SFE exported function headers for the SFE engine.
*/
#ifndef __SFE_API_H
#define __SFE_API_H
/**
* @addtogroup nss_sfe_definitions
* @{
*/
#define SFE_MAX_VLAN_DEPTH 2 /**< Maximum VLAN depth. */
#define SFE_VLAN_ID_NOT_CONFIGURED 0xfff /**< VLAN ID not configured. */
#define SFE_INVALID_VLAN_PCP 0xff /**< VLAN PCP remark is invalid for SAWF (Service Aware Wi-Fi). */
#define SFE_MAX_SERVICE_CLASS_ID 0x80 /**< Maximum service class ID. */
#define SFE_INVALID_SERVICE_CLASS_ID 0xff /**< Service class ID not valid. */
#define SFE_SERVICE_CLASS_STATS_MAX_RETRY 100 /**< Maximum retries for fetching service class statistics. */
#define SFE_INVALID_MSDUQ 0xff /**< Invalid MAC Service Data Unit Queue. */
#define SFE_SPECIAL_INTERFACE_BASE 0x7f00 /**< Special interface base number. */
#define SFE_SPECIAL_INTERFACE_IPV4 (SFE_SPECIAL_INTERFACE_BASE + 1) /**< Interface number for IPv4. */
#define SFE_SPECIAL_INTERFACE_IPV6 (SFE_SPECIAL_INTERFACE_BASE + 2) /**< Interface enumber fo IPv6. */
#define SFE_SPECIAL_INTERFACE_IPSEC (SFE_SPECIAL_INTERFACE_BASE + 3) /**< Interface number for IPSec. */
#define SFE_SPECIAL_INTERFACE_L2TP (SFE_SPECIAL_INTERFACE_BASE + 4) /**< Interface number for L2TP. */
#define SFE_SPECIAL_INTERFACE_PPTP (SFE_SPECIAL_INTERFACE_BASE + 5) /**< Interface number for PPTP. */
/**
* @}
*/
/**
* @addtogroup nss_sfe_flags
* @{
*/
/*
* Rule creation and rule update flags.
*/
#define SFE_RULE_CREATE_FLAG_NO_SEQ_CHECK (1<<0) /**< Do not perform TCP sequence number checks. */
#define SFE_RULE_CREATE_FLAG_BRIDGE_FLOW (1<<1) /**< Rule is for a pure bridge forwarding flow. */
#define SFE_RULE_CREATE_FLAG_ROUTED (1<<2) /**< Rule is for a routed connection. */
#define SFE_RULE_CREATE_FLAG_DSCP_MARKING (1<<3) /**< Rule has DSCP marking configured. */
#define SFE_RULE_CREATE_FLAG_VLAN_MARKING (1<<4) /**< Rule has VLAN marking configured. */
#define SFE_RULE_UPDATE_FLAG_CHANGE_MTU (1<<5) /**< Update MTU of connection interfaces. */
#define SFE_RULE_CREATE_FLAG_ICMP_NO_CME_FLUSH (1<<6) /**< Rule to not flush CME on ICMP packets. */
#define SFE_RULE_CREATE_FLAG_L2_ENCAP (1<<7) /**< Consists of an encapsulating protocol that carries an IPv4 payload within it. */
#define SFE_RULE_CREATE_FLAG_USE_FLOW_BOTTOM_INTERFACE (1<<8) /**< Use flow interface number instead of top interface. */
#define SFE_RULE_CREATE_FLAG_USE_RETURN_BOTTOM_INTERFACE (1<<9) /**< Use return interface number instead of top interface. */
#define SFE_RULE_CREATE_FLAG_FLOW_SRC_INTERFACE_CHECK (1<<10) /**< Check source interface on the flow direction. */
#define SFE_RULE_CREATE_FLAG_RETURN_SRC_INTERFACE_CHECK (1<<11)
/**< Check source interface on the return direction. */
#define SFE_RULE_CREATE_FLAG_FLOW_TRANSMIT_FAST (1<<12) /**< Original flow can be transmitted fast. */
#define SFE_RULE_CREATE_FLAG_RETURN_TRANSMIT_FAST (1<<13) /**< Return flow can be transmitted fast. */
#define SFE_RULE_CREATE_FLAG_FLOW_SRC_INTERFACE_CHECK_NO_FLUSH (1<<14)
/**< Check source interface on the flow direction but do not flush the connection. */
#define SFE_RULE_CREATE_FLAG_RETURN_SRC_INTERFACE_CHECK_NO_FLUSH (1<<15)
/**< Check source interface on the return direction but do not flush the connection. */
/*
* Rule creation validity flags.
*/
#define SFE_RULE_CREATE_CONN_VALID (1<<0) /**< IPv4 connection is valid. */
#define SFE_RULE_CREATE_TCP_VALID (1<<1) /**< TCP protocol fields are valid. */
#define SFE_RULE_CREATE_PPPOE_DECAP_VALID (1<<2) /**< PPPoE decapsulation fields are valid. */
#define SFE_RULE_CREATE_PPPOE_ENCAP_VALID (1<<3) /**< PPPoE encapsulation fields are valid. */
#define SFE_RULE_CREATE_QOS_VALID (1<<4) /**< QoS fields are valid. */
#define SFE_RULE_CREATE_VLAN_VALID (1<<5) /**< VLAN fields are valid. */
#define SFE_RULE_CREATE_DSCP_MARKING_VALID (1<<6) /**< DSCP marking fields are valid. */
#define SFE_RULE_CREATE_VLAN_MARKING_VALID (1<<7) /**< VLAN marking fields are valid. */
#define SFE_RULE_CREATE_DIRECTION_VALID (1<<8) /**< Acceleration direction is valid. */
#define SFE_RULE_CREATE_SRC_MAC_VALID (1<<9) /**< Source MAC address is valid. */
#define SFE_RULE_CREATE_MARK_VALID (1<<10) /**< SKB marking fields are valid. */
/*
* Source MAC address validity flags; used with the mac_valid_flags field in the sfe_ipv4_src_mac_rule structure.
*/
#define SFE_SRC_MAC_FLOW_VALID 0x01
/**< MAC address for the flow interface is valid. */
#define SFE_SRC_MAC_RETURN_VALID 0x02
/**< MAC address for the return interface is valid. */
/**
* @}
*/
/**
* @addtogroup nss_sfe_datatypes
* @{
*/
/*
* 32/64-bit pointer types.
*/
#ifdef __LP64__
typedef uint64_t sfe_ptr_t; /**< 64-bit pointer. */
#else
typedef uint32_t sfe_ptr_t; /**< 32-bit pointer. */
#endif
/**
* Synchronize reason enum.
*/
typedef enum /** @cond */ sfe_rule_sync_reason /** @endcond */ {
SFE_RULE_SYNC_REASON_STATS, /**< Synchronize statistics. */
SFE_RULE_SYNC_REASON_FLUSH, /**< Synchronize to flush an entry. */
SFE_RULE_SYNC_REASON_EVICT, /**< Synchronize to evict an entry. */
SFE_RULE_SYNC_REASON_DESTROY /**< Synchronize to destroy an entry (requested by the connection manager). */
} sfe_rule_sync_reason_t;
/**
* Tx command status.
*/
typedef enum {
SFE_TX_SUCCESS = 0, /**< Success. */
SFE_TX_FAILURE, /**< Failure other than descriptor not available. */
SFE_TX_FAILURE_QUEUE, /**< failure due to descriptor not available. */
SFE_TX_FAILURE_NOT_READY, /**< Failure due to SFE state uninitialized. */
SFE_TX_FAILURE_TOO_LARGE, /**< Command is too large to fit in one message. */
SFE_TX_FAILURE_TOO_SHORT, /**< Command or packet is shorter than expected. */
SFE_TX_FAILURE_NOT_SUPPORTED, /**< Command or packet not accepted for forwarding. */
SFE_TX_FAILURE_BAD_PARAM, /**< Failure due to bad parameters. */
SFE_TX_FAILURE_NOT_ENABLED, /**< Failure due to SFE not enabled. */
} sfe_tx_status_t;
/**
* Common response types.
*/
enum sfe_cmn_response {
SFE_CMN_RESPONSE_ACK, /**< Message acknowledged. */
SFE_CMN_RESPONSE_EVERSION, /**< Version error. */
SFE_CMN_RESPONSE_EINTERFACE, /**< Interface error. */
SFE_CMN_RESPONSE_ELENGTH, /**< Length error. */
SFE_CMN_RESPONSE_EMSG, /**< Message error. */
SFE_CMM_RESPONSE_NOTIFY, /**< Message independant of request. */
SFE_CMN_RESPONSE_LAST /**< Indicates the last item. */
};
/**
* IPv4 bridge/route rule messages.
*/
enum sfe_message_types {
SFE_TX_CREATE_RULE_MSG, /**< IPv4/IPv6 create rule message. */
SFE_TX_DESTROY_RULE_MSG, /**< IPv4/IPv6 destroy rule message. */
SFE_RX_CONN_STATS_SYNC_MSG, /**< IPv4/IPv6 connection statistics synchronize message. */
SFE_TX_CONN_STATS_SYNC_MANY_MSG,/**< IPv4/IPv6 connection statistics synchronize many message. */
SFE_TUN6RD_ADD_UPDATE_PEER, /**< Add/update peer for 6RD tunnel. */
SFE_MAX_MSG_TYPES, /**< IPv4/IPv6 message max type number. */
};
/**
* Connection mark structure.
*/
struct sfe_connection_mark {
int protocol; /**< Protocol number. */
__be32 src_ip[4]; /**< Source IP address. */
__be32 dest_ip[4]; /**< Destination IP address. */
__be16 src_port; /**< Source port number. */
__be16 dest_port; /**< Destination port number. */
u32 mark; /**< Mark value to be updated. */
};
/**
* Common message structure.
*/
struct sfe_cmn_msg {
u16 version; /**< Version ID for the main message format. */
u16 interface; /**< Primary key for all messages. */
enum sfe_cmn_response response; /**< Primary response. */
u32 type; /**< Decentralized request ID used to match response ID. */
u32 error; /**< Decentralized specific error message; response == EMSG. */
sfe_ptr_t cb; /**< Callback pointer. */
sfe_ptr_t app_data; /**< Application data. */
u32 len; /**< Length of the message excluding this header. */
};
/**
* Common 5-tuple structure.
*/
struct sfe_ipv4_5tuple {
__be32 flow_ip; /**< Flow IP address. */
__be32 return_ip; /**< Return IP address. */
__be16 flow_ident; /**< Flow identifier, e.g., TCP/UDP port. */
__be16 return_ident; /**< Return identifier, e.g., TCP/UDP port. */
u8 protocol; /**< Protocol number. */
u8 reserved[3]; /**< Reserved; padding for alignment. */
};
/**
* IPv4 connection rule structure.
*/
struct sfe_ipv4_connection_rule {
u8 flow_mac[6]; /**< Flow MAC address. */
u8 return_mac[6]; /**< Return MAC address. */
s32 flow_interface_num; /**< Flow interface number. */
s32 return_interface_num; /**< Return interface number. */
s32 flow_top_interface_num; /**< Top flow interface number. */
s32 return_top_interface_num; /**< Top return interface number. */
u32 flow_mtu; /**< Flow interface`s MTU. */
u32 return_mtu; /**< Return interface`s MTU. */
__be32 flow_ip_xlate; /**< Translated flow IP address. */
__be32 return_ip_xlate; /**< Translated return IP address. */
__be16 flow_ident_xlate; /**< Translated flow identifier, e.g., port. */
__be16 return_ident_xlate; /**< Translated return identifier, e.g., port. */
};
/**
* TCP connection rule structure.
*/
struct sfe_protocol_tcp_rule {
u32 flow_max_window; /**< Flow direction's largest seen window. */
u32 return_max_window; /**< Return direction's largest seen window. */
u32 flow_end; /**< Flow direction's largest seen sequence + segment length. */
u32 return_end; /**< Return direction's largest seen sequence + segment length. */
u32 flow_max_end; /**< Flow direction's largest seen ack + max(1, win). */
u32 return_max_end; /**< Return direction's largest seen ack + max(1, win). */
u8 flow_window_scale; /**< Flow direction's window scaling factor. */
u8 return_window_scale; /**< Return direction's window scaling factor. */
u16 reserved; /**< Reserved; padding for alignment. */
};
/**
* sfe_pppoe_br_accel_mode_t
* PPPoE bridge acceleration modes.
*/
typedef enum {
SFE_PPPOE_BR_ACCEL_MODE_DISABLED, /**< No acceleration */
SFE_PPPOE_BR_ACCEL_MODE_EN_5T, /**< 5-tuple (src_ip, dest_ip, src_port, dest_port, protocol) acceleration */
SFE_PPPOE_BR_ACCEL_MODE_EN_3T, /**< 3-tuple (src_ip, dest_ip, pppoe session id) acceleration */
SFE_PPPOE_BR_ACCEL_MODE_MAX /**< Indicates the last item */
} __attribute__ ((__packed__)) sfe_pppoe_br_accel_mode_t;
/**
* PPPoE connection rules structure.
*/
struct sfe_pppoe_rule {
u16 flow_pppoe_session_id; /**< Flow direction`s PPPoE session ID. */
u8 flow_pppoe_remote_mac[ETH_ALEN]; /**< Flow direction`s PPPoE server MAC address. */
u16 return_pppoe_session_id; /**< Return direction's PPPoE session ID. */
u8 return_pppoe_remote_mac[ETH_ALEN]; /**< Return direction's PPPoE server MAC address. */
};
/**
* Information for source MAC address rules.
*/
struct sfe_src_mac_rule {
uint32_t mac_valid_flags; /**< MAC address validity flags. */
uint16_t flow_src_mac[3]; /**< Source MAC address for the flow direction. */
uint16_t return_src_mac[3]; /**< Source MAC address for the return direction. */
};
/**
* QoS connection rule structure.
*/
struct sfe_qos_rule {
u32 flow_qos_tag; /**< QoS tag associated with this rule for flow direction. */
u32 return_qos_tag; /**< QoS tag associated with this rule for return direction. */
};
/**
* Mark rule structure.
*/
struct sfe_mark_rule {
u32 flow_mark; /**< SKB mark associated with this rule for flow direction. */
u32 return_mark; /**< SKB mark associated with this rule for return direction. */
};
/**
* DSCP connection rule structure.
*/
struct sfe_dscp_rule {
u8 flow_dscp; /**< Egress DSCP value for flow direction. */
u8 return_dscp; /**< Egress DSCP value for return direction. */
u8 reserved[2]; /**< Reserved; padding for alignment. */
};
/**
* VLAN connection rule structure.
*/
struct sfe_vlan_rule {
u32 ingress_vlan_tag; /**< VLAN tag for ingress packets. */
u32 egress_vlan_tag; /**< VLAN tag for egress packets. */
};
/**
* Acceleration direction rule structure.
* Sometimes it is useful to accelerate traffic in one direction and not in another.
*/
struct sfe_acceleration_direction_rule {
u8 flow_accel; /**< Accelerate in flow direction. */
u8 return_accel; /**< Accelerate in return direction. */
u8 reserved[2]; /**< Reserved; padding for alignment. */
};
/**
* Service class rule information in both directions.
*/
struct sfe_service_class_rule {
uint32_t flow_mark; /**< Service class information in flow direction. */
uint32_t return_mark; /**< Service class information in return direction. */
};
/**
* IPv4 rule create submessage structure.
*/
struct sfe_ipv4_rule_create_msg {
/* Request */
u16 valid_flags; /**< Bit flags associated with paramater validity. */
u16 rule_flags; /**< Bit flags associated with the rule. */
struct sfe_ipv4_5tuple tuple; /**< Holds values of 5-tuple. */
struct sfe_ipv4_connection_rule conn_rule; /**< Basic connection-specific data. */
struct sfe_protocol_tcp_rule tcp_rule; /**< TCP-related acceleration parameters. */
struct sfe_pppoe_rule pppoe_rule; /**< PPPoE-related acceleration parameters. */
struct sfe_qos_rule qos_rule; /**< QoS-related acceleration parameters. */
struct sfe_src_mac_rule src_mac_rule; /**< Source MAC address rule. */
struct sfe_mark_rule mark_rule; /**< SKB mark-related acceleration parameters. */
struct sfe_dscp_rule dscp_rule; /**< DSCP-related acceleration parameters. */
struct sfe_vlan_rule vlan_primary_rule; /**< Primary VLAN-related acceleration parameters. */
struct sfe_vlan_rule vlan_secondary_rule; /**< Secondary VLAN-related acceleration parameters. */
#ifdef CONFIG_XFRM
struct sfe_acceleration_direction_rule direction_rule;
/**< Direction related acceleration parameters. */
#endif
/* Response */
struct sfe_service_class_rule sawf_rule;
/**< Service class related information */
u32 index; /**< Slot ID for cache statistics to host OS. */
};
/**
* IPv4 rule destroy submessage structure.
*/
struct sfe_ipv4_rule_destroy_msg {
struct sfe_ipv4_5tuple tuple; /**< Holds values of 5-tuple. */
};
/**
* The SFE IPv4 rule sync structure.
*/
struct sfe_ipv4_conn_sync {
u32 index; /**< Slot ID for cache statistics to host OS. */
u8 protocol; /**< Protocol number. */
__be32 flow_ip; /**< Flow IP address. */
__be32 flow_ip_xlate; /**< Translated flow IP address. */
__be16 flow_ident; /**< Flow identifier, e.g., port. */
__be16 flow_ident_xlate; /**< Translated flow identifier, e.g., port. */
u32 flow_max_window; /**< Flow direction's largest seen window. */
u32 flow_end; /**< Flow direction's largest seen sequence + segment length. */
u32 flow_max_end; /**< Flow direction's largest seen ack + max(1, win). */
u32 flow_rx_packet_count; /**< Flow interface's Rx packet count. */
u32 flow_rx_byte_count; /**< Flow interface's Rx byte count. */
u32 flow_tx_packet_count; /**< Flow interface's Tx packet count. */
u32 flow_tx_byte_count; /**< Flow interface's Tx byte count. */
u16 flow_pppoe_session_id; /**< Flow interface`s PPPoE session ID. */
u16 flow_pppoe_remote_mac[3]; /**< Flow interface's PPPoE remote server MAC address (if present). */
__be32 return_ip; /**< Return IP address. */
__be32 return_ip_xlate; /**< Translated return IP address */
__be16 return_ident; /**< Return identifier, e.g., port. */
__be16 return_ident_xlate; /**< Translated return identifier, e.g., port. */
u32 return_max_window; /**< Return direction's largest seen window. */
u32 return_end; /**< Return direction's largest seen sequence + segment length. */
u32 return_max_end; /**< Return direction's largest seen ack + max(1, win). */
u32 return_rx_packet_count; /**< Return interface's Rx packet count. */
u32 return_rx_byte_count; /**< Return interface's Rx byte count. */
u32 return_tx_packet_count; /**< Return interface's Tx packet count. */
u32 return_tx_byte_count; /**< Return interface's Tx byte count. */
u16 return_pppoe_session_id; /**< Return interface`s PPPoE session ID. */
u16 return_pppoe_remote_mac[3]; /**< Return interface's PPPoE remote server MAC address (if present). */
u32 inc_ticks; /**< Number of ticks since the last sync. */
u32 reason; /**< Synchronization reason. */
u8 flags; /**< Bit flags associated with the rule. */
u32 qos_tag; /**< QoS tag. */
u32 cause; /**< Flush cause. */
};
/**
* Information for a multiple IPv4 connection statistics synchronization message.
*/
struct sfe_ipv4_conn_sync_many_msg {
/*
* Request
*/
uint16_t index; /**< Request connection statistics from the index. */
uint16_t size; /**< Buffer size of this message. */
/*
* Response
*/
uint16_t next; /**< Firmware response for the next connection to be requested. */
uint16_t count; /**< Number of synchronized connections included in this message. */
struct sfe_ipv4_conn_sync conn_sync[]; /**< Array for the statistics. */
};
/**
* Message structure to send/receive IPv4 bridge/route commands
*/
struct sfe_ipv4_msg {
struct sfe_cmn_msg cm; /**< Message header. */
union {
struct sfe_ipv4_rule_create_msg rule_create; /**< Rule create message. */
struct sfe_ipv4_rule_destroy_msg rule_destroy; /**< Rule destroy message. */
struct sfe_ipv4_conn_sync conn_stats; /**< Connection statistics synchronization message. */
struct sfe_ipv4_conn_sync_many_msg conn_stats_many;
/**< Many connections' statistics synchronization message. */
} msg; /**< IPv4 message. */
};
/**
* @}
*/
/**
* @addtogroup nss_sfe_functions
* @{
*/
/**
* IPv4 message received callback.
*/
typedef void (*sfe_ipv4_msg_callback_t)(void *app_data, struct sfe_ipv4_msg *msg);
/**
* @}
*/
/**
* @addtogroup nss_sfe_datatypes
* @{
*/
/**
* IPv6 5-tuple structure.
*/
struct sfe_ipv6_5tuple {
__be32 flow_ip[4]; /**< Flow IP address. */
__be32 return_ip[4]; /**< Return IP address. */
__be16 flow_ident; /**< Flow identifier, e.g.,TCP/UDP port. */
__be16 return_ident; /**< Return identifier, e.g., TCP/UDP port. */
u8 protocol; /**< Protocol number. */
u8 reserved[3]; /**< Reserved; padding for alignment. */
};
/**
* IPv6 connection rule structure.
*/
struct sfe_ipv6_connection_rule {
u8 flow_mac[6]; /**< Flow MAC address. */
u8 return_mac[6]; /**< Return MAC address. */
s32 flow_interface_num; /**< Flow interface number. */
s32 return_interface_num; /**< Return interface number. */
s32 flow_top_interface_num; /**< Top flow interface number. */
s32 return_top_interface_num; /**< Top return interface number. */
u32 flow_mtu; /**< Flow interface's MTU. */
u32 return_mtu; /**< Return interface's MTU. */
};
/**
* IPv6 rule create submessage structure.
*/
struct sfe_ipv6_rule_create_msg {
/*
* Request
*/
u16 valid_flags; /**< Bit flags associated with parameter validity. */
u16 rule_flags; /**< Bit flags associated with the rule. */
struct sfe_ipv6_5tuple tuple; /**< Holds values of the sfe_ipv6_5tuple tuple. */
struct sfe_ipv6_connection_rule conn_rule; /**< Basic connection-specific data. */
struct sfe_protocol_tcp_rule tcp_rule; /**< Protocol-related acceleration parameters. */
struct sfe_pppoe_rule pppoe_rule; /**< PPPoE-related acceleration parameters. */
struct sfe_qos_rule qos_rule; /**< QoS-related acceleration parameters. */
struct sfe_src_mac_rule src_mac_rule; /**< Source MAC address rule. */
struct sfe_mark_rule mark_rule; /**< SKB mark-related acceleration parameters. */
struct sfe_dscp_rule dscp_rule; /**< DSCP-related acceleration parameters. */
struct sfe_vlan_rule vlan_primary_rule; /**< VLAN-related acceleration parameters. */
struct sfe_vlan_rule vlan_secondary_rule; /**< VLAN-related acceleration parameters. */
#ifdef CONFIG_XFRM
struct sfe_acceleration_direction_rule direction_rule;
/**< Direction-related acceleration parameters. */
#endif
/*
* Response
*/
struct sfe_service_class_rule sawf_rule; /**< Service class related information. */
u32 index; /**< Slot ID for cache statistics to host OS. */
};
/**
* IPv6 rule destroy submessage structure.
*/
struct sfe_ipv6_rule_destroy_msg {
struct sfe_ipv6_5tuple tuple; /**< Holds values of the sfe_ipv6_5tuple tuple */
};
/**
* SFE IPv6 rule sync structure.
*/
struct sfe_ipv6_conn_sync {
u32 index; /**< Slot ID for cache statistics to host OS. */
u8 protocol; /**< Protocol number. */
__be32 flow_ip[4]; /**< Flow IP address. */
__be16 flow_ident; /**< Flow identifier, e.g., port. */
u32 flow_max_window; /**< Flow direction's largest seen window. */
u32 flow_end; /**< Flow direction's largest seen sequence + segment length. */
u32 flow_max_end; /**< Flow direction's largest seen ack + max(1, win). */
u32 flow_rx_packet_count; /**< Flow interface's Rx packet count. */
u32 flow_rx_byte_count; /**< Flow interface's Rx byte count. */
u32 flow_tx_packet_count; /**< Flow interface's Tx packet count. */
u32 flow_tx_byte_count; /**< Flow interface's Tx byte count. */
u16 flow_pppoe_session_id; /**< Flow interface`s PPPoE session ID. */
u16 flow_pppoe_remote_mac[3]; /**< Flow interface's PPPoE remote server MAC address (if present). */
__be32 return_ip[4]; /**< Return IP address. */
__be16 return_ident; /**< Return identifer, e.g., port. */
u32 return_max_window; /**< Return direction's largest seen window. */
u32 return_end; /**< Return direction's largest seen sequence + segment length. */
u32 return_max_end; /**< Return direction's largest seen ack + max(1, win). */
u32 return_rx_packet_count; /**< Return interface's Rx packet count. */
u32 return_rx_byte_count; /**< Return interface's Rx byte count. */
u32 return_tx_packet_count; /**< Return interface's Tx packet count. */
u32 return_tx_byte_count; /**< Return interface's Tx byte count. */
u16 return_pppoe_session_id; /**< Return interface`s PPPoE session ID. */
u16 return_pppoe_remote_mac[3]; /**< Return interface's PPPoE remote server MAC address (if present). */
u32 inc_ticks; /**< Number of ticks since the last sync. */
u32 reason; /**< Sync reason. */
u8 flags; /**< Bit flags associated with the rule. */
u32 qos_tag; /**< QoS tag. */
u32 cause; /**< Flush cause associated with the rule. */
};
/**
* Information for a multiple IPv6 connection statistics synchronization message.
*/
struct sfe_ipv6_conn_sync_many_msg {
/*
* Request:
*/
uint16_t index; /**< Request connection statistics from the index. */
uint16_t size; /**< Buffer size of this message. */
/*
* Response:
*/
uint16_t next; /**< Firmware response for the next connection to be requested. */
uint16_t count; /**< Number of synchronized connections included in this message. */
struct sfe_ipv6_conn_sync conn_sync[]; /**< Array for the statistics. */
};
/**
* Message structure to send/receive IPv6 bridge/route commands.
*/
struct sfe_ipv6_msg {
struct sfe_cmn_msg cm; /**< Message header. */
union {
struct sfe_ipv6_rule_create_msg rule_create;
/**< Rule create message. */
struct sfe_ipv6_rule_destroy_msg rule_destroy;
/**< Rule destroy message. */
struct sfe_ipv6_conn_sync conn_stats;
/**< Statistics synchronization message. */
struct sfe_ipv6_conn_sync_many_msg conn_stats_many;
/**< Many Connections' statistics synchronizaion message. */
} msg; /**< IPv6 message. */
};
/**
* @}
*/
/**
* @addtogroup nss_sfe_functions
* @{
*/
/**
* IPv6 message received callback.
*/
typedef void (*sfe_ipv6_msg_callback_t)(void *app_data, struct sfe_ipv6_msg *msg);
/**
* @}
*/
/**
* @addtogroup nss_sfe_datatypes
* @{
*/
/**
* 6rd tunnel peer address.
*/
struct sfe_tun6rd_set_peer_msg {
__be32 ipv6_address[4]; /**< The peer's IPv6 address. */
__be32 dest; /**< The peer's IPv4 address. */
};
/**
* Message structure to send/receive 6rd tunnel messages.
*/
struct sfe_tun6rd_msg {
struct sfe_cmn_msg cm; /**< Message header. */
union {
struct sfe_tun6rd_set_peer_msg peer; /**< Add or update peer message. */
} msg; /**< 6RD tunnel message. */
};
/**
* SFE context instance.
*/
struct sfe_ctx_instance {
int not_used; /**< Not used. */
};
/**
* @}
*/
/**
* @addtogroup nss_sfe_functions
* @{
*/
/**
* Copy the IPv4 statistics for the given service class.
*
* @param sid Service class ID
* @param bytes Pointer to where byte count should be written.
* @param packets Pointer to where packet count should be written.
*
* @return
* True if successful, false if maximum retries exceeded; bool.
*/
extern bool sfe_service_class_stats_get(uint8_t sid, uint64_t *bytes, uint64_t *packets);
/**
* Gets the maximum number of IPv4 connections supported by the SFE acceleration engine.
*
* @return
* The maximum number of connections that can be accelerated by the SFE.
*/
int sfe_ipv4_max_conn_count(void);
/**
* Transmits an IPv4 message to the SFE.
*
* @param sfe_ctx SFE context.
* @param msg The IPv4 message.
*
* @return
* The status of the Tx operation (#sfe_tx_status_t).
*/
extern sfe_tx_status_t sfe_ipv4_tx(struct sfe_ctx_instance *sfe_ctx, struct sfe_ipv4_msg *msg);
/**
* Registers a notifier callback for IPv4 messages from the SFE.
*
* @param one_rule_cb The callback pointer for one rule.
* @param many_rules_cb The callback pointer for many rules.
* @param app_data The application context for this message.
*
* @return
* The SFE context (#sfe_ctx_instance).
*/
extern struct sfe_ctx_instance *sfe_ipv4_notify_register(sfe_ipv4_msg_callback_t one_rule_cb,
sfe_ipv4_msg_callback_t many_rules_cb,void *app_data);
/**
* Unregisters a notifier callback for IPv4 messages from the SFE.
*
* @return
* None.
*/
extern void sfe_ipv4_notify_unregister(void);
/**
* Initializes an IPv4 message.
*
* @param nim The IPv4 message pointer.
* @param if_num The interface number.
* @param type The type of the message.
* @param len The length of the message.
* @param cb The message callback.
* @param app_data The application context for this message.
*
*/
extern void sfe_ipv4_msg_init(struct sfe_ipv4_msg *nim, u16 if_num, u32 type, u32 len,
sfe_ipv4_msg_callback_t cb, void *app_data);
/**
* Gets the maximum number of IPv6 connections supported by the SFE acceleration engine.
*
* @return
* The maximum number of connections that can be accelerated by the SFE; integer.
*/
int sfe_ipv6_max_conn_count(void);
/**
* Transmits an IPv6 message to the SFE.
*
* @param sfe_ctx The SFE context.
* @param msg The IPv6 message.
*
* @return
* The status of the Tx operation (#sfe_tx_status_t).
*/
extern sfe_tx_status_t sfe_ipv6_tx(struct sfe_ctx_instance *sfe_ctx, struct sfe_ipv6_msg *msg);
/**
* Registers a notifier callback for IPv6 messages from the SFE.
*
* @param one_rule_cb The callback pointer for one rule.
* @param many_rules_cb The callback pointer for many rules.
*
* @return
* The SFE context (#sfe_ctx_instance).
*/
extern struct sfe_ctx_instance *sfe_ipv6_notify_register(sfe_ipv6_msg_callback_t one_rule_cb,
sfe_ipv6_msg_callback_t many_rules_cb,void *app_data);
/**
* Unregisters a notifier callback for IPv6 messages from the SFE.
*/
extern void sfe_ipv6_notify_unregister(void);
/**
* Initializes an IPv6 message.
*
* @param nim The IPv6 message pointer.
* @param if_num The interface number.
* @param type The type of the message.
* @param len The length of the message.
* @param cb The message callback.
* @param app_data The application context for this message.
*
* @return
* None.
*/
extern void sfe_ipv6_msg_init(struct sfe_ipv6_msg *nim, u16 if_num, u32 type, u32 len,
sfe_ipv6_msg_callback_t cb, void *app_data);
/**
* Transmits a 6rd tunnel message to the SFE.
*
* @param sfe_ctx The SFE context pointer.
* @param msg The 6rd tunnel message pointer.
*
* @return
* The status of the Tx operation (#sfe_tx_status_t).
*/
sfe_tx_status_t sfe_tun6rd_tx(struct sfe_ctx_instance *sfe_ctx, struct sfe_tun6rd_msg *msg);
/**
* Initializes a 6rd tunnel message.
*
* @param ncm The 6rd tunnel message pointer.
* @param if_num The interface number.
* @param type The type of the message.
* @param len The length of the message.
* @param cb The message callback.
* @param app_data The application context for this message.
*
* @return
* None.
*/
void sfe_tun6rd_msg_init(struct sfe_tun6rd_msg *ncm, u16 if_num, u32 type, u32 len,
void *cb, void *app_data);
/**
* Indicates whether the l2 feature flag is enabled or disabled.
*
* @return
* True if enabled; false if disabled.
*/
bool sfe_is_l2_feature_enabled(void);
/**
* Updates mark values of an IPv4 connection.
*
* @param mark The mark object.
*
* @return
* None.
*/
void sfe_ipv4_mark_rule_update(struct sfe_connection_mark *mark);
/**
* Updates mark values of an IPv6 connection.
*
* @param mark The mark object.
*
* @return
* None.
*/
void sfe_ipv6_mark_rule_update(struct sfe_connection_mark *mark);
/**
* Gets the acceleration mode of PPPoE bridge.
*
* @return
* The acceleration mode.
*/
sfe_pppoe_br_accel_mode_t sfe_pppoe_get_br_accel_mode(void);
/**
* @}
*/
#endif /* __SFE_API_H */

109
shortcut-fe/fast-classifier/Makefile Normal file
View file

@ -0,0 +1,109 @@
#
# Copyright (c) 2013-2018 The Linux Foundation. All rights reserved.
# Permission to use, copy, modify, and/or distribute this software for
# any purpose with or without fee is hereby granted, provided that the
# above copyright notice and this permission notice appear in all copies.
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
include $(TOPDIR)/rules.mk
include $(INCLUDE_DIR)/kernel.mk
PKG_NAME:=fast-classifier
PKG_RELEASE:=6
include $(INCLUDE_DIR)/package.mk
define KernelPackage/fast-classifier/Default
SECTION:=kernel
CATEGORY:=Kernel modules
SUBMENU:=Network Support
DEPENDS:=+kmod-ipt-conntrack +kmod-shortcut-fe
TITLE:=Kernel driver for FAST Classifier
FILES:=$(PKG_BUILD_DIR)/fast-classifier.ko
KCONFIG:= \
CONFIG_NF_CONNTRACK_EVENTS=y \
CONFIG_NF_CONNTRACK_CHAIN_EVENTS=y \
CONFIG_NF_CONNTRACK_MARK=y \
CONFIG_XFRM=y
CONFLICTS:=kmod-shortcut-fe-drv kmod-shortcut-fe-cm
endef
define KernelPackage/fast-classifier
$(call KernelPackage/fast-classifier/Default)
endef
define KernelPackage/fast-classifier-noload
$(call KernelPackage/fast-classifier/Default)
endef
define KernelPackage/fast-classifier/Default/description
FAST Classifier talks to SFE to make decisions about offloading connections
endef
define KernelPackage/fast-classifier/description
$(call KernelPackage/fast-classifier/Default/description)
endef
define KernelPackage/fast-classifier-noload/description
$(call KernelPackage/fast-classifier/Default/description)
This package does not load fast-classifier at boot by default
endef
define Package/fast-classifier-example
TITLE:=Example user space program for fast-classifier
DEPENDS:=+libnl +kmod-fast-classifier
endef
define Package/fast-classifier-example/description
Example user space program that communicates with fast
classifier kernel module
endef
HAVE_ECM:=$(CONFIG_PACKAGE_kmod-qca-nss-ecm-premium)$(CONFIG_PACKAGE_kmod-qca-nss-ecm-noload)$(CONFIG_PACKAGE_kmod-qca-nss-ecm-premium-noload)$(CONFIG_PACKAGE_kmod-qca-nss-ecm-standard)
define Build/Compile/kmod
+$(MAKE) $(PKG_JOBS) -C "$(LINUX_DIR)" \
$(KERNEL_MAKE_FLAGS) \
$(PKG_MAKE_FLAGS) \
M="$(PKG_BUILD_DIR)" \
CONFIG_FAST_CLASSIFIER=m \
EXTRA_CFLAGS+="-DSFE_SUPPORT_IPV6" \
$(if $(HAVE_ECM),EXTRA_CFLAGS+="-DCONFIG_SFE_ECM" CONFIG_SFE_ECM=y,) \
modules
endef
define Build/Compile/example
$(TARGET_CC) -o $(PKG_BUILD_DIR)/userspace_fast_classifier \
-I $(PKG_BUILD_DIR) \
-I$(STAGING_DIR)/usr/include/libnl \
-I$(STAGING_DIR)/usr/include/libnl3 \
-lnl-genl-3 -lnl-3 \
$(PKG_BUILD_DIR)/nl_classifier_test.c
endef
define Build/Compile
$(Build/Compile/kmod)
$(if $(CONFIG_PACKAGE_fast-classifier-example),$(Build/Compile/example))
endef
define Build/InstallDev
$(INSTALL_DIR) $(1)/usr/include
$(CP) $(PKG_BUILD_DIR)/fast-classifier.h $(1)/usr/include/
endef
define Package/fast-classifier-example/install
$(INSTALL_DIR) $(1)/sbin
$(CP) $(PKG_BUILD_DIR)/userspace_fast_classifier $(1)/sbin/
endef
$(eval $(call KernelPackage,fast-classifier))
#$(eval $(call KernelPackage,fast-classifier-noload))
#$(eval $(call BuildPackage,fast-classifier-example))

10
shortcut-fe/fast-classifier/src/Makefile Normal file
View file

@ -0,0 +1,10 @@
obj-$(CONFIG_FAST_CLASSIFIER) += fast-classifier.o
ifeq ($(SFE_SUPPORT_IPV6),)
SFE_SUPPORT_IPV6=y
endif
ccflags-$(SFE_SUPPORT_IPV6) += -DSFE_SUPPORT_IPV6
ccflags-y += -I$(obj)/../shortcut-fe
obj ?= .

2002
shortcut-fe/fast-classifier/src/fast-classifier.c Normal file
View file

File diff suppressed because it is too large Load diff

57
shortcut-fe/fast-classifier/src/fast-classifier.h Normal file
View file

@ -0,0 +1,57 @@
/*
* User space header to send message to the fast classifier
*
* Copyright (c) 2013,2016 The Linux Foundation. All rights reserved.
* Permission to use, copy, modify, and/or distribute this software for
* any purpose with or without fee is hereby granted, provided that the
* above copyright notice and this permission notice appear in all copies.
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
* OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <linux/if_ether.h>
#define FAST_CLASSIFIER_GENL_VERSION (1)
#define FAST_CLASSIFIER_GENL_NAME "FC"
#define FAST_CLASSIFIER_GENL_MCGRP "FC_MCGRP"
#define FAST_CLASSIFIER_GENL_HDRSIZE (0)
enum {
FAST_CLASSIFIER_A_UNSPEC,
FAST_CLASSIFIER_A_TUPLE,
__FAST_CLASSIFIER_A_MAX,
};
#define FAST_CLASSIFIER_A_MAX (__FAST_CLASSIFIER_A_MAX - 1)
enum {
FAST_CLASSIFIER_C_UNSPEC,
FAST_CLASSIFIER_C_OFFLOAD,
FAST_CLASSIFIER_C_OFFLOADED,
FAST_CLASSIFIER_C_DONE,
__FAST_CLASSIFIER_C_MAX,
};
#define FAST_CLASSIFIER_C_MAX (__FAST_CLASSIFIER_C_MAX - 1)
struct fast_classifier_tuple {
unsigned short ethertype;
unsigned char proto;
union {
struct in_addr in;
struct in6_addr in6;
} src_saddr;
union {
struct in_addr in;
struct in6_addr in6;
} dst_saddr;
unsigned short sport;
unsigned short dport;
unsigned char smac[ETH_ALEN];
unsigned char dmac[ETH_ALEN];
};

281
shortcut-fe/fast-classifier/src/nl_classifier_test.c Normal file
View file

@ -0,0 +1,281 @@
/*
* Copyright (c) 2016 The Linux Foundation. All rights reserved.
* Permission to use, copy, modify, and/or distribute this software for
* any purpose with or without fee is hereby granted, provided that the
* above copyright notice and this permission notice appear in all copies.
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
* OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <netlink/genl/genl.h>
#include <netlink/genl/ctrl.h>
#include <errno.h>
#include <stdio.h>
#include <signal.h>
#include <arpa/inet.h>
#define NL_CLASSIFIER_GENL_VERSION 1
#define NL_CLASSIFIER_GENL_FAMILY "FC"
#define NL_CLASSIFIER_GENL_GROUP "FC_MCGRP"
#define NL_CLASSIFIER_GENL_HDRSIZE 0
enum NL_CLASSIFIER_CMD {
NL_CLASSIFIER_CMD_UNSPEC,
NL_CLASSIFIER_CMD_ACCEL,
NL_CLASSIFIER_CMD_ACCEL_OK,
NL_CLASSIFIER_CMD_CONNECTION_CLOSED,
NL_CLASSIFIER_CMD_MAX,
};
enum NL_CLASSIFIER_ATTR {
NL_CLASSIFIER_ATTR_UNSPEC,
NL_CLASSIFIER_ATTR_TUPLE,
NL_CLASSIFIER_ATTR_MAX,
};
union nl_classifier_tuple_ip {
struct in_addr in;
struct in6_addr in6;
};
struct nl_classifier_tuple {
unsigned short af;
unsigned char proto;
union nl_classifier_tuple_ip src_ip;
union nl_classifier_tuple_ip dst_ip;
unsigned short sport;
unsigned short dport;
unsigned char smac[6];
unsigned char dmac[6];
};
struct nl_classifier_instance {
struct nl_sock *sock;
int family_id;
int group_id;
int stop;
};
struct nl_classifier_instance nl_cls_inst;
static struct nla_policy nl_classifier_genl_policy[(NL_CLASSIFIER_ATTR_MAX+1)] = {
[NL_CLASSIFIER_ATTR_TUPLE] = { .type = NLA_UNSPEC },
};
void nl_classifier_dump_nl_tuple(struct nl_classifier_tuple *tuple)
{
char ip_str[64];
printf("protocol = %s\n", (tuple->proto == IPPROTO_UDP) ? "udp" : ((tuple->proto == IPPROTO_TCP) ? "tcp" : "unknown"));
printf("source ip = %s\n", inet_ntop(tuple->af, &tuple->src_ip, ip_str, sizeof(ip_str)));
printf("destination ip = %s\n", inet_ntop(tuple->af, &tuple->dst_ip, ip_str, sizeof(ip_str)));
printf("source port = %d\n", ntohs(tuple->sport));
printf("destination port = %d\n", ntohs(tuple->dport));
}
int nl_classifier_msg_recv(struct nl_msg *msg, void *arg)
{
struct nlmsghdr *nlh = nlmsg_hdr(msg);
struct genlmsghdr *gnlh = nlmsg_data(nlh);
struct nlattr *attrs[(NL_CLASSIFIER_ATTR_MAX+1)];
genlmsg_parse(nlh, NL_CLASSIFIER_GENL_HDRSIZE, attrs, NL_CLASSIFIER_ATTR_MAX, nl_classifier_genl_policy);
switch (gnlh->cmd) {
case NL_CLASSIFIER_CMD_ACCEL_OK:
printf("Acceleration successful:\n");
nl_classifier_dump_nl_tuple(nla_data(attrs[NL_CLASSIFIER_ATTR_TUPLE]));
return NL_OK;
case NL_CLASSIFIER_CMD_CONNECTION_CLOSED:
printf("Connection is closed:\n");
nl_classifier_dump_nl_tuple(nla_data(attrs[NL_CLASSIFIER_ATTR_TUPLE]));
return NL_OK;
default:
printf("nl classifier received unknow message %d\n", gnlh->cmd);
}
return NL_SKIP;
}
void nl_classifier_offload(struct nl_classifier_instance *inst,
unsigned char proto, unsigned long *src_saddr,
unsigned long *dst_saddr, unsigned short sport,
unsigned short dport, int af)
{
struct nl_msg *msg;
int ret;
struct nl_classifier_tuple classifier_msg;
memset(&classifier_msg, 0, sizeof(classifier_msg));
classifier_msg.af = af;
classifier_msg.proto = proto;
memcpy(&classifier_msg.src_ip, src_saddr, (af == AF_INET ? 4 : 16));
memcpy(&classifier_msg.dst_ip, dst_saddr, (af == AF_INET ? 4 : 16));
classifier_msg.sport = sport;
classifier_msg.dport = dport;
msg = nlmsg_alloc();
if (!msg) {
printf("Unable to allocate message\n");
return;
}
genlmsg_put(msg, NL_AUTO_PID, NL_AUTO_SEQ, inst->family_id,
NL_CLASSIFIER_GENL_HDRSIZE, NLM_F_REQUEST,
NL_CLASSIFIER_CMD_ACCEL, NL_CLASSIFIER_GENL_VERSION);
nla_put(msg, NL_CLASSIFIER_ATTR_TUPLE, sizeof(classifier_msg), &classifier_msg);
ret = nl_send_auto(inst->sock, msg);
if (ret < 0) {
printf("send netlink message failed.\n");
nlmsg_free(msg);
return;
}
nlmsg_free(msg);
printf("nl classifier offload connection successful\n");
}
int nl_classifier_init(struct nl_classifier_instance *inst)
{
int ret;
inst->sock = nl_socket_alloc();
if (!inst->sock) {
printf("Unable to allocation socket.\n");
return -1;
}
genl_connect(inst->sock);
inst->family_id = genl_ctrl_resolve(inst->sock, NL_CLASSIFIER_GENL_FAMILY);
if (inst->family_id < 0) {
printf("Unable to resolve family %s\n", NL_CLASSIFIER_GENL_FAMILY);
goto init_failed;
}
inst->group_id = genl_ctrl_resolve_grp(inst->sock, NL_CLASSIFIER_GENL_FAMILY, NL_CLASSIFIER_GENL_GROUP);
if (inst->group_id < 0) {
printf("Unable to resolve mcast group %s\n", NL_CLASSIFIER_GENL_GROUP);
goto init_failed;
}
ret = nl_socket_add_membership(inst->sock, inst->group_id);
if (ret < 0) {
printf("Unable to add membership\n");
goto init_failed;
}
nl_socket_disable_seq_check(inst->sock);
nl_socket_modify_cb(inst->sock, NL_CB_VALID, NL_CB_CUSTOM, nl_classifier_msg_recv, NULL);
printf("nl classifier init successful\n");
return 0;
init_failed:
if (inst->sock) {
nl_close(inst->sock);
nl_socket_free(inst->sock);
inst->sock = NULL;
}
return -1;
}
void nl_classifier_exit(struct nl_classifier_instance *inst)
{
if (inst->sock) {
nl_close(inst->sock);
nl_socket_free(inst->sock);
inst->sock = NULL;
}
printf("nl classifier exit successful\n");
}
int nl_classifier_parse_arg(int argc, char *argv[], unsigned char *proto, unsigned long *src_saddr,
unsigned long *dst_saddr, unsigned short *sport, unsigned short *dport, int *af)
{
int ret;
unsigned short port;
if (argc < 7) {
printf("help: nl_classifier <v4|v6> <udp|tcp> <source ip> <destination ip> <source port> <destination port>\n");
return -1;
}
if (0 == strncmp(argv[1], "v4", 2)) {
*af = AF_INET;
} else if (0 == strncmp(argv[1], "v6", 2)) {
*af = AF_INET6;
} else {
printf("Address family is not supported");
return -1;
}
if (0 == strncmp(argv[2], "udp", 3)) {
*proto = IPPROTO_UDP;
} else if (0 == strncmp(argv[2], "tcp", 3)) {
*proto = IPPROTO_TCP;
} else {
printf("Protocol is not supported");
return -1;
}
ret = inet_pton(*af, argv[3], src_saddr);
if (ret <= 0) {
printf("source ip has wrong format\n");
return -1;
}
ret = inet_pton(*af, argv[4], dst_saddr);
if (ret <= 0) {
printf("destination ip has wrong format\n");
return -1;
}
port = strtol(argv[5], NULL, 0);
*sport = htons(port);
port = strtol(argv[6], NULL, 0);
*dport = htons(port);
printf("nl classifier parse arguments successful\n");
return 0;
}
int main(int argc, char *argv[])
{
struct nl_classifier_instance *inst = &nl_cls_inst;
unsigned char proto;
unsigned long src_addr[4];
unsigned long dst_addr[4];
unsigned short sport;
unsigned short dport;
int af;
int ret;
ret = nl_classifier_parse_arg(argc, argv, &proto, src_addr, dst_addr, &sport, &dport, &af);
if (ret < 0) {
printf("Failed to parse arguments\n");
return ret;
}
ret = nl_classifier_init(inst);
if (ret < 0) {
printf("Unable to init generic netlink\n");
return ret;
}
nl_classifier_offload(inst, proto, src_addr, dst_addr, sport, dport, af);
/* main loop to listen on message */
while (!inst->stop) {
nl_recvmsgs_default(inst->sock);
}
nl_classifier_exit(inst);
return 0;
}

232
shortcut-fe/fast-classifier/src/userspace_example.c Normal file
View file

@ -0,0 +1,232 @@
/*
* Copyright (c) 2013,2016 The Linux Foundation. All rights reserved.
* Permission to use, copy, modify, and/or distribute this software for
* any purpose with or without fee is hereby granted, provided that the
* above copyright notice and this permission notice appear in all copies.
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
* OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <netlink/genl/genl.h>
#include <netlink/genl/ctrl.h>
#include <errno.h>
#include <stdio.h>
#include <arpa/inet.h>
#include <fast-classifier.h>
static struct nl_sock *sock;
static struct nl_sock *sock_event;
static int family;
static int grp_id;
static struct nla_policy fast_classifier_genl_policy[FAST_CLASSIFIER_A_MAX + 1] = {
[FAST_CLASSIFIER_A_TUPLE] = { .type = NLA_UNSPEC },
};
void dump_fc_tuple(struct fast_classifier_tuple *fc_msg)
{
char src_str[INET_ADDRSTRLEN];
char dst_str[INET_ADDRSTRLEN];
printf("TUPLE: %d, %s, %s, %d, %d"
" SMAC=%02x:%02x:%02x:%02x:%02x:%02x",
" DMAC=%02x:%02x:%02x:%02x:%02x:%02x\n",
fc_msg->proto,
inet_ntop(AF_INET,
&fc_msg->src_saddr.in.s_addr,
src_str,
INET_ADDRSTRLEN),
inet_ntop(AF_INET,
&fc_msg->dst_saddr.in.s_addr,
dst_str,
INET_ADDRSTRLEN),
fc_msg->sport, fc_msg->dport,
fc_msg->smac[0], fc_msg->smac[1], fc_msg->smac[2],
fc_msg->smac[3], fc_msg->smac[4], fc_msg->smac[5],
fc_msg->dmac[0], fc_msg->dmac[1], fc_msg->dmac[2],
fc_msg->dmac[3], fc_msg->dmac[4], fc_msg->dmac[5]);
}
static int parse_cb(struct nl_msg *msg, void *arg)
{
struct nlmsghdr *nlh = nlmsg_hdr(msg);
struct genlmsghdr *gnlh = nlmsg_data(nlh);
struct nlattr *attrs[FAST_CLASSIFIER_A_MAX];
genlmsg_parse(nlh, 0, attrs, FAST_CLASSIFIER_A_MAX, fast_classifier_genl_policy);
switch (gnlh->cmd) {
case FAST_CLASSIFIER_C_OFFLOADED:
printf("Got a offloaded message\n");
dump_fc_tuple(nla_data(attrs[FAST_CLASSIFIER_A_TUPLE]));
return NL_OK;
case FAST_CLASSIFIER_C_DONE:
printf("Got a done message\n");
dump_fc_tuple(nla_data(attrs[FAST_CLASSIFIER_A_TUPLE]));
return NL_OK;
}
return NL_SKIP;
}
int fast_classifier_init(void)
{
int err;
sock = nl_socket_alloc();
if (!sock) {
printf("Unable to allocation socket.\n");
return -1;
}
genl_connect(sock);
sock_event = nl_socket_alloc();
if (!sock_event) {
nl_close(sock);
nl_socket_free(sock);
printf("Unable to allocation socket.\n");
return -1;
}
genl_connect(sock_event);
family = genl_ctrl_resolve(sock, FAST_CLASSIFIER_GENL_NAME);
if (family < 0) {
nl_close(sock_event);
nl_close(sock);
nl_socket_free(sock);
nl_socket_free(sock_event);
printf("Unable to resolve family\n");
return -1;
}
grp_id = genl_ctrl_resolve_grp(sock, FAST_CLASSIFIER_GENL_NAME,
FAST_CLASSIFIER_GENL_MCGRP);
if (grp_id < 0) {
printf("Unable to resolve mcast group\n");
return -1;
}
err = nl_socket_add_membership(sock_event, grp_id);
if (err < 0) {
printf("Unable to add membership\n");
return -1;
}
nl_socket_disable_seq_check(sock_event);
nl_socket_modify_cb(sock_event, NL_CB_VALID, NL_CB_CUSTOM, parse_cb, NULL);
return 0;
}
void fast_classifier_close(void)
{
nl_close(sock_event);
nl_close(sock);
nl_socket_free(sock_event);
nl_socket_free(sock);
}
void fast_classifier_ipv4_offload(unsigned char proto, unsigned long src_saddr,
unsigned long dst_saddr, unsigned short sport,
unsigned short dport)
{
struct nl_msg *msg;
int ret;
#ifdef DEBUG
char src_str[INET_ADDRSTRLEN];
char dst_str[INET_ADDRSTRLEN];
#endif
struct fast_classifier_tuple fc_msg;
#ifdef DEBUG
printf("DEBUG: would offload: %d, %s, %s, %d, %d\n", proto,
inet_ntop(AF_INET, &src_saddr, src_str, INET_ADDRSTRLEN),
inet_ntop(AF_INET, &dst_saddr, dst_str, INET_ADDRSTRLEN),
sport, dport);
#endif
fc_msg.proto = proto;
fc_msg.src_saddr.in.s_addr = src_saddr;
fc_msg.dst_saddr.in.s_addr = dst_saddr;
fc_msg.sport = sport;
fc_msg.dport = dport;
fc_msg.smac[0] = 'a';
fc_msg.smac[1] = 'b';
fc_msg.smac[2] = 'c';
fc_msg.smac[3] = 'd';
fc_msg.smac[4] = 'e';
fc_msg.smac[5] = 'f';
fc_msg.dmac[0] = 'f';
fc_msg.dmac[1] = 'e';
fc_msg.dmac[2] = 'd';
fc_msg.dmac[3] = 'c';
fc_msg.dmac[4] = 'b';
fc_msg.dmac[5] = 'a';
if (fast_classifier_init() < 0) {
printf("Unable to init generic netlink\n");
exit(1);
}
msg = nlmsg_alloc();
if (!msg) {
nl_socket_free(sock);
printf("Unable to allocate message\n");
return;
}
genlmsg_put(msg, NL_AUTO_PID, NL_AUTO_SEQ, family,
FAST_CLASSIFIER_GENL_HDRSIZE, NLM_F_REQUEST,
FAST_CLASSIFIER_C_OFFLOAD, FAST_CLASSIFIER_GENL_VERSION);
nla_put(msg, 1, sizeof(fc_msg), &fc_msg);
ret = nl_send_auto_complete(sock, msg);
nlmsg_free(msg);
if (ret < 0) {
printf("nlmsg_free failed");
nl_close(sock);
nl_socket_free(sock);
return;
}
ret = nl_wait_for_ack(sock);
if (ret < 0) {
printf("wait for ack failed");
nl_close(sock);
nl_socket_free(sock);
return;
}
}
void fast_classifier_listen_for_messages(void)
{
printf("waiting for netlink events\n");
while (1) {
nl_recvmsgs_default(sock_event);
}
}
int main(int argc, char *argv[])
{
if (fast_classifier_init() < 0) {
printf("Unable to init generic netlink\n");
exit(1);
}
fast_classifier_ipv4_offload('a', 0, 0, 0, 0);
/* this never returns */
fast_classifier_listen_for_messages();
fast_classifier_close();
return 0;
}

1838
shortcut-fe/sfe.c
View file

File diff suppressed because it is too large Load diff

331
shortcut-fe/sfe.h
View file

@ -1,331 +0,0 @@
/*
* sfe.h
* Shortcut forwarding engine.
*
* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
* Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#ifndef __SFE_H
#define __SFE_H
/*
* Maximum number of accelerated IPv4 or IPv6 connections
*/
#if defined(SFE_MEM_PROFILE_LOW)
#define SFE_MAX_CONNECTION_NUM 512
#elif defined(SFE_MEM_PROFILE_MEDIUM)
#define SFE_MAX_CONNECTION_NUM 2048
#else
#define SFE_MAX_CONNECTION_NUM 4096
#endif
#define SFE_L2_PARSE_FLAGS_PPPOE_INGRESS 0x01 /* Indicates presence of a valid PPPoE header */
/**
* SAWF_metadata information placement in mark field.
*/
#define SFE_SAWF_VALID_TAG 0xAA
#define SFE_SAWF_TAG_SHIFT 0x18
#define SFE_SAWF_SERVICE_CLASS_SHIFT 0x10
#define SFE_SAWF_SERVICE_CLASS_MASK 0xff
#define SFE_SAWF_MSDUQ_MASK 0xffff
/**
* SAWF_metadata extraction.
*/
#define SFE_GET_SAWF_TAG(x) (x>>SFE_SAWF_TAG_SHIFT)
#define SFE_GET_SAWF_SERVICE_CLASS(x) ((x>>SFE_SAWF_SERVICE_CLASS_SHIFT) & SFE_SAWF_SERVICE_CLASS_MASK)
#define SFE_GET_SAWF_MSDUQ(x) (x & SFE_SAWF_MSDUQ_MASK)
#define SFE_SAWF_TAG_IS_VALID(x) ((x == SFE_SAWF_VALID_TAG) ? true : false)
/*
* IPv6 address structure
*/
struct sfe_ipv6_addr {
__be32 addr[4];
};
typedef union {
__be32 ip;
struct sfe_ipv6_addr ip6[1];
} sfe_ip_addr_t;
typedef enum sfe_sync_reason {
SFE_SYNC_REASON_STATS, /* Sync is to synchronize stats */
SFE_SYNC_REASON_FLUSH, /* Sync is to flush a entry */
SFE_SYNC_REASON_DESTROY /* Sync is to destroy a entry(requested by connection manager) */
} sfe_sync_reason_t;
/*
* VLAN header (aka VLAN tag)
*/
struct sfe_vlan_hdr {
u16 tpid; /* Tag Protocol Identifier */
u16 tci; /* Tag Control Information */
};
/*
* Structure used to store L2 information
*/
struct sfe_l2_info {
u16 parse_flags; /* Flags indicating L2.5 headers presence */
u16 pppoe_session_id; /* PPPOE header offset */
u16 protocol; /* L3 Protocol */
struct sfe_vlan_hdr vlan_hdr[SFE_MAX_VLAN_DEPTH];
/* VLAN tag(s) of ingress packet */
u8 vlan_hdr_cnt; /* Number of VLAN tags in the ingress packet */
};
/*
* Structure used to sync connection stats/state back within the system.
*
* NOTE: The addresses here are NON-NAT addresses, i.e. the true endpoint addressing.
* 'src' is the creator of the connection.
*/
struct sfe_connection_sync {
struct net_device *src_dev;
struct net_device *dest_dev;
int is_v6; /* Is it for ipv6? */
int protocol; /* IP protocol number (IPPROTO_...) */
sfe_ip_addr_t src_ip; /* Non-NAT source address, i.e. the creator of the connection */
sfe_ip_addr_t src_ip_xlate; /* NATed source address */
__be16 src_port; /* Non-NAT source port */
__be16 src_port_xlate; /* NATed source port */
sfe_ip_addr_t dest_ip; /* Non-NAT destination address, i.e. to whom the connection was created */
sfe_ip_addr_t dest_ip_xlate; /* NATed destination address */
__be16 dest_port; /* Non-NAT destination port */
__be16 dest_port_xlate; /* NATed destination port */
u32 src_td_max_window;
u32 src_td_end;
u32 src_td_max_end;
u64 src_packet_count;
u64 src_byte_count;
u32 src_new_packet_count;
u32 src_new_byte_count;
u32 dest_td_max_window;
u32 dest_td_end;
u32 dest_td_max_end;
u64 dest_packet_count;
u64 dest_byte_count;
u32 dest_new_packet_count;
u32 dest_new_byte_count;
u32 reason; /* reason for stats sync message, i.e. destroy, flush, period sync */
u64 delta_jiffies; /* Time to be added to the current timeout to keep the connection alive */
};
/*
* Expose the hook for the receive processing.
*/
extern int (*athrs_fast_nat_recv)(struct sk_buff *skb);
/*
* Expose what should be a static flag in the TCP connection tracker.
*/
extern int nf_ct_tcp_no_window_check;
/*
* Check the fast transmit feasibility.
*/
bool sfe_fast_xmit_check(struct sk_buff *skb, netdev_features_t features);
/*
* This callback will be called in a timer
* at 100 times per second to sync stats back to
* Linux connection track.
*
* A RCU lock is taken to prevent this callback
* from unregistering.
*/
typedef void (*sfe_sync_rule_callback_t)(struct sfe_connection_sync *);
typedef void (*sfe_ipv4_many_sync_callback_t)(struct sfe_ipv4_msg *msg);
typedef void (*sfe_ipv6_many_sync_callback_t)(struct sfe_ipv6_msg *msg);
/*
* IPv4 APIs used by connection manager
*/
int sfe_ipv4_recv(struct net_device *dev, struct sk_buff *skb, struct sfe_l2_info *l2_info, bool tun_outer);
int sfe_ipv4_create_rule(struct sfe_ipv4_rule_create_msg *msg);
void sfe_ipv4_destroy_rule(struct sfe_ipv4_rule_destroy_msg *msg);
void sfe_ipv4_destroy_all_rules_for_dev(struct net_device *dev);
void sfe_ipv4_register_sync_rule_callback(sfe_sync_rule_callback_t callback);
void sfe_ipv4_update_rule(struct sfe_ipv4_rule_create_msg *msg);
bool sfe_dev_has_hw_csum(struct net_device *dev);
bool sfe_ipv4_sync_invoke(uint16_t index);
void sfe_ipv4_register_many_sync_callback(sfe_ipv4_many_sync_callback_t cb);
void sfe_ipv4_stats_convert(struct sfe_ipv4_conn_sync *sync_msg, struct sfe_connection_sync *sis);
#ifdef SFE_SUPPORT_IPV6
/*
* IPv6 APIs used by connection manager
*/
int sfe_ipv6_recv(struct net_device *dev, struct sk_buff *skb, struct sfe_l2_info *l2_info, bool tun_outer);
int sfe_ipv6_create_rule(struct sfe_ipv6_rule_create_msg *msg);
void sfe_ipv6_destroy_rule(struct sfe_ipv6_rule_destroy_msg *msg);
void sfe_ipv6_destroy_all_rules_for_dev(struct net_device *dev);
void sfe_ipv6_register_sync_rule_callback(sfe_sync_rule_callback_t callback);
void sfe_ipv6_update_rule(struct sfe_ipv6_rule_create_msg *msg);
bool sfe_ipv6_sync_invoke(uint16_t index);
void sfe_ipv6_register_many_sync_callback(sfe_ipv6_many_sync_callback_t cb);
void sfe_ipv6_stats_convert(struct sfe_ipv6_conn_sync *sync_msg, struct sfe_connection_sync *sis);
#else
static inline int sfe_ipv6_recv(struct net_device *dev, struct sk_buff *skb, struct sfe_l2_info *l2_info, bool tun_outer)
{
return 0;
}
static inline int sfe_ipv6_create_rule(struct sfe_ipv6_rule_create_msg *msg)
{
return 0;
}
static inline void sfe_ipv6_destroy_rule(struct sfe_ipv6_rule_destroy_msg *msg)
{
return;
}
static inline void sfe_ipv6_destroy_all_rules_for_dev(struct net_device *dev)
{
return;
}
static inline void sfe_ipv6_register_sync_rule_callback(sfe_sync_rule_callback_t callback)
{
return;
}
static inline void sfe_ipv6_update_rule(struct sfe_ipv6_rule_create_msg *msg)
{
return;
}
static inline bool sfe_ipv6_sync_invoke(uint16_t index)
{
return false;
}
static inline void sfe_ipv6_register_many_sync_callback(sfe_ipv6_many_sync_callback_t cb)
{
return;
}
static inline void sfe_ipv6_stats_convert(struct sfe_ipv6_conn_sync *sync_msg, struct sfe_connection_sync *sis)
{
return;
}
#endif
/*
* sfe_ipv6_addr_equal()
* compare ipv6 address
*
* return: 1, equal; 0, no equal
*/
static inline int sfe_ipv6_addr_equal(struct sfe_ipv6_addr *a,
struct sfe_ipv6_addr *b)
{
return a->addr[0] == b->addr[0] &&
a->addr[1] == b->addr[1] &&
a->addr[2] == b->addr[2] &&
a->addr[3] == b->addr[3];
}
/*
* sfe_ipv4_addr_equal()
* compare ipv4 address
*
* return: 1, equal; 0, no equal
*/
#define sfe_ipv4_addr_equal(a, b) ((u32)(a) == (u32)(b))
/*
* sfe_addr_equal()
* compare ipv4 or ipv6 address
*
* return: 1, equal; 0, no equal
*/
static inline int sfe_addr_equal(sfe_ip_addr_t *a,
sfe_ip_addr_t *b, int is_v4)
{
return is_v4 ? sfe_ipv4_addr_equal(a->ip, b->ip) : sfe_ipv6_addr_equal(a->ip6, b->ip6);
}
/*
* sfe_l2_parse_flag_set()
* Set L2 parse flag
*/
static inline void sfe_l2_parse_flag_set(struct sfe_l2_info *l2_info, u16 flag)
{
l2_info->parse_flags |= flag;
}
/*
* sfe_l2_parse_flag_get()
* Get L2 parse flag
*/
static inline u16 sfe_l2_parse_flag_get(struct sfe_l2_info *l2_info)
{
return l2_info->parse_flags;
}
/*
* sfe_l2_parse_flag_check()
* Check L2 parse flag
*/
static inline bool sfe_l2_parse_flag_check(struct sfe_l2_info *l2_info, u16 flag)
{
return !!(l2_info->parse_flags & flag);
}
/*
* sfe_l2_pppoe_session_id_get()
* Get PPPPoE session ID from l2_info
*/
static inline u16 sfe_l2_pppoe_session_id_get(struct sfe_l2_info *l2_info)
{
return l2_info->pppoe_session_id;
}
/*
* sfe_l2_pppoe_session_id_set()
* Set PPPoE session ID to l2_info
*/
static inline void sfe_l2_pppoe_session_id_set(struct sfe_l2_info *l2_info, u16 session_id)
{
l2_info->pppoe_session_id = session_id;
}
/*
* sfe_l2_protocol_get()
* Get L2 protocol
*/
static inline u16 sfe_l2_protocol_get(struct sfe_l2_info *l2_info)
{
return l2_info->protocol;
}
/*
* sfe_l2_protocol_set()
* Set L2 protocol
*/
static inline void sfe_l2_protocol_set(struct sfe_l2_info *l2_info, u16 proto)
{
l2_info->protocol = proto;
}
int sfe_init_if(void);
void sfe_exit_if(void);
#endif /* __SFE_H */

72
shortcut-fe/sfe_debug.h
View file

@ -1,72 +0,0 @@
/*
* sfe_debug.h
* SFE debug macros.
*
* Copyright (c) 2021 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
/*
* The following are debug macros used throughout the SFE.
*
* The DEBUG_LEVEL enables the followings based on its value,
* when dynamic debug option is disabled.
*
* 0 = OFF
* 1 = ASSERTS / ERRORS
* 2 = 1 + WARN
* 3 = 2 + INFO
* 4 = 3 + TRACE
*/
#define DEBUG_LEVEL 2
#if (DEBUG_LEVEL < 1)
#define DEBUG_ASSERT(s, ...)
#define DEBUG_ERROR(s, ...)
#else
#define DEBUG_ASSERT(c, s, ...) if (!(c)) { pr_emerg("ASSERT: %s:%d:" s, __FUNCTION__, __LINE__, ##__VA_ARGS__); BUG(); }
#define DEBUG_ERROR(s, ...) pr_err("%s:%d:" s, __FUNCTION__, __LINE__, ##__VA_ARGS__)
#endif
#if defined(CONFIG_DYNAMIC_DEBUG)
/*
* Compile messages for dynamic enable/disable
*/
#define DEBUG_WARN(s, ...) pr_debug("%s[%d]:" s, __FUNCTION__, __LINE__, ##__VA_ARGS__)
#define DEBUG_INFO(s, ...) pr_debug("%s[%d]:" s, __FUNCTION__, __LINE__, ##__VA_ARGS__)
#define DEBUG_TRACE(s, ...) pr_debug("%s[%d]:" s, __FUNCTION__, __LINE__, ##__VA_ARGS__)
#else
/*
* Statically compile messages at different levels
*/
#if (DEBUG_LEVEL < 2)
#define DEBUG_WARN(s, ...)
#else
#define DEBUG_WARN(s, ...) pr_warn("%s[%d]:" s, __FUNCTION__, __LINE__, ##__VA_ARGS__)
#endif
#if (DEBUG_LEVEL < 3)
#define DEBUG_INFO(s, ...)
#else
#define DEBUG_INFO(s, ...) pr_notice("%s[%d]:" s, __FUNCTION__, __LINE__, ##__VA_ARGS__)
#endif
#if (DEBUG_LEVEL < 4)
#define DEBUG_TRACE(s, ...)
#else
#define DEBUG_TRACE(s, ...) pr_info("%s[%d]:" s, __FUNCTION__, __LINE__, ##__VA_ARGS__)
#endif
#endif

104
shortcut-fe/sfe_init.c
View file

@ -1,104 +0,0 @@
/*
* sfe_init.c
* Shortcut forwarding engine initialization.
*
* Copyright (c) 2021 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <linux/sysfs.h>
#include <linux/skbuff.h>
#include <linux/icmp.h>
#include <net/tcp.h>
#include <linux/etherdevice.h>
#include <linux/version.h>
#include "sfe_api.h"
#include "sfe.h"
#include "sfe_ipv4.h"
#include "sfe_ipv6.h"
int max_ipv4_conn = SFE_MAX_CONNECTION_NUM;
module_param(max_ipv4_conn, int, S_IRUGO);
MODULE_PARM_DESC(max_ipv4_conn, "Max number of IPv4 connections");
int max_ipv6_conn = SFE_MAX_CONNECTION_NUM;
module_param(max_ipv6_conn, int, S_IRUGO);
MODULE_PARM_DESC(max_ipv6_conn, "Max number of IPv6 connections");
/*
* sfe_init()
* Initialize SFE engine.
*/
static int __init sfe_init(void)
{
/*
* Initialize SFE IPv4 engine.
*/
if (sfe_ipv4_init()) {
goto fail0;
}
#ifdef SFE_SUPPORT_IPV6
/*
* Initialize SFE IPv6 engine.
*/
if (sfe_ipv6_init()) {
goto fail1;
}
#endif
/*
* Initialize SFE infrastructure and register SFE hook with Linux stack
*/
if (sfe_init_if()) {
goto fail2;
}
return 0;
fail2:
#ifdef SFE_SUPPORT_IPV6
sfe_ipv6_exit();
fail1:
#endif
sfe_ipv4_exit();
fail0:
return -1;
}
/*
* sfe_exit()
*/
static void __exit sfe_exit(void)
{
sfe_exit_if();
#ifdef SFE_SUPPORT_IPV6
sfe_ipv6_exit();
#endif
sfe_ipv4_exit();
}
module_init(sfe_init)
module_exit(sfe_exit)
MODULE_AUTHOR("Qualcomm Technologies");
MODULE_DESCRIPTION("Shortcut Forwarding Engine");
MODULE_LICENSE("Dual BSD/GPL");

2961
shortcut-fe/sfe_ipv4.c
View file

File diff suppressed because it is too large Load diff

456
shortcut-fe/sfe_ipv4.h
View file

@ -1,456 +0,0 @@
/*
* sfe_ipv4.h
* Shortcut forwarding engine header file for IPv4.
*
* Copyright (c) 2013-2016, 2019-2020, The Linux Foundation. All rights reserved.
* Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#ifndef __SFE_IPV4_H
#define __SFE_IPV4_H
#define SFE_IPV4_DSCP_MASK 0x3
#define SFE_IPV4_DSCP_SHIFT 2
#include <linux/version.h>
/*
* Specifies the lower bound on ACK numbers carried in the TCP header
*/
#define SFE_IPV4_TCP_MAX_ACK_WINDOW 65520
/*
* IPv4 TCP connection match additional data.
*/
struct sfe_ipv4_tcp_connection_match {
u8 win_scale; /* Window scale */
u32 max_win; /* Maximum window size seen */
u32 end; /* Sequence number of the next byte to send (seq + segment length) */
u32 max_end; /* Sequence number of the last byte to ack */
};
/*
* Bit flags for IPv4 connection matching entry.
*/
#define SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_SRC (1<<0)
/* Perform source translation */
#define SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_DEST (1<<1)
/* Perform destination translation */
#define SFE_IPV4_CONNECTION_MATCH_FLAG_NO_SEQ_CHECK (1<<2)
/* Ignore TCP sequence numbers */
#define SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR (1<<3)
/* Fast Ethernet header write */
#define SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_L2_HDR (1<<4)
/* Fast Ethernet header write */
#define SFE_IPV4_CONNECTION_MATCH_FLAG_PRIORITY_REMARK (1<<5)
/* remark priority of SKB */
#define SFE_IPV4_CONNECTION_MATCH_FLAG_DSCP_REMARK (1<<6)
/* remark DSCP of packet */
#define SFE_IPV4_CONNECTION_MATCH_FLAG_CSUM_OFFLOAD (1<<7)
/* checksum offload.*/
#define SFE_IPV4_CONNECTION_MATCH_FLAG_PPPOE_DECAP (1<<8)
/* Indicates that PPPoE should be decapsulated */
#define SFE_IPV4_CONNECTION_MATCH_FLAG_PPPOE_ENCAP (1<<9)
/* Indicates that PPPoE should be encapsulated */
#define SFE_IPV4_CONNECTION_MATCH_FLAG_BRIDGE_FLOW (1<<10)
/* Bridge flow */
#define SFE_IPV4_CONNECTION_MATCH_FLAG_MARK (1<<11)
/* skb mark of the packet */
#define SFE_IPV4_CONNECTION_MATCH_FLAG_INSERT_EGRESS_VLAN_TAG (1<<12)
/* Insert VLAN tag */
#define SFE_IPV4_CONNECTION_MATCH_FLAG_SRC_INTERFACE_CHECK (1<<13)
/* Source interface check */
#define SFE_IPV4_CONNECTION_MATCH_FLAG_PASSTHROUGH (1<<14)
/* passthrough flow: encap/decap to be skipped for this flow */
#define SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT (1<<15)
/* skb go fast xmit */
#define SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED (1<<16)
/* Fast xmit flow checked or not */
#define SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION (1<<17)
/* Fast xmit may be possible for this flow, if SFE check passes */
#define SFE_IPV4_CONNECTION_MATCH_FLAG_SRC_INTERFACE_CHECK_NO_FLUSH (1<<18)
/* Source interface check but do not flush the connection */
/*
* IPv4 connection matching structure.
*/
struct sfe_ipv4_connection_match {
/*
* References to other objects.
*/
struct hlist_node hnode;
struct sfe_ipv4_connection *connection;
struct sfe_ipv4_connection_match *counter_match;
/* Matches the flow in the opposite direction as the one in *connection */
/*
* Characteristics that identify flows that match this rule.
*/
struct net_device *match_dev; /* Network device */
u8 match_protocol; /* Protocol */
__be32 match_src_ip; /* Source IP address */
__be32 match_dest_ip; /* Destination IP address */
__be16 match_src_port; /* Source port/connection ident */
__be16 match_dest_port; /* Destination port/connection ident */
struct udp_sock *up; /* Stores UDP sock information; valid only in decap path */
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0))
const struct net_protocol *proto; /* stores protocol handler; valid only in decap path */
#else
struct net_protocol *proto; /* stores protocol handler; valid only in decap path */
#endif
/*
* Control the operations of the match.
*/
u32 flags; /* Bit flags */
#ifdef CONFIG_NF_FLOW_COOKIE
u32 flow_cookie; /* used flow cookie, for debug */
#endif
#ifdef CONFIG_XFRM
u32 flow_accel; /* The flow accelerated or not */
#endif
/*
* Connection state that we track once we match.
*/
union { /* Protocol-specific state */
struct sfe_ipv4_tcp_connection_match tcp;
} protocol_state;
/*
* VLAN headers
*/
struct sfe_vlan_hdr ingress_vlan_hdr[SFE_MAX_VLAN_DEPTH];
struct sfe_vlan_hdr egress_vlan_hdr[SFE_MAX_VLAN_DEPTH];
/*
* Stats recorded in a sync period. These stats will be added to
* rx_packet_count64/rx_byte_count64 after a sync period.
*/
atomic_t rx_packet_count;
atomic_t rx_byte_count;
/*
* Packet translation information.
*/
__be32 xlate_src_ip; /* Address after source translation */
__be16 xlate_src_port; /* Port/connection ident after source translation */
u16 xlate_src_csum_adjustment;
/* Transport layer checksum adjustment after source translation */
u16 xlate_src_partial_csum_adjustment;
/* Transport layer pseudo header checksum adjustment after source translation */
__be32 xlate_dest_ip; /* Address after destination translation */
__be16 xlate_dest_port; /* Port/connection ident after destination translation */
u16 xlate_dest_csum_adjustment;
/* Transport layer checksum adjustment after destination translation */
u16 xlate_dest_partial_csum_adjustment;
/* Transport layer pseudo header checksum adjustment after destination translation */
/*
* QoS information
*/
u32 priority;
u32 dscp;
u32 mark; /* mark for outgoing packet */
/*
* Packet transmit information.
*/
struct net_device *xmit_dev; /* Network device on which to transmit */
unsigned short int xmit_dev_mtu;
/* Interface MTU */
u16 xmit_dest_mac[ETH_ALEN / 2];
/* Destination MAC address to use when forwarding */
u16 xmit_src_mac[ETH_ALEN / 2];
/* Source MAC address to use when forwarding */
u8 ingress_vlan_hdr_cnt; /* Ingress active vlan headers count */
u8 egress_vlan_hdr_cnt; /* Egress active vlan headers count */
/*
* Summary stats.
*/
u64 rx_packet_count64;
u64 rx_byte_count64;
/*
* PPPoE information
*/
u16 pppoe_session_id;
u8 pppoe_remote_mac[ETH_ALEN];
struct net_device *top_interface_dev; /* Used by tun6rd to store decap VLAN netdevice.*/
/*
* Size of all needed L2 headers
*/
u16 l2_hdr_size;
/*
* xmit device's feature
*/
netdev_features_t features;
bool sawf_valid; /* Indicates mark has valid SAWF information */
};
/*
* Per-connection data structure.
*/
struct sfe_ipv4_connection {
struct sfe_ipv4_connection *next;
/* Pointer to the next entry in a hash chain */
struct sfe_ipv4_connection *prev;
/* Pointer to the previous entry in a hash chain */
int protocol; /* IP protocol number */
__be32 src_ip; /* Src IP addr pre-translation */
__be32 src_ip_xlate; /* Src IP addr post-translation */
__be32 dest_ip; /* Dest IP addr pre-translation */
__be32 dest_ip_xlate; /* Dest IP addr post-translation */
__be16 src_port; /* Src port pre-translation */
__be16 src_port_xlate; /* Src port post-translation */
__be16 dest_port; /* Dest port pre-translation */
__be16 dest_port_xlate; /* Dest port post-translation */
struct sfe_ipv4_connection_match *original_match;
/* Original direction matching structure */
struct net_device *original_dev;
/* Original direction source device */
struct sfe_ipv4_connection_match *reply_match;
/* Reply direction matching structure */
struct net_device *reply_dev; /* Reply direction source device */
u64 last_sync_jiffies; /* Jiffies count for the last sync */
struct sfe_ipv4_connection *all_connections_next;
/* Pointer to the next entry in the list of all connections */
struct sfe_ipv4_connection *all_connections_prev;
/* Pointer to the previous entry in the list of all connections */
u32 debug_read_seq; /* sequence number for debug dump */
bool removed; /* Indicates the connection is removed */
struct rcu_head rcu; /* delay rcu free */
};
/*
* IPv4 connections and hash table size information.
*/
#define SFE_IPV4_CONNECTION_HASH_SHIFT 12
#define SFE_IPV4_CONNECTION_HASH_SIZE (1 << SFE_IPV4_CONNECTION_HASH_SHIFT)
#define SFE_IPV4_CONNECTION_HASH_MASK (SFE_IPV4_CONNECTION_HASH_SIZE - 1)
enum sfe_ipv4_exception_events {
SFE_IPV4_EXCEPTION_EVENT_UDP_HEADER_INCOMPLETE,
SFE_IPV4_EXCEPTION_EVENT_UDP_NO_CONNECTION,
SFE_IPV4_EXCEPTION_EVENT_UDP_IP_OPTIONS_OR_INITIAL_FRAGMENT,
SFE_IPV4_EXCEPTION_EVENT_UDP_SMALL_TTL,
SFE_IPV4_EXCEPTION_EVENT_UDP_NEEDS_FRAGMENTATION,
SFE_IPV4_EXCEPTION_EVENT_TCP_HEADER_INCOMPLETE,
SFE_IPV4_EXCEPTION_EVENT_TCP_NO_CONNECTION_SLOW_FLAGS,
SFE_IPV4_EXCEPTION_EVENT_TCP_NO_CONNECTION_FAST_FLAGS,
SFE_IPV4_EXCEPTION_EVENT_TCP_IP_OPTIONS_OR_INITIAL_FRAGMENT,
SFE_IPV4_EXCEPTION_EVENT_TCP_SMALL_TTL,
SFE_IPV4_EXCEPTION_EVENT_TCP_NEEDS_FRAGMENTATION,
SFE_IPV4_EXCEPTION_EVENT_TCP_FLAGS,
SFE_IPV4_EXCEPTION_EVENT_TCP_SEQ_EXCEEDS_RIGHT_EDGE,
SFE_IPV4_EXCEPTION_EVENT_TCP_SMALL_DATA_OFFS,
SFE_IPV4_EXCEPTION_EVENT_TCP_BAD_SACK,
SFE_IPV4_EXCEPTION_EVENT_TCP_BIG_DATA_OFFS,
SFE_IPV4_EXCEPTION_EVENT_TCP_SEQ_BEFORE_LEFT_EDGE,
SFE_IPV4_EXCEPTION_EVENT_TCP_ACK_EXCEEDS_RIGHT_EDGE,
SFE_IPV4_EXCEPTION_EVENT_TCP_ACK_BEFORE_LEFT_EDGE,
SFE_IPV4_EXCEPTION_EVENT_ICMP_HEADER_INCOMPLETE,
SFE_IPV4_EXCEPTION_EVENT_ICMP_UNHANDLED_TYPE,
SFE_IPV4_EXCEPTION_EVENT_ICMP_IPV4_HEADER_INCOMPLETE,
SFE_IPV4_EXCEPTION_EVENT_ICMP_IPV4_NON_V4,
SFE_IPV4_EXCEPTION_EVENT_ICMP_IPV4_IP_OPTIONS_INCOMPLETE,
SFE_IPV4_EXCEPTION_EVENT_ICMP_IPV4_UDP_HEADER_INCOMPLETE,
SFE_IPV4_EXCEPTION_EVENT_ICMP_IPV4_TCP_HEADER_INCOMPLETE,
SFE_IPV4_EXCEPTION_EVENT_ICMP_IPV4_UNHANDLED_PROTOCOL,
SFE_IPV4_EXCEPTION_EVENT_ICMP_NO_CONNECTION,
SFE_IPV4_EXCEPTION_EVENT_ICMP_FLUSHED_CONNECTION,
SFE_IPV4_EXCEPTION_EVENT_HEADER_INCOMPLETE,
SFE_IPV4_EXCEPTION_EVENT_HEADER_CSUM_BAD,
SFE_IPV4_EXCEPTION_EVENT_BAD_TOTAL_LENGTH,
SFE_IPV4_EXCEPTION_EVENT_NON_V4,
SFE_IPV4_EXCEPTION_EVENT_NON_INITIAL_FRAGMENT,
SFE_IPV4_EXCEPTION_EVENT_DATAGRAM_INCOMPLETE,
SFE_IPV4_EXCEPTION_EVENT_IP_OPTIONS_INCOMPLETE,
SFE_IPV4_EXCEPTION_EVENT_UNHANDLED_PROTOCOL,
SFE_IPV4_EXCEPTION_EVENT_NO_HEADROOM,
SFE_IPV4_EXCEPTION_EVENT_INVALID_PPPOE_SESSION,
SFE_IPV4_EXCEPTION_EVENT_INCORRECT_PPPOE_PARSING,
SFE_IPV4_EXCEPTION_EVENT_PPPOE_NOT_SET_IN_CME,
SFE_IPV4_EXCEPTION_EVENT_PPPOE_BR_NOT_IN_CME,
SFE_IPV4_EXCEPTION_EVENT_INGRESS_VLAN_TAG_MISMATCH,
SFE_IPV4_EXCEPTION_EVENT_INVALID_SRC_IFACE,
SFE_IPV4_EXCEPTION_EVENT_TUN6RD_NO_CONNECTION,
SFE_IPV4_EXCEPTION_EVENT_TUN6RD_NEEDS_FRAGMENTATION,
SFE_IPV4_EXCEPTION_EVENT_TUN6RD_SYNC_ON_FIND,
SFE_IPV4_EXCEPTION_EVENT_GRE_HEADER_INCOMPLETE,
SFE_IPV4_EXCEPTION_EVENT_GRE_NO_CONNECTION,
SFE_IPV4_EXCEPTION_EVENT_GRE_IP_OPTIONS_OR_INITIAL_FRAGMENT,
SFE_IPV4_EXCEPTION_EVENT_GRE_SMALL_TTL,
SFE_IPV4_EXCEPTION_EVENT_GRE_NEEDS_FRAGMENTATION,
SFE_IPV4_EXCEPTION_EVENT_ESP_NO_CONNECTION,
SFE_IPV4_EXCEPTION_EVENT_ESP_IP_OPTIONS_OR_INITIAL_FRAGMENT,
SFE_IPV4_EXCEPTION_EVENT_ESP_NEEDS_FRAGMENTATION,
SFE_IPV4_EXCEPTION_EVENT_ESP_SMALL_TTL,
SFE_IPV4_EXCEPTION_EVENT_LAST
};
/*
* per CPU stats
*/
struct sfe_ipv4_stats {
/*
* Stats recorded in a sync period. These stats will be added to
* connection_xxx64 after a sync period.
*/
u64 connection_create_requests64;
/* Number of IPv4 connection create requests */
u64 connection_create_collisions64;
/* Number of IPv4 connection create requests that collided with existing hash table entries */
u64 connection_create_failures64;
/* Number of IPv4 connection create requests that failed */
u64 connection_destroy_requests64;
/* Number of IPv4 connection destroy requests */
u64 connection_destroy_misses64;
/* Number of IPv4 connection destroy requests that missed our hash table */
u64 connection_match_hash_hits64;
/* Number of IPv4 connection match hash hits */
u64 connection_match_hash_reorders64;
/* Number of IPv4 connection match hash reorders */
u64 connection_flushes64; /* Number of IPv4 connection flushes */
u64 packets_dropped64; /* Number of IPv4 packets dropped */
u64 packets_forwarded64; /* Number of IPv4 packets forwarded */
u64 packets_fast_xmited64; /* Number of IPv4 packets fast transmited */
u64 packets_not_forwarded64; /* Number of IPv4 packets not forwarded */
u64 exception_events64[SFE_IPV4_EXCEPTION_EVENT_LAST];
u64 pppoe_encap_packets_forwarded64; /* Number of IPv4 PPPoE encap packets forwarded */
u64 pppoe_decap_packets_forwarded64; /* Number of IPv4 PPPoE decap packets forwarded */
u64 pppoe_bridge_packets_forwarded64; /* Number of IPv4 PPPoE bridge packets forwarded */
u64 pppoe_bridge_packets_3tuple_forwarded64; /* Number of IPv4 PPPoE bridge packets forwarded based on 3-tuple info */
};
/*
* sfe_ipv4_per_service_class_stats
* Per service class stats
*/
struct sfe_ipv4_per_service_class_stats {
u64 tx_bytes; /* Byte count */
u64 tx_packets; /* Packet count */
seqcount_t seq; /* seq lock for read/write protection */
/*
* TODO : add entries to be collected later.
*/
};
/*
* sfe_ipv4_service_class_stats_db
* stat entries for each service class.
*/
struct sfe_ipv4_service_class_stats_db {
struct sfe_ipv4_per_service_class_stats psc_stats[SFE_MAX_SERVICE_CLASS_ID];
/* Per service class stats */
};
/*
* Per-module structure.
*/
struct sfe_ipv4 {
spinlock_t lock; /* Lock for SMP correctness */
struct sfe_ipv4_connection *all_connections_head;
/* Head of the list of all connections */
struct sfe_ipv4_connection *all_connections_tail;
/* Tail of the list of all connections */
unsigned int num_connections; /* Number of connections */
struct delayed_work sync_dwork; /* Work to sync the statistics */
unsigned int work_cpu; /* The core to run stats sync on */
sfe_sync_rule_callback_t __rcu sync_rule_callback;
sfe_ipv4_many_sync_callback_t __rcu many_sync_callback;
/* Callback function registered by a connection manager for stats syncing */
struct sfe_ipv4_connection *conn_hash[SFE_IPV4_CONNECTION_HASH_SIZE];
/* Connection hash table */
struct hlist_head hlist_conn_match_hash_head[SFE_IPV4_CONNECTION_HASH_SIZE];
/* Connection match hash table */
#ifdef CONFIG_NF_FLOW_COOKIE
struct sfe_flow_cookie_entry sfe_flow_cookie_table[SFE_FLOW_COOKIE_SIZE];
/* flow cookie table*/
flow_cookie_set_func_t flow_cookie_set_func;
/* function used to configure flow cookie in hardware*/
int flow_cookie_enable;
/* Enable/disable flow cookie at runtime */
#endif
struct sfe_ipv4_service_class_stats_db __percpu *stats_pcpu_psc;
/* Database to maintain per cpu per service class statistics */
struct sfe_ipv4_stats __percpu *stats_pcpu;
/* Per CPU statistics. */
struct sfe_ipv4_connection *wc_next; /* Connection list walk pointer for stats sync */
/*
* Control state.
*/
struct kobject *sys_ipv4; /* sysfs linkage */
int debug_dev; /* Major number of the debug char device */
u32 debug_read_seq; /* sequence number for debug dump */
};
/*
* Enumeration of the XML output.
*/
enum sfe_ipv4_debug_xml_states {
SFE_IPV4_DEBUG_XML_STATE_START,
SFE_IPV4_DEBUG_XML_STATE_CONNECTIONS_START,
SFE_IPV4_DEBUG_XML_STATE_CONNECTIONS_CONNECTION,
SFE_IPV4_DEBUG_XML_STATE_CONNECTIONS_END,
SFE_IPV4_DEBUG_XML_STATE_EXCEPTIONS_START,
SFE_IPV4_DEBUG_XML_STATE_EXCEPTIONS_EXCEPTION,
SFE_IPV4_DEBUG_XML_STATE_EXCEPTIONS_END,
SFE_IPV4_DEBUG_XML_STATE_STATS,
SFE_IPV4_DEBUG_XML_STATE_END,
SFE_IPV4_DEBUG_XML_STATE_DONE
};
/*
* XML write state.
*/
struct sfe_ipv4_debug_xml_write_state {
enum sfe_ipv4_debug_xml_states state;
/* XML output file state machine state */
int iter_exception; /* Next exception iterator */
};
typedef bool (*sfe_ipv4_debug_xml_write_method_t)(struct sfe_ipv4 *si, char *buffer, char *msg, size_t *length,
int *total_read, struct sfe_ipv4_debug_xml_write_state *ws);
u16 sfe_ipv4_gen_ip_csum(struct iphdr *iph);
bool sfe_ipv4_service_class_stats_get(uint8_t sid, uint64_t *bytes, uint64_t *packets);
void sfe_ipv4_service_class_stats_inc(struct sfe_ipv4 *si, uint8_t sid, uint64_t bytes);
void sfe_ipv4_exception_stats_inc(struct sfe_ipv4 *si, enum sfe_ipv4_exception_events reason);
bool sfe_ipv4_remove_connection(struct sfe_ipv4 *si, struct sfe_ipv4_connection *c);
void sfe_ipv4_flush_connection(struct sfe_ipv4 *si, struct sfe_ipv4_connection *c, sfe_sync_reason_t reason);
void sfe_ipv4_sync_status(struct sfe_ipv4 *si, struct sfe_ipv4_connection *c, sfe_sync_reason_t reason);
struct sfe_ipv4_connection_match *
sfe_ipv4_find_connection_match_rcu(struct sfe_ipv4 *si, struct net_device *dev, u8 protocol,
__be32 src_ip, __be16 src_port,
__be32 dest_ip, __be16 dest_port);
void sfe_ipv4_exit(void);
int sfe_ipv4_init(void);
#endif /* __SFE_IPV4_H */

295
shortcut-fe/sfe_ipv4_esp.c
View file

@ -1,295 +0,0 @@
/*
* sfe_ipv4_esp.c
* Shortcut forwarding engine - IPv4 ESP implementation
*
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <linux/skbuff.h>
#include <net/protocol.h>
#include <net/ip.h>
#include <linux/etherdevice.h>
#include <linux/lockdep.h>
#include "sfe_debug.h"
#include "sfe_api.h"
#include "sfe.h"
#include "sfe_flow_cookie.h"
#include "sfe_ipv4.h"
#include "sfe_ipv4_esp.h"
/*
* sfe_ipv4_recv_esp()
* Handle ESP packet receives and forwarding
*/
int sfe_ipv4_recv_esp(struct sfe_ipv4 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct iphdr *iph, unsigned int ihl,
bool sync_on_find, bool tun_outer)
{
struct sfe_ipv4_connection_match *cm;
struct net_device *xmit_dev;
struct net_protocol *ipprot;
netdev_features_t features;
bool passthrough;
bool bridge_flow;
bool fast_xmit;
bool hw_csum;
__be32 src_ip;
__be32 dest_ip;
bool ret;
u8 ttl;
/*
* Read the IP address from the iphdr, and set the src/dst ports to 0.
*/
src_ip = iph->saddr;
dest_ip = iph->daddr;
rcu_read_lock();
/*
* Look for a connection match.
*/
#ifdef CONFIG_NF_FLOW_COOKIE
cm = si->sfe_flow_cookie_table[skb->flow_cookie & SFE_FLOW_COOKIE_MASK].match;
if (unlikely(!cm)) {
cm = sfe_ipv4_find_ipv4_connection_match_rcu(si, dev, IPPROTO_ESP, src_ip, 0, dest_ip, 0);
}
#else
cm = sfe_ipv4_find_connection_match_rcu(si, dev, IPPROTO_ESP, src_ip, 0, dest_ip, 0);
#endif
if (unlikely(!cm)) {
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_ESP_NO_CONNECTION);
DEBUG_TRACE("no connection found for esp packet\n");
return 0;
}
/*
* Source interface validate.
*/
if (unlikely((cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_SRC_INTERFACE_CHECK) && (cm->match_dev != dev))) {
struct sfe_ipv4_connection *c = cm->connection;
int ret;
spin_lock_bh(&si->lock);
ret = sfe_ipv4_remove_connection(si, c);
spin_unlock_bh(&si->lock);
if (ret) {
sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_INVALID_SRC_IFACE);
DEBUG_TRACE("flush on wrong source interface check failure\n");
return 0;
}
passthrough = cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_PASSTHROUGH;
bridge_flow = !!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_BRIDGE_FLOW);
/*
* If our packet has been marked as "sync on find" we can't actually
* forward it in the fast path, but now that we've found an associated
* connection we need sync its status before exception it to slow path unless
* it is passthrough (packets not directed to DUT) packet.
* TODO: revisit to ensure that pass through traffic is not bypassing firewall for fragmented cases
*/
if (unlikely(sync_on_find) && !passthrough) {
sfe_ipv4_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_ESP_IP_OPTIONS_OR_INITIAL_FRAGMENT);
DEBUG_TRACE("%px: sfe: sync on find\n", cm);
return 0;
}
/*
* Check if skb was cloned. If it was, unshare it.
*/
if (unlikely(skb_cloned(skb))) {
DEBUG_TRACE("%px: skb is a cloned skb\n", skb);
skb = skb_unshare(skb, GFP_ATOMIC);
if (!skb) {
DEBUG_WARN("Failed to unshare the cloned skb\n");
rcu_read_unlock();
return 0;
}
/*
* Update the iphdr pointer with the unshared skb's data area.
*/
iph = (struct iphdr *)skb->data;
}
/*
* Enable HW csum if rx checksum is verified and xmit interface is CSUM offload capable.
*/
hw_csum = !!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_CSUM_OFFLOAD) && (skb->ip_summed == CHECKSUM_UNNECESSARY);
/*
* proto decap packet.
* Invoke the inet_protocol handler for delivery of the packet.
*/
ipprot = rcu_dereference(cm->proto);
if (likely(ipprot)) {
skb_reset_network_header(skb);
skb_pull(skb, ihl);
skb_reset_transport_header(skb);
xmit_dev = cm->xmit_dev;
skb->dev = xmit_dev;
ret = ipprot->handler(skb);
if (ret) {
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_not_forwarded64);
DEBUG_TRACE("ESP handler returned error %u\n", ret);
return 0;
}
/*
* Update traffic stats.
*/
atomic_inc(&cm->rx_packet_count);
atomic_add(len, &cm->rx_byte_count);
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
return 1;
}
/*
* esp passthrough / ip local out scenarios.
*/
/*
* If our packet is larger than the MTU of the transmit interface then
* we can't forward it easily.
*/
if (unlikely(len > cm->xmit_dev_mtu)) {
sfe_ipv4_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_ESP_NEEDS_FRAGMENTATION);
DEBUG_TRACE("%px: sfe: larger than MTU\n", cm);
return 0;
}
/*
* need to ensure that TTL is >=2.
*/
ttl = iph->ttl;
if (!bridge_flow && (ttl < 2) && passthrough) {
sfe_ipv4_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
DEBUG_TRACE("%px: sfe: TTL too low\n", skb);
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_ESP_SMALL_TTL);
return 0;
}
/*
* decrement TTL by 1.
*/
iph->ttl = (ttl - (u8)(!bridge_flow && !tun_outer));
/*
* Update DSCP
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_DSCP_REMARK)) {
iph->tos = (iph->tos & SFE_IPV4_DSCP_MASK) | cm->dscp;
}
/*
* Replace the IP checksum.
*/
if (likely(hw_csum)) {
skb->ip_summed = CHECKSUM_PARTIAL;
} else {
iph->check = sfe_ipv4_gen_ip_csum(iph);
}
/*
* Update traffic stats.
*/
atomic_inc(&cm->rx_packet_count);
atomic_add(len, &cm->rx_byte_count);
xmit_dev = cm->xmit_dev;
skb->dev = xmit_dev;
/*
* write the layer - 2 header.
*/
if (likely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_L2_HDR)) {
if (unlikely(!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR))) {
dev_hard_header(skb, xmit_dev, ETH_P_IP, cm->xmit_dest_mac, cm->xmit_src_mac, len);
} else {
/*
* For the simple case we write this really fast.
*/
struct ethhdr *eth = (struct ethhdr *)__skb_push(skb, ETH_HLEN);
eth->h_proto = htons(ETH_P_IP);
ether_addr_copy((u8 *)eth->h_dest, (u8 *)cm->xmit_dest_mac);
ether_addr_copy((u8 *)eth->h_source, (u8 *)cm->xmit_src_mac);
}
}
/*
* Update priority of skb
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_PRIORITY_REMARK)) {
skb->priority = cm->priority;
}
/*
* Mark outgoing packet.
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_MARK)) {
skb->mark = cm->mark;
}
/*
* For the first packets, check if it could got fast xmit.
*/
if (unlikely(!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED)
&& (cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION))){
cm->features = netif_skb_features(skb);
if (likely(sfe_fast_xmit_check(skb, cm->features))) {
cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT;
}
cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED;
}
features = cm->features;
fast_xmit = !!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT);
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
prefetch(skb_shinfo(skb));
/*
* We do per packet condition check before we could fast xmit the
* packet.
*/
if (likely(fast_xmit && dev_fast_xmit(skb, xmit_dev, features))) {
this_cpu_inc(si->stats_pcpu->packets_fast_xmited64);
return 1;
}
/*
* Mark that this packet has been fast forwarded.
*/
skb->fast_forwarded = 1;
dev_queue_xmit(skb);
return 1;
}

21
shortcut-fe/sfe_ipv4_esp.h
View file

@ -1,21 +0,0 @@
/*
* sfe_ipv4_esp.h
* Shortcut forwarding engine - IPv4 ESP header file
*
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
int sfe_ipv4_recv_esp(struct sfe_ipv4 *si, struct sk_buff *skb, struct net_device *dev, unsigned int len,
struct iphdr *iph, unsigned int ihl, bool sync_on_find, bool tun_outer);

390
shortcut-fe/sfe_ipv4_gre.c
View file

@ -1,390 +0,0 @@
/*
* sfe_ipv4_gre.c
* Shortcut forwarding engine file for IPv4 GRE
*
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <linux/skbuff.h>
#include <net/protocol.h>
#include <linux/etherdevice.h>
#include <linux/lockdep.h>
#include <net/gre.h>
#include <net/pptp.h>
#include "sfe_debug.h"
#include "sfe_api.h"
#include "sfe.h"
#include "sfe_flow_cookie.h"
#include "sfe_ipv4.h"
#include "sfe_pppoe.h"
#include "sfe_vlan.h"
/*
* sfe_ipv4_recv_gre()
* GRE tunnel packet receive and forwarding.
*/
int sfe_ipv4_recv_gre(struct sfe_ipv4 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct iphdr *iph, unsigned int ihl, bool sync_on_find,
struct sfe_l2_info *l2_info, bool tun_outer)
{
struct sfe_ipv4_connection_match *cm;
struct pptp_gre_header *pptp_hdr;
struct gre_base_hdr *gre_hdr;
struct net_device *xmit_dev;
__be16 dest_port = 0;
bool passthrough;
bool bridge_flow;
__be32 dest_ip;
__be32 src_ip;
bool hw_csum;
bool ret;
u8 ttl;
/*
* Is our packet too short to contain a valid GRE header?
*/
if (unlikely(!pskb_may_pull(skb, sizeof(*gre_hdr) + ihl))) {
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_GRE_HEADER_INCOMPLETE);
DEBUG_TRACE("packet too short for GRE header\n");
return 0;
}
/*
* Read the source and destination IP address.
*/
src_ip = iph->saddr;
dest_ip = iph->daddr;
rcu_read_lock();
/*
* Look for a connection match with 4 tuple if it is PPTP
*/
gre_hdr = (struct gre_base_hdr *)(skb->data + ihl);
if ((gre_hdr->protocol == GRE_PROTO_PPP) && likely(pskb_may_pull(skb, (sizeof(*pptp_hdr) - 8) + ihl))) {
pptp_hdr = (struct pptp_gre_header *)(skb->data + ihl);
dest_port = pptp_hdr->call_id;
}
#ifdef CONFIG_NF_FLOW_COOKIE
cm = si->sfe_flow_cookie_table[skb->flow_cookie & SFE_FLOW_COOKIE_MASK].match;
if (unlikely(!cm)) {
cm = sfe_ipv4_find_connection_match_rcu(si, dev, IPPROTO_GRE, src_ip, 0, dest_ip, dest_port);
}
#else
cm = sfe_ipv4_find_connection_match_rcu(si, dev, IPPROTO_GRE, src_ip, 0, dest_ip, dest_port);
#endif
if (unlikely(!cm)) {
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_GRE_NO_CONNECTION);
DEBUG_INFO("no GRE connection match found dev %s src ip %pI4 dest ip %pI4 port %d\n", dev->name, &src_ip, &dest_ip, ntohs(dest_port));
return 0;
}
/*
* Source interface validate.
*/
if (unlikely((cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_SRC_INTERFACE_CHECK) && (cm->match_dev != dev))) {
if (!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_SRC_INTERFACE_CHECK_NO_FLUSH)) {
struct sfe_ipv4_connection *c = cm->connection;
int ret;
DEBUG_TRACE("flush on source interface check failure\n");
spin_lock_bh(&si->lock);
ret = sfe_ipv4_remove_connection(si, c);
spin_unlock_bh(&si->lock);
if (ret) {
sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
}
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_INVALID_SRC_IFACE);
DEBUG_TRACE("exception the packet on source interface check failure\n");
return 0;
}
passthrough = cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_PASSTHROUGH;
/*
* If our packet has been marked as "sync on find" we can't actually
* forward it in the fast path, but now that we've found an associated
* connection we need sync its status before exception it to slow path unless
* it is passthrough (packets not directed to DUT) packet.
* TODO: revisit to ensure that pass through traffic is not bypassing firewall for fragmented cases
*/
if (unlikely(sync_on_find) && !passthrough) {
sfe_ipv4_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_GRE_IP_OPTIONS_OR_INITIAL_FRAGMENT);
DEBUG_TRACE("%px: sfe: sync on find\n", cm);
return 0;
}
/*
* Do we expect an ingress VLAN tag for this flow?
*/
if (unlikely(!sfe_vlan_validate_ingress_tag(skb, cm->ingress_vlan_hdr_cnt, cm->ingress_vlan_hdr, l2_info))) {
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_INGRESS_VLAN_TAG_MISMATCH);
DEBUG_TRACE("VLAN tag mismatch. skb=%px\n", skb);
return 0;
}
bridge_flow = !!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_BRIDGE_FLOW);
/*
* Does our TTL allow forwarding?
*/
ttl = iph->ttl;
if (!bridge_flow && (ttl < 2) && passthrough) {
sfe_ipv4_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
DEBUG_TRACE("%px: sfe: TTL too low\n", skb);
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_GRE_SMALL_TTL);
return 0;
}
/*
* From this point on we're good to modify the packet.
*/
/*
* Check if skb was cloned. If it was, unshare it. Because
* the data area is going to be written in this path and we don't want to
* change the cloned skb's data section.
*/
if (unlikely(skb_cloned(skb))) {
DEBUG_TRACE("%px: skb is a cloned skb\n", skb);
skb = skb_unshare(skb, GFP_ATOMIC);
if (!skb) {
DEBUG_WARN("Failed to unshare the cloned skb\n");
rcu_read_unlock();
return 1;
}
/*
* Update the iph and udph pointers with the unshared skb's data area.
*/
iph = (struct iphdr *)skb->data;
}
/*
* For PPPoE packets, match server MAC and session id
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_PPPOE_DECAP)) {
struct ethhdr *eth;
bool pppoe_match;
if (unlikely(!sfe_l2_parse_flag_check(l2_info, SFE_L2_PARSE_FLAGS_PPPOE_INGRESS))) {
rcu_read_unlock();
DEBUG_TRACE("%px: PPPoE header not present in packet for PPPoE rule\n", skb);
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_INCORRECT_PPPOE_PARSING);
return 0;
}
eth = eth_hdr(skb);
pppoe_match = (cm->pppoe_session_id == sfe_l2_pppoe_session_id_get(l2_info)) &&
ether_addr_equal((u8*)cm->pppoe_remote_mac, (u8 *)eth->h_source);
if (unlikely(!pppoe_match)) {
DEBUG_TRACE("%px: PPPoE session ID %d and %d or MAC %pM and %pM did not match\n",
skb, cm->pppoe_session_id, sfe_l2_pppoe_session_id_get(l2_info),
cm->pppoe_remote_mac, eth->h_source);
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_INVALID_PPPOE_SESSION);
return 0;
}
skb->protocol = htons(l2_info->protocol);
this_cpu_inc(si->stats_pcpu->pppoe_decap_packets_forwarded64);
} else if (unlikely(sfe_l2_parse_flag_check(l2_info, SFE_L2_PARSE_FLAGS_PPPOE_INGRESS))) {
/*
* If packet contains PPPoE header but CME doesn't contain PPPoE flag yet we are exceptioning
* the packet to linux
*/
if (unlikely(!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_BRIDGE_FLOW))) {
rcu_read_unlock();
DEBUG_TRACE("%px: CME doesn't contain PPPoE flag but packet has PPPoE header\n", skb);
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_PPPOE_NOT_SET_IN_CME);
return 0;
}
/*
* For bridged flows when packet contains PPPoE header, restore the header back and forward
* to xmit interface
*/
__skb_push(skb, (sizeof(struct pppoe_hdr) + sizeof(struct sfe_ppp_hdr)));
this_cpu_inc(si->stats_pcpu->pppoe_bridge_packets_forwarded64);
}
/*
* protocol handler will be valid only in decap-path.
*/
if (cm->proto) {
struct net_protocol *ipprot = cm->proto;
skb_reset_network_header(skb);
skb_pull(skb, ihl);
skb_reset_transport_header(skb);
skb->fast_forwarded = 1;
ret = ipprot->handler(skb);
if (ret) {
this_cpu_inc(si->stats_pcpu->packets_not_forwarded64);
rcu_read_unlock();
DEBUG_TRACE("GRE handler returned error %u\n", ret);
return 1;
}
/*
* Update traffic stats
*/
atomic_inc(&cm->rx_packet_count);
atomic_add(len, &cm->rx_byte_count);
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
rcu_read_unlock();
return 1;
}
/*
* Check if skb has enough headroom to write L2 headers
*/
if (unlikely(skb_headroom(skb) < cm->l2_hdr_size)) {
rcu_read_unlock();
DEBUG_WARN("%px: Not enough headroom: %u\n", skb, skb_headroom(skb));
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_NO_HEADROOM);
return 0;
}
/*
* If our packet is larger than the MTU of the transmit interface then
* we can't forward it easily.
*/
if (unlikely(len > cm->xmit_dev_mtu)) {
sfe_ipv4_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_GRE_NEEDS_FRAGMENTATION);
DEBUG_TRACE("%px: sfe: larger than MTU\n", cm);
return 0;
}
/*
* Decrement our TTL
*/
iph->ttl = (ttl - (u8)(!bridge_flow && !tun_outer));
/*
* Update DSCP
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_DSCP_REMARK)) {
iph->tos = (iph->tos & SFE_IPV4_DSCP_MASK) | cm->dscp;
}
/*
* Enable HW csum if rx checksum is verified and xmit interface is CSUM offload capable.
*/
hw_csum = !!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_CSUM_OFFLOAD) && (skb->ip_summed == CHECKSUM_UNNECESSARY);
/*
* Replace the IP checksum.
*/
if (likely(hw_csum)) {
skb->ip_summed = CHECKSUM_PARTIAL;
} else {
iph->check = sfe_ipv4_gen_ip_csum(iph);
}
/*
* Update traffic stats
*/
atomic_inc(&cm->rx_packet_count);
atomic_add(len, &cm->rx_byte_count);
xmit_dev = cm->xmit_dev;
skb->dev = xmit_dev;
/*
* For PPPoE flows, add PPPoE header before L2 header is added.
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_PPPOE_ENCAP)) {
sfe_pppoe_add_header(skb, cm->pppoe_session_id, PPP_IP);
this_cpu_inc(si->stats_pcpu->pppoe_encap_packets_forwarded64);
}
/*
* Check to see if we need to add VLAN tags
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_INSERT_EGRESS_VLAN_TAG)) {
sfe_vlan_add_tag(skb, cm->egress_vlan_hdr_cnt, cm->egress_vlan_hdr);
}
/*
* For the simple case we write this really fast.
*/
if (cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR) {
struct ethhdr *eth = (struct ethhdr *)__skb_push(skb, ETH_HLEN);
eth->h_proto = skb->protocol;
ether_addr_copy((u8 *)eth->h_dest, (u8 *)cm->xmit_dest_mac);
ether_addr_copy((u8 *)eth->h_source, (u8 *)cm->xmit_src_mac);
} else if (cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_L2_HDR) {
dev_hard_header(skb, xmit_dev, ntohs(skb->protocol), cm->xmit_dest_mac, cm->xmit_src_mac, len);
}
/*
* Update priority of skb.
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_PRIORITY_REMARK)) {
skb->priority = cm->priority;
}
/*
* Mark outgoing packet.
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_MARK)) {
skb->mark = cm->mark;
}
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
rcu_read_unlock();
/*
* We're going to check for GSO flags when we transmit the packet so
* start fetching the necessary cache line now.
*/
prefetch(skb_shinfo(skb));
/*
* Mark that this packet has been fast forwarded.
*/
skb->fast_forwarded = 1;
/*
* Send the packet on its way.
*/
dev_queue_xmit(skb);
return 1;
}

21
shortcut-fe/sfe_ipv4_gre.h
View file

@ -1,21 +0,0 @@
/*
* sfe_ipv4_gre.h
* Shortcut forwarding engine - IPv4 GRE header file
*
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
int sfe_ipv4_recv_gre(struct sfe_ipv4 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct iphdr *iph, unsigned int ihl, bool sync_on_find,
struct sfe_l2_info *l2_info, bool tun_outer);

206
shortcut-fe/sfe_ipv4_icmp.c
View file

@ -1,206 +0,0 @@
/*
* sfe_ipv4_icmp.c
* Shortcut forwarding engine - IPv4 ICMP implementation
*
* Copyright (c) 2013-2016, 2019-2020, The Linux Foundation. All rights reserved.
* Copyright (c) 2021 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <linux/skbuff.h>
#include <linux/ip.h>
#include <net/udp.h>
#include <net/tcp.h>
#include <net/icmp.h>
#include <linux/etherdevice.h>
#include <linux/lockdep.h>
#include "sfe_debug.h"
#include "sfe_api.h"
#include "sfe.h"
#include "sfe_flow_cookie.h"
#include "sfe_ipv4.h"
/*
* sfe_ipv4_recv_icmp()
* Handle ICMP packet receives.
*
* ICMP packets aren't handled as a "fast path" and always have us process them
* through the default Linux stack. What we do need to do is look for any errors
* about connections we are handling in the fast path. If we find any such
* connections then we want to flush their state so that the ICMP error path
* within Linux has all of the correct state should it need it.
*/
int sfe_ipv4_recv_icmp(struct sfe_ipv4 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct iphdr *iph, unsigned int ihl)
{
struct icmphdr *icmph;
struct iphdr *icmp_iph;
unsigned int icmp_ihl_words;
unsigned int icmp_ihl;
u32 *icmp_trans_h;
struct udphdr *icmp_udph;
struct tcphdr *icmp_tcph;
__be32 src_ip;
__be32 dest_ip;
__be16 src_port;
__be16 dest_port;
struct sfe_ipv4_connection_match *cm;
struct sfe_ipv4_connection *c;
u32 pull_len = sizeof(struct icmphdr) + ihl;
bool ret;
/*
* Is our packet too short to contain a valid ICMP header?
*/
len -= ihl;
if (!pskb_may_pull(skb, pull_len)) {
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_ICMP_HEADER_INCOMPLETE);
DEBUG_TRACE("packet too short for ICMP header\n");
return 0;
}
/*
* We only handle "destination unreachable" and "time exceeded" messages.
*/
icmph = (struct icmphdr *)(skb->data + ihl);
if ((icmph->type != ICMP_DEST_UNREACH)
&& (icmph->type != ICMP_TIME_EXCEEDED)) {
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_ICMP_UNHANDLED_TYPE);
DEBUG_TRACE("unhandled ICMP type: 0x%x\n", icmph->type);
return 0;
}
/*
* Do we have the full embedded IP header?
*/
len -= sizeof(struct icmphdr);
pull_len += sizeof(struct iphdr);
if (!pskb_may_pull(skb, pull_len)) {
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_ICMP_IPV4_HEADER_INCOMPLETE);
DEBUG_TRACE("Embedded IP header not complete\n");
return 0;
}
/*
* Is our embedded IP version wrong?
*/
icmp_iph = (struct iphdr *)(icmph + 1);
if (unlikely(icmp_iph->version != 4)) {
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_ICMP_IPV4_NON_V4);
DEBUG_TRACE("IP version: %u\n", icmp_iph->version);
return 0;
}
/*
* Do we have the full embedded IP header, including any options?
*/
icmp_ihl_words = icmp_iph->ihl;
icmp_ihl = icmp_ihl_words << 2;
pull_len += icmp_ihl - sizeof(struct iphdr);
if (!pskb_may_pull(skb, pull_len)) {
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_ICMP_IPV4_IP_OPTIONS_INCOMPLETE);
DEBUG_TRACE("Embedded header not large enough for IP options\n");
return 0;
}
len -= icmp_ihl;
icmp_trans_h = ((u32 *)icmp_iph) + icmp_ihl_words;
/*
* Handle the embedded transport layer header.
*/
switch (icmp_iph->protocol) {
case IPPROTO_UDP:
/*
* We should have 8 bytes of UDP header - that's enough to identify
* the connection.
*/
pull_len += 8;
if (!pskb_may_pull(skb, pull_len)) {
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_ICMP_IPV4_UDP_HEADER_INCOMPLETE);
DEBUG_TRACE("Incomplete embedded UDP header\n");
return 0;
}
icmp_udph = (struct udphdr *)icmp_trans_h;
src_port = icmp_udph->source;
dest_port = icmp_udph->dest;
break;
case IPPROTO_TCP:
/*
* We should have 8 bytes of TCP header - that's enough to identify
* the connection.
*/
pull_len += 8;
if (!pskb_may_pull(skb, pull_len)) {
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_ICMP_IPV4_TCP_HEADER_INCOMPLETE);
DEBUG_TRACE("Incomplete embedded TCP header\n");
return 0;
}
icmp_tcph = (struct tcphdr *)icmp_trans_h;
src_port = icmp_tcph->source;
dest_port = icmp_tcph->dest;
break;
default:
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_ICMP_IPV4_UNHANDLED_PROTOCOL);
DEBUG_TRACE("Unhandled embedded IP protocol: %u\n", icmp_iph->protocol);
return 0;
}
src_ip = icmp_iph->saddr;
dest_ip = icmp_iph->daddr;
rcu_read_lock();
/*
* Look for a connection match. Note that we reverse the source and destination
* here because our embedded message contains a packet that was sent in the
* opposite direction to the one in which we just received it. It will have
* been sent on the interface from which we received it though so that's still
* ok to use.
*/
cm = sfe_ipv4_find_connection_match_rcu(si, dev, icmp_iph->protocol, dest_ip, dest_port, src_ip, src_port);
if (unlikely(!cm)) {
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_ICMP_NO_CONNECTION);
DEBUG_TRACE("no connection found\n");
return 0;
}
/*
* We found a connection so now remove it from the connection list and flush
* its state.
*/
c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv4_remove_connection(si, c);
spin_unlock_bh(&si->lock);
if (ret) {
sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_ICMP_FLUSHED_CONNECTION);
return 0;
}

22
shortcut-fe/sfe_ipv4_icmp.h
View file

@ -1,22 +0,0 @@
/*
* sfe_ipv4_icmp.h
* Shortcut forwarding engine - IPv4 ICMP header file
*
* Copyright (c) 2013-2016, 2019-2020, The Linux Foundation. All rights reserved.
* Copyright (c) 2021 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
int sfe_ipv4_recv_icmp(struct sfe_ipv4 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct iphdr *iph, unsigned int ihl);

204
shortcut-fe/sfe_ipv4_pppoe_br.c
View file

@ -1,204 +0,0 @@
/*
* sfe_ipv4_pppoe_br.c
* Shortcut forwarding engine - IPv4 PPPoE bridge implementation
*
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <linux/skbuff.h>
#include <net/udp.h>
#include <linux/etherdevice.h>
#include <linux/version.h>
#include "sfe_debug.h"
#include "sfe_api.h"
#include "sfe.h"
#include "sfe_ipv4.h"
#include "sfe_pppoe.h"
#include "sfe_vlan.h"
/*
* sfe_ipv4_recv_pppoe_bridge()
* Process PPPoE bridge packets using 3-tuple acceleration
*
*/
int sfe_ipv4_recv_pppoe_bridge(struct sfe_ipv4 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct iphdr *iph, unsigned int ihl, struct sfe_l2_info *l2_info)
{
struct sfe_ipv4_connection_match *cm;
u32 service_class_id;
struct net_device *xmit_dev;
int ret;
bool fast_xmit;
netdev_features_t features;
rcu_read_lock();
cm = sfe_ipv4_find_connection_match_rcu(si, dev, IPPROTO_RAW, iph->saddr, 0, iph->daddr, htons(sfe_l2_pppoe_session_id_get(l2_info)));
if (unlikely(!cm)) {
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_PPPOE_BR_NOT_IN_CME);
DEBUG_TRACE("%px: no connection found in 3-tuple lookup for PPPoE bridge flow\n", skb);
return 0;
}
/*
* Source interface validate.
*/
if (unlikely((cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_SRC_INTERFACE_CHECK) && (cm->match_dev != dev))) {
if (!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_SRC_INTERFACE_CHECK_NO_FLUSH)) {
struct sfe_ipv4_connection *c = cm->connection;
DEBUG_TRACE("flush on source interface check failure\n");
spin_lock_bh(&si->lock);
ret = sfe_ipv4_remove_connection(si, c);
spin_unlock_bh(&si->lock);
if (ret) {
sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
}
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_INVALID_SRC_IFACE);
DEBUG_TRACE("exception the packet on source interface check failure\n");
return 0;
}
/*
* Do we expect an ingress VLAN tag for this flow?
*/
if (unlikely(!sfe_vlan_validate_ingress_tag(skb, cm->ingress_vlan_hdr_cnt, cm->ingress_vlan_hdr, l2_info))) {
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_INGRESS_VLAN_TAG_MISMATCH);
DEBUG_TRACE("VLAN tag mismatch. skb=%px\n", skb);
return 0;
}
/*
* Check if skb has enough headroom to write L2 headers
*/
if (unlikely(skb_headroom(skb) < cm->l2_hdr_size)) {
rcu_read_unlock();
DEBUG_WARN("%px: Not enough headroom: %u\n", skb, skb_headroom(skb));
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_NO_HEADROOM);
return 0;
}
/*
* Restore PPPoE header back
*/
__skb_push(skb, PPPOE_SES_HLEN);
/*
* Update traffic stats.
*/
atomic_inc(&cm->rx_packet_count);
atomic_add(len, &cm->rx_byte_count);
xmit_dev = cm->xmit_dev;
skb->dev = xmit_dev;
/*
* Check to see if we need to add VLAN tags
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_INSERT_EGRESS_VLAN_TAG)) {
sfe_vlan_add_tag(skb, cm->egress_vlan_hdr_cnt, cm->egress_vlan_hdr);
}
/*
* Check to see if we need to write an Ethernet header.
*/
if (likely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_L2_HDR)) {
if (unlikely(!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR))) {
dev_hard_header(skb, xmit_dev, ntohs(skb->protocol),
cm->xmit_dest_mac, cm->xmit_src_mac, len);
} else {
/*
* For the simple case we write this really fast.
*/
struct ethhdr *eth = (struct ethhdr *)__skb_push(skb, ETH_HLEN);
eth->h_proto = skb->protocol;
ether_addr_copy((u8 *)eth->h_dest, (u8 *)cm->xmit_dest_mac);
ether_addr_copy((u8 *)eth->h_source, (u8 *)cm->xmit_src_mac);
}
}
/*
* Update priority of skb.
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_PRIORITY_REMARK)) {
skb->priority = cm->priority;
}
/*
* Mark outgoing packet.
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_MARK)) {
skb->mark = cm->mark;
/*
* Update service class stats if SAWF is valid.
*/
if (likely(cm->sawf_valid)) {
service_class_id = SFE_GET_SAWF_SERVICE_CLASS(cm->mark);
sfe_ipv4_service_class_stats_inc(si, service_class_id, len);
}
}
/*
* For the first packets, check if it could got fast xmit.
*/
if (unlikely(!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED)
&& (cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION))){
cm->features = netif_skb_features(skb);
if (likely(sfe_fast_xmit_check(skb, cm->features))) {
cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT;
}
cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED;
}
features = cm->features;
fast_xmit = !!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT);
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->pppoe_bridge_packets_3tuple_forwarded64);
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
/*
* We're going to check for GSO flags when we transmit the packet so
* start fetching the necessary cache line now.
*/
prefetch(skb_shinfo(skb));
/*
* We do per packet condition check before we could fast xmit the
* packet.
*/
if (likely(fast_xmit && dev_fast_xmit(skb, xmit_dev, features))) {
this_cpu_inc(si->stats_pcpu->packets_fast_xmited64);
return 1;
}
/*
* Mark that this packet has been fast forwarded.
*/
skb->fast_forwarded = 1;
/*
* Send the packet on its way.
*/
dev_queue_xmit(skb);
return 1;
}

20
shortcut-fe/sfe_ipv4_pppoe_br.h
View file

@ -1,20 +0,0 @@
/*
* sfe_ipv4_pppoe_br.h
* Shortcut forwarding engine - IPv4 PPPoE bridge header file
*
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
int sfe_ipv4_recv_pppoe_bridge(struct sfe_ipv4 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct iphdr *iph, unsigned int ihl, struct sfe_l2_info *l2_info);

765
shortcut-fe/sfe_ipv4_tcp.c
View file

@ -1,765 +0,0 @@
/*
* sfe_ipv4_tcp.c
* Shortcut forwarding engine - IPv4 TCP implementation
*
* Copyright (c) 2013-2016, 2019-2020, The Linux Foundation. All rights reserved.
* Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <linux/skbuff.h>
#include <net/tcp.h>
#include <linux/etherdevice.h>
#include <linux/lockdep.h>
#include "sfe_debug.h"
#include "sfe_api.h"
#include "sfe.h"
#include "sfe_flow_cookie.h"
#include "sfe_ipv4.h"
#include "sfe_pppoe.h"
#include "sfe_vlan.h"
/*
* sfe_ipv4_process_tcp_option_sack()
* Parse TCP SACK option and update ack according
*/
static bool sfe_ipv4_process_tcp_option_sack(const struct tcphdr *th, const u32 data_offs,
u32 *ack)
{
u32 length = sizeof(struct tcphdr);
u8 *ptr = (u8 *)th + length;
/*
* Ignore processing if TCP packet has only TIMESTAMP option.
*/
if (likely(data_offs == length + TCPOLEN_TIMESTAMP + 1 + 1)
&& likely(ptr[0] == TCPOPT_NOP)
&& likely(ptr[1] == TCPOPT_NOP)
&& likely(ptr[2] == TCPOPT_TIMESTAMP)
&& likely(ptr[3] == TCPOLEN_TIMESTAMP)) {
return true;
}
/*
* TCP options. Parse SACK option.
*/
while (length < data_offs) {
u8 size;
u8 kind;
ptr = (u8 *)th + length;
kind = *ptr;
/*
* NOP, for padding
* Not in the switch because to fast escape and to not calculate size
*/
if (kind == TCPOPT_NOP) {
length++;
continue;
}
if (kind == TCPOPT_SACK) {
u32 sack = 0;
u8 re = 1 + 1;
size = *(ptr + 1);
if ((size < (1 + 1 + TCPOLEN_SACK_PERBLOCK))
|| ((size - (1 + 1)) % (TCPOLEN_SACK_PERBLOCK))
|| (size > (data_offs - length))) {
return false;
}
re += 4;
while (re < size) {
u32 sack_re;
u8 *sptr = ptr + re;
sack_re = (sptr[0] << 24) | (sptr[1] << 16) | (sptr[2] << 8) | sptr[3];
if (sack_re > sack) {
sack = sack_re;
}
re += TCPOLEN_SACK_PERBLOCK;
}
if (sack > *ack) {
*ack = sack;
}
length += size;
continue;
}
if (kind == TCPOPT_EOL) {
return true;
}
size = *(ptr + 1);
if (size < 2) {
return false;
}
length += size;
}
return true;
}
/*
* sfe_ipv4_recv_tcp()
* Handle TCP packet receives and forwarding.
*/
int sfe_ipv4_recv_tcp(struct sfe_ipv4 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct iphdr *iph, unsigned int ihl, bool sync_on_find, struct sfe_l2_info *l2_info)
{
struct tcphdr *tcph;
__be32 src_ip;
__be32 dest_ip;
__be16 src_port;
__be16 dest_port;
struct sfe_ipv4_connection_match *cm;
struct sfe_ipv4_connection_match *counter_cm;
u8 ttl;
u32 flags;
u32 service_class_id;
struct net_device *xmit_dev;
bool ret;
bool hw_csum;
bool bridge_flow;
bool fast_xmit;
netdev_features_t features;
/*
* Is our packet too short to contain a valid TCP header?
*/
if (unlikely(!pskb_may_pull(skb, (sizeof(struct tcphdr) + ihl)))) {
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_HEADER_INCOMPLETE);
DEBUG_TRACE("packet too short for TCP header\n");
return 0;
}
/*
* Read the IP address and port information. Read the IP header data first
* because we've almost certainly got that in the cache. We may not yet have
* the TCP header cached though so allow more time for any prefetching.
*/
src_ip = iph->saddr;
dest_ip = iph->daddr;
tcph = (struct tcphdr *)(skb->data + ihl);
src_port = tcph->source;
dest_port = tcph->dest;
flags = tcp_flag_word(tcph);
rcu_read_lock();
/*
* Look for a connection match.
*/
#ifdef CONFIG_NF_FLOW_COOKIE
cm = si->sfe_flow_cookie_table[skb->flow_cookie & SFE_FLOW_COOKIE_MASK].match;
if (unlikely(!cm)) {
cm = sfe_ipv4_find_connection_match_rcu(si, dev, IPPROTO_TCP, src_ip, src_port, dest_ip, dest_port);
}
#else
/*
* 5-tuple lookup for TCP flow.
*/
cm = sfe_ipv4_find_connection_match_rcu(si, dev, IPPROTO_TCP, src_ip, src_port, dest_ip, dest_port);
#endif
if (unlikely(!cm)) {
/*
* We didn't get a connection but as TCP is connection-oriented that
* may be because this is a non-fast connection (not running established).
* For diagnostic purposes we differentiate this here.
*/
if (likely((flags & (TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_FIN | TCP_FLAG_ACK)) == TCP_FLAG_ACK)) {
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_NO_CONNECTION_FAST_FLAGS);
DEBUG_TRACE("no connection found - fast flags\n");
return 0;
}
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_NO_CONNECTION_SLOW_FLAGS);
DEBUG_TRACE("no connection found - slow flags: 0x%x\n",
flags & (TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_FIN | TCP_FLAG_ACK));
return 0;
}
/*
* Source interface validate.
*/
if (unlikely((cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_SRC_INTERFACE_CHECK) && (cm->match_dev != dev))) {
if (!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_SRC_INTERFACE_CHECK_NO_FLUSH)) {
struct sfe_ipv4_connection *c = cm->connection;
DEBUG_TRACE("flush on source interface check failure\n");
spin_lock_bh(&si->lock);
ret = sfe_ipv4_remove_connection(si, c);
spin_unlock_bh(&si->lock);
if (ret) {
sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
}
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_INVALID_SRC_IFACE);
DEBUG_TRACE("exception the packet on source interface check failure\n");
return 0;
}
/*
* If our packet has been marked as "sync on find" we can't actually
* forward it in the fast path, but now that we've found an associated
* connection we need sync its status before throw it slow path.
*/
if (unlikely(sync_on_find)) {
sfe_ipv4_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_IP_OPTIONS_OR_INITIAL_FRAGMENT);
DEBUG_TRACE("Sync on find\n");
return 0;
}
#ifdef CONFIG_XFRM
/*
* We can't accelerate the flow on this direction, just let it go
* through the slow path.
*/
if (unlikely(!cm->flow_accel)) {
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_not_forwarded64);
return 0;
}
#endif
/*
* Do we expect an ingress VLAN tag for this flow?
*/
if (unlikely(!sfe_vlan_validate_ingress_tag(skb, cm->ingress_vlan_hdr_cnt, cm->ingress_vlan_hdr, l2_info))) {
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_INGRESS_VLAN_TAG_MISMATCH);
DEBUG_TRACE("VLAN tag mismatch. skb=%px\n", skb);
return 0;
}
bridge_flow = !!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_BRIDGE_FLOW);
/*
* Does our TTL allow forwarding?
*/
if (likely(!bridge_flow)) {
ttl = iph->ttl;
if (unlikely(ttl < 2)) {
sfe_ipv4_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_SMALL_TTL);
DEBUG_TRACE("TTL too low\n");
return 0;
}
}
/*
* If our packet is larger than the MTU of the transmit interface then
* we can't forward it easily.
*/
if (unlikely((len > cm->xmit_dev_mtu) && !skb_is_gso(skb))) {
sfe_ipv4_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_IP_OPTIONS_OR_INITIAL_FRAGMENT);
DEBUG_TRACE("Larger than MTU\n");
return 0;
}
/*
* Look at our TCP flags. Anything missing an ACK or that has RST, SYN or FIN
* set is not a fast path packet.
*/
if (unlikely((flags & (TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_FIN | TCP_FLAG_ACK)) != TCP_FLAG_ACK)) {
struct sfe_ipv4_connection *c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv4_remove_connection(si, c);
spin_unlock_bh(&si->lock);
DEBUG_TRACE("TCP flags: %#x are not fast. %u->%u\n",
htonl(flags), htons(src_port), htons(dest_port));
if (ret) {
sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_FLAGS);
return 0;
}
counter_cm = cm->counter_match;
/*
* Are we doing sequence number checking?
*/
if (likely(!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_NO_SEQ_CHECK))) {
u32 seq;
u32 ack;
u32 sack;
u32 data_offs;
u32 end;
u32 left_edge;
u32 scaled_win;
u32 max_end;
/*
* Is our sequence fully past the right hand edge of the window?
*/
seq = ntohl(tcph->seq);
if (unlikely((s32)(seq - (cm->protocol_state.tcp.max_end + 1)) > 0)) {
struct sfe_ipv4_connection *c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv4_remove_connection(si, c);
spin_unlock_bh(&si->lock);
DEBUG_TRACE("seq: %u exceeds right edge: %u\n",
seq, cm->protocol_state.tcp.max_end + 1);
if (ret) {
sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_SEQ_EXCEEDS_RIGHT_EDGE);
return 0;
}
/*
* Check that our TCP data offset isn't too short.
*/
data_offs = tcph->doff << 2;
if (unlikely(data_offs < sizeof(struct tcphdr))) {
struct sfe_ipv4_connection *c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv4_remove_connection(si, c);
spin_unlock_bh(&si->lock);
DEBUG_TRACE("TCP data offset: %u, too small\n", data_offs);
if (ret) {
sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_SMALL_DATA_OFFS);
return 0;
}
/*
* Update ACK according to any SACK option.
*/
ack = ntohl(tcph->ack_seq);
sack = ack;
if (unlikely(!sfe_ipv4_process_tcp_option_sack(tcph, data_offs, &sack))) {
struct sfe_ipv4_connection *c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv4_remove_connection(si, c);
spin_unlock_bh(&si->lock);
DEBUG_TRACE("TCP option SACK size is wrong\n");
if (ret) {
sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_BAD_SACK);
return 0;
}
/*
* Check that our TCP data offset isn't past the end of the packet.
*/
data_offs += sizeof(struct iphdr);
if (unlikely(len < data_offs)) {
struct sfe_ipv4_connection *c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv4_remove_connection(si, c);
spin_unlock_bh(&si->lock);
DEBUG_TRACE("TCP data offset: %u, past end of packet: %u\n",
data_offs, len);
if (ret) {
sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_BIG_DATA_OFFS);
return 0;
}
end = seq + len - data_offs;
/*
* Is our sequence fully before the left hand edge of the window?
*/
if (unlikely((s32)(end - (cm->protocol_state.tcp.end
- counter_cm->protocol_state.tcp.max_win - 1)) < 0)) {
struct sfe_ipv4_connection *c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv4_remove_connection(si, c);
spin_unlock_bh(&si->lock);
DEBUG_TRACE("seq: %u before left edge: %u\n",
end, cm->protocol_state.tcp.end - counter_cm->protocol_state.tcp.max_win - 1);
if (ret) {
sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_SEQ_BEFORE_LEFT_EDGE);
return 0;
}
/*
* Are we acking data that is to the right of what has been sent?
*/
if (unlikely((s32)(sack - (counter_cm->protocol_state.tcp.end + 1)) > 0)) {
struct sfe_ipv4_connection *c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv4_remove_connection(si, c);
spin_unlock_bh(&si->lock);
DEBUG_TRACE("ack: %u exceeds right edge: %u\n",
sack, counter_cm->protocol_state.tcp.end + 1);
if (ret) {
sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_ACK_EXCEEDS_RIGHT_EDGE);
return 0;
}
/*
* Is our ack too far before the left hand edge of the window?
*/
left_edge = counter_cm->protocol_state.tcp.end
- cm->protocol_state.tcp.max_win
- SFE_IPV4_TCP_MAX_ACK_WINDOW
- 1;
if (unlikely((s32)(sack - left_edge) < 0)) {
struct sfe_ipv4_connection *c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv4_remove_connection(si, c);
spin_unlock_bh(&si->lock);
DEBUG_TRACE("ack: %u before left edge: %u\n", sack, left_edge);
if (ret) {
sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_ACK_BEFORE_LEFT_EDGE);
return 0;
}
/*
* Have we just seen the largest window size yet for this connection? If yes
* then we need to record the new value.
*/
scaled_win = ntohs(tcph->window) << cm->protocol_state.tcp.win_scale;
scaled_win += (sack - ack);
if (unlikely(cm->protocol_state.tcp.max_win < scaled_win)) {
cm->protocol_state.tcp.max_win = scaled_win;
}
/*
* If our sequence and/or ack numbers have advanced then record the new state.
*/
if (likely((s32)(end - cm->protocol_state.tcp.end) >= 0)) {
cm->protocol_state.tcp.end = end;
}
max_end = sack + scaled_win;
if (likely((s32)(max_end - counter_cm->protocol_state.tcp.max_end) >= 0)) {
counter_cm->protocol_state.tcp.max_end = max_end;
}
}
/*
* Check if skb was cloned. If it was, unshare it. Because
* the data area is going to be written in this path and we don't want to
* change the cloned skb's data section.
*/
if (unlikely(skb_cloned(skb))) {
DEBUG_TRACE("%px: skb is a cloned skb\n", skb);
skb = skb_unshare(skb, GFP_ATOMIC);
if (!skb) {
DEBUG_WARN("Failed to unshare the cloned skb\n");
rcu_read_unlock();
return 0;
}
/*
* Update the iph and tcph pointers with the unshared skb's data area.
*/
iph = (struct iphdr *)skb->data;
tcph = (struct tcphdr *)(skb->data + ihl);
}
/*
* Check if skb has enough headroom to write L2 headers
*/
if (unlikely(skb_headroom(skb) < cm->l2_hdr_size)) {
rcu_read_unlock();
DEBUG_WARN("%px: Not enough headroom: %u\n", skb, skb_headroom(skb));
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_NO_HEADROOM);
return 0;
}
/*
* For PPPoE packets, match server MAC and session id
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_PPPOE_DECAP)) {
struct ethhdr *eth;
bool pppoe_match;
if (unlikely(!sfe_l2_parse_flag_check(l2_info, SFE_L2_PARSE_FLAGS_PPPOE_INGRESS))) {
rcu_read_unlock();
DEBUG_TRACE("%px: PPPoE header not present in packet for PPPoE rule\n", skb);
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_INCORRECT_PPPOE_PARSING);
return 0;
}
eth = eth_hdr(skb);
pppoe_match = (cm->pppoe_session_id == sfe_l2_pppoe_session_id_get(l2_info)) &&
ether_addr_equal((u8*)cm->pppoe_remote_mac, (u8 *)eth->h_source);
if (unlikely(!pppoe_match)) {
DEBUG_TRACE("%px: PPPoE session ID %d and %d or MAC %pM and %pM did not match\n",
skb, cm->pppoe_session_id, sfe_l2_pppoe_session_id_get(l2_info),
cm->pppoe_remote_mac, eth->h_source);
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_INVALID_PPPOE_SESSION);
return 0;
}
skb->protocol = htons(l2_info->protocol);
this_cpu_inc(si->stats_pcpu->pppoe_decap_packets_forwarded64);
} else if (unlikely(sfe_l2_parse_flag_check(l2_info, SFE_L2_PARSE_FLAGS_PPPOE_INGRESS))) {
/*
* If packet contains PPPoE header but CME doesn't contain PPPoE flag yet we are exceptioning
* the packet to linux
*/
if (unlikely(!bridge_flow)) {
rcu_read_unlock();
DEBUG_TRACE("%px: CME doesn't contain PPPoE flag but packet has PPPoE header\n", skb);
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_PPPOE_NOT_SET_IN_CME);
return 0;
}
/*
* For bridged flows when packet contains PPPoE header, restore the header back and forward
* to xmit interface
*/
__skb_push(skb, PPPOE_SES_HLEN);
this_cpu_inc(si->stats_pcpu->pppoe_bridge_packets_forwarded64);
}
/*
* From this point on we're good to modify the packet.
*/
/*
* For PPPoE flows, add PPPoE header before L2 header is added.
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_PPPOE_ENCAP)) {
sfe_pppoe_add_header(skb, cm->pppoe_session_id, PPP_IP);
this_cpu_inc(si->stats_pcpu->pppoe_encap_packets_forwarded64);
}
/*
* Update DSCP
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_DSCP_REMARK)) {
iph->tos = (iph->tos & SFE_IPV4_DSCP_MASK) | cm->dscp;
}
/*
* Decrement our TTL.
*/
if (likely(!bridge_flow)) {
iph->ttl = ttl - 1;
}
/*
* Enable HW csum if rx checksum is verified and xmit interface is CSUM offload capable.
* Note: If L4 csum at Rx was found to be incorrect, we (router) should use incremental L4 checksum here
* so that HW does not re-calculate/replace the L4 csum
*/
hw_csum = !!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_CSUM_OFFLOAD) && (skb->ip_summed == CHECKSUM_UNNECESSARY);
/*
* Do we have to perform translations of the source address/port?
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_SRC)) {
u16 tcp_csum;
u32 sum;
iph->saddr = cm->xlate_src_ip;
tcph->source = cm->xlate_src_port;
if (unlikely(!hw_csum)) {
tcp_csum = tcph->check;
if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL)) {
sum = tcp_csum + cm->xlate_src_partial_csum_adjustment;
} else {
sum = tcp_csum + cm->xlate_src_csum_adjustment;
}
sum = (sum & 0xffff) + (sum >> 16);
tcph->check = (u16)sum;
}
}
/*
* Do we have to perform translations of the destination address/port?
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_DEST)) {
u16 tcp_csum;
u32 sum;
iph->daddr = cm->xlate_dest_ip;
tcph->dest = cm->xlate_dest_port;
if (unlikely(!hw_csum)) {
tcp_csum = tcph->check;
if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL)) {
sum = tcp_csum + cm->xlate_dest_partial_csum_adjustment;
} else {
sum = tcp_csum + cm->xlate_dest_csum_adjustment;
}
sum = (sum & 0xffff) + (sum >> 16);
tcph->check = (u16)sum;
}
}
/*
* If HW checksum offload is not possible, full L3 checksum and incremental L4 checksum
* are used to update the packet. Setting ip_summed to CHECKSUM_UNNECESSARY ensures checksum is
* not recalculated further in packet path.
*/
if (likely(hw_csum)) {
skb->ip_summed = CHECKSUM_PARTIAL;
} else {
iph->check = sfe_ipv4_gen_ip_csum(iph);
skb->ip_summed = CHECKSUM_UNNECESSARY;
}
/*
* Update traffic stats.
*/
atomic_inc(&cm->rx_packet_count);
atomic_add(len, &cm->rx_byte_count);
xmit_dev = cm->xmit_dev;
skb->dev = xmit_dev;
/*
* Check to see if we need to add VLAN tags
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_INSERT_EGRESS_VLAN_TAG)) {
sfe_vlan_add_tag(skb, cm->egress_vlan_hdr_cnt, cm->egress_vlan_hdr);
}
/*
* Check to see if we need to write an Ethernet header.
*/
if (likely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_L2_HDR)) {
if (unlikely(!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR))) {
dev_hard_header(skb, xmit_dev, ntohs(skb->protocol),
cm->xmit_dest_mac, cm->xmit_src_mac, len);
} else {
/*
* For the simple case we write this really fast.
*/
struct ethhdr *eth = (struct ethhdr *)__skb_push(skb, ETH_HLEN);
eth->h_proto = skb->protocol;
ether_addr_copy((u8 *)eth->h_dest, (u8 *)cm->xmit_dest_mac);
ether_addr_copy((u8 *)eth->h_source, (u8 *)cm->xmit_src_mac);
}
}
/*
* Update priority of skb.
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_PRIORITY_REMARK)) {
skb->priority = cm->priority;
}
/*
* Mark outgoing packet
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_MARK)) {
skb->mark = cm->mark;
/*
* Update service class stats if SAWF is valid.
*/
if (likely(cm->sawf_valid)) {
service_class_id = SFE_GET_SAWF_SERVICE_CLASS(cm->mark);
sfe_ipv4_service_class_stats_inc(si, service_class_id, len);
}
}
/*
* For the first packets, check if it could got fast xmit.
*/
if (unlikely(!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED)
&& (cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION))){
cm->features = netif_skb_features(skb);
if (likely(sfe_fast_xmit_check(skb, cm->features))) {
cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT;
}
cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED;
}
features = cm->features;
fast_xmit = !!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT);
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
/*
* We're going to check for GSO flags when we transmit the packet so
* start fetching the necessary cache line now.
*/
prefetch(skb_shinfo(skb));
/*
* We do per packet condition check before we could fast xmit the
* packet.
*/
if (likely(fast_xmit)) {
if (likely(!skb_is_gso(skb))) {
if (likely(dev_fast_xmit(skb, xmit_dev, features))) {
this_cpu_inc(si->stats_pcpu->packets_fast_xmited64);
return 1;
}
} else {
cm->flags &= ~SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT;
DEBUG_TRACE("%px: fast xmit disabled for xmit dev %s", skb, xmit_dev->name);
}
}
/*
* Mark that this packet has been fast forwarded.
*/
skb->fast_forwarded = 1;
/*
* Send the packet on its way.
*/
dev_queue_xmit(skb);
return 1;
}

22
shortcut-fe/sfe_ipv4_tcp.h
View file

@ -1,22 +0,0 @@
/*
* sfe_ipv4_tcp.h
* Shortcut forwarding engine - IPv4 TCP header file
*
* Copyright (c) 2013-2016, 2019-2020, The Linux Foundation. All rights reserved.
* Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
int sfe_ipv4_recv_tcp(struct sfe_ipv4 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct iphdr *iph, unsigned int ihl, bool sync_on_find, struct sfe_l2_info *l2_info);

235
shortcut-fe/sfe_ipv4_tun6rd.c
View file

@ -1,235 +0,0 @@
/*
* sfe_ipv4_tun6rd.c
* Shortcut forwarding engine file for IPv4 TUN6RD
*
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <linux/skbuff.h>
#include <linux/etherdevice.h>
#include <linux/version.h>
#include <net/protocol.h>
#include <net/ip.h>
#include "sfe_debug.h"
#include "sfe_api.h"
#include "sfe.h"
#include "sfe_flow_cookie.h"
#include "sfe_ipv4.h"
#include "sfe_vlan.h"
/*
* sfe_ipv4_recv_tun6rd()
* Handle TUN6RD packet receives and forwarding.
*/
int sfe_ipv4_recv_tun6rd(struct sfe_ipv4 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct iphdr *iph, unsigned int ihl,
bool sync_on_find, struct sfe_l2_info *l2_info, bool tun_outer)
{
__be32 src_ip;
__be32 dest_ip;
__be16 src_port = 0;
__be16 dest_port = 0;
struct sfe_ipv4_connection_match *cm;
DEBUG_TRACE("%px: sfe: sfe_ipv4_recv_tun6rd called.\n", skb);
/*
* Read the IP address information. Read the IP header data first
* because we've almost certainly got that in the cache.
*/
src_ip = iph->saddr;
dest_ip = iph->daddr;
rcu_read_lock();
/*
* Look for a connection match.
*/
#ifdef CONFIG_NF_FLOW_COOKIE
cm = si->sfe_flow_cookie_table[skb->flow_cookie & SFE_FLOW_COOKIE_MASK].match;
if (unlikely(!cm)) {
cm = sfe_ipv4_find_connection_match_rcu(si, dev, IPPROTO_IPV6, src_ip, src_port, dest_ip, dest_port);
}
#else
cm = sfe_ipv4_find_connection_match_rcu(si, dev, IPPROTO_IPV6, src_ip, src_port, dest_ip, dest_port);
#endif
if (unlikely(!cm)) {
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TUN6RD_NO_CONNECTION);
DEBUG_TRACE("%px: no tun6rd connection found\n", skb);
return 0;
}
/*
* If our packet has been marked as "sync on find" we will sync the status
* and forward it to slowpath.
*/
if (unlikely(sync_on_find)) {
sfe_ipv4_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TUN6RD_SYNC_ON_FIND);
DEBUG_TRACE("%px: Sync on find\n", skb);
return 0;
}
/*
* If cm->proto is set, it means the decap path.
* Otherwise we forward the packet in encap path.
*/
if(cm->proto) {
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0))
const struct net_protocol *ipprot = cm->proto;
#else
struct net_protocol *ipprot = cm->proto;
#endif
/*
* Do we expect an ingress VLAN tag for this flow?
* Note: We will only have ingress tag check in decap direction.
* Here, no modification is needed, we only check tag match between
* vlan hdr stored in cm and l2_info.
*/
if (unlikely(!sfe_vlan_validate_ingress_tag(skb, cm->ingress_vlan_hdr_cnt, cm->ingress_vlan_hdr, l2_info))) {
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_INGRESS_VLAN_TAG_MISMATCH);
DEBUG_TRACE("VLAN tag mismatch. skb=%px\n"
"cm: %u [0]=%x/%x [1]=%x/%x\n"
"l2_info+: %u [0]=%x/%x [1]=%x/%x\n", skb,
cm->ingress_vlan_hdr_cnt,
htons(cm->ingress_vlan_hdr[0].tpid), cm->ingress_vlan_hdr[0].tci,
htons(cm->ingress_vlan_hdr[1].tpid), cm->ingress_vlan_hdr[1].tci,
l2_info->vlan_hdr_cnt,
htons(l2_info->vlan_hdr[0].tpid), l2_info->vlan_hdr[0].tci,
htons(l2_info->vlan_hdr[1].tpid), l2_info->vlan_hdr[1].tci);
return 0;
}
skb_reset_network_header(skb);
skb_pull(skb, ihl);
skb_reset_transport_header(skb);
/*
* ipprot->handler(skb) will always return 0;
* There is no way to tell whether the packet is dropped later in linux or not.
* Hence here inc the byte/packet count always.
*/
atomic_inc(&cm->rx_packet_count);
atomic_add(len, &cm->rx_byte_count);
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
DEBUG_TRACE("%px: %s decap done \n", skb, __func__);
/*
* Update top interface for tunnel searching.
*/
skb->dev = cm->top_interface_dev;
ipprot->handler(skb);
return 1;
}
/*
* If our packet is larger than the MTU of the transmit interface then
* we can't forward it easily.
*/
if (unlikely(len > cm->xmit_dev_mtu)) {
sfe_ipv4_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TUN6RD_NEEDS_FRAGMENTATION);
DEBUG_TRACE("%px: Larger than mtu\n", skb);
return 0;
}
/*
* Update DSCP
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_DSCP_REMARK)) {
iph->tos = (iph->tos & SFE_IPV4_DSCP_MASK) | cm->dscp;
}
/*
* Update traffic stats.
*/
atomic_inc(&cm->rx_packet_count);
atomic_add(len, &cm->rx_byte_count);
skb->dev = cm->xmit_dev;
/*
* Check to see if we need to add VLAN tags
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_INSERT_EGRESS_VLAN_TAG)) {
/*
* Check if skb has enough headroom to write L2 headers
*/
if (unlikely(skb_headroom(skb) < cm->l2_hdr_size)) {
rcu_read_unlock();
DEBUG_WARN("%px: Not enough headroom: %u\n", skb, skb_headroom(skb));
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_NO_HEADROOM);
return 0;
}
sfe_vlan_add_tag(skb, cm->egress_vlan_hdr_cnt, cm->egress_vlan_hdr);
}
/*
* Check to see if we need to write a header.
*/
if (likely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_L2_HDR)) {
if (unlikely(!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR))) {
dev_hard_header(skb, cm->xmit_dev, ntohs(skb->protocol),
cm->xmit_dest_mac, cm->xmit_src_mac, len);
} else {
struct ethhdr *eth = (struct ethhdr *)__skb_push(skb, ETH_HLEN);
eth->h_proto = skb->protocol;
ether_addr_copy((u8 *)eth->h_dest, (u8 *)cm->xmit_dest_mac);
ether_addr_copy((u8 *)eth->h_source, (u8 *)cm->xmit_src_mac);
}
}
/*
* Update priority of skb.
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_PRIORITY_REMARK)) {
skb->priority = cm->priority;
}
/*
* Mark outgoing packet.
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_MARK)) {
skb->mark = cm->mark;
}
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
/*
* We're going to check for GSO flags when we transmit the packet so
* start fetching the necessary cache line now.
*/
prefetch(skb_shinfo(skb));
/*
* Mark that this packet has been fast forwarded and send it on its way.
*/
skb->fast_forwarded = 1;
dev_queue_xmit(skb);
return 1;
}

22
shortcut-fe/sfe_ipv4_tun6rd.h
View file

@ -1,22 +0,0 @@
/*
* sfe_ipv4_tun6rd.h
* Shortcut forwarding engine header file for IPv4 TUN6RD
*
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
int sfe_ipv4_recv_tun6rd(struct sfe_ipv4 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct iphdr *iph, unsigned int ihl,
bool sync_on_find, struct sfe_l2_info *l2_info, bool tun_outer);

616
shortcut-fe/sfe_ipv4_udp.c
View file

@ -1,616 +0,0 @@
/*
* sfe_ipv4_udp.c
* Shortcut forwarding engine - IPv4 UDP implementation
*
* Copyright (c) 2013-2016, 2019-2020, The Linux Foundation. All rights reserved.
* Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <linux/skbuff.h>
#include <net/udp.h>
#include <linux/etherdevice.h>
#include <linux/lockdep.h>
#include <linux/version.h>
#include "sfe_debug.h"
#include "sfe_api.h"
#include "sfe.h"
#include "sfe_flow_cookie.h"
#include "sfe_ipv4.h"
#include "sfe_pppoe.h"
#include "sfe_vlan.h"
/*
* sfe_ipv4_udp_sk_deliver()
* Deliver the packet to the protocol handler registered with Linux.
* To be called under rcu_read_lock()
* Returns:
* 1 if the packet needs to be passed to Linux.
* 0 if the packet is processed successfully.
* -1 if the packet is dropped in SFE.
*/
static int sfe_ipv4_udp_sk_deliver(struct sk_buff *skb, struct sfe_ipv4_connection_match *cm, unsigned int ihl)
{
struct udp_sock *up;
struct sock *sk;
int ret;
int (*encap_rcv)(struct sock *sk, struct sk_buff *skb);
struct udphdr *uh;
unsigned short ulen;
/*
* Call the decap handler for valid encap_rcv handler.
*/
up = rcu_dereference(cm->up);
encap_rcv = READ_ONCE(up->encap_rcv);
if (!encap_rcv) {
DEBUG_ERROR("%px: sfe: Error: up->encap_rcv is NULL\n", skb);
return 1;
}
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0))
nf_reset(skb);
#else
nf_reset_ct(skb);
#endif
skb_pull(skb, ihl);
skb_reset_transport_header(skb);
sk = (struct sock *)up;
uh = udp_hdr(skb);
ulen = ntohs(uh->len);
if (ulen < sizeof(*uh) || pskb_trim_rcsum(skb, ulen)) {
DEBUG_TRACE("%px: short packet", skb);
goto except;
}
uh = udp_hdr(skb);
/*
* Verify checksum before giving to encap_rcv handler function.
* TODO: The following approach is ignorant for UDPLITE for now.
* Instead, consider calling Linux API to do checksum validation.
*/
if (unlikely(skb->ip_summed != CHECKSUM_UNNECESSARY && skb->ip_summed != CHECKSUM_COMPLETE) && uh->check) {
UDP_SKB_CB(skb)->partial_cov = 0;
UDP_SKB_CB(skb)->cscov = skb->len;
if (skb_checksum_init(skb, IPPROTO_UDP, inet_compute_pseudo)) {
DEBUG_TRACE("%px: checksum initilization failed", skb);
goto except;
}
if (inet_get_convert_csum(sk)) {
skb_checksum_try_convert(skb, IPPROTO_UDP, inet_compute_pseudo);
}
if (udp_lib_checksum_complete(skb)) {
DEBUG_TRACE("%px: udp checksum validation failed", skb);
goto except;
}
DEBUG_TRACE("%px: sfe: udp checksum verified in s/w correctly.\n", skb);
}
/*
* At this point, L4 checksum has already been verified and pkt is going
* to Linux's tunnel decap-handler. Setting ip_summed field to CHECKSUM_NONE,
* to ensure that later packet's inner header checksum is validated correctly.
* TODO: Find the fix to set skb->ip_summed = CHECKSUM_NONE;
*/
/*
* encap_rcv() returns the following value:
* =0 if skb was successfully passed to the encap
* handler or was discarded by it.
* >0 if skb should be passed on to UDP.
* <0 if skb should be resubmitted as proto -N
*/
ret = encap_rcv(sk, skb);
if (unlikely(ret)) {
DEBUG_TRACE("%px: sfe: udp-decap API return error: %d\n", skb, ret);
goto except;
}
return 0;
except:
/*
* The packet could be restored with the original L2 Information for L2
* flow, but it couldn't restore the NATed IP in the packets.
*/
skb_push(skb, ihl);
return 1;
}
/*
* sfe_ipv4_recv_udp()
* Handle UDP packet receives and forwarding.
*/
int sfe_ipv4_recv_udp(struct sfe_ipv4 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct iphdr *iph, unsigned int ihl,
bool sync_on_find, struct sfe_l2_info *l2_info, bool tun_outer)
{
struct udphdr *udph;
__be32 src_ip;
__be32 dest_ip;
__be16 src_port;
__be16 dest_port;
struct sfe_ipv4_connection_match *cm;
u8 ttl;
u32 service_class_id;
struct net_device *xmit_dev;
bool hw_csum;
int err;
bool bridge_flow;
int ret;
bool fast_xmit;
netdev_features_t features;
/*
* Is our packet too short to contain a valid UDP header?
*/
if (unlikely(!pskb_may_pull(skb, (sizeof(struct udphdr) + ihl)))) {
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_UDP_HEADER_INCOMPLETE);
DEBUG_TRACE("%px: packet too short for UDP header\n", skb);
return 0;
}
/*
* Read the IP address and port information. Read the IP header data first
* because we've almost certainly got that in the cache. We may not yet have
* the UDP header cached though so allow more time for any prefetching.
*/
src_ip = iph->saddr;
dest_ip = iph->daddr;
udph = (struct udphdr *)(skb->data + ihl);
src_port = udph->source;
dest_port = udph->dest;
rcu_read_lock();
/*
* Look for a connection match.
*/
#ifdef CONFIG_NF_FLOW_COOKIE
cm = si->sfe_flow_cookie_table[skb->flow_cookie & SFE_FLOW_COOKIE_MASK].match;
if (unlikely(!cm)) {
cm = sfe_ipv4_find_connection_match_rcu(si, dev, IPPROTO_UDP, src_ip, src_port, dest_ip, dest_port);
}
#else
/*
* 5-tuple lookup for UDP flow.
*/
cm = sfe_ipv4_find_connection_match_rcu(si, dev, IPPROTO_UDP, src_ip, src_port, dest_ip, dest_port);
#endif
if (unlikely(!cm)) {
/*
* Try a 4-tuple lookup; required for tunnels like vxlan.
*/
cm = sfe_ipv4_find_connection_match_rcu(si, dev, IPPROTO_UDP, src_ip, 0, dest_ip, dest_port);
if (unlikely(!cm)) {
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_UDP_NO_CONNECTION);
DEBUG_TRACE("%px: sfe: no connection found in 4-tuple lookup.\n", skb);
return 0;
}
}
/*
* Source interface validate.
*/
if (unlikely((cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_SRC_INTERFACE_CHECK) && (cm->match_dev != dev))) {
if (!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_SRC_INTERFACE_CHECK_NO_FLUSH)) {
struct sfe_ipv4_connection *c = cm->connection;
DEBUG_TRACE("flush on source interface check failure\n");
spin_lock_bh(&si->lock);
ret = sfe_ipv4_remove_connection(si, c);
spin_unlock_bh(&si->lock);
if (ret) {
sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
}
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_INVALID_SRC_IFACE);
DEBUG_TRACE("exception the packet on source interface check failure\n");
return 0;
}
/*
* If our packet has been marked as "sync on find" we can't actually
* forward it in the fast path, but now that we've found an associated
* connection we need sync its status before exception it to slow path.
*/
if (unlikely(sync_on_find)) {
sfe_ipv4_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_UDP_IP_OPTIONS_OR_INITIAL_FRAGMENT);
DEBUG_TRACE("%px: sync on find\n", cm);
return 0;
}
#ifdef CONFIG_XFRM
/*
* We can't accelerate the flow on this direction, just let it go
* through the slow path.
*/
if (unlikely(!cm->flow_accel)) {
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_not_forwarded64);
return 0;
}
#endif
/*
* Do we expect an ingress VLAN tag for this flow?
*/
if (unlikely(!sfe_vlan_validate_ingress_tag(skb, cm->ingress_vlan_hdr_cnt, cm->ingress_vlan_hdr, l2_info))) {
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_INGRESS_VLAN_TAG_MISMATCH);
DEBUG_TRACE("VLAN tag mismatch. skb=%px\n", skb);
return 0;
}
bridge_flow = !!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_BRIDGE_FLOW);
/*
* Does our TTL allow forwarding?
*/
if (likely(!bridge_flow)) {
ttl = iph->ttl;
if (unlikely(ttl < 2)) {
sfe_ipv4_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
DEBUG_TRACE("%px: sfe: TTL too low\n", skb);
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_UDP_SMALL_TTL);
return 0;
}
}
/*
* If our packet is larger than the MTU of the transmit interface then
* we can't forward it easily.
*/
if (unlikely((len > cm->xmit_dev_mtu) && (!cm->up))) {
sfe_ipv4_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_UDP_NEEDS_FRAGMENTATION);
DEBUG_TRACE("%px: sfe: larger than MTU\n", cm);
return 0;
}
/*
* Check if skb was cloned. If it was, unshare it. Because
* the data area is going to be written in this path and we don't want to
* change the cloned skb's data section.
*/
if (unlikely(skb_cloned(skb))) {
DEBUG_TRACE("%px: skb is a cloned skb\n", skb);
skb = skb_unshare(skb, GFP_ATOMIC);
if (!skb) {
DEBUG_WARN("%px: Failed to unshare the cloned skb\n", skb);
rcu_read_unlock();
return 0;
}
/*
* Update the iph and udph pointers with the unshared skb's data area.
*/
iph = (struct iphdr *)skb->data;
udph = (struct udphdr *)(skb->data + ihl);
}
/*
* Check if skb has enough headroom to write L2 headers
*/
if (unlikely(skb_headroom(skb) < cm->l2_hdr_size)) {
rcu_read_unlock();
DEBUG_WARN("%px: Not enough headroom: %u\n", skb, skb_headroom(skb));
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_NO_HEADROOM);
return 0;
}
/*
* For PPPoE packets, match server MAC and session id
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_PPPOE_DECAP)) {
struct ethhdr *eth;
bool pppoe_match;
if (unlikely(!sfe_l2_parse_flag_check(l2_info, SFE_L2_PARSE_FLAGS_PPPOE_INGRESS))) {
rcu_read_unlock();
DEBUG_TRACE("%px: PPPoE header not present in packet for PPPoE rule\n", skb);
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_INCORRECT_PPPOE_PARSING);
return 0;
}
eth = eth_hdr(skb);
pppoe_match = (cm->pppoe_session_id == sfe_l2_pppoe_session_id_get(l2_info)) &&
ether_addr_equal((u8*)cm->pppoe_remote_mac, (u8 *)eth->h_source);
if (unlikely(!pppoe_match)) {
DEBUG_TRACE("%px: PPPoE session ID %d and %d or MAC %pM and %pM did not match\n",
skb, cm->pppoe_session_id, sfe_l2_pppoe_session_id_get(l2_info),
cm->pppoe_remote_mac, eth->h_source);
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_INVALID_PPPOE_SESSION);
return 0;
}
skb->protocol = htons(l2_info->protocol);
this_cpu_inc(si->stats_pcpu->pppoe_decap_packets_forwarded64);
} else if (unlikely(sfe_l2_parse_flag_check(l2_info, SFE_L2_PARSE_FLAGS_PPPOE_INGRESS))) {
/*
* If packet contains PPPoE header but CME doesn't contain PPPoE flag yet we are exceptioning
* the packet to linux
*/
if (unlikely(!bridge_flow)) {
rcu_read_unlock();
DEBUG_TRACE("%px: CME doesn't contain PPPoE flag but packet has PPPoE header\n", skb);
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_PPPOE_NOT_SET_IN_CME);
return 0;
}
/*
* For bridged flows when packet contains PPPoE header, restore the header back and forward
* to xmit interface
*/
__skb_push(skb, PPPOE_SES_HLEN);
this_cpu_inc(si->stats_pcpu->pppoe_bridge_packets_forwarded64);
}
/*
* From this point on we're good to modify the packet.
*/
/*
* For PPPoE flows, add PPPoE header before L2 header is added.
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_PPPOE_ENCAP)) {
sfe_pppoe_add_header(skb, cm->pppoe_session_id, PPP_IP);
this_cpu_inc(si->stats_pcpu->pppoe_encap_packets_forwarded64);
}
/*
* Enable HW csum if rx checksum is verified and xmit interface is CSUM offload capable.
* Note: If L4 csum at Rx was found to be incorrect, we (router) should use incremental L4 checksum here
* so that HW does not re-calculate/replace the L4 csum
*/
hw_csum = !!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_CSUM_OFFLOAD) && (skb->ip_summed == CHECKSUM_UNNECESSARY);
/*
* Do we have to perform translations of the source address/port?
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_SRC)) {
u16 udp_csum;
iph->saddr = cm->xlate_src_ip;
udph->source = cm->xlate_src_port;
/*
* Do we have a non-zero UDP checksum? If we do then we need
* to update it.
*/
if (unlikely(!hw_csum)) {
udp_csum = udph->check;
if (likely(udp_csum)) {
u32 sum;
if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL)) {
sum = udp_csum + cm->xlate_src_partial_csum_adjustment;
} else {
sum = udp_csum + cm->xlate_src_csum_adjustment;
}
sum = (sum & 0xffff) + (sum >> 16);
udph->check = (u16)sum;
}
}
}
/*
* Do we have to perform translations of the destination address/port?
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_DEST)) {
u16 udp_csum;
iph->daddr = cm->xlate_dest_ip;
udph->dest = cm->xlate_dest_port;
/*
* Do we have a non-zero UDP checksum? If we do then we need
* to update it.
*/
if (unlikely(!hw_csum)) {
udp_csum = udph->check;
if (likely(udp_csum)) {
u32 sum;
/*
* TODO: Use a common API for below incremental checksum calculation
* for IPv4/IPv6 UDP/TCP
*/
if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL)) {
sum = udp_csum + cm->xlate_dest_partial_csum_adjustment;
} else {
sum = udp_csum + cm->xlate_dest_csum_adjustment;
}
sum = (sum & 0xffff) + (sum >> 16);
udph->check = (u16)sum;
}
}
}
/*
* UDP sock will be valid only in decap-path.
* Call encap_rcv function associated with udp_sock in cm.
*/
if (unlikely(cm->up)) {
/*
* Call decap handler associated with sock.
* Also validates UDP checksum before calling decap handler.
*/
err = sfe_ipv4_udp_sk_deliver(skb, cm, ihl);
if (unlikely(err == 1)) {
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_not_forwarded64);
return 0;
}
/*
* Update traffic stats.
*/
atomic_inc(&cm->rx_packet_count);
atomic_add(len, &cm->rx_byte_count);
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
DEBUG_TRACE("%px: sfe: sfe_ipv4_recv_udp -> encap_rcv done.\n", skb);
return 1;
}
/*
* Decrement our TTL
* Except when called from hook function in post-decap.
*/
if (likely(!bridge_flow)) {
iph->ttl -= (u8)(!tun_outer);
}
/*
* Update DSCP
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_DSCP_REMARK)) {
iph->tos = (iph->tos & SFE_IPV4_DSCP_MASK) | cm->dscp;
}
/*
* If HW checksum offload is not possible, full L3 checksum and incremental L4 checksum
* are used to update the packet. Setting ip_summed to CHECKSUM_UNNECESSARY ensures checksum is
* not recalculated further in packet path.
*/
if (likely(hw_csum)) {
skb->ip_summed = CHECKSUM_PARTIAL;
} else {
iph->check = sfe_ipv4_gen_ip_csum(iph);
}
/*
* Update traffic stats.
*/
atomic_inc(&cm->rx_packet_count);
atomic_add(len, &cm->rx_byte_count);
xmit_dev = cm->xmit_dev;
skb->dev = xmit_dev;
/*
* Check to see if we need to add VLAN tags
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_INSERT_EGRESS_VLAN_TAG)) {
sfe_vlan_add_tag(skb, cm->egress_vlan_hdr_cnt, cm->egress_vlan_hdr);
}
/*
* Check to see if we need to write an Ethernet header.
*/
if (likely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_L2_HDR)) {
if (unlikely(!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR))) {
dev_hard_header(skb, xmit_dev, ntohs(skb->protocol),
cm->xmit_dest_mac, cm->xmit_src_mac, len);
} else {
/*
* For the simple case we write this really fast.
*/
struct ethhdr *eth = (struct ethhdr *)__skb_push(skb, ETH_HLEN);
eth->h_proto = skb->protocol;
ether_addr_copy((u8 *)eth->h_dest, (u8 *)cm->xmit_dest_mac);
ether_addr_copy((u8 *)eth->h_source, (u8 *)cm->xmit_src_mac);
}
}
/*
* Update priority of skb.
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_PRIORITY_REMARK)) {
skb->priority = cm->priority;
}
/*
* Mark outgoing packet.
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_MARK)) {
skb->mark = cm->mark;
/*
* Update service class stats if SAWF is valid.
*/
if (likely(cm->sawf_valid)) {
service_class_id = SFE_GET_SAWF_SERVICE_CLASS(cm->mark);
sfe_ipv4_service_class_stats_inc(si, service_class_id, len);
}
}
/*
* For the first packets, check if it could got fast xmit.
*/
if (unlikely(!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED)
&& (cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION))){
cm->features = netif_skb_features(skb);
if (likely(sfe_fast_xmit_check(skb, cm->features))) {
cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT;
}
cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED;
}
features = cm->features;
fast_xmit = !!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT);
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
/*
* We're going to check for GSO flags when we transmit the packet so
* start fetching the necessary cache line now.
*/
prefetch(skb_shinfo(skb));
/*
* We do per packet condition check before we could fast xmit the
* packet.
*/
if (likely(fast_xmit && dev_fast_xmit(skb, xmit_dev, features))) {
this_cpu_inc(si->stats_pcpu->packets_fast_xmited64);
return 1;
}
/*
* Mark that this packet has been fast forwarded.
*/
skb->fast_forwarded = 1;
/*
* Send the packet on its way.
*/
dev_queue_xmit(skb);
return 1;
}

22
shortcut-fe/sfe_ipv4_udp.h
View file

@ -1,22 +0,0 @@
/*
* sfe_ipv4_udp.h
* Shortcut forwarding engine - IPv4 UDP header file
*
* Copyright (c) 2013-2016, 2019-2020, The Linux Foundation. All rights reserved.
* Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
int sfe_ipv4_recv_udp(struct sfe_ipv4 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct iphdr *iph, unsigned int ihl, bool sync_on_find,
struct sfe_l2_info *l2_info, bool tun_outer);

2905
shortcut-fe/sfe_ipv6.c
View file

File diff suppressed because it is too large Load diff

496
shortcut-fe/sfe_ipv6.h
View file

@ -1,496 +0,0 @@
/*
* sfe_ipv6.h
* Shortcut forwarding engine header file for IPv6.
*
* Copyright (c) 2015-2016, 2019-2020, The Linux Foundation. All rights reserved.
* Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#ifndef __SFE_IPV6_H
#define __SFE_IPV6_H
#define CHAR_DEV_MSG_SIZE 768
#define SFE_IPV6_DSCP_MASK 0xf03f
#define SFE_IPV6_DSCP_SHIFT 2
#define SFE_IPV6_FRAG_OFFSET 0xfff8
/*
* generic IPv6 extension header
*/
struct sfe_ipv6_ext_hdr {
__u8 next_hdr;
__u8 hdr_len;
__u8 padding[6];
};
/*
* Specifies the lower bound on ACK numbers carried in the TCP header
*/
#define SFE_IPV6_TCP_MAX_ACK_WINDOW 65520
/*
* IPv6 TCP connection match additional data.
*/
struct sfe_ipv6_tcp_connection_match {
u8 win_scale; /* Window scale */
u32 max_win; /* Maximum window size seen */
u32 end; /* Sequence number of the next byte to send (seq + segment length) */
u32 max_end; /* Sequence number of the last byte to ack */
};
/*
* Bit flags for IPv6 connection matching entry.
*/
#define SFE_IPV6_CONNECTION_MATCH_FLAG_XLATE_SRC (1<<0)
/* Perform source translation */
#define SFE_IPV6_CONNECTION_MATCH_FLAG_XLATE_DEST (1<<1)
/* Perform destination translation */
#define SFE_IPV6_CONNECTION_MATCH_FLAG_NO_SEQ_CHECK (1<<2)
/* Ignore TCP sequence numbers */
#define SFE_IPV6_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR (1<<3)
/* Fast Ethernet header write */
#define SFE_IPV6_CONNECTION_MATCH_FLAG_WRITE_L2_HDR (1<<4)
/* Fast Ethernet header write */
#define SFE_IPV6_CONNECTION_MATCH_FLAG_PRIORITY_REMARK (1<<5)
/* remark priority of SKB */
#define SFE_IPV6_CONNECTION_MATCH_FLAG_DSCP_REMARK (1<<6)
/* remark DSCP of packet */
#define SFE_IPV6_CONNECTION_MATCH_FLAG_CSUM_OFFLOAD (1<<7)
/* checksum offload.*/
#define SFE_IPV6_CONNECTION_MATCH_FLAG_PPPOE_DECAP (1<<8)
/* Indicates that PPPoE should be decapsulated */
#define SFE_IPV6_CONNECTION_MATCH_FLAG_PPPOE_ENCAP (1<<9)
/* Indicates that PPPoE should be encapsulated */
#define SFE_IPV6_CONNECTION_MATCH_FLAG_BRIDGE_FLOW (1<<10)
/* Bridge flow */
#define SFE_IPV6_CONNECTION_MATCH_FLAG_MARK (1<<11)
/* set skb mark*/
#define SFE_IPV6_CONNECTION_MATCH_FLAG_INSERT_EGRESS_VLAN_TAG (1<<12)
/* Insert VLAN tag */
#define SFE_IPV6_CONNECTION_MATCH_FLAG_SRC_INTERFACE_CHECK (1<<13)
/* Source interface check */
#define SFE_IPV6_CONNECTION_MATCH_FLAG_PASSTHROUGH (1<<14)
/* passthrough flow: encap/decap to be skipped for this flow */
#define SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT (1<<15)
/* go fast xmit*/
#define SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED (1<<16)
/* fast xmit checked or not*/
#define SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION (1<<17)
/* Fast xmit may be possible for this flow, if SFE check passes */
#define SFE_IPV6_CONNECTION_MATCH_FLAG_SRC_INTERFACE_CHECK_NO_FLUSH (1<<18)
/* Source interface check but do not flush the connection */
/*
* IPv6 connection matching structure.
*/
struct sfe_ipv6_connection_match {
/*
* References to other objects.
*/
struct hlist_node hnode;
struct sfe_ipv6_connection *connection;
struct sfe_ipv6_connection_match *counter_match;
/* Matches the flow in the opposite direction as the one in connection */
/*
* Characteristics that identify flows that match this rule.
*/
struct net_device *match_dev; /* Network device */
u8 match_protocol; /* Protocol */
struct sfe_ipv6_addr match_src_ip[1]; /* Source IP address */
struct sfe_ipv6_addr match_dest_ip[1]; /* Destination IP address */
__be16 match_src_port; /* Source port/connection ident */
__be16 match_dest_port; /* Destination port/connection ident */
struct udp_sock *up; /* Stores UDP sock information; valid only in decap path */
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0))
const struct inet6_protocol *proto; /* stores protocol handler; valid only in decap path */
#else
struct inet6_protocol *proto; /* stores protocol handler; valid only in decap path */
#endif
/*
* Control the operations of the match.
*/
u32 flags; /* Bit flags */
#ifdef CONFIG_NF_FLOW_COOKIE
u32 flow_cookie; /* used flow cookie, for debug */
#endif
#ifdef CONFIG_XFRM
u32 flow_accel; /* The flow accelerated or not */
#endif
/*
* Connection state that we track once we match.
*/
union { /* Protocol-specific state */
struct sfe_ipv6_tcp_connection_match tcp;
} protocol_state;
/*
* VLAN headers
*/
struct sfe_vlan_hdr ingress_vlan_hdr[SFE_MAX_VLAN_DEPTH];
struct sfe_vlan_hdr egress_vlan_hdr[SFE_MAX_VLAN_DEPTH];
/*
* Stats recorded in a sync period. These stats will be added to
* rx_packet_count64/rx_byte_count64 after a sync period.
*/
atomic_t rx_packet_count;
atomic_t rx_byte_count;
/*
* Packet translation information.
*/
struct sfe_ipv6_addr xlate_src_ip[1]; /* Address after source translation */
__be16 xlate_src_port; /* Port/connection ident after source translation */
u16 xlate_src_csum_adjustment;
/* Transport layer checksum adjustment after source translation */
struct sfe_ipv6_addr xlate_dest_ip[1]; /* Address after destination translation */
__be16 xlate_dest_port; /* Port/connection ident after destination translation */
u16 xlate_dest_csum_adjustment;
/* Transport layer checksum adjustment after destination translation */
u32 mark; /* mark for outgoing packet */
/*
* QoS information
*/
u32 priority;
u32 dscp;
/*
* Packet transmit information.
*/
struct net_device *xmit_dev; /* Network device on which to transmit */
unsigned short int xmit_dev_mtu;
/* Interface MTU */
u16 xmit_dest_mac[ETH_ALEN / 2];
/* Destination MAC address to use when forwarding */
u16 xmit_src_mac[ETH_ALEN / 2];
/* Source MAC address to use when forwarding */
u8 ingress_vlan_hdr_cnt; /* Ingress active vlan headers count */
u8 egress_vlan_hdr_cnt; /* Egress active vlan headers count */
/*
* Summary stats.
*/
u64 rx_packet_count64;
u64 rx_byte_count64;
/*
* PPPoE information.
*/
u16 pppoe_session_id;
u8 pppoe_remote_mac[ETH_ALEN];
struct net_device *top_interface_dev; /* Used by tunipip6 to store decap VLAN netdevice.*/
/*
* Size of all needed L2 headers
*/
u16 l2_hdr_size;
/*
* xmit device's feature
*/
netdev_features_t features;
bool sawf_valid; /* Indicates mark has valid SAWF information. */
};
/*
* Per-connection data structure.
*/
struct sfe_ipv6_connection {
struct sfe_ipv6_connection *next;
/* Pointer to the next entry in a hash chain */
struct sfe_ipv6_connection *prev;
/* Pointer to the previous entry in a hash chain */
int protocol; /* IP protocol number */
struct sfe_ipv6_addr src_ip[1]; /* Src IP addr pre-translation */
struct sfe_ipv6_addr src_ip_xlate[1]; /* Src IP addr post-translation */
struct sfe_ipv6_addr dest_ip[1]; /* Dest IP addr pre-translation */
struct sfe_ipv6_addr dest_ip_xlate[1]; /* Dest IP addr post-translation */
__be16 src_port; /* Src port pre-translation */
__be16 src_port_xlate; /* Src port post-translation */
__be16 dest_port; /* Dest port pre-translation */
__be16 dest_port_xlate; /* Dest port post-translation */
struct sfe_ipv6_connection_match *original_match;
/* Original direction matching structure */
struct net_device *original_dev;
/* Original direction source device */
struct sfe_ipv6_connection_match *reply_match;
/* Reply direction matching structure */
struct net_device *reply_dev; /* Reply direction source device */
u64 last_sync_jiffies; /* Jiffies count for the last sync */
struct sfe_ipv6_connection *all_connections_next;
/* Pointer to the next entry in the list of all connections */
struct sfe_ipv6_connection *all_connections_prev;
/* Pointer to the previous entry in the list of all connections */
bool removed; /* Indicates the connection is removed */
struct rcu_head rcu; /* delay rcu free */
u32 debug_read_seq; /* sequence number for debug dump */
};
/*
* IPv6 connections and hash table size information.
*/
#define SFE_IPV6_CONNECTION_HASH_SHIFT 12
#define SFE_IPV6_CONNECTION_HASH_SIZE (1 << SFE_IPV6_CONNECTION_HASH_SHIFT)
#define SFE_IPV6_CONNECTION_HASH_MASK (SFE_IPV6_CONNECTION_HASH_SIZE - 1)
enum sfe_ipv6_exception_events {
SFE_IPV6_EXCEPTION_EVENT_UDP_HEADER_INCOMPLETE,
SFE_IPV6_EXCEPTION_EVENT_UDP_NO_CONNECTION,
SFE_IPV6_EXCEPTION_EVENT_UDP_IP_OPTIONS_OR_INITIAL_FRAGMENT,
SFE_IPV6_EXCEPTION_EVENT_UDP_SMALL_TTL,
SFE_IPV6_EXCEPTION_EVENT_UDP_NEEDS_FRAGMENTATION,
SFE_IPV6_EXCEPTION_EVENT_TCP_HEADER_INCOMPLETE,
SFE_IPV6_EXCEPTION_EVENT_TCP_NO_CONNECTION_SLOW_FLAGS,
SFE_IPV6_EXCEPTION_EVENT_TCP_NO_CONNECTION_FAST_FLAGS,
SFE_IPV6_EXCEPTION_EVENT_TCP_IP_OPTIONS_OR_INITIAL_FRAGMENT,
SFE_IPV6_EXCEPTION_EVENT_TCP_SMALL_TTL,
SFE_IPV6_EXCEPTION_EVENT_TCP_NEEDS_FRAGMENTATION,
SFE_IPV6_EXCEPTION_EVENT_TCP_FLAGS,
SFE_IPV6_EXCEPTION_EVENT_TCP_SEQ_EXCEEDS_RIGHT_EDGE,
SFE_IPV6_EXCEPTION_EVENT_TCP_SMALL_DATA_OFFS,
SFE_IPV6_EXCEPTION_EVENT_TCP_BAD_SACK,
SFE_IPV6_EXCEPTION_EVENT_TCP_BIG_DATA_OFFS,
SFE_IPV6_EXCEPTION_EVENT_TCP_SEQ_BEFORE_LEFT_EDGE,
SFE_IPV6_EXCEPTION_EVENT_TCP_ACK_EXCEEDS_RIGHT_EDGE,
SFE_IPV6_EXCEPTION_EVENT_TCP_ACK_BEFORE_LEFT_EDGE,
SFE_IPV6_EXCEPTION_EVENT_ICMP_HEADER_INCOMPLETE,
SFE_IPV6_EXCEPTION_EVENT_ICMP_UNHANDLED_TYPE,
SFE_IPV6_EXCEPTION_EVENT_ICMP_IPV6_HEADER_INCOMPLETE,
SFE_IPV6_EXCEPTION_EVENT_ICMP_IPV6_NON_V6,
SFE_IPV6_EXCEPTION_EVENT_ICMP_IPV6_IP_OPTIONS_INCOMPLETE,
SFE_IPV6_EXCEPTION_EVENT_ICMP_IPV6_UDP_HEADER_INCOMPLETE,
SFE_IPV6_EXCEPTION_EVENT_ICMP_IPV6_TCP_HEADER_INCOMPLETE,
SFE_IPV6_EXCEPTION_EVENT_ICMP_IPV6_UNHANDLED_PROTOCOL,
SFE_IPV6_EXCEPTION_EVENT_ICMP_NO_CONNECTION,
SFE_IPV6_EXCEPTION_EVENT_ICMP_FLUSHED_CONNECTION,
SFE_IPV6_EXCEPTION_EVENT_HEADER_INCOMPLETE,
SFE_IPV6_EXCEPTION_EVENT_BAD_TOTAL_LENGTH,
SFE_IPV6_EXCEPTION_EVENT_NON_V6,
SFE_IPV6_EXCEPTION_EVENT_NON_INITIAL_FRAGMENT,
SFE_IPV6_EXCEPTION_EVENT_DATAGRAM_INCOMPLETE,
SFE_IPV6_EXCEPTION_EVENT_IP_OPTIONS_INCOMPLETE,
SFE_IPV6_EXCEPTION_EVENT_UNHANDLED_PROTOCOL,
SFE_IPV6_EXCEPTION_EVENT_FLOW_COOKIE_ADD_FAIL,
SFE_IPV6_EXCEPTION_EVENT_NO_HEADROOM,
SFE_IPV6_EXCEPTION_EVENT_INVALID_PPPOE_SESSION,
SFE_IPV6_EXCEPTION_EVENT_INCORRECT_PPPOE_PARSING,
SFE_IPV6_EXCEPTION_EVENT_PPPOE_NOT_SET_IN_CME,
SFE_IPV6_EXCEPTION_EVENT_PPPOE_BR_NOT_IN_CME,
SFE_IPV6_EXCEPTION_EVENT_INGRESS_VLAN_TAG_MISMATCH,
SFE_IPV6_EXCEPTION_EVENT_INVALID_SRC_IFACE,
SFE_IPV6_EXCEPTION_EVENT_TUNIPIP6_HEADER_INCOMPLETE,
SFE_IPV6_EXCEPTION_EVENT_TUNIPIP6_NO_CONNECTION,
SFE_IPV6_EXCEPTION_EVENT_TUNIPIP6_IP_OPTIONS_OR_INITIAL_FRAGMENT,
SFE_IPV6_EXCEPTION_EVENT_TUNIPIP6_SMALL_TTL,
SFE_IPV6_EXCEPTION_EVENT_TUNIPIP6_NEEDS_FRAGMENTATION,
SFE_IPV6_EXCEPTION_EVENT_TUNIPIP6_SYNC_ON_FIND,
SFE_IPV6_EXCEPTION_EVENT_GRE_HEADER_INCOMPLETE,
SFE_IPV6_EXCEPTION_EVENT_GRE_NO_CONNECTION,
SFE_IPV6_EXCEPTION_EVENT_GRE_IP_OPTIONS_OR_INITIAL_FRAGMENT,
SFE_IPV6_EXCEPTION_EVENT_GRE_SMALL_TTL,
SFE_IPV6_EXCEPTION_EVENT_GRE_NEEDS_FRAGMENTATION,
SFE_IPV6_EXCEPTION_EVENT_ESP_NO_CONNECTION,
SFE_IPV6_EXCEPTION_EVENT_ESP_IP_OPTIONS_OR_INITIAL_FRAGMENT,
SFE_IPV6_EXCEPTION_EVENT_ESP_NEEDS_FRAGMENTATION,
SFE_IPV6_EXCEPTION_EVENT_ESP_SMALL_TTL,
SFE_IPV6_EXCEPTION_EVENT_LAST
};
/*
* Per CPU stats
*/
struct sfe_ipv6_stats {
/*
* Stats recorded in a sync period. These stats will be added to
* connection_xxx64 after a sync period.
*/
u64 connection_create_requests64;
/* Number of IPv6 connection create requests */
u64 connection_create_collisions64;
/* Number of IPv6 connection create requests that collided with existing hash table entries */
u64 connection_create_failures64;
/* Number of IPv6 connection create requests failures. */
u64 connection_destroy_requests64;
/* Number of IPv6 connection destroy requests */
u64 connection_destroy_misses64;
/* Number of IPv6 connection destroy requests that missed our hash table */
u64 connection_match_hash_hits64;
/* Number of IPv6 connection match hash hits */
u64 connection_match_hash_reorders64;
/* Number of IPv6 connection match hash reorders */
u64 connection_flushes64; /* Number of IPv6 connection flushes */
u64 packets_dropped64; /* Number of IPv4 packets dropped */
u64 packets_forwarded64; /* Number of IPv6 packets forwarded */
u64 packets_fast_xmited64; /* Number of IPv6 packets fast transmited */
u64 packets_not_forwarded64; /* Number of IPv6 packets not forwarded */
u64 exception_events64[SFE_IPV6_EXCEPTION_EVENT_LAST];
u64 pppoe_encap_packets_forwarded64; /* Number of IPv6 PPPoE encap packets forwarded */
u64 pppoe_decap_packets_forwarded64; /* Number of IPv6 PPPoE decap packets forwarded */
u64 pppoe_bridge_packets_forwarded64; /* Number of IPv6 PPPoE decap packets forwarded */
u64 pppoe_bridge_packets_3tuple_forwarded64; /* Number of IPv6 PPPoE bridge packets forwarded based on 3-tuple info */
};
/*
* sfe_ipv6_per_service_class_stats
* Per service class stats
*/
struct sfe_ipv6_per_service_class_stats {
u64 tx_bytes; /* Byte count */
u64 tx_packets; /* Packet count */
seqcount_t seq; /* seq lock for read/write protection */
/*
* TODO : Add the entries to be maintained later.
*/
};
/*
* sfe_ipv6_service_class_stats_db
* Stat entries for each service class.
*/
struct sfe_ipv6_service_class_stats_db{
struct sfe_ipv6_per_service_class_stats psc_stats[SFE_MAX_SERVICE_CLASS_ID];
/* Per service class stats */
};
/*
* Per-module structure.
*/
struct sfe_ipv6 {
spinlock_t lock; /* Lock for SMP correctness */
struct sfe_ipv6_connection *all_connections_head;
/* Head of the list of all connections */
struct sfe_ipv6_connection *all_connections_tail;
/* Tail of the list of all connections */
unsigned int num_connections; /* Number of connections */
struct delayed_work sync_dwork; /* Work to sync the statistics */
unsigned int work_cpu; /* The core to run stats sync on */
sfe_sync_rule_callback_t __rcu sync_rule_callback;
/* Callback function registered by a connection manager for stats syncing */
sfe_ipv6_many_sync_callback_t __rcu many_sync_callback;
/* Callback function registered by a connection manager for many stats syncing */
struct sfe_ipv6_connection *conn_hash[SFE_IPV6_CONNECTION_HASH_SIZE];
/* Connection hash table */
struct hlist_head hlist_conn_match_hash_head[SFE_IPV6_CONNECTION_HASH_SIZE];
#ifdef CONFIG_NF_FLOW_COOKIE
struct sfe_ipv6_flow_cookie_entry sfe_flow_cookie_table[SFE_FLOW_COOKIE_SIZE];
/* flow cookie table*/
sfe_ipv6_flow_cookie_set_func_t flow_cookie_set_func;
/* function used to configure flow cookie in hardware*/
int flow_cookie_enable;
/* Enable/disable flow cookie at runtime */
#endif
struct sfe_ipv6_service_class_stats_db __percpu *stats_pcpu_psc;
/* Database to maintain per cpu per service class statistics */
struct sfe_ipv6_stats __percpu *stats_pcpu;
/* Common SFE counters. */
struct sfe_ipv6_connection *wc_next;
/* The next walk point in the all connection list*/
/*
* Control state.
*/
struct kobject *sys_ipv6; /* sysfs linkage */
int debug_dev; /* Major number of the debug char device */
u32 debug_read_seq; /* sequence number for debug dump */
};
/*
* Enumeration of the XML output.
*/
enum sfe_ipv6_debug_xml_states {
SFE_IPV6_DEBUG_XML_STATE_START,
SFE_IPV6_DEBUG_XML_STATE_CONNECTIONS_START,
SFE_IPV6_DEBUG_XML_STATE_CONNECTIONS_CONNECTION,
SFE_IPV6_DEBUG_XML_STATE_CONNECTIONS_END,
SFE_IPV6_DEBUG_XML_STATE_EXCEPTIONS_START,
SFE_IPV6_DEBUG_XML_STATE_EXCEPTIONS_EXCEPTION,
SFE_IPV6_DEBUG_XML_STATE_EXCEPTIONS_END,
SFE_IPV6_DEBUG_XML_STATE_STATS,
SFE_IPV6_DEBUG_XML_STATE_END,
SFE_IPV6_DEBUG_XML_STATE_DONE
};
/*
* XML write state.
*/
struct sfe_ipv6_debug_xml_write_state {
enum sfe_ipv6_debug_xml_states state;
/* XML output file state machine state */
int iter_exception; /* Next exception iterator */
};
typedef bool (*sfe_ipv6_debug_xml_write_method_t)(struct sfe_ipv6 *si, char *buffer, char *msg, size_t *length,
int *total_read, struct sfe_ipv6_debug_xml_write_state *ws);
/*
* sfe_ipv6_is_ext_hdr()
* check if we recognize ipv6 extension header
*/
static inline bool sfe_ipv6_is_ext_hdr(u8 hdr)
{
return (hdr == NEXTHDR_HOP) ||
(hdr == NEXTHDR_ROUTING) ||
(hdr == NEXTHDR_FRAGMENT) ||
(hdr == NEXTHDR_AUTH) ||
(hdr == NEXTHDR_DEST) ||
(hdr == NEXTHDR_MOBILITY);
}
/*
* sfe_ipv6_change_dsfield()
* change dscp field in IPv6 packet
*/
static inline void sfe_ipv6_change_dsfield(struct ipv6hdr *iph, u8 dscp)
{
__be16 *p = (__be16 *)iph;
*p = ((*p & htons(SFE_IPV6_DSCP_MASK)) | htons((u16)dscp << 4));
}
bool sfe_ipv6_service_class_stats_get(uint8_t sid, uint64_t *bytes, uint64_t *packets);
void sfe_ipv6_exception_stats_inc(struct sfe_ipv6 *si, enum sfe_ipv6_exception_events reason);
void sfe_ipv6_service_class_stats_inc(struct sfe_ipv6 *si, uint8_t sid, uint64_t bytes);
struct sfe_ipv6_connection_match *
sfe_ipv6_find_connection_match_rcu(struct sfe_ipv6 *si, struct net_device *dev, u8 protocol,
struct sfe_ipv6_addr *src_ip, __be16 src_port,
struct sfe_ipv6_addr *dest_ip, __be16 dest_port);
bool sfe_ipv6_remove_connection(struct sfe_ipv6 *si, struct sfe_ipv6_connection *c);
void sfe_ipv6_flush_connection(struct sfe_ipv6 *si,
struct sfe_ipv6_connection *c,
sfe_sync_reason_t reason);
void sfe_ipv6_sync_status(struct sfe_ipv6 *si,
struct sfe_ipv6_connection *c,
sfe_sync_reason_t reason);
void sfe_ipv6_exit(void);
int sfe_ipv6_init(void);
#endif /* __SFE_IPV6_H */

275
shortcut-fe/sfe_ipv6_esp.c
View file

@ -1,275 +0,0 @@
/*
* sfe_ipv6_esp.c
* Shortcut forwarding engine - IPv6 ESP implementation
*
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <linux/skbuff.h>
#include <net/protocol.h>
#include <net/ip6_checksum.h>
#include <linux/etherdevice.h>
#include <linux/version.h>
#include "sfe_debug.h"
#include "sfe_api.h"
#include "sfe.h"
#include "sfe_flow_cookie.h"
#include "sfe_ipv6.h"
#include "sfe_ipv6_esp.h"
/*
* sfe_ipv6_recv_esp()
* Handle ESP packet receives and forwarding
*/
int sfe_ipv6_recv_esp(struct sfe_ipv6 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct ipv6hdr *iph, unsigned int ihl,
bool sync_on_find, bool tun_outer)
{
struct sfe_ipv6_connection_match *cm;
struct sfe_ipv6_addr *src_ip;
struct sfe_ipv6_addr *dest_ip;
struct net_device *xmit_dev;
struct inet6_protocol *ipprot;
netdev_features_t features;
bool bridge_flow;
bool passthrough;
bool fast_xmit;
bool ret;
/*
* Read the IP address from the iphdr, and set the src/dst ports to 0.
*/
src_ip = (struct sfe_ipv6_addr *)iph->saddr.s6_addr32;
dest_ip = (struct sfe_ipv6_addr *)iph->daddr.s6_addr32;
rcu_read_lock();
/*
* Look for a connection match.
*/
#ifdef CONFIG_NF_FLOW_COOKIE
cm = si->sfe_flow_cookie_table[skb->flow_cookie & SFE_FLOW_COOKIE_MASK].match;
if (unlikely(!cm)) {
cm = sfe_ipv6_find_connection_match_rcu(si, dev, IPPROTO_ESP, src_ip, 0, dest_ip, 0);
}
#else
cm = sfe_ipv6_find_connection_match_rcu(si, dev, IPPROTO_ESP, src_ip, 0, dest_ip, 0);
#endif
if (unlikely(!cm)) {
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_ESP_NO_CONNECTION);
DEBUG_TRACE("no connection found for esp packet\n");
return 0;
}
/*
* Source interface validate.
*/
if (unlikely((cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_SRC_INTERFACE_CHECK) && (cm->match_dev != dev))) {
struct sfe_ipv6_connection *c = cm->connection;
int ret;
spin_lock_bh(&si->lock);
ret = sfe_ipv6_remove_connection(si, c);
spin_unlock_bh(&si->lock);
if (ret) {
sfe_ipv6_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_INVALID_SRC_IFACE);
DEBUG_TRACE("flush on wrong source interface check failure\n");
return 0;
}
passthrough = cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_PASSTHROUGH;
bridge_flow = !!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_BRIDGE_FLOW);
/*
* If our packet has beern marked as "sync on find" we can't actually
* forward it in the fast path, but now that we've found an associated
* connection we need sync its status before exception it to slow path. unless
* it is passthrough packet.
* TODO: revisit to ensure that pass through traffic is not bypassing firewall for fragmented cases
*/
if (unlikely(sync_on_find) && !passthrough) {
sfe_ipv6_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_ESP_IP_OPTIONS_OR_INITIAL_FRAGMENT);
DEBUG_TRACE("Sync on find\n");
return 0;
}
/*
* Check if skb was cloned. If it was, unshare it.
*/
if (unlikely(skb_cloned(skb))) {
DEBUG_TRACE("%px: skb is a cloned skb\n", skb);
skb = skb_unshare(skb, GFP_ATOMIC);
if (!skb) {
DEBUG_WARN("Failed to unshare the cloned skb\n");
rcu_read_unlock();
return 0;
}
/*
* Update the iphdr pointer with the unshared skb's data area.
*/
iph = (struct ipv6hdr *)skb->data;
}
/*
* proto decap packet.
* Invoke the inet_protocol handler for delivery of the packet.
*/
ipprot = rcu_dereference(cm->proto);
if (likely(ipprot)) {
skb_reset_network_header(skb);
skb_pull(skb, ihl);
skb_reset_transport_header(skb);
xmit_dev = cm->xmit_dev;
skb->dev = xmit_dev;
ret = ipprot->handler(skb);
if (ret) {
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_not_forwarded64);
DEBUG_TRACE("ESP handler returned error %u\n", ret);
return 0;
}
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
return 1;
}
/*
* esp passthrough / ip local out scenarios
*/
/*
* If our packet is larger than the MTU of the transmit interface then
* we can't forward it easily.
*/
if (unlikely(len > cm->xmit_dev_mtu)) {
sfe_ipv6_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_ESP_NEEDS_FRAGMENTATION);
DEBUG_TRACE("Larger than MTU\n");
return 0;
}
/*
* need to ensure that TTL is >=2.
*/
if (!bridge_flow && (iph->hop_limit < 2) && passthrough) {
sfe_ipv6_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_ESP_SMALL_TTL);
DEBUG_TRACE("hop_limit too low\n");
return 0;
}
/*
* decrement TTL by 1.
*/
iph->hop_limit = iph->hop_limit - (u8)(!bridge_flow && !tun_outer);
/*
* Update DSCP
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_DSCP_REMARK)) {
sfe_ipv6_change_dsfield(iph, cm->dscp);
}
/*
* Update traffic stats.
*/
atomic_inc(&cm->rx_packet_count);
atomic_add(len, &cm->rx_byte_count);
xmit_dev = cm->xmit_dev;
skb->dev = xmit_dev;
/*
* write the layer - 2 header.
*/
if (likely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_WRITE_L2_HDR)) {
if (unlikely(!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR))) {
dev_hard_header(skb, xmit_dev, ETH_P_IPV6, cm->xmit_dest_mac, cm->xmit_src_mac, len);
} else {
/*
* For the simple case we write this really fast.
*/
struct ethhdr *eth = (struct ethhdr *)__skb_push(skb, ETH_HLEN);
eth->h_proto = htons(ETH_P_IPV6);
ether_addr_copy((u8 *)eth->h_dest, (u8 *)cm->xmit_dest_mac);
ether_addr_copy((u8 *)eth->h_source, (u8 *)cm->xmit_src_mac);
}
}
/*
* Update priority of skb.
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_PRIORITY_REMARK)) {
skb->priority = cm->priority;
}
/*
* Mark outgoing packet.
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_MARK)) {
skb->mark = cm->mark;
}
/*
* For the first packets, check if it could got fast xmit.
*/
if (unlikely(!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED)
&& (cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION))){
cm->features = netif_skb_features(skb);
if (likely(sfe_fast_xmit_check(skb, cm->features))) {
cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT;
}
cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED;
}
features = cm->features;
fast_xmit = !!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT);
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
prefetch(skb_shinfo(skb));
/*
* We do per packet condition check before we could fast xmit the
* packet.
*/
if (likely(fast_xmit && dev_fast_xmit(skb, xmit_dev, features))) {
this_cpu_inc(si->stats_pcpu->packets_fast_xmited64);
return 1;
}
/*
* Mark that this packet has been fast forwarded.
*/
skb->fast_forwarded = 1;
dev_queue_xmit(skb);
return 1;
}

21
shortcut-fe/sfe_ipv6_esp.h
View file

@ -1,21 +0,0 @@
/*
* sfe_ipv6_esp.h
* Shortcut forwarding engine - IPv6 ESP header file
*
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
int sfe_ipv6_recv_esp(struct sfe_ipv6 *si, struct sk_buff *skb, struct net_device *dev, unsigned int len,
struct ipv6hdr *iph, unsigned int ihl, bool sync_on_find, bool tun_outer);

356
shortcut-fe/sfe_ipv6_gre.c
View file

@ -1,356 +0,0 @@
/*
* sfe_ipv6_gre.c
* Shortcut forwarding engine file for IPv6 GRE
*
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <linux/skbuff.h>
#include <net/gre.h>
#include <net/protocol.h>
#include <linux/etherdevice.h>
#include <linux/version.h>
#include <net/ip6_checksum.h>
#include "sfe_debug.h"
#include "sfe_api.h"
#include "sfe.h"
#include "sfe_flow_cookie.h"
#include "sfe_ipv6.h"
#include "sfe_pppoe.h"
#include "sfe_vlan.h"
/*
* sfe_ipv6_recv_gre()
* Handle GRE packet receives and forwarding.
*/
int sfe_ipv6_recv_gre(struct sfe_ipv6 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct ipv6hdr *iph, unsigned int ihl, bool sync_on_find,
struct sfe_l2_info *l2_info, bool tun_outer)
{
struct sfe_ipv6_connection_match *cm;
struct sfe_ipv6_addr *dest_ip;
struct sfe_ipv6_addr *src_ip;
struct net_device *xmit_dev;
bool bridge_flow;
bool passthrough;
bool ret;
/*
* Is our packet too short to contain a valid UDP header?
*/
if (!pskb_may_pull(skb, (sizeof(struct gre_base_hdr) + ihl))) {
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_GRE_HEADER_INCOMPLETE);
DEBUG_TRACE("packet too short for GRE header\n");
return 0;
}
/*
* Read the IP address and port information. Read the IP header data first
* because we've almost certainly got that in the cache. We may not yet have
* the UDP header cached though so allow more time for any prefetching.
*/
src_ip = (struct sfe_ipv6_addr *)iph->saddr.s6_addr32;
dest_ip = (struct sfe_ipv6_addr *)iph->daddr.s6_addr32;
rcu_read_lock();
/*
* Look for a connection match.
*/
#ifdef CONFIG_NF_FLOW_COOKIE
cm = si->sfe_flow_cookie_table[skb->flow_cookie & SFE_FLOW_COOKIE_MASK].match;
if (unlikely(!cm)) {
cm = sfe_ipv6_find_connection_match_rcu(si, dev, IPPROTO_GRE, src_ip, 0, dest_ip, 0);
}
#else
cm = sfe_ipv6_find_connection_match_rcu(si, dev, IPPROTO_GRE, src_ip, 0, dest_ip, 0);
#endif
if (unlikely(!cm)) {
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_GRE_NO_CONNECTION);
DEBUG_TRACE("no connection match found dev %s src ip %pI6 dest ip %pI6\n", dev->name, src_ip, dest_ip);
return 0;
}
/*
* Do we expect an ingress VLAN tag for this flow?
*/
if (unlikely(!sfe_vlan_validate_ingress_tag(skb, cm->ingress_vlan_hdr_cnt, cm->ingress_vlan_hdr, l2_info))) {
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_INGRESS_VLAN_TAG_MISMATCH);
DEBUG_TRACE("VLAN tag mismatch. skb=%px\n", skb);
return 0;
}
/*
* Source interface validate.
*/
if (unlikely((cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_SRC_INTERFACE_CHECK) && (cm->match_dev != dev))) {
if (!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_SRC_INTERFACE_CHECK_NO_FLUSH)) {
struct sfe_ipv6_connection *c = cm->connection;
int ret;
DEBUG_TRACE("flush on source interface check failure\n");
spin_lock_bh(&si->lock);
ret = sfe_ipv6_remove_connection(si, c);
spin_unlock_bh(&si->lock);
if (ret) {
sfe_ipv6_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
}
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_INVALID_SRC_IFACE);
DEBUG_TRACE("exception the packet on source interface check failure\n");
return 0;
}
passthrough = cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_PASSTHROUGH;
/*
* If our packet has beern marked as "sync on find" we can't actually
* forward it in the fast path, but now that we've found an associated
* connection we need sync its status before exception it to slow path. unless
* it is passthrough packet.
* TODO: revisit to ensure that pass through traffic is not bypassing firewall for fragmented cases
*/
if (unlikely(sync_on_find) && !passthrough) {
sfe_ipv6_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_GRE_IP_OPTIONS_OR_INITIAL_FRAGMENT);
DEBUG_TRACE("Sync on find\n");
return 0;
}
bridge_flow = !!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_BRIDGE_FLOW);
/*
* Does our hop_limit allow forwarding?
*/
if (!bridge_flow && (iph->hop_limit < 2) && passthrough) {
sfe_ipv6_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_GRE_SMALL_TTL);
DEBUG_TRACE("hop_limit too low\n");
return 0;
}
/*
* Check if skb was cloned. If it was, unshare it. Because
* the data area is going to be written in this path and we don't want to
* change the cloned skb's data section.
*/
if (unlikely(skb_cloned(skb))) {
DEBUG_TRACE("%px: skb is a cloned skb\n", skb);
skb = skb_unshare(skb, GFP_ATOMIC);
if (!skb) {
DEBUG_WARN("Failed to unshare the cloned skb\n");
rcu_read_unlock();
return 1;
}
/*
* Update the iph and udph pointers with the unshared skb's data area.
*/
iph = (struct ipv6hdr *)skb->data;
}
/*
* For PPPoE packets, match server MAC and session id
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_PPPOE_DECAP)) {
struct ethhdr *eth;
bool pppoe_match;
if (unlikely(!sfe_l2_parse_flag_check(l2_info, SFE_L2_PARSE_FLAGS_PPPOE_INGRESS))) {
rcu_read_unlock();
DEBUG_TRACE("%px: PPPoE header not present in packet for PPPoE rule\n", skb);
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_INCORRECT_PPPOE_PARSING);
return 0;
}
eth = eth_hdr(skb);
pppoe_match = (cm->pppoe_session_id == sfe_l2_pppoe_session_id_get(l2_info)) &&
ether_addr_equal((u8*)cm->pppoe_remote_mac, (u8 *)eth->h_source);
if (unlikely(!pppoe_match)) {
DEBUG_TRACE("%px: PPPoE sessions ID %d and %d or MAC %pM and %pM did not match\n",
skb, cm->pppoe_session_id, sfe_l2_pppoe_session_id_get(l2_info),
cm->pppoe_remote_mac, eth->h_source);
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_INVALID_PPPOE_SESSION);
return 0;
}
skb->protocol = htons(l2_info->protocol);
this_cpu_inc(si->stats_pcpu->pppoe_decap_packets_forwarded64);
} else if (unlikely(sfe_l2_parse_flag_check(l2_info, SFE_L2_PARSE_FLAGS_PPPOE_INGRESS))) {
/*
* If packet contains PPPoE header but CME doesn't contain PPPoE flag yet we are exceptioning the packet to linux
*/
if (unlikely(!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_BRIDGE_FLOW))) {
rcu_read_unlock();
DEBUG_TRACE("%px: CME doesn't contain PPPoE flag but packet has PPPoE header\n", skb);
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_PPPOE_NOT_SET_IN_CME);
return 0;
}
/*
* For bridged flows when packet contains PPPoE header, restore the header back and forward to xmit interface
*/
__skb_push(skb, (sizeof(struct pppoe_hdr) + sizeof(struct sfe_ppp_hdr)));
this_cpu_inc(si->stats_pcpu->pppoe_bridge_packets_forwarded64);
}
/*
* protocol handler will be valid only in decap-path.
*/
if (cm->proto) {
struct inet6_protocol *ipprot = cm->proto;
skb_pull(skb, ihl);
skb_reset_transport_header(skb);
skb->fast_forwarded = 1;
ret = ipprot->handler(skb);
if (ret) {
this_cpu_inc(si->stats_pcpu->packets_not_forwarded64);
rcu_read_unlock();
return 1;
}
/*
* Update traffic stats.
*/
atomic_inc(&cm->rx_packet_count);
atomic_add(len, &cm->rx_byte_count);
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
rcu_read_unlock();
DEBUG_TRACE("%p: %s decap done\n",skb, __func__);
return 1;
}
/*
* Check if skb has enough headroom to write L2 headers
*/
if (unlikely(skb_headroom(skb) < cm->l2_hdr_size)) {
rcu_read_unlock();
DEBUG_WARN("%px: Not enough headroom: %u\n", skb, skb_headroom(skb));
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_NO_HEADROOM);
return 0;
}
/*
* If our packet is larger than the MTU of the transmit interface then
* we can't forward it easily.
*/
if (unlikely(len > cm->xmit_dev_mtu)) {
sfe_ipv6_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_GRE_NEEDS_FRAGMENTATION);
DEBUG_TRACE("Larger than MTU\n");
return 0;
}
/*
* Update DSCP
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_DSCP_REMARK)) {
sfe_ipv6_change_dsfield(iph, cm->dscp);
}
iph->hop_limit -= (u8)(!bridge_flow & !tun_outer);
/*
* Update traffic stats.
*/
atomic_inc(&cm->rx_packet_count);
atomic_add(len, &cm->rx_byte_count);
xmit_dev = cm->xmit_dev;
skb->dev = xmit_dev;
/*
* For PPPoE flows, add PPPoE header before L2 header is added.
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_PPPOE_ENCAP)) {
sfe_pppoe_add_header(skb, cm->pppoe_session_id, PPP_IPV6);
this_cpu_inc(si->stats_pcpu->pppoe_encap_packets_forwarded64);
}
/*
* Check to see if we need to add VLAN tags
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_INSERT_EGRESS_VLAN_TAG)) {
sfe_vlan_add_tag(skb, cm->egress_vlan_hdr_cnt, cm->egress_vlan_hdr);
}
if (cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR) {
/*
* For the simple case we write this really fast.
*/
struct ethhdr *eth = (struct ethhdr *)__skb_push(skb, ETH_HLEN);
eth->h_proto = skb->protocol;
ether_addr_copy((u8 *)eth->h_dest, (u8 *)cm->xmit_dest_mac);
ether_addr_copy((u8 *)eth->h_source, (u8 *)cm->xmit_src_mac);
} else if (cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_WRITE_L2_HDR) {
dev_hard_header(skb, xmit_dev, ntohs(skb->protocol),
cm->xmit_dest_mac, cm->xmit_src_mac, len);
}
/*
* Update priority of skb.
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_PRIORITY_REMARK)) {
skb->priority = cm->priority;
}
/*
* Mark outgoing packet.
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_MARK)) {
skb->mark = cm->mark;
}
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
/*
* We're going to check for GSO flags when we transmit the packet so
* start fetching the necessary cache line now.
*/
prefetch(skb_shinfo(skb));
/*
* Mark that this packet has been fast forwarded.
*/
skb->fast_forwarded = 1;
/*
* Send the packet on its way.
*/
dev_queue_xmit(skb);
return 1;
}

22
shortcut-fe/sfe_ipv6_gre.h
View file

@ -1,22 +0,0 @@
/*
* sfe_ipv6_gre.h
* Shortcut forwarding engine header file for IPv6 GRE
*
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
int sfe_ipv6_recv_gre(struct sfe_ipv6 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct ipv6hdr *iph, unsigned int ihl, bool sync_on_find,
struct sfe_l2_info *l2_info, bool tun_outer);

207
shortcut-fe/sfe_ipv6_icmp.c
View file

@ -1,207 +0,0 @@
/*
* sfe_ipv6_icmp.c
* Shortcut forwarding engine file for IPv6 ICMP
*
* Copyright (c) 2015-2016, 2019-2020, The Linux Foundation. All rights reserved.
* Copyright (c) 2021 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <linux/skbuff.h>
#include <linux/ip.h>
#include <net/udp.h>
#include <net/tcp.h>
#include <net/icmp.h>
#include <linux/etherdevice.h>
#include <linux/version.h>
#include "sfe_debug.h"
#include "sfe_api.h"
#include "sfe.h"
#include "sfe_flow_cookie.h"
#include "sfe_ipv6.h"
/*
* sfe_ipv6_recv_icmp()
* Handle ICMP packet receives.
*
* ICMP packets aren't handled as a "fast path" and always have us process them
* through the default Linux stack. What we do need to do is look for any errors
* about connections we are handling in the fast path. If we find any such
* connections then we want to flush their state so that the ICMP error path
* within Linux has all of the correct state should it need it.
*/
int sfe_ipv6_recv_icmp(struct sfe_ipv6 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct ipv6hdr *iph, unsigned int ihl)
{
struct icmp6hdr *icmph;
struct ipv6hdr *icmp_iph;
struct udphdr *icmp_udph;
struct tcphdr *icmp_tcph;
struct sfe_ipv6_addr *src_ip;
struct sfe_ipv6_addr *dest_ip;
__be16 src_port;
__be16 dest_port;
struct sfe_ipv6_connection_match *cm;
struct sfe_ipv6_connection *c;
u8 next_hdr;
bool ret;
/*
* Is our packet too short to contain a valid ICMP header?
*/
len -= ihl;
if (!pskb_may_pull(skb, ihl + sizeof(struct icmp6hdr))) {
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_ICMP_HEADER_INCOMPLETE);
DEBUG_TRACE("packet too short for ICMP header\n");
return 0;
}
/*
* We only handle "destination unreachable" and "time exceeded" messages.
*/
icmph = (struct icmp6hdr *)(skb->data + ihl);
if ((icmph->icmp6_type != ICMPV6_DEST_UNREACH)
&& (icmph->icmp6_type != ICMPV6_TIME_EXCEED)) {
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_ICMP_UNHANDLED_TYPE);
DEBUG_TRACE("unhandled ICMP type: 0x%x\n", icmph->icmp6_type);
return 0;
}
/*
* Do we have the full embedded IP header?
* We should have 8 bytes of next L4 header - that's enough to identify
* the connection.
*/
len -= sizeof(struct icmp6hdr);
ihl += sizeof(struct icmp6hdr);
if (!pskb_may_pull(skb, ihl + sizeof(struct ipv6hdr) + sizeof(struct sfe_ipv6_ext_hdr))) {
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_ICMP_IPV6_HEADER_INCOMPLETE);
DEBUG_TRACE("Embedded IP header not complete\n");
return 0;
}
/*
* Is our embedded IP version wrong?
*/
icmp_iph = (struct ipv6hdr *)(icmph + 1);
if (unlikely(icmp_iph->version != 6)) {
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_ICMP_IPV6_NON_V6);
DEBUG_TRACE("IP version: %u\n", icmp_iph->version);
return 0;
}
len -= sizeof(struct ipv6hdr);
ihl += sizeof(struct ipv6hdr);
next_hdr = icmp_iph->nexthdr;
while (unlikely(sfe_ipv6_is_ext_hdr(next_hdr))) {
struct sfe_ipv6_ext_hdr *ext_hdr;
unsigned int ext_hdr_len;
ext_hdr = (struct sfe_ipv6_ext_hdr *)(skb->data + ihl);
if (next_hdr == NEXTHDR_FRAGMENT) {
struct frag_hdr *frag_hdr = (struct frag_hdr *)ext_hdr;
unsigned int frag_off = ntohs(frag_hdr->frag_off);
if (frag_off & SFE_IPV6_FRAG_OFFSET) {
DEBUG_TRACE("non-initial fragment\n");
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_NON_INITIAL_FRAGMENT);
return 0;
}
}
ext_hdr_len = ext_hdr->hdr_len;
ext_hdr_len <<= 3;
ext_hdr_len += sizeof(struct sfe_ipv6_ext_hdr);
len -= ext_hdr_len;
ihl += ext_hdr_len;
/*
* We should have 8 bytes of next header - that's enough to identify
* the connection.
*/
if (!pskb_may_pull(skb, ihl + sizeof(struct sfe_ipv6_ext_hdr))) {
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_HEADER_INCOMPLETE);
DEBUG_TRACE("extension header %d not completed\n", next_hdr);
return 0;
}
next_hdr = ext_hdr->next_hdr;
}
/*
* Handle the embedded transport layer header.
*/
switch (next_hdr) {
case IPPROTO_UDP:
icmp_udph = (struct udphdr *)(skb->data + ihl);
src_port = icmp_udph->source;
dest_port = icmp_udph->dest;
break;
case IPPROTO_TCP:
icmp_tcph = (struct tcphdr *)(skb->data + ihl);
src_port = icmp_tcph->source;
dest_port = icmp_tcph->dest;
break;
default:
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_ICMP_IPV6_UNHANDLED_PROTOCOL);
DEBUG_TRACE("Unhandled embedded IP protocol: %u\n", next_hdr);
return 0;
}
src_ip = (struct sfe_ipv6_addr *)icmp_iph->saddr.s6_addr32;
dest_ip = (struct sfe_ipv6_addr *)icmp_iph->daddr.s6_addr32;
rcu_read_lock();
/*
* Look for a connection match. Note that we reverse the source and destination
* here because our embedded message contains a packet that was sent in the
* opposite direction to the one in which we just received it. It will have
* been sent on the interface from which we received it though so that's still
* ok to use.
*/
cm = sfe_ipv6_find_connection_match_rcu(si, dev, icmp_iph->nexthdr, dest_ip, dest_port, src_ip, src_port);
if (unlikely(!cm)) {
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_ICMP_NO_CONNECTION);
DEBUG_TRACE("no connection found\n");
return 0;
}
/*
* We found a connection so now remove it from the connection list and flush
* its state.
*/
c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv6_remove_connection(si, c);
spin_unlock_bh(&si->lock);
if (ret) {
sfe_ipv6_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_ICMP_FLUSHED_CONNECTION);
return 0;
}

22
shortcut-fe/sfe_ipv6_icmp.h
View file

@ -1,22 +0,0 @@
/*
* sfe_ipv6_icmp.h
* Shortcut forwarding engine header file for IPv6 ICMP
*
* Copyright (c) 2015-2016, 2019-2020, The Linux Foundation. All rights reserved.
* Copyright (c) 2021 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
int sfe_ipv6_recv_icmp(struct sfe_ipv6 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct ipv6hdr *iph, unsigned int ihl);

207
shortcut-fe/sfe_ipv6_pppoe_br.c
View file

@ -1,207 +0,0 @@
/*
* sfe_ipv6_pppoe_br.c
* Shortcut forwarding engine - IPv6 PPPoE bridge implementation
*
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <linux/skbuff.h>
#include <net/udp.h>
#include <linux/etherdevice.h>
#include <linux/version.h>
#include "sfe_debug.h"
#include "sfe_api.h"
#include "sfe.h"
#include "sfe_ipv6.h"
#include "sfe_pppoe.h"
#include "sfe_vlan.h"
/*
* sfe_ipv6_recv_pppoe_bridge()
* Process PPPoE bridge packets using 3-tuple acceleration
*
*/
int sfe_ipv6_recv_pppoe_bridge(struct sfe_ipv6 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct ipv6hdr *iph, unsigned int ihl, struct sfe_l2_info *l2_info)
{
struct sfe_ipv6_connection_match *cm;
u32 service_class_id;
struct net_device *xmit_dev;
int ret;
bool fast_xmit;
netdev_features_t features;
rcu_read_lock();
cm = sfe_ipv6_find_connection_match_rcu(si, dev, IPPROTO_RAW,
(struct sfe_ipv6_addr *)iph->saddr.s6_addr32, 0,
(struct sfe_ipv6_addr *)iph->daddr.s6_addr32,
htons(sfe_l2_pppoe_session_id_get(l2_info)));
if (unlikely(!cm)) {
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_PPPOE_BR_NOT_IN_CME);
DEBUG_TRACE("%px: no connection found in 3-tuple lookup for PPPoE bridge flow\n", skb);
return 0;
}
/*
* Source interface validate.
*/
if (unlikely((cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_SRC_INTERFACE_CHECK) && (cm->match_dev != dev))) {
if (!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_SRC_INTERFACE_CHECK_NO_FLUSH)) {
struct sfe_ipv6_connection *c = cm->connection;
DEBUG_TRACE("flush on source interface check failure\n");
spin_lock_bh(&si->lock);
ret = sfe_ipv6_remove_connection(si, c);
spin_unlock_bh(&si->lock);
if (ret) {
sfe_ipv6_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
}
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_INVALID_SRC_IFACE);
DEBUG_TRACE("exception the packet on source interface check failure\n");
return 0;
}
/*
* Do we expect an ingress VLAN tag for this flow?
*/
if (unlikely(!sfe_vlan_validate_ingress_tag(skb, cm->ingress_vlan_hdr_cnt, cm->ingress_vlan_hdr, l2_info))) {
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_INGRESS_VLAN_TAG_MISMATCH);
DEBUG_TRACE("VLAN tag mismatch. skb=%px\n", skb);
return 0;
}
/*
* Check if skb has enough headroom to write L2 headers
*/
if (unlikely(skb_headroom(skb) < cm->l2_hdr_size)) {
rcu_read_unlock();
DEBUG_WARN("%px: Not enough headroom: %u\n", skb, skb_headroom(skb));
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_NO_HEADROOM);
return 0;
}
/*
* Restore PPPoE header back
*/
__skb_push(skb, PPPOE_SES_HLEN);
/*
* Update traffic stats.
*/
atomic_inc(&cm->rx_packet_count);
atomic_add(len, &cm->rx_byte_count);
xmit_dev = cm->xmit_dev;
skb->dev = xmit_dev;
/*
* Check to see if we need to add VLAN tags
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_INSERT_EGRESS_VLAN_TAG)) {
sfe_vlan_add_tag(skb, cm->egress_vlan_hdr_cnt, cm->egress_vlan_hdr);
}
/*
* Check to see if we need to write an Ethernet header.
*/
if (likely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_WRITE_L2_HDR)) {
if (unlikely(!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR))) {
dev_hard_header(skb, xmit_dev, ntohs(skb->protocol),
cm->xmit_dest_mac, cm->xmit_src_mac, len);
} else {
/*
* For the simple case we write this really fast.
*/
struct ethhdr *eth = (struct ethhdr *)__skb_push(skb, ETH_HLEN);
eth->h_proto = skb->protocol;
ether_addr_copy((u8 *)eth->h_dest, (u8 *)cm->xmit_dest_mac);
ether_addr_copy((u8 *)eth->h_source, (u8 *)cm->xmit_src_mac);
}
}
/*
* Update priority of skb.
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_PRIORITY_REMARK)) {
skb->priority = cm->priority;
}
/*
* Mark outgoing packet.
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_MARK)) {
skb->mark = cm->mark;
/*
* Update service class stats if SAWF is valid.
*/
if (likely(cm->sawf_valid)) {
service_class_id = SFE_GET_SAWF_SERVICE_CLASS(cm->mark);
sfe_ipv6_service_class_stats_inc(si, service_class_id, len);
}
}
/*
* For the first packets, check if it could got fast xmit.
*/
if (unlikely(!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED)
&& (cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION))){
cm->features = netif_skb_features(skb);
if (likely(sfe_fast_xmit_check(skb, cm->features))) {
cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT;
}
cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED;
}
features = cm->features;
fast_xmit = !!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT);
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->pppoe_bridge_packets_3tuple_forwarded64);
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
/*
* We're going to check for GSO flags when we transmit the packet so
* start fetching the necessary cache line now.
*/
prefetch(skb_shinfo(skb));
/*
* We do per packet condition check before we could fast xmit the
* packet.
*/
if (likely(fast_xmit && dev_fast_xmit(skb, xmit_dev, features))) {
this_cpu_inc(si->stats_pcpu->packets_fast_xmited64);
return 1;
}
/*
* Mark that this packet has been fast forwarded.
*/
skb->fast_forwarded = 1;
/*
* Send the packet on its way.
*/
dev_queue_xmit(skb);
return 1;
}

20
shortcut-fe/sfe_ipv6_pppoe_br.h
View file

@ -1,20 +0,0 @@
/*
* sfe_ipv6_pppoe_br.h
* Shortcut forwarding engine - IPv6 PPPoE bridge header file
*
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
int sfe_ipv6_recv_pppoe_bridge(struct sfe_ipv6 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct ipv6hdr *iph, unsigned int ihl, struct sfe_l2_info *l2_info);

765
shortcut-fe/sfe_ipv6_tcp.c
View file

@ -1,765 +0,0 @@
/*
* sfe_ipv6_tcp.c
* Shortcut forwarding engine file for IPv6 TCP
*
* Copyright (c) 2015-2016, 2019-2020, The Linux Foundation. All rights reserved.
* Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <linux/skbuff.h>
#include <net/tcp.h>
#include <linux/etherdevice.h>
#include <linux/version.h>
#include "sfe_debug.h"
#include "sfe_api.h"
#include "sfe.h"
#include "sfe_flow_cookie.h"
#include "sfe_ipv6.h"
#include "sfe_pppoe.h"
#include "sfe_vlan.h"
/*
* sfe_ipv6_process_tcp_option_sack()
* Parse TCP SACK option and update ack according
*/
static bool sfe_ipv6_process_tcp_option_sack(const struct tcphdr *th, const u32 data_offs,
u32 *ack)
{
u32 length = sizeof(struct tcphdr);
u8 *ptr = (u8 *)th + length;
/*
* Ignore processing if TCP packet has only TIMESTAMP option.
*/
if (likely(data_offs == length + TCPOLEN_TIMESTAMP + 1 + 1)
&& likely(ptr[0] == TCPOPT_NOP)
&& likely(ptr[1] == TCPOPT_NOP)
&& likely(ptr[2] == TCPOPT_TIMESTAMP)
&& likely(ptr[3] == TCPOLEN_TIMESTAMP)) {
return true;
}
/*
* TCP options. Parse SACK option.
*/
while (length < data_offs) {
u8 size;
u8 kind;
ptr = (u8 *)th + length;
kind = *ptr;
/*
* NOP, for padding
* Not in the switch because to fast escape and to not calculate size
*/
if (kind == TCPOPT_NOP) {
length++;
continue;
}
if (kind == TCPOPT_SACK) {
u32 sack = 0;
u8 re = 1 + 1;
size = *(ptr + 1);
if ((size < (1 + 1 + TCPOLEN_SACK_PERBLOCK))
|| ((size - (1 + 1)) % (TCPOLEN_SACK_PERBLOCK))
|| (size > (data_offs - length))) {
return false;
}
re += 4;
while (re < size) {
u32 sack_re;
u8 *sptr = ptr + re;
sack_re = (sptr[0] << 24) | (sptr[1] << 16) | (sptr[2] << 8) | sptr[3];
if (sack_re > sack) {
sack = sack_re;
}
re += TCPOLEN_SACK_PERBLOCK;
}
if (sack > *ack) {
*ack = sack;
}
length += size;
continue;
}
if (kind == TCPOPT_EOL) {
return true;
}
size = *(ptr + 1);
if (size < 2) {
return false;
}
length += size;
}
return true;
}
/*
* sfe_ipv6_recv_tcp()
* Handle TCP packet receives and forwarding.
*/
int sfe_ipv6_recv_tcp(struct sfe_ipv6 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct ipv6hdr *iph, unsigned int ihl, bool sync_on_find, struct sfe_l2_info *l2_info)
{
struct tcphdr *tcph;
struct sfe_ipv6_addr *src_ip;
struct sfe_ipv6_addr *dest_ip;
__be16 src_port;
__be16 dest_port;
struct sfe_ipv6_connection_match *cm;
struct sfe_ipv6_connection_match *counter_cm;
u32 flags;
u32 service_class_id;
struct net_device *xmit_dev;
bool ret;
bool hw_csum;
bool bridge_flow;
bool fast_xmit;
netdev_features_t features;
/*
* Is our packet too short to contain a valid TCP header?
*/
if (!pskb_may_pull(skb, (sizeof(struct tcphdr) + ihl))) {
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_HEADER_INCOMPLETE);
DEBUG_TRACE("packet too short for TCP header\n");
return 0;
}
/*
* Read the IP address and port information. Read the IP header data first
* because we've almost certainly got that in the cache. We may not yet have
* the TCP header cached though so allow more time for any prefetching.
*/
src_ip = (struct sfe_ipv6_addr *)iph->saddr.s6_addr32;
dest_ip = (struct sfe_ipv6_addr *)iph->daddr.s6_addr32;
tcph = (struct tcphdr *)(skb->data + ihl);
src_port = tcph->source;
dest_port = tcph->dest;
flags = tcp_flag_word(tcph);
rcu_read_lock();
/*
* Look for a connection match.
*/
#ifdef CONFIG_NF_FLOW_COOKIE
cm = si->sfe_flow_cookie_table[skb->flow_cookie & SFE_FLOW_COOKIE_MASK].match;
if (unlikely(!cm)) {
cm = sfe_ipv6_find_connection_match_rcu(si, dev, IPPROTO_TCP, src_ip, src_port, dest_ip, dest_port);
}
#else
cm = sfe_ipv6_find_connection_match_rcu(si, dev, IPPROTO_TCP, src_ip, src_port, dest_ip, dest_port);
#endif
if (unlikely(!cm)) {
/*
* We didn't get a connection but as TCP is connection-oriented that
* may be because this is a non-fast connection (not running established).
* For diagnostic purposes we differentiate this here.
*/
if (likely((flags & (TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_FIN | TCP_FLAG_ACK)) == TCP_FLAG_ACK)) {
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_NO_CONNECTION_FAST_FLAGS);
DEBUG_TRACE("no connection found - fast flags\n");
return 0;
}
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_NO_CONNECTION_SLOW_FLAGS);
DEBUG_TRACE("no connection found - slow flags: 0x%x\n",
flags & (TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_FIN | TCP_FLAG_ACK));
return 0;
}
/*
* Source interface validate.
*/
if (unlikely((cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_SRC_INTERFACE_CHECK) && (cm->match_dev != dev))) {
if (!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_SRC_INTERFACE_CHECK_NO_FLUSH)) {
struct sfe_ipv6_connection *c = cm->connection;
DEBUG_TRACE("flush on source interface check failure\n");
spin_lock_bh(&si->lock);
ret = sfe_ipv6_remove_connection(si, c);
spin_unlock_bh(&si->lock);
if (ret) {
sfe_ipv6_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
}
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_INVALID_SRC_IFACE);
DEBUG_TRACE("exception the packet on source interface check failure\n");
return 0;
}
/*
* If our packet has been marked as "sync on find" we can't actually
* forward it in the fast path, but now that we've found an associated
* connection we need sync its status before throw it slow path.
*/
if (unlikely(sync_on_find)) {
sfe_ipv6_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_IP_OPTIONS_OR_INITIAL_FRAGMENT);
DEBUG_TRACE("Sync on find\n");
return 0;
}
#ifdef CONFIG_XFRM
/*
* We can't accelerate the flow on this direction, just let it go
* through the slow path.
*/
if (unlikely(!cm->flow_accel)) {
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_not_forwarded64);
return 0;
}
#endif
/*
* Do we expect an ingress VLAN tag for this flow?
*/
if (unlikely(!sfe_vlan_validate_ingress_tag(skb, cm->ingress_vlan_hdr_cnt, cm->ingress_vlan_hdr, l2_info))) {
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_INGRESS_VLAN_TAG_MISMATCH);
DEBUG_TRACE("VLAN tag mismatch. skb=%px\n", skb);
return 0;
}
bridge_flow = !!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_BRIDGE_FLOW);
/*
* Does our hop_limit allow forwarding?
*/
if (likely(!bridge_flow)) {
if (unlikely(iph->hop_limit < 2)) {
sfe_ipv6_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_SMALL_TTL);
DEBUG_TRACE("hop_limit too low\n");
return 0;
}
}
/*
* If our packet is larger than the MTU of the transmit interface then
* we can't forward it easily.
*/
if (unlikely((len > cm->xmit_dev_mtu) && !skb_is_gso(skb))) {
sfe_ipv6_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_NEEDS_FRAGMENTATION);
DEBUG_TRACE("Larger than MTU\n");
return 0;
}
/*
* Look at our TCP flags. Anything missing an ACK or that has RST, SYN or FIN
* set is not a fast path packet.
*/
if (unlikely((flags & (TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_FIN | TCP_FLAG_ACK)) != TCP_FLAG_ACK)) {
struct sfe_ipv6_connection *c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv6_remove_connection(si, c);
spin_unlock_bh(&si->lock);
DEBUG_TRACE("TCP flags: %#x are not fast. %u->%u skb=%px\n",
htonl(flags), htons(src_port), htons(dest_port), skb);
if (ret) {
sfe_ipv6_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_FLAGS);
return 0;
}
counter_cm = cm->counter_match;
/*
* Are we doing sequence number checking?
*/
if (likely(!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_NO_SEQ_CHECK))) {
u32 seq;
u32 ack;
u32 sack;
u32 data_offs;
u32 end;
u32 left_edge;
u32 scaled_win;
u32 max_end;
/*
* Is our sequence fully past the right hand edge of the window?
*/
seq = ntohl(tcph->seq);
if (unlikely((s32)(seq - (cm->protocol_state.tcp.max_end + 1)) > 0)) {
struct sfe_ipv6_connection *c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv6_remove_connection(si, c);
spin_unlock_bh(&si->lock);
DEBUG_TRACE("seq: %u exceeds right edge: %u\n",
seq, cm->protocol_state.tcp.max_end + 1);
if (ret) {
sfe_ipv6_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_SEQ_EXCEEDS_RIGHT_EDGE);
return 0;
}
/*
* Check that our TCP data offset isn't too short.
*/
data_offs = tcph->doff << 2;
if (unlikely(data_offs < sizeof(struct tcphdr))) {
struct sfe_ipv6_connection *c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv6_remove_connection(si, c);
spin_unlock_bh(&si->lock);
DEBUG_TRACE("TCP data offset: %u, too small\n", data_offs);
if (ret) {
sfe_ipv6_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_SMALL_DATA_OFFS);
return 0;
}
/*
* Update ACK according to any SACK option.
*/
ack = ntohl(tcph->ack_seq);
sack = ack;
if (unlikely(!sfe_ipv6_process_tcp_option_sack(tcph, data_offs, &sack))) {
struct sfe_ipv6_connection *c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv6_remove_connection(si, c);
spin_unlock_bh(&si->lock);
DEBUG_TRACE("TCP option SACK size is wrong\n");
if (ret) {
sfe_ipv6_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_BAD_SACK);
return 0;
}
/*
* Check that our TCP data offset isn't past the end of the packet.
*/
data_offs += sizeof(struct ipv6hdr);
if (unlikely(len < data_offs)) {
struct sfe_ipv6_connection *c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv6_remove_connection(si, c);
spin_unlock_bh(&si->lock);
DEBUG_TRACE("TCP data offset: %u, past end of packet: %u\n",
data_offs, len);
if (ret) {
sfe_ipv6_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_BIG_DATA_OFFS);
return 0;
}
end = seq + len - data_offs;
/*
* Is our sequence fully before the left hand edge of the window?
*/
if (unlikely((s32)(end - (cm->protocol_state.tcp.end
- counter_cm->protocol_state.tcp.max_win - 1)) < 0)) {
struct sfe_ipv6_connection *c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv6_remove_connection(si, c);
spin_unlock_bh(&si->lock);
DEBUG_TRACE("seq: %u before left edge: %u\n",
end, cm->protocol_state.tcp.end - counter_cm->protocol_state.tcp.max_win - 1);
if (ret) {
sfe_ipv6_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_SEQ_BEFORE_LEFT_EDGE);
return 0;
}
/*
* Are we acking data that is to the right of what has been sent?
*/
if (unlikely((s32)(sack - (counter_cm->protocol_state.tcp.end + 1)) > 0)) {
struct sfe_ipv6_connection *c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv6_remove_connection(si, c);
spin_unlock_bh(&si->lock);
DEBUG_TRACE("ack: %u exceeds right edge: %u\n",
sack, counter_cm->protocol_state.tcp.end + 1);
if (ret) {
sfe_ipv6_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_ACK_EXCEEDS_RIGHT_EDGE);
return 0;
}
/*
* Is our ack too far before the left hand edge of the window?
*/
left_edge = counter_cm->protocol_state.tcp.end
- cm->protocol_state.tcp.max_win
- SFE_IPV6_TCP_MAX_ACK_WINDOW
- 1;
if (unlikely((s32)(sack - left_edge) < 0)) {
struct sfe_ipv6_connection *c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv6_remove_connection(si, c);
spin_unlock_bh(&si->lock);
DEBUG_TRACE("ack: %u before left edge: %u\n", sack, left_edge);
if (ret) {
sfe_ipv6_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_ACK_BEFORE_LEFT_EDGE);
return 0;
}
/*
* Have we just seen the largest window size yet for this connection? If yes
* then we need to record the new value.
*/
scaled_win = ntohs(tcph->window) << cm->protocol_state.tcp.win_scale;
scaled_win += (sack - ack);
if (unlikely(cm->protocol_state.tcp.max_win < scaled_win)) {
cm->protocol_state.tcp.max_win = scaled_win;
}
/*
* If our sequence and/or ack numbers have advanced then record the new state.
*/
if (likely((s32)(end - cm->protocol_state.tcp.end) >= 0)) {
cm->protocol_state.tcp.end = end;
}
max_end = sack + scaled_win;
if (likely((s32)(max_end - counter_cm->protocol_state.tcp.max_end) >= 0)) {
counter_cm->protocol_state.tcp.max_end = max_end;
}
}
/*
* Check if skb was cloned. If it was, unshare it. Because
* the data area is going to be written in this path and we don't want to
* change the cloned skb's data section.
*/
if (unlikely(skb_cloned(skb))) {
DEBUG_TRACE("%px: skb is a cloned skb\n", skb);
skb = skb_unshare(skb, GFP_ATOMIC);
if (!skb) {
DEBUG_WARN("Failed to unshare the cloned skb\n");
rcu_read_unlock();
return 0;
}
/*
* Update the iph and tcph pointers with the unshared skb's data area.
*/
iph = (struct ipv6hdr *)skb->data;
tcph = (struct tcphdr *)(skb->data + ihl);
}
/*
* Check if skb has enough headroom to write L2 headers
*/
if (unlikely(skb_headroom(skb) < cm->l2_hdr_size)) {
rcu_read_unlock();
DEBUG_WARN("%px: Not enough headroom: %u\n", skb, skb_headroom(skb));
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_NO_HEADROOM);
return 0;
}
/*
* For PPPoE packets, match server MAC and session id
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_PPPOE_DECAP)) {
struct ethhdr *eth;
bool pppoe_match;
if (unlikely(!sfe_l2_parse_flag_check(l2_info, SFE_L2_PARSE_FLAGS_PPPOE_INGRESS))) {
rcu_read_unlock();
DEBUG_TRACE("%px: PPPoE header not present in packet for PPPoE rule\n", skb);
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_INCORRECT_PPPOE_PARSING);
return 0;
}
eth = eth_hdr(skb);
pppoe_match = (cm->pppoe_session_id == sfe_l2_pppoe_session_id_get(l2_info)) &&
ether_addr_equal((u8*)cm->pppoe_remote_mac, (u8 *)eth->h_source);
if (unlikely(!pppoe_match)) {
DEBUG_TRACE("%px: PPPoE sessions ID %d and %d or MAC %pM and %pM did not match\n",
skb, cm->pppoe_session_id, sfe_l2_pppoe_session_id_get(l2_info),
cm->pppoe_remote_mac, eth->h_source);
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_INVALID_PPPOE_SESSION);
return 0;
}
skb->protocol = htons(l2_info->protocol);
this_cpu_inc(si->stats_pcpu->pppoe_decap_packets_forwarded64);
} else if (unlikely(sfe_l2_parse_flag_check(l2_info, SFE_L2_PARSE_FLAGS_PPPOE_INGRESS))) {
/*
* If packet contains PPPoE header but CME doesn't contain PPPoE flag yet we are exceptioning the packet to linux
*/
if (unlikely(!bridge_flow)) {
rcu_read_unlock();
DEBUG_TRACE("%px: CME doesn't contain PPPoE flag but packet has PPPoE header\n", skb);
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_PPPOE_NOT_SET_IN_CME);
return 0;
}
/*
* For bridged flows when packet contains PPPoE header, restore the header back and forward to xmit interface
*/
__skb_push(skb, PPPOE_SES_HLEN);
this_cpu_inc(si->stats_pcpu->pppoe_bridge_packets_forwarded64);
}
/*
* From this point on we're good to modify the packet.
*/
/*
* For PPPoE flows, add PPPoE header before L2 header is added.
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_PPPOE_ENCAP)) {
sfe_pppoe_add_header(skb, cm->pppoe_session_id, PPP_IPV6);
this_cpu_inc(si->stats_pcpu->pppoe_encap_packets_forwarded64);
}
/*
* Update DSCP
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_DSCP_REMARK)) {
sfe_ipv6_change_dsfield(iph, cm->dscp);
}
/*
* Decrement our hop_limit.
*/
if (likely(!bridge_flow)) {
iph->hop_limit -= 1;
}
/*
* Enable HW csum if rx checksum is verified and xmit interface is CSUM offload capable.
* Note: If L4 csum at Rx was found to be incorrect, we (router) should use incremental L4 checksum here
* so that HW does not re-calculate/replace the L4 csum
*/
hw_csum = !!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_CSUM_OFFLOAD) && (skb->ip_summed == CHECKSUM_UNNECESSARY);
/*
* Do we have to perform translations of the source address/port?
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_XLATE_SRC)) {
u16 tcp_csum;
u32 sum;
iph->saddr.s6_addr32[0] = cm->xlate_src_ip[0].addr[0];
iph->saddr.s6_addr32[1] = cm->xlate_src_ip[0].addr[1];
iph->saddr.s6_addr32[2] = cm->xlate_src_ip[0].addr[2];
iph->saddr.s6_addr32[3] = cm->xlate_src_ip[0].addr[3];
tcph->source = cm->xlate_src_port;
if (unlikely(!hw_csum)) {
tcp_csum = tcph->check;
sum = tcp_csum + cm->xlate_src_csum_adjustment;
sum = (sum & 0xffff) + (sum >> 16);
tcph->check = (u16)sum;
}
}
/*
* Do we have to perform translations of the destination address/port?
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_XLATE_DEST)) {
u16 tcp_csum;
u32 sum;
iph->daddr.s6_addr32[0] = cm->xlate_dest_ip[0].addr[0];
iph->daddr.s6_addr32[1] = cm->xlate_dest_ip[0].addr[1];
iph->daddr.s6_addr32[2] = cm->xlate_dest_ip[0].addr[2];
iph->daddr.s6_addr32[3] = cm->xlate_dest_ip[0].addr[3];
tcph->dest = cm->xlate_dest_port;
if (unlikely(!hw_csum)) {
tcp_csum = tcph->check;
sum = tcp_csum + cm->xlate_dest_csum_adjustment;
sum = (sum & 0xffff) + (sum >> 16);
tcph->check = (u16)sum;
}
}
/*
* If HW checksum offload is not possible, incremental L4 checksum is used to update the packet.
* Setting ip_summed to CHECKSUM_UNNECESSARY ensures checksum is not recalculated further in packet
* path.
*/
if (likely(hw_csum)) {
skb->ip_summed = CHECKSUM_PARTIAL;
} else {
skb->ip_summed = CHECKSUM_UNNECESSARY;
}
/*
* Update traffic stats.
*/
atomic_inc(&cm->rx_packet_count);
atomic_add(len, &cm->rx_byte_count);
xmit_dev = cm->xmit_dev;
skb->dev = xmit_dev;
/*
* Check to see if we need to add VLAN tags
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_INSERT_EGRESS_VLAN_TAG)) {
sfe_vlan_add_tag(skb, cm->egress_vlan_hdr_cnt, cm->egress_vlan_hdr);
}
/*
* Check to see if we need to write an Ethernet header.
*/
if (likely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_WRITE_L2_HDR)) {
if (unlikely(!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR))) {
dev_hard_header(skb, xmit_dev, ntohs(skb->protocol),
cm->xmit_dest_mac, cm->xmit_src_mac, len);
} else {
/*
* For the simple case we write this really fast.
*/
struct ethhdr *eth = (struct ethhdr *)__skb_push(skb, ETH_HLEN);
eth->h_proto = skb->protocol;
ether_addr_copy((u8 *)eth->h_dest, (u8 *)cm->xmit_dest_mac);
ether_addr_copy((u8 *)eth->h_source, (u8 *)cm->xmit_src_mac);
}
}
/*
* Update priority of skb.
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_PRIORITY_REMARK)) {
skb->priority = cm->priority;
}
/*
* Mark outgoing packet
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_MARK)) {
skb->mark = cm->mark;
/*
* Update service class stats if SAWF is valid.
*/
if (likely(cm->sawf_valid)) {
service_class_id = SFE_GET_SAWF_SERVICE_CLASS(cm->mark);
sfe_ipv6_service_class_stats_inc(si, service_class_id, len);
}
}
/*
* For the first packets, check if it could got fast xmit.
*/
if (unlikely(!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED)
&& (cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION))){
cm->features = netif_skb_features(skb);
if (likely(sfe_fast_xmit_check(skb, cm->features))) {
cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT;
}
cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED;
}
features = cm->features;
fast_xmit = !!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT);
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
/*
* We're going to check for GSO flags when we transmit the packet so
* start fetching the necessary cache line now.
*/
prefetch(skb_shinfo(skb));
/*
* We do per packet condition check before we could fast xmit the
* packet.
*/
if (likely(fast_xmit)) {
if (likely(!skb_is_gso(skb))) {
if (likely(dev_fast_xmit(skb, xmit_dev, features))) {
this_cpu_inc(si->stats_pcpu->packets_fast_xmited64);
return 1;
}
} else {
cm->flags &= ~SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT;
DEBUG_TRACE("%px: fast xmit disabled for xmit dev %s", skb, xmit_dev->name);
}
}
/*
* Mark that this packet has been fast forwarded.
*/
skb->fast_forwarded = 1;
/*
* Send the packet on its way.
*/
dev_queue_xmit(skb);
return 1;
}

23
shortcut-fe/sfe_ipv6_tcp.h
View file

@ -1,23 +0,0 @@
/*
* sfe_ipv6_tcp.h
* Shortcut forwarding engine header file for IPv6 TCP
*
* Copyright (c) 2015-2016, 2019-2020, The Linux Foundation. All rights reserved.
* Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
int sfe_ipv6_recv_tcp(struct sfe_ipv6 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct ipv6hdr *iph, unsigned int ihl,
bool sync_on_find,struct sfe_l2_info *l2_info);

260
shortcut-fe/sfe_ipv6_tunipip6.c
View file

@ -1,260 +0,0 @@
/*
* sfe_ipv6_tunipip6.c
* Shortcut forwarding engine file for IPv6 TUNIPIP6
*
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <linux/skbuff.h>
#include <linux/etherdevice.h>
#include <linux/version.h>
#include <net/ip6_checksum.h>
#include <net/protocol.h>
#include "sfe_debug.h"
#include "sfe_api.h"
#include "sfe.h"
#include "sfe_flow_cookie.h"
#include "sfe_ipv6.h"
#include "sfe_vlan.h"
/*
* sfe_ipv6_recv_tunipip6()
* Handle TUNIPIP6 packet receives and forwarding.
*/
int sfe_ipv6_recv_tunipip6(struct sfe_ipv6 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct ipv6hdr *iph, unsigned int ihl,
bool sync_on_find, struct sfe_l2_info *l2_info, bool tun_outer)
{
struct sfe_ipv6_addr *src_ip;
struct sfe_ipv6_addr *dest_ip;
__be16 src_port = 0;
__be16 dest_port = 0;
unsigned int ihl_tmp = sizeof(struct ipv6hdr);
struct sfe_ipv6_connection_match *cm;
bool non_dst = false;
u8 next_hdr;
DEBUG_TRACE("%px: sfe: sfe_ipv6_recv_tunipip6 called.\n", skb);
/*
* Read the IP address information. Read the IP header data first
* because we've almost certainly got that in the cache.
*/
src_ip = (struct sfe_ipv6_addr *)iph->saddr.s6_addr32;
dest_ip = (struct sfe_ipv6_addr *)iph->daddr.s6_addr32;
rcu_read_lock();
/*
* Look for a connection match.
*/
#ifdef CONFIG_NF_FLOW_COOKIE
cm = si->sfe_flow_cookie_table[skb->flow_cookie & SFE_FLOW_COOKIE_MASK].match;
if (unlikely(!cm)) {
cm = sfe_ipv6_find_connection_match_rcu(si, dev, IPPROTO_IPIP, src_ip, src_port, dest_ip, dest_port);
}
#else
cm = sfe_ipv6_find_connection_match_rcu(si, dev, IPPROTO_IPIP, src_ip, src_port, dest_ip, dest_port);
#endif
if (unlikely(!cm)) {
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TUNIPIP6_NO_CONNECTION);
DEBUG_TRACE("%px: no connection found\n", skb);
return 0;
}
next_hdr = iph->nexthdr;
/*
* Try to find an extension header(if any) that is not NEXTHDR_DEST.
*/
while (unlikely(sfe_ipv6_is_ext_hdr(next_hdr))) {
struct sfe_ipv6_ext_hdr *ext_hdr;
unsigned int ext_hdr_len;
if(next_hdr != NEXTHDR_DEST) {
non_dst = true;
break;
}
ext_hdr = (struct sfe_ipv6_ext_hdr *)(skb->data + ihl_tmp);
ext_hdr_len = ext_hdr->hdr_len;
ext_hdr_len <<= 3;
ext_hdr_len += sizeof(struct sfe_ipv6_ext_hdr);
ihl_tmp += ext_hdr_len;
next_hdr = ext_hdr->next_hdr;
}
/*
* If our packet has been marked as "sync on find" we will sync the status
* and forward it to slowpath, except that encap_limit is set for dslite tunnel
* which is embedded in exthdr type NEXTHDR_DEST.
*/
if (unlikely(sync_on_find && non_dst)) {
sfe_ipv6_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TUNIPIP6_SYNC_ON_FIND);
DEBUG_TRACE("%px: Sync on find\n", skb);
return 0;
}
/*
* If cm->proto is set, it means the decap path.
* Otherwise we forward the packet in encap path.
*/
if(cm->proto) {
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0))
const struct inet6_protocol *ipprot = cm->proto;
#else
struct inet6_protocol *ipprot = cm->proto;
#endif
/*
* Do we expect an ingress VLAN tag for this flow?
* Note: We will only have ingress tag check in decap direction.
*/
if (unlikely(!sfe_vlan_validate_ingress_tag(skb, cm->ingress_vlan_hdr_cnt, cm->ingress_vlan_hdr, l2_info))) {
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_INGRESS_VLAN_TAG_MISMATCH);
DEBUG_TRACE("VLAN tag mismatch. skb=%px\n"
"cm: %u [0]=%x/%x [1]=%x/%x\n"
"l2_info+: %u [0]=%x/%x [1]=%x/%x\n", skb,
cm->ingress_vlan_hdr_cnt,
htons(cm->ingress_vlan_hdr[0].tpid), cm->ingress_vlan_hdr[0].tci,
htons(cm->ingress_vlan_hdr[1].tpid), cm->ingress_vlan_hdr[1].tci,
l2_info->vlan_hdr_cnt,
htons(l2_info->vlan_hdr[0].tpid), l2_info->vlan_hdr[0].tci,
htons(l2_info->vlan_hdr[1].tpid), l2_info->vlan_hdr[1].tci);
return 0;
}
skb_reset_network_header(skb);
skb_pull(skb, ihl);
skb_reset_transport_header(skb);
/*
* ipprot->handler(skb) will always return 0;
* There is no way to tell whether the packet is dropped later in linux or not.
* Hence here inc the byte/packet count always.
*/
atomic_inc(&cm->rx_packet_count);
atomic_add(len, &cm->rx_byte_count);
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
rcu_read_unlock();
DEBUG_TRACE("%px: %s decap done \n",skb, __func__);
/*
* Update top interface for tunnel searching.
*/
skb->dev = cm->top_interface_dev;
ipprot->handler(skb);
return 1;
}
/*
* Check if skb has enough headroom to write L2 headers
*/
if (unlikely(skb_headroom(skb) < cm->l2_hdr_size)) {
rcu_read_unlock();
DEBUG_WARN("%px: Not enough headroom: %u\n", skb, skb_headroom(skb));
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_NO_HEADROOM);
return 0;
}
/*
* If our packet is larger than the MTU of the transmit interface then
* we can't forward it easily.
*/
if (unlikely(len > cm->xmit_dev_mtu)) {
sfe_ipv6_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TUNIPIP6_NEEDS_FRAGMENTATION);
DEBUG_TRACE("%px: Larger than mtu\n", skb);
return 0;
}
/*
* Update DSCP
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_DSCP_REMARK)) {
sfe_ipv6_change_dsfield(iph, cm->dscp);
}
/*
* Update traffic stats.
*/
atomic_inc(&cm->rx_packet_count);
atomic_add(len, &cm->rx_byte_count);
skb->dev = cm->xmit_dev;
/*
* Check to see if we need to add VLAN tags
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_INSERT_EGRESS_VLAN_TAG)) {
sfe_vlan_add_tag(skb, cm->egress_vlan_hdr_cnt, cm->egress_vlan_hdr);
}
/*
* Check to see if we need to write a header.
*/
if (likely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_WRITE_L2_HDR)) {
if (unlikely(!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR))) {
dev_hard_header(skb, cm->xmit_dev, ntohs(skb->protocol),
cm->xmit_dest_mac, cm->xmit_src_mac, len);
} else {
struct ethhdr *eth = (struct ethhdr *)__skb_push(skb, ETH_HLEN);
eth->h_proto = skb->protocol;
ether_addr_copy((u8 *)eth->h_dest, (u8 *)cm->xmit_dest_mac);
ether_addr_copy((u8 *)eth->h_source, (u8 *)cm->xmit_src_mac);
}
}
/*
* Update priority of skb.
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_PRIORITY_REMARK)) {
skb->priority = cm->priority;
}
/*
* Mark outgoing packet.
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_MARK)) {
skb->mark = cm->mark;
}
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
/*
* We're going to check for GSO flags when we transmit the packet so
* start fetching the necessary cache line now.
*/
prefetch(skb_shinfo(skb));
/*
* Mark that this packet has been fast forwarded and send it on its way.
*/
skb->fast_forwarded = 1;
dev_queue_xmit(skb);
return 1;
}

22
shortcut-fe/sfe_ipv6_tunipip6.h
View file

@ -1,22 +0,0 @@
/*
* sfe_ipv6_tunipip6.h
* Shortcut forwarding engine header file for IPv6 TUNIPIP6
*
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
int sfe_ipv6_recv_tunipip6(struct sfe_ipv6 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct ipv6hdr *iph, unsigned int ihl,
bool sync_on_find, struct sfe_l2_info *l2_info, bool tun_outer);

589
shortcut-fe/sfe_ipv6_udp.c
View file

@ -1,589 +0,0 @@
/*
* sfe_ipv6_udp.c
* Shortcut forwarding engine file for IPv6 UDP
*
* Copyright (c) 2015-2016, 2019-2020, The Linux Foundation. All rights reserved.
* Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <linux/skbuff.h>
#include <net/udp.h>
#include <linux/etherdevice.h>
#include <linux/version.h>
#include <net/ip6_checksum.h>
#include "sfe_debug.h"
#include "sfe_api.h"
#include "sfe.h"
#include "sfe_flow_cookie.h"
#include "sfe_ipv6.h"
#include "sfe_pppoe.h"
#include "sfe_vlan.h"
/*
* sfe_ipv6_udp_sk_deliver()
* Deliver the packet to the protocol handler registered with Linux.
* To be called under rcu_read_lock()
* Returns:
* 1 if the packet needs to be passed to Linux.
* 0 if the packet is processed successfully.
* -1 if the packet is dropped in SFE.
*/
static int sfe_ipv6_udp_sk_deliver(struct sk_buff *skb, struct sfe_ipv6_connection_match *cm,
unsigned int ihl)
{
int (*encap_rcv)(struct sock *sk, struct sk_buff *skb);
struct udp_sock *up;
struct udphdr *udph;
struct sock *sk;
int ret;
/*
* Call the decap handler
*/
up = rcu_dereference(cm->up);
encap_rcv = READ_ONCE(up->encap_rcv);
if (unlikely(!encap_rcv)) {
DEBUG_ERROR("sfe: Error: up->encap_rcv is NULL\n");
return 1;
}
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0))
nf_reset(skb);
#else
nf_reset_ct(skb);
#endif
skb_pull(skb, ihl);
skb_reset_transport_header(skb);
udph = udp_hdr(skb);
if (unlikely(skb->ip_summed != CHECKSUM_UNNECESSARY) && unlikely(skb->ip_summed != CHECKSUM_COMPLETE)) {
/*
* Set Pseudo Checksum using Linux API
*/
if (unlikely(udp6_csum_init(skb, udp_hdr(skb), IPPROTO_UDP))) {
DEBUG_ERROR("sfe: udp checksum init() failed: %p\n", skb);
kfree_skb(skb);
return -1;
}
/*
* Verify checksum before giving to encap_rcv handler function.
*/
if (unlikely(udp_lib_checksum_complete(skb))) {
DEBUG_ERROR("sfe: Invalid udp checksum: %p\n", skb);
kfree_skb(skb);
return -1;
}
}
/*
* Mark that this packet has been fast forwarded.
*/
sk = (struct sock *)up;
/*
* TODO: Find the fix to set skb->ip_summed = CHECKSUM_NONE;
*/
/*
* encap_rcv() returns the following value:
* =0 if skb was successfully passed to the encap
* handler or was discarded by it.
* >0 if skb should be passed on to UDP.
* <0 if skb should be resubmitted as proto -N
*/
ret = encap_rcv(sk, skb);
if (unlikely(ret)) {
/*
* If encap_rcv fails, vxlan driver drops the packet.
* No need to free the skb here.
*/
DEBUG_ERROR("sfe: udp-decap API return error: %d\n", ret);
return -1;
}
DEBUG_TRACE("sfe: udp-decap API encap_rcv successful\n");
return 0;
}
/*
* sfe_ipv6_recv_udp()
* Handle UDP packet receives and forwarding.
*/
int sfe_ipv6_recv_udp(struct sfe_ipv6 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct ipv6hdr *iph, unsigned int ihl, bool sync_on_find, struct sfe_l2_info *l2_info, bool tun_outer)
{
struct udphdr *udph;
struct sfe_ipv6_addr *src_ip;
struct sfe_ipv6_addr *dest_ip;
__be16 src_port;
__be16 dest_port;
u32 service_class_id;
struct sfe_ipv6_connection_match *cm;
struct net_device *xmit_dev;
int ret;
bool hw_csum;
bool bridge_flow;
bool fast_xmit;
netdev_features_t features;
DEBUG_TRACE("%px: sfe: sfe_ipv6_recv_udp called.\n", skb);
/*
* Is our packet too short to contain a valid UDP header?
*/
if (!pskb_may_pull(skb, (sizeof(struct udphdr) + ihl))) {
sfe_ipv6_exception_stats_inc(si,SFE_IPV6_EXCEPTION_EVENT_UDP_HEADER_INCOMPLETE);
DEBUG_TRACE("packet too short for UDP header\n");
return 0;
}
/*
* Read the IP address and port information. Read the IP header data first
* because we've almost certainly got that in the cache. We may not yet have
* the UDP header cached though so allow more time for any prefetching.
*/
src_ip = (struct sfe_ipv6_addr *)iph->saddr.s6_addr32;
dest_ip = (struct sfe_ipv6_addr *)iph->daddr.s6_addr32;
udph = (struct udphdr *)(skb->data + ihl);
src_port = udph->source;
dest_port = udph->dest;
rcu_read_lock();
/*
* Look for a connection match.
*/
#ifdef CONFIG_NF_FLOW_COOKIE
cm = si->sfe_flow_cookie_table[skb->flow_cookie & SFE_FLOW_COOKIE_MASK].match;
if (unlikely(!cm)) {
cm = sfe_ipv6_find_connection_match_rcu(si, dev, IPPROTO_UDP, src_ip, src_port, dest_ip, dest_port);
}
#else
cm = sfe_ipv6_find_connection_match_rcu(si, dev, IPPROTO_UDP, src_ip, src_port, dest_ip, dest_port);
#endif
if (unlikely(!cm)) {
/*
* Try a 4-tuple lookup; required for tunnels like VxLAN.
*/
cm = sfe_ipv6_find_connection_match_rcu(si, dev, IPPROTO_UDP, src_ip, 0, dest_ip, dest_port);
if (unlikely(!cm)) {
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_UDP_NO_CONNECTION);
DEBUG_TRACE("no connection found\n");
return 0;
}
DEBUG_TRACE("sfe: 4-tuple lookup successful\n");
}
/*
* Do we expect an ingress VLAN tag for this flow?
*/
if (unlikely(!sfe_vlan_validate_ingress_tag(skb, cm->ingress_vlan_hdr_cnt, cm->ingress_vlan_hdr, l2_info))) {
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_INGRESS_VLAN_TAG_MISMATCH);
DEBUG_TRACE("VLAN tag mismatch. skb=%px\n", skb);
return 0;
}
/*
* Source interface validate.
*/
if (unlikely((cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_SRC_INTERFACE_CHECK) && (cm->match_dev != dev))) {
if (!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_SRC_INTERFACE_CHECK_NO_FLUSH)) {
struct sfe_ipv6_connection *c = cm->connection;
DEBUG_TRACE("flush on source interface check failure\n");
spin_lock_bh(&si->lock);
ret = sfe_ipv6_remove_connection(si, c);
spin_unlock_bh(&si->lock);
if (ret) {
sfe_ipv6_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
}
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_INVALID_SRC_IFACE);
DEBUG_TRACE("exception the packet on source interface check failure\n");
return 0;
}
/*
* If our packet has been marked as "sync on find" we can't actually
* forward it in the fast path, but now that we've found an associated
* connection we need sync its status before exception it to slow path.
*/
if (unlikely(sync_on_find)) {
sfe_ipv6_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_UDP_IP_OPTIONS_OR_INITIAL_FRAGMENT);
DEBUG_TRACE("Sync on find\n");
return 0;
}
#ifdef CONFIG_XFRM
/*
* We can't accelerate the flow on this direction, just let it go
* through the slow path.
*/
if (unlikely(!cm->flow_accel)) {
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_not_forwarded64);
return 0;
}
#endif
bridge_flow = !!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_BRIDGE_FLOW);
/*
* Does our hop_limit allow forwarding?
*/
if (likely(!bridge_flow)) {
if (unlikely(iph->hop_limit < 2)) {
sfe_ipv6_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_UDP_SMALL_TTL);
DEBUG_TRACE("hop_limit too low\n");
return 0;
}
}
/*
* If our packet is larger than the MTU of the transmit interface then
* we can't forward it easily.
*/
if (unlikely((len > cm->xmit_dev_mtu) && (!cm->up))) {
sfe_ipv6_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_UDP_NEEDS_FRAGMENTATION);
DEBUG_TRACE("Larger than MTU\n");
return 0;
}
/*
* Check if skb was cloned. If it was, unshare it. Because
* the data area is going to be written in this path and we don't want to
* change the cloned skb's data section.
*/
if (unlikely(skb_cloned(skb))) {
DEBUG_TRACE("%px: skb is a cloned skb\n", skb);
skb = skb_unshare(skb, GFP_ATOMIC);
if (!skb) {
DEBUG_WARN("Failed to unshare the cloned skb\n");
rcu_read_unlock();
return 0;
}
/*
* Update the iph and udph pointers with the unshared skb's data area.
*/
iph = (struct ipv6hdr *)skb->data;
udph = (struct udphdr *)(skb->data + ihl);
}
/*
* Check if skb has enough headroom to write L2 headers
*/
if (unlikely(skb_headroom(skb) < cm->l2_hdr_size)) {
rcu_read_unlock();
DEBUG_WARN("%px: Not enough headroom: %u\n", skb, skb_headroom(skb));
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_NO_HEADROOM);
return 0;
}
/*
* For PPPoE packets, match server MAC and session id
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_PPPOE_DECAP)) {
struct ethhdr *eth;
bool pppoe_match;
if (unlikely(!sfe_l2_parse_flag_check(l2_info, SFE_L2_PARSE_FLAGS_PPPOE_INGRESS))) {
rcu_read_unlock();
DEBUG_TRACE("%px: PPPoE header not present in packet for PPPoE rule\n", skb);
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_INCORRECT_PPPOE_PARSING);
return 0;
}
eth = eth_hdr(skb);
pppoe_match = (cm->pppoe_session_id == sfe_l2_pppoe_session_id_get(l2_info)) &&
ether_addr_equal((u8*)cm->pppoe_remote_mac, (u8 *)eth->h_source);
if (unlikely(!pppoe_match)) {
DEBUG_TRACE("%px: PPPoE sessions ID %d and %d or MAC %pM and %pM did not match\n",
skb, cm->pppoe_session_id, sfe_l2_pppoe_session_id_get(l2_info),
cm->pppoe_remote_mac, eth->h_source);
rcu_read_unlock();
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_INVALID_PPPOE_SESSION);
return 0;
}
skb->protocol = htons(l2_info->protocol);
this_cpu_inc(si->stats_pcpu->pppoe_decap_packets_forwarded64);
} else if (unlikely(sfe_l2_parse_flag_check(l2_info, SFE_L2_PARSE_FLAGS_PPPOE_INGRESS))) {
/*
* If packet contains PPPoE header but CME doesn't contain PPPoE flag yet we are exceptioning the packet to linux
*/
if (unlikely(!bridge_flow)) {
rcu_read_unlock();
DEBUG_TRACE("%px: CME doesn't contain PPPoE flag but packet has PPPoE header\n", skb);
sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_PPPOE_NOT_SET_IN_CME);
return 0;
}
/*
* For bridged flows when packet contains PPPoE header, restore the header back and forward to xmit interface
*/
__skb_push(skb, PPPOE_SES_HLEN);
this_cpu_inc(si->stats_pcpu->pppoe_bridge_packets_forwarded64);
}
/*
* From this point on we're good to modify the packet.
*/
/*
* For PPPoE flows, add PPPoE header before L2 header is added.
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_PPPOE_ENCAP)) {
sfe_pppoe_add_header(skb, cm->pppoe_session_id, PPP_IPV6);
this_cpu_inc(si->stats_pcpu->pppoe_encap_packets_forwarded64);
}
/*
* UDP sock will be valid only in decap-path.
* Call encap_rcv function associated with udp_sock in cm.
*/
if (unlikely(cm->up)) {
/*
* Call decap handler associated with sock.
* Also validates UDP checksum before calling decap handler.
*/
ret = sfe_ipv6_udp_sk_deliver(skb, cm, ihl);
if (unlikely(ret == -1)) {
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_dropped64);
return 1;
} else if (unlikely(ret == 1)) {
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_not_forwarded64);
return 0;
}
/*
* Update traffic stats
*/
atomic_inc(&cm->rx_packet_count);
atomic_add(len, &cm->rx_byte_count);
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
DEBUG_TRACE("%p: sfe: sfe_ipv4_recv_udp -> encap_rcv done.\n", skb);
return 1;
}
/*
* Update DSCP
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_DSCP_REMARK)) {
sfe_ipv6_change_dsfield(iph, cm->dscp);
}
/*
* Decrement our hop_limit.
*/
if (likely(!bridge_flow)) {
iph->hop_limit -= (u8)!tun_outer;
}
/*
* Enable HW csum if rx checksum is verified and xmit interface is CSUM offload capable.
* Note: If L4 csum at Rx was found to be incorrect, we (router) should use incremental L4 checksum here
* so that HW does not re-calculate/replace the L4 csum
*/
hw_csum = !!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_CSUM_OFFLOAD) && (skb->ip_summed == CHECKSUM_UNNECESSARY);
/*
* Do we have to perform translations of the source address/port?
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_XLATE_SRC)) {
u16 udp_csum;
iph->saddr.s6_addr32[0] = cm->xlate_src_ip[0].addr[0];
iph->saddr.s6_addr32[1] = cm->xlate_src_ip[0].addr[1];
iph->saddr.s6_addr32[2] = cm->xlate_src_ip[0].addr[2];
iph->saddr.s6_addr32[3] = cm->xlate_src_ip[0].addr[3];
udph->source = cm->xlate_src_port;
/*
* Do we have a non-zero UDP checksum? If we do then we need
* to update it.
*/
if (unlikely(!hw_csum)) {
udp_csum = udph->check;
if (likely(udp_csum)) {
u32 sum = udp_csum + cm->xlate_src_csum_adjustment;
sum = (sum & 0xffff) + (sum >> 16);
udph->check = (u16)sum;
}
}
}
/*
* Do we have to perform translations of the destination address/port?
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_XLATE_DEST)) {
u16 udp_csum;
iph->daddr.s6_addr32[0] = cm->xlate_dest_ip[0].addr[0];
iph->daddr.s6_addr32[1] = cm->xlate_dest_ip[0].addr[1];
iph->daddr.s6_addr32[2] = cm->xlate_dest_ip[0].addr[2];
iph->daddr.s6_addr32[3] = cm->xlate_dest_ip[0].addr[3];
udph->dest = cm->xlate_dest_port;
/*
* Do we have a non-zero UDP checksum? If we do then we need
* to update it.
*/
if (unlikely(!hw_csum)) {
udp_csum = udph->check;
if (likely(udp_csum)) {
u32 sum = udp_csum + cm->xlate_dest_csum_adjustment;
sum = (sum & 0xffff) + (sum >> 16);
udph->check = (u16)sum;
}
}
}
/*
* If HW checksum offload is not possible, incremental L4 checksum is used to update the packet.
* Setting ip_summed to CHECKSUM_UNNECESSARY ensures checksum is not recalculated further in packet
* path.
*/
if (likely(hw_csum)) {
skb->ip_summed = CHECKSUM_PARTIAL;
}
/*
* Update traffic stats.
*/
atomic_inc(&cm->rx_packet_count);
atomic_add(len, &cm->rx_byte_count);
xmit_dev = cm->xmit_dev;
skb->dev = xmit_dev;
/*
* Check to see if we need to add VLAN tags
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_INSERT_EGRESS_VLAN_TAG)) {
sfe_vlan_add_tag(skb, cm->egress_vlan_hdr_cnt, cm->egress_vlan_hdr);
}
/*
* Check to see if we need to write an Ethernet header.
*/
if (likely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_WRITE_L2_HDR)) {
if (unlikely(!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR))) {
dev_hard_header(skb, xmit_dev, ntohs(skb->protocol),
cm->xmit_dest_mac, cm->xmit_src_mac, len);
} else {
/*
* For the simple case we write this really fast.
*/
struct ethhdr *eth = (struct ethhdr *)__skb_push(skb, ETH_HLEN);
eth->h_proto = skb->protocol;
ether_addr_copy((u8 *)eth->h_dest, (u8 *)cm->xmit_dest_mac);
ether_addr_copy((u8 *)eth->h_source, (u8 *)cm->xmit_src_mac);
}
}
/*
* Update priority of skb.
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_PRIORITY_REMARK)) {
skb->priority = cm->priority;
}
/*
* Mark outgoing packet.
*/
if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_MARK)) {
skb->mark = cm->mark;
/*
* Update service class stats if SAWF is valid.
*/
if (likely(cm->sawf_valid)) {
service_class_id = SFE_GET_SAWF_SERVICE_CLASS(cm->mark);
sfe_ipv6_service_class_stats_inc(si, service_class_id, len);
}
}
/*
* For the first packets, check if it could got fast xmit.
*/
if (unlikely(!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED)
&& (cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION))){
cm->features = netif_skb_features(skb);
if (likely(sfe_fast_xmit_check(skb, cm->features))) {
cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT;
}
cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED;
}
features = cm->features;
fast_xmit = !!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT);
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
/*
* We're going to check for GSO flags when we transmit the packet so
* start fetching the necessary cache line now.
*/
prefetch(skb_shinfo(skb));
/*
* We do per packet condition check before we could fast xmit the
* packet.
*/
if (likely(fast_xmit && dev_fast_xmit(skb, xmit_dev, features))) {
this_cpu_inc(si->stats_pcpu->packets_fast_xmited64);
return 1;
}
/*
* Mark that this packet has been fast forwarded.
*/
skb->fast_forwarded = 1;
/*
* Send the packet on its way.
*/
dev_queue_xmit(skb);
return 1;
}

23
shortcut-fe/sfe_ipv6_udp.h
View file

@ -1,23 +0,0 @@
/*
* sfe_ipv6_udp.h
* Shortcut forwarding engine header file for IPv6 UDP
*
* Copyright (c) 2015-2016, 2019-2020, The Linux Foundation. All rights reserved.
* Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
int sfe_ipv6_recv_udp(struct sfe_ipv6 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct ipv6hdr *iph, unsigned int ihl, bool sync_on_find,
struct sfe_l2_info *l2_info, bool tun_outer);

172
shortcut-fe/sfe_pppoe.c
View file

@ -1,172 +0,0 @@
/*
* sfe_pppoe.c
* API for shortcut forwarding engine PPPoE flows
*
* Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <linux/skbuff.h>
#include <linux/if_pppox.h>
#include <linux/ppp_defs.h>
#include <asm/unaligned.h>
#include "sfe_debug.h"
#include "sfe_api.h"
#include "sfe.h"
#include "sfe_pppoe.h"
/*
* sfe_pppoe_br_accel_mode controls how to accelerate PPPoE bridge flow.
* - SFE_PPPOE_BR_ACCEL_MODE_EN_5T: 5-tuple (src_ip, dest_ip, src_port, dest_port, protocol) acceleration
* - SFE_PPPOE_BR_ACCEL_MODE_EN_3T: 3-tuple (src_ip, dest_ip, PPPoE session id) acceleration
* - SFE_PPPOE_BR_ACCEL_MODE_DISABLED: No acceleration
*/
static sfe_pppoe_br_accel_mode_t sfe_pppoe_br_accel_mode __read_mostly = SFE_PPPOE_BR_ACCEL_MODE_EN_5T;
/*
* sfe_pppoe_get_br_accel_mode()
* Gets PPPoE bridge acceleration mode
*/
sfe_pppoe_br_accel_mode_t sfe_pppoe_get_br_accel_mode(void)
{
return sfe_pppoe_br_accel_mode;
}
EXPORT_SYMBOL(sfe_pppoe_get_br_accel_mode);
/*
* sfe_pppoe_set_br_accel_mode()
* Sets PPPoE bridge acceleration mode
*/
int sfe_pppoe_set_br_accel_mode(sfe_pppoe_br_accel_mode_t mode)
{
if (mode >= SFE_PPPOE_BR_ACCEL_MODE_MAX) {
return -1;
}
sfe_pppoe_br_accel_mode = mode;
return 0;
}
/*
* sfe_pppoe_add_header()
* Add PPPoE header.
*
* skb->data will point to PPPoE header after the function
*/
void sfe_pppoe_add_header(struct sk_buff *skb, u16 pppoe_session_id, u16 ppp_protocol)
{
struct pppoe_hdr *ph;
unsigned char *pp;
unsigned int data_len;
/*
* Insert the PPP header protocol
*/
pp = __skb_push(skb, 2);
put_unaligned_be16(ppp_protocol, pp);
data_len = skb->len;
ph = (struct pppoe_hdr *)__skb_push(skb, sizeof(*ph));
skb_reset_network_header(skb);
/*
* Headers in skb will look like in below sequence
* | PPPoE hdr(6 bytes) | PPP hdr (2 bytes) | L3 hdr |
*
* The length field in the PPPoE header indicates the length of the PPPoE payload which
* consists of a 2-byte PPP header plus a skb->len.
*/
ph->ver = 1;
ph->type = 1;
ph->code = 0;
ph->sid = htons(pppoe_session_id);
ph->length = htons(data_len);
skb->protocol = htons(ETH_P_PPP_SES);
}
/*
* sfe_pppoe_parse_hdr()
* Parse PPPoE header
*
* Returns true if the packet is good for further processing.
*/
bool sfe_pppoe_parse_hdr(struct sk_buff *skb, struct sfe_l2_info *l2_info)
{
unsigned int len;
int pppoe_len;
struct sfe_ppp_hdr *ppp;
struct pppoe_hdr *ph = pppoe_hdr(skb);
/*
* Check that we have space for PPPoE header here.
*/
if (unlikely(!pskb_may_pull(skb, (sizeof(struct pppoe_hdr) + sizeof(struct sfe_ppp_hdr))))) {
DEBUG_TRACE("%px: packet too short for PPPoE header\n", skb);
return false;
}
len = skb->len;
pppoe_len = ntohs(ph->length);
if (unlikely(len < pppoe_len)) {
DEBUG_TRACE("%px: len: %u is too short to %u\n", skb, len, pppoe_len);
return false;
}
ppp = (struct sfe_ppp_hdr *)((u8*)ph + sizeof(*ph));
/*
* Converting PPP protocol values to ether type protocol values
*/
switch(ntohs(ppp->protocol)) {
case PPP_IP:
sfe_l2_protocol_set(l2_info, ETH_P_IP);
break;
case PPP_IPV6:
sfe_l2_protocol_set(l2_info, ETH_P_IPV6);
break;
case PPP_LCP:
DEBUG_TRACE("%px: LCP packets are not supported in SFE\n", skb);
return false;
default:
DEBUG_TRACE("%px: Unsupported protocol : %d in PPP header\n", skb, ntohs(ppp->protocol));
return false;
}
sfe_l2_parse_flag_set(l2_info, SFE_L2_PARSE_FLAGS_PPPOE_INGRESS);
sfe_l2_pppoe_session_id_set(l2_info, ntohs(ph->sid));
/*
* strip PPPoE header
*/
__skb_pull(skb, (sizeof(struct pppoe_hdr) + sizeof(struct sfe_ppp_hdr)));
skb_reset_network_header(skb);
return true;
}
/*
* sfe_pppoe_undo_parse()
* undo changes done to skb during PPPoE parsing
*/
void sfe_pppoe_undo_parse(struct sk_buff *skb, struct sfe_l2_info *l2_info)
{
if (sfe_l2_parse_flag_check(l2_info, SFE_L2_PARSE_FLAGS_PPPOE_INGRESS)) {
__skb_push(skb, (sizeof(struct pppoe_hdr) + sizeof(struct sfe_ppp_hdr)));
}
}

35
shortcut-fe/sfe_pppoe.h
View file

@ -1,35 +0,0 @@
/*
* sfe_pppoe.h
* Shortcut flow acceleration for PPPoE flow
*
* Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#ifndef __SFE_PPPOE_H
#define __SFE_PPPOE_H
#include <linux/ppp_defs.h>
#include <linux/if_pppox.h>
struct sfe_ppp_hdr {
u16 protocol;
};
void sfe_pppoe_add_header(struct sk_buff *skb, u16 pppoe_session_id, u16 ppp_protocol);
bool sfe_pppoe_parse_hdr(struct sk_buff *skb, struct sfe_l2_info *l2_info);
void sfe_pppoe_undo_parse(struct sk_buff *skb, struct sfe_l2_info *l2_info);
int sfe_pppoe_set_br_accel_mode(sfe_pppoe_br_accel_mode_t mode);
#endif /* __SFE_PPPOE_H */

302
shortcut-fe/sfe_pppoe_mgr.c
View file

@ -1,302 +0,0 @@
/*
* Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <linux/version.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/rwlock_types.h>
#include <linux/hashtable.h>
#include <linux/inetdevice.h>
#include <linux/etherdevice.h>
#include <linux/if_arp.h>
#include <linux/if_pppox.h>
#include "sfe_pppoe_mgr.h"
#include "sfe_debug.h"
#define HASH_BUCKET_SIZE 2 /* ( 2^ HASH_BUCKET_SIZE ) == 4 */
static DEFINE_HASHTABLE(pppoe_session_table, HASH_BUCKET_SIZE);
/*
* sfe_pppoe_mgr_get_session_info()
* Retrieve PPPoE session info associated with this netdevice
*/
static bool sfe_pppoe_mgr_get_session_info(struct net_device *dev, struct pppoe_opt *addressing)
{
struct ppp_channel *channel[1] = {NULL};
int px_proto;
int ppp_ch_count;
if (ppp_is_multilink(dev)) {
DEBUG_WARN("%s: channel is multilink PPP\n", dev->name);
return false;
}
ppp_ch_count = ppp_hold_channels(dev, channel, 1);
DEBUG_INFO("%s: PPP hold channel ret %d\n", dev->name, ppp_ch_count);
if (ppp_ch_count != 1) {
DEBUG_WARN("%s: hold channel for netdevice %px failed\n", dev->name, dev);
return false;
}
px_proto = ppp_channel_get_protocol(channel[0]);
if (px_proto != PX_PROTO_OE) {
DEBUG_WARN("%s: session socket is not of type PX_PROTO_OE\n", dev->name);
ppp_release_channels(channel, 1);
return false;
}
if (pppoe_channel_addressing_get(channel[0], addressing)) {
DEBUG_WARN("%s: failed to get addressing information\n", dev->name);
ppp_release_channels(channel, 1);
return false;
}
DEBUG_TRACE("dev=%px %s %d: opt_dev=%px opt_dev_name=%s opt_dev_ifindex=%d opt_ifindex=%d\n",
dev, dev->name, dev->ifindex,
addressing->dev, addressing->dev->name, addressing->dev->ifindex, addressing->ifindex);
/*
* pppoe_channel_addressing_get returns held device.
* So, put it back here.
*/
dev_put(addressing->dev);
ppp_release_channels(channel, 1);
return true;
}
/*
* sfe_pppoe_mgr_remove_session()
* Remove PPPoE session entry from hash table
*/
static void sfe_pppoe_mgr_remove_session(struct sfe_pppoe_mgr_session_entry *entry)
{
struct sfe_pppoe_mgr_session_info *info;
info = &entry->info;
DEBUG_INFO("%px %s %d: Remove PPPoE session entry with session_id=%u server_mac=%pM\n",
entry, entry->dev->name, entry->dev->ifindex,
info->session_id, info->server_mac);
hash_del_rcu(&entry->hash_list);
synchronize_rcu();
kfree(entry);
}
/*
* sfe_pppoe_mgr_add_session()
* Create a PPPoE session entry and add it into hash table
*/
static struct sfe_pppoe_mgr_session_entry *sfe_pppoe_mgr_add_session(struct net_device *dev, struct pppoe_opt *opt)
{
struct sfe_pppoe_mgr_session_entry *entry;
struct sfe_pppoe_mgr_session_info *info;
entry = kzalloc(sizeof(struct sfe_pppoe_mgr_session_entry), GFP_KERNEL);
if (!entry) {
DEBUG_WARN("%px: failed to allocate pppoe session entry\n", dev);
return NULL;
}
info = &entry->info;
/*
* Save session info
*/
info->session_id = (uint16_t)ntohs((uint16_t)opt->pa.sid);
ether_addr_copy(info->server_mac, opt->pa.remote);
entry->dev = dev;
/*
* There is no need for protecting simultaneous addition &
* deletion of pppoe sesion entry as the PPP notifier chain
* call back is called with mutex lock.
*/
hash_add_rcu(pppoe_session_table,
&entry->hash_list,
dev->ifindex);
DEBUG_INFO("%px %s %d: Add PPPoE session entry with session_id=%u server_mac=%pM\n",
entry, dev->name, dev->ifindex,
info->session_id, info->server_mac);
return entry;
}
/*
* sfe_pppoe_mgr_disconnect()
* PPPoE interface's disconnect event handler
*/
static int sfe_pppoe_mgr_disconnect(struct net_device *dev)
{
struct sfe_pppoe_mgr_session_entry *entry;
struct sfe_pppoe_mgr_session_entry *found = NULL;
struct hlist_node *temp;
/*
* check whether the interface is of type PPP
*/
if (dev->type != ARPHRD_PPP || !(dev->flags & IFF_POINTOPOINT)) {
return NOTIFY_DONE;
}
hash_for_each_possible_safe(pppoe_session_table, entry,
temp, hash_list, dev->ifindex) {
if (entry->dev != dev) {
continue;
}
/*
* In the hash list, there must be only one entry match with this net device.
*/
found = entry;
break;
}
if (!found) {
DEBUG_WARN("%px: PPPoE session is not found for %s\n", dev, dev->name);
return NOTIFY_DONE;
}
/*
* Remove entry from hash table
*/
sfe_pppoe_mgr_remove_session(found);
return NOTIFY_DONE;
}
/*
* sfe_pppoe_mgr_connect()
* PPPoE interface's connect event handler
*/
static int sfe_pppoe_mgr_connect(struct net_device *dev)
{
struct pppoe_opt opt;
struct sfe_pppoe_mgr_session_entry *entry;
/*
* check whether the interface is of type PPP
*/
if (dev->type != ARPHRD_PPP || !(dev->flags & IFF_POINTOPOINT)) {
return NOTIFY_DONE;
}
if (sfe_pppoe_mgr_get_session_info(dev, &opt) == false) {
DEBUG_WARN("%px: Unable to get pppoe session info from %s\n", dev, dev->name);
return NOTIFY_DONE;
}
/*
* Create an session entry and add it to hash table
*/
entry = sfe_pppoe_mgr_add_session(dev, &opt);
if (!entry) {
DEBUG_WARN("%s: PPPoE session add failed\n", dev->name);
}
return NOTIFY_DONE;
}
/*
* sfe_pppoe_mgr_channel_notifier_handler()
* PPPoE channel notifier handler.
*/
static int sfe_pppoe_mgr_channel_notifier_handler(struct notifier_block *nb,
unsigned long event,
void *arg)
{
struct net_device *dev = (struct net_device *)arg;
switch (event) {
case PPP_CHANNEL_CONNECT:
DEBUG_INFO("%s: PPP_CHANNEL_CONNECT event\n", dev->name);
return sfe_pppoe_mgr_connect(dev);
case PPP_CHANNEL_DISCONNECT:
DEBUG_INFO("%s: PPP_CHANNEL_DISCONNECT event\n", dev->name);
return sfe_pppoe_mgr_disconnect(dev);
default:
DEBUG_INFO("%s: Unhandled channel event: %lu\n", dev->name, event);
break;
}
return NOTIFY_DONE;
}
struct notifier_block sfe_pppoe_mgr_channel_notifier_nb = {
.notifier_call = sfe_pppoe_mgr_channel_notifier_handler,
};
/*
* sfe_pppoe_mgr_find_session()
* Find pppoe session entry given session ID and server MAC
*/
bool sfe_pppoe_mgr_find_session(uint16_t session_id, uint8_t *server_mac)
{
struct sfe_pppoe_mgr_session_entry *entry;
struct sfe_pppoe_mgr_session_info *info;
struct hlist_node *temp;
int bkt;
hash_for_each_safe(pppoe_session_table, bkt, temp, entry, hash_list) {
info = &entry->info;
if ((uint16_t)info->session_id == session_id &&
ether_addr_equal(info->server_mac, server_mac)) {
return true;
}
}
DEBUG_INFO("PPPoE session entry not found: session_id %d server_mac %pM\n", session_id, server_mac);
return false;
}
/*
* sfe_pppoe_mgr_exit
* PPPoE mgr exit function
*/
void sfe_pppoe_mgr_exit(void)
{
struct sfe_pppoe_mgr_session_entry *entry;
struct hlist_node *temp;
int bkt;
/*
* Unregister the module from the PPP channel events.
*/
ppp_channel_connection_unregister_notify(&sfe_pppoe_mgr_channel_notifier_nb);
hash_for_each_safe(pppoe_session_table, bkt, temp, entry, hash_list) {
sfe_pppoe_mgr_remove_session(entry);
}
}
/*
* sfe_pppoe_mgr_init()
* PPPoE mgr init function
*/
int sfe_pppoe_mgr_init(void)
{
/*
* Register the module to the PPP channel events.
*/
ppp_channel_connection_register_notify(&sfe_pppoe_mgr_channel_notifier_nb);
return 0;
}

49
shortcut-fe/sfe_pppoe_mgr.h
View file

@ -1,49 +0,0 @@
/*
* Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
/*
* sfe_pppoe_mgr.h
* SFE PPPoE mgr definitions
*/
#ifndef _SFE_PPPOE_MGR_H_
#define _SFE_PPPOE_MGR_H_
/*
* struct sfe_pppoe_mgr_session_info
* Structure for PPPoE client driver session info
*/
struct sfe_pppoe_mgr_session_info {
uint32_t session_id; /* PPPoE Session ID */
uint8_t server_mac[ETH_ALEN]; /* PPPoE server's MAC address */
};
/*
* struct sfe_pppoe_mgr_session_entry
* Structure for PPPoE session entry into HASH table
*/
struct sfe_pppoe_mgr_session_entry {
struct sfe_pppoe_mgr_session_info info;
/* Session information */
struct net_device *dev; /* Net device */
struct hlist_node hash_list; /* Hash list for sessions */
};
bool sfe_pppoe_mgr_find_session(uint16_t session_id, uint8_t *server_mac);
int sfe_pppoe_mgr_init(void);
void sfe_pppoe_mgr_exit(void);
#endif

218
shortcut-fe/sfe_vlan.h
View file

@ -1,218 +0,0 @@
/*
* sfe_vlan.h
* Shortcut flow acceleration for 802.1AD/802.1Q flow
*
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#ifndef __SFE_VLAN_H
#define __SFE_VLAN_H
#include <linux/if_vlan.h>
/*
* sfe_vlan_check_and_parse_tag()
*
* case 1: QinQ frame (e.g. outer tag = 88a80032, inner tag = 81000001):
* When entering this function:
* ----+-----------------+-----|-----+-----------+-----+---------
* |DMAC |SMAC |88|a8|00|32|81|00|00|01|08|00|45|00|
* ----+-----------------+-----A-----+-----------+-----+---------
* skb->data
* skb->protocol = ntohs(ETH_P_8021AD)
* skb->vlan_proto = 0
* skb->vlan_tci = 0
* skb->vlan_present = 0
* When exiting:
* ----+-----------------+-----------+-----------+-----+---------
* |DMAC |SMAC |88|a8|00|32|81|00|00|01|08|00|45|00|
* ----+-----------------+-----------+-----------+-----A---------
* skb->data
* skb->protocol = ntohs(ETH_P_IP)
* skb->vlan_proto = 0
* skb->vlan_tci = 0
* skb->vlan_present = 0
* l2_info->vlan_hdr_cnt = 2
* l2_info->vlan_hdr[0].tpid = ntohs(ETH_P_8021AD)
* l2_info->vlan_hdr[0].tci = 0x0032
* l2_info->vlan_hdr[1].tpid = ntohs(ETH_P_8021Q)
* l2_info->vlan_hdr[1].tci = 0x0001
* l2_info->protocol = ETH_P_IP
*
* case 2: 802.1Q frame (e.g. the tag is 81000001):
* When entering this function:
* ----+-----------------+-----|-----+-----+---------
* |DMAC |SMAC |81|00|00|01|08|00|45|00|
* ----+-----------------+-----A-----+-----+---------
* skb->data
* skb->protocol = ntohs(ETH_P_8021Q)
* skb->vlan_proto = 0
* skb->vlan_tci = 0
* skb->vlan_present = 0
* When exiting:
* ----+-----------------+-----------+-----+---------
* |DMAC |SMAC |81|00|00|01|08|00|45|00|
* ----+-----------------+-----------+-----A---------
* skb->data
* skb->protocol = ntohs(ETH_P_IP)
* skb->vlan_proto = 0
* skb->vlan_tci = 0
* skb->vlan_present = 0
* l2_info->vlan_hdr_cnt = 1
* l2_info->vlan_hdr[0].tpid = ntohs(ETH_P_8021Q)
* l2_info->vlan_hdr[0].tci = 0x0001
* l2_info->protocol = ETH_P_IP
*
* case 3: untagged frame
* When entering this function:
* ----+-----------------+-----|---------------------
* |DMAC |SMAC |08|00|45|00|
* ----+-----------------+-----A---------------------
* skb->data
* skb->protocol = ntohs(ETH_P_IP)
* skb->vlan_proto = 0
* skb->vlan_tci = 0
* skb->vlan_present = 0
* When exiting:
* ----+-----------------+-----|---------------------
* |DMAC |SMAC |08|00|45|00|
* ----+-----------------+-----A---------------------
* skb->data
* skb->protocol = ntohs(ETH_P_IP)
* skb->vlan_proto = 0
* skb->vlan_tci = 0
* skb->vlan_present = 0
* l2_info->vlan_hdr_cnt = 0
* l2_info->protocol = ETH_P_IP
*/
static inline bool sfe_vlan_check_and_parse_tag(struct sk_buff *skb, struct sfe_l2_info *l2_info)
{
struct vlan_hdr *vhdr;
while ((skb->protocol == htons(ETH_P_8021AD) || skb->protocol == htons(ETH_P_8021Q)) &&
l2_info->vlan_hdr_cnt < SFE_MAX_VLAN_DEPTH) {
if (unlikely(!pskb_may_pull(skb, VLAN_HLEN))) {
return false;
}
vhdr = (struct vlan_hdr *)skb->data;
l2_info->vlan_hdr[l2_info->vlan_hdr_cnt].tpid = skb->protocol;
l2_info->vlan_hdr[l2_info->vlan_hdr_cnt].tci = ntohs(vhdr->h_vlan_TCI);
skb->protocol = vhdr->h_vlan_encapsulated_proto;
l2_info->vlan_hdr_cnt++;
/*
* strip VLAN header
*/
__skb_pull(skb, VLAN_HLEN);
skb_reset_network_header(skb);
}
l2_info->protocol = htons(skb->protocol);
return true;
}
/*
* sfe_vlan_undo_parse()
* Restore some skb fields which are modified when parsing VLAN tags.
*/
static inline void sfe_vlan_undo_parse(struct sk_buff *skb, struct sfe_l2_info *l2_info)
{
if (l2_info->vlan_hdr_cnt == 0) {
return;
}
skb->protocol = l2_info->vlan_hdr[0].tpid;
__skb_push(skb, l2_info->vlan_hdr_cnt * VLAN_HLEN);
}
/*
* sfe_vlan_validate_ingress_tag()
* Validate ingress packet's VLAN tag
*/
static inline bool sfe_vlan_validate_ingress_tag(
struct sk_buff *skb, u8 count, struct sfe_vlan_hdr *vlan_hdr, struct sfe_l2_info *l2_info)
{
u8 i;
if (likely(!sfe_is_l2_feature_enabled())) {
return true;
}
if (unlikely(count != l2_info->vlan_hdr_cnt)) {
return false;
}
for (i = 0; i < count; i++) {
if (unlikely(vlan_hdr[i].tpid != l2_info->vlan_hdr[i].tpid)) {
return false;
}
if (unlikely((vlan_hdr[i].tci & VLAN_VID_MASK) !=
(l2_info->vlan_hdr[i].tci & VLAN_VID_MASK))) {
return false;
}
}
return true;
}
/*
* sfe_vlan_add_tag()
* Add VLAN tags at skb->data.
* Normally, it is called just before adding 14-byte Ethernet header.
*
* This function does not update skb->mac_header so later code
* needs to call skb_reset_mac_header()/skb_reset_mac_len() to
* get correct skb->mac_header/skb->mac_len.
*
* It assumes:
* - skb->protocol is set
* - skb has enough headroom to write VLAN tags
* - 0 < count <= SFE_MAX_VLAN_DEPTH
*
* When entering (e.g. skb->protocol = ntohs(ETH_P_IP) or ntohs(ETH_P_PPP_SES)):
* -------------------------------+---------------------
* |45|00|...
* -------------------------------A---------------------
* skb->data
* -------------------------------v-----------------+-----+----------
* |11|00|xx|xx|xx|xx|00|21|45|00|...
* -------------------------------+-----------------+-----+----------
*
* When exiting (e.g. to add outer/inner tag = 88a80032/81000001):
* -------------+-----------+-----+---------------------
* |00|32|81|00|00|01|08|00|45|00|05|d8|....
* -------A-----+-----------+-----+---------------------
* skb->data
* -------v-----+-----------+-----+-----------------+-----+----------
* |00|32|81|00|00|01|88|64|11|00|xx|xx|xx|xx|00|21|45|00|
* -------------+-----------+-----+-----------------+-----+----------
* skb->protocol = ntohs(ETH_P_8021AD)
*/
static inline void sfe_vlan_add_tag(struct sk_buff *skb, int count, struct sfe_vlan_hdr *vlan)
{
struct vlan_hdr *vhdr;
int i;
vlan += (count - 1);
for (i = 0; i < count; i++) {
vhdr = (struct vlan_hdr *)skb_push(skb, VLAN_HLEN);
vhdr->h_vlan_TCI = htons(vlan->tci);
vhdr->h_vlan_encapsulated_proto = skb->protocol;
skb->protocol = vlan->tpid;
vlan--;
}
}
#endif /* __SFE_VLAN_H */

88
shortcut-fe/shortcut-fe/Makefile Normal file
View file

@ -0,0 +1,88 @@
#
# Copyright (c) 2013-2018, 2020 The Linux Foundation. All rights reserved.
# Permission to use, copy, modify, and/or distribute this software for
# any purpose with or without fee is hereby granted, provided that the
# above copyright notice and this permission notice appear in all copies.
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
include $(TOPDIR)/rules.mk
include $(INCLUDE_DIR)/kernel.mk
PKG_NAME:=shortcut-fe
PKG_RELEASE:=8
include $(INCLUDE_DIR)/package.mk
define KernelPackage/shortcut-fe
SECTION:=kernel
CATEGORY:=Kernel modules
SUBMENU:=Network Support
DEPENDS:=@IPV6 +kmod-nf-conntrack
TITLE:=Kernel driver for SFE
FILES:= \
$(PKG_BUILD_DIR)/shortcut-fe.ko \
$(PKG_BUILD_DIR)/shortcut-fe-ipv6.ko
KCONFIG:= \
CONFIG_NF_CONNTRACK_EVENTS=y \
CONFIG_NF_CONNTRACK_TIMEOUT=y \
CONFIG_SHORTCUT_FE=y \
CONFIG_XFRM=y
PROVIDES:=$(PKG_NAME)
AUTOLOAD:=$(call AutoLoad,09,shortcut-fe shortcut-fe-ipv6)
endef
define KernelPackage/shortcut-fe/Description
Shortcut is an in-Linux-kernel IP packet forwarding engine.
endef
define KernelPackage/shortcut-fe/install
$(INSTALL_DIR) $(1)/usr/bin
$(INSTALL_BIN) ./files/usr/bin/sfe_dump $(1)/usr/bin
endef
HAVE_ECM:=$(CONFIG_PACKAGE_kmod-qca-nss-ecm-premium)$(CONFIG_PACKAGE_kmod-qca-nss-ecm-noload)$(CONFIG_PACKAGE_kmod-qca-nss-ecm-premium-noload)$(CONFIG_PACKAGE_kmod-qca-nss-ecm-standard)
define KernelPackage/shortcut-fe-cm
SECTION:=kernel
CATEGORY:=Kernel modules
SUBMENU:=Network Support
DEPENDS:=+kmod-ipt-conntrack +kmod-shortcut-fe
TITLE:=Kernel driver for SFE
FILES:=$(PKG_BUILD_DIR)/shortcut-fe-cm.ko
KCONFIG:= \
CONFIG_NF_CONNTRACK_CHAIN_EVENTS=y \
CONFIG_NF_CONNTRACK_EVENTS=y \
CONFIG_XFRM=y
CONFLICTS:=kmod-shortcut-fe-drv
endef
define KernelPackage/shortcut-fe-cm/Description
Simple connection manager for the Shortcut forwarding engine.
endef
define Build/Compile
$(MAKE) $(PKG_JOBS) -C "$(LINUX_DIR)" \
$(KERNEL_MAKE_FLAGS) \
$(PKG_MAKE_FLAGS) \
M="$(PKG_BUILD_DIR)" \
EXTRA_CFLAGS+="-DSFE_SUPPORT_IPV6" SFE_SUPPORT_IPV6=y \
$(if $(HAVE_ECM),EXTRA_CFLAGS+="-DCONFIG_SFE_ECM" CONFIG_SFE_ECM=y,) \
modules
endef
ifneq ($(CONFIG_PACKAGE_kmod-shortcut-fe)$(CONFIG_PACKAGE_kmod-shortcut-fe-cm),)
define Build/InstallDev
$(INSTALL_DIR) $(1)/usr/include/shortcut-fe
$(CP) -rf $(PKG_BUILD_DIR)/sfe.h $(1)/usr/include/shortcut-fe
endef
endif
$(eval $(call KernelPackage,shortcut-fe))
$(eval $(call KernelPackage,shortcut-fe-cm))

51
shortcut-fe/shortcut-fe/files/etc/init.d/shortcut-fe Normal file
View file

@ -0,0 +1,51 @@
#!/bin/sh /etc/rc.common
#
# Copyright (c) 2014-2015 The Linux Foundation. All rights reserved.
# Permission to use, copy, modify, and/or distribute this software for
# any purpose with or without fee is hereby granted, provided that the
# above copyright notice and this permission notice appear in all copies.
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
#SFE connection manager has a lower priority, it should be started after other connection manager
#to detect the existence of connection manager with higher priority
START=70
have_cm() {
[ -d "/sys/kernel/debug/ecm" ] && echo 1 && return
echo 0
}
#load shortcut-fe and connection manager
load_sfe() {
local kernel_version=$(uname -r)
[ -d "/sys/module/shortcut_fe" ] || insmod /lib/modules/$kernel_version/shortcut-fe.ko
[ -d "/sys/module/shortcut_fe_ipv6" ] || insmod /lib/modules/$kernel_version/shortcut-fe-ipv6.ko
[ -e "/lib/modules/$kernel_version/shortcut-fe-cm.ko" ] && {
[ -d /sys/module/shortcut_fe_cm ] || insmod /lib/modules/$kernel_version/shortcut-fe-cm.ko
}
[ -e "/lib/modules/$kernel_version/fast-classifier.ko" ] && {
[ -d /sys/module/fast_classifier ] || insmod /lib/modules/$kernel_version/fast-classifier.ko
}
}
start() {
[ "$(have_cm)" = "0" ] && load_sfe
}
stop() {
[ -d "/sys/module/shortcut_fe_cm" ] && rmmod shortcut_fe_cm
[ -d "/sys/module/shortcut_fe_ipv6" ] && rmmod shortcut_fe_ipv6
[ -d "/sys/module/shortcut_fe" ] && rmmod shortcut_fe
[ -d "/sys/module/shortcut_fe_drv" ] && rmmod shortcut_fe_drv
[ -d "/sys/module/fast_classifier" ] && rmmod fast_classifier
}

35
shortcut-fe/shortcut-fe/files/usr/bin/sfe_dump Normal file
View file

@ -0,0 +1,35 @@
#!/bin/sh
#
# Copyright (c) 2015 The Linux Foundation. All rights reserved.
# Permission to use, copy, modify, and/or distribute this software for
# any purpose with or without fee is hereby granted, provided that the
# above copyright notice and this permission notice appear in all copies.
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
#@sfe_dump
#@example : sfe_dump (ipv4|ipv6)
sfe_dump(){
[ -e "/dev/sfe_ipv4" ] || {
dev_num=$(cat /sys/sfe_ipv4/debug_dev)
mknod /dev/sfe_ipv4 c $dev_num 0
}
[ -e "/dev/sfe_ipv6" ] || {
dev_num=$(cat /sys/sfe_ipv6/debug_dev)
mknod /dev/sfe_ipv6 c $dev_num 0
}
cat /dev/sfe_$1
}
if [ -z "$1" ]; then
sfe_dump ipv4
sfe_dump ipv6
else
sfe_dump $1
fi

14
shortcut-fe/shortcut-fe/src/Kconfig Normal file
View file

@ -0,0 +1,14 @@
#
# Shortcut forwarding engine
#
config SHORTCUT_FE
tristate "Shortcut Forwarding Engine"
depends on NF_CONNTRACK
---help---
Shortcut is a fast in-kernel packet forwarding engine.
To compile this code as a module, choose M here: the module will be
called shortcut-fe.
If unsure, say N.

24
shortcut-fe/shortcut-fe/src/Makefile Normal file
View file

@ -0,0 +1,24 @@
#
# Makefile for Shortcut FE.
#
obj-m += shortcut-fe.o
ifdef SFE_SUPPORT_IPV6
obj-m += shortcut-fe-ipv6.o
endif
obj-m += shortcut-fe-cm.o
shortcut-fe-objs := \
sfe_ipv4.o
ifdef SFE_SUPPORT_IPV6
shortcut-fe-ipv6-objs := \
sfe_ipv6.o
endif
shortcut-fe-cm-objs := \
sfe_cm.o
ccflags-y += -Werror -Wall

89
shortcut-fe/sfe_flow_cookie.h → shortcut-fe/shortcut-fe/src/sfe.h
View file

@ -1,40 +1,75 @@
/*
* sfe_flow_cookie.h
* Flow cookie related callbacks.
*
* Copyright (c) 2013-2016, 2019-2020, The Linux Foundation. All rights reserved.
* Copyright (c) 2021 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
* sfe.h
* Shortcut forwarding engine.
*
* Copyright (c) 2013-2017 The Linux Foundation. All rights reserved.
* Permission to use, copy, modify, and/or distribute this software for
* any purpose with or without fee is hereby granted, provided that the
* above copyright notice and this permission notice appear in all copies.
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
* OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
/*
* The following are debug macros used throughout the SFE.
*
* The DEBUG_LEVEL enables the followings based on its value,
* when dynamic debug option is disabled.
*
* 0 = OFF
* 1 = ASSERTS / ERRORS
* 2 = 1 + WARN
* 3 = 2 + INFO
* 4 = 3 + TRACE
*/
#define DEBUG_LEVEL 2
#if (DEBUG_LEVEL < 1)
#define DEBUG_ASSERT(s, ...)
#define DEBUG_ERROR(s, ...)
#else
#define DEBUG_ASSERT(c, s, ...) if (!(c)) { pr_emerg("ASSERT: %s:%d:" s, __FUNCTION__, __LINE__, ##__VA_ARGS__); BUG(); }
#define DEBUG_ERROR(s, ...) pr_err("%s:%d:" s, __FUNCTION__, __LINE__, ##__VA_ARGS__)
#endif
#if defined(CONFIG_DYNAMIC_DEBUG)
/*
* Compile messages for dynamic enable/disable
*/
#define DEBUG_WARN(s, ...) pr_debug("%s[%d]:" s, __FUNCTION__, __LINE__, ##__VA_ARGS__)
#define DEBUG_INFO(s, ...) pr_debug("%s[%d]:" s, __FUNCTION__, __LINE__, ##__VA_ARGS__)
#define DEBUG_TRACE(s, ...) pr_debug("%s[%d]:" s, __FUNCTION__, __LINE__, ##__VA_ARGS__)
#else
/*
* Statically compile messages at different levels
*/
#if (DEBUG_LEVEL < 2)
#define DEBUG_WARN(s, ...)
#else
#define DEBUG_WARN(s, ...) pr_warn("%s[%d]:" s, __FUNCTION__, __LINE__, ##__VA_ARGS__)
#endif
#if (DEBUG_LEVEL < 3)
#define DEBUG_INFO(s, ...)
#else
#define DEBUG_INFO(s, ...) pr_notice("%s[%d]:" s, __FUNCTION__, __LINE__, ##__VA_ARGS__)
#endif
#if (DEBUG_LEVEL < 4)
#define DEBUG_TRACE(s, ...)
#else
#define DEBUG_TRACE(s, ...) pr_info("%s[%d]:" s, __FUNCTION__, __LINE__, ##__VA_ARGS__)
#endif
#endif
#ifdef CONFIG_NF_FLOW_COOKIE
#define SFE_FLOW_COOKIE_SIZE 2048
#define SFE_FLOW_COOKIE_MASK 0x7ff
struct sfe_ipv4_connection_match;
struct sfe_ipv6_connection_match;
struct sfe_flow_cookie_entry {
struct sfe_ipv4_connection_match *match;
unsigned long last_clean_time;
};
struct sfe_ipv6_flow_cookie_entry {
struct sfe_ipv6_connection_match *match;
unsigned long last_clean_time;
};
typedef int (*flow_cookie_set_func_t)(u32 protocol, __be32 src_ip, __be16 src_port,
__be32 dst_ip, __be16 dst_port, u16 flow_cookie);
/*

195
shortcut-fe/shortcut-fe/src/sfe_backport.h Normal file
View file

@ -0,0 +1,195 @@
/*
* sfe_backport.h
* Shortcut forwarding engine compatible header file.
*
* Copyright (c) 2014-2016 The Linux Foundation. All rights reserved.
* Permission to use, copy, modify, and/or distribute this software for
* any purpose with or without fee is hereby granted, provided that the
* above copyright notice and this permission notice appear in all copies.
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
* OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <linux/version.h>
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0))
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0))
#include <net/netfilter/nf_conntrack_timeout.h>
#else
enum udp_conntrack {
UDP_CT_UNREPLIED,
UDP_CT_REPLIED,
UDP_CT_MAX
};
static inline unsigned int *
nf_ct_timeout_lookup(struct net *net, struct nf_conn *ct,
struct nf_conntrack_l4proto *l4proto)
{
#ifdef CONFIG_NF_CONNTRACK_TIMEOUT
struct nf_conn_timeout *timeout_ext;
unsigned int *timeouts;
timeout_ext = nf_ct_timeout_find(ct);
if (timeout_ext)
timeouts = NF_CT_TIMEOUT_EXT_DATA(timeout_ext);
else
timeouts = l4proto->get_timeouts(net);
return timeouts;
#else
return l4proto->get_timeouts(net);
#endif /*CONFIG_NF_CONNTRACK_TIMEOUT*/
}
#endif /*KERNEL_VERSION(3, 7, 0)*/
#endif /*KERNEL_VERSION(3, 4, 0)*/
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0))
#define sfe_define_post_routing_hook(FN_NAME, HOOKNUM, OPS, SKB, UNUSED, OUT, OKFN) \
static unsigned int FN_NAME(void *priv, \
struct sk_buff *SKB, \
const struct nf_hook_state *state)
#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0))
#define sfe_define_post_routing_hook(FN_NAME, HOOKNUM, OPS, SKB, UNUSED, OUT, OKFN) \
static unsigned int FN_NAME(const struct nf_hook_ops *OPS, \
struct sk_buff *SKB, \
const struct net_device *UNUSED, \
const struct net_device *OUT, \
int (*OKFN)(struct sk_buff *))
#else
#define sfe_define_post_routing_hook(FN_NAME, HOOKNUM, OPS, SKB, UNUSED, OUT, OKFN) \
static unsigned int FN_NAME(unsigned int HOOKNUM, \
struct sk_buff *SKB, \
const struct net_device *UNUSED, \
const struct net_device *OUT, \
int (*OKFN)(struct sk_buff *))
#endif
#define sfe_cm_ipv4_post_routing_hook(HOOKNUM, OPS, SKB, UNUSED, OUT, OKFN) \
sfe_define_post_routing_hook(__sfe_cm_ipv4_post_routing_hook, HOOKNUM, OPS, SKB, UNUSED, OUT, OKFN)
#define sfe_cm_ipv6_post_routing_hook(HOOKNUM, OPS, SKB, UNUSED, OUT, OKFN) \
sfe_define_post_routing_hook(__sfe_cm_ipv6_post_routing_hook, HOOKNUM, OPS, SKB, UNUSED, OUT, OKFN)
#define fast_classifier_ipv4_post_routing_hook(HOOKNUM, OPS, SKB, UNUSED, OUT, OKFN) \
sfe_define_post_routing_hook(__fast_classifier_ipv4_post_routing_hook, HOOKNUM, OPS, SKB, UNUSED, OUT, OKFN)
#define fast_classifier_ipv6_post_routing_hook(HOOKNUM, OPS, SKB, UNUSED, OUT, OKFN) \
sfe_define_post_routing_hook(__fast_classifier_ipv6_post_routing_hook, HOOKNUM, OPS, SKB, UNUSED, OUT, OKFN)
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0))
#define SFE_IPV4_NF_POST_ROUTING_HOOK(fn) \
{ \
.hook = fn, \
.pf = NFPROTO_IPV4, \
.hooknum = NF_INET_POST_ROUTING, \
.priority = NF_IP_PRI_NAT_SRC + 1, \
}
#else
#define SFE_IPV4_NF_POST_ROUTING_HOOK(fn) \
{ \
.hook = fn, \
.owner = THIS_MODULE, \
.pf = NFPROTO_IPV4, \
.hooknum = NF_INET_POST_ROUTING, \
.priority = NF_IP_PRI_NAT_SRC + 1, \
}
#endif
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0))
#define SFE_IPV6_NF_POST_ROUTING_HOOK(fn) \
{ \
.hook = fn, \
.pf = NFPROTO_IPV6, \
.hooknum = NF_INET_POST_ROUTING, \
.priority = NF_IP_PRI_NAT_SRC + 1, \
}
#else
#define SFE_IPV6_NF_POST_ROUTING_HOOK(fn) \
{ \
.hook = fn, \
.owner = THIS_MODULE, \
.pf = NFPROTO_IPV6, \
.hooknum = NF_INET_POST_ROUTING, \
.priority = NF_IP6_PRI_NAT_SRC + 1, \
}
#endif
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 3, 0))
#define SFE_NF_CT_DEFAULT_ZONE (&nf_ct_zone_dflt)
#else
#define SFE_NF_CT_DEFAULT_ZONE NF_CT_DEFAULT_ZONE
#endif
/*
* sfe_dev_get_master
* get master of bridge port, and hold it
*/
static inline struct net_device *sfe_dev_get_master(struct net_device *dev)
{
struct net_device *master;
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0))
rcu_read_lock();
master = netdev_master_upper_dev_get_rcu(dev);
if (master)
dev_hold(master);
rcu_read_unlock();
#else
master = dev->master;
if (master)
dev_hold(master);
#endif
return master;
}
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 11, 0))
#define SFE_DEV_EVENT_PTR(PTR) netdev_notifier_info_to_dev(PTR)
#else
#define SFE_DEV_EVENT_PTR(PTR) (struct net_device *)(PTR)
#endif
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0))
#define SFE_NF_CONN_ACCT(NM) struct nf_conn_acct *NM
#else
#define SFE_NF_CONN_ACCT(NM) struct nf_conn_counter *NM
#endif
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0))
#define SFE_ACCT_COUNTER(NM) ((NM)->counter)
#else
#define SFE_ACCT_COUNTER(NM) (NM)
#endif
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0))
#define sfe_hash_for_each_possible(name, obj, node, member, key) \
hash_for_each_possible(name, obj, member, key)
#else
#define sfe_hash_for_each_possible(name, obj, node, member, key) \
hash_for_each_possible(name, obj, node, member, key)
#endif
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0))
#define sfe_hash_for_each(name, bkt, node, obj, member) \
hash_for_each(name, bkt, obj, member)
#else
#define sfe_hash_for_each(name, bkt, node, obj, member) \
hash_for_each(name, bkt, node, obj, member)
#endif
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0))
#define sfe_dst_get_neighbour(dst, daddr) dst_neigh_lookup(dst, addr)
#else
static inline struct neighbour *
sfe_dst_get_neighbour(struct dst_entry *dst, void *daddr)
{
struct neighbour *neigh = dst_get_neighbour_noref(dst);
if (neigh)
neigh_hold(neigh);
return neigh;
}
#endif

1210
shortcut-fe/shortcut-fe/src/sfe_cm.c Normal file
View file

File diff suppressed because it is too large Load diff

260
shortcut-fe/shortcut-fe/src/sfe_cm.h Normal file
View file

@ -0,0 +1,260 @@
/*
* sfe_cm.h
* Shortcut forwarding engine.
*
* Copyright (c) 2013-2016 The Linux Foundation. All rights reserved.
* Permission to use, copy, modify, and/or distribute this software for
* any purpose with or without fee is hereby granted, provided that the
* above copyright notice and this permission notice appear in all copies.
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
* OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
/*
* connection flags.
*/
#define SFE_CREATE_FLAG_NO_SEQ_CHECK BIT(0)
/* Indicates that we should not check sequence numbers */
#define SFE_CREATE_FLAG_REMARK_PRIORITY BIT(1)
/* Indicates that we should remark priority of skb */
#define SFE_CREATE_FLAG_REMARK_DSCP BIT(2)
/* Indicates that we should remark DSCP of packet */
/*
* IPv6 address structure
*/
struct sfe_ipv6_addr {
__be32 addr[4];
};
typedef union {
__be32 ip;
struct sfe_ipv6_addr ip6[1];
} sfe_ip_addr_t;
/*
* connection creation structure.
*/
struct sfe_connection_create {
int protocol;
struct net_device *src_dev;
struct net_device *dest_dev;
u32 flags;
u32 src_mtu;
u32 dest_mtu;
sfe_ip_addr_t src_ip;
sfe_ip_addr_t src_ip_xlate;
sfe_ip_addr_t dest_ip;
sfe_ip_addr_t dest_ip_xlate;
__be16 src_port;
__be16 src_port_xlate;
__be16 dest_port;
__be16 dest_port_xlate;
u8 src_mac[ETH_ALEN];
u8 src_mac_xlate[ETH_ALEN];
u8 dest_mac[ETH_ALEN];
u8 dest_mac_xlate[ETH_ALEN];
u8 src_td_window_scale;
u32 src_td_max_window;
u32 src_td_end;
u32 src_td_max_end;
u8 dest_td_window_scale;
u32 dest_td_max_window;
u32 dest_td_end;
u32 dest_td_max_end;
u32 mark;
#ifdef CONFIG_XFRM
u32 original_accel;
u32 reply_accel;
#endif
u32 src_priority;
u32 dest_priority;
u32 src_dscp;
u32 dest_dscp;
};
/*
* connection destruction structure.
*/
struct sfe_connection_destroy {
int protocol;
sfe_ip_addr_t src_ip;
sfe_ip_addr_t dest_ip;
__be16 src_port;
__be16 dest_port;
};
typedef enum sfe_sync_reason {
SFE_SYNC_REASON_STATS, /* Sync is to synchronize stats */
SFE_SYNC_REASON_FLUSH, /* Sync is to flush a entry */
SFE_SYNC_REASON_DESTROY /* Sync is to destroy a entry(requested by connection manager) */
} sfe_sync_reason_t;
/*
* Structure used to sync connection stats/state back within the system.
*
* NOTE: The addresses here are NON-NAT addresses, i.e. the true endpoint addressing.
* 'src' is the creator of the connection.
*/
struct sfe_connection_sync {
struct net_device *src_dev;
struct net_device *dest_dev;
int is_v6; /* Is it for ipv6? */
int protocol; /* IP protocol number (IPPROTO_...) */
sfe_ip_addr_t src_ip; /* Non-NAT source address, i.e. the creator of the connection */
sfe_ip_addr_t src_ip_xlate; /* NATed source address */
__be16 src_port; /* Non-NAT source port */
__be16 src_port_xlate; /* NATed source port */
sfe_ip_addr_t dest_ip; /* Non-NAT destination address, i.e. to whom the connection was created */
sfe_ip_addr_t dest_ip_xlate; /* NATed destination address */
__be16 dest_port; /* Non-NAT destination port */
__be16 dest_port_xlate; /* NATed destination port */
u32 src_td_max_window;
u32 src_td_end;
u32 src_td_max_end;
u64 src_packet_count;
u64 src_byte_count;
u32 src_new_packet_count;
u32 src_new_byte_count;
u32 dest_td_max_window;
u32 dest_td_end;
u32 dest_td_max_end;
u64 dest_packet_count;
u64 dest_byte_count;
u32 dest_new_packet_count;
u32 dest_new_byte_count;
u32 reason; /* reason for stats sync message, i.e. destroy, flush, period sync */
u64 delta_jiffies; /* Time to be added to the current timeout to keep the connection alive */
};
/*
* connection mark structure
*/
struct sfe_connection_mark {
int protocol;
sfe_ip_addr_t src_ip;
sfe_ip_addr_t dest_ip;
__be16 src_port;
__be16 dest_port;
u32 mark;
};
/*
* Expose the hook for the receive processing.
*/
extern int (*athrs_fast_nat_recv)(struct sk_buff *skb);
/*
* Expose what should be a static flag in the TCP connection tracker.
*/
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 15, 0)
extern int nf_ct_tcp_no_window_check;
#endif
/*
* This callback will be called in a timer
* at 100 times per second to sync stats back to
* Linux connection track.
*
* A RCU lock is taken to prevent this callback
* from unregistering.
*/
typedef void (*sfe_sync_rule_callback_t)(struct sfe_connection_sync *);
/*
* IPv4 APIs used by connection manager
*/
int sfe_ipv4_recv(struct net_device *dev, struct sk_buff *skb);
int sfe_ipv4_create_rule(struct sfe_connection_create *sic);
void sfe_ipv4_destroy_rule(struct sfe_connection_destroy *sid);
void sfe_ipv4_destroy_all_rules_for_dev(struct net_device *dev);
void sfe_ipv4_register_sync_rule_callback(sfe_sync_rule_callback_t callback);
void sfe_ipv4_update_rule(struct sfe_connection_create *sic);
void sfe_ipv4_mark_rule(struct sfe_connection_mark *mark);
#ifdef SFE_SUPPORT_IPV6
/*
* IPv6 APIs used by connection manager
*/
int sfe_ipv6_recv(struct net_device *dev, struct sk_buff *skb);
int sfe_ipv6_create_rule(struct sfe_connection_create *sic);
void sfe_ipv6_destroy_rule(struct sfe_connection_destroy *sid);
void sfe_ipv6_destroy_all_rules_for_dev(struct net_device *dev);
void sfe_ipv6_register_sync_rule_callback(sfe_sync_rule_callback_t callback);
void sfe_ipv6_update_rule(struct sfe_connection_create *sic);
void sfe_ipv6_mark_rule(struct sfe_connection_mark *mark);
#else
static inline int sfe_ipv6_recv(struct net_device *dev, struct sk_buff *skb)
{
return 0;
}
static inline int sfe_ipv6_create_rule(struct sfe_connection_create *sic)
{
return 0;
}
static inline void sfe_ipv6_destroy_rule(struct sfe_connection_destroy *sid)
{
return;
}
static inline void sfe_ipv6_destroy_all_rules_for_dev(struct net_device *dev)
{
return;
}
static inline void sfe_ipv6_register_sync_rule_callback(sfe_sync_rule_callback_t callback)
{
return;
}
static inline void sfe_ipv6_update_rule(struct sfe_connection_create *sic)
{
return;
}
static inline void sfe_ipv6_mark_rule(struct sfe_connection_mark *mark)
{
return;
}
#endif
/*
* sfe_ipv6_addr_equal()
* compare ipv6 address
*
* return: 1, equal; 0, no equal
*/
static inline int sfe_ipv6_addr_equal(struct sfe_ipv6_addr *a,
struct sfe_ipv6_addr *b)
{
return a->addr[0] == b->addr[0] &&
a->addr[1] == b->addr[1] &&
a->addr[2] == b->addr[2] &&
a->addr[3] == b->addr[3];
}
/*
* sfe_ipv4_addr_equal()
* compare ipv4 address
*
* return: 1, equal; 0, no equal
*/
#define sfe_ipv4_addr_equal(a, b) ((u32)(a) == (u32)(b))
/*
* sfe_addr_equal()
* compare ipv4 or ipv6 address
*
* return: 1, equal; 0, no equal
*/
static inline int sfe_addr_equal(sfe_ip_addr_t *a,
sfe_ip_addr_t *b, int is_v4)
{
return is_v4 ? sfe_ipv4_addr_equal(a->ip, b->ip) : sfe_ipv6_addr_equal(a->ip6, b->ip6);
}

3618
shortcut-fe/shortcut-fe/src/sfe_ipv4.c Normal file
View file

File diff suppressed because it is too large Load diff

3625
shortcut-fe/shortcut-fe/src/sfe_ipv6.c Normal file
View file

File diff suppressed because it is too large Load diff

60
shortcut-fe/simulated-driver/Makefile Normal file
View file

@ -0,0 +1,60 @@
#
# Copyright (c) 2015,2016 The Linux Foundation. All rights reserved.
# Permission to use, copy, modify, and/or distribute this software for
# any purpose with or without fee is hereby granted, provided that the
# above copyright notice and this permission notice appear in all copies.
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
include $(TOPDIR)/rules.mk
include $(INCLUDE_DIR)/kernel.mk
PKG_NAME:=shortcut-fe-simulated-driver
PKG_RELEASE:=1
PKG_SOURCE_URL:=https://source.codeaurora.org/quic/qsdk/oss/lklm/shortcut-fe
PKG_SOURCE_PROTO:=git
PKG_SOURCE_DATE:=2021-03-17
PKG_SOURCE_VERSION:=697977d8d0ccf0ab596e5692d08608a75dd7f33d
PKG_MIRROR_HASH:=659fa82a431e15af797a6c7069faeee02810453ad8b576c51c29f95a1761a045
include $(INCLUDE_DIR)/package.mk
define KernelPackage/shortcut-fe-drv
SECTION:=kernel
CATEGORY:=Kernel modules
SUBMENU:=Network Support
DEPENDS:=@TARGET_ipq806x||TARGET_ipq807x +kmod-shortcut-fe
KCONFIG:= \
CONFIG_NET_CLS_ACT=y \
CONFIG_XFRM=y
TITLE:=Simulated sfe driver for ECM
FILES:=$(PKG_BUILD_DIR)/simulated-driver/shortcut-fe-drv.ko
endef
define KernelPackage/shortcut-fe-drv/Description
Simulated sfe driver which act as an adapter to convert message
between a connection manager and the SFE core engine.
endef
define Build/Compile
$(MAKE) $(PKG_JOBS) -C "$(LINUX_DIR)" \
$(KERNEL_MAKE_FLAGS) \
$(PKG_MAKE_FLAGS) \
M="$(PKG_BUILD_DIR)/simulated-driver" \
EXTRA_CFLAGS="-DSFE_SUPPORT_IPV6" \
modules
endef
define Build/InstallDev
$(INSTALL_DIR) $(1)/usr/include/shortcut-fe
$(CP) -rf $(PKG_BUILD_DIR)/simulated-driver/sfe_drv.h $(1)/usr/include/shortcut-fe
endef
$(eval $(call KernelPackage,shortcut-fe-drv))

11
shortcut-fe/simulated-driver/patches/200-nss-qdisc-support.patch Normal file
View file

@ -0,0 +1,11 @@
--- ./simulated-driver/sfe_drv.c.orig 2020-06-16 12:49:47.680153371 +0800
+++ ./simulated-driver/sfe_drv.c 2020-06-16 12:50:18.540153371 +0800
@@ -1167,7 +1167,7 @@ int sfe_drv_recv(struct sk_buff *skb)
* If ingress Qdisc configured, and packet not processed by ingress Qdisc yet
* We can not accelerate this packet.
*/
- if (dev->ingress_queue && !(skb->tc_verd & TC_NCLS)) {
+ if (dev->ingress_queue && !(skb->tc_verd_qca_nss & TC_NCLS)) {
return 0;
}
#endif