Merge tag 'net-next-6.3' of git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-nextgrafted

Pull networking updates from Jakub Kicinski:
 "Core:

   - Add dedicated kmem_cache for typical/small skb->head, avoid having
     to access struct page at kfree time, and improve memory use.

   - Introduce sysctl to set default RPS configuration for new netdevs.

   - Define Netlink protocol specification format which can be used to
     describe messages used by each family and auto-generate parsers.
     Add tools for generating kernel data structures and uAPI headers.

   - Expose all net/core sysctls inside netns.

   - Remove 4s sleep in netpoll if carrier is instantly detected on
     boot.

   - Add configurable limit of MDB entries per port, and port-vlan.

   - Continue populating drop reasons throughout the stack.

   - Retire a handful of legacy Qdiscs and classifiers.

  Protocols:

   - Support IPv4 big TCP (TSO frames larger than 64kB).

   - Add IP_LOCAL_PORT_RANGE socket option, to control local port range
     on socket by socket basis.

   - Track and report in procfs number of MPTCP sockets used.

   - Support mixing IPv4 and IPv6 flows in the in-kernel MPTCP path
     manager.

   - IPv6: don't check net.ipv6.route.max_size and rely on garbage
     collection to free memory (similarly to IPv4).

   - Support Penultimate Segment Pop (PSP) flavor in SRv6 (RFC8986).

   - ICMP: add per-rate limit counters.

   - Add support for user scanning requests in ieee802154.

   - Remove static WEP support.

   - Support minimal Wi-Fi 7 Extremely High Throughput (EHT) rate
     reporting.

   - WiFi 7 EHT channel puncturing support (client & AP).

  BPF:

   - Add a rbtree data structure following the "next-gen data structure"
     precedent set by recently added linked list, that is, by using
     kfunc + kptr instead of adding a new BPF map type.

   - Expose XDP hints via kfuncs with initial support for RX hash and
     timestamp metadata.

   - Add BPF_F_NO_TUNNEL_KEY extension to bpf_skb_set_tunnel_key to
     better support decap on GRE tunnel devices not operating in collect
     metadata.

   - Improve x86 JIT's codegen for PROBE_MEM runtime error checks.

   - Remove the need for trace_printk_lock for bpf_trace_printk and
     bpf_trace_vprintk helpers.

   - Extend libbpf's bpf_tracing.h support for tracing arguments of
     kprobes/uprobes and syscall as a special case.

   - Significantly reduce the search time for module symbols by
     livepatch and BPF.

   - Enable cpumasks to be used as kptrs, which is useful for tracing
     programs tracking which tasks end up running on which CPUs in
     different time intervals.

   - Add support for BPF trampoline on s390x and riscv64.

   - Add capability to export the XDP features supported by the NIC.

   - Add __bpf_kfunc tag for marking kernel functions as kfuncs.

   - Add cgroup.memory=nobpf kernel parameter option to disable BPF
     memory accounting for container environments.

  Netfilter:

   - Remove the CLUSTERIP target. It has been marked as obsolete for
     years, and we still have WARN splats wrt races of the out-of-band
     /proc interface installed by this target.

   - Add 'destroy' commands to nf_tables. They are identical to the
     existing 'delete' commands, but do not return an error if the
     referenced object (set, chain, rule...) did not exist.

  Driver API:

   - Improve cpumask_local_spread() locality to help NICs set the right
     IRQ affinity on AMD platforms.

   - Separate C22 and C45 MDIO bus transactions more clearly.

   - Introduce new DCB table to control DSCP rewrite on egress.

   - Support configuration of Physical Layer Collision Avoidance (PLCA)
     Reconciliation Sublayer (RS) (802.3cg-2019). Modern version of
     shared medium Ethernet.

   - Support for MAC Merge layer (IEEE 802.3-2018 clause 99). Allowing
     preemption of low priority frames by high priority frames.

   - Add support for controlling MACSec offload using netlink SET.

   - Rework devlink instance refcounts to allow registration and
     de-registration under the instance lock. Split the code into
     multiple files, drop some of the unnecessarily granular locks and
     factor out common parts of netlink operation handling.

   - Add TX frame aggregation parameters (for USB drivers).

   - Add a new attr TCA_EXT_WARN_MSG to report TC (offload) warning
     messages with notifications for debug.

   - Allow offloading of UDP NEW connections via act_ct.

   - Add support for per action HW stats in TC.

   - Support hardware miss to TC action (continue processing in SW from
     a specific point in the action chain).

   - Warn if old Wireless Extension user space interface is used with
     modern cfg80211/mac80211 drivers. Do not support Wireless
     Extensions for Wi-Fi 7 devices at all. Everyone should switch to
     using nl80211 interface instead.

   - Improve the CAN bit timing configuration. Use extack to return
     error messages directly to user space, update the SJW handling,
     including the definition of a new default value that will benefit
     CAN-FD controllers, by increasing their oscillator tolerance.

  New hardware / drivers:

   - Ethernet:
      - nVidia BlueField-3 support (control traffic driver)
      - Ethernet support for imx93 SoCs
      - Motorcomm yt8531 gigabit Ethernet PHY
      - onsemi NCN26000 10BASE-T1S PHY (with support for PLCA)
      - Microchip LAN8841 PHY (incl. cable diagnostics and PTP)
      - Amlogic gxl MDIO mux

   - WiFi:
      - RealTek RTL8188EU (rtl8xxxu)
      - Qualcomm Wi-Fi 7 devices (ath12k)

   - CAN:
      - Renesas R-Car V4H

  Drivers:

   - Bluetooth:
      - Set Per Platform Antenna Gain (PPAG) for Intel controllers.

   - Ethernet NICs:
      - Intel (1G, igc):
         - support TSN / Qbv / packet scheduling features of i226 model
      - Intel (100G, ice):
         - use GNSS subsystem instead of TTY
         - multi-buffer XDP support
         - extend support for GPIO pins to E823 devices
      - nVidia/Mellanox:
         - update the shared buffer configuration on PFC commands
         - implement PTP adjphase function for HW offset control
         - TC support for Geneve and GRE with VF tunnel offload
         - more efficient crypto key management method
         - multi-port eswitch support
      - Netronome/Corigine:
         - add DCB IEEE support
         - support IPsec offloading for NFP3800
      - Freescale/NXP (enetc):
         - support XDP_REDIRECT for XDP non-linear buffers
         - improve reconfig, avoid link flap and waiting for idle
         - support MAC Merge layer
      - Other NICs:
         - sfc/ef100: add basic devlink support for ef100
         - ionic: rx_push mode operation (writing descriptors via MMIO)
         - bnxt: use the auxiliary bus abstraction for RDMA
         - r8169: disable ASPM and reset bus in case of tx timeout
         - cpsw: support QSGMII mode for J721e CPSW9G
         - cpts: support pulse-per-second output
         - ngbe: add an mdio bus driver
         - usbnet: optimize usbnet_bh() by avoiding unnecessary queuing
         - r8152: handle devices with FW with NCM support
         - amd-xgbe: support 10Mbps, 2.5GbE speeds and rx-adaptation
         - virtio-net: support multi buffer XDP
         - virtio/vsock: replace virtio_vsock_pkt with sk_buff
         - tsnep: XDP support

   - Ethernet high-speed switches:
      - nVidia/Mellanox (mlxsw):
         - add support for latency TLV (in FW control messages)
      - Microchip (sparx5):
         - separate explicit and implicit traffic forwarding rules, make
           the implicit rules always active
         - add support for egress DSCP rewrite
         - IS0 VCAP support (Ingress Classification)
         - IS2 VCAP filters (protos, L3 addrs, L4 ports, flags, ToS
           etc.)
         - ES2 VCAP support (Egress Access Control)
         - support for Per-Stream Filtering and Policing (802.1Q,
           8.6.5.1)

   - Ethernet embedded switches:
      - Marvell (mv88e6xxx):
         - add MAB (port auth) offload support
         - enable PTP receive for mv88e6390
      - NXP (ocelot):
         - support MAC Merge layer
         - support for the the vsc7512 internal copper phys
      - Microchip:
         - lan9303: convert to PHYLINK
         - lan966x: support TC flower filter statistics
         - lan937x: PTP support for KSZ9563/KSZ8563 and LAN937x
         - lan937x: support Credit Based Shaper configuration
         - ksz9477: support Energy Efficient Ethernet
      - other:
         - qca8k: convert to regmap read/write API, use bulk operations
         - rswitch: Improve TX timestamp accuracy

   - Intel WiFi (iwlwifi):
      - EHT (Wi-Fi 7) rate reporting
      - STEP equalizer support: transfer some STEP (connection to radio
        on platforms with integrated wifi) related parameters from the
        BIOS to the firmware.

   - Qualcomm 802.11ax WiFi (ath11k):
      - IPQ5018 support
      - Fine Timing Measurement (FTM) responder role support
      - channel 177 support

   - MediaTek WiFi (mt76):
      - per-PHY LED support
      - mt7996: EHT (Wi-Fi 7) support
      - Wireless Ethernet Dispatch (WED) reset support
      - switch to using page pool allocator

   - RealTek WiFi (rtw89):
      - support new version of Bluetooth co-existance

   - Mobile:
      - rmnet: support TX aggregation"

* tag 'net-next-6.3' of git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next: (1872 commits)
  page_pool: add a comment explaining the fragment counter usage
  net: ethtool: fix __ethtool_dev_mm_supported() implementation
  ethtool: pse-pd: Fix double word in comments
  xsk: add linux/vmalloc.h to xsk.c
  sefltests: netdevsim: wait for devlink instance after netns removal
  selftest: fib_tests: Always cleanup before exit
  net/mlx5e: Align IPsec ASO result memory to be as required by hardware
  net/mlx5e: TC, Set CT miss to the specific ct action instance
  net/mlx5e: Rename CHAIN_TO_REG to MAPPED_OBJ_TO_REG
  net/mlx5: Refactor tc miss handling to a single function
  net/mlx5: Kconfig: Make tc offload depend on tc skb extension
  net/sched: flower: Support hardware miss to tc action
  net/sched: flower: Move filter handle initialization earlier
  net/sched: cls_api: Support hardware miss to tc action
  net/sched: Rename user cookie and act cookie
  sfc: fix builds without CONFIG_RTC_LIB
  sfc: clean up some inconsistent indentings
  net/mlx4_en: Introduce flexible array to silence overflow warning
  net: lan966x: Fix possible deadlock inside PTP
  net/ulp: Remove redundant ->clone() test in inet_clone_ulp().
  ...

1 files changed, 681 insertions, 0 deletions

diff --git a/arch/mips/kernel/genex.S b/arch/mips/kernel/genex.S
new file mode 100644
index 000000000..3425df601
--- /dev/null
+++ b/arch/mips/kernel/genex.S
@@ -0,0 +1,681 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 1994 - 2000, 2001, 2003 Ralf Baechle
+ * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
+ * Copyright (C) 2002, 2007  Maciej W. Rozycki
+ * Copyright (C) 2001, 2012 MIPS Technologies, Inc.  All rights reserved.
+ */
+#include <linux/init.h>
+
+#include <asm/asm.h>
+#include <asm/asmmacro.h>
+#include <asm/cacheops.h>
+#include <asm/irqflags.h>
+#include <asm/regdef.h>
+#include <asm/fpregdef.h>
+#include <asm/mipsregs.h>
+#include <asm/stackframe.h>
+#include <asm/sync.h>
+#include <asm/thread_info.h>
+
+	__INIT
+
+/*
+ * General exception vector for all other CPUs.
+ *
+ * Be careful when changing this, it has to be at most 128 bytes
+ * to fit into space reserved for the exception handler.
+ */
+NESTED(except_vec3_generic, 0, sp)
+	.set	push
+	.set	noat
+	mfc0	k1, CP0_CAUSE
+	andi	k1, k1, 0x7c
+#ifdef CONFIG_64BIT
+	dsll	k1, k1, 1
+#endif
+	PTR_L	k0, exception_handlers(k1)
+	jr	k0
+	.set	pop
+	END(except_vec3_generic)
+
+/*
+ * General exception handler for CPUs with virtual coherency exception.
+ *
+ * Be careful when changing this, it has to be at most 256 (as a special
+ * exception) bytes to fit into space reserved for the exception handler.
+ */
+NESTED(except_vec3_r4000, 0, sp)
+	.set	push
+	.set	arch=r4000
+	.set	noat
+	mfc0	k1, CP0_CAUSE
+	li	k0, 31<<2
+	andi	k1, k1, 0x7c
+	.set	push
+	.set	noreorder
+	.set	nomacro
+	beq	k1, k0, handle_vced
+	 li	k0, 14<<2
+	beq	k1, k0, handle_vcei
+#ifdef CONFIG_64BIT
+	 dsll	k1, k1, 1
+#endif
+	.set	pop
+	PTR_L	k0, exception_handlers(k1)
+	jr	k0
+
+	/*
+	 * Big shit, we now may have two dirty primary cache lines for the same
+	 * physical address.  We can safely invalidate the line pointed to by
+	 * c0_badvaddr because after return from this exception handler the
+	 * load / store will be re-executed.
+	 */
+handle_vced:
+	MFC0	k0, CP0_BADVADDR
+	li	k1, -4					# Is this ...
+	and	k0, k1					# ... really needed?
+	mtc0	zero, CP0_TAGLO
+	cache	Index_Store_Tag_D, (k0)
+	cache	Hit_Writeback_Inv_SD, (k0)
+#ifdef CONFIG_PROC_FS
+	PTR_LA	k0, vced_count
+	lw	k1, (k0)
+	addiu	k1, 1
+	sw	k1, (k0)
+#endif
+	eret
+
+handle_vcei:
+	MFC0	k0, CP0_BADVADDR
+	cache	Hit_Writeback_Inv_SD, (k0)		# also cleans pi
+#ifdef CONFIG_PROC_FS
+	PTR_LA	k0, vcei_count
+	lw	k1, (k0)
+	addiu	k1, 1
+	sw	k1, (k0)
+#endif
+	eret
+	.set	pop
+	END(except_vec3_r4000)
+
+	__FINIT
+
+	.align	5	/* 32 byte rollback region */
+LEAF(__r4k_wait)
+	.set	push
+	.set	noreorder
+	/* start of rollback region */
+	LONG_L	t0, TI_FLAGS($28)
+	nop
+	andi	t0, _TIF_NEED_RESCHED
+	bnez	t0, 1f
+	 nop
+	nop
+	nop
+#ifdef CONFIG_CPU_MICROMIPS
+	nop
+	nop
+	nop
+	nop
+#endif
+	.set	MIPS_ISA_ARCH_LEVEL_RAW
+	wait
+	/* end of rollback region (the region size must be power of two) */
+1:
+	jr	ra
+	 nop
+	.set	pop
+	END(__r4k_wait)
+
+	.macro	BUILD_ROLLBACK_PROLOGUE handler
+	FEXPORT(rollback_\handler)
+	.set	push
+	.set	noat
+	MFC0	k0, CP0_EPC
+	PTR_LA	k1, __r4k_wait
+	ori	k0, 0x1f	/* 32 byte rollback region */
+	xori	k0, 0x1f
+	bne	k0, k1, \handler
+	MTC0	k0, CP0_EPC
+	.set pop
+	.endm
+
+	.align	5
+BUILD_ROLLBACK_PROLOGUE handle_int
+NESTED(handle_int, PT_SIZE, sp)
+	.cfi_signal_frame
+#ifdef CONFIG_TRACE_IRQFLAGS
+	/*
+	 * Check to see if the interrupted code has just disabled
+	 * interrupts and ignore this interrupt for now if so.
+	 *
+	 * local_irq_disable() disables interrupts and then calls
+	 * trace_hardirqs_off() to track the state. If an interrupt is taken
+	 * after interrupts are disabled but before the state is updated
+	 * it will appear to restore_all that it is incorrectly returning with
+	 * interrupts disabled
+	 */
+	.set	push
+	.set	noat
+	mfc0	k0, CP0_STATUS
+#if defined(CONFIG_CPU_R3000)
+	and	k0, ST0_IEP
+	bnez	k0, 1f
+
+	mfc0	k0, CP0_EPC
+	.set	noreorder
+	j	k0
+	 rfe
+#else
+	and	k0, ST0_IE
+	bnez	k0, 1f
+
+	eret
+#endif
+1:
+	.set pop
+#endif
+	SAVE_ALL docfi=1
+	CLI
+	TRACE_IRQS_OFF
+
+	LONG_L	s0, TI_REGS($28)
+	LONG_S	sp, TI_REGS($28)
+
+	/*
+	 * SAVE_ALL ensures we are using a valid kernel stack for the thread.
+	 * Check if we are already using the IRQ stack.
+	 */
+	move	s1, sp # Preserve the sp
+
+	/* Get IRQ stack for this CPU */
+	ASM_CPUID_MFC0	k0, ASM_SMP_CPUID_REG
+#if defined(CONFIG_32BIT) || defined(KBUILD_64BIT_SYM32)
+	lui	k1, %hi(irq_stack)
+#else
+	lui	k1, %highest(irq_stack)
+	daddiu	k1, %higher(irq_stack)
+	dsll	k1, 16
+	daddiu	k1, %hi(irq_stack)
+	dsll	k1, 16
+#endif
+	LONG_SRL	k0, SMP_CPUID_PTRSHIFT
+	LONG_ADDU	k1, k0
+	LONG_L	t0, %lo(irq_stack)(k1)
+
+	# Check if already on IRQ stack
+	PTR_LI	t1, ~(_THREAD_SIZE-1)
+	and	t1, t1, sp
+	beq	t0, t1, 2f
+
+	/* Switch to IRQ stack */
+	li	t1, _IRQ_STACK_START
+	PTR_ADD sp, t0, t1
+
+	/* Save task's sp on IRQ stack so that unwinding can follow it */
+	LONG_S	s1, 0(sp)
+2:
+	jal	plat_irq_dispatch
+
+	/* Restore sp */
+	move	sp, s1
+
+	j	ret_from_irq
+#ifdef CONFIG_CPU_MICROMIPS
+	nop
+#endif
+	END(handle_int)
+
+	__INIT
+
+/*
+ * Special interrupt vector for MIPS64 ISA & embedded MIPS processors.
+ * This is a dedicated interrupt exception vector which reduces the
+ * interrupt processing overhead.  The jump instruction will be replaced
+ * at the initialization time.
+ *
+ * Be careful when changing this, it has to be at most 128 bytes
+ * to fit into space reserved for the exception handler.
+ */
+NESTED(except_vec4, 0, sp)
+1:	j	1b			/* Dummy, will be replaced */
+	END(except_vec4)
+
+/*
+ * EJTAG debug exception handler.
+ * The EJTAG debug exception entry point is 0xbfc00480, which
+ * normally is in the boot PROM, so the boot PROM must do an
+ * unconditional jump to this vector.
+ */
+NESTED(except_vec_ejtag_debug, 0, sp)
+	j	ejtag_debug_handler
+#ifdef CONFIG_CPU_MICROMIPS
+	 nop
+#endif
+	END(except_vec_ejtag_debug)
+
+	__FINIT
+
+/*
+ * Vectored interrupt handler.
+ * This prototype is copied to ebase + n*IntCtl.VS and patched
+ * to invoke the handler
+ */
+BUILD_ROLLBACK_PROLOGUE except_vec_vi
+NESTED(except_vec_vi, 0, sp)
+	SAVE_SOME docfi=1
+	SAVE_AT docfi=1
+	.set	push
+	.set	noreorder
+	PTR_LA	v1, except_vec_vi_handler
+FEXPORT(except_vec_vi_lui)
+	lui	v0, 0		/* Patched */
+	jr	v1
+FEXPORT(except_vec_vi_ori)
+	 ori	v0, 0		/* Patched */
+	.set	pop
+	END(except_vec_vi)
+EXPORT(except_vec_vi_end)
+
+/*
+ * Common Vectored Interrupt code
+ * Complete the register saves and invoke the handler which is passed in $v0
+ */
+NESTED(except_vec_vi_handler, 0, sp)
+	SAVE_TEMP
+	SAVE_STATIC
+	CLI
+#ifdef CONFIG_TRACE_IRQFLAGS
+	move	s0, v0
+	TRACE_IRQS_OFF
+	move	v0, s0
+#endif
+
+	LONG_L	s0, TI_REGS($28)
+	LONG_S	sp, TI_REGS($28)
+
+	/*
+	 * SAVE_ALL ensures we are using a valid kernel stack for the thread.
+	 * Check if we are already using the IRQ stack.
+	 */
+	move	s1, sp # Preserve the sp
+
+	/* Get IRQ stack for this CPU */
+	ASM_CPUID_MFC0	k0, ASM_SMP_CPUID_REG
+#if defined(CONFIG_32BIT) || defined(KBUILD_64BIT_SYM32)
+	lui	k1, %hi(irq_stack)
+#else
+	lui	k1, %highest(irq_stack)
+	daddiu	k1, %higher(irq_stack)
+	dsll	k1, 16
+	daddiu	k1, %hi(irq_stack)
+	dsll	k1, 16
+#endif
+	LONG_SRL	k0, SMP_CPUID_PTRSHIFT
+	LONG_ADDU	k1, k0
+	LONG_L	t0, %lo(irq_stack)(k1)
+
+	# Check if already on IRQ stack
+	PTR_LI	t1, ~(_THREAD_SIZE-1)
+	and	t1, t1, sp
+	beq	t0, t1, 2f
+
+	/* Switch to IRQ stack */
+	li	t1, _IRQ_STACK_START
+	PTR_ADD sp, t0, t1
+
+	/* Save task's sp on IRQ stack so that unwinding can follow it */
+	LONG_S	s1, 0(sp)
+2:
+	jalr	v0
+
+	/* Restore sp */
+	move	sp, s1
+
+	j	ret_from_irq
+	END(except_vec_vi_handler)
+
+/*
+ * EJTAG debug exception handler.
+ */
+NESTED(ejtag_debug_handler, PT_SIZE, sp)
+	.set	push
+	.set	noat
+	MTC0	k0, CP0_DESAVE
+	mfc0	k0, CP0_DEBUG
+
+	andi	k0, k0, MIPS_DEBUG_DBP	# Check for SDBBP.
+	beqz	k0, ejtag_return
+
+#ifdef CONFIG_SMP
+1:	PTR_LA	k0, ejtag_debug_buffer_spinlock
+	__SYNC(full, loongson3_war)
+2:	ll	k0, 0(k0)
+	bnez	k0, 2b
+	PTR_LA	k0, ejtag_debug_buffer_spinlock
+	sc	k0, 0(k0)
+	beqz	k0, 1b
+# ifdef CONFIG_WEAK_REORDERING_BEYOND_LLSC
+	sync
+# endif
+
+	PTR_LA	k0, ejtag_debug_buffer
+	LONG_S	k1, 0(k0)
+
+	ASM_CPUID_MFC0 k1, ASM_SMP_CPUID_REG
+	PTR_SRL	k1, SMP_CPUID_PTRSHIFT
+	PTR_SLL	k1, LONGLOG
+	PTR_LA	k0, ejtag_debug_buffer_per_cpu
+	PTR_ADDU k0, k1
+
+	PTR_LA	k1, ejtag_debug_buffer
+	LONG_L	k1, 0(k1)
+	LONG_S	k1, 0(k0)
+
+	PTR_LA	k0, ejtag_debug_buffer_spinlock
+	sw	zero, 0(k0)
+#else
+	PTR_LA	k0, ejtag_debug_buffer
+	LONG_S	k1, 0(k0)
+#endif
+
+	SAVE_ALL
+	move	a0, sp
+	jal	ejtag_exception_handler
+	RESTORE_ALL
+
+#ifdef CONFIG_SMP
+	ASM_CPUID_MFC0 k1, ASM_SMP_CPUID_REG
+	PTR_SRL	k1, SMP_CPUID_PTRSHIFT
+	PTR_SLL	k1, LONGLOG
+	PTR_LA	k0, ejtag_debug_buffer_per_cpu
+	PTR_ADDU k0, k1
+	LONG_L	k1, 0(k0)
+#else
+	PTR_LA	k0, ejtag_debug_buffer
+	LONG_L	k1, 0(k0)
+#endif
+
+ejtag_return:
+	back_to_back_c0_hazard
+	MFC0	k0, CP0_DESAVE
+	.set	mips32
+	deret
+	.set	pop
+	END(ejtag_debug_handler)
+
+/*
+ * This buffer is reserved for the use of the EJTAG debug
+ * handler.
+ */
+	.data
+EXPORT(ejtag_debug_buffer)
+	.fill	LONGSIZE
+#ifdef CONFIG_SMP
+EXPORT(ejtag_debug_buffer_spinlock)
+	.fill	LONGSIZE
+EXPORT(ejtag_debug_buffer_per_cpu)
+	.fill	LONGSIZE * NR_CPUS
+#endif
+	.previous
+
+	__INIT
+
+/*
+ * NMI debug exception handler for MIPS reference boards.
+ * The NMI debug exception entry point is 0xbfc00000, which
+ * normally is in the boot PROM, so the boot PROM must do a
+ * unconditional jump to this vector.
+ */
+NESTED(except_vec_nmi, 0, sp)
+	j	nmi_handler
+#ifdef CONFIG_CPU_MICROMIPS
+	 nop
+#endif
+	END(except_vec_nmi)
+
+	__FINIT
+
+NESTED(nmi_handler, PT_SIZE, sp)
+	.cfi_signal_frame
+	.set	push
+	.set	noat
+	/*
+	 * Clear ERL - restore segment mapping
+	 * Clear BEV - required for page fault exception handler to work
+	 */
+	mfc0	k0, CP0_STATUS
+	ori	k0, k0, ST0_EXL
+	li	k1, ~(ST0_BEV | ST0_ERL)
+	and	k0, k0, k1
+	mtc0	k0, CP0_STATUS
+	_ehb
+	SAVE_ALL
+	move	a0, sp
+	jal	nmi_exception_handler
+	/* nmi_exception_handler never returns */
+	.set	pop
+	END(nmi_handler)
+
+	.macro	__build_clear_none
+	.endm
+
+	.macro	__build_clear_sti
+	TRACE_IRQS_ON
+	STI
+	.endm
+
+	.macro	__build_clear_cli
+	CLI
+	TRACE_IRQS_OFF
+	.endm
+
+	.macro	__build_clear_fpe
+	CLI
+	TRACE_IRQS_OFF
+	.set	push
+	/* gas fails to assemble cfc1 for some archs (octeon).*/ \
+	.set	mips1
+	SET_HARDFLOAT
+	cfc1	a1, fcr31
+	.set	pop
+	.endm
+
+	.macro	__build_clear_msa_fpe
+	CLI
+	TRACE_IRQS_OFF
+	_cfcmsa	a1, MSA_CSR
+	.endm
+
+	.macro	__build_clear_ade
+	MFC0	t0, CP0_BADVADDR
+	PTR_S	t0, PT_BVADDR(sp)
+	KMODE
+	.endm
+
+	.macro __build_clear_gsexc
+	.set	push
+	/*
+	 * We need to specify a selector to access the CP0.Diag1 (GSCause)
+	 * register. All GSExc-equipped processors have MIPS32.
+	 */
+	.set	mips32
+	mfc0	a1, CP0_DIAGNOSTIC1
+	.set	pop
+	TRACE_IRQS_ON
+	STI
+	.endm
+
+	.macro	__BUILD_silent exception
+	.endm
+
+	/* Gas tries to parse the ASM_PRINT argument as a string containing
+	   string escapes and emits bogus warnings if it believes to
+	   recognize an unknown escape code.  So make the arguments
+	   start with an n and gas will believe \n is ok ...  */
+	.macro	__BUILD_verbose nexception
+	LONG_L	a1, PT_EPC(sp)
+#ifdef CONFIG_32BIT
+	ASM_PRINT("Got \nexception at %08lx\012")
+#endif
+#ifdef CONFIG_64BIT
+	ASM_PRINT("Got \nexception at %016lx\012")
+#endif
+	.endm
+
+	.macro	__BUILD_count exception
+	LONG_L	t0,exception_count_\exception
+	LONG_ADDIU	t0, 1
+	LONG_S	t0,exception_count_\exception
+	.comm	exception_count\exception, 8, 8
+	.endm
+
+	.macro	__BUILD_HANDLER exception handler clear verbose ext
+	.align	5
+	NESTED(handle_\exception, PT_SIZE, sp)
+	.cfi_signal_frame
+	.set	noat
+	SAVE_ALL
+	FEXPORT(handle_\exception\ext)
+	__build_clear_\clear
+	.set	at
+	__BUILD_\verbose \exception
+	move	a0, sp
+	jal	do_\handler
+	j	ret_from_exception
+	END(handle_\exception)
+	.endm
+
+	.macro	BUILD_HANDLER exception handler clear verbose
+	__BUILD_HANDLER \exception \handler \clear \verbose _int
+	.endm
+
+	BUILD_HANDLER adel ade ade silent		/* #4  */
+	BUILD_HANDLER ades ade ade silent		/* #5  */
+	BUILD_HANDLER ibe be cli silent			/* #6  */
+	BUILD_HANDLER dbe be cli silent			/* #7  */
+	BUILD_HANDLER bp bp sti silent			/* #9  */
+	BUILD_HANDLER ri ri sti silent			/* #10 */
+	BUILD_HANDLER cpu cpu sti silent		/* #11 */
+	BUILD_HANDLER ov ov sti silent			/* #12 */
+	BUILD_HANDLER tr tr sti silent			/* #13 */
+	BUILD_HANDLER msa_fpe msa_fpe msa_fpe silent	/* #14 */
+#ifdef CONFIG_MIPS_FP_SUPPORT
+	BUILD_HANDLER fpe fpe fpe silent		/* #15 */
+#endif
+	BUILD_HANDLER ftlb ftlb none silent		/* #16 */
+	BUILD_HANDLER gsexc gsexc gsexc silent		/* #16 */
+	BUILD_HANDLER msa msa sti silent		/* #21 */
+	BUILD_HANDLER mdmx mdmx sti silent		/* #22 */
+#ifdef	CONFIG_HARDWARE_WATCHPOINTS
+	/*
+	 * For watch, interrupts will be enabled after the watch
+	 * registers are read.
+	 */
+	BUILD_HANDLER watch watch cli silent		/* #23 */
+#else
+	BUILD_HANDLER watch watch sti verbose		/* #23 */
+#endif
+	BUILD_HANDLER mcheck mcheck cli verbose		/* #24 */
+	BUILD_HANDLER mt mt sti silent			/* #25 */
+	BUILD_HANDLER dsp dsp sti silent		/* #26 */
+	BUILD_HANDLER reserved reserved sti verbose	/* others */
+
+	.align	5
+	LEAF(handle_ri_rdhwr_tlbp)
+	.set	push
+	.set	noat
+	.set	noreorder
+	/* check if TLB contains a entry for EPC */
+	MFC0	k1, CP0_ENTRYHI
+	andi	k1, MIPS_ENTRYHI_ASID | MIPS_ENTRYHI_ASIDX
+	MFC0	k0, CP0_EPC
+	PTR_SRL	k0, _PAGE_SHIFT + 1
+	PTR_SLL	k0, _PAGE_SHIFT + 1
+	or	k1, k0
+	MTC0	k1, CP0_ENTRYHI
+	mtc0_tlbw_hazard
+	tlbp
+	tlb_probe_hazard
+	mfc0	k1, CP0_INDEX
+	.set	pop
+	bltz	k1, handle_ri	/* slow path */
+	/* fall thru */
+	END(handle_ri_rdhwr_tlbp)
+
+	LEAF(handle_ri_rdhwr)
+	.set	push
+	.set	noat
+	.set	noreorder
+	/* MIPS32:    0x7c03e83b: rdhwr v1,$29 */
+	/* microMIPS: 0x007d6b3c: rdhwr v1,$29 */
+	MFC0	k1, CP0_EPC
+#if defined(CONFIG_CPU_MICROMIPS) || defined(CONFIG_CPU_MIPS32_R2) || defined(CONFIG_CPU_MIPS64_R2)
+	and	k0, k1, 1
+	beqz	k0, 1f
+	 xor	k1, k0
+	lhu	k0, (k1)
+	lhu	k1, 2(k1)
+	ins	k1, k0, 16, 16
+	lui	k0, 0x007d
+	b	docheck
+	 ori	k0, 0x6b3c
+1:
+	lui	k0, 0x7c03
+	lw	k1, (k1)
+	ori	k0, 0xe83b
+#else
+	andi	k0, k1, 1
+	bnez	k0, handle_ri
+	 lui	k0, 0x7c03
+	lw	k1, (k1)
+	ori	k0, 0xe83b
+#endif
+	.set	reorder
+docheck:
+	bne	k0, k1, handle_ri	/* if not ours */
+
+isrdhwr:
+	/* The insn is rdhwr.  No need to check CAUSE.BD here. */
+	get_saved_sp	/* k1 := current_thread_info */
+	.set	noreorder
+	MFC0	k0, CP0_EPC
+#if defined(CONFIG_CPU_R3000)
+	ori	k1, _THREAD_MASK
+	xori	k1, _THREAD_MASK
+	LONG_L	v1, TI_TP_VALUE(k1)
+	LONG_ADDIU	k0, 4
+	jr	k0
+	 rfe
+#else
+#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
+	LONG_ADDIU	k0, 4		/* stall on $k0 */
+#else
+	.set	at=v1
+	LONG_ADDIU	k0, 4
+	.set	noat
+#endif
+	MTC0	k0, CP0_EPC
+	/* I hope three instructions between MTC0 and ERET are enough... */
+	ori	k1, _THREAD_MASK
+	xori	k1, _THREAD_MASK
+	LONG_L	v1, TI_TP_VALUE(k1)
+	.set	push
+	.set	arch=r4000
+	eret
+	.set	pop
+#endif
+	.set	pop
+	END(handle_ri_rdhwr)
+
+#ifdef CONFIG_CPU_R4X00_BUGS64
+/* A temporary overflow handler used by check_daddi(). */
+
+	__INIT
+
+	BUILD_HANDLER  daddi_ov daddi_ov none silent	/* #12 */
+#endif