Revision 80ef4464d5e27408685e609d389663aad46644b9 authored by Robert Richter on 20 March 2019, 18:57:23 UTC, committed by Joerg Roedel on 22 March 2019, 11:01:58 UTC
If a 32 bit allocation request is too big to possibly succeed, it early exits with a failure and then should never update max32_alloc_ size. This patch fixes current code, now the size is only updated if the slow path failed while walking the tree. Without the fix the allocation may enter the slow path again even if there was a failure before of a request with the same or a smaller size. Cc: <stable@vger.kernel.org> # 4.20+ Fixes: bee60e94a1e2 ("iommu/iova: Optimise attempts to allocate iova from 32bit address range") Reviewed-by: Robin Murphy <robin.murphy@arm.com> Signed-off-by: Robert Richter <rrichter@marvell.com> Signed-off-by: Joerg Roedel <jroedel@suse.de>
1 parent 4e50ce0
bcm-flexrm-mailbox.c
/*
* Copyright (C) 2017 Broadcom
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation version 2.
*
* This program is distributed "as is" WITHOUT ANY WARRANTY of any
* kind, whether express or implied; without even the implied warranty
* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*/
/*
* Broadcom FlexRM Mailbox Driver
*
* Each Broadcom FlexSparx4 offload engine is implemented as an
* extension to Broadcom FlexRM ring manager. The FlexRM ring
* manager provides a set of rings which can be used to submit
* work to a FlexSparx4 offload engine.
*
* This driver creates a mailbox controller using a set of FlexRM
* rings where each mailbox channel represents a separate FlexRM ring.
*/
#include <asm/barrier.h>
#include <asm/byteorder.h>
#include <linux/atomic.h>
#include <linux/bitmap.h>
#include <linux/debugfs.h>
#include <linux/delay.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/dmapool.h>
#include <linux/err.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/mailbox_controller.h>
#include <linux/mailbox_client.h>
#include <linux/mailbox/brcm-message.h>
#include <linux/module.h>
#include <linux/msi.h>
#include <linux/of_address.h>
#include <linux/of_irq.h>
#include <linux/platform_device.h>
#include <linux/spinlock.h>
/* ====== FlexRM register defines ===== */
/* FlexRM configuration */
#define RING_REGS_SIZE 0x10000
#define RING_DESC_SIZE 8
#define RING_DESC_INDEX(offset) \
((offset) / RING_DESC_SIZE)
#define RING_DESC_OFFSET(index) \
((index) * RING_DESC_SIZE)
#define RING_MAX_REQ_COUNT 1024
#define RING_BD_ALIGN_ORDER 12
#define RING_BD_ALIGN_CHECK(addr) \
(!((addr) & ((0x1 << RING_BD_ALIGN_ORDER) - 1)))
#define RING_BD_TOGGLE_INVALID(offset) \
(((offset) >> RING_BD_ALIGN_ORDER) & 0x1)
#define RING_BD_TOGGLE_VALID(offset) \
(!RING_BD_TOGGLE_INVALID(offset))
#define RING_BD_DESC_PER_REQ 32
#define RING_BD_DESC_COUNT \
(RING_MAX_REQ_COUNT * RING_BD_DESC_PER_REQ)
#define RING_BD_SIZE \
(RING_BD_DESC_COUNT * RING_DESC_SIZE)
#define RING_CMPL_ALIGN_ORDER 13
#define RING_CMPL_DESC_COUNT RING_MAX_REQ_COUNT
#define RING_CMPL_SIZE \
(RING_CMPL_DESC_COUNT * RING_DESC_SIZE)
#define RING_VER_MAGIC 0x76303031
/* Per-Ring register offsets */
#define RING_VER 0x000
#define RING_BD_START_ADDR 0x004
#define RING_BD_READ_PTR 0x008
#define RING_BD_WRITE_PTR 0x00c
#define RING_BD_READ_PTR_DDR_LS 0x010
#define RING_BD_READ_PTR_DDR_MS 0x014
#define RING_CMPL_START_ADDR 0x018
#define RING_CMPL_WRITE_PTR 0x01c
#define RING_NUM_REQ_RECV_LS 0x020
#define RING_NUM_REQ_RECV_MS 0x024
#define RING_NUM_REQ_TRANS_LS 0x028
#define RING_NUM_REQ_TRANS_MS 0x02c
#define RING_NUM_REQ_OUTSTAND 0x030
#define RING_CONTROL 0x034
#define RING_FLUSH_DONE 0x038
#define RING_MSI_ADDR_LS 0x03c
#define RING_MSI_ADDR_MS 0x040
#define RING_MSI_CONTROL 0x048
#define RING_BD_READ_PTR_DDR_CONTROL 0x04c
#define RING_MSI_DATA_VALUE 0x064
/* Register RING_BD_START_ADDR fields */
#define BD_LAST_UPDATE_HW_SHIFT 28
#define BD_LAST_UPDATE_HW_MASK 0x1
#define BD_START_ADDR_VALUE(pa) \
((u32)((((dma_addr_t)(pa)) >> RING_BD_ALIGN_ORDER) & 0x0fffffff))
#define BD_START_ADDR_DECODE(val) \
((dma_addr_t)((val) & 0x0fffffff) << RING_BD_ALIGN_ORDER)
/* Register RING_CMPL_START_ADDR fields */
#define CMPL_START_ADDR_VALUE(pa) \
((u32)((((u64)(pa)) >> RING_CMPL_ALIGN_ORDER) & 0x07ffffff))
/* Register RING_CONTROL fields */
#define CONTROL_MASK_DISABLE_CONTROL 12
#define CONTROL_FLUSH_SHIFT 5
#define CONTROL_ACTIVE_SHIFT 4
#define CONTROL_RATE_ADAPT_MASK 0xf
#define CONTROL_RATE_DYNAMIC 0x0
#define CONTROL_RATE_FAST 0x8
#define CONTROL_RATE_MEDIUM 0x9
#define CONTROL_RATE_SLOW 0xa
#define CONTROL_RATE_IDLE 0xb
/* Register RING_FLUSH_DONE fields */
#define FLUSH_DONE_MASK 0x1
/* Register RING_MSI_CONTROL fields */
#define MSI_TIMER_VAL_SHIFT 16
#define MSI_TIMER_VAL_MASK 0xffff
#define MSI_ENABLE_SHIFT 15
#define MSI_ENABLE_MASK 0x1
#define MSI_COUNT_SHIFT 0
#define MSI_COUNT_MASK 0x3ff
/* Register RING_BD_READ_PTR_DDR_CONTROL fields */
#define BD_READ_PTR_DDR_TIMER_VAL_SHIFT 16
#define BD_READ_PTR_DDR_TIMER_VAL_MASK 0xffff
#define BD_READ_PTR_DDR_ENABLE_SHIFT 15
#define BD_READ_PTR_DDR_ENABLE_MASK 0x1
/* ====== FlexRM ring descriptor defines ===== */
/* Completion descriptor format */
#define CMPL_OPAQUE_SHIFT 0
#define CMPL_OPAQUE_MASK 0xffff
#define CMPL_ENGINE_STATUS_SHIFT 16
#define CMPL_ENGINE_STATUS_MASK 0xffff
#define CMPL_DME_STATUS_SHIFT 32
#define CMPL_DME_STATUS_MASK 0xffff
#define CMPL_RM_STATUS_SHIFT 48
#define CMPL_RM_STATUS_MASK 0xffff
/* Completion DME status code */
#define DME_STATUS_MEM_COR_ERR BIT(0)
#define DME_STATUS_MEM_UCOR_ERR BIT(1)
#define DME_STATUS_FIFO_UNDERFLOW BIT(2)
#define DME_STATUS_FIFO_OVERFLOW BIT(3)
#define DME_STATUS_RRESP_ERR BIT(4)
#define DME_STATUS_BRESP_ERR BIT(5)
#define DME_STATUS_ERROR_MASK (DME_STATUS_MEM_COR_ERR | \
DME_STATUS_MEM_UCOR_ERR | \
DME_STATUS_FIFO_UNDERFLOW | \
DME_STATUS_FIFO_OVERFLOW | \
DME_STATUS_RRESP_ERR | \
DME_STATUS_BRESP_ERR)
/* Completion RM status code */
#define RM_STATUS_CODE_SHIFT 0
#define RM_STATUS_CODE_MASK 0x3ff
#define RM_STATUS_CODE_GOOD 0x0
#define RM_STATUS_CODE_AE_TIMEOUT 0x3ff
/* General descriptor format */
#define DESC_TYPE_SHIFT 60
#define DESC_TYPE_MASK 0xf
#define DESC_PAYLOAD_SHIFT 0
#define DESC_PAYLOAD_MASK 0x0fffffffffffffff
/* Null descriptor format */
#define NULL_TYPE 0
#define NULL_TOGGLE_SHIFT 58
#define NULL_TOGGLE_MASK 0x1
/* Header descriptor format */
#define HEADER_TYPE 1
#define HEADER_TOGGLE_SHIFT 58
#define HEADER_TOGGLE_MASK 0x1
#define HEADER_ENDPKT_SHIFT 57
#define HEADER_ENDPKT_MASK 0x1
#define HEADER_STARTPKT_SHIFT 56
#define HEADER_STARTPKT_MASK 0x1
#define HEADER_BDCOUNT_SHIFT 36
#define HEADER_BDCOUNT_MASK 0x1f
#define HEADER_BDCOUNT_MAX HEADER_BDCOUNT_MASK
#define HEADER_FLAGS_SHIFT 16
#define HEADER_FLAGS_MASK 0xffff
#define HEADER_OPAQUE_SHIFT 0
#define HEADER_OPAQUE_MASK 0xffff
/* Source (SRC) descriptor format */
#define SRC_TYPE 2
#define SRC_LENGTH_SHIFT 44
#define SRC_LENGTH_MASK 0xffff
#define SRC_ADDR_SHIFT 0
#define SRC_ADDR_MASK 0x00000fffffffffff
/* Destination (DST) descriptor format */
#define DST_TYPE 3
#define DST_LENGTH_SHIFT 44
#define DST_LENGTH_MASK 0xffff
#define DST_ADDR_SHIFT 0
#define DST_ADDR_MASK 0x00000fffffffffff
/* Immediate (IMM) descriptor format */
#define IMM_TYPE 4
#define IMM_DATA_SHIFT 0
#define IMM_DATA_MASK 0x0fffffffffffffff
/* Next pointer (NPTR) descriptor format */
#define NPTR_TYPE 5
#define NPTR_TOGGLE_SHIFT 58
#define NPTR_TOGGLE_MASK 0x1
#define NPTR_ADDR_SHIFT 0
#define NPTR_ADDR_MASK 0x00000fffffffffff
/* Mega source (MSRC) descriptor format */
#define MSRC_TYPE 6
#define MSRC_LENGTH_SHIFT 44
#define MSRC_LENGTH_MASK 0xffff
#define MSRC_ADDR_SHIFT 0
#define MSRC_ADDR_MASK 0x00000fffffffffff
/* Mega destination (MDST) descriptor format */
#define MDST_TYPE 7
#define MDST_LENGTH_SHIFT 44
#define MDST_LENGTH_MASK 0xffff
#define MDST_ADDR_SHIFT 0
#define MDST_ADDR_MASK 0x00000fffffffffff
/* Source with tlast (SRCT) descriptor format */
#define SRCT_TYPE 8
#define SRCT_LENGTH_SHIFT 44
#define SRCT_LENGTH_MASK 0xffff
#define SRCT_ADDR_SHIFT 0
#define SRCT_ADDR_MASK 0x00000fffffffffff
/* Destination with tlast (DSTT) descriptor format */
#define DSTT_TYPE 9
#define DSTT_LENGTH_SHIFT 44
#define DSTT_LENGTH_MASK 0xffff
#define DSTT_ADDR_SHIFT 0
#define DSTT_ADDR_MASK 0x00000fffffffffff
/* Immediate with tlast (IMMT) descriptor format */
#define IMMT_TYPE 10
#define IMMT_DATA_SHIFT 0
#define IMMT_DATA_MASK 0x0fffffffffffffff
/* Descriptor helper macros */
#define DESC_DEC(_d, _s, _m) (((_d) >> (_s)) & (_m))
#define DESC_ENC(_d, _v, _s, _m) \
do { \
(_d) &= ~((u64)(_m) << (_s)); \
(_d) |= (((u64)(_v) & (_m)) << (_s)); \
} while (0)
/* ====== FlexRM data structures ===== */
struct flexrm_ring {
/* Unprotected members */
int num;
struct flexrm_mbox *mbox;
void __iomem *regs;
bool irq_requested;
unsigned int irq;
cpumask_t irq_aff_hint;
unsigned int msi_timer_val;
unsigned int msi_count_threshold;
struct brcm_message *requests[RING_MAX_REQ_COUNT];
void *bd_base;
dma_addr_t bd_dma_base;
u32 bd_write_offset;
void *cmpl_base;
dma_addr_t cmpl_dma_base;
/* Atomic stats */
atomic_t msg_send_count;
atomic_t msg_cmpl_count;
/* Protected members */
spinlock_t lock;
DECLARE_BITMAP(requests_bmap, RING_MAX_REQ_COUNT);
u32 cmpl_read_offset;
};
struct flexrm_mbox {
struct device *dev;
void __iomem *regs;
u32 num_rings;
struct flexrm_ring *rings;
struct dma_pool *bd_pool;
struct dma_pool *cmpl_pool;
struct dentry *root;
struct dentry *config;
struct dentry *stats;
struct mbox_controller controller;
};
/* ====== FlexRM ring descriptor helper routines ===== */
static u64 flexrm_read_desc(void *desc_ptr)
{
return le64_to_cpu(*((u64 *)desc_ptr));
}
static void flexrm_write_desc(void *desc_ptr, u64 desc)
{
*((u64 *)desc_ptr) = cpu_to_le64(desc);
}
static u32 flexrm_cmpl_desc_to_reqid(u64 cmpl_desc)
{
return (u32)(cmpl_desc & CMPL_OPAQUE_MASK);
}
static int flexrm_cmpl_desc_to_error(u64 cmpl_desc)
{
u32 status;
status = DESC_DEC(cmpl_desc, CMPL_DME_STATUS_SHIFT,
CMPL_DME_STATUS_MASK);
if (status & DME_STATUS_ERROR_MASK)
return -EIO;
status = DESC_DEC(cmpl_desc, CMPL_RM_STATUS_SHIFT,
CMPL_RM_STATUS_MASK);
status &= RM_STATUS_CODE_MASK;
if (status == RM_STATUS_CODE_AE_TIMEOUT)
return -ETIMEDOUT;
return 0;
}
static bool flexrm_is_next_table_desc(void *desc_ptr)
{
u64 desc = flexrm_read_desc(desc_ptr);
u32 type = DESC_DEC(desc, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
return (type == NPTR_TYPE) ? true : false;
}
static u64 flexrm_next_table_desc(u32 toggle, dma_addr_t next_addr)
{
u64 desc = 0;
DESC_ENC(desc, NPTR_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
DESC_ENC(desc, toggle, NPTR_TOGGLE_SHIFT, NPTR_TOGGLE_MASK);
DESC_ENC(desc, next_addr, NPTR_ADDR_SHIFT, NPTR_ADDR_MASK);
return desc;
}
static u64 flexrm_null_desc(u32 toggle)
{
u64 desc = 0;
DESC_ENC(desc, NULL_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
DESC_ENC(desc, toggle, NULL_TOGGLE_SHIFT, NULL_TOGGLE_MASK);
return desc;
}
static u32 flexrm_estimate_header_desc_count(u32 nhcnt)
{
u32 hcnt = nhcnt / HEADER_BDCOUNT_MAX;
if (!(nhcnt % HEADER_BDCOUNT_MAX))
hcnt += 1;
return hcnt;
}
static void flexrm_flip_header_toggle(void *desc_ptr)
{
u64 desc = flexrm_read_desc(desc_ptr);
if (desc & ((u64)0x1 << HEADER_TOGGLE_SHIFT))
desc &= ~((u64)0x1 << HEADER_TOGGLE_SHIFT);
else
desc |= ((u64)0x1 << HEADER_TOGGLE_SHIFT);
flexrm_write_desc(desc_ptr, desc);
}
static u64 flexrm_header_desc(u32 toggle, u32 startpkt, u32 endpkt,
u32 bdcount, u32 flags, u32 opaque)
{
u64 desc = 0;
DESC_ENC(desc, HEADER_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
DESC_ENC(desc, toggle, HEADER_TOGGLE_SHIFT, HEADER_TOGGLE_MASK);
DESC_ENC(desc, startpkt, HEADER_STARTPKT_SHIFT, HEADER_STARTPKT_MASK);
DESC_ENC(desc, endpkt, HEADER_ENDPKT_SHIFT, HEADER_ENDPKT_MASK);
DESC_ENC(desc, bdcount, HEADER_BDCOUNT_SHIFT, HEADER_BDCOUNT_MASK);
DESC_ENC(desc, flags, HEADER_FLAGS_SHIFT, HEADER_FLAGS_MASK);
DESC_ENC(desc, opaque, HEADER_OPAQUE_SHIFT, HEADER_OPAQUE_MASK);
return desc;
}
static void flexrm_enqueue_desc(u32 nhpos, u32 nhcnt, u32 reqid,
u64 desc, void **desc_ptr, u32 *toggle,
void *start_desc, void *end_desc)
{
u64 d;
u32 nhavail, _toggle, _startpkt, _endpkt, _bdcount;
/* Sanity check */
if (nhcnt <= nhpos)
return;
/*
* Each request or packet start with a HEADER descriptor followed
* by one or more non-HEADER descriptors (SRC, SRCT, MSRC, DST,
* DSTT, MDST, IMM, and IMMT). The number of non-HEADER descriptors
* following a HEADER descriptor is represented by BDCOUNT field
* of HEADER descriptor. The max value of BDCOUNT field is 31 which
* means we can only have 31 non-HEADER descriptors following one
* HEADER descriptor.
*
* In general use, number of non-HEADER descriptors can easily go
* beyond 31. To tackle this situation, we have packet (or request)
* extenstion bits (STARTPKT and ENDPKT) in the HEADER descriptor.
*
* To use packet extension, the first HEADER descriptor of request
* (or packet) will have STARTPKT=1 and ENDPKT=0. The intermediate
* HEADER descriptors will have STARTPKT=0 and ENDPKT=0. The last
* HEADER descriptor will have STARTPKT=0 and ENDPKT=1. Also, the
* TOGGLE bit of the first HEADER will be set to invalid state to
* ensure that FlexRM does not start fetching descriptors till all
* descriptors are enqueued. The user of this function will flip
* the TOGGLE bit of first HEADER after all descriptors are
* enqueued.
*/
if ((nhpos % HEADER_BDCOUNT_MAX == 0) && (nhcnt - nhpos)) {
/* Prepare the header descriptor */
nhavail = (nhcnt - nhpos);
_toggle = (nhpos == 0) ? !(*toggle) : (*toggle);
_startpkt = (nhpos == 0) ? 0x1 : 0x0;
_endpkt = (nhavail <= HEADER_BDCOUNT_MAX) ? 0x1 : 0x0;
_bdcount = (nhavail <= HEADER_BDCOUNT_MAX) ?
nhavail : HEADER_BDCOUNT_MAX;
if (nhavail <= HEADER_BDCOUNT_MAX)
_bdcount = nhavail;
else
_bdcount = HEADER_BDCOUNT_MAX;
d = flexrm_header_desc(_toggle, _startpkt, _endpkt,
_bdcount, 0x0, reqid);
/* Write header descriptor */
flexrm_write_desc(*desc_ptr, d);
/* Point to next descriptor */
*desc_ptr += sizeof(desc);
if (*desc_ptr == end_desc)
*desc_ptr = start_desc;
/* Skip next pointer descriptors */
while (flexrm_is_next_table_desc(*desc_ptr)) {
*toggle = (*toggle) ? 0 : 1;
*desc_ptr += sizeof(desc);
if (*desc_ptr == end_desc)
*desc_ptr = start_desc;
}
}
/* Write desired descriptor */
flexrm_write_desc(*desc_ptr, desc);
/* Point to next descriptor */
*desc_ptr += sizeof(desc);
if (*desc_ptr == end_desc)
*desc_ptr = start_desc;
/* Skip next pointer descriptors */
while (flexrm_is_next_table_desc(*desc_ptr)) {
*toggle = (*toggle) ? 0 : 1;
*desc_ptr += sizeof(desc);
if (*desc_ptr == end_desc)
*desc_ptr = start_desc;
}
}
static u64 flexrm_src_desc(dma_addr_t addr, unsigned int length)
{
u64 desc = 0;
DESC_ENC(desc, SRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
DESC_ENC(desc, length, SRC_LENGTH_SHIFT, SRC_LENGTH_MASK);
DESC_ENC(desc, addr, SRC_ADDR_SHIFT, SRC_ADDR_MASK);
return desc;
}
static u64 flexrm_msrc_desc(dma_addr_t addr, unsigned int length_div_16)
{
u64 desc = 0;
DESC_ENC(desc, MSRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
DESC_ENC(desc, length_div_16, MSRC_LENGTH_SHIFT, MSRC_LENGTH_MASK);
DESC_ENC(desc, addr, MSRC_ADDR_SHIFT, MSRC_ADDR_MASK);
return desc;
}
static u64 flexrm_dst_desc(dma_addr_t addr, unsigned int length)
{
u64 desc = 0;
DESC_ENC(desc, DST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
DESC_ENC(desc, length, DST_LENGTH_SHIFT, DST_LENGTH_MASK);
DESC_ENC(desc, addr, DST_ADDR_SHIFT, DST_ADDR_MASK);
return desc;
}
static u64 flexrm_mdst_desc(dma_addr_t addr, unsigned int length_div_16)
{
u64 desc = 0;
DESC_ENC(desc, MDST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
DESC_ENC(desc, length_div_16, MDST_LENGTH_SHIFT, MDST_LENGTH_MASK);
DESC_ENC(desc, addr, MDST_ADDR_SHIFT, MDST_ADDR_MASK);
return desc;
}
static u64 flexrm_imm_desc(u64 data)
{
u64 desc = 0;
DESC_ENC(desc, IMM_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
DESC_ENC(desc, data, IMM_DATA_SHIFT, IMM_DATA_MASK);
return desc;
}
static u64 flexrm_srct_desc(dma_addr_t addr, unsigned int length)
{
u64 desc = 0;
DESC_ENC(desc, SRCT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
DESC_ENC(desc, length, SRCT_LENGTH_SHIFT, SRCT_LENGTH_MASK);
DESC_ENC(desc, addr, SRCT_ADDR_SHIFT, SRCT_ADDR_MASK);
return desc;
}
static u64 flexrm_dstt_desc(dma_addr_t addr, unsigned int length)
{
u64 desc = 0;
DESC_ENC(desc, DSTT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
DESC_ENC(desc, length, DSTT_LENGTH_SHIFT, DSTT_LENGTH_MASK);
DESC_ENC(desc, addr, DSTT_ADDR_SHIFT, DSTT_ADDR_MASK);
return desc;
}
static u64 flexrm_immt_desc(u64 data)
{
u64 desc = 0;
DESC_ENC(desc, IMMT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
DESC_ENC(desc, data, IMMT_DATA_SHIFT, IMMT_DATA_MASK);
return desc;
}
static bool flexrm_spu_sanity_check(struct brcm_message *msg)
{
struct scatterlist *sg;
if (!msg->spu.src || !msg->spu.dst)
return false;
for (sg = msg->spu.src; sg; sg = sg_next(sg)) {
if (sg->length & 0xf) {
if (sg->length > SRC_LENGTH_MASK)
return false;
} else {
if (sg->length > (MSRC_LENGTH_MASK * 16))
return false;
}
}
for (sg = msg->spu.dst; sg; sg = sg_next(sg)) {
if (sg->length & 0xf) {
if (sg->length > DST_LENGTH_MASK)
return false;
} else {
if (sg->length > (MDST_LENGTH_MASK * 16))
return false;
}
}
return true;
}
static u32 flexrm_spu_estimate_nonheader_desc_count(struct brcm_message *msg)
{
u32 cnt = 0;
unsigned int dst_target = 0;
struct scatterlist *src_sg = msg->spu.src, *dst_sg = msg->spu.dst;
while (src_sg || dst_sg) {
if (src_sg) {
cnt++;
dst_target = src_sg->length;
src_sg = sg_next(src_sg);
} else
dst_target = UINT_MAX;
while (dst_target && dst_sg) {
cnt++;
if (dst_sg->length < dst_target)
dst_target -= dst_sg->length;
else
dst_target = 0;
dst_sg = sg_next(dst_sg);
}
}
return cnt;
}
static int flexrm_spu_dma_map(struct device *dev, struct brcm_message *msg)
{
int rc;
rc = dma_map_sg(dev, msg->spu.src, sg_nents(msg->spu.src),
DMA_TO_DEVICE);
if (rc < 0)
return rc;
rc = dma_map_sg(dev, msg->spu.dst, sg_nents(msg->spu.dst),
DMA_FROM_DEVICE);
if (rc < 0) {
dma_unmap_sg(dev, msg->spu.src, sg_nents(msg->spu.src),
DMA_TO_DEVICE);
return rc;
}
return 0;
}
static void flexrm_spu_dma_unmap(struct device *dev, struct brcm_message *msg)
{
dma_unmap_sg(dev, msg->spu.dst, sg_nents(msg->spu.dst),
DMA_FROM_DEVICE);
dma_unmap_sg(dev, msg->spu.src, sg_nents(msg->spu.src),
DMA_TO_DEVICE);
}
static void *flexrm_spu_write_descs(struct brcm_message *msg, u32 nhcnt,
u32 reqid, void *desc_ptr, u32 toggle,
void *start_desc, void *end_desc)
{
u64 d;
u32 nhpos = 0;
void *orig_desc_ptr = desc_ptr;
unsigned int dst_target = 0;
struct scatterlist *src_sg = msg->spu.src, *dst_sg = msg->spu.dst;
while (src_sg || dst_sg) {
if (src_sg) {
if (sg_dma_len(src_sg) & 0xf)
d = flexrm_src_desc(sg_dma_address(src_sg),
sg_dma_len(src_sg));
else
d = flexrm_msrc_desc(sg_dma_address(src_sg),
sg_dma_len(src_sg)/16);
flexrm_enqueue_desc(nhpos, nhcnt, reqid,
d, &desc_ptr, &toggle,
start_desc, end_desc);
nhpos++;
dst_target = sg_dma_len(src_sg);
src_sg = sg_next(src_sg);
} else
dst_target = UINT_MAX;
while (dst_target && dst_sg) {
if (sg_dma_len(dst_sg) & 0xf)
d = flexrm_dst_desc(sg_dma_address(dst_sg),
sg_dma_len(dst_sg));
else
d = flexrm_mdst_desc(sg_dma_address(dst_sg),
sg_dma_len(dst_sg)/16);
flexrm_enqueue_desc(nhpos, nhcnt, reqid,
d, &desc_ptr, &toggle,
start_desc, end_desc);
nhpos++;
if (sg_dma_len(dst_sg) < dst_target)
dst_target -= sg_dma_len(dst_sg);
else
dst_target = 0;
dst_sg = sg_next(dst_sg);
}
}
/* Null descriptor with invalid toggle bit */
flexrm_write_desc(desc_ptr, flexrm_null_desc(!toggle));
/* Ensure that descriptors have been written to memory */
wmb();
/* Flip toggle bit in header */
flexrm_flip_header_toggle(orig_desc_ptr);
return desc_ptr;
}
static bool flexrm_sba_sanity_check(struct brcm_message *msg)
{
u32 i;
if (!msg->sba.cmds || !msg->sba.cmds_count)
return false;
for (i = 0; i < msg->sba.cmds_count; i++) {
if (((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) ||
(msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C)) &&
(msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT))
return false;
if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) &&
(msg->sba.cmds[i].data_len > SRCT_LENGTH_MASK))
return false;
if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C) &&
(msg->sba.cmds[i].data_len > SRCT_LENGTH_MASK))
return false;
if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_RESP) &&
(msg->sba.cmds[i].resp_len > DSTT_LENGTH_MASK))
return false;
if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT) &&
(msg->sba.cmds[i].data_len > DSTT_LENGTH_MASK))
return false;
}
return true;
}
static u32 flexrm_sba_estimate_nonheader_desc_count(struct brcm_message *msg)
{
u32 i, cnt;
cnt = 0;
for (i = 0; i < msg->sba.cmds_count; i++) {
cnt++;
if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) ||
(msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C))
cnt++;
if (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_RESP)
cnt++;
if (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT)
cnt++;
}
return cnt;
}
static void *flexrm_sba_write_descs(struct brcm_message *msg, u32 nhcnt,
u32 reqid, void *desc_ptr, u32 toggle,
void *start_desc, void *end_desc)
{
u64 d;
u32 i, nhpos = 0;
struct brcm_sba_command *c;
void *orig_desc_ptr = desc_ptr;
/* Convert SBA commands into descriptors */
for (i = 0; i < msg->sba.cmds_count; i++) {
c = &msg->sba.cmds[i];
if ((c->flags & BRCM_SBA_CMD_HAS_RESP) &&
(c->flags & BRCM_SBA_CMD_HAS_OUTPUT)) {
/* Destination response descriptor */
d = flexrm_dst_desc(c->resp, c->resp_len);
flexrm_enqueue_desc(nhpos, nhcnt, reqid,
d, &desc_ptr, &toggle,
start_desc, end_desc);
nhpos++;
} else if (c->flags & BRCM_SBA_CMD_HAS_RESP) {
/* Destination response with tlast descriptor */
d = flexrm_dstt_desc(c->resp, c->resp_len);
flexrm_enqueue_desc(nhpos, nhcnt, reqid,
d, &desc_ptr, &toggle,
start_desc, end_desc);
nhpos++;
}
if (c->flags & BRCM_SBA_CMD_HAS_OUTPUT) {
/* Destination with tlast descriptor */
d = flexrm_dstt_desc(c->data, c->data_len);
flexrm_enqueue_desc(nhpos, nhcnt, reqid,
d, &desc_ptr, &toggle,
start_desc, end_desc);
nhpos++;
}
if (c->flags & BRCM_SBA_CMD_TYPE_B) {
/* Command as immediate descriptor */
d = flexrm_imm_desc(c->cmd);
flexrm_enqueue_desc(nhpos, nhcnt, reqid,
d, &desc_ptr, &toggle,
start_desc, end_desc);
nhpos++;
} else {
/* Command as immediate descriptor with tlast */
d = flexrm_immt_desc(c->cmd);
flexrm_enqueue_desc(nhpos, nhcnt, reqid,
d, &desc_ptr, &toggle,
start_desc, end_desc);
nhpos++;
}
if ((c->flags & BRCM_SBA_CMD_TYPE_B) ||
(c->flags & BRCM_SBA_CMD_TYPE_C)) {
/* Source with tlast descriptor */
d = flexrm_srct_desc(c->data, c->data_len);
flexrm_enqueue_desc(nhpos, nhcnt, reqid,
d, &desc_ptr, &toggle,
start_desc, end_desc);
nhpos++;
}
}
/* Null descriptor with invalid toggle bit */
flexrm_write_desc(desc_ptr, flexrm_null_desc(!toggle));
/* Ensure that descriptors have been written to memory */
wmb();
/* Flip toggle bit in header */
flexrm_flip_header_toggle(orig_desc_ptr);
return desc_ptr;
}
static bool flexrm_sanity_check(struct brcm_message *msg)
{
if (!msg)
return false;
switch (msg->type) {
case BRCM_MESSAGE_SPU:
return flexrm_spu_sanity_check(msg);
case BRCM_MESSAGE_SBA:
return flexrm_sba_sanity_check(msg);
default:
return false;
};
}
static u32 flexrm_estimate_nonheader_desc_count(struct brcm_message *msg)
{
if (!msg)
return 0;
switch (msg->type) {
case BRCM_MESSAGE_SPU:
return flexrm_spu_estimate_nonheader_desc_count(msg);
case BRCM_MESSAGE_SBA:
return flexrm_sba_estimate_nonheader_desc_count(msg);
default:
return 0;
};
}
static int flexrm_dma_map(struct device *dev, struct brcm_message *msg)
{
if (!dev || !msg)
return -EINVAL;
switch (msg->type) {
case BRCM_MESSAGE_SPU:
return flexrm_spu_dma_map(dev, msg);
default:
break;
}
return 0;
}
static void flexrm_dma_unmap(struct device *dev, struct brcm_message *msg)
{
if (!dev || !msg)
return;
switch (msg->type) {
case BRCM_MESSAGE_SPU:
flexrm_spu_dma_unmap(dev, msg);
break;
default:
break;
}
}
static void *flexrm_write_descs(struct brcm_message *msg, u32 nhcnt,
u32 reqid, void *desc_ptr, u32 toggle,
void *start_desc, void *end_desc)
{
if (!msg || !desc_ptr || !start_desc || !end_desc)
return ERR_PTR(-ENOTSUPP);
if ((desc_ptr < start_desc) || (end_desc <= desc_ptr))
return ERR_PTR(-ERANGE);
switch (msg->type) {
case BRCM_MESSAGE_SPU:
return flexrm_spu_write_descs(msg, nhcnt, reqid,
desc_ptr, toggle,
start_desc, end_desc);
case BRCM_MESSAGE_SBA:
return flexrm_sba_write_descs(msg, nhcnt, reqid,
desc_ptr, toggle,
start_desc, end_desc);
default:
return ERR_PTR(-ENOTSUPP);
};
}
/* ====== FlexRM driver helper routines ===== */
static void flexrm_write_config_in_seqfile(struct flexrm_mbox *mbox,
struct seq_file *file)
{
int i;
const char *state;
struct flexrm_ring *ring;
seq_printf(file, "%-5s %-9s %-18s %-10s %-18s %-10s\n",
"Ring#", "State", "BD_Addr", "BD_Size",
"Cmpl_Addr", "Cmpl_Size");
for (i = 0; i < mbox->num_rings; i++) {
ring = &mbox->rings[i];
if (readl(ring->regs + RING_CONTROL) &
BIT(CONTROL_ACTIVE_SHIFT))
state = "active";
else
state = "inactive";
seq_printf(file,
"%-5d %-9s 0x%016llx 0x%08x 0x%016llx 0x%08x\n",
ring->num, state,
(unsigned long long)ring->bd_dma_base,
(u32)RING_BD_SIZE,
(unsigned long long)ring->cmpl_dma_base,
(u32)RING_CMPL_SIZE);
}
}
static void flexrm_write_stats_in_seqfile(struct flexrm_mbox *mbox,
struct seq_file *file)
{
int i;
u32 val, bd_read_offset;
struct flexrm_ring *ring;
seq_printf(file, "%-5s %-10s %-10s %-10s %-11s %-11s\n",
"Ring#", "BD_Read", "BD_Write",
"Cmpl_Read", "Submitted", "Completed");
for (i = 0; i < mbox->num_rings; i++) {
ring = &mbox->rings[i];
bd_read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR);
val = readl_relaxed(ring->regs + RING_BD_START_ADDR);
bd_read_offset *= RING_DESC_SIZE;
bd_read_offset += (u32)(BD_START_ADDR_DECODE(val) -
ring->bd_dma_base);
seq_printf(file, "%-5d 0x%08x 0x%08x 0x%08x %-11d %-11d\n",
ring->num,
(u32)bd_read_offset,
(u32)ring->bd_write_offset,
(u32)ring->cmpl_read_offset,
(u32)atomic_read(&ring->msg_send_count),
(u32)atomic_read(&ring->msg_cmpl_count));
}
}
static int flexrm_new_request(struct flexrm_ring *ring,
struct brcm_message *batch_msg,
struct brcm_message *msg)
{
void *next;
unsigned long flags;
u32 val, count, nhcnt;
u32 read_offset, write_offset;
bool exit_cleanup = false;
int ret = 0, reqid;
/* Do sanity check on message */
if (!flexrm_sanity_check(msg))
return -EIO;
msg->error = 0;
/* If no requests possible then save data pointer and goto done. */
spin_lock_irqsave(&ring->lock, flags);
reqid = bitmap_find_free_region(ring->requests_bmap,
RING_MAX_REQ_COUNT, 0);
spin_unlock_irqrestore(&ring->lock, flags);
if (reqid < 0)
return -ENOSPC;
ring->requests[reqid] = msg;
/* Do DMA mappings for the message */
ret = flexrm_dma_map(ring->mbox->dev, msg);
if (ret < 0) {
ring->requests[reqid] = NULL;
spin_lock_irqsave(&ring->lock, flags);
bitmap_release_region(ring->requests_bmap, reqid, 0);
spin_unlock_irqrestore(&ring->lock, flags);
return ret;
}
/* Determine current HW BD read offset */
read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR);
val = readl_relaxed(ring->regs + RING_BD_START_ADDR);
read_offset *= RING_DESC_SIZE;
read_offset += (u32)(BD_START_ADDR_DECODE(val) - ring->bd_dma_base);
/*
* Number required descriptors = number of non-header descriptors +
* number of header descriptors +
* 1x null descriptor
*/
nhcnt = flexrm_estimate_nonheader_desc_count(msg);
count = flexrm_estimate_header_desc_count(nhcnt) + nhcnt + 1;
/* Check for available descriptor space. */
write_offset = ring->bd_write_offset;
while (count) {
if (!flexrm_is_next_table_desc(ring->bd_base + write_offset))
count--;
write_offset += RING_DESC_SIZE;
if (write_offset == RING_BD_SIZE)
write_offset = 0x0;
if (write_offset == read_offset)
break;
}
if (count) {
ret = -ENOSPC;
exit_cleanup = true;
goto exit;
}
/* Write descriptors to ring */
next = flexrm_write_descs(msg, nhcnt, reqid,
ring->bd_base + ring->bd_write_offset,
RING_BD_TOGGLE_VALID(ring->bd_write_offset),
ring->bd_base, ring->bd_base + RING_BD_SIZE);
if (IS_ERR(next)) {
ret = PTR_ERR(next);
exit_cleanup = true;
goto exit;
}
/* Save ring BD write offset */
ring->bd_write_offset = (unsigned long)(next - ring->bd_base);
/* Increment number of messages sent */
atomic_inc_return(&ring->msg_send_count);
exit:
/* Update error status in message */
msg->error = ret;
/* Cleanup if we failed */
if (exit_cleanup) {
flexrm_dma_unmap(ring->mbox->dev, msg);
ring->requests[reqid] = NULL;
spin_lock_irqsave(&ring->lock, flags);
bitmap_release_region(ring->requests_bmap, reqid, 0);
spin_unlock_irqrestore(&ring->lock, flags);
}
return ret;
}
static int flexrm_process_completions(struct flexrm_ring *ring)
{
u64 desc;
int err, count = 0;
unsigned long flags;
struct brcm_message *msg = NULL;
u32 reqid, cmpl_read_offset, cmpl_write_offset;
struct mbox_chan *chan = &ring->mbox->controller.chans[ring->num];
spin_lock_irqsave(&ring->lock, flags);
/*
* Get current completion read and write offset
*
* Note: We should read completion write pointer atleast once
* after we get a MSI interrupt because HW maintains internal
* MSI status which will allow next MSI interrupt only after
* completion write pointer is read.
*/
cmpl_write_offset = readl_relaxed(ring->regs + RING_CMPL_WRITE_PTR);
cmpl_write_offset *= RING_DESC_SIZE;
cmpl_read_offset = ring->cmpl_read_offset;
ring->cmpl_read_offset = cmpl_write_offset;
spin_unlock_irqrestore(&ring->lock, flags);
/* For each completed request notify mailbox clients */
reqid = 0;
while (cmpl_read_offset != cmpl_write_offset) {
/* Dequeue next completion descriptor */
desc = *((u64 *)(ring->cmpl_base + cmpl_read_offset));
/* Next read offset */
cmpl_read_offset += RING_DESC_SIZE;
if (cmpl_read_offset == RING_CMPL_SIZE)
cmpl_read_offset = 0;
/* Decode error from completion descriptor */
err = flexrm_cmpl_desc_to_error(desc);
if (err < 0) {
dev_warn(ring->mbox->dev,
"ring%d got completion desc=0x%lx with error %d\n",
ring->num, (unsigned long)desc, err);
}
/* Determine request id from completion descriptor */
reqid = flexrm_cmpl_desc_to_reqid(desc);
/* Determine message pointer based on reqid */
msg = ring->requests[reqid];
if (!msg) {
dev_warn(ring->mbox->dev,
"ring%d null msg pointer for completion desc=0x%lx\n",
ring->num, (unsigned long)desc);
continue;
}
/* Release reqid for recycling */
ring->requests[reqid] = NULL;
spin_lock_irqsave(&ring->lock, flags);
bitmap_release_region(ring->requests_bmap, reqid, 0);
spin_unlock_irqrestore(&ring->lock, flags);
/* Unmap DMA mappings */
flexrm_dma_unmap(ring->mbox->dev, msg);
/* Give-back message to mailbox client */
msg->error = err;
mbox_chan_received_data(chan, msg);
/* Increment number of completions processed */
atomic_inc_return(&ring->msg_cmpl_count);
count++;
}
return count;
}
/* ====== FlexRM Debugfs callbacks ====== */
static int flexrm_debugfs_conf_show(struct seq_file *file, void *offset)
{
struct platform_device *pdev = to_platform_device(file->private);
struct flexrm_mbox *mbox = platform_get_drvdata(pdev);
/* Write config in file */
flexrm_write_config_in_seqfile(mbox, file);
return 0;
}
static int flexrm_debugfs_stats_show(struct seq_file *file, void *offset)
{
struct platform_device *pdev = to_platform_device(file->private);
struct flexrm_mbox *mbox = platform_get_drvdata(pdev);
/* Write stats in file */
flexrm_write_stats_in_seqfile(mbox, file);
return 0;
}
/* ====== FlexRM interrupt handler ===== */
static irqreturn_t flexrm_irq_event(int irq, void *dev_id)
{
/* We only have MSI for completions so just wakeup IRQ thread */
/* Ring related errors will be informed via completion descriptors */
return IRQ_WAKE_THREAD;
}
static irqreturn_t flexrm_irq_thread(int irq, void *dev_id)
{
flexrm_process_completions(dev_id);
return IRQ_HANDLED;
}
/* ====== FlexRM mailbox callbacks ===== */
static int flexrm_send_data(struct mbox_chan *chan, void *data)
{
int i, rc;
struct flexrm_ring *ring = chan->con_priv;
struct brcm_message *msg = data;
if (msg->type == BRCM_MESSAGE_BATCH) {
for (i = msg->batch.msgs_queued;
i < msg->batch.msgs_count; i++) {
rc = flexrm_new_request(ring, msg,
&msg->batch.msgs[i]);
if (rc) {
msg->error = rc;
return rc;
}
msg->batch.msgs_queued++;
}
return 0;
}
return flexrm_new_request(ring, NULL, data);
}
static bool flexrm_peek_data(struct mbox_chan *chan)
{
int cnt = flexrm_process_completions(chan->con_priv);
return (cnt > 0) ? true : false;
}
static int flexrm_startup(struct mbox_chan *chan)
{
u64 d;
u32 val, off;
int ret = 0;
dma_addr_t next_addr;
struct flexrm_ring *ring = chan->con_priv;
/* Allocate BD memory */
ring->bd_base = dma_pool_alloc(ring->mbox->bd_pool,
GFP_KERNEL, &ring->bd_dma_base);
if (!ring->bd_base) {
dev_err(ring->mbox->dev,
"can't allocate BD memory for ring%d\n",
ring->num);
ret = -ENOMEM;
goto fail;
}
/* Configure next table pointer entries in BD memory */
for (off = 0; off < RING_BD_SIZE; off += RING_DESC_SIZE) {
next_addr = off + RING_DESC_SIZE;
if (next_addr == RING_BD_SIZE)
next_addr = 0;
next_addr += ring->bd_dma_base;
if (RING_BD_ALIGN_CHECK(next_addr))
d = flexrm_next_table_desc(RING_BD_TOGGLE_VALID(off),
next_addr);
else
d = flexrm_null_desc(RING_BD_TOGGLE_INVALID(off));
flexrm_write_desc(ring->bd_base + off, d);
}
/* Allocate completion memory */
ring->cmpl_base = dma_pool_zalloc(ring->mbox->cmpl_pool,
GFP_KERNEL, &ring->cmpl_dma_base);
if (!ring->cmpl_base) {
dev_err(ring->mbox->dev,
"can't allocate completion memory for ring%d\n",
ring->num);
ret = -ENOMEM;
goto fail_free_bd_memory;
}
/* Request IRQ */
if (ring->irq == UINT_MAX) {
dev_err(ring->mbox->dev,
"ring%d IRQ not available\n", ring->num);
ret = -ENODEV;
goto fail_free_cmpl_memory;
}
ret = request_threaded_irq(ring->irq,
flexrm_irq_event,
flexrm_irq_thread,
0, dev_name(ring->mbox->dev), ring);
if (ret) {
dev_err(ring->mbox->dev,
"failed to request ring%d IRQ\n", ring->num);
goto fail_free_cmpl_memory;
}
ring->irq_requested = true;
/* Set IRQ affinity hint */
ring->irq_aff_hint = CPU_MASK_NONE;
val = ring->mbox->num_rings;
val = (num_online_cpus() < val) ? val / num_online_cpus() : 1;
cpumask_set_cpu((ring->num / val) % num_online_cpus(),
&ring->irq_aff_hint);
ret = irq_set_affinity_hint(ring->irq, &ring->irq_aff_hint);
if (ret) {
dev_err(ring->mbox->dev,
"failed to set IRQ affinity hint for ring%d\n",
ring->num);
goto fail_free_irq;
}
/* Disable/inactivate ring */
writel_relaxed(0x0, ring->regs + RING_CONTROL);
/* Program BD start address */
val = BD_START_ADDR_VALUE(ring->bd_dma_base);
writel_relaxed(val, ring->regs + RING_BD_START_ADDR);
/* BD write pointer will be same as HW write pointer */
ring->bd_write_offset =
readl_relaxed(ring->regs + RING_BD_WRITE_PTR);
ring->bd_write_offset *= RING_DESC_SIZE;
/* Program completion start address */
val = CMPL_START_ADDR_VALUE(ring->cmpl_dma_base);
writel_relaxed(val, ring->regs + RING_CMPL_START_ADDR);
/* Completion read pointer will be same as HW write pointer */
ring->cmpl_read_offset =
readl_relaxed(ring->regs + RING_CMPL_WRITE_PTR);
ring->cmpl_read_offset *= RING_DESC_SIZE;
/* Read ring Tx, Rx, and Outstanding counts to clear */
readl_relaxed(ring->regs + RING_NUM_REQ_RECV_LS);
readl_relaxed(ring->regs + RING_NUM_REQ_RECV_MS);
readl_relaxed(ring->regs + RING_NUM_REQ_TRANS_LS);
readl_relaxed(ring->regs + RING_NUM_REQ_TRANS_MS);
readl_relaxed(ring->regs + RING_NUM_REQ_OUTSTAND);
/* Configure RING_MSI_CONTROL */
val = 0;
val |= (ring->msi_timer_val << MSI_TIMER_VAL_SHIFT);
val |= BIT(MSI_ENABLE_SHIFT);
val |= (ring->msi_count_threshold & MSI_COUNT_MASK) << MSI_COUNT_SHIFT;
writel_relaxed(val, ring->regs + RING_MSI_CONTROL);
/* Enable/activate ring */
val = BIT(CONTROL_ACTIVE_SHIFT);
writel_relaxed(val, ring->regs + RING_CONTROL);
/* Reset stats to zero */
atomic_set(&ring->msg_send_count, 0);
atomic_set(&ring->msg_cmpl_count, 0);
return 0;
fail_free_irq:
free_irq(ring->irq, ring);
ring->irq_requested = false;
fail_free_cmpl_memory:
dma_pool_free(ring->mbox->cmpl_pool,
ring->cmpl_base, ring->cmpl_dma_base);
ring->cmpl_base = NULL;
fail_free_bd_memory:
dma_pool_free(ring->mbox->bd_pool,
ring->bd_base, ring->bd_dma_base);
ring->bd_base = NULL;
fail:
return ret;
}
static void flexrm_shutdown(struct mbox_chan *chan)
{
u32 reqid;
unsigned int timeout;
struct brcm_message *msg;
struct flexrm_ring *ring = chan->con_priv;
/* Disable/inactivate ring */
writel_relaxed(0x0, ring->regs + RING_CONTROL);
/* Set ring flush state */
timeout = 1000; /* timeout of 1s */
writel_relaxed(BIT(CONTROL_FLUSH_SHIFT),
ring->regs + RING_CONTROL);
do {
if (readl_relaxed(ring->regs + RING_FLUSH_DONE) &
FLUSH_DONE_MASK)
break;
mdelay(1);
} while (--timeout);
if (!timeout)
dev_err(ring->mbox->dev,
"setting ring%d flush state timedout\n", ring->num);
/* Clear ring flush state */
timeout = 1000; /* timeout of 1s */
writel_relaxed(0x0, ring->regs + RING_CONTROL);
do {
if (!(readl_relaxed(ring->regs + RING_FLUSH_DONE) &
FLUSH_DONE_MASK))
break;
mdelay(1);
} while (--timeout);
if (!timeout)
dev_err(ring->mbox->dev,
"clearing ring%d flush state timedout\n", ring->num);
/* Abort all in-flight requests */
for (reqid = 0; reqid < RING_MAX_REQ_COUNT; reqid++) {
msg = ring->requests[reqid];
if (!msg)
continue;
/* Release reqid for recycling */
ring->requests[reqid] = NULL;
/* Unmap DMA mappings */
flexrm_dma_unmap(ring->mbox->dev, msg);
/* Give-back message to mailbox client */
msg->error = -EIO;
mbox_chan_received_data(chan, msg);
}
/* Clear requests bitmap */
bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT);
/* Release IRQ */
if (ring->irq_requested) {
irq_set_affinity_hint(ring->irq, NULL);
free_irq(ring->irq, ring);
ring->irq_requested = false;
}
/* Free-up completion descriptor ring */
if (ring->cmpl_base) {
dma_pool_free(ring->mbox->cmpl_pool,
ring->cmpl_base, ring->cmpl_dma_base);
ring->cmpl_base = NULL;
}
/* Free-up BD descriptor ring */
if (ring->bd_base) {
dma_pool_free(ring->mbox->bd_pool,
ring->bd_base, ring->bd_dma_base);
ring->bd_base = NULL;
}
}
static const struct mbox_chan_ops flexrm_mbox_chan_ops = {
.send_data = flexrm_send_data,
.startup = flexrm_startup,
.shutdown = flexrm_shutdown,
.peek_data = flexrm_peek_data,
};
static struct mbox_chan *flexrm_mbox_of_xlate(struct mbox_controller *cntlr,
const struct of_phandle_args *pa)
{
struct mbox_chan *chan;
struct flexrm_ring *ring;
if (pa->args_count < 3)
return ERR_PTR(-EINVAL);
if (pa->args[0] >= cntlr->num_chans)
return ERR_PTR(-ENOENT);
if (pa->args[1] > MSI_COUNT_MASK)
return ERR_PTR(-EINVAL);
if (pa->args[2] > MSI_TIMER_VAL_MASK)
return ERR_PTR(-EINVAL);
chan = &cntlr->chans[pa->args[0]];
ring = chan->con_priv;
ring->msi_count_threshold = pa->args[1];
ring->msi_timer_val = pa->args[2];
return chan;
}
/* ====== FlexRM platform driver ===== */
static void flexrm_mbox_msi_write(struct msi_desc *desc, struct msi_msg *msg)
{
struct device *dev = msi_desc_to_dev(desc);
struct flexrm_mbox *mbox = dev_get_drvdata(dev);
struct flexrm_ring *ring = &mbox->rings[desc->platform.msi_index];
/* Configure per-Ring MSI registers */
writel_relaxed(msg->address_lo, ring->regs + RING_MSI_ADDR_LS);
writel_relaxed(msg->address_hi, ring->regs + RING_MSI_ADDR_MS);
writel_relaxed(msg->data, ring->regs + RING_MSI_DATA_VALUE);
}
static int flexrm_mbox_probe(struct platform_device *pdev)
{
int index, ret = 0;
void __iomem *regs;
void __iomem *regs_end;
struct msi_desc *desc;
struct resource *iomem;
struct flexrm_ring *ring;
struct flexrm_mbox *mbox;
struct device *dev = &pdev->dev;
/* Allocate driver mailbox struct */
mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL);
if (!mbox) {
ret = -ENOMEM;
goto fail;
}
mbox->dev = dev;
platform_set_drvdata(pdev, mbox);
/* Get resource for registers */
iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
if (!iomem || (resource_size(iomem) < RING_REGS_SIZE)) {
ret = -ENODEV;
goto fail;
}
/* Map registers of all rings */
mbox->regs = devm_ioremap_resource(&pdev->dev, iomem);
if (IS_ERR(mbox->regs)) {
ret = PTR_ERR(mbox->regs);
dev_err(&pdev->dev, "Failed to remap mailbox regs: %d\n", ret);
goto fail;
}
regs_end = mbox->regs + resource_size(iomem);
/* Scan and count available rings */
mbox->num_rings = 0;
for (regs = mbox->regs; regs < regs_end; regs += RING_REGS_SIZE) {
if (readl_relaxed(regs + RING_VER) == RING_VER_MAGIC)
mbox->num_rings++;
}
if (!mbox->num_rings) {
ret = -ENODEV;
goto fail;
}
/* Allocate driver ring structs */
ring = devm_kcalloc(dev, mbox->num_rings, sizeof(*ring), GFP_KERNEL);
if (!ring) {
ret = -ENOMEM;
goto fail;
}
mbox->rings = ring;
/* Initialize members of driver ring structs */
regs = mbox->regs;
for (index = 0; index < mbox->num_rings; index++) {
ring = &mbox->rings[index];
ring->num = index;
ring->mbox = mbox;
while ((regs < regs_end) &&
(readl_relaxed(regs + RING_VER) != RING_VER_MAGIC))
regs += RING_REGS_SIZE;
if (regs_end <= regs) {
ret = -ENODEV;
goto fail;
}
ring->regs = regs;
regs += RING_REGS_SIZE;
ring->irq = UINT_MAX;
ring->irq_requested = false;
ring->msi_timer_val = MSI_TIMER_VAL_MASK;
ring->msi_count_threshold = 0x1;
memset(ring->requests, 0, sizeof(ring->requests));
ring->bd_base = NULL;
ring->bd_dma_base = 0;
ring->cmpl_base = NULL;
ring->cmpl_dma_base = 0;
atomic_set(&ring->msg_send_count, 0);
atomic_set(&ring->msg_cmpl_count, 0);
spin_lock_init(&ring->lock);
bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT);
ring->cmpl_read_offset = 0;
}
/* FlexRM is capable of 40-bit physical addresses only */
ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40));
if (ret) {
ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
if (ret)
goto fail;
}
/* Create DMA pool for ring BD memory */
mbox->bd_pool = dma_pool_create("bd", dev, RING_BD_SIZE,
1 << RING_BD_ALIGN_ORDER, 0);
if (!mbox->bd_pool) {
ret = -ENOMEM;
goto fail;
}
/* Create DMA pool for ring completion memory */
mbox->cmpl_pool = dma_pool_create("cmpl", dev, RING_CMPL_SIZE,
1 << RING_CMPL_ALIGN_ORDER, 0);
if (!mbox->cmpl_pool) {
ret = -ENOMEM;
goto fail_destroy_bd_pool;
}
/* Allocate platform MSIs for each ring */
ret = platform_msi_domain_alloc_irqs(dev, mbox->num_rings,
flexrm_mbox_msi_write);
if (ret)
goto fail_destroy_cmpl_pool;
/* Save alloced IRQ numbers for each ring */
for_each_msi_entry(desc, dev) {
ring = &mbox->rings[desc->platform.msi_index];
ring->irq = desc->irq;
}
/* Check availability of debugfs */
if (!debugfs_initialized())
goto skip_debugfs;
/* Create debugfs root entry */
mbox->root = debugfs_create_dir(dev_name(mbox->dev), NULL);
if (IS_ERR_OR_NULL(mbox->root)) {
ret = PTR_ERR_OR_ZERO(mbox->root);
goto fail_free_msis;
}
/* Create debugfs config entry */
mbox->config = debugfs_create_devm_seqfile(mbox->dev,
"config", mbox->root,
flexrm_debugfs_conf_show);
if (IS_ERR_OR_NULL(mbox->config)) {
ret = PTR_ERR_OR_ZERO(mbox->config);
goto fail_free_debugfs_root;
}
/* Create debugfs stats entry */
mbox->stats = debugfs_create_devm_seqfile(mbox->dev,
"stats", mbox->root,
flexrm_debugfs_stats_show);
if (IS_ERR_OR_NULL(mbox->stats)) {
ret = PTR_ERR_OR_ZERO(mbox->stats);
goto fail_free_debugfs_root;
}
skip_debugfs:
/* Initialize mailbox controller */
mbox->controller.txdone_irq = false;
mbox->controller.txdone_poll = false;
mbox->controller.ops = &flexrm_mbox_chan_ops;
mbox->controller.dev = dev;
mbox->controller.num_chans = mbox->num_rings;
mbox->controller.of_xlate = flexrm_mbox_of_xlate;
mbox->controller.chans = devm_kcalloc(dev, mbox->num_rings,
sizeof(*mbox->controller.chans), GFP_KERNEL);
if (!mbox->controller.chans) {
ret = -ENOMEM;
goto fail_free_debugfs_root;
}
for (index = 0; index < mbox->num_rings; index++)
mbox->controller.chans[index].con_priv = &mbox->rings[index];
/* Register mailbox controller */
ret = devm_mbox_controller_register(dev, &mbox->controller);
if (ret)
goto fail_free_debugfs_root;
dev_info(dev, "registered flexrm mailbox with %d channels\n",
mbox->controller.num_chans);
return 0;
fail_free_debugfs_root:
debugfs_remove_recursive(mbox->root);
fail_free_msis:
platform_msi_domain_free_irqs(dev);
fail_destroy_cmpl_pool:
dma_pool_destroy(mbox->cmpl_pool);
fail_destroy_bd_pool:
dma_pool_destroy(mbox->bd_pool);
fail:
return ret;
}
static int flexrm_mbox_remove(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
struct flexrm_mbox *mbox = platform_get_drvdata(pdev);
debugfs_remove_recursive(mbox->root);
platform_msi_domain_free_irqs(dev);
dma_pool_destroy(mbox->cmpl_pool);
dma_pool_destroy(mbox->bd_pool);
return 0;
}
static const struct of_device_id flexrm_mbox_of_match[] = {
{ .compatible = "brcm,iproc-flexrm-mbox", },
{},
};
MODULE_DEVICE_TABLE(of, flexrm_mbox_of_match);
static struct platform_driver flexrm_mbox_driver = {
.driver = {
.name = "brcm-flexrm-mbox",
.of_match_table = flexrm_mbox_of_match,
},
.probe = flexrm_mbox_probe,
.remove = flexrm_mbox_remove,
};
module_platform_driver(flexrm_mbox_driver);
MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>");
MODULE_DESCRIPTION("Broadcom FlexRM mailbox driver");
MODULE_LICENSE("GPL v2");
![swh spinner](/static/img/swh-spinner.gif)
Computing file changes ...