OpenSolaris_b135/lib/udapl/udapl_tavor/tavor/dapl_tavor_wr.c
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#include "dapl.h"
#include "dapl_tavor_wr.h"
#include "dapl_hash.h"
#include "dapl_tavor_ibtf_impl.h"
static dapls_tavor_wrid_entry_t *dapli_tavor_wrid_find_match(
dapls_tavor_workq_hdr_t *, tavor_hw_cqe_t *);
static dapls_tavor_wrid_list_hdr_t *dapli_tavor_wrid_get_list(uint32_t, int);
static void dapli_tavor_wrid_reaplist_add(ib_cq_handle_t,
dapls_tavor_workq_hdr_t *);
static dapls_tavor_workq_hdr_t *dapli_tavor_wrid_wqhdr_find(ib_cq_handle_t,
uint_t, uint_t);
static uint32_t dapli_tavor_wrid_get_wqeaddrsz(dapls_tavor_workq_hdr_t *);
static dapls_tavor_workq_hdr_t *dapli_tavor_wrid_list_reap(
dapls_tavor_wrid_list_hdr_t *);
static dapls_tavor_workq_hdr_t *dapli_tavor_wrid_wqhdr_create(ib_cq_handle_t,
uint_t, uint_t, uint_t);
static void dapli_tavor_wrid_wqhdr_add(dapls_tavor_workq_hdr_t *,
dapls_tavor_wrid_list_hdr_t *);
static void dapli_tavor_wrid_wqhdr_remove(dapls_tavor_workq_hdr_t *,
dapls_tavor_wrid_list_hdr_t *);
static void dapli_tavor_wrid_wqhdr_lock_both(ib_qp_handle_t);
static void dapli_tavor_wrid_wqhdr_unlock_both(ib_qp_handle_t);
static DAT_RETURN dapli_tavor_cq_wqhdr_add(ib_cq_handle_t,
dapls_tavor_workq_hdr_t *);
static void dapli_tavor_cq_wqhdr_remove(ib_cq_handle_t,
dapls_tavor_workq_hdr_t *);
/*
* dapls_tavor_wrid_get_entry()
*/
uint64_t
dapls_tavor_wrid_get_entry(ib_cq_handle_t cq, tavor_hw_cqe_t *cqe,
uint_t send_or_recv, uint_t error, dapls_tavor_wrid_entry_t *wre)
{
dapls_tavor_workq_hdr_t *wq;
dapls_tavor_wrid_entry_t *wre_tmp;
uint64_t wrid;
uint_t qpnum;
/* Lock the list of work queues associated with this CQ */
dapl_os_lock(&cq->cq_wrid_wqhdr_lock);
/* Find the work queue for this QP number (send or receive side) */
qpnum = TAVOR_CQE_QPNUM_GET(cqe);
wq = dapli_tavor_wrid_wqhdr_find(cq, qpnum, send_or_recv);
dapl_os_assert(wq != NULL);
/*
* Regardless of whether the completion is the result of a "success"
* or a "failure", we lock the list of "containers" and attempt to
* search for the the first matching completion (i.e. the first WR
* with a matching WQE addr and size). Once we find it, we pull out
* the "wrid" field and return it (see below). Note: One possible
* future enhancement would be to enable this routine to skip over
* any "unsignaled" completions to go directly to the next "signaled"
* entry on success. XXX
*/
dapl_os_lock(&wq->wq_wrid_lock->wrl_lock);
wre_tmp = dapli_tavor_wrid_find_match(wq, cqe);
/*
* If this is a "successful" completion, then we assert that this
* completion must be a "signaled" completion.
*/
dapl_os_assert(error || (wre_tmp->wr_signaled_dbd &
TAVOR_WRID_ENTRY_SIGNALED));
/*
* If the completion is a "failed" completion, then we save away the
* contents of the entry (into the "wre" field passed in) for use
* in later CQE processing. Note: We use the
* dapli_tavor_wrid_get_wqeaddrsz() function to grab "wqeaddrsz" from
* the next entry in the container.
* This is required for error processing (where updating these fields
* properly is necessary to correct handling of the "error" CQE)
*/
if (error && (wre != NULL)) {
*wre = *wre_tmp;
wre->wr_wqeaddrsz = dapli_tavor_wrid_get_wqeaddrsz(wq);
}
/* Pull out the WRID and return it */
wrid = wre_tmp->wr_wrid;
dapl_os_unlock(&wq->wq_wrid_lock->wrl_lock);
dapl_os_unlock(&cq->cq_wrid_wqhdr_lock);
return (wrid);
}
/*
* dapli_tavor_wrid_find_match()
*/
static dapls_tavor_wrid_entry_t *
dapli_tavor_wrid_find_match(dapls_tavor_workq_hdr_t *wq, tavor_hw_cqe_t *cqe)
{
dapls_tavor_wrid_entry_t *curr = NULL;
dapls_tavor_wrid_list_hdr_t *container;
uint32_t wqeaddr_size;
uint32_t head, tail, size;
int found = 0, last_container;
/* dapl_os_assert(MUTEX_HELD(&wq->wq_wrid_lock)); */
/* Pull the "wqeaddrsz" information from the CQE */
wqeaddr_size = TAVOR_CQE_WQEADDRSZ_GET(cqe);
/*
* Walk the "containers" list(s), find first WR with a matching WQE
* addr. If the current "container" is not the last one on the list,
* i.e. not the current one to which we are posting new WRID entries,
* then we do not attempt to update the "q_head", "q_tail", and
* "q_full" indicators on the main work queue header. We do, however,
* update the "head" and "full" indicators on the individual containers
* as we go. This is imperative because we need to be able to
* determine when the current container has been emptied (so that we
* can move on to the next container).
*/
container = wq->wq_wrid_poll;
while (container != NULL) {
/* Is this the last/only "container" on the list */
last_container = (container != wq->wq_wrid_post) ? 0 : 1;
/*
* First check if we are on an SRQ. If so, we grab the entry
* and break out. Since SRQ wridlist's are never added to
* reaplist, they can only be the last container.
*/
if (container->wl_srq_en) {
dapl_os_assert(last_container == 1);
curr = dapli_tavor_wrid_find_match_srq(container, cqe);
break;
}
/*
* Grab the current "head", "tail" and "size" fields before
* walking the list in the current container. Note: the "size"
* field here must always be a power-of-2. The "full"
* parameter is checked (and updated) here to distinguish the
* "queue full" condition from "queue empty".
*/
head = container->wl_head;
tail = container->wl_tail;
size = container->wl_size;
while ((head != tail) || (container->wl_full)) {
container->wl_full = 0;
curr = &container->wl_wre[head];
head = ((head + 1) & (size - 1));
/*
* If the current entry's "wqeaddrsz" matches the one
* we're searching for, then this must correspond to
* the work request that caused the completion. Set
* the "found" flag and bail out.
*/
if (curr->wr_wqeaddrsz == wqeaddr_size) {
found = 1;
break;
}
}
/*
* If the current container is empty (having reached here the
* "head == tail" condition can only mean that the container
* is empty), then NULL out the "wrid_old_tail" field (see
* tavor_post_send() and tavor_post_recv() for more details)
* and (potentially) remove the current container from future
* searches.
*/
if (head == tail) {
container->wl_wre_old_tail = NULL;
/*
* If this wasn't the last "container" on the chain,
* i.e. the one to which new WRID entries will be
* added, then remove it from the list.
* Note: we don't "lose" the memory pointed to by this
* because we should have already put this container
* on the "reapable" list (from where it will later be
* pulled).
*/
if (!last_container) {
wq->wq_wrid_poll = container->wl_next;
}
}
/* Update the head index for the container */
container->wl_head = head;
/*
* If the entry was found in this container, then continue to
* bail out. Else reset the "curr" pointer and move on to the
* next container (if there is one). Note: the only real
* reason for setting "curr = NULL" here is so that the ASSERT
* below can catch the case where no matching entry was found
* on any of the lists.
*/
if (found) {
break;
} else {
curr = NULL;
container = container->wl_next;
}
}
/*
* Update work queue header's "head" and "full" conditions to match
* the last entry on the container list. (Note: Only if we're pulling
* entries from the last work queue portion of the list, i.e. not from
* the previous portions that may be the "reapable" list.)
*/
if (last_container) {
wq->wq_head = wq->wq_wrid_post->wl_head;
wq->wq_full = wq->wq_wrid_post->wl_full;
}
/* Ensure that we've actually found what we were searching for */
dapl_os_assert(curr != NULL);
return (curr);
}
/*
* tavor_wrid_find_match_srq()
* Context: Can be called from interrupt or base context.
*/
dapls_tavor_wrid_entry_t *
dapli_tavor_wrid_find_match_srq(dapls_tavor_wrid_list_hdr_t *wl,
tavor_hw_cqe_t *cqe)
{
dapls_tavor_wrid_entry_t *wre;
uint32_t wqe_index;
uint32_t wqe_addr;
uint32_t qsize_msk;
uint32_t tail, next_tail;
/* Grab the WQE addr out of the CQE */
wqe_addr = TAVOR_CQE_WQEADDRSZ_GET(cqe) & 0xFFFFFFC0;
/*
* Given the 'wqe_addr' just calculated and the srq buf address, we
* find the 'wqe_index'. The 'wre' returned below contains the WRID
* that we are looking for. This indexes into the wre_list for this
* specific WQE.
*/
wqe_index = TAVOR_SRQ_WQ_INDEX(wl->wl_srq_desc_addr, wqe_addr,
wl->wl_srq_wqesz);
/* ASSERT on impossible wqe_index values */
dapl_os_assert(wqe_index < wl->wl_size);
/* Put this WQE back on the free list */
qsize_msk = wl->wl_size - 1;
tail = wl->wl_freel_tail;
next_tail = (tail + 1) & qsize_msk;
wl->wl_freel_entries++;
dapl_os_assert(wl->wl_freel_entries <= wl->wl_size);
/* Get the descriptor (IO Address) of the WQE to be built */
wl->wl_free_list[tail] = wqe_addr;
wl->wl_freel_tail = next_tail;
/* Using the index, return the Work Request ID Entry (wre) */
wre = &wl->wl_wre[wqe_index];
return (wre);
}
/*
* dapls_tavor_wrid_cq_reap()
*/
void
dapls_tavor_wrid_cq_reap(ib_cq_handle_t cq)
{
dapls_tavor_workq_hdr_t *consume_wqhdr;
dapls_tavor_wrid_list_hdr_t *container, *to_free;
/* dapl_os_assert(MUTEX_HELD(&cq->cq_lock)); */
/* Lock the list of work queues associated with this CQ */
dapl_os_lock(&cq->cq_wrid_wqhdr_lock);
/* Walk the "reapable" list and free up containers */
container = cq->cq_wrid_reap_head;
while (container != NULL) {
to_free = container;
container = container->wl_reap_next;
/*
* If reaping the WRID list containers pulls the last
* container from the given work queue header, then we free
* the work queue header as well.
*/
consume_wqhdr = dapli_tavor_wrid_list_reap(to_free);
if (consume_wqhdr != NULL) {
dapli_tavor_cq_wqhdr_remove(cq, consume_wqhdr);
}
}
/* Once finished reaping, we reset the CQ's reap list */
cq->cq_wrid_reap_head = cq->cq_wrid_reap_tail = NULL;
dapl_os_unlock(&cq->cq_wrid_wqhdr_lock);
}
/*
* dapls_tavor_wrid_cq_force_reap()
*/
void
dapls_tavor_wrid_cq_force_reap(ib_cq_handle_t cq)
{
DAPL_HASH_DATA curr;
DAT_RETURN retval;
dapls_tavor_workq_hdr_t *to_free_wqhdr;
dapls_tavor_wrid_list_hdr_t *container, *to_free;
/* dapl_os_assert(MUTEX_HELD(&cq->cq_lock)); */
/*
* The first step is to walk the "reapable" list and free up those
* containers. This is necessary because the containers on the
* reapable list are not otherwise connected to the work queue headers
* anymore.
*/
dapls_tavor_wrid_cq_reap(cq);
/* Now lock the list of work queues associated with this CQ */
dapl_os_lock(&cq->cq_wrid_wqhdr_lock);
/*
* Walk the list of work queue headers and free up all the WRID list
* containers chained to it. Note: We don't need to grab the locks
* for each of the individual WRID lists here because the only way
* things can be added or removed from the list at this point would be
* through post a work request to a QP. But if we've come this far,
* then we can be assured that there are no longer any QP associated
* with the CQ that we are trying to free.
*/
retval = dapls_hash_iterate(cq->cq_wrid_wqhdr_list,
DAPL_HASH_ITERATE_INIT, &curr);
dapl_os_assert(retval == DAT_SUCCESS);
while (curr != NULL) {
to_free_wqhdr = (dapls_tavor_workq_hdr_t *)curr;
container = ((dapls_tavor_workq_hdr_t *)curr)->wq_wrid_poll;
retval = dapls_hash_iterate(cq->cq_wrid_wqhdr_list,
DAPL_HASH_ITERATE_NEXT, &curr);
dapl_os_assert(retval == DAT_SUCCESS);
while (container != NULL) {
to_free = container;
container = container->wl_next;
/*
* If reaping the WRID list containers pulls the last
* container from the given work queue header, then
* we free the work queue header as well. Note: we
* ignore the return value because we know that the
* work queue header should always be freed once the
* list of containers has come to an end.
*/
(void) dapli_tavor_wrid_list_reap(to_free);
if (container == NULL) {
dapli_tavor_cq_wqhdr_remove(cq, to_free_wqhdr);
}
}
}
dapl_os_lock(&cq->cq_wrid_wqhdr_lock);
}
/*
* dapli_tavor_wrid_get_list()
*/
static dapls_tavor_wrid_list_hdr_t *
dapli_tavor_wrid_get_list(uint32_t qsize, int wrid_for_srq)
{
dapls_tavor_wrid_list_hdr_t *wridlist;
dapls_tavor_wrid_entry_t *wl_wre;
uint32_t *wl_freel;
uint32_t size;
uint32_t wl_wre_size;
uint32_t wl_freel_size;
wridlist = NULL;
wl_wre = NULL;
wl_freel = NULL;
size = wl_wre_size = wl_freel_size = 0;
/*
* The WRID list "container" consists of the dapls_tavor_wrid_list_hdr_t
* which holds the pointers necessary for maintaining the "reapable"
* list, chaining together multiple "containers" old and new, and
* tracking the head, tail, size, etc. for each container. The
* "container" also holds all the tavor_wrid_entry_t's, one for
* each entry on the corresponding work queue.
*/
/*
* For wridlist associated with SRQs the wridlock needs to be
* allocated and initialized here.
*/
size = sizeof (dapls_tavor_wrid_list_hdr_t);
if (wrid_for_srq) {
size = size + sizeof (dapls_tavor_wrid_lock_t);
}
wridlist = dapl_os_alloc(size);
if (wridlist == NULL) {
goto bail;
}
if (wrid_for_srq) {
wridlist->wl_lock = (dapls_tavor_wrid_lock_t *)(
(uintptr_t)wridlist + sizeof (dapls_tavor_wrid_list_hdr_t));
dapl_os_lock_init(&wridlist->wl_lock->wrl_lock);
wridlist->wl_lock->wrl_on_srq = wrid_for_srq;
} else {
wridlist->wl_lock = NULL;
}
wl_wre_size = qsize * sizeof (dapls_tavor_wrid_entry_t);
wl_wre = dapl_os_alloc(wl_wre_size);
if (wl_wre == NULL) {
goto bail;
}
if (wrid_for_srq) { /* memory for the SRQ free list */
wl_freel_size = qsize * sizeof (uint32_t);
wl_freel = dapl_os_alloc(wl_freel_size);
if (wl_freel == NULL) {
goto bail;
}
}
/* Complete the "container" initialization */
wridlist->wl_size = qsize;
wridlist->wl_full = 0;
wridlist->wl_head = 0;
wridlist->wl_tail = 0;
wridlist->wl_wre = wl_wre;
wridlist->wl_wre_old_tail = NULL;
wridlist->wl_reap_next = NULL;
wridlist->wl_next = NULL;
wridlist->wl_prev = NULL;
if (wrid_for_srq) {
wridlist->wl_srq_en = 1;
wridlist->wl_free_list = (uint32_t *)wl_freel;
wridlist->wl_freel_head = 0;
wridlist->wl_freel_tail = 0;
wridlist->wl_freel_entries = qsize;
} else {
wridlist->wl_srq_en = 0;
wridlist->wl_free_list = NULL;
wridlist->wl_freel_head = 0;
wridlist->wl_freel_tail = 0;
wridlist->wl_freel_entries = 0;
wridlist->wl_srq_wqesz = 0;
wridlist->wl_srq_desc_addr = 0;
}
return (wridlist);
bail:
if (wridlist) {
if (wrid_for_srq) {
dapl_os_lock_destroy(&wridlist->wl_lock->wrl_lock);
}
dapl_os_free(wridlist, size);
}
if (wl_wre) {
dapl_os_free(wl_wre, wl_wre_size);
}
if (wl_freel) {
dapl_os_free(wl_freel, wl_freel_size);
}
return (NULL);
}
/*
* dapli_tavor_wrid_reaplist_add()
*/
static void
dapli_tavor_wrid_reaplist_add(ib_cq_handle_t cq, dapls_tavor_workq_hdr_t *wq)
{
/* dapl_os_assert(MUTEX_HELD(&cq->cq_wrid_wqhdr_lock)); */
dapl_os_lock(&wq->wq_wrid_lock->wrl_lock);
/*
* Add the "post" container (the last one on the current chain) to
* the CQ's "reapable" list
*/
if ((cq->cq_wrid_reap_head == NULL) &&
(cq->cq_wrid_reap_tail == NULL)) {
cq->cq_wrid_reap_head = wq->wq_wrid_post;
cq->cq_wrid_reap_tail = wq->wq_wrid_post;
} else {
cq->cq_wrid_reap_tail->wl_reap_next = wq->wq_wrid_post;
cq->cq_wrid_reap_tail = wq->wq_wrid_post;
}
dapl_os_unlock(&wq->wq_wrid_lock->wrl_lock);
}
/*
* dapli_tavor_wrid_wqhdr_find()
*/
static dapls_tavor_workq_hdr_t *
dapli_tavor_wrid_wqhdr_find(ib_cq_handle_t cq, uint_t qpn, uint_t send_or_recv)
{
DAPL_HASH_DATA curr;
DAPL_HASH_KEY key;
DAT_RETURN status;
/* dapl_os_assert(MUTEX_HELD(&cq->cq_wrid_wqhdr_lock)); */
/*
* Walk the CQ's work queue list, trying to find a send or recv queue
* with the same QP number. We do this even if we are going to later
* create a new entry because it helps us easily find the end of the
* list.
*/
key = (DAPL_HASH_KEY)(((uint64_t)send_or_recv << 32) | (uint32_t)qpn);
status = dapls_hash_search(cq->cq_wrid_wqhdr_list, key, &curr);
if (status == DAT_SUCCESS) {
return ((dapls_tavor_workq_hdr_t *)curr);
} else {
return (NULL);
}
}
/*
* dapli_tavor_wrid_get_wqeaddrsz()
*/
static uint32_t
dapli_tavor_wrid_get_wqeaddrsz(dapls_tavor_workq_hdr_t *wq)
{
dapls_tavor_wrid_entry_t *wre;
uint32_t wqeaddrsz;
uint32_t head;
/*
* If the container is empty, then there is no next entry. So just
* return zero. Note: the "head == tail" condition here can only
* mean that the container is empty because we have previously pulled
* something from the container.
*
* If the container is not empty, then find the next entry and return
* the contents of its "wqeaddrsz" field.
*/
if (wq->wq_wrid_poll->wl_head == wq->wq_wrid_poll->wl_tail) {
wqeaddrsz = 0;
} else {
/*
* We don't need to calculate the "next" head pointer here
* because "head" should already point to the next entry on
* the list (since we just pulled something off - in
* dapli_tavor_wrid_find_match() - and moved the head index
* forward.)
*/
head = wq->wq_wrid_poll->wl_head;
wre = &wq->wq_wrid_poll->wl_wre[head];
wqeaddrsz = wre->wr_wqeaddrsz;
}
return (wqeaddrsz);
}
/*
* dapli_tavor_wrid_list_reap()
* Note: The "wqhdr_list_lock" must be held.
*/
static dapls_tavor_workq_hdr_t *
dapli_tavor_wrid_list_reap(dapls_tavor_wrid_list_hdr_t *wridlist)
{
dapls_tavor_workq_hdr_t *wqhdr, *consume_wqhdr = NULL;
dapls_tavor_wrid_list_hdr_t *prev, *next;
/* Get the back pointer to the work queue header (see below) */
wqhdr = wridlist->wl_wqhdr;
dapl_os_lock(&wqhdr->wq_wrid_lock->wrl_lock);
/* Unlink the WRID list "container" from the work queue list */
prev = wridlist->wl_prev;
next = wridlist->wl_next;
if (prev != NULL) {
prev->wl_next = next;
}
if (next != NULL) {
next->wl_prev = prev;
}
/*
* If the back pointer to the work queue header shows that it
* was pointing to the entry we are about to remove, then the work
* queue header is reapable as well.
*/
if ((wqhdr->wq_wrid_poll == wridlist) &&
(wqhdr->wq_wrid_post == wridlist)) {
consume_wqhdr = wqhdr;
}
/* Be sure to update the "poll" and "post" container pointers */
if (wqhdr->wq_wrid_poll == wridlist) {
wqhdr->wq_wrid_poll = next;
}
if (wqhdr->wq_wrid_post == wridlist) {
wqhdr->wq_wrid_post = NULL;
}
/*
* Calculate the size and free the container, for SRQ wridlist is
* freed when srq gets freed
*/
if (!wridlist->wl_srq_en) {
if (wridlist->wl_wre) {
dapl_os_free(wridlist->wl_wre, wridlist->wl_size *
sizeof (dapls_tavor_wrid_entry_t));
}
dapl_os_assert(wridlist->wl_free_list == NULL);
dapl_os_free(wridlist, sizeof (dapls_tavor_wrid_list_hdr_t));
}
dapl_os_unlock(&wqhdr->wq_wrid_lock->wrl_lock);
return (consume_wqhdr);
}
/*
* dapls_tavor_srq_wrid_init()
*/
DAT_RETURN
dapls_tavor_srq_wrid_init(ib_srq_handle_t srq)
{
dapls_tavor_wrid_list_hdr_t *wridlist;
int i;
wridlist = dapli_tavor_wrid_get_list(srq->srq_wq_numwqe, 1);
if (wridlist == NULL) {
srq->srq_wridlist = NULL;
return (DAT_INSUFFICIENT_RESOURCES | DAT_RESOURCE_MEMORY);
}
/* initialize the free list with the descriptor addresses */
wridlist->wl_free_list[0] = srq->srq_wq_desc_addr;
for (i = 1; i < srq->srq_wq_numwqe; i++) {
wridlist->wl_free_list[i] = wridlist->wl_free_list[i-1] +
srq->srq_wq_wqesz;
}
wridlist->wl_srq_wqesz = srq->srq_wq_wqesz;
wridlist->wl_srq_desc_addr = srq->srq_wq_desc_addr;
srq->srq_wridlist = wridlist;
return (DAT_SUCCESS);
}
void
dapls_tavor_srq_wrid_free(ib_srq_handle_t srq)
{
dapls_tavor_wrid_list_hdr_t *wridlist;
size_t size = 0;
wridlist = srq->srq_wridlist;
if (wridlist) {
dapl_os_assert(wridlist->wl_srq_en == 1);
if (wridlist->wl_wre) {
dapl_os_free(wridlist->wl_wre, wridlist->wl_size *
sizeof (dapls_tavor_wrid_entry_t));
}
if (wridlist->wl_free_list) {
dapl_os_free(wridlist->wl_free_list, wridlist->wl_size *
sizeof (uint32_t));
}
if (wridlist->wl_lock) {
dapl_os_assert(wridlist->wl_lock->wrl_on_srq == 1);
dapl_os_lock_destroy(&wridlist->wl_lock->wrl_lock);
size = sizeof (dapls_tavor_wrid_lock_t);
}
size = size; /* pacify lint */
dapl_os_free(wridlist, size +
sizeof (dapls_tavor_wrid_list_hdr_t));
srq->srq_wridlist = NULL;
}
}
/*
* dapls_tavor_wrid_init()
*/
DAT_RETURN
dapls_tavor_wrid_init(ib_qp_handle_t qp)
{
dapls_tavor_workq_hdr_t *swq;
dapls_tavor_workq_hdr_t *rwq;
dapls_tavor_wrid_list_hdr_t *s_wridlist;
dapls_tavor_wrid_list_hdr_t *r_wridlist;
uint_t create_new_swq = 0;
uint_t create_new_rwq = 0;
/*
* For each of this QP's Work Queues, make sure we have a (properly
* initialized) Work Request ID list attached to the relevant
* completion queue. Grab the CQ lock(s) before manipulating the
* lists.
*/
dapli_tavor_wrid_wqhdr_lock_both(qp);
swq = dapli_tavor_wrid_wqhdr_find(qp->qp_sq_cqhdl, qp->qp_num,
TAVOR_WR_SEND);
if (swq == NULL) {
/* Couldn't find matching work queue header, create it */
create_new_swq = 1;
swq = dapli_tavor_wrid_wqhdr_create(qp->qp_sq_cqhdl,
qp->qp_num, TAVOR_WR_SEND, 1);
if (swq == NULL) {
/*
* If we couldn't find/allocate space for the workq
* header, then drop the lock(s) and return failure.
*/
dapli_tavor_wrid_wqhdr_unlock_both(qp);
return (DAT_INSUFFICIENT_RESOURCES);
}
}
qp->qp_sq_wqhdr = swq;
swq->wq_size = qp->qp_sq_numwqe;
swq->wq_head = 0;
swq->wq_tail = 0;
swq->wq_full = 0;
/*
* Allocate space for the dapls_tavor_wrid_entry_t container
*/
s_wridlist = dapli_tavor_wrid_get_list(swq->wq_size, 0);
if (s_wridlist == NULL) {
/*
* If we couldn't allocate space for tracking the WRID
* entries, then cleanup the workq header from above (if
* necessary, i.e. if we created the workq header). Then
* drop the lock(s) and return failure.
*/
if (create_new_swq) {
dapli_tavor_cq_wqhdr_remove(qp->qp_sq_cqhdl, swq);
}
dapli_tavor_wrid_wqhdr_unlock_both(qp);
return (DAT_INSUFFICIENT_RESOURCES | DAT_RESOURCE_MEMORY);
}
s_wridlist->wl_wqhdr = swq;
/* Chain the new WRID list container to the workq hdr list */
dapl_os_lock(&swq->wq_wrid_lock->wrl_lock);
dapli_tavor_wrid_wqhdr_add(swq, s_wridlist);
dapl_os_unlock(&swq->wq_wrid_lock->wrl_lock);
/*
* Now we repeat all the above operations for the receive work queue
*/
rwq = dapli_tavor_wrid_wqhdr_find(qp->qp_rq_cqhdl, qp->qp_num,
TAVOR_WR_RECV);
if (rwq == NULL) {
create_new_rwq = 1;
/* if qp is attached to an SRQ don't need to alloc wrid_lock */
rwq = dapli_tavor_wrid_wqhdr_create(qp->qp_rq_cqhdl,
qp->qp_num, TAVOR_WR_RECV, qp->qp_srq_enabled ? 0 : 1);
if (rwq == NULL) {
/*
* If we couldn't find/allocate space for the workq
* header, then free all the send queue resources we
* just allocated and setup (above), drop the lock(s)
* and return failure.
*/
dapl_os_lock(&swq->wq_wrid_lock->wrl_lock);
dapli_tavor_wrid_wqhdr_remove(swq, s_wridlist);
dapl_os_unlock(&swq->wq_wrid_lock->wrl_lock);
if (create_new_swq) {
dapli_tavor_cq_wqhdr_remove(qp->qp_sq_cqhdl,
swq);
}
dapli_tavor_wrid_wqhdr_unlock_both(qp);
return (DAT_INSUFFICIENT_RESOURCES |
DAT_RESOURCE_MEMORY);
}
}
qp->qp_rq_wqhdr = rwq;
rwq->wq_size = qp->qp_rq_numwqe;
rwq->wq_head = 0;
rwq->wq_tail = 0;
rwq->wq_full = 0;
/*
* Allocate space for the dapls_tavor_wrid_entry_t container
* For qp associated with SRQs the SRQ wridlist is used
*/
if (qp->qp_srq_enabled) {
/* Use existing srq_wridlist pointer */
r_wridlist = qp->qp_srq->srq_wridlist;
dapl_os_assert(r_wridlist != NULL);
/* store the wl_lock in the wqhdr */
rwq->wq_wrid_lock = r_wridlist->wl_lock;
dapl_os_assert(rwq->wq_wrid_lock != NULL);
} else {
/* Allocate memory for the r_wridlist */
r_wridlist = dapli_tavor_wrid_get_list(rwq->wq_size, 0);
}
if (r_wridlist == NULL) {
/*
* If we couldn't allocate space for tracking the WRID
* entries, then cleanup all the stuff from above. Then
* drop the lock(s) and return failure.
*/
dapl_os_lock(&swq->wq_wrid_lock->wrl_lock);
dapli_tavor_wrid_wqhdr_remove(swq, s_wridlist);
dapl_os_unlock(&swq->wq_wrid_lock->wrl_lock);
if (create_new_swq) {
dapli_tavor_cq_wqhdr_remove(qp->qp_sq_cqhdl, swq);
}
if (create_new_rwq) {
dapli_tavor_cq_wqhdr_remove(qp->qp_rq_cqhdl, rwq);
}
dapli_tavor_wrid_wqhdr_unlock_both(qp);
return (DAT_INSUFFICIENT_RESOURCES | DAT_RESOURCE_MEMORY);
}
/* For SRQ based QPs r_wridlist does not point to recv wqhdr */
if (!qp->qp_srq_enabled) {
r_wridlist->wl_wqhdr = rwq;
}
/* Chain the new WRID list "container" to the workq hdr list */
dapl_os_lock(&rwq->wq_wrid_lock->wrl_lock);
dapli_tavor_wrid_wqhdr_add(rwq, r_wridlist);
dapl_os_unlock(&rwq->wq_wrid_lock->wrl_lock);
dapli_tavor_wrid_wqhdr_unlock_both(qp);
return (DAT_SUCCESS);
}
/*
* dapls_tavor_wrid_cleanup()
*/
void
dapls_tavor_wrid_cleanup(DAPL_EP *ep, ib_qp_handle_t qp)
{
/*
* For each of this QP's Work Queues, move the WRID "container" to
* the "reapable" list. Although there may still be unpolled
* entries in these containers, it is not a big deal. We will not
* reap the list until either the Poll CQ command detects an empty
* condition or the CQ itself is freed. Grab the CQ lock(s) before
* manipulating the lists.
*/
dapli_tavor_wrid_wqhdr_lock_both(qp);
dapli_tavor_wrid_reaplist_add(qp->qp_sq_cqhdl, qp->qp_sq_wqhdr);
/*
* Repeat the above operation for the Recv work queue "container".
* However for qps with SRQ we flush the cq entries, remove the
* wridlist and wqhdr.
* Then drop the CQ lock(s) and return
*/
if (qp->qp_srq_enabled) {
/*
* Pull off all (if any) entries for this QP from CQ. This
* only includes entries that have not yet been polled
*/
dapl_os_lock(&qp->qp_rq_wqhdr->wq_wrid_lock->wrl_lock);
DAPL_FLUSH(ep)(qp);
/* Remove wridlist from WQHDR */
dapli_tavor_wrid_wqhdr_remove(qp->qp_rq_wqhdr,
qp->qp_rq_wqhdr->wq_wrid_post);
dapl_os_assert(qp->qp_rq_wqhdr->wq_wrid_post == NULL);
dapl_os_unlock(&qp->qp_rq_wqhdr->wq_wrid_lock->wrl_lock);
/* Free the WQHDR */
dapli_tavor_cq_wqhdr_remove(qp->qp_rq_cqhdl, qp->qp_rq_wqhdr);
} else {
dapli_tavor_wrid_reaplist_add(qp->qp_rq_cqhdl, qp->qp_rq_wqhdr);
}
dapli_tavor_wrid_wqhdr_unlock_both(qp);
}
/*
* dapli_tavor_wrid_wqhdr_create()
*/
static dapls_tavor_workq_hdr_t *
dapli_tavor_wrid_wqhdr_create(ib_cq_handle_t cq, uint_t qpn,
uint_t send_or_recv, uint_t alloc_wrl)
{
dapls_tavor_workq_hdr_t *wqhdr_tmp;
size_t size, aligned_size;
/* dapl_os_assert(MUTEX_HELD(&cq->cq_wrid_wqhdr_lock)); */
/*
* Allocate space for a work queue header structure and initialize it.
* Each work queue header structure includes a "wq_wrid_lock"
* which needs to be initialized.
*
* Note: the address smashing is needed to ensure wq_wrid_lock is
* 8-byte aligned, which is not always the case on 32-bit sparc.
*/
size = (sizeof (dapls_tavor_workq_hdr_t) + 0x7) & ~0x7;
aligned_size = size;
if (alloc_wrl) {
/* for non-srq wqhdr the lock is allocated with the wqhdr */
size = size + sizeof (dapls_tavor_wrid_lock_t);
}
wqhdr_tmp = dapl_os_alloc(size);
if (wqhdr_tmp == NULL) {
return (NULL);
}
if (alloc_wrl) {
wqhdr_tmp->wq_wrid_lock = (dapls_tavor_wrid_lock_t *)
(((uintptr_t)wqhdr_tmp + aligned_size) & ~0x7);
dapl_os_lock_init(&wqhdr_tmp->wq_wrid_lock->wrl_lock);
/* wrl allocated with wqhdr don't have srq enabled */
wqhdr_tmp->wq_wrid_lock->wrl_on_srq = 0;
}
wqhdr_tmp->wq_qpn = qpn;
wqhdr_tmp->wq_send_or_recv = send_or_recv;
wqhdr_tmp->wq_wrid_poll = NULL;
wqhdr_tmp->wq_wrid_post = NULL;
/* Chain the newly allocated work queue header to the CQ's list */
if (dapli_tavor_cq_wqhdr_add(cq, wqhdr_tmp) != DAT_SUCCESS) {
if (alloc_wrl) {
dapl_os_lock_destroy(&wqhdr_tmp->wq_wrid_lock->
wrl_lock);
}
dapl_os_free(wqhdr_tmp, size);
wqhdr_tmp = NULL;
}
return (wqhdr_tmp);
}
/*
* dapli_tavor_wrid_wqhdr_add()
*/
static void
dapli_tavor_wrid_wqhdr_add(dapls_tavor_workq_hdr_t *wqhdr,
dapls_tavor_wrid_list_hdr_t *wridlist)
{
/* dapl_os_assert(MUTEX_HELD(&wqhdr->wq_wrid_lock)); */
/* Chain the new WRID list "container" to the work queue list */
if ((wqhdr->wq_wrid_post == NULL) &&
(wqhdr->wq_wrid_poll == NULL)) {
wqhdr->wq_wrid_poll = wridlist;
wqhdr->wq_wrid_post = wridlist;
} else {
wqhdr->wq_wrid_post->wl_next = wridlist;
wridlist->wl_prev = wqhdr->wq_wrid_post;
wqhdr->wq_wrid_post = wridlist;
}
}
/*
* dapli_tavor_wrid_wqhdr_remove()
* Note: this is only called to remove the most recently added WRID list
* container.
*/
static void
dapli_tavor_wrid_wqhdr_remove(dapls_tavor_workq_hdr_t *wqhdr,
dapls_tavor_wrid_list_hdr_t *wridlist)
{
dapls_tavor_wrid_list_hdr_t *prev, *next;
/* dapl_os_assert(MUTEX_HELD(&wqhdr->wq_wrid_lock)); */
/* Unlink the WRID list "container" from the work queue list */
prev = wridlist->wl_prev;
next = wridlist->wl_next;
if (prev != NULL) {
prev->wl_next = next;
}
if (next != NULL) {
next->wl_prev = prev;
}
/*
* Update any pointers in the work queue hdr that may point to this
* WRID list container
*/
if (wqhdr->wq_wrid_post == wridlist) {
wqhdr->wq_wrid_post = prev;
}
if (wqhdr->wq_wrid_poll == wridlist) {
wqhdr->wq_wrid_poll = NULL;
}
}
/*
* dapli_tavor_wrid_wqhdr_lock_both()
*/
static void
dapli_tavor_wrid_wqhdr_lock_both(ib_qp_handle_t qp)
{
ib_cq_handle_t sq_cq, rq_cq;
sq_cq = qp->qp_sq_cqhdl;
rq_cq = qp->qp_rq_cqhdl;
/*
* If both work queues (send and recv) share a completion queue, then
* grab the common lock. If they use different CQs (hence different
* "cq_wrid_wqhdr_list" locks), then grab the send one first, then the
* receive. We do this consistently and correctly in
* tavor_wrid_wqhdr_unlock_both() below to avoid introducing any kind
* of dead lock condition.
*/
if (sq_cq == rq_cq) {
dapl_os_lock(&sq_cq->cq_wrid_wqhdr_lock);
} else {
dapl_os_lock(&sq_cq->cq_wrid_wqhdr_lock);
dapl_os_lock(&rq_cq->cq_wrid_wqhdr_lock);
}
}
/*
* dapli_tavor_wrid_wqhdr_unlock_both()
*/
static void
dapli_tavor_wrid_wqhdr_unlock_both(ib_qp_handle_t qp)
{
ib_cq_handle_t sq_cq, rq_cq;
sq_cq = qp->qp_sq_cqhdl;
rq_cq = qp->qp_rq_cqhdl;
/*
* See tavor_wrid_wqhdr_lock_both() above for more detail
*/
if (sq_cq == rq_cq) {
dapl_os_unlock(&sq_cq->cq_wrid_wqhdr_lock);
} else {
dapl_os_unlock(&rq_cq->cq_wrid_wqhdr_lock);
dapl_os_unlock(&sq_cq->cq_wrid_wqhdr_lock);
}
}
/*
* dapli_tavor_cq_wqhdr_add()
*/
static DAT_RETURN
dapli_tavor_cq_wqhdr_add(ib_cq_handle_t cq, dapls_tavor_workq_hdr_t *wqhdr)
{
DAPL_HASH_KEY key;
/* dapl_os_assert(MUTEX_HELD(&cq->cq_wrid_wqhdr_lock)); */
/*
* If the CQ's work queue list is empty, then just add it.
* Otherwise, chain it to the beginning of the list.
*/
key = (DAPL_HASH_KEY)(((uint64_t)wqhdr->wq_send_or_recv << 32) |
wqhdr->wq_qpn);
return (dapls_hash_insert(cq->cq_wrid_wqhdr_list, key, wqhdr));
}
/*
* dapli_tavor_cq_wqhdr_remove
*/
static void
dapli_tavor_cq_wqhdr_remove(ib_cq_handle_t cq, dapls_tavor_workq_hdr_t *wqhdr)
{
DAPL_HASH_DATA curr;
DAPL_HASH_KEY key;
size_t size = 0;
/* dapl_os_assert(MUTEX_HELD(&cq->cq_wrid_wqhdr_lock)); */
/* Remove "wqhdr" from the work queue header list on "cq" */
key = (DAPL_HASH_KEY)(((uint64_t)wqhdr->wq_send_or_recv << 32) |
wqhdr->wq_qpn);
(void) dapls_hash_remove(cq->cq_wrid_wqhdr_list, key, &curr);
size = (sizeof (dapls_tavor_workq_hdr_t) + 0x7) & ~0x7;
if (wqhdr->wq_wrid_lock && (!wqhdr->wq_wrid_lock->wrl_on_srq)) {
dapl_os_lock_destroy(&wqhdr->wq_wrid_lock->wrl_lock);
size += sizeof (dapls_tavor_wrid_lock_t);
}
/* Free the memory associated with "wqhdr" */
dapl_os_free(wqhdr, size);
}
/*
* dapls_tavor_srq_wrid_resize() is called to resize the wridlist
* associated with SRQS as a result of dat_srq_resize().
*
* Returns: DAT_TRUE if successful, otherwise DAT_FALSE
*/
DAT_BOOLEAN
dapls_tavor_srq_wrid_resize(ib_srq_handle_t srq_handle, uint32_t new_size)
{
dapls_tavor_wrid_list_hdr_t *wridlist;
dapls_tavor_wrid_entry_t *old_wl_wre;
dapls_tavor_wrid_entry_t *new_wl_wre;
uint32_t *old_wl_freel;
uint32_t *new_wl_freel;
uint32_t old_size;
uint32_t idx;
uint32_t prev_idx;
uint32_t i;
wridlist = srq_handle->srq_wridlist;
if (wridlist == NULL) {
return (DAT_FALSE);
}
dapl_os_assert(wridlist->wl_srq_en);
dapl_os_lock(&wridlist->wl_lock->wrl_lock);
old_wl_wre = wridlist->wl_wre;
old_wl_freel = wridlist->wl_free_list;
old_size = wridlist->wl_size;
new_wl_wre = (dapls_tavor_wrid_entry_t *)dapl_os_alloc(new_size *
sizeof (dapls_tavor_wrid_entry_t));
if (new_wl_wre == NULL) {
goto bail;
}
new_wl_freel = dapl_os_alloc(new_size * sizeof (uint32_t));
if (new_wl_freel == NULL) {
goto bail;
}
/*
* we just need to copy the old WREs to the new array. Since the
* descriptors are relatively addressed the descriptor to index
* mapping doesn't change.
*/
(void) dapl_os_memcpy(&new_wl_wre[0], &old_wl_wre[0],
old_size * sizeof (dapls_tavor_wrid_entry_t));
/*
* Copy the old free list to the new one
*/
idx = wridlist->wl_freel_head;
for (i = 0; i < wridlist->wl_freel_entries; i++) {
new_wl_freel[i] = old_wl_freel[idx];
idx = (idx + 1) % old_size;
}
/*
* Add the new entries in wl_wre to the new free list
*/
idx = wridlist->wl_freel_entries;
new_wl_freel[idx] = wridlist->wl_srq_desc_addr + old_size *
wridlist->wl_srq_wqesz;
prev_idx = idx;
idx = (idx + 1) % new_size;
for (i = 0; i < new_size - old_size - 1; i++) {
new_wl_freel[idx] = new_wl_freel[prev_idx] +
wridlist->wl_srq_wqesz;
prev_idx = idx;
idx = (idx + 1) % new_size;
}
wridlist->wl_size = new_size;
wridlist->wl_wre = new_wl_wre;
wridlist->wl_free_list = new_wl_freel;
wridlist->wl_freel_head = 0;
wridlist->wl_freel_tail = idx;
wridlist->wl_freel_entries = wridlist->wl_freel_entries + new_size -
old_size;
dapl_os_unlock(&wridlist->wl_lock->wrl_lock);
if (old_wl_wre) {
dapl_os_free(old_wl_wre, old_size *
sizeof (dapls_tavor_wrid_entry_t));
}
if (old_wl_freel) {
dapl_os_free(old_wl_freel, old_size * sizeof (uint32_t));
}
return (DAT_TRUE);
bail:
dapl_os_unlock(&wridlist->wl_lock->wrl_lock);
if (new_wl_wre) {
dapl_os_free(new_wl_wre, new_size *
sizeof (dapls_tavor_wrid_entry_t));
}
return (DAT_FALSE);
}