OpenSolaris_b135/lib/udapl/udapl_tavor/common/dapl_cr_callback.c
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved.
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/*
*
* MODULE: dapls_cr_callback.c
*
* PURPOSE: implements passive side connection callbacks
*
* Description: Accepts asynchronous callbacks from the Communications Manager
* for EVDs that have been specified as the connection_evd.
*
* $Id: dapl_cr_callback.c,v 1.58 2003/08/20 14:55:39 sjs2 Exp $
*/
#include "dapl.h"
#include "dapl_evd_util.h"
#include "dapl_cr_util.h"
#include "dapl_ia_util.h"
#include "dapl_sp_util.h"
#include "dapl_ep_util.h"
#include "dapl_adapter_util.h"
/*
* Prototypes
*/
DAT_RETURN dapli_connection_request(
IN ib_cm_handle_t ib_cm_handle,
IN DAPL_SP *sp_ptr,
IN DAPL_PRIVATE *prd_ptr,
IN DAPL_EVD *evd_ptr);
DAPL_EP * dapli_get_sp_ep(
IN ib_cm_handle_t ib_cm_handle,
IN DAPL_SP *sp_ptr,
IN const ib_cm_events_t ib_cm_event);
/*
* dapls_cr_callback
*
* The callback function registered with verbs for passive side of
* connection requests. The interface is specified by cm_api.h
*
*
* Input:
* ib_cm_handle, Handle to CM
* ib_cm_event Specific CM event
* instant_data Private data with DAT ADDRESS header
* context SP pointer
*
* Output:
* None
*
*/
void
dapls_cr_callback(
IN ib_cm_handle_t ib_cm_handle,
IN const ib_cm_events_t ib_cm_event,
IN const void *private_data_ptr, /* event data */
IN const void *context)
{
DAPL_EP *ep_ptr;
DAPL_EVD *evd_ptr;
DAPL_SP *sp_ptr;
DAPL_PRIVATE *prd_ptr;
DAT_EVENT_NUMBER event_type;
DAT_RETURN dat_status;
dapl_dbg_log(DAPL_DBG_TYPE_CM,
"--> dapls_cr_callback! context: 0x%p "
"event: %d cm_handle 0x%llx magic 0x%x\n",
context, ib_cm_event, ib_cm_handle,
((DAPL_HEADER *)context)->magic);
if (((DAPL_HEADER *)context)->magic == DAPL_MAGIC_INVALID) {
return;
}
/*
* Passive side of the connection, context is a SP and
* we need to look up the EP.
*/
dapl_os_assert(((DAPL_HEADER *)context)->magic == DAPL_MAGIC_PSP ||
((DAPL_HEADER *)context)->magic == DAPL_MAGIC_RSP);
sp_ptr = (DAPL_SP *) context;
/*
* CONNECT_REQUEST events create an event on the PSP
* EVD, which will trigger connection processing. The
* sequence is:
* CONNECT_REQUEST Event to SP
* CONNECTED Event to EP
* DISCONNECT Event to EP
*
* Obtain the EP if required and set an event up on the correct EVD.
*/
if (ib_cm_event == IB_CME_CONNECTION_REQUEST_PENDING ||
ib_cm_event == IB_CME_CONNECTION_REQUEST_PENDING_PRIVATE_DATA) {
ep_ptr = NULL;
evd_ptr = sp_ptr->evd_handle;
} else {
ep_ptr = dapli_get_sp_ep(ib_cm_handle, sp_ptr, ib_cm_event);
dapl_os_assert(ep_ptr != NULL);
evd_ptr = (DAPL_EVD *) ep_ptr->param.connect_evd_handle;
dapl_dbg_log(DAPL_DBG_TYPE_CM,
" dapls_cr_callback cont: ep 0x%p evd 0x%p\n",
ep_ptr, evd_ptr);
}
prd_ptr = (DAPL_PRIVATE *)private_data_ptr;
dat_status = DAT_INTERNAL_ERROR; /* init to ERR */
switch (ib_cm_event) {
case IB_CME_CONNECTION_REQUEST_PENDING:
case IB_CME_CONNECTION_REQUEST_PENDING_PRIVATE_DATA: {
/*
* Requests arriving on a disabled SP are immediatly rejected
*/
dapl_os_lock(&sp_ptr->header.lock);
if (sp_ptr->listening == DAT_FALSE) {
dapl_os_unlock(&sp_ptr->header.lock);
dapl_dbg_log(DAPL_DBG_TYPE_CM,
"---> dapls_cr_callback: conn event on down SP\n");
return;
}
if (sp_ptr->header.handle_type == DAT_HANDLE_TYPE_RSP) {
/*
* RSP connections only allow a single connection. Close
* it down NOW so we reject any further connections.
*/
sp_ptr->listening = DAT_FALSE;
}
dapl_os_unlock(&sp_ptr->header.lock);
/*
* Only occurs on the passive side of a connection
* dapli_connection_request will post the connection
* event if appropriate.
*/
dat_status = dapli_connection_request(ib_cm_handle,
sp_ptr, prd_ptr, evd_ptr);
break;
}
case IB_CME_CONNECTED: {
/*
* This is just a notification the connection is now
* established, there isn't any private data to deal with.
*
* Update the EP state and cache a copy of the cm handle,
* then let the user know we are ready to go.
*/
dapl_os_lock(&ep_ptr->header.lock);
if (ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECT_PENDING) {
/*
* If someone pulled the plug on the connection, just
* exit
*/
dapl_os_unlock(&ep_ptr->header.lock);
dat_status = DAT_SUCCESS;
break;
}
dapls_ib_connected(ep_ptr);
ep_ptr->param.ep_state = DAT_EP_STATE_CONNECTED;
ep_ptr->cm_handle = ib_cm_handle;
dapl_os_unlock(&ep_ptr->header.lock);
dat_status = dapls_evd_post_connection_event(
evd_ptr,
DAT_CONNECTION_EVENT_ESTABLISHED,
(DAT_HANDLE) ep_ptr,
((DAPL_CR *)ep_ptr->cr_ptr)->param.private_data_size,
((DAPL_CR *)ep_ptr->cr_ptr)->param.private_data);
/*
* post them to the recv evd now.
* there is a race here - if events arrive after we change
* the ep state to connected and before we process premature
* events
*/
dapls_evd_post_premature_events(ep_ptr);
break;
}
case IB_CME_DISCONNECTED:
case IB_CME_DISCONNECTED_ON_LINK_DOWN: {
/*
* EP is now fully disconnected; initiate any post processing
* to reset the underlying QP and get the EP ready for
* another connection
*/
dapl_os_lock(&ep_ptr->header.lock);
if (ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECTED) {
/* DTO error caused this */
event_type = DAT_CONNECTION_EVENT_BROKEN;
} else {
ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECTED;
dapls_ib_disconnect_clean(ep_ptr, DAT_FALSE,
ib_cm_event);
event_type = DAT_CONNECTION_EVENT_DISCONNECTED;
}
dapls_evd_post_premature_events(ep_ptr);
ep_ptr->cr_ptr = NULL;
dapl_os_unlock(&ep_ptr->header.lock);
/*
* If the user has done an ep_free of the EP, we have been
* waiting for the disconnect event; just clean it up now.
*/
if (ep_ptr->header.magic == DAPL_MAGIC_EP_EXIT) {
(void) dapl_ep_free(ep_ptr);
}
/* If the EP has been freed, the evd_ptr will be NULL */
if (evd_ptr != NULL) {
dat_status = dapls_evd_post_connection_event(
evd_ptr, event_type, (DAT_HANDLE) ep_ptr, 0, 0);
}
break;
}
case IB_CME_DESTINATION_REJECT:
case IB_CME_DESTINATION_REJECT_PRIVATE_DATA:
case IB_CME_DESTINATION_UNREACHABLE: {
/*
* After posting an accept the requesting node has
* stopped talking.
*/
dapl_os_lock(&ep_ptr->header.lock);
ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECTED;
ep_ptr->cm_handle = IB_INVALID_HANDLE;
dapls_ib_disconnect_clean(ep_ptr, DAT_FALSE, ib_cm_event);
dapl_os_unlock(&ep_ptr->header.lock);
dat_status = dapls_evd_post_connection_event(
evd_ptr,
DAT_CONNECTION_EVENT_ACCEPT_COMPLETION_ERROR,
(DAT_HANDLE) ep_ptr, 0, 0);
break;
}
case IB_CME_TOO_MANY_CONNECTION_REQUESTS: {
/*
* DAPL does not deal with this IB error. There is a
* separate OVERFLOW event error if we try to post too many
* events, but we don't propagate this provider error. Not
* all providers generate this error.
*/
break;
}
case IB_CME_LOCAL_FAILURE: {
ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECTED;
dapls_ib_disconnect_clean(ep_ptr, DAT_FALSE, ib_cm_event);
dat_status = dapls_evd_post_connection_event(
evd_ptr,
DAT_CONNECTION_EVENT_BROKEN,
(DAT_HANDLE) ep_ptr, 0, 0);
break;
}
case IB_CME_TIMED_OUT: {
ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECTED;
dapls_ib_disconnect_clean(ep_ptr, DAT_FALSE, ib_cm_event);
dat_status = dapls_evd_post_connection_event(
evd_ptr,
DAT_CONNECTION_EVENT_TIMED_OUT,
(DAT_HANDLE) ep_ptr, 0, 0);
break;
}
default:
dapl_os_assert(0); /* shouldn't happen */
break;
}
if (dat_status != DAT_SUCCESS) {
/* The event post failed; take appropriate action. */
(void) dapls_ib_reject_connection(ib_cm_handle,
IB_CME_LOCAL_FAILURE, sp_ptr);
return;
}
}
/*
* dapli_connection_request
*
* Process a connection request on the Passive side of a connection.
* Create a CR record and link it on to the SP so we can update it
* and free it later. Create an EP if specified by the PSP flags.
*
* Input:
* ib_cm_handle,
* sp_ptr
* event_ptr
* prd_ptr
*
* Output:
* None
*
* Returns
* DAT_INSUFFICIENT_RESOURCES
* DAT_SUCCESS
*
*/
DAT_RETURN
dapli_connection_request(
IN ib_cm_handle_t ib_cm_handle,
IN DAPL_SP *sp_ptr,
IN DAPL_PRIVATE *prd_ptr,
IN DAPL_EVD *evd_ptr)
{
DAT_RETURN dat_status;
DAPL_CR *cr_ptr;
DAPL_EP *ep_ptr;
DAPL_IA *ia_ptr;
DAT_SP_HANDLE sp_handle;
struct sockaddr_in *sv4;
struct sockaddr_in6 *sv6;
uint8_t *sadata;
DAT_COUNT length;
cr_ptr = dapls_cr_alloc(sp_ptr->header.owner_ia);
if (cr_ptr == NULL) {
/* Invoking function will call dapls_ib_cm_reject() */
return (DAT_INSUFFICIENT_RESOURCES);
}
/*
* Set up the CR
*/
cr_ptr->sp_ptr = sp_ptr; /* maintain sp_ptr in case of reject */
cr_ptr->ib_cm_handle = ib_cm_handle;
/*
* Copy the remote address and private data out of the private_data
* payload and put them in a local structure
*/
cr_ptr->param.private_data = cr_ptr->private_data;
cr_ptr->param.remote_ia_address_ptr =
(DAT_IA_ADDRESS_PTR)&cr_ptr->remote_ia_address;
cr_ptr->param.remote_port_qual =
(DAT_PORT_QUAL) prd_ptr->hello_msg.hi_port;
length = (DAT_COUNT) prd_ptr->hello_msg.hi_clen;
cr_ptr->param.private_data_size = length;
(void) dapl_os_memcpy(cr_ptr->private_data,
prd_ptr->private_data, length);
switch (prd_ptr->hello_msg.hi_ipv) {
case AF_INET:
sv4 = (struct sockaddr_in *)&cr_ptr->remote_ia_address;
sv4->sin_family = AF_INET;
sv4->sin_port = prd_ptr->hello_msg.hi_port;
sv4->sin_addr = prd_ptr->hello_msg.hi_v4ipaddr;
break;
case AF_INET6:
sv6 = (struct sockaddr_in6 *)&cr_ptr->remote_ia_address;
sv6->sin6_family = AF_INET6;
sv6->sin6_port = prd_ptr->hello_msg.hi_port;
sv6->sin6_addr = prd_ptr->hello_msg.hi_v6ipaddr;
break;
default:
sadata = (uint8_t *)&cr_ptr->remote_ia_address;
(void) dapl_os_memcpy(sadata, prd_ptr->hello_msg.hi_saaddr,
DAPL_ATS_NBYTES);
break;
}
/* EP will be NULL unless RSP service point */
ep_ptr = (DAPL_EP *) sp_ptr->ep_handle;
if (sp_ptr->psp_flags == DAT_PSP_PROVIDER_FLAG) {
/*
* Never true for RSP connections
*
* Create an EP for the user. If we can't allocate an
* EP we are out of resources and need to tell the
* requestor that we cant help them.
*/
ia_ptr = sp_ptr->header.owner_ia;
ep_ptr = dapl_ep_alloc(ia_ptr, NULL, DAT_FALSE);
if (ep_ptr == NULL) {
dapls_cr_free(cr_ptr);
/* Invoking function will call dapls_ib_cm_reject() */
return (DAT_INSUFFICIENT_RESOURCES);
}
/* Link the EP onto the IA */
dapl_ia_link_ep(ia_ptr, ep_ptr);
}
cr_ptr->param.local_ep_handle = ep_ptr;
if (ep_ptr != NULL) {
/* Assign valid EP fields: RSP and PSP_PROVIDER_FLAG only */
if (sp_ptr->psp_flags == DAT_PSP_PROVIDER_FLAG) {
ep_ptr->param.ep_state =
DAT_EP_STATE_TENTATIVE_CONNECTION_PENDING;
} else { /* RSP */
dapl_os_assert(sp_ptr->header.handle_type ==
DAT_HANDLE_TYPE_RSP);
ep_ptr->param.ep_state =
DAT_EP_STATE_PASSIVE_CONNECTION_PENDING;
}
ep_ptr->cm_handle = ib_cm_handle;
}
/* Post the event. */
/* assign sp_ptr to union to avoid typecast errors from compilers */
sp_handle.psp_handle = (DAT_PSP_HANDLE)sp_ptr;
dat_status = dapls_evd_post_cr_arrival_event(
evd_ptr,
DAT_CONNECTION_REQUEST_EVENT,
sp_handle,
(DAT_IA_ADDRESS_PTR)&sp_ptr->header.owner_ia->hca_ptr->hca_address,
sp_ptr->conn_qual,
(DAT_CR_HANDLE)cr_ptr);
if (dat_status != DAT_SUCCESS) {
dapls_cr_free(cr_ptr);
(void) dapls_ib_reject_connection(ib_cm_handle,
IB_CME_LOCAL_FAILURE, sp_ptr);
return (DAT_INSUFFICIENT_RESOURCES);
}
/* link the CR onto the SP so we can pick it up later */
dapl_sp_link_cr(sp_ptr, cr_ptr);
return (DAT_SUCCESS);
}
/*
* dapli_get_sp_ep
*
* Passive side of a connection is now fully established. Clean
* up resources and obtain the EP pointer associated with a CR in
* the SP
*
* Input:
* ib_cm_handle,
* sp_ptr
*
* Output:
* none
*
* Returns
* ep_ptr
*
*/
DAPL_EP *
dapli_get_sp_ep(
IN ib_cm_handle_t ib_cm_handle,
IN DAPL_SP *sp_ptr,
IN const ib_cm_events_t ib_cm_event)
{
DAPL_CR *cr_ptr;
DAPL_EP *ep_ptr;
/*
* There are potentially multiple connections in progress. Need to
* go through the list and find the one we are interested
* in. There is no guarantee of order. dapl_sp_search_cr
* leaves the CR on the SP queue.
*/
cr_ptr = dapl_sp_search_cr(sp_ptr, ib_cm_handle);
if (cr_ptr == NULL) {
dapl_os_assert(0);
return (NULL);
}
ep_ptr = (DAPL_EP *)cr_ptr->param.local_ep_handle;
dapl_os_assert(!(DAPL_BAD_HANDLE(ep_ptr, DAPL_MAGIC_EP)) ||
ep_ptr->header.magic == DAPL_MAGIC_EP_EXIT);
if (ib_cm_event == IB_CME_DISCONNECTED ||
ib_cm_event == IB_CME_DISCONNECTED_ON_LINK_DOWN) {
/* Remove the CR from the queue */
dapl_sp_remove_cr(sp_ptr, cr_ptr);
/*
* Last event, time to clean up and dispose of the resource
*/
dapls_cr_free(cr_ptr);
/*
* If this SP has been removed from service, free it
* up after the last CR is removed
*/
dapl_os_lock(&sp_ptr->header.lock);
if (sp_ptr->listening != DAT_TRUE &&
sp_ptr->cr_list_count == 0 &&
sp_ptr->state != DAPL_SP_STATE_FREE) {
dapl_dbg_log(DAPL_DBG_TYPE_CM,
"--> dapli_get_sp_ep! disconnect dump sp: %p \n",
sp_ptr);
sp_ptr->state = DAPL_SP_STATE_FREE;
dapl_os_unlock(&sp_ptr->header.lock);
(void) dapls_ib_remove_conn_listener(sp_ptr->
header.owner_ia, sp_ptr);
dapls_ia_unlink_sp((DAPL_IA *)sp_ptr->header.owner_ia,
sp_ptr);
dapls_sp_free_sp(sp_ptr);
} else {
dapl_os_unlock(&sp_ptr->header.lock);
}
}
return (ep_ptr);
}