Skip to content

Commit 47b34ad

Browse files
edgargabrieledgargabriel
committed
smsc/accelerator: add new smsc component
- add smsc component based on the accelerator API IPC functionality. - adjust smsc base selection logic to enable a pass-through mechanism with the new smsc/accelerator component Signed-off-by: Edgar Gabriel <[email protected]>
1 parent 72c952d commit 47b34ad

10 files changed

+580
-14
lines changed

‎opal/mca/btl/sm/btl_sm_component.c

+4
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,10 @@ mca_btl_sm_component_init(int *num_btls, bool enable_progress_threads, bool enab
340340
mca_btl_sm.super.btl_get = mca_btl_sm_get;
341341
mca_btl_sm.super.btl_put = mca_btl_sm_put;
342342

343+
if (mca_smsc_base_has_feature(MCA_SMSC_FEATURE_ACCELERATOR)) {
344+
mca_btl_sm.super.btl_flags |= MCA_BTL_FLAGS_ACCELERATOR_GET;
345+
}
346+
343347
mca_btl_sm.super.btl_bandwidth = 40000; /* Mbs */
344348

345349
if (mca_smsc_base_has_feature(MCA_SMSC_FEATURE_CAN_MAP)) {

‎opal/mca/smsc/accelerator/Makefile.am

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#
2+
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
3+
# University Research and Technology
4+
# Corporation. All rights reserved.
5+
# Copyright (c) 2004-2009 The University of Tennessee and The University
6+
# of Tennessee Research Foundation. All rights
7+
# reserved.
8+
# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
9+
# University of Stuttgart. All rights reserved.
10+
# Copyright (c) 2004-2005 The Regents of the University of California.
11+
# All rights reserved.
12+
# Copyright (c) 2009-2014 Cisco Systems, Inc. All rights reserved.
13+
# Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
14+
# reserved.
15+
# Copyright (c) 2017 IBM Corporation. All rights reserved.
16+
# Copyright (c) 2020-2021 Google, LLC. All rights reserved.
17+
# Copyright (c) 2024 Advanced Micro Devices, Inc. All Rights reserved.
18+
# $COPYRIGHT$
19+
#
20+
# Additional copyrights may follow
21+
#
22+
# $HEADER$
23+
#
24+
25+
EXTRA_DIST = post_configure.sh
26+
27+
AM_CPPFLAGS = $(smsc_accelerator_CPPFLAGS)
28+
29+
30+
libmca_smsc_accelerator_la_sources = \
31+
smsc_accelerator_component.c \
32+
smsc_accelerator_module.c \
33+
smsc_accelerator_internal.h \
34+
smsc_accelerator.h
35+
36+
# Make the output library in this directory, and name it either
37+
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
38+
# (for static builds).
39+
40+
if MCA_BUILD_opal_smsc_accelerator_DSO
41+
component_noinst =
42+
component_install = mca_smsc_accelerator.la
43+
else
44+
component_noinst = libmca_smsc_accelerator.la
45+
component_install =
46+
endif
47+
48+
mcacomponentdir = $(opallibdir)
49+
mcacomponent_LTLIBRARIES = $(component_install)
50+
mca_smsc_accelerator_la_SOURCES = $(libmca_smsc_accelerator_la_sources)
51+
mca_smsc_accelerator_la_LDFLAGS = -module -avoid-version $(smsc_accelerator_LDFLAGS)
52+
mca_smsc_accelerator_la_LIBADD = $(top_builddir)/opal/lib@[email protected] \
53+
$(smsc_accelerator_LIBS)
54+
55+
noinst_LTLIBRARIES = $(component_noinst)
56+
libmca_smsc_accelerator_la_SOURCES = $(libmca_smsc_accelerator_la_sources)
57+
libmca_smsc_accelerator_la_LIBADD = $(smsc_accelerator_LIBS)
58+
libmca_smsc_accelerator_la_LDFLAGS = -module -avoid-version $(smsc_accelerator_LDFLAGS)

‎opal/mca/smsc/accelerator/post_configure.sh

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
DIRECT_CALL_HEADER="opal/mca/smsc/accelerator/smsc_accelerator.h"

‎opal/mca/smsc/accelerator/smsc_accelerator.h

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2+
/*
3+
* Copyright (c) 2024 Advanced Micro Devices, Inc. All Rights reserved.
4+
* $COPYRIGHT$
5+
*
6+
* Additional copyrights may follow
7+
*
8+
* $HEADER$
9+
*/
10+
11+
#ifndef OPAL_MCA_SMSC_ACCELERATOR_H
12+
#define OPAL_MCA_SMSC_ACCELERATOR_H
13+
14+
#include "opal_config.h"
15+
16+
#include "opal/mca/smsc/smsc.h"
17+
18+
mca_smsc_endpoint_t *mca_smsc_accelerator_get_endpoint(opal_proc_t *peer_proc);
19+
void mca_smsc_accelerator_return_endpoint(mca_smsc_endpoint_t *endpoint);
20+
21+
int mca_smsc_accelerator_copy_to(mca_smsc_endpoint_t *endpoint, void *local_address, void *remote_address,
22+
size_t size, void *reg_handle);
23+
24+
int mca_smsc_accelerator_copy_from(mca_smsc_endpoint_t *endpoint, void *local_address,
25+
void *remote_address, size_t size, void *reg_handle);
26+
27+
void *mca_smsc_accelerator_map_peer_region(mca_smsc_endpoint_t *endpoint, uint64_t flags,
28+
void *remote_ptr, size_t size, void **local_ptr);
29+
void mca_smsc_accelerator_unmap_peer_region(void *ctx);
30+
31+
void *mca_smsc_accelerator_register_region(void *local_address, size_t size);
32+
void mca_smsc_accelerator_deregister_region(void *reg_data);
33+
34+
#endif /* OPAL_MCA_SMSC_ACCELERATOR__H */

‎opal/mca/smsc/accelerator/smsc_accelerator_component.c

+133
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2+
/*
3+
* Copyright (c) 2024 Advanced Micro Devices, Inc. All Rights reserved.
4+
* $COPYRIGHT$
5+
*
6+
* Additional copyrights may follow
7+
*
8+
* $HEADER$
9+
*/
10+
#include "opal_config.h"
11+
12+
#include "opal/mca/smsc/base/base.h"
13+
#include "opal/mca/smsc/accelerator/smsc_accelerator_internal.h"
14+
#include "opal/mca/accelerator/accelerator.h"
15+
#include "opal/mca/accelerator/base/base.h"
16+
17+
static int mca_smsc_accelerator_component_register(void);
18+
static int mca_smsc_accelerator_component_open(void);
19+
static int mca_smsc_accelerator_component_close(void);
20+
static int mca_smsc_accelerator_component_query(void);
21+
static mca_smsc_module_t *mca_smsc_accelerator_component_enable(void);
22+
23+
#define MCA_SMSC_ACCELERATOR_DEFAULT_PRIORITY 80
24+
static const int mca_smsc_accelerator_default_priority = MCA_SMSC_ACCELERATOR_DEFAULT_PRIORITY;
25+
26+
mca_smsc_accelerator_component_t mca_smsc_accelerator_component = {
27+
.super = {
28+
.smsc_version = {
29+
MCA_SMSC_DEFAULT_VERSION("accelerator"),
30+
.mca_open_component = mca_smsc_accelerator_component_open,
31+
.mca_close_component = mca_smsc_accelerator_component_close,
32+
.mca_register_component_params = mca_smsc_accelerator_component_register,
33+
},
34+
.priority = MCA_SMSC_ACCELERATOR_DEFAULT_PRIORITY,
35+
.query = mca_smsc_accelerator_component_query,
36+
.enable = mca_smsc_accelerator_component_enable,
37+
},
38+
};
39+
40+
static int mca_smsc_accelerator_component_register(void)
41+
{
42+
mca_smsc_base_register_default_params(&mca_smsc_accelerator_component.super,
43+
mca_smsc_accelerator_default_priority);
44+
return OPAL_SUCCESS;
45+
}
46+
47+
static int mca_smsc_accelerator_component_open(void)
48+
{
49+
return OPAL_SUCCESS;
50+
}
51+
52+
static int mca_smsc_accelerator_component_close(void)
53+
{
54+
if (mca_smsc_accelerator_module.rcache) {
55+
(void) mca_rcache_base_module_destroy(mca_smsc_accelerator_module.rcache);
56+
mca_smsc_accelerator_module.rcache = NULL;
57+
}
58+
59+
return OPAL_SUCCESS;
60+
}
61+
62+
static int mca_smsc_accelerator_component_query(void)
63+
{
64+
if (0 == strcmp(opal_accelerator_base_selected_component.base_version.mca_component_name,
65+
"null")) {
66+
opal_output_verbose(10, opal_smsc_base_framework.framework_output,
67+
"smsc:accelerator:component_query: accelerator component is null: disqualifying myself");
68+
return OPAL_ERROR;
69+
}
70+
71+
if (!opal_accelerator.is_ipc_enabled()) {
72+
opal_output_verbose(10, opal_smsc_base_framework.framework_output,
73+
"smsc:accelerator:component_query: accelerator component does not have support for IPC operations: disqualifying myself");
74+
return OPAL_ERROR;
75+
}
76+
77+
return OPAL_SUCCESS;
78+
}
79+
80+
81+
static int mca_smsc_accelerator_reg(void *reg_data, void *local_address, size_t size,
82+
mca_rcache_base_registration_t *reg_handle)
83+
{
84+
mca_smsc_accelerator_registration_handle_t *reg = (mca_smsc_accelerator_registration_handle_t *) reg_handle;
85+
int ret;
86+
87+
ret = opal_accelerator.get_ipc_handle (mca_smsc_accelerator_module.device_id, local_address,
88+
&reg->ipc_handle);
89+
memcpy (reg->data.handle.accelerator, reg->ipc_handle.handle, SMSC_ACCELERATOR_HANDLE_SIZE);
90+
if (OPAL_SUCCESS != ret ) {
91+
return OPAL_ERROR;
92+
}
93+
94+
return OPAL_SUCCESS;
95+
}
96+
97+
static int mca_smsc_accelerator_dereg(void *reg_data, mca_rcache_base_registration_t *reg_handle)
98+
{
99+
mca_smsc_accelerator_registration_handle_t *reg = (mca_smsc_accelerator_registration_handle_t *) reg_handle;
100+
101+
OBJ_DESTRUCT(&(reg->ipc_handle));
102+
return OPAL_SUCCESS;
103+
}
104+
105+
static mca_smsc_module_t *mca_smsc_accelerator_component_enable(void)
106+
{
107+
if (0 > mca_smsc_accelerator_component.super.priority) {
108+
return NULL;
109+
}
110+
111+
mca_rcache_base_resources_t rcache_resources = {.cache_name = "smsc_accelerator",
112+
.reg_data = NULL,
113+
.sizeof_reg = sizeof(
114+
mca_smsc_accelerator_registration_handle_t),
115+
.register_mem = mca_smsc_accelerator_reg,
116+
.deregister_mem = mca_smsc_accelerator_dereg};
117+
118+
mca_smsc_accelerator_module.rcache = mca_rcache_base_module_create("grdma", NULL, &rcache_resources);
119+
if (NULL == mca_smsc_accelerator_module.rcache) {
120+
return NULL;
121+
}
122+
123+
/* Not set. Will initialize later */
124+
mca_smsc_accelerator_module.device_id = MCA_ACCELERATOR_NO_DEVICE_ID;
125+
126+
mca_smsc_accelerator_module.prev_smsc = mca_smsc;
127+
if ((NULL != mca_smsc_accelerator_module.prev_smsc) &&
128+
(mca_smsc_accelerator_module.prev_smsc->features & MCA_SMSC_FEATURE_CAN_MAP)) {
129+
mca_smsc_accelerator_module.super.features |= MCA_SMSC_FEATURE_CAN_MAP;
130+
}
131+
132+
return &mca_smsc_accelerator_module.super;
133+
}

‎opal/mca/smsc/accelerator/smsc_accelerator_internal.h

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2+
/*
3+
* Copyright (c) 2021 Google, Inc. All rights reserved.
4+
* Copyright (c) 2024 Advanced Micro Devices, Inc. All Rights reserved.
5+
* $COPYRIGHT$
6+
*
7+
* Additional copyrights may follow
8+
*
9+
* $HEADER$
10+
*/
11+
12+
#ifndef OPAL_MCA_SMSC_ACCELERATOR_INTERNAL_H
13+
#define OPAL_MCA_SMSC_ACCELERATOR_INTERNAL_H
14+
15+
#include "opal_config.h"
16+
17+
#include "opal/mca/rcache/base/base.h"
18+
#include "opal/mca/rcache/rcache.h"
19+
#include "opal/mca/smsc/accelerator/smsc_accelerator.h"
20+
#include "opal/mca/accelerator/accelerator.h"
21+
22+
#define SMSC_ACCELERATOR_HANDLE_SIZE IPC_MAX_HANDLE_SIZE
23+
struct mca_smsc_accelerator_registration_data_t {
24+
uint64_t base_addr;
25+
union {
26+
uint8_t accelerator[SMSC_ACCELERATOR_HANDLE_SIZE];
27+
void* host;
28+
} handle;
29+
};
30+
typedef struct mca_smsc_accelerator_registration_data_t mca_smsc_accelerator_registration_data_t;
31+
32+
struct mca_smsc_accelerator_registration_handle_t {
33+
mca_rcache_base_registration_t base;
34+
mca_smsc_accelerator_registration_data_t data;
35+
opal_accelerator_ipc_handle_t ipc_handle;
36+
};
37+
typedef struct mca_smsc_accelerator_registration_handle_t mca_smsc_accelerator_registration_handle_t;
38+
OBJ_CLASS_DECLARATION(mca_smsc_accelerator_registration_handle_t);
39+
40+
#define MCA_SMSC_ACCELERATOR_REG_DATA_TO_HANDLE(data_ptr) \
41+
((mca_smsc_accelerator_registration_handle_t *) ((uintptr_t) data_ptr \
42+
- offsetof(mca_smsc_accelerator_registration_handle_t, \
43+
data)))
44+
45+
46+
struct mca_smsc_accelerator_endpoint_t {
47+
mca_smsc_endpoint_t super;
48+
mca_smsc_endpoint_t *prev_endpoint;
49+
mca_rcache_base_module_t *rcache;
50+
};
51+
typedef struct mca_smsc_accelerator_endpoint_t mca_smsc_accelerator_endpoint_t;
52+
OBJ_CLASS_DECLARATION(mca_smsc_accelerator_endpoint_t);
53+
54+
struct mca_smsc_accelerator_component_t {
55+
mca_smsc_component_t super;
56+
};
57+
typedef struct mca_smsc_accelerator_component_t mca_smsc_accelerator_component_t;
58+
59+
struct mca_smsc_accelerator_module_t {
60+
mca_smsc_module_t super;
61+
mca_smsc_module_t *prev_smsc;
62+
mca_rcache_base_module_t *rcache;
63+
int device_id;
64+
};
65+
typedef struct mca_smsc_accelerator_module_t mca_smsc_accelerator_module_t;
66+
67+
extern mca_smsc_accelerator_module_t mca_smsc_accelerator_module;
68+
extern mca_smsc_accelerator_component_t mca_smsc_accelerator_component;
69+
70+
#endif /* OPAL_MCA_SMSC_ACCELERATOR_INTERNAL_H */

0 commit comments

Comments
 (0)