Skip to content

Commit

Permalink
Merge pull request #7258 from raffenet/4.3.x-update
Browse files Browse the repository at this point in the history 
[4.3.x] misc: Fixes for ch4:ofi and prep for 4.3.0rc3

Approved-by: Hui Zhou <hzhou321@anl.gov>
  • Loading branch information
@raffenet
raffenet authored Jan 14, 2025
2 parents e12effd + 999d6ce commit 4f08552
Show file tree
Hide file tree
Showing 12 changed files with 75 additions and 66 deletions.
5 changes: 5 additions & 0 deletions CHANGES
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,11 @@

# Fix compilation issue with g++ in -std=gnu++20 mode

# Fix bug in MPI_ANY_SOURCE handling observed using the libfabric CXI
provider

# Add NIC information to error messages in ch4:ofi netmod

===============================================================================
Changes in 4.2
===============================================================================
Expand Down
6 changes: 3 additions & 3 deletions maint/extracterrmsgs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -680,12 +680,12 @@ sub ProcessFile
# add longnames since we omit errnames.txt for these
$longnames{"**ofid_$name"} = "OFI call $name failed";
$longnamesDefined{"**ofid_$name"} = "$filename:$linecount";
$longnames{"**ofid_$name %s %d %s %s"} = "OFI call $name failed (%s:%d:%s:%s)";
$longnamesDefined{"**ofid_$name %s %d %s %s"} = "$filename:$linecount";
$longnames{"**ofid_$name %s %s"} = "OFI call $name failed (default nic=%s: %s)";
$longnamesDefined{"**ofid_$name %s %s"} = "$filename:$linecount";
}

$generic_msgs{"**ofid_$name"}++;
$specific_msgs{"**ofid_$name %s %d %s %s"}++;
$specific_msgs{"**ofid_$name %s %s"}++;

next;
}
Expand Down
2 changes: 1 addition & 1 deletion maint/version.m4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
# changing this by playing with diversions, but then we would probably be
# playing with autotools-fire.

m4_define([MPICH_VERSION_m4],[4.3.0rc2])dnl
m4_define([MPICH_VERSION_m4],[4.3.0rc3])dnl
m4_define([MPICH_RELEASE_DATE_m4],[unreleased development copy])dnl

# For libtool ABI versioning rules see:
Expand Down
12 changes: 6 additions & 6 deletions src/mpid/ch4/netmod/ofi/coll/ofi_bcast_tree_rma.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,10 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Bcast_intra_triggered_rma(void *buffer, i
if (0 != strcmp(MPIDI_OFI_global.prov_use[0]->fabric_attr->prov_name, "cxi")) {
do {
ret = fi_cntr_wait(rcv_cntr, 1, 1);
MPIR_ERR_CHKANDJUMP4(ret < 0 && ret != -FI_ETIMEDOUT,
MPIR_ERR_CHKANDJUMP2(ret < 0 && ret != -FI_ETIMEDOUT,
mpi_errno, MPI_ERR_RMA_RANGE,
"**ofid_cntr_wait", "**ofid_cntr_wait %s %d %s %s",
__SHORT_FILE__, __LINE__, __func__, fi_strerror(-ret));
"**ofid_cntr_wait", "**ofid_cntr_wait %s %s",
MPIDI_OFI_DEFAULT_NIC_NAME, fi_strerror(-ret));
MPID_Progress_test(NULL);
} while (ret == -FI_ETIMEDOUT);
} else {
Expand All @@ -90,10 +90,10 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Bcast_intra_triggered_rma(void *buffer, i
if (0 != strcmp(MPIDI_OFI_global.prov_use[0]->fabric_attr->prov_name, "cxi")) {
do {
ret = fi_cntr_wait(snd_cntr, num_children, 1);
MPIR_ERR_CHKANDJUMP4(ret < 0 && ret != -FI_ETIMEDOUT,
MPIR_ERR_CHKANDJUMP2(ret < 0 && ret != -FI_ETIMEDOUT,
mpi_errno, MPI_ERR_RMA_RANGE,
"**ofid_cntr_wait", "**ofid_cntr_wait %s %d %s %s",
__SHORT_FILE__, __LINE__, __func__, fi_strerror(-ret));
"**ofid_cntr_wait", "**ofid_cntr_wait %s %s",
MPIDI_OFI_DEFAULT_NIC_NAME, fi_strerror(-ret));
MPID_Progress_test(NULL);
} while (ret == -FI_ETIMEDOUT);
} else {
Expand Down
12 changes: 6 additions & 6 deletions src/mpid/ch4/netmod/ofi/coll/ofi_bcast_tree_tagged.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,10 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Bcast_intra_triggered_tagged(void *buffer
if (0 != strcmp(MPIDI_OFI_global.prov_use[0]->fabric_attr->prov_name, "cxi")) {
do {
ret = fi_cntr_wait(rcv_cntr, 1, 1);
MPIR_ERR_CHKANDJUMP4(ret < 0 && ret != -FI_ETIMEDOUT,
MPIR_ERR_CHKANDJUMP2(ret < 0 && ret != -FI_ETIMEDOUT,
mpi_errno, MPI_ERR_RMA_RANGE,
"**ofid_cntr_wait", "**ofid_cntr_wait %s %d %s %s",
__SHORT_FILE__, __LINE__, __func__, fi_strerror(-ret));
"**ofid_cntr_wait", "**ofid_cntr_wait %s %s",
MPIDI_OFI_DEFAULT_NIC_NAME, fi_strerror(-ret));
MPID_Progress_test(NULL);
} while (ret == -FI_ETIMEDOUT);
} else {
Expand All @@ -89,10 +89,10 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Bcast_intra_triggered_tagged(void *buffer
if (0 != strcmp(MPIDI_OFI_global.prov_use[0]->fabric_attr->prov_name, "cxi")) {
do {
ret = fi_cntr_wait(snd_cntr, num_children, 1);
MPIR_ERR_CHKANDJUMP4(ret < 0 && ret != -FI_ETIMEDOUT,
MPIR_ERR_CHKANDJUMP2(ret < 0 && ret != -FI_ETIMEDOUT,
mpi_errno, MPI_ERR_RMA_RANGE,
"**ofid_cntr_wait", "**ofid_cntr_wait %s %d %s %s",
__SHORT_FILE__, __LINE__, __func__, fi_strerror(-ret));
"**ofid_cntr_wait", "**ofid_cntr_wait %s %s",
MPIDI_OFI_DEFAULT_NIC_NAME, fi_strerror(-ret));
MPID_Progress_test(NULL);
} while (ret == -FI_ETIMEDOUT);
} else {
Expand Down
12 changes: 6 additions & 6 deletions src/mpid/ch4/netmod/ofi/errnames.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,14 @@
# Most of the libfabric call error names are generated from MPIDI_OFI_CALL macros,
# some of them are explicitly used via MPIR_ERR_CHKANDJUMP4, they need be listed here.
**ofid_cancel:OFI cancel failed
**ofid_cancel %s %d %s %s:OFI cancel failed (%s:%d:%s:%s)
**ofid_cancel %s %s:OFI cancel failed (default nic=%s: %s)
**ofid_cntr_open:OFI Counter open failed
**ofid_cntr_open %s %d %s %s:OFI OFI Counter open failed (%s:%d:%s:%s)
**ofid_cntr_open %s %s:OFI OFI Counter open failed (default nic=%s: %s)
**ofid_cntr_wait:OFI Counter wait failed
**ofid_cntr_wait %s %d %s %s:OFI OFI Counter wait failed (%s:%d:%s:%s)
**ofid_cntr_wait %s %s:OFI OFI Counter wait failed (default nic=%s: %s)
**ofid_enable_trigger:OFI triggered ops enable failed
**ofid_enable_trigger %s %d %s %s:OFI triggered ops enable failed (%s:%d:%s:%s)
**ofid_enable_trigger %s %s:OFI triggered ops enable failed (default nic=%s: %s)
**ofid_issue_trigger:OFI triggered ops issue failed
**ofid_issue_trigger %s %d %s %s:OFI triggered ops issue failed (%s:%d:%s:%s)
**ofid_issue_trigger %s %s:OFI triggered ops issue failed (default nic=%s: %s)
**ofid_poll:OFI poll failed
**ofid_poll %s %d %s %s:OFI poll failed (%s:%d:%s:%s)
**ofid_poll %s %s:OFI poll failed (default nic=%s: %s)
19 changes: 10 additions & 9 deletions src/mpid/ch4/netmod/ofi/ofi_events.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -737,9 +737,10 @@ int MPIDI_OFI_handle_cq_error(int vci, int nic, ssize_t ret)
break;

default:
MPIR_ERR_SETFATALANDJUMP4(mpi_errno, MPI_ERR_OTHER, "**ofid_poll",
"**ofid_poll %s %d %s %s", __SHORT_FILE__,
__LINE__, __func__, fi_strerror(e.err));
MPIR_ERR_SETFATALANDJUMP2(mpi_errno, MPI_ERR_OTHER, "**ofid_poll",
"**ofid_poll %s %s",
MPIDI_OFI_DEFAULT_NIC_NAME,
fi_strerror(e.err));
}

break;
Expand Down Expand Up @@ -782,17 +783,17 @@ int MPIDI_OFI_handle_cq_error(int vci, int nic, ssize_t ret)
break;

default:
MPIR_ERR_SETFATALANDJUMP4(mpi_errno, MPI_ERR_OTHER, "**ofid_poll",
"**ofid_poll %s %d %s %s", __SHORT_FILE__,
__LINE__, __func__, fi_strerror(e.err));
MPIR_ERR_SETFATALANDJUMP2(mpi_errno, MPI_ERR_OTHER, "**ofid_poll",
"**ofid_poll %s %s",
MPIDI_OFI_DEFAULT_NIC_NAME, fi_strerror(e.err));
}

break;

default:
MPIR_ERR_SETFATALANDJUMP4(mpi_errno, MPI_ERR_OTHER, "**ofid_poll",
"**ofid_poll %s %d %s %s", __SHORT_FILE__, __LINE__,
__func__, fi_strerror(errno));
MPIR_ERR_SETFATALANDJUMP2(mpi_errno, MPI_ERR_OTHER, "**ofid_poll",
"**ofid_poll %s %s",
MPIDI_OFI_DEFAULT_NIC_NAME, fi_strerror(errno));
}

fn_exit:
Expand Down
42 changes: 17 additions & 25 deletions src/mpid/ch4/netmod/ofi/ofi_impl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@ ATTRIBUTE((unused));

#define MPIDI_OFI_WIN(win) ((win)->dev.netmod.ofi)

#define MPIDI_OFI_NIC_NAME(nic) (MPIDI_OFI_global.prov_use[nic] ? \
MPIDI_OFI_global.prov_use[nic]->domain_attr->name : "(n/a)")
#define MPIDI_OFI_DEFAULT_NIC_NAME (MPIDI_OFI_NIC_NAME(0))

int MPIDI_OFI_progress_uninlined(int vci);
int MPIDI_OFI_handle_cq_error(int vci, int nic, ssize_t ret);

Expand All @@ -55,18 +59,16 @@ int MPIDI_OFI_handle_cq_error(int vci, int nic, ssize_t ret);
#define MPIDI_OFI_PROGRESS_WHILE(cond, vci) \
while (cond) MPIDI_OFI_PROGRESS(vci)

#define MPIDI_OFI_ERR MPIR_ERR_CHKANDJUMP4
#define MPIDI_OFI_CALL(FUNC,STR) \
#define MPIDI_OFI_ERR MPIR_ERR_CHKANDJUMP2
#define MPIDI_OFI_CALL(FUNC,STR) \
do { \
ssize_t _ret = FUNC; \
MPIDI_OFI_ERR(_ret<0, \
mpi_errno, \
MPI_ERR_OTHER, \
"**ofid_"#STR, \
"**ofid_"#STR" %s %d %s %s", \
__SHORT_FILE__, \
__LINE__, \
__func__, \
"**ofid_"#STR" %s %s", \
MPIDI_OFI_DEFAULT_NIC_NAME, \
fi_strerror(-_ret)); \
} while (0)

Expand All @@ -81,10 +83,8 @@ int MPIDI_OFI_handle_cq_error(int vci, int nic, ssize_t ret);
mpi_errno, \
MPI_ERR_OTHER, \
"**ofid_"#STR, \
"**ofid_"#STR" %s %d %s %s", \
__SHORT_FILE__, \
__LINE__, \
__func__, \
"**ofid_"#STR" %s %s", \
MPIDI_OFI_DEFAULT_NIC_NAME, \
fi_strerror(-_ret)); \
if (_retry > 0) { \
_retry--; \
Expand All @@ -94,9 +94,7 @@ int MPIDI_OFI_handle_cq_error(int vci, int nic, ssize_t ret);
* for recursive locking in more than one lock (currently limited
* to one due to scalar TLS counter), this lock yielding
* operation can be avoided since we are inside a finite loop. */ \
MPIDI_OFI_THREAD_CS_EXIT_VCI_OPTIONAL(vci_); \
mpi_errno = MPIDI_OFI_retry_progress(); \
MPIDI_OFI_THREAD_CS_ENTER_VCI_OPTIONAL(vci_); \
mpi_errno = MPIDI_OFI_retry_progress(vci_, _retry); \
MPIR_ERR_CHECK(mpi_errno); \
} while (1); \
} while (0)
Expand All @@ -113,9 +111,7 @@ int MPIDI_OFI_handle_cq_error(int vci, int nic, ssize_t ret);
_retry--; \
MPIR_ERR_CHKANDJUMP(_retry == 0, mpi_errno, MPIX_ERR_EAGAIN, "**eagain"); \
} \
MPIDI_OFI_THREAD_CS_EXIT_VCI_OPTIONAL(vci_); \
mpi_errno = MPIDI_OFI_retry_progress(); \
MPIDI_OFI_THREAD_CS_ENTER_VCI_OPTIONAL(vci_); \
mpi_errno = MPIDI_OFI_retry_progress(vci_, _retry); \
} \
} while (0)

Expand All @@ -129,10 +125,8 @@ int MPIDI_OFI_handle_cq_error(int vci, int nic, ssize_t ret);
mpi_errno, \
MPI_ERR_OTHER, \
"**ofid_"#STR, \
"**ofid_"#STR" %s %d %s %s", \
__SHORT_FILE__, \
__LINE__, \
__func__, \
"**ofid_"#STR" %s %s", \
MPIDI_OFI_DEFAULT_NIC_NAME, \
fi_strerror(-_ret)); \
mpi_errno = MPIDI_OFI_progress_do_queue(vci_); \
if (mpi_errno != MPI_SUCCESS) \
Expand Down Expand Up @@ -176,10 +170,8 @@ int MPIDI_OFI_handle_cq_error(int vci, int nic, ssize_t ret);
mpi_errno, \
MPI_ERR_OTHER, \
"**ofid_"#STR, \
"**ofid_"#STR" %s %d %s %s", \
__SHORT_FILE__, \
__LINE__, \
__func__, \
"**ofid_"#STR" %s %s", \
MPIDI_OFI_DEFAULT_NIC_NAME, \
fi_strerror(-_ret)); \
} while (0)

Expand Down Expand Up @@ -299,7 +291,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_mr_bind(struct fi_info *prov, struct fid_
#define MPIDI_OFI_LOCAL_MR_KEY 0
#define MPIDI_OFI_COLL_MR_KEY 1
#define MPIDI_OFI_INVALID_MR_KEY 0xFFFFFFFFFFFFFFFFULL
int MPIDI_OFI_retry_progress(void);
int MPIDI_OFI_retry_progress(int vci, int retry);
int MPIDI_OFI_recv_huge_event(int vci, struct fi_cq_tagged_entry *wc, MPIR_Request * rreq);
int MPIDI_OFI_recv_huge_control(int vci, MPIR_Context_id_t comm_id, int rank, int tag,
MPIDI_OFI_huge_remote_info_t * info);
Expand Down
2 changes: 1 addition & 1 deletion src/mpid/ch4/netmod/ofi/ofi_send.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
- name : MPIR_CVAR_CH4_OFI_EAGER_THRESHOLD
category : CH4_OFI
type : int
default : 16384
default : -1
class : none
verbosity : MPI_T_VERBOSITY_USER_BASIC
scope : MPI_T_SCOPE_LOCAL
Expand Down
8 changes: 4 additions & 4 deletions src/mpid/ch4/netmod/ofi/ofi_spawn.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,8 @@ int MPIDI_OFI_dynamic_send(uint64_t remote_gpid, int tag, const void *buf, int s
int rc;
rc = fi_cancel((fid_t) MPIDI_OFI_global.ctx[ctx_idx].tx, (void *) &req.context);
if (rc && rc != -FI_ENOENT) {
MPIR_ERR_CHKANDJUMP4(rc < 0, mpi_errno, MPI_ERR_OTHER, "**ofid_cancel",
"**ofid_cancel %s %d %s %s", __SHORT_FILE__, __LINE__, __func__,
MPIR_ERR_CHKANDJUMP2(rc < 0, mpi_errno, MPI_ERR_OTHER, "**ofid_cancel",
"**ofid_cancel %s %s", MPIDI_OFI_DEFAULT_NIC_NAME,
fi_strerror(-rc));

}
Expand Down Expand Up @@ -112,8 +112,8 @@ int MPIDI_OFI_dynamic_recv(int tag, void *buf, int size, int timeout)
int rc;
rc = fi_cancel((fid_t) MPIDI_OFI_global.ctx[ctx_idx].rx, (void *) &req.context);
if (rc && rc != -FI_ENOENT) {
MPIR_ERR_CHKANDJUMP4(rc < 0, mpi_errno, MPI_ERR_OTHER, "**ofid_cancel",
"**ofid_cancel %s %d %s %s", __SHORT_FILE__, __LINE__, __func__,
MPIR_ERR_CHKANDJUMP2(rc < 0, mpi_errno, MPI_ERR_OTHER, "**ofid_cancel",
"**ofid_cancel %s %s", MPIDI_OFI_DEFAULT_NIC_NAME,
fi_strerror(-rc));

}
Expand Down
6 changes: 3 additions & 3 deletions src/mpid/ch4/netmod/ofi/ofi_win.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,10 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_win_do_progress(MPIR_Win * win, int vci)

if (itercount == 1000 && MPIDI_OFI_COUNTER_WAIT_OBJECTS) {
ret = fi_cntr_wait(MPIDI_OFI_WIN(win).cmpl_cntr, tcount, 0);
MPIR_ERR_CHKANDJUMP4(ret < 0 && ret != -FI_ETIMEDOUT,
MPIR_ERR_CHKANDJUMP2(ret < 0 && ret != -FI_ETIMEDOUT,
mpi_errno, MPI_ERR_RMA_RANGE,
"**ofid_cntr_wait", "**ofid_cntr_wait %s %d %s %s",
__SHORT_FILE__, __LINE__, __func__, fi_strerror(-ret));
"**ofid_cntr_wait", "**ofid_cntr_wait %s %s",
MPIDI_OFI_DEFAULT_NIC_NAME, fi_strerror(-ret));
itercount = 0;
DEBUG_PROGRESS_CHECK;
}
Expand Down
15 changes: 13 additions & 2 deletions src/mpid/ch4/netmod/ofi/util.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,23 @@
#include "ofi_impl.h"
#include "ofi_events.h"

int MPIDI_OFI_retry_progress(void)
int MPIDI_OFI_retry_progress(int vci, int retry)
{
/* We do not call progress on hooks form netmod level
* because it is not reentrant safe.
*/
return MPID_Progress_test(NULL);
int mpi_errno;
/* call global progress sparingly. I assume the netmod progress on its own
* will resolve most of the resource busy issue. Call global progress when
* that is not resolving. */
if ((retry & 0xff) == 0) {
MPIDI_OFI_THREAD_CS_EXIT_VCI_OPTIONAL(vci);
mpi_errno = MPID_Progress_test(NULL);
MPIDI_OFI_THREAD_CS_ENTER_VCI_OPTIONAL(vci);
} else {
mpi_errno = MPIDI_OFI_progress_uninlined(vci);
}
return mpi_errno;
}

typedef struct MPIDI_OFI_mr_key_allocator_t {
Expand Down

0 comments on commit 4f08552

Please sign in to comment.