Skip to content

Commit

Permalink
coll/inter: fix non standard ddt handling
Browse files Browse the repository at this point in the history 
 - correctly handle non zero lower bound ddt
 - correctly handle ddt with size > extent

Thanks Yuki Matsumoto for the report
  • Loading branch information
@ggouaillardet
ggouaillardet committed Jul 7, 2016
1 parent 488d037 commit 3e559a1
Show file tree
Hide file tree
Showing 8 changed files with 82 additions and 123 deletions.
22 changes: 9 additions & 13 deletions ompi/mca/coll/inter/coll_inter_allgather.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2010 University of Houston. All rights reserved.
* Copyright (c) 2015 Research Organization for Information Science
* Copyright (c) 2015-2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
Expand Down Expand Up @@ -49,26 +49,22 @@ mca_coll_inter_allgather_inter(const void *sbuf, int scount,
mca_coll_base_module_t *module)
{
int rank, root = 0, size, rsize, err;
char *ptmp = NULL;
ptrdiff_t slb, sextent, incr;
char *ptmp_free = NULL, *ptmp;
ptrdiff_t gap, span;
ompi_request_t *req[2];

rank = ompi_comm_rank(comm);
size = ompi_comm_size(comm->c_local_comm);
rsize = ompi_comm_remote_size(comm);

/* Perform the gather locally at the root */
err = ompi_datatype_get_extent(sdtype, &slb, &sextent);
if (OMPI_SUCCESS != err) {
return OMPI_ERROR;
}

if ( scount > 0 ) {
incr = sextent * scount;
ptmp = (char*)malloc(size * incr);
if (NULL == ptmp) {
span = opal_datatype_span(&sdtype->super, scount*size, &gap);
ptmp_free = (char*)malloc(span);
if (NULL == ptmp_free) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
ptmp = ptmp_free - gap;

err = comm->c_local_comm->c_coll.coll_gather(sbuf, scount, sdtype,
ptmp, scount, sdtype,
Expand Down Expand Up @@ -112,8 +108,8 @@ mca_coll_inter_allgather_inter(const void *sbuf, int scount,
}

exit:
if (NULL != ptmp) {
free(ptmp);
if (NULL != ptmp_free) {
free(ptmp_free);
}

return err;
Expand Down
33 changes: 12 additions & 21 deletions ompi/mca/coll/inter/coll_inter_allgatherv.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2010 University of Houston. All rights reserved.
* Copyright (c) 2015 Research Organization for Information Science
* Copyright (c) 2015-2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
Expand Down Expand Up @@ -49,10 +49,7 @@ mca_coll_inter_allgatherv_inter(const void *sbuf, int scount,
{
int i, rank, size, size_local, total=0, err;
int *count=NULL,*displace=NULL;
char *ptmp=NULL;
MPI_Aint incr;
MPI_Aint extent;
MPI_Aint lb;
char *ptmp_free=NULL, *ptmp;
ompi_datatype_t *ndtype = NULL;
ompi_request_t *req[2];

Expand Down Expand Up @@ -81,22 +78,19 @@ mca_coll_inter_allgatherv_inter(const void *sbuf, int scount,
for (i = 1; i < size_local; i++) {
displace[i] = displace[i-1] + count[i-1];
}
/* Perform the gatherv locally with the first process as root */
err = ompi_datatype_get_extent(sdtype, &lb, &extent);
if (OMPI_SUCCESS != err) {
err = OMPI_ERROR;
goto exit;
}
incr = 0;
total = 0;
for (i = 0; i < size_local; i++) {
incr = incr + extent*count[i];
total = total + count[i];
}
if ( incr > 0 ) {
ptmp = (char*)malloc(incr);
if (NULL == ptmp) {
if ( total > 0 ) {
ptrdiff_t gap, span;
span = opal_datatype_span(&sdtype->super, total, &gap);
ptmp_free = (char*)malloc(span);
if (NULL == ptmp_free) {
err = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
ptmp = ptmp_free - gap;
}
}
err = comm->c_local_comm->c_coll.coll_gatherv(sbuf, scount, sdtype,
Expand All @@ -111,9 +105,6 @@ mca_coll_inter_allgatherv_inter(const void *sbuf, int scount,
ompi_datatype_commit(&ndtype);

if (0 == rank) {
for (i = 0; i < size_local; i++) {
total = total + count[i];
}
/* Exchange data between roots */
err = MCA_PML_CALL(irecv(rbuf, 1, ndtype, 0,
MCA_COLL_BASE_TAG_ALLGATHERV, comm,
Expand Down Expand Up @@ -144,8 +135,8 @@ mca_coll_inter_allgatherv_inter(const void *sbuf, int scount,
if( NULL != ndtype ) {
ompi_datatype_destroy(&ndtype);
}
if (NULL != ptmp) {
free(ptmp);
if (NULL != ptmp_free) {
free(ptmp_free);
}
if (NULL != displace) {
free(displace);
Expand Down
15 changes: 6 additions & 9 deletions ompi/mca/coll/inter/coll_inter_allreduce.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2007 University of Houston. All rights reserved.
* Copyright (c) 2013 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2015 Research Organization for Information Science
* Copyright (c) 2013 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2015-2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
Expand Down Expand Up @@ -48,23 +48,20 @@ mca_coll_inter_allreduce_inter(const void *sbuf, void *rbuf, int count,
mca_coll_base_module_t *module)
{
int err, rank, root = 0;
ptrdiff_t lb, extent;
char *tmpbuf = NULL, *pml_buffer = NULL;
ompi_request_t *req[2];
ptrdiff_t gap, span;

rank = ompi_comm_rank(comm);

/* Perform the reduction locally */
err = ompi_datatype_get_extent(dtype, &lb, &extent);
if (OMPI_SUCCESS != err) {
return OMPI_ERROR;
}
span = opal_datatype_span(&dtype->super, count, &gap);

tmpbuf = (char *) malloc(count * extent);
tmpbuf = (char *) malloc(span);
if (NULL == tmpbuf) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
pml_buffer = tmpbuf - lb;
pml_buffer = tmpbuf - gap;

err = comm->c_local_comm->c_coll.coll_reduce(sbuf, pml_buffer, count,
dtype, op, root,
Expand Down
27 changes: 11 additions & 16 deletions ompi/mca/coll/inter/coll_inter_gather.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2007 University of Houston. All rights reserved.
* Copyright (c) 2015 Research Organization for Information Science
* Copyright (c) 2015-2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
Expand Down Expand Up @@ -47,11 +47,7 @@ mca_coll_inter_gather_inter(const void *sbuf, int scount,
{
int err;
int rank;
int size,size_local;
char *ptmp = NULL;
MPI_Aint incr;
MPI_Aint extent;
MPI_Aint lb;
int size;

size = ompi_comm_remote_size(comm);
rank = ompi_comm_rank(comm);
Expand All @@ -61,17 +57,18 @@ mca_coll_inter_gather_inter(const void *sbuf, int scount,
err = OMPI_SUCCESS;
} else if (MPI_ROOT != root) {
/* Perform the gather locally with the first process as root */
err = ompi_datatype_get_extent(sdtype, &lb, &extent);
if (OMPI_SUCCESS != err) {
return OMPI_ERROR;
}
char *ptmp_free = NULL, *ptmp;
int size_local;
ptrdiff_t gap, span;

incr = extent * scount;
size_local = ompi_comm_size(comm->c_local_comm);
ptmp = (char*)malloc(size_local * incr);
if (NULL == ptmp) {
span = opal_datatype_span(&sdtype->super, scount*size_local, &gap);

ptmp_free = (char*)malloc(span);
if (NULL == ptmp_free) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
ptmp = ptmp_free - gap;

err = comm->c_local_comm->c_coll.coll_gather(sbuf, scount, sdtype,
ptmp, scount, sdtype,
Expand All @@ -86,9 +83,7 @@ mca_coll_inter_gather_inter(const void *sbuf, int scount,
return err;
}
}
if (NULL != ptmp) {
free(ptmp);
}
free(ptmp_free);
} else {
/* I am the root, loop receiving the data. */
err = MCA_PML_CALL(recv(rbuf, rcount*size, rdtype, 0,
Expand Down
31 changes: 11 additions & 20 deletions ompi/mca/coll/inter/coll_inter_gatherv.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2010 University of Houston. All rights reserved.
* Copyright (c) 2015 Research Organization for Information Science
* Copyright (c) 2015-2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
Expand Down Expand Up @@ -46,10 +46,7 @@ mca_coll_inter_gatherv_inter(const void *sbuf, int scount,
{
int i, rank, size, size_local, total=0, err;
int *count=NULL, *displace=NULL;
char *ptmp=NULL;
MPI_Aint incr;
MPI_Aint extent;
MPI_Aint lb;
char *ptmp_free=NULL, *ptmp;
ompi_datatype_t *ndtype;

if (MPI_PROC_NULL == root) { /* do nothing */
Expand Down Expand Up @@ -92,21 +89,18 @@ mca_coll_inter_gatherv_inter(const void *sbuf, int scount,
displace[i] = displace[i-1] + count[i-1];
}
/* Perform the gatherv locally with the first process as root */
err = ompi_datatype_get_extent(sdtype, &lb, &extent);
if (OMPI_SUCCESS != err) {
err = OMPI_ERROR;
goto exit;
}
incr = 0;
for (i = 0; i < size_local; i++) {
incr = incr + extent*count[i];
total = total + count[i];
}
if ( incr > 0 ) {
ptmp = (char*)malloc(incr);
if (NULL == ptmp) {
if ( total > 0 ) {
ptrdiff_t gap, span;
span = opal_datatype_span(&sdtype->super, total, &gap);
ptmp_free = (char*)malloc(span);
if (NULL == ptmp_free) {
err = OMPI_ERR_OUT_OF_RESOURCE;
goto exit;
}
ptmp = ptmp_free - gap;
}
}
err = comm->c_local_comm->c_coll.coll_gatherv(sbuf, scount, sdtype,
Expand All @@ -118,18 +112,15 @@ mca_coll_inter_gatherv_inter(const void *sbuf, int scount,
}

if (0 == rank) {
for (i = 0; i < size_local; i++) {
total = total + count[i];
}
/* First process sends data to the root */
err = MCA_PML_CALL(send(ptmp, total, sdtype, root,
MCA_COLL_BASE_TAG_GATHERV,
MCA_PML_BASE_SEND_STANDARD, comm));
}

exit:
if (NULL != ptmp) {
free(ptmp);
if (NULL != ptmp_free) {
free(ptmp_free);
}
if (NULL != displace) {
free(displace);
Expand Down
18 changes: 9 additions & 9 deletions ompi/mca/coll/inter/coll_inter_reduce.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2007 University of Houston. All rights reserved.
* Copyright (c) 2013 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2015 Research Organization for Information Science
* Copyright (c) 2013 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2015-2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
Expand Down Expand Up @@ -47,9 +47,6 @@ mca_coll_inter_reduce_inter(const void *sbuf, void *rbuf, int count,
mca_coll_base_module_t *module)
{
int rank, err;
ptrdiff_t true_lb, true_extent, lb, extent;
char *free_buffer = NULL;
char *pml_buffer = NULL;

/* Initialize */
rank = ompi_comm_rank(comm);
Expand All @@ -58,15 +55,18 @@ mca_coll_inter_reduce_inter(const void *sbuf, void *rbuf, int count,
/* do nothing */
err = OMPI_SUCCESS;
} else if (MPI_ROOT != root) {
ptrdiff_t gap, span;
char *free_buffer = NULL;
char *pml_buffer = NULL;

/* Perform the reduce locally with the first process as root */
ompi_datatype_get_extent(dtype, &lb, &extent);
ompi_datatype_get_true_extent(dtype, &true_lb, &true_extent);
span = opal_datatype_span(&dtype->super, count, &gap);

free_buffer = (char*)malloc(true_extent + (count - 1) * extent);
free_buffer = (char*)malloc(span);
if (NULL == free_buffer) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
pml_buffer = free_buffer - true_lb;
pml_buffer = free_buffer - gap;

err = comm->c_local_comm->c_coll.coll_reduce(sbuf, pml_buffer, count,
dtype, op, 0, comm->c_local_comm,
Expand Down
24 changes: 11 additions & 13 deletions ompi/mca/coll/inter/coll_inter_scatter.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2008 University of Houston. All rights reserved.
* Copyright (c) 2015 Research Organization for Information Science
* Copyright (c) 2015-2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
Expand Down Expand Up @@ -44,9 +44,7 @@ mca_coll_inter_scatter_inter(const void *sbuf, int scount,
int root, struct ompi_communicator_t *comm,
mca_coll_base_module_t *module)
{
int rank, size, size_local, err;
char *ptmp = NULL;
ptrdiff_t lb, incr;
int rank, size, err;

/* Initialize */

Expand All @@ -58,18 +56,18 @@ mca_coll_inter_scatter_inter(const void *sbuf, int scount,
err = OMPI_SUCCESS;
} else if (MPI_ROOT != root) {
/* First process receives the data from root */
char *ptmp_free = NULL, *ptmp;
if(0 == rank) {
err = ompi_datatype_get_extent(rdtype, &lb, &incr);
if (OMPI_SUCCESS != err) {
return OMPI_ERROR;
}
int size_local;
ptrdiff_t gap, span;

incr *= rcount;
size_local = ompi_comm_size(comm->c_local_comm);
ptmp = (char*)malloc(size_local * incr);
if (NULL == ptmp) {
span = opal_datatype_span(&rdtype->super, rcount*size_local, &gap);
ptmp_free = malloc(span);
if (NULL == ptmp_free) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
ptmp = ptmp_free - gap;

err = MCA_PML_CALL(recv(ptmp, rcount*size_local, rdtype,
root, MCA_COLL_BASE_TAG_SCATTER,
Expand All @@ -83,8 +81,8 @@ mca_coll_inter_scatter_inter(const void *sbuf, int scount,
rbuf, rcount, rdtype,
0, comm->c_local_comm,
comm->c_local_comm->c_coll.coll_scatter_module);
if (NULL != ptmp) {
free(ptmp);
if (NULL != ptmp_free) {
free(ptmp_free);
}
} else {
/* Root sends data to the first process in the remote group */
Expand Down
Loading

0 comments on commit 3e559a1

Please sign in to comment.