From fcc32e3bc43af8f636bd0798f43c3fd8395f88da Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Sat, 23 May 2026 17:05:41 -0500 Subject: [PATCH] GIO, a new I/O driver --- .github/dependabot.yml | 8 + .github/workflows/mac_mpich.yml | 206 -- .github/workflows/mac_openmpi.yml | 208 -- .github/workflows/main.yml | 238 ++ .github/workflows/netcdf4_adios.yml | 262 ++ .github/workflows/ubuntu_mpich.yml | 215 -- .github/workflows/ubuntu_openmpi.yml | 213 -- .gitignore | 1 + .gitmodules | 3 + DEVELOPER_NOTES.md | 7 +- Makefile.am | 18 +- README.md | 5 +- RELEASE_NOTES | 50 +- SECURITY.md | 37 + benchmarks/C/Makefile.am | 44 +- benchmarks/C/aggregation.c | 120 +- benchmarks/C/netcdf_put_vara.c | 2 +- benchmarks/C/parallel_run.sh | 150 +- benchmarks/C/pnetcdf_put_vara.c | 36 +- benchmarks/C/write_block_read_column.c | 69 +- benchmarks/FLASH-IO/Makefile.am | 43 +- benchmarks/FLASH-IO/parallel_run.sh | 169 +- benchmarks/Makefile.am | 7 +- benchmarks/WRF-IO/Makefile.am | 14 +- benchmarks/WRF-IO/parallel_run.sh | 146 +- benchmarks/WRF-IO/wrf_io.c | 798 ++++- configure.ac | 212 +- examples/C/Makefile.am | 42 +- examples/C/block_cyclic.c | 8 +- examples/C/bput_varn_int64.c | 6 +- examples/C/bput_varn_uint.c | 6 +- examples/C/collective_write.c | 25 +- examples/C/column_wise.c | 10 +- examples/C/create_from_cdl.c | 9 +- examples/C/create_open.c | 4 +- examples/C/fill_mode.c | 8 +- examples/C/flexible_api.c | 8 +- examples/C/flexible_bottom.c | 6 +- examples/C/get_info.c | 4 +- examples/C/get_vara.c | 4 +- examples/C/ghost_cell.c | 6 +- examples/C/global_attributes.c | 10 +- examples/C/hints.c | 20 +- examples/C/i_varn_int64.c | 6 +- examples/C/mpi_subarray.c | 2 +- examples/C/mput.c | 4 +- examples/C/nonblocking_write.c | 52 +- examples/C/nonblocking_write_in_def.c | 16 +- examples/C/parallel_run.sh | 35 +- examples/C/pthread.c | 12 +- examples/C/put_vara.c | 4 +- examples/C/put_varn_float.c | 4 +- examples/C/put_varn_int.c | 4 +- examples/C/req_all.c | 6 +- examples/C/time_var.c | 14 +- examples/C/transpose.c | 6 +- examples/C/transpose2D.c | 8 +- examples/C/vard_bottom.c | 4 +- examples/C/vard_int.c | 6 +- examples/C/vard_mvars.c | 4 +- examples/CXX/Makefile.am | 40 +- examples/CXX/SimpleXyWr.cpp | 2 +- examples/CXX/block_cyclic.cpp | 4 +- examples/CXX/collective_write.cpp | 13 +- examples/CXX/column_wise.cpp | 2 +- examples/CXX/fill_mode.cpp | 4 +- examples/CXX/nonblocking_write.cpp | 17 +- examples/CXX/parallel_run.sh | 34 +- examples/CXX/transpose.cpp | 2 +- examples/F77/Makefile.am | 41 +- examples/F77/get_vara.f | 149 + examples/F77/nonblocking_write.f | 5 +- examples/F77/parallel_run.sh | 34 +- examples/F90/Makefile.am | 42 +- examples/F90/nonblocking_write.f90 | 5 +- examples/F90/parallel_run.sh | 34 +- examples/Makefile.am | 13 +- examples/README.md | 11 +- examples/adios/Makefile.am | 32 +- examples/adios/parallel_run.sh | 42 +- examples/adios/read_metadata.c | 6 +- examples/adios/read_var.c | 4 +- examples/adios/read_var_nb.c | 4 +- examples/burst_buffer/Makefile.am | 34 +- examples/burst_buffer/create_open.c | 2 +- examples/burst_buffer/nonblocking.c | 2 +- examples/burst_buffer/parallel_run.sh | 35 +- examples/parallel_run.sh | 218 ++ examples/tutorial/Makefile.am | 10 +- examples/tutorial/parallel_run.sh | 116 +- examples/tutorial/pnetcdf-write-buffered.c | 2 +- gio | 1 + m4/check_mpi.m4 | 5 +- m4/check_utils.m4 | 2 +- m4/utils.m4 | 25 +- sneak_peek.md | 77 +- src/binding/cxx/ncmpiAtt.cpp | 4 +- src/binding/cxx/ncmpiEnumType.cpp | 4 +- src/binding/cxx/ncmpiOpaqueType.cpp | 4 +- src/binding/cxx/ncmpiVlenType.cpp | 4 +- src/binding/f77/pnetcdf.inc.in | 2 + src/binding/f90/nfmpi_constants.fh.in | 2 + src/dispatchers/Makefile.am | 4 - src/dispatchers/attr_getput.m4 | 4 +- src/dispatchers/attribute.c | 2 +- src/dispatchers/cdl_header_parser.c | 36 +- src/dispatchers/error_codes.c | 327 +- src/dispatchers/file.c | 1459 ++++++-- src/dispatchers/var_getput.m4 | 87 +- src/dispatchers/variable.c | 22 +- src/drivers/common/Makefile.am | 12 +- src/drivers/common/convert_swap.m4 | 4 +- src/drivers/common/dtype_decode.c | 7 +- src/drivers/common/error_adios2nc.c | 6 +- src/drivers/common/error_gio2nc.c | 59 + src/drivers/common/error_mpi2nc.c | 6 +- src/drivers/common/error_posix2nc.c | 6 +- src/drivers/common/mem_alloc.c | 30 +- src/drivers/common/utils.c | 89 +- src/drivers/include/common.h | 9 +- src/drivers/nc4io/Makefile.am | 4 - src/drivers/nc4io/nc4io_driver.c | 2 - src/drivers/nc4io/nc4io_driver.h | 8 +- src/drivers/nc4io/nc4io_file.c | 46 +- src/drivers/nc4io/nc4io_var.c | 27 - src/drivers/ncadios/Makefile.am | 4 - src/drivers/ncadios/ncadios_bp2ncd.c | 4 +- src/drivers/ncadios/ncadios_driver.c | 2 - src/drivers/ncadios/ncadios_driver.h | 10 +- src/drivers/ncadios/ncadios_file.c | 44 +- src/drivers/ncadios/ncadios_lists.c | 2 +- src/drivers/ncadios/ncadios_misc.c | 14 +- src/drivers/ncadios/ncadios_var.c | 29 - src/drivers/ncbbio/Makefile.am | 4 - src/drivers/ncbbio/ncbbio_driver.c | 2 - src/drivers/ncbbio/ncbbio_driver.h | 16 +- src/drivers/ncbbio/ncbbio_file.c | 54 +- src/drivers/ncbbio/ncbbio_log.c | 58 +- src/drivers/ncbbio/ncbbio_log_flush.c | 26 +- src/drivers/ncbbio/ncbbio_log_put.c | 24 +- src/drivers/ncbbio/ncbbio_nonblocking.c | 2 +- src/drivers/ncbbio/ncbbio_var.c | 40 - src/drivers/ncfoo/Makefile.am | 4 - src/drivers/ncfoo/ncfoo_driver.c | 2 - src/drivers/ncfoo/ncfoo_driver.h | 8 +- src/drivers/ncfoo/ncfoo_file.c | 148 +- src/drivers/ncfoo/ncfoo_var.c | 41 - src/drivers/ncmpio/Makefile.am | 9 +- src/drivers/ncmpio/ncmpio_NC.h | 405 ++- src/drivers/ncmpio/ncmpio_attr.m4 | 8 +- src/drivers/ncmpio/ncmpio_close.c | 68 +- src/drivers/ncmpio/ncmpio_create.c | 798 ++++- src/drivers/ncmpio/ncmpio_dim.c | 2 +- src/drivers/ncmpio/ncmpio_driver.c | 2 - src/drivers/ncmpio/ncmpio_driver.h | 11 +- src/drivers/ncmpio/ncmpio_enddef.c | 935 ++--- src/drivers/ncmpio/ncmpio_file_io.c | 885 +++-- src/drivers/ncmpio/ncmpio_file_misc.c | 167 +- src/drivers/ncmpio/ncmpio_filetype.c | 710 ---- src/drivers/ncmpio/ncmpio_fill.c | 414 +-- src/drivers/ncmpio/ncmpio_fstype.c | 232 ++ src/drivers/ncmpio/ncmpio_getput.m4 | 274 +- src/drivers/ncmpio/ncmpio_header_get.c | 235 +- src/drivers/ncmpio/ncmpio_header_put.c | 173 +- src/drivers/ncmpio/ncmpio_i_getput.m4 | 34 +- src/drivers/ncmpio/ncmpio_i_varn.m4 | 57 +- src/drivers/ncmpio/ncmpio_intra_node.c | 3012 ++++++++++++----- src/drivers/ncmpio/ncmpio_open.c | 398 ++- src/drivers/ncmpio/ncmpio_subfile.c | 374 +- src/drivers/ncmpio/ncmpio_sync.c | 132 +- src/drivers/ncmpio/ncmpio_util.c | 923 +++-- src/drivers/ncmpio/ncmpio_var.c | 10 +- src/drivers/ncmpio/ncmpio_vard.c | 436 --- src/drivers/ncmpio/ncmpio_wait.c | 1753 +--------- src/include/dispatch.h | 42 +- src/include/pnc_debug.h | 34 +- src/include/pnetcdf.h.in | 8 +- src/libs/Makefile.am | 3 + src/packaging/pnetcdf.pc.in | 2 +- src/utils/Makefile.am | 2 +- src/utils/ncmpidiff/Makefile.am | 24 +- src/utils/ncmpidiff/cdfdiff.c | 30 +- src/utils/ncmpidiff/ncmpidiff.c | 1022 +----- src/utils/ncmpidiff/ncmpidiff_core.c | 972 ++++++ src/utils/ncmpidiff/ncmpidiff_core.h | 28 + src/utils/ncmpidiff/tst_file.nc | Bin 0 -> 800 bytes src/utils/ncmpidiff/xfail_runs.sh | 38 + src/utils/ncmpidump/ncmpidump.c | 45 +- src/utils/ncmpidump/ncmpidump.h | 5 + src/utils/ncmpidump/vardata.c | 16 +- src/utils/ncmpigen/genlib.c | 11 +- src/utils/ncmpigen/load.c | 10 +- src/utils/ncmpigen/ncmpigen.1 | 11 +- src/utils/ncmpigen/ncmpigentab.c | 8 +- src/utils/ncmpilogdump/ncmpilogdump.m4 | 2 +- src/utils/ncoffsets/ncoffsets.c | 103 +- src/utils/ncvalidator/Makefile.am | 10 +- src/utils/ncvalidator/ncvalidator.c | 397 ++- src/utils/ncvalidator/seq_runs.sh | 46 +- src/utils/ncvalidator/tst_open.c | 10 +- src/utils/ncvalidator/wrap_runs.sh | 52 - src/utils/pnetcdf-config.in | 22 +- test/C/Makefile.am | 41 +- test/C/parallel_run.sh | 79 +- test/C/pres_temp_4D.h | 37 - test/C/pres_temp_4D_rd.c | 229 -- test/C/pres_temp_4D_wr.c | 263 -- test/C/pres_temp_4D_wr_rd.c | 490 +++ test/C/seq_runs.sh | 59 +- test/CXX/Makefile.am | 31 +- test/CXX/nctst.cpp | 23 +- test/CXX/parallel_run.sh | 72 +- test/CXX/seq_runs.sh | 29 + test/CXX/test_classic.cpp | 17 +- test/CXX/wrap_runs.sh | 50 - test/F90/Makefile.am | 72 +- test/F90/f90tst_parallel.f90 | 49 +- test/F90/f90tst_parallel2.f90 | 46 +- test/F90/f90tst_parallel3.f90 | 50 +- test/F90/f90tst_parallel4.f90 | 48 +- test/F90/f90tst_vars.f90 | 50 +- test/F90/f90tst_vars2.f90 | 56 +- test/F90/f90tst_vars3.f90 | 57 +- test/F90/f90tst_vars4.f90 | 52 +- test/F90/parallel_run.sh | 77 +- test/F90/seq_runs.sh | 42 +- test/F90/test_attr_int64.f90 | 45 +- test/F90/test_fill.f90 | 59 +- test/F90/test_intent.f90 | 71 +- test/F90/tst_f90.f90 | 49 +- test/F90/tst_f90_cdf5.f90 | 51 +- test/F90/tst_io.f90 | 81 +- test/F90/tst_types2.f90 | 44 +- test/F90/wrap_runs.sh | 54 - test/Makefile.am | 2 + test/adios/Makefile.am | 21 +- test/adios/att.c | 4 +- test/adios/header.c | 2 +- test/adios/indep.c | 4 +- test/adios/ivar.c | 4 +- test/adios/ivarm.c | 4 +- test/adios/ivars.c | 4 +- test/adios/open.c | 4 +- test/adios/parallel_run.sh | 38 +- test/adios/seq_runs.sh | 33 + test/adios/var.c | 4 +- test/adios/varm.c | 4 +- test/adios/vars.c | 4 +- test/adios/wrap_runs.sh | 41 - test/burst_buffer/Makefile.am | 41 +- test/burst_buffer/bb_bsize.c | 206 +- test/burst_buffer/bb_hints.c | 167 +- test/burst_buffer/bb_many_reqs.c | 161 +- test/burst_buffer/bb_nonblocking.c | 157 +- test/burst_buffer/highdim.c | 201 +- test/burst_buffer/parallel_run.sh | 65 +- test/burst_buffer/seq_runs.sh | 30 + test/burst_buffer/varn.c | 182 +- test/burst_buffer/wrap_runs.sh | 38 - test/cdf_format/Makefile.am | 48 +- test/cdf_format/cdf_type.c | 86 +- test/cdf_format/dim_cdf12.c | 126 +- test/cdf_format/parallel_run.sh | 83 +- test/cdf_format/seq_runs.sh | 47 +- .../cdf_format/{test_cdf1.nc => test_cdf.nc1} | Bin .../cdf_format/{test_cdf2.nc => test_cdf.nc2} | Bin .../{test_netcdf4.nc => test_cdf.nc3} | Bin test/cdf_format/test_cdf.nc4 | Bin 0 -> 263 bytes .../cdf_format/{test_cdf5.nc => test_cdf.nc5} | Bin test/cdf_format/test_inq_format.c | 172 +- test/cdf_format/tst_corrupt.c | 16 +- test/cdf_format/tst_open_cdf5.c | 16 +- test/cdf_format/wrap_runs.sh | 52 - test/cdf_format/xfail_runs.sh | 2 +- test/cdl/Makefile.am | 30 +- test/cdl/parallel_run.sh | 52 +- test/cdl/seq_runs.sh | 33 + test/cdl/tst_cdl_hdr_parser.c | 118 +- test/cdl/wrap_runs.sh | 41 - test/common/Makefile.am | 14 +- test/common/testutils.c | 709 +++- test/common/testutils.h | 94 +- test/common/testutilsf.F90 | 116 +- test/fandc/Makefile.am | 23 +- test/fandc/csnap.c | 29 +- test/fandc/pnctest.c | 10 +- test/header/Makefile.am | 28 +- test/header/header_consistency.c | 131 +- test/header/parallel_run.sh | 71 +- test/header/seq_runs.sh | 45 +- test/largefile/Makefile.am | 55 +- test/largefile/bigrecords.f | 40 +- test/largefile/high_dim_var.c | 102 +- test/largefile/large_attr.c | 196 +- test/largefile/large_coalesce.c | 307 +- test/largefile/large_dims_vars_attrs.c | 106 +- test/largefile/large_files.c | 103 +- test/largefile/large_header.c | 89 +- test/largefile/large_reqs.c | 94 +- test/largefile/large_var.c | 162 +- test/largefile/parallel_run.sh | 50 +- test/largefile/seq_runs.sh | 28 +- test/largefile/tst_cdf5_begin.c | 83 +- test/largefile/tst_flarge.f90 | 38 +- test/largefile/tst_hash_large_ndims.c | 87 +- test/largefile/tst_hash_large_ngattrs.c | 91 +- test/largefile/tst_hash_large_nvars.c | 82 +- test/largefile/wrap_runs.sh | 38 - test/mpi/threads_open.c | 2 +- test/nc4/Makefile.am | 22 +- test/nc4/compressed.c | 16 +- test/nc4/interoperability_rd.m4 | 15 +- test/nc4/interoperability_wr.m4 | 15 +- test/nc4/noclobber.c | 16 +- test/nc4/notsupport.c | 24 +- test/nc4/parallel_run.sh | 29 +- test/nc4/pres_temp_4D.c | 23 +- test/nc4/put_get_all_kinds.m4 | 17 +- test/nc4/rd_compressed.c | 24 +- test/nc4/{wrap_runs.sh => seq_runs.sh} | 25 +- test/nc4/simple_xy.c | 26 +- test/nc4/tst_2_rec_dims.c | 22 +- test/nc4/tst_get_put_size.c | 17 +- test/nc4/tst_rec_vars.c | 25 +- test/nc4/tst_zero_req.c | 25 +- test/nc_test/Makefile.am | 27 +- test/nc_test/nc_test.c | 15 +- test/nc_test/seq_runs.sh | 2 +- test/nc_test/t_nc.c | 34 +- test/nc_test/test_get.m4 | 51 +- test/nc_test/test_iget.m4 | 138 +- test/nc_test/test_iput.m4 | 132 +- test/nc_test/test_put.m4 | 60 +- test/nc_test/test_read.m4 | 62 +- test/nc_test/test_write.m4 | 94 +- test/nc_test/tst_atts.c | 13 +- test/nc_test/tst_atts3.c | 50 +- test/nc_test/tst_misc.c | 11 +- test/nc_test/tst_names.c | 101 +- test/nc_test/tst_nofill.c | 22 +- test/nc_test/tst_norm.c | 24 +- test/nc_test/tst_small.c | 48 +- test/nc_test/util.c | 20 +- test/nc_test/wrap_runs.sh | 30 +- test/nf90_test/Makefile.am | 26 +- test/nf90_test/nf90_test.F90 | 36 +- test/nf90_test/seq_runs.sh | 2 +- test/nf90_test/test_get.m4 | 51 +- test/nf90_test/test_iget.m4 | 51 +- test/nf90_test/test_iput.m4 | 48 +- test/nf90_test/test_put.m4 | 48 +- test/nf_test/Makefile.am | 24 +- test/nf_test/nf_test.F | 33 +- test/nf_test/seq_runs.sh | 2 +- test/nf_test/test_get.m4 | 51 +- test/nf_test/test_iget.m4 | 51 +- test/nf_test/test_iput.m4 | 48 +- test/nf_test/test_put.m4 | 48 +- test/nonblocking/Makefile.am | 60 +- test/nonblocking/bput_varn.m4 | 456 ++- test/nonblocking/column_wise.m4 | 183 +- test/nonblocking/flexible_bput.c | 168 +- test/nonblocking/i_varn_indef.c | 341 +- test/nonblocking/i_varn_int64.c | 376 +- test/nonblocking/interleaved.c | 171 +- test/nonblocking/large_num_reqs.c | 95 +- test/nonblocking/mcoll_perf.c | 115 +- test/nonblocking/mcoll_testf.f90 | 39 +- test/nonblocking/mcoll_testf77.f | 39 +- test/nonblocking/parallel_run.sh | 81 +- test/nonblocking/req_all.c | 94 +- test/nonblocking/seq_runs.sh | 50 +- test/nonblocking/test_bput.c | 136 +- test/nonblocking/test_bputf.f90 | 32 +- test/nonblocking/test_bputf77.f | 32 +- test/nonblocking/wait_after_indep.c | 83 +- test/nonblocking/wrap_runs.sh | 54 - test/parallel_run.sh | 327 ++ test/subfile/Makefile.am | 29 +- test/subfile/parallel_run.sh | 83 +- test/subfile/seq_runs.sh | 64 +- test/subfile/test_subfile.c | 177 +- test/test_installed/README.md | 21 - test/test_installed/batch.sh.in | 29 +- test/test_installed/makefile | 27 +- test/testcases/Makefile.am | 175 +- test/testcases/add_var.c | 119 +- test/testcases/alignment_test.c | 251 +- test/testcases/attrf.f | 36 +- test/testcases/buftype_free.c | 192 +- test/testcases/buftype_freef.f | 36 +- test/testcases/check_striping.c | 125 +- test/testcases/check_type.c | 92 +- test/testcases/collective_error.c | 171 +- test/testcases/erange_fill.m4 | 409 ++- test/testcases/error_precedence.m4 | 430 ++- test/testcases/file_create_open.c | 131 +- test/testcases/flexible.c | 231 +- test/testcases/flexible2.c | 124 +- test/testcases/flexible_api.f | 39 +- test/testcases/flexible_large_count.c | 209 +- test/testcases/flexible_var.c | 115 +- test/testcases/flexible_varm.c | 182 +- test/testcases/inq_num_vars.c | 115 +- test/testcases/inq_num_varsf.f90 | 37 +- test/testcases/inq_recsize.c | 99 +- test/testcases/inq_recsizef.f90 | 35 +- test/testcases/iput_all_kinds.m4 | 459 ++- test/testcases/ivarn.c | 141 +- test/testcases/large_var_cdf5.c | 79 +- test/testcases/last_large_var.c | 133 +- test/testcases/mix_collectives.c | 119 +- test/testcases/modes.c | 180 +- test/testcases/ncmpi_vars_null_stride.c | 240 +- test/testcases/noclobber.c | 103 +- test/testcases/nonblocking.c | 287 +- test/testcases/null_args.m4 | 363 +- test/testcases/one_record.c | 101 +- test/testcases/parallel_run.sh | 121 +- test/testcases/profile.c | 2 +- test/testcases/put_all_kinds.m4 | 240 +- test/testcases/put_parameter.f | 36 +- test/testcases/record.c | 192 +- test/testcases/redef1.c | 152 +- test/testcases/scalar.c | 272 +- test/testcases/seq_runs.sh | 90 +- test/testcases/test_erange.c | 402 ++- test/testcases/test_fillvalue.c | 93 +- test/testcases/test_get_varn.c | 107 +- test/testcases/test_vard.c | 261 +- test/testcases/test_vard_multiple.c | 188 +- test/testcases/test_vard_rec.c | 126 +- test/testcases/test_vardf.F | 35 +- test/testcases/test_vardf90.f90 | 37 +- test/testcases/test_varm.c | 230 +- test/testcases/tst_data_move.c | 648 ++++ test/testcases/tst_def_var_fill.c | 188 +- test/testcases/tst_del_attr.c | 102 +- test/testcases/tst_dimsizes.c | 150 +- test/testcases/tst_free_comm.c | 106 +- test/testcases/tst_grow_data.c | 471 +++ test/testcases/tst_grow_header.c | 215 +- test/testcases/tst_info.c | 88 +- test/testcases/tst_inq_header_size.c | 277 ++ test/testcases/tst_max_var_dims.c | 102 +- test/testcases/tst_multi_redefine.c | 377 +++ test/testcases/tst_pthread.c | 217 +- test/testcases/tst_redefine.c | 666 ++-- test/testcases/tst_symlink.c | 93 +- test/testcases/tst_varn_var1.c | 241 ++ test/testcases/tst_vars_fill.m4 | 145 +- test/testcases/tst_version.c | 67 +- test/testcases/varn_contig.c | 102 +- test/testcases/varn_int.c | 220 +- test/testcases/varn_intf.f | 36 +- test/testcases/varn_real.f90 | 37 +- test/testcases/vectors.c | 81 +- test/testcases/wrap_runs.sh | 55 - 458 files changed, 28089 insertions(+), 20669 deletions(-) create mode 100644 .github/dependabot.yml delete mode 100644 .github/workflows/mac_mpich.yml delete mode 100644 .github/workflows/mac_openmpi.yml create mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/netcdf4_adios.yml delete mode 100644 .github/workflows/ubuntu_mpich.yml delete mode 100644 .github/workflows/ubuntu_openmpi.yml create mode 100644 .gitmodules create mode 100644 SECURITY.md create mode 100644 examples/F77/get_vara.f create mode 100755 examples/parallel_run.sh create mode 160000 gio create mode 100644 src/drivers/common/error_gio2nc.c delete mode 100644 src/drivers/ncmpio/ncmpio_filetype.c create mode 100644 src/drivers/ncmpio/ncmpio_fstype.c delete mode 100644 src/drivers/ncmpio/ncmpio_vard.c create mode 100644 src/utils/ncmpidiff/ncmpidiff_core.c create mode 100644 src/utils/ncmpidiff/ncmpidiff_core.h create mode 100644 src/utils/ncmpidiff/tst_file.nc create mode 100755 src/utils/ncmpidiff/xfail_runs.sh delete mode 100755 src/utils/ncvalidator/wrap_runs.sh delete mode 100644 test/C/pres_temp_4D.h delete mode 100644 test/C/pres_temp_4D_rd.c delete mode 100644 test/C/pres_temp_4D_wr.c create mode 100644 test/C/pres_temp_4D_wr_rd.c create mode 100755 test/CXX/seq_runs.sh delete mode 100755 test/CXX/wrap_runs.sh delete mode 100755 test/F90/wrap_runs.sh create mode 100755 test/adios/seq_runs.sh delete mode 100755 test/adios/wrap_runs.sh create mode 100755 test/burst_buffer/seq_runs.sh delete mode 100755 test/burst_buffer/wrap_runs.sh rename test/cdf_format/{test_cdf1.nc => test_cdf.nc1} (100%) rename test/cdf_format/{test_cdf2.nc => test_cdf.nc2} (100%) rename test/cdf_format/{test_netcdf4.nc => test_cdf.nc3} (100%) create mode 100644 test/cdf_format/test_cdf.nc4 rename test/cdf_format/{test_cdf5.nc => test_cdf.nc5} (100%) delete mode 100755 test/cdf_format/wrap_runs.sh create mode 100755 test/cdl/seq_runs.sh delete mode 100755 test/cdl/wrap_runs.sh delete mode 100755 test/largefile/wrap_runs.sh rename test/nc4/{wrap_runs.sh => seq_runs.sh} (54%) delete mode 100755 test/nonblocking/wrap_runs.sh create mode 100755 test/parallel_run.sh create mode 100644 test/testcases/tst_data_move.c create mode 100644 test/testcases/tst_grow_data.c create mode 100644 test/testcases/tst_inq_header_size.c create mode 100644 test/testcases/tst_multi_redefine.c create mode 100644 test/testcases/tst_varn_var1.c delete mode 100755 test/testcases/wrap_runs.sh diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..cdced8f1b1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,8 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + cooldown: + default-days: 7 \ No newline at end of file diff --git a/.github/workflows/mac_mpich.yml b/.github/workflows/mac_mpich.yml deleted file mode 100644 index 72ca9c9037..0000000000 --- a/.github/workflows/mac_mpich.yml +++ /dev/null @@ -1,206 +0,0 @@ -name: Mac OSX with MPICH - -on: - push: - branches: [ master, test_github_actions ] - paths-ignore: - - '**/*.md' - - '**/*.txt' - - '**/*.1' - - '**/*.jpg' - - '**/*.png' - - 'docs/*' - - 'test/test_installed/*' - pull_request: - branches: [ master, test_github_actions ] - paths-ignore: - - '**/*.md' - - '**/*.txt' - - '**/*.1' - - '**/*.jpg' - - '**/*.png' - - 'docs/*' - - 'test/test_installed/*' - -env: - MPICH_VERSION: 4.3.0 - AUTOCONF_VERSION: 2.71 - AUTOMAKE_VERSION: 1.17 - LIBTOOL_VERSION: 2.5.4 - M4_VERSION: 1.4.19 - -jobs: - build: - runs-on: macos-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v4 - - name: Set up dependencies - run: | - # brew install gcc - # which gcc - # gcc --version - # which gfortran - - name: Clean up git untracked files - run: | - git clean -fx - - name: Build GNU autotools - run: | - export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" - export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" - cd ${GITHUB_WORKSPACE} - wget -q https://ftp.gnu.org/gnu/m4/m4-${M4_VERSION}.tar.gz - gzip -dc m4-${M4_VERSION}.tar.gz | tar -xf - - cd m4-${M4_VERSION} - ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ - --silent - make -s -j 8 install > qout 2>&1 - make -s -j 8 distclean >> qout 2>&1 - cd ${GITHUB_WORKSPACE} - wget -q https://ftp.gnu.org/gnu/autoconf/autoconf-${AUTOCONF_VERSION}.tar.gz - gzip -dc autoconf-${AUTOCONF_VERSION}.tar.gz | tar -xf - - cd autoconf-${AUTOCONF_VERSION} - ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ - --silent - make -s -j 8 install > qout 2>&1 - make -s -j 8 distclean >> qout 2>&1 - cd ${GITHUB_WORKSPACE} - wget -q https://ftp.gnu.org/gnu/automake/automake-${AUTOMAKE_VERSION}.tar.gz - gzip -dc automake-${AUTOMAKE_VERSION}.tar.gz | tar -xf - - cd automake-${AUTOMAKE_VERSION} - ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ - --silent - make -s -j 8 install > qout 2>&1 - make -s -j 8 distclean >> qout 2>&1 - cd ${GITHUB_WORKSPACE} - wget -q https://ftp.gnu.org/gnu/libtool/libtool-${LIBTOOL_VERSION}.tar.gz - gzip -dc libtool-${LIBTOOL_VERSION}.tar.gz | tar -xf - - cd libtool-${LIBTOOL_VERSION} - ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ - --silent - make -s -j 8 install > qout 2>&1 - make -s -j 8 distclean >> qout 2>&1 - - name: Build MPICH - run: | - cd ${GITHUB_WORKSPACE} - rm -rf MPICH ; mkdir MPICH ; cd MPICH - wget -q https://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz - gzip -dc mpich-${MPICH_VERSION}.tar.gz | tar -xf - - cd mpich-${MPICH_VERSION} - ./configure --prefix=${GITHUB_WORKSPACE}/MPICH \ - --silent \ - --enable-romio \ - --with-file-system=ufs \ - --with-device=ch3:sock \ - --disable-fortran \ - CC=gcc - make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1 - make -s -j 8 distclean >> qout 2>&1 - - name: Build PnetCDF - run: | - cd ${GITHUB_WORKSPACE} - export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" - export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" - m4 --version - autoconf --version - automake --version - libtool --version - autoreconf -i - mkdir -p pnetcdf_output - ./configure --enable-option-checking=fatal \ - --enable-profiling \ - pnc_ac_debug=yes \ - --enable-burst_buffering \ - --enable-subfiling \ - --enable-thread-safe \ - --with-pthread \ - --disable-fortran \ - --with-mpi=${GITHUB_WORKSPACE}/MPICH \ - TESTOUTDIR=${GITHUB_WORKSPACE}/pnetcdf_output - make -j 8 tests - - name: Print config.log - if: ${{ always() }} - run: | - cat ${GITHUB_WORKSPACE}/config.log - - name: make check - run: | - cd ${GITHUB_WORKSPACE} - make check - - name: Print test log files - if: ${{ always() }} - run: | - cd ${GITHUB_WORKSPACE} - fname=`find src test examples benchmarks -type f -name "*.log"` - for f in $fname ; do \ - bname=`basename $f` ; \ - if test "x$bname" != xconfig.log ; then \ - echo "-------- dump $f ----------------------------" ; \ - cat $f ; \ - fi ; \ - done - - name: make ptests - run: | - cd ${GITHUB_WORKSPACE} - make ptests - - name: Build PnetCDF (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" - export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" - make distclean - rm -rf pnetcdf_output - mkdir -p pnetcdf_output - ./configure --disable-fortran \ - --with-mpi=${GITHUB_WORKSPACE}/MPICH \ - TESTOUTDIR=${GITHUB_WORKSPACE}/pnetcdf_output - make -j 8 tests - - name: Print config.log (default configuration) - if: ${{ always() }} - run: | - cat ${GITHUB_WORKSPACE}/config.log - - name: make check (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - make check - - name: Print test log files (default configuration) - if: ${{ always() }} - run: | - cd ${GITHUB_WORKSPACE} - fname=`find src test examples benchmarks -type f -name "*.log"` - for f in $fname ; do \ - bname=`basename $f` ; \ - if test "x$bname" != xconfig.log ; then \ - echo "-------- dump $f ----------------------------" ; \ - cat $f ; \ - fi ; \ - done - - name: make ptests (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - make ptests - - name: make distcheck - run: | - cd ${GITHUB_WORKSPACE} - make -j 8 distcheck DISTCHECK_CONFIGURE_FLAGS="--silent --with-mpi=${GITHUB_WORKSPACE}/MPICH" - - name: make install - run: | - cd ${GITHUB_WORKSPACE} - prefix_path=${GITHUB_WORKSPACE}/pnetcdf_install - echo "---- test make install prefix=${prefix_path}" - make install prefix=${prefix_path} - test/tst_install.sh ${prefix_path} - prefix_path="/pnetcdf_install" - destdir_path=${GITHUB_WORKSPACE}/inst - echo "---- test make install prefix=${prefix_path} DESTDIR=${destdir_path}" - make install prefix=${prefix_path} DESTDIR=${destdir_path} - test/tst_install.sh ${prefix_path} ${destdir_path} - - name: Cleanup - if: ${{ always() }} - run: | - cd ${GITHUB_WORKSPACE} - make -s distclean - rm -rf ${GITHUB_WORKSPACE}/pnetcdf_output - rm -rf ${GITHUB_WORKSPACE}/MPICH - rm -rf ${GITHUB_WORKSPACE}/pnetcdf_install - rm -rf ${GITHUB_WORKSPACE}/inst - diff --git a/.github/workflows/mac_openmpi.yml b/.github/workflows/mac_openmpi.yml deleted file mode 100644 index 65fcb10be2..0000000000 --- a/.github/workflows/mac_openmpi.yml +++ /dev/null @@ -1,208 +0,0 @@ -name: Mac OSX with OpenMPI - -on: - push: - branches: [ master, test_github_actions ] - paths-ignore: - - '**/*.md' - - '**/*.txt' - - '**/*.1' - - '**/*.jpg' - - '**/*.png' - - 'docs/*' - - 'test/test_installed/*' - pull_request: - branches: [ master, test_github_actions ] - paths-ignore: - - '**/*.md' - - '**/*.txt' - - '**/*.1' - - '**/*.jpg' - - '**/*.png' - - 'docs/*' - - 'test/test_installed/*' - -env: - OPENMPI_VERSION: 5.0.2 - AUTOCONF_VERSION: 2.71 - AUTOMAKE_VERSION: 1.17 - LIBTOOL_VERSION: 2.5.4 - M4_VERSION: 1.4.19 - -jobs: - build: - runs-on: macos-latest - timeout-minutes: 90 - steps: - - uses: actions/checkout@v4 - - name: Set up dependencies - run: | - # brew install gcc - # which gcc - # gcc --version - # which gfortran - - name: Clean up git untracked files - run: | - git clean -fx - - name: Build GNU autotools - run: | - export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" - export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" - cd ${GITHUB_WORKSPACE} - wget -q https://ftp.gnu.org/gnu/m4/m4-${M4_VERSION}.tar.gz - gzip -dc m4-${M4_VERSION}.tar.gz | tar -xf - - cd m4-${M4_VERSION} - ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ - --silent - make -s -j 8 install > qout 2>&1 - make -s -j 8 distclean >> qout 2>&1 - cd ${GITHUB_WORKSPACE} - wget -q https://ftp.gnu.org/gnu/autoconf/autoconf-${AUTOCONF_VERSION}.tar.gz - gzip -dc autoconf-${AUTOCONF_VERSION}.tar.gz | tar -xf - - cd autoconf-${AUTOCONF_VERSION} - ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ - --silent - make -s -j 8 install > qout 2>&1 - make -s -j 8 distclean >> qout 2>&1 - cd ${GITHUB_WORKSPACE} - wget -q https://ftp.gnu.org/gnu/automake/automake-${AUTOMAKE_VERSION}.tar.gz - gzip -dc automake-${AUTOMAKE_VERSION}.tar.gz | tar -xf - - cd automake-${AUTOMAKE_VERSION} - ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ - --silent - make -s -j 8 install > qout 2>&1 - make -s -j 8 distclean >> qout 2>&1 - cd ${GITHUB_WORKSPACE} - wget -q https://ftp.gnu.org/gnu/libtool/libtool-${LIBTOOL_VERSION}.tar.gz - gzip -dc libtool-${LIBTOOL_VERSION}.tar.gz | tar -xf - - cd libtool-${LIBTOOL_VERSION} - ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ - --silent - make -s -j 8 install > qout 2>&1 - make -s -j 8 distclean >> qout 2>&1 - - name: Build OPENMPI - run: | - cd ${GITHUB_WORKSPACE} - rm -rf OPENMPI ; mkdir OPENMPI ; cd OPENMPI - VER_MAJOR=${OPENMPI_VERSION%.*} - wget -q https://download.open-mpi.org/release/open-mpi/v${VER_MAJOR}/openmpi-${OPENMPI_VERSION}.tar.gz - gzip -dc openmpi-${OPENMPI_VERSION}.tar.gz | tar -xf - - cd openmpi-${OPENMPI_VERSION} - ./configure --prefix=${GITHUB_WORKSPACE}/OPENMPI \ - --silent \ - --with-io-romio-flags="--with-file-system=ufs" \ - --with-hwloc=internal \ - --with-pmix=internal \ - --with-libevent=internal \ - --disable-mpi-fortran \ - CC=gcc - make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1 - make -s -j 8 distclean >> qout 2>&1 - - name: Build PnetCDF - run: | - cd ${GITHUB_WORKSPACE} - export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" - export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" - m4 --version - autoconf --version - automake --version - libtool --version - autoreconf -i - mkdir -p pnetcdf_output - ./configure --enable-option-checking=fatal \ - --enable-profiling \ - pnc_ac_debug=yes \ - --enable-burst_buffering \ - --enable-subfiling \ - --enable-thread-safe \ - --with-pthread \ - --disable-fortran \ - --with-mpi=${GITHUB_WORKSPACE}/OPENMPI \ - TESTOUTDIR=${GITHUB_WORKSPACE}/pnetcdf_output - make -j 8 tests - - name: Print config.log - if: ${{ always() }} - run: | - cat ${GITHUB_WORKSPACE}/config.log - - name: make check - run: | - cd ${GITHUB_WORKSPACE} - make check - - name: Print test log files - if: ${{ always() }} - run: | - cd ${GITHUB_WORKSPACE} - fname=`find src test examples benchmarks -type f -name "*.log"` - for f in $fname ; do \ - bname=`basename $f` ; \ - if test "x$bname" != xconfig.log ; then \ - echo "-------- dump $f ----------------------------" ; \ - cat $f ; \ - fi ; \ - done - - name: make ptests - run: | - cd ${GITHUB_WORKSPACE} - make ptests - - name: Build PnetCDF (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" - export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" - make distclean - rm -rf pnetcdf_output - mkdir -p pnetcdf_output - ./configure --disable-fortran \ - --with-mpi=${GITHUB_WORKSPACE}/OPENMPI \ - TESTOUTDIR=${GITHUB_WORKSPACE}/pnetcdf_output - make -j 8 tests - - name: Print config.log (default configuration) - if: ${{ always() }} - run: | - cat ${GITHUB_WORKSPACE}/config.log - - name: make check (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - make check - - name: Print test log files (default configuration) - if: ${{ always() }} - run: | - cd ${GITHUB_WORKSPACE} - fname=`find src test examples benchmarks -type f -name "*.log"` - for f in $fname ; do \ - bname=`basename $f` ; \ - if test "x$bname" != xconfig.log ; then \ - echo "-------- dump $f ----------------------------" ; \ - cat $f ; \ - fi ; \ - done - - name: make ptests (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - make ptests - - name: make distcheck - run: | - cd ${GITHUB_WORKSPACE} - make -j 8 distcheck DISTCHECK_CONFIGURE_FLAGS="--silent --with-mpi=${GITHUB_WORKSPACE}/OPENMPI" - - name: make install - run: | - cd ${GITHUB_WORKSPACE} - prefix_path=${GITHUB_WORKSPACE}/pnetcdf_install - echo "---- test make install prefix=${prefix_path}" - make install prefix=${prefix_path} - test/tst_install.sh ${prefix_path} - prefix_path="/pnetcdf_install" - destdir_path=${GITHUB_WORKSPACE}/inst - echo "---- test make install prefix=${prefix_path} DESTDIR=${destdir_path}" - make install prefix=${prefix_path} DESTDIR=${destdir_path} - test/tst_install.sh ${prefix_path} ${destdir_path} - - name: Cleanup - if: ${{ always() }} - run: | - cd ${GITHUB_WORKSPACE} - make -s distclean - rm -rf ${GITHUB_WORKSPACE}/pnetcdf_output - rm -rf ${GITHUB_WORKSPACE}/OPENMPI - rm -rf ${GITHUB_WORKSPACE}/pnetcdf_install - rm -rf ${GITHUB_WORKSPACE}/inst - diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000000..5cb26ace56 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,238 @@ +# Test Combinations of Ubuntu/MacOSX, MPICH/OpenMPI, and UFS/Lustre +name: CI - OS and MPI + +permissions: + contents: read + +on: + push: + branches: [ master ] + paths-ignore: + - '**/*.md' + - '**/*.txt' + - '**/*.1' + - '**/*.jpg' + - '**/*.png' + - 'docs/*' + - 'test/test_installed/*' + pull_request: + branches: [ master ] + paths-ignore: + - '**/*.md' + - '**/*.txt' + - '**/*.1' + - '**/*.jpg' + - '**/*.png' + - 'docs/*' + - 'test/test_installed/*' + +env: + LIBTOOL_VERSION: 2.5.4 + MPICH_VERSION: 5.0.1 + OPENMPI_VERSION: 5.0.9 + +jobs: + build: + strategy: + fail-fast: false # This disables the default cancel-on-failure behavior + matrix: + os: [ ubuntu-latest ] # [ubuntu-latest, macos-latest] + mpi_vendor: [ MPICH ] # [ MPICH, OpenMPI ] + fstype: [ ufs, mimic_lustre ] + debug_mode: [ nodebug ] # [ debug, nodebug ] + + runs-on: ${{ matrix.os }} + timeout-minutes: 120 + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v4 + + - name: Install autotools + run: | + set -x + if test ${{ matrix.os }} == ubuntu-latest ; then + sudo apt-get update + sudo apt-get install autoconf + sudo apt-get install automake + sudo apt-get install m4 + + # sudo apt-get install libtool libtool-bin + wget -q https://ftp.gnu.org/gnu/libtool/libtool-${LIBTOOL_VERSION}.tar.gz + gzip -dc libtool-${LIBTOOL_VERSION}.tar.gz | tar -xf - + cd libtool-${LIBTOOL_VERSION} + ./configure --prefix=/usr --silent + sudo make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1 + sudo make -s LIBTOOLFLAGS=--silent V=1 -j 8 distclean >> qout 2>&1 + else + brew update + brew install coreutils + brew install autoconf + brew install automake + brew install libtool + export PATH="/opt/homebrew/opt/libtool/libexec/gnubin:${PATH}" + brew list m4 || brew install m4 + fi + which autoconf + autoconf --version + which automake + automake --version + which libtool + libtool --version + which m4 + m4 --version + + - name: Install MPI compiler vendor - ${{ matrix.mpi_vendor }} + run: | + set -x + if test "${{ matrix.os }}" = "macos-latest" ; then + # Must reinstall gcc to get gfortran installed + brew reinstall gcc + which gcc + gcc --version + which gfortran + gfortran --version + fi + if test "${{ matrix.mpi_vendor }}" = "MPICH" ; then + # MPICH versions older than 4.2.2 do not support the MPI large + # count feature. + echo "Install MPICH ${MPICH_VERSION} in ${GITHUB_WORKSPACE}/MPI" + wget -q https://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz + gzip -dc mpich-${MPICH_VERSION}.tar.gz | tar -xf - + cd mpich-${MPICH_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/MPI \ + --silent \ + --enable-romio \ + --with-file-system=ufs \ + --with-device=ch3:sock \ + --enable-fortran \ + CC=gcc FC=gfortran \ + FFLAGS=-fallow-argument-mismatch \ + FCFLAGS=-fallow-argument-mismatch + make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1 + make -s LIBTOOLFLAGS=--silent V=1 -j 8 distclean >> qout 2>&1 + else # OpenMPI + echo "Install OPENMPI ${OPENMPI_VERSION} in ${GITHUB_WORKSPACE}/MPI" + VER_MAJOR=${OPENMPI_VERSION%.*} + wget -q https://download.open-mpi.org/release/open-mpi/v${VER_MAJOR}/openmpi-${OPENMPI_VERSION}.tar.gz + gzip -dc openmpi-${OPENMPI_VERSION}.tar.gz | tar -xf - + cd openmpi-${OPENMPI_VERSION} + if test "${{ matrix.os }}" = "macos-latest" ; then + # When built on MacOS, additional configure options are required. + EXTRA_OPTS="--with-hwloc=internal --with-pmix=internal --with-libevent=internal" + fi + ./configure --prefix=${GITHUB_WORKSPACE}/MPI \ + --silent \ + --with-io-romio-flags="--with-file-system=ufs" \ + ${EXTRA_OPTS} \ + CC=gcc \ + FC=gfortran \ + FCFLAGS=-fallow-argument-mismatch + make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1 + make -s LIBTOOLFLAGS=--silent V=1 -j 8 distclean >> qout 2>&1 + fi + + - name: Clean up git untracked files + run: | + git clean -fx + + - name: Initialize submodules used by PnetCDF + run: | + set -x + git submodule update --init --recursive + continue-on-error: true # Job continues and stays "green" if this fails + + - name: Build PnetCDF - ${{ matrix.fstype }} + if: ${{ always() }} + run: | + set -x + cd ${GITHUB_WORKSPACE} + if test ${{ matrix.os }} == macos-latest ; then + export PATH="/opt/homebrew/opt/libtool/libexec/gnubin:${PATH}" + fi + which autoconf + autoconf --version + which automake + automake --version + which libtool + libtool --version + which m4 + m4 --version + autoreconf -i + mkdir -p ${GITHUB_WORKSPACE}/pnetcdf_output + if test ${{ matrix.fstype }} == mimic_lustre ; then + export MIMIC_LUSTRE=yes + fi + if test ${{ matrix.debug_mode }} == debug ; then + CONFIG_OPTS="--enable-debug" + else + CONFIG_OPTS="--disable-debug" + fi + MPI_RUN="${GITHUB_WORKSPACE}/MPI/bin/mpiexec -n NP" + if test "${{ matrix.mpi_vendor }}" = "OpenMPI" ; then + MPI_RUN="${GITHUB_WORKSPACE}/MPI/bin/mpirun --host localhost --oversubscribe -np NP" + fi + ./configure --prefix=${GITHUB_WORKSPACE}/PnetCDF \ + pnc_ac_debug=yes \ + --enable-burst_buffering \ + --enable-subfiling \ + --enable-thread-safe \ + --with-pthread \ + ${CONFIG_OPTS} \ + --with-mpi=${GITHUB_WORKSPACE}/MPI \ + TESTOUTDIR=${GITHUB_WORKSPACE}/pnetcdf_output \ + TESTMPIRUN="${MPI_RUN}" + make -s LIBTOOLFLAGS=--silent V=1 -j 8 tests + + - name: Print config.log + if: ${{ always() }} + run: | + cat ${GITHUB_WORKSPACE}/config.log + + - name: make check + run: | + cd ${GITHUB_WORKSPACE} + make -s LIBTOOLFLAGS=--silent V=1 check + + - name: Print test log files + if: ${{ always() }} + run: | + cd ${GITHUB_WORKSPACE} + fname=`find src test examples benchmarks -type f -name "*.log"` + for f in $fname ; do \ + bname=`basename $f` ; \ + if test "x$bname" != xconfig.log ; then \ + echo "" ; \ + echo "-------- dump $f ----------------------------" ; \ + cat $f ; \ + fi ; \ + done + + - name: make ptests + run: | + cd ${GITHUB_WORKSPACE} + make -s LIBTOOLFLAGS=--silent V=1 ptests + + - name: make distcheck + run: | + cd ${GITHUB_WORKSPACE} + make -j 8 distcheck DISTCHECK_CONFIGURE_FLAGS="--silent --with-mpi=${GITHUB_WORKSPACE}/MPI" + + - name: make install + run: | + set -x + cd ${GITHUB_WORKSPACE} + prefix_path=${GITHUB_WORKSPACE}/pnetcdf_install + echo "---- test make install prefix=${prefix_path}" + make -s LIBTOOLFLAGS=--silent V=1 install prefix=${prefix_path} + test/tst_install.sh ${prefix_path} + prefix_path="/pnetcdf_install" + destdir_path=${GITHUB_WORKSPACE}/inst + echo "---- test make install prefix=${prefix_path} DESTDIR=${destdir_path}" + make -s LIBTOOLFLAGS=--silent V=1 install prefix=${prefix_path} DESTDIR=${destdir_path} + test/tst_install.sh ${prefix_path} ${destdir_path} + + - name: Cleanup + if: ${{ always() }} + run: | + cd ${GITHUB_WORKSPACE} + make -s LIBTOOLFLAGS=--silent V=1 distclean diff --git a/.github/workflows/netcdf4_adios.yml b/.github/workflows/netcdf4_adios.yml new file mode 100644 index 0000000000..75d94051c9 --- /dev/null +++ b/.github/workflows/netcdf4_adios.yml @@ -0,0 +1,262 @@ +# Test NetCDF4 and ADIOS +name: CI - NetCDF4 and ADIOS + +permissions: + contents: read + +on: + push: + branches: [ master ] + paths-ignore: + - '**/*.md' + - '**/*.txt' + - '**/*.1' + - '**/*.jpg' + - '**/*.png' + - 'docs/*' + - 'test/test_installed/*' + pull_request: + branches: [ master ] + paths-ignore: + - '**/*.md' + - '**/*.txt' + - '**/*.1' + - '**/*.jpg' + - '**/*.png' + - 'docs/*' + - 'test/test_installed/*' + +env: + LIBTOOL_VERSION: 2.5.4 + MPICH_VERSION: 5.0.1 + HDF5_VERSION: 1.14.6 + NETCDF4_VERSION: 4.9.3 + ADIOS_VERSION: 1.13.1 + +jobs: + build: + strategy: + fail-fast: false # This disables the default cancel-on-failure behavior + matrix: + debug_mode: [ debug ] # [ debug, nodebug ] + externals: [ netcdf4 ] + # disable testing ADIOS as its URL is no longer valid + # externals: [ netcdf4, adios ] + + runs-on: ubuntu-latest + timeout-minutes: 120 + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v4 + + - name: Install autotools + run: | + set -x + sudo apt-get update + sudo apt-get install autoconf + sudo apt-get install automake + sudo apt-get install m4 + # sudo apt-get install libtool libtool-bin + wget -q https://ftp.gnu.org/gnu/libtool/libtool-${LIBTOOL_VERSION}.tar.gz + gzip -dc libtool-${LIBTOOL_VERSION}.tar.gz | tar -xf - + cd libtool-${LIBTOOL_VERSION} + ./configure --prefix=/usr --silent + sudo make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1 + sudo make -s LIBTOOLFLAGS=--silent V=1 -j 8 distclean >> qout 2>&1 + which autoconf + autoconf --version + which automake + automake --version + which libtool + libtool --version + which m4 + m4 --version + + - name: Build and install MPICH + id: build_mpich + run: | + # MPICH versions older than 4.2.2 do not support the MPI large + # count feature. + echo "Install MPICH ${MPICH_VERSION} in ${GITHUB_WORKSPACE}/MPI" + wget -q https://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz + gzip -dc mpich-${MPICH_VERSION}.tar.gz | tar -xf - + cd mpich-${MPICH_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/MPICH \ + --silent \ + --enable-romio \ + --with-file-system=ufs \ + --with-device=ch3:sock \ + --enable-fortran \ + CC=gcc FC=gfortran \ + FFLAGS=-fallow-argument-mismatch \ + FCFLAGS=-fallow-argument-mismatch + make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1 + make -s LIBTOOLFLAGS=--silent V=1 -j 8 distclean >> qout 2>&1 + + - name: Build and install HDF5 + id: build_hdf5 + # NetCDF4 requires HDF5 + if: steps.build_mpich.outcome == 'success' && matrix.externals == 'netcdf4' + run: | + set -x + cd ${GITHUB_WORKSPACE} + rm -rf HDF5 ; mkdir HDF5 ; cd HDF5 + curl -LO https://github.com/HDFGroup/hdf5/releases/download/hdf5_${HDF5_VERSION}/hdf5-${HDF5_VERSION}.tar.gz + tar -zxf hdf5-${HDF5_VERSION}.tar.gz + cd hdf5-${HDF5_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/HDF5 \ + --silent \ + --enable-hl \ + --enable-parallel \ + --enable-build-mode=production \ + --disable-doxygen-doc \ + --disable-doxygen-man \ + --disable-doxygen-html \ + --disable-tools \ + --disable-tests \ + --disable-fortran \ + --disable-cxx \ + CC=${GITHUB_WORKSPACE}/MPICH/bin/mpicc + make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1 + make -s distclean >> qout 2>&1 + + - name: Build and install NetCDF4 + id: build_netcdf4 + if: steps.build_hdf5.outcome == 'success' && matrix.externals == 'netcdf4' + run: | + set -x + cd ${GITHUB_WORKSPACE} + rm -rf NetCDF ; mkdir NetCDF ; cd NetCDF + curl -LO https://github.com/Unidata/netcdf-c/archive/refs/tags/v${NETCDF4_VERSION}.tar.gz + tar -zxf v${NETCDF4_VERSION}.tar.gz + cd netcdf-c-${NETCDF4_VERSION} + ./configure --prefix=${GITHUB_WORKSPACE}/NetCDF \ + --silent \ + --disable-doxygen \ + --disable-mmap \ + --disable-dap \ + --disable-nczarr \ + --disable-nczarr-filters \ + --disable-filter-testing \ + --disable-quantize \ + --disable-byterange \ + CC=${GITHUB_WORKSPACE}/MPICH/bin/mpicc \ + CPPFLAGS="-I${GITHUB_WORKSPACE}/HDF5/include" \ + LDFLAGS="-L${GITHUB_WORKSPACE}/HDF5/lib" \ + LIBS="-lhdf5" + make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1 + make -s distclean >> qout 2>&1 + + - name: Build and install ADIOS + if: steps.build_mpich.outcome == 'success' && matrix.externals == 'adios' + run: | + cd ${GITHUB_WORKSPACE} + export PATH="${GITHUB_WORKSPACE}/MPICH/bin:${PATH}" + wget -q https://users.nccs.gov/~pnorbert/adios-${ADIOS_VERSION}.tar.gz + gzip -dc adios-${ADIOS_VERSION}.tar.gz | tar -xf - + cd adios-${ADIOS_VERSION} + mkdir build && cd build + ../configure --prefix=${GITHUB_WORKSPACE}/ADIOS \ + --silent \ + --with-mpi=${GITHUB_WORKSPACE}/MPICH \ + --disable-fortran + make -j 8 >> qout 2>&1 + make -j 8 install >> qout 2>&1 + + - name: Initialize submodules used by PnetCDF + run: | + set -x + git submodule update --init --recursive + continue-on-error: true # Job continues and stays "green" if this fails + + - name: Build PnetCDF + id: build_pnetcdf + if: steps.build_mpich.outcome == 'success' + run: | + set -x + cd ${GITHUB_WORKSPACE} + which autoconf + autoconf --version + which automake + automake --version + which libtool + libtool --version + which m4 + m4 --version + autoreconf -i + mkdir -p ${GITHUB_WORKSPACE}/pnetcdf_output + if test ${{ matrix.debug_mode }} == debug ; then + CONFIG_OPTS="--enable-debug" + else + CONFIG_OPTS="--disable-debug" + fi + if test ${{ matrix.externals }} == netcdf4 ; then + CONFIG_OPTS+=" --with-netcdf4=${GITHUB_WORKSPACE}/NetCDF" + fi + if test ${{ matrix.externals }} == adios ; then + CONFIG_OPTS+=" --with-adios=${GITHUB_WORKSPACE}/ADIOS" + fi + ./configure --prefix=${GITHUB_WORKSPACE}/PnetCDF \ + pnc_ac_debug=yes \ + ${CONFIG_OPTS} \ + --with-mpi=${GITHUB_WORKSPACE}/MPICH \ + TESTOUTDIR=${GITHUB_WORKSPACE}/pnetcdf_output + make -s LIBTOOLFLAGS=--silent V=1 -j 8 tests + + - name: Print config.log + if: ${{ always() }} + run: | + cat ${GITHUB_WORKSPACE}/config.log + + - name: make check + if: steps.build_pnetcdf.outcome == 'success' + run: | + cd ${GITHUB_WORKSPACE} + make -s LIBTOOLFLAGS=--silent V=1 check + + - name: Print test log files + if: ${{ always() }} + run: | + cd ${GITHUB_WORKSPACE} + fname=`find src test examples benchmarks -type f -name "*.log"` + for f in $fname ; do \ + bname=`basename $f` ; \ + if test "x$bname" != xconfig.log ; then \ + echo "" ; \ + echo "-------- dump $f ----------------------------" ; \ + cat $f ; \ + fi ; \ + done + + - name: make ptests + if: steps.build_pnetcdf.outcome == 'success' + run: | + cd ${GITHUB_WORKSPACE} + make -s LIBTOOLFLAGS=--silent V=1 ptests + + - name: make distcheck + if: steps.build_pnetcdf.outcome == 'success' + run: | + cd ${GITHUB_WORKSPACE} + make -j 8 distcheck DISTCHECK_CONFIGURE_FLAGS="--silent --with-mpi=${GITHUB_WORKSPACE}/MPICH" + + - name: make install + if: steps.build_pnetcdf.outcome == 'success' + run: | + set -x + cd ${GITHUB_WORKSPACE} + prefix_path=${GITHUB_WORKSPACE}/pnetcdf_install + echo "---- test make install prefix=${prefix_path}" + make -s LIBTOOLFLAGS=--silent V=1 install prefix=${prefix_path} + test/tst_install.sh ${prefix_path} + prefix_path="/pnetcdf_install" + destdir_path=${GITHUB_WORKSPACE}/inst + echo "---- test make install prefix=${prefix_path} DESTDIR=${destdir_path}" + make -s LIBTOOLFLAGS=--silent V=1 install prefix=${prefix_path} DESTDIR=${destdir_path} + test/tst_install.sh ${prefix_path} ${destdir_path} + + - name: Cleanup + if: ${{ always() }} + run: | + cd ${GITHUB_WORKSPACE} + make -s LIBTOOLFLAGS=--silent V=1 distclean diff --git a/.github/workflows/ubuntu_mpich.yml b/.github/workflows/ubuntu_mpich.yml deleted file mode 100644 index 09d626f8f1..0000000000 --- a/.github/workflows/ubuntu_mpich.yml +++ /dev/null @@ -1,215 +0,0 @@ -name: ubuntu_mpich - -on: - push: - branches: master - paths-ignore: - - '**/*.md' - - '**/*.txt' - - '**/*.1' - - 'docs/*' - - 'test/test_installed/*' - pull_request: - branches: master - paths-ignore: - - '**/*.md' - - '**/*.txt' - - '**/*.1' - - 'docs/*' - - 'test/test_installed/*' - -env: - MPICH_VERSION: 4.3.0 - AUTOCONF_VERSION: 2.71 - AUTOMAKE_VERSION: 1.17 - LIBTOOL_VERSION: 2.5.4 - M4_VERSION: 1.4.19 - -jobs: - build: - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v4 - - name: Set up dependencies - run: | - sudo apt-get update - # install gfortran - version=12 - sudo add-apt-repository ppa:ubuntu-toolchain-r/test - sudo apt-get update - sudo apt-get install -y gcc-${version} gfortran-${version} - sudo update-alternatives \ - --install /usr/bin/gcc gcc /usr/bin/gcc-${version} 100 \ - --slave /usr/bin/gfortran gfortran /usr/bin/gfortran-${version} \ - --slave /usr/bin/gcov gcov /usr/bin/gcov-${version} - echo "---- gcc/gfortran version ------------------------------" - which gcc - which gfortran - gcc --version - gfortran --version - - name: Clean up git untracked files - run: | - git clean -fx - - name: Build GNU autotools - run: | - export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" - export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" - cd ${GITHUB_WORKSPACE} - wget -q https://ftp.gnu.org/gnu/m4/m4-${M4_VERSION}.tar.gz - gzip -dc m4-${M4_VERSION}.tar.gz | tar -xf - - cd m4-${M4_VERSION} - ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ - --silent - make -s -j 8 install > qout 2>&1 - make -s -j 8 distclean >> qout 2>&1 - cd ${GITHUB_WORKSPACE} - wget -q https://ftp.gnu.org/gnu/autoconf/autoconf-${AUTOCONF_VERSION}.tar.gz - gzip -dc autoconf-${AUTOCONF_VERSION}.tar.gz | tar -xf - - cd autoconf-${AUTOCONF_VERSION} - ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ - --silent - make -s -j 8 install > qout 2>&1 - make -s -j 8 distclean >> qout 2>&1 - cd ${GITHUB_WORKSPACE} - wget -q https://ftp.gnu.org/gnu/automake/automake-${AUTOMAKE_VERSION}.tar.gz - gzip -dc automake-${AUTOMAKE_VERSION}.tar.gz | tar -xf - - cd automake-${AUTOMAKE_VERSION} - ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ - --silent - make -s -j 8 install > qout 2>&1 - make -s -j 8 distclean >> qout 2>&1 - cd ${GITHUB_WORKSPACE} - wget -q https://ftp.gnu.org/gnu/libtool/libtool-${LIBTOOL_VERSION}.tar.gz - gzip -dc libtool-${LIBTOOL_VERSION}.tar.gz | tar -xf - - cd libtool-${LIBTOOL_VERSION} - ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ - --silent - make -s -j 8 install > qout 2>&1 - make -s -j 8 distclean >> qout 2>&1 - - name: Build MPICH - run: | - cd ${GITHUB_WORKSPACE} - echo "Install MPICH ${MPICH_VERSION} in ${GITHUB_WORKSPACE}/MPICH" - rm -rf MPICH ; mkdir MPICH ; cd MPICH - # git clone -q https://github.com/pmodels/mpich.git - # cd mpich - # git submodule update --init - # ./autogen.sh - wget -q https://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz - gzip -dc mpich-${MPICH_VERSION}.tar.gz | tar -xf - - cd mpich-${MPICH_VERSION} - ./configure --prefix=${GITHUB_WORKSPACE}/MPICH \ - --silent \ - --enable-romio \ - --with-file-system=ufs \ - --with-device=ch3:sock \ - --enable-fortran \ - CC=gcc FC=gfortran \ - FFLAGS=-fallow-argument-mismatch \ - FCFLAGS=-fallow-argument-mismatch - make -s LIBTOOLFLAGS=--silent V=1 -j 4 install > qout 2>&1 - make -s -j 4 distclean >> qout 2>&1 - - name: Build PnetCDF - run: | - cd ${GITHUB_WORKSPACE} - export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" - export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" - m4 --version - autoconf --version - automake --version - libtool --version - autoreconf -i - ./configure --prefix=${GITHUB_WORKSPACE}/PnetCDF \ - --enable-option-checking=fatal \ - --enable-profiling \ - pnc_ac_debug=yes \ - --enable-burst_buffering \ - --enable-subfiling \ - --enable-thread-safe \ - --with-pthread \ - --with-mpi=${GITHUB_WORKSPACE}/MPICH - make -j 8 tests - - name: Print config.log - if: ${{ always() }} - run: | - cat ${GITHUB_WORKSPACE}/config.log - - name: make check - run: | - cd ${GITHUB_WORKSPACE} - make check - - name: Print test log files - if: ${{ always() }} - run: | - cd ${GITHUB_WORKSPACE} - fname=`find src test examples benchmarks -type f -name "*.log"` - for f in $fname ; do \ - bname=`basename $f` ; \ - if test "x$bname" != xconfig.log ; then \ - echo "-------- dump $f ----------------------------" ; \ - cat $f ; \ - fi ; \ - done - - name: make ptests - run: | - cd ${GITHUB_WORKSPACE} - make ptests - - name: Build PnetCDF (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" - export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" - make distclean - ./configure --prefix=${GITHUB_WORKSPACE}/PnetCDF \ - --with-mpi=${GITHUB_WORKSPACE}/MPICH - make -j 8 tests - - name: Print config.log (default configuration) - if: ${{ always() }} - run: | - cat ${GITHUB_WORKSPACE}/config.log - - name: make check (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - make check - - name: Print test log files (default configuration) - if: ${{ always() }} - run: | - cd ${GITHUB_WORKSPACE} - fname=`find src test examples benchmarks -type f -name "*.log"` - for f in $fname ; do \ - bname=`basename $f` ; \ - if test "x$bname" != xconfig.log ; then \ - echo "-------- dump $f ----------------------------" ; \ - cat $f ; \ - fi ; \ - done - - name: make ptests (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - make ptests - - name: make distcheck - run: | - cd ${GITHUB_WORKSPACE} - make -j 8 distcheck DISTCHECK_CONFIGURE_FLAGS="--silent --with-mpi=${GITHUB_WORKSPACE}/MPICH" - - name: make install - run: | - cd ${GITHUB_WORKSPACE} - prefix_path=${GITHUB_WORKSPACE}/pnetcdf_install - echo "---- test make install prefix=${prefix_path}" - make install prefix=${prefix_path} - test/tst_install.sh ${prefix_path} - prefix_path="/pnetcdf_install" - destdir_path=${GITHUB_WORKSPACE}/inst - echo "---- test make install prefix=${prefix_path} DESTDIR=${destdir_path}" - make install prefix=${prefix_path} DESTDIR=${destdir_path} - test/tst_install.sh ${prefix_path} ${destdir_path} - - name: Cleanup - if: ${{ always() }} - run: | - cd ${GITHUB_WORKSPACE} - make -s distclean - rm -rf ${GITHUB_WORKSPACE}/pnetcdf_output - rm -rf ${GITHUB_WORKSPACE}/MPICH - rm -rf ${GITHUB_WORKSPACE}/pnetcdf_install - rm -rf ${GITHUB_WORKSPACE}/inst - diff --git a/.github/workflows/ubuntu_openmpi.yml b/.github/workflows/ubuntu_openmpi.yml deleted file mode 100644 index 80f087295e..0000000000 --- a/.github/workflows/ubuntu_openmpi.yml +++ /dev/null @@ -1,213 +0,0 @@ -name: ubuntu_openmpi - -on: - push: - branches: master - paths-ignore: - - '**/*.md' - - '**/*.txt' - - '**/*.1' - - 'docs/*' - - 'test/test_installed/*' - pull_request: - branches: master - paths-ignore: - - '**/*.md' - - '**/*.txt' - - '**/*.1' - - 'docs/*' - - 'test/test_installed/*' - -env: - OPENMPI_VERSION: 5.0.2 - AUTOCONF_VERSION: 2.71 - AUTOMAKE_VERSION: 1.17 - LIBTOOL_VERSION: 2.5.4 - M4_VERSION: 1.4.19 - -jobs: - build: - runs-on: ubuntu-latest - timeout-minutes: 90 - steps: - - uses: actions/checkout@v4 - - name: Set up dependencies - run: | - sudo apt-get update - # install gfortran - version=12 - sudo add-apt-repository ppa:ubuntu-toolchain-r/test - sudo apt-get update - sudo apt-get install -y gcc-${version} gfortran-${version} - sudo update-alternatives \ - --install /usr/bin/gcc gcc /usr/bin/gcc-${version} 100 \ - --slave /usr/bin/gfortran gfortran /usr/bin/gfortran-${version} \ - --slave /usr/bin/gcov gcov /usr/bin/gcov-${version} - echo "---- gcc/gfortran version ------------------------------" - which gcc - which gfortran - gcc --version - gfortran --version - - name: Clean up git untracked files - run: | - git clean -fx - - name: Build GNU autotools - run: | - export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" - export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" - cd ${GITHUB_WORKSPACE} - wget -q https://ftp.gnu.org/gnu/m4/m4-${M4_VERSION}.tar.gz - gzip -dc m4-${M4_VERSION}.tar.gz | tar -xf - - cd m4-${M4_VERSION} - ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ - --silent - make -s -j 8 install > qout 2>&1 - make -s -j 8 distclean >> qout 2>&1 - cd ${GITHUB_WORKSPACE} - wget -q https://ftp.gnu.org/gnu/autoconf/autoconf-${AUTOCONF_VERSION}.tar.gz - gzip -dc autoconf-${AUTOCONF_VERSION}.tar.gz | tar -xf - - cd autoconf-${AUTOCONF_VERSION} - ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ - --silent - make -s -j 8 install > qout 2>&1 - make -s -j 8 distclean >> qout 2>&1 - cd ${GITHUB_WORKSPACE} - wget -q https://ftp.gnu.org/gnu/automake/automake-${AUTOMAKE_VERSION}.tar.gz - gzip -dc automake-${AUTOMAKE_VERSION}.tar.gz | tar -xf - - cd automake-${AUTOMAKE_VERSION} - ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ - --silent - make -s -j 8 install > qout 2>&1 - make -s -j 8 distclean >> qout 2>&1 - cd ${GITHUB_WORKSPACE} - wget -q https://ftp.gnu.org/gnu/libtool/libtool-${LIBTOOL_VERSION}.tar.gz - gzip -dc libtool-${LIBTOOL_VERSION}.tar.gz | tar -xf - - cd libtool-${LIBTOOL_VERSION} - ./configure --prefix=${GITHUB_WORKSPACE}/AUTOTOOLS \ - --silent - make -s -j 8 install > qout 2>&1 - make -s -j 8 distclean >> qout 2>&1 - - name: Build OPENMPI - run: | - cd ${GITHUB_WORKSPACE} - echo "Install OPENMPI ${OPENMPI_VERSION} in ${GITHUB_WORKSPACE}/OPENMPI" - rm -rf OPENMPI ; mkdir OPENMPI ; cd OPENMPI - VER_MAJOR=${OPENMPI_VERSION%.*} - wget -q https://download.open-mpi.org/release/open-mpi/v${VER_MAJOR}/openmpi-${OPENMPI_VERSION}.tar.gz - gzip -dc openmpi-${OPENMPI_VERSION}.tar.gz | tar -xf - - cd openmpi-${OPENMPI_VERSION} - ./configure --prefix=${GITHUB_WORKSPACE}/OPENMPI \ - --silent \ - --with-io-romio-flags="--with-file-system=ufs" \ - CC=gcc \ - FC=gfortran \ - FCFLAGS=-fallow-argument-mismatch - make -s LIBTOOLFLAGS=--silent V=1 -j 4 install > qout 2>&1 - make -s -j 4 distclean >> qout 2>&1 - - name: Build PnetCDF - run: | - cd ${GITHUB_WORKSPACE} - export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" - export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" - m4 --version - autoconf --version - automake --version - libtool --version - autoreconf -i - mkdir -p pnetcdf_output - ./configure --prefix=${GITHUB_WORKSPACE}/PnetCDF \ - --enable-option-checking=fatal \ - --enable-profiling \ - pnc_ac_debug=yes \ - --enable-burst_buffering \ - --enable-subfiling \ - --enable-thread-safe \ - --with-pthread \ - --with-mpi=${GITHUB_WORKSPACE}/OPENMPI \ - TESTOUTDIR=${GITHUB_WORKSPACE}/pnetcdf_output - make -j 8 tests - - name: Print config.log - if: ${{ always() }} - run: | - cat ${GITHUB_WORKSPACE}/config.log - - name: make check - run: | - cd ${GITHUB_WORKSPACE} - make check - - name: Print test log files - if: ${{ always() }} - run: | - cd ${GITHUB_WORKSPACE} - fname=`find src test examples benchmarks -type f -name "*.log"` - for f in $fname ; do \ - bname=`basename $f` ; \ - if test "x$bname" != xconfig.log ; then \ - echo "-------- dump $f ----------------------------" ; \ - cat $f ; \ - fi ; \ - done - - name: make ptests - run: | - cd ${GITHUB_WORKSPACE} - make ptests - - name: Build PnetCDF (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - export PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/bin:${PATH}" - export LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/AUTOTOOLS/lib:${LD_LIBRARY_PATH}" - make distclean - mkdir -p pnetcdf_output - ./configure --prefix=${GITHUB_WORKSPACE}/PnetCDF \ - --with-mpi=${GITHUB_WORKSPACE}/OPENMPI \ - TESTOUTDIR=${GITHUB_WORKSPACE}/pnetcdf_output - make -j 8 tests - - name: Print config.log (default configuration) - if: ${{ always() }} - run: | - cat ${GITHUB_WORKSPACE}/config.log - - name: make check (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - make check - - name: Print test log files (default configuration) - if: ${{ always() }} - run: | - cd ${GITHUB_WORKSPACE} - fname=`find src test examples benchmarks -type f -name "*.log"` - for f in $fname ; do \ - bname=`basename $f` ; \ - if test "x$bname" != xconfig.log ; then \ - echo "-------- dump $f ----------------------------" ; \ - cat $f ; \ - fi ; \ - done - - name: make ptests (default configuration) - run: | - cd ${GITHUB_WORKSPACE} - make ptests - - name: make distcheck - run: | - cd ${GITHUB_WORKSPACE} - make -j 8 distcheck DISTCHECK_CONFIGURE_FLAGS="--silent --with-mpi=${GITHUB_WORKSPACE}/OPENMPI" - - name: make install - run: | - cd ${GITHUB_WORKSPACE} - prefix_path=${GITHUB_WORKSPACE}/pnetcdf_install - echo "---- test make install prefix=${prefix_path}" - make install prefix=${prefix_path} - test/tst_install.sh ${prefix_path} - prefix_path="/pnetcdf_install" - destdir_path=${GITHUB_WORKSPACE}/inst - echo "---- test make install prefix=${prefix_path} DESTDIR=${destdir_path}" - make install prefix=${prefix_path} DESTDIR=${destdir_path} - test/tst_install.sh ${prefix_path} ${destdir_path} - - name: Cleanup - if: ${{ always() }} - run: | - cd ${GITHUB_WORKSPACE} - make -s distclean - rm -rf ${GITHUB_WORKSPACE}/pnetcdf_output - rm -rf ${GITHUB_WORKSPACE}/OPENMPI - rm -rf ${GITHUB_WORKSPACE}/pnetcdf_install - rm -rf ${GITHUB_WORKSPACE}/inst - diff --git a/.gitignore b/.gitignore index 48692018eb..546a6fc421 100644 --- a/.gitignore +++ b/.gitignore @@ -83,6 +83,7 @@ Makefile *.nc1 *.nc2 *.nc5 +!/src/utils/ncmpidiff/tst_file.nc !/src/utils/ncvalidator/bad_*.nc !/src/utils/ncvalidator/bad_*.nc? !/test/cdf_format/bad_*.nc? diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..6a179c9a74 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "gio"] + path = gio + url = git@github.com:wkliao/gio.git diff --git a/DEVELOPER_NOTES.md b/DEVELOPER_NOTES.md index f1b3693a6c..0ca204b832 100644 --- a/DEVELOPER_NOTES.md +++ b/DEVELOPER_NOTES.md @@ -149,12 +149,17 @@ 10. Generate SHA1 checksums * Run command: ``` - openssl sha1 pnetcdf-1.11.0.tar.gz` + openssl sha1 pnetcdf-1.11.0.tar.gz ``` * Example command-line output: ``` SHA1(pnetcdf-1.11.0.tar.gz)= 495d42f0a41abbd09d276262dce0f7c1c535968a ``` + * Or use SHA 256 + ``` + sha256sum pnetcdf-1.11.0.tar.gz + a18a1a43e6c4fd7ef5827dbe90e9dcf1363b758f513af1f1356ed6c651195a9f pnetcdf-1.11.0.tar.gz + ``` 11. Update PnetCDF Web Page * https://github.com/Parallel-NetCDF/Parallel-NetCDF.github.io * Create a new file of release note Parallel-NetCDF.github.io/Release_notes/1.11.0.md. diff --git a/Makefile.am b/Makefile.am index 8e116c08ab..7fc92a868f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -8,11 +8,15 @@ ACLOCAL_AMFLAGS = -I m4 -SUBDIRS = src man examples benchmarks test -DIST_SUBDIRS = src man examples benchmarks test doc +if ENABLE_GIO + GIO_MOD = gio +endif + +SUBDIRS = $(GIO_MOD) src man examples benchmarks test +DIST_SUBDIRS = $(GIO_MOD) src man examples benchmarks test doc if BUILD_DOCS -SUBDIRS += doc + SUBDIRS += doc endif EXTRA_DIST = COPYRIGHT \ @@ -23,6 +27,10 @@ EXTRA_DIST = COPYRIGHT \ m4/foreach.m4 \ m4/utils.m4 +if ! ENABLE_GIO + EXTRA_DIST += gio +endif + # Below is a trick to build all test executables, without running them # tests: # $(MAKE) $(AM_MAKEFLAGS) check TESTS= @@ -127,7 +135,9 @@ install-data-hook: distclean-local: @if [ "$(abs_builddir)" != "$(abs_srcdir)" ] ; then \ for d in $(DIST_SUBDIRS) ; do \ - rmdir $$d || true ; \ + if [ -d $$d ] ; then \ + rmdir $$d || true ; \ + fi \ done ; \ fi diff --git a/README.md b/README.md index bd76d61e78..c1a49c10cb 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ ## PnetCDF source code development repository -[![MPICH](https://github.com/Parallel-NetCDF/PnetCDF/actions/workflows/ubuntu_mpich.yml/badge.svg)](https://github.com/Parallel-NetCDF/PnetCDF/actions/workflows/ubuntu_mpich.yml) -[![OpenMPI](https://github.com/Parallel-NetCDF/PnetCDF/actions/workflows/ubuntu_openmpi.yml/badge.svg)](https://github.com/Parallel-NetCDF/PnetCDF/actions/workflows/ubuntu_openmpi.yml) - +[![CI - OS and MPI](https://github.com/Parallel-NetCDF/PnetCDF/actions/workflows/main.yml/badge.svg)](https://github.com/Parallel-NetCDF/PnetCDF/actions/workflows/main.yml) PnetCDF is a parallel I/O library for accessing [Unidata's NetCDF](http://www.unidata.ucar.edu/software/netcdf) files in @@ -17,6 +15,7 @@ Northwestern University and Argonne National Laboratory. contains more information about PnetCDF. ### PnetCDF official software releases +* The [alpha release of 1.15.0](https://parallel-netcdf.github.io/Release/pnetcdf-1.15.0-alpha.tar.gz) is available on October 27, 2025. * The latest stable release is [pnetcdf-1.14.1.tar.gz](https://parallel-netcdf.github.io/Release/pnetcdf-1.14.1.tar.gz) ([release note](https://github.com/Parallel-NetCDF/Parallel-NetCDF.github.io/blob/master/Release_notes/1.14.1.md)), diff --git a/RELEASE_NOTES b/RELEASE_NOTES index c0e031f48c..112da4eb4a 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -6,6 +6,19 @@ PnetCDF Release Notes Version _PNETCDF_VERSION_ (_PNETCDF_RELEASE_DATE_) ------------------------------------- +* New feature + +* New optimization + +* API deprecated + +* New error code + + +------------------------------------- +Version 1.14.1 (July 31, 2025) +------------------------------------- + * New optimization + When file header extent size grows, moving the data section to a higher file offset has changed to be done in chunks of 16 MB per process. @@ -68,27 +81,28 @@ Version _PNETCDF_VERSION_ (_PNETCDF_RELEASE_DATE_) + There are three ways in PnetCDF for user to set hints to align the starting file offset for the data section (header extent) and record variable section. - 1. through a call to API `nc_header_align_size` by setting arguments of - `h_minfree`, `v_align`, `v_minfree`, and `r_align`. - 2. through an MPI info object passed to calls of `ncmpi_create()` and - `ncmpi_open()`. Hints are `nc_header_align_size`, `nc_var_align_size`, - and `nc_record_align_size`. - 3. through a run-time environment variable `PNETCDF_HINTS`. Hints are - `nc_header_align_size`, `nc_var_align_size`, and `nc_record_align_size`. + 1. through a call to API `ncmpi__enddef` by setting arguments `h_minfree`, + `v_align`, `v_minfree`, and `r_align`. + 2. through passing an MPI info object to a call of `ncmpi_create()` and + `ncmpi_open()`. Hints include `nc_header_align_size`, + `nc_var_align_size`, and `nc_record_align_size`. + 3. through setting hints in an environment variable `PNETCDF_HINTS` at the + run time. Hints include `nc_header_align_size`, `nc_var_align_size`, and + `nc_record_align_size`. + As the same hints may be set by one or more of the above methods, PnetCDF implements the following hint precedence. * `PNETCDF_HINTS` > `ncmpi__enddef()` > `MPI info`. - * 1st priority: hints set in the environment variable `PNETCDF_HINTS`, e.g. - `PNETCDF_HINTS="nc_var_align_size=1048576"`. Making this the first - priority is because it allows to run the same application executable - without source code modification using different alignment settings - through a run-time environment variable. - * 2nd priority: hints set in the MPI info object passed to calls of - `ncmpi_create()` and `ncmpi_open()`, e.g. - `MPI_Info_set("nc_var_align_size", "1048576");`. The reasoning is when a - 3rd-party library built on top of PnetCDF implements its codes using - 'ncmpi__enddef'. An application that uses such 3rd-party library can pass - an MPI info object to it, which further passes the info to PnetCDF. This + * 1st priority: hints set in the run-time environment variable + `PNETCDF_HINTS`, e.g. `PNETCDF_HINTS="nc_var_align_size=1048576"`. Making + this the first priority is because it allows the same application + executable without source code modification to run using different + alignment settings in the run-time environment variable. + * 2nd priority: hints set in the MPI info object, e.g. + `MPI_Info_set("nc_var_align_size", "1048576");`, passed to calls of + `ncmpi_create()` and `ncmpi_open()`. The reasoning is when a 3rd-party + library built on top of PnetCDF may call 'ncmpi__enddef' with its setting + of alignment values. An application that uses such 3rd-party library can + pass an MPI info object to it, which is then passes to PnetCDF. This precedence allows that application to exercise different hints without changing the 3rd-party library's source codes. * 3rd priority: hints used in the arguments of `ncmpi__enddef()`, e.g. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000..5315b6bc07 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,37 @@ +# Security Policy + +Security issues specific to the PnetCDF code base itself have so far been rare. +The issue label `security` is used to identify issues which manifest known +security vulnerabilities. + +Security issues, when discovered, follow the same process as any other bug +fixes. Security issues are triaged and assessed for severity and likelihood. +Work to correct security issues is then scheduled as appropriate. + +Though the project has so far not encountered urgent security vulnerabilities, +should any arise the project will use GitHub's security communication +mechanisms to gather information. + +In the event the PnetCDF user community requires notification of a potential +urgent security vulnerability, our intention is to provide an update on or +about the same time we use our normal communication mechanisms to alert users. + +## Supported Versions + +The supported version of PnetCDF is the *latest* release. +All releases of PnetCDF can be found on the +[download page](https://parallel-netcdf.github.io/wiki/Download.html). + +Any security issues requiring immediate updates to PnetCDF will be made +available, at best, only in the *latest* release but might also only be made +available in the *next* planned release. A planned release of PnetCDF may be +accelerated in order to address a security issue. On very rare occasions, the +PnetCDF project may re-release an already released version solely to address a +specific or severe issue. + +## Reporting a Vulnerability + +Generally, any issues with security implications should be submitted through +the project's [GitHub security](https://github.com/Parallel-NetCDF/PnetCDF/security) +**Report a vulnerability** button. + diff --git a/benchmarks/C/Makefile.am b/benchmarks/C/Makefile.am index 333176cbf5..dfe9cc80c1 100644 --- a/benchmarks/C/Makefile.am +++ b/benchmarks/C/Makefile.am @@ -21,14 +21,18 @@ check_PROGRAMS = aggregation \ TESTS_ENVIRONMENT = export SED="$(SED)"; TESTS_ENVIRONMENT += export srcdir="$(srcdir)"; +TESTS_ENVIRONMENT += export top_builddir="$(top_builddir)"; TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; TESTS_ENVIRONMENT += export TESTMPIRUN="$(TESTMPIRUN)"; -TESTS_ENVIRONMENT += export PNETCDF_DEBUG="$(PNETCDF_DEBUG)"; TESTS_ENVIRONMENT += export TESTPROGRAMS="$(TESTPROGRAMS)"; TESTS_ENVIRONMENT += export check_PROGRAMS="$(check_PROGRAMS)"; -TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER="$(ENABLE_BURST_BUFFER)"; -TESTS_ENVIRONMENT += export ENABLE_NETCDF4="$(ENABLE_NETCDF4)"; + +TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER=@ENABLE_BURST_BUFFER@; +TESTS_ENVIRONMENT += export ENABLE_NETCDF4=@ENABLE_NETCDF4@; +TESTS_ENVIRONMENT += export ENABLE_THREAD_SAFE=@ENABLE_THREAD_SAFE@; +TESTS_ENVIRONMENT += export MIMIC_LUSTRE=@MIMIC_LUSTRE@; +TESTS_ENVIRONMENT += export ENABLE_GIO=@ENABLE_GIO@; NC_FILES = $(check_PROGRAMS:%=$(TESTOUTDIR)/%.nc) \ $(check_PROGRAMS:%=$(TESTOUTDIR)/%.bb.nc) @@ -36,18 +40,48 @@ NC_FILES = $(check_PROGRAMS:%=$(TESTOUTDIR)/%.nc) \ CLEANFILES = core core.* *.gcda *.gcno *.gcov gmon.out \ $(NC_FILES) +check_SCRIPTS = parallel_run.sh + # netcdf_put_vara.c is a NetCDF4 version of pnetcdf_put_vara.c. Both files can # be used to compare NetCDF4 performance against PnetCDF. EXTRA_DIST = parallel_run.sh netcdf_put_vara.c -ptest ptests ptest4: $(check_PROGRAMS) +ptest ptest4: $(check_PROGRAMS) @echo "===========================================================" @echo " $(subdir): Parallel testing on 4 MPI processes" @echo "===========================================================" @$(TESTS_ENVIRONMENT) \ $(srcdir)/parallel_run.sh 4 || exit 1 -ptest2 ptest6 ptest8 ptest10: +ptest2: $(check_PROGRAMS) + @echo "===========================================================" + @echo " $(subdir): Parallel testing on 2 MPI processes" + @echo "===========================================================" + @$(TESTS_ENVIRONMENT) \ + $(srcdir)/parallel_run.sh 2 || exit 1 + +ptest6: $(check_PROGRAMS) + @echo "===========================================================" + @echo " $(subdir): Parallel testing on 6 MPI processes" + @echo "===========================================================" + @$(TESTS_ENVIRONMENT) \ + $(srcdir)/parallel_run.sh 6 || exit 1 + +ptest8: $(check_PROGRAMS) + @echo "===========================================================" + @echo " $(subdir): Parallel testing on 8 MPI processes" + @echo "===========================================================" + @$(TESTS_ENVIRONMENT) \ + $(srcdir)/parallel_run.sh 8 || exit 1 + +ptest10: $(check_PROGRAMS) + @echo "===========================================================" + @echo " $(subdir): Parallel testing on 10 MPI processes" + @echo "===========================================================" + @$(TESTS_ENVIRONMENT) \ + $(srcdir)/parallel_run.sh 10 || exit 1 + +ptests: ptest4 ptest6 # build check targets but not invoke tests-local: all $(check_PROGRAMS) diff --git a/benchmarks/C/aggregation.c b/benchmarks/C/aggregation.c index 3f2f3afccc..9093cb7a17 100644 --- a/benchmarks/C/aggregation.c +++ b/benchmarks/C/aggregation.c @@ -101,7 +101,7 @@ #define ERR(e) { \ if ((e) != NC_NOERR) { \ - printf("Error at line=%d: %s\n", __LINE__, ncmpi_strerror(e)); \ + fprintf(stderr,"Error at line=%d: %s\n", __LINE__, ncmpi_strerror(e)); \ nerrs++; \ } \ } @@ -122,6 +122,7 @@ typedef struct { int star_block; int blocking_io; int double_xtype; + int indep_io; MPI_Offset len; MPI_Offset w_size; MPI_Offset r_size; @@ -287,6 +288,11 @@ int benchmark_write(char *filename, sts = (int*) malloc(sizeof(int) * num_reqs); err = ncmpi_enddef(ncid); ERR(err) + if (cfg->indep_io) { + err = ncmpi_begin_indep_data(ncid); + ERR(err) + } + err = ncmpi_inq_header_size(ncid, &cfg->header_size); ERR(err) err = ncmpi_inq_header_extent(ncid, &cfg->header_extent); ERR(err) end_t = MPI_Wtime(); @@ -306,12 +312,17 @@ int benchmark_write(char *filename, start[1] = cfg->len * (rank / psizes[1]); count[1] = cfg->len; count[2] = cfg->len; - if (cfg->blocking_io) - err = ncmpi_put_vara_double_all(ncid, varid[v], start, + if (cfg->blocking_io) { + if (cfg->indep_io) + err = ncmpi_put_vara_double(ncid, varid[v], start, count, buf[v]); + else + err = ncmpi_put_vara_double_all(ncid, varid[v], start, + count, buf[v]); + } else err = ncmpi_iput_vara_double(ncid, varid[v], start, count, - buf[v], &reqs[k++]); + buf[v], &reqs[k++]); ERR(err) if (debug) DBG_PRINT("block-block", n, i); v++; @@ -323,9 +334,14 @@ int benchmark_write(char *filename, count[2] = cfg->len; stride[1] = 1; stride[2] = nprocs; - if (cfg->blocking_io) - err = ncmpi_put_vars_double_all(ncid, varid[v], start, + if (cfg->blocking_io) { + if (cfg->indep_io) + err = ncmpi_put_vars_double(ncid, varid[v], start, count, stride, buf[v]); + else + err = ncmpi_put_vars_double_all(ncid, varid[v], start, + count, stride, buf[v]); + } else err = ncmpi_iput_vars_double(ncid, varid[v], start, count, stride, buf[v], &reqs[k++]); @@ -338,9 +354,14 @@ int benchmark_write(char *filename, start[2] = 0; count[1] = cfg->len; count[2] = bs_gsizes[2]; - if (cfg->blocking_io) - err = ncmpi_put_vara_double_all(ncid, varid[v], start, + if (cfg->blocking_io) { + if (cfg->indep_io) + err = ncmpi_put_vara_double(ncid, varid[v], start, count, buf[v]); + else + err = ncmpi_put_vara_double_all(ncid, varid[v], start, + count, buf[v]); + } else err = ncmpi_iput_vara_double(ncid, varid[v], start, count, buf[v], &reqs[k++]); @@ -353,9 +374,14 @@ int benchmark_write(char *filename, start[2] = cfg->len * rank; count[1] = sb_gsizes[1]; count[2] = cfg->len; - if (cfg->blocking_io) - err = ncmpi_put_vara_double_all(ncid, varid[v], start, + if (cfg->blocking_io) { + if (cfg->indep_io) + err = ncmpi_put_vara_double(ncid, varid[v], start, count, buf[v]); + else + err = ncmpi_put_vara_double_all(ncid, varid[v], start, + count, buf[v]); + } else err = ncmpi_iput_vara_double(ncid, varid[v], start, count, buf[v], &reqs[k++]); @@ -372,13 +398,13 @@ int benchmark_write(char *filename, if (!cfg->blocking_io) { start_t = end_t; -#ifdef USE_INDEP_MODE - err = ncmpi_begin_indep_data(ncid); ERR(err) - err = ncmpi_wait(ncid, num_reqs, reqs, sts); ERR(err) - err = ncmpi_end_indep_data(ncid); ERR(err) -#else - err = ncmpi_wait_all(ncid, num_reqs, reqs, sts); ERR(err) -#endif + + if (cfg->indep_io) + err = ncmpi_wait(ncid, num_reqs, reqs, sts); + else + err = ncmpi_wait_all(ncid, num_reqs, reqs, sts); + ERR(err) + /* check status of all requests */ for (i=0; iindep_io) { + err = ncmpi_begin_indep_data(ncid); + ERR(err) + } + /* Note that PnetCDF read the file in chunks of size 256KB, thus the read * amount may be more than the file header size */ @@ -518,8 +549,12 @@ int benchmark_read(char *filename, start[1] = cfg->len * (rank / psizes[1]); count[1] = cfg->len; count[2] = cfg->len; - if (cfg->blocking_io) - err = ncmpi_get_vara_double_all(ncid, v, start, count, buf[v]); + if (cfg->blocking_io) { + if (cfg->indep_io) + err = ncmpi_get_vara_double(ncid, v, start, count, buf[v]); + else + err = ncmpi_get_vara_double_all(ncid, v, start, count, buf[v]); + } else err = ncmpi_iget_vara_double(ncid, v, start, count, buf[v], &reqs[k++]); @@ -533,9 +568,14 @@ int benchmark_read(char *filename, count[2] = cfg->len; stride[1] = 1; stride[2] = nprocs; - if (cfg->blocking_io) - err = ncmpi_get_vars_double_all(ncid, v, start, count, + if (cfg->blocking_io) { + if (cfg->indep_io) + err = ncmpi_get_vars_double(ncid, v, start, count, stride, buf[v]); + else + err = ncmpi_get_vars_double_all(ncid, v, start, count, + stride, buf[v]); + } else err = ncmpi_iget_vars_double(ncid, v, start, count, stride, buf[v], &reqs[k++]); @@ -547,8 +587,12 @@ int benchmark_read(char *filename, start[2] = 0; count[1] = cfg->len; count[2] = bs_gsizes[2]; - if (cfg->blocking_io) - err = ncmpi_get_vara_double_all(ncid, v, start, count, buf[v]); + if (cfg->blocking_io) { + if (cfg->indep_io) + err = ncmpi_get_vara_double(ncid, v, start, count, buf[v]); + else + err = ncmpi_get_vara_double_all(ncid, v, start, count, buf[v]); + } else err = ncmpi_iget_vara_double(ncid, v, start, count, buf[v], &reqs[k++]); @@ -560,8 +604,12 @@ int benchmark_read(char *filename, start[2] = cfg->len * rank; count[1] = sb_gsizes[1]; count[2] = cfg->len; - if (cfg->blocking_io) - err = ncmpi_get_vara_double_all(ncid, v, start, count, buf[v]); + if (cfg->blocking_io) { + if (cfg->indep_io) + err = ncmpi_get_vara_double(ncid, v, start, count, buf[v]); + else + err = ncmpi_get_vara_double_all(ncid, v, start, count, buf[v]); + } else err = ncmpi_iget_vara_double(ncid, v, start, count, buf[v], &reqs[k++]); @@ -578,13 +626,12 @@ int benchmark_read(char *filename, if (!cfg->blocking_io) { start_t = end_t; -#ifdef USE_INDEP_MODE - err = ncmpi_begin_indep_data(ncid); ERR(err) - err = ncmpi_wait(ncid, num_reqs, reqs, sts); ERR(err) - err = ncmpi_end_indep_data(ncid); ERR(err) -#else - err = ncmpi_wait_all(ncid, num_reqs, reqs, sts); ERR(err) -#endif + if (cfg->indep_io) + err = ncmpi_wait(ncid, num_reqs, reqs, sts); + else + err = ncmpi_wait_all(ncid, num_reqs, reqs, sts); + ERR(err) + /* check status of all requests */ for (i=0; i 0); diff --git a/benchmarks/C/write_block_read_column.c b/benchmarks/C/write_block_read_column.c index 5048783f4d..c33941b000 100644 --- a/benchmarks/C/write_block_read_column.c +++ b/benchmarks/C/write_block_read_column.c @@ -44,7 +44,7 @@ static int verbose; -#define ERR(e) {if((e)!=NC_NOERR){printf("Error at line=%d: %s\n", __LINE__, ncmpi_strerror(e));nerrs++;}} +#define ERR(e) {if((e)!=NC_NOERR){fprintf(stderr,"Error at line=%d: %s\n", __LINE__, ncmpi_strerror(e));nerrs++;}} /*----< print_info() >------------------------------------------------------*/ static @@ -68,6 +68,7 @@ void print_info(MPI_Info *info_used) /*----< benchmark_write() >---------------------------------------------------*/ static int benchmark_write(char *filename, + int indep_io, MPI_Offset len, MPI_Offset *w_size, MPI_Info *w_info_used, @@ -157,6 +158,12 @@ int benchmark_write(char *filename, } err = ncmpi_enddef(ncid); ERR(err) + + if (indep_io) { + err = ncmpi_begin_indep_data(ncid); + ERR(err) + } + end_t = MPI_Wtime(); timing[2] = end_t - start_t; start_t = end_t; @@ -171,19 +178,31 @@ int benchmark_write(char *filename, for (i=0; i---------------------------------------------------*/ static int benchmark_read(char *filename, + int indep_io, MPI_Offset len, MPI_Offset *r_size, MPI_Info *r_info_used, @@ -241,6 +261,11 @@ int benchmark_read(char *filename, timing[1] = start_t - timing[0]; MPI_Info_free(&info); + if (indep_io) { + err = ncmpi_begin_indep_data(ncid); + ERR(err) + } + err = ncmpi_inq_nvars(ncid, &nvars); ERR(err) err = ncmpi_inq_dimid(ncid, "Y", &dimid[0]); ERR(err) err = ncmpi_inq_dimid(ncid, "X", &dimid[1]); ERR(err) @@ -283,19 +308,31 @@ int benchmark_read(char *filename, for (i=0; inelems = (rank == 0) ? 1 : 0; + if (vars->nelems == 0) vars->count[0] = 0; } else if (ndims == 2) { err = cdl_hdr_inq_dim(hid, dimids[1], NULL, &dim1); CHECK_ERR("cdl_hdr_inq_dim") vars->nelems = (rank == 0) ? dim1 : 0; + if (vars->nelems == 0) vars->count[0] = 0; vars->count[1] = dim1; /* dimension dim1 is not partitioned */ vars->start[1] = 0; /* dimension dim1 is not partitioned */ } @@ -286,7 +308,7 @@ int inquire_vars(int ncid, my_start_x += latitude % psizes[1]; } - if (debug) { + if (verbose && debug) { printf("%2d: rank (%2d, %2d) start %4lld %4lld count %4lld %4lld\n", rank, my_rank_y, my_rank_x, my_start_y, my_start_x, my_count_y, my_count_x); fflush(stdout); @@ -343,10 +365,13 @@ int inquire_vars(int ncid, vars->count[0] = 1; /* time dimension */ /* In WRF, the first dimension is always NC_UNLIMITED */ - if (ndims == 1) + if (ndims == 1) { vars->nelems = (rank == 0) ? 1 : 0; + if (vars->nelems == 0) vars->count[0] = 0; + } else if (ndims == 2) { vars->nelems = (rank == 0) ? dim1 : 0; + if (vars->nelems == 0) vars->count[0] = 0; vars->count[1] = dim1; /* dimension dim1 is not partitioned */ vars->start[1] = 0; /* dimension dim1 is not partitioned */ } @@ -396,16 +421,19 @@ int def_dims_vars(int ncid, { char *name; void *value; - int i, j, err=NC_NOERR, ndims, nvars, nattrs; + int i, j, err=NC_NOERR, rank, ndims, nvars, nattrs; MPI_Offset size, nelems; nc_type xtype; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + /* define dimensions */ /* retrieve the number of dimensions defined in the CDL file */ err = cdl_hdr_inq_ndims(hid, &ndims); CHECK_ERR("cdl_hdr_inq_ndims") - if (debug) printf("dim: ndims %d\n", ndims); + if (verbose && debug && rank == 0) + printf("dim: ndims %d\n", ndims); for (i=0; ixtype, ndims, nattrs); - for (j=0; jdimids[j]); + if (verbose && debug && rank == 0) { + printf("\t var name %s type %s ndims %d nattr %d\n", + vars->name, str_NC_type(vars->xtype), vars->ndims, nattrs); + for (j=0; jndims; j++) { + char dname[64]; + ncmpi_inq_dimname(ncid, vars->dimids[j], dname); + printf("\t\tdimid %d, name: %s\n",vars->dimids[j], dname); + } } for (j=0; jvarid, name, xtype, nelems, value); @@ -470,90 +502,67 @@ int def_dims_vars(int ncid, /* retrieve the number of global attributes */ err = cdl_hdr_inq_nattrs(hid, NC_GLOBAL, &nattrs); CHECK_ERR("cdl_hdr_inq_nattrs") - if (debug) printf("global attrs: nattrs %d\n", nattrs); + if (verbose && debug && rank == 0) + printf("global attrs: nattrs %d\n", nattrs); for (i=0; i= ntimes */ + err = ncmpi_inq_unlimdim(ncid, &unlimdimid); + CHECK_ERR("ncmpi_inq_unlimdim") + err = ncmpi_inq_dimlen(ncid, unlimdimid, &dim_len); + CHECK_ERR("ncmpi_inq_dimlen") + if (dim_len < ntimes) { + if (rank == 0) + fprintf(stderr, "Error: input file expects to have at least %d time records but got %lld\n", ntimes, dim_len); + err = NC_EIO; + goto err_out; + } + err = ncmpi_inq_nvars(ncid, &nvars); CHECK_ERR("ncmpi_inq_nvars") @@ -793,48 +1002,11 @@ int wrf_r_benchmark(char *in_file, err = inquire_vars(ncid, vars, psizes, longitude, latitude, &buf_size); CHECK_ERR("inquire_vars") - if (debug) { + if (verbose && debug && rank == 0) printf("%2d: buf_size %lld\n", rank, buf_size); - fflush(stdout); - } - /* allocate and initialize read buffers */ - MPI_Offset mem_alloc; - if (debug) mem_alloc = 0; - - for (i=0; i= ntimes */ vars[i].start[0] = j; - if (vars[i].xtype == NC_FLOAT) - err = ncmpi_iget_vara_float(ncid, vars[i].varid, vars[i].start, - vars[i].count, vars[i].buf, NULL); - else if (vars[i].xtype == NC_INT) - err = ncmpi_iget_vara_int(ncid, vars[i].varid, vars[i].start, - vars[i].count, vars[i].buf, NULL); - else if (vars[i].xtype == NC_CHAR) - err = ncmpi_iget_vara_text(ncid, vars[i].varid, vars[i].start, - vars[i].count, vars[i].buf, NULL); + if (vars[i].xtype == NC_FLOAT) { + if (blocking) + err = ncmpi_get_vara_float_all(ncid, vars[i].varid, + vars[i].start, vars[i].count, + vars[i].buf); + else + err = ncmpi_iget_vara_float(ncid, vars[i].varid, + vars[i].start, vars[i].count, + vars[i].buf, NULL); + } + else if (vars[i].xtype == NC_INT) { + if (blocking) + err = ncmpi_get_vara_int_all(ncid, vars[i].varid, + vars[i].start, vars[i].count, + vars[i].buf); + else + err = ncmpi_iget_vara_int(ncid, vars[i].varid, + vars[i].start, vars[i].count, + vars[i].buf, NULL); + } + else if (vars[i].xtype == NC_CHAR) { + if (blocking) + err = ncmpi_get_vara_text_all(ncid, vars[i].varid, + vars[i].start, vars[i].count, + vars[i].buf); + else + err = ncmpi_iget_vara_text(ncid, vars[i].varid, + vars[i].start, vars[i].count, + vars[i].buf, NULL); + } CHECK_ERR(vars[i].name) } @@ -872,16 +1064,18 @@ int wrf_r_benchmark(char *in_file, timing[2] += end_t - start_t; start_t = end_t; - if (debug && rank == 0) { + if (verbose && debug && rank == 0) { printf("Flush read requests at end of iteration j=%d\n",j); fflush(stdout); } - /* flush all nonblocking read requests */ - err = ncmpi_wait_all(ncid, NC_REQ_ALL, NULL, NULL); - CHECK_ERR("ncmpi_wait_all") - end_t = MPI_Wtime(); - timing[3] += end_t - start_t; + if (!blocking) { + /* flush all nonblocking read requests */ + err = ncmpi_wait_all(ncid, NC_REQ_ALL, NULL, NULL); + CHECK_ERR("ncmpi_wait_all") + end_t = MPI_Wtime(); + timing[3] += end_t - start_t; + } } /* obtain the accumulated data amount read by this rank */ @@ -904,7 +1098,7 @@ int wrf_r_benchmark(char *in_file, char value[MPI_MAX_INFO_VAL+1]; int flag; - printf("-----------------------------------------------------------\n"); + printf("\n-----------------------------------------------------------\n"); printf("---- WRF-IO read benchmark ----\n"); printf("Input NetCDF file name: %s\n", in_file); printf("Number of MPI processes: %d\n", nprocs); @@ -915,11 +1109,19 @@ int wrf_r_benchmark(char *in_file, printf("Total read amount: %lld B\n", sum_r_size); printf(" %.2f MiB\n", (float)sum_r_size/1048576); printf(" %.2f GiB\n", (float)sum_r_size/1073741824); + if (blocking) + printf("Using PnetCDF blocking APIs\n"); + else + printf("Using PnetCDF non-blocking APIs\n"); double bw = (double)sum_r_size / 1048576; printf("Max open-to-close time: %.4f sec\n", max_t[0]); printf("Max inquire metadata time: %.4f sec\n", max_t[1]); - printf("Max iget posting time: %.4f sec\n", max_t[2]); - printf("Max wait_all time: %.4f sec\n", max_t[3]); + if (blocking) + printf("Max get time: %.4f sec\n", max_t[2]); + else { + printf("Max iget posting time: %.4f sec\n", max_t[2]); + printf("Max wait_all time: %.4f sec\n", max_t[3]); + } printf("Read bandwidth: %.2f MiB/s\n", bw/max_t[0]); printf(" %.2f GiB/s\n", bw/1024.0/max_t[0]); printf("-----------------------------------------------------------\n"); @@ -935,6 +1137,8 @@ int wrf_r_benchmark(char *in_file, printf("MPI-IO hint cb_config_list: %s\n", HINT); MPI_Info_get(info_used, "cb_node_list", MPI_MAX_INFO_VAL, value, &flag); printf("MPI-IO hint cb_node_list: %s\n", HINT); + MPI_Info_get(info_used, "nc_driver", MPI_MAX_INFO_VAL, value, &flag); + printf("PnetCDF hint nc_driver: %s\n", HINT); MPI_Info_get(info_used, "nc_num_aggrs_per_node",MPI_MAX_INFO_VAL, value, &flag); printf("PnetCDF hint nc_num_aggrs_per_node: %s\n", HINT); MPI_Info_get(info_used, "nc_ina_node_list", MPI_MAX_INFO_VAL, value, &flag); @@ -942,7 +1146,7 @@ int wrf_r_benchmark(char *in_file, MPI_Info_get(info_used, "cray_cb_nodes_multiplier", MPI_MAX_INFO_VAL, value, &flag); printf("Hint cray_cb_nodes_multiplier: %s\n", HINT); MPI_Info_get(info_used, "cray_cb_write_lock_mode", MPI_MAX_INFO_VAL, value, &flag); - printf("Hint cray_cb_write_lock_mode: %s\n", HINT); + printf("Hint cray_cb_write_lock_mode: %s\n", HINT); printf("-----------------------------------------------------------\n"); } MPI_Info_free(&info_used); @@ -958,6 +1162,240 @@ int wrf_r_benchmark(char *in_file, } if (err != NC_NOERR) return err; + /* check if there is any PnetCDF internal malloc residue */ + MPI_Offset malloc_size, sum_size; + err = ncmpi_inq_malloc_size(&malloc_size); + if (err == NC_ENOTENABLED) /* --enable-profiling is not set at configure */ + return NC_NOERR; + else if (err == NC_NOERR) { + MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); + if (rank == 0 && sum_size > 0) + printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n", + sum_size); + if (malloc_size > 0) ncmpi_inq_malloc_list(); + } + /* report the PnetCDF internal heap memory allocation high water mark */ + err = ncmpi_inq_malloc_max_size(&malloc_size); + if (err == NC_NOERR) { + MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_MAX, 0, MPI_COMM_WORLD); + if (verbose && rank == 0) + printf("Max heap memory allocated by PnetCDF internally is %.2f MiB\n\n", + (float)sum_size/1048576); + } + fflush(stdout); + + return err; +} + +static +int grow_header_benchmark(char *in_file) +{ + char value[MPI_MAX_INFO_VAL], cb_node_list[MPI_MAX_INFO_VAL], *attr; + int i, err=NC_NOERR, nprocs, rank, ncid, ndims, dimid[3]; + int varid, unlimdimid, nvars, fix_nvars, rec_nvars, len, flag; + double timing, max_t; + MPI_Offset hdr_size, hdr_extent, attr_len, num_rec, longitude, latitude; + MPI_Offset r_amnt[2], w_amnt[2], amnt[2], sum_amnt[2], fix_off, rec_off; + MPI_Offset rec_size, nc_data_move_chunk_size; + MPI_Info info; + + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + /* open input file */ + err = ncmpi_open(MPI_COMM_WORLD, in_file, NC_WRITE, MPI_INFO_NULL, &ncid); + if (err != NC_NOERR) { + fprintf(stderr,"Error at line=%d: opening file %s (%s)\n", + __LINE__, in_file, ncmpi_strerror(err)); + goto err_out; + } + + err = ncmpi_inq_dimid(ncid, "Time", &dimid[0]); + CHECK_ERR("ncmpi_inq_dimid") + err = ncmpi_inq_dimid(ncid, "south_north", &dimid[1]); + CHECK_ERR("ncmpi_inq_dimid") + err = ncmpi_inq_dimlen(ncid, dimid[1], &longitude); + CHECK_ERR("ncmpi_inq_dimlen") + err = ncmpi_inq_dimid(ncid, "west_east", &dimid[2]); + CHECK_ERR("ncmpi_inq_dimid") + err = ncmpi_inq_dimlen(ncid, dimid[2], &latitude); + CHECK_ERR("ncmpi_inq_dimlen") + + err = ncmpi_inq_header_size(ncid, &hdr_size); + CHECK_ERR("ncmpi_inq_header_size") + err = ncmpi_inq_header_extent(ncid, &hdr_extent); + CHECK_ERR("ncmpi_inq_header_extent") + if (verbose && debug && rank == 0) + printf("Line %d: header size %lld extent %lld free space %lld\n", + __LINE__,hdr_size,hdr_extent,hdr_extent-hdr_size); + + /* check number of records in input file */ + err = ncmpi_inq_unlimdim(ncid, &unlimdimid); + CHECK_ERR("ncmpi_inq_unlimdim") + err = ncmpi_inq_dimlen(ncid, unlimdimid, &num_rec); + CHECK_ERR("ncmpi_inq_dimlen") + + err = ncmpi_inq_nvars(ncid, &nvars); + CHECK_ERR("ncmpi_inq_nvars") + + fix_nvars = 0; + rec_nvars = 0; + fix_off = -1; + rec_off = -1; + for (i=0; i 0 && r_amnt[0] > 0) { + printf("Line %d: rank %d r_amnt expect 0 but got %lld\n", + __LINE__,rank,r_amnt[0]); + err = 1; + } + if (rank > 0 && w_amnt[0] > 0) { + printf("Line %d: rank %d w_amnt expect 0 but got %lld\n", + __LINE__,rank,w_amnt[0]); + err = 1; + } + MPI_Allreduce(MPI_IN_PLACE, &err, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); + if (err > 0) goto err_out; + + /* start the timer */ + MPI_Barrier(MPI_COMM_WORLD); + timing = MPI_Wtime(); + + err = ncmpi__enddef(ncid, 0, 0, 0, 0); + CHECK_ERR("ncmpi_enddef") + + timing = MPI_Wtime() - timing; + + err = ncmpi_inq_get_size(ncid, &r_amnt[1]); + CHECK_ERR("ncmpi_inq_get_size") + err = ncmpi_inq_put_size(ncid, &w_amnt[1]); + CHECK_ERR("ncmpi_inq_put_size") + + err = ncmpi_inq_header_size(ncid, &hdr_size); + CHECK_ERR("ncmpi_inq_header_size") + err = ncmpi_inq_header_extent(ncid, &hdr_extent); + CHECK_ERR("ncmpi_inq_header_extent") + if (verbose && debug && rank == 0) + printf("Line %d: header size %lld extent %lld free space %lld\n", + __LINE__,hdr_size,hdr_extent,hdr_extent-hdr_size); + + /* fill the new record variable, so ncmpidiff can run and check */ + for (i=0; i 0); + return (err != NC_NOERR); } diff --git a/configure.ac b/configure.ac index 1426121047..8f9cff689e 100644 --- a/configure.ac +++ b/configure.ac @@ -15,7 +15,7 @@ dnl AC_REVISION([$Revision$])dnl dnl autoconf v2.70 and later is required. See https://github.com/Parallel-NetCDF/PnetCDF/issues/94 dnl autoconf v2.70 was released in 2021-01-28 AC_PREREQ([2.70]) -AC_INIT([PnetCDF], [1.14.1], +AC_INIT([PnetCDF], [1.15.0-alpha], [parallel-netcdf@mcs.anl.gov], [pnetcdf], [https://parallel-netcdf.github.io]) @@ -69,8 +69,8 @@ AM_EXTRA_RECURSIVE_TARGETS([tests]) dnl parse the version numbers to 4 env variables PNETCDF_VERSION_MAJOR=`echo ${PACKAGE_VERSION} | cut -d. -f1` PNETCDF_VERSION_MINOR=`echo ${PACKAGE_VERSION} | cut -d. -f2` -PNETCDF_VERSION_SUB=`echo ${PACKAGE_VERSION} | cut -d. -f3` -PNETCDF_VERSION_PRE=`echo ${PACKAGE_VERSION} | cut -d. -f4` +PNETCDF_VERSION_SUB=`echo ${PACKAGE_VERSION} | cut -d. -f3 | cut -d'-' -f1` +PNETCDF_VERSION_PRE=`echo ${PACKAGE_VERSION} | cut -d'-' -f2` dnl Note major, minor, and sub are required, but pre is not. PNETCDF_VERSION=${PACKAGE_VERSION} @@ -137,20 +137,11 @@ AH_TEMPLATE([NF_INT8_IS_C_], [C type for Fortran INT8]) AH_TEMPLATE([NF_INT8_T], [Type for Fortran INT8]) AH_TEMPLATE([NF_REAL_IS_C_], [C type for Fortran REAL]) AH_TEMPLATE([NO_IEEE_FLOAT], [Does system have IEEE FLOAT]) -dnl AH_TEMPLATE([ENABLE_IN_PLACE_SWAP], [Define if to enable in-place byte swap]) -dnl AH_TEMPLATE([DISABLE_IN_PLACE_SWAP],[Define if to disable in-place byte swap]) -AH_TEMPLATE([ENABLE_SUBFILING], [Define if to enable subfiling feature]) AH_TEMPLATE([ENABLE_NETCDF4], [Define if to enable NetCDF-4 support]) -AH_TEMPLATE([ENABLE_ADIOS], [Define if to enable ADIOS BP read feature]) AH_TEMPLATE([HDF5_VER_GE_1_10_4], [Define if HDF5 version is at least 1.10.4]) AH_TEMPLATE([NETCDF_GE_4_5_0], [Define if NetCDF version is at least 4.5.0]) AH_TEMPLATE([PNC_MALLOC_TRACE], [Define if to enable malloc tracing]) -AH_TEMPLATE([RELAX_COORD_BOUND], [Define if relaxed coordinate check is enabled]) -AH_TEMPLATE([ENABLE_NULL_BYTE_HEADER_PADDING], [Define if to enable strict null-byte padding in file header]) -AH_TEMPLATE([ENABLE_BURST_BUFFER], [Define if to enable burst buffer feature]) AH_TEMPLATE([PNETCDF_PROFILING], [Define if to enable PnetCDF internal performance profiling]) -AH_TEMPLATE([ENABLE_THREAD_SAFE], [Define if to enable thread-safe capability]) -AH_TEMPLATE([ENABLE_REQ_AGGREGATION], [Define if able to support request aggregation in nonblocking routines]) dnl AH_TEMPLATE([HAVE_MPI_COUNT], [Define if type MPI_Count is defined]) AH_TEMPLATE([HAVE_MPI_LARGE_COUNT], [Define if required MPI APIs have arguments of type MPI_Count]) @@ -1175,6 +1166,8 @@ dnl AC_CHECK_FUNCS([memset setlocale sqrt strchr strrchr strtol]) dnl AC_CHECK_LIB([m], [tanh]) dnl UD_CHECK_LIB_MATH +AC_CHECK_HEADERS([unistd.h fcntl.h malloc.h stddef.h sys/types.h limits.h time.h dirent.h getopt.h]) + dnl When using gcc based compiler with -ansi flag, AC_CHECK_FUNCS can still dnl find strdup, but AC_CHECK_DECL cannot. So we check with AC_CHECK_DECL dnl first and then check AC_CHECK_FUNCS. @@ -1266,7 +1259,6 @@ if test "x${debug}" = xyes; then PNETCDF_DEBUG=1 fi AC_SUBST(PNETCDF_DEBUG) -AM_CONDITIONAL(PNETCDF_DEBUG, [test x"$PNETCDF_DEBUG" = x1]) dnl Data type MPI_Offset was first introduced in MPI 2 and since PnetCDF dnl requires MPI-IO (also introduced in MPI 2), there is no need to check @@ -1328,22 +1320,6 @@ AC_DEFINE_UNQUOTED([OFFFMT], ["$MPI_OFFSET_FMT"], [MPI_Offset printf format]) AC_CHECK_SIZEOF([MPI_Aint], [], [#include ]) AM_CONDITIONAL(SIZEOF_MPI_AINT_IS_4, [test x$ac_cv_sizeof_MPI_Aint = x4]) -dnl the nonblocking routines build up lists of requests with MPI_Type_struct. -dnl If MPI_Offset not the same size as MPI_Aint, the arrays passed around will -dnl get mangled. -if test "$ac_cv_sizeof_MPI_Offset" -ne "$ac_cv_sizeof_MPI_Aint"; then - AC_MSG_WARN([ - ----------------------------------------------------------------------- - MPI_Offset and MPI_Aint are detected of different sizes. - The request aggregation feature implemented in non-blocking APIs is - thus disabled. - -----------------------------------------------------------------------]) -else - AC_DEFINE(ENABLE_REQ_AGGREGATION) - ENABLE_REQ_AGGREGATION=1 - AC_SUBST(ENABLE_REQ_AGGREGATION) -fi - dnl check availability of MPI_Count, which is added in MPI 3.0 standard. dnl The new MPI functions added in 3.0 that have arguments of type MPI_Count: dnl MPI_Type_size_x, MPI_Type_get_extent_x, MPI_Type_get_true_extent_x, and @@ -1352,6 +1328,7 @@ dnl AC_CHECK_TYPE([MPI_Count], [], [], [#include ]) dnl if test "x${ac_cv_type_MPI_Count}" = xyes; then dnl AC_DEFINE(HAVE_MPI_COUNT) dnl fi +AC_CHECK_SIZEOF([MPI_Count], [], [#include ]) dnl Because MPI-IO was first introduced in MPI 2.0 and MPI-IO is a requirement dnl of PnetCDF, there is no need to check the below APIs which were also @@ -1377,8 +1354,11 @@ AC_CHECK_FUNCS([MPI_Type_create_subarray_c \ MPI_Type_get_true_extent_c \ MPI_Type_get_envelope_c \ MPI_Type_get_contents_c \ + MPI_Status_set_elements_x \ MPI_Bcast_c \ MPI_Get_count_c \ + MPI_Isend_c \ + MPI_Irecv_c \ MPI_Pack_c \ MPI_Unpack_c \ MPI_File_read_at_c \ @@ -1459,6 +1439,14 @@ if test "$mpi_version" -ge "3" ; then [], [], [[#include ]]) fi +# check some MPI combiner types that are used internally in PnetCDF +UD_CHECK_MPI_CONSTANTS([MPI_COMBINER_DUP, + MPI_COMBINER_SUBARRAY, + MPI_COMBINER_DARRAY, + MPI_COMBINER_INDEXED_BLOCK, + MPI_COMBINER_HINDEXED_BLOCK], + [], [], [[#include ]]) + dnl Check presence of various MPI error classes. Introduced in MPI 2.0. dnl These could be enums, so we have to do compile checks. dnl AC_CHECK_DECLS([MPI_ERR_FILE_EXISTS, @@ -1521,6 +1509,100 @@ dnl UD_CHECK_MPI_DATATYPE(MPI_REAL8) dnl first defined in MPI 1.0 dnl UD_CHECK_MPI_DATATYPE(MPI_DOUBLE_PRECISION) dnl first defined in MPI 1.0 dnl fi +AC_MSG_CHECKING([whether MPI_Waitall takes MPI_STATUSES_IGNORE]) +if test "x${GCC}" = xyes; then + saved_CFLAGS=${CFLAGS} + CFLAGS="-Werror -Wstringop-overflow=2" + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]],[[ + int count; + MPI_Request *reqs; + MPI_Waitall(count, reqs, MPI_STATUSES_IGNORE); + ]])], [MPI_STATUSES_IGNORE=yes], [MPI_STATUSES_IGNORE=no]) + CFLAGS=${saved_CFLAGS} +else + AC_CHECK_DECL([MPI_STATUSES_IGNORE], [MPI_STATUSES_IGNORE=yes], [MPI_STATUSES_IGNORE=no] [[#include ]]) +fi +AC_MSG_RESULT([$MPI_STATUSES_IGNORE]) +if test "x$MPI_STATUSES_IGNORE" = xyes ; then + AC_DEFINE(HAVE_MPI_STATUSES_IGNORE, 1, [Whether MPI_Waitall takes argument MPI_STATUSES_IGNORE]) +fi + +# +# Check for statfs (many) and specifically f_fstypename field (BSD) +# +AC_CHECK_HEADERS(sys/vfs.h sys/param.h sys/mount.h sys/statvfs.h sys/stat.h sys/type.h unistd.h) + +AC_CHECK_FUNCS([statvfs statfs stat]) + +AC_CHECK_MEMBERS([struct statvfs.f_basetype, + struct statfs.f_fstypename, + struct statfs.f_type, + struct stat.st_fstype],[],[], + AC_INCLUDES_DEFAULT + [#ifdef HAVE_SYS_VFS_H + #include + #endif + #ifdef HAVE_SYS_PARAM_H + #include + #endif + #ifdef HAVE_SYS_MOUNT_H + #include + #endif + #ifdef HAVE_SYS_STATFS_H + #include + #endif + #ifdef HAVE_SYS_STAT_H + #include + #endif + #ifdef HAVE_SYS_TYPE_H + #include + #endif + #ifdef HAVE_UNISTD_H + #include + #endif + ]) + +AC_CHECK_TYPE([blksize_t],[],[AC_DEFINE_UNQUOTED([blksize_t],[__blksize_t],[Provide blksize_t if not available]) ], [[ + #ifdef HAVE_SYS_TYPES_H + #include + #endif + #ifdef HAVE_SYS_STAT_H + #include + #endif + #ifdef HAVE_UNISTD_H + #include + #endif]] ) + +AC_CHECK_DECLS([pwrite]) + +# +# Check if Lustre is available by verifying presence of lustre/lustre_user.h +# +has_lustre=no +AC_CHECK_HEADERS([lustre/lustre_user.h linux/lustre/lustre_user.h], + [has_lustre=yes ; break]) +if test "x$has_lustre" = xyes ; then + AC_DEFINE(HAVE_LUSTRE, 1, [Define for LUSTRE]) + LIBS="$LIBS -llustreapi" + # llapi_get_obd_count() can get the total number of available OSTs + AC_CHECK_FUNCS([llapi_get_obd_count]) +fi +AM_CONDITIONAL(HAVE_LUSTRE, [test x$has_lustre = xyes]) + +minicking_lustre=0 +if test "x$has_lustre" = xno ; then + AC_MSG_CHECKING([for whether mimicking Lustre]) + if test "x$MIMIC_LUSTRE" = xyes ; then + AC_DEFINE(MIMIC_LUSTRE, 1, [Define if mimicking LUSTRE file system]) + minicking_lustre=1 + AC_MSG_RESULT(yes) + else + AC_MSG_RESULT(no) + fi +fi +AC_SUBST(MIMIC_LUSTRE, [$minicking_lustre]) +AM_CONDITIONAL(MIMIC_LUSTRE, [test x$minicking_lustre = x1]) + AC_C_CHAR_UNSIGNED AC_C_BIGENDIAN AM_CONDITIONAL(IS_BIGENDIAN, [test x$ac_cv_c_bigendian = xyes]) @@ -1533,23 +1615,10 @@ AC_ARG_ENABLE([in-place-swap], [in_place_swap=${enableval}], [in_place_swap=auto] ) UD_MSG_DEBUG([in_place_swap=$in_place_swap]) -dnl ENABLE_IN_PLACE_SWAP=0 -dnl if test "x${in_place_swap}" = xyes ; then -dnl ENABLE_IN_PLACE_SWAP=1 -dnl fi -dnl AC_DEFINE(ENABLE_IN_PLACE_SWAP, $ENABLE_IN_PLACE_SWAP) -dnl DISABLE_IN_PLACE_SWAP=0 -dnl if test "x${in_place_swap}" = xno ; then -dnl if test "x${ac_cv_c_bigendian}" = xyes ; then -dnl AC_MSG_WARN([--disable-in-place-swap takes no effect on Big Endian]) -dnl in_place_swap=yes -dnl else -dnl DISABLE_IN_PLACE_SWAP=1 -dnl fi -dnl fi -dnl AC_DEFINE(DISABLE_IN_PLACE_SWAP, $DISABLE_IN_PLACE_SWAP) -dnl IN_PLACE_SWAP being -1 corresponds to "auto" +dnl values of IN_PLACE_SWAP: -1 corresponds to "auto" +dnl 0 corresponds to "disable" +dnl 1 corresponds to "enable" IN_PLACE_SWAP=-1 if test "x${ac_cv_c_bigendian}" = xyes && test "x${in_place_swap}" != xauto ; then AC_MSG_WARN([--(en/dis)able-in-place-swap takes no effect on Big Endian]) @@ -1581,6 +1650,7 @@ AC_SUBST(INTENTV)dnl for src/binding/f90/api.fh.in AC_TYPE_SIZE_T AC_TYPE_OFF_T AC_TYPE_SSIZE_T +AC_TYPE_UINT64_T AC_CHECK_TYPES([ptrdiff_t, schar, uchar, ushort, uint, longlong, ulonglong, int64, uint64]) @@ -1739,8 +1809,15 @@ if test "x${debug}" = xyes; then if test "x$?" != x0 ; then CFLAGS="$CFLAGS -g" fi - CFLAGS=`echo $CFLAGS | ${SED} 's/-O. *//g' | ${SED} 's/-fast *//g'` - CFLAGS="$CFLAGS -O0" + + # remove -fast if set by user + CFLAGS=`echo $CFLAGS | ${SED} 's/-fast *//g'` + + # check if -O is set by user, if not, then add -O0 + str_found=`echo "${CFLAGS}" | ${EGREP} -- "-O"` + if test "x$str_found" = x ; then + CFLAGS="$CFLAGS -O0" + fi if test "x${has_mpicxx}" = xyes ; then str_found=`echo "${CXXFLAGS}" | ${EGREP} -- "-g"` @@ -1836,7 +1913,6 @@ AC_ARG_ENABLE([subfiling], ENABLE_SUBFILING=0 if test "x$enable_subfiling" = "xyes" ; then - AC_DEFINE(ENABLE_SUBFILING) ENABLE_SUBFILING=1 fi AC_SUBST(ENABLE_SUBFILING) @@ -1862,7 +1938,6 @@ if test "x${thread_safe}" = xyes ; then -----------------------------------------------------------------------]) fi fi - AC_DEFINE(ENABLE_THREAD_SAFE) ENABLE_THREAD_SAFE=1 fi AC_SUBST(ENABLE_THREAD_SAFE) @@ -2337,7 +2412,6 @@ UD_MSG_DEBUG(relax_coord_bound=$relax_coord_bound) RELAX_COORD_BOUND=0 if test "x$relax_coord_bound" = xyes ; then RELAX_COORD_BOUND=1 - AC_DEFINE(RELAX_COORD_BOUND) fi AC_SUBST(RELAX_COORD_BOUND) AM_CONDITIONAL([RELAX_COORD_BOUND], [test "x$relax_coord_bound" = xyes]) @@ -2377,7 +2451,6 @@ AC_ARG_ENABLE([null-byte-header-padding], ENABLE_NULL_BYTE_HEADER_PADDING=0 if test "x${null_byte_header_padding}" = xyes ; then ENABLE_NULL_BYTE_HEADER_PADDING=1 - AC_DEFINE(ENABLE_NULL_BYTE_HEADER_PADDING) fi AC_SUBST(ENABLE_NULL_BYTE_HEADER_PADDING) @@ -2389,7 +2462,6 @@ AC_ARG_ENABLE([burst-buffering], ENABLE_BURST_BUFFER=0 if test "x$enable_bbdriver" = "xyes" ; then - AC_DEFINE(ENABLE_BURST_BUFFER) ENABLE_BURST_BUFFER=1 fi AC_SUBST(ENABLE_BURST_BUFFER) @@ -2412,7 +2484,6 @@ AC_ARG_WITH(adios, ENABLE_ADIOS=0 if test "x$enable_adios" = "xyes" ; then - AC_DEFINE(ENABLE_ADIOS) ENABLE_ADIOS=1 fi AC_SUBST(ENABLE_ADIOS) @@ -2603,10 +2674,10 @@ else # no name prefix end with ':' FSTYPE_PREFIX= else - # check if name prefix is one of file system types known to ROMIO - romio_known_fstypes=(ufs nfs xfs pvfs2 gpfs panfs lustre daos testfs ime quobyte) + # check if name prefix is one of file system types known to MPI-IO and GIO + known_fstypes=(ufs nfs xfs pvfs2 gpfs panfs lustre daos testfs ime quobyte) known_fstype= - for pre in $romio_known_fstypes ; do + for pre in $known_fstypes ; do if test "$FSTYPE_PREFIX" = $pre ; then known_fstype=$pre break @@ -2622,6 +2693,7 @@ else fi fi AC_SUBST([FSTYPE_PREFIX]) +AC_DEFINE_UNQUOTED(TESTOUTDIR, ["$TESTOUTDIR"], [Output directory for tests]) # SEQ_CC is used to compile programs to be run sequentially, such as # pnetcdf_version, ncoffsets, and ncvalidator @@ -2691,7 +2763,7 @@ dnl Update the version information only immediately before a public release. dnl PnetCDF starts with 1:0:0 (shared library is first supported in 1.9.0) dnl because some package distributors, such as Debian, may have already built dnl PnetCDF with shared libraries. -ABIVERSION="7:0:0" +ABIVERSION="8:0:1" AC_SUBST(ABIVERSION) if test "$enable_versioning" = "yes" ; then ABIVERSIONFLAGS="-version-info \$(ABIVERSION)" @@ -2700,6 +2772,29 @@ else fi AC_SUBST(ABIVERSIONFLAGS) +AC_ARG_ENABLE([gio], + [AS_HELP_STRING([--disable-gio], + [Build without GIO library])], + [enable_gio=${enableval}], [enable_gio=yes] +) + +# check whether or not submodule GIO has been initialized +if ! test -f ${srcdir}/gio/configure.ac ; then + enable_gio=no +fi + +ENABLE_GIO=0 +if test "x$enable_gio" = xyes ; then + ENABLE_GIO=1 + # Tell GIO that it will be built as a sub-module of PnetCDF, + # so 'make install' will skip libgio.so + export GIO_SUB_BUILD=$enable_gio + AC_CONFIG_SUBDIRS([gio]) + AC_SUBST([GIO_PACKAGE], [gio]) +fi +AC_SUBST(ENABLE_GIO) +AM_CONDITIONAL(ENABLE_GIO, [test "x$enable_gio" = xyes]) + dnl AC_CONFIG_HEADERS([src/binding/f77/nfconfig_inc]) AC_CONFIG_FILES(Makefile \ src/Makefile \ @@ -2796,7 +2891,8 @@ echo \ Features: Build static libraries - ${enable_static} Build shared libraries - ${enable_shared} Build Fortran APIs - ${has_fortran} - Build C++ APIs - ${has_mpicxx}" + Build C++ APIs - ${has_mpicxx} + GIO support - ${enable_gio}" if test "x${enable_netcdf4}" = xyes; then echo "\ NetCDF-4 support - enabled" diff --git a/examples/C/Makefile.am b/examples/C/Makefile.am index 009095a774..f3f486e499 100644 --- a/examples/C/Makefile.am +++ b/examples/C/Makefile.am @@ -30,15 +30,12 @@ check_PROGRAMS = collective_write \ get_vara \ transpose \ transpose2D \ - vard_int \ - vard_bottom \ i_varn_int64 \ bput_varn_uint \ bput_varn_int64 \ fill_mode \ ghost_cell \ req_all \ - vard_mvars \ time_var \ create_from_cdl @@ -59,45 +56,56 @@ endif TESTS_ENVIRONMENT = export SED="$(SED)"; TESTS_ENVIRONMENT += export srcdir="$(srcdir)"; +TESTS_ENVIRONMENT += export top_builddir="$(top_builddir)"; TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; TESTS_ENVIRONMENT += export TESTMPIRUN="$(TESTMPIRUN)"; -TESTS_ENVIRONMENT += export PNETCDF_DEBUG="$(PNETCDF_DEBUG)"; TESTS_ENVIRONMENT += export TESTPROGRAMS="$(TESTPROGRAMS)"; TESTS_ENVIRONMENT += export check_PROGRAMS="$(check_PROGRAMS)"; -TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER="$(ENABLE_BURST_BUFFER)"; + +TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER=@ENABLE_BURST_BUFFER@; +TESTS_ENVIRONMENT += export ENABLE_THREAD_SAFE=@ENABLE_THREAD_SAFE@; +TESTS_ENVIRONMENT += export MIMIC_LUSTRE=@MIMIC_LUSTRE@; +TESTS_ENVIRONMENT += export ENABLE_GIO=@ENABLE_GIO@; NC_FILES = $(check_PROGRAMS:%=$(TESTOUTDIR)/%.nc) \ $(check_PROGRAMS:%=$(TESTOUTDIR)/%.bb.nc) CLEANFILES = core core.* *.gcda *.gcno *.gcov gmon.out \ - $(NC_FILES) $(TESTOUTDIR)/pthread.nc.* $(TESTOUTDIR)/testfile.nc + $(TESTOUTDIR)/pthread.nc.* $(TESTOUTDIR)/*.nc -EXTRA_DIST = parallel_run.sh run_c_examples.sh cdl_header.txt +EXTRA_DIST = run_c_examples.sh cdl_header.txt ptest ptest4: $(check_PROGRAMS) @echo "===========================================================" @echo " $(subdir): Parallel testing on 4 MPI processes" @echo "===========================================================" @$(TESTS_ENVIRONMENT) \ - $(srcdir)/parallel_run.sh 4 || exit 1 + $(srcdir)/../parallel_run.sh 4 "C " || exit 1 -ptest8: $(check_PROGRAMS) +ptest3: $(check_PROGRAMS) @echo "===========================================================" - @echo " $(subdir): Parallel testing on 8 MPI processes" + @echo " $(subdir): Parallel testing on 3 MPI processes" @echo "===========================================================" @$(TESTS_ENVIRONMENT) \ - $(srcdir)/parallel_run.sh 8 || exit 1 + $(srcdir)/../parallel_run.sh 3 "C " || exit 1 -ptest3: $(check_PROGRAMS) +ptest6: $(check_PROGRAMS) @echo "===========================================================" - @echo " $(subdir): Parallel testing on 3 MPI processes" + @echo " $(subdir): Parallel testing on 6 MPI processes" + @echo "===========================================================" + @$(TESTS_ENVIRONMENT) \ + $(srcdir)/../parallel_run.sh 6 "C " || exit 1 + +ptest8: $(check_PROGRAMS) + @echo "===========================================================" + @echo " $(subdir): Parallel testing on 8 MPI processes" @echo "===========================================================" @$(TESTS_ENVIRONMENT) \ - $(srcdir)/parallel_run.sh 3 || exit 1 + $(srcdir)/../parallel_run.sh 8 "C " || exit 1 -ptests: ptest3 ptest4 ptest8 -ptest2 ptest6 ptest10: +ptests: ptest4 ptest6 +ptest2 ptest10: # build check targets but not invoke tests-local: all $(check_PROGRAMS) @@ -107,6 +115,6 @@ tests-local: all $(check_PROGRAMS) install-exec-hook: @if test "x$(example_execbindir)" != x ; then \ cp -f $(srcdir)/cdl_header.txt $(DESTDIR)/$(example_execbindir) ; \ - $(SED_I) -e 's|check_PROGRAMS|$(check_PROGRAMS)|g ; s|TESTOUTDIR|$(TESTOUTDIR)|g ; s|TESTMPIRUN|$(TESTMPIRUN)|g ; s|SED_CMD|$(SED)|g ; s|ENABLE_BURST_BUFFER|$(ENABLE_BURST_BUFFER)|g ; s|ENABLE_NETCDF4|$(ENABLE_NETCDF4)|g' $(DESTDIR)/$(example_execbindir)/run_c_examples.sh ; \ + $(SED_I) -e 's|check_PROGRAMS|$(check_PROGRAMS)|g ; s|TESTOUTDIR|$(TESTOUTDIR)|g ; s|TESTMPIRUN|$(TESTMPIRUN)|g ; s|SED_CMD|$(SED)|g ; s|ENABLE_BURST_BUFFER|@ENABLE_BURST_BUFFER@|g ; s|ENABLE_NETCDF4|@ENABLE_NETCDF4@|g' $(DESTDIR)/$(example_execbindir)/run_c_examples.sh ; \ fi diff --git a/examples/C/block_cyclic.c b/examples/C/block_cyclic.c index fa5c2a5edc..c45d29021d 100644 --- a/examples/C/block_cyclic.c +++ b/examples/C/block_cyclic.c @@ -83,7 +83,7 @@ static int verbose; -#define ERR {if(err!=NC_NOERR){printf("Error at %s:%d : %s\n", __FILE__,__LINE__, ncmpi_strerror(err));nerrs++;}} +#define ERR {if(err!=NC_NOERR){fprintf(stderr,"Error at %s:%d : %s\n", __FILE__,__LINE__, ncmpi_strerror(err));nerrs++;}} static void usage(char *argv0) @@ -122,7 +122,7 @@ pnetcdf_check_mem_usage(MPI_Comm comm) sum_size); } else if (err != NC_ENOTENABLED) { - printf("Error at %s:%d: %s\n", __FILE__,__LINE__,ncmpi_strerror(err)); + fprintf(stderr,"Error at %s:%d: %s\n", __FILE__,__LINE__,ncmpi_strerror(err)); nerrs++; } return nerrs; @@ -237,7 +237,7 @@ int main(int argc, char** argv) { /* check status of all requests */ for (i=0; i------------------------------------------------------------*/ diff --git a/examples/C/mput.c b/examples/C/mput.c index 51978655c1..b895e6cfe2 100644 --- a/examples/C/mput.c +++ b/examples/C/mput.c @@ -54,7 +54,7 @@ static int verbose; -#define ERR {if(err!=NC_NOERR){printf("Error at %s:%d : %s\n", __FILE__,__LINE__, ncmpi_strerror(err));nerrs++;}} +#define ERR {if(err!=NC_NOERR){fprintf(stderr,"Error at %s:%d : %s\n", __FILE__,__LINE__, ncmpi_strerror(err));nerrs++;}} static void usage(char *argv0) @@ -93,7 +93,7 @@ pnetcdf_check_mem_usage(MPI_Comm comm) sum_size); } else if (err != NC_ENOTENABLED) { - printf("Error at %s:%d: %s\n", __FILE__,__LINE__,ncmpi_strerror(err)); + fprintf(stderr,"Error at %s:%d: %s\n", __FILE__,__LINE__,ncmpi_strerror(err)); nerrs++; } return nerrs; diff --git a/examples/C/nonblocking_write.c b/examples/C/nonblocking_write.c index 937d0da382..f42c24a95b 100644 --- a/examples/C/nonblocking_write.c +++ b/examples/C/nonblocking_write.c @@ -48,7 +48,7 @@ static int verbose; -#define ERR {if(err!=NC_NOERR){printf("Error at %s:%d : %s\n", __FILE__,__LINE__, ncmpi_strerror(err));nerrs++;}} +#define ERR {if(err!=NC_NOERR){fprintf(stderr,"Error at %s:%d : %s\n", __FILE__,__LINE__, ncmpi_strerror(err));nerrs++;}} static void usage(char *argv0) @@ -90,7 +90,7 @@ pnetcdf_check_mem_usage(MPI_Comm comm) sum_size); } else if (err != NC_ENOTENABLED) { - printf("Error at %s:%d: %s\n", __FILE__,__LINE__,ncmpi_strerror(err)); + fprintf(stderr,"Error at %s:%d: %s\n", __FILE__,__LINE__,ncmpi_strerror(err)); nerrs++; } return nerrs; @@ -126,7 +126,7 @@ int main(int argc, char **argv) extern int optind; extern char *optarg; int i, j, k, err, nerrs=0, debug=0, use_contig_buf=0, use_bput=0; - int nprocs, len=0, nelems, rank; + int nprocs, len=0, nelems, rank, format, rec_bytes; int *sca_buf, *fix_buf[FIX_NVARS], *rec_buf[REC_NVARS]; int gsizes[NDIMS], psizes[NDIMS]; double write_timing, max_write_timing, write_bw; @@ -172,9 +172,14 @@ int main(int argc, char **argv) psizes[0],psizes[1],psizes[2]); starts[0] = 0; - starts[1] = (rank / (psizes[1] * psizes[2])) % psizes[0]; +#if NDIMS == 3 + starts[1] = rank / (psizes[1] * psizes[2]); starts[2] = (rank / psizes[2]) % psizes[1]; starts[3] = rank % psizes[2]; +#elif NDIMS == 2 + starts[1] = rank / psizes[1]; + starts[2] = rank % psizes[1]; +#endif counts[0] = 1; nelems = 1; @@ -185,6 +190,9 @@ int main(int argc, char **argv) counts[i+1] = len; } + if (verbose && rank == 0) + printf("psizes=%d %d %d\n",psizes[0],psizes[1],psizes[2]); + if (verbose && debug) printf("%2d: starts=%2lld %2lld %2lld %2lld counts=%2lld %2lld %2lld %2lld\n", rank, starts[0], starts[1], starts[2], starts[3], @@ -206,7 +214,7 @@ int main(int argc, char **argv) rec_buf[i] = sca_buf + SCA_NVARS + FIX_NVARS * nelems + nelems * i; } else { - /* allocate individual buffers separately +1 ensure non-contiguity*/ + /* allocate individual buffers separately +1 ensure non-contiguity */ sca_buf = (int*) malloc(sizeof(int) * (SCA_NVARS+1)); for (i=0; i 0); } diff --git a/examples/adios/read_var.c b/examples/adios/read_var.c index 867e3fd1fd..881adfdc33 100644 --- a/examples/adios/read_var.c +++ b/examples/adios/read_var.c @@ -44,7 +44,7 @@ static int verbose; /* Handle errors by printing an error message and exiting with a * non-zero status. */ #define ERRCODE 2 -#define ERR(e) {printf("Error: %s\n", nc_strerror(e)); exit(ERRCODE);} +#define ERR(e) {fprintf(stderr,"Error: %s\n", nc_strerror(e)); exit(ERRCODE);} static void usage(char *argv0) @@ -86,7 +86,7 @@ int main(int argc, char** argv) { snprintf(filename, 256, "%s", argv[optind]); else { if (rank==0) { - printf("Error: input file is required\n"); + fprintf(stderr,"Error: input file is required\n"); usage(argv[0]); } MPI_Finalize(); diff --git a/examples/adios/read_var_nb.c b/examples/adios/read_var_nb.c index 3d490cf4a1..2bc1645413 100644 --- a/examples/adios/read_var_nb.c +++ b/examples/adios/read_var_nb.c @@ -44,7 +44,7 @@ static int verbose; /* Handle errors by printing an error message and exiting with a * non-zero status. */ #define ERRCODE 2 -#define ERR(e) {printf("Error: %s\n", nc_strerror(e)); exit(ERRCODE);} +#define ERR(e) {fprintf(stderr,"Error: %s\n", nc_strerror(e)); exit(ERRCODE);} static void usage(char *argv0) @@ -88,7 +88,7 @@ int main(int argc, char** argv) { snprintf(filename, 256, "%s", argv[optind]); else { if (rank==0) { - printf("Error: input file is required\n"); + fprintf(stderr,"Error: input file is required\n"); usage(argv[0]); } MPI_Finalize(); diff --git a/examples/burst_buffer/Makefile.am b/examples/burst_buffer/Makefile.am index 162770fcfe..ff3c5d4b08 100644 --- a/examples/burst_buffer/Makefile.am +++ b/examples/burst_buffer/Makefile.am @@ -27,40 +27,50 @@ DATA_FILES = $(NC_FILES:%=%_*.data) CLEANFILES = $(NC_FILES) core core.* *.gcda *.gcno *.gcov gmon.out \ $(META_FILES) $(DATA_FILES) -EXTRA_DIST = parallel_run.sh - TESTS_ENVIRONMENT = export SED="$(SED)"; TESTS_ENVIRONMENT += export srcdir="$(srcdir)"; +TESTS_ENVIRONMENT += export top_builddir="$(top_builddir)"; TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; TESTS_ENVIRONMENT += export TESTMPIRUN="$(TESTMPIRUN)"; -TESTS_ENVIRONMENT += export PNETCDF_DEBUG="$(PNETCDF_DEBUG)"; TESTS_ENVIRONMENT += export TESTPROGRAMS="$(TESTPROGRAMS)"; TESTS_ENVIRONMENT += export check_PROGRAMS="$(check_PROGRAMS)"; +TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER=@ENABLE_BURST_BUFFER@; +TESTS_ENVIRONMENT += export ENABLE_THREAD_SAFE=@ENABLE_THREAD_SAFE@; +TESTS_ENVIRONMENT += export MIMIC_LUSTRE=@MIMIC_LUSTRE@; +TESTS_ENVIRONMENT += export ENABLE_GIO=@ENABLE_GIO@; + ptest ptest4: $(check_PROGRAMS) @echo "===========================================================" @echo " $(subdir): Parallel testing on 4 MPI processes" @echo "===========================================================" @$(TESTS_ENVIRONMENT) \ - $(srcdir)/parallel_run.sh 4 || exit 1 + $(srcdir)/../parallel_run.sh 4 "BB " || exit 1 -ptest8: $(check_PROGRAMS) +ptest3: $(check_PROGRAMS) @echo "===========================================================" - @echo " $(subdir): Parallel testing on 8 MPI processes" + @echo " $(subdir): Parallel testing on 3 MPI processes" @echo "===========================================================" @$(TESTS_ENVIRONMENT) \ - $(srcdir)/parallel_run.sh 8 || exit 1 + $(srcdir)/../parallel_run.sh 3 "BB "|| exit 1 -ptest3: $(check_PROGRAMS) +ptest6: $(check_PROGRAMS) @echo "===========================================================" - @echo " $(subdir): Parallel testing on 3 MPI processes" + @echo " $(subdir): Parallel testing on 6 MPI processes" + @echo "===========================================================" + @$(TESTS_ENVIRONMENT) \ + $(srcdir)/../parallel_run.sh 6 "BB "|| exit 1 + +ptest8: $(check_PROGRAMS) + @echo "===========================================================" + @echo " $(subdir): Parallel testing on 8 MPI processes" @echo "===========================================================" @$(TESTS_ENVIRONMENT) \ - $(srcdir)/parallel_run.sh 3 || exit 1 + $(srcdir)/../parallel_run.sh 8 "BB "|| exit 1 -ptests: ptest3 ptest4 ptest8 -ptest2 ptest6 ptest10: +ptests: ptest4 ptest6 +ptest2 ptest10: # build check targets but not invoke tests-local: all $(check_PROGRAMS) diff --git a/examples/burst_buffer/create_open.c b/examples/burst_buffer/create_open.c index 8c7b1c6d83..d202b8cfe9 100644 --- a/examples/burst_buffer/create_open.c +++ b/examples/burst_buffer/create_open.c @@ -45,7 +45,7 @@ #include #include -#define ERR {if(err!=NC_NOERR){printf("Error at line %d in %s: %s\n", __LINE__,__FILE__, ncmpi_strerror(err));nerrs++;}} +#define ERR {if(err!=NC_NOERR){fprintf(stderr,"Error at line %d in %s: %s\n", __LINE__,__FILE__, ncmpi_strerror(err));nerrs++;}} static void usage(char *argv0) diff --git a/examples/burst_buffer/nonblocking.c b/examples/burst_buffer/nonblocking.c index 70683f75f5..83794481b8 100644 --- a/examples/burst_buffer/nonblocking.c +++ b/examples/burst_buffer/nonblocking.c @@ -72,7 +72,7 @@ #include #include -#define ERR {if(err!=NC_NOERR){printf("Error at line %d in %s: %s\n", __LINE__,__FILE__, ncmpi_strerror(err));nerrs++;}} +#define ERR {if(err!=NC_NOERR){fprintf(stderr,"Error at line %d in %s: %s\n", __LINE__,__FILE__, ncmpi_strerror(err));nerrs++;}} static void usage(char *argv0) diff --git a/examples/burst_buffer/parallel_run.sh b/examples/burst_buffer/parallel_run.sh index 9fe960c9cf..450c5d77cc 100755 --- a/examples/burst_buffer/parallel_run.sh +++ b/examples/burst_buffer/parallel_run.sh @@ -20,30 +20,39 @@ MPIRUN=`echo ${TESTMPIRUN} | ${SED} -e "s/NP/$1/g"` # let NTHREADS=$1*6-1 NTHREADS=`expr $1 \* 6 - 1` -# echo "PNETCDF_DEBUG = ${PNETCDF_DEBUG}" -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" +if test "x$ENABLE_GIO" = x0 ; then + IO_MODES="mpiio" else - safe_modes="0" + IO_MODES="gio mpiio" fi # prevent user environment setting of PNETCDF_HINTS to interfere unset PNETCDF_HINTS for i in ${check_PROGRAMS} ; do - # echo "---- exec=$i" - for j in ${safe_modes} ; do - for intra_aggr in 0 1 ; do - if test "$j" = 1 ; then # test only in safe mode - export PNETCDF_HINTS="romio_no_indep_rw=true" + for io_mode in $IO_MODES ; do + if test "x$io_mode" = xmpiio ; then + USEMPIO_HINTS="nc_driver=mpiio" else - export PNETCDF_HINTS= + USEMPIO_HINTS="nc_driver=gio" fi + for intra_aggr in 0 1 ; do if test "$intra_aggr" = 1 ; then - export PNETCDF_HINTS="${PNETCDF_HINTS};nc_num_aggrs_per_node=2" + INA_HINTS="nc_num_aggrs_per_node=2" + else + INA_HINTS="nc_num_aggrs_per_node=0" fi - export PNETCDF_SAFE_MODE=$j - # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" + + PNETCDF_HINTS= + if test "x$USEMPIO_HINTS" != x ; then + PNETCDF_HINTS="$USEMPIO_HINTS;$PNETCDF_HINTS" + fi + if test "x$INA_HINTS" != x ; then + PNETCDF_HINTS="$INA_HINTS;$PNETCDF_HINTS" + fi + + export PNETCDF_HINTS="$PNETCDF_HINTS" + # echo "PNETCDF_HINTS=$PNETCDF_HINTS" # echo "${MPIRUN} ./$i -q -b ${TESTOUTDIR} ${TESTOUTDIR}/$i.nc" ${MPIRUN} ./$i -q -b ${TESTOUTDIR} ${TESTOUTDIR}/$i.nc diff --git a/examples/parallel_run.sh b/examples/parallel_run.sh new file mode 100755 index 0000000000..302d92a001 --- /dev/null +++ b/examples/parallel_run.sh @@ -0,0 +1,218 @@ +#!/bin/bash +# +# Copyright (C) 2018, Northwestern University and Argonne National Laboratory +# See COPYRIGHT notice in top-level directory. +# + +# Exit immediately if a command exits with a non-zero status. +# set -e + +DRY_RUN=no +VERBOSE=no + +exe_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + + cmd=`basename $1` + if test "x$MIMIC_LUSTRE" = x1 && test "x$cmd" = xncmpidiff ; then + # echo "export MIMIC_STRIPE_SIZE=1048576" + export MIMIC_STRIPE_SIZE=1048576 + fi + + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $MPIRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $MPIRUN $@ + fi + if test $? != 0 ; then + echo "FAIL: nprocs=$1 ---- $i $TEST_OPTS" + exit 1 + fi + + if test "x$MIMIC_LUSTRE" = x1 && test "x$cmd" = xncmpidiff ; then + # echo "unset MIMIC_STRIPE_SIZE" + unset MIMIC_STRIPE_SIZE + fi +} + +seq_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $TESTSEQRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $TESTSEQRUN $@ + fi +} + +VALIDATOR=../../src/utils/ncvalidator/ncvalidator +NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff + +MPIRUN=`echo ${TESTMPIRUN} | ${SED} -e "s/NP/$1/g"` +# echo "MPIRUN = ${MPIRUN}" +# echo "check_PROGRAMS=${check_PROGRAMS}" + +# remove file system type prefix if there is any +OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` + +# let NTHREADS=$1*6-1 +NTHREADS=`expr $1 \* 6 - 1` + +if test "x$ENABLE_GIO" = x0 ; then + IO_MODES="mpiio" +else + IO_MODES="gio mpiio" +fi + +# prevent user environment setting of PNETCDF_HINTS to interfere +unset PNETCDF_HINTS + +PNETCDF_DEBUG_MODE=`grep PNETCDF_DEBUG_MODE ${top_builddir}/src/include/pnetcdf.h | tr -s ' ' | cut -d ' ' -f 3` + +for i in ${check_PROGRAMS} ; do + # Capture start time in seconds and nanoseconds + start_time=$(date +%s.%1N) + + if test "${PNETCDF_DEBUG_MODE}" = 1 ; then # test only in safe mode + safe_hint=" SAFE" + else + safe_hint="NOSAFE" + fi + OUT_PREFIX="${TESTOUTDIR}/$i" + + for io_mode in $IO_MODES ; do + if test "x$io_mode" = xmpiio ; then + USEMPIO_HINTS="nc_driver=mpiio" + DRIVER_OUT_FILE="${OUT_PREFIX}.mpio" + driver_hint=" MPIO" + else + USEMPIO_HINTS="nc_driver=gio" + DRIVER_OUT_FILE="${OUT_PREFIX}.gio" + driver_hint="GIO" + fi + + for intra_aggr in 0 1 ; do + if test "$intra_aggr" = 1 ; then + INA_HINTS="nc_num_aggrs_per_node=2" + INA_OUT_FILE="${DRIVER_OUT_FILE}.ina" + ina_hint=" INA" + else + INA_HINTS="nc_num_aggrs_per_node=0" + INA_OUT_FILE="${DRIVER_OUT_FILE}" + ina_hint="NOINA" + fi + + OUT_FILE=$INA_OUT_FILE + + PNETCDF_HINTS= + if test "x$USEMPIO_HINTS" != x ; then + PNETCDF_HINTS="$USEMPIO_HINTS;$PNETCDF_HINTS" + fi + if test "x$INA_HINTS" != x ; then + PNETCDF_HINTS="$INA_HINTS;$PNETCDF_HINTS" + fi + if test "x$MIMIC_LUSTRE" != x1 ; then + PNETCDF_HINTS="cb_nodes=2;$PNETCDF_HINTS" + fi + + export PNETCDF_HINTS="$PNETCDF_HINTS" + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line ${LINENO}: PNETCDF_HINTS=$PNETCDF_HINTS" + fi + + TEST_OPTS="$safe_hint $driver_hint $ina_hint" + + CMD_OPT=-q + IN_FILE= + if test "$i" = create_from_cdl ; then + IN_FILE=${srcdir}/cdl_header.txt + fi + + if test "$i" = pthread ; then + # each MPI process created 6 threads + exe_cmd ./$i $CMD_OPT ${OUT_FILE}.nc + for k in `seq 0 ${NTHREADS}` ; do + seq_cmd ${VALIDATOR} -q ${OUT_FILE}.nc.$k + done + continue + elif test "$i" = put_vara ; then + exe_cmd ./$i $CMD_OPT ${OUT_FILE}.nc + seq_cmd ${VALIDATOR} -q ${OUT_FILE}.nc + + exe_cmd ./get_vara $CMD_OPT ${OUT_FILE}.nc + elif test "$i" = get_vara ; then + continue + elif test "$i" = create_from_cdl ; then + # create_from_cdl reads a CDL header file + exe_cmd ./$i $CMD_OPT -o ${OUT_FILE}.nc $IN_FILE + else + exe_cmd ./$i $CMD_OPT ${OUT_FILE}.nc + fi + + seq_cmd ${VALIDATOR} -q ${OUT_FILE}.nc + + if test "x${ENABLE_BURST_BUFFER}" = x1 ; then + saved_PNETCDF_HINTS=${PNETCDF_HINTS} + export PNETCDF_HINTS="${PNETCDF_HINTS};nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" + if test "$i" = create_from_cdl ; then + # create_from_cdl reads a CDL header file + exe_cmd ./$i -q -o ${OUT_FILE}.bb.nc $IN_FILE + else + exe_cmd ./$i $CMD_OPT ${OUT_FILE}.bb.nc + fi + export PNETCDF_HINTS=${saved_PNETCDF_HINTS} + + seq_cmd ${VALIDATOR} -q ${OUT_FILE}.bb.nc + + # compare file header only for large file tests + DIFF_OPT="-q" + if test "$i" = create_from_cdl ; then + DIFF_OPT+=" -h" + fi + exe_cmd ${NCMPIDIFF} $DIFF_OPT $OUT_FILE.nc $OUT_FILE.bb.nc + fi + + if test "x${ENABLE_NETCDF4}" = x1 ; then + exe_cmd ./$i ${OUT_FILE}.nc4 4 + # Validator does not support nc4 + fi + done # intra_aggr + done # io_mode + + if test "$i" = get_vara ; then + continue + fi + + DIFF_OPT="-q" + if test "$i" = create_from_cdl ; then + DIFF_OPT+=" -h" + fi + if test "$i" = pthread ; then + for j in `seq 0 ${NTHREADS}` ; do + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.mpio.ina.nc.$j + if test "x$ENABLE_GIO" = x1 ; then + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.gio.nc.$j + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.gio.nc.$j $OUT_PREFIX.gio.ina.nc.$j + fi + done + else + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.mpio.ina.nc + if test "x$ENABLE_GIO" = x1 ; then + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.gio.nc + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.gio.nc $OUT_PREFIX.gio.ina.nc + fi + fi + + rm -f ${OUTDIR}/$i*nc* + + end_time=$(date +%s.%1N) + + # Calculate difference (requires bc for floating point math) + elapsed_time=$(echo "$end_time - $start_time" | bc) + + fixed_length=48 + printf "*** TESTING %-${fixed_length}s -- pass (%4ss)\n" "$i" "$elapsed_time" + +done # check_PROGRAMS + diff --git a/examples/tutorial/Makefile.am b/examples/tutorial/Makefile.am index 20859a3ff3..44b93ce425 100644 --- a/examples/tutorial/Makefile.am +++ b/examples/tutorial/Makefile.am @@ -64,13 +64,17 @@ endif TESTS_ENVIRONMENT = export SED="$(SED)"; TESTS_ENVIRONMENT += export srcdir="$(srcdir)"; +TESTS_ENVIRONMENT += export top_builddir="$(top_builddir)"; TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; TESTS_ENVIRONMENT += export TESTMPIRUN="$(TESTMPIRUN)"; -TESTS_ENVIRONMENT += export PNETCDF_DEBUG="$(PNETCDF_DEBUG)"; TESTS_ENVIRONMENT += export TESTPROGRAMS="$(TESTPROGRAMS)"; TESTS_ENVIRONMENT += export check_PROGRAMS="$(check_PROGRAMS)"; -TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER="$(ENABLE_BURST_BUFFER)"; + +TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER=@ENABLE_BURST_BUFFER@; +TESTS_ENVIRONMENT += export ENABLE_THREAD_SAFE=@ENABLE_THREAD_SAFE@; +TESTS_ENVIRONMENT += export MIMIC_LUSTRE=@MIMIC_LUSTRE@; +TESTS_ENVIRONMENT += export ENABLE_GIO=@ENABLE_GIO@; NC_FILES = $(check_PROGRAMS:%=$(TESTOUTDIR)/%.nc) \ $(check_PROGRAMS:%=$(TESTOUTDIR)/%.bb.nc) @@ -79,6 +83,8 @@ CLEANFILES = core core.* *.gcda *.gcno *.gcov gmon.out \ $(TESTOUTDIR)/pnetcdf-write-nfiles.*.nc \ $(NC_FILES) +check_SCRIPTS = parallel_run.sh + EXTRA_DIST = parallel_run.sh ptest ptest4: $(check_PROGRAMS) diff --git a/examples/tutorial/parallel_run.sh b/examples/tutorial/parallel_run.sh index bfd3e65a9e..f6db645afd 100755 --- a/examples/tutorial/parallel_run.sh +++ b/examples/tutorial/parallel_run.sh @@ -5,7 +5,24 @@ # # Exit immediately if a command exits with a non-zero status. -set -e +# set -e + +DRY_RUN=no +VERBOSE=no + +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $MPIRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $MPIRUN $@ + fi + if test $? != 0 ; then + echo "FAIL: nprocs=$1 ---- $i $TEST_OPTS" + exit 1 + fi +} VALIDATOR=../../src/utils/ncvalidator/ncvalidator NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff @@ -20,51 +37,58 @@ MPIRUN=`echo ${TESTMPIRUN} | ${SED} -e "s/NP/$1/g"` # let NTHREADS=$1*6-1 NTHREADS=`expr $1 \* 6 - 1` -# echo "PNETCDF_DEBUG = ${PNETCDF_DEBUG}" -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" +if test "x$ENABLE_GIO" = x0 ; then + IO_MODES="mpiio" else - safe_modes="0" + IO_MODES="gio mpiio" fi # prevent user environment setting of PNETCDF_HINTS to interfere unset PNETCDF_HINTS for i in ${check_PROGRAMS} ; do - for j in ${safe_modes} ; do - for intra_aggr in 0 1 ; do - if test "$j" = 1 ; then # test only in safe mode - export PNETCDF_HINTS="romio_no_indep_rw=true" + # Capture start time in seconds and nanoseconds + start_time=$(date +%s.%1N) + + for io_mode in $IO_MODES ; do + if test "x$io_mode" = xmpiio ; then + USEMPIO_HINTS="nc_driver=mpiio" else - export PNETCDF_HINTS= + USEMPIO_HINTS="nc_driver=gio" fi + for intra_aggr in 0 1 ; do if test "$intra_aggr" = 1 ; then - export PNETCDF_HINTS="${PNETCDF_HINTS};nc_num_aggrs_per_node=2" + INA_HINTS="nc_num_aggrs_per_node=2" + else + INA_HINTS="nc_num_aggrs_per_node=0" fi - export PNETCDF_SAFE_MODE=$j - # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" + + PNETCDF_HINTS= + if test "x$USEMPIO_HINTS" != x ; then + PNETCDF_HINTS="$USEMPIO_HINTS;$PNETCDF_HINTS" + fi + if test "x$INA_HINTS" != x ; then + PNETCDF_HINTS="$INA_HINTS;$PNETCDF_HINTS" + fi + if test "x$MIMIC_LUSTRE" != x1 ; then + PNETCDF_HINTS="cb_nodes=2;$PNETCDF_HINTS" + fi + + export PNETCDF_HINTS="$PNETCDF_HINTS" + # echo "PNETCDF_HINTS=$PNETCDF_HINTS" if test $i = "pnetcdf-read-from-master" ; then - ${MPIRUN} ./$i ${TESTOUTDIR}/pnetcdf-write-from-master.nc + run_cmd ./$i ${TESTOUTDIR}/pnetcdf-write-from-master.nc elif test $i = "pnetcdf-read-nfiles" ; then - ${MPIRUN} ./$i ${TESTOUTDIR}/pnetcdf-write-nfiles.nc + run_cmd ./$i ${TESTOUTDIR}/pnetcdf-write-nfiles.nc elif test $i = "pnetcdf-read-standard" ; then - ${MPIRUN} ./$i ${TESTOUTDIR}/pnetcdf-write-standard.nc + run_cmd ./$i ${TESTOUTDIR}/pnetcdf-write-standard.nc elif test $i = "pnetcdf-read-flexible" ; then - ${MPIRUN} ./$i ${TESTOUTDIR}/pnetcdf-write-flexible.nc + run_cmd ./$i ${TESTOUTDIR}/pnetcdf-write-flexible.nc elif test $i = "pnetcdf-read-nb" ; then - ${MPIRUN} ./$i ${TESTOUTDIR}/pnetcdf-write-nb.nc + run_cmd ./$i ${TESTOUTDIR}/pnetcdf-write-nb.nc else - ${MPIRUN} ./$i ${TESTOUTDIR}/$i.nc - fi - if test $? = 0 ; then - if test $i = "pnetcdf-write-bufferedf77" ; then - echo "PASS: F77 parallel run on $1 processes --------------- $i" - elif test $i = "pnetcdf-write-bufferedf" ; then - echo "PASS: F90 parallel run on $1 processes --------------- $i" - else - echo "PASS: C parallel run on $1 processes --------------- $i" - fi + run_cmd ./$i ${TESTOUTDIR}/$i.nc fi if test "$i" = pthread ; then @@ -95,26 +119,17 @@ for i in ${check_PROGRAMS} ; do saved_PNETCDF_HINTS=${PNETCDF_HINTS} export PNETCDF_HINTS="${PNETCDF_HINTS};nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" if test $i = "pnetcdf-read-from-master" ; then - ${MPIRUN} ./$i ${TESTOUTDIR}/pnetcdf-write-from-master.bb.nc + run_cmd ./$i ${TESTOUTDIR}/pnetcdf-write-from-master.bb.nc elif test $i = "pnetcdf-read-nfiles" ; then - ${MPIRUN} ./$i ${TESTOUTDIR}/pnetcdf-write-nfiles.bb.nc + run_cmd ./$i ${TESTOUTDIR}/pnetcdf-write-nfiles.bb.nc elif test $i = "pnetcdf-read-standard" ; then - ${MPIRUN} ./$i ${TESTOUTDIR}/pnetcdf-write-standard.bb.nc + run_cmd ./$i ${TESTOUTDIR}/pnetcdf-write-standard.bb.nc elif test $i = "pnetcdf-read-flexible" ; then - ${MPIRUN} ./$i ${TESTOUTDIR}/pnetcdf-write-flexible.bb.nc + run_cmd ./$i ${TESTOUTDIR}/pnetcdf-write-flexible.bb.nc elif test $i = "pnetcdf-read-nb" ; then - ${MPIRUN} ./$i ${TESTOUTDIR}/pnetcdf-write-nb.bb.nc + run_cmd ./$i ${TESTOUTDIR}/pnetcdf-write-nb.bb.nc else - ${MPIRUN} ./$i ${TESTOUTDIR}/$i.bb.nc - fi - if test $? = 0 ; then - if test $i = "pnetcdf-write-bufferedf77" ; then - echo "PASS: F77 parallel run on $1 processes --------------- $i" - elif test $i = "pnetcdf-write-bufferedf" ; then - echo "PASS: F90 parallel run on $1 processes --------------- $i" - else - echo "PASS: C parallel run on $1 processes --------------- $i" - fi + run_cmd ./$i ${TESTOUTDIR}/$i.bb.nc fi export PNETCDF_HINTS=${saved_PNETCDF_HINTS} @@ -131,18 +146,25 @@ for i in ${check_PROGRAMS} ; do # echo "--- validating file ${TESTOUTDIR}/$i.bb.nc" ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.bb.nc - # echo "--- ncmpidiff $i.nc $i.bb.nc ---" - ${MPIRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/$i.nc ${TESTOUTDIR}/$i.bb.nc + run_cmd ${NCMPIDIFF} -q ${TESTOUTDIR}/$i.nc ${TESTOUTDIR}/$i.bb.nc fi fi if test "x${ENABLE_NETCDF4}" = x1 ; then - # echo "test netCDF-4 feature" - ${MPIRUN} ./$i ${TESTOUTDIR}/$i.nc4 4 + run_cmd ./$i ${TESTOUTDIR}/$i.nc4 4 # Validator does not support nc4 fi done done + + end_time=$(date +%s.%1N) + + # Calculate difference (requires bc for floating point math) + elapsed_time=$(echo "$end_time - $start_time" | bc) + + fixed_length=48 + printf "*** TESTING %-${fixed_length}s -- pass (%4ss)\n" "$i" "$elapsed_time" + done rm -f ${OUTDIR}/pnetcdf-*.nc diff --git a/examples/tutorial/pnetcdf-write-buffered.c b/examples/tutorial/pnetcdf-write-buffered.c index cca1d536b2..ab92d7f3fc 100644 --- a/examples/tutorial/pnetcdf-write-buffered.c +++ b/examples/tutorial/pnetcdf-write-buffered.c @@ -13,7 +13,7 @@ #include #define ERRCODE 2 -#define ERR(e) {printf("Error at line %d: err=%d %s\n", __LINE__, e, ncmpi_strerror(e)); exit(ERRCODE);} +#define ERR(e) {fprintf(stderr,"Error at line %d: err=%d %s\n", __LINE__, e, ncmpi_strerror(e)); exit(ERRCODE);} /*----< main() >------------------------------------------------------------*/ int main(int argc, char **argv) { diff --git a/gio b/gio new file mode 160000 index 0000000000..4e7725c8ae --- /dev/null +++ b/gio @@ -0,0 +1 @@ +Subproject commit 4e7725c8ae050746d8d112e7bbc3decb144a2e65 diff --git a/m4/check_mpi.m4 b/m4/check_mpi.m4 index 3fdcb8867a..ff8caa9771 100644 --- a/m4/check_mpi.m4 +++ b/m4/check_mpi.m4 @@ -206,8 +206,9 @@ AC_DEFUN([LT_MPI_CHECK_SHLIB],[ if test "x$RM" = xrm || test "x$RM" = "x/bin/rm" ; then RM="$RM -f" fi - ac_ltcompile='./libtool --mode=compile $MPICC -c $CFLAGS $CPPFLAGS conftest.$ac_ext -o conftest.lo >&AS_MESSAGE_LOG_FD' - ac_ltlink_la='./libtool --mode=link $MPICC -rpath `pwd` $CFLAGS $LDFLAGS -o libconftest.la conftest.lo $LIBS >&AS_MESSAGE_LOG_FD' + # must add '--tag=CC', otherwise autotool 2.73 failed. + ac_ltcompile='./libtool --tag=CC --mode=compile $MPICC -c $CFLAGS $CPPFLAGS conftest.$ac_ext -o conftest.lo >&AS_MESSAGE_LOG_FD' + ac_ltlink_la='./libtool --tag=CC --mode=link $MPICC -rpath `pwd` $CFLAGS $LDFLAGS -o libconftest.la conftest.lo $LIBS >&AS_MESSAGE_LOG_FD' AS_IF([AC_TRY_EVAL([ac_ltcompile]) && AC_TRY_EVAL([ac_ltlink_la]) && AC_TRY_COMMAND([test -s libconftest.la])], diff --git a/m4/check_utils.m4 b/m4/check_utils.m4 index 02f3c81dc4..e9a0db31f7 100644 --- a/m4/check_utils.m4 +++ b/m4/check_utils.m4 @@ -356,5 +356,5 @@ AC_DEFUN([LT_AC_LINK_SHLIB_IFELSE],[ m4_ifvaln([$3], [$3])]) rm -rf $objdir rm -f conftest* libconftest*[]dnl -])# _AC_LINK_IFELSE +])# LT_AC_LINK_SHLIB_IFELSE diff --git a/m4/utils.m4 b/m4/utils.m4 index 3ef65eb059..25024922f9 100644 --- a/m4/utils.m4 +++ b/m4/utils.m4 @@ -109,6 +109,11 @@ dnl define(`CDF2_ITYPE_LIST', `text, schar, short, int, long, float, double')dnl dnl dnl +dnl dnl dnl +dnl +define(`CDF5_EXTRA_ITYPE_LIST', `uchar, ushort, uint, longlong, ulonglong')dnl +dnl +dnl define(`CollIndep', `ifelse(`$1', `_all', `NC_REQ_COLL', `NC_REQ_INDEP')')dnl define(`ReadWrite', `ifelse(`$1', `get', `NC_REQ_WR', `$1', `iget', `NC_REQ_RD', @@ -204,25 +209,25 @@ define(`IFMT',`ifelse( `$1', `longlong', `%lld', `$1', `ulonglong', `%llu')')dnl dnl -define(`PUT_VAR',`ifdef(`PNETCDF',`ncmpi_put_var_$1_all',`nc_put_var_$1')')dnl +define(`PUT_VAR',`ifdef(`PNETCDF',`ncmpi_put_var_$1$2',`nc_put_var_$1')')dnl dnl -define(`GET_VAR',`ifdef(`PNETCDF',`ncmpi_get_var_$1_all',`nc_get_var_$1')')dnl +define(`GET_VAR',`ifdef(`PNETCDF',`ncmpi_get_var_$1$2',`nc_get_var_$1')')dnl dnl -define(`PUT_VAR1',`ifdef(`PNETCDF',`ncmpi_put_var1_$1_all',`nc_put_var1_$1')')dnl +define(`PUT_VAR1',`ifdef(`PNETCDF',`ncmpi_put_var1_$1$2',`nc_put_var1_$1')')dnl dnl -define(`GET_VAR1',`ifdef(`PNETCDF',`ncmpi_get_var1_$1_all',`nc_get_var1_$1')')dnl +define(`GET_VAR1',`ifdef(`PNETCDF',`ncmpi_get_var1_$1$2',`nc_get_var1_$1')')dnl dnl -define(`PUT_VARA',`ifdef(`PNETCDF',`ncmpi_put_vara_$1_all',`nc_put_vara_$1')')dnl +define(`PUT_VARA',`ifdef(`PNETCDF',`ncmpi_put_vara_$1$2',`nc_put_vara_$1')')dnl dnl -define(`GET_VARA',`ifdef(`PNETCDF',`ncmpi_get_vara_$1_all',`nc_get_vara_$1')')dnl +define(`GET_VARA',`ifdef(`PNETCDF',`ncmpi_get_vara_$1$2',`nc_get_vara_$1')')dnl dnl -define(`PUT_VARS',`ifdef(`PNETCDF',`ncmpi_put_vars_$1_all',`nc_put_vars_$1')')dnl +define(`PUT_VARS',`ifdef(`PNETCDF',`ncmpi_put_vars_$1$2',`nc_put_vars_$1')')dnl dnl -define(`GET_VARS',`ifdef(`PNETCDF',`ncmpi_get_vars_$1_all',`nc_get_vars_$1')')dnl +define(`GET_VARS',`ifdef(`PNETCDF',`ncmpi_get_vars_$1$2',`nc_get_vars_$1')')dnl dnl -define(`PUT_VARM',`ifdef(`PNETCDF',`ncmpi_put_varm_$1_all',`nc_put_varm_$1')')dnl +define(`PUT_VARM',`ifdef(`PNETCDF',`ncmpi_put_varm_$1$2',`nc_put_varm_$1')')dnl dnl -define(`GET_VARM',`ifdef(`PNETCDF',`ncmpi_get_varm_$1_all',`nc_get_varm_$1')')dnl +define(`GET_VARM',`ifdef(`PNETCDF',`ncmpi_get_varm_$1$2',`nc_get_varm_$1')')dnl dnl define(`XTYPE_MAX',`ifelse( `$1', `text', `127', diff --git a/sneak_peek.md b/sneak_peek.md index 3635450639..ece9730550 100644 --- a/sneak_peek.md +++ b/sneak_peek.md @@ -2,11 +2,29 @@ This is essentially a placeholder for the next release note ... ------------------------------------------------------------------------------ -* New features - + none +* New feature + + Intra-node aggregation for read requests is added. This is the complement + to the write counterpart first implemented in version 1.14.0. Now the + intra-node aggregation feature supports both write and read operations. + This feature can be enabled by setting hint `nc_num_aggrs_per_node` to the + desired number of aggregators per compute node. * New optimization - + none + + When creating a new file on the Lustre file system, PnetCDF will try to set + the Lustre file striping count to the number of compute nodes (NUMA nodes) + allocated to the MPI job. See the section of new PnetCDF hints below for + detailed information. + +* New I/O driver + + A new internal I/O driver, named "GIO", is added. It is an alternative I/O + driver to the MPI-IO driver. GIO includes several implementations for + performance improvement, such as: + * automatically sets `cb_nodes` if this hint is not set by the user. + * supports `cb_nodes` to be set to a multiple of the number of OSTs on + Lustre. + * supports hint `cb_buffer_size` to be a multiple of file striping unit + size. + * supports Lustre overstriping through hint `overstriping_ratio`. * New Limitations + none @@ -22,11 +40,26 @@ This is essentially a placeholder for the next release note ... * New APIs + none +* APIs deprecated + + The "vard" APIs introduced in version 1.6.0 are now deprecated. These are + the API family that take an argument of MPI derived data type describing + the file access layout, which is used as the fileview by the underlying + MPI-IO library. The deprecation is because direct file access can be a + security risk and error prone. + * API syntax changes + none * API semantics updates - + none + + API `ncmpi_inq_header_size()` now can be called in the define mode. This + API returns the file header size with metadata defined by the time of the + call. The inquired file header size can be helpful to pick proper values + for arguments `h_minfree`, `v_align`, `v_minfree`, `r_align` when calling + API `ncmpi__enddef()` to make a sufficiently large free space for file + header extent and variable data sections to grow without moving data + already stored in the file, i.e. when adding new variables, dimensions, or + attributes. + See [PR #201](https://github.com/Parallel-NetCDF/PnetCDF/pull/201). * New error code precedence + none @@ -35,10 +68,33 @@ This is essentially a placeholder for the next release note ... + none * New error code - + none + + `NC_EFSTYPE` indicates an error when an invalid file system type is + detected by the GIO library. + + `NC_EDRIVER` indicates an invalid PnetCDF I/O driver is set in I/O hint. + The current valid drivers are "gio" and "mpiio". + + `NC_EFILEVIEW` indicates a PnetCDF internal error when creating an MPI + fileview whose offsets violate the MPI standard requirement of being in a + monotonically non-decreasing order. * New PnetCDF hints - + none + + `nc_data_move_chunk_size` -- When adding new data objects into an existing + file, the data sections may need to be moved to a higher file offset. The + movement is performed in chunks. This hint allows users to customized the + chunk size. The default is 1048576 bytes, i.e. 1 MiB. + See [PR #203](https://github.com/Parallel-NetCDF/PnetCDF/pull/203). + + `nc_file_striping` -- When creating a new file on the Lustre file system, + this hint advises PnetCDF to set the new file's striping configuration. The + hint value is either "auto" or "inherit". The former sets the new file's + striping unit to 1 MiB and striping count to the number of compute nodes + found in the MPI communicator passed to `ncmpi_create()`. The latter sets + the striping of new file to inherit its parent folder's striping settings, + if the folder's striping is set. However, if users also set the MPI-IO hint + `striping_factor` or `striping_unit`, then these MPI-IO hints will take a + higher precedence. This hint's default value is "auto". + See [PR #222](https://github.com/Parallel-NetCDF/PnetCDF/pull/222). + + `nc_driver` -- To select the internal I/O driver. String value of "gio" is + to use the GIO driver, an external library used as a sub-module of PnetCDF, + and "mpiio" the MPI-IO driver. The default is "gio". * New run-time environment variables + none @@ -53,7 +109,9 @@ This is essentially a placeholder for the next release note ... + none * Bug fixes - + none + + Fix data movement when new record variables are added to an existing file + that does not change the starting offset of record variable section. + See [PR #199](https://github.com/Parallel-NetCDF/PnetCDF/pull/199). * New example programs + none @@ -62,7 +120,10 @@ This is essentially a placeholder for the next release note ... + none * New test program - + none + + test/testcases/tst_grow_data.c -- adding new variables by re-entering the + define mode multiple time, but does not cause file header extent to grow. + It also tests a case when adding a new record variable that does not change + the starting offset of the record variable section in the file. * Issues with NetCDF library + none diff --git a/src/binding/cxx/ncmpiAtt.cpp b/src/binding/cxx/ncmpiAtt.cpp index 55fd35a1cb..7432d921cd 100644 --- a/src/binding/cxx/ncmpiAtt.cpp +++ b/src/binding/cxx/ncmpiAtt.cpp @@ -29,8 +29,8 @@ NcmpiAtt::NcmpiAtt() : {} // Constructor for non-null instances. -NcmpiAtt::NcmpiAtt(bool nullObject): - nullObject(nullObject), +NcmpiAtt::NcmpiAtt(bool nullObj): + nullObject(nullObj), groupId(-1), varId(-1) {} diff --git a/src/binding/cxx/ncmpiEnumType.cpp b/src/binding/cxx/ncmpiEnumType.cpp index 1edaacddfb..04935d8014 100644 --- a/src/binding/cxx/ncmpiEnumType.cpp +++ b/src/binding/cxx/ncmpiEnumType.cpp @@ -58,8 +58,8 @@ NcmpiEnumType::NcmpiEnumType(const NcmpiGroup& grp, const string& name): // constructor -NcmpiEnumType::NcmpiEnumType(const NcmpiType& ncmpiType): - NcmpiType(ncmpiType) +NcmpiEnumType::NcmpiEnumType(const NcmpiType& xType): + NcmpiType(xType) { // check the nctype object is the base of an Enum type if(getTypeClass() != NC_ENUM) throw NcmpiException("The NcmpiType object must be the base of an Enum type.",__FILE__,__LINE__); diff --git a/src/binding/cxx/ncmpiOpaqueType.cpp b/src/binding/cxx/ncmpiOpaqueType.cpp index a270dc2c5c..c2256fd6b4 100644 --- a/src/binding/cxx/ncmpiOpaqueType.cpp +++ b/src/binding/cxx/ncmpiOpaqueType.cpp @@ -50,8 +50,8 @@ NcmpiOpaqueType::NcmpiOpaqueType(const NcmpiGroup& grp, const string& name) : // constructor -NcmpiOpaqueType::NcmpiOpaqueType(const NcmpiType& ncmpiType) : - NcmpiType(ncmpiType) +NcmpiOpaqueType::NcmpiOpaqueType(const NcmpiType& xType) : + NcmpiType(xType) { // check the nctype object is the base of a Opaque type if(getTypeClass() != NC_OPAQUE) throw NcmpiException("The NcmpiType object must be the base of an Opaque type.",__FILE__,__LINE__); diff --git a/src/binding/cxx/ncmpiVlenType.cpp b/src/binding/cxx/ncmpiVlenType.cpp index aec268b5f5..59ae86d0b2 100644 --- a/src/binding/cxx/ncmpiVlenType.cpp +++ b/src/binding/cxx/ncmpiVlenType.cpp @@ -58,8 +58,8 @@ NcmpiVlenType::NcmpiVlenType(const NcmpiGroup& grp, const string& name) : {} // constructor -NcmpiVlenType::NcmpiVlenType(const NcmpiType& ncmpiType): - NcmpiType(ncmpiType) +NcmpiVlenType::NcmpiVlenType(const NcmpiType& xType): + NcmpiType(xType) { // check the nctype object is the base of a Vlen type if(getTypeClass() != NC_VLEN) throw NcmpiException("The NcmpiType object must be the base of a Vlen type.",__FILE__,__LINE__); diff --git a/src/binding/f77/pnetcdf.inc.in b/src/binding/f77/pnetcdf.inc.in index 9a5966a4d1..725650f396 100644 --- a/src/binding/f77/pnetcdf.inc.in +++ b/src/binding/f77/pnetcdf.inc.in @@ -14,10 +14,12 @@ integer PNETCDF_VERSION_MAJOR integer PNETCDF_VERSION_MINOR integer PNETCDF_VERSION_SUB + character*16 PNETCDF_VERSION_PRE parameter (PNETCDF_VERSION_MAJOR = @PNETCDF_VERSION_MAJOR@) parameter (PNETCDF_VERSION_MINOR = @PNETCDF_VERSION_MINOR@) parameter (PNETCDF_VERSION_SUB = @PNETCDF_VERSION_SUB@) + parameter (PNETCDF_VERSION_PRE = "@PNETCDF_VERSION_PRE@") ! ! list of PnetCDF options enabled/disabled at configure time diff --git a/src/binding/f90/nfmpi_constants.fh.in b/src/binding/f90/nfmpi_constants.fh.in index ea6d31d371..1d9f6acf6c 100644 --- a/src/binding/f90/nfmpi_constants.fh.in +++ b/src/binding/f90/nfmpi_constants.fh.in @@ -16,6 +16,8 @@ PNETCDF_VERSION_MINOR = @PNETCDF_VERSION_MINOR@, & PNETCDF_VERSION_SUB = @PNETCDF_VERSION_SUB@ + character(len=16), parameter :: PNETCDF_VERSION_PRE = "@PNETCDF_VERSION_PRE@" + ! ! list of PnetCDF options enabled/disabled at configure time ! diff --git a/src/dispatchers/Makefile.am b/src/dispatchers/Makefile.am index d6d9835e7e..56b224d17b 100644 --- a/src/dispatchers/Makefile.am +++ b/src/dispatchers/Makefile.am @@ -13,10 +13,6 @@ AM_CPPFLAGS += -I${top_builddir}/src/include AM_CPPFLAGS += -I${top_srcdir}/src/drivers/include AM_CPPFLAGS += @NETCDF4_INC@ @ADIOS_INC@ -if PNETCDF_DEBUG - AM_CPPFLAGS += -DPNETCDF_DEBUG -endif - noinst_LTLIBRARIES = libdispatchers.la M4FLAGS += -I${top_srcdir}/m4 diff --git a/src/dispatchers/attr_getput.m4 b/src/dispatchers/attr_getput.m4 index dcdca8355a..ac330887ff 100644 --- a/src/dispatchers/attr_getput.m4 +++ b/src/dispatchers/attr_getput.m4 @@ -198,12 +198,10 @@ check_consistency_put(MPI_Comm comm, /* check if buf contents is consistent across all processes */ if (root_nelems > 0) { /* non-scalar attribute */ /* note xsz is aligned, thus must use the exact size of buf */ - int rank, itype_size; + int itype_size; size_t buf_size; void *root_buf; - MPI_Comm_rank(comm, &rank); - /* for attributes, itype is nc_type, so its size is small. Thus, no * need to check against NC_MAX_INT. */ diff --git a/src/dispatchers/attribute.c b/src/dispatchers/attribute.c index d406c92e59..61d2bdd485 100644 --- a/src/dispatchers/attribute.c +++ b/src/dispatchers/attribute.c @@ -119,7 +119,7 @@ ncmpi_inq_attname(int ncid, /*----< ncmpi_copy_att() >---------------------------------------------------*/ /* This is a collective subroutine. - * ncid_out must be in define mode. If varid_in's attribute name has alreay + * ncid_out must be in define mode. If varid_in's attribute name has already * existed in varid_out, it means to overwrite the attribute in varid_out. * In this case, if the space used by varid_in's attribute is larger than * varid_out's, then this API must be called when the file is in define mode. diff --git a/src/dispatchers/cdl_header_parser.c b/src/dispatchers/cdl_header_parser.c index 421999e85b..6d76e20ea3 100644 --- a/src/dispatchers/cdl_header_parser.c +++ b/src/dispatchers/cdl_header_parser.c @@ -85,7 +85,7 @@ typedef long long MPI_Offset; break; \ default: err_str = "Other unknown error"; \ } \ - printf("Error in %s and %d : %s\n", __FILE__,__LINE__, err_str); \ + fprintf(stderr,"Error in %s and %d : %s\n", __FILE__,__LINE__, err_str); \ err = 1; \ goto err_out; \ } \ @@ -95,7 +95,7 @@ typedef long long MPI_Offset; #define LINE_SIZE 1024 #define ERR_FORMAT(msg) { \ - printf("Error in %s at %d: input file format at %s\n", \ + fprintf(stderr,"Error in %s at %d: input file format at %s\n", \ __func__,__LINE__,msg); \ err = NC_ENOTNC; \ goto err_out; \ @@ -182,7 +182,7 @@ int get_dimid(CDL_header *header, return NC_NOERR; } - printf("Error in %s at %d: failed to find dim ID for %s\n", + fprintf(stderr,"Error in %s at %d: failed to find dim ID for %s\n", __func__,__LINE__,name); return NC_EBADDIM; } @@ -303,7 +303,7 @@ int parse_dims(char **bptr, dims->value = (CDL_dim*) realloc(dims->value, sizeof(CDL_dim) * len); if (dims->value == NULL) { - printf("Error in %s at %d: fail to realloc of size %zd (%s)\n", + fprintf(stderr,"Error in %s at %d: fail to realloc of size %zd (%s)\n", __func__,__LINE__,sizeof(CDL_dim)*len,strerror(errno)); return NC_ENOMEM; } @@ -387,7 +387,7 @@ int parse_attr_value(CDL_attr *attrp, attrp->value = (void*) malloc(sizeof(double) * nelems); if (attrp->value == NULL) { - printf("Error in %s at %d: fail to malloc of size %zd (%s)\n", + fprintf(stderr,"Error in %s at %d: fail to malloc of size %zd (%s)\n", __func__,__LINE__,sizeof(double)*nelems,strerror(errno)); return NC_ENOMEM; } @@ -466,7 +466,7 @@ int parse_attr(char **bptr, attrs->value = (CDL_attr*) realloc(attrs->value, sizeof(CDL_attr) * len); if (attrs->value == NULL) { - printf("Error in %s at %d: fail to realloc of size %zd (%s)\n", + fprintf(stderr,"Error in %s at %d: fail to realloc of size %zd (%s)\n", __func__,__LINE__,sizeof(CDL_attr)*len,strerror(errno)); return NC_ENOMEM; } @@ -478,7 +478,7 @@ int parse_attr(char **bptr, attrs->value = (CDL_attr*) realloc(attrs->value, sizeof(CDL_attr) * len); if (attrs->value == NULL) { - printf("Error in %s at %d: fail to realloc of size %zd (%s)\n", + fprintf(stderr,"Error in %s at %d: fail to realloc of size %zd (%s)\n", __func__,__LINE__,sizeof(CDL_attr)*len,strerror(errno)); return NC_ENOMEM; } @@ -543,7 +543,7 @@ int parse_vars(char **bptr, vars->value = (CDL_var*) realloc(vars->value, sizeof(CDL_var) * len); if (vars->value == NULL) { - printf("Error in %s at %d: fail to realloc of size %zd (%s)\n", + fprintf(stderr,"Error in %s at %d: fail to realloc of size %zd (%s)\n", __func__,__LINE__,sizeof(CDL_var)*len,strerror(errno)); return NC_ENOMEM; } @@ -578,7 +578,7 @@ int parse_vars(char **bptr, varp->dimids = (int*) malloc(sizeof(int) * varp->ndims); if (varp->dimids == NULL) { - printf("Error in %s at %d: fail to malloc of size %zd (%s)\n", + fprintf(stderr,"Error in %s at %d: fail to malloc of size %zd (%s)\n", __func__,__LINE__,sizeof(int)*varp->ndims,strerror(errno)); return NC_ENOMEM; } @@ -678,7 +678,7 @@ int cdl_hdr_close(int hid) /*----< cdl_hdr_open() >-----------------------------------------------------*/ /* Reads a CDL header file, parses it, and stores the metadata in an internal - * buffer. Returns an ID for future inquery APIs to retrive the metadata. + * buffer. Returns an ID for future inquiry APIs to retrieve the metadata. */ int cdl_hdr_open(const char *filename, int *hid) @@ -694,13 +694,13 @@ int cdl_hdr_open(const char *filename, fptr = fopen(filename, "r"); if (fptr == NULL) { - printf("Error in %s at %d: fail to open file %s (%s)\n", + fprintf(stderr,"Error in %s at %d: fail to open file %s (%s)\n", __func__,__LINE__,filename,strerror(errno)); return NC_ENOENT; } err = fseek(fptr, 0, SEEK_END); if (err < 0) { - printf("Error in %s at %d: fail to fseek SEEK_END file %s (%s)\n", + fprintf(stderr,"Error in %s at %d: fail to fseek SEEK_END file %s (%s)\n", __func__,__LINE__,filename,strerror(errno)); return NC_EFILE; } @@ -709,26 +709,26 @@ int cdl_hdr_open(const char *filename, long file_size = ftell(fptr); fbuf = (char *) malloc(file_size); if (fbuf == NULL) { - printf("Error in %s at %d: fail to malloc of size %zd (%s)\n", + fprintf(stderr,"Error in %s at %d: fail to malloc of size %zd (%s)\n", __func__,__LINE__,file_size,strerror(errno)); return NC_ENOMEM; } err = fseek(fptr, 0, SEEK_SET); if (err < 0) { - printf("Error in %s at %d: fail to fseek SEEK_SET file %s (%s)\n", + fprintf(stderr,"Error in %s at %d: fail to fseek SEEK_SET file %s (%s)\n", __func__,__LINE__,filename,strerror(errno)); return NC_EFILE; } rlen = fread(fbuf, 1, file_size, fptr); - if (rlen < 0) { - printf("Error in %s at %d: fail to fread file %s (%s)\n", + if (file_size > 0 && rlen == 0) { + fprintf(stderr,"Error in %s at %d: fail to fread file %s (%s)\n", __func__,__LINE__,filename,strerror(errno)); return NC_EFILE; } bptr = fbuf; err = fclose(fptr); if (err == EOF) { - printf("Error in %s at %d: fail to fclose file %s (%s)\n", + fprintf(stderr,"Error in %s at %d: fail to fclose file %s (%s)\n", __func__,__LINE__,filename,strerror(errno)); return NC_EFILE; } @@ -739,7 +739,7 @@ int cdl_hdr_open(const char *filename, header = (CDL_header*) calloc(1, sizeof(CDL_header)); if (header == NULL) { - printf("Error in %s at %d: fail to calloc of size %zd (%s)\n", + fprintf(stderr,"Error in %s at %d: fail to calloc of size %zd (%s)\n", __func__,__LINE__,sizeof(CDL_header),strerror(errno)); return NC_ENOMEM; } diff --git a/src/dispatchers/error_codes.c b/src/dispatchers/error_codes.c index 1a71538f0e..7c31ed9803 100644 --- a/src/dispatchers/error_codes.c +++ b/src/dispatchers/error_codes.c @@ -289,12 +289,20 @@ ncmpi_strerror(int err) return "Variable fill value is inconsistent among processes."; case NC_EMULTIDEFINE_CMODE: return "File create mode is inconsistent among processes."; + case NC_EMULTIDEFINE_HINTS: + return "I/O hints are not consistent among processes."; case NC_EBADLOG: return "Unrecognized burst buffering log file format."; case NC_EFLUSHED: return "Nonblocking requests already flushed."; case NC_EADIOS: - return "unknown ADIOS error."; + return "Unknown ADIOS error."; + case NC_EFSTYPE: + return "File system type not supported by the GIO driver."; + case NC_EDRIVER: + return "Invalid PnetCDF I/O driver."; + case NC_EFILEVIEW: + return "PnetCDF internal error: MPI fileview's offsets are not in a monotonically non-decreasing order."; default: /* check netCDF-3 and netCDF-4 errors */ @@ -587,165 +595,172 @@ ncmpi_strerrno(int err) return unknown_str; } +#define ERR_CODE_STR(err) case (err): return #err; + switch (err) { - case (NC_NOERR): return "NC_NOERR"; - case (NC_EBADID): return "NC_EBADID"; - case (NC_ENFILE): return "NC_ENFILE"; - case (NC_EEXIST): return "NC_EEXIST"; - case (NC_EINVAL): return "NC_EINVAL"; - case (NC_EPERM): return "NC_EPERM"; - case (NC_ENOTINDEFINE): return "NC_ENOTINDEFINE"; - case (NC_EINDEFINE): return "NC_EINDEFINE"; - case (NC_EINVALCOORDS): return "NC_EINVALCOORDS"; - case (NC_EMAXDIMS): return "NC_EMAXDIMS"; - case (NC_ENAMEINUSE): return "NC_ENAMEINUSE"; - case (NC_ENOTATT): return "NC_ENOTATT"; - case (NC_EMAXATTS): return "NC_EMAXATTS"; - case (NC_EBADTYPE): return "NC_EBADTYPE"; - case (NC_EBADDIM): return "NC_EBADDIM"; - case (NC_EUNLIMPOS): return "NC_EUNLIMPOS"; - case (NC_EMAXVARS): return "NC_EMAXVARS"; - case (NC_ENOTVAR): return "NC_ENOTVAR"; - case (NC_EGLOBAL): return "NC_EGLOBAL"; - case (NC_ENOTNC): return "NC_ENOTNC"; - case (NC_ESTS): return "NC_ESTS"; - case (NC_EMAXNAME): return "NC_EMAXNAME"; - case (NC_EUNLIMIT): return "NC_EUNLIMIT"; - case (NC_ENORECVARS): return "NC_ENORECVARS"; - case (NC_ECHAR): return "NC_ECHAR"; - case (NC_EEDGE): return "NC_EEDGE"; - case (NC_ESTRIDE): return "NC_ESTRIDE"; - case (NC_EBADNAME): return "NC_EBADNAME"; - case (NC_ERANGE): return "NC_ERANGE"; - case (NC_ENOMEM): return "NC_ENOMEM"; - case (NC_EVARSIZE): return "NC_EVARSIZE"; - case (NC_EDIMSIZE): return "NC_EDIMSIZE"; - case (NC_ETRUNC): return "NC_ETRUNC"; - case (NC_EAXISTYPE): return "NC_EAXISTYPE"; - case (NC_EDAP): return "NC_EDAP"; - case (NC_ECURL): return "NC_ECURL"; - case (NC_EIO): return "NC_EIO"; - case (NC_ENODATA): return "NC_ENODATA"; - case (NC_EDAPSVC): return "NC_EDAPSVC"; - case (NC_EDAS): return "NC_EDAS"; - case (NC_EDDS): return "NC_EDDS"; - case (NC_EDATADDS): return "NC_EDATADDS"; - case (NC_EDAPURL): return "NC_EDAPURL"; - case (NC_EDAPCONSTRAINT): return "NC_EDAPCONSTRAINT"; - case (NC_ETRANSLATION): return "NC_ETRANSLATION"; - case (NC_EACCESS): return "NC_EACCESS"; - case (NC_EAUTH): return "NC_EAUTH"; - case (NC_ENOTFOUND): return "NC_ENOTFOUND"; - case (NC_ECANTREMOVE): return "NC_ECANTREMOVE"; - case (NC_EINTERNAL): return "NC_EINTERNAL"; - case (NC_EPNETCDF): return "NC_EPNETCDF"; - case (NC_EHDFERR): return "NC_EHDFERR"; - case (NC_ECANTREAD): return "NC_ECANTREAD"; - case (NC_ECANTWRITE): return "NC_ECANTWRITE"; - case (NC_ECANTCREATE): return "NC_ECANTCREATE"; - case (NC_EFILEMETA): return "NC_EFILEMETA"; - case (NC_EDIMMETA): return "NC_EDIMMETA"; - case (NC_EATTMETA): return "NC_EATTMETA"; - case (NC_EVARMETA): return "NC_EVARMETA"; - case (NC_ENOCOMPOUND): return "NC_ENOCOMPOUND"; - case (NC_EATTEXISTS): return "NC_EATTEXISTS"; - case (NC_ENOTNC4): return "NC_ENOTNC4"; - case (NC_ESTRICTNC3): return "NC_ESTRICTNC3"; - case (NC_ENOTNC3): return "NC_ENOTNC3"; - case (NC_ENOPAR): return "NC_ENOPAR"; - case (NC_EPARINIT): return "NC_EPARINIT"; - case (NC_EBADGRPID): return "NC_EBADGRPID"; - case (NC_EBADTYPID): return "NC_EBADTYPID"; - case (NC_ETYPDEFINED): return "NC_ETYPDEFINED"; - case (NC_EBADFIELD): return "NC_EBADFIELD"; - case (NC_EBADCLASS): return "NC_EBADCLASS"; - case (NC_EMAPTYPE): return "NC_EMAPTYPE"; - case (NC_ELATEFILL): return "NC_ELATEFILL"; - case (NC_ELATEDEF): return "NC_ELATEDEF"; - case (NC_EDIMSCALE): return "NC_EDIMSCALE"; - case (NC_ENOGRP): return "NC_ENOGRP"; - case (NC_ESTORAGE): return "NC_ESTORAGE"; - case (NC_EBADCHUNK): return "NC_EBADCHUNK"; - case (NC_ENOTBUILT): return "NC_ENOTBUILT"; - case (NC_EDISKLESS): return "NC_EDISKLESS"; - case (NC_ECANTEXTEND): return "NC_ECANTEXTEND"; - case (NC_EMPI): return "NC_EMPI"; - case (NC_EFILTER): return "NC_EFILTER"; - case (NC_ERCFILE): return "NC_ERCFILE"; - case (NC_ENULLPAD): return "NC_ENULLPAD"; - case (NC_EINMEMORY): return "NC_EINMEMORY"; - case (NC_ENOFILTER): return "NC_ENOFILTER"; + ERR_CODE_STR(NC_NOERR) + ERR_CODE_STR(NC_EBADID) + ERR_CODE_STR(NC_ENFILE) + ERR_CODE_STR(NC_EEXIST) + ERR_CODE_STR(NC_EINVAL) + ERR_CODE_STR(NC_EPERM) + ERR_CODE_STR(NC_ENOTINDEFINE) + ERR_CODE_STR(NC_EINDEFINE) + ERR_CODE_STR(NC_EINVALCOORDS) + ERR_CODE_STR(NC_EMAXDIMS) + ERR_CODE_STR(NC_ENAMEINUSE) + ERR_CODE_STR(NC_ENOTATT) + ERR_CODE_STR(NC_EMAXATTS) + ERR_CODE_STR(NC_EBADTYPE) + ERR_CODE_STR(NC_EBADDIM) + ERR_CODE_STR(NC_EUNLIMPOS) + ERR_CODE_STR(NC_EMAXVARS) + ERR_CODE_STR(NC_ENOTVAR) + ERR_CODE_STR(NC_EGLOBAL) + ERR_CODE_STR(NC_ENOTNC) + ERR_CODE_STR(NC_ESTS) + ERR_CODE_STR(NC_EMAXNAME) + ERR_CODE_STR(NC_EUNLIMIT) + ERR_CODE_STR(NC_ENORECVARS) + ERR_CODE_STR(NC_ECHAR) + ERR_CODE_STR(NC_EEDGE) + ERR_CODE_STR(NC_ESTRIDE) + ERR_CODE_STR(NC_EBADNAME) + ERR_CODE_STR(NC_ERANGE) + ERR_CODE_STR(NC_ENOMEM) + ERR_CODE_STR(NC_EVARSIZE) + ERR_CODE_STR(NC_EDIMSIZE) + ERR_CODE_STR(NC_ETRUNC) + ERR_CODE_STR(NC_EAXISTYPE) + ERR_CODE_STR(NC_EDAP) + ERR_CODE_STR(NC_ECURL) + ERR_CODE_STR(NC_EIO) + ERR_CODE_STR(NC_ENODATA) + ERR_CODE_STR(NC_EDAPSVC) + ERR_CODE_STR(NC_EDAS) + ERR_CODE_STR(NC_EDDS) + ERR_CODE_STR(NC_EDATADDS) + ERR_CODE_STR(NC_EDAPURL) + ERR_CODE_STR(NC_EDAPCONSTRAINT) + ERR_CODE_STR(NC_ETRANSLATION) + ERR_CODE_STR(NC_EACCESS) + ERR_CODE_STR(NC_EAUTH) + ERR_CODE_STR(NC_ENOTFOUND) + ERR_CODE_STR(NC_ECANTREMOVE) + ERR_CODE_STR(NC_EINTERNAL) + ERR_CODE_STR(NC_EPNETCDF) + ERR_CODE_STR(NC_EHDFERR) + ERR_CODE_STR(NC_ECANTREAD) + ERR_CODE_STR(NC_ECANTWRITE) + ERR_CODE_STR(NC_ECANTCREATE) + ERR_CODE_STR(NC_EFILEMETA) + ERR_CODE_STR(NC_EDIMMETA) + ERR_CODE_STR(NC_EATTMETA) + ERR_CODE_STR(NC_EVARMETA) + ERR_CODE_STR(NC_ENOCOMPOUND) + ERR_CODE_STR(NC_EATTEXISTS) + ERR_CODE_STR(NC_ENOTNC4) + ERR_CODE_STR(NC_ESTRICTNC3) + ERR_CODE_STR(NC_ENOTNC3) + ERR_CODE_STR(NC_ENOPAR) + ERR_CODE_STR(NC_EPARINIT) + ERR_CODE_STR(NC_EBADGRPID) + ERR_CODE_STR(NC_EBADTYPID) + ERR_CODE_STR(NC_ETYPDEFINED) + ERR_CODE_STR(NC_EBADFIELD) + ERR_CODE_STR(NC_EBADCLASS) + ERR_CODE_STR(NC_EMAPTYPE) + ERR_CODE_STR(NC_ELATEFILL) + ERR_CODE_STR(NC_ELATEDEF) + ERR_CODE_STR(NC_EDIMSCALE) + ERR_CODE_STR(NC_ENOGRP) + ERR_CODE_STR(NC_ESTORAGE) + ERR_CODE_STR(NC_EBADCHUNK) + ERR_CODE_STR(NC_ENOTBUILT) + ERR_CODE_STR(NC_EDISKLESS) + ERR_CODE_STR(NC_ECANTEXTEND) + ERR_CODE_STR(NC_EMPI) + ERR_CODE_STR(NC_EFILTER) + ERR_CODE_STR(NC_ERCFILE) + ERR_CODE_STR(NC_ENULLPAD) + ERR_CODE_STR(NC_EINMEMORY) + ERR_CODE_STR(NC_ENOFILTER) /* - case (NC_EURL): return "NC_EURL"; - case (NC_ECONSTRAINT): return "NC_ECONSTRAINT"; + ERR_CODE_STR(NC_EURL) + ERR_CODE_STR(NC_ECONSTRAINT) */ - case (NC_ESMALL): return "NC_ESMALL"; - case (NC_ENOTINDEP): return "NC_ENOTINDEP"; - case (NC_EINDEP): return "NC_EINDEP"; - case (NC_EFILE): return "NC_EFILE"; - case (NC_EREAD): return "NC_EREAD"; - case (NC_EWRITE): return "NC_EWRITE"; - case (NC_EOFILE): return "NC_EOFILE"; - case (NC_EMULTITYPES): return "NC_EMULTITYPES"; - case (NC_EIOMISMATCH): return "NC_EIOMISMATCH"; - case (NC_ENEGATIVECNT): return "NC_ENEGATIVECNT"; - case (NC_EUNSPTETYPE): return "NC_EUNSPTETYPE"; - case (NC_EINVAL_REQUEST): return "NC_EINVAL_REQUEST"; - case (NC_EAINT_TOO_SMALL): return "NC_EAINT_TOO_SMALL"; - case (NC_ENOTSUPPORT): return "NC_ENOTSUPPORT"; - case (NC_ENULLBUF): return "NC_ENULLBUF"; - case (NC_EPREVATTACHBUF): return "NC_EPREVATTACHBUF"; - case (NC_ENULLABUF): return "NC_ENULLABUF"; - case (NC_EPENDINGBPUT): return "NC_EPENDINGBPUT"; - case (NC_EINSUFFBUF): return "NC_EINSUFFBUF"; - case (NC_ENOENT): return "NC_ENOENT"; - case (NC_EINTOVERFLOW): return "NC_EINTOVERFLOW"; - case (NC_ENOTENABLED): return "NC_ENOTENABLED"; - case (NC_EBAD_FILE): return "NC_EBAD_FILE"; - case (NC_ENO_SPACE): return "NC_ENO_SPACE"; - case (NC_EQUOTA): return "NC_EQUOTA"; - case (NC_ENULLSTART): return "NC_ENULLSTART"; - case (NC_ENULLCOUNT): return "NC_ENULLCOUNT"; - case (NC_EINVAL_CMODE): return "NC_EINVAL_CMODE"; - case (NC_ETYPESIZE): return "NC_ETYPESIZE"; - case (NC_ETYPE_MISMATCH): return "NC_ETYPE_MISMATCH"; - case (NC_ETYPESIZE_MISMATCH): return "NC_ETYPESIZE_MISMATCH"; - case (NC_ESTRICTCDF2): return "NC_ESTRICTCDF2"; - case (NC_ENOTRECVAR): return "NC_ENOTRECVAR"; - case (NC_ENOTFILL): return "NC_ENOTFILL"; - case (NC_EINVAL_OMODE): return "NC_EINVAL_OMODE"; - case (NC_EPENDING): return "NC_EPENDING"; - case (NC_EMAX_REQ): return "NC_EMAX_REQ"; - case (NC_EBADLOG): return "NC_EBADLOG"; - case (NC_EFLUSHED): return "NC_EFLUSHED"; - case (NC_EADIOS): return "NC_EADIOS"; - - case (NC_EMULTIDEFINE): return "NC_EMULTIDEFINE"; - case (NC_EMULTIDEFINE_OMODE): return "NC_EMULTIDEFINE_OMODE"; - case (NC_EMULTIDEFINE_DIM_NUM): return "NC_EMULTIDEFINE_DIM_NUM"; - case (NC_EMULTIDEFINE_DIM_SIZE): return "NC_EMULTIDEFINE_DIM_SIZE"; - case (NC_EMULTIDEFINE_DIM_NAME): return "NC_EMULTIDEFINE_DIM_NAME"; - case (NC_EMULTIDEFINE_VAR_NUM): return "NC_EMULTIDEFINE_VAR_NUM"; - case (NC_EMULTIDEFINE_VAR_NAME): return "NC_EMULTIDEFINE_VAR_NAME"; - case (NC_EMULTIDEFINE_VAR_NDIMS): return "NC_EMULTIDEFINE_VAR_NDIMS"; - case (NC_EMULTIDEFINE_VAR_DIMIDS): return "NC_EMULTIDEFINE_VAR_DIMIDS"; - case (NC_EMULTIDEFINE_VAR_TYPE): return "NC_EMULTIDEFINE_VAR_TYPE"; - case (NC_EMULTIDEFINE_VAR_LEN): return "NC_EMULTIDEFINE_VAR_LEN"; - case (NC_EMULTIDEFINE_NUMRECS): return "NC_EMULTIDEFINE_NUMRECS"; - case (NC_EMULTIDEFINE_VAR_BEGIN): return "NC_EMULTIDEFINE_VAR_BEGIN"; - case (NC_EMULTIDEFINE_ATTR_NUM): return "NC_EMULTIDEFINE_ATTR_NUM"; - case (NC_EMULTIDEFINE_ATTR_SIZE): return "NC_EMULTIDEFINE_ATTR_SIZE"; - case (NC_EMULTIDEFINE_ATTR_NAME): return "NC_EMULTIDEFINE_ATTR_NAME"; - case (NC_EMULTIDEFINE_ATTR_TYPE): return "NC_EMULTIDEFINE_ATTR_TYPE"; - case (NC_EMULTIDEFINE_ATTR_LEN): return "NC_EMULTIDEFINE_ATTR_LEN"; - case (NC_EMULTIDEFINE_ATTR_VAL): return "NC_EMULTIDEFINE_ATTR_VAL"; - case (NC_EMULTIDEFINE_FNC_ARGS): return "NC_EMULTIDEFINE_FNC_ARGS"; - case (NC_EMULTIDEFINE_FILL_MODE): return "NC_EMULTIDEFINE_FILL_MODE"; - case (NC_EMULTIDEFINE_VAR_FILL_MODE): return "NC_EMULTIDEFINE_VAR_FILL_MODE"; - case (NC_EMULTIDEFINE_VAR_FILL_VALUE): return "NC_EMULTIDEFINE_VAR_FILL_VALUE"; - case (NC_EMULTIDEFINE_CMODE): return "NC_EMULTIDEFINE_CMODE"; + ERR_CODE_STR(NC_ESMALL) + ERR_CODE_STR(NC_ENOTINDEP) + ERR_CODE_STR(NC_EINDEP) + ERR_CODE_STR(NC_EFILE) + ERR_CODE_STR(NC_EREAD) + ERR_CODE_STR(NC_EWRITE) + ERR_CODE_STR(NC_EOFILE) + ERR_CODE_STR(NC_EMULTITYPES) + ERR_CODE_STR(NC_EIOMISMATCH) + ERR_CODE_STR(NC_ENEGATIVECNT) + ERR_CODE_STR(NC_EUNSPTETYPE) + ERR_CODE_STR(NC_EINVAL_REQUEST) + ERR_CODE_STR(NC_EAINT_TOO_SMALL) + ERR_CODE_STR(NC_ENOTSUPPORT) + ERR_CODE_STR(NC_ENULLBUF) + ERR_CODE_STR(NC_EPREVATTACHBUF) + ERR_CODE_STR(NC_ENULLABUF) + ERR_CODE_STR(NC_EPENDINGBPUT) + ERR_CODE_STR(NC_EINSUFFBUF) + ERR_CODE_STR(NC_ENOENT) + ERR_CODE_STR(NC_EINTOVERFLOW) + ERR_CODE_STR(NC_ENOTENABLED) + ERR_CODE_STR(NC_EBAD_FILE) + ERR_CODE_STR(NC_ENO_SPACE) + ERR_CODE_STR(NC_EQUOTA) + ERR_CODE_STR(NC_ENULLSTART) + ERR_CODE_STR(NC_ENULLCOUNT) + ERR_CODE_STR(NC_EINVAL_CMODE) + ERR_CODE_STR(NC_ETYPESIZE) + ERR_CODE_STR(NC_ETYPE_MISMATCH) + ERR_CODE_STR(NC_ETYPESIZE_MISMATCH) + ERR_CODE_STR(NC_ESTRICTCDF2) + ERR_CODE_STR(NC_ENOTRECVAR) + ERR_CODE_STR(NC_ENOTFILL) + ERR_CODE_STR(NC_EINVAL_OMODE) + ERR_CODE_STR(NC_EPENDING) + ERR_CODE_STR(NC_EMAX_REQ) + ERR_CODE_STR(NC_EBADLOG) + ERR_CODE_STR(NC_EFLUSHED) + ERR_CODE_STR(NC_EADIOS) + ERR_CODE_STR(NC_EFSTYPE) + + ERR_CODE_STR(NC_EMULTIDEFINE) + ERR_CODE_STR(NC_EMULTIDEFINE_OMODE) + ERR_CODE_STR(NC_EMULTIDEFINE_DIM_NUM) + ERR_CODE_STR(NC_EMULTIDEFINE_DIM_SIZE) + ERR_CODE_STR(NC_EMULTIDEFINE_DIM_NAME) + ERR_CODE_STR(NC_EMULTIDEFINE_VAR_NUM) + ERR_CODE_STR(NC_EMULTIDEFINE_VAR_NAME) + ERR_CODE_STR(NC_EMULTIDEFINE_VAR_NDIMS) + ERR_CODE_STR(NC_EMULTIDEFINE_VAR_DIMIDS) + ERR_CODE_STR(NC_EMULTIDEFINE_VAR_TYPE) + ERR_CODE_STR(NC_EMULTIDEFINE_VAR_LEN) + ERR_CODE_STR(NC_EMULTIDEFINE_NUMRECS) + ERR_CODE_STR(NC_EMULTIDEFINE_VAR_BEGIN) + ERR_CODE_STR(NC_EMULTIDEFINE_ATTR_NUM) + ERR_CODE_STR(NC_EMULTIDEFINE_ATTR_SIZE) + ERR_CODE_STR(NC_EMULTIDEFINE_ATTR_NAME) + ERR_CODE_STR(NC_EMULTIDEFINE_ATTR_TYPE) + ERR_CODE_STR(NC_EMULTIDEFINE_ATTR_LEN) + ERR_CODE_STR(NC_EMULTIDEFINE_ATTR_VAL) + ERR_CODE_STR(NC_EMULTIDEFINE_FNC_ARGS) + ERR_CODE_STR(NC_EMULTIDEFINE_FILL_MODE) + ERR_CODE_STR(NC_EMULTIDEFINE_VAR_FILL_MODE) + ERR_CODE_STR(NC_EMULTIDEFINE_VAR_FILL_VALUE) + ERR_CODE_STR(NC_EMULTIDEFINE_CMODE) + ERR_CODE_STR(NC_EMULTIDEFINE_HINTS) + ERR_CODE_STR(NC_EDRIVER) + ERR_CODE_STR(NC_EFILEVIEW) + default: - sprintf(unknown_str,"Unknown code %d",err); + sprintf(unknown_str,"Unknown code %d",err); } return unknown_str; } diff --git a/src/dispatchers/file.c b/src/dispatchers/file.c index b68782038b..420f3c0c1e 100644 --- a/src/dispatchers/file.c +++ b/src/dispatchers/file.c @@ -10,7 +10,7 @@ #include #include /* getenv() */ -#include /* strtok(), strtok_r(), strchr(), strcpy(), strdup() */ +#include /* strtok(), strtok_r(), strchr(), strcpy() */ #include /* strcasecmp() */ #include /* open() */ #include /* lseek() */ @@ -18,13 +18,8 @@ #include /* assert() */ #include /* errno */ -#ifdef ENABLE_THREAD_SAFE -#include -static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; -#endif - #ifdef ENABLE_NETCDF4 -#include +#include /* must included before pnetcdf.h if enabled */ #endif #include @@ -32,14 +27,23 @@ static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; #include #include -#ifdef ENABLE_ADIOS +#if PNETCDF_THREAD_SAFE == 1 +#include +static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; +#endif + +#ifndef MAX_INT_LEN +#define MAX_INT_LEN 24 +#endif + +#if PNETCDF_DRIVER_ADIOS == 1 #include "adios_read.h" #include #define BP_MINIFOOTER_SIZE 28 #define BUFREAD64(buf,var) memcpy(&var, buf, 8); if (diff_endian) swap_64(&var); #endif -/* Note accessing the following 3 global variables must be protected by a +/* Note accessing the following 5 global variables must be protected by a * mutex, otherwise it will not be thread safe. */ @@ -52,21 +56,555 @@ static int pnc_numfiles; */ static int ncmpi_default_create_format = NC_FORMAT_CLASSIC; -#define NCMPII_HANDLE_ERROR(func) \ - if (mpireturn != MPI_SUCCESS) { \ - int errorStringLen; \ - char errorString[MPI_MAX_ERROR_STRING]; \ - MPI_Error_string(mpireturn, errorString, &errorStringLen); \ - printf("Error: file %s line %d calling func %s: (%s)\n", \ - __FILE__, __LINE__, func, errorString); \ - } - -#define CHECK_ERRNO(err, func) { \ - if (err != 0) { \ - printf("Error: file %s line %d calling func %s: (%s)\n", \ - __FILE__, __LINE__, func, strerror(err)); \ - goto err_out; \ - } \ +/* attribute to be cached in all communicators */ +static int ncmpi_comm_keyval = MPI_KEYVAL_INVALID; + +/* attribute to be cached in MPI_COMM_SELF */ +static int ncmpi_init_keyval = MPI_KEYVAL_INVALID; + +#define NCMPII_HANDLE_ERROR(func) \ + if (mpireturn != MPI_SUCCESS) { \ + int errorStringLen; \ + char errorString[MPI_MAX_ERROR_STRING]; \ + MPI_Error_string(mpireturn, errorString, &errorStringLen); \ + fprintf(stderr,"Error: file %s line %d calling func %s: (%s)\n", \ + __FILE__, __LINE__, func, errorString); \ + } + +#define CHECK_ERRNO(err, func) { \ + if (err != 0) { \ + fprintf(stderr,"Error: file %s line %d calling func %s: (%s)\n", \ + __FILE__, __LINE__, func, strerror(err)); \ + goto err_out; \ + } \ +} + +/* struct PNC_comm_attr is defined in dispatch.h */ + +/*----< ina_init() >---------------------------------------------------------*/ +/* When the intra-node write aggregation (INA) hint is enabled, this subroutine + * initializes the metadata to be used in intra- and inter-node communication, + * including an intra-node communicator for communication between INA + * aggregators and non-INA aggregators, and an inter-node communicator + * consisting of all the INA aggregators to be used when calling GIO/MPI-IO + * file open and their collective and independent I/O calls. + * + * Processes on the same NUMA node will first be divided into disjoined groups. + * The passed in communicator will be split into sub-communicators, referred to + * as intra-node communicators, one for each INA group. Within a group, the + * process with the lowest rank ID is selected as the group's INA aggregator. + * + * A new MPI communicator consisting of all INA aggregators across all nodes + * will be created, which is referred to as the inter-node communicator. The + * inter-node communicator will be used to open the file, i.e. in the call to + * GIO_open()/MPI_File_open() later. Only the INA aggregators make calls to + * the GIO/MPI-IO APIs to access the file. Thus, this subroutine must be + * called before opening the file and should be called only once in + * ncmpi_create() or ncmpi_open(). + * + * This subroutine performs the following tasks. + * 1. Makes use of the affinity of each MPI process to its NUMA node to + * calculate the number of INA groups within a node and identify the + * membership of every process to its INA group. + * + comm_attr->num_NUMAs is the number of NUMA nodes. + * + comm_attr->NUMA_IDs[nprocs] contains NUMA node IDs of all processes. + * Note comm_attr should have already been established during a call to + * ncmpii_construct_node_list() at the beginning of ncmpi_create() and + * ncmpi_open(). + * 2. Based on hint num_aggrs_per_node and the number of processes per NUMA + * node, calculates the number of INA groups per node and divides processes + * into groups. Select the process with the lowest rank in a group as the + * INA aggregator. + * + comm_attr->is_ina_aggr indicates whether this rank is an INA aggregator + * 3. Create a new MPI communicator by splitting 'comm' into sub-communicators, + * each consisting of processes belonging to the same INA group. + * + comm_attr->ina_intra_comm is the INA intra-node communicator. + * + MPI_Comm_size(comm_attr->ina_intra_comm, &size) is the number of + * processes within an INA group. + * + Rank 0 in comm_attr->ina_intra_comm is the INA aggregator. + * 4. Create a new MPI communicator consisting of all the INA aggregators. + * + comm_attr->ina_inter_comm is the INA inter-node communicator. + * + MPI_Comm_size(comm_attr->ina_inter_comm, &size) is the total number of + * INA aggregators. + * + comm_attr->ina_inter_comm will be used when calling GIO_open()/ + * MPI_File_open(). + * + * The concept of intra-node request aggregation and its performance results + * are presented in the following paper. + * Q. Kang, S. Lee, K. Hou, R. Ross, A. Agrawal, A. Choudhary, and W. Liao. + * Improving MPI Collective I/O for High Volume Non-Contiguous Requests With + * Intra-Node Aggregation. IEEE Transactions on Parallel and Distributed + * Systems, 31(11):2682-2695, November 2020. + */ +static +int ina_init(MPI_Comm comm, + int num_aggrs_per_node, + PNC_comm_attr *comm_attr) +{ + int i, j, k, mpireturn, nprocs, rank, my_node_naggrs, aggr_rank; + int my_node_nprocs, my_node_rank; + int *ina_flags, grp_nprocs, rem; + +#if PNETCDF_PROFILING == 1 + double timing = MPI_Wtime(); +#endif + + if (num_aggrs_per_node == 0) return NC_NOERR; + + MPI_Comm_size(comm, &nprocs); + MPI_Comm_rank(comm, &rank); + +#if PNETCDF_DEBUG_MODE == 1 + /* Note that ill value of num_aggrs_per_node has been checked before + * entering this subroutine. Thus num_aggrs_per_node must be > 0. + */ + assert(num_aggrs_per_node > 0); + assert(comm_attr->numa_comm != MPI_COMM_NULL); +#endif + + /* comm_attr->NUMA_IDs[] has been set in ncmpii_construct_node_list() + * called earlier in ncmpio_create() or ncmpio_open() before entering this + * subroutine. + */ + + /* my_node_nprocs is the number of processes in my NUMA compute node. */ + MPI_Comm_size(comm_attr->numa_comm, &my_node_nprocs); + + /* my_node_rank is the rank ID of this process in my NUMA compute node. */ + MPI_Comm_rank(comm_attr->numa_comm, &my_node_rank); + + /* Make sure the actual number of INA aggregators per node, initially set + * in hint num_aggrs_per_node, is <= my_node_nprocs. In some cases, the + * number of processes allocated to the last few compute nodes can be less + * than others. + */ + my_node_naggrs = MIN(num_aggrs_per_node, my_node_nprocs); + + /* Divide processes in a NUMA node into INA groups and calculate the number + * of processes in each INA group, grp_nprocs. Select the INA aggregator, + * as the process with loweset rank in an INA group, whose local rank in + * comm_attr->numa_comm is 'aggr_rank'. + */ + grp_nprocs = my_node_nprocs / my_node_naggrs; /* no. processes per group */ + rem = my_node_nprocs % my_node_naggrs; + if (rem > 0) { /* non-divisible case */ + grp_nprocs++; + if (my_node_rank < grp_nprocs * rem) + /* Select the first rank of my INA group as INA aggregator. */ + aggr_rank = my_node_rank - my_node_rank % grp_nprocs; + else { + aggr_rank = grp_nprocs * rem; + grp_nprocs--; + aggr_rank = my_node_rank + - (my_node_rank - aggr_rank) % grp_nprocs; + } + } + else /* divisible case */ + aggr_rank = my_node_rank - my_node_rank % grp_nprocs; + + /* whether this rank is an INA aggregator */ + comm_attr->is_ina_aggr = (my_node_rank == aggr_rank); + +#if PNETCDF_DEBUG_MODE == 1 + /* Make sure the number of processes in my INA group does not go beyond + * my_node_nprocs. + */ + assert(grp_nprocs <= my_node_nprocs - aggr_rank); +#endif + + if (comm_attr->ina_intra_comm != MPI_COMM_NULL) { + /* free ina_intra_comm if previous created */ + TRACE_COMM(MPI_Comm_free)(&comm_attr->ina_intra_comm); + if (mpireturn != MPI_SUCCESS) + return ncmpii_error_mpi2nc(mpireturn, "MPI_Comm_free"); + } + comm_attr->ina_intra_comm = MPI_COMM_NULL; + + /* Split NUMA comm into INA intra-node comm based on the assigned INA + * aggregator's rank ID, i.e. processes sharing the same INA aggregator + * form an ina_intra_comm. This process's local rank on the NUMA node + * is my_node_rank and its INA aggregator's rank is aggr_rank. + * + * Special note on when the number of processes in this INA group is 1. In + * this case, there is only one process in this group and thus the + * intra-node aggregation of this group will not perform. However, the + * intra-node communicator of this INA group must still be created. This is + * because MPI_Comm_split is a collective call with the processes running + * on the same NUMA node, i.e. on comm_attr->numa_comm. + * + * The case of grp_nprocs == 1, does not mean intra-node aggregation is + * disabled globally. It just means this group will not perform INA + * aggregation. The indicator of whether intra-node aggregation is globally + * enabled or disabled is 'num_aggrs_per_node', whose value must be kept + * consistent across all processes. It is possible for some groups + * containing only one process, in which case the aggregation is not + * necessarily to perform within those groups. + */ + TRACE_COMM(MPI_Comm_split)(comm_attr->numa_comm, aggr_rank, my_node_rank, + &comm_attr->ina_intra_comm); + if (mpireturn != MPI_SUCCESS) + return ncmpii_error_mpi2nc(mpireturn, "MPI_Comm_split"); + +#if PNETCDF_DEBUG_MODE == 1 + { + /* check whether or not this rank's MPI processor name is the same as + * its INA aggregator's . + */ + int world_rank, intra_rank, mpi_namelen, root_mpi_namelen; + char mpi_name[MPI_MAX_PROCESSOR_NAME]; + char root_mpi_name[MPI_MAX_PROCESSOR_NAME]; + + MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); + MPI_Comm_rank(comm_attr->ina_intra_comm, &intra_rank); + MPI_Get_processor_name(mpi_name, &mpi_namelen); + + memcpy(root_mpi_name, mpi_name, MPI_MAX_PROCESSOR_NAME); + root_mpi_namelen = mpi_namelen + 1; + MPI_Bcast(&root_mpi_namelen, 1, MPI_INT, 0, comm_attr->ina_intra_comm); + MPI_Bcast(root_mpi_name, root_mpi_namelen, MPI_CHAR, 0, comm_attr->ina_intra_comm); + + if (strncmp(root_mpi_name, mpi_name, MPI_MAX_PROCESSOR_NAME)) { + printf("%s at %d: intra-node rank %d (world rank %d) MPI name (%s) != root's (%s)\n", + __func__,__LINE__, intra_rank, world_rank, mpi_name, root_mpi_name); + assert(0); + } + } +#endif + + /* Next step is to construct an inter-node MPI communicator consisting of + * all INA aggregators. It will later be used to call MPI_File_open(), and + * successively only INA aggregators call MPI-IO functions to access the + * file. + */ + + /* construct an array containing ranks of aggregators */ + ina_flags = (int*) malloc(sizeof(int) * nprocs); + TRACE_COMM(MPI_Allgather)(&comm_attr->is_ina_aggr, 1, MPI_INT, ina_flags, + 1, MPI_INT, comm); + + /* Given a comm_attr->NUMA_IDs[], the rank IDs of INA aggregators will + * dependent on the layout of MPI process allocation to the compute nodes. + * The common layouts can be two kinds: + * + cyclic - MPI ranks are assigned to nodes round-robin-ly, + * + block - MPI ranks are assigned to a node and then move on to next. + * + * Below uses an example of nodes=3, nprocs=10, * num_aggrs_per_node=2. + * comm_attr->NUMA_IDs[] should be + * block process allocation: 0,0,0,0,1,1,1,2,2,2 + * cyclic process allocation: 0,1,2,0,1,2,0,1,2,0 + * Accordingly, rank IDs of INA aggregators can be two kinds + * block process allocation: 1,0,1,0,1,0,1,1,0,1 + * cyclic process allocation: 1,1,1,0,0,0,1,1,1,0 + */ + + /* calculate actual number of INA aggregators */ + comm_attr->num_ina_aggrs = 0; + for (j=0; jnum_ina_aggrs += ina_flags[j]; + + /* Collect aggregators' rank IDs and store them in an increasing order of + * node IDs. Note rank IDs in ina_ranks[] are relative to comm (not + * inter-node comm or intra-node comm). + */ + comm_attr->ina_ranks = (int*)malloc(sizeof(int) * comm_attr->num_ina_aggrs); + for (k=0, i=0; inum_NUMAs; i++) { + for (j=0; jNUMA_IDs[j] == i && ina_flags[j] > 0) + comm_attr->ina_ranks[k++] = j; + } + } + free(ina_flags); + + if (comm_attr->ina_inter_comm != MPI_COMM_NULL) { + /* free comm_attr->ina_inter_comm if created previously */ + TRACE_COMM(MPI_Comm_free)(&comm_attr->ina_inter_comm); + if (mpireturn != MPI_SUCCESS) + return ncmpii_error_mpi2nc(mpireturn, "MPI_Comm_free"); + } + comm_attr->ina_inter_comm = MPI_COMM_NULL; + + /* create an inter-node communicator consisting of all INA aggregators */ + MPI_Group origin_group, ina_group; + TRACE_COMM(MPI_Comm_group)(comm, &origin_group); + if (mpireturn != MPI_SUCCESS) + return ncmpii_error_mpi2nc(mpireturn, "MPI_Comm_group"); + TRACE_COMM(MPI_Group_incl)(origin_group, comm_attr->num_ina_aggrs, + comm_attr->ina_ranks, &ina_group); + if (mpireturn != MPI_SUCCESS) + return ncmpii_error_mpi2nc(mpireturn, "MPI_Group_incl"); + TRACE_COMM(MPI_Comm_create)(comm, ina_group, &comm_attr->ina_inter_comm); + if (mpireturn != MPI_SUCCESS) + return ncmpii_error_mpi2nc(mpireturn, "MPI_Comm_create"); + TRACE_COMM(MPI_Group_free)(&ina_group); + if (mpireturn != MPI_SUCCESS) + return ncmpii_error_mpi2nc(mpireturn, "MPI_Group_free"); + TRACE_COMM(MPI_Group_free)(&origin_group); + if (mpireturn != MPI_SUCCESS) + return ncmpii_error_mpi2nc(mpireturn, "MPI_Group_free"); + + /* TODO: automatically determine whether or not to enable intra-node + * aggregation. + * + * The ideal case is it can be determined right before each collective + * write call, because only at that time, the communication pattern is + * known. If the pattern can cause contention, then enable it. Otherwise, + * disable it. + * + * Such mechanism may depends on the followings. + * 1. MPI-IO hint cb_noddes, and striping_unit + * 2. calculate aggregate access region + * 3. If the number of senders to each cb_nodes is very large, then + * intra-node aggregation should be enabled. + * 4. Average of nprocs_per_node across all processes may be a factor for + * determining whether to enable intra-node aggregation. It indicates + * whether the high number of processes are allocated on the same + * node. + */ + +#if PNETCDF_PROFILING == 1 + pnc_ina_init = MPI_Wtime() - timing; +#endif + + return NC_NOERR; +} + +/*----< comm_attr_copy() >---------------------------------------------------*/ +/* A function to be invoked when a communicator is duplicated, which adds a + * reference to the already allocated memory space storing node ID array. + */ +static +int comm_attr_copy(MPI_Comm comm, + int keyval, + void *extra, + void *attr_inP, + void *attr_outP, + int *flag) +{ + PNC_comm_attr *attr_in = (PNC_comm_attr*) attr_inP; + PNC_comm_attr **attr_out = (PNC_comm_attr**)attr_outP; + + if (attr_in == NULL) + return MPI_ERR_KEYVAL; + else + attr_in->ref_count++; + + *attr_out = attr_in; + + *flag = 1; /* make a copy in the new communicator */ + + return MPI_SUCCESS; +} + +/*----< comm_attr_delete() >-------------------------------------------------*/ +/* Callback function to be called when a communicator is freed, which frees the + * allocated memory space of node ID array. + */ +static +int comm_attr_delete(MPI_Comm comm, + int keyval, + void *attr_val, + void *extra) +{ + PNC_comm_attr *attr = (PNC_comm_attr*) attr_val; + + if (attr == NULL) + return MPI_ERR_KEYVAL; + else + attr->ref_count--; + + if (attr->ref_count <= 0) { + /* free the allocated array */ + if (attr->NUMA_IDs != NULL) + free(attr->NUMA_IDs); + if (attr->ina_ranks != NULL) + free(attr->ina_ranks); + if (attr->numa_comm != MPI_COMM_NULL) + MPI_Comm_free(&attr->numa_comm); + if (attr->ina_inter_comm != MPI_COMM_NULL) + MPI_Comm_free(&attr->ina_inter_comm); + if (attr->ina_intra_comm != MPI_COMM_NULL) + MPI_Comm_free(&attr->ina_intra_comm); + free(attr); + } + return MPI_SUCCESS; +} + +/*----< end_call() >---------------------------------------------------------*/ +/* Callback function to be called at MPI_Finalize(), which frees all cached + * attributes. + */ +static +int end_call(MPI_Comm comm, + int keyval, + void *attribute_val, + void *extra_state) +{ + /* Free all keyvals used by PnetCDF */ + + MPI_Comm_free_keyval(&keyval); /* free ncmpi_init_keyval */ + + if (ncmpi_comm_keyval != MPI_KEYVAL_INVALID) + MPI_Comm_free_keyval(&ncmpi_comm_keyval); + + return MPI_SUCCESS; +} + +/*----< set_get_comm_attr() >------------------------------------------------*/ +/* Create/set/get attributes into/from the MPI communicators passed in from + * the user application. + */ +static +int set_get_comm_attr(MPI_Comm comm, + int num_aggrs_per_node, + PNC_comm_attr *attrP) +{ + int err, nprocs; + PNC_comm_attr *attr; + + MPI_Comm_size(comm, &nprocs); + + if (ncmpi_init_keyval == MPI_KEYVAL_INVALID) { + /* This is the first call ever to PnetCDF API. Creating key + * ncmpi_init_keyval is necessary for MPI_Finalize() to free key + * ncmpi_comm_keyval. + */ + err = MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, end_call, + &ncmpi_init_keyval, (void*)0); + if (err != MPI_SUCCESS) + return ncmpii_error_mpi2nc(err, "MPI_Comm_create_keyval"); + + err = MPI_Comm_set_attr(MPI_COMM_SELF, ncmpi_init_keyval, (void*)0); + if (err != MPI_SUCCESS) + return ncmpii_error_mpi2nc(err, "MPI_Comm_set_attr"); + } + + if (ncmpi_comm_keyval == MPI_KEYVAL_INVALID) { + err = MPI_Comm_create_keyval(comm_attr_copy, + comm_attr_delete, + &ncmpi_comm_keyval, NULL); + if (err != MPI_SUCCESS) + return ncmpii_error_mpi2nc(err, "MPI_Comm_create_keyval"); + } + + if (ncmpi_comm_keyval != MPI_KEYVAL_INVALID) { + int found; + + err = MPI_Comm_get_attr(comm, ncmpi_comm_keyval, &attr, &found); + if (err != MPI_SUCCESS) + return ncmpii_error_mpi2nc(err, "MPI_Comm_get_attr"); + + if (!found) { + /* Construct an array storing node IDs of all processes. Note the + * memory allocated for setting the attribute will be freed in + * comm_attr_delete(), a callback function invoked when the + * MPI communicator is freed. + */ + attr = (PNC_comm_attr*) malloc(sizeof(PNC_comm_attr)); + attr->ref_count = 1; + + /* initialize INA metadata of intra-node aggregation */ + attr->num_aggrs_per_node = 0; + attr->num_ina_aggrs = 0; + attr->is_ina_aggr = 0; + attr->ina_ranks = NULL; + attr->numa_comm = MPI_COMM_NULL; + attr->ina_inter_comm = MPI_COMM_NULL; + attr->ina_intra_comm = MPI_COMM_NULL; + + if (nprocs == 1) { + attr->num_NUMAs = 1; + attr->NUMA_IDs = (int*) malloc(sizeof(int)); + attr->NUMA_IDs[0] = 0; + } + else { + /* Constructing NUMA compute node IDs requires communication + * calls to MPI_Comm_split_type(), MPI_Bcast(), and + * MPI_Allgather(). + */ + err = ncmpii_construct_node_list(comm, &attr->num_NUMAs, + &attr->NUMA_IDs, + &attr->numa_comm); + if (err != NC_NOERR) + return err; + + /* If INA is enabled, construct INA metadata */ + err = ina_init(comm, num_aggrs_per_node, attr); + if (err != NC_NOERR) DEBUG_RETURN_ERROR(err) + attr->num_aggrs_per_node = num_aggrs_per_node; + } + + /* FYI. The same key ncmpi_comm_keyval can be added to different + * MPI communicators with same or different values. + */ + err = MPI_Comm_set_attr(comm, ncmpi_comm_keyval, attr); + if (err != MPI_SUCCESS) + return ncmpii_error_mpi2nc(err, "MPI_Comm_set_attr"); + } + else { + if (num_aggrs_per_node == 0) { + /* When INA is disabled, there is no need to retrieved cached + * INA metadata. + */ + *attrP = *attr; /* retrieve num_NUMAs and ids */ + attrP->num_aggrs_per_node = 0; + attrP->num_ina_aggrs = 0; + attrP->is_ina_aggr = 01; + attrP->ina_ranks = NULL; + attrP->ina_inter_comm = MPI_COMM_NULL; + attrP->ina_intra_comm = MPI_COMM_NULL; + return NC_NOERR; + } + + /* When num_aggrs_per_node has changed, the cached attribute must + * be reset. MPI standard says calling MPI_Comm_set_attr() will + * trigger MPI_Comm_delete_attr() first, which will free all + * allocated space, e.g. NUMA_IDs[] and numa_comm. Thus, we must + * preserve attr->num_NUMAs, attr->NUMA_IDs[], and attr->numa_comm. + */ + if (num_aggrs_per_node != attr->num_aggrs_per_node) { + PNC_comm_attr *new_attr; + + /* updating an attribute must delete it first and set again */ + new_attr = (PNC_comm_attr*) malloc(sizeof(PNC_comm_attr)); + + /* copy out attributes that remains with the communicator */ + new_attr->ref_count = attr->ref_count; + new_attr->num_NUMAs = attr->num_NUMAs; + new_attr->NUMA_IDs = (int*) malloc(sizeof(int) * nprocs); + memcpy(new_attr->NUMA_IDs, attr->NUMA_IDs, sizeof(int)*nprocs); + + /* Must duplicate numa_comm that has been established, because + * it will be freed at the call to MPI_Comm_set_attr() below. + * Once created, numa_comm remains fixed with 'comm'. Unlike + * ina_inter_comm and ina_intra_comm, numa_comm does not change + * when num_aggrs_per_node changes. + */ + MPI_Comm_dup(attr->numa_comm, &new_attr->numa_comm); + + /* update with new INA metadata */ + new_attr->num_aggrs_per_node = num_aggrs_per_node; + new_attr->num_ina_aggrs = 0; + new_attr->is_ina_aggr = 0; + new_attr->ina_ranks = NULL; + new_attr->ina_inter_comm = MPI_COMM_NULL; + new_attr->ina_intra_comm = MPI_COMM_NULL; + + /* re-construct INA metadata */ + err = ina_init(comm, num_aggrs_per_node, new_attr); + if (err != NC_NOERR) DEBUG_RETURN_ERROR(err) + + err = MPI_Comm_set_attr(comm, ncmpi_comm_keyval, new_attr); + if (err != MPI_SUCCESS) + return ncmpii_error_mpi2nc(err, "MPI_Comm_set_attr"); + attr = new_attr; + } + } + + /* copy contents */ + *attrP = *attr; + } + + return NC_NOERR; } /*----< new_id_PNCList() >---------------------------------------------------*/ @@ -80,7 +618,7 @@ new_id_PNCList(int *new_id, PNC *pncp) { int i, err=NC_NOERR, perr=0; -#ifdef ENABLE_THREAD_SAFE +#if PNETCDF_THREAD_SAFE == 1 perr = pthread_mutex_lock(&lock); CHECK_ERRNO(perr, "pthread_mutex_lock") #endif @@ -99,7 +637,7 @@ new_id_PNCList(int *new_id, PNC *pncp) } } } -#ifdef ENABLE_THREAD_SAFE +#if PNETCDF_THREAD_SAFE == 1 perr = pthread_mutex_unlock(&lock); CHECK_ERRNO(perr, "pthread_mutex_unlock") @@ -114,7 +652,7 @@ del_from_PNCList(int ncid) { int perr=0; -#ifdef ENABLE_THREAD_SAFE +#if PNETCDF_THREAD_SAFE == 1 perr = pthread_mutex_lock(&lock); CHECK_ERRNO(perr, "pthread_mutex_lock") #endif @@ -123,7 +661,7 @@ del_from_PNCList(int ncid) pnc_filelist[ncid] = NULL; pnc_numfiles--; -#ifdef ENABLE_THREAD_SAFE +#if PNETCDF_THREAD_SAFE == 1 perr = pthread_mutex_unlock(&lock); CHECK_ERRNO(perr, "pthread_mutex_unlock") @@ -140,7 +678,7 @@ PNC_check_id(int ncid, PNC **pncp) assert(pncp != NULL); -#ifdef ENABLE_THREAD_SAFE +#if PNETCDF_THREAD_SAFE == 1 perr = pthread_mutex_lock(&lock); CHECK_ERRNO(perr, "pthread_mutex_lock") #endif @@ -150,7 +688,7 @@ PNC_check_id(int ncid, PNC **pncp) else *pncp = pnc_filelist[ncid]; -#ifdef ENABLE_THREAD_SAFE +#if PNETCDF_THREAD_SAFE == 1 perr = pthread_mutex_unlock(&lock); CHECK_ERRNO(perr, "pthread_mutex_unlock") @@ -160,19 +698,25 @@ PNC_check_id(int ncid, PNC **pncp) } /*----< construct_info() >---------------------------------------------------*/ +/* This subroutine reads I/O hints from the environment variable PNETCDF_HINTS, + * if set at the run time. The value of PNETCDF_HINTS is a character string + * consisting of one or more hints separated by ";" and each hint is in the + * form of "hint=value". E.g. "cb_nodes=16;cb_config_list=*:6". + * + * Hints set in PNETCDF_HINTS environment variable takes the highest precedence + * over hints set in the MPI info object passed from the application programs. + * + * When use_info is MPI_INFO_NULL and PNETCDF_HINTS is empty, MPI_INFO_NULL + * will be returned as new_info. + */ static void combine_env_hints(MPI_Info user_info, /* IN */ MPI_Info *new_info) /* OUT: may be MPI_INFO_NULL */ { char *warn_str="Warning: skip ill-formed hint set in PNETCDF_HINTS"; char *env_str; + char *hdr_align_val=NULL, *var_align_val=NULL; - /* take hints from the environment variable PNETCDF_HINTS, a string of - * hints separated by ";" and each hint is in the form of hint=value. E.g. - * "cb_nodes=16;cb_config_list=*:6". If this environment variable is set, - * it overrides the same hints that were set by MPI_Info_set() called in - * the application program. - */ if (user_info != MPI_INFO_NULL) MPI_Info_dup(user_info, new_info); /* ignore error */ else @@ -182,17 +726,17 @@ combine_env_hints(MPI_Info user_info, /* IN */ if ((env_str = getenv("PNETCDF_HINTS")) != NULL) { #ifdef USE_STRTOK_R char *env_str_cpy, *env_str_saved, *hint, *key; - env_str_cpy = strdup(env_str); + env_str_cpy = NCI_Strdup(env_str); env_str_saved = env_str_cpy; hint = strtok_r(env_str_cpy, ";", &env_str_saved); while (hint != NULL) { - char *hint_saved = strdup(hint); + char *hint_saved = NCI_Strdup(hint); char *val = strchr(hint, '='); if (val == NULL) { /* ill-formed hint */ if (NULL != strtok(hint, " \t")) printf("%s: '%s'\n", warn_str, hint_saved); /* else case: ignore white-spaced hints */ - free(hint_saved); + NCI_Free(hint_saved); hint = strtok_r(NULL, ";", &env_str_saved); /* get next hint */ continue; } @@ -203,18 +747,24 @@ combine_env_hints(MPI_Info user_info, /* IN */ else { if (*new_info == MPI_INFO_NULL) MPI_Info_create(new_info); /* ignore error */ - MPI_Info_set(*new_info, key, val); /* override or add */ + + if (!strcmp(key, "nc_header_align_size")) + hdr_align_val = NCI_Strdup(val); + else if (!strcmp(key, "nc_var_align_size")) + var_align_val = NCI_Strdup(val); + else + MPI_Info_set(*new_info, key, val); /* override or add */ } /* printf("env hint: key=%s val=%s\n",key,val); */ hint = strtok_r(NULL, ";", &env_str_saved); - free(hint_saved); + NCI_Free(hint_saved); } - free(env_str_cpy); + NCI_Free(env_str_cpy); #else char *env_str_cpy, *hint, *next_hint, *key, *val, *deli; char *hint_saved=NULL; - env_str_cpy = strdup(env_str); + env_str_cpy = NCI_Strdup(env_str); next_hint = env_str_cpy; do { @@ -225,14 +775,14 @@ combine_env_hints(MPI_Info user_info, /* IN */ next_hint = deli + 1; } else next_hint = "\0"; - if (hint_saved != NULL) free(hint_saved); + if (hint_saved != NULL) NCI_Free(hint_saved); /* skip all-blank hint */ - hint_saved = strdup(hint); + hint_saved = NCI_Strdup(hint); if (strtok(hint, " \t") == NULL) continue; - free(hint_saved); - hint_saved = strdup(hint); /* save hint for error message */ + NCI_Free(hint_saved); + hint_saved = NCI_Strdup(hint); /* save hint for error message */ deli = strchr(hint, '='); if (deli == NULL) { /* ill-formed hint */ @@ -257,15 +807,72 @@ combine_env_hints(MPI_Info user_info, /* IN */ } if (*new_info == MPI_INFO_NULL) MPI_Info_create(new_info); /* ignore error */ - MPI_Info_set(*new_info, key, val); /* override or add */ + + if (!strcmp(key, "nc_header_align_size")) + hdr_align_val = NCI_Strdup(val); + else if (!strcmp(key, "nc_var_align_size")) + var_align_val = NCI_Strdup(val); + else + MPI_Info_set(*new_info, key, val); /* override or add */ } while (*next_hint != '\0'); - if (hint_saved != NULL) free(hint_saved); - free(env_str_cpy); + if (hint_saved != NULL) NCI_Free(hint_saved); + NCI_Free(env_str_cpy); #endif + + /* nc_var_align_size supersedes nc_header_align_size */ + if (var_align_val != NULL) { + MPI_Info_set(*new_info, "nc_var_align_size", var_align_val); + MPI_Info_set(*new_info, "nc_header_align_size", var_align_val); + } + else if (hdr_align_val != NULL) { + MPI_Info_set(*new_info, "nc_var_align_size", hdr_align_val); + MPI_Info_set(*new_info, "nc_header_align_size", hdr_align_val); + } } /* return no error as all hints are advisory */ + + if (hdr_align_val != NULL) NCI_Free(hdr_align_val); + if (var_align_val != NULL) NCI_Free(var_align_val); +} + +/*----< set_env_mode() >-----------------------------------------------------*/ +static +void set_env_mode(int *env_mode) +{ + char *env_str; + +#if PNETCDF_DEBUG_MODE == 1 + fSet(*env_mode, NC_MODE_SAFE); + /* When debug mode is enabled at the configure time, safe mode is by + * default enabled. This can be overwritten by the run-time environment + * variable PNETCDF_SAFE_MODE. + */ +#endif + /* get environment variable PNETCDF_SAFE_MODE + * if it is set to 1, then we perform a strict parameter consistent test + */ + if ((env_str = getenv("PNETCDF_SAFE_MODE")) != NULL) { + if (*env_str == '0') fClr(*env_mode, NC_MODE_SAFE); + else fSet(*env_mode, NC_MODE_SAFE); + /* if PNETCDF_SAFE_MODE is set but without a value, *env_str can + * be '\0' (null character). In this case, safe mode is enabled */ + } + + /* get environment variable PNETCDF_RELAX_COORD_BOUND + * if it is set to 0, then we perform a strict start bound check + */ +#if PNETCDF_RELAX_COORD_BOUND == 1 + fSet(*env_mode, NC_MODE_STRICT_COORD_BOUND); +#endif + if ((env_str = getenv("PNETCDF_RELAX_COORD_BOUND")) != NULL) { + if (*env_str == '0') fClr(*env_mode, NC_MODE_STRICT_COORD_BOUND); + else fSet(*env_mode, NC_MODE_STRICT_COORD_BOUND); + /* if PNETCDF_RELAX_COORD_BOUND is set but without a value, *env_str + * can be '\0' (null character). This is equivalent to setting + * PNETCDF_RELAX_COORD_BOUND to 1 */ + } } /*----< ncmpi_create() >-----------------------------------------------------*/ @@ -277,74 +884,106 @@ ncmpi_create(MPI_Comm comm, MPI_Info info, int *ncidp) { - int rank, nprocs, status=NC_NOERR, err; - int safe_mode=0, mpireturn, relax_coord_bound, format; - char *env_str; - MPI_Info combined_info; + char value[MPI_MAX_INFO_VAL]; + int rank, nprocs, status=NC_NOERR, err, flag; + int env_mode=0, mpireturn, format, num_aggrs_per_node; + MPI_Info combined_info=MPI_INFO_NULL; void *ncp; PNC *pncp; PNC_driver *driver; + PNC_comm_attr comm_attr; #ifdef BUILD_DRIVER_FOO int enable_foo_driver=0; #endif -#ifdef ENABLE_BURST_BUFFER +#if PNETCDF_BURST_BUFFERING == 1 int enable_bb_driver=0; #endif - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &nprocs); - -#ifdef PNETCDF_DEBUG - safe_mode = 1; - /* When debug mode is enabled at the configure time, safe_mode is by - * default enabled. This can be overwritten by the run-time environment - * variable PNETCDF_SAFE_MODE */ +#if PNETCDF_PROFILING == 1 + { + int i; + pnc_num_aggrs_per_node = 0; + pnc_ina_init = 0; + pnc_ina_flatten = 0; + pnc_ina_npairs_put = 0; + pnc_ina_npairs_get = 0; + + for (i=0; ipath = NULL; + + /* The first thing is to duplicate the MPI communicator (even if comm is + * MPI_COMM_WORLD or MPI_COMM_SELF) before any communication can be made + * within PnetCDF. This is because users may use 'comm' to do other + * point-to-point communication while PnetCDF also makes some MPI + * communication calls. When this happened, user's communication can mess + * up with the PnetCDF's communication, particularly when in independent + * data mode. Once comm is duplicated, we pass pncp->comm to PnetCDF + * drivers, so there is no need for a driver to duplicate it again. */ - if ((env_str = getenv("PNETCDF_SAFE_MODE")) != NULL) { - if (*env_str == '0') safe_mode = 0; - else safe_mode = 1; - /* if PNETCDF_SAFE_MODE is set but without a value, *env_str can - * be '\0' (null character). In this case, safe_mode is enabled */ + pncp->comm = MPI_COMM_NULL; + mpireturn = MPI_Comm_dup(comm, &pncp->comm); + if (mpireturn != MPI_SUCCESS) { + err = ncmpii_error_mpi2nc(mpireturn, "MPI_Comm_dup"); + goto err_out; } - /* get environment variable PNETCDF_RELAX_COORD_BOUND - * if it is set to 0, then we perform a strict start bound check + /* Rudimentary check path's validity. + * + * Note MPI standard's requirement for filename argument is: " ... all + * processes must provide filenames that reference the same file. ... The + * user is responsible for ensuring that a single file is referenced by the + * filename argument, as it may be impossible for an implementation to + * detect this type of namespace error." */ -#ifndef RELAX_COORD_BOUND - relax_coord_bound = 0; -#else - relax_coord_bound = 1; -#endif - if ((env_str = getenv("PNETCDF_RELAX_COORD_BOUND")) != NULL) { - if (*env_str == '0') relax_coord_bound = 0; - else relax_coord_bound = 1; - /* if PNETCDF_RELAX_COORD_BOUND is set but without a value, *env_str - * can be '\0' (null character). This is equivalent to setting - * relax_coord_bound to 1 */ + if (path == NULL || *path == '\0') { + if (status == NC_NOERR) status = NC_EBAD_FILE; + goto err_out; } - /* path's validity is checked in MPI-IO with error code MPI_ERR_BAD_FILE - * path consistency is checked in MPI-IO with error code MPI_ERR_NOT_SAME - */ - if (path == NULL || *path == '\0') DEBUG_RETURN_ERROR(NC_EBAD_FILE) + MPI_Comm_rank(pncp->comm, &rank); + MPI_Comm_size(pncp->comm, &nprocs); + + if (rank == 0) + set_env_mode(&env_mode); + + /* duplicate file path */ + pncp->path = (char*) NCI_Strdup(path); + if (pncp->path == NULL) { + DEBUG_ASSIGN_ERROR(status, NC_ENOMEM) + goto err_out; + } if (nprocs > 1) { /* Check cmode consistency */ - int root_cmode = cmode; /* only root's matters */ - TRACE_COMM(MPI_Bcast)(&root_cmode, 1, MPI_INT, 0, comm); + int modes[2] = {cmode, env_mode}; /* only root's matters */ + + TRACE_COMM(MPI_Bcast)(&modes, 2, MPI_INT, 0, pncp->comm); NCMPII_HANDLE_ERROR("MPI_Bcast") /* Overwrite cmode with root's cmode */ - if (root_cmode != cmode) { - cmode = root_cmode; + if (modes[0] != cmode) { + cmode = modes[0]; DEBUG_ASSIGN_ERROR(status, NC_EMULTIDEFINE_CMODE) } - if (safe_mode) { /* sync status among all processes */ + env_mode = modes[1]; + if (fIsSet(env_mode, NC_MODE_SAFE)) { + /* sync status among all processes */ err = status; - TRACE_COMM(MPI_Allreduce)(&err, &status, 1, MPI_INT, MPI_MIN, comm); + TRACE_COMM(MPI_Allreduce)(&err, &status, 1, MPI_INT, MPI_MIN, + pncp->comm); NCMPII_HANDLE_ERROR("MPI_Allreduce") } /* continue to use root's cmode to create the file, but will report @@ -354,30 +993,70 @@ ncmpi_create(MPI_Comm comm, /* combine user's MPI info and PNETCDF_HINTS env variable */ combine_env_hints(info, &combined_info); -#ifdef BUILD_DRIVER_FOO + num_aggrs_per_node = 0; + if (nprocs > 1 && combined_info != MPI_INFO_NULL) { + /* check if INA hint is enabled */ + MPI_Info_get(combined_info, "nc_num_aggrs_per_node", + MPI_MAX_INFO_VAL-1, value, &flag); + if (flag) { + int ival; + errno = 0; /* errno must set to zero before calling atoi */ + ival = atoi(value); + if (errno == 0 && ival >= 0) + num_aggrs_per_node = ival; + } + } +#if PNETCDF_PROFILING == 1 + pnc_num_aggrs_per_node = num_aggrs_per_node; +#endif + +#if PNETCDF_THREAD_SAFE == 1 + int perr; + perr = pthread_mutex_lock(&lock); + if (perr != 0) + printf("Warning in file %s line %d: pthread_mutex_lock() failed (%s)\n", + __FILE__, __LINE__, strerror(perr)); +#endif + + /* creating communicator attributes must be protected by a mutex */ + set_get_comm_attr(pncp->comm, num_aggrs_per_node, &comm_attr); + /* ignore error, as it is not a critical error */ + +#if PNETCDF_DEBUG_MODE == 1 + if (num_aggrs_per_node == 0) + assert(comm_attr.ina_intra_comm == MPI_COMM_NULL); + else + assert(comm_attr.ina_intra_comm != MPI_COMM_NULL); +#endif + +#if PNETCDF_THREAD_SAFE == 1 + perr = pthread_mutex_unlock(&lock); + if (perr != 0) + printf("Warning in file %s line %d: pthread_mutex_unlock() failed (%s)\n", + __FILE__, __LINE__, strerror(perr)); +#endif + if (combined_info == MPI_INFO_NULL) MPI_Info_create(&combined_info); - { - char value[MPI_MAX_INFO_VAL]; - int flag; + /* add this rank's NUMA node ID */ + snprintf(value, MAX_INT_LEN, "%d", comm_attr.NUMA_IDs[rank]); + MPI_Info_set(combined_info, "NUMA_ID", value); + +#ifdef BUILD_DRIVER_FOO + if (combined_info != MPI_INFO_NULL) { /* check if nc_foo_driver is enabled */ MPI_Info_get(combined_info, "nc_foo_driver", MPI_MAX_INFO_VAL-1, - value, &flag); + value, &flag); if (flag && strcasecmp(value, "enable") == 0) enable_foo_driver = 1; } #endif -#ifdef ENABLE_BURST_BUFFER - if (combined_info == MPI_INFO_NULL) - MPI_Info_create(&combined_info); - { - char value[MPI_MAX_INFO_VAL]; - int flag; - +#if PNETCDF_BURST_BUFFERING == 1 + if (combined_info != MPI_INFO_NULL) { /* check if nc_burst_buf is enabled */ MPI_Info_get(combined_info, "nc_burst_buf", MPI_MAX_INFO_VAL-1, - value, &flag); + value, &flag); if (flag && strcasecmp(value, "enable") == 0) enable_bb_driver = 1; } @@ -387,30 +1066,27 @@ ncmpi_create(MPI_Comm comm, * which is later used to select the right driver. */ -#ifdef ENABLE_NETCDF4 +#if PNETCDF_DRIVER_NETCDF4 == 1 /* It is illegal to have NC_64BIT_OFFSET & NC_64BIT_DATA & NC_NETCDF4 */ if ((cmode & (NC_64BIT_OFFSET|NC_NETCDF4)) == (NC_64BIT_OFFSET|NC_NETCDF4) || (cmode & (NC_64BIT_DATA|NC_NETCDF4)) == (NC_64BIT_DATA|NC_NETCDF4)) { - if (combined_info != MPI_INFO_NULL) - MPI_Info_free(&combined_info); - DEBUG_RETURN_ERROR(NC_EINVAL_CMODE) + DEBUG_ASSIGN_ERROR(status, NC_EINVAL_CMODE) + goto err_out; } #else if (cmode & NC_NETCDF4) { - if (combined_info != MPI_INFO_NULL) - MPI_Info_free(&combined_info); - DEBUG_RETURN_ERROR(NC_ENOTBUILT) + DEBUG_ASSIGN_ERROR(status, NC_ENOTBUILT) + goto err_out; } #endif /* It is illegal to have both NC_64BIT_OFFSET & NC_64BIT_DATA */ if ((cmode & (NC_64BIT_OFFSET|NC_64BIT_DATA)) == (NC_64BIT_OFFSET|NC_64BIT_DATA)) { - if (combined_info != MPI_INFO_NULL) - MPI_Info_free(&combined_info); - DEBUG_RETURN_ERROR(NC_EINVAL_CMODE) + DEBUG_ASSIGN_ERROR(status, NC_EINVAL_CMODE) + goto err_out; } /* Check if cmode contains format specific flag */ @@ -439,14 +1115,14 @@ ncmpi_create(MPI_Comm comm, cmode |= NC_NETCDF4 | NC_CLASSIC_MODEL; } -#ifdef ENABLE_NETCDF4 +#if PNETCDF_DRIVER_NETCDF4 == 1 if (format == NC_FORMAT_NETCDF4 || format == NC_FORMAT_NETCDF4_CLASSIC) { driver = nc4io_inq_driver(); -#ifdef ENABLE_BURST_BUFFER +#if PNETCDF_BURST_BUFFERING == 1 /* Burst buffering does not support NetCDF-4 files yet. * If hint nc_burst_buf is enabled in combined_info, disable it. */ - if (enable_bb_driver == 1) + if (enable_bb_driver == 1 && combined_info != MPI_INFO_NULL) MPI_Info_set(combined_info, "nc_burst_buf", "disable"); enable_bb_driver = 0; #endif @@ -458,7 +1134,7 @@ ncmpi_create(MPI_Comm comm, driver = ncfoo_inq_driver(); else #endif -#ifdef ENABLE_BURST_BUFFER +#if PNETCDF_BURST_BUFFERING == 1 if (enable_bb_driver) driver = ncbbio_inq_driver(); else @@ -466,60 +1142,27 @@ ncmpi_create(MPI_Comm comm, /* default is the driver built on top of MPI-IO */ driver = ncmpio_inq_driver(); - /* allocate a new PNC object */ - pncp = (PNC*) NCI_Malloc(sizeof(PNC)); - if (pncp == NULL) { - *ncidp = -1; - if (combined_info != MPI_INFO_NULL) - MPI_Info_free(&combined_info); - DEBUG_RETURN_ERROR(NC_ENOMEM) - } + pncp->flag = NC_MODE_DEF | NC_MODE_CREATE; + fSet(pncp->flag, env_mode); /* generate a new nc file ID from NCPList */ err = new_id_PNCList(ncidp, pncp); if (err != NC_NOERR) { if (combined_info != MPI_INFO_NULL) MPI_Info_free(&combined_info); - return err; + DEBUG_ASSIGN_ERROR(status, err) + goto err_out; } - /* Duplicate comm, because users may free it (though unlikely). Note - * MPI_Comm_dup() is collective. We pass pncp->comm to drivers, so there - * is no need for a driver to duplicate it again. - */ - if (comm != MPI_COMM_WORLD && comm != MPI_COMM_SELF) { - mpireturn = MPI_Comm_dup(comm, &pncp->comm); - if (mpireturn != MPI_SUCCESS) - return ncmpii_error_mpi2nc(mpireturn, "MPI_Comm_dup"); - } - else - pncp->comm = comm; - /* calling the driver's create subroutine */ - err = driver->create(pncp->comm, path, cmode, *ncidp, combined_info, &ncp); + err = driver->create(pncp->comm, pncp->path, cmode, *ncidp, env_mode, + combined_info, comm_attr, &ncp); if (status == NC_NOERR) status = err; - if (combined_info != MPI_INFO_NULL) MPI_Info_free(&combined_info); if (status != NC_NOERR && status != NC_EMULTIDEFINE_CMODE) { del_from_PNCList(*ncidp); - if (pncp->comm != MPI_COMM_WORLD && pncp->comm != MPI_COMM_SELF) - MPI_Comm_free(&pncp->comm); /* a collective call */ - NCI_Free(pncp); - *ncidp = -1; - return status; + goto err_out; } - /* fill in pncp members */ - pncp->path = (char*) NCI_Malloc(strlen(path)+1); - if (pncp->path == NULL) { - driver->close(ncp); /* close file and ignore error */ - del_from_PNCList(*ncidp); - if (pncp->comm != MPI_COMM_WORLD && pncp->comm != MPI_COMM_SELF) - MPI_Comm_free(&pncp->comm); /* a collective call */ - NCI_Free(pncp); - *ncidp = -1; - DEBUG_RETURN_ERROR(NC_ENOMEM) - } - strcpy(pncp->path, path); pncp->mode = cmode; pncp->driver = driver; pncp->ndims = 0; @@ -527,12 +1170,27 @@ ncmpi_create(MPI_Comm comm, pncp->nvars = 0; pncp->nrec_vars = 0; pncp->vars = NULL; - pncp->flag = NC_MODE_DEF | NC_MODE_CREATE; pncp->ncp = ncp; pncp->format = format; - if (safe_mode) pncp->flag |= NC_MODE_SAFE; - if (!relax_coord_bound) pncp->flag |= NC_MODE_STRICT_COORD_BOUND; + if (fIsSet(env_mode, NC_MODE_SAFE)) + pncp->flag |= NC_MODE_SAFE; + + if (fIsSet(env_mode, NC_MODE_STRICT_COORD_BOUND)) + pncp->flag |= NC_MODE_STRICT_COORD_BOUND; + +err_out: + if (combined_info != MPI_INFO_NULL) + MPI_Info_free(&combined_info); + + if (status != NC_NOERR && status != NC_EMULTIDEFINE_CMODE) { + if (pncp->comm != MPI_COMM_NULL) + MPI_Comm_free(&pncp->comm); /* a collective call */ + if (pncp->path != NULL) + NCI_Free(pncp->path); + NCI_Free(pncp); + *ncidp = -1; + } return status; } @@ -548,59 +1206,78 @@ ncmpi_open(MPI_Comm comm, MPI_Info info, int *ncidp) /* OUT */ { - int i, j, nalloc, rank, nprocs, format, status=NC_NOERR, err; - int safe_mode=0, mpireturn, relax_coord_bound, DIMIDS[NDIMS_], *dimids; - char *env_str; - MPI_Info combined_info; + char value[MPI_MAX_INFO_VAL]; + int i, j, nalloc, rank, nprocs, format, status=NC_NOERR, err, flag; + int env_mode=0, mpireturn, DIMIDS[NDIMS_], *dimids, num_aggrs_per_node; + MPI_Info combined_info=MPI_INFO_NULL; void *ncp; PNC *pncp; PNC_driver *driver; + PNC_comm_attr comm_attr; #ifdef BUILD_DRIVER_FOO int enable_foo_driver=0; #endif -#ifdef ENABLE_BURST_BUFFER +#if PNETCDF_BURST_BUFFERING == 1 int enable_bb_driver=0; #endif - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &nprocs); +#if PNETCDF_PROFILING == 1 + pnc_num_aggrs_per_node = 0; + pnc_ina_init = 0; + pnc_ina_flatten = 0; + pnc_ina_npairs_put = 0; + pnc_ina_npairs_get = 0; -#ifdef PNETCDF_DEBUG - safe_mode = 1; - /* When debug mode is enabled at the configure time, safe_mode is by - * default enabled. This can be overwritten by the run-time environment - * variable PNETCDF_SAFE_MODE */ + for (i=0; ipath = NULL; + + /* The first thing is to duplicate the MPI communicator (even if comm is + * MPI_COMM_WORLD or MPI_COMM_SELF) before any communication can be made + * within PnetCDF. This is because users may use 'comm' to do other + * point-to-point communication while PnetCDF also makes some MPI + * communication calls. When this happened, user's communication can mess + * up with the PnetCDF's communication, particularly when in independent + * data mode. Once comm is duplicated, we pass pncp->comm to PnetCDF + * drivers, so there is no need for a driver to duplicate it again. */ - if ((env_str = getenv("PNETCDF_SAFE_MODE")) != NULL) { - if (*env_str == '0') safe_mode = 0; - else safe_mode = 1; - /* if PNETCDF_SAFE_MODE is set but without a value, *env_str can - * be '\0' (null character). In this case, safe_mode is enabled */ + pncp->comm = MPI_COMM_NULL; + mpireturn = MPI_Comm_dup(comm, &pncp->comm); + if (mpireturn != MPI_SUCCESS) { + err = ncmpii_error_mpi2nc(mpireturn, "MPI_Comm_dup"); + if (status == NC_NOERR) status = err; + goto err_out; } - /* get environment variable PNETCDF_RELAX_COORD_BOUND - * if it is set to 0, then we perform a strict start bound check + /* Rudimentary check path's validity. + * + * Note MPI standard's requirement for filename argument is: " ... all + * processes must provide filenames that reference the same file. ... The + * user is responsible for ensuring that a single file is referenced by the + * filename argument, as it may be impossible for an implementation to + * detect this type of namespace error." */ -#ifndef RELAX_COORD_BOUND - relax_coord_bound = 0; -#else - relax_coord_bound = 1; -#endif - if ((env_str = getenv("PNETCDF_RELAX_COORD_BOUND")) != NULL) { - if (*env_str == '0') relax_coord_bound = 0; - else relax_coord_bound = 1; - /* if PNETCDF_RELAX_COORD_BOUND is set but without a value, *env_str - * can be '\0' (null character). This is equivalent to setting - * relax_coord_bound to 1 */ + if (path == NULL || *path == '\0') { + if (status == NC_NOERR) status = NC_EBAD_FILE; + goto err_out; } - /* path's validity is checked in MPI-IO with error code MPI_ERR_BAD_FILE - * path consistency is checked in MPI-IO with error code MPI_ERR_NOT_SAME - */ - if (path == NULL || *path == '\0') DEBUG_RETURN_ERROR(NC_EBAD_FILE) + MPI_Comm_rank(pncp->comm, &rank); + MPI_Comm_size(pncp->comm, &nprocs); + + if (rank == 0) + set_env_mode(&env_mode); /* Check the file signature to tell the file format which is later used to * select the right driver. @@ -609,105 +1286,160 @@ ncmpi_open(MPI_Comm comm, if (rank == 0) { err = ncmpi_inq_file_format(path, &format); if (err != NC_NOERR) { - if (nprocs == 1) return err; + if (nprocs == 1) { + if (status == NC_NOERR) status = err; + goto err_out; + } format = err; } else if (format == NC_FORMAT_UNKNOWN) { - if (nprocs == 1) DEBUG_RETURN_ERROR(NC_ENOTNC) + if (nprocs == 1) { + if (status == NC_NOERR) status = NC_ENOTNC; + goto err_out; + } format = NC_ENOTNC; } -#ifndef ENABLE_NETCDF4 +#if PNETCDF_DRIVER_NETCDF4 == 0 else if (format == NC_FORMAT_NETCDF4 || format == NC_FORMAT_NETCDF4_CLASSIC) { - if (nprocs == 1) DEBUG_RETURN_ERROR(NC_ENOTBUILT) + if (nprocs == 1) { + if (status == NC_NOERR) status = NC_ENOTBUILT; + goto err_out; + } format = NC_ENOTBUILT; } #endif } if (nprocs > 1) { /* root broadcasts format and omode */ - int root_omode, msg[2]; - - msg[0] = format; /* only root's matters (format or error code) */ + int modes[3] = {format, omode, env_mode}; - /* Check omode consistency: + /* Check consistency: + * Note only root's values matter, format, omode, env_mode. * Note if omode contains NC_NOWRITE, it is equivalent to NC_CLOBBER. * In pnetcdf.h, they both are defined the same value, 0. - * Only root's omode matters. */ - msg[1] = omode; /* only root's matters */ - TRACE_COMM(MPI_Bcast)(&msg, 2, MPI_INT, 0, comm); + TRACE_COMM(MPI_Bcast)(&modes, 3, MPI_INT, 0, pncp->comm); NCMPII_HANDLE_ERROR("MPI_Bcast") /* check format error (a fatal error, must return now) */ - format = msg[0]; - if (format < 0) return format; /* all netCDF errors are negative */ + format = modes[0]; + if (format < 0) { /* all netCDF errors are negative */ + if (status == NC_NOERR) status = format; + goto err_out; + } /* check omode consistency */ - root_omode = msg[1]; - if (root_omode != omode) { - omode = root_omode; - DEBUG_ASSIGN_ERROR(status, NC_EMULTIDEFINE_OMODE) + if (modes[1] != omode) { + omode = modes[1]; + if (status == NC_NOERR) status = NC_EMULTIDEFINE_OMODE; } - if (safe_mode) { /* sync status among all processes */ + env_mode = modes[2]; + if (fIsSet(env_mode, NC_MODE_SAFE)) { + /* sync status among all processes */ err = status; - TRACE_COMM(MPI_Allreduce)(&err, &status, 1, MPI_INT, MPI_MIN, comm); + TRACE_COMM(MPI_Allreduce)(&err, &status, 1, MPI_INT, MPI_MIN, + pncp->comm); NCMPII_HANDLE_ERROR("MPI_Allreduce") } /* continue to use root's omode to open the file, but will report omode - * inconsistency error, if there is any */ + * inconsistency error, if there is any + */ } /* combine user's MPI info and PNETCDF_HINTS env variable */ combine_env_hints(info, &combined_info); -#ifdef BUILD_DRIVER_FOO + num_aggrs_per_node = 0; + if (nprocs > 1 && combined_info != MPI_INFO_NULL) { + /* check if INA hint is enabled */ + MPI_Info_get(combined_info, "nc_num_aggrs_per_node", + MPI_MAX_INFO_VAL-1, value, &flag); + if (flag) { + int ival; + errno = 0; /* errno must set to zero before calling atoi */ + ival = atoi(value); + if (errno == 0 && ival >= 0) + num_aggrs_per_node = ival; + } + } +#if PNETCDF_PROFILING == 1 + pnc_num_aggrs_per_node = num_aggrs_per_node; +#endif + +#if PNETCDF_THREAD_SAFE == 1 + int perr; + perr = pthread_mutex_lock(&lock); + if (perr != 0) + printf("Warning in file %s line %d: pthread_mutex_lock() failed (%s)\n", + __FILE__, __LINE__, strerror(perr)); +#endif + + /* creating communicator attributes must be protected by a mutex */ + set_get_comm_attr(pncp->comm, num_aggrs_per_node, &comm_attr); + /* ignore error, as it is not a critical error */ + +#if PNETCDF_DEBUG_MODE == 1 + if (num_aggrs_per_node == 0) + assert(comm_attr.ina_intra_comm == MPI_COMM_NULL); + else + assert(comm_attr.ina_intra_comm != MPI_COMM_NULL); +#endif + +#if PNETCDF_THREAD_SAFE == 1 + perr = pthread_mutex_unlock(&lock); + if (perr != 0) + printf("Warning in file %s line %d: pthread_mutex_unlock() failed (%s)\n", + __FILE__, __LINE__, strerror(perr)); +#endif + if (combined_info == MPI_INFO_NULL) MPI_Info_create(&combined_info); - { - char value[MPI_MAX_INFO_VAL]; - int flag; + /* add this rank's NUMA node ID */ + snprintf(value, MAX_INT_LEN, "%d", comm_attr.NUMA_IDs[rank]); + MPI_Info_set(combined_info, "NUMA_ID", value); + +#ifdef BUILD_DRIVER_FOO + if (combined_info != MPI_INFO_NULL) { /* check if nc_foo_driver is enabled */ MPI_Info_get(combined_info, "nc_foo_driver", MPI_MAX_INFO_VAL-1, - value, &flag); + value, &flag); if (flag && strcasecmp(value, "enable") == 0) enable_foo_driver = 1; - } #endif -#ifdef ENABLE_BURST_BUFFER - if (combined_info == MPI_INFO_NULL) - MPI_Info_create(&combined_info); - { - char value[MPI_MAX_INFO_VAL]; - int flag; - +#if PNETCDF_BURST_BUFFERING == 1 + if (combined_info != MPI_INFO_NULL) { /* check if nc_burst_buf is enabled */ MPI_Info_get(combined_info, "nc_burst_buf", MPI_MAX_INFO_VAL-1, - value, &flag); + value, &flag); if (flag && strcasecmp(value, "enable") == 0) enable_bb_driver = 1; } #endif -#ifdef ENABLE_NETCDF4 +#if PNETCDF_DRIVER_NETCDF4 == 1 if (format == NC_FORMAT_NETCDF4_CLASSIC || format == NC_FORMAT_NETCDF4) { driver = nc4io_inq_driver(); -#ifdef ENABLE_BURST_BUFFER +#if PNETCDF_BURST_BUFFERING == 1 /* Burst buffering does not support NetCDF-4 files yet. * If hint nc_burst_buf is enabled in combined_info, disable it. */ - if (enable_bb_driver == 1) - MPI_Info_set(combined_info, "nc_burst_buf", "disable"); + if (enable_bb_driver == 1) { + if (combined_info != MPI_INFO_NULL) + MPI_Info_set(combined_info, "nc_burst_buf", "disable"); + } enable_bb_driver = 0; #endif } else #else - if (format == NC_FORMAT_NETCDF4_CLASSIC || format == NC_FORMAT_NETCDF4) - DEBUG_RETURN_ERROR(NC_ENOTBUILT) + if (format == NC_FORMAT_NETCDF4_CLASSIC || format == NC_FORMAT_NETCDF4) { + if (status == NC_NOERR) status = NC_ENOTBUILT; + goto err_out; + } else #endif #ifdef BUILD_DRIVER_FOO @@ -715,7 +1447,7 @@ ncmpi_open(MPI_Comm comm, driver = ncfoo_inq_driver(); else #endif -#ifdef ENABLE_BURST_BUFFER +#if PNETCDF_BURST_BUFFERING == 1 if (enable_bb_driver) driver = ncbbio_inq_driver(); else @@ -727,66 +1459,47 @@ ncmpi_open(MPI_Comm comm, format == NC_FORMAT_CDF5) { driver = ncmpio_inq_driver(); } -#ifdef ENABLE_ADIOS +#if PNETCDF_DRIVER_ADIOS == 1 else if (format == NC_FORMAT_BP) { driver = ncadios_inq_driver(); } #endif - else /* unrecognized file format */ - DEBUG_RETURN_ERROR(NC_ENOTNC) + else { /* unrecognized file format */ + if (status == NC_NOERR) status = NC_ENOTNC; + goto err_out; + } } - /* allocate a PNC object */ - pncp = (PNC*) NCI_Malloc(sizeof(PNC)); - if (pncp == NULL) { - *ncidp = -1; - DEBUG_RETURN_ERROR(NC_ENOMEM) + pncp->flag = 0; + fSet(pncp->flag, env_mode); + + /* duplicate file path */ + pncp->path = (char*) NCI_Strdup(path); + if (pncp->path == NULL) { + if (status == NC_NOERR) status = NC_ENOMEM; + goto err_out; } /* generate a new nc file ID from NCPList */ err = new_id_PNCList(ncidp, pncp); - if (err != NC_NOERR) return err; - - /* Duplicate comm, because users may free it (though unlikely). Note - * MPI_Comm_dup() is collective. We pass pncp->comm to drivers, so there - * is no need for a driver to duplicate it again. - */ - if (comm != MPI_COMM_WORLD && comm != MPI_COMM_SELF) { - mpireturn = MPI_Comm_dup(comm, &pncp->comm); - if (mpireturn != MPI_SUCCESS) - return ncmpii_error_mpi2nc(mpireturn, "MPI_Comm_dup"); + if (err != NC_NOERR) { + if (status == NC_NOERR) status = err; + goto err_out; } - else - pncp->comm = comm; /* calling the driver's open subroutine */ - err = driver->open(pncp->comm, path, omode, *ncidp, combined_info, &ncp); + err = driver->open(pncp->comm, pncp->path, omode, *ncidp, env_mode, + combined_info, comm_attr, &ncp); if (status == NC_NOERR) status = err; - if (combined_info != MPI_INFO_NULL) MPI_Info_free(&combined_info); if (status != NC_NOERR && status != NC_EMULTIDEFINE_OMODE && status != NC_ENULLPAD) { /* NC_EMULTIDEFINE_OMODE and NC_ENULLPAD are not fatal error. We * continue the rest open procedure */ del_from_PNCList(*ncidp); - if (pncp->comm != MPI_COMM_WORLD && pncp->comm != MPI_COMM_SELF) - MPI_Comm_free(&pncp->comm); /* a collective call */ - NCI_Free(pncp); - *ncidp = -1; - return status; + goto err_out; } /* fill in pncp members */ - pncp->path = (char*) NCI_Malloc(strlen(path)+1); - if (pncp->path == NULL) { - driver->close(ncp); /* close file and ignore error */ - del_from_PNCList(*ncidp); - if (pncp->comm != MPI_COMM_WORLD && pncp->comm != MPI_COMM_SELF) - MPI_Comm_free(&pncp->comm); /* a collective call */ - NCI_Free(pncp); - *ncidp = -1; - DEBUG_RETURN_ERROR(NC_ENOMEM) - } - strcpy(pncp->path, path); pncp->mode = omode; pncp->driver = driver; pncp->ndims = 0; @@ -794,31 +1507,43 @@ ncmpi_open(MPI_Comm comm, pncp->nvars = 0; pncp->nrec_vars = 0; pncp->vars = NULL; - pncp->flag = 0; pncp->ncp = ncp; pncp->format = format; - if (!fIsSet(omode, NC_WRITE)) pncp->flag |= NC_MODE_RDONLY; - if (safe_mode) pncp->flag |= NC_MODE_SAFE; - if (!relax_coord_bound) pncp->flag |= NC_MODE_STRICT_COORD_BOUND; + if (!fIsSet(omode, NC_WRITE)) + pncp->flag |= NC_MODE_RDONLY; + + if (fIsSet(env_mode, NC_MODE_SAFE)) + pncp->flag |= NC_MODE_SAFE; + + if (fIsSet(env_mode, NC_MODE_STRICT_COORD_BOUND)) + pncp->flag |= NC_MODE_STRICT_COORD_BOUND; /* inquire number of dimensions, variables defined and rec dim ID */ err = driver->inq(pncp->ncp, &pncp->ndims, &pncp->nvars, NULL, &pncp->unlimdimid); - if (err != NC_NOERR) goto fn_exit; + if (err != NC_NOERR) { + driver->close(ncp); /* close file and ignore error */ + del_from_PNCList(*ncidp); + if (status == NC_NOERR) status = err; + goto err_out; + } - if (pncp->nvars == 0) return status; /* no variable defined in the file */ + if (pncp->nvars == 0) /* no variable defined in the file */ + goto err_out; - /* make a copy of variable metadata at the dispatcher layer, because - * sanity check is done at the dispatcher layer + /* make a copy of variable metadata at the dispatcher layer, because sanity + * check is done at the dispatcher layer */ /* allocate chunk size for pncp->vars[] */ nalloc = PNETCDF_RNDUP(pncp->nvars, PNC_VARS_CHUNK); pncp->vars = NCI_Malloc(sizeof(PNC_var) * nalloc); if (pncp->vars == NULL) { - DEBUG_ASSIGN_ERROR(err, NC_ENOMEM) - goto fn_exit; + driver->close(ncp); /* close file and ignore error */ + del_from_PNCList(*ncidp); + if (status == NC_NOERR) status = NC_ENOMEM; + goto err_out; } dimids = DIMIDS; @@ -855,6 +1580,7 @@ ncmpi_open(MPI_Comm comm, } if (pncp->vars[i].recdim >= 0) pncp->nrec_vars++; } + if (err != NC_NOERR) { /* error happens in loop i */ assert(i < pncp->nvars); for (j=0; j<=i; j++) { @@ -862,24 +1588,116 @@ ncmpi_open(MPI_Comm comm, NCI_Free(pncp->vars[j].shape); } NCI_Free(pncp->vars); + driver->close(ncp); /* close file and ignore error */ + del_from_PNCList(*ncidp); + if (status == NC_NOERR) status = err; } if (dimids != DIMIDS) NCI_Free(dimids); -fn_exit: - if (err != NC_NOERR) { - driver->close(ncp); /* close file and ignore error */ - if (pncp->comm != MPI_COMM_WORLD && pncp->comm != MPI_COMM_SELF) +err_out: + if (combined_info != MPI_INFO_NULL) + MPI_Info_free(&combined_info); + + if (status != NC_NOERR && status != NC_EMULTIDEFINE_OMODE && + status != NC_ENULLPAD) { + if (pncp->comm != MPI_COMM_NULL) MPI_Comm_free(&pncp->comm); /* a collective call */ - del_from_PNCList(*ncidp); - NCI_Free(pncp->path); + if (pncp->path != NULL) + NCI_Free(pncp->path); NCI_Free(pncp); *ncidp = -1; - if (status == NC_NOERR) status = err; } return status; } +#if PNETCDF_PROFILING == 1 + +int pnc_num_aggrs_per_node; +double pnc_ina_init; +double pnc_ina_flatten; +double pnc_ina_put[NTIMERS]; +double pnc_ina_get[NTIMERS]; +MPI_Count pnc_ina_npairs_put; +MPI_Count pnc_ina_npairs_get; +MPI_Count pnc_ina_mem_put[NTIMERS]; +MPI_Count pnc_ina_mem_get[NTIMERS]; + +static +void print_profiled(MPI_Comm comm) +{ + int i, rank; + double max_t[NTIMERS]; + MPI_Count max_c[NTIMERS]; + + MPI_Comm_rank(comm, &rank); + + /* print intra-node aggregation timing breakdown */ + if (pnc_num_aggrs_per_node > 0) { + double timing, max_MiB[NTIMERS], wr_total, rd_total; + MPI_Count count; + + wr_total = pnc_ina_init + pnc_ina_flatten; + for (i=0; i 0) { + printf("PNC INA put npairs=%lld mem=%.1f %.1f %.1f %.1f %.1f %.1f (MiB)\n", + pnc_ina_npairs_put, + max_MiB[0],max_MiB[1],max_MiB[2],max_MiB[3],max_MiB[4],max_MiB[5]); + printf("PNC INA put time: init %.2f flat %.2f MD %.2f sort %.2f post %.2f wait %.2f setview %.2f total %.2f (write %.2f)\n", + pnc_ina_init,pnc_ina_flatten, + pnc_ina_put[0],pnc_ina_put[1],pnc_ina_put[2],pnc_ina_put[3],pnc_ina_put[4], + wr_total,pnc_ina_put[5]); + } + + MPI_Reduce(&pnc_ina_npairs_get, &count, 1, MPI_COUNT, MPI_MAX, 0, comm); + pnc_ina_npairs_get = count; + + MPI_Reduce(pnc_ina_get, max_t, NTIMERS, MPI_DOUBLE, MPI_MAX, 0, comm); + for (i=0; i 0) { + printf("PNC INA get npairs=%lld mem=%.1f %.1f %.1f %.1f %.1f %.1f (MiB)\n", + pnc_ina_npairs_get, + max_MiB[0],max_MiB[1],max_MiB[2],max_MiB[3],max_MiB[4],max_MiB[5]); + printf("PNC INA get time: init %.2f flat %.2f MD %.2f sort %.2f post %.2f wait %.2f total %.2f (read %.2f)\n", + pnc_ina_init,pnc_ina_flatten, + pnc_ina_get[0],pnc_ina_get[1],pnc_ina_get[3],pnc_ina_get[4], + rd_total,pnc_ina_get[2]); + } + } +} +#endif + /*----< ncmpi_close() >------------------------------------------------------*/ /* This is a collective subroutine. */ int @@ -898,9 +1716,12 @@ ncmpi_close(int ncid) /* Remove from the PNCList, even if err != NC_NOERR */ del_from_PNCList(ncid); +#if PNETCDF_PROFILING == 1 + print_profiled(pncp->comm); +#endif + /* free the PNC object */ - if (pncp->comm != MPI_COMM_WORLD && pncp->comm != MPI_COMM_SELF) - MPI_Comm_free(&pncp->comm); /* a collective call */ + MPI_Comm_free(&pncp->comm); /* a collective call */ NCI_Free(pncp->path); for (i=0; invars; i++) @@ -1099,8 +1920,7 @@ ncmpi_abort(int ncid) del_from_PNCList(ncid); /* free the PNC object */ - if (pncp->comm != MPI_COMM_WORLD && pncp->comm != MPI_COMM_SELF) - MPI_Comm_free(&pncp->comm); /* a collective call */ + MPI_Comm_free(&pncp->comm); /* a collective call */ NCI_Free(pncp->path); for (i=0; invars; i++) @@ -1166,7 +1986,7 @@ ncmpi_inq_format(int ncid, return NC_NOERR; } -#ifdef ENABLE_ADIOS +#if PNETCDF_DRIVER_ADIOS == 1 static void swap_64(void *data) { uint64_t *dest = (uint64_t*) data; @@ -1242,18 +2062,19 @@ ncmpi_inq_file_format(const char *filename, errno = 0; rlen = read(fd, signature, 8); if (rlen != 8) { - close(fd); /* ignore error */ +#if PNETCDF_DEBUG_MODE == 1 if (rlen == 0 && errno == 0) fprintf(stderr, "Error in %s at %d: empty file %s\n", __func__,__LINE__,filename); else fprintf(stderr, "Error in %s at %d: fail to read signature of file %s\n", __func__,__LINE__,filename); +#endif + close(fd); /* ignore error */ DEBUG_RETURN_ERROR(NC_EFILE) } - if (close(fd) == -1) { + if (close(fd) == -1) DEBUG_RETURN_ERROR(NC_EFILE) - } if (memcmp(signature, cdf_signature, 3) == 0) { if (signature[3] == 5) *formatp = NC_FORMAT_CDF5; @@ -1289,7 +2110,7 @@ ncmpi_inq_file_format(const char *filename, * to H5Aget_name(). For now, we do not distinguish * NC_CLASSIC_MODEL, but simply return NETCDF4 format. */ -#ifdef ENABLE_NETCDF4 +#if PNETCDF_DRIVER_NETCDF4 == 1 int err, ncid; err = nc_open(path, NC_NOWRITE, &ncid); if (err != NC_NOERR) DEBUG_RETURN_ERROR(err) @@ -1303,7 +2124,7 @@ ncmpi_inq_file_format(const char *filename, } } -#ifdef ENABLE_ADIOS +#if PNETCDF_DRIVER_ADIOS == 1 /* check if the file is a BP. */ if (*formatp == NC_FORMAT_UNKNOWN) { off_t fsize; @@ -1352,7 +2173,7 @@ ncmpi_inq_file_format(const char *filename, 0 < h3 && h3 < fsize && h1 < h2 && h2 < h3){ /* basic footer check is passed, now we try to open the file with - * ADIOS library to make sure it is indeed a BP formated file + * ADIOS library to make sure it is indeed a BP file */ ADIOS_FILE *fp; fp = adios_read_open_file(path, ADIOS_READ_METHOD_BP, @@ -1388,14 +2209,14 @@ ncmpi_inq_version(int ncid, int *nc_mode) else if (pncp->format == NC_FORMAT_CLASSIC) *nc_mode = NC_CLASSIC_MODEL; -#ifdef ENABLE_NETCDF4 +#if PNETCDF_DRIVER_NETCDF4 == 1 else if (pncp->format == NC_FORMAT_NETCDF4) *nc_mode = NC_NETCDF4; else if (pncp->format == NC_FORMAT_NETCDF4_CLASSIC) *nc_mode = NC_NETCDF4 | NC_CLASSIC_MODEL; #endif -#ifdef ENABLE_ADIOS +#if PNETCDF_DRIVER_ADIOS == 1 else if (pncp->format == NC_FORMAT_BP) *nc_mode = NC_BP; #endif @@ -1471,12 +2292,6 @@ ncmpi_inq_path(int ncid, err = PNC_check_id(ncid, &pncp); if (err != NC_NOERR) return err; -#if 0 - /* calling the subroutine that implements ncmpi_inq_path() */ - return pncp->driver->inq_misc(pncp->ncp, pathlen, path, NULL, NULL, - NULL, NULL, NULL, NULL, NULL, NULL, - NULL, NULL, NULL, NULL, NULL); -#endif if (pathlen != NULL) { if (pncp->path == NULL) *pathlen = 0; else *pathlen = (int)strlen(pncp->path); @@ -1502,7 +2317,7 @@ ncmpi_inq_num_fix_vars(int ncid, int *num_fix_varsp) if (num_fix_varsp == NULL) return NC_NOERR; -#ifdef ENABLE_NETCDF4 +#if PNETCDF_DRIVER_NETCDF4 == 1 if (pncp->format == NC_FORMAT_NETCDF4 || pncp->format == NC_FORMAT_NETCDF4_CLASSIC) { /* calling the subroutine that implements ncmpi_inq_num_fix_vars() */ @@ -1540,7 +2355,7 @@ ncmpi_inq_num_rec_vars(int ncid, int *num_rec_varsp) if (num_rec_varsp == NULL) return NC_NOERR; -#ifdef ENABLE_NETCDF4 +#if PNETCDF_DRIVER_NETCDF4 == 1 if (pncp->format == NC_FORMAT_NETCDF4 || pncp->format == NC_FORMAT_NETCDF4_CLASSIC) { /* calling the subroutine that implements ncmpi_inq_num_rec_vars() */ @@ -1780,7 +2595,7 @@ ncmpi_set_default_format(int format, int *old_formatp) { int err=NC_NOERR, perr=0; -#ifdef ENABLE_THREAD_SAFE +#if PNETCDF_THREAD_SAFE == 1 perr = pthread_mutex_lock(&lock); CHECK_ERRNO(perr, "pthread_mutex_lock") #endif @@ -1802,7 +2617,7 @@ ncmpi_set_default_format(int format, int *old_formatp) err = NC_NOERR; } -#ifdef ENABLE_THREAD_SAFE +#if PNETCDF_THREAD_SAFE == 1 perr = pthread_mutex_unlock(&lock); CHECK_ERRNO(perr, "pthread_mutex_unlock") @@ -1821,14 +2636,14 @@ ncmpi_inq_default_format(int *formatp) if (formatp == NULL) DEBUG_RETURN_ERROR(NC_EINVAL) -#ifdef ENABLE_THREAD_SAFE +#if PNETCDF_THREAD_SAFE == 1 perr = pthread_mutex_lock(&lock); CHECK_ERRNO(perr, "pthread_mutex_lock") #endif *formatp = ncmpi_default_create_format; -#ifdef ENABLE_THREAD_SAFE +#if PNETCDF_THREAD_SAFE == 1 perr = pthread_mutex_unlock(&lock); CHECK_ERRNO(perr, "pthread_mutex_unlock") @@ -1847,7 +2662,7 @@ ncmpi_inq_files_opened(int *num, /* cannot be NULL */ if (num == NULL) DEBUG_RETURN_ERROR(NC_EINVAL) -#ifdef ENABLE_THREAD_SAFE +#if PNETCDF_THREAD_SAFE == 1 perr = pthread_mutex_lock(&lock); CHECK_ERRNO(perr, "pthread_mutex_lock") #endif @@ -1863,7 +2678,7 @@ ncmpi_inq_files_opened(int *num, /* cannot be NULL */ } } } -#ifdef ENABLE_THREAD_SAFE +#if PNETCDF_THREAD_SAFE == 1 perr = pthread_mutex_unlock(&lock); CHECK_ERRNO(perr, "pthread_mutex_unlock") diff --git a/src/dispatchers/var_getput.m4 b/src/dispatchers/var_getput.m4 index f2757a9fed..d7756b0c7b 100644 --- a/src/dispatchers/var_getput.m4 +++ b/src/dispatchers/var_getput.m4 @@ -56,23 +56,24 @@ define(`GOTO_CHECK',`{ DEBUG_ASSIGN_ERROR(err, $1) goto err_check; }')dnl } +#include /*----< check_EINVALCOORDS() >-----------------------------------------------*/ static -int check_EINVALCOORDS(int strict_coord_bound, +int check_EINVALCOORDS(int relax_coord_bound, MPI_Offset start, MPI_Offset count, MPI_Offset shape) { - if (strict_coord_bound) { - if (start < 0 || start >= shape) - DEBUG_RETURN_ERROR(NC_EINVALCOORDS) - } - else { + if (relax_coord_bound) { if (start < 0 || start > shape) DEBUG_RETURN_ERROR(NC_EINVALCOORDS) if (start == shape && count > 0) DEBUG_RETURN_ERROR(NC_EINVALCOORDS) } + else { + if (start < 0 || start >= shape) + DEBUG_RETURN_ERROR(NC_EINVALCOORDS) + } return NC_NOERR; } @@ -105,7 +106,7 @@ int check_EEDGE(const MPI_Offset *start, return NC_NOERR; } -#ifdef PNETCDF_DEBUG +#if PNETCDF_DEBUG_MODE == 1 #define DEBUG_PRINT_EEDGE(dim) { \ if (err != NC_NOERR) { \ char *_env_str = getenv("PNETCDF_VERBOSE_DEBUG_MODE"); \ @@ -116,11 +117,11 @@ int check_EEDGE(const MPI_Offset *start, pncp->driver->inq_var(pncp->ncp, varid, name, NULL, NULL, \ NULL, NULL, NULL, NULL, NULL); \ if (stride != NULL) \ - fprintf(stderr, "Rank %d: NC_EEDGE variable %s: shape[%d]="OFFFMT" but start[%d]="OFFFMT" count[%d]="OFFFMT" stride[%d]="OFFFMT"\n", \ - _rank, name, dim, shape[dim], dim, start[dim], dim, count[dim], dim, stride[dim]); \ + fprintf(stderr, "Rank %d in %s (%d): NC_EEDGE variable %s: shape[%d]="OFFFMT" but start[%d]="OFFFMT" count[%d]="OFFFMT" stride[%d]="OFFFMT"\n", \ + _rank, __func__, __LINE__, name, dim, shape[dim], dim, start[dim], dim, count[dim], dim, stride[dim]); \ else \ - fprintf(stderr, "Rank %d: NC_EEDGE variable %s: shape[%d]="OFFFMT" but start[%d]="OFFFMT" count[%d]="OFFFMT"\n", \ - _rank, name, dim, shape[dim], dim, start[dim], dim, count[dim]); \ + fprintf(stderr, "Rank %d in %s (%d): NC_EEDGE variable %s: shape[%d]="OFFFMT" but start[%d]="OFFFMT" count[%d]="OFFFMT"\n", \ + _rank, __func__, __LINE__, name, dim, shape[dim], dim, start[dim], dim, count[dim]); \ } \ } \ } @@ -170,7 +171,7 @@ int check_start_count_stride(PNC *pncp, MPI_Offset len = (count == NULL) ? 1 : count[0]; if (shape[0] == 0 && len > 0) /* no record yet */ DEBUG_RETURN_ERROR(NC_EINVALCOORDS) - err = check_EINVALCOORDS(pncp->flag & NC_MODE_STRICT_COORD_BOUND, + err = check_EINVALCOORDS(fIsSet(pncp->flag, NC_MODE_STRICT_COORD_BOUND), start[0], len, shape[0]); if (err != NC_NOERR) return err; } @@ -181,7 +182,7 @@ int check_start_count_stride(PNC *pncp, ndims = pncp->vars[varid].ndims; for (i=firstDim; iflag & NC_MODE_STRICT_COORD_BOUND, + err = check_EINVALCOORDS(fIsSet(pncp->flag, NC_MODE_STRICT_COORD_BOUND), start[i], len, shape[i]); if (err != NC_NOERR) return err; } @@ -241,7 +242,7 @@ int sanity_check(PNC *pncp, int varid, IO_type io, /* get/put/iget/iput/bput */ MPI_Datatype itype, /* internal data type */ - int isColl) /* collective or indepdnent API */ + int isColl) /* collective or independent API */ { /* for put APIs */ if (io == API_PUT || io == API_IPUT || io == API_BPUT) @@ -732,7 +733,7 @@ IAPINAME($1,$2,$3)(int ncid, if (err != NC_NOERR) return err; ifelse(`$1',`bput',` -#ifdef ENABLE_BURST_BUFFER +#if PNETCDF_BURST_BUFFERING == 1 if (pncp->driver == ncbbio_inq_driver()) reqMode = NC_REQ_NBI; else @@ -852,7 +853,7 @@ INAPINAME($1,$2)(int ncid, if (num == 0) return NC_NOERR; ifelse(`$1',`bput',` -#ifdef ENABLE_BURST_BUFFER +#if PNETCDF_BURST_BUFFERING == 1 if (pncp->driver == ncbbio_inq_driver()) reqMode = NC_REQ_NBI; else @@ -939,57 +940,9 @@ ncmpi_$1_vard$2(int ncid, MPI_Offset bufcount, MPI_Datatype buftype) /* data type of the buffer */ { - int err, status, reqMode=0; - PNC *pncp; - - /* check if ncid is valid. - * For invalid ncid, we must return error now, as there is no way to - * continue with invalid ncp. However, collective APIs might hang if this - * error occurs only on a subset of processes - */ - err = PNC_check_id(ncid, &pncp); - if (err != NC_NOERR) return err; - - err = sanity_check(pncp, varid, IO_TYPE($1), MPI_DATATYPE_NULL, IS_COLL($2)); - - /* when bufcount == NC_COUNT_IGNORE, buftype must be an MPI predefined datatype */ - if (err == NC_NOERR && - buftype != MPI_DATATYPE_NULL && bufcount == NC_COUNT_IGNORE && - buftype != MPI_CHAR && - buftype != MPI_SIGNED_CHAR && buftype != MPI_UNSIGNED_CHAR && - buftype != MPI_SHORT && buftype != MPI_UNSIGNED_SHORT && - buftype != MPI_INT && buftype != MPI_UNSIGNED && - buftype != MPI_FLOAT && buftype != MPI_DOUBLE && - buftype != MPI_LONG_LONG_INT && buftype != MPI_UNSIGNED_LONG_LONG && - buftype != MPI_LONG) - DEBUG_ASSIGN_ERROR(err, NC_EINVAL) - - ifelse(`$2',`', - `/* for independent API, return now if error encountered or zero request */ - if (err != NC_NOERR) return err; - if (buftype != MPI_DATATYPE_NULL && bufcount == 0) return NC_NOERR;', - `/* In safe mode, check errors across all processes */ - if (pncp->flag & NC_MODE_SAFE) { - err = allreduce_error(pncp, err); - if (err != NC_NOERR) return err; - } - else if (err == NC_EPERM || err == NC_EINDEFINE || err == NC_EINDEP || - err == NC_ENOTINDEP) /* cannot continue if fatal errors */ - return err; - else if (err != NC_NOERR) { /* other errors, participate collective call */ - int nprocs; - MPI_Comm_size(pncp->comm, &nprocs); - if (nprocs == 1) return err; - reqMode |= NC_REQ_ZERO; - }') - - reqMode |= IO_MODE($1) | NC_REQ_BLK | NC_REQ_FLEX | COLL_MODE($2); - - /* calling the subroutine that implements ncmpi_$1_vard$2() */ - status = pncp->driver->$1_vard(pncp->ncp, varid, filetype, buf, - bufcount, buftype, reqMode); - - return ifelse(`$2',`',`status;',`(err != NC_NOERR) ? err : status; /* first error encountered */') + fprintf(stderr, + "PnetCDF vard APIs have been deprecated since 1.15.0 release.\n"); + return NC_ENOTSUPPORT; } ') dnl diff --git a/src/dispatchers/variable.c b/src/dispatchers/variable.c index ba8c92e16e..be6fc83d70 100644 --- a/src/dispatchers/variable.c +++ b/src/dispatchers/variable.c @@ -391,11 +391,11 @@ ncmpi_inq_vartype(int ncid, /* IN: file ID */ *xtypep = pncp->vars[varid].xtype; return NC_NOERR; -#if 0 - /* calling the subroutine that implements ncmpi_inq_vartype() */ - return pncp->driver->inq_var(pncp->ncp, varid, NULL, xtypep, NULL, - NULL, NULL, NULL, NULL, NULL); -#endif + /* This can also be achieved by calling the subroutine that implements + * ncmpi_inq_vartype( + return pncp->driver->inq_var(pncp->ncp, varid, NULL, xtypep, NULL, + NULL, NULL, NULL, NULL, NULL); + */ } /*----< ncmpi_inq_varndims() >-----------------------------------------------*/ @@ -423,11 +423,11 @@ ncmpi_inq_varndims(int ncid, /* IN: file ID */ *ndimsp = pncp->vars[varid].ndims; return NC_NOERR; -#if 0 - /* calling the subroutine that implements ncmpi_inq_varndims() */ - return pncp->driver->inq_var(pncp->ncp, varid, NULL, NULL, ndimsp, - NULL, NULL, NULL, NULL, NULL); -#endif + /* This can also be achieved by calling the subroutine that implements + * ncmpi_inq_varndims() + return pncp->driver->inq_var(pncp->ncp, varid, NULL, NULL, ndimsp, + NULL, NULL, NULL, NULL, NULL); + */ } /*----< ncmpi_inq_vardimid() >-----------------------------------------------*/ @@ -590,6 +590,8 @@ ncmpi_fill_var_rec(int ncid, return ncmpii_error_mpi2nc(mpireturn, "MPI_Allreduce"); if (minE != NC_NOERR) return minE; } + else if (err != NC_NOERR) + return err; /* calling the subroutine that implements ncmpi_fill_var_rec() */ return pncp->driver->fill_var_rec(pncp->ncp, varid, recno); diff --git a/src/drivers/common/Makefile.am b/src/drivers/common/Makefile.am index bcab9f3882..44e137fc4d 100644 --- a/src/drivers/common/Makefile.am +++ b/src/drivers/common/Makefile.am @@ -13,8 +13,8 @@ AM_CPPFLAGS += -I$(top_srcdir)/src/drivers/include AM_CPPFLAGS += -I$(top_builddir)/src/drivers/include AM_CPPFLAGS += @ADIOS_INC@ -if PNETCDF_DEBUG - AM_CPPFLAGS += -DPNETCDF_DEBUG +if ENABLE_GIO + AM_CPPFLAGS += -I${top_builddir}/gio/src endif noinst_LTLIBRARIES = libcommon.la @@ -34,15 +34,19 @@ C_SRCS = utf8proc.c \ mem_alloc.c \ dtype_decode.c \ create_imaptype.c \ - error_mpi2nc.c \ check_name.c \ pack_unpack.c \ utils.c \ error_posix2nc.c \ + error_mpi2nc.c \ hash_map.c +if ENABLE_GIO + C_SRCS += error_gio2nc.c +endif + if ENABLE_ADIOS - C_SRCS += error_adios2nc.c + C_SRCS += error_adios2nc.c endif libcommon_la_SOURCES = $(C_SRCS) $(H_SRCS) diff --git a/src/drivers/common/convert_swap.m4 b/src/drivers/common/convert_swap.m4 index 80c3f5d2a4..6e6294cee0 100644 --- a/src/drivers/common/convert_swap.m4 +++ b/src/drivers/common/convert_swap.m4 @@ -361,9 +361,9 @@ ncmpii_put_cast_swap(int format, /* NC_FORMAT_CDF2/NC_FORMAT_CDF5 */ /* allocate xbuf if necessary */ if (need_cast -#ifndef ENABLE_IN_PLACE_SWAP +#if PNETCDF_BYTE_SWAP_IN_PLACE != 1 || (need_swap && !isNewBuf -#ifndef DISABLE_IN_PLACE_SWAP +#if PNETCDF_BYTE_SWAP_IN_PLACE != 0 && nbytes <= NC_BYTE_SWAP_BUFFER_SIZE #endif ) diff --git a/src/drivers/common/dtype_decode.c b/src/drivers/common/dtype_decode.c index 3c4143e361..e2396fb643 100644 --- a/src/drivers/common/dtype_decode.c +++ b/src/drivers/common/dtype_decode.c @@ -74,8 +74,11 @@ dtype_filter(MPI_Datatype type) #if (SIZEOF_LONG == 4) MPI_Datatype uint_4byte; #endif + #if (SIZEOF_INT == 4) +#if (defined(ENABLE_FORTRAN) && (ENABLE_FORTRAN == 1)) || (SIZEOF_LONG == 4) MPI_Datatype int_4byte = MPI_INT; +#endif #if (SIZEOF_LONG == 4) uint_4byte = MPI_UNSIGNED; #endif @@ -97,12 +100,14 @@ dtype_filter(MPI_Datatype type) if (type == MPI_INTEGER4) /* a Fortran datatype */ return int_4byte; #endif + #if (SIZEOF_LONG == 4) if (type == MPI_LONG) return int_4byte; if (type == MPI_UNSIGNED_LONG) return uint_4byte; #endif + #if (SIZEOF_INT == 4) if (type == MPI_INT) return MPI_INT; @@ -299,7 +304,7 @@ int ncmpii_dtype_decode(MPI_Datatype dtype, MPI_Datatype ptype, *array_of_dtypes=NULL; MPI_Aint *array_of_adds=NULL; #ifdef HAVE_MPI_LARGE_COUNT - MPI_Count num_ints, num_adds, num_larges, num_dtypes, *array_of_larges; + MPI_Count num_ints, num_adds, num_larges, num_dtypes, *array_of_larges=NULL; int *distribs, *dargs, *psizes; MPI_Count *gzises; #else diff --git a/src/drivers/common/error_adios2nc.c b/src/drivers/common/error_adios2nc.c index f01b782d97..8edbd2a71b 100644 --- a/src/drivers/common/error_adios2nc.c +++ b/src/drivers/common/error_adios2nc.c @@ -57,13 +57,13 @@ int ncmpii_error_adios2nc(int adios_err, char *err_msg) /* extra error mes errstr = adios_errmsg(); if (err_msg == NULL) err_msg = ""; -#ifdef PNETCDF_DEBUG +#if PNETCDF_DEBUG_MODE == 1 /* report the world rank */ int rank; MPI_Comm_rank(MPI_COMM_WORLD, &rank); - printf("rank %d: IO error (%s) : %s\n", rank, err_msg, errstr); + fprintf(stderr,"rank %d: IO error (%s) : %s\n", rank, err_msg, errstr); #else - printf("IO error (%s) : %s\n", err_msg, errstr); + fprintf(stderr,"IO error (%s) : %s\n", err_msg, errstr); #endif return NC_EFILE; /* other unknown file I/O error */ diff --git a/src/drivers/common/error_gio2nc.c b/src/drivers/common/error_gio2nc.c new file mode 100644 index 0000000000..a7e9ce6864 --- /dev/null +++ b/src/drivers/common/error_gio2nc.c @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2026, Northwestern University and Argonne National Laboratory + * See COPYRIGHT notice in top-level directory. + */ + +#ifdef HAVE_CONFIG_H +# include +#endif + +#include +#ifdef HAVE_STDLIB_H +#include +#endif + +#include + +#include +#include + +/*----< ncmpii_error_gio2nc() -----------------------------------------------*/ +/* translate GIO error codes to PnetCDF/netCDF error codes */ +int ncmpii_error_gio2nc(int gio_err, /* returned value from GIO call */ + const char *err_msg) /* extra error message */ +{ + const char *dump_str = (err_msg == NULL) ? "" : err_msg; + + switch (gio_err) { + case GIO_NOERR: return NC_NOERR; + case GIO_EEXIST: return NC_EEXIST; + case GIO_EINVAL: return NC_EINVAL; + case GIO_EPERM: return NC_EPERM; + case GIO_ENOMEM: return NC_ENOMEM; + case GIO_EACCESS: return NC_EACCESS; + case GIO_ENEGATIVECNT: return NC_ENEGATIVECNT; + case GIO_EFILE: return NC_EFILE; + case GIO_ENOENT: return NC_ENOENT; + case GIO_EBAD_FILE: return NC_EBAD_FILE; + case GIO_ENO_SPACE: return NC_ENO_SPACE; + case GIO_EQUOTA: return NC_EQUOTA; + case GIO_EFSTYPE: return NC_EFSTYPE; + case GIO_EMULTIDEFINE_OMODE: return NC_EMULTIDEFINE_OMODE; + case GIO_EMULTIDEFINE_FNC_ARGS: return NC_EMULTIDEFINE_FNC_ARGS; + case GIO_EMULTIDEFINE_HINTS: return NC_EMULTIDEFINE_HINTS; + case GIO_EFILEVIEW: return NC_EFILEVIEW; + } + +#if PNETCDF_DEBUG_MODE == 1 + /* report the world rank */ + int rank; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + fprintf(stderr,"rank %d: GIO error (%s) : %s\n", rank, dump_str, GIO_strerror(gio_err)); +#else + fprintf(stderr,"GIO error (%s) : %s\n", dump_str, GIO_strerror(gio_err)); +#endif + + return NC_EFILE; /* other unknown file I/O error */ +} + + diff --git a/src/drivers/common/error_mpi2nc.c b/src/drivers/common/error_mpi2nc.c index 2d4f3981c2..79de19e9b1 100644 --- a/src/drivers/common/error_mpi2nc.c +++ b/src/drivers/common/error_mpi2nc.c @@ -66,13 +66,13 @@ int ncmpii_error_mpi2nc(int mpi_errorcode, /* returned value from MPI call */ */ MPI_Error_string(mpi_errorcode, errorString, &errorStringLen); -#ifdef PNETCDF_DEBUG +#if PNETCDF_DEBUG_MODE == 1 /* report the world rank */ int rank; MPI_Comm_rank(MPI_COMM_WORLD, &rank); - printf("rank %d: MPI error (%s) : %s\n", rank, dump_str, errorString); + fprintf(stderr,"rank %d: MPI error (%s) : %s\n", rank, dump_str, errorString); #else - printf("MPI error (%s) : %s\n", dump_str, errorString); + fprintf(stderr,"MPI error (%s) : %s\n", dump_str, errorString); #endif return NC_EFILE; /* other unknown file I/O error */ diff --git a/src/drivers/common/error_posix2nc.c b/src/drivers/common/error_posix2nc.c index 740bbe3d4a..57dc27bf21 100644 --- a/src/drivers/common/error_posix2nc.c +++ b/src/drivers/common/error_posix2nc.c @@ -41,13 +41,13 @@ int ncmpii_error_posix2nc(char *err_msg) /* extra error message */ /* other errors that currently have no corresponding PnetCDF error codes */ if (err_msg == NULL) err_msg = ""; -#ifdef PNETCDF_DEBUG +#if PNETCDF_DEBUG_MODE == 1 /* report the world rank */ int rank; MPI_Comm_rank(MPI_COMM_WORLD, &rank); - printf("rank %d: IO error (%s) : %s\n", rank, err_msg, errorString); + fprintf(stderr,"rank %d: IO error (%s) : %s\n", rank, err_msg, errorString); #else - printf("IO error (%s) : %s\n", err_msg, errorString); + fprintf(stderr,"IO error (%s) : %s\n", err_msg, errorString); #endif return NC_EFILE; /* other unknown file I/O error */ diff --git a/src/drivers/common/mem_alloc.c b/src/drivers/common/mem_alloc.c index 279dd44b4e..71b962c92e 100644 --- a/src/drivers/common/mem_alloc.c +++ b/src/drivers/common/mem_alloc.c @@ -9,13 +9,15 @@ NCI_Malloc(size) NCI_Calloc(nelems, esize) NCI_Realloc(ptr, size) + NCI_Strdup(ptr) NCI_Free(ptr) In macro.h, they are macro-replaced to - NCI_Malloc_fn(size, __LINE__, __FILE__) and - NCI_Calloc_fn(nelems, esize, __LINE__, __FILE__) and + NCI_Malloc_fn(size, __LINE__, __func__, __FILE__) + NCI_Calloc_fn(nelems, esize, __LINE__, __func__, __FILE__) NCI_Realloc_fn(ptr, size, __LINE__, __func__, __FILE__) - NCI_Free_fn(ptr,__LINE__,__FILE__). + NCI_Strdup_fn(ptr, __LINE__, __func__, __FILE__) + NCI_Free_fn(ptr, __LINE__, __func__, __FILE__). */ #ifdef HAVE_CONFIG_H @@ -52,7 +54,7 @@ static void *ncmpii_mem_root; static size_t ncmpii_mem_alloc; static size_t ncmpii_max_mem_alloc; -#ifdef ENABLE_THREAD_SAFE +#if PNETCDF_THREAD_SAFE == 1 /* updating the binary tree used in tfind()/tsearch()/tdelete() is not * thread-safe, protect these subroutines with a mutex */ #include @@ -105,7 +107,7 @@ void ncmpii_add_mem_entry(void *buf, const char *func, const char *filename) { -#ifdef ENABLE_THREAD_SAFE +#if PNETCDF_THREAD_SAFE == 1 pthread_mutex_lock(&lock); #endif @@ -132,7 +134,7 @@ void ncmpii_add_mem_entry(void *buf, ncmpii_mem_alloc += size; ncmpii_max_mem_alloc = MAX(ncmpii_max_mem_alloc, ncmpii_mem_alloc); } -#ifdef ENABLE_THREAD_SAFE +#if PNETCDF_THREAD_SAFE == 1 pthread_mutex_unlock(&lock); #endif } @@ -144,7 +146,7 @@ int ncmpii_del_mem_entry(void *buf) { int err=0; -#ifdef ENABLE_THREAD_SAFE +#if PNETCDF_THREAD_SAFE == 1 pthread_mutex_lock(&lock); #endif /* use C tsearch utility */ @@ -179,7 +181,7 @@ int ncmpii_del_mem_entry(void *buf) fprintf(stderr, "Error at line %d file %s: ncmpii_mem_root is NULL\n", __LINE__,__FILE__); fn_exit: -#ifdef ENABLE_THREAD_SAFE +#if PNETCDF_THREAD_SAFE == 1 pthread_mutex_unlock(&lock); #endif return err; @@ -388,11 +390,11 @@ void NCI_Free_fn(void *ptr, int ncmpi_inq_malloc_size(MPI_Offset *size) { #ifdef PNC_MALLOC_TRACE -#ifdef ENABLE_THREAD_SAFE +#if PNETCDF_THREAD_SAFE == 1 pthread_mutex_lock(&lock); #endif *size = (MPI_Offset)ncmpii_mem_alloc; -#ifdef ENABLE_THREAD_SAFE +#if PNETCDF_THREAD_SAFE == 1 pthread_mutex_unlock(&lock); #endif return NC_NOERR; @@ -407,11 +409,11 @@ int ncmpi_inq_malloc_size(MPI_Offset *size) int ncmpi_inq_malloc_max_size(MPI_Offset *size) { #ifdef PNC_MALLOC_TRACE -#ifdef ENABLE_THREAD_SAFE +#if PNETCDF_THREAD_SAFE == 1 pthread_mutex_lock(&lock); #endif *size = (MPI_Offset)ncmpii_max_mem_alloc; -#ifdef ENABLE_THREAD_SAFE +#if PNETCDF_THREAD_SAFE == 1 pthread_mutex_unlock(&lock); #endif return NC_NOERR; @@ -426,13 +428,13 @@ int ncmpi_inq_malloc_max_size(MPI_Offset *size) int ncmpi_inq_malloc_list(void) { #ifdef PNC_MALLOC_TRACE -#ifdef ENABLE_THREAD_SAFE +#if PNETCDF_THREAD_SAFE == 1 pthread_mutex_lock(&lock); #endif /* check if malloc tree is empty */ if (ncmpii_mem_root != NULL) twalk(ncmpii_mem_root, walker); -#ifdef ENABLE_THREAD_SAFE +#if PNETCDF_THREAD_SAFE == 1 pthread_mutex_unlock(&lock); #endif return NC_NOERR; diff --git a/src/drivers/common/utils.c b/src/drivers/common/utils.c index e60b6e30af..6a5ea83453 100644 --- a/src/drivers/common/utils.c +++ b/src/drivers/common/utils.c @@ -61,7 +61,7 @@ ncmpii_xlen_nc_type(nc_type xtype, int *size) } } -/* File system types recognized by ROMIO in MPICH 4.0.0 */ +/* File system types recognized by ROMIO in MPICH 4.0.0, and by PnetCDF */ static const char* fstypes[] = {"ufs", "nfs", "xfs", "pvfs2", "gpfs", "panfs", "lustre", "daos", "testfs", "ime", "quobyte", NULL}; /* Return a pointer to filename by removing the file system type prefix name if @@ -91,3 +91,90 @@ char* ncmpii_remove_file_system_type_prefix(const char *filename) return ret_filename; } +/*----< ncmpii_construct_node_list() >---------------------------------------*/ +/* This subroutine is a collective call. It finds the affinity of MPI processes + * to their shared-memory compute nodes (NUMA) and returns the followings: + * num_NUMAs: Number of NUMA nodes + * numa_ids[nprocs]: node IDs of each rank, must be freed by the caller. + * hwcomm: sub-communicator split based on NUMA hardware. It is the caller's + * responsibility to free it. + */ +int +ncmpii_construct_node_list(MPI_Comm comm, + int *num_NUMAs, /* OUT: */ + int **numa_ids, /* OUT: [nprocs] */ + MPI_Comm *hwcomm) /* OUT: NUMA communicator */ +{ + char *err_msg="No error"; + int i, err, rank, nprocs, numa_id, *ids; + + MPI_Comm_size(comm, &nprocs); + MPI_Comm_rank(comm, &rank); + + *num_NUMAs = 0; + *numa_ids = NULL; + +#if 1 + /* split comm based on NUMA nodes (processes sharing memory) */ + err = MPI_Comm_split_type(comm, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL, + hwcomm); +#else + /* Below code fragment is from MPI standard 4.0's example 7.3: + * Splitting MPI_COMM_WORLD into NUMANode subcommunicators. + */ + MPI_Info_set(info, "mpi_hw_resource_type" , "NUMANode"); + err = MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_HW_GUIDED, + rank, info, &hwcomm); + + /* Below code fragment is from MPI standard 5.0's example 7.3: + * Splitting MPI_COMM_WORLD into NUMANode subcommunicators. + */ + MPI_Info_set(info, "mpi_hw_resource_type" , "hwloc://NUMANode"); + err = MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_HW_GUIDED, + rank, info, &hwcomm); + + /* Below code fragment is from MPI standard 5.0's example 7.4: + * Splitting MPI_COMM_WORLD into NUMANode subcommunicators. + */ + MPI_Info_set(info, "mpi_hw_resource_type", "hwloc://NUMANode"); + err = MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_RESOURCE_GUIDED, + rank, info, &hwcomm); +#endif + + if (err != MPI_SUCCESS) { + err_msg = "MPI_Comm_split_type()"; + goto err_out; + } + + if (*hwcomm == MPI_COMM_NULL) { + err_msg = "MPI_Comm_split_type() hwcomm NULL"; + goto err_out; + } + + /* Use hwcomm's root's rank as this process's NUMA node ID */ + numa_id = rank; + MPI_Bcast(&numa_id, 1, MPI_INT, 0, *hwcomm); + + /* Gather all NUMA node IDs */ + *numa_ids = (int*) malloc(sizeof(int) * nprocs); + MPI_Allgather(&numa_id, 1, MPI_INT, *numa_ids, 1, MPI_INT, comm); + + /* Count number of unique IDs and reassign NUMA ID */ + ids = (int*) calloc(nprocs, sizeof(int)); + *num_NUMAs = 0; + for (i=0; i int -nc4io_create(MPI_Comm comm, - const char *path, - int cmode, - int ncid, - MPI_Info info, - void **ncpp) /* OUT */ +nc4io_create(MPI_Comm comm, + const char *path, + int cmode, + int ncid, + int env_mode, + MPI_Info info, + PNC_comm_attr node_ids, /* node IDs of all processes */ + void **ncpp) /* OUT */ { char *filename; int err, ncidtmp; @@ -84,12 +86,7 @@ nc4io_create(MPI_Comm comm, nc4p = (NC_nc4*) NCI_Malloc(sizeof(NC_nc4)); if (nc4p == NULL) DEBUG_RETURN_ERROR(NC_ENOMEM) - nc4p->path = (char*) NCI_Malloc(strlen(filename)+1); - if (nc4p->path == NULL) { - NCI_Free(nc4p); - DEBUG_RETURN_ERROR(NC_ENOMEM) - } - strcpy(nc4p->path, filename); + nc4p->path = path; /* reuse path duplicated in dispatch layer */ nc4p->mode = cmode | NC_WRITE; nc4p->flag = NC_MODE_DEF; nc4p->ncid = ncid; @@ -108,12 +105,14 @@ nc4io_create(MPI_Comm comm, } int -nc4io_open(MPI_Comm comm, - const char *path, - int omode, - int ncid, - MPI_Info info, - void **ncpp) +nc4io_open(MPI_Comm comm, + const char *path, + int omode, + int ncid, + int env_mode, + MPI_Info info, + PNC_comm_attr node_ids, /* node IDs of all processes */ + void **ncpp) /* OUT */ { char *filename; int err, ncidtmp; @@ -130,19 +129,14 @@ nc4io_open(MPI_Comm comm, * NC_MPIIO is ignored in 4.6.2 and after. */ omode |= NC_MPIIO; - err = nc_open_par(path, omode, comm, info, &ncidtmp); + err = nc_open_par(filename, omode, comm, info, &ncidtmp); if (err != NC_NOERR) DEBUG_RETURN_ERROR(err); /* Create a NC_nc4 object and save its driver pointer */ nc4p = (NC_nc4*) NCI_Malloc(sizeof(NC_nc4)); if (nc4p == NULL) DEBUG_RETURN_ERROR(NC_ENOMEM) - nc4p->path = (char*) NCI_Malloc(strlen(filename)+1); - if (nc4p->path == NULL) { - NCI_Free(nc4p); - DEBUG_RETURN_ERROR(NC_ENOMEM) - } - strcpy(nc4p->path, filename); + nc4p->path = path; /* reuse path duplicated in dispatch layer */ nc4p->mode = omode; nc4p->flag = 0; nc4p->ncid = ncid; @@ -177,7 +171,6 @@ nc4io_close(void *ncdp) if (nc4p->mpiinfo != MPI_INFO_NULL) MPI_Info_free(&nc4p->mpiinfo); - NCI_Free(nc4p->path); NCI_Free(nc4p); return err; @@ -306,7 +299,6 @@ nc4io_abort(void *ncdp) if (nc4p->mpiinfo != MPI_INFO_NULL) MPI_Info_free(&nc4p->mpiinfo); - NCI_Free(nc4p->path); NCI_Free(nc4p); return err; diff --git a/src/drivers/nc4io/nc4io_var.c b/src/drivers/nc4io/nc4io_var.c index 077235cc6d..a383596567 100644 --- a/src/drivers/nc4io/nc4io_var.c +++ b/src/drivers/nc4io/nc4io_var.c @@ -36,9 +36,6 @@ * ncmpi_iget_varn_() : dispatcher->iget_varn() * ncmpi_iput_varn_() : dispatcher->iput_varn() * ncmpi_bput_varn_() : dispatcher->bput_varn() - * - * ncmpi_get_vard() : dispatcher->get_vard() - * ncmpi_put_vard() : dispatcher->put_vard() */ #ifdef HAVE_CONFIG_H @@ -288,27 +285,3 @@ nc4io_bput_varn(void *ncdp, DEBUG_RETURN_ERROR(NC_ENOTSUPPORT); } -int -nc4io_get_vard(void *ncdp, - int varid, - MPI_Datatype filetype, - void *buf, - MPI_Offset bufcount, - MPI_Datatype buftype, - int reqMode) -{ - DEBUG_RETURN_ERROR(NC_ENOTSUPPORT) -} - -int -nc4io_put_vard(void *ncdp, - int varid, - MPI_Datatype filetype, - const void *buf, - MPI_Offset bufcount, - MPI_Datatype buftype, - int reqMode) -{ - DEBUG_RETURN_ERROR(NC_ENOTSUPPORT) -} - diff --git a/src/drivers/ncadios/Makefile.am b/src/drivers/ncadios/Makefile.am index 7db5483cd5..6c37a860e8 100644 --- a/src/drivers/ncadios/Makefile.am +++ b/src/drivers/ncadios/Makefile.am @@ -16,10 +16,6 @@ AM_CPPFLAGS += -I${srcdir}/adios_headers/core/transforms AM_CPPFLAGS += -I${srcdir}/adios_headers/transforms AM_CPPFLAGS += @ADIOS_INC@ -if PNETCDF_DEBUG - AM_CPPFLAGS += -DPNETCDF_DEBUG -endif - noinst_LTLIBRARIES = libncadios.la M4FLAGS += -I${top_srcdir}/m4 diff --git a/src/drivers/ncadios/ncadios_bp2ncd.c b/src/drivers/ncadios/ncadios_bp2ncd.c index d985ec093f..413e7d7404 100644 --- a/src/drivers/ncadios/ncadios_bp2ncd.c +++ b/src/drivers/ncadios/ncadios_bp2ncd.c @@ -31,7 +31,7 @@ #include "ncadios_driver.h" #include -#define ERR(e){if(e){printf("Error:%d\n",e);return 2;}} +#define ERR(e){if(e){fprintf(stderr,"Error:%d\n",e);return 2;}} #define DIVIDER "\t---------------------------------\n" struct var_dim @@ -525,7 +525,7 @@ int ncadiosi_parse_header_bp2ncd (NC_ad *ncid) pg = pg_root; while (pg) { - int i,j; + int j; int var_dims_count = 0; struct var_dim * var_dims = 0; diff --git a/src/drivers/ncadios/ncadios_driver.c b/src/drivers/ncadios/ncadios_driver.c index dcf0e17178..ee831f7b14 100644 --- a/src/drivers/ncadios/ncadios_driver.c +++ b/src/drivers/ncadios/ncadios_driver.c @@ -55,8 +55,6 @@ static PNC_driver ncadios_driver = { ncadios_put_var, ncadios_get_varn, ncadios_put_varn, - ncadios_get_vard, - ncadios_put_vard, ncadios_iget_var, ncadios_iput_var, ncadios_bput_var, diff --git a/src/drivers/ncadios/ncadios_driver.h b/src/drivers/ncadios/ncadios_driver.h index b8aff5f757..38c69bde72 100644 --- a/src/drivers/ncadios/ncadios_driver.h +++ b/src/drivers/ncadios/ncadios_driver.h @@ -77,7 +77,7 @@ typedef struct NC_ad NC_ad; /* forward reference */ struct NC_ad { int mode; /* file _open/_create mode */ int flag; /* define/data/collective/indep mode */ - char *path; /* path name */ + const char *path; /* path name */ MPI_Comm comm; /* MPI communicator */ ADIOS_FILE *fp; /* ADIOS file pointer */ int *ndims; /* Number of dims in each var */ @@ -91,10 +91,14 @@ struct NC_ad { }; extern int -ncadios_create(MPI_Comm comm, const char *path, int cmode, int ncid, MPI_Info info, void **ncdp); +ncadios_create(MPI_Comm comm, const char *path, int cmode, int ncid, + int env_mode, MPI_Info info, PNC_comm_attr node_ids, + void **ncdp); extern int -ncadios_open(MPI_Comm comm, const char *path, int omode, int ncid, MPI_Info info, void **ncdp); +ncadios_open(MPI_Comm comm, const char *path, int omode, int ncid, + int env_mode, MPI_Info info, PNC_comm_attr node_ids, + void **ncdp); extern int ncadios_close(void *ncdp); diff --git a/src/drivers/ncadios/ncadios_file.c b/src/drivers/ncadios/ncadios_file.c index bede8e461d..c0e596234e 100644 --- a/src/drivers/ncadios/ncadios_file.c +++ b/src/drivers/ncadios/ncadios_file.c @@ -47,24 +47,28 @@ #include int -ncadios_create(MPI_Comm comm, - const char *path, - int cmode, - int ncid, - MPI_Info info, - void **ncpp) /* OUT */ +ncadios_create(MPI_Comm comm, + const char *path, + int cmode, + int ncid, + int env_mode, + MPI_Info info, + PNC_comm_attr node_ids, /* node IDs of all processes */ + void **ncpp) /* OUT */ { /* Read only driver */ DEBUG_RETURN_ERROR(NC_ENOTSUPPORT); } int -ncadios_open(MPI_Comm comm, - const char *path, - int omode, - int ncid, - MPI_Info info, - void **ncpp) +ncadios_open(MPI_Comm comm, + const char *path, + int omode, + int ncid, + int env_mode, + MPI_Info info, + PNC_comm_attr node_ids, /* node IDs of all processes */ + void **ncpp) /* OUT */ { int err, parse_done=0; NC_ad *ncadp; @@ -77,15 +81,10 @@ ncadios_open(MPI_Comm comm, ncadp = (NC_ad*) NCI_Malloc(sizeof(NC_ad)); if (ncadp == NULL) DEBUG_RETURN_ERROR(NC_ENOMEM) - ncadp->path = (char*) NCI_Malloc(strlen(path) + 1); - if (ncadp->path == NULL) { - NCI_Free(ncadp); - DEBUG_RETURN_ERROR(NC_ENOMEM) - } - strcpy(ncadp->path, path); - ncadp->mode = omode; - ncadp->flag = 0; - ncadp->comm = comm; + ncadp->path = path; /* reuse path duplicated in dispatch layer */ + ncadp->mode = omode; + ncadp->flag = 0; + ncadp->comm = comm; ncadp->getsize = 0; MPI_Comm_rank(ncadp->comm, &(ncadp->rank)); @@ -165,7 +164,6 @@ ncadios_close(void *ncdp) } ncadiosi_get_list_free(&(ncadp->getlist)); - NCI_Free(ncadp->path); NCI_Free(ncadp); return err; @@ -284,7 +282,7 @@ ncadios_inq_misc(void *ncdp, NC_ad *ncadp = (NC_ad*)ncdp; if (pathlen != NULL){ - *pathlen = strlen(ncadp->path); + *pathlen = (int)strlen(ncadp->path); } if (path != NULL){ diff --git a/src/drivers/ncadios/ncadios_lists.c b/src/drivers/ncadios/ncadios_lists.c index 3bf4e29cb4..3618ab0994 100644 --- a/src/drivers/ncadios/ncadios_lists.c +++ b/src/drivers/ncadios/ncadios_lists.c @@ -73,7 +73,7 @@ int ncadiosi_var_list_add(NC_ad_var_list *list, NC_ad_var data) { if (list->nalloc == 0){ list->nalloc = 16; - list->data = NCI_Malloc(sizeof(NC_ad_varp) * list->nalloc); + list->data = NCI_Malloc(sizeof(NC_ad_var) * list->nalloc); } else if (list->nalloc == id){ list->nalloc *= 2; diff --git a/src/drivers/ncadios/ncadios_misc.c b/src/drivers/ncadios/ncadios_misc.c index 82ac3a8f0b..777d32c238 100644 --- a/src/drivers/ncadios/ncadios_misc.c +++ b/src/drivers/ncadios/ncadios_misc.c @@ -47,25 +47,25 @@ nc_type ncadios_to_nc_type(enum ADIOS_DATATYPES atype){ case adios_string: return NC_CHAR; case adios_complex: -#ifdef PNETCDF_DEBUG +#if PNETCDF_DEBUG_MODE == 1 printf("Warning: unsupported adios type: adios_string_array\n"); fflush(stdout); #endif return NC_BYTE; case adios_double_complex: -#ifdef PNETCDF_DEBUG +#if PNETCDF_DEBUG_MODE == 1 printf("Warning: unsupported adios type: adios_string_array\n"); fflush(stdout); #endif return NC_BYTE; case adios_string_array: -#ifdef PNETCDF_DEBUG +#if PNETCDF_DEBUG_MODE == 1 printf("Warning: unsupported adios type: adios_string_array\n"); fflush(stdout); #endif return NC_BYTE; case adios_unknown: -#ifdef PNETCDF_DEBUG +#if PNETCDF_DEBUG_MODE == 1 printf("Warning: unsupported adios type: adios_unknown\n"); fflush(stdout); #endif @@ -104,19 +104,19 @@ MPI_Datatype ncadios_to_mpi_type(enum ADIOS_DATATYPES atype){ case adios_string_array: return MPI_CHAR; case adios_complex: -#ifdef PNETCDF_DEBUG +#if PNETCDF_DEBUG_MODE == 1 printf("Warning: unsupported adios type: adios_string_array\n"); fflush(stdout); #endif return NC_BYTE; case adios_double_complex: -#ifdef PNETCDF_DEBUG +#if PNETCDF_DEBUG_MODE == 1 printf("Warning: unsupported adios type: adios_string_array\n"); fflush(stdout); #endif return NC_BYTE; case adios_unknown: -#ifdef PNETCDF_DEBUG +#if PNETCDF_DEBUG_MODE == 1 printf("Warning: unsupported adios type: adios_unknown\n"); fflush(stdout); #endif diff --git a/src/drivers/ncadios/ncadios_var.c b/src/drivers/ncadios/ncadios_var.c index 315aa0d7af..91ba32bcff 100644 --- a/src/drivers/ncadios/ncadios_var.c +++ b/src/drivers/ncadios/ncadios_var.c @@ -35,9 +35,6 @@ * ncmpi_iget_varn_() : dispatcher->iget_varn() * ncmpi_iput_varn_() : dispatcher->iput_varn() * ncmpi_bput_varn_() : dispatcher->bput_varn() - * - * ncmpi_get_vard() : dispatcher->get_vard() - * ncmpi_put_vard() : dispatcher->put_vard() */ #ifdef HAVE_CONFIG_H @@ -337,29 +334,3 @@ ncadios_bput_varn(void *ncdp, DEBUG_RETURN_ERROR(NC_ENOTSUPPORT); } -int -ncadios_get_vard(void *ncdp, - int varid, - MPI_Datatype filetype, - void *buf, - MPI_Offset bufcount, - MPI_Datatype buftype, - int reqMode) -{ - /* ADIOS has no vard interface */ - DEBUG_RETURN_ERROR(NC_ENOTSUPPORT); -} - -int -ncadios_put_vard(void *ncdp, - int varid, - MPI_Datatype filetype, - const void *buf, - MPI_Offset bufcount, - MPI_Datatype buftype, - int reqMode) -{ - /* Read only driver */ - DEBUG_RETURN_ERROR(NC_ENOTSUPPORT); -} - diff --git a/src/drivers/ncbbio/Makefile.am b/src/drivers/ncbbio/Makefile.am index 4b08f999b2..50f4bc7190 100644 --- a/src/drivers/ncbbio/Makefile.am +++ b/src/drivers/ncbbio/Makefile.am @@ -13,10 +13,6 @@ AM_CPPFLAGS += -I${top_builddir}/src/include AM_CPPFLAGS += -I${top_srcdir}/src/drivers/include AM_CPPFLAGS += -I${top_builddir}/src/drivers/include -if PNETCDF_DEBUG - AM_CPPFLAGS += -DPNETCDF_DEBUG -endif - noinst_LTLIBRARIES = libncbbio.la M4FLAGS += -I${top_srcdir}/m4 diff --git a/src/drivers/ncbbio/ncbbio_driver.c b/src/drivers/ncbbio/ncbbio_driver.c index 356a7880b2..812567504b 100644 --- a/src/drivers/ncbbio/ncbbio_driver.c +++ b/src/drivers/ncbbio/ncbbio_driver.c @@ -55,8 +55,6 @@ static PNC_driver ncbbio_driver = { ncbbio_put_var, ncbbio_get_varn, ncbbio_put_varn, - ncbbio_get_vard, - ncbbio_put_vard, ncbbio_iget_var, ncbbio_iput_var, ncbbio_bput_var, diff --git a/src/drivers/ncbbio/ncbbio_driver.h b/src/drivers/ncbbio/ncbbio_driver.h index b585d2e05c..9ec74b95f4 100644 --- a/src/drivers/ncbbio/ncbbio_driver.h +++ b/src/drivers/ncbbio/ncbbio_driver.h @@ -173,7 +173,7 @@ typedef struct NC_bb { MPI_Offset recdimsize; MPI_Offset flushbuffersize; MPI_Offset maxentrysize; -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 /* Profiling information */ MPI_Offset total_data; MPI_Offset total_meta; @@ -196,7 +196,7 @@ typedef struct NC_bb { int mode; /* file _open/_create mode */ int flag; /* define/data/collective/indep mode */ int ncid; - char *path; /* path name */ + const char *path; /* path name */ MPI_Comm comm; /* MPI communicator */ MPI_Comm logcomm; /* MPI communicator */ MPI_Info info; @@ -259,11 +259,15 @@ int ncbbio_sharedfile_seek (NC_bb_sharedfile *f, off_t offset, int whence); void ncbbio_extract_hint (NC_bb *ncbbp, MPI_Info info); void ncbbio_export_hint (NC_bb *ncbbp, MPI_Info *info); -extern int ncbbio_create ( - MPI_Comm comm, const char *path, int cmode, int ncid, MPI_Info info, void **ncdp); +extern +int ncbbio_create(MPI_Comm comm, const char *path, int cmode, int ncid, + int env_mode, MPI_Info info, PNC_comm_attr node_ids, + void **ncdp); -extern int ncbbio_open ( - MPI_Comm comm, const char *path, int omode, int ncid, MPI_Info info, void **ncdp); +extern +int ncbbio_open(MPI_Comm comm, const char *path, int omode, int ncid, + int env_mode, MPI_Info info, PNC_comm_attr node_ids, + void **ncdp); extern int ncbbio_close (void *ncdp); diff --git a/src/drivers/ncbbio/ncbbio_file.c b/src/drivers/ncbbio/ncbbio_file.c index 271f1b969d..8f6c608d7f 100644 --- a/src/drivers/ncbbio/ncbbio_file.c +++ b/src/drivers/ncbbio/ncbbio_file.c @@ -49,12 +49,14 @@ #include int -ncbbio_create(MPI_Comm comm, - const char *path, - int cmode, - int ncid, - MPI_Info info, - void **ncpp) /* OUT */ +ncbbio_create(MPI_Comm comm, + const char *path, + int cmode, + int ncid, + int env_mode, + MPI_Info info, + PNC_comm_attr node_ids, /* node IDs of all processes */ + void **ncpp) /* OUT */ { int err; void *ncp=NULL; @@ -65,19 +67,15 @@ ncbbio_create(MPI_Comm comm, driver = ncmpio_inq_driver(); if (driver == NULL) DEBUG_RETURN_ERROR(NC_ENOTNC) - err = driver->create(comm, path, cmode, ncid, info, &ncp); + err = driver->create(comm, path, cmode, ncid, env_mode, info, node_ids, + &ncp); if (err != NC_NOERR) return err; /* Create a NC_bb object and save its driver pointer */ ncbbp = (NC_bb*) NCI_Malloc(sizeof(NC_bb)); if (ncbbp == NULL) DEBUG_RETURN_ERROR(NC_ENOMEM) - ncbbp->path = (char*) NCI_Malloc(strlen(path)+1); - if (ncbbp->path == NULL) { - NCI_Free(ncbbp); - DEBUG_RETURN_ERROR(NC_ENOMEM) - } - strcpy(ncbbp->path, path); + ncbbp->path = path; /* reuse path duplicated in dispatch layer */ ncbbp->mode = cmode; ncbbp->ncmpio_driver = driver; /* ncmpio driver */ ncbbp->ncid = ncid; @@ -104,12 +102,14 @@ ncbbio_create(MPI_Comm comm, } int -ncbbio_open(MPI_Comm comm, - const char *path, - int omode, - int ncid, - MPI_Info info, - void **ncpp) +ncbbio_open(MPI_Comm comm, + const char *path, + int omode, + int ncid, + int env_mode, + MPI_Info info, + PNC_comm_attr node_ids, /* node IDs of all processes */ + void **ncpp) /* OUT */ { int err; void *ncp=NULL; @@ -119,19 +119,14 @@ ncbbio_open(MPI_Comm comm, driver = ncmpio_inq_driver(); if (driver == NULL) DEBUG_RETURN_ERROR(NC_ENOTNC) - err = driver->open(comm, path, omode, ncid, info, &ncp); + err = driver->open(comm, path, omode, ncid, env_mode, info, node_ids, &ncp); if (err != NC_NOERR) return err; /* Create a NC_bb object and save its driver pointer */ ncbbp = (NC_bb*) NCI_Malloc(sizeof(NC_bb)); if (ncbbp == NULL) DEBUG_RETURN_ERROR(NC_ENOMEM) - ncbbp->path = (char*) NCI_Malloc(strlen(path)+1); - if (ncbbp->path == NULL) { - NCI_Free(ncbbp); - DEBUG_RETURN_ERROR(NC_ENOMEM) - } - strcpy(ncbbp->path, path); + ncbbp->path = path; /* reuse path duplicated in dispatch layer */ ncbbp->mode = omode; ncbbp->ncmpio_driver = driver; /* ncmpio driver */ ncbbp->ncid = ncid; @@ -220,7 +215,6 @@ ncbbio_close(void *ncdp) /* Cleanup NC-bb object */ if (ncbbp->info != MPI_INFO_NULL) MPI_Info_free(&ncbbp->info); - NCI_Free(ncbbp->path); NCI_Free(ncbbp); return status; @@ -375,6 +369,11 @@ ncbbio_begin_indep_data(void *ncdp) int err; NC_bb *ncbbp = (NC_bb*)ncdp; + err = ncbbio_sync(ncdp); + if (err != NC_NOERR) return err; + + MPI_Barrier(ncbbp->comm); + err = ncbbp->ncmpio_driver->begin_indep_data(ncbbp->ncp); if (err != NC_NOERR) return err; @@ -439,7 +438,6 @@ ncbbio_abort(void *ncdp) if (ncbbp->info != MPI_INFO_NULL) MPI_Info_free(&ncbbp->info); - NCI_Free(ncbbp->path); NCI_Free(ncbbp); return status; diff --git a/src/drivers/ncbbio/ncbbio_log.c b/src/drivers/ncbbio/ncbbio_log.c index 606f602568..05d340f326 100644 --- a/src/drivers/ncbbio/ncbbio_log.c +++ b/src/drivers/ncbbio/ncbbio_log.c @@ -16,6 +16,8 @@ #include /* opendir() closedir() */ #include #include +#include /* dirname() */ + #include #include #include @@ -29,18 +31,17 @@ int ncbbio_log_create(NC_bb* ncbbp, __attribute__((unused)) MPI_Info info) { int rank, np, err, flag, masterrank, procname_len; - char logbase[NC_LOG_MAX_PATH], basename[NC_LOG_MAX_PATH]; + char *logbase=NULL, *basename=NULL; char procname[MPI_MAX_PROCESSOR_NAME]; - char *abspath, *fname, *path, *fdir = NULL; - char *logbasep = "."; -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) + char *fname, *path, *fdir = NULL, *logbasep; +#if PNETCDF_PROFILING == 1 double t1, t2; #endif DIR *logdir; ssize_t headersize; NC_bb_metadataheader *headerp; -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 t1 = MPI_Wtime(); #endif @@ -68,7 +69,7 @@ int ncbbio_log_create(NC_bb* ncbbp, /* Determine log file name * Log file name is $(bufferdir)$(basename)_$(ncid)_$(rank).{meta/data} * filepath is absolute path to the cdf file - * If buffer directory is not set, we use the same directory as the NetCDF file + * If buffer directory is not set, we use the same directory as the file */ /* Read environment variable for burst buffer path */ @@ -86,21 +87,9 @@ int ncbbio_log_create(NC_bb* ncbbp, logbasep = ncmpii_remove_file_system_type_prefix(ncbbp->logbase); } else { - size_t i = strlen(path); - fdir = (char*)NCI_Malloc(sizeof(char) * (i + 1)); - strncpy(fdir, path, i + 1); - /* Search for first '\' from the back */ - for (i--; i > -1; i--) { - if (fdir[i] == '/') { - fdir[i + 1] = '\0'; - break; - } - } - - /* If directory is fund, use it as logbase */ - if (i >= 0) { - logbasep = fdir; - } + fdir = strdup(path); + logbasep = dirname(fdir); + if (logbasep == NULL) logbasep = "."; /* Warn if log base not set by user */ if (rank == 0) { @@ -121,13 +110,13 @@ int ncbbio_log_create(NC_bb* ncbbp, closedir(logdir); /* Resolve absolute path */ - abspath = realpath(path, basename); - if (abspath == NULL) { + basename = realpath(path, NULL); + if (basename == NULL) { /* Can not resolve absolute path */ DEBUG_RETURN_ERROR(NC_EBAD_FILE); } - abspath = realpath(logbasep, logbase); - if (abspath == NULL) { + logbase = realpath(logbasep, NULL); + if (logbase == NULL) { /* Can not resolve absolute path */ DEBUG_RETURN_ERROR(NC_EBAD_FILE); } @@ -194,7 +183,7 @@ int ncbbio_log_create(NC_bb* ncbbp, /* Set log file descriptor to NULL */ -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 /* Performance counters */ ncbbp->total_data = 0; ncbbp->total_meta = 0; @@ -296,7 +285,7 @@ int ncbbio_log_create(NC_bb* ncbbp, ncbbp->datalogsize = 8; -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 t2 = MPI_Wtime(); ncbbp->total_time += t2 - t1; ncbbp->create_time += t2 - t1; @@ -305,6 +294,9 @@ int ncbbio_log_create(NC_bb* ncbbp, ncbbp->total_data += 8; #endif + if (basename != NULL) free(basename); + if (logbase != NULL) free(logbase); + return NC_NOERR; } @@ -317,7 +309,7 @@ int ncbbio_log_enddef(NC_bb *ncbbp) int err; NC_bb_metadataheader *headerp; -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 double t1, t2; t1 = MPI_Wtime(); #endif @@ -345,7 +337,7 @@ int ncbbio_log_enddef(NC_bb *ncbbp) } } -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 t2 = MPI_Wtime(); ncbbp->total_time += t2 - t1; ncbbp->enddef_time += t2 - t1; @@ -365,7 +357,7 @@ int ncbbio_log_close(NC_bb *ncbbp, int err, status=NC_NOERR; NC_bb_metadataheader* headerp; -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 unsigned long long total_data; unsigned long long total_meta; unsigned long long buffer_size; @@ -420,7 +412,7 @@ int ncbbio_log_close(NC_bb *ncbbp, MPI_Comm_free(&(ncbbp->logcomm)); } -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 t2 = MPI_Wtime(); ncbbp->total_time += t2 - t1; ncbbp->close_time += t2 - t1; @@ -497,7 +489,7 @@ int ncbbio_log_flush(NC_bb* ncbbp) int err, status = NC_NOERR; NC_bb_metadataheader *headerp; -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 double t1, t2; t1 = MPI_Wtime(); #endif @@ -547,7 +539,7 @@ int ncbbio_log_flush(NC_bb* ncbbp) ncbbp->datalogsize = 8; -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 t2 = MPI_Wtime(); ncbbp->total_time += t2 - t1; ncbbp->flush_time += t2 - t1; diff --git a/src/drivers/ncbbio/ncbbio_log_flush.c b/src/drivers/ncbbio/ncbbio_log_flush.c index 444be0226b..43218042f4 100644 --- a/src/drivers/ncbbio/ncbbio_log_flush.c +++ b/src/drivers/ncbbio/ncbbio_log_flush.c @@ -84,7 +84,7 @@ int ncbbio_log_flush_core(NC_bb *ncbbp) { char *databuffer, *databufferoff; NC_bb_metadataheader *headerp; NC_bb_metadataptr *ip; -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 double t1, t2, t3, t4; t1 = MPI_Wtime(); @@ -106,7 +106,7 @@ int ncbbio_log_flush_core(NC_bb *ncbbp) { databuffersize = ncbbp->maxentrysize; } -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 if ((size_t)(ncbbp->max_buffer) < databuffersize){ ncbbp->max_buffer = databuffersize; } @@ -184,14 +184,14 @@ int ncbbio_log_flush_core(NC_bb *ncbbp) { * We read only what needed by pending requests */ if (dataread < databufferused){ -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 t2 = MPI_Wtime(); #endif err = ncbbio_sharedfile_read(ncbbp->datalog_fd, databuffer + dataread, databufferused - dataread); if (err != NC_NOERR){ return err; } -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 t3 = MPI_Wtime(); ncbbp->flush_data_rd_time += t3 - t2; #endif @@ -211,14 +211,14 @@ int ncbbio_log_flush_core(NC_bb *ncbbp) { * We read only what needed by pending requests */ if (dataread < databufferused){ -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 t2 = MPI_Wtime(); #endif err = ncbbio_sharedfile_read(ncbbp->datalog_fd, databuffer + dataread, databufferused - dataread); if (err != NC_NOERR){ return err; } -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 t3 = MPI_Wtime(); ncbbp->flush_data_rd_time += t3 - t2; #endif @@ -250,7 +250,7 @@ int ncbbio_log_flush_core(NC_bb *ncbbp) { stride = NULL; } -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 t2 = MPI_Wtime(); #endif @@ -260,7 +260,7 @@ int ncbbio_log_flush_core(NC_bb *ncbbp) { status = err; } -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 t3 = MPI_Wtime(); ncbbp->flush_put_time += t3 - t2; #endif @@ -294,7 +294,7 @@ int ncbbio_log_flush_core(NC_bb *ncbbp) { return err; } -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 t2 = MPI_Wtime(); #endif @@ -309,7 +309,7 @@ int ncbbio_log_flush_core(NC_bb *ncbbp) { status = err; } -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 t3 = MPI_Wtime(); ncbbp->flush_put_time += t3 - t2; #endif @@ -324,7 +324,7 @@ int ncbbio_log_flush_core(NC_bb *ncbbp) { entryp = (NC_bb_metadataentry*)(((char*)entryp) + entryp->esize); } -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 t2 = MPI_Wtime(); #endif /* @@ -340,7 +340,7 @@ int ncbbio_log_flush_core(NC_bb *ncbbp) { status = err; } -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 t3 = MPI_Wtime(); ncbbp->flush_wait_time += t3 - t2; #endif @@ -388,7 +388,7 @@ int ncbbio_log_flush_core(NC_bb *ncbbp) { if (starts != NULL) NCI_Free(starts); -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 t4 = MPI_Wtime(); ncbbp->flush_replay_time += t4 - t1; #endif diff --git a/src/drivers/ncbbio/ncbbio_log_put.c b/src/drivers/ncbbio/ncbbio_log_put.c index a2018a0774..4fbdb33c7e 100644 --- a/src/drivers/ncbbio/ncbbio_log_put.c +++ b/src/drivers/ncbbio/ncbbio_log_put.c @@ -76,7 +76,7 @@ int ncbbio_log_put_var(NC_bb *ncbbp, NC_bb_metadataentry *entryp; NC_bb_metadataheader *headerp; -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 double t1, t2, t3, t4, t5; t1 = MPI_Wtime(); #endif @@ -121,7 +121,7 @@ int ncbbio_log_put_var(NC_bb *ncbbp, err = mpitype2logtype(buftype, &itype); if (err != NC_NOERR){ -#ifdef PNETCDF_DEBUG +#if PNETCDF_DEBUG_MODE == 1 int name_len; char type_name[MPI_MAX_OBJECT_NAME]; MPI_Type_get_name(buftype, type_name, &name_len); @@ -199,7 +199,7 @@ int ncbbio_log_put_var(NC_bb *ncbbp, ncbbio_metaidx_add(ncbbp, (NC_bb_metadataentry*)((char*)entryp - (char*)(ncbbp->metadata.buffer))); -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 t2 = MPI_Wtime(); #endif @@ -210,7 +210,7 @@ int ncbbio_log_put_var(NC_bb *ncbbp, err = ncbbio_sharedfile_write(ncbbp->datalog_fd, buf, put_size); if (err != NC_NOERR) return err; -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 t3 = MPI_Wtime(); #endif @@ -228,7 +228,7 @@ int ncbbio_log_put_var(NC_bb *ncbbp, err = ncbbio_sharedfile_write(ncbbp->metalog_fd, buffer, esize); if (err != NC_NOERR) return err; -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 t4 = MPI_Wtime(); #endif @@ -239,7 +239,7 @@ int ncbbio_log_put_var(NC_bb *ncbbp, SIZEOF_MPI_OFFSET, 56); if (err != NC_NOERR) return err; -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 t5 = MPI_Wtime(); ncbbp->put_data_wr_time += t3 - t2; ncbbp->put_meta_wr_time += t4 - t3; @@ -281,7 +281,7 @@ int ncbbio_log_put_varn(NC_bb *ncbbp, NC_bb_metadataentry *entryp; NC_bb_metadataheader *headerp; -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 double t1, t2, t3, t4, t5; t1 = MPI_Wtime(); #endif @@ -299,7 +299,7 @@ int ncbbio_log_put_varn(NC_bb *ncbbp, /* Convert to ext type */ err = mpitype2logtype(buftype, &itype); if (err != NC_NOERR){ -#ifdef PNETCDF_DEBUG +#if PNETCDF_DEBUG_MODE == 1 int name_len; char type_name[MPI_MAX_OBJECT_NAME]; MPI_Type_get_name(buftype, type_name, &name_len); @@ -418,7 +418,7 @@ int ncbbio_log_put_varn(NC_bb *ncbbp, ncbbio_metaidx_add(ncbbp, (NC_bb_metadataentry*)((char*)entryp - (char*)(ncbbp->metadata.buffer))); -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 t2 = MPI_Wtime(); #endif @@ -435,7 +435,7 @@ int ncbbio_log_put_varn(NC_bb *ncbbp, err = ncbbio_sharedfile_write(ncbbp->datalog_fd, buf, total_put_size); if (err != NC_NOERR) return err; -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 t3 = MPI_Wtime(); #endif @@ -453,7 +453,7 @@ int ncbbio_log_put_varn(NC_bb *ncbbp, err = ncbbio_sharedfile_write(ncbbp->metalog_fd, buffer, esize); if (err != NC_NOERR) return err; -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 t4 = MPI_Wtime(); #endif @@ -464,7 +464,7 @@ int ncbbio_log_put_varn(NC_bb *ncbbp, SIZEOF_MPI_OFFSET, 56); if (err != NC_NOERR) return err; -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) +#if PNETCDF_PROFILING == 1 t5 = MPI_Wtime(); ncbbp->put_data_wr_time += t3 - t2; ncbbp->put_meta_wr_time += t4 - t3; diff --git a/src/drivers/ncbbio/ncbbio_nonblocking.c b/src/drivers/ncbbio/ncbbio_nonblocking.c index 8a1c2b31f6..be00c480c6 100644 --- a/src/drivers/ncbbio/ncbbio_nonblocking.c +++ b/src/drivers/ncbbio/ncbbio_nonblocking.c @@ -221,7 +221,7 @@ int ncbbio_handle_put_req(NC_bb *ncbbp, int reqid, int *stat) * If it do, we have mising log entry or corrupt metadata index */ if (!req->ready) { - printf("Fatal error: nonblocking request not in log file\n"); + fprintf(stderr,"Fatal error: nonblocking request not in log file\n"); MPI_Abort(MPI_COMM_WORLD, -1); } diff --git a/src/drivers/ncbbio/ncbbio_var.c b/src/drivers/ncbbio/ncbbio_var.c index 74e3dd274a..ffa86af027 100644 --- a/src/drivers/ncbbio/ncbbio_var.c +++ b/src/drivers/ncbbio/ncbbio_var.c @@ -36,9 +36,6 @@ * ncmpi_iget_varn_() : dispatcher->iget_varn() * ncmpi_iput_varn_() : dispatcher->iput_varn() * ncmpi_bput_varn_() : dispatcher->bput_varn() - * - * ncmpi_get_vard() : dispatcher->get_vard() - * ncmpi_put_vard() : dispatcher->put_vard() */ #ifdef HAVE_CONFIG_H @@ -574,40 +571,3 @@ ncbbio_bput_varn(void *ncdp, bufcount, buftype, reqid, reqMode); } -int -ncbbio_get_vard(void *ncdp, - int varid, - MPI_Datatype filetype, - void *buf, - MPI_Offset bufcount, - MPI_Datatype buftype, - int reqMode) -{ - int err, status = NC_NOERR; - NC_bb *ncbbp = (NC_bb*)ncdp; - - /* Flush on read */ - if (ncbbp->inited) - status = ncbbio_log_flush(ncbbp); - - err = ncbbp->ncmpio_driver->get_vard(ncbbp->ncp, varid, filetype, buf, - bufcount, buftype, reqMode); - return (status == NC_NOERR) ? err : status; -} - -int -ncbbio_put_vard(void *ncdp, - int varid, - MPI_Datatype filetype, - const void *buf, - MPI_Offset bufcount, - MPI_Datatype buftype, - int reqMode) -{ - NC_bb *ncbbp = (NC_bb*)ncdp; - - /* BB driver does not support vard */ - return ncbbp->ncmpio_driver->put_vard(ncbbp->ncp, varid, filetype, buf, - bufcount, buftype, reqMode); -} - diff --git a/src/drivers/ncfoo/Makefile.am b/src/drivers/ncfoo/Makefile.am index 04641373b6..023e007b17 100644 --- a/src/drivers/ncfoo/Makefile.am +++ b/src/drivers/ncfoo/Makefile.am @@ -13,10 +13,6 @@ AM_CPPFLAGS += -I${top_builddir}/src/include AM_CPPFLAGS += -I${top_srcdir}/src/drivers/include AM_CPPFLAGS += -I${top_builddir}/src/drivers/include -if PNETCDF_DEBUG - AM_CPPFLAGS += -DPNETCDF_DEBUG -endif - noinst_LTLIBRARIES = libncfoo.la M4FLAGS += -I${top_srcdir}/m4 diff --git a/src/drivers/ncfoo/ncfoo_driver.c b/src/drivers/ncfoo/ncfoo_driver.c index 23d2448c9c..bb399e23a2 100644 --- a/src/drivers/ncfoo/ncfoo_driver.c +++ b/src/drivers/ncfoo/ncfoo_driver.c @@ -56,8 +56,6 @@ static PNC_driver ncfoo_driver = { ncfoo_put_var, ncfoo_get_varn, ncfoo_put_varn, - ncfoo_get_vard, - ncfoo_put_vard, ncfoo_iget_var, ncfoo_iput_var, ncfoo_bput_var, diff --git a/src/drivers/ncfoo/ncfoo_driver.h b/src/drivers/ncfoo/ncfoo_driver.h index b266ffa513..3f9f49b698 100644 --- a/src/drivers/ncfoo/ncfoo_driver.h +++ b/src/drivers/ncfoo/ncfoo_driver.h @@ -15,17 +15,19 @@ typedef struct NC_foo NC_foo; /* forward reference */ struct NC_foo { int mode; /* file _open/_create mode */ int flag; /* define/data/collective/indep mode */ - char *path; /* path name */ + const char *path; /* path name */ MPI_Comm comm; /* MPI communicator */ void *ncp; /* pointer to driver's internal object */ struct PNC_driver *driver; }; extern int -ncfoo_create(MPI_Comm comm, const char *path, int cmode, int ncid, MPI_Info info, void **ncdp); +ncfoo_create(MPI_Comm comm, const char *path, int cmode, int ncid, + int env_mode, MPI_Info info, PNC_comm_attr node_ids, void **ncdp); extern int -ncfoo_open(MPI_Comm comm, const char *path, int omode, int ncid, MPI_Info info, void **ncdp); +ncfoo_open(MPI_Comm comm, const char *path, int omode, int ncid, + int env_mode, MPI_Info info, PNC_comm_attr node_ids, void **ncdp); extern int ncfoo_close(void *ncdp); diff --git a/src/drivers/ncfoo/ncfoo_file.c b/src/drivers/ncfoo/ncfoo_file.c index d4cc2e22fc..f09b7c0fdd 100644 --- a/src/drivers/ncfoo/ncfoo_file.c +++ b/src/drivers/ncfoo/ncfoo_file.c @@ -47,35 +47,32 @@ #include int -ncfoo_create(MPI_Comm comm, - const char *path, - int cmode, - int ncid, - MPI_Info info, - void **ncpp) /* OUT */ +ncfoo_create(MPI_Comm comm, + const char *path, + int cmode, + int ncid, + int env_mode, + MPI_Info info, + PNC_comm_attr node_ids, /* node IDs of all processes */ + void **ncpp) /* OUT */ { int err; void *ncp=NULL; NC_foo *foo; PNC_driver *driver=NULL; - /* TODO: use comde to determine the true driver */ + /* TODO: use cmode to determine the true driver */ driver = ncmpio_inq_driver(); if (driver == NULL) return NC_ENOTNC; - err = driver->create(comm, path, cmode, ncid, info, &ncp); + err = driver->create(comm, path, cmode, ncid, env_mode, info, node_ids, &ncp); if (err != NC_NOERR) return err; /* Create a NC_foo object and save its driver pointer */ foo = (NC_foo*) NCI_Malloc(sizeof(NC_foo)); if (foo == NULL) DEBUG_RETURN_ERROR(NC_ENOMEM) - foo->path = (char*) NCI_Malloc(strlen(path)+1); - if (foo->path == NULL) { - NCI_Free(foo); - DEBUG_RETURN_ERROR(NC_ENOMEM) - } - strcpy(foo->path, path); + foo->path = path; /* reuse path duplicated in dispatch layer */ foo->mode = cmode; foo->driver = driver; foo->flag = 0; @@ -88,12 +85,14 @@ ncfoo_create(MPI_Comm comm, } int -ncfoo_open(MPI_Comm comm, - const char *path, - int omode, - int ncid, - MPI_Info info, - void **ncpp) +ncfoo_open(MPI_Comm comm, + const char *path, + int omode, + int ncid, + int env_mode, + MPI_Info info, + PNC_comm_attr node_ids, /* node IDs of all processes */ + void **ncpp) /* OUT */ { int err, format; void *ncp=NULL; @@ -110,19 +109,14 @@ ncfoo_open(MPI_Comm comm, } if (driver == NULL) return NC_ENOTNC; - err = driver->open(comm, path, omode, ncid, info, &ncp); + err = driver->open(comm, path, omode, ncid, env_mode, info, node_ids, &ncp); if (err != NC_NOERR) return err; /* Create a NC_foo object and save its driver pointer */ foo = (NC_foo*) NCI_Malloc(sizeof(NC_foo)); if (foo == NULL) DEBUG_RETURN_ERROR(NC_ENOMEM) - foo->path = (char*) NCI_Malloc(strlen(path)+1); - if (foo->path == NULL) { - NCI_Free(foo); - DEBUG_RETURN_ERROR(NC_ENOMEM) - } - strcpy(foo->path, path); + foo->path = path; /* reuse path duplicated in dispatch layer */ foo->mode = omode; foo->driver = driver; foo->flag = 0; @@ -144,7 +138,6 @@ ncfoo_close(void *ncdp) err = foo->driver->close(foo->ncp); - NCI_Free(foo->path); NCI_Free(foo); return err; @@ -153,13 +146,9 @@ ncfoo_close(void *ncdp) int ncfoo_enddef(void *ncdp) { - int err; NC_foo *foo = (NC_foo*)ncdp; - err = foo->driver->enddef(foo->ncp); - if (err != NC_NOERR) return err; - - return NC_NOERR; + return foo->driver->enddef(foo->ncp); } int @@ -169,50 +158,34 @@ ncfoo__enddef(void *ncdp, MPI_Offset v_minfree, MPI_Offset r_align) { - int err; NC_foo *foo = (NC_foo*)ncdp; - err = foo->driver->_enddef(foo->ncp, h_minfree, v_align, v_minfree, - r_align); - if (err != NC_NOERR) return err; - - return NC_NOERR; + return foo->driver->_enddef(foo->ncp, h_minfree, v_align, v_minfree, + r_align); } int ncfoo_redef(void *ncdp) { - int err; NC_foo *foo = (NC_foo*)ncdp; - err = foo->driver->redef(foo->ncp); - if (err != NC_NOERR) return err; - - return NC_NOERR; + return foo->driver->redef(foo->ncp); } int ncfoo_begin_indep_data(void *ncdp) { - int err; NC_foo *foo = (NC_foo*)ncdp; - err = foo->driver->begin_indep_data(foo->ncp); - if (err != NC_NOERR) return err; - - return NC_NOERR; + return foo->driver->begin_indep_data(foo->ncp); } int ncfoo_end_indep_data(void *ncdp) { - int err; NC_foo *foo = (NC_foo*)ncdp; - err = foo->driver->end_indep_data(foo->ncp); - if (err != NC_NOERR) return err; - - return NC_NOERR; + return foo->driver->end_indep_data(foo->ncp); } int @@ -225,7 +198,6 @@ ncfoo_abort(void *ncdp) err = foo->driver->abort(foo->ncp); - NCI_Free(foo->path); NCI_Free(foo); return err; @@ -238,13 +210,9 @@ ncfoo_inq(void *ncdp, int *nattsp, int *xtendimp) { - int err; NC_foo *foo = (NC_foo*)ncdp; - err = foo->driver->inq(foo->ncp, ndimsp, nvarsp, nattsp, xtendimp); - if (err != NC_NOERR) return err; - - return NC_NOERR; + return foo->driver->inq(foo->ncp, ndimsp, nvarsp, nattsp, xtendimp); } int @@ -265,16 +233,12 @@ ncfoo_inq_misc(void *ncdp, MPI_Offset *usage, MPI_Offset *buf_size) { - int err; NC_foo *foo = (NC_foo*)ncdp; - err = foo->driver->inq_misc(foo->ncp, pathlen, path, num_fix_varsp, - num_rec_varsp, striping_size, striping_count, - header_size, header_extent, recsize, put_size, - get_size, info_used, nreqs, usage, buf_size); - if (err != NC_NOERR) return err; - - return NC_NOERR; + return foo->driver->inq_misc(foo->ncp, pathlen, path, num_fix_varsp, + num_rec_varsp, striping_size, striping_count, + header_size, header_extent, recsize, put_size, + get_size, info_used, nreqs, usage, buf_size); } int @@ -283,13 +247,9 @@ ncfoo_cancel(void *ncdp, int *req_ids, int *statuses) { - int err; NC_foo *foo = (NC_foo*)ncdp; - err = foo->driver->cancel(foo->ncp, num_req, req_ids, statuses); - if (err != NC_NOERR) return err; - - return NC_NOERR; + return foo->driver->cancel(foo->ncp, num_req, req_ids, statuses); } int @@ -299,13 +259,9 @@ ncfoo_wait(void *ncdp, int *statuses, int reqMode) { - int err; NC_foo *foo = (NC_foo*)ncdp; - err = foo->driver->wait(foo->ncp, num_reqs, req_ids, statuses, reqMode); - if (err != NC_NOERR) return err; - - return NC_NOERR; + return foo->driver->wait(foo->ncp, num_reqs, req_ids, statuses, reqMode); } int @@ -313,13 +269,9 @@ ncfoo_set_fill(void *ncdp, int fill_mode, int *old_fill_mode) { - int err; NC_foo *foo = (NC_foo*)ncdp; - err = foo->driver->set_fill(foo->ncp, fill_mode, old_fill_mode); - if (err != NC_NOERR) return err; - - return NC_NOERR; + return foo->driver->set_fill(foo->ncp, fill_mode, old_fill_mode); } int @@ -327,13 +279,9 @@ ncfoo_fill_var_rec(void *ncdp, int varid, MPI_Offset recno) { - int err; NC_foo *foo = (NC_foo*)ncdp; - err = foo->driver->fill_var_rec(foo->ncp, varid, recno); - if (err != NC_NOERR) return err; - - return NC_NOERR; + return foo->driver->fill_var_rec(foo->ncp, varid, recno); } int @@ -342,48 +290,32 @@ ncfoo_def_var_fill(void *ncdp, int no_fill, const void *fill_value) { - int err; NC_foo *foo = (NC_foo*)ncdp; - err = foo->driver->def_var_fill(foo->ncp, varid, no_fill, fill_value); - if (err != NC_NOERR) return err; - - return NC_NOERR; + return foo->driver->def_var_fill(foo->ncp, varid, no_fill, fill_value); } int ncfoo_sync_numrecs(void *ncdp) { - int err; NC_foo *foo = (NC_foo*)ncdp; - err = foo->driver->sync_numrecs(foo->ncp); - if (err != NC_NOERR) return err; - - return NC_NOERR; + return foo->driver->sync_numrecs(foo->ncp); } int ncfoo_sync(void *ncdp) { - int err; NC_foo *foo = (NC_foo*)ncdp; - err = foo->driver->sync(foo->ncp); - if (err != NC_NOERR) return err; - - return NC_NOERR; + return foo->driver->sync(foo->ncp); } int ncfoo_flush(void *ncdp) { - int err; NC_foo *foo = (NC_foo*)ncdp; - err = foo->driver->flush(foo->ncp); - if (err != NC_NOERR) return err; - - return NC_NOERR; + return foo->driver->flush(foo->ncp); } diff --git a/src/drivers/ncfoo/ncfoo_var.c b/src/drivers/ncfoo/ncfoo_var.c index c428ea3065..f8b4ca8d74 100644 --- a/src/drivers/ncfoo/ncfoo_var.c +++ b/src/drivers/ncfoo/ncfoo_var.c @@ -36,9 +36,6 @@ * ncmpi_iget_varn_() : dispatcher->iget_varn() * ncmpi_iput_varn_() : dispatcher->iput_varn() * ncmpi_bput_varn_() : dispatcher->bput_varn() - * - * ncmpi_get_vard() : dispatcher->get_vard() - * ncmpi_put_vard() : dispatcher->put_vard() */ #ifdef HAVE_CONFIG_H @@ -398,41 +395,3 @@ ncfoo_bput_varn(void *ncdp, return NC_NOERR; } -int -ncfoo_get_vard(void *ncdp, - int varid, - MPI_Datatype filetype, - void *buf, - MPI_Offset bufcount, - MPI_Datatype buftype, - int reqMode) -{ - int err; - NC_foo *foo = (NC_foo*)ncdp; - - err = foo->driver->get_vard(foo->ncp, varid, filetype, buf, bufcount, - buftype, reqMode); - if (err != NC_NOERR) return err; - - return NC_NOERR; -} - -int -ncfoo_put_vard(void *ncdp, - int varid, - MPI_Datatype filetype, - const void *buf, - MPI_Offset bufcount, - MPI_Datatype buftype, - int reqMode) -{ - int err; - NC_foo *foo = (NC_foo*)ncdp; - - err = foo->driver->put_vard(foo->ncp, varid, filetype, buf, bufcount, - buftype, reqMode); - if (err != NC_NOERR) return err; - - return NC_NOERR; -} - diff --git a/src/drivers/ncmpio/Makefile.am b/src/drivers/ncmpio/Makefile.am index c1afe76c19..63ef698d17 100644 --- a/src/drivers/ncmpio/Makefile.am +++ b/src/drivers/ncmpio/Makefile.am @@ -13,8 +13,8 @@ AM_CPPFLAGS += -I${top_builddir}/src/include AM_CPPFLAGS += -I${top_srcdir}/src/drivers/include AM_CPPFLAGS += -I${top_builddir}/src/drivers/include -if PNETCDF_DEBUG - AM_CPPFLAGS += -DPNETCDF_DEBUG +if ENABLE_GIO + AM_CPPFLAGS += -I${top_builddir}/gio/src endif noinst_LTLIBRARIES = libncmpio.la @@ -46,14 +46,13 @@ C_SRCS = ncmpio_driver.c \ ncmpio_dim.c \ ncmpio_var.c \ ncmpio_bput.c \ - ncmpio_filetype.c \ ncmpio_wait.c \ - ncmpio_vard.c \ ncmpio_fill.c \ ncmpio_util.c \ ncmpio_hash_func.c \ ncmpio_file_io.c \ - ncmpio_intra_node.c + ncmpio_intra_node.c \ + ncmpio_fstype.c $(M4_SRCS:.m4=.c): Makefile diff --git a/src/drivers/ncmpio/ncmpio_NC.h b/src/drivers/ncmpio/ncmpio_NC.h index 0e2c71e814..b0fdc0c878 100644 --- a/src/drivers/ncmpio/ncmpio_NC.h +++ b/src/drivers/ncmpio/ncmpio_NC.h @@ -17,13 +17,43 @@ #include #include "ncmpio_driver.h" +#if PNETCDF_DRIVER_GIO == 1 +#include "gio.h" +#endif + +#define PNC_PERM 0666 /* file creation permission mask */ + +#define PNC_FS_UFS 152 /* Unix file system */ +#define PNC_FS_LUSTRE 163 /* Lustre file system */ + +#define PNC_DRIVER_GIO 0 /* Use GIO driver (default if enabled) */ +#define PNC_DRIVER_MPIIO 1 /* Use MPI-IO driver */ + +#define PNCIO_STRIPING_AUTO -1 +#define PNCIO_STRIPING_INHERIT 0 + +typedef struct { + MPI_Offset size; /* total size in bytes, i.e. sum of len[*] */ + MPI_Offset count; /* number of off-len pairs */ + MPI_Offset *off; /* [count] byte offsets */ + MPI_Offset *len; /* [count] block lengths in bytes */ +} PNCIO_View; + +/* default free space in the file header section. */ #define NC_DEFAULT_H_MINFREE 0 -#define NC_DEFAULT_V_ALIGN 512 + +/* default free space in the fix-sized variable section. */ #define NC_DEFAULT_V_MINFREE 0 + +/* default alignment for the starting offset of record variable section. */ #define NC_DEFAULT_R_ALIGN 4 -#define FILE_ALIGNMENT_DEFAULT 512 -#define FILE_ALIGNMENT_LB 4 +/* The default file header extent size is aligned with NC_DEFAULT_V_ALIGN. + * This default will be overwritten by hint nc_header_align_size or + * nc_var_align_size. Note when both hints are set by users, hint + * nc_var_align_size supersedes nc_header_align_size. + */ +#define NC_DEFAULT_V_ALIGN 512 /* MPI_OFFSET datatype was introduced in MPI 2.2 */ #if MPI_VERSION < 3 @@ -50,13 +80,25 @@ /* ncmpi_create/ncmpi_open set up header to be 'chunksize' big and to grow * by 'chunksize' as new items added. This used to be 4k. 256k lets us read - * in an entire climate header in one go */ -#define PNC_DEFAULT_CHUNKSIZE 262144 + * in an entire climate header in one go. This default will be overwritten by + * hint nc_header_read_chunk_size. + */ +#define PNC_HDR_READ_CHUNK_SIZE 262144 + +/* When file header grows or variables need to be moved to higher file offsets, + * data movement is performed in chunks of size PNC_MOVE_CHUNK_SIZE each per + * process. If the number of chunks is larger than the number of processes, + * carry out the data movement in multiple rounds. This default will be + * overwritten by hint nc_data_move_chunk_size. + */ +#define PNC_DATA_MOVE_CHUNK_SIZE 1048576 /* default size of temporal buffer to pack noncontiguous user buffers for MPI * collective read and write during ncmpi_wait/wait_all(). On some systems, * e.g. Cray KNL, using contiguous user buffers in collective I/O is much - * faster than noncontiguous. */ + * faster than noncontiguous. This default will be overwritten by hint + * nc_ibuf_size. + */ #define PNC_DEFAULT_IBUF_SIZE 16777216 /* when variable's nctype is NC_CHAR, I/O buffer's MPI type must be MPI_CHAR @@ -64,9 +106,8 @@ #define NCMPII_ECHAR(nctype, mpitype) ((((nctype) == NC_CHAR) == ((mpitype) != MPI_CHAR)) ? NC_ECHAR : NC_NOERR) /* - * The extern size of an empty - * netcdf version 1 file. - * The initial value of ncp->xsz. + * The external size of an empty netcdf version 1 file is 32 bytes, which is + * also set as the initial value of ncp->xsz. */ #define MIN_NC_XSZ 32 @@ -88,14 +129,18 @@ typedef enum { att_list = ABSENT | NC_ATTRIBUTE nelems [attr ...] var_list = ABSENT | NC_VARIABLE nelems [var ...] ABSENT = ZERO ZERO // Means list is not present - ZERO = \x00 \x00 \x00 \x00 // 32-bit zero + ZERO = \x00 \x00 \x00 \x00 // 32-bit zero for CDF-1/2 + // 64-bit zero for CDF-5 Minimum happens when nothing is defined, i.e. magic -- 4 bytes - numrecs -- 4 bytes for CDF-1 and CDF-2, 8 bytes for CDF-5 - dim_list = ABSENT -- 8 bytes - gatt_list = ABSENT -- 8 bytes - var_list = ABSENT -- 8 bytes + numrecs -- 4 bytes for CDF-1 and CDF-2, 8 bytes for CDF-5 + dim_list = ABSENT -- 8 bytes for CDF-1 and CDF-2, 12 bytes for CDF-5 + gatt_list = ABSENT -- 8 bytes for CDF-1 and CDF-2, 12 bytes for CDF-5 + var_list = ABSENT -- 8 bytes for CDF-1 and CDF-2, 12 bytes for CDF-5 + + Thus, the minimum file size for CDF-1/2 files are of size 32 bytes, and CDF-5 + files 48 bytes. */ typedef struct NC NC; /* forward reference */ @@ -156,7 +201,7 @@ typedef struct { * specifications can be of type 8-byte integers. */ typedef struct NC_dimarray { - int ndefined; /* number of defined dimensions */ + int ndefined; /* no. defined dimensions */ int unlimited_id; /* -1 for not defined, otherwise >= 0 */ NC_dim **value; int hash_size; @@ -180,7 +225,7 @@ ncmpio_dup_NC_dimarray(NC_dimarray *ncap, const NC_dimarray *ref); * NC attribute */ typedef struct { - MPI_Offset nelems; /* number of attribute elements */ + MPI_Offset nelems; /* no. attribute elements */ MPI_Offset xsz; /* amount of space at xvalue (4-byte aligned) */ nc_type xtype; /* external NC data type of the attribute */ size_t name_len; /* strlen(name) for faster string compare */ @@ -199,7 +244,7 @@ typedef struct { * specifications can be of type 8-byte integers. */ typedef struct NC_attrarray { - int ndefined; /* number of defined attributes */ + int ndefined; /* no. defined attributes */ NC_attr **value; int hash_size; NC_nametable *nameT; @@ -238,7 +283,7 @@ typedef struct { int no_fill; /* whether fill mode is disabled */ size_t name_len;/* strlen(name) for faster string compare */ char *name; /* name of the variable */ - int ndims; /* number of dimensions */ + int ndims; /* no. dimensions */ int *dimids; /* [ndims] array of dimension IDs */ MPI_Offset *shape; /* [ndims] dim->size of each dim shape[0] == NC_UNLIMITED if record variable */ @@ -248,7 +293,7 @@ typedef struct { total size in bytes of the array variable. For record variable, this is the record size */ NC_attrarray attrs; /* attribute array */ -#ifdef ENABLE_SUBFILING +#if PNETCDF_SUBFILING == 1 int num_subfiles; int ndims_org; /* ndims before subfiling */ int *dimids_org; /* dimids before subfiling */ @@ -268,8 +313,8 @@ typedef struct { */ /* note: we only allow less than 2^31-1 variables defined in a file */ typedef struct NC_vararray { - int ndefined; /* number of defined variables */ - int num_rec_vars;/* number of defined record variables */ + int ndefined; /* no. defined variables */ + int num_rec_vars;/* no. defined record variables */ NC_var **value; int hash_size; NC_nametable *nameT; @@ -319,15 +364,15 @@ typedef struct NC_lead_req { int flag; /* bit-wise OR of the above NC_REQ_* flags */ int id; /* even number for write, odd for read */ int nonlead_off; /* start index in the non-lead queue */ - int nonlead_num; /* number of non-lead requests */ + int nonlead_num; /* no. non-lead requests */ int abuf_index; /* index in the abuf occupy_table. -1 means not using attached buffer */ void *buf; /* user buffer */ void *xbuf; /* buffer in external type, may be == buf */ NC_var *varp; /* pointer to NC variable object */ - MPI_Offset nelems; /* total number of array elements requested */ + MPI_Offset nelems; /* total no. array elements requested */ MPI_Offset max_rec; /* highest record requested */ - MPI_Offset bufcount; /* number of buftype in this request */ + MPI_Offset bufcount; /* no. buftype in this request */ MPI_Offset *start; /* [varp->ndims*3] for start/count/stride */ MPI_Datatype buftype; /* user defined derived data type */ MPI_Datatype itype; /* internal element data type in buftype */ @@ -338,10 +383,11 @@ typedef struct NC_lead_req { typedef struct NC_req { MPI_Offset offset_start; /* starting offset of aggregate access region */ MPI_Offset offset_end; /* ending offset of aggregate access region */ - MPI_Offset nelems; /* number of array elements requested */ + MPI_Offset nelems; /* no. array elements requested */ MPI_Offset *start; /* [varp->ndims*3] for start/count/stride */ void *xbuf; /* buffer in external type, used in file I/O calls */ int lead_off; /* start index in the lead queue */ + MPI_Aint npairs; /* no. flattened offset-length pairs */ } NC_req; #define NC_ABUF_DEFAULT_TABLE_SIZE 128 @@ -369,93 +415,150 @@ typedef struct NC_buf { #define NC_HSYNC 0x200000 /* synchronise whole header on change */ #define NC_NDIRTY 0x400000 /* numrecs has changed */ #define NC_HDIRTY 0x800000 /* header info has changed */ -#define NC_HCOLL 0x000001 /* write header collectively */ struct NC { - int ncid; /* file ID */ - int flags; /* various modes, i.e. define/data, fill, - indep/coll, header dirty, etc */ - int iomode; /* cmode or omode used in ncmpi_create/open */ - int mpiomode; /* mode used in MPI_File_open, passed from - * collective open to independent open */ - int format; /* 1, 2, or 5 corresponding to CDF-1, 2, or 5 */ - int safe_mode; /* 0 or 1, for parameter consistency check */ -#ifdef ENABLE_SUBFILING + int ncid; /* file ID */ + int flags; /* various modes, i.e. define/data, fill, indep/coll, + * header dirty, etc. + */ + int nc_amode; /* cmode or omode used in ncmpi_create/open */ + int mpi_amode; /* mode used in MPI_File_open, passed from collective + * open to independent open + */ + int format; /* 1, 2, or 5 corresponding to CDF-1, 2, or 5 */ + int is_open; /* 0: not open yet 1: is open. When INA is enabled, only + * INA aggregators' is_open is set to 1 after file + * create/open. Non-INA aggregators' is_open 0. Once a + * non-INA aggregators makes an independent I/O call, its + * is_open will be set to 1. When INA is disabled, all + * processes' is_open is set to 1 after file create/open. + */ +#if PNETCDF_SUBFILING == 1 int subfile_mode; /* 0 or 1, for disable/enable subfiling */ - int num_subfiles; /* number of subfiles */ + int num_subfiles; /* no. subfiles */ struct NC *ncp_sf; /* ncp of subfile */ MPI_Comm comm_sf; /* subfile MPI communicator */ + PNC_comm_attr comm_attr_sf; /* node IDs of subfile MPI communicator */ #endif - int striping_unit; /* stripe size of the file */ - int chunk; /* chunk size for reading header, one chunk at a time */ - MPI_Offset v_align; /* alignment of the beginning of fixed-size variables */ - MPI_Offset r_align; /* file alignment for record variable section */ - MPI_Offset env_v_align; /* v_align set in environment variable */ - MPI_Offset env_r_align; /* r_align set in environment variable */ - MPI_Offset info_v_align;/* v_align set in MPI Info object */ - MPI_Offset info_r_align;/* r_align set in MPI Info object */ - MPI_Offset h_minfree; /* pad at the end of the header section */ - MPI_Offset v_minfree; /* pad at the end of the data section for fixed-size variables */ - MPI_Offset ibuf_size; /* packing buffer size for flushing noncontig - user buffer during wait */ - MPI_Offset xsz; /* size of this file header, <= var[0].begin */ - MPI_Offset begin_var; /* file offset of the first fixed-size variable, - if no fixed-sized variable, it is the offset - of first record variable. This value is also - the size of file header extent. */ - MPI_Offset begin_rec; /* file offset of the first 'record' */ - - MPI_Offset recsize; /* length of 'record': sum of single record sizes - of all the record variables */ - MPI_Offset numrecs; /* number of 'records' allocated */ - MPI_Offset put_size; /* amount of writes committed so far in bytes */ - MPI_Offset get_size; /* amount of reads committed so far in bytes */ - - MPI_Comm comm; /* MPI communicator */ - int rank; /* MPI rank of this process */ - int nprocs; /* number of MPI processes */ - MPI_Info mpiinfo; /* used MPI info object */ - MPI_File collective_fh; /* file handle for collective mode */ - MPI_File independent_fh; /* file handle for independent mode */ - - NC_dimarray dims; /* dimensions defined */ - NC_attrarray attrs; /* global attributes defined */ - NC_vararray vars; /* variables defined */ - - int hash_size_attr; /* hash table size for non-global attributes */ - - int maxGetReqID; /* max get request ID */ - int maxPutReqID; /* max put request ID */ - int numLeadGetReqs; /* number of pending lead get requests */ - int numLeadPutReqs; /* number of pending lead put requests */ - NC_lead_req *get_lead_list; /* list of lead nonblocking read requests */ - NC_lead_req *put_lead_list; /* list of lead nonblocking write requests */ - - int numGetReqs; /* number of pending nonblocking get requests */ - int numPutReqs; /* number of pending nonblocking put requests */ - NC_req *get_list; /* list of nonblocking read requests */ - NC_req *put_list; /* list of nonblocking write requests */ - - NC_buf *abuf; /* attached buffer, used by bput APIs */ - - char *path; /* file name */ - struct NC *old; /* contains the previous NC during redef. */ - - /* Below are used for intra-node aggregation */ - int num_aggrs_per_node; /* number of aggregators per compute node. Set - through a user hint. 0 to disable the - intra-node aggregation, -1 to let PnetCDF to - decide. This value must be the same among all - processes. + + int hdr_chunk; /* chunk size used when reading file header, which + * is read one chunk at a time + */ + int data_chunk; /* chunk size for moving data to higher offsets */ + int file_striping; /* hint for setting file striping configuration, + * available values are PNCIO_STRIPING_AUTO and + * PNCIO_STRIPING_INHERIT + */ + int striping_unit; /* file striping size */ + int striping_factor; /* file striping count */ + + MPI_Offset v_align; /* hint of file offset alignment for setting the + * beginning of fixed-size variable section + */ + MPI_Offset r_align; /* hint of file offset alignment for setting the + * beginning of record variable section */ - int my_aggr; /* rank ID of my aggregator */ - int num_nonaggrs; /* number of non-aggregators assigned */ - int *nonaggr_ranks; /* ranks of assigned non-aggregators */ -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) - double aggr_time; + MPI_Offset info_v_align; /* v_align set in MPI Info object */ + MPI_Offset info_r_align; /* r_align set in MPI Info object */ + MPI_Offset h_minfree; /* hint for padding at the end of the header + * section + */ + MPI_Offset v_minfree; /* hint for padding at the end of fixed-size + * variable section + */ + MPI_Offset ibuf_size; /* hint for an internal buffer size used to pack + * non-contiguous user write buffer into a + * contiguous one, when the buffer size is smaller + * than this value. Default PNC_DEFAULT_IBUF_SIZE + */ + + MPI_Offset xsz; /* size of this file header, <= var[0].begin */ + MPI_Offset begin_var; /* file offset of the 1st fixed-size variable, + * if no fixed-sized variable, it is the offset + * of first record variable. This value is also + * the size of file header extent. + */ + MPI_Offset begin_rec; /* file offset of the first record variable */ + + MPI_Offset fix_end; /* end offset of last fix-sized variable. This may + * be different from begin_rec, due to padding. + */ + + MPI_Offset recsize; /* length of a record: sum of single record sizes + * of all the record variables + */ + MPI_Offset numrecs; /* number of records allocated */ + MPI_Offset put_size; /* amount of writes committed so far in bytes */ + MPI_Offset get_size; /* amount of reads committed so far in bytes */ + + MPI_Comm comm; /* MPI communicator, duplicated */ + int rank; /* MPI rank of this process within comm */ + int nprocs; /* number of MPI processes in comm */ + + PNC_comm_attr comm_attr; /* attributes cached in communicator, comm. See + * PNC_comm_attr defined in include/dispatch.h + */ + + int NUMA_ID; /* this rank's NUMA node ID */ + MPI_Info info; /* hints used by MPI/GIO */ + + MPI_File mpio_fh_coll; /* MPI-IO file handle for collective mode */ + MPI_File mpio_fh_indep; /* MPI-IO file handle for independent mode */ +#if PNETCDF_DRIVER_GIO == 1 + GIO_File gio_fh; /* GIO file handler */ #endif + +/* TODO: remove unused */ +PNCIO_View file_view; + + int driver; /* I/O driver: PNC_DRIVER_GIO or + * PNC_DRIVER_MPIIO + */ + int fstype; /* file system type: PNCIO_FS_LUSTRE or + * PNCIO_FS_UFS + */ + + NC_dimarray dims; /* dimensions defined */ + NC_attrarray attrs; /* global attributes defined */ + NC_vararray vars; /* variables defined */ + + int hash_size_attr; /* hash table size for non-global attributes */ + + int maxGetReqID; /* max get request ID */ + int maxPutReqID; /* max put request ID */ + int numLeadGetReqs; /* no. pending lead get requests */ + int numLeadPutReqs; /* no. pending lead put requests */ + NC_lead_req *get_lead_list; /* list of lead nonblocking read requests */ + NC_lead_req *put_lead_list; /* list of lead nonblocking write requests */ + + int numGetReqs; /* no. pending nonblocking get requests */ + int numPutReqs; /* no. pending nonblocking put requests */ + NC_req *get_list; /* list of nonblocking read requests */ + NC_req *put_list; /* list of nonblocking write requests */ + + NC_buf *abuf; /* attached buffer, used by bput APIs */ + + const char *path; /* file name (duplicated from user's) */ + struct NC *old; /* NC metadata object containing the previous one being + * re-defined during ncmpi_redef() + */ + + int num_aggrs_per_node; /* Number of intra-node aggregators per compute + * node, set through a user hint. 0 to disable the + * intra-node aggregation, -1 to let PnetCDF to + * decide.This value must be the same among all + * processes. + */ }; +typedef struct bufferinfo { + NC *ncp; + MPI_Offset offset; /* current read/write offset in the file */ + char *base; /* beginning of read/write buffer */ + char *pos; /* current position in buffer */ + char *end; /* end position of buffer */ +} bufferinfo; + #define NC_readonly(ncp) fIsSet((ncp)->flags, NC_MODE_RDONLY) #define NC_IsNew(ncp) fIsSet((ncp)->flags, NC_MODE_CREATE) #define NC_indef(ncp) fIsSet((ncp)->flags, NC_MODE_DEF) @@ -474,9 +577,6 @@ struct NC { (NC_EMULTIDEFINE_FIRST >= (err) && (err) >= NC_EMULTIDEFINE_LAST) /* Begin defined in nc.c ----------------------------------------------------*/ -extern void -ncmpio_free_NC(NC *ncp); - extern int ncmpio_NC_check_vlen(NC_var *varp, MPI_Offset vlen_max); @@ -487,20 +587,6 @@ extern int ncmpio_NC_check_voffs(NC *ncp); /* Begin defined in ncmpio_header_get.c -------------------------------------*/ -typedef struct bufferinfo { - MPI_Comm comm; - MPI_File collective_fh; - MPI_Offset get_size; /* amount of file read n bytes so far */ - MPI_Offset offset; /* current read/write offset in the file */ - int chunk; /* chunk size for reading the header */ - int version; /* 1, 2, and 5 for CDF-1, 2, and 5 respectively */ - int safe_mode;/* 0: disabled, 1: enabled */ - int coll_mode;/* 0: independent, 1: collective */ - char *base; /* beginning of read/write buffer */ - char *pos; /* current position in buffer */ - char *end; /* end position of buffer */ -} bufferinfo; - extern MPI_Offset ncmpio_hdr_len_NC(const NC *ncp); @@ -515,23 +601,9 @@ extern int ncmpio_write_header(NC *ncp); /* Begin defined in ncmpio_sync.c -------------------------------------------*/ -extern int -ncmpio_file_sync(NC *ncp); - extern int ncmpio_write_numrecs(NC *ncp, MPI_Offset new_numrecs); -/* Begin defined in ncmpio_filetype.c ---------------------------------------*/ -extern int -ncmpio_filetype_create_vars(const NC* ncp, const NC_var* varp, - const MPI_Offset start[], const MPI_Offset count[], - const MPI_Offset stride[], MPI_Offset *offset, - MPI_Datatype *filetype, int *is_filetype_contig); - -extern int -ncmpio_file_set_view(const NC *ncp, MPI_File fh, MPI_Offset *offset, - MPI_Datatype filetype); - /* Begin defined in ncmpio_igetput.m4 ---------------------------------------*/ extern int ncmpio_abuf_malloc(NC *ncp, MPI_Offset nbytes, void **buf, int *abuf_index); @@ -607,17 +679,16 @@ ncmpio_inq_var_fill(NC_var *varp, void *fill_value); extern int ncmpio_fill_vars(NC *ncp); -/* Begin defined in ncmpio_nonblocking.c ------------------------------------*/ -extern int -ncmpio_getput_zero_req(NC *ncp, int rw_flag); - -/* Begin defined in ncmpio_close.c */ -extern int -ncmpio_close_files(NC *ncp, int doUnlink); +/* Begin defined in ncmpio_close.c ------------------------------------------*/ +extern void +ncmpio_free_NC(NC *ncp); /* Begin defined in ncmpio_utils.c ------------------------------------------*/ extern void -ncmpio_set_pnetcdf_hints(NC *ncp, MPI_Info user_info, MPI_Info info_used); +ncmpio_hint_extract(NC *ncp, MPI_Info info); + +extern void +ncmpio_hint_set(NC *ncp, MPI_Info info); extern int ncmpio_NC_check_name(const char *name, int file_ver); @@ -644,23 +715,53 @@ ncmpio_unpack_xbuf(int format, NC_var *varp, MPI_Offset bufcount, MPI_Datatype etype, MPI_Datatype imaptype, int need_convert, int need_swap, void *buf, void *xbuf); +extern int +ncmpio_calc_off(const NC *ncp, const NC_var *varp, const MPI_Offset *start, + MPI_Offset *offset); + +extern int +ncmpio_calc_start_end(const NC *ncp, const NC_var *varp, + const MPI_Offset *start, const MPI_Offset *count, + const MPI_Offset *stride, MPI_Offset *start_off, + MPI_Offset *end_off); + +extern +int ncmpio_type_contiguous(MPI_Offset count, MPI_Datatype *newType); + +extern int +ncmpio_type_create_hindexed(MPI_Offset count, MPI_Offset *off, MPI_Offset *len, + MPI_Datatype *newType); + /* Begin defined in ncmpio_file_io.c ----------------------------------------*/ + +extern MPI_Offset +ncmpio_file_read(NC *ncp, int coll_indep, void *buf, + PNCIO_View file_view, PNCIO_View buf_view); + +extern MPI_Offset +ncmpio_file_write(NC *ncp, int coll_indep, const void *buf, + PNCIO_View file_view, PNCIO_View buf_view); + extern int -ncmpio_read_write(NC *ncp, int rw_flag, int coll_indep, MPI_Offset offset, - MPI_Offset buf_count, MPI_Datatype buf_type, void *buf, - int buftype_is_contig); +ncmpio_file_close(NC *ncp); -/* Begin defined in ncmpio_intranode.c --------------------------------------*/ extern int -ncmpio_intra_node_aggr_init(NC *ncp); +ncmpio_file_delete(NC *ncp); extern int -ncmpio_intra_node_aggregation_nreqs(NC *ncp, int mode, int num_reqs, - NC_req *put_list, MPI_Offset newnumrecs); +ncmpio_file_sync(NC *ncp); + +/* Begin defined in ncmpio_intranode.c --------------------------------------*/ +extern int +ncmpio_ina_nreqs(NC *ncp, int mode, int num_reqs, NC_req *put_list, + MPI_Offset newnumrecs); +extern int +ncmpio_ina_req(NC *ncp, int mode, const NC_var *varp, const MPI_Offset *start, + const MPI_Offset *count, const MPI_Offset *stride, + MPI_Offset nbytes, void *buf); + +/* Begin defined in ncmpio_fstype.c -----------------------------------------*/ extern int -ncmpio_intra_node_aggregation(NC *ncp, int mode, NC_var *varp, - const MPI_Offset *start, const MPI_Offset *count, - const MPI_Offset *stride, MPI_Offset bufCount, - MPI_Datatype bufType, void *buf); +ncmpio_FileSysType(const char *filename); #endif /* H_NC */ diff --git a/src/drivers/ncmpio/ncmpio_attr.m4 b/src/drivers/ncmpio/ncmpio_attr.m4 index 5b969ded70..46c0b3733f 100644 --- a/src/drivers/ncmpio/ncmpio_attr.m4 +++ b/src/drivers/ncmpio/ncmpio_attr.m4 @@ -477,7 +477,7 @@ ncmpio_rename_att(void *ncdp, err_check: if (nname != NULL) NCI_Free(nname); - if (ncp->safe_mode && ncp->nprocs > 1) { + if (fIsSet(ncp->flags, NC_MODE_SAFE) && ncp->nprocs > 1) { int minE, mpireturn; /* check error code across processes */ @@ -597,7 +597,7 @@ ncmpio_copy_att(void *ncdp_in, } err_check: - if (ncp_out->safe_mode && ncp_out->nprocs > 1) { + if (fIsSet(ncp_out->flags, NC_MODE_SAFE) && ncp_out->nprocs > 1) { int minE, mpireturn; /* check the error code across processes */ @@ -710,7 +710,7 @@ ncmpio_del_att(void *ncdp, err_check: if (nname != NULL) NCI_Free(nname); - if (ncp->safe_mode && ncp->nprocs > 1) { + if (fIsSet(ncp->flags, NC_MODE_SAFE) && ncp->nprocs > 1) { int minE, mpireturn; /* find min error code across processes */ @@ -1044,7 +1044,7 @@ ncmpio_put_att(void *ncdp, } err_check: - if (ncp->safe_mode && ncp->nprocs > 1) { + if (fIsSet(ncp->flags, NC_MODE_SAFE) && ncp->nprocs > 1) { /* check the error code across processes */ int minE, mpireturn; diff --git a/src/drivers/ncmpio/ncmpio_close.c b/src/drivers/ncmpio/ncmpio_close.c index ec79088e81..ab46cc1b62 100644 --- a/src/drivers/ncmpio/ncmpio_close.c +++ b/src/drivers/ncmpio/ncmpio_close.c @@ -28,11 +28,11 @@ #include #include #include "ncmpio_NC.h" -#ifdef ENABLE_SUBFILING +#if PNETCDF_SUBFILING == 1 #include "ncmpio_subfile.h" #endif -/*----< ncmpio_free_NC() >----------------------------------------------------*/ +/*----< ncmpio_free_NC() >---------------------------------------------------*/ void ncmpio_free_NC(NC *ncp) { @@ -42,56 +42,20 @@ ncmpio_free_NC(NC *ncp) ncmpio_free_NC_attrarray(&ncp->attrs); ncmpio_free_NC_vararray(&ncp->vars); - /* The only case that ncp->mpiinfo is MPI_INFO_NULL is when exiting endef - * from a redef. All other cases reaching here are from ncmpi_close, in - * which case ncp->mpiinfo is never MPI_INFO_NULL. + /* The only case that ncp->info is MPI_INFO_NULL is when exiting + * enddef() from a redef(). All other cases reaching here are from + * ncmpi_close, in which case ncp->info is never MPI_INFO_NULL. */ - if (ncp->mpiinfo != MPI_INFO_NULL) MPI_Info_free(&ncp->mpiinfo); + if (ncp->info != MPI_INFO_NULL) MPI_Info_free(&ncp->info); if (ncp->get_list != NULL) NCI_Free(ncp->get_list); if (ncp->put_list != NULL) NCI_Free(ncp->put_list); if (ncp->abuf != NULL) NCI_Free(ncp->abuf); - if (ncp->path != NULL) NCI_Free(ncp->path); - if (ncp->nonaggr_ranks != NULL) NCI_Free(ncp->nonaggr_ranks); NCI_Free(ncp); } -/*----< ncmpio_close_files() >-----------------------------------------------*/ -int -ncmpio_close_files(NC *ncp, int doUnlink) { - char *mpi_name; - int mpireturn; - - assert(ncp != NULL); /* this should never occur */ - - if (ncp->independent_fh != MPI_FILE_NULL) { - TRACE_IO(MPI_File_close, (&ncp->independent_fh)); - if (mpireturn != MPI_SUCCESS) - return ncmpii_error_mpi2nc(mpireturn, mpi_name); - } - - if (ncp->nprocs > 1 && ncp->collective_fh != MPI_FILE_NULL) { - TRACE_IO(MPI_File_close, (&ncp->collective_fh)); - if (mpireturn != MPI_SUCCESS) - return ncmpii_error_mpi2nc(mpireturn, mpi_name); - } - - if (doUnlink) { - /* called from ncmpi_abort, if the file is being created and is still - * in define mode, the file is deleted */ - if (ncp->rank == 0) { - TRACE_IO(MPI_File_delete, ((char *)ncp->path, ncp->mpiinfo)); - if (mpireturn != MPI_SUCCESS) - return ncmpii_error_mpi2nc(mpireturn, mpi_name); - } - if (ncp->nprocs > 1) - MPI_Barrier(ncp->comm); - } - return NC_NOERR; -} - -/*----< ncmpio_close() >------------------------------------------------------*/ +/*----< ncmpio_close() >-----------------------------------------------------*/ /* This function is collective */ int ncmpio_close(void *ncdp) @@ -121,7 +85,7 @@ ncmpio_close(void *ncdp) /* if entering this function in collective data mode, we do not have to * update header in file, as file header is always up-to-date */ -#ifdef ENABLE_SUBFILING +#if PNETCDF_SUBFILING == 1 /* ncmpio__enddef() will update ncp->num_subfiles */ /* TODO: should check ncid_sf? */ /* if the file has subfiles, close them first */ @@ -159,8 +123,8 @@ ncmpio_close(void *ncdp) } #endif - /* calling MPI_File_close() */ - err = ncmpio_close_files(ncp, 0); + /* close the file */ + err = ncmpio_file_close(ncp); if (status == NC_NOERR) status = err; /* file is open for write and no variable has been defined */ @@ -169,7 +133,7 @@ ncmpio_close(void *ncdp) if (ncp->nprocs > 1) MPI_Barrier(ncp->comm); if (ncp->rank == 0) { - /* ignore all errors, as unexpected file size if not a fatal error */ + /* ignore all errors, as unexpected file size is not fatal */ #ifdef HAVE_TRUNCATE /* when calling POSIX I/O, remove file type prefix from file name */ char *path = ncmpii_remove_file_system_type_prefix(ncp->path); @@ -187,7 +151,13 @@ ncmpio_close(void *ncdp) #else MPI_File fh; int mpireturn; +#ifdef MPICH_VERSION + /* MPICH recognizes file system type acronym prefixed to the file name */ TRACE_IO(MPI_File_open, (MPI_COMM_SELF, ncp->path, MPI_MODE_RDWR, MPI_INFO_NULL, &fh)); +#else + char *path = ncmpii_remove_file_system_type_prefix(ncp->path); + TRACE_IO(MPI_File_open, (MPI_COMM_SELF, path, MPI_MODE_RDWR, MPI_INFO_NULL, &fh)); +#endif if (mpireturn == MPI_SUCCESS) { /* obtain file size */ MPI_Offset *file_size; @@ -219,6 +189,10 @@ ncmpio_close(void *ncdp) if (ncp->nprocs > 1) MPI_Barrier(ncp->comm); } + /* collectively return the same error code */ + if (ncp->nprocs > 1) + MPI_Allreduce(MPI_IN_PLACE, &status, 1, MPI_INT, MPI_MIN, ncp->comm); + /* free up space occupied by the header metadata */ ncmpio_free_NC(ncp); diff --git a/src/drivers/ncmpio/ncmpio_create.c b/src/drivers/ncmpio/ncmpio_create.c index e5cee83d3f..5de0bc876c 100644 --- a/src/drivers/ncmpio/ncmpio_create.c +++ b/src/drivers/ncmpio/ncmpio_create.c @@ -8,7 +8,6 @@ * This file implements the corresponding APIs defined in src/dispatchers/file.c * * ncmpi_create() : dispatcher->create() - * ncmpi_open() : dispatcher->open() */ #ifdef HAVE_CONFIG_H @@ -24,36 +23,124 @@ #include /* lstat(), open() */ #include /* lstat(), open() */ #include /* lstat(), access(), unlink(), open(), close() */ -#include /* open() */ +#include /* open(), O_CREAT, O_RDWR, O_RDONLY */ #endif +#include /* dirname() */ +#include #include +#if PNETCDF_DRIVER_GIO == 1 +#include +#endif + #include #include #include "ncmpio_NC.h" + +#ifdef HAVE_LUSTRE +/* /usr/include/lustre/lustreapi.h + * /usr/include/linux/lustre/lustre_user.h + */ +#include + +// #define PNETCDF_LUSTRE_DEBUG + +#define PATTERN_STR(pattern, int_str) ( \ + (pattern == LLAPI_LAYOUT_DEFAULT) ? "LLAPI_LAYOUT_DEFAULT" : \ + (pattern == LLAPI_LAYOUT_RAID0) ? "LLAPI_LAYOUT_RAID0" : \ + (pattern == LLAPI_LAYOUT_WIDE) ? "LLAPI_LAYOUT_WIDE" : \ + (pattern == LLAPI_LAYOUT_MDT) ? "LLAPI_LAYOUT_MDT" : \ + (pattern == LLAPI_LAYOUT_OVERSTRIPING) ? "LLAPI_LAYOUT_OVERSTRIPING" : \ + (pattern == LLAPI_LAYOUT_SPECIFIC) ? "LLAPI_LAYOUT_SPECIFIC" : \ + int_str) + +/*----< lustre_get_striping() >----------------------------------------------*/ +static +void lustre_get_striping(const char *path, + uint64_t *stripe_count, + uint64_t *stripe_size) +{ + char *dirc, *dname; + int err, fd; + struct llapi_layout *layout; + + /* Retrieve the file striping settings from the parent folder. */ + dirc = NCI_Strdup(path); + dname = dirname(dirc); /* folder name */ + + fd = open(dname, O_RDONLY, PNC_PERM); + + layout = llapi_layout_get_by_fd(fd, LLAPI_LAYOUT_GET_COPY); + if (layout == NULL) { +#ifdef PNETCDF_LUSTRE_DEBUG + fprintf(stderr,"Error at %s (%d) llapi_layout_get_by_fd() fails\n", + __FILE__, __LINE__); +#endif + goto err_out; + } + + if (stripe_count != NULL && *stripe_count == 0) { + /* obtain file striping count */ + err = llapi_layout_stripe_count_get(layout, stripe_count); + if (err != 0) { +#ifdef PNETCDF_LUSTRE_DEBUG + char int_str[32]; + snprintf(int_str, 32, "%lu", *stripe_count); + fprintf(stderr,"Error at %s (%d) llapi_layout_stripe_count_get() fails to get stripe count %s\n", + __FILE__, __LINE__, PATTERN_STR(*stripe_count, int_str)); +#endif + goto err_out; + } + } + + if (stripe_size != NULL && *stripe_size == 0) { + /* obtain file striping unit size */ + err = llapi_layout_stripe_size_get(layout, stripe_size); + if (err != 0) { +#ifdef PNETCDF_LUSTRE_DEBUG + char int_str[32]; + snprintf(int_str, 32, "%lu", *stripe_size); + fprintf(stderr,"Error at %s (%d) llapi_layout_stripe_size_get() fails to get stripe size %s\n", + __FILE__,__LINE__, PATTERN_STR(*stripe_size, int_str)); +#endif + goto err_out; + } + } + +err_out: + if (layout != NULL) llapi_layout_free(layout); + + close(fd); +} +#endif + /*----< ncmpio_create() >----------------------------------------------------*/ int -ncmpio_create(MPI_Comm comm, - const char *path, - int cmode, - int ncid, - MPI_Info user_info, /* user's and env info combined */ - void **ncpp) +ncmpio_create(MPI_Comm comm, + const char *path, + int cmode, + int ncid, + int env_mode, + MPI_Info user_info, /* user's and env info combined */ + PNC_comm_attr comm_attr, /* node IDs and INA metadata */ + void **ncpp) /* OUT */ { - char *env_str, *filename, *mpi_name; - int rank, nprocs, mpiomode, err, mpireturn, default_format, file_exist=1; - int use_trunc=1; - MPI_File fh; - MPI_Info info_used; + char *filename, value[MPI_MAX_INFO_VAL + 1], *mpi_name; + int rank, nprocs, mpi_amode, err, mpireturn, default_format, file_exist=1; + int use_trunc=1, flag, striping_info[2]; + MPI_File fh=MPI_FILE_NULL; NC *ncp=NULL; *ncpp = NULL; + MPI_Comm_rank(comm, &rank); + MPI_Comm_size(comm, &nprocs); + /* Note path's validity and cmode consistency have been checked in - * ncmpi_create() in src/dispatchers/file.c and - * path consistency will be done in MPI_File_open */ + * ncmpi_create() in src/dispatchers/file.c. + */ /* First, check whether cmode is valid or supported ---------------------*/ @@ -65,26 +152,100 @@ ncmpio_create(MPI_Comm comm, /* Check cmode for other illegal flags already done in dispatcher layer */ - /* Get default format, in case cmode does not include either - * NC_64BIT_OFFSET or NC_64BIT_DATA */ + /* Get default format, in case cmode does not include either NC_64BIT_DATA + * or NC_64BIT_OFFSET. + */ ncmpi_inq_default_format(&default_format); - /* Handle file clobber --------------------------------------------------*/ - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &nprocs); + /* allocate buffer for header object NC and initialize its contents */ + ncp = (NC*) NCI_Calloc(1, sizeof(NC)); + if (ncp == NULL) DEBUG_RETURN_ERROR(NC_ENOMEM) + + *ncpp = (void*)ncp; + + ncp->ncid = ncid; + ncp->comm = comm; /* reuse comm duplicated in dispatch layer */ + ncp->rank = rank; + ncp->nprocs = nprocs; + + /* Extract hints from user_info. + * + * Three PnetCDF hints must and have been be extracted before calling + * MPI_File_open() or GIO_open(). + * + nc_driver: whether to use MPI-IO or GIO driver. + * + nc_num_aggrs_per_node: number of processes per node to be the INA + * aggregators, which determines the MPI communicator to be passed + * to GIO_open() and MPI_File_open(). + * + nc_striping: whether to inherit parent folder's striping. + * + * Four PnetCDF hints must be extracted before ncmpio_enddef(). They are + * used when creating dimensions, attributes, and variables. + * + nc_hash_size_dim: hash table size for dimensions + * + nc_hash_size_var: hash table size for variables + * + nc_hash_size_gattr: hash table size for global attributes + * + nc_hash_size_vattr: hash table size for non-global attributes + * + * The remaining PnetCDF hints are used at ncmpio_enddef() and after. + * + nc_var_align_size ncp->info_v_align + * + nc_header_align_size ncp->info_v_align + * + nc_record_align_size ncp->info_r_align + * + nc_header_read_chunk_size ncp->hdr_chunk + * + nc_in_place_swap fSet(ncp->flags, NC_MODE_SWAP_ON) + * + nc_ibuf_size ncp->ibuf_size + * + pnetcdf_subfiling ncp->subfile_mode/num_subfiles + * + nc_data_move_chunk_size ncp->data_chunk + * + * Note after a call to MPI_File_open() returns, any hints that are not + * used by MPI_File_open() will be discarded, which includes all PnetCDF + * hints. Therefore, we must at first call MPI_File_get_info() to obtain + * an info object containing all the hints used by MPI-IO, and then add + * the PnetCDF hints into the info object (ncp->info). So ncp->info + * can be used in ncmpi_inq_file_info() to return all hints to users. + * + * MPI_File_open() will add new hints, such as those related to file + * striping and I/O aggregators. + * + cb_nodes + * + striping_unit + * + striping_factor + * + start_iodevice + * + * GIO_open() will add new hints, such as those related to file striping + * and I/O aggregators. + * + cb_nodes + * + cb_node_list + * + striping_unit + * + striping_factor + * + start_iodevice + * + lustre_overstriping_ratio + * + lustre_num_osts + */ + ncmpio_hint_extract(ncp, user_info); - mpiomode = MPI_MODE_RDWR | MPI_MODE_CREATE; + if (rank == 0) + /* Check file system type. If the given file does not exist, check its + * parent folder. This info will be used to set file striping hints. + */ + ncp->fstype = ncmpio_FileSysType(path); + + /* Setting file open mode in mpi_amode which may later be needed in + * ncmpi_begin_indep_data() to open file in the independent data mode. + */ + mpi_amode = MPI_MODE_RDWR | MPI_MODE_CREATE; - /* remove the file system type prefix name if there is any. For example, + /* Remove the file system type prefix name if there is any. For example, * when path = "lustre:/home/foo/testfile.nc", remove "lustre:" to make * filename pointing to "/home/foo/testfile.nc", so it can be used in POSIX - * access() below + * access() below. */ filename = ncmpii_remove_file_system_type_prefix(path); - /* Check if the file already exists, if lstat() or access() is available */ + /* In case of file clobber mode, we first check if the file already exists, + * through a call to lstat() or access() if they are is available. If not, + * we call MPI_File_open() to try opening the file. If the file exists, we + * will cal MPI_File_set_size() to truncate the file. + */ #ifdef HAVE_LSTAT - /* call lstat() to check the file if exists and if is a symbolic link */ + /* Call lstat() to check the file if exists and if is a symbolic link */ if (rank == 0) { struct stat st_buf; st_buf.st_mode = 0; @@ -92,21 +253,22 @@ ncmpio_create(MPI_Comm comm, if (lstat(filename, &st_buf) == -1) file_exist = 0; errno = 0; /* reset errno */ - /* If the file is a regular file, not a symbolic link, then we can - * delete the file first and later create it when calling - * MPI_File_open() with MPI_MODE_CREATE. It is OK to delete and then - * re-create the file if the file is a regular file. If there are other - * files symbolically linked to this file, then their links will still - * point to this file after it is re-created. + /* If the file is a regular file, not a symbolic link, then we delete + * the file first and later create it (by calling MPI_File_open() with + * MPI_MODE_CREATE). In this case, it is faster to delete and re-create + * the file, as truncating a file to zero size is more expensive. * * If the file is a symbolic link, then we cannot delete the file, as - * the link will be gone. + * the link will be gone. If the file is deleted and there are other + * files symbolically linked to this file, then their links will become + * invalid. */ if (S_ISREG(st_buf.st_mode)) use_trunc = 0; } #elif defined HAVE_ACCESS - /* if access() is available, use it to check whether file already exists - * rank 0 calls access() and broadcasts file_exist */ + /* If access() is available, use it to check whether file already exists, + * by having rank 0 to call access() and broadcast file_exist. + */ if (rank == 0) { if (access(filename, F_OK) == -1) file_exist = 0; errno = 0; /* reset errno */ @@ -114,53 +276,89 @@ ncmpio_create(MPI_Comm comm, #endif if (fIsSet(cmode, NC_NOCLOBBER)) { - /* check if file exists: NC_EEXIST is returned if the file already - * exists and NC_NOCLOBBER mode is used in ncmpi_create */ + /* Error NC_EEXIST will be returned, if the file already exists. */ #ifdef HAVE_ACCESS - if (nprocs > 1) - TRACE_COMM(MPI_Bcast)(&file_exist, 1, MPI_INT, 0, comm); - if (file_exist) DEBUG_RETURN_ERROR(NC_EEXIST) + if (nprocs > 1) { + int msg[2] = {file_exist, ncp->fstype}; + TRACE_COMM(MPI_Bcast)(msg, 2, MPI_INT, 0, comm); + file_exist = msg[0]; + ncp->fstype = msg[1]; + } + if (file_exist) { + NCI_Free(ncp); + DEBUG_RETURN_ERROR(NC_EEXIST) + } #else - /* add MPI_MODE_EXCL mode for MPI_File_open to check file existence */ - fSet(mpiomode, MPI_MODE_EXCL); + if (nprocs > 1) + TRACE_COMM(MPI_Bcast)(&ncp->fstype, 1, MPI_INT, 0, comm); + + /* Add MPI_MODE_EXCL mode for MPI_File_open, so it can error out, if + * the file exists. + */ + fSet(mpi_amode, MPI_MODE_EXCL); errno = 0; /* reset errno, as MPI_File_open may change it */ #endif } - else { /* NC_CLOBBER is the default mode in create */ - /* rank 0 truncates or deletes the file and ignores error code. - * Note calling MPI_File_set_size is expensive as it calls truncate() + else { + /* NC_CLOBBER is the default mode in ncmpi_create(). Below, rank 0 + * truncates or deletes the file and ignores error code. Note in some + * implementation of MPI-IO, calling MPI_File_set_size() can be very + * expensive as it may call truncate() by all ranks. */ err = NC_NOERR; if (rank == 0 && file_exist) { if (!use_trunc) { /* delete the file */ #ifdef HAVE_UNLINK - /* unlink() is likely faster then truncate(), but may be still - * expensive + /* unlink() is likely faster then truncate(). However, unlink() + * can be expensive when the file size is large. For example, + * it took 1.1061 seconds to delete a file of size 27.72 GiB + * on Perlmutter at NERSC. */ err = unlink(filename); if (err < 0 && errno != ENOENT) /* ignore ENOENT: file not exist */ - DEBUG_ASSIGN_ERROR(err, NC_EFILE) /* other error */ + DEBUG_ASSIGN_ERROR(err, NC_EFILE) /* report other error */ else err = NC_NOERR; #else - err = NC_NOERR; - TRACE_IO(MPI_File_delete, ((char *)path, MPI_INFO_NULL)); - if (mpireturn != MPI_SUCCESS) { - int errorclass; - MPI_Error_class(mpireturn, &errorclass); - if (errorclass != MPI_ERR_NO_SUCH_FILE) - /* ignore file not exist */ - err = ncmpii_error_mpi2nc(mpireturn, mpi_name); + if (ncp->driver == PNC_DRIVER_MPIIO) { + err = NC_NOERR; +#ifdef MPICH_VERSION + /* MPICH recognizes file system type acronym prefixed to + * the file name. Other MPI implementations may not. + */ + TRACE_IO(MPI_File_delete, (path, MPI_INFO_NULL)); +#else + TRACE_IO(MPI_File_delete, (filename, MPI_INFO_NULL)); +#endif + if (mpireturn != MPI_SUCCESS) { + int errorclass; + MPI_Error_class(mpireturn, &errorclass); + if (errorclass != MPI_ERR_NO_SUCH_FILE) + /* ignore file not exist */ + err = ncmpii_error_mpi2nc(mpireturn, mpi_name); + } } +#if PNETCDF_DRIVER_GIO == 1 + else if (ncp->driver == PNC_DRIVER_GIO) + err = GIO_delete(path); +#endif + else + err = NC_EDRIVER; #endif } - else { /* file is not a regular file, truncate it to zero size */ + else { + /* If file is not a regular file (e.g. a symbolic link), we + * cannot delete it and must truncate it to zero size. In this + * case, file open mode needs to remove MPI_MODE_CREATE. + */ + mpi_amode = MPI_MODE_RDWR; + #ifdef HAVE_TRUNCATE - err = truncate(filename, 0); /* can be expensive */ + err = truncate(filename, 0); /* truncate() may be expensive */ if (err < 0 && errno != ENOENT) /* ignore ENOENT: file not exist */ - DEBUG_ASSIGN_ERROR(err, NC_EFILE) /* other error */ + DEBUG_ASSIGN_ERROR(err, NC_EFILE) /* report other error */ else err = NC_NOERR; #elif defined HAVE_OPEN @@ -173,82 +371,93 @@ ncmpio_create(MPI_Comm comm, DEBUG_ASSIGN_ERROR(err, NC_EFILE) } #else - /* call MPI_File_set_size() to truncate the file. Note this can - * be expensive. + /* When all POSIX system calls are not available, the last + * resort is to call MPI_File_set_size() to truncate the file. + * Note in some ROMIO versions that make all processes to call + * truncate(), this option may be expensive. */ - err = NC_NOERR; - TRACE_IO(MPI_File_open, (MPI_COMM_SELF, (char *)path, MPI_MODE_RDWR, MPI_INFO_NULL, &fh)); - if (mpireturn != MPI_SUCCESS) { - int errorclass; - MPI_Error_class(mpireturn, &errorclass); - err = ncmpii_error_mpi2nc(mpireturn, mpi_name); - } - else { - TRACE_IO(MPI_File_set_size, (fh, 0)); /* can be expensive */ + if (ncp->driver == PNC_DRIVER_MPIIO) { + err = NC_NOERR; +#ifdef MPICH_VERSION + /* MPICH recognizes file system type acronym prefixed to + * the file name. Other MPI implementations may not. + */ + TRACE_IO(MPI_File_open, (MPI_COMM_SELF, path, + MPI_MODE_RDWR, MPI_INFO_NULL, &fh)); +#else + TRACE_IO(MPI_File_open, (MPI_COMM_SELF, filename, + MPI_MODE_RDWR, MPI_INFO_NULL, &fh)); +#endif if (mpireturn != MPI_SUCCESS) { int errorclass; MPI_Error_class(mpireturn, &errorclass); err = ncmpii_error_mpi2nc(mpireturn, mpi_name); } else { - TRACE_IO(MPI_File_close, (&fh)); + /* MPI_File_set_size() can be expensive */ + TRACE_IO(MPI_File_set_size, (fh, 0)); if (mpireturn != MPI_SUCCESS) { int errorclass; MPI_Error_class(mpireturn, &errorclass); err = ncmpii_error_mpi2nc(mpireturn, mpi_name); } + else { + TRACE_IO(MPI_File_close, (&fh)); + if (mpireturn != MPI_SUCCESS) { + int errorclass; + MPI_Error_class(mpireturn, &errorclass); + err = ncmpii_error_mpi2nc(mpireturn, mpi_name); + } + } } } +#if PNETCDF_DRIVER_GIO == 1 + else if (ncp->driver == PNC_DRIVER_GIO) { + GIO_File gio_fh; + err = GIO_open(MPI_COMM_SELF, path, O_RDWR, MPI_INFO_NULL, + &gio_fh); + if (err == NC_NOERR) + GIO_set_size(gio_fh, 0); /* may be expensive */ + else + GIO_close(&gio_fh); + } +#endif + else + err = NC_EDRIVER; #endif } if (errno == ENOENT) errno = 0; /* reset errno */ } - /* all processes must wait here until file deletion is completed */ - if (nprocs > 1) - TRACE_COMM(MPI_Bcast)(&err, 1, MPI_INT, 0, comm); + /* All processes must wait here until file clobbering by root process + * is completed. Note mpi_amode may be modified by removing + * MPI_MODE_CREATE when the file to be clobbered is a symbolic link. + */ + if (nprocs > 1) { + int msg[3] = {err, mpi_amode, ncp->fstype}; + TRACE_COMM(MPI_Bcast)(&msg, 3, MPI_INT, 0, comm); + err = msg[0]; + mpi_amode = msg[1]; + ncp->fstype = msg[2]; + } if (err != NC_NOERR) return err; } + /* Now file has been clobbered, i.e. deleted if it is not a symbolic link. + * If it is a symbolic link, it now has been truncated to zero size. + */ - /* create file collectively -------------------------------------------- */ - TRACE_IO(MPI_File_open, (comm, (char *)path, mpiomode, user_info, &fh)); - if (mpireturn != MPI_SUCCESS) { -#ifndef HAVE_ACCESS - if (fIsSet(cmode, NC_NOCLOBBER)) { - /* This is the case when NC_NOCLOBBER is used in file creation and - * function access() is not available. MPI_MODE_EXCL is set in open - * mode. When MPI_MODE_EXCL is used and the file already exists, - * MPI-IO should return error class MPI_ERR_FILE_EXISTS. But, some - * MPI-IO implementations (older ROMIO) do not correctly return - * this error class. In this case, we can do the followings: check - * errno to see if it set to EEXIST. Note usually rank 0 makes the - * file open call and can be the only one having errno set. - */ - if (nprocs > 1) - TRACE_COMM(MPI_Bcast)(&errno, 1, MPI_INT, 0, comm); - if (errno == EEXIST) DEBUG_RETURN_ERROR(NC_EEXIST) - } -#endif - return ncmpii_error_mpi2nc(mpireturn, mpi_name); - /* for NC_NOCLOBBER, MPI_MODE_EXCL was added to mpiomode. If the file - * already exists, MPI-IO should return error class MPI_ERR_FILE_EXISTS - * which PnetCDF will return error code NC_EEXIST. This is checked - * inside of ncmpii_error_mpi2nc() - */ - } - else - /* reset errno, as MPI_File_open may change it, even for MPI_SUCCESS */ - errno = 0; + ncp->path = path; /* reuse path duplicated in dispatch layer */ + ncp->mpi_amode = mpi_amode; + ncp->info = MPI_INFO_NULL; - /* get the I/O hints used/modified by MPI-IO */ - TRACE_IO(MPI_File_get_info, (fh, &info_used)); - if (mpireturn != MPI_SUCCESS) - return ncmpii_error_mpi2nc(mpireturn, mpi_name); +#if PNETCDF_DRIVER_GIO == 1 + ncp->gio_fh = NULL; /* when using GIO driver */ +#endif - /* Now the file has been successfully created, allocate/set NC object */ + /* For file create, ignore NC_NOWRITE if set in cmode argument. */ + ncp->nc_amode = cmode | NC_WRITE; - /* allocate buffer for header object NC and initialize its contents */ - ncp = (NC*) NCI_Calloc(1, sizeof(NC)); - if (ncp == NULL) DEBUG_RETURN_ERROR(NC_ENOMEM) + ncp->mpio_fh_coll = MPI_FILE_NULL; + ncp->mpio_fh_indep = MPI_FILE_NULL; /* set the file format version based on the create mode, cmode */ if (fIsSet(cmode, NC_64BIT_DATA)) ncp->format = 5; @@ -259,6 +468,7 @@ ncmpio_create(MPI_Comm comm, else ncp->format = 1; } + /* indicate this is from ncmpi_create */ fSet(ncp->flags, NC_MODE_CREATE); /* create automatically enter write mode */ fClr(ncp->flags, NC_MODE_RDONLY); @@ -267,70 +477,336 @@ ncmpio_create(MPI_Comm comm, /* PnetCDF default mode is no fill */ fClr(ncp->flags, NC_MODE_FILL); - ncp->ncid = ncid; + /* incorporate modes set in environment variables */ + fSet(ncp->flags, env_mode); - /* chunk size for reading header, set to default before check hints */ - ncp->chunk = PNC_DEFAULT_CHUNKSIZE; + /* initialize unlimited_id as no unlimited dimension yet defined */ + ncp->dims.unlimited_id = -1; - /* calculate the true header size (not-yet aligned) - * No need to do this now. - * ncp->xsz = ncmpio_hdr_len_NC(ncp); + /* comm_attr has been constructed at the dispatchers. + * + * comm_attr.NUMA_IDs[] stores the NUMA compute node IDs of all MPI ranks + * of this comm (the MPI communicator passed from the user application). It + * is a keyval attribute cached in the communicator, comm. See how + * comm_attr.NUMA_IDs[] was constructed in src/dispatchers/file.c. + * + * When the intra-node aggregation (INA) is enabled, commm_attr also stores + * an intra-node communicator (containing processes belonging to its INA + * group) and an inter-node communicator (containing all the INA + * aggregators). + * + * The inter-node communicator will be used to call GIO_open() or + * MPI_File_open(), which means only the INA aggregators will perform + * collective I/O to the file. */ + ncp->comm_attr = comm_attr; - /* initialize unlimited_id as no unlimited dimension yet defined */ - ncp->dims.unlimited_id = -1; +#if PNETCDF_DEBUG_MODE == 1 + if (ncp->num_aggrs_per_node == 0) + assert(comm_attr.ina_intra_comm == MPI_COMM_NULL); + else + assert(comm_attr.ina_intra_comm != MPI_COMM_NULL); +#endif - /* buffer to pack noncontiguous user buffers when calling wait() */ - ncp->ibuf_size = PNC_DEFAULT_IBUF_SIZE; + /* When the total number of aggregators >= number of processes, disable + * intra-node aggregation. + */ + if (ncp->num_aggrs_per_node * comm_attr.num_NUMAs >= ncp->nprocs) + ncp->num_aggrs_per_node = 0; - /* Extract PnetCDF specific I/O hints from user_info and set default hint - * values into info_used. Note some MPI libraries, such as MPICH 3.3.1 and - * priors fail to preserve user hints that are not recognized by the MPI - * libraries. + /* The value of ncp->num_aggrs_per_node = 0, or > 0 is an indicator of + * whether the INA feature is disabled or enabled globally. Thus, + * ncp->num_aggrs_per_node is always consistent among all processes. */ - ncmpio_set_pnetcdf_hints(ncp, user_info, info_used); - - /* For file create, ignore if NC_NOWRITE set in cmode by user */ - ncp->iomode = cmode | NC_WRITE; - ncp->comm = comm; /* reuse comm duplicated in dispatch layer */ - ncp->mpiinfo = info_used; /* is not MPI_INFO_NULL */ - ncp->mpiomode = mpiomode; - ncp->rank = rank; - ncp->nprocs = nprocs; - ncp->collective_fh = fh; - ncp->independent_fh = (nprocs > 1) ? MPI_FILE_NULL : fh; - ncp->path = (char*) NCI_Malloc(strlen(path) + 1); - strcpy(ncp->path, path); - -#ifdef PNETCDF_DEBUG - /* PNETCDF_DEBUG is set at configure time, which will be overwritten by - * the run-time environment variable PNETCDF_SAFE_MODE */ - ncp->safe_mode = 1; + if (ncp->num_aggrs_per_node > 0) { +#if PNETCDF_DEBUG_MODE == 1 + if (comm_attr.is_ina_aggr) /* INA aggregator */ + assert(comm_attr.ina_inter_comm != MPI_COMM_NULL); + else + assert(comm_attr.ina_inter_comm == MPI_COMM_NULL); #endif - /* If environment variable PNETCDF_SAFE_MODE is set to 1, then we perform - * a strict consistent test, i.e. arguments used in def_dim/def_var APIs - */ - if ((env_str = getenv("PNETCDF_SAFE_MODE")) != NULL) { - if (*env_str == '0') ncp->safe_mode = 0; - else ncp->safe_mode = 1; - /* if PNETCDF_SAFE_MODE is set but without a value, *env_str can - * be '\0' (null character). In this case, safe_mode is enabled */ + + /* As non-aggregators will call MPI_File_open() or GIO_open(), we now + * can replace comm with ina_inter_comm. Note 'comm' is a local + * variable of this subroutine. + */ + comm = comm_attr.ina_inter_comm; + + /* For non-INA aggregators, comm_attr.ina_inter_comm is MPI_COMM_NULL. + * Because the codes below till label 'after_open' are for INA + * aggregators to open the file and obtain a file handler, non-INA + * aggregators do not participate and thus do not make use of comm. + */ + if (comm == MPI_COMM_NULL) { + if (user_info != MPI_INFO_NULL) + MPI_Info_dup(user_info, &ncp->info); + goto after_open; /* non-INA aggregators skip to 'after_open' */ + } + +#if PNETCDF_DEBUG_MODE == 1 + assert(comm_attr.is_ina_aggr); /* INA aggregator */ +#endif + + /* As non-aggregators will not perform any file I/O, we now can replace + * nprocs with ina_inter_comm's size. Note 'nprocs' is a local variable + * of this subroutine. + */ + MPI_Comm_size(comm, &nprocs); } - /* determine whether to enable intra-node aggregation and set up all - * intra-node aggregation metadata. - * ncp->num_aggrs_per_node = 0, or non-zero indicates whether this feature - * is enabled globally for all processes. - * ncp->my_aggr = -1 or >= 0 indicates whether aggregation is effectively - * enabled for the aggregation group of this process. + /* create file collectively -------------------------------------------- */ + if (ncp->driver == PNC_DRIVER_MPIIO) { + /* If hint nc_file_striping is set to "auto" and hint striping_factor + * is not set by the user, then set hint striping_factor to + * ncp->comm_attr.num_NUMAs. + */ + if (ncp->file_striping == PNCIO_STRIPING_AUTO) { + int striping_factor=0; + if (user_info != MPI_INFO_NULL) { + MPI_Info_get(user_info, "striping_factor", MPI_MAX_INFO_VAL-1, + value, &flag); + if (flag) + striping_factor = atoi(value); + } + if (striping_factor == 0) { + sprintf(value, "%d", ncp->comm_attr.num_NUMAs); + MPI_Info_set(user_info, "striping_factor", value); + } + } + else { /* ncp->file_striping == PNCIO_STRIPING_INHERIT */ + if (user_info != MPI_INFO_NULL) { + striping_info[0] = striping_info[1] = 0; + + /* check if hint striping_factor is set by the user */ + MPI_Info_get(user_info, "striping_factor", MPI_MAX_INFO_VAL-1, + value, &flag); + if (flag) + striping_info[0] = atoi(value); + + /* check if hint striping_unit is set by the user */ + MPI_Info_get(user_info, "striping_unit", MPI_MAX_INFO_VAL-1, + value, &flag); + if (flag) + striping_info[1] = atoi(value); + +#ifdef HAVE_LUSTRE + uint64_t striping_factor, striping_unit; + striping_factor = striping_info[0]; + striping_unit = striping_info[1]; + /* When either striping_factor or striping_unit is not set, but + * not both, retrieve folder's striping factor or unit in order + * to inherit the missing one. + */ + if (ncp->rank == 0 && + striping_factor * striping_unit == 0 && + striping_factor + striping_unit > 0) { + /* rank 0 retrieves folder's striping settings */ + lustre_get_striping(filename, &striping_factor, + &striping_unit); + /* error is ignored, if there is any */ + striping_info[0] = striping_factor; + striping_info[1] = striping_unit; + } + MPI_Bcast(striping_info, 2, MPI_INT, 0, comm); +#endif + + if (striping_info[0] > 0) { + sprintf(value, "%d", striping_info[0]); + MPI_Info_set(user_info, "striping_factor", value); + } + + if (striping_info[1] > 0) { + sprintf(value, "%d", striping_info[1]); + MPI_Info_set(user_info, "striping_info", value); + } + } + } + +#ifdef MPICH_VERSION + /* MPICH recognizes file system type acronym prefixed to file names */ + TRACE_IO(MPI_File_open, (comm, path, mpi_amode, user_info, &fh)); +#else + TRACE_IO(MPI_File_open, (comm, filename, mpi_amode, user_info, &fh)); +#endif + if (mpireturn != MPI_SUCCESS) { +#ifndef HAVE_ACCESS + if (fIsSet(cmode, NC_NOCLOBBER)) { + /* This is the case when NC_NOCLOBBER is used in file creation + * and function access() is not available. MPI_MODE_EXCL is set + * in open mode. When MPI_MODE_EXCL is used and the file + * already exists, MPI-IO should return error class + * MPI_ERR_FILE_EXISTS. But, some MPI-IO implementations (older + * ROMIO) do not correctly return this error class. In this + * case, we can do the followings: check errno to see if it set + * to EEXIST. Note usually rank 0 makes the file open call and + * can be the only one having errno set. + */ + if (nprocs > 1) + TRACE_COMM(MPI_Bcast)(&errno, 1, MPI_INT, 0, comm); + if (errno == EEXIST) { + NCI_Free(ncp); + DEBUG_FOPEN_ERROR(NC_EEXIST) + } + } +#endif + err = ncmpii_error_mpi2nc(mpireturn, "MPI_File_open"); + DEBUG_FOPEN_ERROR(err); + /* for NC_NOCLOBBER, MPI_MODE_EXCL was added to mpi_amode. If the + * file already exists, MPI-IO should return error class + * MPI_ERR_FILE_EXISTS which PnetCDF will return error code + * NC_EEXIST. This is checked inside of ncmpii_error_mpi2nc() + */ + } + else + /* reset errno, as MPI_File_open may change it, even if it returns + * MPI_SUCCESS + */ + errno = 0; + + /* Now the file has been successfully created */ + ncp->mpio_fh_coll = fh; + ncp->mpio_fh_indep = (nprocs == 1) ? fh : MPI_FILE_NULL; + + /* Now the file has been successfully created, obtain the I/O hints + * used/modified by MPI-IO. + */ + TRACE_IO(MPI_File_get_info, (fh, &ncp->info)); + if (mpireturn != MPI_SUCCESS) { + err = ncmpii_error_mpi2nc(mpireturn, mpi_name); + DEBUG_FOPEN_ERROR(err); + } + } +#if PNETCDF_DRIVER_GIO == 1 + else if (ncp->driver == PNC_DRIVER_GIO) { + /* Use GIO driver. + * + * Note when INA is enabled, only the INA aggregators call GIO_open(). + * Non-INA aggregators have skipped to 'after_open' from above, with + * their 'comm', a local variable of this subroutine, remaining + * MPI_COMM_NULL (ncp->comm is always assigned to comm). + */ + + /* 'use_trunc' also indicates whether the file has already existed as a + * symbolic link, For a symbolic link file, we cannot add O_CREAT. + */ + int amode = (mpi_amode & MPI_MODE_CREATE) ? O_CREAT|O_RDWR : O_RDWR; + err = GIO_open(comm, path, amode, user_info, &ncp->gio_fh); + if (err != GIO_NOERR) { + err = ncmpii_error_gio2nc(err, "GIO_open"); + DEBUG_FOPEN_ERROR(err); + } + + /* Now the file has been successfully created, obtain the I/O hints + * used/modified by GIO driver. + */ + err = GIO_get_info(ncp->gio_fh, &ncp->info); + if (err != NC_NOERR) DEBUG_FOPEN_ERROR(err) + } +#endif + else + DEBUG_FOPEN_ERROR(NC_EDRIVER); + +after_open: + ncp->striping_unit = 0; + ncp->striping_factor = 0; + + /* All processes must obtain striping_unit and striping_factor consistent + * across all processes in order to fulfill ncmpi_inq_striping() and + * striping_unit is also used to set ncp->data_chunk if hint + * nc_data_move_chunk_size is not set by the user. + * + * When INA is enabled, only INA aggregators have valid hints striping_unit + * and striping_factor stored in their ncp->info, returned from the call to + * MPI_File_get_info() or GIO_get_info(). When non-INA aggregators have + * never call MPI_File_open() or GIO_open(), only happened when performing + * independent I/O, they do not have valid file striping hints store in + * their ncp->info. We need to make INA root to broadcast these 2 info to + * its INA group members. It is OK to have all processes extract these 2 + * hints below. In this case, the INA aggregators' hints will be valid and + * non-INA aggregators' hints will be overwritten by their INA aggregator's + * after the call of MPI_Bcast() later. + * + * When INA is disabled, all processes extract hints. */ - ncp->my_aggr = -1; - if (ncp->num_aggrs_per_node != 0) { - err = ncmpio_intra_node_aggr_init(ncp); - if (err != NC_NOERR) return err; + MPI_Info_get(ncp->info, "striping_unit", MPI_MAX_INFO_VAL-1, value, &flag); + striping_info[0] = 0; + if (flag) { + errno = 0; /* errno must set to zero before calling strtoll */ + striping_info[0] = (int)strtol(value,NULL,10); + if (errno != 0) striping_info[0] = 0; } - *ncpp = (void*)ncp; + MPI_Info_get(ncp->info, "striping_factor", MPI_MAX_INFO_VAL-1, value, + &flag); + striping_info[1] = 0; + if (flag) { + errno = 0; /* errno must set to zero before calling strtoll */ + striping_info[1] = (int)strtol(value,NULL,10); + if (errno != 0) striping_info[1] = 0; + } + + if (ncp->num_aggrs_per_node > 0) { + /* When INA is enabled, each INA aggregator broadcasts the file + * striping hints to its INA group members. + * + * Note ncp->num_aggrs_per_node may have been adjusted above. + */ + int intra_nprocs, intra_rank; + +#if PNETCDF_DEBUG_MODE == 1 + assert(comm_attr.ina_intra_comm != MPI_COMM_NULL); +#endif + MPI_Comm_size(comm_attr.ina_intra_comm, &intra_nprocs); + + if (intra_nprocs > 1) { + MPI_Info_get(ncp->info, "striping_unit", MPI_MAX_INFO_VAL-1, value, + &flag); + striping_info[0] = 0; + if (flag) { + errno = 0; /* errno must set to zero before calling strtoll */ + striping_info[0] = (int)strtol(value,NULL,10); + if (errno != 0) striping_info[0] = 0; + } + + MPI_Info_get(ncp->info, "striping_factor", MPI_MAX_INFO_VAL-1, + value, &flag); + striping_info[1] = 0; + if (flag) { + errno = 0; /* errno must set to zero before calling strtoll */ + striping_info[1] = (int)strtol(value,NULL,10); + if (errno != 0) striping_info[1] = 0; + } + + MPI_Comm_rank(comm_attr.ina_intra_comm, &intra_rank); + + MPI_Bcast(striping_info, 2, MPI_INT, 0, comm_attr.ina_intra_comm); + + if (intra_rank > 0) { + /* non-INA aggregators have not set these to ncp->info */ + sprintf(value, "%d", striping_info[0]); + MPI_Info_set(ncp->info, "striping_unit", value); + sprintf(value, "%d", striping_info[1]); + MPI_Info_set(ncp->info, "striping_factor", value); + } + } + } + + ncp->striping_unit = striping_info[0]; + ncp->striping_factor = striping_info[1]; + + if (ncp->data_chunk == -1) + /* if hint nc_data_move_chunk_size is not set by the user */ + ncp->data_chunk = (ncp->striping_unit > 0) ? ncp->striping_unit + : PNC_DATA_MOVE_CHUNK_SIZE; + + /* Add PnetCDF hints into ncp->info. This step is necessary, because the + * underneath MPI-IO may discard hints it does not recognize, which include + * all PnetCDF hints. PnetCDF hints need to be added to info, so users can + * inquire them. + */ + ncmpio_hint_set(ncp, ncp->info); + + /* comm_attr.NUMA_IDs[] is no longer needed once the file is opened. */ return NC_NOERR; } diff --git a/src/drivers/ncmpio/ncmpio_dim.c b/src/drivers/ncmpio/ncmpio_dim.c index 6d44dd1c91..273f9a4940 100644 --- a/src/drivers/ncmpio/ncmpio_dim.c +++ b/src/drivers/ncmpio/ncmpio_dim.c @@ -346,7 +346,7 @@ ncmpio_rename_dim(void *ncdp, #endif err_check: - if (ncp->safe_mode && ncp->nprocs > 1) { + if (fIsSet(ncp->flags, NC_MODE_SAFE) && ncp->nprocs > 1) { /* check the error so far across processes */ int status, mpireturn; diff --git a/src/drivers/ncmpio/ncmpio_driver.c b/src/drivers/ncmpio/ncmpio_driver.c index d28d0fccd7..ac11887e5f 100644 --- a/src/drivers/ncmpio/ncmpio_driver.c +++ b/src/drivers/ncmpio/ncmpio_driver.c @@ -57,8 +57,6 @@ static PNC_driver ncmpio_driver = { ncmpio_put_var, ncmpio_get_varn, ncmpio_put_varn, - ncmpio_get_vard, - ncmpio_put_vard, ncmpio_iget_var, ncmpio_iput_var, ncmpio_bput_var, diff --git a/src/drivers/ncmpio/ncmpio_driver.h b/src/drivers/ncmpio/ncmpio_driver.h index f072b5d420..5838b0b855 100644 --- a/src/drivers/ncmpio/ncmpio_driver.h +++ b/src/drivers/ncmpio/ncmpio_driver.h @@ -12,10 +12,14 @@ #include extern int -ncmpio_create(MPI_Comm comm, const char *path, int cmode, int ncid, MPI_Info info, void **ncdp); +ncmpio_create(MPI_Comm comm, const char *path, int cmode, int ncid, + int env_mode, MPI_Info info, PNC_comm_attr node_ids, + void **ncdp); extern int -ncmpio_open(MPI_Comm comm, const char *path, int omode, int ncid, MPI_Info info, void **ncdp); +ncmpio_open(MPI_Comm comm, const char *path, int omode, int ncid, + int env_mode, MPI_Info info, PNC_comm_attr node_ids, + void **ncdp); extern int ncmpio_close(void *ncdp); @@ -24,7 +28,8 @@ extern int ncmpio_enddef(void *ncdp); extern int -ncmpio__enddef(void *ncdp, MPI_Offset h_minfree, MPI_Offset v_align, MPI_Offset v_minfree, MPI_Offset r_align); +ncmpio__enddef(void *ncdp, MPI_Offset h_minfree, MPI_Offset v_align, + MPI_Offset v_minfree, MPI_Offset r_align); extern int ncmpio_redef(void *ncdp); diff --git a/src/drivers/ncmpio/ncmpio_enddef.c b/src/drivers/ncmpio/ncmpio_enddef.c index efc99657a6..52e3b30a87 100644 --- a/src/drivers/ncmpio/ncmpio_enddef.c +++ b/src/drivers/ncmpio/ncmpio_enddef.c @@ -26,18 +26,12 @@ #include #include #include "ncmpio_NC.h" -#ifdef ENABLE_SUBFILING +#if PNETCDF_SUBFILING == 1 #include "ncmpio_subfile.h" #endif -/* Divide the amount of data to be moved into chunks of size MOVE_UNIT each, - * and assign chunks to all processes. If the number of chunks is larger than - * the number of processes, carry out the data movement in multiple rounds. - */ -#define MOVE_UNIT 16777216 - #ifdef USE_POSIX_IO_TO_MOVE -/*----< move_file_block() >-------------------------------------------------*/ +/*----< move_file_block() >--------------------------------------------------*/ /* Call POSIX I/O subroutines to move data */ #include /* open() */ #include /* open() */ @@ -56,19 +50,22 @@ move_file_block(NC *ncp, off_t off_last, off_from, off_to; char *path = ncmpii_remove_file_system_type_prefix(ncp->path); + /* check if this is a valid move request */ + if (to == from || nbytes == 0) return NC_NOERR; + rank = ncp->rank; nprocs = ncp->nprocs; /* buf will be used as a temporal buffer to move data in chunks, i.e. * read a chunk and later write to the new location */ - buf = NCI_Malloc(MOVE_UNIT); + buf = NCI_Malloc(ncp->data_chunk); if (buf == NULL) DEBUG_RETURN_ERROR(NC_ENOMEM) - p_units = MOVE_UNIT * nprocs; + p_units = (size_t)ncp->data_chunk * nprocs; num_moves = nbytes / p_units; if (nbytes % p_units) num_moves++; - off_last = (num_moves - 1) * p_units + rank * MOVE_UNIT; + off_last = (num_moves - 1) * p_units + (size_t)rank * ncp->data_chunk; off_from = from + off_last; off_to = to + off_last; mv_amnt = nbytes % p_units; @@ -79,8 +76,8 @@ move_file_block(NC *ncp, if (nbytes >= p_units) do_open = 1; else { - MPI_Offset n_units = nbytes / MOVE_UNIT; - if (nbytes % MOVE_UNIT) n_units++; + MPI_Offset n_units = nbytes / ncp->data_chunk; + if (nbytes % ncp->data_chunk) n_units++; if (rank < n_units) do_open = 1; } @@ -97,16 +94,16 @@ move_file_block(NC *ncp, if (mv_amnt == p_units) { /* each rank moves amount of chunk_size */ - chunk_size = MOVE_UNIT; + chunk_size = ncp->data_chunk; } else { /* when total move amount is less than p_units */ - size_t num_chunks = mv_amnt / MOVE_UNIT; - if (mv_amnt % MOVE_UNIT) num_chunks++; + size_t num_chunks = mv_amnt / ncp->data_chunk; + if (mv_amnt % ncp->data_chunk) num_chunks++; if (rank < num_chunks) { - chunk_size = MOVE_UNIT; - if (rank == num_chunks - 1 && mv_amnt % MOVE_UNIT > 0) - chunk_size = mv_amnt % MOVE_UNIT; + chunk_size = ncp->data_chunk; + if (rank == num_chunks - 1 && mv_amnt % ncp->data_chunk > 0) + chunk_size = mv_amnt % ncp->data_chunk; assert(chunk_size > 0); } else @@ -118,8 +115,8 @@ move_file_block(NC *ncp, get_size = pread(fd, buf, chunk_size, off_from); if (get_size < 0) { fprintf(stderr, - "Error at %s line %d: pread file %s offset "OFFFMT" size %zd (%s)\n", - __func__,__LINE__,path,off_from,chunk_size,strerror(errno)); + "Error at %s line %d: pread file %s offset %lld size %zd (%s)\n", + __func__,__LINE__,path,(long long)off_from,chunk_size,strerror(errno)); DEBUG_RETURN_ERROR(NC_EREAD) } ncp->get_size += get_size; @@ -138,8 +135,8 @@ move_file_block(NC *ncp, put_size = pwrite(fd, buf, get_size, off_to); if (put_size < 0) { fprintf(stderr, - "Error at %s line %d: pwrite file %s offset "OFFFMT" size %zd (%s)\n", - __func__,__LINE__,path,off_to,get_size,strerror(errno)); + "Error at %s line %d: pwrite file %s offset %lld size %zd (%s)\n", + __func__,__LINE__,path,(long long)off_to,get_size,strerror(errno)); DEBUG_RETURN_ERROR(NC_EREAD) } ncp->put_size += put_size; @@ -159,169 +156,157 @@ move_file_block(NC *ncp, return status; } #else -/*----< move_file_block() >-------------------------------------------------*/ -/* Call MPI I/O subroutines to move data */ +/*----< move_file_block() >--------------------------------------------------*/ +/* Call GIO/MPI collective I/O subroutines to move data. + * GIO/MPI collective I/O subroutines themselves may switch to use independent + * subroutines if the file access pattern meet the condition. + */ static int move_file_block(NC *ncp, MPI_Offset to, /* destination starting file offset */ MPI_Offset from, /* source starting file offset */ MPI_Offset nbytes) /* amount to be moved */ { - char *mpi_name; - int rank, nprocs, mpireturn, err, status=NC_NOERR, do_coll; + int rank, align_rank, nprocs, status=NC_NOERR; void *buf; - size_t num_moves, mv_amnt, p_units; - MPI_Offset off_last, off_from, off_to; - MPI_Status mpistatus; - MPI_File fh; + MPI_Offset mv_amnt, p_units, end_off, end_block; + MPI_Offset off_last, off_from, off_to, rlen, wlen; + MPI_Comm comm; - rank = ncp->rank; - nprocs = ncp->nprocs; + /* check if this is a valid move request */ + if (to == from || nbytes == 0) return NC_NOERR; - /* collective_fh can be used in either MPI independent or collective I/O - * APIs to move data, within this subroutine. - */ - fh = ncp->collective_fh; + if (ncp->num_aggrs_per_node > 0) { + /* When intra-node aggregation is enabled, only INA aggregators perform + * the data movement. + */ + if (ncp->comm_attr.ina_inter_comm == MPI_COMM_NULL) + return NC_NOERR; - /* MPI-IO fileview has been reset in ncmpi_redef() to make the entire file - * visible - */ + comm = ncp->comm_attr.ina_inter_comm; + MPI_Comm_rank(comm, &rank); + MPI_Comm_size(comm, &nprocs); + } + else { + /* When intra-node aggregation is disabled, all processes perform the + * data movement. + */ + comm = ncp->comm; + rank = ncp->rank; + nprocs = ncp->nprocs; + } - /* Use MPI collective I/O subroutines to move data, only if nproc > 1 and - * MPI-IO hint "romio_no_indep_rw" is set to true. Otherwise, use MPI - * independent I/O subroutines, as the data partitioned among processes are - * not interleaved and thus need no collective I/O. - */ - do_coll = (ncp->nprocs > 1 && fIsSet(ncp->flags, NC_HCOLL)); +// printf("%s at %d: nbytes %lld from %lld to %lld\n",__func__,__LINE__,nbytes,from,to); +// fflush(stdout); MPI_Barrier(comm); fflush(stdout); MPI_Barrier(comm); + + /* align file access for all ranks */ + align_rank = (to / ncp->data_chunk + rank) % nprocs; /* buf will be used as a temporal buffer to move data in chunks, i.e. * read a chunk and later write to the new location */ - buf = NCI_Malloc(MOVE_UNIT); + buf = NCI_Malloc(ncp->data_chunk); if (buf == NULL) DEBUG_RETURN_ERROR(NC_ENOMEM) - p_units = MOVE_UNIT * nprocs; - num_moves = nbytes / p_units; - if (nbytes % p_units) num_moves++; - off_last = (num_moves - 1) * p_units + rank * MOVE_UNIT; - off_from = from + off_last; - off_to = to + off_last; - mv_amnt = nbytes % p_units; - if (mv_amnt == 0 && nbytes > 0) mv_amnt = p_units; + /* movement must start from the last p_units toward to the 1st */ + p_units = (MPI_Offset)ncp->data_chunk * nprocs; + end_off = to + nbytes; + end_block = end_off % p_units; + if (end_block == 0) end_block = p_units; + off_last = end_off - end_block; + + /* align file writes for all ranks (reads will not be aligned) */ + if (align_rank < end_block / ncp->data_chunk) + mv_amnt = ncp->data_chunk; + else if (end_block % ncp->data_chunk > 0 && + align_rank == end_block / ncp->data_chunk) + mv_amnt = end_block % ncp->data_chunk; + else + mv_amnt = 0; - /* move the data section starting from its tail toward its beginning */ - while (nbytes > 0) { - int chunk_size, get_size=0; + /* set the 1st read-write pair */ + off_to = off_last + (MPI_Offset)align_rank * ncp->data_chunk; + off_from = off_to - (to - from); - if (mv_amnt == p_units) { - /* each rank moves amount of chunk_size */ - chunk_size = MOVE_UNIT; - } - else { - /* when total move amount is less than p_units */ - size_t num_chunks = mv_amnt / MOVE_UNIT; - if (mv_amnt % MOVE_UNIT) num_chunks++; - if (rank < num_chunks) { - chunk_size = MOVE_UNIT; - if (rank == num_chunks - 1 && mv_amnt % MOVE_UNIT > 0) - chunk_size = mv_amnt % MOVE_UNIT; - assert(chunk_size > 0); - } - else - chunk_size = 0; - } + if (off_from < from) + off_from = from; - /* explicitly initialize mpistatus object to 0. For zero-length read, - * MPI_Get_count may report incorrect result for some MPICH version, - * due to the uninitialized MPI_Status object passed to MPI-IO calls. - * Thus we initialize it above to work around. - */ - memset(&mpistatus, 0, sizeof(MPI_Status)); - mpireturn = MPI_SUCCESS; + if (off_to < to) { + mv_amnt -= to - off_to; + if (mv_amnt < 0) mv_amnt = 0; + off_to = to; + } - /* read from file at off_from for amount of chunk_size */ - if (do_coll) { - TRACE_IO(MPI_File_read_at_all, (fh, off_from, buf, chunk_size, - MPI_BYTE, &mpistatus)); - } - else if (chunk_size > 0) { - TRACE_IO(MPI_File_read_at, (fh, off_from, buf, chunk_size, - MPI_BYTE, &mpistatus)); - } - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, mpi_name); - if (status == NC_NOERR && err == NC_EFILE) - DEBUG_ASSIGN_ERROR(status, NC_EREAD) - get_size = chunk_size; - } - else if (chunk_size > 0) { - /* for zero-length read, MPI_Get_count may report incorrect result - * for some MPICH version, due to the uninitialized MPI_Status - * object passed to MPI-IO calls. Thus we initialize it above to - * work around. See MPICH ticket: - * https://trac.mpich.org/projects/mpich/ticket/2332 - * - * Update the number of bytes read since file open. - * Because each rank reads and writes no more than one chunk_size - * at a time and chunk_size is < NC_MAX_INT, it is OK to call - * MPI_Get_count, instead of MPI_Get_count_c. - */ - MPI_Get_count(&mpistatus, MPI_BYTE, &get_size); - ncp->get_size += get_size; - } + /* pad the remaining of last p_units */ + nbytes += p_units - end_block; - /* to prevent from one rank's write run faster than other's read */ - if (ncp->nprocs > 1) MPI_Barrier(ncp->comm); + /* No need to set fileview, as fileview has been reset in ncmpi_redef() to + * make the entire file visible. + */ + + MPI_Offset f_off, f_len, b_off=0, b_len; + PNCIO_View f_view, b_view; - /* explicitly initialize mpistatus object to 0. For zero-length read, - * MPI_Get_count may report incorrect result for some MPICH version, - * due to the uninitialized MPI_Status object passed to MPI-IO calls. - * Thus we initialize it above to work around. + /* file view and buffer view used to move data are always contiguous */ + f_view.off = &f_off; + f_view.len = &f_len; + b_view.off = &b_off; + b_view.len = &b_len; + + while (nbytes > 0) { + /* read from file at off_from for amount of mv_amnt */ + f_off = off_from; + f_len = mv_amnt; + b_len = mv_amnt; + f_view.count = (mv_amnt == 0) ? 0 : 1; + b_view.count = (mv_amnt == 0) ? 0 : 1; + + rlen = ncmpio_file_read(ncp, NC_REQ_COLL, buf, f_view, b_view); + if (status == NC_NOERR && rlen < 0) status = (int)rlen; + + /* To prevent from one rank's write run faster than other's read, a + * barrier is required. Even with collective read/write, some processes + * may exit MPI_File_read_at_all()/GIO_read_at_all() than others and + * start writing to file while others are still reading it. */ - memset(&mpistatus, 0, sizeof(MPI_Status)); - mpireturn = MPI_SUCCESS; + if (nprocs > 1) MPI_Barrier(comm); - /* Write to new location at off_to for amount of get_size. Assuming the - * call to MPI_Get_count() above returns the actual amount of data read - * from the file, i.e. get_size. + /* Write to new location at off_to for amount of rlen, the actual read + * amount is rlen. */ - if (do_coll) { - TRACE_IO(MPI_File_write_at_all, (fh, off_to, buf, - get_size /* NOT chunk_size */, - MPI_BYTE, &mpistatus)); - } - else if (get_size > 0) { - TRACE_IO(MPI_File_write_at, (fh, off_to, buf, - get_size /* NOT chunk_size */, - MPI_BYTE, &mpistatus)); - } - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, mpi_name); - if (status == NC_NOERR && err == NC_EFILE) - DEBUG_ASSIGN_ERROR(status, NC_EWRITE) - } - else if (get_size > 0) { - /* update the number of bytes written since file open. - * Because each rank reads and writes no more than one chunk_size - * at a time and chunk_size is < NC_MAX_INT, it is OK to call - * MPI_Get_count, instead of MPI_Get_count_c. - */ - int put_size; - mpireturn = MPI_Get_count(&mpistatus, MPI_BYTE, &put_size); - if (mpireturn != MPI_SUCCESS || put_size == MPI_UNDEFINED) - ncp->put_size += get_size; /* or chunk_size */ - else - ncp->put_size += put_size; - } + wlen = 0; + + /* even when rlen == 0, must still participate */ + /* file view is contiguous */ + f_off = off_to; + f_len = rlen; + b_len = rlen; + f_view.count = (rlen == 0) ? 0 : 1; + b_view.count = (rlen == 0) ? 0 : 1; + + wlen = ncmpio_file_write(ncp, NC_REQ_COLL, buf, f_view, b_view); + if (status == NC_NOERR && wlen < 0) status = (int)wlen; /* move on to the next round */ - mv_amnt = p_units; - off_from -= mv_amnt; - off_to -= mv_amnt; - nbytes -= mv_amnt; + nbytes -= p_units; + off_from -= p_units; + off_to -= p_units; + + /* mv_amnt becomes ncp->data_chunk in the 2nd and later rounds */ + mv_amnt = ncp->data_chunk; + + /* special treatment for the 1st p_units */ + if (off_to < to) { + mv_amnt -= to - off_to; + if (mv_amnt < 0) mv_amnt = 0; + off_to = to; + } + if (off_from < from) + off_from = from; } NCI_Free(buf); + return status; } #endif @@ -371,12 +356,13 @@ move_record_vars(NC *ncp, NC *old) { * ncp->numrecs ---- number of records (set only if new file) */ static int -NC_begins(NC *ncp) +NC_begins(NC *ncp, + MPI_Offset v_align, + MPI_Offset r_align) { int i, j, mpireturn; MPI_Offset end_var=0; NC_var *last = NULL; - NC_var *first_var = NULL; /* first "non-record" var */ /* For CDF-1 and 2 formats, a variable's "begin" in the header is 4 bytes. * For CDF-5, it is 8 bytes. @@ -385,7 +371,7 @@ NC_begins(NC *ncp) /* get the true header size (not header extent) */ ncp->xsz = ncmpio_hdr_len_NC(ncp); - if (ncp->safe_mode && ncp->nprocs > 1) { + if (fIsSet(ncp->flags, NC_MODE_SAFE) && ncp->nprocs > 1) { /* this consistency check is redundant as metadata is kept consistent * at all time when safe mode is on */ @@ -411,28 +397,90 @@ NC_begins(NC *ncp) if (status != NC_NOERR) DEBUG_RETURN_ERROR(status) } + if (ncp->vars.ndefined == 0) { + /* There is no variable defined, ignore alignment and set header extent + * to header size. + */ + ncp->begin_var = MAX(ncp->begin_var, ncp->xsz); + ncp->begin_rec = ncp->begin_var; + ncp->recsize = 0; + ncp->numrecs = 0; + ncp->v_align = 0; + ncp->r_align = 0; + return NC_NOERR; + } + + /* calculate a good file extent alignment size based on user hints. + * The precedence of hints: + * + 1st priority: hints set in the environment variable PNETCDF_HINTS, + * i.e. nc_var_align_size and nc_record_align_size + * e.g. PNETCDF_HINTS="nc_var_align_size=1024" + * + 2nd priority: hints set in the MPI info objects passed into calls to + * ncmpi_create() and ncmpi_open() + * e.g. MPI_Info_set("nc_var_align_size", "1024"); + * + 3rd priority: hints passed from arguments of ncmpi__enddef() + * i.e. v_align and r_align + * e.g. ncmpi__enddef(..., v_align=1024,...) + * + * Default values + * NC_DEFAULT_H_MINFREE for h_minfree + * NC_DEFAULT_V_ALIGN for v_align + * NC_DEFAULT_V_MINFREE for v_minfree + * NC_DEFAULT_R_ALIGN for r_align + */ + + /* determine header extent (alignment for the data section) */ + if (ncp->info_v_align == -1) { + /* hint nc_var_align_size is not set */ + + /* argument v_align is set by user */ + if (v_align > 0) + ncp->v_align = D_RNDUP(v_align, 4); + else + ncp->v_align = NC_DEFAULT_V_ALIGN; + } + else + ncp->v_align = D_RNDUP(ncp->info_v_align, 4); + + /* determine alignment for record variable section */ + if (ncp->info_r_align == -1) { + /* hint nc_record_align_size is not set */ + + /* argument r_align is set by user */ + if (r_align > 0) + ncp->r_align = D_RNDUP(r_align, 4); + else if (ncp->vars.ndefined > ncp->vars.num_rec_vars) + ncp->r_align = NC_DEFAULT_R_ALIGN; + else + ncp->r_align = NC_DEFAULT_V_ALIGN; + } + else + ncp->r_align = D_RNDUP(ncp->info_r_align, 4); + /* This function is called in ncmpi_enddef(), which can happen either when * creating a new file and first time call to ncmpi_enddef(), or other * case, e.g. opening an existing file, calling ncmpi_redef(), and then * ncmpi_enddef(). For the former case, ncp->begin_var == 0. For the latter - * case, ncp->begin_var must be > 0, as it is the orignial header extent. + * case, ncp->begin_var must be > 0, as it is the original header extent. * We increase begin_var only if the new header size grows out of its * original extent, or the start of variable section is not aligned as * requested by ncp->v_align. Note ncp->xsz is header size and * ncp->begin_var is header extent. Growth of header extent must also * respect the minimum header free space requested by user. */ - ncp->begin_var = MAX(ncp->begin_var, ncp->xsz + ncp->h_minfree); - /* align header extent */ - if (ncp->vars.ndefined > 0) - ncp->begin_var = D_RNDUP(ncp->begin_var, ncp->v_align); - else /* no variable defined, ignore alignment and set header extent to - * header size */ - ncp->begin_var = MAX(ncp->begin_var, ncp->xsz); + /* warrant a free space at the end of header section */ + ncp->begin_var = MAX(ncp->begin_var, ncp->xsz + ncp->h_minfree); + /* Previously begin_var may be calculated using a different h_minfree and + * v_align. Thus it can be larger than this round's calculation. + */ if (ncp->old != NULL) - assert(ncp->begin_var >= ncp->old->begin_var); + ncp->begin_var = MAX(ncp->begin_var, ncp->old->begin_var); + + /* align header extent if there are fix-sized variables */ + if (ncp->vars.ndefined > ncp->vars.num_rec_vars) + ncp->begin_var = D_RNDUP(ncp->begin_var, ncp->v_align); /* ncp->begin_var is the aligned starting file offset of the first * variable (also data section), which is the extent of file header @@ -447,8 +495,6 @@ NC_begins(NC *ncp) /* skip record variables on this pass */ if (IS_RECVAR(ncp->vars.value[i])) continue; - if (first_var == NULL) first_var = ncp->vars.value[i]; - /* for CDF-1 check if over the file size limit 32-bit integer */ if (ncp->format == 1 && end_var > NC_MAX_INT) DEBUG_RETURN_ERROR(NC_EVARSIZE) @@ -464,8 +510,8 @@ NC_begins(NC *ncp) if (j < ncp->old->vars.ndefined) { if (ncp->vars.value[i]->begin < ncp->old->vars.value[j]->begin) /* the first ncp->vars.ndefined non-record variables should - be the same. If the new begin is smaller, reuse the old - begin */ + * be the same. If the new begin is smaller, reuse the old + * begin */ ncp->vars.value[i]->begin = ncp->old->vars.value[j]->begin; j++; } @@ -473,52 +519,43 @@ NC_begins(NC *ncp) /* end_var is the end offset of variable i */ end_var = ncp->vars.value[i]->begin + ncp->vars.value[i]->len; } + /* end_var now is pointing to the end of last fix-sized variable */ + + ncp->fix_end = D_RNDUP(end_var, 4); - /* end_var now is pointing to the end of last non-record variable */ + /* warrant a free space at the end of fix-sized variable section */ + if (ncp->vars.ndefined > ncp->vars.num_rec_vars) + ncp->begin_rec = ncp->fix_end + ncp->v_minfree; + else /* Ignore v_minfree when there is no fix-sized variable. */ + ncp->begin_rec = ncp->fix_end; - /* only (re)calculate begin_rec if there is no sufficient space at end of - * non-record variables or if the start of record variables is not aligned - * as requested by ncp->r_align. + /* Previously begin_rec may be calculated using a different v_minfree and + * r_align. Thus it can be larger than this round's calculation. */ - if (ncp->vars.ndefined > ncp->vars.num_rec_vars) { - if (ncp->begin_rec < end_var + ncp->v_minfree) - ncp->begin_rec = end_var + ncp->v_minfree; - } - else { /* if there is no fix-sized variable, ignore v_minfree */ - if (ncp->begin_rec < end_var) - ncp->begin_rec = end_var; - } + if (ncp->old != NULL) + ncp->begin_rec = MAX(ncp->begin_rec, ncp->old->begin_rec); - ncp->begin_rec = D_RNDUP(ncp->begin_rec, 4); + /* align the starting offset of record variable section */ + ncp->begin_rec = D_RNDUP(ncp->begin_rec, ncp->r_align); - /* Align the starting offset for record variable section. - * Ignore ncp->r_align, if there is no fix-sized variable. - */ - if (ncp->r_align > 1 && ncp->vars.ndefined > ncp->vars.num_rec_vars) - ncp->begin_rec = D_RNDUP(ncp->begin_rec, ncp->r_align); + /* When there is no fix_sized variable, set begin_var == begin_rec */ + if (ncp->vars.ndefined == ncp->vars.num_rec_vars) + ncp->begin_var = ncp->begin_rec; if (ncp->old != NULL) { - /* check whether the new begin_rec is smaller */ - if (ncp->begin_rec < ncp->old->begin_rec) - ncp->begin_rec = ncp->old->begin_rec; + assert(ncp->begin_var >= ncp->old->begin_var); + assert(ncp->begin_rec >= ncp->old->begin_rec); } - if (first_var != NULL) ncp->begin_var = first_var->begin; - else ncp->begin_var = ncp->begin_rec; - - end_var = ncp->begin_rec; - /* end_var now is pointing to the beginning of record variables - * note that this can be larger than the end of last non-record variable + /* Alignment r_align is only applicable to the record variable section, + * not individual record variables. */ - ncp->recsize = 0; - - /* The alignment is only applicable to the section of record variables, - * rather than individual record variables. + /* Loop through record variables and calculate the starting offset of each + * record variable. */ - - /* loop thru vars, second pass is for the 'record' vars, - * re-calculate the starting offset for each record variable */ + end_var = ncp->begin_rec; + ncp->recsize = 0; for (j=0, i=0; ivars.ndefined; i++) { if (!IS_RECVAR(ncp->vars.value[i])) /* skip non-record variables on this pass */ @@ -558,13 +595,12 @@ NC_begins(NC *ncp) last = ncp->vars.value[i]; } - /* - * for special case (Check CDF-1 and CDF-2 file format specifications.) + /* For special case (Check CDF-1 and CDF-2 file format specifications.) * "A special case: Where there is exactly one record variable, we drop the * requirement that each record be four-byte aligned, so in this case there * is no record padding." */ - if (last != NULL) { + if (last != NULL) { /* i.e. at least one record variable */ if (ncp->recsize == last->len) { /* exactly one record variable, pack value */ ncp->recsize = *last->dsizes * last->xsz; @@ -576,13 +612,15 @@ NC_begins(NC *ncp) #endif } -/* below is only needed if alignment is performed on record variables */ #if 0 - /* + /* This code block is to align individual record variable, which is no + * longer needed. + * * for special case of exactly one record variable, pack value + * + * if there is exactly one record variable, then there is no need to + * pad for alignment -- there's nothing after it. */ - /* if there is exactly one record variable, then there is no need to - * pad for alignment -- there's nothing after it */ if (last != NULL && ncp->recsize == last->len) ncp->recsize = *last->dsizes * last->xsz; #endif @@ -598,30 +636,23 @@ NC_begins(NC *ncp) * Write out the header * 1. Call ncmpio_hdr_put_NC() to copy the header object, ncp, to a buffer. * 2. Process rank 0 writes the header to file. + * + * Writing file header is always done by root process only, by calling the + * independent write subroutines. */ static int write_NC(NC *ncp) { - char *mpi_name; - int status=NC_NOERR, mpireturn, err, is_coll; + int status=NC_NOERR; MPI_Offset i, header_wlen, ntimes; - MPI_File fh; - MPI_Status mpistatus; assert(!NC_readonly(ncp)); - /* Depending on whether NC_HCOLL is set, writing file header can be done - * through either MPI collective or independent write call. - * When * ncp->nprocs == 1, ncp->collective_fh == ncp->independent_fh - */ - is_coll = (ncp->nprocs > 1 && fIsSet(ncp->flags, NC_HCOLL)) ? 1 : 0; - fh = ncp->collective_fh; - /* In NC_begins(), root's ncp->xsz and ncp->begin_var, root's header * size and extent, have been broadcast (sync-ed) among processes. */ -#ifdef ENABLE_NULL_BYTE_HEADER_PADDING +#if PNETCDF_NULL_BYTE_HEADER_PADDING == 1 /* NetCDF classic file formats require the file header null-byte padded. * PnetCDF's default is not to write the padding area (between ncp->xsz and * ncp->begin_var). When this padding feature is enabled, we write the @@ -649,9 +680,19 @@ write_NC(NC *ncp) /* only rank 0's header gets written to the file */ if (ncp->rank == 0) { char *buf=NULL, *buf_ptr; - MPI_Offset offset, remain; + MPI_Offset offset, remain, f_off, f_len, b_off=0, b_len; + PNCIO_View f_view, b_view; + + /* Both file view and buffer view are contiguous */ + f_view.count = 1; + f_view.off = &f_off; + f_view.len = &f_len; -#ifdef ENABLE_NULL_BYTE_HEADER_PADDING + b_view.count = 1; + b_view.off = &b_off; + b_view.len = &b_len; + +#if PNETCDF_NULL_BYTE_HEADER_PADDING == 1 /* NetCDF classic file formats require the file header null-byte * padded. Thus we must calloc a buffer of size equal to file header * extent. @@ -673,64 +714,39 @@ write_NC(NC *ncp) /* rank 0's fileview already includes the file header */ - /* explicitly initialize mpistatus object to 0. For zero-length read, - * MPI_Get_count may report incorrect result for some MPICH version, - * due to the uninitialized MPI_Status object passed to MPI-IO calls. - * Thus we initialize it above to work around. - */ - memset(&mpistatus, 0, sizeof(MPI_Status)); - /* write the header in chunks */ offset = 0; remain = header_wlen; buf_ptr = buf; for (i=0; iput_size += bufCount; - else - ncp->put_size += put_size; - } - offset += bufCount; - buf_ptr += bufCount; - remain -= bufCount; + MPI_Offset wlen; + b_len = MIN(remain, NC_MAX_INT); + + /* file view is contiguous */ + f_off = offset; + f_len = b_len; + + wlen = ncmpio_file_write(ncp, NC_REQ_INDEP, buf_ptr, f_view, b_view); + + offset += b_len; + buf_ptr += b_len; + remain -= b_len; + + if (status == NC_NOERR && wlen < 0) status = (int)wlen; } NCI_Free(buf); } - else if (fIsSet(ncp->flags, NC_HCOLL)) { - /* other processes participate the collective call */ - for (i=0; isafe_mode == 1 && ncp->nprocs > 1) { + if (fIsSet(ncp->flags, NC_MODE_SAFE) && ncp->nprocs > 1) { /* broadcast root's status, because only root writes to the file */ - int root_status = status; + int mpireturn, root_status = status; TRACE_COMM(MPI_Bcast)(&root_status, 1, MPI_INT, 0, ncp->comm); - /* root's write has failed, which is more serious than inconsistency */ - if (root_status == NC_EWRITE) DEBUG_ASSIGN_ERROR(status, NC_EWRITE) + if (mpireturn != MPI_SUCCESS) + status = ncmpii_error_mpi2nc(mpireturn, "MPI_Bcast"); + else if (root_status == NC_EWRITE) + /* root's write has failed, more serious than inconsistency */ + DEBUG_ASSIGN_ERROR(status, NC_EWRITE) } fClr(ncp->flags, NC_NDIRTY); @@ -745,15 +761,15 @@ write_NC(NC *ncp) * do not get error and proceed to the next subroutine call. */ #define CHECK_ERROR(err) { \ - if (ncp->safe_mode == 1 && ncp->nprocs > 1) { \ - int status; \ - TRACE_COMM(MPI_Allreduce)(&err, &status, 1, MPI_INT, MPI_MIN, \ + if (fIsSet(ncp->flags, NC_MODE_SAFE) && ncp->nprocs > 1) { \ + int min_err; \ + TRACE_COMM(MPI_Allreduce)(&err, &min_err, 1, MPI_INT, MPI_MIN, \ ncp->comm); \ if (mpireturn != MPI_SUCCESS) { \ err = ncmpii_error_mpi2nc(mpireturn, "MPI_Allreduce"); \ DEBUG_RETURN_ERROR(err) \ } \ - if (status != NC_NOERR) return status; \ + if (min_err != NC_NOERR) return min_err; \ } \ else if (err != NC_NOERR) \ return err; \ @@ -952,7 +968,7 @@ ncmpio_NC_check_voffs(NC *ncp) for (i=0, j=0; ivars.ndefined; i++) { NC_var *varp = ncp->vars.value[i]; if (varp->begin < ncp->xsz) { - if (ncp->safe_mode) { + if (fIsSet(ncp->flags, NC_MODE_SAFE)) { printf("Variable %s begin offset ("OFFFMT") is less than file header extent ("OFFFMT")\n", varp->name, varp->begin, ncp->xsz); } @@ -979,7 +995,7 @@ ncmpio_NC_check_voffs(NC *ncp) max_var_end = var_off_len[0].off + var_off_len[0].len; for (i=1; isafe_mode) { + if (fIsSet(ncp->flags, NC_MODE_SAFE)) { NC_var *var_cur = ncp->vars.value[var_off_len[i].ID]; NC_var *var_prv = ncp->vars.value[var_off_len[i-1].ID]; printf("Variable %s begin offset ("OFFFMT") overlaps variable %s (begin="OFFFMT", length="OFFFMT")\n", @@ -993,7 +1009,7 @@ ncmpio_NC_check_voffs(NC *ncp) } if (ncp->begin_rec < max_var_end) { - if (ncp->safe_mode) + if (fIsSet(ncp->flags, NC_MODE_SAFE)) printf("Record variable section begin ("OFFFMT") is less than fixed-size variable section end ("OFFFMT")\n", ncp->begin_rec, max_var_end); NCI_Free(var_off_len); @@ -1027,7 +1043,7 @@ ncmpio_NC_check_voffs(NC *ncp) for (i=1; ivars.num_rec_vars; i++) { if (var_off_len[i].off < var_off_len[i-1].off + var_off_len[i-1].len) { - if (ncp->safe_mode) { + if (fIsSet(ncp->flags, NC_MODE_SAFE)) { NC_var *var_cur = ncp->vars.value[var_off_len[i].ID]; NC_var *var_prv = ncp->vars.value[var_off_len[i-1].ID]; printf("Variable %s begin offset ("OFFFMT") overlaps variable %s (begin="OFFFMT", length="OFFFMT")\n", @@ -1050,7 +1066,7 @@ ncmpio_NC_check_voffs(NC *ncp) if (IS_RECVAR(varp)) continue; if (varp->begin < prev_off) { - if (ncp->safe_mode) { + if (fIsSet(ncp->flags, NC_MODE_SAFE)) { if (i == 0) printf("Variable \"%s\" begin offset ("OFFFMT") is less than header extent ("OFFFMT")\n", varp->name, varp->begin, prev_off); @@ -1065,7 +1081,7 @@ ncmpio_NC_check_voffs(NC *ncp) } if (ncp->begin_rec < prev_off) { - if (ncp->safe_mode) + if (fIsSet(ncp->flags, NC_MODE_SAFE)) printf("Record variable section begin offset ("OFFFMT") is less than fixed-size variable section end offset ("OFFFMT")\n", ncp->begin_rec, prev_off); DEBUG_RETURN_ERROR(NC_ENOTNC) @@ -1082,7 +1098,7 @@ ncmpio_NC_check_voffs(NC *ncp) if (!IS_RECVAR(varp)) continue; if (varp->begin < prev_off) { - if (ncp->safe_mode) { + if (fIsSet(ncp->flags, NC_MODE_SAFE)) { printf("Variable \"%s\" begin offset ("OFFFMT") is less than previous variable end offset ("OFFFMT")\n", varp->name, varp->begin, prev_off); if (i == 0) @@ -1102,88 +1118,6 @@ ncmpio_NC_check_voffs(NC *ncp) return NC_NOERR; } -/*----< read_hints() >-------------------------------------------------------*/ -/* check only the following hints set in environment variable PNETCDF_HINTS or - * MPI_Info object passed to ncmpi_create() and ncmpi_open(). - * nc_header_align_size, nc_var_align_size, and nc_record_align_size - */ -static void -read_hints(NC *ncp) -{ - char *warn_str="Warning: skip ill-formed hint set in PNETCDF_HINTS"; - char *env_str, *env_str_cpy, *hint, *next_hint, *key, *val, *deli; - char *hint_saved=NULL; - - /* reset hints from environment variable PNETCDF_HINTS */ - ncp->env_v_align = -1; - ncp->env_r_align = -1; - - /* get hints from the environment variable PNETCDF_HINTS, a string of - * hints separated by ";" and each hint is in the form of hint=value. E.g. - * "cb_nodes=16;cb_config_list=*:6". If this environment variable is set, - * it overrides the same hints that were set by MPI_Info_set() called in - * the application program. - */ - env_str = getenv("PNETCDF_HINTS"); - if (env_str == NULL) return; - - env_str_cpy = strdup(env_str); - next_hint = env_str_cpy; - - do { - hint = next_hint; - deli = strchr(hint, ';'); - if (deli != NULL) { - *deli = '\0'; /* add terminate char */ - next_hint = deli + 1; - } - else next_hint = "\0"; - if (hint_saved != NULL) free(hint_saved); - - /* skip all-blank hint */ - hint_saved = strdup(hint); - if (strtok(hint, " \t") == NULL) continue; - - free(hint_saved); - hint_saved = strdup(hint); /* save hint for error message */ - - deli = strchr(hint, '='); - if (deli == NULL) { /* ill-formed hint */ - printf("%s: '%s'\n", warn_str, hint_saved); - continue; - } - *deli = '\0'; - - /* hint key */ - key = strtok(hint, "= \t"); - if (key == NULL || NULL != strtok(NULL, "= \t")) { - /* expect one token before = */ - printf("%s: '%s'\n", warn_str, hint_saved); - continue; - } - - /* hint value */ - val = strtok(deli+1, "= \t"); - if (NULL != strtok(NULL, "= \t")) { /* expect one token before = */ - printf("%s: '%s'\n", warn_str, hint_saved); - continue; - } - - if (!strcmp(key, "nc_header_align_size") && ncp->env_v_align == -1) - ncp->env_v_align = atoll(val); - else if (!strcmp(key, "nc_var_align_size")) - ncp->env_v_align = atoll(val); - else if (!strcmp(key, "nc_record_align_size")) - ncp->env_r_align = atoll(val); - - } while (*next_hint != '\0'); - - if (hint_saved != NULL) free(hint_saved); - free(env_str_cpy); - - /* return no error as all hints are advisory */ -} - /*----< ncmpio__enddef() >---------------------------------------------------*/ /* This is a collective subroutine. * h_minfree Sets the pad at the end of the "header" section, i.e. at least @@ -1203,7 +1137,7 @@ ncmpio__enddef(void *ncdp, MPI_Offset v_minfree, MPI_Offset r_align) { - int i, num_fix_vars, mpireturn, err=NC_NOERR, status=NC_NOERR; + int i, mpireturn, err=NC_NOERR, status=NC_NOERR; char value[MPI_MAX_INFO_VAL]; MPI_Offset saved_begin_var; NC *ncp = (NC*)ncdp; @@ -1217,117 +1151,23 @@ ncmpio__enddef(void *ncdp, * called from ncmpio_enddef(). */ - /* check hints from environment variable PNETCDF_HINTS, or MPI info */ - read_hints(ncp); + /* Checking hints from environment variable PNETCDF_HINTS and MPI info has + * been done at the dispatcher, which calls combine_env_hints() at the + * early stage of ncmpi_create/ncmpi_open. + */ /* sanity check for NC_ENOTINDEFINE, NC_EINVAL, NC_EMULTIDEFINE_FNC_ARGS - * has been done at dispatchers */ + * has been done at dispatchers + */ ncp->h_minfree = (h_minfree < 0) ? NC_DEFAULT_H_MINFREE : h_minfree; ncp->v_minfree = (v_minfree < 0) ? NC_DEFAULT_V_MINFREE : v_minfree; - /* calculate a good file extent alignment size based on user hints. - * The precedence of hints: - * + 1st priority: hints set in the environment variable PNETCDF_HINTS, - * i.e. nc_var_align_size and nc_record_align_size - * e.g. PNETCDF_HINTS="nc_var_align_size=1024" - * + 2nd priority: hints set in the MPI info objects passed into calls to - * ncmpi_create() and ncmpi_open() - * e.g. MPI_Info_set("nc_var_align_size", "1024"); - * + 3rd priority: hints passed from arguments of ncmpi__enddef() - * i.e. v_align and r_align - * e.g. ncmpi__enddef(..., v_align=1024,...) - * - * Default values - * NC_DEFAULT_H_MINFREE for h_minfree - * NC_DEFAULT_V_ALIGN for v_align - * NC_DEFAULT_V_MINFREE for v_minfree - * NC_DEFAULT_R_ALIGN for r_align - */ - - num_fix_vars = ncp->vars.ndefined - ncp->vars.num_rec_vars; - - /* determine header extent (alignment for the data section) */ - if (ncp->env_v_align == -1) { - /* hint nc_var_align_size is not set in PNETCDF_HINTS */ - ncp->v_align = -1; - - if (num_fix_vars == 0 && ncp->env_r_align != -1) - /* if no fix-sizes variable, try use env_r_align */ - ncp->v_align = ncp->env_r_align; - - if (ncp->v_align < 0) { /* ncp->v_align is still not set */ - if (ncp->info_v_align >= 0) - /* use hint set in MPI info passed to ncmpi_create/ncmpi_open */ - ncp->v_align = ncp->info_v_align; - else if (v_align >= 0) - /* valid v_align is passed from ncmpi__enddef */ - ncp->v_align = v_align; - } - - if (ncp->v_align < 0) { /* ncp->v_align is still not set */ - if (ncp->old != NULL) - /* if enter from redefine mode, reuse one set in old header */ - ncp->v_align = ncp->old->v_align; - else /* default */ - ncp->v_align = NC_DEFAULT_V_ALIGN; - } - } - else /* hint nc_var_align_size is set in PNETCDF_HINTS, use it and - * ignore v_align passed from ncmpi__enddef(). - */ - ncp->v_align = ncp->env_v_align; - - /* determine alignment for record variable section */ - if (ncp->env_r_align == -1) { - /* hint nc_record_align_size is not set in PNETCDF_HINTS */ - ncp->r_align = -1; - - if (ncp->info_r_align >= 0) - /* use hint set in MPI info passed to ncmpi_create/ncmpi_open */ - ncp->r_align = ncp->info_r_align; - else if (r_align >= 0) - /* valid r_align is passed from ncmpi__enddef */ - ncp->r_align = r_align; - - if (ncp->r_align == -1) { /* ncp->r_align is still not set */ - if (ncp->old != NULL) - /* reuse one set in old header */ - ncp->r_align = ncp->old->r_align; - else - ncp->r_align = NC_DEFAULT_R_ALIGN; - } - } - else - /* hint nc_record_align_size is set in PNETCDF_HINTS, use it and - * ignore r_align passed from ncmpi__enddef(). - */ - ncp->r_align = ncp->env_r_align; - - /* all CDF formats require 4-bytes alignment */ - if (ncp->v_align == 0) ncp->v_align = 4; - else ncp->v_align = D_RNDUP(ncp->v_align, 4); - if (ncp->r_align == 0) ncp->r_align = 4; - else ncp->r_align = D_RNDUP(ncp->r_align, 4); - - /* reflect the hint changes to the MPI info object, so the user can inquire - * what the true hint values are being used - */ - sprintf(value, OFFFMT, ncp->v_align); - MPI_Info_set(ncp->mpiinfo, "nc_var_align_size", value); - sprintf(value, OFFFMT, ncp->r_align); - MPI_Info_set(ncp->mpiinfo, "nc_record_align_size", value); - -#ifdef ENABLE_SUBFILING - sprintf(value, "%d", ncp->num_subfiles); - MPI_Info_set(ncp->mpiinfo, "nc_num_subfiles", value); +#if PNETCDF_SUBFILING == 1 if (ncp->num_subfiles > 1) { /* TODO: should return subfile-related msg when there's an error */ err = ncmpio_subfile_partition(ncp); CHECK_ERROR(err) } -#else - MPI_Info_set(ncp->mpiinfo, "pnetcdf_subfiling", "disable"); - MPI_Info_set(ncp->mpiinfo, "nc_num_subfiles", "0"); #endif /* check whether sizes of all variables are legal */ @@ -1344,25 +1184,25 @@ ncmpio__enddef(void *ncdp, * all processes. */ saved_begin_var = ncp->begin_var; - err = NC_begins(ncp); + err = NC_begins(ncp, v_align, r_align); if (err != NC_NOERR) /* restore the original begin_var when failed */ ncp->begin_var = saved_begin_var; CHECK_ERROR(err) - if (ncp->safe_mode) { + if (fIsSet(ncp->flags, NC_MODE_SAFE)) { /* check whether variable begins are in an increasing order. * This check is for debugging purpose. */ err = ncmpio_NC_check_voffs(ncp); CHECK_ERROR(err) } -#ifdef ENABLE_SUBFILING +#if PNETCDF_SUBFILING == 1 if (ncp->num_subfiles > 1) { /* get ncp info for the subfile */ - err = NC_begins(ncp->ncp_sf); + err = NC_begins(ncp->ncp_sf, v_align, r_align); CHECK_ERROR(err) - if (ncp->safe_mode) { + if (fIsSet(ncp->flags, NC_MODE_SAFE)) { /* check whether variable begins are in an increasing order. * This check is for debugging purpose. */ err = ncmpio_NC_check_voffs(ncp->ncp_sf); @@ -1373,8 +1213,9 @@ ncmpio__enddef(void *ncdp, if (ncp->old != NULL && ncp->vars.ndefined > 0) { /* The current define mode was entered from ncmpi_redef, not from - * ncmpi_create. We must check if header extent has grown. - * This only needs to be done when there are variables defined. + * ncmpi_create. We must individually check if the three sections of + * header, fix-sized, and record variables have grown. This is only + * required when there are variables defined. */ int mov_done=0; MPI_Offset nbytes; @@ -1387,111 +1228,65 @@ ncmpio__enddef(void *ncdp, /* ncp->numrecs has already sync-ed in ncmpi_redef */ - if (ncp->begin_var > ncp->old->begin_var && - ncp->begin_rec - ncp->begin_var == - ncp->old->begin_rec - ncp->old->begin_var && - ncp->vars.num_rec_vars == ncp->old->vars.num_rec_vars) { - /* When header extent grows, if the distance between the starting - * offsets of fix-sized and record variable sections remains the - * same, and no new record variable has been added, then the entire - * data section can be moved as a single contiguous block to a - * higher file offset. - */ - - /* Make sure all processes finish their I/O before any process - * starts to read the data section. - */ - if (ncp->nprocs > 1) MPI_Barrier(ncp->comm); + /* Make sure all processes finish their I/O before any process starts + * to read the data section. + */ + if (ncp->nprocs > 1) MPI_Barrier(ncp->comm); - /* amount of data section to be moved */ - nbytes = ncp->old->begin_rec - ncp->old->begin_var - + ncp->old->recsize * ncp->old->numrecs; + mov_done = 0; - err = move_file_block(ncp, ncp->begin_var, ncp->old->begin_var, - nbytes); - if (status == NC_NOERR) status = err; - mov_done = 1; - } - else { - if (ncp->begin_rec > ncp->old->begin_rec) { - /* beginning of record variable section grows. The entire - * record variable section must be moved to a higher file - * offset. - */ - - /* Make sure all processes finish their I/O before any process - * starts to read the data section. + /* move record variable section first */ + if (ncp->begin_rec > ncp->old->begin_rec || + ncp->vars.num_rec_vars > ncp->old->vars.num_rec_vars) { + /* It is possible begin_rec remain the same after adding new record + * variables, e.g. when both header extent and fix-sized variable + * section did not grow. + */ + if (ncp->vars.num_rec_vars == ncp->old->vars.num_rec_vars) { + /* No new record variable is added. Move the entire record + * variable section as a single data chunk. */ - if (ncp->nprocs > 1) MPI_Barrier(ncp->comm); - - if (ncp->vars.num_rec_vars == ncp->old->vars.num_rec_vars) { - /* no new record variable has been added, then the entire - * record variable section can be moved as a single - * contiguous block - */ - - /* amount of data to be moved */ + if (ncp->vars.num_rec_vars > 0) { nbytes = ncp->old->recsize * ncp->old->numrecs; - err = move_file_block(ncp, ncp->begin_rec, ncp->old->begin_rec, nbytes); if (status == NC_NOERR) status = err; } - else { - /* new record variables have been added. Must move one - * record at a time, because all records of record - * variables are stored interleaved in the file. - */ - err = move_record_vars(ncp, ncp->old); - if (status == NC_NOERR) status = err; - } - mov_done = 1; } + else { + /* Move one record variable at a time */ + err = move_record_vars(ncp, ncp->old); + if (status == NC_NOERR) status = err; + } + mov_done = 1; + } - if (ncp->begin_var > ncp->old->begin_var) { - /* beginning of fix-sized variable section grows. The fix-sized - * variable section must be moved to a higher file offset. - */ - - /* Make sure all processes finish their I/O before any process - * starts to read the data section. - */ - if (!mov_done && ncp->nprocs > 1) MPI_Barrier(ncp->comm); + /* Move fix-sized variable section when starting offset grows and there + * are fix-sized variables defined. + */ + if (ncp->begin_var > ncp->old->begin_var && + ncp->old->vars.ndefined > ncp->old->vars.num_rec_vars) { - /* First, find the size of fix-sized variable section, i.e. - * from the last fix-sized variable's begin and len. Note there - * may be some free space at the end of fix-sized variable - * section that need not be moved. - */ - MPI_Offset end_var = ncp->old->begin_var; - for (i=ncp->old->vars.ndefined-1; i>=0; i--) { - if (!IS_RECVAR(ncp->old->vars.value[i])) { - end_var = ncp->old->vars.value[i]->begin - + ncp->old->vars.value[i]->len; - break; - } - } - /* amount of data to be moved */ - nbytes = end_var - ncp->old->begin_var; + nbytes = ncp->old->fix_end - ncp->old->begin_var; - err = move_file_block(ncp, ncp->begin_var, ncp->old->begin_var, - nbytes); - if (status == NC_NOERR) status = err; - mov_done = 1; - } + err = move_file_block(ncp, ncp->begin_var, ncp->old->begin_var, + nbytes); + if (status == NC_NOERR) status = err; + mov_done = 1; } - /* to prevent some ranks run faster than others and start to read - * after exiting ncmpi_enddef(), while some processes are still moving - * the data section + /* To prevent some ranks run faster than others and start to read after + * exiting ncmpi_enddef(), while some processes are still moving the + * data section */ if (mov_done && ncp->nprocs > 1) MPI_Barrier(ncp->comm); } /* ... ncp->old != NULL */ /* first sync header objects in memory across all processes, and then root - * writes the header to file. Note safe_mode error check will be done in - * write_NC() */ + * writes the header to file. Note safe mode error check will be done in + * write_NC(). + */ status = write_NC(ncp); /* we should continue to exit define mode, even if header is inconsistent @@ -1500,7 +1295,7 @@ ncmpio__enddef(void *ncdp, * be considered fatal, as inconsistency is about the data structure, * rather then contents (such as attribute values) */ -#ifdef ENABLE_SUBFILING +#if PNETCDF_SUBFILING == 1 /* write header to subfile */ if (ncp->num_subfiles > 1) { err = write_NC(ncp->ncp_sf); @@ -1520,11 +1315,29 @@ ncmpio__enddef(void *ncdp, } fClr(ncp->flags, NC_MODE_CREATE | NC_MODE_DEF); -#ifdef ENABLE_SUBFILING +#if PNETCDF_SUBFILING == 1 if (ncp->num_subfiles > 1) fClr(ncp->ncp_sf->flags, NC_MODE_CREATE | NC_MODE_DEF); #endif + if (ncp->info != MPI_INFO_NULL) { + /* reflect the hint changes to the MPI info object, so the user can + * inquire what the true hint values are being used + */ + sprintf(value, OFFFMT, ncp->v_align); + MPI_Info_set(ncp->info, "nc_var_align_size", value); + sprintf(value, OFFFMT, ncp->r_align); + MPI_Info_set(ncp->info, "nc_record_align_size", value); + +#if PNETCDF_SUBFILING == 1 + sprintf(value, "%d", ncp->num_subfiles); + MPI_Info_set(ncp->info, "nc_num_subfiles", value); +#else + MPI_Info_set(ncp->info, "pnetcdf_subfiling", "disable"); + MPI_Info_set(ncp->info, "nc_num_subfiles", "0"); +#endif + } + return status; } diff --git a/src/drivers/ncmpio/ncmpio_file_io.c b/src/drivers/ncmpio/ncmpio_file_io.c index 681cfd599e..4023585da0 100644 --- a/src/drivers/ncmpio/ncmpio_file_io.c +++ b/src/drivers/ncmpio/ncmpio_file_io.c @@ -10,320 +10,701 @@ #include #include #include /* memset() */ +#include /* O_RDONLY, O_RDWR */ +#include /* INT_MAX */ + +#include #include +#if PNETCDF_DRIVER_GIO == 1 +#include +#endif + #include #include #include "ncmpio_NC.h" -/*----< ncmpio_read_write() >------------------------------------------------*/ -int -ncmpio_read_write(NC *ncp, - int rw_flag, /* NC_REQ_WR or NC_REQ_RD */ - int coll_indep, /* NC_REQ_COLL or NC_REQ_INDEP */ - MPI_Offset offset, - MPI_Offset buf_count, - MPI_Datatype buf_type, - void *buf, - int buftype_is_contig) +/*----< get_count() >--------------------------------------------------------*/ +/* This subroutine is independent. On success, the number of bytes read/written + * is returned (zero indicates nothing was read/written). Like POSIX read()/ + * write(), it is not an error if this number is smaller than the number of + * bytes requested. On error, a negative value, an NC error code, is returned. + * + * This subroutine is called only when using MPI-IO driver. + */ +static +MPI_Offset get_count(MPI_Status *mpistatus, + MPI_Datatype datatype) { - char *mpi_name; - int status=NC_NOERR, err=NC_NOERR, mpireturn; - MPI_Status mpistatus; - MPI_File fh; - MPI_Offset req_size; + int mpireturn; + + if (datatype == MPI_DATATYPE_NULL) return 0; #ifdef HAVE_MPI_TYPE_SIZE_C - MPI_Count btype_size; + MPI_Count type_size; /* MPI_Type_size_c is introduced in MPI 4.0 */ - mpireturn = MPI_Type_size_c(buf_type, &btype_size); - mpi_name = "MPI_Type_size_c"; + MPI_Type_size_c(datatype, &type_size); #elif defined(HAVE_MPI_TYPE_SIZE_X) - MPI_Count btype_size; + MPI_Count type_size; /* MPI_Type_size_x is introduced in MPI 3.0 */ - mpireturn = MPI_Type_size_x(buf_type, &btype_size); - mpi_name = "MPI_Type_size_x"; + MPI_Type_size_x(datatype, &type_size); #else - int btype_size; - mpireturn = MPI_Type_size(buf_type, &btype_size); - mpi_name = "MPI_Type_size"; + int type_size; + MPI_Type_size(datatype, &type_size); #endif - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, mpi_name); - /* return the first encountered error if there is any */ - err = (err == NC_EFILE) ? NC_EREAD : err; - } - else if (btype_size == MPI_UNDEFINED) { -#ifdef PNETCDF_DEBUG - fprintf(stderr,"%d: %s line %d: btype_size MPI_UNDEFINED buf_count="OFFFMT"\n", - ncp->rank, __func__,__LINE__,buf_count); + +#ifdef HAVE_MPI_GET_COUNT_C + MPI_Count count; + mpireturn = MPI_Get_count_c(mpistatus, datatype, &count); +#else + int count; + mpireturn = MPI_Get_count(mpistatus, datatype, &count); #endif - DEBUG_ASSIGN_ERROR(err, NC_EINTOVERFLOW) + + if (mpireturn != MPI_SUCCESS || count == MPI_UNDEFINED) + /* In case of partial read/write, MPI_Get_elements() is supposed to be + * called to obtain the number of type map elements actually read/ + * written in order to calculate the true read/write amount. Below + * skips this step and simply returns the partial read/write amount. + * See an example usage of MPI_Get_count() in Example 5.12 from MPI + * standard document. + */ + return NC_EFILE; + + return (MPI_Offset)count * type_size; +} + +/*----< ncmpio_file_close() >------------------------------------------------*/ +/* + * This function is collective. + */ +int +ncmpio_file_close(NC *ncp) +{ + int err=NC_NOERR; + + if (ncp->driver == PNC_DRIVER_MPIIO) { + char *mpi_name; + int mpireturn; + + if (ncp->mpio_fh_indep != ncp->mpio_fh_coll && + ncp->mpio_fh_indep != MPI_FILE_NULL) { + TRACE_IO(MPI_File_close, (&ncp->mpio_fh_indep)); + if (mpireturn != MPI_SUCCESS) + err = ncmpii_error_mpi2nc(mpireturn, mpi_name); + } + + if (ncp->mpio_fh_coll != MPI_FILE_NULL) { + TRACE_IO(MPI_File_close, (&ncp->mpio_fh_coll)); + if (mpireturn != MPI_SUCCESS) + err = ncmpii_error_mpi2nc(mpireturn, mpi_name); + } } +#if PNETCDF_DRIVER_GIO == 1 + else if (ncp->driver == PNC_DRIVER_GIO) { + if (ncp->gio_fh != NULL) + /* When INA is enabled, non-INA aggregators' gio_fh may be NULL */ + err = GIO_close(&ncp->gio_fh); + ncp->gio_fh = NULL; + } +#endif + else + err = NC_EDRIVER; - if (err != NC_NOERR) { - if (coll_indep == NC_REQ_COLL) { - DEBUG_ASSIGN_ERROR(status, err) - /* write nothing, but participate the collective call */ - buf_count = 0; + return err; +} + +/*----< ncmpio_file_delete() >-----------------------------------------------*/ +/* + * This function is collective. + * + * This subroutine is called only from ncmpi_abort. When the file is being + * created and an error occurs, the program is still in define mode. In this + * case, the file is deleted. + */ +int +ncmpio_file_delete(NC *ncp) +{ + int err=NC_NOERR; + + if (ncp->rank == 0) { + if (ncp->driver == PNC_DRIVER_MPIIO) { + char *mpi_name; + int mpireturn; +#ifdef MPICH_VERSION + /* MPICH recognizes file system type acronym prefixed to the file + * name. + */ + TRACE_IO(MPI_File_delete, ((char *)ncp->path, ncp->info)); +#else + /* Remove the file system type prefix name if there is any, because + * some MPI libraries do not recognize such prefix. For example, + * when path = "lustre:/home/foo/testfile.nc", remove "lustre:" to + * make filename pointing to "/home/foo/testfile.nc". + */ + char *path = ncmpii_remove_file_system_type_prefix(ncp->path); + TRACE_IO(MPI_File_delete, (path, ncp->info)); +#endif + if (mpireturn != MPI_SUCCESS) + err = ncmpii_error_mpi2nc(mpireturn, mpi_name); + } +#if PNETCDF_DRIVER_GIO == 1 + else if (ncp->driver == PNC_DRIVER_GIO) { + err = GIO_delete(ncp->path); + if (err != GIO_NOERR) + err = ncmpii_error_gio2nc(err, "GIO_delete"); } +#endif else - DEBUG_RETURN_ERROR(err) + err = NC_EDRIVER; } - /* request size in bytes, may be > NC_MAX_INT */ - req_size = buf_count * btype_size; + if (ncp->nprocs > 1) + MPI_Bcast(&err, 1, MPI_INT, 0, ncp->comm); + + return err; +} + +/*----< ncmpio_file_sync() >-------------------------------------------------*/ +/* This function must be called collectively, no matter if it is in collective + * or independent data mode. + */ +int +ncmpio_file_sync(NC *ncp) { + char *mpi_name; + int mpireturn; - /* explicitly initialize mpistatus object to 0. For zero-length read, - * MPI_Get_count may report incorrect result for some MPICH version, - * due to the uninitialized MPI_Status object passed to MPI-IO calls. - * Thus we initialize it above to work around. +#if PNETCDF_DRIVER_GIO == 1 + if (ncp->driver == PNC_DRIVER_GIO) { + int err=NC_NOERR; + if (ncp->gio_fh != NULL) { + err = GIO_sync(ncp->gio_fh); + if (err != GIO_NOERR) + err = ncmpii_error_gio2nc(err, "GIO_sync"); + } + return err; + } +#endif + if (ncp->driver != PNC_DRIVER_MPIIO) + return NC_EDRIVER; + + /* the remaining of this subroutine are for when using MPI-IO */ + + if (ncp->mpio_fh_indep != MPI_FILE_NULL) { + TRACE_IO(MPI_File_sync, (ncp->mpio_fh_indep)); + if (mpireturn != MPI_SUCCESS) + return ncmpii_error_mpi2nc(mpireturn, mpi_name); + } + /* when nprocs == 1, ncp->mpio_fh_coll == ncp->mpio_fh_indep */ + if (ncp->nprocs == 1) return NC_NOERR; + + /* When intra-node aggregation is enabled, non-aggregator's + * ncp->mpio_fh_coll is always MPI_FILE_NULL. When disabled, + * ncp->mpio_fh_coll on all ranks is never MPI_FILE_NULL as collective + * mode is default in PnetCDF. */ - memset(&mpistatus, 0, sizeof(MPI_Status)); + if (ncp->mpio_fh_coll != MPI_FILE_NULL) { + TRACE_IO(MPI_File_sync, (ncp->mpio_fh_coll)); + if (mpireturn != MPI_SUCCESS) + return ncmpii_error_mpi2nc(mpireturn, mpi_name); + } - if (coll_indep == NC_REQ_COLL) - fh = ncp->collective_fh; - else - fh = ncp->independent_fh; + /* Barrier is not necessary ... + TRACE_COMM(MPI_Barrier)(ncp->comm); + */ - if (rw_flag == NC_REQ_RD) { - void *xbuf=buf; - MPI_Datatype xbuf_type=buf_type; + return NC_NOERR; +} -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count xlen = (MPI_Count)buf_count; -#else - int xlen = (int)buf_count; +/*----< ncmpio_file_read() >-------------------------------------------------*/ +/* For zero-sized requests, file_view.count == buf_view.count == 0. + * For non-zero sized requests, file_view.count > 0 and buf_view.count > 0 and + * both file_view's off and len should not be NULL. The same for buf_view. + * Accumulated amount of file_view and buf_view should be equal. + * + * This subroutine may be called from either collective or independent data + * mode. Argument 'coll_indep' indicates the caller intends to perform a + * collective or independent get request, regardless of which data mode the + * program is currently on. + */ +MPI_Offset +ncmpio_file_read(NC *ncp, + int coll_indep, /* NC_REQ_INDEP or NC_REQ_COLL */ + void *buf, + PNCIO_View file_view, + PNCIO_View buf_view) +{ + char *xbuf; + int i, status=NC_NOERR, err=NC_NOERR; + MPI_Offset rlen=0, off_zero=0, f_amnt, b_amnt; + PNCIO_View orig_buf_view; + + /* If zero-sized request and independent read, this rank can return now. */ + if (buf_view.count == 0 && coll_indep == NC_REQ_INDEP) + return NC_NOERR; - if (buf_count > NC_MAX_INT) { - if (coll_indep == NC_REQ_COLL) { -#ifdef PNETCDF_DEBUG - fprintf(stderr,"%d: %s line %d: NC_EINTOVERFLOW buf_count="OFFFMT"\n", - ncp->rank, __func__,__LINE__,buf_count); +#if PNETCDF_DEBUG_MODE == 1 + assert(file_view.count >= 0); + if (file_view.count > 0) { + assert(file_view.off != NULL); + assert(file_view.len != NULL); + } + assert(buf_view.count >= 0); + if (buf_view.count > 0) { + assert(buf_view.off != NULL); + assert(buf_view.len != NULL); + } #endif - DEBUG_ASSIGN_ERROR(status, NC_EINTOVERFLOW) - /* write nothing, but participate the collective call */ - xlen = 0; - } - else - DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) - } + + /* Calculate file_view amount in bytes, may be > NC_MAX_INT */ + for (f_amnt=0, i=0; i NC_MAX_INT */ + for (b_amnt=0, i=0; i 0 && !buftype_is_contig && req_size <= ncp->ibuf_size) { - /* if read buffer is noncontiguous and size is < ncp->ibuf_size, - * allocate a temporary buffer and use it to read, as some MPI, - * e.g. Cray on KNL, can be significantly slow when read buffer is - * noncontiguous. - */ -#ifdef HAVE_MPI_LARGE_COUNT - xbuf_type = MPI_BYTE; - xlen = (MPI_Count)req_size; -#else - if (req_size > NC_MAX_INT) { - mpireturn = MPI_Type_contiguous(xlen, buf_type, &xbuf_type); - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, "MPI_Type_contiguous"); - if (coll_indep == NC_REQ_COLL) - DEBUG_ASSIGN_ERROR(status, err) - else - DEBUG_RETURN_ERROR(err) - } - MPI_Type_commit(&xbuf_type); - xlen = 1; + /* Save the original buf_view.count, as it may be modified when + * b_amnt <= ibuf_size, original buf_view is required to unpack the read + * data to user read buffer. + */ + orig_buf_view = buf_view; + + xbuf = (char*)buf; + + if (buf_view.count > 1 && b_amnt <= ncp->ibuf_size) { + /* If this read buffer is noncontiguous and amount is less than + * ncp->ibuf_size, we allocate a temporary contiguous buffer and use it + * to read from the file. Later it is unpacked to user buffer. As some + * MPI, e.g. Cray on KNL, can be significantly slow when the read + * buffer is noncontiguous. The only case of read buffer being + * noncontiguous is when nonblocking API ncmpi_wait/wait_all() is + * called and INA is disabled. + * + * Note ncp->ibuf_size is never > NC_MAX_INT. + */ + xbuf = (char*) NCI_Malloc(b_amnt); + + /* mark buf_view is contiguous */ + buf_view.count = 1; + buf_view.off = &off_zero; + buf_view.len = &b_amnt; + } + + if (ncp->driver == PNC_DRIVER_MPIIO) { + char *xbuf_ptr, *mpi_name; + int bufCount, set_file_view, mpireturn; + MPI_Offset disp; + MPI_File fh; + MPI_Status mpistatus; + MPI_Datatype fileType=MPI_BYTE, bufType=MPI_BYTE; + + memset(&mpistatus, 0, sizeof(MPI_Status)); + + /* when ncp->nprocs == 1, ncp->mpio_fh_coll == ncp->mpio_fh_indep */ + fh = (ncp->nprocs > 1 && !fIsSet(ncp->flags, NC_MODE_INDEP)) + ? ncp->mpio_fh_coll : ncp->mpio_fh_indep; + + /* When in collective data mode, if a process makes an independent put + * call, skip setting the file view. This must come from subroutines + * that would like to access file header or perform data section + * movement at ncmpi_enddef(), due to new metadata was added. In these + * cases, their file views are always contiguous. However, argument + * 'disp' to be used in MPI_File_read_at() call must be set to + * file_view.off[0]. + * + * In all other cases, we must set the file view. + */ + if (!fIsSet(ncp->flags, NC_MODE_INDEP) && coll_indep == NC_REQ_INDEP) { + set_file_view = 0; + assert(file_view.count == 1); + } + else + set_file_view = 1; + + disp = 0; + if (file_view.count == 1) { /* no need to create fileType */ + disp = file_view.off[0]; + } + else if (file_view.count > 1) { + /* Construct a file type for setting the file view. */ + err = ncmpio_type_create_hindexed(file_view.count, file_view.off, + file_view.len, &fileType); + if (err != NC_NOERR && status == NC_NOERR) status = err; + } + + if (set_file_view) { + TRACE_IO(MPI_File_set_view, (fh, disp, MPI_BYTE, fileType, + "native", MPI_INFO_NULL)); + if (mpireturn != MPI_SUCCESS) { + err = ncmpii_error_mpi2nc(mpireturn, mpi_name); + if (status == NC_NOERR) status = err; } + } + if (fileType != MPI_BYTE) MPI_Type_free(&fileType); + + /* Construct a derived data type describing user buffer data layout to + * be used in MPI_File_read_at(). + */ + bufCount = 0; + xbuf_ptr = xbuf; + if (buf_view.count == 1) { /* buffer view is contiguous */ + if (buf_view.len[0] <= INT_MAX) + bufCount = (int)buf_view.len[0]; else { - xbuf_type = MPI_BYTE; - xlen = (int)req_size; + /* Try creating bufType */ + err = ncmpio_type_contiguous(buf_view.len[0], &bufType); + if (err == NC_NOERR) + bufCount = 1; + else { /* make this a zero-size request */ + bufCount = 0; + if (status == NC_NOERR) + status = err; + } } -#endif - xbuf = NCI_Malloc((size_t)req_size); + xbuf_ptr += buf_view.off[0]; } + else if (buf_view.count > 1) { + err = ncmpio_type_create_hindexed(buf_view.count, buf_view.off, + buf_view.len, &bufType); + if (err == NC_NOERR) + bufCount = 1; + else if (status == NC_NOERR) + status = err; + } + /* else is for zero-sized request */ + + if (set_file_view) disp = 0; if (ncp->nprocs > 1 && coll_indep == NC_REQ_COLL) { -#ifdef HAVE_MPI_LARGE_COUNT - TRACE_IO(MPI_File_read_at_all_c, (fh, offset, xbuf, xlen, xbuf_type, &mpistatus)); -#else - TRACE_IO(MPI_File_read_at_all, (fh, offset, xbuf, xlen, xbuf_type, &mpistatus)); -#endif - } else { -#ifdef HAVE_MPI_LARGE_COUNT - TRACE_IO(MPI_File_read_at_c, (fh, offset, xbuf, xlen, xbuf_type, &mpistatus)); -#else - TRACE_IO(MPI_File_read_at, (fh, offset, xbuf, xlen, xbuf_type, &mpistatus)); -#endif + /* call MPI collective read */ + TRACE_IO(MPI_File_read_at_all, (fh, disp, xbuf_ptr, bufCount, + bufType, &mpistatus)); } + else { + /* call MPI independent read */ + TRACE_IO(MPI_File_read_at, (fh, disp, xbuf_ptr, bufCount, + bufType, &mpistatus)); + } + if (mpireturn != MPI_SUCCESS) { err = ncmpii_error_mpi2nc(mpireturn, mpi_name); - /* return the first encountered error if there is any */ - if (status == NC_NOERR) { - err = (err == NC_EFILE) ? NC_EREAD : err; - DEBUG_ASSIGN_ERROR(status, err) - } + if (err == NC_EFILE) DEBUG_ASSIGN_ERROR(err, NC_EREAD) + if (status == NC_NOERR) status = err; } - else { - /* update the number of bytes read since file open */ -#ifdef HAVE_MPI_GET_COUNT_C - MPI_Count get_size; - MPI_Get_count_c(&mpistatus, MPI_BYTE, &get_size); - ncp->get_size += get_size; -#else - int get_size; - mpireturn = MPI_Get_count(&mpistatus, xbuf_type, &get_size); - if (mpireturn != MPI_SUCCESS || get_size == MPI_UNDEFINED) - ncp->get_size += req_size; - else { -#ifdef HAVE_MPI_TYPE_SIZE_X - /* MPI_Type_size_x is introduced in MPI 3.0 */ - mpireturn = MPI_Type_size_x(xbuf_type, &btype_size); -#else - mpireturn = MPI_Type_size(xbuf_type, &btype_size); -#endif - if (mpireturn != MPI_SUCCESS || get_size == MPI_UNDEFINED) - ncp->get_size += req_size; - else - ncp->get_size += btype_size * get_size; + else + /* update the number of bytes read */ + rlen = get_count(&mpistatus, MPI_BYTE); + + if (set_file_view) /* reset the file view to entire file visible */ + MPI_File_set_view(fh, 0, MPI_BYTE, MPI_BYTE, "native", + MPI_INFO_NULL); + + if (bufType != MPI_BYTE) MPI_Type_free(&bufType); + } +#if PNETCDF_DRIVER_GIO == 1 + else if (ncp->driver == PNC_DRIVER_GIO) { + + if (ncp->gio_fh == GIO_FILE_NULL) { + /* If this process has not opened the file, this process must be a + * non-INA aggregator calling from an independent get API. + */ + assert(coll_indep == NC_REQ_INDEP); + + /* Check file open mode */ + int amode = fIsSet(ncp->flags, NC_MODE_RDONLY) ? O_RDONLY : O_RDWR; + err = GIO_open(MPI_COMM_SELF, ncp->path, amode, ncp->info, + &ncp->gio_fh); + if (err != GIO_NOERR) { + err = ncmpii_error_gio2nc(err, "GIO_open"); + DEBUG_FOPEN_ERROR(err); + if (status == NC_NOERR) status = err; } + } + + if (ncp->nprocs > 1 && coll_indep == NC_REQ_COLL) + rlen = GIO_read_all(ncp->gio_fh, xbuf, + file_view.count, + file_view.off, + file_view.len, + buf_view.count, + buf_view.off, + buf_view.len); + else + rlen = GIO_read(ncp->gio_fh, xbuf, + file_view.count, + file_view.off, + file_view.len, + buf_view.count, + buf_view.off, + buf_view.len); + + if (status == NC_NOERR && rlen < 0) status = (int)rlen; + } #endif + else { + if (status == NC_NOERR) status = NC_EDRIVER; + } + + if (xbuf != buf) { /* unpack contiguous xbuf to noncontiguous buf */ + char *in_ptr, *out_ptr; + in_ptr = xbuf; + + for (i=0; i= 0) ncp->get_size += rlen; + + return (status == NC_NOERR) ? rlen : status; +} + +/*----< ncmpio_file_write() >------------------------------------------------*/ +/* For zero-sized requests, file_view.count == buf_view.count == 0. + * For non-zero sized requests, file_view.count > 0 and buf_view.count > 0 and + * both file_view's off and len should not be NULL. The same for buf_view. + * Accumulated amount of file_view and buf_view should be equal. + * + * This subroutine may be called from either collective or independent data + * mode. Argument 'coll_indep' indicates the caller intends to perform a + * collective or independent put request, regardless of which data mode the + * program is currently on. + */ +MPI_Offset +ncmpio_file_write(NC *ncp, + int coll_indep, /* NC_REQ_INDEP or NC_REQ_COLL */ + const void *buf, + PNCIO_View file_view, + PNCIO_View buf_view) +{ + char *xbuf; + int i, status=NC_NOERR, err=NC_NOERR; + MPI_Offset wlen=0, off_zero=0, f_amnt, b_amnt; + + /* If zero-sized request and independent write, this rank can return now. */ + if (buf_view.count == 0 && coll_indep == NC_REQ_INDEP) + return NC_NOERR; + +#if PNETCDF_DEBUG_MODE == 1 + assert(file_view.count >= 0); + if (file_view.count > 0) { + assert(file_view.off != NULL); + assert(file_view.len != NULL); + } + assert(buf_view.count >= 0); + if (buf_view.count > 0) { + assert(buf_view.off != NULL); + assert(buf_view.len != NULL); + } #endif + + /* Calculate file_view amount in bytes, may be > NC_MAX_INT */ + for (f_amnt=0, i=0; i NC_MAX_INT */ + for (b_amnt=0, i=0; i 1 && b_amnt <= ncp->ibuf_size) { + /* If this write buffer is noncontiguous and amount is less than + * ncp->ibuf_size, we pack it into a temporary contiguous buffer and + * use it to write to the file. As some MPI, e.g. Cray on KNL, can be + * significantly slow when the write buffer is noncontiguous. The only + * case of write buffer being noncontiguous is when nonblocking API + * ncmpi_wait/wait_all() is called and INA is disabled. + * + * Note ncp->ibuf_size is never > NC_MAX_INT. + */ + char *in_ptr, *out_ptr; + xbuf = NCI_Malloc(b_amnt); + out_ptr = xbuf; + + for (i=0; idriver == PNC_DRIVER_MPIIO) { + char *xbuf_ptr, *mpi_name; + int bufCount, set_file_view, mpireturn; + MPI_Offset disp; + MPI_File fh; + MPI_Status mpistatus; + MPI_Datatype fileType=MPI_BYTE, bufType=MPI_BYTE; + + memset(&mpistatus, 0, sizeof(MPI_Status)); + + /* when ncp->nprocs == 1, ncp->mpio_fh_coll == ncp->mpio_fh_indep */ + fh = (ncp->nprocs > 1 && !fIsSet(ncp->flags, NC_MODE_INDEP)) + ? ncp->mpio_fh_coll : ncp->mpio_fh_indep; + + /* When in collective data mode, if a process makes an independent put + * call, skip setting the file view. This must come from subroutines + * that would like to access file header or perform data section + * movement at ncmpi_enddef(), due to new metadata was added. In these + * cases, their file views are always contiguous. + * + * In all other cases, we must set the file view. + */ + if (!fIsSet(ncp->flags, NC_MODE_INDEP) && coll_indep == NC_REQ_INDEP) { + set_file_view = 0; + assert(file_view.count == 1); + } + else + set_file_view = 1; + + disp = 0; + if (file_view.count == 1) { /* no need to create fileType */ + disp = file_view.off[0]; + } + else if (file_view.count > 1) { + /* Construct a file type for setting the file view. */ + err = ncmpio_type_create_hindexed(file_view.count, file_view.off, + file_view.len, &fileType); + if (err != NC_NOERR && status == NC_NOERR) status = err; + } + + if (set_file_view) { + TRACE_IO(MPI_File_set_view, (fh, disp, MPI_BYTE, fileType, + "native", MPI_INFO_NULL)); if (mpireturn != MPI_SUCCESS) { err = ncmpii_error_mpi2nc(mpireturn, mpi_name); - if (coll_indep == NC_REQ_COLL) - DEBUG_ASSIGN_ERROR(status, err) - else - DEBUG_RETURN_ERROR(err) + if (status == NC_NOERR) status = err; } - NCI_Free(xbuf); } - if (xbuf_type != buf_type && xbuf_type != MPI_BYTE) - MPI_Type_free(&xbuf_type); - } else { /* NC_REQ_WR */ - void *xbuf=buf; - MPI_Datatype xbuf_type=buf_type; - -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count xlen = (MPI_Count)buf_count; -#else - int xlen = (int)buf_count; - if (buf_count > NC_MAX_INT) { - if (coll_indep == NC_REQ_COLL) { -#ifdef PNETCDF_DEBUG - fprintf(stderr,"%d: %s line %d: NC_EINTOVERFLOW buf_count="OFFFMT"\n", - ncp->rank, __func__,__LINE__,buf_count); -#endif - DEBUG_ASSIGN_ERROR(status, NC_EINTOVERFLOW) - /* write nothing, but participate the collective call */ - xlen = 0; - } - else - DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) - } -#endif + if (fileType != MPI_BYTE) MPI_Type_free(&fileType); - if (xlen > 0 && !buftype_is_contig && req_size <= ncp->ibuf_size) { - /* if write buffer is noncontiguous and size is < ncp->ibuf_size, - * allocate a temporary buffer and use it to write, as some MPI, - * e.g. Cray on KNL, can be significantly slow when write buffer is - * noncontiguous. - */ -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count pos=0; - xbuf_type = MPI_BYTE; - xlen = (MPI_Count)req_size; - xbuf = NCI_Malloc(req_size); - mpireturn = MPI_Pack_c(buf, (MPI_Count)buf_count, buf_type, xbuf, - (MPI_Count)req_size, &pos, MPI_COMM_SELF); - mpi_name = "MPI_Pack_c"; -#else - if (req_size > NC_MAX_INT) { - /* skip packing write data into a temp buffer */ - xlen = (int)buf_count; - xbuf_type = buf_type; - mpireturn = MPI_SUCCESS; - } + /* Construct a derived data type describing user buffer data layout to + * be used in MPI_File_write_at(). + */ + bufCount = 0; + xbuf_ptr = xbuf; + if (buf_view.count == 1) { /* buffer view is contiguous */ + if (buf_view.len[0] <= INT_MAX) + bufCount = (int)buf_view.len[0]; else { - int pos=0; - xbuf_type = MPI_BYTE; - xlen = (int)req_size; - xbuf = NCI_Malloc(xlen); - mpireturn = MPI_Pack(buf, (int)buf_count, buf_type, xbuf, - xlen, &pos, MPI_COMM_SELF); - mpi_name = "MPI_Pack"; - } -#endif - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, mpi_name); - if (coll_indep == NC_REQ_COLL) - DEBUG_ASSIGN_ERROR(status, err) - else - DEBUG_RETURN_ERROR(err) + /* Try creating bufType */ + err = ncmpio_type_contiguous(buf_view.len[0], &bufType); + if (err == NC_NOERR) + bufCount = 1; + else { /* make this a zero-size request */ + bufCount = 0; + if (status == NC_NOERR) + status = err; + } } + xbuf_ptr += buf_view.off[0]; } + else if (buf_view.count > 1) { + err = ncmpio_type_create_hindexed(buf_view.count, buf_view.off, + buf_view.len, &bufType); + if (err == NC_NOERR) + bufCount = 1; + else if (status == NC_NOERR) + status = err; + } + /* else is for zero-sized request */ + + if (set_file_view) disp = 0; if (ncp->nprocs > 1 && coll_indep == NC_REQ_COLL) { -#ifdef HAVE_MPI_LARGE_COUNT - TRACE_IO(MPI_File_write_at_all_c, (fh, offset, xbuf, xlen, xbuf_type, &mpistatus)); -#else - TRACE_IO(MPI_File_write_at_all, (fh, offset, xbuf, xlen, xbuf_type, &mpistatus)); -#endif - } else { -#ifdef HAVE_MPI_LARGE_COUNT - TRACE_IO(MPI_File_write_at_c, (fh, offset, xbuf, xlen, xbuf_type, &mpistatus)); -#else - TRACE_IO(MPI_File_write_at, (fh, offset, xbuf, xlen, xbuf_type, &mpistatus)); -#endif + /* call MPI collective write */ + TRACE_IO(MPI_File_write_at_all, (fh, disp, xbuf_ptr, bufCount, + bufType, &mpistatus)); } + else { + /* call MPI independent write */ + TRACE_IO(MPI_File_write_at, (fh, disp, xbuf_ptr, bufCount, + bufType, &mpistatus)); + } + if (mpireturn != MPI_SUCCESS) { err = ncmpii_error_mpi2nc(mpireturn, mpi_name); - /* return the first encountered error if there is any */ - if (status == NC_NOERR) { - err = (err == NC_EFILE) ? NC_EWRITE : err; - DEBUG_ASSIGN_ERROR(status, err) - } + if (err == NC_EFILE) DEBUG_ASSIGN_ERROR(err, NC_EWRITE) + if (status == NC_NOERR) status = err; } - else { - /* update the number of bytes written since file open */ -#ifdef HAVE_MPI_GET_COUNT_C - MPI_Count put_size; - MPI_Get_count_c(&mpistatus, MPI_BYTE, &put_size); - ncp->put_size += put_size; -#else - int put_size; - mpireturn = MPI_Get_count(&mpistatus, xbuf_type, &put_size); - if (mpireturn != MPI_SUCCESS || put_size == MPI_UNDEFINED) - ncp->put_size += req_size; - else { -#ifdef HAVE_MPI_TYPE_SIZE_X - /* MPI_Type_size_x is introduced in MPI 3.0 */ - mpireturn = MPI_Type_size_x(xbuf_type, &btype_size); -#else - mpireturn = MPI_Type_size(xbuf_type, &btype_size); -#endif - if (mpireturn != MPI_SUCCESS || put_size == MPI_UNDEFINED) - ncp->put_size += req_size; - else - ncp->put_size += btype_size * put_size; + else + /* update the number of bytes written */ + wlen = get_count(&mpistatus, MPI_BYTE); + + if (set_file_view) /* reset the file view to entire file visible */ + MPI_File_set_view(fh, 0, MPI_BYTE, MPI_BYTE, "native", + MPI_INFO_NULL); + + if (bufType != MPI_BYTE) MPI_Type_free(&bufType); + } +#if PNETCDF_DRIVER_GIO == 1 + else if (ncp->driver == PNC_DRIVER_GIO) { + + if (ncp->gio_fh == GIO_FILE_NULL) { + /* If this process has not opened the file, this process must be a + * non-INA aggregator calling from an independent put API. + */ + assert(coll_indep == NC_REQ_INDEP); + + /* Check file open mode */ + int amode = fIsSet(ncp->flags, NC_MODE_RDONLY) ? O_RDONLY : O_RDWR; + err = GIO_open(MPI_COMM_SELF, ncp->path, amode, ncp->info, + &ncp->gio_fh); + if (err != GIO_NOERR) { + err = ncmpii_error_gio2nc(err, "GIO_open"); + DEBUG_FOPEN_ERROR(err); + if (status == NC_NOERR) status = err; } -#endif } - if (xbuf != buf) NCI_Free(xbuf); - if (xbuf_type != buf_type && xbuf_type != MPI_BYTE) - MPI_Type_free(&xbuf_type); + + if (ncp->nprocs > 1 && coll_indep == NC_REQ_COLL) + wlen = GIO_write_all(ncp->gio_fh, xbuf, + file_view.count, + file_view.off, + file_view.len, + buf_view.count, + buf_view.off, + buf_view.len); + else + wlen = GIO_write(ncp->gio_fh, xbuf, + file_view.count, + file_view.off, + file_view.len, + buf_view.count, + buf_view.off, + buf_view.len); + + if (status == NC_NOERR && wlen < 0) status = (int)wlen; + } +#endif + else { + if (status == NC_NOERR) status = NC_EDRIVER; } - return status; + if (xbuf != buf) NCI_Free(xbuf); + + /* update the number of bytes written since file open */ + if (wlen >= 0) ncp->put_size += wlen; + + return (status == NC_NOERR) ? wlen : status; } diff --git a/src/drivers/ncmpio/ncmpio_file_misc.c b/src/drivers/ncmpio/ncmpio_file_misc.c index 932b5027f7..e954e75de6 100644 --- a/src/drivers/ncmpio/ncmpio_file_misc.c +++ b/src/drivers/ncmpio/ncmpio_file_misc.c @@ -24,6 +24,7 @@ #include #include #include /* strcpy() */ +#include /* unlink() */ #include #include @@ -59,19 +60,16 @@ dup_NC(const NC *ref) return NULL; } - if (ref->nonaggr_ranks != NULL) { - size_t len = sizeof(int) * ncp->num_nonaggrs; - ncp->nonaggr_ranks = (int*) NCI_Malloc(len); - memcpy(ncp->nonaggr_ranks, ref->nonaggr_ranks, len); - } + /* copy communicator attribute over */ + ncp->comm_attr = ref->comm_attr; /* fields below should not copied from ref */ - ncp->comm = MPI_COMM_NULL; - ncp->mpiinfo = MPI_INFO_NULL; - ncp->get_list = NULL; - ncp->put_list = NULL; - ncp->abuf = NULL; - ncp->path = NULL; + ncp->comm = MPI_COMM_NULL; + ncp->info = MPI_INFO_NULL; + ncp->get_list = NULL; + ncp->put_list = NULL; + ncp->abuf = NULL; + ncp->path = NULL; return ncp; } @@ -81,8 +79,7 @@ dup_NC(const NC *ref) int ncmpio_redef(void *ncdp) { - char *mpi_name; - int err, status=NC_NOERR, mpireturn; + int status=NC_NOERR; NC *ncp = (NC*)ncdp; #if 0 @@ -100,30 +97,13 @@ ncmpio_redef(void *ncdp) if (NC_indep(ncp)) /* exit independent mode, if in independent mode */ ncmpio_end_indep_data(ncp); - /* duplicate a header to be used in enddef() for checking if header grows */ + /* duplicate header to be used in enddef() for checking if header grows */ ncp->old = dup_NC(ncp); if (ncp->old == NULL) DEBUG_RETURN_ERROR(NC_ENOMEM) /* we are now entering define mode */ fSet(ncp->flags, NC_MODE_DEF); - /* must reset fileview as header extent may later change in enddef() */ - TRACE_IO(MPI_File_set_view, (ncp->collective_fh, 0, MPI_BYTE, - MPI_BYTE, "native", MPI_INFO_NULL)); - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, mpi_name); - DEBUG_ASSIGN_ERROR(status, err) - } - - if (ncp->independent_fh != MPI_FILE_NULL) { - TRACE_IO(MPI_File_set_view, (ncp->independent_fh, 0, MPI_BYTE, - MPI_BYTE, "native", MPI_INFO_NULL)); - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, mpi_name); - DEBUG_ASSIGN_ERROR(status, err) - } - } - return status; } @@ -132,7 +112,6 @@ ncmpio_redef(void *ncdp) int ncmpio_begin_indep_data(void *ncdp) { - char *mpi_name; NC *ncp = (NC*)ncdp; if (NC_indef(ncp)) /* must not be in define mode */ @@ -146,25 +125,59 @@ ncmpio_begin_indep_data(void *ncdp) */ /* If users want a stronger data consistency, ncmpi_sync() should be called - * following this subroutine. */ + * following this subroutine. + */ /* raise independent flag */ fSet(ncp->flags, NC_MODE_INDEP); - /* PnetCDF's default mode is collective. MPI file handle, collective_fh, + /* Barrier is necessary to prevent non-aggregators from calling open() + * before the file is being collectively created by the aggregators. + * It is also necessary when fill mode is enabled. A barrier prevents one + * process start to do I/O while some other processes are still filling + * the variables. + */ + MPI_Barrier(ncp->comm); + +#if PNETCDF_DRIVER_GIO == 1 + if (ncp->driver == PNC_DRIVER_GIO) { + /* GIO driver implements open-on-demand mechanism. */ + return NC_NOERR; + } +#endif + + /* PnetCDF's default mode is collective. MPI file handle, mpio_fh_coll, * will never be MPI_FILE_NULL. We must use a separate MPI file handle * opened with MPI_COMM_SELF, because MPI_File_set_view is a collective * call and accessing a subarray requires a call to MPI_File_set_view. * In independent data mode, no collective MPI operation can be implicitly * called. */ - if (ncp->independent_fh == MPI_FILE_NULL) { + if (ncp->mpio_fh_indep == MPI_FILE_NULL) { + char *mpi_name; int mpireturn; - TRACE_IO(MPI_File_open, (MPI_COMM_SELF, ncp->path, - ncp->mpiomode, ncp->mpiinfo, - &ncp->independent_fh)); +#ifdef MPICH_VERSION + /* MPICH recognizes file system type acronym prefixed to file name */ + TRACE_IO(MPI_File_open, (MPI_COMM_SELF, ncp->path, ncp->mpi_amode, + ncp->info, &ncp->mpio_fh_indep)); +#else + char *path = ncmpii_remove_file_system_type_prefix(ncp->path); + TRACE_IO(MPI_File_open, (MPI_COMM_SELF, path, ncp->mpi_amode, + ncp->info, &ncp->mpio_fh_indep)); +#endif if (mpireturn != MPI_SUCCESS) return ncmpii_error_mpi2nc(mpireturn, mpi_name); + + /* for those ranks whose info is NULL, retrieve info */ + if (ncp->info == MPI_INFO_NULL) { + /* get the I/O hints used/modified by MPI-IO */ + mpireturn = MPI_File_get_info(ncp->mpio_fh_indep, &ncp->info); + if (mpireturn != MPI_SUCCESS) + return ncmpii_error_mpi2nc(mpireturn, mpi_name); + + /* Copy MPI-IO hints into ncp->info */ + ncmpio_hint_set(ncp, ncp->info); + } } return NC_NOERR; } @@ -185,7 +198,7 @@ ncmpio_end_indep_data(void *ncdp) if (NC_indef(ncp)) /* must not be in define mode */ DEBUG_RETURN_ERROR(NC_EINDEFINE) - if (!NC_indep(ncp)) /* already in collective ata mode */ + if (!NC_indep(ncp)) /* already in collective data mode */ return NC_NOERR; /* starting from 1.9.0, calling end_indep_data() in collective data * mode is no longer considered illegal @@ -215,7 +228,7 @@ ncmpio_abort(void *ncdp) { /* * In data mode, same as ncmpi_close(). - * In define mode, descard new definition. + * In define mode, discard new definition. * If file is just created, remove the file. */ int status=NC_NOERR, err, doUnlink = 0; @@ -242,9 +255,14 @@ ncmpio_abort(void *ncdp) } /* close the file */ - err = ncmpio_close_files(ncp, doUnlink); + err = ncmpio_file_close(ncp); if (status == NC_NOERR ) status = err; + if (doUnlink) { + err = ncmpio_file_delete(ncp); + status = (status == NC_NOERR) ? err : status; + } + /* free up space occupied by the header metadata */ ncmpio_free_NC(ncp); @@ -289,7 +307,7 @@ ncmpio_inq_misc(void *ncdp, MPI_Offset *usage, MPI_Offset *buf_size) { - int i, flag, mpireturn; + int i, mpireturn; char value[MPI_MAX_INFO_VAL]; NC *ncp=(NC*)ncdp; @@ -335,27 +353,11 @@ ncmpio_inq_misc(void *ncdp, /* obtain file (system) striping settings, striping size and count, if they * are available from MPI-IO hint. Otherwise, 0s are returned. */ - if (striping_size != NULL) { - MPI_Info_get(ncp->mpiinfo, "striping_unit", MPI_MAX_INFO_VAL-1, - value, &flag); - *striping_size = 0; - if (flag) { - errno = 0; /* errno must set to zero before calling strtoll */ - *striping_size = (int)strtol(value,NULL,10); - if (errno != 0) *striping_size = 0; - } - } + if (striping_size != NULL) + *striping_size = ncp->striping_unit; - if (striping_count != NULL) { - MPI_Info_get(ncp->mpiinfo, "striping_factor", MPI_MAX_INFO_VAL-1, - value, &flag); - *striping_count = 0; - if (flag) { - errno = 0; /* errno must set to zero before calling strtoll */ - *striping_count = (int)strtol(value,NULL,10); - if (errno != 0) *striping_count = 0; - } - } + if (striping_count != NULL) + *striping_count = ncp->striping_factor; /* the amount of writes, in bytes, committed to file system so far */ if (put_size != NULL) *put_size = ncp->put_size; @@ -365,20 +367,30 @@ ncmpio_inq_misc(void *ncdp, if (recsize != NULL) *recsize = ncp->recsize; - if (header_size != NULL) *header_size = ncp->xsz; + if (header_size != NULL) { + if (NC_indef(ncp)) + /* When called in define mode, calculate and return the current + * header size. Cannot do the same for header extent, as the empty + * space depends on arguments h_minfree and v_align of + * ncmpi__enddef(). + */ + *header_size = ncmpio_hdr_len_NC(ncp); + else + *header_size = ncp->xsz; + } if (header_extent != NULL) *header_extent = ncp->begin_var; if (info_used != NULL) { - mpireturn = MPI_Info_dup(ncp->mpiinfo, info_used); + mpireturn = MPI_Info_dup(ncp->info, info_used); if (mpireturn != MPI_SUCCESS) return ncmpii_error_mpi2nc(mpireturn, "MPI_Info_dup"); - /* PnetCDF hints have been added to ncp->mpiinfo at ncmpi_enddef. + /* PnetCDF hints have been added to ncp->info at ncmpi_enddef. * * Note MPI implementations may choose to ignore unrecognized hints and * MPI_File_get_info() may returns no PnetCDF hints. We need to add the - * PnbetCDF hints explicitly to the info object brefore returning it to + * PnetCDF hints explicitly to the info object before returning it to * user. */ @@ -388,9 +400,12 @@ ncmpio_inq_misc(void *ncdp, sprintf(value, OFFFMT, ncp->r_align); MPI_Info_set(*info_used, "nc_record_align_size", value); - sprintf(value, "%d", ncp->chunk); + sprintf(value, "%d", ncp->hdr_chunk); MPI_Info_set(*info_used, "nc_header_read_chunk_size", value); + sprintf(value, "%d", ncp->data_chunk); + MPI_Info_set(*info_used, "nc_data_move_chunk_size", value); + if (fIsSet(ncp->flags, NC_MODE_SWAP_ON)) MPI_Info_set(*info_used, "nc_in_place_swap", "enable"); else if (fIsSet(ncp->flags, NC_MODE_SWAP_OFF)) @@ -401,7 +416,7 @@ ncmpio_inq_misc(void *ncdp, sprintf(value, OFFFMT, ncp->ibuf_size); MPI_Info_set(*info_used, "nc_ibuf_size", value); -#ifdef ENABLE_SUBFILING +#if PNETCDF_SUBFILING == 1 if (ncp->subfile_mode) MPI_Info_set(*info_used, "pnetcdf_subfiling", "enable"); else @@ -444,17 +459,27 @@ int ncmpi_delete(const char *filename, MPI_Info info) { + int err = NC_NOERR; + +#ifdef HAVE_UNLINK + char *path = ncmpii_remove_file_system_type_prefix(filename); + err = unlink(path); + if (err != 0) + err = ncmpii_error_posix2nc("unlink"); +#else char *mpi_name; - int err=NC_NOERR, mpireturn; + int mpireturn; - TRACE_IO(MPI_File_delete, ((char*)filename, info)); + TRACE_IO(MPI_File_delete, (filename, info)); if (mpireturn != MPI_SUCCESS) err = ncmpii_error_mpi2nc(mpireturn, mpi_name); +#endif + return err; } -/*----< ncmpio_flush() >------------------------------------------------------*/ +/*----< ncmpio_flush() >-----------------------------------------------------*/ /* This API is a collective subroutine, and must be called in data mode */ int diff --git a/src/drivers/ncmpio/ncmpio_filetype.c b/src/drivers/ncmpio/ncmpio_filetype.c deleted file mode 100644 index 828ab41325..0000000000 --- a/src/drivers/ncmpio/ncmpio_filetype.c +++ /dev/null @@ -1,710 +0,0 @@ -/* - * Copyright (C) 2003, Northwestern University and Argonne National Laboratory - * See COPYRIGHT notice in top-level directory. - */ -/* $Id$ */ - -#ifdef HAVE_CONFIG_H -# include -#endif - -#include -#ifdef HAVE_STDLIB_H -#include -#endif -#include - -#include - -#include -#include -#include -#include "ncmpio_NC.h" - - -#if SIZEOF_MPI_AINT != SIZEOF_MPI_OFFSET -/*----< check_recsize_too_big() >--------------------------------------------*/ -/* Because recsize (sum of single record of all record variables) will be used - * to as the "offset stride" when constructing the file view, we must ensure - * there is no integer overflow. Note records of all variables are interleaved, - * for example record i of all record variables followed by record i+1 of all - * record variables, the stride across two records of a variable can be too - * large for a 4-byte integer to represent. - */ -static int -check_recsize_too_big(MPI_Offset recsize) -{ - int ret = NC_NOERR; - - if (recsize != (MPI_Aint)recsize) { - fprintf(stderr, "Type overflow: unable to read/write multiple records in this dataset\non this platform. Please either access records of this record variable\none-at-a-time or run on a 64 bit platform\n"); - DEBUG_ASSIGN_ERROR(ret, NC_ESMALL) - } - /* the assert here might be harsh, but without it, users will get corrupt - * data. Now, we just skip this request to avoid this assertion. */ - /* assert (recsize == (MPI_Aint)recsize); */ - return ret; -} -#endif - -/*----< is_request_contiguous() >-------------------------------------------*/ -/* check if a request, represented by start[] and count[], is contiguous in - * file. - */ -static int -is_request_contiguous(int isRecVar, - int numRecVars, - int ndims, - const MPI_Offset *shape, - const MPI_Offset *start, - const MPI_Offset *count) -{ - int i, j, most_sig_dim; - - if (ndims == 0) return 1; /* this variable is a scalar */ - - for (i=0; i 1, - * then this request is noncontiguous. - */ - if (numRecVars > 1) { - if (count[0] > 1) return 0; - - /* continue to check from dimension ndims-1 up to dimension 1 */ - most_sig_dim = 1; - } - /* if there is only one record variable, then we need to check - * dimension ndims-1 up to dimension 0 */ - } - - for (i=ndims-1; i>most_sig_dim; i--) { - /* find the first count[i] that is not equal to the entire dimension */ - if (count[i] < shape[i]) { - /* the request is contiguous only if count[i-1], count[i-2], ... - * count[0] are all 1s. */ - for (j=i-1; j>=most_sig_dim; j--) { - if (count[j] > 1) - return 0; - } - break; - } -#if 0 - else { /* count[i] == shape[i] */ - /* when accessing the entire dimension, start[i] must be 0 */ - if (start[i] != 0) return 0; /* actually this should be error */ - } -#endif - } - return 1; -} - -/*----< type_create_subarray64() >-------------------------------------------*/ -/* This subroutine is to achieve the same result as MPI_Type_create_subarray() - * but it takes arguments in type of MPI_Offset, instead of int. It also - * checked for any possible 4-byte integer overflow. - */ -static int -type_create_subarray64(int ndims, - const MPI_Offset *array_of_sizes, /* [ndims] */ - const MPI_Offset *array_of_subsizes, /* [ndims] */ - const MPI_Offset *array_of_starts, /* [ndims] */ - int order, - MPI_Datatype oldtype, - MPI_Datatype *newtype) -{ - int i, err=NC_NOERR, mpireturn; - - if (ndims == 0) DEBUG_RETURN_ERROR(NC_EDIMMETA) - -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count *sizes, *subsizes, *starts; - - sizes = (MPI_Count*) NCI_Malloc(sizeof(MPI_Count) * ndims * 3); - subsizes = sizes + ndims; - starts = subsizes + ndims; - for (i=0; i NC_MAX_INT || array_of_starts[i] > NC_MAX_INT) { - big_int = 1; - break; - } - } - - if (big_int == 0) { - int *sizes, *subsizes, *starts; - /* none of dimensions > 2^31-1, we can safely use - * MPI_Type_create_subarray */ - sizes = (int*) NCI_Malloc(sizeof(int) * ndims * 3); - subsizes = sizes + ndims; - starts = subsizes + ndims; - for (i=0; i 2^31-1 and we cannot use - * MPI_Type_create_subarray() to create the newtype, - * as its arguments array_of_sizes[] and array_of_starts[] are of - * type int. One solution is to use a combination of - * MPI_Type_create_hvector(), MPI_Type_create_hindexed(), - * MPI_Type_create_resized(), and MPI_Type_struct(), as one - * of their arguments, stride and indices[], are of type MPI_Aint - * (a possible 8-byte integer) that can be used to store the value - * of the dimension whose size is > 2^31-1. However, on a machine - * where MPI_Aint is 4-byte integer, those MPI_Aint arguments will - * cause overflow. - */ -#if SIZEOF_MPI_AINT != SIZEOF_MPI_OFFSET - DEBUG_RETURN_ERROR(NC_EAINT_TOO_SMALL) -#endif - - mpireturn = MPI_Type_get_extent(oldtype, &lb, &extent); - if (mpireturn != MPI_SUCCESS) - return ncmpii_error_mpi2nc(mpireturn, "MPI_Type_get_extent"); - array_size = extent; - for (i=0; i NC_MAX_INT) /* check int overflow */ - DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) - blklens[1] = (int)array_of_subsizes[0]; - disps[1] = extent * array_of_starts[0]; - - /* take advantage of disps argument is of type MPI_Aint */ - mpireturn = MPI_Type_create_hindexed(1, &blklens[1], &disps[1], oldtype, &type1); - if (mpireturn != MPI_SUCCESS) - return ncmpii_error_mpi2nc(mpireturn, "MPI_Type_create_hindexed"); - MPI_Type_commit(&type1); - - /* add holes in the beginning and tail of type1 */ - mpireturn = MPI_Type_create_resized(type1, 0, array_size, newtype); - if (mpireturn != MPI_SUCCESS) - return ncmpii_error_mpi2nc(mpireturn, "MPI_Type_create_resized"); - MPI_Type_free(&type1); - - return NC_NOERR; - } - /* now, ndims > 1 */ - - /* first create a datatype for the least 2 significant dimensions */ - - /* count and blocklength arguments in MPI_Type_create_hvector() are of - * type int. We need to check for integer overflow */ - int count, blocklength; - - if (array_of_subsizes[ndims-2] > NC_MAX_INT || - array_of_subsizes[ndims-1] > NC_MAX_INT) /* check int overflow */ - DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) - - count = (int)array_of_subsizes[ndims-2]; - blocklength = (int)array_of_subsizes[ndims-1]; - stride = array_of_sizes[ndims-1] * extent; - - mpireturn = MPI_Type_create_hvector(count, blocklength, stride, oldtype, &type1); - if (mpireturn != MPI_SUCCESS) - return ncmpii_error_mpi2nc(mpireturn, "MPI_Type_create_hvector"); - - MPI_Type_commit(&type1); - - /* now iterate through the rest dimensions */ - for (i=ndims-3; i>=0; i--) { - if (array_of_subsizes[i] > NC_MAX_INT) /* check int overflow */ - DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) - - count = (int)array_of_subsizes[i]; - stride *= array_of_sizes[i+1]; - - mpireturn = MPI_Type_create_hvector(count, 1, stride, type1, &type2); - if (mpireturn != MPI_SUCCESS) - return ncmpii_error_mpi2nc(mpireturn, "MPI_Type_create_hvector"); - - MPI_Type_commit(&type2); - MPI_Type_free(&type1); - type1 = type2; - } - - /* disps[0] is the displacement to the beginning of the global array */ - disps[0] = 0; - - /* disps[1] is the first byte displacement of the subarray */ - disps[1] = array_of_starts[ndims-1] * extent; - size = extent; - for (i=ndims-2; i>=0; i--) { - size *= array_of_sizes[i+1]; - disps[1] += size * array_of_starts[i]; - } - - /* disps[2] is the size of the global array */ - disps[2] = array_size; - - /* make filetype the same as calling MPI_Type_create_subarray() */ - /* adjust LB and UB without using MPI_LB or MPI_UB */ - mpireturn = MPI_Type_create_hindexed(1, blklens, &disps[1], type1, &type2); - if (mpireturn != MPI_SUCCESS) - return ncmpii_error_mpi2nc(mpireturn, "MPI_Type_create_hindexed"); - MPI_Type_commit(&type2); - mpireturn = MPI_Type_create_resized(type2, disps[0], disps[2], newtype); - if (mpireturn != MPI_SUCCESS) - return ncmpii_error_mpi2nc(mpireturn, "MPI_Type_create_resized"); - MPI_Type_free(&type2); - MPI_Type_free(&type1); - - return NC_NOERR; -#endif -} - -/*----< filetype_create_vara() >--------------------------------------------*/ -static int -filetype_create_vara(const NC *ncp, - const NC_var *varp, - const MPI_Offset *start, - const MPI_Offset *count, - MPI_Offset *offset_ptr, /* OUT */ - MPI_Datatype *filetype_ptr, /* OUT */ - int *is_filetype_contig) /* OUT */ -{ - int status, mpireturn; - MPI_Offset offset; - MPI_Datatype filetype, xtype; - - *offset_ptr = varp->begin; - *filetype_ptr = MPI_BYTE; - if (is_filetype_contig != NULL) *is_filetype_contig = 1; - - /* when varp is a scalar, no need to create a filetype */ - if (varp->ndims == 0) return NC_NOERR; - - /* if the request is contiguous in file, no need to create a filetype */ - if (is_request_contiguous(IS_RECVAR(varp), ncp->vars.num_rec_vars, - varp->ndims, varp->shape, start, count)) { - /* find the starting file offset of this request */ - status = ncmpio_first_offset(ncp, varp, start, &offset); - *offset_ptr = offset; - return status; - } - - /* hereinafter fileview is noncontiguous, i.e. filetype != MPI_BYTE */ - if (is_filetype_contig != NULL) *is_filetype_contig = 0; - offset = varp->begin; - - /* element MPI data type of variable */ - xtype = ncmpii_nc2mpitype(varp->xtype); - - /* previously, request size has been checked and it must > 0 */ - if (IS_RECVAR(varp)) { -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count blocklength; -#else - int blocklength; -#endif - MPI_Datatype rectype=MPI_BYTE; - -#if SIZEOF_MPI_AINT != SIZEOF_MPI_OFFSET - /* check overflow only if MPI_Aint is smaller than MPI_Offset */ - status = check_recsize_too_big(ncp->recsize); - if (status != NC_NOERR) return status; -#endif - offset += start[0] * ncp->recsize; - - if (varp->ndims > 1) { - /* when ndims > 1, we first construct a subarray type for a - * single record, i.e. for dimensions 1 ... ndims-1 */ - status = type_create_subarray64(varp->ndims-1, varp->shape+1, - count+1, start+1, MPI_ORDER_C, - xtype, &rectype); - if (status != NC_NOERR) return status; - - MPI_Type_commit(&rectype); - blocklength = 1; - } - else { /* no subarray datatype is needed */ - blocklength = varp->xsz; - } - -#ifdef HAVE_MPI_LARGE_COUNT - /* concatenate number of count[0] subarray types into filetype */ - mpireturn = MPI_Type_create_hvector_c(count[0], blocklength, ncp->recsize, - rectype, &filetype); - if (mpireturn != MPI_SUCCESS) - return ncmpii_error_mpi2nc(mpireturn, "MPI_Type_create_hvector_c"); -#else - /* check overflow, because 1st argument of hvector is of type int */ - if (count[0] > NC_MAX_INT) DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) - - /* concatenate number of count[0] subarray types into filetype */ - mpireturn = MPI_Type_create_hvector((int)count[0], blocklength, ncp->recsize, - rectype, &filetype); - if (mpireturn != MPI_SUCCESS) - return ncmpii_error_mpi2nc(mpireturn, "MPI_Type_create_hvector"); -#endif - - if (rectype != MPI_BYTE) MPI_Type_free(&rectype); - } - else { /* for non-record variable, just create a subarray datatype */ - status = type_create_subarray64(varp->ndims, varp->shape, count, start, - MPI_ORDER_C, xtype, &filetype); - if (status != NC_NOERR) return status; - } - MPI_Type_commit(&filetype); - - *offset_ptr = offset; - *filetype_ptr = filetype; - - return NC_NOERR; -} - -/*----< stride_flatten() >----------------------------------------------------*/ -/* flatten a stride request into a list of offset-length pairs stored in - * blocklens[] and disps[] - */ -static int -stride_flatten(const NC_var *varp, - MPI_Offset recsize, /* record size */ - const MPI_Offset *start, /* [ndim] starts of subarray */ - const MPI_Offset *count, /* [ndim] counts of subarray */ - const MPI_Offset *stride, /* [ndim] strides of subarray */ - MPI_Offset *nblocks, /* OUT: number of blocks */ - MPI_Offset **blocklens,/* OUT: length of each block */ - MPI_Aint **disps) /* OUT: displacement of each block */ -{ - int i, j, k, isRecVar, ndims, el_size; - MPI_Offset *dimlen, seg_len, nstride, array_len, off, subarray_len; - - *blocklens = NULL; - *disps = NULL; - - /* whether record variable */ - isRecVar = IS_RECVAR(varp); - - ndims = varp->ndims; - - /* scalar variables have been handled before this subroutine is called */ - assert (ndims > 0); - - /* array element size */ - el_size = varp->xsz; - - /* dimension lengths */ - dimlen = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset) * ndims); - - for (i=0; ishape[i]; - /* for record variable, set dimlen[0] to the record size */ - if (isRecVar) dimlen[0] = recsize; - - /* calculate the number of offset-length pairs */ - *nblocks = (stride[ndims-1] == 1) ? 1 : count[ndims-1]; - for (i=0; i 0) { - /* array_len is global array size from lowest up to ndims */ - array_len *= dimlen[ndims]; - - /* off is the global array offset for this dimension, ndims-1 - * For record variable, dimlen[0] is the sum of single record sizes - */ - if (ndims == 1 && isRecVar) off = start[0] * dimlen[0]; - else off = start[ndims-1] * array_len; - - /* update all offsets from lowest up to dimension ndims-1 */ - for (j=0; j--------------------------------------*/ -int -ncmpio_filetype_create_vars(const NC *ncp, - const NC_var *varp, - const MPI_Offset *start, - const MPI_Offset *count, - const MPI_Offset *stride, - MPI_Offset *offset_ptr, /* OUT */ - MPI_Datatype *filetype_ptr, /* OUT */ - int *is_filetype_contig) /* OUT */ -{ - int err=NC_NOERR, mpireturn, dim, isLargeReq; - MPI_Aint *disps; - MPI_Offset i, nblocks, nelems, *blocklens; - MPI_Datatype filetype=MPI_BYTE; - - if (stride == NULL) - return filetype_create_vara(ncp, varp, start, count, offset_ptr, - filetype_ptr, is_filetype_contig); - - /* check if a true vars (skip stride[] when count[] == 1) */ - for (dim=0; dimndims; dim++) - if (count[dim] > 1 && stride[dim] > 1) - break; - - if (dim == varp->ndims) /* not a true vars */ - return filetype_create_vara(ncp, varp, start, count, offset_ptr, - filetype_ptr, is_filetype_contig); - - /* now stride[] indicates a non-contiguous fileview */ - - /* calculate request amount */ - nelems = 1; - for (dim=0; dimndims; dim++) nelems *= count[dim]; - - /* starting file offset of the variable */ - *offset_ptr = varp->begin; - - /* when nelems == 0 or varp is a scalar, i.e. varp->ndims == 0, no need to - * create a filetype - */ - if (varp->ndims == 0 || nelems == 0) { - *filetype_ptr = MPI_BYTE; - if (is_filetype_contig != NULL) *is_filetype_contig = 1; - return NC_NOERR; - } - - /* hereinafter fileview is noncontiguous, i.e. filetype != MPI_BYTE */ - if (is_filetype_contig != NULL) *is_filetype_contig = 0; - - /* flatten stride access into list of offsets and lengths stored in - * disps[] and blocklens[], respectively. - */ - stride_flatten(varp, ncp->recsize, start, count, stride, &nblocks, - &blocklens, &disps); - - /* the flattened list allows one single call to hindexed constructor. - * We cannot use MPI_Type_indexed because displacement for the record - * dimension may not be a multiple of varp->xtype - */ - isLargeReq = 0; - if (nblocks > NC_MAX_INT) - isLargeReq = 1; - else { - for (i=0; i NC_MAX_INT) { - isLargeReq = 1; - break; - } - } - } - - if (isLargeReq) { -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count *blocklens_c, *disps_c; - blocklens_c = (MPI_Count*) NCI_Malloc(sizeof(MPI_Count) * nblocks); - disps_c = (MPI_Count*) NCI_Malloc(sizeof(MPI_Count) * nblocks); - for (i=0; i---------------------------------------------*/ -/* This function handles the special case for root process for setting its - * file view: to keeps the whole file header visible to the root process. This - * is because the root process may update the number of records or attributes - * into the file header while in data mode. In PnetCDF design, only root - * process can read/write the file header. - * This function is collective if called in collective data mode - */ -int -ncmpio_file_set_view(const NC *ncp, - MPI_File fh, - MPI_Offset *offset, /* IN/OUT */ - MPI_Datatype filetype) -{ - char *mpi_name; - int err, mpireturn, status=NC_NOERR; - - if (filetype == MPI_BYTE) { - /* filetype is a contiguous space, make the whole file visible */ - TRACE_IO(MPI_File_set_view, (fh, 0, MPI_BYTE, MPI_BYTE, - "native", MPI_INFO_NULL)); - return NC_NOERR; - } - - if (ncp->rank == 0) { - /* prepend the whole file header to filetype */ - MPI_Datatype root_filetype=MPI_BYTE, ftypes[2]; -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count blocklens[2]; - MPI_Count disps[2]; - blocklens[0] = ncp->begin_var; -#else - int blocklens[2]; - MPI_Aint disps[2]; - - /* check if header size > 2^31 */ - if (ncp->begin_var > NC_MAX_INT) { - DEBUG_ASSIGN_ERROR(status, NC_EINTOVERFLOW); - goto err_out; - } - - blocklens[0] = (int)ncp->begin_var; -#endif - - /* first block is the header extent */ - disps[0] = 0; - ftypes[0] = MPI_BYTE; - - /* second block is filetype, the subarray request(s) to the variable */ - blocklens[1] = 1; - disps[1] = *offset; - ftypes[1] = filetype; - -#if !defined(HAVE_MPI_LARGE_COUNT) && (SIZEOF_MPI_AINT != SIZEOF_MPI_OFFSET) - if (*offset > NC_MAX_INT) { - DEBUG_ASSIGN_ERROR(status, NC_EINTOVERFLOW); - goto err_out; - } -#endif - -#ifdef HAVE_MPI_LARGE_COUNT - mpireturn = MPI_Type_create_struct_c(2, blocklens, disps, ftypes, - &root_filetype); - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, "MPI_Type_create_struct_c"); - if (status == NC_NOERR) status = err; - } -#else - mpireturn = MPI_Type_create_struct(2, blocklens, disps, ftypes, - &root_filetype); - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, "MPI_Type_create_struct"); - if (status == NC_NOERR) status = err; - } -#endif - MPI_Type_commit(&root_filetype); - -#ifndef HAVE_MPI_LARGE_COUNT -err_out: -#endif - TRACE_IO(MPI_File_set_view, (fh, 0, MPI_BYTE, root_filetype, "native", - MPI_INFO_NULL)); - if (root_filetype != MPI_BYTE) - MPI_Type_free(&root_filetype); - - /* now update the explicit offset to be used in MPI-IO call later */ - *offset = ncp->begin_var; - } - else { - TRACE_IO(MPI_File_set_view, (fh, *offset, MPI_BYTE, filetype, "native", - MPI_INFO_NULL)); - /* the explicit offset is already set in fileview */ - *offset = 0; - } - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, mpi_name); - if (status == NC_NOERR) status = err; - } - - return status; -} - diff --git a/src/drivers/ncmpio/ncmpio_fill.c b/src/drivers/ncmpio/ncmpio_fill.c index e392a366d9..c738efd3af 100644 --- a/src/drivers/ncmpio/ncmpio_fill.c +++ b/src/drivers/ncmpio/ncmpio_fill.c @@ -144,13 +144,31 @@ fill_var_rec(NC *ncp, NC_var *varp, MPI_Offset recno) /* record number */ { - char *mpi_name; - int err, status=NC_NOERR, mpireturn; void *buf; - MPI_Offset var_len, start, count, offset; - MPI_File fh; - MPI_Status mpistatus; - MPI_Datatype bufType; + int err, status=NC_NOERR, mpireturn; + MPI_Offset var_len, start, count, offset, wlen, f_off, b_off=0, fill_len; + PNCIO_View f_view, b_view; + + f_view.off = &f_off; + f_view.len = &fill_len; + b_view.off = &b_off; + b_view.len = &fill_len; + + /* When intra-node aggregation is enabled, use the communicator consisting + * of aggregators in comm, nprocs, and rank. Non-aggregators do not + * participate the fill operation. + */ + MPI_Comm comm = ncp->comm; + int nprocs = ncp->nprocs; + int rank = ncp->rank; + if (ncp->num_aggrs_per_node > 0) { + comm = ncp->comm_attr.ina_inter_comm; + if (comm == MPI_COMM_NULL) /* non-INA aggregator */ + return NC_NOERR; + + MPI_Comm_size(comm, &nprocs); + MPI_Comm_rank(comm, &rank); + } if (varp->ndims == 0) /* scalar variable */ var_len = 1; @@ -162,14 +180,14 @@ fill_var_rec(NC *ncp, var_len = varp->dsizes[0]; /* divide total number of elements of this variable among all processes */ - count = var_len / ncp->nprocs; - start = count * ncp->rank; - if (ncp->rank < var_len % ncp->nprocs) { - start += ncp->rank; + count = var_len / nprocs; + start = count * rank; + if (rank < var_len % nprocs) { + start += rank; count++; } else { - start += var_len % ncp->nprocs; + start += var_len % nprocs; } /* allocate buffer space */ @@ -179,7 +197,7 @@ fill_var_rec(NC *ncp, err = fill_var_buf(varp, count, buf); if (err != NC_NOERR) { NCI_Free(buf); - count = 0; /* still participate collective calls below */ + /* still participate collective calls below */ status = err; } @@ -189,68 +207,42 @@ fill_var_rec(NC *ncp, offset += ncp->recsize * recno; offset += start * varp->xsz; - /* when ncp->nprocs == 1, we keep I/O mode in independent mode at all time */ - fh = ncp->collective_fh; - - /* make the entire file visible */ - TRACE_IO(MPI_File_set_view, (fh, 0, MPI_BYTE, MPI_BYTE, "native", - MPI_INFO_NULL)); - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, mpi_name); - if (status == NC_NOERR) status = err; - } - count *= varp->xsz; - bufType = MPI_BYTE; + /* Note if count > NC_MAX_INT and MPI large-count feature is not support. + * the call to ncmpio_file_write() will return NC_EINTOVERFLOW. + */ -#ifndef HAVE_MPI_LARGE_COUNT - if (count > NC_MAX_INT) { - DEBUG_ASSIGN_ERROR(err, NC_EINTOVERFLOW) - if (status == NC_NOERR) status = err; - count = 0; /* participate collective write with 0-length request */ + /* both file view and buffer view are contiguous */ + if (status == NC_NOERR) { + f_off = offset; + fill_len = count; + f_view.count = b_view.count = 1; + } + else { /* make this a zero-sized request */ + fill_len = 0; + f_view.count = b_view.count = 0; } -#endif /* write to variable collectively */ - if (ncp->nprocs > 1) { -#ifdef HAVE_MPI_LARGE_COUNT - TRACE_IO(MPI_File_write_at_all_c, (fh, offset, buf, (MPI_Count)count, - bufType, &mpistatus)); -#else - TRACE_IO(MPI_File_write_at_all, (fh, offset, buf, (int)count, - bufType, &mpistatus)); -#endif - } - else { -#ifdef HAVE_MPI_LARGE_COUNT - TRACE_IO(MPI_File_write_at_c, (fh, offset, buf, (MPI_Count)count, - bufType, &mpistatus)); -#else - TRACE_IO(MPI_File_write_at, (fh, offset, buf, (int)count, - bufType, &mpistatus)); -#endif - } + wlen = ncmpio_file_write(ncp, NC_REQ_COLL, buf, f_view, b_view); + if (status == NC_NOERR && wlen < 0) status = (int)wlen; + NCI_Free(buf); - if (bufType != MPI_BYTE) MPI_Type_free(&bufType); - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, mpi_name); - if (status == NC_NOERR) status = err; - } if (status != NC_NOERR) return status; if (IS_RECVAR(varp)) { /* update header's number of records in memory */ - /* recno may be differenet among, if safe mode is disabled. In + /* recno may be different among processes, if safe mode is disabled. In * addition, recno can be > or < then ncp->numrecs. We need to sync * first before update numrecs field in file. * * First, find the max numrecs among all processes. */ MPI_Offset max_numrecs=recno+1; - if (ncp->nprocs > 1) { + if (nprocs > 1) { TRACE_COMM(MPI_Allreduce)(MPI_IN_PLACE, &max_numrecs, 1, MPI_OFFSET, - MPI_MAX, ncp->comm); + MPI_MAX, comm); if (mpireturn != MPI_SUCCESS) { err = ncmpii_error_mpi2nc(mpireturn, "MPI_Allreduce"); if (status == NC_NOERR) status = err; @@ -287,7 +279,7 @@ fillerup(NC *ncp) indx = ncmpio_NC_findattr(&ncp->vars.value[i]->attrs, "_FillValue"); /* only if filling this variable is requested. Fill mode can be - * enabled by 2 ways: explictly call to ncmpi_def_var_fill() or put + * enabled by 2 ways: explicitly call to ncmpi_def_var_fill() or put * the attribute named "_FillValue" */ if (ncp->vars.value[i]->no_fill && indx == -1) continue; @@ -342,7 +334,8 @@ fill_added_recs(NC *ncp, NC *old_ncp) continue; /* check if attribute _FillValue is defined */ - indx = ncmpio_NC_findattr(&ncp->vars.value[varid]->attrs, "_FillValue"); + indx = ncmpio_NC_findattr(&ncp->vars.value[varid]->attrs, + "_FillValue"); /* only if filling this variable is requested */ if (ncp->vars.value[varid]->no_fill && indx == -1) continue; @@ -363,23 +356,28 @@ fill_added_recs(NC *ncp, NC *old_ncp) static int fillerup_aggregate(NC *ncp, NC *old_ncp) { - int i, j, k, mpireturn, err, status=NC_NOERR; - int start_vid, recno, nVarsFill; - char *buf_ptr, *noFill, *mpi_name; - void *buf; + int i, j, k, err, status=NC_NOERR, start_vid, recno, nVarsFill; + char *buf, *buf_ptr, *noFill; size_t nsegs; - MPI_Offset buf_len, var_len, nrecs, start, *count; - MPI_Datatype filetype, bufType; - MPI_File fh; - MPI_Status mpistatus; + MPI_Offset var_len, nrecs, start, wlen; NC_var *varp; + MPI_Offset b_off=0, b_len; + PNCIO_View f_view, b_view; -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count *blocklengths, *offset; -#else - int *blocklengths; - MPI_Aint *offset; -#endif + /* When intra-node aggregation is enabled, use the communicator consisting + * of aggregators in comm, nprocs, and rank. Non-aggregators do not + * participate the fill operation. + */ + int nprocs = ncp->nprocs; + int rank = ncp->rank; + if (ncp->num_aggrs_per_node > 0) { + if (ncp->comm_attr.ina_inter_comm == MPI_COMM_NULL) + /* non-INA aggregator */ + return NC_NOERR; + + MPI_Comm_size(ncp->comm_attr.ina_inter_comm, &nprocs); + MPI_Comm_rank(ncp->comm_attr.ina_inter_comm, &rank); + } /* find the starting vid for newly added variables */ start_vid = 0; @@ -397,12 +395,17 @@ fillerup_aggregate(NC *ncp, NC *old_ncp) * variables' fill modes and overwrite local's if an inconsistency is found * Note ncp->vars.ndefined is already made consistent by this point. */ - if (ncp->nprocs > 1) { + MPI_Comm comm = (ncp->num_aggrs_per_node > 0) + ? ncp->comm_attr.ina_inter_comm : ncp->comm; + + if (nprocs > 1) { + int mpireturn; + for (i=start_vid; ivars.ndefined; i++) noFill[i-start_vid] = (char)(ncp->vars.value[i]->no_fill); TRACE_COMM(MPI_Bcast)(noFill, (ncp->vars.ndefined - start_vid), - MPI_BYTE, 0, ncp->comm); + MPI_BYTE, 0, comm); if (mpireturn != MPI_SUCCESS) return ncmpii_error_mpi2nc(mpireturn, "MPI_Bcast"); @@ -425,15 +428,11 @@ fillerup_aggregate(NC *ncp, NC *old_ncp) /* find the number of write segments (upper bound) */ nsegs = (size_t)(ncp->vars.ndefined + ncp->vars.num_rec_vars * nrecs); - count = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset) * nsegs); -#ifdef HAVE_MPI_LARGE_COUNT - offset = (MPI_Count*) NCI_Malloc(sizeof(MPI_Count) * nsegs); -#else - offset = (MPI_Aint*) NCI_Malloc(sizeof(MPI_Aint) * nsegs); -#endif + f_view.off = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset) * nsegs); + f_view.len = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset) * nsegs); - /* calculate each segment's offset and count */ - buf_len = 0; /* total write amount, used to allocate buffer */ + /* calculate each segment's offset and length */ + b_len = 0; /* total write amount, used to allocate buffer */ j = 0; for (i=start_vid; ivars.ndefined; i++) { if (noFill[i-start_vid]) continue; @@ -446,29 +445,23 @@ fillerup_aggregate(NC *ncp, NC *old_ncp) else var_len = varp->dsizes[0]; /* divide evenly total number of variable's elements among processes */ - count[j] = var_len / ncp->nprocs; - start = count[j] * ncp->rank; - if (ncp->rank < var_len % ncp->nprocs) { - start += ncp->rank; - count[j]++; + f_view.len[j] = var_len / nprocs; + start = f_view.len[j] * rank; + if (rank < var_len % nprocs) { + start += rank; + f_view.len[j]++; } else - start += var_len % ncp->nprocs; + start += var_len % nprocs; /* calculate the starting file offset */ start *= varp->xsz; - start += varp->begin; - offset[j] = (MPI_Aint)start; - if (start != offset[j]) { - DEBUG_ASSIGN_ERROR(err, NC_EINTOVERFLOW) - if (status == NC_NOERR) status = err; - noFill[i-start_vid] = 1; /* skip this variable */ - continue; - } + f_view.off[j] = varp->begin + start; + /* add up the buffer size */ - buf_len += count[j] * varp->xsz; + b_len += f_view.len[j] * varp->xsz; - j++; /* increase j even when count[j] is zero */ + j++; /* increase j even when f_view.len[j] is zero */ } /* loop thru all record variables to find the aggregated write amount */ @@ -483,48 +476,36 @@ fillerup_aggregate(NC *ncp, NC *old_ncp) else var_len = varp->dsizes[1]; /* divide total number of variable's elements among all processes */ - count[j] = var_len / ncp->nprocs; - start = count[j] * ncp->rank; - if (ncp->rank < var_len % ncp->nprocs) { - start += ncp->rank; - count[j]++; + f_view.len[j] = var_len / nprocs; + start = f_view.len[j] * rank; + if (rank < var_len % nprocs) { + start += rank; + f_view.len[j]++; } else - start += var_len % ncp->nprocs; + start += var_len % nprocs; /* calculate the starting file offset */ start *= varp->xsz; - start += varp->begin + ncp->recsize * recno; - offset[j] = (MPI_Aint)start; - if (start != offset[j]) { - DEBUG_ASSIGN_ERROR(err, NC_EINTOVERFLOW) - if (status == NC_NOERR) status = err; - noFill[i-start_vid] = 1; /* skip this variable */ - continue; - } + f_view.off[j] = varp->begin + ncp->recsize * recno + start; + /* add up the buffer size */ - buf_len += count[j] * varp->xsz; + b_len += f_view.len[j] * varp->xsz; - j++; /* increase j even when count[j] is zero */ + j++; /* increase j even when f_view.len[j] is zero */ } } /* j is now the number of valid write segments */ if (status == NC_NOERR && j == 0) { NCI_Free(noFill); - NCI_Free(count); - NCI_Free(offset); + NCI_Free(f_view.len); + NCI_Free(f_view.off); return NC_NOERR; } - /* allocate one contiguous buffer space for all writes */ -#ifdef HAVE_MPI_LARGE_COUNT - blocklengths = (MPI_Count*) NCI_Malloc(sizeof(MPI_Count) * j); -#else - blocklengths = (int*) NCI_Malloc(sizeof(int) * j); -#endif - buf = NCI_Malloc((size_t)buf_len); - buf_ptr = (char*)buf; + buf = (char*) NCI_Malloc((size_t)b_len); + buf_ptr = buf; /* fill write buffers for fixed-size variables first */ j = k = 0; @@ -534,29 +515,19 @@ fillerup_aggregate(NC *ncp, NC *old_ncp) varp = ncp->vars.value[i]; if (IS_RECVAR(varp)) continue; - if (k < j) { /* coalesce count[] and offset[] */ - count[k] = count[j]; - offset[k] = offset[j]; + if (k < j) { /* coalesce f_view.off[] and f_view.len[] */ + f_view.len[k] = f_view.len[j]; + f_view.off[k] = f_view.off[j]; } j++; - err = fill_var_buf(varp, count[k], buf_ptr); + err = fill_var_buf(varp, f_view.len[k], buf_ptr); if (err != NC_NOERR) { if (status == NC_NOERR) status = err; continue; /* skip this request */ } - count[k] *= varp->xsz; -#ifdef HAVE_MPI_LARGE_COUNT - blocklengths[k] = (MPI_Count)count[k]; -#else - if (count[k] != (int)count[k]) { - DEBUG_ASSIGN_ERROR(err, NC_EINTOVERFLOW) - if (status == NC_NOERR) status = err; - continue; /* skip this request */ - } - blocklengths[k] = (int)count[k]; -#endif - buf_ptr += count[k]; + f_view.len[k] *= varp->xsz; + buf_ptr += f_view.len[k]; k++; } /* k is the number of valid write requests thus far */ @@ -569,123 +540,80 @@ fillerup_aggregate(NC *ncp, NC *old_ncp) varp = ncp->vars.value[i]; if (!IS_RECVAR(varp)) continue; - if (k < j) { /* coalesce count[] and offset[] */ - count[k] = count[j]; - offset[k] = offset[j]; + if (k < j) { /* coalesce f_view.off[] and f_view.len[] */ + f_view.len[k] = f_view.len[j]; + f_view.off[k] = f_view.off[j]; } j++; - err = fill_var_buf(varp, count[k], buf_ptr); + err = fill_var_buf(varp, f_view.len[k], buf_ptr); if (err != NC_NOERR) { if (status == NC_NOERR) status = err; continue; /* skip this request */ } - count[k] *= varp->xsz; -#ifdef HAVE_MPI_LARGE_COUNT - blocklengths[k] = (MPI_Count)count[k]; -#else - if (count[k] != (int)count[k]) { - DEBUG_ASSIGN_ERROR(err, NC_EINTOVERFLOW) - if (status == NC_NOERR) status = err; - continue; /* skip this request */ - } - blocklengths[k] = (int)count[k]; -#endif - buf_ptr += count[k]; + f_view.len[k] *= varp->xsz; + buf_ptr += f_view.len[k]; k++; } } /* k is the number of valid write requests */ NCI_Free(noFill); - if (k == 0) { - filetype = MPI_BYTE; + /* Remove entries whose f_view.len[i] == 0. This happens when the size of + * fix-sized variable or record variable section is too small, such that + * some processes are not assigned data to fille. + * + * In the meantime, coalesce offset-length pairs. + */ + for (j=0; j 0) { + f_view.len[0] = f_view.len[j]; + f_view.off[0] = f_view.off[j]; + } + j++; /* next check starts from j */ + break; + } } + + if (f_view.len[0] == 0) + k = 0; /* all zero requests */ else { - /* create fileview: a list of contiguous segment for each variable */ -#ifdef HAVE_MPI_LARGE_COUNT - mpireturn = MPI_Type_create_hindexed_c(k, blocklengths, offset, - MPI_BYTE, &filetype); -#else - mpireturn = MPI_Type_create_hindexed(k, blocklengths, offset, - MPI_BYTE, &filetype); -#endif - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, "MPI_Type_hindexed"); - /* return the first encountered error if there is any */ - if (status == NC_NOERR) status = err; + i = 0; + for (; jcollective_fh; - - TRACE_IO(MPI_File_set_view, (fh, 0, MPI_BYTE, filetype, "native", - MPI_INFO_NULL)); - if (k > 0) MPI_Type_free(&filetype); - - bufType = MPI_BYTE; - if (buf_len > NC_MAX_INT) { -#ifdef HAVE_MPI_LARGE_COUNT - mpireturn = MPI_Type_contiguous_c((MPI_Count)buf_len, MPI_BYTE, - &bufType); - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, "MPI_Type_contiguous_c"); - /* return the first encountered error if there is any */ - if (status == NC_NOERR) status = err; - buf_len = 0; - } - else { - MPI_Type_commit(&bufType); - buf_len = 1; - } -#else - if (status == NC_NOERR) - DEBUG_ASSIGN_ERROR(status, NC_EINTOVERFLOW) - - buf_len = 0; /* participate collective write with 0-length request */ -#endif - } + /* write buffer is contiguous */ + b_view.count = (b_len > 0) ? 1 : 0; + b_view.off = &b_off; + b_view.len = &b_len; /* write to variable collectively */ - if (ncp->nprocs > 1) { -#ifdef HAVE_MPI_LARGE_COUNT - TRACE_IO(MPI_File_write_at_all_c, (fh, 0, buf, (MPI_Count)buf_len, - bufType, &mpistatus)); -#else - TRACE_IO(MPI_File_write_at_all, (fh, 0, buf, (int)buf_len, - bufType, &mpistatus)); -#endif - } - else { -#ifdef HAVE_MPI_LARGE_COUNT - TRACE_IO(MPI_File_write_at_c, (fh, 0, buf, (MPI_Count)buf_len, - bufType, &mpistatus)); -#else - TRACE_IO(MPI_File_write_at, (fh, 0, buf, (int)buf_len, - bufType, &mpistatus)); -#endif - } + wlen = ncmpio_file_write(ncp, NC_REQ_COLL, buf, f_view, b_view); + if (status == NC_NOERR && wlen < 0) status = (int)wlen; + /* free allocated resources */ NCI_Free(buf); - if (bufType != MPI_BYTE) MPI_Type_free(&bufType); - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, mpi_name); - if (status == NC_NOERR) status = err; - } + if (f_view.len != NULL) NCI_Free(f_view.len); + if (f_view.off != NULL) NCI_Free(f_view.off); - TRACE_IO(MPI_File_set_view, (fh, 0, MPI_BYTE, MPI_BYTE, "native", - MPI_INFO_NULL)); - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, mpi_name); - if (status == NC_NOERR) status = err; - } return status; } @@ -753,7 +681,8 @@ ncmpio_fill_var_rec(void *ncdp, } err_check: - if (ncp->safe_mode && ncp->nprocs > 1) { /* consistency check */ + if (fIsSet(ncp->flags, NC_MODE_SAFE) && ncp->nprocs > 1) { + /* consistency check */ int root_varid, status, mpireturn; MPI_Offset root_recno; @@ -788,9 +717,9 @@ ncmpio_fill_var_rec(void *ncdp, } /*----< ncmpio_set_fill() >--------------------------------------------------*/ -/* this API is collective, must be called in define mode, contrary to netCDF - * where nc_set_fill() can also be called in data mode. The reason of PnetCDF - * enforcing this requirement is because PnetCDF only fills fixed-size +/* This API is collective, and must be called in the define mode, contrary to + * netCDF where nc_set_fill() can also be called in data mode. The reason of + * PnetCDF enforcing this requirement is because PnetCDF only fills fixed-size * variables at ncmpi_enddef() and record variables in ncmpi_fill_var_rec(). */ int @@ -801,7 +730,7 @@ ncmpio_set_fill(void *ncdp, int i, mpireturn, oldmode; NC *ncp = (NC*)ncdp; - if (ncp->safe_mode && ncp->nprocs > 1) { + if (fIsSet(ncp->flags, NC_MODE_SAFE) && ncp->nprocs > 1) { int err, status, root_fill_mode=fill_mode; TRACE_COMM(MPI_Bcast)(&root_fill_mode, 1, MPI_INT, 0, ncp->comm); @@ -809,7 +738,7 @@ ncmpio_set_fill(void *ncdp, return ncmpii_error_mpi2nc(mpireturn, "MPI_Bcast"); if (fill_mode != root_fill_mode) - /* dataset's fill mode is inconsistent with root's */ + /* the file's fill mode is inconsistent with root's */ DEBUG_ASSIGN_ERROR(err, NC_EMULTIDEFINE_FILL_MODE) else err = NC_NOERR; @@ -840,7 +769,8 @@ ncmpio_set_fill(void *ncdp, /* once the file's fill mode is set, any new variables defined after this * call will check NC_dofill(ncp) and set their no_fill accordingly. See - * ncmpi_def_var() */ + * ncmpi_def_var() + */ return NC_NOERR; } @@ -860,7 +790,7 @@ ncmpio_def_var_fill(void *ncdp, /* sanity check for ncdp and varid has been done in dispatchers */ varp = ncp->vars.value[varid]; - if (ncp->safe_mode && ncp->nprocs > 1) { + if (fIsSet(ncp->flags, NC_MODE_SAFE) && ncp->nprocs > 1) { int root_ids[3], my_fill_null, minE, mpireturn; /* check if varid, no_fill, fill_value, are consistent */ @@ -946,7 +876,7 @@ ncmpio_inq_var_fill(NC_var *varp, /* retrieve user-defined attribute _FillValue */ xp = ncap->value[i]->xvalue; - /* value stored in xvalue is in external representation, may need byte-swap */ + /* value stored in xvalue is in external representation, need byte-swap */ switch(varp->xtype) { case NC_CHAR: return ncmpix_getn_text (&xp, 1, (char*)fill_value); case NC_BYTE: return ncmpix_getn_NC_BYTE_schar (&xp, 1, (signed char*)fill_value); diff --git a/src/drivers/ncmpio/ncmpio_fstype.c b/src/drivers/ncmpio/ncmpio_fstype.c new file mode 100644 index 0000000000..bc4de51238 --- /dev/null +++ b/src/drivers/ncmpio/ncmpio_fstype.c @@ -0,0 +1,232 @@ +/* + * Copyright (C) 2026, Northwestern University and Argonne National Laboratory + * See COPYRIGHT notice in top-level directory. + */ + +#ifdef HAVE_CONFIG_H +# include +#endif + +#include +#include +#include /* readlink() */ +#include /* strrchr(), strncpy(), strncmp() */ +#include /* strncasecmp() */ +#include +#include +#include /* open(), O_CREAT */ +#include /* open() */ + +#ifdef HAVE_LIMITS_H +#include +#endif +#ifndef PATH_MAX +#define PATH_MAX 65535 +#endif + +#ifdef HAVE_SYS_VFS_H +#include +#endif +#ifdef HAVE_SYS_STATVFS_H +#include +#endif +#ifdef HAVE_SYS_PARAM_H +#include /* struct statfs */ +#endif +#ifdef HAVE_SYS_MOUNT_H +#include /* struct statfs */ +#endif +#ifdef HAVE_SYS_STAT_H +#include /* open(), fstat(), lstat(), stat() */ +#endif + +#include +#include + +/* In a strict ANSI environment, S_ISLNK may not be defined. Fix that here. + * We assume that S_ISLNK is *always* defined as a macro. If that is not + * universally true, then add a test to the configure that tries to link + * a program that references S_ISLNK + */ +#if !defined(S_ISLNK) +#if defined(S_IFLNK) +/* Check for the link bit */ +#define S_ISLNK(mode) ((mode) & S_IFLNK) +#else +/* no way to check if it is a link, so say false */ +#define S_ISLNK(mode) 0 +#endif +#endif /* !(S_ISLNK) */ + +/* Returns a string, the parent directory of a given filename. + * The caller should free the memory located returned by this subroutine. + */ +static +void parentdir(const char *filename, char **dirnamep) +{ + int err; + char *dir = NULL, *slash; + struct stat statbuf; + + err = lstat(filename, &statbuf); + + if (err || (!S_ISLNK(statbuf.st_mode))) { + /* No such file, or file is not a link; these are the "normal" cases + * where we can just return the parent directory. + */ + dir = NCI_Strdup(filename); + } else { + /* filename is a symlink. We've presumably already tried to stat it + * and found it to be missing (dangling link), but this code doesn't + * care if the target is really there or not. + */ + ssize_t namelen; + char *linkbuf; + + linkbuf = NCI_Malloc(PATH_MAX + 1); + namelen = readlink(filename, linkbuf, PATH_MAX + 1); + if (namelen == -1) { + /* Something strange has happened between the time that we + * determined that this was a link and the time that we attempted + * to read it; punt and use the old name. + */ + dir = NCI_Strdup(filename); + } else { + /* successfully read the link */ + linkbuf[namelen] = '\0'; /* readlink doesn't null terminate */ + dir = NCI_Strdup(linkbuf); + } + NCI_Free(linkbuf); + } + + slash = strrchr(dir, '/'); + if (!slash) + strncpy(dir, ".", 2); + else { + if (slash == dir) + *(dir + 1) = '\0'; + else + *slash = '\0'; + } + + *dirnamep = dir; + return; +} + +#define UNKNOWN_SUPER_MAGIC (0xDEADBEEF) +#ifndef LL_SUPER_MAGIC +#define LL_SUPER_MAGIC 0x0BD00BD0 +#endif + +static int check_statfs(const char *filename, int64_t * file_id) +{ + int err = 0; + +#ifdef HAVE_STRUCT_STATVFS_WITH_F_BASETYPE + /* rare: old solaris machines */ + struct statvfs vfsbuf; +#endif +#if defined(HAVE_STRUCT_STATFS_F_TYPE) || defined(HAVE_STRUCT_STATFS_F_FSTYPENAME) + /* common fs-detection logic for any modern POSIX-compliant environment, + * with the one wrinkle that some platforms (Darwin, BSD) give us a file + * system as a string, not an identifier */ + struct statfs fsbuf; +#endif + + *file_id = UNKNOWN_SUPER_MAGIC; + +#ifdef HAVE_STRUCT_STATVFS_WITH_F_BASETYPE + err = statvfs(filename, &vfsbuf); + if (err == 0) + *file_id = vfsbuf.f_basetype; +#endif + + /* remember above how I said 'statfs with f_type' was the common linux-y + * way to report file system type? Darwin (and probably the BSDs) *also* + * uses f_type but it is "reserved" and does not give us anything + * meaningful. Fine. If configure detects f_type we'll use it here and on + * those "reserved" platforms we'll ignore that result and check the + * f_fstypename field. + */ + +#ifdef HAVE_STRUCT_STATFS_F_TYPE + err = statfs(filename, &fsbuf); + if (err == 0) { + *file_id = fsbuf.f_type; + return 0; + } +#endif + +#ifdef HAVE_STRUCT_STATFS_F_FSTYPENAME + /* these stat routines store the file system type in a string */ + err = statfs(filename, &fsbuf); + if (err == 0 && !strncasecmp(fsbuf.f_fstypename, "lustre", 6)) { + *file_id = LL_SUPER_MAGIC; + return 0; + } +#endif + +#ifdef HAVE_STRUCT_STAT_ST_FSTYPE + struct stat sbuf; + err = stat(filename, &sbuf); + if (err == 0) { + *file_id = sbuf.st_fstype; + return 0; + } +#endif + return err; +} + +/* Check if file system type from file name, using a system-dependent function + * call. + */ +int ncmpio_FileSysType(const char *filename) +{ + + int err, retry_cnt; + int64_t file_id=UNKNOWN_SUPER_MAGIC; + + char *colon = strchr(filename, ':'); + if (colon != NULL) { /* there is a prefix end with : */ + if (!strncmp(filename, "lustre", 6)) + return PNC_FS_LUSTRE; + else if (!strncmp(filename, "ufs", 3)) + return PNC_FS_UFS; + else + return 0; + } +#ifdef MIMIC_LUSTRE + return PNC_FS_LUSTRE; +#endif + + /* NFS can get stuck and end up returning ESTALE "forever" */ + +#define MAX_ESTALE_RETRY 10000 + + retry_cnt = 0; + do { + err = check_statfs(filename, &file_id); + } while (err && (errno == ESTALE) && retry_cnt++ < MAX_ESTALE_RETRY); + + if (err) { + /* ENOENT may be returned in two cases: + * 1) no directory entry for "filename" + * 2) "filename" is a dangling symbolic link + * + * parentdir() tries to deal with both cases. + */ + if (errno == ENOENT) { + char *dir; + parentdir(filename, &dir); + err = check_statfs(dir, &file_id); + NCI_Free(dir); + } else + return 0; + } + + if (file_id == LL_SUPER_MAGIC) + return PNC_FS_LUSTRE; + else + return PNC_FS_UFS; /* UFS support if we don't know what else to use */ +} + diff --git a/src/drivers/ncmpio/ncmpio_getput.m4 b/src/drivers/ncmpio/ncmpio_getput.m4 index 363701cac2..a2850ed812 100644 --- a/src/drivers/ncmpio/ncmpio_getput.m4 +++ b/src/drivers/ncmpio/ncmpio_getput.m4 @@ -40,7 +40,7 @@ dnl #include #include #include "ncmpio_NC.h" -#ifdef ENABLE_SUBFILING +#if PNETCDF_SUBFILING == 1 #include "ncmpio_subfile.h" #endif @@ -50,14 +50,14 @@ dnl buf (user buffer of internal data type) lbuf (contiguous buffer packed from buf based on buftype) cbuf (contiguous buffer packed from lbuf based on imap) - xbuf (contiguous buffer in external data type, type-casted/byte-swapped + xbuf (contiguous buffer in external data type, type-cast/byte-swapped from cbuf, ready to be used in MPI_File_write to write to file) For read requests: xbuf (contiguous buffer to be used in MPI_File_read to read from file. Its contents are in external data type) - cbuf (contiguous buffer type-casted/byte-swapped from xbuf, its contents - are in internal data type) + cbuf (contiguous buffer type-cast/byte-swapped from xbuf, its contents are + in internal data type) lbuf (contiguous buffer unpacked from cbuf based on imap) buf (user buffer, unpacked from lbuf based on buftype) @@ -118,10 +118,18 @@ put_varm(NC *ncp, void *xbuf=NULL; int mpireturn, err=NC_NOERR, status=NC_NOERR, buftype_is_contig; int el_size, need_convert=0, need_swap=0, need_swap_back_buf=0; - int coll_indep, xtype_is_contig=1, can_swap_in_place; - MPI_Offset nelems=0, bnelems=0, nbytes=0, offset=0; - MPI_Datatype itype, xtype=MPI_BYTE, imaptype, filetype=MPI_BYTE; - MPI_File fh; + int can_swap_in_place; + MPI_Offset nelems=0, bnelems=0, nbytes=0; + MPI_Datatype itype, imaptype; + + if (varp == NULL) { /* zero-sized request */ + itype = MPI_BYTE; + el_size = 0; + bnelems = 0; + nbytes = 0; + buftype_is_contig = 0; + goto err_check; + } /* decode buftype to obtain the followings: * itype: element data type (MPI primitive type) in buftype @@ -135,20 +143,10 @@ put_varm(NC *ncp, * el_size: byte size of itype * buftype_is_contig: whether buftype is contiguous */ - if (varp == NULL) { /* zero-sized request */ - itype = MPI_BYTE; - el_size = 0; - bnelems = 0; - nbytes = 0; - buftype_is_contig = 0; - } - else { - err = ncmpii_buftype_decode(varp->ndims, varp->xtype, count, bufcount, - buftype, &itype, &el_size, &bnelems, - &nbytes, &buftype_is_contig); - if (err != NC_NOERR) goto err_check; - } - xtype_is_contig = buftype_is_contig; + err = ncmpii_buftype_decode(varp->ndims, varp->xtype, count, bufcount, + buftype, &itype, &el_size, &bnelems, &nbytes, + &buftype_is_contig); + if (err != NC_NOERR) goto err_check; if (buftype == MPI_DATATYPE_NULL) { /* buftype and bufcount are ignored */ bufcount = bnelems; @@ -164,7 +162,7 @@ put_varm(NC *ncp, /* When bnelems > NC_MAX_INT, we construct a datatype to bypass the * limitation of MPI file read/write APIs on the argument "count" of type - * int. See ncmpio_read_write() in ncmpio_file_io.c + * int. See ncmpi_file_read()/ncmpi_file_write() in ncmpio_file_io.c * * Note not all MPI-IO libraries support single requests larger than * NC_MAX_INT. In this case, MPI-IO should report an error. @@ -174,10 +172,15 @@ put_varm(NC *ncp, goto err_check; /* check if type conversion and Endianness byte swap is needed */ - if (varp != NULL) { /* non-zero-sized request */ - need_convert = ncmpii_need_convert(ncp->format, varp->xtype, itype); - need_swap = NEED_BYTE_SWAP(varp->xtype, itype); - } + need_convert = ncmpii_need_convert(ncp->format, varp->xtype, itype); + need_swap = NEED_BYTE_SWAP(varp->xtype, itype); + + /* check whether this is a true varm call, if yes, imaptype will be a + * newly created MPI derived data type, otherwise MPI_DATATYPE_NULL + */ + imaptype = MPI_DATATYPE_NULL; + err = ncmpii_create_imaptype(varp->ndims, count, imap, itype, &imaptype); + if (err != NC_NOERR) goto err_check; /* check if in-place byte swap can be enabled */ can_swap_in_place = 1; @@ -190,25 +193,18 @@ put_varm(NC *ncp, else if (! fIsSet(ncp->flags, NC_MODE_SWAP_ON)) { /* auto mode, as user does not explicitly enable it */ if (nbytes <= NC_BYTE_SWAP_BUFFER_SIZE) - /* If write amount is small, disable in-place swap. - * This is because the user buffer may be immutable. In this - * case, in-place swap will cause segmentation fault. Immutable - * buffers are usually small. */ + /* If write amount is small, disable in-place swap. This is + * because the user buffer may be immutable. In this case, + * in-place swap will cause segmentation fault. Immutable + * buffers are usually small. + */ can_swap_in_place = 0; } } - /* check whether this is a true varm call, if yes, imaptype will be a - * newly created MPI derived data type, otherwise MPI_DATATYPE_NULL - */ - imaptype = MPI_DATATYPE_NULL; - if (varp != NULL) { /* non-zero-sized request */ - err = ncmpii_create_imaptype(varp->ndims, count, imap, itype, &imaptype); - if (err != NC_NOERR) goto err_check; - } - - if (!need_convert && imaptype == MPI_DATATYPE_NULL && - (!need_swap || (can_swap_in_place && buftype_is_contig))) { + if (!need_convert && imaptype == MPI_DATATYPE_NULL && buftype_is_contig && + (!need_swap || can_swap_in_place)) + { /* reuse buftype, bufcount, buf in later MPI file write */ xbuf = buf; if (need_swap) { @@ -216,17 +212,17 @@ put_varm(NC *ncp, need_swap_back_buf = 1; } } - else if (varp != NULL) { + else { xbuf = NCI_Malloc((size_t)nbytes); if (xbuf == NULL) { DEBUG_ASSIGN_ERROR(err, NC_ENOMEM) goto err_check; } need_swap_back_buf = 0; - xtype_is_contig = 1; - /* pack buf to xbuf, byte-swap and type-convert on xbuf, which - * will later be used in MPI file write */ + /* Pack buf to xbuf, byte-swap and type-convert on xbuf, which will + * later be used in MPI file write. + */ err = ncmpio_pack_xbuf(ncp->format, varp, bufcount, buftype, buftype_is_contig, bnelems, itype, el_size, imaptype, need_convert, need_swap, nbytes, buf, @@ -238,16 +234,14 @@ put_varm(NC *ncp, } } - /* Set nelems and xtype which will be used in MPI read/write */ - if (buf != xbuf && varp != NULL) { + /* Set nelems which will be used in MPI read/write */ + if (buf != xbuf) { /* xbuf is a contiguous buffer */ - xtype = ncmpii_nc2mpitype(varp->xtype); nelems = bnelems; } else { /* we can safely use bufcount and buftype in MPI File read/write */ nelems = (bufcount == NC_COUNT_IGNORE) ? bnelems : bufcount; - xtype = buftype; } err_check: @@ -263,63 +257,11 @@ err_check: */ nbytes = 0; nelems = 0; - filetype = MPI_BYTE; - xtype = MPI_BYTE; } - if (fIsSet(reqMode, NC_REQ_COLL) && ncp->my_aggr >= 0 && ncp->nprocs > 1) { - /* intra-node write aggregation must be in collective mode */ - void *wbuf = (nbytes == 0) ? NULL : xbuf; - err = ncmpio_intra_node_aggregation(ncp, NC_REQ_WR, varp, start, count, - stride, nelems, xtype, wbuf); - if (status == NC_NOERR) status = err; - } - else { - if (nbytes > 0) { - /* Create the filetype for this request and calculate the beginning - * file offset for this request. If this request is contiguous in - * file, then set filetype == MPI_BYTE. Otherwise filetype will - * be an MPI derived data type. - */ - err = ncmpio_filetype_create_vars(ncp, varp, start, count, stride, - &offset, &filetype, NULL); - if (err != NC_NOERR) { - nbytes = 0; - nelems = 0; - filetype = MPI_BYTE; - xtype = MPI_BYTE; - if (status == NC_NOERR) status = err; - } - } - - /* TODO: if record variables are too big to store the stride between - * records in an MPI_Aint, then we will have to process this one record - * at a time. - */ - - fh = ncp->independent_fh; - coll_indep = NC_REQ_INDEP; - if (ncp->nprocs > 1 && fIsSet(reqMode, NC_REQ_COLL)) { - fh = ncp->collective_fh; - coll_indep = NC_REQ_COLL; - } - - /* MPI_File_set_view is collective */ - err = ncmpio_file_set_view(ncp, fh, &offset, filetype); - if (err != NC_NOERR) { - nelems = 0; /* skip this request */ - if (status == NC_NOERR) status = err; - } - if (filetype != MPI_BYTE) MPI_Type_free(&filetype); - - /* xtype is the element data type (MPI primitive type) in xbuf to be - * written to the variable defined in file. Note data stored in xbuf - * is in the external data type, ready to be written to file. - */ - err = ncmpio_read_write(ncp, NC_REQ_WR, coll_indep, offset, nelems, - xtype, xbuf, xtype_is_contig); - if (status == NC_NOERR) status = err; - } + err = ncmpio_ina_req(ncp, NC_REQ_WR, varp, start, count, stride, nbytes, + xbuf); + if (status == NC_NOERR) status = err; /* done with xbuf */ if (xbuf != NULL && xbuf != buf) NCI_Free(xbuf); @@ -340,7 +282,8 @@ err_check: new_numrecs = start[0] + (count[0] - 1) * stride[0] + 1; /* note new_numrecs can be smaller than ncp->numrecs when this - * write request writes existing records */ + * write request writes existing records + */ } if (fIsSet(reqMode, NC_REQ_COLL)) { @@ -357,8 +300,9 @@ err_check: if (status == NC_NOERR) status = err; } } - /* In collective mode, ncp->numrecs is always sync-ed among - processes */ + /* In collective data mode, ncp->numrecs is always sync-ed among + * processes + */ if (ncp->numrecs < max_numrecs) { err = ncmpio_write_numrecs(ncp, max_numrecs); if (status == NC_NOERR) status = err; @@ -396,11 +340,19 @@ get_varm(NC *ncp, int reqMode) /* WR/RD/COLL/INDEP */ { void *xbuf=NULL; - int err=NC_NOERR, status=NC_NOERR, coll_indep, xtype_is_contig=1; + int err=NC_NOERR, status=NC_NOERR; int el_size, buftype_is_contig, need_swap=0, need_convert=0; - MPI_Offset nelems=0, bnelems=0, nbytes=0, offset=0; - MPI_Datatype itype, xtype=MPI_BYTE, filetype=MPI_BYTE, imaptype=MPI_DATATYPE_NULL; - MPI_File fh; + MPI_Offset nelems=0, bnelems=0, nbytes=0; + MPI_Datatype itype, imaptype=MPI_DATATYPE_NULL; + + if (varp == NULL) { /* zero-sized request */ + itype = MPI_BYTE; + el_size = 0; + bnelems = 0; + nbytes = 0; + buftype_is_contig = 0; + goto err_check; + } /* decode buftype to see if we can use buf to read from file. * itype: element data type (MPI primitive type) in buftype @@ -415,10 +367,9 @@ get_varm(NC *ncp, * buftype_is_contig: whether buftype is contiguous */ err = ncmpii_buftype_decode(varp->ndims, varp->xtype, count, bufcount, - buftype, &itype, &el_size, &bnelems, - &nbytes, &buftype_is_contig); + buftype, &itype, &el_size, &bnelems, &nbytes, + &buftype_is_contig); if (err != NC_NOERR) goto err_check; - xtype_is_contig = buftype_is_contig; if (buftype == MPI_DATATYPE_NULL) { /* buftype and bufcount are ignored */ bufcount = bnelems; @@ -434,7 +385,7 @@ get_varm(NC *ncp, /* When bnelems > NC_MAX_INT, we construct a datatype to bypass the * limitation of MPI file read/write APIs on the argument "count" of type - * int. See ncmpio_read_write() in ncmpio_file_io.c + * int. See ncmpi_file_read()/ncmpi_file_write() in ncmpio_file_io.c * * Note not all MPI-IO libraries support single requests larger than * NC_MAX_INT. In this case, MPI-IO should report an error. @@ -462,31 +413,30 @@ get_varm(NC *ncp, * For condition 2, imap is checked in ncmpii_create_imaptype() */ if (!need_convert && imaptype == MPI_DATATYPE_NULL && - (!need_swap || buftype_is_contig)) { + !need_swap && buftype_is_contig) + { /* reuse buftype, bufcount, buf in later MPI file read */ xbuf = buf; } else { /* allocate xbuf for reading */ xbuf = NCI_Malloc((size_t)nbytes); - xtype_is_contig = 1; if (xbuf == NULL) { DEBUG_ASSIGN_ERROR(err, NC_ENOMEM) goto err_check; } } /* Note xbuf is the buffer to be used in MPI read calls, and hence its - * contents are in the external type */ + * contents are in the external type. + */ - /* Set nelems and xtype which will be used in MPI read/write */ + /* Set nelems which will be used in MPI read/write */ if (buf != xbuf) { /* xbuf is a contiguous buffer */ nelems = bnelems; - xtype = ncmpii_nc2mpitype(varp->xtype); } else { /* we can safely use bufcount and buftype in MPI File read/write */ nelems = (bufcount == NC_COUNT_IGNORE) ? bnelems : bufcount; - xtype = buftype; } err_check: @@ -496,57 +446,15 @@ err_check: /* for independent API, this process returns now */ if (fIsSet(reqMode, NC_REQ_INDEP)) return err; - /* for collective API, this process needs to participate the - * collective I/O operations, but with zero-length request + /* for collective API, this process needs to participate the collective + * I/O operations, but with zero-length request */ - filetype = MPI_BYTE; - xtype = MPI_BYTE; nbytes = 0; nelems = 0; } - else { - /* Create the filetype for this request and calculate the beginning - * file offset for this request. If this request is contiguous in file, - * then set filetype == MPI_BYTE. Otherwise filetype will be an MPI - * derived data type. - */ - err = ncmpio_filetype_create_vars(ncp, varp, start, count, stride, - &offset, &filetype, NULL); - if (err != NC_NOERR) { - filetype = MPI_BYTE; - xtype = MPI_BYTE; - nbytes = 0; - nelems = 0; - if (status == NC_NOERR) status = err; - } - } - - /* TODO: if record variables are too big (so big that we cannot store the - * stride between records in an MPI_Aint, for example) then we will - * have to process this one record at a time. - */ - - fh = ncp->independent_fh; - coll_indep = NC_REQ_INDEP; - if (ncp->nprocs > 1 && fIsSet(reqMode, NC_REQ_COLL)) { - fh = ncp->collective_fh; - coll_indep = NC_REQ_COLL; - } - - /* MPI_File_set_view is collective */ - err = ncmpio_file_set_view(ncp, fh, &offset, filetype); - if (err != NC_NOERR) { - nelems = 0; /* skip this request */ - if (status == NC_NOERR) status = err; - } - if (filetype != MPI_BYTE) MPI_Type_free(&filetype); - /* xtype is the element data type (MPI primitive type) in xbuf to be - * read from the variable defined in file. Note xbuf will contain data read - * from the file and hence is in the external data type. - */ - err = ncmpio_read_write(ncp, NC_REQ_RD, coll_indep, offset, nelems, xtype, - xbuf, xtype_is_contig); + err = ncmpio_ina_req(ncp, NC_REQ_RD, varp, start, count, stride, nbytes, + xbuf); if (status == NC_NOERR) status = err; if (nelems > 0) { @@ -557,7 +465,9 @@ err_check: if (status == NC_NOERR) status = err; } - if (xbuf != buf) NCI_Free(xbuf); + if (varp != NULL && xbuf != buf) + /* xbuf may be allocated only if this is a non-zero-sized request */ + NCI_Free(xbuf); return status; } @@ -599,7 +509,19 @@ ncmpio_$1_var(void *ncdp, { NC *ncp=(NC*)ncdp; - NC_var *varp=NULL; + NC_var *varp; + + /* Check if this is a true zero-sized request. Note NC_REQ_ZERO is added to + * reqMode only when an error is detected at the dispatcher level. + */ + if (!fIsSet(reqMode, NC_REQ_ZERO)) { + int i; + for (i=0; ivars.value[varid]->ndims; i++) + if (count[i] == 0) { + reqMode |= NC_REQ_ZERO; + break; + } + } /* sanity check has been done at dispatchers */ @@ -608,18 +530,16 @@ ncmpio_$1_var(void *ncdp, * write, they still need to participate the communication part of the * intra-node aggregation operation. */ - ifelse(`$1',`put',`if (ncp->my_aggr >= 0) - return $1_varm(ncp, NULL, NULL, NULL, NULL, imap, NULL, 0, buftype, reqMode);') - - /* this collective API has a zero-length request */ - return ncmpio_getput_zero_req(ncp, reqMode); + return $1_varm(ncp, NULL, NULL, NULL, NULL, imap, NULL, 0, + buftype, reqMode); } /* obtain NC_var object pointer, varp. Note sanity check for ncdp and - * varid has been done in dispatchers */ + * varid has been done in dispatchers + */ varp = ncp->vars.value[varid]; -#ifdef ENABLE_SUBFILING +#if PNETCDF_SUBFILING == 1 /* call a separate routine if variable is stored in subfiles */ if (varp->num_subfiles > 1) { if (imap != NULL) { diff --git a/src/drivers/ncmpio/ncmpio_header_get.c b/src/drivers/ncmpio/ncmpio_header_get.c index 6ddd89bc10..43faadbaa8 100644 --- a/src/drivers/ncmpio/ncmpio_header_get.c +++ b/src/drivers/ncmpio/ncmpio_header_get.c @@ -34,7 +34,7 @@ static int compute_var_shape(NC *ncp) { - int i, err; + int i, err, last_fix; NC_var *first_var = NULL; /* first "non-record" var */ NC_var *first_rec = NULL; /* first "record" var */ @@ -44,6 +44,7 @@ compute_var_shape(NC *ncp) ncp->begin_rec = ncp->xsz; ncp->recsize = 0; + last_fix = -1; for (i=0; ivars.ndefined; i++) { /* ncp->vars.value[i]->len will be recomputed from dimensions in * ncmpio_NC_var_shape64() */ @@ -62,8 +63,14 @@ compute_var_shape(NC *ncp) */ ncp->begin_rec = ncp->vars.value[i]->begin + ncp->vars.value[i]->len; + last_fix = i; } } + if (last_fix >= 0) + ncp->fix_end = ncp->vars.value[last_fix]->begin + + ncp->vars.value[last_fix]->len; + else + ncp->fix_end = ncp->begin_var; if (first_rec != NULL) { if (ncp->begin_rec > first_rec->begin) @@ -316,103 +323,92 @@ hdr_len_NC_vararray(const NC_vararray *ncap, /*----< hdr_fetch() >--------------------------------------------------------*/ /* Fetch the next header chunk. The chunk buffer, pointed by gbp->base, is of - * size 'gbp->chunk' bytes. Be careful not to overwrite leftover (yet to be - * used) data in the buffer before fetching a new chunk. + * size 'gbp->ncp->hdr_chunk' bytes. Be careful not to overwrite leftover (yet + * to be used) data in the buffer before fetching a new chunk. */ static int hdr_fetch(bufferinfo *gbp) { - char *mpi_name; int rank, nprocs, err=NC_NOERR, mpireturn; - MPI_Status mpistatus; assert(gbp->base != NULL); - MPI_Comm_size(gbp->comm, &nprocs); - MPI_Comm_rank(gbp->comm, &rank); - if (rank == 0) { + MPI_Comm_size(gbp->ncp->comm, &nprocs); + MPI_Comm_rank(gbp->ncp->comm, &rank); + + if (rank == 0) { /* only root reads file header */ char *readBuf; - int readLen; size_t slack; + MPI_Offset rlen, file_off=gbp->offset, buf_off=0, readLen; + PNCIO_View file_view, buf_view; + + /* For fetching file header, both file view and buffer view are + * contiguous. + */ + file_view.count = 1; + file_view.off = &file_off; + file_view.len = &readLen; + + buf_view.count = 1; + buf_view.off = &buf_off; + buf_view.len = &readLen; /* any leftover data in the buffer */ - slack = gbp->chunk - (gbp->pos - gbp->base); - if (slack == gbp->chunk) slack = 0; + slack = gbp->ncp->hdr_chunk - (gbp->pos - gbp->base); + if (slack == gbp->ncp->hdr_chunk) slack = 0; - /* When gbp->chunk == (gbp->pos - gbp->base), all data in the buffer has - * been consumed. If not, then read additional header of size - * (gbp->chunk - slack) into a contiguous buffer, pointed by gbp->base + - * slack. + /* When gbp->ncp->hdr_chunk == (gbp->pos - gbp->base), all data in the + * buffer has been consumed. If not, then read additional header of + * size (gbp->ncp->hdr_chunk - slack) into a contiguous buffer, pointed + * by gbp->base + slack. */ readBuf = gbp->base; - readLen = gbp->chunk; + readLen = gbp->ncp->hdr_chunk; if (slack > 0) { /* move slack to beginning of the buffer, gbp->base */ memmove(gbp->base, gbp->pos, slack); readBuf += slack; readLen -= slack; } - /* explicitly initialize mpistatus object to 0. For zero-length read, - * MPI_Get_count may report incorrect result for some MPICH version, - * due to the uninitialized MPI_Status object passed to MPI-IO calls. - */ - memset(&mpistatus, 0, sizeof(MPI_Status)); - - /* fileview is already entire file visible and MPI_File_read_at does - not change the file pointer */ - if (gbp->coll_mode == 1) { /* collective read */ - TRACE_IO(MPI_File_read_at_all, (gbp->collective_fh, gbp->offset, readBuf, - readLen, MPI_BYTE, &mpistatus)); - } - else { - TRACE_IO(MPI_File_read_at, (gbp->collective_fh, gbp->offset, readBuf, - readLen, MPI_BYTE, &mpistatus)); - } - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, mpi_name); - if (err == NC_EFILE) DEBUG_ASSIGN_ERROR(err, NC_EREAD) - } - else { - /* Obtain the actual read amount. It may be smaller than readLen, - * when the remaining file size is smaller than read chunk size. - * Because each MPI File_read reads amount of readLen bytes, and - * readLen <= read chunk size which is <= NC_MAX_INT, calling - * MPI_Get_count() is sufficient. No need to call MPI_Get_count_c() - */ - int get_size; - MPI_Get_count(&mpistatus, MPI_BYTE, &get_size); - gbp->get_size += get_size; + file_off = gbp->offset; + rlen = ncmpio_file_read(gbp->ncp, NC_REQ_INDEP, readBuf, file_view, buf_view); - /* If actual read amount is shorter than readLen, then we zero-out - * the remaining buffer. This is because the MPI_Bcast below - * broadcasts a buffer of a fixed size, gbp->chunk. Without zeroing + if (rlen > 0) { + /* rlen is the actual read amount. It may be smaller than readLen, + * when the remaining file size is smaller than readLen. When + * actual read amount is smaller than readLen, then we zero-out the + * remaining buffer. This is because the MPI_Bcast below broadcasts + * a buffer of a fixed size, gbp->ncp->hdr_chunk. Without zeroing * out, valgrind will complain about the uninitialized values. */ - if (get_size < readLen) - memset(readBuf + get_size, 0, readLen - get_size); + if (rlen < readLen) + memset(readBuf + rlen, 0, readLen - rlen); + + /* update the number of bytes read since file open */ + gbp->ncp->get_size += rlen; } + else if (rlen < 0) + err = (int)rlen; + /* only root process reads file header, keeps track of current read * file pointer location */ - gbp->offset += readLen; - } - else if (gbp->coll_mode == 1) { /* collective read */ - /* other processes participate the collective call */ - TRACE_IO(MPI_File_read_at_all, (gbp->collective_fh, 0, NULL, - 0, MPI_BYTE, &mpistatus)); - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, mpi_name); - if (err == NC_EFILE) DEBUG_ASSIGN_ERROR(err, NC_EREAD) - } + gbp->offset += rlen; } - if (gbp->safe_mode == 1 && nprocs > 1) { - TRACE_COMM(MPI_Bcast)(&err, 1, MPI_INT, 0, gbp->comm); + if (fIsSet(gbp->ncp->flags, NC_MODE_SAFE) && nprocs > 1) { + TRACE_COMM(MPI_Bcast)(&err, 1, MPI_INT, 0, gbp->ncp->comm); + if (mpireturn != MPI_SUCCESS) + return ncmpii_error_mpi2nc(mpireturn, "MPI_Bcast"); if (err != NC_NOERR) return err; } /* broadcast root's read (full or partial header) to other processes */ - if (nprocs > 1) - TRACE_COMM(MPI_Bcast)(gbp->base, gbp->chunk, MPI_BYTE, 0, gbp->comm); + if (nprocs > 1) { + TRACE_COMM(MPI_Bcast)(gbp->base, gbp->ncp->hdr_chunk, MPI_BYTE, 0, + gbp->ncp->comm); + if (mpireturn != MPI_SUCCESS) + return ncmpii_error_mpi2nc(mpireturn, "MPI_Bcast"); + } gbp->pos = gbp->base; @@ -503,7 +499,7 @@ hdr_get_nc_type(bufferinfo *gbp, nc_type *xtypep) if (xtype < NC_BYTE) DEBUG_RETURN_ERROR(NC_EBADTYPE) - if (gbp->version < 5) { + if (gbp->ncp->format < 5) { if (xtype > NC_DOUBLE) DEBUG_RETURN_ERROR(NC_EBADTYPE) } @@ -536,7 +532,7 @@ hdr_get_NC_name(bufferinfo *gbp, char **namep, size_t *name_len) *namep = NULL; /* get nelems (string length of name) */ - if (gbp->version < 5) { + if (gbp->ncp->format < 5) { uint tmp; err = hdr_get_uint32(gbp, &tmp); if (err != NC_NOERR) return err; @@ -564,7 +560,7 @@ hdr_get_NC_name(bufferinfo *gbp, char **namep, size_t *name_len) */ padding = PNETCDF_RNDUP(nchars, X_ALIGN) - nchars; - bufremain = gbp->chunk - (gbp->pos - gbp->base); + bufremain = gbp->ncp->hdr_chunk - (gbp->pos - gbp->base); cpos = *namep; @@ -585,7 +581,7 @@ hdr_get_NC_name(bufferinfo *gbp, char **namep, size_t *name_len) *namep = NULL; return err; } - bufremain = gbp->chunk; + bufremain = gbp->ncp->hdr_chunk; } } @@ -618,11 +614,11 @@ hdr_get_NC_name(bufferinfo *gbp, char **namep, size_t *name_len) * and discussion in NetCDF Github issue * https://github.com/Unidata/netcdf-c/issues/657. */ -#ifdef ENABLE_NULL_BYTE_HEADER_PADDING +#if PNETCDF_NULL_BYTE_HEADER_PADDING == 1 char pad[X_ALIGN-1]; memset(pad, 0, X_ALIGN-1); if (memcmp(gbp->pos, pad, padding) != 0) { -#ifdef PNETCDF_DEBUG +#if PNETCDF_DEBUG_MODE == 1 fprintf(stderr,"Error in file %s func %s line %d: NetCDF header corrupted, non-zero padding found\n",__FILE__,__func__,__LINE__); #endif DEBUG_ASSIGN_ERROR(err, NC_ENULLPAD) /* not a fatal error */ @@ -659,7 +655,7 @@ hdr_get_NC_dim(bufferinfo *gbp, int unlimited_id, NC_dim **dimpp) else if (err != NC_NOERR) return err; /* get dim_length */ - if (gbp->version < 5) { + if (gbp->ncp->format < 5) { uint tmp; err = hdr_get_uint32(gbp, &tmp); dim_length = (MPI_Offset)tmp; @@ -730,7 +726,7 @@ hdr_get_NC_dimarray(bufferinfo *gbp, NC_dimarray *ncap) if (err != NC_NOERR) return err; /* read nelems (number of dimensions) from gbp buffer */ - if (gbp->version < 5) { /* nelems is */ + if (gbp->ncp->format < 5) { /* nelems is */ uint tmp; err = hdr_get_uint32(gbp, &tmp); if (err != NC_NOERR) return err; @@ -760,7 +756,7 @@ hdr_get_NC_dimarray(bufferinfo *gbp, NC_dimarray *ncap) /* Now, ndefined > 0, tag must be NC_DIMENSION */ if (tag != NC_DIMENSION) { -#ifdef PNETCDF_DEBUG +#if PNETCDF_DEBUG_MODE == 1 fprintf(stderr,"Error in file %s func %s line %d: NetCDF header corrupted, expecting tag NC_DIMENSION but got %d\n",__FILE__,__func__,__LINE__,tag); #endif DEBUG_RETURN_ERROR(NC_ENOTNC) @@ -809,8 +805,8 @@ hdr_get_NC_attrV(bufferinfo *gbp, NC_attr *attrp) nbytes = attrp->nelems * xsz; padding = attrp->xsz - nbytes; - bufremain = gbp->chunk - (gbp->pos - gbp->base); - /* gbp->chunk is the read chunk size, which is of type 4-byte int. + bufremain = gbp->ncp->hdr_chunk - (gbp->pos - gbp->base); + /* gbp->ncp->hdr_chunk is the read chunk size, which is of type 4-byte int. * thus bufremain should be less than INT_MAX */ /* get values */ @@ -823,10 +819,9 @@ hdr_get_NC_attrV(bufferinfo *gbp, NC_attr *attrp) value = (void *)((char *)value + attcount); bufremain -= attcount; } else { - int err; err = hdr_fetch(gbp); if (err != NC_NOERR) return err; - bufremain = gbp->chunk; + bufremain = gbp->ncp->hdr_chunk; } } @@ -859,11 +854,11 @@ hdr_get_NC_attrV(bufferinfo *gbp, NC_attr *attrp) * and discussion in NetCDF Github issue * https://github.com/Unidata/netcdf-c/issues/657. */ -#ifdef ENABLE_NULL_BYTE_HEADER_PADDING +#if PNETCDF_NULL_BYTE_HEADER_PADDING == 1 char pad[X_ALIGN-1]; memset(pad, 0, X_ALIGN-1); if (memcmp(gbp->pos, pad, padding) != 0) { -#ifdef PNETCDF_DEBUG +#if PNETCDF_DEBUG_MODE == 1 fprintf(stderr,"Error in file %s func %s line %d: NetCDF header corrupted, non-zero padding found\n",__FILE__,__func__,__LINE__); #endif DEBUG_ASSIGN_ERROR(err, NC_ENULLPAD) @@ -906,7 +901,7 @@ hdr_get_NC_attr(bufferinfo *gbp, NC_attr **attrpp) } /* get nelems */ - if (gbp->version < 5) { + if (gbp->ncp->format < 5) { uint tmp; err = hdr_get_uint32(gbp, &tmp); nelems = (MPI_Offset)tmp; @@ -977,7 +972,7 @@ hdr_get_NC_attrarray(bufferinfo *gbp, NC_attrarray *ncap) if (err != NC_NOERR) return err; /* read nelems (number of attributes) from gbp buffer */ - if (gbp->version < 5) { /* nelems is */ + if (gbp->ncp->format < 5) { /* nelems is */ uint tmp; err = hdr_get_uint32(gbp, &tmp); if (err != NC_NOERR) return err; @@ -1005,7 +1000,7 @@ hdr_get_NC_attrarray(bufferinfo *gbp, NC_attrarray *ncap) /* Now, ndefined > 0, tag must be NC_ATTRIBUTE */ if (tag != NC_ATTRIBUTE) { -#ifdef PNETCDF_DEBUG +#if PNETCDF_DEBUG_MODE == 1 fprintf(stderr,"Error in file %s func %s line %d: NetCDF header corrupted, expecting tag NC_ATTRIBUTE but got %d\n",__FILE__,__func__,__LINE__,tag); #endif DEBUG_RETURN_ERROR(NC_ENOTNC) @@ -1061,7 +1056,7 @@ hdr_get_NC_var(bufferinfo *gbp, else if (err != NC_NOERR) return err; /* nelems (number of dimensions) */ - if (gbp->version < 5) { + if (gbp->ncp->format < 5) { uint tmp; err = hdr_get_uint32(gbp, &tmp); if (err != NC_NOERR) { @@ -1099,7 +1094,7 @@ hdr_get_NC_var(bufferinfo *gbp, /* get [dimid ...] */ for (dim=0; dimversion < 5) { + if (gbp->ncp->format < 5) { uint tmp; err = hdr_get_uint32(gbp, &tmp); if (err != NC_NOERR) break; @@ -1135,7 +1130,7 @@ hdr_get_NC_var(bufferinfo *gbp, ncmpii_xlen_nc_type(varp->xtype, &varp->xsz); /* get vsize */ - if (gbp->version < 5) { + if (gbp->ncp->format < 5) { uint tmp; err = hdr_get_uint32(gbp, &tmp); varp->len = (MPI_Offset)tmp; @@ -1164,7 +1159,7 @@ hdr_get_NC_var(bufferinfo *gbp, */ /* get begin */ - if (gbp->version == 1) { + if (gbp->ncp->format == 1) { uint tmp; err = hdr_get_uint32(gbp, &tmp); varp->begin = (MPI_Offset)tmp; @@ -1223,7 +1218,7 @@ hdr_get_NC_vararray(bufferinfo *gbp, if (err != NC_NOERR) return err; /* read nelems (number of variables) from gbp buffer */ - if (gbp->version < 5) { /* nelems is */ + if (gbp->ncp->format < 5) { /* nelems is */ uint tmp; err = hdr_get_uint32(gbp, &tmp); if (err != NC_NOERR) return err; @@ -1251,7 +1246,7 @@ hdr_get_NC_vararray(bufferinfo *gbp, /* Now, ndefined > 0, tag must be NC_VARIABLE */ if (tag != NC_VARIABLE) { -#ifdef PNETCDF_DEBUG +#if PNETCDF_DEBUG_MODE == 1 fprintf(stderr,"Error in file %s func %s line %d: NetCDF header corrupted, expecting tag NC_VARIABLE but got %d\n",__FILE__,__func__,__LINE__,tag); #endif DEBUG_RETURN_ERROR(NC_ENOTNC) @@ -1339,30 +1334,22 @@ ncmpio_hdr_get_NC(NC *ncp) assert(ncp != NULL); /* Initialize the get buffer that stores the header read from the file */ - getbuf.comm = ncp->comm; - getbuf.collective_fh = ncp->collective_fh; - getbuf.get_size = 0; - getbuf.offset = 0; /* read from start of the file */ - getbuf.safe_mode = ncp->safe_mode; - if (ncp->nprocs > 1 && fIsSet(ncp->flags, NC_HCOLL)) - getbuf.coll_mode = 1; - else - getbuf.coll_mode = 0; - - /* CDF-5's minimum header size is 4 bytes more than CDF-1 and CDF-2's */ - getbuf.chunk = PNETCDF_RNDUP( MAX(MIN_NC_XSZ+4, ncp->chunk), X_ALIGN ); - - getbuf.base = (char*) NCI_Malloc(getbuf.chunk); - getbuf.pos = getbuf.base; - getbuf.end = getbuf.base + getbuf.chunk; - - /* Fetch the next header chunk. The chunk is 'gbp->chunk' bytes big */ + getbuf.ncp = ncp; + getbuf.offset = 0; /* read from start of the file */ + getbuf.base = (char*) NCI_Malloc(getbuf.ncp->hdr_chunk); + getbuf.pos = getbuf.base; + getbuf.end = getbuf.base + getbuf.ncp->hdr_chunk; + + /* Fetch the first chunk of file header. The chunk size is of size + * 'gbp->ncp->hdr_chunk' bytes. Its value can be customized by setting the + * PnetCDF I/O hint 'nc_header_read_chunk_size'. + */ err = hdr_fetch(&getbuf); if (err != NC_NOERR) return err; /* processing the header from getbuf, the get buffer */ - /* First get the file format information, magic */ + /* First get the file format signature, "magic" = "CDF"+version */ err = ncmpix_getn_text((const void **)(&getbuf.pos), NC_MAGIC_LEN, magic); if (err != NC_NOERR) return err; @@ -1373,7 +1360,7 @@ ncmpio_hdr_get_NC(NC *ncp) ncmpix_getn_text((const void **)(&getbuf.pos), 8, signature); if (memcmp(signature, hdf5_signature, 8) == 0) { DEBUG_ASSIGN_ERROR(err, NC_ENOTNC3) - if (ncp->safe_mode) + if (fIsSet(ncp->flags, NC_MODE_SAFE)) fprintf(stderr,"Error: file %s is HDF5 format\n",ncp->path); } else @@ -1381,20 +1368,20 @@ ncmpio_hdr_get_NC(NC *ncp) goto fn_exit; } - /* check version number in last byte of magic */ - if (magic[3] == 0x1) { - getbuf.version = ncp->format = 1; - } else if (magic[3] == 0x2) { - getbuf.version = ncp->format = 2; - } else if (magic[3] == 0x5) { - getbuf.version = ncp->format = 5; - } else { + /* check format version number in last byte of magic */ + if (magic[3] == 0x1) + ncp->format = 1; + else if (magic[3] == 0x2) + ncp->format = 2; + else if (magic[3] == 0x5) + ncp->format = 5; + else { NCI_Free(getbuf.base); DEBUG_RETURN_ERROR(NC_ENOTNC) /* not a netCDF file */ } - /* get numrecs from getbuf into ncp */ - if (getbuf.version < 5) { + /* After "magic" is "numrecs", get numrecs from getbuf into ncp */ + if (getbuf.ncp->format < 5) { uint tmp=0; err = hdr_get_uint32(&getbuf, &tmp); if (err != NC_NOERR) goto fn_exit; @@ -1407,19 +1394,22 @@ ncmpio_hdr_get_NC(NC *ncp) ncp->numrecs = (MPI_Offset)tmp; } +#if PNETCDF_DEBUG_MODE == 1 + /* getbuf.pos now should be 4 bytes for CDF-1 and 2, 8 bytes for CDF-5 */ assert(getbuf.pos < getbuf.end); +#endif - /* get dim_list from getbuf into ncp */ + /* After "numrecs" is "dim_list", get dim_list from getbuf into ncp */ err = hdr_get_NC_dimarray(&getbuf, &ncp->dims); if (err == NC_ENULLPAD) status = NC_ENULLPAD; /* non-fatal error */ else if (err != NC_NOERR) goto fn_exit; - /* get gatt_list from getbuf into ncp */ + /* After "dim_list" is "gatt_list", get gatt_list from getbuf into ncp */ err = hdr_get_NC_attrarray(&getbuf, &ncp->attrs); if (err == NC_ENULLPAD) status = NC_ENULLPAD; /* non-fatal error */ else if (err != NC_NOERR) goto fn_exit; - /* get var_list from getbuf into ncp */ + /* After "gatt_list" is "var_list", get var_list from getbuf into ncp */ err = hdr_get_NC_vararray(&getbuf, &ncp->vars, ncp->dims.ndefined); if (err == NC_ENULLPAD) status = NC_ENULLPAD; /* non-fatal error */ else if (err != NC_NOERR) goto fn_exit; @@ -1449,7 +1439,6 @@ ncmpio_hdr_get_NC(NC *ncp) if (err != NC_NOERR) goto fn_exit; fn_exit: - ncp->get_size += getbuf.get_size; NCI_Free(getbuf.base); return (err == NC_NOERR) ? status : err; diff --git a/src/drivers/ncmpio/ncmpio_header_put.c b/src/drivers/ncmpio/ncmpio_header_put.c index 8daf88c678..a2bafbb66f 100644 --- a/src/drivers/ncmpio/ncmpio_header_put.c +++ b/src/drivers/ncmpio/ncmpio_header_put.c @@ -49,7 +49,7 @@ hdr_put_NC_name(bufferinfo *pbp, size_t nchars = strlen(name); /* copy nelems */ - if (pbp->version < 5) + if (pbp->ncp->format < 5) err = ncmpix_put_uint32((void**)(&pbp->pos), (uint)nchars); else err = ncmpix_put_uint64((void**)(&pbp->pos), (uint64)nchars); @@ -78,7 +78,7 @@ hdr_put_NC_dim(bufferinfo *pbp, if (err != NC_NOERR) return err; /* copy dim_length */ - if (pbp->version < 5) { + if (pbp->ncp->format < 5) { /* TODO: Isn't checking dimension size already done in def_dim()? */ if (dimp->size > NC_MAX_INT) DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) @@ -116,7 +116,7 @@ hdr_put_NC_dimarray(bufferinfo *pbp, if (status != NC_NOERR) return status; /* put a ZERO or ZERO64 depending on which CDF format */ - if (pbp->version < 5) + if (pbp->ncp->format < 5) status = ncmpix_put_uint32((void**)(&pbp->pos), 0); else status = ncmpix_put_uint64((void**)(&pbp->pos), 0); @@ -128,7 +128,7 @@ hdr_put_NC_dimarray(bufferinfo *pbp, if (status != NC_NOERR) return status; /* copy nelems */ - if (pbp->version < 5) + if (pbp->ncp->format < 5) status = ncmpix_put_uint32((void**)(&pbp->pos), (uint)ncap->ndefined); else status = ncmpix_put_uint64((void**)(&pbp->pos), (uint64)ncap->ndefined); @@ -175,7 +175,7 @@ hdr_put_NC_attrV(bufferinfo *pbp, sz = attrp->nelems * xsz; padding = attrp->xsz - sz; - if (pbp->version < 5 && sz > NC_MAX_INT) + if (pbp->ncp->format < 5 && sz > NC_MAX_INT) DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) memcpy(pbp->pos, attrp->xvalue, (size_t)sz); @@ -214,7 +214,7 @@ hdr_put_NC_attr(bufferinfo *pbp, if (status != NC_NOERR) return status; /* copy nelems */ - if (pbp->version < 5) { + if (pbp->ncp->format < 5) { if (attrp->nelems > NC_MAX_INT) DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) status = ncmpix_put_uint32((void**)(&pbp->pos), (uint)attrp->nelems); @@ -258,7 +258,7 @@ hdr_put_NC_attrarray(bufferinfo *pbp, if (status != NC_NOERR) return status; /* put a ZERO or ZERO64 depending on which CDF format */ - if (pbp->version < 5) + if (pbp->ncp->format < 5) status = ncmpix_put_uint32((void**)(&pbp->pos), 0); else status = ncmpix_put_uint64((void**)(&pbp->pos), 0); @@ -270,7 +270,7 @@ hdr_put_NC_attrarray(bufferinfo *pbp, if (status != NC_NOERR) return status; /* copy nelems */ - if (pbp->version < 5) + if (pbp->ncp->format < 5) status = ncmpix_put_uint32((void**)(&pbp->pos), (uint)ncap->ndefined); else status = ncmpix_put_uint64((void**)(&pbp->pos), (uint64)ncap->ndefined); @@ -314,7 +314,7 @@ hdr_put_NC_var(bufferinfo *pbp, if (status != NC_NOERR) return status; /* copy nelems */ - if (pbp->version < 5) + if (pbp->ncp->format < 5) status = ncmpix_put_uint32((void**)(&pbp->pos), (uint)varp->ndims); else status = ncmpix_put_uint64((void**)(&pbp->pos), (uint64)varp->ndims); @@ -322,7 +322,7 @@ hdr_put_NC_var(bufferinfo *pbp, /* copy [dimid ...] */ for (i=0; indims; i++) { - if (pbp->version < 5) + if (pbp->ncp->format < 5) status = ncmpix_put_uint32((void**)(&pbp->pos), (uint)varp->dimids[i]); else status = ncmpix_put_uint64((void**)(&pbp->pos), (uint64)varp->dimids[i]); @@ -341,7 +341,7 @@ hdr_put_NC_var(bufferinfo *pbp, /* in CDF-1 and CDF-2, a variable's size in the header is a 32-bit integer * in CDF-5, it is a 64-bit integer */ - if (pbp->version < 5) { + if (pbp->ncp->format < 5) { /* Special case, when there is no record variable, the last fixed-size * variable can be larger than 2 GiB if its file starting offset is * less than 2 GiB. This checking has already been done in the call @@ -367,7 +367,7 @@ hdr_put_NC_var(bufferinfo *pbp, /* in CDF-1 header, a variable's starting file offset is a 32-bit integer * in CDF-2 and CDF-5, it is a 64-bit integer */ - if (pbp->version == 1) { + if (pbp->ncp->format == 1) { if (varp->begin > NC_MAX_INT) DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) status = ncmpix_put_uint32((void**)(&pbp->pos), (uint)varp->begin); @@ -407,7 +407,7 @@ hdr_put_NC_vararray(bufferinfo *pbp, if (status != NC_NOERR) return status; /* put a ZERO or ZERO64 depending on which CDF format */ - if (pbp->version < 5) + if (pbp->ncp->format < 5) status = ncmpix_put_uint32((void**)(&pbp->pos), 0); else status = ncmpix_put_uint64((void**)(&pbp->pos), 0); @@ -419,7 +419,7 @@ hdr_put_NC_vararray(bufferinfo *pbp, if (status != NC_NOERR) return status; /* copy nelems */ - if (pbp->version < 5) + if (pbp->ncp->format < 5) status = ncmpix_put_uint32((void**)(&pbp->pos), (uint)ncap->ndefined); else status = ncmpix_put_uint64((void**)(&pbp->pos), (uint64)ncap->ndefined); @@ -441,20 +441,14 @@ hdr_put_NC_vararray(bufferinfo *pbp, int ncmpio_hdr_put_NC(NC *ncp, void *buf) { - int status; + int err; bufferinfo putbuf; MPI_Offset nrecs=0; - putbuf.comm = ncp->comm; - putbuf.collective_fh = ncp->collective_fh; - putbuf.offset = 0; - putbuf.pos = buf; - putbuf.base = buf; - putbuf.safe_mode = ncp->safe_mode; - if (ncp->nprocs > 1 && fIsSet(ncp->flags, NC_HCOLL)) - putbuf.coll_mode = 1; - else - putbuf.coll_mode = 0; + putbuf.ncp = ncp; + putbuf.offset = 0; + putbuf.pos = buf; + putbuf.base = buf; /* netCDF file format: * netcdf_file = header data @@ -462,43 +456,37 @@ ncmpio_hdr_put_NC(NC *ncp, void *buf) */ /* copy "magic", 4 characters */ - if (ncp->format == 5) { - putbuf.version = 5; - status = ncmpix_putn_text((void **)(&putbuf.pos), sizeof(ncmagic5), ncmagic5); - } - else if (ncp->format == 2) { - putbuf.version = 2; - status = ncmpix_putn_text((void **)(&putbuf.pos), sizeof(ncmagic2), ncmagic2); - } - else { - putbuf.version = 1; - status = ncmpix_putn_text((void **)(&putbuf.pos), sizeof(ncmagic1), ncmagic1); - } - if (status != NC_NOERR) return status; + if (ncp->format == 5) + err = ncmpix_putn_text((void **)(&putbuf.pos), sizeof(ncmagic5), ncmagic5); + else if (ncp->format == 2) + err = ncmpix_putn_text((void **)(&putbuf.pos), sizeof(ncmagic2), ncmagic2); + else + err = ncmpix_putn_text((void **)(&putbuf.pos), sizeof(ncmagic1), ncmagic1); + if (err != NC_NOERR) return err; /* copy numrecs, number of records */ nrecs = ncp->numrecs; if (ncp->format < 5) { if (nrecs > NC_MAX_INT) DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) - status = ncmpix_put_uint32((void**)(&putbuf.pos), (uint)nrecs); + err = ncmpix_put_uint32((void**)(&putbuf.pos), (uint)nrecs); } else { - status = ncmpix_put_uint64((void**)(&putbuf.pos), (uint64)nrecs); + err = ncmpix_put_uint64((void**)(&putbuf.pos), (uint64)nrecs); } - if (status != NC_NOERR) return status; + if (err != NC_NOERR) return err; /* copy dim_list */ - status = hdr_put_NC_dimarray(&putbuf, &ncp->dims); - if (status != NC_NOERR) return status; + err = hdr_put_NC_dimarray(&putbuf, &ncp->dims); + if (err != NC_NOERR) return err; /* copy gatt_list */ - status = hdr_put_NC_attrarray(&putbuf, &ncp->attrs); - if (status != NC_NOERR) return status; + err = hdr_put_NC_attrarray(&putbuf, &ncp->attrs); + if (err != NC_NOERR) return err; /* copy var_list */ - status = hdr_put_NC_vararray(&putbuf, &ncp->vars); - if (status != NC_NOERR) return status; + err = hdr_put_NC_vararray(&putbuf, &ncp->vars); + if (err != NC_NOERR) return err; return NC_NOERR; } @@ -514,11 +502,16 @@ ncmpio_hdr_put_NC(NC *ncp, void *buf) */ int ncmpio_write_header(NC *ncp) { - char *mpi_name; - int status=NC_NOERR, mpireturn, err; + int status=NC_NOERR, mpireturn; size_t i, ntimes; - MPI_File fh; - MPI_Status mpistatus; + +#if 0 + PNCIO_View buf_view; + buf_view.count = 0; /* indicating buffer in this request is contiguous */ + /* Note buf_view.size will be set to the file header size in this + * subroutine, a positive value. + */ +#endif /* Write the entire header to the file. This function may be called from * a rename API. In that case, we cannot just change the variable name in @@ -526,10 +519,6 @@ int ncmpio_write_header(NC *ncp) * all metadata following the new name must be moved ahead. */ - fh = ncp->collective_fh; - if (NC_indep(ncp)) /* called in independent data mode */ - fh = ncp->independent_fh; - /* update file header size, as this subroutine may be called from a rename * API (var or attribute) and the new name is smaller/bigger which changes * the header size. We recalculate ncp->xsz by getting the un-aligned size @@ -540,10 +529,21 @@ int ncmpio_write_header(NC *ncp) if (ncp->xsz % NC_MAX_INT) ntimes++; if (ncp->rank == 0) { /* only root writes to file header */ - MPI_Offset offset; - size_t remain; - size_t bufLen = PNETCDF_RNDUP(ncp->xsz, X_ALIGN); char *buf, *buf_ptr; + size_t bufLen = PNETCDF_RNDUP(ncp->xsz, X_ALIGN); + MPI_Offset offset, file_off, buf_off=0, writeLen, remain; + PNCIO_View file_view, buf_view; + + /* For writing to file header, both file view and buffer view are + * contiguous. + */ + file_view.count = 1; + file_view.off = &file_off; + file_view.len = &writeLen; + + buf_view.count = 1; + buf_view.off = &buf_off; + buf_view.len = &writeLen; buf = NCI_Malloc(bufLen); /* header's write buffer */ @@ -554,57 +554,28 @@ int ncmpio_write_header(NC *ncp) remain = ncp->xsz; buf_ptr = buf; for (i=0; iflags, NC_HCOLL)) { /* header collective write */ - TRACE_IO(MPI_File_write_at_all, (fh, offset, buf_ptr, writeLen, - MPI_BYTE, &mpistatus)); - } - else { /* header independent write */ - TRACE_IO(MPI_File_write_at, (fh, offset, buf_ptr, writeLen, - MPI_BYTE, &mpistatus)); - } - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, mpi_name); - if (status == NC_NOERR) { - err = (err == NC_EFILE) ? NC_EWRITE : err; - DEBUG_ASSIGN_ERROR(status, err) - } - } - else { - /* update the number of bytes written since file open. - * Because each MPI write writes no more than NC_MAX_INT, - * calling MPI_Get_count() is sufficient. No need to call - * MPI_Get_count_c() - */ - int put_size; - mpireturn = MPI_Get_count(&mpistatus, MPI_BYTE, &put_size); - if (mpireturn != MPI_SUCCESS || put_size == MPI_UNDEFINED) - ncp->put_size += ncp->xsz; - else - ncp->put_size += writeLen; - } offset += writeLen; buf_ptr += writeLen; remain -= writeLen; } NCI_Free(buf); } - else if (fIsSet(ncp->flags, NC_HCOLL)) { /* header collective write */ - /* collective write: other processes participate the collective call */ - for (i=0; isafe_mode == 1) { + if (fIsSet(ncp->flags, NC_MODE_SAFE)) { /* broadcast root's status, because only root writes to the file */ int root_status = status; TRACE_COMM(MPI_Bcast)(&root_status, 1, MPI_INT, 0, ncp->comm); diff --git a/src/drivers/ncmpio/ncmpio_i_getput.m4 b/src/drivers/ncmpio/ncmpio_i_getput.m4 index 7f624207d3..631323c388 100644 --- a/src/drivers/ncmpio/ncmpio_i_getput.m4 +++ b/src/drivers/ncmpio/ncmpio_i_getput.m4 @@ -122,6 +122,11 @@ ncmpio_add_record_requests(NC_lead_req *lead_list, reqs[i].lead_off = reqs[0].lead_off; reqs[i].xbuf = xbuf; xbuf += rec_bufsize; + + /* copy the number of flattened offset-length pairs */ + reqs[i].npairs = reqs[0].npairs; + reqs[i].offset_start = reqs[0].offset_start; + reqs[i].offset_end = reqs[0].offset_end; } return NC_NOERR; @@ -142,7 +147,7 @@ ncmpio_igetput_varm(NC *ncp, int reqMode) { void *xbuf=NULL; - int i, err=NC_NOERR, abuf_index=-1, isize, xsize, new_nreqs, rem; + int i, j, err=NC_NOERR, abuf_index=-1, isize, xsize, new_nreqs, rem; int mpireturn, buftype_is_contig=1, need_convert, free_xbuf=0; int need_swap, can_swap_in_place, need_swap_back_buf=0; MPI_Offset nelems=0, nbytes, *ptr; @@ -192,7 +197,8 @@ ncmpio_igetput_varm(NC *ncp, /* When bufcount > NC_MAX_INT, we construct a datatype to bypass the * limitation of MPI file read/write APIs on the argument "count" of - * type int. See ncmpio_read_write() in ncmpio_file_io.c + * type int. See ncmpi_file_read()/ncmpi_file_write() in + * ncmpio_file_io.c * * Note not all MPI-IO libraries support single requests larger than * NC_MAX_INT. In this case, MPI-IO should report an error. @@ -520,9 +526,13 @@ ncmpio_igetput_varm(NC *ncp, } /* allocate a single array for non-leads to store start/count/stride */ + req->npairs = 0; if (varp->ndims == 0) { /* scalar variable, start may be NULL */ lead_req->start = NULL; req->start = NULL; + req->npairs = 1; + req->offset_start = 0; /* relative to var's begin */ + req->offset_end = varp->xsz; } else if (stride == NULL) { size_t memChunk = varp->ndims * SIZEOF_MPI_OFFSET; @@ -536,6 +546,12 @@ ncmpio_igetput_varm(NC *ncp, memcpy(ptr, start, memChunk); ptr += varp->ndims; memcpy(ptr, count, memChunk); + + /* calculate number of flattened offset-length pairs */ + req->npairs = 1; + j = IS_RECVAR(varp) ? 1 : 0; + for (i=j; indims-1; i++) + req->npairs *= count[i]; } else { size_t memChunk = varp->ndims * SIZEOF_MPI_OFFSET; @@ -551,12 +567,24 @@ ncmpio_igetput_varm(NC *ncp, memcpy(ptr, count, memChunk); ptr += varp->ndims; memcpy(ptr, stride, memChunk); + + /* calculate number of flattened offset-length pairs */ + req->npairs = (stride[varp->ndims-1] == 1) ? 1 : count[varp->ndims-1]; + j = IS_RECVAR(varp) ? 1 : 0; + for (i=j; indims-1; i++) + req->npairs *= count[i]; } /* set the properties of non-lead request */ req->xbuf = xbuf; req->nelems = nelems; + /* special treatment when there is only one offset-length pair */ + if (req->npairs == 1 && varp->ndims > 0) { + ncmpio_calc_off(ncp, varp, start, &req->offset_start); + req->offset_end = req->nelems * varp->xsz; + } + if (IS_RECVAR(varp)) { /* save the last record number accessed */ if (stride == NULL) @@ -576,6 +604,8 @@ ncmpio_igetput_varm(NC *ncp, : ncp->get_lead_list; req->nelems /= count[0]; + if (req->npairs == 1) + req->offset_end = req->nelems * varp->xsz; /* add (count[0]-1) number of (sub)requests */ ncmpio_add_record_requests(lead_list, req, count[0], stride); diff --git a/src/drivers/ncmpio/ncmpio_i_varn.m4 b/src/drivers/ncmpio/ncmpio_i_varn.m4 index be9af9752c..8bad268f01 100644 --- a/src/drivers/ncmpio/ncmpio_i_varn.m4 +++ b/src/drivers/ncmpio/ncmpio_i_varn.m4 @@ -452,6 +452,12 @@ igetput_varn(NC *ncp, lead_req->max_rec = -1; lead_req->nonlead_num = new_nreqs; +#if 0 +MPI_Aint addr; +MPI_Get_address(lead_req->xbuf, &addr); +printf("%s at %d: lead_req xbuf=%ld nelems=%lld\n",__func__,__LINE__, addr,lead_req->nelems); +#endif + /* varn APIs have no argument stride */ fSet(lead_req->flag, NC_REQ_STRIDE_NULL); @@ -466,6 +472,8 @@ igetput_varn(NC *ncp, xbufp = (char*)xbuf; for (i=0; inpairs = 0; + if (req_nelems[i] == 0) continue; /* ignore this 0-length request i */ req->nelems = req_nelems[i]; @@ -473,11 +481,17 @@ igetput_varn(NC *ncp, req->xbuf = xbufp; xbufp += req_nelems[i] * xsize; +#if 0 +MPI_Get_address(req->xbuf, &addr); +printf("%s at %d: req i=%d xbuf=%ld off=%ld nelems=%lld\n",__func__,__LINE__, i,addr,(char*)req->xbuf - (char*)xbuf,req->nelems); +#endif + /* copy starts[i] and counts[i] over to req */ req->start = start_ptr; memcpy(start_ptr, starts[i], memChunk); start_ptr += varp->ndims; /* count[] */ if (counts == NULL || counts[i] == NULL) { + /* counts == NULL, equivalent to all 1s */ for (j=0; jndims; j++) start_ptr[j] = 1; /* start_ptr is now counts[] */ } @@ -492,6 +506,24 @@ igetput_varn(NC *ncp, if (counts == NULL || counts[i] == NULL) num_rec = 1; else num_rec = counts[i][0]; + /* calculate number of flattened offset-length pairs */ + req->npairs = 1; + if (counts == NULL || counts[i] == NULL) { + /* equivalent to all multiple var1 APIs */ + ncmpio_calc_off(ncp, varp, starts[i], &req->offset_start); + // req->offset_end = req->offset_start + varp->xsz; + req->offset_end = varp->xsz; + } + else { + for (j=1; jndims-1; j++) + req->npairs *= counts[i][j]; + /* special treatment for when there is only one pair */ + if (req->npairs == 1) { + ncmpio_calc_off(ncp, varp, starts[i], &req->offset_start); + req->offset_end = req->nelems * varp->xsz; + } + } + max_rec = starts[i][0] + num_rec; lead_req->max_rec = MAX(lead_req->max_rec, max_rec); @@ -506,6 +538,11 @@ igetput_varn(NC *ncp, lead_list = (fIsSet(reqMode, NC_REQ_WR)) ? ncp->put_lead_list : ncp->get_lead_list; + + req->nelems /= counts[i][0]; + if (req->npairs == 1) + req->offset_end = req->nelems * varp->xsz; + /* append (counts[i][0]-1) number of requests to the queue */ ncmpio_add_record_requests(lead_list, req, counts[i][0], NULL); start_ptr += (counts[i][0] - 1) * 2 * varp->ndims; @@ -514,8 +551,26 @@ igetput_varn(NC *ncp, else req++; } - else + else { + /* calculate number of flattened offset-length pairs */ + req->npairs = 1; + if (counts == NULL || counts[i] == NULL) { + /* equivalent to all multiple var1 APIs */ + ncmpio_calc_off(ncp, varp, starts[i], &req->offset_start); + // req->offset_end = req->offset_start + varp->xsz; + req->offset_end = varp->xsz; + } + else { + for (j=0; jndims-1; j++) + req->npairs *= counts[i][j]; + /* special treatment for when there is only one pair */ + if (req->npairs == 1) { + ncmpio_calc_off(ncp, varp, starts[i], &req->offset_start); + req->offset_end = req->nelems * varp->xsz; + } + } req++; + } } if (reqid != NULL) *reqid = lead_req->id; diff --git a/src/drivers/ncmpio/ncmpio_intra_node.c b/src/drivers/ncmpio/ncmpio_intra_node.c index 90d613eb47..afa1c08790 100644 --- a/src/drivers/ncmpio/ncmpio_intra_node.c +++ b/src/drivers/ncmpio/ncmpio_intra_node.c @@ -3,96 +3,89 @@ * See COPYRIGHT notice in top-level directory. * * This file contains the implementation of intra-node aggregation feature, - * which is designed for the I/O patterns that contain many noncontiguous - * requests interleaved among processes, and spreading across a wide range of - * file space. It is particularly useful when the number of MPI processes - * allocated to a compute node is large. + * which is designed to improve performance for I/O patterns that contain many + * noncontiguous requests interleaved among processes, with a wide aggregate + * access region on each process which results in an all-to-many personalized + * communication with each MPI I/O aggregator receiving data from all MPI + * processes. By reducing the number of processes per node participating the + * all-to-many communication (i.e. the 'all' part), this feature effectively + * reduces the communication contention, which often happens to jobs that run + * on a large the number of MPI processes per compute node. * - * This feature is enabled by setting the PnetCDF hint 'nc_num_aggrs_per_node' - * to a positive integral value indicating the desired number of processes per - * compute node to be selected as the intra-node I/O aggregators. Each process - * is assigned a unique aggregator. The non-aggregators send their requests to - * the assigned aggregators, and then the aggregators make MPI-IO requests to - * the file. + * Users can enable this feature by setting the PnetCDF I/O hint named + * 'nc_num_aggrs_per_node' to a positive integral value, indicating the desired + * number of processes per compute node to be selected as the intra-node I/O + * aggregators. Processes running on the same node are divided into groups. + * The process with the lowest rank ID is selected as the I/O aggregator of + * that group. Non-aggregators send their requests to their aggregators, and + * then the aggregators make I/O requests to the file, i.e. only aggregators + * make MPI-IO calls. * - * Such strategy can effectively reduce communication congestion due to many - * pending asynchronous messages produced in the collective write inside of - * MPI-IO. + * Because communication within a node can be achieved by memory copy operation + * and thus its cost is expected to be much lower than the inter-node + * communication, this feature can effectively reduce the communication + * congestion or exhaustion of message queues, due to many pending asynchronous + * messages produced in the two-phase I/O, the strategy used to implement + * MPI collective I/O. * - * The concept of intra-node request aggregation is based on the paper: + * The concept of intra-node request aggregation and its performance results + * are presented in the following paper. * Q. Kang, S. Lee, K. Hou, R. Ross, A. Agrawal, A. Choudhary, and W. Liao. * Improving MPI Collective I/O for High Volume Non-Contiguous Requests With * Intra-Node Aggregation. IEEE Transactions on Parallel and Distributed - * Systems (TPDS), 31(11):2682-2695, November 2020. + * Systems, 31(11):2682-2695, November 2020. */ #ifdef HAVE_CONFIG_H -# include +#include #endif #include #include #include /* strcmp() strdup() */ +#include /* INT_MAX */ #include #include + #include #include #include #include "ncmpio_NC.h" -#ifdef HAVE_MPI_LARGE_COUNT -#define SWAP(offsets, lengths, bufAddr, x, y) { \ - MPI_Count aint; \ - MPI_Count cint; \ - MPI_Count d0 = (x) - offsets; \ - MPI_Count d1 = (y) - offsets; \ - if (d0 != d1) { \ - cint = *(x) ; *(x) = *(y) ; *(y) = cint ; \ - cint = lengths[d0] ; lengths[d0] = lengths[d1] ; lengths[d1] = cint ; \ - aint = bufAddr[d0] ; bufAddr[d0] = bufAddr[d1] ; bufAddr[d1] = aint ; \ - } \ -} -#else +/* swap values of x and y */ +#define SWAP1(x, y, tmp) { tmp = x ; x = y; y = tmp ; } + +/* swap elements of arrays x, y, and corresponding lengths and bufAddr */ #define SWAP(offsets, lengths, bufAddr, x, y) { \ - int int4; \ - MPI_Aint aint; \ - MPI_Aint d0 = (x) - offsets; \ - MPI_Aint d1 = (y) - offsets; \ + MPI_Offset tmp; \ + MPI_Offset d0 = (x) - offsets; \ + MPI_Offset d1 = (y) - offsets; \ if (d0 != d1) { \ - aint = *(x) ; *(x) = *(y) ; *(y) = aint ; \ - int4 = lengths[d0] ; lengths[d0] = lengths[d1] ; lengths[d1] = int4 ; \ - aint = bufAddr[d0] ; bufAddr[d0] = bufAddr[d1] ; bufAddr[d1] = aint ; \ + SWAP1(*(x), *(y), tmp); \ + SWAP1(lengths[d0], lengths[d1], tmp); \ + if (bufAddr != NULL) \ + SWAP1(bufAddr[d0], bufAddr[d1], tmp); \ } \ } -#endif #define MEDIAN(a,b,c) ((*(a) < *(b)) ? \ ((*(b) < *(c)) ? (b) : ((*(a) < *(c)) ? (c) : (a))) : \ ((*(b) > *(c)) ? (b) : ((*(a) < *(c)) ? (a) : (c)))) + /*----< qsort_off_len_buf() >------------------------------------------------*/ -/* Sort three arrays of offsets, lengths, and buffer addresses based on the - * increasing order of offsets. This code is based on the qsort routine from - * Bentley & McIlroy's "Engineering a Sort Function". +/* Sort three arrays of offsets, lengths, and buffer addresses based on array + * offsets into an increasing order. This code is based on the qsort routine + * from Bentley & McIlroy's "Engineering a Sort Function". */ static void -qsort_off_len_buf(MPI_Aint num, -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count *offsets, - MPI_Count *lengths, -#else - MPI_Aint *offsets, - int *lengths, -#endif - MPI_Aint *bufAddr) +qsort_off_len_buf(MPI_Offset num, + MPI_Offset *offsets, + MPI_Offset *lengths, + MPI_Offset *bufAddr) { -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count *pa, *pb, *pc, *pd, *pl, *pm, *pn, cmp_result, swap_cnt; -#else - MPI_Aint *pa, *pb, *pc, *pd, *pl, *pm, *pn, cmp_result, swap_cnt; -#endif - MPI_Aint i, r; + MPI_Offset *pa, *pb, *pc, *pd, *pl, *pm, *pn, cmp_result, swap_cnt, i, r; while (1) { swap_cnt = 0; @@ -145,17 +138,20 @@ qsort_off_len_buf(MPI_Aint num, pn = offsets + num; r = MIN(pa - offsets, pb - pa); - for (i=0; i 1) qsort_off_len_buf(r, offsets, lengths, bufAddr); if ((r = pd - pc) > 1) { /* Iterate rather than recursively call self to save stack space */ lengths = lengths + (num - r); - bufAddr = bufAddr + (num - r); + if (bufAddr != NULL) + bufAddr = bufAddr + (num - r); offsets = pn - r; num = r; } @@ -164,276 +160,168 @@ qsort_off_len_buf(MPI_Aint num, } } -/*----< ncmpio_init_intra_node_aggr() >--------------------------------------*/ -/* When intra-node write aggregation is enabled, processes on the same node - * will be divided into groups. The number of groups is the number of - * aggregators on that node. The rank IDs of each group must be established. +/*----< heap_merge() >-------------------------------------------------------*/ +/* Heapify(a, i, heapsize); Algorithm from Cormen et al. pg. 143 modified for a + * heap with smallest element at root. The recursion has been removed so that + * there are no function calls. Function calls are too expensive. * - * 1. Find information about MPI processes and their affinity to compute node. - * 2. Determine whether self process is an intra-node aggregator. - * 3. For an aggregator, find the number of non-aggregators assigned to it and - * construct rank IDs of assigned non-aggregators. - * 4. For a non-aggregator, find the rank ID of its assigned aggregator. + * Requirement: all individual offsets lists must be already sorted !!! + * + * count[] contains the number of offset-blklens-bufAddr triplets per process + * offsets[] contains offsets of nprocs processes appending one after another + * blklens[] contains lengths of nprocs processes + * bufAddr[] contains buffer addresses of nprocs processes + * nelems is the total number of offset-blklens-bufAddr triplets */ -int -ncmpio_intra_node_aggr_init(NC *ncp) +static +void heap_merge(int nprocs, + const MPI_Offset *count, /* [nprocs] */ + MPI_Offset *offsets, /* IN/OUT: [nelems] */ + MPI_Offset *blklens, /* IN/OUT: [nelems] */ + MPI_Offset *bufAddr) /* IN/OUT: [nelems] */ { - char my_procname[MPI_MAX_PROCESSOR_NAME], **all_procnames=NULL; - int i, j, k, my_procname_len, num_nodes, root=0; - int *node_ids=NULL, *all_procname_lens=NULL, *nprocs_per_node; - int naggrs_my_node, num_nonaggrs; - int my_rank_index, *ranks_my_node, my_node_id, nprocs_my_node; - - /* initialize parameters of local-node aggregation */ - ncp->my_aggr = -1; /* rank ID of my aggregator */ - ncp->num_nonaggrs = 0; /* number of non-aggregators assigned */ - ncp->nonaggr_ranks = NULL; /* ranks of assigned non-aggregators */ - -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) - ncp->aggr_time = 0.0; -#endif - - if (ncp->num_aggrs_per_node == 0 || ncp->num_aggrs_per_node == ncp->nprocs) - /* disable intra-node aggregation */ - return NC_NOERR; - -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) - double timing = MPI_Wtime(); -#endif - - /* allocate space for storing the rank IDs of non-aggregators assigned to - * this rank. Note ncp->nonaggr_ranks[] will be freed when closing the - * file, if allocated. - */ - num_nonaggrs = ncp->nprocs / ncp->num_aggrs_per_node + 1; - ncp->nonaggr_ranks = (int*) NCI_Malloc(sizeof(int) * num_nonaggrs); - - /* Collect info about compute nodes in order to select I/O aggregators. - * Note my_procname is null character terminated, but my_procname_len - * does not include the null character. - */ - MPI_Get_processor_name(my_procname, &my_procname_len); - my_procname_len++; /* to include terminate null character */ - - if (ncp->rank == root) { - /* root collects all procnames */ - all_procnames = (char **) NCI_Malloc(sizeof(char*) * ncp->nprocs); - if (all_procnames == NULL) - DEBUG_RETURN_ERROR(NC_ENOMEM) - - all_procname_lens = (int *) NCI_Malloc(sizeof(int) * ncp->nprocs); - if (all_procname_lens == NULL) { - NCI_Free(all_procnames); - DEBUG_RETURN_ERROR(NC_ENOMEM) + typedef struct { + MPI_Offset *off_list; + MPI_Offset *len_list; + MPI_Offset *addr_list; + MPI_Offset count; + } heap_struct; + + heap_struct *a, tmp; + int i, j, heapsize, l, r, k, smallest; + size_t sum; + MPI_Offset nelems, *srt_addr=NULL, *srt_off, *srt_len; + + for (nelems=0, i=0; icomm); - - if (ncp->rank == root) { - int *disp; - size_t alloc_size = 0; - - for (i=0; inprocs; i++) - alloc_size += all_procname_lens[i]; - - all_procnames[0] = (char *) NCI_Malloc(alloc_size); - if (all_procnames[0] == NULL) { - NCI_Free(all_procname_lens); - NCI_Free(all_procnames); - DEBUG_RETURN_ERROR(NC_ENOMEM) - } + nprocs = j; /* some count[i] may be zero */ - /* Construct displacement array for the MPI_Gatherv, as each process - * may have a different length for its process name. - */ - disp = (int *) NCI_Malloc(sizeof(int) * ncp->nprocs); - disp[0] = 0; - for (i=1; inprocs; i++) { - all_procnames[i] = all_procnames[i - 1] + all_procname_lens[i - 1]; - disp[i] = disp[i - 1] + all_procname_lens[i - 1]; - } +#define SWAP_HEAP(x, y, tmp) { tmp = x ; x = y ; y = tmp ; } - /* gather all process names */ - MPI_Gatherv(my_procname, my_procname_len, MPI_CHAR, - all_procnames[0], all_procname_lens, disp, MPI_CHAR, - root, ncp->comm); + heapsize = nprocs; - NCI_Free(disp); - NCI_Free(all_procname_lens); - } else - /* send process name to root */ - MPI_Gatherv(my_procname, my_procname_len, MPI_CHAR, - NULL, NULL, NULL, MPI_CHAR, root, ncp->comm); + /* Build a heap out of the first element from each list, with the smallest + * element of the heap at the root. The first for loop is to find and move + * the smallest a[*].off_list[0] to a[0]. + */ + for (i = heapsize / 2 - 1; i >= 0; i--) { + k = i; + for (;;) { + r = 2 * (k + 1); + l = r - 1; + if (l < heapsize && a[l].off_list[0] < a[k].off_list[0]) + smallest = l; + else + smallest = k; - /* each MPI process's compute node ID */ - node_ids = (int *) NCI_Malloc(sizeof(int) * ncp->nprocs); + if (r < heapsize && a[r].off_list[0] < a[smallest].off_list[0]) + smallest = r; - if (ncp->rank == root) { - /* all_procnames[] can tell us the number of nodes and number of - * processes per node. - */ - char **node_names; - int last; - - /* array of pointers pointing to unique host names (compute nodes) */ - node_names = (char **) NCI_Malloc(sizeof(char*) * ncp->nprocs); - - /* number of MPI processes running on each node */ - nprocs_per_node = (int *) NCI_Malloc(sizeof(int) * ncp->nprocs); - - /* calculate nprocs_per_node[] and node_ids[] */ - last = 0; - num_nodes = 0; /* number of unique compute nodes */ - for (i=0; inprocs; i++) { - k = last; - for (j=0; jnprocs, MPI_INT, root, ncp->comm); - - /* my_node_id is this rank's node ID */ - my_node_id = node_ids[ncp->rank]; - - /* nprocs_my_node: the number of processes in my nodes - * ranks_my_node[]: rank IDs of all processes in my node. - * my_rank_index points to ranks_my_node[] where - * ranks_my_node[my_rank_index] == ncp->rank + /* The heap keeps the smallest element in its first element, i.e. + * a[0].off_list[0]. */ - ranks_my_node = (int*) NCI_Malloc(sizeof(int) * ncp->nprocs); - my_rank_index = -1; - nprocs_my_node = 0; - for (i=0; inprocs; i++) { - if (node_ids[i] == my_node_id) { - if (i == ncp->rank) - my_rank_index = nprocs_my_node; - ranks_my_node[nprocs_my_node] = i; - nprocs_my_node++; + j = 0; + for (i = 0; i < nelems; i++) { + /* extract smallest element from heap, i.e. the root */ + srt_off[i] = a[0].off_list[0]; + srt_len[i] = a[0].len_list[0]; + if (bufAddr != NULL) + srt_addr[i] = a[0].addr_list[0]; + a[0].count--; + + if (!a[0].count) { + a[0] = a[heapsize - 1]; + heapsize--; + } else { + a[0].off_list++; + a[0].len_list++; + if (bufAddr != NULL) + a[0].addr_list++; } - } - assert(my_rank_index >= 0); - - /* Now, ranks_my_node[my_rank_index] == ncp->rank */ - NCI_Free(node_ids); - - /* make sure number of aggregators in my node <= nprocs_my_node */ - naggrs_my_node = MIN(ncp->num_aggrs_per_node, nprocs_my_node); + /* Heapify(a, 0, heapsize); */ + k = 0; + for (;;) { + r = 2 * (k + 1); + l = r - 1; + if (l < heapsize && a[l].off_list[0] < a[k].off_list[0]) + smallest = l; + else + smallest = k; - /* calculate the number of non-aggregators assigned to an aggregator. - * Note num_nonaggrs includes self. - */ - num_nonaggrs = nprocs_my_node / naggrs_my_node; - if (nprocs_my_node % naggrs_my_node) num_nonaggrs++; - - if (num_nonaggrs == 1) - /* disable aggregation if the number of non-aggregators assigned to - * this aggregator is 1. Note num_nonaggrs includes self. It is - * possible for aggregation enabled or disabled on different nodes and - * even different aggregation groups on the same node. - * - * Use whether ncp->my_aggr < 0 to tell if aggregation is disabled or - * enabled. - */ - ncp->my_aggr = -1; - else { - /* find the rank ID of aggregator assigned to this rank */ - ncp->my_aggr = ranks_my_node[my_rank_index - my_rank_index % num_nonaggrs]; + if (r < heapsize && a[r].off_list[0] < a[smallest].off_list[0]) + smallest = r; - if (ncp->my_aggr == ncp->rank) { /* this rank is an aggregator */ - /* Set the number of non-aggregators assigned to this rank. For the - * last group, make sure it does not go beyond nprocs_my_node. - */ - ncp->num_nonaggrs = MIN(num_nonaggrs, nprocs_my_node - my_rank_index); - if (ncp->num_nonaggrs == 1) - /* disable aggregation, as this aggregation group contains only - * self rank - */ - ncp->my_aggr = -1; - else - /* copy the rank IDs over to ncp->nonaggr_ranks[] */ - memcpy(ncp->nonaggr_ranks, - ranks_my_node + my_rank_index, - sizeof(int) * num_nonaggrs); + if (smallest != k) { + SWAP_HEAP(a[k], a[smallest], tmp); + k = smallest; + } else + break; } } - NCI_Free(ranks_my_node); - - if (ncp->my_aggr < 0) { - /* free ncp->nonaggr_ranks if aggregation is not enabled */ - NCI_Free(ncp->nonaggr_ranks); - ncp->nonaggr_ranks = NULL; - } - - /* TODO: For automatically determine Whether to enable intra-node write - * aggregation, this should be done right before each collective write - * call. - * 1. obtain hint cb_noddes, and striping_unit - * 2. calculate aggregate access region - * In each round of two-phase I/O, when the number of senders to each - * cb_nodes is very large, then intra-node aggregation should be enabled. - * Average of all nprocs_per_node may be a factor for determining whether - * to enable intra-node aggregation. It indicates whether the high number - * of processes are allocated on the same node. - */ -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) - ncp->aggr_time = MPI_Wtime() - timing; -#endif + memcpy(offsets, srt_off, sizeof(MPI_Offset) * nelems); + memcpy(blklens, srt_len, sizeof(MPI_Offset) * nelems); + if (bufAddr != NULL) + memcpy(bufAddr, srt_addr, sizeof(MPI_Offset) * nelems); - return NC_NOERR; + NCI_Free(a); + if (bufAddr != NULL) NCI_Free(srt_addr); + NCI_Free(srt_len); + NCI_Free(srt_off); } /*----< flatten_subarray() >-------------------------------------------------*/ -/* flatten a subarray request into a list of offset-length pairs */ +/* Flatten a subarray request, specified by start[], count[], and stride[] into + * a list of file offset-length pairs, offsets[] and lengths[]. + */ static int -flatten_subarray(int ndim, /* number of dimensions */ - int el_size, /* array element size */ - MPI_Offset var_begin, /* starting file offset */ - const MPI_Offset *dimlen, /* [ndim] dimension lengths */ - const MPI_Offset *start, /* [ndim] starts of subarray */ - const MPI_Offset *count, /* [ndim] counts of subarray */ - const MPI_Offset *stride, /* [ndim] strides of subarray */ - MPI_Aint *npairs, /* OUT: num of off-len pairs */ -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count *offsets, /* OUT: array of offsets */ - MPI_Count *lengths /* OUT: array of lengths */ -#else - MPI_Aint *offsets, /* OUT: array of offsets */ - int *lengths /* OUT: array of lengths */ -#endif - ) +flatten_subarray(int ndim, /* number of dimensions */ + int el_size, /* array element size */ + MPI_Offset var_begin, /* starting file offset */ + const MPI_Offset *dimlen, /* [ndim] dimension lengths */ + const MPI_Offset *start, /* [ndim] starts of subarray */ + const MPI_Offset *count, /* [ndim] counts of subarray */ + const MPI_Offset *stride, /* [ndim] strides of subarray */ + MPI_Offset *npairs, /* OUT: num of off-len pairs */ + MPI_Offset *offsets, /* OUT: array of offsets */ + MPI_Offset *lengths) /* OUT: array of lengths */ { int i, j; - MPI_Offset length, nstride, array_len, off, subarray_len; size_t idx=0, idx0; + MPI_Offset length, nstride, array_len, off, subarray_len; *npairs = 0; if (ndim < 0) return NC_NOERR; @@ -503,74 +391,68 @@ flatten_subarray(int ndim, /* number of dimensions */ subarray_len *= count[ndim]; } + /* offsets[] and lengths[] may be coalesceable, but this will be delayed + * until this subroutine callers, flatten_req() and flatten_nreqs(), where + * both fix-sized and record variables have been flattened. + */ + return NC_NOERR; } -/*----< flatten_req() >-----------------------------------------------------*/ -/* flatten one write request into offset-length pairs. - * offsets and lengths are allocated here and need to be freed by the caller +/*----< flatten_req() >------------------------------------------------------*/ +/* Flatten one subarray request into offset-length pairs. Arrays offsets and + * lengths are allocated in this subroutine and need to be freed by the caller. */ static int -flatten_req(NC *ncp, - NC_var *varp, - const MPI_Offset *start, - const MPI_Offset *count, - const MPI_Offset *stride, - MPI_Aint *num_pairs, /* OUT: number of off-len pairs */ -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count **offsets, /* OUT: array of flattened offsets */ - MPI_Count **lengths /* OUT: array of flattened lengths */ -#else - MPI_Aint **offsets, /* OUT: array of flattened offsets */ - int **lengths /* OUT: array of flattened lengths */ -#endif - ) +flatten_req(const NC *ncp, + const NC_var *varp, + const MPI_Offset *start, + const MPI_Offset *count, + const MPI_Offset *stride, + int *is_incr, /* OUT: are offsets incrementing */ + PNCIO_View *file_view) /* OUT: flattened file view */ { - int j, err=NC_NOERR, ndims; - MPI_Aint num, idx; - MPI_Offset var_begin, *shape, count0, *ones=NULL; + int i, j, err=NC_NOERR, ndims; + MPI_Offset num, idx, var_begin, prev_end_off, *shape, count0, *ones=NULL; - *num_pairs = 0; /* total number of offset-length pairs */ + file_view->count = 0; /* number of offset-length pairs */ + file_view->off = NULL; /* array of offsets */ + file_view->len = NULL; /* array of lengths */ - /* Count the number off-len pairs, so we can malloc a contiguous memory - * space for storing off-len pairs - */ - if (varp->ndims == 0) { /* scalar variable */ -#ifdef HAVE_MPI_LARGE_COUNT - *offsets = (MPI_Count*)NCI_Malloc(sizeof(MPI_Count)); - *lengths = (MPI_Count*)NCI_Malloc(sizeof(MPI_Count)); -#else - *offsets = (MPI_Aint*)NCI_Malloc(sizeof(MPI_Aint)); - *lengths = (int*) NCI_Malloc(sizeof(int)); -#endif - (*offsets)[0] = varp->begin; - (*lengths)[0] = varp->xsz; - *num_pairs = 1; + /* Count the number offset-length pairs */ + + if (varp->ndims == 0) { /* scalar variable has 1 pair offset-length */ + file_view->count = 1; + file_view->off = (MPI_Offset*)NCI_Malloc(sizeof(MPI_Offset)); + file_view->len = (MPI_Offset*)NCI_Malloc(sizeof(MPI_Offset)); + file_view->off[0] = varp->begin; + file_view->len[0] = varp->xsz; return NC_NOERR; } - else if (varp->ndims == 1 && IS_RECVAR(varp)) { /* scalar variable */ - num = count[0]; + + if (varp->ndims == 1 && IS_RECVAR(varp)) { + /* scalar record variable has 1 pair offset-length */ + file_view->count = count[0]; } else { - num = 1; + file_view->count = 1; if (stride != NULL && stride[varp->ndims-1] > 1) - num = count[varp->ndims-1]; /* count of last dimension */ - for (j=0; jndims-1; j++) - num *= count[j]; /* all count[] except the last dimension */ + /* count of last dimension */ + file_view->count = count[varp->ndims-1]; + + for (i=0; indims-1; i++) + /* all count[] except the last dimension */ + file_view->count *= count[i]; } - *num_pairs = num; -#ifdef HAVE_MPI_LARGE_COUNT - *offsets = (MPI_Count*)NCI_Malloc(sizeof(MPI_Count) * num); - *lengths = (MPI_Count*)NCI_Malloc(sizeof(MPI_Count) * num); -#else - *offsets = (MPI_Aint*)NCI_Malloc(sizeof(MPI_Aint) * num); - *lengths = (int*) NCI_Malloc(sizeof(int) * num); -#endif + file_view->off = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset) * + file_view->count); + file_view->len = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset) * + file_view->count); if (stride == NULL) { /* equivalent to {1, 1, ..., 1} */ ones = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset) * varp->ndims); - for (j=0; jndims; j++) ones[j] = 1; + for (i=0; indims; i++) ones[i] = 1; } ndims = varp->ndims; @@ -589,104 +471,138 @@ flatten_req(NC *ncp, count0 = 1; idx = 0; - for (j=0; jxsz, var_begin, shape, start, count, (stride == NULL) ? ones : stride, - &num, /* OUT: num of off-len pairs */ - *offsets + idx, /* OUT: array of offsets */ - *lengths + idx); /* OUT: array of lengths */ + &num, /* OUT: num of off-len pairs */ + file_view->off+idx, /* OUT: array of offsets */ + file_view->len+idx); /* OUT: array of lengths */ + if (err != NC_NOERR) goto err_out; + if (num == 0) continue; + + /* check if file_view->off[] are in an increasing order */ + for (j=0; j file_view->off[idx+j]) + *is_incr = 0; /* file_view->off are not incrementing */ + else + prev_end_off = file_view->off[idx+j]; + } + idx += num; - assert(idx <= *num_pairs); + assert(idx <= file_view->count); if (IS_RECVAR(varp)) var_begin += ncp->recsize; } - if (ones != NULL) - NCI_Free(ones); + + file_view->count = idx; + + /* check if the offsets-lengths can be coalesced */ + for (i=0, j=1; jcount; j++) { + if (file_view->off[i] + file_view->len[i] == file_view->off[j]) + file_view->len[i] += file_view->len[j]; + else { + i++; + if (i < j) { + file_view->off[i] = file_view->off[j]; + file_view->len[i] = file_view->len[j]; + } + } + } + file_view->count = i + 1; + +err_out: + if (ones != NULL) NCI_Free(ones); + + if (err != NC_NOERR) { + NCI_Free(file_view->off); + NCI_Free(file_view->len); + file_view->off = NULL; + file_view->len = NULL; + file_view->count = 0; + } return err; } -/*----< flatten_reqs() >-----------------------------------------------------*/ -/* flatten all write requests into offset-length pairs. - * offsets and lengths are allocated here and need to be freed by the caller +/*----< flatten_nreqs() >----------------------------------------------------*/ +/* Flatten multiple subarray requests into file offset-length pairs. Arrays + * offsets and lengths are allocated here and need to be freed by the caller. */ static int -flatten_reqs(NC *ncp, - int num_reqs, /* IN: # requests */ - const NC_req *reqs, /* [num_reqs] requests */ - MPI_Aint *num_pairs, /* OUT: total number of off-len pairs */ -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count **offsets, /* OUT: array of flattened offsets */ - MPI_Count **lengths /* OUT: array of flattened lengths */ -#else - MPI_Aint **offsets, /* OUT: array of flattened offsets */ - int **lengths /* OUT: array of flattened lengths */ -#endif - ) +flatten_nreqs(const NC *ncp, + int reqMode, /* IN: NC_REQ_RD or NC_REQ_WR */ + int num_reqs, /* IN: # requests */ + const NC_req *reqs, /* [num_reqs] requests */ + int *is_incr, /* OUT: are offsets incrementing */ + PNCIO_View *file_view) /* OUT: flattened file view */ { - int i, j, status=NC_NOERR, ndims, max_ndims=0; - MPI_Aint num, idx; - MPI_Offset *start, *count, *shape, *stride, *ones; + int i, j, err=NC_NOERR, ndims, max_ndims=0; + MPI_Offset num, idx, prev_end_off; + MPI_Offset *start, *count, *shape, *stride, *ones=NULL; - *num_pairs = 0; /* total number of offset-length pairs */ + file_view->count = 0; /* total number of offset-length pairs */ + file_view->off = NULL; + file_view->len = NULL; + + if (num_reqs == 0) return NC_NOERR; /* Count the number off-len pairs from reqs[], so we can malloc a * contiguous memory space for storing off-len pairs */ for (i=0; iput_lead_list + reqs[i].lead_off; - ndims = lead->varp->ndims; - max_ndims = MAX(max_ndims, ndims); - if (ndims > 0) { - start = reqs[i].start; - count = start + ndims; - stride = count + ndims; - } + /* reqs[i].npairs is the number of offset-length pairs of this request, + * calculated in ncmpio_igetput_varm() and igetput_varn() + */ + file_view->count += reqs[i].npairs; + if (fIsSet(reqMode, NC_REQ_WR)) + ndims = ncp->put_lead_list[reqs[i].lead_off].varp->ndims; else - start = count = stride = NULL; - - /* for record variable, each reqs[] is within a record */ - if (IS_RECVAR(lead->varp)) { - ndims--; - start++; - count++; - stride++; - } - if (fIsSet(lead->flag, NC_REQ_STRIDE_NULL)) stride = NULL; - - if (ndims < 0) continue; - if (ndims == 0) { /* 1D record variable */ - (*num_pairs)++; - continue; - } - num = 1; - if (stride != NULL && stride[ndims-1] > 1) - num = count[ndims-1]; /* count of last dimension */ - for (j=0; jget_lead_list[reqs[i].lead_off].varp->ndims; + max_ndims = MAX(max_ndims, ndims); } /* now we can allocate a contiguous memory space for the off-len pairs */ -#ifdef HAVE_MPI_LARGE_COUNT - *offsets = (MPI_Count*)NCI_Malloc(sizeof(MPI_Count) * (*num_pairs)); - *lengths = (MPI_Count*)NCI_Malloc(sizeof(MPI_Count) * (*num_pairs)); -#else - *offsets = (MPI_Aint*)NCI_Malloc(sizeof(MPI_Aint) * (*num_pairs)); - *lengths = (int*) NCI_Malloc(sizeof(int) * (*num_pairs)); -#endif - idx = 0; + file_view->off = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset) * + file_view->count); + file_view->len = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset) * + file_view->count); ones = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset) * max_ndims); for (i=0; iput_lead_list + reqs[i].lead_off; + NC_lead_req *lead; + if (fIsSet(reqMode, NC_REQ_WR)) + lead = ncp->put_lead_list + reqs[i].lead_off; + else + lead = ncp->get_lead_list + reqs[i].lead_off; + + if (reqs[i].npairs == 1) { + /* When reqs[i] contains only one offset-length pair, re-use + * reqs[i].offset_start, which has been generated earlier at a call + * to ncmpio_intra_node_aggregation_nreqs(). + */ + file_view->off[idx] = reqs[i].offset_start; + file_view->len[idx] = reqs[i].nelems * lead->varp->xsz; + + /* check if file_view->off[] are in an increasing order */ + if (prev_end_off > file_view->off[idx]) + *is_incr = 0; /* file_view->off are not incrementing */ + else + prev_end_off = file_view->off[idx]; + idx++; + continue; + } ndims = lead->varp->ndims; if (ndims > 0) { @@ -715,616 +631,1832 @@ flatten_reqs(NC *ncp, if (fIsSet(lead->flag, NC_REQ_STRIDE_NULL)) stride = NULL; - /* flatten each request into a list of offset-length pairs and - * append to the end of offsets and lengths + /* flatten each request into a list of offset-length pairs and append + * to the end of offsets and lengths */ - flatten_subarray(ndims, lead->varp->xsz, var_begin, shape, - start, count, (stride == NULL) ? ones : stride, - &num, /* OUT: number of off-len pairs */ - *offsets + idx, /* OUT: array of offsets */ - *lengths + idx); /* OUT: array of lengths */ + err = flatten_subarray(ndims, lead->varp->xsz, var_begin, shape, + start, count, (stride == NULL) ? ones : stride, + &num, /* OUT: number of off-len pairs */ + file_view->off+idx, /* OUT: array of offsets */ + file_view->len+idx); /* OUT: array of lengths */ + if (err != NC_NOERR) + goto err_out; + + /* check if file_view->off[] are in an increasing order */ + for (j=0; j file_view->off[idx+j]) + *is_incr = 0; /* offsets are not incrementing */ + else + prev_end_off = file_view->off[idx+j]; + } idx += num; } - NCI_Free(ones); + + file_view->count = idx; + + /* check if the offsets-lengths can be coalesced */ + for (i=0, j=1; jcount; j++) { + if (file_view->off[i] + file_view->len[i] == file_view->off[j]) + file_view->len[i] += file_view->len[j]; + else { + i++; + if (i < j) { + file_view->off[i] = file_view->off[j]; + file_view->len[i] = file_view->len[j]; + } + } + } + file_view->count = i + 1; for (i=0; iput_lead_list + reqs[i].lead_off; + NC_lead_req *lead; + if (fIsSet(reqMode, NC_REQ_WR)) + lead = ncp->put_lead_list + reqs[i].lead_off; + else + lead = ncp->get_lead_list + reqs[i].lead_off; if (fIsSet(lead->flag, NC_REQ_TO_FREE)) { NCI_Free(lead->start); lead->start = NULL; } } - return status; +err_out: + if (ones != NULL) NCI_Free(ones); + + if (err != NC_NOERR) { + NCI_Free(file_view->off); + NCI_Free(file_view->len); + file_view->off = NULL; + file_view->len = NULL; + file_view->count = 0; + } + + return err; } -/*----< construct_buf_type() >-----------------------------------------------*/ -/* construct an MPI derived datatype for I/O buffers from the request list, by - * concatenate all buffers. +/*----< flat_buf_type() >----------------------------------------------------*/ +/* Scan the nonblocking requests, pointed by reqs, and build the offset-length + * pairs of all buffers, xbuf. Note xbuf in each nonblocking request is a + * contiguous buffer (packed from the user buffer for the write operations). + * For record variables, if a user request is accessing more than one record, + * the request is split into into multiple NC_req objects, one for each record. */ static int -construct_buf_type(const NC *ncp, - int num_reqs, /* IN: # requests */ - const NC_req *reqs, /* [num_reqs] requests */ - MPI_Aint *bufLen, /* OUT: buffer size in bytes */ - MPI_Datatype *bufType) /* OUT: buffer datatype */ +flat_buf_type(const NC *ncp, + int reqMode, /* IN: NC_REQ_RD or NC_REQ_WR */ + int num_reqs, /* IN: number of requests */ + const NC_req *reqs, /* IN: [num_reqs] requests */ + PNCIO_View *buf_view, /* OUT: flattened buffer view */ + void **buf) /* OUT: pointer to I/O buffer */ { - int i, err, mpireturn, status=NC_NOERR; + int i, j, err=NC_NOERR; NC_lead_req *lead; + MPI_Aint addr, addr0; -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count *blocklens = (MPI_Count*)NCI_Malloc(sizeof(MPI_Count) * num_reqs); - MPI_Count *disps = (MPI_Count*)NCI_Malloc(sizeof(MPI_Count) * num_reqs); -#else - int *blocklens = (int*) NCI_Malloc(sizeof(int) * num_reqs); - MPI_Aint *disps = (MPI_Aint*) NCI_Malloc(sizeof(MPI_Aint) * num_reqs); -#endif + buf_view->size = 0; + buf_view->count = 0; + buf_view->off = NULL; + buf_view->len = NULL; - *bufLen = 0; - for (i=0; ioff = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset) * num_reqs); + if (buf_view->off == NULL) return NC_ENOMEM; + buf_view->len = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset) * num_reqs); + if (buf_view->len == NULL) return NC_ENOMEM; - /* blocklens[] in bytes */ - lead = ncp->put_lead_list + reqs[i].lead_off; - blocklens[i] = reqs[i].nelems * lead->varp->xsz; + *buf = reqs[0].xbuf; - *bufLen += blocklens[i]; - } + lead = (fIsSet(reqMode, NC_REQ_WR)) ? ncp->put_lead_list + : ncp->get_lead_list; - /* construct buffer derived datatype */ -#ifdef HAVE_MPI_LARGE_COUNT - mpireturn = MPI_Type_create_hindexed_c(num_reqs, blocklens, disps, - MPI_BYTE, bufType); -#else - mpireturn = MPI_Type_create_hindexed(num_reqs, blocklens, disps, - MPI_BYTE, bufType); -#endif - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, "MPI_Type_create_hindexed"); - /* return the first encountered error if there is any */ - if (status == NC_NOERR) status = err; + MPI_Get_address(lead[reqs[0].lead_off].xbuf, &addr0); + + /* set buf_view->off[0] and buf_view->len[0] */ + MPI_Get_address(reqs[0].xbuf, &addr0); /* displacement uses MPI_BOTTOM */ + buf_view->off[0] = 0; + + /* buf_view->len[] are in bytes */ + buf_view->len[0] = reqs[0].nelems * lead[reqs[0].lead_off].varp->xsz; + + buf_view->size = buf_view->len[0]; + for (i=0, j=1; joff[j] = addr - addr0; + + /* buf_view->len[] are in bytes */ + buf_view->len[j] = reqs[j].nelems * lead[reqs[j].lead_off].varp->xsz; - *bufType = MPI_DATATYPE_NULL; + /* accumulate buffer type size */ + buf_view->size += buf_view->len[j]; + + /* coalesce the off-len pairs */ + if (buf_view->off[i] + buf_view->len[i] == buf_view->off[j]) + buf_view->len[i] += buf_view->len[j]; + else { + i++; + if (i < j) { + buf_view->off[i] = buf_view->off[j]; + buf_view->len[i] = buf_view->len[j]; + } + } } - else { - MPI_Type_commit(bufType); -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count typeSize; - MPI_Type_size_c(*bufType, &typeSize); -#else - int typeSize; - MPI_Type_size(*bufType, &typeSize); -#endif - assert(typeSize == *bufLen); + /* After coalescing, the true number of requests may be reduced, but should + * still be > 0. + */ + + if (i + 1 < num_reqs) { + num_reqs = i + 1; /* num_reqs is reduced */ + buf_view->off = (MPI_Offset*)NCI_Realloc(buf_view->off, + sizeof(MPI_Offset) * num_reqs); + if (buf_view->off == NULL) return NC_ENOMEM; + buf_view->len = (MPI_Offset*)NCI_Realloc(buf_view->len, + sizeof(MPI_Offset) * num_reqs); + if (buf_view->len == NULL) return NC_ENOMEM; } + assert(num_reqs > 0); - NCI_Free(blocklens); - NCI_Free(disps); + buf_view->count = num_reqs; - return status; + return err; } -/*----< intra_node_aggregation() >-------------------------------------------*/ -/* This is a collective call */ -static int -intra_node_aggregation(NC *ncp, - MPI_Aint num_pairs, -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count *offsets, - MPI_Count *lengths, -#else - MPI_Aint *offsets, - int *lengths, -#endif - MPI_Offset bufCount, - MPI_Datatype bufType, - void *buf) +/*----< ina_collect_md() >---------------------------------------------------*/ +/* Within each intra-node aggregation group, the aggregator collects request + * metadata from the non-INA aggregators into meta[], including: + * 1. the number of offset-length pairs of each INA group member + * 2. offsets array of each INA group member + * 3. lengths array of each INA group member + * 4. INA aggregator's file_view will be updated: + * INA aggregator's file_view->count will be the total number of + * offset-length pairs of this INA group. + * INA aggregator's file_view->off will contain all the vile_view.off of + * this INA group (appending). + * INA aggregator's file_view->len will contain all the vile_view.len of + * this INA group (appending). + */ +static +int ina_collect_md(MPI_Comm comm, /* INA intra-node MPI communicator */ + MPI_Offset *meta, + PNCIO_View *file_view) { - int i, j, err, mpireturn, status=NC_NOERR, nreqs; - char *recv_buf=NULL, *wr_buf = NULL; - MPI_Aint npairs=0, *msg; - MPI_Offset offset=0, buf_count; - MPI_Datatype recvTypes, fileType=MPI_BYTE; - MPI_File fh; + int i, err, rank, nprocs, mpireturn, status=NC_NOERR, nreqs; MPI_Request *req=NULL; -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) - double timing = MPI_Wtime(); -#endif -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count bufLen; - MPI_Type_size_c(bufType, &bufLen); -#else - int bufLen; - MPI_Type_size(bufType, &bufLen); -#endif - bufLen *= bufCount; + MPI_Offset num_pairs=meta[0]; - /* First, tell aggregator how much to receive by sending: - * (num_pairs and bufLen). The message size to be sent by this rank - * is num_pairs * 2 * sizeof(MPI_Offset) + bufLen - */ - if (ncp->rank == ncp->my_aggr) - msg = (MPI_Aint*) NCI_Malloc(sizeof(MPI_Aint) * ncp->num_nonaggrs * 2); - else - msg = (MPI_Aint*) NCI_Malloc(sizeof(MPI_Aint) * 2); +#if PNETCDF_DEBUG_MODE == 1 + assert(comm != MPI_COMM_NULL); +#endif - msg[0] = num_pairs; - msg[1] = bufLen; + MPI_Comm_size(comm, &nprocs); + MPI_Comm_rank(comm, &rank); - /* Aggregator collects each non-aggregator's num_pairs and bufLen */ - if (ncp->rank == ncp->my_aggr) { - req = (MPI_Request*)NCI_Malloc(sizeof(MPI_Request) * ncp->num_nonaggrs); + /* Aggregator collects metadata of all INA group members */ + if (rank == 0) { + req = (MPI_Request*)NCI_Malloc(sizeof(MPI_Request) * nprocs); nreqs = 0; - for (i=1; inum_nonaggrs; i++) - MPI_Irecv(msg + i*2, 2, MPI_AINT, ncp->nonaggr_ranks[i], 0, - ncp->comm, &req[nreqs++]); + for (i=1; i 0) { +#ifdef HAVE_MPI_STATUSES_IGNORE + TRACE_COMM(MPI_Waitall)(nreqs, req, MPI_STATUSES_IGNORE); +#else + MPI_Status *statuses = (MPI_Status *) + NCI_Malloc(nreqs * sizeof(MPI_Status)); + TRACE_COMM(MPI_Waitall)(nreqs, req, statuses); + NCI_Free(statuses); +#endif + if (mpireturn != MPI_SUCCESS) { + err = ncmpii_error_mpi2nc(mpireturn,"MPI_Waitall"); + /* return the first encountered error if there is any */ + if (status == NC_NOERR) status = err; + } } } - else { /* non-aggregator */ - MPI_Send(msg, 2, MPI_AINT, ncp->my_aggr, 0, ncp->comm); - if (num_pairs == 0) - NCI_Free(msg); - } + else /* non-aggregator */ + TRACE_COMM(MPI_Send)(meta, 3, MPI_OFFSET, 0, 0, comm); - /* Aggregator collects offset-length pairs from non-aggregators */ - if (ncp->rank == ncp->my_aggr) { - /* calculate the total number of offset-length pairs */ - npairs = num_pairs; - for (i=1; inum_nonaggrs; i++) npairs += msg[i*2]; + /* Secondly, aggregators collect offset-length pairs from all its INA group + * members. + */ + if (rank == 0) { + MPI_Aint aint; + MPI_Offset bklens[2], disps[2]; + MPI_Datatype recvType; -#ifdef HAVE_MPI_LARGE_COUNT - if (npairs > num_pairs) { - /* realloc to store all pairs in a contiguous buffer */ - offsets = (MPI_Count*) NCI_Realloc(offsets, sizeof(MPI_Count) * npairs); - lengths = (MPI_Count*) NCI_Realloc(lengths, sizeof(MPI_Count) * npairs); - } -#else - if (npairs > num_pairs) { - /* realloc to store all pairs in a contiguous buffer */ - offsets = (MPI_Aint*) NCI_Realloc(offsets, sizeof(MPI_Aint) * npairs); - lengths = (int*) NCI_Realloc(lengths, sizeof(int) * npairs); + /* calculate the total number of offset-length pairs to receive */ + for (i=1; icount += meta[i*3]; + + /* Realloc this INA aggregator's file_view->off and file_view->len to + * receive the non-INA aggregators' file_view so they can be in a + * contiguous buffer. + */ + if (file_view->count > num_pairs) { + file_view->off = (MPI_Offset*) NCI_Realloc(file_view->off, + sizeof(MPI_Offset) * file_view->count); + file_view->len = (MPI_Offset*) NCI_Realloc(file_view->len, + sizeof(MPI_Offset) * file_view->count); } -#endif + /* To minimize number of MPI recv calls per non-aggregator, below + * creates a derived datatype, recvType, to combine file_view->off and + * file_view->len into one MPI_Irecv call. + */ nreqs = 0; -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Aint aint; - MPI_Count bklens[2]; - MPI_Count disps[2]; - - MPI_Get_address(offsets, &aint); - disps[0] = MPI_Aint_add(aint, sizeof(MPI_Count) * msg[0]); - MPI_Get_address(lengths, &aint); - disps[1] = MPI_Aint_add(aint, sizeof(MPI_Count) * msg[0]); - for (i=1; inum_nonaggrs; i++) { - if (msg[i*2] == 0) continue; - bklens[0] = msg[i*2] * sizeof(MPI_Count); - bklens[1] = msg[i*2] * sizeof(MPI_Count); - mpireturn = MPI_Type_create_hindexed_c(2, bklens, disps, MPI_BYTE, - &recvTypes); - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn,"MPI_Type_create_hindexed_c"); - /* return the first encountered error if there is any */ - if (status == NC_NOERR) status = err; - } - else { - mpireturn = MPI_Type_commit(&recvTypes); - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn,"MPI_Type_commit"); - /* return the first encountered error if there is any */ - if (status == NC_NOERR) status = err; - } - } + MPI_Get_address(file_view->off, &aint); + disps[0] = MPI_Aint_add(aint, sizeof(MPI_Offset) * meta[0]); + MPI_Get_address(file_view->len, &aint); + disps[1] = MPI_Aint_add(aint, sizeof(MPI_Offset) * meta[0]); + for (i=1; inonaggr_ranks[i], - 0, ncp->comm, &req[nreqs]); - MPI_Type_free(&recvTypes); + TRACE_COMM(MPI_Irecv)(MPI_BOTTOM, 1, recvType, i, 0, comm, + &req[nreqs++]); + MPI_Type_free(&recvType); disps[0] = MPI_Aint_add(disps[0], bklens[0]); disps[1] = MPI_Aint_add(disps[1], bklens[1]); - nreqs++; } + + if (nreqs > 0) { +#ifdef HAVE_MPI_STATUSES_IGNORE + TRACE_COMM(MPI_Waitall)(nreqs, req, MPI_STATUSES_IGNORE); #else - int bklens[2]; - MPI_Aint aint, disps[2]; - - MPI_Get_address(offsets, &aint); - disps[0] = MPI_Aint_add(aint, sizeof(MPI_Aint) * msg[0]); - MPI_Get_address(lengths, &aint); - disps[1] = MPI_Aint_add(aint, sizeof(int) * msg[0]); - for (i=1; inum_nonaggrs; i++) { - if (msg[i*2] == 0) continue; - bklens[0] = msg[i*2] * sizeof(MPI_Aint); - bklens[1] = msg[i*2] * sizeof(int); - mpireturn = MPI_Type_create_hindexed(2, bklens, disps, MPI_BYTE, - &recvTypes); + MPI_Status *statuses = (MPI_Status *) + NCI_Malloc(nreqs * sizeof(MPI_Status)); + TRACE_COMM(MPI_Waitall)(nreqs, req, statuses); + NCI_Free(statuses); +#endif if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn,"MPI_Type_create_hindexed"); + err = ncmpii_error_mpi2nc(mpireturn,"MPI_Waitall"); /* return the first encountered error if there is any */ if (status == NC_NOERR) status = err; } - else { - mpireturn = MPI_Type_commit(&recvTypes); - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn,"MPI_Type_commit"); - /* return the first encountered error if there is any */ - if (status == NC_NOERR) status = err; - } - } - /* post to receive offset-length pairs from non-aggregators */ - MPI_Irecv(MPI_BOTTOM, 1, recvTypes, ncp->nonaggr_ranks[i], - 0, ncp->comm, &req[nreqs]); - MPI_Type_free(&recvTypes); - - disps[0] = MPI_Aint_add(disps[0], bklens[0]); - disps[1] = MPI_Aint_add(disps[1], bklens[1]); - nreqs++; - } -#endif - mpireturn = MPI_Waitall(nreqs, req, MPI_STATUSES_IGNORE); - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn,"MPI_Waitall"); - /* return the first encountered error if there is any */ - if (status == NC_NOERR) status = err; } + NCI_Free(req); } else if (num_pairs > 0) { /* non-aggregator */ - /* send offset-length pairs data to the aggregator */ -#ifdef HAVE_MPI_LARGE_COUNT + /* To minimize number of MPI send calls to the aggregator, below + * creates a derived datatype, sendType, to combine file_view->off and + * file_view->len into one MPI_Send call. + */ MPI_Aint aint; - MPI_Count bklens[2]; - MPI_Count disps[2]; + MPI_Offset bklens[2], disps[2]; + MPI_Datatype sendType; - bklens[0] = msg[0] * sizeof(MPI_Count); + bklens[0] = meta[0] * sizeof(MPI_Offset); bklens[1] = bklens[0]; - MPI_Get_address(offsets, &aint); + MPI_Get_address(file_view->off, &aint); disps[0] = aint; - MPI_Get_address(lengths, &aint); + MPI_Get_address(file_view->len, &aint); disps[1] = aint; - mpireturn = MPI_Type_create_hindexed_c(2, bklens, disps, MPI_BYTE, - &recvTypes); - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn,"MPI_Type_create_hindexed_c"); - /* return the first encountered error if there is any */ - if (status == NC_NOERR) status = err; - } - else { - mpireturn = MPI_Type_commit(&recvTypes); - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn,"MPI_Type_commit"); - /* return the first encountered error if there is any */ - if (status == NC_NOERR) status = err; - } - } - MPI_Send_c(MPI_BOTTOM, 1, recvTypes, ncp->my_aggr, 0, ncp->comm); - MPI_Type_free(&recvTypes); -#else - int bklens[2]; - MPI_Aint disps[2]; - - bklens[0] = msg[0] * sizeof(MPI_Aint); - bklens[1] = msg[0] * sizeof(int); - MPI_Get_address(offsets, &disps[0]); - MPI_Get_address(lengths, &disps[1]); - mpireturn = MPI_Type_create_hindexed(2, bklens, disps, MPI_BYTE, - &recvTypes); - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn,"MPI_Type_create_hindexed"); - /* return the first encountered error if there is any */ - if (status == NC_NOERR) status = err; + err = ncmpio_type_create_hindexed(2, disps, bklens, &sendType); + if (status == NC_NOERR) status = err; + + TRACE_COMM(MPI_Send)(MPI_BOTTOM, 1, sendType, 0, 0, comm); + MPI_Type_free(&sendType); + } + + return status; +} + +/*----< ina_put() >----------------------------------------------------------*/ +/* This subroutine implements the intra-node aggregation for write operations. + * It also handles the case when INA is disabled and independent writes when + * INA is enabled. Note heap space allocated in file_view and buf_view will be + * freed by end of this subroutine. + */ +static +int ina_put(NC *ncp, + int is_incr, /* if file_view.off[] are incremental */ + PNCIO_View file_view, + PNCIO_View buf_view, + void *buf) /* user write buffer */ +{ + char *recv_buf=NULL, *wr_buf = NULL, *mpi_name; + int i, j, err, mpireturn, status=NC_NOERR, rank, nprocs, coalesceable=0; + MPI_Offset *meta=NULL, wr_amnt=0, *bufAddr=NULL; + MPI_Offset saved_put_fview_count, *saved_put_fview_len; + MPI_Comm intra_comm; + PNCIO_View put_fview, put_bview; + +#if PNETCDF_PROFILING == 1 + double endT, startT = MPI_Wtime(); + MPI_Offset mem_max; + ncmpi_inq_malloc_size(&mem_max); + // ncmpi_inq_malloc_max_size(&mem_max); + pnc_ina_mem_put[0] = MAX(pnc_ina_mem_put[0], mem_max); +#endif + + /* When a process makes an independent I/O call, we must set INA intra-node + * communicator, intra_comm, to MPI_COMM_SELF, as if self rank is an INA + * aggregator and the INA group size is of 1. + */ + intra_comm = ncp->comm_attr.ina_intra_comm; + if (intra_comm == MPI_COMM_NULL || fIsSet(ncp->flags, NC_MODE_INDEP)) { + /* Either INA is disabled or this is an independent request */ + intra_comm = MPI_COMM_SELF; + nprocs = 1; + rank = 0; + } + else { + MPI_Comm_size(intra_comm, &nprocs); + MPI_Comm_rank(intra_comm, &rank); + } + + /* put_fview and put_bview will be used when calling ncmpio_file_write() */ + put_fview = file_view; + put_bview = buf_view; + + /* Each aggregator's first step is to collect metadata from all INA group + * members about their request's file offset-length pairs, write amount, + * and whether the offsets are in an incremental order. This step is + * necessary for the next step which is to collect all members' file + * offset-length pairs. + * + * Once ina_collect_md() returns, this INA aggregator's file_view.count + * increases, file_view.off and file_view.len have grown to include the + * ones from all the INA group members (appending one after another). + * + * For write operation, keeping the original offset-length pairs is not + * necessary, as they will later be sorted and coalesced before calling + * MPI-IO or GIO file write. + */ + if (rank == 0) + meta = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset) * nprocs * 3); + else + meta = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset) * 3); + + meta[0] = file_view.count; + meta[1] = buf_view.size; + meta[2] = is_incr; + + /* The INA aggregator collects request metadata from all its INA group + * members. Note only the INA aggregator's file_view will get updated + * after returned from ina_collect_md(). + */ + if (nprocs > 1) { + err = ina_collect_md(intra_comm, meta, &put_fview); + if (err != NC_NOERR) { + NCI_Free(meta); + return err; } - else { - mpireturn = MPI_Type_commit(&recvTypes); - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn,"MPI_Type_commit"); - /* return the first encountered error if there is any */ - if (status == NC_NOERR) status = err; + } + + /* For write operation, the non-aggregator now can start sending their + * write data to its aggregator. + */ + if (rank > 0) { /* non-aggregator */ + if (buf_view.count > 0) { + MPI_Datatype sendType=MPI_BYTE; + + /* Non-INA aggregator sends write data to its INA aggregator */ + /* If this rank has non-zero sized request, it constructs an MPI + * derived datatype and call MPI_send to send write data from its + * INA aggregator. + */ + if (buf_view.count > 1) { + err = ncmpio_type_create_hindexed(buf_view.count, buf_view.off, + buf_view.len, &sendType); + if (status == NC_NOERR) + status = err; + + /* When err != NC_NOERR, sendType is set to MPI_DATATYPE_NULL + * which will trigger an error when calling MPI_Send(). + */ + TRACE_COMM(MPI_Send)(buf, 1, sendType, 0, 0, intra_comm); + mpi_name = "MPI_Send"; } + else { /* buf_view.count == 1 */ + if (buf_view.size > INT_MAX) { +#ifdef HAVE_MPI_LARGE_COUNT + TRACE_COMM(MPI_Send_c)(buf, buf_view.size, MPI_BYTE, 0, 0, + intra_comm); + mpi_name = "MPI_Send_c"; +#else + if (status == NC_NOERR) status = NC_EINTOVERFLOW; +#endif + } + else { + TRACE_COMM(MPI_Send)(buf, (int)buf_view.size, MPI_BYTE, + 0, 0, intra_comm); + mpi_name = "MPI_Send"; + } + } + + if (mpireturn != MPI_SUCCESS && status == NC_NOERR) + status = ncmpii_error_mpi2nc(mpireturn, mpi_name); + + if (sendType != MPI_BYTE && sendType != MPI_DATATYPE_NULL) + MPI_Type_free(&sendType); + + /* free space allocated for buf_view */ + NCI_Free(buf_view.off); + NCI_Free(buf_view.len); + } + + /* free space allocated for file_view */ + if (file_view.count > 0) { + NCI_Free(file_view.off); + NCI_Free(file_view.len); } - MPI_Send(MPI_BOTTOM, 1, recvTypes, ncp->my_aggr, 0, ncp->comm); - MPI_Type_free(&recvTypes); + + /* Non-INA aggregators are done here, as only INA aggregators make a + * call to ncmpio_file_write() to write data to the file. Non-INA + * aggregators do not participate file I/O calls, except when + * performing independent writes, in which case intra_comm is + * temporarily set to MPI_COMM_SELF. + */ + NCI_Free(meta); + return status; + } + + /* The remaining of this subroutine is for aggregators only -------------*/ + +#if PNETCDF_PROFILING == 1 + ncmpi_inq_malloc_size(&mem_max); + // ncmpi_inq_malloc_max_size(&mem_max); + pnc_ina_mem_put[1] = MAX(pnc_ina_mem_put[1], mem_max); + + endT = MPI_Wtime(); + pnc_ina_put[0] += endT - startT; /* collect MD */ + startT = endT; #endif - NCI_Free(msg); + + /* Once an INA aggregator collected metadata from all its INA group + * members, it must construct new file view and buffer view whose + * offset-length pairs are coalesced, with overlaps removed, and abide by + * the MPI standard requirement on offsets being in a monotonically + * non-decreasing order. + * + * MPI-IO has the following requirements about filetype. + * 1. The (flattened) displacements (of a filetype) are not required to be + * distinct, but they cannot be negative, and they must be monotonically + * non-decreasing. + * 2. If the file is opened for writing, neither the etype nor the filetype + * is permitted to contain overlapping regions. + */ + + if (put_fview.count == 0) goto do_write; + + /* Now this aggregator has received all offset-length pairs from its + * non-aggregators. If this INA group makes a non-zero sized request, the + * first step is to check if a sorting of file offsets is necessary. + */ + char *ptr; + int nreqs, indv_sorted, do_sort, overlap; + MPI_Request *req=NULL; + MPI_Offset recv_amnt; + + /* Check whether or not all INA group members' file_view.off[] are + * individually sorted. + */ + indv_sorted = 1; + do_sort = 0; + for (i=-1,j=0; j 0) + /* i is the 1st member whose file_view.count > 0 */ + i = j; + if (meta[j*3+2] == 0) { + /* member j's file_view.off are not sorted */ + indv_sorted = 0; + do_sort = 1; + break; + } + } + /* i is the first INA group member whose file_view.count > 0, and + * j is the first INA group member whose is_incr is false + */ + + if (i >= 0 && indv_sorted == 1) { + /* Even when file_view.off[] of all INA group members are individually + * sorted, we still need to check if offsets are interleaved. If + * interleaved, we must sort all offset-length pairs. + */ + MPI_Offset prev_end_off, sum; + + assert(meta[i*3+2] == 1); + + sum = meta[i*3]; + + /* prev_end_off is the last offset of INA group member i */ + prev_end_off = put_fview.off[sum-1]; + + /* check if file_view.off among INA group members are interleaved */ + for (++i; i put_fview.off[sum]) { + do_sort = 1; /* indicate put_fview.off are not incrementing */ + break; + } + /* move on to the next member */ + sum += meta[i*3]; + prev_end_off = put_fview.off[sum-1]; + } } - /* - * TODO, define a datatype to combine sends of offset-length pairs with the - * write data into a single send call. + /* Construct an array of buffer addresses containing a mapping of the + * buffer used to receive write data from non-aggregators and the buffer + * used to write to file. bufAddr[] is calculated based on the assumption + * that the write buffer of this aggregator is contiguous, i.e. + * buf_view.count <= 1. For non-aggregators, their write data will always + * be received into a contiguous buffer. */ - nreqs = 0; - if (ncp->rank == ncp->my_aggr) { - /* calculate the total write account */ - buf_count = bufLen; - for (i=1; inum_nonaggrs; i++) buf_count += msg[i*2 + 1]; - - /* Allocate receive buffer, which will be sorted into an increasing - * order based on the file offsets. Thus, after sorting pack recv_buf - * to wr_buf to avoid creating another buffer datatype. + bufAddr = (MPI_Offset*)NCI_Malloc(sizeof(MPI_Offset) * put_fview.count); + bufAddr[0] = 0; + for (i=1; i 0) { - recv_buf = (char*) NCI_Malloc(buf_count); - wr_buf = (char*) NCI_Malloc(buf_count); + if (indv_sorted) { + /* Interleaved offsets are found but individual offsets are already + * sorted. This is commonly seen from the checkerboard domain + * partitioning pattern. In this case, heap_merge() is faster to + * merge all offsets into one single sorted offset list. Note + * count[] must be initialized, so it can be used in heap_merge() + */ + MPI_Offset *count; + count = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset) * nprocs); + for (i=0; i= + put_fview.off[j] + put_fview.len[j]) { + /* segment i completely covers segment j, skip j */ + overlap = 1; + continue; } - /* First, pack self write data into front of the recv_buf */ - if (bufLen > 0) { - if (bufType == MPI_BYTE) - memcpy(recv_buf, buf, bufLen); - else { - void *inbuf = (buf == NULL) ? MPI_BOTTOM : buf; -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count position=0; - MPI_Count incount = (buf == NULL) ? 1 : bufCount; - MPI_Pack_c(inbuf, incount, bufType, recv_buf, bufLen, &position, - MPI_COMM_SELF); -#else - int position=0; - int incount = (buf == NULL) ? 1 : bufCount; - MPI_Pack(inbuf, incount, bufType, recv_buf, bufLen, &position, - MPI_COMM_SELF); + MPI_Offset gap = put_fview.off[i] + put_fview.len[i] - put_fview.off[j]; + if (gap >= 0) { /* overlap detected, merge j into i */ + /* when gap > 0, pairs i and j overlap + * when gap == 0, pairs i and j are contiguous + */ + if (gap > 0) overlap = 1; + wr_amnt += put_fview.len[j] - gap; /* subtract overlapped amount */ + if (bufAddr[i] + put_fview.len[i] == bufAddr[j] + gap) { + /* buffers i and j are contiguous, merge j into i and + * subtract overlapped amount. + */ + put_fview.len[i] += put_fview.len[j] - gap; + } + else { /* buffers are not contiguous, reduce j's len */ + coalesceable = 1; + put_fview.off[i+1] = put_fview.off[j] + gap; + put_fview.len[i+1] = put_fview.len[j] - gap; + bufAddr[i+1] = bufAddr[j] + gap; + i++; + } + } + else { /* i and j do not overlap */ + wr_amnt += put_fview.len[j]; + i++; + if (i < j) { + put_fview.off[i] = put_fview.off[j]; + put_fview.len[i] = put_fview.len[j]; + bufAddr[i] = bufAddr[j]; + } + } + } + + /* Now put_fview.off[], put_fview.len[], bufAddr[] are coalesced and no + * overlap. Update put_fview.count. + */ + put_fview.count = i+1; + + /* If put_fview can be further coalesced, a new set of offsets and lengths + * must be allocated for put_fview. These new offsets and lengths cannot be + * used for buf_view, because the buffer addresses may not be coalesceable + * even the corresponding put_fview can. Thus the old offsets must be kept + * to construct buf_view. + * + * Note put_fview.len can be updated in place, because it will not be used + * by buf_view. + */ + + /* buf_view to be used in a call to ncmpio_file_write() later will use + * bufAddr[] and put_fview.len[], as it offset-length pairs. Because + * put_fview.count and put_fview.len may change below if coalesceable is + * true, we now save them for later use. + */ + saved_put_fview_len = put_fview.len; + saved_put_fview_count = put_fview.count; + + if (coalesceable) { /* put_fview can be further coalesced */ + size_t cpy_amnt; + MPI_Offset *file_len; + + cpy_amnt = sizeof(MPI_Offset) * put_fview.count; + file_len = (MPI_Offset*) NCI_Malloc(cpy_amnt); + memcpy(file_len, put_fview.len, cpy_amnt); + put_fview.len = file_len; + + for (i=0, j=1; j 0) { - char *ptr = recv_buf + bufLen; - for (i=1; inum_nonaggrs; i++) { - if (msg[i*2 + 1] == 0) continue; + /* update number of offset-length pairs */ + put_fview.count = i+1; + } + +#if PNETCDF_DEBUG_MODE == 1 + /* check if put_fview's offset-lengths have been coalesced */ + for (i=1; i 0); + assert(put_fview.off[i-1] < put_fview.off[i]); + assert(put_fview.off[i-1] + put_fview.len[i-1] < put_fview.off[i]); + } +#endif + +#if PNETCDF_PROFILING == 1 + ncmpi_inq_malloc_size(&mem_max); + // ncmpi_inq_malloc_max_size(&mem_max); + pnc_ina_mem_put[2] = MAX(pnc_ina_mem_put[2], mem_max); + pnc_ina_npairs_put = MAX(pnc_ina_npairs_put, put_fview.count); + + endT = MPI_Wtime(); + pnc_ina_put[1] += endT - startT; /* sorting */ + startT = endT; +#endif + + /* Allocate receive buffer. Once write data from non-aggregators have + * received into recv_buf, it is packed into wr_buf. Then, wr_buf is + * used to call MPI-IO/GIO file write. Note the wr_buf is always + * contiguous. + * + * When nprocs == 1, wr_buf is set to buf and together with buf_view are + * directly passed to ncmpio_file_write() call. + * + * If file offset-length pairs have not been re-ordered, i.e. sorted + * and overlaps removed, and this aggregator will not receive any write + * data from its non-aggregators, then we can use user's buffer, buf, + * to call MPI-IO/GIO to write to the file, without allocating an + * additional temporary buffer. + */ + if (!do_sort && buf_view.size == recv_amnt && !overlap) + recv_buf = buf; + else + recv_buf = (char*) NCI_Malloc(recv_amnt); + + if (recv_buf != buf) { + /* Copy this aggregator's write data into front of recv_buf */ + char *recv_ptr=recv_buf; + for (j=0; j INT_MAX) { #ifdef HAVE_MPI_LARGE_COUNT - MPI_Irecv_c(ptr, msg[i*2 + 1], MPI_BYTE, ncp->nonaggr_ranks[i], - 0, ncp->comm, &req[nreqs++]); + TRACE_COMM(MPI_Irecv_c)(ptr, meta[i*3 + 1], MPI_BYTE, i, 0, + intra_comm, &req[nreqs++]); + mpi_name = "MPI_Irecv_c"; #else - MPI_Irecv(ptr, msg[i*2 + 1], MPI_BYTE, ncp->nonaggr_ranks[i], - 0, ncp->comm, &req[nreqs++]); + if (status == NC_NOERR) status = NC_EINTOVERFLOW; #endif - ptr += msg[i*2 + 1]; - } - mpireturn = MPI_Waitall(nreqs, req, MPI_STATUSES_IGNORE); - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn,"MPI_Waitall"); - /* return the first encountered error if there is any */ - if (status == NC_NOERR) status = err; - } } - NCI_Free(req); - NCI_Free(msg); + else { + TRACE_COMM(MPI_Irecv)(ptr, (int)meta[i*3 + 1], MPI_BYTE, i, 0, + intra_comm, &req[nreqs++]); + mpi_name = "MPI_Irecv"; + } + + if (mpireturn != MPI_SUCCESS && status == NC_NOERR) + status = ncmpii_error_mpi2nc(mpireturn, mpi_name); + + ptr += meta[i*3 + 1]; } - else if (bufLen > 0) { - /* send write data to the aggregator */ - void *buf_ptr = (buf == NULL) ? MPI_BOTTOM : buf; -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count num = (buf == NULL) ? 1 : bufCount; - MPI_Send_c(buf_ptr, num, bufType, ncp->my_aggr, 0, ncp->comm); + +#if PNETCDF_PROFILING == 1 + ncmpi_inq_malloc_size(&mem_max); + // ncmpi_inq_malloc_max_size(&mem_max); + pnc_ina_mem_put[3] = MAX(pnc_ina_mem_put[3], mem_max); + + endT = MPI_Wtime(); + pnc_ina_put[2] += endT - startT; /* post irecv */ + startT = endT; +#endif + + if (nreqs > 0) { +#ifdef HAVE_MPI_STATUSES_IGNORE + TRACE_COMM(MPI_Waitall)(nreqs, req, MPI_STATUSES_IGNORE); #else - int num = (buf == NULL) ? 1 : bufCount; - MPI_Send(buf_ptr, num, bufType, ncp->my_aggr, 0, ncp->comm); + MPI_Status *statuses = (MPI_Status *) + NCI_Malloc(nreqs * sizeof(MPI_Status)); + TRACE_COMM(MPI_Waitall)(nreqs, req, statuses); + NCI_Free(statuses); #endif - NCI_Free(offsets); - NCI_Free(lengths); + if (mpireturn != MPI_SUCCESS) { + err = ncmpii_error_mpi2nc(mpireturn,"MPI_Waitall"); + /* return the first encountered error if there is any */ + if (status == NC_NOERR) status = err; + } } + NCI_Free(req); - /* aggregator sorts the offset-length pairs, along with the buffer */ - if (ncp->rank == ncp->my_aggr && npairs > 0) { +#if PNETCDF_PROFILING == 1 + endT = MPI_Wtime(); + pnc_ina_put[3] += endT - startT; /* wait */ + startT = endT; +#endif - /* construct array of buffer addresses */ - MPI_Aint *bufAddr = (MPI_Aint*)NCI_Malloc(sizeof(MPI_Aint) * npairs); - bufAddr[0] = 0; - for (i=1; i= offsets[j] + lengths[j]) - /* segment i completely covers segment j, skip j */ - continue; + wr_buf = NCI_Malloc(wr_amnt); + ptr = wr_buf; + + /* Copy write data into wr_buf, a contiguous buffer. */ + for (j=0; jflags, NC_MODE_INDEP)) ? NC_REQ_INDEP + : NC_REQ_COLL; + + MPI_Offset wlen; + + wlen = ncmpio_file_write(ncp, coll_indep, wr_buf, put_fview, put_bview); + if (wlen < 0) { + if (status == NC_NOERR) status = (int)wlen; + wr_amnt = 0; + } + + if (wr_buf != buf) NCI_Free(wr_buf); + + /* Free bufAddr if it is not used by put_bview.off */ + if (bufAddr != NULL && bufAddr != put_bview.off) NCI_Free(bufAddr); + + if (put_bview.len != NULL && put_bview.len != buf_view.len) + NCI_Free(put_bview.len); + if (put_bview.off != NULL && put_bview.off != buf_view.off) + NCI_Free(put_bview.off); + + /* free space allocated for file_view and buf_view */ + if (put_fview.count > 0) { + NCI_Free(put_fview.off); + /* put_fview.len and put_bview.len may share the same address */ + if (put_fview.len != put_bview.len) NCI_Free(put_fview.len); + } + if (buf_view.count > 0) { + NCI_Free(buf_view.off); + NCI_Free(buf_view.len); + } + +#if PNETCDF_PROFILING == 1 + ncmpi_inq_malloc_size(&mem_max); + // ncmpi_inq_malloc_max_size(&mem_max); + pnc_ina_mem_put[5] = MAX(pnc_ina_mem_put[5], mem_max); + + endT = MPI_Wtime(); + pnc_ina_put[5] += endT - startT; /* write */ + + if (rank > 0) { /* set timers to zeros for non-INA aggregators */ + for (i=0; i-------------------------------------------------------*/ +static +size_t bin_search(MPI_Offset key, + MPI_Offset nmemb, + const MPI_Offset *base) +{ + MPI_Offset low, high; + + /* only one element */ + if (nmemb == 1) + return (base[0] <= key) ? 0 : -1; + + /* check the 1st element */ + if (base[0] <= key && key < base[1]) + return 0; + + low = 1; + high = nmemb - 1; + + while (low <= high) { + size_t mid = low + (high - low) / 2; + if (base[mid] == key) + return mid; + if (base[mid] < key) + low = mid + 1; + else + high = mid - 1; + } + return (low - 1); +} + +/*----< ina_get() >----------------------------------------------------------*/ +/* This subroutine implements the intra-node aggregation for read operations. + * It also handles the case when INA is disabled and independent reads when INA + * is enabled. Note heap space allocated in file_view and buf_view will be + * freed by end of this subroutine. + */ +static +int ina_get(NC *ncp, + int is_incr, /* if file_view.off[] are incremental */ + PNCIO_View file_view, + PNCIO_View buf_view, + void *buf) /* user read buffer */ +{ + char *rd_buf = NULL; + int i, j, err, mpireturn, status=NC_NOERR, nprocs, rank, nreqs; + int do_sort=0, indv_sorted=1, overlap=0, coalesceable; + MPI_Offset *meta=NULL, *blks=NULL, *disps=NULL; + MPI_Offset max_npairs, send_amnt=0, rd_amnt=0, off_start; + MPI_Request *req=NULL; + MPI_Comm intra_comm; + PNCIO_View get_fview, get_bview, orig_fview; + +#if PNETCDF_PROFILING == 1 + double endT, startT = MPI_Wtime(); + MPI_Offset mem_max; + ncmpi_inq_malloc_size(&mem_max); + // ncmpi_inq_malloc_max_size(&mem_max); + pnc_ina_mem_get[0] = MAX(pnc_ina_mem_get[0], mem_max); +#endif + + /* When a process makes an independent I/O call, we must set INA intra-node + * communicator, intra_comm, to MPI_COMM_SELF, as if self rank is an INA + * aggregator and the INA group size is of 1. + */ + intra_comm = ncp->comm_attr.ina_intra_comm; + if (intra_comm == MPI_COMM_NULL || fIsSet(ncp->flags, NC_MODE_INDEP)) { + /* Either INA is disabled or this is an independent request */ + intra_comm = MPI_COMM_SELF; + nprocs = 1; + rank = 0; + } + else { + MPI_Comm_size(intra_comm, &nprocs); + MPI_Comm_rank(intra_comm, &rank); + } + + /* get_fview and get_bview will be used when calling ncmpio_file_read() */ + get_fview = file_view; + get_bview = buf_view; + + /* Each aggregator's first step is to collect metadata from all INA group + * members about their request's file offset-length pairs, write amount, + * and whether the offsets are in an incremental order. This step is + * necessary for the next step which is to collect all members' file + * offset-length pairs. + * + * Once ina_collect_md() returns, this INA aggregator's file_view.count + * increases, file_view.off and file_view.len have grown to include the + * ones from all the INA group members (appending one after another). + * + * For read operation, the original file offset-length pairs must be kept, + * as they are required to unpack file read data into messages for sending + * them to the INA group members. The buf_view's offset-length pairs to be + * passed to ncmpio_file read() will be modified to be sorted into an + * incremental order and coalesced. + */ + if (rank == 0) + meta = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset) * nprocs * 3); + else + meta = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset) * 3); + + meta[0] = file_view.count; + meta[1] = buf_view.size; + meta[2] = is_incr; + + /* This INA aggregator's file_view.count must be preserved, so it can be + * used to unpack read data into its user buffer. Below the call to + * ina_collect_md() will increase/expand this INA aggregator's + * file_view.count, file_view.off[] and file_view.len[]. + */ + orig_fview.count = file_view.count; + + /* The INA aggregator collects request metadata from all its INA group + * members. Note only the INA aggregator's file_view will get updated + * after returned from ina_collect_md(). + */ + if (nprocs > 1) { + err = ina_collect_md(intra_comm, meta, &get_fview); + if (err != NC_NOERR) { + NCI_Free(meta); + return err; + } + } - MPI_Offset gap = offsets[i] + lengths[i] - offsets[j]; - if (gap >= 0) { /* segments i and j overlaps */ - if (bufAddr[i] + lengths[i] == bufAddr[j] + gap) { - /* buffers i and j are contiguous, merge j to i */ - lengths[i] += lengths[j] - gap; + /* For read operation, the non-INA aggregator members now can start + * receiving their read data from its aggregator. + */ + if (rank > 0) { /* not an INA aggregator */ + if (buf_view.count > 0) { + char *mpi_name; + MPI_Status st; + MPI_Datatype recvType=MPI_BYTE; + + /* If this rank has non-zero sized request, it constructs an MPI + * derived datatype and call MPI_recv to receive read data from its + * INA aggregator. + */ + if (buf_view.count > 1) { + err = ncmpio_type_create_hindexed(buf_view.count, buf_view.off, + buf_view.len, &recvType); + if (status == NC_NOERR) + status = err; + + /* When err != NC_NOERR, recvType is set to MPI_DATATYPE_NULL + * which will trigger an error when calling MPI_Send(). + */ + TRACE_COMM(MPI_Recv)(buf, 1, recvType, 0, 0, intra_comm, &st); + mpi_name = "MPI_Recv"; + } + else { /* buf_view.count == 1 */ + if (buf_view.size > INT_MAX) { +#ifdef HAVE_MPI_LARGE_COUNT + TRACE_COMM(MPI_Recv_c)(buf, buf_view.size, MPI_BYTE, 0, 0, + intra_comm, &st); + mpi_name = "MPI_Recv_c"; +#else + if (status == NC_NOERR) status = NC_EINTOVERFLOW; +#endif } - else { /* buffers are not contiguous, reduce j's len */ - offsets[i+1] = offsets[j] + gap; - lengths[i+1] = lengths[j] - gap; - bufAddr[i+1] = bufAddr[j] + gap; - i++; + else { + TRACE_COMM(MPI_Recv)(buf, (int)buf_view.size, MPI_BYTE, + 0, 0, intra_comm, &st); + mpi_name = "MPI_Recv"; } } - else { /* i and j do not overlap */ - i++; - if (i < j) { - offsets[i] = offsets[j]; - lengths[i] = lengths[j]; - bufAddr[i] = bufAddr[j]; - } + + if (mpireturn != MPI_SUCCESS && status == NC_NOERR) + status = ncmpii_error_mpi2nc(mpireturn, mpi_name); + + if (recvType != MPI_BYTE && recvType != MPI_DATATYPE_NULL) + MPI_Type_free(&recvType); + + /* free space allocated for buf_view */ + NCI_Free(buf_view.off); + NCI_Free(buf_view.len); + } + + /* free space allocated for file_view */ + if (file_view.count > 0) { + NCI_Free(file_view.off); + NCI_Free(file_view.len); + } + + /* Non-INA aggregators are now done, as they do not participate MPI-IO + * or GIO file read (neither collective nor independent). + */ + NCI_Free(meta); + return status; + } + + /* The remaining of this subroutine is for INA aggregators only. */ + +#if PNETCDF_PROFILING == 1 + ncmpi_inq_malloc_size(&mem_max); + // ncmpi_inq_malloc_max_size(&mem_max); + pnc_ina_mem_get[1] = MAX(pnc_ina_mem_get[1], mem_max); + + endT = MPI_Wtime(); + pnc_ina_get[0] += endT - startT; /* collect MD */ + startT = endT; +#endif + + if (get_fview.count == 0) { + /* This INA aggregation group has zero data to read, but this + * aggregator must participate the collective I/O calls. + */ + goto do_read; + } + +#if PNETCDF_PROFILING == 1 + ncmpi_inq_malloc_size(&mem_max); + // ncmpi_inq_malloc_max_size(&mem_max); + pnc_ina_mem_get[2] = MAX(pnc_ina_mem_get[2], mem_max); +#endif + + /* Once an INA aggregator collected metadata from all its INA group + * members, it must construct new file view and buffer view whose + * offset-length pairs are coalesced, with overlaps removed, and abide by + * the MPI standard requirement on offsets being in a monotonically + * non-decreasing order. + * + * MPI-IO has the following requirements about filetype. + * 1. The (flattened) displacements (of a filetype) are not required to be + * distinct, but they cannot be negative, and they must be monotonically + * non-decreasing. + * 2. If the file is opened for writing, neither the etype nor the filetype + * is permitted to contain overlapping regions. + */ + + /* Now this INA aggregator has received all offset-length pairs from + * its non-aggregators. At first, it checks if a sorting is necessary. + * + * First check whether or not all INA group members' file_view.off[] are + * individually sorted. + */ + indv_sorted = 1; + for (i=-1,j=0; j 0) + /* i is the 1st member whose file_view.count > 0 */ + i = j; + if (meta[j*3+2] == 0) { + /* member j's file_view.off are not sorted */ + indv_sorted = 0; + do_sort = 1; + break; + } + } + /* i is the first INA group member whose file_view.count > 0, and + * j is the first INA group member whose is_incr is false + */ + + if (i >= 0 && indv_sorted == 1) { + /* Even when file_view.off[] of all INA group members are individually + * sorted, we still need to check if offsets are interleaved. If + * interleaved, we must sort all offset-length pairs. + */ + MPI_Offset prev_end_off, sum; + + assert(meta[i*3+2] == 1); + + sum = meta[i*3]; + + /* prev_end_off is the last offset of INA group member i */ + prev_end_off = get_fview.off[sum-1]; + + /* check if the file_view.off are interleaved among INA group members */ + for (++i; i get_fview.off[sum]) { + /* get_fview.off[sum] is the non-aggregator i' 1st offset */ + do_sort = 1; /* get_fview.off are not incrementing */ + break; } + /* move on to the next member */ + sum += meta[i*3]; + prev_end_off = get_fview.off[sum-1]; } - /* update number of pairs, now all off-len pairs are not overlapped */ - npairs = i+1; + } - /* pack recv_buf, data received from non-aggregators, into wr_buf, a - * contiguous buffer, wr_buf, which will later be used in a call to - * MPI_File_write_at_all() + if (do_sort) + coalesceable = 1; + else { + /* Check if get_fview.off[] and get_fview.len[] are coalesceable. */ + coalesceable = 0; + send_amnt = get_fview.len[0]; + for (j=1; j= get_fview.off[j]) { + coalesceable = 1; + break; + } + } + } + + if (coalesceable) { + /* For read operations, the INA aggregator's current get_fview.off[] + * and get_fview.len[] collected from all its INA group members must be + * kept untouched, because the later sorting and coalescing will be + * performed on get_fview, messing up the original ones are needed to + * construct MPI datatype for the INA aggregator to send the data read + * from file to its INA group members. + */ + size_t alloc_amnt = sizeof(MPI_Offset) * get_fview.count; + orig_fview.off = (MPI_Offset*) NCI_Malloc(alloc_amnt); + orig_fview.len = (MPI_Offset*) NCI_Malloc(alloc_amnt); + memcpy(orig_fview.off, get_fview.off, alloc_amnt); + memcpy(orig_fview.len, get_fview.len, alloc_amnt); + } + else { + /* Skip allocating orig_fview only when get_fview will not be altered, + * i.e. its offset-length pairs sorted and coalesced. + */ + overlap = 0; + rd_amnt = send_amnt; + orig_fview.off = get_fview.off; + orig_fview.len = get_fview.len; + } + + if (do_sort) { + /* Sort get_fview.off[] into an increasing order. Note during sorting, + * get_fview.len[] is also moved together with their corresponding + * get_fview.off[]. */ - char *ptr = wr_buf; - buf_count = 0; - if (npairs > 0) { - memcpy(ptr, recv_buf + bufAddr[0], lengths[0]); - ptr += lengths[0]; - buf_count = lengths[0]; + if (indv_sorted) { + /* Interleaved offsets are found in the aggregated get_fview.off[], + * but individual get_fview.off[] are already sorted. This is + * commonly seen from the checkerboard domain partitioning pattern. + * In this case, heap_merge() is faster to merge all get_fview.off + * into one single sorted offset list. Note count[] must be + * initialized, so it can be used in heap_merge() + */ + MPI_Offset *count; + count = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset) * nprocs); + for (i=0; i= 0) { /* overlap detected, merge j into i */ + /* when gap > 0, pairs i and j overlap + * when gap == 0, pairs i and j are contiguous + */ + MPI_Offset i_end, j_end; + + if (gap > 0) overlap = 1; + + i_end = get_fview.off[i] + get_fview.len[i]; + j_end = get_fview.off[j] + get_fview.len[j]; + if (i_end < j_end) { + get_fview.len[i] += j_end - i_end; + rd_amnt += j_end - i_end; + } + /* else: j is entirely covered by i */ } - else { + else { /* j and i are not overlapped */ + rd_amnt += get_fview.len[j]; i++; if (i < j) { - offsets[i] = offsets[j]; - lengths[i] = lengths[j]; + get_fview.off[i] = get_fview.off[j]; + get_fview.len[i] = get_fview.len[j]; } } } - NCI_Free(bufAddr); - if (recv_buf != NULL) NCI_Free(recv_buf); - /* update number of pairs, now all off-len pairs are not overlapped */ - npairs = i+1; + /* update get_fview.count after coalesce */ + get_fview.count = i+1; + } - if (npairs == 1) { - /* No need to create fileType if writing to a contiguous space */ - offset = offsets[0]; +#if PNETCDF_PROFILING == 1 + ncmpi_inq_malloc_size(&mem_max); + // ncmpi_inq_malloc_max_size(&mem_max); + pnc_ina_mem_get[2] = MAX(pnc_ina_mem_get[2], mem_max); + pnc_ina_npairs_get = MAX(pnc_ina_npairs_get, get_fview.count); + + endT = MPI_Wtime(); + pnc_ina_get[1] += endT - startT; /* sorting */ + startT = endT; + +#endif + +do_read: + + /* Allocate read buffer and send buffer. Once data are read from file into + * rd_buf. MPI derived datatypes will be constructed for sending the read + * data to each INA group member. rd_buf will be directly used to send so + * no additional memory allocation is necessary. + * + * Note rd_amnt may not be the same as send_amnt, as there can be overlaps + * between adjacent file offset-length pairs after sorting, with overlaps + * removed. + * + * If this INA aggregator does not have to send any read data to its INA + * group members, i.e. all its non-INA aggregator members have zero-sized + * requests, and the file offset-length pairs have not been re-ordered, + * i.e. sorted and overlaps removed, then we can use user's buffer, buf, to + * call ncmpio_file_read() to read from the file, without allocating an + * additional temporary buffer. + */ + if (!do_sort && buf_view.size == send_amnt && !overlap) { + /* Only this INA aggregator has non-zero sized data to read and + * its buf_view does not required to be sorted and overlaps removed. + */ + get_bview = buf_view; + rd_buf = buf; + } + else { + /* Allocate a read buffer to store data read from the file. */ + if (rd_amnt > 0) { + rd_buf = (char*) NCI_Malloc(rd_amnt); + + get_bview.count = 1; + get_bview.off = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset)); + get_bview.len = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset)); + get_bview.off[0] = 0; + get_bview.len[0] = rd_amnt; } else { -#ifdef HAVE_MPI_LARGE_COUNT - /* construct fileview */ - mpireturn = MPI_Type_create_hindexed_c(npairs, lengths, offsets, - MPI_BYTE, &fileType); + rd_buf = NULL; + get_bview.count = 0; + get_bview.off = NULL; + get_bview.len = NULL; + } + get_bview.size = rd_amnt; + } -#else - /* construct fileview */ - mpireturn = MPI_Type_create_hindexed(npairs, lengths, offsets, - MPI_BYTE, &fileType); + int coll_indep = (fIsSet(ncp->flags, NC_MODE_INDEP)) ? NC_REQ_INDEP + : NC_REQ_COLL; + + MPI_Offset rlen; + rlen = ncmpio_file_read(ncp, coll_indep, rd_buf, get_fview, get_bview); + if (rlen < 0) { + if (status == NC_NOERR) status = (int)rlen; + rd_amnt = 0; + } + + if (get_fview.count == 0) { + /* This INA aggregation group has zero data to read. */ + if (meta != NULL) NCI_Free(meta); + return status; + } + +#if PNETCDF_PROFILING == 1 + ncmpi_inq_malloc_size(&mem_max); + // ncmpi_inq_malloc_max_size(&mem_max); + pnc_ina_mem_get[3] = MAX(pnc_ina_mem_get[3], mem_max); + + endT = MPI_Wtime(); + pnc_ina_get[2] += endT - startT; + startT = endT; #endif - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn,"MPI_Type_create_hindexed"); - /* return the first encountered error if there is any */ - if (status == NC_NOERR) status = err; - } - else { - mpireturn = MPI_Type_commit(&fileType); - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn,"MPI_Type_commit"); - /* return the first encountered error if there is any */ - if (status == NC_NOERR) status = err; - } + + /* If sorting has been performed, the orders of get_fview.off[] and + * get_fview.len[] may no longer be the same as the original ones. We must + * use binary search to find the offset-length pair from get_fview that + * contains each INA group member's offset-length pair to construct a send + * buffer datatype, a view layout to the read buffer, rd_buf, so the data + * can be directly sent from rd_buf. + */ + if (rd_buf != buf) { + /* First, the INA aggregator takes case of its own read, by coping the + * read data to its user buffer. Note get_fview.off[] has been sorted + * in an incremental order. + * + * When the offset-length pairs of read buffer have been sorted or the + * read buffer size is smaller than the total read amount, we must + * search and copy from read buffer to self's user buffer. + */ + char *ptr=NULL, *tmp_buf=NULL; + size_t m=0, k, scan_off=0; + + /* If this INA aggregator's user buffer is contiguous, then we have + * used buf as the file read buffer. If not, allocate a temporary + * buffer, copy the read data over, and then unpacking it to the user + * buffer. + */ + if (buf_view.count <= 1) + ptr = buf; + else if (buf_view.size > 0) + ptr = tmp_buf = (char*) NCI_Malloc(buf_view.size); + + for (j=0; j 0 && buf_view.count > 1) { + char *buf_ptr=tmp_buf; + for (j=0; jaggr_time += MPI_Wtime() - timing; +#if PNETCDF_PROFILING == 1 + ncmpi_inq_malloc_size(&mem_max); + // ncmpi_inq_malloc_max_size(&mem_max); + pnc_ina_mem_get[4] = MAX(pnc_ina_mem_get[4], mem_max); + + endT = MPI_Wtime(); + pnc_ina_get[3] += endT - startT; + startT = endT; +#endif + + if (nreqs > 0) { +#ifdef HAVE_MPI_STATUSES_IGNORE + TRACE_COMM(MPI_Waitall)(nreqs, req, MPI_STATUSES_IGNORE); +#else + MPI_Status *statuses = (MPI_Status *) + NCI_Malloc(nreqs * sizeof(MPI_Status)); + TRACE_COMM(MPI_Waitall)(nreqs, req, statuses); + NCI_Free(statuses); #endif + if (mpireturn != MPI_SUCCESS) { + err = ncmpii_error_mpi2nc(mpireturn,"MPI_Waitall"); + /* return the first encountered error if there is any */ + if (status == NC_NOERR) status = err; + } + } + NCI_Free(blks); + NCI_Free(disps); - if (ncp->rank != ncp->my_aggr) /* non-aggregator writes nothing */ - buf_count = 0; +fn_exit: + if (rd_buf != NULL && rd_buf != buf) { + NCI_Free(get_bview.len); + NCI_Free(get_bview.off); + NCI_Free(rd_buf); + } - /* Only aggregators writes non-zero sized of data to the file. The - * non-aggregators participate the collective write call with zero-length - * write requests. - */ - fh = ncp->collective_fh; + if (orig_fview.len != NULL && orig_fview.len != get_fview.len) + NCI_Free(orig_fview.len); + if (orig_fview.off != NULL && orig_fview.off != get_fview.off) + NCI_Free(orig_fview.off); + if (req != NULL) NCI_Free(req); + if (meta != NULL) NCI_Free(meta); + + /* free space allocated for get_fview and buf_view */ + if (get_fview.count > 0) { + NCI_Free(get_fview.off); + NCI_Free(get_fview.len); + } - /* set the MPI-IO fileview, this is a collective call */ - err = ncmpio_file_set_view(ncp, fh, &offset, fileType); - if (fileType != MPI_BYTE) MPI_Type_free(&fileType); - if (err != NC_NOERR) { - if (status == NC_NOERR) status = err; - buf_count = 0; + if (buf_view.count > 0) { + NCI_Free(buf_view.off); + NCI_Free(buf_view.len); } - /* call MPI_File_write_at_all */ - err = ncmpio_read_write(ncp, NC_REQ_WR, NC_REQ_COLL, offset, buf_count, - MPI_BYTE, wr_buf, 1); - if (status == NC_NOERR) status = err; +#if PNETCDF_PROFILING == 1 + ncmpi_inq_malloc_size(&mem_max); + // ncmpi_inq_malloc_max_size(&mem_max); + pnc_ina_mem_get[5] = MAX(pnc_ina_mem_get[5], mem_max); - if (wr_buf != NULL) NCI_Free(wr_buf); + endT = MPI_Wtime(); + pnc_ina_get[4] += endT - startT; + + if (rank > 0) { /* set timers to zeros for non-INA aggregators */ + for (i=0; i------------------------------*/ -/* This is a collective call */ +/*----< req_compare() >------------------------------------------------------*/ +/* This subroutine is used to sort the string file offsets of reqs[] */ +static int +req_compare(const void *a, const void *b) +{ + if (((NC_req*)a)->offset_start > ((NC_req*)b)->offset_start) return (1); + if (((NC_req*)a)->offset_start < ((NC_req*)b)->offset_start) return (-1); + return (0); +} + +/*----< ncmpio_ina_nreqs() >-------------------------------------------------*/ +/* This subroutine handles PnetCDF's requests made from non-blocking APIs, + * which contain multiple requests to one or more variable. The input arguments + * are described below. + * reqMode: NC_REQ_RD for read request and NC_REQ_WR for write. + * num_reqs: number of elements in array req_list. + * req_list[]: stores pending requests from non-blocking API calls, which is + * used to construct file offset-length pairs and user buffer + * datatype. + * newnumrecs: number of new records + * + * Note even when INA is disabled, this subroutine is still called by all + * nonblocking put/get APIs. + */ int -ncmpio_intra_node_aggregation_nreqs(NC *ncp, - int reqMode, - int num_reqs, - NC_req *put_list, - MPI_Offset newnumrecs) +ncmpio_ina_nreqs(NC *ncp, + int reqMode, + int num_reqs, + NC_req *req_list, + MPI_Offset newnumrecs) { - int err, status=NC_NOERR; - MPI_Aint bufLen, num_pairs; -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count *offsets=NULL, *lengths=NULL; -#else - MPI_Aint *offsets=NULL; - int *lengths=NULL; -#endif - MPI_Datatype bufType=MPI_BYTE; -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) + int err, status=NC_NOERR, is_incr=1; + void *buf=NULL; + PNCIO_View file_view, buf_view; + +#if PNETCDF_PROFILING == 1 double timing = MPI_Wtime(); #endif - /* currently supports write requests only */ - if (fIsSet(reqMode, NC_REQ_RD)) return NC_NOERR; + /* populate reqs[].offset_start, starting offset of each request */ + NC_req *reqs = req_list; + int i, descreasing=0; + for (i=0; iget_lead_list + : ncp->put_lead_list; + lead += reqs[i].lead_off; + varp = lead->varp; - assert(ncp->my_aggr >= 0); + if (varp->ndims == 0) { /* scalar variable */ + reqs[i].offset_start += varp->begin; + } + else if (reqs[i].npairs == 1) { /* only one offset-length pair */ + MPI_Offset off = varp->begin; + + if (IS_RECVAR(varp)) off += reqs[i].start[0] * ncp->recsize; + + reqs[i].offset_start += off; + } + else { + /* start/count/stride have been allocated in a contiguous array */ + MPI_Offset *count, *stride, offset_end; + count = reqs[i].start + varp->ndims; + stride = (fIsSet(lead->flag, NC_REQ_STRIDE_NULL)) ? NULL : + count + varp->ndims; + + /* calculate access range of this request */ + ncmpio_calc_start_end(ncp, varp, reqs[i].start, count, stride, + &reqs[i].offset_start, &offset_end); + } + /* check if offset_start are in a monotonic non-decreasing order */ + if (i > 0 && reqs[i].offset_start < reqs[i-1].offset_start) + descreasing = 1; + } - /* construct file offset-length pairs - * num_pairs: total number of off-len pairs - * offsets: array of flattened offsets - * lengths: array of flattened lengths + /* If a decreasing order is found, sort reqs[] based on reqs[].offset_start + * into an increasing order. */ - if (num_reqs > 0) - flatten_reqs(ncp, num_reqs, put_list, &num_pairs, &offsets, &lengths); - else - num_pairs = 0; + if (descreasing) + qsort(reqs, (size_t)num_reqs, sizeof(NC_req), req_compare); + + /* construct file_view, the file access offset-length pairs + * file_view.count: total number of offset_length pairs + * file_view.off[]: array of file offsets + * file_view.len[]: array of lengths + * is_incr: whether file_view.off[] are incremental + */ + err = flatten_nreqs(ncp, reqMode, num_reqs, reqs, &is_incr, &file_view); + if (status == NC_NOERR) status = err; - /* construct write buffer datatype, bufType. - * bufLen is the buffer size in bytes + /* Note offsets lengths may contain overlaps between consecutive pairs when + * the user's requests contain overlaps. They, if exist, will be resolved + * later in ina_put() and ina_get(). */ - if (num_reqs > 0) { - construct_buf_type(ncp, num_reqs, put_list, &bufLen, &bufType); - bufLen = 1; - } - else - bufLen = 0; - if (put_list != NULL) - NCI_Free(put_list); + /* Populate buf_view, which contains metadata describing the user buffer + * from the nonblocking requests. + */ + err = flat_buf_type(ncp, reqMode, num_reqs, reqs, &buf_view, &buf); + if (status == NC_NOERR) status = err; + +#if PNETCDF_DEBUG_MODE == 1 + if (num_reqs > 0) assert(buf_view.count > 0); +#endif -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) - ncp->aggr_time += MPI_Wtime() - timing; + if (req_list != NULL) { + /* All metadata in req_list have been extracted to set file_view and + * buf_view. It is now safe to release the space allocated by req_list. + */ + NCI_Free(req_list); + } + +#if PNETCDF_PROFILING == 1 + pnc_ina_flatten += MPI_Wtime() - timing; #endif - err = intra_node_aggregation(ncp, num_pairs, offsets, lengths, bufLen, - bufType, NULL); + /* Perform intra-node aggregation and file I/O. Note that the space + * allocated for file_view and buf_view will be freed at the end of + * ina_put() and ina_get(). + */ + if (fIsSet(reqMode, NC_REQ_WR)) + err = ina_put(ncp, is_incr, file_view, buf_view, buf); + else + err = ina_get(ncp, is_incr, file_view, buf_view, buf); if (status == NC_NOERR) status = err; - /* free and reset bufType */ - if (bufType != MPI_BYTE && bufType != MPI_DATATYPE_NULL) - MPI_Type_free(&bufType); - /* Update the number of records if new records have been created. * For nonblocking APIs, there is no way for a process to know whether * others write to a record variable or not. Note newnumrecs has been @@ -1344,58 +2476,88 @@ ncmpio_intra_node_aggregation_nreqs(NC *ncp, return status; } -/*----< ncmpio_intra_node_aggregation() >------------------------------------*/ -/* This is a collective call */ +/*----< ncmpio_ina_req() >---------------------------------------------------*/ +/* This subroutine handles a single request made by blocking APIs, involving + * only one variable. Below describe the subroutine arguments. + * reqMode: NC_REQ_RD for read request and NC_REQ_WR for write. + * varp: pointer to the variable struct. + * start[]: starting offsets + * count[]: counts along each dimension + * stride[]: stride along each dimension + * buf_len: size of I/O buffer in bytes + * buf: pointer to the user buffer + * + * Note even when INA is disabled, this subroutine is still called by all + * blocking put/get APIs. + */ int -ncmpio_intra_node_aggregation(NC *ncp, - int reqMode, - NC_var *varp, - const MPI_Offset *start, - const MPI_Offset *count, - const MPI_Offset *stride, - MPI_Offset bufCount, - MPI_Datatype bufType, - void *buf) +ncmpio_ina_req(NC *ncp, + int reqMode, + const NC_var *varp, + const MPI_Offset *start, + const MPI_Offset *count, + const MPI_Offset *stride, + MPI_Offset buf_len, + void *buf) { - int err, status=NC_NOERR; - MPI_Aint num_pairs; -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count *offsets=NULL, *lengths=NULL; -#else - MPI_Aint *offsets=NULL; - int *lengths=NULL; -#endif -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) + int err, status=NC_NOERR, is_incr=1; + PNCIO_View file_view, buf_view; + +#if PNETCDF_PROFILING == 1 double timing = MPI_Wtime(); #endif - /* currently supports write requests only */ - if (fIsSet(reqMode, NC_REQ_RD)) return NC_NOERR; - - if (buf == NULL) /* zero-length request */ - return intra_node_aggregation(ncp, 0, NULL, NULL, 0, MPI_BYTE, NULL); - - /* construct file offset-length pairs - * num_pairs: total number of off-len pairs - * offsets: array of flattened offsets - * lengths: array of flattened lengths - */ - err = flatten_req(ncp, varp, start, count, stride, &num_pairs, &offsets, - &lengths); - if (err != NC_NOERR) { - num_pairs = 0; - if (offsets != NULL) - NCI_Free(offsets); - offsets = NULL; + file_view.count = 0; + file_view.off = NULL; + file_view.len = NULL; + buf_view.count = 0; + buf_view.off = NULL; + buf_view.len = NULL; + buf_view.size = 0; + + if (buf_len > 0 && buf != NULL) { + /* construct file_view, the file access offset-length pairs + * file_view.count: total number of offset_length pairs + * file_view.off[]: array of file offsets + * file_view.len[]: array of lengths + * is_incr: whether file_view.off[] are incremental + */ + err = flatten_req(ncp, varp, start, count, stride, &is_incr, + &file_view); + if (err == NC_NOERR) { + /* buffer passed to blocking APIs is always contiguous */ + buf_view.count = 1; + buf_view.off = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset)); + buf_view.len = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset)); + buf_view.off[0] = 0; + buf_view.len[0] = buf_len; + buf_view.size = buf_len; + } + else { /* make this zero-sized request */ + file_view.count = 0; + } + status = err; } - status = err; + /* The else case indicates this is a zero-length request. + * When in collective data mode, this rank must still participate + * collective calls. When INA is enabled, this rank tells its aggregator + * that it has no I/O data. When INA is disabled, this rank must + * participate other collective file call. + */ -#if defined(PNETCDF_PROFILING) && (PNETCDF_PROFILING == 1) - ncp->aggr_time += MPI_Wtime() - timing; +#if PNETCDF_PROFILING == 1 + pnc_ina_flatten += MPI_Wtime() - timing; #endif - err = intra_node_aggregation(ncp, num_pairs, offsets, lengths, bufCount, - bufType, buf); + /* Perform intra-node aggregation and file I/O. Note that the space + * allocated for file_view and buf_view will be freed at the end of + * ina_put() and ina_get(). + */ + if (fIsSet(reqMode, NC_REQ_WR)) + err = ina_put(ncp, is_incr, file_view, buf_view, buf); + else + err = ina_get(ncp, is_incr, file_view, buf_view, buf); + if (status == NC_NOERR) status = err; return status; diff --git a/src/drivers/ncmpio/ncmpio_open.c b/src/drivers/ncmpio/ncmpio_open.c index a24726ee90..8a35ab31d8 100644 --- a/src/drivers/ncmpio/ncmpio_open.c +++ b/src/drivers/ncmpio/ncmpio_open.c @@ -16,165 +16,384 @@ #include #include +#include #include /* strcpy() */ #ifdef HAVE_ACCESS #include /* access() */ #endif +#include /* O_CREAT, O_RDWR, O_RDONLY */ +#include #include +#if PNETCDF_DRIVER_GIO == 1 +#include +#endif + #include #include #include "ncmpio_NC.h" -#ifdef ENABLE_SUBFILING +#if PNETCDF_SUBFILING == 1 #include "ncmpio_subfile.h" #endif /*----< ncmpio_open() >------------------------------------------------------*/ int -ncmpio_open(MPI_Comm comm, - const char *path, - int omode, - int ncid, - MPI_Info user_info, /* user's and env info combined */ - void **ncpp) +ncmpio_open(MPI_Comm comm, + const char *path, + int omode, + int ncid, + int env_mode, + MPI_Info user_info, /* user's and env info combined */ + PNC_comm_attr comm_attr, /* node IDs and INA metadata */ + void **ncpp) { - char *env_str, *mpi_name; - int i, mpiomode, err, status=NC_NOERR, mpireturn; - MPI_File fh; - MPI_Info info_used; + char value[MPI_MAX_INFO_VAL + 1], *mpi_name; + int i, rank, nprocs, mpi_amode, err, status=NC_NOERR, mpireturn, flag; + int striping_info[2]; + MPI_File fh=MPI_FILE_NULL; NC *ncp=NULL; *ncpp = NULL; + MPI_Comm_rank(comm, &rank); + MPI_Comm_size(comm, &nprocs); + /* Note path's validity and omode consistency have been checked in - * ncmpi_open() in src/dispatchers/file.c and - * path consistency will be done in MPI_File_open */ + * ncmpi_open() in src/dispatchers/file.c. + */ /* First, check whether omode is valid or supported ---------------------*/ + /* NC_DISKLESS is not supported yet */ if (omode & NC_DISKLESS) DEBUG_RETURN_ERROR(NC_EINVAL_OMODE) /* NC_MMAP is not supported yet */ if (omode & NC_MMAP) DEBUG_RETURN_ERROR(NC_EINVAL_OMODE) -#if 0 && defined(HAVE_ACCESS) - if (mpiomode == MPI_MODE_RDONLY) { /* file should already exit */ - int rank, file_exist; - MPI_Comm_rank(comm, &rank); - if (rank == 0) { - if (access(path, F_OK) == 0) file_exist = 1; - else file_exist = 0; - } - TRACE_COMM(MPI_Bcast)(&file_exist, 1, MPI_INT, 0, comm); - if (!file_exist) DEBUG_RETURN_ERROR(NC_ENOENT) - } -#endif + /* allocate buffer for header object NC and initialize its contents */ + ncp = (NC*) NCI_Calloc(1, sizeof(NC)); + if (ncp == NULL) DEBUG_RETURN_ERROR(NC_ENOMEM) - /* open file collectively ---------------------------------------------- */ - mpiomode = fIsSet(omode, NC_WRITE) ? MPI_MODE_RDWR : MPI_MODE_RDONLY; + *ncpp = (void*)ncp; - TRACE_IO(MPI_File_open, (comm, (char *)path, mpiomode, user_info, &fh)); - if (mpireturn != MPI_SUCCESS) - return ncmpii_error_mpi2nc(mpireturn, mpi_name); + ncp->ncid = ncid; + ncp->comm = comm; /* reuse comm duplicated in dispatch layer */ + ncp->rank = rank; + ncp->nprocs = nprocs; + ncp->info = MPI_INFO_NULL; - /* get the file info used/modified by MPI-IO */ - TRACE_IO(MPI_File_get_info, (fh, &info_used)); - if (mpireturn != MPI_SUCCESS) - return ncmpii_error_mpi2nc(mpireturn, mpi_name); + /* Extract hints from user_info. Two hints must be extracted now in order + * to continue: + * nc_driver: whether to user MPI-IO or GIO driver. + * nc_num_aggrs_per_node: number of processes per node to be INA + * aggregators. + * + * ncp->driver is initialized in ncmpio_hint_extract(). + * ncp->fstype is set in ncmpio_FileSysType(). + */ + ncmpio_hint_extract(ncp, user_info); - /* Now the file has been successfully opened, allocate/set NC object */ + if (rank == 0) + /* Check file system type. If the given file does not exist, check its + * parent folder. + */ + ncp->fstype = ncmpio_FileSysType(path); - /* path's validity and omode consistency have been checked in ncmpi_open() - * in src/dispatchers/file.c */ + MPI_Bcast(&ncp->fstype, 1, MPI_INT, 0, ncp->comm); - /* allocate buffer for header object NC */ - ncp = (NC*) NCI_Calloc(1, sizeof(NC)); - if (ncp == NULL) DEBUG_RETURN_ERROR(NC_ENOMEM) + ncp->path = path; /* reuse path duplicated in dispatch layer */ + ncp->nc_amode = omode; + +#if PNETCDF_DRIVER_GIO == 1 + ncp->gio_fh = NULL; /* when using GIO driver */ +#endif + + ncp->mpio_fh_coll = MPI_FILE_NULL; + ncp->mpio_fh_indep = MPI_FILE_NULL; + + /* Setting file open mode in mpi_amode which may later be needed in + * ncmpi_begin_indep_data() to open file for independent data mode. + */ + mpi_amode = fIsSet(omode, NC_WRITE) ? MPI_MODE_RDWR : MPI_MODE_RDONLY; + ncp->mpi_amode = mpi_amode; /* PnetCDF default fill mode is no fill */ fClr(ncp->flags, NC_MODE_FILL); + + /* set read-only mode */ if (!fIsSet(omode, NC_WRITE)) fSet(ncp->flags, NC_MODE_RDONLY); - ncp->ncid = ncid; + fSet(ncp->flags, env_mode); - /* chunk size for reading header (set default before check hints) */ - ncp->chunk = PNC_DEFAULT_CHUNKSIZE; + /* comm_attr has been constructed at the dispatchers. + * + * comm_attr.NUMA_IDs[] stores the NUMA compute node IDs of all MPI ranks + * of this comm (the MPI communicator passed from the user application). It + * is a keyval attribute cached in the communicator, comm. See how + * comm_attr.NUMA_IDs[] was constructed in src/dispatchers/file.c. + * + * When the intra-node aggregation (INA) is enabled, commm_attr also stores + * an intra-node communicator (containing processes belonging to its INA + * group) and an inter-node communicator (containing all the INA + * aggregators). + * + * The inter-node communicator will be used to call GIO_open() or + * MPI_File_open(), which means only the INA aggregators will perform + * collective I/O to the file. + */ + ncp->comm_attr = comm_attr; - /* buffer to pack noncontiguous user buffers when calling wait() */ - ncp->ibuf_size = PNC_DEFAULT_IBUF_SIZE; +#if PNETCDF_DEBUG_MODE == 1 + if (ncp->num_aggrs_per_node == 0) + assert(comm_attr.ina_intra_comm == MPI_COMM_NULL); + else + assert(comm_attr.ina_intra_comm != MPI_COMM_NULL); +#endif - /* Extract PnetCDF specific I/O hints from user_info and set default hint - * values into info_used. Note some MPI libraries, such as MPICH 3.3.1 and - * priors fail to preserve user hints that are not recogniozed by the MPI - * libraries. + /* When the total number of aggregators >= number of processes, disable + * intra-node aggregation. */ - ncmpio_set_pnetcdf_hints(ncp, user_info, info_used); - - ncp->iomode = omode; - ncp->comm = comm; /* reuse comm duplicated in dispatch layer */ - MPI_Comm_rank(comm, &ncp->rank); - MPI_Comm_size(comm, &ncp->nprocs); - ncp->mpiinfo = info_used; /* is not MPI_INFO_NULL */ - ncp->mpiomode = mpiomode; - ncp->collective_fh = fh; - ncp->independent_fh = (ncp->nprocs > 1) ? MPI_FILE_NULL : fh; - ncp->path = (char*) NCI_Malloc(strlen(path) + 1); - strcpy(ncp->path, path); - -#ifdef PNETCDF_DEBUG - /* PNETCDF_DEBUG is set at configure time, which will be overwritten by - * the run-time environment variable PNETCDF_SAFE_MODE */ - ncp->safe_mode = 1; + if (ncp->num_aggrs_per_node * comm_attr.num_NUMAs >= ncp->nprocs) + ncp->num_aggrs_per_node = 0; + + /* ncp->num_aggrs_per_node = 0, or > 0 indicates whether this feature + * is disabled or enabled globally for all processes. + */ + if (ncp->num_aggrs_per_node > 0) { +#if PNETCDF_DEBUG_MODE == 1 + if (comm_attr.is_ina_aggr) /* INA aggregator */ + assert(comm_attr.ina_inter_comm != MPI_COMM_NULL); + else + assert(comm_attr.ina_inter_comm == MPI_COMM_NULL); +#endif + + /* As non-aggregators will call MPI_File_open() or GIO_open(), we now + * can replace comm with ina_inter_comm. Note 'comm' is a local + * variable of this subroutine. + */ + comm = comm_attr.ina_inter_comm; + + /* For non-INA aggregators, comm_attr.ina_inter_comm is MPI_COMM_NULL. + * Because the codes below till label 'after_open' are for INA + * aggregators to open the file and obtain a file handler, non-INA + * aggregators do not participate and thus do not make use of comm. + */ + if (comm == MPI_COMM_NULL) { + if (user_info != MPI_INFO_NULL) + MPI_Info_dup(user_info, &ncp->info); + goto after_open; /* non-INA aggregators skip to 'after_open' */ + } + +#if PNETCDF_DEBUG_MODE == 1 + assert(comm_attr.is_ina_aggr); /* INA aggregator */ +#endif + + /* As non-aggregators will not perform any file I/O, we now can replace + * nprocs with ina_inter_comm's size. Note 'nprocs' is a local variable + * of this subroutine. + */ + + /* As non-aggregators will not perform any file I/O, we now can replace + * nprocs with ina_inter_comm's size. + */ + MPI_Comm_size(comm, &nprocs); + } + + /* open file collectively ---------------------------------------------- */ + if (ncp->driver == PNC_DRIVER_MPIIO) { +#ifdef MPICH_VERSION + /* MPICH recognizes file system type acronym prefixed to file names */ + TRACE_IO(MPI_File_open, (comm, path, mpi_amode, user_info, &fh)); +#else + /* Remove the file system type prefix name if there is any, because + * some MPI libraries do not recognize such prefix. For example, when + * path = "lustre:/home/foo/testfile.nc", remove "lustre:" to make + * filename pointing to "/home/foo/testfile.nc". + */ + char *filename; + filename = ncmpii_remove_file_system_type_prefix(path); + TRACE_IO(MPI_File_open, (comm, filename, mpi_amode, user_info, &fh)); #endif - /* If environment variable PNETCDF_SAFE_MODE is set to 1, then we perform - * a strict consistent test, i.e. arguments used in def_dim/def_var APIs + if (mpireturn != MPI_SUCCESS) { + err = ncmpii_error_mpi2nc(mpireturn, mpi_name); + DEBUG_FOPEN_ERROR(err); + } + + /* Now the file has been successfully opened */ + ncp->mpio_fh_coll = fh; + ncp->mpio_fh_indep = (nprocs > 1) ? MPI_FILE_NULL : fh; + + /* get the I/O hints used/modified by MPI-IO */ + TRACE_IO(MPI_File_get_info, (fh, &ncp->info)); + if (mpireturn != MPI_SUCCESS) { + err = ncmpii_error_mpi2nc(mpireturn, mpi_name); + DEBUG_FOPEN_ERROR(err); + } + } +#if PNETCDF_DRIVER_GIO == 1 + else if (ncp->driver == PNC_DRIVER_GIO) { + /* Use GIO driver. + * + * Note when INA is enabled, only the INA aggregators call GIO_open(). + * Non-INA aggregators have skipped to 'after_open' from above, with + * their 'comm', a local variable of this subroutine, remaining + * MPI_COMM_NULL (ncp->comm is always assigned to comm). + */ + int amode = fIsSet(omode, NC_WRITE) ? O_RDWR : O_RDONLY; + err = GIO_open(comm, path, amode, user_info, &ncp->gio_fh); + if (err != GIO_NOERR) { + err = ncmpii_error_gio2nc(err, "GIO_open"); + DEBUG_FOPEN_ERROR(err); + } + + /* Now the file has been successfully opened, obtain the I/O hints + * used/modified by GIO driver. + */ + err = GIO_get_info(ncp->gio_fh, &ncp->info); + if (err != NC_NOERR) DEBUG_FOPEN_ERROR(err); + } +#endif + else + DEBUG_FOPEN_ERROR(NC_EDRIVER); + +after_open: + ncp->striping_unit = 0; + ncp->striping_factor = 0; + + /* All processes must obtain striping_unit and striping_factor consistent + * across all processes in order to fulfill ncmpi_inq_striping() and + * striping_unit is also used to set ncp->data_chunk if hint + * nc_data_move_chunk_size is not set by the user. + * + * When INA is enabled, only INA aggregators have valid hints striping_unit + * and striping_factor stored in their ncp->info, returned from the call to + * MPI_File_get_info() or GIO_get_info(). When non-INA aggregators have + * never call MPI_File_open() or GIO_open(), only happened when performing + * independent I/O, they do not have valid file striping hints store in + * their ncp->info. We need to make INA root to broadcast these 2 info to + * its INA group members. It is OK to have all processes extract these 2 + * hints below. In this case, the INA aggregators' hints will be valid and + * non-INA aggregators' hints will be overwritten by their INA aggregator's + * after the call of MPI_Bcast() later. + * + * When INA is disabled, all processes extract hints. */ - if ((env_str = getenv("PNETCDF_SAFE_MODE")) != NULL) { - if (*env_str == '0') ncp->safe_mode = 0; - else ncp->safe_mode = 1; - /* if PNETCDF_SAFE_MODE is set but without a value, *env_str can - * be '\0' (null character). In this case, safe_mode is enabled */ + MPI_Info_get(ncp->info, "striping_unit", MPI_MAX_INFO_VAL-1, value, &flag); + striping_info[0] = 0; + if (flag) { + errno = 0; /* errno must set to zero before calling strtoll */ + striping_info[0] = (int)strtol(value,NULL,10); + if (errno != 0) striping_info[0] = 0; + } + + MPI_Info_get(ncp->info, "striping_factor", MPI_MAX_INFO_VAL-1, value, + &flag); + striping_info[1] = 0; + if (flag) { + errno = 0; /* errno must set to zero before calling strtoll */ + striping_info[1] = (int)strtol(value,NULL,10); + if (errno != 0) striping_info[1] = 0; + } + + if (ncp->num_aggrs_per_node > 0) { + /* When INA is enabled, each INA aggregator broadcasts the file + * striping hints to its INA group members. + * + * Note ncp->num_aggrs_per_node may have been adjusted above. + */ + int intra_nprocs, intra_rank; + +#if PNETCDF_DEBUG_MODE == 1 + assert(comm_attr.ina_intra_comm != MPI_COMM_NULL); +#endif + MPI_Comm_size(comm_attr.ina_intra_comm, &intra_nprocs); + + if (intra_nprocs > 1) { + MPI_Info_get(ncp->info, "striping_unit", MPI_MAX_INFO_VAL-1, value, + &flag); + striping_info[0] = 0; + if (flag) { + errno = 0; /* errno must set to zero before calling strtoll */ + striping_info[0] = (int)strtol(value,NULL,10); + if (errno != 0) striping_info[0] = 0; + } + + MPI_Info_get(ncp->info, "striping_factor", MPI_MAX_INFO_VAL-1, + value, &flag); + striping_info[1] = 0; + if (flag) { + errno = 0; /* errno must set to zero before calling strtoll */ + striping_info[1] = (int)strtol(value,NULL,10); + if (errno != 0) striping_info[1] = 0; + } + + MPI_Comm_rank(comm_attr.ina_intra_comm, &intra_rank); + + MPI_Bcast(striping_info, 2, MPI_INT, 0, comm_attr.ina_intra_comm); + + if (intra_rank > 0) { + /* non-INA aggregators have not set these to ncp->info */ + sprintf(value, "%d", striping_info[0]); + MPI_Info_set(ncp->info, "striping_unit", value); + sprintf(value, "%d", striping_info[1]); + MPI_Info_set(ncp->info, "striping_factor", value); + } + } } - /* read header from file into NC object pointed by ncp -------------------*/ + ncp->striping_unit = striping_info[0]; + ncp->striping_factor = striping_info[1]; + + if (ncp->data_chunk == -1) + /* if hint nc_data_move_chunk_size is not set by the user */ + ncp->data_chunk = (ncp->striping_unit > 0) ? ncp->striping_unit + : PNC_DATA_MOVE_CHUNK_SIZE; + + /* Add PnetCDF hints into ncp->info. This step is necessary, because the + * underneath MPI-IO may discard hints it does not recognize, which include + * all PnetCDF hints. PnetCDF hints need to be added to info, so users can + * inquire them. + */ + ncmpio_hint_set(ncp, ncp->info); + + /* read header from file into NC object pointed by ncp ------------------*/ err = ncmpio_hdr_get_NC(ncp); if (err == NC_ENULLPAD) status = NC_ENULLPAD; /* non-fatal error */ else if (err != NC_NOERR) { /* fatal error */ - ncmpio_close_files(ncp, 0); + ncmpio_file_close(ncp); ncmpio_free_NC(ncp); - return err; + DEBUG_RETURN_ERROR(err); } -#ifdef ENABLE_SUBFILING +#if PNETCDF_SUBFILING == 1 if (ncp->subfile_mode) { /* check subfiling attribute */ err = ncmpio_get_att(ncp, NC_GLOBAL, "_PnetCDF_SubFiling.num_subfiles", &ncp->num_subfiles, MPI_INT); if (err == NC_NOERR && ncp->num_subfiles > 1) { - int i; /* ignore error NC_ENOTATT if this attribute is not defined */ for (i=0; ivars.ndefined; i++) { /* variables may have different numbers of subfiles */ err = ncmpio_get_att(ncp, i, "_PnetCDF_SubFiling.num_subfiles", &ncp->vars.value[i]->num_subfiles,MPI_INT); if (err == NC_ENOTATT) continue; - if (err != NC_NOERR) return err; + if (err != NC_NOERR) DEBUG_FOPEN_ERROR(err); if (ncp->vars.value[i]->num_subfiles > 1) { /* find the orginal ndims of variable i */ err = ncmpio_get_att(ncp,i,"_PnetCDF_SubFiling.ndims_org", &ncp->vars.value[i]->ndims_org,MPI_INT); - if (err != NC_NOERR) return err; + if (err != NC_NOERR) DEBUG_FOPEN_ERROR(err); ncp->vars.value[i]->dimids_org = (int*) NCI_Malloc( ncp->vars.value[i]->ndims_org * SIZEOF_INT); err = ncmpio_get_att(ncp,i,"_PnetCDF_SubFiling.dimids_org", ncp->vars.value[i]->dimids_org, MPI_INT); - if (err != NC_NOERR) return err; + if (err != NC_NOERR) DEBUG_FOPEN_ERROR(err); } } /* open subfile */ err = ncmpio_subfile_open(ncp); - if (err != NC_NOERR) return err; + if (err != NC_NOERR) DEBUG_FOPEN_ERROR(err); } else ncp->num_subfiles = 0; } @@ -191,21 +410,6 @@ ncmpio_open(MPI_Comm comm, ncp->vars.value[i]->attrs.hash_size = ncp->hash_size_attr; #endif - /* determine whether to enable intra-node aggregation and set up all - * intra-node aggregation metadata. - * ncp->num_aggrs_per_node = 0, or non-zero indicates whether this feature - * is enabled globally for all processes. - * ncp->my_aggr = -1 or >= 0 indicates whether aggregation is effectively - * enabled for the aggregation group of this process. - */ - ncp->my_aggr = -1; - if (ncp->num_aggrs_per_node != 0) { - err = ncmpio_intra_node_aggr_init(ncp); - if (err != NC_NOERR) return err; - } - - *ncpp = (void*)ncp; - return status; } diff --git a/src/drivers/ncmpio/ncmpio_subfile.c b/src/drivers/ncmpio/ncmpio_subfile.c index e1be70ec7c..dcf8fbbcba 100644 --- a/src/drivers/ncmpio/ncmpio_subfile.c +++ b/src/drivers/ncmpio/ncmpio_subfile.c @@ -86,12 +86,266 @@ static int ncmpio_itoa(int val, char* buf) } #endif +/* ina_init() below is copied from ../../dispatchers/file.c */ + +/*----< ina_init() >---------------------------------------------------------*/ +/* When the intra-node write aggregation (INA) hint is enabled, this subroutine + * initializes the metadata to be used in intra- and inter-node communication, + * including an intra-node communicator for communication between INA + * aggregators and non-INA aggregators, and an inter-node communicator + * consisting of all the INA aggregators to be used when calling GIO/MPI-IO + * file open and their collective and independent I/O calls. + * + * Processes on the same NUMA node will first be divided into disjoined groups. + * The passed in communicator will be split into sub-communicators, referred to + * as intra-node communicators, one for each INA group. Within a group, the + * process with the lowest rank ID is selected as the group's INA aggregator. + * + * A new MPI communicator consisting of all INA aggregators across all nodes + * will be created, which is referred to as the inter-node communicator. The + * inter-node communicator will be used to open the file, i.e. in the call to + * GIO_open()/MPI_File_open() later. Only the INA aggregators make calls to + * the GIO/MPI-IO APIs to access the file. Thus, this subroutine must be + * called before opening the file and should be called only once in + * ncmpi_create() or ncmpi_open(). + * + * This subroutine performs the following tasks. + * 1. Makes use of the affinity of each MPI process to its NUMA node to + * calculate the number of INA groups within a node and identify the + * membership of every process to its INA group. + * + comm_attr->num_NUMAs is the number of NUMA nodes. + * + comm_attr->NUMA_IDs[nprocs] contains NUMA node IDs of all processes. + * Note comm_attr should have already been established during a call to + * ncmpii_construct_node_list() at the beginning of ncmpi_create() and + * ncmpi_open(). + * 2. Based on hint num_aggrs_per_node and the number of processes per NUMA + * node, calculates the number of INA groups per node and divides processes + * into groups. Select the process with the lowest rank in a group as the + * INA aggregator. + * + comm_attr->is_ina_aggr indicates whether this rank is an INA aggregator + * 3. Create a new MPI communicator by splitting 'comm' into sub-communicators, + * each consisting of processes belonging to the same INA group. + * + comm_attr->ina_intra_comm is the INA intra-node communicator. + * + MPI_Comm_size(comm_attr->ina_intra_comm, &size) is the number of + * processes within an INA group. + * + Rank 0 in comm_attr->ina_intra_comm is the INA aggregator. + * 4. Create a new MPI communicator consisting of all the INA aggregators. + * + comm_attr->ina_inter_comm is the INA inter-node communicator. + * + MPI_Comm_size(comm_attr->ina_inter_comm, &size) is the total number of + * INA aggregators. + * + comm_attr->ina_inter_comm will be used when calling GIO_open()/ + * MPI_File_open(). + */ +static +int ina_init(MPI_Comm comm, + int num_aggrs_per_node, + PNC_comm_attr *comm_attr) +{ + int i, j, k, mpireturn, nprocs, rank, my_node_naggrs, aggr_rank; + int my_node_nprocs, my_node_rank; + int *ina_flags, grp_nprocs, rem; + +#if PNETCDF_PROFILING == 1 + double timing = MPI_Wtime(); +#endif + + if (num_aggrs_per_node == 0) return NC_NOERR; + + MPI_Comm_size(comm, &nprocs); + MPI_Comm_rank(comm, &rank); + +#if PNETCDF_DEBUG_MODE == 1 + /* Note that ill value of num_aggrs_per_node has been checked before + * entering this subroutine. Thus num_aggrs_per_node must be > 0. + */ + assert(num_aggrs_per_node > 0); + assert(comm_attr->numa_comm != MPI_COMM_NULL); +#endif + + /* comm_attr->NUMA_IDs[] has been set in ncmpii_construct_node_list() + * called earlier in ncmpio_create() or ncmpio_open() before entering this + * subroutine. + */ + + /* my_node_nprocs is the number of processes in my NUMA compute node. */ + MPI_Comm_size(comm_attr->numa_comm, &my_node_nprocs); + + /* my_node_rank is the rank ID of this process in my NUMA compute node. */ + MPI_Comm_rank(comm_attr->numa_comm, &my_node_rank); + + /* Make sure the actual number of INA aggregators per node, initially set + * in hint num_aggrs_per_node, is <= my_node_nprocs. In some cases, the + * number of processes allocated to the last few compute nodes can be less + * than others. + */ + my_node_naggrs = MIN(num_aggrs_per_node, my_node_nprocs); + + /* Divide processes in a NUMA node into INA groups and calculate the number + * of processes in each INA group, grp_nprocs. Select the INA aggregator, + * as the process with loweset rank in an INA group, whose local rank in + * comm_attr->numa_comm is 'aggr_rank'. + */ + grp_nprocs = my_node_nprocs / my_node_naggrs; /* no. processes per group */ + rem = my_node_nprocs % my_node_naggrs; + if (rem > 0) { /* non-divisible case */ + grp_nprocs++; + if (my_node_rank < grp_nprocs * rem) + /* Select the first rank of my INA group as INA aggregator. */ + aggr_rank = my_node_rank - my_node_rank % grp_nprocs; + else { + aggr_rank = grp_nprocs * rem; + grp_nprocs--; + aggr_rank = my_node_rank + - (my_node_rank - aggr_rank) % grp_nprocs; + } + } + else /* divisible case */ + aggr_rank = my_node_rank - my_node_rank % grp_nprocs; + + /* whether this rank is an INA aggregator */ + comm_attr->is_ina_aggr = (my_node_rank == aggr_rank); + +#if PNETCDF_DEBUG_MODE == 1 + /* Make sure the number of processes in my INA group does not go beyond + * my_node_nprocs. + */ + assert(grp_nprocs <= my_node_nprocs - aggr_rank); +#endif + + if (comm_attr->ina_intra_comm != MPI_COMM_NULL) { + /* free ina_intra_comm if previous created */ + TRACE_COMM(MPI_Comm_free)(&comm_attr->ina_intra_comm); + if (mpireturn != MPI_SUCCESS) + return ncmpii_error_mpi2nc(mpireturn, "MPI_Comm_free"); + } + comm_attr->ina_intra_comm = MPI_COMM_NULL; + + /* Split NUMA comm into INA intra-node comm based on the assigned INA + * aggregator's rank ID, i.e. processes sharing the same INA aggregator + * form an ina_intra_comm. This process's local rank on the NUMA node + * is my_node_rank and its INA aggregator's rank is aggr_rank. + * + * Special note on when the number of processes in this INA group is 1. In + * this case, there is only one process in this group and thus the + * intra-node aggregation of this group will not perform. However, the + * intra-node communicator of this INA group must still be created. This is + * because MPI_Comm_split is a collective call with the processes running + * on the same NUMA node, i.e. on comm_attr->numa_comm. + * + * The case of grp_nprocs == 1, does not mean intra-node aggregation is + * disabled globally. It just means this group will not perform INA + * aggregation. The indicator of whether intra-node aggregation is globally + * enabled or disabled is 'num_aggrs_per_node', whose value must be kept + * consistent across all processes. It is possible for some groups + * containing only one process, in which case the aggregation is not + * necessarily to perform within those groups. + */ + TRACE_COMM(MPI_Comm_split)(comm_attr->numa_comm, aggr_rank, my_node_rank, + &comm_attr->ina_intra_comm); + + /* Next step is to construct an inter-node MPI communicator consisting of + * all INA aggregators. It will later be used to call MPI_File_open(), and + * successively only INA aggregators call MPI-IO functions to access the + * file. + */ + + /* construct an array containing ranks of aggregators */ + ina_flags = (int*) malloc(sizeof(int) * nprocs); + TRACE_COMM(MPI_Allgather)(&comm_attr->is_ina_aggr, 1, MPI_INT, ina_flags, + 1, MPI_INT, comm); + + /* Given a comm_attr->NUMA_IDs[], the rank IDs of INA aggregators will + * dependent on the layout of MPI process allocation to the compute nodes. + * The common layouts can be two kinds: + * + cyclic - MPI ranks are assigned to nodes round-robin-ly, + * + block - MPI ranks are assigned to a node and then move on to next. + * + * Below uses an example of nodes=3, nprocs=10, * num_aggrs_per_node=2. + * comm_attr->NUMA_IDs[] should be + * block process allocation: 0,0,0,0,1,1,1,2,2,2 + * cyclic process allocation: 0,1,2,0,1,2,0,1,2,0 + * Accordingly, rank IDs of INA aggregators can be two kinds + * block process allocation: 1,0,1,0,1,0,1,1,0,1 + * cyclic process allocation: 1,1,1,0,0,0,1,1,1,0 + */ + + /* calculate actual number of INA aggregators */ + comm_attr->num_ina_aggrs = 0; + for (j=0; jnum_ina_aggrs += ina_flags[j]; + + /* Collect aggregators' rank IDs and store them in an increasing order of + * node IDs. Note rank IDs in ina_ranks[] are relative to comm (not + * inter-node comm or intra-node comm). + */ + comm_attr->ina_ranks = (int*)malloc(sizeof(int) * comm_attr->num_ina_aggrs); + for (k=0, i=0; inum_NUMAs; i++) { + for (j=0; jNUMA_IDs[j] == i && ina_flags[j] > 0) + comm_attr->ina_ranks[k++] = j; + } + } + free(ina_flags); + + if (comm_attr->ina_inter_comm != MPI_COMM_NULL) { + /* free comm_attr->ina_inter_comm if created previously */ + TRACE_COMM(MPI_Comm_free)(&comm_attr->ina_inter_comm); + if (mpireturn != MPI_SUCCESS) + return ncmpii_error_mpi2nc(mpireturn, "MPI_Comm_free"); + } + comm_attr->ina_inter_comm = MPI_COMM_NULL; + + /* create an inter-node communicator consisting of all INA aggregators */ + MPI_Group origin_group, ina_group; + TRACE_COMM(MPI_Comm_group)(comm, &origin_group); + if (mpireturn != MPI_SUCCESS) + return ncmpii_error_mpi2nc(mpireturn, "MPI_Comm_group"); + TRACE_COMM(MPI_Group_incl)(origin_group, comm_attr->num_ina_aggrs, + comm_attr->ina_ranks, &ina_group); + if (mpireturn != MPI_SUCCESS) + return ncmpii_error_mpi2nc(mpireturn, "MPI_Group_incl"); + TRACE_COMM(MPI_Comm_create)(comm, ina_group, &comm_attr->ina_inter_comm); + if (mpireturn != MPI_SUCCESS) + return ncmpii_error_mpi2nc(mpireturn, "MPI_Comm_create"); + TRACE_COMM(MPI_Group_free)(&ina_group); + if (mpireturn != MPI_SUCCESS) + return ncmpii_error_mpi2nc(mpireturn, "MPI_Group_free"); + TRACE_COMM(MPI_Group_free)(&origin_group); + if (mpireturn != MPI_SUCCESS) + return ncmpii_error_mpi2nc(mpireturn, "MPI_Group_free"); + + /* TODO: automatically determine whether or not to enable intra-node + * aggregation. + * + * The ideal case is it can be determined right before each collective + * write call, because only at that time, the communication pattern is + * known. If the pattern can cause contention, then enable it. Otherwise, + * disable it. + * + * Such mechanism may depends on the followings. + * 1. MPI-IO hint cb_noddes, and striping_unit + * 2. calculate aggregate access region + * 3. If the number of senders to each cb_nodes is very large, then + * intra-node aggregation should be enabled. + * 4. Average of nprocs_per_node across all processes may be a factor for + * determining whether to enable intra-node aggregation. It indicates + * whether the high number of processes are allocated on the same + * node. + */ + +#if PNETCDF_PROFILING == 1 + pnc_ina_init = MPI_Wtime() - timing; +#endif + + return NC_NOERR; +} + /*----< subfile_create() >---------------------------------------------------*/ static int subfile_create(NC *ncp) { - int myrank, nprocs, color, status=NC_NOERR, mpireturn; - char path_sf[1024]; + char value[MPI_MAX_INFO_VAL], path_sf[1024]; + int myrank, nprocs, color, err, status=NC_NOERR, mpireturn, flag; double ratio; MPI_Info info=MPI_INFO_NULL; @@ -124,23 +378,49 @@ subfile_create(NC *ncp) sprintf(path_sf, "%s.subfile_%i.%s", ncp->path, color, "nc"); -/* MPI_Info_create(&info); - MPI_Info_set(info, "romio_lustre_start_iodevice", offset); - MPI_Info_set(info, "striping_factor", "1"); -*/ + MPI_Info_get(ncp->info, "nc_num_aggrs_per_node", MPI_MAX_INFO_VAL-1, + value, &flag); + if (flag) + MPI_Info_set(info, "nc_num_aggrs_per_node", value); + + ncp->comm_attr_sf.num_aggrs_per_node = 0; + ncp->comm_attr_sf.num_ina_aggrs = 0; + ncp->comm_attr_sf.is_ina_aggr = 0; + ncp->comm_attr_sf.ina_ranks = NULL; + ncp->comm_attr_sf.numa_comm = MPI_COMM_NULL; + ncp->comm_attr_sf.ina_inter_comm = MPI_COMM_NULL; + ncp->comm_attr_sf.ina_intra_comm = MPI_COMM_NULL; + + /* Constructing NUMA compute node IDs requires communication calls to + * MPI_Comm_split_type(), MPI_Bcast(), and MPI_Allgather(). + */ + ncmpii_construct_node_list(ncp->comm_sf, &ncp->comm_attr_sf.num_NUMAs, + &ncp->comm_attr_sf.NUMA_IDs, + &ncp->comm_attr_sf.numa_comm); + + /* If INA is enabled, construct INA metadata */ + err = ina_init(ncp->comm_sf, ncp->num_aggrs_per_node, &ncp->comm_attr_sf); + if (err != NC_NOERR) { + ncp->comm_attr_sf.num_aggrs_per_node = 0; + status = err; + } + else + ncp->comm_attr_sf.num_aggrs_per_node = ncp->num_aggrs_per_node; void *ncp_sf; - status = ncmpio_create(ncp->comm_sf, path_sf, ncp->iomode, ncp->ncid, info, - &ncp_sf); - if (status != NC_NOERR && myrank == 0) - fprintf(stderr, "%s: error in creating file(%s): %s\n", - __func__, path_sf, ncmpi_strerror(status)); - ncp->ncp_sf = (NC*) ncp_sf; -/* + err = ncmpio_create(ncp->comm_sf, path_sf, ncp->nc_amode, ncp->ncid, + ncp->flags, info, ncp->comm_attr_sf, &ncp_sf); + if (err != NC_NOERR) { + if (status == NC_NOERR) status = err; + if (myrank == 0) + fprintf(stderr, "%s at %d: error in creating file(%s): %s\n", + __func__,__LINE__, path_sf, ncmpi_strerror(status)); + } + ncp->ncp_sf = (NC*)ncp_sf; + MPI_Info_free(&info); -*/ return status; } @@ -149,9 +429,10 @@ subfile_create(NC *ncp) int ncmpio_subfile_open(NC *ncp) { - int myrank, nprocs, color, status=NC_NOERR, mpireturn; - char path_sf[1024]; + char value[MPI_MAX_INFO_VAL], path_sf[1024]; + int myrank, nprocs, color, err, status=NC_NOERR, mpireturn, flag; double ratio; + MPI_Info info=MPI_INFO_NULL; MPI_Comm_rank(ncp->comm, &myrank); MPI_Comm_size(ncp->comm, &nprocs); @@ -186,11 +467,50 @@ ncmpio_subfile_open(NC *ncp) /* sprintf(path_sf, "%s%d/%s", path, color, file); */ sprintf(path_sf, "%s.subfile_%i.%s", ncp->path, color, "nc"); + MPI_Info_create(&info); + MPI_Info_get(ncp->info, "nc_num_aggrs_per_node", MPI_MAX_INFO_VAL-1, + value, &flag); + if (flag) + MPI_Info_set(info, "nc_num_aggrs_per_node", value); + + ncp->comm_attr_sf.num_aggrs_per_node = 0; + ncp->comm_attr_sf.num_ina_aggrs = 0; + ncp->comm_attr_sf.is_ina_aggr = 0; + ncp->comm_attr_sf.ina_ranks = NULL; + ncp->comm_attr_sf.numa_comm = MPI_COMM_NULL; + ncp->comm_attr_sf.ina_inter_comm = MPI_COMM_NULL; + ncp->comm_attr_sf.ina_intra_comm = MPI_COMM_NULL; + + /* Constructing NUMA compute node IDs requires communication calls to + * MPI_Comm_split_type(), MPI_Bcast(), and MPI_Allgather(). + */ + ncmpii_construct_node_list(ncp->comm_sf, &ncp->comm_attr_sf.num_NUMAs, + &ncp->comm_attr_sf.NUMA_IDs, + &ncp->comm_attr_sf.numa_comm); + + /* If INA is enabled, construct INA metadata */ + err = ina_init(ncp->comm_sf, ncp->num_aggrs_per_node, &ncp->comm_attr_sf); + if (err != NC_NOERR) { + ncp->comm_attr_sf.num_aggrs_per_node = 0; + status = err; + } + else + ncp->comm_attr_sf.num_aggrs_per_node = ncp->num_aggrs_per_node; + void *ncp_sf; - status = ncmpio_open(ncp->comm_sf, path_sf, ncp->iomode, ncp->ncid, - MPI_INFO_NULL, &ncp_sf); - ncp->ncp_sf = (NC*) ncp_sf; + err = ncmpio_open(ncp->comm_sf, path_sf, ncp->nc_amode, ncp->ncid, + ncp->flags, MPI_INFO_NULL, ncp->comm_attr_sf, &ncp_sf); + if (err != NC_NOERR) { + if (status == NC_NOERR) status = err; + if (myrank == 0) + fprintf(stderr, "%s at %d: error in opening file(%s): %s\n", + __func__,__LINE__, path_sf, ncmpi_strerror(status)); + } + ncp->ncp_sf = (NC*)ncp_sf; + + MPI_Info_free(&info); + return status; } @@ -200,10 +520,22 @@ int ncmpio_subfile_close(NC *ncp) int status = NC_NOERR; if (ncp->ncp_sf != NULL) { + if (ncp->comm_attr_sf.NUMA_IDs != NULL) + free(ncp->comm_attr_sf.NUMA_IDs); + status = ncmpio_close(ncp->ncp_sf); if (status != NC_NOERR) return status; ncp->ncp_sf = NULL; MPI_Comm_free(&ncp->comm_sf); + + if (ncp->comm_attr_sf.numa_comm != MPI_COMM_NULL) + MPI_Comm_free(&ncp->comm_attr_sf.numa_comm); + if (ncp->comm_attr_sf.ina_inter_comm != MPI_COMM_NULL) + MPI_Comm_free(&ncp->comm_attr_sf.ina_inter_comm); + if (ncp->comm_attr_sf.ina_intra_comm != MPI_COMM_NULL) + MPI_Comm_free(&ncp->comm_attr_sf.ina_intra_comm); + if (ncp->comm_attr_sf.ina_ranks != NULL) + free(ncp->comm_attr_sf.ina_ranks); /* malloc()-ed */ } /* reset values to 0 */ @@ -315,7 +647,7 @@ int ncmpio_subfile_partition(NC *ncp) if (dpp[vpp[i]->dimids[par_dim_id]]->size/ncp->num_subfiles > 0 && vpp[i]->ndims >= par_dim_id+1 && vpp[i]->ndims >= SUBFILING_MIN_NDIMS) { - int varid, j, jj, k; + int varid, jj, k; int var_ndims = vpp[i]->ndims; /* keep org ndims */ int dimids[var_ndims]; char *key[ncp->num_subfiles][var_ndims]; @@ -1003,7 +1335,6 @@ ncmpio_subfile_getput_vars(NC *ncp, for (i=0; i #include "ncmpio_NC.h" -/*----< ncmpio_file_sync() >-------------------------------------------------*/ -/* This function must be called collectively, no matter if it is in collective - * or independent data mode. - */ -int -ncmpio_file_sync(NC *ncp) { - char *mpi_name; - int mpireturn; - - if (ncp->independent_fh != MPI_FILE_NULL) { - TRACE_IO(MPI_File_sync, (ncp->independent_fh)); - if (mpireturn != MPI_SUCCESS) - return ncmpii_error_mpi2nc(mpireturn, mpi_name); - } - /* when nprocs == 1, ncp->collective_fh == ncp->independent_fh */ - if (ncp->nprocs == 1) return NC_NOERR; - - /* ncp->collective_fh is never MPI_FILE_NULL as collective mode is - * default in PnetCDF */ - TRACE_IO(MPI_File_sync, (ncp->collective_fh)); - if (mpireturn != MPI_SUCCESS) - return ncmpii_error_mpi2nc(mpireturn, mpi_name); - - /* Barrier is not necessary ... - TRACE_COMM(MPI_Barrier)(ncp->comm); - */ - - return NC_NOERR; -} - #define NC_NUMRECS_OFFSET 4 /*----< ncmpio_write_numrecs() >---------------------------------------------*/ -/* root process writes the new record number into file. +/* Only root process writes the new record number into file. * This function is called by: * 1. ncmpio_sync_numrecs * 2. collective nonblocking wait API, if the new number of records is bigger @@ -69,32 +39,43 @@ int ncmpio_write_numrecs(NC *ncp, MPI_Offset new_numrecs) { - char *mpi_name; - int mpireturn, err; - MPI_File fh; - MPI_Status mpistatus; + int err=NC_NOERR; - if (!fIsSet(ncp->flags, NC_HCOLL) && ncp->rank > 0) - /* Only root process writes numrecs in file */ - return NC_NOERR; - - /* return now if there is no record variabled defined */ + /* return now if there is no record variable defined */ if (ncp->vars.num_rec_vars == 0) return NC_NOERR; - fh = ncp->independent_fh; - if (ncp->nprocs > 1 && !NC_indep(ncp)) - fh = ncp->collective_fh; + /* When intra-node aggregation is enabled, non-aggregators do not + * participate any collective calls below. + */ + if (ncp->num_aggrs_per_node > 0 && !ncp->comm_attr.is_ina_aggr) + return NC_NOERR; + + if (ncp->rank > 0) { + /* Currently in independent data mode */ + if (NC_indep(ncp)) + return NC_NOERR; - if (ncp->rank > 0 && fIsSet(ncp->flags, NC_HCOLL)) { - /* other processes participate the collective call */ - TRACE_IO(MPI_File_write_at_all, (fh, 0, NULL, 0, MPI_BYTE, &mpistatus)); - return (mpireturn == MPI_SUCCESS) ? NC_NOERR : - ncmpii_error_mpi2nc(mpireturn, mpi_name); + /* If not requiring all MPI-IO calls to be collective, non-root + * processes can return now. This is because only root process writes + * numrecs to the file header. + */ + return NC_NOERR; } + /* codes below run by root only */ if (new_numrecs > ncp->numrecs || NC_ndirty(ncp)) { int len; char pos[8], *buf=pos; + MPI_Offset wlen, file_off, file_len, buf_off=0, buf_len; + PNCIO_View file_view, buf_view; + + /* both file_view and buf_view are contiguous */ + file_view.count = 1; + file_view.off = &file_off; + file_view.len = &file_len; + buf_view.count = 1; + buf_view.off = &buf_off; + buf_view.len = &buf_len; /* update ncp->numrecs */ if (new_numrecs > ncp->numrecs) ncp->numrecs = new_numrecs; @@ -113,44 +94,25 @@ ncmpio_write_numrecs(NC *ncp, } /* ncmpix_put_xxx advances the 1st argument with size len */ - /* explicitly initialize mpistatus object to 0. For zero-length read, - * MPI_Get_count may report incorrect result for some MPICH version, - * due to the uninitialized MPI_Status object passed to MPI-IO calls. - * Thus we initialize it above to work around. - */ - memset(&mpistatus, 0, sizeof(MPI_Status)); - - /* root's file view always includes the entire file header */ - if (fIsSet(ncp->flags, NC_HCOLL) && ncp->nprocs > 1) { - TRACE_IO(MPI_File_write_at_all, (fh, NC_NUMRECS_OFFSET, (void*)pos, - len, MPI_BYTE, &mpistatus)); - } - else { - TRACE_IO(MPI_File_write_at, (fh, NC_NUMRECS_OFFSET, (void*)pos, - len, MPI_BYTE, &mpistatus)); - } - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, mpi_name); - if (err == NC_EFILE) DEBUG_RETURN_ERROR(NC_EWRITE) - } - else { - /* update the number of bytes written since file open. - * Because the above MPI write writes either 4 or 8 bytes, - * calling MPI_Get_count() is sufficient. No need to call - * MPI_Get_count_c() + if (ncp->num_aggrs_per_node > 0 && !ncp->comm_attr.is_ina_aggr) + /* When intra-node aggregation is enabled, non-aggregators do not + * participate the collective call. */ - int put_size; - mpireturn = MPI_Get_count(&mpistatus, MPI_BYTE, &put_size); - if (mpireturn != MPI_SUCCESS || put_size == MPI_UNDEFINED) - ncp->put_size += len; - else - ncp->put_size += put_size; - } + return NC_NOERR; + + file_off = NC_NUMRECS_OFFSET; + file_len = len; + buf_len = len; + + wlen = ncmpio_file_write(ncp, NC_REQ_INDEP, (void*)pos, file_view, buf_view); + if (wlen < 0) + DEBUG_RETURN_ERROR((int)wlen) } - return NC_NOERR; + + return err; } -/*----< ncmpio_sync_numrecs() >-----------------------------------------------*/ +/*----< ncmpio_sync_numrecs() >----------------------------------------------*/ /* Synchronize the number of records in memory among all processes and write * numrecs to file. * This function is called by: @@ -185,7 +147,7 @@ ncmpio_sync_numrecs(void *ncdp) else /* if called in independent mode, we force sync in memory */ set_NC_ndirty(ncp); - /* return now if there is no record variabled defined */ + /* return now if there is no record variable defined */ if (ncp->vars.num_rec_vars == 0) return NC_NOERR; /* find the max numrecs among all processes @@ -202,7 +164,7 @@ ncmpio_sync_numrecs(void *ncdp) /* root process writes max_numrecs to file */ status = ncmpio_write_numrecs(ncp, max_numrecs); - if (ncp->nprocs > 1 && ncp->safe_mode == 1) { + if (ncp->nprocs > 1 && fIsSet(ncp->flags, NC_MODE_SAFE)) { /* broadcast root's status, because only root writes to the file */ int root_status = status; TRACE_COMM(MPI_Bcast)(&root_status, 1, MPI_INT, 0, ncp->comm); @@ -212,7 +174,7 @@ ncmpio_sync_numrecs(void *ncdp) if (root_status == NC_EWRITE) DEBUG_ASSIGN_ERROR(status, NC_EWRITE) } - /* update numrecs in all processes's memory */ + /* update numrecs in all processes' memory */ ncp->numrecs = max_numrecs; /* clear numrecs dirty bit */ diff --git a/src/drivers/ncmpio/ncmpio_util.c b/src/drivers/ncmpio/ncmpio_util.c index 8034f9f0be..f2fd7769b2 100644 --- a/src/drivers/ncmpio/ncmpio_util.c +++ b/src/drivers/ncmpio/ncmpio_util.c @@ -12,276 +12,418 @@ #include /* strtoll() is first introduced in C99 */ #include /* strcpy() */ #include /* strcasecmp() */ +#include /* INT_MAX */ #include #include + #include #include #include +#include #include "ncmpio_NC.h" -/*----< ncmpio_set_pnetcdf_hints() >-----------------------------------------*/ -/* this is where the I/O hints designated to pnetcdf are extracted and their - * default values are set. +#define MAX_INT_LEN 24 + +/*----< ncmpio_hint_extract() >----------------------------------------------*/ +/* Extract hints from info. Argument info is the info object set by application + * user and passed to ncmpi_create() or ncmpi_open(). For those PnetCDF hints + * are not set in info, their default values are used. */ -void ncmpio_set_pnetcdf_hints(NC *ncp, - MPI_Info user_info, - MPI_Info info_used) +void ncmpio_hint_extract(NC *ncp, + MPI_Info info) { char value[MPI_MAX_INFO_VAL]; - int flag; + int flag, ival; + long long llval; + MPI_Offset var_align_val, hdr_align_val; - if (user_info == MPI_INFO_NULL) flag = 0; + assert(ncp != NULL); - /* Note info_used cannot be MPI_INFO_NULL, as it is returned from a call to - * MPI_File_get_info() - */ - assert(info_used != MPI_INFO_NULL); + ncp->info_v_align = -1; /* -1 indicates not set */ + ncp->info_r_align = -1; /* -1 indicates not set */ + + /* chunk size for reading header (set default before check hints) */ + ncp->hdr_chunk = PNC_HDR_READ_CHUNK_SIZE; + + /* chunk size for moving variables to higher offsets */ + ncp->data_chunk = -1; + + /* buffer to pack noncontiguous user buffers when calling wait() */ + ncp->ibuf_size = PNC_DEFAULT_IBUF_SIZE; + +#if PNETCDF_SUBFILING == 1 + ncp->subfile_mode = 0; + ncp->num_subfiles = 0; + ncp->comm_attr_sf.num_NUMAs = 0; + ncp->comm_attr_sf.NUMA_IDs = NULL; +#endif + + ncp->dims.hash_size = PNC_HSIZE_DIM; + ncp->vars.hash_size = PNC_HSIZE_VAR; + ncp->attrs.hash_size = PNC_HSIZE_GATTR; + ncp->hash_size_attr = PNC_HSIZE_VATTR; + + /* number of INA aggregators per compute node */ + ncp->num_aggrs_per_node = 0; + + ncp->driver = PNC_DRIVER_MPIIO; + + /* default I/O driver is GIO, if it is not disabled at configure time */ +#if PNETCDF_DRIVER_GIO == 1 + ncp->driver = PNC_DRIVER_GIO; +#endif + + if (info == MPI_INFO_NULL) return; /* nc_var_align_size, and r_align take effect when a file is created, or * opened and later adding more metadata or variable data */ - ncp->info_v_align = -1; /* -1 indicates not set */ - if (user_info != MPI_INFO_NULL) { - /* aligns starting file offsets of entire data section */ - MPI_Info_get(user_info, "nc_var_align_size", MPI_MAX_INFO_VAL-1, - value, &flag); - if (flag) { - errno = 0; /* errno must set to zero before calling strtoll */ - ncp->info_v_align = strtoll(value, NULL, 10); - if (errno != 0) ncp->info_v_align = -1; - else if (ncp->info_v_align < 0) ncp->info_v_align = -1; - } + /* aligns starting file offsets of entire data section */ + var_align_val = -1; + MPI_Info_get(info, "nc_var_align_size", MPI_MAX_INFO_VAL-1, value, &flag); + if (flag) { + errno = 0; /* errno must set to zero before calling strtoll */ + llval = strtoll(value, NULL, 10); + if (errno == 0 && llval >= 0) + var_align_val = llval; } - if (ncp->info_v_align == -1) - sprintf(value, "%d", FILE_ALIGNMENT_DEFAULT); - else - sprintf(value, OFFFMT, ncp->info_v_align); - MPI_Info_set(info_used, "nc_var_align_size", value); - if (user_info != MPI_INFO_NULL) { - /* Hint nc_header_align_size is now deprecated. But for backward - * compatibility, let's still check. - */ - MPI_Offset info_h_align = -1; - MPI_Info_get(user_info, "nc_header_align_size", MPI_MAX_INFO_VAL-1, - value, &flag); - if (flag) { - errno = 0; /* errno must set to zero before calling strtoll */ - info_h_align = strtoll(value, NULL, 10); - if (errno != 0) info_h_align = -1; - else if (info_h_align < 0) info_h_align = -1; - } - /* if nc_header_align_size is set and nc_var_align_size is not set, - * replace hint nc_var_align_size with the value of info_h_align. - */ - if (info_h_align >= 0 && ncp->info_v_align == -1) { - ncp->info_v_align = info_h_align; - sprintf(value, OFFFMT, ncp->info_v_align); - MPI_Info_set(info_used, "nc_var_align_size", value); + /* Hint nc_header_align_size is now deprecated. But for backward + * compatibility, let's still check. + */ + hdr_align_val = -1; + MPI_Info_get(info, "nc_header_align_size", MPI_MAX_INFO_VAL-1, + value, &flag); + if (flag) { + errno = 0; /* errno must set to zero before calling strtoll */ + llval = strtoll(value, NULL, 10); + if (errno == 0 && llval >= 0) { + /* if nc_header_align_size is set and nc_var_align_size is not set, + * replace hint nc_var_align_size with the value of info_h_align. + */ + if (llval >= 0 && ncp->info_v_align == -1) + hdr_align_val = llval; } } - ncp->info_r_align = -1; - if (user_info != MPI_INFO_NULL) { - /* aligns starting file offset of the record variable section */ - MPI_Info_get(user_info, "nc_record_align_size", MPI_MAX_INFO_VAL-1, - value, &flag); - if (flag) { - errno = 0; /* errno must set to zero before calling strtoll */ - ncp->info_r_align = strtoll(value, NULL, 10); - if (errno != 0) ncp->info_r_align = -1; - else if (ncp->info_r_align < 0) ncp->info_r_align = -1; - } + /* hint nc_var_align_size supersedes nc_header_align_size */ + if (var_align_val > 0) + ncp->info_v_align = var_align_val; + else if (hdr_align_val > 0) + ncp->info_v_align = hdr_align_val; + + /* aligns starting file offset of the record variable section */ + MPI_Info_get(info, "nc_record_align_size", MPI_MAX_INFO_VAL-1, + value, &flag); + if (flag) { + errno = 0; /* errno must set to zero before calling strtoll */ + llval = strtoll(value, NULL, 10); + if (errno == 0 && llval >= 0) + ncp->info_r_align = llval; } - if (ncp->info_r_align == -1) - sprintf(value, "%d", FILE_ALIGNMENT_DEFAULT); - else - sprintf(value, OFFFMT, ncp->info_r_align); - MPI_Info_set(info_used, "nc_record_align_size", value); - - ncp->chunk = PNC_DEFAULT_CHUNKSIZE; - if (user_info != MPI_INFO_NULL) { - /* header reading chunk size */ - MPI_Info_get(user_info, "nc_header_read_chunk_size", MPI_MAX_INFO_VAL-1, - value, &flag); - if (flag) { - int chunk; - errno = 0; /* errno must set to zero before calling strtoll */ - chunk = atoi(value); - if (errno != 0) ncp->chunk = 0; - else if (ncp->chunk < 0) - ncp->chunk = 0; - else if (chunk > NC_MAX_INT) /* limit to NC_MAX_INT */ - ncp->chunk = NC_MAX_INT; + + /* header reading chunk size */ + MPI_Info_get(info, "nc_header_read_chunk_size", MPI_MAX_INFO_VAL-1, + value, &flag); + if (flag) { + errno = 0; /* errno must set to zero before calling strtoll */ + llval = strtoll(value, NULL, 10); + if (errno == 0) { + if (llval < 0) + ncp->hdr_chunk = PNC_HDR_READ_CHUNK_SIZE; + else if (llval > NC_MAX_INT) /* limit to NC_MAX_INT */ + ncp->hdr_chunk = NC_MAX_INT; + else if (llval < 48) + /* minimum file sizes: 32 bytes for CDF-1 and 2, 48 for CDF5 */ + ncp->hdr_chunk = 48; + else + ncp->hdr_chunk = (int)llval; + + /* CDF-5's minimum header size is 4 bytes more than CDF-1 2's */ + ncp->hdr_chunk = PNETCDF_RNDUP(MAX(MIN_NC_XSZ+4, ncp->hdr_chunk), + X_ALIGN); } } - sprintf(value, "%d", ncp->chunk); - MPI_Info_set(info_used, "nc_header_read_chunk_size", value); - strcpy(value, "auto"); - if (user_info != MPI_INFO_NULL) { - /* setting in-place byte swap (matters only for Little Endian) */ - MPI_Info_get(user_info, "nc_in_place_swap", MPI_MAX_INFO_VAL-1, value, &flag); - if (flag) { - if (strcasecmp(value, "enable") == 0) { - fClr(ncp->flags, NC_MODE_SWAP_OFF); - fSet(ncp->flags, NC_MODE_SWAP_ON); - } - else if (strcasecmp(value, "disable") == 0) { - fClr(ncp->flags, NC_MODE_SWAP_ON); - fSet(ncp->flags, NC_MODE_SWAP_OFF); - } - else if (strcasecmp(value, "auto") == 0) { - fClr(ncp->flags, NC_MODE_SWAP_ON); - fClr(ncp->flags, NC_MODE_SWAP_OFF); - } + /* setting in-place byte swap (matters only for Little Endian) */ + MPI_Info_get(info, "nc_in_place_swap", MPI_MAX_INFO_VAL-1, value, &flag); + if (flag) { + if (strcasecmp(value, "enable") == 0) { + fClr(ncp->flags, NC_MODE_SWAP_OFF); + fSet(ncp->flags, NC_MODE_SWAP_ON); + } + else if (strcasecmp(value, "disable") == 0) { + fClr(ncp->flags, NC_MODE_SWAP_ON); + fSet(ncp->flags, NC_MODE_SWAP_OFF); + } + else if (strcasecmp(value, "auto") == 0) { + fClr(ncp->flags, NC_MODE_SWAP_ON); + fClr(ncp->flags, NC_MODE_SWAP_OFF); } } - MPI_Info_set(info_used, "nc_in_place_swap", value); - if (user_info != MPI_INFO_NULL) { - /* temporal buffer size used to pack noncontiguous aggregated user - * buffers when calling ncmpi_wait/wait_all, Default 16 MiB - */ - MPI_Info_get(user_info, "nc_ibuf_size", MPI_MAX_INFO_VAL-1, value, - &flag); - if (flag) { - MPI_Offset ibuf_size; - errno = 0; /* errno must set to zero before calling strtoll */ - ibuf_size = strtoll(value, NULL, 10); - if (errno == 0 && ibuf_size >= 0) ncp->ibuf_size = ibuf_size; - } + /* Temporal buffer size used to pack non-contiguous aggregated user buffers + * when calling ncmpi_wait/wait_all. Default PNC_DEFAULT_IBUF_SIZE. + */ + MPI_Info_get(info, "nc_ibuf_size", MPI_MAX_INFO_VAL-1, value, &flag); + if (flag) { + errno = 0; /* errno must set to zero before calling strtoll */ + llval = strtoll(value, NULL, 10); + if (errno == 0 && llval >= 0) + ncp->ibuf_size = llval; } - sprintf(value, OFFFMT, ncp->ibuf_size); - MPI_Info_set(info_used, "nc_ibuf_size", value); -#ifdef ENABLE_SUBFILING - ncp->subfile_mode = 0; - if (user_info != MPI_INFO_NULL) { - MPI_Info_get(user_info, "pnetcdf_subfiling", MPI_MAX_INFO_VAL-1, - value, &flag); - if (flag) { - if (strcasecmp(value, "enable") == 0) - ncp->subfile_mode = 1; +#if PNETCDF_SUBFILING == 1 + MPI_Info_get(info, "pnetcdf_subfiling", MPI_MAX_INFO_VAL-1, value, &flag); + if (flag) { + if (strcasecmp(value, "enable") == 0) + ncp->subfile_mode = 1; + else { + ncp->subfile_mode = 0; + ncp->num_subfiles = 0; } } - if (ncp->subfile_mode) - MPI_Info_set(info_used, "pnetcdf_subfiling", "enable"); - else - MPI_Info_set(info_used, "pnetcdf_subfiling", "disable"); - ncp->num_subfiles = 0; - if (user_info != MPI_INFO_NULL) { - MPI_Info_get(user_info, "nc_num_subfiles", MPI_MAX_INFO_VAL-1, - value, &flag); + if (ncp->subfile_mode == 1) { + MPI_Info_get(info, "nc_num_subfiles", MPI_MAX_INFO_VAL-1, value, &flag); if (flag) { - errno = 0; - ncp->num_subfiles = atoi(value); - if (errno != 0) ncp->num_subfiles = 0; - else if (ncp->num_subfiles < 0) ncp->num_subfiles = 0; + errno = 0; /* errno must set to zero before calling atoi */ + ival = atoi(value); + if (errno == 0 && ival >= 0) + ncp->num_subfiles = ival; } } - sprintf(value, "%d", ncp->num_subfiles); - MPI_Info_set(info_used, "nc_num_subfiles", value); - - if (ncp->subfile_mode == 0) ncp->num_subfiles = 0; -#else - MPI_Info_set(info_used, "pnetcdf_subfiling", "disable"); - MPI_Info_set(info_used, "nc_num_subfiles", "0"); #endif - if (user_info != MPI_INFO_NULL) { - /* If romio_no_indep_rw is set to true, let all processes participate - * the read/write file header using MPI collective APIs, where only - * rank 0 has non-zero request count. - */ - MPI_Info_get(user_info, "romio_no_indep_rw", MPI_MAX_INFO_VAL-1, - value, &flag); - if (flag) { - if (strcasecmp(value, "true") == 0) - fSet((ncp)->flags, NC_HCOLL); - } + /* Hash table size for dimensions */ + MPI_Info_get(info, "nc_hash_size_dim", MPI_MAX_INFO_VAL-1, value, &flag); + if (flag) { + errno = 0; /* errno must set to zero before calling atoi */ + ival = atoi(value); + if (errno == 0 && ival >= 0) + ncp->dims.hash_size = ival; } - ncp->dims.hash_size = PNC_HSIZE_DIM; - if (user_info != MPI_INFO_NULL) { - /* Hash table size for dimensions */ - MPI_Info_get(user_info, "nc_hash_size_dim", MPI_MAX_INFO_VAL-1, - value, &flag); - if (flag) { - errno = 0; /* errno must set to zero before calling atoi */ - ncp->dims.hash_size = atoi(value); - if (errno != 0 || ncp->dims.hash_size < 0) - ncp->dims.hash_size = PNC_HSIZE_DIM; - } + /* Hash table size for variables */ + MPI_Info_get(info, "nc_hash_size_var", MPI_MAX_INFO_VAL-1, value, &flag); + if (flag) { + errno = 0; /* errno must set to zero before calling atoi */ + ival = atoi(value); + if (errno == 0 && ival >= 0) + ncp->vars.hash_size = ival; } - sprintf(value, "%d", ncp->dims.hash_size); - MPI_Info_set(info_used, "nc_hash_size_dim", value); - ncp->vars.hash_size = PNC_HSIZE_VAR; - if (user_info != MPI_INFO_NULL) { - /* Hash table size for variables */ - MPI_Info_get(user_info, "nc_hash_size_var", MPI_MAX_INFO_VAL-1, - value, &flag); - if (flag) { - errno = 0; /* errno must set to zero before calling atoi */ - ncp->vars.hash_size = atoi(value); - if (errno != 0 || ncp->vars.hash_size < 0) - ncp->vars.hash_size = PNC_HSIZE_VAR; - } + /* Hash table size for global attributes */ + MPI_Info_get(info, "nc_hash_size_gattr", MPI_MAX_INFO_VAL-1, value, &flag); + if (flag) { + errno = 0; /* errno must set to zero before calling atoi */ + ival = atoi(value); + if (errno == 0 && ival >= 0) + ncp->attrs.hash_size = ival; } - sprintf(value, "%d", ncp->vars.hash_size); - MPI_Info_set(info_used, "nc_hash_size_var", value); - ncp->attrs.hash_size = PNC_HSIZE_GATTR; - if (user_info != MPI_INFO_NULL) { - /* Hash table size for global attributes */ - MPI_Info_get(user_info, "nc_hash_size_gattr", MPI_MAX_INFO_VAL-1, - value, &flag); + /* Hash table size for non-global attributes */ + MPI_Info_get(info, "nc_hash_size_vattr", MPI_MAX_INFO_VAL-1, value, &flag); + if (flag) { + errno = 0; /* errno must set to zero before calling atoi */ + ival = atoi(value); + if (errno == 0 && ival >= 0) + ncp->hash_size_attr = ival; + } + + /* Number of INA aggregators per compute node. */ + if (ncp->nprocs > 1) { + MPI_Info_get(info, "nc_num_aggrs_per_node", MPI_MAX_INFO_VAL-1, value, + &flag); if (flag) { errno = 0; /* errno must set to zero before calling atoi */ - ncp->attrs.hash_size = atoi(value); - if (errno != 0 || ncp->attrs.hash_size < 0) - ncp->attrs.hash_size = PNC_HSIZE_GATTR; + ival = atoi(value); + if (errno == 0 && ival >= 0) + ncp->num_aggrs_per_node = ival; } } - sprintf(value, "%d", ncp->attrs.hash_size); - MPI_Info_set(info_used, "nc_hash_size_gattr", value); - ncp->hash_size_attr = PNC_HSIZE_VATTR; - if (user_info != MPI_INFO_NULL) { - /* Hash table size for non-global attributes */ - MPI_Info_get(user_info, "nc_hash_size_vattr", MPI_MAX_INFO_VAL-1, - value, &flag); - if (flag) { - errno = 0; /* errno must set to zero before calling atoi */ - ncp->hash_size_attr = atoi(value); - if (errno != 0 || ncp->hash_size_attr < 0) - ncp->hash_size_attr = PNC_HSIZE_VATTR; + /* If user explicitly want to use MPI-IO instead of GIO driver, then set + * PnetCDF I/O hint "nc_driver" to "mpiio". + */ + MPI_Info_get(info, "nc_driver", MPI_MAX_INFO_VAL-1, value, &flag); + if (flag) { + if (strcasecmp(value, "gio") == 0) + ncp->driver = PNC_DRIVER_GIO; + else if (strcasecmp(value, "mpiio") == 0) + ncp->driver = PNC_DRIVER_MPIIO; + } + + /* Data movement chunk size when variables need to be moved to higher file + * offsets. + */ + MPI_Info_get(info, "nc_data_move_chunk_size", MPI_MAX_INFO_VAL-1, value, + &flag); + if (flag) { + errno = 0; /* errno must set to zero before calling strtoll */ + llval = strtoll(value, NULL, 10); + if (errno == 0) { + if (llval < 0) + ncp->data_chunk = -1; + else if (llval > NC_MAX_INT) /* limit to NC_MAX_INT */ + ncp->data_chunk = NC_MAX_INT; + else + ncp->data_chunk = (int)llval; } } - sprintf(value, "%d", ncp->hash_size_attr); - MPI_Info_set(info_used, "nc_hash_size_vattr", value); - ncp->num_aggrs_per_node = 0; - if (user_info != MPI_INFO_NULL) { - /* Hash table size for non-global attributes */ - MPI_Info_get(user_info, "nc_num_aggrs_per_node", MPI_MAX_INFO_VAL-1, - value, &flag); - if (flag) { - errno = 0; /* errno must set to zero before calling atoi */ - ncp->num_aggrs_per_node = atoi(value); - if (errno != 0 || ncp->num_aggrs_per_node < 0) - ncp->num_aggrs_per_node = 0; + /* When creating a file, inherit file striping from the parent folder or + * let PnetCDF to decide. + */ + ncp->file_striping = PNCIO_STRIPING_AUTO; + MPI_Info_get(info, "nc_file_striping", MPI_MAX_INFO_VAL-1, value, &flag); + if (flag && strcasecmp(value, "inherit") == 0) + ncp->file_striping = PNCIO_STRIPING_INHERIT; + + /* Check and set hint NUMA_ID, which is set by the dispatcher. */ + MPI_Info_get(info, "NUMA_ID", MPI_MAX_INFO_VAL-1, value, &flag); + if (flag) { + errno = 0; /* errno must set to zero before calling atoi */ + ival = atoi(value); + if (errno == 0 && ival >= 0) + ncp->NUMA_ID = ival; + } +} + +/*----< ncmpio_hint_set() >--------------------------------------------------*/ +/* Insert PnetCDF hints into info. Argument info is the info object returned + * from an earlier call to MPI_File_get_info(). This subroutine is necessary, + * because the underneath MPI-IO may discard hints it does not recognize, which + * include all PnetCDF hints. PnetCDF hints need to be added to info, so users + * can inquire them. + */ +void ncmpio_hint_set(NC *ncp, + MPI_Info info) +{ + char int_str[MAX_INT_LEN]; + + assert(ncp != NULL); + assert(info != MPI_INFO_NULL); + + /* nc_var_align_size, and r_align take effect when a file is created, or + * opened and later adding more metadata or variable data + */ + + /* aligns starting file offsets of entire data section */ + if (ncp->info_v_align != -1) { + snprintf(int_str, MAX_INT_LEN, OFFFMT, ncp->info_v_align); + MPI_Info_set(info, "nc_var_align_size", int_str); + } + + /* aligns starting file offset of the record variable section */ + if (ncp->info_r_align != -1) { + snprintf(int_str, MAX_INT_LEN, OFFFMT, ncp->info_r_align); + MPI_Info_set(info, "nc_record_align_size", int_str); + } + + /* header reading chunk size */ + snprintf(int_str, MAX_INT_LEN, "%d", ncp->hdr_chunk); + MPI_Info_set(info, "nc_header_read_chunk_size", int_str); + + /* variable movement chunk size */ + snprintf(int_str, MAX_INT_LEN, "%d", ncp->data_chunk); + MPI_Info_set(info, "nc_data_move_chunk_size", int_str); + + /* setting in-place byte swap (matters only for Little Endian) */ + int swap_on = fIsSet(ncp->flags, NC_MODE_SWAP_ON); + int swap_off = fIsSet(ncp->flags, NC_MODE_SWAP_OFF); + if (!swap_on && !swap_off) + MPI_Info_set(info, "nc_in_place_swap", "auto"); + else if (swap_on) + MPI_Info_set(info, "nc_in_place_swap", "enable"); + else + MPI_Info_set(info, "nc_in_place_swap", "disable"); + + /* Temporal buffer size used to pack non-contiguous aggregated user buffers + * when calling ncmpi_wait/wait_all. Default PNC_DEFAULT_IBUF_SIZE. + */ + snprintf(int_str, MAX_INT_LEN, OFFFMT, ncp->ibuf_size); + MPI_Info_set(info, "nc_ibuf_size", int_str); + +#if PNETCDF_SUBFILING == 1 + if (ncp->subfile_mode) + MPI_Info_set(info, "pnetcdf_subfiling", "enable"); + else + MPI_Info_set(info, "pnetcdf_subfiling", "disable"); + + snprintf(int_str, MAX_INT_LEN, "%d", ncp->num_subfiles); + MPI_Info_set(info, "nc_num_subfiles", int_str); +#endif + + /* Hash table size for dimensions */ + snprintf(int_str, MAX_INT_LEN, "%d", ncp->dims.hash_size); + MPI_Info_set(info, "nc_hash_size_dim", int_str); + + /* Hash table size for variables */ + snprintf(int_str, MAX_INT_LEN, "%d", ncp->vars.hash_size); + MPI_Info_set(info, "nc_hash_size_var", int_str); + + /* Hash table size for global attributes */ + snprintf(int_str, MAX_INT_LEN, "%d", ncp->attrs.hash_size); + MPI_Info_set(info, "nc_hash_size_gattr", int_str); + + /* Hash table size for non-global attributes */ + snprintf(int_str, MAX_INT_LEN, "%d", ncp->hash_size_attr); + MPI_Info_set(info, "nc_hash_size_vattr", int_str); + + /* Whether to use MPI-IO or GIO driver. */ + if (ncp->driver == PNC_DRIVER_MPIIO) + MPI_Info_set(info, "nc_driver", "mpiio"); +#if PNETCDF_DRIVER_GIO == 1 + else if (ncp->driver == PNC_DRIVER_GIO) + MPI_Info_set(info, "nc_driver", "gio"); +#endif + + if (ncp->num_aggrs_per_node > 0) { + /* Number of INA aggregators per compute node. */ + snprintf(int_str, MAX_INT_LEN, "%d", ncp->num_aggrs_per_node); + MPI_Info_set(info, "nc_num_aggrs_per_node", int_str); + + /* Add hint "ina_ranks", list of INA aggregators' rank IDs. + * Note this hint is just informative, not user changeable. + */ + if (ncp->comm_attr.ina_ranks != NULL) { + char value[MPI_MAX_INFO_VAL]; + int i; + + snprintf(value, MAX_INT_LEN, "%d", ncp->comm_attr.ina_ranks[0]); + for (i=1; icomm_attr.num_ina_aggrs; i++) { + snprintf(int_str, sizeof(int_str), " %d", ncp->comm_attr.ina_ranks[i]); + if (strlen(value) + strlen(int_str) >= MPI_MAX_INFO_VAL-5) { + strcat(value, " ..."); + break; + } + strcat(value, int_str); + } + MPI_Info_set(info, "nc_ina_node_list", value); } } - sprintf(value, "%d", ncp->num_aggrs_per_node); - MPI_Info_set(info_used, "nc_num_aggrs_per_node", value); + else /* Update hint "num_aggrs_per_node" to indicate disabled. */ + MPI_Info_set(info, "nc_num_aggrs_per_node", "0"); + + /* When creating a file, inherit file striping from the parent folder or + * let PnetCDF to decide. + */ + if (ncp->file_striping == PNCIO_STRIPING_AUTO) + MPI_Info_set(info, "nc_file_striping", "auto"); + else + MPI_Info_set(info, "nc_file_striping", "inherit"); + + if (ncp->NUMA_ID >= 0) { + /* This rank's NUMA_ID, will be used by GIO. */ + snprintf(int_str, MAX_INT_LEN, "%d", ncp->NUMA_ID); + MPI_Info_set(info, "NUMA_ID", int_str); + } } -/*----< ncmpio_first_offset() >-----------------------------------------------*/ +/*----< ncmpio_first_offset() >----------------------------------------------*/ /* Returns the file offset of the first variable element accessed by this * request. Note zero-length request should never call this subroutine. */ @@ -474,8 +616,8 @@ ncmpio_pack_xbuf(int fmt, /* NC_FORMAT_CDF2 NC_FORMAT_CDF5 etc. */ /* check byte size of buf (internal representation) */ ibuf_size = nelems * el_size; - /* Step 1: if buftype is not contiguous, i.e. a noncontiguous MPI - * derived datatype, pack buf into a contiguous buffer, lbuf, + /* Step 1: if buftype is not contiguous, i.e. a noncontiguous MPI derived + * datatype, pack buf into a contiguous buffer, lbuf, */ if (!buftype_is_contig) { /* buftype is not contiguous */ if (imaptype == MPI_DATATYPE_NULL && !need_convert) @@ -494,7 +636,8 @@ ncmpio_pack_xbuf(int fmt, /* NC_FORMAT_CDF2 NC_FORMAT_CDF5 etc. */ #ifdef HAVE_MPI_LARGE_COUNT MPI_Count position = 0; mpireturn = MPI_Pack_c(buf, (MPI_Count)bufcount, buftype, lbuf, - (MPI_Count)ibuf_size, &position, MPI_COMM_SELF); + (MPI_Count)ibuf_size, &position, + MPI_COMM_SELF); if (mpireturn != MPI_SUCCESS) return ncmpii_error_mpi2nc(mpireturn, "MPI_Pack_c"); #else @@ -503,8 +646,8 @@ ncmpio_pack_xbuf(int fmt, /* NC_FORMAT_CDF2 NC_FORMAT_CDF5 etc. */ if (free_lbuf) NCI_Free(lbuf); DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) } - mpireturn = MPI_Pack(buf, (int)bufcount, buftype, lbuf, (int)ibuf_size, - &position, MPI_COMM_SELF); + mpireturn = MPI_Pack(buf, (int)bufcount, buftype, lbuf, + (int)ibuf_size, &position, MPI_COMM_SELF); if (mpireturn != MPI_SUCCESS) return ncmpii_error_mpi2nc(mpireturn, "MPI_Pack"); #endif @@ -556,8 +699,8 @@ ncmpio_pack_xbuf(int fmt, /* NC_FORMAT_CDF2 NC_FORMAT_CDF5 etc. */ else /* not a true varm call: reuse lbuf */ cbuf = lbuf; - /* Step 3: type-convert and byte-swap cbuf to xbuf, and xbuf will be - * used in MPI write function to write to file + /* Step 3: type-convert and byte-swap cbuf to xbuf, and xbuf will be used + * in MPI write function to write to file */ if (need_convert) { /* user buf type does not match nc var type defined in file */ @@ -604,12 +747,12 @@ ncmpio_pack_xbuf(int fmt, /* NC_FORMAT_CDF2 NC_FORMAT_CDF5 etc. */ break; } /* The only error codes returned from the above switch block are - * NC_EBADTYPE or NC_ERANGE. Bad varp->xtype and itype have been sanity - * checked at the dispatchers, so NC_EBADTYPE is not possible. Thus, - * the only possible error is NC_ERANGE. NC_ERANGE can be caused by - * one or more elements of buf that is out of range representable by - * the external data type, it is not considered a fatal error. This - * request must continue to finish. + * NC_EBADTYPE or NC_ERANGE. Bad varp->xtype and itype have been sanity + * checked at the dispatchers, so NC_EBADTYPE is not possible. Thus, + * the only possible error is NC_ERANGE. NC_ERANGE can be caused by + * one or more elements of buf that is out of range representable by + * the external data type, it is not considered a fatal error. This + * request must continue to finish. */ if (free_cbuf) NCI_Free(cbuf); if (free_lbuf) NCI_Free(lbuf); @@ -730,12 +873,12 @@ ncmpio_unpack_xbuf(int fmt, /* NC_FORMAT_CDF2 NC_FORMAT_CDF5 etc. */ break; } /* The only error codes returned from the above switch block are - * NC_EBADTYPE or NC_ERANGE. Bad varp->xtype and itype have been sanity - * checked at the dispatchers, so NC_EBADTYPE is not possible. Thus, - * the only possible error is NC_ERANGE. NC_ERANGE can be caused by - * one or more elements of buf that is out of range representable by - * the external data type, it is not considered a fatal error. This - * request must continue to finish. + * NC_EBADTYPE or NC_ERANGE. Bad varp->xtype and itype have been sanity + * checked at the dispatchers, so NC_EBADTYPE is not possible. Thus, + * the only possible error is NC_ERANGE. NC_ERANGE can be caused by + * one or more elements of buf that is out of range representable by + * the external data type, it is not considered a fatal error. This + * request must continue to finish. */ } else { @@ -785,30 +928,36 @@ ncmpio_unpack_xbuf(int fmt, /* NC_FORMAT_CDF2 NC_FORMAT_CDF5 etc. */ MPI_Type_free(&imaptype); } - /* unpacked lbuf into buf based on buftype -----------------------------*/ - if (!buftype_is_contig && lbuf != buf) { - /* no need unpack when buftype is used in MPI_File_read (lbuf == buf) */ + /* Unpacked lbuf into buf based on buftype. Note no need to unpack when + * buftype is used in MPI_File_read, i.e. lbuf == buf. + */ + if (lbuf != buf) { + if (buftype_is_contig) + memcpy(buf, lbuf, ibuf_size); + else { /* buftye is not contiguous */ #ifdef HAVE_MPI_LARGE_COUNT - MPI_Count position = 0; - mpireturn = MPI_Unpack_c(lbuf, (MPI_Count)ibuf_size, &position, buf, - (MPI_Count)bufcount, buftype, MPI_COMM_SELF); - if (mpireturn != MPI_SUCCESS) - return ncmpii_error_mpi2nc(mpireturn, "MPI_Unpack_c"); -#else - if (bufcount > NC_MAX_INT) { - if (err == NC_NOERR) - DEBUG_ASSIGN_ERROR(err, NC_EINTOVERFLOW) - } - else { - int position = 0; - if (ibuf_size > NC_MAX_INT) - DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) - mpireturn = MPI_Unpack(lbuf, (int)ibuf_size, &position, buf, - (int)bufcount, buftype, MPI_COMM_SELF); + MPI_Count position = 0; + mpireturn = MPI_Unpack_c(lbuf, (MPI_Count)ibuf_size, &position, + buf, (MPI_Count)bufcount, buftype, + MPI_COMM_SELF); if (mpireturn != MPI_SUCCESS) - return ncmpii_error_mpi2nc(mpireturn, "MPI_Unpack"); - } + return ncmpii_error_mpi2nc(mpireturn, "MPI_Unpack_c"); +#else + if (bufcount > NC_MAX_INT) { + if (err == NC_NOERR) + DEBUG_ASSIGN_ERROR(err, NC_EINTOVERFLOW) + } + else { + int position = 0; + if (ibuf_size > NC_MAX_INT) + DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) + mpireturn = MPI_Unpack(lbuf, (int)ibuf_size, &position, buf, + (int)bufcount, buftype, MPI_COMM_SELF); + if (mpireturn != MPI_SUCCESS) + return ncmpii_error_mpi2nc(mpireturn, "MPI_Unpack"); + } #endif + } } if (free_cbuf) NCI_Free(cbuf); if (free_lbuf) NCI_Free(lbuf); @@ -816,3 +965,273 @@ ncmpio_unpack_xbuf(int fmt, /* NC_FORMAT_CDF2 NC_FORMAT_CDF5 etc. */ return err; } +/*----< ncmpio_calc_off() >--------------------------------------------------*/ +/* Returns the starting file offset of a subarray request. + * Note zero-length request should never call this subroutine. + * Only a single offset-length pair will call this subroutine. + */ +int +ncmpio_calc_off(const NC *ncp, + const NC_var *varp, + const MPI_Offset *start, /* [varp->ndims] */ + MPI_Offset *offset) /* OUT: start offset */ +{ + int i, ndims = varp->ndims; /* number of dimensions of this variable */ + + /* + * varp->dsizes[] is computed from right to left product of shape + * For example, a 3D array of size 5x4x3 in C order, + * For fixed-size variable: dsizes[0]=60 dsizes[1]=12 dsizes[2]=3 + * For record variable: dsizes[0]=12 dsizes[1]=12 dsizes[2]=3 + */ + if (IS_RECVAR(varp)) { + *offset = 0; + if (ndims > 1) { + /* start from the least significant dimension */ + *offset = start[ndims-1]; + /* the remaining dimensions */ + for (i=ndims-2; i>0; i--) + *offset += start[i]*varp->dsizes[i+1]; + } + *offset *= varp->xsz; /* offset in bytes */ + } + else { + /* first handle the least significant dimension */ + *offset = start[ndims-1]; + /* remaining dimensions till the most significant dimension */ + for (i=ndims-2; i>=0; i--) + *offset += start[i] * varp->dsizes[i+1]; + *offset *= varp->xsz; /* offset in bytes */ + } + + return NC_NOERR; +} + +/*----< ncmpio_calc_start_end() >--------------------------------------------*/ +/* Returns the file offsets of access range of this request: starting file + * offset and end offset (exclusive). + * Note zero-length request should never call this subroutine. + */ +int +ncmpio_calc_start_end(const NC *ncp, + const NC_var *varp, + const MPI_Offset *start, /* [varp->ndims] */ + const MPI_Offset *count, /* [varp->ndims] */ + const MPI_Offset *stride, /* [varp->ndims] */ + MPI_Offset *start_off, /* OUT: start offset */ + MPI_Offset *end_off) /* OUT: end offset */ +{ + int i, ndims = varp->ndims; /* number of dimensions of this variable */ + + /* + * varp->dsizes[] is computed from right to left product of shape + * For example, a 3D array of size 5x4x3 in C order, + * For fixed-size variable: dsizes[0]=60 dsizes[1]=12 dsizes[2]=3 + * For record variable: dsizes[0]=12 dsizes[1]=12 dsizes[2]=3 + */ + if (IS_RECVAR(varp)) { + *start_off = 0; + *end_off = 0; + if (stride == NULL) { + if (ndims > 1) { + /* least significant dimension */ + *start_off = start[ndims-1]; + *end_off = start[ndims-1]+(count[ndims-1]-1); + /* the remaining dimensions */ + for (i=ndims-2; i>0; i--) { + *start_off += start[i]*varp->dsizes[i+1]; + *end_off += (start[i]+(count[i]-1))*varp->dsizes[i+1]; + } + } + *start_off *= varp->xsz; /* offset in bytes */ + *end_off *= varp->xsz; + /* handle the unlimited, most significant dimension */ + *start_off += start[0] * ncp->recsize; + *end_off += (start[0]+(count[0]-1)) * ncp->recsize; + } + else { + if (ndims > 1) { + /* least significant dimension */ + *start_off = start[ndims-1]; + *end_off = start[ndims-1]+(count[ndims-1]-1)*stride[ndims-1]; + /* the remaining dimensions */ + for (i=ndims-2; i>0; i--) { + *start_off += start[i]*varp->dsizes[i+1]; + *end_off += (start[i]+(count[i]-1)*stride[i]) * + varp->dsizes[i+1]; + } + } + *start_off *= varp->xsz; /* offset in bytes */ + *end_off *= varp->xsz; + /* handle the unlimited, most significant dimension */ + *start_off += start[0] * ncp->recsize; + *end_off += (start[0]+(count[0]-1)*stride[0]) * ncp->recsize; + } + } + else { + if (stride == NULL) { + /* first handle the least significant dimension */ + *start_off = start[ndims-1]; + *end_off = start[ndims-1] + (count[ndims-1]-1); + /* remaining dimensions till the most significant dimension */ + for (i=ndims-2; i>=0; i--) { + *start_off += start[i] * varp->dsizes[i+1]; + *end_off += (start[i]+(count[i]-1)) * varp->dsizes[i+1]; + } + } + else { + /* first handle the least significant dimension */ + *start_off = start[ndims-1]; + *end_off = start[ndims-1]+(count[ndims-1]-1)*stride[ndims-1]; + /* remaining dimensions till the most significant dimension */ + for (i=ndims-2; i>=0; i--) { + *start_off += start[i] * varp->dsizes[i+1]; + *end_off += (start[i]+(count[i]-1)*stride[i])*varp->dsizes[i+1]; + } + } + *start_off *= varp->xsz; /* offset in bytes */ + *end_off *= varp->xsz; + } + *start_off += varp->begin; /* beginning file offset of this variable */ + *end_off += varp->begin + varp->xsz; + + return NC_NOERR; +} + +/*----< ncmpio_type_contiguous() >-------------------------------------------*/ +int ncmpio_type_contiguous(MPI_Offset count, + MPI_Datatype *newType) +{ + char *mpi_name; + int err; + + *newType = MPI_BYTE; + + if (count == 0) return NC_NOERR; + +#ifdef HAVE_MPI_LARGE_COUNT + err = MPI_Type_contiguous_c((MPI_Count)count, MPI_BYTE, newType); + mpi_name = "MPI_Type_contiguous_c"; +#else + if (count > INT_MAX) { + *newType = MPI_DATATYPE_NULL; + return NC_EINTOVERFLOW; + } + + err = MPI_Type_contiguous((int)count, MPI_BYTE, newType); + mpi_name = "MPI_Type_contiguous"; +#endif + + if (err != MPI_SUCCESS) { + *newType = MPI_DATATYPE_NULL; + return ncmpii_error_mpi2nc(err, mpi_name); + } + else { + err = MPI_Type_commit(newType); + if (err != MPI_SUCCESS) { + MPI_Type_free(newType); + *newType = MPI_DATATYPE_NULL; + return ncmpii_error_mpi2nc(err,"MPI_Type_commit"); + } + } + + return NC_NOERR; +} + +/*----< ncmpio_type_create_hindexed() >--------------------------------------*/ +int ncmpio_type_create_hindexed(MPI_Offset count, + MPI_Offset *off, + MPI_Offset *len, + MPI_Datatype *newType) +{ + char *mpi_name; + int err; + + *newType = MPI_BYTE; + + if (count == 0) return NC_NOERR; + +#ifdef HAVE_MPI_LARGE_COUNT + /* Note argument array_of_displacements[] in MPI_Type_create_hindexed_c() + * is of type 'MPI_Count'. + */ + MPI_Count *disp, *blklen; +#if SIZEOF_MPI_OFFSET == SIZEOF_MPI_COUNT + disp = off; + blklen = len; +#else + MPI_Count j; + disp = (MPI_Count*) NCI_Malloc(sizeof(MPI_Count) * count); + blklen = (MPI_Count*) NCI_Malloc(sizeof(MPI_Count) * count); + for (j=0; j INT_MAX) { + /* count argument in MPI_Type_create_hindexed() is of type int */ + *newType = MPI_DATATYPE_NULL; + DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) + } + +#if SIZEOF_MPI_AINT == SIZEOF_MPI_OFFSET + disp = (MPI_Aint*) off; +#else + MPI_Offset j; + disp = (MPI_Aint*) NCI_Malloc(sizeof(MPI_Aint) * count); + for (j=0; j INT_MAX) { + *newType = MPI_DATATYPE_NULL; + DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) + } + blklen[k] = (int)len[k]; + } +#endif + + err = MPI_Type_create_hindexed((int)count, blklen, disp, MPI_BYTE, newType); + mpi_name = "MPI_Type_create_hindexed"; + +#if SIZEOF_MPI_AINT != SIZEOF_MPI_OFFSET + NCI_Free(disp); +#endif +#if SIZEOF_INT != SIZEOF_MPI_OFFSET + NCI_Free(blklen); +#endif +#endif + + if (err != MPI_SUCCESS) { + *newType = MPI_DATATYPE_NULL; + return ncmpii_error_mpi2nc(err, mpi_name); + } + else { + err = MPI_Type_commit(newType); + if (err != MPI_SUCCESS) { + MPI_Type_free(newType); + *newType = MPI_DATATYPE_NULL; + return ncmpii_error_mpi2nc(err,"MPI_Type_commit"); + } + } + + return NC_NOERR; +} + diff --git a/src/drivers/ncmpio/ncmpio_var.c b/src/drivers/ncmpio/ncmpio_var.c index df66a52c0c..95579aea3b 100644 --- a/src/drivers/ncmpio/ncmpio_var.c +++ b/src/drivers/ncmpio/ncmpio_var.c @@ -41,7 +41,7 @@ ncmpio_free_NC_var(NC_var *varp) ncmpio_free_NC_attrarray(&varp->attrs); NCI_Free(varp->name); -#ifdef ENABLE_SUBFILING +#if PNETCDF_SUBFILING == 1 if (varp->num_subfiles > 1) /* deallocate it */ NCI_Free(varp->dimids_org); #endif @@ -415,7 +415,7 @@ ncmpio_def_var(void *ncdp, ncp->vars.ndefined++; err_check: - if (ncp->safe_mode && ncp->nprocs > 1) { + if (fIsSet(ncp->flags, NC_MODE_SAFE) && ncp->nprocs > 1) { int minE, mpireturn; /* First check the error code across processes */ @@ -524,7 +524,7 @@ ncmpio_inq_var(void *ncdp, *xtypep = varp->xtype; if (ndimsp != NULL) { -#ifdef ENABLE_SUBFILING +#if PNETCDF_SUBFILING == 1 /* varp->num_subfiles is already set during open or enddef */ if (varp->num_subfiles > 1) *ndimsp = varp->ndims_org; @@ -533,7 +533,7 @@ ncmpio_inq_var(void *ncdp, *ndimsp = varp->ndims; } if (dimids != NULL) { /* copy dim IDs for non-scalar */ -#ifdef ENABLE_SUBFILING +#if PNETCDF_SUBFILING == 1 /* varp->dimids_org is already set during open or enddef */ if (varp->num_subfiles > 1 && varp->ndims_org > 0) memcpy(dimids, varp->dimids_org, sizeof(int) * varp->ndims_org); @@ -600,7 +600,7 @@ ncmpio_rename_var(void *ncdp, #endif err_check: - if (ncp->safe_mode && ncp->nprocs > 1) { + if (fIsSet(ncp->flags, NC_MODE_SAFE) && ncp->nprocs > 1) { int minE, mpireturn; /* First check error code so far across processes */ diff --git a/src/drivers/ncmpio/ncmpio_vard.c b/src/drivers/ncmpio/ncmpio_vard.c deleted file mode 100644 index 7f3fe12248..0000000000 --- a/src/drivers/ncmpio/ncmpio_vard.c +++ /dev/null @@ -1,436 +0,0 @@ -/* - * Copyright (C) 2014, Northwestern University and Argonne National Laboratory - * See COPYRIGHT notice in top-level directory. - */ -/* $Id$ */ - -/* - * This file implements the corresponding APIs defined in - * src/dispatchers/var_getput.m4 - * - * ncmpi_get_vard() : dispatcher->get_vard() - * ncmpi_put_vard() : dispatcher->put_vard() - */ - -#ifdef HAVE_CONFIG_H -# include -#endif - -#include -#include -#ifdef HAVE_STDLIB_H -#include -#endif -#include /* memcpy() */ -#include - -#include - -#include -#include -#include "ncmpio_NC.h" - -/*----< getput_vard() >------------------------------------------------------*/ -/* Starting from 1.9.1, it is possible to read/write multiple variables in a - * single call to vard. Because it is difficult to tell the variable boundaries - * from a single filetype, unless it is flattened, we enforce the following - * requirements for vard APIs. - * 1. The element data type of filetype must be the conform with the NC - * external data type of the variable, i.e. MPI_FLOAT <--> NC_FLOAT. - * If the requirement is violated, NC_ETYPE_MISMATCH will be returned. - * 2. All variables accessed by filetype must be of the same data type. - * NC_EMULTITYPES will be returned if violated. - * 3. buftype must contain only one element data type. Otherwise NC_EMULTITYPES - * will be returned. - */ -static int -getput_vard(NC *ncp, - NC_var *varp, - MPI_Datatype filetype, /* access layout in the file */ - void *buf, - MPI_Offset bufcount, - MPI_Datatype buftype, /* data type of the buffer */ - int reqMode) -{ - void *xbuf=NULL; - int mpireturn, status=NC_NOERR, err=NC_NOERR, xtype_is_contig=1; - int el_size, buftype_is_contig=0, need_swap_back_buf=0; - int need_convert=0, need_swap=0, coll_indep, rw_flag; - MPI_File fh; - MPI_Offset nelems=0, fnelems=0, bnelems=0, offset=0; - MPI_Datatype etype=MPI_DATATYPE_NULL, xtype=MPI_BYTE; - MPI_Offset filetype_size=0; -#ifdef HAVE_MPI_TYPE_SIZE_C - MPI_Count true_lb=0, true_ub=0, true_extent=0; - MPI_Count type_size; -#elif defined(HAVE_MPI_TYPE_SIZE_X) - MPI_Count true_lb=0, true_ub=0, true_extent=0; - MPI_Count type_size; -#else - MPI_Aint true_lb=0, true_ub=0, true_extent=0; - int type_size; -#endif - -#ifdef ENABLE_SUBFILING - /* call a separate routine if variable is stored in subfiles */ - if (varp->num_subfiles > 1) { - printf("This feature for subfiling is yet to implement\n"); - DEBUG_RETURN_ERROR(NC_ENOTSUPPORT) - } -#endif - - /* First check filetype whether it meets the requirement */ - if (filetype == MPI_DATATYPE_NULL) { - /* This is actually an invalid filetype that can cause - * MPI_File_set_view to fail. In PnetCDF, we simply consider this as - * a zero-length request. - */ - filetype_size = 0; - goto err_check; - } - - /* obtain size of filetype and its true upper bound. - * filetype's lb may not always be 0 (e.g. created by constructor - * MPI_Type_create_hindexed), we need to find the true last byte accessed - * by this request, true_ub, in order to calculate new_numrecs. - */ -#ifdef HAVE_MPI_TYPE_SIZE_C - /* MPI_Type_size_c is introduced in MPI 4.0 */ - mpireturn = MPI_Type_size_c(filetype, &type_size); -#elif defined(HAVE_MPI_TYPE_SIZE_X) - /* MPI_Type_size_x is introduced in MPI 3.0 */ - mpireturn = MPI_Type_size_x(filetype, &type_size); -#else - /* PROBLEM: In MPI_Type_size(), argument filetype_size is a 4-byte integer, - * cannot be used for large filetypes. Prior to MPI 3.0 standard, argument - * "size" of MPI_Type_size is of type int. When int overflows, the returned - * value in argument "size" may be a negative. */ - mpireturn = MPI_Type_size(filetype, &type_size); -#endif - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, "MPI_Type_size"); - goto err_check; - } - if (type_size == MPI_UNDEFINED) { /* int overflow */ - DEBUG_ASSIGN_ERROR(err, NC_EINTOVERFLOW) - goto err_check; - } - else if (type_size == 0) /* zero-length request */ - goto err_check; - - filetype_size = type_size; - -#ifdef HAVE_MPI_TYPE_GET_TRUE_EXTENT_C - /* MPI_Type_get_true_extent_c is introduced in MPI 4.0 */ - mpireturn = MPI_Type_get_true_extent_c(filetype, &true_lb, &true_extent); -#elif defined(HAVE_MPI_TYPE_GET_TRUE_EXTENT_X) - /* MPI_Type_get_true_extent_x is introduced in MPI 3.0 */ - mpireturn = MPI_Type_get_true_extent_x(filetype, &true_lb, &true_extent); -#else - mpireturn = MPI_Type_get_true_extent(filetype, &true_lb, &true_extent); -#endif - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, "MPI_Type_get_true_extent"); - goto err_check; - } - - true_ub = true_lb + true_extent; - - /* get the corresponding MPI datatype of variable external type */ - xtype = ncmpii_nc2mpitype(varp->xtype); - - /* find the element type of filetype. ncmpii_dtype_decode() checks - * NC_EMULTITYPES */ - err = ncmpii_dtype_decode(filetype, &etype, NULL, &fnelems, NULL, NULL); - if (err != NC_NOERR) goto err_check; - - /* element type of filetype must be the same as variable's NC type */ - if (etype != xtype) { - DEBUG_ASSIGN_ERROR(err, NC_ETYPE_MISMATCH) - goto err_check; - } - - /* done with checking filetype, now check buftype */ - - if (bufcount == 0 && buftype != MPI_DATATYPE_NULL) { - /* if this process has nothing to read/write */ - goto err_check; - } - - if (buftype == MPI_DATATYPE_NULL) { - /* In this case, the request size is the same as filetype */ - buftype = etype = xtype; - mpireturn = MPI_Type_size(buftype, &el_size); - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, "MPI_Type_size"); - goto err_check; - } - bufcount = filetype_size / el_size; - buftype_is_contig = 1; - bnelems = bufcount; - } - else { - /* find the element type of filetype. ncmpii_dtype_decode() checks - * NC_EMULTITYPES */ - err = ncmpii_dtype_decode(buftype, &etype, &el_size, &bnelems, NULL, - &buftype_is_contig); - if (err != NC_NOERR) goto err_check; - - /* type conversion between non-char and char is not allowed */ - err = NCMPII_ECHAR(varp->xtype, etype); - if (err != NC_NOERR) goto err_check; - - bnelems *= bufcount; - - /* filetype's number of elements must be equal to request's */ - if (fnelems != bnelems) { - DEBUG_ASSIGN_ERROR(err, NC_EIOMISMATCH) - goto err_check; - } - } - xtype_is_contig = buftype_is_contig; - - /* check if type conversion and Endianness byte swap is needed */ - need_convert = ncmpii_need_convert(ncp->format, varp->xtype, etype); - need_swap = NEED_BYTE_SWAP(varp->xtype, etype); - - if (fIsSet(reqMode, NC_REQ_WR)) { - /* check if in-place byte swap can be enabled */ - int can_swap_in_place = 1; - if (need_swap) { - if (fIsSet(ncp->flags, NC_MODE_SWAP_OFF)) - /* in-place byte swap is disabled by user through PnetCDF hint - * 'nc_in_place_swap'. - */ - can_swap_in_place = 0; - else if (! fIsSet(ncp->flags, NC_MODE_SWAP_ON)) { - /* auto mode, as user does not explicitly enable it */ - if (filetype_size <= NC_BYTE_SWAP_BUFFER_SIZE) - /* If write amount is small, disable in-place swap. - * This is because the user buffer may be immutable. - * In this case, in-place swap will cause segmentation - * fault. Immutable buffers are usually small. */ - can_swap_in_place = 0; - } - } - - if (!need_convert && - (!need_swap || (can_swap_in_place && buftype_is_contig))) { - /* reuse buftype, bufcount, buf in later MPI file write */ - xbuf = buf; - if (need_swap) { - ncmpii_in_swapn(xbuf, bnelems, varp->xsz); - need_swap_back_buf = 1; - } - } - else { /* must allocate a buffer to convert/swap/pack */ - xbuf = NCI_Malloc((size_t)filetype_size); - if (xbuf == NULL) { - DEBUG_ASSIGN_ERROR(err, NC_ENOMEM) - goto err_check; - } - need_swap_back_buf = 0; - xtype_is_contig = 1; - - /* pack buf to xbuf, byte-swap and type-convert on xbuf, which - * will later be used in MPI file write */ - err = ncmpio_pack_xbuf(ncp->format, varp, bufcount, buftype, - buftype_is_contig, bnelems, etype, el_size, - MPI_DATATYPE_NULL, need_convert, need_swap, - filetype_size, buf, xbuf); - if (err != NC_NOERR && err != NC_ERANGE) { - if (xbuf != buf) NCI_Free(xbuf); - xbuf = NULL; - goto err_check; - } - } - } - else { /* read request */ - if (!need_convert && (!need_swap || buftype_is_contig)) { - /* reuse buftype, bufcount, buf in later MPI file read */ - xbuf = buf; - } - else { /* must allocate a buffer to convert/swap/unpack */ - xbuf = NCI_Malloc((size_t)filetype_size); - if (xbuf == NULL) { - DEBUG_ASSIGN_ERROR(err, NC_ENOMEM) - goto err_check; - } - xtype_is_contig = 1; - } - } - - /* Set nelems and xtype which will be used in MPI read/write */ - if (buf != xbuf) { - /* xbuf is a malloc-ed contiguous buffer */ - nelems = bnelems; - } - else { - /* we can safely use bufcount and buftype in MPI File read/write. - * Note buftype may be noncontiguous. */ - nelems = bufcount; - xtype = buftype; - } - - /* set fileview's displacement to the variable's starting file offset */ - offset = varp->begin; - -err_check: - /* check error before going into a collective call. - * If an error has been detected on one or more processes, we'll still - * conduct a zero-byte operation (everyone has to participate in the - * collective I/O call) but return the error at the end. NC_ERANGE is not a - * fatal error, we proceed with write request. - */ - if ((err != NC_NOERR && err != NC_ERANGE) || bufcount == 0 || - filetype_size == 0) { - if (fIsSet(reqMode, NC_REQ_INDEP)) { - if (need_swap_back_buf) - /* byte-swap back to buf's original contents */ - ncmpii_in_swapn(buf, bnelems, varp->xsz); - if (xbuf != NULL && xbuf != buf) NCI_Free(xbuf); - return err; - } - /* for NC_REQ_COLL, this process must participate successive collective - * MPI-IO calls as a zero-length request. - */ - offset = 0; - bufcount = 0; - nelems = 0; - filetype_size = 0; - filetype = MPI_BYTE; - buftype = MPI_BYTE; - xtype = MPI_BYTE; - etype = MPI_BYTE; - } - status = err; - - /* when ncp->nprocs == 1, ncp->collective_fh == ncp->independent_fh */ - fh = ncp->independent_fh; - coll_indep = NC_REQ_INDEP; - if (ncp->nprocs > 1 && fIsSet(reqMode, NC_REQ_COLL)) { - fh = ncp->collective_fh; - coll_indep = NC_REQ_COLL; - } - - /* set the MPI-IO fileview, this is a collective call */ - err = ncmpio_file_set_view(ncp, fh, &offset, filetype); - if (err != NC_NOERR) { - if (status == NC_NOERR) status = err; - nelems = 0; /* skip this request */ - } - - rw_flag = (fIsSet(reqMode, NC_REQ_RD)) ? NC_REQ_RD : NC_REQ_WR; - - err = ncmpio_read_write(ncp, rw_flag, coll_indep, offset, nelems, - xtype, xbuf, xtype_is_contig); - if (status == NC_NOERR) status = err; - - /* No longer need to reset the file view, as the root's fileview includes - * the whole file header. - MPI_File_set_view(fh, 0, MPI_BYTE, MPI_BYTE, "native", MPI_INFO_NULL); - */ - - if (fIsSet(reqMode, NC_REQ_RD)) { - if (filetype_size == 0) return status; - - /* swap/convert/unpack xbuf into user buffer, buf, when necessary */ - err = ncmpio_unpack_xbuf(ncp->format, varp, bufcount, buftype, - buftype_is_contig, bnelems, etype, - MPI_DATATYPE_NULL, need_convert, need_swap, - buf, xbuf); - if (status == NC_NOERR) status = err; - } - else { /* write request */ - if (need_swap_back_buf) /* byte-swap back to buf's original contents */ - ncmpii_in_swapn(buf, bnelems, varp->xsz); - - if (IS_RECVAR(varp)) { - /* update header's number of records */ - MPI_Offset new_numrecs = true_ub / ncp->recsize; - if (true_ub % ncp->recsize) new_numrecs++; - - if (fIsSet(reqMode, NC_REQ_INDEP)) { - /* For independent put, we delay the sync for numrecs until - * the next collective call, such as end_indep(), sync(), - * enddef(), or close(). This is because if we update numrecs - * to file now, race condition can happen. Note numrecs in - * memory may be inconsistent and obsolete till then. - */ - if (ncp->numrecs < new_numrecs) { - ncp->numrecs = new_numrecs; - set_NC_ndirty(ncp); - } - } - else { /* NC_REQ_COLL: sync numrecs in memory and file */ - /* new_numrecs may be different among processes. - * First, find the max numrecs among all processes. - */ - MPI_Offset max_numrecs = new_numrecs; - if (ncp->nprocs > 1) { - TRACE_COMM(MPI_Allreduce)(&new_numrecs, &max_numrecs, 1, - MPI_OFFSET, MPI_MAX, ncp->comm); - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, "MPI_Allreduce"); - if (status == NC_NOERR) status = err; - } - } - /* In collective mode, ncp->numrecs is always sync-ed among - processes */ - if (ncp->numrecs < max_numrecs) { - err = ncmpio_write_numrecs(ncp, max_numrecs); - if (status == NC_NOERR) status = err; - ncp->numrecs = max_numrecs; - } - } - } - } - - if (xbuf != NULL && xbuf != buf) NCI_Free(xbuf); - - return status; -} - -/*----< ncmpio_get_vard() >--------------------------------------------------*/ -int -ncmpio_get_vard(void *ncdp, - int varid, - MPI_Datatype filetype, /* access layout in file */ - void *buf, - MPI_Offset bufcount, - MPI_Datatype buftype, /* data type of the buffer */ - int reqMode) -{ - NC *ncp=(NC*)ncdp; - - if (fIsSet(reqMode, NC_REQ_ZERO) && fIsSet(reqMode, NC_REQ_COLL)) - /* this collective API has a zero-length request */ - return ncmpio_getput_zero_req(ncp, reqMode); - - /* Note sanity check for ncdp and varid has been done in dispatchers */ - - return getput_vard(ncp, ncp->vars.value[varid], filetype, buf, bufcount, - buftype, reqMode); -} - -/*----< ncmpio_put_vard() >--------------------------------------------------*/ -int -ncmpio_put_vard(void *ncdp, - int varid, - MPI_Datatype filetype, /* access layout in the file */ - const void *buf, - MPI_Offset bufcount, - MPI_Datatype buftype, /* data type of the buffer */ - int reqMode) -{ - NC *ncp=(NC*)ncdp; - - if (fIsSet(reqMode, NC_REQ_ZERO) && fIsSet(reqMode, NC_REQ_COLL)) - /* this collective API has a zero-length request */ - return ncmpio_getput_zero_req(ncp, reqMode); - - /* Note sanity check for ncdp and varid has been done in dispatchers */ - - return getput_vard(ncp, ncp->vars.value[varid], filetype, (void*)buf, - bufcount, buftype, reqMode); -} diff --git a/src/drivers/ncmpio/ncmpio_wait.c b/src/drivers/ncmpio/ncmpio_wait.c index fc635acfac..743c53e886 100644 --- a/src/drivers/ncmpio/ncmpio_wait.c +++ b/src/drivers/ncmpio/ncmpio_wait.c @@ -34,71 +34,6 @@ NetCDF XDR Level xbuf (XDR I/O buffer) */ -/* Prototypes for functions used only in this file */ -static int wait_getput(NC *ncp, int num_reqs, NC_req *reqs, int rw_flag, - int coll_indep, MPI_Offset newnumrecs); - -static int mgetput(NC *ncp, int num_reqs, NC_req *reqs, int rw_flag, - int coll_indep); - -/*----< ncmpio_getput_zero_req() >-------------------------------------------*/ -/* This function is called when this process has zero-length I/O request and - * must participate all the MPI collective calls involved in the collective - * APIs and wait_all(), which include setting fileview, collective read/write, - * another setting fileview. - * - * This function is collective. - */ -int -ncmpio_getput_zero_req(NC *ncp, int reqMode) -{ - char *mpi_name; - int err, mpireturn, status=NC_NOERR; - MPI_Status mpistatus; - MPI_File fh; - - /* do nothing if this came from an independent API */ - if (fIsSet(reqMode, NC_REQ_INDEP)) return NC_NOERR; - - fh = ncp->collective_fh; - - TRACE_IO(MPI_File_set_view, (fh, 0, MPI_BYTE, MPI_BYTE, "native", MPI_INFO_NULL)); - - if (fIsSet(reqMode, NC_REQ_RD)) { - if (ncp->nprocs > 1) { - TRACE_IO(MPI_File_read_at_all, (fh, 0, NULL, 0, MPI_BYTE, &mpistatus)); - } - else { - TRACE_IO(MPI_File_read_at, (fh, 0, NULL, 0, MPI_BYTE, &mpistatus)); - } - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, mpi_name); - err = (err == NC_EFILE) ? NC_EREAD : err; - DEBUG_ASSIGN_ERROR(status, err) - } - } else { /* write request */ - if (ncp->nprocs > 1) { - TRACE_IO(MPI_File_write_at_all, (fh, 0, NULL, 0, MPI_BYTE, &mpistatus)); - } - else { - TRACE_IO(MPI_File_write_at, (fh, 0, NULL, 0, MPI_BYTE, &mpistatus)); - } - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, mpi_name); - err = (err == NC_EFILE) ? NC_EWRITE : err; - DEBUG_ASSIGN_ERROR(status, err) - } - } - - /* No longer need to reset the file view, as the root's fileview includes - * the whole file header. - TRACE_IO(MPI_File_set_view, (fh, 0, MPI_BYTE, MPI_BYTE, "native", - MPI_INFO_NULL)); - */ - - return status; -} - /*----< abuf_coalesce() >----------------------------------------------------*/ /* this function should be called after all bput requests have been served */ static int @@ -332,280 +267,6 @@ ncmpio_cancel(void *ncdp, return status; } -/*----< construct_filetypes() >----------------------------------------------*/ -/* concatenate the requests into a single MPI derived filetype */ -static int -construct_filetypes(NC *ncp, - NC_lead_req *lead_list, /* NC_REQ_WR or NC_REQ_RD */ - int num_reqs, -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count *blocklens, /* [num_reqs] temp buffer */ - MPI_Count *disps, /* [num_reqs] temp buffer */ -#else - int *blocklens, /* [num_reqs] temp buffer */ - MPI_Aint *disps, /* [num_reqs] temp buffer */ -#endif - NC_req *reqs, /* [num_reqs] */ - MPI_Datatype *filetype) /* OUT */ -{ - int i, j, err, status=NC_NOERR, all_ftype_contig=1, last_contig_req; - int mpireturn; - MPI_Datatype *ftypes; - - if (num_reqs <= 0) { /* for participating collective call */ - *filetype = MPI_BYTE; - return NC_NOERR;; - } - - /* hereinafter, num_reqs > 0 */ - ftypes = (MPI_Datatype*) NCI_Malloc(sizeof(MPI_Datatype) * num_reqs); - - /* create a filetype for each request */ - last_contig_req = -1; /* index of the last contiguous request */ - j = 0; /* index of last valid ftypes */ - for (i=0; ivarp->ndims; - - ftypes[j] = MPI_BYTE; /* in case the call below failed */ - - if (ndims == 0) { /* scalar variable */ -#if SIZEOF_MPI_AINT < SIZEOF_MPI_OFFSET - if (lead->varp->begin > NC_MAX_INT) { - DEBUG_ASSIGN_ERROR(err, NC_EINTOVERFLOW) - fSet(lead->flag, NC_REQ_SKIP); /* skip this request */ - if ( lead->status != NULL && - *lead->status == NC_NOERR) - *lead->status = err; - if (status == NC_NOERR) - status = err; /* report first error */ - } -#endif - disps[j] = lead->varp->begin; - is_ftype_contig = 1; - } - else { /* non-scalar variable */ - MPI_Offset offset, *count, *stride; - count = reqs[i].start + ndims; - stride = fIsSet(lead->flag, NC_REQ_STRIDE_NULL) ? - NULL : count + ndims; - - err = ncmpio_filetype_create_vars(ncp, - lead->varp, - reqs[i].start, - count, - stride, - &offset, - &ftypes[j], - &is_ftype_contig); - -#if SIZEOF_MPI_AINT < SIZEOF_MPI_OFFSET - if (err == NC_NOERR && offset > NC_MAX_INT) - DEBUG_ASSIGN_ERROR(err, NC_EINTOVERFLOW) -#endif - disps[j] = (MPI_Aint)offset; - - if (err != NC_NOERR) { - fSet(lead->flag, NC_REQ_SKIP); /* skip this request */ - if ( lead->status != NULL && - *lead->status == NC_NOERR) - *lead->status = err; - if (status == NC_NOERR) status = err; /* report first error */ - continue; - } - } - - if (is_ftype_contig) { - MPI_Offset coalesced_len; - - /* No need to construct a filetype */ - coalesced_len = lead->varp->xsz * reqs[i].nelems; - -#ifdef HAVE_MPI_LARGE_COUNT - blocklens[j] = coalesced_len; -#else - if (coalesced_len > NC_MAX_INT) { - DEBUG_ASSIGN_ERROR(err, NC_EINTOVERFLOW) - if (status == NC_NOERR) - status = err; /* report first error */ - coalesced_len = 0; - } - blocklens[j] = (int)coalesced_len; -#endif - if (last_contig_req >= 0) - coalesced_len += blocklens[last_contig_req]; -#ifdef HAVE_MPI_LARGE_COUNT - if (last_contig_req >= 0 && - disps[j] - disps[last_contig_req] == - blocklens[last_contig_req]) { - blocklens[last_contig_req] = coalesced_len; - j--; - } - else last_contig_req = j; -#else - /* if coalesced_len overflows 4-byte int, then skip coalescing */ - if (coalesced_len < NC_MAX_INT && last_contig_req >= 0 && - disps[j] - disps[last_contig_req] == - blocklens[last_contig_req]) { - blocklens[last_contig_req] = (int)coalesced_len; - j--; - } - else last_contig_req = j; -#endif - } - else { - /* we will construct a filetype, set blocklen to 1 */ - blocklens[j] = 1; - last_contig_req = -1; - all_ftype_contig = 0; - } - } - /* j is the new num_reqs */ - num_reqs = j; - - if (status != NC_NOERR) { - /* even if error occurs, we still must participate the collective - call to MPI_File_set_view() */ - *filetype = MPI_BYTE; - } - else if (num_reqs == 1 && disps[0] == 0) { - if (ftypes[0] == MPI_BYTE) - *filetype = MPI_BYTE; - else { - mpireturn = MPI_Type_dup(ftypes[0], filetype); - if (mpireturn != MPI_SUCCESS) - err = ncmpii_error_mpi2nc(mpireturn, "MPI_Type_dup"); - } - } - else { /* if (num_reqs > 1 || (num_reqs == 1 && disps[0] > 0)) */ - /* all ftypes[] created fine, now concatenate all ftypes[] */ - if (all_ftype_contig) { -#ifdef HAVE_MPI_LARGE_COUNT - mpireturn = MPI_Type_create_hindexed_c(num_reqs, blocklens, disps, - MPI_BYTE, filetype); -#else - mpireturn = MPI_Type_create_hindexed(num_reqs, blocklens, disps, - MPI_BYTE, filetype); -#endif - if (mpireturn != MPI_SUCCESS) - err = ncmpii_error_mpi2nc(mpireturn, "MPI_Type_create_hindexed"); - else { - MPI_Type_commit(filetype); - err = NC_NOERR; - } - } - else { -#ifdef HAVE_MPI_LARGE_COUNT - mpireturn = MPI_Type_create_struct_c(num_reqs, blocklens, disps, - ftypes, filetype); -#else - mpireturn = MPI_Type_create_struct(num_reqs, blocklens, disps, - ftypes, filetype); -#endif - if (mpireturn != MPI_SUCCESS) - err = ncmpii_error_mpi2nc(mpireturn, "MPI_Type_create_struct"); - else { - MPI_Type_commit(filetype); - err = NC_NOERR; - } - } - - if (err != NC_NOERR) *filetype = MPI_BYTE; - if (status == NC_NOERR) status = err; /* report the first error */ - } - - for (i=0; i--------------------------------------------*/ -/* the input requests, reqs[], are non-interleaving requests */ -static int -construct_buffertypes(NC_lead_req *lead_list, - int num_reqs, -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count *blocklens, /* [num_reqs] temp buffer */ - MPI_Count *disps, /* [num_reqs] temp buffer */ -#else - int *blocklens, /* [num_reqs] temp buffer */ - MPI_Aint *disps, /* [num_reqs] temp buffer */ -#endif - NC_req *reqs, /* [num_reqs] */ - MPI_Datatype *buf_type) /* OUT */ -{ - int i, j, k, status=NC_NOERR, mpireturn; - MPI_Aint a0, ai; - - *buf_type = MPI_BYTE; - if (num_reqs == 0) return NC_NOERR; - - /* create the I/O buffer derived data type */ - - /* calculate blocklens[], and disps[] */ - for (i=0, j=0; iflag, NC_REQ_SKIP)) continue; - - req_size = lead->varp->xsz; - if (lead->varp->ndims > 0) { /* non-scalar variable */ - MPI_Offset *count = reqs[i].start + lead->varp->ndims; - if (!IS_RECVAR(lead->varp)) req_size *= count[0]; - for (k=1; kvarp->ndims; k++) req_size *= count[k]; - } - -#ifdef HAVE_MPI_LARGE_COUNT - blocklens[j] = req_size; -#else - /* check int overflow */ - if (req_size > NC_MAX_INT) { /* skip this request */ - fSet(lead->flag, NC_REQ_SKIP); - DEBUG_ASSIGN_ERROR(status, NC_EINTOVERFLOW) - continue; - } - blocklens[j] = (int)req_size; -#endif - - MPI_Get_address(reqs[i].xbuf, &ai); - if (j == 0) a0 = ai; - disps[j] = MPI_Aint_diff(ai, a0); - j++; - } - /* update num_reqs to number of valid requests */ - num_reqs = j; - - if (num_reqs > 0) { - /* concatenate buffer addresses into a single buffer type */ -#ifdef HAVE_MPI_LARGE_COUNT - mpireturn = MPI_Type_create_hindexed_c(num_reqs, blocklens, disps, - MPI_BYTE, buf_type); -#else - mpireturn = MPI_Type_create_hindexed(num_reqs, blocklens, disps, - MPI_BYTE, buf_type); -#endif - if (mpireturn != MPI_SUCCESS) { - int err = ncmpii_error_mpi2nc(mpireturn,"MPI_Type_create_hindexed"); - /* return the first encountered error if there is any */ - if (status == NC_NOERR) status = err; - } - else - MPI_Type_commit(buf_type); - } - - return status; -} - /*----< extract_reqs() >-----------------------------------------------------*/ /* extract requests from the queues into new queues to be committed. * Input value of num_reqs can be NC_REQ_ALL, NC_GET_REQ_ALL, or NC_PUT_REQ_ALL @@ -797,7 +458,7 @@ extract_reqs(NC *ncp, req_ids[i] == ncp->put_lead_list[j].id) { memcpy(put_list_ptr, ncp->put_list + ncp->put_lead_list[j].nonlead_off, - ncp->put_lead_list[j].nonlead_num * sizeof(NC_req)); + sizeof(NC_req) * ncp->put_lead_list[j].nonlead_num); put_list_ptr += ncp->put_lead_list[j].nonlead_num; req_ids[i] = NC_REQ_NULL; break; @@ -810,7 +471,7 @@ extract_reqs(NC *ncp, req_ids[i] == ncp->get_lead_list[j].id) { memcpy(get_list_ptr, ncp->get_list + ncp->get_lead_list[j].nonlead_off, - ncp->get_lead_list[j].nonlead_num * sizeof(NC_req)); + sizeof(NC_req) * ncp->get_lead_list[j].nonlead_num); get_list_ptr += ncp->get_lead_list[j].nonlead_num; req_ids[i] = NC_REQ_NULL; break; @@ -988,24 +649,40 @@ req_commit(NC *ncp, } /* carry out writes and reads separately (writes first) */ - if (do_write > 0) { - - if (ncp->my_aggr >= 0 && coll_indep == NC_REQ_COLL && ncp->nprocs > 1) - /* intra-node write aggregation must be in collective mode */ - err = ncmpio_intra_node_aggregation_nreqs(ncp, NC_REQ_WR, - num_w_reqs, put_list, - newnumrecs); - else - err = wait_getput(ncp, num_w_reqs, put_list, NC_REQ_WR, coll_indep, - newnumrecs); - put_list = NULL; /* has been freed in wait_getput() */ + err = ncmpio_ina_nreqs(ncp, NC_REQ_WR, num_w_reqs, put_list, + newnumrecs); + put_list = NULL; /* has been freed in the above call */ + + /* Update the number of records if new records have been created. + * For nonblocking APIs, there is no way for a process to know whether + * others write to a record variable or not. Note newnumrecs has been + * sync-ed and always >= ncp->numrecs. + */ + if (coll_indep == NC_REQ_COLL) { + if (newnumrecs > ncp->numrecs) { + /* update new record number in file. Note newnumrecs is already + * sync-ed among all processes and in collective mode + * ncp->numrecs is always sync-ed in memory among processes, + * thus no need another MPI_Allreduce to sync it. */ + err = ncmpio_write_numrecs(ncp, newnumrecs); + if (status == NC_NOERR) status = err; + /* retain the first error if there is any */ + if (ncp->numrecs < newnumrecs) ncp->numrecs = newnumrecs; + } + } + else { /* NC_REQ_INDEP */ + if (ncp->numrecs < newnumrecs) { + ncp->numrecs = newnumrecs; + set_NC_ndirty(ncp); + /* delay numrecs sync until end_indep, redef or close */ + } + } } - if (do_read > 0) { - err = wait_getput(ncp, num_r_reqs, get_list, NC_REQ_RD, coll_indep, - newnumrecs); - get_list = NULL; /* has been freed in wait_getput() */ + err = ncmpio_ina_nreqs(ncp, NC_REQ_RD, num_r_reqs, get_list, + newnumrecs); + get_list = NULL; /* has been freed in the above call */ } /* retain the first error status */ @@ -1142,7 +819,6 @@ ncmpio_wait(void *ncdp, coll_indep = (fIsSet(reqMode, NC_REQ_INDEP)) ? NC_REQ_INDEP : NC_REQ_COLL; -#ifdef ENABLE_REQ_AGGREGATION /* check collective or independent mode */ if (coll_indep == NC_REQ_INDEP && !NC_indep(ncp)) DEBUG_RETURN_ERROR(NC_ENOTINDEP) @@ -1152,1368 +828,5 @@ ncmpio_wait(void *ncdp, if (coll_indep == NC_REQ_INDEP && num_reqs == 0) return NC_NOERR; return req_commit(ncp, num_reqs, req_ids, statuses, coll_indep); -#else - /* If request aggregation is disabled, we call an independent wait() for - * each request - */ - int i, status=NC_NOERR, err; - - if (coll_indep == NC_REQ_INDEP) { - /* This is called from ncmpi_wait(), which is an independent call - * Argument num_reqs can be NC_REQ_ALL which means to flush all pending - * nonblocking requests. In this case, arguments req_ids and statuses - * will be ignored. - * Argument num_reqs must either be NC_REQ_ALL, NC_GET_REQ_ALL, - * NC_PUT_REQ_ALL, or a non-negative value. - * Argument statuses can be NULL, meaning the caller only cares about - * the error code returned by this call, but not the statuses of - * individual nonblocking requests. - */ - if (num_reqs == 0) return NC_NOERR; - - /* This is called from ncmpi_wait which must be called in independent - * data mode, illegal in collective mode. - */ - if (!NC_indep(ncp)) DEBUG_RETURN_ERROR(NC_ENOTINDEP); - - if (coll_indep == NC_REQ_INDEP && num_reqs == 0) return NC_NOERR; - } - else { - /* This is called from ncmpi_wait_all(), which is a collective call - * Argument num_reqs can be NC_REQ_ALL which means to flush all pending - * nonblocking requests. In this case, arguments req_ids and statuses - * will be ignored. - * Argument num_reqs must either be NC_REQ_ALL, NC_GET_REQ_ALL, - * NC_PUT_REQ_ALL, or a non-negative value. - * Argument statuses can be NULL, meaning the caller only cares about - * the error code returned by this call, but not the statuses of - * individual nonblocking requests. - */ - /* the following line CANNOT be added, because ncmpi_wait_all() is a - * collective call, all processes must participate some MPI collective - * operations used later on. - */ - /* if (num_reqs == 0) return NC_NOERR; */ - - /* This is called from ncmpi_wait_all which must be called in - * collective data mode, illegal in independent mode. This also - * ensures the program will returns back to collective mode. - */ - if (NC_indep(ncp)) DEBUG_RETURN_ERROR(NC_EINDEP); - - /* must enter independent mode, as num_reqs may be different among - processes */ - err = ncmpio_begin_indep_data(ncp); - if (status == NC_NOERR) status = err; - } - - if (num_reqs <= NC_REQ_ALL) { /* flush all get or put pending requests */ - if (num_reqs == NC_REQ_ALL || num_reqs == NC_GET_REQ_ALL) { - while (ncp->numLeadGetReqs) { - /* commit one request at a time. Note ncp->numLeadGetReqs - * will be descreased in req_commit() - */ - err = req_commit(ncp, 1, &ncp->get_lead_list[0].id, NULL, - NC_REQ_INDEP); - if (status == NC_NOERR) status = err; - } - } - if (num_reqs == NC_REQ_ALL || num_reqs == NC_PUT_REQ_ALL) { - while (ncp->numLeadPutReqs) { - /* commit one request at a time. Note ncp->numLeadPutReqs - * will be descreased in req_commit() - */ - err = req_commit(ncp, 1, &ncp->put_lead_list[0].id, NULL, - NC_REQ_INDEP); - if (status == NC_NOERR) status = err; - } - } - } - else { - for (i=0; i------------------------------------------------------*/ -/* used for sorting the offsets of the off_len array */ -static int -off_compare(const void *a, const void *b) -{ - if (((off_len*)a)->off > ((off_len*)b)->off) return 1; - if (((off_len*)a)->off < ((off_len*)b)->off) return -1; - return 0; -} - -/*----< vars_flatten() >-----------------------------------------------------*/ -/* flatten a subarray request into a list of offset-length pairs */ -static MPI_Offset -vars_flatten(int ndim, /* number of dimensions */ - int el_size, /* array element size */ - MPI_Offset offset, /* starting file offset of variable */ - MPI_Offset *dimlen, /* [ndim] dimension lengths */ - MPI_Aint buf_addr,/* starting buffer address */ - MPI_Offset *start, /* [ndim] starts of subarray */ - MPI_Offset *count, /* [ndim] counts of subarray */ - MPI_Offset *stride, /* [ndim] strides of subarray */ - MPI_Offset *nseg, /* OUT: number of segments */ - off_len *seg) /* OUT: array of segments */ -{ - int i, j, to_free_stride=0; - MPI_Offset seg_len, nstride, array_len, off, subarray_len; - off_len *ptr=seg, *seg0; - - *nseg = 0; - if (ndim < 0) return *nseg; - - if (ndim == 0) { /* scalar record variable */ - *nseg = 1; - seg->off = offset; - seg->len = el_size; - seg->buf_addr = buf_addr; - return *nseg; - } - - if (stride == NULL) { /* equivalent to {1, 1, ..., 1} */ - stride = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset) * ndim); - for (i=0; i= 1 - Q: Is it legal if any stride[] <= 0 ? */ - - /* calculate the number of offset-length pairs */ - *nseg = (stride[ndim-1] == 1) ? 1 : count[ndim-1]; - for (i=0; ioff = off; - ptr->len = seg_len; - ptr->buf_addr = buf_addr; - buf_addr += seg_len; - off += stride[ndim-1] * el_size; - ptr++; - } - ndim--; - - subarray_len = nstride; - array_len = 1; - /* for higher dimensions */ - while (ndim > 0) { - /* array_len is global array size from lowest up to ndim */ - array_len *= dimlen[ndim]; - - /* off is the global array offset for this dimension, ndim-1 */ - off = start[ndim-1] * array_len * el_size; - - /* update all offsets from lowest up to dimension ndim-1 */ - seg0 = seg; - for (j=0; joff += off; - seg0++; - } - - /* update each plan subarray of dimension ndim-1 */ - off = array_len * stride[ndim-1] * el_size; - for (i=1; ioff = seg0->off + off; - ptr->len = seg_len; - ptr->buf_addr = buf_addr; - buf_addr += seg_len; - ptr++; - seg0++; - } - off += array_len * stride[ndim-1] * el_size; - } - ndim--; /* move to next higher dimension */ - subarray_len *= count[ndim]; - } - if (to_free_stride) NCI_Free(stride); - - return *nseg; -} - -/*----< merge_requests() >---------------------------------------------------*/ -/* flatten all requests into offset-length pairs, sort them into an increasing - * order, and resolve the overlapped offset-length pairs. - */ -static int -merge_requests(NC *ncp, - NC_lead_req *lead_list, - int num_reqs, - NC_req *reqs, /* [num_reqs] */ - void **buf, /* OUT: 1st I/O buf addr */ - MPI_Offset *nsegs, /* OUT: no. off-len pairs */ - off_len **segs) /* OUT: [*nsegs] */ -{ - int i, j, status=NC_NOERR, ndims; - MPI_Offset nseg, *start, *count, *shape, *stride; - MPI_Aint addr, buf_addr; - - *nsegs = 0; /* total number of offset-length pairs */ - *segs = NULL; /* array of offset-length pairs */ - - /* note invalid requests have been removed in wait_getput() */ - *buf = reqs[0].xbuf; /* I/O buffer of first request */ - - /* buf_addr is the buffer address of the first request */ - MPI_Get_address(reqs[0].xbuf, &buf_addr); - - /* Count the number off-len pairs from reqs[], so we can malloc a - * contiguous memory space for storing off-len pairs - */ - for (i=0; ivarp->ndims; - if (ndims > 0) { - start = reqs[i].start; - count = start + ndims; - stride = count + ndims; - } - else - start = count = stride = NULL; - - /* for record variable, each reqs[] is within a record */ - if (IS_RECVAR(lead->varp)) { - ndims--; - start++; - count++; - stride++; - } - if (fIsSet(lead->flag, NC_REQ_STRIDE_NULL)) stride = NULL; - - if (ndims < 0) continue; - if (ndims == 0) { /* 1D record variable */ - (*nsegs)++; - continue; - } - nseg = 1; - if (stride != NULL && stride[ndims-1] > 1) - nseg = count[ndims-1]; /* count of last dimension */ - for (j=0; jvarp->ndims; - if (ndims > 0) { - start = reqs[i].start; - count = start + ndims; - stride = count + ndims; - } - else - start = count = stride = NULL; - - shape = lead->varp->shape; - - /* find the starting file offset for this variable */ - var_begin = lead->varp->begin; - - /* for record variable, each reqs[] is within a record */ - if (IS_RECVAR(lead->varp)) { - ndims--; - start++; - count++; - stride++; - shape++; - /* find the starting file offset for this record */ - var_begin += reqs[i].start[0] * ncp->recsize; - } - - if (fIsSet(lead->flag, NC_REQ_STRIDE_NULL)) stride = NULL; - - /* flatten each request to a list of offset-length pairs */ - vars_flatten(ndims, lead->varp->xsz, var_begin, shape, - addr, start, count, stride, - &nseg, /* OUT: number of offset-length pairs */ - seg_ptr); /* OUT: array of offset-length pairs */ - seg_ptr += nseg; /* append the list to the end of segs array */ - } - - /* check if (*segs)[].off are in an increasing order */ - for (i=1; i<*nsegs; i++) { - if ((*segs)[i-1].off > (*segs)[i].off) - break; - } - if (i < *nsegs) /* not in an increasing order */ - /* sort the off-len array, segs[], in an increasing order */ - qsort(*segs, (size_t)(*nsegs), sizeof(off_len), off_compare); - - /* merge the overlapped requests, skip the overlapped regions for those - * requests with higher j indices (i.e. requests with lower j indices - * win the writes to the overlapped regions) - */ - for (i=0, j=1; j<*nsegs; j++) { - if ((*segs)[i].off + (*segs)[i].len >= (*segs)[j].off + (*segs)[j].len) - /* segment i completely covers segment j, skip j */ - continue; - - MPI_Offset gap = (*segs)[i].off + (*segs)[i].len - (*segs)[j].off; - if (gap >= 0) { /* segments i and j overlaps */ - if (MPI_Aint_add((*segs)[i].buf_addr, (*segs)[i].len) == - MPI_Aint_add((*segs)[j].buf_addr, gap)) { - /* buffers i and j are contiguous, merge j to i */ - (*segs)[i].len = MPI_Aint_add((*segs)[i].len, (*segs)[j].len - gap); - } - else { /* buffers are not contiguous, reduce j's len */ - (*segs)[i+1].off = (*segs)[j].off + gap; - (*segs)[i+1].len = (*segs)[j].len - gap; - (*segs)[i+1].buf_addr = MPI_Aint_add((*segs)[j].buf_addr, gap); - i++; - } - } - else { /* i and j do not overlap */ - i++; - if (i < j) (*segs)[i] = (*segs)[j]; - } - } - - /* update number of segments, now all off-len pairs are not overlapped */ - *nsegs = i+1; - - return status; -} - -/*----< type_create_off_len() >----------------------------------------------*/ -static int -type_create_off_len(MPI_Offset nsegs, /* no. off-len pairs */ - off_len *segs, /* [nsegs] off-len pairs (sorted) */ - MPI_Datatype *filetype, /* OUT */ - MPI_Datatype *buf_type) /* OUT */ -{ - int i, j, mpireturn; - MPI_Offset next_off, next_len, true_nsegs; -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count *blocklens; - MPI_Count *disps; -#else - int *blocklens; - MPI_Aint *disps; -#endif - - assert(nsegs > 0); - - /* create the file view MPI derived data type by concatenating the sorted - offset-length pairs */ - - /* For filetype, the segs[].off can be further coalesced. For example, - * when writing a consecutive columns of a 2D array, even though the I/O - * buffer addresses may not be able to coalesced, the file offsets on - * the same row can be coalesced. Thus, first calculate the length of - * coalesced off-len pairs (the memory space needed for malloc) - */ - next_off = segs[0].off; - next_len = segs[0].len; - for (j=0,i=1; i NC_MAX_INT) { - NCI_Free(disps); - NCI_Free(blocklens); - DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) - } - disps[0] = segs[0].off; - blocklens[0] = (int)segs[0].len; - for (j=0,i=1; i NC_MAX_INT) { - NCI_Free(disps); - NCI_Free(blocklens); - DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) - } - if (disps[j] + blocklens[j] == segs[i].off) - /* j and i are contiguous */ - blocklens[j] += (int)segs[i].len; - /* TODO: take care of 4-byte int overflow problem */ - else { - j++; - disps[j] = segs[i].off; - blocklens[j] = (int)segs[i].len; - } - } - /* Now j+1 is the coalesced length */ - - mpireturn = MPI_Type_create_hindexed(j+1, blocklens, disps, MPI_BYTE, - filetype); -#endif - if (mpireturn != MPI_SUCCESS) { - *filetype = MPI_BYTE; - *buf_type = MPI_BYTE; - NCI_Free(disps); - NCI_Free(blocklens); - return ncmpii_error_mpi2nc(mpireturn, "MPI_Type_create_hindexed"); - } - MPI_Type_commit(filetype); - - /* create the I/O buffer derived data type from the I/O buffer's - offset-length pairs */ - - /* Although it is unlikely buffers can be coalesced, it will not harm to - check it out */ - next_off = segs[0].buf_addr; - next_len = segs[0].len; - for (j=0,i=1; i NC_MAX_INT) { - NCI_Free(disps); - NCI_Free(blocklens); - DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) - } - disps[0] = segs[0].buf_addr; - blocklens[0] = (int)segs[0].len; - for (j=0,i=1; i NC_MAX_INT) { - NCI_Free(disps); - NCI_Free(blocklens); - DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) - } - if (disps[j] + blocklens[j] == segs[i].buf_addr) - /* j and i are contiguous */ - blocklens[j] += (int)segs[i].len; - else { - j++; - disps[j] = segs[i].buf_addr; - blocklens[j] = (int)segs[i].len; - } - } - /* j+1 is the coalesced length */ - - mpireturn = MPI_Type_create_hindexed(j+1, blocklens, disps, MPI_BYTE, - buf_type); -#endif - NCI_Free(disps); - NCI_Free(blocklens); - if (mpireturn != MPI_SUCCESS) { - if (*filetype != MPI_BYTE) MPI_Type_free(filetype); - *filetype = MPI_BYTE; - *buf_type = MPI_BYTE; - return ncmpii_error_mpi2nc(mpireturn, "MPI_Type_create_hindexed"); - } - MPI_Type_commit(buf_type); - - return NC_NOERR; -} - -/*----< req_compare() >------------------------------------------------------*/ -/* used to sort the the string file offsets of reqs[] */ -static int -req_compare(const void *a, const void *b) -{ - if (((NC_req*)a)->offset_start > ((NC_req*)b)->offset_start) return (1); - if (((NC_req*)a)->offset_start < ((NC_req*)b)->offset_start) return (-1); - return (0); -} - -/*----< req_aggregation() >--------------------------------------------------*/ -/* aggregate multiple read/write (non-contiguous) requests and call MPI-IO - */ -static int -req_aggregation(NC *ncp, - int num_reqs, /* IN/OUT: # requests */ - NC_req *reqs, /* [num_reqs] sorted requests, to be freed - in this subroutine */ - int rw_flag, /* NC_REQ_WR or NC_REQ_RD */ - int coll_indep, /* NC_REQ_COLL or NC_REQ_INDEP */ - int interleaved) /* interleaved in reqs[] */ -{ - int i, gtype, err, status=NC_NOERR, ngroups, mpireturn, buf_len; - int *group_index, *group_type, numLeadReqs; -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count *blocklens, *f_blocklens, *b_blocklens; - MPI_Count *disps, *f_disps, *b_disps; -#else - int *blocklens, *f_blocklens, *b_blocklens; - MPI_Aint *disps, *f_disps, *b_disps; -#endif - NC_lead_req *lead_list; - void *buf; /* point to starting buffer, used by MPI-IO call */ - MPI_Aint b_begin, b_addr; - MPI_Datatype filetype, buf_type, *ftypes, *btypes; - MPI_File fh; - MPI_Offset max_end, offset; - - if (num_reqs == 0) { /* only NC_REQ_COLL can reach here for 0 request */ - assert(coll_indep == NC_REQ_COLL); - /* simply participate the collective call */ - return ncmpio_getput_zero_req(ncp, rw_flag); - } - if (! interleaved) { - /* concatenate all filetypes into a single one and do I/O */ - return mgetput(ncp, num_reqs, reqs, rw_flag, coll_indep); - } - /* now some request's aggregate access region is interleaved with other's */ - - /* divide the requests into groups. - * Two types of groups: one contains requests that all are not interleaved - * and the other contains requests that any 2 consecutive requests are - * interleaved. All requests will be aggregated into one and carried out - * by a single MPI-IO call. - * This approach is because MPI collective I/O requires each process's - * fileview must contain only monotonic non-decreasing file offsets. Thus - * if the nonblocking requests interleave with each other (although not - * overlap), then we cannot simply concatenate the filetypes of individual - * requests. Codes below flatten the requests of "interleaved" groups - * into offset-length pairs, sorts, and merges them into an aggregated - * filetype. Similar for building an aggregated I/O buffer type. - */ - - /* first calculate the number of groups, so group_index and group_type can - be malloc-ed. Group type: 0 for non-interleaved group and 1 for - interleaved group. - */ -#define INTERLEAVED 1 -#define NONINTERLEAVED 0 - - assert(num_reqs > 1); - ngroups = 1; - gtype = (reqs[0].offset_end > reqs[1].offset_start) ? - INTERLEAVED : NONINTERLEAVED; - max_end = MAX(reqs[0].offset_end, reqs[1].offset_end); - for (i=1; i reqs[i+1].offset_start) { - /* Done with this NONINTERLEAVED group. Continue to construct - * next group, starting from reqs[i], which will be INTERLEAVED. */ - ngroups++; - gtype = INTERLEAVED; - max_end = MAX(reqs[i].offset_end, reqs[i+1].offset_end); - } - else if (gtype == INTERLEAVED) { - if (max_end <= reqs[i+1].offset_start) { - /* Done with this INTERLEAVED group. Continue to construct - * next group. First check whether the next group is - * INTERLEAVED or NONINTERLEAVED. */ - gtype = NONINTERLEAVED; - if (i+2 < num_reqs && - reqs[i+1].offset_end > reqs[i+2].offset_start) - gtype = INTERLEAVED; /* next group is also interleaved */ - ngroups++; - max_end = reqs[i+1].offset_end; - } - else - max_end = MAX(max_end, reqs[i+1].offset_end); - } - } - - group_index = (int*) NCI_Malloc(sizeof(int) * (ngroups+1) * 2); - group_type = group_index + (ngroups+1); - - /* calculate the starting index of each group and determine group type */ - ngroups = 1; - gtype = (reqs[0].offset_end > reqs[1].offset_start) ? - INTERLEAVED : NONINTERLEAVED; - max_end = MAX(reqs[0].offset_end, reqs[1].offset_end); - group_index[0] = 0; - group_type[0] = gtype; - for (i=1; i reqs[i+1].offset_start) { - /* Done with this NONINTERLEAVED group. Continue to construct - * next group, which will be INTERLEAVED. */ - /* reqs[i] starts a new interleaved group */ - group_index[ngroups] = i; - gtype = INTERLEAVED; - group_type[ngroups] = gtype; - ngroups++; - max_end = MAX(reqs[i].offset_end, reqs[i+1].offset_end); - } - else if (gtype == INTERLEAVED) { - if (max_end <= reqs[i+1].offset_start) { - /* Done with this INTERLEAVED group. Continue to construct - * next group. First check whether the next group is - * INTERLEAVED or NONINTERLEAVED. */ - gtype = NONINTERLEAVED; - if (i+2 < num_reqs && - reqs[i+1].offset_end > reqs[i+2].offset_start) - gtype = INTERLEAVED; /* next group is also interleaved */ - /* the interleaved group ends with reqs[i] */ - group_index[ngroups] = i+1; - group_type[ngroups] = gtype; - ngroups++; - max_end = reqs[i+1].offset_end; - } - else - max_end = MAX(max_end, reqs[i+1].offset_end); - } - } - group_index[ngroups] = num_reqs; /* to indicate end of groups */ - - /* for each group, construct one filetype by concatenating if the group - * is non-interleaved and by flatten/sort/merge if the group is - * interleaved. At the end, all ngroups filetypes are concatenated into - * a single filetype. Similar for constructing buffer types. - * Then use one collective I/O to commit. - */ - - ftypes = (MPI_Datatype*) NCI_Malloc(sizeof(MPI_Datatype) * ngroups * 2); - btypes = ftypes + ngroups; - - /* temp buffers, used by multiple calls to construct_filetypes() */ -#ifdef HAVE_MPI_LARGE_COUNT - blocklens = (MPI_Count*) NCI_Malloc(sizeof(MPI_Count) * num_reqs); - disps = (MPI_Count*) NCI_Malloc(sizeof(MPI_Count) * num_reqs); - f_blocklens = (MPI_Count*) NCI_Malloc(sizeof(MPI_Count) * ngroups); - f_disps = (MPI_Count*) NCI_Malloc(sizeof(MPI_Count) * ngroups); - b_blocklens = (MPI_Count*) NCI_Malloc(sizeof(MPI_Count) * ngroups); - b_disps = (MPI_Count*) NCI_Malloc(sizeof(MPI_Count) * ngroups); -#else - blocklens = (int*) NCI_Malloc(sizeof(int) * num_reqs); - disps = (MPI_Aint*) NCI_Malloc(sizeof(MPI_Aint) * num_reqs); - f_blocklens = (int*) NCI_Malloc(sizeof(int) * ngroups); - f_disps = (MPI_Aint*) NCI_Malloc(sizeof(MPI_Aint) * ngroups); - b_blocklens = (int*) NCI_Malloc(sizeof(int) * ngroups); - b_disps = (MPI_Aint*) NCI_Malloc(sizeof(MPI_Aint) * ngroups); -#endif - - buf = reqs[0].xbuf; /* the buffer of 1st request */ - b_disps[0] = 0; /* relative to address of 1st buf */ - MPI_Get_address(buf, &b_begin); - - lead_list = (rw_flag == NC_REQ_RD) ? ncp->get_lead_list - : ncp->put_lead_list; - /* for each group, build a filetype and a buffer type in ftypes[i] and - btypes[i] */ - for (i=0; i 0); - - /* sges[] will be used to construct fileview and buffer type */ - err = type_create_off_len(nsegs, segs, &ftypes[i], &btypes[i]); - /* preserve the previous error if there is any */ - if (status == NC_NOERR) status = err; - NCI_Free(segs); - if (err != NC_NOERR) { /* skip this group */ - ftypes[i] = btypes[i] = MPI_BYTE; - b_blocklens[i] = 0; - f_blocklens[i] = 0; - continue; - } - f_blocklens[i] = 1; - } - - if (i > 0) { - /* get the buffer address of the first request in this group */ - MPI_Get_address(g_reqs[0].xbuf, &b_addr); - b_disps[i] = MPI_Aint_diff(b_addr, b_begin); /* to 1st buffer of 1st group*/ - } - b_blocklens[i] = 1; - } - NCI_Free(disps); - NCI_Free(blocklens); - NCI_Free(group_index); - - buf_len=1; - - if (ngroups == 1) { - /* use ftypes[0] and btypes[0] directly */ - filetype = ftypes[0]; - buf_type = btypes[0]; - } - else { - /* concatenate all ftypes[] to filetype */ -#ifdef HAVE_MPI_LARGE_COUNT - mpireturn = MPI_Type_create_struct_c(ngroups, f_blocklens, f_disps, - ftypes, &filetype); -#else - mpireturn = MPI_Type_create_struct(ngroups, f_blocklens, f_disps, - ftypes, &filetype); -#endif - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, "MPI_Type_create_struct"); - /* return the first encountered error if there is any */ - if (status == NC_NOERR) status = err; - - buf_len = 0; /* skip this request */ - filetype = MPI_BYTE; - } - else - MPI_Type_commit(&filetype); - - for (i=0; inumLeadGetReqs - : ncp->numLeadPutReqs; - for (i=0; iindependent_fh; - if (ncp->nprocs > 1 && coll_indep == NC_REQ_COLL) - fh = ncp->collective_fh; - - /* set the MPI-IO fileview, this is a collective call */ - offset = 0; - err = ncmpio_file_set_view(ncp, fh, &offset, filetype); - if (filetype != MPI_BYTE) MPI_Type_free(&filetype); - if (err != NC_NOERR) { - if (status == NC_NOERR) status = err; - if (coll_indep == NC_REQ_INDEP) return status; - buf_len = 0; - } - - /* call MPI_File_read_at_all/MPI_File_write_at_all */ - err = ncmpio_read_write(ncp, rw_flag, coll_indep, offset, buf_len, buf_type, - buf, ((buf_type == MPI_BYTE) ? 1 : 0)); - if (status == NC_NOERR) status = err; - - if (buf_type != MPI_BYTE) MPI_Type_free(&buf_type); - - /* No longer need to reset the file view, as the root's fileview includes - * the whole file header. - TRACE_IO(MPI_File_set_view, (fh, 0, MPI_BYTE, MPI_BYTE, "native", - MPI_INFO_NULL)); - */ - - return status; -} - -/*----< calculate_access_range() >-------------------------------------------*/ -/* Returns the file offsets of access range of this request: starting file - * offset and end offset (exclusive). - * Note zero-length request should never call this subroutine. - */ -static int -calculate_access_range(const NC *ncp, - const NC_var *varp, - const MPI_Offset *start, /* [varp->ndims] */ - const MPI_Offset *count, /* [varp->ndims] */ - const MPI_Offset *stride, /* [varp->ndims] */ - MPI_Offset *start_off, /* OUT: start offset */ - MPI_Offset *end_off) /* OUT: end offset */ -{ - int i, ndims = varp->ndims; /* number of dimensions of this variable */ - - /* - * varp->dsizes[] is computed from right to left product of shape - * For example, a 3D array of size 5x4x3 in C order, - * For fixed-size variable: dsizes[0]=60 dsizes[1]=12 dsizes[2]=3 - * For record variable: dsizes[0]=12 dsizes[1]=12 dsizes[2]=3 - */ - if (IS_RECVAR(varp)) { - *start_off = 0; - *end_off = 0; - if (stride == NULL) { - if (ndims > 1) { - /* least significant dimension */ - *start_off = start[ndims-1]; - *end_off = start[ndims-1]+(count[ndims-1]-1); - /* the remaining dimensions */ - for (i=ndims-2; i>0; i--) { - *start_off += start[i]*varp->dsizes[i+1]; - *end_off += (start[i]+(count[i]-1))*varp->dsizes[i+1]; - } - } - *start_off *= varp->xsz; /* offset in bytes */ - *end_off *= varp->xsz; - /* handle the unlimited, most significant dimension */ - *start_off += start[0] * ncp->recsize; - *end_off += (start[0]+(count[0]-1)) * ncp->recsize; - } - else { - if (ndims > 1) { - /* least significant dimension */ - *start_off = start[ndims-1]; - *end_off = start[ndims-1]+(count[ndims-1]-1)*stride[ndims-1]; - /* the remaining dimensions */ - for (i=ndims-2; i>0; i--) { - *start_off += start[i]*varp->dsizes[i+1]; - *end_off += (start[i]+(count[i]-1)*stride[i]) * - varp->dsizes[i+1]; - } - } - *start_off *= varp->xsz; /* offset in bytes */ - *end_off *= varp->xsz; - /* handle the unlimited, most significant dimension */ - *start_off += start[0] * ncp->recsize; - *end_off += (start[0]+(count[0]-1)*stride[0]) * ncp->recsize; - } - } - else { - if (stride == NULL) { - /* first handle the least significant dimension */ - *start_off = start[ndims-1]; - *end_off = start[ndims-1] + (count[ndims-1]-1); - /* remaining dimensions till the most significant dimension */ - for (i=ndims-2; i>=0; i--) { - *start_off += start[i] * varp->dsizes[i+1]; - *end_off += (start[i]+(count[i]-1)) * varp->dsizes[i+1]; - } - } - else { - /* first handle the least significant dimension */ - *start_off = start[ndims-1]; - *end_off = start[ndims-1]+(count[ndims-1]-1)*stride[ndims-1]; - /* remaining dimensions till the most significant dimension */ - for (i=ndims-2; i>=0; i--) { - *start_off += start[i] * varp->dsizes[i+1]; - *end_off += (start[i]+(count[i]-1)*stride[i])*varp->dsizes[i+1]; - } - } - *start_off *= varp->xsz; /* offset in bytes */ - *end_off *= varp->xsz; - } - *start_off += varp->begin; /* beginning file offset of this variable */ - *end_off += varp->begin + varp->xsz; - - return NC_NOERR; } -/*----< wait_getput() >------------------------------------------------------*/ -static int -wait_getput(NC *ncp, - int num_reqs, /* number of non-lead requests */ - NC_req *reqs, /* array of non-lead requests */ - int rw_flag, /* NC_REQ_WR or NC_REQ_RD */ - int coll_indep, /* NC_REQ_COLL or NC_REQ_INDEP */ - MPI_Offset newnumrecs) /* new number of records */ -{ - int i, err, status=NC_NOERR, interleaved=0, descreasing=0; - NC_lead_req *lead_list; - - /* move the offset calculation from request posting API calls to wait call, - * such that posting a nonblocking request can be made in define mode - */ - lead_list = (rw_flag == NC_REQ_RD) ? ncp->get_lead_list - : ncp->put_lead_list; - for (i=0; ivarp; - - if (varp->ndims == 0) { /* scalar variable */ - reqs[i].offset_start = varp->begin; - reqs[i].offset_end = varp->begin + varp->xsz; - } - else { - /* start/count/stride have been allocated in a contiguous array */ - MPI_Offset *count, *stride; - count = reqs[i].start + varp->ndims; - stride = (fIsSet(lead->flag, NC_REQ_STRIDE_NULL)) ? NULL : - count + varp->ndims; - - /* calculate access range of this request */ - calculate_access_range(ncp, varp, reqs[i].start, count, stride, - &reqs[i].offset_start, &reqs[i].offset_end); - } - if (i > 0) { - /* check if offset_start are in a monotonic nondecreasing order */ - if (reqs[i].offset_start < reqs[i-1].offset_start) - descreasing = interleaved = 1; /* possible interleaved */ - else if (reqs[i].offset_start < reqs[i-1].offset_end) - interleaved = 1; /* possible interleaved */ - } - } - - /* If a decreasing order is found, sort reqs[] based on reqs[].offset_start - * into an increasing order */ - if (descreasing) - qsort(reqs, (size_t)num_reqs, sizeof(NC_req), req_compare); - - /* check sorted requests for true interleaved */ - if (interleaved) { /* only if possible interleaved */ - interleaved = 0; - for (i=1; i= ncp->numrecs. - */ - if (rw_flag == NC_REQ_WR) { - if (coll_indep == NC_REQ_COLL) { - if (newnumrecs > ncp->numrecs) { - /* update new record number in file. Note newnumrecs is already - * sync-ed among all processes and in collective mode - * ncp->numrecs is always sync-ed in memory among processes, - * thus no need another MPI_Allreduce to sync it. */ - err = ncmpio_write_numrecs(ncp, newnumrecs); - if (status == NC_NOERR) status = err; - /* retain the first error if there is any */ - if (ncp->numrecs < newnumrecs) ncp->numrecs = newnumrecs; - } - } - else { /* NC_REQ_INDEP */ - if (ncp->numrecs < newnumrecs) { - ncp->numrecs = newnumrecs; - set_NC_ndirty(ncp); - /* delay numrecs sync until end_indep, redef or close */ - } - } - } - - return status; -} - -/*----< mgetput() >----------------------------------------------------------*/ -/* Before reaching to this subroutine, all the filetypes in the request array - * are sorted in a non-decreasing order and not interleaved. This subroutine - * concatenates the filetypes into a single fileview and calls MPI-IO function. - * This subroutine also concatenates user buffertypes into a single derived - * data type to be used in the MPI read/write function call. - */ -static int -mgetput(NC *ncp, - int num_reqs, /* IN: number of requests */ - NC_req *reqs, /* [num_reqs] non-lead request list, to be freed - in this subroutine */ - int rw_flag, /* NC_REQ_WR or NC_REQ_RD */ - int coll_indep) /* NC_REQ_COLL or NC_REQ_INDEP */ -{ - int i, j, numLeadReqs, status=NC_NOERR, mpireturn, err; - void *buf=NULL; - NC_lead_req *lead_list; - MPI_Datatype filetype, buf_type=MPI_BYTE; - MPI_Offset offset=0, buf_count=0; - MPI_File fh; - -#ifdef HAVE_MPI_LARGE_COUNT - MPI_Count *blocklens; - MPI_Count *disps; - blocklens = (MPI_Count*) NCI_Malloc(sizeof(MPI_Count) * num_reqs); - disps = (MPI_Count*) NCI_Malloc(sizeof(MPI_Count) * num_reqs); -#else - int *blocklens; - MPI_Aint *disps; - blocklens = (int*) NCI_Malloc(sizeof(int) * num_reqs); - disps = (MPI_Aint*) NCI_Malloc(sizeof(MPI_Aint) * num_reqs); -#endif - - lead_list = (rw_flag == NC_REQ_RD) ? ncp->get_lead_list - : ncp->put_lead_list; - - /* construct an MPI file type by concatenating fileviews of all requests */ - status = construct_filetypes(ncp, lead_list, num_reqs, blocklens, disps, - reqs, &filetype); - if (status != NC_NOERR) { /* if failed, skip this request */ - if (coll_indep == NC_REQ_INDEP) { - NCI_Free(blocklens); - NCI_Free(disps); - NCI_Free(reqs); - return status; - } - - /* For collective I/O, we still need to participate the successive - collective calls: setview/read/write */ - filetype = MPI_BYTE; - buf = NULL; - buf_count = 0; - NCI_Free(disps); - NCI_Free(blocklens); - goto mpi_io; - } - - /* now construct buffer datatype */ - if (num_reqs == 1) { - NC_lead_req *lead = lead_list + reqs[0].lead_off; - if (fIsSet(lead->flag, NC_REQ_SKIP)) - buf_count = 0; - else { -#ifdef HAVE_MPI_LARGE_COUNT - buf_count = reqs[0].nelems * lead->varp->xsz; -#else - MPI_Offset req_size = reqs[0].nelems * lead->varp->xsz; - if (req_size > NC_MAX_INT) { /* skip this request */ - if (status == NC_NOERR) - DEBUG_ASSIGN_ERROR(status, NC_EINTOVERFLOW) - fSet(lead->flag, NC_REQ_SKIP); - buf_count = 0; /* skip this request */ - } - else - buf_count = req_size; -#endif - } - buf = reqs[0].xbuf; - } - else if (num_reqs > 1) { /* create the I/O buffer derived data type */ - int last_contig_req; - MPI_Aint a0=0, ai, a_last_contig; - - last_contig_req = 0; /* index of the last contiguous request */ - buf = NULL; - /* process only valid requests */ - for (j=0, i=0; iflag, NC_REQ_SKIP)) - continue; /* skip invalid request */ - - req_size = reqs[i].nelems * lead->varp->xsz; - -#ifdef HAVE_MPI_LARGE_COUNT - blocklens[j] = req_size; -#else - /* check int overflow */ - if (req_size > NC_MAX_INT) { /* int overflows, skip this request */ - if (status == NC_NOERR) /* keep the 1st encountered error */ - DEBUG_ASSIGN_ERROR(status, NC_EINTOVERFLOW) - fSet(lead->flag, NC_REQ_SKIP); - continue; /* skip this request */ - } - blocklens[j] = (int)req_size; -#endif - - MPI_Get_address(reqs[i].xbuf, &ai); - if (j == 0) { /* first valid request */ - a_last_contig = a0 = ai; - buf = reqs[i].xbuf; - disps[0] = 0; - j = 1; - continue; - } - disps[j] = MPI_Aint_diff(ai, a0); - - req_size = blocklens[last_contig_req]; - req_size += blocklens[j]; -#ifdef HAVE_MPI_LARGE_COUNT - if (MPI_Aint_diff(ai, a_last_contig) == blocklens[last_contig_req]) { - /* user buffer of request j is contiguous from j-1 - * we coalesce j to j-1 */ - blocklens[last_contig_req] += blocklens[j]; - } -#else - /* if req_size overflows 4-byte int, then skip coalescing */ - if (req_size <= NC_MAX_INT && - MPI_Aint_diff(ai, a_last_contig) == blocklens[last_contig_req]) { - /* user buffer of request j is contiguous from j-1 - * we coalesce j to j-1 */ - blocklens[last_contig_req] += blocklens[j]; - } -#endif - else if (j > 0) { - /* not contiguous from request last_contig_req */ - last_contig_req++; - a_last_contig = ai; - disps[last_contig_req] = MPI_Aint_diff(ai, a0); - blocklens[last_contig_req] = blocklens[i]; - } - j++; - } - - /* last_contig_req is the index of last contiguous request */ - if (last_contig_req == 0) { - /* user buffers can be concatenated into a contiguous buffer */ - buf_type = MPI_BYTE; - buf_count = blocklens[0]; - } - else { - /* after possible concatenating the user buffers, the true number - * of non-contiguous buffers is last_contig_req+1 */ - int num_contig_reqs = last_contig_req+1; - - /* concatenate buffer addresses into a single buffer type */ -#ifdef HAVE_MPI_LARGE_COUNT - mpireturn = MPI_Type_create_hindexed_c(num_contig_reqs, blocklens, - disps, MPI_BYTE, &buf_type); -#else - mpireturn = MPI_Type_create_hindexed(num_contig_reqs, blocklens, - disps, MPI_BYTE, &buf_type); -#endif - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn,"MPI_Type_create_hindexed"); - /* return the first encountered error if there is any */ - if (status == NC_NOERR) status = err; - } - else { - mpireturn = MPI_Type_commit(&buf_type); - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn,"MPI_Type_commit"); - /* return the first encountered error if there is any */ - if (status == NC_NOERR) status = err; - } - } - - buf_count = 1; - } - } - /* if (buf_type == MPI_BYTE) then the whole buf is contiguous */ - - NCI_Free(disps); - NCI_Free(blocklens); - - /* Free up memory space allocated before entering MPI-IO calls, as MPI-IO - * flattens the fileview and buftype which can take some space. The - * non-lead request list, reqs, is no longer used after fileview and buftype - * have been constructed. In addition, the start arrays allocated at lead - * requests are no longer used. - */ - numLeadReqs = (rw_flag == NC_REQ_RD) ? ncp->numLeadGetReqs - : ncp->numLeadPutReqs; - for (i=0; iindependent_fh; - if (ncp->nprocs > 1 && coll_indep == NC_REQ_COLL) - fh = ncp->collective_fh; - - /* set the MPI-IO fileview, this is a collective call */ - err = ncmpio_file_set_view(ncp, fh, &offset, filetype); - if (filetype != MPI_BYTE) MPI_Type_free(&filetype); - if (err != NC_NOERR) { - if (status == NC_NOERR) status = err; - if (coll_indep == NC_REQ_INDEP) return status; - buf_count = 0; - } - - /* call MPI_File_read_at_all/MPI_File_write_at_all */ - err = ncmpio_read_write(ncp, rw_flag, coll_indep, offset, buf_count, - buf_type, buf, ((buf_type == MPI_BYTE) ? 1 : 0)); - if (status == NC_NOERR) status = err; - - if (buf_type != MPI_BYTE) MPI_Type_free(&buf_type); - - /* No longer need to reset the file view, as the root's fileview includes - * the whole file header. - TRACE_IO(MPI_File_set_view, (fh, 0, MPI_BYTE, MPI_BYTE, "native", - MPI_INFO_NULL)); - */ - - return status; -} diff --git a/src/include/dispatch.h b/src/include/dispatch.h index 8276a4245a..873e9fc6ec 100644 --- a/src/include/dispatch.h +++ b/src/include/dispatch.h @@ -22,7 +22,7 @@ #define NC_REQ_NBI 0x00000100 /* nonblocking iget/iput API */ #define NC_REQ_NBB 0x00000200 /* nonblocking bput API */ -#define NC_MODE_RDONLY 0x00001000 /* file is opned in read-only mode */ +#define NC_MODE_RDONLY 0x00001000 /* file is opened in read-only mode */ #define NC_MODE_DEF 0x00002000 /* in define mode */ #define NC_MODE_INDEP 0x00004000 /* in independent data mode */ #define NC_MODE_CREATE 0x00008000 /* just created and still in define mode */ @@ -44,10 +44,28 @@ typedef enum { API_VARM } NC_api; +typedef struct { + int ref_count; /* this comm's attribute reference count */ + int num_NUMAs; /* number of NUMA compute nodes */ + int *NUMA_IDs; /* [nprocs] NUMA node ID of each MPI process */ + MPI_Comm numa_comm; /* hardware NUMA communicator */ + int num_aggrs_per_node; /* number of INA aggregators per node */ + int num_ina_aggrs; /* total number of INA aggregators overall */ + int is_ina_aggr; /* Whether this rank is an INA aggregator */ + int *ina_ranks; /* all INA aggregators' rank IDs in this comm */ + MPI_Comm ina_inter_comm; /* inter-node communicator of INA aggregators */ + MPI_Comm ina_intra_comm; /* intra-node communicator within an INA group. + * being MPI_COMM_NULL or not also indicates + * whether INA is disable or not. + */ +} PNC_comm_attr; + struct PNC_driver { /* APIs manipulate files */ - int (*create)(MPI_Comm, const char*, int, int, MPI_Info, void**); - int (*open)(MPI_Comm, const char*, int, int, MPI_Info, void**); + int (*create)(MPI_Comm, const char*, int, int, int, MPI_Info, + PNC_comm_attr, void**); + int (*open)(MPI_Comm, const char*, int, int, int, MPI_Info, + PNC_comm_attr, void**); int (*close)(void*); int (*enddef)(void*); int (*_enddef)(void*,MPI_Offset,MPI_Offset,MPI_Offset,MPI_Offset); @@ -92,9 +110,6 @@ struct PNC_driver { int (*get_varn)(void*,int,int,MPI_Offset* const*,MPI_Offset* const*,void*,MPI_Offset,MPI_Datatype,int); int (*put_varn)(void*,int,int,MPI_Offset* const*,MPI_Offset* const*,const void*,MPI_Offset,MPI_Datatype,int); - int (*get_vard)(void*,int,MPI_Datatype,void*,MPI_Offset,MPI_Datatype,int); - int (*put_vard)(void*,int,MPI_Datatype,const void*,MPI_Offset,MPI_Datatype,int); - int (*iget_var)(void*,int,const MPI_Offset*,const MPI_Offset*,const MPI_Offset*,const MPI_Offset*,void*,MPI_Offset,MPI_Datatype,int*,int); int (*iput_var)(void*,int,const MPI_Offset*,const MPI_Offset*,const MPI_Offset*,const MPI_Offset*,const void*,MPI_Offset,MPI_Datatype,int*,int); int (*bput_var)(void*,int,const MPI_Offset*,const MPI_Offset*,const MPI_Offset*,const MPI_Offset*,const void*,MPI_Offset,MPI_Datatype,int*,int); @@ -147,7 +162,7 @@ extern PNC_driver* ncmpio_inq_driver(void); extern PNC_driver* nc4io_inq_driver(void); -extern PNC_driver* ncadios_inq_driver(void); +extern PNC_driver* ncadios_inq_driver(void); extern PNC_driver* ncfoo_inq_driver(void); @@ -155,4 +170,17 @@ extern PNC_driver* ncbbio_inq_driver(void); extern int PNC_check_id(int ncid, PNC **pncp); +#if PNETCDF_PROFILING == 1 +#define NTIMERS 10 +extern int pnc_num_aggrs_per_node; +extern double pnc_ina_init; +extern double pnc_ina_flatten; +extern double pnc_ina_put[NTIMERS]; +extern double pnc_ina_get[NTIMERS]; +extern MPI_Count pnc_ina_npairs_put; +extern MPI_Count pnc_ina_npairs_get; +extern MPI_Count pnc_ina_mem_put[NTIMERS]; +extern MPI_Count pnc_ina_mem_get[NTIMERS]; +#endif + #endif /* H_PNC_DISPATCH */ diff --git a/src/include/pnc_debug.h b/src/include/pnc_debug.h index 9762448964..50c4496989 100644 --- a/src/include/pnc_debug.h +++ b/src/include/pnc_debug.h @@ -44,7 +44,17 @@ } \ } -#ifdef PNETCDF_DEBUG +#if PNETCDF_DEBUG_MODE == 1 + +/* PNETCDF_VERBOSE_DEBUG_MODE environment variable can be used to print the + * location in the source code where the error code is originated, no matter + * the error is intended or not. This run-time environment variable only takes + * effect only when PnetCDF is configure with debug mode, i.e. --enable-debug + * is set at the configure command line. This feature is mainly for PnetCDF + * developers, who are warned that enabling this mode may result in a lot of + * debugging messages printed in stderr. + */ + #define DEBUG_RETURN_ERROR(err) { \ char *_env_str = getenv("PNETCDF_VERBOSE_DEBUG_MODE"); \ if (_env_str != NULL && *_env_str != '0') { \ @@ -55,6 +65,26 @@ } \ return err; \ } +#define DEBUG_FOPEN_ERROR(err) { \ + char *_env_str = getenv("PNETCDF_VERBOSE_DEBUG_MODE"); \ + if (_env_str != NULL && *_env_str != '0') { \ + int _rank; \ + MPI_Comm_rank(MPI_COMM_WORLD, &_rank); \ + fprintf(stderr, "Rank %d: %s error at line %d of %s in %s\n", \ + _rank,ncmpi_strerrno(err),__LINE__,__func__,__FILE__); \ + } \ + return err; \ +} +#define DEBUG_RETURN_ERROR_MSG(err, msg) { \ + char *_env_str = getenv("PNETCDF_VERBOSE_DEBUG_MODE"); \ + if (_env_str != NULL && *_env_str != '0') { \ + int _rank; \ + MPI_Comm_rank(MPI_COMM_WORLD, &_rank); \ + fprintf(stderr, "Rank %d: %s error at line %d of %s in %s (%s)\n", \ + _rank,ncmpi_strerrno(err),__LINE__,__func__,__FILE__, msg); \ + } \ + return err; \ +} #define DEBUG_ASSIGN_ERROR(status, err) { \ char *_env_str = getenv("PNETCDF_VERBOSE_DEBUG_MODE"); \ if (_env_str != NULL && *_env_str != '0') { \ @@ -76,6 +106,8 @@ } #else #define DEBUG_RETURN_ERROR(err) return err; +#define DEBUG_RETURN_ERROR_MSG(err, msg) return err; +#define DEBUG_FOPEN_ERROR(err) return err; #define DEBUG_ASSIGN_ERROR(status, err) status = err; #define DEBUG_TRACE_ERROR(err) #endif diff --git a/src/include/pnetcdf.h.in b/src/include/pnetcdf.h.in index b363276b6c..097d2cfac1 100644 --- a/src/include/pnetcdf.h.in +++ b/src/include/pnetcdf.h.in @@ -16,6 +16,7 @@ #define PNETCDF_VERSION_MAJOR @PNETCDF_VERSION_MAJOR@ #define PNETCDF_VERSION_MINOR @PNETCDF_VERSION_MINOR@ #define PNETCDF_VERSION_SUB @PNETCDF_VERSION_SUB@ +#define PNETCDF_VERSION_PRE "@PNETCDF_VERSION_PRE@" #define PNETCDF_RELEASE_DATE "DIST_DATE" /* List of PnetCDF features enabled/disabled at configure time. @@ -34,6 +35,7 @@ #define PNETCDF_THREAD_SAFE @ENABLE_THREAD_SAFE@ #define PNETCDF_DRIVER_NETCDF4 @ENABLE_NETCDF4@ #define PNETCDF_DRIVER_ADIOS @ENABLE_ADIOS@ +#define PNETCDF_DRIVER_GIO @ENABLE_GIO@ #if defined(__cplusplus) extern "C" { @@ -657,6 +659,9 @@ by the desired type. */ #define NC_EBADLOG (-238) /**< Unrecognized log file format */ #define NC_EFLUSHED (-239) /**< Nonblocking request has already been flushed. It is too late to cancel */ #define NC_EADIOS (-240) /**< unknown ADIOS error */ +#define NC_EFSTYPE (-241) /**< File system type not supported by the GIO driver */ +#define NC_EDRIVER (-242) /**< Invalid PnetCDF I/O driver */ +#define NC_EFILEVIEW (-243) /**< PnetCDF internal error: file view is not monotonically non-decreasing */ /* add new error here */ /* header inconsistency errors start from -250 */ @@ -684,9 +689,10 @@ by the desired type. */ #define NC_EMULTIDEFINE_VAR_FILL_MODE (-271) /**< inconsistent variable fill mode */ #define NC_EMULTIDEFINE_VAR_FILL_VALUE (-272) /**< inconsistent variable fill value */ #define NC_EMULTIDEFINE_CMODE (-273) /**< inconsistent file create modes among processes */ +#define NC_EMULTIDEFINE_HINTS (-274) /**< inconsistent I/O hints among processes */ #define NC_EMULTIDEFINE_FIRST NC_EMULTIDEFINE -#define NC_EMULTIDEFINE_LAST NC_EMULTIDEFINE_CMODE +#define NC_EMULTIDEFINE_LAST NC_EMULTIDEFINE_HINTS /* backward compatible with PnetCDF 1.3.1 and earlier */ #define NC_ECMODE NC_EMULTIDEFINE_OMODE diff --git a/src/libs/Makefile.am b/src/libs/Makefile.am index a932f20f56..3e602f4de7 100644 --- a/src/libs/Makefile.am +++ b/src/libs/Makefile.am @@ -35,6 +35,9 @@ endif if ENABLE_ADIOS libpnetcdf_la_LIBADD += ../drivers/ncadios/libncadios.la endif +if ENABLE_GIO + libpnetcdf_la_LIBADD += ${top_builddir}/gio/src/libgio.la +endif nodist_EXTRA_libpnetcdf_la_SOURCES = dummyc.c diff --git a/src/packaging/pnetcdf.pc.in b/src/packaging/pnetcdf.pc.in index aba430cd1f..47e9da03fb 100644 --- a/src/packaging/pnetcdf.pc.in +++ b/src/packaging/pnetcdf.pc.in @@ -47,5 +47,5 @@ Version: @PNETCDF_VERSION@ Cflags: -I${includedir} Libs: -L${libdir} -lpnetcdf Libs.private: -Requires: @NETCDF4_PACKAGE@ @ADIOS_PACKAGE@ +Requires: @GIO_PACKAGE@ @NETCDF4_PACKAGE@ @ADIOS_PACKAGE@ Requires.private: diff --git a/src/utils/Makefile.am b/src/utils/Makefile.am index a0b931fe46..d1fd61a315 100644 --- a/src/utils/Makefile.am +++ b/src/utils/Makefile.am @@ -16,7 +16,7 @@ endif # The script shows the end users how pnetcdf is built bin_SCRIPTS = pnetcdf-config -EXTRA_DIST = pnetcdf-config.in README.md +EXTRA_DIST = README.md # For VPATH build (parallel build), try delete all sub-directories distclean-local: diff --git a/src/utils/ncmpidiff/Makefile.am b/src/utils/ncmpidiff/Makefile.am index 0cbc72adb5..3b3f64b2a3 100644 --- a/src/utils/ncmpidiff/Makefile.am +++ b/src/utils/ncmpidiff/Makefile.am @@ -11,8 +11,13 @@ AM_CPPFLAGS += -I$(top_builddir)/src/include bin_PROGRAMS = ncmpidiff cdfdiff +check_PROGRAMS = $(bin_PROGRAMS) + +noinst_LTLIBRARIES = libncmpidiff_core.la +libncmpidiff_core_la_SOURCES = ncmpidiff_core.c + ncmpidiff_SOURCES = ncmpidiff.c -ncmpidiff_LDADD = $(top_builddir)/src/libs/libpnetcdf.la +ncmpidiff_LDADD = $(noinst_LTLIBRARIES) $(top_builddir)/src/libs/libpnetcdf.la ncmpidiff_LDADD += @NETCDF4_LDFLAGS@ @ADIOS_LDFLAGS@ @NETCDF4_LIBS@ @ADIOS_LIBS@ cdfdiff_SOURCES = cdfdiff.c @@ -25,6 +30,23 @@ $(top_builddir)/src/libs/libpnetcdf.la: dist_man_MANS = ncmpidiff.1 cdfdiff.1 +AM_TESTS_ENVIRONMENT = export check_PROGRAMS="$(check_PROGRAMS)"; +AM_TESTS_ENVIRONMENT += export SED="$(SED)"; +AM_TESTS_ENVIRONMENT += export srcdir="$(srcdir)"; +AM_TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; +AM_TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; +AM_TESTS_ENVIRONMENT += export TESTMPIRUN="$(TESTMPIRUN)"; + +TESTS = $(check_PROGRAMS) +XFAIL_TESTS = $(check_PROGRAMS) +TEST_EXTENSIONS = .sh +LOG_COMPILER = $(srcdir)/xfail_runs.sh +SH_LOG_COMPILER = + +check_SCRIPTS=xfail_runs.sh + +EXTRA_DIST = ncmpidiff_core.h xfail_runs.sh tst_file.nc + CLEANFILES = core core.* *.gcda *.gcno *.gcov gmon.out dist-hook: diff --git a/src/utils/ncmpidiff/cdfdiff.c b/src/utils/ncmpidiff/cdfdiff.c index be19bc0c7e..5656a0fa84 100644 --- a/src/utils/ncmpidiff/cdfdiff.c +++ b/src/utils/ncmpidiff/cdfdiff.c @@ -187,9 +187,9 @@ struct vspec { /*----< get_var_names() >-----------------------------------------------------*/ static void -get_var_names(char *optarg, struct vspec* vspecp) +get_var_names(char *opt_arg, struct vspec* vspecp) { - char *cp=optarg, **cpp; + char *cp=opt_arg, **cpp; int nvars = 1; /* compute number of variable names in comma-delimited list */ @@ -203,7 +203,7 @@ get_var_names(char *optarg, struct vspec* vspecp) cpp = vspecp->names; /* copy variable names into list */ - for (cp = strtok(optarg, ","); + for (cp = strtok(opt_arg, ","); cp != NULL; cp = strtok((char *) NULL, ",")) { @@ -237,11 +237,11 @@ get_type(int type) /*----< main() >--------------------------------------------------------------*/ int main(int argc, char **argv) { + /* int verbose; is defined as a locally global variable in ncvalidator.c */ extern char *optarg; extern int optind; - char *str, *ptr; size_t nbytes; - int i, j, k, m, n, c, err, verbose, quiet, isDiff; + int i, j, k, m, n, c, err, quiet, isDiff; int fd[2], nvars[2], ndims[2], nattrs[2], check_tolerance; int cmp_nvars, check_header, check_variable_list, check_entire_file; long long numVarDIFF=0, numHeadDIFF=0, numDIFF; @@ -264,7 +264,8 @@ int main(int argc, char **argv) var_list.nvars = 0; check_tolerance = 0; - while ((c = getopt(argc, argv, "bhqv:t:")) != -1) + while ((c = getopt(argc, argv, "bhqv:t:")) != -1) { + char *str, *ptr; switch(c) { case 'h': /* compare header only */ check_header = 1; @@ -301,12 +302,19 @@ int main(int argc, char **argv) usage(argv[0]); break; } + } /* quiet mode overwrites verbose */ if (quiet) verbose = 0; if (argc - optind != 2) usage(argv[0]); + if (strcmp(argv[optind], argv[optind+1]) == 0) { + fprintf(stderr,"Error: two input file names are identical (%s) ... exit\n", + argv[optind]); + exit(1); + } + if (verbose && check_tolerance) { printf("Tolerance absolute difference = %e\n", tolerance_difference); printf("Tolerance ratio difference = %e\n", tolerance_ratio); @@ -394,9 +402,9 @@ int main(int argc, char **argv) /* compare file header */ if (check_header) { - NC_attr *attr[2]; - NC_dim *dim[2]; - NC_var *var[2]; + NC_attr *attr[2]={NULL, NULL}; + NC_dim *dim[2]={NULL, NULL}; + NC_var *var[2]={NULL, NULL}; /* compare number of dimensions defined */ if (ndims[0] != ndims[1]) { @@ -844,7 +852,7 @@ int main(int argc, char **argv) if (j == nvars[0]) { if (!check_header) { if (!quiet) - printf("WARN: variable \"%s\" s not found in %s\n", + printf("WARN: variable \"%s\" is not found in %s\n", var_list.names[i],argv[optind]); numVarDIFF++; } @@ -859,7 +867,7 @@ int main(int argc, char **argv) if (j == nvars[1]) { if (!check_header) { if (!quiet) - printf("WARN: variable \"%s\" s not found in %s\n", + printf("WARN: variable \"%s\" is not found in %s\n", var_list.names[i],argv[optind+1]); numVarDIFF++; } diff --git a/src/utils/ncmpidiff/ncmpidiff.c b/src/utils/ncmpidiff/ncmpidiff.c index 8f1fe4f203..1ca506b128 100644 --- a/src/utils/ncmpidiff/ncmpidiff.c +++ b/src/utils/ncmpidiff/ncmpidiff.c @@ -23,27 +23,15 @@ #include #include #include -#include -#include /* INFINITY */ +#include /* stat() */ +#include /* stat() */ +#include /* stat() */ +#include /* errno */ +#include /* INFINITY */ #include #include - -#ifndef ubyte -#define ubyte unsigned char -#endif -#ifndef ushort -#define ushort unsigned short -#endif -#ifndef uint -#define uint unsigned int -#endif -#ifndef int64 -#define int64 long long -#endif -#ifndef uint64 -#define uint64 unsigned long long -#endif +#include #define OOM_ERROR { \ @@ -51,213 +39,6 @@ exit(1); \ } -#define HANDLE_ERROR { \ - if (err != NC_NOERR) { \ - fprintf(stderr, "Error at line %d of file %s (%s)\n", __LINE__, \ - __FILE__, ncmpi_strerror(err)); \ - MPI_Abort(MPI_COMM_WORLD, -1); \ - exit(-1); \ - } \ -} - -#define CHECK_GLOBAL_ATT_DIFF_CHAR { \ - int pos; \ - char *b1 = (char *)calloc((attlen[0] + 1) * 2, sizeof(char)); \ - char *b2 = b1 + attlen[0] + 1; \ - if (!b1) OOM_ERROR \ - err = ncmpi_get_att_text(ncid[0], NC_GLOBAL, name[0], b1); \ - HANDLE_ERROR \ - err = ncmpi_get_att_text(ncid[1], NC_GLOBAL, name[0], b2); \ - HANDLE_ERROR \ - for (pos=0; pos= 0) ? (x) : (-x) -#define UABS(x) (x) - -#define CHECK_VAR_DIFF(type, func, xabs) { \ - int pos, isDiff, worst = -1; \ - type *b1, *b2; \ - b1 = (type *)calloc(varsize * 2, sizeof(type)); \ - if (!b1) OOM_ERROR \ - b2 = b1 + varsize; \ - err = ncmpi_get_vara_##func(ncid[0], varid1, start, shape, b1); \ - HANDLE_ERROR \ - err = ncmpi_get_vara_##func(ncid[1], varid2, start, shape, b2); \ - HANDLE_ERROR \ - if (!check_tolerance) { \ - for (pos=0; pos abs_b2) ? abs_b1 : abs_b2; \ - diff = b1[pos] - b2[pos]; \ - diff = (diff >= 0) ? diff : -diff; \ - ratio = diff / abs_max; \ - if (diff <= tolerance_difference || ratio <= tolerance_ratio) \ - continue; \ - /* fail to meet both tolerance errors */ \ - worst = pos; \ - break; \ - } \ - } \ - if (pos != varsize || worst != -1) { /* diff is found */ \ - double v1, v2; \ - if (ndims[0] == 0) { /* scalar variable */ \ - if (worst == -1) \ - printf("DIFF: scalar variable \"%s\" of type \"%s\"\n", \ - name[0], get_type(xtype[0])); \ - else { \ - v1 = b1[worst]; \ - v2 = b2[worst]; \ - printf("DIFF (tolerance): scalar variable \"%s\" of type \"%s\" of value %g vs %g (difference = %e)\n", \ - name[0], get_type(xtype[0]), v1, v2, v1-v2); \ - } \ - } else { \ - int _i; \ - MPI_Offset *diffStart; \ - diffStart = (MPI_Offset*) malloc(sizeof(MPI_Offset) * ndims[0]); \ - if (worst != -1) pos = worst; \ - v1 = b1[pos]; \ - v2 = b2[pos]; \ - for (_i=ndims[0]-1; _i>=0; _i--) { \ - diffStart[_i] = pos % shape[_i] + start[_i]; \ - pos /= shape[_i]; \ - } \ - if (worst == -1) \ - printf("DIFF: variable \"%s\" of type \"%s\" at element ["OFFFMT, \ - name[0], get_type(xtype[0]), diffStart[0]); \ - else \ - printf("DIFF (tolerance): variable \"%s\" of type \"%s\" at element ["OFFFMT, \ - name[0], get_type(xtype[0]), diffStart[0]); \ - for (_i=1; _i-------------------------------------------------------------*/ +/* File system types recognized by ROMIO in MPICH 4.0.0, and by PnetCDF */ +static const char* fstypes[] = {"ufs", "nfs", "xfs", "pvfs2", "gpfs", "panfs", "lustre", "daos", "testfs", "ime", "quobyte", NULL}; + +/* Return a pointer to filename by removing the file system type prefix name if + * there is any. For example, when filename = "lustre:/home/foo/testfile.nc", + * remove "lustre:" to return a pointer to "/home/foo/testfile.nc", so the name + * can be used in POSIX open() calls. + */ +static +char* ncmpii_remove_file_system_type_prefix(const char *filename) +{ + char *ret_filename = (char*)filename; + + if (filename == NULL) return NULL; + + if (strchr(filename, ':') != NULL) { /* there is a prefix end with ':' */ + /* check if prefix is one of recognized file system types */ + int i=0; + while (fstypes[i] != NULL) { + size_t prefix_len = strlen(fstypes[i]); + if (!strncmp(filename, fstypes[i], prefix_len)) { /* found */ + ret_filename += prefix_len + 1; + break; + } + i++; + } + } + + return ret_filename; +} + +/*----< usage() >------------------------------------------------------------*/ static void usage(int rank, char *progname) { @@ -298,25 +110,26 @@ struct vspec { char **names; /* [nvars] */ }; -/*----< get_var_names() >-----------------------------------------------------*/ +/*----< get_var_names() >----------------------------------------------------*/ static void -get_var_names(char *optarg, struct vspec* vspecp) +get_var_names(char *opt_arg, + int *nvars, + char ***names) { - char *cp=optarg, **cpp; - int nvars = 1; + char *cp=opt_arg, **cpp; + *nvars = 1; /* compute number of variable names in comma-delimited list */ - vspecp->nvars = 1; while (*cp++) if (*cp == ',') - nvars++; + (*nvars)++; - vspecp->names = (char **) calloc((size_t)nvars, sizeof(char*)); - if (!vspecp->names) OOM_ERROR + *names = (char **) calloc((size_t)*nvars, sizeof(char*)); + if (!*names) OOM_ERROR - cpp = vspecp->names; + cpp = *names; /* copy variable names into list */ - for (cp = strtok(optarg, ","); + for (cp = strtok(opt_arg, ","); cp != NULL; cp = strtok((char *) NULL, ",")) { @@ -324,59 +137,45 @@ get_var_names(char *optarg, struct vspec* vspecp) if (!*cpp) OOM_ERROR cpp++; } - vspecp->nvars = nvars; } -/*----< get_type() >----------------------------------------------------------*/ -static char* -get_type(int type) -{ - switch (type) { - case NC_BYTE: return "NC_BYTE"; - case NC_CHAR: return "NC_CHAR"; - case NC_SHORT: return "NC_SHORT"; - case NC_INT: return "NC_INT"; - case NC_FLOAT: return "NC_FLOAT"; - case NC_DOUBLE: return "NC_DOUBLE"; - case NC_UBYTE: return "NC_UBYTE"; - case NC_USHORT: return "NC_USHORT"; - case NC_UINT: return "NC_UINT"; - case NC_INT64: return "NC_INT64"; - case NC_UINT64: return "NC_UINT64"; - } - return "NC_NAT"; -} - -/*----< main() >--------------------------------------------------------------*/ +/*----< main() >-------------------------------------------------------------*/ int main(int argc, char **argv) { extern char *optarg; extern int optind; - char *name[2]; - int i, j, c, err, rank, nprocs, verbose, quiet, check_tolerance; - int ncid[2], ndims[2], nvars[2], natts[2], recdim[2], *dimids[2], fmt[2]; - int cmp_nvars, check_header, check_variable_list, check_entire_file; - long long numVarDIFF=0, numHeadDIFF=0, varDIFF, numDIFF; + char cmd_opts[1024], **var_names; + int i, c, rank, nprocs, verbose, quiet, check_tolerance; + int first_diff, ncid[2], num_vars; + int check_header, check_variable_list, check_entire_file; + long long numDIFF; double tolerance_ratio, tolerance_difference; - MPI_Offset *shape=NULL, varsize, *start=NULL; - MPI_Offset attlen[2], dimlen[2]; MPI_Comm comm=MPI_COMM_WORLD; MPI_Info info = MPI_INFO_NULL; - nc_type xtype[2]; - struct vspec var_list; MPI_Init(&argc, &argv); MPI_Comm_size(comm, &nprocs); MPI_Comm_rank(comm, &rank); + if (nprocs == 1) + strcpy(cmd_opts, "ncmpidiff"); + else + sprintf(cmd_opts, "Rank %d: ncmpidiff", rank); + + for (i=1; i 0 && ndims[1] > 0) { - if (verbose) - printf("Dimension:\n"); - } else - goto cmp_vars; - - /* check dimensions in 1st file also appear in 2nd file */ - for (i=0; i 0 && nvars[1] > 0) { - if (verbose) - printf("Variables:\n"); - } else - goto cmp_exit; - - /* check variables defined in 1st file and also in 2nd file */ - for (i=0; i 0) { + for (i=0; i 0 && dimids[0][0] == recdim[0]) { /* record variable */ - err = ncmpi_inq_dimlen(ncid[0], recdim[0], &shape[0]); - HANDLE_ERROR - if (shape[0] == 0) { - /* No record has been written to the file, skip comparison */ - free(shape); - free(dimids[0]); - free(dimids[1]); - continue; - } - } - - /* calculate read amount of this process in start[] and shape[] */ - for (j=0; j= nprocs) { - MPI_Offset dimLen = shape[j]; - shape[j] = dimLen / nprocs; - start[j] = shape[j] * rank; - if (rank < dimLen % nprocs) { - start[j] += rank; - shape[j]++; - } - else - start[j] += dimLen % nprocs; - break; - } - } - /* if none of shape[*] >= nprocs, then let all processes compare the - * whole variable - */ - - varsize = 1; - /* block partition the variable along the 1st dimension */ - for (j=0; j= 0) { - err = ncmpi_close(ncid[i]); - HANDLE_ERROR - } - } - if (info != MPI_INFO_NULL) - MPI_Info_free(&info); - - /* summary of the difference */ - MPI_Reduce(&numVarDIFF, &varDIFF, 1, MPI_LONG_LONG_INT, MPI_SUM, 0, comm); - if (rank == 0 && !quiet) { - if (check_header) { - if (numHeadDIFF == 0) - printf("Headers of two files are the same\n"); - else - printf("Number of differences in header %lld\n",numHeadDIFF); - } - if (check_variable_list) { - if (varDIFF == 0) - printf("Compared variable(s) are the same\n"); - else - printf("Compared variables(s) has %lld differences\n",varDIFF); - } - if (check_entire_file) { - if (varDIFF == 0) - printf("All variables of two files are the same\n"); - else - printf("Number of differences in variables %lld\n",varDIFF); - } - } - - if (rank == 0) numDIFF = varDIFF + numHeadDIFF; - MPI_Bcast(&numDIFF, 1, MPI_LONG_LONG_INT, 0, comm); MPI_Finalize(); + exit((numDIFF == 0) ? EXIT_SUCCESS : EXIT_FAILURE); } diff --git a/src/utils/ncmpidiff/ncmpidiff_core.c b/src/utils/ncmpidiff/ncmpidiff_core.c new file mode 100644 index 0000000000..7a4de648b5 --- /dev/null +++ b/src/utils/ncmpidiff/ncmpidiff_core.c @@ -0,0 +1,972 @@ +/* + * Copyright (C) 2025, Northwestern University and Argonne National Laboratory + * See COPYRIGHT notice in top-level directory. + */ + +/* This core subroutines of utility program ncmpidiff. It compares header and + * variables of two files regardless the define order of the variables and + * attributes. + */ + +#ifdef HAVE_CONFIG_H +# include +#endif + +#include +#include +#include +#include +#include /* stat() */ +#include /* stat() */ +#include /* stat() */ +#include /* errno */ +#include /* INFINITY */ +#include /* basename() */ + +#include + +#include +#include + +#include + +#if PNETCDF_PROFILING == 1 +#include +#endif + +#ifndef ubyte +#define ubyte unsigned char +#endif +#ifndef ushort +#define ushort unsigned short +#endif +#ifndef uint +#define uint unsigned int +#endif +#ifndef int64 +#define int64 long long +#endif +#ifndef uint64 +#define uint64 unsigned long long +#endif + +#define PRINT_CMD_OPTS \ + if (first_diff && cmd_opts != NULL) { \ + printf("%s\n", cmd_opts); \ + first_diff = 0; \ + } + +#define OOM_ERROR { \ + fprintf(stderr, "Error: calloc() out of memory at line %d\n",__LINE__); \ + exit(1); \ +} + +#define HANDLE_ERROR { \ + if (err != NC_NOERR) { \ + fprintf(stderr, "Error at %s line %d: (%s)\n", basename(__FILE__), \ + __LINE__, ncmpi_strerror(err)); \ + MPI_Abort(MPI_COMM_WORLD, -1); \ + exit(-1); \ + } \ +} + +#define HANDLE_FILE_ERR(filename) { \ + if (err != NC_NOERR) { \ + fprintf(stderr, "Error at %s line %d: input file %s (%s)\n", \ + basename(__FILE__), __LINE__,filename, ncmpi_strerror(err)); \ + MPI_Abort(MPI_COMM_WORLD, -1); \ + exit(-1); \ + } \ +} + +#define CHECK_GLOBAL_ATT_DIFF_CHAR { \ + int pos; \ + char *b1 = (char *)calloc((attlen[0] + 1) * 2, sizeof(char)); \ + char *b2 = b1 + attlen[0] + 1; \ + if (!b1) OOM_ERROR \ + err = ncmpi_get_att_text(ncid[0], NC_GLOBAL, name[0], b1); \ + HANDLE_ERROR \ + err = ncmpi_get_att_text(ncid[1], NC_GLOBAL, name[0], b2); \ + HANDLE_ERROR \ + for (pos=0; pos= 0) ? (x) : (-x) +#define UABS(x) (x) + +#define CHECK_VAR_DIFF(type, func, xabs) { \ + int pos, isDiff, worst = -1; \ + type *b1, *b2; \ + b1 = (type *)calloc(varsize * 2, sizeof(type)); \ + if (!b1) OOM_ERROR \ + b2 = b1 + varsize; \ + err = ncmpi_get_vara_##func(ncid[0], varid1, start, shape, b1); \ + HANDLE_ERROR \ + err = ncmpi_get_vara_##func(ncid[1], varid2, start, shape, b2); \ + HANDLE_ERROR \ + if (!check_tolerance) { \ + for (pos=0; pos abs_b2) ? abs_b1 : abs_b2; \ + diff = b1[pos] - b2[pos]; \ + diff = (diff >= 0) ? diff : -diff; \ + ratio = diff / abs_max; \ + if (diff <= tolerance_difference || ratio <= tolerance_ratio) \ + continue; \ + /* fail to meet both tolerance errors */ \ + worst = pos; \ + break; \ + } \ + } \ + if (pos != varsize || worst != -1) { /* diff is found */ \ + double v1, v2; \ + if (ndims[0] == 0) { /* scalar variable */ \ + PRINT_CMD_OPTS \ + if (worst == -1) \ + printf("DIFF: scalar variable \"%s\" of type \"%s\"\n", \ + name[0], get_type(xtype[0])); \ + else { \ + v1 = b1[worst]; \ + v2 = b2[worst]; \ + printf("DIFF (tolerance): scalar variable \"%s\" of type \"%s\" of value %g vs %g (difference = %e)\n", \ + name[0], get_type(xtype[0]), v1, v2, v1-v2); \ + } \ + } else { \ + int _i; \ + MPI_Offset *diffStart; \ + diffStart = (MPI_Offset*) malloc(sizeof(MPI_Offset) * ndims[0]); \ + if (worst != -1) pos = worst; \ + v1 = b1[pos]; \ + v2 = b2[pos]; \ + for (_i=ndims[0]-1; _i>=0; _i--) { \ + diffStart[_i] = pos % shape[_i] + start[_i]; \ + pos /= shape[_i]; \ + } \ + PRINT_CMD_OPTS \ + if (worst == -1) \ + printf("DIFF: variable \"%s\" of type \"%s\" at element ["OFFFMT, \ + name[0], get_type(xtype[0]), diffStart[0]); \ + else \ + printf("DIFF (tolerance): variable \"%s\" of type \"%s\" at element ["OFFFMT, \ + name[0], get_type(xtype[0]), diffStart[0]); \ + for (_i=1; _i----------------------------------------------------------*/ +static char* +get_type(int type) +{ + switch (type) { + case NC_BYTE: return "NC_BYTE"; + case NC_CHAR: return "NC_CHAR"; + case NC_SHORT: return "NC_SHORT"; + case NC_INT: return "NC_INT"; + case NC_FLOAT: return "NC_FLOAT"; + case NC_DOUBLE: return "NC_DOUBLE"; + case NC_UBYTE: return "NC_UBYTE"; + case NC_USHORT: return "NC_USHORT"; + case NC_UINT: return "NC_UINT"; + case NC_INT64: return "NC_INT64"; + case NC_UINT64: return "NC_UINT64"; + } + return "NC_NAT"; +} + +/*----< ncmpidiff_core() >---------------------------------------------------*/ +MPI_Offset ncmpidiff_core(const char *file1, + const char *file2, + MPI_Comm comm, + MPI_Info info, + int verbose, + int quiet, + int check_header, + int check_variable_list, + int check_entire_file, + int num_vars, + char **var_names, + int check_tolerance, + int first_diff, + char *cmd_opts, + double tolerance_difference, + double tolerance_ratio) +{ + char *name[2]; + int i, j, err, rank, nprocs; + int ncid[2], ndims[2], nvars[2], natts[2], recdim[2], *dimids[2], fmt[2]; + int cmp_nvars; + long long numVarDIFF=0, numHeadDIFF=0, varDIFF, numDIFF; + MPI_Offset *shape=NULL, varsize, *start=NULL; + MPI_Offset attlen[2], dimlen[2]; + nc_type xtype[2]; + + MPI_Comm_size(comm, &nprocs); + MPI_Comm_rank(comm, &rank); + + ncid[0] = ncid[1] = -1; + + if (verbose && rank == 0) { + printf("First file: %s\n", file1); + printf("Second file: %s\n", file2); + } + + if (strcmp(file1, file2) == 0) { + if (rank == 0) + fprintf(stderr,"Error: two input file names are identical (%s) ... exit\n", file1); + return NC_EINVAL; + } + + /* compare file format */ + err = ncmpi_inq_file_format(file1, &fmt[0]); + HANDLE_FILE_ERR(file1) + err = ncmpi_inq_file_format(file2, &fmt[1]); + HANDLE_FILE_ERR(file2) + + if (fmt[0] != fmt[1]) { + if (!quiet && rank == 0) + printf("DIFF: file format (CDF-%d) != (CDF-%d)\n",fmt[0], fmt[1]); + numHeadDIFF++; + /* even formats are different, we continue to compare the contents + * of the files (headers and variables). + */ + } + + /* open files and retrieve headers into memory buffers */ + name[0] = (char*) calloc(NC_MAX_NAME, 1); + if (!name[0]) OOM_ERROR + name[1] = (char*) calloc(NC_MAX_NAME, 1); + if (!name[1]) OOM_ERROR + + /* open files */ + err = ncmpi_open(comm, file1, NC_NOWRITE, info, &ncid[0]); + HANDLE_FILE_ERR(file1) + err = ncmpi_open(comm, file2, NC_NOWRITE, info, &ncid[1]); + HANDLE_FILE_ERR(file2) + + /* retrieve metadata */ + err = ncmpi_inq(ncid[0], &ndims[0], &nvars[0], &natts[0], &recdim[0]); + HANDLE_ERROR + err = ncmpi_inq(ncid[1], &ndims[1], &nvars[1], &natts[1], &recdim[1]); + HANDLE_ERROR + + /* compare file header */ + if (check_header && rank == 0) { /* only root checks header */ + int attnump; + + /* compare number of dimensions defined */ + if (ndims[0] != ndims[1]) { + if (!quiet) + printf("DIFF: number of dimensions (%d) != (%d)\n",ndims[0], ndims[1]); + numHeadDIFF++; + } + else if (verbose) + printf("SAME: number of dimensions (%d)\n",ndims[0]); + + /* compare number of variables defined */ + if (nvars[0] != nvars[1]) { + if (!quiet) + printf("DIFF: number of variables (%d) != (%d)\n",nvars[0], nvars[1]); + numHeadDIFF++; + } + else if (verbose) + printf("SAME: number of variables (%d)\n",nvars[0]); + + /* compare number of global attributes defined */ + if (natts[0] != natts[1]) { + if (!quiet) + printf("DIFF: number of global attributes (%d) != (%d)\n",natts[0], natts[1]); + numHeadDIFF++; + } + else if (verbose) + printf("SAME: number of global attributes (%d)\n",natts[0]); + + /* compare attributes defined in 1st file and also in 2nd file */ + for (i=0; i 0 && ndims[1] > 0) { + if (verbose) + printf("Dimension:\n"); + } else + goto cmp_vars; + + /* check dimensions in 1st file also appear in 2nd file */ + for (i=0; i 0 && nvars[1] > 0) { + if (verbose) + printf("Variables:\n"); + } else + goto cmp_exit; + + /* check variables defined in 1st file and also in 2nd file */ + for (i=0; i 0 && dimids[0][0] == recdim[0]) { /* record variable */ + err = ncmpi_inq_dimlen(ncid[0], recdim[0], &shape[0]); + HANDLE_ERROR + if (shape[0] == 0) { + /* No record has been written to the file, skip comparison */ + free(shape); + free(dimids[0]); + free(dimids[1]); + continue; + } + } + + /* calculate read amount of this process in start[] and shape[] */ + for (j=0; j= nprocs) { + MPI_Offset dimLen = shape[j]; + shape[j] = dimLen / nprocs; + start[j] = shape[j] * rank; + if (rank < dimLen % nprocs) { + start[j] += rank; + shape[j]++; + } + else + start[j] += dimLen % nprocs; + break; + } + } + /* if none of shape[*] >= nprocs, then let all processes compare the + * whole variable + */ + + varsize = 1; + /* block partition the variable along the 1st dimension */ + for (j=0; j= 0) { +#if PNETCDF_PROFILING == 1 + /* disable printing of profiling timers */ + pnc_ina_npairs_put = pnc_ina_npairs_get = 0; +#endif + err = ncmpi_close(ncid[i]); + HANDLE_ERROR + } + } + + /* summary of the difference */ + MPI_Reduce(&numVarDIFF, &varDIFF, 1, MPI_LONG_LONG_INT, MPI_SUM, 0, comm); + if (rank == 0 && !quiet) { + if (check_header) { + if (numHeadDIFF == 0) + printf("Headers of two files are the same\n"); + else + printf("Number of differences in header %lld\n",numHeadDIFF); + } + if (check_variable_list) { + if (varDIFF == 0) + printf("Compared variable(s) are the same\n"); + else + printf("Compared variables(s) has %lld differences\n",varDIFF); + } + if (check_entire_file) { + if (varDIFF == 0) + printf("All variables of two files are the same\n"); + else + printf("Number of differences in variables %lld\n",varDIFF); + } + } + + if (rank == 0) numDIFF = varDIFF + numHeadDIFF; + MPI_Bcast(&numDIFF, 1, MPI_LONG_LONG_INT, 0, comm); + + return numDIFF; +} diff --git a/src/utils/ncmpidiff/ncmpidiff_core.h b/src/utils/ncmpidiff/ncmpidiff_core.h new file mode 100644 index 0000000000..d98faaefb2 --- /dev/null +++ b/src/utils/ncmpidiff/ncmpidiff_core.h @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2025, Northwestern University and Argonne National Laboratory + * See COPYRIGHT notice in top-level directory. + */ + +#include + +#ifndef OFFFMT +#define OFFFMT "%lld" +#endif + +extern +MPI_Offset ncmpidiff_core(const char *file1, + const char *file2, + MPI_Comm comm, + MPI_Info info, + int verbose, + int quiet, + int check_header, + int check_variable_list, + int check_entire_file, + int num_vars, + char **var_names, + int check_tolerance, + int first_diff, + char *cmd_opts, + double tolerance_difference, + double tolerance_ratio); diff --git a/src/utils/ncmpidiff/tst_file.nc b/src/utils/ncmpidiff/tst_file.nc new file mode 100644 index 0000000000000000000000000000000000000000..f6fca87d2e1582473f8f394b89e9d6bcfcc4c451 GIT binary patch literal 800 zcmeH@xeCHi5Jkt0fcgQJ`2-UM#m2ZGF4$|Mh`6LsAued=r=_qE5x>cMU-Du~v3B5O zPTt%%gtXe7goq?T7NkJ>;BFS^ti(9zY5FMeJR0FWW9{w_JMY2?cNB)Zt=TxPOsizt zR?V)|%2m@YT2;()VaD(l^XX~k;Z@yD}xO1`!bENxZ(vH}4`554h$D*Sz2%t*!{U QP0)J*%KLDjd;kaZH_~81+5i9m literal 0 HcmV?d00001 diff --git a/src/utils/ncmpidiff/xfail_runs.sh b/src/utils/ncmpidiff/xfail_runs.sh new file mode 100755 index 0000000000..1095c8ab56 --- /dev/null +++ b/src/utils/ncmpidiff/xfail_runs.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory +# See COPYRIGHT notice in top-level directory. +# + +# Exit immediately if a command exits with a non-zero status. +set -e + +DRY_RUN=no +VERBOSE=yes + +if test "x$1" = "x./ncmpidiff" ; then + RUN_CMD=`echo ${TESTMPIRUN} | ${SED} -e "s/NP/4/g"` +else + RUN_CMD=$TESTSEQRUN +fi + +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $RUN_CMD $@" + fi + if test "x$DRY_RUN" = xno ; then + $RUN_CMD $@ + fi +} + +exe_name=`basename $1` + +# prevent user environment setting of PNETCDF_HINTS to interfere +unset PNETCDF_HINTS + +# ncmpidiff/ncdiff should error out when two input file names are identical. +IDENT_FILE=${srcdir}/tst_file.nc + +run_cmd $1 ${IDENT_FILE} ${IDENT_FILE} + diff --git a/src/utils/ncmpidump/ncmpidump.c b/src/utils/ncmpidump/ncmpidump.c index be3d72482b..7a38c231b0 100644 --- a/src/utils/ncmpidump/ncmpidump.c +++ b/src/utils/ncmpidump/ncmpidump.c @@ -15,6 +15,11 @@ #include #include /* open() */ #include /* read(), close() */ + +#if defined(HAVE_GETOPT_H) && HAVE_GETOPT_H == 1 +#include /* getopt() */ +#endif + #include /* errno */ #include @@ -23,7 +28,7 @@ #include "dumplib.h" #include "vardata.h" -#ifdef ENABLE_ADIOS +#if PNETCDF_DRIVER_ADIOS == 1 #include "adios_read.h" #include #define BP_MINIFOOTER_SIZE 28 @@ -51,16 +56,16 @@ static void pr_att_string(size_t len, const char* string); static void pr_att_vals(nc_type type, size_t len, const double* vals); static void pr_att(int ncid, int varid, const char *varname, int ia); static void do_ncdump(const char* path, struct fspec* specp); -static void make_lvars(char* optarg, struct fspec* fspecp); -static void set_sigdigs( const char* optarg); -static void set_precision( const char *optarg); +static void make_lvars(char* opt_arg, struct fspec* fspecp); +static void set_sigdigs( const char* opt_arg); +static void set_precision( const char *opt_arg); int main(int argc, char** argv); #define STREQ(a, b) (*(a) == *(b) && strcmp((a), (b)) == 0) char *progname; -#ifdef ENABLE_ADIOS +#if PNETCDF_DRIVER_ADIOS == 1 unsigned int bp_ver; #endif @@ -417,11 +422,11 @@ do_ncdump(const char *path, struct fspec* specp) Printf ("%s file format: CDF-5 (big variables)\n", specp->name); else if (NC_mode == NC_64BIT_OFFSET) Printf ("%s file format: CDF-2 (large file)\n", specp->name); -#ifdef ENABLE_NETCDF4 +#if PNETCDF_DRIVER_NETCDF4 == 1 else if (NC_mode == NC_NETCDF4) Printf ("%s file format: NetCDF-4\n", specp->name); #endif -#ifdef ENABLE_ADIOS +#if PNETCDF_DRIVER_ADIOS == 1 else if (NC_mode == NC_BP) Printf ("%s file format: ADIOS BP Ver. %u\n", specp->name, bp_ver); #endif @@ -441,11 +446,11 @@ do_ncdump(const char *path, struct fspec* specp) Printf ("// file format: CDF-5 (big variables)\n"); else if (NC_mode == NC_64BIT_OFFSET) Printf ("// file format: CDF-2 (large file)\n"); -#ifdef ENABLE_NETCDF4 +#if PNETCDF_DRIVER_NETCDF4 == 1 else if (NC_mode == NC_NETCDF4) Printf ("// file format: NetCDF-4\n"); #endif -#ifdef ENABLE_ADIOS +#if PNETCDF_DRIVER_ADIOS == 1 else if (NC_mode == NC_BP) Printf ("// file format: ADIOS BP Ver. %u\n", bp_ver); #endif @@ -611,9 +616,9 @@ do_ncdump(const char *path, struct fspec* specp) static void -make_lvars(char *optarg, struct fspec* fspecp) +make_lvars(char *opt_arg, struct fspec* fspecp) { - char *cp = optarg; + char *cp = opt_arg; int nvars = 1; char ** cpp; @@ -628,7 +633,7 @@ make_lvars(char *optarg, struct fspec* fspecp) cpp = fspecp->lvars; /* copy variable names into list */ - for (cp = strtok(optarg, ","); + for (cp = strtok(opt_arg, ","); cp != NULL; cp = strtok((char *) NULL, ",")) { @@ -647,15 +652,15 @@ make_lvars(char *optarg, struct fspec* fspecp) * command-line and update the default data formats appropriately. */ static void -set_sigdigs(const char *optarg) +set_sigdigs(const char *opt_arg) { char *ptr1 = 0; char *ptr2 = 0; int flt_digits = FLT_DIGITS; /* default floating-point digits */ int dbl_digits = DBL_DIGITS; /* default double-precision digits */ - if (optarg != 0 && (int) strlen(optarg) > 0 && optarg[0] != ',') - flt_digits = (int)strtol(optarg, &ptr1, 10); + if (opt_arg != 0 && (int) strlen(opt_arg) > 0 && opt_arg[0] != ',') + flt_digits = (int)strtol(opt_arg, &ptr1, 10); if (flt_digits < 1 || flt_digits > 20) error("unreasonable value for float significant digits: %d", @@ -679,15 +684,15 @@ set_sigdigs(const char *optarg) * and update the default data formats appropriately. */ static void -set_precision(const char *optarg) +set_precision(const char *opt_arg) { char *ptr1 = 0; char *ptr2 = 0; int flt_digits = FLT_DIGITS; /* default floating-point digits */ int dbl_digits = DBL_DIGITS; /* default double-precision digits */ - if (optarg != 0 && (int) strlen(optarg) > 0 && optarg[0] != ',') { - flt_digits = (int)strtol(optarg, &ptr1, 10); + if (opt_arg != 0 && (int) strlen(opt_arg) > 0 && opt_arg[0] != ',') { + flt_digits = (int)strtol(opt_arg, &ptr1, 10); float_precision_specified = 1; } @@ -716,7 +721,7 @@ enum FILE_KIND { UNKNOWN }; -#ifdef ENABLE_ADIOS +#if PNETCDF_DRIVER_ADIOS == 1 static int adios_parse_version (char *footer, unsigned int *version, int *diff_endianness) { unsigned int test = 1; /* If high bit set, big endian */ @@ -771,7 +776,7 @@ enum FILE_KIND check_file_signature(char *path) else if (signature[3] == 2) return CDF2; else if (signature[3] == 1) return CDF1; } -#ifdef ENABLE_ADIOS +#if PNETCDF_DRIVER_ADIOS == 1 else{ off_t fsize; int diff_endian; diff --git a/src/utils/ncmpidump/ncmpidump.h b/src/utils/ncmpidump/ncmpidump.h index 290c16e91a..1d1604f543 100644 --- a/src/utils/ncmpidump/ncmpidump.h +++ b/src/utils/ncmpidump/ncmpidump.h @@ -15,10 +15,15 @@ #define Printf (void) printf +#if defined(HAVE_STDBOOL_H) && HAVE_STDBOOL_H == 1 +#include /* type false and true */ +typedef bool boolean; +#else typedef int boolean; #if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 202311L enum {false=0, true=1}; #endif +#endif struct ncdim { /* dimension */ char name[NC_MAX_NAME]; diff --git a/src/utils/ncmpidump/vardata.c b/src/utils/ncmpidump/vardata.c index 11310a9f91..b8cec8b995 100644 --- a/src/utils/ncmpidump/vardata.c +++ b/src/utils/ncmpidump/vardata.c @@ -82,9 +82,9 @@ static double double_eps; static float float_epsilon(void) { - float float_eps; + float val_float_eps; #ifndef NO_FLOAT_H - float_eps = FLT_EPSILON; + val_float_eps = FLT_EPSILON; #else /* NO_FLOAT_H */ { float etop, ebot, eps; @@ -103,19 +103,19 @@ float_epsilon(void) ebot = eps; eps = ebot + (etop - ebot)/two; } - float_eps = two * etop; + val_float_eps = two * etop; } #endif /* NO_FLOAT_H */ - return float_eps; + return val_float_eps; } static double double_epsilon(void) { - double double_eps; + double val_double_eps; #ifndef NO_FLOAT_H - double_eps = DBL_EPSILON; + val_double_eps = DBL_EPSILON; #else /* NO_FLOAT_H */ { double etop, ebot, eps; @@ -134,10 +134,10 @@ double_epsilon(void) ebot = eps; eps = ebot + (etop - ebot)/two; } - double_eps = two * etop; + val_double_eps = two * etop; } #endif /* NO_FLOAT_H */ - return double_eps; + return val_double_eps; } diff --git a/src/utils/ncmpigen/genlib.c b/src/utils/ncmpigen/genlib.c index 1e558a92fb..26ac82d890 100644 --- a/src/utils/ncmpigen/genlib.c +++ b/src/utils/ncmpigen/genlib.c @@ -1548,7 +1548,6 @@ cl_fortran(void) } fline(stmnt); if (v->type != NC_CHAR) { - char *sp; sprintf(stmnt, "%s %s(", ncftype(v->type), v->lname); /* reverse dimensions for FORTRAN */ @@ -1582,12 +1581,12 @@ cl_fortran(void) if (v->has_data) { fline(v->data_stmnt); } else { /* generate data statement for FILL record */ - MPI_Offset rec_len = 1; + MPI_Offset rec_size = 1; for (idim = 1; idim < v->ndims; idim++) { - rec_len *= dims[v->dims[idim]].size; + rec_size *= dims[v->dims[idim]].size; } sprintf(stmnt,"data %s /%lu * %s/", v->lname, - (unsigned long) rec_len, + (unsigned long) rec_size, f_fill_name(v->type)); fline(stmnt); } @@ -1695,9 +1694,9 @@ close_netcdf(void) void -check_err(int stat, const char *ncmpi_func, const char *calling_func, int lineno, const char *calling_file) { +check_err(int stat, const char *ncmpi_func, const char *calling_func, int linenum, const char *calling_file) { if (stat != NC_NOERR) { - fprintf(stderr, "ncmpigen error when calling %s in %s() at line %d of %s: %s\n", ncmpi_func, calling_func, lineno, calling_file, ncmpi_strerror(stat)); + fprintf(stderr, "ncmpigen error when calling %s in %s() at line %d of %s: %s\n", ncmpi_func, calling_func, linenum, calling_file, ncmpi_strerror(stat)); derror_count++; } } diff --git a/src/utils/ncmpigen/load.c b/src/utils/ncmpigen/load.c index 69fe54e3a9..788450aa26 100644 --- a/src/utils/ncmpigen/load.c +++ b/src/utils/ncmpigen/load.c @@ -394,7 +394,7 @@ fstrcat( */ static void f_var_init( - int varnum, /* which variable */ + int varid, /* which variable */ void *rec_start /* start of data */ ) { @@ -415,9 +415,9 @@ f_var_init( int ival; /* load variable with data values */ - sprintf(stmnt, "data %s /",vars[varnum].lname); + sprintf(stmnt, "data %s /",vars[varid].lname); stmnt_len = strlen(stmnt); - switch (vars[varnum].type) { + switch (vars[varid].type) { case NC_BYTE: charvalp = (char *) rec_start; for (ival = 0; ival < var_len-1; ival++) { @@ -524,10 +524,10 @@ f_var_init( /* For record variables, store data statement for later use; otherwise, just print it. */ - if (vars[varnum].ndims > 0 && vars[varnum].dims[0] == rec_dim) { + if (vars[varid].ndims > 0 && vars[varid].dims[0] == rec_dim) { char *dup_stmnt = (char*) emalloc(strlen(stmnt)+1); strcpy(dup_stmnt, stmnt); /* ULTRIX missing strdup */ - vars[varnum].data_stmnt = dup_stmnt; + vars[varid].data_stmnt = dup_stmnt; } else { fline(stmnt); } diff --git a/src/utils/ncmpigen/ncmpigen.1 b/src/utils/ncmpigen/ncmpigen.1 index 4e42a5affe..e67b3035ed 100644 --- a/src/utils/ncmpigen/ncmpigen.1 +++ b/src/utils/ncmpigen/ncmpigen.1 @@ -355,8 +355,6 @@ For example the following are all acceptable \fIdouble\fP constants: 1.d .fi .RE -.SH DATE -PNETCDF_RELEASE_DATE .SH BUGS .LP The programs generated by \fBncmpigen\fP when using the \fB-c\fP or \fB-f\fP @@ -371,3 +369,12 @@ concatenated into a single array of characters, since netCDF cannot represent an array of variable-length strings in one netCDF variable. .LP NetCDF and CDL do not yet support a type corresponding to a 64-bit integer. +.SH "SEE ALSO" +.LP +.BR ncmpidiff (1), +.BR ncmpidump (1), +.BR ncgen (1), +.BR pnetcdf (3) +.SH DATE +PNETCDF_RELEASE_DATE +.LP diff --git a/src/utils/ncmpigen/ncmpigentab.c b/src/utils/ncmpigen/ncmpigentab.c index 117e7d4945..0699867887 100644 --- a/src/utils/ncmpigen/ncmpigentab.c +++ b/src/utils/ncmpigen/ncmpigentab.c @@ -1,6 +1,8 @@ +/* #ifndef lint static const char yysccsid[] = "@(#)yaccpar 1.9 (Berkeley) 02/21/93"; #endif +*/ #include #include @@ -617,7 +619,6 @@ static int yygrowstack(void) #define YYABORT goto yyabort #define YYREJECT goto yyabort #define YYACCEPT goto yyaccept -#define YYERROR goto yyerrlab int yyparse(void) { @@ -686,11 +687,6 @@ yyparse(void) yyerror("syntax error"); -#ifdef lint - goto yyerrlab; -#endif - -yyerrlab: ++yynerrs; yyinrecovery: diff --git a/src/utils/ncmpilogdump/ncmpilogdump.m4 b/src/utils/ncmpilogdump/ncmpilogdump.m4 index 78e293131c..391049518c 100644 --- a/src/utils/ncmpilogdump/ncmpilogdump.m4 +++ b/src/utils/ncmpilogdump/ncmpilogdump.m4 @@ -227,7 +227,7 @@ fn_exit: case NC_NOERR: break; default: - printf("Unknown error\n"); + fprintf(stderr,"Unknown error\n"); } return 0; diff --git a/src/utils/ncoffsets/ncoffsets.c b/src/utils/ncoffsets/ncoffsets.c index 977f199dd2..0dc1ff75f7 100644 --- a/src/utils/ncoffsets/ncoffsets.c +++ b/src/utils/ncoffsets/ncoffsets.c @@ -14,6 +14,7 @@ #include /* read() */ #include /* assert() */ #include /* check for Endianness, uint32_t*/ +#include /* uint32_t, uint64_t */ static int is_little_endian; @@ -225,38 +226,29 @@ static int check_little_endian(void) { (((a) & 0x00FF000000000000ULL) >> 40) | \ (((a) & 0xFF00000000000000ULL) >> 56) ) -static void -swap4b(void *val) -{ - uint32_t *op = (uint32_t*)val; - *op = SWAP4B(*op); -} - -static void -swap8b(unsigned long long *val) -{ - uint64_t *op = (uint64_t*)val; - *op = SWAP8B(*op); -} static unsigned long long get_uint64(bufferinfo *gbp) { - /* retrieve a 64bit unisgned integer and return it as unsigned long long */ - unsigned long long tmp; - memcpy(&tmp, gbp->pos, 8); - if (is_little_endian) swap8b(&tmp); - gbp->pos = (char*)gbp->pos + 8; - return tmp; + /* retrieve a 64bit unsigned integer and return it as unsigned long long */ + uint64_t tmp; + memcpy(&tmp, gbp->pos, sizeof(uint64_t)); + + if (is_little_endian) tmp = SWAP8B(tmp); + + gbp->pos = (char*)gbp->pos + sizeof(uint64_t); + return (unsigned long long)tmp; } static unsigned int get_uint32(bufferinfo *gbp) { - /* retrieve a 32bit unisgned integer and return it as unsigned int */ - unsigned int tmp; - memcpy(&tmp, gbp->pos, 4); - if (is_little_endian) swap4b(&tmp); - gbp->pos = (char*)gbp->pos + 4; - return tmp; + /* retrieve a 32bit unsigned integer and return it as unsigned int */ + uint32_t tmp; + memcpy(&tmp, gbp->pos, sizeof(uint32_t)); + + if (is_little_endian) tmp = SWAP4B(tmp); + + gbp->pos = (char*)gbp->pos + sizeof(uint32_t); + return (unsigned int)tmp; } static int @@ -471,8 +463,7 @@ ncmpii_NC_computeshapes(NC *ncp) ncp->recsize += (*vpp)->len; } else { - if (first_var == NULL) - first_var = *vpp; + if (first_var == NULL) first_var = *vpp; /* * Overwritten each time thru. * Usually overwritten in first_rec != NULL clause. @@ -691,7 +682,7 @@ hdr_len_NC_var(const NC_var *varp, */ sz = hdr_len_NC_name(varp->name, sizeof_t); /* name */ sz += sizeof_t; /* nelems */ - sz += sizeof_t * varp->ndims; /* [dimid ...] */ + sz += (long long)sizeof_t * varp->ndims; /* [dimid ...] */ sz += hdr_len_NC_attrarray(&varp->attrs, sizeof_t); /* vatt_list */ sz += X_SIZEOF_NC_TYPE; /* nc_type */ sz += sizeof_t; /* vsize */ @@ -1749,31 +1740,6 @@ ncmpii_free_NC(NC *ncp) free(ncp->path); } -#if 0 -const char * -ncmpi_strerror(int err) -{ - switch(err) - { - case NC_NOERR: - return "No error"; - case NC_EINVAL: - return "NetCDF: Invalid argument"; - case NC_EBADDIM: - return "NetCDF: Invalid dimension ID or name"; - case NC_EUNLIMPOS: - return "NetCDF: NC_UNLIMITED in the wrong index"; - case NC_ENOTNC: - return "NetCDF: Unknown file format"; - case NC_EVARSIZE: - return "NetCDF: One or more variable sizes violate format constraints"; - default: - printf("Unknown error code %d\n",err); - return "Unknown error code"; - } -} -#endif - const char * ncmpii_err_code_name(int err) { @@ -1787,7 +1753,7 @@ ncmpii_err_code_name(int err) case NC_EUNLIMIT: return "NC_EUNLIMIT"; case NC_EVARSIZE : return "NC_EVARSIZE"; default: - printf("Unknown error code %d\n",err); + fprintf(stderr,"Unknown error code %d\n",err); return "Unknown error code"; } } @@ -1802,9 +1768,9 @@ struct fspec { }; static void -make_lvars(char *optarg, struct fspec* fspecp) +make_lvars(char *opt_arg, struct fspec* fspecp) { - char *cp = optarg; + char *cp = opt_arg; int nvars = 1; char ** cpp; @@ -1819,7 +1785,7 @@ make_lvars(char *optarg, struct fspec* fspecp) cpp = fspecp->lvars; /* copy variable names into list */ - for (cp = strtok(optarg, ","); + for (cp = strtok(opt_arg, ","); cp != NULL; cp = strtok((char *) NULL, ",")) { @@ -1879,8 +1845,9 @@ static void usage(char *cmd) { char *help = -"Usage: %s [-h] | [-x] | [-sgr] [-v var1[,...]] file\n" +"Usage: %s [-h | -d | -x | -s | -g | -r | -v var1[,...]] file\n" " [-h] Print help\n" +" [-d] Enable debug mode (verbose output)\n" " [-v var1[,...]] Output for variable(s) ,... only\n" " [-s] Output variable size. For record variables, output\n" " the size of one record only\n" @@ -1898,7 +1865,7 @@ int main(int argc, char *argv[]) { extern int optind; extern char *optarg; - char *filename, *env_str; + char *filename; int i, j, err, opt; int print_var_size=0, print_gap=0, check_gap=0, print_all_rec=0; NC *ncp; @@ -1906,8 +1873,10 @@ int main(int argc, char *argv[]) fspecp = (struct fspec*) calloc(1, sizeof(struct fspec)); + verbose_debug = 0; + /* get command-line arguments */ - while ((opt = getopt(argc, argv, "v:sghqxr")) != EOF) { + while ((opt = getopt(argc, argv, "v:dsghqxr")) != EOF) { switch(opt) { case 'v': make_lvars(optarg, fspecp); break; @@ -1919,6 +1888,8 @@ int main(int argc, char *argv[]) break; case 'x': check_gap = 1; break; + case 'd': verbose_debug = 1; + break; case 'h': default: usage(argv[0]); free(fspecp); @@ -1937,17 +1908,13 @@ int main(int argc, char *argv[]) } filename = argv[optind]; /* required argument */ - verbose_debug = 0; - env_str = getenv("PNETCDF_VERBOSE_DEBUG_MODE"); - if (env_str != NULL && *env_str != '0') verbose_debug = 1; - /* find Endianness of the running machine */ is_little_endian = check_little_endian(); /* open file */ int fd = open(filename, O_RDONLY, 0666); if (fd == -1) { - printf("Error: file open %s (%s)\n",filename,strerror(errno)); + fprintf(stderr,"Error: file open %s (%s)\n",filename,strerror(errno)); exit(1); } @@ -1992,7 +1959,7 @@ int main(int argc, char *argv[]) } } if (j == ncp->vars.ndefined) { - printf("Error: variable %s not found\n",fspecp->lvars[i]); + fprintf(stderr,"Error: variable %s not found\n",fspecp->lvars[i]); free(fspecp->varids); free(fspecp->varp); for (i=0; inlvars; i++) @@ -2070,7 +2037,7 @@ int main(int argc, char *argv[]) /* print fixed-size variables first */ if (num_fix_vars) printf("\nfixed-size variables:\n"); for (i=0; inlvars; i++) { - int j, ndims, cdots; + int ndims, cdots; char type_str[16], str[1024], *line; size_t lineLen; long long size; @@ -2162,7 +2129,7 @@ int main(int argc, char *argv[]) /* print record variables */ if (num_rec_vars) printf("\nrecord variables:\n"); for (i=0; inlvars; i++) { - int j, ndims, cdots; + int ndims, cdots; char type_str[16], str[1024], *line; size_t lineLen; long long var_begin, var_end, size, numrecs; diff --git a/src/utils/ncvalidator/Makefile.am b/src/utils/ncvalidator/Makefile.am index ee40ce2e7b..2f3a1f6bbb 100644 --- a/src/utils/ncvalidator/Makefile.am +++ b/src/utils/ncvalidator/Makefile.am @@ -11,10 +11,6 @@ SUFFIXES = .o .c AM_CPPFLAGS = -I$(top_srcdir)/src/include AM_CPPFLAGS += -I$(top_builddir)/src/include -if PNETCDF_DEBUG - AM_CPPFLAGS += -DPNETCDF_DEBUG -endif - bin_PROGRAMS = ncvalidator ncvalidator_SOURCES = ncvalidator.c @@ -93,10 +89,12 @@ TESTS_ENVIRONMENT += ENOTNC_FILES="$(ENOTNC_FILES)" ; export ENOTNC_FILES; TESTS_ENVIRONMENT += EVARSIZE_FILES="$(EVARSIZE_FILES)" ; export EVARSIZE_FILES; TESTS_ENVIRONMENT += TST_HDF5_FILES="$(TST_HDF5_FILES)" ; export TST_HDF5_FILES; -TESTS = xfail_runs.sh wrap_runs.sh +TESTS = xfail_runs.sh seq_runs.sh XFAIL_TESTS = xfail_runs.sh -EXTRA_DIST = xfail_runs.sh wrap_runs.sh \ +check_SCRIPTS = xfail_runs.sh seq_runs.sh + +EXTRA_DIST = xfail_runs.sh seq_runs.sh \ $(ENULLPAD_FILES) $(EMAXVARS_FILES) \ $(EUNLIMIT_FILES) $(ENOTNC_FILES) $(EVARSIZE_FILES) \ $(TST_HDF5_FILES) diff --git a/src/utils/ncvalidator/ncvalidator.c b/src/utils/ncvalidator/ncvalidator.c index da58bcf6c4..420e1f6277 100644 --- a/src/utils/ncvalidator/ncvalidator.c +++ b/src/utils/ncvalidator/ncvalidator.c @@ -14,6 +14,7 @@ #include /* check for Endianness, uint32_t*/ #include #include /* errno */ +#include /* uint32_t, uint64_t */ #define X_ALIGN 4 #define X_INT_MAX 2147483647 @@ -55,14 +56,14 @@ static const char *off_limit = "https://docs.unidata.ucar.edu/nug/current/file_s #define IS_RECVAR(vp) ((vp)->shape != NULL ? (*(vp)->shape == NC_UNLIMITED) : 0 ) -#ifdef PNETCDF_DEBUG +#if PNETCDF_DEBUG_MODE == 1 #define DEBUG_RETURN_ERROR(err) { \ - if (verbose) printf("\t(Error %s at line %d in file %s)\n", \ + if (verbose) fprintf(stderr,"\t(Error %s at line %d in file %s)\n", \ #err,__LINE__,__FILE__); \ return err; \ } #define DEBUG_ASSIGN_ERROR(status, err) { \ - if (verbose) printf("\t(Error %s at line %d in file %s)\n", \ + if (verbose) fprintf(stderr,"\t(Error %s at line %d in file %s)\n", \ #err,__LINE__,__FILE__); \ status = err; \ } @@ -230,40 +231,30 @@ static int check_little_endian(void) return (*((uint8_t*)(&i))) == 0x67; } -static void -swap4b(void *val) -{ - uint32_t *op = (uint32_t*)val; - *op = SWAP4B(*op); -} - -static void -swap8b(unsigned long long *val) -{ - uint64_t *op = (uint64_t*)val; - *op = SWAP8B(*op); -} - static unsigned long long get_uint64(bufferinfo *gbp) { /* retrieve a 64bit unsigned integer and return it as unsigned long long */ - unsigned long long tmp; - memcpy(&tmp, gbp->pos, 8); - if (gbp->is_little_endian) swap8b(&tmp); - gbp->pos = (char*)gbp->pos + 8; /* advance gbp->pos 8 bytes */ - return tmp; + uint64_t tmp; + memcpy(&tmp, gbp->pos, sizeof(uint64_t)); + + if (gbp->is_little_endian) tmp = SWAP8B(tmp); + + gbp->pos = (char*)gbp->pos + sizeof(uint64_t); /* advance gbp->pos 8 bytes */ + return (unsigned long long)tmp; } static unsigned int get_uint32(bufferinfo *gbp) { /* retrieve a 32bit unsigned integer and return it as unsigned int */ - unsigned int tmp; - memcpy(&tmp, gbp->pos, 4); - if (gbp->is_little_endian) swap4b(&tmp); - gbp->pos = (char*)gbp->pos + 4; /* advance gbp->pos 4 bytes */ - return tmp; + uint32_t tmp; + memcpy(&tmp, gbp->pos, sizeof(uint32_t)); + + if (gbp->is_little_endian) tmp = SWAP4B(tmp); + + gbp->pos = (char*)gbp->pos + sizeof(uint32_t); /* advance gbp->pos 4 bytes */ + return (unsigned int)tmp; } static void @@ -668,14 +659,14 @@ var_shape64(NC_var *varp, const NC_dim *dimp; if (varp->dimids[i] < 0) { - if (verbose) printf("Error:\n"); - if (verbose) printf("\t%s: dimension ID [%d] invalid (%d)\n",loc,i,varp->dimids[i]); + if (verbose) fprintf(stderr,"Error:\n"); + if (verbose) fprintf(stderr,"\t%s: dimension ID [%d] invalid (%d)\n",loc,i,varp->dimids[i]); DEBUG_RETURN_ERROR(NC_EBADDIM); } if (varp->dimids[i] >= ((dims != NULL) ? dims->ndefined : 1)) { - if (verbose) printf("Error:\n"); - if (verbose) printf("\t%s: dimension ID [%d] (%d) larger than defined (%d)\n",loc,i,varp->dimids[i], ((dims != NULL) ? dims->ndefined : 1)); + if (verbose) fprintf(stderr,"Error:\n"); + if (verbose) fprintf(stderr,"\t%s: dimension ID [%d] (%d) larger than defined (%d)\n",loc,i,varp->dimids[i], ((dims != NULL) ? dims->ndefined : 1)); DEBUG_RETURN_ERROR(NC_EBADDIM); } @@ -686,8 +677,8 @@ var_shape64(NC_var *varp, /* check for record variable, only the highest dimension can * be unlimited */ if (varp->shape[i] == NC_UNLIMITED && i != 0) { - if (verbose) printf("Error:\n"); - if (verbose) printf("\t%s: dimension ID [%d] is NC_UNLIMITED in the wrong index\n",loc,i); + if (verbose) fprintf(stderr,"Error:\n"); + if (verbose) fprintf(stderr,"\t%s: dimension ID [%d] is NC_UNLIMITED in the wrong index\n",loc,i); DEBUG_RETURN_ERROR(NC_EUNLIMPOS); } } @@ -762,13 +753,13 @@ compute_var_shape(NC *ncp) /* check if dimids are valid */ for (j=0; jvars.value[i]->ndims; j++) { if (ncp->vars.value[i]->dimids[j] < 0) { - if (verbose) printf("Error:\n"); - if (verbose) printf("\t%s: dimension ID [%d] invalid (%d)\n",xloc,i,ncp->vars.value[i]->dimids[i]); + if (verbose) fprintf(stderr,"Error:\n"); + if (verbose) fprintf(stderr,"\t%s: dimension ID [%d] invalid (%d)\n",xloc,i,ncp->vars.value[i]->dimids[i]); DEBUG_RETURN_ERROR(NC_EBADDIM) /* dimid is not defined */ } else if (ncp->vars.value[i]->dimids[j] >= ncp->dims.ndefined) { - if (verbose) printf("Error:\n"); - if (verbose) printf("\t%s: dimension ID [%d] (%d) larger than defined (%d)\n",xloc,i,ncp->vars.value[i]->dimids[i], ncp->dims.ndefined); + if (verbose) fprintf(stderr,"Error:\n"); + if (verbose) fprintf(stderr,"\t%s: dimension ID [%d] (%d) larger than defined (%d)\n",xloc,i,ncp->vars.value[i]->dimids[i], ncp->dims.ndefined); DEBUG_RETURN_ERROR(NC_EBADDIM); } } @@ -790,8 +781,8 @@ compute_var_shape(NC *ncp) if (first_rec != NULL) { if (ncp->begin_rec > first_rec->begin) { - if (verbose) printf("Error:\n"); - if (verbose) printf("\tbegin of record section (%lld) greater than the begin of first record (%lld)\n",ncp->begin_rec, first_rec->begin); + if (verbose) fprintf(stderr,"Error:\n"); + if (verbose) fprintf(stderr,"\tbegin of record section (%lld) greater than the begin of first record (%lld)\n",ncp->begin_rec, first_rec->begin); DEBUG_RETURN_ERROR(NC_ENOTNC) /* not a netCDF file or corrupted */ } @@ -809,23 +800,23 @@ compute_var_shape(NC *ncp) ncp->begin_var = ncp->begin_rec; if (ncp->begin_var <= 0) { - if (verbose) printf("Error:\n"); - if (verbose) printf("\tbegin of variable section (%lld) is negative\n",ncp->begin_var); + if (verbose) fprintf(stderr,"Error:\n"); + if (verbose) fprintf(stderr,"\tbegin of variable section (%lld) is negative\n",ncp->begin_var); DEBUG_RETURN_ERROR(NC_ENOTNC) /* not a netCDF file or corrupted */ } else if (ncp->xsz > ncp->begin_var) { - if (verbose) printf("Error:\n"); - if (verbose) printf("\tfile header size (%lld) is larger than the begin of data section (%lld)\n",ncp->xsz, ncp->begin_var); + if (verbose) fprintf(stderr,"Error:\n"); + if (verbose) fprintf(stderr,"\tfile header size (%lld) is larger than the begin of data section (%lld)\n",ncp->xsz, ncp->begin_var); DEBUG_RETURN_ERROR(NC_ENOTNC) /* not a netCDF file or corrupted */ } else if (ncp->begin_rec <= 0) { - if (verbose) printf("Error:\n"); - if (verbose) printf("\tbegin of record section (%lld) is zero or negative\n",ncp->begin_rec); + if (verbose) fprintf(stderr,"Error:\n"); + if (verbose) fprintf(stderr,"\tbegin of record section (%lld) is zero or negative\n",ncp->begin_rec); DEBUG_RETURN_ERROR(NC_ENOTNC) /* not a netCDF file or corrupted */ } else if (ncp->begin_var > ncp->begin_rec) { - if (verbose) printf("Error:\n"); - if (verbose) printf("\tbegin of data section (%lld) is larger than record section (%lld)\n",ncp->begin_var, ncp->begin_rec); + if (verbose) fprintf(stderr,"Error:\n"); + if (verbose) fprintf(stderr,"\tbegin of data section (%lld) is larger than record section (%lld)\n",ncp->begin_var, ncp->begin_rec); DEBUG_RETURN_ERROR(NC_ENOTNC) /* not a netCDF file or corrupted */ } @@ -842,18 +833,18 @@ val_repair(int fd, off_t offset, size_t len, void *buf) if (-1 == lseek(fd, offset, SEEK_SET)) { if (verbose) - printf("Error at line %d: lseek %s\n",__LINE__,strerror(errno)); + fprintf(stderr,"Error at line %d: lseek %s\n",__LINE__,strerror(errno)); return -1; } nn = write(fd, buf, len); if (nn == -1) { if (verbose) - printf("Error at line %d: write %s\n",__LINE__,strerror(errno)); + fprintf(stderr,"Error at line %d: write %s\n",__LINE__,strerror(errno)); return -1; } if (nn != len) { if (verbose) - printf("Error at line %d: writing %zd bytes but only %zd written\n", + fprintf(stderr,"Error at line %d: writing %zd bytes but only %zd written\n", __LINE__,len, nn); return -1; } @@ -937,15 +928,15 @@ val_get_NC_tag(int fd, bufferinfo *gbp, NC_tag *tagp, const char *loc) case 12: *tagp = NC_ATTRIBUTE; break; default: *tagp = NC_INVALID; - if (verbose) printf("Error @ [0x%8.8zx]:\n", err_addr); - if (verbose) printf("\tInvalid NC component tag (%d)\n",tag); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", err_addr); + if (verbose) fprintf(stderr,"\tInvalid NC component tag (%d)\n",tag); return NC_ENOTNC; } return NC_NOERR; fn_exit: - if (verbose) printf("Error @ [0x%8.8zx]:\n", err_addr); - if (verbose) printf("\t%s: Fail to read NC component tag\n",loc); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", err_addr); + if (verbose) fprintf(stderr,"\t%s: Fail to read NC component tag\n",loc); return status; } @@ -962,7 +953,7 @@ hdr_get_NON_NEG(int fd, bufferinfo *gbp, long long *sp) sizeof_NON_NEG = (gbp->version < 5) ? 4 : 8; status = val_check_buffer(fd, gbp, sizeof_NON_NEG); if (status != NC_NOERR) { - if (verbose) printf("%d-byte size is expected for ", sizeof_NON_NEG); + if (verbose) fprintf(stderr,"%d-byte size is expected for ", sizeof_NON_NEG); return status; } if (gbp->version < 5) @@ -1002,8 +993,8 @@ hdr_get_name(int fd, err_addr = ERR_ADDR; err = hdr_get_NON_NEG(fd, gbp, &nchars); if (err != NC_NOERR) { - if (verbose) printf("Error @ [0x%8.8zx]:\n", err_addr); - if (verbose) printf("\t%s: Failed to read name string length\n", loc); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", err_addr); + if (verbose) fprintf(stderr,"\t%s: Failed to read name string length\n", loc); return err; } *name_len = nchars; @@ -1030,8 +1021,8 @@ hdr_get_name(int fd, err_addr = ERR_ADDR; err = val_fetch(fd, gbp); if (err != NC_NOERR) { - if (verbose) printf("Error @ [0x%8.8zx]:\n", err_addr); - if (verbose) printf("\t%s - fetching name string\n", loc); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", err_addr); + if (verbose) fprintf(stderr,"\t%s - fetching name string\n", loc); free(*namep); *namep = NULL; return err; @@ -1044,8 +1035,8 @@ hdr_get_name(int fd, err_addr = ERR_ADDR; err = val_check_buffer(fd, gbp, padding); if (err != NC_NOERR) { - if (verbose) printf("Error @ [0x%8.8zx]:\n", err_addr); - if (verbose) printf("\t%s - fetching name string padding\n", loc); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", err_addr); + if (verbose) fprintf(stderr,"\t%s - fetching name string padding\n", loc); free(*namep); *namep = NULL; return err; @@ -1053,13 +1044,13 @@ hdr_get_name(int fd, memset(pad, 0, X_ALIGN-1); if (memcmp(gbp->pos, pad, padding) != 0) { /* This is considered not a fatal error, we continue to validate */ - if (verbose) printf("Error @ [0x%8.8zx]:\n", err_addr); - if (verbose) printf("\t%s \"%s\": name padding is non-null byte\n", loc, *namep); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", err_addr); + if (verbose) fprintf(stderr,"\t%s \"%s\": name padding is non-null byte\n", loc, *namep); DEBUG_ASSIGN_ERROR(err, NC_ENULLPAD) if (repair) { val_repair(fd, err_addr, (size_t)padding, (void*)nada); if (verbose) - printf("\t%s \"%s\": name padding error has been **repaired**\n",loc,*namep); + fprintf(stderr,"\t%s \"%s\": name padding error has been **repaired**\n",loc,*namep); } } gbp->pos = (void *)((char *)gbp->pos + padding); @@ -1088,16 +1079,16 @@ val_get_NC_dim(int fd, bufferinfo *gbp, NC_dim **dimpp, NC_dimarray *ncap) { err_addr = ERR_ADDR; err = hdr_get_NON_NEG(fd, gbp, &dim_length); if (err != NC_NOERR) { /* frees dimp */ - if (verbose) printf("Error @ [0x%8.8zx]:\n", err_addr); - if (verbose) printf("\tDimension \"%s\": Failed to read dimension size\n",name); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", err_addr); + if (verbose) fprintf(stderr,"\tDimension \"%s\": Failed to read dimension size\n",name); free(name); return err; } /* check if unlimited_id already set */ if (ncap->unlimited_id != -1 && dim_length == 0) { - if (verbose) printf("Error @ [0x%8.8zx]:\n", err_addr); - if (verbose) printf("\tDimension \"%s\": NC_UNLIMITED dimension already found (\"%s\")\n",name,ncap->value[ncap->unlimited_id]->name); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", err_addr); + if (verbose) fprintf(stderr,"\tDimension \"%s\": NC_UNLIMITED dimension already found (\"%s\")\n",name,ncap->value[ncap->unlimited_id]->name); free(name); return NC_EUNLIMIT; } @@ -1149,14 +1140,14 @@ val_get_NC_dimarray(int fd, bufferinfo *gbp, NC_dimarray *ncap, long long numrec nelems_err_addr = ERR_ADDR; err = hdr_get_NON_NEG(fd, gbp, &tmp); if (err != NC_NOERR) { - if (verbose) printf("Error @ [0x%8.8zx]:\n", nelems_err_addr); - if (verbose) printf("\tFailed to read tag NC_DIMENSION\n"); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", nelems_err_addr); + if (verbose) fprintf(stderr,"\tFailed to read tag NC_DIMENSION\n"); return err; } if (tmp > NC_MAX_DIMS) { /* number of allowable defined dimensions NC_MAX_DIMS */ - if (verbose) printf("Error @ [0x%8.8zx]:\n", nelems_err_addr); - if (verbose) printf("\tNumber of dimensions (%lld) defined in file exceeds NC_MAX_DIMS (%d)\n",tmp,NC_MAX_DIMS); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", nelems_err_addr); + if (verbose) fprintf(stderr,"\tNumber of dimensions (%lld) defined in file exceeds NC_MAX_DIMS (%d)\n",tmp,NC_MAX_DIMS); DEBUG_RETURN_ERROR(NC_EMAXDIMS) } ncap->ndefined = (int)tmp; @@ -1174,15 +1165,15 @@ val_get_NC_dimarray(int fd, bufferinfo *gbp, NC_dimarray *ncap, long long numrec return NC_NOERR; #if 0 if (tag != ABSENT) { - if (verbose) printf("Error @ [0x%8.8zx]:\n", err_addr); - if (verbose) printf("\tInvalid NC component tag, while ABSENT is expected for "); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", err_addr); + if (verbose) fprintf(stderr,"\tInvalid NC component tag, while ABSENT is expected for "); DEBUG_RETURN_ERROR(NC_ENOTNC) } #endif } else { if (tag != NC_DIMENSION) { - if (verbose) printf("Error @ [0x%8.8zx]:\n", tag_err_addr); - if (verbose) printf("\tInvalid NC component tag (%d), expecting NC_DIMENSION (%d)\n",tag,NC_DIMENSION); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", tag_err_addr); + if (verbose) fprintf(stderr,"\tInvalid NC component tag (%d), expecting NC_DIMENSION (%d)\n",tag,NC_DIMENSION); DEBUG_RETURN_ERROR(NC_ENOTNC) } if (trace) { @@ -1250,13 +1241,13 @@ val_get_nc_type(int fd, return NC_NOERR; read_err_exit: - if (verbose) printf("Error @ [0x%8.8zx]:\n", err_addr); - if (verbose) printf("\t%s: Failed to read NC data type\n",loc); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", err_addr); + if (verbose) fprintf(stderr,"\t%s: Failed to read NC data type\n",loc); return status; err_exit: - if (verbose) printf("Error @ [0x%8.8zx]:\n", err_addr); - if (verbose) printf("\t%s: Unknown NC data type (%u)\n",loc, xtype); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", err_addr); + if (verbose) fprintf(stderr,"\t%s: Unknown NC data type (%u)\n",loc, xtype); DEBUG_RETURN_ERROR(NC_EBADTYPE) } @@ -1294,8 +1285,8 @@ val_get_NC_attrV(int fd, err_addr = ERR_ADDR; status = val_fetch(fd, gbp); if (status != NC_NOERR) { - if (verbose) printf("Error @ [0x%8.8zx]:\n", err_addr); - if (verbose) printf("\t%s: Failed to fetch next chunk into a buffer\n", loc); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", err_addr); + if (verbose) fprintf(stderr,"\t%s: Failed to fetch next chunk into a buffer\n", loc); return status; } bufremain = gbp->size; @@ -1306,8 +1297,8 @@ val_get_NC_attrV(int fd, memset(pad, 0, X_ALIGN-1); if (memcmp(gbp->pos, pad, padding) != 0) { /* This is considered not a fatal error, we continue to validate */ - if (verbose) printf("Error @ [0x%8.8zx]:\n", ERR_ADDR); - if (verbose) printf("\t%s: value padding is non-null byte\n", loc); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", ERR_ADDR); + if (verbose) fprintf(stderr,"\t%s: value padding is non-null byte\n", loc); DEBUG_ASSIGN_ERROR(status, NC_ENULLPAD) if (repair) { val_repair(fd, ERR_ADDR, (size_t)padding, (void*)nada); @@ -1397,7 +1388,7 @@ val_get_NC_attr(int fd, NC_attr **attrpp, const char *loc) { - char *name=NULL, xloc[1024]; + char *name=NULL, xloc[2048]; int err, status=NC_NOERR; size_t err_addr, name_len; nc_type xtype; @@ -1420,8 +1411,8 @@ val_get_NC_attr(int fd, err_addr = ERR_ADDR; err = hdr_get_NON_NEG(fd, gbp, &nelems); if (err != NC_NOERR) { - if (verbose) printf("Error @ [0x%8.8zx]:\n", err_addr); - if (verbose) printf("\t%s: Failed to read attribute length\n",xloc); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", err_addr); + if (verbose) fprintf(stderr,"\t%s: Failed to read attribute length\n",xloc); if (name != NULL) free(name); return err; } @@ -1483,14 +1474,14 @@ val_get_NC_attrarray(int fd, nelems_err_addr = ERR_ADDR; err = hdr_get_NON_NEG(fd, gbp, &tmp); if (err != NC_NOERR) { - if (verbose) printf("Error @ [0x%8.8zx]:\n", nelems_err_addr); - if (verbose) printf("\tFailed to read tag NC_ATTRIBUTE\n"); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", nelems_err_addr); + if (verbose) fprintf(stderr,"\tFailed to read tag NC_ATTRIBUTE\n"); return err; } if (tmp > NC_MAX_ATTRS) { /* number of allowable defined attributes NC_MAX_ATTRS */ - if (verbose) printf("Error @ [0x%8.8zx]:\n", nelems_err_addr); - if (verbose) printf("\t%s attributes: number of attributes (%lld) exceeds NC_MAX_ATTRS (%d)\n",loc,tmp,NC_MAX_ATTRS); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", nelems_err_addr); + if (verbose) fprintf(stderr,"\t%s attributes: number of attributes (%lld) exceeds NC_MAX_ATTRS (%d)\n",loc,tmp,NC_MAX_ATTRS); return NC_EMAXATTS; } ncap->ndefined = (int)tmp; @@ -1515,16 +1506,16 @@ val_get_NC_attrarray(int fd, return NC_NOERR; #if 0 if (tag != ABSENT) { - if (verbose) printf("Error @ [0x%8.8zx]:\n", err_addr); - if (verbose) printf("\tInvalid NC component tag, while ABSENT is expected for "); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", err_addr); + if (verbose) fprintf(stderr,"\tInvalid NC component tag, while ABSENT is expected for "); DEBUG_RETURN_ERROR(NC_ENOTNC) } #endif } else { sprintf(xloc, "%s attribute", loc); if (tag != NC_ATTRIBUTE) { - if (verbose) printf("Error @ [0x%8.8zx]:\n", tag_err_addr); - if (verbose) printf("\t%s: Invalid NC component tag (%d), expecting NC_ATTRIBUTE (%d)\n",xloc,tag,NC_ATTRIBUTE); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", tag_err_addr); + if (verbose) fprintf(stderr,"\t%s: Invalid NC component tag (%d), expecting NC_ATTRIBUTE (%d)\n",xloc,tag,NC_ATTRIBUTE); DEBUG_RETURN_ERROR(NC_ENOTNC) } if (trace) { @@ -1648,8 +1639,8 @@ val_get_NC_var(int fd, err_addr = ERR_ADDR; err = hdr_get_NON_NEG(fd, gbp, &ndims64); if (err != NC_NOERR) { - if (verbose) printf("Error @ [0x%8.8zx]:\n", err_addr); - if (verbose) printf("\t%s: Failed to read number of dimensions\n",xloc); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", err_addr); + if (verbose) fprintf(stderr,"\t%s: Failed to read number of dimensions\n",xloc); if (name != NULL) free(name); return err; } @@ -1657,8 +1648,8 @@ val_get_NC_var(int fd, /* cannot be more than NC_MAX_VAR_DIMS */ if (ndims64 > NC_MAX_VAR_DIMS) { - if (verbose) printf("Error:\n"); - if (verbose) printf("\t%s: number of dimensions (%lld) larger than NC_MAX_VAR_DIMS (%d)\n",xloc,ndims64,NC_MAX_VAR_DIMS); + if (verbose) fprintf(stderr,"Error:\n"); + if (verbose) fprintf(stderr,"\t%s: number of dimensions (%lld) larger than NC_MAX_VAR_DIMS (%d)\n",xloc,ndims64,NC_MAX_VAR_DIMS); if (name != NULL) free(name); DEBUG_RETURN_ERROR(NC_EMAXDIMS) } @@ -1680,8 +1671,8 @@ val_get_NC_var(int fd, err = val_check_buffer(fd, gbp, (gbp->version < 5 ? 4 : 8)); if (err != NC_NOERR) { if (trace) printf("\n"); - if (verbose) printf("Error @ [0x%8.8zx]:\n", ERR_ADDR); - if (verbose) printf("\t%s: Fail to read dimid[%d]\n",xloc,dim); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", ERR_ADDR); + if (verbose) fprintf(stderr,"\t%s: Fail to read dimid[%d]\n",xloc,dim); free_NC_var(varp); return err; } @@ -1696,8 +1687,8 @@ val_get_NC_var(int fd, /* dimid should be < f_ndims (num of dimensions defined in file) */ if (dimid >= f_ndims) { if (trace) printf("\n"); - if (verbose) printf("Error @ [0x%8.8zx]:\n", err_addr); - if (verbose) printf("\t%s \"%s\": dimid[%d]=%d is larger than the number of dimensions defined in file (%d)\n",loc,name,dim,dimid,f_ndims); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", err_addr); + if (verbose) fprintf(stderr,"\t%s \"%s\": dimid[%d]=%d is larger than the number of dimensions defined in file (%d)\n",loc,name,dim,dimid,f_ndims); free_NC_var(varp); DEBUG_RETURN_ERROR(NC_EBADDIM) } @@ -1733,8 +1724,8 @@ val_get_NC_var(int fd, err_addr = ERR_ADDR; err = hdr_get_NON_NEG(fd, gbp, &varp->len); if (err != NC_NOERR) { - if (verbose) printf("Error @ [0x%8.8zx]:\n", err_addr); - if (verbose) printf("\t%s: Failed to read vsize\n",xloc); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", err_addr); + if (verbose) fprintf(stderr,"\t%s: Failed to read vsize\n",xloc); free_NC_var(varp); return err; } @@ -1742,8 +1733,8 @@ val_get_NC_var(int fd, err = val_check_buffer(fd, gbp, (gbp->version == 1 ? 4 : 8)); if (err != NC_NOERR) { - if (verbose) printf("Error @ [0x%8.8zx]:\n", ERR_ADDR); - if (verbose) printf("\t%s: Fail to read begin\n",xloc); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", ERR_ADDR); + if (verbose) fprintf(stderr,"\t%s: Fail to read begin\n",xloc); free_NC_var(varp); return err; } @@ -1799,14 +1790,14 @@ val_get_NC_vararray(int fd, nelems_err_addr = ERR_ADDR; err = hdr_get_NON_NEG(fd, gbp, &tmp); if (err != NC_NOERR) { - if (verbose) printf("Error @ [0x%8.8zx]:\n", nelems_err_addr); - if (verbose) printf("\tFailed to read tag NC_VARIABLE\n"); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", nelems_err_addr); + if (verbose) fprintf(stderr,"\tFailed to read tag NC_VARIABLE\n"); return err; } if (tmp > NC_MAX_VARS) { /* number of allowable defined variables NC_MAX_VARS */ - if (verbose) printf("Error @ [0x%8.8zx]:\n", nelems_err_addr); - if (verbose) printf("\tNumber of variables (%lld) exceeds NC_MAX_VARS (%d)\n",tmp,NC_MAX_VARS); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", nelems_err_addr); + if (verbose) fprintf(stderr,"\tNumber of variables (%lld) exceeds NC_MAX_VARS (%d)\n",tmp,NC_MAX_VARS); DEBUG_RETURN_ERROR(NC_EMAXVARS); } ncap->ndefined = (int)tmp; @@ -1822,15 +1813,15 @@ val_get_NC_vararray(int fd, return NC_NOERR; #if 0 if (tag != ABSENT) { - if (verbose) printf("Error @ [0x%8.8zx]:\n", err_addr); - if (verbose) printf("\tInvalid NC component tag, while ABSENT is expected for "); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", err_addr); + if (verbose) fprintf(stderr,"\tInvalid NC component tag, while ABSENT is expected for "); DEBUG_RETURN_ERROR(NC_ENOTNC) } #endif } else { if (tag != NC_VARIABLE) { - if (verbose) printf("Error @ [0x%8.8zx]:\n", tag_err_addr); - if (verbose) printf("\tInvalid NC component tag (%d), expecting NC_VARIABLE (%d)\n",tag,NC_VARIABLE); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", tag_err_addr); + if (verbose) fprintf(stderr,"\tInvalid NC component tag (%d), expecting NC_VARIABLE (%d)\n",tag,NC_VARIABLE); DEBUG_RETURN_ERROR(NC_ENOTNC) } if (trace) printf("\ttag = NC_VARIABLE\n"); @@ -1901,8 +1892,8 @@ val_NC_check_vlens(NC *ncp) and format 2. */ long long ii, vlen_max, rec_vars_count; long long large_fix_vars_count, large_rec_vars_count; - long long first_large_fix_var, first_large_rec_var; - long long second_large_fix_var, second_large_rec_var; + long long first_large_fix_var=0, first_large_rec_var=0; + long long second_large_fix_var=0, second_large_rec_var=0; int last = 0; if (ncp->vars.ndefined == 0) @@ -1927,10 +1918,10 @@ val_NC_check_vlens(NC *ncp) if (ncp->format >= 5) { /* CDF-5 */ if (verbose) { NC_var *varp = (*vpp); - printf("Error:\n"); - printf("\tvar %s",varp->name); + fprintf(stderr,"Error:\n"); + fprintf(stderr,"\tvar %s",varp->name); DUMP_DIMS(varp) - printf(": variable size greater than max (%lld) allowable by CDF-%d\n",vlen_max,ncp->format); + fprintf(stderr,": variable size greater than max (%lld) allowable by CDF-%d\n",vlen_max,ncp->format); } DEBUG_RETURN_ERROR(NC_EVARSIZE) } @@ -1948,21 +1939,21 @@ val_NC_check_vlens(NC *ncp) if (large_fix_vars_count > 1) { /* only one "too-large" variable allowed */ if (verbose) { NC_var *varp; - printf("Error:\n"); - printf("\tInput file contains %lld large fixed-size variables\n",large_fix_vars_count); - printf("\tCDF-%d format allows only one large fixed-size variable\n",ncp->format); + fprintf(stderr,"Error:\n"); + fprintf(stderr,"\tInput file contains %lld large fixed-size variables\n",large_fix_vars_count); + fprintf(stderr,"\tCDF-%d format allows only one large fixed-size variable\n",ncp->format); varp = ncp->vars.value[first_large_fix_var]; - printf("\tThe 1st large fixed-size variable is %s",varp->name); + fprintf(stderr,"\tThe 1st large fixed-size variable is %s",varp->name); DUMP_DIMS(varp) - printf("\n"); + fprintf(stderr,"\n"); varp = ncp->vars.value[second_large_fix_var]; - printf("\tThe 2nd large fixed-size variable is %s\n",varp->name); + fprintf(stderr,"\tThe 2nd large fixed-size variable is %s\n",varp->name); DUMP_DIMS(varp) - printf("\n"); + fprintf(stderr,"\n"); if (ncp->format == 1) - printf("\tSee: %s\n", fmt_limit); + fprintf(stderr,"\tSee: %s\n", fmt_limit); else if (ncp->format == 2) - printf("\tSee: %s\n", off_limit); + fprintf(stderr,"\tSee: %s\n", off_limit); } DEBUG_RETURN_ERROR(NC_EVARSIZE) } @@ -1970,16 +1961,16 @@ val_NC_check_vlens(NC *ncp) /* The only "too-large" variable must be the last one defined */ if (large_fix_vars_count == 1 && last == 0) { if (verbose) { - printf("Error:\n"); - printf("\tCDF-%d format allows only one large fixed-size variable which must be defined last and there is no record variable\n",ncp->format); + fprintf(stderr,"Error:\n"); + fprintf(stderr,"\tCDF-%d format allows only one large fixed-size variable which must be defined last and there is no record variable\n",ncp->format); NC_var *varp = ncp->vars.value[first_large_fix_var]; - printf("\tThe large fixed-size variable is %s",varp->name); + fprintf(stderr,"\tThe large fixed-size variable is %s",varp->name); DUMP_DIMS(varp) - printf("\n"); + fprintf(stderr,"\n"); if (ncp->format == 1) - printf("\tSee: %s\n", fmt_limit); + fprintf(stderr,"\tSee: %s\n", fmt_limit); else if (ncp->format == 2) - printf("\tSee: %s\n", off_limit); + fprintf(stderr,"\tSee: %s\n", off_limit); } DEBUG_RETURN_ERROR(NC_EVARSIZE) } @@ -1990,13 +1981,13 @@ val_NC_check_vlens(NC *ncp) * allowed */ if (large_fix_vars_count == 1) { if (verbose) { - printf("Error:\n"); - printf("\tInput file contains 1 large fixed-size variables and %lld record variables\n", rec_vars_count); - printf("\tCDF-%d format allows only one large fixed-size variable which must be defined last and there is no record variable\n",ncp->format); + fprintf(stderr,"Error:\n"); + fprintf(stderr,"\tInput file contains 1 large fixed-size variables and %lld record variables\n", rec_vars_count); + fprintf(stderr,"\tCDF-%d format allows only one large fixed-size variable which must be defined last and there is no record variable\n",ncp->format); if (ncp->format == 1) - printf("\tSee: %s\n", fmt_limit); + fprintf(stderr,"\tSee: %s\n", fmt_limit); else if (ncp->format == 2) - printf("\tSee: %s\n", off_limit); + fprintf(stderr,"\tSee: %s\n", off_limit); } DEBUG_RETURN_ERROR(NC_EVARSIZE) } @@ -2012,10 +2003,10 @@ val_NC_check_vlens(NC *ncp) if (ncp->format >= 5) { /* CDF-5 */ if (verbose) { NC_var *varp = (*vpp); - printf("Error:\n"); - printf("\tvar %s",varp->name); + fprintf(stderr,"Error:\n"); + fprintf(stderr,"\tvar %s",varp->name); DUMP_DIMS(varp) - printf(": variable size greater than max (%lld) allowable by CDF-%d\n",vlen_max,ncp->format); + fprintf(stderr,": variable size greater than max (%lld) allowable by CDF-%d\n",vlen_max,ncp->format); } DEBUG_RETURN_ERROR(NC_EVARSIZE) } @@ -2034,21 +2025,21 @@ val_NC_check_vlens(NC *ncp) if (large_rec_vars_count > 1) { /* only one "too-large" variable allowed */ if (verbose) { NC_var *varp; - printf("Error:\n"); - printf("\tInput file contains %lld large record variables\n",large_rec_vars_count); + fprintf(stderr,"Error:\n"); + fprintf(stderr,"\tInput file contains %lld large record variables\n",large_rec_vars_count); varp = ncp->vars.value[first_large_rec_var]; - printf("\tThe 1st large record variable is %s",varp->name); + fprintf(stderr,"\tThe 1st large record variable is %s",varp->name); DUMP_DIMS(varp) - printf("\n"); + fprintf(stderr,"\n"); varp = ncp->vars.value[second_large_rec_var]; - printf("\tThe 2nd large record variable is %s",varp->name); + fprintf(stderr,"\tThe 2nd large record variable is %s",varp->name); DUMP_DIMS(varp) - printf("\n"); - printf("\tCDF-%d format allows only one large record variable\n",ncp->format); + fprintf(stderr,"\n"); + fprintf(stderr,"\tCDF-%d format allows only one large record variable\n",ncp->format); if (ncp->format == 1) - printf("\tSee: %s\n", fmt_limit); + fprintf(stderr,"\tSee: %s\n", fmt_limit); else if (ncp->format == 2) - printf("\tSee: %s\n", off_limit); + fprintf(stderr,"\tSee: %s\n", off_limit); } DEBUG_RETURN_ERROR(NC_EVARSIZE) } @@ -2056,16 +2047,16 @@ val_NC_check_vlens(NC *ncp) /* and it has to be the last one */ if (large_rec_vars_count == 1 && last == 0) { if (verbose) { - printf("Error:\n"); + fprintf(stderr,"Error:\n"); NC_var *varp = ncp->vars.value[first_large_rec_var]; - printf("\tThe 1st large record variable that is not defined last is %s",varp->name); + fprintf(stderr,"\tThe 1st large record variable that is not defined last is %s",varp->name); DUMP_DIMS(varp) - printf("\n"); - printf("\tCDF-%d format allows only one large record variable and it must be defined last\n",ncp->format); + fprintf(stderr,"\n"); + fprintf(stderr,"\tCDF-%d format allows only one large record variable and it must be defined last\n",ncp->format); if (ncp->format == 1) - printf("\tSee: %s\n", fmt_limit); + fprintf(stderr,"\tSee: %s\n", fmt_limit); else if (ncp->format == 2) - printf("\tSee: %s\n", off_limit); + fprintf(stderr,"\tSee: %s\n", off_limit); } DEBUG_RETURN_ERROR(NC_EVARSIZE) } @@ -2116,8 +2107,8 @@ val_NC_check_voff(NC *ncp) NC_var *varp = ncp->vars.value[i]; if (varp->begin < ncp->begin_var) { if (verbose) { - printf("Error - variable begin offset:\n"); - printf("\tvar \"%s\" begin offset (%lld) is less than file header extent (%lld)\n", + fprintf(stderr,"Error - variable begin offset:\n"); + fprintf(stderr,"\tvar \"%s\" begin offset (%lld) is less than file header extent (%lld)\n", varp->name, varp->begin, ncp->begin_var); } DEBUG_ASSIGN_ERROR(status, NC_ENOTNC) @@ -2145,8 +2136,8 @@ val_NC_check_voff(NC *ncp) if (verbose) { NC_var *var_cur = ncp->vars.value[var_off_len[i].ID]; NC_var *var_prv = ncp->vars.value[var_off_len[i-1].ID]; - printf("Error - variable begin offset:\n"); - printf("\tvar \"%s\" begin offset (%lld) overlaps var %s (begin=%lld, length=%lld)\n", + fprintf(stderr,"Error - variable begin offset:\n"); + fprintf(stderr,"\tvar \"%s\" begin offset (%lld) overlaps var %s (begin=%lld, length=%lld)\n", var_cur->name, var_cur->begin, var_prv->name, var_prv->begin, var_prv->len); } DEBUG_ASSIGN_ERROR(status, NC_ENOTNC) @@ -2156,8 +2147,8 @@ val_NC_check_voff(NC *ncp) } if (ncp->begin_rec < max_var_end) { - if (verbose) printf("Error:\n"); - if (verbose) printf("\tRecord variable section begin offset (%lld) is less than fixed-size variable section end offset (%lld)\n", ncp->begin_rec, max_var_end); + if (verbose) fprintf(stderr,"Error:\n"); + if (verbose) fprintf(stderr,"\tRecord variable section begin offset (%lld) is less than fixed-size variable section end offset (%lld)\n", ncp->begin_rec, max_var_end); DEBUG_ASSIGN_ERROR(status, NC_ENOTNC) } free(var_off_len); @@ -2191,8 +2182,8 @@ val_NC_check_voff(NC *ncp) if (verbose) { NC_var *var_cur = ncp->vars.value[var_off_len[i].ID]; NC_var *var_prv = ncp->vars.value[var_off_len[i-1].ID]; - printf("Error - variable begin offset:\n"); - printf("\tvar \"%s\" begin offset (%lld) overlaps var %s (begin=%lld, length=%lld)\n", + fprintf(stderr,"Error - variable begin offset:\n"); + fprintf(stderr,"\tvar \"%s\" begin offset (%lld) overlaps var %s (begin=%lld, length=%lld)\n", var_cur->name, var_cur->begin, var_prv->name, var_prv->begin, var_prv->len); } DEBUG_ASSIGN_ERROR(status, NC_ENOTNC) @@ -2212,11 +2203,11 @@ val_NC_check_voff(NC *ncp) if (varp->begin < prev_off) { if (verbose) { - printf("Error - variable begin offset orders:\n"); + fprintf(stderr,"Error - variable begin offset orders:\n"); if (i == 0) - printf("\tvar \"%s\" begin offset (%lld) is less than header extent (%lld)\n", varp->name, varp->begin, prev_off); + fprintf(stderr,"\tvar \"%s\" begin offset (%lld) is less than header extent (%lld)\n", varp->name, varp->begin, prev_off); else - printf("\tvar \"%s\" begin offset (%lld) is less than previous variable \"%s\" end offset (%lld)\n", varp->name, varp->begin, ncp->vars.value[prev]->name, prev_off); + fprintf(stderr,"\tvar \"%s\" begin offset (%lld) is less than previous variable \"%s\" end offset (%lld)\n", varp->name, varp->begin, ncp->vars.value[prev]->name, prev_off); } DEBUG_ASSIGN_ERROR(status, NC_ENOTNC) } @@ -2225,8 +2216,8 @@ val_NC_check_voff(NC *ncp) } if (ncp->begin_rec < prev_off) { - if (verbose) printf("Error:\n"); - if (verbose) printf("\tRecord variable section begin offset (%lld) is less than fixed-size variable section end offset (%lld)\n", ncp->begin_rec, prev_off); + if (verbose) fprintf(stderr,"Error:\n"); + if (verbose) fprintf(stderr,"\tRecord variable section begin offset (%lld) is less than fixed-size variable section end offset (%lld)\n", ncp->begin_rec, prev_off); DEBUG_ASSIGN_ERROR(status, NC_ENOTNC) } @@ -2242,11 +2233,11 @@ val_NC_check_voff(NC *ncp) if (varp->begin < prev_off) { if (verbose) { - printf("Error:\n"); + fprintf(stderr,"Error:\n"); if (i == 0) - printf("Variable \"%s\" begin offset (%lld) is less than record variable section begin offset (%lld)\n", varp->name, varp->begin, prev_off); + fprintf(stderr,"Variable \"%s\" begin offset (%lld) is less than record variable section begin offset (%lld)\n", varp->name, varp->begin, prev_off); else - printf("Variable \"%s\" begin offset (%lld) is less than previous variable \"%s\" end offset (%lld)\n", varp->name, varp->begin, ncp->vars.value[prev]->name, prev_off); + fprintf(stderr,"Variable \"%s\" begin offset (%lld) is less than previous variable \"%s\" end offset (%lld)\n", varp->name, varp->begin, ncp->vars.value[prev]->name, prev_off); } DEBUG_ASSIGN_ERROR(status, NC_ENOTNC) } @@ -2289,8 +2280,8 @@ val_get_NC(int fd, NC *ncp) getbuf.pos = (char*)getbuf.pos + 4; if (memcmp(magic, ncmagic, 3) != 0) { - if (verbose) printf("Error: Unknow file signature\n"); - if (verbose) printf("\tExpecting \"CDF1\", \"CDF2\", or \"CDF5\", but got \"%4s\"\n",magic); + if (verbose) fprintf(stderr,"Error: Unknown file signature\n"); + if (verbose) fprintf(stderr,"\tExpecting \"CDF1\", \"CDF2\", or \"CDF5\", but got \"%4s\"\n",magic); status = NC_ENOTNC; goto fn_exit; } @@ -2306,8 +2297,8 @@ val_get_NC(int fd, NC *ncp) getbuf.version = 5; ncp->format = 5; } else { - if (verbose) printf("Error: Unknow file signature\n"); - if (verbose) printf("\tExpecting \"CDF1\", \"CDF2\", or \"CDF5\", but got \"%4s\"\n",magic); + if (verbose) fprintf(stderr,"Error: Unknown file signature\n"); + if (verbose) fprintf(stderr,"\tExpecting \"CDF1\", \"CDF2\", or \"CDF5\", but got \"%4s\"\n",magic); status = NC_ENOTNC; goto fn_exit; } @@ -2319,8 +2310,8 @@ val_get_NC(int fd, NC *ncp) err_addr = 4; status = val_check_buffer(fd, &getbuf, (getbuf.version < 5) ? 4 : 8); if (status != NC_NOERR) { - if (verbose) printf("Error @ [0x%8.8zx]:\n", err_addr); - if (verbose) printf("\tFailed to read the number of records\n"); + if (verbose) fprintf(stderr,"Error @ [0x%8.8zx]:\n", err_addr); + if (verbose) fprintf(stderr,"\tFailed to read the number of records\n"); status = NC_ENOTNC; goto fn_exit; } @@ -2399,22 +2390,24 @@ val_get_NC(int fd, NC *ncp) goto fn_exit; } - /* check zero padding in the blank space betwee header size and extent */ + /* check zero padding in the blank space between end of file header and + * beginning of data section + */ if (repair && ncp->begin_var - ncp->xsz > 0) { - size_t i, gap = ncp->begin_var - ncp->xsz; + size_t gap = ncp->begin_var - ncp->xsz; ssize_t readLen; char *buf = (char*) malloc(gap); if (-1 == lseek(fd, ncp->xsz, SEEK_SET)) { if (verbose) - printf("Error at line %d: lseek %s\n",__LINE__,strerror(errno)); + fprintf(stderr,"Error at line %d: lseek %s\n",__LINE__,strerror(errno)); free(buf); goto fn_exit; } readLen = read(fd, buf, gap); if (readLen == -1) { if (verbose) - printf("Error at line %d: read %s\n",__LINE__,strerror(errno)); + fprintf(stderr,"Error at line %d: read %s\n",__LINE__,strerror(errno)); free(buf); status = -1; goto fn_exit; @@ -2448,7 +2441,7 @@ val_get_NC(int fd, NC *ncp) #ifndef BUILD_CDFDIFF -/* File system types recognized by ROMIO in MPICH 4.0.0 */ +/* File system types recognized by ROMIO in MPICH 4.0.0, and by PnetCDF */ static const char* fstypes[] = {"ufs", "nfs", "xfs", "pvfs2", "gpfs", "panfs", "lustre", "daos", "testfs", "ime", "quobyte", NULL}; /* Return a pointer to filename by removing the file system type prefix name if @@ -2503,7 +2496,7 @@ check_signature(char *filename) exit(1); } if (rlen != 8) { /* file size less than 8 bytes */ - printf("Error at line %d: invalid file %s\n",__LINE__,filename); + fprintf(stderr,"Error at line %d: invalid file %s\n",__LINE__,filename); close(fd); /* ignore error */ exit(1); } @@ -2535,9 +2528,9 @@ check_signature(char *filename) if (rlen == 8) { /* HDF5 signature found */ if (verbose) { - printf("Error: Input file is in HDF format\n"); - printf(" 'ncvalidator' only validates NetCDF classic files\n"); - printf(" Consider 'h5check', the HDF5 format checker\n"); + fprintf(stderr,"Error: Input file is in HDF format\n"); + fprintf(stderr," 'ncvalidator' only validates NetCDF classic files\n"); + fprintf(stderr," Consider 'h5check', the HDF5 format checker\n"); } return NC_FORMAT_NETCDF4; } @@ -2602,7 +2595,7 @@ int main(int argc, char **argv) if (fmt != NC_FORMAT_CDF5 && fmt != NC_FORMAT_CDF2 && fmt != NC_FORMAT_CLASSIC) { if (fmt == NC_FORMAT_UNKNOWN && verbose) - printf("File \"%s\" format is unknown\n",filename); + fprintf(stderr,"File \"%s\" format is unknown\n",filename); return EXIT_FAILURE; } @@ -2620,7 +2613,7 @@ int main(int argc, char **argv) ncp = (NC*) calloc(1, sizeof(NC)); if (ncp == NULL) { status = NC_ENOMEM; - if (verbose) printf("Error at line %d when calling calloc()\n",__LINE__); + if (verbose) fprintf(stderr,"Error at line %d when calling calloc()\n",__LINE__); goto prog_exit; } @@ -2634,7 +2627,7 @@ int main(int argc, char **argv) /* check file size */ if (-1 == fstat(fd, &ncfilestat)) { - if (verbose) printf("Error at line %d fstat (%s)\n",__LINE__,strerror(errno)); + if (verbose) fprintf(stderr,"Error at line %d fstat (%s)\n",__LINE__,strerror(errno)); status = NC_EFILE; goto prog_exit; } @@ -2643,8 +2636,8 @@ int main(int argc, char **argv) expect_fsize = ncp->begin_rec + ncp->recsize * ncp->numrecs; if (expect_fsize < ncfilestat.st_size) { if (verbose) { - printf("Warning: file size (%lld) is larger than expected (%lld)!\n",(long long)ncfilestat.st_size, expect_fsize); - printf("\tbegin_rec=%lld recsize=%lld numrecs=%lld ncfilestat.st_size=%lld\n",ncp->begin_rec, ncp->recsize, ncp->numrecs, (long long) ncfilestat.st_size); + fprintf(stderr,"Warning: file size (%lld) is larger than expected (%lld)!\n",(long long)ncfilestat.st_size, expect_fsize); + fprintf(stderr,"\tbegin_rec=%lld recsize=%lld numrecs=%lld ncfilestat.st_size=%lld\n",ncp->begin_rec, ncp->recsize, ncp->numrecs, (long long) ncfilestat.st_size); } } else if (expect_fsize > ncfilestat.st_size) { @@ -2652,14 +2645,14 @@ int main(int argc, char **argv) * less than expected, then this is due to partial data written * to the variable while the file is in no fill mode */ if (verbose) { - printf("Warning:\n"); - printf("\tfile size (%lld) is less than expected (%lld)!\n",(long long)ncfilestat.st_size, expect_fsize); + fprintf(stderr,"Warning:\n"); + fprintf(stderr,"\tfile size (%lld) is less than expected (%lld)!\n",(long long)ncfilestat.st_size, expect_fsize); } } } else { /* either there is no record variable or no record is written */ /* Assuming variables' begins do not follow their define order, find - * max end offset among all fixed-size varaibles. + * max end offset among all fixed-size variables. */ long long expect_fsize = MAX(ncp->begin_var, ncp->begin_rec); expect_fsize = MAX(expect_fsize, ncp->xsz); @@ -2679,8 +2672,8 @@ int main(int argc, char **argv) /* if file header are valid and the only error is the file size * less than expected, then this is due to partial data written * to the variable while the file is in no fill mode */ - if (verbose) printf("Warning:\n"); - if (verbose) printf("\tfile size (%lld) is less than expected (%lld)!\n",(long long)ncfilestat.st_size, expect_fsize); + if (verbose) fprintf(stderr,"Warning:\n"); + if (verbose) fprintf(stderr,"\tfile size (%lld) is less than expected (%lld)!\n",(long long)ncfilestat.st_size, expect_fsize); } } @@ -2698,10 +2691,10 @@ int main(int argc, char **argv) if (status == NC_NOERR) printf("File \"%s\" is a valid NetCDF classic CDF-%d file.\n",filename, fmt); else { - printf("File \"%s\" fails to conform with classic CDF-%d file format specifications\n",filename, fmt); + fprintf(stderr,"File \"%s\" fails to conform with classic CDF-%d file format specifications\n",filename, fmt); if (repair) { - printf("and it has been repaired in place to remove the errors.\n"); - printf("Please run \"%s %s\" to validate again.\n",argv[0],filename); + fprintf(stderr,"and it has been repaired in place to remove the errors.\n"); + fprintf(stderr,"Please run \"%s %s\" to validate again.\n",argv[0],filename); } } } diff --git a/src/utils/ncvalidator/seq_runs.sh b/src/utils/ncvalidator/seq_runs.sh index 0e7f6316d3..d18b827bff 100755 --- a/src/utils/ncvalidator/seq_runs.sh +++ b/src/utils/ncvalidator/seq_runs.sh @@ -1,38 +1,32 @@ #!/bin/sh # -# Copyright (C) 2003, Northwestern University and Argonne National Laboratory +# Copyright (C) 2017, Northwestern University and Argonne National Laboratory # See COPYRIGHT notice in top-level directory. # # Exit immediately if a command exits with a non-zero status. -# set -e -# Cannot use "set -e" here, as the tests here all return errors. - -VALIDATOR=ncvalidator -if [ -z "${VALIDATOR}" ]; then - echo "Failed: variable VALIDATOR id not defined" - exit 1 -fi -if [ ! -f ${VALIDATOR} ]; then - echo "Failed: file ${VALIDATOR} does not exit" - exit 1 -fi +set -e # prevent user environment setting of PNETCDF_HINTS to interfere unset PNETCDF_HINTS -for i in ${BAD_FILES} ; do - if [ ! -f ${srcdir}/$i ]; then - echo "Failed: input test file ${srcdir}/$i does not exit" - exit 1 - fi - ${TESTSEQRUN} ./${VALIDATOR} ${srcdir}/$i - ret=$? - # capture exit status of VALIDATOR command - if [ ${ret} -ne 1 ]; then - echo "Failed: ${VALIDATOR} ${srcdir}/$i" - exit 1 - fi +for i in ${ENULLPAD_FILES} ; do + ${TESTSEQRUN} ./tst_open ${srcdir}/$i NC_ENULLPAD +done + +for i in ${EMAXVARS_FILES} ; do + ${TESTSEQRUN} ./tst_open ${srcdir}/$i NC_EMAXVARS +done + +for i in ${EUNLIMIT_FILES} ; do + ${TESTSEQRUN} ./tst_open ${srcdir}/$i NC_EUNLIMIT +done + +for i in ${ENOTNC_FILES} ; do + ${TESTSEQRUN} ./tst_open ${srcdir}/$i NC_ENOTNC +done + +for i in ${EVARSIZE_FILES} ; do + ${TESTSEQRUN} ./tst_open ${srcdir}/$i NC_EVARSIZE done -echo "SUCCESS: ${VALIDATOR} to detect files fail to conform CDF formats" diff --git a/src/utils/ncvalidator/tst_open.c b/src/utils/ncvalidator/tst_open.c index aa34b82f81..83c28441c5 100644 --- a/src/utils/ncvalidator/tst_open.c +++ b/src/utils/ncvalidator/tst_open.c @@ -26,7 +26,7 @@ #define CHECK_ERR { \ if (err != NC_NOERR) { \ nerrs++; \ - printf("Error at line %d in %s: input file %s (%s)\n", \ + fprintf(stderr,"Error at line %d in %s: input file %s (%s)\n", \ __LINE__,__FILE__,filename,ncmpi_strerrno(err)); \ } \ } @@ -34,7 +34,7 @@ #define EXP_ERR(exp) { \ if (err != exp) { \ nerrs++; \ - printf("Error at line %d in %s: expected_errno %s but got %s\n", \ + fprintf(stderr,"Error at line %d in %s: expected_errno %s but got %s\n", \ __LINE__,__FILE__,ncmpi_strerrno(exp), ncmpi_strerrno(err)); \ } \ } @@ -61,7 +61,7 @@ int main(int argc, char** argv) { MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (argc != 3) { - if (!rank) printf("Usage: %s [filename] expected_errno\n",argv[0]); + if (!rank) fprintf(stderr,"Usage: %s [filename] expected_errno\n",argv[0]); goto fn_exit; } snprintf(filename, 256, "%s", argv[1]); @@ -76,7 +76,7 @@ int main(int argc, char** argv) { } err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); -#ifndef ENABLE_NULL_BYTE_HEADER_PADDING +#if PNETCDF_NULL_BYTE_HEADER_PADDING == 0 if (expected_errno == NC_ENULLPAD) CHECK_ERR else #endif @@ -96,7 +96,7 @@ int main(int argc, char** argv) { fn_exit: MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { - if (nerrs) printf("fail with %d mismatches\n",nerrs); + if (nerrs) fprintf(stderr,"fail with %d mismatches\n",nerrs); else printf("pass\n"); } diff --git a/src/utils/ncvalidator/wrap_runs.sh b/src/utils/ncvalidator/wrap_runs.sh deleted file mode 100755 index 39c5a985e4..0000000000 --- a/src/utils/ncvalidator/wrap_runs.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2017, Northwestern University and Argonne National Laboratory -# See COPYRIGHT notice in top-level directory. -# - -# Exit immediately if a command exits with a non-zero status. -set -e - -# prevent user environment setting of PNETCDF_HINTS to interfere -unset PNETCDF_HINTS - -for i in ${ENULLPAD_FILES} ; do - for j in 0 1 ; do - export PNETCDF_SAFE_MODE=$j - # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - ${TESTSEQRUN} ./tst_open ${srcdir}/$i NC_ENULLPAD - done -done - -for i in ${EMAXVARS_FILES} ; do - for j in 0 1 ; do - export PNETCDF_SAFE_MODE=$j - # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - ${TESTSEQRUN} ./tst_open ${srcdir}/$i NC_EMAXVARS - done -done - -for i in ${EUNLIMIT_FILES} ; do - for j in 0 1 ; do - export PNETCDF_SAFE_MODE=$j - # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - ${TESTSEQRUN} ./tst_open ${srcdir}/$i NC_EUNLIMIT - done -done - -for i in ${ENOTNC_FILES} ; do - for j in 0 1 ; do - export PNETCDF_SAFE_MODE=$j - # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - ${TESTSEQRUN} ./tst_open ${srcdir}/$i NC_ENOTNC - done -done - -for i in ${EVARSIZE_FILES} ; do - for j in 0 1 ; do - export PNETCDF_SAFE_MODE=$j - # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - ${TESTSEQRUN} ./tst_open ${srcdir}/$i NC_EVARSIZE - done -done - diff --git a/src/utils/pnetcdf-config.in b/src/utils/pnetcdf-config.in index 4892f27211..97922fac50 100644 --- a/src/utils/pnetcdf-config.in +++ b/src/utils/pnetcdf-config.in @@ -66,9 +66,12 @@ enable_relax_coord_bound="@RELAX_COORD_BOUND@" is_bigendian="@ac_cv_c_bigendian@" in_place_swap="@in_place_swap@" enable_bbdriver="@ENABLE_BURST_BUFFER@" -enable_netcdf4="@ENABLE_NETCDF4@" -enable_adios="@ENABLE_ADIOS@" +enable_gio="@ENABLE_GIO@" +# GIO is built as a sub-module of PnetCDF, thus it is not necessary to add +# GIO install paths to LDFLAGS or LIBS. + +enable_netcdf4="@ENABLE_NETCDF4@" if test "x$enable_netcdf4" = x1; then # NetCDF4 header files are required to build PnetCDF, but not PnetCDF applications # CPPFLAGS="$CPPFLAGS @NETCDF4_INC@" @@ -76,6 +79,7 @@ if test "x$enable_netcdf4" = x1; then LIBS="@NETCDF4_LIBS@ $LIBS" fi +enable_adios="@ENABLE_ADIOS@" if test "x$enable_adios" = x1; then # ADIOS header files are required to build PnetCDF, but not PnetCDF applications LDFLAGS="@ADIOS_LDFLAGS@ $LDFLAGS" @@ -111,6 +115,7 @@ Available values for OPTION include: --fcflags Fortran 9x compiler flags used to build PnetCDF --ldflags Linker flags used to build PnetCDF --libs Extra libraries used to build PnetCDF + --gio Whether GIO support is enabled or disabled --netcdf4 Whether NetCDF-4 support is enabled or disabled --adios Whether ADIOS support is enabled or disabled --relax-coord-bound Whether using a relaxed coordinate boundary check @@ -139,6 +144,11 @@ all() echo echo " --has-c++ -> $has_cxx" echo " --has-fortran -> $has_fortran" +if test "x$enable_gio" = x1; then + echo " --gio -> enabled" +else + echo " --gio -> disabled" +fi if test "x$enable_netcdf4" = x1; then echo " --netcdf4 -> enabled" else @@ -330,6 +340,14 @@ while test $# -gt 0; do # echo "$FCLIBS" | sed -e 's/^[ \t]*//' # ;; + --gio) +if test "x$enable_gio" = x1; then + echo "enabled" +else + echo "disabled" +fi + ;; + --netcdf4) if test "x$enable_netcdf4" = x1; then echo "enabled" diff --git a/test/C/Makefile.am b/test/C/Makefile.am index 4d0668b6fd..15273145f5 100644 --- a/test/C/Makefile.am +++ b/test/C/Makefile.am @@ -13,57 +13,48 @@ AM_DEFAULT_SOURCE_EXT = .c AM_CPPFLAGS = -I$(top_srcdir)/src/include AM_CPPFLAGS += -I$(srcdir)/../common AM_CPPFLAGS += -I$(top_builddir)/src/include +AM_CPPFLAGS += -I$(top_srcdir)/src/utils/ncmpidiff -LDADD = $(top_builddir)/src/libs/libpnetcdf.la ../common/libtestutils.la +LDADD = ../common/libtestutils.la +LDADD += $(top_builddir)/src/libs/libpnetcdf.la LDADD += @NETCDF4_LDFLAGS@ @ADIOS_LDFLAGS@ @NETCDF4_LIBS@ @ADIOS_LIBS@ if DECL_MPI_OFFSET AM_CPPFLAGS += -DHAVE_DECL_MPI_OFFSET endif -TESTPROGRAMS = pres_temp_4D_wr \ - pres_temp_4D_rd - -pres_temp_4D_wr_SOURCES = pres_temp_4D_wr.c pres_temp_4D.h -pres_temp_4D_rd_SOURCES = pres_temp_4D_rd.c pres_temp_4D.h - -check_PROGRAMS = $(TESTPROGRAMS) +check_PROGRAMS = pres_temp_4D_wr_rd # autimake 1.11.3 has not yet implemented AM_TESTS_ENVIRONMENT # For newer versions, we can use AM_TESTS_ENVIRONMENT instead TESTS_ENVIRONMENT = export SED="$(SED)"; TESTS_ENVIRONMENT += export srcdir="$(srcdir)"; +TESTS_ENVIRONMENT += export top_builddir="$(top_builddir)"; TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; TESTS_ENVIRONMENT += export TESTMPIRUN="$(TESTMPIRUN)"; -TESTS_ENVIRONMENT += export PNETCDF_DEBUG="$(PNETCDF_DEBUG)"; -TESTS_ENVIRONMENT += export TESTPROGRAMS="$(TESTPROGRAMS)"; TESTS_ENVIRONMENT += export check_PROGRAMS="$(check_PROGRAMS)"; -TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER="$(ENABLE_BURST_BUFFER)"; -TESTS_ENVIRONMENT += export ENABLE_NETCDF4="$(ENABLE_NETCDF4)"; -TESTS = seq_runs.sh +TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER=@ENABLE_BURST_BUFFER@; +TESTS_ENVIRONMENT += export ENABLE_NETCDF4=@ENABLE_NETCDF4@; +TESTS_ENVIRONMENT += export ENABLE_THREAD_SAFE=@ENABLE_THREAD_SAFE@; +TESTS_ENVIRONMENT += export MIMIC_LUSTRE=@MIMIC_LUSTRE@; + +TESTS = $(check_PROGRAMS) TEST_EXTENSIONS = .sh -# LOG_COMPILER = $(srcdir)/wrap_runs.sh -# SH_LOG_COMPILER = +LOG_COMPILER = $(srcdir)/seq_runs.sh +SH_LOG_COMPILER = -# Dependency: pres_temp_4D_rd reads the output file from pres_temp_4D_wr. -# To support parallel "make -jN check", we add dependencies below and automake -# states it currently works only for tests that end in one of the suffixes -# listed in TEST_EXTENSIONS. $(EXEEXT) are appended to work around. -pres_temp_4D_rd$(EXEEXT).log: pres_temp_4D_wr$(EXEEXT).log +check_SCRIPTS = seq_runs.sh parallel_run.sh EXTRA_DIST = seq_runs.sh parallel_run.sh -CLEANFILES = $(TESTOUTDIR)/pres_temp_4D.nc \ - $(TESTOUTDIR)/pres_temp_4D.nc4 \ - $(TESTOUTDIR)/pres_temp_4D.bb.nc \ - core core.* *.gcda *.gcno *.gcov gmon.out +CLEANFILES = core core.* *.gcda *.gcno *.gcov gmon.out ../common/libtestutils.la: set -e; cd ../common && $(MAKE) $(MFLAGS) tests -ptest ptests ptest4: $(TESTPROGRAMS) +ptest ptests ptest4: $(check_PROGRAMS) @echo "===========================================================" @echo " $(subdir): Parallel testing on 4 MPI processes" @echo "===========================================================" diff --git a/test/C/parallel_run.sh b/test/C/parallel_run.sh index 9fe1f41a8a..20630dabc0 100755 --- a/test/C/parallel_run.sh +++ b/test/C/parallel_run.sh @@ -1,73 +1,50 @@ #!/bin/bash # -# Copyright (C) 2018, Northwestern University and Argonne National Laboratory +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory # See COPYRIGHT notice in top-level directory. # # Exit immediately if a command exits with a non-zero status. set -e -VALIDATOR=../../src/utils/ncvalidator/ncvalidator -NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff +DRY_RUN=no +VERBOSE=no -# remove file system type prefix if there is any -OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $MPIRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $MPIRUN $@ + fi +} MPIRUN=`echo ${TESTMPIRUN} | ${SED} -e "s/NP/$1/g"` # echo "MPIRUN = ${MPIRUN}" - -# echo "PNETCDF_DEBUG = ${PNETCDF_DEBUG}" -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" -else - safe_modes="0" -fi +# echo "check_PROGRAMS=${check_PROGRAMS}" # prevent user environment setting of PNETCDF_HINTS to interfere unset PNETCDF_HINTS -for j in ${safe_modes} ; do -for intra_aggr in 0 1 ; do - if test "$j" = 1 ; then # test only in safe mode - export PNETCDF_HINTS="romio_no_indep_rw=true" - else - export PNETCDF_HINTS= - fi - if test "$intra_aggr" = 1 ; then - export PNETCDF_HINTS="${PNETCDF_HINTS};nc_num_aggrs_per_node=2" - fi - export PNETCDF_SAFE_MODE=$j - # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - ${MPIRUN} ./pres_temp_4D_wr ${TESTOUTDIR}/pres_temp_4D.nc - ${MPIRUN} ./pres_temp_4D_rd ${TESTOUTDIR}/pres_temp_4D.nc - # echo "--- validating file ${TESTOUTDIR}/pres_temp_4D.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/pres_temp_4D.nc - # echo "" +if test "x$MIMIC_LUSTRE" != x1 ; then + PNETCDF_HINTS="cb_nodes=2" +fi + +for i in ${check_PROGRAMS} ; do - if test "x${ENABLE_BURST_BUFFER}" = x1 ; then - # echo "---- test burst buffering feature" - saved_PNETCDF_HINTS=${PNETCDF_HINTS} - export PNETCDF_HINTS="${PNETCDF_HINTS};nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" - ${MPIRUN} ./pres_temp_4D_wr ${TESTOUTDIR}/pres_temp_4D.bb.nc - ${MPIRUN} ./pres_temp_4D_rd ${TESTOUTDIR}/pres_temp_4D.bb.nc - export PNETCDF_HINTS=${saved_PNETCDF_HINTS} + # SECONDS=0 + # start_ns=$(date +%s.%4N) - # echo "--- validating file ${TESTOUTDIR}/pres_temp_4D.bb.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/pres_temp_4D.bb.nc + exe_name=`basename $i` - # echo "--- ncmpidiff pres_temp_4D.nc pres_temp_4D.bb.nc ---" - ${MPIRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/pres_temp_4D.nc ${TESTOUTDIR}/pres_temp_4D.bb.nc - fi + run_cmd ./$i -q -o ${TESTOUTDIR}/${exe_name}.nc - if test "x${ENABLE_NETCDF4}" = x1 ; then - # echo "test netCDF-4 feature" - ${MPIRUN} ./pres_temp_4D_wr ${TESTOUTDIR}/pres_temp_4D.nc4 4 - ${MPIRUN} ./pres_temp_4D_rd ${TESTOUTDIR}/pres_temp_4D.nc4 - # Validator does not support nc4 - fi -done -done + # # echo "Elapsed: $SECONDS seconds" + # end_ns=$(date +%s.%4N) + # # Calculate difference (requires bc for floating point math) + # elapsed_ns=$(echo "$end_ns - $start_ns" | bc) + # echo "Elapsed time: ${elapsed_ns} seconds" -rm -f ${OUTDIR}/*.nc -rm -f ${OUTDIR}/*.nc4 +done # check_PROGRAMS diff --git a/test/C/pres_temp_4D.h b/test/C/pres_temp_4D.h deleted file mode 100644 index f70677c069..0000000000 --- a/test/C/pres_temp_4D.h +++ /dev/null @@ -1,37 +0,0 @@ -/* We are writing and reading 4D data, a 2 x 6 x 12 lvl-lat-lon grid, with 2 - * timesteps of data. */ -#define NDIMS 4 -#define NLAT 6 -#define NLON 12 -#define LAT_NAME "latitude" -#define LON_NAME "longitude" -#define NREC 2 -#define REC_NAME "time" -#define LVL_NAME "level" -#define NLVL 10 - -/* Names of things. */ -#define PRES_NAME "pressure" -#define TEMP_NAME "temperature" -#define UNITS "units" -#define DEGREES_EAST "degrees_east" -#define DEGREES_NORTH "degrees_north" - -/* These are used to construct some example data, and to calculate the values - * we expect to find. */ -#define SAMPLE_PRESSURE 900.0 -#define SAMPLE_TEMP 9.0 -#define START_LAT 25.0 -#define START_LON -125.0 - -/* For the units attributes. */ -#define UNITS "units" -#define PRES_UNITS "hPa" -#define TEMP_UNITS "celsius" -#define LAT_UNITS "degrees_north" -#define LON_UNITS "degrees_east" -#define MAX_ATT_LEN 80 - -/* This is the name of the data file we will create and read back. */ -#define FILE_NAME "pres_temp_4D.nc" - diff --git a/test/C/pres_temp_4D_rd.c b/test/C/pres_temp_4D_rd.c deleted file mode 100644 index 84257c3b09..0000000000 --- a/test/C/pres_temp_4D_rd.c +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Copyright (C) 2003, Northwestern University and Argonne National Laboratory - * See COPYRIGHT notice in top-level directory. - */ -/* $Id$ */ - -/* - This is an example which reads some 4D pressure and - temperatures. The data file read by this program is produced by the - companion program pres_temp_4D_wr.c. It is intended to illustrate - the use of the netCDF C API. - - This program is part of the netCDF tutorial: - http://www.unidata.ucar.edu/software/netcdf/docs/netcdf-tutorial - - Full documentation of the netCDF C API can be found at: - http://www.unidata.ucar.edu/software/netcdf/docs/netcdf-c - - $Id$ -*/ - -#include -#include -#include -#include /* basename() */ -#include -#include -#include - -#include "pres_temp_4D.h" - -int main(int argc, char **argv) -{ - int rank, nprocs, ncid, pres_varid, temp_varid; - int lat_varid, lon_varid; - - /* The start and count arrays will tell the netCDF library where to - read our data. */ - MPI_Offset start[NDIMS], count[NDIMS]; - - /* Program variables to hold the data we will read. We will only - need enough space to hold one timestep of data; one record. */ - float **pres_in = NULL; /* [NLVL/nprocs][NLAT][NLON] */ - float **temp_in = NULL; /* [NLVL/nprocs][NLAT][NLON] */ - - /* These program variables hold the latitudes and longitudes. */ - float lats[NLAT], lons[NLON]; - - /* Loop indexes. */ - int lvl, lat, lon, rec, i = 0; - - /* Error handling. */ - int err, nerrs = 0; - - char *filename = FILE_NAME; - - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - if (argc > 3) { - if (!rank) - printf("Usage: %s [filename]\n", argv[0]); - MPI_Finalize(); - return 1; - } - - if (argc > 1) filename = argv[1]; - - /* Open the file. */ - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); - if (err != NC_NOERR) { /* fatal error */ - if (rank == 0) - fprintf(stderr,"Error: failed to open file %s (%s)\n",filename,ncmpi_strerror(err)); - goto err_out; - } - - if (rank == 0) { - char *cmd_str = (char *)malloc(strlen(argv[0]) + 256); - int format; - err = ncmpi_inq_format(ncid, &format); CHECK_ERR - if (format == NC_FORMAT_NETCDF4) - sprintf(cmd_str, "*** TESTING C %s for reading NetCDF-4 file", basename(argv[0])); - else - sprintf(cmd_str, "*** TESTING C %s for reading classic file", basename(argv[0])); - printf("%-66s ------ ", cmd_str); - free(cmd_str); - } - - /* Get the varids of the latitude and longitude coordinate variables. */ - err = ncmpi_inq_varid(ncid, LAT_NAME, &lat_varid); - CHECK_ERR - err = ncmpi_inq_varid(ncid, LON_NAME, &lon_varid); - CHECK_ERR - - /* Read the coordinate variable data. */ - memset(lats, 0, sizeof(float) * NLAT); - memset(lons, 0, sizeof(float) * NLON); - err = ncmpi_get_var_float_all(ncid, lat_varid, &lats[0]); - CHECK_ERR - err = ncmpi_get_var_float_all(ncid, lon_varid, &lons[0]); - CHECK_ERR - - /* Check the coordinate variable data. */ - for (lat = 0; lat < NLAT; lat++) { - float exp = START_LAT + 5. * lat; - if (lats[lat] != exp) { - printf("\nError at line %d in %s: lats[%d] expect %.1f but got %.1f\n", - __LINE__, __FILE__, lat, exp, lats[lat]); - nerrs++; - break; - } - } - for (lon = 0; lon < NLON; lon++) { - float exp = START_LON + 5. * lon; - if (lons[lon] != START_LON + 5. * lon) { - printf("\nError at line %d in %s: lons[%d] expect %.1f but got %.1f\n", - __LINE__, __FILE__, lon, exp, lons[lon]); - nerrs++; - break; - } - } - - /* Get the varids of the pressure and temperature netCDF variables. */ - err = ncmpi_inq_varid(ncid, PRES_NAME, &pres_varid); - CHECK_ERR - err = ncmpi_inq_varid(ncid, TEMP_NAME, &temp_varid); - CHECK_ERR - - /* Read the data. Since we know the contents of the file we know that the - * data arrays in this program are the correct size to hold one timestep. - */ - count[0] = 1; - count[2] = NLAT; - count[3] = NLON; - start[2] = 0; - start[3] = 0; - - /* divide NLVL dimension among processes */ - count[1] = NLVL / nprocs; - start[1] = count[1] * rank; - if (rank < NLVL % nprocs) { - start[1] += rank; - count[1]++; - } - else { - start[1] += NLVL % nprocs; - } - if (count[1] == 0) - start[1] = 0; - - /* allocate read buffers */ - pres_in = (float **)malloc(sizeof(float *) * count[1] * 2); - temp_in = pres_in + count[1]; - if (count[1] > 0) { - pres_in[0] = (float *)malloc(sizeof(float) * count[1] * 2 * NLAT * NLON); - temp_in[0] = pres_in[0] + count[1] * NLAT * NLON; - for (i = 1; i < count[1]; i++) { - pres_in[i] = pres_in[i - 1] + NLAT * NLON; - temp_in[i] = temp_in[i - 1] + NLAT * NLON; - } - } - - /* Read and check one record at a time. */ - for (rec = 0; rec < NREC; rec++) { - start[0] = rec; - err = ncmpi_get_vara_float_all(ncid, pres_varid, start, count, &pres_in[0][0]); - CHECK_ERR - err = ncmpi_get_vara_float_all(ncid, temp_varid, start, count, &temp_in[0][0]); - CHECK_ERR - - /* Check the data. */ - i = (int)start[1] * NLAT * NLON; - for (lvl = 0; lvl < count[1]; lvl++) - for (lat = 0; lat < NLAT; lat++) - for (lon = 0; lon < NLON; lon++) { - float exp = SAMPLE_PRESSURE + i; - int indx = lat * NLON + lon; - if (pres_in[lvl][indx] != exp) { - printf("\nError at line %d in %s: %s[%d][%d][%d][%d] expect %.1f but got %.1f\n", - __LINE__, __FILE__, PRES_NAME, rec, lvl, lat, lon, exp, pres_in[lvl][indx]); - nerrs++; - goto fn_exit; - } - exp = SAMPLE_TEMP + i; - if (temp_in[lvl][indx] != exp) { - printf("\nError at line %d in %s: %s[%d][%d][%d][%d] expect %.1f but got %.1f\n", - __LINE__, __FILE__, TEMP_NAME, rec, lvl, lat, lon, exp, temp_in[lvl][indx]); - nerrs++; - goto fn_exit; - } - i++; - } - } /* next record */ - -fn_exit: - /* Close the file. */ - err = ncmpi_close(ncid); - CHECK_ERR - - if (pres_in != NULL) { - if (pres_in[0] != NULL) - free(pres_in[0]); - free(pres_in); - } - - /* check if there is any malloc residue */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } - -err_out: - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) - printf(FAIL_STR, nerrs); - else - printf(PASS_STR); - } - - MPI_Finalize(); - return (nerrs > 0); -} diff --git a/test/C/pres_temp_4D_wr.c b/test/C/pres_temp_4D_wr.c deleted file mode 100644 index 8039d0dcdb..0000000000 --- a/test/C/pres_temp_4D_wr.c +++ /dev/null @@ -1,263 +0,0 @@ -/* - * Copyright (C) 2003, Northwestern University and Argonne National Laboratory - * See COPYRIGHT notice in top-level directory. - */ -/* $Id$ */ - -/* - This is an example program which writes some 4D pressure and - temperatures. It is intended to illustrate the use of the netCDF - C API. The companion program pres_temp_4D_rd.c shows how - to read the netCDF data file created by this program. - - This program is part of the netCDF tutorial: - http://www.unidata.ucar.edu/software/netcdf/docs/netcdf-tutorial - - Full documentation of the netCDF C API can be found at: - http://www.unidata.ucar.edu/software/netcdf/docs/netcdf-c - - $Id$ -*/ - -#include -#include -#include -#include /* basename() */ -#include -#include -#include - -#include "pres_temp_4D.h" - -int main(int argc, char **argv) -{ - /* IDs for the netCDF file, dimensions, and variables. */ - int nprocs, rank, nerrs = 0; - int ncid; - int lon_dimid, lat_dimid, lvl_dimid, rec_dimid; - int lat_varid, lon_varid, pres_varid, temp_varid; - int dimids[NDIMS]; - int format = NC_FORMAT_CLASSIC; - - /* The start and count arrays will tell the netCDF library where to - write our data. */ - MPI_Offset start[NDIMS], count[NDIMS]; - - /* Program variables to hold the data we will write out. We will only - need enough space to hold one timestep of data; one record. */ - float **pres_out; /* [NLVL/nprocs][NLAT][NLON] */ - float **temp_out; /* [NLVL/nprocs][NLAT][NLON] */ - - /* These program variables hold the latitudes and longitudes. */ - float lats[NLAT], lons[NLON]; - - /* Loop indexes. */ - int lvl, lat, lon, rec, i = 0; - - /* Error handling. */ - int err; - - char *filename = FILE_NAME; - - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - if (argc > 3) { - if (!rank) - printf("Usage: %s [filename]\n", argv[0]); - MPI_Finalize(); - return 1; - } - - if (argc > 1) filename = argv[1]; - - if (argc > 2 && atoi(argv[2]) == 4) - format = NC_FORMAT_NETCDF4; - - if (rank == 0) { - char *cmd_str = (char *)malloc(strlen(argv[0]) + 256); - if (format == NC_FORMAT_NETCDF4) - sprintf(cmd_str, "*** TESTING C %s for writing NetCDF-4 file", basename(argv[0])); - else - sprintf(cmd_str, "*** TESTING C %s for writing classic file", basename(argv[0])); - printf("%-66s ------ ", cmd_str); - free(cmd_str); - } - - /* Create some pretend data. If this wasn't an example program, we - * would have some real data to write, for example, model - * output. - */ - for (lat = 0; lat < NLAT; lat++) - lats[lat] = START_LAT + 5. * lat; - for (lon = 0; lon < NLON; lon++) - lons[lon] = START_LON + 5. * lon; - - /* Set format. */ - err = ncmpi_set_default_format(format, NULL); - CHECK_ERR - - /* Create the file. */ - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, MPI_INFO_NULL, &ncid); - CHECK_ERR - - /* Define the dimensions. The record dimension is defined to have - * unlimited length - it can grow as needed. In this example it is - * the time dimension.*/ - err = ncmpi_def_dim(ncid, LVL_NAME, NLVL, &lvl_dimid); - CHECK_ERR - err = ncmpi_def_dim(ncid, LAT_NAME, NLAT, &lat_dimid); - CHECK_ERR - err = ncmpi_def_dim(ncid, LON_NAME, NLON, &lon_dimid); - CHECK_ERR - err = ncmpi_def_dim(ncid, REC_NAME, NC_UNLIMITED, &rec_dimid); - CHECK_ERR - - /* Define the coordinate variables. We will only define coordinate - variables for lat and lon. Ordinarily we would need to provide - an array of dimension IDs for each variable's dimensions, but - since coordinate variables only have one dimension, we can - simply provide the address of that dimension ID (&lat_dimid) and - similarly for (&lon_dimid). */ - err = ncmpi_def_var(ncid, LAT_NAME, NC_FLOAT, 1, &lat_dimid, &lat_varid); - CHECK_ERR - err = ncmpi_def_var(ncid, LON_NAME, NC_FLOAT, 1, &lon_dimid, &lon_varid); - CHECK_ERR - - /* Assign units attributes to coordinate variables. */ - err = ncmpi_put_att_text(ncid, lat_varid, UNITS, - strlen(DEGREES_NORTH), DEGREES_NORTH); - CHECK_ERR - err = ncmpi_put_att_text(ncid, lon_varid, UNITS, - strlen(DEGREES_EAST), DEGREES_EAST); - CHECK_ERR - - /* The dimids array is used to pass the dimids of the dimensions of - the netCDF variables. Both of the netCDF variables we are - creating share the same four dimensions. In C, the - unlimited dimension must come first on the list of dimids. */ - dimids[0] = rec_dimid; - dimids[1] = lvl_dimid; - dimids[2] = lat_dimid; - dimids[3] = lon_dimid; - - /* Define the netCDF variables for the pressure and temperature - * data. */ - err = ncmpi_def_var(ncid, PRES_NAME, NC_FLOAT, NDIMS, dimids, &pres_varid); - CHECK_ERR - err = ncmpi_def_var(ncid, TEMP_NAME, NC_FLOAT, NDIMS, dimids, &temp_varid); - CHECK_ERR - - /* Assign units attributes to the netCDF variables. */ - err = ncmpi_put_att_text(ncid, pres_varid, UNITS, - strlen(PRES_UNITS), PRES_UNITS); - CHECK_ERR - err = ncmpi_put_att_text(ncid, temp_varid, UNITS, - strlen(TEMP_UNITS), TEMP_UNITS); - CHECK_ERR - - /* End define mode. */ - err = ncmpi_enddef(ncid); - CHECK_ERR - - err = ncmpi_begin_indep_data(ncid); - /* Write the coordinate variable data. This will put the latitudes - and longitudes of our data grid into the netCDF file. */ - if (rank == 0) { - err = ncmpi_put_var_float(ncid, lat_varid, &lats[0]); - CHECK_ERR - err = ncmpi_put_var_float(ncid, lon_varid, &lons[0]); - CHECK_ERR - } - err = ncmpi_end_indep_data(ncid); - CHECK_ERR - - /* These settings tell netcdf to write one timestep of data. (The - setting of start[0] inside the loop below tells netCDF which - &data[0][0][0]); - timestep to write.) */ - count[0] = 1; - count[2] = NLAT; - count[3] = NLON; - start[2] = 0; - start[3] = 0; - - /* divide NLVL dimension among processes */ - count[1] = NLVL / nprocs; - start[1] = count[1] * rank; - if (rank < NLVL % nprocs) { - start[1] += rank; - count[1]++; - } - else { - start[1] += NLVL % nprocs; - } - if (count[1] == 0) - start[1] = 0; - - /* allocate write buffers */ - pres_out = (float **)malloc(sizeof(float *) * count[1] * 2); - temp_out = pres_out + count[1]; - if (count[1] > 0) { - pres_out[0] = (float *)malloc(sizeof(float) * count[1] * 2 * NLAT * NLON); - temp_out[0] = pres_out[0] + count[1] * NLAT * NLON; - for (i = 1; i < count[1]; i++) { - pres_out[i] = pres_out[i - 1] + NLAT * NLON; - temp_out[i] = temp_out[i - 1] + NLAT * NLON; - } - } - - /* initialize write buffers */ - i = (int)start[1] * NLAT * NLON; - for (lvl = 0; lvl < count[1]; lvl++) - for (lat = 0; lat < NLAT; lat++) - for (lon = 0; lon < NLON; lon++) { - pres_out[lvl][lat * NLON + lon] = SAMPLE_PRESSURE + i; - temp_out[lvl][lat * NLON + lon] = SAMPLE_TEMP + i++; - } - - /* Write the pretend data. This will write our surface pressure and - surface temperature data. The arrays only hold one timestep worth - of data. We will just rewrite the same data for each timestep. In - a real application, the data would change between timesteps. */ - - for (rec = 0; rec < NREC; rec++) { - start[0] = rec; - err = ncmpi_put_vara_float_all(ncid, pres_varid, start, count, &pres_out[0][0]); - CHECK_ERR - err = ncmpi_put_vara_float_all(ncid, temp_varid, start, count, &temp_out[0][0]); - CHECK_ERR - } - - /* Close the file. */ - err = ncmpi_close(ncid); - CHECK_ERR - - if (count[1] > 0) - free(pres_out[0]); - free(pres_out); - - /* check if there is any malloc residue */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } - - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) - printf(FAIL_STR, nerrs); - else - printf(PASS_STR); - } - - MPI_Finalize(); - - return (nerrs > 0); -} diff --git a/test/C/pres_temp_4D_wr_rd.c b/test/C/pres_temp_4D_wr_rd.c new file mode 100644 index 0000000000..217cd24d04 --- /dev/null +++ b/test/C/pres_temp_4D_wr_rd.c @@ -0,0 +1,490 @@ +/* + * Copyright (C) 2003, Northwestern University and Argonne National Laboratory + * See COPYRIGHT notice in top-level directory. + */ + +/* + This is an example program which writes some 4D pressure and + temperatures. It is intended to illustrate the use of the netCDF + C API. The companion program pres_temp_4D_rd.c shows how + to read the netCDF data file created by this program. + + This program is part of the netCDF tutorial: + http://www.unidata.ucar.edu/software/netcdf/docs/netcdf-tutorial + + Full documentation of the netCDF C API can be found at: + http://www.unidata.ucar.edu/software/netcdf/docs/netcdf-c +*/ + +#include +#include +#include + +#include +#include +#include + +/* We are writing and reading 4D data, a 2 x 6 x 12 lvl-lat-lon grid, with 2 + * timesteps of data. */ +#define NDIMS 4 +#define NLAT 6 +#define NLON 12 +#define LAT_NAME "latitude" +#define LON_NAME "longitude" +#define NREC 2 +#define REC_NAME "time" +#define LVL_NAME "level" +#define NLVL 512 + +/* Names of things. */ +#define PRES_NAME "pressure" +#define TEMP_NAME "temperature" +#define UNITS "units" +#define DEGREES_EAST "degrees_east" +#define DEGREES_NORTH "degrees_north" + +/* These are used to construct some example data, and to calculate the values + * we expect to find. */ +#define SAMPLE_PRESSURE 900.0 +#define SAMPLE_TEMP 9.0 +#define START_LAT 25.0 +#define START_LON -125.0 + +/* For the units attributes. */ +#define UNITS "units" +#define PRES_UNITS "hPa" +#define TEMP_UNITS "celsius" +#define LAT_UNITS "degrees_north" +#define LON_UNITS "degrees_east" +#define MAX_ATT_LEN 80 + +/* This is the name of the data file we will create and read back. */ +#define FILE_NAME "pres_temp_4D.nc" + + +/*----< pres_temp_4D_wr() >--------------------------------------------------*/ +int pres_temp_4D_wr(const char *filename, + int format, + int coll_io, + MPI_Info info) +{ + /* IDs for the netCDF file, dimensions, and variables. */ + int nprocs, rank, nerrs = 0, ncid; + int lon_dimid, lat_dimid, lvl_dimid, rec_dimid; + int lat_varid, lon_varid, pres_varid, temp_varid; + int dimids[NDIMS]; + + /* The start and count arrays will tell the netCDF library where to + write our data. */ + MPI_Offset start[NDIMS], count[NDIMS]; + + /* Program variables to hold the data we will write out. We will only + need enough space to hold one timestep of data; one record. */ + float **pres_out=NULL; /* [NLVL/nprocs][NLAT][NLON] */ + float **temp_out=NULL; /* [NLVL/nprocs][NLAT][NLON] */ + + /* These program variables hold the latitudes and longitudes. */ + float lats[NLAT], lons[NLON]; + + /* Loop indexes. */ + int lvl, lat, lon, rec, i = 0; + + /* Error handling. */ + int err; + + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + /* Create some pretend data. If this wasn't an example program, we + * would have some real data to write, for example, model + * output. + */ + for (lat = 0; lat < NLAT; lat++) + lats[lat] = START_LAT + 5. * lat; + for (lon = 0; lon < NLON; lon++) + lons[lon] = START_LON + 5. * lon; + + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + /* Create the file. */ + err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, info, &ncid); + CHECK_ERR + + /* Define the dimensions. The record dimension is defined to have + * unlimited length - it can grow as needed. In this example it is + * the time dimension.*/ + err = ncmpi_def_dim(ncid, LVL_NAME, NLVL, &lvl_dimid); + CHECK_ERR + err = ncmpi_def_dim(ncid, LAT_NAME, NLAT, &lat_dimid); + CHECK_ERR + err = ncmpi_def_dim(ncid, LON_NAME, NLON, &lon_dimid); + CHECK_ERR + err = ncmpi_def_dim(ncid, REC_NAME, NC_UNLIMITED, &rec_dimid); + CHECK_ERR + + /* Define the coordinate variables. We will only define coordinate + variables for lat and lon. Ordinarily we would need to provide + an array of dimension IDs for each variable's dimensions, but + since coordinate variables only have one dimension, we can + simply provide the address of that dimension ID (&lat_dimid) and + similarly for (&lon_dimid). */ + err = ncmpi_def_var(ncid, LAT_NAME, NC_FLOAT, 1, &lat_dimid, &lat_varid); + CHECK_ERR + err = ncmpi_def_var(ncid, LON_NAME, NC_FLOAT, 1, &lon_dimid, &lon_varid); + CHECK_ERR + + /* Assign units attributes to coordinate variables. */ + err = ncmpi_put_att_text(ncid, lat_varid, UNITS, + strlen(DEGREES_NORTH), DEGREES_NORTH); + CHECK_ERR + err = ncmpi_put_att_text(ncid, lon_varid, UNITS, + strlen(DEGREES_EAST), DEGREES_EAST); + CHECK_ERR + + /* The dimids array is used to pass the dimids of the dimensions of + the netCDF variables. Both of the netCDF variables we are + creating share the same four dimensions. In C, the + unlimited dimension must come first on the list of dimids. */ + dimids[0] = rec_dimid; + dimids[1] = lvl_dimid; + dimids[2] = lat_dimid; + dimids[3] = lon_dimid; + + /* Define the netCDF variables for the pressure and temperature + * data. */ + err = ncmpi_def_var(ncid, PRES_NAME, NC_FLOAT, NDIMS, dimids, &pres_varid); + CHECK_ERR + err = ncmpi_def_var(ncid, TEMP_NAME, NC_FLOAT, NDIMS, dimids, &temp_varid); + CHECK_ERR + + /* Assign units attributes to the netCDF variables. */ + err = ncmpi_put_att_text(ncid, pres_varid, UNITS, + strlen(PRES_UNITS), PRES_UNITS); + CHECK_ERR + err = ncmpi_put_att_text(ncid, temp_varid, UNITS, + strlen(TEMP_UNITS), TEMP_UNITS); + CHECK_ERR + + /* End define mode. */ + err = ncmpi_enddef(ncid); + CHECK_ERR + + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + + /* Write the coordinate variable data. This will put the latitudes + and longitudes of our data grid into the netCDF file. */ + if (rank == 0) { + err = ncmpi_put_var_float(ncid, lat_varid, &lats[0]); + CHECK_ERR + err = ncmpi_put_var_float(ncid, lon_varid, &lons[0]); + CHECK_ERR + } + + if (coll_io) { + err = ncmpi_end_indep_data(ncid); + CHECK_ERR + } + + /* These settings tell netcdf to write one timestep of data. (The + setting of start[0] inside the loop below tells netCDF which + &data[0][0][0]); + timestep to write.) */ + count[0] = 1; + count[2] = NLAT; + count[3] = NLON; + start[2] = 0; + start[3] = 0; + + /* divide NLVL dimension among processes */ + count[1] = NLVL / nprocs; + start[1] = count[1] * rank; + if (rank < NLVL % nprocs) { + start[1] += rank; + count[1]++; + } + else { + start[1] += NLVL % nprocs; + } + if (count[1] == 0) + start[1] = 0; + + /* allocate write buffers */ + pres_out = (float **)malloc(sizeof(float *) * count[1] * 2); + temp_out = pres_out + count[1]; + if (count[1] > 0) { + pres_out[0] = (float *)malloc(sizeof(float) * count[1] * 2 * NLAT * NLON); + temp_out[0] = pres_out[0] + count[1] * NLAT * NLON; + for (i = 1; i < count[1]; i++) { + pres_out[i] = pres_out[i - 1] + NLAT * NLON; + temp_out[i] = temp_out[i - 1] + NLAT * NLON; + } + } + + /* initialize write buffers */ + i = (int)start[1] * NLAT * NLON; + for (lvl = 0; lvl < count[1]; lvl++) + for (lat = 0; lat < NLAT; lat++) + for (lon = 0; lon < NLON; lon++) { + pres_out[lvl][lat * NLON + lon] = SAMPLE_PRESSURE + i; + temp_out[lvl][lat * NLON + lon] = SAMPLE_TEMP + i++; + } + + /* Write the pretend data. This will write our surface pressure and + surface temperature data. The arrays only hold one timestep worth + of data. We will just rewrite the same data for each timestep. In + a real application, the data would change between timesteps. */ + + for (rec = 0; rec < NREC; rec++) { + start[0] = rec; + if (coll_io) + err = ncmpi_put_vara_float_all(ncid, pres_varid, start, count, &pres_out[0][0]); + else + err = ncmpi_put_vara_float(ncid, pres_varid, start, count, &pres_out[0][0]); + CHECK_ERR + if (coll_io) + err = ncmpi_put_vara_float_all(ncid, temp_varid, start, count, &temp_out[0][0]); + else + err = ncmpi_put_vara_float(ncid, temp_varid, start, count, &temp_out[0][0]); + CHECK_ERR + } + + /* Close the file. */ + err = ncmpi_close(ncid); + CHECK_ERR + + if (pres_out != NULL) { + if (count[1] > 0) + free(pres_out[0]); + free(pres_out); + } + + return nerrs; +} + +/*----< pres_temp_4D_rd() >--------------------------------------------------*/ +int pres_temp_4D_rd(const char *filename, + int coll_io, + MPI_Info info) +{ + int rank, nprocs, ncid, pres_varid, temp_varid, lat_varid, lon_varid; + + /* The start and count arrays will tell the netCDF library where to + read our data. */ + MPI_Offset start[NDIMS], count[NDIMS]; + + /* Program variables to hold the data we will read. We will only + need enough space to hold one timestep of data; one record. */ + float **pres_in = NULL; /* [NLVL/nprocs][NLAT][NLON] */ + float **temp_in = NULL; /* [NLVL/nprocs][NLAT][NLON] */ + + /* These program variables hold the latitudes and longitudes. */ + float lats[NLAT], lons[NLON]; + + /* Loop indexes. */ + int lvl, lat, lon, rec, i = 0; + + /* Error handling. */ + int err, nerrs = 0; + + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + /* Open the file. */ + err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); + if (err != NC_NOERR) { /* fatal error */ + if (rank == 0) + fprintf(stderr,"Error: failed to open file %s (%s)\n",filename,ncmpi_strerror(err)); + nerrs++; + goto err_out; + } + + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + + /* Get the varids of the latitude and longitude coordinate variables. */ + err = ncmpi_inq_varid(ncid, LAT_NAME, &lat_varid); + CHECK_ERR + err = ncmpi_inq_varid(ncid, LON_NAME, &lon_varid); + CHECK_ERR + + /* Read the coordinate variable data. */ + memset(lats, 0, sizeof(float) * NLAT); + memset(lons, 0, sizeof(float) * NLON); + if (coll_io) { + err = ncmpi_get_var_float_all(ncid, lat_varid, &lats[0]); + CHECK_ERR + err = ncmpi_get_var_float_all(ncid, lon_varid, &lons[0]); + CHECK_ERR + } + else { + err = ncmpi_get_var_float(ncid, lat_varid, &lats[0]); + CHECK_ERR + err = ncmpi_get_var_float(ncid, lon_varid, &lons[0]); + CHECK_ERR + } + + /* Check the coordinate variable data. */ + for (lat = 0; lat < NLAT; lat++) { + float exp = START_LAT + 5. * lat; + if (lats[lat] != exp) { + fprintf(stderr,"\nError at line %d in %s: %s[%d] expect %.1f but got %.1f\n", + __LINE__, __FILE__, LAT_NAME, lat, exp, lats[lat]); + nerrs++; + break; + } + } + for (lon = 0; lon < NLON; lon++) { + float exp = START_LON + 5. * lon; + if (lons[lon] != START_LON + 5. * lon) { + fprintf(stderr,"\nError at line %d in %s: %s[%d] expect %.1f but got %.1f\n", + __LINE__, __FILE__, LON_NAME, lon, exp, lons[lon]); + nerrs++; + break; + } + } + + /* Get the varids of the pressure and temperature netCDF variables. */ + err = ncmpi_inq_varid(ncid, PRES_NAME, &pres_varid); + CHECK_ERR + err = ncmpi_inq_varid(ncid, TEMP_NAME, &temp_varid); + CHECK_ERR + + /* Read the data. Since we know the contents of the file we know that the + * data arrays in this program are the correct size to hold one timestep. + */ + count[0] = 1; + count[2] = NLAT; + count[3] = NLON; + start[2] = 0; + start[3] = 0; + + /* divide NLVL dimension among processes */ + count[1] = NLVL / nprocs; + start[1] = count[1] * rank; + if (rank < NLVL % nprocs) { + start[1] += rank; + count[1]++; + } + else { + start[1] += NLVL % nprocs; + } + if (count[1] == 0) + start[1] = 0; + + /* allocate read buffers */ + pres_in = (float **)malloc(sizeof(float *) * count[1] * 2); + temp_in = pres_in + count[1]; + if (count[1] > 0) { + pres_in[0] = (float *)malloc(sizeof(float) * count[1] * 2 * NLAT * NLON); + temp_in[0] = pres_in[0] + count[1] * NLAT * NLON; + for (i = 1; i < count[1]; i++) { + pres_in[i] = pres_in[i - 1] + NLAT * NLON; + temp_in[i] = temp_in[i - 1] + NLAT * NLON; + } + } + + /* Read and check one record at a time. */ + for (rec = 0; rec < NREC; rec++) { + start[0] = rec; + if (coll_io) + err = ncmpi_get_vara_float_all(ncid, pres_varid, start, count, &pres_in[0][0]); + else + err = ncmpi_get_vara_float(ncid, pres_varid, start, count, &pres_in[0][0]); + CHECK_ERR + if (coll_io) + err = ncmpi_get_vara_float_all(ncid, temp_varid, start, count, &temp_in[0][0]); + else + err = ncmpi_get_vara_float(ncid, temp_varid, start, count, &temp_in[0][0]); + CHECK_ERR + + /* Check the data. */ + i = (int)start[1] * NLAT * NLON; + for (lvl = 0; lvl < count[1]; lvl++) + for (lat = 0; lat < NLAT; lat++) + for (lon = 0; lon < NLON; lon++) { + float exp = SAMPLE_PRESSURE + i; + int indx = lat * NLON + lon; + if (pres_in[lvl][indx] != exp) { + fprintf(stderr,"\nError at line %d in %s: %s[%d][%d][%d][%d] expect %.1f but got %.1f\n", + __LINE__, __FILE__, PRES_NAME, rec, lvl, lat, lon, exp, pres_in[lvl][indx]); + nerrs++; + goto loop_exit; + } + exp = SAMPLE_TEMP + i; + if (temp_in[lvl][indx] != exp) { + fprintf(stderr,"\nError at line %d in %s: %s[%d][%d][%d][%d] expect %.1f but got %.1f\n", + __LINE__, __FILE__, TEMP_NAME, rec, lvl, lat, lon, exp, temp_in[lvl][indx]); + nerrs++; + goto loop_exit; + } + i++; + } + +loop_exit: + MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); + if (nerrs > 0) break; + } /* next record */ + + /* Close the file. */ + err = ncmpi_close(ncid); + CHECK_ERR + + if (pres_in != NULL) { + if (pres_in[0] != NULL) + free(pres_in[0]); + free(pres_in); + } + +err_out: + return nerrs; +} + +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) +{ + int nerrs; + + nerrs = pres_temp_4D_wr(out_path, format, coll_io, info); + if (nerrs > 0) return nerrs; + + MPI_Barrier(MPI_COMM_WORLD); + + nerrs = pres_temp_4D_rd(out_path, coll_io, info); + if (nerrs > 0) return nerrs; + + return 0; +} + +int main(int argc, char **argv) { + + int err; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "write/read netCDF file", opt, test_io); + + MPI_Finalize(); + + return err; +} + diff --git a/test/C/seq_runs.sh b/test/C/seq_runs.sh index 1098e28969..eb3c73faf6 100755 --- a/test/C/seq_runs.sh +++ b/test/C/seq_runs.sh @@ -1,58 +1,29 @@ -#!/bin/sh +#!/bin/bash # -# Copyright (C) 2003, Northwestern University and Argonne National Laboratory +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory # See COPYRIGHT notice in top-level directory. # # Exit immediately if a command exits with a non-zero status. set -e -VALIDATOR=../../src/utils/ncvalidator/ncvalidator -NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff +DRY_RUN=no +VERBOSE=no -# remove file system type prefix if there is any -OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $TESTSEQRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $TESTSEQRUN $@ + fi +} -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" -else - safe_modes="0" -fi +exe_name=`basename $1` # prevent user environment setting of PNETCDF_HINTS to interfere unset PNETCDF_HINTS -for j in ${safe_modes} ; do - export PNETCDF_SAFE_MODE=$j - # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - ${TESTSEQRUN} ./pres_temp_4D_wr ${TESTOUTDIR}/pres_temp_4D.nc - ${TESTSEQRUN} ./pres_temp_4D_rd ${TESTOUTDIR}/pres_temp_4D.nc - # echo "--- validating file ${TESTOUTDIR}/pres_temp_4D.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/pres_temp_4D.nc - # echo "" +run_cmd ./$1 -q -o ${TESTOUTDIR}/${exe_name}.nc - if test "x${ENABLE_BURST_BUFFER}" = x1 ; then - echo "" - echo "---- testing burst buffering" - export PNETCDF_HINTS="nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" - ${TESTSEQRUN} ./pres_temp_4D_wr ${TESTOUTDIR}/pres_temp_4D.bb.nc - ${TESTSEQRUN} ./pres_temp_4D_rd ${TESTOUTDIR}/pres_temp_4D.bb.nc - unset PNETCDF_HINTS - # echo "--- validating file ${TESTOUTDIR}/pres_temp_4D.bb.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/pres_temp_4D.bb.nc - # echo "" - - # echo "--- ncmpidiff pres_temp_4D.nc pres_temp_4D.bb.nc ---" - ${TESTSEQRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/pres_temp_4D.nc ${TESTOUTDIR}/pres_temp_4D.bb.nc - fi - # echo "" - - if test "${ENABLE_NETCDF4}" = 1 ; then - ${TESTSEQRUN} ./pres_temp_4D_wr ${TESTOUTDIR}/pres_temp_4D.nc4 4 - ${TESTSEQRUN} ./pres_temp_4D_rd ${TESTOUTDIR}/pres_temp_4D.nc4 - # Validator does not support nc4 - fi - # echo "" -done -rm -f ${OUTDIR}/*.nc -rm -f ${OUTDIR}/*.nc4 diff --git a/test/CXX/Makefile.am b/test/CXX/Makefile.am index 532133e059..07673e07df 100644 --- a/test/CXX/Makefile.am +++ b/test/CXX/Makefile.am @@ -15,47 +15,46 @@ AM_CPPFLAGS += -I$(top_builddir)/src/binding/cxx AM_CPPFLAGS += -I$(srcdir)/../common AM_CPPFLAGS += -I$(top_builddir)/src/include -LDADD = $(top_builddir)/src/libs/libpnetcdf.la ../common/libtestutils.la +LDADD = ../common/libtestutils.la $(top_builddir)/src/libs/libpnetcdf.la LDADD += @NETCDF4_LDFLAGS@ @ADIOS_LDFLAGS@ @NETCDF4_LIBS@ @ADIOS_LIBS@ if DECL_MPI_OFFSET AM_CPPFLAGS += -DHAVE_DECL_MPI_OFFSET endif -TESTPROGRAMS = nctst \ - test_classic - -check_PROGRAMS = $(TESTPROGRAMS) +check_PROGRAMS = nctst \ + test_classic # autimake 1.11.3 has not yet implemented AM_TESTS_ENVIRONMENT # For newer versions, we can use AM_TESTS_ENVIRONMENT instead TESTS_ENVIRONMENT = export SED="$(SED)"; TESTS_ENVIRONMENT += export srcdir="$(srcdir)"; +TESTS_ENVIRONMENT += export top_builddir="$(top_builddir)"; TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; TESTS_ENVIRONMENT += export TESTMPIRUN="$(TESTMPIRUN)"; -TESTS_ENVIRONMENT += export PNETCDF_DEBUG="$(PNETCDF_DEBUG)"; -TESTS_ENVIRONMENT += export TESTPROGRAMS="$(TESTPROGRAMS)"; TESTS_ENVIRONMENT += export check_PROGRAMS="$(check_PROGRAMS)"; -TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER="$(ENABLE_BURST_BUFFER)"; -TESTS_ENVIRONMENT += export ENABLE_NETCDF4="$(ENABLE_NETCDF4)"; -TESTS = $(TESTPROGRAMS) -LOG_COMPILER = $(srcdir)/wrap_runs.sh +TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER=@ENABLE_BURST_BUFFER@; +TESTS_ENVIRONMENT += export ENABLE_NETCDF4=@ENABLE_NETCDF4@; +TESTS_ENVIRONMENT += export ENABLE_THREAD_SAFE=@ENABLE_THREAD_SAFE@; +TESTS_ENVIRONMENT += export MIMIC_LUSTRE=@MIMIC_LUSTRE@; + +TESTS = $(check_PROGRAMS) +LOG_COMPILER = $(srcdir)/seq_runs.sh -EXTRA_DIST = wrap_runs.sh parallel_run.sh +check_SCRIPTS = seq_runs.sh parallel_run.sh -NC_FILES = $(TESTPROGRAMS:%=$(TESTOUTDIR)/%.nc) \ - $(TESTPROGRAMS:%=$(TESTOUTDIR)/%.bb.nc) +EXTRA_DIST = seq_runs.sh parallel_run.sh -CLEANFILES = $(NC_FILES) \ +CLEANFILES = $(TESTOUTDIR)/*.nc \ core core.* *.gcda *.gcno *.gcov gmon.out a.out ../common/libtestutils.la: set -e; cd ../common && $(MAKE) $(MFLAGS) tests -ptest ptests ptest4: $(TESTPROGRAMS) +ptest ptests ptest4: $(check_PROGRAMS) @echo "===========================================================" @echo " $(subdir): Parallel testing on 4 MPI processes" @echo "===========================================================" diff --git a/test/CXX/nctst.cpp b/test/CXX/nctst.cpp index 84cd1a4add..f7138ac001 100644 --- a/test/CXX/nctst.cpp +++ b/test/CXX/nctst.cpp @@ -362,7 +362,7 @@ cdl_name(const char* path) while (*(cp-1) != '/' && cp != path) // assumes UNIX path separator cp--; - static char np[NC_MAX_NAME]; + static char np[NC_MAX_NAME+1]; strncpy(&np[0], cp, NC_MAX_NAME); char* ep = np + strlen(np); @@ -498,7 +498,7 @@ void dump(const MPI_Comm &comm, const char* path) } /* Test everything for classic, 64-bit offset, 64-bit data, and netCDF4 files. */ -#ifdef ENABLE_NETCDF4 +#if PNETCDF_DRIVER_NETCDF4 == 1 #define NUM_FORMATS (4) #else #define NUM_FORMATS (3) @@ -509,12 +509,16 @@ main(int argc, char* argv[]) // test new netCDF interface { char filename[256]; int rank, nprocs; + double timing; MPI_Init(&argc, &argv); + + timing = MPI_Wtime(); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - if (argc > 2) { + if (argc < 2) { if (!rank) printf("Usage: %s [filename]\n",argv[0]); MPI_Finalize(); return 1; @@ -525,13 +529,13 @@ main(int argc, char* argv[]) // test new netCDF interface if (rank == 0) { std::ostringstream cmd_str; cmd_str << "*** TESTING C++ " << basename(argv[0]) << - " for APIs with different netCDF formats"; - printf("%-66s ------ ", cmd_str.str().c_str()); + " - APIs with different netCDF formats"; + printf("%-63s -- ", cmd_str.str().c_str()); } // Set up the format constants. NcmpiFile::FileFormat format[NUM_FORMATS] = -#ifdef ENABLE_NETCDF4 +#if PNETCDF_DRIVER_NETCDF4 == 1 {NcmpiFile::classic, NcmpiFile::classic2, NcmpiFile::nc4, NcmpiFile::classic5}; #else {NcmpiFile::classic, NcmpiFile::classic2, NcmpiFile::classic5}; @@ -568,13 +572,16 @@ main(int argc, char* argv[]) // test new netCDF interface if (err == NC_NOERR) { MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", + printf("heap memory allocated by PnetCDF internally has " OFFFMT " bytes yet to be freed\n", sum_size); } + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); + if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf(PASS_STR, timing); } MPI_Finalize(); diff --git a/test/CXX/parallel_run.sh b/test/CXX/parallel_run.sh index 4f887ad275..3c8574ee09 100755 --- a/test/CXX/parallel_run.sh +++ b/test/CXX/parallel_run.sh @@ -1,73 +1,41 @@ #!/bin/bash # -# Copyright (C) 2018, Northwestern University and Argonne National Laboratory +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory # See COPYRIGHT notice in top-level directory. # # Exit immediately if a command exits with a non-zero status. set -e -VALIDATOR=../../src/utils/ncvalidator/ncvalidator -NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff +DRY_RUN=no +VERBOSE=no -# remove file system type prefix if there is any -OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $MPIRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $MPIRUN $@ + fi +} MPIRUN=`echo ${TESTMPIRUN} | ${SED} -e "s/NP/$1/g"` # echo "MPIRUN = ${MPIRUN}" -# echo "TESTPROGRAMS=${TESTPROGRAMS}" - -# echo "PNETCDF_DEBUG = ${PNETCDF_DEBUG}" -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" -else - safe_modes="0" -fi +# echo "check_PROGRAMS=${check_PROGRAMS}" # prevent user environment setting of PNETCDF_HINTS to interfere unset PNETCDF_HINTS -for i in ${TESTPROGRAMS} ; do - for j in ${safe_modes} ; do - for intra_aggr in 0 1 ; do - if test "$j" = 1 ; then # test only in safe mode - export PNETCDF_HINTS="romio_no_indep_rw=true" - else - export PNETCDF_HINTS= - fi - if test "$intra_aggr" = 1 ; then - export PNETCDF_HINTS="${PNETCDF_HINTS};nc_num_aggrs_per_node=2" - fi - export PNETCDF_SAFE_MODE=$j - # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - ${MPIRUN} ./$i ${TESTOUTDIR}/$i.nc - - # echo "--- validating file ${TESTOUTDIR}/$i.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.nc - # echo "" +if test "x$MIMIC_LUSTRE" != x1 ; then + PNETCDF_HINTS="cb_nodes=2" +fi - if test "x${ENABLE_BURST_BUFFER}" = x1 ; then - # echo "---- test burst buffering feature" - saved_PNETCDF_HINTS=${PNETCDF_HINTS} - export PNETCDF_HINTS="${PNETCDF_HINTS};nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" - ${MPIRUN} ./$i ${TESTOUTDIR}/$i.bb.nc - export PNETCDF_HINTS=${saved_PNETCDF_HINTS} +for i in ${check_PROGRAMS} ; do - # echo "--- validating file ${TESTOUTDIR}/$i.bb.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.bb.nc + exe_name=`basename $i` - # echo "--- ncmpidiff $i.nc $i.bb.nc ---" - ${MPIRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/$i.nc ${TESTOUTDIR}/$i.bb.nc - fi + run_cmd ./$i -q ${TESTOUTDIR}/${exe_name}.nc - if test "x${ENABLE_NETCDF4}" = x1 ; then - # echo "test netCDF-4 feature" - ${MPIRUN} ./$i ${TESTOUTDIR}/$i.nc - # Validator does not support NetCDF-4 format - fi - done - done - rm -f ${OUTDIR}/$i.nc - rm -f ${OUTDIR}/$i.bb.nc -done +done # check_PROGRAMS diff --git a/test/CXX/seq_runs.sh b/test/CXX/seq_runs.sh new file mode 100755 index 0000000000..4537b472c1 --- /dev/null +++ b/test/CXX/seq_runs.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory +# See COPYRIGHT notice in top-level directory. +# + +# Exit immediately if a command exits with a non-zero status. +set -e + +DRY_RUN=no +VERBOSE=no + +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $TESTSEQRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $TESTSEQRUN $@ + fi +} + +exe_name=`basename $1` + +# prevent user environment setting of PNETCDF_HINTS to interfere +unset PNETCDF_HINTS + +run_cmd ./$1 ${TESTOUTDIR}/${exe_name}.nc + diff --git a/test/CXX/test_classic.cpp b/test/CXX/test_classic.cpp index c600181af9..5ebfc5e87c 100644 --- a/test/CXX/test_classic.cpp +++ b/test/CXX/test_classic.cpp @@ -15,10 +15,14 @@ int main( int argc, char *argv[] ) { char filename[256]; int rank, nerrs=0, verbose=0; + double timing; MPI_Init(&argc, &argv); + + timing = MPI_Wtime(); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); - if (argc > 2) { + if (argc < 2) { if (!rank) printf("Usage: %s [filename]\n",argv[0]); MPI_Finalize(); return 1; @@ -32,8 +36,8 @@ int main( int argc, char *argv[] ) std::ostringstream cmd_str; cmd_str << "*** TESTING C++ " << basename(argv[0]) << - " for creation of classic format file"; - printf("%-66s ------ ", cmd_str.str().c_str()); + " - creation of classic format file"; + printf("%-63s -- ", cmd_str.str().c_str()); } try @@ -88,13 +92,16 @@ int main( int argc, char *argv[] ) if (err == NC_NOERR) { MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", + printf("heap memory allocated by PnetCDF internally has " OFFFMT " bytes yet to be freed\n", sum_size); } + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); + if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf(PASS_STR, timing); } MPI_Finalize(); diff --git a/test/CXX/wrap_runs.sh b/test/CXX/wrap_runs.sh deleted file mode 100755 index d34ca7b47f..0000000000 --- a/test/CXX/wrap_runs.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2003, Northwestern University and Argonne National Laboratory -# See COPYRIGHT notice in top-level directory. -# - -# Exit immediately if a command exits with a non-zero status. -set -e - -VALIDATOR=../../src/utils/ncvalidator/ncvalidator -NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff - -outfile=`basename $1` - -# remove file system type prefix if there is any -OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` - -# echo "PNETCDF_DEBUG = ${PNETCDF_DEBUG}" -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" -else - safe_modes="0" -fi - -# prevent user environment setting of PNETCDF_HINTS to interfere -unset PNETCDF_HINTS - -for j in ${safe_modes} ; do - export PNETCDF_SAFE_MODE=$j - # echo "---- set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - ${TESTSEQRUN} ./$1 ${TESTOUTDIR}/$outfile.nc - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$outfile.nc - # echo "" - - if test "x${ENABLE_BURST_BUFFER}" = x1 ; then - echo "" - echo "---- testing burst buffering" - - export PNETCDF_HINTS="nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" - ${TESTSEQRUN} $1 ${TESTOUTDIR}/$outfile.bb.nc - unset PNETCDF_HINTS - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$outfile.bb.nc - - # echo "--- ncmpidiff $outfile.nc $outfile.bb.nc ---" - ${TESTSEQRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/$outfile.nc ${TESTOUTDIR}/$outfile.bb.nc - fi -done -rm -f ${OUTDIR}/$outfile.nc -rm -f ${OUTDIR}/$outfile.bb.nc - diff --git a/test/F90/Makefile.am b/test/F90/Makefile.am index 4d48127679..35fd597571 100644 --- a/test/F90/Makefile.am +++ b/test/F90/Makefile.am @@ -13,7 +13,7 @@ AM_DEFAULT_SOURCE_EXT = .f90 AM_FCFLAGS = $(FC_MODINC)$(top_builddir)/src/binding/f90 \ $(FC_MODINC)../common $(FFREEFORMFLAG) -LDADD = $(top_builddir)/src/libs/libpnetcdf.la ../common/libtestutils.la +LDADD = ../common/libtestutils.la $(top_builddir)/src/libs/libpnetcdf.la LDADD += @NETCDF4_LDFLAGS@ @ADIOS_LDFLAGS@ @NETCDF4_LIBS@ @ADIOS_LIBS@ # suppress warning messages when using NAG Fortran compiler @@ -28,89 +28,89 @@ if NAGFORT AM_FCFLAGS += -w=uparam endif -TESTPROGRAMS = tst_f90 \ - f90tst_vars \ - tst_types2 \ - tst_f90_cdf5 \ - f90tst_vars2 \ - f90tst_vars3 \ - f90tst_vars4 \ - test_intent \ - test_attr_int64 \ - test_fill - -PARALLEL_PROGS = f90tst_parallel \ +check_PROGRAMS = tst_f90 \ + f90tst_vars \ + tst_types2 \ + tst_f90_cdf5 \ + f90tst_vars2 \ + f90tst_vars3 \ + f90tst_vars4 \ + test_intent \ + test_attr_int64 \ + test_fill \ + f90tst_parallel \ f90tst_parallel2 \ f90tst_parallel3 \ - f90tst_parallel4 - -check_PROGRAMS = $(TESTPROGRAMS) \ - $(PARALLEL_PROGS) \ + f90tst_parallel4 \ tst_io # autimake 1.11.3 has not yet implemented AM_TESTS_ENVIRONMENT # For newer versions, we can use AM_TESTS_ENVIRONMENT instead -# AM_TESTS_ENVIRONMENT = TESTPROGRAMS="$(TESTPROGRAMS)" ; export TESTPROGRAMS; +# AM_TESTS_ENVIRONMENT = check_PROGRAMS="$(check_PROGRAMS)" ; export check_PROGRAMS; # AM_TESTS_ENVIRONMENT += TESTSEQRUN="$(TESTSEQRUN)" ; export TESTSEQRUN; # AM_TESTS_ENVIRONMENT += TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)" ; export TESTOUTDIR; TESTS_ENVIRONMENT = export SED="$(SED)"; TESTS_ENVIRONMENT += export srcdir="$(srcdir)"; +TESTS_ENVIRONMENT += export top_builddir="$(top_builddir)"; TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; TESTS_ENVIRONMENT += export TESTMPIRUN="$(TESTMPIRUN)"; -TESTS_ENVIRONMENT += export PNETCDF_DEBUG="$(PNETCDF_DEBUG)"; -TESTS_ENVIRONMENT += export TESTPROGRAMS="$(TESTPROGRAMS)"; TESTS_ENVIRONMENT += export check_PROGRAMS="$(check_PROGRAMS)"; -TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER="$(ENABLE_BURST_BUFFER)"; -TESTS_ENVIRONMENT += export PARALLEL_PROGS="$(PARALLEL_PROGS)"; -TESTS = $(TESTPROGRAMS) seq_runs.sh +TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER=@ENABLE_BURST_BUFFER@; +TESTS_ENVIRONMENT += export ENABLE_THREAD_SAFE=@ENABLE_THREAD_SAFE@; +TESTS_ENVIRONMENT += export MIMIC_LUSTRE=@MIMIC_LUSTRE@; + +TESTS = $(check_PROGRAMS) TEST_EXTENSIONS = .sh -LOG_COMPILER = $(srcdir)/wrap_runs.sh +LOG_COMPILER = $(srcdir)/seq_runs.sh SH_LOG_COMPILER = -EXTRA_DIST = seq_runs.sh wrap_runs.sh parallel_run.sh +check_SCRIPTS = seq_runs.sh parallel_run.sh -NC_FILES = $(check_PROGRAMS:%=$(TESTOUTDIR)/%.nc) \ - $(check_PROGRAMS:%=$(TESTOUTDIR)/%.bb.nc) +EXTRA_DIST = seq_runs.sh parallel_run.sh -CLEANFILES = *.$(FC_MODEXT) core core.* *.gcda *.gcno *.gcov gmon.out \ - $(TESTOUTDIR)/tst_io1.nc $(TESTOUTDIR)/tst_io1.nc0 \ - $(TESTOUTDIR)/testfile.nc $(NC_FILES) +CLEANFILES = *.$(FC_MODEXT) core core.* *.gcda *.gcno *.gcov gmon.out ../common/libtestutils.la: set -e; cd ../common && $(MAKE) $(MFLAGS) tests -ptest ptest4: $(PARALLEL_PROGS) +ptest ptest4: $(check_PROGRAMS) @echo "===========================================================" @echo " $(subdir): Parallel testing on 4 MPI processes" @echo "===========================================================" @$(TESTS_ENVIRONMENT) \ $(srcdir)/parallel_run.sh 4 || exit 1 -ptest2: $(PARALLEL_PROGS) +ptest2: $(check_PROGRAMS) @echo "===========================================================" @echo " $(subdir): Parallel testing on 2 MPI processes" @echo "===========================================================" @$(TESTS_ENVIRONMENT) \ $(srcdir)/parallel_run.sh 2 || exit 1 -ptest8: $(PARALLEL_PROGS) +ptest6: $(check_PROGRAMS) + @echo "===========================================================" + @echo " $(subdir): Parallel testing on 6 MPI processes" + @echo "===========================================================" + @$(TESTS_ENVIRONMENT) \ + $(srcdir)/parallel_run.sh 6 || exit 1 + +ptest8: $(check_PROGRAMS) @echo "===========================================================" @echo " $(subdir): Parallel testing on 8 MPI processes" @echo "===========================================================" @$(TESTS_ENVIRONMENT) \ $(srcdir)/parallel_run.sh 8 || exit 1 -ptest10: $(PARALLEL_PROGS) +ptest10: $(check_PROGRAMS) @echo "===========================================================" @echo " $(subdir): Parallel testing on 10 MPI processes" @echo "===========================================================" @$(TESTS_ENVIRONMENT) \ $(srcdir)/parallel_run.sh 10 || exit 1 -ptests: ptest2 ptest4 ptest8 ptest10 -ptest6: +ptests: ptest4 ptest8 # build check targets but not invoke tests-local: all $(check_PROGRAMS) diff --git a/test/F90/f90tst_parallel.f90 b/test/F90/f90tst_parallel.f90 index d6223a334a..725e1aa976 100644 --- a/test/F90/f90tst_parallel.f90 +++ b/test/F90/f90tst_parallel.f90 @@ -56,28 +56,35 @@ program f90tst_parallel integer :: mode_flag integer :: nvars, ngatts, ndims, unlimdimid, file_format integer :: x, y - integer :: p, my_rank, err, ierr, get_args + integer :: nprocs, my_rank, err, ierr, get_args integer(KIND=MPI_OFFSET_KIND) :: start(MAX_DIMS), count(MAX_DIMS) integer(KIND=MPI_OFFSET_KIND) :: nx_ll, ny_ll - character(LEN=256) filename, cmd, msg + character(LEN=256) out_path, in_path, cmd, msg + logical keep_files + double precision timing call MPI_Init(ierr) + + timing = MPI_Wtime() + call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr) - call MPI_Comm_size(MPI_COMM_WORLD, p, ierr) + call MPI_Comm_size(MPI_COMM_WORLD, nprocs, ierr) - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any if (my_rank .EQ. 0) then - filename = FILE_NAME - err = get_args(cmd, filename) + out_path = FILE_NAME + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr) -! if (p .ne. 4 .AND. my_rank .eq. 0) then -! print *, 'Warning: ',trim(cmd),' is design to run on 4 processes.' -! endif + ! if (nprocs .ne. 4 .AND. my_rank .eq. 0) then + ! print *, 'Warning: ',trim(cmd),' is design to run on 4 processes.' + ! endif ! Create some pretend data. do x = 1, NX / 2 @@ -88,7 +95,7 @@ program f90tst_parallel ! Create the netCDF file. mode_flag = IOR(NF90_CLOBBER, NF90_64BIT_DATA) - call handle_err(nf90mpi_create(MPI_COMM_WORLD, filename, mode_flag, MPI_INFO_NULL, ncid)) + call handle_err(nf90mpi_create(MPI_COMM_WORLD, out_path, mode_flag, MPI_INFO_NULL, ncid)) ! Define the dimensions. nx_ll = NX @@ -100,6 +107,9 @@ program f90tst_parallel ! Define the variable. call handle_err(nf90mpi_def_var(ncid, "data", NF90_INT, dimids, varid)) + ! fill with default fill value + call handle_err(nf90mpi_def_var_fill(ncid, varid, 0, NF90_FILL_INT)) + call handle_err(nf90mpi_enddef(ncid)) ! Determine what part of the variable will be written for this @@ -125,7 +135,7 @@ program f90tst_parallel call handle_err(nf90mpi_close(ncid)) ! Reopen the file. - call handle_err(nf90mpi_open(MPI_COMM_WORLD, filename, nf90_nowrite, MPI_INFO_NULL, ncid)) + call handle_err(nf90mpi_open(MPI_COMM_WORLD, out_path, nf90_nowrite, MPI_INFO_NULL, ncid)) ! Check some stuff out. call handle_err(nf90mpi_inquire(ncid, ndims, nvars, ngatts, unlimdimid, file_format)) @@ -150,12 +160,21 @@ program f90tst_parallel ! Close the file. call handle_err(nf90mpi_close(ncid)) + 999 timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, & + MPI_DOUBLE_PRECISION, MPI_MAX, & + MPI_COMM_WORLD, ierr) + if (my_rank .eq. 0) then - msg = '*** TESTING F90 '//trim(cmd) - call pass_fail(0, msg) + if (.NOT. keep_files) then + err = nf90mpi_delete(out_path, MPI_INFO_NULL) + end if + + msg = '*** TESTING F90 '//trim(cmd) + call pass_fail(0, msg, timing) endif - 999 call MPI_Finalize(ierr) + call MPI_Finalize(ierr) contains diff --git a/test/F90/f90tst_parallel2.f90 b/test/F90/f90tst_parallel2.f90 index b7e4a0e37c..51de6c579a 100644 --- a/test/F90/f90tst_parallel2.f90 +++ b/test/F90/f90tst_parallel2.f90 @@ -60,28 +60,35 @@ program f90tst_parallel integer :: mode_flag, old_fillmode integer :: nvars, ngatts, ndims, unlimdimid, file_format integer :: x, y - integer :: p, my_rank, err, ierr, get_args + integer :: nprocs, my_rank, err, ierr, get_args integer(KIND=MPI_OFFSET_KIND) :: start(MAX_DIMS), count(MAX_DIMS), stride(MAX_DIMS) integer(KIND=MPI_OFFSET_KIND) :: nx_ll, ny_ll - character(LEN=256) filename, cmd, msg + character(LEN=256) out_path, in_path, cmd, msg + logical keep_files + double precision timing call MPI_Init(ierr) + + timing = MPI_Wtime() + call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr) - call MPI_Comm_size(MPI_COMM_WORLD, p, ierr) + call MPI_Comm_size(MPI_COMM_WORLD, nprocs, ierr) - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any if (my_rank .EQ. 0) then - filename = FILE_NAME - err = get_args(cmd, filename) + out_path = FILE_NAME + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr) -! if (p .ne. 4 .AND. my_rank .eq. 0) then -! print *, 'Warning: ',trim(cmd),' is design to run on 4 processes.' -! endif + ! if (nprocs .ne. 4 .AND. my_rank .eq. 0) then + ! print *, 'Warning: ',trim(cmd),' is design to run on 4 processes.' + ! endif ! Create some pretend data. do x = 1, NX / 4 @@ -92,7 +99,7 @@ program f90tst_parallel ! Create the netCDF file. mode_flag = IOR(NF90_CLOBBER, NF90_64BIT_DATA) - call handle_err(nf90mpi_create(MPI_COMM_WORLD, filename, mode_flag, MPI_INFO_NULL, ncid)) + call handle_err(nf90mpi_create(MPI_COMM_WORLD, out_path, mode_flag, MPI_INFO_NULL, ncid)) ! Define the dimensions. nx_ll = NX @@ -139,7 +146,7 @@ program f90tst_parallel call handle_err(nf90mpi_close(ncid)) ! Reopen the file. - call handle_err(nf90mpi_open(MPI_COMM_WORLD, filename, nf90_nowrite, MPI_INFO_NULL, ncid)) + call handle_err(nf90mpi_open(MPI_COMM_WORLD, out_path, nf90_nowrite, MPI_INFO_NULL, ncid)) ! Check some stuff out. call handle_err(nf90mpi_inquire(ncid, ndims, nvars, ngatts, unlimdimid, file_format)) @@ -165,12 +172,21 @@ program f90tst_parallel ! Close the file. call handle_err(nf90mpi_close(ncid)) + 999 timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, & + MPI_DOUBLE_PRECISION, MPI_MAX, & + MPI_COMM_WORLD, ierr) + if (my_rank .eq. 0) then - msg = '*** TESTING F90 '//trim(cmd)//' for strided access' - call pass_fail(0, msg) + if (.NOT. keep_files) then + err = nf90mpi_delete(out_path, MPI_INFO_NULL) + end if + + msg = '*** TESTING F90 '//trim(cmd)//' - strided access' + call pass_fail(0, msg, timing) endif - 999 call MPI_Finalize(ierr) + call MPI_Finalize(ierr) contains ! This subroutine handles errors by printing an error message and diff --git a/test/F90/f90tst_parallel3.f90 b/test/F90/f90tst_parallel3.f90 index 08bd615a0a..1bdba8884a 100644 --- a/test/F90/f90tst_parallel3.f90 +++ b/test/F90/f90tst_parallel3.f90 @@ -55,29 +55,36 @@ program f90tst_parallel3 integer (kind=EightByteInt) :: int64_out(HALF_NY, HALF_NX), int64_in(HALF_NY, HALF_NX) integer :: nvars, ngatts, ndims, unlimdimid, file_format integer :: x, y, v - integer :: p, my_rank, err, ierr, get_args, old_mode + integer :: nprocs, my_rank, err, ierr, get_args, old_mode integer(KIND=MPI_OFFSET_KIND) :: start(MAX_DIMS), count(MAX_DIMS) integer :: cmode integer(KIND=MPI_OFFSET_KIND) :: nx_ll, ny_ll - character(LEN=256) filename, cmd, msg + character(LEN=256) out_path, in_path, cmd, msg + logical keep_files + double precision timing call MPI_Init(ierr) + + timing = MPI_Wtime() + call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr) - call MPI_Comm_size(MPI_COMM_WORLD, p, ierr) + call MPI_Comm_size(MPI_COMM_WORLD, nprocs, ierr) - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any if (my_rank .EQ. 0) then - filename = FILE_NAME - err = get_args(cmd, filename) + out_path = FILE_NAME + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) -! if (p .ne. 4 .AND. my_rank .eq. 0) then -! print *, 'Warning: ',trim(cmd),' is design to run on 4 processes.' -! endif + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr) + + ! if (nprocs .ne. 4 .AND. my_rank .eq. 0) then + ! print *, 'Warning: ',trim(cmd),' is design to run on 4 processes.' + ! endif ! Create some pretend data. do x = 1, HALF_NX @@ -93,7 +100,7 @@ program f90tst_parallel3 ! Create the netCDF file. cmode = IOR(NF90_CLOBBER, NF90_64BIT_DATA) - call check(nf90mpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, ncid)) + call check(nf90mpi_create(MPI_COMM_WORLD, out_path, cmode, MPI_INFO_NULL, ncid)) ! Define the dimensions. nx_ll = NX @@ -147,7 +154,7 @@ program f90tst_parallel3 call check(nf90mpi_close(ncid)) ! Reopen the file. - call check(nf90mpi_open(MPI_COMM_WORLD, filename, nf90_nowrite, MPI_INFO_NULL, ncid)) + call check(nf90mpi_open(MPI_COMM_WORLD, out_path, nf90_nowrite, MPI_INFO_NULL, ncid)) ! Check some stuff out. call check(nf90mpi_inquire(ncid, ndims, nvars, ngatts, unlimdimid, file_format)) @@ -183,12 +190,21 @@ program f90tst_parallel3 ! Close the file. call check(nf90mpi_close(ncid)) - if (my_rank .eq. 0) then - msg = '*** TESTING F90 '//trim(cmd) - call pass_fail(0, msg) - endif + 999 timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, & + MPI_DOUBLE_PRECISION, MPI_MAX, & + MPI_COMM_WORLD, ierr) + + if (my_rank .eq. 0) then + if (.NOT. keep_files) then + err = nf90mpi_delete(out_path, MPI_INFO_NULL) + end if + + msg = '*** TESTING F90 '//trim(cmd) + call pass_fail(0, msg, timing) + endif - 999 call MPI_Finalize(ierr) + call MPI_Finalize(ierr) contains ! This subroutine handles errors by printing an error message and diff --git a/test/F90/f90tst_parallel4.f90 b/test/F90/f90tst_parallel4.f90 index b545fa8474..c618d5acbe 100644 --- a/test/F90/f90tst_parallel4.f90 +++ b/test/F90/f90tst_parallel4.f90 @@ -17,38 +17,47 @@ program f90tst integer :: dimid(3) integer(KIND=MPI_OFFSET_KIND) :: start(3), count(3) real :: f(3) - character(LEN=256) filename, cmd, msg + character(LEN=256) out_path, in_path, cmd, msg + logical keep_files + double precision timing + + call MPI_Init(ierr) + + timing = MPI_Wtime() - call MPI_INIT(ierr) call MPI_COMM_RANK(MPI_COMM_WORLD, my_rank, ierr) call MPI_COMM_SIZE(MPI_COMM_WORLD, nprocs, ierr) - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any if (my_rank .EQ. 0) then - filename = FILE_NAME - err = get_args(cmd, filename) + out_path = FILE_NAME + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr) -! if (nprocs .ne. 8 .AND. my_rank .eq. 0) then -! print *, 'Warning: ',trim(cmd),' is design to run on 8 processes.' -! endif + ! if (nprocs .ne. 8 .AND. my_rank .eq. 0) then + ! print *, 'Warning: ',trim(cmd),' is design to run on 8 processes.' + ! endif nmode = IOR(NF90_CLOBBER,NF90_64BIT_DATA) - call handle_err(nf90mpi_create(MPI_COMM_WORLD, filename, nmode, MPI_INFO_NULL, fh)) + call handle_err(nf90mpi_create(MPI_COMM_WORLD, out_path, nmode, MPI_INFO_NULL, fh)) call handle_err(nf90mpi_def_dim(fh, 'dim1', 6_MPI_OFFSET_KIND, dimid(1))) call handle_err(nf90mpi_def_dim(fh, 'dim2', 4_MPI_OFFSET_KIND, dimid(2))) call handle_err(nf90mpi_def_dim(fh, 'dim3', 1_MPI_OFFSET_KIND, dimid(3))) - call handle_err(nf90mpi_def_var(fh, 'var1', NF90_DOUBLE, dimid, varid)) - call handle_err(nf90mpi_enddef(fh)) + ! fill with default fill value + call handle_err(nf90mpi_def_var_fill(fh, varid, 0, NF90_FILL_DOUBLE)) + + call handle_err(nf90mpi_enddef(fh)) do i=1,3 f(i) = my_rank*3+i @@ -68,7 +77,7 @@ program f90tst call handle_err(nf90mpi_close(fh)) ! Reopen the file and check it. - call handle_err(nf90mpi_open(MPI_COMM_WORLD, filename, NF90_NOWRITE, MPI_INFO_NULL, fh)) + call handle_err(nf90mpi_open(MPI_COMM_WORLD, out_path, NF90_NOWRITE, MPI_INFO_NULL, fh)) call handle_err(nf90mpi_get_var_all(fh, varid, f, start=start, count=count)) @@ -83,12 +92,21 @@ program f90tst call handle_err(nf90mpi_close(fh)) + 999 timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, & + MPI_DOUBLE_PRECISION, MPI_MAX, & + MPI_COMM_WORLD, ierr) + if (my_rank .eq. 0) then + if (.NOT. keep_files) then + err = nf90mpi_delete(out_path, MPI_INFO_NULL) + end if + msg = '*** TESTING F90 '//trim(cmd) - call pass_fail(0, msg) + call pass_fail(0, msg, timing) endif - 999 call MPI_Finalize(ierr) + call MPI_Finalize(ierr) contains ! This subroutine handles errors by printing an error message and diff --git a/test/F90/f90tst_vars.f90 b/test/F90/f90tst_vars.f90 index 7f43a69089..78faf601d0 100644 --- a/test/F90/f90tst_vars.f90 +++ b/test/F90/f90tst_vars.f90 @@ -31,24 +31,34 @@ program f90tst_vars integer, parameter :: CACHE_SIZE = 1000000 integer :: info, err, ierr, get_args integer(KIND=MPI_OFFSET_KIND) :: nx_ll, ny_ll - character(LEN=256) filename, cmd, msg - integer my_rank, p + character(LEN=256) out_path, in_path, cmd, msg + integer my_rank, nprocs + logical keep_files + double precision timing call MPI_Init(ierr) + + timing = MPI_Wtime() + call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr) - call MPI_Comm_size(MPI_COMM_WORLD, p, ierr) + call MPI_Comm_size(MPI_COMM_WORLD, nprocs, ierr) - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any + cmd = ' ' if (my_rank .EQ. 0) then - filename = FILE_NAME - err = get_args(cmd, filename) + out_path = FILE_NAME + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr) + + if (my_rank > 0) goto 999; -! if (p .ne. 1 .AND. my_rank .eq. 0) then +! if (nprocs .ne. 1 .AND. my_rank .eq. 0) then ! print *, 'Warning: ',trim(cmd),' is design to run on 1 process' ! endif @@ -65,7 +75,7 @@ program f90tst_vars ! Create the netCDF file. mode_flag = IOR(NF90_CLOBBER, NF90_64BIT_DATA) - call handle_err(nf90mpi_create(MPI_COMM_WORLD, filename, mode_flag, info, ncid)) + call handle_err(nf90mpi_create(MPI_COMM_SELF, out_path, mode_flag, info, ncid)) call MPI_Info_free(info, ierr) ! Define the dimensions. @@ -78,6 +88,9 @@ program f90tst_vars ! Define the variable. call handle_err(nf90mpi_def_var(ncid, "data", NF90_INT, dimids, varid)) + ! fill with default fill value + call handle_err(nf90mpi_def_var_fill(ncid, varid, 0, NF90_FILL_INT)) + ! With classic model netCDF-4 file, enddef must be called. call handle_err(nf90mpi_enddef(ncid)) @@ -91,7 +104,7 @@ program f90tst_vars call handle_err(nf90mpi_close(ncid)) ! Reopen the file. - call handle_err(nf90mpi_open(MPI_COMM_WORLD, filename, nf90_nowrite, MPI_INFO_NULL, ncid)) + call handle_err(nf90mpi_open(MPI_COMM_SELF, out_path, nf90_nowrite, MPI_INFO_NULL, ncid)) ! Check some stuff out. call handle_err(nf90mpi_inquire(ncid, ndims, nvars, ngatts, unlimdimid, file_format)) @@ -116,10 +129,21 @@ program f90tst_vars ! Close the file. call handle_err(nf90mpi_close(ncid)) - msg = '*** TESTING F90 '//trim(cmd)//' for def_var API' - if (my_rank .eq. 0) call pass_fail(0, msg) + 999 timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, & + MPI_DOUBLE_PRECISION, MPI_MAX, & + MPI_COMM_WORLD, ierr) + + if (my_rank .eq. 0) then + if (.NOT. keep_files) then + err = nf90mpi_delete(out_path, MPI_INFO_NULL) + end if + + msg = '*** TESTING F90 '//trim(cmd)//' - def_var API' + call pass_fail(0, msg, timing) + end if - 999 call MPI_Finalize(ierr) + call MPI_Finalize(ierr) contains ! This subroutine handles errors by printing an error message and diff --git a/test/F90/f90tst_vars2.f90 b/test/F90/f90tst_vars2.f90 index d2011970ed..4ce95a111a 100644 --- a/test/F90/f90tst_vars2.f90 +++ b/test/F90/f90tst_vars2.f90 @@ -24,9 +24,8 @@ program f90tst_vars2 ! We need these ids and other gunk for netcdf. integer :: ncid, varid1, varid2, varid3, varid4, varid5, dimids(MAX_DIMS) - integer :: x_dimid, y_dimid + integer :: x, y, x_dimid, y_dimid, old_fillmode integer :: nvars, ngatts, ndims, unlimdimid, file_format - integer :: x, y integer, parameter :: DEFLATE_LEVEL = 4 integer, parameter :: EightByteInt = selected_int_kind(18) integer (kind = EightByteInt) :: TOE_SAN_VALUE = 2147483648_EightByteInt @@ -45,26 +44,36 @@ program f90tst_vars2 integer (kind = EightByteInt) :: toe_san_in(1) integer :: cmode, err, ierr, get_args integer(KIND=MPI_OFFSET_KIND) :: nx_ll, ny_ll - character(LEN=256) filename, cmd, msg - integer my_rank, p + character(LEN=256) out_path, in_path, cmd, msg + integer my_rank, nprocs + logical keep_files + double precision timing call MPI_Init(ierr) + + timing = MPI_Wtime() + call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr) - call MPI_Comm_size(MPI_COMM_WORLD, p, ierr) + call MPI_Comm_size(MPI_COMM_WORLD, nprocs, ierr) - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any + cmd = ' ' if (my_rank .EQ. 0) then - filename = FILE_NAME - err = get_args(cmd, filename) + out_path = FILE_NAME + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr) + + if (my_rank > 0) goto 999 -! if (p .ne. 1 .AND. my_rank .eq. 0) then -! print *, 'Warning: ',trim(cmd),' is design to run on 1 process' -! endif + ! if (nprocs .ne. 1 .AND. my_rank .eq. 0) then + ! print *, 'Warning: ',trim(cmd),' is design to run on 1 process' + ! endif ! Create some pretend data. do x = 1, NX @@ -78,7 +87,7 @@ program f90tst_vars2 ! Create the netCDF file. cmode = IOR(NF90_CLOBBER, NF90_64BIT_DATA) - call check(nf90mpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, ncid)) + call check(nf90mpi_create(MPI_COMM_SELF, out_path, cmode, MPI_INFO_NULL, ncid)) ! Define the dimensions. nx_ll = NX @@ -94,6 +103,8 @@ program f90tst_vars2 call check(nf90mpi_def_var(ncid, VAR4_NAME, NF90_INT, x_dimid, varid4)) call check(nf90mpi_def_var(ncid, VAR5_NAME, NF90_INT, dimids, varid5)) + call check(nf90mpi_set_fill(ncid, NF90_FILL, old_fillmode)) + call check(nf90mpi_enddef(ncid)) ! enter independent data mode @@ -110,7 +121,7 @@ program f90tst_vars2 call check(nf90mpi_close(ncid)) ! Reopen the file. - call check(nf90mpi_open(MPI_COMM_WORLD, filename, nf90_nowrite, MPI_INFO_NULL, ncid)) + call check(nf90mpi_open(MPI_COMM_SELF, out_path, nf90_nowrite, MPI_INFO_NULL, ncid)) ! Check some stuff out. call check(nf90mpi_inquire(ncid, ndims, nvars, ngatts, unlimdimid, file_format)) @@ -173,10 +184,21 @@ program f90tst_vars2 ! Close the file. call check(nf90mpi_close(ncid)) - msg = '*** TESTING F90 '//trim(cmd)//' for def_var API' - if (my_rank .eq. 0) call pass_fail(0, msg) + 999 timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, & + MPI_DOUBLE_PRECISION, MPI_MAX, & + MPI_COMM_WORLD, ierr) + + if (my_rank .eq. 0) then + if (.NOT. keep_files) then + err = nf90mpi_delete(out_path, MPI_INFO_NULL) + end if + + msg = '*** TESTING F90 '//trim(cmd)//' - def_var API' + call pass_fail(0, msg, timing) + end if - 999 call MPI_Finalize(ierr) + call MPI_Finalize(ierr) contains ! This subroutine handles errors by printing an error message and diff --git a/test/F90/f90tst_vars3.f90 b/test/F90/f90tst_vars3.f90 index 88fadc5692..9edec5ffa5 100644 --- a/test/F90/f90tst_vars3.f90 +++ b/test/F90/f90tst_vars3.f90 @@ -24,9 +24,8 @@ program f90tst_vars3 ! We need these ids and other gunk for netcdf. integer :: ncid, varid1, varid2, varid3, varid4, varid5, dimids(MAX_DIMS) - integer :: x_dimid, y_dimid + integer :: x, y, x_dimid, y_dimid, old_fillmode integer :: nvars, ngatts, ndims, unlimdimid, file_format - integer :: x, y integer, parameter :: DEFAULT_CACHE_NELEMS = 10000, DEFAULT_CACHE_SIZE = 1000000 integer, parameter :: DEFAULT_CACHE_PREEMPTION = 22 integer, parameter :: DEFLATE_LEVEL = 4 @@ -46,26 +45,37 @@ program f90tst_vars3 integer (kind = EightByteInt) :: toe_san_in(1) integer :: cmode, err, ierr, get_args integer(KIND=MPI_OFFSET_KIND) :: nx_ll, ny_ll - character(LEN=256) filename, cmd, msg - integer my_rank, p + character(LEN=256) out_path, in_path, cmd, msg + integer my_rank, nprocs + logical keep_files + double precision timing call MPI_Init(ierr) + + timing = MPI_Wtime() + call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr) - call MPI_Comm_size(MPI_COMM_WORLD, p, ierr) + call MPI_Comm_size(MPI_COMM_WORLD, nprocs, ierr) - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any + cmd = ' ' if (my_rank .EQ. 0) then - filename = FILE_NAME - err = get_args(cmd, filename) + out_path = FILE_NAME + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr) + + ! This program is design to run on 1 process' + if (my_rank > 0) goto 999 -! if (p .ne. 1 .AND. my_rank .eq. 0) then -! print *, 'Warning: ',trim(cmd),' is design to run on 1 process' -! endif + ! if (nprocs .ne. 1 .AND. my_rank .eq. 0) then + ! print *, 'Warning: ',trim(cmd),' is design to run on 1 process' + ! endif ! Create some pretend data. do x = 1, NX @@ -79,7 +89,7 @@ program f90tst_vars3 ! Create the netCDF file. cmode = IOR(NF90_CLOBBER, NF90_64BIT_DATA) - call check(nf90mpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, ncid)) + call check(nf90mpi_create(MPI_COMM_SELF, out_path, cmode, MPI_INFO_NULL, ncid)) ! Define the dimensions. nx_ll = NX @@ -95,6 +105,8 @@ program f90tst_vars3 call check(nf90mpi_def_var(ncid, VAR4_NAME, NF90_INT, x_dimid, varid4)) call check(nf90mpi_def_var(ncid, VAR5_NAME, NF90_INT, dimids, varid5)) + call check(nf90mpi_set_fill(ncid, NF90_FILL, old_fillmode)) + call check(nf90mpi_enddef(ncid)) call check(nf90mpi_begin_indep_data(ncid)) @@ -109,7 +121,7 @@ program f90tst_vars3 call check(nf90mpi_close(ncid)) ! Reopen the file. - call check(nf90mpi_open(MPI_COMM_WORLD, filename, nf90_nowrite, MPI_INFO_NULL, ncid)) + call check(nf90mpi_open(MPI_COMM_SELF, out_path, nf90_nowrite, MPI_INFO_NULL, ncid)) ! Check some stuff out. call check(nf90mpi_inquire(ncid, ndims, nvars, ngatts, unlimdimid, file_format)) @@ -172,10 +184,21 @@ program f90tst_vars3 ! Close the file. call check(nf90mpi_close(ncid)) - msg = '*** TESTING F90 '//trim(cmd)//' for def_var API' - if (my_rank .eq. 0) call pass_fail(0, msg) + 999 timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, & + MPI_DOUBLE_PRECISION, MPI_MAX, & + MPI_COMM_WORLD, ierr) + + if (my_rank .eq. 0) then + if (.NOT. keep_files) then + err = nf90mpi_delete(out_path, MPI_INFO_NULL) + end if + + msg = '*** TESTING F90 '//trim(cmd)//' - def_var API' + call pass_fail(0, msg, timing) + end if - 999 call MPI_Finalize(ierr) + call MPI_Finalize(ierr) contains ! This subroutine handles errors by printing an error message and diff --git a/test/F90/f90tst_vars4.f90 b/test/F90/f90tst_vars4.f90 index 1104246e38..0afbfb8ac9 100644 --- a/test/F90/f90tst_vars4.f90 +++ b/test/F90/f90tst_vars4.f90 @@ -22,33 +22,42 @@ program f90tst_vars4 ! We need these ids and other gunk for netcdf. integer :: ncid, varid, dimids(MAX_DIMS) - integer :: x_dimid, y_dimid + integer :: x, y, x_dimid, y_dimid, old_fillmode integer :: mode_flag integer :: nvars, ngatts, ndims, unlimdimid, file_format - integer :: x, y integer, parameter :: CACHE_SIZE = 1000000 integer :: xtype_in, natts_in, dimids_in(MAX_DIMS) character (len = NF90_MAX_NAME) :: name_in integer :: err, ierr, get_args integer(KIND=MPI_OFFSET_KIND) :: nx_ll, ny_ll - character(LEN=256) filename, cmd, msg - integer my_rank, p + character(LEN=256) out_path, in_path, cmd, msg + integer my_rank, nprocs + logical keep_files + double precision timing call MPI_Init(ierr) + + timing = MPI_Wtime() + call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr) - call MPI_Comm_size(MPI_COMM_WORLD, p, ierr) + call MPI_Comm_size(MPI_COMM_WORLD, nprocs, ierr) - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any + cmd = ' ' if (my_rank .EQ. 0) then - filename = FILE_NAME - err = get_args(cmd, filename) + out_path = FILE_NAME + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr) + + if (my_rank > 0) goto 999 -! if (p .ne. 1 .AND. my_rank .eq. 0) then +! if (nprocs .ne. 1 .AND. my_rank .eq. 0) then ! print *, 'Warning: ',trim(cmd),' is design to run on 1 process' ! endif @@ -61,7 +70,7 @@ program f90tst_vars4 ! Create the netCDF file. mode_flag = IOR(NF90_CLOBBER, NF90_64BIT_DATA) - call handle_err(nf90mpi_create(MPI_COMM_WORLD, filename, mode_flag, MPI_INFO_NULL, ncid)) + call handle_err(nf90mpi_create(MPI_COMM_SELF, out_path, mode_flag, MPI_INFO_NULL, ncid)) ! Define the dimensions. nx_ll = NX @@ -73,6 +82,8 @@ program f90tst_vars4 ! Define the variable. call handle_err(nf90mpi_def_var(ncid, 'data', NF90_INT, dimids, varid)) + call handle_err(nf90mpi_set_fill(ncid, NF90_FILL, old_fillmode)) + ! enddef must be called. call handle_err(nf90mpi_enddef(ncid)) @@ -85,7 +96,7 @@ program f90tst_vars4 call handle_err(nf90mpi_close(ncid)) ! Reopen the file. - call handle_err(nf90mpi_open(MPI_COMM_WORLD, filename, nf90_nowrite, MPI_INFO_NULL, ncid)) + call handle_err(nf90mpi_open(MPI_COMM_SELF, out_path, nf90_nowrite, MPI_INFO_NULL, ncid)) ! Check some stuff out. call handle_err(nf90mpi_inquire(ncid, ndims, nvars, ngatts, unlimdimid, file_format)) @@ -110,10 +121,21 @@ program f90tst_vars4 ! Close the file. call handle_err(nf90mpi_close(ncid)) - msg = '*** TESTING F90 '//trim(cmd)//' for def_var API' - if (my_rank .eq. 0) call pass_fail(0, msg) + 999 timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, & + MPI_DOUBLE_PRECISION, MPI_MAX, & + MPI_COMM_WORLD, ierr) + + if (my_rank .eq. 0) then + if (.NOT. keep_files) then + err = nf90mpi_delete(out_path, MPI_INFO_NULL) + end if + + msg = '*** TESTING F90 '//trim(cmd)//' - def_var API' + call pass_fail(0, msg, timing) + end if - 999 call MPI_Finalize(ierr) + call MPI_Finalize(ierr) contains ! This subroutine handles errors by printing an error message and diff --git a/test/F90/parallel_run.sh b/test/F90/parallel_run.sh index da29ea3bc1..553d5e2342 100755 --- a/test/F90/parallel_run.sh +++ b/test/F90/parallel_run.sh @@ -1,75 +1,46 @@ #!/bin/bash # -# Copyright (C) 2018, Northwestern University and Argonne National Laboratory +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory # See COPYRIGHT notice in top-level directory. # # Exit immediately if a command exits with a non-zero status. set -e -VALIDATOR=../../src/utils/ncvalidator/ncvalidator -NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff +DRY_RUN=no +VERBOSE=no -# remove file system type prefix if there is any -OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $MPIRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $MPIRUN $@ + fi +} MPIRUN=`echo ${TESTMPIRUN} | ${SED} -e "s/NP/$1/g"` # echo "MPIRUN = ${MPIRUN}" -# echo "PARALLEL_PROGS=${PARALLEL_PROGS}" - -# echo "PNETCDF_DEBUG = ${PNETCDF_DEBUG}" -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" -else - safe_modes="0" -fi +# echo "check_PROGRAMS=${check_PROGRAMS}" # prevent user environment setting of PNETCDF_HINTS to interfere unset PNETCDF_HINTS -for i in ${PARALLEL_PROGS} ; do - for j in ${safe_modes} ; do - for intra_aggr in 0 1 ; do - if test "$j" = 1 ; then # test only in safe mode - export PNETCDF_HINTS="romio_no_indep_rw=true" - else - export PNETCDF_HINTS= - fi - if test "$intra_aggr" = 1 ; then - export PNETCDF_HINTS="${PNETCDF_HINTS};nc_num_aggrs_per_node=2" - fi - export PNETCDF_SAFE_MODE=$j - # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - ${MPIRUN} ./$i ${TESTOUTDIR}/$i.nc +if test "x$MIMIC_LUSTRE" != x1 ; then + PNETCDF_HINTS="cb_nodes=2" +fi - # echo "--- validating file ${TESTOUTDIR}/$i.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.nc - # echo "" +for i in ${check_PROGRAMS} ; do - if test "x${ENABLE_BURST_BUFFER}" = x1 ; then - # echo "---- test burst buffering feature" - saved_PNETCDF_HINTS=${PNETCDF_HINTS} - export PNETCDF_HINTS="${PNETCDF_HINTS};nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" - ${MPIRUN} ./$i ${TESTOUTDIR}/$i.bb.nc - export PNETCDF_HINTS=${saved_PNETCDF_HINTS} + exe_name=`basename $i` - # echo "--- validating file ${TESTOUTDIR}/$i.bb.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.bb.nc + if test "x$exe_name" = xtst_io ; then + run_cmd ./$i -q -o ${TESTOUTDIR} + continue + fi - if test "$1" != "./tst_flarge" ; then - # echo "--- ncmpidiff $i.nc $i.bb.nc ---" - ${MPIRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/$i.nc ${TESTOUTDIR}/$i.bb.nc - fi - fi + run_cmd ./$i -q -o ${TESTOUTDIR}/${exe_name}.nc - if test "x${ENABLE_NETCDF4}" = x1 ; then - # echo "test netCDF-4 feature" - ${MPIRUN} ./$i ${TESTOUTDIR}/$i.nc4 4 - # Validator does not support nc4 - fi - done - done - rm -f ${OUTDIR}/$i.nc - rm -f ${OUTDIR}/$i.bb.nc -done +done # check_PROGRAMS diff --git a/test/F90/seq_runs.sh b/test/F90/seq_runs.sh index 9a816ad2d6..86a10d2e14 100755 --- a/test/F90/seq_runs.sh +++ b/test/F90/seq_runs.sh @@ -1,35 +1,33 @@ -#!/bin/sh +#!/bin/bash # -# Copyright (C) 2003, Northwestern University and Argonne National Laboratory +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory # See COPYRIGHT notice in top-level directory. # # Exit immediately if a command exits with a non-zero status. set -e -VALIDATOR=../../src/utils/ncvalidator/ncvalidator -NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff +DRY_RUN=no +VERBOSE=no -# prevent user environment setting of PNETCDF_HINTS to interfere -unset PNETCDF_HINTS +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $TESTSEQRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $TESTSEQRUN $@ + fi +} -${TESTSEQRUN} ./tst_io ${TESTOUTDIR} -${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/tst_io1.nc -# remove file system type prefix if there is any -OUTDIR=$(echo $TESTOUTDIR | cut -d: -f2) -mv ${OUTDIR}/tst_io1.nc ${OUTDIR}/tst_io1.nc0 +exe_name=`basename $1` -if test "x${ENABLE_BURST_BUFFER}" = x1 ; then - echo "" - echo "---- testing burst buffering" - export PNETCDF_HINTS="nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" - ${TESTSEQRUN} ./tst_io ${TESTOUTDIR} - unset PNETCDF_HINTS - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/tst_io1.nc +# prevent user environment setting of PNETCDF_HINTS to interfere +unset PNETCDF_HINTS - # echo "--- ncmpidiff tst_io1.nc0 tst_io1.nc ---" - ${TESTSEQRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/tst_io1.nc0 ${TESTOUTDIR}/tst_io1.nc +if test "x$exe_name" = xtst_io ; then + run_cmd ./$1 -q -o ${TESTOUTDIR} +else + run_cmd ./$1 -q -o ${TESTOUTDIR}/${exe_name}.nc fi -rm -f ${OUTDIR}/tst_io1.nc0 -rm -f ${OUTDIR}/tst_io1.nc diff --git a/test/F90/test_attr_int64.f90 b/test/F90/test_attr_int64.f90 index 19b19183c8..2dd3c659fa 100644 --- a/test/F90/test_attr_int64.f90 +++ b/test/F90/test_attr_int64.f90 @@ -18,7 +18,7 @@ subroutine check(err, message) if (err .NE. NF90_NOERR) then write(6,*) trim(message), trim(nf90mpi_strerror(err)) msg = '*** TESTING F90 test_attr_int64.f90 ' - call pass_fail(1, msg) + call pass_fail(1, msg, 0) STOP 2 end if end subroutine check @@ -28,27 +28,35 @@ program main use pnetcdf implicit none - character(LEN=256) filename, cmd, msg + character(LEN=256) out_path, in_path, cmd, msg integer rank, err, ierr, ncid, cmode, get_args, xtype, varid integer(kind=MPI_OFFSET_KIND) :: buf integer,parameter :: INT2_KIND = selected_int_kind(4) integer fillmode + logical keep_files + double precision timing + + call MPI_Init(ierr) + + timing = MPI_Wtime() - call MPI_Init(err) call MPI_Comm_rank(MPI_COMM_WORLD, rank, err) - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any + cmd = ' ' if (rank .EQ. 0) then - filename = 'testfile.nc' - err = get_args(cmd, filename) + out_path = 'testfile.nc' + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr) cmode = IOR(NF90_64BIT_DATA, NF90_CLOBBER) - err = nf90mpi_create(MPI_COMM_WORLD, filename, cmode, & + err = nf90mpi_create(MPI_COMM_WORLD, out_path, cmode, & MPI_INFO_NULL, ncid) call check(err, 'In nf90mpi_create: ') @@ -61,7 +69,7 @@ program main if (xtype .NE. NF90_INT64) then msg = '*** TESTING F90 test_attr_int64.f90 ' - call pass_fail(1, msg) + call pass_fail(1, msg, 0) STOP 2 endif @@ -85,7 +93,7 @@ program main if (err .NE. NF90_EBADTYPE) then 10 FORMAT(A,I3) write(msg,10) '*** test_attr_int64.f90 expects NF90_EBADTYPE but got ', err - call pass_fail(1, msg) + call pass_fail(1, msg, 0) STOP 2 endif @@ -95,9 +103,20 @@ program main err = nf90mpi_close(ncid) call check(err, 'In nf90mpi_close: ') - msg = '*** TESTING F90 '//trim(cmd)//' for scalar attr of INT64 ' - if (rank .eq. 0) call pass_fail(0, msg) + 999 timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, & + MPI_DOUBLE_PRECISION, MPI_MAX, & + MPI_COMM_WORLD, ierr) + + if (rank .eq. 0) then + if (.NOT. keep_files) then + err = nf90mpi_delete(out_path, MPI_INFO_NULL) + end if + + msg = '*** TESTING F90 '//trim(cmd)//' - scalar attr of INT64 ' + call pass_fail(0, msg, timing) + end if - 999 call MPI_Finalize(err) + call MPI_Finalize(err) end program main diff --git a/test/F90/test_fill.f90 b/test/F90/test_fill.f90 index a35f524670..fbcd6a6dfd 100644 --- a/test/F90/test_fill.f90 +++ b/test/F90/test_fill.f90 @@ -18,25 +18,28 @@ subroutine check(err, message) if (err .NE. NF90_NOERR) then write(6,*) trim(message), trim(nf90mpi_strerror(err)) msg = '*** TESTING F90 test_fill.f90 ' - call pass_fail(1, msg) + call pass_fail(1, msg, 0) STOP 2 end if end subroutine check - integer function tst_fmt(filename, mode) + integer function tst_fmt(out_path, mode) use mpi use pnetcdf implicit none - character(LEN=256) filename - integer i, err, ierr, rank + character(LEN=256) out_path + integer i, err, ierr, rank, nprocs integer :: ncid, mode, cmode, dimid(1), varid integer(kind=MPI_OFFSET_KIND) :: start(1) integer(kind=MPI_OFFSET_KIND) :: count(1) + integer(kind=MPI_OFFSET_KIND) :: dim_len integer(kind=MPI_OFFSET_KIND), parameter :: len = 3 integer, parameter :: k = selected_int_kind(18) integer(kind=k) :: buf(len) + logical keep_files + call MPI_Comm_size(MPI_COMM_WORLD, nprocs, ierr) call MPI_Comm_rank(MPI_COMM_WORLD, rank, ierr) tst_fmt = 0 @@ -47,11 +50,12 @@ integer function tst_fmt(filename, mode) ! create netcdf file cmode = IOR(mode, NF90_CLOBBER) - err = nf90mpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, ncid) + err = nf90mpi_create(MPI_COMM_WORLD, out_path, cmode, MPI_INFO_NULL, ncid) call check(err, 'In nf90mpi_create: ') tst_fmt = tst_fmt + err - err = nf90mpi_def_dim(ncid, "dim", len, dimid(1)) + dim_len = len * nprocs + err = nf90mpi_def_dim(ncid, "dim", dim_len, dimid(1)) call check(err, 'In nf90mpi_def_dim: ') tst_fmt = tst_fmt + err @@ -74,7 +78,7 @@ integer function tst_fmt(filename, mode) tst_fmt = tst_fmt + err ! Write buf - start(1) = 1 + start(1) = len * rank + 1 count(1) = len err = nf90mpi_put_var_all(ncid, varid, buf, start, count) call check(err, 'In nf90mpi_put_var_all: ') @@ -90,35 +94,54 @@ program test use mpi use pnetcdf implicit none - character(LEN=256) filename, cmd, msg + character(LEN=256) out_path, in_path, cmd, msg integer err, ierr, nerrs, rank, get_args, tst_fmt + logical keep_files + double precision timing + + call MPI_Init(ierr) + + timing = MPI_Wtime() - call MPI_Init(err) call MPI_Comm_rank(MPI_COMM_WORLD, rank, ierr) - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any + cmd = ' ' if (rank .EQ. 0) then - filename = 'testfile.nc' - err = get_args(cmd, filename) + out_path = 'testfile.nc' + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, err) + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, err) + + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr) nerrs = 0 if (PNETCDF_DRIVER_NETCDF4 .EQ. 1) then - err = tst_fmt(filename, NF90_NETCDF4) + err = tst_fmt(out_path, NF90_NETCDF4) nerrs = nerrs + err endif - err = tst_fmt(filename, NF90_64BIT_DATA) + err = tst_fmt(out_path, NF90_64BIT_DATA) nerrs = nerrs + err - msg = '*** TESTING F90 '//trim(cmd)//' for _FillValue ' - if (rank .eq. 0) call pass_fail(nerrs, msg) + 999 timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, & + MPI_DOUBLE_PRECISION, MPI_MAX, & + MPI_COMM_WORLD, ierr) + + if (rank .eq. 0) then + if (.NOT. keep_files) then + err = nf90mpi_delete(out_path, MPI_INFO_NULL) + end if + + msg = '*** TESTING F90 '//trim(cmd)//' - _FillValue ' + call pass_fail(0, msg, timing) + end if - 999 call MPI_Finalize(err) + call MPI_Finalize(err) end program test diff --git a/test/F90/test_intent.f90 b/test/F90/test_intent.f90 index 70612ca86b..6bdecf206e 100644 --- a/test/F90/test_intent.f90 +++ b/test/F90/test_intent.f90 @@ -21,7 +21,7 @@ subroutine check(err, message) if (err .NE. NF90_NOERR) then write(6,*) trim(message), trim(nf90mpi_strerror(err)) msg = '*** TESTING F90 test_intent.f90 ' - call pass_fail(1, msg) + call pass_fail(1, msg, 0) STOP 2 end if end subroutine check @@ -36,12 +36,13 @@ program main FourByteInt = selected_int_kind(9), & EightByteInt = selected_int_kind(18) - character(LEN=256) filename, cmd, msg + character(LEN=256) out_path, in_path, cmd, msg integer err, ierr, rank, get_args integer cmode, ncid, varid, dimid(1), req(1), status(1) integer(kind=MPI_OFFSET_KIND) start(1) integer(kind=MPI_OFFSET_KIND) count(1) integer(kind=MPI_OFFSET_KIND) bufsize + logical keep_files character(LEN=3) cbuf integer(KIND=OneByteInt) i1buf(3) @@ -58,23 +59,30 @@ program main PARAMETER( rbuf=(/1.0,2.0,3.0/)) PARAMETER( dbuf=(/1.0,2.0,3.0/)) PARAMETER(i8buf=(/1_EightByteInt,2_EightByteInt,3_EightByteInt/)) + double precision timing call MPI_Init(ierr) + + timing = MPI_Wtime() + call MPI_Comm_rank(MPI_COMM_WORLD, rank, ierr) - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any + cmd = ' ' if (rank .EQ. 0) then - filename = 'testfile.nc' - err = get_args(cmd, filename) + out_path = 'testfile.nc' + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr) ! create file, truncate it if exists cmode = IOR(NF90_CLOBBER, NF90_64BIT_DATA) - err = nf90mpi_create(MPI_COMM_WORLD, filename, cmode, & + err = nf90mpi_create(MPI_COMM_WORLD, out_path, cmode, & MPI_INFO_NULL, ncid) call check(err, 'In nf90mpi_create: ') @@ -109,42 +117,57 @@ program main call check(err, 'In nfmpi_put_att_int8: ') ! define a variable of an integer array of size 3 in the nc file - err = nfmpi_def_dim(ncid, 'X', 3_MPI_OFFSET_KIND, dimid(1)) - call check(err, 'In nfmpi_def_dim: ') + err = nf90mpi_def_dim(ncid, 'X', 3_MPI_OFFSET_KIND, dimid(1)) + call check(err, 'In nf90mpi_def_dim: ') + + err = nf90mpi_def_var(ncid, 'var', NF90_INT, dimid, varid) + call check(err, 'In nf90mpi_def_var: ') - err = nfmpi_def_var(ncid, 'var', NF90_INT, 1, dimid, varid) - call check(err, 'In nfmpi_def_var: ') + ! fill with default fill value + err = nf90mpi_def_var_fill(ncid, varid, 0, NF90_FILL_INT) + call check(err, 'In nf90mpi_def_var_fill: ') - err = nfmpi_enddef(ncid) - call check(err, 'In nfmpi_enddef: ') + err = nf90mpi_enddef(ncid) + call check(err, 'In nf90mpi_enddef: ') ! bufsize must be max of data type converted before and after bufsize = 3*4 - err = nfmpi_buffer_attach(ncid, bufsize) - call check(err, 'In nfmpi_buffer_attach: ') + err = nf90mpi_buffer_attach(ncid, bufsize) + call check(err, 'In nf90mpi_buffer_attach: ') start(1) = 1 count(1) = 3 - err = nfmpi_bput_vara_int(ncid, varid, start, count, ibuf, req(1)) + err = nfmpi_bput_vara_int(ncid, varid, start, count, ibuf(1:), req(1)) call check(err, 'In nfmpi_bput_vara_int: ') - err = nfmpi_wait_all(ncid, 1, req, status) - call check(err, 'In nfmpi_wait_all: ') + err = nf90mpi_wait_all(ncid, 1, req, status) + call check(err, 'In nf90mpi_wait_all: ') if (status(1) .ne. NF90_NOERR) then - print*,'Error at bput status ', nfmpi_strerror(status(1)) + print*,'Error at bput status ', nf90mpi_strerror(status(1)) endif - err = nfmpi_buffer_detach(ncid) - call check(err, 'In nfmpi_buffer_detach: ') + err = nf90mpi_buffer_detach(ncid) + call check(err, 'In nf90mpi_buffer_detach: ') ! close the file err = nf90mpi_close(ncid) call check(err, 'In nf90mpi_close: ') - msg = '*** TESTING F90 '//trim(cmd)//' for INTENT modifier ' - if (rank .eq. 0) call pass_fail(0, msg) + 999 timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, & + MPI_DOUBLE_PRECISION, MPI_MAX, & + MPI_COMM_WORLD, ierr) + + if (rank .eq. 0) then + if (.NOT. keep_files) then + err = nf90mpi_delete(out_path, MPI_INFO_NULL) + end if + + msg = '*** TESTING F90 '//trim(cmd)//' - INTENT modifier ' + call pass_fail(0, msg, timing) + end if - 999 call MPI_Finalize(ierr) + call MPI_Finalize(ierr) end program main diff --git a/test/F90/tst_f90.f90 b/test/F90/tst_f90.f90 index 3369556c9a..4b96cf5dd2 100644 --- a/test/F90/tst_f90.f90 +++ b/test/F90/tst_f90.f90 @@ -72,26 +72,36 @@ program netcdfTest character (len = 20) frTimeUnits real (kind = FourByteReal), dimension(numLats) :: latVarBuf real (kind = FourByteReal), dimension(numLons) :: lonVarBuf - character(LEN=256) filename, cmd, msg - integer my_rank, p, info + character(LEN=256) out_path, in_path, cmd, msg + integer my_rank, nprocs, info integer i, nformats, old_format integer formats(2) + logical keep_files + double precision timing call MPI_Init(ierr) + + timing = MPI_Wtime() + call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr) - call MPI_Comm_size(MPI_COMM_WORLD, p, ierr) + call MPI_Comm_size(MPI_COMM_WORLD, nprocs, ierr) - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any + cmd = ' ' if (my_rank .EQ. 0) then - filename = FILE_NAME - err = get_args(cmd, filename) + out_path = FILE_NAME + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr) + + if (my_rank > 0) goto 999; -! if (p .ne. 1 .AND. my_rank .eq. 0) then +! if (nprocs .ne. 1 .AND. my_rank .eq. 0) then ! print *, 'Warning: ',trim(cmd),' is design to run on 1 process' ! endif @@ -114,7 +124,7 @@ program netcdfTest ! call MPI_Info_set(info, "romio_pvfs2_posix_write", "enable",ierr) ! Create the file - call check(nf90mpi_create(MPI_COMM_WORLD, filename, nf90_clobber, info, ncFileID)) + call check(nf90mpi_create(MPI_COMM_SELF, out_path, nf90_clobber, info, ncFileID)) timeStringLen = LEN(timeString) @@ -183,15 +193,15 @@ program netcdfTest call check(nf90mpi_put_var_all(ncFileID, pressVarID, pressure(:, :, 2:2), & start = (/ 1_EightByteInt, 1_EightByteInt, 2_EightByteInt /)) ) - call check(nfmpi_begin_indep_data(ncFileID)) + call check(nf90mpi_begin_indep_data(ncFileID)) scalarVarBuf = 10 call check(nf90mpi_put_var(ncFileID, scalarVarID, scalarVarBuf)) - call check(nfmpi_end_indep_data(ncFileID)) + call check(nf90mpi_end_indep_data(ncFileID)) call check(nf90mpi_close(ncFileID)) ! Now open the file to read and check a few values - call check(nf90mpi_open(MPI_COMM_WORLD, filename, NF90_NOWRITE, info, ncFileID)) + call check(nf90mpi_open(MPI_COMM_SELF, out_path, NF90_NOWRITE, info, ncFileID)) call check(nf90mpi_inq_varid(ncFileID,"frtime",frTimeVarID)) call check(nf90mpi_get_att(ncFileID,frTimeVarID,"units",frTimeUnits)) if(frTimeUnits .ne. "hours") then @@ -202,10 +212,21 @@ program netcdfTest call MPI_Info_free(info, ierr) enddo + 999 timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, & + MPI_DOUBLE_PRECISION, MPI_MAX, & + MPI_COMM_WORLD, ierr) + + if (my_rank .eq. 0) then + if (.NOT. keep_files) then + err = nf90mpi_delete(out_path, MPI_INFO_NULL) + end if + msg = '*** TESTING F90 '//trim(cmd) - if (my_rank .eq. 0) call pass_fail(0, msg) + call pass_fail(0, msg, timing) + end if - 999 call MPI_Finalize(ierr) + call MPI_Finalize(ierr) contains ! Internal subroutine - checks error status after each netcdf, prints out text message each time diff --git a/test/F90/tst_f90_cdf5.f90 b/test/F90/tst_f90_cdf5.f90 index eae87baeef..ddd2c29703 100644 --- a/test/F90/tst_f90_cdf5.f90 +++ b/test/F90/tst_f90_cdf5.f90 @@ -14,56 +14,77 @@ program tst_f90_nc4 integer :: fh, cmode, err, ierr, dimid, varid, ndim, nvar, get_args character (len = *), parameter :: FILE_NAME = "tst_f90_nc4.nc" integer(KIND=MPI_OFFSET_KIND) :: ten=10 - character(LEN=256) filename, cmd, msg - integer my_rank, p, fillmode + character(LEN=256) out_path, in_path, cmd, msg + integer my_rank, nprocs, fillmode + logical keep_files + double precision timing call MPI_Init(ierr) + + timing = MPI_Wtime() + call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr) - call MPI_Comm_size(MPI_COMM_WORLD, p, ierr) + call MPI_Comm_size(MPI_COMM_WORLD, nprocs, ierr) - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any + cmd = ' ' if (my_rank .EQ. 0) then - filename = FILE_NAME - err = get_args(cmd, filename) + out_path = FILE_NAME + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr) -! if (p .ne. 1 .AND. my_rank .eq. 0) then + if (my_rank > 0) goto 999; + +! if (nprocs .ne. 1 .AND. my_rank .eq. 0) then ! print *, 'Warning: ',trim(cmd),' is design to run on 1 process' ! endif cmode = IOR(NF90_CLOBBER, NF90_64BIT_DATA) - call check(nf90mpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, fh)) + call check(nf90mpi_create(MPI_COMM_SELF, out_path, cmode, MPI_INFO_NULL, fh)) call check(nf90mpi_def_dim(fh, 'fred', ten, dimid)) call check(nf90mpi_def_var(fh, 'john', NF90_INT, (/dimid/), varid)) call check(nf90mpi_set_fill(fh, NF90_FILL, fillmode)) call check(nf90mpi_close(fh)) ! Check the file. - call check(nf90mpi_open(MPI_COMM_WORLD, filename, NF90_WRITE, MPI_INFO_NULL, fh)) + call check(nf90mpi_open(MPI_COMM_SELF, out_path, NF90_WRITE, MPI_INFO_NULL, fh)) call check(nf90mpi_inquire(fh, nDimensions = ndim, nVariables = nvar)) if (nvar .ne. 1 .or. ndim .ne. 1) stop 3 call check(nf90mpi_close(fh)) - call check(nf90mpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, fh)) + call check(nf90mpi_create(MPI_COMM_SELF, out_path, cmode, MPI_INFO_NULL, fh)) call check(nf90mpi_def_dim(fh, 'fred', ten, dimid)) call check(nf90mpi_def_var(fh, 'john', NF90_INT, (/dimid/), varid)) call check(nf90mpi_set_fill(fh, NF90_FILL, fillmode)) call check(nf90mpi_close(fh)) ! Check the file. - call check(nf90mpi_open(MPI_COMM_WORLD, filename, NF90_WRITE, MPI_INFO_NULL, fh)) + call check(nf90mpi_open(MPI_COMM_SELF, out_path, NF90_WRITE, MPI_INFO_NULL, fh)) call check(nf90mpi_inquire(fh, nDimensions = ndim, nVariables = nvar)) if (nvar .ne. 1 .or. ndim .ne. 1) stop 3 call check(nf90mpi_close(fh)) - msg = '*** TESTING F90 '//trim(cmd) - if (my_rank .eq. 0) call pass_fail(0, msg) + 999 timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, & + MPI_DOUBLE_PRECISION, MPI_MAX, & + MPI_COMM_WORLD, ierr) + + if (my_rank .eq. 0) then + if (.NOT. keep_files) then + err = nf90mpi_delete(out_path, MPI_INFO_NULL) + end if + + msg = '*** TESTING F90 '//trim(cmd) + call pass_fail(0, msg, timing) + end if - 999 call MPI_Finalize(ierr) + call MPI_Finalize(ierr) contains ! This subroutine handles errors by printing an error message and diff --git a/test/F90/tst_io.f90 b/test/F90/tst_io.f90 index b0555374da..0d7c1042c5 100644 --- a/test/F90/tst_io.f90 +++ b/test/F90/tst_io.f90 @@ -30,26 +30,35 @@ program tst_io ! needed for netcdf integer :: ncid, x1id, x2id, x3id, x4id, vrid ! integer :: vrids, vridt, vridu, vridv, vridw, vridx, vridy, vridz - character(LEN=256) dirpath, cmd, msg - integer my_rank, p + character(LEN=256) out_path, in_path, cmd, msg + integer my_rank, nprocs integer i, nformats, old_format integer formats(2) + logical keep_files + double precision timing call MPI_Init(ierr) + + timing = MPI_Wtime() + + call MPI_Comm_size(MPI_COMM_WORLD, nprocs, ierr) call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr) - call MPI_Comm_size(MPI_COMM_WORLD, p, ierr) - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any if (my_rank .EQ. 0) then - dirpath = '.' - err = get_args(cmd, dirpath) + out_path = '.' + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(dirpath, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr) + + if (my_rank > 0) goto 999; -! if (p .ne. 1 .AND. my_rank .eq. 0) then +! if (nprocs .ne. 1 .AND. my_rank .eq. 0) then ! print *, 'Warning: ',trim(cmd),' is design to run on 1 process' ! endif @@ -80,13 +89,13 @@ program tst_io do i = nformats, 2 call check(nf90mpi_set_default_format(formats(i), old_format), 10) - ! call setupNetCDF (trim(dirpath)//'/'//nclFilenm1, ncid, vrid, x, prsz1, prsz2, prsz3, prsz4, & - call setupNetCDF (trim(dirpath)//'/'//nclFilenm1, ncid, vrid, prsz1, prsz2, prsz3, prsz4, & + ! call setupNetCDF (trim(out_path)//'/'//nclFilenm1, ncid, vrid, x, prsz1, prsz2, prsz3, prsz4, & + call setupNetCDF (trim(out_path)//'/'//nclFilenm1, ncid, vrid, prsz1, prsz2, prsz3, prsz4, & x1id, x2id, x3id, x4id, NF90_CLOBBER, 20) call system_clock(start) - call check(nfmpi_begin_indep_data(ncid), 11) + call check(nf90mpi_begin_indep_data(ncid), 11) call check (NF90MPI_PUT_VAR(ncid, vrid, x), 18) - call check(nfmpi_end_indep_data(ncid), 12) + call check(nf90mpi_end_indep_data(ncid), 12) call system_clock(now) ncint1 = now - start ! print 3, size, "MB"," netcdf write = ", ncint1 * clockRate, & @@ -97,12 +106,12 @@ program tst_io call system_clock(start) do i1 = 1, repct - ! call setupNetCDF (trim(dirpath)//'/'//nclFilenm1, ncid, vrid, x, prsz1, prsz2, prsz3, prsz4, & - call setupNetCDF (trim(dirpath)//'/'//nclFilenm1, ncid, vrid, prsz1, prsz2, prsz3, prsz4, & + ! call setupNetCDF (trim(out_path)//'/'//nclFilenm1, ncid, vrid, x, prsz1, prsz2, prsz3, prsz4, & + call setupNetCDF (trim(out_path)//'/'//nclFilenm1, ncid, vrid, prsz1, prsz2, prsz3, prsz4, & x1id, x2id, x3id, x4id, NF90_CLOBBER, 130) - call check(nfmpi_begin_indep_data(ncid), 11) + call check(nf90mpi_begin_indep_data(ncid), 11) call check (NF90MPI_PUT_VAR(ncid, vrid, x), 23 + i1) - call check(nfmpi_end_indep_data(ncid), 11) + call check(nf90mpi_end_indep_data(ncid), 11) call check (NF90MPI_CLOSE(ncid), 15) enddo call system_clock(now) @@ -112,23 +121,23 @@ program tst_io ! 4 format("Time for", i5, "MB", i3, a22, i7, " msec. Spd ratio = ", f5.2) ! call system_clock(start) - ! call setupNetCDF (trim(dirpath)//'/'//nclFilenm3, ncid, vrids, s, prsz1, prsz2, prsz3, prsz4, & + ! call setupNetCDF (trim(out_path)//'/'//nclFilenm3, ncid, vrids, s, prsz1, prsz2, prsz3, prsz4, & ! x1id, x2id, x3id, x4id, NF90_CLOBBER, 20) - ! call setupNetCDF (trim(dirpath)//'/'//nclFilenm4, ncid, vridt, t, prsz1, prsz2, prsz3, prsz4, & + ! call setupNetCDF (trim(out_path)//'/'//nclFilenm4, ncid, vridt, t, prsz1, prsz2, prsz3, prsz4, & ! x1id, x2id, x3id, x4id, NF90_CLOBBER, 30) - ! call setupNetCDF (trim(dirpath)//'/'//nclFilenm5, ncid, vridu, u, prsz1, prsz2, prsz3, prsz4, & + ! call setupNetCDF (trim(out_path)//'/'//nclFilenm5, ncid, vridu, u, prsz1, prsz2, prsz3, prsz4, & ! x1id, x2id, x3id, x4id, NF90_CLOBBER, 40) - ! call setupNetCDF (trim(dirpath)//'/'//nclFilenm6, ncid, vridv, v, prsz1, prsz2, prsz3, prsz4, & + ! call setupNetCDF (trim(out_path)//'/'//nclFilenm6, ncid, vridv, v, prsz1, prsz2, prsz3, prsz4, & ! x1id, x2id, x3id, x4id, NF90_CLOBBER, 50) - ! call setupNetCDF (trim(dirpath)//'/'//nclFilenm7, ncid, vridw, w, prsz1, prsz2, prsz3, prsz4, & + ! call setupNetCDF (trim(out_path)//'/'//nclFilenm7, ncid, vridw, w, prsz1, prsz2, prsz3, prsz4, & ! x1id, x2id, x3id, x4id, NF90_CLOBBER, 60) - ! call setupNetCDF (trim(dirpath)//'/'//nclFilenm8, ncid, vridx, x, prsz1, prsz2, prsz3, prsz4, & + ! call setupNetCDF (trim(out_path)//'/'//nclFilenm8, ncid, vridx, x, prsz1, prsz2, prsz3, prsz4, & ! x1id, x2id, x3id, x4id, NF90_CLOBBER, 70) - ! call setupNetCDF (trim(dirpath)//'/'//nclFilenm9, ncid, vridy, y, prsz1, prsz2, prsz3, prsz4, & + ! call setupNetCDF (trim(out_path)//'/'//nclFilenm9, ncid, vridy, y, prsz1, prsz2, prsz3, prsz4, & ! x1id, x2id, x3id, x4id, NF90_CLOBBER, 80) - ! call setupNetCDF (trim(dirpath)//'/'//nclFilenm10, ncid, vridz, z, prsz1, prsz2, prsz3, prsz4, & + ! call setupNetCDF (trim(out_path)//'/'//nclFilenm10, ncid, vridz, z, prsz1, prsz2, prsz3, prsz4, & ! x1id, x2id, x3id, x4id, NF90_CLOBBER, 90) - ! call check(nfmpi_begin_indep_data(ncid), 11) + ! call check(nf90mpi_begin_indep_data(ncid), 11) ! call check (NF90MPI_PUT_VAR(ncid, vrids, s), 118) ! call check (NF90MPI_PUT_VAR(ncid, vridt, t), 119) ! call check (NF90MPI_PUT_VAR(ncid, vridu, u), 120) @@ -137,17 +146,29 @@ program tst_io ! call check (NF90MPI_PUT_VAR(ncid, vridx, x), 123) ! call check (NF90MPI_PUT_VAR(ncid, vridy, y), 124) ! call check (NF90MPI_PUT_VAR(ncid, vridz, z), 125) - ! call check(nfmpi_end_indep_data(ncid), 11) + ! call check(nf90mpi_end_indep_data(ncid), 11) ! call system_clock(now) ! ncint3 = now - start ! call check (NF90MPI_CLOSE(ncid), 16) ! print 4, size, 8, " netcdf file writes = ", ncint3 * clockRate, & ! real(ncint3)/real(wrint3); enddo - msg = '*** TESTING F90 '//trim(cmd) - if (my_rank .eq. 0) call pass_fail(0, msg) - 999 call MPI_Finalize(ierr) + 999 timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, & + MPI_DOUBLE_PRECISION, MPI_MAX, & + MPI_COMM_WORLD, ierr) + + if (my_rank .eq. 0) then + if (.NOT. keep_files) then + err = nf90mpi_delete(trim(out_path)//'/'//nclFilenm1, MPI_INFO_NULL) + end if + + msg = '*** TESTING F90 '//trim(cmd) + call pass_fail(0, msg, timing) + end if + + call MPI_Finalize(ierr) contains subroutine check (st, n) ! checks the return error code @@ -170,7 +191,7 @@ subroutine setupNetCDF(fn, nc, vr, d1, d2, d3, d4, do1, do2, & integer, intent(inout) :: nc integer, dimension(4) :: dimids (4) - call check (NF90MPI_CREATE (MPI_COMM_WORLD, fn, stat, MPI_INFO_NULL, nc), deb + 1) + call check (NF90MPI_CREATE (MPI_COMM_SELF, fn, stat, MPI_INFO_NULL, nc), deb + 1) call check (NF90MPI_DEF_DIM(nc, "d1", d1, do1), deb + 2) call check (NF90MPI_DEF_DIM(nc, "d2", d2, do2), deb + 3) call check (NF90MPI_DEF_DIM(nc, "d3", d3, do3), deb + 4) diff --git a/test/F90/tst_types2.f90 b/test/F90/tst_types2.f90 index f1506eb76e..3e40549d2e 100644 --- a/test/F90/tst_types2.f90 +++ b/test/F90/tst_types2.f90 @@ -34,26 +34,31 @@ program tst_types2 integer :: cmode, err, ierr, get_args integer(KIND=MPI_OFFSET_KIND) :: dlen_ll - character(LEN=256) filename, cmd, msg - integer my_rank, p - logical verbose + character(LEN=256) out_path, in_path, cmd, msg + integer my_rank, nprocs + logical verbose, keep_files + double precision timing call MPI_Init(ierr) + + timing = MPI_Wtime() + call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr) - call MPI_Comm_size(MPI_COMM_WORLD, p, ierr) + call MPI_Comm_size(MPI_COMM_WORLD, nprocs, ierr) - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any + cmd = ' ' if (my_rank .EQ. 0) then - filename = FILE_NAME - err = get_args(cmd, filename) + out_path = FILE_NAME + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) verbose = .FALSE. - if (p .ne. 4 .AND. my_rank .eq. 0 .AND. verbose) then + if (nprocs .ne. 4 .AND. my_rank .eq. 0 .AND. verbose) then print *, 'Warning: ',trim(cmd),' is design to run on 4 processes.' endif @@ -123,7 +128,7 @@ program tst_types2 ! Create the netCDF file. cmode = IOR(NF90_CLOBBER, NF90_64BIT_DATA) - call check(nf90mpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, ncid)) + call check(nf90mpi_create(MPI_COMM_WORLD, out_path, cmode, MPI_INFO_NULL, ncid)) ! Define dimensions. dlen_ll = DLEN @@ -189,7 +194,7 @@ program tst_types2 call check(nf90mpi_close(ncid)) ! Reopen the netCDF file. - call check(nf90mpi_open(MPI_COMM_WORLD, filename, nf90_nowrite, MPI_INFO_NULL, ncid)) + call check(nf90mpi_open(MPI_COMM_WORLD, out_path, nf90_nowrite, MPI_INFO_NULL, ncid)) ! Read in the large numbers. call check(nf90mpi_get_var_all(ncid, varid1, data1_in)) @@ -272,10 +277,21 @@ program tst_types2 ! Close the file. call check(nf90mpi_close(ncid)) - msg = '*** TESTING F90 '//trim(cmd)//' for 64-bit integer types' - if (my_rank .eq. 0) call pass_fail(0, msg) + 999 timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, & + MPI_DOUBLE_PRECISION, MPI_MAX, & + MPI_COMM_WORLD, ierr) + + if (my_rank .eq. 0) then + if (.NOT. keep_files) then + err = nf90mpi_delete(out_path, MPI_INFO_NULL) + end if + + msg = '*** TESTING F90 '//trim(cmd)//' - 64-bit integer types' + call pass_fail(0, msg, timing) + end if - 999 call MPI_Finalize(ierr) + call MPI_Finalize(ierr) ! This subroutine handles errors by printing an error message and ! exiting with a non-zero status. diff --git a/test/F90/wrap_runs.sh b/test/F90/wrap_runs.sh deleted file mode 100755 index 716aacf063..0000000000 --- a/test/F90/wrap_runs.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2003, Northwestern University and Argonne National Laboratory -# See COPYRIGHT notice in top-level directory. -# - -# Exit immediately if a command exits with a non-zero status. -set -e - -VALIDATOR=../../src/utils/ncvalidator/ncvalidator -NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff - -outfile=`basename $1` - -# remove file system type prefix if there is any -OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` - -# echo "PNETCDF_DEBUG = ${PNETCDF_DEBUG}" -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" -else - safe_modes="0" -fi - -# prevent user environment setting of PNETCDF_HINTS to interfere -unset PNETCDF_HINTS - -for j in ${safe_modes} ; do - export PNETCDF_SAFE_MODE=$j - # echo "---- set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - ${TESTSEQRUN} $1 ${TESTOUTDIR}/$outfile.nc - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$outfile.nc - # echo "" - - if test "x${ENABLE_BURST_BUFFER}" = x1 ; then - echo "" - echo "---- testing burst buffering" - - export PNETCDF_HINTS="nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" - ${TESTSEQRUN} $1 ${TESTOUTDIR}/$outfile.bb.nc - unset PNETCDF_HINTS - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$outfile.bb.nc - - # running ncmpidiff on dim_cdf12 on one process requires more than 2 GB - # memory. Use more processes to check. Disable the check for now. - if test "$1" != "./tst_flarge" ; then - # echo "--- ncmpidiff $outfile.nc $outfile.bb.nc ---" - ${TESTSEQRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/$outfile.nc ${TESTOUTDIR}/$outfile.bb.nc - fi - fi -done -rm -f ${OUTDIR}/$outfile.nc -rm -f ${OUTDIR}/$outfile.bb.nc - diff --git a/test/Makefile.am b/test/Makefile.am index f5eb9d5b4a..7d94074039 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -64,6 +64,8 @@ else PTEST_SUBDIRS = $(SUBDIRS) endif +EXTRA_DIST = parallel_run.sh + ptest: @for d in $(PTEST_SUBDIRS) ; do \ $(MAKE) $(MFLAGS) -C $$d ptest $$* || exit 1 ; \ diff --git a/test/adios/Makefile.am b/test/adios/Makefile.am index 7a670fa3e7..acfea84bcb 100644 --- a/test/adios/Makefile.am +++ b/test/adios/Makefile.am @@ -12,7 +12,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/src/include AM_CPPFLAGS += -I$(srcdir)/../common AM_CPPFLAGS += -I$(top_builddir)/src/include -LDADD = $(top_builddir)/src/libs/libpnetcdf.la ../common/libtestutils.la +LDADD = ../common/libtestutils.la $(top_builddir)/src/libs/libpnetcdf.la LDADD += @NETCDF4_LDFLAGS@ @ADIOS_LDFLAGS@ @NETCDF4_LIBS@ @ADIOS_LIBS@ -lm if DECL_MPI_OFFSET @@ -43,7 +43,7 @@ $(M4_SRCS:.m4=.c): Makefile .m4.c: $(M4) $(AM_M4FLAGS) $(M4FLAGS) $< >$@ -# all programs in TESTPROGRAMS will be run by wrap_runs.sh +# all programs in TESTPROGRAMS will be run by seq_runs.sh # others in check_PROGRAMS but not in TESTPROGRAMS will be run by seq_runs.sh # Those are the ones need special treatment check_PROGRAMS = $(TESTPROGRAMS) @@ -55,26 +55,29 @@ check_PROGRAMS = $(TESTPROGRAMS) # AM_TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; TESTS_ENVIRONMENT = export SED="$(SED)"; TESTS_ENVIRONMENT += export srcdir="$(srcdir)"; +TESTS_ENVIRONMENT += export top_builddir="$(top_builddir)"; TESTS_ENVIRONMENT += export TESTOUTDIR="$(TESTOUTDIR)"; TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; TESTS_ENVIRONMENT += export TESTMPIRUN="$(TESTMPIRUN)"; -TESTS_ENVIRONMENT += export PNETCDF_DEBUG="$(PNETCDF_DEBUG)"; TESTS_ENVIRONMENT += export TESTPROGRAMS="$(TESTPROGRAMS)"; TESTS_ENVIRONMENT += export check_PROGRAMS="$(check_PROGRAMS)"; TESTS_ENVIRONMENT += export ADIOS_VER_GE_1132="$(ADIOS_VER_GE_1132)"; +TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER=@ENABLE_BURST_BUFFER@; +TESTS_ENVIRONMENT += export ENABLE_THREAD_SAFE=@ENABLE_THREAD_SAFE@; +TESTS_ENVIRONMENT += export MIMIC_LUSTRE=@MIMIC_LUSTRE@; +TESTS_ENVIRONMENT += export ENABLE_GIO=@ENABLE_GIO@; + TESTS = $(TESTPROGRAMS) TEST_EXTENSIONS = .sh -LOG_COMPILER = $(srcdir)/wrap_runs.sh +LOG_COMPILER = $(srcdir)/seq_runs.sh SH_LOG_COMPILER = -NC_FILES = $(TESTPROGRAMS:%=$(TESTOUTDIR)/%.nc) +check_SCRIPTS = seq_runs.sh parallel_run.sh -CLEANFILES = $(M4_SRCS:.m4=.c) \ - $(TESTOUTDIR)/put_get_all_kinds.nc.cdf4 \ - $(NC_FILES) testfile.nc +CLEANFILES = $(M4_SRCS:.m4=.c) -EXTRA_DIST = $(M4_SRCS) wrap_runs.sh parallel_run.sh \ +EXTRA_DIST = $(M4_SRCS) seq_runs.sh parallel_run.sh \ arrays_big.bp arrays.bp attributes_big.bp attributes.bp ../common/libtestutils.la: diff --git a/test/adios/att.c b/test/adios/att.c index 102edd6c10..8946493de5 100644 --- a/test/adios/att.c +++ b/test/adios/att.c @@ -23,7 +23,7 @@ /* Handle errors by printing an error message and exiting with a * non-zero status. */ #define ERRCODE 2 -#define ERR(e) {printf("Error: %s\n", nc_strerror(e)); exit(ERRCODE);} +#define ERR(e) {fprintf(stderr,"Error: %s\n", nc_strerror(e)); exit(ERRCODE);} int main(int argc, char** argv) { int nerrs=0, rank, nprocs, err; @@ -130,7 +130,7 @@ int main(int argc, char** argv) { MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf("pass\n"); } MPI_Finalize(); diff --git a/test/adios/header.c b/test/adios/header.c index 52a3892456..b7f7c9008d 100644 --- a/test/adios/header.c +++ b/test/adios/header.c @@ -130,7 +130,7 @@ int main(int argc, char** argv) { MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf("pass\n"); } MPI_Finalize(); diff --git a/test/adios/indep.c b/test/adios/indep.c index c352b77f90..61ac55c1ff 100644 --- a/test/adios/indep.c +++ b/test/adios/indep.c @@ -32,7 +32,7 @@ /* Handle errors by printing an error message and exiting with a * non-zero status. */ #define ERRCODE 2 -#define ERR(e) {printf("Error: %s\n", nc_strerror(e)); exit(ERRCODE);} +#define ERR(e) {fprintf(stderr,"Error: %s\n", nc_strerror(e)); exit(ERRCODE);} double data[NX][NY], datat[NY][NX]; @@ -99,7 +99,7 @@ int main(int argc, char** argv) { MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf("pass\n"); } MPI_Finalize(); diff --git a/test/adios/ivar.c b/test/adios/ivar.c index 07e75db54a..1985559c01 100644 --- a/test/adios/ivar.c +++ b/test/adios/ivar.c @@ -32,7 +32,7 @@ /* Handle errors by printing an error message and exiting with a * non-zero status. */ #define ERRCODE 2 -#define ERR(e) {printf("Error: %s\n", nc_strerror(e)); exit(ERRCODE);} +#define ERR(e) {fprintf(stderr,"Error: %s\n", nc_strerror(e)); exit(ERRCODE);} int main(int argc, char** argv) { int i, nerrs=0, rank, nprocs, err; @@ -211,7 +211,7 @@ int main(int argc, char** argv) { MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf("pass\n"); } MPI_Finalize(); diff --git a/test/adios/ivarm.c b/test/adios/ivarm.c index c7ba84a516..04a806ceee 100644 --- a/test/adios/ivarm.c +++ b/test/adios/ivarm.c @@ -32,7 +32,7 @@ /* Handle errors by printing an error message and exiting with a * non-zero status. */ #define ERRCODE 2 -#define ERR(e) {printf("Error: %s\n", nc_strerror(e)); exit(ERRCODE);} +#define ERR(e) {fprintf(stderr,"Error: %s\n", nc_strerror(e)); exit(ERRCODE);} double data[NX][NY], datat[NY][NX]; @@ -95,7 +95,7 @@ int main(int argc, char** argv) { MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf("pass\n"); } MPI_Finalize(); diff --git a/test/adios/ivars.c b/test/adios/ivars.c index ba7a783bc6..15c073650e 100644 --- a/test/adios/ivars.c +++ b/test/adios/ivars.c @@ -32,7 +32,7 @@ /* Handle errors by printing an error message and exiting with a * non-zero status. */ #define ERRCODE 2 -#define ERR(e) {printf("Error: %s\n", nc_strerror(e)); exit(ERRCODE);} +#define ERR(e) {fprintf(stderr,"Error: %s\n", nc_strerror(e)); exit(ERRCODE);} int main(int argc, char** argv) { char filename[256]; @@ -95,7 +95,7 @@ int main(int argc, char** argv) { MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf("pass\n"); } MPI_Finalize(); diff --git a/test/adios/open.c b/test/adios/open.c index a1c7117869..e1bfb31f1c 100644 --- a/test/adios/open.c +++ b/test/adios/open.c @@ -27,7 +27,7 @@ /* Handle errors by printing an error message and exiting with a * non-zero status. */ #define ERRCODE 2 -#define ERR(e) {printf("Error: %s\n", nc_strerror(e)); exit(ERRCODE);} +#define ERR(e) {fprintf(stderr,"Error: %s\n", nc_strerror(e)); exit(ERRCODE);} int main(int argc, char** argv) { char filename[256]; @@ -64,7 +64,7 @@ int main(int argc, char** argv) { MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf("pass\n"); } MPI_Finalize(); diff --git a/test/adios/parallel_run.sh b/test/adios/parallel_run.sh index 612fd7591a..db99ed23eb 100755 --- a/test/adios/parallel_run.sh +++ b/test/adios/parallel_run.sh @@ -14,29 +14,43 @@ MPIRUN=`echo ${TESTMPIRUN} | ${SED} -e "s/NP/$1/g"` # echo "MPIRUN = ${MPIRUN}" # echo "check_PROGRAMS=${check_PROGRAMS}" -# echo "PNETCDF_DEBUG = ${PNETCDF_DEBUG}" -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" +if test "x$ENABLE_GIO" = x0 ; then + IO_MODES="mpiio" else - safe_modes="0" + IO_MODES="gio mpiio" fi # prevent user environment setting of PNETCDF_HINTS to interfere unset PNETCDF_HINTS for i in ${check_PROGRAMS} ; do - for j in ${safe_modes} ; do - for intra_aggr in 0 1 ; do - if test "$j" = 1 ; then # test only in safe mode - export PNETCDF_HINTS="romio_no_indep_rw=true" + for io_mode in $IO_MODES ; do + if test "x$io_mode" = xmpiio ; then + USEMPIO_HINTS="nc_driver=mpiio" else - export PNETCDF_HINTS= + USEMPIO_HINTS="nc_driver=gio" fi + for intra_aggr in 0 1 ; do if test "$intra_aggr" = 1 ; then - export PNETCDF_HINTS="${PNETCDF_HINTS};nc_num_aggrs_per_node=2" + INA_HINTS="nc_num_aggrs_per_node=2" + else + INA_HINTS="nc_num_aggrs_per_node=0" + fi + + PNETCDF_HINTS= + if test "x$USEMPIO_HINTS" != x ; then + PNETCDF_HINTS="$USEMPIO_HINTS;$PNETCDF_HINTS" fi - export PNETCDF_SAFE_MODE=$j - # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" + if test "x$INA_HINTS" != x ; then + PNETCDF_HINTS="$INA_HINTS;$PNETCDF_HINTS" + fi + if test "x$MIMIC_LUSTRE" != x1 ; then + PNETCDF_HINTS="cb_nodes=2;$PNETCDF_HINTS" + fi + + export PNETCDF_HINTS="$PNETCDF_HINTS" + # echo "PNETCDF_HINTS=$PNETCDF_HINTS" + if test "$i" = open ; then ${MPIRUN} ./$i ${srcdir}/arrays.bp ${MPIRUN} ./$i ${srcdir}/attributes.bp diff --git a/test/adios/seq_runs.sh b/test/adios/seq_runs.sh new file mode 100755 index 0000000000..595f760f19 --- /dev/null +++ b/test/adios/seq_runs.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# +# Copyright (C) 2019, Northwestern University and Argonne National Laboratory +# See COPYRIGHT notice in top-level directory. +# + +# Exit immediately if a command exits with a non-zero status. +set -e + +VALIDATOR=../../src/utils/ncvalidator/ncvalidator +NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff + +# prevent user environment setting of PNETCDF_HINTS to interfere +unset PNETCDF_HINTS + +# PNETCDF_HINTS= +# export PNETCDF_HINTS="$PNETCDF_HINTS" +# echo "PNETCDF_HINTS=$PNETCDF_HINTS" + +if test "$1" = ./open ; then + ${TESTSEQRUN} $1 ${srcdir}/arrays.bp + ${TESTSEQRUN} $1 ${srcdir}/attributes.bp + ${TESTSEQRUN} $1 ${srcdir}/arrays_big.bp + echo ${ADIOS_VER_GE_1132} + if test ${ADIOS_VER_GE_1132} = 1 ; then + ${TESTSEQRUN} $1 ${srcdir}/attributes_big.bp + fi +elif test "$1" = ./att ; then + ${TESTSEQRUN} $1 ${srcdir}/attributes.bp +else + ${TESTSEQRUN} $1 ${srcdir}/arrays.bp +fi + diff --git a/test/adios/var.c b/test/adios/var.c index ecf86d53a5..43c2a0d2bb 100644 --- a/test/adios/var.c +++ b/test/adios/var.c @@ -32,7 +32,7 @@ /* Handle errors by printing an error message and exiting with a * non-zero status. */ #define ERRCODE 2 -#define ERR(e) {printf("Error: %s\n", nc_strerror(e)); exit(ERRCODE);} +#define ERR(e) {fprintf(stderr,"Error: %s\n", nc_strerror(e)); exit(ERRCODE);} int main(int argc, char** argv) { char filename[256], tmp[1024]; @@ -154,7 +154,7 @@ int main(int argc, char** argv) { MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf("pass\n"); } MPI_Finalize(); diff --git a/test/adios/varm.c b/test/adios/varm.c index a60d2aa909..3018b2f15d 100644 --- a/test/adios/varm.c +++ b/test/adios/varm.c @@ -32,7 +32,7 @@ /* Handle errors by printing an error message and exiting with a * non-zero status. */ #define ERRCODE 2 -#define ERR(e) {printf("Error: %s\n", nc_strerror(e)); exit(ERRCODE);} +#define ERR(e) {fprintf(stderr,"Error: %s\n", nc_strerror(e)); exit(ERRCODE);} double data[NX][NY], datat[NY][NX]; @@ -95,7 +95,7 @@ int main(int argc, char** argv) { MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf("pass\n"); } MPI_Finalize(); diff --git a/test/adios/vars.c b/test/adios/vars.c index 37fbf25365..7842bd3e69 100644 --- a/test/adios/vars.c +++ b/test/adios/vars.c @@ -32,7 +32,7 @@ /* Handle errors by printing an error message and exiting with a * non-zero status. */ #define ERRCODE 2 -#define ERR(e) {printf("Error: %s\n", nc_strerror(e)); exit(ERRCODE);} +#define ERR(e) {fprintf(stderr,"Error: %s\n", nc_strerror(e)); exit(ERRCODE);} int main(int argc, char** argv) { char filename[256]; @@ -92,7 +92,7 @@ int main(int argc, char** argv) { MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf("pass\n"); } MPI_Finalize(); diff --git a/test/adios/wrap_runs.sh b/test/adios/wrap_runs.sh deleted file mode 100755 index e619098d06..0000000000 --- a/test/adios/wrap_runs.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2019, Northwestern University and Argonne National Laboratory -# See COPYRIGHT notice in top-level directory. -# - -# Exit immediately if a command exits with a non-zero status. -set -e - -VALIDATOR=../../src/utils/ncvalidator/ncvalidator -NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff - - -# echo "PNETCDF_DEBUG = ${PNETCDF_DEBUG}" -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" -else - safe_modes="0" -fi - -# prevent user environment setting of PNETCDF_HINTS to interfere -unset PNETCDF_HINTS - -for j in ${safe_modes} ; do - export PNETCDF_SAFE_MODE=$j - # echo "---- set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - if test "$1" = ./open ; then - ${TESTSEQRUN} $1 ${srcdir}/arrays.bp - ${TESTSEQRUN} $1 ${srcdir}/attributes.bp - ${TESTSEQRUN} $1 ${srcdir}/arrays_big.bp - echo ${ADIOS_VER_GE_1132} - if test ${ADIOS_VER_GE_1132} = 1 ; then - ${TESTSEQRUN} $1 ${srcdir}/attributes_big.bp - fi - elif test "$1" = ./att ; then - ${TESTSEQRUN} $1 ${srcdir}/attributes.bp - else - ${TESTSEQRUN} $1 ${srcdir}/arrays.bp - fi -done - diff --git a/test/burst_buffer/Makefile.am b/test/burst_buffer/Makefile.am index dde2d18ebc..5cb957d8cd 100644 --- a/test/burst_buffer/Makefile.am +++ b/test/burst_buffer/Makefile.am @@ -14,7 +14,7 @@ AM_CPPFLAGS = -I$(srcdir)/../common AM_CPPFLAGS += -I$(top_srcdir)/src/include AM_CPPFLAGS += -I$(top_builddir)/src/include -LDADD = $(top_builddir)/src/libs/libpnetcdf.la ../common/libtestutils.la +LDADD = ../common/libtestutils.la $(top_builddir)/src/libs/libpnetcdf.la LDADD += @NETCDF4_LDFLAGS@ @ADIOS_LDFLAGS@ @NETCDF4_LIBS@ @ADIOS_LIBS@ AM_CFLAGS = @@ -32,66 +32,63 @@ check_PROGRAMS = bb_bsize \ highdim \ varn -EXTRA_DIST = wrap_runs.sh parallel_run.sh - -NC_FILES = $(check_PROGRAMS:%=$(TESTOUTDIR)/%.nc) -META_FILES = $(NC_FILES:%=%_*.meta) -DATA_FILES = $(NC_FILES:%=%_*.data) - -CLEANFILES = $(NC_FILES) core core.* *.gcda *.gcno *.gcov gmon.out \ - $(META_FILES) $(DATA_FILES) - ../common/libtestutils.la: set -e; cd ../common && $(MAKE) $(MFLAGS) tests -TESTPROGRAMS = $(check_PROGRAMS) - # autimake 1.11.3 has not yet implemented AM_TESTS_ENVIRONMENT # For newer versions, we can use AM_TESTS_ENVIRONMENT instead -# AM_TESTS_ENVIRONMENT = export TESTPROGRAMS="$(TESTPROGRAMS)"; +# AM_TESTS_ENVIRONMENT = export check_PROGRAMS="$(check_PROGRAMS)"; # AM_TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; # AM_TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; TESTS_ENVIRONMENT = export SED="$(SED)"; TESTS_ENVIRONMENT += export srcdir="$(srcdir)"; +TESTS_ENVIRONMENT += export top_builddir="$(top_builddir)"; TESTS_ENVIRONMENT += export TESTOUTDIR="$(TESTOUTDIR)"; TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; TESTS_ENVIRONMENT += export TESTMPIRUN="$(TESTMPIRUN)"; -TESTS_ENVIRONMENT += export PNETCDF_DEBUG="$(PNETCDF_DEBUG)"; -TESTS_ENVIRONMENT += export TESTPROGRAMS="$(TESTPROGRAMS)"; TESTS_ENVIRONMENT += export check_PROGRAMS="$(check_PROGRAMS)"; -TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER="$(ENABLE_BURST_BUFFER)"; +TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER=@ENABLE_BURST_BUFFER@; +TESTS_ENVIRONMENT += export ENABLE_THREAD_SAFE=@ENABLE_THREAD_SAFE@; +TESTS_ENVIRONMENT += export MIMIC_LUSTRE=@MIMIC_LUSTRE@; + +TESTS = $(check_PROGRAMS) TEST_EXTENSIONS = .sh -LOG_COMPILER = $(srcdir)/wrap_runs.sh +LOG_COMPILER = $(srcdir)/seq_runs.sh SH_LOG_COMPILER = -TESTS = $(TESTPROGRAMS) +check_SCRIPTS = seq_runs.sh parallel_run.sh + +EXTRA_DIST = seq_runs.sh parallel_run.sh + +CLEANFILES = core core.* *.gcda *.gcno *.gcov gmon.out \ + $(TESTOUTDIR)/*.meta $(TESTOUTDIR)/*data # Some of these tests are designed to run on one processes, # Run them on 4 processes to see if they can handle well -ptest ptest4: $(TESTPROGRAMS) +ptest ptest4: $(check_PROGRAMS) @echo "===========================================================" @echo " $(subdir): Parallel testing on 4 MPI processes" @echo "===========================================================" @$(TESTS_ENVIRONMENT) \ $(srcdir)/parallel_run.sh 4 || exit 1 -ptest2: $(TESTPROGRAMS) +ptest2: $(check_PROGRAMS) @echo "===========================================================" @echo " $(subdir): Parallel testing on 2 MPI processes" @echo "===========================================================" @$(TESTS_ENVIRONMENT) \ $(srcdir)/parallel_run.sh 2 || exit 1 -ptest6: $(TESTPROGRAMS) +ptest6: $(check_PROGRAMS) @echo "===========================================================" @echo " $(subdir): Parallel testing on 6 MPI processes" @echo "===========================================================" @$(TESTS_ENVIRONMENT) \ $(srcdir)/parallel_run.sh 6 || exit 1 -ptests: ptest2 ptest4 ptest6 +ptests: ptest4 ptest6 ptest8 ptest10: # build check targets but not invoke diff --git a/test/burst_buffer/bb_bsize.c b/test/burst_buffer/bb_bsize.c index 2891f42edf..61a30f2d21 100644 --- a/test/burst_buffer/bb_bsize.c +++ b/test/burst_buffer/bb_bsize.c @@ -23,89 +23,64 @@ #include #include #include -#include +#include /* dirname() */ #define SIZE 1024 int buffer[SIZE * SIZE]; char bsize[32]; -int main(int argc, char *argv[]) { - int i, ret = NC_NOERR, nerr = 0; - int rank, np; - int ncid, varid; - int dimid[2]; - char *filename; +static +int test_bb(const char *out_path, + int coll_io, + MPI_Info info) +{ + char *folder, *dup_out_path; + int i, err = NC_NOERR, nerrs = 0; + int rank, np, ncid, varid, dimid[2]; MPI_Offset start[2], count[2]; - MPI_Info info; - /* Initialize MPI */ - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &np); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n", argv[0]); - MPI_Finalize(); - return 1; - } - - /* Determine test file name */ - if (argc > 1) - filename = argv[1]; - else - filename = "testfile.nc"; - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for checking request > buffer size", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - - /* Initialize file info */ - MPI_Info_create(&info); + /* add file info */ MPI_Info_set(info, "nc_burst_buf", "enable"); + /* Set default buffer size to 1/16 of the rows */ sprintf(bsize, "%u", (unsigned int)(SIZE * SIZE / 16 * sizeof(int))); MPI_Info_set(info, "nc_burst_buf_flush_buffer_size", bsize); + MPI_Info_set(info, "nc_burst_buf_overwrite", "enable"); + + dup_out_path = strdup(out_path); + folder = dirname(dup_out_path); + if (folder == NULL) + MPI_Info_set(info, "nc_burst_buf_dirname", "."); + else + MPI_Info_set(info, "nc_burst_buf_dirname", folder); + free(dup_out_path); + /* Create new netcdf file */ - ret = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, info, &ncid); - if (ret != NC_NOERR) { - printf("Error at line %d in %s: ncmpi_create: %d\n", __LINE__, __FILE__, ret); - nerr++; - goto ERROR; - } + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR /* Define dimensions */ - ret = ncmpi_def_dim(ncid, "X", SIZE * np, dimid); - if (ret != NC_NOERR) { - printf("Error at line %d in %s: ncmpi_def_dim: %d\n", __LINE__, __FILE__, ret); - nerr++; - goto ERROR; - } - ret = ncmpi_def_dim(ncid, "Y", SIZE, dimid + 1); - if (ret != NC_NOERR) { - printf("Error at line %d in %s: ncmpi_def_dim: %d\n", __LINE__, __FILE__, ret); - nerr++; - goto ERROR; - } + err = ncmpi_def_dim(ncid, "X", SIZE * np, dimid); + CHECK_ERR + err = ncmpi_def_dim(ncid, "Y", SIZE, dimid + 1); + CHECK_ERR /* Define variable */ - ret = ncmpi_def_var(ncid, "M", NC_INT, 2, dimid, &varid); - if (ret != NC_NOERR) { - printf("Error at line %d in %s: ncmpi_def_var: %d\n", __LINE__, __FILE__, ret); - nerr++; - goto ERROR; - } + err = ncmpi_def_var(ncid, "M", NC_INT, 2, dimid, &varid); + CHECK_ERR /* Switch to data mode */ - ret = ncmpi_enddef(ncid); - if (ret != NC_NOERR) { - printf("Error at line %d in %s: ncmpi_enddef: %d\n", __LINE__, __FILE__, ret); - nerr++; - goto ERROR; + err = ncmpi_enddef(ncid); + CHECK_ERR + + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR } /* Initialize buffer */ @@ -118,12 +93,11 @@ int main(int argc, char *argv[]) { start[1] = 0; count[0] = SIZE / 8; count[1] = SIZE; - ret = ncmpi_put_vara_int_all(ncid, varid, start, count, buffer); - if (ret != NC_NOERR) { - printf("Error at line %d in %s: ncmpi_put_vara_int: %d\n", __LINE__, __FILE__, ret); - nerr++; - goto ERROR; - } + if (coll_io) + err = ncmpi_put_vara_int_all(ncid, varid, start, count, buffer); + else + err = ncmpi_put_vara_int(ncid, varid, start, count, buffer); + CHECK_ERR /* Write remaining rows */ start[0] = SIZE * rank + SIZE / 8; @@ -131,12 +105,11 @@ int main(int argc, char *argv[]) { count[0] = 1; count[1] = SIZE; for (; start[0] < SIZE * (rank + 1); start[0]++) { - ret = ncmpi_put_vara_int_all(ncid, varid, start, count, buffer); - if (ret != NC_NOERR) { - printf("Error at line %d in %s: ncmpi_put_vara_int: %d\n", __LINE__, __FILE__, ret); - nerr++; - goto ERROR; - } + if (coll_io) + err = ncmpi_put_vara_int_all(ncid, varid, start, count, buffer); + else + err = ncmpi_put_vara_int(ncid, varid, start, count, buffer); + CHECK_ERR } /* @@ -148,48 +121,77 @@ int main(int argc, char *argv[]) { start[1] = 0; count[0] = SIZE; count[1] = SIZE; - ret = ncmpi_get_vara_int_all(ncid, varid, start, count, buffer); - if (ret != NC_NOERR) { - printf("Error at line %d in %s: ncmpi_get_vara_int: %d\n", __LINE__, __FILE__, ret); - nerr++; - goto ERROR; - } + if (coll_io) + err = ncmpi_get_vara_int_all(ncid, varid, start, count, buffer); + else + err = ncmpi_get_vara_int(ncid, varid, start, count, buffer); + CHECK_ERR /* Verify the result */ for (i = 0; i < SIZE * SIZE; i++) { if (buffer[i] != rank + 1) { - nerr++; - printf("Error at line %d in %s: expecting buffer[%d] = %d but got %d\n", __LINE__, __FILE__, i, rank + 1, buffer[i]); + fprintf(stderr,"Error at line %d in %s: expecting buffer[%d] = %d but got %d\n", + __LINE__, __FILE__, i, rank + 1, buffer[i]); + nerrs++; + goto err_out; } } /* Close the file */ - ret = ncmpi_close(ncid); - if (ret != NC_NOERR) { - printf("Error at line %d in %s: ncmpi_close: %d\n", __LINE__, __FILE__, ret); - nerr++; - goto ERROR; - } + err = ncmpi_close(ncid); + CHECK_ERR - MPI_Info_free(&info); +err_out: + return nerrs; +} - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - ret = ncmpi_inq_malloc_size(&malloc_size); - if (ret == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", sum_size); - } +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) +{ + int err=NC_NOERR; + MPI_Info local_info; + + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + MPI_Info_dup(info, &local_info); + err = test_bb(out_path, coll_io, local_info); + MPI_Info_free(&local_info); + + MPI_Info_dup(info, &local_info); + MPI_Info_set(local_info, "nc_burst_buf_shared_logs", "enable"); + err = test_bb(out_path, coll_io, local_info); + MPI_Info_free(&local_info); + + return err; +} -ERROR: - MPI_Allreduce(MPI_IN_PLACE, &nerr, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerr) printf(FAIL_STR, nerr); - else printf(PASS_STR); - } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "request size > buffer size", opt, test_io); MPI_Finalize(); - return nerr > 0; + return err; } diff --git a/test/burst_buffer/bb_hints.c b/test/burst_buffer/bb_hints.c index 4a3fa054d6..d2d544f317 100644 --- a/test/burst_buffer/bb_hints.c +++ b/test/burst_buffer/bb_hints.c @@ -23,103 +23,128 @@ #include #include #include -#include /* basename() */ +#include /* dirname() */ #include #include -int main(int argc, char** argv) { - char filename[256]; - int rank, nprocs, err, flag, nerrs=0; - int log_enabled; - int ncid; - MPI_Info info, infoused; +static +int test_bb(const char *out_path, + MPI_Info info) +{ + char *folder, *dup_out_path; + int err, flag, nerrs=0, ncid; + MPI_Info infoused; char hint[MPI_MAX_INFO_VAL]; - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for checking offsets of new variables ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - - MPI_Info_create(&info); + MPI_Info_set(info, "nc_burst_buf", "enable"); MPI_Info_set(info, "nc_burst_buf_del_on_close", "disable"); MPI_Info_set(info, "nc_burst_buf_flush_buffer_size", "256"); /* MPI_Info_set(info, "nc_burst_buf_dirname", "()@^$@!(_&$)@(#%%&)(*#$"); */ - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, info, &ncid); CHECK_ERR - err = ncmpi_inq_file_info(ncid, &infoused); CHECK_ERR + MPI_Info_set(info, "nc_burst_buf_overwrite", "enable"); - MPI_Info_get(infoused, "nc_burst_buf", MPI_MAX_INFO_VAL - 1, hint, &flag); - if (flag && strcasecmp(hint, "enable") == 0) - log_enabled = 1; + dup_out_path = strdup(out_path); + folder = dirname(dup_out_path); + if (folder == NULL) + MPI_Info_set(info, "nc_burst_buf_dirname", "."); else - log_enabled = 0; - - if (log_enabled) { - MPI_Info_get(infoused, "nc_burst_buf_del_on_close", MPI_MAX_INFO_VAL - 1, hint, &flag); - if (flag) { - if (strcmp(hint, "disable") != 0) { - printf("Error at line %d: unexpected nc_burst_buf_del_on_close = %s, but got %s\n", __LINE__, "disable", hint); - nerrs++; - } - } - else{ - printf("Error at line %d: nc_burst_buf_del_on_close is not set\n", __LINE__); + MPI_Info_set(info, "nc_burst_buf_dirname", folder); + free(dup_out_path); + + /* Create new netcdf file */ + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR + + err = ncmpi_inq_file_info(ncid, &infoused); CHECK_ERR + + MPI_Info_get(infoused, "nc_burst_buf_del_on_close", MPI_MAX_INFO_VAL - 1, hint, &flag); + if (flag) { + if (strcmp(hint, "disable") != 0) { + fprintf(stderr,"Error at line %d: unexpected nc_burst_buf_del_on_close = %s, but got %s\n", + __LINE__, "disable", hint); nerrs++; + goto err_out; } + } + else{ + fprintf(stderr,"Error at line %d: nc_burst_buf_del_on_close is not set\n", __LINE__); + nerrs++; + goto err_out; + } - MPI_Info_get(infoused, "nc_burst_buf_flush_buffer_size", MPI_MAX_INFO_VAL - 1, hint, &flag); - if (flag) { - if (strcmp(hint, "256") != 0) { - printf("Error at line %d: unexpected nc_burst_buf_flush_buffer_size = %s, but got %s\n", __LINE__, "256", hint); - nerrs++; - } - } - else{ - printf("Error at line %d: nc_burst_buf_flush_buffer_size is not set\n", __LINE__); + MPI_Info_get(infoused, "nc_burst_buf_flush_buffer_size", MPI_MAX_INFO_VAL - 1, hint, &flag); + if (flag) { + if (strcmp(hint, "256") != 0) { + fprintf(stderr,"Error at line %d: unexpected nc_burst_buf_flush_buffer_size = %s, but got %s\n", + __LINE__, "256", hint); nerrs++; + goto err_out; } } + else{ + fprintf(stderr,"Error at line %d: nc_burst_buf_flush_buffer_size is not set\n", __LINE__); + nerrs++; + goto err_out; + } err = ncmpi_enddef(ncid); CHECK_ERR err = ncmpi_close(ncid); CHECK_ERR - MPI_Info_free(&info); MPI_Info_free(&infoused); - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - } +err_out: + return nerrs; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) +{ + int err=NC_NOERR; + MPI_Info local_info; + + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + MPI_Info_dup(info, &local_info); + err = test_bb(out_path, local_info); + MPI_Info_free(&local_info); + + MPI_Info_dup(info, &local_info); + MPI_Info_set(local_info, "nc_burst_buf_shared_logs", "enable"); + err = test_bb(out_path, local_info); + MPI_Info_free(&local_info); + + return err; +} + +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "burt buffering hints", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/burst_buffer/bb_many_reqs.c b/test/burst_buffer/bb_many_reqs.c index a379134ac7..bfc0439868 100644 --- a/test/burst_buffer/bb_many_reqs.c +++ b/test/burst_buffer/bb_many_reqs.c @@ -23,56 +23,48 @@ #define NREQ 2048 #define NROUND 4 -int main(int argc, char *argv[]) { - int i, j, err, nerrs = 0; - int rank, np; - int ncid, varid; +static +int test_bb(const char *out_path, + int coll_io, + MPI_Info info) +{ + char *folder, *dup_out_path; + int i, j, err, nerrs = 0, rank, np, ncid, varid, dimid[2]; int *buffer, *reqs, *stat; - int dimid[2]; - char *filename; MPI_Offset start[2]; - MPI_Info info; - /* Initialize MPI */ - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &np); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n", argv[0]); - MPI_Finalize(); - return 1; - } + MPI_Info_set(info, "nc_burst_buf", "enable"); + MPI_Info_set(info, "nc_burst_buf_overwrite", "enable"); - /* Determine test file name */ - if (argc > 1) - filename = argv[1]; + dup_out_path = strdup(out_path); + folder = dirname(dup_out_path); + if (folder == NULL) + MPI_Info_set(info, "nc_burst_buf_dirname", "."); else - filename = "testfile.nc"; - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for burst buffer big requests", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - - /* Initialize file info */ - MPI_Info_create(&info); - MPI_Info_set(info, "nc_burst_buf", "enable"); + MPI_Info_set(info, "nc_burst_buf_dirname", folder); + free(dup_out_path); /* Create new netcdf file */ - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR /* Define dimensions */ - err = ncmpi_def_dim(ncid, "X", np, dimid); CHECK_ERR - err = ncmpi_def_dim(ncid, "Y", NREQ, dimid + 1); CHECK_ERR + err = ncmpi_def_dim(ncid, "X", np, dimid); CHECK_ERR + err = ncmpi_def_dim(ncid, "Y", NREQ, dimid + 1); CHECK_ERR /* Define variable */ - err = ncmpi_def_var(ncid, "M", NC_INT, 2, dimid, &varid); CHECK_ERR + err = ncmpi_def_var(ncid, "M", NC_INT, 2, dimid, &varid); CHECK_ERR /* Switch to data mode */ - err = ncmpi_enddef(ncid); CHECK_ERR + err = ncmpi_enddef(ncid); CHECK_ERR + + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } /* Prepare Buffer */ reqs = (int*)malloc(sizeof(int) * NREQ); @@ -87,15 +79,22 @@ int main(int argc, char *argv[]) { for (i = 0; i < NREQ; i++) { start[0] = rank; start[1] = i; - err = ncmpi_iput_var1_int(ncid, varid, start, buffer + i, reqs + i); CHECK_ERR + err = ncmpi_iput_var1_int(ncid, varid, start, buffer + i, reqs + i); CHECK_ERR } - err = ncmpi_wait_all(ncid, NREQ, reqs, stat); CHECK_ERR + if (coll_io) + err = ncmpi_wait_all(ncid, NREQ, reqs, stat); + else + err = ncmpi_wait(ncid, NREQ, reqs, stat); + CHECK_ERR for (i = 0; i < NREQ; i++) { - err = stat[i]; CHECK_ERR + err = stat[i]; CHECK_ERR } for (i = 0; i < NREQ; i++) { if (reqs[i] != NC_REQ_NULL) { - printf("Error at line %d in %s: expecting reqs[%d] = NC_REQ_NULL but got %d\n", __LINE__, __FILE__, i, reqs[i]); + fprintf(stderr,"Error at line %d in %s: expecting reqs[%d] = NC_REQ_NULL but got %d\n", + __LINE__, __FILE__, i, reqs[i]); + nerrs++; + goto err_out; } } @@ -104,48 +103,92 @@ int main(int argc, char *argv[]) { for (i = 0; i < NREQ; i++ ) { start[0] = rank; start[1] = i; - err = ncmpi_iget_var1_int(ncid, varid, start, buffer + i, reqs + i); CHECK_ERR + err = ncmpi_iget_var1_int(ncid, varid, start, buffer + i, reqs + i); CHECK_ERR } - err = ncmpi_wait_all(ncid, NREQ, reqs, stat); CHECK_ERR + if (coll_io) + err = ncmpi_wait_all(ncid, NREQ, reqs, stat); + else + err = ncmpi_wait(ncid, NREQ, reqs, stat); + CHECK_ERR for (i = 0; i < NREQ; i++) { - err = stat[i]; CHECK_ERR + err = stat[i]; CHECK_ERR } for (i = 0; i < NREQ; i++) { if (buffer[i] != rank + 1) { - printf("Error at line %d in %s: expecting buffer[%d] = %d but got %d\n", __LINE__, __FILE__, i, rank + 1, buffer[i]); + fprintf(stderr,"Error at line %d in %s: expecting buffer[%d] = %d but got %d\n", + __LINE__, __FILE__, i, rank + 1, buffer[i]); + nerrs++; + goto err_out; } } for (i = 0; i < NREQ; i++) { if (reqs[i] != NC_REQ_NULL) { - printf("Error at line %d in %s: expecting reqs[%d] = NC_REQ_NULL but got %d\n", __LINE__, __FILE__, i, reqs[i]); + fprintf(stderr,"Error at line %d in %s: expecting reqs[%d] = NC_REQ_NULL but got %d\n", + __LINE__, __FILE__, i, reqs[i]); + nerrs++; + goto err_out; } } } /* Close the file */ - err = ncmpi_close(ncid); CHECK_ERR + err = ncmpi_close(ncid); CHECK_ERR - MPI_Info_free(&info); free(buffer); free(reqs); free(stat); - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", sum_size); - } +err_out: + return nerrs; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR, nerrs); - else printf(PASS_STR); - } +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) +{ + int err=NC_NOERR; + MPI_Info local_info; + + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + MPI_Info_dup(info, &local_info); + err = test_bb(out_path, coll_io, local_info); + MPI_Info_free(&local_info); + + MPI_Info_dup(info, &local_info); + MPI_Info_set(local_info, "nc_burst_buf_shared_logs", "enable"); + err = test_bb(out_path, coll_io, local_info); + MPI_Info_free(&local_info); + + return err; +} + +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "big requests", opt, test_io); MPI_Finalize(); - return nerrs > 0; + return err; } diff --git a/test/burst_buffer/bb_nonblocking.c b/test/burst_buffer/bb_nonblocking.c index 57cd858022..b1dbe8490b 100644 --- a/test/burst_buffer/bb_nonblocking.c +++ b/test/burst_buffer/bb_nonblocking.c @@ -18,111 +18,134 @@ #include #include -int main(int argc, char *argv[]) { - int err, tmp, nerrs = 0; - int rank, np; - int ncid, varid; +static +int test_bb(const char *out_path, + MPI_Info info) +{ + char *folder, *dup_out_path; + int err, tmp, nerrs = 0, rank, np, ncid, varid, dimid[2]; int buffer, req1, req2, stat; - int dimid[2]; - char *filename; MPI_Offset start[2]; - MPI_Info info; - /* Initialize MPI */ - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &np); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n", argv[0]); - MPI_Finalize(); - return 1; - } + /* add file info */ + MPI_Info_set(info, "nc_burst_buf", "enable"); + MPI_Info_set(info, "nc_burst_buf_overwrite", "enable"); - /* Determine test file name */ - if (argc > 1) - filename = argv[1]; + dup_out_path = strdup(out_path); + folder = dirname(dup_out_path); + if (folder == NULL) + MPI_Info_set(info, "nc_burst_buf_dirname", "."); else - filename = "testfile.nc"; - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for when requests are > buffer size", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - - /* Initialize file info */ - MPI_Info_create(&info); - MPI_Info_set(info, "nc_burst_buf", "enable"); + MPI_Info_set(info, "nc_burst_buf_dirname", folder); + free(dup_out_path); /* Create new netcdf file */ - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR /* Define dimensions */ - err = ncmpi_def_dim(ncid, "X", np, dimid); CHECK_ERR - err = ncmpi_def_dim(ncid, "Y", 4, dimid + 1); CHECK_ERR + err = ncmpi_def_dim(ncid, "X", np, dimid); CHECK_ERR + err = ncmpi_def_dim(ncid, "Y", 4, dimid + 1); CHECK_ERR /* Define variable */ - err = ncmpi_def_var(ncid, "M", NC_INT, 2, dimid, &varid); CHECK_ERR + err = ncmpi_def_var(ncid, "M", NC_INT, 2, dimid, &varid); CHECK_ERR + + err = ncmpi_set_fill(ncid, NC_FILL, NULL); CHECK_ERR /* Switch to data mode */ - err = ncmpi_enddef(ncid); CHECK_ERR + err = ncmpi_enddef(ncid); CHECK_ERR buffer = rank + 1; - start[0] = 0; + /* each process writes to a different row */ + start[0] = rank; start[1] = 0; - err = ncmpi_iput_var1_int(ncid, varid, start, &buffer, &req1); CHECK_ERR + err = ncmpi_iput_var1_int(ncid, varid, start, &buffer, &req1); CHECK_ERR start[1] = 1; - err = ncmpi_iput_var1_int(ncid, varid, start, &buffer, &req2); CHECK_ERR + err = ncmpi_iput_var1_int(ncid, varid, start, &buffer, &req2); CHECK_ERR start[1] = 0; - err = ncmpi_get_var1_int_all(ncid, varid, start, &buffer); CHECK_ERR + err = ncmpi_get_var1_int_all(ncid, varid, start, &buffer); CHECK_ERR start[1] = 1; - err = ncmpi_get_var1_int_all(ncid, varid, start, &buffer); CHECK_ERR - err = ncmpi_cancel(ncid, 1, &req1, &stat); CHECK_ERR + err = ncmpi_get_var1_int_all(ncid, varid, start, &buffer); CHECK_ERR + err = ncmpi_cancel(ncid, 1, &req1, &stat); CHECK_ERR err = stat; EXP_ERR(NC_EFLUSHED) - err = ncmpi_wait_all(ncid, 1, &req2, &stat); CHECK_ERR - err = stat; CHECK_ERR + err = ncmpi_wait_all(ncid, 1, &req2, &stat); CHECK_ERR + err = stat; CHECK_ERR start[1] = 2; - err = ncmpi_iput_var1_int(ncid, varid, start, &buffer, &req1); CHECK_ERR + err = ncmpi_iput_var1_int(ncid, varid, start, &buffer, &req1); CHECK_ERR start[1] = 3; - err = ncmpi_iput_var1_int(ncid, varid, start, &buffer, &req2); CHECK_ERR + err = ncmpi_iput_var1_int(ncid, varid, start, &buffer, &req2); CHECK_ERR tmp = req1; - err = ncmpi_cancel(ncid, 1, &req1, &stat); CHECK_ERR - err = stat; CHECK_ERR + err = ncmpi_cancel(ncid, 1, &req1, &stat); CHECK_ERR + err = stat; CHECK_ERR start[1] = 2; - err = ncmpi_get_var1_int_all(ncid, varid, start, &buffer); CHECK_ERR + err = ncmpi_get_var1_int_all(ncid, varid, start, &buffer); CHECK_ERR start[1] = 3; - err = ncmpi_get_var1_int_all(ncid, varid, start, &buffer); CHECK_ERR + err = ncmpi_get_var1_int_all(ncid, varid, start, &buffer); CHECK_ERR req1 = tmp; - err = ncmpi_wait_all(ncid, 1, &req1, &stat); CHECK_ERR + err = ncmpi_wait_all(ncid, 1, &req1, &stat); CHECK_ERR err = stat; EXP_ERR(NC_EINVAL_REQUEST) - err = ncmpi_wait_all(ncid, 1, &req2, &stat); CHECK_ERR - err = stat; CHECK_ERR + err = ncmpi_wait_all(ncid, 1, &req2, &stat); CHECK_ERR + err = stat; CHECK_ERR /* Close the file */ - err = ncmpi_close(ncid); CHECK_ERR + err = ncmpi_close(ncid); CHECK_ERR - MPI_Info_free(&info); + return nerrs; +} + +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) +{ + int err=NC_NOERR; + MPI_Info local_info; + + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + MPI_Info_dup(info, &local_info); + err = test_bb(out_path, local_info); + MPI_Info_free(&local_info); + + MPI_Info_dup(info, &local_info); + MPI_Info_set(local_info, "nc_burst_buf_shared_logs", "enable"); + err = test_bb(out_path, local_info); + MPI_Info_free(&local_info); + + MPI_Barrier(MPI_COMM_WORLD); + return err; +} + +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", sum_size); - } + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR, nerrs); - else printf(PASS_STR); - } + err = tst_main(argc, argv, "nonblocking APIs", opt, test_io); MPI_Finalize(); - return nerrs > 0; + return err; } diff --git a/test/burst_buffer/highdim.c b/test/burst_buffer/highdim.c index 6957f6c0f1..f42e187cd9 100644 --- a/test/burst_buffer/highdim.c +++ b/test/burst_buffer/highdim.c @@ -58,44 +58,21 @@ #define BSIZE 1024 * 1024 -int main(int argc, char *argv[]) +static +int test_bb(const char *out_path, + int coll_io, + MPI_Info info) { - char *filename=NULL, dimname[64]; - int i, ret=NC_NOERR, nerr=0; - int rank, np, ndims; - int ncid, varid; + char *folder, *dup_out_path, dimname[64]; + int i, err=NC_NOERR, nerrs=0, rank, np, ndims, ncid, varid; int *dimid=NULL, *buffer=NULL; long long j; MPI_Offset *start=NULL, *count=NULL, *stride=NULL; - /* Initialize MPI */ - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &np); - if (argc > 3) { - if (!rank) printf("Usage: %s [filename]\n", argv[0]); - MPI_Finalize(); - return 1; - } - - /* Determine ndims and test file name */ - if (argc > 1) - filename = strdup(argv[1]); - else - filename = strdup("testfile.nc"); - ndims = DIM; - if (argc > 2) - ndims = atoi(argv[2]); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for high dimensional variables", basename(argv[0])); - printf("%-66s ------ ", cmd_str); - fflush(stdout); - free(cmd_str); - } /* Allocate buffers */ dimid = (int*)malloc(sizeof(int) * ndims); @@ -104,9 +81,9 @@ int main(int argc, char *argv[]) stride = (MPI_Offset*)malloc(sizeof(MPI_Offset) * ndims); buffer = (int*)malloc(sizeof(int) * BSIZE); if (dimid == NULL || start == NULL || count == NULL || stride == NULL || buffer == NULL) { - printf("Error at line %d in %s: malloc error\n", __LINE__, __FILE__); - nerr++; - goto ERROR; + fprintf(stderr,"Error at line %d in %s: malloc error\n", __LINE__, __FILE__); + nerrs++; + goto err_out; } /* Initialize buffers and calculate share among processes @@ -133,104 +110,130 @@ int main(int argc, char *argv[]) start[0] = rank; stride[0] = np; + MPI_Info_set(info, "nc_burst_buf", "enable"); + MPI_Info_set(info, "nc_burst_buf_overwrite", "enable"); + + dup_out_path = strdup(out_path); + folder = dirname(dup_out_path); + if (folder == NULL) + MPI_Info_set(info, "nc_burst_buf_dirname", "."); + else + MPI_Info_set(info, "nc_burst_buf_dirname", folder); + free(dup_out_path); + /* Create new netcdf file */ - ret = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, MPI_INFO_NULL, &ncid); - if (ret != NC_NOERR) { - printf("Error at line %d in %s: ncmpi_create: %d\n", __LINE__, __FILE__, ret); - nerr++; - goto ERROR; - } + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR /* Define dimensions */ for (i = 0; i < ndims; i++) { sprintf(dimname, "D%d", i); /* Submatrix of each process stack along the first dimension */ - if (i == 0) { - ret = ncmpi_def_dim(ncid, dimname, count[i] * np, dimid + i); - } - else{ - ret = ncmpi_def_dim(ncid, dimname, count[i], dimid + i); - } - if (ret != NC_NOERR) { - printf("Error at line %d in %s: ncmpi_enddef: %d\n", __LINE__, __FILE__, ret); - nerr++; - goto ERROR; - } + if (i == 0) + err = ncmpi_def_dim(ncid, dimname, count[i] * np, dimid + i); + else + err = ncmpi_def_dim(ncid, dimname, count[i], dimid + i); + CHECK_ERR } /* Define variable */ - ret = ncmpi_def_var(ncid, "M", NC_INT, ndims, dimid, &varid); - if (ret != NC_NOERR) { - printf("Error at line %d in %s: ncmpi_def_var: %d\n", __LINE__, __FILE__, ret); - nerr++; - goto ERROR; - } + err = ncmpi_def_var(ncid, "M", NC_INT, ndims, dimid, &varid); + CHECK_ERR /* Switch to data mode */ - ret = ncmpi_enddef(ncid); - if (ret != NC_NOERR) { - printf("Error at line %d in %s: ncmpi_enddef: %d\n", __LINE__, __FILE__, ret); - nerr++; - goto ERROR; + err = ncmpi_enddef(ncid); + CHECK_ERR + + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR } /* Write variable */ - ret = ncmpi_put_vars_int_all(ncid, varid, start, count, stride, buffer); - if (ret != NC_NOERR) { - printf("Error at line %d in %s: ncmpi_put_vars_int: %d\n", __LINE__, __FILE__, ret); - nerr++; - goto ERROR; - } + if (coll_io) + err = ncmpi_put_vars_int_all(ncid, varid, start, count, stride, buffer); + else + err = ncmpi_put_vars_int(ncid, varid, start, count, stride, buffer); + CHECK_ERR /* Read it back */ - ret = ncmpi_get_vars_int_all(ncid, varid, start, count, stride, buffer); - if (ret != NC_NOERR) { - printf("Error at line %d in %s: ncmpi_get_vars_int: %d\n", __LINE__, __FILE__, ret); - nerr++; - goto ERROR; - } + if (coll_io) + err = ncmpi_get_vars_int_all(ncid, varid, start, count, stride, buffer); + else + err = ncmpi_get_vars_int(ncid, varid, start, count, stride, buffer); + CHECK_ERR /* Verify the result */ for (i = 0; i < BSIZE; i++) { if (buffer[i] != rank + 1) { - nerr++; - printf("Error at line %d in %s: expecting buffer[%d] = %d but got %d\n", __LINE__, __FILE__, i, rank + 1, buffer[i]); + fprintf(stderr,"Error at line %d in %s: expecting buffer[%d] = %d but got %d\n", + __LINE__, __FILE__, i, rank + 1, buffer[i]); + nerrs++; + goto err_out; } } /* Close the file */ - ret = ncmpi_close(ncid); - if (ret != NC_NOERR) { - printf("Error at line %d in %s: ncmpi_close: %d\n", __LINE__, __FILE__, ret); - nerr++; - goto ERROR; - } - - - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - ret = ncmpi_inq_malloc_size(&malloc_size); - if (ret == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", sum_size); - } - -ERROR: - MPI_Allreduce(MPI_IN_PLACE, &nerr, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerr) printf(FAIL_STR, nerr); - else printf(PASS_STR); - } + err = ncmpi_close(ncid); + CHECK_ERR if (start != NULL) free(start); if (count != NULL) free(count); if (stride != NULL) free(stride); if (dimid != NULL) free(dimid); if (buffer != NULL) free(buffer); - if (filename != NULL) free(filename); + +err_out: + return nerrs; +} + +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) +{ + int err=NC_NOERR; + MPI_Info local_info; + + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + MPI_Info_dup(info, &local_info); + err = test_bb(out_path, coll_io, local_info); + MPI_Info_free(&local_info); + + MPI_Info_dup(info, &local_info); + MPI_Info_set(local_info, "nc_burst_buf_shared_logs", "enable"); + err = test_bb(out_path, coll_io, local_info); + MPI_Info_free(&local_info); + + return err; +} + +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "high dimensional variables", opt, test_io); MPI_Finalize(); - return nerr > 0; + return err; } diff --git a/test/burst_buffer/parallel_run.sh b/test/burst_buffer/parallel_run.sh index d726ee7601..ad26c7a5b8 100755 --- a/test/burst_buffer/parallel_run.sh +++ b/test/burst_buffer/parallel_run.sh @@ -1,66 +1,41 @@ #!/bin/bash # -# Copyright (C) 2018, Northwestern University and Argonne National Laboratory +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory # See COPYRIGHT notice in top-level directory. # # Exit immediately if a command exits with a non-zero status. set -e -VALIDATOR=../../src/utils/ncvalidator/ncvalidator -NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff +DRY_RUN=no +VERBOSE=no -# remove file system type prefix if there is any -OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $MPIRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $MPIRUN $@ + fi +} MPIRUN=`echo ${TESTMPIRUN} | ${SED} -e "s/NP/$1/g"` # echo "MPIRUN = ${MPIRUN}" -# echo "TESTPROGRAMS=${TESTPROGRAMS}" - -# echo "PNETCDF_DEBUG = ${PNETCDF_DEBUG}" -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" -else - safe_modes="0" -fi +# echo "check_PROGRAMS=${check_PROGRAMS}" # prevent user environment setting of PNETCDF_HINTS to interfere unset PNETCDF_HINTS -for i in ${TESTPROGRAMS} ; do - for j in ${safe_modes} ; do - for intra_aggr in 0 1 ; do - if test "$j" = 1 ; then # test only in safe mode - export PNETCDF_HINTS="romio_no_indep_rw=true" - else - export PNETCDF_HINTS= - fi - if test "$intra_aggr" = 1 ; then - export PNETCDF_HINTS="${PNETCDF_HINTS};nc_num_aggrs_per_node=2" - fi - export PNETCDF_SAFE_MODE=$j - # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" +if test "x$MIMIC_LUSTRE" != x1 ; then + PNETCDF_HINTS="cb_nodes=2" +fi - saved_PNETCDF_HINTS=${PNETCDF_HINTS} - export PNETCDF_HINTS="${PNETCDF_HINTS};nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" - rm -f ${OUTDIR}/$i.nc - ${MPIRUN} ./$i ${TESTOUTDIR}/$i.nc - export PNETCDF_HINTS=${saved_PNETCDF_HINTS} +for i in ${check_PROGRAMS} ; do - # echo "--- validating file ${TESTOUTDIR}/$i.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.nc + exe_name=`basename $i` - export PNETCDF_HINTS="${PNETCDF_HINTS};nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable;nc_burst_buf_shared_logs=enable" - rm -f ${OUTDIR}/$i.nc - ${MPIRUN} ./$i ${TESTOUTDIR}/$i.nc - export PNETCDF_HINTS=${saved_PNETCDF_HINTS} + run_cmd ./$i -q -o ${TESTOUTDIR}/${exe_name}.nc - # echo "--- validating file ${TESTOUTDIR}/$i.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.nc - done - done - rm -f ${OUTDIR}/$i.nc - rm -f ${OUTDIR}/$i.nc*.data - rm -f ${OUTDIR}/$i.nc*.meta -done +done # check_PROGRAMS diff --git a/test/burst_buffer/seq_runs.sh b/test/burst_buffer/seq_runs.sh new file mode 100755 index 0000000000..fb850c9388 --- /dev/null +++ b/test/burst_buffer/seq_runs.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory +# See COPYRIGHT notice in top-level directory. +# + +# Exit immediately if a command exits with a non-zero status. +set -e + +DRY_RUN=no +VERBOSE=no + +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $TESTSEQRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $TESTSEQRUN $@ + fi +} + +exe_name=`basename $1` + +# prevent user environment setting of PNETCDF_HINTS to interfere +unset PNETCDF_HINTS + +run_cmd ./$1 -q -o ${TESTOUTDIR}/${exe_name}.nc + + diff --git a/test/burst_buffer/varn.c b/test/burst_buffer/varn.c index 81e93063e9..92b931aa65 100644 --- a/test/burst_buffer/varn.c +++ b/test/burst_buffer/varn.c @@ -18,58 +18,49 @@ #include #include -int main(int argc, char *argv[]) { - int i; - int err, nerrs = 0; - int rank, np; - int ncid, varid; - int dimid[2]; - int buffer[10]; - char *filename; +static +int test_bb(const char *out_path, + int coll_io, + MPI_Info info) +{ + char *folder, *dup_out_path; + int i, err, nerrs = 0, rank, np, ncid, varid, dimid[2], buffer[10]; MPI_Offset starts[10][2], counts[10][2]; MPI_Offset *Starts[10], *Counts[10]; - MPI_Info info; - /* Initialize MPI */ - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &np); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n", argv[0]); - MPI_Finalize(); - return 1; - } + /* add file info */ + MPI_Info_set(info, "nc_burst_buf", "enable"); + MPI_Info_set(info, "nc_burst_buf_overwrite", "enable"); - /* Determine test file name */ - if (argc > 1) - filename = argv[1]; + dup_out_path = strdup(out_path); + folder = dirname(dup_out_path); + if (folder == NULL) + MPI_Info_set(info, "nc_burst_buf_dirname", "."); else - filename = "testfile.nc"; - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for when requests are > buffer size", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - - /* Initialize file info */ - MPI_Info_create(&info); - MPI_Info_set(info, "nc_burst_buf", "enable"); + MPI_Info_set(info, "nc_burst_buf_dirname", folder); + free(dup_out_path); /* Create new netcdf file */ - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR /* Define dimensions */ - err = ncmpi_def_dim(ncid, "X", np, dimid); CHECK_ERR - err = ncmpi_def_dim(ncid, "Y", 10, dimid + 1); CHECK_ERR + err = ncmpi_def_dim(ncid, "X", np, dimid); CHECK_ERR + err = ncmpi_def_dim(ncid, "Y", 10, dimid + 1); CHECK_ERR /* Define variable */ - err = ncmpi_def_var(ncid, "M", NC_INT, 2, dimid, &varid); CHECK_ERR + err = ncmpi_def_var(ncid, "M", NC_INT, 2, dimid, &varid); CHECK_ERR /* Switch to data mode */ - err = ncmpi_enddef(ncid); CHECK_ERR + err = ncmpi_enddef(ncid); CHECK_ERR + + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } for(i = 0; i < 10; i++){ starts[i][0] = rank; @@ -82,22 +73,46 @@ int main(int argc, char *argv[]) { } /* Standard varn */ - err = ncmpi_put_varn_int_all(ncid, varid, 10, Starts, Counts, buffer); CHECK_ERR - err = ncmpi_get_varn_int_all(ncid, varid, 10, Starts, Counts, buffer); CHECK_ERR + if (coll_io) + err = ncmpi_put_varn_int_all(ncid, varid, 10, Starts, Counts, buffer); + else + err = ncmpi_put_varn_int(ncid, varid, 10, Starts, Counts, buffer); + CHECK_ERR + + for (i=0; i<10; i++) buffer[0] = -1; + if (coll_io) + err = ncmpi_get_varn_int_all(ncid, varid, 10, Starts, Counts, buffer); + else + err = ncmpi_get_varn_int(ncid, varid, 10, Starts, Counts, buffer); + CHECK_ERR for(i = 0; i < 10; i++){ if (buffer[i] != rank + i){ + fprintf(stderr,"Error at line %d in %s: expecting buffer[%d] = %d but got %d\n", + __LINE__, __FILE__, i, rank + 1, buffer[i]); nerrs++; - printf("Error at line %d in %s: expecting buffer[%d] = %d but got %d\n", __LINE__, __FILE__, i, rank + 1, buffer[i]); + goto err_out; } } /* NULL counts */ - err = ncmpi_put_varn_int_all(ncid, varid, 10, Starts, NULL, buffer); CHECK_ERR - err = ncmpi_get_varn_int_all(ncid, varid, 10, Starts, NULL, buffer); CHECK_ERR + if (coll_io) + err = ncmpi_put_varn_int_all(ncid, varid, 10, Starts, NULL, buffer); + else + err = ncmpi_put_varn_int(ncid, varid, 10, Starts, NULL, buffer); + CHECK_ERR + + for (i=0; i<10; i++) buffer[0] = -1; + if (coll_io) + err = ncmpi_get_varn_int_all(ncid, varid, 10, Starts, NULL, buffer); + else + err = ncmpi_get_varn_int(ncid, varid, 10, Starts, NULL, buffer); + CHECK_ERR for(i = 0; i < 10; i++){ if (buffer[i] != rank + i){ + fprintf(stderr,"Error at line %d in %s: expecting buffer[%d] = %d but got %d\n", + __LINE__, __FILE__, i, rank + 1, buffer[i]); nerrs++; - printf("Error at line %d in %s: expecting buffer[%d] = %d but got %d\n", __LINE__, __FILE__, i, rank + 1, buffer[i]); + goto err_out; } } @@ -105,36 +120,81 @@ int main(int argc, char *argv[]) { for(i = 0; i < 10; i += 2){ Counts[i] = (MPI_Offset*)counts[i]; } - err = ncmpi_put_varn_int_all(ncid, varid, 10, Starts, Counts, buffer); CHECK_ERR - err = ncmpi_get_varn_int_all(ncid, varid, 10, Starts, Counts, buffer); CHECK_ERR + if (coll_io) + err = ncmpi_put_varn_int_all(ncid, varid, 10, Starts, Counts, buffer); + else + err = ncmpi_put_varn_int(ncid, varid, 10, Starts, Counts, buffer); + CHECK_ERR + + for (i=0; i<10; i++) buffer[0] = -1; + if (coll_io) + err = ncmpi_get_varn_int_all(ncid, varid, 10, Starts, Counts, buffer); + else + err = ncmpi_get_varn_int(ncid, varid, 10, Starts, Counts, buffer); + CHECK_ERR for(i = 0; i < 10; i++){ if (buffer[i] != rank + i){ + fprintf(stderr,"Error at line %d in %s: expecting buffer[%d] = %d but got %d\n", + __LINE__, __FILE__, i, rank + 1, buffer[i]); nerrs++; - printf("Error at line %d in %s: expecting buffer[%d] = %d but got %d\n", __LINE__, __FILE__, i, rank + 1, buffer[i]); + goto err_out; } } /* Close the file */ - err = ncmpi_close(ncid); CHECK_ERR + err = ncmpi_close(ncid); CHECK_ERR - MPI_Info_free(&info); +err_out: + return nerrs; +} - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", sum_size); - } +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) +{ + int err=NC_NOERR; + MPI_Info local_info; + + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + MPI_Info_dup(info, &local_info); + err = test_bb(out_path, coll_io, local_info); + MPI_Info_free(&local_info); + + MPI_Info_dup(info, &local_info); + MPI_Info_set(local_info, "nc_burst_buf_shared_logs", "enable"); + err = test_bb(out_path, coll_io, local_info); + MPI_Info_free(&local_info); + + return err; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR, nerrs); - else printf(PASS_STR); - } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "varn API", opt, test_io); MPI_Finalize(); - return nerrs > 0; + return err; } diff --git a/test/burst_buffer/wrap_runs.sh b/test/burst_buffer/wrap_runs.sh deleted file mode 100755 index 308ccfc19a..0000000000 --- a/test/burst_buffer/wrap_runs.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2003, Northwestern University and Argonne National Laboratory -# See COPYRIGHT notice in top-level directory. -# - -# Exit immediately if a command exits with a non-zero status. -set -e - -VALIDATOR=../../src/utils/ncvalidator/ncvalidator -outfile=`basename $1` - -# remove file system type prefix if there is any -OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` - -# echo "PNETCDF_DEBUG = ${PNETCDF_DEBUG}" -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" -else - safe_modes="0" -fi - -# prevent user environment setting of PNETCDF_HINTS to interfere -unset PNETCDF_HINTS - -for j in ${safe_modes} ; do - export PNETCDF_SAFE_MODE=$j - # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - export PNETCDF_HINTS="nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" - ${TESTSEQRUN} $1 ${TESTOUTDIR}/$outfile.nc - unset PNETCDF_HINTS - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$outfile.nc -done - -rm -f ${OUTDIR}/$outfile.nc -rm -f ${OUTDIR}/$outfile.nc_0_0.data -rm -f ${OUTDIR}/$outfile.nc_0_0.meta - diff --git a/test/cdf_format/Makefile.am b/test/cdf_format/Makefile.am index a8d18918fe..e25cc116cd 100644 --- a/test/cdf_format/Makefile.am +++ b/test/cdf_format/Makefile.am @@ -2,8 +2,6 @@ # Copyright (C) 2003, Northwestern University and Argonne National Laboratory # See COPYRIGHT notice in top-level directory. # -# $Id$ -# # @configure_input@ SUFFIXES = .o .c @@ -11,8 +9,10 @@ SUFFIXES = .o .c AM_CPPFLAGS = -I$(top_srcdir)/src/include AM_CPPFLAGS += -I$(srcdir)/../common AM_CPPFLAGS += -I$(top_builddir)/src/include +AM_CPPFLAGS += -I$(top_srcdir)/src/utils/ncmpidiff -LDADD = $(top_builddir)/src/libs/libpnetcdf.la ../common/libtestutils.la +LDADD = ../common/libtestutils.la +LDADD += $(top_builddir)/src/libs/libpnetcdf.la LDADD += @NETCDF4_LDFLAGS@ @ADIOS_LDFLAGS@ @NETCDF4_LIBS@ @ADIOS_LIBS@ if DECL_MPI_OFFSET @@ -22,31 +22,35 @@ if DECL_MPI_OFFSET # AM_FFLAGS += $(FC_DEFINE)HAVE_DECL_MPI_OFFSET endif -TESTPROGRAMS = test_inq_format \ - cdf_type \ - dim_cdf12 - -check_PROGRAMS = $(TESTPROGRAMS) tst_open_cdf5 tst_corrupt +check_PROGRAMS = test_inq_format \ + cdf_type \ + dim_cdf12 \ + tst_open_cdf5 \ + tst_corrupt # autimake 1.11.3 has not yet implemented AM_TESTS_ENVIRONMENT # For newer versions, we can use AM_TESTS_ENVIRONMENT instead -# AM_TESTS_ENVIRONMENT = export TESTPROGRAMS="$(TESTPROGRAMS)"; +# AM_TESTS_ENVIRONMENT = export check_PROGRAMS="$(check_PROGRAMS)"; # AM_TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; # AM_TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; TESTS_ENVIRONMENT = export SED="$(SED)"; TESTS_ENVIRONMENT += export srcdir="$(srcdir)"; +TESTS_ENVIRONMENT += export top_builddir="$(top_builddir)"; TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; TESTS_ENVIRONMENT += export TESTMPIRUN="$(TESTMPIRUN)"; -TESTS_ENVIRONMENT += export PNETCDF_DEBUG="$(PNETCDF_DEBUG)"; -TESTS_ENVIRONMENT += export TESTPROGRAMS="$(TESTPROGRAMS)"; TESTS_ENVIRONMENT += export check_PROGRAMS="$(check_PROGRAMS)"; -TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER="$(ENABLE_BURST_BUFFER)"; TESTS_ENVIRONMENT += export BAD_FILES="$(BAD_FILES)"; -TESTS = cdf_type dim_cdf12 seq_runs.sh xfail_runs.sh +TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER=@ENABLE_BURST_BUFFER@; +TESTS_ENVIRONMENT += export ENABLE_THREAD_SAFE=@ENABLE_THREAD_SAFE@; +TESTS_ENVIRONMENT += export MIMIC_LUSTRE=@MIMIC_LUSTRE@; + +# During "make check", programs set in TESTS will be run by +# seq_runs.sh individually. Each produces its own log file. +TESTS = $(check_PROGRAMS) xfail_runs.sh TEST_EXTENSIONS = .sh -LOG_COMPILER = $(srcdir)/wrap_runs.sh +LOG_COMPILER = $(srcdir)/seq_runs.sh SH_LOG_COMPILER = BAD_FILES = bad_begin.nc5 \ @@ -57,20 +61,18 @@ BAD_FILES = bad_begin.nc5 \ XFAIL_TESTS = xfail_runs.sh -CLEANFILES = $(TESTOUTDIR)/cdf_type.nc \ - $(TESTOUTDIR)/cdf_type.bb.nc \ - $(TESTOUTDIR)/dim_cdf12.nc \ - $(TESTOUTDIR)/dim_cdf12.bb.nc \ - core core.* *.gcda *.gcno *.gcov gmon.out +check_SCRIPTS = seq_runs.sh parallel_run.sh xfail_runs.sh + +CLEANFILES = core core.* *.gcda *.gcno *.gcov gmon.out -EXTRA_DIST = wrap_runs.sh seq_runs.sh xfail_runs.sh parallel_run.sh \ - test_cdf1.nc test_cdf2.nc test_cdf5.nc \ - test_netcdf4.nc $(BAD_FILES) +EXTRA_DIST = seq_runs.sh xfail_runs.sh parallel_run.sh \ + test_cdf.nc1 test_cdf.nc2 test_cdf.nc3 test_cdf.nc4 test_cdf.nc5 \ + $(BAD_FILES) ../common/libtestutils.la: set -e; cd ../common && $(MAKE) $(MFLAGS) tests -ptest ptests ptest4: $(TESTPROGRAMS) +ptest ptests ptest4: $(check_PROGRAMS) @echo "===========================================================" @echo " $(subdir): Parallel testing on 4 MPI processes" @echo "===========================================================" diff --git a/test/cdf_format/cdf_type.c b/test/cdf_format/cdf_type.c index 6d66e6aa3d..e7b664f8a8 100644 --- a/test/cdf_format/cdf_type.c +++ b/test/cdf_format/cdf_type.c @@ -23,14 +23,14 @@ #define CHECK_ERR { \ if (err != NC_NOERR) { \ nerrs++; \ - printf("Error at line %d in %s: (%s)\n", \ + fprintf(stderr,"Error at line %d in %s: (%s)\n", \ __LINE__,__FILE__,nc_strerror(err)); \ } \ } #define EXP_ERR(exp) { \ if (err != exp) { \ nerrs++; \ - printf("Error at line %d in %s: expecting %d but got %d\n", \ + fprintf(stderr,"Error at line %d in %s: expecting %d but got %d\n", \ __LINE__,__FILE__,exp, err); \ } \ } @@ -80,13 +80,13 @@ int test_attr_types(const char *filename, err = PutAttInt(ncid, NC_GLOBAL, name, xtype[i], 1, &attr); #ifdef TEST_NETCDF if (err != NC_EBADTYPE) { - printf("Error at line %d in %s: expect NC_EBADTYPE but got %d\n", + fprintf(stderr,"Error at line %d in %s: expect NC_EBADTYPE but got %d\n", __LINE__, __FILE__, err); nerrs++; } #else if (err != NC_ESTRICTCDF2) { - printf("Error at line %d in %s: expect NC_ESTRICTCDF2 but got %s\n", + fprintf(stderr,"Error at line %d in %s: expect NC_ESTRICTCDF2 but got %s\n", __LINE__, __FILE__,NC_ERR_CODE_NAME(err)); nerrs++; } @@ -101,8 +101,8 @@ int test_attr_types(const char *filename, /*----< test_var_types() >----------------------------------------------------*/ static -int test_var_types(char *filename, - int format) +int test_var_types(const char *filename, + int format) { int i, err, rank, ncid, cmode, nerrs=0; int dimid, varid[5]; @@ -120,13 +120,13 @@ int test_var_types(char *filename, err = DefVar(ncid, name, xtype[i], 1, &dimid, &varid[i]); #ifdef TEST_NETCDF if (err != NC_EBADTYPE) { - printf("Error at line %d in %s: expect NC_EBADTYPE but got %d\n", + fprintf(stderr,"Error at line %d in %s: expect NC_EBADTYPE but got %d\n", __LINE__, __FILE__, err); nerrs++; } #else if (err != NC_ESTRICTCDF2) { - printf("Error at line %d in %s: expect NC_ESTRICTCDF2 but got %s\n", + fprintf(stderr,"Error at line %d in %s: expect NC_ESTRICTCDF2 but got %s\n", __LINE__, __FILE__,NC_ERR_CODE_NAME(err)); nerrs++; } @@ -137,6 +137,57 @@ int test_var_types(char *filename, return nerrs; } +/*----< test_io() >----------------------------------------------------------*/ +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, /* ignored */ + int coll_io, /* ignored */ + MPI_Info info) +{ + int nerrs; + + nerrs = test_attr_types(out_path, 0); /* CDF-1 */ + if (nerrs > 0) return nerrs; + + nerrs = test_attr_types(out_path, NC_64BIT_OFFSET); /* CDF-2 */ + if (nerrs > 0) return nerrs; + + nerrs = test_var_types(out_path, 0); /* CDF-1 */ + if (nerrs > 0) return nerrs; + + nerrs = test_var_types(out_path, NC_64BIT_OFFSET); /* CDF-2 */ + if (nerrs > 0) return nerrs; + + return 0; +} + +#ifndef TEST_NETCDF +/*----< main() >--------------------------------------------------------------*/ +int main(int argc, char **argv) { + + int err, formats[] = {0}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 0; /* skip test intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "CDF-5 dtype in CDF-1 and 2", opt, test_io); + + MPI_Finalize(); + + return err; +} +#else /*----< main() >--------------------------------------------------------------*/ int main(int argc, char **argv) { @@ -167,27 +218,10 @@ int main(int argc, char **argv) nerrs += test_var_types(filename, 0); nerrs += test_var_types(filename, NC_64BIT_OFFSET); -#ifdef TEST_NETCDF if (nerrs) printf("fail with %d mismatches\n",nerrs); else printf("pass\n"); -#else - MPI_Offset malloc_size, sum_size; - int err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } - - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } -#endif MPI_Finalize(); return (nerrs > 0); } - +#endif diff --git a/test/cdf_format/dim_cdf12.c b/test/cdf_format/dim_cdf12.c index cc8be2599c..561c71dc7a 100644 --- a/test/cdf_format/dim_cdf12.c +++ b/test/cdf_format/dim_cdf12.c @@ -4,7 +4,6 @@ * See COPYRIGHT notice in top-level directory. * *********************************************************************/ -/* $Id$ */ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * @@ -59,64 +58,49 @@ #include #include -int main(int argc, char** argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, /* ignored */ + int coll_io, /* ignored */ + MPI_Info info) { - char filename[256]; int rank, nprocs, err, nerrs=0; int ncid, cmode, varid, dimid[3]; - MPI_Info info=MPI_INFO_NULL; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); /* Note this test program must use the 512-byte alignment setting */ - MPI_Info_create(&info); + /* use the 512-byte fixed-size variable starting file offset alignment, * which is also the header extent align size */ MPI_Info_set(info, "nc_var_align_size", "512"); - /* get command-line arguments */ - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for defining dim in CDF-1/2 format ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - /* create a new CDF-1 file ----------------------------------------------*/ cmode = NC_CLOBBER; /* max dimension size for CDF-1 file is NC_MAX_INT */ - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, cmode, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "Y", (MPI_Offset)1+NC_MAX_INT, &dimid[0]); EXP_ERR(NC_EDIMSIZE) err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR err = ncmpi_close(ncid); CHECK_ERR - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); CHECK_ERR + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_NOWRITE, info, &ncid); CHECK_ERR err = ncmpi_close(ncid); CHECK_ERR /* use the max dimension size to define a 1D variable */ - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, cmode, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR err = ncmpi_def_var(ncid, "var0", NC_CHAR, 1, dimid, &varid); CHECK_ERR err = ncmpi_close(ncid); CHECK_ERR - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); CHECK_ERR + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_NOWRITE, info, &ncid); CHECK_ERR err = ncmpi_close(ncid); CHECK_ERR /* use the max dimension size to define a 1D variable, followed by * another variable to make the file size > NC_MAX_INT */ - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, cmode, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR err = ncmpi_def_var(ncid, "var0", NC_CHAR, 1, &dimid[0], &varid); CHECK_ERR @@ -127,19 +111,19 @@ int main(int argc, char** argv) /* use the max dimension size - 1024 to define a 1D variable, followed * by another variable to make the file size < 2147483647 */ - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, cmode, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT-1024, &dimid[0]); CHECK_ERR err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR err = ncmpi_def_var(ncid, "var0", NC_CHAR, 1, &dimid[0], &varid); CHECK_ERR err = ncmpi_def_var(ncid, "var1", NC_INT, 1, &dimid[1], &varid); CHECK_ERR err = ncmpi_close(ncid); CHECK_ERR - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); CHECK_ERR + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_NOWRITE, info, &ncid); CHECK_ERR err = ncmpi_close(ncid); CHECK_ERR /* define the first variable of type short that makes the file size > * NC_MAX_INT. error should be reported in ncmpi_enddef() or * ncmpi_close() */ - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, cmode, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR err = ncmpi_def_var(ncid, "var0", NC_SHORT, 1, &dimid[0], &varid); CHECK_ERR @@ -148,7 +132,7 @@ int main(int argc, char** argv) EXP_ERR(NC_EVARSIZE) /* define two variables to make the file size just < 2147483647 */ - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, cmode, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT-512-8, &dimid[0]); CHECK_ERR err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR err = ncmpi_def_var(ncid, "var0", NC_CHAR, 1, &dimid[0], &varid); CHECK_ERR @@ -156,7 +140,7 @@ int main(int argc, char** argv) err = ncmpi_close(ncid); CHECK_ERR /* define two variables to make the file size just > NC_MAX_INT */ - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, cmode, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT/2+1, &dimid[0]); CHECK_ERR err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR err = ncmpi_def_var(ncid, "var0", NC_INT, 1, &dimid[0], &varid); CHECK_ERR @@ -168,37 +152,37 @@ int main(int argc, char** argv) cmode = NC_CLOBBER | NC_64BIT_OFFSET; /* max dimension size for CDF-2 file is NC_MAX_INT */ - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, cmode, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "Y", (MPI_Offset)1+NC_MAX_INT, &dimid[0]); EXP_ERR(NC_EDIMSIZE) err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR err = ncmpi_close(ncid); CHECK_ERR - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); CHECK_ERR + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_NOWRITE, info, &ncid); CHECK_ERR err = ncmpi_close(ncid); CHECK_ERR /* use the max dimension size to define a 1D variable */ - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, cmode, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR err = ncmpi_def_var(ncid, "var0", NC_CHAR, 1, dimid, &varid); CHECK_ERR err = ncmpi_close(ncid); CHECK_ERR - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); CHECK_ERR + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_NOWRITE, info, &ncid); CHECK_ERR err = ncmpi_close(ncid); CHECK_ERR /* use the max dimension size to define a 1D variable, followed by * another variable to make the file size > 2 * NC_MAX_INT */ - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, cmode, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR err = ncmpi_def_var(ncid, "var0", NC_CHAR, 1, &dimid[0], &varid); CHECK_ERR err = ncmpi_def_var(ncid, "var1", NC_INT, 1, &dimid[1], &varid); CHECK_ERR err = ncmpi_close(ncid); CHECK_ERR - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); CHECK_ERR + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_NOWRITE, info, &ncid); CHECK_ERR err = ncmpi_close(ncid); CHECK_ERR /* define the first variable of type short that makes the file size > * 4294967295. error should be reported in ncmpi_enddef() or * ncmpi_close() */ - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, cmode, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR err = ncmpi_def_var(ncid, "var0", NC_SHORT, 1, &dimid[0], &varid); CHECK_ERR @@ -207,7 +191,7 @@ int main(int argc, char** argv) EXP_ERR(NC_EVARSIZE) /* define 2 1D int variables of dimension size > NC_MAX_INT */ - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, cmode, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT, &dimid[0]); CHECK_ERR err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR err = ncmpi_def_var(ncid, "var0", NC_INT, 1, &dimid[0], &varid); CHECK_ERR @@ -215,7 +199,7 @@ int main(int argc, char** argv) err = ncmpi_close(ncid); EXP_ERR(NC_EVARSIZE) - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, cmode, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT/2+1, &dimid[0]); CHECK_ERR err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR err = ncmpi_def_var(ncid, "var0", NC_INT, 1, &dimid[0], &varid); CHECK_ERR @@ -226,7 +210,7 @@ int main(int argc, char** argv) /* No record variable can require more than 2^32 - 4 bytes of storage for * each record's worth of data, unless it is the last record variable. */ - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, cmode, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "Z", NC_UNLIMITED, &dimid[0]); CHECK_ERR err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT/64, &dimid[1]); CHECK_ERR err = ncmpi_def_dim(ncid, "X", 64, &dimid[2]); CHECK_ERR @@ -236,7 +220,7 @@ int main(int argc, char** argv) EXP_ERR(NC_EVARSIZE) /* test large record variable that is not defined last */ - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, cmode, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "Z", NC_UNLIMITED, &dimid[0]); CHECK_ERR err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT/64, &dimid[1]); CHECK_ERR err = ncmpi_def_dim(ncid, "X", 64, &dimid[2]); CHECK_ERR @@ -245,38 +229,44 @@ int main(int argc, char** argv) err = ncmpi_close(ncid); EXP_ERR(NC_EVARSIZE) - /* Note for developers: keep the last test that produces no error, so the - * output file can be tested by ncvalidator in wrap_runs.sh - */ - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, cmode, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "Y", NC_MAX_INT/2, &dimid[0]); CHECK_ERR err = ncmpi_def_dim(ncid, "X", 2, &dimid[1]); CHECK_ERR err = ncmpi_def_var(ncid, "var0", NC_INT, 1, &dimid[0], &varid); CHECK_ERR err = ncmpi_def_var(ncid, "var1", NC_INT, 1, &dimid[1], &varid); CHECK_ERR err = ncmpi_close(ncid); CHECK_ERR - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); CHECK_ERR + + /* Note for developers: add ncmpi_open() at the end is to validate the file + * created above. + */ + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_NOWRITE, info, &ncid); CHECK_ERR err = ncmpi_close(ncid); CHECK_ERR - MPI_Info_free(&info); - - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } - - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + return nerrs; +} + +/*----< main() >--------------------------------------------------------------*/ +int main(int argc, char **argv) { + + int err, formats[] = {0}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 0; /* skip test intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ + + err = tst_main(argc, argv, "defining dim in CDF-1/2 format", opt, test_io); MPI_Finalize(); - return (nerrs > 0); + + return err; } diff --git a/test/cdf_format/parallel_run.sh b/test/cdf_format/parallel_run.sh index 9f95d0813d..36fa59fb55 100755 --- a/test/cdf_format/parallel_run.sh +++ b/test/cdf_format/parallel_run.sh @@ -1,78 +1,49 @@ #!/bin/bash # -# Copyright (C) 2018, Northwestern University and Argonne National Laboratory +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory # See COPYRIGHT notice in top-level directory. # # Exit immediately if a command exits with a non-zero status. set -e -VALIDATOR=../../src/utils/ncvalidator/ncvalidator -NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff +DRY_RUN=no +VERBOSE=no -# remove file system type prefix if there is any -OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $MPIRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $MPIRUN $@ + fi +} MPIRUN=`echo ${TESTMPIRUN} | ${SED} -e "s/NP/$1/g"` # echo "MPIRUN = ${MPIRUN}" # echo "check_PROGRAMS=${check_PROGRAMS}" -# echo "srcdir = ${srcdir}" - -# echo "PNETCDF_DEBUG = ${PNETCDF_DEBUG}" -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" -else - safe_modes="0" -fi # prevent user environment setting of PNETCDF_HINTS to interfere unset PNETCDF_HINTS -for j in ${safe_modes} ; do -for intra_aggr in 0 1 ; do - if test "$j" = 1 ; then # test only in safe mode - export PNETCDF_HINTS="romio_no_indep_rw=true" - else - export PNETCDF_HINTS= - fi - if test "$intra_aggr" = 1 ; then - export PNETCDF_HINTS="${PNETCDF_HINTS};nc_num_aggrs_per_node=2" - fi - export PNETCDF_SAFE_MODE=$j - # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - ${MPIRUN} ./test_inq_format ${srcdir} - ${MPIRUN} ./cdf_type ${TESTOUTDIR}/cdf_type.nc - ${MPIRUN} ./dim_cdf12 ${TESTOUTDIR}/dim_cdf12.nc - - # echo "--- validating file ${TESTOUTDIR}/cdf_type.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/cdf_type.nc - # echo "--- validating file ${TESTOUTDIR}/dim_cdf12.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/dim_cdf12.nc +if test "x$MIMIC_LUSTRE" != x1 ; then + PNETCDF_HINTS="cb_nodes=2" +fi - if test "x${ENABLE_BURST_BUFFER}" = x1 ; then - # echo "---- test burst buffering feature" - saved_PNETCDF_HINTS=${PNETCDF_HINTS} - export PNETCDF_HINTS="${PNETCDF_HINTS};nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" - ${MPIRUN} ./test_inq_format ${srcdir} - ${MPIRUN} ./cdf_type ${TESTOUTDIR}/cdf_type.bb.nc - ${MPIRUN} ./dim_cdf12 ${TESTOUTDIR}/dim_cdf12.bb.nc - export PNETCDF_HINTS=${saved_PNETCDF_HINTS} +for i in ${check_PROGRAMS} ; do - # echo "--- validating file ${TESTOUTDIR}/cdf_type.bb.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/cdf_type.bb.nc - # echo "--- validating file ${TESTOUTDIR}/dim_cdf12.bb.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/dim_cdf12.bb.nc + exe_name=`basename $i` - # echo "--- ncmpidiff cdf_type.nc cdf_type.bb.nc ---" - ${MPIRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/cdf_type.nc ${TESTOUTDIR}/cdf_type.bb.nc - # echo "--- ncmpidiff dim_cdf12.nc dim_cdf12.bb.nc ---" - # ${MPIRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/dim_cdf12.nc ${TESTOUTDIR}/dim_cdf12.bb.nc - fi -done -done + if test "x$exe_name" = "xtest_inq_format" ; then + run_cmd ./$i -q -i ${srcdir} + elif test "x$exe_name" = "xtst_corrupt" ; then + run_cmd ./$i ${srcdir} + elif test "x$exe_name" = "xtst_open_cdf5" ; then + run_cmd ./$i ${srcdir}/bad_begin.nc5 + else + run_cmd ./$i -q -o ${TESTOUTDIR}/${exe_name}.nc + fi -rm -f ${OUTDIR}/dim_cdf12.nc -rm -f ${OUTDIR}/cdf_type.nc -rm -f ${OUTDIR}/dim_cdf12.bb.nc -rm -f ${OUTDIR}/cdf_type.bb.nc +done # check_PROGRAMS diff --git a/test/cdf_format/seq_runs.sh b/test/cdf_format/seq_runs.sh index 732f08d2d3..cfdf158376 100755 --- a/test/cdf_format/seq_runs.sh +++ b/test/cdf_format/seq_runs.sh @@ -1,32 +1,39 @@ -#!/bin/sh +#!/bin/bash # -# Copyright (C) 2003, Northwestern University and Argonne National Laboratory +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory # See COPYRIGHT notice in top-level directory. # # Exit immediately if a command exits with a non-zero status. set -e -# prevent user environment setting of PNETCDF_HINTS to interfere -unset PNETCDF_HINTS - -${TESTSEQRUN} ./test_inq_format ${srcdir} - -# the followings check files with corrupted header -${TESTSEQRUN} ./tst_open_cdf5 ${srcdir}/bad_begin.nc5 -${TESTSEQRUN} ./tst_corrupt ${srcdir} +DRY_RUN=no +VERBOSE=no -# echo "" +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $TESTSEQRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $TESTSEQRUN $@ + fi +} -if test "x${ENABLE_BURST_BUFFER}" = x1 ; then - echo "" - echo "---- testing burst buffering" +exe_name=`basename $1` - export PNETCDF_HINTS="nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" - - ${TESTSEQRUN} ./test_inq_format ${srcdir} +# prevent user environment setting of PNETCDF_HINTS to interfere +unset PNETCDF_HINTS - # the followings check files with corrupted header - ${TESTSEQRUN} ./tst_open_cdf5 ${srcdir}/bad_begin.nc5 - ${TESTSEQRUN} ./tst_corrupt ${srcdir} +if test "x$exe_name" = xtest_inq_format ; then + run_cmd ./$1 -q -i ${srcdir} +elif test "x$exe_name" = xtst_open_cdf5 ; then + # check files with corrupted header + run_cmd ./$1 ${srcdir}/bad_begin.nc5 +elif test "x$exe_name" = xtst_corrupt ; then + # check files with corrupted header + run_cmd ./$1 ${srcdir} +else + run_cmd ./$1 -q -o ${TESTOUTDIR}/${exe_name}.nc fi + diff --git a/test/cdf_format/test_cdf1.nc b/test/cdf_format/test_cdf.nc1 similarity index 100% rename from test/cdf_format/test_cdf1.nc rename to test/cdf_format/test_cdf.nc1 diff --git a/test/cdf_format/test_cdf2.nc b/test/cdf_format/test_cdf.nc2 similarity index 100% rename from test/cdf_format/test_cdf2.nc rename to test/cdf_format/test_cdf.nc2 diff --git a/test/cdf_format/test_netcdf4.nc b/test/cdf_format/test_cdf.nc3 similarity index 100% rename from test/cdf_format/test_netcdf4.nc rename to test/cdf_format/test_cdf.nc3 diff --git a/test/cdf_format/test_cdf.nc4 b/test/cdf_format/test_cdf.nc4 new file mode 100644 index 0000000000000000000000000000000000000000..c3a19dba0855a5e3ea860497b696dc56bda92795 GIT binary patch literal 263 zcmeD5aB<`1lHy|G;9!7(|4_ir2oW)WN@$+An&a=`62!!FhDix5#*D57rWHnWF)@G* z5tU(J0jgzYVt~n@(>$6WvzQrpfDouOJ}=ogzPO|)Gr5F80O&N3ZUzPg1|}d0GLgp> zL^Ct+F>rvD`#A>`vqGuMX{(BMtP)Jo+ literal 0 HcmV?d00001 diff --git a/test/cdf_format/test_cdf5.nc b/test/cdf_format/test_cdf.nc5 similarity index 100% rename from test/cdf_format/test_cdf5.nc rename to test/cdf_format/test_cdf.nc5 diff --git a/test/cdf_format/test_inq_format.c b/test/cdf_format/test_inq_format.c index e6ec67adf5..937c147277 100644 --- a/test/cdf_format/test_inq_format.c +++ b/test/cdf_format/test_inq_format.c @@ -10,151 +10,89 @@ #include #include #include /* basename() */ +#include /* getopt() */ + #include #include #include -int main(int argc, char **argv) { - char dir_name[256], filename[512]; - int err, rank, nerrs=0, format, ncid; - - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); +static +int test_io(const char *out_path, /* ignored */ + const char *in_path, + int format, + int coll_io, + MPI_Info info) +{ + char filename[512]; + int err, nerrs=0, fmt, ncid; - if (argc > 2) { - if (!rank) printf("Usage: %s dir_name\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(dir_name, 256, "%s", argv[1]); - else strcpy(dir_name, "."); - MPI_Bcast(dir_name, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for inquiring file formats ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); - free(cmd_str); - } + sprintf(filename,"%s/test_cdf.nc%d",in_path, format); + // printf("%s at %d: input filename %s\n",basename(__FILE__),__LINE__,filename); - /* test CDF-1 -----------------------------------------------------------*/ - sprintf(filename,"%s/test_cdf1.nc",dir_name); err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, - &ncid); CHECK_ERR - - /* test NULL argument */ - err = ncmpi_inq_format(ncid, NULL); CHECK_ERR - - err = ncmpi_inq_format(ncid, &format); CHECK_ERR - if (format != NC_FORMAT_CLASSIC) { - printf("Error at line %d in %s: expecting CDF-1 format for file %s but got %d\n", - __LINE__,__FILE__,filename,format); - nerrs++; + &ncid); + if (format == NC_FORMAT_NETCDF4 && PNETCDF_DRIVER_NETCDF4 == 0) { + EXP_ERR(NC_ENOTBUILT) + return 0; } - err = ncmpi_close(ncid); CHECK_ERR - - /* test NULL argument */ - err = ncmpi_inq_file_format(filename, NULL); CHECK_ERR - - err = ncmpi_inq_file_format(filename, &format); CHECK_ERR - if (format != NC_FORMAT_CLASSIC) { - printf("Error at line %d in %s: expecting CDF-1 format for file %s but got %d\n", - __LINE__,__FILE__,filename,format); - nerrs++; + else { + CHECK_ERR + if (err != NC_NOERR) { + int rank; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + fprintf(stderr,"Error in %s at %d: rank %d failed to open file %s\n", + basename(__FILE__), __LINE__, rank, filename); + return 1; + } } - /* test CDF-2 -----------------------------------------------------------*/ - sprintf(filename,"%s/test_cdf2.nc",dir_name); - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, - &ncid); CHECK_ERR - /* test NULL argument */ err = ncmpi_inq_format(ncid, NULL); CHECK_ERR - err = ncmpi_inq_format(ncid, &format); CHECK_ERR - if (format != NC_FORMAT_CDF2) { - printf("Error at line %d in %s: expecting CDF-2 format for file %s but got %d\n", - __LINE__,__FILE__,filename,format); - nerrs++; - } - err = ncmpi_close(ncid); CHECK_ERR + err = ncmpi_inq_format(ncid, &fmt); CHECK_ERR - err = ncmpi_inq_file_format(filename, &format); CHECK_ERR - if (format != NC_FORMAT_CDF2) { - printf("Error at line %d in %s: expecting CDF-2 format for file %s but got %d\n", - __LINE__,__FILE__,filename,format); + if (fmt != format) { + fprintf(stderr,"Error in %s at %d: expect CDF-%d format for file %s but got %d\n", + __FILE__,__LINE__,format,filename,fmt); nerrs++; } - - /* test CDF-5 -----------------------------------------------------------*/ - sprintf(filename,"%s/test_cdf5.nc",dir_name); - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, - &ncid); CHECK_ERR + err = ncmpi_close(ncid); CHECK_ERR /* test NULL argument */ - err = ncmpi_inq_format(ncid, NULL); CHECK_ERR + err = ncmpi_inq_file_format(filename, NULL); CHECK_ERR - err = ncmpi_inq_format(ncid, &format); CHECK_ERR - if (format != NC_FORMAT_CDF5) { - printf("Error at line %d in %s: expecting CDF-5 format for file %s but got %d\n", - __LINE__,__FILE__,filename,format); - nerrs++; - } - err = ncmpi_close(ncid); CHECK_ERR + err = ncmpi_inq_file_format(filename, &fmt); CHECK_ERR - err = ncmpi_inq_file_format(filename, &format); CHECK_ERR - if (format != NC_FORMAT_CDF5) { - printf("Error at line %d in %s: expecting CDF-5 format for file %s but got %d\n", - __LINE__,__FILE__,filename,format); + if (fmt != format) { + fprintf(stderr,"Error in %s at %d: expect CDF-%d format for file %s but got %d\n", + __FILE__,__LINE__,format,filename,fmt); nerrs++; } - /* test NetCDF4 --------------------------------------------------------*/ - sprintf(filename,"%s/test_netcdf4.nc",dir_name); - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); - if (PNETCDF_DRIVER_NETCDF4 == 0) - EXP_ERR(NC_ENOTBUILT) - else { /* NetCDF-4 is enabled */ - CHECK_ERR - - /* test NULL argument */ - err = ncmpi_inq_format(ncid, NULL); CHECK_ERR + return nerrs; +} - err = ncmpi_inq_format(ncid, &format); CHECK_ERR - if (format != NC_FORMAT_NETCDF4) { - printf("Error at line %d in %s: expecting NetCDF-4 format for file %s but got %d\n", - __LINE__,__FILE__,filename,format); - nerrs++; - } - err = ncmpi_close(ncid); CHECK_ERR +int main(int argc, char **argv) { - /* test NULL argument */ - err = ncmpi_inq_file_format(filename, NULL); CHECK_ERR + int err; + loop_opts opt; - err = ncmpi_inq_file_format(filename, &format); CHECK_ERR - if (format != NC_FORMAT_NETCDF4) { - printf("Error at line %d in %s: expecting NETCDF4 format for file %s but got %d\n", - __LINE__,__FILE__,filename,format); - nerrs++; - } - } + MPI_Init(&argc, &argv); - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 0; /* skip test intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = false;/* skip ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + err = tst_main(argc, argv, "inquiring file formats", opt, test_io); MPI_Finalize(); - return (nerrs > 0); + + return err; } + diff --git a/test/cdf_format/tst_corrupt.c b/test/cdf_format/tst_corrupt.c index ce13d916de..89102b8dc7 100644 --- a/test/cdf_format/tst_corrupt.c +++ b/test/cdf_format/tst_corrupt.c @@ -32,14 +32,14 @@ #define CHECK_ERR { \ if (err != NC_NOERR) { \ nerrs++; \ - printf("Error at line %d in %s: (%s)\n", \ + fprintf(stderr,"Error at line %d in %s: (%s)\n", \ __LINE__,__FILE__,nc_strerror(err)); \ } \ } #define EXP_ERR(exp) { \ if (err != exp) { \ nerrs++; \ - printf("Error at line %d in %s: expecting %d but got %d\n", \ + fprintf(stderr,"Error at line %d in %s: expecting %d but got %d\n", \ __LINE__,__FILE__,exp, err); \ } \ } @@ -63,8 +63,12 @@ int main(int argc, char** argv) { char filename[1024], dirname[512]; int i, rank, err, ncid, nerrs=0; + double timing; MPI_Init(&argc, &argv); + + timing = MPI_Wtime(); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (argc != 2) { @@ -78,9 +82,9 @@ int main(int argc, char** argv) { if (rank == 0) { char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); sprintf(cmd_str, - "*** TESTING C %s for checking corrupted file header ", + "*** TESTING C %s - check corrupted file header", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); + printf("%-63s -- ", cmd_str); fflush(stdout); free(cmd_str); } @@ -165,10 +169,12 @@ int main(int argc, char** argv) { if (malloc_size > 0) ncmpi_inq_malloc_list(); } + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf(PASS_STR, timing); } #endif diff --git a/test/cdf_format/tst_open_cdf5.c b/test/cdf_format/tst_open_cdf5.c index ce02a5b797..eca3eaa086 100644 --- a/test/cdf_format/tst_open_cdf5.c +++ b/test/cdf_format/tst_open_cdf5.c @@ -54,14 +54,18 @@ int main(int argc, char** argv) { char filename[256]; int nerrs=0, rank, nprocs, err, ncid; + double timing; MPI_Init(&argc, &argv); + + timing = MPI_Wtime(); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); if (argc > 2) { if (!rank) printf("Usage: %s [filename]\n",argv[0]); - goto fn_exit; + goto err_out; } if (argc == 2) snprintf(filename, 256, "%s", argv[1]); else strcpy(filename, FILE_NAME); @@ -69,9 +73,9 @@ int main(int argc, char** argv) { if (rank == 0) { char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); sprintf(cmd_str, - "*** TESTING C %s for checking begins in corrupted header", + "*** TESTING C %s - begins in corrupted header", basename(argv[0])); - printf("%-66s --- ", cmd_str); fflush(stdout); + printf("%-63s -- ", cmd_str); fflush(stdout); free(cmd_str); } @@ -93,11 +97,13 @@ int main(int argc, char** argv) { if (malloc_size > 0) ncmpi_inq_malloc_list(); } -fn_exit: +err_out: + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf(PASS_STR, timing); } MPI_Finalize(); diff --git a/test/cdf_format/wrap_runs.sh b/test/cdf_format/wrap_runs.sh deleted file mode 100755 index c749c7dbe7..0000000000 --- a/test/cdf_format/wrap_runs.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2003, Northwestern University and Argonne National Laboratory -# See COPYRIGHT notice in top-level directory. -# - -# Exit immediately if a command exits with a non-zero status. -set -e - -VALIDATOR=../../src/utils/ncvalidator/ncvalidator -NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff - -outfile=`basename $1` - -# remove file system type prefix if there is any -OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` - -# echo "PNETCDF_DEBUG = ${PNETCDF_DEBUG}" -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" -else - safe_modes="0" -fi - -# prevent user environment setting of PNETCDF_HINTS to interfere -unset PNETCDF_HINTS - -for j in ${safe_modes} ; do - export PNETCDF_SAFE_MODE=$j - # echo "---- set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - ${TESTSEQRUN} $1 ${TESTOUTDIR}/$outfile.nc - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$outfile.nc - - if test "x${ENABLE_BURST_BUFFER}" = x1 ; then - echo "" - echo "---- testing burst buffering" - - export PNETCDF_HINTS="nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" - ${TESTSEQRUN} $1 ${TESTOUTDIR}/$outfile.bb.nc - unset PNETCDF_HINTS - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$outfile.bb.nc - - # running ncmpidiff on dim_cdf12 on one process requires more than 2 GB - # memory. Use more processes to check. Disable the check for now. - if test "$1" != "./dim_cdf12" ; then - # echo "--- ncmpidiff $outfile.nc $outfile.bb.nc ---" - ${TESTSEQRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/$outfile.nc ${TESTOUTDIR}/$outfile.bb.nc - fi - fi -done -rm -f ${OUTDIR}/$outfile.nc -rm -f ${OUTDIR}/$outfile.bb.nc diff --git a/test/cdf_format/xfail_runs.sh b/test/cdf_format/xfail_runs.sh index 7fddf05284..aaee1d3379 100755 --- a/test/cdf_format/xfail_runs.sh +++ b/test/cdf_format/xfail_runs.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # Copyright (C) 2003, Northwestern University and Argonne National Laboratory # See COPYRIGHT notice in top-level directory. diff --git a/test/cdl/Makefile.am b/test/cdl/Makefile.am index 493d5ccd21..0cc559bec8 100644 --- a/test/cdl/Makefile.am +++ b/test/cdl/Makefile.am @@ -11,45 +11,47 @@ AM_DEFAULT_SOURCE_EXT = .c AM_CPPFLAGS = -I$(top_srcdir)/src/include AM_CPPFLAGS += -I$(srcdir)/../common AM_CPPFLAGS += -I$(top_builddir)/src/include +AM_CPPFLAGS += -I$(top_srcdir)/src/utils/ncmpidiff -LDADD = $(top_builddir)/src/libs/libpnetcdf.la ../common/libtestutils.la +LDADD = ../common/libtestutils.la +LDADD += $(top_builddir)/src/libs/libpnetcdf.la LDADD += @NETCDF4_LDFLAGS@ @ADIOS_LDFLAGS@ @NETCDF4_LIBS@ @ADIOS_LIBS@ if DECL_MPI_OFFSET AM_CPPFLAGS += -DHAVE_DECL_MPI_OFFSET endif -TESTPROGRAMS = tst_cdl_hdr_parser - -check_PROGRAMS = $(TESTPROGRAMS) +check_PROGRAMS = tst_cdl_hdr_parser # autimake 1.11.3 has not yet implemented AM_TESTS_ENVIRONMENT # For newer versions, we can use AM_TESTS_ENVIRONMENT instead TESTS_ENVIRONMENT = export SED="$(SED)"; TESTS_ENVIRONMENT += export srcdir="$(srcdir)"; +TESTS_ENVIRONMENT += export top_builddir="$(top_builddir)"; TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; TESTS_ENVIRONMENT += export TESTMPIRUN="$(TESTMPIRUN)"; -TESTS_ENVIRONMENT += export PNETCDF_DEBUG="$(PNETCDF_DEBUG)"; -TESTS_ENVIRONMENT += export TESTPROGRAMS="$(TESTPROGRAMS)"; TESTS_ENVIRONMENT += export check_PROGRAMS="$(check_PROGRAMS)"; -TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER="$(ENABLE_BURST_BUFFER)"; -TESTS_ENVIRONMENT += export ENABLE_NETCDF4="$(ENABLE_NETCDF4)"; -TESTS = $(TESTPROGRAMS) +TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER=@ENABLE_BURST_BUFFER@; +TESTS_ENVIRONMENT += export ENABLE_THREAD_SAFE=@ENABLE_THREAD_SAFE@; +TESTS_ENVIRONMENT += export MIMIC_LUSTRE=@MIMIC_LUSTRE@; + +TESTS = $(check_PROGRAMS) TEST_EXTENSIONS = .sh -LOG_COMPILER = $(srcdir)/wrap_runs.sh +LOG_COMPILER = $(srcdir)/seq_runs.sh SH_LOG_COMPILER = -EXTRA_DIST = wrap_runs.sh parallel_run.sh cdl_header.txt +check_SCRIPTS = seq_runs.sh parallel_run.sh + +EXTRA_DIST = seq_runs.sh parallel_run.sh cdl_header.txt -CLEANFILES = $(TESTOUTDIR)/tst_cdl_hdr_parser.nc \ - core core.* *.gcda *.gcno *.gcov gmon.out +CLEANFILES = core core.* *.gcda *.gcno *.gcov gmon.out ../common/libtestutils.la: set -e; cd ../common && $(MAKE) $(MFLAGS) tests -ptest ptests ptest4: $(TESTPROGRAMS) +ptest ptests ptest4: $(check_PROGRAMS) @echo "===========================================================" @echo " $(subdir): Parallel testing on 4 MPI processes" @echo "===========================================================" diff --git a/test/cdl/parallel_run.sh b/test/cdl/parallel_run.sh index 4920c61be5..f76139f04f 100755 --- a/test/cdl/parallel_run.sh +++ b/test/cdl/parallel_run.sh @@ -1,50 +1,46 @@ #!/bin/bash # -# Copyright (C) 2025, Northwestern University and Argonne National Laboratory +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory # See COPYRIGHT notice in top-level directory. # # Exit immediately if a command exits with a non-zero status. set -e -VALIDATOR=../../src/utils/ncvalidator/ncvalidator -NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff +DRY_RUN=no +VERBOSE=no -# remove file system type prefix if there is any -OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $MPIRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $MPIRUN $@ + fi +} MPIRUN=`echo ${TESTMPIRUN} | ${SED} -e "s/NP/$1/g"` # echo "MPIRUN = ${MPIRUN}" # echo "check_PROGRAMS=${check_PROGRAMS}" -# echo "PNETCDF_DEBUG = ${PNETCDF_DEBUG}" -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" -else - safe_modes="0" -fi - # prevent user environment setting of PNETCDF_HINTS to interfere unset PNETCDF_HINTS +if test "x$MIMIC_LUSTRE" != x1 ; then + PNETCDF_HINTS="cb_nodes=2" +fi + for i in ${check_PROGRAMS} ; do - CMD_OPTS=${TESTOUTDIR}/$i.nc - if test $i = "tst_cdl_hdr_parser" ; then - CMD_OPTS="-q -o ${TESTOUTDIR}/$i.nc ${srcdir}/cdl_header.txt" + exe_name=`basename $i` + + if test "x$i" = "xtst_cdl_hdr_parser" ; then + run_cmd ./$i -q -o ${TESTOUTDIR}/${exe_name}.nc -i ${srcdir}/cdl_header.txt + continue fi - for j in ${safe_modes} ; do - if test "$j" = 1 ; then # test only in safe mode - export PNETCDF_HINTS="romio_no_indep_rw=true" - else - export PNETCDF_HINTS= - fi - export PNETCDF_SAFE_MODE=$j - # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - ${MPIRUN} ./$i ${CMD_OPTS} - - done - rm -f ${OUTDIR}/$i.nc -done + run_cmd ./$i -q -o ${TESTOUTDIR}/${exe_name}.nc + +done # check_PROGRAMS diff --git a/test/cdl/seq_runs.sh b/test/cdl/seq_runs.sh new file mode 100755 index 0000000000..e8a5a55c2d --- /dev/null +++ b/test/cdl/seq_runs.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory +# See COPYRIGHT notice in top-level directory. +# + +# Exit immediately if a command exits with a non-zero status. +set -e + +DRY_RUN=no +VERBOSE=no + +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $TESTSEQRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $TESTSEQRUN $@ + fi +} + +exe_name=`basename $1` + +# prevent user environment setting of PNETCDF_HINTS to interfere +unset PNETCDF_HINTS + +if test "x$1" = "x./tst_cdl_hdr_parser" ; then + run_cmd ./$1 -q -o ${TESTOUTDIR}/${exe_name}.nc -i ${srcdir}/cdl_header.txt +else + run_cmd ./$1 -q -o ${TESTOUTDIR}/${exe_name}.nc +fi + diff --git a/test/cdl/tst_cdl_hdr_parser.c b/test/cdl/tst_cdl_hdr_parser.c index e2ef5d2c25..6a6deb6828 100644 --- a/test/cdl/tst_cdl_hdr_parser.c +++ b/test/cdl/tst_cdl_hdr_parser.c @@ -27,81 +27,40 @@ #include -/*----< usage() >------------------------------------------------------------*/ -static void usage (char *argv0) { - char *help = "Usage: %s [OPTION] FILE\n\ - [-h] Print this help message\n\ - [-v] Verbose mode\n\ - [-o path] Output netCDF file path\n\ - FILE: Input CDL file path\n"; - fprintf (stderr, help, argv0); -} - -int main(int argc, char **argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) { - extern int optind; - char *outfile, *infile; - int i, j, rank, err=0, nerrs=0, hid, verbose; + char *name; + int i, j, rank, err=0, nerrs=0, hid, ncid, verbose; + int ndims, *dimids, nvars, nattrs; + void *value; + nc_type xtype; + MPI_Offset size, nelems; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); - verbose = 1; - infile = NULL; - outfile = NULL; - - /* get command-line arguments */ - while ((i = getopt(argc, argv, "hqo:")) != EOF) - switch(i) { - case 'q': verbose = 0; - break; - case 'o': outfile = strdup(optarg); - break; - case 'h': - default: if (rank==0) usage(argv[0]); - MPI_Finalize(); - return 1; - } - if (outfile == NULL) outfile = strdup("testfile.nc"); - - if (argv[optind] == NULL) { - if (rank == 0) usage(argv[0]); - MPI_Finalize(); - return 1; - } - - infile = strdup(argv[optind]); + verbose = 0; - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for CDL header parser ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - - err = cdl_hdr_open(infile, &hid); + err = cdl_hdr_open(in_path, &hid); if (err != NC_NOERR) exit(1); if (verbose) printf("==================================================\n"); /* create a new netcdf file */ - int ncid, cmode, format; - err = cdl_hdr_inq_format(hid, &format); CHECK_ERR if (verbose) printf("CDF file format: CDF-%d\n", format); - cmode = NC_CLOBBER; - if (format == 2) cmode |= NC_64BIT_OFFSET; - else if (format == 5) cmode |= NC_64BIT_DATA; - err = ncmpi_create(MPI_COMM_WORLD, outfile, cmode, MPI_INFO_NULL, &ncid); + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); CHECK_ERR - char *name; - int ndims, *dimids, nvars, nattrs; - void *value; - nc_type xtype; - MPI_Offset size, nelems; + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR /* define dimensions */ err = cdl_hdr_inq_ndims(hid, &ndims); CHECK_ERR @@ -173,27 +132,30 @@ int main(int argc, char **argv) err = cdl_hdr_close(hid); CHECK_ERR - if (infile != NULL) free(infile); - if (outfile != NULL) free(outfile); - - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + return nerrs; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +int main(int argc, char **argv) { + + int err; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ + + err = tst_main(argc, argv, "CDL header parser", opt, test_io); MPI_Finalize(); - return (nerrs > 0); + + return err; } diff --git a/test/cdl/wrap_runs.sh b/test/cdl/wrap_runs.sh deleted file mode 100755 index 98de428706..0000000000 --- a/test/cdl/wrap_runs.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2025, Northwestern University and Argonne National Laboratory -# See COPYRIGHT notice in top-level directory. -# - -# Exit immediately if a command exits with a non-zero status. -set -e - -VALIDATOR=../../src/utils/ncvalidator/ncvalidator -NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff - -outfile=`basename $1` - -CMD_OPTS= -if test $outfile = "tst_cdl_hdr_parser" ; then - CMD_OPTS="-q -o ${TESTOUTDIR}/$outfile.nc ${srcdir}/cdl_header.txt" -fi - -# remove file system type prefix if there is any -OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` - -# echo "PNETCDF_DEBUG = ${PNETCDF_DEBUG}" -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" -else - safe_modes="0" -fi - -# prevent user environment setting of PNETCDF_HINTS to interfere -unset PNETCDF_HINTS - -for j in ${safe_modes} ; do - export PNETCDF_SAFE_MODE=$j - # echo "---- set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - ${TESTSEQRUN} $1 $CMD_OPTS - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$outfile.nc - -done -rm -f ${OUTDIR}/$outfile.nc - diff --git a/test/common/Makefile.am b/test/common/Makefile.am index fc3abb0ea4..f5a11210a9 100644 --- a/test/common/Makefile.am +++ b/test/common/Makefile.am @@ -11,9 +11,15 @@ SUFFIXES = .a .o .c .F90 .h AM_DEFAULT_SOURCE_EXT = .c AM_CPPFLAGS = -I$(top_srcdir)/src/include AM_CPPFLAGS += -I$(top_builddir)/src/include +AM_CPPFLAGS += -I$(top_srcdir)/src/utils/ncmpidiff + +LDADD = ${top_builddir}/src/libs/libpnetcdf.la +LDADD += @NETCDF4_LDFLAGS@ @ADIOS_LDFLAGS@ @NETCDF4_LIBS@ @ADIOS_LIBS@ -lm check_LTLIBRARIES = libtestutils.la +libtestutils_la_LIBADD = $(top_builddir)/src/utils/ncmpidiff/libncmpidiff_core.la + libtestutils_la_SOURCES = testutils.c testutils.h CLEANFILES = core.* *.gcda *.gcno *.gcov gmon.out @@ -23,12 +29,13 @@ CLEANFILES = core.* *.gcda *.gcno *.gcov gmon.out if HAS_FORTRAN check_LTLIBRARIES += libtestutilsf.la libtestutilsf_la_SOURCES = testutilsf.F90 -libtestutils_la_LIBADD = libtestutilsf.la +libtestutils_la_LIBADD += libtestutilsf.la + CLEANFILES += testutilsf.mod endif AM_FFLAGS = -AM_FCFLAGS = +AM_FCFLAGS = ${FC_MODINC}$(top_builddir)/src/binding/f90 $(FFREEFORMFLAG) if DECL_GET_ENVIRONMENT_VARIABLE AM_FCFLAGS += $(FC_DEFINE)HAS_GET_ENVIRONMENT_VARIABLE @@ -52,6 +59,9 @@ endif # $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ # $(AM_FCFLAGS) $(FCFLAGS) +$(top_builddir)/src/utils/ncmpidiff/libncmpidiff_core.la: + set -e; cd $(top_builddir)/src/utils/ncmpidiff && $(MAKE) $(MFLAGS) + # build check targets but not invoke tests-local: all $(check_LTLIBRARIES) diff --git a/test/common/testutils.c b/test/common/testutils.c index 3e37aa84bd..d2a7dc7420 100644 --- a/test/common/testutils.c +++ b/test/common/testutils.c @@ -6,12 +6,25 @@ #include +#include /* basename() */ #include #include /* strchr(), strerror(), strdup(), strcpy(), strlen() */ +#include /* getopt(), stat() */ +#include /* stat() */ +#include /* stat() */ + #include #include #include "testutils.h" +#include + +#if PNETCDF_DRIVER_NETCDF4 == 1 +int nc_formats[5] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_NETCDF4, + NC_FORMAT_NETCDF4_CLASSIC, NC_FORMAT_64BIT_DATA}; +#else +int nc_formats[3] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; +#endif char* nc_err_code_name(int err) { @@ -26,148 +39,150 @@ char* nc_err_code_name(int err) return unknown_str; } +#define ERR_CODE_STR(err) case (err): return #err; + switch (err) { - case (NC_NOERR): return "NC_NOERR"; - case (NC_EBADID): return "NC_EBADID"; - case (NC_ENFILE): return "NC_ENFILE"; - case (NC_EEXIST): return "NC_EEXIST"; - case (NC_EINVAL): return "NC_EINVAL"; - case (NC_EPERM): return "NC_EPERM"; - case (NC_ENOTINDEFINE): return "NC_ENOTINDEFINE"; - case (NC_EINDEFINE): return "NC_EINDEFINE"; - case (NC_EINVALCOORDS): return "NC_EINVALCOORDS"; - case (NC_EMAXDIMS): return "NC_EMAXDIMS"; - case (NC_ENAMEINUSE): return "NC_ENAMEINUSE"; - case (NC_ENOTATT): return "NC_ENOTATT"; - case (NC_EMAXATTS): return "NC_EMAXATTS"; - case (NC_EBADTYPE): return "NC_EBADTYPE"; - case (NC_EBADDIM): return "NC_EBADDIM"; - case (NC_EUNLIMPOS): return "NC_EUNLIMPOS"; - case (NC_EMAXVARS): return "NC_EMAXVARS"; - case (NC_ENOTVAR): return "NC_ENOTVAR"; - case (NC_EGLOBAL): return "NC_EGLOBAL"; - case (NC_ENOTNC): return "NC_ENOTNC"; - case (NC_ESTS): return "NC_ESTS"; - case (NC_EMAXNAME): return "NC_EMAXNAME"; - case (NC_EUNLIMIT): return "NC_EUNLIMIT"; - case (NC_ENORECVARS): return "NC_ENORECVARS"; - case (NC_ECHAR): return "NC_ECHAR"; - case (NC_EEDGE): return "NC_EEDGE"; - case (NC_ESTRIDE): return "NC_ESTRIDE"; - case (NC_EBADNAME): return "NC_EBADNAME"; - case (NC_ERANGE): return "NC_ERANGE"; - case (NC_ENOMEM): return "NC_ENOMEM"; - case (NC_EVARSIZE): return "NC_EVARSIZE"; - case (NC_EDIMSIZE): return "NC_EDIMSIZE"; - case (NC_ETRUNC): return "NC_ETRUNC"; - case (NC_EAXISTYPE): return "NC_EAXISTYPE"; - case (NC_EDAP): return "NC_EDAP"; - case (NC_ECURL): return "NC_ECURL"; - case (NC_EIO): return "NC_EIO"; - case (NC_ENODATA): return "NC_ENODATA"; - case (NC_EDAPSVC): return "NC_EDAPSVC"; - case (NC_EDAS): return "NC_EDAS"; - case (NC_EDDS): return "NC_EDDS"; - case (NC_EDATADDS): return "NC_EDATADDS"; - case (NC_EDAPURL): return "NC_EDAPURL"; - case (NC_EDAPCONSTRAINT): return "NC_EDAPCONSTRAINT"; - case (NC_ETRANSLATION): return "NC_ETRANSLATION"; - case (NC_EACCESS): return "NC_EACCESS"; - case (NC_EAUTH): return "NC_EAUTH"; - case (NC_ENOTFOUND): return "NC_ENOTFOUND"; - case (NC_ECANTREMOVE): return "NC_ECANTREMOVE"; - case (NC_EHDFERR): return "NC_EHDFERR"; - case (NC_ECANTREAD): return "NC_ECANTREAD"; - case (NC_ECANTWRITE): return "NC_ECANTWRITE"; - case (NC_ECANTCREATE): return "NC_ECANTCREATE"; - case (NC_EFILEMETA): return "NC_EFILEMETA"; - case (NC_EDIMMETA): return "NC_EDIMMETA"; - case (NC_EATTMETA): return "NC_EATTMETA"; - case (NC_EVARMETA): return "NC_EVARMETA"; - case (NC_ENOCOMPOUND): return "NC_ENOCOMPOUND"; - case (NC_EATTEXISTS): return "NC_EATTEXISTS"; - case (NC_ENOTNC4): return "NC_ENOTNC4"; - case (NC_ESTRICTNC3): return "NC_ESTRICTNC3"; - case (NC_ENOTNC3): return "NC_ENOTNC3"; - case (NC_ENOPAR): return "NC_ENOPAR"; - case (NC_EPARINIT): return "NC_EPARINIT"; - case (NC_EBADGRPID): return "NC_EBADGRPID"; - case (NC_EBADTYPID): return "NC_EBADTYPID"; - case (NC_ETYPDEFINED): return "NC_ETYPDEFINED"; - case (NC_EBADFIELD): return "NC_EBADFIELD"; - case (NC_EBADCLASS): return "NC_EBADCLASS"; - case (NC_EMAPTYPE): return "NC_EMAPTYPE"; - case (NC_ELATEFILL): return "NC_ELATEFILL"; - case (NC_ELATEDEF): return "NC_ELATEDEF"; - case (NC_EDIMSCALE): return "NC_EDIMSCALE"; - case (NC_ENOGRP): return "NC_ENOGRP"; - case (NC_ESTORAGE): return "NC_ESTORAGE"; - case (NC_EBADCHUNK): return "NC_EBADCHUNK"; - case (NC_ENOTBUILT): return "NC_ENOTBUILT"; - case (NC_EDISKLESS): return "NC_EDISKLESS"; - case (NC_ECANTEXTEND): return "NC_ECANTEXTEND"; - case (NC_EMPI): return "NC_EMPI"; - // case (NC_EURL): return "NC_EURL"; - // case (NC_ECONSTRAINT): return "NC_ECONSTRAINT"; - case (NC_ESMALL): return "NC_ESMALL"; - case (NC_ENOTINDEP): return "NC_ENOTINDEP"; - case (NC_EINDEP): return "NC_EINDEP"; - case (NC_EFILE): return "NC_EFILE"; - case (NC_EREAD): return "NC_EREAD"; - case (NC_EWRITE): return "NC_EWRITE"; - case (NC_EOFILE): return "NC_EOFILE"; - case (NC_EMULTITYPES): return "NC_EMULTITYPES"; - case (NC_EIOMISMATCH): return "NC_EIOMISMATCH"; - case (NC_ENEGATIVECNT): return "NC_ENEGATIVECNT"; - case (NC_EUNSPTETYPE): return "NC_EUNSPTETYPE"; - case (NC_EINVAL_REQUEST): return "NC_EINVAL_REQUEST"; - case (NC_EAINT_TOO_SMALL): return "NC_EAINT_TOO_SMALL"; - case (NC_ENOTSUPPORT): return "NC_ENOTSUPPORT"; - case (NC_ENULLBUF): return "NC_ENULLBUF"; - case (NC_EPREVATTACHBUF): return "NC_EPREVATTACHBUF"; - case (NC_ENULLABUF): return "NC_ENULLABUF"; - case (NC_EPENDINGBPUT): return "NC_EPENDINGBPUT"; - case (NC_EINSUFFBUF): return "NC_EINSUFFBUF"; - case (NC_ENOENT): return "NC_ENOENT"; - case (NC_EINTOVERFLOW): return "NC_EINTOVERFLOW"; - case (NC_ENOTENABLED): return "NC_ENOTENABLED"; - case (NC_EBAD_FILE): return "NC_EBAD_FILE"; - case (NC_ENO_SPACE): return "NC_ENO_SPACE"; - case (NC_EQUOTA): return "NC_EQUOTA"; - case (NC_ENULLSTART): return "NC_ENULLSTART"; - case (NC_ENULLCOUNT): return "NC_ENULLCOUNT"; - case (NC_EINVAL_CMODE): return "NC_EINVAL_CMODE"; - case (NC_EINVAL_OMODE): return "NC_EINVAL_OMODE"; - case (NC_ETYPESIZE): return "NC_ETYPESIZE"; - case (NC_ETYPE_MISMATCH): return "NC_ETYPE_MISMATCH"; - case (NC_ETYPESIZE_MISMATCH): return "NC_ETYPESIZE_MISMATCH"; - case (NC_ESTRICTCDF2): return "NC_ESTRICTCDF2"; - case (NC_ENOTRECVAR): return "NC_ENOTRECVAR"; - case (NC_ENOTFILL): return "NC_ENOTFILL"; - case (NC_EMULTIDEFINE): return "NC_EMULTIDEFINE"; - case (NC_EMULTIDEFINE_OMODE): return "NC_EMULTIDEFINE_OMODE"; - case (NC_EMULTIDEFINE_CMODE): return "NC_EMULTIDEFINE_CMODE"; - case (NC_EMULTIDEFINE_DIM_NUM): return "NC_EMULTIDEFINE_DIM_NUM"; - case (NC_EMULTIDEFINE_DIM_SIZE): return "NC_EMULTIDEFINE_DIM_SIZE"; - case (NC_EMULTIDEFINE_DIM_NAME): return "NC_EMULTIDEFINE_DIM_NAME"; - case (NC_EMULTIDEFINE_VAR_NUM): return "NC_EMULTIDEFINE_VAR_NUM"; - case (NC_EMULTIDEFINE_VAR_NAME): return "NC_EMULTIDEFINE_VAR_NAME"; - case (NC_EMULTIDEFINE_VAR_NDIMS): return "NC_EMULTIDEFINE_VAR_NDIMS"; - case (NC_EMULTIDEFINE_VAR_DIMIDS): return "NC_EMULTIDEFINE_VAR_DIMIDS"; - case (NC_EMULTIDEFINE_VAR_TYPE): return "NC_EMULTIDEFINE_VAR_TYPE"; - case (NC_EMULTIDEFINE_VAR_LEN): return "NC_EMULTIDEFINE_VAR_LEN"; - case (NC_EMULTIDEFINE_NUMRECS): return "NC_EMULTIDEFINE_NUMRECS"; - case (NC_EMULTIDEFINE_VAR_BEGIN): return "NC_EMULTIDEFINE_VAR_BEGIN"; - case (NC_EMULTIDEFINE_ATTR_NUM): return "NC_EMULTIDEFINE_ATTR_NUM"; - case (NC_EMULTIDEFINE_ATTR_SIZE): return "NC_EMULTIDEFINE_ATTR_SIZE"; - case (NC_EMULTIDEFINE_ATTR_NAME): return "NC_EMULTIDEFINE_ATTR_NAME"; - case (NC_EMULTIDEFINE_ATTR_TYPE): return "NC_EMULTIDEFINE_ATTR_TYPE"; - case (NC_EMULTIDEFINE_ATTR_LEN): return "NC_EMULTIDEFINE_ATTR_LEN"; - case (NC_EMULTIDEFINE_ATTR_VAL): return "NC_EMULTIDEFINE_ATTR_VAL"; - case (NC_EMULTIDEFINE_FNC_ARGS): return "NC_EMULTIDEFINE_FNC_ARGS"; - case (NC_EMULTIDEFINE_FILL_MODE): return "NC_EMULTIDEFINE_FILL_MODE"; - case (NC_EMULTIDEFINE_VAR_FILL_MODE): return "NC_EMULTIDEFINE_VAR_FILL_MODE"; - case (NC_EMULTIDEFINE_VAR_FILL_VALUE): return "NC_EMULTIDEFINE_VAR_FILL_VALUE"; + ERR_CODE_STR(NC_NOERR) + ERR_CODE_STR(NC_EBADID) + ERR_CODE_STR(NC_ENFILE) + ERR_CODE_STR(NC_EEXIST) + ERR_CODE_STR(NC_EINVAL) + ERR_CODE_STR(NC_EPERM) + ERR_CODE_STR(NC_ENOTINDEFINE) + ERR_CODE_STR(NC_EINDEFINE) + ERR_CODE_STR(NC_EINVALCOORDS) + ERR_CODE_STR(NC_EMAXDIMS) + ERR_CODE_STR(NC_ENAMEINUSE) + ERR_CODE_STR(NC_ENOTATT) + ERR_CODE_STR(NC_EMAXATTS) + ERR_CODE_STR(NC_EBADTYPE) + ERR_CODE_STR(NC_EBADDIM) + ERR_CODE_STR(NC_EUNLIMPOS) + ERR_CODE_STR(NC_EMAXVARS) + ERR_CODE_STR(NC_ENOTVAR) + ERR_CODE_STR(NC_EGLOBAL) + ERR_CODE_STR(NC_ENOTNC) + ERR_CODE_STR(NC_ESTS) + ERR_CODE_STR(NC_EMAXNAME) + ERR_CODE_STR(NC_EUNLIMIT) + ERR_CODE_STR(NC_ENORECVARS) + ERR_CODE_STR(NC_ECHAR) + ERR_CODE_STR(NC_EEDGE) + ERR_CODE_STR(NC_ESTRIDE) + ERR_CODE_STR(NC_EBADNAME) + ERR_CODE_STR(NC_ERANGE) + ERR_CODE_STR(NC_ENOMEM) + ERR_CODE_STR(NC_EVARSIZE) + ERR_CODE_STR(NC_EDIMSIZE) + ERR_CODE_STR(NC_ETRUNC) + ERR_CODE_STR(NC_EAXISTYPE) + ERR_CODE_STR(NC_EDAP) + ERR_CODE_STR(NC_ECURL) + ERR_CODE_STR(NC_EIO) + ERR_CODE_STR(NC_ENODATA) + ERR_CODE_STR(NC_EDAPSVC) + ERR_CODE_STR(NC_EDAS) + ERR_CODE_STR(NC_EDDS) + ERR_CODE_STR(NC_EDATADDS) + ERR_CODE_STR(NC_EDAPURL) + ERR_CODE_STR(NC_EDAPCONSTRAINT) + ERR_CODE_STR(NC_ETRANSLATION) + ERR_CODE_STR(NC_EACCESS) + ERR_CODE_STR(NC_EAUTH) + ERR_CODE_STR(NC_ENOTFOUND) + ERR_CODE_STR(NC_ECANTREMOVE) + ERR_CODE_STR(NC_EHDFERR) + ERR_CODE_STR(NC_ECANTREAD) + ERR_CODE_STR(NC_ECANTWRITE) + ERR_CODE_STR(NC_ECANTCREATE) + ERR_CODE_STR(NC_EFILEMETA) + ERR_CODE_STR(NC_EDIMMETA) + ERR_CODE_STR(NC_EATTMETA) + ERR_CODE_STR(NC_EVARMETA) + ERR_CODE_STR(NC_ENOCOMPOUND) + ERR_CODE_STR(NC_EATTEXISTS) + ERR_CODE_STR(NC_ENOTNC4) + ERR_CODE_STR(NC_ESTRICTNC3) + ERR_CODE_STR(NC_ENOTNC3) + ERR_CODE_STR(NC_ENOPAR) + ERR_CODE_STR(NC_EPARINIT) + ERR_CODE_STR(NC_EBADGRPID) + ERR_CODE_STR(NC_EBADTYPID) + ERR_CODE_STR(NC_ETYPDEFINED) + ERR_CODE_STR(NC_EBADFIELD) + ERR_CODE_STR(NC_EBADCLASS) + ERR_CODE_STR(NC_EMAPTYPE) + ERR_CODE_STR(NC_ELATEFILL) + ERR_CODE_STR(NC_ELATEDEF) + ERR_CODE_STR(NC_EDIMSCALE) + ERR_CODE_STR(NC_ENOGRP) + ERR_CODE_STR(NC_ESTORAGE) + ERR_CODE_STR(NC_EBADCHUNK) + ERR_CODE_STR(NC_ENOTBUILT) + ERR_CODE_STR(NC_EDISKLESS) + ERR_CODE_STR(NC_ECANTEXTEND) + ERR_CODE_STR(NC_EMPI) + // ERR_CODE_STR(NC_EURL) + // ERR_CODE_STR(NC_ECONSTRAINT) + ERR_CODE_STR(NC_ESMALL) + ERR_CODE_STR(NC_ENOTINDEP) + ERR_CODE_STR(NC_EINDEP) + ERR_CODE_STR(NC_EFILE) + ERR_CODE_STR(NC_EREAD) + ERR_CODE_STR(NC_EWRITE) + ERR_CODE_STR(NC_EOFILE) + ERR_CODE_STR(NC_EMULTITYPES) + ERR_CODE_STR(NC_EIOMISMATCH) + ERR_CODE_STR(NC_ENEGATIVECNT) + ERR_CODE_STR(NC_EUNSPTETYPE) + ERR_CODE_STR(NC_EINVAL_REQUEST) + ERR_CODE_STR(NC_EAINT_TOO_SMALL) + ERR_CODE_STR(NC_ENOTSUPPORT) + ERR_CODE_STR(NC_ENULLBUF) + ERR_CODE_STR(NC_EPREVATTACHBUF) + ERR_CODE_STR(NC_ENULLABUF) + ERR_CODE_STR(NC_EPENDINGBPUT) + ERR_CODE_STR(NC_EINSUFFBUF) + ERR_CODE_STR(NC_ENOENT) + ERR_CODE_STR(NC_EINTOVERFLOW) + ERR_CODE_STR(NC_ENOTENABLED) + ERR_CODE_STR(NC_EBAD_FILE) + ERR_CODE_STR(NC_ENO_SPACE) + ERR_CODE_STR(NC_EQUOTA) + ERR_CODE_STR(NC_ENULLSTART) + ERR_CODE_STR(NC_ENULLCOUNT) + ERR_CODE_STR(NC_EINVAL_CMODE) + ERR_CODE_STR(NC_EINVAL_OMODE) + ERR_CODE_STR(NC_ETYPESIZE) + ERR_CODE_STR(NC_ETYPE_MISMATCH) + ERR_CODE_STR(NC_ETYPESIZE_MISMATCH) + ERR_CODE_STR(NC_ESTRICTCDF2) + ERR_CODE_STR(NC_ENOTRECVAR) + ERR_CODE_STR(NC_ENOTFILL) + ERR_CODE_STR(NC_EMULTIDEFINE) + ERR_CODE_STR(NC_EMULTIDEFINE_OMODE) + ERR_CODE_STR(NC_EMULTIDEFINE_CMODE) + ERR_CODE_STR(NC_EMULTIDEFINE_DIM_NUM) + ERR_CODE_STR(NC_EMULTIDEFINE_DIM_SIZE) + ERR_CODE_STR(NC_EMULTIDEFINE_DIM_NAME) + ERR_CODE_STR(NC_EMULTIDEFINE_VAR_NUM) + ERR_CODE_STR(NC_EMULTIDEFINE_VAR_NAME) + ERR_CODE_STR(NC_EMULTIDEFINE_VAR_NDIMS) + ERR_CODE_STR(NC_EMULTIDEFINE_VAR_DIMIDS) + ERR_CODE_STR(NC_EMULTIDEFINE_VAR_TYPE) + ERR_CODE_STR(NC_EMULTIDEFINE_VAR_LEN) + ERR_CODE_STR(NC_EMULTIDEFINE_NUMRECS) + ERR_CODE_STR(NC_EMULTIDEFINE_VAR_BEGIN) + ERR_CODE_STR(NC_EMULTIDEFINE_ATTR_NUM) + ERR_CODE_STR(NC_EMULTIDEFINE_ATTR_SIZE) + ERR_CODE_STR(NC_EMULTIDEFINE_ATTR_NAME) + ERR_CODE_STR(NC_EMULTIDEFINE_ATTR_TYPE) + ERR_CODE_STR(NC_EMULTIDEFINE_ATTR_LEN) + ERR_CODE_STR(NC_EMULTIDEFINE_ATTR_VAL) + ERR_CODE_STR(NC_EMULTIDEFINE_FNC_ARGS) + ERR_CODE_STR(NC_EMULTIDEFINE_FILL_MODE) + ERR_CODE_STR(NC_EMULTIDEFINE_VAR_FILL_MODE) + ERR_CODE_STR(NC_EMULTIDEFINE_VAR_FILL_VALUE) default: sprintf(unknown_str,"Unknown code %d",err); } @@ -285,3 +300,415 @@ char* remove_file_system_type_prefix(const char *filename) return ret_filename; } +int is_relax_coord_bound(void) +{ + char *env_str; + int relax_coord_bound; + +#if PNETCDF_RELAX_COORD_BOUND == 1 + relax_coord_bound = 1; +#else + relax_coord_bound = 0; +#endif + if ((env_str = getenv("PNETCDF_RELAX_COORD_BOUND")) != NULL) { + /* the env variable is set */ + if (*env_str == '0') relax_coord_bound = 0; + else relax_coord_bound = 1; + } + + return relax_coord_bound; +} + +void +static tst_main_usage(char *argv0) +{ + char *base_name = basename(argv0); + char *help = + "Usage: %s [OPTIONS]...[filename]\n" + " [-h] Print help\n" + " [-q] quiet mode\n" + " [-k] Keep output files (default: no)\n" + " [-i in_path]: input file path (default: NULL)\n" + " [-o out_path]: output netCDF file name (default: %s.nc)\n"; + fprintf(stderr, help, base_name, base_name); +} + +int tst_main(int argc, + char **argv, + char *msg, /* short description about the test */ + loop_opts opt, /* test options */ + int (*tst_body)(const char*,const char*,int,int,MPI_Info)) +{ + extern int optind; + extern char *optarg; + char *in_path=NULL, *out_path=NULL, *out_dir; + + /* IDs for the netCDF file, dimensions, and variables. */ + int nprocs, rank, err, nerrs=0, keep_files, quiet; + int i, a, d, b, s, m, u; + int s_ina, e_ina, s_drv, e_drv, s_bb, e_bb, s_mod, e_mod, s_ds, e_ds; + int s_ibuf, e_ibuf; + + MPI_Info info=MPI_INFO_NULL; + double timing = MPI_Wtime(); + +#if PNETCDF_PROFILING == 1 + double itiming[256]; int k=0; +#endif + + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + keep_files = 0; + quiet = 0; + + while ((i = getopt(argc, argv, "hqki:o:")) != EOF) + switch(i) { + case 'q': + quiet = 1; + break; + case 'k': + keep_files = 1; + break; + case 'i': + in_path = strdup(optarg); + break; + case 'o': + out_path = strdup(optarg); + break; + case 'h': + default: if (rank==0) tst_main_usage(argv[0]); + MPI_Finalize(); + exit(1); + } + +#ifndef TESTOUTDIR + out_dir = getenv("TESTOUTDIR"); + if (out_dir == NULL) out_dir = "."; +#else + out_dir = TESTOUTDIR; +#endif + + if (out_path == NULL) + out_path = strdup("testfile.nc"); +#if 0 + else { + /* check if filename is a directory */ + struct stat sb; + + snprintf(filename, 256, "%s", argv[optind]); + if (stat(filename, &sb) == 0 && S_ISDIR(sb.st_mode)) + append_suffix = 0; + } +#endif + + if (rank == 0) { + char *cmd_str = (char *)malloc(strlen(argv[0]) + 256); + sprintf(cmd_str, "*** TESTING C %s - %s", basename(argv[0]), msg); + printf("%-63s -- ", cmd_str); + free(cmd_str); + } + + char cmd_opts[64]; + sprintf(cmd_opts, "Rank %d: ncmpidiff", rank); + + char *ptr = strrchr(out_path, '.'); + if (ptr != NULL) *ptr = '\0'; + + MPI_Info_create(&info); + + /* Set common I/O hints. Using smaller values for hints below can make + * the tests more rigorous. + */ + MPI_Info_set(info, "ind_wr_buffer_size", "60"); + MPI_Info_set(info, "ind_rd_buffer_size", "70"); + MPI_Info_set(info, "cb_buffer_size", "500"); + +#define SET_OPT(key) { \ + if (opt.key == 2) { \ + s_ ## key = 1; e_##key = 0; \ + } \ + else if (opt.key == 1) { \ + s_##key = 1; e_##key = 1; \ + } \ + else { /* #key == 0 */ \ + s_##key = 0; e_##key = 0; \ + } \ +} + + SET_OPT(ina) /* test intra-node aggregation */ + SET_OPT(drv) /* test MPI-IO, GIO drivers */ + SET_OPT(ibuf) /* test hint nc_ibuf_size */ + SET_OPT(mod) /* test collective/independent data mode */ + SET_OPT(bb) /* test of burst-buffering feature */ + +#if PNETCDF_DRIVER_GIO == 0 + s_drv = e_drv = 1; /* skip testing GIO driver */ +#endif + +#if !defined(PNETCDF_BURST_BUFFERING) || PNETCDF_BURST_BUFFERING == 0 + s_bb = e_bb = 0; /* skip testing burst buffering */ +#endif + + s_ds = 1; e_ds = 0; /* test both date sieving enabled and disabled */ + + for (i=0; i 0) + sprintf(ext, "nc%d", opt.formats[i]); + else /* for tests that are not testing different CDF versions */ + strcpy(ext, "nc"); + + /* For indice a,d,b,s,m, 0 is PnetCDF's default setting */ + for (a=s_ina; a>=e_ina; a--) { + for (d=s_drv; d>=e_drv; d--) { + for (u=s_ibuf; u>=e_ibuf; u--) { + for (b=s_bb; b>=e_bb; b--) { + for (s=s_ds; s>=e_ds; s--) { + for (m=s_mod; m>=e_mod; m--) { + +#if PNETCDF_PROFILING == 1 + MPI_Barrier(MPI_COMM_WORLD); + itiming[k] = MPI_Wtime(); +#endif + + sprintf(out_filename, "%s.%s", out_path, ext); + + /* Whether or not to enable the intra-node aggregation */ + if (a == 0) { /* PnetCDF's default */ + MPI_Info_set(info, "nc_num_aggrs_per_node", "0"); + strcat(out_filename, ".noina"); + } else { + MPI_Info_set(info, "nc_num_aggrs_per_node", "2"); + strcat(out_filename, ".ina"); + } + + /* Use MPI-IO or GIO driver */ + if (d == 0) { /* PnetCDF's default */ + MPI_Info_set(info, "nc_driver", "gio"); + strcat(out_filename, ".gio"); + } else { + MPI_Info_set(info, "nc_driver", "mpiio"); + strcat(out_filename, ".mpio"); + } + + /* Set hint nc_ibuf_size */ + if (u == 0) { /* test nc_ibuf_size = 1 MB */ + /* PnetCDF's default PNC_DEFAULT_IBUF_SIZE 16777216 */ + MPI_Info_set(info, "nc_ibuf_size", "16777216"); + strcat(out_filename, ".ibuf"); + } else { /* do not use internal buffering */ + MPI_Info_set(info, "nc_ibuf_size", "0"); + strcat(out_filename, ".noibuf"); + } + + /* Whether or not to enable the burst buffering */ + if (b == 0) { /* PnetCDF's default */ + MPI_Info_set(info, "nc_burst_buf", "disable"); + strcat(out_filename, ".nobb"); + } + else { + MPI_Info_set(info, "nc_burst_buf", "enable"); + MPI_Info_set(info, "nc_burst_buf_dirname", out_dir); + MPI_Info_set(info, "nc_burst_buf_overwrite", "enable"); + strcat(out_filename, ".bb"); + } + + /* Whether or not to enable data sieving */ + if (s == 0) { /* PnetCDF's default */ + MPI_Info_set(info, "romio_ds_read", "automatic"); + MPI_Info_set(info, "romio_ds_write", "automatic"); + strcat(out_filename, ".ds"); + } + else { + MPI_Info_set(info, "romio_ds_read", "disable"); + MPI_Info_set(info, "romio_ds_write", "disable"); + strcat(out_filename, ".nods"); + } + + /* Test PnetCDF collective or independent APIs */ + if (m == 0) /* collective data mode */ + strcat(out_filename, ".coll_mod"); + else /* independent data mode */ + strcat(out_filename, ".indep_mod"); + + /* NetCDF4 does not allow to extend number of record numbers in + * independent data mode. NC_ECANTEXTEND will be returned. + */ + if (m == 1 && + (opt.formats[i] == NC_FORMAT_NETCDF4_CLASSIC || + opt.formats[i] == NC_FORMAT_NETCDF4)) + continue; + +#define RUN_ERR(msg, fname) {\ + printf("\n%s %-44s (INA=%s drv=%s ibuf=%s BB=%s DS=%s mode=%s)\n", \ + msg, fname, (a)?"yes":"no", (d)?"mpio":"gio", (u)?"no":"yes", \ + (b)?"yes":"no", (s)?"no":"auto", (m)?"indp":"coll"); \ +} + +#define DIFF_ERR(msg, f1, f2) {\ + printf("\n%s %-44s %s (INA=%s drv=%s ibuf=%s BB=%s DS=%s mode=%s)\n", \ + msg, f1, f2, (a)?"yes":"no", (d)?"mpio":"gio", (u)?"no":"yes", \ + (b)?"yes":"no", (s)?"no":"auto", (m)?"indp":"coll"); \ +} + + double time_body = MPI_Wtime(); + if (!quiet && rank == 0) + RUN_ERR("Testing", out_filename) + + /* tst_body() is the core of test program */ + int coll_io = (m == 0); + nerrs = tst_body(out_filename, in_path, opt.formats[i], coll_io, + info); + MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_MAX, + MPI_COMM_WORLD); + if (nerrs > 0) { + fflush(stdout); + if (rank == 0) + RUN_ERR("\nFAILED", out_filename) + goto err_out; + } + + if (!quiet) { + time_body = MPI_Wtime() - time_body; + MPI_Allreduce(MPI_IN_PLACE, &time_body, 1, MPI_DOUBLE, MPI_MAX, + MPI_COMM_WORLD); + if (rank == 0) + printf(" (%.2fs)\n", time_body); + } + +#if PNETCDF_PROFILING == 1 + itiming[k] = MPI_Wtime() - itiming[k]; k++; +#endif + /* wait for all processes to complete */ + MPI_Barrier(MPI_COMM_WORLD); + + /* run ncmpidiff to compare output files */ + if (base_file == NULL) { /* skip first file */ + base_file = strdup(out_filename); + goto skip_diff; + } + + if (!opt.hdr_diff) goto skip_diff; + + if (strcmp(base_file, out_filename)) { + int check_header=1, check_entire_file, first_diff=1; + + check_entire_file = (opt.var_diff) ? 1 : 0; + + /* ncmpidiff does not support netCDF4 files */ + if (opt.formats[i] == NC_FORMAT_NETCDF4_CLASSIC || + opt.formats[i] == NC_FORMAT_NETCDF4) { + goto skip_diff; + } + + if (!quiet && rank == 0) + DIFF_ERR("ncmpidiff", out_filename, base_file) + +#ifdef MIMIC_LUSTRE + /* use a larger stripe size when running ncmpidiff is faster */ + setenv("MIMIC_STRIPE_SIZE", "1048576", 1); +#else + unsetenv("MIMIC_STRIPE_SIZE"); +#endif + /* running ncmpidiff also validates the file header */ + MPI_Offset numDIFF; + numDIFF = ncmpidiff_core(out_filename, base_file, + MPI_COMM_WORLD, info, 0, + quiet, check_header, 0, + check_entire_file, 0, NULL, 0, + first_diff, cmd_opts, 0, 0); + + /* check error so it can error out collectively */ + MPI_Allreduce(MPI_IN_PLACE, &numDIFF, 1, MPI_OFFSET, MPI_MAX, + MPI_COMM_WORLD); + if (numDIFF > 0) { + if (rank == 0) + DIFF_ERR("FAILED: ncmpidiff", out_filename, base_file) + + nerrs = 1; + goto err_out; + } + + /* wait for all ranks to complete diff before file delete */ + MPI_Barrier(MPI_COMM_WORLD); + } +skip_diff: + if (!keep_files && base_file != NULL && strcmp(base_file, out_filename)) { + if (rank == 0) + ncmpi_delete(out_filename, MPI_INFO_NULL); + + /* wait for deletion to complete before next iteration */ + MPI_Barrier(MPI_COMM_WORLD); + } + } /* loop m */ + } /* loop s */ + } /* loop b */ + } /* loop u */ + } /* loop d */ + } /* loop a */ + + if (base_file != NULL) { + if (!keep_files) { + if (rank == 0) + ncmpi_delete(base_file, MPI_INFO_NULL); + + /* all ranks wait for root to complete file deletion */ + MPI_Barrier(MPI_COMM_WORLD); + } + free(base_file); + } + } + MPI_Info_free(&info); + + /* check if there is any malloc residue */ + MPI_Offset malloc_size, sum_size; + err = ncmpi_inq_malloc_size(&malloc_size); + if (err == NC_NOERR) { + MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); + if (rank == 0 && sum_size > 0) + printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", + sum_size); + if (malloc_size > 0) ncmpi_inq_malloc_list(); + } + +err_out: + if (in_path != NULL) free(in_path); + if (out_path != NULL) free(out_path); + + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); + MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + +#if PNETCDF_PROFILING == 1 + MPI_Allreduce(MPI_IN_PLACE, itiming, 256, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + if (rank == 0 && !quiet) { + for (i=0; i 0); +} + +/*----< pnc_fmt_string() >---------------------------------------------------*/ +char* pnc_fmt_string(int format) +{ + switch(format) { + case NC_FORMAT_CLASSIC: return "NC_FORMAT_CLASSIC"; + case NC_FORMAT_64BIT_OFFSET: return "NC_FORMAT_64BIT_OFFSET"; + case NC_FORMAT_NETCDF4: return "NC_FORMAT_NETCDF4"; + case NC_FORMAT_NETCDF4_CLASSIC: return "NC_FORMAT_NETCDF4_CLASSIC"; + case NC_FORMAT_64BIT_DATA: return "NC_FORMAT_64BIT_DATA"; + default: return "UNKNOWN"; + } +} diff --git a/test/common/testutils.h b/test/common/testutils.h index ef3d2593f8..38d6f8f9ec 100644 --- a/test/common/testutils.h +++ b/test/common/testutils.h @@ -15,32 +15,47 @@ #include #include #include +#include + +#if defined(HAVE_STDBOOL_H) && HAVE_STDBOOL_H == 1 +#include /* type false and true */ +typedef bool boolean; +#else +typedef int boolean; +#if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 202311L +enum {false=0, true=1}; +#endif +#endif #define MODE_COLL 1 #define MODE_INDEP 0 +#ifndef OFFFMT +#define OFFFMT "%lld" +#endif + #define CHECK_ERR { \ if (err != NC_NOERR) { \ - nerrs++; \ - printf("Error at line %d in %s: (%s)\n", \ + fprintf(stderr,"Error at line %d in %s: (%s)\n", \ __LINE__,__FILE__,ncmpi_strerrno(err)); \ + assert(0); \ } \ } #define CHECK_ERR_ALL { \ if (err != NC_NOERR) { \ nerrs++; \ - printf("Error at line %d in %s: (%s)\n", \ + fprintf(stderr,"Error at line %d in %s: (%s)\n", \ __LINE__,__FILE__,ncmpi_strerrno(err)); \ } \ MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); \ - if (nerrs > 0) goto fn_exit; \ + if (nerrs > 0) goto err_out; \ } #define CHECK_ERROUT { \ if (err != NC_NOERR) { \ nerrs++; \ - printf("Error at line %d in %s: (%s)\n", \ + fprintf(stderr,"Error at line %d in %s: (%s)\n", \ __LINE__,__FILE__,ncmpi_strerrno(err)); \ goto err_out; \ } \ @@ -49,16 +64,16 @@ #define CHECK_FATAL_ERR { \ if (err != NC_NOERR) { \ nerrs++; \ - printf("Error at line %d in %s: (%s)\n", \ + fprintf(stderr,"Error at line %d in %s: (%s)\n", \ __LINE__,__FILE__,ncmpi_strerrno(err)); \ - goto fn_exit; \ + assert(0); \ } \ } #define EXP_ERR(exp) { \ if (err != exp) { \ nerrs++; \ - printf("Error at line %d in %s: expecting %s but got %s\n", \ + fprintf(stderr,"Error at line %d in %s: expecting %s but got %s\n", \ __LINE__,__FILE__,ncmpi_strerrno(exp), ncmpi_strerrno(err)); \ } \ } @@ -66,26 +81,29 @@ #define CHECK_EXP_ERR_ALL(exp) { \ if (err != exp) { \ nerrs++; \ - printf("Error at line %d in %s: expecting %s but got %s\n", \ + fprintf(stderr,"Error at line %d in %s: expecting %s but got %s\n", \ __LINE__,__FILE__,ncmpi_strerrno(exp), ncmpi_strerrno(err)); \ } \ MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); \ - if (nerrs > 0) goto fn_exit; \ + if (nerrs > 0) goto err_out; \ } +#define CHECK_NERRS_ALL if (nerrs != 0) assert(0); +/* #define CHECK_NERRS_ALL { \ MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); \ - if (nerrs > 0) goto fn_exit; \ + if (nerrs > 0) assert(0) err_out; \ } +*/ int inq_env_hint(char *hint_key, char **hint_value); -#ifdef PNETCDF_DEBUG -#define PASS_STR "\x1b[32mpass\x1b[0m\n" +#if PNETCDF_DEBUG_MODE == 1 +#define PASS_STR "\x1b[32mpass\x1b[0m (%4.1fs)\n" #define SKIP_STR "\x1b[32mskip\x1b[0m\n" #define FAIL_STR "\x1b[31mfail\x1b[0m with %d mismatches\n" #else -#define PASS_STR "pass\n" +#define PASS_STR "pass (%4.1fs)\n" #define SKIP_STR "skip\n" #define FAIL_STR "fail with %d mismatches\n" #endif @@ -95,9 +113,20 @@ int inq_env_hint(char *hint_key, char **hint_value); char err_string[MPI_MAX_ERROR_STRING+1]; \ int err_len; \ MPI_Error_string(err, err_string, &err_len); \ - printf("MPI Error at file %s line %d (%s)\n",__FILE__,__LINE__,err_string); \ + fprintf(stderr,"MPI Error at file %s line %d (%s)\n",__FILE__,__LINE__,err_string); \ } +#ifndef PNETCDF_DRIVER_NETCDF4 +#error PNETCDF_DRIVER_NETCDF4 should be defined in pnetcdf.h +#endif +#if PNETCDF_DRIVER_NETCDF4 == 1 +extern int nc_formats[5]; +#else +extern int nc_formats[3]; +#endif + +extern char* pnc_fmt_string(int format); + extern char* nc_err_code_name(int err); #if MPI_VERSION < 3 @@ -114,6 +143,41 @@ extern char *strdup(const char *s); extern int strcasecmp(const char *s1, const char *s2); #endif +extern char* remove_file_system_type_prefix(const char *filename); +extern +int is_relax_coord_bound(void); + +extern +void tst_usage(char *argv0); + +typedef struct { + char *in_path; /* input file path for read tests */ + int num_fmts; /* number of file formats: CDF 1/2/3/4/5 */ + int *formats; /* [num_fmts] max are {NC_FORMAT_CLASSIC, + NC_FORMAT_64BIT_OFFSET, + NC_FORMAT_NETCDF4, + NC_FORMAT_NETCDF4_CLASSIC, + NC_FORMAT_64BIT_DATA}; */ + /* below 4 options: + * 2 for testing both non-default and defaulte settings. + * 1 for testing non-default setting only. + * 0 for testing defaulte setting only. + */ + int ina; /* test of intra-node aggregation */ + int drv; /* test of GIO and MPI-IO drivers */ + int ibuf; /* test of hint nc_ibuf_size */ + int bb; /* test of burst-buffering feature */ + int mod; /* test of independent data mode */ + + boolean hdr_diff; /* run ncmpidiff for file header */ + boolean var_diff; /* run ncmpidiff for variables + * (disabled automatically when hdr_diff == false) */ +} loop_opts; + +extern +int tst_main(int argc, char **argv, char *msg, loop_opts opt, + int (*tst_body)(const char*, const char*, int, int, MPI_Info)); + #endif diff --git a/test/common/testutilsf.F90 b/test/common/testutilsf.F90 index ca08e381cf..b88e0cd35e 100644 --- a/test/common/testutilsf.F90 +++ b/test/common/testutilsf.F90 @@ -6,19 +6,28 @@ ! ! $Id$ + subroutine fusage(cmd) + implicit none + character(len=*) cmd + + print*,'Usage: ',trim(cmd),' [OPTIONS]' + print*,' [-h] Print help' + print*,' [-q] quiet mode' + print*,' [-k] Keep output files (default: no)' + print*,' [-i in_path]: input file path (default: NULL)' + print*,' [-o out_path]: output netCDF file name (default: %s.nc)' + end subroutine fusage + ! This function gets the executable name and output file name from the ! command line. - integer function get_args(cmd, filename) -#ifdef NAGFOR - USE F90_UNIX_ENV, only : iargc, getarg + integer function get_args(cmd, out_path, in_path, keep_files) implicit none -#else - implicit none - integer iargc -#endif - integer argc, i - character(len=*) cmd, filename - character(len=256) full_cmd + character(len=*) cmd, out_path, in_path + character(len=256) :: full_cmd, arg + logical :: keep_files, skip_next + integer :: i, j, n_args + + keep_files = .false. get_args = 1 call getarg(0, full_cmd) @@ -31,44 +40,70 @@ integer function get_args(cmd, filename) cmd(:) = full_cmd(i+1:) endif - argc = IARGC() - if (argc .GT. 1) then - print*,'Usage: ',trim(cmd),' [filename]' - get_args = 0 - return - endif - if (argc .EQ. 1) call getarg(1, filename) + n_args = command_argument_count() + + skip_next = .false. + do j = 1, n_args + if (skip_next) then + skip_next = .false. + cycle + end if + + call get_command_argument(j, arg) + arg = trim(arg) ! Remove trailing spaces + + if (arg == "-k") then + keep_files = .true. + else if (arg == "-i") then + if (j < n_args) then + call get_command_argument(j+1, arg) + in_path = trim(arg) + skip_next = .true. + end if + else if (arg == "-o") then + if (j < n_args) then + call get_command_argument(j+1, arg) + out_path = trim(arg) + skip_next = .true. + end if + else if (arg == "-h") then + call fusage(cmd) + return + end if + end do + end function get_args ! This function prints the pass/fail message on screen - subroutine pass_fail(nerrs, msg) + subroutine pass_fail(nerrs, msg, timing) implicit none integer nerrs character(len=*) msg + double precision timing ! local variables CHARACTER ESC PARAMETER (ESC=char(27)) -#ifdef PNETCDF_DEBUG - CHARACTER (LEN=20) PASS_STR, FAIL_STR - PARAMETER (PASS_STR='------ '//ESC//'[32mpass'//ESC//'[0m') - PARAMETER (FAIL_STR='------ '//ESC//'[31mfail'//ESC//'[0m') -#else + ! screen output in color + ! CHARACTER (LEN=20) PASS_STR, FAIL_STR + ! PARAMETER (PASS_STR='-- '//ESC//'[32mpass'//ESC//'[0m (') + ! PARAMETER (FAIL_STR='-- '//ESC//'[31mfail'//ESC//'[0m') CHARACTER (LEN=11) PASS_STR, FAIL_STR - PARAMETER (PASS_STR='------ pass') - PARAMETER (FAIL_STR='------ fail') -#endif + PARAMETER (PASS_STR='-- pass (') + PARAMETER (FAIL_STR='-- fail') if (nerrs .EQ. 0) then - write(*,"(A67,A)") msg, PASS_STR + write(*,"(A64,A,F4.1,A)") msg, trim(PASS_STR), timing, 's)' else - write(*,"(A67,A)") msg, FAIL_STR + write(*,"(A64,A)") msg, trim(FAIL_STR) endif end subroutine pass_fail subroutine get_env(hint_str, value) + implicit none character(len=*) hint_str, value + #ifdef HAS_GET_ENVIRONMENT_VARIABLE call Get_Environment_Variable(hint_str, Value=value) #else @@ -76,3 +111,28 @@ subroutine get_env(hint_str, value) #endif end subroutine get_env + LOGICAL FUNCTION relax_coord_bound_f() + use pnetcdf + implicit none + + character(len=256) :: env_str, env_val + integer :: ierr + + if (PNETCDF_RELAX_COORD_BOUND == 1) then + relax_coord_bound_f = .TRUE. + else + relax_coord_bound_f = .FALSE. + endif + env_str = "PNETCDF_RELAX_COORD_BOUND" + call get_environment_variable(env_str, value=env_val, status=ierr) + + if (ierr == 0) THEN + ! Environment variable is set + if (env_val(1:1) == '1') then + relax_coord_bound_f = .TRUE. + else + relax_coord_bound_f = .FALSE. + endif + endif + END FUNCTION relax_coord_bound_f + diff --git a/test/fandc/Makefile.am b/test/fandc/Makefile.am index dc3f81432a..80c71381ea 100644 --- a/test/fandc/Makefile.am +++ b/test/fandc/Makefile.am @@ -16,7 +16,7 @@ AM_FFLAGS = -I$(top_builddir)/src/binding/f77 AM_FCFLAGS = $(FC_MODINC)$(top_builddir)/src/binding/f90 \ $(FC_MODINC)$(srcdir)/../common -LDADD = $(top_builddir)/src/libs/libpnetcdf.la ../common/libtestutils.la +LDADD = ../common/libtestutils.la $(top_builddir)/src/libs/libpnetcdf.la LDADD += @NETCDF4_LDFLAGS@ @ADIOS_LDFLAGS@ @NETCDF4_LIBS@ @ADIOS_LIBS@ -lm # suppress warning messages when using NAG Fortran compiler @@ -47,11 +47,11 @@ if DECL_MPI_OFFSET AM_FCFLAGS += $(FC_DEFINE)HAVE_DECL_MPI_OFFSET endif -TESTPROGRAMS = pnctest \ - csnap +check_PROGRAMS = pnctest \ + csnap if HAS_FORTRAN - TESTPROGRAMS += pnf_test pnctestf fixedform + check_PROGRAMS += pnf_test pnctestf fixedform pnf_test_SOURCES = pnf_test.f pnf_test_FFLAGS = $(FFIXEDFORMFLAG) $(AM_FFLAGS) pnctestf_SOURCES = pnctestf.f @@ -59,31 +59,32 @@ if HAS_FORTRAN fixedform_SOURCES = fixedform.f90 fixedform_FCFLAGS = $(FFIXEDFORMFLAG) $(AM_FCFLAGS) $(AM_FFLAGS) if HAVE_F77_SUPPORT_FREEFORM - TESTPROGRAMS += freeform + check_PROGRAMS += freeform freeform_SOURCES = freeform.f freeform_FFLAGS = $(FFREEFORMFLAG) $(AM_FFLAGS) endif endif -check_PROGRAMS = $(TESTPROGRAMS) - # autimake 1.11.3 has not yet implemented AM_TESTS_ENVIRONMENT # For newer versions, we can use AM_TESTS_ENVIRONMENT instead -# AM_TESTS_ENVIRONMENT = TESTPROGRAMS="$(TESTPROGRAMS)" ; export TESTPROGRAMS; +# AM_TESTS_ENVIRONMENT = check_PROGRAMS="$(check_PROGRAMS)" ; export check_PROGRAMS; # AM_TESTS_ENVIRONMENT += TESTSEQRUN="$(TESTSEQRUN)" ; export TESTSEQRUN; # AM_TESTS_ENVIRONMENT += TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)" ; export TESTOUTDIR; TESTS_ENVIRONMENT = export SED="$(SED)"; TESTS_ENVIRONMENT += export srcdir="$(srcdir)"; +TESTS_ENVIRONMENT += export top_builddir="$(top_builddir)"; TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; TESTS_ENVIRONMENT += export TESTMPIRUN="$(TESTMPIRUN)"; -TESTS_ENVIRONMENT += export PNETCDF_DEBUG="$(PNETCDF_DEBUG)"; -TESTS_ENVIRONMENT += export TESTPROGRAMS="$(TESTPROGRAMS)"; TESTS_ENVIRONMENT += export check_PROGRAMS="$(check_PROGRAMS)"; +TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER=@ENABLE_BURST_BUFFER@; +TESTS_ENVIRONMENT += export ENABLE_THREAD_SAFE=@ENABLE_THREAD_SAFE@; +TESTS_ENVIRONMENT += export MIMIC_LUSTRE=@MIMIC_LUSTRE@; + # programs in this folder are just for testing compile error/warning messages, # not for running -# TESTS = $(TESTPROGRAMS) +# TESTS = $(check_PROGRAMS) EXTRA_DIST = README diff --git a/test/fandc/csnap.c b/test/fandc/csnap.c index 2443086e5e..6528503b7b 100644 --- a/test/fandc/csnap.c +++ b/test/fandc/csnap.c @@ -64,7 +64,7 @@ int pe_coords[3]; /* Cartesian PE coords */ /*** function prototypes ***/ -void find_locnx(MPI_Offset nx, int mype, int totpes, MPI_Offset *locnx, MPI_Offset *xbegin); +void find_locnx(MPI_Offset nx, int rank, int nprocs, MPI_Offset *locnx, MPI_Offset *xbegin); void write_file(char *filename, double *t); void read_file(char *filename, double *t); void get_fields(double *tt, double *smf); @@ -79,8 +79,12 @@ int main(int argc, char *argv[]) { double rates_l[4], rates_g[4]; int i, rank; char filename[256]; + double timing; + + MPI_Init(&argc, &argv); + + timing = MPI_Wtime(); - MPI_Init(&argc,&argv); MPI_Comm_size(MPI_COMM_WORLD,&totpes); MPI_Comm_rank(MPI_COMM_WORLD,&rank); @@ -96,11 +100,12 @@ int main(int argc, char *argv[]) { if (rank == 0) { char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); sprintf(cmd_str, "*** TESTING C %s for 3D array write/read ", argv[0]); - printf("%-66s ------ ", cmd_str); fflush(stdout); + printf("%-64s -- ", cmd_str); fflush(stdout); free(cmd_str); } + if (filename[0] == '\0') { - printf(PASS_STR); + printf(FAIL_STR, 1); fprintf(stderr,"Error: invalid filename, Exiting ...\n"); MPI_Finalize(); return 1; @@ -195,10 +200,12 @@ int main(int argc, char *argv[]) { sum_size); } + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf(PASS_STR, timing); } MPI_Finalize(); @@ -390,14 +397,14 @@ void read_file(char *filename, double *t) { } -void find_locnx(MPI_Offset nx, int mype, int totpes, MPI_Offset *locnx, MPI_Offset *xbegin) { +void find_locnx(MPI_Offset nx, int rank, int nprocs, MPI_Offset *locnx, MPI_Offset *xbegin) { MPI_Offset xremain; - *locnx = nx / totpes; - xremain = nx - totpes*(*locnx); - if (mype < xremain) (*locnx)++; - *xbegin = mype*(nx/totpes) + xremain; - if (mype < xremain) *xbegin += mype - xremain; + *locnx = nx / nprocs; + xremain = nx - nprocs*(*locnx); + if (rank < xremain) (*locnx)++; + *xbegin = rank*(nx/nprocs) + xremain; + if (rank < xremain) *xbegin += rank - xremain; } diff --git a/test/fandc/pnctest.c b/test/fandc/pnctest.c index b85bd91703..dfafdc0fbe 100644 --- a/test/fandc/pnctest.c +++ b/test/fandc/pnctest.c @@ -21,8 +21,12 @@ int main (int argc, char *argv[]) zero is specified */ MPI_Offset TOTSIZ_3D[3] = { 10, 20, 30 }; MPI_Comm comm_cart; + double timing; + + MPI_Init(&argc, &argv); + + timing = MPI_Wtime(); - MPI_Init (&argc, &argv); MPI_Comm_size (MPI_COMM_WORLD, &totpes); MPI_Comm_size (MPI_COMM_WORLD, &rank); @@ -57,10 +61,12 @@ int main (int argc, char *argv[]) sum_size); } + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf(PASS_STR, timing); } MPI_Finalize(); diff --git a/test/header/Makefile.am b/test/header/Makefile.am index 8080f6885a..a7ad21676f 100644 --- a/test/header/Makefile.am +++ b/test/header/Makefile.am @@ -12,7 +12,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/src/include AM_CPPFLAGS += -I$(srcdir)/../common AM_CPPFLAGS += -I$(top_builddir)/src/include -LDADD = $(top_builddir)/src/libs/libpnetcdf.la ../common/libtestutils.la +LDADD = ../common/libtestutils.la $(top_builddir)/src/libs/libpnetcdf.la LDADD += @NETCDF4_LDFLAGS@ @ADIOS_LDFLAGS@ @NETCDF4_LIBS@ @ADIOS_LIBS@ if DECL_MPI_OFFSET @@ -23,34 +23,36 @@ if DECL_MPI_OFFSET # AM_FCFLAGS += $(FC_DEFINE)HAVE_DECL_MPI_OFFSET endif -TESTPROGRAMS = header_consistency - -check_PROGRAMS = $(TESTPROGRAMS) +check_PROGRAMS = header_consistency # autimake 1.11.3 has not yet implemented AM_TESTS_ENVIRONMENT # For newer versions, we can use AM_TESTS_ENVIRONMENT instead -# AM_TESTS_ENVIRONMENT = export TESTPROGRAMS="$(TESTPROGRAMS)"; +# AM_TESTS_ENVIRONMENT = export check_PROGRAMS="$(check_PROGRAMS)"; # AM_TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; # AM_TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; TESTS_ENVIRONMENT = export SED="$(SED)"; TESTS_ENVIRONMENT += export srcdir="$(srcdir)"; +TESTS_ENVIRONMENT += export top_builddir="$(top_builddir)"; TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; TESTS_ENVIRONMENT += export TESTMPIRUN="$(TESTMPIRUN)"; -TESTS_ENVIRONMENT += export PNETCDF_DEBUG="$(PNETCDF_DEBUG)"; -TESTS_ENVIRONMENT += export TESTPROGRAMS="$(TESTPROGRAMS)"; TESTS_ENVIRONMENT += export check_PROGRAMS="$(check_PROGRAMS)"; -TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER="$(ENABLE_BURST_BUFFER)"; + +TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER=@ENABLE_BURST_BUFFER@; +TESTS_ENVIRONMENT += export ENABLE_THREAD_SAFE=@ENABLE_THREAD_SAFE@; +TESTS_ENVIRONMENT += export MIMIC_LUSTRE=@MIMIC_LUSTRE@; # consistency check should be run on more than one process -# TESTS = seq_runs.sh +TESTS = $(check_PROGRAMS) +TEST_EXTENSIONS = .sh +LOG_COMPILER = $(srcdir)/seq_runs.sh +SH_LOG_COMPILER = -EXTRA_DIST = seq_runs.sh parallel_run.sh +check_SCRIPTS = seq_runs.sh parallel_run.sh -NC_FILES = $(TESTPROGRAMS:%=$(TESTOUTDIR)/%.nc) \ - $(TESTPROGRAMS:%=$(TESTOUTDIR)/%.bb.nc) +EXTRA_DIST = seq_runs.sh parallel_run.sh -CLEANFILES = $(NC_FILES) core core.* *.gcda *.gcno *.gcov gmon.out +CLEANFILES = core core.* *.gcda *.gcno *.gcov gmon.out ../common/libtestutils.la: set -e; cd ../common && $(MAKE) $(MFLAGS) tests diff --git a/test/header/header_consistency.c b/test/header/header_consistency.c index 62b05923b5..66c5146d70 100644 --- a/test/header/header_consistency.c +++ b/test/header/header_consistency.c @@ -13,13 +13,16 @@ #include #include #include /* basename() */ +#include + #include + #include #include #define EXP_ERR2(e, exp1, exp2) { \ if (e != exp1 && e != exp2 && e != NC_EFILE) { \ - printf("Error at line %d in %s: expecting error code %s or %s but got %s\n", \ + fprintf(stderr,"Error at line %d in %s: expecting error code %s or %s but got %s\n", \ __LINE__, __FILE__, ncmpi_strerrno(exp1), ncmpi_strerrno(exp2), ncmpi_strerrno(e)); \ nerrs++; \ } \ @@ -28,14 +31,14 @@ #define EXP_SAFE_ERR(expect) { \ if (safe_mode) { \ if (err != NC_EMULTIDEFINE && err != expect) { \ - printf("Error at line %d in %s: expecting error code NC_EMULTIDEFINE or %s but got %s\n", \ + fprintf(stderr,"Error at line %d in %s: expecting error code NC_EMULTIDEFINE or %s but got %s\n", \ __LINE__, __FILE__, ncmpi_strerrno(expect), ncmpi_strerrno(err)); \ nerrs++; \ } \ } \ else if (rank > 0) { \ if (err != expect) { \ - printf("Error at line %d in %s: expecting error code %s but got %s\n", \ + fprintf(stderr,"Error at line %d in %s: expecting error code %s but got %s\n", \ __LINE__, __FILE__, ncmpi_strerrno(expect), ncmpi_strerrno(err)); \ nerrs++; \ } \ @@ -44,11 +47,9 @@ /*----< test_open_mode() >----------------------------------------------------*/ static -int test_open_mode(char *filename, int safe_mode) +int test_open_mode(const char *filename, MPI_Comm comm, int safe_mode, MPI_Info info) { int err, rank, ncid, cmode, omode, nerrs=0; - MPI_Info info=MPI_INFO_NULL; - MPI_Comm comm=MPI_COMM_WORLD; MPI_Comm_rank(comm, &rank); @@ -84,11 +85,9 @@ int test_open_mode(char *filename, int safe_mode) /*----< test_dim() >----------------------------------------------------------*/ static -int test_dim(char *filename, int safe_mode) +int test_dim(const char *filename, MPI_Comm comm, int safe_mode, MPI_Info info) { int err, rank, ncid, cmode, dimid1, dimid2, dimid3, nerrs=0; - MPI_Info info=MPI_INFO_NULL; - MPI_Comm comm=MPI_COMM_WORLD; MPI_Comm_rank(comm, &rank); cmode = NC_CLOBBER|NC_64BIT_OFFSET; @@ -129,19 +128,18 @@ int test_dim(char *filename, int safe_mode) CHECK_ERR err = ncmpi_close(ncid); CHECK_ERR + return nerrs; } /*----< test_attr() >---------------------------------------------------------*/ static -int test_attr(char *filename, int safe_mode) +int test_attr(const char *filename, MPI_Comm comm, int safe_mode, MPI_Info info) { int err, rank, ncid, cmode, nerrs=0; char gattr[128]; int int_attr; float flt_attr; - MPI_Info info=MPI_INFO_NULL; - MPI_Comm comm=MPI_COMM_WORLD; MPI_Comm_rank(comm, &rank); cmode = NC_CLOBBER|NC_64BIT_OFFSET; @@ -197,14 +195,12 @@ int test_attr(char *filename, int safe_mode) /*----< test_var() >----------------------------------------------------------*/ static -int test_var(char *filename, int safe_mode) +int test_var(const char *filename, MPI_Comm comm, int safe_mode, MPI_Info info) { int err, rank, ncid, cmode, nerrs=0; int dimid[3], varid1, int_attr; float flt_attr; char name[128], var_attr[128]; - MPI_Info info=MPI_INFO_NULL; - MPI_Comm comm=MPI_COMM_WORLD; MPI_Comm_rank(comm, &rank); cmode = NC_CLOBBER|NC_64BIT_OFFSET; @@ -343,35 +339,41 @@ int test_var(char *filename, int safe_mode) return nerrs; } -/*----< main() >--------------------------------------------------------------*/ -int main(int argc, char **argv) +/*----< test_io() >----------------------------------------------------------*/ +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, /* ignored */ + int coll_io, /* ignored */ + MPI_Info info) { - char *filename="testfile.nc", *mode[2] = {"0", "1"}; + char *mode[2] = {"0", "1"}; int i, rank, nprocs, verbose, nerrs=0; + MPI_Comm comm=MPI_COMM_NULL; - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); - if (nprocs < 2) { - if (!rank) printf("This program is for running 2 or more processes. Exiting ...\n"); - MPI_Finalize(); - return 1; - } + if (nprocs == 1) return 0; - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) filename = argv[1]; + assert(in_path == NULL); + + /* This program is designed to run on 2 MPI processes */ + if (nprocs > 2) { + /* Make MPI calls to create a new communicator. */ + int new_ranks[2]={0,1}; + MPI_Group origin_group, group; - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for header consistency", basename(argv[0])); - printf("%-66s ------ ", cmd_str); - free(cmd_str); + MPI_Comm_group(MPI_COMM_WORLD, &origin_group); + MPI_Group_incl(origin_group, 2, new_ranks, &group); + MPI_Comm_create(MPI_COMM_WORLD, group, &comm); + MPI_Group_free(&group); + MPI_Group_free(&origin_group); } + else + comm = MPI_COMM_WORLD; + + if (rank >= 2) goto err_out; verbose = 1; for (i=verbose; i>=0; i--) { @@ -381,32 +383,51 @@ int main(int argc, char **argv) * PNETCDF_SAFE_MODE to 0. */ setenv("PNETCDF_SAFE_MODE", mode[i], 1); - nerrs += test_open_mode(filename, i); - nerrs += test_dim(filename, i); + nerrs = test_open_mode(out_path, comm, i, info); + if (nerrs > 0) goto err_out; - nerrs += test_attr(filename, i); + nerrs = test_dim(out_path, comm, i, info); + if (nerrs > 0) goto err_out; - nerrs += test_var(filename, i); - } + nerrs = test_attr(out_path, comm, i, info); + if (nerrs > 0) goto err_out; - MPI_Offset malloc_size, sum_size; - int err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); + nerrs = test_var(out_path, comm, i, info); + if (nerrs > 0) goto err_out; } - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +err_out: + if (comm != MPI_COMM_WORLD && comm != MPI_COMM_NULL) + MPI_Comm_free(&comm); + + return nerrs; +} + +int main(int argc, char **argv) { + + /* This test program does not support NetCDF4 option */ + int err, nprocs, formats[] = {0}; + + loop_opts opt; + + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 0; /* skip test intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = (nprocs > 1) ? true : false; /* run ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ + + err = tst_main(argc, argv, "header consistency", opt, test_io); MPI_Finalize(); - return (nerrs > 0); + + return err; } diff --git a/test/header/parallel_run.sh b/test/header/parallel_run.sh index a4d0770fa1..ad26c7a5b8 100755 --- a/test/header/parallel_run.sh +++ b/test/header/parallel_run.sh @@ -1,74 +1,41 @@ #!/bin/bash # -# Copyright (C) 2018, Northwestern University and Argonne National Laboratory +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory # See COPYRIGHT notice in top-level directory. # # Exit immediately if a command exits with a non-zero status. set -e -VALIDATOR=../../src/utils/ncvalidator/ncvalidator -NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff +DRY_RUN=no +VERBOSE=no -# remove file system type prefix if there is any -OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $MPIRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $MPIRUN $@ + fi +} MPIRUN=`echo ${TESTMPIRUN} | ${SED} -e "s/NP/$1/g"` # echo "MPIRUN = ${MPIRUN}" # echo "check_PROGRAMS=${check_PROGRAMS}" -# echo "PNETCDF_DEBUG = ${PNETCDF_DEBUG}" -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" -else - safe_modes="0" -fi - # prevent user environment setting of PNETCDF_HINTS to interfere unset PNETCDF_HINTS -for i in ${check_PROGRAMS} ; do - for j in ${safe_modes} ; do - for intra_aggr in 0 1 ; do - if test "$j" = 1 ; then # test only in safe mode - export PNETCDF_HINTS="romio_no_indep_rw=true" - else - export PNETCDF_HINTS= - fi - if test "$intra_aggr" = 1 ; then - export PNETCDF_HINTS="${PNETCDF_HINTS};nc_num_aggrs_per_node=2" - fi - export PNETCDF_SAFE_MODE=$j - # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - ${MPIRUN} ./$i ${TESTOUTDIR}/$i.nc - - # echo "--- validating file ${TESTOUTDIR}/$i.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.nc - # echo "" +if test "x$MIMIC_LUSTRE" != x1 ; then + PNETCDF_HINTS="cb_nodes=2" +fi - if test "x${ENABLE_BURST_BUFFER}" = x1 ; then - # echo "---- test burst buffering feature" - saved_PNETCDF_HINTS=${PNETCDF_HINTS} - export PNETCDF_HINTS="${PNETCDF_HINTS};nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" - ${MPIRUN} ./$i ${TESTOUTDIR}/$i.bb.nc - export PNETCDF_HINTS=${saved_PNETCDF_HINTS} +for i in ${check_PROGRAMS} ; do - # echo "--- validating file ${TESTOUTDIR}/$i.bb.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.bb.nc + exe_name=`basename $i` - # burst buffering does not support nonblocking requests in define mode - # echo "--- ncmpidiff $i.nc $i.bb.nc ---" - ${MPIRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/$i.nc ${TESTOUTDIR}/$i.bb.nc - fi + run_cmd ./$i -q -o ${TESTOUTDIR}/${exe_name}.nc - if test "x${ENABLE_NETCDF4}" = x1 ; then - # echo "test netCDF-4 feature" - ${MPIRUN} ./$i ${TESTOUTDIR}/$i.nc4 4 - # Validator does not support nc4 - fi - done - done - rm -f ${OUTDIR}/$i.nc - rm -f ${OUTDIR}/$i.bb.nc -done +done # check_PROGRAMS diff --git a/test/header/seq_runs.sh b/test/header/seq_runs.sh index 475c8da8ac..eb3c73faf6 100755 --- a/test/header/seq_runs.sh +++ b/test/header/seq_runs.sh @@ -1,44 +1,29 @@ -#!/bin/sh +#!/bin/bash # -# Copyright (C) 2003, Northwestern University and Argonne National Laboratory +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory # See COPYRIGHT notice in top-level directory. # # Exit immediately if a command exits with a non-zero status. set -e -VALIDATOR=../../src/utils/ncvalidator/ncvalidator -NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff +DRY_RUN=no +VERBOSE=no -outfile=`basename $1` +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $TESTSEQRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $TESTSEQRUN $@ + fi +} -# echo "PNETCDF_DEBUG = ${PNETCDF_DEBUG}" -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" -else - safe_modes="0" -fi +exe_name=`basename $1` # prevent user environment setting of PNETCDF_HINTS to interfere unset PNETCDF_HINTS -# header consistency tests are designed to run on more than one MPI process -for j in ${safe_modes} ; do - export PNETCDF_SAFE_MODE=$j - # echo "---- set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - ${TESTSEQRUN} $1 ${TESTOUTDIR}/$outfile.nc - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$outfile.nc +run_cmd ./$1 -q -o ${TESTOUTDIR}/${exe_name}.nc - - if test "x${ENABLE_BURST_BUFFER}" = x1 ; then - echo "" - echo "---- testing burst buffering" - export PNETCDF_HINTS="nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" - ${TESTSEQRUN} $1 ${TESTOUTDIR}/$outfile.bb.nc - unset PNETCDF_HINTS - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$outfile.bb.nc - - # echo "--- ncmpidiff $outfile.nc $outfile.bb.nc ---" - ${TESTSEQRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/$outfile.nc ${TESTOUTDIR}/$outfile.bb.nc - fi -done diff --git a/test/largefile/Makefile.am b/test/largefile/Makefile.am index d74134b761..d9cea7463f 100644 --- a/test/largefile/Makefile.am +++ b/test/largefile/Makefile.am @@ -14,7 +14,7 @@ AM_CPPFLAGS = -I$(srcdir)/../common AM_CPPFLAGS += -I$(top_srcdir)/src/include AM_CPPFLAGS += -I$(top_builddir)/src/include -LDADD = $(top_builddir)/src/libs/libpnetcdf.la ../common/libtestutils.la +LDADD = ../common/libtestutils.la $(top_builddir)/src/libs/libpnetcdf.la LDADD += @NETCDF4_LDFLAGS@ @ADIOS_LDFLAGS@ @NETCDF4_LIBS@ @ADIOS_LIBS@ if IS_BIGENDIAN @@ -24,62 +24,61 @@ if IS_BIGENDIAN # AM_FCFLAGS += $(FC_DEFINE)WORDS_BIGENDIAN endif -TESTPROGRAMS = large_files \ - large_var \ - large_attr \ - large_dims_vars_attrs \ - high_dim_var \ - tst_cdf5_begin \ - tst_hash_large_ndims \ - tst_hash_large_nvars \ - tst_hash_large_ngattrs \ - large_coalesce \ - large_header \ - large_reqs +check_PROGRAMS = large_files \ + large_var \ + large_attr \ + large_dims_vars_attrs \ + high_dim_var \ + tst_cdf5_begin \ + tst_hash_large_ndims \ + tst_hash_large_nvars \ + tst_hash_large_ngattrs \ + large_coalesce \ + large_header \ + large_reqs if HAS_FORTRAN - TESTPROGRAMS += bigrecords + check_PROGRAMS += bigrecords bigrecords_SOURCES = bigrecords.f AM_FFLAGS = -I$(top_builddir)/src/binding/f77 $(FFIXEDFORMFLAG) if HAVE_MPI_MOD - TESTPROGRAMS += tst_flarge + check_PROGRAMS += tst_flarge tst_flarge_SOURCES = tst_flarge.f90 AM_FCFLAGS = $(FC_MODINC)$(top_builddir)/src/binding/f90 \ $(FC_MODINC)../common $(FFREEFORMFLAG) endif endif -EXTRA_DIST = wrap_runs.sh parallel_run.sh +check_SCRIPTS = seq_runs.sh parallel_run.sh -NC_FILES = $(TESTPROGRAMS:%=$(TESTOUTDIR)/%.nc) \ - $(TESTPROGRAMS:%=$(TESTOUTDIR)/%.bb.nc) +EXTRA_DIST = seq_runs.sh parallel_run.sh -CLEANFILES = $(NC_FILES) core core.* *.gcda *.gcno *.gcov gmon.out +CLEANFILES = *.nc core core.* *.gcda *.gcno *.gcov gmon.out ../common/libtestutils.la: set -e; cd ../common && $(MAKE) $(MFLAGS) tests -check_PROGRAMS = $(TESTPROGRAMS) - # autimake 1.11.3 has not yet implemented AM_TESTS_ENVIRONMENT # For newer versions, we can use AM_TESTS_ENVIRONMENT instead -# AM_TESTS_ENVIRONMENT = export TESTPROGRAMS="$(TESTPROGRAMS)"; +# AM_TESTS_ENVIRONMENT = export check_PROGRAMS="$(check_PROGRAMS)"; # AM_TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; # AM_TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; TESTS_ENVIRONMENT = export SED="$(SED)"; TESTS_ENVIRONMENT += export srcdir="$(srcdir)"; +TESTS_ENVIRONMENT += export top_builddir="$(top_builddir)"; TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; TESTS_ENVIRONMENT += export TESTMPIRUN="$(TESTMPIRUN)"; -TESTS_ENVIRONMENT += export PNETCDF_DEBUG="$(PNETCDF_DEBUG)"; -TESTS_ENVIRONMENT += export TESTPROGRAMS="$(TESTPROGRAMS)"; TESTS_ENVIRONMENT += export check_PROGRAMS="$(check_PROGRAMS)"; -TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER="$(ENABLE_BURST_BUFFER)"; -TESTS_ENVIRONMENT += export ENABLE_NETCDF4="$(ENABLE_NETCDF4)"; + +TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER=@ENABLE_BURST_BUFFER@; +TESTS_ENVIRONMENT += export ENABLE_NETCDF4=@ENABLE_NETCDF4@; +TESTS_ENVIRONMENT += export ENABLE_THREAD_SAFE=@ENABLE_THREAD_SAFE@; +TESTS_ENVIRONMENT += export MIMIC_LUSTRE=@MIMIC_LUSTRE@; TESTS = $(check_PROGRAMS) TEST_EXTENSIONS = .sh -LOG_COMPILER = $(srcdir)/wrap_runs.sh +LOG_COMPILER = $(srcdir)/seq_runs.sh SH_LOG_COMPILER = # Some of these tests are designed to run on one processes, @@ -96,7 +95,7 @@ ptests: ptest4 ptest2 ptest6 ptest8 ptest10: # build check targets but not invoke -tests-local: all $(TESTPROGRAMS) +tests-local: all $(check_PROGRAMS) .PHONY: ptest ptests ptest2 ptest4 ptest6 ptest8 ptest10 diff --git a/test/largefile/bigrecords.f b/test/largefile/bigrecords.f index 51715e8ed8..08114b6d9d 100644 --- a/test/largefile/bigrecords.f +++ b/test/largefile/bigrecords.f @@ -88,29 +88,36 @@ program main + 377.5, 367.59, 360.06, 353.85999, 348.66, 342.5, 336, 328.5, 320, + 310, 300, 290, 280, 270, 260, 250, 240, 230, 220, 210, 199.10001/ - character(LEN=256) filename, cmd, msg - + character(LEN=256) out_path, in_path, cmd, msg + logical keep_files + double precision timing ! attribute vectors ! enter define mode ! iret = nf_create('pressure_19010101_000000.nc', OR(NF_CLOBBER,NF_64BIT_OFFSET), ncid) call MPI_INIT(ierr) + + timing = MPI_Wtime() + call MPI_COMM_RANK(MPI_COMM_WORLD, myid, ierr) call MPI_COMM_SIZE(MPI_COMM_WORLD, numprocs, ierr) if (myid .EQ. 0) then - filename = "testfile.nc" - err = get_args(cmd, filename) + out_path = "testfile.nc" + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, + ierr) call MPI_Bcast(cmd, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, + ierr) - iret = nfmpi_create( MPI_COMM_WORLD, filename, + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, + + ierr) + + iret = nfmpi_create( MPI_COMM_WORLD, out_path, + IOR(NF_CLOBBER,NF_64BIT_DATA), + MPI_INFO_NULL, ncid) @@ -232,7 +239,7 @@ program main ! todo: insert code to re-open dataset, read time variable all at once ! iret = nfmpi_open ( MPI_COMM_SELF, - + filename, + + out_path, + IOR(NF_CLOBBER,NF_64BIT_DATA), + MPI_INFO_NULL, + ncid) @@ -259,10 +266,21 @@ program main ! write(6,*) "Error: time array was ", time ! endif - msg = '*** TESTING F77 '//cmd(1:XTRIM(cmd))//' for NF_64BIT_DATA' - if (myid .eq. 0) call pass_fail(0, msg) + 999 timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, + + MPI_DOUBLE_PRECISION, MPI_MAX, + + MPI_COMM_WORLD, ierr) + + if (myid .eq. 0) then + if (.NOT. keep_files) then + err = nfmpi_delete(out_path, MPI_INFO_NULL) + end if + + msg='*** TESTING F77 '//cmd(1:XTRIM(cmd))//' for NF_64BIT_DATA' + call pass_fail(0, msg, timing) + endif - 999 call MPI_FINALIZE(ierr) + call MPI_FINALIZE(ierr) end ! program main subroutine writerecs(cmd,ncid,time_id) @@ -336,7 +354,7 @@ subroutine check_err(cmd, msg, iret) print *, msg, nfmpi_strerror(iret) msg = '*** TESTING F77 '//cmd(1:XTRIM(cmd))// + ' for NF_64BIT_DATA' - call pass_fail(1, msg) + call pass_fail(1, msg, timing) stop 2 endif end ! subroutine check_err diff --git a/test/largefile/high_dim_var.c b/test/largefile/high_dim_var.c index 482e9d2688..43ac7cb6a4 100644 --- a/test/largefile/high_dim_var.c +++ b/test/largefile/high_dim_var.c @@ -24,39 +24,29 @@ #define DIMLEN 3 #define NRECS 4 -int main(int argc, char** argv) { - char filename[256], name[32]; +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) +{ + char name[32]; size_t nelms; short *buffer; - int i, j, cmode, rank, nprocs, err, nerrs=0; + int i, j, rank, nprocs, err, nerrs=0; int ncid, fvarid[NVARS], rvarid[NVARS], dimids[NDIMS], rdimids[NDIMS]; MPI_Offset start[NDIMS], count[NDIMS], stride[NDIMS]; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - memset(filename, 0, 256); - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for vars APIs on high-dim variables ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR - cmode = NC_CLOBBER; - cmode |= NC_64BIT_DATA; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, - &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR /* define dimensions */ err = ncmpi_def_dim(ncid, "rdim", NC_UNLIMITED, &rdimids[0]); CHECK_ERR @@ -76,7 +66,6 @@ int main(int argc, char** argv) { err = ncmpi_set_fill(ncid, NC_FILL, NULL); CHECK_ERR err = ncmpi_enddef(ncid); CHECK_ERR - if (err != NC_NOERR) goto fn_exit; #ifdef STRONGER_CONSISTENCY ncmpi_sync(ncid); @@ -88,17 +77,18 @@ int main(int argc, char** argv) { for (i=1; i 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - } - if (malloc_size > 0) ncmpi_inq_malloc_list(); -#endif - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +err_out: + return nerrs; +} + +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 0; /* skip test intra-node aggregation */ + opt.drv = 0; /* test GIO driver only */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ + + err = tst_main(argc, argv, "vars APIs on high-dim variables", opt, test_io); MPI_Finalize(); - return (nerrs > 0); + + return err; } diff --git a/test/largefile/large_attr.c b/test/largefile/large_attr.c index c032c6ba5c..576b551af8 100644 --- a/test/largefile/large_attr.c +++ b/test/largefile/large_attr.c @@ -28,67 +28,67 @@ #include #include -int main(int argc, char** argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) { - char filename[256], *name, *buf; + char *name, *buf; size_t i; - int rank, nprocs, err, nerrs=0; - int ncid, cmode, varid, dimid; + int err, nerrs=0, ncid, varid, dimid; MPI_Offset nelems, inq_nelems; - MPI_Info info=MPI_INFO_NULL; -#ifdef PNC_MALLOC_TRACE - int verbose=0; -#endif + int rank, nprocs, color; + MPI_Comm comm=MPI_COMM_NULL; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - /* get command-line arguments */ - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for one large ATTR", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); + color = 1; + + if (nprocs > 2) { + /* run on 2 ranks only, as this test allocates memory > 4GB per rank */ + /* split MPI_COMM_WORLD based on 'color' and use the same rank order */ + color = (rank < 2) ? 1 : 0; + MPI_Comm_split(MPI_COMM_WORLD, color, rank, &comm); } + else + comm = MPI_COMM_WORLD; + + if (!color) goto err_out; nelems = (MPI_Offset)NC_MAX_INT + 17; buf = (char*) malloc(nelems); + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + /* create a new file and put a large global attribute -------------------*/ - cmode = NC_CLOBBER | NC_64BIT_DATA; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); + err = ncmpi_create(comm, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR /* put a large (> 2GiB) global attribute */ for (i=0; i 0) goto err_out; /* open the file and read back the large global attribute ---------------*/ - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); + err = ncmpi_open(comm, out_path, NC_NOWRITE, info, &ncid); CHECK_ERR - if (err != NC_NOERR) goto err_out; err = ncmpi_inq_attlen(ncid, NC_GLOBAL, "large_attr", &inq_nelems); + CHECK_ERR if (inq_nelems != nelems) { - printf("Error at %s line %d: expecting attr nelems "OFFFMT" but got "OFFFMT"\n", + fprintf(stderr,"Error at %s line %d: expecting attr nelems "OFFFMT" but got "OFFFMT"\n", __FILE__,__LINE__,nelems,inq_nelems); nerrs++; + goto err_out; } for (i=0; i 0) goto err_out; /* create a new file and put a large local attribute -------------------*/ - cmode = NC_CLOBBER | NC_64BIT_DATA; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); + err = ncmpi_create(comm, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "time", NC_UNLIMITED, &dimid); @@ -122,15 +121,14 @@ int main(int argc, char** argv) /* put a large (> 2GiB) global attribute */ for (i=0; i 0) goto err_out; /* open the file and read back the large global attribute ---------------*/ - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); + err = ncmpi_open(comm, out_path, NC_NOWRITE, info, &ncid); CHECK_ERR if (err != NC_NOERR) goto err_out; @@ -139,9 +137,10 @@ int main(int argc, char** argv) err = ncmpi_inq_attlen(ncid, varid, "large_attr", &inq_nelems); if (inq_nelems != nelems) { - printf("Error at %s line %d: expecting attr len "OFFFMT" but got "OFFFMT"\n", + fprintf(stderr,"Error at %s line %d: expecting attr len "OFFFMT" but got "OFFFMT"\n", __FILE__,__LINE__,nelems,inq_nelems); nerrs++; + goto err_out; } for (i=0; i 0) goto err_out; /* create a new file and put 2 global attributes, total size > 2 GiB ----*/ nelems /= 2; - cmode = NC_CLOBBER | NC_64BIT_DATA; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); + err = ncmpi_create(comm, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR /* put two global attributes (total size > 2GiB) */ for (i=0; i 0) goto err_out; /* open the file and read back the large global attributes --------------*/ - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); + err = ncmpi_open(comm, out_path, NC_NOWRITE, info, &ncid); CHECK_ERR if (err != NC_NOERR) goto err_out; name = "large_attr_0"; err = ncmpi_inq_attlen(ncid, NC_GLOBAL, name, &inq_nelems); if (inq_nelems != nelems) { - printf("Error at %s line %d: expecting attr %s nelems "OFFFMT" but got "OFFFMT"\n", + fprintf(stderr,"Error at %s line %d: expecting attr %s nelems "OFFFMT" but got "OFFFMT"\n", __FILE__,__LINE__,name, nelems,inq_nelems); nerrs++; + goto err_out; } for (i=0; i 0) goto err_out; /* create a new file and put 2 local attributes, total size > 2 GiB -----*/ - cmode = NC_CLOBBER | NC_64BIT_DATA; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); + err = ncmpi_create(comm, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "time", NC_UNLIMITED, &dimid); @@ -250,20 +246,19 @@ int main(int argc, char** argv) /* put two local attributes (total size > 2GiB) */ name = "large_attr_0"; err = ncmpi_put_att_text(ncid, varid, name, nelems, buf); - if (!(cmode & NC_64BIT_DATA)) EXP_ERR(NC_EINVAL) + if (format != NC_FORMAT_64BIT_DATA) EXP_ERR(NC_EINVAL) else CHECK_ERR name = "large_attr_1"; err = ncmpi_put_att_text(ncid, varid, name, nelems, buf); - if (!(cmode & NC_64BIT_DATA)) EXP_ERR(NC_EINVAL) + if (format != NC_FORMAT_64BIT_DATA) EXP_ERR(NC_EINVAL) else CHECK_ERR err = ncmpi_enddef(ncid); CHECK_ERR err = ncmpi_close(ncid); CHECK_ERR - if (nerrs > 0) goto err_out; /* open the file and read back the two local attributes -----------------*/ - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); + err = ncmpi_open(comm, out_path, NC_NOWRITE, info, &ncid); CHECK_ERR if (err != NC_NOERR) goto err_out; @@ -273,9 +268,10 @@ int main(int argc, char** argv) name = "large_attr_0"; err = ncmpi_inq_attlen(ncid, varid, name, &inq_nelems); if (inq_nelems != nelems) { - printf("Error at %s line %d: expecting attr %s len "OFFFMT" but got "OFFFMT"\n", + fprintf(stderr,"Error at %s line %d: expecting attr %s len "OFFFMT" but got "OFFFMT"\n", __FILE__,__LINE__,name,nelems,inq_nelems); nerrs++; + goto err_out; } for (i=0; i 0) goto err_out; free(buf); -#ifdef PNC_MALLOC_TRACE - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) { - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - } - } - if (malloc_size > 0) ncmpi_inq_malloc_list(); +err_out: + if (comm != MPI_COMM_WORLD && comm != MPI_COMM_NULL) + MPI_Comm_free(&comm); - if (verbose) { - err = ncmpi_inq_malloc_max_size(&malloc_size); - printf("\n%d: PnetCDF internal memory footprint high water mark %.2f MB\n", - rank, (float)malloc_size/1048576); - } -#endif + return nerrs; +} -err_out: - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 0; /* skip test intra-node aggregation */ + opt.drv = 0; /* test GIO driver only */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ + + err = tst_main(argc, argv, "one large ATTR", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/largefile/large_coalesce.c b/test/largefile/large_coalesce.c index 6b4ece55dc..f5a5c44bd8 100644 --- a/test/largefile/large_coalesce.c +++ b/test/largefile/large_coalesce.c @@ -26,36 +26,33 @@ #define TWO_G 2147483648LL #define ONE_G 1073741824LL -int main(int argc, char** argv) +static +int test_io_nc5(const char *out_path, + MPI_Info global_info) { - char filename[256]; unsigned char *buf; - int rank, nprocs, err, nerrs=0; - int ncid, cmode, varid, dimid[2], req[3], st[3]; + int rank, nprocs, color, err, nerrs=0; + int ncid, varid, dimid[2], req[3], st[3]; MPI_Offset start[2], count[2]; MPI_Info info; size_t i; + MPI_Comm comm=MPI_COMM_NULL; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - /* get command-line arguments */ - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for skip filetype buftype coalesce ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); + color = 1; + + if (nprocs > 2) { + /* run on 2 ranks only, as this test allocates memory > 4GB per rank */ + /* split MPI_COMM_WORLD based on 'color' and use the same rank order */ + color = (rank < 2) ? 1 : 0; + MPI_Comm_split(MPI_COMM_WORLD, color, rank, &comm); } + else + comm = MPI_COMM_WORLD; + + if (!color) goto err_out; buf = (unsigned char*) calloc(TWO_G+1024,1); if (buf == NULL) { @@ -64,19 +61,15 @@ int main(int argc, char** argv) return 1; } - MPI_Info_create(&info); + MPI_Info_dup(global_info, &info); MPI_Info_set(info, "romio_cb_write", "enable"); MPI_Info_set(info, "romio_ds_read", "disable"); /* run slow without it */ - /* silence iternal debug messages */ + /* silence internal debug messages */ setenv("PNETCDF_SAFE_MODE", "0", 1); -#ifdef ENABLE_NETCDF4 - /* Test for NetCDF 4 first as ncvalidator checks only read classic files */ - /* create a new file for writing ----------------------------------------*/ - cmode = NC_CLOBBER | NC_NETCDF4; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); + err = ncmpi_create(comm, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR /* define dimensions */ @@ -101,76 +94,6 @@ int main(int argc, char** argv) start[0] = rank; count[0] = 1; - start[1] = 0; - count[1] = 10; - err = ncmpi_put_vara_uchar_all(ncid, varid, start, count, buf); - CHECK_ERR - - /* 2nd request is not contiguous from the first */ - start[1] = 1024; - count[1] = ONE_G-1024; - err = ncmpi_put_vara_uchar_all(ncid, varid, start, count, buf+1024); - CHECK_ERR - - /* make file access and write buffer of 3rd request contiguous from the 2nd - * request to check whether the internal fileview and buftype coalescing - * are skipped */ - start[1] = ONE_G; - count[1] = ONE_G+1024; - err = ncmpi_put_vara_uchar_all(ncid, varid, start, count, buf+ONE_G); - CHECK_ERR - - start[1] = 0; - count[1] = 10; - err = ncmpi_get_vara_uchar_all(ncid, varid, start, count, buf); - CHECK_ERR - - start[1] = 1024; - count[1] = ONE_G-1024; - err = ncmpi_get_vara_uchar_all(ncid, varid, start, count, buf+1024); - CHECK_ERR - - start[1] = ONE_G; - count[1] = ONE_G+1024; - err = ncmpi_get_vara_uchar_all(ncid, varid, start, count, buf+ONE_G); - CHECK_ERR - - err = ncmpi_close(ncid); CHECK_ERR - - /* check if open to read header fine */ - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); CHECK_ERR - err = ncmpi_close(ncid); CHECK_ERR -#endif - /* Test classic format */ - - /* create a new file for writing ----------------------------------------*/ - cmode = NC_CLOBBER | NC_64BIT_DATA; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); - CHECK_ERR - MPI_Info_free(&info); - - /* define dimensions */ - err = ncmpi_def_dim(ncid, "NPROCS", nprocs, &dimid[0]); - CHECK_ERR - - err = ncmpi_def_dim(ncid, "X", TWO_G+1024, &dimid[1]); - CHECK_ERR - - /* define a big 1D variable of ubyte type */ - err = ncmpi_def_var(ncid, "big_var", NC_UBYTE, 2, dimid, &varid); - CHECK_ERR - - /* do not forget to exit define mode */ - err = ncmpi_enddef(ncid); - CHECK_ERR - - /* now we are in data mode */ - for (i=0; i<20; i++) buf[ONE_G-10+i] = 'a'+i; - for (i=0; i<20; i++) buf[TWO_G-10+i] = 'A'+i; - - start[0] = rank; - count[0] = 1; - start[1] = 0; count[1] = 10; err = ncmpi_iput_vara_uchar(ncid, varid, start, count, buf, &req[0]); @@ -259,31 +182,185 @@ int main(int argc, char** argv) err = ncmpi_close(ncid); CHECK_ERR /* check if open for reading header */ - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); CHECK_ERR + err = ncmpi_open(comm, out_path, NC_NOWRITE, info, &ncid); CHECK_ERR err = ncmpi_close(ncid); CHECK_ERR free(buf); -#ifdef PNC_MALLOC_TRACE - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); + MPI_Info_free(&info); + +err_out: + if (comm != MPI_COMM_WORLD && comm != MPI_COMM_NULL) + MPI_Comm_free(&comm); + + return nerrs; +} + +static +int test_io_nc4(const char *out_path, + MPI_Info global_info) +{ + unsigned char *buf; + int rank, nprocs, color, err, nerrs=0; + int ncid, varid, dimid[2]; + MPI_Offset start[2], count[2]; + MPI_Info info; + size_t i; + MPI_Comm comm=MPI_COMM_NULL; + + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + color = 1; + + if (nprocs > 2) { + /* run on 2 ranks only, as this test allocates memory > 4GB per rank */ + /* split MPI_COMM_WORLD based on 'color' and use the same rank order */ + color = (rank < 2) ? 1 : 0; + MPI_Comm_split(MPI_COMM_WORLD, color, rank, &comm); } - if (malloc_size > 0) ncmpi_inq_malloc_list(); -#endif + else + comm = MPI_COMM_WORLD; - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + if (!color) goto err_out; + + buf = (unsigned char*) calloc(TWO_G+1024,1); + if (buf == NULL) { + printf("malloc failed for size "OFFFMT"\n", TWO_G+1024); + MPI_Finalize(); + return 1; } - MPI_Finalize(); - return (nerrs > 0); + MPI_Info_dup(global_info, &info); + MPI_Info_set(info, "romio_cb_write", "enable"); + MPI_Info_set(info, "romio_ds_read", "disable"); /* run slow without it */ + + /* silence internal debug messages */ + setenv("PNETCDF_SAFE_MODE", "0", 1); + + /* Test for NetCDF 4 first as ncvalidator checks only read classic files */ + + /* create a new file for writing ----------------------------------------*/ + err = ncmpi_create(comm, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR + + /* define dimensions */ + err = ncmpi_def_dim(ncid, "NPROCS", nprocs, &dimid[0]); + CHECK_ERR + + err = ncmpi_def_dim(ncid, "X", TWO_G+1024, &dimid[1]); + CHECK_ERR + + /* define a big 1D variable of ubyte type */ + err = ncmpi_def_var(ncid, "big_var", NC_UBYTE, 2, dimid, &varid); + CHECK_ERR + + /* do not forget to exit define mode */ + err = ncmpi_enddef(ncid); + CHECK_ERR + + /* now we are in data mode */ + for (i=0; i<20; i++) buf[ONE_G-10+i] = 'a'+i; + for (i=0; i<20; i++) buf[TWO_G-10+i] = 'A'+i; + + start[0] = rank; + count[0] = 1; + + start[1] = 0; + count[1] = 10; + err = ncmpi_put_vara_uchar_all(ncid, varid, start, count, buf); + CHECK_ERR + + /* 2nd request is not contiguous from the first */ + start[1] = 1024; + count[1] = ONE_G-1024; + err = ncmpi_put_vara_uchar_all(ncid, varid, start, count, buf+1024); + CHECK_ERR + + /* make file access and write buffer of 3rd request contiguous from the 2nd + * request to check whether the internal fileview and buftype coalescing + * are skipped */ + start[1] = ONE_G; + count[1] = ONE_G+1024; + err = ncmpi_put_vara_uchar_all(ncid, varid, start, count, buf+ONE_G); + CHECK_ERR + + start[1] = 0; + count[1] = 10; + err = ncmpi_get_vara_uchar_all(ncid, varid, start, count, buf); + CHECK_ERR + + start[1] = 1024; + count[1] = ONE_G-1024; + err = ncmpi_get_vara_uchar_all(ncid, varid, start, count, buf+1024); + CHECK_ERR + + start[1] = ONE_G; + count[1] = ONE_G+1024; + err = ncmpi_get_vara_uchar_all(ncid, varid, start, count, buf+ONE_G); + CHECK_ERR + + err = ncmpi_close(ncid); CHECK_ERR + + /* check if open to read header fine */ + err = ncmpi_open(comm, out_path, NC_NOWRITE, info, &ncid); CHECK_ERR + err = ncmpi_close(ncid); CHECK_ERR + + free(buf); + + MPI_Info_free(&info); + +err_out: + if (comm != MPI_COMM_WORLD && comm != MPI_COMM_NULL) + MPI_Comm_free(&comm); + + return nerrs; } +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) +{ + int err; + + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + if (format == NC_FORMAT_NETCDF4) + return test_io_nc4(out_path, info); + else + return test_io_nc5(out_path, info); +} + +int main(int argc, char **argv) { + + int err; +#if PNETCDF_DRIVER_NETCDF4 == 1 + int formats[] = {NC_FORMAT_NETCDF4, NC_FORMAT_64BIT_DATA}; +#else + int formats[] = {NC_FORMAT_64BIT_DATA}; +#endif + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 0; /* skip test intra-node aggregation */ + opt.drv = 0; /* test GIO driver only */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ + + err = tst_main(argc, argv, "skip filetype buftype coalesce", opt, test_io); + + MPI_Finalize(); + + return err; +} diff --git a/test/largefile/large_dims_vars_attrs.c b/test/largefile/large_dims_vars_attrs.c index e924c5087d..f04101508c 100644 --- a/test/largefile/large_dims_vars_attrs.c +++ b/test/largefile/large_dims_vars_attrs.c @@ -27,42 +27,43 @@ #define LARGE_NUM 102400 -int main(int argc, char** argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) { - char filename[256], str[32]; - int i, rank, nprocs, err, nerrs=0; - int ncid, cmode, *varid, *dimids, intBuf[1]; -#ifdef PNC_MALLOC_TRACE - int verbose=0; -#endif + char str[32]; + int i, rank, nprocs, color, err, nerrs=0; + int ncid, *varid, *dimids, intBuf[1]; + MPI_Comm comm=MPI_COMM_NULL; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - /* get command-line arguments */ - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for large DIMS, VARS, ATTRS ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); + color = 1; + + if (nprocs > 2) { + /* run on 2 ranks only, as this test allocates memory > 4GB per rank */ + /* split MPI_COMM_WORLD based on 'color' and use the same rank order */ + color = (rank < 2) ? 1 : 0; + MPI_Comm_split(MPI_COMM_WORLD, color, rank, &comm); } + else + comm = MPI_COMM_WORLD; + + if (!color) goto err_out; dimids = (int*) malloc(sizeof(int) * LARGE_NUM); varid = (int*) malloc(sizeof(int) * LARGE_NUM); + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + /* create a new file for writing ----------------------------------------*/ - cmode = NC_CLOBBER; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + err = ncmpi_create(comm, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR for (i=0; i 0) { - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - } - } - if (malloc_size > 0) ncmpi_inq_malloc_list(); +err_out: + if (comm != MPI_COMM_WORLD && comm != MPI_COMM_NULL) + MPI_Comm_free(&comm); - if (verbose) { - err = ncmpi_inq_malloc_max_size(&malloc_size); - printf("\n%d: PnetCDF internal memory footprint high water mark %.2f MB\n", - rank, (float)malloc_size/1048576); - } -#endif + return nerrs; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 0; /* skip test intra-node aggregation */ + opt.drv = 0; /* test GIO driver only */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ + + err = tst_main(argc, argv, "large DIMS, VARS, ATTRS", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/largefile/large_files.c b/test/largefile/large_files.c index 74b30484b2..0c90ac8c2e 100644 --- a/test/largefile/large_files.c +++ b/test/largefile/large_files.c @@ -16,14 +16,7 @@ #include #include -#define CHECK_ERR { \ - if (err != NC_NOERR) { \ - nerrs++; \ - printf("Error at line %d in %s: (%s)\n", \ - __LINE__,__FILE__,ncmpi_strerrno(err)); \ - goto fn_exit; \ - } \ -} +#include #define NUMRECS 1 #define I_LEN 4104 @@ -31,10 +24,14 @@ #define K_LEN 1023 #define N_LEN 2 -static int -test_large_file(char *filename, int fmt_flag) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) { - int err, nerrs=0, ncid, varid, x_id; + int err, nerrs=0, rank, ncid, varid, x_id; int n, rec, i, j, k, dims[4]; MPI_Offset start[4] = {0, 0, 0, 0}; MPI_Offset count[4] = {1, 1, J_LEN, K_LEN}; @@ -42,13 +39,19 @@ test_large_file(char *filename, int fmt_flag) /* I/O buffers */ signed char *buf; - printf("\n*** Testing large files, slowly.\n"); - printf("*** Creating large file %s...", filename); + // printf("\n*** Testing large files, slowly.\n"); + // printf("*** Creating large file %s...", out_path); + + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + if (rank > 0) goto err_out; + + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR - err = ncmpi_create(MPI_COMM_SELF, filename, NC_CLOBBER|fmt_flag, - MPI_INFO_NULL, &ncid); + err = ncmpi_create(MPI_COMM_SELF, out_path, NC_CLOBBER, info, &ncid); if (err != NC_NOERR) { - printf("Error at line %d in %s: (%s)\n", __LINE__,__FILE__,ncmpi_strerrno(err)); + fprintf(stderr,"Error at line %d in %s: (%s)\n", __LINE__,__FILE__,ncmpi_strerrno(err)); return 1; } @@ -97,10 +100,10 @@ test_large_file(char *filename, int fmt_flag) } err = ncmpi_close(ncid); CHECK_ERR - printf("ok\n"); - printf("*** Reading large file %s...", filename); + // printf("ok\n"); + // printf("*** Reading large file %s...", out_path); - err = ncmpi_open(MPI_COMM_SELF, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); + err = ncmpi_open(MPI_COMM_SELF, out_path, NC_NOWRITE, MPI_INFO_NULL, &ncid); CHECK_ERR /* read variables and check their contents */ @@ -120,10 +123,10 @@ test_large_file(char *filename, int fmt_flag) for (j=0; j 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - memset(filename, 0, 256); - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); +int main(int argc, char **argv) { - if (rank > 0) goto prog_exit; + int err; + int formats[] = {NC_FORMAT_64BIT_DATA}; + loop_opts opt; - /* Test NetCDF 4 first as ncvalidator checks only classic files */ -#ifdef ENABLE_NETCDF4 - nerrs += test_large_file(filename, NC_NETCDF4); -#endif + MPI_Init(&argc, &argv); - /* Test CDF-5 format */ - nerrs += test_large_file(filename, NC_64BIT_DATA); + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 0; /* test GIO driver only */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ - if (nerrs == 0) { - printf("ok\n"); - printf("*** Tests successful!\n"); - } - else - printf("\n*** Tests failed!\n"); + err = tst_main(argc, argv, "> 4 GiB file", opt, test_io); -prog_exit: MPI_Finalize(); - return (nerrs > 0); + + return err; } diff --git a/test/largefile/large_header.c b/test/largefile/large_header.c index a376496609..e2eb7c3f23 100644 --- a/test/largefile/large_header.c +++ b/test/largefile/large_header.c @@ -22,37 +22,26 @@ #include #include -int main(int argc, char** argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) { - char filename[256]; int rank, nprocs, err, nerrs=0; - int ncid, cmode, dimid, varid, buf; + int ncid, dimid, varid, buf; MPI_Offset extent, start; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - /* get command-line arguments */ - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for large header ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR /* create a new file for writing ----------------------------------------*/ - cmode = NC_CLOBBER | NC_64BIT_DATA; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR /* define a dimension of size = nprocs */ @@ -79,10 +68,8 @@ int main(int argc, char** argv) err = ncmpi_close(ncid); CHECK_ERR - if (err != NC_NOERR) goto err_out; - - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, - &ncid); CHECK_ERR + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_NOWRITE, info, &ncid); + CHECK_ERR /* inquire ID of the variable */ err = ncmpi_inq_varid(ncid, "var1", &varid); @@ -94,35 +81,37 @@ int main(int argc, char** argv) CHECK_ERR if (buf != rank) { - nerrs++; - printf("Error at line %d in %s: expecting read buf %d but got %d\n", + fprintf(stderr,"Error at line %d in %s: expecting read buf %d but got %d\n", __LINE__,__FILE__,rank,buf); + nerrs++; } err = ncmpi_close(ncid); CHECK_ERR -#ifdef PNC_MALLOC_TRACE - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) { - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - } - } - if (malloc_size > 0) ncmpi_inq_malloc_list(); -#endif - -err_out: - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + return nerrs; +} + +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ + + err = tst_main(argc, argv, "large header", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/largefile/large_reqs.c b/test/largefile/large_reqs.c index b55cbc4f9f..a61dff00e1 100644 --- a/test/largefile/large_reqs.c +++ b/test/largefile/large_reqs.c @@ -30,10 +30,10 @@ static int verbose; static -int tst_one_var(char *filename, MPI_Comm comm) +int tst_one_var(MPI_Comm comm, const char *filename, MPI_Info info) { size_t i, buf_len; - int rank, nprocs, err, nerrs=0, ncid, cmode, varid, dimid[3], psize[2]; + int rank, nprocs, err, nerrs=0, ncid, varid, dimid[3], psize[2]; int *buf; MPI_Offset start[3], count[3]; @@ -46,8 +46,7 @@ int tst_one_var(char *filename, MPI_Comm comm) /* Test classic CDF-5 format */ /* create a new file for writing ----------------------------------------*/ - cmode = NC_CLOBBER | NC_64BIT_DATA; - err = ncmpi_create(comm, filename, cmode, MPI_INFO_NULL, &ncid); + err = ncmpi_create(comm, filename, NC_CLOBBER, info, &ncid); CHECK_ERR /* define dimensions Z, Y, and X */ @@ -150,11 +149,11 @@ int tst_one_var(char *filename, MPI_Comm comm) #define LEN 1024 static -int tst_vars(char *filename, MPI_Comm comm) +int tst_vars(MPI_Comm comm, const char *filename, MPI_Info info) { size_t i, buf_len; int rank, nprocs, err, nerrs=0, *buf, *buf_ptr; - int ncid, cmode, *varid, dimid[3], gap, psize[2]={0,0}; + int ncid, *varid, dimid[3], gap, psize[2]={0,0}; MPI_Offset start[3], count[3]; MPI_Comm_size(comm, &nprocs); @@ -166,8 +165,7 @@ int tst_vars(char *filename, MPI_Comm comm) /* Test classic CDF-5 format */ /* create a new file for writing ----------------------------------------*/ - cmode = NC_CLOBBER | NC_64BIT_DATA; - err = ncmpi_create(comm, filename, cmode, MPI_INFO_NULL, &ncid); + err = ncmpi_create(comm, filename, NC_CLOBBER, info, &ncid); CHECK_ERR /* define dimensions Z, Y, and X */ @@ -269,36 +267,24 @@ int tst_vars(char *filename, MPI_Comm comm) return nerrs; } -int main(int argc, char** argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) { - char filename[256]; - int rank, nprocs, nerrs=0, color; + int rank, nprocs, err, nerrs=0, color; MPI_Comm comm; -#ifdef PNC_MALLOC_TRACE - int err; -#endif + verbose = 0; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - /* get command-line arguments */ - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for large requests ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR color = 1; @@ -313,34 +299,38 @@ int main(int argc, char** argv) if (color) { /* test one big variable */ - nerrs += tst_one_var(filename, comm); + nerrs += tst_one_var(comm, out_path, info); /* test a large number of smaller variables */ - nerrs += tst_vars(filename, comm); + nerrs += tst_vars(comm, out_path, info); } if (comm != MPI_COMM_WORLD) MPI_Comm_free(&comm); -#ifdef PNC_MALLOC_TRACE - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - } - if (malloc_size > 0) ncmpi_inq_malloc_list(); -#endif + return nerrs; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 0; /* skip test intra-node aggregation */ + opt.drv = 0; /* test GIO driver only */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ + + err = tst_main(argc, argv, "large requests", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/largefile/large_var.c b/test/largefile/large_var.c index de12b0a187..84b88a2fd3 100644 --- a/test/largefile/large_var.c +++ b/test/largefile/large_var.c @@ -58,47 +58,27 @@ swapn(void *buf, } #endif -int main(int argc, char** argv) +static +int test_io_nc4(const char *out_path, + MPI_Info global_info) { - char filename[256]; size_t bufsize; - int i, j, rank, nprocs, err, nerrs=0, expected; - int ncid, cmode, varid, dimid[3], req[3], st[3], *buf, *buf_ptr; - MPI_Offset offset, var_offset, start[3], count[3]; - MPI_File fh; - MPI_Status status; + int i, rank, nprocs, err, nerrs=0, expected; + int ncid, varid, dimid[3], *buf; + MPI_Offset start[3], count[3]; MPI_Info info; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - /* get command-line arguments */ - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for writing to a large variable ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - - MPI_Info_create(&info); + MPI_Info_dup(global_info, &info); MPI_Info_set(info, "romio_ds_write", "disable"); MPI_Info_set(info, "romio_ds_read", "disable"); -#ifdef ENABLE_NETCDF4 /* Test NetCDF-4 feature */ + /* create a new file for writing ----------------------------------------*/ - cmode = NC_CLOBBER | NC_NETCDF4; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR /* define dimensions Z, Y, and X */ @@ -151,7 +131,7 @@ int main(int argc, char** argv) err = ncmpi_close(ncid); CHECK_ERR /* open the same file and read back for validation */ - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_NOWRITE, info, &ncid); CHECK_ERR err = ncmpi_inq_varid(ncid, "var", &varid); CHECK_ERR @@ -177,7 +157,7 @@ int main(int argc, char** argv) /* check if the contents of buf are expected */ for (i=0; i 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - } - if (malloc_size > 0) ncmpi_inq_malloc_list(); + return nerrs; +} + +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) +{ + int err; + + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + if (format == NC_FORMAT_NETCDF4) + return test_io_nc4(out_path, info); + + return test_io_nc5(out_path, info); +} + +int main(int argc, char **argv) { + + int err; +#if PNETCDF_DRIVER_NETCDF4 == 1 + int formats[] = {NC_FORMAT_NETCDF4, NC_FORMAT_64BIT_DATA}; +#else + int formats[] = {NC_FORMAT_64BIT_DATA}; #endif + loop_opts opt; - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ + + err = tst_main(argc, argv, "writing to a large variable", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/largefile/parallel_run.sh b/test/largefile/parallel_run.sh index 8df4783e3f..ad26c7a5b8 100755 --- a/test/largefile/parallel_run.sh +++ b/test/largefile/parallel_run.sh @@ -1,47 +1,41 @@ #!/bin/bash # -# Copyright (C) 2018, Northwestern University and Argonne National Laboratory +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory # See COPYRIGHT notice in top-level directory. # # Exit immediately if a command exits with a non-zero status. set -e -VALIDATOR=../../src/utils/ncvalidator/ncvalidator -NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff +DRY_RUN=no +VERBOSE=no -# remove file system type prefix if there is any -OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $MPIRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $MPIRUN $@ + fi +} MPIRUN=`echo ${TESTMPIRUN} | ${SED} -e "s/NP/$1/g"` # echo "MPIRUN = ${MPIRUN}" # echo "check_PROGRAMS=${check_PROGRAMS}" -# turn off safe mode for large tests -export PNETCDF_SAFE_MODE=0 - # prevent user environment setting of PNETCDF_HINTS to interfere unset PNETCDF_HINTS +if test "x$MIMIC_LUSTRE" != x1 ; then + PNETCDF_HINTS="cb_nodes=2" +fi + for i in ${check_PROGRAMS} ; do - # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - ${MPIRUN} ./$i ${TESTOUTDIR}/$i.nc - - # echo "--- validating file ${TESTOUTDIR}/$i.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.nc - rm -f ${OUTDIR}/$i.nc - # echo "" - - if test "x${ENABLE_BURST_BUFFER}" = x1 ; then - # echo "---- test burst buffering feature" - export PNETCDF_HINTS="nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" - ${MPIRUN} ./$i ${TESTOUTDIR}/$i.bb.nc - unset PNETCDF_HINTS - - # echo "--- validating file ${TESTOUTDIR}/$i.bb.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.bb.nc - fi - - rm -f ${OUTDIR}/$i.bb.nc -done + + exe_name=`basename $i` + + run_cmd ./$i -q -o ${TESTOUTDIR}/${exe_name}.nc + +done # check_PROGRAMS diff --git a/test/largefile/seq_runs.sh b/test/largefile/seq_runs.sh index e28955773d..eb3c73faf6 100755 --- a/test/largefile/seq_runs.sh +++ b/test/largefile/seq_runs.sh @@ -1,25 +1,29 @@ -#!/bin/sh +#!/bin/bash # -# Copyright (C) 2003, Northwestern University and Argonne National Laboratory +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory # See COPYRIGHT notice in top-level directory. # # Exit immediately if a command exits with a non-zero status. set -e -VALIDATOR=../../src/utils/ncvalidator/ncvalidator +DRY_RUN=no +VERBOSE=no -# remove file system type prefix if there is any -OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $TESTSEQRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $TESTSEQRUN $@ + fi +} -# disable safe mode, as large tests already run slow -export PNETCDF_SAFE_MODE=0 +exe_name=`basename $1` # prevent user environment setting of PNETCDF_HINTS to interfere unset PNETCDF_HINTS -for i in ${TESTPROGRAMS}; do - ${TESTSEQRUN} ./$i ${TESTOUTDIR}/$i.nc - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.nc - rm -f ${OUTDIR}/$i.nc -done +run_cmd ./$1 -q -o ${TESTOUTDIR}/${exe_name}.nc + diff --git a/test/largefile/tst_cdf5_begin.c b/test/largefile/tst_cdf5_begin.c index b965fba707..8f1cd208c2 100644 --- a/test/largefile/tst_cdf5_begin.c +++ b/test/largefile/tst_cdf5_begin.c @@ -33,14 +33,14 @@ #define CHECK_ERR { \ if (err != NC_NOERR) { \ nerrs++; \ - printf("Error at line %d in %s: (%s)\n", \ + fprintf(stderr,"Error at line %d in %s: (%s)\n", \ __LINE__,__FILE__,nc_strerror(err)); \ } \ } #define EXP_ERR(exp) { \ if (err != exp) { \ nerrs++; \ - printf("Error at line %d in %s: expecting %d but got %d\n", \ + fprintf(stderr,"Error at line %d in %s: expecting %d but got %d\n", \ __LINE__,__FILE__,exp, err); \ } \ } @@ -83,33 +83,26 @@ typedef MPI_Offset len_t; * contents of the possible overlaps between the two variables. */ -int main(int argc, char** argv) { - char filename[256]; +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) +{ int i, err, rank, nprocs, nerrs=0, ncid, dimid[2], varid[2]; short buf[10]; len_t start[1], count[1]; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for checking CDF-5 writes", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR - err = FileCreate(MPI_COMM_WORLD, filename, NC_CLOBBER|NC_64BIT_DATA, - MPI_INFO_NULL, &ncid); CHECK_ERR + err = FileCreate(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR err = DefDim(ncid, "dim0", NC_MAX_UINT, &dimid[0]); CHECK_ERR err = DefDim(ncid, "dim1", 10, &dimid[1]); CHECK_ERR @@ -135,38 +128,50 @@ int main(int argc, char** argv) { err = GetVaraShort(ncid, varid[0], start, count,buf); CHECK_ERR for (i=0; i<10; i++) { if (buf[i] != i) { - printf("Error at buf[%d] expect %d but got %hd\n",i,i,buf[i]); + fprintf(stderr,"Error at buf[%d] expect %d but got %hd\n",i,i,buf[i]); nerrs++; } } err = FileClose(ncid); CHECK_ERR /* check if open to read header fine */ - err = FileOpen(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); CHECK_ERR + err = FileOpen(MPI_COMM_WORLD, out_path, NC_NOWRITE, info, &ncid); + CHECK_ERR err = FileClose(ncid); CHECK_ERR #ifdef TEST_NETCDF if (nerrs) printf("fail with %d mismatches\n",nerrs); else printf("pass\n"); -#else -#ifdef PNC_MALLOC_TRACE - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR && malloc_size > 0) /* this test is for running 1 process */ - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - malloc_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); #endif - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + return nerrs; +} + +int main(int argc, char **argv) { + + int err; +#if PNETCDF_DRIVER_NETCDF4 == 1 + int formats[] = {NC_FORMAT_NETCDF4, NC_FORMAT_64BIT_DATA}; +#else + int formats[] = {NC_FORMAT_64BIT_DATA}; #endif + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ + + err = tst_main(argc, argv, "CDF-5 writes", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/largefile/tst_flarge.f90 b/test/largefile/tst_flarge.f90 index 7d557add52..f72cd92b4e 100644 --- a/test/largefile/tst_flarge.f90 +++ b/test/largefile/tst_flarge.f90 @@ -26,22 +26,29 @@ program tst_flarge integer :: cmode, err, ierr, get_args double precision dbl_buf(1) integer(KIND=MPI_OFFSET_KIND) :: start(1), count(1) - character(LEN=256) filename, cmd, msg + character(LEN=256) out_path, in_path, cmd, msg integer my_rank, p + logical keep_files + double precision timing call MPI_Init(ierr) + + timing = MPI_Wtime() + call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr) call MPI_Comm_size(MPI_COMM_WORLD, p, ierr) - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any if (my_rank .EQ. 0) then - filename = FILE_NAME - err = get_args(cmd, filename) + out_path = FILE_NAME + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr) ! if (p .ne. 1 .AND. my_rank .eq. 0) then ! print *, 'Warning: ',trim(cmd),' is design to run on 1 process' @@ -49,7 +56,7 @@ program tst_flarge ! Create the file with 2 NF90_DOUBLE vars, each with one really long dimension. cmode = IOR(NF90_CLOBBER, NF90_64BIT_DATA) - call check(nf90mpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, ncFileID)) + call check(nf90mpi_create(MPI_COMM_WORLD, out_path, cmode, MPI_INFO_NULL, ncFileID)) call check(nf90mpi_def_dim(ncFileID, dimName, BIG_DIMENSION, dimID)) call check(nf90mpi_def_var(ncFileID, var1Name, nf90_double, (/ dimID /), varID1) ) call check(nf90mpi_def_var(ncFileID, var2Name, nf90_double, (/ dimID /), varID2) ) @@ -70,7 +77,7 @@ program tst_flarge call check(nf90mpi_close(ncFileID)) ! Now open the file to read and check a few values - call check(nf90mpi_open(MPI_COMM_WORLD, filename, NF90_NOWRITE, MPI_INFO_NULL, ncFileID)) + call check(nf90mpi_open(MPI_COMM_WORLD, out_path, NF90_NOWRITE, MPI_INFO_NULL, ncFileID)) call check(nf90mpi_begin_indep_data(ncFileID)) start(1) = 1 call check(nf90mpi_get_var(ncFileID, VarID1, val1_in, start)) @@ -83,10 +90,21 @@ program tst_flarge call check(nf90mpi_close(ncFileID)) - msg = '*** TESTING F90 '//trim(cmd)//' for large files' - if (my_rank .eq. 0) call pass_fail(0, msg) + 999 timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, & + MPI_DOUBLE_PRECISION, MPI_MAX, & + MPI_COMM_WORLD, ierr) + + if (my_rank .eq. 0) then + if (.NOT. keep_files) then + err = nf90mpi_delete(out_path, MPI_INFO_NULL) + end if + + msg = '*** TESTING F90 '//trim(cmd)//' for large files' + call pass_fail(0, msg, timing) + endif - 999 call MPI_Finalize(ierr) + call MPI_Finalize(ierr) contains ! Internal subroutine - checks error status after each netcdf, prints out text message each time diff --git a/test/largefile/tst_hash_large_ndims.c b/test/largefile/tst_hash_large_ndims.c index f58eb41c00..c1111a2ef9 100644 --- a/test/largefile/tst_hash_large_ndims.c +++ b/test/largefile/tst_hash_large_ndims.c @@ -23,49 +23,34 @@ #define NDIMS 400000 -int main(int argc, char** argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) { - char filename[256]; - int i, rank, nprocs, err, nerrs=0, ncid, cmode, dimid, verbose=1; + size_t j; + int rank, nprocs, err, nerrs=0, ncid, dimid, verbose=0; double timing[3], max_timing[3]; #ifdef PNC_MALLOC_TRACE - MPI_Offset malloc_size[2], sum_size, max_size[2]; + MPI_Offset malloc_size[2], max_size[2]; #endif - MPI_Info info; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - /* get command-line arguments */ - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for hasing large ndims ", - basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - if (verbose && rank == 0) printf("\nNDIMS = %d\n", NDIMS); - MPI_Info_create(&info); MPI_Info_set(info, "nc_hash_size_dim", "2048"); - /* create a new file for writing ----------------------------------------*/ - cmode = NC_CLOBBER | NC_64BIT_DATA; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); CHECK_ERR - MPI_Info_free(&info); + /* create a new file for writing ----------------------------------------*/ + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR #ifdef PNC_MALLOC_TRACE err = ncmpi_inq_malloc_size(&malloc_size[0]); CHECK_ERR @@ -82,9 +67,9 @@ int main(int argc, char** argv) MPI_Barrier(MPI_COMM_WORLD); timing[0] = MPI_Wtime(); - for (i=0; i 0) { - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - } - } if (malloc_size[0] > 0) ncmpi_inq_malloc_list(); #endif @@ -163,13 +139,30 @@ int main(int argc, char** argv) if (verbose && rank == 0) printf("Time ncmpi_def_dim = %.4f ncmpi_enddef = %.4f ncmpi_close = %.4f\n", max_timing[0],max_timing[1],max_timing[2]); - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + return nerrs; +} + +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 0; /* skip test intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ + + err = tst_main(argc, argv, " hashing large ndims", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/largefile/tst_hash_large_ngattrs.c b/test/largefile/tst_hash_large_ngattrs.c index c6732c5d5f..710bf3ab88 100644 --- a/test/largefile/tst_hash_large_ngattrs.c +++ b/test/largefile/tst_hash_large_ngattrs.c @@ -23,49 +23,33 @@ #define NATTRS 400000 -int main(int argc, char** argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) { - char filename[256]; - int i, rank, nprocs, err, nerrs=0, ncid, cmode, verbose=1; + size_t j; + int rank, err, nerrs=0, ncid, verbose=0; double timing[3], max_timing[3]; #ifdef PNC_MALLOC_TRACE - MPI_Offset malloc_size[2], sum_size, max_size[2]; + MPI_Offset malloc_size[2], max_size[2]; #endif - MPI_Info info; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - - /* get command-line arguments */ - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for hashing of large gattr ", - basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } if (verbose && rank == 0) printf("\nNATTRS = %d\n", NATTRS); - MPI_Info_create(&info); MPI_Info_set(info, "nc_hash_size_gattr", "2048"); - /* create a new file for writing ----------------------------------------*/ - cmode = NC_CLOBBER | NC_64BIT_DATA; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); CHECK_ERR - MPI_Info_free(&info); + /* create a new file for writing ----------------------------------------*/ + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR #ifdef PNC_MALLOC_TRACE err = ncmpi_inq_malloc_size(&malloc_size[0]); CHECK_ERR @@ -82,10 +66,11 @@ int main(int argc, char** argv) MPI_Barrier(MPI_COMM_WORLD); timing[0] = MPI_Wtime(); - for (i=0; i 0) { - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - } - } if (malloc_size[0] > 0) ncmpi_inq_malloc_list(); #endif @@ -163,13 +139,30 @@ int main(int argc, char** argv) if (verbose && rank == 0) printf("Time ncmpi_put_att = %.4f ncmpi_enddef = %.4f ncmpi_close = %.4f\n", max_timing[0],max_timing[1],max_timing[2]); - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + return nerrs; +} + +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 0; /* skip test intra-node aggregation */ + opt.drv = 0; /* test GIO driver only */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ + + err = tst_main(argc, argv, "hashing of large number of gattr", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/largefile/tst_hash_large_nvars.c b/test/largefile/tst_hash_large_nvars.c index 5040eb8aac..321bad8aab 100644 --- a/test/largefile/tst_hash_large_nvars.c +++ b/test/largefile/tst_hash_large_nvars.c @@ -23,52 +23,36 @@ #define NVARS 400000 -int main(int argc, char** argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) { - char filename[256]; - int i, rank, nprocs, err, nerrs=0, ncid, cmode, dimid, *varid, verbose=1; + int i, rank, nprocs, err, nerrs=0, ncid, dimid, *varid, verbose=0; double timing[4], max_timing[4]; #ifdef PNC_MALLOC_TRACE - MPI_Offset malloc_size[2], sum_size, max_size[2]; + MPI_Offset malloc_size[2], max_size[2]; #endif - MPI_Info info; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - /* get command-line arguments */ - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for hashing of large nvars ", - basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - if (verbose && rank == 0) printf("\nNVARS = %d\n", NVARS); varid = (int*) malloc(sizeof(int) * NVARS); - MPI_Info_create(&info); MPI_Info_set(info, "nc_hash_size_var", "2048"); MPI_Info_set(info, "nc_hash_size_vattr", "2"); - /* create a new file for writing ----------------------------------------*/ - cmode = NC_CLOBBER | NC_64BIT_DATA; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); CHECK_ERR - MPI_Info_free(&info); + /* create a new file for writing ----------------------------------------*/ + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR #ifdef PNC_MALLOC_TRACE err = ncmpi_inq_malloc_size(&malloc_size[0]); CHECK_ERR @@ -181,15 +165,6 @@ int main(int argc, char** argv) printf("After ncmpi_close, PnetCDF memory footprint %4lld B\n", max_size[0]); } - - /* check if PnetCDF freed all internal malloc */ - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size[0], &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) { - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - } - } if (malloc_size[0] > 0) ncmpi_inq_malloc_list(); #endif @@ -198,13 +173,30 @@ int main(int argc, char** argv) printf("Time ncmpi_def_var = %.4f ncmpi_put_att = %.4f ncmpi_enddef = %.4f ncmpi_close = %.4f\n", max_timing[0],max_timing[1],max_timing[2],max_timing[3]); - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + return nerrs; +} + +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 0; /* skip test intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ + + err = tst_main(argc, argv, "hashing of large number of vars", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/largefile/wrap_runs.sh b/test/largefile/wrap_runs.sh deleted file mode 100755 index d9295eb10f..0000000000 --- a/test/largefile/wrap_runs.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2003, Northwestern University and Argonne National Laboratory -# See COPYRIGHT notice in top-level directory. -# - -# Exit immediately if a command exits with a non-zero status. -set -e - -VALIDATOR=../../src/utils/ncvalidator/ncvalidator -outfile=`basename $1` - -# remove file system type prefix if there is any -OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` - -# disable safe mode, as large tests already run slow -export PNETCDF_SAFE_MODE=0 - -# prevent user environment setting of PNETCDF_HINTS to interfere -unset PNETCDF_HINTS - -${TESTSEQRUN} $1 ${TESTOUTDIR}/$outfile.nc -${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$outfile.nc -rm -f ${OUTDIR}/$outfile.nc - -# echo "" - -if test "x${ENABLE_BURST_BUFFER}" = x1 ; then - echo "" - echo "---- testing burst buffering" - export PNETCDF_HINTS="nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" - ${TESTSEQRUN} $1 ${TESTOUTDIR}/$outfile.nc - unset PNETCDF_HINTS - - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$outfile.nc - - rm -f ${OUTDIR}/$outfile.nc -fi diff --git a/test/mpi/threads_open.c b/test/mpi/threads_open.c index a7d5dc048d..b3841921fd 100644 --- a/test/mpi/threads_open.c +++ b/test/mpi/threads_open.c @@ -29,7 +29,7 @@ char errorString[MPI_MAX_ERROR_STRING]; \ MPI_Comm_rank(MPI_COMM_WORLD, &rank); \ MPI_Error_string(err, errorString, &errorStringLen); \ - printf("rank %d: MPI error (%s) : %s\n", rank, err_msg, errorString); \ + fprintf(stderr,"rank %d: MPI error (%s) : %s\n", rank, err_msg, errorString); \ assert(err == MPI_SUCCESS); \ } \ } diff --git a/test/nc4/Makefile.am b/test/nc4/Makefile.am index b25c30dd9e..d6334aa180 100644 --- a/test/nc4/Makefile.am +++ b/test/nc4/Makefile.am @@ -15,7 +15,7 @@ AM_CPPFLAGS += -I$(srcdir)/../common AM_CPPFLAGS += -I$(top_builddir)/src/include AM_CPPFLAGS += @NETCDF4_INC@ -LDADD = $(top_builddir)/src/libs/libpnetcdf.la ../common/libtestutils.la +LDADD = ../common/libtestutils.la $(top_builddir)/src/libs/libpnetcdf.la LDADD += @NETCDF4_LDFLAGS@ @ADIOS_LDFLAGS@ @NETCDF4_LIBS@ @ADIOS_LIBS@ -lm if DECL_MPI_OFFSET @@ -66,25 +66,29 @@ check_PROGRAMS = $(TESTPROGRAMS) # AM_TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; TESTS_ENVIRONMENT = export SED="$(SED)"; TESTS_ENVIRONMENT += export srcdir="$(srcdir)"; +TESTS_ENVIRONMENT += export top_builddir="$(top_builddir)"; TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; TESTS_ENVIRONMENT += export TESTMPIRUN="$(TESTMPIRUN)"; -TESTS_ENVIRONMENT += export PNETCDF_DEBUG="$(PNETCDF_DEBUG)"; TESTS_ENVIRONMENT += export TESTPROGRAMS="$(TESTPROGRAMS)"; TESTS_ENVIRONMENT += export check_PROGRAMS="$(check_PROGRAMS)"; +TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER=@ENABLE_BURST_BUFFER@; +TESTS_ENVIRONMENT += export ENABLE_NETCDF4=@ENABLE_NETCDF4@; +TESTS_ENVIRONMENT += export ENABLE_THREAD_SAFE=@ENABLE_THREAD_SAFE@; +TESTS_ENVIRONMENT += export MIMIC_LUSTRE=@MIMIC_LUSTRE@; +TESTS_ENVIRONMENT += export ENABLE_GIO=@ENABLE_GIO@; + TESTS = $(TESTPROGRAMS) TEST_EXTENSIONS = .sh -LOG_COMPILER = $(srcdir)/wrap_runs.sh +LOG_COMPILER = $(srcdir)/seq_runs.sh SH_LOG_COMPILER = -NC_FILES = $(TESTPROGRAMS:%=$(TESTOUTDIR)/%.nc) +CLEANFILES = $(M4_SRCS:.m4=.c) -CLEANFILES = $(M4_SRCS:.m4=.c) \ - $(TESTOUTDIR)/put_get_all_kinds.nc.cdf4 \ - $(NC_FILES) testfile.nc +check_SCRIPTS = seq_runs.sh parallel_run.sh -EXTRA_DIST = $(M4_SRCS) wrap_runs.sh parallel_run.sh +EXTRA_DIST = $(M4_SRCS) seq_runs.sh parallel_run.sh ../common/libtestutils.la: set -e; cd ../common && $(MAKE) $(MFLAGS) tests @@ -115,7 +119,7 @@ ptest6: $(check_PROGRAMS) @$(TESTS_ENVIRONMENT) \ $(srcdir)/parallel_run.sh 6 || exit 1 -ptests: ptest2 ptest4 ptest6 +ptests: ptest4 ptest6 ptest8 ptest10: # build check targets but not invoke diff --git a/test/nc4/compressed.c b/test/nc4/compressed.c index e9e18f9d8a..793dd189cf 100644 --- a/test/nc4/compressed.c +++ b/test/nc4/compressed.c @@ -15,12 +15,16 @@ #define FNAME "gzip_example.nc" int main(int argc, char **argv) { - int err, nerrs=0, rank, np; + int err, nerrs=0, rank; int ncid, ndims, nvars; char *dir_name=".", filename[512]; + double timing; + MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &np); + + timing = MPI_Wtime(); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (argc > 2) { @@ -32,8 +36,8 @@ int main(int argc, char **argv) { if (rank == 0) { char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for reading compressed file", basename(argv[0])); - printf("%-66s ------ ", cmd_str); + sprintf(cmd_str, "*** TESTING C %s - reading compressed file", basename(argv[0])); + printf("%-63s -- ", cmd_str); free(cmd_str); } @@ -60,10 +64,12 @@ int main(int argc, char **argv) { sum_size); } + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf(PASS_STR, timing); } MPI_Finalize(); diff --git a/test/nc4/interoperability_rd.m4 b/test/nc4/interoperability_rd.m4 index 0bfa86d80c..1acdc82c5d 100644 --- a/test/nc4/interoperability_rd.m4 +++ b/test/nc4/interoperability_rd.m4 @@ -75,7 +75,7 @@ int pnc_rd_$2(int rank, int ncid, int vid, int *did){ /* Compare result */ for(i = 0; i < 5; i++){ if (buf[i] != ans[i]){ - printf("Error at line %d in %s: expecting buf[%d] = %$1 but got %$1\n", __LINE__, __FILE__, i, ans[i], buf[i]); + fprintf(stderr,"Error at line %d in %s: expecting buf[%d] = %$1 but got %$1\n", __LINE__, __FILE__, i, ans[i], buf[i]); } } @@ -149,7 +149,12 @@ int main(int argc, char **argv) { int did[2]; char *filename = FILE_NAME; + double timing; + MPI_Init(&argc, &argv); + + timing = MPI_Wtime(); + MPI_Comm_size(MPI_COMM_WORLD, &np); MPI_Comm_rank(MPI_COMM_WORLD, &rank); @@ -162,8 +167,8 @@ int main(int argc, char **argv) { if (rank == 0) { char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for interoperability file", basename(argv[0])); - printf("%-66s ------ ", cmd_str); + sprintf(cmd_str, "*** TESTING C %s - interoperability file", basename(argv[0])); + printf("%-63s -- ", cmd_str); free(cmd_str); } @@ -227,10 +232,12 @@ err_out: sum_size); } + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf(PASS_STR, timing); } MPI_Finalize(); diff --git a/test/nc4/interoperability_wr.m4 b/test/nc4/interoperability_wr.m4 index fcbf47cdd7..16bc22ceb5 100644 --- a/test/nc4/interoperability_wr.m4 +++ b/test/nc4/interoperability_wr.m4 @@ -77,7 +77,7 @@ int nc_rd_$2(int rank, int ncid, int vid, int *did){ /* Compare result */ for(i = 0; i < 5; i++){ if (buf[i] != ans[i]){ - printf("Error at line %d in %s: expecting buf[%d] = %$1 but got %$1\n", __LINE__, __FILE__, i, ans[i], buf[i]); + fprintf(stderr,"Error at line %d in %s: expecting buf[%d] = %$1 but got %$1\n", __LINE__, __FILE__, i, ans[i], buf[i]); } } @@ -150,7 +150,12 @@ int main(int argc, char **argv) { int did[2]; char *filename = FILE_NAME; + double timing; + MPI_Init(&argc, &argv); + + timing = MPI_Wtime(); + MPI_Comm_size(MPI_COMM_WORLD, &np); MPI_Comm_rank(MPI_COMM_WORLD, &rank); @@ -163,8 +168,8 @@ int main(int argc, char **argv) { if (rank == 0) { char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for interoperability file", basename(argv[0])); - printf("%-66s ------ ", cmd_str); + sprintf(cmd_str, "*** TESTING C %s - interoperability file", basename(argv[0])); + printf("%-63s -- ", cmd_str); free(cmd_str); } @@ -231,10 +236,12 @@ foreach(`dt', (`(`0', `schar', `char')', dnl sum_size); } + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf(PASS_STR, timing); } MPI_Finalize(); diff --git a/test/nc4/noclobber.c b/test/nc4/noclobber.c index 6bad7447ab..6841f4aece 100644 --- a/test/nc4/noclobber.c +++ b/test/nc4/noclobber.c @@ -19,10 +19,14 @@ int main(int argc, char **argv) { char filename[256]; - int err, nerrs=0, ncid, cmode, rank, nprocs; + int err, nerrs=0, ncid, cmode, rank; + + double timing; MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + timing = MPI_Wtime(); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (argc > 2) { @@ -36,8 +40,8 @@ int main(int argc, char **argv) { if (rank == 0) { char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for NC_NOCLOBBER and NC_EEXIST ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); + sprintf(cmd_str, "*** TESTING C %s - NC_NOCLOBBER and NC_EEXIST ", basename(argv[0])); + printf("%-63s -- ", cmd_str); fflush(stdout); free(cmd_str); } @@ -68,10 +72,12 @@ int main(int argc, char **argv) { if (malloc_size > 0) ncmpi_inq_malloc_list(); } + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf(PASS_STR, timing); } MPI_Finalize(); diff --git a/test/nc4/notsupport.c b/test/nc4/notsupport.c index 2165067b80..55a1abdb95 100644 --- a/test/nc4/notsupport.c +++ b/test/nc4/notsupport.c @@ -27,7 +27,7 @@ #define EXPECT_ERR(err_no1) \ if (err != err_no1) { \ nerrs++; \ - printf("Error at line %d in %s: expect error code %s but got %s\n", \ + fprintf(stderr,"Error at line %d in %s: expect error code %s but got %s\n", \ __LINE__,__FILE__,ncmpi_strerrno(err_no1),ncmpi_strerrno(err)); \ } @@ -36,7 +36,7 @@ int main(int argc, char** argv) { char filename[256]; - int rank, nprocs, err, nerrs=0; + int rank, err, nerrs=0; int ncid, cmode, varid, dimid, buf; int v1, v2; MPI_Comm comm=MPI_COMM_WORLD; @@ -44,9 +44,13 @@ int main(int argc, char** argv) { MPI_Offset start[1] = {0}; MPI_Datatype btype; + double timing; + MPI_Init(&argc, &argv); + + timing = MPI_Wtime(); + MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &nprocs); if (argc > 2) { if (!rank) printf("Usage: %s [filename]\n",argv[0]); @@ -59,8 +63,8 @@ int main(int argc, char** argv) { if (rank == 0) { char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for error NC_ENOTSUPPORT ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); + sprintf(cmd_str, "*** TESTING C %s - error NC_ENOTSUPPORT ", basename(argv[0])); + printf("%-63s -- ", cmd_str); fflush(stdout); free(cmd_str); } @@ -102,12 +106,12 @@ int main(int argc, char** argv) { err = MPI_Type_vector(1, 2, 1, MPI_INT, &btype); if (err != MPI_SUCCESS ){ nerrs++; - printf("Error at line %d in %s: %d\n", __LINE__, __FILE__, err); + fprintf(stderr,"Error at line %d in %s: %d\n", __LINE__, __FILE__, err); } err = MPI_Type_commit(&btype); if (err != MPI_SUCCESS ){ nerrs++; - printf("Error at line %d in %s: %d\n", __LINE__, __FILE__, err); + fprintf(stderr,"Error at line %d in %s: %d\n", __LINE__, __FILE__, err); } else { err = ncmpi_put_var1_all(ncid, varid, start, &buf, 1, btype); EXPECT_ERR(NC_ENOTSUPPORT); @@ -116,7 +120,7 @@ int main(int argc, char** argv) { err = MPI_Type_free(&btype); if (err != MPI_SUCCESS ){ nerrs++; - printf("Error at line %d in %s: %d\n", __LINE__, __FILE__, err); + fprintf(stderr,"Error at line %d in %s: %d\n", __LINE__, __FILE__, err); } } @@ -142,10 +146,12 @@ int main(int argc, char** argv) { sum_size); } + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, comm); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf(PASS_STR, timing); } MPI_Finalize(); diff --git a/test/nc4/parallel_run.sh b/test/nc4/parallel_run.sh index 6e0dfa371b..0b049cfba1 100755 --- a/test/nc4/parallel_run.sh +++ b/test/nc4/parallel_run.sh @@ -14,32 +14,17 @@ MPIRUN=`echo ${TESTMPIRUN} | ${SED} -e "s/NP/$1/g"` # echo "MPIRUN = ${MPIRUN}" # echo "check_PROGRAMS=${check_PROGRAMS}" -# echo "PNETCDF_DEBUG = ${PNETCDF_DEBUG}" -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" -else - safe_modes="0" -fi - # prevent user environment setting of PNETCDF_HINTS to interfere unset PNETCDF_HINTS for i in ${check_PROGRAMS} ; do - for j in ${safe_modes} ; do - for intra_aggr in 0 1 ; do - if test "$j" = 1 ; then # test only in safe mode - export PNETCDF_HINTS="romio_no_indep_rw=true" - else - export PNETCDF_HINTS= - fi - if test "$intra_aggr" = 1 ; then - export PNETCDF_HINTS="${PNETCDF_HINTS};nc_num_aggrs_per_node=2" - fi - export PNETCDF_SAFE_MODE=$j - # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - ${MPIRUN} ./$i ${TESTOUTDIR}/$i.nc - done - done + PNETCDF_HINTS= + if test "x$MIMIC_LUSTRE" != x1 ; then + PNETCDF_HINTS="cb_nodes=2;$PNETCDF_HINTS" + fi + export PNETCDF_HINTS="$PNETCDF_HINTS" + + ${MPIRUN} ./$i ${TESTOUTDIR}/$i.nc rm -f ${OUTDIR}/$i.nc rm -f ${OUTDIR}/$i.nc.cdf4 done diff --git a/test/nc4/pres_temp_4D.c b/test/nc4/pres_temp_4D.c index d37acfde48..2fb1365e4e 100644 --- a/test/nc4/pres_temp_4D.c +++ b/test/nc4/pres_temp_4D.c @@ -283,13 +283,13 @@ pres_temp_4D_rd(char *filename, int rank, int nprocs) /* Check the coordinate variable data. */ for (lat = 0; lat < NLAT; lat++) if (lats[lat] != START_LAT + 5.*lat) { - printf("\nError at line %d in %s: expect %e but got %e\n", __LINE__,__FILE__, START_LAT+5.*lat,lats[lat]); + fprintf(stderr,"\nError at line %d in %s: expect %e but got %e\n", __LINE__,__FILE__, START_LAT+5.*lat,lats[lat]); nerrs++; goto fn_exit; } for (lon = 0; lon < NLON; lon++) if (lons[lon] != START_LON + 5.*lon) { - printf("\nError at line %d in %s: expect %e but got %e\n", __LINE__,__FILE__, START_LON+5.*lon,lons[lon]); + fprintf(stderr,"\nError at line %d in %s: expect %e but got %e\n", __LINE__,__FILE__, START_LON+5.*lon,lons[lon]); nerrs++; goto fn_exit; } @@ -349,12 +349,12 @@ pres_temp_4D_rd(char *filename, int rank, int nprocs) for (lat = 0; lat < NLAT; lat++) for (lon = 0; lon < NLON; lon++) { if (pres_in[lvl][lat*NLON+lon] != SAMPLE_PRESSURE + i) { - printf("\nError at line %d in %s: expect %e but got %e\n", __LINE__,__FILE__, SAMPLE_PRESSURE+i,pres_in[lvl][lat*NLON+lon]); + fprintf(stderr,"\nError at line %d in %s: expect %e but got %e\n", __LINE__,__FILE__, SAMPLE_PRESSURE+i,pres_in[lvl][lat*NLON+lon]); nerrs++; goto fn_exit; } if (temp_in[lvl][lat*NLON+lon] != SAMPLE_TEMP + i) { - printf("\nError at line %d in %s: expect %e but got %e\n", __LINE__,__FILE__, SAMPLE_TEMP+i,temp_in[lvl][lat*NLON+lon]); + fprintf(stderr,"\nError at line %d in %s: expect %e but got %e\n", __LINE__,__FILE__, SAMPLE_TEMP+i,temp_in[lvl][lat*NLON+lon]); nerrs++; goto fn_exit; } @@ -382,7 +382,12 @@ main(int argc, char ** argv) char filename[256]; int nprocs, rank, err, nerrs=0; + double timing; + MPI_Init(&argc, &argv); + + timing = MPI_Wtime(); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); MPI_Comm_rank(MPI_COMM_WORLD, &rank); @@ -397,8 +402,8 @@ main(int argc, char ** argv) if (rank == 0) { char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for NetCDF4 file", basename(argv[0])); - printf("%-66s ------ ", cmd_str); + sprintf(cmd_str, "*** TESTING C %s - NetCDF4 file", basename(argv[0])); + printf("%-63s -- ", cmd_str); free(cmd_str); } @@ -417,10 +422,12 @@ main(int argc, char ** argv) sum_size); } + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + if (nerrs) fprintf(stderr,FAIL_STR,nerrs); + else printf(PASS_STR, timing); } MPI_Finalize(); diff --git a/test/nc4/put_get_all_kinds.m4 b/test/nc4/put_get_all_kinds.m4 index f5b1527482..632053aa5d 100644 --- a/test/nc4/put_get_all_kinds.m4 +++ b/test/nc4/put_get_all_kinds.m4 @@ -118,7 +118,7 @@ define(`TEST_CDF_FORMAT',dnl sprintf(fname, "%s.cdf%d",filename, $1); err = ncmpi_create(MPI_COMM_WORLD, fname, cmode, info, &ncid); if (err != NC_NOERR) { - printf("Error at line %d in %s: ncmpi_create() file %s (%s)\n", + fprintf(stderr,"Error at line %d in %s: ncmpi_create() file %s (%s)\n", __LINE__,__FILE__,fname,ncmpi_strerror(err)); MPI_Abort(MPI_COMM_WORLD, -1); exit(1); @@ -155,7 +155,12 @@ int main(int argc, char **argv) MPI_Offset startM[NDIMS], countM[NDIMS]; MPI_Info info; - MPI_Init(&argc,&argv); + double timing; + + MPI_Init(&argc, &argv); + + timing = MPI_Wtime(); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); @@ -170,8 +175,8 @@ int main(int argc, char **argv) if (rank == 0) { char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for all kinds put APIs ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); + sprintf(cmd_str, "*** TESTING C %s - all kinds put APIs ", basename(argv[0])); + printf("%-63s -- ", cmd_str); fflush(stdout); free(cmd_str); } @@ -242,10 +247,12 @@ int main(int argc, char **argv) sum_size); } + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf(PASS_STR, timing); } MPI_Finalize(); diff --git a/test/nc4/rd_compressed.c b/test/nc4/rd_compressed.c index 19b7551081..7da36cddaa 100644 --- a/test/nc4/rd_compressed.c +++ b/test/nc4/rd_compressed.c @@ -63,12 +63,16 @@ static int create_nc4(char *filename) int main(int argc, char **argv) { char filename[512]; - int i, err, nerrs=0, rank, np; + int i, err, nerrs=0, rank; int ncid, ndims, nvars, varid, dimids[3], *buf; MPI_Offset bufLen; + double timing; + MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &np); + + timing = MPI_Wtime(); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (argc > 2) { @@ -81,8 +85,8 @@ int main(int argc, char **argv) { if (rank == 0) { char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for reading compressed NetCDF4 file", basename(argv[0])); - printf("%-66s ------ ", cmd_str); + sprintf(cmd_str, "*** TESTING C %s - reading compressed NetCDF4 file", basename(argv[0])); + printf("%-63s -- ", cmd_str); free(cmd_str); } @@ -111,13 +115,13 @@ int main(int argc, char **argv) { /* check if number of dimensions defined in the file is expected */ if (ndims != 3) { - printf("Error: expect ndims=3 but got %d\n", ndims); + fprintf(stderr,"Error: expect ndims=3 but got %d\n", ndims); nerrs++; goto fn_exit; } /* check if number of variables defined in the file is expected */ if (nvars != 1) { - printf("Error: expect nvars=1 but got %d\n", nvars); + fprintf(stderr,"Error: expect nvars=1 but got %d\n", nvars); nerrs++; goto fn_exit; } @@ -128,7 +132,7 @@ int main(int argc, char **argv) { /* check if number of dimensions of the variable is expected */ err = ncmpi_inq_varndims(ncid, varid, &ndims); CHECK_ERR if (ndims != 3) { - printf("Error: expect variable's ndims=3 but got %d\n", ndims); + fprintf(stderr,"Error: expect variable's ndims=3 but got %d\n", ndims); nerrs++; goto fn_exit; } @@ -150,7 +154,7 @@ int main(int argc, char **argv) { /* check the contents of variable */ for (i=0; i 2) { if (!rank) printf("Usage: %s [filename]\n",argv[0]); @@ -74,9 +78,9 @@ int main(int argc, char** argv) { if (rank == 0) { char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); sprintf(cmd_str, - "*** TESTING C %s for opening and reading a netcdf4 file", + "*** TESTING C %s - opening and reading a netcdf4 file", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); + printf("%-63s -- ", cmd_str); fflush(stdout); free(cmd_str); } @@ -124,19 +128,19 @@ int main(int argc, char** argv) { /* Check if number of dimensions matches */ err = ncmpi_inq_ndims(ncid, &ndim); CHECK_ERR if (ndim != 2) { - printf("Error at line %d in %s: expect ndim = %d, but got %d\n", + fprintf(stderr,"Error at line %d in %s: expect ndim = %d, but got %d\n", __LINE__, __FILE__, 2, ndim); } /* Check if dimension size matches */ err = ncmpi_inq_dim(ncid, x_dimid, tmp, &dlen); CHECK_ERR if (dlen != NX) { - printf("Error at line %d in %s: expect X dim size = %d, but got %d\n", + fprintf(stderr,"Error at line %d in %s: expect X dim size = %d, but got %d\n", __LINE__, __FILE__, NX, (int)dlen); } err = ncmpi_inq_dim(ncid, y_dimid, tmp, &dlen); CHECK_ERR if (dlen != NY) { - printf("Error at line %d in %s: expect X dim size = %d, but got %d\n", + fprintf(stderr,"Error at line %d in %s: expect X dim size = %d, but got %d\n", __LINE__, __FILE__, NY, (int)dlen); } @@ -145,7 +149,7 @@ int main(int argc, char** argv) { for(i = 0; i < NX; i++){ for(j = 0; j < NY; j++){ if (data_out[i][j] != data_in[i][j]){ - printf("Error at line %d in %s: expect data_in[%d][%d] = %d, but got %d\n", + fprintf(stderr,"Error at line %d in %s: expect data_in[%d][%d] = %d, but got %d\n", __LINE__, __FILE__, i, j, data_out[i][j], data_in[i][j]); } } @@ -162,10 +166,12 @@ int main(int argc, char** argv) { malloc_size); fn_exit: + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf(PASS_STR, timing); } MPI_Finalize(); diff --git a/test/nc4/tst_2_rec_dims.c b/test/nc4/tst_2_rec_dims.c index 2f8d40ae9c..3b7e11b3d6 100644 --- a/test/nc4/tst_2_rec_dims.c +++ b/test/nc4/tst_2_rec_dims.c @@ -26,16 +26,20 @@ int main(int argc, char** argv) { char filename[512]; - int i, rank, nprocs, err, nerrs=0, buf[16]; + int i, rank, err, nerrs=0, buf[16]; int ncid, mode, dimids[2], varid, num_rec_vars; MPI_Comm comm=MPI_COMM_WORLD; MPI_Info info=MPI_INFO_NULL; MPI_Offset recsize; size_t start[2], count[2]; + double timing; + MPI_Init(&argc, &argv); + + timing = MPI_Wtime(); + MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &nprocs); if (argc > 2) { if (!rank) printf("Usage: %s [filename]\n",argv[0]); @@ -48,8 +52,8 @@ int main(int argc, char** argv) { if (rank == 0) { char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for reading file with 2 rec dims ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); + sprintf(cmd_str, "*** TESTING C %s - reading file with 2 rec dims ", basename(argv[0])); + printf("%-63s -- ", cmd_str); fflush(stdout); free(cmd_str); } @@ -81,21 +85,21 @@ int main(int argc, char** argv) { err = ncmpi_inq_num_rec_vars(ncid, &num_rec_vars); CHECK_ERR if (num_rec_vars != 1){ - printf("Error at line %d of %s: expect num_rec_vars %d but got %d\n", + fprintf(stderr,"Error at line %d of %s: expect num_rec_vars %d but got %d\n", __LINE__,__FILE__,2,num_rec_vars); nerrs++; } err = ncmpi_inq_num_fix_vars(ncid, &num_rec_vars); CHECK_ERR if (num_rec_vars != 0){ - printf("Error at line %d of %s: expect num_fix_vars %d but got %d\n", + fprintf(stderr,"Error at line %d of %s: expect num_fix_vars %d but got %d\n", __LINE__,__FILE__,0,num_rec_vars); nerrs++; } err = ncmpi_inq_recsize(ncid, &recsize); CHECK_ERR if (recsize != 4){ - printf("Error at line %d of %s: expect recsize "OFFFMT" but got "OFFFMT"\n", + fprintf(stderr,"Error at line %d of %s: expect recsize "OFFFMT" but got "OFFFMT"\n", __LINE__,__FILE__, (MPI_Offset)4, recsize); nerrs++; } @@ -112,10 +116,12 @@ int main(int argc, char** argv) { sum_size); } + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, comm); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf(PASS_STR, timing); } MPI_Finalize(); diff --git a/test/nc4/tst_get_put_size.c b/test/nc4/tst_get_put_size.c index 9f05ff2488..90599afbee 100644 --- a/test/nc4/tst_get_put_size.c +++ b/test/nc4/tst_get_put_size.c @@ -31,7 +31,12 @@ int main(int argc, char** argv) { MPI_Offset start[2]; MPI_Offset size; + double timing; + MPI_Init(&argc, &argv); + + timing = MPI_Wtime(); + MPI_Comm_rank(comm, &rank); MPI_Comm_size(comm, &nprocs); @@ -46,8 +51,8 @@ int main(int argc, char** argv) { if (rank == 0) { char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for get size and put size ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); + sprintf(cmd_str, "*** TESTING C %s - get size and put size ", basename(argv[0])); + printf("%-63s -- ", cmd_str); fflush(stdout); free(cmd_str); } @@ -72,7 +77,7 @@ int main(int argc, char** argv) { err = ncmpi_put_var1_int_all(ncid, varid[0], start, buf); CHECK_ERR err = ncmpi_inq_put_size(ncid, &size); CHECK_ERR if (size != sizeof(int) * (i + 1)){ - printf("Error at line %d of %s: expect put_size = %ld but got "OFFFMT"\n", + fprintf(stderr,"Error at line %d of %s: expect put_size = %ld but got "OFFFMT"\n", __LINE__,__FILE__,sizeof(int) * (i + 1),size); nerrs++; } @@ -80,7 +85,7 @@ int main(int argc, char** argv) { err = ncmpi_get_var1_int_all(ncid, varid[0], start, buf); CHECK_ERR err = ncmpi_inq_get_size(ncid, &size); CHECK_ERR if (size != sizeof(int) * (i + 1)){ - printf("Error at line %d of %s: expect put_size = %ld but got "OFFFMT"\n", + fprintf(stderr,"Error at line %d of %s: expect put_size = %ld but got "OFFFMT"\n", __LINE__,__FILE__,sizeof(int) * (i + 1),size); nerrs++; } @@ -98,10 +103,12 @@ int main(int argc, char** argv) { sum_size); } + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, comm); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf(PASS_STR, timing); } MPI_Finalize(); diff --git a/test/nc4/tst_rec_vars.c b/test/nc4/tst_rec_vars.c index 3286881350..4f886e2165 100644 --- a/test/nc4/tst_rec_vars.c +++ b/test/nc4/tst_rec_vars.c @@ -30,7 +30,12 @@ int main(int argc, char** argv) { MPI_Info info=MPI_INFO_NULL; MPI_Offset start[1], count[1]; + double timing; + MPI_Init(&argc, &argv); + + timing = MPI_Wtime(); + MPI_Comm_rank(comm, &rank); MPI_Comm_size(comm, &nprocs); @@ -45,8 +50,8 @@ int main(int argc, char** argv) { if (rank == 0) { char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for record variables to NetCDF4 file ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); + sprintf(cmd_str, "*** TESTING C %s - record variables to NetCDF4 file", basename(argv[0])); + printf("%-63s -- ", cmd_str); fflush(stdout); free(cmd_str); } @@ -74,28 +79,28 @@ int main(int argc, char** argv) { err = ncmpi_inq_dimlen(ncid, dimid[0], start); CHECK_ERR if (start[0] != (MPI_Offset)nprocs){ - printf("Error at line %d of %s: expect NC_UNLIMITED dimension X of len "OFFFMT" but got "OFFFMT"\n", + fprintf(stderr,"Error at line %d of %s: expect NC_UNLIMITED dimension X of len "OFFFMT" but got "OFFFMT"\n", __LINE__,__FILE__,(MPI_Offset)nprocs,start[0]); nerrs++; } err = ncmpi_inq_num_rec_vars(ncid, &v1); CHECK_ERR if (v1 != 2){ - printf("Error at line %d of %s: expect num_rec_vars %d but got %d\n", + fprintf(stderr,"Error at line %d of %s: expect num_rec_vars %d but got %d\n", __LINE__,__FILE__,2,v1); nerrs++; } err = ncmpi_inq_num_fix_vars(ncid, &v1); CHECK_ERR if (v1 != 0){ - printf("Error at line %d of %s: expect num_fix_vars %d but got %d\n", + fprintf(stderr,"Error at line %d of %s: expect num_fix_vars %d but got %d\n", __LINE__,__FILE__,0,v1); nerrs++; } err = ncmpi_inq_recsize(ncid, start); CHECK_ERR if (start[0] != 8){ - printf("Error at line %d of %s: expect recsize "OFFFMT" but got "OFFFMT"\n", + fprintf(stderr,"Error at line %d of %s: expect recsize "OFFFMT" but got "OFFFMT"\n", __LINE__,__FILE__, (MPI_Offset)8, start[0]); nerrs++; } @@ -111,7 +116,7 @@ int main(int argc, char** argv) { buf[0] = -1; err = ncmpi_get_vara_int_all(ncid, varid[0], start, count, buf); CHECK_ERR if (buf[0] != rank + 1){ - printf("Error at line %d of %s: expect buf = %d but got %d\n", + fprintf(stderr,"Error at line %d of %s: expect buf = %d but got %d\n", __LINE__,__FILE__, rank + 1, buf[0]); nerrs++; } @@ -119,7 +124,7 @@ int main(int argc, char** argv) { start[0] = nprocs - rank - 1; err = ncmpi_get_vara_int_all(ncid, varid[1], start, count, buf); CHECK_ERR if (buf[0] != rank + 1){ - printf("Error at line %d of %s: expect buf = %d but got %d\n", + fprintf(stderr,"Error at line %d of %s: expect buf = %d but got %d\n", __LINE__,__FILE__, rank + 1, buf[0]); nerrs++; } @@ -136,10 +141,12 @@ int main(int argc, char** argv) { sum_size); } + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, comm); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf(PASS_STR, timing); } MPI_Finalize(); diff --git a/test/nc4/tst_zero_req.c b/test/nc4/tst_zero_req.c index 04d780c1ca..1a8929a22a 100644 --- a/test/nc4/tst_zero_req.c +++ b/test/nc4/tst_zero_req.c @@ -33,7 +33,7 @@ tst_fmt(char *filename, int cmode) cmode |= NC_CLOBBER; err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); if (err != NC_NOERR) { - printf("Error at %s:%d ncmpi_create %s\n",__FILE__,__LINE__, + fprintf(stderr,"Error at %s:%d ncmpi_create %s\n",__FILE__,__LINE__, ncmpi_strerrno(err)); return 1; } @@ -67,7 +67,7 @@ tst_fmt(char *filename, int cmode) err = ncmpi_inq_dimlen(ncid, dimid[0], &num_records); CHECK_ERR if (num_records != Y_LEN) { - printf("Error at %s:%d: expect num_records=%d but got "OFFFMT"\n", + fprintf(stderr,"Error at %s:%d: expect num_records=%d but got "OFFFMT"\n", __FILE__,__LINE__,Y_LEN,num_records); nerrs++; } @@ -77,7 +77,7 @@ tst_fmt(char *filename, int cmode) err = ncmpi_inq_dimlen(ncid, dimid[0], &num_records); CHECK_ERR if (num_records != Y_LEN) { - printf("Error at %s:%d: expect num_records=%d but got "OFFFMT"\n", + fprintf(stderr,"Error at %s:%d: expect num_records=%d but got "OFFFMT"\n", __FILE__,__LINE__,Y_LEN,num_records); nerrs++; } @@ -96,7 +96,7 @@ tst_fmt(char *filename, int cmode) int k= i*X_LEN+j; int expect=k+(rank * 100); if (buf[k] != expect) { - printf("Error at %s:%d: expect buf[%d]=%d but got %d\n", + fprintf(stderr,"Error at %s:%d: expect buf[%d]=%d but got %d\n", __FILE__,__LINE__,k,expect,buf[k]); nerrs++; i = Y_LEN; @@ -115,7 +115,7 @@ tst_fmt(char *filename, int cmode) int k=i*X_LEN+j; int expect=k+((rank+1)%nprocs * 100); if (buf[k] != expect) { - printf("Error at %s:%d: expect buf[%d]=%d but got %d\n", + fprintf(stderr,"Error at %s:%d: expect buf[%d]=%d but got %d\n", __FILE__,__LINE__,k,expect,buf[k]); nerrs++; i = Y_LEN; @@ -134,7 +134,12 @@ int main(int argc, char **argv) { char filename[256]; int rank=0, nerrs=0; - MPI_Init(&argc,&argv); + double timing; + + MPI_Init(&argc, &argv); + + timing = MPI_Wtime(); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (argc > 2) { @@ -148,8 +153,8 @@ int main(int argc, char **argv) { if (rank == 0) { char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for zero-length request ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); + sprintf(cmd_str, "*** TESTING C %s - zero-length request ", basename(argv[0])); + printf("%-63s -- ", cmd_str); fflush(stdout); free(cmd_str); } @@ -177,10 +182,12 @@ int main(int argc, char **argv) { if (malloc_size > 0) ncmpi_inq_malloc_list(); } + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf(PASS_STR, timing); } MPI_Finalize(); diff --git a/test/nc_test/Makefile.am b/test/nc_test/Makefile.am index 8cd608d83d..66a833ade6 100644 --- a/test/nc_test/Makefile.am +++ b/test/nc_test/Makefile.am @@ -14,7 +14,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/src/include AM_CPPFLAGS += -I$(srcdir)/../common AM_CPPFLAGS += -I$(top_builddir)/src/include -LDADD = $(top_builddir)/src/libs/libpnetcdf.la ../common/libtestutils.la +LDADD = ../common/libtestutils.la $(top_builddir)/src/libs/libpnetcdf.la LDADD += @NETCDF4_LDFLAGS@ @ADIOS_LDFLAGS@ @NETCDF4_LIBS@ @ADIOS_LIBS@ -lm if DECL_MPI_OFFSET @@ -62,32 +62,29 @@ check_PROGRAMS = $(TESTPROGRAMS) tst_nofill nc_test # AM_TESTS_ENVIRONMENT += TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)" ; export TESTOUTDIR; TESTS_ENVIRONMENT = export SED="$(SED)"; TESTS_ENVIRONMENT += export srcdir="$(srcdir)"; +TESTS_ENVIRONMENT += export top_builddir="$(top_builddir)"; TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; TESTS_ENVIRONMENT += export TESTMPIRUN="$(TESTMPIRUN)"; -TESTS_ENVIRONMENT += export PNETCDF_DEBUG="$(PNETCDF_DEBUG)"; TESTS_ENVIRONMENT += export TESTPROGRAMS="$(TESTPROGRAMS)"; TESTS_ENVIRONMENT += export check_PROGRAMS="$(check_PROGRAMS)"; -TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER="$(ENABLE_BURST_BUFFER)"; -TESTS_ENVIRONMENT += export ENABLE_NETCDF4="$(ENABLE_NETCDF4)"; +TESTS_ENVIRONMENT += export ENABLE_GIO=@ENABLE_GIO@; + +TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER=@ENABLE_BURST_BUFFER@; +TESTS_ENVIRONMENT += export ENABLE_NETCDF4=@ENABLE_NETCDF4@; +TESTS_ENVIRONMENT += export ENABLE_THREAD_SAFE=@ENABLE_THREAD_SAFE@; +TESTS_ENVIRONMENT += export MIMIC_LUSTRE=@MIMIC_LUSTRE@; TESTS = $(TESTPROGRAMS) seq_runs.sh TEST_EXTENSIONS = .sh LOG_COMPILER = $(srcdir)/wrap_runs.sh SH_LOG_COMPILER = -NC_FILES = $(TESTPROGRAMS:%=$(TESTOUTDIR)/%.nc) \ - $(TESTPROGRAMS:%=$(TESTOUTDIR)/%.bb.nc) - CLEANFILES = tests.h $(M4_SRCS:.m4=.c) \ - $(TESTOUTDIR)/test.nc \ - $(TESTOUTDIR)/scratch.nc \ - $(TESTOUTDIR)/tooth-fairy.nc \ - $(TESTOUTDIR)/tst_nofill.nc.fill \ - $(TESTOUTDIR)/tst_nofill.nc.nofill \ - $(TESTOUTDIR)/tst_atts3.nc.2 \ - $(TESTOUTDIR)/tst_atts3.bb.nc.2 \ - core core.* *.gcda *.gcno *.gcov gmon.out $(NC_FILES) + $(TESTOUTDIR)/test.nc \ + core core.* *.gcda *.gcno *.gcov gmon.out + +check_SCRIPTS = seq_runs.sh wrap_runs.sh EXTRA_DIST = error.h $(M4_SRCS) $(M4_HFILES) README seq_runs.sh wrap_runs.sh diff --git a/test/nc_test/nc_test.c b/test/nc_test/nc_test.c index 3d34ff9f7e..aea941f2f2 100644 --- a/test/nc_test/nc_test.c +++ b/test/nc_test/nc_test.c @@ -69,7 +69,7 @@ MPI_Info info; static void usage(char *progname) { -#ifdef ENABLE_NETCDF4 +#if PNETCDF_DRIVER_NETCDF4 == 1 error("%s [-hrv245] [-n ]\n", progname); #else error("%s [-hrv25] [-n ]\n", progname); @@ -79,7 +79,7 @@ usage(char *progname) error(" [-v] Verbose mode\n" ); error(" [-2] (with -c) create file with CDF-2 format\n" ); error(" [-5] (with -c) create file with CDF-5 format\n" ); -#ifdef ENABLE_NETCDF4 +#if PNETCDF_DRIVER_NETCDF4 == 1 error(" [-4] (with -c) create file with NetCDF-4 classic-model format\n" ); #endif error(" [-n ] max. number of messages per test (Default: 8)\n"); @@ -145,8 +145,12 @@ main(int argc, char *argv[]) (void) signal(SIGFPE, SIG_IGN); #endif + double timing; + MPI_Init(&argc, &argv); + timing = MPI_Wtime(); + cdf_format = 1; /* 1: CDF-1, 2: CDF-2 5: CDF-5 */ read_only = 0; /* assume may write in test dir as default */ verbose = 0; @@ -187,7 +191,7 @@ main(int argc, char *argv[]) #ifndef ENABLE_NETCDF4 if (cdf_format == 4) { - printf("Error: NetCDF-4 support is not enabled at configure time\n"); + fprintf(stderr,"Error: NetCDF-4 support is not enabled at configure time\n"); MPI_Finalize(); return 1; } @@ -579,8 +583,11 @@ main(int argc, char *argv[]) fn_exit: MPI_Info_free(&info); + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); + if (nfailsTotal == 0) { - printf(PASS_STR); + printf(PASS_STR, timing); } else { print("\n%s: expects 0 failures ... ",argv[0]); diff --git a/test/nc_test/seq_runs.sh b/test/nc_test/seq_runs.sh index e8cd748cb8..4dcbe71629 100755 --- a/test/nc_test/seq_runs.sh +++ b/test/nc_test/seq_runs.sh @@ -34,7 +34,7 @@ rm -f ${OUTDIR}/tooth-fairy.nc ${OUTDIR}/scratch.nc ${OUTDIR}/test.nc ${TESTSEQRUN} ./nc_test -2 -d ${TESTOUTDIR} ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/test.nc -if test "${ENABLE_NETCDF4}" = 1 ; then +if test "x${ENABLE_NETCDF4}" = x1 ; then rm -f ${OUTDIR}/tooth-fairy.nc ${OUTDIR}/scratch.nc ${OUTDIR}/test.nc ${TESTSEQRUN} ./nc_test -4 -d ${TESTOUTDIR} # Validator does not support nc4 diff --git a/test/nc_test/t_nc.c b/test/nc_test/t_nc.c index c15b8d52d7..ffe4a14e41 100644 --- a/test/nc_test/t_nc.c +++ b/test/nc_test/t_nc.c @@ -77,7 +77,7 @@ union getret }; -#define ERR {if (err != NC_NOERR) {printf("Error at %s line %d: %s\n",__func__,__LINE__,ncmpi_strerror(err)); return 1;}} +#define ERR {if (err != NC_NOERR) {fprintf(stderr,"Error at %s line %d: %s\n",__func__,__LINE__,ncmpi_strerror(err)); return 1;}} static void chkgot(nc_type type, union getret got, double check) @@ -112,32 +112,32 @@ static MPI_Offset sizes[] = { NC_UNLIMITED, SIZE_1 , SIZE_2 }; static const char * const dim_names[] = { "record", "ixx", "iyy"}; static int -createtestdims(int cdfid, size_t num_dims, const MPI_Offset *sizes, const char * const dim_names[]) +createtestdims(int cdfid, size_t ndims, const MPI_Offset *dim_sizes, const char * const names[]) { int dimid, err; - while(num_dims-- != 0) + while(ndims-- != 0) { - err = ncmpi_def_dim(cdfid, *dim_names++, *sizes, &dimid); ERR - sizes++; + err = ncmpi_def_dim(cdfid, *names++, *dim_sizes, &dimid); ERR + dim_sizes++; } return 0; } static int -testdims(int cdfid, size_t num_dims, MPI_Offset *sizes, const char * const dim_names[]) +testdims(int cdfid, size_t ndims, MPI_Offset *dim_sizes, const char * const names[]) { int ii, err; MPI_Offset size; char cp[NC_MAX_NAME]; - for(ii=0; (size_t) ii < num_dims; ii++, sizes++) + for(ii=0; (size_t) ii < ndims; ii++, dim_sizes++) { err = ncmpi_inq_dim(cdfid, ii, cp, &size); ERR - if( size != *sizes) + if( size != *dim_sizes) (void) fprintf(stderr, "%d: %lu != %lu\n", - ii, (unsigned long)size, (unsigned long)*sizes); - if ( size != *sizes) return 1; - if ( strcmp(cp, *dim_names++) != 0) return 1; + ii, (unsigned long)size, (unsigned long)*dim_sizes); + if ( size != *dim_sizes) return 1; + if ( strcmp(cp, *names++) != 0) return 1; } return 0; } @@ -195,11 +195,11 @@ static struct tcdfvar { #define NUM_TESTVARS 6 static int -createtestvars(int id, const struct tcdfvar *testvars, size_t count) +createtestvars(int id, const struct tcdfvar *vars, size_t count) { int ii, err; int varid; - const struct tcdfvar *vp = testvars; + const struct tcdfvar *vp = vars; for(ii = 0; (size_t) ii < count; ii++, vp++ ) { @@ -617,8 +617,12 @@ int main(int argc, char *argv[]) { char filename[256]; int rank, nprocs, cmode, err, nerrs=0; + double timing; MPI_Init(&argc, &argv); + + timing = MPI_Wtime(); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); MPI_Comm_rank(MPI_COMM_WORLD, &rank); @@ -660,10 +664,12 @@ int main(int argc, char *argv[]) sum_size); } + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf(PASS_STR, timing); } MPI_Finalize(); diff --git a/test/nc_test/test_get.m4 b/test/nc_test/test_get.m4 index 8474d1ab03..f66c9e3016 100644 --- a/test/nc_test/test_get.m4 +++ b/test/nc_test/test_get.m4 @@ -492,13 +492,16 @@ ifelse(`$1',`uchar',`ifdef(`PNETCDF',,``#'endif')') start[j] = 0; continue; } -#ifndef RELAX_COORD_BOUND - IF (err != NC_EINVALCOORDS) /* not allowed even when edge[j]==0 */ - EXPECT_ERR(NC_EINVALCOORDS, err) -#else - IF (err != NC_NOERR) /* allowed when edge[j]==0 */ - EXPECT_ERR(NC_NOERR, err) -#endif + + if (is_relax_coord_bound()) { + IF (err != NC_NOERR) /* allowed when edge[j]==0 */ + EXPECT_ERR(NC_NOERR, err) + } + else { + IF (err != NC_EINVALCOORDS) /* not allowed even when edge[j]==0 */ + EXPECT_ERR(NC_EINVALCOORDS, err) + } + start[j] = var_shape[i][j]+1; /* should cause NC_EINVALCOORDS */ err = GetVara($1)(ncid, i, start, edge, value); IF (err != NC_EINVALCOORDS) @@ -763,13 +766,16 @@ ifdef(`PNETCDF',`dnl start[j] = 0; continue; } -#ifndef RELAX_COORD_BOUND - IF (err != NC_EINVALCOORDS) /* not allowed even when edge[j]==0 */ - EXPECT_ERR(NC_EINVALCOORDS, err) -#else - IF (err != NC_NOERR) /* allowed when edge[j]==0 */ - EXPECT_ERR(NC_NOERR, err) -#endif + + if (is_relax_coord_bound()) { + IF (err != NC_NOERR) /* allowed when edge[j]==0 */ + EXPECT_ERR(NC_NOERR, err) + } + else { + IF (err != NC_EINVALCOORDS) /* not allowed even when edge[j]==0 */ + EXPECT_ERR(NC_EINVALCOORDS, err) + } + start[j] = var_shape[i][j]+1; /* should cause NC_EINVALCOORDS */ err = GetVars($1)(ncid, i, start, edge, stride, value); IF (err != NC_EINVALCOORDS) @@ -1063,13 +1069,16 @@ ifdef(`PNETCDF',`dnl start[j] = 0; continue; } -#ifndef RELAX_COORD_BOUND - IF (err != NC_EINVALCOORDS) /* not allowed even when edge[j]==0 */ - EXPECT_ERR(NC_EINVALCOORDS, err) -#else - IF (err != NC_NOERR) /* allowed when edge[j]==0 */ - EXPECT_ERR(NC_NOERR, err) -#endif + + if (is_relax_coord_bound()) { + IF (err != NC_NOERR) /* allowed when edge[j]==0 */ + EXPECT_ERR(NC_NOERR, err) + } + else { + IF (err != NC_EINVALCOORDS) /* not allowed even when edge[j]==0 */ + EXPECT_ERR(NC_EINVALCOORDS, err) + } + start[j] = var_shape[i][j]+1; /* should cause NC_EINVALCOORDS */ err = GetVarm($1)(ncid, i, start, edge, stride, imap, value); IF (err != NC_EINVALCOORDS) diff --git a/test/nc_test/test_iget.m4 b/test/nc_test/test_iget.m4 index 2d07ace88c..d51c009cea 100644 --- a/test/nc_test/test_iget.m4 +++ b/test/nc_test/test_iget.m4 @@ -627,14 +627,18 @@ ifdef(`PNETCDF',`dnl if (var_dimid[i][j] == RECDIM) continue; /* skip record dim */ start[j] = var_shape[i][j]; err = APIFunc(iget_vara)(ncid, i, start, edge, value, 0, datatype, &reqid); -#ifndef RELAX_COORD_BOUND - IF (err != NC_EINVALCOORDS) /* not allowed even when edge[j]==0 */ - EXPECT_ERR(NC_EINVALCOORDS, err) -#else - IF (err != NC_NOERR) /* allowed when edge[j]==0 */ - EXPECT_ERR(NC_NOERR, err) -#endif - ELSE_NOK + + if (is_relax_coord_bound()) { + IF (err != NC_NOERR) /* allowed when edge[j]==0 */ + EXPECT_ERR(NC_NOERR, err) + ELSE_NOK + } + else { + IF (err != NC_EINVALCOORDS) /* not allowed even when edge[j]==0 */ + EXPECT_ERR(NC_EINVALCOORDS, err) + ELSE_NOK + } + start[j] = var_shape[i][j]+1; /* out of boundary check */ err = APIFunc(iget_vara)(ncid, i, start, edge, value, 1, datatype, &reqid); IF (err != NC_EINVALCOORDS) @@ -865,17 +869,21 @@ ifdef(`PNETCDF',`dnl start[j] = 0; continue; } -#ifndef RELAX_COORD_BOUND - IF (err != NC_EINVALCOORDS) /* not allowed even when edge[j]==0 */ - EXPECT_ERR(NC_EINVALCOORDS, err) -#else - IF (err != NC_NOERR) /* allowed when edge[j]==0 */ - EXPECT_ERR(NC_NOERR, err) - err = APIFunc(wait_all)(ncid, 1, &reqid, &st); - assert(err == st); - IF (err != NC_NOERR) EXPECT_ERR(NC_NOERR, err) -#endif - ELSE_NOK + + if (is_relax_coord_bound()) { + IF (err != NC_NOERR) /* allowed when edge[j]==0 */ + EXPECT_ERR(NC_NOERR, err) + err = APIFunc(wait_all)(ncid, 1, &reqid, &st); + assert(err == st); + IF (err != NC_NOERR) EXPECT_ERR(NC_NOERR, err) + ELSE_NOK + } + else { + IF (err != NC_EINVALCOORDS) /* not allowed even when edge[j]==0 */ + EXPECT_ERR(NC_EINVALCOORDS, err) + ELSE_NOK + } + start[j] = var_shape[i][j]+1; /* out of boundary check */ err = iGetVara($1)(ncid, i, start, edge, value, &reqid); IF (err != NC_EINVALCOORDS) @@ -1108,14 +1116,18 @@ ifdef(`PNETCDF',`dnl if (var_dimid[i][j] == RECDIM) continue; /* skip record dim */ start[j] = var_shape[i][j]; err = APIFunc(iget_vars)(ncid, i, start, edge, stride, value, 0, datatype, &reqid); -#ifndef RELAX_COORD_BOUND - IF (err != NC_EINVALCOORDS) /* not allowed even when edge[j]==0 */ - EXPECT_ERR(NC_EINVALCOORDS, err) -#else - IF (err != NC_NOERR) /* allowed when edge[j]==0 */ - EXPECT_ERR(NC_NOERR, err) -#endif - ELSE_NOK + + if (is_relax_coord_bound()) { + IF (err != NC_NOERR) /* allowed when edge[j]==0 */ + EXPECT_ERR(NC_NOERR, err) + ELSE_NOK + } + else { + IF (err != NC_EINVALCOORDS) /* not allowed even when edge[j]==0 */ + EXPECT_ERR(NC_EINVALCOORDS, err) + ELSE_NOK + } + start[j] = var_shape[i][j]+1; /* out of boundary check */ err = APIFunc(iget_vars)(ncid, i, start, edge, stride, value, 1, datatype, &reqid); IF (err != NC_EINVALCOORDS) @@ -1378,17 +1390,21 @@ ifdef(`PNETCDF',`dnl start[j] = 0; continue; } -#ifndef RELAX_COORD_BOUND - IF (err != NC_EINVALCOORDS) /* not allowed even when edge[j]==0 */ - EXPECT_ERR(NC_EINVALCOORDS, err) -#else - IF (err != NC_NOERR) /* allowed when edge[j]==0 */ - EXPECT_ERR(NC_NOERR, err) - err = APIFunc(wait_all)(ncid, 1, &reqid, &st); - assert(err == st); - IF (err != NC_NOERR) EXPECT_ERR(NC_NOERR, err) -#endif - ELSE_NOK + + if (is_relax_coord_bound()) { + IF (err != NC_NOERR) /* allowed when edge[j]==0 */ + EXPECT_ERR(NC_NOERR, err) + err = APIFunc(wait_all)(ncid, 1, &reqid, &st); + assert(err == st); + IF (err != NC_NOERR) EXPECT_ERR(NC_NOERR, err) + ELSE_NOK + } + else { + IF (err != NC_EINVALCOORDS) /* not allowed even when edge[j]==0 */ + EXPECT_ERR(NC_EINVALCOORDS, err) + ELSE_NOK + } + start[j] = var_shape[i][j]+1; /* out of boundary check */ err = iGetVars($1)(ncid, i, start, edge, stride, value, &reqid); IF (err != NC_EINVALCOORDS) @@ -1643,14 +1659,18 @@ ifdef(`PNETCDF',`dnl if (var_dimid[i][j] == RECDIM) continue; /* skip record dim */ start[j] = var_shape[i][j]; err = APIFunc(iget_varm)(ncid, i, start, edge, stride, imap, value, 0, datatype, &reqid); -#ifndef RELAX_COORD_BOUND - IF (err != NC_EINVALCOORDS) /* not allowed even when edge[j]==0 */ - EXPECT_ERR(NC_EINVALCOORDS, err) -#else - IF (err != NC_NOERR) /* allowed when edge[j]==0 */ - EXPECT_ERR(NC_NOERR, err) -#endif - ELSE_NOK + + if (is_relax_coord_bound()) { + IF (err != NC_NOERR) /* allowed when edge[j]==0 */ + EXPECT_ERR(NC_NOERR, err) + ELSE_NOK + } + else { + IF (err != NC_EINVALCOORDS) /* not allowed even when edge[j]==0 */ + EXPECT_ERR(NC_EINVALCOORDS, err) + ELSE_NOK + } + start[j] = var_shape[i][j]+1; /* out of boundary check */ err = APIFunc(iget_varm)(ncid, i, start, edge, stride, imap, value, 1, datatype, &reqid); IF (err != NC_EINVALCOORDS) @@ -1920,17 +1940,21 @@ ifdef(`PNETCDF',`dnl start[j] = 0; continue; } -#ifndef RELAX_COORD_BOUND - IF (err != NC_EINVALCOORDS) /* not allowed even when edge[j]==0 */ - EXPECT_ERR(NC_EINVALCOORDS, err) -#else - IF (err != NC_NOERR) /* allowed when edge[j]==0 */ - EXPECT_ERR(NC_NOERR, err) - err = APIFunc(wait_all)(ncid, 1, &reqid, &st); - assert(err == st); - IF (err != NC_NOERR) EXPECT_ERR(NC_NOERR, err) -#endif - ELSE_NOK + + if (is_relax_coord_bound()) { + IF (err != NC_NOERR) /* allowed when edge[j]==0 */ + EXPECT_ERR(NC_NOERR, err) + err = APIFunc(wait_all)(ncid, 1, &reqid, &st); + assert(err == st); + IF (err != NC_NOERR) EXPECT_ERR(NC_NOERR, err) + ELSE_NOK + } + else { + IF (err != NC_EINVALCOORDS) /* not allowed even when edge[j]==0 */ + EXPECT_ERR(NC_EINVALCOORDS, err) + ELSE_NOK + } + start[j] = var_shape[i][j]+1; /* out of boundary check */ err = iGetVarm($1)(ncid, i, start, edge, stride, imap, value, &reqid); IF (err != NC_EINVALCOORDS) diff --git a/test/nc_test/test_iput.m4 b/test/nc_test/test_iput.m4 index d9f0da58ce..7bd902eea9 100644 --- a/test/nc_test/test_iput.m4 +++ b/test/nc_test/test_iput.m4 @@ -76,12 +76,12 @@ define(`CheckRange3', #include "tests.h" static double -hash2nc(const nc_type var_type, int var_rank, MPI_Offset *index) +hash2nc(const nc_type xtype, int v_rank, MPI_Offset *index) { double min; double max; - switch (var_type) { + switch (xtype) { /* no type conversion will happen for NC_CHAR, use in-memory limits */ case NC_CHAR: min = CHAR_MIN; max = (double)CHAR_MAX; break; case NC_BYTE: min = X_BYTE_MIN; max = (double)X_BYTE_MAX; break; @@ -98,16 +98,16 @@ hash2nc(const nc_type var_type, int var_rank, MPI_Offset *index) return NC_EBADTYPE; } - return MAX(min, MIN(max, hash(var_type, var_rank, index))); + return MAX(min, MIN(max, hash(xtype, v_rank, index))); } static int -dbls2ncs(size_t nels, int var_type, double *inBuf, void *outBuf) +dbls2ncs(size_t nels, int xtype, double *inBuf, void *outBuf) { size_t i; char *p = (char*)outBuf; for (i=0; i -#define ERR {if (err != NC_NOERR) {printf("Error at %s line %d: %s\n",__func__,__LINE__,ncmpi_strerror(err)); nerrs++;}} +#define ERR {if (err != NC_NOERR) {fprintf(stderr,"Error at %s line %d: %s\n",__func__,__LINE__,ncmpi_strerror(err)); nerrs++;}} #define ERRV {printf("Unexpected result at %s line %d\n",__func__,__LINE__); nerrs++;} static int verbose; @@ -135,21 +135,6 @@ tst_atts3(char *filename, int cmode) { char filename2[256]; int err, nerrs=0; - signed char schar_in[ATT_LEN], schar_out[ATT_LEN] = {NC_MIN_BYTE, 1, NC_MAX_BYTE}; - unsigned char uchar_in[ATT_LEN]; - short short_in[ATT_LEN], short_out[ATT_LEN] = {NC_MIN_SHORT, -128, NC_MAX_SHORT}; - int int_in[ATT_LEN], int_out[ATT_LEN] = {-100000, 127, 100000}; - float float_in[ATT_LEN], float_out[ATT_LEN] = {-0.5, 0.25, 0.125}; - double double_in[ATT_LEN], double_out[ATT_LEN] = {-0.25, .5, 0.125}; - long long longlong_in[ATT_LEN] = {-1LL, -1LL, -1LL}; -#ifdef USE_NETCDF4 - long long_in[ATT_LEN]; - unsigned short ushort_in[ATT_LEN], ushort_out[ATT_LEN] = {0, 128, NC_MAX_USHORT}; - unsigned int uint_in[ATT_LEN], uint_out[ATT_LEN] = {0, 128, NC_MAX_UINT}; - long long longlong_out[ATT_LEN] = {-3123456789LL, 128LL, 3123456789LL}; - unsigned long long ulonglong_in[ATT_LEN] = {NC_MAX_UINT64, NC_MAX_UINT64, NC_MAX_UINT64}; - unsigned long long ulonglong_out[ATT_LEN] = {0LL, 128LL, 3123456789LL}; -#endif (void) signal(SIGFPE, SIG_IGN); @@ -186,12 +171,18 @@ tst_atts3(char *filename, int cmode) if (verbose) printf("ok\n"); if (verbose) printf("*** testing simple global atts..."); { - int ncid; + char *speech_in; + int i, ncid; nc_type att_type; MPI_Offset att_len; - int i; - char *speech_in; + unsigned char uchar_in[ATT_LEN]; + signed char schar_in[ATT_LEN], schar_out[ATT_LEN] = {NC_MIN_BYTE, 1, NC_MAX_BYTE}; + int int_in[ATT_LEN], int_out[ATT_LEN] = {-100000, 127, 100000}; + short short_in[ATT_LEN], short_out[ATT_LEN] = {NC_MIN_SHORT, -128, NC_MAX_SHORT}; + float float_in[ATT_LEN], float_out[ATT_LEN] = {-0.5, 0.25, 0.125}; + double double_in[ATT_LEN], double_out[ATT_LEN] = {-0.25, .5, 0.125}; + long long longlong_in[ATT_LEN] = {-1LL, -1LL, -1LL}; /* This won't work, because classic files can't create these types. */ err=ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL,&ncid); ERR @@ -254,8 +245,14 @@ tst_atts3(char *filename, int cmode) if (verbose) printf("*** testing attribute data type conversions..."); { - int ncid; - int i; + int i, ncid; + unsigned char uchar_in[ATT_LEN]; + signed char schar_in[ATT_LEN], schar_out[ATT_LEN] = {NC_MIN_BYTE, 1, NC_MAX_BYTE}; + short short_in[ATT_LEN], short_out[ATT_LEN] = {NC_MIN_SHORT, -128, NC_MAX_SHORT}; + int int_in[ATT_LEN], int_out[ATT_LEN] = {-100000, 127, 100000}; + float float_in[ATT_LEN], float_out[ATT_LEN] = {-0.5, 0.25, 0.125}; + double double_in[ATT_LEN], double_out[ATT_LEN] = {-0.25, .5, 0.125}; + long long longlong_in[ATT_LEN] = {-1LL, -1LL, -1LL}; /* Reopen the file and try different type conversions. */ err=ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); ERR @@ -410,8 +407,6 @@ tst_atts3(char *filename, int cmode) { int ncid; - /*int int_in[ATT_LEN], int_out[ATT_LEN] = {NC_MIN_INT, 128, NC_MAX_INT};*/ - /* Create a file with a global attribute of each type of zero length. */ err=ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL,&ncid); ERR err=ncmpi_put_att_text(ncid, NC_GLOBAL, ATT_TEXT_NAME, 0, NULL); ERR @@ -428,7 +423,6 @@ tst_atts3(char *filename, int cmode) int ncid; signed char schar_in[ATT_LEN]; short short_in[ATT_LEN]; - /*int int_in[ATT_LEN], int_out[ATT_LEN] = {NC_MIN_INT, 128, NC_MAX_INT};*/ int int_in[ATT_LEN]; float float_in[ATT_LEN]; double double_in[ATT_LEN]; @@ -762,8 +756,12 @@ int main(int argc, char *argv[]) { char filename[256]; int cmode, rank, nprocs, err, nerrs=0; + double timing; MPI_Init(&argc, &argv); + + timing = MPI_Wtime(); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); MPI_Comm_rank(MPI_COMM_WORLD, &rank); @@ -804,9 +802,11 @@ int main(int argc, char *argv[]) sum_size); } + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf(PASS_STR, timing); } MPI_Finalize(); diff --git a/test/nc_test/tst_misc.c b/test/nc_test/tst_misc.c index 4f23fcf719..743d12ba07 100644 --- a/test/nc_test/tst_misc.c +++ b/test/nc_test/tst_misc.c @@ -29,8 +29,12 @@ main(int argc, char **argv) { char *cmd_str, *path, filename[256]; int rank, nprocs, err, nerrs=0, ncid; + double timing; MPI_Init(&argc, &argv); + + timing = MPI_Wtime(); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); MPI_Comm_rank(MPI_COMM_WORLD, &rank); @@ -84,7 +88,7 @@ main(int argc, char **argv) for index i == 2. Not sure why, but this is a work around. */ if(openstat != NC_ENOTNC && openstat != NC_ENOENT && openstat != NC_EFILE) { /* older version of OpenMPI and MPICH may return MPI_ERR_IO instead of MPI_ERR_NO_SUCH_FILE */ - printf("Error %s line %d: Expecting error code %d or %d but got %d\n", + fprintf(stderr,"Error %s line %d: Expecting error code %d or %d but got %d\n", __FILE__,__LINE__,NC_ENOTNC,NC_ENOENT,openstat); nerrs++; } @@ -106,10 +110,13 @@ main(int argc, char **argv) } fn_exit: + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); + MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf(PASS_STR, timing); } MPI_Finalize(); diff --git a/test/nc_test/tst_names.c b/test/nc_test/tst_names.c index cc81615cd7..6d7c486311 100644 --- a/test/nc_test/tst_names.c +++ b/test/nc_test/tst_names.c @@ -22,8 +22,8 @@ #include -#define ERROR {printf("Error at line %d: %s\n",__LINE__,ncmpi_strerror(res)); nerrs++;} -#define ERRORI {printf("Error at line %d (loop=%d): %s\n",__LINE__,i,ncmpi_strerror(res)); nerrs++;} +#define ERROR {fprintf(stderr,"Error at line %d: %s\n",__LINE__,ncmpi_strerror(res)); nerrs++;} +#define ERRORI {fprintf(stderr,"Error at line %d (loop=%d): %s\n",__LINE__,i,ncmpi_strerror(res)); nerrs++;} /* The data file we will create. */ #define NDIMS 1 @@ -75,11 +75,11 @@ main(int argc, char **argv) "has ascii_del_\x7f_in name", /* Invalid UTF-8 of various sorts, thanks to Markus Kuhn */ "\xA0\xB0\xC0\xD0", - "xyz\x80", /* unexpected continuation bytes */ + "xyz\x80", /* unexpected continuation bytes */ "\x80xyz", "xyz\xBF", "\xBFxyz", - "\xC0xyz", /* lonely start characters */ + "\xC0xyz", /* lonely start characters */ "x\xC0yz", "xy\xC0z", "xyz\xC0", @@ -125,7 +125,7 @@ main(int argc, char **argv) "x\xFDyz", "xy\xFDz", "xyz\xFD", - "\xC0\xC0xy", /* last continuation byte missing */ + "\xC0\xC0xy", /* last continuation byte missing */ "x\xC0\xC0y", "xy\xC0\xC0", "\xDF\xDFxy", @@ -149,7 +149,7 @@ main(int argc, char **argv) "x\xFC\x80\x80\x80\x80", "\xFD\x80\x80\x80\x80x", "x\xFD\x80\x80\x80\x80", - "\xFExyz", /* impossible bytes */ + "\xFExyz", /* impossible bytes */ "x\xFEyz", "xy\xFEz", "xyz\xFE", @@ -157,7 +157,7 @@ main(int argc, char **argv) "x\xFFyz", "xy\xFFz", "xyz\xFF", - "\xC0\xAFxy", /* overlong sequences */ + "\xC0\xAFxy", /* overlong sequences */ "x\xC0\xAFy", "xy\xC0\xAF", "\xE0\x80\xAFx", @@ -179,7 +179,7 @@ main(int argc, char **argv) "x\xF8\x87\xBF\xBF\xBF", "\xFC\x83\xBF\xBF\xBF\xBFx", "x\xFC\x83\xBF\xBF\xBF\xBF", - "x\xC0\x80", /* overlong NULs */ + "x\xC0\x80", /* overlong NULs */ "x\xE0\x80\x80", "x\xF0\x80\x80\x80", "x\xF8\x80\x80\x80\x80", @@ -202,7 +202,7 @@ main(int argc, char **argv) "x\xED\xAF\xBF\xED\xBF\xBF" #if 0 /* The two below is legal since UTF8PROC_VERSION_MAJOR 2 */ - "x\xEF\xBF\xBE", /* other illegal code positions */ + "x\xEF\xBF\xBE", /* other illegal code positions */ "x\xEF\xBF\xBF" #endif }; @@ -225,8 +225,12 @@ main(int argc, char **argv) char filename[256]; int rank, nprocs, err, nerrs=0; + double timing; MPI_Init(&argc, &argv); + + timing = MPI_Wtime(); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); MPI_Comm_rank(MPI_COMM_WORLD, &rank); @@ -252,63 +256,66 @@ main(int argc, char **argv) printf("*** switching to netCDF %s format...", format_names[j]); #endif if((res = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER|cmode[j], MPI_INFO_NULL, &ncid))) - ERROR + ERROR /* Define dimensions, variables, and attributes with various - * acceptable names */ + * acceptable names */ for (i = 0; i < NUM_GOOD; i++) { - if ((res = ncmpi_def_dim(ncid, valid[i], DIMLEN, &dimid))) - ERRORI + if ((res = ncmpi_def_dim(ncid, valid[i], DIMLEN, &dimid))) + ERRORI - dimids[i] = dimid; - /* Define variable with same name */ - if ((res = ncmpi_def_var(ncid, valid[i], NC_FLOAT, NDIMS, &dimids[i], &varid))) - ERRORI - varids[i] = varid; - /* Define variable and global attributes with same name and value */ - if ((res = ncmpi_put_att_text(ncid, varid, valid[i], strlen(valid[i]), valid[i]))) - ERRORI - if ((res = ncmpi_put_att_double(ncid, NC_GLOBAL, valid[i], NC_DOUBLE, NATTVALS, attvals))) - ERRORI + dimids[i] = dimid; + /* Define variable with same name */ + if ((res = ncmpi_def_var(ncid, valid[i], NC_FLOAT, NDIMS, &dimids[i], &varid))) + ERRORI + if ((res = ncmpi_def_var_fill(ncid, varid, 0, NULL))) + ERRORI + + varids[i] = varid; + /* Define variable and global attributes with same name and value */ + if ((res = ncmpi_put_att_text(ncid, varid, valid[i], strlen(valid[i]), valid[i]))) + ERRORI + if ((res = ncmpi_put_att_double(ncid, NC_GLOBAL, valid[i], NC_DOUBLE, NATTVALS, attvals))) + ERRORI } /* Try defining dimensions, variables, and attributes with various - * bad names and make sure these are rejected */ + * bad names and make sure these are rejected */ for (i = 0; i < NUM_BAD; i++) { - if ((res = ncmpi_def_dim(ncid, notvalid[i], DIMLEN, &dimid)) != NC_EBADNAME) + if ((res = ncmpi_def_dim(ncid, notvalid[i], DIMLEN, &dimid)) != NC_EBADNAME) ERRORI - if ((res = ncmpi_def_var(ncid, notvalid[i], NC_FLOAT, NDIMS, dimids, &varid)) != NC_EBADNAME) + if ((res = ncmpi_def_var(ncid, notvalid[i], NC_FLOAT, NDIMS, dimids, &varid)) != NC_EBADNAME) ERRORI - if ((res = ncmpi_put_att_text(ncid, varid, notvalid[i], strlen(attstring), attstring)) != NC_EBADNAME) + if ((res = ncmpi_put_att_text(ncid, varid, notvalid[i], strlen(attstring), attstring)) != NC_EBADNAME) ERRORI - if ((res = ncmpi_put_att_double(ncid, NC_GLOBAL, notvalid[i], NC_DOUBLE, NATTVALS, attvals)) != NC_EBADNAME) + if ((res = ncmpi_put_att_double(ncid, NC_GLOBAL, notvalid[i], NC_DOUBLE, NATTVALS, attvals)) != NC_EBADNAME) ERRORI } if ((res = ncmpi_enddef(ncid))) - ERROR + ERROR if ((res = ncmpi_close(ncid))) - ERROR + ERROR /* Check it out, make sure all objects with good names were defined OK */ if ((res = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid))) - ERROR + ERROR for (i = 0; i < NUM_GOOD; i++) { - MPI_Offset attlen; - if ((res = ncmpi_inq_dimid(ncid, valid[i], &dimid)) || dimid != dimids[i]) - ERRORI - if ((res = ncmpi_inq_varid(ncid, valid[i], &varid)) || varid != varids[i]) - ERRORI - res = ncmpi_inq_attlen(ncid, varid, valid[i], &attlen); - if ((res = ncmpi_get_att_text(ncid, varid, valid[i], attstr_in))) - ERRORI - attstr_in[attlen] = '\0'; - if (strcmp(valid[i], attstr_in) != 0) - ERRORI - if ((res = ncmpi_get_att_double(ncid, NC_GLOBAL, valid[i], attvals_in)) || attvals[0] != attvals_in[0]) - ERRORI + MPI_Offset attlen; + if ((res = ncmpi_inq_dimid(ncid, valid[i], &dimid)) || dimid != dimids[i]) + ERRORI + if ((res = ncmpi_inq_varid(ncid, valid[i], &varid)) || varid != varids[i]) + ERRORI + res = ncmpi_inq_attlen(ncid, varid, valid[i], &attlen); + if ((res = ncmpi_get_att_text(ncid, varid, valid[i], attstr_in))) + ERRORI + attstr_in[attlen] = '\0'; + if (strcmp(valid[i], attstr_in) != 0) + ERRORI + if ((res = ncmpi_get_att_double(ncid, NC_GLOBAL, valid[i], attvals_in)) || attvals[0] != attvals_in[0]) + ERRORI } if ((res = ncmpi_close(ncid))) - ERROR + ERROR } /* check if PnetCDF freed all internal malloc */ @@ -321,10 +328,12 @@ main(int argc, char **argv) sum_size); } + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf(PASS_STR, timing); } MPI_Finalize(); diff --git a/test/nc_test/tst_nofill.c b/test/nc_test/tst_nofill.c index 149c3a3d9f..cc26bb7d57 100644 --- a/test/nc_test/tst_nofill.c +++ b/test/nc_test/tst_nofill.c @@ -22,7 +22,7 @@ #include -#define ERR {if (err != NC_NOERR) {printf("Error at %s line %d: %s\n",__func__,__LINE__,ncmpi_strerror(err)); return 1;}} +#define ERR {if (err != NC_NOERR) {fprintf(stderr,"Error at %s line %d: %s\n",__func__,__LINE__,ncmpi_strerror(err)); return 1;}} static void check_err(const int stat, const int line, const char *file) { @@ -351,8 +351,12 @@ main(int argc, char **argv) { char *filename, *fill_filename, *nofill_filename; int rank, nprocs, err, nerrs=0; + double timing; MPI_Init(&argc, &argv); + + timing = MPI_Wtime(); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); MPI_Comm_rank(MPI_COMM_WORLD, &rank); @@ -402,7 +406,7 @@ main(int argc, char **argv) /* How many values in this variable to compare? */ err = ncmpi_inq_varndims(ncid1, varid, &ndims); ERR dimids = malloc(sizeof(int) * (ndims + 1)); - if (!dimids) printf("Error in file %s line %d\n",__FILE__,__LINE__); + if (!dimids) fprintf(stderr,"Error in file %s line %d\n",__FILE__,__LINE__); err = ncmpi_inq_vardimid (ncid1, varid, dimids); ERR nvals = 1; for(dim = 0; dim < ndims; dim++) { @@ -418,9 +422,9 @@ main(int argc, char **argv) double *data1, *data2; /* Allocate space to hold values in both files */ data1 = malloc(sizeof(double) * (nvals + 1)); - if (!data1) printf("Error in file %s line %d\n",__FILE__,__LINE__); + if (!data1) fprintf(stderr,"Error in file %s line %d\n",__FILE__,__LINE__); data2 = malloc(sizeof(double) * (nvals + 1)); - if (!data2) printf("Error in file %s line %d\n",__FILE__,__LINE__); + if (!data2) fprintf(stderr,"Error in file %s line %d\n",__FILE__,__LINE__); /* Read in values */ err = ncmpi_get_var_double_all(ncid1, varid, data1); ERR err = ncmpi_get_var_double_all(ncid2, varid, data2); ERR @@ -440,9 +444,9 @@ main(int argc, char **argv) char *data1, *data2; /* Allocate space to hold values in both files */ data1 = malloc(sizeof(char) * (nvals + 1)); - if (!data1) printf("Error in file %s line %d\n",__FILE__,__LINE__); + if (!data1) fprintf(stderr,"Error in file %s line %d\n",__FILE__,__LINE__); data2 = malloc(sizeof(char) * (nvals + 1)); - if (!data2) printf("Error in file %s line %d\n",__FILE__,__LINE__); + if (!data2) fprintf(stderr,"Error in file %s line %d\n",__FILE__,__LINE__); /* Read in values */ err = ncmpi_get_var_text_all(ncid1, varid, data1); ERR err = ncmpi_get_var_text_all(ncid2, varid, data2); ERR @@ -461,7 +465,7 @@ main(int argc, char **argv) } free(dimids); } - if(badvars > 0) printf("Error in file %s line %d\n",__FILE__,__LINE__); + if(badvars > 0) fprintf(stderr,"Error in file %s line %d\n",__FILE__,__LINE__); err = ncmpi_close(ncid1); ERR err = ncmpi_close(ncid2); ERR } @@ -476,10 +480,12 @@ main(int argc, char **argv) sum_size); } + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf(PASS_STR, timing); } free(fill_filename); diff --git a/test/nc_test/tst_norm.c b/test/nc_test/tst_norm.c index e3418e2337..a5ee389496 100644 --- a/test/nc_test/tst_norm.c +++ b/test/nc_test/tst_norm.c @@ -26,7 +26,7 @@ #define NDIMS 1 #define NX 18 -#define ERR {if (err != NC_NOERR) {printf("Error at %s line %d: %s\n",__func__,__LINE__,ncmpi_strerror(err)); return 1;}} +#define ERR {if (err != NC_NOERR) {fprintf(stderr,"Error at %s line %d: %s\n",__func__,__LINE__,ncmpi_strerror(err)); return 1;}} static int tst_norm(char *filename, int cmode) @@ -126,12 +126,12 @@ int tst_norm(char *filename, int cmode) * should have been normalized in library, so these are attempts to * create duplicate netCDF objects. */ if ((err = ncmpi_def_dim(ncid, NNAME, NX, &dimid)) != NC_ENAMEINUSE) { - printf("Error at line %d: expecting error code %d but got %d\n",__LINE__,NC_ENAMEINUSE,err); + fprintf(stderr,"Error at line %d: expecting error code %d but got %d\n",__LINE__,NC_ENAMEINUSE,err); return 1; } if ((err=ncmpi_def_var(ncid, NNAME, NC_CHAR, NDIMS, dimids, &varid)) != NC_ENAMEINUSE) { - printf("Error at line %d: expecting error code %d but got %d\n",__LINE__,NC_ENAMEINUSE,err); + fprintf(stderr,"Error at line %d: expecting error code %d but got %d\n",__LINE__,NC_ENAMEINUSE,err); return 1; } err = ncmpi_enddef(ncid); ERR @@ -147,19 +147,19 @@ int tst_norm(char *filename, int cmode) err = strncmp(NNAME, name_in, NNAMELEN); ERR err = ncmpi_inq_varid(ncid, NNAME, &varid_in); ERR if ((err = ncmpi_inq_dimid(ncid, UNAME, &dimid_in)) || dimid != dimid_in) - {printf("Error at line %d\n",__LINE__);return 1;} + {fprintf(stderr,"Error at line %d\n",__LINE__);return 1;} if ((err = ncmpi_inq_dimid(ncid, NNAME, &dimid_in)) || dimid != dimid_in) - {printf("Error at line %d\n",__LINE__);return 1;} + {fprintf(stderr,"Error at line %d\n",__LINE__);return 1;} err = ncmpi_inq_att(ncid, varid, UNITS, &att_type, &att_len); ERR if ( att_type != NC_CHAR || att_len != UNAMELEN) - {printf("Error at line %d\n",__LINE__);return 1;} + {fprintf(stderr,"Error at line %d\n",__LINE__);return 1;} err = ncmpi_get_att_text(ncid, varid, UNITS, strings_in); ERR strings_in[UNAMELEN] = '\0'; err = strncmp(UNAME, strings_in, UNAMELEN); ERR if ((err = ncmpi_inq_attid(ncid, varid, UNAME, &attnum_in)) || ATTNUM != attnum_in) - {printf("Error at line %d\n",__LINE__);return 1;} + {fprintf(stderr,"Error at line %d\n",__LINE__);return 1;} if ((err = ncmpi_inq_attid(ncid, varid, NNAME, &attnum_in)) || ATTNUM != attnum_in) - {printf("Error at line %d\n",__LINE__);return 1;} + {fprintf(stderr,"Error at line %d\n",__LINE__);return 1;} err = ncmpi_close(ncid); ERR return 0; @@ -170,8 +170,12 @@ main(int argc, char **argv) { char filename[256]; int rank, nprocs, cmode, err, nerrs=0; + double timing; MPI_Init(&argc, &argv); + + timing = MPI_Wtime(); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); MPI_Comm_rank(MPI_COMM_WORLD, &rank); @@ -209,10 +213,12 @@ main(int argc, char **argv) sum_size); } + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf(PASS_STR, timing); } MPI_Finalize(); diff --git a/test/nc_test/tst_small.c b/test/nc_test/tst_small.c index 14e4960190..e363d42afa 100644 --- a/test/nc_test/tst_small.c +++ b/test/nc_test/tst_small.c @@ -27,7 +27,7 @@ #define ATT_NAME "Atom" #define MAX_LEN 16 -#define ERR {if (err != NC_NOERR) {printf("Error at %s line %d: %s\n",__func__,__LINE__,ncmpi_strerror(err)); return 1;}} +#define ERR {if (err != NC_NOERR) {fprintf(stderr,"Error at %s line %d: %s\n",__func__,__LINE__,ncmpi_strerror(err)); return 1;}} static int test_small_atts(const char *testfile, int cmode) @@ -60,11 +60,11 @@ test_small_atts(const char *testfile, int cmode) /* Reopen the file and check it. */ err=ncmpi_open(MPI_COMM_WORLD, testfile, NC_NOWRITE, MPI_INFO_NULL, &ncid); ERR err=ncmpi_inq(ncid, &ndims, &nvars, &natts, &unlimdimid); ERR - if (ndims != 0 && nvars != 0 && natts != 1 && unlimdimid != -1) {printf("Error at line %d\n",__LINE__);return 1;} + if (ndims != 0 && nvars != 0 && natts != 1 && unlimdimid != -1) {fprintf(stderr,"Error at line %d\n",__LINE__);return 1;} err=ncmpi_inq_attlen(ncid, NC_GLOBAL, ATT_NAME, &len_in); ERR - if (len_in != t + 1) {printf("Error at line %d\n",__LINE__);return 1;} + if (len_in != t + 1) {fprintf(stderr,"Error at line %d\n",__LINE__);return 1;} err=ncmpi_get_att_text(ncid, NC_GLOBAL, ATT_NAME, att_in); ERR - if (strncmp(att_in, att, t)) {printf("Error at line %d\n",__LINE__);return 1;} + if (strncmp(att_in, att, t)) {fprintf(stderr,"Error at line %d\n",__LINE__);return 1;} err=ncmpi_close(ncid); ERR } } @@ -121,10 +121,10 @@ test_small_unlim(const char *testfile, int cmode) /* Reopen the file and check it. */ err=ncmpi_open(MPI_COMM_WORLD, testfile, NC_NOWRITE, MPI_INFO_NULL, &ncid); ERR err=ncmpi_inq(ncid, &ndims, &nvars, &natts, &unlimdimid); ERR - if (ndims != 2 && nvars != 1 && natts != 0 && unlimdimid != 0) {printf("Error at line %d\n",__LINE__);return 1;} + if (ndims != 2 && nvars != 1 && natts != 0 && unlimdimid != 0) {fprintf(stderr,"Error at line %d\n",__LINE__);return 1;} err=ncmpi_get_var_text_all(ncid, varid, (char *)data_in); ERR for (i = 0; i < NUM_VALS; i++) - /* if (strncmp(data[i], data_in[i], STR_LEN)) {printf("Error at line %d\n",__LINE__);return 1;} */ + /* if (strncmp(data[i], data_in[i], STR_LEN)) {fprintf(stderr,"Error at line %d\n",__LINE__);return 1;} */ if (strncmp(data[i], data_in[i], STR_LEN)) { printf("i=%d data=%s data_in=%s\n",i,data[i],data_in[i]); } @@ -170,10 +170,10 @@ test_small_fixed(const char *testfile, int cmode) /* Reopen the file and check it. */ err=ncmpi_open(MPI_COMM_WORLD, testfile, NC_NOWRITE, MPI_INFO_NULL, &ncid); ERR err=ncmpi_inq(ncid, &ndims, &nvars, &natts, &unlimdimid); ERR - if (ndims != 2 && nvars != 1 && natts != 0 && unlimdimid != -1) {printf("Error at line %d\n",__LINE__);return 1;} + if (ndims != 2 && nvars != 1 && natts != 0 && unlimdimid != -1) {fprintf(stderr,"Error at line %d\n",__LINE__);return 1;} err=ncmpi_get_var_text_all(ncid, varid, (char *)data_in); ERR for (i = 0; i < NUM_VALS; i++) - if (strncmp(data[i], data_in[i], STR_LEN)) {printf("Error at line %d\n",__LINE__);return 1;} + if (strncmp(data[i], data_in[i], STR_LEN)) {fprintf(stderr,"Error at line %d\n",__LINE__);return 1;} err=ncmpi_close(ncid); ERR return 0; } @@ -204,9 +204,9 @@ test_small_one(const char *testfile, int cmode) /* Reopen the file and check it. */ err=ncmpi_open(MPI_COMM_WORLD, testfile, NC_NOWRITE, MPI_INFO_NULL, &ncid); ERR err=ncmpi_inq(ncid, &ndims, &nvars, &natts, &unlimdimid); ERR - if (ndims != 1 && nvars != 1 && natts != 0 && unlimdimid != 0) {printf("Error at line %d\n",__LINE__);return 1;} + if (ndims != 1 && nvars != 1 && natts != 0 && unlimdimid != 0) {fprintf(stderr,"Error at line %d\n",__LINE__);return 1;} err=ncmpi_get_var_text_all(ncid, varid, &data_in); ERR - if (data_in != data) {printf("Error at line %d\n",__LINE__);return 1;} + if (data_in != data) {fprintf(stderr,"Error at line %d\n",__LINE__);return 1;} err=ncmpi_close(ncid); ERR return 0; } @@ -251,11 +251,11 @@ test_one_growing(const char *testfile, int cmode) /* Reopen the file and check it. */ err=ncmpi_open(MPI_COMM_WORLD, testfile, NC_NOWRITE, MPI_INFO_NULL, &ncid); ERR err=ncmpi_inq_dimlen(ncid, 0, &len_in); ERR - if (len_in != r + 1) {printf("Error at line %d\n",__LINE__);return 1;} + if (len_in != r + 1) {fprintf(stderr,"Error at line %d\n",__LINE__);return 1;} index[0] = r; err=ncmpi_begin_indep_data(ncid); ERR err=ncmpi_get_var1_text(ncid, 0, index, &data_in); ERR - if (data_in != data[r]) {printf("Error at line %d\n",__LINE__);return 1;} + if (data_in != data[r]) {fprintf(stderr,"Error at line %d\n",__LINE__);return 1;} err=ncmpi_close(ncid); ERR } /* Next record. */ } @@ -304,13 +304,13 @@ test_one_growing_with_att(const char *testfile, int cmode) /* Reopen the file and check it. */ err=ncmpi_open(MPI_COMM_WORLD, testfile, NC_NOWRITE, MPI_INFO_NULL, &ncid); ERR err=ncmpi_inq_dimlen(ncid, 0, &len_in); ERR - if (len_in != r + 1) {printf("Error at line %d\n",__LINE__);return 1;} + if (len_in != r + 1) {fprintf(stderr,"Error at line %d\n",__LINE__);return 1;} index[0] = r; err=ncmpi_begin_indep_data(ncid); ERR err=ncmpi_get_var1_text(ncid, 0, index, &data_in); ERR - if (data_in != data[r]) {printf("Error at line %d\n",__LINE__);return 1;} + if (data_in != data[r]) {fprintf(stderr,"Error at line %d\n",__LINE__);return 1;} err=ncmpi_get_att_text(ncid, varid, att_name, &data_in); ERR - if (data_in != data[r]) {printf("Error at line %d\n",__LINE__);return 1;} + if (data_in != data[r]) {fprintf(stderr,"Error at line %d\n",__LINE__);return 1;} err=ncmpi_close(ncid); ERR } /* Next record. */ return 0; @@ -363,13 +363,13 @@ test_two_growing_with_att(const char *testfile, int cmode) /* Reopen the file and check it. */ err=ncmpi_open(MPI_COMM_WORLD, testfile, NC_NOWRITE, MPI_INFO_NULL, &ncid); ERR err=ncmpi_inq_dimlen(ncid, 0, &len_in); ERR - if (len_in != r + 1) {printf("Error at line %d\n",__LINE__);return 1;} + if (len_in != r + 1) {fprintf(stderr,"Error at line %d\n",__LINE__);return 1;} index[0] = r; err=ncmpi_begin_indep_data(ncid); ERR for (v = 0; v < NUM_VARS; v++) { err=ncmpi_get_var1_text(ncid, varid[v], index, &data_in); ERR - if (data_in != data[r]) {printf("Error at line %d\n",__LINE__);return 1;} + if (data_in != data[r]) {fprintf(stderr,"Error at line %d\n",__LINE__);return 1;} } err=ncmpi_close(ncid); ERR } /* Next record. */ @@ -403,11 +403,11 @@ test_one_with_att(const char *testfile, int cmode) /* Reopen the file and check it. */ err=ncmpi_open(MPI_COMM_WORLD, testfile, NC_NOWRITE, MPI_INFO_NULL, &ncid); ERR err=ncmpi_inq(ncid, &ndims, &nvars, &natts, &unlimdimid); ERR - if (ndims != 1 && nvars != 1 && natts != 0 && unlimdimid != 0) {printf("Error at line %d\n",__LINE__);return 1;} + if (ndims != 1 && nvars != 1 && natts != 0 && unlimdimid != 0) {fprintf(stderr,"Error at line %d\n",__LINE__);return 1;} err=ncmpi_get_var_text_all(ncid, varid, &data_in); ERR - if (data_in != data) {printf("Error at line %d\n",__LINE__);return 1;} + if (data_in != data) {fprintf(stderr,"Error at line %d\n",__LINE__);return 1;} err=ncmpi_get_att_text(ncid, NC_GLOBAL, ATT_NAME, &data_in); ERR - if (data_in != data) {printf("Error at line %d\n",__LINE__);return 1;} + if (data_in != data) {fprintf(stderr,"Error at line %d\n",__LINE__);return 1;} err=ncmpi_close(ncid); ERR return 0; } @@ -417,8 +417,12 @@ int main(int argc, char *argv[]) char filename[256]; int i, rank, nprocs, err, nerrs=0; int cmode[NUM_FORMATS]={0, NC_64BIT_OFFSET, NC_64BIT_DATA}; + double timing; MPI_Init(&argc, &argv); + + timing = MPI_Wtime(); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); MPI_Comm_rank(MPI_COMM_WORLD, &rank); @@ -490,10 +494,12 @@ int main(int argc, char *argv[]) sum_size); } + timing = MPI_Wtime() - timing; + MPI_Allreduce(MPI_IN_PLACE, &timing, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + else printf(PASS_STR, timing); } MPI_Finalize(); diff --git a/test/nc_test/util.c b/test/nc_test/util.c index 71345fb89e..a43c2881e4 100644 --- a/test/nc_test/util.c +++ b/test/nc_test/util.c @@ -809,7 +809,7 @@ put_vars(int ncid, int numVars) MPI_Offset index[MAX_RANK]; int i, j, err, allInRange; double value[MAX_NELS]; - char text[MAX_NELS]; + char txt_buf[MAX_NELS]; int bb_enabled=0; { int flag; @@ -832,14 +832,14 @@ put_vars(int ncid, int numVars) IF (err != NC_NOERR) error("toMixedBase"); if (var_name[i][0] == 'c') { /* var_type[i] is NC_CHAR */ assert(var_type[i] == NC_CHAR); - text[j] = hash(var_type[i], var_rank[i], index); + txt_buf[j] = hash(var_type[i], var_rank[i], index); } else { value[j] = hash(var_type[i], var_rank[i], index); allInRange = allInRange && inRange(value[j], var_type[i]); } } if (var_name[i][0] == 'c') { - err = ncmpi_put_vara_text_all(ncid, i, start, var_shape[i], text); + err = ncmpi_put_vara_text_all(ncid, i, start, var_shape[i], txt_buf); IF (err != NC_NOERR) error("ncmpi_put_vara_text_all: %s", ncmpi_strerror(err)); } else { @@ -929,7 +929,7 @@ int check_vars(int ncid, int numVars) { MPI_Offset index[MAX_RANK]; - char text, name[NC_MAX_NAME]; + char txt_buf, name[NC_MAX_NAME]; int i, j, err; int nok = 0; /* count of valid comparisons */ int isChar, ndims, dimids[MAX_RANK]; @@ -961,12 +961,12 @@ check_vars(int ncid, int numVars) error("error in toMixedBase 2"); expect = hash( var_type[i], var_rank[i], index ); if (isChar) { - err = ncmpi_get_var1_text_all(ncid, i, index, &text); + err = ncmpi_get_var1_text_all(ncid, i, index, &txt_buf); IF (err != NC_NOERR) error("ncmpi_get_var1_text_all: %s", ncmpi_strerror(err)); - IF (text != (char)expect) { + IF (txt_buf != (char)expect) { error("Var %s (varid=%d) value[%d] read %d not that expected %d ", - var_name[i], i, j, text, (char)expect); + var_name[i], i, j, txt_buf, (char)expect); print_n_size_t(var_rank[i], index); } else { nok++; @@ -999,7 +999,7 @@ check_vars(int ncid, int numVars) void check_atts(int ncid, int numGatts, int numVars) { - char name[NC_MAX_NAME], text[MAX_NELS]; + char name[NC_MAX_NAME], txt_buf[MAX_NELS]; int i, j, err; /* status */ nc_type xtype; MPI_Offset k, length, ndx[1]; @@ -1021,13 +1021,13 @@ check_atts(int ncid, int numGatts, int numVars) IF (length != ATT_LEN(i,j)) error("ncmpi_inq_att: unexpected length"); if (xtype == NC_CHAR) { - err = ncmpi_get_att_text(ncid, i, name, text); + err = ncmpi_get_att_text(ncid, i, name, txt_buf); IF (err != NC_NOERR) error("ncmpi_get_att_text: %s", ncmpi_strerror(err)); for (k = 0; k < ATT_LEN(i,j); k++) { ndx[0] = k; expect = hash(xtype, -1, ndx); - if (text[k] != (char)expect) { + if (txt_buf[k] != (char)expect) { error("ncmpi_get_att_text: unexpected value"); } else { nok++; diff --git a/test/nc_test/wrap_runs.sh b/test/nc_test/wrap_runs.sh index 343accd9fa..dcdefa8630 100755 --- a/test/nc_test/wrap_runs.sh +++ b/test/nc_test/wrap_runs.sh @@ -15,19 +15,29 @@ outfile=`basename $1` # remove file system type prefix if there is any OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` -# echo "PNETCDF_DEBUG = ${PNETCDF_DEBUG}" -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" +# prevent user environment setting of PNETCDF_HINTS to interfere +unset PNETCDF_HINTS + +if test "x$ENABLE_GIO" = x0 ; then + IO_MODES="mpiio" else - safe_modes="0" + IO_MODES="gio mpiio" fi -# prevent user environment setting of PNETCDF_HINTS to interfere -unset PNETCDF_HINTS +for io_mode in $IO_MODES ; do + if test "x$io_mode" = xmpiio ; then + USEMPIO_HINTS="nc_driver=mpiio" + else + USEMPIO_HINTS="nc_driver=gio" + fi + + PNETCDF_HINTS= + if test "x$USEMPIO_HINTS" != x ; then + PNETCDF_HINTS="$USEMPIO_HINTS;$PNETCDF_HINTS" + fi -for j in ${safe_modes} ; do - export PNETCDF_SAFE_MODE=$j - # echo "---- set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" + export PNETCDF_HINTS="$PNETCDF_HINTS" + # echo "PNETCDF_HINTS=$PNETCDF_HINTS" ${TESTSEQRUN} $1 ${TESTOUTDIR}/$outfile.nc ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$outfile.nc @@ -37,7 +47,7 @@ for j in ${safe_modes} ; do echo "" echo "---- testing burst buffering" - export PNETCDF_HINTS="nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" + export PNETCDF_HINTS="$PNETCDF_HINTS;nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" ${TESTSEQRUN} $1 ${TESTOUTDIR}/$outfile.bb.nc unset PNETCDF_HINTS ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$outfile.bb.nc diff --git a/test/nf90_test/Makefile.am b/test/nf90_test/Makefile.am index 4c405947fe..2daeddafb5 100644 --- a/test/nf90_test/Makefile.am +++ b/test/nf90_test/Makefile.am @@ -19,7 +19,7 @@ AM_FCFLAGS = $(FC_MODINC)$(top_builddir)/src/binding/f90 \ $(FC_MODINC)../common $(FFREEFORMFLAG) AM_CFLAGS = -LDADD = $(top_builddir)/src/libs/libpnetcdf.la ../common/libtestutils.la +LDADD = ../common/libtestutils.la $(top_builddir)/src/libs/libpnetcdf.la LDADD += @NETCDF4_LDFLAGS@ @ADIOS_LDFLAGS@ @NETCDF4_LIBS@ @ADIOS_LIBS@ # suppress warning messages when using NAG Fortran compiler @@ -34,16 +34,6 @@ if NAGFORT AM_FCFLAGS += -w=uparam endif -if RELAX_COORD_BOUND - AM_CFLAGS += -DRELAX_COORD_BOUND - AM_FCFLAGS += $(FC_DEFINE)RELAX_COORD_BOUND -endif - -if ENABLE_NETCDF4 - AM_CFLAGS += -DENABLE_NETCDF4 - AM_FCFLAGS += $(FC_DEFINE)ENABLE_NETCDF4 -endif - M4SRCS = test_get.m4 \ test_put.m4 \ test_iget.m4 \ @@ -78,24 +68,26 @@ check_PROGRAMS = $(TESTPROGRAMS) # AM_TESTS_ENVIRONMENT += TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)" ; export TESTOUTDIR; TESTS_ENVIRONMENT = export SED="$(SED)"; TESTS_ENVIRONMENT += export srcdir="$(srcdir)"; +TESTS_ENVIRONMENT += export top_builddir="$(top_builddir)"; TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; TESTS_ENVIRONMENT += export TESTMPIRUN="$(TESTMPIRUN)"; -TESTS_ENVIRONMENT += export PNETCDF_DEBUG="$(PNETCDF_DEBUG)"; TESTS_ENVIRONMENT += export TESTPROGRAMS="$(TESTPROGRAMS)"; TESTS_ENVIRONMENT += export check_PROGRAMS="$(check_PROGRAMS)"; -TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER="$(ENABLE_BURST_BUFFER)"; -TESTS_ENVIRONMENT += export ENABLE_NETCDF4="$(ENABLE_NETCDF4)"; + +TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER=@ENABLE_BURST_BUFFER@; +TESTS_ENVIRONMENT += export ENABLE_NETCDF4=@ENABLE_NETCDF4@; +TESTS_ENVIRONMENT += export ENABLE_THREAD_SAFE=@ENABLE_THREAD_SAFE@; +TESTS_ENVIRONMENT += export MIMIC_LUSTRE=@MIMIC_LUSTRE@; TESTS = seq_runs.sh TEST_EXTENSIONS = .sh CLEANFILES = $(M4SRCS_F90) \ - $(TESTOUTDIR)/scratch.nc \ - $(TESTOUTDIR)/test.nc \ - $(TESTOUTDIR)/tooth-fairy.nc \ core core.* *.gcda *.gcno *.gcov gmon.out +check_SCRIPTS = seq_runs.sh + EXTRA_DIST = $(M4SRCS) README seq_runs.sh ../common/libtestutils.la: diff --git a/test/nf90_test/nf90_test.F90 b/test/nf90_test/nf90_test.F90 index 7f304977e2..f7740d9fe5 100644 --- a/test/nf90_test/nf90_test.F90 +++ b/test/nf90_test/nf90_test.F90 @@ -36,9 +36,9 @@ subroutine usage() call error(' [-1] test CDF-1 format' ) call error(' [-2] test CDF-2 format' ) call error(' [-5] test CDF-5 format' ) -#ifdef ENABLE_NETCDF4 - call error(' [-4] test NetCDF-4 classic-model format' ) -#endif + if (PNETCDF_DRIVER_NETCDF4 == 1) then + call error(' [-4] test NetCDF-4 classic-model format' ) + endif call error(' [-r] Just do read-only tests' ) call error( & ' [-d directory] directory for storing input/output files' ) @@ -47,9 +47,10 @@ subroutine usage() ' [-n ] max. number of messages per test (Default: 20)') end - subroutine report_test + subroutine report_test(timing) implicit none character(LEN=1024) msg + double precision timing #include "tests.inc" if (cdf_format .EQ. 4) then @@ -63,7 +64,9 @@ subroutine report_test write(*,*) trim(PROGNAME)//' expects to see 0 failure ... '//& 'Total number of failures: ', nfailsTotal endif - call pass_fail(nfailsTotal, msg) + + timing = MPI_Wtime() - timing + call pass_fail(nfailsTotal, msg, timing) end subroutine test(name, func) @@ -88,7 +91,7 @@ subroutine test(name, func) print *, ' ' print *, ' ### ', nfails, ' FAILURES TESTING ', name, & '! Stop ... ###' - call report_test + call report_test(0) stop 2 end if end @@ -427,7 +430,12 @@ program nf90_test external test_nf90mpi_set_default_format external nc_ignorefpe + double precision timing + call MPI_INIT(err) + + timing = MPI_Wtime() + comm = MPI_COMM_WORLD call nc_ignorefpe(1) @@ -501,14 +509,14 @@ program nf90_test end if 1 continue -#ifndef ENABLE_NETCDF4 - if (cdf_format .EQ. 4) then - call error( & - "Error: NetCDF-4 support is not enabled at configure time") - call MPI_Finalize(err) - stop 1 + if (PNETCDF_DRIVER_NETCDF4 == 0) then + if (cdf_format .EQ. 4) then + call error( & + "Error: NetCDF-4 support is not enabled at configure time") + call MPI_Finalize(err) + stop 1 + endif endif -#endif call MPI_Info_create(info, err) ! call MPI_Info_set(info, "romio_pvfs2_posix_write", "enable",err) @@ -865,7 +873,7 @@ program nf90_test call MPI_Info_free(info, err) - call report_test + call report_test(timing) ! if (nfailsTotal .eq. 0) call ud_exit(0) call ud_exit(0) diff --git a/test/nf90_test/seq_runs.sh b/test/nf90_test/seq_runs.sh index ade7a6ad3a..c862fce875 100755 --- a/test/nf90_test/seq_runs.sh +++ b/test/nf90_test/seq_runs.sh @@ -36,7 +36,7 @@ rm -f ${OUTDIR}/tooth-fairy.nc ${TESTSEQRUN} ./nf90_test -5 -d ${TESTOUTDIR} ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/test.nc -if test "${ENABLE_NETCDF4}" = 1 ; then +if test "x${ENABLE_NETCDF4}" = x1 ; then rm -f ${OUTDIR}/test.nc rm -f ${OUTDIR}/scratch.nc rm -f ${OUTDIR}/tooth-fairy.nc diff --git a/test/nf90_test/test_get.m4 b/test/nf90_test/test_get.m4 index 75b8e36369..56f2b37ea7 100644 --- a/test/nf90_test/test_get.m4 +++ b/test/nf90_test/test_get.m4 @@ -335,7 +335,7 @@ define([TEST_NFMPI_GET_VARA],[dnl integer(kind=MPI_OFFSET_KIND) edge(MAX_RANK) integer(kind=MPI_OFFSET_KIND) index(MAX_RANK) integer(kind=MPI_OFFSET_KIND) mid(MAX_RANK) - logical canConvert + logical canConvert, relax_coord_bound, relax_coord_bound_f DATATYPE($1, value, MAX_NELS) doubleprecision expect(MAX_NELS) doubleprecision val @@ -394,13 +394,14 @@ define([TEST_NFMPI_GET_VARA],[dnl if (err .ne. NF90_ECHAR) & call errore('wrong type: ', err) else -#ifdef RELAX_COORD_BOUND - if (err .ne. NF90_NOERR) & - call error(ErrFunc(err)) -#else - if (err .ne. NF90_EINVALCOORDS) & - call errore('bad start: ', err) -#endif + relax_coord_bound = relax_coord_bound_f() + if (relax_coord_bound) then + if (err .ne. NF90_NOERR) & + call error(ErrFunc(err)) + else + if (err .ne. NF90_EINVALCOORDS) & + call errore('bad start: ', err) + endif endif start(j) = var_shape(j,i) + 2 err = GetVar(ncid, i, value, start, edge) @@ -545,7 +546,7 @@ define([TEST_NFMPI_GET_VARS],dnl integer(kind=MPI_OFFSET_KIND) count(MAX_RANK) integer(kind=MPI_OFFSET_KIND) sstride(MAX_RANK) integer(kind=MPI_OFFSET_KIND) stride(MAX_RANK) - logical canConvert + logical canConvert, relax_coord_bound, relax_coord_bound_f DATATYPE($1, value, MAX_NELS) doubleprecision expect(MAX_NELS) doubleprecision val @@ -625,13 +626,14 @@ define([TEST_NFMPI_GET_VARS],dnl if (err .ne. NF90_ECHAR) & call errore('wrong type: ', err) else -#ifdef RELAX_COORD_BOUND - if (err .ne. NF90_NOERR) & - call error(ErrFunc(err)) -#else - if (err .ne. NF90_EINVALCOORDS) & - call errore('bad start: ', err) -#endif + relax_coord_bound = relax_coord_bound_f() + if (relax_coord_bound) then + if (err .ne. NF90_NOERR) & + call error(ErrFunc(err)) + else + if (err .ne. NF90_EINVALCOORDS) & + call errore('bad start: ', err) + endif endif start(j) = var_shape(j,i) + 2 err = GetVar(ncid, i, value, start, edge, stride) @@ -818,7 +820,7 @@ define([TEST_NFMPI_GET_VARM],dnl integer(kind=MPI_OFFSET_KIND) sstride(MAX_RANK) integer(kind=MPI_OFFSET_KIND) stride(MAX_RANK) integer(kind=MPI_OFFSET_KIND) imap(MAX_RANK) - logical canConvert + logical canConvert, relax_coord_bound, relax_coord_bound_f DATATYPE($1, value, MAX_NELS) doubleprecision expect(MAX_NELS) doubleprecision val @@ -904,13 +906,14 @@ define([TEST_NFMPI_GET_VARM],dnl if (err .ne. NF90_ECHAR) & call errore('wrong type: ', err) else -#ifdef RELAX_COORD_BOUND - if (err .ne. NF90_NOERR) & - call error(ErrFunc(err)) -#else - if (err .ne. NF90_EINVALCOORDS) & - call errore('bad start: ', err) -#endif + relax_coord_bound = relax_coord_bound_f() + if (relax_coord_bound) then + if (err .ne. NF90_NOERR) & + call error(ErrFunc(err)) + else + if (err .ne. NF90_EINVALCOORDS) & + call errore('bad start: ', err) + endif endif start(j) = var_shape(j,i) + 2 err = GetVar(ncid, i, value, start, edge, stride, imap) diff --git a/test/nf90_test/test_iget.m4 b/test/nf90_test/test_iget.m4 index fff6559272..5e2d4eedde 100644 --- a/test/nf90_test/test_iget.m4 +++ b/test/nf90_test/test_iget.m4 @@ -313,7 +313,7 @@ define([TEST_NFMPI_IGET_VARA],[dnl integer(kind=MPI_OFFSET_KIND) edge(MAX_RANK) integer(kind=MPI_OFFSET_KIND) index(MAX_RANK) integer(kind=MPI_OFFSET_KIND) mid(MAX_RANK) - logical canConvert + logical canConvert, relax_coord_bound, relax_coord_bound_f DATATYPE($1, value, (MAX_NELS)) doubleprecision expect(MAX_NELS) doubleprecision val @@ -383,13 +383,14 @@ define([TEST_NFMPI_IGET_VARA],[dnl if (err .ne. NF90_ECHAR) & call errore('wrong type: ', err) else -#ifdef RELAX_COORD_BOUND - if (err .ne. NF90_NOERR) & - call error(ErrFunc(err)) -#else - if (err .ne. NF90_EINVALCOORDS) & - call errore('bad start: ', err) -#endif + relax_coord_bound = relax_coord_bound_f() + if (relax_coord_bound) then + if (err .ne. NF90_NOERR) & + call error(ErrFunc(err)) + else + if (err .ne. NF90_EINVALCOORDS) & + call errore('bad start: ', err) + endif endif start(j) = var_shape(j,i) + 2 err = GetVar(ncid, i, value,reqid(1), start, edge) @@ -542,7 +543,7 @@ define([TEST_NFMPI_IGET_VARS],dnl integer(kind=MPI_OFFSET_KIND) count(MAX_RANK) integer(kind=MPI_OFFSET_KIND) sstride(MAX_RANK) integer(kind=MPI_OFFSET_KIND) stride(MAX_RANK) - logical canConvert + logical canConvert, relax_coord_bound, relax_coord_bound_f DATATYPE($1, value, (MAX_NELS)) doubleprecision expect(MAX_NELS) doubleprecision val @@ -623,13 +624,14 @@ define([TEST_NFMPI_IGET_VARS],dnl if (err .ne. NF90_ECHAR) & call errore('wrong type: ', err) else -#ifdef RELAX_COORD_BOUND - if (err .ne. NF90_NOERR) & - call error(ErrFunc(err)) -#else - if (err .ne. NF90_EINVALCOORDS) & - call errore('bad start: ', err) -#endif + relax_coord_bound = relax_coord_bound_f() + if (relax_coord_bound) then + if (err .ne. NF90_NOERR) & + call error(ErrFunc(err)) + else + if (err .ne. NF90_EINVALCOORDS) & + call errore('bad start: ', err) + endif endif start(j) = var_shape(j,i) + 2 err = GetVar(ncid, i, value,reqid(1), start, edge, stride) @@ -813,7 +815,7 @@ define([TEST_NFMPI_IGET_VARM],dnl integer(kind=MPI_OFFSET_KIND) sstride(MAX_RANK) integer(kind=MPI_OFFSET_KIND) stride(MAX_RANK) integer(kind=MPI_OFFSET_KIND) imap(MAX_RANK) - logical canConvert + logical canConvert, relax_coord_bound, relax_coord_bound_f DATATYPE($1, value, (MAX_NELS)) doubleprecision expect(MAX_NELS) doubleprecision val @@ -895,13 +897,14 @@ define([TEST_NFMPI_IGET_VARM],dnl if (err .ne. NF90_ECHAR) & call errore('wrong type: ', err) else -#ifdef RELAX_COORD_BOUND - if (err .ne. NF90_NOERR) & - call error(ErrFunc(err)) -#else - if (err .ne. NF90_EINVALCOORDS) & - call errore('bad start: ', err) -#endif + relax_coord_bound = relax_coord_bound_f() + if (relax_coord_bound) then + if (err .ne. NF90_NOERR) & + call error(ErrFunc(err)) + else + if (err .ne. NF90_EINVALCOORDS) & + call errore('bad start: ', err) + endif endif start(j) = var_shape(j,i) + 2 err = GetVar(ncid, i, value,reqid(1), start, edge, stride, imap) diff --git a/test/nf90_test/test_iput.m4 b/test/nf90_test/test_iput.m4 index c65dbde787..3143cfd7ad 100644 --- a/test/nf90_test/test_iput.m4 +++ b/test/nf90_test/test_iput.m4 @@ -486,6 +486,7 @@ define([TEST_NFMPI_IPUT_VARA],dnl integer(kind=MPI_OFFSET_KIND) index(MAX_RANK) logical canConvert !/* Both text or both numeric */ logical allInExtRange !/* all values within external range? */ + logical relax_coord_bound, relax_coord_bound_f DATATYPE($1, value, (MAX_NELS)) doubleprecision val integer ud_shift @@ -577,13 +578,14 @@ define([TEST_NFMPI_IPUT_VARA],dnl if (err .ne. NF90_ECHAR) & call errore('conversion: ', err) else -#ifdef RELAX_COORD_BOUND - if (err .ne. NF90_NOERR) & - call error(ErrFunc(err)) -#else - if (err .ne. NF90_EINVALCOORDS) & - call errore('bad start: ', err) -#endif + relax_coord_bound = relax_coord_bound_f() + if (relax_coord_bound) then + if (err .ne. NF90_NOERR) & + call error(ErrFunc(err)) + else + if (err .ne. NF90_EINVALCOORDS) & + call errore('bad start: ', err) + endif endif start(j) = var_shape(j,i) + 2 err = iPutVar(ncid, i, value,reqid(1), start, edge) @@ -723,6 +725,7 @@ define([TEST_NFMPI_IPUT_VARS],dnl integer(kind=MPI_OFFSET_KIND) stride(MAX_RANK) logical canConvert !/* Both text or both numeric */ logical allInExtRange !/* all values within external range? */ + logical relax_coord_bound, relax_coord_bound_f DATATYPE($1, value, (MAX_NELS)) doubleprecision val integer ud_shift @@ -825,13 +828,14 @@ define([TEST_NFMPI_IPUT_VARS],dnl if (err .ne. NF90_ECHAR) & call errore('conversion: ', err) else -#ifdef RELAX_COORD_BOUND - if (err .ne. NF90_NOERR) & - call error(ErrFunc(err)) -#else - if (err .ne. NF90_EINVALCOORDS) & - call errore('bad start: ', err) -#endif + relax_coord_bound = relax_coord_bound_f() + if (relax_coord_bound) then + if (err .ne. NF90_NOERR) & + call error(ErrFunc(err)) + else + if (err .ne. NF90_EINVALCOORDS) & + call errore('bad start: ', err) + endif endif start(j) = var_shape(j,i) + 2 err = iPutVar(ncid, i, value,reqid(1), start, edge, stride) @@ -1003,6 +1007,7 @@ define([TEST_NFMPI_IPUT_VARM],dnl integer(kind=MPI_OFFSET_KIND) imap(MAX_RANK) logical canConvert !/* Both text or both numeric */ logical allInExtRange !/* all values within external range? */ + logical relax_coord_bound, relax_coord_bound_f DATATYPE($1, value, (MAX_NELS)) doubleprecision val integer ud_shift @@ -1106,13 +1111,14 @@ define([TEST_NFMPI_IPUT_VARM],dnl if (err .ne. NF90_ECHAR) & call errore('conversion: ', err) else -#ifdef RELAX_COORD_BOUND - if (err .ne. NF90_NOERR) & - call error(ErrFunc(err)) -#else - if (err .ne. NF90_EINVALCOORDS) & - call errore('bad start: ', err) -#endif + relax_coord_bound = relax_coord_bound_f() + if (relax_coord_bound) then + if (err .ne. NF90_NOERR) & + call error(ErrFunc(err)) + else + if (err .ne. NF90_EINVALCOORDS) & + call errore('bad start: ', err) + endif endif start(j) = var_shape(j,i) + 2 err = iPutVar(ncid, i, value,reqid(1), start, edge, stride, imap) diff --git a/test/nf90_test/test_put.m4 b/test/nf90_test/test_put.m4 index 857934c013..5e6b5671a9 100644 --- a/test/nf90_test/test_put.m4 +++ b/test/nf90_test/test_put.m4 @@ -751,6 +751,7 @@ define([TEST_NFMPI_PUT_VARA],dnl integer(kind=MPI_OFFSET_KIND) index(MAX_RANK) logical canConvert !/* Both text or both numeric */ logical allInExtRange !/* all values within external range? */ + logical relax_coord_bound, relax_coord_bound_f DATATYPE($1, value, (MAX_NELS)) doubleprecision val integer ud_shift @@ -859,13 +860,14 @@ define([TEST_NFMPI_PUT_VARA],dnl if (err .ne. NF90_ECHAR) & call errore('wrong type: ', err) else -#ifdef RELAX_COORD_BOUND - if (err .ne. NF90_NOERR) & - call errore('PutVarAll($1): ',err) -#else - if (err .ne. NF90_EINVALCOORDS) & - call errore('bad start: ', err) -#endif + relax_coord_bound = relax_coord_bound_f() + if (relax_coord_bound) then + if (err .ne. NF90_NOERR) & + call errore('PutVarAll($1): ',err) + else + if (err .ne. NF90_EINVALCOORDS) & + call errore('bad start: ', err) + endif endif ! make start() way out of bounds start(j) = var_shape(j,i) + 2 @@ -995,6 +997,7 @@ define([TEST_NFMPI_PUT_VARS],dnl integer(kind=MPI_OFFSET_KIND) stride(MAX_RANK) logical canConvert !/* Both text or both numeric */ logical allInExtRange !/* all values within external range? */ + logical relax_coord_bound, relax_coord_bound_f DATATYPE($1, value, (MAX_NELS)) doubleprecision val integer ud_shift @@ -1115,13 +1118,14 @@ define([TEST_NFMPI_PUT_VARS],dnl if (err .ne. NF90_ECHAR) & call errore('wrong type: ', err) else -#ifdef RELAX_COORD_BOUND - if (err .ne. NF90_NOERR) & - call errore('PutVarAll($1): ',err) -#else - if (err .ne. NF90_EINVALCOORDS) & - call errore('bad start: ', err) -#endif + relax_coord_bound = relax_coord_bound_f() + if (relax_coord_bound) then + if (err .ne. NF90_NOERR) & + call errore('PutVarAll($1): ',err) + else + if (err .ne. NF90_EINVALCOORDS) & + call errore('bad start: ', err) + endif endif ! make start() way out of bounds start(j) = var_shape(j,i) + 2 @@ -1284,6 +1288,7 @@ define([TEST_NFMPI_PUT_VARM],dnl integer(kind=MPI_OFFSET_KIND) imap(MAX_RANK) logical canConvert !/* Both text or both numeric */ logical allInExtRange !/* all values within external range? */ + logical relax_coord_bound, relax_coord_bound_f DATATYPE($1, value, (MAX_NELS)) doubleprecision val integer ud_shift @@ -1410,13 +1415,14 @@ define([TEST_NFMPI_PUT_VARM],dnl if (err .ne. NF90_ECHAR) & call errore('wrong type: ', err) else -#ifdef RELAX_COORD_BOUND - if (err .ne. NF90_NOERR) & - call errore('PutVarAll($1): ',err) -#else - if (err .ne. NF90_EINVALCOORDS) & - call errore('bad start: ', err) -#endif + relax_coord_bound = relax_coord_bound_f() + if (relax_coord_bound) then + if (err .ne. NF90_NOERR) & + call errore('PutVarAll($1): ',err) + else + if (err .ne. NF90_EINVALCOORDS) & + call errore('bad start: ', err) + endif endif ! make start() way out of bounds start(j) = var_shape(j,i) + 2 diff --git a/test/nf_test/Makefile.am b/test/nf_test/Makefile.am index 9532b07185..1a3493c949 100644 --- a/test/nf_test/Makefile.am +++ b/test/nf_test/Makefile.am @@ -17,7 +17,7 @@ AM_CPPFLAGS = -I$(top_builddir)/src/include \ AM_CFLAGS = AM_FFLAGS = $(FFIXEDFORMFLAG) -LDADD = $(top_builddir)/src/libs/libpnetcdf.la ../common/libtestutils.la +LDADD = ../common/libtestutils.la $(top_builddir)/src/libs/libpnetcdf.la LDADD += @NETCDF4_LDFLAGS@ @ADIOS_LDFLAGS@ @NETCDF4_LIBS@ @ADIOS_LIBS@ # suppress warning messages when using NAG Fortran compiler @@ -51,14 +51,6 @@ endif if HAVE_F77_INT8 M4FFLAGS += -DHAVE_F77_INT8 endif -if RELAX_COORD_BOUND - AM_CFLAGS += -DRELAX_COORD_BOUND - AM_FFLAGS += $(FC_DEFINE)RELAX_COORD_BOUND -endif -if ENABLE_NETCDF4 - AM_CFLAGS += -DENABLE_NETCDF4 - AM_FFLAGS += $(FC_DEFINE)ENABLE_NETCDF4 -endif M4SRCS = test_get.m4 \ test_put.m4 \ @@ -86,14 +78,17 @@ check_PROGRAMS = $(TESTPROGRAMS) # AM_TESTS_ENVIRONMENT += TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)" ; export TESTOUTDIR; TESTS_ENVIRONMENT = export SED="$(SED)"; TESTS_ENVIRONMENT += export srcdir="$(srcdir)"; +TESTS_ENVIRONMENT += export top_builddir="$(top_builddir)"; TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; TESTS_ENVIRONMENT += export TESTMPIRUN="$(TESTMPIRUN)"; -TESTS_ENVIRONMENT += export PNETCDF_DEBUG="$(PNETCDF_DEBUG)"; TESTS_ENVIRONMENT += export TESTPROGRAMS="$(TESTPROGRAMS)"; TESTS_ENVIRONMENT += export check_PROGRAMS="$(check_PROGRAMS)"; -TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER="$(ENABLE_BURST_BUFFER)"; -TESTS_ENVIRONMENT += export ENABLE_NETCDF4="$(ENABLE_NETCDF4)"; + +TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER=@ENABLE_BURST_BUFFER@; +TESTS_ENVIRONMENT += export ENABLE_NETCDF4=@ENABLE_NETCDF4@; +TESTS_ENVIRONMENT += export ENABLE_THREAD_SAFE=@ENABLE_THREAD_SAFE@; +TESTS_ENVIRONMENT += export MIMIC_LUSTRE=@MIMIC_LUSTRE@; TESTS = seq_runs.sh TEST_EXTENSIONS = .sh @@ -104,11 +99,10 @@ nodist_nf_test_SOURCES = $(M4SRCS:.m4=.F) $(M4SRCS:.m4=.F): Makefile CLEANFILES = $(M4SRCS:.m4=.F) \ - $(TESTOUTDIR)/scratch.nc \ - $(TESTOUTDIR)/test.nc \ - $(TESTOUTDIR)/tooth-fairy.nc \ core core.* *.gcda *.gcno *.gcov gmon.out +check_SCRIPTS = seq_runs.sh + EXTRA_DIST = $(M4SRCS) $(HFILES) README seq_runs.sh ../common/libtestutils.la: diff --git a/test/nf_test/nf_test.F b/test/nf_test/nf_test.F index 45b344c52c..5ec86fc394 100644 --- a/test/nf_test/nf_test.F +++ b/test/nf_test/nf_test.F @@ -25,9 +25,9 @@ subroutine usage() call error(' [-1] test CDF-1 format' ) call error(' [-2] test CDF-2 format' ) call error(' [-5] test CDF-5 format' ) -#ifdef ENABLE_NETCDF4 - call error(' [-4] test NetCDF-4 classic-model format' ) -#endif + if (PNETCDF_DRIVER_NETCDF4 .EQ. 1) then + call error(' [-4] test NetCDF-4 classic-model format' ) + endif call error(' [-r] Just do read-only tests' ) call error( + ' [-d directory] directory for storing input/output files' ) @@ -37,9 +37,10 @@ subroutine usage() end - subroutine report_test + subroutine report_test(timing) implicit none character(LEN=1024) msg + double precision timing #include "tests.inc" integer MY_LEN_TRIM @@ -57,7 +58,7 @@ subroutine report_test + ' expects to see 0 failure ... '// + 'Total number of failures: ', nfailsTotal endif - call pass_fail(nfailsTotal, msg) + call pass_fail(nfailsTotal, msg, timing) end subroutine test(name, func) @@ -422,7 +423,12 @@ program nf_test external test_nfmpi_set_default_format external nc_ignorefpe + double precision timing + call MPI_INIT(err) + + timing = MPI_Wtime() + comm = MPI_COMM_WORLD call nc_ignorefpe(1) @@ -498,14 +504,14 @@ program nf_test end if 1 continue -#ifndef ENABLE_NETCDF4 - if (cdf_format .EQ. 4) then - call error( - + "Error: NetCDF-4 support is not enabled at configure time") - call MPI_Finalize(err) - stop 1 + if (PNETCDF_DRIVER_NETCDF4 == 0) then + if (cdf_format .EQ. 4) then + call error( + + "Error: NetCDF-4 feature is disabled at configure time") + call MPI_Finalize(err) + stop 1 + endif endif -#endif numGatts = 6 numVars = 136 @@ -865,7 +871,8 @@ program nf_test call MPI_Info_free(info, err) - call report_test + timing = MPI_Wtime() - timing + call report_test(timing) ! if (nfailsTotal .eq. 0) call ud_exit(0) call ud_exit(0) diff --git a/test/nf_test/seq_runs.sh b/test/nf_test/seq_runs.sh index f76a7a28c8..08c5e4bdc0 100755 --- a/test/nf_test/seq_runs.sh +++ b/test/nf_test/seq_runs.sh @@ -30,7 +30,7 @@ rm -f ${OUTDIR}/tooth-fairy.nc ${TESTSEQRUN} ./nf_test -2 -d ${TESTOUTDIR} ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/test.nc -if test "${ENABLE_NETCDF4}" = 1 ; then +if test "x${ENABLE_NETCDF4}" = x1 ; then rm -f ${OUTDIR}/test.nc rm -f ${OUTDIR}/scratch.nc rm -f ${OUTDIR}/tooth-fairy.nc diff --git a/test/nf_test/test_get.m4 b/test/nf_test/test_get.m4 index 7ae87007ab..acfc9ab399 100644 --- a/test/nf_test/test_get.m4 +++ b/test/nf_test/test_get.m4 @@ -320,7 +320,7 @@ define([TEST_NFMPI_GET_VARA],[dnl integer*8 edge(MAX_RANK) integer*8 index(MAX_RANK) integer*8 mid(MAX_RANK) - logical canConvert + logical canConvert, relax_coord_bound, relax_coord_bound_f DATATYPE($1, value, (MAX_NELS)) doubleprecision expect(MAX_NELS) doubleprecision val @@ -386,13 +386,14 @@ C /* there is nothing to get (edge(j).eq.0) */ if (err .NE. NF_ECHAR) + call errore('wrong type: ', err) else -#ifdef RELAX_COORD_BOUND - if (err .NE. NF_NOERR) - + call error(ErrFunc(err)) -#else - if (err .NE. NF_EINVALCOORDS) - + call errore('bad start: ', err) -#endif + relax_coord_bound = relax_coord_bound_f() + if (relax_coord_bound) then + if (err .NE. NF_NOERR) + + call error(ErrFunc(err)) + else + if (err .NE. NF_EINVALCOORDS) + + call errore('bad start: ', err) + endif endif start(j) = var_shape(j,i) + 2 err = GetVara($1)(ncid, i, @@ -548,7 +549,7 @@ define([TEST_NFMPI_GET_VARS],dnl integer*8 count(MAX_RANK) integer*8 sstride(MAX_RANK) integer*8 stride(MAX_RANK) - logical canConvert + logical canConvert, relax_coord_bound, relax_coord_bound_f DATATYPE($1, value, (MAX_NELS)) doubleprecision expect(MAX_NELS) doubleprecision val @@ -636,13 +637,14 @@ C /* there is nothing to get (edge(j).eq.0) */ if (err .NE. NF_ECHAR) + call errore('wrong type: ', err) else -#ifdef RELAX_COORD_BOUND - if (err .NE. NF_NOERR) - + call error(ErrFunc(err)) -#else - if (err .NE. NF_EINVALCOORDS) - + call errore('bad start: ', err) -#endif + relax_coord_bound = relax_coord_bound_f() + if (relax_coord_bound) then + if (err .NE. NF_NOERR) + + call error(ErrFunc(err)) + else + if (err .NE. NF_EINVALCOORDS) + + call errore('bad start: ', err) + endif endif start(j) = var_shape(j,i) + 2 err = GetVars($1)(ncid, i, @@ -832,7 +834,7 @@ define([TEST_NFMPI_GET_VARM],dnl integer*8 sstride(MAX_RANK) integer*8 stride(MAX_RANK) integer*8 imap(MAX_RANK) - logical canConvert + logical canConvert, relax_coord_bound, relax_coord_bound_f DATATYPE($1, value, (MAX_NELS)) doubleprecision expect(MAX_NELS) doubleprecision val @@ -926,13 +928,14 @@ C /* there is nothing to get (edge(j).eq.0) */ if (err .NE. NF_ECHAR) + call errore('wrong type: ', err) else -#ifdef RELAX_COORD_BOUND - if (err .NE. NF_NOERR) - + call error(ErrFunc(err)) -#else - if (err .NE. NF_EINVALCOORDS) - + call errore('bad start: ', err) -#endif + relax_coord_bound = relax_coord_bound_f() + if (relax_coord_bound) then + if (err .NE. NF_NOERR) + + call error(ErrFunc(err)) + else + if (err .NE. NF_EINVALCOORDS) + + call errore('bad start: ', err) + endif endif start(j) = var_shape(j,i) + 2 err = GetVarm($1)(ncid, i, diff --git a/test/nf_test/test_iget.m4 b/test/nf_test/test_iget.m4 index d188c8d5fe..69c399d0e0 100644 --- a/test/nf_test/test_iget.m4 +++ b/test/nf_test/test_iget.m4 @@ -322,7 +322,7 @@ define([TEST_NFMPI_IGET_VARA],[dnl integer*8 edge(MAX_RANK) integer*8 index(MAX_RANK) integer*8 mid(MAX_RANK) - logical canConvert + logical canConvert, relax_coord_bound, relax_coord_bound_f DATATYPE($1, value, (MAX_NELS)) doubleprecision expect(MAX_NELS) doubleprecision val @@ -399,13 +399,14 @@ C /* there is nothing to get (edge(j).eq.0) */ if (err .NE. NF_ECHAR) + call errore('wrong type: ', err) else -#ifdef RELAX_COORD_BOUND - if (err .ne. NF_NOERR) - + call error(ErrFunc(err)) -#else - if (err .ne. NF_EINVALCOORDS) - + call errore('bad start: ', err) -#endif + relax_coord_bound = relax_coord_bound_f() + if (relax_coord_bound) then + if (err .ne. NF_NOERR) + + call error(ErrFunc(err)) + else + if (err .ne. NF_EINVALCOORDS) + + call errore('bad start: ', err) + endif endif start(j) = var_shape(j,i) + 2 err = iGetVara($1)(ncid, i, @@ -559,7 +560,7 @@ define([TEST_NFMPI_IGET_VARS],dnl integer*8 count(MAX_RANK) integer*8 sstride(MAX_RANK) integer*8 stride(MAX_RANK) - logical canConvert + logical canConvert, relax_coord_bound, relax_coord_bound_f DATATYPE($1, value, (MAX_NELS)) doubleprecision expect(MAX_NELS) doubleprecision val @@ -648,13 +649,14 @@ C /* there is nothing to get (edge(j).eq.0) */ if (err .NE. NF_ECHAR) + call errore('wrong type: ', err) else -#ifdef RELAX_COORD_BOUND - if (err .ne. NF_NOERR) - + call error(ErrFunc(err)) -#else - if (err .ne. NF_EINVALCOORDS) - + call errore('bad start: ', err) -#endif + relax_coord_bound = relax_coord_bound_f() + if (relax_coord_bound) then + if (err .ne. NF_NOERR) + + call error(ErrFunc(err)) + else + if (err .ne. NF_EINVALCOORDS) + + call errore('bad start: ', err) + endif endif start(j) = var_shape(j,i) + 2 err = iGetVars($1)(ncid, i, @@ -843,7 +845,7 @@ define([TEST_NFMPI_IGET_VARM],dnl integer*8 sstride(MAX_RANK) integer*8 stride(MAX_RANK) integer*8 imap(MAX_RANK) - logical canConvert + logical canConvert, relax_coord_bound, relax_coord_bound_f DATATYPE($1, value, (MAX_NELS)) doubleprecision expect(MAX_NELS) doubleprecision val @@ -933,13 +935,14 @@ C /* there is nothing to get (edge(j).eq.0) */ if (err .NE. NF_ECHAR) + call errore('wrong type: ', err) else -#ifdef RELAX_COORD_BOUND - if (err .ne. NF_NOERR) - + call error(ErrFunc(err)) -#else - if (err .ne. NF_EINVALCOORDS) - + call errore('bad start: ', err) -#endif + relax_coord_bound = relax_coord_bound_f() + if (relax_coord_bound) then + if (err .ne. NF_NOERR) + + call error(ErrFunc(err)) + else + if (err .ne. NF_EINVALCOORDS) + + call errore('bad start: ', err) + endif endif start(j) = var_shape(j,i) + 2 err = iGetVarm($1)(ncid, i, diff --git a/test/nf_test/test_iput.m4 b/test/nf_test/test_iput.m4 index 3fa39f3bb6..d364cae4f2 100644 --- a/test/nf_test/test_iput.m4 +++ b/test/nf_test/test_iput.m4 @@ -501,6 +501,7 @@ define([TEST_NFMPI_IPUT_VARA],dnl integer*8 index(MAX_RANK) logical canConvert !/* Both text or both numeric */ logical allInExtRange !/* all values within external range? */ + logical relax_coord_bound, relax_coord_bound_f DATATYPE($1, value, (MAX_NELS)) doubleprecision val integer ud_shift @@ -599,13 +600,14 @@ C /* Check correct error returned even when nothing to put */ if (err .ne. NF_ECHAR) + call errore('wrong type: ', err) else -#ifdef RELAX_COORD_BOUND - if (err .ne. NF_NOERR) - + call error(ErrFunc(err)) -#else - if (err .ne. NF_EINVALCOORDS) - + call errore('bad start: ', err) -#endif + relax_coord_bound = relax_coord_bound_f() + if (relax_coord_bound) then + if (err .ne. NF_NOERR) + + call error(ErrFunc(err)) + else + if (err .ne. NF_EINVALCOORDS) + + call errore('bad start: ', err) + endif endif start(j) = var_shape(j,i) + 2 err = iPutVara($1)(ncid, i, @@ -751,6 +753,7 @@ define([TEST_NFMPI_IPUT_VARS],dnl integer*8 stride(MAX_RANK) logical canConvert !/* Both text or both numeric */ logical allInExtRange !/* all values within external range? */ + logical relax_coord_bound, relax_coord_bound_f DATATYPE($1, value, (MAX_NELS)) doubleprecision val integer ud_shift @@ -861,13 +864,14 @@ C /* Check correct error returned even when nothing to put */ if (err .ne. NF_ECHAR) + call errore('wrong type: ', err) else -#ifdef RELAX_COORD_BOUND - if (err .ne. NF_NOERR) - + call error(ErrFunc(err)) -#else - if (err .ne. NF_EINVALCOORDS) - + call errore('bad start: ', err) -#endif + relax_coord_bound = relax_coord_bound_f() + if (relax_coord_bound) then + if (err .ne. NF_NOERR) + + call error(ErrFunc(err)) + else + if (err .ne. NF_EINVALCOORDS) + + call errore('bad start: ', err) + endif endif start(j) = var_shape(j,i) + 2 err = iPutVars($1)(ncid, i, @@ -1045,6 +1049,7 @@ define([TEST_NFMPI_IPUT_VARM],dnl integer*8 imap(MAX_RANK) logical canConvert !/* Both text or both numeric */ logical allInExtRange !/* all values within external range? */ + logical relax_coord_bound, relax_coord_bound_f DATATYPE($1, value, (MAX_NELS)) doubleprecision val integer ud_shift @@ -1156,13 +1161,14 @@ C /* Check correct error returned even when nothing to put */ if (err .ne. NF_ECHAR) + call errore('wrong type: ', err) else -#ifdef RELAX_COORD_BOUND - if (err .ne. NF_NOERR) - + call error(ErrFunc(err)) -#else - if (err .ne. NF_EINVALCOORDS) - + call errore('bad start: ', err) -#endif + relax_coord_bound = relax_coord_bound_f() + if (relax_coord_bound) then + if (err .ne. NF_NOERR) + + call error(ErrFunc(err)) + else + if (err .ne. NF_EINVALCOORDS) + + call errore('bad start: ', err) + endif endif start(j) = var_shape(j,i) + 2 err = iPutVarm($1)(ncid, i, diff --git a/test/nf_test/test_put.m4 b/test/nf_test/test_put.m4 index 7ea68ff073..0ca97e5f4f 100644 --- a/test/nf_test/test_put.m4 +++ b/test/nf_test/test_put.m4 @@ -729,6 +729,7 @@ define([TEST_NFMPI_PUT_VARA],dnl integer*8 index(MAX_RANK) logical canConvert !/* Both text or both numeric */ logical allInExtRange !/* all values within external range? */ + logical relax_coord_bound, relax_coord_bound_f DATATYPE($1, value, (MAX_NELS)) doubleprecision val integer ud_shift @@ -843,13 +844,14 @@ C /* Check correct error returned even when nothing to put */ if (err .ne. NF_ECHAR) + call errore('conversion: ', err) else -#ifdef RELAX_COORD_BOUND - if (err .ne. NF_NOERR) - + call errore('expect no error: ', err) -#else - if (err .ne. NF_EINVALCOORDS) - + call errore('bad start: ', err) -#endif + relax_coord_bound = relax_coord_bound_f() + if (relax_coord_bound) then + if (err .ne. NF_NOERR) + + call errore('expect no error: ', err) + else + if (err .ne. NF_EINVALCOORDS) + + call errore('bad start: ', err) + endif endif start(j) = var_shape(j,i) + 2 err = PutVaraAll($1)(ncid, i, @@ -980,6 +982,7 @@ define([TEST_NFMPI_PUT_VARS],dnl integer*8 stride(MAX_RANK) logical canConvert !/* Both text or both numeric */ logical allInExtRange !/* all values within external range? */ + logical relax_coord_bound, relax_coord_bound_f DATATYPE($1, value, (MAX_NELS)) doubleprecision val integer ud_shift @@ -1099,13 +1102,14 @@ C /* Check correct error returned even when nothing to put */ if (err .ne. NF_ECHAR) + call errore('conversion: ', err) else -#ifdef RELAX_COORD_BOUND - if (err .ne. NF_NOERR) - + call errore('expect no error: ', err) -#else - if (err .ne. NF_EINVALCOORDS) - + call errore('bad start: ', err) -#endif + relax_coord_bound = relax_coord_bound_f() + if (relax_coord_bound) then + if (err .ne. NF_NOERR) + + call errore('expect no error: ', err) + else + if (err .ne. NF_EINVALCOORDS) + + call errore('bad start: ', err) + endif endif start(j) = var_shape(j,i) + 2 err = PutVarsAll($1)(ncid, i, @@ -1269,6 +1273,7 @@ define([TEST_NFMPI_PUT_VARM],dnl integer*8 imap(MAX_RANK) logical canConvert !/* Both text or both numeric */ logical allInExtRange !/* all values within external range? */ + logical relax_coord_bound, relax_coord_bound_f DATATYPE($1, value, (MAX_NELS)) doubleprecision val integer ud_shift @@ -1388,13 +1393,14 @@ C /* Check correct error returned even when nothing to put */ if (err .ne. NF_ECHAR) + call errore('conversion: ', err) else -#ifdef RELAX_COORD_BOUND - if (err .ne. NF_NOERR) - + call errore('expect no error: ', err) -#else - if (err .ne. NF_EINVALCOORDS) - + call errore('bad start: ', err) -#endif + relax_coord_bound = relax_coord_bound_f() + if (relax_coord_bound) then + if (err .ne. NF_NOERR) + + call errore('expect no error: ', err) + else + if (err .ne. NF_EINVALCOORDS) + + call errore('bad start: ', err) + endif endif start(j) = var_shape(j,i) + 2 err = PutVarmAll($1)(ncid, i, diff --git a/test/nonblocking/Makefile.am b/test/nonblocking/Makefile.am index 568154932d..f4b0f25382 100644 --- a/test/nonblocking/Makefile.am +++ b/test/nonblocking/Makefile.am @@ -19,7 +19,7 @@ AM_FCFLAGS = $(FC_MODINC)$(top_builddir)/src/binding/f90 \ $(FC_MODINC)$(srcdir)/../common $(FFREEFORMFLAG) AM_FFLAGS = -I$(top_builddir)/src/binding/f77 $(FFIXEDFORMFLAG) -LDADD = ${top_builddir}/src/libs/libpnetcdf.la ../common/libtestutils.la +LDADD = ../common/libtestutils.la ${top_builddir}/src/libs/libpnetcdf.la LDADD += @NETCDF4_LDFLAGS@ @ADIOS_LDFLAGS@ @NETCDF4_LIBS@ @ADIOS_LIBS@ -lm # suppress warning messages when using NAG Fortran compiler @@ -50,19 +50,19 @@ if DECL_MPI_OFFSET AM_FCFLAGS += $(FC_DEFINE)HAVE_DECL_MPI_OFFSET endif -TESTPROGRAMS = test_bput \ - interleaved \ - i_varn_int64 \ - flexible_bput \ - wait_after_indep \ - req_all \ - i_varn_indef \ - large_num_reqs +M4_SRCS = bput_varn.m4 \ + column_wise.m4 -M4_SRCS = bput_varn.m4 \ - column_wise.m4 - -TESTPROGRAMS += $(M4_SRCS:.m4=) +check_PROGRAMS = test_bput \ + interleaved \ + i_varn_int64 \ + flexible_bput \ + wait_after_indep \ + req_all \ + i_varn_indef \ + large_num_reqs \ + mcoll_perf \ + $(M4_SRCS:.m4=) $(M4_SRCS:.m4=.c): Makefile @@ -80,49 +80,47 @@ nodist_bput_varn_SOURCES = bput_varn.c nodist_column_wise_SOURCES = column_wise.c if HAS_FORTRAN - TESTPROGRAMS += mcoll_testf77 \ - test_bputf77 + check_PROGRAMS += mcoll_testf77 \ + test_bputf77 mcoll_testf77_SOURCES = mcoll_testf77.f test_bputf77_SOURCES = test_bputf77.f if HAVE_MPI_MOD - TESTPROGRAMS += mcoll_testf \ - test_bputf + check_PROGRAMS += mcoll_testf \ + test_bputf mcoll_testf_SOURCES = mcoll_testf.f90 test_bputf_SOURCES = test_bputf.f90 endif endif -check_PROGRAMS = $(TESTPROGRAMS) mcoll_perf - # autimake 1.11.3 has not yet implemented AM_TESTS_ENVIRONMENT # For newer versions, we can use AM_TESTS_ENVIRONMENT instead -# AM_TESTS_ENVIRONMENT = TESTPROGRAMS="$(TESTPROGRAMS)" ; export TESTPROGRAMS; +# AM_TESTS_ENVIRONMENT = check_PROGRAMS="$(check_PROGRAMS)" ; export check_PROGRAMS; # AM_TESTS_ENVIRONMENT += TESTSEQRUN="$(TESTSEQRUN)" ; export TESTSEQRUN; # AM_TESTS_ENVIRONMENT += TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)" ; export TESTOUTDIR; TESTS_ENVIRONMENT = export SED="$(SED)"; TESTS_ENVIRONMENT += export srcdir="$(srcdir)"; +TESTS_ENVIRONMENT += export top_builddir="$(top_builddir)"; TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; TESTS_ENVIRONMENT += export TESTMPIRUN="$(TESTMPIRUN)"; -TESTS_ENVIRONMENT += export PNETCDF_DEBUG="$(PNETCDF_DEBUG)"; -TESTS_ENVIRONMENT += export TESTPROGRAMS="$(TESTPROGRAMS)"; TESTS_ENVIRONMENT += export check_PROGRAMS="$(check_PROGRAMS)"; -TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER="$(ENABLE_BURST_BUFFER)"; -TESTS = $(TESTPROGRAMS) seq_runs.sh +TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER=@ENABLE_BURST_BUFFER@; +TESTS_ENVIRONMENT += export ENABLE_THREAD_SAFE=@ENABLE_THREAD_SAFE@; +TESTS_ENVIRONMENT += export MIMIC_LUSTRE=@MIMIC_LUSTRE@; + +TESTS = $(check_PROGRAMS) TEST_EXTENSIONS = .sh -LOG_COMPILER = $(srcdir)/wrap_runs.sh +LOG_COMPILER = $(srcdir)/seq_runs.sh SH_LOG_COMPILER = -NC_FILES = $(TESTPROGRAMS:%=$(TESTOUTDIR)/%.nc) \ - $(TESTPROGRAMS:%=$(TESTOUTDIR)/%.bb.nc) +CLEANFILES = $(M4_SRCS:.m4=.c) $(TESTOUTDIR)/*.nc \ + core core.* *.gcda *.gcno *.gcov gmon.out -CLEANFILES = $(M4_SRCS:.m4=.c) core core.* *.gcda *.gcno *.gcov gmon.out \ - $(TESTOUTDIR)/testfile*.nc $(NC_FILES) \ - $(TESTOUTDIR)/mcoll_perf.nc.* $(TESTOUTDIR)/mcoll_perf.bb.nc.* +check_SCRIPTS = seq_runs.sh parallel_run.sh -EXTRA_DIST = $(M4_SRCS) seq_runs.sh wrap_runs.sh parallel_run.sh +EXTRA_DIST = $(M4_SRCS) seq_runs.sh parallel_run.sh ../common/libtestutils.la: set -e; cd ../common && $(MAKE) $(MFLAGS) tests diff --git a/test/nonblocking/bput_varn.m4 b/test/nonblocking/bput_varn.m4 index 8cc4a9541c..6f83b10b42 100644 --- a/test/nonblocking/bput_varn.m4 +++ b/test/nonblocking/bput_varn.m4 @@ -35,28 +35,28 @@ dnl * data: * * var0 = - * 13, 13, 13, 11, 11, 10, 10, 12, 11, 11, - * 10, 12, 12, 12, 13, 11, 11, 12, 12, 12, - * 11, 11, 12, 13, 13, 13, 10, 10, 11, 11, - * 10, 10, 10, 12, 11, 11, 11, 13, 13, 13 ; + * 3, 3, 3, 1, 1, 0, 0, 2, 1, 1, + * 0, 2, 2, 2, 3, 1, 1, 2, 2, 2, + * 1, 1, 2, 3, 3, 3, 0, 0, 1, 1, + * 0, 0, 0, 2, 1, 1, 1, 3, 3, 3 ; * var1 = - * 12, 12, 12, 10, 10, 13, 13, 11, 10, 10, - * 13, 11, 11, 11, 12, 10, 10, 11, 11, 11, - * 10, 10, 11, 12, 12, 12, 13, 13, 10, 10, - * 13, 13, 13, 11, 10, 10, 10, 12, 12, 12 ; + * 2, 2, 2, 0, 0, 3, 3, 1, 0, 0, + * 3, 1, 1, 1, 2, 0, 0, 1, 1, 1, + * 0, 0, 1, 2, 2, 2, 3, 3, 0, 0, + * 3, 3, 3, 1, 0, 0, 0, 2, 2, 2 ; * * var2 = - * 11, 11, 11, 13, 13, 12, 12, 10, 13, 13, - * 12, 10, 10, 10, 11, 13, 13, 10, 10, 10, - * 13, 13, 10, 11, 11, 11, 12, 12, 13, 13, - * 12, 12, 12, 10, 13, 13, 13, 11, 11, 11 ; + * 1, 1, 1, 3, 3, 2, 2, 0, 3, 3, + * 2, 0, 0, 0, 1, 3, 3, 0, 0, 0, + * 3, 3, 0, 1, 1, 1, 2, 2, 3, 3, + * 2, 2, 2, 0, 3, 3, 3, 1, 1, 1 ; * * var3 = - * 10, 10, 10, 12, 12, 11, 11, 13, 12, 12, - * 11, 13, 13, 13, 10, 12, 12, 13, 13, 13, - * 12, 12, 13, 10, 10, 10, 11, 11, 12, 12, - * 11, 11, 11, 13, 12, 12, 12, 10, 10, 10 ; + * 0, 0, 0, 2, 2, 1, 1, 3, 2, 2, + * 1, 3, 3, 3, 0, 2, 2, 3, 3, 3, + * 2, 2, 3, 0, 0, 0, 1, 1, 2, 2, + * 1, 1, 1, 3, 2, 2, 2, 0, 0, 0 ; * } * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -68,6 +68,8 @@ dnl #include #include /* strcpy() */ #include /* basename() */ +#include /* getopt() */ + #include #include @@ -76,6 +78,7 @@ dnl #define NLOOPS 4 #define MAX_NREQS 6 #define NDIMS 2 +#define NVARS 4 #define NY 4 #define NX 10 @@ -89,9 +92,10 @@ include(`utils.m4')dnl int _i; \ for (_i=0; _i<(n); _i++) { \ if ((a)[_i] != NC_NOERR) { \ - printf("Error at line %d in %s: err[%d] %s\n", __LINE__, __FILE__, _i, \ + fprintf(stderr,"Error at line %d in %s: err[%d] %s\n", __LINE__, __FILE__, _i, \ ncmpi_strerrno((a)[_i])); \ nerrs++; \ + goto err_out; \ } \ } \ } @@ -107,7 +111,7 @@ check_num_pending_reqs(int ncid, int expected, int lineno) err = ncmpi_inq_nreqs(ncid, &n_pendings); CHECK_ERR if (n_pendings != expected) { - printf("Error at line %d in %s: expect %d pending requests but got %d\n", + fprintf(stderr,"Error at line %d in %s: expect %d pending requests but got %d\n", lineno, __FILE__, expected, n_pendings); nerrs++; } @@ -134,14 +138,14 @@ int check_attached_buffer_usage(int ncid, err = ncmpi_inq_buffer_size(ncid, &buf_size); CHECK_ERR if (expected_size != buf_size) { - printf("Error at line %d in %s: expect buffer size "OFFFMT" but got "OFFFMT"\n", + fprintf(stderr,"Error at line %d in %s: expect buffer size "OFFFMT" but got "OFFFMT"\n", lineno, __FILE__,expected_size, buf_size); nerrs++; } err = ncmpi_inq_buffer_usage(ncid, &usage); CHECK_ERR if (expected_usage != usage) { - printf("Error at line %d in %s: expect buffer usage "OFFFMT" but got "OFFFMT"\n", + fprintf(stderr,"Error at line %d in %s: expect buffer usage "OFFFMT" but got "OFFFMT"\n", lineno, __FILE__,expected_usage, usage); nerrs++; } @@ -162,70 +166,86 @@ void permute(MPI_Offset *a, MPI_Offset *b) define(`TEST_BPUT_VARN',`dnl static -int clear_file_contents_$1(int ncid, int *varid) +int clear_file_contents_$1(int ncid, int *varid, int coll_io) { - int i, err, nerrs=0, rank; + int i, err, nerrs=0; $1 *w_buffer = ($1*) malloc(sizeof($1) * NY*NX); - for (i=0; i 4) MPI_Barrier(MPI_COMM_WORLD); - for (i=0; i<4; i++) { - for (j=0; j= 10+nprocs) continue; - if (r_buffer[j] != expected[i][j]) { - printf("Expected read $1 buf[%d][%d]=%d, but got %d\n", - i,j,(int)expected[i][j],(int)r_buffer[j]); + $1 exp = (expected[i][j] >= nprocs) ? fillv : expected[i][j]; + if (r_buffer[j] != exp) { + char var_name[16]; + ncmpi_inq_varname(ncid, varid[i], var_name); + printf("Expected var %s read $1 buf[%d][%d]=%d, but got %d\n", + var_name,i,j,(int)exp,(int)r_buffer[j]); nerrs++; - goto fn_exit; + break; } } + MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD); + if (nerrs > 0) goto fn_exit; } fn_exit: free(r_buffer); @@ -233,10 +253,10 @@ fn_exit: } static int -test_bput_varn_$1(char *filename, int cdf) +test_bput_varn_$1(const char *out_path, int format, int coll_io, MPI_Info info) { - int i, j, k, rank, err, nerrs=0, bb_enabled; - int ncid, cmode, varid[NLOOPS], dimid[2], nreqs, reqs[NLOOPS], sts[NLOOPS]; + int i, j, k, rank, err, nerrs=0, bb_enabled, fmt; + int ncid, varid[NLOOPS], dimid[2], nreqs, reqs[NLOOPS], sts[NLOOPS]; int req_lens[NLOOPS], my_nsegs[NLOOPS], num_segs[NLOOPS] = {4, 6, 5, 4}; $1 *buffer[NLOOPS]; @@ -276,13 +296,12 @@ test_bput_varn_$1(char *filename, int cdf) MPI_Comm_rank(MPI_COMM_WORLD, &rank); + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + /* create a new file for writing ----------------------------------------*/ - cmode = NC_CLOBBER; - if (cdf == NC_FORMAT_CDF2) - cmode |= NC_64BIT_OFFSET; - else if (cdf == NC_FORMAT_CDF5) - cmode |= NC_64BIT_DATA; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR { @@ -299,15 +318,6 @@ test_bput_varn_$1(char *filename, int cdf) MPI_Info_free(&infoused); } - /* create a global array of size NY * NX */ - err = ncmpi_def_dim(ncid, "Y", NY, &dimid[0]); CHECK_ERR - err = ncmpi_def_dim(ncid, "X", NX, &dimid[1]); CHECK_ERR - err = ncmpi_def_var(ncid, "var0", NC_TYPE($1), NDIMS, dimid, &varid[0]); CHECK_ERR - err = ncmpi_def_var(ncid, "var1", NC_TYPE($1), NDIMS, dimid, &varid[1]); CHECK_ERR - err = ncmpi_def_var(ncid, "var2", NC_TYPE($1), NDIMS, dimid, &varid[2]); CHECK_ERR - err = ncmpi_def_var(ncid, "var3", NC_TYPE($1), NDIMS, dimid, &varid[3]); CHECK_ERR - err = ncmpi_enddef(ncid); CHECK_ERR - /* allocate space for starts and counts */ starts[0] = (MPI_Offset**) malloc(sizeof(MPI_Offset*) * 4 * 6); counts[0] = (MPI_Offset**) malloc(sizeof(MPI_Offset*) * 4 * 6); @@ -338,17 +348,32 @@ test_bput_varn_$1(char *filename, int cdf) } } + /* only rank 0, 1, 2, and 3 do I/O: + * each of ranks 0 to 3 write nreqs nonblocking requests */ + nreqs = 4; + if (rank >= 4) nreqs = 0; + + /* create a global array of size NY * NX */ + err = ncmpi_def_dim(ncid, "Y", NY, &dimid[0]); CHECK_ERR + err = ncmpi_def_dim(ncid, "X", NX, &dimid[1]); CHECK_ERR + for (i=0; i= 4) nreqs = 0; - /* bufsize must be max of data type converted before and after */ MPI_Offset bufsize = 0; @@ -364,7 +389,7 @@ test_bput_varn_$1(char *filename, int cdf) /* allocate I/O buffer and initialize its contents */ buffer[i] = ($1*) malloc(sizeof($1) * req_lens[i]); - for (j=0; j 0) goto err_out; /* write using varn API, one bput call per variable */ for (i=0; i 0) goto err_out; + nerrs += check_num_pending_reqs(ncid, nreqs, __LINE__); - if (!bb_enabled) + if (nerrs > 0) goto err_out; + if (!bb_enabled) { /* burst buffering driver does not use attached memory */ nerrs += check_attached_buffer_usage(ncid, bufsize, bufsize, __LINE__); + if (nerrs > 0) goto err_out; + } - err = ncmpi_wait_all(ncid, nreqs, reqs, sts); + if (coll_io) + err = ncmpi_wait_all(ncid, nreqs, reqs, sts); + else + err = ncmpi_wait(ncid, nreqs, reqs, sts); ERRS(nreqs, sts) - if (!bb_enabled) + if (!coll_io) { + /* When in independent I/O mode, some processes may read before other + * write completion from previous writes. + */ + err = ncmpi_flush(ncid); + if (err != NC_NOERR) goto err_out; + MPI_Barrier(MPI_COMM_WORLD); + } + + if (!bb_enabled) { /* now usgae of attached memory should be 0 */ nerrs += check_attached_buffer_usage(ncid, bufsize, 0, __LINE__); + if (nerrs > 0) goto err_out; + } /* all processes read entire variables back and check contents */ - nerrs += check_contents_for_fail_$1(ncid, varid); + nerrs += check_contents_for_fail_$1(ncid, varid, coll_io); + if (nerrs > 0) goto err_out; /* permute write order: so starts[*] are not in an increasing order: * swap segment 0 with segment 2 and swap segment 1 with segment 3 @@ -422,88 +472,140 @@ test_bput_varn_$1(char *filename, int cdf) } /* write using varn API, one bput call per variable */ - nerrs += clear_file_contents_$1(ncid, varid); + nerrs += clear_file_contents_$1(ncid, varid, coll_io); + if (nerrs > 0) goto err_out; for (i=0; i 0) goto err_out; + nerrs += check_num_pending_reqs(ncid, nreqs, __LINE__); - if (!bb_enabled) + if (nerrs > 0) goto err_out; + if (!bb_enabled) { /* burst buffering driver does not use attached memory */ nerrs += check_attached_buffer_usage(ncid, bufsize, bufsize, __LINE__); + if (nerrs > 0) goto err_out; + } - err = ncmpi_wait_all(ncid, nreqs, reqs, sts); + if (coll_io) + err = ncmpi_wait_all(ncid, nreqs, reqs, sts); + else + err = ncmpi_wait(ncid, nreqs, reqs, sts); ERRS(nreqs, sts) - if (!bb_enabled) + if (!coll_io) { + /* When in independent I/O mode, some processes may read before other + * write completion from previous writes. + */ + err = ncmpi_flush(ncid); + if (err != NC_NOERR) goto err_out; + MPI_Barrier(MPI_COMM_WORLD); + } + + if (!bb_enabled) { /* now usgae of attached memory should be 0 */ nerrs += check_attached_buffer_usage(ncid, bufsize, 0, __LINE__); + if (nerrs > 0) goto err_out; + } /* all processes read entire variables back and check contents */ - nerrs += check_contents_for_fail_$1(ncid, varid); + nerrs += check_contents_for_fail_$1(ncid, varid, coll_io); + if (nerrs > 0) goto err_out; for (i=0; i 0) goto err_out; + nerrs += check_num_pending_reqs(ncid, nreqs, __LINE__); - if (!bb_enabled) + if (!bb_enabled) { /* burst buffering driver does not use attached memory */ nerrs += check_attached_buffer_usage(ncid, bufsize, bufsize, __LINE__); + if (nerrs > 0) goto err_out; + } - err = ncmpi_wait_all(ncid, nreqs, reqs, sts); + if (coll_io) + err = ncmpi_wait_all(ncid, nreqs, reqs, sts); + else + err = ncmpi_wait(ncid, nreqs, reqs, sts); ERRS(nreqs, sts) + if (!coll_io) { + /* When in independent I/O mode, some processes may read before other + * write completion from previous writes. + */ + err = ncmpi_flush(ncid); + if (err != NC_NOERR) goto err_out; + MPI_Barrier(MPI_COMM_WORLD); + } + /* check if write buffer contents have been altered */ for (i=0; i 0) goto err_out; - if (!bb_enabled) + if (!bb_enabled) { /* now usgae of attached memory should be 0 */ nerrs += check_attached_buffer_usage(ncid, bufsize, 0, __LINE__); + if (nerrs > 0) goto err_out; + } /* all processes read entire variables back and check contents */ - nerrs += check_contents_for_fail_$1(ncid, varid); + nerrs += check_contents_for_fail_$1(ncid, varid, coll_io); + if (nerrs > 0) goto err_out; /* permute back to original order */ for (i=0; i 0) goto err_out; for (i=0; i 0) goto err_out; + nerrs += check_num_pending_reqs(ncid, nreqs, __LINE__); - if (!bb_enabled) + if (!bb_enabled) { /* burst buffering driver does not use attached memory */ nerrs += check_attached_buffer_usage(ncid, bufsize, bufsize, __LINE__); + if (nerrs > 0) goto err_out; + } - err = ncmpi_wait_all(ncid, nreqs, reqs, sts); + if (coll_io) + err = ncmpi_wait_all(ncid, nreqs, reqs, sts); + else + err = ncmpi_wait(ncid, nreqs, reqs, sts); ERRS(nreqs, sts) + if (!coll_io) { + /* When in independent I/O mode, some processes may read before other + * write completion from previous writes. + */ + err = ncmpi_flush(ncid); + if (err != NC_NOERR) goto err_out; + MPI_Barrier(MPI_COMM_WORLD); + } + /* check if write buffer contents have been altered */ for (i=0; i 0) goto err_out; - if (!bb_enabled) + if (!bb_enabled) { /* now usgae of attached memory should be 0 */ nerrs += check_attached_buffer_usage(ncid, bufsize, 0, __LINE__); + if (nerrs > 0) goto err_out; + } /* all processes read entire variables back and check contents */ - nerrs += check_contents_for_fail_$1(ncid, varid); + nerrs += check_contents_for_fail_$1(ncid, varid, coll_io); +err_out: /* free the buffer space for bput */ if (bufsize > 0) { err = ncmpi_buffer_detach(ncid); CHECK_ERR @@ -570,6 +700,9 @@ test_bput_varn_$1(char *filename, int cdf) err = ncmpi_inq_buffer_usage(ncid, &bufsize); EXP_ERR(NC_ENULLABUF) + err = ncmpi_inq_format(ncid, &fmt); + assert(format == fmt); + err = ncmpi_close(ncid); CHECK_ERR for (i=0; i 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for bput_varn ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); - free(cmd_str); - } - - for (i=0; i<3; i++) { - nerrs += test_bput_varn_text(filename, cdf_formats[i]); - nerrs += test_bput_varn_schar(filename, cdf_formats[i]); - nerrs += test_bput_varn_short(filename, cdf_formats[i]); - nerrs += test_bput_varn_int(filename, cdf_formats[i]); - nerrs += test_bput_varn_float(filename, cdf_formats[i]); - nerrs += test_bput_varn_double(filename, cdf_formats[i]); - if (cdf_formats[i] == NC_FORMAT_CDF5) { - nerrs += test_bput_varn_uchar(filename, cdf_formats[i]); - nerrs += test_bput_varn_ushort(filename, cdf_formats[i]); - nerrs += test_bput_varn_uint(filename, cdf_formats[i]); - nerrs += test_bput_varn_longlong(filename, cdf_formats[i]); - nerrs += test_bput_varn_ulonglong(filename, cdf_formats[i]); - } +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) +{ + int err; + + err = TEST_DATA_TYPE(text); if (err > 0) return err; + err = TEST_DATA_TYPE(schar); if (err > 0) return err; + err = TEST_DATA_TYPE(short); if (err > 0) return err; + err = TEST_DATA_TYPE(int); if (err > 0) return err; + err = TEST_DATA_TYPE(float); if (err > 0) return err; + err = TEST_DATA_TYPE(double); if (err > 0) return err; + if (format == NC_FORMAT_64BIT_DATA) { + err = TEST_DATA_TYPE(uchar); if (err > 0) return err; + err = TEST_DATA_TYPE(ushort); if (err > 0) return err; + err = TEST_DATA_TYPE(uint); if (err > 0) return err; + err = TEST_DATA_TYPE(longlong); if (err > 0) return err; + err = TEST_DATA_TYPE(ulonglong); if (err > 0) return err; } - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + return 0; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 0; /* disable burst-buffering */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "bput_varn", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/nonblocking/column_wise.m4 b/test/nonblocking/column_wise.m4 index 9bc15958c6..3acaf1fd1f 100644 --- a/test/nonblocking/column_wise.m4 +++ b/test/nonblocking/column_wise.m4 @@ -25,7 +25,7 @@ * * % m4 column_wise.m4 > column_wise.c * % mpicc -O2 -o column_wise column_wise.c -lpnetcdf - * % mpiexec -l -n 4 ./column_wise /pvfs2/wkliao/testfile.nc + * % mpiexec -l -n 4 ./column_wise -l 4 /pvfs2/wkliao/testfile.nc * 0: 0: myOff= 0 myNX= 4 * 1: 1: myOff= 4 myNX= 4 * 2: 2: myOff= 8 myNX= 4 @@ -67,37 +67,40 @@ #include #include /* strcpy() */ #include /* basename() */ +#include /* getopt() */ + #include #include #include #define NY 10 -#define NX 4 +#define NX 70 typedef char text; include(`foreach.m4')dnl include(`utils.m4')dnl +define(`TEST_DATA_TYPE',`test_column_wise_$1(out_path, format, coll_io, info)') + define(`TEST_COLUMN_WISE',`dnl static -int test_column_wise_$1(char *filename, int cdf) +int test_column_wise_$1(const char *out_path, int format, int coll_io, MPI_Info info) { int i, j, nerrs=0, rank, nprocs, err, myNX, G_NX, myOff, num_reqs; - int ncid, cmode, varid, dimid[2], *reqs, *sts; + int fmt, ncid, varid, dimid[2], *reqs, *sts; $1 **buf; MPI_Offset start[2], count[2]; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - cmode = NC_CLOBBER; - if (cdf == NC_FORMAT_CDF2) - cmode |= NC_64BIT_OFFSET; - else if (cdf == NC_FORMAT_CDF5) - cmode |= NC_64BIT_DATA; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR /* the global array is NY * (NX * nprocs) */ @@ -124,6 +127,11 @@ int test_column_wise_$1(char *filename, int cdf) */ err = ncmpi_flush(ncid); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + /* initialize the buffer with rank ID. Also make the case interesting, by allocatsing buffersd separately */ for (i=0; i 0) return err; + err = TEST_DATA_TYPE(schar); if (err > 0) return err; + err = TEST_DATA_TYPE(short); if (err > 0) return err; + err = TEST_DATA_TYPE(int); if (err > 0) return err; + err = TEST_DATA_TYPE(float); if (err > 0) return err; + err = TEST_DATA_TYPE(double); if (err > 0) return err; + if (format == NC_FORMAT_CDF5) { + err = TEST_DATA_TYPE(uchar); if (err > 0) return err; + err = TEST_DATA_TYPE(ushort); if (err > 0) return err; + err = TEST_DATA_TYPE(uint); if (err > 0) return err; + err = TEST_DATA_TYPE(longlong); if (err > 0) return err; + err = TEST_DATA_TYPE(ulonglong); if (err > 0) return err; + } - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); + return 0; +} - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for iput/iget interleaved access ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); - free(cmd_str); - } +int main(int argc, char **argv) { - for (i=0; i<3; i++) { - nerrs += test_column_wise_text(filename, cdf_formats[i]); - if (nerrs > 0) break; - nerrs += test_column_wise_schar(filename, cdf_formats[i]); - if (nerrs > 0) break; - nerrs += test_column_wise_short(filename, cdf_formats[i]); - if (nerrs > 0) break; - nerrs += test_column_wise_int(filename, cdf_formats[i]); - if (nerrs > 0) break; - nerrs += test_column_wise_float(filename, cdf_formats[i]); - if (nerrs > 0) break; - nerrs += test_column_wise_double(filename, cdf_formats[i]); - if (nerrs > 0) break; - if (cdf_formats[i] == NC_FORMAT_CDF5) { - nerrs += test_column_wise_uchar(filename, cdf_formats[i]); - if (nerrs > 0) break; - nerrs += test_column_wise_ushort(filename, cdf_formats[i]); - if (nerrs > 0) break; - nerrs += test_column_wise_uint(filename, cdf_formats[i]); - if (nerrs > 0) break; - nerrs += test_column_wise_longlong(filename, cdf_formats[i]); - if (nerrs > 0) break; - nerrs += test_column_wise_ulonglong(filename, cdf_formats[i]); - if (nerrs > 0) break; - } - } + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + MPI_Init(&argc, &argv); - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "iput/iget interleaved access", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/nonblocking/flexible_bput.c b/test/nonblocking/flexible_bput.c index 468af74b6c..1234b71fd3 100644 --- a/test/nonblocking/flexible_bput.c +++ b/test/nonblocking/flexible_bput.c @@ -21,7 +21,7 @@ * * % mpicc -O2 -o flexible_bput flexible_bput.c -lpnetcdf * - * % mpiexec -l -n 4 ./flexible_bput /pvfs2/wkliao/testfile.nc + * % mpiexec -l -n 4 ./flexible_bput -l 4 /pvfs2/wkliao/testfile.nc * * % ncmpidump /pvfs2/wkliao/testfile.nc * netcdf testfile { @@ -48,14 +48,16 @@ #include #include /* strcpy() */ #include /* basename() */ +#include /* getopt() */ #include + #include #include #include #define NY 6 -#define NX 4 +#define NX 70 #define GHOST 2 #define INIT_PUT_BUF(buf) \ @@ -75,16 +77,18 @@ if (i < GHOST || GHOST+array_of_subsizes[0] <= i || \ j < GHOST || GHOST+array_of_subsizes[1] <= j) { \ if (buf[i][j] != -1) { \ - printf("Error at line %d in %s: put buffer altered buffer[%d][%d]=%f\n", \ + fprintf(stderr,"Error at line %d in %s: put buffer altered buffer[%d][%d]=%f\n", \ __LINE__,__FILE__,i,j,(double)buf[i][j]); \ nerrs++; \ + goto err_out; \ } \ } \ else { \ if (buf[i][j] != (i-GHOST)*array_of_subsizes[1]+(j-GHOST)) { \ - printf("Error at line %d in %s: put buffer altered buffer[%d][%d]=%f\n", \ + fprintf(stderr,"Error at line %d in %s: put buffer altered buffer[%d][%d]=%f\n", \ __LINE__,__FILE__,i,j,(double)buf[i][j]); \ nerrs++; \ + goto err_out; \ } \ } \ } \ @@ -101,55 +105,58 @@ if (i < GHOST || GHOST+array_of_subsizes[0] <= i || \ j < GHOST || GHOST+array_of_subsizes[1] <= j) { \ if (buf[i][j] != -2) { \ - printf("Error at line %d in %s: unexpected get buffer[%d][%d]=%f\n", \ - __LINE__,__FILE__,i,j,(double)buf[i][j]); \ + fprintf(stderr,"Error at line %d in %s: expect buffer[%d][%d] to be %d but got %f\n", \ + __LINE__,__FILE__,i,j,-2,(double)buf[i][j]); \ nerrs++; \ + goto err_out; \ } \ } \ else { \ - if (buf[i][j] != (i-GHOST)*array_of_subsizes[1]+(j-GHOST)) { \ - printf("Error at line %d in %s: unexpected get buffer[%d][%d]=%f\n", \ - __LINE__,__FILE__,i,j,(double)buf[i][j]); \ + int exp = (i-GHOST)*array_of_subsizes[1]+(j-GHOST); \ + if (buf[i][j] != exp) { \ + fprintf(stderr,"Error at line %d in %s: expect buffer[%d][%d] to be %d but got %f\n", \ + __LINE__,__FILE__,i,j,exp,(double)buf[i][j]); \ nerrs++; \ + goto err_out; \ } \ } \ } \ } -int main(int argc, char** argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) { - char filename[256]; int i, j, rank, nprocs, err, nerrs=0, req, status; - int ncid, cmode, varid, dimid[2]; + int ncid, varid, dimid[2]; int array_of_sizes[2], array_of_subsizes[2], array_of_starts[2]; - int buf_int[NX+2*GHOST][NY+2*GHOST]; - double buf_dbl[NX+2*GHOST][NY+2*GHOST]; + int **buf_int; + double **buf_dbl; MPI_Offset start[2], count[2], stride[2], imap[2]; MPI_Datatype subarray; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for flexible bput_varm ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); - free(cmd_str); - } + buf_int = (int**) malloc(sizeof(int*) * (NX+2*GHOST)); + buf_int[0] = (int*) malloc(sizeof(int) * (NX+2*GHOST) * (NY+2*GHOST)); + for (i=1; i<(NX+2*GHOST); i++) + buf_int[i] = buf_int[i-1] + (NY+2*GHOST); + + buf_dbl = (double**) malloc(sizeof(double*) * (NX+2*GHOST)); + buf_dbl[0] = (double*) malloc(sizeof(double) * (NX+2*GHOST) * (NY+2*GHOST)); + for (i=1; i<(NX+2*GHOST); i++) + buf_dbl[i] = buf_dbl[i-1] + (NY+2*GHOST); + + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR /* create a new file for writing ----------------------------------------*/ - cmode = NC_CLOBBER | NC_64BIT_DATA; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR /* define 2 dimensions */ @@ -160,6 +167,11 @@ int main(int argc, char** argv) err = ncmpi_def_var(ncid, "var", NC_DOUBLE, 2, dimid, &varid); CHECK_ERR err = ncmpi_enddef(ncid); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + start[0] = 0; start[1] = NX * rank; count[0] = NY; count[1] = NX; stride[0] = 1; stride[1] = 1; @@ -184,13 +196,18 @@ int main(int argc, char** argv) for (i=0; i<2; i++) bufsize *= count[i]; err = ncmpi_buffer_attach(ncid, bufsize); CHECK_ERR - err = ncmpi_bput_varm(ncid, varid, start, count, stride, imap, buf_int, + err = ncmpi_bput_varm(ncid, varid, start, count, stride, imap, buf_int[0], 1, subarray, &req); CHECK_ERR /* check if the contents of put buffer are altered */ CHECK_PUT_BUF(buf_int) - err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR + if (coll_io) + err = ncmpi_wait_all(ncid, 1, &req, &status); + else + err = ncmpi_wait(ncid, 1, &req, &status); + + CHECK_ERR err = status; CHECK_ERR /* check the contents of put buffer are altered */ @@ -198,15 +215,29 @@ int main(int argc, char** argv) err = ncmpi_buffer_detach(ncid); CHECK_ERR + /* file sync before reading */ + err = ncmpi_sync(ncid); + CHECK_ERR + MPI_Barrier(MPI_COMM_WORLD); + /* read back using a blocking get_varm flexible API ---------------------*/ /* initiate get buffer contents */ INIT_GET_BUF(buf_int) - /* calling a blocking flexible API */ - err = ncmpi_get_varm_all(ncid, varid, start, count, stride, imap, buf_int, - 1, subarray); + if (!coll_io) { + /* calling a blocking flexible API */ + err = ncmpi_end_indep_data(ncid); + CHECK_ERR + } + err = ncmpi_get_varm_all(ncid, varid, start, count, stride, imap, + buf_int[0], 1, subarray); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + /* check the contents of get buffer */ CHECK_GET_BUF(buf_int) @@ -224,14 +255,18 @@ int main(int argc, char** argv) err = ncmpi_buffer_attach(ncid, bufsize); CHECK_ERR - err = ncmpi_bput_varm(ncid, varid, start, count, stride, imap, buf_dbl, + err = ncmpi_bput_varm(ncid, varid, start, count, stride, imap, buf_dbl[0], 1, subarray, &req); CHECK_ERR /* check the contents of put buffer are altered */ CHECK_PUT_BUF(buf_dbl) - err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR + if (coll_io) + err = ncmpi_wait_all(ncid, 1, &req, &status); + else + err = ncmpi_wait(ncid, 1, &req, &status); + CHECK_ERR err = status; CHECK_ERR /* check the contents of put buffer are altered */ @@ -244,36 +279,51 @@ int main(int argc, char** argv) INIT_GET_BUF(buf_dbl) /* calling a blocking flexible API */ - err = ncmpi_get_varm_all(ncid, varid, start, count, stride, imap, buf_dbl, - 1, subarray); + if (coll_io) + err = ncmpi_get_varm_all(ncid, varid, start, count, stride, imap, + buf_dbl[0], 1, subarray); + else + err = ncmpi_get_varm(ncid, varid, start, count, stride, imap, + buf_dbl[0], 1, subarray); CHECK_ERR /* check the contents of get buffer */ CHECK_GET_BUF(buf_dbl) +err_out: MPI_Type_free(&subarray); err = ncmpi_close(ncid); CHECK_ERR - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) { - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - } - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + free(buf_int[0]); + free(buf_int); + free(buf_dbl[0]); + free(buf_dbl); - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + return nerrs; +} + +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "flexible bput_varm", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/nonblocking/i_varn_indef.c b/test/nonblocking/i_varn_indef.c index 9617d99ad5..43d7a0daad 100644 --- a/test/nonblocking/i_varn_indef.c +++ b/test/nonblocking/i_varn_indef.c @@ -8,10 +8,10 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * This example tests posting nonblocking varn APIs, including - * ncmpi_iput_varn_longlong(), ncmpi_iget_varn_longlong(), ncmpi_iput_varn(), + * ncmpi_iput_varn_int(), ncmpi_iget_varn_int(), ncmpi_iput_varn(), * and ncmpi_iget_varn(), in define mode. * It first writes a sequence of requests with arbitrary array indices and - * lengths to four variables of type NC_INT64, and reads back. + * lengths to four variables of type NC_INT, and reads back. * * The compile and run commands are given below, together with an ncmpidump of * the output file. @@ -74,7 +74,7 @@ int _i; \ for (_i=0; _i<(n); _i++) { \ if ((a)[_i] != NC_NOERR) { \ - printf("Error at line %d in %s: err[%d] %s\n", __LINE__, __FILE__, _i, \ + fprintf(stderr,"Error at line %d in %s: err[%d] %s\n", __LINE__, __FILE__, _i, \ ncmpi_strerrno((a)[_i])); \ nerrs++; \ } \ @@ -82,59 +82,48 @@ } static -int clear_file_contents(int ncid, int *varid) -{ - int i, err, nerrs=0, rank; - long long *w_buffer = (long long*) malloc(sizeof(long long) * NY*NX); - for (i=0; i 4) MPI_Barrier(MPI_COMM_WORLD); for (i=0; i<4; i++) { for (j=0; j= nprocs) continue; if (r_buffer[j] != expected[i][j]) { - printf("Error at line %d in %s: Expected read buf[%d][%d]=%lld, but got %lld\n", + fprintf(stderr,"Error at line %d in %s: Expected read buf[%d][%d]=%d, but got %d\n", lineno,__FILE__,i,j,expected[i][j],r_buffer[j]); nerrs++; } @@ -152,7 +141,7 @@ check_num_pending_reqs(int ncid, int expected, int lineno) err = ncmpi_inq_nreqs(ncid, &n_pendings); CHECK_ERR if (n_pendings != expected) { - printf("Error at line %d in %s: expect %d pending requests but got %d\n", + fprintf(stderr,"Error at line %d in %s: expect %d pending requests but got %d\n", lineno, __FILE__, expected, n_pendings); nerrs++; } @@ -170,15 +159,19 @@ void permute(MPI_Offset *a, MPI_Offset *b) } } -int main(int argc, char** argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) { - char filename[256], *varname[4]; + char *varname[4]; int i, j, k, rank, nprocs, err, nerrs=0, bufsize=0; - int ncid, cmode, varid[4], dimid[2], nreqs, reqs[12], sts[4]; - long long *buffer[4], *cbuffer[4], *rbuffer[4]; + int ncid, varid[4], dimid[2], nreqs, reqs[12], sts[4]; + int *buffer[4], *cbuffer[4], *rbuffer[4]; int num_segs[4] = {4, 6, 5, 4}; int req_lens[4], my_nsegs[4]; - int bb_enabled=0; MPI_Datatype buftype[4]; MPI_Offset **starts[4], **counts[4]; @@ -212,26 +205,9 @@ int main(int argc, char** argv) - - - X X X - - - - - - - - - - - X X X */ - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for iput/iget varn in define mode ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); - free(cmd_str); - } - #ifdef DEBUG if (nprocs != 4 && rank == 0) printf("Warning: %s is intended to run on 4 processes\n",argv[0]); @@ -289,7 +265,7 @@ int main(int argc, char** argv) } /* allocate I/O buffer and initialize its contents */ - buffer[i] = (long long*) malloc(sizeof(long long) * req_lens[i]); + buffer[i] = (int*) malloc(sizeof(int) * req_lens[i]); for (j=0; j0) cbuffer[0] = (long long*) malloc(sizeof(long long) * bufsize); + if (bufsize>0) cbuffer[0] = (int*) malloc(sizeof(int) * bufsize); for (i=1; i 0) { - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - } - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + return nerrs; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "iput/iget varn in define mode", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/nonblocking/i_varn_int64.c b/test/nonblocking/i_varn_int64.c index 52700d8b1d..3e029ec31c 100644 --- a/test/nonblocking/i_varn_int64.c +++ b/test/nonblocking/i_varn_int64.c @@ -101,7 +101,7 @@ #define FATAL_ERR \ if (err != NC_NOERR) { \ - printf("Error at line %d in %s: %s\n", __LINE__, __FILE__, ncmpi_strerrno(err)); \ + fprintf(stderr,"Error at line %d in %s: %s\n", __LINE__, __FILE__, ncmpi_strerrno(err)); \ exit(1); \ } @@ -109,15 +109,15 @@ int _i; \ for (_i=0; _i<(n); _i++) { \ if ((a)[_i] != NC_NOERR) { \ - printf("Error at line %d in %s: err[%d] %s\n", __LINE__, __FILE__, _i, \ + fprintf(stderr,"Error at line %d in %s: err[%d] %s\n", __LINE__, __FILE__, _i, \ ncmpi_strerrno((a)[_i])); \ - nerrs++; \ + assert(0); \ } \ } \ } static -int clear_file_contents(int ncid, int *varid) +int clear_file_contents(int ncid, int *varid, int coll_io) { int i, err, rank, nerrs=0; MPI_Offset start[2], count[2]; @@ -127,6 +127,8 @@ int clear_file_contents(int ncid, int *varid) MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Barrier(MPI_COMM_WORLD); + start[0] = start[1] = count[0] = count[1] = 0; if (rank == 0) { /* only rank 0 writes */ count[0] = NY; @@ -138,21 +140,25 @@ int clear_file_contents(int ncid, int *varid) err = ncmpi_iput_vara_longlong(ncid, varid[i], start, count, w_buffer, NULL); CHECK_ERR } - err = ncmpi_wait_all(ncid, NC_REQ_ALL, NULL, NULL); + if (coll_io) + err = ncmpi_wait_all(ncid, NC_REQ_ALL, NULL, NULL); + else + err = ncmpi_wait(ncid, NC_REQ_ALL, NULL, NULL); CHECK_ERR - free(w_buffer); - /* When using burst buffering, flush the log to prevent new value being * skipped due to overlaping domain */ err = ncmpi_flush(ncid); CHECK_ERR + MPI_Barrier(MPI_COMM_WORLD); + + free(w_buffer); return nerrs; } static -int check_contents_for_fail(int ncid, int *varid) +int check_contents_for_fail(int ncid, int *varid, int coll_io) { /* all processes read entire variables back and check contents */ int i, j, nerrs=0, err, nprocs; @@ -176,20 +182,26 @@ int check_contents_for_fail(int ncid, int *varid) long long *r_buffer = (long long*) malloc(sizeof(long long) * NY*NX); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - if (nprocs > 4) MPI_Barrier(MPI_COMM_WORLD); + + /* file sync before reading */ + err = ncmpi_sync(ncid); CHECK_ERR + MPI_Barrier(MPI_COMM_WORLD); for (i=0; i<4; i++) { for (j=0; j= nprocs) continue; if (r_buffer[j] != expected[i][j]) { - printf("Error at line %d in %s: xxpect read buf[%d][%d]=%lld, but got %lld\n", + fprintf(stderr,"Error at line %d in %s: expect read buf[%d][%d]=%lld, but got %lld\n", __LINE__,__FILE__,i,j,expected[i][j],r_buffer[j]); - nerrs++; + assert(0); } } } @@ -205,9 +217,9 @@ check_num_pending_reqs(int ncid, int expected, int lineno) err = ncmpi_inq_nreqs(ncid, &n_pendings); CHECK_ERR if (n_pendings != expected) { - printf("Error at line %d in %s: expect %d pending requests but got %d\n", + fprintf(stderr,"Error at line %d in %s: expect %d pending requests but got %d\n", lineno, __FILE__, expected, n_pendings); - nerrs++; + assert(0); } return nerrs; } @@ -224,11 +236,11 @@ void permute(MPI_Offset *a, MPI_Offset *b) } static int -test_varn(int ncid, int rank, int *varid) +test_varn(int ncid, int rank, int *varid, int coll_io) { int i, j, k, err, nerrs=0, bufsize=0; - int nreqs, reqs[4], sts[4]; - long long *buffer[4], *cbuffer[4]; + int nreqs=0, reqs[4], sts[4]; + long long *wbuf[4], *c_wbuf[4], *rbuf[4], *c_rbuf[4]; int num_segs[4] = {4, 6, 5, 4}; int req_lens[4], my_nsegs[4]; MPI_Offset **starts[4], **counts[4]; @@ -263,6 +275,9 @@ test_varn(int ncid, int rank, int *varid) - - - - - - - X X X */ + c_wbuf[0] = NULL; + c_rbuf[0] = NULL; + /* allocate space for starts and counts */ starts[0] = (MPI_Offset**) malloc(sizeof(MPI_Offset*) * 4 * 6); counts[0] = (MPI_Offset**) malloc(sizeof(MPI_Offset*) * 4 * 6); @@ -297,9 +312,9 @@ test_varn(int ncid, int rank, int *varid) err = ncmpi_iput_varn_longlong(ncid, varid[0], 1, NULL, NULL, NULL, &reqs[0]); if (err != NC_ENULLSTART) { - printf("expecting error code NC_ENULLSTART but got %s\n", + fprintf(stderr,"expecting error code NC_ENULLSTART but got %s\n", nc_err_code_name(err)); - nerrs++; + assert(0); } /* only rank 0, 1, 2, and 3 do I/O: @@ -318,71 +333,89 @@ test_varn(int ncid, int rank, int *varid) } /* allocate I/O buffer and initialize its contents */ - buffer[i] = (long long*) malloc(sizeof(long long) * req_lens[i]); - for (j=0; j0) { + c_wbuf[0] = (long long*) malloc(sizeof(long long) * bufsize); + c_rbuf[0] = (long long*) malloc(sizeof(long long) * bufsize); + for (i=1; i0) { - cbuffer[0] = (long long*) malloc(sizeof(long long) * bufsize); - for (i=1; i0) free(cbuffer[0]); - for (i=0; i 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for iput/iget varn ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); - free(cmd_str); - } #ifdef DEBUG + int nprocs; + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); if (nprocs != 4 && rank == 0) printf("Warning: %s is intended to run on 4 processes\n",argv[0]); #endif + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + /* create a new file for writing ----------------------------------------*/ - cmode = NC_CLOBBER | NC_64BIT_DATA; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); FATAL_ERR /* create fixed-size variables of size NY * NX */ @@ -598,8 +646,13 @@ int main(int argc, char** argv) err = ncmpi_def_var(ncid, "var3", NC_INT64, NDIMS, dimid, &varid[3]); CHECK_ERR err = ncmpi_enddef(ncid); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + /* test fixed-size variables */ - nerrs += test_varn(ncid, rank, varid); + nerrs += test_varn(ncid, rank, varid, coll_io); err = ncmpi_redef(ncid); CHECK_ERR @@ -611,30 +664,61 @@ int main(int argc, char** argv) err = ncmpi_def_var(ncid, "t_var3", NC_INT64, NDIMS, dimid, &varid[3]); CHECK_ERR err = ncmpi_enddef(ncid); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + /* test record variables */ - nerrs += test_varn(ncid, rank, varid); + nerrs += test_varn(ncid, rank, varid, coll_io); err = ncmpi_close(ncid); CHECK_ERR - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) { - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - } - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + return nerrs; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) +{ + int nerrs; - MPI_Finalize(); - return (nerrs > 0); + /* test without setting hint nc_data_move_chunk_size */ + nerrs = i_varn_int64(out_path, format, coll_io, info); + if (nerrs > 0) return nerrs; + + /* test with setting hint nc_data_move_chunk_size */ + MPI_Info_set(info, "nc_data_move_chunk_size", "100"); + nerrs = i_varn_int64(out_path, format, coll_io, info); + if (nerrs > 0) return nerrs; + + return 0; } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "iput/iget varn", opt, test_io); + + MPI_Finalize(); + + return err; +} diff --git a/test/nonblocking/interleaved.c b/test/nonblocking/interleaved.c index 606b74be57..da309e1aba 100644 --- a/test/nonblocking/interleaved.c +++ b/test/nonblocking/interleaved.c @@ -78,53 +78,45 @@ #define CHECK_CONTENTS(buf, exp) { \ if (buf[i] != (exp)) { \ - printf("Error at line %d in %s: put buffer[%d] altered to %d, expect %d\n", \ + fprintf(stderr,"Error at line %d in %s: put buffer[%d] altered to %d, expect %d\n", \ __LINE__,__FILE__,i,buf[i],(exp)); \ nerrs++; \ } \ } -int main(int argc, char** argv) + +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info global_info) { - char filename[256]; int i, j, rank, nprocs, err, nerrs=0, expected; - int ncid, cmode, varid[2], dimid[2], req[4], st[4], *buf; - int *buf0, *buf1, *buf2; + int ncid, varid[2], dimid[2], req[4], st[4], *buf=NULL; + int *buf0=NULL, *buf1=NULL, *buf2=NULL; size_t len; MPI_Offset start[2], count[2]; MPI_Info info; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); /* this program is intended to run on one process */ - if (rank) goto fn_exit; - - /* get command-line arguments */ - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for writing interleaved fileviews ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); - free(cmd_str); - } + if (rank) goto err_out; - MPI_Info_create(&info); + MPI_Info_dup(global_info, &info); MPI_Info_set(info, "romio_cb_write", "disable"); MPI_Info_set(info, "ind_wr_buffer_size", "8"); /* these 2 hints are required to cause a core dump if r1758 fix is not * presented */ + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + /* create a new file for writing ----------------------------------------*/ - cmode = NC_CLOBBER | NC_64BIT_DATA; - err = ncmpi_create(MPI_COMM_SELF, filename, cmode, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_SELF, out_path, NC_CLOBBER, info, &ncid); + CHECK_FATAL_ERR MPI_Info_free(&info); @@ -142,12 +134,21 @@ int main(int argc, char** argv) /* do not forget to exit define mode */ err = ncmpi_enddef(ncid); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + /* now we are in data mode */ buf = (int*) malloc(sizeof(int) * NY*NX); /* fill the entire variable var0 with -1s */ for (i=0; i 0) { - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", malloc_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } +err_out: + MPI_Barrier(MPI_COMM_WORLD); -fn_exit: - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + return nerrs; +} + +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "writing interleaved fileviews", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/nonblocking/large_num_reqs.c b/test/nonblocking/large_num_reqs.c index 3ea4e3211f..d13d86da5f 100644 --- a/test/nonblocking/large_num_reqs.c +++ b/test/nonblocking/large_num_reqs.c @@ -22,34 +22,25 @@ #define FILE_NAME "testfile.nc" #define NUM_REQS 1100 /* a number greater than NC_REQUEST_CHUNK */ -/*----< main() >------------------------------------------------------------*/ -int main(int argc, char **argv) { +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) +{ int i, ncid, dimid[2], varid, err, nerrs=0, rank, nprocs; int *buf, *req, *status; - char filename[256]; MPI_Offset start[2], count[2]; - MPI_Info info=MPI_INFO_NULL; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for large number of iput/iget ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); - free(cmd_str); - } + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "Y", NC_UNLIMITED, &dimid[0]); CHECK_ERR #define STRESS_ROMIO @@ -66,6 +57,11 @@ int main(int argc, char **argv) { err = ncmpi_def_var(ncid, "var", NC_INT, 2, dimid, &varid); CHECK_ERR err = ncmpi_enddef(ncid); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + req = (int*) malloc(sizeof(int) * NUM_REQS * 2); status = req + NUM_REQS; @@ -85,13 +81,22 @@ int main(int argc, char **argv) { start[0] += 3; } - err = ncmpi_wait_all(ncid, NUM_REQS, req, status); CHECK_ERR + if (coll_io) + err = ncmpi_wait_all(ncid, NUM_REQS, req, status); + else + err = ncmpi_wait(ncid, NUM_REQS, req, status); + CHECK_ERR + /* check each iput status */ for (i=0; i 0) { - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - } - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + return nerrs; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "large number of iput/iget", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/nonblocking/mcoll_perf.c b/test/nonblocking/mcoll_perf.c index 9d0cf1f4a2..5314e2ffe7 100644 --- a/test/nonblocking/mcoll_perf.c +++ b/test/nonblocking/mcoll_perf.c @@ -300,19 +300,26 @@ int ncmpi_diff(char *filename1, char *filename2) } -int main(int argc, char **argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) { + /* file sync before reading */ + extern int optind; + extern char *optarg; int i, j, array_of_gsizes[3]; int nprocs, **buf, rank; MPI_Offset bufcount; int array_of_psizes[3]; int err, nerrs=0; MPI_Offset array_of_starts[3], stride[3]; - char fbasename[256], filename[512]; + char filename[512]; char filename1[512], filename2[512], filename3[512]; char dimname[20], varname[20]; int ncid, dimids0[3], dimids1[3], rank_dim[3], *varid; - MPI_Info info; MPI_Offset **starts, **counts; MPI_Offset *bufcounts; int ndims = 3; @@ -324,60 +331,50 @@ int main(int argc, char **argv) int *sts; int *buf_var; int nvars2; + int keep_files; /* int buf_var[32] ={1, 1, 1, 1, 2, 2, 2, 2, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 3, 3, 3, 3, 4, 4, 4, 4}; */ - MPI_Init(&argc, &argv); + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); verbose = 0; - if (argc > 2) { - if (!rank) printf("Usage: %s [file base name]\n",argv[0]); - MPI_Finalize(); - nerrs++; goto fn_exit; - } - if (argc == 2) snprintf(fbasename, 256, "%s", argv[1]); - else strcpy(fbasename, "testfile"); - MPI_Bcast(fbasename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for mput/iput APIs ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); - free(cmd_str); - } + keep_files = 0; - length = 2; + length = 10; array_of_gsizes[0] = array_of_gsizes[1] = array_of_gsizes[2] = length; nvars = 4; buf = (int **)malloc(sizeof(int*)*nvars); if (buf == NULL){ - printf("buf malloc error\n"); + fprintf(stderr,"buf malloc error\n"); nerrs++; goto fn_exit; } bufcounts = (MPI_Offset *)malloc(sizeof(MPI_Offset)*nvars); if (bufcounts == NULL){ - printf("bufcounts malloc error\n"); + fprintf(stderr,"bufcounts malloc error\n"); nerrs++; goto fn_exit; } starts = (MPI_Offset **)malloc(sizeof(MPI_Offset*)*nvars); if (starts== NULL){ - printf("starts malloc error\n"); + fprintf(stderr,"starts malloc error\n"); nerrs++; goto fn_exit; } counts = (MPI_Offset **)malloc(sizeof(MPI_Offset*)*nvars); if (counts == NULL){ - printf("counts malloc error\n"); + fprintf(stderr,"counts malloc error\n"); nerrs++; goto fn_exit; } datatype_list = (MPI_Datatype*)malloc(sizeof(MPI_Datatype)*nvars); if (datatype_list == NULL){ - printf("counts malloc error\n"); + fprintf(stderr,"counts malloc error\n"); nerrs++; goto fn_exit; } @@ -387,12 +384,12 @@ int main(int argc, char **argv) for (i=0; i nprocs) ? nvars : nprocs; varid = (int *)malloc(sizeof(int)*nvars2); if (varid == NULL){ - printf("varid malloc error\n"); + fprintf(stderr,"varid malloc error\n"); nerrs++; goto fn_exit; } - MPI_Info_create(&info); /* MPI_Info_set(info, "romio_pvfs2_posix_write", "enable"); MPI_Info_set(info, "group_cyclic_fd", "enable"); MPI_Info_set(info, "cb_buffer_size", "1024"); MPI_Info_set(info, "cb_buffer_size", "16777216"); - MPI_Info_set(info, "romio_no_indep_rw", "true"); MPI_Info_set(info, "romio_cb_write", "true"); */ for (k=0; k<=9; k++) { - sprintf(filename, "%s.%d.%d.%d.nc", fbasename, length, nvars, k); + sprintf(filename, "%s.%d.%d.%d.nc", out_path, length, nvars, k); if (k==0) strcpy(filename1, filename); else if (k==7) @@ -674,13 +669,19 @@ printf("filename2=%s filename3=%s\n",filename2, filename3); if (rank == 0 && err == NC_NOERR && verbose) printf("\t OK\n"); */ - } else { - if (rank == 0 && verbose) + } else if (rank == 0 && verbose) printf("\t OK\n"); - } } } + if (rank == 0 && !keep_files) { + for (k=0; k<=9; k++) { + sprintf(filename, "%s.%d.%d.%d.nc", out_path, length, nvars, k); + unlink(filename); + } + } + MPI_Barrier(MPI_COMM_WORLD); + /* int nkeys; MPI_Info_get_nkeys(info, &nkeys); @@ -697,8 +698,6 @@ printf("filename2=%s filename3=%s\n",filename2, filename3); } */ - MPI_Info_free(&info); - for (i=0; i 0) { - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - } - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } +fn_exit: + return nerrs; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = false;/* skip ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ + + err = tst_main(argc, argv, "mput/iput APIs", opt, test_io); -fn_exit: MPI_Finalize(); - return (nerrs > 0); + + return err; } diff --git a/test/nonblocking/mcoll_testf.f90 b/test/nonblocking/mcoll_testf.f90 index 0f079e863f..c126781cb0 100644 --- a/test/nonblocking/mcoll_testf.f90 +++ b/test/nonblocking/mcoll_testf.f90 @@ -51,7 +51,8 @@ program Mcoll_Testf ! determined by MPI where a ! zero is specified integer rank, Write_File - character(len=256) :: filename, cmd, msg + character(len=256) :: out_path, in_path, cmd, msg + logical keep_files real*4 filsiz real*4 rdt_l(2) @@ -67,22 +68,29 @@ program Mcoll_Testf ! data TOTSIZ_3D / 256, 256, 256 / data TOTSIZ_3D / 8, 8, 8 / + double precision timing + ! ---------------- ! Begin execution. ! ---------------- - call MPI_Init (ierr) + call MPI_Init(ierr) + + timing = MPI_Wtime() + call MPI_Comm_Size(MPI_COMM_WORLD, totpes, ierr) call MPI_Comm_rank(MPI_COMM_WORLD, rank, ierr) if (rank .EQ. 0) then - filename = 'testfile.nc' - err = get_args(cmd, filename) + out_path = 'testfile.nc' + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD,ierr) call MPI_Dims_Create (totpes, 3, numpes, ierr) @@ -146,7 +154,7 @@ program Mcoll_Testf locsiz = locsiz_3d(1) * locsiz_3d(2) * locsiz_3d(3) ! =============== - ierr = Write_File(filename, NWRITES, comm_cart, & + ierr = Write_File(out_path, NWRITES, comm_cart, & istart, jstart, kstart, locsiz, locsiz_3d, & TOTSIZ_3D, wrt_l) if (ierr .NE. NF90_NOERR) then @@ -180,9 +188,18 @@ program Mcoll_Testf call MPI_Comm_Free (comm_cart, ierr) + timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, & + MPI_DOUBLE_PRECISION, MPI_MAX, & + MPI_COMM_WORLD, ierr) + if (rank .EQ. 0) then - msg='*** TESTING F90 '//trim(cmd)//' for nf90mpi_iput_var API' - call pass_fail(0, msg) + if (.NOT. keep_files) then + err = nf90mpi_delete(out_path, MPI_INFO_NULL) + end if + + msg='*** TESTING F90 '//trim(cmd)//' - nf90mpi_iput_var APIs' + call pass_fail(0, msg, timing) endif 999 call MPI_Finalize(ierr) @@ -192,7 +209,7 @@ end program Mcoll_Testf ! ------------ - integer function Write_File(filename, nwrites, comm_cart, & + integer function Write_File(out_path, nwrites, comm_cart, & istart, jstart, kstart, locsiz, & locsiz_3d, totsiz_3d, wrt_l) @@ -204,7 +221,7 @@ integer function Write_File(filename, nwrites, comm_cart, & ! Argument declarations. ! ---------------------- - character (len=*) filename + character (len=*) out_path integer nwrites integer comm_cart INTEGER(KIND=MPI_OFFSET_KIND) istart, jstart, kstart @@ -261,7 +278,7 @@ integer function Write_File(filename, nwrites, comm_cart, & call MPI_Info_create(info, ierr) ! call MPI_Info_set(info, "romio_pvfs2_posix_write","enable",ierr) - Write_File = nf90mpi_create(comm_cart, filename, NF90_CLOBBER, & + Write_File = nf90mpi_create(comm_cart, out_path, NF90_CLOBBER, & info, ncid) if (Write_File .NE. NF90_NOERR) return diff --git a/test/nonblocking/mcoll_testf77.f b/test/nonblocking/mcoll_testf77.f index b5b7c6abee..3598d3f7df 100644 --- a/test/nonblocking/mcoll_testf77.f +++ b/test/nonblocking/mcoll_testf77.f @@ -69,7 +69,8 @@ program Mcoll_Testf ! zero is specified integer rank, Write_File - character(LEN=256) filename, cmd, msg + character(LEN=256) out_path, in_path, cmd, msg + logical keep_files real*4 filsiz @@ -87,24 +88,31 @@ program Mcoll_Testf ! data TOTSIZ_3D / 256, 256, 256 / data TOTSIZ_3D / 8, 8, 8 / + double precision timing + ! ---------------- ! Begin execution. ! ---------------- - call MPI_Init (ierr) + call MPI_Init(ierr) + + timing = MPI_Wtime() + call MPI_Comm_Size(MPI_COMM_WORLD, totpes, ierr) call MPI_Comm_rank(MPI_COMM_WORLD, rank, ierr) if (rank .EQ. 0) then - filename = "testfile.nc" - err = get_args(cmd, filename) + out_path = "testfile.nc" + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, + ierr) + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD,ierr) + call MPI_Dims_Create (totpes, 3, numpes, ierr) call MPI_Cart_Create(MPI_COMM_WORLD, 3, numpes, isperiodic, @@ -169,7 +177,7 @@ program Mcoll_Testf locsiz = locsiz_3d(1) * locsiz_3d(2) * locsiz_3d(3) ! =============== - ierr = Write_File(filename, NWRITES, comm_cart, + ierr = Write_File(out_path, NWRITES, comm_cart, + istart, jstart, kstart, locsiz, locsiz_3d, + TOTSIZ_3D, wrt_l) if (ierr .NE. NF_NOERR) then @@ -204,9 +212,18 @@ program Mcoll_Testf call MPI_Comm_Free (comm_cart, ierr) + timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, + + MPI_DOUBLE_PRECISION, MPI_MAX, + + MPI_COMM_WORLD, ierr) + if (rank .EQ. 0) then - msg = '*** TESTING F77 '//cmd(1:XTRIM(cmd))//' for iput API' - call pass_fail(0, msg) + if (.NOT. keep_files) then + err = nfmpi_delete(out_path, MPI_INFO_NULL) + end if + + msg = '*** TESTING F77 '//cmd(1:XTRIM(cmd))//' - iput API' + call pass_fail(0, msg, timing) endif 999 call MPI_Finalize (ierr) @@ -217,7 +234,7 @@ program Mcoll_Testf ! ------------ - integer function Write_File(filename, nwrites, comm_cart, + integer function Write_File(out_path, nwrites, comm_cart, + istart, jstart, kstart, locsiz, + locsiz_3d, totsiz_3d, wrt_l) @@ -229,7 +246,7 @@ integer function Write_File(filename, nwrites, comm_cart, ! Argument declarations. ! ---------------------- - character(LEN=*) filename + character(LEN=*) out_path integer nwrites integer comm_cart integer*8 istart, jstart, kstart @@ -291,7 +308,7 @@ integer function Write_File(filename, nwrites, comm_cart, call MPI_Info_create(info, ierr) ! call MPI_Info_set(info, "romio_pvfs2_posix_write", "enable",ierr) - Write_File = nfmpi_create(comm_cart, filename, NF_CLOBBER, + Write_File = nfmpi_create(comm_cart, out_path, NF_CLOBBER, + info, ncid) if (Write_File .NE. NF_NOERR) return diff --git a/test/nonblocking/parallel_run.sh b/test/nonblocking/parallel_run.sh index 25ffe7650b..ad26c7a5b8 100755 --- a/test/nonblocking/parallel_run.sh +++ b/test/nonblocking/parallel_run.sh @@ -1,84 +1,41 @@ #!/bin/bash # -# Copyright (C) 2018, Northwestern University and Argonne National Laboratory +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory # See COPYRIGHT notice in top-level directory. # # Exit immediately if a command exits with a non-zero status. set -e -VALIDATOR=../../src/utils/ncvalidator/ncvalidator -NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff +DRY_RUN=no +VERBOSE=no -# remove file system type prefix if there is any -OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $MPIRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $MPIRUN $@ + fi +} MPIRUN=`echo ${TESTMPIRUN} | ${SED} -e "s/NP/$1/g"` # echo "MPIRUN = ${MPIRUN}" # echo "check_PROGRAMS=${check_PROGRAMS}" -# echo "PNETCDF_DEBUG = ${PNETCDF_DEBUG}" -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" -else - safe_modes="0" -fi - # prevent user environment setting of PNETCDF_HINTS to interfere unset PNETCDF_HINTS -for i in ${check_PROGRAMS} ; do - for j in ${safe_modes} ; do - for intra_aggr in 0 1 ; do - if test "$j" = 1 ; then # test only in safe mode - export PNETCDF_HINTS="romio_no_indep_rw=true" - else - export PNETCDF_HINTS= - fi - if test "$intra_aggr" = 1 ; then - export PNETCDF_HINTS="${PNETCDF_HINTS};nc_num_aggrs_per_node=2" - fi - export PNETCDF_SAFE_MODE=$j - # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - ${MPIRUN} ./$i ${TESTOUTDIR}/$i.nc - - if test "$i" != mcoll_perf ; then - # echo "--- validating file ${TESTOUTDIR}/$i.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.nc - fi - # echo "" - - if test "x${ENABLE_BURST_BUFFER}" = x1 ; then - # echo "---- test burst buffering feature" - saved_PNETCDF_HINTS=${PNETCDF_HINTS} - export PNETCDF_HINTS="${PNETCDF_HINTS};nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" - ${MPIRUN} ./$i ${TESTOUTDIR}/$i.bb.nc - export PNETCDF_HINTS=${saved_PNETCDF_HINTS} +if test "x$MIMIC_LUSTRE" != x1 ; then + PNETCDF_HINTS="cb_nodes=2" +fi - if test "$i" = mcoll_perf ; then - continue - fi +for i in ${check_PROGRAMS} ; do - # echo "--- validating file ${TESTOUTDIR}/$i.bb.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.bb.nc + exe_name=`basename $i` - # burst buffering does not support nonblocking requests in define mode - if test $i != "i_varn_indef" ; then - # echo "--- ncmpidiff $i.nc $i.bb.nc ---" - ${MPIRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/$i.nc ${TESTOUTDIR}/$i.bb.nc - fi - fi + run_cmd ./$i -q -o ${TESTOUTDIR}/${exe_name}.nc - if test "x${ENABLE_NETCDF4}" = x1 ; then - # echo "test netCDF-4 feature" - ${MPIRUN} ./$i ${TESTOUTDIR}/$i.nc4 4 - # Validator does not support nc4 - fi - done - done - rm -f ${OUTDIR}/$i.nc - rm -f ${OUTDIR}/$i.bb.nc - rm -f ${OUTDIR}/$i.nc.* - rm -f ${OUTDIR}/$i.bb.nc.* -done +done # check_PROGRAMS diff --git a/test/nonblocking/req_all.c b/test/nonblocking/req_all.c index bf793f13f3..662b14662a 100644 --- a/test/nonblocking/req_all.c +++ b/test/nonblocking/req_all.c @@ -9,9 +9,9 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * This example shows how to use NC_REQ_ALL in nonblocking I/O operations. * The program writes 2 arrays by calling the nonblocking APIs with NULLs for - * argument request ID. When calling ncmpi_wait_all(), NC_REQ_ALL is used to - * commit all the pending requests without checking the individual statuses of - * the requests. + * argument request ID. When calling ncmpi_wait_all()/ncmpi_wait(), NC_REQ_ALL + * is used to commit all the pending requests without checking the individual + * statuses of the requests. * * To compile: * mpicc -O2 req_all.c -o req_all -lpnetcdf @@ -65,38 +65,28 @@ #define NY 8 #define NX 2 -int main(int argc, char** argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) { - char filename[256]; int i, j, rank, nprocs, nerrs=0, err; - int ncid, cmode, varid[2], dimid[2], buf_int[NY][NX]; + int ncid, varid[2], dimid[2], buf_int[NY][NX]; float buf_flt[NY][NX]; MPI_Offset global_ny, global_nx; MPI_Offset start[2], count[2]; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for NC_REQ_ALL ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); - free(cmd_str); - } + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR /* create a new file for writing ----------------------------------------*/ - cmode = NC_CLOBBER; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR /* the global array is NY * (NX * nprocs) */ global_ny = NY; @@ -121,6 +111,11 @@ int main(int argc, char** argv) /* do not forget to exit define mode */ err = ncmpi_enddef(ncid); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + /* now we are in data mode */ start[0] = 0; start[1] = NX * rank; @@ -130,18 +125,22 @@ int main(int argc, char** argv) err = ncmpi_iput_vara_int(ncid, varid[0], start, count, &buf_int[0][0], NULL); CHECK_ERR err = ncmpi_iput_vara_float(ncid, varid[1], start, count, &buf_flt[0][0], NULL); CHECK_ERR - err = ncmpi_wait_all(ncid, NC_REQ_ALL, NULL, NULL); CHECK_ERR + if (coll_io) + err = ncmpi_wait_all(ncid, NC_REQ_ALL, NULL, NULL); + else + err = ncmpi_wait(ncid, NC_REQ_ALL, NULL, NULL); + CHECK_ERR /* check if write buffer contents have been altered */ for (i=0; i 0) { - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - } - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + return nerrs; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "NC_REQ_ALL", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/nonblocking/seq_runs.sh b/test/nonblocking/seq_runs.sh index aa47560d30..eb3c73faf6 100755 --- a/test/nonblocking/seq_runs.sh +++ b/test/nonblocking/seq_runs.sh @@ -1,47 +1,29 @@ -#!/bin/sh +#!/bin/bash # -# Copyright (C) 2003, Northwestern University and Argonne National Laboratory +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory # See COPYRIGHT notice in top-level directory. # -# "set -x" expands variables and prints a little + sign before the line - # Exit immediately if a command exits with a non-zero status. set -e -VALIDATOR=../../src/utils/ncvalidator/ncvalidator -NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff +DRY_RUN=no +VERBOSE=no + +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $TESTSEQRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $TESTSEQRUN $@ + fi +} -# remove file system type prefix if there is any -OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` +exe_name=`basename $1` # prevent user environment setting of PNETCDF_HINTS to interfere unset PNETCDF_HINTS -${TESTSEQRUN} ./mcoll_perf ${TESTOUTDIR}/testfile -# seq is not available on FreeBSD otherwise we can use: for j in `seq 0 9` -for j in 0 1 2 3 4 5 6 7 8 9 ; do - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/testfile.2.4.$j.nc -done - -# echo "" - -if test "x${ENABLE_BURST_BUFFER}" = x1 ; then - echo "" - echo "---- testing burst buffering" - export PNETCDF_HINTS="nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" - ${TESTSEQRUN} ./mcoll_perf ${TESTOUTDIR}/testfile_bb - unset PNETCDF_HINTS - for j in 0 1 2 3 4 5 6 7 8 9 ; do - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/testfile_bb.2.4.$j.nc - - # echo "--- ncmpidiff testfile.2.4.$j.nc testfile_bb.2.4.$j.nc ---" - ${TESTSEQRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/testfile.2.4.$j.nc ${TESTOUTDIR}/testfile_bb.2.4.$j.nc - done -fi - -for j in 0 1 2 3 4 5 6 7 8 9 ; do - rm -f ${OUTDIR}/testfile.2.4.$j.nc - rm -f ${OUTDIR}/testfile_bb.2.4.$j.nc -done +run_cmd ./$1 -q -o ${TESTOUTDIR}/${exe_name}.nc diff --git a/test/nonblocking/test_bput.c b/test/nonblocking/test_bput.c index 35ec82667e..d14b4beaaa 100644 --- a/test/nonblocking/test_bput.c +++ b/test/nonblocking/test_bput.c @@ -15,67 +15,54 @@ #include -#define FILE_NAME "testfile.nc" - -/*----< main() >------------------------------------------------------------*/ -int main(int argc, char **argv) { - int i, j, ncid, dimid[2], varid, err, nerrs=0, rank, nprocs, bb_enabled; - int req[2], status[2]; +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) +{ + char hint[MPI_MAX_INFO_VAL]; + int i, j, ncid, dimid[2], varid, err, nerrs=0, rank, bb_enabled; + int flag, req[2], status[2]; float var[4][6]; - char filename[256]; MPI_Offset bufsize, start[2], count[2], stride[2], imap[2]; - MPI_Info info; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); #ifdef DEBUG + int nprocs; + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); if (nprocs > 1 && rank == 0) printf("Warning: %s is designed to run on 1 process\n", argv[0]); #endif - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for bput API ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); - free(cmd_str); - } + if (rank) goto err_out; - MPI_Info_create(&info); - /* MPI_Info_set(info, "romio_pvfs2_posix_write","enable"); */ + MPI_Info_get(info, "nc_burst_buf", MPI_MAX_INFO_VAL - 1, hint, &flag); + if (flag && strcasecmp(hint, "enable") == 0) + bb_enabled = 1; + else + bb_enabled = 0; - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER | NC_64BIT_DATA, info, &ncid); CHECK_ERR - MPI_Info_free(&info); + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR - { - int flag; - char hint[MPI_MAX_INFO_VAL]; - MPI_Info infoused; - - ncmpi_inq_file_info(ncid, &infoused); - MPI_Info_get(infoused, "nc_burst_buf", MPI_MAX_INFO_VAL - 1, hint, &flag); - if (flag && strcasecmp(hint, "enable") == 0) - bb_enabled = 1; - else - bb_enabled = 0; - MPI_Info_free(&infoused); - } + err = ncmpi_create(MPI_COMM_SELF, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR /* define a variable of a 6 x 4 integer array in the nc file */ err = ncmpi_def_dim(ncid, "Y", 6, &dimid[0]); CHECK_ERR err = ncmpi_def_dim(ncid, "X", 4, &dimid[1]); CHECK_ERR - err = ncmpi_def_var(ncid, "var", NC_INT64, 2, dimid, &varid); CHECK_ERR + err = ncmpi_def_var(ncid, "var", NC_INT, 2, dimid, &varid); CHECK_ERR err = ncmpi_enddef(ncid); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + /* set the contents of the write buffer var, a 4 x 6 float array 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, @@ -85,7 +72,7 @@ int main(int argc, char **argv) { for (j=0; j<4; j++) for (i=0; i<6; i++) var[j][i] = 50.5 + j*6+i; /* bufsize must be max of data type converted before and after */ - bufsize = 4*6*sizeof(long long); + bufsize = 4*6*sizeof(int); err = ncmpi_buffer_attach(ncid, bufsize); CHECK_ERR /* write var to the NC variable in the matrix transposed way */ @@ -104,9 +91,9 @@ int main(int argc, char **argv) { for (j=0; j<4; j++) for (i=0; i<6; i++) { if (var[j][i] != 50.5 + j*6+i) { - printf("Error at line %d in %s: put buffer[%d][%d]=%f altered, should be %f\n", + fprintf(stderr,"Error at line %d in %s: put buffer[%d][%d]=%f altered, should be %f\n", __LINE__,__FILE__,j,i,var[j][i],50.5+j*6+i); - nerrs++; + assert(0); } } @@ -118,13 +105,17 @@ int main(int argc, char **argv) { for (j=0; j<4; j++) for (i=0; i<6; i++) { if (var[j][i] != 50.5 + j*6+i) { - printf("Error at line %d in %s: put buffer[%d][%d]=%f altered, should be %f\n", + fprintf(stderr,"Error at line %d in %s: put buffer[%d][%d]=%f altered, should be %f\n", __LINE__,__FILE__,j,i,var[j][i],50.5+j*6+i); - nerrs++; + assert(0); } } - err = ncmpi_wait_all(ncid, 2, req, status); CHECK_ERR + if (coll_io) + err = ncmpi_wait_all(ncid, 2, req, status); + else + err = ncmpi_wait(ncid, 2, req, status); + CHECK_ERR /* check each bput status */ for (i=0; i<2; i++) { @@ -149,10 +140,9 @@ int main(int argc, char **argv) { for (i=0; i<6; i++) { if (var[j][i] != 50.5+j*6+i) { /* this error is a pnetcdf internal error, if occurs */ - printf("Error at line %d in %s: put buffer[%d][%d]=%f altered, should be %f\n", + fprintf(stderr,"Error at line %d in %s: put buffer[%d][%d]=%f altered, should be %f\n", __LINE__,__FILE__,j,i,var[j][i],50.5+j*6+i); - nerrs++; - break; + assert(0); } } } @@ -167,25 +157,33 @@ int main(int argc, char **argv) { err = ncmpi_close(ncid); CHECK_ERR - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) { - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - } - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } +err_out: + MPI_Barrier(MPI_COMM_WORLD); - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + return nerrs; +} + +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "bput API", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/nonblocking/test_bputf.f90 b/test/nonblocking/test_bputf.f90 index 56b9eaefa0..6dfccfae4b 100644 --- a/test/nonblocking/test_bputf.f90 +++ b/test/nonblocking/test_bputf.f90 @@ -34,21 +34,28 @@ program main integer(kind=MPI_OFFSET_KIND) bufsize, inq_bufsize integer(kind=MPI_OFFSET_KIND) usage, acc_usage real var(6,4) - character(len=256) :: filename, cmd, msg + character(len=256) :: out_path, in_path, cmd, msg character(len=512) :: hints + logical keep_files + double precision timing + + call MPI_Init(ierr) + + timing = MPI_Wtime() - call MPI_INIT(ierr) call MPI_COMM_RANK(MPI_COMM_WORLD, rank, ierr) call MPI_COMM_SIZE(MPI_COMM_WORLD, nprocs, ierr) if (rank .EQ. 0) then - filename = "testfile.nc" - err = get_args(cmd, filename) + out_path = "testfile.nc" + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD,ierr) verbose = .FALSE. if (nprocs > 1 .AND. rank .EQ. 0 .AND. verbose) then @@ -66,7 +73,7 @@ program main ! call MPI_Info_set(info, "romio_pvfs2_posix_write","enable",ierr) cmode = IOR(NF90_CLOBBER, NF90_64BIT_DATA) - err = nf90mpi_create(MPI_COMM_WORLD, filename, cmode, & + err = nf90mpi_create(MPI_COMM_WORLD, out_path, cmode, & info, ncid) call check(err, 'Error at nf90mpi_create ') @@ -233,9 +240,18 @@ program main err = nf90mpi_close(ncid) call check(err, 'Error at nf90mpi_close ') + timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, & + MPI_DOUBLE_PRECISION, MPI_MAX, & + MPI_COMM_WORLD, ierr) + if (rank .EQ. 0) then - msg = '*** TESTING F90 '//trim(cmd)//' for bput_var' - call pass_fail(no_err, msg) + if (.NOT. keep_files) then + err = nf90mpi_delete(out_path, MPI_INFO_NULL) + end if + + msg = '*** TESTING F90 '//trim(cmd)//' - bput_var' + call pass_fail(no_err, msg, timing) endif 999 CALL MPI_Finalize(ierr) diff --git a/test/nonblocking/test_bputf77.f b/test/nonblocking/test_bputf77.f index 564b56c696..d45f9cfc14 100644 --- a/test/nonblocking/test_bputf77.f +++ b/test/nonblocking/test_bputf77.f @@ -44,24 +44,31 @@ program main integer*8 imap(2) integer*8 dim_size, bufsize, inq_bufsize real var(6,4) - character(LEN=256) filename, cmd, msg + character(LEN=256) out_path, in_path, cmd, msg integer*8 usage, acc_usage character(LEN=512) hints + logical keep_files + double precision timing + + call MPI_Init(ierr) + + timing = MPI_Wtime() - call MPI_INIT(ierr) call MPI_COMM_RANK(MPI_COMM_WORLD, rank, ierr) call MPI_COMM_SIZE(MPI_COMM_WORLD, nprocs, ierr) if (rank .EQ. 0) then - filename = "testfile.nc" - err = get_args(cmd, filename) + out_path = "testfile.nc" + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, + ierr) + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD,ierr) + verbose = .FALSE. if (nprocs .GT. 1 .AND. rank .EQ. 0 .AND. verbose) then print*,'Warning: ',cmd(1:XTRIM(cmd)), @@ -78,7 +85,7 @@ program main ! call MPI_Info_set(info, "romio_pvfs2_posix_write","enable",ierr) cmode = IOR(NF_CLOBBER, NF_64BIT_DATA) - err = nfmpi_create(MPI_COMM_WORLD, filename, cmode, + err = nfmpi_create(MPI_COMM_WORLD, out_path, cmode, + info, ncid) call check(err, 'Error at nfmpi_create ') @@ -251,10 +258,19 @@ program main err = nfmpi_close(ncid) call check(err, 'Error at nfmpi_close ') + timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, + + MPI_DOUBLE_PRECISION, MPI_MAX, + + MPI_COMM_WORLD, ierr) + if (rank .EQ. 0) then + if (.NOT. keep_files) then + err = nfmpi_delete(out_path, MPI_INFO_NULL) + end if + msg = '*** TESTING F77 '//cmd(1:XTRIM(cmd))// - + ' for bput_varm_real API' - call pass_fail(no_err, msg) + + ' - bput_varm_real API' + call pass_fail(no_err, msg, timing) endif 999 CALL MPI_Finalize(ierr) diff --git a/test/nonblocking/wait_after_indep.c b/test/nonblocking/wait_after_indep.c index 5351a3f7ea..adabda7983 100644 --- a/test/nonblocking/wait_after_indep.c +++ b/test/nonblocking/wait_after_indep.c @@ -26,41 +26,31 @@ #define NX 10 #define NDIMS 2 -int main(int argc, char** argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) { - char filename[256]; int i, j, rank, nprocs, err, nerrs=0; int ncid, varid, dimid[2], req, st; MPI_Offset start[2], count[2], stride[2]; - unsigned char buffer[NY][NX]; + signed char buffer[NY][NX]; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for ncmpi_end_indep_data ", basename(argv[0])); - printf("%-66s ------ ",cmd_str); - free(cmd_str); - } - - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER|NC_64BIT_DATA, - MPI_INFO_NULL, &ncid); + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "Y", NC_UNLIMITED, &dimid[0]); CHECK_ERR err = ncmpi_def_dim(ncid, "X", NX*nprocs, &dimid[1]); CHECK_ERR - err = ncmpi_def_var(ncid, "var", NC_UBYTE, NDIMS, dimid, &varid); CHECK_ERR + err = ncmpi_def_var(ncid, "var", NC_BYTE, NDIMS, dimid, &varid); CHECK_ERR err = ncmpi_enddef(ncid); CHECK_ERR for (i=0; i 0) { - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - } - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + return nerrs; +} + +int main(int argc, char **argv) { - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "ncmpi_end_indep_data()", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/nonblocking/wrap_runs.sh b/test/nonblocking/wrap_runs.sh deleted file mode 100755 index 3fdd5e99c3..0000000000 --- a/test/nonblocking/wrap_runs.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2003, Northwestern University and Argonne National Laboratory -# See COPYRIGHT notice in top-level directory. -# - -# Exit immediately if a command exits with a non-zero status. -set -e - -VALIDATOR=../../src/utils/ncvalidator/ncvalidator -NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff - -outfile=`basename $1` - -# remove file system type prefix if there is any -OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` - -# echo "PNETCDF_DEBUG = ${PNETCDF_DEBUG}" -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" -else - safe_modes="0" -fi - -# prevent user environment setting of PNETCDF_HINTS to interfere -unset PNETCDF_HINTS - -for j in ${safe_modes} ; do - export PNETCDF_SAFE_MODE=$j - # echo "---- set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - ${TESTSEQRUN} $1 ${TESTOUTDIR}/$outfile.nc - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$outfile.nc - # echo "" - - if test "x${ENABLE_BURST_BUFFER}" = x1 ; then - echo "" - echo "---- testing burst buffering" - - export PNETCDF_HINTS="nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" - ${TESTSEQRUN} $1 ${TESTOUTDIR}/$outfile.bb.nc - unset PNETCDF_HINTS - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$outfile.bb.nc - - # burst buffering does not support nonblocking requests in define mode - if test $1 != "./i_varn_indef" ; then - # echo "--- ncmpidiff $outfile.nc $outfile.bb.nc ---" - ${TESTSEQRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/$outfile.nc ${TESTOUTDIR}/$outfile.bb.nc - fi - fi -done -rm -f ${OUTDIR}/$outfile.nc -rm -f ${OUTDIR}/$outfile.bb.nc - - diff --git a/test/parallel_run.sh b/test/parallel_run.sh new file mode 100755 index 0000000000..03f9abbc5d --- /dev/null +++ b/test/parallel_run.sh @@ -0,0 +1,327 @@ +#!/bin/bash +# +# Copyright (C) 2018, Northwestern University and Argonne National Laboratory +# See COPYRIGHT notice in top-level directory. +# + +# Exit immediately if a command exits with a non-zero status. +set -e + +DRY_RUN=no +VERBOSE=no + +exe_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + + saved_PNETCDF_HINTS= + cmd=`basename $1` + if test "x$MIMIC_LUSTRE" = x1 && test "x$cmd" = xncmpidiff ; then + # echo "export MIMIC_STRIPE_SIZE=1048576" + export MIMIC_STRIPE_SIZE=1048576 + saved_PNETCDF_HINTS=$PNETCDF_HINTS + unset PNETCDF_HINTS + fi + + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $MPIRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $MPIRUN $@ + fi + + if test "x$MIMIC_LUSTRE" = x1 && test "x$cmd" = xncmpidiff ; then + # echo "unset MIMIC_STRIPE_SIZE" + unset MIMIC_STRIPE_SIZE + export PNETCDF_HINTS=$saved_PNETCDF_HINTS + fi +} + +seq_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $TESTSEQRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $TESTSEQRUN $@ + fi +} + +VALIDATOR=../../src/utils/ncvalidator/ncvalidator +NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff + +MPIRUN=`echo ${TESTMPIRUN} | ${SED} -e "s/NP/$1/g"` +# echo "MPIRUN = ${MPIRUN}" +# echo "check_PROGRAMS=${check_PROGRAMS}" + +# remove file system type prefix if there is any +OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` + +# let NTHREADS=$1*6-1 +NTHREADS=`expr $1 \* 6 - 1` + +PNETCDF_DEBUG_MODE=`grep PNETCDF_DEBUG_MODE ${top_builddir}/src/include/pnetcdf.h | tr -s ' ' | cut -d ' ' -f 3` +# echo "${LINENO}: PNETCDF_DEBUG_MODE = ${PNETCDF_DEBUG_MODE}" +if test "x${PNETCDF_DEBUG_MODE}" = x1 ; then + safe_modes="0 1" +else + safe_modes="0" +fi + +if test "x$ENABLE_GIO" = x0 ; then + IO_MODES="gio mpiio" +else + IO_MODES="mpiio" +fi + +# prevent user environment setting of PNETCDF_HINTS to interfere +unset PNETCDF_HINTS + +fixed_length=23 + +for i in ${check_PROGRAMS} ; do + if test "$i" = pres_temp_4D_rd ; then + # running pres_temp_4D_rd is a part of pres_temp_4D_wr + continue + fi + if test "$i" = tst_io ; then + # this is designed to run 1 process + continue + fi + if test "$i" = tst_version ; then + # this program read only and creates no output file + exe_cmd ./$i + continue + fi + if test "$i" = tst_open_cdf5 ; then + # this program read only and creates no output file + exe_cmd ./$i ${srcdir}/bad_begin.nc5 + continue + fi + if test "$i" = tst_corrupt ; then + # this program read only and creates no output file + exe_cmd ./$i ${srcdir} + continue + fi + + for j in ${safe_modes} ; do + if test "$j" = 1 ; then # test only in safe mode + safe_hint=" SAFE" + else + safe_hint="NOSAFE" + fi + OUT_PREFIX="${TESTOUTDIR}/$i" + + for io_mode in $IO_MODES ; do + if test "x$io_mode" = xmpiio ; then + USEMPIO_HINTS="nc_driver=mpiio" + DRIVER_OUT_FILE="${OUT_PREFIX}.mpio" + driver_hint=" MPIO" + else + USEMPIO_HINTS="nc_driver=gio" + DRIVER_OUT_FILE="${OUT_PREFIX}.gio" + driver_hint="GIO" + fi + for intra_aggr in 0 1 ; do + if test "$intra_aggr" = 1 ; then + INA_HINTS="nc_num_aggrs_per_node=2" + INA_OUT_FILE="${DRIVER_OUT_FILE}.ina" + ina_hint=" INA" + else + INA_HINTS="nc_num_aggrs_per_node=0" + INA_OUT_FILE="${DRIVER_OUT_FILE}" + ina_hint="NOINA" + fi + + OUT_FILE=$INA_OUT_FILE + TEST_OPTS="$safe_hint $driver_hint $ina_hint" + + if [[ "$i" == *"vard"* ]] ; then + if test "x$io_mode" == xmpiio || test "x$intra_aggr" == x1 ; then + # vard APIs have deprecated + continue + fi + fi + + # More rigorous tests using a small moving chunk size + PNETCDF_HINTS="nc_data_move_chunk_size=100" + + if test "x$USEMPIO_HINTS" != x ; then + PNETCDF_HINTS="$USEMPIO_HINTS;$PNETCDF_HINTS" + fi + if test "x$INA_HINTS" != x ; then + PNETCDF_HINTS="$INA_HINTS;$PNETCDF_HINTS" + fi + if test "x$MIMIC_LUSTRE" != x1 ; then + PNETCDF_HINTS="cb_nodes=2;$PNETCDF_HINTS" + fi + + export PNETCDF_HINTS="$PNETCDF_HINTS" + export PNETCDF_SAFE_MODE=$j + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line ${LINENO}: PNETCDF_SAFE_MODE=$PNETCDF_SAFE_MODE PNETCDF_HINTS=$PNETCDF_HINTS" + fi + + if test "$i" = tst_pthread ; then + # each MPI process created 6 threads + exe_cmd ./$i ${OUT_FILE}.nc + for k in `seq 0 ${NTHREADS}` ; do + seq_cmd ${VALIDATOR} -q ${OUT_FILE}.nc.$k + rm -f ${OUTDIR}/tst_pthread.nc.$k + done + continue + elif test "$i" = pres_temp_4D_wr ; then + exe_cmd ./$i ${OUT_FILE}.nc + exe_cmd ./pres_temp_4D_rd ${OUT_FILE}.nc + elif test "$i" = pres_temp_4D_rd ; then + continue + elif test "$i" = test_inq_format ; then + exe_cmd ./$i ${srcdir} + continue + elif test "$i" = "tst_cdl_hdr_parser" ; then + exe_cmd ./$i -q -o ${OUT_FILE}.nc ${srcdir}/cdl_header.txt + continue + elif test "$i" = mcoll_perf ; then + exe_cmd ./$i ${OUT_FILE} + else + exe_cmd ./$i ${OUT_FILE}.nc + fi + + # put_all_kinds and iput_all_kinds output 3 files + if test "$i" = put_all_kinds -o "$i" = iput_all_kinds ; then + for k in 1 2 5 ; do + seq_cmd ${VALIDATOR} -q ${OUT_FILE}.nc$k + done + elif test "$i" = mcoll_perf ; then + for j in `seq 0 9` ; do + ext="2.4.$j.nc" + seq_cmd ${VALIDATOR} -q ${OUT_FILE}.$ext + done + else + seq_cmd ${VALIDATOR} -q ${OUT_FILE}.nc + fi + + if test "x${ENABLE_BURST_BUFFER}" = x1 ; then + # echo "${LINENO}: ---- test burst buffering feature" + saved_PNETCDF_HINTS=${PNETCDF_HINTS} + export PNETCDF_HINTS="${PNETCDF_HINTS};nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" + if test "$i" = mcoll_perf ; then + exe_cmd ./$i ${OUT_FILE}.bb + else + exe_cmd ./$i ${OUT_FILE}.bb.nc + fi + export PNETCDF_HINTS=${saved_PNETCDF_HINTS} + + # put_all_kinds and iput_all_kinds output 3 files + if test "$i" = put_all_kinds -o "$i" = iput_all_kinds ; then + for k in 1 2 5 ; do + seq_cmd ${VALIDATOR} -q ${OUT_FILE}.bb.nc$k + exe_cmd ${NCMPIDIFF} -q ${OUT_FILE}.nc$k ${OUT_FILE}.bb.nc$k + done + continue + elif test "$i" = mcoll_perf ; then + for j in `seq 0 9` ; do + ext="2.4.$j.nc" + bb_ext="bb.2.4.$j.nc" + seq_cmd ${VALIDATOR} -q ${OUT_FILE}.$bb_ext + exe_cmd ${NCMPIDIFF} -q ${OUT_FILE}.$ext ${OUT_FILE}.$bb_ext + done + continue + else + seq_cmd ${VALIDATOR} -q ${OUT_FILE}.bb.nc + fi + + # compare file header only for large file tests + DIFF_OPT="-q" + if test "$i" = last_large_var || + test "$i" = dim_cdf12 || + test "$i" = tst_cdl_hdr_parser || + test "$i" = bigrecords || + test "$i" = high_dim_var || + test "$i" = large_attr || + test "$i" = large_coalesce || + test "$i" = large_dims_vars_attrs || + test "$i" = large_files || + test "$i" = large_header || + test "$i" = large_reqs || + test "$i" = large_var || + test "$i" = tst_cdf5_begin || + test "$i" = tst_flarge || + test "$i" = tst_hash_large_ndims || + test "$i" = tst_hash_large_ngattrs || + test "$i" = tst_hash_large_nvars ; then + DIFF_OPT+=" -h" + fi + exe_cmd ${NCMPIDIFF} $DIFF_OPT $OUT_FILE.nc $OUT_FILE.bb.nc + fi + + if test "x${ENABLE_NETCDF4}" = x1 ; then + if test "$i" = tst_grow_data ; then + continue + fi + exe_cmd ./$i ${OUT_FILE}.nc4 4 + # Validator does not support nc4 + fi + done # intra_aggr + done # io_mode + + if [[ "$i" == *"vard"* ]] ; then + continue + fi + + DIFF_OPT="-q" + if test "$i" = last_large_var || + test "$i" = dim_cdf12 || + test "$i" = tst_cdl_hdr_parser || + test "$i" = bigrecords || + test "$i" = high_dim_var || + test "$i" = large_attr || + test "$i" = large_coalesce || + test "$i" = large_dims_vars_attrs || + test "$i" = large_files || + test "$i" = large_header || + test "$i" = large_reqs || + test "$i" = large_var || + test "$i" = tst_cdf5_begin || + test "$i" = tst_flarge || + test "$i" = tst_hash_large_ndims || + test "$i" = tst_hash_large_ngattrs || + test "$i" = tst_hash_large_nvars ; then + DIFF_OPT+=" -h" + fi + if test "$i" = test_inq_format ; then + continue + fi + if test "$i" = put_all_kinds || test "$i" = iput_all_kinds ; then + for j in 1 2 5; do + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc$j $OUT_PREFIX.mpio.ina.nc$j + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc$j $OUT_PREFIX.gio.nc$j + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.gio.nc$j $OUT_PREFIX.gio.ina.nc$j + done + elif test "$i" = tst_pthread ; then + for j in `seq 0 ${NTHREADS}` ; do + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.mpio.ina.nc.$j + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc.$j $OUT_PREFIX.gio.nc.$j + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.gio.nc.$j $OUT_PREFIX.gio.ina.nc.$j + done + elif test "$i" = mcoll_perf ; then + for j in `seq 0 9` ; do + ext="2.4.$j.nc" + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.$ext $OUT_PREFIX.mpio.ina.$ext + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.$ext $OUT_PREFIX.gio.$ext + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.gio.$ext $OUT_PREFIX.gio.ina.$ext + done + else + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.mpio.ina.nc + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.mpio.nc $OUT_PREFIX.gio.nc + exe_cmd $NCMPIDIFF $DIFF_OPT $OUT_PREFIX.gio.nc $OUT_PREFIX.gio.ina.nc + fi + + done # safe_modes + + if test "x$i" = xpres_temp_4D_wr ; then + rm -f ${OUTDIR}/pres_temp_4D*.nc* + else + rm -f ${OUTDIR}/$i*nc* + fi +done # check_PROGRAMS + diff --git a/test/subfile/Makefile.am b/test/subfile/Makefile.am index ed268755cd..9fc2075628 100644 --- a/test/subfile/Makefile.am +++ b/test/subfile/Makefile.am @@ -14,7 +14,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/src/include AM_CPPFLAGS += -I$(srcdir)/../common AM_CPPFLAGS += -I$(top_builddir)/src/include -LDADD = $(top_builddir)/src/libs/libpnetcdf.la ../common/libtestutils.la +LDADD = ../common/libtestutils.la $(top_builddir)/src/libs/libpnetcdf.la LDADD += @NETCDF4_LDFLAGS@ @ADIOS_LDFLAGS@ @NETCDF4_LIBS@ @ADIOS_LIBS@ if DECL_MPI_OFFSET @@ -25,30 +25,29 @@ if DECL_MPI_OFFSET # AM_FCFLAGS += $(FC_DEFINE)HAVE_DECL_MPI_OFFSET endif -TESTPROGRAMS = test_subfile - -check_PROGRAMS = $(TESTPROGRAMS) +check_PROGRAMS = test_subfile TESTS_ENVIRONMENT = export SED="$(SED)"; TESTS_ENVIRONMENT += export srcdir="$(srcdir)"; +TESTS_ENVIRONMENT += export top_builddir="$(top_builddir)"; TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; TESTS_ENVIRONMENT += export TESTMPIRUN="$(TESTMPIRUN)"; -TESTS_ENVIRONMENT += export PNETCDF_DEBUG="$(PNETCDF_DEBUG)"; -TESTS_ENVIRONMENT += export TESTPROGRAMS="$(TESTPROGRAMS)"; TESTS_ENVIRONMENT += export check_PROGRAMS="$(check_PROGRAMS)"; -TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER="$(ENABLE_BURST_BUFFER)"; -TESTS = seq_runs.sh +TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER=@ENABLE_BURST_BUFFER@; +TESTS_ENVIRONMENT += export ENABLE_THREAD_SAFE=@ENABLE_THREAD_SAFE@; +TESTS_ENVIRONMENT += export MIMIC_LUSTRE=@MIMIC_LUSTRE@; + +TESTS = $(check_PROGRAMS) TEST_EXTENSIONS = .sh +LOG_COMPILER = $(srcdir)/seq_runs.sh +SH_LOG_COMPILER = + +CLEANFILES = $(TESTOUTDIR)/*.subfile_0.nc $(TESTOUTDIR)/*.subfile_1.nc \ + core core.* *.gcda *.gcno *.gcov gmon.out -CLEANFILES = core core.* *.gcda *.gcno *.gcov gmon.out \ - $(TESTOUTDIR)/test_subfile.nc \ - $(TESTOUTDIR)/test_subfile.nc.subfile_0.nc \ - $(TESTOUTDIR)/test_subfile.nc.subfile_1.nc \ - $(TESTOUTDIR)/test_subfile.bb.nc \ - $(TESTOUTDIR)/test_subfile.bb.nc.subfile_0.nc \ - $(TESTOUTDIR)/test_subfile.bb.nc.subfile_1.nc +check_SCRIPTS = seq_runs.sh parallel_run.sh EXTRA_DIST = README.md seq_runs.sh parallel_run.sh diff --git a/test/subfile/parallel_run.sh b/test/subfile/parallel_run.sh index dea9874ade..ad26c7a5b8 100755 --- a/test/subfile/parallel_run.sh +++ b/test/subfile/parallel_run.sh @@ -1,86 +1,41 @@ #!/bin/bash # -# Copyright (C) 2018, Northwestern University and Argonne National Laboratory +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory # See COPYRIGHT notice in top-level directory. # # Exit immediately if a command exits with a non-zero status. set -e -VALIDATOR=../../src/utils/ncvalidator/ncvalidator -NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff +DRY_RUN=no +VERBOSE=no -# remove file system type prefix if there is any -OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $MPIRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $MPIRUN $@ + fi +} MPIRUN=`echo ${TESTMPIRUN} | ${SED} -e "s/NP/$1/g"` # echo "MPIRUN = ${MPIRUN}" # echo "check_PROGRAMS=${check_PROGRAMS}" -# echo "PNETCDF_DEBUG = ${PNETCDF_DEBUG}" -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" -else - safe_modes="0" -fi - # prevent user environment setting of PNETCDF_HINTS to interfere unset PNETCDF_HINTS -for i in ${check_PROGRAMS} ; do - for j in ${safe_modes} ; do - for intra_aggr in 0 1 ; do - if test "$j" = 1 ; then # test only in safe mode - export PNETCDF_HINTS="romio_no_indep_rw=true" - else - export PNETCDF_HINTS= - fi - if test "$intra_aggr" = 1 ; then - export PNETCDF_HINTS="${PNETCDF_HINTS};nc_num_aggrs_per_node=2" - fi - export PNETCDF_SAFE_MODE=$j - # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - ${MPIRUN} ./$i -f ${TESTOUTDIR}/$i.nc -s 2 - - # echo "--- validating file ${TESTOUTDIR}/$i.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.nc - # echo "--- validating file ${TESTOUTDIR}/$i.nc.subfile_0.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.nc.subfile_0.nc - # echo "--- validating file ${TESTOUTDIR}/$i.nc.subfile_1.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.nc.subfile_1.nc - # echo "" +if test "x$MIMIC_LUSTRE" != x1 ; then + PNETCDF_HINTS="cb_nodes=2" +fi - # skip burst buffering test, as it has not supported subfiling yet - continue +for i in ${check_PROGRAMS} ; do - if test "x${ENABLE_BURST_BUFFER}" = x1 ; then - # echo "---- test burst buffering feature" - saved_PNETCDF_HINTS=${PNETCDF_HINTS} - export PNETCDF_HINTS="${PNETCDF_HINTS};nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" - ${MPIRUN} ./$i -f ${TESTOUTDIR}/$i.bb.nc -s 2 - export PNETCDF_HINTS=${saved_PNETCDF_HINTS} + exe_name=`basename $i` - # echo "--- validating file ${TESTOUTDIR}/$i.bb.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.bb.nc - # echo "--- validating file ${TESTOUTDIR}/$i.bb.nc.subfile_0.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.bb.nc.subfile_0.nc - # echo "--- validating file ${TESTOUTDIR}/$i.bb.nc.subfile_1.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.bb.nc.subfile_1.nc + run_cmd ./$i -q -o ${TESTOUTDIR}/${exe_name}.nc - # echo "--- ncmpidiff $i.nc $i.bb.nc ---" - ${MPIRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/$i.nc ${TESTOUTDIR}/$i.bb.nc - # echo "--- ncmpidiff $i.nc.subfile_0.nc $i.bb.nc.subfile_0.nc ---" - ${MPIRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/$i.nc.subfile_0.nc ${TESTOUTDIR}/$i.bb.nc.subfile_0.nc - # echo "--- ncmpidiff $i.nc.subfile_1.nc $i.bb.nc.subfile_1.nc ---" - ${MPIRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/$i.nc.subfile_1.nc ${TESTOUTDIR}/$i.bb.nc.subfile_1.nc - fi - done - done - rm -f ${OUTDIR}/$i.nc - rm -f ${OUTDIR}/$i.nc.subfile_0.nc - rm -f ${OUTDIR}/$i.nc.subfile_1.nc - rm -f ${OUTDIR}/$i.bb.nc - rm -f ${OUTDIR}/$i.bb.nc.subfile_0.nc - rm -f ${OUTDIR}/$i.bb.nc.subfile_1.nc -done +done # check_PROGRAMS diff --git a/test/subfile/seq_runs.sh b/test/subfile/seq_runs.sh index 46a34d3842..eb3c73faf6 100755 --- a/test/subfile/seq_runs.sh +++ b/test/subfile/seq_runs.sh @@ -1,63 +1,29 @@ -#!/bin/sh +#!/bin/bash # -# Copyright (C) 2003, Northwestern University and Argonne National Laboratory +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory # See COPYRIGHT notice in top-level directory. # # Exit immediately if a command exits with a non-zero status. set -e -VALIDATOR=../../src/utils/ncvalidator/ncvalidator -NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff +DRY_RUN=no +VERBOSE=no -# remove file system type prefix if there is any -OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $TESTSEQRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $TESTSEQRUN $@ + fi +} -# echo "PNETCDF_DEBUG = ${PNETCDF_DEBUG}" -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" -else - safe_modes="0" -fi +exe_name=`basename $1` # prevent user environment setting of PNETCDF_HINTS to interfere unset PNETCDF_HINTS -for i in ${TESTPROGRAMS} ; do - for j in ${safe_modes} ; do - export PNETCDF_SAFE_MODE=$j - # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - ${TESTSEQRUN} ./$i -f ${TESTOUTDIR}/$i.nc -s 2 - - # echo "--- validating file ${TESTOUTDIR}/$i.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.nc - # echo "--- validating file ${TESTOUTDIR}/$i.nc.subfile_0.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.nc.subfile_0.nc - # echo "" - - # skip burst buffering test, as it has not supported subfiling yet - continue - - if test "x${ENABLE_BURST_BUFFER}" = x1 ; then - echo "" - echo "---- testing burst buffering" - - export PNETCDF_HINTS="nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" - ${TESTSEQRUN} ./$i -f ${TESTOUTDIR}/$i.bb.nc -s 2 - unset PNETCDF_HINTS - - # echo "--- validating file ${TESTOUTDIR}/$i.bb.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.bb.nc - # echo "--- validating file ${TESTOUTDIR}/$i.bb.nc.subfile_0.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.bb.nc.subfile_0.nc - - # echo "--- ncmpidiff $i.nc.subfile_0.nc $i.bb.nc.subfile_0.nc ---" - ${MPIRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/$i.nc.subfile_0.nc ${TESTOUTDIR}/$i.bb.nc.subfile_0.nc - fi - done - rm -f ${OUTDIR}/$i.nc - rm -f ${OUTDIR}/$i.nc.subfile_0.nc - rm -f ${OUTDIR}/$i.bb.nc - rm -f ${OUTDIR}/$i.bb.nc.subfile_0.nc -done +run_cmd ./$1 -q -o ${TESTOUTDIR}/${exe_name}.nc diff --git a/test/subfile/test_subfile.c b/test/subfile/test_subfile.c index 356be8df94..221e0847ac 100644 --- a/test/subfile/test_subfile.c +++ b/test/subfile/test_subfile.c @@ -25,130 +25,73 @@ Array size 128^3. For other array sizes, change array_of_gsizes below. */ -int main(int argc, char **argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) { - int opt, verbose=0; extern char *optarg; extern int optind; - int i, j, array_of_gsizes[3]; - int nprocs, len, **buf, rank; - MPI_Offset bufcount; - int array_of_psizes[3]; - int err; - MPI_Offset array_of_starts[3]; - char *fbasename=NULL; char dimname[20], varname[20]; - int ncid, dimids0[3], rank_dim[3], *varid=NULL; - MPI_Info info=MPI_INFO_NULL, info_used=MPI_INFO_NULL; - MPI_Offset **starts_list, **count_list; + int i, j, err, nerrs=0, verbose=0, nprocs, rank, ncid, *varid=NULL; + int ndims=3, ngatts, unlimdimid, dimids0[3], rank_dim[3]; + int num_files, **buf, array_of_psizes[3], array_of_gsizes[3]; + MPI_Offset bufcount, array_of_starts[3], **starts_list, **count_list; MPI_Offset *bufcount_list; - int ndims=3, nvars=1, ngatts, unlimdimid; MPI_Datatype *datatype_list; - int length = 8; - double stim, write_tim, new_write_tim, write_bw; - double read_tim, new_read_tim, read_bw; - double open_tim, new_open_tim; + MPI_Info info_used=MPI_INFO_NULL; + double stim, write_tim, new_write_tim, write_bw, read_bw; + double open_tim, new_open_tim, read_tim, new_read_tim; double close_tim, new_close_tim; + int num_sf = 2; int par_dim_id = 0; /* default is 0 */ int do_read = 0; - int nerrs=0; + int nvars = 1; + int length = 8; - MPI_Init(&argc,&argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - /* process 0 takes the file name as a command-line argument and - broadcasts it to other processes */ - if (rank == 0) { - while ((opt = getopt(argc, argv, "f:s:p:n:l:r")) != EOF) { - switch (opt) { - case 'f': fbasename = optarg; - break; - case 's': num_sf = (int)strtol(optarg,NULL,10); - break; - case 'r': do_read = 1; - break; - case 'p': par_dim_id = (int)strtol(optarg,NULL,10); - break; - case 'n': nvars = (int)strtol(optarg,NULL,10); - break; - case 'l': length = (int)strtol(optarg,NULL,10); - break; - default: - break; - } - } - if (fbasename == NULL) { - fprintf(stderr, "\n*# Usage: test_subfile -f pathname -s num_sf -p par_dim_id \n\n"); - nerrs++; - } - } - MPI_Bcast(&nerrs, 1, MPI_INT, 0, MPI_COMM_WORLD); - if (nerrs > 0) { - MPI_Finalize(); - return 1; - } - - if (rank == 0) { - len = (fbasename == NULL) ? 0 : (int)strlen(fbasename); - MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD); - } - else { - MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD); - fbasename = (char *) malloc(len+1); - } - MPI_Bcast(fbasename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD); - MPI_Bcast(&num_sf, 1, MPI_INT, 0, MPI_COMM_WORLD); - MPI_Bcast(&par_dim_id, 1, MPI_INT, 0, MPI_COMM_WORLD); - MPI_Bcast(&nvars, 1, MPI_INT, 0, MPI_COMM_WORLD); - MPI_Bcast(&do_read, 1, MPI_INT, 0, MPI_COMM_WORLD); - MPI_Bcast(&length, 1, MPI_INT, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for subfiling", basename(argv[0])); - printf("%-66s ------ ", cmd_str); - free(cmd_str); - } - array_of_gsizes[0] = array_of_gsizes[1] = array_of_gsizes[2] = length; buf = (int **)malloc(sizeof(int*) * nvars); if (buf == NULL){ - printf("buf malloc error\n"); + fprintf(stderr,"buf malloc error\n"); nerrs++; goto fn_exit; } bufcount_list = (MPI_Offset *)malloc(sizeof(MPI_Offset)*nvars); if (bufcount_list == NULL){ - printf("bufcount_list malloc error\n"); + fprintf(stderr,"bufcount_list malloc error\n"); nerrs++; goto fn_exit; } starts_list = (MPI_Offset **)malloc(sizeof(MPI_Offset *)*nvars); if (starts_list== NULL){ - printf("starts_list malloc error\n"); + fprintf(stderr,"starts_list malloc error\n"); nerrs++; goto fn_exit; } count_list = (MPI_Offset **)malloc(sizeof(MPI_Offset *)*nvars); if (count_list == NULL){ - printf("count_list malloc error\n"); + fprintf(stderr,"count_list malloc error\n"); nerrs++; goto fn_exit; } datatype_list = (MPI_Datatype*)malloc(sizeof(MPI_Datatype)*nvars); if (datatype_list == NULL){ - printf("count_list malloc error\n"); + fprintf(stderr,"count_list malloc error\n"); nerrs++; goto fn_exit; } for (i=0; i 0) free(fbasename); - MPI_Offset malloc_size, sum_size; int nfiles, ncids[10]; /* NULL argument test */ @@ -397,22 +340,46 @@ int main(int argc, char **argv) CHECK_ERR if (nfiles > 0) printf("nfiles %d still opened\n",nfiles); - /* check for any PnetCDF internal malloc residues */ - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - } + /* open the subfiles to validate the file format */ + MPI_Barrier(MPI_COMM_WORLD); + if (rank > 0) goto fn_exit; - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); + num_files = (nprocs < num_sf) ? 1 : num_sf; + for (i=0; i 0); + + return err; } diff --git a/test/test_installed/README.md b/test/test_installed/README.md index 62987cfd76..37b57dd17e 100644 --- a/test/test_installed/README.md +++ b/test/test_installed/README.md @@ -109,12 +109,6 @@ distribution. All test programs are designed to run on 4 MPI processes. cc -L/opt/cray/pe/parallel-netcdf/1.12.3.1/lib test_fillvalue.o testutils.o -lpnetcdf -o test_fillvalue cc -I/opt/cray/pe/parallel-netcdf/1.12.3.1/include -I../common -c ../testcases/test_get_varn.c cc -L/opt/cray/pe/parallel-netcdf/1.12.3.1/lib test_get_varn.o testutils.o -lpnetcdf -o test_get_varn - cc -I/opt/cray/pe/parallel-netcdf/1.12.3.1/include -I../common -c ../testcases/test_vard.c - cc -L/opt/cray/pe/parallel-netcdf/1.12.3.1/lib test_vard.o testutils.o -lpnetcdf -o test_vard - cc -I/opt/cray/pe/parallel-netcdf/1.12.3.1/include -I../common -c ../testcases/test_vard_multiple.c - cc -L/opt/cray/pe/parallel-netcdf/1.12.3.1/lib test_vard_multiple.o testutils.o -lpnetcdf -o test_vard_multiple - cc -I/opt/cray/pe/parallel-netcdf/1.12.3.1/include -I../common -c ../testcases/test_vard_rec.c - cc -L/opt/cray/pe/parallel-netcdf/1.12.3.1/lib test_vard_rec.o testutils.o -lpnetcdf -o test_vard_rec cc -I/opt/cray/pe/parallel-netcdf/1.12.3.1/include -I../common -c ../testcases/test_varm.c cc -L/opt/cray/pe/parallel-netcdf/1.12.3.1/lib test_varm.o testutils.o -lpnetcdf -o test_varm cc -I/opt/cray/pe/parallel-netcdf/1.12.3.1/include -I../common -c ../testcases/tst_def_var_fill.c @@ -182,10 +176,6 @@ distribution. All test programs are designed to run on 4 MPI processes. cc -L/opt/cray/pe/parallel-netcdf/1.12.3.1/lib transpose2D.o testutils.o -lpnetcdf -o transpose2D cc -I/opt/cray/pe/parallel-netcdf/1.12.3.1/include -I../common -c ../../examples/C/transpose.c cc -L/opt/cray/pe/parallel-netcdf/1.12.3.1/lib transpose.o testutils.o -lpnetcdf -o transpose - cc -I/opt/cray/pe/parallel-netcdf/1.12.3.1/include -I../common -c ../../examples/C/vard_int.c - cc -L/opt/cray/pe/parallel-netcdf/1.12.3.1/lib vard_int.o testutils.o -lpnetcdf -o vard_int - cc -I/opt/cray/pe/parallel-netcdf/1.12.3.1/include -I../common -c ../../examples/C/vard_mvars.c - cc -L/opt/cray/pe/parallel-netcdf/1.12.3.1/lib vard_mvars.o testutils.o -lpnetcdf -o vard_mvars ftn -I/opt/cray/pe/parallel-netcdf/1.12.3.1/include -I../common -w -fallow-argument-mismatch -c ../../examples/F77/block_cyclic.f -o block_cyclic.77o ftn -I/opt/cray/pe/parallel-netcdf/1.12.3.1/include -I../common -w -fallow-argument-mismatch -o block_cyclic.exe77 block_cyclic.77o utils.o -L/opt/cray/pe/parallel-netcdf/1.12.3.1/lib -lpnetcdf ftn -I/opt/cray/pe/parallel-netcdf/1.12.3.1/include -I../common -w -fallow-argument-mismatch -c ../../examples/F77/bput_varn_int8.f -o bput_varn_int8.77o @@ -214,8 +204,6 @@ distribution. All test programs are designed to run on 4 MPI processes. ftn -I/opt/cray/pe/parallel-netcdf/1.12.3.1/include -I../common -w -fallow-argument-mismatch -o time_var.exe77 time_var.77o utils.o -L/opt/cray/pe/parallel-netcdf/1.12.3.1/lib -lpnetcdf ftn -I/opt/cray/pe/parallel-netcdf/1.12.3.1/include -I../common -w -fallow-argument-mismatch -c ../../examples/F77/transpose.f -o transpose.77o ftn -I/opt/cray/pe/parallel-netcdf/1.12.3.1/include -I../common -w -fallow-argument-mismatch -o transpose.exe77 transpose.77o utils.o -L/opt/cray/pe/parallel-netcdf/1.12.3.1/lib -lpnetcdf - ftn -I/opt/cray/pe/parallel-netcdf/1.12.3.1/include -I../common -w -fallow-argument-mismatch -c ../../examples/F77/vard_int.f -o vard_int.77o - ftn -I/opt/cray/pe/parallel-netcdf/1.12.3.1/include -I../common -w -fallow-argument-mismatch -o vard_int.exe77 vard_int.77o utils.o -L/opt/cray/pe/parallel-netcdf/1.12.3.1/lib -lpnetcdf ftn -I/opt/cray/pe/parallel-netcdf/1.12.3.1/include -I../common -w -fallow-argument-mismatch -c ../../examples/F90/block_cyclic.f90 -o block_cyclic.90o ftn -I/opt/cray/pe/parallel-netcdf/1.12.3.1/include -I../common -w -fallow-argument-mismatch -o block_cyclic.exe90 block_cyclic.90o utils.o -L/opt/cray/pe/parallel-netcdf/1.12.3.1/lib -lpnetcdf ftn -I/opt/cray/pe/parallel-netcdf/1.12.3.1/include -I../common -w -fallow-argument-mismatch -c ../../examples/F90/column_wise.f90 -o column_wise.90o @@ -238,8 +226,6 @@ distribution. All test programs are designed to run on 4 MPI processes. ftn -I/opt/cray/pe/parallel-netcdf/1.12.3.1/include -I../common -w -fallow-argument-mismatch -o put_varn_real.exe90 put_varn_real.90o utils.o -L/opt/cray/pe/parallel-netcdf/1.12.3.1/lib -lpnetcdf ftn -I/opt/cray/pe/parallel-netcdf/1.12.3.1/include -I../common -w -fallow-argument-mismatch -c ../../examples/F90/transpose.f90 -o transpose.90o ftn -I/opt/cray/pe/parallel-netcdf/1.12.3.1/include -I../common -w -fallow-argument-mismatch -o transpose.exe90 transpose.90o utils.o -L/opt/cray/pe/parallel-netcdf/1.12.3.1/lib -lpnetcdf - ftn -I/opt/cray/pe/parallel-netcdf/1.12.3.1/include -I../common -w -fallow-argument-mismatch -c ../../examples/F90/vard_int.f90 -o vard_int.90o - ftn -I/opt/cray/pe/parallel-netcdf/1.12.3.1/include -I../common -w -fallow-argument-mismatch -o vard_int.exe90 vard_int.90o utils.o -L/opt/cray/pe/parallel-netcdf/1.12.3.1/lib -lpnetcdf ``` @@ -298,9 +284,6 @@ distribution. All test programs are designed to run on 4 MPI processes. *** TESTING C test_erange for checking for NC_ERANGE ------ pass *** TESTING C test_fillvalue for _FillValue for NC_GLOBAL ------ pass *** TESTING C test_get_varn for get_varn ------ pass - *** TESTING C test_vard for vard put and get ------ pass - *** TESTING C test_vard_multiple for vard to 2 variables ------ pass - *** TESTING C test_vard_rec for vard put on record var ------ pass *** TESTING C test_varm for get/put varm ------ pass *** TESTING C tst_def_var_fill for def_var_fill ------ pass *** TESTING C tst_dimsizes for defining max dimension sizes ------ pass @@ -336,8 +319,6 @@ distribution. All test programs are designed to run on 4 MPI processes. *** TESTING C examples/C/time_var ------ pass *** TESTING C examples/C/transpose2D ------ pass *** TESTING C examples/C/transpose ------ pass - *** TESTING C examples/C/vard_int ------ pass - *** TESTING C examples/C/vard_mvars ------ pass *** TESTING F77 examples/F77/block_cyclic.exe77 ------ pass *** TESTING F77 examples/F77/bput_varn_int8.exe77 ------ pass *** TESTING F77 examples/F77/column_wise.exe77 ------ pass @@ -352,7 +333,6 @@ distribution. All test programs are designed to run on 4 MPI processes. *** TESTING F77 examples/F77/put_varn_real.exe77 ------ pass *** TESTING F77 examples/F77/time_var.exe77 ------ pass *** TESTING F77 examples/F77/transpose.exe77 ------ pass - *** TESTING F77 examples/F77/vard_int.exe77 ------ pass *** TESTING F90 examples/F90/block_cyclic.exe90 ------ pass *** TESTING F90 examples/F90/column_wise.exe90 ------ pass *** TESTING F90 examples/F90/fill_mode.exe90 ------ pass @@ -364,7 +344,6 @@ distribution. All test programs are designed to run on 4 MPI processes. *** TESTING F90 examples/F90/put_varn_int.exe90 ------ pass *** TESTING F90 examples/F90/put_varn_real.exe90 ------ pass *** TESTING F90 examples/F90/transpose.exe90 ------ pass - *** TESTING F90 examples/F90/vard_int.exe90 ------ pass Total number of tested programs: 105 diff --git a/test/test_installed/batch.sh.in b/test/test_installed/batch.sh.in index 16bba66c77..afddbf1ff8 100755 --- a/test/test_installed/batch.sh.in +++ b/test/test_installed/batch.sh.in @@ -14,23 +14,31 @@ cd $PWD # exit immediately when any of the runs failed set -e +if test "x$SLURM_JOB_QOS" = x ; then + DRY_RUN=1 +else + DRY_RUN=0 +fi + export LD_LIBRARY_PATH=__PNETCDF_DIR__/lib:$LD_LIBRARY_PATH OUT_DIR=__OUT_DIR__ +export TESTOUTDIR=$OUT_DIR + NP=4 num_pass=0 for exec in __TEST_PROGS__ ; do - OUTFILE="$OUT_DIR/$exec.nc" - CMD_OPTS= - if test "x$exec" = xpres_temp_4D_rd ; then - OUTFILE="$OUT_DIR/pres_temp_4D_wr.nc" - elif test "x$exec" = xtst_cdl_hdr_parser ; then - CMD_OPTS="-q ../cdl/cdl_header.txt -o" + CMD="__RUN_CMD__ -n $NP $exec -q -o $OUT_DIR/$exec.nc" + if test "x$exec" = "xtst_cdl_hdr_parser" ; then + CMD="$CMD -i ../cdl/cdl_header.txt" + fi + + echo "CMD=$CMD" + if test $DRY_RUN = 0 ; then + eval "$CMD" fi - CMD="__RUN_CMD__ -n $NP $exec $CMD_OPTS $OUTFILE" - eval "$CMD" num_pass=$(expr $num_pass + 1) done @@ -43,7 +51,10 @@ for exec in __EXAMPLE_PROGS__ ; do CMD_OPTS="../../examples/C/cdl_header.txt -o" fi CMD="__RUN_CMD__ -n $NP $exec -q $CMD_OPTS $OUTFILE" - eval "$CMD" + echo "CMD=$CMD" + if test $DRY_RUN = 0 ; then + eval "$CMD" + fi extension="${exec##*.}" if test "x$extension" = xexe90 ; then printf '*** TESTING F90 %-50s ------ pass\n' "examples/F90/$exec" diff --git a/test/test_installed/makefile b/test/test_installed/makefile index b643b4925f..f83e5d1d9b 100644 --- a/test/test_installed/makefile +++ b/test/test_installed/makefile @@ -1,16 +1,16 @@ .SUFFIXES: .o .exe77 .exe90 .77o .90o .testcases77o .PRECIOUS: %.o %.77o %.90o %.testcases77o -CPPFLAGS = -I$(PNETCDF_DIR)/include -I../common +PNETCDF_SRC = ../.. +CPPFLAGS = -I$(PNETCDF_DIR)/include -I../common -I$(PNETCDF_SRC)/src/utils/ncmpidiff CFLAGS = $(CPPFLAGS) FCFLAGS = -w -fallow-argument-mismatch LDFLAGS = -L$(PNETCDF_DIR)/lib -LDLIBS = testutils.o -lpnetcdf +LDLIBS = ncmpidiff_core.o testutils.o -lpnetcdf TEST_PROGS = -C_src = ../C/pres_temp_4D_wr.c \ - ../C/pres_temp_4D_rd.c +C_src = ../C/pres_temp_4D_wr_rd.c TEST_PROGS += $(C_src:../C/%.c=%) %.o: ../C/%.c $(CC) $(CFLAGS) -c $< @@ -61,9 +61,6 @@ testcases_src = ../testcases/add_var.c \ ../testcases/test_erange.c \ ../testcases/test_fillvalue.c \ ../testcases/test_get_varn.c \ - ../testcases/test_vard.c \ - ../testcases/test_vard_multiple.c \ - ../testcases/test_vard_rec.c \ ../testcases/test_varm.c \ ../testcases/tst_def_var_fill.c \ ../testcases/tst_del_attr.c \ @@ -109,10 +106,7 @@ examples_C_src = ../../examples/C/block_cyclic.c \ ../../examples/C/put_varn_int.c \ ../../examples/C/time_var.c \ ../../examples/C/transpose2D.c \ - ../../examples/C/transpose.c \ - ../../examples/C/vard_bottom.c \ - ../../examples/C/vard_int.c \ - ../../examples/C/vard_mvars.c + ../../examples/C/transpose.c EXAMPLE_PROGS += $(examples_C_src:../../examples/C/%.c=%) %.o: ../../examples/C/%.c $(CC) $(CFLAGS) -c $< @@ -146,8 +140,7 @@ examples_F77_src = ../../examples/F77/block_cyclic.f \ ../../examples/F77/put_varn_int.f \ ../../examples/F77/put_varn_real.f \ ../../examples/F77/time_var.f \ - ../../examples/F77/transpose.f \ - ../../examples/F77/vard_int.f + ../../examples/F77/transpose.f EXAMPLE_PROGS += $(examples_F77_src:../../examples/F77/%.f=%.exe77) %.77o: ../../examples/F77/%.f @@ -163,11 +156,10 @@ examples_F90_src = ../../examples/F90/block_cyclic.f90 \ ../../examples/F90/put_var.f90 \ ../../examples/F90/put_varn_int.f90 \ ../../examples/F90/put_varn_real.f90 \ - ../../examples/F90/transpose.f90 \ - ../../examples/F90/vard_int.f90 + ../../examples/F90/transpose.f90 EXAMPLE_PROGS += $(examples_F90_src:../../examples/F90/%.f90=%.exe90) -all: env_check testutils.o utils.o $(TEST_PROGS) $(EXAMPLE_PROGS) batch.sh interactive.sh +all: env_check ncmpidiff_core.o testutils.o utils.o $(TEST_PROGS) $(EXAMPLE_PROGS) batch.sh interactive.sh f90tst_parallel.exe90: ../F90/f90tst_parallel.f90 testutilsf.o $(FC) $(CPPFLAGS) $(FCFLAGS) -o $@ $< testutilsf.o $(LDFLAGS) -lpnetcdf @@ -217,6 +209,9 @@ env_check: testutils.o: ../common/testutils.c $(CC) $(CFLAGS) -I../common -c $< -o $@ +ncmpidiff_core.o: ../../src/utils/ncmpidiff/ncmpidiff_core.c + $(CC) $(CFLAGS) -I../../src/utils/ncmpidiff -c $< -o $@ + testutilsf.o: ../common/testutilsf.F90 $(FC) $(CPPFLAGS) $(FCFLAGS) -c $< -o $@ diff --git a/test/testcases/Makefile.am b/test/testcases/Makefile.am index 5c8a8a25c0..0f3b7fd8a6 100644 --- a/test/testcases/Makefile.am +++ b/test/testcases/Makefile.am @@ -19,7 +19,7 @@ AM_FCFLAGS = $(FC_MODINC)$(top_builddir)/src/binding/f90 \ $(FC_MODINC)$(srcdir)/../common $(FFREEFORMFLAG) AM_FFLAGS = -I$(top_builddir)/src/binding/f77 $(FFIXEDFORMFLAG) -LDADD = ${top_builddir}/src/libs/libpnetcdf.la ../common/libtestutils.la +LDADD = ../common/libtestutils.la ${top_builddir}/src/libs/libpnetcdf.la LDADD += @NETCDF4_LDFLAGS@ @ADIOS_LDFLAGS@ @NETCDF4_LIBS@ @ADIOS_LIBS@ -lm if SIZEOF_MPI_AINT_IS_4 @@ -54,51 +54,57 @@ if NAGFORT AM_FCFLAGS += -w=uparam endif -TESTPROGRAMS = file_create_open \ - ncmpi_vars_null_stride \ - vectors \ - collective_error \ - test_varm \ - alignment_test \ - flexible \ - flexible2 \ - flexible_varm \ - nonblocking \ - noclobber \ - record \ - inq_num_vars \ - varn_int \ - modes \ - one_record \ - inq_recsize \ - test_vard \ - test_vard_rec \ - test_vard_multiple \ - varn_contig \ - ivarn \ - check_striping \ - add_var \ - buftype_free \ - last_large_var \ - check_type \ - test_erange \ - scalar \ - null_args \ - tst_dimsizes \ - mix_collectives \ - large_var_cdf5 \ - tst_max_var_dims \ - tst_info \ - tst_vars_fill \ - tst_def_var_fill \ - test_fillvalue \ - error_precedence \ - tst_free_comm \ - flexible_var \ - test_get_varn \ - tst_del_attr \ - tst_redefine \ - tst_grow_header +check_PROGRAMS = file_create_open \ + ncmpi_vars_null_stride \ + vectors \ + collective_error \ + test_varm \ + alignment_test \ + flexible \ + flexible2 \ + flexible_varm \ + nonblocking \ + noclobber \ + record \ + inq_num_vars \ + varn_int \ + modes \ + one_record \ + inq_recsize \ + varn_contig \ + ivarn \ + check_striping \ + add_var \ + buftype_free \ + last_large_var \ + check_type \ + test_erange \ + scalar \ + null_args \ + tst_dimsizes \ + mix_collectives \ + large_var_cdf5 \ + tst_max_var_dims \ + tst_info \ + tst_vars_fill \ + tst_def_var_fill \ + test_fillvalue \ + error_precedence \ + tst_free_comm \ + flexible_var \ + test_get_varn \ + tst_del_attr \ + tst_redefine \ + tst_grow_header \ + tst_varn_var1 \ + tst_multi_redefine \ + tst_grow_data \ + tst_inq_header_size \ + tst_data_move \ + put_all_kinds \ + redef1 \ + iput_all_kinds \ + tst_version M4_SRCS = put_all_kinds.m4 \ erange_fill.m4 \ @@ -114,17 +120,21 @@ nodist_iput_all_kinds_SOURCES = iput_all_kinds.c nodist_error_precedence_SOURCES = error_precedence.c nodist_null_args_SOURCES = null_args.c +if ENABLE_THREAD_SAFE + check_PROGRAMS += tst_pthread +endif + if TEST_SYMLINK - TESTPROGRAMS += tst_symlink + check_PROGRAMS += tst_symlink endif if TEST_LARGE_COUNT - TESTPROGRAMS += flexible_large_count + check_PROGRAMS += flexible_large_count endif if ENABLE_ERANGE_FILL M4FLAGS += -DERANGE_FILL - TESTPROGRAMS += erange_fill + check_PROGRAMS += erange_fill endif M4FLAGS += -I${top_srcdir}/m4 @@ -138,89 +148,56 @@ $(M4_SRCS:.m4=.c): Makefile set -e; cd ../common && $(MAKE) $(MFLAGS) tests if HAS_FORTRAN - TESTPROGRAMS += varn_intf \ + check_PROGRAMS += varn_intf \ attrf \ buftype_freef \ put_parameter \ - test_vardf \ flexible_api varn_intf_SOURCES = varn_intf.f attrf_SOURCES = attrf.f buftype_freef_SOURCES = buftype_freef.f put_parameter_SOURCES = put_parameter.f - test_vardf_SOURCES = test_vardf.F flexible_api_SOURCES = flexible_api.f if HAVE_MPI_MOD - TESTPROGRAMS += inq_num_varsf \ - inq_recsizef \ - test_vardf90 \ - varn_real + check_PROGRAMS += inq_num_varsf \ + inq_recsizef \ + varn_real inq_num_varsf_SOURCES = inq_num_varsf.f90 inq_recsizef_SOURCES = inq_recsizef.f90 - test_vardf90_SOURCES = test_vardf90.f90 varn_real_SOURCES = varn_real.f90 endif endif -# all programs in TESTPROGRAMS will be run by wrap_runs.sh -# others in check_PROGRAMS but not in TESTPROGRAMS will be run by seq_runs.sh -# Those are the ones need special treatment -check_PROGRAMS = $(TESTPROGRAMS) \ - put_all_kinds \ - redef1 \ - iput_all_kinds \ - tst_version - -# autimake 1.11.3 has not yet implemented AM_TESTS_ENVIRONMENT +# automake 1.11.3 has not yet implemented AM_TESTS_ENVIRONMENT # For newer versions, we can use AM_TESTS_ENVIRONMENT instead -# AM_TESTS_ENVIRONMENT = TESTPROGRAMS="$(TESTPROGRAMS)" ; export TESTPROGRAMS; +# AM_TESTS_ENVIRONMENT = check_PROGRAMS="$(check_PROGRAMS)" ; export check_PROGRAMS; # AM_TESTS_ENVIRONMENT += TESTSEQRUN="$(TESTSEQRUN)" ; export TESTSEQRUN; # AM_TESTS_ENVIRONMENT += TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)" ; export TESTOUTDIR; TESTS_ENVIRONMENT = export SED="$(SED)"; TESTS_ENVIRONMENT += export srcdir="$(srcdir)"; +TESTS_ENVIRONMENT += export top_builddir="$(top_builddir)"; TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; TESTS_ENVIRONMENT += export TESTMPIRUN="$(TESTMPIRUN)"; -TESTS_ENVIRONMENT += export PNETCDF_DEBUG="$(PNETCDF_DEBUG)"; -TESTS_ENVIRONMENT += export TESTPROGRAMS="$(TESTPROGRAMS)"; TESTS_ENVIRONMENT += export check_PROGRAMS="$(check_PROGRAMS)"; -TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER="$(ENABLE_BURST_BUFFER)"; -TESTS_ENVIRONMENT += export ENABLE_THREAD_SAFE="$(ENABLE_THREAD_SAFE)"; -TESTS = $(TESTPROGRAMS) seq_runs.sh +TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER=@ENABLE_BURST_BUFFER@; +TESTS_ENVIRONMENT += export ENABLE_THREAD_SAFE=@ENABLE_THREAD_SAFE@; +TESTS_ENVIRONMENT += export MIMIC_LUSTRE=@MIMIC_LUSTRE@; + +TESTS = $(check_PROGRAMS) TEST_EXTENSIONS = .sh -LOG_COMPILER = $(srcdir)/wrap_runs.sh +LOG_COMPILER = $(srcdir)/seq_runs.sh SH_LOG_COMPILER = -NC_FILES = $(check_PROGRAMS:%=$(TESTOUTDIR)/%.nc) \ - $(check_PROGRAMS:%=$(TESTOUTDIR)/%.nc1) \ - $(check_PROGRAMS:%=$(TESTOUTDIR)/%.nc2) \ - $(check_PROGRAMS:%=$(TESTOUTDIR)/%.nc3) \ - $(check_PROGRAMS:%=$(TESTOUTDIR)/%.nc4) \ - $(check_PROGRAMS:%=$(TESTOUTDIR)/%.nc5) \ - $(check_PROGRAMS:%=$(TESTOUTDIR)/%.bb.nc) \ - $(check_PROGRAMS:%=$(TESTOUTDIR)/%.bb.nc1) \ - $(check_PROGRAMS:%=$(TESTOUTDIR)/%.bb.nc2) \ - $(check_PROGRAMS:%=$(TESTOUTDIR)/%.bb.nc3) \ - $(check_PROGRAMS:%=$(TESTOUTDIR)/%.bb.nc4) \ - $(check_PROGRAMS:%=$(TESTOUTDIR)/%.bb.nc5) - -if ENABLE_THREAD_SAFE - check_PROGRAMS += tst_pthread -endif - -BURST_BUFFER_FILES = $(NC_FILES:=_*.meta) $(NC_FILES:=_*.data) +CLEANFILES = $(M4_SRCS:.m4=.c) $(TESTOUTDIR)/tst_pthread.nc* \ + core core.* *.gcda *.gcno *.gcov gmon.out -CLEANFILES = $(M4_SRCS:.m4=.c) core core.* *.gcda *.gcno *.gcov gmon.out \ - $(TESTOUTDIR)/redef1.nc $(TESTOUTDIR)/redef1.bb.nc \ - $(TESTOUTDIR)/redef2.nc \ - $(TESTOUTDIR)/tst_pthread.nc.* $(TESTOUTDIR)/testfile.nc* \ - $(NC_FILES) +check_SCRIPTS = seq_runs.sh parallel_run.sh -EXTRA_DIST = $(M4_SRCS) seq_runs.sh redef-good.ncdump \ - wrap_runs.sh parallel_run.sh +EXTRA_DIST = $(M4_SRCS) seq_runs.sh parallel_run.sh # Some of these tests are designed to run on one process, # Run them on 4 processes to see if they can handle well diff --git a/test/testcases/add_var.c b/test/testcases/add_var.c index 002942014f..1ebee10632 100644 --- a/test/testcases/add_var.c +++ b/test/testcases/add_var.c @@ -29,26 +29,22 @@ #include -static int -tst_fmt(char *filename, int cmode) +static +int add_var(const char *out_path, + int format, + int coll_io, + MPI_Info info) { - char fname[256], var_name[256]; + char var_name[256]; int i, nvars, err, nerrs=0; int ncid, varid, dimid[2]; MPI_Offset prev_off; - if (cmode == 0) sprintf(fname,"%s",filename); - else if (cmode & NC_64BIT_OFFSET) sprintf(fname,"%s%d",filename,2); - else if (cmode & NC_64BIT_DATA) sprintf(fname,"%s%d",filename,5); - else if (cmode & NC_NETCDF4) { - if (cmode & NC_CLASSIC_MODEL) - sprintf(fname,"%s%d",filename,4); - else - sprintf(fname,"%s%d",filename,3); - } + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR - cmode |= NC_CLOBBER; - err = ncmpi_create(MPI_COMM_WORLD, fname, cmode, MPI_INFO_NULL, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR /* define dimensions */ err = ncmpi_def_dim(ncid, "dim_1", 5, &dimid[0]); CHECK_ERR @@ -58,6 +54,7 @@ tst_fmt(char *filename, int cmode) for (i=0; i<10; i++) { sprintf(var_name, "var_%d", i); err = ncmpi_def_var(ncid, var_name, NC_INT, 2, dimid, &varid); CHECK_ERR + err = ncmpi_def_var_fill(ncid, varid, 0, NULL); CHECK_ERR } err = ncmpi_enddef(ncid); CHECK_ERR @@ -70,11 +67,13 @@ tst_fmt(char *filename, int cmode) /* add 2 new variables */ err = ncmpi_def_var(ncid, "new_var1", NC_INT, 2, dimid, &varid); CHECK_ERR + err = ncmpi_def_var_fill(ncid, varid, 0, NULL); CHECK_ERR err = ncmpi_def_var(ncid, "new_var2", NC_FLOAT, 2, dimid, &varid); CHECK_ERR + err = ncmpi_def_var_fill(ncid, varid, 0, NULL); CHECK_ERR err = ncmpi_enddef(ncid); CHECK_ERR err = ncmpi_inq_nvars(ncid, &nvars); CHECK_ERR - if (cmode & NC_NETCDF4) { + if (format == NC_FORMAT_NETCDF4 || format == NC_FORMAT_NETCDF4_CLASSIC) { err = ncmpi_inq_varoffset(ncid, 0, &prev_off); EXP_ERR(NC_ENOTSUPPORT) } else { @@ -84,7 +83,7 @@ tst_fmt(char *filename, int cmode) err = ncmpi_inq_varoffset(ncid, i, &off); CHECK_ERR if (off < prev_off + 5*4*4) { /* each variable is of size 5*4*4 bytes */ err = ncmpi_inq_varname(ncid, i, var_name); CHECK_ERR - printf("Error at line %d in %s: variable %s offset is set incorrectly\n", + fprintf(stderr,"Error at line %d in %s: variable %s offset is set incorrectly\n", __LINE__,__FILE__,var_name); nerrs++; } @@ -96,63 +95,47 @@ tst_fmt(char *filename, int cmode) return nerrs; } -int main(int argc, char** argv) { - char filename[256], *hint_value; - int rank, err, nerrs=0, bb_enabled=0; +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) +{ + int nerrs; - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); + /* test without setting hint nc_data_move_chunk_size */ + nerrs = add_var(out_path, format, coll_io, info); + if (nerrs > 0) return nerrs; - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for checking offsets of new variables ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } + /* test with setting hint nc_data_move_chunk_size */ + MPI_Info_set(info, "nc_data_move_chunk_size", "100"); + nerrs = add_var(out_path, format, coll_io, info); + if (nerrs > 0) return nerrs; - /* check whether burst buffering is enabled */ - if (inq_env_hint("nc_burst_buf", &hint_value)) { - if (strcasecmp(hint_value, "enable") == 0) bb_enabled = 1; - free(hint_value); - } + return 0; +} - nerrs += tst_fmt(filename, 0); - nerrs += tst_fmt(filename, NC_64BIT_OFFSET); - if (!bb_enabled) { -#ifdef ENABLE_NETCDF4 - nerrs += tst_fmt(filename, NC_NETCDF4); - nerrs += tst_fmt(filename, NC_NETCDF4 | NC_CLASSIC_MODEL); -#endif - } - nerrs += tst_fmt(filename, NC_64BIT_DATA); - - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } +int main(int argc, char **argv) { - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + int err; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "checking offsets of new variables", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/testcases/alignment_test.c b/test/testcases/alignment_test.c index c04a2377cc..1e1dec2c11 100644 --- a/test/testcases/alignment_test.c +++ b/test/testcases/alignment_test.c @@ -31,14 +31,20 @@ #include #define NVARS 8 -#define NX 5 +#define NX 70 -static int tst_mode(char *filename, - int mode) +#define TEST_FIXED_VAR +#define TEST_RECORD_VAR + +static +int alignment_test(const char *out_path, + int format, + int coll_io, + MPI_Info global_info) { - int i, j, rank, nprocs, err, verbose=0, nerrs=0; - int ncid, cmode, varid[NVARS], dimid[2], *buf; char str[32]; + int i, j, rank, nprocs, err, verbose=0, nerrs=0; + int ncid, varid[NVARS], dimid[2], *buf=NULL; MPI_Offset start[2], count[2]; MPI_Offset new_var_off[NVARS*2], old_var_off[NVARS*2]; MPI_Offset header_size[2], header_extent[2]; @@ -47,35 +53,40 @@ static int tst_mode(char *filename, MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + MPI_Info_dup(global_info, &info); + + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + /* create a new file for writing ----------------------------------------*/ - cmode = NC_CLOBBER | NC_64BIT_DATA; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); - CHECK_ERROUT + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR /* define dimension */ err = ncmpi_def_dim(ncid, "Y", NC_UNLIMITED, &dimid[0]); CHECK_ERR err = ncmpi_def_dim(ncid, "X", NX*nprocs, &dimid[1]); CHECK_ERR -#define TEST_FIXED_VAR -#define TEST_RECORD_VAR /* Odd numbers are fixed variables, even numbers are record variables */ for (i=0; i 0) return nerrs; - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for alignment ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } + /* test with setting hint nc_data_move_chunk_size */ + MPI_Info_set(info, "nc_data_move_chunk_size", "100"); + nerrs = alignment_test(out_path, format, coll_io, info); + if (nerrs > 0) return nerrs; - nerrs += tst_mode(filename, MODE_COLL); - if (nerrs > 0) goto err_out; - - nerrs += tst_mode(filename, MODE_INDEP); - if (nerrs > 0) goto err_out; - - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + return 0; +} -err_out: - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "alignment hints", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/testcases/attrf.f b/test/testcases/attrf.f index a3e4fce108..ac572ec6ee 100644 --- a/test/testcases/attrf.f +++ b/test/testcases/attrf.f @@ -34,8 +34,8 @@ subroutine check(err, message, nerrs) ! It is a good idea to check returned value for possible error if (err .NE. NF_NOERR) then write(6,*) message(1:XTRIM(message)), nfmpi_strerror(err) - msg = '*** TESTING F77 attrf.f for attribute overflow ' - call pass_fail(1, msg) + msg = '*** TESTING F77 attrf.f - attribute overflow ' + call pass_fail(1, msg, 0) nerrs = nerrs + 1 end if end ! subroutine check @@ -51,32 +51,40 @@ program main integer*2 buf_int2 integer*8 buf_int8, one - character(LEN=256) filename, cmd, msg + character(LEN=256) out_path, in_path, cmd, msg integer ncid, err, ierr, nerrs, nprocs, rank, get_args integer*8 malloc_size, sum_size + logical keep_files + double precision timing call MPI_Init(ierr) + + timing = MPI_Wtime() + call MPI_Comm_rank(MPI_COMM_WORLD, rank, ierr) call MPI_Comm_size(MPI_COMM_WORLD, nprocs, ierr) one = 1 one_flt = 1.0 - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any if (rank .EQ. 0) then - filename = "testfile.nc" - err = get_args(cmd, filename) + out_path = "testfile.nc" + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, + MPI_COMM_WORLD, ierr) + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, + + ierr) + nerrs = 0 cmode = IOR(NF_CLOBBER,NF_64BIT_DATA) - err = nfmpi_create(MPI_COMM_WORLD, filename, cmode, + err = nfmpi_create(MPI_COMM_WORLD, out_path, cmode, + MPI_INFO_NULL, ncid) call check(err, 'In nfmpi_create: ', nerrs) @@ -184,10 +192,18 @@ program main + sum_size, ' bytes yet to be freed' endif + timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, + + MPI_DOUBLE_PRECISION, MPI_MAX, + + MPI_COMM_WORLD, ierr) if (rank .eq. 0) then + if (.NOT. keep_files) then + err = nfmpi_delete(out_path, MPI_INFO_NULL) + end if + msg = '*** TESTING F77 '//cmd(1:XTRIM(cmd))// - + ' for attribute overflow ' - call pass_fail(nerrs, msg) + + ' - attribute overflow ' + call pass_fail(nerrs, msg, timing) endif 999 call MPI_Finalize(ierr) diff --git a/test/testcases/buftype_free.c b/test/testcases/buftype_free.c index 90ffc7645c..8e5ecb619f 100644 --- a/test/testcases/buftype_free.c +++ b/test/testcases/buftype_free.c @@ -1,8 +1,6 @@ /* * Copyright (C) 2015, Northwestern University and Argonne National Laboratory * See COPYRIGHT notice in top-level directory. - * - * $Id$ */ /* @@ -21,39 +19,32 @@ #include #define NY 4 -#define NX 4 - -/*----< main() >------------------------------------------------------------*/ -int main(int argc, char **argv) { - - char filename[256]; - int i, j, err, ncid, varid[4], dimids[2], req[4], st[4], nerrs=0; - int rank, nprocs, buf[4][(NY+4)*(NX+4)]; - int gsize[2], subsize[2], a_start[2], ghost; +#define NX 100 +#define NVARS 4 +#define NGHOSTS 2 + +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) +{ + int i, j, k, err, ncid, nerrs=0, rank, nprocs; + int varid[NVARS], dimids[2], req[NVARS], st[NVARS], *buf[NVARS]; + int gsize[2], subsize[2], a_start[2], ghost; MPI_Offset start[2], count[2]; - MPI_Datatype buftype[4]; + MPI_Datatype buftype[NVARS]; - MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); MPI_Comm_rank(MPI_COMM_WORLD, &rank); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for free buftype in flexible API ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, MPI_INFO_NULL, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR /* define a 2D array */ err = ncmpi_def_dim(ncid, "Y", NY*nprocs, &dimids[0]); CHECK_ERR @@ -64,49 +55,91 @@ int main(int argc, char **argv) { err = ncmpi_def_var(ncid, "var3", NC_INT, 2, dimids, &varid[3]); CHECK_ERR err = ncmpi_enddef(ncid); CHECK_ERR - /* initialize the contents of the array */ - for (i=0; i<4; i++) for (j=0; j<(NY+4)*(NX+4); j++) buf[i][j] = rank+10; + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + /* initialize the contents of the array */ start[0] = NY*rank; start[1] = 0; count[0] = NY; count[1] = NX; - err = ncmpi_put_vara_int_all(ncid, varid[0], start, count, buf[0]); CHECK_ERR - err = ncmpi_put_vara_int_all(ncid, varid[1], start, count, buf[1]); CHECK_ERR - err = ncmpi_put_vara_int_all(ncid, varid[2], start, count, buf[2]); CHECK_ERR - err = ncmpi_put_vara_int_all(ncid, varid[3], start, count, buf[3]); CHECK_ERR + for (i=0; i 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); + /* check contents of read buffers */ + for (i=0; i= gsize[0]-ghost || + k < ghost || k >= gsize[1]-ghost) + exp = -1; + else + exp = (int)((j-ghost)*count[1]+(k-ghost) + rank*10); + if (buf[i][j*gsize[1]+k] != exp) { + fprintf(stderr,"Error at %d: var %d expect buf[%d][%d] = %d but got %d\n", + __LINE__, i, j, k, exp, buf[i][j*gsize[1]+k]); + nerrs++; + goto err_out; + } + } + } } - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + for (i=0; i 0); +err_out: + return nerrs; } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "free buftype in flexible API", opt, test_io); + + MPI_Finalize(); + + return err; +} diff --git a/test/testcases/buftype_freef.f b/test/testcases/buftype_freef.f index 18e06cc679..7fba21ee9a 100644 --- a/test/testcases/buftype_freef.f +++ b/test/testcases/buftype_freef.f @@ -35,8 +35,8 @@ subroutine check(err, message) ! It is a good idea to check returned value for possible error if (err .NE. NF_NOERR) then write(6,*) message(1:XTRIM(message)), nfmpi_strerror(err) - msg = '*** TESTING F77 buftype_freef.f for flexible API ' - call pass_fail(1, msg) + msg = '*** TESTING F77 buftype_freef.f - flexible API ' + call pass_fail(1, msg, 0) STOP 2 end if end ! subroutine check @@ -50,29 +50,37 @@ program main integer*8 NX, NY PARAMETER(NREQS=4, NX=4, NY=4) - character(LEN=256) filename, cmd, msg, varname, str + character(LEN=256) out_path, in_path, cmd, msg, varname, str integer i, j, err, ierr, nprocs, rank, nerrs, get_args integer ncid, ghost integer buf(64,4), varid(4), dimid(2), req(4), st(4) integer buftype(4), gsize(2), subsize(2), a_start(2) integer*8 start(2), count(2) integer*8 one, malloc_size, sum_size + logical keep_files + double precision timing call MPI_Init(ierr) + + timing = MPI_Wtime() + call MPI_Comm_rank(MPI_COMM_WORLD, rank, ierr) call MPI_Comm_size(MPI_COMM_WORLD, nprocs, ierr) - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any if (rank .EQ. 0) then - filename = "testfile.nc" - err = get_args(cmd, filename) + out_path = "testfile.nc" + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, + MPI_COMM_WORLD, ierr) + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, + + ierr) + nerrs = 0 ! initialize I/O buffer @@ -83,7 +91,7 @@ program main enddo ! create file, truncate it if exists - err = nfmpi_create(MPI_COMM_WORLD, filename, NF_CLOBBER, + err = nfmpi_create(MPI_COMM_WORLD, out_path, NF_CLOBBER, + MPI_INFO_NULL, ncid) call check(err, 'In nfmpi_create:') @@ -167,10 +175,18 @@ program main + sum_size, ' bytes yet to be freed' endif + timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, + + MPI_DOUBLE_PRECISION, MPI_MAX, + + MPI_COMM_WORLD, ierr) if (rank .eq. 0) then + if (.NOT. keep_files) then + err = nfmpi_delete(out_path, MPI_INFO_NULL) + end if + msg = '*** TESTING F77 '//cmd(1:XTRIM(cmd))// - + ' for flexible API ' - call pass_fail(nerrs, msg) + + ' - flexible API ' + call pass_fail(nerrs, msg, timing) endif 999 call MPI_Finalize(ierr) diff --git a/test/testcases/check_striping.c b/test/testcases/check_striping.c index ff36e0215a..6c4ecee47d 100644 --- a/test/testcases/check_striping.c +++ b/test/testcases/check_striping.c @@ -29,19 +29,30 @@ #include -static int -tst_fmt(char *filename, int cmode) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) { int err, nerrs=0, ncid, fmt; - int striping_size=0, striping_count=0, root_striping_size, root_striping_count; + int striping_size=0, striping_count=0; + int root_striping_size, root_striping_count; - cmode |= NC_CLOBBER; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); CHECK_ERR + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + /* check file create case */ + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR err = ncmpi_enddef(ncid); CHECK_ERR err = ncmpi_inq_format(ncid, &fmt); CHECK_ERR + err = ncmpi_inq_striping(ncid, &striping_size, &striping_count); - if (fmt == NC_FORMAT_NETCDF4 || fmt == NC_FORMAT_NETCDF4_CLASSIC) + if (format == NC_FORMAT_NETCDF4 || format == NC_FORMAT_NETCDF4_CLASSIC) EXP_ERR(NC_ENOTSUPPORT) else CHECK_ERR @@ -53,76 +64,72 @@ tst_fmt(char *filename, int cmode) err = MPI_Bcast(&root_striping_count, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_ERR(err) if (root_striping_size != striping_size) { - printf("Error at line %d in %s: inconsistent striping_size (root=%d local=%d)\n", + fprintf(stderr,"Error at line %d in %s: inconsistent striping_size (root=%d local=%d)\n", __LINE__,__FILE__, root_striping_size, striping_size); nerrs++; } if (root_striping_count != striping_count) { - printf("Error at line %d in %s: inconsistent striping_count (root=%d local=%d)\n", + fprintf(stderr,"Error at line %d in %s: inconsistent striping_count (root=%d local=%d)\n", __LINE__,__FILE__, root_striping_count, striping_count); nerrs++; } err = ncmpi_close(ncid); CHECK_ERR - return nerrs; -} + /* check file open case */ + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_NOWRITE, info, &ncid); + CHECK_ERR -int main(int argc, char** argv) { - char filename[256], *hint_value; - int rank, err, nerrs=0, bb_enabled=0; + err = ncmpi_inq_format(ncid, &fmt); CHECK_ERR - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); + err = ncmpi_inq_striping(ncid, &striping_size, &striping_count); + if (format == NC_FORMAT_NETCDF4 || format == NC_FORMAT_NETCDF4_CLASSIC) + EXP_ERR(NC_ENOTSUPPORT) + else + CHECK_ERR - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; + root_striping_size = striping_size; + root_striping_count = striping_count; + err = MPI_Bcast(&root_striping_size, 1, MPI_INT, 0, MPI_COMM_WORLD); + MPI_ERR(err) + err = MPI_Bcast(&root_striping_count, 1, MPI_INT, 0, MPI_COMM_WORLD); + MPI_ERR(err) + if (root_striping_size != striping_size) { + fprintf(stderr,"Error at line %d in %s: inconsistent striping_size (root=%d local=%d)\n", + __LINE__,__FILE__, root_striping_size, striping_size); + nerrs++; } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for striping info ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); + if (root_striping_count != striping_count) { + fprintf(stderr,"Error at line %d in %s: inconsistent striping_count (root=%d local=%d)\n", + __LINE__,__FILE__, root_striping_count, striping_count); + nerrs++; } - /* check whether burst buffering is enabled */ - if (inq_env_hint("nc_burst_buf", &hint_value)) { - if (strcasecmp(hint_value, "enable") == 0) bb_enabled = 1; - free(hint_value); - } + err = ncmpi_close(ncid); CHECK_ERR - nerrs += tst_fmt(filename, 0); - nerrs += tst_fmt(filename, NC_64BIT_OFFSET); - if (!bb_enabled) { -#ifdef ENABLE_NETCDF4 - nerrs += tst_fmt(filename, NC_NETCDF4); - nerrs += tst_fmt(filename, NC_NETCDF4 | NC_CLASSIC_MODEL); -#endif - } - nerrs += tst_fmt(filename, NC_64BIT_DATA); - - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + return nerrs; +} + +int main(int argc, char **argv) { + + int err; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = false;/* skip ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ + + err = tst_main(argc, argv, "inquire striping info", opt, test_io); - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/testcases/check_type.c b/test/testcases/check_type.c index 36a559b400..4f6f03d216 100644 --- a/test/testcases/check_type.c +++ b/test/testcases/check_type.c @@ -28,7 +28,7 @@ #define EXP_ERR_N_TYPE(expect_err,itype,etype) { \ if (err != expect_err) { \ - printf("Error at line %d in %s: itype=%9s etype=%-9s err=%s\n", \ + fprintf(stderr,"Error at line %d in %s: itype=%9s etype=%-9s err=%s\n", \ __LINE__,__FILE__,itype,etype_name(etype),ncmpi_strerrno(err)); \ nerrs++; \ } \ @@ -53,19 +53,26 @@ static char* etype_name(nc_type etype) { } } -static int -tst_fmt(char *filename, int cmode) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) { char *varname[12], buf[1024], attname[256]; int i, err, nerrs=0, ncid, dimid, varid[12], max_type; - if (cmode == 0 || cmode == NC_64BIT_OFFSET || cmode & NC_CLASSIC_MODEL) - max_type = NC_DOUBLE; - else + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + if (format == NC_FORMAT_64BIT_DATA) max_type = NC_UINT64; + else + max_type = NC_DOUBLE; - cmode |= NC_CLOBBER; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "x", 2, &dimid); CHECK_ERR varname[0] = "var_nat"; @@ -173,65 +180,26 @@ tst_fmt(char *filename, int cmode) return nerrs; } -int main(int argc, char* argv[]) -{ - char filename[256], *hint_value; - int err, nerrs=0, rank, bb_enabled=0; - - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); +int main(int argc, char **argv) { - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for checking for type conflict ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } + int err; + loop_opts opt; - /* check whether burst buffering is enabled */ - if (inq_env_hint("nc_burst_buf", &hint_value)) { - if (strcasecmp(hint_value, "enable") == 0) bb_enabled = 1; - free(hint_value); - } + MPI_Init(&argc, &argv); - nerrs += tst_fmt(filename, 0); - nerrs += tst_fmt(filename, NC_64BIT_OFFSET); - if (!bb_enabled) { -#ifdef ENABLE_NETCDF4 - nerrs += tst_fmt(filename, NC_NETCDF4); - nerrs += tst_fmt(filename, NC_NETCDF4 | NC_CLASSIC_MODEL); -#endif - } - nerrs += tst_fmt(filename, NC_64BIT_DATA); - - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + err = tst_main(argc, argv, "checking for type conflict", opt, test_io); MPI_Finalize(); - return (nerrs > 0); + return err; } - diff --git a/test/testcases/collective_error.c b/test/testcases/collective_error.c index 6a22088a01..6d850ffe46 100644 --- a/test/testcases/collective_error.c +++ b/test/testcases/collective_error.c @@ -32,9 +32,15 @@ #include static -int test_collective_error(char *filename, int safe_mode, int cmode) +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) { - int rank, nproc, ncid, err, nerrs=0, varid, dimids[1], req, status; + char *val; + int rank, nproc, ncid, err, nerrs=0, varid, dimids[1], req, status, exp; + int safe_mode=0; double buf[2]; MPI_Offset start[1], count[1]; MPI_Comm comm=MPI_COMM_WORLD; @@ -42,22 +48,30 @@ int test_collective_error(char *filename, int safe_mode, int cmode) MPI_Comm_rank(comm, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nproc); + val = getenv("PNETCDF_SAFE_MODE"); + if (val != NULL && atoi(val) == 1) safe_mode = 1; +#if PNETCDF_DEBUG_MODE == 1 + if (val == NULL && !safe_mode) safe_mode = 1; +#endif + + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + /* Create a 2 element vector of doubles */ - cmode |= NC_CLOBBER; - err = ncmpi_create(comm, filename, cmode, MPI_INFO_NULL, &ncid); CHECK_ERR - err = ncmpi_def_dim(ncid, "dim", 2, &dimids[0]); CHECK_ERR - err = ncmpi_def_var(ncid, "var", NC_DOUBLE, 1, dimids, &varid); CHECK_ERR - err = ncmpi_enddef(ncid); CHECK_ERR + err = ncmpi_create(comm, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR_ALL + err = ncmpi_def_dim(ncid, "dim", 2, &dimids[0]); CHECK_ERR_ALL + err = ncmpi_def_var(ncid, "var", NC_DOUBLE, 1, dimids, &varid); CHECK_ERR_ALL + err = ncmpi_enddef(ncid); CHECK_ERR_ALL if (rank == 0) { start[0] = 0; count[0] = 2; } else if (rank == 1) { -#if defined(PNETCDF_RELAX_COORD_BOUND) && PNETCDF_RELAX_COORD_BOUND==1 - start[0] = 3; /* illegal for a start > defined shape */ -#else - start[0] = 2; /* illegal for a start >= defined shape */ -#endif + if (is_relax_coord_bound()) + start[0] = 3; /* illegal for a start > defined shape */ + else + start[0] = 2; /* illegal for a start >= defined shape */ count[0] = 0; } else { @@ -70,111 +84,92 @@ int test_collective_error(char *filename, int safe_mode, int cmode) err = ncmpi_put_vara_all(ncid, varid, start, count, buf, count[0], MPI_DOUBLE); - if ((safe_mode && nproc > 1) || rank == 1) EXP_ERR(NC_EINVALCOORDS) - else EXP_ERR(NC_NOERR) + if ((safe_mode && nproc > 1) || rank == 1) exp = NC_EINVALCOORDS; + else exp = NC_NOERR; + CHECK_EXP_ERR_ALL(exp) /* check if user put buffer contents altered */ if (buf[0] != 1.0) { - printf("Error at line %d in %s: user put buffer[%d] altered from %f to %f\n", + fprintf(stderr,"Error at line %d in %s: user put buffer[%d] altered from %f to %f\n", __LINE__,__FILE__,0, 1.0, buf[0]); nerrs++; } if (buf[1] != 2.0) { - printf("Error at line %d in %s: user put buffer[%d] altered from %f to %f\n", + fprintf(stderr,"Error at line %d in %s: user put buffer[%d] altered from %f to %f\n", __LINE__,__FILE__,1, 2.0, buf[1]); nerrs++; } err = ncmpi_put_vara_double_all(ncid, varid, start, count, buf); - if ((safe_mode && nproc > 1) || rank == 1) EXP_ERR(NC_EINVALCOORDS) - else EXP_ERR(NC_NOERR) + if ((safe_mode && nproc > 1) || rank == 1) exp = NC_EINVALCOORDS; + else exp = NC_NOERR; + CHECK_EXP_ERR_ALL(exp) /* check if user put buffer contents altered */ if (buf[0] != 1.0) { - printf("Error at line %d in %s: user put buffer[%d] altered from %f to %f\n", + fprintf(stderr,"Error at line %d in %s: user put buffer[%d] altered from %f to %f\n", __LINE__,__FILE__,0, 1.0, buf[0]); nerrs++; } if (buf[1] != 2.0) { - printf("Error at line %d in %s: user put buffer[%d] altered from %f to %f\n", + fprintf(stderr,"Error at line %d in %s: user put buffer[%d] altered from %f to %f\n", __LINE__,__FILE__,1, 2.0, buf[1]); nerrs++; } - if (!(cmode & NC_NETCDF4)) { + if (format != NC_FORMAT_NETCDF4 && format != NC_FORMAT_NETCDF4_CLASSIC) { err = ncmpi_iput_vara_double(ncid, varid, start, count, buf, &req); - if (rank == 1) - EXP_ERR(NC_EINVALCOORDS) - else - EXP_ERR(NC_NOERR) + exp = (rank == 1) ? NC_EINVALCOORDS : NC_NOERR; + EXP_ERR(exp) - err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR + err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR_ALL /* check if user put buffer contents altered */ if (buf[0] != 1.0) { - printf("Error at line %d in %s: user put buffer[%d] altered from %f to %f\n", + fprintf(stderr,"Error at line %d in %s: user put buffer[%d] altered from %f to %f\n", __LINE__,__FILE__,0, 1.0, buf[0]); nerrs++; } if (buf[1] != 2.0) { - printf("Error at line %d in %s: user put buffer[%d] altered from %f to %f\n", + fprintf(stderr,"Error at line %d in %s: user put buffer[%d] altered from %f to %f\n", __LINE__,__FILE__,1, 2.0, buf[1]); nerrs++; } } + /* file sync before reading */ + err = ncmpi_sync(ncid); + CHECK_ERR + MPI_Barrier(MPI_COMM_WORLD); + err = ncmpi_get_vara_all(ncid, varid, start, count, buf, count[0], MPI_DOUBLE); - if ((safe_mode && nproc > 1) || rank == 1) EXP_ERR(NC_EINVALCOORDS) - else EXP_ERR(NC_NOERR) + if ((safe_mode && nproc > 1) || rank == 1) exp = NC_EINVALCOORDS; + else exp = NC_NOERR; + CHECK_EXP_ERR_ALL(exp) err = ncmpi_get_vara_double_all(ncid, varid, start, count, buf); - if ((safe_mode && nproc > 1) || rank == 1) EXP_ERR(NC_EINVALCOORDS) - else EXP_ERR(NC_NOERR) + if ((safe_mode && nproc > 1) || rank == 1) exp = NC_EINVALCOORDS; + else exp = NC_NOERR; + CHECK_EXP_ERR_ALL(exp) - if (!(cmode & NC_NETCDF4)) { + if (format != NC_FORMAT_NETCDF4 && format != NC_FORMAT_NETCDF4_CLASSIC) { err = ncmpi_iget_vara_double(ncid, varid, start, count, buf, &req); - if (rank == 1) - EXP_ERR(NC_EINVALCOORDS) - else - EXP_ERR(NC_NOERR) + exp = (rank == 1) ? NC_EINVALCOORDS : NC_NOERR; + EXP_ERR(exp) - err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR + err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR_ALL } - err = ncmpi_close(ncid); CHECK_ERR + err = ncmpi_close(ncid); CHECK_ERR_ALL +err_out: return nerrs; } -int main(int argc, char *argv[]) +#if 0 { - char filename[256], *hint_value; - int rank, err, nerrs=0, bb_enabled=0; - - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for collective abort ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - - /* check whether burst buffering is enabled */ - if (inq_env_hint("nc_burst_buf", &hint_value)) { - if (strcasecmp(hint_value, "enable") == 0) bb_enabled = 1; - free(hint_value); - } + int err, nerrs=0; /* test in non-safe mode */ setenv("PNETCDF_SAFE_MODE", "0", 1); @@ -183,7 +178,7 @@ int main(int argc, char *argv[]) nerrs += test_collective_error(filename, 0, NC_64BIT_OFFSET); if (nerrs) goto err_out; if (!bb_enabled) { -#ifdef ENABLE_NETCDF4 +#if PNETCDF_DRIVER_NETCDF4 == 1 nerrs += test_collective_error(filename, 0, NC_NETCDF4); if (nerrs) goto err_out; nerrs += test_collective_error(filename, 0, NC_NETCDF4 | NC_CLASSIC_MODEL); @@ -200,7 +195,7 @@ int main(int argc, char *argv[]) nerrs += test_collective_error(filename, 1, NC_64BIT_OFFSET); if (nerrs) goto err_out; if (!bb_enabled) { -#ifdef ENABLE_NETCDF4 +#if PNETCDF_DRIVER_NETCDF4 == 1 nerrs += test_collective_error(filename, 1, NC_NETCDF4); if (nerrs) goto err_out; nerrs += test_collective_error(filename, 1, NC_NETCDF4 | NC_CLASSIC_MODEL); @@ -210,24 +205,30 @@ int main(int argc, char *argv[]) nerrs += test_collective_error(filename, 1, NC_64BIT_DATA); if (nerrs) goto err_out; - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + return (nerrs > 0); +} +#endif -err_out: - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +int main(int argc, char **argv) { + + int err; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "collective abort", opt, test_io); MPI_Finalize(); - return (nerrs > 0); + + return err; } diff --git a/test/testcases/erange_fill.m4 b/test/testcases/erange_fill.m4 index b9a1f6321c..97872e758c 100644 --- a/test/testcases/erange_fill.m4 +++ b/test/testcases/erange_fill.m4 @@ -39,6 +39,8 @@ dnl */ #define LEN 12 +static int bb_enabled; + include(`foreach.m4')dnl include(`utils.m4')dnl @@ -77,121 +79,137 @@ define(`ITYPE_SIZE',`ifelse( `$1', `longlong', `8',dnl `$1', `ulonglong', `8')')dnl +define(`CHECK_DEFAULT_FILL_VALUE',` + err = ncmpi_inq_varid(ncid, "var_$1", &varid); CHECK_ERR + if (coll_io) + err = GET_VAR($1,_all)(ncid, varid, buf_$1); + else + err = GET_VAR($1)(ncid, varid, buf_$1); + CHECK_ERR + for (i=0; i 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for erange elements are filled ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) +{ + char hint[MPI_MAX_INFO_VAL]; + int err, nerrs=0, flag; - /*---- CDF-2 format -----------------------------------------------------*/ - /* ncmpi_set_default_format(NC_FORMAT_CLASSIC, NULL); */ - ncmpi_set_default_format(NC_FORMAT_CDF2, NULL); + MPI_Info_get(info, "nc_burst_buf", MPI_MAX_INFO_VAL-1, hint, &flag); + if (flag && strcasecmp(hint, "enable") == 0) + bb_enabled = 1; + else + bb_enabled = 0; - nerrs += test_default_fill_mode(filename); + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR - foreach(`itype', (CDF2_ITYPE_LIST), ` - _CAT(`nerrs += test_default_fill_',itype)'`(filename);') + nerrs += test_default_fill_mode(out_path, format, coll_io, info); - fillv=99; - foreach(`itype', (CDF2_ITYPE_LIST), ` - _CAT(`nerrs += test_user_fill_',itype)'`(filename, (itype)fillv);') + nerrs += test_user_fill_mode(out_path, format, coll_io, info); /* test put ERANGE values */ foreach(`itype', (uchar,short,int,float,double), ` - _CAT(`nerrs += test_erange_put_schar_',itype)'`(filename);') + _CAT(`nerrs += test_erange_put_schar_',itype)'`(out_path, coll_io, info);') foreach(`itype', (ushort,int,uint,float,double), ` - _CAT(`nerrs += test_erange_put_short_',itype)'`(filename);') + _CAT(`nerrs += test_erange_put_short_',itype)'`(out_path, coll_io, info);') foreach(`itype', (float,double), ` - _CAT(`nerrs += test_erange_put_int_',itype)'`(filename);') + _CAT(`nerrs += test_erange_put_int_',itype)'`(out_path, coll_io, info);') - nerrs += test_erange_put_float_double(filename); + nerrs += test_erange_put_float_double(out_path, coll_io, info); /* test get ERANGE values */ foreach(`itype', (short,int,float,double), ` - _CAT(`nerrs += test_erange_get_',itype)'`_schar(filename);') + _CAT(`nerrs += test_erange_get_',itype)'`_schar(out_path, coll_io, info);') foreach(`itype', (schar,short,int,float,double), ` - _CAT(`nerrs += test_erange_get_',itype)'`_uchar(filename);') + _CAT(`nerrs += test_erange_get_',itype)'`_uchar(out_path, coll_io, info);') foreach(`itype', (int,float,double), ` - _CAT(`nerrs += test_erange_get_',itype)'`_short(filename);') + _CAT(`nerrs += test_erange_get_',itype)'`_short(out_path, coll_io, info);') foreach(`itype', (float,double), ` - _CAT(`nerrs += test_erange_get_',itype)'`_int(filename);') + _CAT(`nerrs += test_erange_get_',itype)'`_int(out_path, coll_io, info);') - nerrs += test_erange_get_double_float(filename); + nerrs += test_erange_get_double_float(out_path, coll_io, info); /*---- CDF-5 format -----------------------------------------------------*/ - ncmpi_set_default_format(NC_FORMAT_CDF5, NULL); + if (format == NC_FORMAT_CDF5) { - nerrs += test_default_fill_mode(filename); + /* test put ERANGE values */ + foreach(`itype', (ushort,uint,longlong,ulonglong), ` + _CAT(`nerrs += test_erange_put_schar_',itype)'`(out_path, coll_io, info);') - foreach(`itype', (ITYPE_LIST), ` - _CAT(`nerrs += test_default_fill_',itype)'`(filename);') + foreach(`itype', (schar,short,ushort,int,uint,float,double,longlong,ulonglong), ` + _CAT(`nerrs += test_erange_put_uchar_',itype)'`(out_path, coll_io, info);') - fillv=99; - foreach(`itype', (ITYPE_LIST), ` - _CAT(`nerrs += test_user_fill_',itype)'`(filename, (itype)fillv);') + foreach(`itype', (longlong,ulonglong), ` + _CAT(`nerrs += test_erange_put_short_',itype)'`(out_path, coll_io, info);') - /* test put ERANGE values */ - foreach(`itype', (uchar,short,ushort,int,uint,float,double,longlong,ulonglong), ` - _CAT(`nerrs += test_erange_put_schar_',itype)'`(filename);') + foreach(`itype', (short,int,uint,float,double,longlong,ulonglong), ` + _CAT(`nerrs += test_erange_put_ushort_',itype)'`(out_path, coll_io, info);') - foreach(`itype', (schar,short,ushort,int,uint,float,double,longlong,ulonglong), ` - _CAT(`nerrs += test_erange_put_uchar_',itype)'`(filename);') + foreach(`itype', (uint,longlong,ulonglong), ` + _CAT(`nerrs += test_erange_put_int_',itype)'`(out_path, coll_io, info);') - foreach(`itype', (ushort,int,uint,float,double,longlong,ulonglong), ` - _CAT(`nerrs += test_erange_put_short_',itype)'`(filename);') + foreach(`itype', (int,float,double,longlong,ulonglong), ` + _CAT(`nerrs += test_erange_put_uint_',itype)'`(out_path, coll_io, info);') - foreach(`itype', (short,int,uint,float,double,longlong,ulonglong), ` - _CAT(`nerrs += test_erange_put_ushort_',itype)'`(filename);') + nerrs += test_erange_put_float_double(out_path, coll_io, info); - foreach(`itype', (uint,float,double,longlong,ulonglong), ` - _CAT(`nerrs += test_erange_put_int_',itype)'`(filename);') + /* test get ERANGE values */ + foreach(`itype', (uchar,ushort,uint,longlong,ulonglong), ` + _CAT(`nerrs += test_erange_get_',itype)'`_schar(out_path, coll_io, info);') - foreach(`itype', (int,float,double,longlong,ulonglong), ` - _CAT(`nerrs += test_erange_put_uint_',itype)'`(filename);') + foreach(`itype', (ushort,uint,longlong,ulonglong), ` + _CAT(`nerrs += test_erange_get_',itype)'`_uchar(out_path, coll_io, info);') - nerrs += test_erange_put_float_double(filename); + foreach(`itype', (ushort,uint,longlong,ulonglong), ` + _CAT(`nerrs += test_erange_get_',itype)'`_short(out_path, coll_io, info);') - /* test get ERANGE values */ - foreach(`itype', (uchar,short,ushort,int,uint,float,double,longlong,ulonglong), ` - _CAT(`nerrs += test_erange_get_',itype)'`_schar(filename);') + foreach(`itype', (short,int,uint,float,double,longlong,ulonglong), ` + _CAT(`nerrs += test_erange_get_',itype)'`_ushort(out_path, coll_io, info);') - foreach(`itype', (schar,short,ushort,int,uint,float,double,longlong,ulonglong), ` - _CAT(`nerrs += test_erange_get_',itype)'`_uchar(filename);') + foreach(`itype', (uint,longlong,ulonglong), ` + _CAT(`nerrs += test_erange_get_',itype)'`_int(out_path, coll_io, info);') - foreach(`itype', (ushort,int,uint,float,double,longlong,ulonglong), ` - _CAT(`nerrs += test_erange_get_',itype)'`_short(filename);') + foreach(`itype', (int,float,double,longlong,ulonglong), ` + _CAT(`nerrs += test_erange_get_',itype)'`_uint(out_path, coll_io, info);') + } - foreach(`itype', (short,int,uint,float,double,longlong,ulonglong), ` - _CAT(`nerrs += test_erange_get_',itype)'`_ushort(filename);') + return nerrs; +} - foreach(`itype', (uint,float,double,longlong,ulonglong), ` - _CAT(`nerrs += test_erange_get_',itype)'`_int(filename);') +int main(int argc, char **argv) { - foreach(`itype', (int,float,double,longlong,ulonglong), ` - _CAT(`nerrs += test_erange_get_',itype)'`_uint(filename);') + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; - nerrs += test_erange_get_double_float(filename); + MPI_Init(&argc, &argv); - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + err = tst_main(argc, argv, "erange elements are filled", opt, test_io); MPI_Finalize(); - return (nerrs == 0) ? 0 : 1; -} + return err; +} diff --git a/test/testcases/error_precedence.m4 b/test/testcases/error_precedence.m4 index 6b26b0c387..2a4595afb0 100644 --- a/test/testcases/error_precedence.m4 +++ b/test/testcases/error_precedence.m4 @@ -40,14 +40,14 @@ dnl #define CHECK_ERR { \ if (err != NC_NOERR) { \ nerrs++; \ - printf("Error at line %d in %s: (%s)\n", \ + fprintf(stderr,"Error at line %d in %s: (%s)\n", \ __LINE__,__FILE__,nc_strerror(err)); \ } \ } #define EXP_ERR(exp) { \ if (err != exp) { \ nerrs++; \ - printf("Error at line %d in %s: expecting " #exp " but got %d\n", \ + fprintf(stderr,"Error at line %d in %s: expecting " #exp " but got %d\n", \ __LINE__,__FILE__, err); \ } \ } @@ -77,8 +77,8 @@ dnl #define EndDef_ ncmpi__enddef #define FileClose ncmpi_close #define StrError ncmpi_strerror -#define FileCreate(a,b,c) ncmpi_create(MPI_COMM_WORLD,a,b,MPI_INFO_NULL,c) -#define FileOpen(a,b,c) ncmpi_open(MPI_COMM_WORLD,a,b,MPI_INFO_NULL,c) +#define FileCreate(a,b,c) ncmpi_create(MPI_COMM_WORLD,a,b,info,c) +#define FileOpen(a,b,c) ncmpi_open(MPI_COMM_WORLD,a,b,info,c) #define API(kind) ncmpi_##kind #define API_ALL(kind) ncmpi_##kind##_all #endif @@ -131,9 +131,9 @@ define(`EXTRA_ITYPES',`uchar,ushort,uint,longlong,ulonglong')dnl define(`TEST_FORMAT',dnl `dnl static int -test_format_nc$1(char *filename) +test_format_nc$1(const char *out_path, int coll_io, MPI_Info info) { - int err, nerrs=0, ncid, cmode, dimids[2]; + int err, nerrs=0, ncid, dimids[2]; MPI_Offset start[2], count[2]; #ifdef TEST_NETCDF ptrdiff_t stride[2]; @@ -160,16 +160,10 @@ test_format_nc$1(char *filename) dnl #define NC_FORMAT_64BIT_DATA (5) /* create a new file */ - ifelse(`$1',`2',`cmode = NC_CLOBBER | NC_64BIT_OFFSET;', - `$1',`5',`cmode = NC_CLOBBER | NC_64BIT_DATA;', - `$1',`3',`cmode = NC_CLOBBER | NC_NETCDF4;', - `$1',`4',`cmode = NC_CLOBBER | NC_NETCDF4 | NC_CLASSIC_MODEL;', - `cmode = NC_CLOBBER;')dnl - - err=FileCreate(filename, cmode, &ncid); + err=FileCreate(out_path, NC_CLOBBER, &ncid); if (err != NC_NOERR) { - printf("Error at line %d in %s: FileCreate() file %s (%s)\n", - __LINE__,__FILE__,filename,StrError(err)); + fprintf(stderr,"Error at line %d in %s: FileCreate() file %s (%s)\n", + __LINE__,__FILE__,out_path,StrError(err)); MPI_Abort(MPI_COMM_WORLD, -1); exit(1); } @@ -196,6 +190,9 @@ test_format_nc$1(char *filename) foreach(`itype',(text, TYPE_LIST),`_CAT(` err=API(def_var)(ncid,"var_'itype`",NC_TYPE(itype),2,dimids,&vid_',itype`); CHECK_ERR')') + foreach(`itype',(text, TYPE_LIST),`_CAT(` + err=API(def_var_fill)(ncid, vid_'itype`, 0, NULL); CHECK_ERR')') + /* For put attribute APIs, the error precedence is the following: * NC_EBADID, NC_EPERM, NC_ENOTVAR, NC_EBADNAME, NC_EBADTYPE, NC_ECHAR, * NC_EINVAL, NC_ENOTINDEFINE, NC_ERANGE @@ -267,22 +264,44 @@ test_format_nc$1(char *filename) err=API(del_att)(ncid,vid_'itype`,`"att_'itype`"'); CHECK_ERR')') /* test put_var APIs in define mode */ - ifelse(`$1',`3',`',`/* test NC_EINDEFINE */dnl - foreach(`itype',(text, TYPE_LIST),`_CAT(` - err=API_ALL(put_var_'itype`) (ncid,-999,NULL); EXP_ERR(NC_EINDEFINE) - err=API_ALL(put_var1_'itype`)(ncid,-999,NULL,NULL); EXP_ERR(NC_EINDEFINE) - err=API_ALL(put_vara_'itype`)(ncid,-999,NULL,NULL,NULL); EXP_ERR(NC_EINDEFINE) - err=API_ALL(put_vars_'itype`)(ncid,-999,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINDEFINE) - err=API_ALL(put_varm_'itype`)(ncid,-999,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINDEFINE)')')') + if (coll_io) { + ifelse(`$1',`3',`',`/* test NC_EINDEFINE */dnl + foreach(`itype',(text, TYPE_LIST),`_CAT(` + err=API_ALL(put_var_'itype`) (ncid,-999,NULL); EXP_ERR(NC_EINDEFINE) + err=API_ALL(put_var1_'itype`)(ncid,-999,NULL,NULL); EXP_ERR(NC_EINDEFINE) + err=API_ALL(put_vara_'itype`)(ncid,-999,NULL,NULL,NULL); EXP_ERR(NC_EINDEFINE) + err=API_ALL(put_vars_'itype`)(ncid,-999,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINDEFINE) + err=API_ALL(put_varm_'itype`)(ncid,-999,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINDEFINE)')')') + } + else { + ifelse(`$1',`3',`',`/* test NC_EINDEFINE */dnl + foreach(`itype',(text, TYPE_LIST),`_CAT(` + err=API(put_var_'itype`) (ncid,-999,NULL); EXP_ERR(NC_EINDEFINE) + err=API(put_var1_'itype`)(ncid,-999,NULL,NULL); EXP_ERR(NC_EINDEFINE) + err=API(put_vara_'itype`)(ncid,-999,NULL,NULL,NULL); EXP_ERR(NC_EINDEFINE) + err=API(put_vars_'itype`)(ncid,-999,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINDEFINE) + err=API(put_varm_'itype`)(ncid,-999,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINDEFINE)')')') + } /* test put_var APIs in define mode */ - ifelse(`$1',`3',`',`/* test NC_EINDEFINE */dnl - foreach(`itype',(text, TYPE_LIST),`_CAT(` - err=API_ALL(get_var_'itype`) (ncid,-999,NULL); EXP_ERR(NC_EINDEFINE) - err=API_ALL(get_var1_'itype`)(ncid,-999,NULL,NULL); EXP_ERR(NC_EINDEFINE) - err=API_ALL(get_vara_'itype`)(ncid,-999,NULL,NULL,NULL); EXP_ERR(NC_EINDEFINE) - err=API_ALL(get_vars_'itype`)(ncid,-999,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINDEFINE) - err=API_ALL(get_varm_'itype`)(ncid,-999,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINDEFINE)')')') + if (coll_io) { + ifelse(`$1',`3',`',`/* test NC_EINDEFINE */dnl + foreach(`itype',(text, TYPE_LIST),`_CAT(` + err=API_ALL(get_var_'itype`) (ncid,-999,NULL); EXP_ERR(NC_EINDEFINE) + err=API_ALL(get_var1_'itype`)(ncid,-999,NULL,NULL); EXP_ERR(NC_EINDEFINE) + err=API_ALL(get_vara_'itype`)(ncid,-999,NULL,NULL,NULL); EXP_ERR(NC_EINDEFINE) + err=API_ALL(get_vars_'itype`)(ncid,-999,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINDEFINE) + err=API_ALL(get_varm_'itype`)(ncid,-999,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINDEFINE)')')') + } + else { + ifelse(`$1',`3',`',`/* test NC_EINDEFINE */dnl + foreach(`itype',(text, TYPE_LIST),`_CAT(` + err=API(get_var_'itype`) (ncid,-999,NULL); EXP_ERR(NC_EINDEFINE) + err=API(get_var1_'itype`)(ncid,-999,NULL,NULL); EXP_ERR(NC_EINDEFINE) + err=API(get_vara_'itype`)(ncid,-999,NULL,NULL,NULL); EXP_ERR(NC_EINDEFINE) + err=API(get_vars_'itype`)(ncid,-999,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINDEFINE) + err=API(get_varm_'itype`)(ncid,-999,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINDEFINE)')')') + } /* test NC_EBADID */ err=EndDef(-999); EXP_ERR(NC_EBADID) @@ -291,6 +310,11 @@ test_format_nc$1(char *filename) /* leave define mode and enter data mode */ err=EndDef(ncid); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + /* attribute att_text has been deleted */ foreach(`itype',(text, TYPE_LIST),`_CAT(` err=API(inq_att)(ncid,vid_'itype`,`"att_'itype`"',NULL,NULL); EXP_ERR(NC_ENOTATT)')') @@ -320,74 +344,146 @@ test_format_nc$1(char *filename) */ /* test NC_EBADID */dnl - foreach(`itype',(text, TYPE_LIST),`_CAT(` - err=API_ALL(put_var_'itype`) (-999,-999,NULL); EXP_ERR(NC_EBADID) - err=API_ALL(put_var1_'itype`)(-999,-999,NULL,NULL); EXP_ERR(NC_EBADID) - err=API_ALL(put_vara_'itype`)(-999,-999,NULL,NULL,NULL); EXP_ERR(NC_EBADID) - err=API_ALL(put_vars_'itype`)(-999,-999,NULL,NULL,NULL,NULL); EXP_ERR(NC_EBADID) - err=API_ALL(put_varm_'itype`)(-999,-999,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_EBADID) - err=API_ALL(get_var_'itype`) (-999,-999,NULL); EXP_ERR(NC_EBADID) - err=API_ALL(get_var1_'itype`)(-999,-999,NULL,NULL); EXP_ERR(NC_EBADID) - err=API_ALL(get_vara_'itype`)(-999,-999,NULL,NULL,NULL); EXP_ERR(NC_EBADID) - err=API_ALL(get_vars_'itype`)(-999,-999,NULL,NULL,NULL,NULL); EXP_ERR(NC_EBADID) - err=API_ALL(get_varm_'itype`)(-999,-999,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_EBADID) -')') + if (coll_io) { + foreach(`itype',(text, TYPE_LIST),`_CAT(` + err=API_ALL(put_var_'itype`) (-999,-999,NULL); EXP_ERR(NC_EBADID) + err=API_ALL(put_var1_'itype`)(-999,-999,NULL,NULL); EXP_ERR(NC_EBADID) + err=API_ALL(put_vara_'itype`)(-999,-999,NULL,NULL,NULL); EXP_ERR(NC_EBADID) + err=API_ALL(put_vars_'itype`)(-999,-999,NULL,NULL,NULL,NULL); EXP_ERR(NC_EBADID) + err=API_ALL(put_varm_'itype`)(-999,-999,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_EBADID) + err=API_ALL(get_var_'itype`) (-999,-999,NULL); EXP_ERR(NC_EBADID) + err=API_ALL(get_var1_'itype`)(-999,-999,NULL,NULL); EXP_ERR(NC_EBADID) + err=API_ALL(get_vara_'itype`)(-999,-999,NULL,NULL,NULL); EXP_ERR(NC_EBADID) + err=API_ALL(get_vars_'itype`)(-999,-999,NULL,NULL,NULL,NULL); EXP_ERR(NC_EBADID) + err=API_ALL(get_varm_'itype`)(-999,-999,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_EBADID) + ')') + } + else { + foreach(`itype',(text, TYPE_LIST),`_CAT(` + err=API(put_var_'itype`) (-999,-999,NULL); EXP_ERR(NC_EBADID) + err=API(put_var1_'itype`)(-999,-999,NULL,NULL); EXP_ERR(NC_EBADID) + err=API(put_vara_'itype`)(-999,-999,NULL,NULL,NULL); EXP_ERR(NC_EBADID) + err=API(put_vars_'itype`)(-999,-999,NULL,NULL,NULL,NULL); EXP_ERR(NC_EBADID) + err=API(put_varm_'itype`)(-999,-999,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_EBADID) + err=API(get_var_'itype`) (-999,-999,NULL); EXP_ERR(NC_EBADID) + err=API(get_var1_'itype`)(-999,-999,NULL,NULL); EXP_ERR(NC_EBADID) + err=API(get_vara_'itype`)(-999,-999,NULL,NULL,NULL); EXP_ERR(NC_EBADID) + err=API(get_vars_'itype`)(-999,-999,NULL,NULL,NULL,NULL); EXP_ERR(NC_EBADID) + err=API(get_varm_'itype`)(-999,-999,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_EBADID) + ')') + } /* test NC_ENOTVAR */dnl - foreach(`itype',(text, TYPE_LIST),`_CAT(` - err=API_ALL(put_var_'itype`) (ncid,-999,NULL); EXP_ERR(NC_ENOTVAR) - err=API_ALL(put_var1_'itype`)(ncid,-999,NULL,NULL); EXP_ERR(NC_ENOTVAR) - err=API_ALL(put_vara_'itype`)(ncid,-999,NULL,NULL,NULL); EXP_ERR(NC_ENOTVAR) - err=API_ALL(put_vars_'itype`)(ncid,-999,NULL,NULL,NULL,NULL); EXP_ERR(NC_ENOTVAR) - err=API_ALL(put_varm_'itype`)(ncid,-999,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_ENOTVAR) - err=API_ALL(get_var_'itype`) (ncid,-999,NULL); EXP_ERR(NC_ENOTVAR) - err=API_ALL(get_var1_'itype`)(ncid,-999,NULL,NULL); EXP_ERR(NC_ENOTVAR) - err=API_ALL(get_vara_'itype`)(ncid,-999,NULL,NULL,NULL); EXP_ERR(NC_ENOTVAR) - err=API_ALL(get_vars_'itype`)(ncid,-999,NULL,NULL,NULL,NULL); EXP_ERR(NC_ENOTVAR) - err=API_ALL(get_varm_'itype`)(ncid,-999,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_ENOTVAR) -')') + if (coll_io) { + foreach(`itype',(text, TYPE_LIST),`_CAT(` + err=API_ALL(put_var_'itype`) (ncid,-999,NULL); EXP_ERR(NC_ENOTVAR) + err=API_ALL(put_var1_'itype`)(ncid,-999,NULL,NULL); EXP_ERR(NC_ENOTVAR) + err=API_ALL(put_vara_'itype`)(ncid,-999,NULL,NULL,NULL); EXP_ERR(NC_ENOTVAR) + err=API_ALL(put_vars_'itype`)(ncid,-999,NULL,NULL,NULL,NULL); EXP_ERR(NC_ENOTVAR) + err=API_ALL(put_varm_'itype`)(ncid,-999,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_ENOTVAR) + err=API_ALL(get_var_'itype`) (ncid,-999,NULL); EXP_ERR(NC_ENOTVAR) + err=API_ALL(get_var1_'itype`)(ncid,-999,NULL,NULL); EXP_ERR(NC_ENOTVAR) + err=API_ALL(get_vara_'itype`)(ncid,-999,NULL,NULL,NULL); EXP_ERR(NC_ENOTVAR) + err=API_ALL(get_vars_'itype`)(ncid,-999,NULL,NULL,NULL,NULL); EXP_ERR(NC_ENOTVAR) + err=API_ALL(get_varm_'itype`)(ncid,-999,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_ENOTVAR) + ')') + } + else { + foreach(`itype',(text, TYPE_LIST),`_CAT(` + err=API(put_var_'itype`) (ncid,-999,NULL); EXP_ERR(NC_ENOTVAR) + err=API(put_var1_'itype`)(ncid,-999,NULL,NULL); EXP_ERR(NC_ENOTVAR) + err=API(put_vara_'itype`)(ncid,-999,NULL,NULL,NULL); EXP_ERR(NC_ENOTVAR) + err=API(put_vars_'itype`)(ncid,-999,NULL,NULL,NULL,NULL); EXP_ERR(NC_ENOTVAR) + err=API(put_varm_'itype`)(ncid,-999,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_ENOTVAR) + err=API(get_var_'itype`) (ncid,-999,NULL); EXP_ERR(NC_ENOTVAR) + err=API(get_var1_'itype`)(ncid,-999,NULL,NULL); EXP_ERR(NC_ENOTVAR) + err=API(get_vara_'itype`)(ncid,-999,NULL,NULL,NULL); EXP_ERR(NC_ENOTVAR) + err=API(get_vars_'itype`)(ncid,-999,NULL,NULL,NULL,NULL); EXP_ERR(NC_ENOTVAR) + err=API(get_varm_'itype`)(ncid,-999,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_ENOTVAR) + ')') + } /* test NC_EINVALCOORDS */ start[0] = Y_LEN; start[1] = X_LEN; - foreach(`itype',(text, TYPE_LIST),`_CAT(` - err=API_ALL(put_var1_'itype`)(ncid,vid_'itype`,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) - err=API_ALL(put_vara_'itype`)(ncid,vid_'itype`,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) - err=API_ALL(put_vars_'itype`)(ncid,vid_'itype`,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) - err=API_ALL(put_varm_'itype`)(ncid,vid_'itype`,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) - err=API_ALL(put_var1_'itype`)(ncid,vid_'itype`,start,NULL); EXP_ERR(NC_EINVALCOORDS) - err=API_ALL(put_vara_'itype`)(ncid,vid_'itype`,start,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) - err=API_ALL(put_vars_'itype`)(ncid,vid_'itype`,start,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) - err=API_ALL(put_varm_'itype`)(ncid,vid_'itype`,start,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) - err=API_ALL(get_var1_'itype`)(ncid,vid_'itype`,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) - err=API_ALL(get_vara_'itype`)(ncid,vid_'itype`,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) - err=API_ALL(get_vars_'itype`)(ncid,vid_'itype`,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) - err=API_ALL(get_varm_'itype`)(ncid,vid_'itype`,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) - err=API_ALL(get_var1_'itype`)(ncid,vid_'itype`,start,NULL); EXP_ERR(NC_EINVALCOORDS) - err=API_ALL(get_vara_'itype`)(ncid,vid_'itype`,start,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) - err=API_ALL(get_vars_'itype`)(ncid,vid_'itype`,start,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) - err=API_ALL(get_varm_'itype`)(ncid,vid_'itype`,start,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) -')') + if (coll_io) { + foreach(`itype',(text, TYPE_LIST),`_CAT(` + err=API_ALL(put_var1_'itype`)(ncid,vid_'itype`,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API_ALL(put_vara_'itype`)(ncid,vid_'itype`,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API_ALL(put_vars_'itype`)(ncid,vid_'itype`,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API_ALL(put_varm_'itype`)(ncid,vid_'itype`,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API_ALL(put_var1_'itype`)(ncid,vid_'itype`,start,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API_ALL(put_vara_'itype`)(ncid,vid_'itype`,start,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API_ALL(put_vars_'itype`)(ncid,vid_'itype`,start,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API_ALL(put_varm_'itype`)(ncid,vid_'itype`,start,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API_ALL(get_var1_'itype`)(ncid,vid_'itype`,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API_ALL(get_vara_'itype`)(ncid,vid_'itype`,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API_ALL(get_vars_'itype`)(ncid,vid_'itype`,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API_ALL(get_varm_'itype`)(ncid,vid_'itype`,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API_ALL(get_var1_'itype`)(ncid,vid_'itype`,start,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API_ALL(get_vara_'itype`)(ncid,vid_'itype`,start,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API_ALL(get_vars_'itype`)(ncid,vid_'itype`,start,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API_ALL(get_varm_'itype`)(ncid,vid_'itype`,start,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + ')') + } + else { + foreach(`itype',(text, TYPE_LIST),`_CAT(` + err=API(put_var1_'itype`)(ncid,vid_'itype`,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API(put_vara_'itype`)(ncid,vid_'itype`,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API(put_vars_'itype`)(ncid,vid_'itype`,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API(put_varm_'itype`)(ncid,vid_'itype`,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API(put_var1_'itype`)(ncid,vid_'itype`,start,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API(put_vara_'itype`)(ncid,vid_'itype`,start,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API(put_vars_'itype`)(ncid,vid_'itype`,start,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API(put_varm_'itype`)(ncid,vid_'itype`,start,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API(get_var1_'itype`)(ncid,vid_'itype`,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API(get_vara_'itype`)(ncid,vid_'itype`,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API(get_vars_'itype`)(ncid,vid_'itype`,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API(get_varm_'itype`)(ncid,vid_'itype`,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API(get_var1_'itype`)(ncid,vid_'itype`,start,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API(get_vara_'itype`)(ncid,vid_'itype`,start,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API(get_vars_'itype`)(ncid,vid_'itype`,start,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + err=API(get_varm_'itype`)(ncid,vid_'itype`,start,NULL,NULL,NULL,NULL); EXP_ERR(NC_EINVALCOORDS) + ')') + } /* test NC_EEDGE */ start[0] = 0; start[1] = 0; count[0] = Y_LEN; count[1] = X_LEN + 1; - foreach(`itype',(text, TYPE_LIST),`_CAT(` - err=API_ALL(put_vara_'itype`)(ncid,vid_'itype`,start,NULL, NULL); EXP_ERR(NC_EEDGE) - err=API_ALL(put_vars_'itype`)(ncid,vid_'itype`,start,NULL, NULL,NULL); EXP_ERR(NC_EEDGE) - err=API_ALL(put_varm_'itype`)(ncid,vid_'itype`,start,NULL, NULL,NULL,NULL); EXP_ERR(NC_EEDGE) - err=API_ALL(put_vara_'itype`)(ncid,vid_'itype`,start,count,NULL); EXP_ERR(NC_EEDGE) - err=API_ALL(put_vars_'itype`)(ncid,vid_'itype`,start,count,NULL,NULL); EXP_ERR(NC_EEDGE) - err=API_ALL(put_varm_'itype`)(ncid,vid_'itype`,start,count,NULL,NULL,NULL); EXP_ERR(NC_EEDGE) - err=API_ALL(get_vara_'itype`)(ncid,vid_'itype`,start,NULL, NULL); EXP_ERR(NC_EEDGE) - err=API_ALL(get_vars_'itype`)(ncid,vid_'itype`,start,NULL, NULL,NULL); EXP_ERR(NC_EEDGE) - err=API_ALL(get_varm_'itype`)(ncid,vid_'itype`,start,NULL, NULL,NULL,NULL); EXP_ERR(NC_EEDGE) - err=API_ALL(get_vara_'itype`)(ncid,vid_'itype`,start,count,NULL); EXP_ERR(NC_EEDGE) - err=API_ALL(get_vars_'itype`)(ncid,vid_'itype`,start,count,NULL,NULL); EXP_ERR(NC_EEDGE) - err=API_ALL(get_varm_'itype`)(ncid,vid_'itype`,start,count,NULL,NULL,NULL); EXP_ERR(NC_EEDGE) -')') + if (coll_io) { + foreach(`itype',(text, TYPE_LIST),`_CAT(` + err=API_ALL(put_vara_'itype`)(ncid,vid_'itype`,start,NULL, NULL); EXP_ERR(NC_EEDGE) + err=API_ALL(put_vars_'itype`)(ncid,vid_'itype`,start,NULL, NULL,NULL); EXP_ERR(NC_EEDGE) + err=API_ALL(put_varm_'itype`)(ncid,vid_'itype`,start,NULL, NULL,NULL,NULL); EXP_ERR(NC_EEDGE) + err=API_ALL(put_vara_'itype`)(ncid,vid_'itype`,start,count,NULL); EXP_ERR(NC_EEDGE) + err=API_ALL(put_vars_'itype`)(ncid,vid_'itype`,start,count,NULL,NULL); EXP_ERR(NC_EEDGE) + err=API_ALL(put_varm_'itype`)(ncid,vid_'itype`,start,count,NULL,NULL,NULL); EXP_ERR(NC_EEDGE) + err=API_ALL(get_vara_'itype`)(ncid,vid_'itype`,start,NULL, NULL); EXP_ERR(NC_EEDGE) + err=API_ALL(get_vars_'itype`)(ncid,vid_'itype`,start,NULL, NULL,NULL); EXP_ERR(NC_EEDGE) + err=API_ALL(get_varm_'itype`)(ncid,vid_'itype`,start,NULL, NULL,NULL,NULL); EXP_ERR(NC_EEDGE) + err=API_ALL(get_vara_'itype`)(ncid,vid_'itype`,start,count,NULL); EXP_ERR(NC_EEDGE) + err=API_ALL(get_vars_'itype`)(ncid,vid_'itype`,start,count,NULL,NULL); EXP_ERR(NC_EEDGE) + err=API_ALL(get_varm_'itype`)(ncid,vid_'itype`,start,count,NULL,NULL,NULL); EXP_ERR(NC_EEDGE) + ')') + } + else { + foreach(`itype',(text, TYPE_LIST),`_CAT(` + err=API(put_vara_'itype`)(ncid,vid_'itype`,start,NULL, NULL); EXP_ERR(NC_EEDGE) + err=API(put_vars_'itype`)(ncid,vid_'itype`,start,NULL, NULL,NULL); EXP_ERR(NC_EEDGE) + err=API(put_varm_'itype`)(ncid,vid_'itype`,start,NULL, NULL,NULL,NULL); EXP_ERR(NC_EEDGE) + err=API(put_vara_'itype`)(ncid,vid_'itype`,start,count,NULL); EXP_ERR(NC_EEDGE) + err=API(put_vars_'itype`)(ncid,vid_'itype`,start,count,NULL,NULL); EXP_ERR(NC_EEDGE) + err=API(put_varm_'itype`)(ncid,vid_'itype`,start,count,NULL,NULL,NULL); EXP_ERR(NC_EEDGE) + err=API(get_vara_'itype`)(ncid,vid_'itype`,start,NULL, NULL); EXP_ERR(NC_EEDGE) + err=API(get_vars_'itype`)(ncid,vid_'itype`,start,NULL, NULL,NULL); EXP_ERR(NC_EEDGE) + err=API(get_varm_'itype`)(ncid,vid_'itype`,start,NULL, NULL,NULL,NULL); EXP_ERR(NC_EEDGE) + err=API(get_vara_'itype`)(ncid,vid_'itype`,start,count,NULL); EXP_ERR(NC_EEDGE) + err=API(get_vars_'itype`)(ncid,vid_'itype`,start,count,NULL,NULL); EXP_ERR(NC_EEDGE) + err=API(get_varm_'itype`)(ncid,vid_'itype`,start,count,NULL,NULL,NULL); EXP_ERR(NC_EEDGE) + ')') + } /* test NC_ESTRIDE */ start[0] = start[1] = 0; @@ -395,22 +491,38 @@ test_format_nc$1(char *filename) count[1] = X_LEN; stride[0] = -1; stride[1] = -1; - foreach(`itype',(text, TYPE_LIST),`_CAT(` - err=API_ALL(put_vars_'itype`)(ncid,vid_'itype`,start,count,stride,NULL); EXP_ERR(NC_ESTRIDE) - err=API_ALL(put_varm_'itype`)(ncid,vid_'itype`,start,count,stride,NULL,NULL); EXP_ERR(NC_ESTRIDE) - err=API_ALL(get_vars_'itype`)(ncid,vid_'itype`,start,count,stride,NULL); EXP_ERR(NC_ESTRIDE) - err=API_ALL(get_varm_'itype`)(ncid,vid_'itype`,start,count,stride,NULL,NULL); EXP_ERR(NC_ESTRIDE) -')') + if (coll_io) { + foreach(`itype',(text, TYPE_LIST),`_CAT(` + err=API_ALL(put_vars_'itype`)(ncid,vid_'itype`,start,count,stride,NULL); EXP_ERR(NC_ESTRIDE) + err=API_ALL(put_varm_'itype`)(ncid,vid_'itype`,start,count,stride,NULL,NULL); EXP_ERR(NC_ESTRIDE) + err=API_ALL(get_vars_'itype`)(ncid,vid_'itype`,start,count,stride,NULL); EXP_ERR(NC_ESTRIDE) + err=API_ALL(get_varm_'itype`)(ncid,vid_'itype`,start,count,stride,NULL,NULL); EXP_ERR(NC_ESTRIDE) + ')') + } + else { + foreach(`itype',(text, TYPE_LIST),`_CAT(` + err=API(put_vars_'itype`)(ncid,vid_'itype`,start,count,stride,NULL); EXP_ERR(NC_ESTRIDE) + err=API(put_varm_'itype`)(ncid,vid_'itype`,start,count,stride,NULL,NULL); EXP_ERR(NC_ESTRIDE) + err=API(get_vars_'itype`)(ncid,vid_'itype`,start,count,stride,NULL); EXP_ERR(NC_ESTRIDE) + err=API(get_varm_'itype`)(ncid,vid_'itype`,start,count,stride,NULL,NULL); EXP_ERR(NC_ESTRIDE) + ')') + } /* close the file */ err=FileClose(-999); EXP_ERR(NC_EBADID) + + /* file sync before reading */ + err = ncmpi_sync(ncid); + CHECK_ERR + MPI_Barrier(MPI_COMM_WORLD); + err=FileClose(ncid); CHECK_ERR /* open the file with read-only permission */ - err=FileOpen(filename, NC_NOWRITE, &ncid); + err=FileOpen(out_path, NC_NOWRITE, &ncid); if (err != NC_NOERR) { - printf("Error at line %d in %s: FileOpen() file %s (%s)\n", - __LINE__,__FILE__,filename,StrError(err)); + fprintf(stderr,"Error at line %d in %s: FileOpen() file %s (%s)\n", + __LINE__,__FILE__,out_path,StrError(err)); MPI_Abort(MPI_COMM_WORLD, -1); exit(1); } @@ -418,6 +530,11 @@ test_format_nc$1(char *filename) /* test NC_EPERM */ err=ReDef(ncid); EXP_ERR(NC_EPERM) + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + /* test NC_EPERM for attribute APIs */dnl err=API(put_att_text) (-999,-999,NULL,-999,NULL); EXP_ERR(NC_EBADID) err=API(put_att_text) (ncid,-999,NULL,-999,NULL); EXP_ERR(NC_EPERM) @@ -439,12 +556,22 @@ test_format_nc$1(char *filename) err=API(put_att_'itype`)(ncid,vid_'itype`,`"att_'itype`"',NC_TYPE(itype),1,NULL); EXP_ERR(NC_EPERM)')') /* test NC_EPERM */dnl - foreach(`itype',(text, TYPE_LIST),`_CAT(` - err=API_ALL(put_var_'itype`) (ncid,-999,NULL); EXP_ERR(NC_EPERM) - err=API_ALL(put_var1_'itype`)(ncid,-999,NULL,NULL); EXP_ERR(NC_EPERM) - err=API_ALL(put_vara_'itype`)(ncid,-999,NULL,NULL,NULL); EXP_ERR(NC_EPERM) - err=API_ALL(put_vars_'itype`)(ncid,-999,NULL,NULL,NULL,NULL); EXP_ERR(NC_EPERM) - err=API_ALL(put_varm_'itype`)(ncid,-999,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_EPERM)')') + if (coll_io) { + foreach(`itype',(text, TYPE_LIST),`_CAT(` + err=API_ALL(put_var_'itype`) (ncid,-999,NULL); EXP_ERR(NC_EPERM) + err=API_ALL(put_var1_'itype`)(ncid,-999,NULL,NULL); EXP_ERR(NC_EPERM) + err=API_ALL(put_vara_'itype`)(ncid,-999,NULL,NULL,NULL); EXP_ERR(NC_EPERM) + err=API_ALL(put_vars_'itype`)(ncid,-999,NULL,NULL,NULL,NULL); EXP_ERR(NC_EPERM) + err=API_ALL(put_varm_'itype`)(ncid,-999,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_EPERM)')') + } + else { + foreach(`itype',(text, TYPE_LIST),`_CAT(` + err=API(put_var_'itype`) (ncid,-999,NULL); EXP_ERR(NC_EPERM) + err=API(put_var1_'itype`)(ncid,-999,NULL,NULL); EXP_ERR(NC_EPERM) + err=API(put_vara_'itype`)(ncid,-999,NULL,NULL,NULL); EXP_ERR(NC_EPERM) + err=API(put_vars_'itype`)(ncid,-999,NULL,NULL,NULL,NULL); EXP_ERR(NC_EPERM) + err=API(put_varm_'itype`)(ncid,-999,NULL,NULL,NULL,NULL,NULL); EXP_ERR(NC_EPERM)')') + } /* close the file */ err=FileClose(-999); EXP_ERR(NC_EBADID) @@ -468,65 +595,66 @@ test_format_nc$1(char *filename) TEST_FORMAT(1) TEST_FORMAT(2) TEST_FORMAT(5) -#if defined(ENABLE_NETCDF4) || defined(TEST_NETCDF) TEST_FORMAT(3) TEST_FORMAT(4) -#endif -/*----< main() >------------------------------------------------------------*/ -int main(int argc, char **argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) { - char filename[256]; - int rank=0, nerrs=0; + char val[MPI_MAX_INFO_VAL]; + int err, nerrs=0, flag; - MPI_Init(&argc,&argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); + /* check whether burst buffering is enabled */ + MPI_Info_get(info, "nc_burst_buf", MPI_MAX_INFO_VAL - 1, val, &flag); + if (flag && strcasecmp(val, "enable") == 0 && + (format == NC_FORMAT_NETCDF4 || format == NC_FORMAT_NETCDF4_CLASSIC)) + /* does not work for NetCDF4 files when burst-buffering is enabled */ + return 0; - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for error precedence ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } verbose = 0; - /* test all file formats separately */ - nerrs += test_format_nc1(filename); - nerrs += test_format_nc2(filename); -#if defined(ENABLE_NETCDF4) || defined(TEST_NETCDF) - nerrs += test_format_nc3(filename); /* NC_FORMAT_NETCDF4 */ - nerrs += test_format_nc4(filename); /* NC_FORMAT_NETCDF4_CLASSIC */ -#endif - nerrs += test_format_nc5(filename); - -#ifndef TEST_NETCDF - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - int err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + if (format == NC_FORMAT_CLASSIC) + nerrs = test_format_nc1(out_path, coll_io, info); + else if (format == NC_FORMAT_64BIT_OFFSET) + nerrs = test_format_nc2(out_path, coll_io, info); + else if (format == NC_FORMAT_NETCDF4) + nerrs = test_format_nc3(out_path, coll_io, info); + else if (format == NC_FORMAT_NETCDF4_CLASSIC) + nerrs = test_format_nc4(out_path, coll_io, info); + else if (format == NC_FORMAT_64BIT_DATA) + nerrs = test_format_nc5(out_path, coll_io, info); - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } -#endif - MPI_Finalize(); - return (nerrs > 0); + return nerrs; } +int main(int argc, char **argv) { + + int err; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "error precedence", opt, test_io); + + MPI_Finalize(); + + return err; +} diff --git a/test/testcases/file_create_open.c b/test/testcases/file_create_open.c index e4b40e435f..cd3fb61320 100644 --- a/test/testcases/file_create_open.c +++ b/test/testcases/file_create_open.c @@ -10,76 +10,99 @@ #include #include #include -#include /* basename() */ #include #include #include -int main(int argc, char **argv) +static +int file_op(const char *out_path, + MPI_Info info) { - char filename[512]; - int i, err, nprocs, rank, nerrs=0, ncid; - int format[3] = {0, NC_64BIT_OFFSET, NC_64BIT_DATA}; + int err, nerrs=0, ncid; - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); + /* Create a new file */ + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 512, "%s", argv[1]); - else sprintf(filename, "%s.nc", argv[0]); - - if (rank == 0) { - char *cmd_str = (char *)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for file create", basename(argv[0])); - printf("%-66s ------ ", cmd_str); - free(cmd_str); - } + /* Close the file. */ + err = ncmpi_close(ncid); + CHECK_ERR - for (i=0; i<3; i++) { - /* Create a new file */ - int cmode = NC_CLOBBER | format[i]; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); - CHECK_ERR + /* Open the file */ + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_WRITE, info, &ncid); + CHECK_ERR - /* Close the file. */ - err = ncmpi_close(ncid); - CHECK_ERR + /* Close the file. */ + err = ncmpi_close(ncid); + CHECK_ERR - /* Open the file */ - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_WRITE, MPI_INFO_NULL, &ncid); - CHECK_ERR + return (nerrs > 0); +} - /* Close the file. */ - err = ncmpi_close(ncid); - CHECK_ERR - } +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) +{ + int err, nerrs=0; + MPI_Info info_dup=MPI_INFO_NULL; - /* check if there is any malloc residue */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + MPI_Info_dup(info, &info_dup); + + MPI_Info_set(info_dup, "nc_header_align_size", "100"); + MPI_Info_set(info_dup, "nc_var_align_size", "200"); + MPI_Info_set(info_dup, "nc_record_align_size", "300"); + + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + err = file_op(out_path, info_dup); + if (err != 0) nerrs++; + +#ifdef MPICH_VERSION + /* MPICH recognizes file system type acronym prefixed to the file name */ + if (strncmp("ufs:", out_path, 4)) { + char *prefix_fname; + prefix_fname = (char*) malloc(strlen(out_path) + 10); + sprintf(prefix_fname, "ufs:%s", out_path); + + err = file_op(prefix_fname, info_dup); + if (err != 0) nerrs++; - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) - printf(FAIL_STR, nerrs); - else - printf(PASS_STR); + free(prefix_fname); } +#endif - MPI_Finalize(); + if (info_dup != MPI_INFO_NULL) MPI_Info_free(&info_dup); return (nerrs > 0); } + +int main(int argc, char **argv) { + + int err; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ + + err = tst_main(argc, argv, "file create/open", opt, test_io); + + MPI_Finalize(); + + return err; +} + diff --git a/test/testcases/flexible.c b/test/testcases/flexible.c index e1d6b10dc7..767d0834b6 100644 --- a/test/testcases/flexible.c +++ b/test/testcases/flexible.c @@ -42,13 +42,22 @@ #include #define NY 2 -#define NX 5 - -/*----< main() >------------------------------------------------------------*/ -int main(int argc, char **argv) { - - char filename[256]; - int i, j, err, ncid, varid1, varid2, varid3, dimids[2], debug=0; +#define NX 70 + +#define INDEP_MODE 0 +#define COLL_MODE 1 + +static int debug; + +/*----< tst_io() >-----------------------------------------------------------*/ +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) +{ + int i, j, err, ncid, varid1, varid2, varid3, dimids[2]; int rank, nprocs, blocklengths[2], buf[NY][NX], *bufptr; int *ncbuf, req, st, nerrs=0; int array_of_sizes[2], array_of_subsizes[2], array_of_starts[2]; @@ -56,28 +65,15 @@ int main(int argc, char **argv) { MPI_Aint a0, a1, disps[2]; MPI_Datatype buftype; - MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); MPI_Comm_rank(MPI_COMM_WORLD, &rank); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for flexible put and get ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, MPI_INFO_NULL, - &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR /* define a 2D array */ err = ncmpi_def_dim(ncid, "Y", NC_UNLIMITED, &dimids[0]); CHECK_ERR @@ -108,6 +104,11 @@ int main(int argc, char **argv) { ncmpi_sync(ncid); #endif + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + /* initialize the contents of the array */ for (j=0; j 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + return nerrs; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "flexible APIs", opt, test_io); MPI_Finalize(); - return (nerrs > 0); + + return err; } diff --git a/test/testcases/flexible2.c b/test/testcases/flexible2.c index da265fda3d..1bb1c3d909 100644 --- a/test/testcases/flexible2.c +++ b/test/testcases/flexible2.c @@ -7,6 +7,8 @@ /* $Id$ */ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * + * Similar to flexible.c, this program tests APIs with a need of type conversion. * * This program tests PnetCDF flexible APIs, ncmpi_put_vara_all(), * ncmpi_iput_vara() to write two 2D array variables (one is of 4-byte @@ -84,41 +86,32 @@ #define NZ 5 #define NY 5 -#define NX 5 +#define NX 70 -int main(int argc, char** argv) +/*----< tst_io() >-----------------------------------------------------------*/ +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) { - char filename[256]; int i, j, rank, nprocs, err, nerrs=0, req, status, ghost_len=3; - int ncid, cmode, varid0, varid1, dimid[3], *buf_zy, verbose=0; + int ncid, varid0, varid1, dimid[3], *buf_zy, verbose=0; int array_of_sizes[2], array_of_subsizes[2], array_of_starts[2]; double *buf_yx; MPI_Offset start[2], count[2]; - MPI_Datatype subarray; + MPI_Datatype subarray; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for flexible APIs ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR /* create a new file for writing ----------------------------------------*/ - cmode = NC_CLOBBER | NC_64BIT_DATA; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR /* define 3 dimensions */ @@ -132,6 +125,11 @@ int main(int argc, char** argv) err = ncmpi_def_var(ncid, "var_yx", NC_FLOAT, 2, &dimid[1], &varid1); CHECK_ERR err = ncmpi_enddef(ncid); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + /* var_zy is partitioned along Z dimension */ array_of_sizes[0] = NZ + 2*ghost_len; array_of_sizes[1] = NY + 2*ghost_len; @@ -158,20 +156,32 @@ int main(int argc, char** argv) start[0] = NZ * rank; start[1] = 0; count[0] = NZ; count[1] = NY; /* calling a blocking flexible API */ - err = ncmpi_put_vara_all(ncid, varid0, start, count, buf_zy, 1, subarray); + if (coll_io) + err = ncmpi_put_vara_all(ncid, varid0, start, count, buf_zy, 1, subarray); + else + err = ncmpi_put_vara(ncid, varid0, start, count, buf_zy, 1, subarray); CHECK_ERR /* check the contents of put buffer */ for (i=0; i 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } +err_out: + return nerrs; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "flexible API + type conversion", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/testcases/flexible_api.f b/test/testcases/flexible_api.f index 1c7c0990ac..f3f6c196ca 100644 --- a/test/testcases/flexible_api.f +++ b/test/testcases/flexible_api.f @@ -27,8 +27,8 @@ subroutine check(err, message) ! It is a good idea to check returned value for possible error if (err .NE. NF_NOERR) then write(6,*) message(1:XTRIM(message)), nfmpi_strerror(err) - msg = '*** TESTING F77 flexible_api.f for flexible API ' - call pass_fail(1, msg) + msg = '*** TESTING F77 flexible_api.f - flexible API ' + call pass_fail(1, msg, 0) STOP 2 end if end ! subroutine check @@ -38,7 +38,7 @@ program main include "mpif.h" include "pnetcdf.inc" - character(LEN=256) filename, cmd, msg + character(LEN=256) out_path, in_path, cmd, msg integer XTRIM integer err, ierr, nerrs, nprocs, rank, i, j integer cmode, ncid, varid, dimid(2), ghost_len, get_args @@ -50,26 +50,33 @@ program main integer array_of_sizes(2), array_of_subsizes(2) integer array_of_starts(2) integer*8 malloc_size, sum_size - logical verbose + logical verbose, keep_files + double precision timing + + call MPI_Init(ierr) + + timing = MPI_Wtime() - call MPI_Init(err) call MPI_Comm_rank(MPI_COMM_WORLD, rank, err) call MPI_Comm_size(MPI_COMM_WORLD, nprocs, err) - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any if (rank .EQ. 0) then verbose = .TRUE. - filename = "testfile.nc" - ierr = get_args(cmd, filename) + out_path = "testfile.nc" + ierr = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(ierr, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, err) if (ierr .EQ. 0) goto 999 call MPI_Bcast(verbose, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, + err) - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, + MPI_COMM_WORLD, err) + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, + + ierr) + nerrs = 0 ! set parameters @@ -103,7 +110,7 @@ program main ! create file, truncate it if exists cmode = IOR(NF_CLOBBER, NF_64BIT_DATA) - err = nfmpi_create(MPI_COMM_WORLD, filename, cmode, + err = nfmpi_create(MPI_COMM_WORLD, out_path, cmode, + MPI_INFO_NULL, ncid) call check(err, 'In nfmpi_create: ') @@ -291,10 +298,18 @@ program main + sum_size, ' bytes yet to be freed' endif + timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, + + MPI_DOUBLE_PRECISION, MPI_MAX, + + MPI_COMM_WORLD, ierr) if (rank .eq. 0) then + if (.NOT. keep_files) then + err = nfmpi_delete(out_path, MPI_INFO_NULL) + end if + msg = '*** TESTING F77 '//cmd(1:XTRIM(cmd))// - + ' for bufcount=NF_COUNT_IGNORE & buftype predefined' - call pass_fail(nerrs, msg) + + ' - bufcount=NF_COUNT_IGNORE & buftype predefined' + call pass_fail(nerrs, msg, timing) endif 999 call MPI_Finalize(ierr) diff --git a/test/testcases/flexible_large_count.c b/test/testcases/flexible_large_count.c index 9c225518ae..4c65979ce1 100644 --- a/test/testcases/flexible_large_count.c +++ b/test/testcases/flexible_large_count.c @@ -79,7 +79,7 @@ for (i=0; i 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for flexible var APIs ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR /* create a new file for writing ----------------------------------------*/ - cmode = NC_CLOBBER | NC_64BIT_DATA; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); - CHECK_ERROUT + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR /* define 2 dimensions */ - err = ncmpi_def_dim(ncid, "Y", NY, &dimid[0]); CHECK_ERROUT - err = ncmpi_def_dim(ncid, "X", NX, &dimid[1]); CHECK_ERROUT + err = ncmpi_def_dim(ncid, "Y", NY, &dimid[0]); CHECK_ERR + err = ncmpi_def_dim(ncid, "X", NX, &dimid[1]); CHECK_ERR - /* define a variable of size NY * (NX * nprocs) */ - err = ncmpi_def_var(ncid, "var", NC_DOUBLE, 2, dimid, &varid); CHECK_ERROUT - err = ncmpi_enddef(ncid); CHECK_ERROUT + /* define a variable of size NY * NX */ + err = ncmpi_def_var(ncid, "var", NC_DOUBLE, 2, dimid, &varid); CHECK_ERR + err = ncmpi_enddef(ncid); CHECK_ERR - /* var is partitioned along X dimension in a matrix transported way */ + /* var is written in a matrix transported way */ array_of_sizes[0] = NY + 2*GHOST; array_of_sizes[1] = NX + 2*GHOST; array_of_subsizes[0] = NY; @@ -211,7 +200,7 @@ int main(int argc, char** argv) else /* other ranks write 0-sized data */ err = ncmpi_put_vara_all(ncid, varid, start, count, buf_ghost, 0, MPI_INT); MPI_Allreduce(MPI_IN_PLACE, &err, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD); - CHECK_ERROUT + CHECK_ERR /* check the contents of put buffer. They should not be altered. */ CHECK_PUT_BUF_GHOST(NY, NX, GHOST, buf_ghost) @@ -219,7 +208,7 @@ int main(int argc, char** argv) /* read back and check the contents written in the file ----------------*/ INIT_GET_BUF(NY, NX, buf_ghost) err = ncmpi_get_var_all(ncid, varid, buf_ghost, bufcount, subarray); - CHECK_ERROUT + CHECK_ERR /* check the contents of get buffer */ CHECK_GET_BUF_GHOST(NY, NX, GHOST, buf_ghost) @@ -227,9 +216,9 @@ int main(int argc, char** argv) /* read back using a non-blocking flexible API --------------------------*/ INIT_GET_BUF(NY, NX, buf_ghost) err = ncmpi_iget_var(ncid, varid, buf_ghost, bufcount, subarray, &req); - CHECK_ERROUT - err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERROUT - err = status; CHECK_ERROUT + CHECK_ERR + err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR + err = status; CHECK_ERR /* check the contents of get buffer */ CHECK_GET_BUF_GHOST(NY, NX, GHOST, buf_ghost) @@ -240,17 +229,17 @@ int main(int argc, char** argv) /* calling a nonblocking put_var flexible API --------------------------*/ if (rank == 0) { /* only rank 0 writes to the variable */ err = ncmpi_iput_var(ncid, varid, buf_ghost, bufcount, subarray, &req); - CHECK_ERROUT + CHECK_ERR /* check the contents of put buffer. They should not be altered. */ CHECK_PUT_BUF_GHOST(NY, NX, GHOST, buf_ghost) } - err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERROUT - err = status; CHECK_ERROUT + err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR + err = status; CHECK_ERR /* read back and check the contents written in the file ----------------*/ INIT_GET_BUF(NY, NX, buf_ghost) err = ncmpi_get_var_all(ncid, varid, buf_ghost, bufcount, subarray); - CHECK_ERROUT + CHECK_ERR /* check the contents of get buffer */ CHECK_GET_BUF_GHOST(NY, NX, GHOST, buf_ghost) @@ -258,9 +247,9 @@ int main(int argc, char** argv) /* read back using a non-blocking flexible API --------------------------*/ INIT_GET_BUF(NY, NX, buf_ghost) err = ncmpi_iget_var(ncid, varid, buf_ghost, bufcount, subarray, &req); - CHECK_ERROUT - err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERROUT - err = status; CHECK_ERROUT + CHECK_ERR + err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR + err = status; CHECK_ERR /* check the contents of get buffer */ CHECK_GET_BUF_GHOST(NY, NX, GHOST, buf_ghost) @@ -279,7 +268,7 @@ int main(int argc, char** argv) else /* other ranks write 0-sized data */ err = ncmpi_put_vara_all(ncid, varid, start, count, buf, 0, MPI_INT); MPI_Allreduce(MPI_IN_PLACE, &err, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD); - CHECK_ERROUT + CHECK_ERR /* check the contents of put buffer. They should not be altered. */ CHECK_PUT_BUF(NY, NX, buf) @@ -287,7 +276,7 @@ int main(int argc, char** argv) /* read back and check the contents written in the file ----------------*/ INIT_GET_BUF(NY, NX, buf) err = ncmpi_get_var_all(ncid, varid, buf, bufcount, MPI_INT); - CHECK_ERROUT + CHECK_ERR /* check the contents of get buffer */ CHECK_GET_BUF(NY, NX, buf) @@ -295,9 +284,9 @@ int main(int argc, char** argv) /* read back using a non-blocking flexible API --------------------------*/ INIT_GET_BUF(NY, NX, buf) err = ncmpi_iget_var(ncid, varid, buf, bufcount, MPI_INT, &req); - CHECK_ERROUT - err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERROUT - err = status; CHECK_ERROUT + CHECK_ERR + err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR + err = status; CHECK_ERR /* check the contents of get buffer */ CHECK_GET_BUF(NY, NX, buf) @@ -308,17 +297,17 @@ int main(int argc, char** argv) /* calling a nonblocking put_var flexible API --------------------------*/ if (rank == 0) { /* only rank 0 writes to the variable */ err = ncmpi_iput_var(ncid, varid, buf, bufcount, MPI_INT, &req); - CHECK_ERROUT + CHECK_ERR /* check the contents of put buffer. They should not be altered. */ CHECK_PUT_BUF(NY, NX, buf) } - err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERROUT - err = status; CHECK_ERROUT + err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR + err = status; CHECK_ERR /* read back and check the contents written in the file ----------------*/ INIT_GET_BUF(NY, NX, buf) err = ncmpi_get_var_all(ncid, varid, buf, bufcount, MPI_INT); - CHECK_ERROUT + CHECK_ERR /* check the contents of get buffer */ CHECK_GET_BUF(NY, NX, buf) @@ -326,18 +315,23 @@ int main(int argc, char** argv) /* read back using a non-blocking flexible API --------------------------*/ INIT_GET_BUF(NY, NX, buf) err = ncmpi_iget_var(ncid, varid, buf, bufcount, MPI_INT, &req); - CHECK_ERROUT - err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERROUT - err = status; CHECK_ERROUT + CHECK_ERR + err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR + err = status; CHECK_ERR /* check the contents of get buffer */ CHECK_GET_BUF(NY, NX, buf) + /* file sync before reading */ + err = ncmpi_sync(ncid); + CHECK_ERR + MPI_Barrier(MPI_COMM_WORLD); + /*----------------------------------------------------------------------*/ /*---- test independent I/O mode ---------------------------------------*/ /*----------------------------------------------------------------------*/ err = ncmpi_begin_indep_data(ncid); - CHECK_ERROUT + CHECK_ERR /*----------------------------------------------------------------------*/ /*---- test using bufcount == 1 with ghost cells -----------------------*/ @@ -350,19 +344,19 @@ int main(int argc, char** argv) /* calling a blocking put_var flexible API -----------------------------*/ if (rank == 0) { /* only rank 0 writes to the variable */ err = ncmpi_put_var(ncid, varid, buf_ghost, bufcount, subarray); - CHECK_ERROUT + CHECK_ERR /* check the contents of put buffer. They should not be altered. */ CHECK_PUT_BUF_GHOST(NY, NX, GHOST, buf_ghost) } /* file sync is required for non-zero ranks to see the data in file */ err = ncmpi_sync(ncid); - CHECK_ERROUT + CHECK_ERR /* read back and check the contents written in the file ----------------*/ INIT_GET_BUF(NY, NX, buf_ghost) err = ncmpi_get_var(ncid, varid, buf_ghost, bufcount, subarray); - CHECK_ERROUT + CHECK_ERR /* check the contents of get buffer */ CHECK_GET_BUF_GHOST(NY, NX, GHOST, buf_ghost) @@ -370,9 +364,9 @@ int main(int argc, char** argv) /* read back using a non-blocking flexible API --------------------------*/ INIT_GET_BUF(NY, NX, buf_ghost) err = ncmpi_iget_var(ncid, varid, buf_ghost, bufcount, subarray, &req); - CHECK_ERROUT - err = ncmpi_wait(ncid, 1, &req, &status); CHECK_ERROUT - err = status; CHECK_ERROUT + CHECK_ERR + err = ncmpi_wait(ncid, 1, &req, &status); CHECK_ERR + err = status; CHECK_ERR /* check the contents of get buffer */ CHECK_GET_BUF_GHOST(NY, NX, GHOST, buf_ghost) @@ -383,17 +377,17 @@ int main(int argc, char** argv) /* calling a nonblocking put_var flexible API --------------------------*/ if (rank == 0) { /* only rank 0 writes to the variable */ err = ncmpi_iput_var(ncid, varid, buf_ghost, bufcount, subarray, &req); - CHECK_ERROUT + CHECK_ERR /* check the contents of put buffer. They should not be altered. */ CHECK_PUT_BUF_GHOST(NY, NX, GHOST, buf_ghost) } - err = ncmpi_wait(ncid, 1, &req, &status); CHECK_ERROUT - err = status; CHECK_ERROUT + err = ncmpi_wait(ncid, 1, &req, &status); CHECK_ERR + err = status; CHECK_ERR /* read back and check the contents written in the file ----------------*/ INIT_GET_BUF(NY, NX, buf_ghost) err = ncmpi_get_var(ncid, varid, buf_ghost, bufcount, subarray); - CHECK_ERROUT + CHECK_ERR /* check the contents of get buffer */ CHECK_GET_BUF_GHOST(NY, NX, GHOST, buf_ghost) @@ -401,9 +395,9 @@ int main(int argc, char** argv) /* read back using a non-blocking flexible API --------------------------*/ INIT_GET_BUF(NY, NX, buf_ghost) err = ncmpi_iget_var(ncid, varid, buf_ghost, bufcount, subarray, &req); - CHECK_ERROUT - err = ncmpi_wait(ncid, 1, &req, &status); CHECK_ERROUT - err = status; CHECK_ERROUT + CHECK_ERR + err = ncmpi_wait(ncid, 1, &req, &status); CHECK_ERR + err = status; CHECK_ERR /* check the contents of get buffer */ CHECK_GET_BUF_GHOST(NY, NX, GHOST, buf_ghost) @@ -419,19 +413,19 @@ int main(int argc, char** argv) /* calling a blocking put_var flexible API -----------------------------*/ if (rank == 0) { /* only rank 0 writes to the variable */ err = ncmpi_put_var(ncid, varid, buf, bufcount, MPI_INT); - CHECK_ERROUT + CHECK_ERR /* check the contents of put buffer. They should not be altered. */ CHECK_PUT_BUF(NY, NX, buf) } /* file sync is required for non-zero ranks to see the data in file */ err = ncmpi_sync(ncid); - CHECK_ERROUT + CHECK_ERR /* read back and check the contents written in the file ----------------*/ INIT_GET_BUF(NY, NX, buf) err = ncmpi_get_var(ncid, varid, buf, bufcount, MPI_INT); - CHECK_ERROUT + CHECK_ERR /* check the contents of get buffer */ CHECK_GET_BUF(NY, NX, buf) @@ -439,9 +433,9 @@ int main(int argc, char** argv) /* read back using a non-blocking flexible API --------------------------*/ INIT_GET_BUF(NY, NX, buf) err = ncmpi_iget_var(ncid, varid, buf, bufcount, MPI_INT, &req); - CHECK_ERROUT - err = ncmpi_wait(ncid, 1, &req, &status); CHECK_ERROUT - err = status; CHECK_ERROUT + CHECK_ERR + err = ncmpi_wait(ncid, 1, &req, &status); CHECK_ERR + err = status; CHECK_ERR /* check the contents of get buffer */ CHECK_GET_BUF(NY, NX, buf) @@ -452,17 +446,17 @@ int main(int argc, char** argv) /* calling a nonblocking put_var flexible API --------------------------*/ if (rank == 0) { /* only rank 0 writes to the variable */ err = ncmpi_iput_var(ncid, varid, buf, bufcount, MPI_INT, &req); - CHECK_ERROUT + CHECK_ERR /* check the contents of put buffer. They should not be altered. */ CHECK_PUT_BUF(NY, NX, buf) } - err = ncmpi_wait(ncid, 1, &req, &status); CHECK_ERROUT - err = status; CHECK_ERROUT + err = ncmpi_wait(ncid, 1, &req, &status); CHECK_ERR + err = status; CHECK_ERR /* read back and check the contents written in the file ----------------*/ INIT_GET_BUF(NY, NX, buf) err = ncmpi_get_var(ncid, varid, buf, bufcount, MPI_INT); - CHECK_ERROUT + CHECK_ERR /* check the contents of get buffer */ CHECK_GET_BUF(NY, NX, buf) @@ -470,36 +464,41 @@ int main(int argc, char** argv) /* read back using a non-blocking flexible API --------------------------*/ INIT_GET_BUF(NY, NX, buf) err = ncmpi_iget_var(ncid, varid, buf, bufcount, MPI_INT, &req); - CHECK_ERROUT - err = ncmpi_wait(ncid, 1, &req, &status); CHECK_ERROUT - err = status; CHECK_ERROUT + CHECK_ERR + err = ncmpi_wait(ncid, 1, &req, &status); CHECK_ERR + err = status; CHECK_ERR /* check the contents of get buffer */ CHECK_GET_BUF(NY, NX, buf) MPI_Type_free(&subarray); - err = ncmpi_close(ncid); CHECK_ERROUT - - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + err = ncmpi_close(ncid); CHECK_ERR -err_out: - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + return nerrs; +} + +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "flexible var APIs", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/testcases/flexible_var.c b/test/testcases/flexible_var.c index ceb712eb29..b861156028 100644 --- a/test/testcases/flexible_var.c +++ b/test/testcases/flexible_var.c @@ -53,8 +53,12 @@ #include +/* Smaller NY and NX are for demonstration, producing outputs as shown above. #define NY 6 #define NX 4 +*/ +#define NY 32 +#define NX 128 #define GHOST 2 #define PRINT_PUT_BUF(Y, X, buf) { \ @@ -83,7 +87,7 @@ for (i=0; i 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for flexible var APIs ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - /* create a new file for writing ----------------------------------------*/ - cmode = NC_CLOBBER | NC_64BIT_DATA; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERROUT /* define 2 dimensions */ @@ -485,25 +472,61 @@ int main(int argc, char** argv) err = ncmpi_close(ncid); CHECK_ERROUT - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } - err_out: - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } - - MPI_Finalize(); return (nerrs > 0); } +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) +{ + int err, nerrs=0; + MPI_Info info_dup; + + MPI_Info_dup(info, &info_dup); + + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + nerrs = tst_io(out_path, info_dup); + if (nerrs > 0) return nerrs; + + /* disable PnetCDF internal buffering */ + MPI_Info_set(info_dup, "nc_ibuf_size", "0"); + + nerrs = tst_io(out_path, info_dup); + if (nerrs > 0) return nerrs; + + MPI_Info_free(&info_dup); + + return nerrs; +} + +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "flexible var APIs", opt, test_io); + + MPI_Finalize(); + + return err; +} diff --git a/test/testcases/flexible_varm.c b/test/testcases/flexible_varm.c index 03cb2ce161..87f7787076 100644 --- a/test/testcases/flexible_varm.c +++ b/test/testcases/flexible_varm.c @@ -21,6 +21,9 @@ * * % mpiexec -l -n 4 ./flexible_varm /pvfs2/wkliao/testfile.nc * + * When NY and NX are set to 6 and 4 respectively, the contents of output file + * are shown below. + * * % ncmpidump /pvfs2/wkliao/testfile.nc * netcdf testfile { * // file format: CDF-5 (big variables) @@ -52,8 +55,8 @@ #include -#define NY 6 -#define NX 4 +#define NY 32 +#define NX 128 #define GHOST 2 #define INIT_PUT_BUF \ @@ -72,17 +75,21 @@ for (j=0; j 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for flexible varm APIs ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR /* create a new file for writing ----------------------------------------*/ - cmode = NC_CLOBBER | NC_64BIT_DATA; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR /* define 2 dimensions */ @@ -157,6 +161,11 @@ int main(int argc, char** argv) err = ncmpi_def_var(ncid, "var", NC_DOUBLE, 2, dimid, &varid); CHECK_ERR err = ncmpi_enddef(ncid); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + start[0] = 0; start[1] = NX * rank; count[0] = NY; count[1] = NX; stride[0] = 1; stride[1] = 1; @@ -176,7 +185,11 @@ int main(int argc, char** argv) /* calling a blocking put_varm flexible API -----------------------------*/ /* initiate put buffer contents */ INIT_PUT_BUF - err = ncmpi_put_varm_all(ncid, varid, start, count, stride, imap, buf, + if (coll_io) + err = ncmpi_put_varm_all(ncid, varid, start, count, stride, imap, buf, + 1, subarray); + else + err = ncmpi_put_varm(ncid, varid, start, count, stride, imap, buf, 1, subarray); CHECK_ERR @@ -189,18 +202,35 @@ int main(int argc, char** argv) err = ncmpi_iput_varm(ncid, varid, start, count, stride, imap, buf, 1, subarray, &req); CHECK_ERR - err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR + + if (coll_io) + err = ncmpi_wait_all(ncid, 1, &req, &status); + else + err = ncmpi_wait(ncid, 1, &req, &status); + CHECK_ERR err = status; CHECK_ERR /* check the contents of put buffer */ CHECK_PUT_BUF + /* When running in independent data mode, flushing writes is necessary + * before reading the data back. + */ + MPI_Barrier(MPI_COMM_WORLD); + err = ncmpi_sync(ncid); + CHECK_ERR + MPI_Barrier(MPI_COMM_WORLD); + /* read back using a blocking get_varm flexible API ---------------------*/ /* initiate get buffer contents */ INIT_GET_BUF /* calling a blocking flexible API */ - err = ncmpi_get_varm_all(ncid, varid, start, count, stride, imap, buf, + if (coll_io) + err = ncmpi_get_varm_all(ncid, varid, start, count, stride, imap, buf, + 1, subarray); + else + err = ncmpi_get_varm(ncid, varid, start, count, stride, imap, buf, 1, subarray); CHECK_ERR @@ -215,34 +245,68 @@ int main(int argc, char** argv) err = ncmpi_iget_varm(ncid, varid, start, count, stride, imap, buf, 1, subarray, &req); CHECK_ERR - err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR + if (coll_io) + err = ncmpi_wait_all(ncid, 1, &req, &status); + else + err = ncmpi_wait(ncid, 1, &req, &status); + CHECK_ERR err = status; CHECK_ERR /* check the contents of get buffer */ CHECK_GET_BUF - MPI_Type_free(&subarray); - +err_out: err = ncmpi_close(ncid); CHECK_ERR - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + if (subarray != MPI_DATATYPE_NULL) MPI_Type_free(&subarray); - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + return nerrs; +} - MPI_Finalize(); - return (nerrs > 0); +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) +{ + int nerrs=0; + + nerrs = tst_varm(out_path, in_path, format, coll_io, info); + if (nerrs > 0) goto err_out; + + /* disable PnetCDF internal buffering */ + MPI_Info_set(info, "nc_ibuf_size", "0"); + + nerrs = tst_varm(out_path, in_path, format, coll_io, info); + if (nerrs > 0) goto err_out; + +err_out: + return nerrs; } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "flexible varm APIs", opt, test_io); + + MPI_Finalize(); + + return err; +} diff --git a/test/testcases/inq_num_vars.c b/test/testcases/inq_num_vars.c index 6cc8f44d4d..381651abb9 100644 --- a/test/testcases/inq_num_vars.c +++ b/test/testcases/inq_num_vars.c @@ -33,9 +33,9 @@ static int check_num_vars(int ncid, - int expected_nvars, - int expected_num_rec_vars, - int expected_num_fix_vars) + int expected_nvars, + int expected_num_rec_vars, + int expected_num_fix_vars) { int err, nerrs=0, nvars, num_rec_vars, num_fix_vars; @@ -49,32 +49,46 @@ int check_num_vars(int ncid, err = ncmpi_inq_num_fix_vars(ncid, &num_fix_vars); CHECK_ERR if (nvars != expected_nvars) { - printf("Error at line %d in %s: expecting %d number of variables defined, but got %d\n", + fprintf(stderr,"Error at line %d in %s: expecting %d number of variables defined, but got %d\n", __LINE__,__FILE__,expected_nvars, nvars); nerrs++; } if (num_rec_vars != expected_num_rec_vars) { - printf("Error at line %d in %s: expecting %d number of record variables defined, but got %d\n", + fprintf(stderr,"Error at line %d in %s: expecting %d number of record variables defined, but got %d\n", __LINE__,__FILE__,expected_num_rec_vars, num_rec_vars); nerrs++; } if (num_fix_vars != expected_num_fix_vars) { - printf("Error at line %d in %s: expecting %d number of fixed-size variables defined, but got %d\n", + fprintf(stderr,"Error at line %d in %s: expecting %d number of fixed-size variables defined, but got %d\n", __LINE__,__FILE__,expected_num_fix_vars, num_fix_vars); nerrs++; } return nerrs; } -static int -tst_fmt(char *filename, int cmode) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) { - int nerrs=0, err, ncid, varid[7], dimid[3]; - MPI_Info info=MPI_INFO_NULL; + char val[MPI_MAX_INFO_VAL]; + int nerrs=0, err, flag, ncid, varid[7], dimid[3]; + + /* check whether burst buffering is enabled */ + MPI_Info_get(info, "nc_burst_buf", MPI_MAX_INFO_VAL - 1, val, &flag); + if (flag && strcasecmp(val, "enable") == 0 && + (format == NC_FORMAT_NETCDF4 || format == NC_FORMAT_NETCDF4_CLASSIC)) + /* does not work for NetCDF4 files when burst-buffering is enabled */ + return 0; + + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR /* create a new file for writing ----------------------------------------*/ - cmode |= NC_CLOBBER; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR /* define dimension and variable */ err = ncmpi_def_dim(ncid, "REC_DIM", NC_UNLIMITED, &dimid[0]); CHECK_ERR @@ -109,10 +123,15 @@ tst_fmt(char *filename, int cmode) nerrs += check_num_vars(ncid, 7, 4, 3); + /* file sync before reading */ + err = ncmpi_sync(ncid); + CHECK_ERR + MPI_Barrier(MPI_COMM_WORLD); + err = ncmpi_close(ncid); CHECK_ERR /* open the file for reading --------------------------------------------*/ - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_NOWRITE, info, &ncid); CHECK_ERR nerrs += check_num_vars(ncid, 7, 4, 3); @@ -122,64 +141,26 @@ tst_fmt(char *filename, int cmode) return nerrs; } -int main(int argc, char** argv) { - char filename[256], *hint_value; - int nerrs=0, rank, err, bb_enabled=0; - - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); +int main(int argc, char **argv) { - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - goto fn_exit; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for no. record/fixed variables", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } + int err; + loop_opts opt; - /* printf("PnetCDF version string: \"%s\"\n", ncmpi_inq_libvers()); */ + MPI_Init(&argc, &argv); - /* check whether burst buffering is enabled */ - if (inq_env_hint("nc_burst_buf", &hint_value)) { - if (strcasecmp(hint_value, "enable") == 0) bb_enabled = 1; - free(hint_value); - } + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ - nerrs += tst_fmt(filename, 0); - nerrs += tst_fmt(filename, NC_64BIT_OFFSET); - if (!bb_enabled) { -#ifdef ENABLE_NETCDF4 - nerrs += tst_fmt(filename, NC_NETCDF4); - nerrs += tst_fmt(filename, NC_NETCDF4 | NC_CLASSIC_MODEL); -#endif - } - nerrs += tst_fmt(filename, NC_64BIT_DATA); - - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + err = tst_main(argc, argv, "number of variables", opt, test_io); - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } - -fn_exit: MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/testcases/inq_num_varsf.f90 b/test/testcases/inq_num_varsf.f90 index 85c689aa95..7ba9385d73 100644 --- a/test/testcases/inq_num_varsf.f90 +++ b/test/testcases/inq_num_varsf.f90 @@ -46,8 +46,8 @@ subroutine check(err, message) ! It is a good idea to check returned value for possible error if (err .NE. NF90_NOERR) then write(6,*) trim(message), trim(nf90mpi_strerror(err)) - msg = '*** TESTING F90 inq_num_varsf.f90 for no. record/fixed variables' - call pass_fail(1, msg) + msg = '*** TESTING F90 inq_num_varsf.f90 - no. record/fixed variables' + call pass_fail(1, msg, 0) call MPI_Abort(MPI_COMM_WORLD, -1, err) end if end subroutine check @@ -57,29 +57,36 @@ program main use pnetcdf implicit none - character(LEN=256) filename, cmd, msg + character(LEN=256) out_path, in_path, cmd, msg integer err, ierr, nprocs, rank, cmode, ncid, get_args integer varid(7), dimid(3), dimid_1D(1), dimid_2D(2) integer nerrs, nvars, num_rec_vars, num_fix_vars, old_mode integer(kind=MPI_OFFSET_KIND) malloc_size, sum_size + logical keep_files + double precision timing call MPI_Init(ierr) + + timing = MPI_Wtime() + call MPI_Comm_rank(MPI_COMM_WORLD, rank, ierr) call MPI_Comm_size(MPI_COMM_WORLD, nprocs, ierr) - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any if (rank .EQ. 0) then - filename = "testfile.nc" - err = get_args(cmd, filename) + out_path = "testfile.nc" + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr) ! create file, truncate it if exists cmode = IOR(NF90_CLOBBER, NF90_64BIT_OFFSET) - err = nf90mpi_create(MPI_COMM_WORLD, filename, cmode, & + err = nf90mpi_create(MPI_COMM_WORLD, out_path, cmode, & MPI_INFO_NULL, ncid) call check(err, 'In nf90mpi_create: ') @@ -152,7 +159,7 @@ program main call check(err, 'In nf90mpi_close: ') ! open the file just now created - err = nf90mpi_open(MPI_COMM_WORLD, filename, NF90_NOWRITE, & + err = nf90mpi_open(MPI_COMM_WORLD, out_path, NF90_NOWRITE, & MPI_INFO_NULL, ncid) call check(err, 'In nf90mpi_open: ') @@ -193,10 +200,18 @@ program main sum_size, ' bytes yet to be freed' endif + timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, & + MPI_DOUBLE_PRECISION, MPI_MAX, & + MPI_COMM_WORLD, ierr) if (rank .eq. 0) then + if (.NOT. keep_files) then + err = nfmpi_delete(out_path, MPI_INFO_NULL) + end if + msg = '*** TESTING F90 '//trim(cmd)// & - ' for no. record/fixed variables' - call pass_fail(nerrs, msg) + ' - no. record/fixed variables' + call pass_fail(nerrs, msg, timing) endif 999 call MPI_Finalize(ierr) diff --git a/test/testcases/inq_recsize.c b/test/testcases/inq_recsize.c index fec23ab2b9..72f97e7f71 100644 --- a/test/testcases/inq_recsize.c +++ b/test/testcases/inq_recsize.c @@ -30,17 +30,30 @@ #include -static int -tst_fmt(char *filename, int cmode) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) { - int nerrs=0, err; - int ncid, varid[7], dimid[3]; + char val[MPI_MAX_INFO_VAL]; + int nerrs=0, err, flag, ncid, varid[7], dimid[3]; MPI_Offset expected_recsize, recsize; - MPI_Info info=MPI_INFO_NULL; + + /* check whether burst buffering is enabled */ + MPI_Info_get(info, "nc_burst_buf", MPI_MAX_INFO_VAL - 1, val, &flag); + if (flag && strcasecmp(val, "enable") == 0 && + (format == NC_FORMAT_NETCDF4 || format == NC_FORMAT_NETCDF4_CLASSIC)) + /* does not work for NetCDF4 files when burst-buffering is enabled */ + return 0; + + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR /* create a new file for writing ----------------------------------------*/ - cmode |= NC_CLOBBER; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR /* define dimension and variable */ err = ncmpi_def_dim(ncid, "REC_DIM", NC_UNLIMITED, &dimid[0]); CHECK_ERR @@ -75,7 +88,7 @@ tst_fmt(char *filename, int cmode) err = ncmpi_inq_recsize(ncid, &recsize); CHECK_ERR if (expected_recsize != recsize) { - printf("Error at line %d in %s: expecting record size "OFFFMT" but got "OFFFMT"\n", + fprintf(stderr,"Error at line %d in %s: expecting record size "OFFFMT" but got "OFFFMT"\n", __LINE__,__FILE__,expected_recsize, recsize); nerrs++; } @@ -85,64 +98,26 @@ tst_fmt(char *filename, int cmode) return nerrs; } -int main(int argc, char** argv) { - char filename[256], *hint_value; - int nerrs=0, rank, err, bb_enabled=0; - - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - goto fn_exit; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for inquiring record size ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } +int main(int argc, char **argv) { - /* printf("PnetCDF version string: \"%s\"\n", ncmpi_inq_libvers()); */ + int err; + loop_opts opt; - /* check whether burst buffering is enabled */ - if (inq_env_hint("nc_burst_buf", &hint_value)) { - if (strcasecmp(hint_value, "enable") == 0) bb_enabled = 1; - free(hint_value); - } + MPI_Init(&argc, &argv); - nerrs += tst_fmt(filename, 0); - nerrs += tst_fmt(filename, NC_64BIT_OFFSET); - if (!bb_enabled) { -#ifdef ENABLE_NETCDF4 - nerrs += tst_fmt(filename, NC_NETCDF4); - nerrs += tst_fmt(filename, NC_NETCDF4 | NC_CLASSIC_MODEL); -#endif - } - nerrs += tst_fmt(filename, NC_64BIT_DATA); - - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + err = tst_main(argc, argv, "inquiring record size", opt, test_io); -fn_exit: MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/testcases/inq_recsizef.f90 b/test/testcases/inq_recsizef.f90 index 208b04df2f..8c7e37f9c5 100644 --- a/test/testcases/inq_recsizef.f90 +++ b/test/testcases/inq_recsizef.f90 @@ -29,8 +29,8 @@ subroutine check(err, message) ! It is a good idea to check returned value for possible error if (err .NE. NF90_NOERR) then write(6,*) trim(message), trim(nf90mpi_strerror(err)) - msg = '*** TESTING F90 inq_recsizef.f90 for inquiring record size' - call pass_fail(1, msg) + msg = '*** TESTING F90 inq_recsizef.f90 - inquiring record size' + call pass_fail(1, msg, 0) call MPI_Abort(MPI_COMM_WORLD, -1, err) end if end subroutine check @@ -40,29 +40,36 @@ program main use pnetcdf implicit none - character(LEN=256) filename, cmd, msg + character(LEN=256) out_path, in_path, cmd, msg integer err, ierr, nprocs, rank, cmode, ncid, nerrs, get_args integer varid(7), dimid(3), dimid_1D(1), dimid_2D(2), old_mode integer(kind=MPI_OFFSET_KIND) expected_recsize, recsize integer(kind=MPI_OFFSET_KIND) malloc_size, sum_size + logical keep_files + double precision timing call MPI_Init(ierr) + + timing = MPI_Wtime() + call MPI_Comm_rank(MPI_COMM_WORLD, rank, ierr) call MPI_Comm_size(MPI_COMM_WORLD, nprocs, ierr) - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any if (rank .EQ. 0) then - filename = "testfile.nc" - err = get_args(cmd, filename) + out_path = "testfile.nc" + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr) ! create file, truncate it if exists cmode = IOR(NF90_CLOBBER, NF90_64BIT_OFFSET) - err = nf90mpi_create(MPI_COMM_WORLD, filename, cmode, & + err = nf90mpi_create(MPI_COMM_WORLD, out_path, cmode, & MPI_INFO_NULL, ncid) call check(err, 'In nf90mpi_create: ') @@ -138,9 +145,17 @@ program main sum_size, ' bytes yet to be freed' endif + timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, & + MPI_DOUBLE_PRECISION, MPI_MAX, & + MPI_COMM_WORLD, ierr) if (rank .eq. 0) then - msg = '*** TESTING F90 '//trim(cmd)//' for inquiring record size' - call pass_fail(nerrs, msg) + if (.NOT. keep_files) then + err = nfmpi_delete(out_path, MPI_INFO_NULL) + end if + + msg = '*** TESTING F90 '//trim(cmd)//' - inquiring record size' + call pass_fail(nerrs, msg, timing) endif 999 call MPI_Finalize(ierr) diff --git a/test/testcases/iput_all_kinds.m4 b/test/testcases/iput_all_kinds.m4 index 81be7b3447..b4f31f10d2 100644 --- a/test/testcases/iput_all_kinds.m4 +++ b/test/testcases/iput_all_kinds.m4 @@ -9,7 +9,6 @@ dnl * See COPYRIGHT notice in top-level directory. * *********************************************************************/ -/* $Id: transpose.c 3078 2017-05-29 22:46:50Z wkliao $ */ #include #include @@ -21,7 +20,9 @@ dnl #include #define NDIMS 3 -#define LEN 2 +#define LEN 13 + +static int debug; include(`foreach.m4')dnl include(`utils.m4')dnl @@ -46,12 +47,153 @@ include(`utils.m4')dnl #define ulonglong unsigned long long #endif +#define INIT_BUF(buf, len) { \ + for (i=0; i YXZ */ + dimidsT[0] = dimids[1]; dimidsT[1] = dimids[2]; dimidsT[2] = dimids[0]; + err = ncmpi_def_var(ncid, "varm_$1", NC_TYPE($1), NDIMS, dimidsT, &varm_id); + CHECK_ERR + + return err; +} +')dnl + +foreach(`itype',(`text,schar,uchar,short,ushort,int,uint,long,float,double,longlong,ulonglong'),`DEFINE_VARS(itype)') + define(`TEST_NON_BLOCKING_PUT',dnl `dnl static int non_blocking_put_$1(int rank, + int coll_io, + int ncid, + MPI_Offset *gsize, + MPI_Offset *start, + MPI_Offset *count, + MPI_Offset *startS, + MPI_Offset *countS, + MPI_Offset *stride, + MPI_Offset *startM, + MPI_Offset *countM, + MPI_Offset *imap) +{ + int i, err=NC_NOERR, exp; + int var1_id, vara_id, vars_id, varm_id; + MPI_Offset start1[1]; + size_t bufsize, bufsizeS, bufsizeM; + + err = ncmpi_inq_varid(ncid, "var1_$1", &var1_id); CHECK_ERR + err = ncmpi_inq_varid(ncid, "vara_$1", &vara_id); CHECK_ERR + err = ncmpi_inq_varid(ncid, "vars_$1", &vars_id); CHECK_ERR + err = ncmpi_inq_varid(ncid, "varm_$1", &varm_id); CHECK_ERR + + bufsize = bufsizeS = bufsizeM = 1; + for (i=0; i YXZ */ - dimidsT[0] = dimids[1]; dimidsT[1] = dimids[2]; dimidsT[2] = dimids[0]; - err = ncmpi_def_var(ncid, "varm_$1", NC_TYPE($1), NDIMS, dimidsT, &varm_id); CHECK_ERR + MALLOC_ITYPE($1, *buf1, 1) + MALLOC_ITYPE($1, *bufa, bufsize) + MALLOC_ITYPE($1, *bufs, bufsizeS) + MALLOC_ITYPE($1, *bufm, bufsizeM) - /* exit the define mode */ - err = ncmpi_enddef(ncid); CHECK_ERR + ZERO_OUT_BUF(buf1, 1) + ZERO_OUT_BUF(bufa, bufsize) + ZERO_OUT_BUF(bufs, bufsizeS) + ZERO_OUT_BUF(bufm, bufsizeM) - /* write the whole variable in parallel */ + err = ncmpi_inq_varid(ncid, "var1_$1", &var1_id); CHECK_ERR + err = ncmpi_inq_varid(ncid, "vara_$1", &vara_id); CHECK_ERR + err = ncmpi_inq_varid(ncid, "vars_$1", &vars_id); CHECK_ERR + err = ncmpi_inq_varid(ncid, "varm_$1", &varm_id); CHECK_ERR + + /* write the variable in parallel */ start1[0] = rank; - err = `ncmpi_iput_var1_'ifelse(`$1',`text',`$1',`double')(ncid, var1_id, start1, buf, NULL); CHECK_ERR + err = `ncmpi_iget_var1_'$1(ncid, var1_id, start1, buf1, NULL); CHECK_ERR + + err = `ncmpi_iget_vara_'$1(ncid, vara_id, start, count, bufa, NULL); CHECK_ERR + + err = `ncmpi_iget_vars_'$1(ncid, vars_id, startS, countS, stride, bufs, NULL); CHECK_ERR + + err = `ncmpi_iget_varm_'$1(ncid, varm_id, startM, countM, NULL, imap, bufm, NULL); CHECK_ERR - err = `ncmpi_iput_vara_'ifelse(`$1',`text',`$1',`double')(ncid, vara_id, start, count, buf, NULL); CHECK_ERR + /* commit all nonblocking requests */ + if (!coll_io) + err = ncmpi_wait(ncid, NC_REQ_ALL, NULL, NULL); + else + err = ncmpi_wait_all(ncid, NC_REQ_ALL, NULL, NULL); + CHECK_ERR - err = `ncmpi_iput_vars_'ifelse(`$1',`text',`$1',`double')(ncid, vars_id, startS, countS, stride, buf, NULL); CHECK_ERR + /* Check read contents */ + exp = (rank + 1) % 128; + if (buf1[0] != exp) { + fprintf(stderr,"Error %s at %d: buf1 expects %.f but got %.f\n", + __func__,__LINE__, (float)exp, (float)buf1[0]); + CHECK_ERR + } + for (i=0; i------------------------------------------------------------*/ -int main(int argc, char **argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) { - char filename[256], fname[512], *cbuf; - int i, j, k, rank, nprocs, ncid, bufsize, err, nerrs=0, cmode; + int i, rank, nprocs, ncid, err=NC_NOERR; int psize[NDIMS], dimids[NDIMS], dim_rank[NDIMS]; - double *buf; MPI_Offset _nprocs; MPI_Offset gsize[NDIMS], stride[NDIMS], imap[NDIMS]; MPI_Offset start[NDIMS], count[NDIMS]; MPI_Offset startS[NDIMS], countS[NDIMS]; MPI_Offset startM[NDIMS], countM[NDIMS]; - MPI_Info info; - MPI_Init(&argc,&argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for all kinds put APIs ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - /* calculate number of processes along each dimension */ for (i=0; i YXZ: (this is borrowed from examples/C/transpose.c */ imap[1] = 1; imap[0] = count[2]; imap[2] = count[1]*count[2]; startM[0] = start[1]; startM[1] = start[2]; startM[2] = start[0]; countM[0] = count[1]; countM[1] = count[2]; countM[2] = count[0]; + if (debug) + printf("startM=%lld %lld %lld countM=%lld %lld %lld stride=%lld %lld %lld imap=%lld %lld %lld\n", + start[0],start[1],start[2],count[0],count[1],count[2], + stride[0],stride[1],stride[2], imap[0],imap[1],imap[2]); - /* test CDF-1, 2, and 5 formats separately */ - TEST_CDF_FORMAT(NC_FORMAT_CLASSIC) - TEST_CDF_FORMAT(NC_FORMAT_64BIT_OFFSET) - TEST_CDF_FORMAT(NC_FORMAT_64BIT_DATA) - - free(cbuf); - free(buf); - MPI_Info_free(&info); - - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + MPI_Info_set(info, "nc_var_align_size", "1"); - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR - MPI_Finalize(); - return (nerrs > 0); + TEST_CDF_FORMAT_PUT + TEST_CDF_FORMAT_GET + + return err; } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "all kinds iput APIs", opt, test_io); + + MPI_Finalize(); + + return err; +} diff --git a/test/testcases/ivarn.c b/test/testcases/ivarn.c index abb818e064..532a17b058 100644 --- a/test/testcases/ivarn.c +++ b/test/testcases/ivarn.c @@ -87,7 +87,7 @@ int check_int_buf(int *buffer, int lineno) else if (nprocs == 3) { if (i == 12) break; } if (buffer[i] != expected[i]) { - printf("Error at line %d: expected read buf[%d]=%d, but got %d\n", + fprintf(stderr,"Error at line %d: expected read buf[%d]=%d, but got %d\n", lineno,i,expected[i],buffer[i]); return 1; } @@ -116,7 +116,7 @@ int check_flt_buf(float *buffer, float extra, int lineno) else if (nprocs == 3) { if (i == 12) break; } if (buffer[i] != expected[i]) { - printf("Error at line %d: expected read buf[%d]=%.1f, but got %.1f\n", + fprintf(stderr,"Error at line %d: expected read buf[%d]=%.1f, but got %.1f\n", lineno,i,expected[i],buffer[i]); return 1; } @@ -145,7 +145,7 @@ int check_dbl_buf(double *buffer, double extra, int lineno) else if (nprocs == 3) { if (i == 12) break; } if (buffer[i] != expected[i]) { - printf("Error at line %d: expected read buf[%d]=%.1f, but got %.1f\n", + fprintf(stderr,"Error at line %d: expected read buf[%d]=%.1f, but got %.1f\n", lineno,i,expected[i],buffer[i]); return 1; } @@ -153,11 +153,15 @@ int check_dbl_buf(double *buffer, double extra, int lineno) return 0; } -int main(int argc, char** argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) { - char filename[256]; int i, rank, nprocs, err, nerrs=0; - int ncid, cmode, dimid[2]; + int ncid, dimid[2]; int vari0001, vari0002, varr0001, varr0002, vard0001, vard0002; MPI_Offset **starts, **counts; int req[LEN], st[LEN], num_reqs=0; @@ -165,34 +169,20 @@ int main(int argc, char** argv) float rbuf1[LEN], rbuf2[LEN]; double dbuf1[LEN], dbuf2[LEN]; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for ncmpi_iput_varn_() ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - #ifdef DEBUG if (nprocs != 4 && rank == 0) printf("Warning: %s is intended to run on 4 processes\n",argv[0]); #endif + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + /* create a new file for writing ----------------------------------------*/ - cmode = NC_CLOBBER | NC_64BIT_DATA; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "dim000001", LEN, &dimid[1]); CHECK_ERR err = ncmpi_def_dim(ncid, "time", NC_UNLIMITED, &dimid[0]); CHECK_ERR @@ -223,6 +213,11 @@ int main(int argc, char** argv) err = ncmpi_fill_var_rec(ncid, vard0002, 0); CHECK_ERR } + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + #ifdef STRONGER_CONSISTENCY ncmpi_sync(ncid); MPI_Barrier(MPI_COMM_WORLD); @@ -406,11 +401,20 @@ int main(int argc, char** argv) */ } - err = ncmpi_wait_all(ncid, num_reqs, req, st); CHECK_ERR + if (coll_io) + err = ncmpi_wait_all(ncid, num_reqs, req, st); + else + err = ncmpi_wait(ncid, num_reqs, req, st); + CHECK_ERR for (i=0; i 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + return nerrs; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "ncmpi_iput_varn_()", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/testcases/large_var_cdf5.c b/test/testcases/large_var_cdf5.c index 0c39afca0d..930f07a453 100644 --- a/test/testcases/large_var_cdf5.c +++ b/test/testcases/large_var_cdf5.c @@ -28,33 +28,20 @@ #include -int main(int argc, char** argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) { - char filename[256]; - int rank, nprocs, err, nerrs=0, ncid, dimid[2], varid[2]; + int err, nerrs=0, ncid, dimid[2], varid[2]; - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for large var in CDF-5", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER|NC_64BIT_DATA, - MPI_INFO_NULL, &ncid); CHECK_ERR + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "dim0", NC_UNLIMITED, &dimid[0]); CHECK_ERR err = ncmpi_def_dim(ncid, "dim1", NC_MAX_INT64, &dimid[1]); CHECK_ERR @@ -67,24 +54,30 @@ int main(int argc, char** argv) err = ncmpi_set_fill(ncid, NC_NOFILL, NULL); CHECK_ERR err = ncmpi_close(ncid); CHECK_ERR - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } - - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + return nerrs; +} + +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ + + err = tst_main(argc, argv, "large var in CDF-5", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/testcases/last_large_var.c b/test/testcases/last_large_var.c index 458b9cf07a..dd5fd592cb 100644 --- a/test/testcases/last_large_var.c +++ b/test/testcases/last_large_var.c @@ -56,14 +56,14 @@ #define CHECK_ERR { \ if (err != NC_NOERR) { \ nerrs++; \ - printf("Error at line %d in %s: (%s)\n", \ + fprintf(stderr,"Error at line %d in %s: (%s)\n", \ __LINE__,__FILE__,nc_strerror(err)); \ } \ } #define EXP_ERR(exp) { \ if (err != exp) { \ nerrs++; \ - printf("Error at line %d in %s: expecting %d but got %d\n", \ + fprintf(stderr,"Error at line %d in %s: expecting %d but got %d\n", \ __LINE__,__FILE__,exp, err); \ } \ } @@ -94,13 +94,13 @@ #endif static -int check_last_var(char *filename) +int check_last_var(const char *filename, MPI_Info info) { int err, nerrs=0, ncid, cmode, fill_mode, varid, dimid[4]; /* create a new file ---------------------------------------------------*/ cmode = NC_CLOBBER; - err = FileCreate(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + err = FileCreate(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR err = DefDim(ncid, "Y", NC_UNLIMITED, &dimid[0]); CHECK_ERR @@ -120,13 +120,13 @@ int check_last_var(char *filename) } static -int check_fix_var(char *filename) +int check_fix_var(const char *filename, MPI_Info info) { int err, nerrs=0, ncid, cmode, fill_mode, varid, dimid[4]; /* create a new CDF-1 file ----------------------------------------------*/ cmode = NC_CLOBBER; - err = FileCreate(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + err = FileCreate(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR err = DefDim(ncid, "X", 536870911, &dimid[0]); CHECK_ERR @@ -144,7 +144,7 @@ int check_fix_var(char *filename) /* create a new CDF-2 file ----------------------------------------------*/ cmode = NC_CLOBBER | NC_64BIT_OFFSET; - err = FileCreate(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + err = FileCreate(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR err = DefDim(ncid, "X", 536870911, &dimid[0]); CHECK_ERR @@ -165,13 +165,13 @@ int check_fix_var(char *filename) } static -int check_fix_rec_var(char *filename) +int check_fix_rec_var(const char *filename, MPI_Info info) { int err, nerrs=0, ncid, cmode, fill_mode, varid, dimid[4]; /* create a new file ---------------------------------------------------*/ cmode = NC_CLOBBER; - err = FileCreate(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + err = FileCreate(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR err = DefDim(ncid, "Y", NC_UNLIMITED, &dimid[0]); CHECK_ERR @@ -196,13 +196,13 @@ int check_fix_rec_var(char *filename) * record is less than 2 GiB - 4. */ static -int check_rec_var(char *filename, int cmode) +int check_rec_var(const char *filename, int cmode, MPI_Info info) { int err, nerrs=0, ncid, fill_mode, varid, dimid[3]; /* create a new file ---------------------------------------------------*/ cmode |= NC_CLOBBER; - err = FileCreate(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + err = FileCreate(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR err = DefDim(ncid, "Z", NC_UNLIMITED, &dimid[0]); CHECK_ERR @@ -218,7 +218,7 @@ int check_rec_var(char *filename, int cmode) /* create a new file ---------------------------------------------------*/ cmode |= NC_CLOBBER; - err = FileCreate(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + err = FileCreate(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR err = DefDim(ncid, "Z", NC_UNLIMITED, &dimid[0]); CHECK_ERR @@ -234,7 +234,7 @@ int check_rec_var(char *filename, int cmode) /* create a new file ---------------------------------------------------*/ cmode |= NC_CLOBBER; - err = FileCreate(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + err = FileCreate(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR err = DefDim(ncid, "Z", NC_UNLIMITED, &dimid[0]); CHECK_ERR @@ -262,13 +262,13 @@ int check_rec_var(char *filename, int cmode) * this variable must be less than about 2 GiB. */ static -int check_not_last_var(char *filename) +int check_not_last_var(const char *filename, MPI_Info info) { int err, nerrs=0, ncid, cmode, fill_mode, varid, dimid[4]; /* create a new file ---------------------------------------------------*/ cmode = NC_CLOBBER; - err = FileCreate(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + err = FileCreate(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR err = DefDim(ncid, "Y", NC_UNLIMITED, &dimid[0]); CHECK_ERR @@ -288,13 +288,13 @@ int check_not_last_var(char *filename) } static -int check_add_var(char *filename) +int check_add_var(const char *filename, MPI_Info info) { int err, nerrs=0, ncid, cmode, fill_mode, varid, dimid[4]; /* create a new file ---------------------------------------------------*/ cmode = NC_CLOBBER; - err = FileCreate(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + err = FileCreate(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR err = DefDim(ncid, "Y", NC_UNLIMITED, &dimid[0]); CHECK_ERR @@ -320,13 +320,13 @@ int check_add_var(char *filename) } static -int check_var_offset(char *filename) +int check_var_offset(const char *filename, MPI_Info info) { int err, nerrs=0, ncid, cmode, fill_mode, varid, dimid[4]; /* create a new file ---------------------------------------------------*/ cmode = NC_CLOBBER; - err = FileCreate(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + err = FileCreate(MPI_COMM_WORLD, filename, cmode, info, &ncid); CHECK_ERR err = DefDim(ncid, "Y", NC_UNLIMITED, &dimid[0]); CHECK_ERR @@ -359,64 +359,57 @@ int check_var_offset(char *filename) return nerrs; } -int main(int argc, char** argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, /* ignored */ + int coll_io, /* ignored */ + MPI_Info info) { - char filename[256]; - int rank=0, nprocs=1, err, nerrs=0; + int nerrs=0; - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for last large var in CDF-1/2", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - - nerrs += check_fix_var(filename); - nerrs += check_last_var(filename); - nerrs += check_fix_rec_var(filename); - nerrs += check_rec_var(filename, 0); - nerrs += check_rec_var(filename, NC_64BIT_OFFSET); - nerrs += check_rec_var(filename, NC_64BIT_DATA); - nerrs += check_not_last_var(filename); - nerrs += check_add_var(filename); - nerrs += check_var_offset(filename); + nerrs += check_fix_var(out_path, info); + nerrs += check_last_var(out_path, info); + nerrs += check_fix_rec_var(out_path, info); + + nerrs += check_rec_var(out_path, 0, info); + nerrs += check_rec_var(out_path, NC_64BIT_OFFSET, info); + nerrs += check_rec_var(out_path, NC_64BIT_DATA, info); + + nerrs += check_not_last_var(out_path, info); + nerrs += check_add_var(out_path, info); + nerrs += check_var_offset(out_path, info); #ifdef TEST_NETCDF if (nerrs) printf("fail with %d mismatches\n",nerrs); else printf("pass\n"); -#else - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } - - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } #endif - MPI_Finalize(); - return (nerrs > 0); + return nerrs; } +int main(int argc, char **argv) { + + int err; + int formats[] = {0}; + + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ + + err = tst_main(argc, argv, "last large var in CDF-1/2", opt, test_io); + + MPI_Finalize(); + + return err; +} diff --git a/test/testcases/mix_collectives.c b/test/testcases/mix_collectives.c index 583a295997..30ed7847c3 100644 --- a/test/testcases/mix_collectives.c +++ b/test/testcases/mix_collectives.c @@ -13,10 +13,11 @@ #include -/*----< main() >------------------------------------------------------------*/ -int main(int argc, char **argv) +static +int mix_collectives(const char *out_path, + int format, + MPI_Info info) { - char filename[256]; int i, j, err, nerrs=0, rank, nprocs; int ncid, dimid[2], varid, varids[4]; MPI_Offset start[2], count[2], stride[2], imap[2]; @@ -35,32 +36,19 @@ int main(int argc, char **argv) 204, NC_FILL_INT, 205, NC_FILL_INT, 304, 310, 316, 322, NC_FILL_INT, NC_FILL_INT, NC_FILL_INT, NC_FILL_INT, 305, 311, 317, 323}; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for get/put varm ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - #ifdef DEBUG if (nprocs > 1 && rank == 0) printf("Warning: %s is designed to run on 1 process\n", argv[0]); #endif - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER | NC_64BIT_DATA, - MPI_INFO_NULL, &ncid); CHECK_ERR + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR /* define a variable of a 6 x 4 integer array in the nc file */ err = ncmpi_def_dim(ncid, "Y", 12, &dimid[0]); CHECK_ERR @@ -165,7 +153,7 @@ int main(int argc, char **argv) err = ncmpi_close(ncid); CHECK_ERR - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_WRITE, MPI_INFO_NULL, &ncid); CHECK_ERR + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_WRITE, info, &ncid); CHECK_ERR err = ncmpi_inq_varid(ncid, "var", &varid); CHECK_ERR @@ -177,11 +165,11 @@ int main(int argc, char **argv) if (nprocs == 4) { for (i=0; i<96; i++) { if (check_buf[i] != g_buf[i]) { - printf("Error at line %d in %s: expecting var[%d]=%d but got %d\n", + fprintf(stderr,"Error at line %d in %s: expecting var[%d]=%d but got %d\n", __LINE__,__FILE__,i,g_buf[i],check_buf[i]); nerrs++; free(check_buf); - goto err_out; + goto syn_err; } } } @@ -193,7 +181,7 @@ int main(int argc, char **argv) start[0] = 1; start[1] = 1; err = ncmpi_get_var1_int_all(ncid, varid, start, &buf[0][0]); CHECK_ERR if (buf[0][0] != rank) { - printf("Error at line %d in %s: expecting buf[0][0]=%d but got %d\n", + fprintf(stderr,"Error at line %d in %s: expecting buf[0][0]=%d but got %d\n", __LINE__,__FILE__,rank,buf[0][0]); nerrs++; } @@ -204,10 +192,10 @@ int main(int argc, char **argv) err = ncmpi_get_vara_int_all(ncid, varid, start, count, &buf[0][0]); CHECK_ERR for (j=0; j<6; j++) for (i=0; i<4; i++) { if (buf[j][i] != j*4+i + rank*100) { - printf("Error at line %d in %s: expecting var[%d]=%d but got %d\n", + fprintf(stderr,"Error at line %d in %s: expecting var[%d]=%d but got %d\n", __LINE__,__FILE__,i, j*4+i + rank*100, buf[j][i]); nerrs++; - goto err_out; + goto syn_err; } } } @@ -219,10 +207,10 @@ int main(int argc, char **argv) int *val = &buf[0][0]; for (j=0; j 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + return nerrs; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) +{ + int nerrs; - MPI_Finalize(); - return (nerrs > 0); + /* test without setting hint nc_data_move_chunk_size */ + nerrs = mix_collectives(out_path, format, info); + if (nerrs > 0) return nerrs; + + /* test with setting hint nc_data_move_chunk_size */ + MPI_Info_set(info, "nc_data_move_chunk_size", "100"); + nerrs = mix_collectives(out_path, format, info); + if (nerrs > 0) return nerrs; + + return 0; } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "mixed kind collective APIs", opt, test_io); + + MPI_Finalize(); + + return err; +} diff --git a/test/testcases/modes.c b/test/testcases/modes.c index 23d04099e7..73e973a410 100644 --- a/test/testcases/modes.c +++ b/test/testcases/modes.c @@ -28,12 +28,16 @@ #define EXPECT_ERR2(err_no1, err_no2) \ if (err != err_no1 && err != err_no2) { \ nerrs++; \ - printf("Error at line %d in %s: expect error code %s or %s but got %s\n", \ + fprintf(stderr,"Error at line %d in %s: expect error code %s or %s but got %s\n", \ __LINE__,__FILE__,ncmpi_strerrno(err_no1),ncmpi_strerrno(err_no2),ncmpi_strerrno(err)); \ } static -int check_modes(char *filename) +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, /* ignored */ + int coll_io, /* ignored */ + MPI_Info info) { int rank, err, nerrs=0, ncid, cmode; char *path; @@ -43,27 +47,32 @@ int check_modes(char *filename) /* delete the file and ignore error */ /* remove the file system type prefix name if there is any. For example, - * when filename = "lustre:/home/foo/testfile.nc", remove "lustre:" to make + * when out_path = "lustre:/home/foo/testfile.nc", remove "lustre:" to make * path pointing to "/home/foo/testfile.nc", so it can be used in POSIX * unlink() and access() below */ - path = remove_file_system_type_prefix(filename); + path = remove_file_system_type_prefix(out_path); + + MPI_Barrier(MPI_COMM_WORLD); if (rank == 0) unlink(path); + MPI_Barrier(MPI_COMM_WORLD); /* create a new file and test various cmodes ----------------------------*/ /* It is illegal to use both NC_64BIT_OFFSET and NC_64BIT_DATA together */ cmode = NC_CLOBBER | NC_64BIT_OFFSET | NC_64BIT_DATA; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + err = ncmpi_create(MPI_COMM_WORLD, out_path, cmode, info, &ncid); EXP_ERR(NC_EINVAL_CMODE) + MPI_Barrier(MPI_COMM_WORLD); + /* The file should not be created */ if (rank == 0) { if (access(path, F_OK) == 0) { - printf("Error at %s:%d : file (%s) should not be created\n", - __FILE__,__LINE__, filename); + fprintf(stderr,"Error at %s:%d : file (%s) should not be created\n", + __FILE__,__LINE__, out_path); nerrs++; /* delete the file and ignore error */ unlink(path); @@ -72,47 +81,61 @@ int check_modes(char *filename) } MPI_Barrier(MPI_COMM_WORLD); -#ifdef ENABLE_NETCDF4 - /* It is illegal to use both NC_64BIT_OFFSET and NC_NETCDF4 together */ - cmode = NC_CLOBBER | NC_64BIT_OFFSET | NC_NETCDF4; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); - EXP_ERR(NC_EINVAL_CMODE) + if (format == NC_FORMAT_NETCDF4 || format == NC_FORMAT_NETCDF4_CLASSIC) { + /* It is illegal to use both NC_64BIT_OFFSET and NC_NETCDF4 together */ + if (format == NC_FORMAT_NETCDF4_CLASSIC) + cmode = NC_CLOBBER | NC_64BIT_OFFSET | NC_NETCDF4 | NC_CLASSIC_MODEL; + else + cmode = NC_CLOBBER | NC_64BIT_OFFSET | NC_NETCDF4; - /* The file should not be created */ - if (rank == 0) { - if (access(path, F_OK) == 0) { - printf("Error at %s:%d : file (%s) should not be created\n", - __FILE__,__LINE__, filename); - nerrs++; - /* delete the file and ignore error */ - unlink(path); + err = ncmpi_create(MPI_COMM_WORLD, out_path, cmode, info, &ncid); + EXP_ERR(NC_EINVAL_CMODE) + + MPI_Barrier(MPI_COMM_WORLD); + + /* The file should not be created */ + if (rank == 0) { + if (access(path, F_OK) == 0) { + fprintf(stderr,"Error at %s:%d : file (%s) should not be created\n", + __FILE__,__LINE__, out_path); + nerrs++; + /* delete the file and ignore error */ + unlink(path); + } + /* else : file does not exist */ } - /* else : file does not exist */ - } - MPI_Barrier(MPI_COMM_WORLD); + MPI_Barrier(MPI_COMM_WORLD); - /* It is illegal to use both NC_64BIT_DATA and NC_NETCDF4 together */ - cmode = NC_CLOBBER | NC_64BIT_DATA | NC_NETCDF4; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); - EXP_ERR(NC_EINVAL_CMODE) + /* It is illegal to use both NC_64BIT_DATA and NC_NETCDF4 together */ + if (format == NC_FORMAT_NETCDF4_CLASSIC) + cmode = NC_CLOBBER | NC_64BIT_DATA | NC_NETCDF4 | NC_CLASSIC_MODEL; + else + cmode = NC_CLOBBER | NC_64BIT_DATA | NC_NETCDF4; - /* The file should not be created */ - if (rank == 0) { - if (access(path, F_OK) == 0) { - printf("Error at %s:%d : file (%s) should not be created\n", - __FILE__,__LINE__, filename); - nerrs++; - /* delete the file and ignore error */ - unlink(path); + err = ncmpi_create(MPI_COMM_WORLD, out_path, cmode, info, &ncid); + EXP_ERR(NC_EINVAL_CMODE) + + MPI_Barrier(MPI_COMM_WORLD); + + /* The file should not be created */ + if (rank == 0) { + if (access(path, F_OK) == 0) { + fprintf(stderr,"Error at %s:%d : file (%s) should not be created\n", + __FILE__,__LINE__, out_path); + nerrs++; + /* delete the file and ignore error */ + unlink(path); + } + /* else : file does not exist */ } - /* else : file does not exist */ + MPI_Barrier(MPI_COMM_WORLD); } - MPI_Barrier(MPI_COMM_WORLD); -#endif /* Collectively opening a non-existing file for read, expect error code * NC_ENOENT on all processes */ - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_NOWRITE, info, &ncid); + + MPI_Barrier(MPI_COMM_WORLD); /* When using MVAPICH2 2.2, its Lustre driver adds O_CREAT to all open * calls. This is considered a bug in an MPI-IO implementation. Due to this @@ -130,8 +153,8 @@ int check_modes(char *filename) /* The file should not be created */ if (rank == 0) { if (access(path, F_OK) == 0) { - printf("Error at line %d in %s: file (%s) should not be created\n", - __LINE__,__FILE__, filename); + fprintf(stderr,"Error at line %d in %s: file (%s) should not be created\n", + __LINE__,__FILE__, out_path); nerrs++; /* delete the file and ignore error */ unlink(path); @@ -143,7 +166,9 @@ int check_modes(char *filename) /* Collectively opening a non-existing file for write, expect error code * NC_ENOENT on all processes */ - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_WRITE, MPI_INFO_NULL, &ncid); + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_WRITE, info, &ncid); + + MPI_Barrier(MPI_COMM_WORLD); /* When using MVAPICH2 2.2, its Lustre driver adds O_CREAT to all open * calls. This is considered a bug in an MPI-IO implementation. Due to this @@ -161,8 +186,8 @@ int check_modes(char *filename) /* The file should not be created */ if (rank == 0) { if (access(path, F_OK) == 0) { - printf("Error at line %d in %s: file (%s) should not be created\n", - __LINE__,__FILE__, filename); + fprintf(stderr,"Error at line %d in %s: file (%s) should not be created\n", + __LINE__,__FILE__, out_path); nerrs++; /* delete the file and ignore error */ unlink(path); @@ -172,66 +197,33 @@ int check_modes(char *filename) } MPI_Barrier(MPI_COMM_WORLD); - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, MPI_INFO_NULL, &ncid); + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR err = ncmpi_close(ncid); CHECK_ERR return nerrs; } -int main(int argc, char** argv) -{ - char *filename=NULL; - int len, rank, err, nerrs=0; - - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); +int main(int argc, char **argv) { - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) filename = strdup(argv[1]); - else filename = strdup("testfile.nc"); - len = (int)strlen(filename) + 1; - MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD); - MPI_Bcast(filename, len, MPI_CHAR, 0, MPI_COMM_WORLD); + int err; + loop_opts opt; - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for file create/open modes ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } + MPI_Init(&argc, &argv); - /* test under safe mode enabled */ - setenv("PNETCDF_SAFE_MODE", "1", 1); - nerrs += check_modes(filename); - - /* test under safe mode disabled */ - setenv("PNETCDF_SAFE_MODE", "0", 1); - nerrs += check_modes(filename); - - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = false;/* skip ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } - free(filename); + err = tst_main(argc, argv, "file create/open modes", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/testcases/ncmpi_vars_null_stride.c b/test/testcases/ncmpi_vars_null_stride.c index 0cfd81aacd..ac0e936170 100644 --- a/test/testcases/ncmpi_vars_null_stride.c +++ b/test/testcases/ncmpi_vars_null_stride.c @@ -18,9 +18,10 @@ /* check if user put buffer contents altered */ #define CHECK_PUT_BUF \ for (i=0; i 0); } -int main(int argc, char **argv) -{ - char filename[256], *hint_value; - int rank, err, nerrs=0, bb_enabled=0; - - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); +int main(int argc, char **argv) { - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for NULL stride ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } + int err; + loop_opts opt; - /* check whether burst buffering is enabled */ - if (inq_env_hint("nc_burst_buf", &hint_value)) { - if (strcasecmp(hint_value, "enable") == 0) bb_enabled = 1; - free(hint_value); - } + MPI_Init(&argc, &argv); - nerrs += tst_fmt(filename, 0); - if (nerrs) goto fn_exit; - nerrs += tst_fmt(filename, NC_64BIT_OFFSET); - if (nerrs) goto fn_exit; - if (!bb_enabled) { -#ifdef ENABLE_NETCDF4 - nerrs += tst_fmt(filename, NC_NETCDF4); - if (nerrs) goto fn_exit; - nerrs += tst_fmt(filename, NC_NETCDF4 | NC_CLASSIC_MODEL); - if (nerrs) goto fn_exit; -#endif - } - nerrs += tst_fmt(filename, NC_64BIT_DATA); - if (nerrs) goto fn_exit; - - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ -fn_exit: - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + err = tst_main(argc, argv, "NULL stride", opt, test_io); MPI_Finalize(); - return (nerrs > 0); + + return err; } + diff --git a/test/testcases/noclobber.c b/test/testcases/noclobber.c index 23f17598b9..c5935cec37 100644 --- a/test/testcases/noclobber.c +++ b/test/testcases/noclobber.c @@ -20,86 +20,63 @@ #include -static int -tst_fmt(char *filename, int fmt_flag) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) { - int err, nerrs=0, ncid, cmode; + int err, nerrs=0, ncid; MPI_Barrier(MPI_COMM_WORLD); + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + /* create a file if it does not exist */ - cmode = NC_CLOBBER | fmt_flag; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR + + err = ncmpi_enddef(ncid); CHECK_ERR + + /* file sync before reading */ + err = ncmpi_sync(ncid); + CHECK_ERR + MPI_Barrier(MPI_COMM_WORLD); + err = ncmpi_close(ncid); CHECK_ERR /* now the file exists, test if PnetCDF can return correct error code */ - cmode = NC_NOCLOBBER | fmt_flag; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_NOCLOBBER, info, &ncid); EXP_ERR(NC_EEXIST) /* err == NC_EOFILE */ + MPI_Barrier(MPI_COMM_WORLD); + return nerrs; } int main(int argc, char **argv) { - char *filename, *hint_value; - int err, nerrs=0, len, rank, bb_enabled=0; + + int err; + loop_opts opt; MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) filename = strdup(argv[1]); - else filename = strdup("testfile.nc"); - len = (int)strlen(filename) + 1; - MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD); - MPI_Bcast(filename, len, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for NC_NOCLOBBER and NC_EEXIST ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - - /* check whether burst buffering is enabled */ - if (inq_env_hint("nc_burst_buf", &hint_value)) { - if (strcasecmp(hint_value, "enable") == 0) bb_enabled = 1; - free(hint_value); - } - - nerrs += tst_fmt(filename, 0); - nerrs += tst_fmt(filename, NC_64BIT_OFFSET); - if (!bb_enabled) { -#ifdef ENABLE_NETCDF4 - nerrs += tst_fmt(filename, NC_NETCDF4); - nerrs += tst_fmt(filename, NC_NETCDF4|NC_CLASSIC_MODEL); -#endif - } - nerrs += tst_fmt(filename, NC_64BIT_DATA); - - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } - - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } - free(filename); + + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "NC_NOCLOBBER and NC_EEXIST", opt, test_io); MPI_Finalize(); - return (nerrs > 0); + + return err; } diff --git a/test/testcases/nonblocking.c b/test/testcases/nonblocking.c index 169d7f45b2..8589241660 100644 --- a/test/testcases/nonblocking.c +++ b/test/testcases/nonblocking.c @@ -1,31 +1,44 @@ /* * Copyright (C) 2013, Northwestern University and Argonne National Laboratory * See COPYRIGHT notice in top-level directory. - * - * $Id$ */ /* * This program tests the use of nonblocking API. * The write buffer is a 2D array of size NY x NX - * It writes the 2nd row of the memory buffer to the 1st row of the variable - * array in file. Then it writes the 1st row of the memory buffer to the - * 2nd row of the variable array in file. + * It writes the NY-th row of the memory buffer to the 1st row of the variable + * array in file and (NY-1)-th row of the memory buffer to the 2nd row of the + * variable in the file, and so one. Testing for reading is the opposite of + * write. * - * The expected reults from the output file contents are: - * (when running on 1 MPI process) + The expected reults from the output file contents are: * + * % mpiexec -n 4 ./nonblocking * % ncmpidump testfile.nc * netcdf testfile { * // file format: CDF-1 * dimensions: - * Y = UNLIMITED ; // (2 currently) + * Y = UNLIMITED ; // (16 currently) * X = 5 ; * variables: * int VAR(Y, X) ; * data: * * var = + * 3, 3, 3, 3, 3, + * 2, 2, 2, 2, 2, + * 1, 1, 1, 1, 1, + * 0, 0, 0, 0, 0, + * 3, 3, 3, 3, 3, + * 2, 2, 2, 2, 2, + * 1, 1, 1, 1, 1, + * 0, 0, 0, 0, 0, + * 3, 3, 3, 3, 3, + * 2, 2, 2, 2, 2, + * 1, 1, 1, 1, 1, + * 0, 0, 0, 0, 0, + * 3, 3, 3, 3, 3, + * 2, 2, 2, 2, 2, * 1, 1, 1, 1, 1, * 0, 0, 0, 0, 0 ; * } @@ -43,44 +56,24 @@ #define NY 4 #define NX 5 -/*----< main() >------------------------------------------------------------*/ -int main(int argc, char **argv) { - - char filename[256]; - int i, j, err, ncid, varid, dimids[2], req[2], st[2], nerrs=0; - int rank, nprocs, buf[NY+1][NX]; +int tst_iput(const char *out_path, + int format, + int coll_io, + MPI_Info info) +{ + int i, j, err, ncid, varid, dimids[2], req[NY], st[NY], nerrs=0; + int rank, nprocs, buf[NY+1][NX]; MPI_Offset start[2], count[2]; - MPI_Info info; - MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); MPI_Comm_rank(MPI_COMM_WORLD, &rank); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for using ncmpi_iput_vara_int() ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - - MPI_Info_create(&info); - /* When using PVFS2, unexpected buffer value error message might occur. - * This is due to a possible bug in ADIOI_PVFS2_OldWriteStrided() when - * filetype is contiguous and buftype is non-contiguous. - * Fix: Add ROMIO hint to force ADIO driever to use POSIX I/O */ - /* MPI_Info_set(info, "romio_pvfs2_posix_write", "enable"); */ + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, info, &ncid); CHECK_ERR - MPI_Info_free(&info); + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); + CHECK_FATAL_ERR /* define a 2D array */ err = ncmpi_def_dim(ncid, "Y", NC_UNLIMITED, &dimids[0]); CHECK_ERR @@ -88,79 +81,221 @@ int main(int argc, char **argv) { err = ncmpi_def_var(ncid, "var", NC_INT, 2, dimids, &varid); CHECK_ERR err = ncmpi_enddef(ncid); CHECK_ERR + if (!coll_io) { + err = ncmpi_sync(ncid); CHECK_ERR + } + /* initialize the contents of the array */ for (j=0; j 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); +fn_exit: + return nerrs; +} + +int tst_iget(const char *in_path, + int coll_io, + MPI_Info info) +{ + int i, j, err, ncid, varid, req[NY], st[NY], nerrs=0; + int rank, nprocs, buf[NY+1][NX]; + MPI_Offset start[2], count[2]; + + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + /* open the same file and read back for validate */ + err = ncmpi_open(MPI_COMM_WORLD, in_path, NC_NOWRITE, info, &ncid); + CHECK_FATAL_ERR + + err = ncmpi_inq_varid(ncid, "VAR", &varid); CHECK_ERR + + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + + /* initialize the contents of the array to a different value */ + for (j=0; j 0) goto err_out; + + nerrs = tst_iget(out_path, coll_io, info); + if (nerrs > 0) goto err_out; + + /* disable PnetCDF internal buffering */ + MPI_Info_set(info, "nc_ibuf_size", "0"); + + nerrs = tst_iput(out_path, format, coll_io, info); + if (nerrs > 0) goto err_out; + + nerrs = tst_iget(out_path, coll_io, info); + if (nerrs > 0) goto err_out; + +err_out: + return nerrs; +} + +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "ncmpi_iput_vara_int()", opt, test_io); + MPI_Finalize(); - return (nerrs > 0); + + return err; } diff --git a/test/testcases/null_args.m4 b/test/testcases/null_args.m4 index 65484f3802..190a0f17aa 100644 --- a/test/testcases/null_args.m4 +++ b/test/testcases/null_args.m4 @@ -66,186 +66,191 @@ define(`TEST_NULL_ARGS',` memset($1_buf, 0, 100*sizeof($1)); /*---- test put_var1 ---- */ - err = ncmpi_put_var1_$1_all(ncid, vid_$1, start, $1_buf); - EXP_ERR_MSG(NC_NOERR, "put_var1_$1_all") + err = ncmpi_put_var1_$1$2(ncid, vid_$1, start, $1_buf); + EXP_ERR_MSG(NC_NOERR, "put_var1_$1$2") - err = ncmpi_put_var1_$1_all(ncid, vid_$1, NULL, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "put_var1_$1_all start=NULL") + err = ncmpi_put_var1_$1$2(ncid, vid_$1, NULL, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "put_var1_$1$2 start=NULL") /*---- test put_vara ---- */ - err = ncmpi_put_vara_$1_all(ncid, vid_$1, start, count, $1_buf); - EXP_ERR_MSG(NC_NOERR, "put_vara_$1_all") + err = ncmpi_put_vara_$1$2(ncid, vid_$1, start, count, $1_buf); + EXP_ERR_MSG(NC_NOERR, "put_vara_$1$2") - err = ncmpi_put_vara_$1_all(ncid, vid_$1, NULL, count, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "put_vara_$1_all start=NULL") + err = ncmpi_put_vara_$1$2(ncid, vid_$1, NULL, count, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "put_vara_$1$2 start=NULL") - err = ncmpi_put_vara_$1_all(ncid, vid_$1, start, NULL, $1_buf); - EXP_ERR_MSG(NC_EEDGE, "put_vara_$1_all count=NULL") + err = ncmpi_put_vara_$1$2(ncid, vid_$1, start, NULL, $1_buf); + EXP_ERR_MSG(NC_EEDGE, "put_vara_$1$2 count=NULL") - err = ncmpi_put_vara_$1_all(ncid, vid_$1, NULL, NULL, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "put_vara_$1_all start=count=NULL") + err = ncmpi_put_vara_$1$2(ncid, vid_$1, NULL, NULL, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "put_vara_$1$2 start=count=NULL") /*---- test put_vars ---- */ - err = ncmpi_put_vars_$1_all(ncid, vid_$1, start, count, stride, $1_buf); - EXP_ERR_MSG(NC_NOERR, "put_vars_$1_all") + err = ncmpi_put_vars_$1$2(ncid, vid_$1, start, count, stride, $1_buf); + EXP_ERR_MSG(NC_NOERR, "put_vars_$1$2") - err = ncmpi_put_vars_$1_all(ncid, vid_$1, NULL, count, stride, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "put_vars_$1_all start=NULL") + err = ncmpi_put_vars_$1$2(ncid, vid_$1, NULL, count, stride, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "put_vars_$1$2 start=NULL") - err = ncmpi_put_vars_$1_all(ncid, vid_$1, start, NULL, stride, $1_buf); - EXP_ERR_MSG(NC_EEDGE, "put_vars_$1_all count=NULL") + err = ncmpi_put_vars_$1$2(ncid, vid_$1, start, NULL, stride, $1_buf); + EXP_ERR_MSG(NC_EEDGE, "put_vars_$1$2 count=NULL") - err = ncmpi_put_vars_$1_all(ncid, vid_$1, start, count, NULL, $1_buf); - EXP_ERR_MSG(NC_NOERR, "put_vars_$1_all stride=NULL") + err = ncmpi_put_vars_$1$2(ncid, vid_$1, start, count, NULL, $1_buf); + EXP_ERR_MSG(NC_NOERR, "put_vars_$1$2 stride=NULL") - err = ncmpi_put_vars_$1_all(ncid, vid_$1, NULL, NULL, stride, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "put_vars_$1_all start=count=NULL") + err = ncmpi_put_vars_$1$2(ncid, vid_$1, NULL, NULL, stride, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "put_vars_$1$2 start=count=NULL") - err = ncmpi_put_vars_$1_all(ncid, vid_$1, NULL, count, NULL, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "put_vars_$1_all start=stride=NULL") + err = ncmpi_put_vars_$1$2(ncid, vid_$1, NULL, count, NULL, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "put_vars_$1$2 start=stride=NULL") - err = ncmpi_put_vars_$1_all(ncid, vid_$1, start, NULL, NULL, $1_buf); - EXP_ERR_MSG(NC_EEDGE, "put_vars_$1_all count=stride=NULL") + err = ncmpi_put_vars_$1$2(ncid, vid_$1, start, NULL, NULL, $1_buf); + EXP_ERR_MSG(NC_EEDGE, "put_vars_$1$2 count=stride=NULL") - err = ncmpi_put_vars_$1_all(ncid, vid_$1, NULL, NULL, NULL, $1_buf); + err = ncmpi_put_vars_$1$2(ncid, vid_$1, NULL, NULL, NULL, $1_buf); EXP_ERR_MSG(NC_EINVALCOORDS, "put_vars start=count=stride=NULL") /*---- test put_varm ---- */ - err = ncmpi_put_varm_$1_all(ncid, vid_$1, start, count, stride, imap, $1_buf); - EXP_ERR_MSG(NC_NOERR, "put_varm_$1_all") + err = ncmpi_put_varm_$1$2(ncid, vid_$1, start, count, stride, imap, $1_buf); + EXP_ERR_MSG(NC_NOERR, "put_varm_$1$2") - err = ncmpi_put_varm_$1_all(ncid, vid_$1, NULL, count, stride, imap, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "put_varm_$1_all start=NULL") + err = ncmpi_put_varm_$1$2(ncid, vid_$1, NULL, count, stride, imap, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "put_varm_$1$2 start=NULL") - err = ncmpi_put_varm_$1_all(ncid, vid_$1, start, NULL, stride, imap, $1_buf); - EXP_ERR_MSG(NC_EEDGE, "put_varm_$1_all count=NULL") + err = ncmpi_put_varm_$1$2(ncid, vid_$1, start, NULL, stride, imap, $1_buf); + EXP_ERR_MSG(NC_EEDGE, "put_varm_$1$2 count=NULL") - err = ncmpi_put_varm_$1_all(ncid, vid_$1, start, count, NULL, imap, $1_buf); - EXP_ERR_MSG(NC_NOERR, "put_varm_$1_all stride=NULL") + err = ncmpi_put_varm_$1$2(ncid, vid_$1, start, count, NULL, imap, $1_buf); + EXP_ERR_MSG(NC_NOERR, "put_varm_$1$2 stride=NULL") - err = ncmpi_put_varm_$1_all(ncid, vid_$1, start, count, stride, NULL, $1_buf); - EXP_ERR_MSG(NC_NOERR, "put_varm_$1_all imap=NULL") + err = ncmpi_put_varm_$1$2(ncid, vid_$1, start, count, stride, NULL, $1_buf); + EXP_ERR_MSG(NC_NOERR, "put_varm_$1$2 imap=NULL") - err = ncmpi_put_varm_$1_all(ncid, vid_$1, NULL, NULL, stride, imap, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "put_varm_$1_all start=count=NULL") + err = ncmpi_put_varm_$1$2(ncid, vid_$1, NULL, NULL, stride, imap, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "put_varm_$1$2 start=count=NULL") - err = ncmpi_put_varm_$1_all(ncid, vid_$1, NULL, count, NULL, imap, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "put_varm_$1_all start=stride=NULL") + err = ncmpi_put_varm_$1$2(ncid, vid_$1, NULL, count, NULL, imap, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "put_varm_$1$2 start=stride=NULL") - err = ncmpi_put_varm_$1_all(ncid, vid_$1, NULL, count, stride, NULL, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "put_varm_$1_all start=imap=NULL") + err = ncmpi_put_varm_$1$2(ncid, vid_$1, NULL, count, stride, NULL, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "put_varm_$1$2 start=imap=NULL") - err = ncmpi_put_varm_$1_all(ncid, vid_$1, start, NULL, NULL, imap, $1_buf); - EXP_ERR_MSG(NC_EEDGE, "put_varm_$1_all count=stride=NULL") + err = ncmpi_put_varm_$1$2(ncid, vid_$1, start, NULL, NULL, imap, $1_buf); + EXP_ERR_MSG(NC_EEDGE, "put_varm_$1$2 count=stride=NULL") - err = ncmpi_put_varm_$1_all(ncid, vid_$1, start, NULL, stride, NULL, $1_buf); - EXP_ERR_MSG(NC_EEDGE, "put_varm_$1_all count=imap=NULL") + err = ncmpi_put_varm_$1$2(ncid, vid_$1, start, NULL, stride, NULL, $1_buf); + EXP_ERR_MSG(NC_EEDGE, "put_varm_$1$2 count=imap=NULL") - err = ncmpi_put_varm_$1_all(ncid, vid_$1, start, count, NULL, NULL, $1_buf); - EXP_ERR_MSG(NC_NOERR, "put_varm_$1_all stride=imap=NULL") + err = ncmpi_put_varm_$1$2(ncid, vid_$1, start, count, NULL, NULL, $1_buf); + EXP_ERR_MSG(NC_NOERR, "put_varm_$1$2 stride=imap=NULL") - err = ncmpi_put_varm_$1_all(ncid, vid_$1, NULL, NULL, NULL, imap, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "put_varm_$1_all start=count=stride=NULL") + err = ncmpi_put_varm_$1$2(ncid, vid_$1, NULL, NULL, NULL, imap, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "put_varm_$1$2 start=count=stride=NULL") - err = ncmpi_put_varm_$1_all(ncid, vid_$1, NULL, NULL, stride, NULL, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "put_varm_$1_all start=count=imap=NULL") + err = ncmpi_put_varm_$1$2(ncid, vid_$1, NULL, NULL, stride, NULL, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "put_varm_$1$2 start=count=imap=NULL") - err = ncmpi_put_varm_$1_all(ncid, vid_$1, start, NULL, NULL, NULL, $1_buf); - EXP_ERR_MSG(NC_EEDGE, "put_varm_$1_all count=stride=imap=NULL") + err = ncmpi_put_varm_$1$2(ncid, vid_$1, start, NULL, NULL, NULL, $1_buf); + EXP_ERR_MSG(NC_EEDGE, "put_varm_$1$2 count=stride=imap=NULL") - err = ncmpi_put_varm_$1_all(ncid, vid_$1, NULL, NULL, NULL, NULL, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "put_varm_$1_all start=count=stride=imap=NULL") + err = ncmpi_put_varm_$1$2(ncid, vid_$1, NULL, NULL, NULL, NULL, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "put_varm_$1$2 start=count=stride=imap=NULL") + + /* file sync before reading */ + err = ncmpi_sync(ncid); + CHECK_ERR + MPI_Barrier(MPI_COMM_WORLD); /*---- test get_var1 ---- */ - err = ncmpi_get_var1_$1_all(ncid, vid_$1, start, $1_buf); - EXP_ERR_MSG(NC_NOERR, "get_var1_$1_all") + err = ncmpi_get_var1_$1$2(ncid, vid_$1, start, $1_buf); + EXP_ERR_MSG(NC_NOERR, "get_var1_$1$2") - err = ncmpi_get_var1_$1_all(ncid, vid_$1, NULL, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "get_var1_$1_all start=NULL") + err = ncmpi_get_var1_$1$2(ncid, vid_$1, NULL, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "get_var1_$1$2 start=NULL") /*---- test get_vara ---- */ - err = ncmpi_get_vara_$1_all(ncid, vid_$1, start, count, $1_buf); - EXP_ERR_MSG(NC_NOERR, "get_vara_$1_all") + err = ncmpi_get_vara_$1$2(ncid, vid_$1, start, count, $1_buf); + EXP_ERR_MSG(NC_NOERR, "get_vara_$1$2") - err = ncmpi_get_vara_$1_all(ncid, vid_$1, NULL, count, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "get_vara_$1_all start=NULL") + err = ncmpi_get_vara_$1$2(ncid, vid_$1, NULL, count, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "get_vara_$1$2 start=NULL") - err = ncmpi_get_vara_$1_all(ncid, vid_$1, start, NULL, $1_buf); - EXP_ERR_MSG(NC_EEDGE, "get_vara_$1_all count=NULL") + err = ncmpi_get_vara_$1$2(ncid, vid_$1, start, NULL, $1_buf); + EXP_ERR_MSG(NC_EEDGE, "get_vara_$1$2 count=NULL") - err = ncmpi_get_vara_$1_all(ncid, vid_$1, NULL, NULL, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "get_vara_$1_all start=count=NULL") + err = ncmpi_get_vara_$1$2(ncid, vid_$1, NULL, NULL, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "get_vara_$1$2 start=count=NULL") /*---- test get_vars ---- */ - err = ncmpi_get_vars_$1_all(ncid, vid_$1, start, count, stride, $1_buf); - EXP_ERR_MSG(NC_NOERR, "get_vars_$1_all") + err = ncmpi_get_vars_$1$2(ncid, vid_$1, start, count, stride, $1_buf); + EXP_ERR_MSG(NC_NOERR, "get_vars_$1$2") - err = ncmpi_get_vars_$1_all(ncid, vid_$1, NULL, count, stride, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "get_vars_$1_all start=NULL") + err = ncmpi_get_vars_$1$2(ncid, vid_$1, NULL, count, stride, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "get_vars_$1$2 start=NULL") - err = ncmpi_get_vars_$1_all(ncid, vid_$1, start, NULL, stride, $1_buf); - EXP_ERR_MSG(NC_EEDGE, "get_vars_$1_all count=NULL") + err = ncmpi_get_vars_$1$2(ncid, vid_$1, start, NULL, stride, $1_buf); + EXP_ERR_MSG(NC_EEDGE, "get_vars_$1$2 count=NULL") - err = ncmpi_get_vars_$1_all(ncid, vid_$1, start, count, NULL, $1_buf); - EXP_ERR_MSG(NC_NOERR, "get_vars_$1_all stride=NULL") + err = ncmpi_get_vars_$1$2(ncid, vid_$1, start, count, NULL, $1_buf); + EXP_ERR_MSG(NC_NOERR, "get_vars_$1$2 stride=NULL") - err = ncmpi_get_vars_$1_all(ncid, vid_$1, NULL, NULL, stride, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "get_vars_$1_all start=count=NULL") + err = ncmpi_get_vars_$1$2(ncid, vid_$1, NULL, NULL, stride, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "get_vars_$1$2 start=count=NULL") - err = ncmpi_get_vars_$1_all(ncid, vid_$1, NULL, count, NULL, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "get_vars_$1_all start=stride=NULL") + err = ncmpi_get_vars_$1$2(ncid, vid_$1, NULL, count, NULL, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "get_vars_$1$2 start=stride=NULL") - err = ncmpi_get_vars_$1_all(ncid, vid_$1, start, NULL, NULL, $1_buf); - EXP_ERR_MSG(NC_EEDGE, "get_vars_$1_all count=stride=NULL") + err = ncmpi_get_vars_$1$2(ncid, vid_$1, start, NULL, NULL, $1_buf); + EXP_ERR_MSG(NC_EEDGE, "get_vars_$1$2 count=stride=NULL") - err = ncmpi_get_vars_$1_all(ncid, vid_$1, NULL, NULL, NULL, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "get_vars_$1_all start=count=stride=NULL") + err = ncmpi_get_vars_$1$2(ncid, vid_$1, NULL, NULL, NULL, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "get_vars_$1$2 start=count=stride=NULL") /*---- test get_varm ---- */ - err = ncmpi_get_varm_$1_all(ncid, vid_$1, start, count, stride, imap, $1_buf); - EXP_ERR_MSG(NC_NOERR, "get_varm_$1_all") + err = ncmpi_get_varm_$1$2(ncid, vid_$1, start, count, stride, imap, $1_buf); + EXP_ERR_MSG(NC_NOERR, "get_varm_$1$2") - err = ncmpi_get_varm_$1_all(ncid, vid_$1, NULL, count, stride, imap, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "get_varm_$1_all start=NULL") + err = ncmpi_get_varm_$1$2(ncid, vid_$1, NULL, count, stride, imap, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "get_varm_$1$2 start=NULL") - err = ncmpi_get_varm_$1_all(ncid, vid_$1, start, NULL, stride, imap, $1_buf); - EXP_ERR_MSG(NC_EEDGE, "get_varm_$1_all count=NULL") + err = ncmpi_get_varm_$1$2(ncid, vid_$1, start, NULL, stride, imap, $1_buf); + EXP_ERR_MSG(NC_EEDGE, "get_varm_$1$2 count=NULL") - err = ncmpi_get_varm_$1_all(ncid, vid_$1, start, count, NULL, imap, $1_buf); - EXP_ERR_MSG(NC_NOERR, "get_varm_$1_all stride=NULL") + err = ncmpi_get_varm_$1$2(ncid, vid_$1, start, count, NULL, imap, $1_buf); + EXP_ERR_MSG(NC_NOERR, "get_varm_$1$2 stride=NULL") - err = ncmpi_get_varm_$1_all(ncid, vid_$1, start, count, stride, NULL, $1_buf); - EXP_ERR_MSG(NC_NOERR, "get_varm_$1_all imap=NULL") + err = ncmpi_get_varm_$1$2(ncid, vid_$1, start, count, stride, NULL, $1_buf); + EXP_ERR_MSG(NC_NOERR, "get_varm_$1$2 imap=NULL") - err = ncmpi_get_varm_$1_all(ncid, vid_$1, NULL, NULL, stride, imap, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "get_varm_$1_all start=count=NULL") + err = ncmpi_get_varm_$1$2(ncid, vid_$1, NULL, NULL, stride, imap, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "get_varm_$1$2 start=count=NULL") - err = ncmpi_get_varm_$1_all(ncid, vid_$1, NULL, count, NULL, imap, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "get_varm_$1_all start=stride=NULL") + err = ncmpi_get_varm_$1$2(ncid, vid_$1, NULL, count, NULL, imap, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "get_varm_$1$2 start=stride=NULL") - err = ncmpi_get_varm_$1_all(ncid, vid_$1, NULL, count, stride, NULL, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "get_varm_$1_all start=imap=NULL") + err = ncmpi_get_varm_$1$2(ncid, vid_$1, NULL, count, stride, NULL, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "get_varm_$1$2 start=imap=NULL") - err = ncmpi_get_varm_$1_all(ncid, vid_$1, start, NULL, NULL, imap, $1_buf); - EXP_ERR_MSG(NC_EEDGE, "get_varm_$1_all count=stride=NULL") + err = ncmpi_get_varm_$1$2(ncid, vid_$1, start, NULL, NULL, imap, $1_buf); + EXP_ERR_MSG(NC_EEDGE, "get_varm_$1$2 count=stride=NULL") - err = ncmpi_get_varm_$1_all(ncid, vid_$1, start, NULL, stride, NULL, $1_buf); - EXP_ERR_MSG(NC_EEDGE, "get_varm_$1_all count=imap=NULL") + err = ncmpi_get_varm_$1$2(ncid, vid_$1, start, NULL, stride, NULL, $1_buf); + EXP_ERR_MSG(NC_EEDGE, "get_varm_$1$2 count=imap=NULL") - err = ncmpi_get_varm_$1_all(ncid, vid_$1, start, count, NULL, NULL, $1_buf); - EXP_ERR_MSG(NC_NOERR, "get_varm_$1_all stride=imap=NULL") + err = ncmpi_get_varm_$1$2(ncid, vid_$1, start, count, NULL, NULL, $1_buf); + EXP_ERR_MSG(NC_NOERR, "get_varm_$1$2 stride=imap=NULL") - err = ncmpi_get_varm_$1_all(ncid, vid_$1, NULL, NULL, NULL, imap, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "get_varm_$1_all start=count=stride=NULL") + err = ncmpi_get_varm_$1$2(ncid, vid_$1, NULL, NULL, NULL, imap, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "get_varm_$1$2 start=count=stride=NULL") - err = ncmpi_get_varm_$1_all(ncid, vid_$1, NULL, NULL, stride, NULL, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "get_varm_$1_all start=count=imap=NULL") + err = ncmpi_get_varm_$1$2(ncid, vid_$1, NULL, NULL, stride, NULL, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "get_varm_$1$2 start=count=imap=NULL") - err = ncmpi_get_varm_$1_all(ncid, vid_$1, start, NULL, NULL, NULL, $1_buf); - EXP_ERR_MSG(NC_EEDGE, "get_varm_$1_all count=stride=imap=NULL") + err = ncmpi_get_varm_$1$2(ncid, vid_$1, start, NULL, NULL, NULL, $1_buf); + EXP_ERR_MSG(NC_EEDGE, "get_varm_$1$2 count=stride=imap=NULL") - err = ncmpi_get_varm_$1_all(ncid, vid_$1, NULL, NULL, NULL, NULL, $1_buf); - EXP_ERR_MSG(NC_EINVALCOORDS, "get_varm_$1_all start=count=stride=imap=NULL") + err = ncmpi_get_varm_$1$2(ncid, vid_$1, NULL, NULL, NULL, NULL, $1_buf); + EXP_ERR_MSG(NC_EINVALCOORDS, "get_varm_$1$2 start=count=stride=imap=NULL") ')dnl define(`CDF5_ITYPES',`schar,uchar,short,ushort,int,uint,long,float,double,longlong,ulonglong')dnl @@ -255,7 +260,7 @@ define(`EXTRA_ITYPES',`uchar,ushort,uint,longlong,ulonglong')dnl define(`TEST_FORMAT',dnl `dnl static int -test_format_nc$1(char *filename) +test_format_nc$1(const char *filename, int format, int coll_io, MPI_Info info) { int err, nerrs=0, ncid, cmode, dimid[2]; MPI_Offset start[2], count[2], stride[2], imap[2]; @@ -284,7 +289,7 @@ test_format_nc$1(char *filename) `$1',`4',`cmode = NC_CLOBBER | NC_NETCDF4 | NC_CLASSIC_MODEL;', `cmode = NC_CLOBBER;')dnl - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, info, &ncid); EXP_ERR_MSG(NC_NOERR, "create") err = ncmpi_def_dim(ncid, "Y", NC_UNLIMITED, &dimid[0]); @@ -295,17 +300,36 @@ test_format_nc$1(char *filename) /* define variables */dnl foreach(`itype',(text, TYPE_LIST),`_CAT(` err = ncmpi_def_var(ncid,"var_'itype`",NC_TYPE(itype),2,dimid,&vid_',itype`); - EXP_ERR_MSG(NC_NOERR,"def_var")')') + EXP_ERR_MSG(NC_NOERR,"def_var") + err = ncmpi_def_var_fill(ncid, vid_'itype`, 0, NULL); + EXP_ERR_MSG(NC_NOERR,"def_var_fill")')') err = ncmpi_enddef(ncid); EXP_ERR_MSG(NC_NOERR,"enddef") + if (format != NC_FORMAT_NETCDF4 && format != NC_FORMAT_NETCDF4_CLASSIC) { + /* fill the 1st record of all variables */dnl + foreach(`itype',(text, TYPE_LIST),`_CAT(` + err = ncmpi_fill_var_rec(ncid, vid_'itype`, 0); + EXP_ERR_MSG(NC_NOERR,"fill_var_rec")')') + } + + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + start[0] = start[1] = 0; count[0] = count[1] = 1; stride[0] = stride[1] = 1; imap[0] = imap[1] = 1; - foreach(`itype',(text, TYPE_LIST),`TEST_NULL_ARGS(itype)') + if (coll_io) { + foreach(`itype',(text, TYPE_LIST),`TEST_NULL_ARGS(itype, _all)') + } + else { + foreach(`itype',(text, TYPE_LIST),`TEST_NULL_ARGS(itype)') + } err_out: err = ncmpi_close(ncid); @@ -318,67 +342,64 @@ err_out: TEST_FORMAT(1) TEST_FORMAT(2) TEST_FORMAT(5) -#ifdef ENABLE_NETCDF4 TEST_FORMAT(3) TEST_FORMAT(4) -#endif -int main(int argc, char **argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) { - char filename[256], *hint_value;; - int err, nerrs=0, rank, bb_enabled=0; + char val[MPI_MAX_INFO_VAL]; + int err, nerrs=0, flag; - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); + /* check whether burst buffering is enabled */ + MPI_Info_get(info, "nc_burst_buf", MPI_MAX_INFO_VAL - 1, val, &flag); + if (flag && strcasecmp(val, "enable") == 0 && + (format == NC_FORMAT_NETCDF4 || format == NC_FORMAT_NETCDF4_CLASSIC)) + /* does not work for NetCDF4 files when burst-buffering is enabled */ + return 0; + + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + if (format == NC_FORMAT_CLASSIC) + nerrs += test_format_nc1(out_path, format, coll_io, info); + else if (format == NC_FORMAT_64BIT_OFFSET) + nerrs += test_format_nc2(out_path, format, coll_io, info); + else if (format == NC_FORMAT_NETCDF4) + nerrs += test_format_nc3(out_path, format, coll_io, info); + else if (format == NC_FORMAT_NETCDF4_CLASSIC) + nerrs += test_format_nc4(out_path, format, coll_io, info); + else if (format == NC_FORMAT_64BIT_OFFSET) + nerrs += test_format_nc5(out_path, format, coll_io, info); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for NULL arguments ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } + return nerrs; +} - /* check whether burst buffering is enabled */ - if (inq_env_hint("nc_burst_buf", &hint_value)) { - if (strcasecmp(hint_value, "enable") == 0) bb_enabled = 1; - free(hint_value); - } +int main(int argc, char **argv) { - nerrs += test_format_nc1(filename); - nerrs += test_format_nc2(filename); - if (!bb_enabled) { -#ifdef ENABLE_NETCDF4 - nerrs += test_format_nc3(filename); - nerrs += test_format_nc4(filename); -#endif - } - nerrs += test_format_nc5(filename); - - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + int err; + loop_opts opt; - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = false;/* skip ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ + + err = tst_main(argc, argv, "NULL arguments", opt, test_io); MPI_Finalize(); - return (nerrs > 0); + + return err; } diff --git a/test/testcases/one_record.c b/test/testcases/one_record.c index 0af91fec97..1b6506efe8 100644 --- a/test/testcases/one_record.c +++ b/test/testcases/one_record.c @@ -26,40 +26,30 @@ #define STR_LEN 19 #define NUM_VALS 2 -/*----< main() >------------------------------------------------------------*/ -int main(int argc, char **argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) { - char filename[256]; - int i, err, nerrs=0, rank, nprocs, cmode; + int i, err, nerrs=0, rank, nprocs; int ncid, dimids[2], varid; char data[NUM_VALS][STR_LEN + 1], data_in[NUM_VALS*STR_LEN]; MPI_Offset start[2]; MPI_Offset count[2]; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for only one record variable ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - strcpy(data[0], "2005-04-11_12:00:00"); /* 19 bytes not a multiply of 4 */ strcpy(data[1], "2005-04-11_13:00:00"); - cmode = NC_CLOBBER; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); CHECK_ERR + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "time", NC_UNLIMITED, dimids); CHECK_ERR err = ncmpi_def_dim(ncid, "text_dim", STR_LEN, &dimids[1]); CHECK_ERR @@ -70,54 +60,79 @@ int main(int argc, char **argv) err = ncmpi_def_var(ncid, "text_var", NC_CHAR, 2, dimids, &varid); CHECK_ERR err = ncmpi_enddef(ncid); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } /* Write some records of var data. */ count[0] = 1; count[1] = STR_LEN; start[0] = 0; start[1] = 0; for (i=0; i 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + return nerrs; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +int main(int argc, char **argv) { + + int err; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "only one record variable", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/testcases/parallel_run.sh b/test/testcases/parallel_run.sh index 0abc8b12fb..f479a28ed4 100755 --- a/test/testcases/parallel_run.sh +++ b/test/testcases/parallel_run.sh @@ -1,116 +1,51 @@ #!/bin/bash # -# Copyright (C) 2018, Northwestern University and Argonne National Laboratory +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory # See COPYRIGHT notice in top-level directory. # # Exit immediately if a command exits with a non-zero status. set -e -VALIDATOR=../../src/utils/ncvalidator/ncvalidator -NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff +DRY_RUN=no +VERBOSE=no + +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $MPIRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $MPIRUN $@ + fi +} MPIRUN=`echo ${TESTMPIRUN} | ${SED} -e "s/NP/$1/g"` # echo "MPIRUN = ${MPIRUN}" # echo "check_PROGRAMS=${check_PROGRAMS}" -# remove file system type prefix if there is any -OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` - -# let NTHREADS=$1*6-1 -NTHREADS=`expr $1 \* 6 - 1` - -# echo "PNETCDF_DEBUG = ${PNETCDF_DEBUG}" -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" -else - safe_modes="0" -fi - # prevent user environment setting of PNETCDF_HINTS to interfere unset PNETCDF_HINTS -for i in ${check_PROGRAMS} ; do - for j in ${safe_modes} ; do - for intra_aggr in 0 1 ; do - if test "$j" = 1 ; then # test only in safe mode - export PNETCDF_HINTS="romio_no_indep_rw=true" - else - export PNETCDF_HINTS= - fi - if test "$intra_aggr" = 1 ; then - export PNETCDF_HINTS="${PNETCDF_HINTS};nc_num_aggrs_per_node=2" - fi - export PNETCDF_SAFE_MODE=$j - # echo "set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - - if test "$i" = tst_version ; then - ${MPIRUN} ./tst_version - continue - fi - - if test "$i" = tst_pthread ; then - # each MPI process created 6 threads - ${MPIRUN} ./tst_pthread ${TESTOUTDIR}/tst_pthread.nc - for k in `seq 0 ${NTHREADS}` ; do - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/tst_pthread.nc.$k - rm -f ${OUTDIR}/tst_pthread.nc.$k - done - continue - fi - - ${MPIRUN} ./$i ${TESTOUTDIR}/$i.nc +PNETCDF_HINTS= +if test "x$MIMIC_LUSTRE" != x1 ; then + PNETCDF_HINTS="cb_nodes=2" +fi - # put_all_kinds and iput_all_kinds output 3 files - if test "$i" = put_all_kinds -o "$i" = iput_all_kinds ; then - for k in 1 2 5 ; do - # echo "--- validating file ${TESTOUTDIR}/$i.nc$k" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.nc$k - done - else - # echo "--- validating file ${TESTOUTDIR}/$i.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.nc - fi - # echo "" +for i in ${check_PROGRAMS} ; do - if test "x${ENABLE_BURST_BUFFER}" = x1 ; then - # echo "---- test burst buffering feature" - saved_PNETCDF_HINTS=${PNETCDF_HINTS} - export PNETCDF_HINTS="${PNETCDF_HINTS};nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" - ${MPIRUN} ./$i ${TESTOUTDIR}/$i.bb.nc - export PNETCDF_HINTS=${saved_PNETCDF_HINTS} + # # SECONDS=0 + # start_ns=$(date +%s.%4N) - # put_all_kinds and iput_all_kinds output 3 files - if test "$i" = put_all_kinds -o "$i" = iput_all_kinds ; then - for k in 1 2 5 ; do - # echo "--- validating file ${TESTOUTDIR}/$i.bb.nc$k" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.bb.nc$k - # echo "--- ncmpidiff $i.nc$k $i.bb.nc$k ---" - ${MPIRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/$i.nc$k ${TESTOUTDIR}/$i.bb.nc$k - done - continue - else - # echo "--- validating file ${TESTOUTDIR}/$i.bb.nc" - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$i.bb.nc - fi + exe_name=`basename $i` - # skip ncmpidiff for large file - if test "$i" = last_large_var ; then - continue - fi + run_cmd ./$i -q -o ${TESTOUTDIR}/${exe_name}.nc - # echo "--- ncmpidiff $i.nc $i.bb.nc ---" - ${MPIRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/$i.nc ${TESTOUTDIR}/$i.bb.nc - fi + # # echo "Elapsed: $SECONDS seconds" + # end_ns=$(date +%s.%4N) + # # Calculate difference (requires bc for floating point math) + # elapsed_ns=$(echo "$end_ns - $start_ns" | bc) + # echo "Elapsed time: ${elapsed_ns} seconds" - if test "x${ENABLE_NETCDF4}" = x1 ; then - # echo "test netCDF-4 feature" - ${MPIRUN} ./$i ${TESTOUTDIR}/$i.nc4 4 - # Validator does not support nc4 - fi - done - done - rm -f ${OUTDIR}/$i.nc* - rm -f ${OUTDIR}/$i.bb.nc* -done +done # check_PROGRAMS diff --git a/test/testcases/profile.c b/test/testcases/profile.c index 4162be254a..c28737bf4d 100644 --- a/test/testcases/profile.c +++ b/test/testcases/profile.c @@ -332,7 +332,7 @@ static int test_ivarn(int ncid) if (err != NC_NOERR) { for (i=0; i<4; i++) { if (st[i] != NC_NOERR) { - printf("Error at line %d in %s: st[%d] %s\n", + fprintf(stderr,"Error at line %d in %s: st[%d] %s\n", __FILE__,__LINE__,i,ncmpi_strerror(st[i])); } } diff --git a/test/testcases/put_all_kinds.m4 b/test/testcases/put_all_kinds.m4 index 8cbac915d8..f4e018095a 100644 --- a/test/testcases/put_all_kinds.m4 +++ b/test/testcases/put_all_kinds.m4 @@ -22,7 +22,7 @@ dnl #include #define NDIMS 3 -#define LEN 2 +#define LEN 7 include(`foreach.m4')dnl include(`utils.m4')dnl @@ -47,12 +47,43 @@ include(`utils.m4')dnl #define ulonglong unsigned long long #endif +define(`DEFINE_VARS',dnl +`dnl +static int +def_vars_$1(int rank, + int ncid, + int *dimids) +{ + int err=NC_NOERR; + int var1_id, vara_id, vars_id, varm_id; + int dimid, dimidsT[NDIMS]; + + err = ncmpi_inq_dimid(ncid, "nprocs", &dimid); CHECK_ERR + + err = ncmpi_def_var(ncid, "var1_$1", NC_TYPE($1), 1, &dimid, &var1_id); + CHECK_ERR + err = ncmpi_def_var(ncid, "vara_$1", NC_TYPE($1), NDIMS, dimids, &vara_id); + CHECK_ERR + err = ncmpi_def_var(ncid, "vars_$1", NC_TYPE($1), NDIMS, dimids, &vars_id); + CHECK_ERR + + /* define variable with transposed file layout: ZYX -> YXZ */ + dimidsT[0] = dimids[1]; dimidsT[1] = dimids[2]; dimidsT[2] = dimids[0]; + err = ncmpi_def_var(ncid, "varm_$1", NC_TYPE($1), NDIMS, dimidsT, &varm_id); + CHECK_ERR + + return err; +} +')dnl + +foreach(`itype',(`text,schar,uchar,short,ushort,int,uint,long,float,double,longlong,ulonglong'),`DEFINE_VARS(itype)') + define(`TEST_BLOCKING_PUT',dnl `dnl static int blocking_put_$1(int rank, + int coll_io, int ncid, - int *dimids, MPI_Offset *start, MPI_Offset *count, MPI_Offset *startS, @@ -63,34 +94,40 @@ blocking_put_$1(int rank, MPI_Offset *imap, ifelse(`$1',`text',`char',`double') *buf) { - int err, nerrs=0; + int err=NC_NOERR, nerrs=0; int var1_id, vara_id, vars_id, varm_id; - int dimid, dimidsT[NDIMS]; MPI_Offset start1[1]; - /* re-enter define mode, so we can add more variables */ - err = ncmpi_redef(ncid); CHECK_ERR - err = ncmpi_inq_dimid(ncid, "nprocs", &dimid); CHECK_ERR - err = ncmpi_def_var(ncid, "var1_$1", NC_TYPE($1), 1, &dimid, &var1_id); CHECK_ERR - err = ncmpi_def_var(ncid, "vara_$1", NC_TYPE($1), NDIMS, dimids, &vara_id); CHECK_ERR - err = ncmpi_def_var(ncid, "vars_$1", NC_TYPE($1), NDIMS, dimids, &vars_id); CHECK_ERR - - /* define variable with transposed file layout: ZYX -> YXZ */ - dimidsT[0] = dimids[1]; dimidsT[1] = dimids[2]; dimidsT[2] = dimids[0]; - err = ncmpi_def_var(ncid, "varm_$1", NC_TYPE($1), NDIMS, dimidsT, &varm_id); CHECK_ERR - - /* exit the define mode */ - err = ncmpi_enddef(ncid); CHECK_ERR + err = ncmpi_inq_varid(ncid, "var1_$1", &var1_id); CHECK_ERR + err = ncmpi_inq_varid(ncid, "vara_$1", &vara_id); CHECK_ERR + err = ncmpi_inq_varid(ncid, "vars_$1", &vars_id); CHECK_ERR + err = ncmpi_inq_varid(ncid, "varm_$1", &varm_id); CHECK_ERR /* write the whole variable in parallel */ start1[0] = rank; - err = `ncmpi_put_var1_'ifelse(`$1',`text',`$1',`double')`_all'(ncid, var1_id, start1, buf); CHECK_ERR + if (coll_io) + err = `ncmpi_put_var1_'ifelse(`$1',`text',`$1',`double')`_all'(ncid, var1_id, start1, buf); + else + err = `ncmpi_put_var1_'ifelse(`$1',`text',`$1',`double')(ncid, var1_id, start1, buf); + CHECK_ERR - err = `ncmpi_put_vara_'ifelse(`$1',`text',`$1',`double')`_all'(ncid, vara_id, start, count, buf); CHECK_ERR + if (coll_io) + err = `ncmpi_put_vara_'ifelse(`$1',`text',`$1',`double')`_all'(ncid, vara_id, start, count, buf); + else + err = `ncmpi_put_vara_'ifelse(`$1',`text',`$1',`double')(ncid, vara_id, start, count, buf); + CHECK_ERR - err = `ncmpi_put_vars_'ifelse(`$1',`text',`$1',`double')`_all'(ncid, vars_id, startS, countS, stride, buf); CHECK_ERR + if (coll_io) + err = `ncmpi_put_vars_'ifelse(`$1',`text',`$1',`double')`_all'(ncid, vars_id, startS, countS, stride, buf); + else + err = `ncmpi_put_vars_'ifelse(`$1',`text',`$1',`double')(ncid, vars_id, startS, countS, stride, buf); + CHECK_ERR - err = `ncmpi_put_varm_'ifelse(`$1',`text',`$1',`double')`_all'(ncid, varm_id, startM, countM, NULL, imap, buf); CHECK_ERR + if (coll_io) + err = `ncmpi_put_varm_'ifelse(`$1',`text',`$1',`double')`_all'(ncid, varm_id, startM, countM, NULL, imap, buf); + else + err = `ncmpi_put_varm_'ifelse(`$1',`text',`$1',`double')(ncid, varm_id, startM, countM, NULL, imap, buf); + CHECK_ERR return nerrs; } @@ -100,20 +137,12 @@ foreach(`itype',(`text,schar,uchar,short,ushort,int,uint,long,float,double,longl define(`TEST_CDF_FORMAT',dnl `dnl -/* create a new $1 file */ - cmode = NC_CLOBBER; - ifelse(`$1',`NC_FORMAT_64BIT_OFFSET', `cmode |= NC_64BIT_OFFSET;', - `$1',`NC_FORMAT_64BIT_DATA', `cmode |= NC_64BIT_DATA;', - `$1',`NC_FORMAT_NETCDF4_CLASSIC',`cmode |= NC_NETCDF4 | NC_CLASSIC_MODEL;', - `$1',`NC_FORMAT_NETCDF4', `cmode |= NC_NETCDF4;') - - sprintf(fname, "%s%d",filename, $1); - err = ncmpi_create(MPI_COMM_WORLD, fname, cmode, info, &ncid); + /* create a new file */ + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); if (err != NC_NOERR) { - printf("Error at line %d in %s: ncmpi_create() file %s (%s)\n", + fprintf(stderr,"Error at line %d in %s: ncmpi_create() file %s (%s)\n", __LINE__,__FILE__,fname,ncmpi_strerror(err)); - MPI_Abort(MPI_COMM_WORLD, -1); - exit(1); + CHECK_ERR } /* define dimensions */ @@ -121,66 +150,62 @@ define(`TEST_CDF_FORMAT',dnl err = ncmpi_def_dim(ncid, "Z", gsize[0], &dimids[0]); CHECK_ERR err = ncmpi_def_dim(ncid, "Y", gsize[1], &dimids[1]); CHECK_ERR err = ncmpi_def_dim(ncid, "X", gsize[2], &dimids[2]); CHECK_ERR + + nerrs += def_vars_text(rank, ncid, dimids); CHECK_ERR + foreach(`itype',(`schar,short,int,long,float,double'),` + _CAT(`nerrs += def_vars_',itype)'`(rank, ncid, dimids); CHECK_ERR') + + if (format == NC_FORMAT_64BIT_DATA) { + foreach(`itype', + (`uchar,ushort,uint,longlong,ulonglong'),` + _CAT(`nerrs += def_vars_',itype)'`(rank, ncid, dimids); CHECK_ERR') + } + + /* define variables */ err = ncmpi_enddef(ncid); - nerrs += blocking_put_text(rank, ncid, dimids, start, count, + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + + nerrs += blocking_put_text(rank, coll_io, ncid, start, count, startS, countS, stride, startM, countM, imap, cbuf); foreach(`itype',(`schar,short,int,long,float,double'),` - _CAT(`nerrs += blocking_put_',itype)'`(rank, ncid, dimids, start, count, - startS, countS, stride, startM, countM, imap, buf);') - -ifelse(`$1', `NC_FORMAT_CLASSIC',`', - `$1', `NC_FORMAT_64BIT_OFFSET',`', - `$1', `NC_FORMAT_NETCDF4_CLASSIC',`',` - foreach(`itype', - (`uchar,ushort,uint,longlong,ulonglong'),` - _CAT(`nerrs += blocking_put_',itype)'`(rank, ncid, dimids, start, count, - startS, countS, stride, startM, countM, imap, buf);')') + _CAT(`nerrs += blocking_put_',itype)'`(rank, coll_io, ncid, start, count, + startS, countS, stride, startM, countM, imap, buf);') + + if (format == NC_FORMAT_64BIT_DATA) { + foreach(`itype', + (`uchar,ushort,uint,longlong,ulonglong'),` + _CAT(`nerrs += blocking_put_',itype)'`(rank, coll_io, ncid, start, + count, startS, countS, stride, startM, countM, imap, buf);') + } /* close the file */ err = ncmpi_close(ncid); CHECK_ERR ')dnl -/*----< main() >------------------------------------------------------------*/ -int main(int argc, char **argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) { - char filename[256], fname[512], *hint_value, *cbuf; - int i, j, k, rank, nprocs, ncid, bufsize, err, nerrs=0, cmode; - int bb_enabled=0, psize[NDIMS], dimids[NDIMS], dim_rank[NDIMS]; + char fname[512], *cbuf; + int i, j, k, rank, nprocs, ncid, bufsize, err, nerrs=0; + int psize[NDIMS], dimids[NDIMS], dim_rank[NDIMS]; double *buf; MPI_Offset gsize[NDIMS], stride[NDIMS], imap[NDIMS]; MPI_Offset start[NDIMS], count[NDIMS]; MPI_Offset startS[NDIMS], countS[NDIMS]; MPI_Offset startM[NDIMS], countM[NDIMS]; - MPI_Info info; - MPI_Init(&argc,&argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for all kinds put APIs ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - - /* check whether burst buffering is enabled */ - if (inq_env_hint("nc_burst_buf", &hint_value)) { - if (strcasecmp(hint_value, "enable") == 0) bb_enabled = 1; - free(hint_value); - } - /* calculate number of processes along each dimension */ for (i=0; i 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + return nerrs; +} + +int main(int argc, char **argv) { + + int err; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "all kinds put APIs", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/testcases/put_parameter.f b/test/testcases/put_parameter.f index 5f3d78da29..c3a4641519 100644 --- a/test/testcases/put_parameter.f +++ b/test/testcases/put_parameter.f @@ -61,8 +61,8 @@ subroutine check(err, message) ! It is a good idea to check returned value for possible error if (err .NE. NF_NOERR) then write(6,*) message(1:XTRIM(message)), nfmpi_strerror(err) - msg = '*** TESTING F77 put_parameter.f for immutable put ' - call pass_fail(1, msg) + msg = '*** TESTING F77 put_parameter.f - immutable put ' + call pass_fail(1, msg, 0) STOP 2 end if end ! subroutine check @@ -76,32 +76,40 @@ program main PARAMETER(NX=4) data buffer /5,6,7,8/ - character(LEN=256) filename, cmd, msg + character(LEN=256) out_path, in_path, cmd, msg integer err, ierr, nprocs, rank, nerrs, get_args, XTRIM integer cmode, ncid, varid(2), dimid(2) integer*8 len_ll, start(2), count(2) integer*8 malloc_size, sum_size + logical keep_files + double precision timing call MPI_Init(ierr) + + timing = MPI_Wtime() + call MPI_Comm_rank(MPI_COMM_WORLD, rank, ierr) call MPI_Comm_size(MPI_COMM_WORLD, nprocs, ierr) - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any if (rank .EQ. 0) then - filename = "testfile.nc" - err = get_args(cmd, filename) + out_path = "testfile.nc" + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, + MPI_COMM_WORLD, ierr) + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, + + ierr) + nerrs = 0 ! create file, truncate it if exists cmode = IOR(NF_CLOBBER, NF_64BIT_OFFSET) - err = nfmpi_create(MPI_COMM_WORLD, filename, cmode, + err = nfmpi_create(MPI_COMM_WORLD, out_path, cmode, + MPI_INFO_NULL, ncid) call check(err, 'In nfmpi_create: ') @@ -163,10 +171,18 @@ program main + sum_size, ' bytes yet to be freed' endif + timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, + + MPI_DOUBLE_PRECISION, MPI_MAX, + + MPI_COMM_WORLD, ierr) if (rank .eq. 0) then + if (.NOT. keep_files) then + err = nfmpi_delete(out_path, MPI_INFO_NULL) + end if + msg = '*** TESTING F77 '//cmd(1:XTRIM(cmd))// - + ' for using immutable write buf ' - call pass_fail(nerrs, msg) + + ' - using immutable write buf ' + call pass_fail(nerrs, msg, timing) endif 999 call MPI_Finalize(ierr) diff --git a/test/testcases/record.c b/test/testcases/record.c index 170e1bfe39..5f00d96559 100644 --- a/test/testcases/record.c +++ b/test/testcases/record.c @@ -35,15 +35,17 @@ #include static -int test_only_record_var_1D(char *filename, int cmode) +int test_only_record_var_1D(const char *out_path, int format, MPI_Info info) { int ncid, varid, dimid, buf[20], err, nerrs=0; MPI_Offset start[1], count[1], length; - MPI_Info info=MPI_INFO_NULL; + + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR /* create a new file for writing ----------------------------------------*/ - cmode |= NC_CLOBBER; - err = ncmpi_create(MPI_COMM_SELF, filename, cmode, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_SELF, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR /* define dimension and variable */ err = ncmpi_def_dim(ncid, "REC_DIM", NC_UNLIMITED, &dimid); CHECK_ERR @@ -62,12 +64,15 @@ int test_only_record_var_1D(char *filename, int cmode) err = ncmpi_inq_dimlen(ncid, dimid, &length); CHECK_ERR if (length != 2) { - printf("Error at line %d in %s: expecting 2 records, but got "OFFFMT" record(s)\n", + fprintf(stderr,"Error at line %d in %s: expecting 2 records, but got "OFFFMT" record(s)\n", __LINE__,__FILE__,length); nerrs++; } - if (nerrs == 0 && !(cmode & NC_NETCDF4)) { /* test independent data mode */ + /* test independent data mode */ + if (nerrs == 0 && format != NC_FORMAT_NETCDF4 && + format != NC_FORMAT_NETCDF4_CLASSIC) { + err = ncmpi_begin_indep_data(ncid); CHECK_ERR /* write the 4th record */ buf[0] = 93; @@ -81,7 +86,7 @@ int test_only_record_var_1D(char *filename, int cmode) err = ncmpi_inq_dimlen(ncid, dimid, &length); CHECK_ERR if (length != 4) { - printf("Error at line %d in %s: expecting 4 records, but got "OFFFMT" record(s)\n", + fprintf(stderr,"Error at line %d in %s: expecting 4 records, but got "OFFFMT" record(s)\n", __LINE__,__FILE__,length); nerrs++; } @@ -92,15 +97,17 @@ int test_only_record_var_1D(char *filename, int cmode) } static -int test_only_record_var_3D(char *filename, int cmode) +int test_only_record_var_3D(const char *out_path, int format, MPI_Info info) { int i, ncid, varid, dimid[3], buf[20], err, nerrs=0; MPI_Offset start[3], count[3], length; - MPI_Info info=MPI_INFO_NULL; + + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR /* create a new file for writing ----------------------------------------*/ - cmode |= NC_CLOBBER; - err = ncmpi_create(MPI_COMM_SELF, filename, cmode, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_SELF, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR /* define dimension and variable */ err = ncmpi_def_dim(ncid, "REC_DIM", NC_UNLIMITED, &dimid[0]); CHECK_ERR @@ -123,12 +130,15 @@ int test_only_record_var_3D(char *filename, int cmode) err = ncmpi_inq_dimlen(ncid, dimid[0], &length); CHECK_ERR if (length != 2) { - printf("Error at line %d in %s: expecting 2 records, but got "OFFFMT" record(s)\n", + fprintf(stderr,"Error at line %d in %s: expecting 2 records, but got "OFFFMT" record(s)\n", __LINE__,__FILE__,length); nerrs++; } - if (nerrs == 0 && !(cmode & NC_NETCDF4)) { /* test independent data mode */ + /* test independent data mode */ + if (nerrs == 0 && format != NC_FORMAT_NETCDF4 && + format != NC_FORMAT_NETCDF4_CLASSIC) { + err = ncmpi_begin_indep_data(ncid); CHECK_ERR /* write the 4th record */ for (i=0; i<20; i++) buf[i] = 93; @@ -142,7 +152,7 @@ int test_only_record_var_3D(char *filename, int cmode) err = ncmpi_inq_dimlen(ncid, dimid[0], &length); CHECK_ERR if (length != 4) { - printf("Error at line %d in %s: expecting 4 records, but got "OFFFMT" record(s)\n", + fprintf(stderr,"Error at line %d in %s: expecting 4 records, but got "OFFFMT" record(s)\n", __LINE__,__FILE__,length); nerrs++; } @@ -153,15 +163,17 @@ int test_only_record_var_3D(char *filename, int cmode) } static -int test_two_record_var(char *filename, int cmode) +int test_two_record_var(const char *out_path, int format, MPI_Info info) { int i, ncid, varid[2], dimid[3], buf[20], err, nerrs=0; MPI_Offset start[3], count[3], length; - MPI_Info info=MPI_INFO_NULL; + + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR /* create a new file for writing ----------------------------------------*/ - cmode |= NC_CLOBBER; - err = ncmpi_create(MPI_COMM_SELF, filename, cmode, info, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_SELF, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR /* define dimension and variable */ err = ncmpi_def_dim(ncid, "REC_DIM", NC_UNLIMITED, &dimid[0]); CHECK_ERR @@ -183,21 +195,21 @@ int test_two_record_var(char *filename, int cmode) err = ncmpi_inq_dimlen(ncid, dimid[0], &length); CHECK_ERR if (length != 2) { - printf("Error at line %d in %s: expecting 2 records, but got "OFFFMT" record(s)\n", + fprintf(stderr,"Error at line %d in %s: expecting 2 records, but got "OFFFMT" record(s)\n", __LINE__,__FILE__,length); nerrs++; } if (nerrs == 0) { /* test independent data mode */ /* writing new records, HDF5 requires collective I/O */ - if (!(cmode & NC_NETCDF4)) { + if (format != NC_FORMAT_NETCDF4 && format != NC_FORMAT_NETCDF4_CLASSIC) { err = ncmpi_begin_indep_data(ncid); CHECK_ERR } /* write the 4th record */ buf[0] = 93; start[0] = 3; count[0] = 1; - if (!(cmode & NC_NETCDF4)) { + if (format != NC_FORMAT_NETCDF4 && format != NC_FORMAT_NETCDF4_CLASSIC) { err = ncmpi_put_vara_int(ncid, varid[0], start, count, buf); CHECK_ERR } else { err = ncmpi_put_vara_int_all(ncid, varid[0], start, count, buf); CHECK_ERR @@ -206,7 +218,7 @@ int test_two_record_var(char *filename, int cmode) /* write the 3rd and 4th records */ buf[0] = 92; buf[1] = 93; start[0] = 2; count[0] = 2; - if (!(cmode & NC_NETCDF4)) { + if (format != NC_FORMAT_NETCDF4 && format != NC_FORMAT_NETCDF4_CLASSIC) { err = ncmpi_put_vara_int(ncid, varid[0], start, count, buf); CHECK_ERR } else { err = ncmpi_put_vara_int_all(ncid, varid[0], start, count, buf); CHECK_ERR @@ -214,11 +226,11 @@ int test_two_record_var(char *filename, int cmode) err = ncmpi_inq_dimlen(ncid, dimid[0], &length); CHECK_ERR if (length != 4) { - printf("Error at line %d in %s: expecting 4 records, but got "OFFFMT" record(s)\n", + fprintf(stderr,"Error at line %d in %s: expecting 4 records, but got "OFFFMT" record(s)\n", __LINE__,__FILE__,length); nerrs++; } - if (!(cmode & NC_NETCDF4)) { + if (format != NC_FORMAT_NETCDF4 && format != NC_FORMAT_NETCDF4_CLASSIC) { err = ncmpi_end_indep_data(ncid); CHECK_ERR } } @@ -237,21 +249,21 @@ int test_two_record_var(char *filename, int cmode) err = ncmpi_inq_dimlen(ncid, dimid[0], &length); CHECK_ERR if (length != 4) { - printf("Error at line %d in %s: expecting 4 records, but got "OFFFMT" record(s)\n", + fprintf(stderr,"Error at line %d in %s: expecting 4 records, but got "OFFFMT" record(s)\n", __LINE__,__FILE__,length); nerrs++; } if (nerrs == 0) { /* test independent data mode */ /* writing new records, HDF5 requires collective I/O */ - if (!(cmode & NC_NETCDF4)) { + if (format != NC_FORMAT_NETCDF4 && format != NC_FORMAT_NETCDF4_CLASSIC) { err = ncmpi_begin_indep_data(ncid); CHECK_ERR } /* write the 4th record */ for (i=0; i<20; i++) buf[i] = 93; start[0] = 3; - if (!(cmode & NC_NETCDF4)) { + if (format != NC_FORMAT_NETCDF4 && format != NC_FORMAT_NETCDF4_CLASSIC) { err = ncmpi_put_vara_int(ncid, varid[1], start, count, buf); CHECK_ERR } else { err = ncmpi_put_vara_int_all(ncid, varid[1], start, count, buf); CHECK_ERR @@ -260,7 +272,7 @@ int test_two_record_var(char *filename, int cmode) /* write the 3rd record */ for (i=0; i<20; i++) buf[i] = 92; start[0] = 2; - if (!(cmode & NC_NETCDF4)) { + if (format != NC_FORMAT_NETCDF4 && format != NC_FORMAT_NETCDF4_CLASSIC) { err = ncmpi_put_vara_int(ncid, varid[1], start, count, buf); CHECK_ERR } else { err = ncmpi_put_vara_int_all(ncid, varid[1], start, count, buf); CHECK_ERR @@ -268,11 +280,11 @@ int test_two_record_var(char *filename, int cmode) err = ncmpi_inq_dimlen(ncid, dimid[0], &length); CHECK_ERR if (length != 4) { - printf("Error at line %d in %s: expecting 4 records, but got "OFFFMT" record(s)\n", + fprintf(stderr,"Error at line %d in %s: expecting 4 records, but got "OFFFMT" record(s)\n", __LINE__,__FILE__,length); nerrs++; } - if (!(cmode & NC_NETCDF4)) { + if (format != NC_FORMAT_NETCDF4 && format != NC_FORMAT_NETCDF4_CLASSIC) { err = ncmpi_end_indep_data(ncid); CHECK_ERR } } @@ -280,92 +292,56 @@ int test_two_record_var(char *filename, int cmode) return nerrs; } -int main(int argc, char** argv) { - char filename[256], *hint_value; - int nerrs=0, rank, nprocs, err, bb_enabled=0; +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) +{ + char val[MPI_MAX_INFO_VAL]; + int err=0, rank, flag; + + /* check whether burst buffering is enabled */ + MPI_Info_get(info, "nc_burst_buf", MPI_MAX_INFO_VAL - 1, val, &flag); + if (flag && strcasecmp(val, "enable") == 0 && + (format == NC_FORMAT_NETCDF4 || format == NC_FORMAT_NETCDF4_CLASSIC)) + /* does not work for NetCDF4 files when burst-buffering is enabled */ + return 0; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + if (rank >= 1) return 0; /* this test is for running 1 process */ - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - goto fn_exit; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for write records in reversed order", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } + err = test_only_record_var_1D(out_path, format, info); + if (err > 0) return err; + err = test_only_record_var_3D(out_path, format, info); + if (err > 0) return err; + err = test_two_record_var(out_path, format, info); + if (err > 0) return err; - if (rank >= 1) goto fn_exit; /* this test is for running 1 process */ + return err; +} - /* check whether burst buffering is enabled */ - if (inq_env_hint("nc_burst_buf", &hint_value)) { - if (strcasecmp(hint_value, "enable") == 0) bb_enabled = 1; - free(hint_value); - } +int main(int argc, char **argv) { - /* CDF-1: test only one 1D record variable */ - nerrs += test_only_record_var_1D(filename, 0); - /* CDF-1: test only one 3D record variable */ - nerrs += test_only_record_var_3D(filename, 0); - /* CDF-1: test two record variables */ - nerrs += test_two_record_var(filename, 0); - - /* CDF-2: test only one 1D record variable */ - nerrs += test_only_record_var_1D(filename, NC_64BIT_OFFSET); - /* CDF-2: test only one 3D record variable */ - nerrs += test_only_record_var_3D(filename, NC_64BIT_OFFSET); - /* CDF-2: test two record variables */ - nerrs += test_two_record_var(filename, NC_64BIT_OFFSET); - - if (!bb_enabled) { -#ifdef USE_NETCDF4 - /* NETCDF4: test only one 1D record variable */ - nerrs += test_only_record_var_1D(filename, NC_NETCDF4); - /* NETCDF4: test only one 3D record variable */ - nerrs += test_only_record_var_3D(filename, NC_NETCDF4); - /* NETCDF4: test two record variables */ - nerrs += test_two_record_var(filename, NC_NETCDF4); - - /* NETCDF4_CLASSIC: test only one 1D record variable */ - nerrs += test_only_record_var_1D(filename, NC_NETCDF4|NC_CLASSIC_MODEL); - /* NETCDF4_CLASSIC: test only one 3D record variable */ - nerrs += test_only_record_var_3D(filename, NC_NETCDF4|NC_CLASSIC_MODEL); - /* NETCDF4_CLASSIC: test two record variables */ - nerrs += test_two_record_var(filename, NC_NETCDF4|NC_CLASSIC_MODEL); -#endif - } + int err; + loop_opts opt; - /* CDF-5: test only one 1D record variable */ - nerrs += test_only_record_var_1D(filename, NC_64BIT_DATA); - /* CDF-5: test only one 3D record variable */ - nerrs += test_only_record_var_3D(filename, NC_64BIT_DATA); - /* CDF-5: test two record variables */ - nerrs += test_two_record_var(filename, NC_64BIT_DATA); - - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR && malloc_size > 0) { /* this test is for running 1 process */ - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - malloc_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + MPI_Init(&argc, &argv); -fn_exit: - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "write records in reversed order", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/testcases/redef1.c b/test/testcases/redef1.c index 21774f4267..51a15352e4 100644 --- a/test/testcases/redef1.c +++ b/test/testcases/redef1.c @@ -26,10 +26,13 @@ #include -static int -tst_fmt(char *filename, int cmode) +static +int redef1(const char *out_path, + int format, + int coll_io, + MPI_Info info) { - int i, j, k, rank, ncid, err, nerrs=0; + int i, j, k, nprocs, rank, ncid, err, nerrs=0; int dim0id, dim1id, dim5id, dim9id, dim2id, dimsid[2], dims2id[2]; int varid, var3id, var4id, var2id; int *data; @@ -38,11 +41,21 @@ tst_fmt(char *filename, int cmode) MPI_Offset start[2], count[2]; MPI_Comm comm = MPI_COMM_WORLD; - MPI_Comm_rank(comm, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + +#ifdef DEBUG + if (nprocs > 1 && rank == 0) + printf("Warning: %s is designed to run on 1 process\n", + basename(__FILE__)); +#endif + + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR /* Test NetCDF 4 first as ncvalidator checks only classic formats */ - cmode |= NC_CLOBBER; - err = ncmpi_create(comm, filename, cmode, MPI_INFO_NULL, &ncid); CHECK_ERR + err = ncmpi_create(comm, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "dim0", len0, &dim0id); CHECK_ERR err = ncmpi_def_dim(ncid, "dim1", len1, &dim1id); CHECK_ERR @@ -63,6 +76,11 @@ tst_fmt(char *filename, int cmode) err = ncmpi_enddef(ncid); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + /* put data */ start[0] = 0; start[1] = 0; @@ -75,7 +93,10 @@ tst_fmt(char *filename, int cmode) for (j=0; j 0) count[0] = count[1] = 0; - err = ncmpi_put_vara_int_all(ncid, varid, start, count, &data[0]); + if (coll_io) + err = ncmpi_put_vara_int_all(ncid, varid, start, count, &data[0]); + else + err = ncmpi_put_vara_int(ncid, varid, start, count, &data[0]); CHECK_ERR free(data); @@ -87,7 +108,10 @@ tst_fmt(char *filename, int cmode) for (j=0; j 0) count[0] = count[1] = 0; - err = ncmpi_put_vara_int_all(ncid, var3id, start, count, &data[0]); + if (coll_io) + err = ncmpi_put_vara_int_all(ncid, var3id, start, count, &data[0]); + else + err = ncmpi_put_vara_int(ncid, var3id, start, count, &data[0]); CHECK_ERR free(data); @@ -99,13 +123,21 @@ tst_fmt(char *filename, int cmode) for (j=0; j 0) count[0] = count[1] = 0; - err = ncmpi_put_vara_int_all(ncid, var4id, start, count, &data[0]); + if (coll_io) + err = ncmpi_put_vara_int_all(ncid, var4id, start, count, &data[0]); + else + err = ncmpi_put_vara_int(ncid, var4id, start, count, &data[0]); CHECK_ERR free(data); + /* file sync before file close and re-open it */ + err = ncmpi_sync(ncid); + CHECK_ERR + MPI_Barrier(MPI_COMM_WORLD); + err = ncmpi_close(ncid); CHECK_ERR - err = ncmpi_open(comm, filename, NC_WRITE, MPI_INFO_NULL, &ncid); + err = ncmpi_open(comm, out_path, NC_WRITE, info, &ncid); CHECK_ERR err = ncmpi_redef(ncid); CHECK_ERR @@ -119,6 +151,11 @@ tst_fmt(char *filename, int cmode) err = ncmpi_enddef(ncid); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + start[0] = 0; start[1] = 0; count[0] = len0; @@ -131,7 +168,10 @@ tst_fmt(char *filename, int cmode) k++; } if (rank > 0) count[0] = count[1] = 0; - err = ncmpi_put_vara_double_all(ncid, var2id, start, count, &dbl_data[0]); + if (coll_io) + err = ncmpi_put_vara_double_all(ncid, var2id, start, count, &dbl_data[0]); + else + err = ncmpi_put_vara_double(ncid, var2id, start, count, &dbl_data[0]); CHECK_ERR free(dbl_data); @@ -140,69 +180,47 @@ tst_fmt(char *filename, int cmode) return nerrs; } -int main(int argc, char** argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) { - char filename[256], *hint_value; - int commsize, rank, err, nerrs=0, bb_enabled=0; - MPI_Comm comm = MPI_COMM_WORLD; + int nerrs; - MPI_Init(&argc, &argv); - MPI_Comm_size(comm, &commsize); - MPI_Comm_rank(comm, &rank); + /* test without setting hint nc_data_move_chunk_size */ + nerrs = redef1(out_path, format, coll_io, info); + if (nerrs > 0) return nerrs; - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "redef2.nc"); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for entering re-define mode ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } + /* test with setting hint nc_data_move_chunk_size */ + MPI_Info_set(info, "nc_data_move_chunk_size", "100"); + nerrs = redef1(out_path, format, coll_io, info); + if (nerrs > 0) return nerrs; -#ifdef DEBUG - if (commsize > 1 && rank == 0) - printf("Warning: %s is designed to run on 1 process\n",argv[0]); -#endif + return 0; +} - /* check whether burst buffering is enabled */ - if (inq_env_hint("nc_burst_buf", &hint_value)) { - if (strcasecmp(hint_value, "enable") == 0) bb_enabled = 1; - free(hint_value); - } +int main(int argc, char **argv) { - nerrs += tst_fmt(filename, 0); - nerrs += tst_fmt(filename, NC_64BIT_OFFSET); - if (!bb_enabled) { -#ifdef ENABLE_NETCDF4 - nerrs += tst_fmt(filename, NC_NETCDF4); - nerrs += tst_fmt(filename, NC_NETCDF4 | NC_CLASSIC_MODEL); -#endif - } - nerrs += tst_fmt(filename, NC_64BIT_DATA); - - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + int err; + loop_opts opt; - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 0; /* test GIO driver only */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "entering re-define mode", opt, test_io); MPI_Finalize(); - return (nerrs > 0); + + return err; } diff --git a/test/testcases/scalar.c b/test/testcases/scalar.c index 098b6f5801..6870eb4905 100644 --- a/test/testcases/scalar.c +++ b/test/testcases/scalar.c @@ -20,19 +20,27 @@ static int -tst_fmt(char *filename, int cmode) +tst_fmt(const char *out_path, int format, int coll_io, MPI_Info info) { - int err, nerrs=0, ncid, varid, buf; + int err, ncid, varid, buf; MPI_Offset start[1], count[1], stride[1], imap[1]; - /* Test NetCDF-4 first as ncvalidator checks only classic formats */ - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER | cmode, - MPI_INFO_NULL, &ncid); CHECK_ERR + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR /* define a scalar variable of integer type */ err = ncmpi_def_var(ncid, "scalar_var", NC_INT, 0, NULL, &varid); CHECK_ERR err = ncmpi_enddef(ncid); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + buf = 1; start[0] = 1; count[0] = 2; @@ -40,71 +48,133 @@ tst_fmt(char *filename, int cmode) imap[0] = 2; /* put */ - err = ncmpi_put_var1_int_all(ncid, varid, NULL, &buf); CHECK_ERR - err = ncmpi_put_var1_int_all(ncid, varid, start, &buf); CHECK_ERR - - err = ncmpi_put_vara_int_all(ncid, varid, start, count, &buf); CHECK_ERR - err = ncmpi_put_vara_int_all(ncid, varid, NULL, count, &buf); CHECK_ERR - err = ncmpi_put_vara_int_all(ncid, varid, start, NULL, &buf); CHECK_ERR - err = ncmpi_put_vara_int_all(ncid, varid, NULL, NULL, &buf); CHECK_ERR - - err = ncmpi_put_vars_int_all(ncid, varid, start, count, stride, &buf); CHECK_ERR - err = ncmpi_put_vars_int_all(ncid, varid, NULL, count, stride, &buf); CHECK_ERR - err = ncmpi_put_vars_int_all(ncid, varid, start, NULL, stride, &buf); CHECK_ERR - err = ncmpi_put_vars_int_all(ncid, varid, start, count, NULL, &buf); CHECK_ERR - err = ncmpi_put_vars_int_all(ncid, varid, NULL, NULL, NULL, &buf); CHECK_ERR + if (coll_io) { + err = ncmpi_put_var1_int_all(ncid, varid, NULL, &buf); CHECK_ERR + err = ncmpi_put_var1_int_all(ncid, varid, start, &buf); CHECK_ERR + + err = ncmpi_put_vara_int_all(ncid, varid, start, count, &buf); CHECK_ERR + err = ncmpi_put_vara_int_all(ncid, varid, NULL, count, &buf); CHECK_ERR + err = ncmpi_put_vara_int_all(ncid, varid, start, NULL, &buf); CHECK_ERR + err = ncmpi_put_vara_int_all(ncid, varid, NULL, NULL, &buf); CHECK_ERR + + err = ncmpi_put_vars_int_all(ncid, varid, start, count, stride, &buf); CHECK_ERR + err = ncmpi_put_vars_int_all(ncid, varid, NULL, count, stride, &buf); CHECK_ERR + err = ncmpi_put_vars_int_all(ncid, varid, start, NULL, stride, &buf); CHECK_ERR + err = ncmpi_put_vars_int_all(ncid, varid, start, count, NULL, &buf); CHECK_ERR + err = ncmpi_put_vars_int_all(ncid, varid, NULL, NULL, NULL, &buf); CHECK_ERR + + err = ncmpi_put_varm_int_all(ncid, varid, start, count, stride, imap, &buf); CHECK_ERR + err = ncmpi_put_varm_int_all(ncid, varid, NULL, NULL, NULL, NULL, &buf); CHECK_ERR + } + else { + err = ncmpi_put_var1_int(ncid, varid, NULL, &buf); CHECK_ERR + err = ncmpi_put_var1_int(ncid, varid, start, &buf); CHECK_ERR + + err = ncmpi_put_vara_int(ncid, varid, start, count, &buf); CHECK_ERR + err = ncmpi_put_vara_int(ncid, varid, NULL, count, &buf); CHECK_ERR + err = ncmpi_put_vara_int(ncid, varid, start, NULL, &buf); CHECK_ERR + err = ncmpi_put_vara_int(ncid, varid, NULL, NULL, &buf); CHECK_ERR + + err = ncmpi_put_vars_int(ncid, varid, start, count, stride, &buf); CHECK_ERR + err = ncmpi_put_vars_int(ncid, varid, NULL, count, stride, &buf); CHECK_ERR + err = ncmpi_put_vars_int(ncid, varid, start, NULL, stride, &buf); CHECK_ERR + err = ncmpi_put_vars_int(ncid, varid, start, count, NULL, &buf); CHECK_ERR + err = ncmpi_put_vars_int(ncid, varid, NULL, NULL, NULL, &buf); CHECK_ERR + + err = ncmpi_put_varm_int(ncid, varid, start, count, stride, imap, &buf); CHECK_ERR + err = ncmpi_put_varm_int(ncid, varid, NULL, NULL, NULL, NULL, &buf); CHECK_ERR + } - err = ncmpi_put_varm_int_all(ncid, varid, start, count, stride, imap, &buf); CHECK_ERR - err = ncmpi_put_varm_int_all(ncid, varid, NULL, NULL, NULL, NULL, &buf); CHECK_ERR + /* file sync before reading */ + err = ncmpi_sync(ncid); + CHECK_ERR + MPI_Barrier(MPI_COMM_WORLD); err = ncmpi_close(ncid); CHECK_ERR - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); CHECK_ERR + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_NOWRITE, info, &ncid); CHECK_ERR + + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } err = ncmpi_inq_varid(ncid, "scalar_var", &varid); CHECK_ERR /* get */ - err = ncmpi_get_var1_int_all(ncid, varid, NULL, &buf); CHECK_ERR - err = ncmpi_get_var1_int_all(ncid, varid, start, &buf); CHECK_ERR - - err = ncmpi_get_vara_int_all(ncid, varid, start, count, &buf); CHECK_ERR - err = ncmpi_get_vara_int_all(ncid, varid, NULL, count, &buf); CHECK_ERR - err = ncmpi_get_vara_int_all(ncid, varid, start, NULL, &buf); CHECK_ERR - err = ncmpi_get_vara_int_all(ncid, varid, NULL, NULL, &buf); CHECK_ERR - - err = ncmpi_get_vars_int_all(ncid, varid, start, count, stride, &buf); CHECK_ERR - err = ncmpi_get_vars_int_all(ncid, varid, NULL, count, stride, &buf); CHECK_ERR - err = ncmpi_get_vars_int_all(ncid, varid, start, NULL, stride, &buf); CHECK_ERR - err = ncmpi_get_vars_int_all(ncid, varid, start, count, NULL, &buf); CHECK_ERR - err = ncmpi_get_vars_int_all(ncid, varid, NULL, NULL, NULL, &buf); CHECK_ERR - - err = ncmpi_get_varm_int_all(ncid, varid, start, count, stride, imap, &buf); CHECK_ERR - err = ncmpi_get_varm_int_all(ncid, varid, NULL, NULL, NULL, NULL, &buf); CHECK_ERR + if (coll_io) { + err = ncmpi_get_var1_int_all(ncid, varid, NULL, &buf); CHECK_ERR + err = ncmpi_get_var1_int_all(ncid, varid, start, &buf); CHECK_ERR + + err = ncmpi_get_vara_int_all(ncid, varid, start, count, &buf); CHECK_ERR + err = ncmpi_get_vara_int_all(ncid, varid, NULL, count, &buf); CHECK_ERR + err = ncmpi_get_vara_int_all(ncid, varid, start, NULL, &buf); CHECK_ERR + err = ncmpi_get_vara_int_all(ncid, varid, NULL, NULL, &buf); CHECK_ERR + + err = ncmpi_get_vars_int_all(ncid, varid, start, count, stride, &buf); CHECK_ERR + err = ncmpi_get_vars_int_all(ncid, varid, NULL, count, stride, &buf); CHECK_ERR + err = ncmpi_get_vars_int_all(ncid, varid, start, NULL, stride, &buf); CHECK_ERR + err = ncmpi_get_vars_int_all(ncid, varid, start, count, NULL, &buf); CHECK_ERR + err = ncmpi_get_vars_int_all(ncid, varid, NULL, NULL, NULL, &buf); CHECK_ERR + + err = ncmpi_get_varm_int_all(ncid, varid, start, count, stride, imap, &buf); CHECK_ERR + err = ncmpi_get_varm_int_all(ncid, varid, NULL, NULL, NULL, NULL, &buf); CHECK_ERR + } + else { + err = ncmpi_get_var1_int(ncid, varid, NULL, &buf); CHECK_ERR + err = ncmpi_get_var1_int(ncid, varid, start, &buf); CHECK_ERR + + err = ncmpi_get_vara_int(ncid, varid, start, count, &buf); CHECK_ERR + err = ncmpi_get_vara_int(ncid, varid, NULL, count, &buf); CHECK_ERR + err = ncmpi_get_vara_int(ncid, varid, start, NULL, &buf); CHECK_ERR + err = ncmpi_get_vara_int(ncid, varid, NULL, NULL, &buf); CHECK_ERR + + err = ncmpi_get_vars_int(ncid, varid, start, count, stride, &buf); CHECK_ERR + err = ncmpi_get_vars_int(ncid, varid, NULL, count, stride, &buf); CHECK_ERR + err = ncmpi_get_vars_int(ncid, varid, start, NULL, stride, &buf); CHECK_ERR + err = ncmpi_get_vars_int(ncid, varid, start, count, NULL, &buf); CHECK_ERR + err = ncmpi_get_vars_int(ncid, varid, NULL, NULL, NULL, &buf); CHECK_ERR + + err = ncmpi_get_varm_int(ncid, varid, start, count, stride, imap, &buf); CHECK_ERR + err = ncmpi_get_varm_int(ncid, varid, NULL, NULL, NULL, NULL, &buf); CHECK_ERR + } err = ncmpi_close(ncid); CHECK_ERR - return nerrs; + + return 0; } #define WAIT_CHECK { \ CHECK_ERR \ - err = ncmpi_wait_all(ncid, NC_REQ_ALL, NULL, NULL); \ + if (coll_io) \ + err = ncmpi_wait_all(ncid, NC_REQ_ALL, NULL, NULL); \ + else \ + err = ncmpi_wait(ncid, NC_REQ_ALL, NULL, NULL); \ CHECK_ERR \ } static int -tst_fmt_nb(char *filename, int cmode) +tst_fmt_nb(const char *out_path, int format, int coll_io, MPI_Info info) { - int err, nerrs=0, ncid, varid, buf; + int err, ncid, varid, buf; MPI_Offset start[1], count[1], stride[1], imap[1]; - /* No test NetCDF-4 as nonblocking APIs are not defined in NetCDF-4 */ - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER | cmode, - MPI_INFO_NULL, &ncid); CHECK_ERR + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR /* define a scalar variable of integer type */ err = ncmpi_def_var(ncid, "scalar_var", NC_INT, 0, NULL, &varid); CHECK_ERR err = ncmpi_enddef(ncid); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + buf = 1; start[0] = 1; count[0] = 2; @@ -129,9 +199,19 @@ tst_fmt_nb(char *filename, int cmode) err = ncmpi_iput_varm_int(ncid, varid, start, count, stride, imap, &buf, NULL); WAIT_CHECK err = ncmpi_iput_varm_int(ncid, varid, NULL, NULL, NULL, NULL, &buf, NULL); WAIT_CHECK + /* file sync before reading */ + err = ncmpi_sync(ncid); + CHECK_ERR + MPI_Barrier(MPI_COMM_WORLD); + err = ncmpi_close(ncid); CHECK_ERR - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); CHECK_ERR + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_NOWRITE, info, &ncid); CHECK_ERR + + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } err = ncmpi_inq_varid(ncid, "scalar_var", &varid); CHECK_ERR @@ -154,79 +234,65 @@ tst_fmt_nb(char *filename, int cmode) err = ncmpi_iget_varm_int(ncid, varid, NULL, NULL, NULL, NULL, &buf, NULL); WAIT_CHECK err = ncmpi_close(ncid); CHECK_ERR - return nerrs; + + return 0; } -/*----< main() >------------------------------------------------------------*/ -int main(int argc, char **argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) { - char filename[256], *hint_value; - int err, nerrs=0, rank, nprocs, bb_enabled=0; + char val[MPI_MAX_INFO_VAL]; + int nerrs=0, flag; - MPI_Init(&argc, &argv); +#ifdef DEBUG + int rank, nprocs; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for get/put scalar variables ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - -#ifdef DEBUG if (nprocs > 1 && rank == 0) printf("Warning: %s is designed to run on 1 process\n", argv[0]); #endif /* check whether burst buffering is enabled */ - if (inq_env_hint("nc_burst_buf", &hint_value)) { - if (strcasecmp(hint_value, "enable") == 0) bb_enabled = 1; - free(hint_value); - } + MPI_Info_get(info, "nc_burst_buf", MPI_MAX_INFO_VAL - 1, val, &flag); + if (flag && strcasecmp(val, "enable") == 0 && + (format == NC_FORMAT_NETCDF4 || format == NC_FORMAT_NETCDF4_CLASSIC)) + /* does not work for NetCDF4 files when burst-buffering is enabled */ + return 0; + + /* test blocking APIs */ + nerrs += tst_fmt(out_path, format, coll_io, info); /* test nonblocking APIs */ - nerrs += tst_fmt_nb(filename, 0); - nerrs += tst_fmt_nb(filename, NC_64BIT_OFFSET); - nerrs += tst_fmt_nb(filename, NC_64BIT_DATA); + if (format != NC_FORMAT_NETCDF4 && format != NC_FORMAT_NETCDF4_CLASSIC) + nerrs += tst_fmt_nb(out_path, format, coll_io, info); - /* test blocking APIs */ - nerrs += tst_fmt(filename, 0); - nerrs += tst_fmt(filename, NC_64BIT_OFFSET); - if (!bb_enabled) { -#ifdef ENABLE_NETCDF4 - nerrs += tst_fmt(filename, NC_NETCDF4); - nerrs += tst_fmt(filename, NC_NETCDF4 | NC_CLASSIC_MODEL); -#endif - } - nerrs += tst_fmt(filename, NC_64BIT_DATA); - - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + return nerrs; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +int main(int argc, char **argv) { + + int err; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "get/put scalar variables", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/testcases/seq_runs.sh b/test/testcases/seq_runs.sh index 270536cc6b..ad84f4164a 100755 --- a/test/testcases/seq_runs.sh +++ b/test/testcases/seq_runs.sh @@ -1,80 +1,38 @@ -#!/bin/sh +#!/bin/bash # -# Copyright (C) 2003, Northwestern University and Argonne National Laboratory +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory # See COPYRIGHT notice in top-level directory. # # Exit immediately if a command exits with a non-zero status. set -e -VALIDATOR=../../src/utils/ncvalidator/ncvalidator +DRY_RUN=no +VERBOSE=no -# prevent user environment setting of PNETCDF_HINTS to interfere -unset PNETCDF_HINTS - -${TESTSEQRUN} ./tst_version - -${TESTSEQRUN} ./put_all_kinds ${TESTOUTDIR}/put_all_kinds.nc -${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/put_all_kinds.nc1 -${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/put_all_kinds.nc2 -${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/put_all_kinds.nc5 - -${TESTSEQRUN} ./iput_all_kinds ${TESTOUTDIR}/iput_all_kinds.nc -${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/iput_all_kinds.nc1 -${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/iput_all_kinds.nc2 -${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/iput_all_kinds.nc5 - -NCMPIGEN=../../src/utils/ncmpigen/ncmpigen -NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff - -# remove the file system type prefix name if there is any. -OUT_PATH=`echo "$TESTOUTDIR" | cut -d: -f2-` +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $TESTSEQRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $TESTSEQRUN $@ + fi +} -rm -f ${OUT_PATH}/testfile.nc ${OUT_PATH}/redef1.nc -${TESTSEQRUN} ${NCMPIGEN} -v 5 -o ${TESTOUTDIR}/redef1.nc ${srcdir}/redef-good.ncdump -${TESTSEQRUN} ./redef1 ${TESTOUTDIR}/testfile.nc -${TESTSEQRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/testfile.nc ${TESTOUTDIR}/redef1.nc -# diff -q ${OUT_PATH}/testfile.nc ${OUT_PATH}/redef1.nc +exe_name=`basename $1` -${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/testfile.nc -rm -f ${OUT_PATH}/redef1.nc -rm -f ${OUT_PATH}/testfile.nc - -# echo "" - -if test "x${ENABLE_BURST_BUFFER}" = x1 ; then - echo "" - echo "---- testing burst buffering" - - # Run using burst buffer driver - export PNETCDF_HINTS="nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" - ${TESTSEQRUN} ./put_all_kinds ${TESTOUTDIR}/put_all_kinds.bb.nc - ${TESTSEQRUN} ./iput_all_kinds ${TESTOUTDIR}/iput_all_kinds.bb.nc - unset PNETCDF_HINTS +# prevent user environment setting of PNETCDF_HINTS to interfere +unset PNETCDF_HINTS - # Compare - for i in 1 2 5 ; do - ${TESTSEQRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/put_all_kinds.nc$i ${TESTOUTDIR}/put_all_kinds.bb.nc$i - ${TESTSEQRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/iput_all_kinds.nc$i ${TESTOUTDIR}/iput_all_kinds.bb.nc$i - done +# vard APIs have deprecated +if test "x$exe_name" = xtest_vard || + test "x$exe_name" = xtest_vard_multiple || + test "x$exe_name" = xtest_vard_rec || + test "x$exe_name" = xtest_vardf90 || + test "x$exe_name" = xtest_vardf ; then + export PNETCDF_HINTS="nc_driver=mpiio" fi -rm -f ${OUT_PATH}/put_all_kinds.nc* -rm -f ${OUT_PATH}/put_all_kinds.bb.nc* -rm -f ${OUT_PATH}/iput_all_kinds.nc* -rm -f ${OUT_PATH}/iput_all_kinds.bb.nc* - -# echo "" -if test "${ENABLE_THREAD_SAFE}" = 1 ; then - # echo "---- testing thread safety" - for j in 0 1 ; do - export PNETCDF_SAFE_MODE=$j - # echo "---- set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" +run_cmd ./$1 -q -o ${TESTOUTDIR}/${exe_name}.nc - ${TESTSEQRUN} ./tst_pthread ${TESTOUTDIR}/tst_pthread.nc - for i in 0 1 2 3 4 5 ; do - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/tst_pthread.nc.$i - rm -f ${OUT_PATH}/tst_pthread.nc.$i - done - done -fi diff --git a/test/testcases/test_erange.c b/test/testcases/test_erange.c index 5d354f6711..a0f8d1d246 100644 --- a/test/testcases/test_erange.c +++ b/test/testcases/test_erange.c @@ -10,7 +10,7 @@ * This program tests whether NC_ERANGE error code can be reported correctly. * Note in CDF-1 and CDF-2, a special case is made to NOT report NC_ERANGE * when the variable is of NC_BYTE type and the calling APIs are of uchar. See - * http://www.unidata.ucar.edu/software/netcdf/docs/data_type.html#type_conversion + * https://docs.unidata.ucar.edu/nug/current/md_types.html#data_type * * In CDF-5, NC_ERANGE is checked for when the external data type mismatches the * internal one. @@ -38,38 +38,51 @@ #include static -int test_cdf12(char *filename, int bb_enabled, int cmode) +int test_cdf12(const char *filename, int format, int coll_io, MPI_Info info) { - int err, nerrs=0, ncid, vid, fvid, dvid, dimid; - unsigned char uc[1]; - signed char sc[1]; - int si[1]; + char val[MPI_MAX_INFO_VAL]; + int err, nerrs=0, ncid, vid, fvid, dvid, dimid, flag, bb_enabled=0; + unsigned char uc; + signed char sc; + int si; double dbuf; - cmode |= NC_CLOBBER; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); CHECK_ERR + /* check whether burst buffering is enabled */ + MPI_Info_get(info, "nc_burst_buf", MPI_MAX_INFO_VAL - 1, val, &flag); + if (flag && strcasecmp(val, "enable") == 0) { + bb_enabled = 1; + /* does not work for NetCDF4 files when burst-buffering is enabled */ + if (format == NC_FORMAT_NETCDF4 || format == NC_FORMAT_NETCDF4_CLASSIC) + return 0; + } + + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, info, &ncid); CHECK_ERR /* for CDF-1 and CDF-2, a special case is made: there is no NC_ERANGE * error can occur converting between NC_BYTE and unsigned char. - * http://www.unidata.ucar.edu/software/netcdf/docs/data_type.html#type_conversion + * https://docs.unidata.ucar.edu/nug/current/md_types.html#data_type * In brief, NC_BYTE is signed in all signed CDF-2 APIs, and unsigned in * all unsigned APIs. In CDF-2, there is only one unsigned API, _uchar. */ - uc[0] = 255; - err = ncmpi_put_att_uchar(ncid, NC_GLOBAL, "att1", NC_BYTE, 1, uc); CHECK_ERR - uc[0] = 0; /* initialize with a number that is not 0 */ - err = ncmpi_get_att_uchar(ncid, NC_GLOBAL, "att1", uc); CHECK_ERR - if (uc[0] != 255) { - printf("Error at line %d: unexpected read value %d (expecting 255)\n",__LINE__,(int)uc[0]); - nerrs++; + uc = 255; + err = ncmpi_put_att_uchar(ncid, NC_GLOBAL, "att1", NC_BYTE, 1, &uc); CHECK_ERR + uc = 0; /* initialize with a number that is not 0 */ + err = ncmpi_get_att_uchar(ncid, NC_GLOBAL, "att1", &uc); CHECK_ERR + if (uc != 255) { + fprintf(stderr,"Error at line %d: unexpected read value %d (expecting 255)\n",__LINE__,(int)uc); + assert(0); } - sc[0] = 3; /* initialize with a number that is not -1 or -0 */ + sc = 3; /* initialize with a number that is not -1 or -0 */ /* No NC_ERANGE as the internal and external types are considered the same */ - err = ncmpi_get_att_schar(ncid, NC_GLOBAL, "att1", sc); CHECK_ERR - if ( sc[0] != -1 /* 2-complement bit representation */ - && sc[0] != -0) { /* 1-complement bit representation */ - printf("Error at line %d: unexpected read value %d (expecting 255)\n",__LINE__,(int)uc[0]); - nerrs++; + err = ncmpi_get_att_schar(ncid, NC_GLOBAL, "att1", &sc); CHECK_ERR + if ( sc != -1 /* 2-complement bit representation */ + && sc != -0) { /* 1-complement bit representation */ + fprintf(stderr,"Error at line %d: unexpected read value %d (expecting 255)\n",__LINE__,(int)sc); + assert(0); } /* expect NC_ERANGE */ @@ -81,15 +94,15 @@ int test_cdf12(char *filename, int bb_enabled, int cmode) err = ncmpi_put_att_double(ncid, NC_GLOBAL, "attd", NC_DOUBLE, 1, &dbuf); CHECK_ERR -#if defined(PNETCDF_ERANGE_FILL) && PNETCDF_ERANGE_FILL == 1 - if (! (cmode & NC_NETCDF4)) { +#if PNETCDF_ERANGE_FILL == 1 + if (format != NC_FORMAT_NETCDF4 && format != NC_FORMAT_NETCDF4_CLASSIC) { float fbuf; /* read back attf and expect NC_FILL_FLOAT */ dbuf = 0.0; err = ncmpi_get_att_double(ncid, NC_GLOBAL, "attf", &dbuf); CHECK_ERR if (dbuf != NC_FILL_FLOAT) { - printf("Error at line %d: unexpected read value %f (expecting %f)\n",__LINE__,dbuf,NC_FILL_FLOAT); - nerrs++; + fprintf(stderr,"Error at line %d: unexpected read value %f (expecting %f)\n",__LINE__,dbuf,NC_FILL_FLOAT); + assert(0); } /* read back attd as float and expect NC_ERANGE */ err = ncmpi_get_att_float(ncid, NC_GLOBAL, "attd", &fbuf); EXP_ERR(NC_ERANGE) @@ -97,72 +110,143 @@ int test_cdf12(char *filename, int bb_enabled, int cmode) #endif err = ncmpi_def_dim(ncid, "x", 1, &dimid); CHECK_ERR - err = ncmpi_def_var(ncid, "var_byte", NC_BYTE, 1, &dimid, &vid); CHECK_ERR + + /* NC_BYTE is an 8-bit signed integer, i.e. C type of signed char */ + err = ncmpi_def_var(ncid, "var_byte", NC_BYTE, 1, &dimid, &vid); + CHECK_ERR err = ncmpi_def_var(ncid, "var_flt", NC_FLOAT, 1, &dimid, &fvid); CHECK_ERR err = ncmpi_def_var(ncid, "var_dbl", NC_DOUBLE, 1, &dimid, &dvid); CHECK_ERR err = ncmpi_enddef(ncid); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + /* No NC_ERANGE should be returned for CDF-1 and 2 */ - uc[0] = 255; - err = ncmpi_put_var_uchar_all(ncid, vid, uc); CHECK_ERR - uc[0] = 3; /* initialize with a number that is not -1 or -0 */ - err = ncmpi_get_var_uchar_all(ncid, vid, uc); CHECK_ERR - if (uc[0] != 255) { - printf("Error at line %d: unexpected read value %d (expecting 255)\n",__LINE__,(int)uc[0]); - nerrs++; + /* range of unsigned char is from 0 to 255. PnetCDF should detect the value + * being out of range. But because NetCDF standard makes a special case for + * CDF-1 and CDF-2, which treats external NC_BYTE as unsigned char with no + * type conversion. + * + * https://docs.unidata.ucar.edu/nug/current/md_types.html#data_type + * The _uchar and _schar functions were introduced in netCDF-3 to eliminate + * an ambiguity, and support both signed and unsigned byte data. In + * netCDF-2, whether the external NC_BYTE type represented signed or + * unsigned values was left up to the user. In netcdf-3, we treat NC_BYTE + * as signed for the purposes of conversion to short, int, long, float, or + * double. (Of course, no conversion takes place when the internal type is + * signed char.) In the _uchar functions, we treat NC_BYTE as if it were + * unsigned. Thus, no NC_ERANGE error can occur converting between NC_BYTE + * and unsigned char. + */ + uc = 255; + if (coll_io) + err = ncmpi_put_var_uchar_all(ncid, vid, &uc); + else + err = ncmpi_put_var_uchar(ncid, vid, &uc); + CHECK_ERR + + uc = 3; /* set read buffer to a different value before read */ + if (coll_io) + err = ncmpi_get_var_uchar_all(ncid, vid, &uc); + else + err = ncmpi_get_var_uchar(ncid, vid, &uc); + CHECK_ERR + + /* In netCDF-2, whether the external NC_BYTE type represented signed or + * unsigned values was left up to the user. + */ + if (format != NC_FORMAT_CLASSIC && format != NC_FORMAT_64BIT_OFFSET && uc != 255) { + fprintf(stderr,"Error %s at %d: uc[0] expect 255 but got %d\n", + basename(__FILE__),__LINE__,(int)uc); + assert(0); } /* No NC_ERANGE should be returned for CDF-1 and 2 */ - sc[0] = -128; - err = ncmpi_put_var_schar_all(ncid, vid, sc); CHECK_ERR - sc[0] = 0; - err = ncmpi_get_var_schar_all(ncid, vid, sc); CHECK_ERR - if (sc[0] != -128) { - printf("Error at line %d: unexpected read value %d (expecting -128)\n",__LINE__,(int)sc[0]); - nerrs++; + sc = -128; + if (coll_io) + err = ncmpi_put_var_schar_all(ncid, vid, &sc); + else + err = ncmpi_put_var_schar(ncid, vid, &sc); + CHECK_ERR + + sc = 0; /* set read buffer to a different value before read */ + if (coll_io) + err = ncmpi_get_var_schar_all(ncid, vid, &sc); + else + err = ncmpi_get_var_schar(ncid, vid, &sc); + CHECK_ERR + + if (format != NC_FORMAT_CLASSIC && format != NC_FORMAT_64BIT_OFFSET && sc != -128) { + fprintf(stderr,"Error %s at %d: sc expect -128 but got %d\n", + basename(__FILE__),__LINE__,(int)sc); + assert(0); } /* expect NC_ERANGE */ - si[0] = -129; - err = ncmpi_put_var_int_all(ncid, vid, si); + si = -129; + if (coll_io) + err = ncmpi_put_var_int_all(ncid, vid, &si); + else + err = ncmpi_put_var_int(ncid, vid, &si); if (bb_enabled) { CHECK_ERR err = ncmpi_flush(ncid); } EXP_ERR(NC_ERANGE) - if (si[0] != -129) { /* check if put buffer content is altered */ - printf("Error at line %d: put buffer content altered %d (expecting -128)\n",__LINE__,si[0]); - nerrs++; + if (si != -129) { /* check if put buffer content is altered */ + fprintf(stderr,"Error %s at %d: si expect -129 but got %d\n", + basename(__FILE__),__LINE__,si); + assert(0); } /* expect NC_ERANGE */ - si[0] = 256; - err = ncmpi_put_var_int_all(ncid, vid, si); + si = 256; + if (coll_io) + err = ncmpi_put_var_int_all(ncid, vid, &si); + else + err = ncmpi_put_var_int(ncid, vid, &si); if (bb_enabled) { CHECK_ERR err = ncmpi_flush(ncid); } EXP_ERR(NC_ERANGE) - if (si[0] != 256) { /* check if put buffer content is altered */ - printf("Error at line %d: put buffer content altered %d (expecting 256)\n",__LINE__,si[0]); - nerrs++; + if (si != 256) { /* check if put buffer content is altered */ + fprintf(stderr,"Error %s at %d: si expect 256 but got %d\n", + basename(__FILE__),__LINE__,si); + assert(0); } /* expect no error */ - si[0] = -128; - err = ncmpi_put_var_int_all(ncid, vid, si); CHECK_ERR - si[0] = 0; - err = ncmpi_get_var_int_all(ncid, vid, si); CHECK_ERR - if (si[0] != -128) { - printf("Error at line %d: unexpected read value %d (expecting -128)\n",__LINE__,si[0]); - nerrs++; + si = -128; + if (coll_io) + err = ncmpi_put_var_int_all(ncid, vid, &si); + else + err = ncmpi_put_var_int(ncid, vid, &si); + CHECK_ERR + + si = 0; /* set read buffer to a different value before read */ + if (coll_io) + err = ncmpi_get_var_int_all(ncid, vid, &si); + else + err = ncmpi_get_var_int(ncid, vid, &si); + CHECK_ERR + + if (format != NC_FORMAT_CLASSIC && format != NC_FORMAT_64BIT_OFFSET && si != -128) { + fprintf(stderr,"Error %s at %d: si expect -128 but got %d\n", + basename(__FILE__),__LINE__,si); + assert(0); } /* expect NC_ERANGE */ dbuf = NC_MAX_DOUBLE/2.0; - err = ncmpi_put_var_double_all(ncid, fvid, &dbuf); + if (coll_io) + err = ncmpi_put_var_double_all(ncid, fvid, &dbuf); + else + err = ncmpi_put_var_double(ncid, fvid, &dbuf); if (bb_enabled) { CHECK_ERR err = ncmpi_flush(ncid); @@ -170,21 +254,36 @@ int test_cdf12(char *filename, int bb_enabled, int cmode) EXP_ERR(NC_ERANGE) /* write a value > NC_MAX_FLOAT */ - err = ncmpi_put_var_double_all(ncid, dvid, &dbuf); CHECK_ERR + if (coll_io) + err = ncmpi_put_var_double_all(ncid, dvid, &dbuf); + else + err = ncmpi_put_var_double(ncid, dvid, &dbuf); + CHECK_ERR + -#if defined(PNETCDF_ERANGE_FILL) && PNETCDF_ERANGE_FILL == 1 - if (! (cmode & NC_NETCDF4)) { +#if PNETCDF_ERANGE_FILL == 1 + if (format != NC_FORMAT_NETCDF4 && format != NC_FORMAT_NETCDF4_CLASSIC) { float fbuf; /* read back and expect NC_FILL_FLOAT */ - dbuf = 0.0; - err = ncmpi_get_var_double_all(ncid, fvid, &dbuf); CHECK_ERR + dbuf = 0.0; /* set read buffer to a different value before read */ + if (coll_io) + err = ncmpi_get_var_double_all(ncid, fvid, &dbuf); + else + err = ncmpi_get_var_double(ncid, fvid, &dbuf); + CHECK_ERR if (dbuf != NC_FILL_FLOAT) { - printf("Error at line %d: unexpected read value %f (expecting %f)\n",__LINE__,dbuf,NC_FILL_FLOAT); - nerrs++; + fprintf(stderr,"Error %s at %d: dbuf expect NC_FILL_FLOAT (%f) but got %f\n", + basename(__FILE__),__LINE__,NC_FILL_FLOAT,dbuf); + assert(0); } /* read back dvid as float and expect NC_ERANGE */ - err = ncmpi_get_var_float_all(ncid, dvid, &fbuf); EXP_ERR(NC_ERANGE) + fbuf = 0.0; /* set read buffer to a different value before read */ + if (coll_io) + err = ncmpi_get_var_float_all(ncid, dvid, &fbuf); + else + err = ncmpi_get_var_float(ncid, dvid, &fbuf); + EXP_ERR(NC_ERANGE) } #endif @@ -194,123 +293,146 @@ int test_cdf12(char *filename, int bb_enabled, int cmode) } static -int test_cdf345(char *filename, int bb_enabled, int cmode) +int test_cdf345(const char *filename, int format, int coll_io, MPI_Info info) { - int err, nerrs=0, ncid, uc_vid, sc_vid, dimid; - unsigned char uc[1]; - signed char sc[1]; + char val[MPI_MAX_INFO_VAL]; + int err, nerrs=0, ncid, uc_vid, sc_vid, dimid, flag, bb_enabled=0; + unsigned char uc; + signed char sc; - cmode |= NC_CLOBBER; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); CHECK_ERR + /* check whether burst buffering is enabled */ + MPI_Info_get(info, "nc_burst_buf", MPI_MAX_INFO_VAL - 1, val, &flag); + if (flag && strcasecmp(val, "enable") == 0) { + bb_enabled = 1; + /* does not work for NetCDF4 files when burst-buffering is enabled */ + if (format == NC_FORMAT_NETCDF4 || format == NC_FORMAT_NETCDF4_CLASSIC) + return 0; + } + + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, info, &ncid); CHECK_ERR /* CDF-5 considers NC_BYTE a signed 1-byte integer and NC_UBYTE an * unsigned 1-byte integer. The special case in CDF-2 for skipping * NC_ERANGE checking for converting between NC_BYTE and unsigned * char is no longer held. */ - uc[0] = 255; - err = ncmpi_put_att_uchar(ncid, NC_GLOBAL, "att1", NC_UBYTE, 1, uc); CHECK_ERR + uc = 255; + err = ncmpi_put_att_uchar(ncid, NC_GLOBAL, "att1", NC_UBYTE, 1, &uc); CHECK_ERR /* in CDF-5, get 255 to a schar buffer should result in NC_ERANGE */ - err = ncmpi_get_att_schar(ncid, NC_GLOBAL, "att1", sc); EXP_ERR(NC_ERANGE) + err = ncmpi_get_att_schar(ncid, NC_GLOBAL, "att1", &sc); EXP_ERR(NC_ERANGE) - sc[0] = -1; /* a value should cause NC_ERANGE */ - err = ncmpi_put_att_schar(ncid, NC_GLOBAL, "att2", NC_UBYTE, 1, sc); EXP_ERR(NC_ERANGE) + sc = -1; /* a value should cause NC_ERANGE */ + err = ncmpi_put_att_schar(ncid, NC_GLOBAL, "att2", NC_UBYTE, 1, &sc); EXP_ERR(NC_ERANGE) err = ncmpi_def_dim(ncid, "x", 1, &dimid); CHECK_ERR err = ncmpi_def_var(ncid, "var_ubyte", NC_UBYTE, 1, &dimid, &uc_vid); CHECK_ERR err = ncmpi_def_var(ncid, "var_byte", NC_BYTE, 1, &dimid, &sc_vid); CHECK_ERR err = ncmpi_enddef(ncid); CHECK_ERR - uc[0] = 255; - err = ncmpi_put_var_uchar_all(ncid, uc_vid, uc); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + + uc = 255; + if (coll_io) + err = ncmpi_put_var_uchar_all(ncid, uc_vid, &uc); + else + err = ncmpi_put_var_uchar(ncid, uc_vid, &uc); + CHECK_ERR /* in CDF-5, get 255 to an schar should result in NC_ERANGE */ - err = ncmpi_get_var_schar_all(ncid, uc_vid, sc); EXP_ERR(NC_ERANGE) + if (coll_io) + err = ncmpi_get_var_schar_all(ncid, uc_vid, &sc); + else + err = ncmpi_get_var_schar(ncid, uc_vid, &sc); + EXP_ERR(NC_ERANGE) - sc[0] = -1; /* in CDF-5, put -1 to an uchar should result in NC_ERANGE */ - err = ncmpi_put_var_schar_all(ncid, uc_vid, sc); + sc = -1; /* in CDF-5, put -1 to an uchar should result in NC_ERANGE */ + if (coll_io) + err = ncmpi_put_var_schar_all(ncid, uc_vid, &sc); + else + err = ncmpi_put_var_schar(ncid, uc_vid, &sc); if (bb_enabled) { CHECK_ERR err = ncmpi_flush(ncid); } EXP_ERR(NC_ERANGE) - uc[0] = 255; /* in CDF-5, put 255 to a schar should result in NC_ERANGE */ - err = ncmpi_put_var_uchar_all(ncid, sc_vid, uc); + uc = 255; /* in CDF-5, put 255 to a schar should result in NC_ERANGE */ + if (coll_io) + err = ncmpi_put_var_uchar_all(ncid, sc_vid, &uc); + else + err = ncmpi_put_var_uchar(ncid, sc_vid, &uc); if (bb_enabled) { CHECK_ERR err = ncmpi_flush(ncid); } EXP_ERR(NC_ERANGE) - sc[0] = -1; - err = ncmpi_put_var_schar_all(ncid, sc_vid, sc); CHECK_ERR - uc[0] = 0; /* in CDF-5, get -1 to an uchar should result in NC_ERANGE */ - err = ncmpi_get_var_uchar_all(ncid, sc_vid, uc); EXP_ERR(NC_ERANGE) + sc = -1; + if (coll_io) + err = ncmpi_put_var_schar_all(ncid, sc_vid, &sc); + else + err = ncmpi_put_var_schar(ncid, sc_vid, &sc); + CHECK_ERR + + uc = 0; /* in CDF-5, get -1 to an uchar should result in NC_ERANGE */ + if (coll_io) + err = ncmpi_get_var_uchar_all(ncid, sc_vid, &uc); + else + err = ncmpi_get_var_uchar(ncid, sc_vid, &uc); + EXP_ERR(NC_ERANGE) err = ncmpi_close(ncid); CHECK_ERR return nerrs; } -int main(int argc, char* argv[]) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) { - char filename[256], *hint_value; - int err, nerrs=0, rank, bb_enabled=0; + int nerrs=0; - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); + if (format == NC_FORMAT_CLASSIC || + format == NC_FORMAT_64BIT_OFFSET || + format == NC_FORMAT_NETCDF4_CLASSIC) + nerrs += test_cdf12(out_path, format, coll_io, info); + else + nerrs += test_cdf345(out_path, format, coll_io, info); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for checking for NC_ERANGE ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } + return nerrs; +} - /* check whether burst buffering is enabled */ - if (inq_env_hint("nc_burst_buf", &hint_value)) { - if (strcasecmp(hint_value, "enable") == 0) bb_enabled = 1; - free(hint_value); - } +int main(int argc, char **argv) { - nerrs += test_cdf12(filename, bb_enabled, 0); - nerrs += test_cdf12(filename, bb_enabled, NC_64BIT_OFFSET); -#if ENABLE_NETCDF4 - if (!bb_enabled) { - nerrs += test_cdf12(filename, bb_enabled, NC_NETCDF4 | NC_CLASSIC_MODEL); - nerrs += test_cdf345(filename, bb_enabled, NC_NETCDF4); - } -#endif - nerrs += test_cdf345(filename, bb_enabled, NC_64BIT_DATA); - - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + int err; + loop_opts opt; - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "NC_ERANGE", opt, test_io); MPI_Finalize(); - return (nerrs > 0); + + return err; } diff --git a/test/testcases/test_fillvalue.c b/test/testcases/test_fillvalue.c index f79b85048c..14ce03cd6b 100644 --- a/test/testcases/test_fillvalue.c +++ b/test/testcases/test_fillvalue.c @@ -38,15 +38,30 @@ #include -static int -tst_fmt(char *filename, int cmode) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) { - int err, nerrs=0, ncid, varid, int_buf; + char val[MPI_MAX_INFO_VAL]; + int err, nerrs=0, flag, ncid, varid, int_buf; float flt_buf; + /* check whether burst buffering is enabled */ + MPI_Info_get(info, "nc_burst_buf", MPI_MAX_INFO_VAL - 1, val, &flag); + if (flag && strcasecmp(val, "enable") == 0 && + (format == NC_FORMAT_NETCDF4 || format == NC_FORMAT_NETCDF4_CLASSIC)) + /* does not work for NetCDF4 files when burst-buffering is enabled */ + return 0; + + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + /* create a file */ - cmode |= NC_CLOBBER; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR flt_buf = 1.234; @@ -71,61 +86,25 @@ tst_fmt(char *filename, int cmode) } int main(int argc, char **argv) { - char filename[256], *hint_value; - int err, nerrs=0, rank, bb_enabled=0; + + int err; + loop_opts opt; MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for _FillValue for NC_GLOBAL ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - /* check whether burst buffering is enabled */ - if (inq_env_hint("nc_burst_buf", &hint_value)) { - if (strcasecmp(hint_value, "enable") == 0) bb_enabled = 1; - free(hint_value); - } - - nerrs += tst_fmt(filename, 0); - nerrs += tst_fmt(filename, NC_64BIT_OFFSET); - if (!bb_enabled) { -#ifdef ENABLE_NETCDF4 - nerrs += tst_fmt(filename, NC_NETCDF4); - nerrs += tst_fmt(filename, NC_NETCDF4 | NC_CLASSIC_MODEL); -#endif - } - nerrs += tst_fmt(filename, NC_64BIT_DATA); - - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } - - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "_FillValue for NC_GLOBAL", opt, test_io); MPI_Finalize(); - return (nerrs > 0); + + return err; } diff --git a/test/testcases/test_get_varn.c b/test/testcases/test_get_varn.c index 2d377f3d79..0ecd0d1e24 100644 --- a/test/testcases/test_get_varn.c +++ b/test/testcases/test_get_varn.c @@ -22,38 +22,26 @@ #define NDIMS 3 -int main(int argc, char** argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) { - int i, j, rank, nprocs, err, nerrs = 0; + int i, j, rank, err, nerrs = 0; int ncid, varid, num_reqs; double *buffer; float *fbuffer; MPI_Offset r_len, **starts = NULL, **counts = NULL; MPI_Offset st[3], ct[3]; - char filename[256]; int dimids[3]; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for get_varn ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } #ifdef DEBUG + int nprocs; + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); if (nprocs != 4 && rank == 0) printf("Warning: %s is designed to run on 4 process\n",argv[0]); #endif @@ -69,13 +57,22 @@ int main(int argc, char** argv) * float lnfm(time, lat, lon) ; * } */ - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER|NC_64BIT_DATA, MPI_INFO_NULL, &ncid); CHECK_ERR + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "time", NC_UNLIMITED, &dimids[0]); CHECK_ERR err = ncmpi_def_dim(ncid, "lat", 94, &dimids[1]); CHECK_ERR err = ncmpi_def_dim(ncid, "lon", 192, &dimids[2]); CHECK_ERR err = ncmpi_def_var(ncid, "lnfm", NC_FLOAT, 3, dimids, &varid); CHECK_ERR err = ncmpi_enddef(ncid); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + st[0] = rank*2; st[1] = 0; st[2] = 0; @@ -85,15 +82,30 @@ int main(int argc, char** argv) ct[2] = 192; float *scramble = (float*) calloc(ct[0]*ct[1]*ct[2], sizeof(float)); - err = ncmpi_put_vara_float_all(ncid, varid, st, ct, scramble); CHECK_ERR + if (coll_io) + err = ncmpi_put_vara_float_all(ncid, varid, st, ct, scramble); + else + err = ncmpi_put_vara_float(ncid, varid, st, ct, scramble); + CHECK_ERR + + /* file sync before reading */ + err = ncmpi_sync(ncid); + CHECK_ERR + MPI_Barrier(MPI_COMM_WORLD); + err = ncmpi_close(ncid); CHECK_ERR free(scramble); /* now we can finally exercise the read path of this record varable */ - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_NOWRITE, info, &ncid); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + /* pick 2 requests for 4 processes */ /* num_reqs = 1; => works fine*/ num_reqs = 2; @@ -136,8 +148,11 @@ int main(int argc, char** argv) /* set the buffer pointers to different offsets to the I/O buffer */ varid = 0; /* only one variable in lnfm.nc */ - err = ncmpi_get_varn_double_all(ncid, varid, num_reqs, starts, counts, buffer); - /* err = ncmpi_get_varn_float_all(ncid, varid, num_reqs, starts, counts, fbuffer); */ + if (coll_io) + err = ncmpi_get_varn_double_all(ncid, varid, num_reqs, starts, counts, buffer); + /* err = ncmpi_get_varn_float_all(ncid, varid, num_reqs, starts, counts, fbuffer); */ + else + err = ncmpi_get_varn_double(ncid, varid, num_reqs, starts, counts, buffer); CHECK_ERR err = ncmpi_close(ncid); @@ -160,24 +175,30 @@ int main(int argc, char** argv) free(starts); free(counts); - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + return nerrs; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "get_varn", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/testcases/test_vard.c b/test/testcases/test_vard.c index 5ea0ded0f5..8ec0fde8eb 100644 --- a/test/testcases/test_vard.c +++ b/test/testcases/test_vard.c @@ -64,6 +64,7 @@ if (buf[j][i] != val+i) { \ printf("line %d: expecting buf[%d][%d]=%d but got %d\n",__LINE__,j,i,val+i,buf[j][i]); \ nerrs++; \ + goto fn_exit; \ } \ } \ } \ @@ -74,12 +75,14 @@ if (buf[j][i] != rank*100+j*10+i) { \ printf("line %d: expecting buf[%d][%d]=%d but got %d\n",__LINE__,j,i,rank*100+j*10+i,(int)buf[j][i]); \ nerrs++; \ + goto fn_exit; \ } \ } \ } static int get_var_and_verify(int ncid, + int coll_io, int varid, MPI_Offset *start, MPI_Offset *count, @@ -97,7 +100,11 @@ int get_var_and_verify(int ncid, for (j=0; j------------------------------------------------------------*/ -int main(int argc, char **argv) { - - char filename[256]; +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) +{ int i, j, err, ncid, varid0, varid1, varid2, dimids[2], nerrs=0; int rank, nprocs, blocklengths[2], **buf, *bufptr; int array_of_sizes[2], array_of_subsizes[2], array_of_starts[2]; - int buftype_size, expected_put_size, format; - float **flt_buf, *flt_bufptr; - double **dbl_buf, *dbl_bufptr; + int buftype_size, expected_put_size, fmt; + float **flt_buf=NULL, *flt_bufptr; + double **dbl_buf=NULL, *dbl_bufptr; MPI_Offset start[2], count[2], header_size, put_size, new_put_size; MPI_Aint a0, a1, disps[2]; MPI_Datatype buftype, ghost_buftype, rec_filetype, fix_filetype; MPI_Datatype flt_buftype, dbl_buftype; - MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); MPI_Comm_rank(MPI_COMM_WORLD, &rank); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for vard put and get ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - /* construct various MPI derived data types */ buf = (int**)malloc(sizeof(int*) * NY); @@ -196,7 +208,7 @@ int main(int argc, char **argv) { disps[1] = a1 - a0; bufptr = buf[1]; err = MPI_Type_create_hindexed(2, blocklengths, disps, MPI_INT, &buftype); - if (err != MPI_SUCCESS) printf("MPI error MPI_Type_create_hindexed\n"); + if (err != MPI_SUCCESS) fprintf(stderr,"MPI error MPI_Type_create_hindexed\n"); MPI_Type_commit(&buftype); start[0] = 0; start[1] = NX*rank; @@ -226,9 +238,13 @@ int main(int argc, char **argv) { MPI_INT, &ghost_buftype); MPI_Type_commit(&ghost_buftype); + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + /* create a new file for write */ - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, MPI_INFO_NULL, - &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR /* define a 2D array */ err = ncmpi_def_dim(ncid, "REC_DIM", NC_UNLIMITED, &dimids[0]); CHECK_ERR @@ -246,6 +262,11 @@ int main(int argc, char **argv) { err = ncmpi_fill_var_rec(ncid, varid2, 0); CHECK_ERR err = ncmpi_fill_var_rec(ncid, varid2, 1); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + /* create a file type for the record variable */ int *array_of_blocklengths=(int*) malloc(sizeof(int) * count[0]); MPI_Aint *array_of_displacements=(MPI_Aint*) malloc(sizeof(MPI_Aint) * count[0]); @@ -271,21 +292,25 @@ int main(int argc, char **argv) { err = ncmpi_inq_put_size(ncid, &put_size); CHECK_ERR /* write the record variable */ - err = ncmpi_put_vard_all(ncid, varid0, rec_filetype, bufptr, 1, buftype); CHECK_ERR + if (coll_io) + err = ncmpi_put_vard_all(ncid, varid0, rec_filetype, bufptr, 1, buftype); + else + err = ncmpi_put_vard(ncid, varid0, rec_filetype, bufptr, 1, buftype); + CHECK_ERR /* check if put_size is correctly reported */ err = ncmpi_inq_put_size(ncid, &new_put_size); CHECK_ERR MPI_Type_size(buftype, &buftype_size); - err = ncmpi_inq_format(ncid, &format); CHECK_ERR + err = ncmpi_inq_format(ncid, &fmt); CHECK_ERR expected_put_size = buftype_size; /* for writing a record variable, root process will update numrec to the * file header, However, because the first 2 records have been filled * above, root process need not write to file header. - if (rank == 0) expected_put_size += (format == NC_FORMAT_CDF5) ? 8 : 4; + if (rank == 0) expected_put_size += (fmt == NC_FORMAT_CDF5) ? 8 : 4; */ if (expected_put_size != new_put_size - put_size) { - printf("Error at line %d in %s: unexpected put size ("OFFFMT") reported, expecting %d\n", + fprintf(stderr,"Error at line %d in %s: unexpected put size ("OFFFMT") reported, expecting %d\n", __LINE__,__FILE__,new_put_size-put_size, expected_put_size); nerrs++; } @@ -299,13 +324,17 @@ int main(int argc, char **argv) { err = ncmpi_inq_put_size(ncid, &put_size); CHECK_ERR /* write the fixed-size variable */ - err = ncmpi_put_vard_all(ncid, varid1, fix_filetype, bufptr, 1, buftype); CHECK_ERR + if (coll_io) + err = ncmpi_put_vard_all(ncid, varid1, fix_filetype, bufptr, 1, buftype); + else + err = ncmpi_put_vard(ncid, varid1, fix_filetype, bufptr, 1, buftype); + CHECK_ERR /* check if put_size is correctly reported */ err = ncmpi_inq_put_size(ncid, &new_put_size); CHECK_ERR expected_put_size = buftype_size; if (expected_put_size != new_put_size - put_size) { - printf("Error at line %d in %s: unexpected put size ("OFFFMT") reported, expecting %d\n", + fprintf(stderr,"Error at line %d in %s: unexpected put size ("OFFFMT") reported, expecting %d\n", __LINE__,__FILE__,new_put_size-put_size, expected_put_size); nerrs++; } @@ -326,17 +355,27 @@ int main(int argc, char **argv) { err = ncmpi_rename_var(ncid, varid0, "rec_var"); CHECK_ERR err = ncmpi_end_indep_data(ncid); CHECK_ERR + /* file sync before reading */ + err = ncmpi_sync(ncid); + CHECK_ERR + MPI_Barrier(MPI_COMM_WORLD); + err = ncmpi_close(ncid); CHECK_ERR /* open the same file and read back for validate */ - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_WRITE, MPI_INFO_NULL, - &ncid); CHECK_ERR + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_WRITE, info, &ncid); + CHECK_ERR + + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } err = ncmpi_inq_varid(ncid, "rec_var", &varid0); CHECK_ERR err = ncmpi_inq_varid(ncid, "fix_var", &varid1); CHECK_ERR - nerrs += get_var_and_verify(ncid, varid0, start, count, buf, buftype, ghost_buftype, rec_filetype); - nerrs += get_var_and_verify(ncid, varid1, start, count, buf, buftype, ghost_buftype, fix_filetype); + nerrs += get_var_and_verify(ncid, coll_io, varid0, start, count, buf, buftype, ghost_buftype, rec_filetype); + nerrs += get_var_and_verify(ncid, coll_io, varid1, start, count, buf, buftype, ghost_buftype, fix_filetype); /* test type conversion from float to int */ flt_buf = (float**)malloc(sizeof(float*) * NY); @@ -353,39 +392,55 @@ int main(int argc, char **argv) { disps[1] = a1 - a0; flt_bufptr = flt_buf[1]; err = MPI_Type_create_hindexed(2, blocklengths, disps, MPI_FLOAT, &flt_buftype); - if (err != MPI_SUCCESS) printf("MPI error MPI_Type_create_hindexed\n"); + if (err != MPI_SUCCESS) fprintf(stderr,"MPI error MPI_Type_create_hindexed\n"); MPI_Type_commit(&flt_buftype); /* write the record variable with type conversion from float to int */ - err = ncmpi_put_vard_all(ncid, varid0, rec_filetype, flt_bufptr, 1, flt_buftype); CHECK_ERR + if (coll_io) + err = ncmpi_put_vard_all(ncid, varid0, rec_filetype, flt_bufptr, 1, flt_buftype); + else + err = ncmpi_put_vard(ncid, varid0, rec_filetype, flt_bufptr, 1, flt_buftype); + CHECK_ERR CHECK_VALUE(flt_buf) - nerrs += get_var_and_verify(ncid, varid0, start, count, buf, buftype, ghost_buftype, rec_filetype); + nerrs += get_var_and_verify(ncid, coll_io, varid0, start, count, buf, buftype, ghost_buftype, rec_filetype); /* read the record variable with type conversion from int to float */ for (j=0; j= 4) expected = -1; else if (j<2 || j >= NX+2) expected = -1; if (schar_buf[i*array_of_sizes[1]+j] != expected) { - printf("Error at line %d in %s: expecting schar_buf[%d][%d]=%d but got %d\n", + fprintf(stderr,"Error at line %d in %s: expecting schar_buf[%d][%d]=%d but got %d\n", __LINE__,__FILE__,i,j,expected,schar_buf[i*array_of_sizes[1]+j]); nerrs++; } } free(schar_buf); +fn_exit: MPI_Type_free(&rec_filetype); MPI_Type_free(&fix_filetype); MPI_Type_free(&buftype); @@ -495,28 +573,43 @@ int main(int argc, char **argv) { free(array_of_blocklengths); free(array_of_displacements); free(buf[0]); free(buf); - free(flt_buf[0]); free(flt_buf); - free(dbl_buf[0]); free(dbl_buf); + if (flt_buf != NULL) { + free(flt_buf[0]); + free(flt_buf); + } + if (dbl_buf != NULL) { + free(dbl_buf[0]); + free(dbl_buf); + } err = ncmpi_close(ncid); CHECK_ERR - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + return nerrs; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 0; /* skip test intra-node aggregation */ + opt.drv = 1; /* test MPI-IO driver only */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "vard APIs", opt, test_io); MPI_Finalize(); - return (nerrs > 0); + + return err; } diff --git a/test/testcases/test_vard_multiple.c b/test/testcases/test_vard_multiple.c index 1e47dbeb48..6948fee03d 100644 --- a/test/testcases/test_vard_multiple.c +++ b/test/testcases/test_vard_multiple.c @@ -69,43 +69,29 @@ if ((buf)[j*NX+i] != (base)+rank*100+j*10+i) { \ printf("line %d: expecting buf[%d*NX+%d]=%d but got %d\n",\ __LINE__,j,i,(base)+rank*100+j*10+i,(buf)[j*NX+i]); \ - nerrs++; \ + assert(0); \ } \ } \ } -/*----< main() >------------------------------------------------------------*/ -int main(int argc, char **argv) { - - char filename[256]; - int i, j, err, ncid, varid[4], dimids[3], nerrs=0, unlimit_dimid; - int rank, nprocs, *buf[2]; - int array_of_sizes[2], array_of_subsizes[2], array_of_starts[2]; - int array_of_blocklengths[NY]; - MPI_Offset len, recsize, start[2], count[2], offset[2]; - MPI_Aint a0, a1, array_of_displacements[NY]; +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) +{ + int i, j, err, ncid, varid[4], dimids[3], nerrs=0, unlimit_dimid; + int rank, nprocs, *buf[2]; + int array_of_sizes[2], array_of_subsizes[2], array_of_starts[2]; + int array_of_blocklengths[NY]; + MPI_Offset len, recsize, start[2], count[2], offset[2]; + MPI_Aint a0, a1, array_of_displacements[NY]; MPI_Datatype buftype, vtype[2], filetype; - MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); MPI_Comm_rank(MPI_COMM_WORLD, &rank); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for vard to 2 variables ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - buf[0] = (int*)malloc(sizeof(int) * NY * NX); for (j=0; j 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + return nerrs; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 0; /* skip test intra-node aggregation */ + opt.drv = 1; /* test MPI-IO driver only */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "2 var in one call", opt, test_io); MPI_Finalize(); - return (nerrs > 0); + + return err; } diff --git a/test/testcases/test_vard_rec.c b/test/testcases/test_vard_rec.c index b5c75f04a0..66b19ece19 100644 --- a/test/testcases/test_vard_rec.c +++ b/test/testcases/test_vard_rec.c @@ -12,6 +12,9 @@ * to test the fix to bug reported by Jim Edwards in r3675. * * % mpiexec -n 4 test_vard_rec + * + * When setting NX to 3, below shows the expected file contents. + * * % ncmpidump /pvfs2/wkliao/testfile.nc * netcdf testfile { * // file format: CDF-1 @@ -38,43 +41,33 @@ #include #define NY 2 -#define NX 3 - -/*----< main() >------------------------------------------------------------*/ -int main(int argc, char **argv) { - - char filename[256]; +#define NX 100 + +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) +{ int i, j, err, nerrs=0, ncid, varid, dimids[2], unlimit_dimid; int rank, nprocs, verbose, array_of_blocklengths[2], buf[NY][NX]; MPI_Offset recsize, len; MPI_Aint array_of_displacements[2]; MPI_Datatype rec_filetype; - MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); MPI_Comm_rank(MPI_COMM_WORLD, &rank); verbose = 0; - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for vard put on record var ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR /* create a new file for write */ - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, MPI_INFO_NULL, - &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR /* define a 2D array */ err = ncmpi_def_dim(ncid, "REC_DIM", NC_UNLIMITED, &dimids[0]); CHECK_ERR @@ -82,6 +75,11 @@ int main(int argc, char **argv) { err = ncmpi_def_var(ncid, "rec_var", NC_INT, 2, dimids, &varid); CHECK_ERR err = ncmpi_enddef(ncid); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + /* initialize the contents of the array */ for (j=0; j 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + return nerrs; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 0; /* skip test intra-node aggregation */ + opt.drv = 1; /* test MPI-IO driver only */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "vard put on record var", opt, test_io); MPI_Finalize(); - return (nerrs > 0); + + return err; } diff --git a/test/testcases/test_vardf.F b/test/testcases/test_vardf.F index 44e30fa998..77f35432c2 100644 --- a/test/testcases/test_vardf.F +++ b/test/testcases/test_vardf.F @@ -64,7 +64,7 @@ subroutine check(err, message) if (err .NE. NF_NOERR) then write(6,*) message(1:XTRIM(message)), nfmpi_strerror(err) msg = '*** TESTING F77 test_vardf.f for vard API ' - call pass_fail(1, msg) + call pass_fail(1, msg, 0) call MPI_Abort(MPI_COMM_WORLD, -1, err) end if end ! subroutine check @@ -193,7 +193,7 @@ program main implicit none include "mpif.h" include "pnetcdf.inc" - character(LEN=256) filename, cmd, msg + character(LEN=256) out_path, in_path, cmd, msg integer err, ierr, nprocs, rank, i, j, get_args integer cmode, ncid, varid0, varid1, varid2, dimid(2), nerrs integer NX, NY, XTRIM, old_fillmode @@ -209,24 +209,31 @@ program main #else integer*8 a0, a1, disps(2) #endif + logical keep_files + double precision timing call MPI_Init(ierr) + + timing = MPI_Wtime() + call MPI_Comm_rank(MPI_COMM_WORLD, rank, ierr) call MPI_Comm_size(MPI_COMM_WORLD, nprocs, ierr) one = 1 two = 2 - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any if (rank .EQ. 0) then - filename = "testfile.nc" - err = get_args(cmd, filename) + out_path = "testfile.nc" + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, + MPI_COMM_WORLD, ierr) + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, + + ierr) nerrs = 0 start(1) = NX * rank @@ -278,7 +285,7 @@ program main ! create file, truncate it if exists cmode = NF_CLOBBER - err = nfmpi_create(MPI_COMM_WORLD, filename, cmode, + err = nfmpi_create(MPI_COMM_WORLD, out_path, cmode, + MPI_INFO_NULL, ncid) call check(err, 'In nfmpi_create: ') @@ -374,7 +381,7 @@ program main call check(err, 'In nfmpi_close: ') ! open the same file and read back for validate */ - err = nfmpi_open(MPI_COMM_WORLD, filename, NF_NOWRITE, + err = nfmpi_open(MPI_COMM_WORLD, out_path, NF_NOWRITE, + MPI_INFO_NULL, ncid) call check(err, 'In nfmpi_open: ') @@ -413,10 +420,18 @@ program main + sum_size, ' bytes yet to be freed' endif + timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, + + MPI_DOUBLE_PRECISION, MPI_MAX, + + MPI_COMM_WORLD, ierr) if (rank .eq. 0) then + if (.NOT. keep_files) then + err = nfmpi_delete(out_path, MPI_INFO_NULL) + end if + msg = '*** TESTING F77 '//cmd(1:XTRIM(cmd))// - + ' for vard API ' - call pass_fail(nerrs, msg) + + ' - vard API ' + call pass_fail(nerrs, msg, timing) endif 999 call MPI_Finalize(ierr) diff --git a/test/testcases/test_vardf90.f90 b/test/testcases/test_vardf90.f90 index 8a78b3fdfe..80d740f03e 100644 --- a/test/testcases/test_vardf90.f90 +++ b/test/testcases/test_vardf90.f90 @@ -53,8 +53,8 @@ subroutine check(err, message) ! It is a good idea to check returned value for possible error if (err .NE. NF90_NOERR) then write(6,*) trim(message), trim(nf90mpi_strerror(err)) - msg = '*** TESTING F90 test_vardf90.f90 for vard API ' - call pass_fail(1, msg) + msg = '*** TESTING F90 test_vardf90.f90 - vard API ' + call pass_fail(1, msg, 0) call MPI_Abort(MPI_COMM_WORLD, -1, err) end if end subroutine check @@ -182,7 +182,7 @@ program main use pnetcdf implicit none - character(LEN=256) filename, cmd, msg + character(LEN=256) out_path, in_path, cmd, msg integer err, ierr, nprocs, rank, i, j, get_args integer cmode, ncid, varid0, varid1, varid2, dimid(2), nerrs integer NX, NY, old_fillmode @@ -194,20 +194,27 @@ program main integer(kind=MPI_OFFSET_KIND) start(2), count(2), recno integer(kind=MPI_OFFSET_KIND) len, malloc_size, sum_size, recsize integer(kind=MPI_ADDRESS_KIND) a0, a1, disps(2) + logical keep_files + double precision timing call MPI_Init(ierr) + + timing = MPI_Wtime() + call MPI_Comm_rank(MPI_COMM_WORLD, rank, ierr) call MPI_Comm_size(MPI_COMM_WORLD, nprocs, ierr) - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any if (rank .EQ. 0) then - filename = "testfile.nc" - err = get_args(cmd, filename) + out_path = "testfile.nc" + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr) nerrs = 0 @@ -260,7 +267,7 @@ program main ! create file, truncate it if exists cmode = NF90_CLOBBER - err = nf90mpi_create(MPI_COMM_WORLD, filename, cmode, & + err = nf90mpi_create(MPI_COMM_WORLD, out_path, cmode, & MPI_INFO_NULL, ncid) call check(err, 'In nf90mpi_create: ') @@ -354,7 +361,7 @@ program main call check(err, 'In nf90mpi_close: ') ! open the same file and read back for validate */ - err = nf90mpi_open(MPI_COMM_WORLD, filename, NF90_NOWRITE, & + err = nf90mpi_open(MPI_COMM_WORLD, out_path, NF90_NOWRITE, & MPI_INFO_NULL, ncid) call check(err, 'In nf90mpi_open: ') @@ -391,9 +398,17 @@ program main sum_size, ' bytes yet to be freed' endif + timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, & + MPI_DOUBLE_PRECISION, MPI_MAX, & + MPI_COMM_WORLD, ierr) if (rank .eq. 0) then - msg = '*** TESTING F90 '//trim(cmd)//' for vard API ' - call pass_fail(nerrs, msg) + if (.NOT. keep_files) then + err = nfmpi_delete(out_path, MPI_INFO_NULL) + end if + + msg = '*** TESTING F90 '//trim(cmd)//' - vard API ' + call pass_fail(nerrs, msg, timing) endif 999 call MPI_Finalize(ierr) diff --git a/test/testcases/test_varm.c b/test/testcases/test_varm.c index 0248ec4000..084ed9c257 100644 --- a/test/testcases/test_varm.c +++ b/test/testcases/test_varm.c @@ -14,6 +14,8 @@ #include +static int verbose; + static int check_read_contents(float *rh) { @@ -25,10 +27,9 @@ check_read_contents(float *rh) for (i=0; i<6; i++) { for (j=0; j<4; j++) { if (rh[j*6+i] != k) { -#ifdef PRINT_ERR_ON_SCREEN - printf("Error at %s:%d : expect rh[%d][%d]=%f but got %f\n", - __FILE__,__LINE__,j,i,k,rh[j*6+i]); -#endif + if (verbose) + fprintf(stderr,"Error at %s:%d : expect rh[%d][%d]=%f but got %f\n", + __FILE__,__LINE__,j,i,k,rh[j*6+i]); return 1; } k += 1.0; @@ -71,11 +72,10 @@ check_write_contents(signed char *varT) for (j=0; j<4; j++) { for (i=0; i<6; i++) { if (varT[j*6+i] != j*6+i + 50) { -#ifdef PRINT_ERR_ON_SCREEN - /* this error is a pnetcdf internal error, if occurs */ - printf("Error at line %d in %s: expecting varT[%d][%d]=%d but got %d\n", - __LINE__,__FILE__,j,i,j*6+i + 50,varT[j*6+i]); -#endif + if (verbose) + /* this error is a pnetcdf internal error, if occurs */ + fprintf(stderr,"Error at line %d in %s: expecting varT[%d][%d]=%d but got %d\n", + __LINE__,__FILE__,j,i,j*6+i + 50,varT[j*6+i]); return 1; } } @@ -83,10 +83,14 @@ check_write_contents(signed char *varT) return 0; } -static int -tst_fmt(char *filename, int cmode) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) { - int i, j, err, nerrs=0, rank, nprocs; + int i, j, err, nerrs=0, rank, nprocs, debug=0; int ncid, dimid[2], varid, req, status; MPI_Offset start[2], count[2], stride[2], imap[2]; int var[6][4]; @@ -96,8 +100,16 @@ tst_fmt(char *filename, int cmode) MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - cmode |= NC_CLOBBER; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); CHECK_ERR + if (debug && nprocs > 1 && rank == 0) + printf("Warning: %s is designed to run on 1 process\n", + basename(__FILE__)); + + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); + CHECK_FATAL_ERR /* define a variable of a 6 x 4 integer array in the nc file */ err = ncmpi_def_dim(ncid, "Y", 6, &dimid[0]); CHECK_ERR @@ -105,6 +117,11 @@ tst_fmt(char *filename, int cmode) err = ncmpi_def_var(ncid, "var", NC_INT, 2, dimid, &varid); CHECK_ERR err = ncmpi_enddef(ncid); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + /* create a 6 x 4 integer variable in the file with contents: 0, 1, 2, 3, 4, 5, 6, 7, @@ -118,13 +135,28 @@ tst_fmt(char *filename, int cmode) start[0] = 0; start[1] = 0; count[0] = 6; count[1] = 4; if (rank > 0) count[0] = count[1] = 0; - err = ncmpi_put_vara_int_all(ncid, varid, start, count, &var[0][0]); CHECK_ERR + if (coll_io) + err = ncmpi_put_vara_int_all(ncid, varid, start, count, &var[0][0]); + else + err = ncmpi_put_vara_int(ncid, varid, start, count, &var[0][0]); + CHECK_ERR if (nprocs > 1) MPI_Barrier(MPI_COMM_WORLD); + /* file sync before reading */ + err = ncmpi_sync(ncid); + CHECK_ERR + MPI_Barrier(MPI_COMM_WORLD); + err = ncmpi_close(ncid); CHECK_ERR - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); CHECK_ERR + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_NOWRITE, info, &ncid); + CHECK_FATAL_ERR + + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } err = ncmpi_inq_varid(ncid, "var", &varid); CHECK_ERR @@ -133,16 +165,22 @@ tst_fmt(char *filename, int cmode) stride[0] = 1; stride[1] = 1; imap[0] = 1; imap[1] = 6; /* would be {4, 1} if not transposing */ - if (cmode & NC_NETCDF4) { + if (format == NC_FORMAT_NETCDF4 || format == NC_FORMAT_NETCDF4_CLASSIC) { for (i=0; i<6; i++) for (j=0; j<4; j++) rh[j][i] = -1.0; - err = ncmpi_get_varm_float_all(ncid, varid, start, count, stride, imap, - &rh[0][0]); CHECK_ERR + if (coll_io) + err = ncmpi_get_varm_float_all(ncid, varid, start, count, stride, imap, &rh[0][0]); + else + err = ncmpi_get_varm_float(ncid, varid, start, count, stride, imap, &rh[0][0]); + CHECK_ERR nerrs += check_read_contents(&rh[0][0]); /* test when stride == NULL and imap != NULL */ for (i=0; i<6; i++) for (j=0; j<4; j++) rh[j][i] = -1.0; - err = ncmpi_get_varm_float_all(ncid, varid, start, count, NULL, imap, - &rh[0][0]); CHECK_ERR + if (coll_io) + err = ncmpi_get_varm_float_all(ncid, varid, start, count, NULL, imap, &rh[0][0]); + else + err = ncmpi_get_varm_float(ncid, varid, start, count, NULL, imap, &rh[0][0]); + CHECK_ERR nerrs += check_read_contents(&rh[0][0]); } else { @@ -150,7 +188,11 @@ tst_fmt(char *filename, int cmode) for (i=0; i<6; i++) for (j=0; j<4; j++) rh[j][i] = -1.0; err = ncmpi_iget_varm_float(ncid, varid, start, count, stride, imap, &rh[0][0], &req); CHECK_ERR - err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR + if (coll_io) + err = ncmpi_wait_all(ncid, 1, &req, &status); + else + err = ncmpi_wait(ncid, 1, &req, &status); + CHECK_ERR err = status; CHECK_ERR nerrs += check_read_contents(&rh[0][0]); @@ -158,26 +200,42 @@ tst_fmt(char *filename, int cmode) for (i=0; i<6; i++) for (j=0; j<4; j++) rh[j][i] = -1.0; err = ncmpi_iget_varm_float(ncid, varid, start, count, NULL, imap, &rh[0][0], &req); CHECK_ERR - err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR + if (coll_io) + err = ncmpi_wait_all(ncid, 1, &req, &status); + else + err = ncmpi_wait(ncid, 1, &req, &status); + CHECK_ERR err = status; CHECK_ERR nerrs += check_read_contents(&rh[0][0]); /* test blocking API */ for (i=0; i<6; i++) for (j=0; j<4; j++) rh[j][i] = -1.0; - err = ncmpi_get_varm_float_all(ncid, varid, start, count, stride, imap, - &rh[0][0]); CHECK_ERR + if (coll_io) + err = ncmpi_get_varm_float_all(ncid, varid, start, count, stride, imap, &rh[0][0]); + else + err = ncmpi_get_varm_float(ncid, varid, start, count, stride, imap, &rh[0][0]); + CHECK_ERR nerrs += check_read_contents(&rh[0][0]); /* test when stride == NULL and imap != NULL */ for (i=0; i<6; i++) for (j=0; j<4; j++) rh[j][i] = -1.0; - err = ncmpi_get_varm_float_all(ncid, varid, start, count, NULL, imap, - &rh[0][0]); CHECK_ERR + if (coll_io) + err = ncmpi_get_varm_float_all(ncid, varid, start, count, NULL, imap, &rh[0][0]); + else + err = ncmpi_get_varm_float(ncid, varid, start, count, NULL, imap, &rh[0][0]); + CHECK_ERR nerrs += check_read_contents(&rh[0][0]); } err = ncmpi_close(ncid); CHECK_ERR - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_WRITE, MPI_INFO_NULL, &ncid); CHECK_ERR + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_WRITE, info, &ncid); + CHECK_FATAL_ERR + + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } err = ncmpi_inq_varid(ncid, "var", &varid); CHECK_ERR @@ -186,7 +244,11 @@ tst_fmt(char *filename, int cmode) start[0] = 0; start[1] = 0; count[0] = 6; count[1] = 4; if (rank > 0) count[0] = count[1] = 0; - err = ncmpi_put_vara_int_all(ncid, varid, start, count, &var[0][0]); CHECK_ERR + if (coll_io) + err = ncmpi_put_vara_int_all(ncid, varid, start, count, &var[0][0]); + else + err = ncmpi_put_vara_int(ncid, varid, start, count, &var[0][0]); + CHECK_ERR /* set the contents of the write buffer varT, a 4 x 6 char array 50, 51, 52, 53, 54, 55, @@ -203,34 +265,51 @@ tst_fmt(char *filename, int cmode) imap[0] = 1; imap[1] = 6; /* would be {4, 1} if not transposing */ if (rank > 0) count[0] = count[1] = 0; - if (cmode & NC_NETCDF4) { - err = ncmpi_put_varm_schar_all(ncid, varid, start, count, stride, - imap, &varT[0][0]); CHECK_ERR + if (format == NC_FORMAT_NETCDF4 || format == NC_FORMAT_NETCDF4_CLASSIC) { + if (coll_io) + err = ncmpi_put_varm_schar_all(ncid, varid, start, count, stride, imap, &varT[0][0]); + else + err = ncmpi_put_varm_schar(ncid, varid, start, count, stride, imap, &varT[0][0]); + CHECK_ERR nerrs += check_write_contents(&varT[0][0]); } else { /* test nonblocking API */ err = ncmpi_iput_varm_schar(ncid, varid, start, count, stride, imap, &varT[0][0], &req); CHECK_ERR - err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR + if (coll_io) + err = ncmpi_wait_all(ncid, 1, &req, &status); + else + err = ncmpi_wait(ncid, 1, &req, &status); + CHECK_ERR err = status; CHECK_ERR nerrs += check_write_contents(&varT[0][0]); /* test when stride == NULL and imap != NULL */ err = ncmpi_iput_varm_schar(ncid, varid, start, count, NULL, imap, &varT[0][0], &req); CHECK_ERR - err = ncmpi_wait_all(ncid, 1, &req, &status); CHECK_ERR + if (coll_io) + err = ncmpi_wait_all(ncid, 1, &req, &status); + else + err = ncmpi_wait(ncid, 1, &req, &status); + CHECK_ERR err = status; CHECK_ERR nerrs += check_write_contents(&varT[0][0]); /* test blocking API */ - err = ncmpi_put_varm_schar_all(ncid, varid, start, count, stride, imap, - &varT[0][0]); CHECK_ERR + if (coll_io) + err = ncmpi_put_varm_schar_all(ncid, varid, start, count, stride, imap, &varT[0][0]); + else + err = ncmpi_put_varm_schar(ncid, varid, start, count, stride, imap, &varT[0][0]); + CHECK_ERR nerrs += check_write_contents(&varT[0][0]); /* test when stride == NULL and imap != NULL */ - err = ncmpi_put_varm_schar_all(ncid, varid, start, count, NULL, imap, - &varT[0][0]); CHECK_ERR + if (coll_io) + err = ncmpi_put_varm_schar_all(ncid, varid, start, count, NULL, imap, &varT[0][0]); + else + err = ncmpi_put_varm_schar(ncid, varid, start, count, NULL, imap, &varT[0][0]); + CHECK_ERR nerrs += check_write_contents(&varT[0][0]); } @@ -240,69 +319,26 @@ tst_fmt(char *filename, int cmode) } /*----< main() >------------------------------------------------------------*/ -int main(int argc, char **argv) -{ - char filename[256], *hint_value; - int err, nerrs=0, rank, nprocs, bb_enabled=0; - - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for get/put varm ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } +int main(int argc, char **argv) { -#ifdef DEBUG - if (nprocs > 1 && rank == 0) - printf("Warning: %s is designed to run on 1 process\n", argv[0]); -#endif + int err; + loop_opts opt; - /* check whether burst buffering is enabled */ - if (inq_env_hint("nc_burst_buf", &hint_value)) { - if (strcasecmp(hint_value, "enable") == 0) bb_enabled = 1; - free(hint_value); - } + MPI_Init(&argc, &argv); - nerrs += tst_fmt(filename, 0); - nerrs += tst_fmt(filename, NC_64BIT_OFFSET); - if (!bb_enabled) { -#ifdef ENABLE_NETCDF4 - nerrs += tst_fmt(filename, NC_NETCDF4); - nerrs += tst_fmt(filename, NC_NETCDF4 | NC_CLASSIC_MODEL); -#endif - } - nerrs += tst_fmt(filename, NC_64BIT_DATA); - - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + err = tst_main(argc, argv, "get/put varm", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/testcases/tst_data_move.c b/test/testcases/tst_data_move.c new file mode 100644 index 0000000000..fecde2fc18 --- /dev/null +++ b/test/testcases/tst_data_move.c @@ -0,0 +1,648 @@ +/* + * Copyright (C) 2025, Northwestern University and Argonne National Laboratory + * See COPYRIGHT notice in top-level directory. + */ + +/* + * This program checks whether the data movement subroutines are called in the + * following scenario. + * 1. There is a sufficiently large free space in the header extent and fix-sized + * variable section, such that all successive adding new variables will not + * change the starting offsets of both fix-sized and record variable sections. + * 2. Add new fix-sized variables should not call the data movement subroutines. + * 3. When there is only one record, i.e. unlimited dimension size == 1, adding + * a new record variable should not call the data movement subroutines. + * 4. When there are two records, i.e. unlimited dimension size == 2, adding a + * new a new record variable SHOULD call the data movement subroutines, but + * only to move the 2nd record to a higher offset, and by moving the entire + * record, not individual record variables separately. + */ + +#include +#include +#include +#include /* strcasecmp() */ +#include /* basename() */ +#include +#include + +#include + +static int debug; + +#define LON 100 +#define LAT 100 +#define NVARS 10 + +#define PRINT_VAR_OFF \ + for (i=0; i 0 && old_var_off[i] != new_var_off[i]) { \ + fprintf(stderr,"Error: %s var %d old offset %6lld != new offset %6lld\n", \ + kind, i, old_var_off[i], new_var_off[i]); \ + nerrs++; \ + goto err_out; \ + } \ + if (debug && rank == 0) \ + printf("nvars=%d: %s var %d old offset %6lld new offset %6lld\n", \ + nvars, kind, i, old_var_off[i], new_var_off[i]); \ + old_var_off[i] = new_var_off[i]; \ + } + + +static +int read_back_check(int ncid, + int coll_io, + MPI_Offset *start, + MPI_Offset *count) +{ + char name[64]; + int i, j, k, err, nerrs=0, rank, nvars, ndims, tdim, *int_buf; + float *flt_buf; + MPI_Offset nelems, nrecords; + + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + nelems = count[1] * count[2]; + int_buf = (int*) malloc(sizeof(int) * nelems); + flt_buf = (float*) malloc(sizeof(float) * nelems); + + err = ncmpi_inq_nvars(ncid, &nvars); + CHECK_ERR + + err = ncmpi_inq_dimid(ncid, "time", &tdim); + CHECK_ERR + + err = ncmpi_inq_dimlen(ncid, tdim, &nrecords); + CHECK_ERR + + if (debug && rank == 0) printf("number of records=%lld\n",nrecords); + + for (i=0; i 0) { + fprintf(stderr,"Error at rank %d line %d: read_back_check() failed\n",rank, __LINE__); + nerrs++; + goto err_out; + } + + fix_vars_size = 0; + for (i=0; i 0) { + fprintf(stderr,"Error at rank %d line %d: read_back_check() failed\n",rank, __LINE__); + nerrs++; + goto err_out; + } + + if (debug && rank == 0) { + MPI_Offset fix_off, rec_off; + err = ncmpi_inq_varoffset(ncid, varid_1st_fix, &fix_off); + CHECK_ERR + printf("Line %d: fix_nvars=%d rec_nvars=%d \n", __LINE__,fix_nvars,rec_nvars); + printf("Line %d: first fix-sized var offset %6lld\n", __LINE__,fix_off); + err = ncmpi_inq_varoffset(ncid, varid_1st_rec, &rec_off); + CHECK_ERR + printf("Line %d: sum of all fix-sized vars %6lld\n", __LINE__,fix_vars_size); + printf("Line %d: fix-sized var free space %6lld\n", __LINE__, + rec_off - (fix_off + fix_vars_size)); + printf("Line %d: 1st record offset %6lld\n", __LINE__,rec_off); + } + + /* enter define mode and add a new record variable */ + err = ncmpi_redef(ncid); CHECK_ERR + + if (debug && rank == 0) { + printf("\n===================================================\n"); + printf("%s at %d: rec_nvars=%d adding a new record variable\n", + fname,__LINE__,rec_nvars); + } + + /* add a new record variable */ + err = ncmpi_def_var(ncid, "snow", NC_FLOAT, 3, dimids, &varid[5]); + CHECK_ERR + nvars++; + rec_nvars++; + + err = ncmpi_enddef(ncid); CHECK_ERR + + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + + err = ncmpi_inq_header_size(ncid, &h_size); CHECK_ERR + err = ncmpi_inq_header_extent(ncid, &h_extent); CHECK_ERR + err = ncmpi_inq_recsize(ncid, &rec_size); CHECK_ERR + if (debug && rank == 0) { + printf("%s at %d: after adding a new record variable\n", + fname,__LINE__); + printf("\t\t\theader size=%lld extent=%lld rec_size=%lld\n", + h_size, h_extent,rec_size); + } + + start[0] = 0; + for (i=0; i 0) { + fprintf(stderr,"Error at rank %d line %d: read_back_check() failed\n",rank, __LINE__); + nerrs++; + goto err_out; + } + + if (debug && rank == 0) { + MPI_Offset fix_off, rec_off; + err = ncmpi_inq_varoffset(ncid, varid_1st_fix, &fix_off); + CHECK_ERR + printf("Line %d: fix_nvars=%d rec_nvars=%d \n", __LINE__,fix_nvars,rec_nvars); + printf("Line %d: first fix-sized var offset %6lld\n", __LINE__,fix_off); + err = ncmpi_inq_varoffset(ncid, varid_1st_rec, &rec_off); + CHECK_ERR + printf("Line %d: sum of all fix-sized vars %6lld\n", __LINE__,fix_vars_size); + printf("Line %d: fix-sized var free space %6lld\n", __LINE__, + rec_off - (fix_off + fix_vars_size)); + printf("Line %d: 1st record offset %6lld\n", __LINE__,rec_off); + printf("Line %d: 2nd record offset %6lld\n", __LINE__,rec_off+rec_size); + } + + /* write 2nd record */ + if (debug && rank == 0) + printf("\n%s at %d: increment to 2 records\n", fname,__LINE__); + + start[0] = 1; + + for (i=0; i 0) { + fprintf(stderr,"Error at rank %d line %d: read_back_check() failed\n",rank, __LINE__); + nerrs++; + goto err_out; + } + + if (debug && rank == 0) { + MPI_Offset fix_off, rec_off; + err = ncmpi_inq_varoffset(ncid, varid_1st_fix, &fix_off); + CHECK_ERR + printf("Line %d: fix_nvars=%d rec_nvars=%d \n", __LINE__,fix_nvars,rec_nvars); + printf("Line %d: first fix-sized var offset %6lld\n", __LINE__,fix_off); + err = ncmpi_inq_varoffset(ncid, varid_1st_rec, &rec_off); + CHECK_ERR + printf("Line %d: sum of all fix-sized vars %6lld\n", __LINE__,fix_vars_size); + printf("Line %d: fix-sized var free space %6lld\n", __LINE__, + rec_off - (fix_off + fix_vars_size)); + printf("Line %d: 1st record offset %6lld\n", __LINE__,rec_off); + printf("Line %d: 2nd record offset %6lld\n", __LINE__,rec_off+rec_size); + } + +err_out: + err = ncmpi_close(ncid); CHECK_ERR + + if (int_buf != NULL) free(int_buf); + if (flt_buf != NULL) free(flt_buf); + return nerrs; +} + +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) +{ + int nerrs; + + /* test without setting hint nc_data_move_chunk_size */ + nerrs = tst_data_move(out_path, format, coll_io, info); + if (nerrs > 0) return nerrs; + + /* test with setting hint nc_data_move_chunk_size */ + MPI_Info_set(info, "nc_data_move_chunk_size", "5432"); + nerrs = tst_data_move(out_path, format, coll_io, info); + if (nerrs > 0) return nerrs; + + return 0; +} + +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "growing data section", opt, test_io); + + MPI_Finalize(); + + return err; +} diff --git a/test/testcases/tst_def_var_fill.c b/test/testcases/tst_def_var_fill.c index 88c98ec6b7..022364b279 100644 --- a/test/testcases/tst_def_var_fill.c +++ b/test/testcases/tst_def_var_fill.c @@ -31,21 +31,21 @@ #define NX 5 static int -tst_fmt(char *filename, int cmode) +tst_fmt(const char *out_path, int format, int coll_io, MPI_Info info) { int i, j, rank, nprocs, err, nerrs=0; - int ncid, format, varid[2], dimid[2], expect, *buf; + int ncid, fmt, varid[2], dimid[2], expect, *buf; MPI_Offset start[2], count[2]; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - /* allocate I/O buffer */ - buf = (int*) malloc(sizeof(int) * NY*NX); + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR /* create a new file for writing ------------------------------------*/ - cmode |= NC_CLOBBER; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR /* define dimension */ err = ncmpi_def_dim(ncid, "Y", NY, &dimid[0]); CHECK_ERR @@ -61,6 +61,11 @@ tst_fmt(char *filename, int cmode) err = ncmpi_enddef(ncid); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + #ifdef STRONGER_CONSISTENCY err = ncmpi_sync(ncid); CHECK_ERR MPI_Barrier(MPI_COMM_WORLD); @@ -68,6 +73,8 @@ tst_fmt(char *filename, int cmode) #endif /* initialize I/O buffer */ + buf = (int*) malloc(sizeof(int) * NY*NX); + for (i=0; i 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for def_var_fill ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } + nerrs = tst_fmt(out_path, format, coll_io, info); - /* check whether burst buffering is enabled */ - if (inq_env_hint("nc_burst_buf", &hint_value)) { - if (strcasecmp(hint_value, "enable") == 0) bb_enabled = 1; - free(hint_value); - } + return nerrs; +} - nerrs += tst_fmt(filename, 0); - nerrs += tst_fmt(filename, NC_64BIT_OFFSET); - if (!bb_enabled) { -#ifdef ENABLE_NETCDF4 - nerrs += tst_fmt(filename, NC_NETCDF4); - nerrs += tst_fmt(filename, NC_NETCDF4 | NC_CLASSIC_MODEL); -#endif - } - nerrs += tst_fmt(filename, NC_64BIT_DATA); - - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } +int main(int argc, char **argv) { - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + int err; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "def_var_fill", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/testcases/tst_del_attr.c b/test/testcases/tst_del_attr.c index 29c48e2d84..8313b236a0 100644 --- a/test/testcases/tst_del_attr.c +++ b/test/testcases/tst_del_attr.c @@ -33,14 +33,18 @@ #define ATTR_EXP_ERR(expect_err,name) { \ if (err != expect_err) { \ - printf("Error at line %d in %s: attr=%s err=%s\n", \ + fprintf(stderr,"Error at line %d in %s: attr=%s err=%s\n", \ __LINE__,__FILE__,name,ncmpi_strerrno(err)); \ nerrs++; \ } \ } -static int -tst_fmt(char *filename, int cmode) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) { char *attr_name[12] = {"attr_NC_NAT", "attr_NC_BYTE", @@ -60,17 +64,22 @@ tst_fmt(char *filename, int cmode) MPI_Comm_rank(MPI_COMM_WORLD, &rank); - if (cmode == 0 || cmode == NC_64BIT_OFFSET || cmode & NC_CLASSIC_MODEL) + if (format == NC_FORMAT_CLASSIC || + format == NC_FORMAT_64BIT_OFFSET || + format == NC_FORMAT_NETCDF4_CLASSIC) max_type = NC_DOUBLE; else max_type = NC_UINT64; for (i=0; i<1024; i++) buf[i]=0; + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + for (i=NC_BYTE; i<=max_type; i++) { /* create a new file (or truncate it to 0 length) */ - cmode |= NC_CLOBBER; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR if (i == NC_CHAR) { for (j=0; j<3; j++) buf[j]='a'+j; @@ -84,7 +93,7 @@ tst_fmt(char *filename, int cmode) err = ncmpi_close(ncid); CHECK_ERR /* reopen the file */ - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_WRITE, MPI_INFO_NULL, &ncid); CHECK_ERR + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_WRITE, info, &ncid); CHECK_ERR err = ncmpi_redef(ncid); CHECK_ERR err = ncmpi_del_att(ncid, NC_GLOBAL, attr_name[i]); @@ -93,7 +102,7 @@ tst_fmt(char *filename, int cmode) /* call enddef to recalculate the header size */ err = ncmpi_enddef(ncid); CHECK_ERR - if (!(cmode & NC_NETCDF4)) { + if (format != NC_FORMAT_NETCDF4 && format != NC_FORMAT_NETCDF4_CLASSIC) { /* obtained updated header size */ err = ncmpi_inq_header_size(ncid, &header_size); CHECK_ERR } @@ -111,87 +120,50 @@ tst_fmt(char *filename, int cmode) off_t file_size; /* remove file type prefix substring */ - char *fname = remove_file_system_type_prefix(filename); + char *fname = remove_file_system_type_prefix(out_path); int fd = open(fname, O_RDONLY, 0666); if (fd == -1) { - printf("Error: file open %s (%s)\n",fname,strerror(errno)); + fprintf(stderr,"Error: file open %s (%s)\n",fname,strerror(errno)); return 1; } /* obtain file size */ file_size = lseek(fd, 0, SEEK_END); - if (!(cmode & NC_NETCDF4) && file_size != header_size) + if (format != NC_FORMAT_NETCDF4 && format != NC_FORMAT_NETCDF4_CLASSIC && + file_size != header_size) printf("Warning: expected file size "OFFFMT" but got %lld\n", header_size, (long long)file_size); close(fd); } } + return nerrs; } -int main(int argc, char* argv[]) -{ - char filename[256], *hint_value; - int err, nerrs=0, rank, bb_enabled=0; +int main(int argc, char **argv) { - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for testing delete attr ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } + int err; + loop_opts opt; - /* check whether burst buffering is enabled */ - if (inq_env_hint("nc_burst_buf", &hint_value)) { - if (strcasecmp(hint_value, "enable") == 0) bb_enabled = 1; - free(hint_value); - } + MPI_Init(&argc, &argv); - nerrs += tst_fmt(filename, 0); - nerrs += tst_fmt(filename, NC_64BIT_OFFSET); - if (!bb_enabled) { -#ifdef ENABLE_NETCDF4 - nerrs += tst_fmt(filename, NC_NETCDF4); - nerrs += tst_fmt(filename, NC_NETCDF4 | NC_CLASSIC_MODEL); -#endif - } - nerrs += tst_fmt(filename, NC_64BIT_DATA); - - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + err = tst_main(argc, argv, "delete attr", opt, test_io); MPI_Finalize(); - return (nerrs > 0); + return err; } - diff --git a/test/testcases/tst_dimsizes.c b/test/testcases/tst_dimsizes.c index 441afc1a70..cbd145706e 100644 --- a/test/testcases/tst_dimsizes.c +++ b/test/testcases/tst_dimsizes.c @@ -48,109 +48,93 @@ * MPI_Offset is a signed long long. */ -int -main(int argc, char **argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) { - char filename[256]; - int rank, nprocs, err, nerrs=0; - int ncid, dimid; + int err, nerrs=0, fmt, ncid, dimid; MPI_Offset dimsize; - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for defining max dimension sizes ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - - /* Writing Max Dimension Size For NC_CLASSIC */ - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, MPI_INFO_NULL, &ncid); CHECK_ERR - dimsize = DIMMAXCLASSIC; - err = ncmpi_def_dim(ncid, "testdim", dimsize, &dimid); CHECK_ERR + /* Writing Max Dimension Size */ + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR dimsize = -1; err = ncmpi_def_dim(ncid, "testdim1", dimsize, &dimid); EXP_ERR(NC_EDIMSIZE) - dimsize = (MPI_Offset)DIMMAXCLASSIC+1; - err = ncmpi_def_dim(ncid, "testdim1", dimsize, &dimid); EXP_ERR(NC_EDIMSIZE) - err = ncmpi_close(ncid); CHECK_ERR - /* Reading Max Dimension Size For NC_CLASSIC */ - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOCLOBBER, MPI_INFO_NULL, &ncid); CHECK_ERR - err = ncmpi_inq_dimid(ncid, "testdim", &dimid); CHECK_ERR - err = ncmpi_inq_dimlen(ncid, dimid, &dimsize); CHECK_ERR - if (dimsize != DIMMAXCLASSIC) { - printf("Error at line %d in %s: expecting dimsize %d but got "OFFFMT"\n", __LINE__,__FILE__,DIMMAXCLASSIC,dimsize); - nerrs++; + if (format == NC_FORMAT_CLASSIC) { + dimsize = DIMMAXCLASSIC; + err = ncmpi_def_dim(ncid, "testdim", dimsize, &dimid); CHECK_ERR + dimsize = (MPI_Offset)DIMMAXCLASSIC+1; + err = ncmpi_def_dim(ncid, "testdim1", dimsize, &dimid); EXP_ERR(NC_EDIMSIZE) + } + else if (format == NC_FORMAT_64BIT_OFFSET) { + dimsize = DIMMAX64OFFSET; + err = ncmpi_def_dim(ncid, "testdim", dimsize, &dimid); CHECK_ERR + dimsize = (MPI_Offset)DIMMAX64OFFSET+1; + err = ncmpi_def_dim(ncid, "testdim1", dimsize, &dimid); EXP_ERR(NC_EDIMSIZE) + } + else { + dimsize = DIMMAX64DATA; + err = ncmpi_def_dim(ncid, "testdim", dimsize, &dimid); CHECK_ERR + dimsize = -1; + err = ncmpi_def_dim(ncid, "testdim1", dimsize, &dimid); EXP_ERR(NC_EDIMSIZE) } - err = ncmpi_close(ncid); CHECK_ERR - /* Writing Max Dimension Size For NC_64BIT_OFFSET */ - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER | NC_64BIT_OFFSET, MPI_INFO_NULL, &ncid); CHECK_ERR - dimsize = DIMMAX64OFFSET; - err = ncmpi_def_dim(ncid, "testdim", dimsize, &dimid); CHECK_ERR - dimsize = -1; - err = ncmpi_def_dim(ncid, "testdim1", dimsize, &dimid); EXP_ERR(NC_EDIMSIZE) - dimsize = (MPI_Offset)DIMMAX64OFFSET+1; - err = ncmpi_def_dim(ncid, "testdim1", dimsize, &dimid); EXP_ERR(NC_EDIMSIZE) err = ncmpi_close(ncid); CHECK_ERR - /* Reading Max Dimension Size For NC_64BIT_OFFSET */ - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOCLOBBER, MPI_INFO_NULL, &ncid); CHECK_ERR + /* Reading Max Dimension Size */ + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_NOCLOBBER, info, &ncid); CHECK_ERR + err = ncmpi_inq_format(ncid, &fmt); CHECK_ERR err = ncmpi_inq_dimid(ncid, "testdim", &dimid); CHECK_ERR err = ncmpi_inq_dimlen(ncid, dimid, &dimsize); CHECK_ERR - if (dimsize != DIMMAX64OFFSET) { - printf("Error at line %d in %s: expecting dimsize %d but got "OFFFMT"\n", __LINE__,__FILE__,DIMMAX64OFFSET,dimsize); + if (format == NC_FORMAT_CLASSIC && dimsize != DIMMAXCLASSIC) { + fprintf(stderr,"Error at line %d in %s: expecting dimsize %d but got "OFFFMT"\n", + __LINE__,__FILE__,DIMMAXCLASSIC,dimsize); nerrs++; } - err = ncmpi_close(ncid); CHECK_ERR - - /* Writing Max Dimension Size For NC_64BIT_DATA */ - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER | NC_64BIT_DATA, MPI_INFO_NULL, &ncid); CHECK_ERR - dimsize = DIMMAX64DATA; - err = ncmpi_def_dim(ncid, "testdim", dimsize, &dimid); CHECK_ERR - dimsize = -1; - err = ncmpi_def_dim(ncid, "testdim1", dimsize, &dimid); EXP_ERR(NC_EDIMSIZE) - err = ncmpi_close(ncid); CHECK_ERR - - /* Reading Max Dimension Size For NC_64BIT_DATA */ - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOCLOBBER, MPI_INFO_NULL, &ncid); CHECK_ERR - err = ncmpi_inq_dimid(ncid, "testdim", &dimid); CHECK_ERR - err = ncmpi_inq_dimlen(ncid, dimid, &dimsize); CHECK_ERR - if (dimsize != DIMMAX64DATA) { - printf("Error at line %d in %s: expecting dimsize %lld but got "OFFFMT"\n", __LINE__,__FILE__,(long long)DIMMAX64DATA,dimsize); + else if (format == NC_FORMAT_64BIT_OFFSET && dimsize != DIMMAX64OFFSET) { + fprintf(stderr,"Error at line %d in %s: expecting dimsize %d but got "OFFFMT"\n", + __LINE__,__FILE__,DIMMAX64OFFSET,dimsize); nerrs++; } + else if (format == NC_FORMAT_64BIT_DATA && dimsize != DIMMAX64DATA) { + fprintf(stderr,"Error at line %d in %s: expecting dimsize %lld but got "OFFFMT"\n", + __LINE__,__FILE__,(long long)DIMMAX64DATA,dimsize); + nerrs++; + } + err = ncmpi_close(ncid); CHECK_ERR - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + return nerrs; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ + + err = tst_main(argc, argv, "defining max dimension sizes", opt, test_io); MPI_Finalize(); - return (nerrs > 0); + + return err; } diff --git a/test/testcases/tst_free_comm.c b/test/testcases/tst_free_comm.c index 966bd9705e..50017d029d 100644 --- a/test/testcases/tst_free_comm.c +++ b/test/testcases/tst_free_comm.c @@ -23,110 +23,66 @@ #include -static int -tst_fmt(char *fname, int cmode) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) { - int nerrs=0, err, exp_err=NC_NOERR, ncid; + int nerrs=0, err, ncid; MPI_Comm comm=MPI_COMM_NULL; - MPI_Info info=MPI_INFO_NULL; - -#ifndef ENABLE_NETCDF4 - if (cmode & NC_NETCDF4) - exp_err = NC_ENOTBUILT; -#endif /* duplicate MPI_COMM_WORLD */ MPI_Comm_dup(MPI_COMM_WORLD, &comm); - /* create MPI I/O hints */ - MPI_Info_create(&info); - - if (! (cmode & NC_NETCDF4)) - /* this hint may cause H5Fflush() to hang */ - MPI_Info_set(info, "romio_no_indep_rw", "true"); + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR /* create a file */ - cmode |= NC_CLOBBER; - err = ncmpi_create(comm, fname, cmode, info, &ncid); EXP_ERR(exp_err) - if (err == NC_ENOTBUILT) goto fn_exit; + err = ncmpi_create(comm, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR MPI_Comm_free(&comm); comm = MPI_COMM_NULL; - MPI_Info_free(&info); info = MPI_INFO_NULL; err = ncmpi_close(ncid); CHECK_ERR - /* open the file */ - /* duplicate MPI_COMM_WORLD */ MPI_Comm_dup(MPI_COMM_WORLD, &comm); - /* create MPI I/O hints */ - MPI_Info_create(&info); - MPI_Info_set(info, "romio_no_indep_rw", "true"); - - err = ncmpi_open(comm, fname, NC_NOWRITE, info, &ncid); CHECK_ERR + /* open the file */ + err = ncmpi_open(comm, out_path, NC_NOWRITE, info, &ncid); CHECK_ERR MPI_Comm_free(&comm); comm = MPI_COMM_NULL; - MPI_Info_free(&info); info = MPI_INFO_NULL; err = ncmpi_close(ncid); CHECK_ERR -fn_exit: if (comm != MPI_COMM_NULL) MPI_Comm_free(&comm); - if (info != MPI_INFO_NULL) MPI_Info_free(&info); + return nerrs; } -/*----< main() >------------------------------------------------------------*/ int main(int argc, char **argv) { - char filename[256]; - int err, nerrs=0, rank, nprocs; + int err; + loop_opts opt; MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for freeing MPI communicator ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - - nerrs += tst_fmt(filename, 0); - nerrs += tst_fmt(filename, NC_64BIT_OFFSET); - nerrs += tst_fmt(filename, NC_NETCDF4); - nerrs += tst_fmt(filename, NC_NETCDF4|NC_CLASSIC_MODEL); - nerrs += tst_fmt(filename, NC_64BIT_DATA); - - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } - - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = false;/* skip ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ + + err = tst_main(argc, argv, "freeing MPI communicator", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/testcases/tst_grow_data.c b/test/testcases/tst_grow_data.c new file mode 100644 index 0000000000..01fba8ca3b --- /dev/null +++ b/test/testcases/tst_grow_data.c @@ -0,0 +1,471 @@ +/* + * Copyright (C) 2025, Northwestern University and Argonne National Laboratory + * See COPYRIGHT notice in top-level directory. + */ + +/* + * This program tests adding new fix-sized and record variables by re-entering + * the define mode, without growing file header. + */ + +#include +#include +#include +#include /* strcasecmp() */ +#include /* basename() */ +#include +#include + +#include + +static int debug; + +#define LON 100 +#define LAT 100 +#define NVARS 10 + +#define PRINT_VAR_OFF \ + for (i=0; i 0) { + fprintf(stderr,"Error at rank %d line %d: read_back_check() failed\n",rank, __LINE__); + nerrs++; + goto err_out; + } + + for (i=0; i 0) { + fprintf(stderr,"Error at rank %d line %d: read_back_check() failed\n",rank, __LINE__); + nerrs++; + goto err_out; + } + + /* enter define mode and add a new record variable */ + err = ncmpi_redef(ncid); CHECK_ERR + + /* add a new record variable */ + err = ncmpi_def_var(ncid, "snow", NC_FLOAT, 3, dimids, &varid[5]); + CHECK_ERR + nvars++; + + err = ncmpi_enddef(ncid); CHECK_ERR + + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + + err = ncmpi_inq_header_size(ncid, &h_size); CHECK_ERR + err = ncmpi_inq_header_extent(ncid, &h_extent); CHECK_ERR + if (debug && rank == 0) + printf("%s at %d: header size=%lld extent=%lld\n", fname,__LINE__, + h_size, h_extent); + + start[0] = 0; + for (i=0; i 0) { + fprintf(stderr,"Error at rank %d line %d: read_back_check() failed\n",rank, __LINE__); + nerrs++; + goto err_out; + } + + err = ncmpi_close(ncid); CHECK_ERR + + for (i=0; i<2; i++) + if (int_buf[i] != NULL) free(int_buf[i]); + for (i=0; i<4; i++) + if (flt_buf[i] != NULL) free(flt_buf[i]); + +err_out: + return nerrs; +} + +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) +{ + int nerrs; + + /* test without setting hint nc_data_move_chunk_size */ + nerrs = tst_grow_data(out_path, format, coll_io, info); + if (nerrs > 0) return nerrs; + + /* test with setting hint nc_data_move_chunk_size */ + MPI_Info_set(info, "nc_data_move_chunk_size", "4096"); + nerrs = tst_grow_data(out_path, format, coll_io, info); + if (nerrs > 0) return nerrs; + + return 0; +} + +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "growing data section", opt, test_io); + + MPI_Finalize(); + + return err; +} diff --git a/test/testcases/tst_grow_header.c b/test/testcases/tst_grow_header.c index 76b72343e6..f165a2fe4b 100644 --- a/test/testcases/tst_grow_header.c +++ b/test/testcases/tst_grow_header.c @@ -50,6 +50,7 @@ static int verbose; static int check_vars(MPI_Comm comm, int ncid, + int coll_io, MPI_Offset *start, MPI_Offset *count) { @@ -68,7 +69,10 @@ check_vars(MPI_Comm comm, err = ncmpi_inq_vardimid(ncid, id, dimids); CHECK_ERROUT if (dimids[0] == rec_dim) continue; - err = ncmpi_get_vara_int_all(ncid, id, start+1, count+1, buf); + if (coll_io) + err = ncmpi_get_vara_int_all(ncid, id, start+1, count+1, buf); + else + err = ncmpi_get_vara_int(ncid, id, start+1, count+1, buf); CHECK_ERR for (j=0; j 0) { \ - printf("Error at line %d in %s: variable contents unexpected\n", \ + fprintf(stderr,"Error at line %d in %s: variable contents unexpected\n", \ __LINE__,__FILE__ ); \ goto err_out; \ } \ @@ -181,19 +188,21 @@ check_vars(MPI_Comm comm, } #define WRITE_FIX_VAR(id) { \ for (i=0; i 0) return nerrs; - verbose = 0; + /* test with setting hint nc_data_move_chunk_size */ + MPI_Info_set(info, "nc_data_move_chunk_size", "4096"); + nerrs = tst_grow_header(out_path, format, coll_io, info); + if (nerrs > 0) return nerrs; - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "tst_grow_header.nc"); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for header grow ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - if (verbose) printf("\n"); - } - cmode[0] = 0; - cmode[1] = NC_64BIT_OFFSET; - cmode[2] = NC_64BIT_DATA; + return 0; +} - for (i=0; i<3; i++) { - nerrs += tst_fmt(filename, cmode[i]); - if (nerrs > 0) goto main_exit; - } +int main(int argc, char **argv) { - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; -main_exit: - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "header grow", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/testcases/tst_info.c b/test/testcases/tst_info.c index a3080f78ee..b6df37aaa9 100644 --- a/test/testcases/tst_info.c +++ b/test/testcases/tst_info.c @@ -38,7 +38,7 @@ #define CHECK_HINT(hint) { \ MPI_Info_get(info_used, hint, len, value, &flag); \ if (!flag) { \ - printf("Error: hint \"%s\" is missing\n", hint); \ + fprintf(stderr,"Error: hint \"%s\" is missing\n", hint); \ nerrs++; \ } \ } @@ -65,29 +65,23 @@ int check_pnetcdf_hints(int ncid) return nerrs; } -int main(int argc, char** argv) { - char filename[256], value[MPI_MAX_INFO_VAL]; +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* not used */ + MPI_Info global_info) /* not used */ +{ + char value[MPI_MAX_INFO_VAL]; int ncid1, ncid2, rank, err, nerrs=0, len, flag, varid; MPI_Offset header_size, header_extent, expect; MPI_Info info, info_used; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for merging env info ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR /* create a new file and keep it opened to make ncmpii_mem_root not NULL */ err = ncmpi_create(MPI_COMM_WORLD, "dummy", NC_CLOBBER, MPI_INFO_NULL, &ncid1); CHECK_ERR @@ -110,7 +104,7 @@ int main(int argc, char** argv) { MPI_Info_set(info, "nc_var_align_size", "197"); /* size in bytes */ /* create another new file using a non-NULL MPI info --------------------*/ - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, info, &ncid2); CHECK_ERR + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid2); CHECK_ERR MPI_Info_free(&info); @@ -156,12 +150,12 @@ int main(int argc, char** argv) { MPI_Info_get(info_used, "nc_var_align_size", len+1, value, &flag); expect = PNETCDF_RNDUP(197, 4); if (expect != strtoll(value,NULL,10)) { - printf("Error: nc_var_align_size expect "OFFFMT" but got %lld\n", + fprintf(stderr,"Error: nc_var_align_size expect "OFFFMT" but got %lld\n", expect, strtoll(value,NULL,10)); nerrs++; } } else { - printf("Error: hint \"nc_var_align_size\" is missing\n"); + fprintf(stderr,"Error: hint \"nc_var_align_size\" is missing\n"); nerrs++; } @@ -169,7 +163,7 @@ int main(int argc, char** argv) { if (flag) { MPI_Info_get(info_used, "romio_ds_write", len+1, value, &flag); if (strcasecmp("disable", value)) { - printf("Error: romio_ds_write expect \"disable\" but got \"%s\"\n", + fprintf(stderr,"Error: romio_ds_write expect \"disable\" but got \"%s\"\n", value); nerrs++; } @@ -178,21 +172,21 @@ int main(int argc, char** argv) { MPI_Info_get_valuelen(info_used, "pnetcdf_subfiling", &len, &flag); if (flag) { MPI_Info_get(info_used, "pnetcdf_subfiling", len+1, value, &flag); -#ifdef ENABLE_SUBFILING +#if PNETCDF_SUBFILING == 1 if (strcasecmp("enable", value)) { - printf("Error: pnetcdf_subfiling expect \"enable\" but got \"%s\"\n", + fprintf(stderr,"Error: pnetcdf_subfiling expect \"enable\" but got \"%s\"\n", value); nerrs++; } #else if (strcasecmp("disable", value)) { - printf("Error: pnetcdf_subfiling expect \"disable\" but got \"%s\"\n", + fprintf(stderr,"Error: pnetcdf_subfiling expect \"disable\" but got \"%s\"\n", value); nerrs++; } #endif } else { - printf("Error: hint \"pnetcdf_subfiling\" is missing\n"); + fprintf(stderr,"Error: hint \"pnetcdf_subfiling\" is missing\n"); nerrs++; } MPI_Info_free(&info_used); @@ -206,7 +200,7 @@ int main(int argc, char** argv) { } /* re-open the file and get the MPI info object */ - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid1); CHECK_ERR + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_NOWRITE, MPI_INFO_NULL, &ncid1); CHECK_ERR /* retrieve MPI info object and check if all PnetCDF recognizable hints are * present */ @@ -214,24 +208,30 @@ int main(int argc, char** argv) { err = ncmpi_close(ncid1); CHECK_ERR - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + return nerrs; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 0; /* skip test intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "merging env info", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/testcases/tst_inq_header_size.c b/test/testcases/tst_inq_header_size.c new file mode 100644 index 0000000000..fe9b527cb8 --- /dev/null +++ b/test/testcases/tst_inq_header_size.c @@ -0,0 +1,277 @@ +/* + * Copyright (C) 2025, Northwestern University and Argonne National Laboratory + * See COPYRIGHT notice in top-level directory. + */ + +/* + * This program tests a call to ncmpi_inq_header_size() when in the define + * mode, which should calculate and return the latest file header size. This + * can be useful for application users to decide how much free space to be + * preserved in the file header section, i.e. by setting argument h_minfree + * and/or v_align when calling ncmpi__enddef(). + */ + +#include +#include +#include +#include /* strcasecmp() */ +#include /* basename() */ +#include +#include + +#include + +static int debug; + +static +int tst_inq_header_size(const char *out_path, + int format, + MPI_Info info) +{ + char *str; + int err, nerrs=0, rank, ncid, dimids[2], varid, int_buf; + float flt_buf; + double *dbl_buf; + MPI_Offset old_h_size, old_h_extent, new_h_size, new_h_extent; + + debug = 0; + + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + /* create a file */ + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); + CHECK_ERR + + flt_buf = 1.234; + err = ncmpi_put_att(ncid, NC_GLOBAL, "_FillValue", NC_FLOAT, 1, &flt_buf); + CHECK_ERR + + err = ncmpi_def_dim(ncid, "X", 10, &dimids[0]); CHECK_ERR + CHECK_ERR + + err = ncmpi_def_var(ncid, "int_var", NC_INT, 1, dimids, &varid); + CHECK_ERR + + err = ncmpi_put_att(ncid, varid, "_FillValue", NC_FLOAT, 1, &flt_buf); + EXP_ERR(NC_EBADTYPE) + + int_buf = 5678; + err = ncmpi_put_att(ncid, varid, "_FillValue", NC_INT, 1, &int_buf); + CHECK_ERR + + err = ncmpi_def_var(ncid, "dbl_var", NC_DOUBLE, 1, dimids, &varid); + CHECK_ERR + + err = ncmpi_def_var(ncid, "short_var", NC_SHORT, 1, dimids, &varid); + CHECK_ERR + + err = ncmpi_set_fill(ncid, NC_FILL, NULL); CHECK_ERR + + err = ncmpi_inq_header_size(ncid, &old_h_size); CHECK_ERR + + err = ncmpi_inq_header_extent(ncid, &old_h_extent); CHECK_ERR + + if (debug && rank == 0) + printf("%s at %d: header size=%lld extent=%lld\n", __FILE__,__LINE__, + old_h_size, old_h_extent); + + if (old_h_extent != 0) { + fprintf(stderr,"Error at %d: expect file extent size to be 0 but got %lld\n", + __LINE__, old_h_extent); + nerrs++; + goto err_out; + } + + err = ncmpi_enddef(ncid); CHECK_ERR + + err = ncmpi_inq_header_size(ncid, &new_h_size); CHECK_ERR + + err = ncmpi_inq_header_extent(ncid, &new_h_extent); CHECK_ERR + + if (debug && rank == 0) + printf("%s at %d: header size=%lld extent=%lld\n", __FILE__,__LINE__, + new_h_size, new_h_extent); + + if (new_h_size != old_h_size) { + fprintf(stderr,"Error at %d: expect file header size %lld but got %lld\n", + __LINE__, old_h_size, new_h_size); + nerrs++; + goto err_out; + } + + if (new_h_extent <= old_h_extent) { + fprintf(stderr,"Error at %d: expect file extent size > %lld but got %lld\n", + __LINE__, old_h_extent, new_h_extent); + nerrs++; + goto err_out; + } + + /* file sync before reading */ + err = ncmpi_sync(ncid); + CHECK_ERR + MPI_Barrier(MPI_COMM_WORLD); + + err = ncmpi_close(ncid); CHECK_ERR + + old_h_size = new_h_size; + old_h_extent = new_h_extent; + + /* open the file */ + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_WRITE, info, &ncid); + CHECK_ERR + + err = ncmpi_inq_header_size(ncid, &new_h_size); CHECK_ERR + + err = ncmpi_inq_header_extent(ncid, &new_h_extent); CHECK_ERR + + if (debug && rank == 0) + printf("%s at %d: header size=%lld extent=%lld\n", __FILE__,__LINE__, + new_h_size, new_h_extent); + + if (new_h_size != old_h_size) { + fprintf(stderr,"Error at %d: expect file header size %lld but got %lld\n", + __LINE__, old_h_size, new_h_size); + nerrs++; + goto err_out; + } + + if (new_h_extent != old_h_extent) { + fprintf(stderr,"Error at %d: expect file extent size %lld but got %lld\n", + __LINE__, old_h_extent, new_h_extent); + nerrs++; + goto err_out; + } + + old_h_size = new_h_size; + old_h_extent = new_h_extent; + + /* enter define mode and add new a dimension and a variable */ + err = ncmpi_redef(ncid); CHECK_ERR + + str = "new global attribute of text data type"; + err = ncmpi_put_att_text(ncid, NC_GLOBAL, "global_attr", strlen(str), str); + CHECK_ERR + + err = ncmpi_def_dim(ncid, "time", NC_UNLIMITED, &dimids[0]); CHECK_ERR + CHECK_ERR + + err = ncmpi_def_dim(ncid, "Y", 10, &dimids[1]); CHECK_ERR + CHECK_ERR + + err = ncmpi_def_var(ncid, "new_int_var", NC_INT, 2, dimids, &varid); + CHECK_ERR + + dbl_buf = (double*) calloc(16, sizeof(double)); + err = ncmpi_put_att_double(ncid, varid, "attr", NC_DOUBLE, 16, dbl_buf); + CHECK_ERR + free(dbl_buf); + + err = ncmpi_inq_header_size(ncid, &new_h_size); CHECK_ERR + + err = ncmpi_inq_header_extent(ncid, &new_h_extent); CHECK_ERR + + if (debug && rank == 0) + printf("%s at %d: header size=%lld extent=%lld\n", __FILE__,__LINE__, + new_h_size, new_h_extent); + + if (new_h_size <= old_h_size) { + fprintf(stderr,"Error at %d: expect file header size > %lld but got %lld\n", + __LINE__, old_h_size, new_h_size); + nerrs++; + goto err_out; + } + + if (new_h_extent != old_h_extent) { + fprintf(stderr,"Error at %d: expect file extent size %lld but got %lld\n", + __LINE__, old_h_extent, new_h_extent); + nerrs++; + goto err_out; + } + + if (new_h_size > old_h_extent) + /* header size grows beyond the current file extent size */ + err = ncmpi__enddef(ncid, 0, 512, 0, 0); + else + err = ncmpi_enddef(ncid); + CHECK_ERR + + old_h_size = new_h_size; + old_h_extent = new_h_extent; + + err = ncmpi_inq_header_size(ncid, &new_h_size); CHECK_ERR + + err = ncmpi_inq_header_extent(ncid, &new_h_extent); CHECK_ERR + + if (debug && rank == 0) + printf("%s at %d: header size=%lld extent=%lld\n", __FILE__,__LINE__, + new_h_size, new_h_extent); + + if (new_h_size != old_h_size) { + fprintf(stderr,"Error at %d: expect file header size %lld but got %lld\n", + __LINE__, old_h_size, new_h_size); + nerrs++; + goto err_out; + } + + if (new_h_extent < old_h_extent) { + fprintf(stderr,"Error at %d: expect file extent size >= %lld but got %lld\n", + __LINE__, old_h_extent, new_h_extent); + nerrs++; + goto err_out; + } + + err = ncmpi_close(ncid); CHECK_ERR + +err_out: + return nerrs; +} + +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) +{ + int nerrs; + + /* test without setting hint nc_data_move_chunk_size */ + nerrs = tst_inq_header_size(out_path, format, info); + if (nerrs > 0) return nerrs; + + /* test with setting hint nc_data_move_chunk_size */ + MPI_Info_set(info, "nc_data_move_chunk_size", "100"); + nerrs = tst_inq_header_size(out_path, format, info); + if (nerrs > 0) return nerrs; + + return 0; +} + +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "in define mode", opt, test_io); + + MPI_Finalize(); + + return err; +} diff --git a/test/testcases/tst_max_var_dims.c b/test/testcases/tst_max_var_dims.c index a2f256db4a..1fcf4870ab 100644 --- a/test/testcases/tst_max_var_dims.c +++ b/test/testcases/tst_max_var_dims.c @@ -26,43 +26,30 @@ #include -int main(int argc, char** argv) { - char filename[256]; - int rank, nprocs, nerrs=0; - int err, ncid; #if NC_MAX_VAR_DIMS < INT_MAX - int i, varid, *dimid; -#endif - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for checking NC_MAX_VAR_DIMS ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) +{ + int err, nerrs=0, ncid, varid, *dimid; + size_t i; -#if NC_MAX_VAR_DIMS < INT_MAX - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, MPI_INFO_NULL, &ncid); CHECK_ERR + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR /* define dimensions */ dimid = (int*) malloc(sizeof(int) * (NC_MAX_VAR_DIMS+2)); err = ncmpi_def_dim(ncid, "dim0", NC_UNLIMITED, &dimid[0]); CHECK_ERR err = ncmpi_def_dim(ncid, "dim1", 1, &dimid[1]); CHECK_ERR - for (i=2; i 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } - - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } -#else - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, MPI_INFO_NULL, &ncid); CHECK_ERR - err = ncmpi_close(ncid); CHECK_ERR - if (rank == 0) printf(SKIP_STR); -#endif + return nerrs; +} + +int main(int argc, char **argv) { + + int err=0; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ + + err = tst_main(argc, argv, "checking NC_MAX_VAR_DIMS", opt, test_io); MPI_Finalize(); - return (nerrs > 0); + + return err; } +#else +int main(int argc, char **argv) { + int rank; + + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + if (rank == 0) + printf("*** TESTING C tst_max_var_dims - checking NC_MAX_VAR_DIMS -- skip\n"); + + MPI_Finalize(); + return 0; +} +#endif diff --git a/test/testcases/tst_multi_redefine.c b/test/testcases/tst_multi_redefine.c new file mode 100644 index 0000000000..12734ed4d0 --- /dev/null +++ b/test/testcases/tst_multi_redefine.c @@ -0,0 +1,377 @@ +/* + * Copyright (C) 2025, Northwestern University and Argonne National Laboratory + * See COPYRIGHT notice in top-level directory. + * + */ + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * + * This program tests entering define modes multiple times, growing header + * extension causing moving data section to plances of higher offsets, and + * checking the contents of all variables defined. + * + * The compile and run commands are given below. + * + * % mpicc -g -o tst_multi_redefine tst_multi_redefine.c -lpnetcdf + * + * % mpiexec -l -n 4 ./tst_multi_redefine testfile.nc + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include +#include +#include +#include /* strcasecmp() */ +#include /* basename() */ +#include /* getopt() */ + +#include + +#include + +#define NROUNDS 2 +#define NY 10 +#define NX 10 + +static int verbose; + +typedef unsigned char uchar; +typedef unsigned short ushort; +typedef unsigned int uint; +typedef long long longlong; +typedef unsigned long long ulonglong; + +#define END_DEF { \ + err = ncmpi_enddef(ncid); CHECK_ERROUT \ + if (!coll_io) { \ + err = ncmpi_begin_indep_data(ncid); \ + CHECK_ERR \ + } \ + err = ncmpi_inq_header_size(ncid, &new_hdr_size); CHECK_ERROUT \ + err = ncmpi_inq_header_extent(ncid, &new_hdr_ext); CHECK_ERROUT \ + if (verbose && rank == 0) { \ + printf("Add var %2d: header size grows from %4lld to %4lld\n", \ + k, old_hdr_size, new_hdr_size); \ + if (new_hdr_ext > old_hdr_ext) \ + printf("Add var %2d: header extension grows from %4lld to %4lld\n", \ + k, old_hdr_ext, new_hdr_ext); \ + } \ + old_hdr_size = new_hdr_size; \ + old_hdr_ext = new_hdr_ext; \ +} + +#define EXP_VAL(nn) ((j + nn + rank) % NC_MAX_BYTE) + +#define DEF_VAR(xtype) { \ + err = ncmpi_redef(ncid); CHECK_ERROUT \ + sprintf(str, "fix_var_%d", k); \ + err = ncmpi_def_var(ncid, str, xtype, 2, dimids+1, &fix_varids[k]); \ + CHECK_ERROUT \ + sprintf(str, "attribute of fix-sized variable %d", k); \ + err = ncmpi_put_att_text(ncid, fix_varids[k], "attr", strlen(str), str); \ + CHECK_ERROUT \ + sprintf(str, "rec_var_%d", k); \ + err = ncmpi_def_var(ncid, str, xtype, 3, dimids, &rec_varids[k]); \ + CHECK_ERROUT \ + sprintf(str, "attribute of record variable %d", k); \ + err = ncmpi_put_att_text(ncid, rec_varids[k], "attr", strlen(str), str); \ + CHECK_ERROUT \ + END_DEF \ +} + +#define PUT_BUF_CHAR { \ + char *buf = (char*) malloc(sizeof(char) * nelems); \ + for (j=0; j 0) return nerrs; + + /* test with setting hint nc_data_move_chunk_size */ + MPI_Info_set(info, "nc_data_move_chunk_size", "1024"); + nerrs = tst_multi_redefine(out_path, format, coll_io, info); + if (nerrs > 0) return nerrs; + + return 0; +} + +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "re-enter define mode", opt, test_io); + + MPI_Finalize(); + + return err; +} diff --git a/test/testcases/tst_pthread.c b/test/testcases/tst_pthread.c index 432472323c..5cc0024e9a 100644 --- a/test/testcases/tst_pthread.c +++ b/test/testcases/tst_pthread.c @@ -28,7 +28,7 @@ #include -#ifdef ENABLE_THREAD_SAFE +#if PNETCDF_THREAD_SAFE == 1 #include #ifndef PTHREAD_BARRIER_SERIAL_THREAD @@ -138,6 +138,7 @@ static pthread_barrier_t barr; typedef struct { int id; /* globally unique thread ID */ char fname[256]; /* output file name base */ + int coll_io; /* collective or indepndent data mode */ } thread_arg; /*----< thread_func() >------------------------------------------------------*/ @@ -146,13 +147,14 @@ void* thread_func(void *arg) { char filename[512]; int i, id, nprocs, cmode, err=0, nerrs=0, ncid, *ret, dimid[2], varid[2]; - int *ibuf; + int *ibuf, coll_io; double *dbuf; MPI_Offset start[2], count[2]; MPI_Info info; /* make a unique file name for each thread */ id = ((thread_arg*)arg)->id; + coll_io = ((thread_arg*)arg)->coll_io; sprintf(filename, "%s.%d", ((thread_arg*)arg)->fname, id); /* allocate I/O buffers and initialize their contents */ @@ -184,22 +186,43 @@ void* thread_func(void *arg) err = ncmpi_enddef(ncid); CHECK_ERR /* now we are in data mode */ + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + /* write a record to the record variable */ start[0] = 0; /* first record */ start[1] = 0; count[0] = 1; count[1] = NX; - err = ncmpi_put_vara_int_all(ncid, varid[0], start, count, ibuf); CHECK_ERR + if (coll_io) + err = ncmpi_put_vara_int_all(ncid, varid[0], start, count, ibuf); + else + err = ncmpi_put_vara_int(ncid, varid[0], start, count, ibuf); + CHECK_ERR /* write another record to the record variable */ start[0] = 2; /* third record */ start[1] = 0; count[0] = 1; count[1] = NX; - err = ncmpi_put_vara_int_all(ncid, varid[0], start, count, ibuf); CHECK_ERR + if (coll_io) + err = ncmpi_put_vara_int_all(ncid, varid[0], start, count, ibuf); + else + err = ncmpi_put_vara_int(ncid, varid[0], start, count, ibuf); + CHECK_ERR /* write to the fixed-size variable */ - err = ncmpi_put_var_double_all(ncid, varid[1], dbuf); CHECK_ERR + if (coll_io) + err = ncmpi_put_var_double_all(ncid, varid[1], dbuf); + else + err = ncmpi_put_var_double(ncid, varid[1], dbuf); + CHECK_ERR + + /* file sync before closing file */ + err = ncmpi_sync(ncid); + CHECK_ERR err = ncmpi_close(ncid); CHECK_ERR @@ -219,6 +242,12 @@ void* thread_func(void *arg) sprintf(filename, "%s.%d", ((thread_arg*)arg)->fname, id); err = ncmpi_open(MPI_COMM_SELF, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); CHECK_ERR + + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + err = ncmpi_inq_varid(ncid, "ivar", &varid[0]); CHECK_ERR err = ncmpi_inq_varid(ncid, "dvar", &varid[1]); CHECK_ERR @@ -228,10 +257,14 @@ void* thread_func(void *arg) start[1] = 0; count[0] = 1; count[1] = NX; - err = ncmpi_get_vara_int_all(ncid, varid[0], start, count, ibuf); CHECK_ERR + if (coll_io) + err = ncmpi_get_vara_int_all(ncid, varid[0], start, count, ibuf); + else + err = ncmpi_get_vara_int(ncid, varid[0], start, count, ibuf); + CHECK_ERR for (i=0; i-------------------------------------------------------------*/ -int main(int argc, char **argv) { - char filename[256]; - int i, err, nerrs=0, rank, providedT; +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) +{ + int i, err, nerrs=0, rank, ncid; thread_arg t_arg[NTHREADS]; /* must be unique to each thread */ - -#ifdef ENABLE_THREAD_SAFE pthread_t threads[NTHREADS]; - MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &providedT); -#else - MPI_Init(&argc, &argv); -#endif MPI_Comm_rank(MPI_COMM_WORLD, &rank); - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for thread safety ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - -#ifdef ENABLE_THREAD_SAFE -#ifdef DEBUG - if (rank == 0) { - switch (providedT) { - case MPI_THREAD_SINGLE: printf("Support MPI_THREAD_SINGLE\n"); - break; - case MPI_THREAD_FUNNELED: printf("Support MPI_THREAD_FUNNELED\n"); - break; - case MPI_THREAD_SERIALIZED: printf("Support MPI_THREAD_SERIALIZED\n"); - break; - case MPI_THREAD_MULTIPLE: printf("Support MPI_THREAD_MULTIPLE\n"); - break; - default: printf("Error MPI_Init_thread()\n"); break; - } - } -#endif - if (providedT != MPI_THREAD_MULTIPLE) { - if (!rank) { - char fname[512]; - printf("\nWarning: MPI provided thread support level is less than MPI_THREAD_MULTIPLE ---- skip this test\n"); - for (i=0; i 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); + MPI_Barrier(MPI_COMM_WORLD); + + /* open the newly created files to validate and then delete them */ + for (i=0; i 0); + + return err; } diff --git a/test/testcases/tst_redefine.c b/test/testcases/tst_redefine.c index 6f9330079e..0875d9b8a0 100644 --- a/test/testcases/tst_redefine.c +++ b/test/testcases/tst_redefine.c @@ -1,17 +1,25 @@ /* * Copyright (C) 2024, Northwestern University and Argonne National Laboratory * See COPYRIGHT notice in top-level directory. - * */ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * This program tests all alignment features available from PnetCDF: - * 1. v_align: header extent alignment (starting file offset of data section) - * 2. h_minfree: header free space - * 3. r_align: record variable section alignment - * 4. v_minfree: free space between record variable section and the end of last - * fix-sized variable. + * 1. h_minfree: free space in the header section, i.e. + * (header extent) - (header size) >= h_minfree + * 2. v_align: alignment of the beginning of the fix-size variable section, i.e. + * (header extent) % v_align == 0 + * If no fixed-size variable is defined, v_align is ignored. + * Default value of v_align is 512 (NC_DEFAULT_V_ALIGN defined in file + * src/drivers/ncmpio/ncmpio_NC.h. + * 3. v_minfree: free space between the end of last fix-sized variable and the + * record variable section. + * If no fixed-size variable is defined, v_minfree is ignored. + * 4. r_align: alignment of the beginning of the record variable section. + * If no fixed-size variable is defined, default value of r_align is + * 512, i.e. NC_DEFAULT_V_ALIGN. + * Otherwise, default value of r_align is 4. * * Tests are done by reentering the define mode multiple times. * @@ -28,35 +36,66 @@ #include #include /* strcasecmp() */ #include /* basename() */ +#include + #include +#ifdef STAND_ALONE +#define OFFFMT "%lld" +#define CHECK_ERR { \ + if (err != NC_NOERR) { \ + fprintf(stderr,"Error at line %d in %s: (%s)\n", \ + __LINE__,__FILE__,ncmpi_strerrno(err)); \ + assert(0); \ + } \ +} +#define CHECK_ERROUT { \ + if (err != NC_NOERR) { \ + nerrs++; \ + fprintf(stderr,"Error at line %d in %s: (%s)\n", \ + __LINE__,__FILE__,ncmpi_strerrno(err)); \ + goto err_out; \ + } \ +} +#else #include +#endif #define LEN 101 +#ifndef MAX +#define MAX(x, y) ((x) > (y) ? (x) : (y)) +#endif #define RNDUP(x, unit) ((((x) + (unit) - 1) / (unit)) * (unit)) static int verbose; -#define CHECK_VAL(ncid, varid, ii, val, expect) { \ - if (val != expect) { \ - char name[16]; \ - err = ncmpi_inq_varname(ncid, varid, name); \ - CHECK_ERROUT \ - printf("%s line %d: var %s i=%d expecting %d but got %d\n", \ - __func__,__LINE__,name,ii,expect,val); \ - nerrs++; \ - goto err_out; \ - } \ +#define CHECK_VAL(ncid, varid, ii, val, expect) { \ + if (val != expect) { \ + char name[16]; \ + int ndims; \ + err = ncmpi_inq_varndims(ncid, varid, &ndims); \ + CHECK_ERROUT \ + err = ncmpi_inq_varname(ncid, varid, name); \ + CHECK_ERROUT \ + if (ndims == 1) \ + printf("%s line %d: var %s[%d] expecting %d but got %d\n", \ + __func__,__LINE__,name,ii,expect,val); \ + else /* record variable */ \ + printf("%s line %d: var %s[%d][%d] expecting %d but got %d\n", \ + __func__,__LINE__,name,ii/LEN,ii%LEN,expect,val); \ + nerrs++; \ + goto err_out; \ + } \ } /*----< check_vars() >-------------------------------------------------------*/ /* read back variables from file and check their contents */ static int -check_vars(MPI_Comm comm, int ncid, int *varid) +check_vars(MPI_Comm comm, int ncid, int *varid, int coll_io) { - int i, nerrs=0, err, rank, *buf=NULL, nvars; - MPI_Offset start[2], count[2]; + int i, j, nerrs=0, err, rank, *buf[4], nvars, off_val; + MPI_Offset start[2], count[2], bufLen[4]; MPI_Comm_rank(comm, &rank); @@ -67,42 +106,86 @@ check_vars(MPI_Comm comm, int ncid, int *varid) start[1] = rank * LEN; count[1] = LEN; - buf = (int*) malloc(sizeof(int) * 2 * LEN); + buf[0] = (int*) malloc(sizeof(int) * 2 * LEN * 4); + for (i=0; i<2*LEN*4; i++) buf[0][i] = -1; /* check record variables */ count[0] = 2; - for (i=0; i 0) { \ - printf("Error at line %d in %s: check_vars failed\n", \ + fprintf(stderr,"Error at line %d in %s: check_vars failed\n", \ __LINE__,__FILE__); \ goto err_out; \ } \ @@ -179,29 +262,49 @@ check_vars(MPI_Comm comm, int ncid, int *varid) __LINE__,hsize, hsize+growth); \ } +/* Make sure NC_DEFAULT_V_ALIGN is set to the same value as the one defined in + * src/drivers/ncmpio/ncmpio_NC.h. + */ +#define NC_DEFAULT_V_ALIGN 512 + #define CHECK_ALIGNMENTS { \ /* hints set in MPI info precede ncmpi__enddef */ \ v_align = (env_v_align) ? env_v_align : \ - (!has_fix_vars && env_r_align) ? env_r_align : \ - (info_v_align) ? info_v_align : v_align; \ + (info_v_align) ? info_v_align : \ + (v_align > 0) ? v_align : NC_DEFAULT_V_ALIGN; \ r_align = (env_r_align) ? env_r_align : \ - (info_r_align) ? info_r_align : r_align; \ + (info_r_align) ? info_r_align : \ + (r_align > 0) ? r_align : \ + (has_fix_vars) ? 4 : NC_DEFAULT_V_ALIGN;\ + if (h_minfree == -1) h_minfree = 0; \ + if (v_minfree == -1) v_minfree = 0; \ exp_hsize = old_hsize + increment; \ - exp_extent = RNDUP(exp_hsize + h_minfree, v_align); \ - old_extent = RNDUP(old_extent, v_align); \ - exp_extent = (exp_extent < old_extent) ? old_extent : exp_extent; \ + exp_extent = exp_hsize + h_minfree; \ if (has_fix_vars) { \ - exp_r_begin = RNDUP(exp_extent + fix_v_size + v_minfree, r_align); \ - old_r_begin = RNDUP(old_r_begin, r_align); \ + exp_extent = MAX(exp_extent, old_extent); \ + exp_extent = RNDUP(exp_extent, v_align); \ + exp_r_begin = exp_extent + fix_v_size + v_minfree; \ + exp_r_begin = MAX(exp_r_begin, old_r_begin); \ + exp_r_begin = RNDUP(exp_r_begin, r_align); \ + } else { \ + exp_r_begin = MAX(exp_extent, old_r_begin); \ + exp_r_begin = RNDUP(exp_r_begin, r_align); \ + exp_extent = exp_r_begin; \ } \ - else /* v_minfree and r_align are ignored */ \ - exp_r_begin = exp_extent; \ - exp_r_begin = (exp_r_begin < old_r_begin) ? old_r_begin : exp_r_begin; \ - exp_h_free = exp_extent - exp_hsize; \ - exp_v_free = exp_r_begin - (exp_extent + fix_v_size); \ + exp_h_free = exp_extent - exp_hsize; \ + exp_v_free = exp_r_begin - (exp_extent + fix_v_size); \ CHECK_HEADER_SIZE \ } +#define PRINT_HINTS \ + if (verbose && rank == 0) { \ + printf("\n========================================\n"); \ + printf(" Line %d old hsize %lld extent %lld r_begin %lld has_fix_vars %d\n", \ + __LINE__,hsize,extent,r_begin, has_fix_vars); \ + printf(" Line %d ncmpi__enddef(h_minfree %lld v_align %lld v_minfree %lld r_align %lld) increment %lld\n", \ + __LINE__, h_minfree, v_align, v_minfree, r_align, increment); \ + } + /* test alignments hints * 1. set in environment variable PNETCDF_HINTS, * 2. set in ncmpi__enddef() @@ -209,22 +312,23 @@ check_vars(MPI_Comm comm, int ncid, int *varid) * Note precedence of hints: PNETCDF_HINTS > ncmpi__enddef() > MPI info. */ static int -tst_fmt(char *filename, - int cmode, +tst_fmt(const char *out_path, + int coll_io, + MPI_Info global_info, int has_fix_vars, MPI_Offset *env_align, /* [3] 0 means unset in PNETCDF_HINTS */ MPI_Offset *info_align) /* [3] 0 means unset in MPI info */ { int i, rank, nprocs, ncid, err, nerrs=0; - int *buf, dimid[3], varid[4]; + int *buf[4]={NULL,NULL,NULL,NULL}, dimid[3], varid[4]; MPI_Info info=MPI_INFO_NULL; - MPI_Offset start[2], count[2], increment, fix_v_size; + MPI_Offset bufLen[4], start[2], count[2], increment, fix_v_size; - MPI_Offset hsize=0, old_hsize, exp_hsize; - MPI_Offset extent=0, old_extent, exp_extent; + MPI_Offset hsize=0, old_hsize=-1, exp_hsize=-1; + MPI_Offset extent=0, old_extent=-1, exp_extent=-1; MPI_Offset h_free=0, old_h_free, exp_h_free; MPI_Offset v_free=0, old_v_free, exp_v_free; - MPI_Offset r_begin=0, old_r_begin, exp_r_begin; + MPI_Offset r_begin=0, old_r_begin=-1, exp_r_begin=-1; MPI_Offset h_minfree, v_align, v_minfree, r_align; MPI_Offset env_h_align=0, env_v_align=0, env_r_align=0; MPI_Offset info_h_align=0, info_v_align=0, info_r_align=0; @@ -234,9 +338,11 @@ tst_fmt(char *filename, MPI_Comm_rank(comm, &rank); MPI_Comm_size(comm, &nprocs); + MPI_Info_dup(global_info, &info); + if (verbose && rank == 0) - printf("---- cmode=%d has_fix_vars=%d env_align=%s info_align=%s\n", - cmode,has_fix_vars,(env_align==NULL)?"NULL":"SET", + printf("---- has_fix_vars=%d env_align=%s info_align=%s\n", + has_fix_vars,(env_align==NULL)?"NULL":"SET", (info_align==NULL)?"NULL":"SET"); if (env_align != NULL) { @@ -250,7 +356,6 @@ tst_fmt(char *filename, info_h_align = info_align[0]; /* 0 means unset in MPI info */ info_v_align = info_align[1]; /* 0 means unset in MPI info */ info_r_align = info_align[2]; /* 0 means unset in MPI info */ - MPI_Info_create(&info); if (info_h_align) { sprintf(str, OFFFMT, info_h_align); MPI_Info_set(info, "nc_header_align_size", str); @@ -266,14 +371,12 @@ tst_fmt(char *filename, if (info_v_align == 0) info_v_align = info_h_align; } if (verbose && rank == 0) - printf("---- cmode=%d has_fix_vars=%d env_align="OFFFMT" "OFFFMT" "OFFFMT" info_align="OFFFMT" "OFFFMT" "OFFFMT"\n", - cmode,has_fix_vars,env_h_align,env_v_align,env_r_align, + printf("---- has_fix_vars=%d env_align="OFFFMT" "OFFFMT" "OFFFMT" info_align="OFFFMT" "OFFFMT" "OFFFMT"\n", + has_fix_vars,env_h_align,env_v_align,env_r_align, info_h_align,info_v_align,info_r_align); - /* create a new file */ - cmode |= NC_CLOBBER; - err = ncmpi_create(comm, filename, cmode, info, &ncid); CHECK_ERR + err = ncmpi_create(comm, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "time", NC_UNLIMITED, &dimid[0]); CHECK_ERR err = ncmpi_def_dim(ncid, "dim", LEN*nprocs, &dimid[1]); CHECK_ERR @@ -287,53 +390,96 @@ tst_fmt(char *filename, } err = ncmpi_put_att_text(ncid, NC_GLOBAL, "attr", 0, NULL); CHECK_ERR + increment = 0; h_minfree = v_minfree = v_align = r_align = -1; + PRINT_HINTS err = ncmpi_enddef(ncid); CHECK_ERR + buf[0] = (int*) malloc(sizeof(int) * 2 * LEN * 4); + + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + /* write to all variables, 2 records */ start[0] = 0; start[1] = rank * LEN; count[0] = 2; count[1] = LEN; - buf = (int*) malloc(sizeof(int) * count[0] * count[1]); + bufLen[0] = count[0]*count[1]; + for (i=0; i 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "tst_redefine.nc"); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for header alignment ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - if (verbose) printf("\n"); - } - cmode[0] = 0; - cmode[1] = NC_64BIT_OFFSET; - cmode[2] = NC_64BIT_DATA; + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + /* retrieve value of environment variable PNETCDF_HINTS */ + saved_env = getenv("PNETCDF_HINTS"); + if (verbose && rank == 0 && saved_env != NULL) + printf("PNETCDF_HINTS=%s\n",saved_env); + + /* No alignment hints should be set in the environment variable + * PNETCDF_HINTS (there can be other kinds) and the info object passed into + * this subroutine (there can be other hints) before running this test + * program. Check seq_runs.sh and parallel_run.sh first before running this + * test program. + */ for (has_fix_vars=1; has_fix_vars>=0; has_fix_vars--) { - /* No hints set in environment variable PNETCDF_HINTS. - * No hints set in MPI Info object. - */ - unsetenv("PNETCDF_HINTS"); - for (i=0; i<3; i++) { - nerrs += tst_fmt(filename, cmode[i], has_fix_vars, NULL, NULL); - if (nerrs > 0) goto main_exit; - } + /* Test when there is no alignment hints set at all */ + nerrs += tst_fmt(out_path, coll_io, info, has_fix_vars, NULL, NULL); + if (nerrs > 0) goto err_out; + + /* Test when there is no alignment hints set in environment variable + * PNETCDF_HINTS and set alignment hints in MPI Info object. + */ info_align[0] = 28; /* 7 x 4 */ info_align[1] = 44; /* 11 x 4 */ info_align[2] = 52; /* 13 x 4 */ - /* No hints set in environment variable PNETCDF_HINTS. - * Hints set in MPI Info object. - */ - for (i=0; i<3; i++) { - nerrs += tst_fmt(filename, cmode[i], has_fix_vars, NULL, - info_align); - if (nerrs > 0) goto main_exit; - } + nerrs += tst_fmt(out_path, coll_io, info, has_fix_vars, NULL, info_align); + if (nerrs > 0) goto err_out; + /* Set hints in environment variable PNETCDF_HINTS, but no hints set in + * MPI Info object. + */ env_align[0] = 68; /* 17 x 4 */ env_align[1] = 76; /* 19 x 4 */ env_align[2] = 92; /* 23 x 4 */ @@ -566,23 +756,20 @@ int main(int argc, char** argv) env_align[0], env_align[1], env_align[2]); setenv("PNETCDF_HINTS", str, 1); - /* Set hints in environment variable PNETCDF_HINTS. - * No hints set in MPI Info object. - */ - for (i=0; i<3; i++) { - nerrs += tst_fmt(filename, cmode[i], has_fix_vars, env_align, NULL); - if (nerrs > 0) goto main_exit; - } + nerrs += tst_fmt(out_path, coll_io, info, has_fix_vars, env_align, NULL); + if (nerrs > 0) goto err_out; - /* Set hints in environment variable PNETCDF_HINTS. - * Set hints in MPI Info object. + /* Test if the alignment hints set in environment variable + * PNETCDF_HINTS take precedence over hints set in MPI Info object, when + * Hints are both set in environment variable PNETCDF_HINTS and in MPI + * Info object. */ - for (i=0; i<3; i++) { - nerrs += tst_fmt(filename, cmode[i], has_fix_vars, env_align, - info_align); - if (nerrs > 0) goto main_exit; - } + nerrs += tst_fmt(out_path, coll_io, info, has_fix_vars, env_align, info_align); + if (nerrs > 0) goto err_out; + /* Test a different set of alignment hints set in environment variable + * PNETCDF_HINTS. + */ env_align[0] = 68; /* 17 x 4 */ env_align[1] = 0; env_align[2] = 92; /* 23 x 4 */ @@ -590,15 +777,12 @@ int main(int argc, char** argv) env_align[0], env_align[2]); setenv("PNETCDF_HINTS", str, 1); - /* Set hints in environment variable PNETCDF_HINTS. - * No hints set in MPI Info object. - */ - for (i=0; i<3; i++) { - nerrs += tst_fmt(filename, cmode[i], has_fix_vars, env_align, - info_align); - if (nerrs > 0) goto main_exit; - } + nerrs += tst_fmt(out_path, coll_io, info, has_fix_vars, env_align, info_align); + if (nerrs > 0) goto err_out; + /* Test a different set of alignment hints set in environment variable + * PNETCDF_HINTS. + */ env_align[0] = 0; env_align[1] = 76; /* 19 x 4 */ env_align[2] = 92; /* 23 x 4 */ @@ -606,65 +790,113 @@ int main(int argc, char** argv) env_align[1], env_align[2]); setenv("PNETCDF_HINTS", str, 1); - /* Set hints in environment variable PNETCDF_HINTS. - * No hints set in MPI Info object. - */ - for (i=0; i<3; i++) { - nerrs += tst_fmt(filename, cmode[i], has_fix_vars, env_align, - info_align); - if (nerrs > 0) goto main_exit; - } + nerrs += tst_fmt(out_path, coll_io, info, has_fix_vars, env_align, info_align); + if (nerrs > 0) goto err_out; + /* Test a different set of alignment hints set in environment variable + * PNETCDF_HINTS. + */ env_align[0] = 0; env_align[1] = 76; /* 19 x 4 */ env_align[2] = 0; sprintf(str, "nc_var_align_size="OFFFMT"\n", env_align[1]); setenv("PNETCDF_HINTS", str, 1); - /* Set hints in environment variable PNETCDF_HINTS. - * No hints set in MPI Info object. - */ - for (i=0; i<3; i++) { - nerrs += tst_fmt(filename, cmode[i], has_fix_vars, env_align, - info_align); - if (nerrs > 0) goto main_exit; - } + nerrs += tst_fmt(out_path, coll_io, info, has_fix_vars, env_align, info_align); + if (nerrs > 0) goto err_out; + /* Test a different set of alignment hints set in environment variable + * PNETCDF_HINTS. + */ env_align[0] = 0; /* 17 x 4 */ env_align[1] = 0; env_align[2] = 92; /* 23 x 4 */ sprintf(str, "nc_record_align_size="OFFFMT"\n", env_align[2]); setenv("PNETCDF_HINTS", str, 1); - /* Set hints in environment variable PNETCDF_HINTS. - * No hints set in MPI Info object. - */ - for (i=0; i<3; i++) { - nerrs += tst_fmt(filename, cmode[i], has_fix_vars, env_align, - info_align); - if (nerrs > 0) goto main_exit; - } - } + nerrs += tst_fmt(out_path, coll_io, info, has_fix_vars, env_align, info_align); + if (nerrs > 0) goto err_out; - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); + /* restore original values set in environment variable PNETCDF_HINTS */ + if (saved_env != NULL) setenv("PNETCDF_HINTS", saved_env, 1); + else unsetenv("PNETCDF_HINTS"); } -main_exit: - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); +err_out: + return nerrs; +} + +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) +{ + int nerrs; + + /* test without setting hint nc_data_move_chunk_size */ + nerrs = tst_redefine(out_path, format, coll_io, info); + if (nerrs > 0) return nerrs; + + /* test with setting hint nc_data_move_chunk_size */ + MPI_Info_set(info, "nc_data_move_chunk_size", "300"); + nerrs = tst_redefine(out_path, format, coll_io, info); + if (nerrs > 0) return nerrs; + + return 0; +} + +#ifdef STAND_ALONE +int main(int argc, char **argv) { + + int err, rank, format, coll_io; + MPI_Info info; + + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + if (argc < 2) { + if (rank == 0) + printf("Usage: %s out_path\n",argv[0]); + MPI_Finalize(); + return 1; } + format = NC_FORMAT_CLASSIC; + coll_io = 1; + MPI_Info_create(&info); + + err = test_io(argv[1], NULL, format, coll_io, info); + + MPI_Info_free(&info); + MPI_Finalize(); - return (nerrs > 0); + + return err; } +#else +int main(int argc, char **argv) { + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "header alignment", opt, test_io); + + MPI_Finalize(); + + return err; +} +#endif diff --git a/test/testcases/tst_symlink.c b/test/testcases/tst_symlink.c index 828035bc25..32989beacd 100644 --- a/test/testcases/tst_symlink.c +++ b/test/testcases/tst_symlink.c @@ -32,36 +32,23 @@ } \ } -int main(int argc, char **argv) { - char *filename, *symlink_fname, *fname; - int err, nerrs=0, len, rank, ncid, verbose=0; +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) +{ + char *symlink_fname, *fname; + int err, nerrs=0, rank, ncid, verbose=0; struct stat statbuf; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) filename = strdup(argv[1]); - else filename = strdup("testfile.nc"); - len = (int)strlen(filename) + 1; - MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD); - MPI_Bcast(filename, len, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for NC_CLOBBER on symlink file ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - /* remove file system type prefix substring */ - fname = remove_file_system_type_prefix(filename); + fname = remove_file_system_type_prefix(out_path); - symlink_fname = (char*) malloc(strlen(filename) + 10); + symlink_fname = (char*) malloc(strlen(out_path) + 10); /* create a regular file and a symbolic link to it */ err = 0; @@ -91,17 +78,18 @@ int main(int argc, char **argv) { sync(); } MPI_Bcast(&err, 1, MPI_INT, 0, MPI_COMM_WORLD); - if (err != 0) { - nerrs++; - goto fn_exit; - } + if (err != 0) return 1; MPI_Barrier(MPI_COMM_WORLD); /* symlink_fname may have file system type prefix */ - sprintf(symlink_fname, "%s.symlink", filename); + sprintf(symlink_fname, "%s.symlink", out_path); + + /* Set file format */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR /* create a file in NC_CLOBBER mode */ - err = ncmpi_create(MPI_COMM_WORLD, symlink_fname, NC_CLOBBER, MPI_INFO_NULL, &ncid); + err = ncmpi_create(MPI_COMM_WORLD, symlink_fname, NC_CLOBBER, info, &ncid); CHECK_ERR err = ncmpi_close(ncid); CHECK_ERR @@ -130,27 +118,32 @@ int main(int argc, char **argv) { sync(); } - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } - - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } - free(filename); free(symlink_fname); -fn_exit: - MPI_Finalize(); - return (nerrs > 0); + return nerrs; } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = false;/* skip ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ + + err = tst_main(argc, argv, "NC_CLOBBER on symlink file", opt, test_io); + + MPI_Finalize(); + + return err; +} diff --git a/test/testcases/tst_varn_var1.c b/test/testcases/tst_varn_var1.c new file mode 100644 index 0000000000..b1fdb3b8d2 --- /dev/null +++ b/test/testcases/tst_varn_var1.c @@ -0,0 +1,241 @@ +/********************************************************************* + * + * Copyright (C) 2025, Northwestern University and Argonne National Laboratory + * See COPYRIGHT notice in top-level directory. + * + *********************************************************************/ + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * This example tests a single call of ncmpi_put_varn_int_all() to write a + * sequence of requests with arbitrary array indices, all with length == 1. + * + * The compile and run commands are given below, together with an ncmpidump of + * the output file. + * + * % mpicc -O2 -o tst_varn_var1 tst_varn_var1.c -lpnetcdf + * % mpiexec -n 4 ./tst_varn_var1 /pvfs2/wkliao/testfile.nc + * % ncmpidump /pvfs2/wkliao/testfile.nc + * netcdf testfile { + * // file format: CDF-5 (big variables) + * dimensions: + * Y = 4 ; + * X = 10 ; + * time = UNLIMITED ; // (4 currently) + * variables: + * int fix_var(Y, X) ; + * int rec_var(time, X) ; + * data: + * + * fix_var = + * 0, _, -1, _, -2, _, -3, _, -4, _, 0, _, -1, _, -2, _, -3, _, -4, _, + * 0, _, -1, _, -2, _, -3, _, -4, _, 0, _, -1, _, -2, _, -3, _, -4, _, + * 0, _, -1, _, -2, _, -3, _, -4, _, 0, _, -1, _, -2, _, -3, _, -4, _, + * 0, _, -1, _, -2, _, -3, _, -4, _, 0, _, -1, _, -2, _, -3, _, -4, _ ; + * + * rec_var = + * 0, _, -1, _, -2, _, -3, _, -4, _, 0, _, -1, _, -2, _, -3, _, -4, _, + * 0, _, -1, _, -2, _, -3, _, -4, _, 0, _, -1, _, -2, _, -3, _, -4, _, + * 0, _, -1, _, -2, _, -3, _, -4, _, 0, _, -1, _, -2, _, -3, _, -4, _, + * 0, _, -1, _, -2, _, -3, _, -4, _, 0, _, -1, _, -2, _, -3, _, -4, _ ; + * } + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include +#include +#include /* strcpy(), memset() */ +#include /* basename() */ +#include +#include + +#include + +#define NY 40 +#define NX 40 +#define NDIMS 2 + +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) +{ + int i, j, k, rank, nprocs, err, nerrs=0; + int ncid, varid[2], dimid[2], nreqs, req, *buf; + MPI_Offset **starts=NULL; + + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + buf = (int*) malloc(sizeof(int) * NY * NX); + + nreqs = NY * NX * nprocs; + starts = (MPI_Offset**) malloc(sizeof(MPI_Offset*) * nreqs); + starts[0] = (MPI_Offset*) calloc(nreqs * NDIMS, sizeof(MPI_Offset)); + for (i=1; i 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for strided put with fill mode on", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } + foreach(`itype', (schar,short,int,float,double), ` + _CAT(`nerrs = test_vars_',itype)'`(out_path, coll_io, info); + if (nerrs > 0) goto err_out;') - /* check whether burst buffering is enabled */ - if (inq_env_hint("nc_burst_buf", &hint_value)) { - if (strcasecmp(hint_value, "enable") == 0) bb_enabled = 1; - free(hint_value); + if (format == NC_FORMAT_CDF5 || format == NC_FORMAT_NETCDF4) { + foreach(`itype', (uchar,ushort,uint,longlong,ulonglong), ` + _CAT(`nerrs = test_vars_',itype)'`(out_path, coll_io, info);' + if (nerrs > 0) goto err_out;) } - ncmpi_set_default_format(NC_FORMAT_CLASSIC, NULL); - foreach(`itype', (schar,short,int,float,double), ` - _CAT(`nerrs += test_vars_',itype)'`(filename);') +err_out: + return nerrs; +} - ncmpi_set_default_format(NC_FORMAT_CDF2, NULL); - foreach(`itype', (schar,short,int,float,double), ` - _CAT(`nerrs += test_vars_',itype)'`(filename);') +int main(int argc, char **argv) { - if (!bb_enabled) { -#ifdef ENABLE_NETCDF4 - ncmpi_set_default_format(NC_FORMAT_NETCDF4_CLASSIC, NULL); - foreach(`itype', (schar,short,int,float,double), ` - _CAT(`nerrs += test_vars_',itype)'`(filename);') + int err; + loop_opts opt; - ncmpi_set_default_format(NC_FORMAT_NETCDF4, NULL); - foreach(`itype', (schar,uchar,short,ushort,int,uint,float,double,longlong,ulonglong), ` - _CAT(`nerrs += test_vars_',itype)'`(filename);') -#endif - } + MPI_Init(&argc, &argv); - ncmpi_set_default_format(NC_FORMAT_CDF5, NULL); - foreach(`itype', (schar,uchar,short,ushort,int,uint,float,double,longlong,ulonglong), ` - _CAT(`nerrs += test_vars_',itype)'`(filename);') - - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + opt.num_fmts = sizeof(nc_formats) / sizeof(int); + opt.formats = nc_formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + err = tst_main(argc, argv, "put vars with fill mode on", opt, test_io); MPI_Finalize(); - return (nerrs > 0); + + return err; } diff --git a/test/testcases/tst_version.c b/test/testcases/tst_version.c index eee7d65fbb..b43ea70389 100644 --- a/test/testcases/tst_version.c +++ b/test/testcases/tst_version.c @@ -2,7 +2,7 @@ * Copyright (C) 2019, Northwestern University and Argonne National Laboratory * See COPYRIGHT notice in top-level directory. * - * Check whether PnetCDF version string returned from ncmpi_inq_libvers() + * Check whether PnetCDF version string returned from ncmpi_inq_libvers() * matches the constant PNETCDF_VERSION defined in header file pnetcdf.h. * */ @@ -15,56 +15,57 @@ #include -/*----< main() >------------------------------------------------------------*/ -int main(int argc, char **argv) +static +int test_io(const char *out_path, /* ignored */ + const char *in_path, /* ignored */ + int format, /* ignored */ + int coll_io, /* ignored */ + MPI_Info info) /* ignored */ { char *str, *pnetcdf_version_str; - int err, nerrs=0, rank; - - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for PnetCDF library version ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } + int nerrs=0; str = (char*) malloc(strlen(ncmpi_inq_libvers())+1); strcpy(str, ncmpi_inq_libvers()); pnetcdf_version_str = strtok(str, " "); if (pnetcdf_version_str == NULL) { - printf("\nError: ncmpi_inq_libvers() returns ill form string %s\n", + fprintf(stderr,"\nError: ncmpi_inq_libvers() returns ill form string %s\n", ncmpi_inq_libvers()); nerrs++; } else if (strcmp(pnetcdf_version_str, PNETCDF_VERSION)) { - printf("\nError: ncmpi_inq_libvers() returns %s does not match with PNETCDF_VERSION %s\n", + fprintf(stderr,"\nError: ncmpi_inq_libvers() returns %s does not match with PNETCDF_VERSION %s\n", pnetcdf_version_str, PNETCDF_VERSION); nerrs++; } free(str); - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + return nerrs; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +int main(int argc, char **argv) { + + int err; + int formats[] = {0}; + + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 0; /* skip test intra-node aggregation */ + opt.drv = 0; /* test GIO driver only */ + opt.ibuf = 0; /* test default hint nc_ibuf_size only */ + opt.bb = 0; /* skip test burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = false;/* skip ncmpidiff for file header */ + opt.var_diff = false;/* skip ncmpidiff for variables */ + + err = tst_main(argc, argv, "PnetCDF library version", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/testcases/varn_contig.c b/test/testcases/varn_contig.c index 164d717af0..271397276b 100644 --- a/test/testcases/varn_contig.c +++ b/test/testcases/varn_contig.c @@ -70,41 +70,31 @@ int check_contents_for_fail(int *buffer) return 0; } -int main(int argc, char** argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) { - char filename[256]; int i, rank, nprocs, err, nerrs=0; - int ncid, cmode, varid[3], dimid[2], num_reqs, *buffer, *r_buffer; + int ncid, varid[3], dimid[2], num_reqs, *buffer, *r_buffer; MPI_Offset w_len, **starts=NULL, **counts=NULL; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for put_varn with contig fileview", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - #ifdef DEBUG if (nprocs != 4 && rank == 0) printf("Warning: %s is intended to run on 4 processes\n",argv[0]); #endif + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + /* create a new file for writing ----------------------------------------*/ - cmode = NC_CLOBBER | NC_64BIT_DATA; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR /* create a global array of size NY * NX */ @@ -125,6 +115,11 @@ int main(int argc, char** argv) err = ncmpi_sync(ncid); CHECK_ERR + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + /* pick arbitrary numbers of requests for 4 processes */ num_reqs = 0; if (rank == 0) num_reqs = 4; @@ -202,21 +197,34 @@ int main(int argc, char** argv) for (i=0; i 4) MPI_Barrier(MPI_COMM_WORLD); /* read back and check contents */ memset(r_buffer, 0, NY*NX*sizeof(int)); - err = ncmpi_get_var_int_all(ncid, varid[0], r_buffer); + if (coll_io) + err = ncmpi_get_var_int_all(ncid, varid[0], r_buffer); + else + err = ncmpi_get_var_int(ncid, varid[0], r_buffer); CHECK_ERR nerrs += check_contents_for_fail(r_buffer); @@ -232,24 +240,32 @@ int main(int argc, char** argv) free(counts); } - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + return nerrs; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } + +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "put_varn with contig fileview", opt, test_io); MPI_Finalize(); - return (nerrs > 0); -} + return err; +} diff --git a/test/testcases/varn_int.c b/test/testcases/varn_int.c index f824690867..09fe9da865 100644 --- a/test/testcases/varn_int.c +++ b/test/testcases/varn_int.c @@ -23,11 +23,11 @@ * X = 10 ; * REC_DIM = UNLIMITED ; // (4 currently) * variables: - * int var(Y, X) ; + * int fix_var(Y, X) ; * int rec_var(REC_DIM, X) ; * data: * - * var = + * fix_var = * 13, 13, 13, 11, 11, 10, 10, 12, 11, 11, * 10, 12, 12, 12, 13, 11, 11, 12, 12, 12, * 11, 11, 12, 13, 13, 13, 10, 10, 11, 11, @@ -55,9 +55,9 @@ #define NDIMS 2 static -int check_contents_for_fail(int *buffer) +int check_contents_for_fail(char *var_name, int *buffer) { - int i, nprocs; + int i, err=0, nprocs; int expected[NY*NX] = {13, 13, 13, 11, 11, 10, 10, 12, 11, 11, 10, 12, 12, 12, 13, 11, 11, 12, 12, 12, 11, 11, 12, 13, 13, 13, 10, 10, 11, 11, @@ -67,14 +67,17 @@ int check_contents_for_fail(int *buffer) /* check if the contents of buf are expected */ for (i=0; i= nprocs) continue; + if (expected[i] >= nprocs+10) continue; if (buffer[i] != expected[i]) { - printf("Expected read buf[%d]=%d, but got %d\n", - i,expected[i],buffer[i]); - return 1; + fprintf(stderr,"Error: expect %s[%d]=%d, but got %d\n", + var_name, i,expected[i],buffer[i]); + err = 1; + break; } } - return 0; + MPI_Allreduce(MPI_IN_PLACE, &err, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); + + return (err > 0); } static @@ -87,41 +90,23 @@ void permute(MPI_Offset a[NDIMS], MPI_Offset b[NDIMS]) } } -int main(int argc, char** argv) +#define INDEP_MODE 0 +#define COLL_MODE 1 + +static +int tst_io(const char *filename, + int coll_io, + MPI_Info info) { - char filename[256]; int i, j, rank, nprocs, err, nerrs=0; - int ncid, cmode, varid[3], dimid[2], num_reqs, *buffer, *r_buffer; + int ncid, varid[2], dimid[2], num_reqs=0, *buffer=NULL, *r_buffer=NULL; MPI_Offset w_len, **starts=NULL, **counts=NULL; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - MPI_Bcast(filename, 256, MPI_CHAR, 0, MPI_COMM_WORLD); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for ncmpi_put_varn_int_all() ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - -#ifdef DEBUG - if (nprocs != 4 && rank == 0) - printf("Warning: %s is intended to run on 4 processes\n",argv[0]); -#endif - /* create a new file for writing ----------------------------------------*/ - cmode = NC_CLOBBER | NC_64BIT_DATA; - err = ncmpi_create(MPI_COMM_WORLD, filename, cmode, MPI_INFO_NULL, &ncid); + err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, info, &ncid); CHECK_ERR /* create a global array of size NY * NX */ @@ -129,7 +114,7 @@ int main(int argc, char** argv) CHECK_ERR err = ncmpi_def_dim(ncid, "X", NX, &dimid[1]); CHECK_ERR - err = ncmpi_def_var(ncid, "var", NC_INT, NDIMS, dimid, &varid[0]); + err = ncmpi_def_var(ncid, "fix_var", NC_INT, NDIMS, dimid, &varid[0]); CHECK_ERR err = ncmpi_def_dim(ncid, "REC_DIM", NC_UNLIMITED, &dimid[0]); CHECK_ERR @@ -148,6 +133,11 @@ int main(int argc, char** argv) } } + if (!coll_io) { + err = ncmpi_begin_indep_data(ncid); + CHECK_ERR + } + /* pick arbitrary numbers of requests for 4 processes */ num_reqs = 0; if (rank == 0) num_reqs = 4; @@ -231,29 +221,49 @@ int main(int argc, char** argv) for (i=0; i 4) MPI_Barrier(MPI_COMM_WORLD); + + if (nprocs > 4 || !coll_io) { + /* This program was designed to run on 4 processes. If running on more + * than 4, then we need to sync writes before reading, especially for + * processes of rank >= 4. Similarly, when running in independent data + * mode, flushing writes is necessary before reading the data back. + */ + MPI_Barrier(MPI_COMM_WORLD); + err = ncmpi_sync(ncid); + CHECK_ERR + MPI_Barrier(MPI_COMM_WORLD); + } /* read back and check contents */ memset(r_buffer, 0, NY*NX*sizeof(int)); - err = ncmpi_get_var_int_all(ncid, varid[0], r_buffer); + if (coll_io) + err = ncmpi_get_var_int_all(ncid, varid[0], r_buffer); + else + err = ncmpi_get_var_int(ncid, varid[0], r_buffer); CHECK_ERR - nerrs += check_contents_for_fail(r_buffer); + nerrs += check_contents_for_fail("fix_var", r_buffer); if (nerrs > 0) goto err_out; /* permute write order */ @@ -263,34 +273,55 @@ int main(int argc, char** argv) } /* write using varn API */ - err = ncmpi_put_varn_int_all(ncid, varid[1], num_reqs, starts, counts, buffer); + if (coll_io) + err = ncmpi_put_varn_int_all(ncid, varid[1], num_reqs, starts, counts, buffer); + else + err = ncmpi_put_varn_int(ncid, varid[1], num_reqs, starts, counts, buffer); CHECK_ERR /* check if user put buffer contents altered */ for (i=0; i 4 || !coll_io) { + /* This program was designed to run on 4 processes. If running on more + * than 4, then we need to sync writes before reading, especially for + * processes of rank >= 4. Similarly, when running in independent data + * mode, flushing writes is necessary before reading the data back. + */ + MPI_Barrier(MPI_COMM_WORLD); + err = ncmpi_sync(ncid); + CHECK_ERR + MPI_Barrier(MPI_COMM_WORLD); + } + /* read back using get_var API and check contents */ memset(r_buffer, 0, NY*NX*sizeof(int)); - err = ncmpi_get_var_int_all(ncid, varid[1], r_buffer); + if (coll_io) + err = ncmpi_get_var_int_all(ncid, varid[1], r_buffer); + else + err = ncmpi_get_var_int(ncid, varid[1], r_buffer); CHECK_ERR - nerrs += check_contents_for_fail(r_buffer); + nerrs += check_contents_for_fail("rec_var", r_buffer); if (nerrs > 0) goto err_out; /* read back using get_varn API and check contents */ for (i=0; i 0) { free(starts[0]); free(counts[0]); @@ -336,24 +370,68 @@ int main(int argc, char** argv) free(counts); } - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + return nerrs; +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, + MPI_Info info) +{ + int err, nerrs=0; + MPI_Info info_dup; - MPI_Finalize(); - return (nerrs > 0); + MPI_Info_dup(info, &info_dup); + + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + +#ifdef DEBUG + if (nprocs != 4 && rank == 0) + printf("Warning: %s is intended to run on 4 processes\n", + basename(__FILE__)); +#endif + + nerrs = tst_io(out_path, coll_io, info_dup); + if (nerrs > 0) goto err_out; + + /* disable PnetCDF internal buffering */ + MPI_Info_set(info, "nc_ibuf_size", "0"); + + nerrs = tst_io(out_path, coll_io, info_dup); + if (nerrs > 0) goto err_out; + +err_out: + MPI_Info_free(&info_dup); + + return nerrs; } +int main(int argc, char **argv) { + + int err; + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 2; /* collective and independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "ncmpi_put_varn_int_all()", opt, test_io); + + MPI_Finalize(); + + return err; +} diff --git a/test/testcases/varn_intf.f b/test/testcases/varn_intf.f index b1dc33f84e..db9ce6d7a1 100644 --- a/test/testcases/varn_intf.f +++ b/test/testcases/varn_intf.f @@ -67,8 +67,8 @@ subroutine check(err, message) ! It is a good idea to check returned value for possible error if (err .NE. NF_NOERR) then write(6,*) message(1:XTRIM(message)), nfmpi_strerror(err) - msg = '*** TESTING F77 varn_intf.f for varn API ' - call pass_fail(1, msg) + msg = '*** TESTING F77 varn_intf.f - varn API ' + call pass_fail(1, msg, 0) STOP 2 end if end ! subroutine check @@ -82,9 +82,10 @@ program main integer*8 NX, NY PARAMETER(NDIMS=2, NX=4, NY=10) - character(LEN=256) filename, cmd, msg + character(LEN=256) out_path, in_path, cmd, msg integer i, j, err, ierr, nprocs, rank, nerrs, get_args integer cmode, ncid, varid, dimid(NDIMS), num_reqs + logical keep_files integer*8 w_len, w_req_len integer*8 starts(NDIMS, 13) @@ -92,22 +93,29 @@ program main integer*8 malloc_size, sum_size integer buffer(13) integer old_fillmode + double precision timing call MPI_Init(ierr) + + timing = MPI_Wtime() + call MPI_Comm_rank(MPI_COMM_WORLD, rank, ierr) call MPI_Comm_size(MPI_COMM_WORLD, nprocs, ierr) - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any if (rank .EQ. 0) then - filename = "testfile.nc" - err = get_args(cmd, filename) + out_path = "testfile.nc" + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, + MPI_COMM_WORLD, ierr) + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, + + ierr) + nerrs = 0 if (.FALSE. .AND. nprocs .NE. 4 .AND. rank .EQ. 0) @@ -116,7 +124,7 @@ program main ! create file, truncate it if exists cmode = IOR(NF_CLOBBER, NF_64BIT_DATA) - err = nfmpi_create(MPI_COMM_WORLD, filename, cmode, + err = nfmpi_create(MPI_COMM_WORLD, out_path, cmode, + MPI_INFO_NULL, ncid) call check(err, 'In nfmpi_create: ') @@ -286,10 +294,18 @@ program main + sum_size, ' bytes yet to be freed' endif + timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, + + MPI_DOUBLE_PRECISION, MPI_MAX, + + MPI_COMM_WORLD, ierr) if (rank .eq. 0) then + if (.NOT. keep_files) then + err = nfmpi_delete(out_path, MPI_INFO_NULL) + end if + msg = '*** TESTING F77 '//cmd(1:XTRIM(cmd))// - + ' for varn API ' - call pass_fail(nerrs, msg) + + ' - varn API ' + call pass_fail(nerrs, msg, timing) endif 999 call MPI_Finalize(ierr) diff --git a/test/testcases/varn_real.f90 b/test/testcases/varn_real.f90 index 7ec3a2eb40..02f2a3ef3a 100644 --- a/test/testcases/varn_real.f90 +++ b/test/testcases/varn_real.f90 @@ -42,8 +42,8 @@ subroutine check(err, message) ! It is a good idea to check returned value for possible error if (err .NE. NF90_NOERR) then write(6,*) message, trim(nf90mpi_strerror(err)) - msg = '*** TESTING F90 varn_real.f90 for varn API ' - call pass_fail(1, msg) + msg = '*** TESTING F90 varn_real.f90 - varn API ' + call pass_fail(1, msg, 0) ! call MPI_Abort(MPI_COMM_WORLD, -1, err) STOP 2 end if @@ -57,7 +57,7 @@ program main integer NDIMS PARAMETER(NDIMS=2) - character(LEN=256) filename, cmd, msg + character(LEN=256) out_path, in_path, cmd, msg integer rank, nprocs, err, ierr, num_reqs, get_args integer ncid, cmode, varid, dimid(2), y, x, i, j, nerrs integer old_fillmode @@ -69,23 +69,30 @@ program main integer(kind=MPI_OFFSET_KIND), allocatable :: starts(:,:) integer(kind=MPI_OFFSET_KIND), allocatable :: counts(:,:) integer(kind=MPI_OFFSET_KIND) malloc_size, sum_size + logical keep_files + double precision timing + + call MPI_Init(ierr) + + timing = MPI_Wtime() NY = 4 NX = 10 - call MPI_Init(ierr) call MPI_Comm_rank(MPI_COMM_WORLD, rank, ierr) call MPI_Comm_size(MPI_COMM_WORLD, nprocs, ierr) - ! take filename from command-line argument if there is any + ! take out_path from command-line argument if there is any if (rank .EQ. 0) then - filename = "testfile.nc" - err = get_args(cmd, filename) + out_path = "testfile.nc" + err = get_args(cmd, out_path, in_path, keep_files) endif call MPI_Bcast(err, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) if (err .EQ. 0) goto 999 - call MPI_Bcast(filename, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + call MPI_Bcast(out_path, 256, MPI_CHARACTER, 0, MPI_COMM_WORLD, ierr) + + call MPI_Bcast(keep_files, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr) nerrs = 0 @@ -95,7 +102,7 @@ program main ! create file, truncate it if exists cmode = IOR(NF90_CLOBBER, NF90_64BIT_DATA) - err = nf90mpi_create(MPI_COMM_WORLD, filename, cmode, & + err = nf90mpi_create(MPI_COMM_WORLD, out_path, cmode, & MPI_INFO_NULL, ncid) call check(err, 'In nf90mpi_create: ') @@ -306,9 +313,17 @@ program main sum_size, ' bytes yet to be freed' endif + timing = MPI_Wtime() - timing + call MPI_Allreduce(MPI_IN_PLACE, timing, 1, & + MPI_DOUBLE_PRECISION, MPI_MAX, & + MPI_COMM_WORLD, ierr) if (rank .eq. 0) then - msg = '*** TESTING F90 '//trim(cmd)//' for varn API ' - call pass_fail(nerrs, msg) + if (.NOT. keep_files) then + err = nfmpi_delete(out_path, MPI_INFO_NULL) + end if + + msg = '*** TESTING F90 '//trim(cmd)//' - varn API ' + call pass_fail(nerrs, msg, timing) endif 999 call MPI_Finalize(ierr) diff --git a/test/testcases/vectors.c b/test/testcases/vectors.c index 0b7ef5e717..b82d3bdb47 100644 --- a/test/testcases/vectors.c +++ b/test/testcases/vectors.c @@ -21,7 +21,12 @@ #define BLOCKLEN 3 #define STRIDE 5 -int main(int argc, char ** argv) +static +int test_io(const char *out_path, + const char *in_path, /* ignored */ + int format, + int coll_io, /* ignored */ + MPI_Info info) { int ncid, dimid, varid, rank, nprocs; MPI_Datatype vtype, rtype, usertype; @@ -31,39 +36,29 @@ int main(int argc, char ** argv) int count = 25; double pi = 3.14159; MPI_Offset start, acount; - char filename[256]; - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - if (argc > 2) { - if (!rank) printf("Usage: %s [filename]\n",argv[0]); - MPI_Finalize(); - return 1; - } - if (argc == 2) snprintf(filename, 256, "%s", argv[1]); - else strcpy(filename, "testfile.nc"); - - if (rank == 0) { - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for put_vara/get_vara ", basename(argv[0])); - printf("%-66s ------ ", cmd_str); fflush(stdout); - free(cmd_str); - } - #ifdef DEBUG if (nprocs > 2 && rank == 0) printf("Warning: %s is designed to run on 1 process\n",argv[0]); #endif - err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, MPI_INFO_NULL, &ncid); + /* Set format. */ + err = ncmpi_set_default_format(format, NULL); + CHECK_ERR + + err = ncmpi_create(MPI_COMM_WORLD, out_path, NC_CLOBBER, info, &ncid); CHECK_ERR err = ncmpi_def_dim(ncid, "50k", 1024*50, &dimid); CHECK_ERR err = ncmpi_def_var(ncid, "vector", NC_DOUBLE, 1, &dimid, &varid); CHECK_ERR + err = ncmpi_def_var_fill(ncid, varid, 0, NULL); + CHECK_ERR + err = ncmpi_enddef(ncid); CHECK_ERR @@ -90,10 +85,14 @@ int main(int argc, char ** argv) CHECK_ERR } + /* file sync before reading */ + err = ncmpi_sync(ncid); + CHECK_ERR + err = ncmpi_close(ncid); CHECK_ERR - err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, MPI_INFO_NULL, &ncid); + err = ncmpi_open(MPI_COMM_WORLD, out_path, NC_NOWRITE, info, &ncid); CHECK_ERR err = ncmpi_begin_indep_data(ncid); CHECK_ERR @@ -120,23 +119,33 @@ int main(int argc, char ** argv) free(userbuf); free(cmpbuf); - /* check if PnetCDF freed all internal malloc */ - MPI_Offset malloc_size, sum_size; - err = ncmpi_inq_malloc_size(&malloc_size); - if (err == NC_NOERR) { - MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD); - if (rank == 0 && sum_size > 0) - printf("heap memory allocated by PnetCDF internally has "OFFFMT" bytes yet to be freed\n", - sum_size); - if (malloc_size > 0) ncmpi_inq_malloc_list(); - } + return (nerrs > 0); +} - MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - if (rank == 0) { - if (nerrs) printf(FAIL_STR,nerrs); - else printf(PASS_STR); - } +int main(int argc, char **argv) { + + int err; + + /* flexible APIs are not supported in NetCDF4 */ + int formats[] = {NC_FORMAT_CLASSIC, NC_FORMAT_64BIT_OFFSET, NC_FORMAT_64BIT_DATA}; + + loop_opts opt; + + MPI_Init(&argc, &argv); + + opt.num_fmts = sizeof(formats) / sizeof(int); + opt.formats = formats; + opt.ina = 2; /* enable and disable intra-node aggregation */ + opt.drv = 2; /* test GIO and MPI-IO driver */ + opt.ibuf = 2; /* enable and disable hint nc_ibuf_size */ + opt.bb = 2; /* enable and disable burst-buffering feature */ + opt.mod = 0; /* skip test independent data mode */ + opt.hdr_diff = true; /* run ncmpidiff for file header */ + opt.var_diff = true; /* run ncmpidiff for variables */ + + err = tst_main(argc, argv, "put_vara/get_vara", opt, test_io); MPI_Finalize(); - return (nerrs > 0); + + return err; } diff --git a/test/testcases/wrap_runs.sh b/test/testcases/wrap_runs.sh deleted file mode 100755 index 280216b4df..0000000000 --- a/test/testcases/wrap_runs.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2003, Northwestern University and Argonne National Laboratory -# See COPYRIGHT notice in top-level directory. -# - -# Exit immediately if a command exits with a non-zero status. -set -e - -VALIDATOR=../../src/utils/ncvalidator/ncvalidator -NCMPIDIFF=../../src/utils/ncmpidiff/ncmpidiff - -outfile=`basename $1` - -# remove file system type prefix if there is any -OUTDIR=`echo "$TESTOUTDIR" | cut -d: -f2-` - -# echo "PNETCDF_DEBUG = ${PNETCDF_DEBUG}" -if test ${PNETCDF_DEBUG} = 1 ; then - safe_modes="0 1" -else - safe_modes="0" -fi - -# prevent user environment setting of PNETCDF_HINTS to interfere -unset PNETCDF_HINTS - -for j in ${safe_modes} ; do - export PNETCDF_SAFE_MODE=$j - # echo "---- set PNETCDF_SAFE_MODE ${PNETCDF_SAFE_MODE}" - ${TESTSEQRUN} $1 ${TESTOUTDIR}/$outfile.nc - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$outfile.nc - # echo "" - - if test "x${ENABLE_BURST_BUFFER}" = x1 ; then - echo "" - echo "---- testing burst buffering" - - export PNETCDF_HINTS="nc_burst_buf=enable;nc_burst_buf_dirname=${TESTOUTDIR};nc_burst_buf_overwrite=enable" - ${TESTSEQRUN} $1 ${TESTOUTDIR}/$outfile.bb.nc - unset PNETCDF_HINTS - ${TESTSEQRUN} ${VALIDATOR} -q ${TESTOUTDIR}/$outfile.bb.nc - - # running ncmpidiff on last_large_var on one process requires more than - # 2 GB memory. Use more processes to check. Disable the check for now. - if test "$1" != "./last_large_var" ; then - # echo "--- ncmpidiff $outfile.nc $outfile.bb.nc ---" - ${TESTSEQRUN} ${NCMPIDIFF} -q ${TESTOUTDIR}/$outfile.nc ${TESTOUTDIR}/$outfile.bb.nc - fi - fi -done - -rm -f ${OUTDIR}/$outfile.nc* -rm -f ${OUTDIR}/$outfile.bb.nc* -