Skip to content

Record_Component.make_empty() #538

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ax3l merged 2 commits into from
Oct 28, 2020
Merged

Record_Component.make_empty() #538

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4666008
Add Python bindings for RecordComponent::makeEmpty()
franzpoeschel Jul 25, 2019
51d7e57
Fix whitespaces
franzpoeschel Oct 19, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions include/openPMD/RecordComponent.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,16 @@ class RecordComponent : public BaseRecordComponent
template< typename T >
RecordComponent& makeEmpty( uint8_t dimensions );

/**
* @brief Non-template overload of RecordComponent::makeEmpty().
* Uses the passed openPMD datatype to determine the template parameter.
*
* @param dt The datatype of which to create an empty dataset.
* @param dimensions The dimensionality of the dataset.
* @return RecordComponent&
*/
RecordComponent& makeEmpty( Datatype dt, uint8_t dimensions );

/** Returns true if this is an empty record component
*
* An empty record component has a defined dimensionality but zero extent
Expand Down
26 changes: 26 additions & 0 deletions src/RecordComponent.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,32 @@ RecordComponent::getExtent() const
return m_dataset->extent;
}

namespace detail
{
struct MakeEmpty
{
template< typename T >
RecordComponent& operator()( RecordComponent & rc, uint8_t dimensions )
{
return rc.makeEmpty< T >( dimensions );
}

template< unsigned int N >
RecordComponent& operator()( RecordComponent &, uint8_t )
{
throw std::runtime_error(
"RecordComponent::makeEmpty: Unknown datatype." );
}
};
}

RecordComponent&
RecordComponent::makeEmpty( Datatype dt, uint8_t dimensions )
{
static detail::MakeEmpty me;
return switchType< RecordComponent & >( dt, me, *this, dimensions );
}

RecordComponent&
RecordComponent::makeEmpty( Dataset d )
{
Expand Down
14 changes: 14 additions & 0 deletions src/binding/python/RecordComponent.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -546,6 +546,20 @@ void init_RecordComponent(py::module &m) {
py::arg("value"))
.def("make_constant", &RecordComponent::makeConstant<bool>,
py::arg("value"))
.def("make_empty",
[]( RecordComponent & rc, Datatype dt, uint8_t dimensionality )
{
return rc.makeEmpty( dt, dimensionality );
},
py::arg("datatype"), py::arg("dimensionality"))
.def("make_empty",
[](
RecordComponent & rc,
pybind11::dtype const dt,
uint8_t dimensionality )
{
return rc.makeEmpty( dtype_from_numpy( dt ), dimensionality );
})

// TODO if we also want to support scalar arrays, we have to switch
// py::array for py::buffer as in Attributable
Expand Down
18 changes: 18 additions & 0 deletions test/SerialIOTest.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,13 +286,19 @@ empty_dataset_test( std::string file_ending )
series.iterations[ 1 ].meshes[ "rho" ][ "makeEmpty_dim_7_int" ];
auto makeEmpty_dim_7_long =
series.iterations[ 1 ].meshes[ "rho" ][ "makeEmpty_dim_7_bool" ];
auto makeEmpty_dim_7_int_alt =
series.iterations[ 1 ].meshes[ "rho" ][ "makeEmpty_dim_7_int_alt" ];
auto makeEmpty_dim_7_long_alt =
series.iterations[ 1 ].meshes[ "rho" ][ "makeEmpty_dim_7_bool_alt" ];
auto makeEmpty_resetDataset_dim3 =
series.iterations[ 1 ].meshes[ "rho" ][ "makeEmpty_resetDataset_dim3" ];
auto makeEmpty_resetDataset_dim3_notallzero =
series.iterations[ 1 ]
.meshes[ "rho" ][ "makeEmpty_resetDataset_dim3_notallzero" ];
makeEmpty_dim_7_int.makeEmpty< int >( 7 );
makeEmpty_dim_7_long.makeEmpty< long >( 7 );
makeEmpty_dim_7_int_alt.makeEmpty( Datatype::INT, 7 );
makeEmpty_dim_7_long_alt.makeEmpty( Datatype::LONG, 7 );
makeEmpty_resetDataset_dim3.resetDataset(
Dataset( Datatype::LONG, Extent( 3, 0 ) ) );
makeEmpty_resetDataset_dim3_notallzero.resetDataset(
Expand All @@ -315,6 +321,10 @@ empty_dataset_test( std::string file_ending )
series.iterations[ 1 ].meshes[ "rho" ][ "makeEmpty_dim_7_int" ];
auto makeEmpty_dim_7_long =
series.iterations[ 1 ].meshes[ "rho" ][ "makeEmpty_dim_7_bool" ];
auto makeEmpty_dim_7_int_alt =
series.iterations[ 1 ].meshes[ "rho" ][ "makeEmpty_dim_7_int_alt" ];
auto makeEmpty_dim_7_long_alt =
series.iterations[ 1 ].meshes[ "rho" ][ "makeEmpty_dim_7_bool_alt" ];
auto makeEmpty_resetDataset_dim3 =
series.iterations[ 1 ].meshes[ "rho" ][ "makeEmpty_resetDataset_dim3" ];
auto makeEmpty_resetDataset_dim3_notallzero =
Expand All @@ -328,6 +338,14 @@ empty_dataset_test( std::string file_ending )
REQUIRE(makeEmpty_dim_7_long.getExtent() == Extent(7, 0));
REQUIRE(isSame(makeEmpty_dim_7_long.getDatatype(), determineDatatype< long >()));

REQUIRE(makeEmpty_dim_7_int_alt.getDimensionality() == 7);
REQUIRE(makeEmpty_dim_7_int_alt.getExtent() == Extent(7, 0));
REQUIRE(isSame(makeEmpty_dim_7_int_alt.getDatatype(), determineDatatype< int >()));

REQUIRE(makeEmpty_dim_7_long_alt.getDimensionality() == 7);
REQUIRE(makeEmpty_dim_7_long_alt.getExtent() == Extent(7, 0));
REQUIRE(isSame(makeEmpty_dim_7_long_alt.getDatatype(), determineDatatype< long >()));

REQUIRE(makeEmpty_resetDataset_dim3.getDimensionality() == 3);
REQUIRE(makeEmpty_resetDataset_dim3.getExtent() == Extent(3, 0));
REQUIRE(isSame(makeEmpty_resetDataset_dim3.getDatatype(), Datatype::LONG));
Expand Down
156 changes: 156 additions & 0 deletions test/python/unittest/API/APITest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -550,6 +550,162 @@ def testConstantRecords(self):
for ext in io.file_extensions:
self.makeConstantRoundTrip(ext)

def makeEmptyRoundTrip(self, file_ending):
# write
series = io.Series(
"unittest_py_empty_API." + file_ending,
io.Access_Type.create
)

ms = series.iterations[0].meshes
SCALAR = io.Mesh_Record_Component.SCALAR
DT = io.Datatype

ms["CHAR"][SCALAR].make_empty(DT.CHAR, 1)
Comment thread
ax3l marked this conversation as resolved.
ms["UCHAR"][SCALAR].make_empty(DT.UCHAR, 2)
ms["SHORT"][SCALAR].make_empty(DT.SHORT, 3)
ms["INT"][SCALAR].make_empty(DT.INT, 4)
ms["LONG"][SCALAR].make_empty(DT.LONG, 5)
ms["LONGLONG"][SCALAR].make_empty(DT.LONGLONG, 6)
ms["USHORT"][SCALAR].make_empty(DT.USHORT, 7)
ms["UINT"][SCALAR].make_empty(DT.UINT, 8)
ms["ULONG"][SCALAR].make_empty(DT.ULONG, 9)
ms["ULONGLONG"][SCALAR].make_empty(DT.ULONGLONG, 10)
ms["FLOAT"][SCALAR].make_empty(DT.FLOAT, 11)
ms["DOUBLE"][SCALAR].make_empty(DT.DOUBLE, 12)
ms["LONG_DOUBLE"][SCALAR].make_empty(DT.LONG_DOUBLE, 13)

if found_numpy:
ms["int16"][SCALAR].make_empty(np.dtype("int16"), 14)
ms["int32"][SCALAR].make_empty(np.dtype("int32"), 15)
ms["int64"][SCALAR].make_empty(np.dtype("int64"), 16)
ms["uint16"][SCALAR].make_empty(np.dtype("uint16"), 17)
ms["uint32"][SCALAR].make_empty(np.dtype("uint32"), 18)
ms["uint64"][SCALAR].make_empty(np.dtype("uint64"), 19)
ms["single"][SCALAR].make_empty(np.dtype("single"), 20)
ms["np_double"][SCALAR].make_empty(np.dtype("double"), 21)

# flush and close file
del series

# read back
series = io.Series(
"unittest_py_empty_API." + file_ending,
io.Access_Type.read_only
)

ms = series.iterations[0].meshes

self.assertEqual(
ms["CHAR"][SCALAR].shape,
[0 for _ in range(1)]
)
self.assertEqual(
ms["UCHAR"][SCALAR].shape,
[0 for _ in range(2)]
)
self.assertEqual(
ms["SHORT"][SCALAR].shape,
[0 for _ in range(3)]
)
self.assertEqual(
ms["INT"][SCALAR].shape,
[0 for _ in range(4)]
)
self.assertEqual(
ms["LONG"][SCALAR].shape,
[0 for _ in range(5)]
)
self.assertEqual(
ms["LONGLONG"][SCALAR].shape,
[0 for _ in range(6)]
)
self.assertEqual(
ms["USHORT"][SCALAR].shape,
[0 for _ in range(7)]
)
self.assertEqual(
ms["UINT"][SCALAR].shape,
[0 for _ in range(8)]
)
self.assertEqual(
ms["ULONG"][SCALAR].shape,
[0 for _ in range(9)]
)
self.assertEqual(
ms["ULONGLONG"][SCALAR].shape,
[0 for _ in range(10)]
)
self.assertEqual(
ms["FLOAT"][SCALAR].shape,
[0 for _ in range(11)]
)
self.assertEqual(
ms["DOUBLE"][SCALAR].shape,
[0 for _ in range(12)]
)
self.assertEqual(
ms["LONG_DOUBLE"][SCALAR].shape,
[0 for _ in range(13)]
)

if found_numpy:
self.assertEqual(
ms["int16"][SCALAR].shape,
[0 for _ in range(14)]
)
self.assertEqual(
ms["int32"][SCALAR].shape,
[0 for _ in range(15)]
)
self.assertEqual(
ms["int64"][SCALAR].shape,
[0 for _ in range(16)]
)
self.assertEqual(
ms["uint16"][SCALAR].shape,
[0 for _ in range(17)]
)
self.assertEqual(
ms["uint32"][SCALAR].shape,
[0 for _ in range(18)]
)
self.assertEqual(
ms["uint64"][SCALAR].shape,
[0 for _ in range(19)]
)
self.assertEqual(
ms["single"][SCALAR].shape,
[0 for _ in range(20)]
)
self.assertEqual(
ms["np_double"][SCALAR].shape,
[0 for _ in range(21)]
)

# test datatypes for fixed-sized types only
if found_numpy:
self.assertTrue(ms["int16"][SCALAR].dtype == np.dtype("int16"))
self.assertTrue(ms["int32"][SCALAR].dtype == np.dtype("int32"))
self.assertTrue(ms["int64"][SCALAR].dtype == np.dtype("int64"))
self.assertTrue(ms["uint16"][SCALAR].dtype == np.dtype("uint16"))
self.assertTrue(ms["uint32"][SCALAR].dtype == np.dtype("uint32"))
self.assertTrue(ms["uint64"][SCALAR].dtype == np.dtype("uint64"))
self.assertTrue(ms["single"][SCALAR].dtype == np.dtype("single"))
self.assertTrue(
ms["np_double"][SCALAR].dtype == np.dtype("double"))

def testEmptyRecords(self):
backend_filesupport = {
'json': 'json',
'hdf5': 'h5',
'adios1': 'bp',
'adios2': 'bp'
}
for b in io.variants:
if io.variants[b] is True and b in backend_filesupport:
self.makeEmptyRoundTrip(backend_filesupport[b])

def testData(self):
""" Test IO on data containing particles and meshes."""

Expand Down