Coverage for /dolfinx-env/lib/python3.14/site-packages/io4dolfinx/original_checkpoint.py: 100%
187 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-21 11:50 +0000
1# Copyright (C) 2024 Jørgen Schartum Dokken
2#
3# This file is part of io4dolfinx
4#
5# SPDX-License-Identifier: MIT
7from __future__ import annotations
9import logging
10import typing
11from pathlib import Path
13from mpi4py import MPI
15import dolfinx
16import fenicsx_compat
17import numpy as np
19from .backends import FileMode, get_backend
20from .comm_helpers import all_to_all, numpy_to_mpi
21from .structures import FunctionData, MeshData
22from .utils import (
23 compute_insert_position,
24 compute_local_range,
25 index_owner,
26 unroll_dofmap,
27 unroll_insert_position,
28)
30__all__ = ["write_function_on_input_mesh", "write_mesh_input_order"]
31logger = logging.getLogger(__name__)
34def create_original_mesh_data(mesh: dolfinx.mesh.Mesh) -> MeshData:
35 """
36 Store data locally on output process
37 """
39 # 1. Send cell indices owned by current process to the process which owned its input
41 # Get the input cell index for cells owned by this process
42 num_owned_cells = mesh.topology.index_map(mesh.topology.dim).size_local
43 original_cell_index = mesh.topology.original_cell_index[:num_owned_cells]
45 # Compute owner of cells on this process based on the original cell index
46 num_cells_global = mesh.topology.index_map(mesh.topology.dim).size_global
47 output_cell_owner = index_owner(mesh.comm, original_cell_index, num_cells_global)
48 local_cell_range = compute_local_range(mesh.comm, num_cells_global)
50 # Compute outgoing edges from current process to outputting process
51 # Computes the number of cells sent to each process at the same time
52 cell_destinations, _send_cells_per_proc = np.unique(output_cell_owner, return_counts=True)
53 send_cells_per_proc = _send_cells_per_proc.astype(np.int32)
54 del _send_cells_per_proc
55 assert isinstance(mesh.comm, MPI.Intracomm)
56 cell_to_output_comm = mesh.comm.Create_dist_graph(
57 [mesh.comm.rank],
58 [len(cell_destinations)],
59 cell_destinations.tolist(),
60 reorder=False,
61 )
62 cell_sources, cell_dests, _ = cell_to_output_comm.Get_dist_neighbors()
63 assert np.allclose(cell_dests, cell_destinations)
65 # Compute number of recieving cells
66 recv_cells_per_proc = np.zeros_like(cell_sources, dtype=np.int32)
67 send_cells_per_proc = send_cells_per_proc.astype(np.int32)
68 all_to_all(cell_to_output_comm, send_cells_per_proc, recv_cells_per_proc)
70 assert recv_cells_per_proc.sum() == local_cell_range[1] - local_cell_range[0]
71 # Pack and send cell indices (used for mapping topology dofmap later)
72 cell_insert_position = compute_insert_position(
73 output_cell_owner, cell_destinations, send_cells_per_proc
74 )
75 send_cells = np.empty_like(cell_insert_position, dtype=np.int64)
76 send_cells[cell_insert_position] = original_cell_index
77 recv_cells = np.empty(recv_cells_per_proc.sum(), dtype=np.int64)
78 send_cells_msg = [send_cells, send_cells_per_proc, MPI.INT64_T]
79 recv_cells_msg = [recv_cells, recv_cells_per_proc, MPI.INT64_T]
80 cell_to_output_comm.Neighbor_alltoallv(send_cells_msg, recv_cells_msg)
81 del send_cells_msg, recv_cells_msg, send_cells
83 # Map received cells to the local index
84 local_cell_index = recv_cells - local_cell_range[0]
86 # 2. Create dofmap based on original geometry indices and re-order in the same order as original
87 # cell indices on output process
89 # Get original node index for all nodes (including ghosts) and convert dofmap to these indices
90 original_node_index = mesh.geometry.input_global_indices
91 _, num_nodes_per_cell = fenicsx_compat.dofmap(mesh).shape
92 local_geometry_dofmap = fenicsx_compat.dofmap(mesh)[:num_owned_cells, :]
93 global_geometry_dofmap = original_node_index[local_geometry_dofmap.reshape(-1)]
95 # Unroll insert position for geometry dofmap
96 dofmap_insert_position = unroll_insert_position(cell_insert_position, num_nodes_per_cell)
98 # Create and commmnicate connecitivity in original geometry indices
99 send_geometry_dofmap = np.empty_like(dofmap_insert_position, dtype=np.int64)
100 send_geometry_dofmap[dofmap_insert_position] = global_geometry_dofmap
101 del global_geometry_dofmap
102 send_sizes_dofmap = send_cells_per_proc * num_nodes_per_cell
103 recv_sizes_dofmap = recv_cells_per_proc * num_nodes_per_cell
104 recv_geometry_dofmap = np.empty(recv_sizes_dofmap.sum(), dtype=np.int64)
105 send_geometry_dofmap_msg = [send_geometry_dofmap, send_sizes_dofmap, MPI.INT64_T]
106 recv_geometry_dofmap_msg = [recv_geometry_dofmap, recv_sizes_dofmap, MPI.INT64_T]
107 cell_to_output_comm.Neighbor_alltoallv(send_geometry_dofmap_msg, recv_geometry_dofmap_msg)
108 del send_geometry_dofmap_msg, recv_geometry_dofmap_msg
110 # Reshape dofmap and sort by original cell index
111 recv_dofmap = recv_geometry_dofmap.reshape(-1, num_nodes_per_cell)
112 sorted_recv_dofmap = np.empty_like(recv_dofmap)
113 sorted_recv_dofmap[local_cell_index] = recv_dofmap
115 # 3. Move geometry coordinates to input process
116 # Compute outgoing edges from current process and create neighbourhood communicator
117 # Also create number of outgoing cells at the same time
118 num_owned_nodes = mesh.geometry.index_map().size_local
119 num_nodes_global = mesh.geometry.index_map().size_global
120 output_node_owner = index_owner(
121 mesh.comm, original_node_index[:num_owned_nodes], num_nodes_global
122 )
124 node_destinations, _send_nodes_per_proc = np.unique(output_node_owner, return_counts=True)
125 send_nodes_per_proc = _send_nodes_per_proc.astype(np.int32)
126 del _send_nodes_per_proc
128 geometry_to_owner_comm = mesh.comm.Create_dist_graph(
129 [mesh.comm.rank],
130 [len(node_destinations)],
131 node_destinations.tolist(),
132 reorder=False,
133 )
135 node_sources, node_dests, _ = geometry_to_owner_comm.Get_dist_neighbors()
136 assert np.allclose(node_dests, node_destinations)
138 # Compute send node insert positions
139 send_nodes_position = compute_insert_position(
140 output_node_owner, node_destinations, send_nodes_per_proc
141 )
142 unrolled_nodes_positiion = unroll_insert_position(send_nodes_position, 3)
144 send_coordinates = np.empty_like(unrolled_nodes_positiion, dtype=mesh.geometry.x.dtype)
145 send_coordinates[unrolled_nodes_positiion] = mesh.geometry.x[:num_owned_nodes, :].reshape(-1)
147 # Send and recieve geometry sizes
148 send_coordinate_sizes = (send_nodes_per_proc * 3).astype(np.int32)
149 recv_coordinate_sizes = np.zeros_like(node_sources, dtype=np.int32)
150 all_to_all(geometry_to_owner_comm, send_coordinate_sizes, recv_coordinate_sizes)
152 # Send node coordinates
153 recv_coordinates = np.empty(recv_coordinate_sizes.sum(), dtype=mesh.geometry.x.dtype)
154 mpi_type = numpy_to_mpi[recv_coordinates.dtype.type]
155 send_coord_msg = [send_coordinates, send_coordinate_sizes, mpi_type]
156 recv_coord_msg = [recv_coordinates, recv_coordinate_sizes, mpi_type]
157 geometry_to_owner_comm.Neighbor_alltoallv(send_coord_msg, recv_coord_msg)
158 del send_coord_msg, recv_coord_msg
160 # Send node ordering for reordering the coordinates on output process
161 send_nodes = np.empty(num_owned_nodes, dtype=np.int64)
162 send_nodes[send_nodes_position] = original_node_index[:num_owned_nodes]
164 recv_indices = np.empty(recv_coordinate_sizes.sum() // 3, dtype=np.int64)
165 send_nodes_msg = [send_nodes, send_nodes_per_proc, MPI.INT64_T]
166 recv_nodes_msg = [recv_indices, recv_coordinate_sizes // 3, MPI.INT64_T]
167 geometry_to_owner_comm.Neighbor_alltoallv(send_nodes_msg, recv_nodes_msg)
169 # Compute local ording of received nodes
170 local_node_range = compute_local_range(mesh.comm, num_nodes_global)
171 recv_indices -= local_node_range[0]
173 # Sort geometry based on input index and strip to gdim
174 gdim = mesh.geometry.dim
175 recv_nodes = recv_coordinates.reshape(-1, 3)
176 _geometry = np.empty(recv_nodes.shape, dtype=mesh.geometry.x.dtype)
177 _geometry[recv_indices, :] = recv_nodes
178 geometry = _geometry[:, :gdim].copy()
179 del _geometry, recv_nodes
181 assert local_node_range[1] - local_node_range[0] == geometry.shape[0]
182 cmap = fenicsx_compat.cmap(mesh)
184 cell_to_output_comm.Free()
185 geometry_to_owner_comm.Free()
187 # NOTE: Could in theory store partitioning information, but would not work nicely
188 # as one would need to read this data rather than the xdmffile.
189 # NOTE: Local geometry type hint skip is only required on DOLFINX<0.10 where
190 # proper `dolfinx.mesh.Geometry` wrapper doesn't exist
191 return MeshData(
192 local_geometry=geometry, # type: ignore[arg-type]
193 local_geometry_pos=local_node_range,
194 num_nodes_global=num_nodes_global,
195 local_topology=sorted_recv_dofmap,
196 local_topology_pos=local_cell_range,
197 num_cells_global=num_cells_global,
198 cell_type=mesh.topology.cell_name(),
199 degree=cmap.degree,
200 lagrange_variant=cmap.variant,
201 store_partition=False,
202 partition_processes=None,
203 ownership_array=None,
204 ownership_offset=None,
205 partition_range=None,
206 partition_global=None,
207 )
210def create_function_data_on_original_mesh(
211 u: dolfinx.fem.Function, name: typing.Optional[str] = None
212) -> FunctionData:
213 """
214 Create data object to save with ADIOS2
215 """
216 mesh = u.function_space.mesh
218 # Compute what cells owned by current process should be sent to what output process
219 # FIXME: Cache this
220 num_owned_cells = mesh.topology.index_map(mesh.topology.dim).size_local
221 original_cell_index = mesh.topology.original_cell_index[:num_owned_cells]
223 # Compute owner of cells on this process based on the original cell index
224 num_cells_global = mesh.topology.index_map(mesh.topology.dim).size_global
225 output_cell_owner = index_owner(mesh.comm, original_cell_index, num_cells_global)
226 local_cell_range = compute_local_range(mesh.comm, num_cells_global)
228 # Compute outgoing edges from current process to outputting process
229 # Computes the number of cells sent to each process at the same time
230 cell_destinations, _send_cells_per_proc = np.unique(output_cell_owner, return_counts=True)
231 send_cells_per_proc = _send_cells_per_proc.astype(np.int32)
232 del _send_cells_per_proc
233 assert isinstance(mesh.comm, MPI.Intracomm)
234 cell_to_output_comm = mesh.comm.Create_dist_graph(
235 [mesh.comm.rank],
236 [len(cell_destinations)],
237 cell_destinations.tolist(),
238 reorder=False,
239 )
240 cell_sources, cell_dests, _ = cell_to_output_comm.Get_dist_neighbors()
241 assert np.allclose(cell_dests, cell_destinations)
243 # Compute number of recieving cells
244 recv_cells_per_proc = np.zeros_like(cell_sources, dtype=np.int32)
245 send_cells_per_proc = send_cells_per_proc.astype(np.int32)
246 all_to_all(cell_to_output_comm, send_cells_per_proc, recv_cells_per_proc)
247 assert recv_cells_per_proc.sum() == local_cell_range[1] - local_cell_range[0]
249 # Pack and send cell indices (used for mapping topology dofmap later)
250 cell_insert_position = compute_insert_position(
251 output_cell_owner, cell_destinations, send_cells_per_proc
252 )
253 send_cells = np.empty_like(cell_insert_position, dtype=np.int64)
254 send_cells[cell_insert_position] = original_cell_index
255 recv_cells = np.empty(recv_cells_per_proc.sum(), dtype=np.int64)
256 send_cells_msg = [send_cells, send_cells_per_proc, MPI.INT64_T]
257 recv_cells_msg = [recv_cells, recv_cells_per_proc, MPI.INT64_T]
258 cell_to_output_comm.Neighbor_alltoallv(send_cells_msg, recv_cells_msg)
259 del send_cells_msg, recv_cells_msg
261 # Map received cells to the local index
262 local_cell_index = recv_cells - local_cell_range[0]
264 # Pack and send cell permutation info
265 cell_permutation_info = fenicsx_compat.cell_permutation_info(mesh.topology)[:num_owned_cells]
266 send_perm = np.empty_like(send_cells, dtype=np.uint32)
267 send_perm[cell_insert_position] = cell_permutation_info
268 recv_perm = np.empty_like(recv_cells, dtype=np.uint32)
269 send_perm_msg = [send_perm, send_cells_per_proc, MPI.UINT32_T]
270 recv_perm_msg = [recv_perm, recv_cells_per_proc, MPI.UINT32_T]
271 cell_to_output_comm.Neighbor_alltoallv(send_perm_msg, recv_perm_msg)
272 cell_permutation_info = np.empty_like(recv_perm)
273 cell_permutation_info[local_cell_index] = recv_perm
275 # 2. Extract function data (array is the same, keeping global indices from DOLFINx)
276 # Dofmap is moved by the original cell index similar to the mesh geometry dofmap
277 dofmap = u.function_space.dofmap
278 dmap = dofmap.list
279 num_dofs_per_cell = dmap.shape[1]
280 dofmap_bs = dofmap.bs
281 index_map_bs = dofmap.index_map_bs
283 # Unroll dofmap for block size
284 unrolled_dofmap = unroll_dofmap(dofmap.list[:num_owned_cells, :], dofmap_bs)
285 dmap_loc = (unrolled_dofmap // index_map_bs).reshape(-1)
286 dmap_rem = (unrolled_dofmap % index_map_bs).reshape(-1)
288 # Convert imap index to global index
289 imap_global = dofmap.index_map.local_to_global(dmap_loc)
290 dofmap_global = (imap_global * index_map_bs + dmap_rem).reshape(unrolled_dofmap.shape)
291 assert len(dofmap_global.shape) >= 2
292 num_dofs_per_cell = dofmap_global.shape[1]
293 dofmap_insert_position = unroll_insert_position(cell_insert_position, num_dofs_per_cell)
295 # Create and send array for global dofmap
296 send_function_dofmap = np.empty(len(dofmap_insert_position), dtype=np.int64)
297 send_function_dofmap[dofmap_insert_position] = dofmap_global.reshape(-1)
298 send_sizes_dofmap = send_cells_per_proc * num_dofs_per_cell
299 recv_size_dofmap = recv_cells_per_proc * num_dofs_per_cell
300 recv_function_dofmap = np.empty(recv_size_dofmap.sum(), dtype=np.int64)
301 cell_to_output_comm.Neighbor_alltoallv(
302 [send_function_dofmap, send_sizes_dofmap, MPI.INT64_T],
303 [recv_function_dofmap, recv_size_dofmap, MPI.INT64_T],
304 )
306 shaped_dofmap = recv_function_dofmap.reshape(
307 local_cell_range[1] - local_cell_range[0], num_dofs_per_cell
308 ).copy()
309 _final_dofmap = np.empty_like(shaped_dofmap)
310 _final_dofmap[local_cell_index] = shaped_dofmap
311 final_dofmap = _final_dofmap.reshape(-1)
313 # Get offsets of dofmap
314 num_cells_local = local_cell_range[1] - local_cell_range[0]
315 num_dofs_local_dmap = num_cells_local * num_dofs_per_cell
316 dofmap_imap = fenicsx_compat.create_index_map(
317 mesh.comm,
318 num_dofs_local_dmap,
319 np.empty(0, dtype=np.int64),
320 np.empty(0, dtype=np.int32),
321 )
322 local_dofmap_offsets = np.arange(num_cells_local + 1, dtype=np.int64)
323 local_dofmap_offsets[:] *= num_dofs_per_cell
324 local_dofmap_offsets[:] += dofmap_imap.local_range[0]
326 num_dofs_local = dofmap.index_map.size_local * dofmap.index_map_bs
327 num_dofs_global = dofmap.index_map.size_global * dofmap.index_map_bs
328 local_range = np.asarray(dofmap.index_map.local_range, dtype=np.int64) * dofmap.index_map_bs
329 func_name = name if name is not None else u.name
330 cell_to_output_comm.Free()
331 return FunctionData(
332 cell_permutations=cell_permutation_info,
333 local_cell_range=local_cell_range,
334 num_cells_global=num_cells_global,
335 dofmap_array=final_dofmap,
336 dofmap_offsets=local_dofmap_offsets,
337 values=u.x.array[:num_dofs_local].copy(),
338 dof_range=local_range,
339 num_dofs_global=num_dofs_global,
340 dofmap_range=dofmap_imap.local_range,
341 global_dofs_in_dofmap=dofmap_imap.size_global,
342 name=func_name,
343 )
346def write_function_on_input_mesh(
347 filename: Path | str,
348 u: dolfinx.fem.Function,
349 time: float = 0.0,
350 name: typing.Optional[str] = None,
351 mode: FileMode = FileMode.append,
352 backend_args: dict[str, typing.Any] | None = None,
353 backend: str | None = None,
354):
355 """
356 Write function checkpoint (to be read with the input mesh).
358 Note:
359 Requires backend to implement :meth:`io4dolfinx.backends.IOBackend.write_function`.
361 Args:
362 filename: The filename to write to
363 u: The function to checkpoint
364 time: Time-stamp associated with function at current write step
365 mode: The mode to use (write or append)
366 name: Name of function. If None, the name of the function is used.
367 backend_args: Arguments to backend
368 backend: Choice of backend module
369 """
370 logger.debug(
371 f"Writing function on input mesh to {filename} at time {time} with name {name or u.name}"
372 )
373 logger.debug(f"Using backend {backend} with arguments {backend_args} and mode {mode}")
374 mesh = u.function_space.mesh
375 function_data = create_function_data_on_original_mesh(u, name)
376 fname = Path(filename)
378 backend_cls = get_backend(backend)
379 backend_args = backend_cls.get_default_backend_args(backend_args)
380 backend_cls.write_function(
381 fname,
382 mesh.comm,
383 function_data,
384 time=time,
385 mode=mode,
386 backend_args=backend_args,
387 )
390def write_mesh_input_order(
391 filename: Path | str,
392 mesh: dolfinx.mesh.Mesh,
393 time: float = 0.0,
394 mode: FileMode = FileMode.write,
395 backend: str | None = None,
396 backend_args: dict[str, typing.Any] | None = None,
397):
398 """
399 Write mesh to checkpoint file in original input ordering.
401 Note:
402 Requires backend to implement :meth:`io4dolfinx.backends.IOBackend.write_mesh`.
404 Args:
405 filename: The filename to write to
406 mesh: Mesh to checkpoint
407 time: Time-stamp associated with function at current write step
408 mode: The mode to use (write or append)
409 name: Name of function. If None, the name of the function is used.
410 backend_args: Arguments to backend
411 backend: Choice of backend module
412 """
413 logger.debug(f"Writing mesh in input order to {filename} at time {time}")
414 logger.debug(f"Using backend {backend} with arguments {backend_args} and mode {mode}")
415 mesh_data = create_original_mesh_data(mesh)
416 fname = Path(filename)
418 backend_cls = get_backend(backend)
419 backend_args = backend_cls.get_default_backend_args(backend_args)
420 backend_cls.write_mesh(
421 fname,
422 mesh.comm,
423 mesh_data,
424 backend_args=backend_args,
425 mode=mode,
426 time=time,
427 )