TrioCFD 1.9.9_beta
TrioCFD documentation
Loading...
Searching...
No Matches
Device_test.cpp
1/****************************************************************************
2* Copyright (c) 2025, CEA
3* All rights reserved.
4*
5* Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
6* 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
7* 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
8* 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
9*
10* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
11* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
12* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
13*
14*****************************************************************************/
15
16#include <TRUSTTrav.h>
17#include <TRUSTTab_parts.h>
18#include <Device.h>
19#include <string>
20#include <sstream>
21#include <comm_incl.h>
22#include <DeviceMemory.h>
23
24// Device_test: intensively tests the methods of the Device.h interface:
25bool self_tested_ = false;
26void self_test()
27{
28 if (self_tested_)
29 return;
30 else
31 self_tested_ = true;
32 {
33 // local_operations_vect_bis_generic
34 // Critical unit test case:
35 {
36 DoubleTab a(2);
37 a(0) = 1;
38 a(1) = 2;
39 mapToDevice(a);
40 double mp = mp_norme_vect(a);
41 double sol = sqrt(Process::nproc() * 5);
42 if (mp != sol)
43 {
44 Cerr << "What! " << mp << " != " << sol << finl;
45 Cerr
46 << "Weird bug seen during a sum reduction with Kokkos::parallel_reduce for TRUST production build on Cuda 11.6. Fixed with Cuda 12.1 (or before...)."
47 << finl;
48 Cerr << "Update Cuda please !" << finl;
49 Process::exit();
50 }
51 }
52 }
53#ifndef NDEBUG
54 // Test mapToDevice(arr)
55 // Status
56 // Status before Status after Copy ?
57 // DataLocation::HostOnly DataLocation::HostDevice Yes
58 // Host DataLocation::HostDevice Yes
59 // DataLocation::HostDevice DataLocation::HostDevice No
60 // Device Device No
61 {
62 DoubleTab a(10);
63 assert(a.get_data_location() == DataLocation::HostOnly);
64 mapToDevice(a);
65 assert(a.get_data_location() == DataLocation::HostDevice);
66 a[1]=0;
67 assert(a.get_data_location() == DataLocation::Host);
68 mapToDevice(a);
69 assert(a.get_data_location() == DataLocation::HostDevice);
70 }
71
72 {
73 DoubleTab a(10);
74 a = 23;
75 assert(a.get_data_location() == DataLocation::HostOnly);
76 {
77 DoubleTab b;
78 b.ref(a);
79 mapToDevice(b);
80 assert(b.get_data_location() == DataLocation::HostDevice);
81 assert(a.get_data_location() == DataLocation::HostDevice);
82 }
83 }
84
85 int N = 10;
86 // Test access methods on the device:
87 DoubleTab inco(N);
88 inco = 1;
89 mapToDevice(inco); // copy
90 assert(inco.get_data_location() == DataLocation::HostDevice);
91 assert(inco.ref_count() == 1);
92 {
93 // Example: 1st operator
94 DoubleTab a;
95 a.ref(inco); // Must take the state of inco
96 assert(a.get_data_location() == DataLocation::HostDevice);
97 assert(a.ref_count() == 2);
98 assert(inco.ref_count() == 2);
99 DoubleTab b(N);
100
101 CDoubleArrView a_v = static_cast<const ArrOfDouble&>(a).view_ro();
102 DoubleArrView b_v = static_cast<ArrOfDouble&>(b).view_wo();
103 Kokkos::parallel_for(N, KOKKOS_LAMBDA(const int i)
104 {
105 b_v[i] = a_v[i];
106 });
107 Kokkos::fence();
108
109 const DoubleTab& const_b = b;
110 const DoubleTab& const_a = a;
111 assert(const_b[5] == const_a[5]);
112 assert(const_b[5] == 1);
113 //assert(b[5] == a[5]); // Argh double& TRUSTArray<double>::operator[](int i) called for a and thus falls back to host
114 // How to detect whether operator[](int i) is used for read or write? Possible? No, unless const is used.
115 assert(a.get_data_location() == DataLocation::HostDevice);
116 assert(b.get_data_location() == DataLocation::HostDevice);
117 assert(inco.get_data_location() == DataLocation::HostDevice);
118 }
119 assert(inco.get_data_location() == DataLocation::HostDevice);
120 assert(inco.ref_count() == 1);
121 {
122 // Example: 2nd operator
123 DoubleTab a;
124 a.ref(inco); // Must take the state of inco which is still DataLocation::HostDevice
125 assert(a.get_data_location() == DataLocation::HostDevice);
126 assert(a.ref_count() == 2);
127 assert(inco.ref_count() == 2);
128
129 DoubleTab b(N);
130 CDoubleArrView a_v = static_cast<const ArrOfDouble&>(a).view_ro();
131 DoubleArrView b_v = static_cast<ArrOfDouble&>(b).view_wo();
132 Kokkos::parallel_for(N, KOKKOS_LAMBDA(const int i)
133 {
134 b_v[i] = a_v[i];
135 });
136 Kokkos::fence();
137
138 const DoubleTab& const_b = b;
139 const DoubleTab& const_a = a;
140 assert(const_b[5] == const_a[5]);
141 assert(const_b[5] == 1);
142 assert(a.get_data_location() == DataLocation::HostDevice);
143 assert(b.get_data_location() == DataLocation::HostDevice);
144 }
145 assert(inco.ref_count() == 1);
146
147 // Update of the unknown on the device:
148 inco += 1;
149 assert(inco.get_data_location() == DataLocation::Device);
150 assert(inco.ref_count() == 1);
151 {
152 // Next time step, new operator:
153 DoubleTab a;
154 a.ref(inco); // Must take the state of inco
155 assert(a.get_data_location() == DataLocation::Device);
156 assert(a.ref_count() == 2);
157 assert(inco.ref_count() == 2);
158
159 DoubleTab b(N);
160 CDoubleArrView a_v = static_cast<const ArrOfDouble&>(a).view_ro();
161 DoubleArrView b_v = static_cast<ArrOfDouble&>(b).view_wo();
162 Kokkos::parallel_for(N, KOKKOS_LAMBDA(const int i)
163 {
164 b_v[i] = a_v[i];
165 });
166 Kokkos::fence();
167 const DoubleTab& const_b = b;
168 const DoubleTab& const_a = a;
169 assert(const_b[5] == const_a[5]);
170 assert(const_b[5] == 2);
171 assert(a.get_data_location() == DataLocation::HostDevice);
172 assert(b.get_data_location() == DataLocation::HostDevice);
173 assert(inco.get_data_location() == DataLocation::HostDevice); // Because a refs inco
174 }
175 assert(inco.get_data_location() == DataLocation::HostDevice);
176 // Update of the unknown on the device
177 {
178 inco += 1;
179 assert(inco.get_data_location() == DataLocation::Device);
180
181 DoubleTab a;
182 a.ref(inco); // Must take the state of inco
183 assert(a.get_data_location() == DataLocation::Device);
184 assert(a.ref_count() == 2);
185 assert(inco.ref_count() == 2);
186
187 DoubleTab b(N);
188 CDoubleArrView a_v = static_cast<const ArrOfDouble&>(a).view_ro();
189 DoubleArrView b_v = static_cast<ArrOfDouble&>(b).view_wo();
190 Kokkos::parallel_for(N, KOKKOS_LAMBDA(const int i)
191 {
192 b_v[i] = a_v[i];
193 });
194 Kokkos::fence();
195 const DoubleTab& const_b = b;
196 const DoubleTab& const_a = a;
197 assert(const_b[5] == const_a[5]);
198 assert(const_b[5] == 3);
199 assert(a.get_data_location() == DataLocation::HostDevice);
200 assert(b.get_data_location() == DataLocation::HostDevice);
201 assert(inco.get_data_location() == DataLocation::HostDevice);
202 }
203 assert(inco.get_data_location() == DataLocation::HostDevice);
204
205 // Test ArrOfDouble operations on GPU
206 {
207 ArrOfDouble a(10), b(10);
208 a=1;
209 b=2;
210 mapToDevice(a);
211 mapToDevice(b);
212 b+=a; // TRUSTArray& operator+=(const TRUSTArray& y) on the device
213 b+=3; // TRUSTArray& operator+=(const _TYPE_ dy)
214 b-=2; // TRUSTArray& operator-=(const _TYPE_ dy)
215 b-=a; // TRUSTArray& operator-=(const TRUSTArray& y)
216 // ToDo fix: Multiple definition of 'nvkernel__ZN10TRUST
217 //b*=10; // TRUSTArray& operator*= (const _TYPE_ dy)
218 //b/=2; // TRUSTArray& operator/= (const _TYPE_ dy)
219 assert(a.get_data_location() == DataLocation::HostDevice);
220 assert(b.get_data_location() == DataLocation::Device);
221 const ArrOfDouble& const_b = b;
222 // Operations on the device:
223 // Return to host to verify the result
224 copyFromDevice(b);
225 assert(const_b[0] == 3);
226 }
227
228 // Constructors on device:
229 {
230 DoubleTab a(10);
231 allocateOnDevice(a);
232 assert(a.get_data_location() == DataLocation::Device);
233 a = 1; // On the device
234 assert(a.get_data_location() == DataLocation::Device);
235 DoubleTab b(a); // b must also be allocated on the device and the copy done on the device
236 assert(b.get_data_location() == DataLocation::Device);
237 copyFromDevice(b);
238 const ArrOfDouble& const_b = b;
239 assert(const_b[0] == 1);
240 assert(const_b[b.size() - 1] == 1);
241 }
242 {
243 // Array copies:
244 DoubleTab a(10);
245 a=1;
246 mapToDevice(a); // a on the device
247 DoubleTab b;
248 b = a; // b must also be allocated/filled on the device by copying a:
249 assert(b.get_data_location() == DataLocation::Device);
250 const ArrOfDouble& const_b = b;
251 copyFromDevice(b);
252 assert(const_b[0] == 1);
253 assert(const_b[b.size() - 1] == 1);
254 }
255 // operator_vect_vect_generic for DoubleTab::operator+-*/
256 {
257 DoubleTab a(10), b(10);
258 const ArrOfDouble& const_a = a;
259 const ArrOfDouble& const_b = b;
260 a=1;
261 b=10;
262 mapToDevice(a);
263 assert(a.get_data_location() == DataLocation::HostDevice);
264 mapToDevice(b);
265 assert(b.get_data_location() == DataLocation::HostDevice);
266 a=b; // TRUSTArray<_TYPE_>::inject_array(v) done on the device (a=10)
267 assert(a.get_data_location() == DataLocation::Device);
268 a+=b; // operator_vect_vect_generic(ADD_) done on the device (a=20)
269 assert(a.get_data_location() == DataLocation::Device);
270 b-=a; // operator_vect_vect_generic(SUB_) done on the device (b=-10)
271 assert(b.get_data_location() == DataLocation::Device);
272 // Return to host to verify results
273 copyFromDevice(a);
274 copyFromDevice(b);
275 assert(a.get_data_location() == DataLocation::HostDevice);
276 assert(b.get_data_location() == DataLocation::HostDevice);
277 assert(const_a[0] == 20);
278 assert(const_a[a.size()-1] == 20);
279 assert(const_b[0] == -10);
280 assert(const_b[b.size()-1] == -10);
281 }
282 // DoubleTrav
283 {
284 DoubleTrav a(10*N);
285 a = 1;
286 mapToDevice(a); // copy
287 assert(a.get_data_location() == DataLocation::HostDevice);
288 assert(a.ref_count() == 1);
289 DeviceMemory::printMemoryMap();
290 }
291 // Second DoubleTrav
292 {
293 DeviceMemory::printMemoryMap();
294 DoubleTrav a(10*N); // a is initialized to 0 on the device because the previous DoubleTrav was HostDevice!
295 assert(a.get_data_location() == DataLocation::Device);
296 const ArrOfDouble& const_a = a;
297 assert(const_a[0] == 0);
298 }
299 // Copy constructor DoubleTab
300 {
301 DoubleTab a(N);
302 a=-1;
303 mapToDevice(a); // On the device
304 DoubleTrav b(a); // b must be on the device (=0)
305 assert(b.get_data_location()==DataLocation::Device);
306 b+=1; // Operation must be done on the device (=1)
307 assert(b.get_data_location()==DataLocation::Device);
308 copyFromDevice(b);
309 const ArrOfDouble& const_b = b;
310 assert(const_b[0] == 1);
311 assert(const_b[b.size()-1] == 1);
312 }
313 // max/min methods
314 {
315 DoubleTab a(3);
316 a(0)=1;
317 a(1)=3;
318 a(2)=-10;
319 mapToDevice(a);
320 // Change on the host for testing:
321 a.data()[0]=0;
322 a.data()[1]=0;
323 a.data()[2]=0;
324 a.set_data_location(DataLocation::Device);
325 assert(local_max_vect(a)==3);
326 assert(local_min_vect(a)==-10);
327 assert(local_max_abs_vect(a)==10);
328 assert(local_min_abs_vect(a)==1);
329 //assert(local_imax_vect(a)==1);
330 //assert(local_imin_vect(a)==2);
331 assert(a.isDataOnDevice());
332 // Test on host the two methods imin,imax not ported to GPU:
333 copyFromDevice(a);
334 assert(a.get_data_location()==DataLocation::HostDevice);
335 assert(local_imax_vect(a)==1);
336 assert(local_imin_vect(a)==2);
337 }
338 // ref_array
339 {
340 DoubleTab a(N);
341 a=1;
342 mapToDevice(a); // On the device
343 assert(a.get_data_location()==DataLocation::HostDevice);
344 DoubleTab b;
345 b.ref_array(a);
346 assert(b.get_data_location()==DataLocation::HostDevice); // b must be on the device
347 }
348 // ref_tab/ref_array on a chunk array
349 {
350 DoubleTab a(2*N);
351 a=1;
352 DoubleTab b;
353 b.ref_tab(a, 0, N); // partial reference on a
354 mapToDevice(b); // On the device
355 assert(b.get_data_location()==DataLocation::HostDevice); // b must be on the device
356 assert(a.get_data_location()==DataLocation::HostDevice); // a must be on the device
357
358 DoubleArrView a_v = static_cast<ArrOfDouble&>(a).view_wo();
359 Kokkos::parallel_for(2*N, KOKKOS_LAMBDA(const int i)
360 {
361 a_v[i] = 2;
362 });
363 Kokkos::fence();
364 // Return to the device and verify that a was completely on the device:
365 assert(a.get_data_location()==DataLocation::Device);
366 assert(a(0)==2);
367 assert(a.get_data_location()==DataLocation::Host);
368 assert(a(2*N-1)==2);
369 }
370 double * ptr_host;
371 {
372 DoubleTab a(N);
373 a=1;
374 {
375 DoubleTab b;
376 b.ref_array(a);
377 mapToDevice(b); // On the device
378 assert(a.data()==b.data());
379 assert(b.get_data_location() == DataLocation::HostDevice);
380 assert(a.get_data_location() == DataLocation::HostDevice); // a is also considered on the device
381 }
382 assert(a.get_data_location() == DataLocation::HostDevice);
383 ptr_host = a.data();
384 assert(isAllocatedOnDevice(ptr_host)); // Check that the array has a memory area on the device
385 }
386 assert(!isAllocatedOnDevice(ptr_host)); // Check that the array no longer has a memory area on the device
387 // Test of resize_array_ in TRUSTArray<_TYPE_, _SIZE_>::resize_array_
388 {
389 DoubleTab a(1);
390 mapToDevice(a);
391 a=1;
392 a.resize(2);
393 a+=1;
394 assert(a.isDataOnDevice());
395 assert(a(0)==2); // Check that after resize data on device is keep
396 assert(a(1)==1); // Check that resize initialize new elements to 0
397 }
398 // ToDo: How to handle DoubleTab_Parts? Not easy, so for now
399 // the copy constructor does a copyFromDevice on the DoubleTab...
400 /*
401 {
402 DoubleTab pression;
403 mapToDevice(pression);
404 DoubleTab_parts P0P1(pression); // P0P1 Device
405 DoubleTab& P0 = P0P1[0];
406 DoubleTab& P1 = P0P1[1];
407 double moyenne_K = mp_moyenne_vect(P0); // P0 DataLocation::HostDevice but not P1!
408 P1 -= moyenne_K; // Big problem as P1 is always on Device!
409 }
410 */
411 //if (Process::me()==0) std::cerr << ptr_host << std::endl;
412
413 // DoubleTrav copy Constructor:
414 {
415 DoubleTrav b(100);
416 b = 123;
417 mapToDevice(b);
418 DoubleVect b2(b);
419 assert(b2.get_mem_storage() == STORAGE::TEMP_STORAGE);
420 assert(b2.get_data_location() == DataLocation::Device);
421
422 DoubleVect b3(b2);
423 assert(b3.get_mem_storage() == STORAGE::TEMP_STORAGE);
424 assert(b3.get_data_location() == DataLocation::Device);
425 }
426
427 {
428 // Provisional: reproduce the deadlock?
429 //DoubleTab a(10);
430 //allocateOnDevice(a);
431 //allocateOnDevice(a);
432 }
433#endif
434}
435
436
Process::nproc
static int nproc()
Returns the number of processors in the current group. See Comm_Group::nproc() and PE_Groups::current...
Definition Process.cpp:102
Process::exit
static void exit(int exit_code=-1)
Exit routine for TRUST within a Kokkos region.
Definition Process.cpp:466
TRUSTArray::get_data_location
DataLocation get_data_location()
Definition TRUSTArray.h:245
TRUSTArray::ref_count
int ref_count() const
Definition TRUSTArray.tpp:196
TRUSTArray::data
_TYPE_ * data()
Definition TRUSTArray.tpp:173
TRUSTArray::set_data_location
void set_data_location(DataLocation flag)
Definition TRUSTArray.h:247
TRUSTArray::isDataOnDevice
bool isDataOnDevice() const
Definition TRUSTArray_device.tpp:47
TRUSTTab::ref
virtual void ref(const TRUSTTab &)
Definition TRUSTTab.tpp:308
TRUSTTab::ref_array
void ref_array(TRUSTArray< _TYPE_, _SIZE_ > &, _SIZE_ start=0, _SIZE_ sz=-1) override
Definition TRUSTTab.tpp:332
TRUSTTab::ref_tab
virtual void ref_tab(TRUSTTab &, _SIZE_ start_line=0, _SIZE_ nb_lines=-1)
Definition TRUSTTab.tpp:345
TRUSTTab::resize
void resize(_SIZE_ n, RESIZE_OPTIONS opt=RESIZE_OPTIONS::COPY_INIT)
Definition TRUSTTab.tpp:469
TRUSTVect::size
_SIZE_ size() const
Definition TRUSTVect.tpp:45
DeviceMemory::printMemoryMap
static void printMemoryMap()
Definition DeviceMemory.cpp:128