source: sans/Dev/trunk/NCNR_User_Procedures/Reduction/VSANS/VC_HDF5_VSANS_Utils.ipf @ 961

Last change on this file since 961 was 961, checked in by srkline, 7 years ago

removed the function "distance" from RectAnnulAvg? to avoid potential conflicts with the same-named field in Nexus files.The function only occured locally in that one file.

File size: 15.8 KB
Line 
1#pragma rtGlobals=1             // Use modern global access method.
2
3//
4// This file has lots of utility procedures to be able to read/write
5// test HDF5 files for SANS and VSANS in Nexus format
6//
7// It doesn't have the NICE logs, but has everything that I
8// can think of here.
9//
10
11//
12// There's an odd sequence of steps that need to be done to use HDFGateway
13// to most easily write out attributes, and be able to read then back in.
14// Rather awkward, but it works without me needing to go nuts with an atomistic
15// understanding of HDF5. This is necessary for me to be able to write out
16// Nexus files, which is important both for testing, and for simulation.
17//
18// This is done as a set of macros that need to be applied in a specific sequence
19// in order to be able to generate HDF5___xref
20// Ideally, the HDF5___xref wave can be saved/imported to make proper saving possible
21// once the "tree" is finalized.
22//
23//
24// The basic saving routines are here, just as Proc rather than Macro menu clutter
25//
26
27
28// Next... think of all of R/W access needed
29//
30// the simple read/write works...
31// linear_data does not seem to need to be transposed at all
32//
33//  -- this seems too easy. what am I doing wrong? Is something getting garbled when I
34// write back any single values back to the file
35//
36// -- try a string value next
37// -- then start to write the generic get/write functions
38//
39
40//
41// do I ditch the RealsRead/IntegersRead/TextRead? It makes little sense now.
42// maybe copy a "dataInfo" folder/subfolders. can't keep them all (bloat)
43// but then what about multiple files added together?
44//
45
46
47Macro Load_Nexus_V_Template()
48        H_HDF5Gate_Read_Raw("")
49        String tmpStr=root:file_name  //SRK - so I can get the file name that was loaded
50       
51// this is the folder name
52        String base_name = StringFromList(0,File_Name,".")
53        RenameDataFolder $("root:"+base_name), V_Nexus_Template
54End
55
56
57Macro Fill_Nexus_V_Template()
58        H_Fill_VSANS_Template_wSim()
59End
60
61
62Macro Save_Nexus_V_Template()
63        H_HDF5Gate_Write_Raw("root:V_Nexus_Template:","")
64End
65
66
67
68
69
70
71
72
73
74
75
76
77
78/////////////////////////////
79
80/////////////below is largely depricated, ugly dance to be able to "fake" a file from Igor
81// which was not complete anyways.
82
83
84
85Macro IgorOnly_Setup_VSANS_Struct()
86
87        // lays out the tree and fills with dummy values
88        H_Setup_VSANS_Structure()
89       
90        // writes in the attributes
91        H_Fill_VSANS_Attributes()
92       
93        // fill in with VCALC simulation bits
94        H_Fill_VSANS_wSim()
95       
96End
97
98Macro IgorOnly_Save_VSANS_Nexus(fileName)
99        String fileName="Test_VSANS_file"
100
101        // save as HDF5 (no attributes saved yet)
102        Save_VSANS_file("root:VSANS_file", fileName+".h5")
103       
104        // read in a data file using the gateway-- reads from the home path
105        H_HDF5Gate_Read_Raw(fileName+".h5")
106       
107        // after reading in a "partial" file using the gateway (to generate the xref)
108        // Save the xref to disk (for later use)
109        Save_HDF5___xref("root:"+fileName,"HDF5___xref")
110       
111        // after you've generated the HDF5___xref, load it in and copy it
112        // to the necessary folder location.
113        Copy_HDF5___xref("root:VSANS_file", "HDF5___xref")
114       
115        // writes out the contents of a data folder using the gateway
116        H_HDF5Gate_Write_Raw("root:VSANS_file", fileName+".h5")
117
118        // re-load the data file using the gateway-- reads from the home path
119        // now with attributes
120        H_HDF5Gate_Read_Raw(fileName+".h5")
121       
122End
123
124
125
126Macro IgorOnly_Setup_SANS_Struct()
127
128        // lays out the tree and fills with dummy values
129        H_Setup_SANS_Structure()
130       
131        // writes in the attributes
132        H_Fill_SANS_Attributes()
133       
134        // fill in with VCALC simulation bits
135        H_Fill_SANS_wSim()
136
137End
138
139Macro IgorOnly_Save_SANS_Nexus(fileName)
140        String fileName="Test_SANS_file"
141       
142        // save as HDF5 (no attributes saved yet) (save_VSANS is actually generic HDF...)
143        Save_VSANS_file("root:SANS_file", fileName+".h5")
144       
145        // read in a data file using the gateway-- reads from the home path
146        H_HDF5Gate_Read_Raw(fileName+".h5")
147       
148        // after reading in a "partial" file using the gateway (to generate the xref)
149        // Save the xref to disk (for later use)
150        Save_HDF5___xref("root:"+fileName,"HDF5___xref")
151       
152        // after you've generated the HDF5___xref, load it in and copy it
153        // to the necessary folder location.
154        Copy_HDF5___xref("root:SANS_file", "HDF5___xref")
155       
156        // writes out the contents of a data folder using the gateway
157        H_HDF5Gate_Write_Raw("root:SANS_file", fileName+".h5")
158
159        // re-load the data file using the gateway-- reads from the home path
160        // now with attributes
161        H_HDF5Gate_Read_Raw(fileName+".h5")
162
163
164End
165
166
167// TODO
168// currently, there are no dummy fill values or attributes for the fake DIV file
169//
170Macro Setup_VSANS_DIV_Struct()
171
172        // lays out the tree and fills with dummy values
173        H_Setup_VSANS_DIV_Structure()
174       
175        // writes in the attributes
176//      H_Fill_VSANS_Attributes()
177       
178        // fill in with VCALC simulation bits
179//      H_Fill_VSANS_wSim()
180       
181End
182
183Macro Save_VSANS_DIV_Nexus(fileName)
184        String fileName="Test_VSANS_DIV_file"
185
186        // save as HDF5 (no attributes saved yet)
187        Save_VSANS_file("root:VSANS_DIV_file", fileName+".h5")
188       
189        // read in a data file using the gateway-- reads from the home path
190        H_HDF5Gate_Read_Raw(fileName+".h5")
191       
192        // after reading in a "partial" file using the gateway (to generate the xref)
193        // Save the xref to disk (for later use)
194        Save_HDF5___xref("root:"+fileName,"HDF5___xref")
195       
196        // after you've generated the HDF5___xref, load it in and copy it
197        // to the necessary folder location.
198        Copy_HDF5___xref("root:VSANS_DIV_file", "HDF5___xref")
199       
200        // writes out the contents of a data folder using the gateway
201        H_HDF5Gate_Write_Raw("root:VSANS_DIV_file", fileName+".h5")
202
203        // re-load the data file using the gateway-- reads from the home path
204        // now with attributes
205        H_HDF5Gate_Read_Raw(fileName+".h5")
206       
207End
208
209
210
211
212
213
214
215
216
217
218
219
220
221//
222// saves a specified folder, with a given filename.
223// saves to the home path
224//
225Proc Save_VSANS_file(dfPath, filename)
226        String dfPath   ="root:VSANS_file"              // e.g., "root:FolderA" or ":"
227        String filename = "Test_VSANS_file.h5"
228       
229        H_NXSANS_SaveGroupAsHDF5(dfPath, filename)
230End
231
232
233//     
234// this is my procedure to save the folders to HDF5, once I've filled the folder tree
235//
236// this does NOT save attributes, but gets the folder structure correct
237//
238Function H_NXSANS_SaveGroupAsHDF5(dfPath, filename)
239        String dfPath   // e.g., "root:FolderA" or ":"
240        String filename
241
242        Variable result = 0     // 0 means no error
243       
244        Variable fileID
245        HDF5CreateFile/P=home /O /Z fileID as filename
246        if (V_flag != 0)
247                Print "HDF5CreateFile failed"
248                return -1
249        endif
250
251        HDF5SaveGroup /IGOR=0 /O /R /Z $dfPath, fileID, "."
252//      HDF5SaveGroup /O /R /Z $dfPath, fileID, "."
253        if (V_flag != 0)
254                Print "HDF5SaveGroup failed"
255                result = -1
256        endif
257       
258        HDF5CloseFile fileID
259
260        return result
261End
262
263// passing null file string presents a dialog
264// these two procedures will use the full Xrefs, so they can write out full
265// files. They are, however, slow
266Proc Read_Nexus_Xref()
267        H_HDF5Gate_Read_Raw("")
268End
269
270Proc Write_Nexus_Xref()
271        H_HDF5Gate_Write_Raw()
272End
273
274//
275// writes out the contents of a data folder using the gateway
276// -- the HDF5___xref wave must be present at the top level for it to
277//    write out anything.
278//
279Proc H_HDF5Gate_Write_Raw(dfPath,filename)
280        String dfPath   = "root:"       // e.g., "root:FolderA" or ":"
281        String filename = "Test_VSANS_file.h5"
282//      Prompt dfPath, "Data Folder",popup,H_GetAList(4)       
283//      Prompt fileName, "File name"
284       
285//      DoPrompt "Folder and file to write", dfPath,fileName
286        // Check our work so far.
287        // If something prints, there was an error above.
288        print H5GW_ValidateFolder(dfPath)
289
290        print H5GW_WriteHDF5(dfPath, filename)
291       
292        SetDataFolder root:
293End
294
295//
296// read in a data file using the gateway
297//
298// reads from the home path
299//
300Proc H_HDF5Gate_Read_Raw(file)
301        String file
302       
303//      NewDataFolder/O/S root:newdata
304        Print H5GW_ReadHDF5("", file)   // reads into current folder
305        SetDataFolder root:
306End
307
308
309
310// this is in PlotUtils, but duplicated here
311// utility used in the "PlotSmeared...() macros to get a list of data folders
312//
313//1:    Waves.
314//2:    Numeric variables.
315//3:    String variables.
316//4:    Data folders.
317Function/S H_GetAList(type)
318        Variable type
319       
320        SetDataFolder root:
321       
322        String objName,str=""
323        Variable index = 0
324        do
325                objName = GetIndexedObjName(":", type, index)
326                if (strlen(objName) == 0)
327                        break
328                endif
329                //Print objName
330                str += objName + ";"
331                index += 1
332        while(1)
333       
334        // remove myGlobals, Packages, etc. from the folder list
335        if(type==4)
336                str = RemoveFromList("myGlobals", str , ";" )
337                str = RemoveFromList("Packages", str, ";")
338        endif
339       
340        return(str)
341End
342
343//
344// after reading in a "partial" file using the gateway (to generate the xref)
345// Save the xref to disk (for later use)
346//
347Proc Save_HDF5___xref(dfPath, filename)
348        String dfPath   ="root:VSANS_file"              // e.g., "root:FolderA" or ":"
349        String filename = "HDF5___xref"
350
351        Save/T/P=home $(dfPath+":HDF5___xref") as "HDF5___xref.itx"
352
353//      Copy_HDF5___xref(dfPath, filename)
354       
355        SetDataFolder root:
356End
357
358//
359// after you've generated the HDF5___xref, load it in and copy it to
360// the necessary folder location.
361// - then all is set to *really* write out the file correctly, including the attributes
362//
363Proc Copy_HDF5___xref(dfPath, filename)
364        String dfPath   ="root:VSANS_file"              // e.g., "root:FolderA" or ":"
365        String filename = "HDF5___xref"
366
367        if(exists(filename) != 1)
368                //load it in   
369                LoadWave/T/P=home/O "HDF5___xref.itx"
370        endif
371       
372        Duplicate/O HDF5___xref, $(dfPath+":HDF5___xref")
373       
374        SetDataFolder root:
375End
376
377
378
379//////// testing procedures, may be of use, maybe not //////////////
380
381
382//////// Two procedures that test out Pete Jemain's HDF5Gateway
383//
384// This works fine, but it may not be terribly compatible with the way NICE will eventually
385// write out the data files. I'll have very little control over that and I'll need to cobble together
386// a bunch of fixes to cover up their mistakes.
387//
388// Using Nick Hauser's code as a starting point may be a lot more painful, but more flexible in the end.
389//
390// I'm completely baffled about what to do with attributes. Are they needed, is this the best way to deal
391// with them, do I care about reading them in, and if I do, why?
392//
393Proc H_HDF5Gate_WriteTest()
394
395        // create the folder structure
396        NewDataFolder/O/S root:mydata
397        NewDataFolder/O sasentry
398        NewDataFolder/O :sasentry:sasdata
399
400        // create the waves
401        Make/O :sasentry:sasdata:I0
402        Make/O :sasentry:sasdata:Q0
403
404        Make/O/N=0 Igor___folder_attributes
405        Make/O/N=0 :sasentry:Igor___folder_attributes
406        Make/O/N=0 :sasentry:sasdata:Igor___folder_attributes
407
408        // create the attributes
409        Note/K Igor___folder_attributes, "producer=IgorPro\rNX_class=NXroot"
410        Note/K :sasentry:Igor___folder_attributes, "NX_class=NXentry"
411        Note/K :sasentry:sasdata:Igor___folder_attributes, "NX_class=NXdata"
412        Note/K :sasentry:sasdata:I0, "units=1/cm\rsignal=1\rtitle=reduced intensity"
413        Note/K :sasentry:sasdata:Q0, "units=1/A\rtitle=|scattering vector|"
414
415        // create the cross-reference mapping
416        Make/O/T/N=(5,2) HDF5___xref
417        Edit/K=0 'HDF5___xref';DelayUpdate
418        HDF5___xref[0][1] = ":"
419        HDF5___xref[1][1] = ":sasentry"
420        HDF5___xref[2][1] = ":sasentry:sasdata"
421        HDF5___xref[3][1] = ":sasentry:sasdata:I0"
422        HDF5___xref[4][1] = ":sasentry:sasdata:Q0"
423        HDF5___xref[0][0] = "/"
424        HDF5___xref[1][0] = "/sasentry"
425        HDF5___xref[2][0] = "/sasentry/sasdata"
426        HDF5___xref[3][0] = "/sasentry/sasdata/I"
427        HDF5___xref[4][0] = "/sasentry/sasdata/Q"
428
429        // Check our work so far.
430        // If something prints, there was an error above.
431        print H5GW_ValidateFolder("root:mydata")
432
433        // set I0 and Q0 to your data
434
435        print H5GW_WriteHDF5("root:mydata", "mydata.h5")
436       
437        SetDataFolder root:
438End
439
440
441//
442// given a filename of a SANS data filename of the form
443// name.anything
444// returns the name as a string without the ".fbdfasga" extension
445//
446// returns the input string if a "." can't be found (maybe it wasn't there)
447//
448Function/S H_RemoveDotExtension(item)
449        String item
450        String invalid = item   //
451        Variable num=-1
452       
453        //find the "dot"
454        String runStr=""
455        Variable pos = strsearch(item,".",0)
456        if(pos == -1)
457                //"dot" not found
458                return (invalid)
459        else
460                //found, get all of the characters preceeding it
461                runStr = item[0,pos-1]
462                return (runStr)
463        Endif
464End
465
466
467//
468// Writing attributes to a group and to a dataset.
469// example from the WM documentation for HDF5SaveData
470//
471Function DemoAttributes(w)
472        Wave w
473
474        Variable result = 0     // 0 means no error
475       
476        // Create file
477        Variable fileID
478        HDF5CreateFile/P=home /O /Z fileID as "Test.h5"
479        if (V_flag != 0)
480                Print "HDF5CreateFile failed"
481                return -1
482        endif
483
484        // Write an attribute to the root group
485        Make /FREE /T /N=1 groupAttribute = "This is a group attribute"
486        HDF5SaveData /A="GroupAttribute" groupAttribute, fileID, "/"
487       
488        // Save wave as dataset
489        HDF5SaveData /O /Z w, fileID    // Uses wave name as dataset name
490        if (V_flag != 0)
491                Print "HDF5SaveData failed"
492                result = -1
493        endif
494
495        // Write an attribute to the dataset
496        Make /FREE /T /N=1 datasetAttribute = "This is a dataset attribute"
497        String datasetName = NameOfWave(w)
498        HDF5SaveData /A="DatasetAttribute" datasetAttribute, fileID, datasetName
499
500        HDF5CloseFile fileID
501       
502        return result
503End     // The attribute waves are automatically killed since they are free waves
504
505
506
507//     
508//Function H_Test_HDFWriteTrans(fname,val)
509//      String fname
510//      Variable val
511//     
512//     
513//      String str
514//      PathInfo home
515//      str = S_path
516//     
517//      H_WriteTransmissionToHeader(str+fname,val)
518//     
519//      return(0)
520//End
521//
522//Function H_WriteTransmissionToHeader(fname,trans)
523//      String fname
524//      Variable trans
525//     
526//      Make/O/D/N=1 wTmpWrite
527//      String groupName = "/Sample"    //      /Run1/Sample becomes groupName /Run1/Run1/Sample
528//      String varName = "TRNS"
529//      wTmpWrite[0] = trans //
530//
531//      variable err
532//      err = HDFWrite_Wave(fname, groupName, varName, wTmpWrite)
533//      KillWaves wTmpWrite
534//     
535//      //err not handled here
536//             
537//      return(0)
538//End
539
540
541
542Function H_Test_ListAttributes(fname,groupName)
543        String fname,groupName
544        Variable trans
545       
546//      Make/O/D/N=1 wTmpWrite
547//      String groupName = "/Sample"    //      /Run1/Sample becomes groupName /Run1/Run1/Sample
548//      String varName = "TRNS"
549//      wTmpWrite[0] = trans //
550        String str
551        PathInfo home
552        str = S_path
553       
554        variable err
555        err = H_HDF_ListAttributes(str+fname, groupName)
556       
557        //err not handled here
558               
559        return(0)
560End
561
562Function H_HDF_ListAttributes(fname, groupName)
563        String fname, groupName
564       
565        variable err=0, fileID,groupID
566        String cDF = getDataFolder(1), temp
567        String NXentry_name, attrValue=""
568       
569        STRUCT HDF5DataInfo di  // Defined in HDF5 Browser.ipf.
570        InitHDF5DataInfo(di)    // Initialize structure.
571       
572        try     
573                HDF5OpenFile /Z fileID  as fname  //open file read-write
574                if(!fileID)
575                        err = 1
576                        abort "HDF5 file does not exist"
577                endif
578               
579                HDF5OpenGroup /Z fileID , groupName, groupID
580
581        //      (QUOKKA) !! At the moment, there is no entry for sample thickness in our data file
582        //      therefore create new HDF5 group to enable write / patch command
583        //      comment out the following group creation once thickness appears in revised file
584       
585                if(!groupID)
586                        HDF5CreateGroup /Z fileID, groupName, groupID
587                        //err = 1
588                        //abort "HDF5 group does not exist"
589                else
590
591//                      HDF5AttributeInfo(fileID, "/", 1, "file_name", 0, di)
592                        HDF5AttributeInfo(fileID, "/", 1, "NeXus_version", 0, di)
593                        Print di
594
595//                      see the HDF5 Browser  for how to get the actual <value> of the attribute. See GetPreviewString in
596//        or in FillGroupAttributesList or in FillDatasetAttributesList (from FillLists)
597//                      it seems to be ridiculously complex to get such a simple bit of information - the HDF5BrowserData STRUCT
598//                      needs to be filled first. Ugh.
599                        attrValue = GetPreviewString(fileID, 1, di, "/entry", "cucumber")
600                        Print "attrValue = ",attrValue
601                       
602                       
603                        //get attributes and save them
604                        HDF5ListAttributes/TYPE=1 /Z fileID, groupName          //TYPE=1 means that we're referencing a group, not a dataset
605                        Print "S_HDF5ListAttributes = ", S_HDF5ListAttributes
606                       
607                        // passing the groupID works too, then the group name is not needed                     
608                        HDF5ListAttributes/TYPE=1 /Z groupID, "."               //TYPE=1 means that we're referencing a group, not a dataset
609                        Print "S_HDF5ListAttributes = ", S_HDF5ListAttributes
610                endif
611        catch
612
613                // catch any aborts here
614               
615        endtry
616       
617        if(groupID)
618                HDF5CloseGroup /Z groupID
619        endif
620       
621        if(fileID)
622                HDF5CloseFile /Z fileID
623        endif
624
625        setDataFolder $cDF
626        return err
627end
628
629
630/////////////// end of the testing procedures ////////////////
Note: See TracBrowser for help on using the repository browser.