source: sans/Dev/trunk/NCNR_User_Procedures/Reduction/VSANS/VC_HDF5_VSANS_Utils.ipf @ 960

Last change on this file since 960 was 960, checked in by srkline, 7 years ago

more work with SANS and VSANS Nexus files, still trying to organize the r/w routines for just the basic testing of the first pass of NICE-generated files. Ditched the idea of generating the files from Igor, since DAS_log is not included. Now the plan is to start from a NICE/Nexus template. All still EXTREMELY fluid at this point.

File size: 15.8 KB
Line 
1#pragma rtGlobals=1             // Use modern global access method.
2
3//
4// This file has lots of utility procedures to be able to read/write
5// test HDF5 files for SANS and VSANS in Nexus format
6//
7// It doesn't have the NICE logs, but has everything that I
8// can think of here.
9//
10
11//
12// There's an odd sequence of steps that need to be done to use HDFGateway
13// to most easily write out attributes, and be able to read then back in.
14// Rather awkward, but it works without me needing to go nuts with an atomistic
15// understanding of HDF5. This is necessary for me to be able to write out
16// Nexus files, which is important both for testing, and for simulation.
17//
18// This is done as a set of macros that need to be applied in a specific sequence
19// in order to be able to generate HDF5___xref
20// Ideally, the HDF5___xref wave can be saved/imported to make proper saving possible
21// once the "tree" is finalized.
22//
23//
24// The basic saving routines are here, just as Proc rather than Macro menu clutter
25//
26
27
28// Next... think of all of R/W access needed
29//
30// the simple read/write works...
31// linear_data does not seem to need to be transposed at all
32//
33//  -- this seems too easy. what am I doing wrong? Is something getting garbled when I
34// write back any single values back to the file
35//
36// -- try a string value next
37// -- then start to write the generic get/write functions
38//
39
40//
41// do I ditch the RealsRead/IntegersRead/TextRead? It makes little sense now.
42// maybe copy a "dataInfo" folder/subfolders. can't keep them all (bloat)
43// but then what about multiple files added together?
44//
45
46
47Macro Load_Nexus_V_Template()
48        H_HDF5Gate_Read_Raw("")
49        String tmpStr=root:file_name  //SRK - so I can get the file name
50       
51// this is the folder name
52        String base_name = StringFromList(0,File_Name,".")
53//      Print base_name
54        RenameDataFolder $("root:"+base_name), V_Nexus_Template
55End
56
57
58Macro Fill_Nexus_V_Template()
59
60End
61
62
63Macro Save_Nexus_V_Template()
64        H_HDF5Gate_Write_Raw()
65End
66
67
68
69
70
71
72
73
74
75
76
77
78
79/////////////////////////////
80
81/////////////below is largely depricated, ugly dance to be able to "fake" a file from Igor
82// which was not complete anyways.
83
84
85
86Macro IgorOnly_Setup_VSANS_Struct()
87
88        // lays out the tree and fills with dummy values
89        H_Setup_VSANS_Structure()
90       
91        // writes in the attributes
92        H_Fill_VSANS_Attributes()
93       
94        // fill in with VCALC simulation bits
95        H_Fill_VSANS_wSim()
96       
97End
98
99Macro IgorOnly_Save_VSANS_Nexus(fileName)
100        String fileName="Test_VSANS_file"
101
102        // save as HDF5 (no attributes saved yet)
103        Save_VSANS_file("root:VSANS_file", fileName+".h5")
104       
105        // read in a data file using the gateway-- reads from the home path
106        H_HDF5Gate_Read_Raw(fileName+".h5")
107       
108        // after reading in a "partial" file using the gateway (to generate the xref)
109        // Save the xref to disk (for later use)
110        Save_HDF5___xref("root:"+fileName,"HDF5___xref")
111       
112        // after you've generated the HDF5___xref, load it in and copy it
113        // to the necessary folder location.
114        Copy_HDF5___xref("root:VSANS_file", "HDF5___xref")
115       
116        // writes out the contents of a data folder using the gateway
117        H_HDF5Gate_Write_Raw("root:VSANS_file", fileName+".h5")
118
119        // re-load the data file using the gateway-- reads from the home path
120        // now with attributes
121        H_HDF5Gate_Read_Raw(fileName+".h5")
122       
123End
124
125
126
127Macro IgorOnly_Setup_SANS_Struct()
128
129        // lays out the tree and fills with dummy values
130        H_Setup_SANS_Structure()
131       
132        // writes in the attributes
133        H_Fill_SANS_Attributes()
134       
135        // fill in with VCALC simulation bits
136        H_Fill_SANS_wSim()
137
138End
139
140Macro IgorOnly_Save_SANS_Nexus(fileName)
141        String fileName="Test_SANS_file"
142       
143        // save as HDF5 (no attributes saved yet) (save_VSANS is actually generic HDF...)
144        Save_VSANS_file("root:SANS_file", fileName+".h5")
145       
146        // read in a data file using the gateway-- reads from the home path
147        H_HDF5Gate_Read_Raw(fileName+".h5")
148       
149        // after reading in a "partial" file using the gateway (to generate the xref)
150        // Save the xref to disk (for later use)
151        Save_HDF5___xref("root:"+fileName,"HDF5___xref")
152       
153        // after you've generated the HDF5___xref, load it in and copy it
154        // to the necessary folder location.
155        Copy_HDF5___xref("root:SANS_file", "HDF5___xref")
156       
157        // writes out the contents of a data folder using the gateway
158        H_HDF5Gate_Write_Raw("root:SANS_file", fileName+".h5")
159
160        // re-load the data file using the gateway-- reads from the home path
161        // now with attributes
162        H_HDF5Gate_Read_Raw(fileName+".h5")
163
164
165End
166
167
168// TODO
169// currently, there are no dummy fill values or attributes for the fake DIV file
170//
171Macro Setup_VSANS_DIV_Struct()
172
173        // lays out the tree and fills with dummy values
174        H_Setup_VSANS_DIV_Structure()
175       
176        // writes in the attributes
177//      H_Fill_VSANS_Attributes()
178       
179        // fill in with VCALC simulation bits
180//      H_Fill_VSANS_wSim()
181       
182End
183
184Macro Save_VSANS_DIV_Nexus(fileName)
185        String fileName="Test_VSANS_DIV_file"
186
187        // save as HDF5 (no attributes saved yet)
188        Save_VSANS_file("root:VSANS_DIV_file", fileName+".h5")
189       
190        // read in a data file using the gateway-- reads from the home path
191        H_HDF5Gate_Read_Raw(fileName+".h5")
192       
193        // after reading in a "partial" file using the gateway (to generate the xref)
194        // Save the xref to disk (for later use)
195        Save_HDF5___xref("root:"+fileName,"HDF5___xref")
196       
197        // after you've generated the HDF5___xref, load it in and copy it
198        // to the necessary folder location.
199        Copy_HDF5___xref("root:VSANS_DIV_file", "HDF5___xref")
200       
201        // writes out the contents of a data folder using the gateway
202        H_HDF5Gate_Write_Raw("root:VSANS_DIV_file", fileName+".h5")
203
204        // re-load the data file using the gateway-- reads from the home path
205        // now with attributes
206        H_HDF5Gate_Read_Raw(fileName+".h5")
207       
208End
209
210
211
212
213
214
215
216
217
218
219
220
221
222//
223// saves a specified folder, with a given filename.
224// saves to the home path
225//
226Proc Save_VSANS_file(dfPath, filename)
227        String dfPath   ="root:VSANS_file"              // e.g., "root:FolderA" or ":"
228        String filename = "Test_VSANS_file.h5"
229       
230        H_NXSANS_SaveGroupAsHDF5(dfPath, filename)
231End
232
233
234//     
235// this is my procedure to save the folders to HDF5, once I've filled the folder tree
236//
237// this does NOT save attributes, but gets the folder structure correct
238//
239Function H_NXSANS_SaveGroupAsHDF5(dfPath, filename)
240        String dfPath   // e.g., "root:FolderA" or ":"
241        String filename
242
243        Variable result = 0     // 0 means no error
244       
245        Variable fileID
246        HDF5CreateFile/P=home /O /Z fileID as filename
247        if (V_flag != 0)
248                Print "HDF5CreateFile failed"
249                return -1
250        endif
251
252        HDF5SaveGroup /IGOR=0 /O /R /Z $dfPath, fileID, "."
253//      HDF5SaveGroup /O /R /Z $dfPath, fileID, "."
254        if (V_flag != 0)
255                Print "HDF5SaveGroup failed"
256                result = -1
257        endif
258       
259        HDF5CloseFile fileID
260
261        return result
262End
263
264// passing null file string presents a dialog
265// these two procedures will use the full Xrefs, so they can write out full
266// files. They are, however, slow
267Proc Read_Nexus_Xref()
268        H_HDF5Gate_Read_Raw("")
269End
270
271Proc Write_Nexus_Xref()
272        H_HDF5Gate_Write_Raw()
273End
274
275//
276// writes out the contents of a data folder using the gateway
277// -- the HDF5___xref wave must be present at the top level for it to
278//    write out anything.
279//
280Proc H_HDF5Gate_Write_Raw(dfPath,filename)
281        String dfPath   = "root:"       // e.g., "root:FolderA" or ":"
282        String filename = "Test_VSANS_file.h5"
283//      Prompt dfPath, "Data Folder",popup,H_GetAList(4)       
284//      Prompt fileName, "File name"
285       
286//      DoPrompt "Folder and file to write", dfPath,fileName
287        // Check our work so far.
288        // If something prints, there was an error above.
289        print H5GW_ValidateFolder(dfPath)
290
291        print H5GW_WriteHDF5(dfPath, filename)
292       
293        SetDataFolder root:
294End
295
296//
297// read in a data file using the gateway
298//
299// reads from the home path
300//
301Proc H_HDF5Gate_Read_Raw(file)
302        String file
303       
304//      NewDataFolder/O/S root:newdata
305        Print H5GW_ReadHDF5("", file)   // reads into current folder
306        SetDataFolder root:
307End
308
309
310
311// this is in PlotUtils, but duplicated here
312// utility used in the "PlotSmeared...() macros to get a list of data folders
313//
314//1:    Waves.
315//2:    Numeric variables.
316//3:    String variables.
317//4:    Data folders.
318Function/S H_GetAList(type)
319        Variable type
320       
321        SetDataFolder root:
322       
323        String objName,str=""
324        Variable index = 0
325        do
326                objName = GetIndexedObjName(":", type, index)
327                if (strlen(objName) == 0)
328                        break
329                endif
330                //Print objName
331                str += objName + ";"
332                index += 1
333        while(1)
334       
335        // remove myGlobals, Packages, etc. from the folder list
336        if(type==4)
337                str = RemoveFromList("myGlobals", str , ";" )
338                str = RemoveFromList("Packages", str, ";")
339        endif
340       
341        return(str)
342End
343
344//
345// after reading in a "partial" file using the gateway (to generate the xref)
346// Save the xref to disk (for later use)
347//
348Proc Save_HDF5___xref(dfPath, filename)
349        String dfPath   ="root:VSANS_file"              // e.g., "root:FolderA" or ":"
350        String filename = "HDF5___xref"
351
352        Save/T/P=home $(dfPath+":HDF5___xref") as "HDF5___xref.itx"
353
354//      Copy_HDF5___xref(dfPath, filename)
355       
356        SetDataFolder root:
357End
358
359//
360// after you've generated the HDF5___xref, load it in and copy it to
361// the necessary folder location.
362// - then all is set to *really* write out the file correctly, including the attributes
363//
364Proc Copy_HDF5___xref(dfPath, filename)
365        String dfPath   ="root:VSANS_file"              // e.g., "root:FolderA" or ":"
366        String filename = "HDF5___xref"
367
368        if(exists(filename) != 1)
369                //load it in   
370                LoadWave/T/P=home/O "HDF5___xref.itx"
371        endif
372       
373        Duplicate/O HDF5___xref, $(dfPath+":HDF5___xref")
374       
375        SetDataFolder root:
376End
377
378
379
380//////// testing procedures, may be of use, maybe not //////////////
381
382
383//////// Two procedures that test out Pete Jemain's HDF5Gateway
384//
385// This works fine, but it may not be terribly compatible with the way NICE will eventually
386// write out the data files. I'll have very little control over that and I'll need to cobble together
387// a bunch of fixes to cover up their mistakes.
388//
389// Using Nick Hauser's code as a starting point may be a lot more painful, but more flexible in the end.
390//
391// I'm completely baffled about what to do with attributes. Are they needed, is this the best way to deal
392// with them, do I care about reading them in, and if I do, why?
393//
394Proc H_HDF5Gate_WriteTest()
395
396        // create the folder structure
397        NewDataFolder/O/S root:mydata
398        NewDataFolder/O sasentry
399        NewDataFolder/O :sasentry:sasdata
400
401        // create the waves
402        Make/O :sasentry:sasdata:I0
403        Make/O :sasentry:sasdata:Q0
404
405        Make/O/N=0 Igor___folder_attributes
406        Make/O/N=0 :sasentry:Igor___folder_attributes
407        Make/O/N=0 :sasentry:sasdata:Igor___folder_attributes
408
409        // create the attributes
410        Note/K Igor___folder_attributes, "producer=IgorPro\rNX_class=NXroot"
411        Note/K :sasentry:Igor___folder_attributes, "NX_class=NXentry"
412        Note/K :sasentry:sasdata:Igor___folder_attributes, "NX_class=NXdata"
413        Note/K :sasentry:sasdata:I0, "units=1/cm\rsignal=1\rtitle=reduced intensity"
414        Note/K :sasentry:sasdata:Q0, "units=1/A\rtitle=|scattering vector|"
415
416        // create the cross-reference mapping
417        Make/O/T/N=(5,2) HDF5___xref
418        Edit/K=0 'HDF5___xref';DelayUpdate
419        HDF5___xref[0][1] = ":"
420        HDF5___xref[1][1] = ":sasentry"
421        HDF5___xref[2][1] = ":sasentry:sasdata"
422        HDF5___xref[3][1] = ":sasentry:sasdata:I0"
423        HDF5___xref[4][1] = ":sasentry:sasdata:Q0"
424        HDF5___xref[0][0] = "/"
425        HDF5___xref[1][0] = "/sasentry"
426        HDF5___xref[2][0] = "/sasentry/sasdata"
427        HDF5___xref[3][0] = "/sasentry/sasdata/I"
428        HDF5___xref[4][0] = "/sasentry/sasdata/Q"
429
430        // Check our work so far.
431        // If something prints, there was an error above.
432        print H5GW_ValidateFolder("root:mydata")
433
434        // set I0 and Q0 to your data
435
436        print H5GW_WriteHDF5("root:mydata", "mydata.h5")
437       
438        SetDataFolder root:
439End
440
441
442//
443// given a filename of a SANS data filename of the form
444// name.anything
445// returns the name as a string without the ".fbdfasga" extension
446//
447// returns the input string if a "." can't be found (maybe it wasn't there)
448//
449Function/S H_RemoveDotExtension(item)
450        String item
451        String invalid = item   //
452        Variable num=-1
453       
454        //find the "dot"
455        String runStr=""
456        Variable pos = strsearch(item,".",0)
457        if(pos == -1)
458                //"dot" not found
459                return (invalid)
460        else
461                //found, get all of the characters preceeding it
462                runStr = item[0,pos-1]
463                return (runStr)
464        Endif
465End
466
467
468//
469// Writing attributes to a group and to a dataset.
470// example from the WM documentation for HDF5SaveData
471//
472Function DemoAttributes(w)
473        Wave w
474
475        Variable result = 0     // 0 means no error
476       
477        // Create file
478        Variable fileID
479        HDF5CreateFile/P=home /O /Z fileID as "Test.h5"
480        if (V_flag != 0)
481                Print "HDF5CreateFile failed"
482                return -1
483        endif
484
485        // Write an attribute to the root group
486        Make /FREE /T /N=1 groupAttribute = "This is a group attribute"
487        HDF5SaveData /A="GroupAttribute" groupAttribute, fileID, "/"
488       
489        // Save wave as dataset
490        HDF5SaveData /O /Z w, fileID    // Uses wave name as dataset name
491        if (V_flag != 0)
492                Print "HDF5SaveData failed"
493                result = -1
494        endif
495
496        // Write an attribute to the dataset
497        Make /FREE /T /N=1 datasetAttribute = "This is a dataset attribute"
498        String datasetName = NameOfWave(w)
499        HDF5SaveData /A="DatasetAttribute" datasetAttribute, fileID, datasetName
500
501        HDF5CloseFile fileID
502       
503        return result
504End     // The attribute waves are automatically killed since they are free waves
505
506
507
508//     
509//Function H_Test_HDFWriteTrans(fname,val)
510//      String fname
511//      Variable val
512//     
513//     
514//      String str
515//      PathInfo home
516//      str = S_path
517//     
518//      H_WriteTransmissionToHeader(str+fname,val)
519//     
520//      return(0)
521//End
522//
523//Function H_WriteTransmissionToHeader(fname,trans)
524//      String fname
525//      Variable trans
526//     
527//      Make/O/D/N=1 wTmpWrite
528//      String groupName = "/Sample"    //      /Run1/Sample becomes groupName /Run1/Run1/Sample
529//      String varName = "TRNS"
530//      wTmpWrite[0] = trans //
531//
532//      variable err
533//      err = HDFWrite_Wave(fname, groupName, varName, wTmpWrite)
534//      KillWaves wTmpWrite
535//     
536//      //err not handled here
537//             
538//      return(0)
539//End
540
541
542
543Function H_Test_ListAttributes(fname,groupName)
544        String fname,groupName
545        Variable trans
546       
547//      Make/O/D/N=1 wTmpWrite
548//      String groupName = "/Sample"    //      /Run1/Sample becomes groupName /Run1/Run1/Sample
549//      String varName = "TRNS"
550//      wTmpWrite[0] = trans //
551        String str
552        PathInfo home
553        str = S_path
554       
555        variable err
556        err = H_HDF_ListAttributes(str+fname, groupName)
557       
558        //err not handled here
559               
560        return(0)
561End
562
563Function H_HDF_ListAttributes(fname, groupName)
564        String fname, groupName
565       
566        variable err=0, fileID,groupID
567        String cDF = getDataFolder(1), temp
568        String NXentry_name, attrValue=""
569       
570        STRUCT HDF5DataInfo di  // Defined in HDF5 Browser.ipf.
571        InitHDF5DataInfo(di)    // Initialize structure.
572       
573        try     
574                HDF5OpenFile /Z fileID  as fname  //open file read-write
575                if(!fileID)
576                        err = 1
577                        abort "HDF5 file does not exist"
578                endif
579               
580                HDF5OpenGroup /Z fileID , groupName, groupID
581
582        //      (QUOKKA) !! At the moment, there is no entry for sample thickness in our data file
583        //      therefore create new HDF5 group to enable write / patch command
584        //      comment out the following group creation once thickness appears in revised file
585       
586                if(!groupID)
587                        HDF5CreateGroup /Z fileID, groupName, groupID
588                        //err = 1
589                        //abort "HDF5 group does not exist"
590                else
591
592//                      HDF5AttributeInfo(fileID, "/", 1, "file_name", 0, di)
593                        HDF5AttributeInfo(fileID, "/", 1, "NeXus_version", 0, di)
594                        Print di
595
596//                      see the HDF5 Browser  for how to get the actual <value> of the attribute. See GetPreviewString in
597//        or in FillGroupAttributesList or in FillDatasetAttributesList (from FillLists)
598//                      it seems to be ridiculously complex to get such a simple bit of information - the HDF5BrowserData STRUCT
599//                      needs to be filled first. Ugh.
600                        attrValue = GetPreviewString(fileID, 1, di, "/entry", "cucumber")
601                        Print "attrValue = ",attrValue
602                       
603                       
604                        //get attributes and save them
605                        HDF5ListAttributes/TYPE=1 /Z fileID, groupName          //TYPE=1 means that we're referencing a group, not a dataset
606                        Print "S_HDF5ListAttributes = ", S_HDF5ListAttributes
607                       
608                        // passing the groupID works too, then the group name is not needed                     
609                        HDF5ListAttributes/TYPE=1 /Z groupID, "."               //TYPE=1 means that we're referencing a group, not a dataset
610                        Print "S_HDF5ListAttributes = ", S_HDF5ListAttributes
611                endif
612        catch
613
614                // catch any aborts here
615               
616        endtry
617       
618        if(groupID)
619                HDF5CloseGroup /Z groupID
620        endif
621       
622        if(fileID)
623                HDF5CloseFile /Z fileID
624        endif
625
626        setDataFolder $cDF
627        return err
628end
629
630
631/////////////// end of the testing procedures ////////////////
Note: See TracBrowser for help on using the repository browser.